# General access
User-agent: *
Allow: /
Disallow: /api/
# Raw JSON behind the reader (~12 MB). Nothing links to it and it is not in the
# sitemap, so it is not crawled today — this is a guard against the day one of
# those URLs becomes discoverable. It does NOT affect the mobile app: robots.txt
# governs crawlers, not app clients, and the app fetches these at runtime.
Disallow: /data/

# AI Crawlers - explicitly allowed for CONTENT.
#
# A named user-agent group REPLACES the `*` group rather than adding to it, so
# each of these needs its own Disallow lines — without them these ten bots were
# the only clients allowed to crawl /api/, which nobody intended.
#
# /data/ is deliberately NOT disallowed here: it is the same Quran text these
# bots can already read in the pages, and clean JSON quotes Kanz-ul-Iman more
# faithfully than scraped HTML. That is a mission call, left to Saad.
User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /api/

User-agent: Bingbot
Allow: /
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /api/

Sitemap: https://ummesaad.org/sitemap.xml