# robots.txt — SGExamPapers.com
# Updated 2026-07-15 (Tier 3): allow all crawlers including AI bots; reference llms.txt.

# ============================================================
# Default — open access for all crawlers (including AI bots).
# We WANT to be cited by ChatGPT, Perplexity, Claude, Gemini, etc.
# ============================================================
User-agent: *
Allow: /

# Block internal/admin helpers if any appear in future
Disallow: /api/
Disallow: /_/

# Tell crawlers where the sitemap + AI-discovery files live
Sitemap: https://sgexampapers.com/sitemap.xml

# AI-engine discovery (auto-consumed by ChatGPT, Claude, Perplexity when discovered)
# Format: https://llmstxt.org/
Sitemap: https://sgexampapers.com/llms.txt
Sitemap: https://sgexampapers.com/llms-full.txt
Sitemap: https://sgexampapers.com/pdfs.json

# Specific user-agents — open access (no override needed; included for documentation)
# GPTBot (OpenAI / ChatGPT)
User-agent: GPTBot
Allow: /

# ClaudeBot / anthropic-ai
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google-Extended (Gemini AI training)
User-agent: Google-Extended
Allow: /

# Common Crawl (used by many LLMs)
User-agent: CCBot
Allow: /