# https://fronts.ai/robots.txt
#
# Cloudflare may PREPEND managed rules that Disallow AI crawlers (GPTBot, ClaudeBot, etc.).
# Those blocks must be turned off in the dashboard or crawlers never reach this file:
#   Security → Settings → Bots → "Instruct AI bot traffic with robots.txt" → Off
#   Security → Bots → "AI Scrapers and Crawlers" → Allow (or Off)
# See docs/cloudflare-ai-crawlers.md

# --- Traditional search ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# --- AI search / discovery (separate from model-training crawlers below) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

# --- Other research / extended crawlers (optional; not required for AI Overviews) ---
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

# --- Model-training crawlers (independent of search bots above) ---
# Block training scrapers; OAI-SearchBot / Claude-SearchBot handle AI search discovery.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: *
Allow: /

# Marketing routes (`pages/sitemap.xml.tsx`) and directory tag pages (`tags-sitemap.xml`).
Sitemap: https://fronts.ai/sitemap.xml
Sitemap: https://fronts.ai/tags-sitemap.xml