# TrackRadar.ai - Robots.txt
# Last updated: 2026-03-11
# https://trackradar.ai/robots.txt

# Default rules for all search engines
User-agent: *
Allow: /
Allow: /tools/
Allow: /compare/
Allow: /blog/
Allow: /use-cases/
Allow: /legal/

# Block authenticated/private areas
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /discover/
Disallow: /my/
Disallow: /analytics
Disallow: /notifications
Disallow: /test/

# Block low-value pages (preserve crawl budget)
Disallow: /signin
Disallow: /signup
Disallow: /beta
Disallow: /waitlist
Disallow: /status
Disallow: /subscription

# AI Search Engines (Explicitly Allow)
# These bots are used by ChatGPT, Claude, Perplexity, and other AI search platforms

# OpenAI ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /discover/
Disallow: /my/

# Anthropic Claude
User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /discover/
Disallow: /my/

User-agent: Anthropic-AI
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /discover/
Disallow: /my/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard
Disallow: /discover/
Disallow: /my/

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin/

# Common Search Engines (Already allowed by default, but specified for clarity)
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# Additional AI Search bots (added 2026-04-17)

# OpenAI ChatGPT browsing
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /my/
Disallow: /admin/

# OpenAI Search bot
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /my/
Disallow: /admin/

# Anthropic Claude Search
User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /my/
Disallow: /admin/

# Perplexity User
User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /my/
Disallow: /admin/

# Meta AI (Facebook/Instagram)
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /my/
Disallow: /admin/

# Amazon AI
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /my/
Disallow: /admin/

# Sitemap location
Sitemap: https://trackradar.ai/sitemap.xml
Sitemap: https://trackradar.ai/tracklists-sitemap.xml

# LLM-friendly site description
# See: https://llmstxt.org
# https://trackradar.ai/llms.txt

# AI crawler access policy + content licensing
# See: https://github.com/spawning-inc/ai.txt
# https://trackradar.ai/.well-known/ai.txt