# robots.txt for themindreader.ai

User-agent: *
Disallow: /api/
Disallow: /private/
Disallow: /server/

# Allow crawling of public pages and render assets
Allow: /
Allow: /_next/static/
Allow: /_next/image
Allow: /api/blogslug

# Explicit allow rules for AI search crawlers
User-agent: GPTBot
Disallow: /api/
Disallow: /private/
Disallow: /server/
Allow: /

User-agent: OAI-SearchBot
Disallow: /api/
Disallow: /private/
Disallow: /server/
Allow: /

User-agent: ChatGPT-User
Disallow: /api/
Disallow: /private/
Disallow: /server/
Allow: /

User-agent: ClaudeBot
Disallow: /api/
Disallow: /private/
Disallow: /server/
Allow: /

User-agent: PerplexityBot
Disallow: /api/
Disallow: /private/
Disallow: /server/
Allow: /

# Explicit block rules for training-focused crawlers
User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

# Sitemap location
Sitemap: https://themindreader.ai/sitemap.xml
Sitemap: https://themindreader.ai/api/blogslug