# https://vibeengines.com/robots.txt

# Default: allow all standard crawlers
User-agent: *
Allow: /
# Block raw static handbook, roadmap, systemdesign, ai-systemdesign, and
# algorithm URLs (canonicals live at /handbook/<slug>, /roadmap/<slug>,
# /systemdesign/<slug>, /ai-system-design/<slug>, /algorithm/<slug>); 301
# redirects in firebase.json also dedupe these — this is belt-and-braces.
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

# Allow major AI / generative crawlers (signals opt-in for AI Overviews,
# Perplexity citations, ChatGPT browsing, Claude search, etc.)
User-agent: GPTBot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: ChatGPT-User
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: OAI-SearchBot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: ClaudeBot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Claude-Web
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: PerplexityBot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Perplexity-User
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Google-Extended
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Applebot-Extended
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Bingbot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: DuckDuckBot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

# Additional AI / answer-engine crawlers — same policy: index the canonical
# pages, skip the raw static duplicates.
User-agent: meta-externalagent
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Amazonbot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Bytespider
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: cohere-ai
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: cohere-training-data-crawler
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Diffbot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: YouBot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Timpibot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

User-agent: Google-CloudVertexBot
Allow: /
Disallow: /handbooks/
Disallow: /roadmaps/
Disallow: /systemdesigns/
Disallow: /aisystemdesigns/
Disallow: /papers/
Disallow: /algorithms/
Disallow: /labs/
Disallow: /challenges/

# llms.txt is the curated entry point for LLM consumers.
Sitemap: https://vibeengines.com/sitemap.xml