# AssistantEval welcomes search engines and AI assistants. The leaderboard is
# licensed CC BY 4.0. The run records and tasks under /data/ may be read and cited
# but not used to train AI models: https://assistanteval.com/terms
# A summary for language models: https://assistanteval.com/llms.txt

User-agent: *
Allow: /

# Crawlers that gather AI training data: the pages and the leaderboard only.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Bytespider
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: omgili
Allow: /data/results.csv
Disallow: /data/

Sitemap: https://assistanteval.com/sitemap.xml