# All crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /onboarding/
Disallow: /redeem/
Disallow: /login/
Disallow: /signup/
Disallow: /forgot-password/
Disallow: /reset-password/

# Googlebot
User-agent: Googlebot
Allow: /

# Bingbot
User-agent: Bingbot
Allow: /

# OAI-SearchBot (search results in SearchGPT and OpenAI search features — not used for training)
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User (user-initiated actions in ChatGPT and Custom GPTs — not used for training)
User-agent: ChatGPT-User
Allow: /

# GPTBot (used to crawl content for training generative AI models)
User-agent: GPTBot
Allow: /

# ClaudeBot (crawls content for Anthropic AI model training)
User-agent: ClaudeBot
Allow: /

# Claude-User (user-initiated browsing in Claude conversations — not used for training)
User-agent: Claude-User
Allow: /

# Claude-SearchBot (search result quality optimization — not used for training)
User-agent: Claude-SearchBot
Allow: /

# Sitemaps
Sitemap: https://scrapegraphai.com/sitemap.xml