# Samarthya IAS - Robots.txt Configuration
# This file helps search engines crawl and index the website

# Allow all search engines
User-agent: *
Allow: /

# Specific permissions for Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0.1

# Specific permissions for Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# GEO / AEO crawlers (explicitly allowed)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Block bad bots (optional - uncomment if needed)
# User-agent: AhrefsBot
# Disallow: /
# User-agent: SemrushBot
# Disallow: /
# User-agent: DotBot
# Disallow: /

# Disallow access to private/temporary files
Disallow: /admin/
Disallow: /temp/
Disallow: /*?
Disallow: /*.php$

# Allow access to CSS and JS files
Allow: /*.css$
Allow: /*.js$
Allow: /img/
Allow: /resources/

# Specify the sitemap location
Sitemap: https://samarthyaias.com/sitemap.xml

# AI crawler discovery file
Allow: /LLM.txt

# Crawl delay (in seconds) - prevents server overload
Crawl-delay: 0.5

# Request rate (pages per second for bots)
Request-rate: 30/1m