# Robots.txt for gjalla.io
# Optimized for AI crawlers and search engines

User-agent: *
Allow: /

# Block access to private/admin areas
Disallow: /admin/
Disallow: /workspace/
Disallow: /profile/
Disallow: /api/
Disallow: /auth/
Disallow: /payment/
Disallow: /onboarding/

# Crawl delay (optional, for respectful crawling)
Crawl-delay: 1

# Sitemap and RSS feed location
#
# Two sitemaps on purpose, NOT one merged file. The Astro build emits its own
# index covering every blog post and every /docs page, each listed in the
# trailing-slash form those pages declare canonical. Copying those URLs into
# sitemap.xml as well would declare each page twice, in two forms, and the
# second form would contradict the page's own canonical tag.
#
# A sitemap may normally only list URLs under its own path, which would stop
# /blog/sitemap-index.xml from covering /docs. Declaring it here lifts that
# restriction, so it is authoritative for both.
Sitemap: https://gjalla.io/sitemap.xml
Sitemap: https://gjalla.io/blog/sitemap-index.xml
Sitemap: https://gjalla.io/blog/rss.xml

# AI crawlers — explicit allow
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /