# robots.txt for ceaksan.com
# https://ceaksan.com/robots.txt

User-agent: *
Allow: /

# Content Signals - AI usage preferences (https://contentsignals.org/)
Content-Signal: ai-train=no, search=yes, ai-input=no

# Disallow admin routes
Disallow: /admin/
Disallow: /tr/admin/
Disallow: /en/admin/

# Disallow API routes
Disallow: /api/

# Disallow member zones (requires authentication)
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Disallow internal Astro assets (not needed for SEO)
Disallow: /_astro/
Disallow: /_server-islands/

# Disallow Cloudflare internal paths
Disallow: /cdn-cgi/

# Allow all other content (blog posts, courses, pages, etc.)
Allow: /tr/
Allow: /en/
Allow: /tr/contents/
Allow: /en/contents/
Allow: /tr/courses/
Allow: /en/courses/

# ============================================
# AI Crawler Rules
# Content is licensed under CC BY-NC-SA 4.0
# See /llms.txt for usage terms and attribution requirements
# ============================================

# OpenAI GPTBot - ChatGPT, SearchGPT
User-agent: GPTBot
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Anthropic ClaudeBot
User-agent: ClaudeBot
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Anthropic Claude-Web
User-agent: Claude-Web
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Google Gemini / Bard
User-agent: Google-Extended
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Cohere AI
User-agent: cohere-ai
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Meta AI
User-agent: FacebookBot
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# Common Crawl (used by many LLMs)
User-agent: CCBot
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=no
Disallow: /api/
Disallow: /admin/
Disallow: /tr/member-zone
Disallow: /en/member-zone

# ============================================
# Sitemap & LLM Index
# ============================================

# Sitemap location
Sitemap: https://ceaksan.com/sitemap-index.xml

# LLM-friendly index (see llms.txt spec)
# https://ceaksan.com/llms.txt