# Lorkid robots.txt

Sitemap: https://sas-lorkid.com

# Standard crawling rules + IA crawler explicit allowlist

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /?s=
Disallow: /feed/
Disallow: /?rest_route=
Crawl-delay: 1

# Declare sitemap for all crawlers
Sitemap: https://sas-lorkid.com/sitemap.xml

# ===== IA/LLM CRAWLERS EXPLICIT ALLOWLIST =====
# These crawlers index Lorkid content for AI model training + RAG use
# Allowing them boosts brand visibility (GPT, Claude, Perplexity citations)

# Anthropic Claude crawlers (CORRECTED - was wrong as anthropic-ai)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# OpenAI crawlers
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Perplexity AI crawler
User-agent: PerplexityBot
Allow: /

# Google AI crawlers
User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Mistral AI
User-agent: MistralBot
Allow: /

# Others
User-agent: CCBot
Allow: /

User-agent: AppleBot-Extended
Allow: /

User-agent: phindbot
Allow: /

User-agent: RerankCrawler
Allow: /

# Limit aggressive crawlers
User-agent: MJ12bot
Crawl-delay: 5
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 5
Disallow: /

# Note: robots.txt is a courtesy protocol. Malicious actors and aggressive
# crawlers will ignore these rules. For critical content protection, use:
# - Authentication (wp-admin already protected)
# - Rate limiting at .htaccess or WAF level (Cloudflare, ModSecurity)
# - noindex meta tag for sensitive pages