# robots.txt for KAI Signals Platform
# Updated for optimal SEO and AI crawler access
# Last Updated: February 2026

# LLM Optimization File
# For AI/LLM-specific information see: /llms.txt

# Sitemaps
Sitemap: https://kai-signals.com/sitemap.xml
Sitemap: https://kai-signals.com/sitemap-en-US.xml
Sitemap: https://kai-signals.com/sitemap-pt-PT.xml

# Default Rules for All Bots
User-agent: *
# Allow only culture-prefixed URLs
Allow: /en-US/
Allow: /pt-PT/
# Disallow root-level pages (they redirect to culture URLs)
Disallow: /About
Disallow: /Contact
Disallow: /Pricing
Disallow: /FAQ
Disallow: /Terms
Disallow: /Privacy
Disallow: /Disclaimer
Disallow: /Blog$
Disallow: /Forum$
# Disallow private areas
Disallow: /Admin/
Disallow: /Identity/
Disallow: /Account/
Disallow: /Subscription/
Disallow: /Payments/
Disallow: /Notifications/
Disallow: /Activity/
Disallow: /Referral/
Disallow: /Telegram/
Disallow: /api/
# Allow AI resource files explicitly
Allow: /ai-info.json
Allow: /ai-info.md
Allow: /ai-qa.md
Allow: /ai-index.html
Allow: /llms.txt
Allow: /humans.txt
Allow: /robots.txt
# Disallow other query parameters and dynamic endpoints
Disallow: /*?*action=
Disallow: /*?*filter=
Disallow: /*?*pageNumber=
Disallow: /*?*search=
Disallow: /*?*returnUrl=

# Specific Bot Rules for Improved Crawling
User-agent: Googlebot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 0

# AI Search Engines - Explicitly Allow

# ChatGPT/GPTBot (OpenAI)
User-agent: GPTBot
Allow: /en-US/
Allow: /pt-PT/
Allow: /en-US/About
Allow: /pt-PT/About
Allow: /en-US/Pricing
Allow: /pt-PT/Pricing
Allow: /en-US/FAQ
Allow: /pt-PT/FAQ
Allow: /en-US/Blog/
Allow: /pt-PT/Blog/
Disallow: /Admin/
Disallow: /api/
Disallow: /Account/
Crawl-delay: 1

# Google Gemini (Google Bard)
User-agent: Google-Extended
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 0

# Anthropic Claude
User-agent: anthropic-ai
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /en-US/
Allow: /pt-PT/
Allow: /en-US/About
Allow: /pt-PT/About
Allow: /en-US/Pricing
Allow: /pt-PT/Pricing
Allow: /en-US/FAQ
Allow: /pt-PT/FAQ
Allow: /en-US/Blog/
Allow: /pt-PT/Blog/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# Microsoft Copilot (Bing)
User-agent: Bingbot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 0

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 2

# Anthropic AI (Additional)
User-agent: Claude-Web
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# Cohere AI
User-agent: cohere-ai
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# You.com AI Search
User-agent: YouBot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

User-agent: Bingbot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 0

# Amazon Alexa
User-agent: Amazonbot
Allow: /en-US/
Allow: /pt-PT/
Allow: /en-US/FAQ
Allow: /pt-PT/FAQ
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# Apple Siri / Applebot
User-agent: Applebot
Allow: /en-US/
Allow: /pt-PT/
Allow: /en-US/FAQ
Allow: /pt-PT/FAQ
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# Meta AI (Facebook)
User-agent: FacebookBot
User-agent: facebookexternalhit
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# Yandex (Russian search + Alice assistant)
User-agent: Yandex
User-agent: YandexBot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 2

# Baidu (Chinese search)
User-agent: Baiduspider
Allow: /en-US/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 2

# DuckDuckGo AI
User-agent: DuckDuckBot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# Diffbot (AI-powered web data extraction)
User-agent: Diffbot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 1

# ByteDance/TikTok AI (Bytespider)
User-agent: Bytespider
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 2

# Huawei PetalBot (AI search)
User-agent: PetalBot
Allow: /en-US/
Allow: /pt-PT/
Disallow: /Admin/
Disallow: /api/
Crawl-delay: 2

# Block Bad Bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /