# =============================================================================
# Mexico Travel & Leisure - robots.txt
# =============================================================================

# Default: Allow all legitimate crawlers
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://www.mexicotravelandleisure.com/sitemap-index.xml

# AI crawler guide (llms.txt standard)
# https://www.mexicotravelandleisure.com/llms.txt

# =============================================================================
# SEARCH ENGINES - Explicitly allowed (for clarity)
# =============================================================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Slurp
Allow: /

# =============================================================================
# SOCIAL MEDIA BOTS - Allowed for link previews
# =============================================================================

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# =============================================================================
# AI BOTS - Selective blocking
# =============================================================================

# ALLOW: Google AI (powers AI Overviews in search results - good for traffic)
User-agent: Google-Extended
Allow: /

# ALLOW: Perplexity (AI search engine that sends referral traffic)
User-agent: PerplexityBot
Allow: /

# ALLOW: ChatGPT browsing agent (sends referral traffic - 885 sessions/mo)
# This is the user-facing ChatGPT browse/search feature, NOT the training crawler
User-agent: ChatGPT-User
Allow: /

# ALLOW: OpenAI's search indexing bot (for ChatGPT search results)
User-agent: OAI-SearchBot
Allow: /

# BLOCK: OpenAI's GPTBot (training crawler only - doesn't send traffic back)
User-agent: GPTBot
Disallow: /

# BLOCK: Common Crawl (used for AI training datasets)
User-agent: CCBot
Disallow: /

# BLOCK: Anthropic's crawler
User-agent: anthropic-ai
Disallow: /

# BLOCK: Cohere AI
User-agent: cohere-ai
Disallow: /

# BLOCK: Meta/Facebook AI training
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# BLOCK: Apple AI training (Applebot-Extended is for AI, regular Applebot is for Siri/Spotlight)
User-agent: Applebot-Extended
Disallow: /

# BLOCK: Bytedance/TikTok AI
User-agent: Bytespider
Disallow: /

# =============================================================================
# KNOWN BAD BOTS / SCRAPERS - Block
# =============================================================================

# SEO tool scrapers (high volume, no value)
# Allow the project-owned Ahrefs Site Audit crawler while keeping the
# general backlink crawler blocked.
User-agent: AhrefsSiteAudit
Allow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Content scrapers
User-agent: MegaIndex
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Aggressive crawlers
User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /

# =============================================================================
# NOTES
# =============================================================================
# - This file controls which bots can crawl, not who can access the site
# - Malicious bots ignore robots.txt - use Vercel Firewall for actual blocking
# - Google-Extended controls AI training but Googlebot still indexes normally
# - Review quarterly as new AI bots emerge
# Last updated: 2026-08-27