# robots.txt - Optimized for Search + AI Engines
# Anyro (@anyrxo) - anyrxo.com
# Last Updated: 2025-11-24

# ============================================
# DEFAULT USER AGENT
# ============================================
User-agent: *
Allow: /
Allow: /logos/
Allow: /index.html
Disallow: /admin/
Disallow: /private/

# Crawl delay
Crawl-delay: 0

# ============================================
# TRADITIONAL SEARCH ENGINES
# ============================================

# Google Bot
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /logos/
Allow: /

# Bing Bot
User-agent: Bingbot
Allow: /
Crawl-delay: 0

# Yandex
User-agent: Yandex
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# ============================================
# AI SEARCH ENGINE BOTS - FULL ACCESS
# ============================================

# OpenAI (ChatGPT)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl (trains many AI models)
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Apple (Siri, Spotlight)
User-agent: Applebot
Allow: /

# Amazon (Alexa)
User-agent: Amazonbot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Brave Search
User-agent: Brave-Indexer
Allow: /

# ============================================
# SEO/RESEARCH TOOLS (Rate Limited)
# ============================================

# Ahrefs
User-agent: AhrefsSiteAudit
Allow: /
Crawl-delay: 2

User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

# Semrush
User-agent: SemrushBot
Allow: /
Crawl-delay: 10

# Moz
User-agent: rogerbot
Allow: /
Crawl-delay: 5
User-agent: dotbot
Allow: /
Crawl-delay: 5

# Majestic
User-agent: MJ12bot
Allow: /
Crawl-delay: 10

# ============================================
# SITEMAPS
# ============================================
Sitemap: https://anyrxo.com/sitemap.xml

# AI-specific content indexes
Sitemap: https://anyrxo.com/llms.txt
Sitemap: https://anyrxo.com/ai.txt

# ============================================
# SECURITY - HONEYPOTS & BLOCKED PATHS
# ============================================

# Note: These are traps for malicious bots
# Accessing these will be logged and may result in legal action

# Block common attack vectors
Disallow: /admin.php
Disallow: /wp-config.php
Disallow: /phpinfo.php
Disallow: /.env
Disallow: /.env.example
Disallow: /config.json
Disallow: /.git-credentials
Disallow: /wp-admin/
Disallow: /administrator/
Disallow: /phpmyadmin/
Disallow: /backup/
Disallow: /db-backup/
Disallow: /database/
Disallow: /.git/
Disallow: /.claude/

# ============================================
# HOST
# ============================================
Host: https://anyrxo.com