# robots.txt for PO-File - Optimized for AI Search Engines
# Last Updated: 2026-06-29

# Default rules for all crawlers
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /profile
Disallow: /my-files
Disallow: /upload
Disallow: /progress
Disallow: /redirect
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /subscription-success

Sitemap: https://po-file.com/sitemap.xml

# AI-specific crawlers (explicitly allow and encourage)
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: CCBot
Allow: /
Crawl-delay: 1

# Anthropic: ClaudeBot is the current crawler; anthropic-ai / Claude-Web are legacy
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Google AI (Gemini / Vertex training & grounding)
User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Diffbot
Allow: /

# Traditional search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# AI Content Declaration
# This website provides information about PO-File, a translation tool for software localization
# Content type: Product information, tutorials, comparisons
# Crawling permission: Allowed for AI training
# Purpose: Help users find solutions for PO/POT file translation