User-agent: *
Allow: /
Allow: /api/openapi.json
Disallow: /api/

# AI Crawlers — Welcome
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

# Throttled crawlers — Crawl-delay in seconds
User-agent: meta-externalagent
Crawl-delay: 2

User-agent: facebookexternalhit
Crawl-delay: 2

User-agent: Amazonbot
Crawl-delay: 2

User-agent: Applebot
Crawl-delay: 2

User-agent: Bytespider
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

User-agent: AhrefsBot
Crawl-delay: 5

User-agent: MJ12bot
Crawl-delay: 5

User-agent: DotBot
Crawl-delay: 5

User-agent: PetalBot
Crawl-delay: 5

Sitemap: https://tldfyi.com/sitemap.xml