# =======================
# SEARCH ENGINES — allowed with restrictions
# =======================

User-agent: Googlebot
Disallow: /nogooglebot/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /*w_*
Disallow: /*q_auto*
Disallow: /*f_auto*
Disallow: /*v1/*

User-agent: bingbot
Crawl-delay: 10
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /*w_*
Disallow: /*q_auto*
Disallow: /*f_auto*
Disallow: /*v1/*

# =======================
# SOCIAL / PREVIEW BOTS — allow full crawl for link previews
# =======================

User-agent: Twitterbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: facebookexternalhit
Allow: /

# =======================
# AI TRAINING BOTS — block all
# =======================

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot
Disallow: /

User-agent: YandexRenderResourcesBot
Disallow: /

# =======================
# SEO / DATA SCRAPER BOTS — block all
# =======================

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeekportBot
Disallow: /

# =======================
# COMPLIANCE / FUNCTIONAL BOTS — allow
# =======================

User-agent: OneTrustBot
Allow: /

# =======================
# CATCH-ALL — block image transforms and API from everything else
# =======================

User-agent: *
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /*w_*
Disallow: /*q_auto*
Disallow: /*f_auto*
Disallow: /*v1/*

# =======================
# SITEMAPS
# =======================


Sitemap: https://www.usparatf.org/sitemap.xml


Sitemap: https://www.usparatf.org/video_sitemap.xml