# robots.txt for https://pfas-free.eu (WordPress optimized + LLM & bot rules)

# Global rules for all bots
User-agent: *
# Disallow core WordPress admin and system directories
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /cgi-bin/
Disallow: /xmlrpc.php

# Disallow sensitive or duplicate content
Disallow: /private/
Disallow: /staging/
Disallow: /backup/
Disallow: /tmp/

# Disallow specific pages and sections
Disallow: /privacy-policy/
Disallow: /terms/
Disallow: /cookie-policy-eu/
Disallow: /news/

# Allow media, themes, and essential assets
Allow: /wp-content/uploads/
Allow: /wp-content/themes/

# Optional: Block common URL parameters used for sorting or tracking
Disallow: /*?s=
Disallow: /*?replytocom=
Disallow: /*?share=

# Crawl-delay for all bots (in seconds)
Crawl-delay: 10

# Googlebot specific rules
User-agent: Googlebot
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Disallow: /wp-admin/
Crawl-delay: 5

# Bingbot specific rules
User-agent: Bingbot
Crawl-delay: 10

# AI / LLM bots (example: GPTBot, ClaudeBot, MPTBot)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: MPTBot
Disallow: /wp-admin/
Disallow: /private/
Disallow: /staging/
Disallow: /backup/
Disallow: /tmp/
Disallow: /privacy-policy/
Disallow: /terms/
Disallow: /cookie-policy-eu/
Disallow: /news/
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Crawl-delay: 15

# Sitemap location for crawlers
Sitemap: https://pfas-free.eu/sitemap_index.xml

# Last updated: 2025-10-10