# Licensed Trade Charity Robots.txt
# Last updated: October 2025
# Purpose: Optimise for AI search whilst protecting resources

# ===== AI SEARCH CRAWLERS - FULL ACCESS =====
# These power AI-assisted search results
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: CCBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: YouBot
Allow: /

# ===== SEARCH ENGINES - STANDARD ACCESS =====
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: Baiduspider
Allow: /
Crawl-delay: 0

# ===== BLOCK RESOURCE-HEAVY BOTS =====
User-agent: AdsBot
User-agent: AdsBot-Google
User-agent: Mediapartners-Google
Disallow: /

User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
Crawl-delay: 86400

# ===== GENERAL CRAWLERS =====
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/wpo-plugins-tables-list.json
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /?s=
Disallow: /search/
Disallow: *?replytocom=
Disallow: */feed/
Disallow: */comments/
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: *.css
Allow: *.js
Allow: *.jpg
Allow: *.jpeg
Allow: *.png
Allow: *.gif
Allow: *.svg
Allow: *.webp

# ===== SITEMAP DECLARATION =====
Sitemap: https://www.licensedtradecharity.org.uk/sitemap_index.xml
Sitemap: https://www.licensedtradecharity.org.uk/post-sitemap.xml