# REE Waste — robots.txt
# Crawl optimization for search engines and AI systems

User-agent: *
Allow: /
Allow: /services
Allow: /services/*
Allow: /areas
Allow: /areas/*
Allow: /pricing
Allow: /blog
Allow: /blog/*
Allow: /about
Allow: /contact
Allow: /book-rubbish-removal

# Disallow utility pages (session-based, private, or temporary)
Disallow: /book
Disallow: /cart
Disallow: /account
Disallow: /account/*
Disallow: /adminrocket_login
Disallow: /adminrocket_login/*

# Disallow parameter URLs (duplicate content risk)
Disallow: /*?*
Disallow: /*#*

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Sitemap
Sitemap: https://reewaste.co.uk/sitemap.xml
Sitemap: https://reewaste.co.uk/robots.txt

# Specific directives for Googlebot
User-agent: Googlebot
Allow: /
Allow: /services
Allow: /areas
Allow: /pricing
Allow: /blog
Allow: /book-rubbish-removal
Disallow: /book
Disallow: /cart
Disallow: /account
Disallow: /adminrocket_login
Crawl-delay: 0

# Bing/MS Search
User-agent: Bingbot
Allow: /
Disallow: /book
Disallow: /cart
Disallow: /account
Crawl-delay: 1

# AI Crawlers (ChatGPT, Claude, Gemini, Perplexity, etc.)
User-agent: GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Googlebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: PetalBot
Allow: /

# Respect for bad actors / scrapers
User-agent: MJ12bot
Allow: /

# Note: No noindex directives here — handled via meta tags on each page
# /book, /account, /cart are set to noindex via SEOHead component