# TEST-ROBOTS-ORIGIN-2026
# ==========================================
# OpenAI crawlers
# ==========================================

# Required for ChatGPT Ads landing-page review
User-agent: OAI-AdsBot
Allow: /

# Recommended for ChatGPT search visibility
User-agent: OAI-SearchBot
Allow: /

# Optional: allow ChatGPT to fetch pages when a user asks it to visit them
User-agent: ChatGPT-User
Allow: /

# Do not allow OpenAI model-training crawler
User-agent: GPTBot
Disallow: /


# ==========================================
# Other AI / training crawlers blocked
# ==========================================

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /


# ==========================================
# General crawler rules
# ==========================================

User-agent: *
Allow: /
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/woocommerce_transient_files/
Disallow: /wp-content/uploads/woocommerce_uploads/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


# ==========================================
# Sitemaps
# ==========================================

Sitemap: https://gcomsolutions.co.uk/sitemap_index.xml
Sitemap: https://gcomsolutions.co.uk/sitemap.xml