# ---------------------------------------------------------------------------
# AI crawlers
#
# Policy: block crawlers that TRAIN models on our content, but ALLOW the
# real-time "answer" fetchers AI assistants use to read and cite a page when
# a user asks about it. Those drive referral traffic and align with our MCP
# server (https://pawlisty.com/mcp). See docs/mcp-server.md.
#
# NOTE: Cloudflare's managed robots.txt block (prepended at the edge) also
# blocks the training crawlers below — but it does NOT touch the answer
# fetchers, so this file is what keeps them allowed. Keep both in sync.
# ---------------------------------------------------------------------------

# --- AI training crawlers: blocked ---
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Cohere-ai
Disallow: /

# --- AI real-time answer fetchers: ALLOWED (referral traffic + MCP alignment) ---
# Intentionally NOT listed here so they fall under "User-agent: *  Allow: /"
# below (admin/assets are still excluded there):
#   ChatGPT-User    – ChatGPT browses on behalf of a user
#   OAI-SearchBot   – indexes pages for ChatGPT Search
#   PerplexityBot   – indexes pages so Perplexity can cite them
#   Perplexity-User – Perplexity fetches a page for a specific user query

# FacebookBot allowed — needed for link previews on auto-posted listings
# User-agent: FacebookBot
# Disallow: /

# meta-webindexer is Meta's search-index crawler — not needed (we don't
# rely on Facebook Search for discovery) and it walks the /search filter
# matrix hard enough to saturate PHP-FPM.
User-agent: meta-webindexer
Disallow: /

# Search Engines - Allow
# /assets/, /css/ and /js/ are intentionally NOT disallowed: pages load
# scripts and styles from them, and Google needs those to render a page.
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /vendor/

# Faceted/sorted listing variants carry <meta robots="noindex"> already
# (settings.seo.no_index_*), so crawling them only burns crawl budget that
# listings, breed hubs and breed×city pages need. Plain pagination stays
# crawlable so deep listings are still discovered via ?page=N.
Disallow: /search?
Disallow: /location/*?
Disallow: /category/*?
Allow: /location/*?page=
Allow: /category/*?page=
Disallow: /main.php
Disallow: /index.php
Disallow: /mix-manifest.json
Disallow: /*?display=*
Disallow: /locale/en
Disallow: /locale/fr
Disallow: /locale/es
Disallow: /locale/ar
Disallow: /locale/pt
Disallow: /locale/de
Disallow: /locale/it
Disallow: /locale/tr
Disallow: /locale/ru
Disallow: /locale/hi
Disallow: /locale/bn
Disallow: /locale/zh
Disallow: /locale/ja
Disallow: /locale/he
Disallow: /locale/th
Disallow: /locale/ro
Disallow: /locale/ka
Disallow: /auth/connect/facebook
Disallow: /auth/connect/linkedin
Disallow: /auth/connect/twitter
Disallow: /auth/connect/google


Sitemap: https://pawlisty.com/us/sitemaps.xml