# GP Sourcing — Selective crawling policy
# Allow search engines + AI bots, block scrapers
#
# ⚠️ robots.txt precedence: a crawler obeys ONLY the most-specific group whose
# User-agent it matches, and IGNORES `User-agent: *` entirely. The previous
# version gave Googlebot/Bingbot/etc. their own `Allow: /` group with NO
# disallows, so every protective `Disallow:` under `*` (/api/, /sourcing/,
# /account, /share/…) was silently ignored by Google — it crawled /api/og/*
# (→ 404s + 5xx), /sourcing, /account and /share. Fix: the search + AI bots now
# share ONE group that carries the disallows. Multiple `User-agent:` lines with
# no rules between them form a single combined group (Google-supported).

# ── Search engines + AI/LLM crawlers — ALLOWED, but app/API/auth surfaces kept
#    out of the index (saves crawl budget on an 80k-URL property; these surfaces
#    also carry noindex meta, belt-and-suspenders).
#    OpenAI: GPTBot = training corpus, OAI-SearchBot = ChatGPT search index —
#    both needed for full ChatGPT visibility (platform.openai.com/docs/bots).
#    Perplexity: PerplexityBot = background index, Perplexity-User = live browse —
#    both needed (docs.perplexity.ai/guides/bots).
#    `Allow: /api/og/` overrides `Disallow: /api/` so the per-entity OpenGraph
#    images referenced in the image sitemap stay fetchable (longer path wins).
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: YandexBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: AhrefsBot
User-agent: SemrushBot
Allow: /
Allow: /api/og/
Disallow: /sourcing/
Disallow: /account
Disallow: /account/
Disallow: /checkout/
Disallow: /login
Disallow: /signup
Disallow: /api/
Disallow: /share/

# ── Social preview bots — ALLOWED everything, no restrictions. They only fetch
#    og:image + meta for a URL a user explicitly shared (no open crawling), and
#    they must reach /api/og/* + /share/* to render link cards.
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
Allow: /

# ── Scraping tools — BLOCKED
User-agent: python-requests
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: wget
Disallow: /

User-agent: curl
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: libwww-perl
Disallow: /

User-agent: Java
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ── Default: every other bot — allow crawling of public content, block the
#    same app/API/auth surfaces as the search-engine group above.
User-agent: *
Allow: /
Allow: /api/og/
Disallow: /sourcing/
Disallow: /account
Disallow: /account/
Disallow: /checkout/
Disallow: /login
Disallow: /signup
Disallow: /api/
Disallow: /share/

# Sitemap-index points to the per-chunk sitemaps (static / GPs / companies x N).
# The auto-generated /sitemap.xml from Next.js 16 generateSitemaps() is broken
# (returns 404), so we serve our own index at /sitemap-index.xml.
Sitemap: https://www.gp-intel.com/sitemap-index.xml