# https://www.robotstxt.org/robotstxt.html

User-agent: *
# Block API and server-side routes — bots should never hit these
Disallow: /api/
# Block auth flows — no indexable content and avoids crawl budget waste
Disallow: /auth/
# Block parameterised finder URLs (/finder?q=dress etc.) — search result pages
# have no unique value and waste crawl budget. /finder itself stays indexable.
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Allow everything else — product, outfit, feed, finder, and static pages are all indexable
Allow: /

Sitemap: https://hypsh.com/sitemap.xml

# ---------------------------------------------------------------------------
# AI crawlers — training bots blocked, retrieval/search bots allowed.
#
# Training bots scrape content to build AI models — we get no citation value
# from them. Retrieval bots are what actually surfaces the site in ChatGPT
# Shopping, Perplexity, Claude, etc. — blocking them removes us from AI answers.
#
# Note: Bytespider ignores robots.txt ~90% of the time; block at WAF/edge too.
#
# Content-Signal (https://contentsignals.org, IETF draft-romm-aipref-contentsignals)
# states the same policy as a machine-readable usage preference, for crawlers that
# read it instead of (or as well as) matching their own user-agent group:
#   search=yes    search index + links/snippets
#   ai-input=yes  retrieval / grounding for AI answers (the citations we want)
#   ai-train=no   training or fine-tuning models
# It is repeated in every allowed group for the same reason the Disallow lines
# are: a crawler reads only its own most specific group (RFC 9309).
# ---------------------------------------------------------------------------

# -- Block: training scrapers (no citation benefit) --

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# -- Allow: retrieval and search indexing bots (these drive AI citations) --
#
# Each group repeats the wildcard group's Disallow lines on purpose. A crawler
# obeys only the SINGLE most specific group that matches it (RFC 9309) — named
# groups replace the `*` group, they do not extend it. Omitting /finder? here
# would hand these bots the unbounded ?q= permutation space that line 10 exists
# to keep out. check_prerender_bots.py asserts this holds.

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

# Note: Meta-ExternalFetcher (retrieval) is the counterpart to
# Meta-ExternalAgent (training, blocked above) — they are different bots.
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

User-agent: Google-CloudVertexBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /finder?
Content-Signal: search=yes, ai-input=yes, ai-train=no

# These four were already permitted by the `User-agent: *` / `Allow: /` rule
# above; they are named explicitly so this file and the nginx prerender map in
# tools/nginx-prerender.conf.snippet can be diffed against each other. A bot
# allowed here but absent there is told to crawl and then served an empty SPA
# shell, which is the failure this pairing exists to make visible.