# RBX Invest — Roblox inventory and reseller analytics platform
# https://rbxinvest.com

User-agent: *
Allow: /
# Content Signals (contentsignals.org): search, AI retrieval and AI training
# are all permitted on public pages. Note N-0001.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# No page is disallowed. Note N-0002.
# The catalog lists (/items, /players, /creators, /collections/*) and their
# numbered pages are open. Their sort, filter, page-size and search-box
# parameters are not crawled: each one re-orders or narrows the same list.
# Note N-0007.
Disallow: /items?q=
Disallow: /items?*&q=
Disallow: /items?f=
Disallow: /items?*&f=
Disallow: /items?sort=
Disallow: /items?*&sort=
Disallow: /items?dir=
Disallow: /items?*&dir=
Disallow: /items?type=
Disallow: /items?*&type=
Disallow: /items?avail=
Disallow: /items?*&avail=
Disallow: /items?window=
Disallow: /items?*&window=
Disallow: /items?avg=
Disallow: /items?*&avg=
Disallow: /items?asset=
Disallow: /items?*&asset=
Disallow: /players?metric=
Disallow: /players?*&metric=
Disallow: /players?size=
Disallow: /players?*&size=
Disallow: /creators?q=
Disallow: /creators?*&q=
Disallow: /creators?sort=
Disallow: /creators?*&sort=
Disallow: /creators?dir=
Disallow: /creators?*&dir=
Disallow: /collections/*?sort=
Disallow: /collections/*?*&sort=

# Major search bots. Each group repeats the list-parameter lines above, since a
# crawler reads only the group that names it. Note N-0007.
User-agent: Googlebot
Allow: /
Disallow: /items?q=
Disallow: /items?*&q=
Disallow: /items?f=
Disallow: /items?*&f=
Disallow: /items?sort=
Disallow: /items?*&sort=
Disallow: /items?dir=
Disallow: /items?*&dir=
Disallow: /items?type=
Disallow: /items?*&type=
Disallow: /items?avail=
Disallow: /items?*&avail=
Disallow: /items?window=
Disallow: /items?*&window=
Disallow: /items?avg=
Disallow: /items?*&avg=
Disallow: /items?asset=
Disallow: /items?*&asset=
Disallow: /players?metric=
Disallow: /players?*&metric=
Disallow: /players?size=
Disallow: /players?*&size=
Disallow: /creators?q=
Disallow: /creators?*&q=
Disallow: /creators?sort=
Disallow: /creators?*&sort=
Disallow: /creators?dir=
Disallow: /creators?*&dir=
Disallow: /collections/*?sort=
Disallow: /collections/*?*&sort=

User-agent: Bingbot
Allow: /
Disallow: /items?q=
Disallow: /items?*&q=
Disallow: /items?f=
Disallow: /items?*&f=
Disallow: /items?sort=
Disallow: /items?*&sort=
Disallow: /items?dir=
Disallow: /items?*&dir=
Disallow: /items?type=
Disallow: /items?*&type=
Disallow: /items?avail=
Disallow: /items?*&avail=
Disallow: /items?window=
Disallow: /items?*&window=
Disallow: /items?avg=
Disallow: /items?*&avg=
Disallow: /items?asset=
Disallow: /items?*&asset=
Disallow: /players?metric=
Disallow: /players?*&metric=
Disallow: /players?size=
Disallow: /players?*&size=
Disallow: /creators?q=
Disallow: /creators?*&q=
Disallow: /creators?sort=
Disallow: /creators?*&sort=
Disallow: /creators?dir=
Disallow: /creators?*&dir=
Disallow: /collections/*?sort=
Disallow: /collections/*?*&sort=

# Social link preview bots
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# AI search bots. Allow citations and summaries of public pages.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /
# Note N-0003.
Crawl-delay: 2

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

# Site index for language models (llmstxt.org): https://rbxinvest.com/llms.txt
#
# Item, creator, player and key pages are also served as Markdown: append .md
# to the path, or send Accept: text/markdown to the normal URL. Those URLs are
# noindex. Documentation: https://rbxinvest.com/api. Note N-0004.

# Sitemap index; the child sitemaps are listed inside it. Note N-0005.
Sitemap: https://rbxinvest.com/sitemap.xml