# Laso Finance - Robots.txt
# https://www.robotstxt.org/robotstxt.html

# Content Signals (https://contentsignals.org/) declare how this site's content
# may be used once it has been fetched. They express preferences, not access
# control -- the Allow/Disallow rules below govern crawling.
#
#   search=yes     Build a search index and link back to us.
#   ai-input=yes   Use a fetched page as input to an AI answer (RAG, grounding,
#                  agent tool use). Our docs, SKILL.md and llms.txt exist for
#                  exactly this, so it is a deliberate yes.
#   ai-train=yes   Use this content to train or fine-tune a model. Every page
#                  here is public product documentation we want models to know,
#                  and Google-Extended is a training-only control, so withholding
#                  this signal reads as "blocked" to agent-readiness checkers.
#
# A crawler that matches a specific User-agent group ignores the "*" group
# entirely, so the directive is repeated in every group it should apply to.

# Default rules for all crawlers
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin
Disallow: /devin-login
Allow: /for-llms

# AI/LLM Crawlers - Welcome!
# We provide structured content for AI systems at /llms.txt and /llms-full.txt

User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: Anthropic-AI
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

# Claude-Web and Anthropic-AI above are retired names, kept for older tooling
# that still sends them. These three are the current Anthropic agents:
# ClaudeBot crawls, Claude-User fetches when a person asks Claude to read a
# page, and Claude-SearchBot indexes for search. Claude-User is the one that
# matters most here: it is what arrives when someone points their agent at
# SKILL.md.
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /SKILL.md
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /SKILL.md
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /SKILL.md
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

User-agent: Cohere-AI
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /for-llms
Disallow: /troubleshoot
Disallow: /usa/rel
Disallow: /international/rel
Disallow: /admin

# Block Internet Archive
User-agent: ia_archiver
Disallow: /

# Sitemaps and AI context files
Sitemap: https://laso.finance/sitemap.xml

# AI-Optimized Content
# llms.txt: https://laso.finance/llms.txt (summary)
# llms-full.txt: https://laso.finance/llms-full.txt (comprehensive)
# For LLMs page: https://laso.finance/for-llms (human-readable)