# Concurred Robots.txt

# Content-Signal directives (contentsignals.org / draft-romm-aipref-contentsignals)
# Declare AI content usage preferences: search is welcome, AI training and
# on-the-fly ingestion as AI inputs are not permitted without explicit agreement.
Content-Signal: search=yes, ai-train=no, ai-input=no

User-agent: *
Content-Signal: search=yes, ai-train=no, ai-input=no
Allow: /
Disallow: /api/

# Public skill pack for agent builders (SDK-docs-like artifacts).
# Explicit Allow so the Content-Signal above isn't read as an ambiguous
# "don't fetch skills for your AI build tool" — fetching these is the
# whole point. Immutable versioned paths under /skills/v<semver>/* and
# a rolling /skills/latest/* alias.
Allow: /skills/

# AI/LLM documentation (llmstxt.org standard)
# https://concurred.ai/llms.txt
# https://concurred.ai/llms-full.txt

# Sitemap location
Sitemap: https://concurred.ai/sitemap.xml

# Crawl-delay for different bots
User-agent: Googlebot
Crawl-delay: 0

User-agent: Bingbot
Crawl-delay: 1

# Block common bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /