# robots.txt for nairabhi.com
# Default policy: public pages are crawlable; non-page endpoints are limited.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /oauth/
Disallow: /mcp
Allow: /llms.txt
Allow: /.well-known/api-catalog
Allow: /.well-known/agent-skills/
Allow: /.well-known/mcp/server-card.json
Content-Signal: ai-train=no, search=yes, ai-input=yes

# Search and user-requested retrieval bots are allowed.
User-agent: OAI-SearchBot
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: ChatGPT-User
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Claude-Web
Allow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

# Training or broad corpus crawlers are not allowed to crawl this site.
User-agent: GPTBot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Google-Extended
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Amazonbot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: anthropic-ai
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Bytespider
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: CCBot
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: Applebot-Extended
Disallow: /
Content-Signal: ai-train=no, search=yes, ai-input=yes

Sitemap: https://nairabhi.com/sitemap.xml