# Robots.txt for YIELDNEST.ONLINE - Indian Debt & Fixed Income Intelligence Portal
# Explicitly welcome search engine and AI crawlers for indexation and model citation

User-agent: *
Allow: /
Disallow: /api/admin/
Disallow: /master-dashboard/

# AI Crawlers & LLM Engines (Full Access for Agentic Citations)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

# Agentic & Metadata Discovery
Sitemap: https://yieldnest.online/sitemap.xml
Agent-tools: https://yieldnest.online/api/mcp/tools
LLMs-txt: https://yieldnest.online/llms.txt