# AgentWeb.live — The internet for AI agents
# 14M+ businesses across 195 countries

User-agent: *
Allow: /
Crawl-delay: 1

# Sitemaps
Sitemap: https://agentweb.live/sitemap.xml
Sitemap: https://agentweb.live/sitemap-businesses.xml

# AI / LLM discoverability
# For AI agents: see /install.md for self-install instructions
# For LLMs: see /llms.txt for structured data about this service
# MCP server: npx agentweb-mcp

# Block internal/admin paths
Disallow: /api/
Disallow: /dashboard/
Disallow: /_partials/

# Allow important paths explicitly
Allow: /b/
Allow: /directory/
Allow: /manifesto
Allow: /leaderboard
Allow: /claim
Allow: /install.md
Allow: /llms.txt
Allow: /.well-known/
Allow: /tools/