# robots.txt for https://ahmedinnov.com
# https://www.rfc-editor.org/rfc/rfc9309

# Default policy — every well-behaved crawler
User-agent: *
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Content-Signal — declares AI usage preferences for this content.
# https://contentsignals.org/
# Ahmed Mohamed Portfolio: training=yes, search=yes, ai-input=yes
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# ---------------------------------------------------------------------
# Explicit allow for major AI crawlers
# All AI crawlers get full access to content + AI discovery endpoints
# ---------------------------------------------------------------------

# OpenAI — training crawler
User-agent: GPTBot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# OpenAI — SearchGPT in-product crawler
User-agent: OAI-SearchBot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# OpenAI — ChatGPT on-demand fetches
User-agent: ChatGPT-User
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Anthropic — Claude Web/Claude
User-agent: Claude-Web
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

User-agent: ClaudeBot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

User-agent: anthropic-ai
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Google — Generative AI / AI Overviews / Gemini
User-agent: Google-Extended
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Microsoft — Copilot
User-agent: Bingbot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Common Crawl — feeds many LLMs
User-agent: CCBot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Meta / Facebook AI crawlers
User-agent: FacebookBot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

User-agent: meta-externalagent
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Cohere
User-agent: cohere-ai
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Mistral AI
User-agent: MistralAI-User
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /
Disallow: /api/

# Diffbot — structured data crawler
User-agent: Diffbot
Allow: /api/md
Allow: /llms.txt
Allow: /.well-known/
Allow: /

# ---------------------------------------------------------------------
# Bots we explicitly block (aggressive scrapers with no AI policy)
# ---------------------------------------------------------------------
User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ---------------------------------------------------------------------
# AI discovery resources
# ---------------------------------------------------------------------
# llms.txt (plain-text AI content declaration)
# https://ahmedinnov.com/llms.txt
# https://ahmedinnov.com/.well-known/llms.txt
#
# Markdown for Agents
# https://ahmedinnov.com/api/md
#
# MCP server discovery
# https://ahmedinnov.com/.well-known/mcp.json
#
# OpenAI plugin descriptor
# https://ahmedinnov.com/.well-known/ai-plugin.json
#
# Agent metadata
# https://ahmedinnov.com/.well-known/agent.json
#
# OpenAPI 3.1 specification
# https://ahmedinnov.com/openapi.json

# ---------------------------------------------------------------------
# Sitemap + canonical host
# ---------------------------------------------------------------------
Sitemap: https://ahmedinnov.com/sitemap.xml
Host: ahmedinnov.com