# AiEGIS robots.txt — 2026-05-25 expansion
# Additive only: every Disallow from the prior version is preserved verbatim.
# AI-bot allowlist consensus 2026: explicit Allow per agent so a future
# default-deny on User-agent: * does not silently exclude these crawlers.

# --- Baseline (everyone) ---
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# --- AI search crawlers (allowed; respect the same enrolment/admin disallows) ---

# OpenAI search index (ChatGPT search). Distinct from GPTBot (training).
# Source: developers.openai.com/api/docs/bots
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Anthropic search index.
# Source: support.claude.com/en/articles/8896518
User-agent: Claude-SearchBot
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Anthropic user-triggered fetcher (Claude users asking the assistant to read a URL).
User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Anthropic training crawler. Allowing AI engines to learn AiEGIS framing is the goal.
User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Legacy Anthropic-AI token (still respected by some Anthropic crawlers).
User-agent: Anthropic-AI
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Google's AI-training token (Gemini / Vertex). Separate from Googlebot search.
User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Perplexity index crawler. Source: docs.perplexity.ai/docs/resources/perplexity-crawlers
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Perplexity user-triggered fetcher.
User-agent: Perplexity-User
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Apple Intelligence training token (separate from Applebot search).
User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# Bing's general crawler — also feeds Copilot answers.
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /admin
Disallow: /enrol/
Disallow: /v1/enrol/
Disallow: /grid/team/
Disallow: /grid/api/

# --- Indexes ---
Sitemap: https://aiegis.ie/sitemap.xml