# Leads to Sales — robots.txt
# Last updated: 2026-05-02
# Mirror of /robots (app/robots.ts). The dynamic Next.js route is the source of truth;
# this file is kept for tooling that fetches the static file directly.
#
# Policy: every listed crawler is welcomed onto all public pages.
# /api/og is explicitly allowed everywhere so social/AI bots can fetch
# dynamic OpenGraph share images, while the rest of /api/ stays private.

# ─────────────────────────────────────────────
# AI search engines & training crawlers — explicitly allowed
# ─────────────────────────────────────────────

# OpenAI — ChatGPT search index
User-agent: GPTBot
Allow: /
Allow: /api/og
Disallow: /api/

# OpenAI — ChatGPT browsing tool (live fetch on user request)
User-agent: ChatGPT-User
Allow: /
Allow: /api/og
Disallow: /api/

# OpenAI — SearchGPT crawler
User-agent: OAI-SearchBot
Allow: /
Allow: /api/og
Disallow: /api/

# Google — AI Overviews / Gemini / Vertex AI training opt-in
User-agent: Google-Extended
Allow: /
Allow: /api/og
Disallow: /api/

# Google — classic search
User-agent: Googlebot
Allow: /
Allow: /api/og
Disallow: /api/

# Microsoft Bing — search & Copilot
User-agent: Bingbot
Allow: /
Allow: /api/og
Disallow: /api/

# Apple — Spotlight, Siri, Safari Suggestions
User-agent: Applebot
Allow: /
Allow: /api/og
Disallow: /api/

# Apple — Apple Intelligence training opt-in
User-agent: Applebot-Extended
Allow: /
Allow: /api/og
Disallow: /api/

# Perplexity — search index
User-agent: PerplexityBot
Allow: /
Allow: /api/og
Disallow: /api/

# Perplexity — user-triggered fetch (alternate spellings observed in the wild)
User-agent: PerplexityBot-User
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Allow: /api/og
Disallow: /api/

# Anthropic — Claude search & training crawlers (alternate spellings of Claude-Web)
User-agent: ClaudeBot
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: Claude-Web
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: ClaudeWeb
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: Claude-User
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Allow: /api/og
Disallow: /api/

# Common Crawl — feeds many open AI datasets
User-agent: CCBot
Allow: /
Allow: /api/og
Disallow: /api/

# Amazon — Alexa / Rufus / AGI training
User-agent: Amazonbot
Allow: /
Allow: /api/og
Disallow: /api/

# Meta — Llama training & live fetch
User-agent: Meta-ExternalAgent
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: Meta-ExternalFetcher
Allow: /
Allow: /api/og
Disallow: /api/

User-agent: FacebookBot
Allow: /
Allow: /api/og
Disallow: /api/

# ByteDance — Doubao / TikTok search
User-agent: Bytespider
Allow: /
Allow: /api/og
Disallow: /api/

# DuckDuckGo / DuckAssist
User-agent: DuckDuckBot
Allow: /
Allow: /api/og
Disallow: /api/

# Yandex — search & YandexGPT
User-agent: YandexBot
Allow: /
Allow: /api/og
Disallow: /api/

# Cohere — Coral / Command R training
User-agent: Cohere-AI
Allow: /
Allow: /api/og
Disallow: /api/

# Mistral — Le Chat browsing
User-agent: MistralAI-User
Allow: /
Allow: /api/og
Disallow: /api/

# DeepSeek
User-agent: DeepSeekBot
Allow: /
Allow: /api/og
Disallow: /api/

# You.com
User-agent: you.com
Allow: /
Allow: /api/og
Disallow: /api/

# ─────────────────────────────────────────────
# Social preview crawlers — need /api/og for share cards
# ─────────────────────────────────────────────

# X / Twitter share previews
User-agent: Twitterbot
Allow: /
Allow: /api/og
Disallow: /api/

# Facebook / Instagram link previews
User-agent: facebookexternalhit
Allow: /
Allow: /api/og
Disallow: /api/

# LinkedIn share previews
User-agent: LinkedInBot
Allow: /
Allow: /api/og
Disallow: /api/

# Slack link unfurling
User-agent: Slackbot
Allow: /
Allow: /api/og
Disallow: /api/

# Discord link embeds
User-agent: Discordbot
Allow: /
Allow: /api/og
Disallow: /api/

# ─────────────────────────────────────────────
# Default policy for everything else
# ─────────────────────────────────────────────
User-agent: *
Allow: /
Allow: /api/og
Disallow: /api/
Disallow: /dev/

Sitemap: https://leadstosalesagency.com/sitemap.xml
Host: https://leadstosalesagency.com