# humanswith.ai — robots.txt
# We sell AI visibility. We WANT every AI to crawl. Explicit allowlist for
# the engines our buyers use; default Allow: / for everyone else.

User-agent: *
Allow: /
Disallow: /404

# OpenAI — both retrieval (search) and training discovery
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /

# Anthropic — all four Claude user-agents (training, search, user-initiated, agent)
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# Google — Search Generative Experience / AI Overviews
User-agent: Googlebot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: GoogleOther
Allow: /

# Microsoft — Copilot / Bing AI
User-agent: bingbot
Allow: /
User-agent: BingPreview
Allow: /

# Apple — Spotlight AI / Apple Intelligence
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# Yandex Neuro / Alice (RU AI search)
User-agent: YandexBot
Allow: /
User-agent: YandexAI
Allow: /
User-agent: YandexAdditional
Allow: /

# DeepSeek
User-agent: DeepSeekBot
Allow: /

# ByteDance — Doubao/Kimi adjacent
User-agent: Bytespider
Allow: /

# Amazon — Alexa+/Q
User-agent: Amazonbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Meta — Meta AI assistant + Llama-* training discovery
User-agent: Meta-ExternalAgent
Allow: /
User-agent: Meta-ExternalFetcher
Allow: /
User-agent: FacebookBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Kagi (paid AI search; tiny but high-intent buyer audience)
User-agent: Kagibot
Allow: /

# Common Crawl — used as training source by many AI providers (CCBot)
User-agent: CCBot
Allow: /

# Hugging Face crawler
User-agent: huggingface-crawler
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

Sitemap: https://humanswith.ai/sitemap-index.xml