# robots.txt - templates.appwt.com
# AppWT Web & AI Solutions (AppWT LLC)
# Maximum visibility: All search engines and AI crawlers welcome
# Last updated: 2026-02-15

# ==============================================
# SEARCH ENGINES
# ==============================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: BingPreview
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Yandex
Allow: /

# ==============================================
# USA TIER 1: DOMINANT AI PLATFORMS
# ==============================================

# OpenAI / ChatGPT - 75.91% US market share
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google Gemini / AI Overviews
User-agent: Google-Extended
Allow: /

User-agent: Googlebot-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Microsoft Copilot
User-agent: BingBot
Allow: /

User-agent: BingPreview
Allow: /

# ==============================================
# USA TIER 2: MAJOR AI PLAYERS
# ==============================================

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: FacebookExternalHit
Allow: /

# xAI Grok
User-agent: xAI-Grok
Allow: /

# ==============================================
# USA TIER 3: SPECIALTY AI
# ==============================================

# Apple Intelligence / Siri
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: SiriBot
Allow: /

# Amazon Alexa / Q
User-agent: Amazonbot
Allow: /

User-agent: AlexaBot
Allow: /

# Microsoft Cortana
User-agent: CortanaBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# Phind AI
User-agent: PhindBot
Allow: /

# Brave Search
User-agent: BraveBot
Allow: /

# ==============================================
# CANADA
# ==============================================

# Cohere
User-agent: cohere-ai
Allow: /

User-agent: CohereBot
Allow: /

# ==============================================
# EUROPE
# ==============================================

# Mistral AI (France)
User-agent: MistralBot
Allow: /

# Stability AI (UK)
User-agent: StabilityBot
Allow: /

# ==============================================
# CHINA
# ==============================================

# DeepSeek
User-agent: DeepSeekBot
Allow: /

# ByteDance / TikTok
User-agent: Bytespider
Allow: /

User-agent: ByteDance
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# Alibaba
User-agent: AlibabaBot
Allow: /

User-agent: YisouSpider
Allow: /

# Tencent
User-agent: Sosospider
Allow: /

User-agent: TencentTraveler
Allow: /

# Qihoo 360
User-agent: 360Spider
Allow: /

# Huawei
User-agent: PetalBot
Allow: /

# Sogou
User-agent: Sogou
Allow: /

# ==============================================
# ASIA-PACIFIC
# ==============================================

# Naver (South Korea)
User-agent: Yeti
Allow: /

User-agent: NaverBot
Allow: /

# Samsung
User-agent: SamsungBot
Allow: /

# LINE (Japan)
User-agent: LINEBot
Allow: /

# ==============================================
# AI DATA COLLECTION
# ==============================================

User-agent: Omgilibot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: VelenPublicWebCrawler
Allow: /

User-agent: img2dataset
Allow: /

User-agent: Kangaroo Bot
Allow: /

User-agent: GitHubCopilotBot
Allow: /

# ==============================================
# DEFAULT: ALLOW ALL
# ==============================================

User-agent: *
Allow: /
Disallow: /includes/
Disallow: /generate/
Disallow: /cgi-bin/

# Sitemaps
Sitemap: https://templates.appwt.com/sitemap.xml

# =====================================================
# AI DISCOVERY & TRANSPARENCY FILES
# =====================================================
# llms.txt: AI platform discovery and structured information
# llms-full.txt: Comprehensive AI information file (detailed)
# See: https://templates.appwt.com/llms.txt
# See: https://templates.appwt.com/llms-full.txt
# security.txt: https://templates.appwt.com/.well-known/security.txt

llms.txt: https://templates.appwt.com/llms.txt
llms-full.txt: https://templates.appwt.com/llms-full.txt