# Robots.txt for shamonjgifts.com
# AI Visibility Optimized - 48+ AI Crawlers
# Updated: 2026-07-28 | Generated by AppWT Web & AI Solutions

# =====================================================
# xAI / GROK - LISTED FIRST BY POLICY
# Grok leads the allowlist on every property AppWT maintains.
# Group order does not change crawler behaviour, because each
# crawler selects its own most-specific group. This is a house
# standard ordering, not a functional gate.
# =====================================================

User-agent: Grok
Allow: /

User-agent: xAI-Bot
Allow: /

# =====================================================
# USA TIER 1: DOMINANT (75%+ market share)
# =====================================================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google Gemini / AI Overviews
User-agent: Google-Extended
Allow: /

User-agent: Googlebot-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Microsoft Copilot / Bing
User-agent: Bingbot
Allow: /

User-agent: BingPreview
Allow: /

# =====================================================
# USA TIER 2: MAJOR PLAYERS
# =====================================================

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: FacebookExternalHit
Allow: /

# xAI Grok
User-agent: xAI-Grok
Allow: /

# =====================================================
# USA TIER 3: SPECIALTY
# =====================================================

# Apple Intelligence / Siri
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: SiriBot
Allow: /

# Amazon Alexa / Q
User-agent: Amazonbot
Allow: /

User-agent: AlexaBot
Allow: /

# Microsoft Cortana
User-agent: CortanaBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# Phind AI
User-agent: PhindBot
Allow: /

# Brave Search
User-agent: BraveBot
Allow: /

# =====================================================
# CANADA
# =====================================================

User-agent: cohere-ai
Allow: /

User-agent: CohereBot
Allow: /

# =====================================================
# EUROPE
# =====================================================

User-agent: MistralBot
Allow: /

User-agent: StabilityBot
Allow: /

# =====================================================
# CHINA
# =====================================================

User-agent: DeepSeekBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: ByteDance
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: AlibabaBot
Allow: /

User-agent: YisouSpider
Allow: /

User-agent: Sosospider
Allow: /

User-agent: TencentTraveler
Allow: /

User-agent: 360Spider
Allow: /

User-agent: PetalBot
Allow: /

User-agent: Sogou
Allow: /

# =====================================================
# ASIA-PACIFIC
# =====================================================

User-agent: Yeti
Allow: /

User-agent: NaverBot
Allow: /

User-agent: SamsungBot
Allow: /

User-agent: LINEBot
Allow: /

# =====================================================
# AI DATA COLLECTION & TRAINING
# =====================================================

User-agent: Omgilibot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: VelenPublicWebCrawler
Allow: /

User-agent: img2dataset
Allow: /

User-agent: Kangaroo Bot
Allow: /

User-agent: GitHubCopilotBot
Allow: /

# =====================================================
# SEARCH ENGINES
# =====================================================

User-agent: Googlebot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

# =====================================================
# DEFAULT RULE
# =====================================================

User-agent: *
Allow: /

# Block sensitive areas
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /feed/
Disallow: /*?*

# Allow important content
Allow: /wp-content/uploads/

# =====================================================
# SITEMAP & AI DISCOVERY FILES
# =====================================================

# WordPress core answers /sitemap.xml with a 301 to /wp-sitemap.xml. Declaring the
# redirect target directly means crawlers reach the served artifact in one hop, and an
# auditor that correctly refuses to follow redirects still finds a sitemap.
Sitemap: https://shamonjgifts.com/wp-sitemap.xml

# AI DISCOVERY & TRANSPARENCY FILES
llms.txt: https://shamonjgifts.com/llms.txt
llms-full.txt: https://shamonjgifts.com/llms-full.txt
business-facts: https://shamonjgifts.com/business-facts/