# Innovation Vista robots.txt
# Allowing reputable AI & search crawlers, Disallowing unnecessary/bad bots

# Default rules
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /articles/page/
Disallow: /global/
Disallow: /affiliates/
Disallow: /referrals/
Disallow: /white-label/

# --- Explicit allow for major LLM crawlers ---
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /  

# --- Allow major search engines ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: SEBot-WA
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

User-agent: AhrefsBot
Allow: /

# --- Disallow known high-volume SEO crawlers ---
User-agent: Baiduspider
Disallow: / 

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Seoscanners
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Sitemap
Sitemap: https://innovationvista.com/sitemap_index.xml