# Socioverse Robots.txt
# Website: https://www.socioverse.io
# Last Updated: 2026-03-22

# ============================================
# Default rules for all crawlers
# ============================================
User-agent: *
Allow: /
Allow: /home
Allow: /pricing
Allow: /features
Allow: /api
Allow: /signup
Allow: /login
Allow: /about
Allow: /contact
Allow: /terms
Allow: /privacy
Allow: /shipping
Allow: /refunds
Allow: /docs
Allow: /promo
Allow: /vs
Allow: /vs/
Allow: /for
Allow: /for/
Allow: /use-cases
Allow: /use-cases/

# Block authenticated/private areas
Disallow: /dashboard/
Disallow: /onboarding/
Disallow: /instagram/callback
Disallow: /admin/

# Block API and internal endpoints
Disallow: /api/
Disallow: /.well-known/

# Block redirect shortcuts (prevent duplicate content indexing)
Disallow: /content-studio
Disallow: /automations
Disallow: /messages
Disallow: /templates
Disallow: /instagram
Disallow: /billing
Disallow: /settings

# Block utility pages (no SEO value)
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /unsubscribe

# Block query parameters that create duplicate content
Disallow: /*?ref=
Disallow: /*?utm_

# Crawl delay for polite crawling
Crawl-delay: 1

# ============================================
# Googlebot (no crawl-delay, Google ignores it)
# ============================================
User-agent: Googlebot
Allow: /
Allow: /home
Allow: /signup
Allow: /login
Allow: /about
Allow: /contact
Allow: /terms
Allow: /privacy
Allow: /shipping
Allow: /refunds
Allow: /docs
Allow: /promo
Disallow: /dashboard/
Disallow: /onboarding/
Disallow: /instagram/callback
Disallow: /admin/
Disallow: /api/
Disallow: /.well-known/
Disallow: /content-studio
Disallow: /automations
Disallow: /messages
Disallow: /templates
Disallow: /instagram
Disallow: /billing
Disallow: /settings
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /unsubscribe

# Googlebot-Image
User-agent: Googlebot-Image
Allow: /
Allow: /assets/
Allow: /images/
Disallow: /dashboard/
Disallow: /api/

# ============================================
# Bingbot
# ============================================
User-agent: Bingbot
Allow: /
Allow: /home
Allow: /signup
Allow: /login
Allow: /about
Allow: /contact
Allow: /terms
Allow: /privacy
Allow: /shipping
Allow: /refunds
Allow: /docs
Allow: /promo
Disallow: /dashboard/
Disallow: /onboarding/
Disallow: /instagram/callback
Disallow: /admin/
Disallow: /api/
Disallow: /.well-known/
Disallow: /content-studio
Disallow: /automations
Disallow: /messages
Disallow: /templates
Disallow: /instagram
Disallow: /billing
Disallow: /settings
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /unsubscribe
Crawl-delay: 2

# ============================================
# Yandex
# ============================================
User-agent: Yandex
Allow: /
Disallow: /dashboard/
Disallow: /onboarding/
Disallow: /api/
Disallow: /admin/
Crawl-delay: 3

# ============================================
# Social media crawlers - allow for rich previews
# ============================================
User-agent: facebookexternalhit
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: Twitterbot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: LinkedInBot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: WhatsApp
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: Slackbot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: Discordbot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: TelegramBot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: Pinterest
Allow: /
Disallow: /dashboard/
Disallow: /api/

# ============================================
# AI crawlers - allow public content for LLM indexing
# ============================================
User-agent: GPTBot
Allow: /
Allow: /home
Allow: /about
Allow: /docs
Disallow: /dashboard/
Disallow: /api/
Disallow: /onboarding/

User-agent: Google-Extended
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /dashboard/
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /dashboard/
Disallow: /api/

# ============================================
# SEO tool crawlers - rate limited
# ============================================
User-agent: SemrushBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Crawl-delay: 10

User-agent: AhrefsBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Crawl-delay: 10

User-agent: rogerbot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Crawl-delay: 5

User-agent: DataForSeoBot
Allow: /
Disallow: /dashboard/
Disallow: /api/
Crawl-delay: 10

# ============================================
# Block aggressive/malicious bots
# ============================================
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SearchmetricsBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: ltx71
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Sogou
Disallow: /

# ============================================
# Sitemap & Host
# ============================================
Sitemap: https://www.socioverse.io/sitemap.xml
Host: https://www.socioverse.io