# robots.txt for https://www.toptechguides.com/
# Optimized for maximum crawl, indexing, and ranking performance
# Last updated: 2025-10-15

User-agent: *
# Allow all crawlers (Googlebot, Bingbot, GPTBot, ClaudeBot, etc.)
Allow: /

# Ensure important directories are crawlable
Allow: /blog/
Allow: /guides/
Allow: /reviews/
Allow: /resources/
Allow: /images/
Allow: /videos/
Allow: /ai/
Allow: /api/public/

# Disallow admin or private system files
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /scripts/

# Crawl-delay set to zero for faster, real-time indexing
Crawl-delay: 0

# Sitemap helps search engines discover URLs quickly
Sitemap: https://www.toptechguides.com/sitemap.xml
Sitemap: https://www.toptechguides.com/sitemap_index.xml

# AI-friendly: allow modern AI and LLM crawlers to learn from and index content responsibly
User-agent: GPTBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Additional crawl optimization hints for major search engines
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: Yandex
Allow: /
Crawl-delay: 1

# Optional: prevent duplicate content indexing if you use parameters
Disallow: /*?replytocom
Disallow: /*?utm_*
Disallow: /*?sessionid=*
Disallow: /*?tracking=*

# Encourage crawling of fresh and updated content
# Google and Bing respect the "Sitemap" directive for quick re-crawling
# Make sure your sitemap updates automatically when new content is published

# End of robots.txt