# robots.txt for minespider.com
# Crawl directives for SEO and LLM-friendly access

# Block pagination query strings
User-agent: *
Disallow: /*?*page=
Disallow: /build/

# Allow Ahrefs crawlers
User-agent: AhrefsBot
Allow: /

User-agent: AhrefsSiteAudit
Allow: /

# Allow OpenAI's GPTBot
User-agent: GPTBot
Disallow:

# Allow Anthropic's ClaudeBot
User-agent: AnthropicBot
Disallow:

# Allow Google's LLM crawler
User-agent: Google-Extended
Disallow:

# Allow Perplexity AI bot
User-agent: PerplexityBot
Disallow:

# Sitemap for faster discovery
Sitemap: https://www.minespider.com/sitemap.xml

Sitemap: https://www.minespider.com/sitemap.xml