# Robots.txt - Rankzik Website
# Allow legitimate search engines, block aggressive crawlers

# Allow major search engines
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
Allow: /

# Block aggressive crawlers and scrapers
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
Disallow: /

# Block AI scrapers (optional - uncomment if needed)
# User-agent: GPTBot
# User-agent: ChatGPT-User
# User-agent: CCBot
# User-agent: anthropic-ai
# User-agent: Claude-Web
# Disallow: /

# Default rule for all other bots
User-agent: *
Crawl-delay: 10

# Sitemap location
Sitemap: https://www.rankzik.com/sitemap.xml
Sitemap: https://www.rankzik.com/sitemap1.xml