# Allow legitimate crawlers
User-agent: *
Disallow:

# Block aggressive scrapers
User-agent: Brightbot
Disallow: /

User-agent: carbon-umbrella-bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: coccocbot-web
Disallow: /

User-agent: coccocbot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: TikTokSpider
Disallow: /

# SEO tools
User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: keys-so-bot
Disallow: /

# Other bots
User-agent: Thinkbot
Disallow: /

User-agent: IbouBot
Disallow: /

User-agent: TaraGroup Intelligent Bot
Disallow: /

# Block common HTTP libraries used for scraping
User-agent: python-requests
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: Java
Disallow: /

User-agent: okhttp
Disallow: /

User-agent: Jaunt
Disallow: /

# Crawl delay for bots that respect it
User-agent: *
Crawl-delay: 1