User-agent: *

Sitemap: https://www.dica33.it/sitemap.xml

Disallow: /*.xls$
Disallow: /*.doc$
Disallow: /*.swf$
Disallow: /*.rtf$


# Moz
User-agent: rogerbot
Disallow: /

# Ahrefs
User-agent: AhrefsBot
Disallow: /

# Semrush
User-agent: SemrushBot
Disallow: /

# Majestic
User-agent: MJ12bot
Disallow: /

# Babbar
User-agent: Barkrowler
Disallow: /

# Screaming Frog
User-agent: Screaming Frog SEO Spider
Disallow: /

# Sistrix
User-agent: Sistrix
Disallow: /

# SEOkicks
User-agent: SEOkicks
Disallow: /

# Serpstat
User-agent: serpstatbot
Disallow: /

# Netpeak
User-agent: NetpeakSpider
Disallow: /

# DotBot (used by Moz & others)
User-agent: dotbot
Disallow: /

# BLEXBot (often aggressive)
User-agent: BLEXBot
Disallow: /

# ZoomInfo
User-agent: ZoominfoBot
Disallow: /

# Linkpad
User-agent: LinkpadBot
Disallow: /

# DataForSEO
User-agent: DataForSeoBot
Disallow: /

# ltx71 (data miner)
User-agent: ltx71
Disallow: /

# MegaIndex.ru (Russian link miner)
User-agent: MegaIndex.ru
Disallow: /

# Domain crawler (Generic)
User-agent: domaincrawler
Disallow: /