# -------------------------
# Block Specific Aggressive or Unknown Bots
# -------------------------
User-agent: SemrushBot
Disallow: /

User-agent: Edg/132.0.0.0
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Python/3.11 aiohttp/3.9.3
Disallow: /

# -------------------------
# Known Search Engine Bots
# -------------------------
User-agent: Googlebot
Allow: /pub/static/
Allow: /media/
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /report/
Disallow: /*.php$
Disallow: /*SID=
# Note: Googlebot ignores Crawl-delay

User-agent: Bingbot
Crawl-delay: 10
Allow: /pub/static/
Allow: /media/
Disallow: /checkout/
Disallow: /customer/
Disallow: /catalogsearch/
Disallow: /report/
Disallow: /*.php$
Disallow: /*SID=

# -------------------------
# General Bot Rules
# -------------------------
User-agent: *
Crawl-delay: 10

Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /*?dir*
Disallow: /*?dir=desc
Disallow: /*?dir=asc
Disallow: /*?limit=all
Disallow: /*?mode*
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /phpserver/
Disallow: /pub/
Disallow: /tag/
Disallow: /review/

# Allow pagination
Allow: /*?p=

# -------------------------
# Sitemap Locations
# -------------------------



Sitemap: https://romdevices.ro/sitemap.xml