User-Agent: *
# Paths we don't want to index
Disallow: */index.php/
Disallow: */catalog/category/view/
Disallow: */catalog/product/view/
Disallow: */customer/
Disallow: */checkout/
# Paths that can be safely ignored (no clean URLs)
Disallow: /*?___SID=
Disallow: /*store=
Disallow: /*.php$
# Filetypes we don't want to index
Disallow: /*.pdf
#Prevent crawl errors with cdn
Disallow: /cdn-cgi/
# Problem is mostly related to layered nav with query params and blog, allow only paging
Allow: /*?p=
Disallow: /*?p=*&
Disallow: /*?
# allow versioned static assets for bot rendering
Allow: /*?v=*
# allow google merchant center
Allow: /*utm_source*
Allow: /*utm_medium*
Allow: /*utm_content*
# we want to avoid duplicate content for blog WSZDEV-1202
Disallow: /blog/cat/*
# we want to save crawling budget for this category WSZDEV-1253
Disallow: /angebote*
# we want to save crawling budget for noindex search results
Disallow: */catalogsearch/
Sitemap: https://werkzeugstore24.de/sitemap-index.xml

# allow google AdsBot (https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers)
User-agent: AdsBot-Google
Disallow: