# robots.txt voor kleren.com ############################## # 1. Algemene blokkades voor WordPress/WooCommerce ############################## User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /?s= # Kies hier: óf blokkeren van paginatie, óf in meta noindex toepassen Disallow: /page/*/ # Blokkeer affiliate-redirects Disallow: /go/ # Filter-/sorter-parameters (vermijd oneindige URL-varianten) Disallow: /*?orderby= Disallow: /*?filter_ Disallow: /*?min_price= Disallow: /*?max_price= Disallow: /*?ep_filter* Disallow: /*&ep_filter* Disallow: /*?*ep_filter Disallow: /*&ep_filter Disallow: /*?*min_price= Disallow: /*?*max_price= Disallow: /*?*filter_ Disallow: /*?*orderby= Disallow: /*&orderby= Disallow: /*?s= Disallow: /*&s= # (optioneel) Crawl-delay voor generieke crawlers # Crawl-delay: 1 ############################## # 2. Specifieke belangrijke crawlers ############################## User-agent: Googlebot Disallow: /go/ User-agent: Bingbot Disallow: /go/ User-agent: SemrushBot Disallow: /go/ Crawl-delay: 5 User-agent: AhrefsBot Disallow: /go/ Crawl-delay: 5 User-agent: Applebot Disallow: /go/ Crawl-delay: 10 ############################## # 3. Bots die je wilt blokkeren ############################## User-agent: BLEXBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: SEOkicks Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: Yandex Disallow: / # … voeg extra bots toe die je nÃet wilt toelaten … ############################## # 4. Sitemap-vermelding ############################## Sitemap: https://kleren.com/sitemap-statics-pending/sitemap_index_static.xml