# allow crawlers plus delaying each successive spider request User-agent: * #Crawl-delay: 10 Disallow: /*jsessionid* Disallow: /myprofile/ Disallow: /checkout/ #Disallow: /assets/ # Disallow all ../?parentCategoryID=cat... #Disallow: *parentCategoryId* #Specific URLs Disallow: /?oid=* Disallow: /dummy/ Disallow: /dummy2/ Disallow: /cat/dummy/ Disallow: /cat/dummy2/ Disallow: /cat/proionta/ Disallow: /product/dummy/ Disallow: /product/dummy2/ Disallow: /companyinfo/public/storeLocations.jsp Disallow: /companyinfo/storeLocations.jsp Disallow: /searchResults.jsp* Disallow: /templates/largeImagePopup.jsp* # Blocking bad link checker robots User-agent: AhrefsBot Disallow: / User-agent: Alexibot Disallow: / User-agent: MJ12bot Disallow: / User-agent: SurveyBot Disallow: / User-agent: Xenu’s Disallow: / User-agent: Xenu’s Link Sleuth 1.1c Disallow: / User-agent: rogerbot Disallow:/