# www.robotstxt.org/ # www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 User-agent: * # Allow crawling of main content Allow: / # Disallow AJAX endpoints from being indexed Disallow: /product/loadPrice/ Disallow: /product/loadMoreReviews/ Disallow: /product/getLocationsAndStorage/ Disallow: /product/productReviews/ Disallow: /product/ad/ Disallow: /categories/categoryitems/ Disallow: /categories/productfilters/ Disallow: /categories/categoryinfo Disallow: /categories/productfiltersmobile Disallow: /categories/menumobile Disallow: /footer Disallow: /settings/ Disallow: /api/ Disallow: /cms/ # Disallow filtered/sorted URLs to prevent duplicate content Disallow: /*?*orderby= Disallow: /*?*orderdir= Disallow: /*?*minmaxprice Disallow: /*?*field_ # Disallow cart and checkout pages Disallow: /orders/cart Disallow: /orders/checkout Disallow: /orders/cart/ Disallow: /orders/checkout/ # Disallow admin and user account pages Disallow: /myaccount Disallow: /myaccount/ Disallow: /login/ # Allow important pages Allow: /product/ Allow: /categories/ # Sitemap location Sitemap: /sitemap