# Crawlers Setup

# LSD - Update 27-03-2018

crawl-delay: 10

User-agent: *

# Website Sitemap

Sitemap: http://lsd.co.uk/sitemap/sitemap.xml


# Allowable Index

Allow: /*?p=

Allow: /


# Directories

Disallow: /404/

Disallow: /app/

Disallow: /cgi-bin/

Disallow: /downloader/

Disallow: /includes/

#Disallow: /js/

Disallow: /lib/

Disallow: /magento/

Disallow: /pkginfo/

Disallow: /report/

#Disallow: /skin/

Disallow: /stats/

Disallow: /var/

Disallow: /new/

Disallow: /pdffree/

Disallow: /errors/


# Paths (clean URLs)

Disallow: /index.php/

Disallow: /catalog/product/upload/option_id/

Disallow: /catalog/product_compare/

Disallow: /catalog/product/gallary/

Disallow: /catalog/category/view/

Disallow: /catalog/product/view/

Disallow: /catalogsearch/

Disallow: /checkout/

Disallow: /control/

Disallow: /contacts/

Disallow: /customer/

Disallow: /customize/

Disallow: /newsletter/

Disallow: /poll/

Disallow: /review/

Disallow: /sendfriend/

Disallow: /tag/

Disallow: /uk/cat/

Disallow: /view_product.php

Disallow: /uk/tag/

Disallow: /wishlist/
#Disallow: /quickview/



# Files

Disallow: /cron.php

Disallow: /cron.sh

Disallow: /error_log

Disallow: /install.php

Disallow: /LICENSE.html

Disallow: /LICENSE.txt

Disallow: /LICENSE_AFL.txt

Disallow: /STATUS.txt



# Paths (no clean URLs)

#Disallow: /*.js$

#Disallow: /*.css$

Disallow: /*.php$

Disallow: /*?p=*&

Disallow: /*?SID=



User-agent: *

Disallow: /errors/
Disallow: /category/
Disallow: /product/
Disallow: /LSD/
Disallow: /catalogsearch/advanced/
Disallow: /index.php*
Disallow: /?___SID=U
Disallow: /delivery-details/?___SID=U
Disallow: /indian-stone-paving?dir=asc&limit=90&mode=grid&natural_manmade=423&order=name&paving_finish=24&paving_material=16&paving_thickness_range=417


User-agent: Googlebot-image
Allow: /media/
Allow: /js/
Allow: /skin/