# /robots.txt file for http://webcrawler.com/ # For domain: http://www.ecommerce-web-store.com # All robots will spider the domain User-agent: * # Disallow directories and files Disallow: /_lib/ Disallow: /cp/ Disallow: /admin/ Disallow: /images/ Disallow: /storefront/ #Disallow: /under_construction.php