###############################
#
#
User-agent: *
Sitemap: http://www.thebusinessintelligenceguide.com/sitemap.xml
#
#
User-agent: *
#
# list folders robots are not allowed to index 
#

Disallow: /cgi-bin/
Disallow: /library/
Disallow: /blog/wp-admin/
Disallow: /blog/wp-includes/
Disallow: /books

#
# list specific files robots are not allowed to index
#
Disallow: /400.shtml
Disallow: /401.shtml
Disallow: /403.shtml
Disallow: /404.shtml
Disallow: /500.shtml
Disallow: /500.php
Disallow: /default.html
Disallow: /.htaccess
Disallow: /php.ini
Disallow: /error_log
Disallow: /section.php
# 
#
###############################