Sitemap: http://www.theukhighstreet.com/google_index.xml

User-agent: *
Crawl-delay: 2
Disallow: /perl/newjump.cgi
Disallow: /perl/login/community.cgi
Disallow: /perl/jump.cgi
Disallow: /perl/pjump.cgi
Disallow: /perl/like.cgi
#Disallow: /*_J\d+$
#Disallow: /*_LJ\d+$
#Disallow: /*_PJ\d+$
Disallow: /perl/bad_link.cgi
Disallow: /perl/adverts.pl
Disallow: /error/
Disallow: /cgi-bin/
Disallow: /pics/
Disallow: /images/
Disallow: /perl/sitestats.cgi
Disallow: /count.js
Disallow: /perl/stats/
Disallow: /stats/
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: ShopWiki
Disallow: /

User-agent: spbot
Disallow: /

User-agent: MJ12bot
Disallow: / 

User-agent: Sosospider 
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: pompos
Disallow: /
#User-agent: ia_archiver
#Disallow: /

User-agent: niXXiebot
Disallow: /

User-agent: Googlebot
#Disallow: /rss/
#Disallow: /perl/page.cgi?g=
Disallow: /perl/newjump.cgi
Disallow: /perl/login/community.cgi
Disallow: /perl/pjump.cgi
Disallow: /perl/jump.cgi
Disallow: like.cgi
#Disallow: /*_LJ\d+$
#Disallow: /*_J\d+$
#Disallow: /*_PJ\d+$
Disallow: /perl/bad_link.cgi
Disallow: /perl/adverts.pl
Disallow: /error/
Disallow: /cgi-bin/
Disallow: /pics/
Disallow: /images/
Disallow: /perl/sitestats.cgi
Disallow: /count.js
Disallow: /perl/stats/
Disallow: /stats/
Disallow: /*.gif$
Disallow: /*.jpg$