#robots.txt file 10/07/03

#Bot Trap, Disallow All Bots
User-agent: *
Disallow: /spyder-pit/

#Disallow asterias 03/06/04.  Streaming media bot, hit me 736 times in one shot
User-agent: asterias
Disallow: /

#Disallow ia_archiver completely, bad bot
User-agent: ia_archiver
Disallow: /

#Disallow scooter, 
#Disallow TEMP to conserve bandwidth, 10/07/03
User-agent: Scooter
Disallow: /

#Disallow Inktomi_Slurp, uses to much bandwidth (allow 9/21, 0845) disallow, 10/07/03
User-agent: Slurp
Disallow: /

#Disallow turnitinbot
User-agent: turnitinbot
Disallow: /

#Disallow WISENutbot (Looksmart), does not respect robots file
User-agent: WISENutbot 
Disallow: /

#Disallow grub-client, does not respect robots file
User-agent: grub-client 
Disallow: /

#Disallow NPBot
User-agent: NPBot
Disallow: /

#Disallow alamaden, well behaved, respects robtos file/Violated txt 
User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

#Disallow Jeeves, uses to much band width, allow 10/07/03
#User-agent: Jeeves
#Disallow: /

#Disallow AskJeeves
User-agent: AskJeeves
Disallow: /

#Disallow Zyborg, does not respect robots file (IP Ban)
User-agent: ZyBorg/1.0 
Disallow:/

#Disallow Zyborg, does not respect robots file (IP ban)
User-agent: ZyBorg
Disallow:/

#Disallow gigabot
User-agent: gigabot
Disallow:/

#szukacz robot, well behaved
#User-agent: szukacz
#Disallow: /

#Road runner,  Seems to be well mannered
#User-agent: Road Runner: ImageScape Robot (lim@cs.leidenuniv.nl)
#Disallow:/

#Disallow Fast Web Crawler, follows robots.txt, disallow to conserve bandwidth, 10/07/03
User-agent: fast
Disallow: /

*Photo/Image index bot, seems to respect txt
*User-agent: psbot
*Disallow: /

#Disallow Bai Du Spider.  China
User-agent: baiduspider 
Disallow: / 


#Disallow ALL access to Files
User-agent: *
#Disallow: /Mofak
#Disallow: /Phyl&Larry'sPlace
Disallow: /VetsDay

User-Agent: The Knowledge AI
Disallow: /