# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: webcrawler
Disallow: /

User-agent: lycra
Disallow: /

User-agent: Openfind Robot
Disallow: /

User-agent: *
Disallow: /

User-agent: Openfind+Robot/1.1A2
Disallow: /

User-agent: Jabot
Disallow: /

User-agent: Steeler
Disallow: /

User-agent: NaverRobot
Disallow: /

User-agent: dloader(NaverRobot)
Disallow: /

User-agent: dloader
Disallow: /

User-agent: baiduspider
Disallow: /