User-agent: ia_archiver
Disallow: /~www/
Disallow: /~vlad/
Disallow: /~web011/
Disallow: /~web012/
Disallow: /~web103/
Disallow: /~web105/
Disallow: /~web106/
Disallow: /~web1105/
Disallow: /~web1106/
Disallow: /~web111/
Disallow: /~web1110/
Disallow: /~web112/
Disallow: /~web1120/
Disallow: /~web121/
Disallow: /~web122/
Disallow: /~web1220/
Disallow: /~web1300/
Disallow: /~web135/
Disallow: /~web1350/
Disallow: /~web160/
Disallow: /~web1600/
Disallow: /~web171/
Disallow: /~web1710/
Disallow: /~web189/
Disallow: /~web191/
Disallow: /~web1910/
Disallow: /~web192/
Disallow: /~web1920/
Disallow: /~web213/
Disallow: /~web2130/
Disallow: /~web221/
Disallow: /~web2210/
Disallow: /~web222/
Disallow: /~web2220/
Disallow: /~web223/
Disallow: /~web224/
Disallow: /~web231/
Disallow: /~web2310/
Disallow: /~web2710/
Disallow: /~web293/
Disallow: /~web2930/
Disallow: /~web294/
Disallow: /~web2940/
Disallow: /~web304/
Disallow: /~web3040/
Disallow: /~web311/
Disallow: /~web332/
Disallow: /~web335/
Disallow: /~web336/
Disallow: /~web3360/
Disallow: /~web356/
Disallow: /~web401/
Disallow: /~web403/
Disallow: /~web413/
Disallow: /~web4130/
Disallow: /~web414/
Disallow: /~web418/
Disallow: /~web4180/
Disallow: /~web420/
Disallow: /~web422/
Disallow: /~web424/
Disallow: /~web4240/
Disallow: /~web425/
Disallow: /~web428/
Disallow: /~web431/
Disallow: /~web4310/
Disallow: /~web432/
Disallow: /~web433/
Disallow: /~web434/
Disallow: /~web441/
Disallow: /~web4410/
Disallow: /~web452/
Disallow: /~web455/
Disallow: /~web471/
Disallow: /~web4710/
Disallow: /~web472/
Disallow: /~web481/
Disallow: /~web4h/
Disallow: /~web561/
Disallow: /~web611/
Disallow: /~web6110/
Disallow: /~web612/
Disallow: /~web613/
Disallow: /~web6130/
Disallow: /~web617/
Disallow: /~web6170/
Disallow: /~web6180/
Disallow: /~web6210/
Disallow: /~web622/
Disallow: /~web634/
Disallow: /~web6490/
Disallow: /~web651/
Disallow: /~web671/
Disallow: /~web6710/
Disallow: /~web672/
Disallow: /~web712/
Disallow: /~web7170/
Disallow: /~web761/
Disallow: /~web762/
Disallow: /~web7620/
Disallow: /~web767/

User-agent: *
Disallow: /~www/
Crawl-delay: 120
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/logon
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile
Disallow: /twiki/bin/view/Sandbox
Disallow: /twiki/bin/view/*?

Disallow: /twiki/bin/survey
Disallow: /twiki/bin/twikicvs
Disallow: /twiki/bin/awstats/

Disallow: /twiki/bin/b/bin/


User-agent: iecrawler
User-agent: ie_crawler
User-agent: e-SocietyRobot
Disallow: /

User-agent: msnbot
Crawl-delay: 120
Disallow: /~www/
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/logon
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile
Disallow: /twiki/bin/view/Sandbox
Disallow: /twiki/bin/view/*?

Disallow: /twiki/bin/survey
Disallow: /twiki/bin/twikicvs
Disallow: /twiki/bin/awstats/

Disallow: /twiki/bin/b/bin/

User-agent: Slurp
# Custom extension: (see http://help.yahoo.com/help/us/ysearch/slurp/slurp-03.html)
Crawl-delay: 120
Disallow: /~www/
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/logon
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/upload
Disallow: /twiki/bin/viewauth
Disallow: /twiki/bin/viewfile
Disallow: /twiki/bin/view/Sandbox
Disallow: /twiki/bin/view/*?

Disallow: /twiki/bin/survey
Disallow: /twiki/bin/twikicvs
Disallow: /twiki/bin/awstats/

Disallow: /twiki/bin/b/bin/