2013-06-25 20:56:31 +02:00
|
|
|
# block all spiders by default
|
|
|
|
|
User-agent: *
|
2012-10-24 18:53:21 +02:00
|
|
|
Disallow: /
|
2013-06-25 20:56:31 +02:00
|
|
|
|
|
|
|
|
# but allow major ones
|
|
|
|
|
User-agent: Googlebot
|
|
|
|
|
Allow: /
|
|
|
|
|
|
|
|
|
|
User-agent: Slurp
|
|
|
|
|
Allow: /
|
|
|
|
|
|
|
|
|
|
User-Agent: msnbot
|
|
|
|
|
Disallow:
|
|
|
|
|
|
|
|
|
|
User-agent: Baiduspider
|
2012-10-24 18:53:21 +02:00
|
|
|
Disallow: /
|
