20 lines
241 B
Plaintext
20 lines
241 B
Plaintext
# block all spiders by default
|
|
User-agent: *
|
|
Disallow: /
|
|
|
|
# but allow major ones
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
|
|
User-agent: Slurp
|
|
Allow: /
|
|
|
|
User-Agent: msnbot
|
|
Disallow:
|
|
|
|
User-agent: Baiduspider
|
|
Disallow: /
|
|
|
|
User-agent: *
|
|
Disallow: /search
|