From b1e9e359a34e3303fb89d48baba3787ae10beba5 Mon Sep 17 00:00:00 2001
From: joshua stein <jcs@jcs.org>
Date: Tue, 25 Jun 2013 13:56:31 -0500
Subject: [PATCH] block all spiders by default, allow major ones

---
 public/robots.txt | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/public/robots.txt b/public/robots.txt
index 3d1e938..eeda15b 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,5 +1,16 @@
-# no caching
-User-agent: ia_archiver
+# block all spiders by default
+User-agent: *
 Disallow: /
-User-agent: archive.org_bot
+
+# but allow major ones
+User-agent: Googlebot
+Allow: /
+
+User-agent: Slurp
+Allow: /
+
+User-Agent: msnbot
+Disallow: 
+
+User-agent: Baiduspider
 Disallow: /