From 29f0b6a944f09599cdb86bf12cbe81414a586af0 Mon Sep 17 00:00:00 2001
From: Carl Chenet <chaica@ohmytux.com>
Date: Tue, 16 May 2017 19:50:36 +0200
Subject: [PATCH] let spiders crawl all over us

---
 app/views/layouts/application.html.erb |  1 -
 public/robots.txt                      | 17 -----------------
 2 files changed, 18 deletions(-)

diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index cece3ac..1e8c220 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -8,7 +8,6 @@
   <link rel="apple-touch-icon" sizes="72x72" href="/apple-touch-icon-144.png" />
   <link rel="apple-touch-icon" sizes="144x144" href="/apple-touch-icon-144.png" />
   <meta name="viewport" content="width=device-width, initial-scale=1" />
-  <meta name="robots" content="noarchive,noodp,noydir" />
   <meta name="referrer" content="always" />
   <meta name="theme-color" content="#AC130D" />
   <% if @meta_tags %>
diff --git a/public/robots.txt b/public/robots.txt
index aa94cad..87a97c0 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,19 +1,2 @@
-# block all spiders by default
-User-agent: *
-Disallow: /
-
-# but allow major ones
-User-agent: Googlebot
-Allow: /
-
-User-agent: Slurp
-Allow: /
-
-User-Agent: msnbot
-Disallow: 
-
-User-agent: Baiduspider
-Disallow: /
-
 User-agent: *
 Disallow: /search