diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index cece3ac..1e8c220 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -8,7 +8,6 @@ - <% if @meta_tags %> diff --git a/public/robots.txt b/public/robots.txt index aa94cad..87a97c0 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,19 +1,2 @@ -# block all spiders by default -User-agent: * -Disallow: / - -# but allow major ones -User-agent: Googlebot -Allow: / - -User-agent: Slurp -Allow: / - -User-Agent: msnbot -Disallow: - -User-agent: Baiduspider -Disallow: / - User-agent: * Disallow: /search