let spiders crawl all over us
This commit is contained in:
parent
60ec1d6250
commit
29f0b6a944
|
@ -8,7 +8,6 @@
|
|||
<link rel="apple-touch-icon" sizes="72x72" href="/apple-touch-icon-144.png" />
|
||||
<link rel="apple-touch-icon" sizes="144x144" href="/apple-touch-icon-144.png" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta name="robots" content="noarchive,noodp,noydir" />
|
||||
<meta name="referrer" content="always" />
|
||||
<meta name="theme-color" content="#AC130D" />
|
||||
<% if @meta_tags %>
|
||||
|
|
|
@ -1,19 +1,2 @@
|
|||
# block all spiders by default
|
||||
User-agent: *
|
||||
Disallow: /
|
||||
|
||||
# but allow major ones
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
User-agent: Slurp
|
||||
Allow: /
|
||||
|
||||
User-Agent: msnbot
|
||||
Disallow:
|
||||
|
||||
User-agent: Baiduspider
|
||||
Disallow: /
|
||||
|
||||
User-agent: *
|
||||
Disallow: /search
|
||||
|
|
Loading…
Reference in a new issue