diff options
| -rw-r--r-- | Gruntfile.js | 2 | ||||
| -rw-r--r-- | docs/robots.txt | 4 |
2 files changed, 4 insertions, 2 deletions
diff --git a/Gruntfile.js b/Gruntfile.js index 39ba49ada..43f923fb7 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -291,7 +291,7 @@ module.exports = function (grunt) { docs: {}, netlify: { options: { - raw: 'github: true\nbaseurl: ""' + raw: 'github: true\nbaseurl: ""\nnetlify: true' } }, github: { diff --git a/docs/robots.txt b/docs/robots.txt index aef3fa1f6..c68014eda 100644 --- a/docs/robots.txt +++ b/docs/robots.txt @@ -3,7 +3,9 @@ # www.robotstxt.org/ +{% if site.netlify != true and site.github == true -%} # Allow crawling of all content +{%- endif %} User-agent: * -Disallow: +Disallow:{% if site.netlify == true or site.github != true %} /{% endif %} Sitemap: {{ site.url }}/sitemap.xml |