diff options
| -rw-r--r-- | thatcomputerscientist/urls.py | 3 |
1 files changed, 1 insertions, 2 deletions
diff --git a/thatcomputerscientist/urls.py b/thatcomputerscientist/urls.py index cf572262..d3c80341 100644 --- a/thatcomputerscientist/urls.py +++ b/thatcomputerscientist/urls.py @@ -34,8 +34,7 @@ sitemaps = { handler404 = 'thatcomputerscientist.error_handler.custom_404' def robots_txt(request): - content = """ -User-agent: * + content = """User-agent: * Crawl-delay: 5 Disallow: /repositories/*/ Allow: /repositories/$ |