diff options
| author | Bobby <[email protected]> | 2025-06-15 05:39:38 +0530 |
|---|---|---|
| committer | Bobby <[email protected]> | 2025-06-15 05:39:38 +0530 |
| commit | a4ec6447bf8a55722773225d24d365b11b17ee26 (patch) | |
| tree | f8ab27480c5419fe39cd9a3d5187e8bd77e2e82b | |
| parent | 370ef9590ed5b974723193045903ae88d17cf45c (diff) | |
| download | thatcomputerscientist-a4ec6447bf8a55722773225d24d365b11b17ee26.tar.xz thatcomputerscientist-a4ec6447bf8a55722773225d24d365b11b17ee26.zip | |
| -rw-r--r-- | thatcomputerscientist/urls.py | 3 |
1 files changed, 1 insertions, 2 deletions
diff --git a/thatcomputerscientist/urls.py b/thatcomputerscientist/urls.py index cf572262..d3c80341 100644 --- a/thatcomputerscientist/urls.py +++ b/thatcomputerscientist/urls.py @@ -34,8 +34,7 @@ sitemaps = { handler404 = 'thatcomputerscientist.error_handler.custom_404' def robots_txt(request): - content = """ -User-agent: * + content = """User-agent: * Crawl-delay: 5 Disallow: /repositories/*/ Allow: /repositories/$ |