From a4ec6447bf8a55722773225d24d365b11b17ee26 Mon Sep 17 00:00:00 2001 From: Bobby Date: Sun, 15 Jun 2025 05:39:38 +0530 Subject: updated robots.txt --- thatcomputerscientist/urls.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/thatcomputerscientist/urls.py b/thatcomputerscientist/urls.py index cf572262..d3c80341 100644 --- a/thatcomputerscientist/urls.py +++ b/thatcomputerscientist/urls.py @@ -34,8 +34,7 @@ sitemaps = { handler404 = 'thatcomputerscientist.error_handler.custom_404' def robots_txt(request): - content = """ -User-agent: * + content = """User-agent: * Crawl-delay: 5 Disallow: /repositories/*/ Allow: /repositories/$ -- cgit v1.2.3