From 370ef9590ed5b974723193045903ae88d17cf45c Mon Sep 17 00:00:00 2001
From: Bobby <bobbyskhs@gmail.com>
Date: Sun, 15 Jun 2025 05:38:29 +0530
Subject: added robots.txt

---
 manage.py                     |  0
 runserver.sh                  |  0
 thatcomputerscientist/urls.py | 11 +++++++++++
 3 files changed, 11 insertions(+)
 mode change 100755 => 100644 manage.py
 mode change 100755 => 100644 runserver.sh

diff --git a/manage.py b/manage.py
old mode 100755
new mode 100644
diff --git a/runserver.sh b/runserver.sh
old mode 100755
new mode 100644
diff --git a/thatcomputerscientist/urls.py b/thatcomputerscientist/urls.py
index 32d9a06c..cf572262 100644
--- a/thatcomputerscientist/urls.py
+++ b/thatcomputerscientist/urls.py
@@ -17,6 +17,7 @@ from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.contrib.sitemaps.views import sitemap
+from django.http import HttpResponse
 from django.urls import include, path
 
 from .sitemaps import (CategorySitemap, GithubSitemap, PostSitemap,
@@ -32,6 +33,15 @@ sitemaps = {
 
 handler404 = 'thatcomputerscientist.error_handler.custom_404'
 
+def robots_txt(request):
+    content = """
+User-agent: *
+Crawl-delay: 5
+Disallow: /repositories/*/
+Allow: /repositories/$
+"""
+    return HttpResponse(content, content_type="text/plain")
+
 urlpatterns = [
     path('', include('blog.urls', namespace='blog')),
     path('users', include('users.urls', namespace='users')),
@@ -40,6 +50,7 @@ urlpatterns = [
     path('ignis', include(('ignis.urls', 'ignis'), namespace='ignis')),
     path('admin/', admin.site.urls),
     path('sitemap.xml', sitemap, {'sitemaps': sitemaps}, name='django.contrib.sitemaps.views.sitemap'),
+    path('robots.txt', robots_txt),
 ]
 
 urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
-- 
cgit v1.2.3