added robots.txt

3 files changed, 11 insertions, 0 deletions

diff --git a/manage.py b/manage.py
index c65e5392..c65e5392 100755..100644
--- a/manage.py
+++ b/manage.py
diff --git a/runserver.sh b/runserver.sh
index 5a84a06c..5a84a06c 100755..100644
--- a/runserver.sh
+++ b/runserver.sh
diff --git a/thatcomputerscientist/urls.py b/thatcomputerscientist/urls.py
index 32d9a06c..cf572262 100644
--- a/thatcomputerscientist/urls.py
+++ b/thatcomputerscientist/urls.py
@@ -17,6 +17,7 @@ from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.contrib.sitemaps.views import sitemap
+from django.http import HttpResponse
 from django.urls import include, path
 
 from .sitemaps import (CategorySitemap, GithubSitemap, PostSitemap,
@@ -32,6 +33,15 @@ sitemaps = {
 
 handler404 = 'thatcomputerscientist.error_handler.custom_404'
 
+def robots_txt(request):
+    content = """
+User-agent: *
+Crawl-delay: 5
+Disallow: /repositories/*/
+Allow: /repositories/$
+"""
+    return HttpResponse(content, content_type="text/plain")
+
 urlpatterns = [
     path('', include('blog.urls', namespace='blog')),
     path('users', include('users.urls', namespace='users')),
@@ -40,6 +50,7 @@ urlpatterns = [
     path('ignis', include(('ignis.urls', 'ignis'), namespace='ignis')),
     path('admin/', admin.site.urls),
     path('sitemap.xml', sitemap, {'sitemaps': sitemaps}, name='django.contrib.sitemaps.views.sitemap'),
+    path('robots.txt', robots_txt),
 ]
 
 urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)