[extensions-web] Added robots.txt

[Yuri Konotopov <ykonotopov src gnome org>]

commit 018d034c12fb415d80d575330954b7ad0e9b5b39
Author: Yuri Konotopov <ykonotopov gnome org>
Date:   Fri Dec 2 18:58:18 2016 +0300

    Added robots.txt

 sweettooth/templates/robots.txt |   12 ++++++++++++
 sweettooth/urls.py              |    4 ++--
 2 files changed, 14 insertions(+), 2 deletions(-)
---
diff --git a/sweettooth/templates/robots.txt b/sweettooth/templates/robots.txt
new file mode 100644
index 0000000..8e4482c
--- /dev/null
+++ b/sweettooth/templates/robots.txt
@@ -0,0 +1,12 @@
+User-agent: *
+Crawl-delay: 0.5
+Disallow: /errors/
+Disallow: /admin/
+Disallow: /edit/
+Disallow: /detail/
+Disallow: /set-status/
+Disallow: /adjust-popularity/
+Disallow: /extension-query/
+Disallow: /extension-info/
+Disallow: /download-extension/
+Disallow: /update-info/
diff --git a/sweettooth/urls.py b/sweettooth/urls.py
index 7e159fb..70aba0b 100644
--- a/sweettooth/urls.py
+++ b/sweettooth/urls.py
@@ -7,7 +7,7 @@ from django.http import HttpResponse
 
 from django.contrib import admin
 from django.views import static
-from django.contrib.staticfiles import urls as static_urls
+from django.views.generic.base import TemplateView
 admin.autodiscover()
 
 urlpatterns = patterns('',
@@ -21,7 +21,7 @@ urlpatterns = patterns('',
     url(r'^admin/', include(admin.site.urls)),
     url(r'^comments/', include('sweettooth.ratings.urls')),
     url(r'^comments/', include('django_comments.urls')),
-
+    url(r'^robots\.txt$', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')),
 )
 
 if settings.DEBUG: