[extensions-web/wip/api/v1: 8/12] api: initial implementation of /extensions node




commit eae50defcbf0d12bbab06cb4dc0120cce2fd81aa
Author: Yuri Konotopov <ykonotopov gnome org>
Date:   Sun Nov 22 12:25:15 2020 +0400

    api: initial implementation of /extensions node
    
    Xapian usage is dropped with this commit in favour of Elasticsearch.
    Actually there is no proper support of Xapian for DRF or Django exists,
    but only generic python bindings. Support for Xapian in Haystack is very
    poor. Support of Elasticsearch in Haystack is almost dead also.
    On the other hand django-elasticsearch-dsl is developed actively for now.

 openshift/docker/docker-compose.yml                |  12 ++
 requirements.txt                                   |  23 ++
 sweettooth/api/v1/urls.py                          |  12 +-
 sweettooth/extensions/documents.py                 |  50 +++++
 .../management/commands/indexextensions.py         |  19 --
 .../management/commands/searchextensions.py        |  38 ----
 sweettooth/extensions/search.py                    |  87 --------
 sweettooth/extensions/urls.py                      |   6 +-
 sweettooth/extensions/views.py                     | 236 ++++++++++++++-------
 sweettooth/settings.py                             |  12 +-
 10 files changed, 270 insertions(+), 225 deletions(-)
---
diff --git a/openshift/docker/docker-compose.yml b/openshift/docker/docker-compose.yml
index ac2e73ef..62ee726a 100644
--- a/openshift/docker/docker-compose.yml
+++ b/openshift/docker/docker-compose.yml
@@ -38,8 +38,10 @@ services:
       EGO_STATIC_ROOT: /extensions-web/www/static-files
     depends_on:
       - db
+      - search
     links:
       - db
+      - search
     volumes:
       - "static:/extensions-web/www"
   frontend:
@@ -56,7 +58,17 @@ services:
       - "8080:8080"
     volumes:
       - "static:/extensions-web/www"
+  search:
+    image: elasticsearch:7.9.1
+    restart: always
+    environment:
+      "discovery.type": single-node
+    ports:
+      - "127.0.0.1:49200:9200"
+    volumes:
+      - search:/usr/share/elasticsearch/data
 
 volumes:
   database:
+  search:
   static:
diff --git a/requirements.txt b/requirements.txt
index 98b6a490..f3ee3b12 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,12 +7,18 @@ djangorestframework==3.11.0 \
 django-autoslug==1.9.6 \
     --hash=sha256:3c55c18e2a5aab67c3a8d488056422b93c1984a197387c97952e92836f98c73c \
     --hash=sha256:f9d2ef0d98dd97ba4d5ea358cc74bfee3e3498e2dee29f396d604173eaf07e47
+django-filter==2.2.0 \
+    --hash=sha256:558c727bce3ffa89c4a7a0b13bc8976745d63e5fd576b3a9a851650ef11c401b \
+    --hash=sha256:c3deb57f0dd7ff94d7dce52a047516822013e2b441bed472b722a317658cfd14
 django-contrib-comments==1.9.2 \
     --hash=sha256:b83320a86081a76bc0570e6cc0f924c0ced40b46ae9f5dd783ab2c745b449529 \
     --hash=sha256:d1232bade3094de07dcc205fc833204384e71ba9d30caadcb5bb2882ce8e8d31
 django-registration==3.1 \
     --hash=sha256:071dce4f4348ff1da338a0a09bda8e34d44c001ab569777e4c88ee21ce5c499e \
     --hash=sha256:2437a098e6e06983e4b4b442680b1c49a2e979f1a0ef3a504beb3a84ff36131d
+django-elasticsearch-dsl==7.1.4 \
+    --hash=sha256:5bbd49a9acb51c08fbbb7fba9beee7c9ce73b481af718cc57e4c8ac3d561888f \
+    --hash=sha256:e733ccfd0e8b83ad6896d812a2c15ccbd563caf4ebf30fd31640538ad2de8e26
 Pygments==2.6.1 \
     --hash=sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44 \
     --hash=sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324
@@ -50,3 +56,20 @@ confusable_homoglyphs==3.2.0 \
 six==1.14.0 \
     --hash=sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a \
     --hash=sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c
+# django-elasticsearch-dsl
+elasticsearch-dsl==7.2.1 \
+    --hash=sha256:1e345535164cb684de4b825e1d0daf81b75554b30d3905446584a9e4af0cc3e7 \
+    --hash=sha256:593c01822a03e3e84b87753c78edb833f4b2bfafcd52089841bd8f99b7e74ccd
+python-dateutil==2.8.1 \
+    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
+elasticsearch==7.9.1 \
+    --hash=sha256:5e08776fbb30c6e92408c7fa8c37d939210d291475ae2f364f0497975918b6fe \
+    --hash=sha256:8c7e2374f53ee1b891ff2804116e0c7fb517585d6d5788ba668686bbc9d82e2d
+# elasticsearch
+certifi==2020.6.20 \
+    --hash=sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3 \
+    --hash=sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41
+urllib3==1.25.10 \
+    --hash=sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a \
+    --hash=sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461
diff --git a/sweettooth/api/v1/urls.py b/sweettooth/api/v1/urls.py
index aa9849a3..fc980983 100644
--- a/sweettooth/api/v1/urls.py
+++ b/sweettooth/api/v1/urls.py
@@ -13,9 +13,19 @@ from django.urls import path
 from rest_framework.routers import SimpleRouter
 
 from sweettooth.api.v1.views import HelloView
+from sweettooth.extensions.views import ExtensionsViewSet
 from sweettooth.users.views import UserProfileDetailView
 
-urlpatterns = [
+# Create a router and register our viewsets with it.
+router = SimpleRouter()
+router.register(
+    r'v1/extensions',
+    ExtensionsViewSet,
+    basename='extension',
+)
+
+urlpatterns = router.urls
+urlpatterns += [
     path('v1/hello/', HelloView.as_view()),
     path('v1/profile/<int:pk>/',
          UserProfileDetailView.as_view(),
diff --git a/sweettooth/extensions/documents.py b/sweettooth/extensions/documents.py
new file mode 100644
index 00000000..a1ee7ee7
--- /dev/null
+++ b/sweettooth/extensions/documents.py
@@ -0,0 +1,50 @@
+"""
+    GNOME Shell extensions repository
+    Copyright (C) 2020  Yuri Konotopov <ykonotopov gnome org>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+"""
+
+import logging
+
+from django_elasticsearch_dsl import Document
+from django_elasticsearch_dsl.registries import registry
+
+from .models import Extension
+
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
+
+@registry.register_document
+class ExtensionDocument(Document):
+    class Index:
+        name = 'extensions'
+
+    class Django:
+        model = Extension
+
+        # The fields of the model you want to be indexed in Elasticsearch
+        fields = [
+            'uuid',
+            'name',
+            'description',
+            'created',
+            'downloads',
+            'popularity',
+        ]
+
+    def get_queryset(self):
+        return super(ExtensionDocument, self).get_queryset().select_related(
+            'creator'
+        )
+
+    def prepare_creator(self, extension):
+        return extension.creator.username
+
+    @staticmethod
+    def document_fields():
+        return ['uuid', 'name', 'description', 'creator']
diff --git a/sweettooth/extensions/search.py b/sweettooth/extensions/search.py
index fe6d6de1..e69de29b 100644
--- a/sweettooth/extensions/search.py
+++ b/sweettooth/extensions/search.py
@@ -1,87 +0,0 @@
-
-import xapian
-
-from functools import reduce
-from django.conf import settings
-from django.db.models import signals
-
-from sweettooth.extensions.models import Extension, ExtensionVersion
-from sweettooth.extensions.models import reviewed, extension_updated
-
-def index_extension(extension):
-    if extension.latest_version is None:
-        return
-
-    db = xapian.WritableDatabase(settings.XAPIAN_DB_PATH, xapian.DB_CREATE_OR_OPEN)
-
-    termgen = xapian.TermGenerator()
-    termgen.set_stemmer(xapian.Stem("en"))
-
-    doc = xapian.Document()
-    termgen.set_document(doc)
-
-    termgen.index_text(extension.name, 10)
-    termgen.index_text(extension.uuid)
-    termgen.index_text(extension.description)
-
-    doc.set_data(str(extension.pk))
-
-    idterm = "Q%s" % (extension.pk,)
-    doc.add_boolean_term(idterm)
-    for shell_version in extension.visible_shell_version_map.keys():
-        doc.add_boolean_term("V%s" % (shell_version,))
-
-    db.replace_document(idterm, doc)
-
-def delete_extension(extension):
-    db = xapian.WritableDatabase(settings.XAPIAN_DB_PATH, xapian.DB_CREATE_OR_OPEN)
-    idterm = "Q%s" % (extension.pk,)
-    db.delete_document(idterm)
-
-
-def reviewed_handler(sender, request, version, review, **kwargs):
-    index_extension(version.extension)
-reviewed.connect(reviewed_handler)
-
-def extension_updated_handler(extension, **kwargs):
-    index_extension(extension)
-extension_updated.connect(extension_updated_handler)
-
-def post_extension_delete_handler(instance, **kwargs):
-    delete_extension(instance)
-signals.post_delete.connect(post_extension_delete_handler, sender=Extension)
-
-def post_version_save_handler(instance, **kwargs):
-    index_extension(instance.extension)
-signals.post_save.connect(post_version_save_handler, sender=ExtensionVersion)
-
-def combine_queries(op, queries):
-    def make_query(left, right):
-        return xapian.Query(op, left, right)
-    return reduce(make_query, queries)
-
-def make_version_queries(versions):
-    queries = [xapian.Query("V%s" % (v.version_string,)) for v in versions]
-    return combine_queries(xapian.Query.OP_OR, queries)
-
-def enquire(querystring, versions=None):
-    try:
-        db = xapian.Database(settings.XAPIAN_DB_PATH)
-    except xapian.DatabaseOpeningError:
-        return None
-
-    qp = xapian.QueryParser()
-    qp.set_stemming_strategy(qp.STEM_SOME)
-    qp.set_stemmer(xapian.Stem("en"))
-    qp.set_database(db)
-
-    query = qp.parse_query(querystring, qp.FLAG_PARTIAL)
-
-    if versions:
-        query = xapian.Query(xapian.Query.OP_FILTER,
-                             query,
-                             make_version_queries(versions))
-
-    enquiry = xapian.Enquire(db)
-    enquiry.set_query(query)
-    return db, enquiry
diff --git a/sweettooth/extensions/urls.py b/sweettooth/extensions/urls.py
index e90b33a3..02614a38 100644
--- a/sweettooth/extensions/urls.py
+++ b/sweettooth/extensions/urls.py
@@ -1,6 +1,6 @@
 
 from django.conf.urls import include, url
-from django.views.generic.base import TemplateView
+from django.views.generic.base import RedirectView, TemplateView
 
 from sweettooth.extensions import views, models, feeds
 
@@ -18,8 +18,6 @@ ajax_patterns = [
 ]
 
 shell_patterns = [
-    url(r'^extension-query/', views.ajax_query_view, name='extensions-query'),
-
     url(r'^extension-info/', views.ajax_details_view),
 
     url(r'^download-extension/(?P<uuid>.+)\.shell-extension\.zip$',
@@ -29,7 +27,7 @@ shell_patterns = [
 ]
 
 urlpatterns = [
-    url(r'^$', TemplateView.as_view(template_name='extensions/list.html'), name='extensions-index'),
+    url(r'^$', RedirectView.as_view(url='/', permanent=False), name='extensions-index'),
 
     url(r'^about/$', TemplateView.as_view(template_name='extensions/about.html'), name='extensions-about'),
 
diff --git a/sweettooth/extensions/views.py b/sweettooth/extensions/views.py
index dd2c58a4..38e43416 100644
--- a/sweettooth/extensions/views.py
+++ b/sweettooth/extensions/views.py
@@ -10,25 +10,175 @@
 """
 
 import json
-from math import ceil
+from functools import reduce
 
 from django.core.exceptions import ValidationError
 from django.core.paginator import Paginator, InvalidPage
 from django.contrib.auth.decorators import login_required
 from django.contrib import messages
 from django.db import transaction
-from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, 
HttpResponseServerError, Http404
+from django.http import (
+    JsonResponse,
+    HttpResponse,
+    HttpResponseBadRequest,
+    HttpResponseForbidden,
+    HttpResponseServerError,
+    Http404
+)
 from django.shortcuts import get_object_or_404, redirect, render
 from django.template.loader import render_to_string
 from django.views.decorators.http import require_POST
 from django.urls import reverse
 
-from sweettooth.exceptions import DatabaseErrorWithMessages
-from sweettooth.extensions import models, search
-from sweettooth.extensions.forms import ImageUploadForm, UploadForm
+from django_filters.rest_framework import DjangoFilterBackend
+
+from rest_framework import filters, mixins, viewsets, status
+from rest_framework.decorators import action
+from rest_framework.pagination import PageNumberPagination
+from rest_framework.parsers import JSONParser
+from rest_framework.response import Response
+
 
 from sweettooth.decorators import ajax_view, model_view
+from sweettooth.exceptions import DatabaseErrorWithMessages
+from sweettooth.extensions import models, serializers
+from sweettooth.extensions.documents import ExtensionDocument
+from sweettooth.extensions.forms import ImageUploadForm, UploadForm
 from sweettooth.extensions.templatetags.extension_icon import extension_icon
+from sweettooth import settings
+
+
+class ExtensionsPagination(PageNumberPagination):
+    page_size = settings.REST_FRAMEWORK['PAGE_SIZE']
+    page_size_query_param = 'page_size'
+    max_page_size = 100
+
+
+class ExtensionsViewSet(mixins.ListModelMixin,
+                        mixins.RetrieveModelMixin,
+                        viewsets.GenericViewSet):
+    lookup_field = 'uuid'
+    lookup_value_regex = '[-a-zA-Z0-9@._]+'
+    serializer_class = serializers.ExtensionSerializer
+    pagination_class = ExtensionsPagination
+    filter_backends = [DjangoFilterBackend, filters.OrderingFilter]
+    ordering_fields = ['created', 'downloads', 'popularity', '?']
+    page_size = 25
+    page_size_query_param = 'page_size'
+    max_page_size = 100
+
+    def get_queryset(self):
+        uuids = self.request.query_params.getlist('uuid[]')
+        status = self.request.query_params.get('status')
+        queryset = models.Extension.objects.all()
+
+        if uuids:
+            queryset = queryset.filter(uuid__in=uuids)
+
+        if status is not None and status.isnumeric():
+            status = int(status)
+
+        if status in models.STATUSES:
+            queryset = queryset.filter(versions__status=status).distinct()
+
+        return queryset
+
+    @action(methods=['post'], detail=False, parser_classes=[JSONParser])
+    def updates(self, request):
+        installed = self.request.data.get('installed')
+        shell_version = self.request.data.get('shell_version')
+        version_validation_enabled = self.request.data.get('version_validation_enabled', False)
+
+        if not installed:
+            return Response()
+
+        extensions = models.Extension.objects.filter(uuid__in=installed.keys())
+
+        result = {}
+        for uuid, version in installed.items():
+            try:
+                version = int(version)
+            except (KeyError, TypeError):
+                continue
+            except ValueError:
+                version = 1
+
+            extension = reduce(
+                lambda x, y, uuid=uuid: (
+                    x if x and x.uuid == uuid else
+                    y if y and y.uuid == uuid else
+                    None
+                ),
+                list(extensions)
+            )
+
+            if not extension:
+                continue
+
+            try:
+                version = extension.versions.get(version=version)
+            except models.ExtensionVersion.DoesNotExist:
+                # Skip unknown versions
+                continue
+
+            proper_version = grab_proper_extension_version(
+                extension,
+                shell_version,
+                version_validation_enabled
+            )
+
+            if proper_version is not None and proper_version.version != version.version:
+                result[uuid] = {
+                    'action': 'change',
+                    'version': proper_version.version,
+                }
+
+        return Response(result)
+
+    @action(methods=['get'], detail=False, url_path='search/(?P<query>[^/.]+)')
+    def search(self, request, query=None):
+        try:
+            page = int(self.request.query_params.get(self.pagination_class.page_query_param, 1))
+            page_size = max(
+                1,
+                min(
+                    self.max_page_size,
+                    int(self.request.query_params.get(self.page_size_query_param, self.page_size))
+                )
+            )
+        except Exception as ex:
+            print(ex)
+            return Response(status=status.HTTP_400_BAD_REQUEST)
+
+        if not query:
+            return Response(status=status.HTTP_400_BAD_REQUEST)
+
+        queryset = ExtensionDocument.search().query(
+            "multi_match",
+            query=query,
+            fields=ExtensionDocument.document_fields()
+        )
+
+        ordering = self.request.query_params.get('ordering')
+        ordering_field = (
+            ordering
+            if not ordering or ordering[0] != '-'
+            else ordering[1:]
+        )
+        if ordering and ordering_field in self.ordering_fields:
+            queryset = queryset.sort(ordering)
+
+        paginator = Paginator(queryset.to_queryset(), page_size)
+
+        return Response({
+            'count': paginator.count,
+            'results': self.serializer_class(
+                [sr for sr in paginator.page(page).object_list],
+                many=True,
+                context={'request': request}
+            ).data
+        })
+
 
 def get_versions_for_version_strings(version_strings):
     def get_version(major, minor, point):
@@ -55,7 +205,7 @@ def get_versions_for_version_strings(version_strings):
         if base_version:
             yield base_version
 
-def grab_proper_extension_version(extension, shell_version, disable_version_validation=False):
+def grab_proper_extension_version(extension, shell_version, version_validation_enabled=False):
     def get_best_shell_version():
         visible_versions = extension.visible_versions
 
@@ -82,13 +232,13 @@ def grab_proper_extension_version(extension, shell_version, disable_version_vali
 
     shell_versions = set(get_versions_for_version_strings([shell_version]))
     if not shell_versions:
-        return get_best_shell_version() if disable_version_validation else None
+        return get_best_shell_version() if not version_validation_enabled else None
 
     versions = extension.visible_versions.filter(shell_versions__in=shell_versions)
     if versions.count() < 1:
-        return get_best_shell_version() if disable_version_validation else None
-    else:
-        return versions.order_by('-version')[0]
+        return get_best_shell_version() if not version_validation_enabled else None
+
+    return versions.order_by('-version')[0]
 
 def find_extension_version_from_params(extension, params):
     vpk = params.get('version_tag', '')
@@ -97,7 +247,7 @@ def find_extension_version_from_params(extension, params):
                                                                                                     "false"] 
else True
 
     if shell_version:
-        return grab_proper_extension_version(extension, shell_version, disable_version_validation)
+        return grab_proper_extension_version(extension, shell_version, not disable_version_validation)
     elif vpk:
         try:
             return extension.visible_versions.get(pk=int(vpk))
@@ -154,7 +304,7 @@ def shell_update(request):
             # The user may have a newer version than what's on the site.
             continue
 
-        proper_version = grab_proper_extension_version(extension, shell_version, disable_version_validation)
+        proper_version = grab_proper_extension_version(extension, shell_version, not 
disable_version_validation)
 
         if proper_version is not None:
             if version < proper_version.version:
@@ -212,68 +362,6 @@ def ajax_query_params_query(request, versions, n_per_page):
 
     return page_obj.object_list, paginator.num_pages
 
-def ajax_query_search_query(request, versions, n_per_page):
-    querystring = request.GET.get('search', '')
-
-    database, enquire = search.enquire(querystring, versions)
-
-    page = request.GET.get('page', 1)
-    try:
-        offset = (int(page) - 1) * n_per_page
-    except ValueError:
-        raise Http404()
-
-    if n_per_page == -1:
-        mset = enquire.get_mset(offset, database.get_doccount())
-        num_pages = 1
-    else:
-        mset = enquire.get_mset(offset, n_per_page, 5 * n_per_page)
-        num_pages = int(ceil(float(mset.get_matches_lower_bound()) / n_per_page))
-
-    pks = [match.document.get_data().decode('utf-8') for match in mset]
-
-    # filter doesn't guarantee an order, so we need to get all the
-    # possible models then look them up to get the ordering
-    # returned by xapian. This hits the database all at once, rather
-    # than pagesize times.
-    extension_lookup = {}
-    for extension in models.Extension.objects.filter(pk__in=pks):
-        extension_lookup[str(extension.pk)] = extension
-
-    extensions = [extension_lookup[pk] for pk in pks]
-
-    return extensions, num_pages
-
-@ajax_view
-def ajax_query_view(request):
-    try:
-        n_per_page = int(request.GET['n_per_page'])
-        if n_per_page == 1000:
-            from django.conf import settings
-            # This is GNOME Software request. Let's redirect it to static file
-            return redirect((settings.STATIC_URL + "extensions.json"), permanent=True)
-
-        n_per_page = min(n_per_page, 25)
-    except (KeyError, ValueError):
-        n_per_page = 10
-
-    version_strings = request.GET.getlist('shell_version')
-    if version_strings and version_strings not in (['all'], ['-1']):
-        versions = set(get_versions_for_version_strings(version_strings))
-    else:
-        versions = None
-
-    if request.GET.get('search',  ''):
-        func = ajax_query_search_query
-    else:
-        func = ajax_query_params_query
-
-    object_list, num_pages = func(request, versions, n_per_page)
-
-    return dict(extensions=[ajax_details(e) for e in object_list],
-                total=len(object_list),
-                numpages=num_pages)
-
 @model_view(models.Extension)
 def extension_view(request, obj, **kwargs):
     extension, versions = obj, obj.visible_versions
diff --git a/sweettooth/settings.py b/sweettooth/settings.py
index 6fff1077..9cec5aaf 100644
--- a/sweettooth/settings.py
+++ b/sweettooth/settings.py
@@ -13,8 +13,6 @@ SITE_ROOT = os.path.dirname(os.path.abspath(__file__))
 
 BASE_DIR = os.path.dirname(SITE_ROOT)
 
-XAPIAN_DB_PATH = os.getenv('EGO_XAPIAN_DB') or os.path.join(BASE_DIR, 'xapian.db')
-
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/stable/howto/deployment/checklist/
 
@@ -47,6 +45,9 @@ INSTALLED_APPS = (
     'django.contrib.messages',
 
     'rest_framework',
+    'django_filters',
+
+    'django_elasticsearch_dsl',
 
     'sweettooth.api',
     'sweettooth.extensions',
@@ -110,6 +111,13 @@ DATABASES = {
     'default': dj_database_url.config(env="EGO_DATABASE_URL", default="sqlite://./test.db")
 }
 
+ELASTICSEARCH_DSL = {
+    'default': {
+        'hosts': os.getenv('EGO_ELASTIC_ADDRESS') or 'localhost:9200'
+    },
+}
+
+ELASTICSEARCH_DSL_AUTOSYNC = False
 
 # Internationalization
 # https://docs.djangoproject.com/en/stable/topics/i18n/


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]