[extensions-web/wip/api/v1: 38/45] api: initial implementation of /extensions node
- From: Yuri Konotopov <ykonotopov src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [extensions-web/wip/api/v1: 38/45] api: initial implementation of /extensions node
- Date: Sun, 16 Jan 2022 11:27:29 +0000 (UTC)
commit acca34121f44018e5cf446cf6345726c783e0d1c
Author: Yuri Konotopov <ykonotopov gnome org>
Date: Sun Nov 22 12:25:15 2020 +0400
api: initial implementation of /extensions node
Xapian usage is dropped with this commit in favour of Elasticsearch.
Actually there is no proper support of Xapian for DRF or Django exists,
but only generic python bindings. Support for Xapian in Haystack is very
poor. Support of Elasticsearch in Haystack is almost dead also.
On the other hand django-elasticsearch-dsl is developed actively for now.
openshift/docker/docker-compose.yml | 11 +
requirements.txt | 23 ++
sweettooth/api/v1/urls.py | 12 +-
sweettooth/extensions/documents.py | 50 +++++
.../management/commands/indexextensions.py | 19 --
.../management/commands/searchextensions.py | 38 ----
sweettooth/extensions/search.py | 87 --------
sweettooth/extensions/urls.py | 6 +-
sweettooth/extensions/views.py | 236 ++++++++++++++-------
sweettooth/settings.py | 12 +-
10 files changed, 269 insertions(+), 225 deletions(-)
---
diff --git a/openshift/docker/docker-compose.yml b/openshift/docker/docker-compose.yml
index 740956f..d0bab71 100644
--- a/openshift/docker/docker-compose.yml
+++ b/openshift/docker/docker-compose.yml
@@ -37,8 +37,10 @@ services:
EGO_STATIC_ROOT: /extensions-web/www/static-files
depends_on:
- db
+ - search
links:
- db
+ - search
volumes:
- "static:/extensions-web/www"
frontend:
@@ -54,7 +56,16 @@ services:
- "8080:8080"
volumes:
- "static:/extensions-web/www"
+ search:
+ image: elasticsearch:7.9.1
+ environment:
+ "discovery.type": single-node
+ ports:
+ - "127.0.0.1:49200:9200"
+ volumes:
+ - search:/usr/share/elasticsearch/data
volumes:
database:
+ search:
static:
diff --git a/requirements.txt b/requirements.txt
index 0cfe3b3..fa139db 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,12 +7,18 @@ djangorestframework==3.13.1 \
django-autoslug==1.9.8 \
--hash=sha256:26459eeddec207e307c55777a10fc25d17f4978753695340b16a17ed248a6f70 \
--hash=sha256:bae66c27d35615f472865b99c4d107f3b3add3d22ee337e84960fc07694abd45
+django-filter==21.1 \
+ --hash=sha256:632a251fa8f1aadb4b8cceff932bb52fe2f826dd7dfe7f3eac40e5c463d6836e \
+ --hash=sha256:f4a6737a30104c98d2e2a5fb93043f36dd7978e0c7ddc92f5998e85433ea5063
django-contrib-comments==2.1.0 \
--hash=sha256:d82f1d04690550df026553053903deec0c52dc54212e1b79241b08f0355cff2c \
--hash=sha256:e02c7341ea1f4bcdfa347851dbf5e632d3e591d84b4f77de2f90b93398897f3c
django-registration==3.2 \
--hash=sha256:2ea8c7d89a8760ccde41dfd335aa28ba89073d09aab5a0f5f3d7c8c148fcc518 \
--hash=sha256:e79fdbfa22bfaf4182efccb6604391391a7de19438d2669c5b9520a7708efbd2
+django-elasticsearch-dsl==7.2.2 \
+ --hash=sha256:3c58a254a6318b169eb904d41d802924b99ea8e53ddc2c596ebba90506cf47fa \
+ --hash=sha256:811d3909b3387fd55c19d9bbcf0e9a9b234f085df3f8422d59e7519a5f733e0e
Pygments==2.11.2 \
--hash=sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65 \
--hash=sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a
@@ -44,3 +50,20 @@ sqlparse==0.4.2 \
confusable_homoglyphs==3.2.0 \
--hash=sha256:3b4a0d9fa510669498820c91a0bfc0c327568cecec90648cf3819d4a6fc6a751 \
--hash=sha256:e3ce611028d882b74a5faa69e3cbb5bd4dcd9f69936da6e73d33eda42c917944
+# django-elasticsearch-dsl
+elasticsearch-dsl==7.4.0 \
+ --hash=sha256:046ea10820b94c075081b528b4526c5bc776bda4226d702f269a5f203232064b \
+ --hash=sha256:c4a7b93882918a413b63bed54018a1685d7410ffd8facbc860ee7fd57f214a6d
+python-dateutil==2.8.2 \
+ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
+ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
+elasticsearch==7.16.3 \
+ --hash=sha256:213eaae9937924a9c523792748e7b0fc7ea2041d32a646538cbe81e52f852e1f \
+ --hash=sha256:8adf8bc351ed55df7296be1009d38a1c999c0abc7d8700fa88533f1ad6087c5e
+# elasticsearch
+certifi==2021.10.8 \
+ --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \
+ --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569
+urllib3==1.26.8 \
+ --hash=sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed \
+ --hash=sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c
diff --git a/sweettooth/api/v1/urls.py b/sweettooth/api/v1/urls.py
index aa9849a..fc98098 100644
--- a/sweettooth/api/v1/urls.py
+++ b/sweettooth/api/v1/urls.py
@@ -13,9 +13,19 @@ from django.urls import path
from rest_framework.routers import SimpleRouter
from sweettooth.api.v1.views import HelloView
+from sweettooth.extensions.views import ExtensionsViewSet
from sweettooth.users.views import UserProfileDetailView
-urlpatterns = [
+# Create a router and register our viewsets with it.
+router = SimpleRouter()
+router.register(
+ r'v1/extensions',
+ ExtensionsViewSet,
+ basename='extension',
+)
+
+urlpatterns = router.urls
+urlpatterns += [
path('v1/hello/', HelloView.as_view()),
path('v1/profile/<int:pk>/',
UserProfileDetailView.as_view(),
diff --git a/sweettooth/extensions/documents.py b/sweettooth/extensions/documents.py
new file mode 100644
index 0000000..a1ee7ee
--- /dev/null
+++ b/sweettooth/extensions/documents.py
@@ -0,0 +1,50 @@
+"""
+ GNOME Shell extensions repository
+ Copyright (C) 2020 Yuri Konotopov <ykonotopov gnome org>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+"""
+
+import logging
+
+from django_elasticsearch_dsl import Document
+from django_elasticsearch_dsl.registries import registry
+
+from .models import Extension
+
+# Get an instance of a logger
+logger = logging.getLogger(__name__)
+
+
+@registry.register_document
+class ExtensionDocument(Document):
+ class Index:
+ name = 'extensions'
+
+ class Django:
+ model = Extension
+
+ # The fields of the model you want to be indexed in Elasticsearch
+ fields = [
+ 'uuid',
+ 'name',
+ 'description',
+ 'created',
+ 'downloads',
+ 'popularity',
+ ]
+
+ def get_queryset(self):
+ return super(ExtensionDocument, self).get_queryset().select_related(
+ 'creator'
+ )
+
+ def prepare_creator(self, extension):
+ return extension.creator.username
+
+ @staticmethod
+ def document_fields():
+ return ['uuid', 'name', 'description', 'creator']
diff --git a/sweettooth/extensions/search.py b/sweettooth/extensions/search.py
index fe6d6de..e69de29 100644
--- a/sweettooth/extensions/search.py
+++ b/sweettooth/extensions/search.py
@@ -1,87 +0,0 @@
-
-import xapian
-
-from functools import reduce
-from django.conf import settings
-from django.db.models import signals
-
-from sweettooth.extensions.models import Extension, ExtensionVersion
-from sweettooth.extensions.models import reviewed, extension_updated
-
-def index_extension(extension):
- if extension.latest_version is None:
- return
-
- db = xapian.WritableDatabase(settings.XAPIAN_DB_PATH, xapian.DB_CREATE_OR_OPEN)
-
- termgen = xapian.TermGenerator()
- termgen.set_stemmer(xapian.Stem("en"))
-
- doc = xapian.Document()
- termgen.set_document(doc)
-
- termgen.index_text(extension.name, 10)
- termgen.index_text(extension.uuid)
- termgen.index_text(extension.description)
-
- doc.set_data(str(extension.pk))
-
- idterm = "Q%s" % (extension.pk,)
- doc.add_boolean_term(idterm)
- for shell_version in extension.visible_shell_version_map.keys():
- doc.add_boolean_term("V%s" % (shell_version,))
-
- db.replace_document(idterm, doc)
-
-def delete_extension(extension):
- db = xapian.WritableDatabase(settings.XAPIAN_DB_PATH, xapian.DB_CREATE_OR_OPEN)
- idterm = "Q%s" % (extension.pk,)
- db.delete_document(idterm)
-
-
-def reviewed_handler(sender, request, version, review, **kwargs):
- index_extension(version.extension)
-reviewed.connect(reviewed_handler)
-
-def extension_updated_handler(extension, **kwargs):
- index_extension(extension)
-extension_updated.connect(extension_updated_handler)
-
-def post_extension_delete_handler(instance, **kwargs):
- delete_extension(instance)
-signals.post_delete.connect(post_extension_delete_handler, sender=Extension)
-
-def post_version_save_handler(instance, **kwargs):
- index_extension(instance.extension)
-signals.post_save.connect(post_version_save_handler, sender=ExtensionVersion)
-
-def combine_queries(op, queries):
- def make_query(left, right):
- return xapian.Query(op, left, right)
- return reduce(make_query, queries)
-
-def make_version_queries(versions):
- queries = [xapian.Query("V%s" % (v.version_string,)) for v in versions]
- return combine_queries(xapian.Query.OP_OR, queries)
-
-def enquire(querystring, versions=None):
- try:
- db = xapian.Database(settings.XAPIAN_DB_PATH)
- except xapian.DatabaseOpeningError:
- return None
-
- qp = xapian.QueryParser()
- qp.set_stemming_strategy(qp.STEM_SOME)
- qp.set_stemmer(xapian.Stem("en"))
- qp.set_database(db)
-
- query = qp.parse_query(querystring, qp.FLAG_PARTIAL)
-
- if versions:
- query = xapian.Query(xapian.Query.OP_FILTER,
- query,
- make_version_queries(versions))
-
- enquiry = xapian.Enquire(db)
- enquiry.set_query(query)
- return db, enquiry
diff --git a/sweettooth/extensions/urls.py b/sweettooth/extensions/urls.py
index 3f65755..e655a47 100644
--- a/sweettooth/extensions/urls.py
+++ b/sweettooth/extensions/urls.py
@@ -1,7 +1,7 @@
from django.conf.urls import include
from django.urls import re_path
-from django.views.generic.base import TemplateView
+from django.views.generic.base import RedirectView, TemplateView
from sweettooth.extensions import views, models, feeds
@@ -19,8 +19,6 @@ ajax_patterns = [
]
shell_patterns = [
- re_path(r'^extension-query/', views.ajax_query_view, name='extensions-query'),
-
re_path(r'^extension-info/', views.ajax_details_view),
re_path(r'^download-extension/(?P<uuid>.+)\.shell-extension\.zip$',
@@ -30,7 +28,7 @@ shell_patterns = [
]
urlpatterns = [
- re_path(r'^$', TemplateView.as_view(template_name='extensions/list.html'), name='extensions-index'),
+ re_path(r'^$', RedirectView.as_view(url='/', permanent=False), name='extensions-index'),
re_path(r'^about/$', TemplateView.as_view(template_name='extensions/about.html'),
name='extensions-about'),
diff --git a/sweettooth/extensions/views.py b/sweettooth/extensions/views.py
index 7e341fb..3d06154 100644
--- a/sweettooth/extensions/views.py
+++ b/sweettooth/extensions/views.py
@@ -10,26 +10,176 @@
"""
import json
-from math import ceil
+from functools import reduce
from django.core.exceptions import ValidationError
from django.core.paginator import Paginator, InvalidPage
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.db import transaction
-from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest, HttpResponseForbidden,
HttpResponseServerError, Http404
+from django.http import (
+ JsonResponse,
+ HttpResponse,
+ HttpResponseBadRequest,
+ HttpResponseForbidden,
+ HttpResponseServerError,
+ Http404
+)
from django.shortcuts import get_object_or_404, redirect, render
from django.template.loader import render_to_string
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django.urls import reverse
-from sweettooth.exceptions import DatabaseErrorWithMessages
-from sweettooth.extensions import models, search
-from sweettooth.extensions.forms import ImageUploadForm, UploadForm
+from django_filters.rest_framework import DjangoFilterBackend
+
+from rest_framework import filters, mixins, viewsets, status
+from rest_framework.decorators import action
+from rest_framework.pagination import PageNumberPagination
+from rest_framework.parsers import JSONParser
+from rest_framework.response import Response
+
from sweettooth.decorators import ajax_view, model_view
+from sweettooth.exceptions import DatabaseErrorWithMessages
+from sweettooth.extensions import models, serializers
+from sweettooth.extensions.documents import ExtensionDocument
+from sweettooth.extensions.forms import ImageUploadForm, UploadForm
from sweettooth.extensions.templatetags.extension_icon import extension_icon
+from sweettooth import settings
+
+
+class ExtensionsPagination(PageNumberPagination):
+ page_size = settings.REST_FRAMEWORK['PAGE_SIZE']
+ page_size_query_param = 'page_size'
+ max_page_size = 100
+
+
+class ExtensionsViewSet(mixins.ListModelMixin,
+ mixins.RetrieveModelMixin,
+ viewsets.GenericViewSet):
+ lookup_field = 'uuid'
+ lookup_value_regex = '[-a-zA-Z0-9@._]+'
+ serializer_class = serializers.ExtensionSerializer
+ pagination_class = ExtensionsPagination
+ filter_backends = [DjangoFilterBackend, filters.OrderingFilter]
+ ordering_fields = ['created', 'downloads', 'popularity', '?']
+ page_size = 25
+ page_size_query_param = 'page_size'
+ max_page_size = 100
+
+ def get_queryset(self):
+ uuids = self.request.query_params.getlist('uuid[]')
+ status = self.request.query_params.get('status')
+ queryset = models.Extension.objects.all()
+
+ if uuids:
+ queryset = queryset.filter(uuid__in=uuids)
+
+ if status is not None and status.isnumeric():
+ status = int(status)
+
+ if status in models.STATUSES:
+ queryset = queryset.filter(versions__status=status).distinct()
+
+ return queryset
+
+ @action(methods=['post'], detail=False, parser_classes=[JSONParser])
+ def updates(self, request):
+ installed = self.request.data.get('installed')
+ shell_version = self.request.data.get('shell_version')
+ version_validation_enabled = self.request.data.get('version_validation_enabled', False)
+
+ if not installed:
+ return Response()
+
+ extensions = models.Extension.objects.filter(uuid__in=installed.keys())
+
+ result = {}
+ for uuid, version in installed.items():
+ try:
+ version = int(version)
+ except (KeyError, TypeError):
+ continue
+ except ValueError:
+ version = 1
+
+ extension = reduce(
+ lambda x, y, uuid=uuid: (
+ x if x and x.uuid == uuid else
+ y if y and y.uuid == uuid else
+ None
+ ),
+ list(extensions)
+ )
+
+ if not extension:
+ continue
+
+ try:
+ version = extension.versions.get(version=version)
+ except models.ExtensionVersion.DoesNotExist:
+ # Skip unknown versions
+ continue
+
+ proper_version = grab_proper_extension_version(
+ extension,
+ shell_version,
+ version_validation_enabled
+ )
+
+ if proper_version is not None and proper_version.version != version.version:
+ result[uuid] = {
+ 'action': 'change',
+ 'version': proper_version.version,
+ }
+
+ return Response(result)
+
+ @action(methods=['get'], detail=False, url_path='search/(?P<query>[^/.]+)')
+ def search(self, request, query=None):
+ try:
+ page = int(self.request.query_params.get(self.pagination_class.page_query_param, 1))
+ page_size = max(
+ 1,
+ min(
+ self.max_page_size,
+ int(self.request.query_params.get(self.page_size_query_param, self.page_size))
+ )
+ )
+ except Exception as ex:
+ print(ex)
+ return Response(status=status.HTTP_400_BAD_REQUEST)
+
+ if not query:
+ return Response(status=status.HTTP_400_BAD_REQUEST)
+
+ queryset = ExtensionDocument.search().query(
+ "multi_match",
+ query=query,
+ fields=ExtensionDocument.document_fields()
+ )
+
+ ordering = self.request.query_params.get('ordering')
+ ordering_field = (
+ ordering
+ if not ordering or ordering[0] != '-'
+ else ordering[1:]
+ )
+ if ordering and ordering_field in self.ordering_fields:
+ queryset = queryset.sort(ordering)
+
+ paginator = Paginator(queryset.to_queryset(), page_size)
+
+ return Response({
+ 'count': paginator.count,
+ 'results': self.serializer_class(
+ [sr for sr in paginator.page(page).object_list],
+ many=True,
+ context={'request': request}
+ ).data
+ })
+
def get_versions_for_version_strings(version_strings):
def get_version(major, minor, point):
@@ -62,7 +212,7 @@ def get_versions_for_version_strings(version_strings):
yield base_version
-def grab_proper_extension_version(extension, shell_version, disable_version_validation=False):
+def grab_proper_extension_version(extension, shell_version, version_validation_enabled=False):
def get_best_shell_version():
visible_versions = extension.visible_versions
@@ -89,13 +239,13 @@ def grab_proper_extension_version(extension, shell_version, disable_version_vali
shell_versions = set(get_versions_for_version_strings([shell_version]))
if not shell_versions:
- return get_best_shell_version() if disable_version_validation else None
+ return get_best_shell_version() if not version_validation_enabled else None
versions = extension.visible_versions.filter(shell_versions__in=shell_versions)
if versions.count() < 1:
- return get_best_shell_version() if disable_version_validation else None
- else:
- return versions.order_by('-version')[0]
+ return get_best_shell_version() if not version_validation_enabled else None
+
+ return versions.order_by('-version')[0]
def find_extension_version_from_params(extension, params):
vpk = params.get('version_tag', '')
@@ -104,7 +254,7 @@ def find_extension_version_from_params(extension, params):
"false"]
else True
if shell_version:
- return grab_proper_extension_version(extension, shell_version, disable_version_validation)
+ return grab_proper_extension_version(extension, shell_version, not disable_version_validation)
elif vpk:
try:
return extension.visible_versions.get(pk=int(vpk))
@@ -163,7 +313,7 @@ def shell_update(request):
continue
try:
- proper_version = grab_proper_extension_version(extension, shell_version,
disable_version_validation)
+ proper_version = grab_proper_extension_version(extension, shell_version, not
disable_version_validation)
except models.InvalidShellVersion:
return HttpResponseBadRequest()
@@ -223,68 +373,6 @@ def ajax_query_params_query(request, versions, n_per_page):
return page_obj.object_list, paginator.num_pages
-def ajax_query_search_query(request, versions, n_per_page):
- querystring = request.GET.get('search', '')
-
- database, enquire = search.enquire(querystring, versions)
-
- page = request.GET.get('page', 1)
- try:
- offset = (int(page) - 1) * n_per_page
- except ValueError:
- raise Http404()
-
- if n_per_page == -1:
- mset = enquire.get_mset(offset, database.get_doccount())
- num_pages = 1
- else:
- mset = enquire.get_mset(offset, n_per_page, 5 * n_per_page)
- num_pages = int(ceil(float(mset.get_matches_lower_bound()) / n_per_page))
-
- pks = [match.document.get_data().decode('utf-8') for match in mset]
-
- # filter doesn't guarantee an order, so we need to get all the
- # possible models then look them up to get the ordering
- # returned by xapian. This hits the database all at once, rather
- # than pagesize times.
- extension_lookup = {}
- for extension in models.Extension.objects.filter(pk__in=pks):
- extension_lookup[str(extension.pk)] = extension
-
- extensions = [extension_lookup[pk] for pk in pks]
-
- return extensions, num_pages
-
-@ajax_view
-def ajax_query_view(request):
- try:
- n_per_page = int(request.GET['n_per_page'])
- if n_per_page == 1000:
- from django.conf import settings
- # This is GNOME Software request. Let's redirect it to static file
- return redirect((settings.STATIC_URL + "extensions.json"), permanent=True)
-
- n_per_page = min(n_per_page, 25)
- except (KeyError, ValueError):
- n_per_page = 10
-
- version_strings = request.GET.getlist('shell_version')
- if version_strings and version_strings not in (['all'], ['-1']):
- versions = set(get_versions_for_version_strings(version_strings))
- else:
- versions = None
-
- if request.GET.get('search', ''):
- func = ajax_query_search_query
- else:
- func = ajax_query_params_query
-
- object_list, num_pages = func(request, versions, n_per_page)
-
- return dict(extensions=[ajax_details(e) for e in object_list],
- total=len(object_list),
- numpages=num_pages)
-
@model_view(models.Extension)
def extension_view(request, obj, **kwargs):
extension, versions = obj, obj.visible_versions
diff --git a/sweettooth/settings.py b/sweettooth/settings.py
index aeb87e0..1810984 100644
--- a/sweettooth/settings.py
+++ b/sweettooth/settings.py
@@ -13,8 +13,6 @@ SITE_ROOT = os.path.dirname(os.path.abspath(__file__))
BASE_DIR = os.path.dirname(SITE_ROOT)
-XAPIAN_DB_PATH = os.getenv('EGO_XAPIAN_DB') or os.path.join(BASE_DIR, 'xapian.db')
-
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/stable/howto/deployment/checklist/
@@ -47,6 +45,9 @@ INSTALLED_APPS = (
'django.contrib.messages',
'rest_framework',
+ 'django_filters',
+
+ 'django_elasticsearch_dsl',
'sweettooth.api',
'sweettooth.extensions',
@@ -110,6 +111,13 @@ DATABASES = {
'default': dj_database_url.config(env="EGO_DATABASE_URL", default="sqlite://./test.db")
}
+ELASTICSEARCH_DSL = {
+ 'default': {
+ 'hosts': os.getenv('EGO_ELASTIC_ADDRESS') or 'localhost:9200'
+ },
+}
+
+ELASTICSEARCH_DSL_AUTOSYNC = False
# Internationalization
# https://docs.djangoproject.com/en/stable/topics/i18n/
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
