GNOME.org
 

[odrs-web/oscp] Allow deleting and joining components

commit 25d2787d1db1f1c80812c3b932c4c3b2e15fb1ef
Author: Richard Hughes <richard hughsie com>
Date:   Thu Jul 4 17:22:10 2019 +0100

    Allow deleting and joining components
    
    This also means we can return many more reviews when an app has been renamed.

 app_data/migrations/versions/ef03b3a98056_.py | 175 ++++++++++++++++++++++++++
 app_data/migrations/versions/f32bd8265c3b_.py |  24 ++++
 app_data/odrs/models.py                       |  32 ++++-
 app_data/odrs/templates/components.html       |  46 ++++++-
 app_data/odrs/tests/odrs_test.py              |  46 +++++++
 app_data/odrs/views_admin.py                  | 100 +++++++++++++++
 app_data/odrs/views_api.py                    |   7 ++
 7 files changed, 424 insertions(+), 6 deletions(-)
---
diff --git a/app_data/migrations/versions/ef03b3a98056_.py b/app_data/migrations/versions/ef03b3a98056_.py
new file mode 100644
index 0000000..75c02a5
--- /dev/null
+++ b/app_data/migrations/versions/ef03b3a98056_.py
@@ -0,0 +1,175 @@
+"""
+
+Revision ID: ef03b3a98056
+Revises: f32bd8265c3b
+Create Date: 2019-07-05 12:53:17.236904
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ef03b3a98056'
+down_revision = 'f32bd8265c3b'
+
+from odrs import db
+from odrs.models import Component
+
+def upgrade():
+
+    # get all existing components
+    components = {}
+    for component in db.session.query(Component).\
+                        filter(Component.app_id != '').\
+                        order_by(Component.app_id.asc()).all():
+        components[component.app_id] = component
+
+    # guessed, thanks Canonical! :/
+    for app_id in components:
+        if not app_id.startswith('io.snapcraft.'):
+            continue
+        if components[app_id].component_id_parent:
+            continue
+        name, _ = app_id[13:].rsplit('-', maxsplit=1)
+        parent = components.get(name + '.desktop')
+        if not parent:
+            continue
+        print('adding snapcraft parent for {} -> {}'.format(components[app_id].app_id,
+                                                            parent.app_id))
+        parent.adopt(components[app_id])
+
+    # from appstream-glib
+    mapping = {
+        'baobab.desktop': 'org.gnome.baobab.desktop',
+        'bijiben.desktop': 'org.gnome.bijiben.desktop',
+        'cheese.desktop': 'org.gnome.Cheese.desktop',
+        'devhelp.desktop': 'org.gnome.Devhelp.desktop',
+        'epiphany.desktop': 'org.gnome.Epiphany.desktop',
+        'file-roller.desktop': 'org.gnome.FileRoller.desktop',
+        'font-manager.desktop': 'org.gnome.FontManager.desktop',
+        'gcalctool.desktop': 'gnome-calculator.desktop',
+        'gcm-viewer.desktop': 'org.gnome.ColorProfileViewer.desktop',
+        'geary.desktop': 'org.gnome.Geary.desktop',
+        'gedit.desktop': 'org.gnome.gedit.desktop',
+        'glchess.desktop': 'gnome-chess.desktop',
+        'glines.desktop': 'five-or-more.desktop',
+        'gnect.desktop': 'four-in-a-row.desktop',
+        'gnibbles.desktop': 'gnome-nibbles.desktop',
+        'gnobots2.desktop': 'gnome-robots.desktop',
+        'gnome-2048.desktop': 'org.gnome.gnome-2048.desktop',
+        'gnome-boxes.desktop': 'org.gnome.Boxes.desktop',
+        'gnome-calculator.desktop': 'org.gnome.Calculator.desktop',
+        'gnome-clocks.desktop': 'org.gnome.clocks.desktop',
+        'gnome-contacts.desktop': 'org.gnome.Contacts.desktop',
+        'gnome-dictionary.desktop': 'org.gnome.Dictionary.desktop',
+        'gnome-disks.desktop': 'org.gnome.DiskUtility.desktop',
+        'gnome-documents.desktop': 'org.gnome.Documents.desktop',
+        'gnome-font-viewer.desktop': 'org.gnome.font-viewer.desktop',
+        'gnome-maps.desktop': 'org.gnome.Maps.desktop',
+        'gnome-nibbles.desktop': 'org.gnome.Nibbles.desktop',
+        'gnome-photos.desktop': 'org.gnome.Photos.desktop',
+        'gnome-power-statistics.desktop': 'org.gnome.PowerStats.desktop',
+        'gnome-screenshot.desktop': 'org.gnome.Screenshot.desktop',
+        'gnome-software.desktop': 'org.gnome.Software.desktop',
+        'gnome-sound-recorder.desktop': 'org.gnome.SoundRecorder.desktop',
+        'gnome-terminal.desktop': 'org.gnome.Terminal.desktop',
+        'gnome-weather.desktop': 'org.gnome.Weather.Application.desktop',
+        'gnomine.desktop': 'gnome-mines.desktop',
+        'gnotravex.desktop': 'gnome-tetravex.desktop',
+        'gnotski.desktop': 'gnome-klotski.desktop',
+        'gtali.desktop': 'tali.desktop',
+        'hitori.desktop': 'org.gnome.Hitori.desktop',
+        'latexila.desktop': 'org.gnome.latexila.desktop',
+        'lollypop.desktop': 'org.gnome.Lollypop.desktop',
+        'nautilus.desktop': 'org.gnome.Nautilus.desktop',
+        'polari.desktop': 'org.gnome.Polari.desktop',
+        'sound-juicer.desktop': 'org.gnome.SoundJuicer.desktop',
+        'totem.desktop': 'org.gnome.Totem.desktop',
+        'akregator.desktop': 'org.kde.akregator.desktop',
+        'apper.desktop': 'org.kde.apper.desktop',
+        'ark.desktop': 'org.kde.ark.desktop',
+        'blinken.desktop': 'org.kde.blinken.desktop',
+        'cantor.desktop': 'org.kde.cantor.desktop',
+        'digikam.desktop': 'org.kde.digikam.desktop',
+        'dolphin.desktop': 'org.kde.dolphin.desktop',
+        'dragonplayer.desktop': 'org.kde.dragonplayer.desktop',
+        'filelight.desktop': 'org.kde.filelight.desktop',
+        'gwenview.desktop': 'org.kde.gwenview.desktop',
+        'juk.desktop': 'org.kde.juk.desktop',
+        'kajongg.desktop': 'org.kde.kajongg.desktop',
+        'kalgebra.desktop': 'org.kde.kalgebra.desktop',
+        'kalzium.desktop': 'org.kde.kalzium.desktop',
+        'kamoso.desktop': 'org.kde.kamoso.desktop',
+        'kanagram.desktop': 'org.kde.kanagram.desktop',
+        'kapman.desktop': 'org.kde.kapman.desktop',
+        'kapptemplate.desktop': 'org.kde.kapptemplate.desktop',
+        'kbruch.desktop': 'org.kde.kbruch.desktop',
+        'kdevelop.desktop': 'org.kde.kdevelop.desktop',
+        'kfind.desktop': 'org.kde.kfind.desktop',
+        'kgeography.desktop': 'org.kde.kgeography.desktop',
+        'kgpg.desktop': 'org.kde.kgpg.desktop',
+        'khangman.desktop': 'org.kde.khangman.desktop',
+        'kig.desktop': 'org.kde.kig.desktop',
+        'kiriki.desktop': 'org.kde.kiriki.desktop',
+        'kiten.desktop': 'org.kde.kiten.desktop',
+        'klettres.desktop': 'org.kde.klettres.desktop',
+        'klipper.desktop': 'org.kde.klipper.desktop',
+        'KMail2.desktop': 'org.kde.kmail.desktop',
+        'kmplot.desktop': 'org.kde.kmplot.desktop',
+        'kollision.desktop': 'org.kde.kollision.desktop',
+        'kolourpaint.desktop': 'org.kde.kolourpaint.desktop',
+        'konsole.desktop': 'org.kde.konsole.desktop',
+        'Kontact.desktop': 'org.kde.kontact.desktop',
+        'korganizer.desktop': 'org.kde.korganizer.desktop',
+        'krita.desktop': 'org.kde.krita.desktop',
+        'kshisen.desktop': 'org.kde.kshisen.desktop',
+        'kstars.desktop': 'org.kde.kstars.desktop',
+        'ksudoku.desktop': 'org.kde.ksudoku.desktop',
+        'ktouch.desktop': 'org.kde.ktouch.desktop',
+        'ktp-log-viewer.desktop': 'org.kde.ktplogviewer.desktop',
+        'kturtle.desktop': 'org.kde.kturtle.desktop',
+        'kwordquiz.desktop': 'org.kde.kwordquiz.desktop',
+        'marble.desktop': 'org.kde.marble.desktop',
+        'okteta.desktop': 'org.kde.okteta.desktop',
+        'parley.desktop': 'org.kde.parley.desktop',
+        'partitionmanager.desktop': 'org.kde.PartitionManager.desktop',
+        'picmi.desktop': 'org.kde.picmi.desktop',
+        'rocs.desktop': 'org.kde.rocs.desktop',
+        'showfoto.desktop': 'org.kde.showfoto.desktop',
+        'skrooge.desktop': 'org.kde.skrooge.desktop',
+        'step.desktop': 'org.kde.step.desktop',
+        'yakuake.desktop': 'org.kde.yakuake.desktop',
+        'colorhug-ccmx.desktop': 'com.hughski.ColorHug.CcmxLoader.desktop',
+        'colorhug-flash.desktop': 'com.hughski.ColorHug.FlashLoader.desktop',
+        'dconf-editor.desktop': 'ca.desrt.dconf-editor.desktop',
+        'feedreader.desktop': 'org.gnome.FeedReader.desktop',
+        'qtcreator.desktop': 'org.qt-project.qtcreator.desktop',
+    }
+    for app_id in mapping:
+        if not app_id in components:
+            continue
+        app_id_new = mapping[app_id]
+        if not app_id_new in components:
+            continue
+        if components[app_id].component_id_parent:
+            continue
+        print('adding legacy parent for {} -> {}'.format(components[app_id].app_id,
+                                                         components[app_id_new].app_id))
+        components[app_id_new].adopt(components[app_id])
+
+    # upstream drops the .desktop sometimes
+    for app_id in components:
+        if components[app_id].component_id_parent:
+            continue
+        app_id_new = app_id.replace('.desktop', '')
+        if app_id == app_id_new:
+            continue
+        if not app_id_new in components:
+            continue
+        print('adding parent for {} -> {}'.format(components[app_id].app_id,
+                                                  components[app_id_new].app_id))
+        components[app_id_new].adopt(components[app_id])
+
+    # done
+    db.session.commit()
+
+def downgrade():
+    pass
diff --git a/app_data/migrations/versions/f32bd8265c3b_.py b/app_data/migrations/versions/f32bd8265c3b_.py
new file mode 100644
index 0000000..3e8725e
--- /dev/null
+++ b/app_data/migrations/versions/f32bd8265c3b_.py
@@ -0,0 +1,24 @@
+"""
+
+Revision ID: f32bd8265c3b
+Revises: a22c286d8094
+Create Date: 2019-07-04 16:35:39.673744
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f32bd8265c3b'
+down_revision = 'a22c286d8094'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+def upgrade():
+    op.add_column('components', sa.Column('component_id_parent', sa.Integer(), nullable=True))
+    op.create_foreign_key('components_ibfk_4', 'components', 'components', ['component_id_parent'], 
['component_id'])
+
+
+def downgrade():
+    op.drop_constraint('components_ibfk_4', 'components', type_='foreignkey')
+    op.drop_column('components', 'component_id_parent')
diff --git a/app_data/odrs/models.py b/app_data/odrs/models.py
index ea7ca53..8103c99 100644
--- a/app_data/odrs/models.py
+++ b/app_data/odrs/models.py
@@ -141,17 +141,37 @@ class Component(db.Model):
     __table_args__ = {'mysql_character_set': 'utf8mb4'}
 
     component_id = Column(Integer, primary_key=True, nullable=False, unique=True)
+    component_id_parent = Column(Integer, ForeignKey('components.component_id'))
     app_id = Column(Text)
     fetch_cnt = Column(Integer, default=0)
     review_cnt = Column(Integer, default=1)
 
-    reviews = relationship('Review', back_populates='component')
+    reviews = relationship('Review',
+                           back_populates='component',
+                           cascade='all,delete-orphan')
+    parent = relationship('Component',
+                          uselist=False,
+                          remote_side='Component.component_id',
+                          backref='children',
+                          lazy='joined')
 
     def __init__(self, app_id):
         self.app_id = app_id
         self.fetch_cnt = 0
         self.review_cnt = 1
 
+    def adopt(self, child):
+
+        # set the child parent
+        child.component_id_parent = self.component_id
+
+        # adopt any of the childs existing children
+        adopted = 0
+        for component in child.children:
+            component.component_id_parent = self.component_id
+            adopted += 1
+        return adopted
+
     def __repr__(self):
         return 'Component object %s' % self.component_id
 
@@ -179,7 +199,9 @@ class Review(db.Model):
     reported = Column(Integer, default=0)
 
     user = relationship('User', back_populates='reviews')
-    component = relationship('Component', back_populates='reviews', lazy='joined')
+    component = relationship('Component',   # the one used for submit()
+                             back_populates='reviews',
+                             lazy='joined')
     votes = relationship('Vote',
                          back_populates='review',
                          cascade='all,delete-orphan')
@@ -197,6 +219,12 @@ class Review(db.Model):
         self.rating = 0
         self.reported = 0
 
+    @property
+    def component_parent(self):
+        if self.component.parent:
+            return self.component.parent
+        return self.component
+
     def _generate_keywords(self):
 
         # tokenize anything the user can specify
diff --git a/app_data/odrs/templates/components.html b/app_data/odrs/templates/components.html
index 2dea5bd..b88a3fd 100644
--- a/app_data/odrs/templates/components.html
+++ b/app_data/odrs/templates/components.html
@@ -10,20 +10,58 @@
   There are no components stored.
 </p>
 {% else %}
+<form method="post" action="{{url_for('admin_component_join2')}}">
 <table class="table table-hover table-responsive">
   <tr class="row">
-    <th class="col-sm-1">AppStream ID</th>
-    <th class="col-sm-2">Review Count</th>
-    <th class="col-sm-2">Fetch Count</th>
+    <th class="col-sm-4">AppStream ID</th>
+    <th class="col-sm-3">Parent</th>
+    <th class="col-sm-1">Reviews</th>
+    <th class="col-sm-1">Fetches</th>
+    <th class="col-sm-1">
+      <button type="submit" class="btn btn-primary" class="submit">Join</button>
+    </th>
+    <th class="col-sm-2">&nbsp;</th>
   </tr>
 {% for component in components %}
   <tr class="row">
-    <td>{{component.app_id}}</td>
+    <td>
+{% if component.parent %}
+      <span class="text-muted">{{component.app_id}}</span>
+{% else %}
+      {{component.app_id}}
+{% endif %}
+    </td>
+    <td>
+{% if component.parent %}
+      {{component.parent.app_id}}
+{% else %}
+      None
+{% endif %}
+    </td>
     <td>{{component.review_cnt}}</td>
     <td>{{component.fetch_cnt}}</td>
+    <td>
+      P: <input type="radio" name="parent" value="{{component.app_id}}">
+      C: <input type="checkbox" name="child" value="{{component.app_id}}">
+    </td>
+    <td>
+      <a class="btn btn-danger btn-block"
+         href="{{url_for('.admin_component_delete', component_id=component.component_id)}}">
+         Delete {{component.reviews|length}} reviews</a>
+    </td>
   </tr>
 {% endfor %}
 </table>
+</form>
+
+<div class="alert alert-info" role="alert">
+  <strong>Tip!</strong> You can also remove duplicates using:
+  <code>{{url_for('.admin_component_join',
+                  component_id_parent='parent',
+                  component_id_child='child',
+                  _external=True)}}</code>
+</div>
+
 {% endif %}
 
 {% endblock %}
diff --git a/app_data/odrs/tests/odrs_test.py b/app_data/odrs/tests/odrs_test.py
index a595265..d28a6b3 100644
--- a/app_data/odrs/tests/odrs_test.py
+++ b/app_data/odrs/tests/odrs_test.py
@@ -219,6 +219,52 @@ class OdrsTest(unittest.TestCase):
         data = {'locale': locale, 'value': value, 'description': description, 'severity': severity}
         return self.app.post('/admin/taboo/add', data=data, follow_redirects=True)
 
+    def test_admin_components(self):
+
+        self.review_submit()
+        self.review_submit(app_id='inkscape-ubuntu-lts.desktop')
+        self.login()
+        rv = self.app.get('/admin/component/all')
+        assert b'inkscape.desktop' in rv.data, rv.data
+        assert b'inkscape-ubuntu-lts.desktop' in rv.data, rv.data
+
+        rv = self.app.get('/admin/component/join/notgoingtoexist.desktop/inkscape-ubuntu-lts.desktop', 
follow_redirects=True)
+        assert b'No parent component found' in rv.data, rv.data
+        rv = self.app.get('/admin/component/join/inkscape.desktop/notgoingtoexist.desktop', 
follow_redirects=True)
+        assert b'No child component found' in rv.data, rv.data
+        rv = self.app.get('/admin/component/join/inkscape.desktop/inkscape.desktop', follow_redirects=True)
+        assert b'Parent and child components were the same' in rv.data, rv.data
+        rv = self.app.get('/admin/component/join/inkscape.desktop/inkscape-ubuntu-lts.desktop', 
follow_redirects=True)
+        assert b'Joined components' in rv.data, rv.data
+
+        # again
+        rv = self.app.get('/admin/component/join/inkscape.desktop/inkscape-ubuntu-lts.desktop', 
follow_redirects=True)
+        assert b'Parent and child already set up' in rv.data, rv.data
+
+        # delete inkscape.desktop
+        rv = self._api_review_delete()
+        assert b'removed review #1' in rv.data, rv.data
+
+        # still match for the alternate name
+        self.review_fetch()
+
+    def test_admin_component_delete(self):
+        self.review_submit()
+        self.review_submit(app_id='inkscape-ubuntu-lts.desktop')
+        self.login()
+
+        # delete one, causing the review to get deleted too
+        rv = self.app.get('/admin/component/delete/99999', follow_redirects=True)
+        assert b'Unable to find component' in rv.data, rv.data
+        rv = self.app.get('/admin/component/delete/2', follow_redirects=True)
+        assert b'Deleted component with 1 reviews' in rv.data, rv.data
+
+        # still match for the alternate name
+        self.review_fetch()
+
+        rv = self.app.get('/admin/component/delete/1', follow_redirects=True)
+        assert b'Deleted component with 1 reviews' in rv.data, rv.data
+
     def test_admin_taboo(self):
 
         self.login()
diff --git a/app_data/odrs/views_admin.py b/app_data/odrs/views_admin.py
index 902982f..9589e5b 100644
--- a/app_data/odrs/views_admin.py
+++ b/app_data/odrs/views_admin.py
@@ -715,6 +715,106 @@ def admin_component_show_all():
                 order_by(Component.review_cnt.asc()).all()
     return render_template('components.html', components=components)
 
+@app.route('/admin/component/join/<component_id_parent>/<component_id_child>')
+@login_required
+def admin_component_join(component_id_parent, component_id_child):
+    """
+    Join components.
+    """
+    # security check
+    if not current_user.is_admin:
+        flash('Unable to join components', 'error')
+        return redirect(url_for('.odrs_index'))
+    parent = db.session.query(Component).filter(Component.app_id == component_id_parent).first()
+    if not parent:
+        flash('No parent component found', 'warning')
+        return redirect(url_for('.admin_component_show_all'))
+    child = db.session.query(Component).filter(Component.app_id == component_id_child).first()
+    if not child:
+        flash('No child component found', 'warning')
+        return redirect(url_for('.admin_component_show_all'))
+    if parent.component_id == child.component_id:
+        flash('Parent and child components were the same', 'warning')
+        return redirect(url_for('.admin_component_show_all'))
+    if parent.component_id == child.component_id_parent:
+        flash('Parent and child already set up', 'warning')
+        return redirect(url_for('.admin_component_show_all'))
+
+    # return best message
+    adopted = parent.adopt(child)
+    db.session.commit()
+    if adopted:
+        flash('Joined components, adopting {} additional components'.format(adopted), 'info')
+    else:
+        flash('Joined components', 'info')
+    return redirect(url_for('.admin_component_show_all'))
+
+
+@app.route('/admin/component/join', methods=['POST'])
+@login_required
+def admin_component_join2():
+    """ Change details about the any user """
+
+    # security check
+    if not current_user.is_admin:
+        flash('Unable to join components', 'error')
+        return redirect(url_for('.odrs_index'))
+
+    # set each thing in turn
+    parent = None
+    children = []
+    for key in request.form:
+        if key == 'parent':
+            parent = db.session.query(Component).\
+                        filter(Component.app_id == request.form[key]).first()
+        if key == 'child':
+            for component_id in request.form.getlist(key):
+                child = db.session.query(Component).\
+                            filter(Component.app_id == component_id).first()
+                if child:
+                    children.append(child)
+    if not parent:
+        flash('No parent component found', 'warning')
+        return redirect(url_for('.admin_component_show_all'))
+    if not children:
+        flash('No child components found', 'warning')
+        return redirect(url_for('.admin_component_show_all'))
+
+    # adopt each child
+    adopted = 0
+    for child in children:
+        if parent.component_id == child.component_id:
+            child.component_id_parent = None
+            continue
+        adopted += parent.adopt(child)
+    db.session.commit()
+    if adopted:
+        flash('Joined {} components, '
+              'adopting {} additional components'.format(len(children),
+                                                         adopted), 'info')
+    else:
+        flash('Joined {} components'.format(len(children)), 'info')
+    return redirect(url_for('.admin_component_show_all'))
+
+@app.route('/admin/component/delete/<int:component_id>')
+@login_required
+def admin_component_delete(component_id):
+    """
+    Delete component, and any reviews.
+    """
+    if not current_user.is_admin:
+        flash('Unable to delete component', 'error')
+        return redirect(url_for('.odrs_index'))
+    component = db.session.query(Component).filter(Component.component_id == component_id).first()
+    if not component:
+        flash('Unable to find component', 'error')
+        return redirect(url_for('.admin_component_show_all'))
+
+    flash('Deleted component with {} reviews'.format(len(component.reviews)), 'info')
+    db.session.delete(component)
+    db.session.commit()
+    return redirect(url_for('.admin_component_show_all'))
+
 @app.route('/admin/vote/<review_id>/<val_str>')
 @login_required
 def admin_vote(review_id, val_str):
diff --git a/app_data/odrs/views_api.py b/app_data/odrs/views_api.py
index 79ff4d2..dd82590 100644
--- a/app_data/odrs/views_api.py
+++ b/app_data/odrs/views_api.py
@@ -223,6 +223,13 @@ def api_fetch():
     app_ids = [item['app_id']]
     if 'compat_ids' in item:
         app_ids.extend(item['compat_ids'])
+    if component:
+        if component.parent:
+            if component.parent.app_id not in app_ids:
+                app_ids.append(component.parent.app_id)
+        for child in component.children:
+            if child.app_id not in app_ids:
+                app_ids.append(child.app_id)
     reviews = db.session.query(Review).\
                     join(Component).\
                     filter(Component.app_id.in_(app_ids)).\
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]