[gnome-software/wip/rancell/reviews: 20/33] Set ratings from Ubuntu review stats. Disable local ratings plugin on Ubuntu



commit c01830bb971d78bda0d7be1f42629c772e55608b
Author: Robert Ancell <robert ancell canonical com>
Date:   Fri Jan 15 15:00:58 2016 +1300

    Set ratings from Ubuntu review stats. Disable local ratings plugin on Ubuntu

 src/plugins/Makefile.am                |    2 +-
 src/plugins/gs-plugin-local-ratings.c  |    8 +
 src/plugins/gs-plugin-ubuntu-reviews.c |  469 ++++++++++++++++++++++++++++++--
 3 files changed, 459 insertions(+), 20 deletions(-)
---
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 874a7a3..39d3548 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -169,7 +169,7 @@ libgs_plugin_local_ratings_la_LDFLAGS = -module -avoid-version
 libgs_plugin_local_ratings_la_CFLAGS = $(GS_PLUGIN_CFLAGS) $(WARN_CFLAGS)
 
 libgs_plugin_ubuntu_reviews_la_SOURCES = gs-plugin-ubuntu-reviews.c
-libgs_plugin_ubuntu_reviews_la_LIBADD = $(GS_PLUGIN_LIBS) $(SOUP_LIBS) $(SQLITE_LIBS) $(JSON_GLIB_LIBS) 
$(OAUTH_LIBS)
+libgs_plugin_ubuntu_reviews_la_LIBADD = $(GS_PLUGIN_LIBS) $(SOUP_LIBS) $(JSON_GLIB_LIBS) $(OAUTH_LIBS) 
$(SQLITE_LIBS)
 libgs_plugin_ubuntu_reviews_la_LDFLAGS = -module -avoid-version
 libgs_plugin_ubuntu_reviews_la_CFLAGS = $(GS_PLUGIN_CFLAGS) $(WARNINGFLAGS_C)
 
diff --git a/src/plugins/gs-plugin-local-ratings.c b/src/plugins/gs-plugin-local-ratings.c
index 67cc1fb..dd1bd86 100644
--- a/src/plugins/gs-plugin-local-ratings.c
+++ b/src/plugins/gs-plugin-local-ratings.c
@@ -61,6 +61,14 @@ gs_plugin_initialize (GsPlugin *plugin)
 
        /* create private area */
        plugin->priv = GS_PLUGIN_GET_PRIVATE (GsPluginPrivate);
+
+       /* Don't run on Ubuntu - it has it's own review plugin */
+       if (gs_plugin_check_distro_id (plugin, "ubuntu")) {
+               gs_plugin_set_enabled (plugin, FALSE);
+               g_debug ("disabling '%s' as we're on Ubuntu", plugin->name);
+               return;
+       }
+
        plugin->priv->db_path = g_build_filename (g_get_user_data_dir (),
                                                  "gnome-software",
                                                  "hardcoded-ratings.db",
diff --git a/src/plugins/gs-plugin-ubuntu-reviews.c b/src/plugins/gs-plugin-ubuntu-reviews.c
index 4554201..c4c430a 100644
--- a/src/plugins/gs-plugin-ubuntu-reviews.c
+++ b/src/plugins/gs-plugin-ubuntu-reviews.c
@@ -25,14 +25,26 @@
 #include <libsoup/soup.h>
 #include <json-glib/json-glib.h>
 #include <oauth.h>
+#include <sqlite3.h>
 
 #include <gs-plugin.h>
 #include <gs-utils.h>
 
 struct GsPluginPrivate {
+       gchar                   *db_path;
+       sqlite3                 *db;
+       gsize                    db_loaded;
        SoupSession             *session;
 };
 
+typedef struct {
+       gint64           one_star_count;
+       gint64           two_star_count;
+       gint64           three_star_count;
+       gint64           four_star_count;
+       gint64           five_star_count;
+} Histogram;
+
 const gchar *
 gs_plugin_get_name (void)
 {
@@ -41,6 +53,10 @@ gs_plugin_get_name (void)
 
 #define UBUNTU_REVIEWS_SERVER          "https://reviews.ubuntu.com/reviews";
 
+/* Download new stats every three months */
+// FIXME: Much shorter time?
+#define REVIEW_STATS_AGE_MAX           (60 * 60 * 24 * 7 * 4 * 3)
+
 void
 gs_plugin_initialize (GsPlugin *plugin)
 {
@@ -53,6 +69,11 @@ gs_plugin_initialize (GsPlugin *plugin)
                g_debug ("disabling '%s' as we're not Ubuntu", plugin->name);
                return;
        }
+
+       plugin->priv->db_path = g_build_filename (g_get_user_data_dir (),
+                                                 "gnome-software",
+                                                 "ubuntu-reviews.db",
+                                                 NULL);
 }
 
 const gchar **
@@ -65,6 +86,8 @@ gs_plugin_get_deps (GsPlugin *plugin)
 void
 gs_plugin_destroy (GsPlugin *plugin)
 {
+       if (plugin->priv->db != NULL)
+               sqlite3_close (plugin->priv->db);
        if (plugin->priv->session != NULL)
                g_object_unref (plugin->priv->session);
 }
@@ -91,6 +114,344 @@ setup_networking (GsPlugin *plugin, GError **error)
        return TRUE;
 }
 
+static gint
+get_timestamp_sqlite_cb (void *data, gint argc,
+                        gchar **argv, gchar **col_name)
+{
+       gint64 *timestamp = (gint64 *) data;
+       *timestamp = g_ascii_strtoll (argv[0], NULL, 10);
+       return 0;
+}
+
+static gboolean
+set_package_stats (GsPlugin *plugin,
+                  const gchar *package_name,
+                  Histogram *histogram,
+                  GError **error)
+{
+       char *error_msg = NULL;
+       gint result;
+       g_autofree gchar *statement = NULL;
+
+       statement = g_strdup_printf ("INSERT OR REPLACE INTO review_stats (package_name, "
+                                    "one_star_count, two_star_count, three_star_count, "
+                                     "four_star_count, five_star_count) "
+                                    "VALUES ('%s', '%" G_GINT64_FORMAT "', '%" G_GINT64_FORMAT"', '%" 
G_GINT64_FORMAT "', '%" G_GINT64_FORMAT "', '%" G_GINT64_FORMAT "');",
+                                    package_name, histogram->one_star_count, histogram->two_star_count,
+                                    histogram->three_star_count, histogram->four_star_count, 
histogram->five_star_count);
+       result = sqlite3_exec (plugin->priv->db, statement, NULL, NULL, &error_msg);
+       if (result != SQLITE_OK) {
+               g_set_error (error,
+                            GS_PLUGIN_ERROR,
+                            GS_PLUGIN_ERROR_FAILED,
+                            "SQL error: %s", error_msg);
+               sqlite3_free (error_msg);
+               return FALSE;
+       }
+
+       return TRUE;
+}
+
+static gboolean
+set_timestamp (GsPlugin *plugin,
+              const gchar *type,
+              GError **error)
+{
+       char *error_msg = NULL;
+       gint result;
+       g_autofree gchar *statement = NULL;
+
+       statement = g_strdup_printf ("INSERT OR REPLACE INTO timestamps (key, value) "
+                                    "VALUES ('%s', '%" G_GINT64_FORMAT "');",
+                                    type,
+                                    g_get_real_time () / G_USEC_PER_SEC);
+       result = sqlite3_exec (plugin->priv->db, statement, NULL, NULL, &error_msg);
+       if (result != SQLITE_OK) {
+               g_set_error (error,
+                            GS_PLUGIN_ERROR,
+                            GS_PLUGIN_ERROR_FAILED,
+                            "SQL error: %s", error_msg);
+               sqlite3_free (error_msg);
+               return FALSE;
+       }
+       return TRUE;
+}
+
+static gint
+get_rating_sqlite_cb (void *data,
+                     gint argc,
+                     gchar **argv,
+                     gchar **col_name)
+{
+       Histogram *histogram = (Histogram *) data;
+       histogram->one_star_count = g_ascii_strtoll (argv[0], NULL, 10);
+       histogram->two_star_count = g_ascii_strtoll (argv[1], NULL, 10);
+       histogram->three_star_count = g_ascii_strtoll (argv[2], NULL, 10);
+       histogram->four_star_count = g_ascii_strtoll (argv[3], NULL, 10);
+       histogram->five_star_count = g_ascii_strtoll (argv[4], NULL, 10);
+       return 0;
+}
+
+static gboolean
+get_rating (GsPlugin *plugin,
+           const gchar *package_name,
+           gint *rating,
+           GError **error)
+{
+       Histogram histogram = { 0, 0, 0, 0, 0 };
+       gchar *error_msg = NULL;
+       gint result, n_ratings;
+       g_autofree gchar *statement = NULL;
+
+       /* Get histogram from the database */
+       statement = g_strdup_printf ("SELECT one_star_count, two_star_count, three_star_count, 
four_star_count, five_star_count FROM review_stats "
+                                    "WHERE package_name = '%s'", package_name);
+       result = sqlite3_exec (plugin->priv->db,
+                              statement,
+                              get_rating_sqlite_cb,
+                              &histogram,
+                              &error_msg);
+       if (result != SQLITE_OK) {
+               g_set_error (error,
+                            GS_PLUGIN_ERROR,
+                            GS_PLUGIN_ERROR_FAILED,
+                            "SQL error: %s", error_msg);
+               sqlite3_free (error_msg);
+               return FALSE;
+       }
+
+       /* Convert to a rating */
+       // FIXME: Convert to a Wilson score
+       n_ratings = histogram.one_star_count + histogram.two_star_count + histogram.three_star_count + 
histogram.four_star_count + histogram.five_star_count;
+       if (n_ratings == 0)
+               *rating = -1;
+       else
+               *rating = ((histogram.one_star_count * 20) + (histogram.two_star_count * 40) + 
(histogram.three_star_count * 60) + (histogram.four_star_count * 80) + (histogram.five_star_count * 100)) / 
n_ratings;
+g_warning ("%s %zi %zi %zi %zi %zi / %d -> %d", package_name, histogram.one_star_count, 
histogram.two_star_count, histogram.three_star_count, histogram.four_star_count, histogram.five_star_count, 
n_ratings, *rating);
+
+       return TRUE;
+}
+
+static gboolean
+parse_histogram (const gchar *text, Histogram *histogram)
+{
+       JsonParser *parser = NULL;
+       JsonArray *array;
+       gboolean result = FALSE;
+
+       /* Histogram is a five element JSON array, e.g. "[1, 3, 5, 8, 4]" */
+       parser = json_parser_new ();
+       if (!json_parser_load_from_data (parser, text, -1, NULL))
+               goto out;
+       if (!JSON_NODE_HOLDS_ARRAY (json_parser_get_root (parser)))
+               goto out;
+       array = json_node_get_array (json_parser_get_root (parser));
+       if (json_array_get_length (array) != 5)
+               goto out;
+       histogram->one_star_count = json_array_get_int_element (array, 0);
+       histogram->two_star_count = json_array_get_int_element (array, 1);
+       histogram->three_star_count = json_array_get_int_element (array, 2);
+       histogram->four_star_count = json_array_get_int_element (array, 3);
+       histogram->five_star_count = json_array_get_int_element (array, 4);
+       result = TRUE;
+
+out:
+       g_clear_object (&parser);
+
+       return result;
+}
+
+static gboolean
+parse_review_entry (JsonNode *node, const gchar **package_name, Histogram *histogram)
+{
+       JsonObject *object;
+       const gchar *name = NULL, *histogram_text = NULL;
+
+       if (!JSON_NODE_HOLDS_OBJECT (node))
+               return FALSE;
+
+       object = json_node_get_object (node);
+
+       name = json_object_get_string_member (object, "package_name");
+       histogram_text = json_object_get_string_member (object, "histogram");
+       if (!name || !histogram_text)
+               return FALSE;
+
+       if (!parse_histogram (histogram_text, histogram))
+               return FALSE;
+       *package_name = name;
+
+       return TRUE;
+}
+
+static gboolean
+parse_review_entries (GsPlugin *plugin, const gchar *text, GError **error)
+{
+       JsonParser *parser = NULL;
+       JsonArray *array;
+       gint i;
+       gboolean result = FALSE;
+
+       parser = json_parser_new ();
+       if (!json_parser_load_from_data (parser, text, -1, error))
+               goto out;
+       if (!JSON_NODE_HOLDS_ARRAY (json_parser_get_root (parser)))
+               goto out;
+       array = json_node_get_array (json_parser_get_root (parser));
+       for (i = 0; i < json_array_get_length (array); i++) {
+               const gchar *package_name;
+               Histogram histogram;
+
+               /* Read in from JSON... (skip bad entries) */
+               if (!parse_review_entry (json_array_get_element (array, i), &package_name, &histogram))
+                       continue;
+
+               /* ...write into the database (abort everything if can't write) */
+               if (!set_package_stats (plugin, package_name, &histogram, error))
+                       goto out;
+       }
+       result = TRUE;
+
+out:
+       g_clear_object (&parser);
+
+       return result;
+}
+
+static gboolean
+download_review_stats (GsPlugin *plugin, GError **error)
+{
+       guint status_code;
+       g_autofree gchar *uri = NULL;
+       g_autoptr(SoupMessage) msg = NULL;
+       g_auto(GStrv) split = NULL;
+
+       /* Get the review stats using HTTP */
+       uri = g_strdup_printf ("%s/api/1.0/review-stats/any/any/",
+                              UBUNTU_REVIEWS_SERVER);
+       msg = soup_message_new (SOUP_METHOD_GET, uri);
+       if (!setup_networking (plugin, error))
+               return FALSE;
+       status_code = soup_session_send_message (plugin->priv->session, msg);
+       if (status_code != SOUP_STATUS_OK) {
+               g_set_error (error,
+                            GS_PLUGIN_ERROR,
+                            GS_PLUGIN_ERROR_FAILED,
+                            "Failed to download Ubuntu reviews dump: %s",
+                            soup_status_get_phrase (status_code));
+               return FALSE;
+       }
+
+       /* Extract the stats from the data */
+       if (!parse_review_entries (plugin, msg->response_body->data, error))
+               return FALSE;
+
+       /* Record the time we downloaded it */
+       return set_timestamp (plugin, "stats_mtime", error);
+}
+
+static gboolean
+load_database (GsPlugin *plugin, GError **error)
+{
+       const gchar *statement;
+       gboolean rebuild_ratings = FALSE;
+       char *error_msg = NULL;
+       gint result;
+       gint64 stats_mtime = 0;
+       gint64 now;
+       g_autoptr(GError) error_local = NULL;
+
+       g_debug ("trying to open database '%s'", plugin->priv->db_path);
+       if (!gs_mkdir_parent (plugin->priv->db_path, error))
+               return FALSE;
+       result = sqlite3_open (plugin->priv->db_path, &plugin->priv->db);
+       if (result != SQLITE_OK) {
+               g_set_error (error,
+                            GS_PLUGIN_ERROR,
+                            GS_PLUGIN_ERROR_FAILED,
+                            "Can't open Ubuntu review statistics database: %s",
+                            sqlite3_errmsg (plugin->priv->db));
+               return FALSE;
+       }
+
+       /* We don't need to keep doing fsync */
+       sqlite3_exec (plugin->priv->db, "PRAGMA synchronous=OFF",
+                     NULL, NULL, NULL);
+
+       /* Create a table to store the stats */
+       result = sqlite3_exec (plugin->priv->db, "SELECT * FROM review_stats LIMIT 1", NULL, NULL, 
&error_msg);
+       if (result != SQLITE_OK) {
+               g_debug ("creating table to repair: %s", error_msg);
+               sqlite3_free (error_msg);
+               statement = "CREATE TABLE review_stats ("
+                           "package_name TEXT PRIMARY KEY,"
+                           "one_star_count INTEGER DEFAULT 0,"
+                           "two_star_count INTEGER DEFAULT 0,"
+                           "three_star_count INTEGER DEFAULT 0,"
+                           "four_star_count INTEGER DEFAULT 0,"
+                           "five_star_count INTEGER DEFAULT 0);";
+               sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
+               rebuild_ratings = TRUE;
+       }
+
+       /* Create a table to store local reviews */
+       result = sqlite3_exec (plugin->priv->db, "SELECT * FROM reviews LIMIT 1", NULL, NULL, &error_msg);
+       if (result != SQLITE_OK) {
+               g_debug ("creating table to repair: %s", error_msg);
+               sqlite3_free (error_msg);
+               statement = "CREATE TABLE reviews ("
+                           "package_name TEXT PRIMARY KEY,"
+                           "id TEXT,"
+                           "version TEXT,"
+                           "date TEXT,"
+                           "rating INTEGER,"
+                            "summary TEXT,"
+                            "text TEXT);";
+               sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
+               rebuild_ratings = TRUE;
+       }
+
+       /* Create a table to store timestamps */
+       result = sqlite3_exec (plugin->priv->db,
+                              "SELECT value FROM timestamps WHERE key = 'stats_mtime' LIMIT 1",
+                              get_timestamp_sqlite_cb, &stats_mtime,
+                              &error_msg);
+       if (result != SQLITE_OK) {
+               g_debug ("creating table to repair: %s", error_msg);
+               sqlite3_free (error_msg);
+               statement = "CREATE TABLE timestamps ("
+                           "key TEXT PRIMARY KEY,"
+                           "value INTEGER DEFAULT 0);";
+               sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
+
+               /* Set the time of database creation */
+               if (!set_timestamp (plugin, "stats_ctime", error))
+                       return FALSE;
+       }
+
+       /* Download data if we have none or it is out of date */
+       now = g_get_real_time () / G_USEC_PER_SEC;
+       if (stats_mtime == 0 || rebuild_ratings) {
+               g_debug ("No Ubuntu review statistics");
+               if (!download_review_stats (plugin, &error_local)) {
+                       g_warning ("Failed to get Ubuntu review statistics: %s",
+                                  error_local->message);
+                       return TRUE;
+               }
+       } else if (now - stats_mtime > REVIEW_STATS_AGE_MAX) {
+               g_debug ("Ubuntu review statistics was %" G_GINT64_FORMAT
+                        " days old, so regetting",
+                        (now - stats_mtime) / ( 60 * 60 * 24));
+               if (!download_review_stats (plugin, error))
+                       return FALSE;
+       } else {
+               g_debug ("Ubuntu review statistics %" G_GINT64_FORMAT
+                        " days old, so no need to redownload",
+                        (now - stats_mtime) / ( 60 * 60 * 24));
+       }
+       return TRUE;
+}
+
 static GDateTime *
 parse_date_time (const gchar *text)
 {
@@ -213,6 +574,74 @@ download_reviews (GsPlugin *plugin, GsApp *app, const gchar *package_name, GErro
        return TRUE;
 }
 
+static gboolean
+refine_rating (GsPlugin *plugin, GsApp *app, GError **error)
+{
+       GPtrArray *sources;
+       guint i;
+
+       /* Load database once */
+       if (g_once_init_enter (&plugin->priv->db_loaded)) {
+               gboolean ret = load_database (plugin, error);
+               g_once_init_leave (&plugin->priv->db_loaded, TRUE);
+               if (!ret)
+                       return FALSE;
+       }
+
+       /* Skip if already has a rating */
+       if (gs_app_get_rating (app) != -1)
+               return TRUE;
+
+       sources = gs_app_get_sources (app);
+       for (i = 0; i < sources->len; i++) {
+               const gchar *package_name;
+               gint rating;
+               gboolean ret;
+
+               /* If we have a local review, use that as the rating */
+               // FIXME
+
+               /* Otherwise use the statistics */
+               package_name = g_ptr_array_index (sources, i);
+               ret = get_rating (plugin, package_name, &rating, error);
+               if (!ret)
+                       return FALSE;
+               if (rating != -1) {
+                       g_debug ("ubuntu-reviews setting rating on %s to %i%%",
+                                package_name, rating);
+                       gs_app_set_rating (app, rating);
+                       if (rating > 80)
+                               gs_app_add_kudo (app, GS_APP_KUDO_POPULAR);
+               }
+       }
+
+       return TRUE;
+}
+
+static gboolean
+refine_reviews (GsPlugin *plugin, GsApp *app, GError **error)
+{
+       GPtrArray *sources;
+       guint i;
+
+       /* Skip if already has reviews */
+       if (gs_app_get_reviews (app)->len > 0)
+               return TRUE;
+
+       sources = gs_app_get_sources (app);
+       for (i = 0; i < sources->len; i++) {
+               const gchar *package_name;
+               gboolean ret;
+
+               package_name = g_ptr_array_index (sources, i);
+               ret = download_reviews (plugin, app, package_name, error);
+               if (!ret)
+                       return FALSE;
+       }
+
+       return TRUE;
+}
+
 gboolean
 gs_plugin_refine (GsPlugin *plugin,
                  GList **list,
@@ -221,33 +650,22 @@ gs_plugin_refine (GsPlugin *plugin,
                  GError **error)
 {
        GList *l;
-
-       /* We only update reviews */
-       if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_REVIEWS) == 0)
-               return TRUE;
+       gboolean ret = TRUE;
 
        for (l = *list; l != NULL; l = l->next) {
                GsApp *app = GS_APP (l->data);
-               GPtrArray *reviews, *sources;
-               guint i;
-
-               reviews = gs_app_get_reviews (app);
-               if (reviews->len > 0)
-                       continue;
 
-               sources = gs_app_get_sources (app);
-               for (i = 0; i < sources->len; i++) {
-                       const gchar *package_name;
-                       gboolean ret;
-
-                       package_name = g_ptr_array_index (sources, i);
-                       ret = download_reviews (plugin, app, package_name, error);
-                       if (!ret)
+               if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_RATING) != 0) {
+                       if (!refine_rating (plugin, app, error))
+                               return FALSE;
+               }
+               if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_REVIEWS) != 0) {
+                       if (!refine_reviews (plugin, app, error))
                                return FALSE;
                }
        }
 
-       return TRUE;
+       return ret;
 }
 
 static void
@@ -338,6 +756,9 @@ set_package_review (GsPlugin *plugin,
        else
                n_stars = 1;
 
+       /* Write review into database so we can easily access it */
+       // FIXME
+
        /* Load OAuth token */
        // FIXME needs to integrate with GNOME Online Accounts / libaccounts
        config = g_key_file_new ();
@@ -355,6 +776,7 @@ set_package_review (GsPlugin *plugin,
                return FALSE;
        }
 
+       /* Create message for reviews.ubuntu.com */
        uri = g_strdup_printf ("%s/api/1.0/reviews/", UBUNTU_REVIEWS_SERVER);
        msg = soup_message_new (SOUP_METHOD_POST, uri);
        builder = json_builder_new ();
@@ -373,6 +795,7 @@ set_package_review (GsPlugin *plugin,
        g_object_unref (builder);
        sign_message (msg, OA_HMAC, oauth_consumer_key, oauth_consumer_secret, oauth_token, 
oauth_token_secret);
 
+       /* Send to the server */
        status_code = soup_session_send_message (plugin->priv->session, msg);
        if (status_code != SOUP_STATUS_OK) {
                g_set_error (error,
@@ -402,6 +825,14 @@ gs_plugin_app_set_review (GsPlugin *plugin,
        review = gs_app_get_self_review (app);
        g_return_val_if_fail (review != NULL, FALSE);
 
+       /* Load database once */
+       if (g_once_init_enter (&plugin->priv->db_loaded)) {
+               gboolean ret = load_database (plugin, error);
+               g_once_init_leave (&plugin->priv->db_loaded, TRUE);
+               if (!ret)
+                       return FALSE;
+       }
+
        /* get the package name */
        sources = gs_app_get_sources (app);
        if (sources->len == 0) {


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]