[gnome-software/wip/rancell/reviews-3-18] Set ratings from Ubuntu review stats. Disable local ratings plugin on Ubuntu
- From: Robert Ancell <rancell src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-software/wip/rancell/reviews-3-18] Set ratings from Ubuntu review stats. Disable local ratings plugin on Ubuntu
- Date: Fri, 15 Jan 2016 02:01:10 +0000 (UTC)
commit 208a82e78dcf23c14470bab250bd6c35c28f581e
Author: Robert Ancell <robert ancell canonical com>
Date: Fri Jan 15 15:00:58 2016 +1300
Set ratings from Ubuntu review stats. Disable local ratings plugin on Ubuntu
src/plugins/Makefile.am | 2 +-
src/plugins/gs-plugin-local-ratings.c | 8 +
src/plugins/gs-plugin-ubuntu-reviews.c | 471 ++++++++++++++++++++++++++++++--
3 files changed, 461 insertions(+), 20 deletions(-)
---
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index fb4bf67..cf21b8a 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -134,7 +134,7 @@ libgs_plugin_local_ratings_la_LDFLAGS = -module -avoid-version
libgs_plugin_local_ratings_la_CFLAGS = $(GS_PLUGIN_CFLAGS) $(WARN_CFLAGS)
libgs_plugin_ubuntu_reviews_la_SOURCES = gs-plugin-ubuntu-reviews.c
-libgs_plugin_ubuntu_reviews_la_LIBADD = $(GS_PLUGIN_LIBS) $(SOUP_LIBS) $(SQLITE_LIBS) $(JSON_GLIB_LIBS)
$(OAUTH_LIBS)
+libgs_plugin_ubuntu_reviews_la_LIBADD = $(GS_PLUGIN_LIBS) $(SOUP_LIBS) $(JSON_GLIB_LIBS) $(OAUTH_LIBS)
$(SQLITE_LIBS)
libgs_plugin_ubuntu_reviews_la_LDFLAGS = -module -avoid-version
libgs_plugin_ubuntu_reviews_la_CFLAGS = $(GS_PLUGIN_CFLAGS) $(WARNINGFLAGS_C)
diff --git a/src/plugins/gs-plugin-local-ratings.c b/src/plugins/gs-plugin-local-ratings.c
index ef8c199..b1de63c 100644
--- a/src/plugins/gs-plugin-local-ratings.c
+++ b/src/plugins/gs-plugin-local-ratings.c
@@ -50,6 +50,14 @@ gs_plugin_initialize (GsPlugin *plugin)
{
/* create private area */
plugin->priv = GS_PLUGIN_GET_PRIVATE (GsPluginPrivate);
+
+ /* Don't run on Ubuntu - it has it's own review plugin */
+ if (gs_plugin_check_distro_id (plugin, "ubuntu")) {
+ gs_plugin_set_enabled (plugin, FALSE);
+ g_debug ("disabling '%s' as we're on Ubuntu", plugin->name);
+ return;
+ }
+
plugin->priv->db_path = g_build_filename (g_get_user_data_dir (),
"gnome-software",
"hardcoded-ratings.db",
diff --git a/src/plugins/gs-plugin-ubuntu-reviews.c b/src/plugins/gs-plugin-ubuntu-reviews.c
index 4554201..feda832 100644
--- a/src/plugins/gs-plugin-ubuntu-reviews.c
+++ b/src/plugins/gs-plugin-ubuntu-reviews.c
@@ -25,14 +25,26 @@
#include <libsoup/soup.h>
#include <json-glib/json-glib.h>
#include <oauth.h>
+#include <sqlite3.h>
#include <gs-plugin.h>
#include <gs-utils.h>
struct GsPluginPrivate {
+ gchar *db_path;
+ sqlite3 *db;
+ gsize db_loaded;
SoupSession *session;
};
+typedef struct {
+ gint64 one_star_count;
+ gint64 two_star_count;
+ gint64 three_star_count;
+ gint64 four_star_count;
+ gint64 five_star_count;
+} Histogram;
+
const gchar *
gs_plugin_get_name (void)
{
@@ -41,6 +53,10 @@ gs_plugin_get_name (void)
#define UBUNTU_REVIEWS_SERVER "https://reviews.ubuntu.com/reviews";
+/* Download new stats every three months */
+// FIXME: Much shorter time?
+#define REVIEW_STATS_AGE_MAX (60 * 60 * 24 * 7 * 4 * 3)
+
void
gs_plugin_initialize (GsPlugin *plugin)
{
@@ -53,6 +69,11 @@ gs_plugin_initialize (GsPlugin *plugin)
g_debug ("disabling '%s' as we're not Ubuntu", plugin->name);
return;
}
+
+ plugin->priv->db_path = g_build_filename (g_get_user_data_dir (),
+ "gnome-software",
+ "ubuntu-reviews.db",
+ NULL);
}
const gchar **
@@ -65,6 +86,8 @@ gs_plugin_get_deps (GsPlugin *plugin)
void
gs_plugin_destroy (GsPlugin *plugin)
{
+ if (plugin->priv->db != NULL)
+ sqlite3_close (plugin->priv->db);
if (plugin->priv->session != NULL)
g_object_unref (plugin->priv->session);
}
@@ -91,6 +114,344 @@ setup_networking (GsPlugin *plugin, GError **error)
return TRUE;
}
+static gint
+get_timestamp_sqlite_cb (void *data, gint argc,
+ gchar **argv, gchar **col_name)
+{
+ gint64 *timestamp = (gint64 *) data;
+ *timestamp = g_ascii_strtoll (argv[0], NULL, 10);
+ return 0;
+}
+
+static gboolean
+set_package_stats (GsPlugin *plugin,
+ const gchar *package_name,
+ Histogram *histogram,
+ GError **error)
+{
+ char *error_msg = NULL;
+ gint result;
+ g_autofree gchar *statement = NULL;
+
+ statement = g_strdup_printf ("INSERT OR REPLACE INTO review_stats (package_name, "
+ "one_star_count, two_star_count, three_star_count, "
+ "four_star_count, five_star_count) "
+ "VALUES ('%s', '%" G_GINT64_FORMAT "', '%" G_GINT64_FORMAT"', '%"
G_GINT64_FORMAT "', '%" G_GINT64_FORMAT "', '%" G_GINT64_FORMAT "');",
+ package_name, histogram->one_star_count, histogram->two_star_count,
+ histogram->three_star_count, histogram->four_star_count,
histogram->five_star_count);
+ result = sqlite3_exec (plugin->priv->db, statement, NULL, NULL, &error_msg);
+ if (result != SQLITE_OK) {
+ g_set_error (error,
+ GS_PLUGIN_ERROR,
+ GS_PLUGIN_ERROR_FAILED,
+ "SQL error: %s", error_msg);
+ sqlite3_free (error_msg);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+set_timestamp (GsPlugin *plugin,
+ const gchar *type,
+ GError **error)
+{
+ char *error_msg = NULL;
+ gint result;
+ g_autofree gchar *statement = NULL;
+
+ statement = g_strdup_printf ("INSERT OR REPLACE INTO timestamps (key, value) "
+ "VALUES ('%s', '%" G_GINT64_FORMAT "');",
+ type,
+ g_get_real_time () / G_USEC_PER_SEC);
+ result = sqlite3_exec (plugin->priv->db, statement, NULL, NULL, &error_msg);
+ if (result != SQLITE_OK) {
+ g_set_error (error,
+ GS_PLUGIN_ERROR,
+ GS_PLUGIN_ERROR_FAILED,
+ "SQL error: %s", error_msg);
+ sqlite3_free (error_msg);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static gint
+get_rating_sqlite_cb (void *data,
+ gint argc,
+ gchar **argv,
+ gchar **col_name)
+{
+ Histogram *histogram = (Histogram *) data;
+ histogram->one_star_count = g_ascii_strtoll (argv[0], NULL, 10);
+ histogram->two_star_count = g_ascii_strtoll (argv[1], NULL, 10);
+ histogram->three_star_count = g_ascii_strtoll (argv[2], NULL, 10);
+ histogram->four_star_count = g_ascii_strtoll (argv[3], NULL, 10);
+ histogram->five_star_count = g_ascii_strtoll (argv[4], NULL, 10);
+ return 0;
+}
+
+static gboolean
+get_rating (GsPlugin *plugin,
+ const gchar *package_name,
+ gint *rating,
+ GError **error)
+{
+ Histogram histogram = { 0, 0, 0, 0, 0 };
+ gchar *error_msg = NULL;
+ gint result, n_ratings;
+ g_autofree gchar *statement = NULL;
+
+ /* Get histogram from the database */
+ statement = g_strdup_printf ("SELECT one_star_count, two_star_count, three_star_count,
four_star_count, five_star_count FROM review_stats "
+ "WHERE package_name = '%s'", package_name);
+ result = sqlite3_exec (plugin->priv->db,
+ statement,
+ get_rating_sqlite_cb,
+ &histogram,
+ &error_msg);
+ if (result != SQLITE_OK) {
+ g_set_error (error,
+ GS_PLUGIN_ERROR,
+ GS_PLUGIN_ERROR_FAILED,
+ "SQL error: %s", error_msg);
+ sqlite3_free (error_msg);
+ return FALSE;
+ }
+
+ /* Convert to a rating */
+ // FIXME: Convert to a Wilson score
+ n_ratings = histogram.one_star_count + histogram.two_star_count + histogram.three_star_count +
histogram.four_star_count + histogram.five_star_count;
+ if (n_ratings == 0)
+ *rating = -1;
+ else
+ *rating = ((histogram.one_star_count * 20) + (histogram.two_star_count * 40) +
(histogram.three_star_count * 60) + (histogram.four_star_count * 80) + (histogram.five_star_count * 100)) /
n_ratings;
+g_warning ("%s %zi %zi %zi %zi %zi / %d -> %d", package_name, histogram.one_star_count,
histogram.two_star_count, histogram.three_star_count, histogram.four_star_count, histogram.five_star_count,
n_ratings, *rating);
+
+ return TRUE;
+}
+
+static gboolean
+parse_histogram (const gchar *text, Histogram *histogram)
+{
+ JsonParser *parser = NULL;
+ JsonArray *array;
+ gboolean result = FALSE;
+
+ /* Histogram is a five element JSON array, e.g. "[1, 3, 5, 8, 4]" */
+ parser = json_parser_new ();
+ if (!json_parser_load_from_data (parser, text, -1, NULL))
+ goto out;
+ if (!JSON_NODE_HOLDS_ARRAY (json_parser_get_root (parser)))
+ goto out;
+ array = json_node_get_array (json_parser_get_root (parser));
+ if (json_array_get_length (array) != 5)
+ goto out;
+ histogram->one_star_count = json_array_get_int_element (array, 0);
+ histogram->two_star_count = json_array_get_int_element (array, 1);
+ histogram->three_star_count = json_array_get_int_element (array, 2);
+ histogram->four_star_count = json_array_get_int_element (array, 3);
+ histogram->five_star_count = json_array_get_int_element (array, 4);
+ result = TRUE;
+
+out:
+ g_clear_object (&parser);
+
+ return result;
+}
+
+static gboolean
+parse_review_entry (JsonNode *node, const gchar **package_name, Histogram *histogram)
+{
+ JsonObject *object;
+ const gchar *name = NULL, *histogram_text = NULL;
+
+ if (!JSON_NODE_HOLDS_OBJECT (node))
+ return FALSE;
+
+ object = json_node_get_object (node);
+
+ name = json_object_get_string_member (object, "package_name");
+ histogram_text = json_object_get_string_member (object, "histogram");
+ if (!name || !histogram_text)
+ return FALSE;
+
+ if (!parse_histogram (histogram_text, histogram))
+ return FALSE;
+ *package_name = name;
+
+ return TRUE;
+}
+
+static gboolean
+parse_review_entries (GsPlugin *plugin, const gchar *text, GError **error)
+{
+ JsonParser *parser = NULL;
+ JsonArray *array;
+ gint i;
+ gboolean result = FALSE;
+
+ parser = json_parser_new ();
+ if (!json_parser_load_from_data (parser, text, -1, error))
+ goto out;
+ if (!JSON_NODE_HOLDS_ARRAY (json_parser_get_root (parser)))
+ goto out;
+ array = json_node_get_array (json_parser_get_root (parser));
+ for (i = 0; i < json_array_get_length (array); i++) {
+ const gchar *package_name;
+ Histogram histogram;
+
+ /* Read in from JSON... (skip bad entries) */
+ if (!parse_review_entry (json_array_get_element (array, i), &package_name, &histogram))
+ continue;
+
+ /* ...write into the database (abort everything if can't write) */
+ if (!set_package_stats (plugin, package_name, &histogram, error))
+ goto out;
+ }
+ result = TRUE;
+
+out:
+ g_clear_object (&parser);
+
+ return result;
+}
+
+static gboolean
+download_review_stats (GsPlugin *plugin, GError **error)
+{
+ guint status_code;
+ g_autofree gchar *uri = NULL;
+ g_autoptr(SoupMessage) msg = NULL;
+ g_auto(GStrv) split = NULL;
+
+ /* Get the review stats using HTTP */
+ uri = g_strdup_printf ("%s/api/1.0/review-stats/any/any/",
+ UBUNTU_REVIEWS_SERVER);
+ msg = soup_message_new (SOUP_METHOD_GET, uri);
+ if (!setup_networking (plugin, error))
+ return FALSE;
+ status_code = soup_session_send_message (plugin->priv->session, msg);
+ if (status_code != SOUP_STATUS_OK) {
+ g_set_error (error,
+ GS_PLUGIN_ERROR,
+ GS_PLUGIN_ERROR_FAILED,
+ "Failed to download Ubuntu reviews dump: %s",
+ soup_status_get_phrase (status_code));
+ return FALSE;
+ }
+
+ /* Extract the stats from the data */
+ if (!parse_review_entries (plugin, msg->response_body->data, error))
+ return FALSE;
+
+ /* Record the time we downloaded it */
+ return set_timestamp (plugin, "stats_mtime", error);
+}
+
+static gboolean
+load_database (GsPlugin *plugin, GError **error)
+{
+ const gchar *statement;
+ gboolean rebuild_ratings = FALSE;
+ char *error_msg = NULL;
+ gint result;
+ gint64 stats_mtime = 0;
+ gint64 now;
+ g_autoptr(GError) error_local = NULL;
+
+ g_debug ("trying to open database '%s'", plugin->priv->db_path);
+ if (!gs_mkdir_parent (plugin->priv->db_path, error))
+ return FALSE;
+ result = sqlite3_open (plugin->priv->db_path, &plugin->priv->db);
+ if (result != SQLITE_OK) {
+ g_set_error (error,
+ GS_PLUGIN_ERROR,
+ GS_PLUGIN_ERROR_FAILED,
+ "Can't open Ubuntu review statistics database: %s",
+ sqlite3_errmsg (plugin->priv->db));
+ return FALSE;
+ }
+
+ /* We don't need to keep doing fsync */
+ sqlite3_exec (plugin->priv->db, "PRAGMA synchronous=OFF",
+ NULL, NULL, NULL);
+
+ /* Create a table to store the stats */
+ result = sqlite3_exec (plugin->priv->db, "SELECT * FROM review_stats LIMIT 1", NULL, NULL,
&error_msg);
+ if (result != SQLITE_OK) {
+ g_debug ("creating table to repair: %s", error_msg);
+ sqlite3_free (error_msg);
+ statement = "CREATE TABLE review_stats ("
+ "package_name TEXT PRIMARY KEY,"
+ "one_star_count INTEGER DEFAULT 0,"
+ "two_star_count INTEGER DEFAULT 0,"
+ "three_star_count INTEGER DEFAULT 0,"
+ "four_star_count INTEGER DEFAULT 0,"
+ "five_star_count INTEGER DEFAULT 0);";
+ sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
+ rebuild_ratings = TRUE;
+ }
+
+ /* Create a table to store local reviews */
+ result = sqlite3_exec (plugin->priv->db, "SELECT * FROM reviews LIMIT 1", NULL, NULL, &error_msg);
+ if (result != SQLITE_OK) {
+ g_debug ("creating table to repair: %s", error_msg);
+ sqlite3_free (error_msg);
+ statement = "CREATE TABLE reviews ("
+ "package_name TEXT PRIMARY KEY,"
+ "id TEXT,"
+ "version TEXT,"
+ "date TEXT,"
+ "rating INTEGER,"
+ "summary TEXT,"
+ "text TEXT);";
+ sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
+ rebuild_ratings = TRUE;
+ }
+
+ /* Create a table to store timestamps */
+ result = sqlite3_exec (plugin->priv->db,
+ "SELECT value FROM timestamps WHERE key = 'stats_mtime' LIMIT 1",
+ get_timestamp_sqlite_cb, &stats_mtime,
+ &error_msg);
+ if (result != SQLITE_OK) {
+ g_debug ("creating table to repair: %s", error_msg);
+ sqlite3_free (error_msg);
+ statement = "CREATE TABLE timestamps ("
+ "key TEXT PRIMARY KEY,"
+ "value INTEGER DEFAULT 0);";
+ sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
+
+ /* Set the time of database creation */
+ if (!set_timestamp (plugin, "stats_ctime", error))
+ return FALSE;
+ }
+
+ /* Download data if we have none or it is out of date */
+ now = g_get_real_time () / G_USEC_PER_SEC;
+ if (stats_mtime == 0 || rebuild_ratings) {
+ g_debug ("No Ubuntu review statistics");
+ if (!download_review_stats (plugin, &error_local)) {
+ g_warning ("Failed to get Ubuntu review statistics: %s",
+ error_local->message);
+ return TRUE;
+ }
+ } else if (now - stats_mtime > REVIEW_STATS_AGE_MAX) {
+ g_debug ("Ubuntu review statistics was %" G_GINT64_FORMAT
+ " days old, so regetting",
+ (now - stats_mtime) / ( 60 * 60 * 24));
+ if (!download_review_stats (plugin, error))
+ return FALSE;
+ } else {
+ g_debug ("Ubuntu review statistics %" G_GINT64_FORMAT
+ " days old, so no need to redownload",
+ (now - stats_mtime) / ( 60 * 60 * 24));
+ }
+ return TRUE;
+}
+
static GDateTime *
parse_date_time (const gchar *text)
{
@@ -213,6 +574,76 @@ download_reviews (GsPlugin *plugin, GsApp *app, const gchar *package_name, GErro
return TRUE;
}
+static gboolean
+refine_rating (GsPlugin *plugin, GsApp *app, GError **error)
+{
+ GPtrArray *sources;
+ guint i;
+
+ /* Load database once */
+ if (g_once_init_enter (&plugin->priv->db_loaded)) {
+ gboolean ret = load_database (plugin, error);
+ g_once_init_leave (&plugin->priv->db_loaded, TRUE);
+ if (!ret)
+ return FALSE;
+ }
+
+ /* Skip if already has a rating */
+ if (gs_app_get_rating (app) != -1)
+ return TRUE;
+
+ sources = gs_app_get_sources (app);
+ for (i = 0; i < sources->len; i++) {
+ const gchar *package_name;
+ gint rating;
+ gboolean ret;
+
+ /* If we have a local review, use that as the rating */
+ // FIXME
+
+ /* Otherwise use the statistics */
+ package_name = g_ptr_array_index (sources, i);
+ ret = get_rating (plugin, package_name, &rating, error);
+ if (!ret)
+ return FALSE;
+ if (rating != -1) {
+ g_debug ("ubuntu-reviews setting rating on %s to %i%%",
+ package_name, rating);
+ gs_app_set_rating (app, rating);
+ gs_app_set_rating_confidence (app, 100);
+ gs_app_set_rating_kind (app, GS_APP_RATING_KIND_SYSTEM);
+ if (rating > 80)
+ gs_app_add_kudo (app, GS_APP_KUDO_POPULAR);
+ }
+ }
+
+ return TRUE;
+}
+
+static gboolean
+refine_reviews (GsPlugin *plugin, GsApp *app, GError **error)
+{
+ GPtrArray *sources;
+ guint i;
+
+ /* Skip if already has reviews */
+ if (gs_app_get_reviews (app)->len > 0)
+ return TRUE;
+
+ sources = gs_app_get_sources (app);
+ for (i = 0; i < sources->len; i++) {
+ const gchar *package_name;
+ gboolean ret;
+
+ package_name = g_ptr_array_index (sources, i);
+ ret = download_reviews (plugin, app, package_name, error);
+ if (!ret)
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
gboolean
gs_plugin_refine (GsPlugin *plugin,
GList **list,
@@ -221,33 +652,22 @@ gs_plugin_refine (GsPlugin *plugin,
GError **error)
{
GList *l;
-
- /* We only update reviews */
- if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_REVIEWS) == 0)
- return TRUE;
+ gboolean ret = TRUE;
for (l = *list; l != NULL; l = l->next) {
GsApp *app = GS_APP (l->data);
- GPtrArray *reviews, *sources;
- guint i;
-
- reviews = gs_app_get_reviews (app);
- if (reviews->len > 0)
- continue;
- sources = gs_app_get_sources (app);
- for (i = 0; i < sources->len; i++) {
- const gchar *package_name;
- gboolean ret;
-
- package_name = g_ptr_array_index (sources, i);
- ret = download_reviews (plugin, app, package_name, error);
- if (!ret)
+ if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_RATING) != 0) {
+ if (!refine_rating (plugin, app, error))
+ return FALSE;
+ }
+ if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_REVIEWS) != 0) {
+ if (!refine_reviews (plugin, app, error))
return FALSE;
}
}
- return TRUE;
+ return ret;
}
static void
@@ -338,6 +758,9 @@ set_package_review (GsPlugin *plugin,
else
n_stars = 1;
+ /* Write review into database so we can easily access it */
+ // FIXME
+
/* Load OAuth token */
// FIXME needs to integrate with GNOME Online Accounts / libaccounts
config = g_key_file_new ();
@@ -355,6 +778,7 @@ set_package_review (GsPlugin *plugin,
return FALSE;
}
+ /* Create message for reviews.ubuntu.com */
uri = g_strdup_printf ("%s/api/1.0/reviews/", UBUNTU_REVIEWS_SERVER);
msg = soup_message_new (SOUP_METHOD_POST, uri);
builder = json_builder_new ();
@@ -373,6 +797,7 @@ set_package_review (GsPlugin *plugin,
g_object_unref (builder);
sign_message (msg, OA_HMAC, oauth_consumer_key, oauth_consumer_secret, oauth_token,
oauth_token_secret);
+ /* Send to the server */
status_code = soup_session_send_message (plugin->priv->session, msg);
if (status_code != SOUP_STATUS_OK) {
g_set_error (error,
@@ -402,6 +827,14 @@ gs_plugin_app_set_review (GsPlugin *plugin,
review = gs_app_get_self_review (app);
g_return_val_if_fail (review != NULL, FALSE);
+ /* Load database once */
+ if (g_once_init_enter (&plugin->priv->db_loaded)) {
+ gboolean ret = load_database (plugin, error);
+ g_once_init_leave (&plugin->priv->db_loaded, TRUE);
+ if (!ret)
+ return FALSE;
+ }
+
/* get the package name */
sources = gs_app_get_sources (app);
if (sources->len == 0) {
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
