[gnome-software/wip/rancell/ubuntu-ratings-3-18: 9/19] Can finally download stats from server
- From: Robert Ancell <rancell src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-software/wip/rancell/ubuntu-ratings-3-18: 9/19] Can finally download stats from server
- Date: Fri, 11 Dec 2015 02:35:21 +0000 (UTC)
commit 12596c90aa7d4db5a80d44b5365544aaab6bb441
Author: Robert Ancell <robert ancell canonical com>
Date: Tue Nov 24 22:44:52 2015 +1300
Can finally download stats from server
src/plugins/gs-plugin-ubuntu-reviews.c | 360 ++++++++++++++++++++++----------
1 files changed, 245 insertions(+), 115 deletions(-)
---
diff --git a/src/plugins/gs-plugin-ubuntu-reviews.c b/src/plugins/gs-plugin-ubuntu-reviews.c
index 8a75284..6e5a203 100644
--- a/src/plugins/gs-plugin-ubuntu-reviews.c
+++ b/src/plugins/gs-plugin-ubuntu-reviews.c
@@ -36,13 +36,21 @@ struct GsPluginPrivate {
sqlite3 *db;
};
+typedef struct {
+ gint64 one_star_count;
+ gint64 two_star_count;
+ gint64 three_star_count;
+ gint64 four_star_count;
+ gint64 five_star_count;
+} Histogram;
+
/**
* gs_plugin_get_name:
*/
const gchar *
gs_plugin_get_name (void)
{
- return "ubuntu-ratings";
+ return "ubuntu-reviews";
}
#define GS_PLUGIN_UBUNTU_REVIEWS_SERVER "https://reviews.ubuntu.com/reviews";
@@ -95,11 +103,8 @@ gs_plugin_destroy (GsPlugin *plugin)
g_object_unref (plugin->priv->session);
}
-/**
- * gs_plugin_setup_networking:
- */
static gboolean
-gs_plugin_setup_networking (GsPlugin *plugin, GError **error)
+setup_networking (GsPlugin *plugin, GError **error)
{
/* already set up */
if (plugin->priv->session != NULL)
@@ -132,38 +137,59 @@ gs_plugin_app_set_rating (GsPlugin *plugin,
return FALSE;
}
-/**
- * gs_plugin_ubuntu_reviews_timestamp_cb:
- **/
static gint
-gs_plugin_ubuntu_reviews_timestamp_cb (void *data, gint argc,
- gchar **argv, gchar **col_name)
+get_timestamp_sqlite_cb (void *data, gint argc,
+ gchar **argv, gchar **col_name)
{
gint64 *timestamp = (gint64 *) data;
*timestamp = g_ascii_strtoll (argv[0], NULL, 10);
return 0;
}
+static gboolean
+set_package_stats (GsPlugin *plugin,
+ const gchar *package_name,
+ Histogram *histogram,
+ GError **error)
+{
+ char *error_msg = NULL;
+ gint result;
+ g_autofree gchar *statement = NULL;
+
+ statement = g_strdup_printf ("INSERT OR REPLACE INTO review_stats (package_name, "
+ "one_star_count, two_star_count, three_star_count, "
+ "four_star_count, five_star_count) "
+ "VALUES ('%s', '%" G_GINT64_FORMAT "', '%" G_GINT64_FORMAT"', '%"
G_GINT64_FORMAT "', '%" G_GINT64_FORMAT "', '%" G_GINT64_FORMAT "');",
+ package_name, histogram->one_star_count, histogram->two_star_count,
+ histogram->three_star_count, histogram->four_star_count,
histogram->five_star_count);
+ result = sqlite3_exec (plugin->priv->db, statement, NULL, NULL, &error_msg);
+ if (result != SQLITE_OK) {
+ g_set_error (error,
+ GS_PLUGIN_ERROR,
+ GS_PLUGIN_ERROR_FAILED,
+ "SQL error: %s", error_msg);
+ sqlite3_free (error_msg);
+ return FALSE;
+ }
+
+ return TRUE;
+}
-/**
- * gs_plugin_ubuntu_reviews_set_timestamp:
- */
static gboolean
-gs_plugin_ubuntu_reviews_set_timestamp (GsPlugin *plugin,
- const gchar *type,
- GError **error)
+set_timestamp (GsPlugin *plugin,
+ const gchar *type,
+ GError **error)
{
char *error_msg = NULL;
- gint rc;
+ gint result;
g_autofree gchar *statement = NULL;
- /* insert the entry */
statement = g_strdup_printf ("INSERT OR REPLACE INTO timestamps (key, value) "
"VALUES ('%s', '%" G_GINT64_FORMAT "');",
type,
g_get_real_time () / G_USEC_PER_SEC);
- rc = sqlite3_exec (plugin->priv->db, statement, NULL, NULL, &error_msg);
- if (rc != SQLITE_OK) {
+ result = sqlite3_exec (plugin->priv->db, statement, NULL, NULL, &error_msg);
+ if (result != SQLITE_OK) {
g_set_error (error,
GS_PLUGIN_ERROR,
GS_PLUGIN_ERROR_FAILED,
@@ -174,32 +200,106 @@ gs_plugin_ubuntu_reviews_set_timestamp (GsPlugin *plugin,
return TRUE;
}
-/**
- * gs_plugin_ubuntu_review_stats_download:
- **/
static gboolean
-gs_plugin_ubuntu_review_stats_download (GsPlugin *plugin, GError **error)
+parse_histogram (const gchar *text, Histogram *histogram)
+{
+ JsonParser *parser = NULL;
+ JsonArray *array;
+ gboolean result = FALSE;
+
+ /* Histogram is a five element JSON array, e.g. "[1, 3, 5, 8, 4]" */
+ parser = json_parser_new ();
+ if (!json_parser_load_from_data (parser, text, -1, NULL))
+ goto out;
+ if (!JSON_NODE_HOLDS_ARRAY (json_parser_get_root (parser)))
+ goto out;
+ array = json_node_get_array (json_parser_get_root (parser));
+ if (json_array_get_length (array) != 5)
+ goto out;
+ histogram->one_star_count = json_array_get_int_element (array, 0);
+ histogram->two_star_count = json_array_get_int_element (array, 1);
+ histogram->three_star_count = json_array_get_int_element (array, 2);
+ histogram->four_star_count = json_array_get_int_element (array, 3);
+ histogram->five_star_count = json_array_get_int_element (array, 4);
+ result = TRUE;
+
+out:
+ g_clear_object (&parser);
+
+ return result;
+}
+
+static gboolean
+parse_review_entry (JsonNode *node, const gchar **package_name, Histogram *histogram)
+{
+ JsonObject *object;
+ const gchar *name = NULL, *histogram_text = NULL;
+
+ if (!JSON_NODE_HOLDS_OBJECT (node))
+ return FALSE;
+
+ object = json_node_get_object (node);
+
+ name = json_object_get_string_member (object, "package_name");
+ histogram_text = json_object_get_string_member (object, "histogram");
+ if (!name || !histogram_text)
+ return FALSE;
+
+ if (!parse_histogram (histogram_text, histogram))
+ return FALSE;
+ *package_name = name;
+
+ return TRUE;
+}
+
+static gboolean
+parse_review_entries (GsPlugin *plugin, const gchar *text, GError **error)
+{
+ JsonParser *parser = NULL;
+ JsonArray *array;
+ gint i;
+ gboolean result = FALSE;
+
+ parser = json_parser_new ();
+ if (!json_parser_load_from_data (parser, text, -1, error))
+ goto out;
+ if (!JSON_NODE_HOLDS_ARRAY (json_parser_get_root (parser)))
+ goto out;
+ array = json_node_get_array (json_parser_get_root (parser));
+ for (i = 0; i < json_array_get_length (array); i++) {
+ const gchar *package_name;
+ Histogram histogram;
+
+ /* Read in from JSON... (skip bad entries) */
+ if (!parse_review_entry (json_array_get_element (array, i), &package_name, &histogram))
+ continue;
+
+ /* ...write into the database (abort everything if can't write) */
+ if (!set_package_stats (plugin, package_name, &histogram, error))
+ goto out;
+ }
+ result = TRUE;
+
+out:
+ g_clear_object (&parser);
+
+ return result;
+}
+
+static gboolean
+download_review_stats (GsPlugin *plugin, GError **error)
{
- JsonParser *parser;
- JsonReader *reader;
- gdouble count_sum = 0;
- guint i;
guint status_code;
g_autofree gchar *uri = NULL;
g_autoptr(SoupMessage) msg = NULL;
- g_autoptr(GPtrArray) items = NULL;
g_auto(GStrv) split = NULL;
- /* create the GET data */
+ /* Get the review stats using HTTP */
uri = g_strdup_printf ("%s/api/1.0/review-stats/any/any/",
GS_PLUGIN_UBUNTU_REVIEWS_SERVER);
msg = soup_message_new (SOUP_METHOD_GET, uri);
-
- /* ensure networking is set up */
- if (!gs_plugin_setup_networking (plugin, error))
+ if (!setup_networking (plugin, error))
return FALSE;
-
- /* set sync request */
status_code = soup_session_send_message (plugin->priv->session, msg);
if (status_code != SOUP_STATUS_OK) {
g_set_error (error,
@@ -210,59 +310,21 @@ gs_plugin_ubuntu_review_stats_download (GsPlugin *plugin, GError **error)
return FALSE;
}
- /* process the JSON data */
- parser = json_parser_new ();
- if (!json_parser_load_from_data (parser, msg->response_body->data, -1, error)) {
- g_object_unref (parser);
+ /* Extract the stats from the data */
+ if (!parse_review_entries (plugin, msg->response_body->data, error))
return FALSE;
- }
- reader = json_reader_new (json_parser_get_root (parser));
- if (!json_reader_is_array (reader)) {
- g_object_unref (reader);
- g_object_unref (parser);
- return FALSE;
- }
- for (i = 0; i < json_reader_count_elements (reader); i++) {
- json_reader_read_element (reader, i);
- if (json_reader_is_object (reader)) {
- const gchar *package_name;
- gint rating;
-
- json_reader_read_member (reader, "package_name");
- package_name = json_reader_get_string_value (reader);
- json_reader_end_member (reader);
- json_reader_read_member (reader, "ratings_average");
- /* convert star ratings *->10, **->30, ***->50, ****->70, *****->90 */
- rating = round (20 * g_ascii_strtod (json_reader_get_string_value (reader), NULL) -
10);
- json_reader_end_member (reader);
- }
- json_reader_end_element (reader);
- }
- g_object_unref (reader);
-
- /* no suitable data? */
- if (items->len == 0) {
- g_set_error_literal (error,
- GS_PLUGIN_ERROR,
- GS_PLUGIN_ERROR_FAILED,
- "Failed to get data from Ubuntu reviews");
- return FALSE;
- }
-
- return FALSE;
+ /* Record the time we downloaded it */
+ return set_timestamp (plugin, "mtime", error);
}
-/**
- * gs_plugin_ubuntu_ratings_load_db
- **/
static gboolean
-gs_plugin_ubuntu_review_stats_load_db (GsPlugin *plugin, GError **error)
+load_database (GsPlugin *plugin, GError **error)
{
const gchar *statement;
gboolean rebuild_ratings = FALSE;
char *error_msg = NULL;
- gint rc;
+ gint result;
gint64 mtime = 0;
gint64 now;
g_autoptr(GError) error_local = NULL;
@@ -270,8 +332,8 @@ gs_plugin_ubuntu_review_stats_load_db (GsPlugin *plugin, GError **error)
g_debug ("trying to open database '%s'", plugin->priv->db_path);
if (!gs_mkdir_parent (plugin->priv->db_path, error))
return FALSE;
- rc = sqlite3_open (plugin->priv->db_path, &plugin->priv->db);
- if (rc != SQLITE_OK) {
+ result = sqlite3_open (plugin->priv->db_path, &plugin->priv->db);
+ if (result != SQLITE_OK) {
g_set_error (error,
GS_PLUGIN_ERROR,
GS_PLUGIN_ERROR_FAILED,
@@ -280,36 +342,37 @@ gs_plugin_ubuntu_review_stats_load_db (GsPlugin *plugin, GError **error)
return FALSE;
}
- /* we don't need to keep doing fsync */
+ /* We don't need to keep doing fsync */
sqlite3_exec (plugin->priv->db, "PRAGMA synchronous=OFF",
NULL, NULL, NULL);
- /* create ratings if required */
- rc = sqlite3_exec (plugin->priv->db,
- "SELECT vote_count FROM ratings LIMIT 1",
- gs_plugin_ubuntu_reviews_timestamp_cb, &mtime,
- &error_msg);
- if (rc != SQLITE_OK) {
+ /* Create a table to store the stats */
+ result = sqlite3_exec (plugin->priv->db,
+ "SELECT one_star_count FROM review_stats LIMIT 1",
+ NULL, NULL,
+ &error_msg);
+ if (result != SQLITE_OK) {
g_debug ("creating table to repair: %s", error_msg);
sqlite3_free (error_msg);
- statement = "DROP TABLE IF EXISTS ratings;";
+ statement = "DROP TABLE IF EXISTS review_stats;";
sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
- statement = "CREATE TABLE ratings ("
- "pkgname TEXT PRIMARY KEY,"
- "rating INTEGER DEFAULT 0,"
- "vote_count INTEGER DEFAULT 0,"
- "user_count INTEGER DEFAULT 0,"
- "confidence INTEGER DEFAULT 0);";
+ statement = "CREATE TABLE review_stats ("
+ "package_name TEXT PRIMARY KEY,"
+ "one_star_count INTEGER DEFAULT 0,"
+ "two_star_count INTEGER DEFAULT 0,"
+ "three_star_count INTEGER DEFAULT 0,"
+ "four_star_count INTEGER DEFAULT 0,"
+ "five_star_count INTEGER DEFAULT 0);";
sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
rebuild_ratings = TRUE;
}
- /* create timestamps if required */
- rc = sqlite3_exec (plugin->priv->db,
- "SELECT value FROM timestamps WHERE key = 'mtime' LIMIT 1",
- gs_plugin_ubuntu_reviews_timestamp_cb, &mtime,
- &error_msg);
- if (rc != SQLITE_OK) {
+ /* Create a table to store timestamps */
+ result = sqlite3_exec (plugin->priv->db,
+ "SELECT value FROM timestamps WHERE key = 'mtime' LIMIT 1",
+ get_timestamp_sqlite_cb, &mtime,
+ &error_msg);
+ if (result != SQLITE_OK) {
g_debug ("creating table to repair: %s", error_msg);
sqlite3_free (error_msg);
statement = "CREATE TABLE timestamps ("
@@ -317,17 +380,16 @@ gs_plugin_ubuntu_review_stats_load_db (GsPlugin *plugin, GError **error)
"value INTEGER DEFAULT 0);";
sqlite3_exec (plugin->priv->db, statement, NULL, NULL, NULL);
- /* reset the timestamp */
- if (!gs_plugin_ubuntu_reviews_set_timestamp (plugin, "ctime", error))
+ /* Set the time of database creation */
+ if (!set_timestamp (plugin, "ctime", error))
return FALSE;
}
- /* no data */
+ /* Download data if we have none or it is out of date */
now = g_get_real_time () / G_USEC_PER_SEC;
if (mtime == 0 || rebuild_ratings) {
g_debug ("No ubuntu-reviews data");
- /* this should not be fatal */
- if (!gs_plugin_ubuntu_review_stats_download (plugin, &error_local)) {
+ if (!download_review_stats (plugin, &error_local)) {
g_warning ("Failed to get ubuntu-reviews data: %s",
error_local->message);
return TRUE;
@@ -336,7 +398,7 @@ gs_plugin_ubuntu_review_stats_load_db (GsPlugin *plugin, GError **error)
g_debug ("ubuntu-reviews data was %" G_GINT64_FORMAT
" days old, so regetting",
(now - mtime) / ( 60 * 60 * 24));
- if (!gs_plugin_ubuntu_review_stats_download (plugin, error))
+ if (!download_review_stats (plugin, error))
return FALSE;
} else {
g_debug ("ubuntu-reviews data %" G_GINT64_FORMAT
@@ -346,6 +408,60 @@ gs_plugin_ubuntu_review_stats_load_db (GsPlugin *plugin, GError **error)
return TRUE;
}
+static gint
+get_rating_sqlite_cb (void *data,
+ gint argc,
+ gchar **argv,
+ gchar **col_name)
+{
+ Histogram *histogram = (Histogram *) data;
+ histogram->one_star_count = g_ascii_strtoll (argv[0], NULL, 10);
+ histogram->two_star_count = g_ascii_strtoll (argv[1], NULL, 10);
+ histogram->three_star_count = g_ascii_strtoll (argv[2], NULL, 10);
+ histogram->four_star_count = g_ascii_strtoll (argv[3], NULL, 10);
+ histogram->five_star_count = g_ascii_strtoll (argv[4], NULL, 10);
+ return 0;
+}
+
+static gboolean
+get_rating (GsPlugin *plugin,
+ const gchar *package_name,
+ gint *rating,
+ GError **error)
+{
+ Histogram histogram = { 0, 0, 0, 0, 0 };
+ gchar *error_msg = NULL;
+ gint result, n_ratings;
+ g_autofree gchar *statement = NULL;
+
+ /* Get histogram from the database */
+ statement = g_strdup_printf ("SELECT one_star_count, two_star_count, three_star_count,
four_star_count, five_star_count FROM review_stats "
+ "WHERE package_name = '%s'", package_name);
+ result = sqlite3_exec (plugin->priv->db,
+ statement,
+ get_rating_sqlite_cb,
+ &histogram,
+ &error_msg);
+ if (result != SQLITE_OK) {
+ g_set_error (error,
+ GS_PLUGIN_ERROR,
+ GS_PLUGIN_ERROR_FAILED,
+ "SQL error: %s", error_msg);
+ sqlite3_free (error_msg);
+ return FALSE;
+ }
+
+ /* Convert to a rating */
+ // FIXME: Convert to a Wilson score
+ n_ratings = histogram.one_star_count + histogram.two_star_count + histogram.three_star_count +
histogram.four_star_count + histogram.five_star_count;
+ if (n_ratings == 0)
+ *rating = -1;
+ else
+ *rating = ((histogram.one_star_count * 10) + (histogram.two_star_count * 30) +
(histogram.three_star_count * 50) + (histogram.four_star_count * 70) + (histogram.five_star_count * 90)) /
n_ratings;
+
+ return TRUE;
+}
+
/**
* gs_plugin_refine:
*/
@@ -358,18 +474,13 @@ gs_plugin_refine (GsPlugin *plugin,
{
GList *l;
- for (l = *list; l != NULL; l = l->next) {
- GsApp *app = GS_APP (l->data);
- g_printerr (" %s\n", gs_app_get_id (app));
- }
-
/* We only update ratings */
if ((flags & GS_PLUGIN_REFINE_FLAGS_REQUIRE_RATING) == 0)
return TRUE;
/* Load once */
if (g_once_init_enter (&plugin->priv->loaded)) {
- gboolean ret = gs_plugin_ubuntu_review_stats_load_db (plugin, error);
+ gboolean ret = load_database (plugin, error);
g_once_init_leave (&plugin->priv->loaded, TRUE);
if (!ret)
return FALSE;
@@ -377,10 +488,29 @@ gs_plugin_refine (GsPlugin *plugin,
for (l = *list; l != NULL; l = l->next) {
GsApp *app = GS_APP (l->data);
- if (gs_app_get_id (app) == NULL)
- continue;
+ GPtrArray *sources;
+ guint i;
+
if (gs_app_get_rating (app) != -1)
continue;
+
+ sources = gs_app_get_sources (app);
+ for (i = 0; i < sources->len; i++) {
+ const gchar *package_name;
+ gint rating;
+ gboolean ret;
+
+ package_name = g_ptr_array_index (sources, i);
+ ret = get_rating (plugin, package_name, &rating, error);
+ if (!ret)
+ return FALSE;
+ if (rating != -1) {
+ g_debug ("ubuntu-reviews setting rating on %s to %i%%",
+ package_name, rating);
+ gs_app_set_rating (app, rating);
+ gs_app_set_rating_kind (app, GS_APP_RATING_KIND_SYSTEM);
+ }
+ }
}
return TRUE;
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
