[epiphany/wip/google-safe-browsing: 36/37] gsb-storage: Don't hardcode Linux threat lists

[Gabriel Ivașcu <gabrielivascu src gnome org>]

commit 8270c47ad3b92034ccde503d965d86b2a1d0b929
Author: Gabriel Ivascu <gabrielivascu gnome org>
Date:   Mon Oct 2 17:50:43 2017 +0200

    gsb-storage: Don't hardcode Linux threat lists

 lib/safe-browsing/ephy-gsb-service.c |   62 +++++++++++++++++++++
 lib/safe-browsing/ephy-gsb-storage.c |   97 ++++++++++++++++-----------------
 lib/safe-browsing/ephy-gsb-storage.h |    2 +
 3 files changed, 111 insertions(+), 50 deletions(-)
---
diff --git a/lib/safe-browsing/ephy-gsb-service.c b/lib/safe-browsing/ephy-gsb-service.c
index 8a0bc7f..99befa4 100644
--- a/lib/safe-browsing/ephy-gsb-service.c
+++ b/lib/safe-browsing/ephy-gsb-service.c
@@ -212,6 +212,62 @@ ephy_gsb_service_schedule_update (EphyGSBService *self)
   LOG ("Next update scheduled in %ld seconds", interval);
 }
 
+static GList *
+ephy_gsb_service_fetch_threat_lists (EphyGSBService *self)
+{
+  GList *retval = NULL;
+  JsonNode *body_node;
+  JsonObject *body_obj;
+  JsonArray *threat_lists;
+  JsonObject *descriptor;
+  const char *threat_type;
+  const char *platform_type;
+  const char *threat_entry_type;
+  SoupMessage *msg;
+  char *url;
+
+  g_assert (EPHY_IS_GSB_SERVICE (self));
+
+  url = g_strdup_printf ("%sthreatLists?key=%s", API_PREFIX, self->api_key);
+  msg = soup_message_new (SOUP_METHOD_GET, url);
+  soup_session_send_message (self->session, msg);
+
+  if (msg->status_code != 200) {
+    LOG ("Failed to fetch the threat lists from the server, got: %u, %s",
+         msg->status_code, msg->response_body->data);
+    g_object_unref (msg);
+    return NULL;
+  }
+
+  body_node = json_from_string (msg->response_body->data, NULL);
+  body_obj = json_node_get_object (body_node);
+
+  if (json_object_has_non_null_array_member (body_obj, "threatLists")) {
+    threat_lists = json_object_get_array_member (body_obj, "threatLists");
+    for (guint i = 0; i < json_array_get_length (threat_lists); i++) {
+      descriptor = json_array_get_object_element (threat_lists, i);
+      platform_type = json_object_get_string_member (descriptor, "platformType");
+
+      /* Filter out non-Linux threats. */
+      if (g_strcmp0 (platform_type, "LINUX") != 0)
+        continue;
+
+      threat_type = json_object_get_string_member (descriptor, "threatType");
+      threat_entry_type = json_object_get_string_member (descriptor, "threatEntryType");
+      retval = g_list_prepend (retval, ephy_gsb_threat_list_new (threat_type,
+                                                                 platform_type,
+                                                                 threat_entry_type,
+                                                                 NULL));
+    }
+  }
+
+  g_free (url);
+  g_object_unref (msg);
+  json_node_unref (body_node);
+
+  return g_list_reverse (retval);
+}
+
 static void
 ephy_gsb_service_update_thread (GTask          *task,
                                 EphyGSBService *self,
@@ -231,6 +287,12 @@ ephy_gsb_service_update_thread (GTask          *task,
 
   ephy_gsb_storage_delete_old_full_hashes (self->storage);
 
+  /* Fetch and store new threat lists, if any. */
+  threat_lists = ephy_gsb_service_fetch_threat_lists (self);
+  for (GList *l = threat_lists; l && l->data; l = l->next)
+    ephy_gsb_storage_insert_threat_list (self->storage, l->data);
+  g_list_free_full (threat_lists, (GDestroyNotify)ephy_gsb_threat_list_free);
+
   threat_lists = ephy_gsb_storage_get_threat_lists (self->storage);
   if (!threat_lists) {
     self->next_list_updates_time = CURRENT_TIME + DEFAULT_WAIT_TIME;
diff --git a/lib/safe-browsing/ephy-gsb-storage.c b/lib/safe-browsing/ephy-gsb-storage.c
index f61bc42..695a9ea 100644
--- a/lib/safe-browsing/ephy-gsb-storage.c
+++ b/lib/safe-browsing/ephy-gsb-storage.c
@@ -36,22 +36,9 @@
  */
 #define BATCH_SIZE 199
 
-/* Increment schema version if you:
- * 1) Modify the database table structure.
- * 2) Add new threat lists below.
- */
+/* Increment schema version if you modify the database table structure. */
 #define SCHEMA_VERSION 1
 
-/* The available Linux threat lists of Google Safe Browsing API v4.
- * The format is {THREAT_TYPE, PLATFORM_TYPE, THREAT_ENTRY_TYPE}.
- */
-static const char * const gsb_linux_threat_lists[][3] = {
-  {GSB_THREAT_TYPE_MALWARE,            "LINUX", "URL"},
-  {GSB_THREAT_TYPE_SOCIAL_ENGINEERING, "LINUX", "URL"},
-  {GSB_THREAT_TYPE_UNWANTED_SOFTWARE,  "LINUX", "URL"},
-  {GSB_THREAT_TYPE_MALWARE,            "LINUX", "IP_RANGE"},
-};
-
 struct _EphyGSBStorage {
   GObject parent_instance;
 
@@ -210,9 +197,7 @@ ephy_gsb_storage_init_metadata_table (EphyGSBStorage *self)
 static gboolean
 ephy_gsb_storage_init_threats_table (EphyGSBStorage *self)
 {
-  EphySQLiteStatement *statement;
   GError *error = NULL;
-  GString *string;
   const char *sql;
 
   g_assert (EPHY_IS_GSB_STORAGE (self));
@@ -235,40 +220,6 @@ ephy_gsb_storage_init_threats_table (EphyGSBStorage *self)
     return FALSE;
   }
 
-  sql = "INSERT INTO threats (threat_type, platform_type, threat_entry_type) VALUES ";
-  string = g_string_new (sql);
-  for (guint i = 0; i < G_N_ELEMENTS (gsb_linux_threat_lists); i++)
-    g_string_append (string, "(?, ?, ?),");
-  /* Remove trailing comma character. */
-  g_string_erase (string, string->len - 1, -1);
-
-  statement = ephy_sqlite_connection_create_statement (self->db, string->str, &error);
-  g_string_free (string, TRUE);
-
-  if (error) {
-    g_warning ("Failed to create threats table insert statement: %s", error->message);
-    g_error_free (error);
-    return FALSE;
-  }
-
-  for (guint i = 0; i < G_N_ELEMENTS (gsb_linux_threat_lists); i++) {
-    EphyGSBThreatList *list = ephy_gsb_threat_list_new (gsb_linux_threat_lists[i][0],
-                                                        gsb_linux_threat_lists[i][1],
-                                                        gsb_linux_threat_lists[i][2],
-                                                        NULL);
-    bind_threat_list_params (statement, list, i * 3, i * 3 + 1, i * 3 + 2, -1);
-    ephy_gsb_threat_list_free (list);
-  }
-
-  ephy_sqlite_statement_step (statement, &error);
-  g_object_unref (statement);
-
-  if (error) {
-    g_warning ("Failed to insert initial data into threats table: %s", error->message);
-    g_error_free (error);
-    return FALSE;
-  }
-
   return TRUE;
 }
 
@@ -674,6 +625,52 @@ ephy_gsb_storage_set_metadata (EphyGSBStorage *self,
 }
 
 /**
+ * ephy_gsb_storage_insert_threat_list:
+ * @self: an #EphyGSBStorage
+ * @list: an #EphyGSBThreatList
+ *
+ * Insert a threat lists into the local database. If the combination
+ * THREAT_TYPE/PLATFORM_TYPE/THREAT_ENTRY_TYPE already exists in the
+ * database, then this function does nothing. The client state is ignored.
+ * Use ephy_gsb_storage_update_client_state() if you need to update the
+ * client state.
+ **/
+void
+ephy_gsb_storage_insert_threat_list (EphyGSBStorage    *self,
+                                     EphyGSBThreatList *list)
+{
+  EphySQLiteStatement *statement;
+  GError *error = NULL;
+  const char *sql;
+
+  g_assert (EPHY_IS_GSB_STORAGE (self));
+  g_assert (list);
+
+  sql = "INSERT OR IGNORE INTO threats "
+        "(threat_type, platform_type, threat_entry_type, client_state) "
+        "VALUES (?, ?, ?, ?)";
+  statement = ephy_sqlite_connection_create_statement (self->db, sql, &error);
+  if (error) {
+    g_warning ("Failed to create insert threat list statement: %s", error->message);
+    g_error_free (error);
+    return;
+  }
+
+  if (!bind_threat_list_params (statement, list, 0, 1, 2, -1)) {
+    g_object_unref (statement);
+    return;
+  }
+
+  ephy_sqlite_statement_step (statement, &error);
+  if (error) {
+    g_warning ("Failed to execute insert threat list statement: %s", error->message);
+    g_error_free (error);
+  }
+
+  g_object_unref (statement);
+}
+
+/**
  * ephy_gsb_storage_get_threat_lists:
  * @self: an #EphyGSBStorage
  *
diff --git a/lib/safe-browsing/ephy-gsb-storage.h b/lib/safe-browsing/ephy-gsb-storage.h
index ed41a7e..056b4e7 100644
--- a/lib/safe-browsing/ephy-gsb-storage.h
+++ b/lib/safe-browsing/ephy-gsb-storage.h
@@ -38,6 +38,8 @@ gint64          ephy_gsb_storage_get_metadata                   (EphyGSBStorage
 void            ephy_gsb_storage_set_metadata                   (EphyGSBStorage *self,
                                                                  const char     *key,
                                                                  gint64          value);
+void            ephy_gsb_storage_insert_threat_list             (EphyGSBStorage    *self,
+                                                                 EphyGSBThreatList *list);
 GList          *ephy_gsb_storage_get_threat_lists               (EphyGSBStorage *self);
 char           *ephy_gsb_storage_compute_checksum               (EphyGSBStorage    *self,
                                                                  EphyGSBThreatList *list);