GNOME.org
 

[epiphany/wip/google-safe-browsing: 12/21] gsb-service: Add function to query fullHashes:find endpoint

commit dc57a176c2938bcda43d3d4f494d2a83377baf07
Author: Gabriel Ivascu <gabrielivascu gnome org>
Date:   Sun Sep 17 17:40:42 2017 +0300

    gsb-service: Add function to query fullHashes:find endpoint

 lib/safe-browsing/ephy-gsb-service.c |  142 ++++++++++++++++++++++++++++++++++
 lib/safe-browsing/ephy-gsb-storage.c |   47 +++++++++++
 lib/safe-browsing/ephy-gsb-storage.h |   61 ++++++++-------
 lib/safe-browsing/ephy-gsb-utils.c   |  119 ++++++++++++++++++++++++++++
 lib/safe-browsing/ephy-gsb-utils.h   |    4 +
 5 files changed, 344 insertions(+), 29 deletions(-)
---
diff --git a/lib/safe-browsing/ephy-gsb-service.c b/lib/safe-browsing/ephy-gsb-service.c
index 9d97d66..ee79959 100644
--- a/lib/safe-browsing/ephy-gsb-service.c
+++ b/lib/safe-browsing/ephy-gsb-service.c
@@ -57,6 +57,37 @@ static GParamSpec *obj_properties[LAST_PROP];
 
 static gboolean ephy_gsb_service_update (EphyGSBService *self);
 
+typedef struct {
+  EphyGSBService *service;
+  GList          *prefixes;
+} FindFullHashesData;
+
+static FindFullHashesData *
+find_full_hashes_data_new (EphyGSBService *service,
+                                 GList          *prefixes)
+{
+  FindFullHashesData *data;
+
+  g_assert (EPHY_IS_GSB_SERVICE (service));
+  g_assert (prefixes);
+
+  data = g_slice_new (FindFullHashesData);
+  data->service = g_object_ref (service);
+  data->prefixes = g_list_copy_deep (prefixes, (GCopyFunc)g_bytes_ref, NULL);
+
+  return data;
+}
+
+static void
+find_full_hashes_data_free (FindFullHashesData *data)
+{
+  g_assert (data);
+
+  g_object_unref (data->service);
+  g_list_free_full (data->prefixes, (GDestroyNotify)g_bytes_unref);
+  g_slice_free (FindFullHashesData, data);
+}
+
 static inline gboolean
 json_object_has_non_null_string_member (JsonObject *object,
                                         const char *member)
@@ -66,6 +97,14 @@ json_object_has_non_null_string_member (JsonObject *object,
 }
 
 static inline gboolean
+json_object_has_non_null_object_member (JsonObject *object,
+                                        const char *member)
+{
+  return json_object_has_member (object, member) &&
+         json_object_get_object_member (object, member) != NULL;
+}
+
+static inline gboolean
 json_object_has_non_null_array_member (JsonObject *object,
                                        const char *member)
 {
@@ -384,3 +423,106 @@ ephy_gsb_service_new (const char *api_key,
 
   return service;
 }
+
+static void
+ephy_gsb_service_find_full_hashes_cb (SoupSession *session,
+                                      SoupMessage *msg,
+                                      gpointer     user_data)
+{
+  FindFullHashesData *data = (FindFullHashesData *)user_data;
+  EphyGSBService *self = data->service;
+  JsonNode *body_node;
+  JsonObject *body_obj;
+  JsonArray *matches;
+  const char *negative_duration;
+  double duration;
+
+  if (msg->status_code != 200) {
+    LOG ("Cannot update full hashes. Server responded: %u, %s",
+         msg->status_code, msg->response_body->data);
+    goto out;
+  }
+
+  body_node = json_from_string (msg->response_body->data, NULL);
+  body_obj = json_node_get_object (body_node);
+  matches = json_object_get_array_member (body_obj, "matches");
+
+  for (guint i = 0; i < json_array_get_length (matches); i++) {
+    EphyGSBThreatList *list;
+    JsonObject *match = json_array_get_object_element (matches, i);
+    const char *threat_type = json_object_get_string_member (match, "threatType");
+    const char *platform_type = json_object_get_string_member (match, "platformType");
+    const char *threat_entry_type = json_object_get_string_member (match, "threatEntryType");
+    JsonObject *threat = json_object_get_object_member (match, "threat");
+    const char *hash_b64 = json_object_get_string_member (threat, "hash");
+    const char *positive_duration;
+    char *malware_threat_type = NULL;
+    guint8 *hash;
+    gsize length;
+
+    list = ephy_gsb_threat_list_new (threat_type, platform_type, threat_entry_type, NULL, 0);
+    hash = g_base64_decode (hash_b64, &length);
+    positive_duration = json_object_get_string_member (match, "cacheDuration");
+    sscanf (positive_duration, "%lfs", &duration);
+
+    if (json_object_has_non_null_object_member (match, "threatEntryMetadata")) {
+      JsonObject *threat_entry_metadata = json_object_get_object_member (match, "threatEntryMetadata");
+      malware_threat_type = ephy_gsb_utils_get_metadata_entry (threat_entry_metadata, "malware_threat_type");
+    }
+
+    ephy_gsb_storage_insert_full_hash (self->storage, list, hash, floor (duration), malware_threat_type);
+
+    g_free (hash);
+    g_free (malware_threat_type);
+    ephy_gsb_threat_list_free (list);
+  }
+
+  /* Update negative cache duration. */
+  negative_duration = json_object_get_string_member (body_obj, "negativeCacheDuration");
+  sscanf (negative_duration, "%lfs", &duration);
+  for (GList *l = data->prefixes; l && l->data; l = l->next)
+    ephy_gsb_storage_update_hash_prefix_expiration (self->storage, l->data, floor (duration));
+
+  /* TODO: Handle minimumWaitDuration. */
+
+  json_node_unref (body_node);
+out:
+  find_full_hashes_data_free (data);
+}
+
+static void
+ephy_gsb_service_find_full_hashes (EphyGSBService *self,
+                                   GList          *prefixes)
+{
+  SoupMessage *msg;
+  GList *threat_lists;
+  JsonNode *body_node;
+  JsonObject *body_obj;
+  char *url;
+  char *body;
+
+  g_assert (EPHY_IS_GSB_SERVICE (self));
+  g_assert (ephy_gsb_storage_is_operable (self->storage));
+  g_assert (prefixes);
+
+  LOG ("Updating full hashes of %u prefixes", g_list_length (prefixes));
+
+  threat_lists = ephy_gsb_storage_get_threat_lists (self->storage);
+  body_obj = ephy_gsb_utils_make_full_hashes_request (threat_lists, prefixes);
+
+  body_node = json_node_new (JSON_NODE_OBJECT);
+  json_node_set_object (body_node, body_obj);
+  body = json_to_string (body_node, FALSE);
+
+  url = g_strdup_printf ("%sfullHashes:find?key=%s", API_PREFIX, self->api_key);
+  msg = soup_message_new (SOUP_METHOD_POST, url);
+  soup_message_set_request (msg, "application/json", SOUP_MEMORY_TAKE, body, strlen (body));
+  soup_session_queue_message (self->session, msg,
+                              ephy_gsb_service_find_full_hashes_cb,
+                              find_full_hashes_data_new (self, prefixes));
+
+  g_free (url);
+  json_object_unref (body_obj);
+  json_node_unref (body_node);
+  g_list_free_full (threat_lists, (GDestroyNotify)ephy_gsb_threat_list_free);
+}
diff --git a/lib/safe-browsing/ephy-gsb-storage.c b/lib/safe-browsing/ephy-gsb-storage.c
index 67a9bff..d41ea02 100644
--- a/lib/safe-browsing/ephy-gsb-storage.c
+++ b/lib/safe-browsing/ephy-gsb-storage.c
@@ -1348,3 +1348,50 @@ out:
   if (error)
     g_error_free (error);
 }
+
+void
+ephy_gsb_storage_update_hash_prefix_expiration (EphyGSBStorage *self,
+                                                GBytes         *prefix,
+                                                gint64          duration)
+{
+  EphySQLiteStatement *statement = NULL;
+  GError *error = NULL;
+  const char *sql;
+
+  g_assert (EPHY_IS_GSB_STORAGE (self));
+  g_assert (self->is_operable);
+  g_assert (prefix);
+
+  sql = "UPDATE hash_prefix "
+        "SET negative_expires_at=(CAST(strftime('%s', 'now') AS INT)) + ? "
+        "WHERE value=?";
+  statement = ephy_sqlite_connection_create_statement (self->db, sql, &error);
+  if (error) {
+    g_warning ("Failed to create update hash prefix statement: %s", error->message);
+    goto out;
+  }
+
+  ephy_sqlite_statement_bind_int64 (statement, 0, duration, &error);
+  if (error) {
+    g_warning ("Failed to bind int64 in update hash prefix statement: %s", error->message);
+    goto out;
+  }
+  ephy_sqlite_statement_bind_blob (statement, 1,
+                                   g_bytes_get_data (prefix, NULL),
+                                   g_bytes_get_size (prefix),
+                                   &error);
+  if (error) {
+    g_warning ("Failed to bind blob in update hash prefix statement: %s", error->message);
+    goto out;
+  }
+
+  ephy_sqlite_statement_step (statement, &error);
+  if (error)
+    g_warning ("Failed to execute update hash prefix statement: %s", error->message);
+
+out:
+  if (statement)
+    g_object_unref (statement);
+  if (error)
+    g_error_free (error);
+}
diff --git a/lib/safe-browsing/ephy-gsb-storage.h b/lib/safe-browsing/ephy-gsb-storage.h
index 6f64f2e..4cd5981 100644
--- a/lib/safe-browsing/ephy-gsb-storage.h
+++ b/lib/safe-browsing/ephy-gsb-storage.h
@@ -30,34 +30,37 @@ G_BEGIN_DECLS
 
 G_DECLARE_FINAL_TYPE (EphyGSBStorage, ephy_gsb_storage, EPHY, GSB_STORAGE, GObject)
 
-EphyGSBStorage *ephy_gsb_storage_new                    (const char *db_path);
-gboolean        ephy_gsb_storage_is_operable            (EphyGSBStorage *self);
-gint64          ephy_gsb_storage_get_next_update_time   (EphyGSBStorage *self);
-void            ephy_gsb_storage_set_next_update_time   (EphyGSBStorage *self,
-                                                         gint64          next_update_time);
-GList          *ephy_gsb_storage_get_threat_lists       (EphyGSBStorage *self);
-char           *ephy_gsb_storage_compute_checksum       (EphyGSBStorage    *self,
-                                                         EphyGSBThreatList *list);
-void            ephy_gsb_storage_update_client_state    (EphyGSBStorage    *self,
-                                                         EphyGSBThreatList *list,
-                                                         gboolean           clear);
-void            ephy_gsb_storage_clear_hash_prefixes    (EphyGSBStorage    *self,
-                                                         EphyGSBThreatList *list);
-void            ephy_gsb_storage_delete_hash_prefixes   (EphyGSBStorage    *self,
-                                                         EphyGSBThreatList *list,
-                                                         JsonArray         *indices);
-void            ephy_gsb_storage_insert_hash_prefixes   (EphyGSBStorage    *self,
-                                                         EphyGSBThreatList *list,
-                                                         gsize              prefix_len,
-                                                         const char        *prefixes_b64);
-GList          *ephy_gsb_storage_lookup_hash_prefixes   (EphyGSBStorage *self,
-                                                         GList          *cues);
-GList          *ephy_gsb_storage_lookup_full_hashes     (EphyGSBStorage *self,
-                                                         GList          *hashes);
-void            ephy_gsb_storage_insert_full_hash       (EphyGSBStorage    *self,
-                                                         EphyGSBThreatList *list,
-                                                         const guint8      *hash,
-                                                         gint64             duration,
-                                                         const char        *malware_threat_type);
+EphyGSBStorage *ephy_gsb_storage_new                            (const char *db_path);
+gboolean        ephy_gsb_storage_is_operable                    (EphyGSBStorage *self);
+gint64          ephy_gsb_storage_get_next_update_time           (EphyGSBStorage *self);
+void            ephy_gsb_storage_set_next_update_time           (EphyGSBStorage *self,
+                                                                 gint64          next_update_time);
+GList          *ephy_gsb_storage_get_threat_lists               (EphyGSBStorage *self);
+char           *ephy_gsb_storage_compute_checksum               (EphyGSBStorage    *self,
+                                                                 EphyGSBThreatList *list);
+void            ephy_gsb_storage_update_client_state            (EphyGSBStorage    *self,
+                                                                 EphyGSBThreatList *list,
+                                                                 gboolean           clear);
+void            ephy_gsb_storage_clear_hash_prefixes            (EphyGSBStorage    *self,
+                                                                 EphyGSBThreatList *list);
+void            ephy_gsb_storage_delete_hash_prefixes           (EphyGSBStorage    *self,
+                                                                 EphyGSBThreatList *list,
+                                                                 JsonArray         *indices);
+void            ephy_gsb_storage_insert_hash_prefixes           (EphyGSBStorage    *self,
+                                                                 EphyGSBThreatList *list,
+                                                                 gsize              prefix_len,
+                                                                 const char        *prefixes_b64);
+GList          *ephy_gsb_storage_lookup_hash_prefixes           (EphyGSBStorage *self,
+                                                                 GList          *cues);
+GList          *ephy_gsb_storage_lookup_full_hashes             (EphyGSBStorage *self,
+                                                                 GList          *hashes);
+void            ephy_gsb_storage_insert_full_hash               (EphyGSBStorage    *self,
+                                                                 EphyGSBThreatList *list,
+                                                                 const guint8      *hash,
+                                                                 gint64             duration,
+                                                                 const char        *malware_threat_type);
+void            ephy_gsb_storage_update_hash_prefix_expiration  (EphyGSBStorage *self,
+                                                                 GBytes         *prefix,
+                                                                 gint64          duration);
 
 G_END_DECLS
diff --git a/lib/safe-browsing/ephy-gsb-utils.c b/lib/safe-browsing/ephy-gsb-utils.c
index 7c0ff30..da70958 100644
--- a/lib/safe-browsing/ephy-gsb-utils.c
+++ b/lib/safe-browsing/ephy-gsb-utils.c
@@ -201,6 +201,125 @@ ephy_gsb_utils_make_list_updates_request (GList *threat_lists)
   return retval;
 }
 
+JsonObject *
+ephy_gsb_utils_make_full_hashes_request (GList *threat_lists,
+                                         GList *hash_prefixes)
+{
+  GHashTable *threat_types_set;
+  GHashTable *platform_types_set;
+  GHashTable *threat_entry_types_set;
+  GList *threat_types_list;
+  GList *platform_types_list;
+  GList *threat_entry_types_list;
+  JsonArray *threat_types;
+  JsonArray *platform_types;
+  JsonArray *threat_entry_types;
+  JsonArray *threat_entries;
+  JsonArray *client_states;
+  JsonObject *threat_info;
+  JsonObject *retval;
+
+  g_assert (threat_lists);
+  g_assert (hash_prefixes);
+
+  client_states = json_array_new ();
+  threat_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+  platform_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+  threat_entry_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+
+  for (GList *l = threat_lists; l && l->data; l = l->next) {
+    EphyGSBThreatList *list = (EphyGSBThreatList *)l->data;
+
+    if (!g_hash_table_contains (threat_types_set, list->threat_type))
+      g_hash_table_add (threat_types_set, list->threat_type);
+    if (!g_hash_table_contains (platform_types_set, list->platform_type))
+      g_hash_table_add (platform_types_set, list->platform_type);
+    if (!g_hash_table_contains (threat_entry_types_set, list->threat_entry_type))
+      g_hash_table_add (threat_entry_types_set, list->threat_entry_type);
+
+    json_array_add_string_element (client_states, list->client_state);
+  }
+
+  threat_types = json_array_new ();
+  threat_types_list = g_hash_table_get_keys (threat_types_set);
+  for (GList *l = threat_types_list; l && l->data; l = l->next)
+    json_array_add_string_element (threat_types, (const char *)l->data);
+
+  platform_types = json_array_new ();
+  platform_types_list = g_hash_table_get_keys (platform_types_set);
+  for (GList *l = platform_types_list; l && l->data; l = l->next)
+    json_array_add_string_element (platform_types, (const char *)l->data);
+
+  threat_entry_types = json_array_new ();
+  threat_entry_types_list = g_hash_table_get_keys (threat_entry_types_set);
+  for (GList *l = threat_entry_types_list; l && l->data; l = l->next)
+    json_array_add_string_element (threat_entry_types, (const char *)l->data);
+
+  threat_entries = json_array_new ();
+  for (GList *l = hash_prefixes; l && l->data; l = l->next) {
+    JsonObject *threat_entry = json_object_new ();
+    char *hash = g_base64_encode (g_bytes_get_data (l->data, NULL),
+                                  g_bytes_get_size (l->data));
+
+    json_object_set_string_member (threat_entry, "hash", hash);
+    json_array_add_object_element (threat_entries, threat_entry);
+
+    g_free (hash);
+  }
+
+  threat_info = json_object_new ();
+  json_object_set_array_member (threat_info, "threatTypes", threat_types);
+  json_object_set_array_member (threat_info, "platformTypes", platform_types);
+  json_object_set_array_member (threat_info, "threatEntryTypes", threat_entry_types);
+  json_object_set_array_member (threat_info, "threatEntries", threat_entries);
+
+  retval = json_object_new ();
+  json_object_set_object_member (retval, "client", ephy_gsb_utils_make_client_info ());
+  json_object_set_array_member (retval, "clientStates", client_states);
+  json_object_set_object_member (retval, "threatInfo", threat_info);
+  json_object_set_null_member (retval, "apiClient");
+
+  g_list_free (threat_types_list);
+  g_list_free (platform_types_list);
+  g_list_free (threat_entry_types_list);
+  g_hash_table_unref (threat_types_set);
+  g_hash_table_unref (platform_types_set);
+  g_hash_table_unref (threat_entry_types_set);
+
+  return retval;
+}
+
+char *
+ephy_gsb_utils_get_metadata_entry (JsonObject *threat_entry_metadata,
+                                   const char *metadata_key)
+{
+  JsonArray *entries;
+  gsize length;
+
+  g_assert (threat_entry_metadata);
+  g_assert (metadata_key);
+
+  if (!json_object_has_member (threat_entry_metadata, "entries"))
+    return NULL;
+
+  entries = json_object_get_array_member (threat_entry_metadata, "entries");
+  for (guint i = 0; i < json_array_get_length (entries); i++) {
+    JsonObject *entry = json_array_get_object_element (entries, i);
+    const char *key_b64 = json_object_get_string_member (entry, "key");
+    const char *value_b64 = json_object_get_string_member (entry, "value");
+    char *key = (char *)g_base64_decode (key_b64, &length);
+
+    if (!g_strcmp0 (key, metadata_key)) {
+      g_free (key);
+      return (char *)g_base64_decode (value_b64, &length);
+    }
+
+    g_free (key);
+  }
+
+  return NULL;
+}
+
 static char *
 ephy_gsb_utils_full_unescape (const char *part)
 {
diff --git a/lib/safe-browsing/ephy-gsb-utils.h b/lib/safe-browsing/ephy-gsb-utils.h
index 9b69cc1..079f071 100644
--- a/lib/safe-browsing/ephy-gsb-utils.h
+++ b/lib/safe-browsing/ephy-gsb-utils.h
@@ -77,6 +77,10 @@ EphyGSBHashFullLookup   *ephy_gsb_hash_full_lookup_new            (const guint8
 void                    ephy_gsb_hash_full_lookup_free            (EphyGSBHashFullLookup *lookup);
 
 JsonObject              *ephy_gsb_utils_make_list_updates_request (GList *threat_lists);
+JsonObject              *ephy_gsb_utils_make_full_hashes_request  (GList *threat_lists,
+                                                                   GList *hash_prefixes);
+char                    *ephy_gsb_utils_get_metadata_entry        (JsonObject *threat_entry_metadata,
+                                                                   const char *metadata_key);
 
 char                    *ephy_gsb_utils_canonicalize              (const char  *url,
                                                                    char       **host_out,
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]