[epiphany/wip/google-safe-browsing: 12/18] gsb-service: Add function to query fullHashes:find endpoint
- From: Gabriel Ivașcu <gabrielivascu src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [epiphany/wip/google-safe-browsing: 12/18] gsb-service: Add function to query fullHashes:find endpoint
- Date: Tue, 19 Sep 2017 16:31:29 +0000 (UTC)
commit 06e49ad2c10262ebaf869e3ac1a0fa65d7486810
Author: Gabriel Ivascu <gabrielivascu gnome org>
Date: Sun Sep 17 17:40:42 2017 +0300
gsb-service: Add function to query fullHashes:find endpoint
lib/safe-browsing/ephy-gsb-service.c | 155 ++++++++++++++++++++++++++++++++++
lib/safe-browsing/ephy-gsb-storage.c | 47 ++++++++++
lib/safe-browsing/ephy-gsb-storage.h | 61 +++++++-------
lib/safe-browsing/ephy-gsb-utils.c | 119 ++++++++++++++++++++++++++
lib/safe-browsing/ephy-gsb-utils.h | 4 +
5 files changed, 357 insertions(+), 29 deletions(-)
---
diff --git a/lib/safe-browsing/ephy-gsb-service.c b/lib/safe-browsing/ephy-gsb-service.c
index fc2bc8d..ddc1599 100644
--- a/lib/safe-browsing/ephy-gsb-service.c
+++ b/lib/safe-browsing/ephy-gsb-service.c
@@ -57,6 +57,37 @@ static GParamSpec *obj_properties[LAST_PROP];
static gboolean ephy_gsb_service_update (EphyGSBService *self);
+typedef struct {
+ EphyGSBService *service;
+ GList *prefixes;
+} FindFullHashesData;
+
+static FindFullHashesData *
+find_full_hashes_data_new (EphyGSBService *service,
+ GList *prefixes)
+{
+ FindFullHashesData *data;
+
+ g_assert (EPHY_IS_GSB_SERVICE (service));
+ g_assert (prefixes);
+
+ data = g_slice_new (FindFullHashesData);
+ data->service = g_object_ref (service);
+ data->prefixes = g_list_copy_deep (prefixes, (GCopyFunc)g_bytes_ref, NULL);
+
+ return data;
+}
+
+static void
+find_full_hashes_data_free (FindFullHashesData *data)
+{
+ g_assert (data);
+
+ g_object_unref (data->service);
+ g_list_free_full (data->prefixes, (GDestroyNotify)g_bytes_unref);
+ g_slice_free (FindFullHashesData, data);
+}
+
static inline gboolean
json_object_has_non_null_string_member (JsonObject *object,
const char *member)
@@ -66,6 +97,14 @@ json_object_has_non_null_string_member (JsonObject *object,
}
static inline gboolean
+json_object_has_non_null_object_member (JsonObject *object,
+ const char *member)
+{
+ return json_object_has_member (object, member) &&
+ json_object_get_object_member (object, member) != NULL;
+}
+
+static inline gboolean
json_object_has_non_null_array_member (JsonObject *object,
const char *member)
{
@@ -391,3 +430,119 @@ ephy_gsb_service_new (const char *api_key,
return service;
}
+
+static void
+ephy_gsb_service_find_full_hashes_cb (SoupSession *session,
+ SoupMessage *msg,
+ gpointer user_data)
+{
+ FindFullHashesData *data = (FindFullHashesData *)user_data;
+ JsonNode *body_node = NULL;
+ JsonObject *body_obj;
+ JsonArray *matches;
+ GError *error = NULL;
+ const char *negative_duration;
+ double duration;
+
+ if (msg->status_code != 200) {
+ LOG ("Cannot update full hashes. Server responded: %u, %s",
+ msg->status_code, msg->response_body->data);
+ goto out;
+ }
+
+ body_node = json_from_string (msg->response_body->data, &error);
+ if (error) {
+ LOG ("Cannot update full hashes. Response is not a valid JSON: %s", error->message);
+ g_error_free (error);
+ goto out;
+ }
+
+ body_obj = json_node_get_object (body_node);
+ matches = json_object_get_array_member (body_obj, "matches");
+
+ for (guint i = 0; i < json_array_get_length (matches); i++) {
+ EphyGSBThreatList *list;
+ JsonObject *match = json_array_get_object_element (matches, i);
+ const char *threat_type = json_object_get_string_member (match, "threatType");
+ const char *platform_type = json_object_get_string_member (match, "platformType");
+ const char *threat_entry_type = json_object_get_string_member (match, "threatEntryType");
+ JsonObject *threat = json_object_get_object_member (match, "threat");
+ const char *hash_b64 = json_object_get_string_member (threat, "hash");
+ const char *positive_duration;
+ char *malware_threat_type = NULL;
+ guint8 *hash;
+ gsize length;
+
+ list = ephy_gsb_threat_list_new (threat_type, platform_type, threat_entry_type, NULL, 0);
+ hash = g_base64_decode (hash_b64, &length);
+ positive_duration = json_object_get_string_member (match, "cacheDuration");
+ sscanf (positive_duration, "%lfs", &duration);
+
+ if (json_object_has_non_null_object_member (match, "threatEntryMetadata")) {
+ JsonObject *threat_entry_metadata = json_object_get_object_member (match, "threatEntryMetadata");
+ malware_threat_type = ephy_gsb_utils_get_metadata_entry (threat_entry_metadata, "malware_threat_type");
+ }
+
+ ephy_gsb_storage_insert_full_hash (data->service->storage,
+ list, hash,
+ floor (duration),
+ malware_threat_type);
+
+ g_free (hash);
+ g_free (malware_threat_type);
+ ephy_gsb_threat_list_free (list);
+ }
+
+ /* Update negative cache duration. */
+ negative_duration = json_object_get_string_member (body_obj, "negativeCacheDuration");
+ sscanf (negative_duration, "%lfs", &duration);
+ for (GList *l = data->prefixes; l && l->data; l = l->next) {
+ ephy_gsb_storage_update_hash_prefix_expiration (data->service->storage,
+ l->data,
+ floor (duration));
+ }
+
+ /* TODO: Handle minimumWaitDuration. */
+
+out:
+ if (body_node)
+ json_node_unref (body_node);
+ find_full_hashes_data_free (data);
+}
+
+static void
+ephy_gsb_service_find_full_hashes (EphyGSBService *self,
+ GList *prefixes)
+{
+ SoupMessage *msg;
+ GList *threat_lists;
+ JsonNode *body_node;
+ JsonObject *body_obj;
+ char *url;
+ char *body;
+
+ g_assert (EPHY_IS_GSB_SERVICE (self));
+ g_assert (ephy_gsb_storage_is_operable (self->storage));
+ g_assert (prefixes);
+
+ LOG ("Updating full hashes of %u prefixes", g_list_length (prefixes));
+
+ threat_lists = ephy_gsb_storage_get_threat_lists (self->storage);
+ body_obj = ephy_gsb_utils_make_full_hashes_request (threat_lists, prefixes);
+
+ body_node = json_node_new (JSON_NODE_OBJECT);
+ json_node_set_object (body_node, body_obj);
+ body = json_to_string (body_node, TRUE);
+
+ url = g_strdup_printf ("%sfullHashes:find?key=%s", API_PREFIX, self->api_key);
+ msg = soup_message_new (SOUP_METHOD_POST, url);
+ soup_message_set_request (msg, "application/json", SOUP_MEMORY_TAKE, body, strlen (body));
+ soup_session_queue_message (self->session, msg,
+ ephy_gsb_service_find_full_hashes_cb,
+ find_full_hashes_data_new (self, prefixes));
+
+ g_free (url);
+ json_object_unref (body_obj);
+ json_node_unref (body_node);
+ g_list_free_full (threat_lists, (GDestroyNotify)ephy_gsb_threat_list_free);
+}
diff --git a/lib/safe-browsing/ephy-gsb-storage.c b/lib/safe-browsing/ephy-gsb-storage.c
index 67a9bff..d41ea02 100644
--- a/lib/safe-browsing/ephy-gsb-storage.c
+++ b/lib/safe-browsing/ephy-gsb-storage.c
@@ -1348,3 +1348,50 @@ out:
if (error)
g_error_free (error);
}
+
+void
+ephy_gsb_storage_update_hash_prefix_expiration (EphyGSBStorage *self,
+ GBytes *prefix,
+ gint64 duration)
+{
+ EphySQLiteStatement *statement = NULL;
+ GError *error = NULL;
+ const char *sql;
+
+ g_assert (EPHY_IS_GSB_STORAGE (self));
+ g_assert (self->is_operable);
+ g_assert (prefix);
+
+ sql = "UPDATE hash_prefix "
+ "SET negative_expires_at=(CAST(strftime('%s', 'now') AS INT)) + ? "
+ "WHERE value=?";
+ statement = ephy_sqlite_connection_create_statement (self->db, sql, &error);
+ if (error) {
+ g_warning ("Failed to create update hash prefix statement: %s", error->message);
+ goto out;
+ }
+
+ ephy_sqlite_statement_bind_int64 (statement, 0, duration, &error);
+ if (error) {
+ g_warning ("Failed to bind int64 in update hash prefix statement: %s", error->message);
+ goto out;
+ }
+ ephy_sqlite_statement_bind_blob (statement, 1,
+ g_bytes_get_data (prefix, NULL),
+ g_bytes_get_size (prefix),
+ &error);
+ if (error) {
+ g_warning ("Failed to bind blob in update hash prefix statement: %s", error->message);
+ goto out;
+ }
+
+ ephy_sqlite_statement_step (statement, &error);
+ if (error)
+ g_warning ("Failed to execute update hash prefix statement: %s", error->message);
+
+out:
+ if (statement)
+ g_object_unref (statement);
+ if (error)
+ g_error_free (error);
+}
diff --git a/lib/safe-browsing/ephy-gsb-storage.h b/lib/safe-browsing/ephy-gsb-storage.h
index 6f64f2e..4cd5981 100644
--- a/lib/safe-browsing/ephy-gsb-storage.h
+++ b/lib/safe-browsing/ephy-gsb-storage.h
@@ -30,34 +30,37 @@ G_BEGIN_DECLS
G_DECLARE_FINAL_TYPE (EphyGSBStorage, ephy_gsb_storage, EPHY, GSB_STORAGE, GObject)
-EphyGSBStorage *ephy_gsb_storage_new (const char *db_path);
-gboolean ephy_gsb_storage_is_operable (EphyGSBStorage *self);
-gint64 ephy_gsb_storage_get_next_update_time (EphyGSBStorage *self);
-void ephy_gsb_storage_set_next_update_time (EphyGSBStorage *self,
- gint64 next_update_time);
-GList *ephy_gsb_storage_get_threat_lists (EphyGSBStorage *self);
-char *ephy_gsb_storage_compute_checksum (EphyGSBStorage *self,
- EphyGSBThreatList *list);
-void ephy_gsb_storage_update_client_state (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- gboolean clear);
-void ephy_gsb_storage_clear_hash_prefixes (EphyGSBStorage *self,
- EphyGSBThreatList *list);
-void ephy_gsb_storage_delete_hash_prefixes (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- JsonArray *indices);
-void ephy_gsb_storage_insert_hash_prefixes (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- gsize prefix_len,
- const char *prefixes_b64);
-GList *ephy_gsb_storage_lookup_hash_prefixes (EphyGSBStorage *self,
- GList *cues);
-GList *ephy_gsb_storage_lookup_full_hashes (EphyGSBStorage *self,
- GList *hashes);
-void ephy_gsb_storage_insert_full_hash (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- const guint8 *hash,
- gint64 duration,
- const char *malware_threat_type);
+EphyGSBStorage *ephy_gsb_storage_new (const char *db_path);
+gboolean ephy_gsb_storage_is_operable (EphyGSBStorage *self);
+gint64 ephy_gsb_storage_get_next_update_time (EphyGSBStorage *self);
+void ephy_gsb_storage_set_next_update_time (EphyGSBStorage *self,
+ gint64 next_update_time);
+GList *ephy_gsb_storage_get_threat_lists (EphyGSBStorage *self);
+char *ephy_gsb_storage_compute_checksum (EphyGSBStorage *self,
+ EphyGSBThreatList *list);
+void ephy_gsb_storage_update_client_state (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ gboolean clear);
+void ephy_gsb_storage_clear_hash_prefixes (EphyGSBStorage *self,
+ EphyGSBThreatList *list);
+void ephy_gsb_storage_delete_hash_prefixes (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ JsonArray *indices);
+void ephy_gsb_storage_insert_hash_prefixes (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ gsize prefix_len,
+ const char *prefixes_b64);
+GList *ephy_gsb_storage_lookup_hash_prefixes (EphyGSBStorage *self,
+ GList *cues);
+GList *ephy_gsb_storage_lookup_full_hashes (EphyGSBStorage *self,
+ GList *hashes);
+void ephy_gsb_storage_insert_full_hash (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ const guint8 *hash,
+ gint64 duration,
+ const char *malware_threat_type);
+void ephy_gsb_storage_update_hash_prefix_expiration (EphyGSBStorage *self,
+ GBytes *prefix,
+ gint64 duration);
G_END_DECLS
diff --git a/lib/safe-browsing/ephy-gsb-utils.c b/lib/safe-browsing/ephy-gsb-utils.c
index 14a1167..c2bf37f 100644
--- a/lib/safe-browsing/ephy-gsb-utils.c
+++ b/lib/safe-browsing/ephy-gsb-utils.c
@@ -200,6 +200,125 @@ ephy_gsb_utils_make_list_updates_request (GList *threat_lists)
return retval;
}
+JsonObject *
+ephy_gsb_utils_make_full_hashes_request (GList *threat_lists,
+ GList *hash_prefixes)
+{
+ GHashTable *threat_types_set;
+ GHashTable *platform_types_set;
+ GHashTable *threat_entry_types_set;
+ GList *threat_types_list;
+ GList *platform_types_list;
+ GList *threat_entry_types_list;
+ JsonArray *threat_types;
+ JsonArray *platform_types;
+ JsonArray *threat_entry_types;
+ JsonArray *threat_entries;
+ JsonArray *client_states;
+ JsonObject *threat_info;
+ JsonObject *retval;
+
+ g_assert (threat_lists);
+ g_assert (hash_prefixes);
+
+ client_states = json_array_new ();
+ threat_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+ platform_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+ threat_entry_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+
+ for (GList *l = threat_lists; l && l->data; l = l->next) {
+ EphyGSBThreatList *list = (EphyGSBThreatList *)l->data;
+
+ if (!g_hash_table_contains (threat_types_set, list->threat_type))
+ g_hash_table_add (threat_types_set, list->threat_type);
+ if (!g_hash_table_contains (platform_types_set, list->platform_type))
+ g_hash_table_add (platform_types_set, list->platform_type);
+ if (!g_hash_table_contains (threat_entry_types_set, list->threat_entry_type))
+ g_hash_table_add (threat_entry_types_set, list->threat_entry_type);
+
+ json_array_add_string_element (client_states, list->client_state);
+ }
+
+ threat_types = json_array_new ();
+ threat_types_list = g_hash_table_get_keys (threat_types_set);
+ for (GList *l = threat_types_list; l && l->data; l = l->next)
+ json_array_add_string_element (threat_types, (const char *)l->data);
+
+ platform_types = json_array_new ();
+ platform_types_list = g_hash_table_get_keys (platform_types_set);
+ for (GList *l = platform_types_list; l && l->data; l = l->next)
+ json_array_add_string_element (platform_types, (const char *)l->data);
+
+ threat_entry_types = json_array_new ();
+ threat_entry_types_list = g_hash_table_get_keys (threat_entry_types_set);
+ for (GList *l = threat_entry_types_list; l && l->data; l = l->next)
+ json_array_add_string_element (threat_entry_types, (const char *)l->data);
+
+ threat_entries = json_array_new ();
+ for (GList *l = hash_prefixes; l && l->data; l = l->next) {
+ JsonObject *threat_entry = json_object_new ();
+ char *hash = g_base64_encode (g_bytes_get_data (l->data, NULL),
+ g_bytes_get_size (l->data));
+
+ json_object_set_string_member (threat_entry, "hash", hash);
+ json_array_add_object_element (threat_entries, threat_entry);
+
+ g_free (hash);
+ }
+
+ threat_info = json_object_new ();
+ json_object_set_array_member (threat_info, "threatTypes", threat_types);
+ json_object_set_array_member (threat_info, "platformTypes", platform_types);
+ json_object_set_array_member (threat_info, "threatEntryTypes", threat_entry_types);
+ json_object_set_array_member (threat_info, "threatEntries", threat_entries);
+
+ retval = json_object_new ();
+ json_object_set_object_member (retval, "client", ephy_gsb_utils_make_client_info ());
+ json_object_set_array_member (retval, "clientStates", client_states);
+ json_object_set_object_member (retval, "threatInfo", threat_info);
+ json_object_set_null_member (retval, "apiClient");
+
+ g_list_free (threat_types_list);
+ g_list_free (platform_types_list);
+ g_list_free (threat_entry_types_list);
+ g_hash_table_unref (threat_types_set);
+ g_hash_table_unref (platform_types_set);
+ g_hash_table_unref (threat_entry_types_set);
+
+ return retval;
+}
+
+char *
+ephy_gsb_utils_get_metadata_entry (JsonObject *threat_entry_metadata,
+ const char *metadata_key)
+{
+ JsonArray *entries;
+ gsize length;
+
+ g_assert (threat_entry_metadata);
+ g_assert (metadata_key);
+
+ if (!json_object_has_member (threat_entry_metadata, "entries"))
+ return NULL;
+
+ entries = json_object_get_array_member (threat_entry_metadata, "entries");
+ for (guint i = 0; i < json_array_get_length (entries); i++) {
+ JsonObject *entry = json_array_get_object_element (entries, i);
+ const char *key_b64 = json_object_get_string_member (entry, "key");
+ const char *value_b64 = json_object_get_string_member (entry, "value");
+ char *key = (char *)g_base64_decode (key_b64, &length);
+
+ if (!g_strcmp0 (key, metadata_key)) {
+ g_free (key);
+ return (char *)g_base64_decode (value_b64, &length);
+ }
+
+ g_free (key);
+ }
+
+ return NULL;
+}
+
static char *
ephy_gsb_utils_full_unescape (const char *part)
{
diff --git a/lib/safe-browsing/ephy-gsb-utils.h b/lib/safe-browsing/ephy-gsb-utils.h
index 9b69cc1..079f071 100644
--- a/lib/safe-browsing/ephy-gsb-utils.h
+++ b/lib/safe-browsing/ephy-gsb-utils.h
@@ -77,6 +77,10 @@ EphyGSBHashFullLookup *ephy_gsb_hash_full_lookup_new (const guint8
void ephy_gsb_hash_full_lookup_free (EphyGSBHashFullLookup *lookup);
JsonObject *ephy_gsb_utils_make_list_updates_request (GList *threat_lists);
+JsonObject *ephy_gsb_utils_make_full_hashes_request (GList *threat_lists,
+ GList *hash_prefixes);
+char *ephy_gsb_utils_get_metadata_entry (JsonObject *threat_entry_metadata,
+ const char *metadata_key);
char *ephy_gsb_utils_canonicalize (const char *url,
char **host_out,
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
