[epiphany/wip/google-safe-browsing: 10/12] gsb-service: Add function to update full hashes
- From: Gabriel Ivașcu <gabrielivascu src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [epiphany/wip/google-safe-browsing: 10/12] gsb-service: Add function to update full hashes
- Date: Mon, 18 Sep 2017 17:26:08 +0000 (UTC)
commit 74c893423c23173994e3b50de9495ada7c16cf6a
Author: Gabriel Ivascu <gabrielivascu gnome org>
Date: Sun Sep 17 17:40:42 2017 +0300
gsb-service: Add function to update full hashes
lib/safe-browsing/ephy-gsb-service.c | 187 ++++++++++++++++++++++++++++++---
lib/safe-browsing/ephy-gsb-storage.c | 47 +++++++++
lib/safe-browsing/ephy-gsb-storage.h | 61 ++++++-----
lib/safe-browsing/ephy-gsb-utils.c | 119 +++++++++++++++++++++
lib/safe-browsing/ephy-gsb-utils.h | 4 +
5 files changed, 372 insertions(+), 46 deletions(-)
---
diff --git a/lib/safe-browsing/ephy-gsb-service.c b/lib/safe-browsing/ephy-gsb-service.c
index 692e9b1..8f18355 100644
--- a/lib/safe-browsing/ephy-gsb-service.c
+++ b/lib/safe-browsing/ephy-gsb-service.c
@@ -55,7 +55,38 @@ enum {
static GParamSpec *obj_properties[LAST_PROP];
-static gboolean ephy_gsb_service_update (EphyGSBService *self);
+static gboolean ephy_gsb_service_update_db (EphyGSBService *self);
+
+typedef struct {
+ EphyGSBService *service;
+ GList *prefixes;
+} UpdateFullHashesAsyncData;
+
+static UpdateFullHashesAsyncData *
+update_full_hashes_async_data_new (EphyGSBService *service,
+ GList *prefixes)
+{
+ UpdateFullHashesAsyncData *data;
+
+ g_assert (EPHY_IS_GSB_SERVICE (service));
+ g_assert (prefixes);
+
+ data = g_slice_new (UpdateFullHashesAsyncData);
+ data->service = g_object_ref (service);
+ data->prefixes = g_list_copy_deep (prefixes, (GCopyFunc)g_bytes_ref, NULL);
+
+ return data;
+}
+
+static void
+update_full_hashes_async_data_free (UpdateFullHashesAsyncData *data)
+{
+ g_assert (data);
+
+ g_object_unref (data->service);
+ g_list_free_full (data->prefixes, (GDestroyNotify)g_bytes_unref);
+ g_slice_free (UpdateFullHashesAsyncData, data);
+}
static inline gboolean
json_object_has_non_null_string_member (JsonObject *object,
@@ -66,6 +97,14 @@ json_object_has_non_null_string_member (JsonObject *object,
}
static inline gboolean
+json_object_has_non_null_object_member (JsonObject *object,
+ const char *member)
+{
+ return json_object_has_member (object, member) &&
+ json_object_get_object_member (object, member) != NULL;
+}
+
+static inline gboolean
json_object_has_non_null_array_member (JsonObject *object,
const char *member)
{
@@ -74,24 +113,24 @@ json_object_has_non_null_array_member (JsonObject *object,
}
static void
-ephy_gsb_service_schedule_update (EphyGSBService *self,
- gint64 interval)
+ephy_gsb_service_schedule_update_db (EphyGSBService *self,
+ gint64 interval)
{
g_assert (EPHY_IS_GSB_SERVICE (self));
g_assert (ephy_gsb_storage_is_operable (self->storage));
g_assert (interval > 0);
self->source_id = g_timeout_add_seconds (interval,
- (GSourceFunc)ephy_gsb_service_update,
+ (GSourceFunc)ephy_gsb_service_update_db,
self);
LOG ("Next update scheduled in %ld seconds", interval);
}
static void
-ephy_gsb_service_update_thread (GTask *task,
- EphyGSBService *self,
- gpointer task_data,
- GCancellable *cancellable)
+ephy_gsb_service_update_db_thread (GTask *task,
+ EphyGSBService *self,
+ gpointer task_data,
+ GCancellable *cancellable)
{
GError *error = NULL;
JsonNode *body_node;
@@ -222,19 +261,19 @@ out:
}
static void
-ephy_gsb_service_update_finished_cb (EphyGSBService *self,
- GAsyncResult *result,
- gpointer user_data)
+ephy_gsb_service_update_db_finished_cb (EphyGSBService *self,
+ GAsyncResult *result,
+ gpointer user_data)
{
gint64 next_update_time;
next_update_time = g_task_propagate_int (G_TASK (result), NULL);
- ephy_gsb_service_schedule_update (self, next_update_time - CURRENT_TIME);
+ ephy_gsb_service_schedule_update_db (self, next_update_time - CURRENT_TIME);
self->is_updating = FALSE;
}
static gboolean
-ephy_gsb_service_update (EphyGSBService *self)
+ephy_gsb_service_update_db (EphyGSBService *self)
{
GTask *task;
@@ -243,15 +282,129 @@ ephy_gsb_service_update (EphyGSBService *self)
self->is_updating = TRUE;
task = g_task_new (self, NULL,
- (GAsyncReadyCallback)ephy_gsb_service_update_finished_cb,
+ (GAsyncReadyCallback)ephy_gsb_service_update_db_finished_cb,
NULL);
- g_task_run_in_thread (task, (GTaskThreadFunc)ephy_gsb_service_update_thread);
+ g_task_run_in_thread (task, (GTaskThreadFunc)ephy_gsb_service_update_db_thread);
g_object_unref (task);
return G_SOURCE_REMOVE;
}
static void
+ephy_gsb_service_update_full_hashes_cb (SoupSession *session,
+ SoupMessage *msg,
+ gpointer user_data)
+{
+ UpdateFullHashesAsyncData *data = user_data;
+ JsonNode *body_node;
+ JsonObject *body_obj;
+ JsonArray *matches;
+ GError *error = NULL;
+ const char *negative_duration;
+ double duration;
+
+ if (msg->status_code != 200) {
+ LOG ("Cannot update full hashes. Server responded: %u, %s",
+ msg->status_code, msg->response_body->data);
+ return;
+ }
+
+ body_node = json_from_string (msg->response_body->data, &error);
+ if (error) {
+ LOG ("Cannot update full hashes. Response is not a valid JSON: %s", error->message);
+ g_error_free (error);
+ return;
+ }
+
+ body_obj = json_node_get_object (body_node);
+ matches = json_object_get_array_member (body_obj, "matches");
+
+ for (guint i = 0; i < json_array_get_length (matches); i++) {
+ EphyGSBThreatList *list;
+ JsonObject *match = json_array_get_object_element (matches, i);
+ const char *threat_type = json_object_get_string_member (match, "threatType");
+ const char *platform_type = json_object_get_string_member (match, "platformType");
+ const char *threat_entry_type = json_object_get_string_member (match, "threatEntryType");
+ JsonObject *threat = json_object_get_object_member (match, "threat");
+ const char *hash_b64 = json_object_get_string_member (threat, "hash");
+ const char *positive_duration;
+ char *malware_threat_type = NULL;
+ guint8 *hash;
+ gsize length;
+
+ list = ephy_gsb_threat_list_new (threat_type, platform_type, threat_entry_type, NULL, 0);
+ hash = g_base64_decode (hash_b64, &length);
+ positive_duration = json_object_get_string_member (match, "cacheDuration");
+ sscanf (positive_duration, "%lfs", &duration);
+
+ if (json_object_has_non_null_object_member (match, "threatEntryMetadata")) {
+ JsonObject *threat_entry_metadata = json_object_get_object_member (match, "threatEntryMetadata");
+ malware_threat_type = ephy_gsb_utils_get_metadata_entry (threat_entry_metadata, "malware_threat_type");
+ }
+
+ ephy_gsb_storage_insert_full_hash (data->service->storage,
+ list, hash,
+ floor (duration),
+ malware_threat_type);
+
+ g_free (hash);
+ g_free (malware_threat_type);
+ ephy_gsb_threat_list_free (list);
+ }
+
+ /* Update negative cache duration. */
+ negative_duration = json_object_get_string_member (body_obj, "negativeCacheDuration");
+ sscanf (negative_duration, "%lfs", &duration);
+ for (GList *l = data->prefixes; l && l->data; l = l->next) {
+ ephy_gsb_storage_update_hash_prefix_expiration (data->service->storage,
+ l->data,
+ floor (duration));
+ }
+
+ /* TODO: Handle minimumWaitDuration. */
+
+ json_node_unref (body_node);
+ update_full_hashes_async_data_free (data);
+}
+
+static void
+ephy_gsb_service_update_full_hashes (EphyGSBService *self,
+ GList *prefixes)
+{
+ SoupMessage *msg;
+ GList *threat_lists;
+ JsonNode *body_node;
+ JsonObject *body_obj;
+ char *url;
+ char *body;
+
+ g_assert (EPHY_IS_GSB_SERVICE (self));
+ g_assert (ephy_gsb_storage_is_operable (self->storage));
+ g_assert (prefixes);
+
+ LOG ("Updating full hashes of %u prefixes", g_list_length (prefixes));
+
+ threat_lists = ephy_gsb_storage_get_threat_lists (self->storage);
+ body_obj = ephy_gsb_utils_make_full_hashes_request (threat_lists, prefixes);
+
+ body_node = json_node_new (JSON_NODE_OBJECT);
+ json_node_set_object (body_node, body_obj);
+ body = json_to_string (body_node, TRUE);
+
+ url = g_strdup_printf ("%sfullHashes:find?key=%s", API_PREFIX, self->api_key);
+ msg = soup_message_new (SOUP_METHOD_POST, url);
+ soup_message_set_request (msg, "application/json", SOUP_MEMORY_TAKE, body, strlen (body));
+ soup_session_queue_message (self->session, msg,
+ ephy_gsb_service_update_full_hashes_cb,
+ update_full_hashes_async_data_new (self, prefixes));
+
+ g_free (url);
+ json_object_unref (body_obj);
+ json_node_unref (body_node);
+ g_list_free_full (threat_lists, (GDestroyNotify)ephy_gsb_threat_list_free);
+}
+
+static void
ephy_gsb_service_set_property (GObject *object,
guint prop_id,
const GValue *value,
@@ -334,9 +487,9 @@ ephy_gsb_service_constructed (GObject *object)
interval = ephy_gsb_storage_get_next_update_time (self->storage) - CURRENT_TIME;
if (interval <= 0)
- ephy_gsb_service_update (self);
+ ephy_gsb_service_update_db (self);
else
- ephy_gsb_service_schedule_update (self, interval);
+ ephy_gsb_service_schedule_update_db (self, interval);
}
static void
diff --git a/lib/safe-browsing/ephy-gsb-storage.c b/lib/safe-browsing/ephy-gsb-storage.c
index 1e87fa3..6038523 100644
--- a/lib/safe-browsing/ephy-gsb-storage.c
+++ b/lib/safe-browsing/ephy-gsb-storage.c
@@ -1350,3 +1350,50 @@ out:
if (error)
g_error_free (error);
}
+
+void
+ephy_gsb_storage_update_hash_prefix_expiration (EphyGSBStorage *self,
+ GBytes *prefix,
+ gint64 duration)
+{
+ EphySQLiteStatement *statement = NULL;
+ GError *error = NULL;
+ const char *sql;
+
+ g_assert (EPHY_IS_GSB_STORAGE (self));
+ g_assert (self->is_operable);
+ g_assert (prefix);
+
+ sql = "UPDATE hash_prefix "
+ "SET negative_expires_at=(CAST(strftime('%s', 'now') AS INT)) + ? "
+ "WHERE value=?";
+ statement = ephy_sqlite_connection_create_statement (self->db, sql, &error);
+ if (error) {
+ g_warning ("Failed to create update hash prefix statement: %s", error->message);
+ goto out;
+ }
+
+ ephy_sqlite_statement_bind_int64 (statement, 0, duration, &error);
+ if (error) {
+ g_warning ("Failed to bind int64 in update hash prefix statement: %s", error->message);
+ goto out;
+ }
+ ephy_sqlite_statement_bind_blob (statement, 1,
+ g_bytes_get_data (prefix, NULL),
+ g_bytes_get_size (prefix),
+ &error);
+ if (error) {
+ g_warning ("Failed to bind blob in update hash prefix statement: %s", error->message);
+ goto out;
+ }
+
+ ephy_sqlite_statement_step (statement, &error);
+ if (error)
+ g_warning ("Failed to execute update hash prefix statement: %s", error->message);
+
+out:
+ if (statement)
+ g_object_unref (statement);
+ if (error)
+ g_error_free (error);
+}
diff --git a/lib/safe-browsing/ephy-gsb-storage.h b/lib/safe-browsing/ephy-gsb-storage.h
index 6f64f2e..4cd5981 100644
--- a/lib/safe-browsing/ephy-gsb-storage.h
+++ b/lib/safe-browsing/ephy-gsb-storage.h
@@ -30,34 +30,37 @@ G_BEGIN_DECLS
G_DECLARE_FINAL_TYPE (EphyGSBStorage, ephy_gsb_storage, EPHY, GSB_STORAGE, GObject)
-EphyGSBStorage *ephy_gsb_storage_new (const char *db_path);
-gboolean ephy_gsb_storage_is_operable (EphyGSBStorage *self);
-gint64 ephy_gsb_storage_get_next_update_time (EphyGSBStorage *self);
-void ephy_gsb_storage_set_next_update_time (EphyGSBStorage *self,
- gint64 next_update_time);
-GList *ephy_gsb_storage_get_threat_lists (EphyGSBStorage *self);
-char *ephy_gsb_storage_compute_checksum (EphyGSBStorage *self,
- EphyGSBThreatList *list);
-void ephy_gsb_storage_update_client_state (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- gboolean clear);
-void ephy_gsb_storage_clear_hash_prefixes (EphyGSBStorage *self,
- EphyGSBThreatList *list);
-void ephy_gsb_storage_delete_hash_prefixes (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- JsonArray *indices);
-void ephy_gsb_storage_insert_hash_prefixes (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- gsize prefix_len,
- const char *prefixes_b64);
-GList *ephy_gsb_storage_lookup_hash_prefixes (EphyGSBStorage *self,
- GList *cues);
-GList *ephy_gsb_storage_lookup_full_hashes (EphyGSBStorage *self,
- GList *hashes);
-void ephy_gsb_storage_insert_full_hash (EphyGSBStorage *self,
- EphyGSBThreatList *list,
- const guint8 *hash,
- gint64 duration,
- const char *malware_threat_type);
+EphyGSBStorage *ephy_gsb_storage_new (const char *db_path);
+gboolean ephy_gsb_storage_is_operable (EphyGSBStorage *self);
+gint64 ephy_gsb_storage_get_next_update_time (EphyGSBStorage *self);
+void ephy_gsb_storage_set_next_update_time (EphyGSBStorage *self,
+ gint64 next_update_time);
+GList *ephy_gsb_storage_get_threat_lists (EphyGSBStorage *self);
+char *ephy_gsb_storage_compute_checksum (EphyGSBStorage *self,
+ EphyGSBThreatList *list);
+void ephy_gsb_storage_update_client_state (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ gboolean clear);
+void ephy_gsb_storage_clear_hash_prefixes (EphyGSBStorage *self,
+ EphyGSBThreatList *list);
+void ephy_gsb_storage_delete_hash_prefixes (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ JsonArray *indices);
+void ephy_gsb_storage_insert_hash_prefixes (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ gsize prefix_len,
+ const char *prefixes_b64);
+GList *ephy_gsb_storage_lookup_hash_prefixes (EphyGSBStorage *self,
+ GList *cues);
+GList *ephy_gsb_storage_lookup_full_hashes (EphyGSBStorage *self,
+ GList *hashes);
+void ephy_gsb_storage_insert_full_hash (EphyGSBStorage *self,
+ EphyGSBThreatList *list,
+ const guint8 *hash,
+ gint64 duration,
+ const char *malware_threat_type);
+void ephy_gsb_storage_update_hash_prefix_expiration (EphyGSBStorage *self,
+ GBytes *prefix,
+ gint64 duration);
G_END_DECLS
diff --git a/lib/safe-browsing/ephy-gsb-utils.c b/lib/safe-browsing/ephy-gsb-utils.c
index 14a1167..c2bf37f 100644
--- a/lib/safe-browsing/ephy-gsb-utils.c
+++ b/lib/safe-browsing/ephy-gsb-utils.c
@@ -200,6 +200,125 @@ ephy_gsb_utils_make_list_updates_request (GList *threat_lists)
return retval;
}
+JsonObject *
+ephy_gsb_utils_make_full_hashes_request (GList *threat_lists,
+ GList *hash_prefixes)
+{
+ GHashTable *threat_types_set;
+ GHashTable *platform_types_set;
+ GHashTable *threat_entry_types_set;
+ GList *threat_types_list;
+ GList *platform_types_list;
+ GList *threat_entry_types_list;
+ JsonArray *threat_types;
+ JsonArray *platform_types;
+ JsonArray *threat_entry_types;
+ JsonArray *threat_entries;
+ JsonArray *client_states;
+ JsonObject *threat_info;
+ JsonObject *retval;
+
+ g_assert (threat_lists);
+ g_assert (hash_prefixes);
+
+ client_states = json_array_new ();
+ threat_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+ platform_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+ threat_entry_types_set = g_hash_table_new (g_str_hash, g_str_equal);
+
+ for (GList *l = threat_lists; l && l->data; l = l->next) {
+ EphyGSBThreatList *list = (EphyGSBThreatList *)l->data;
+
+ if (!g_hash_table_contains (threat_types_set, list->threat_type))
+ g_hash_table_add (threat_types_set, list->threat_type);
+ if (!g_hash_table_contains (platform_types_set, list->platform_type))
+ g_hash_table_add (platform_types_set, list->platform_type);
+ if (!g_hash_table_contains (threat_entry_types_set, list->threat_entry_type))
+ g_hash_table_add (threat_entry_types_set, list->threat_entry_type);
+
+ json_array_add_string_element (client_states, list->client_state);
+ }
+
+ threat_types = json_array_new ();
+ threat_types_list = g_hash_table_get_keys (threat_types_set);
+ for (GList *l = threat_types_list; l && l->data; l = l->next)
+ json_array_add_string_element (threat_types, (const char *)l->data);
+
+ platform_types = json_array_new ();
+ platform_types_list = g_hash_table_get_keys (platform_types_set);
+ for (GList *l = platform_types_list; l && l->data; l = l->next)
+ json_array_add_string_element (platform_types, (const char *)l->data);
+
+ threat_entry_types = json_array_new ();
+ threat_entry_types_list = g_hash_table_get_keys (threat_entry_types_set);
+ for (GList *l = threat_entry_types_list; l && l->data; l = l->next)
+ json_array_add_string_element (threat_entry_types, (const char *)l->data);
+
+ threat_entries = json_array_new ();
+ for (GList *l = hash_prefixes; l && l->data; l = l->next) {
+ JsonObject *threat_entry = json_object_new ();
+ char *hash = g_base64_encode (g_bytes_get_data (l->data, NULL),
+ g_bytes_get_size (l->data));
+
+ json_object_set_string_member (threat_entry, "hash", hash);
+ json_array_add_object_element (threat_entries, threat_entry);
+
+ g_free (hash);
+ }
+
+ threat_info = json_object_new ();
+ json_object_set_array_member (threat_info, "threatTypes", threat_types);
+ json_object_set_array_member (threat_info, "platformTypes", platform_types);
+ json_object_set_array_member (threat_info, "threatEntryTypes", threat_entry_types);
+ json_object_set_array_member (threat_info, "threatEntries", threat_entries);
+
+ retval = json_object_new ();
+ json_object_set_object_member (retval, "client", ephy_gsb_utils_make_client_info ());
+ json_object_set_array_member (retval, "clientStates", client_states);
+ json_object_set_object_member (retval, "threatInfo", threat_info);
+ json_object_set_null_member (retval, "apiClient");
+
+ g_list_free (threat_types_list);
+ g_list_free (platform_types_list);
+ g_list_free (threat_entry_types_list);
+ g_hash_table_unref (threat_types_set);
+ g_hash_table_unref (platform_types_set);
+ g_hash_table_unref (threat_entry_types_set);
+
+ return retval;
+}
+
+char *
+ephy_gsb_utils_get_metadata_entry (JsonObject *threat_entry_metadata,
+ const char *metadata_key)
+{
+ JsonArray *entries;
+ gsize length;
+
+ g_assert (threat_entry_metadata);
+ g_assert (metadata_key);
+
+ if (!json_object_has_member (threat_entry_metadata, "entries"))
+ return NULL;
+
+ entries = json_object_get_array_member (threat_entry_metadata, "entries");
+ for (guint i = 0; i < json_array_get_length (entries); i++) {
+ JsonObject *entry = json_array_get_object_element (entries, i);
+ const char *key_b64 = json_object_get_string_member (entry, "key");
+ const char *value_b64 = json_object_get_string_member (entry, "value");
+ char *key = (char *)g_base64_decode (key_b64, &length);
+
+ if (!g_strcmp0 (key, metadata_key)) {
+ g_free (key);
+ return (char *)g_base64_decode (value_b64, &length);
+ }
+
+ g_free (key);
+ }
+
+ return NULL;
+}
+
static char *
ephy_gsb_utils_full_unescape (const char *part)
{
diff --git a/lib/safe-browsing/ephy-gsb-utils.h b/lib/safe-browsing/ephy-gsb-utils.h
index 9b69cc1..079f071 100644
--- a/lib/safe-browsing/ephy-gsb-utils.h
+++ b/lib/safe-browsing/ephy-gsb-utils.h
@@ -77,6 +77,10 @@ EphyGSBHashFullLookup *ephy_gsb_hash_full_lookup_new (const guint8
void ephy_gsb_hash_full_lookup_free (EphyGSBHashFullLookup *lookup);
JsonObject *ephy_gsb_utils_make_list_updates_request (GList *threat_lists);
+JsonObject *ephy_gsb_utils_make_full_hashes_request (GList *threat_lists,
+ GList *hash_prefixes);
+char *ephy_gsb_utils_get_metadata_entry (JsonObject *threat_entry_metadata,
+ const char *metadata_key);
char *ephy_gsb_utils_canonicalize (const char *url,
char **host_out,
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
