[gnome-online-miners/wip/facebook] src: add a ownCloud miner

[Álvaro Peña <alvaropg src gnome org>]

commit 9aa7bd9e3db86acdbeea70eca33cbab2827b7991
Author: Debarshi Ray <debarshir gnome org>
Date:   Wed Aug 14 16:40:47 2013 +0200

    src: add a ownCloud miner
    
    https://bugzilla.gnome.org/show_bug.cgi?id=706158

 src/Makefile.am               |   25 +++
 src/gom-owncloud-miner-main.c |   33 +++
 src/gom-owncloud-miner.c      |  448 +++++++++++++++++++++++++++++++++++++++++
 src/gom-owncloud-miner.h      |   72 +++++++
 4 files changed, 578 insertions(+), 0 deletions(-)
---
diff --git a/src/Makefile.am b/src/Makefile.am
index 4b104ed..2741cef 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -34,6 +34,7 @@ libexec_PROGRAMS = \
     gom-facebook-miner \
     gom-flickr-miner \
     gom-gdata-miner \
+    gom-owncloud-miner \
     gom-zpj-miner \
     $(NULL)
 
@@ -115,6 +116,30 @@ gom_gdata_miner_LDADD = \
     $(TRACKER_LIBS) \
     $(NULL)
 
+gom_owncloud_miner_SOURCES = \
+    gom-owncloud-miner-main.c \
+    gom-owncloud-miner.c \
+    gom-owncloud-miner.h \
+    $(NULL)
+
+gom_owncloud_miner_CPPFLAGS = \
+    -DG_LOG_DOMAIN=\"Gom\" \
+    -DG_DISABLE_DEPRECATED \
+    -I$(top_srcdir)/src \
+    $(GIO_CFLAGS) \
+    $(GLIB_CFLAGS) \
+    $(GOA_CFLAGS) \
+    $(TRACKER_CFLAGS) \
+    $(NULL)
+
+gom_owncloud_miner_LDADD = \
+    libgom-1.0.la  \
+    $(GIO_LIBS) \
+    $(GLIB_LIBS) \
+    $(GOA_LIBS) \
+    $(TRACKER_LIBS) \
+    $(NULL)
+
 gom_zpj_miner_SOURCES = \
     gom-zpj-miner-main.c \
     gom-zpj-miner.c \
diff --git a/src/gom-owncloud-miner-main.c b/src/gom-owncloud-miner-main.c
new file mode 100644
index 0000000..c7ce506
--- /dev/null
+++ b/src/gom-owncloud-miner-main.c
@@ -0,0 +1,33 @@
+/*
+ * GNOME Online Miners - crawls through your online content
+ * Copyright (c) 2013 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * Author: Debarshi Ray <debarshir gnome org>
+ *
+ */
+
+#include "config.h"
+
+#define INSIDE_MINER
+#define MINER_NAME "OWNCLOUD"
+#define MINER_TYPE GOM_TYPE_OWNCLOUD_MINER
+#define MINER_BUS_NAME "org.gnome.OnlineMiners.Owncloud"
+#define MINER_OBJECT_PATH "/org/gnome/OnlineMiners/Owncloud"
+
+#include "gom-owncloud-miner.h"
+#include "gom-miner-main.c"
diff --git a/src/gom-owncloud-miner.c b/src/gom-owncloud-miner.c
new file mode 100644
index 0000000..1cce24b
--- /dev/null
+++ b/src/gom-owncloud-miner.c
@@ -0,0 +1,448 @@
+/*
+ * GNOME Online Miners - crawls through your online content
+ * Copyright (c) 2013 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * Author: Debarshi Ray <debarshir gnome org>
+ *
+ */
+
+#include "config.h"
+
+#include <goa/goa.h>
+
+#include "gom-owncloud-miner.h"
+#include "gom-utils.h"
+
+#define MINER_IDENTIFIER "gd:owncloud:miner:8a409711-8fea-4eda-a417-f140ffc6d8f3"
+
+G_DEFINE_TYPE (GomOwncloudMiner, gom_owncloud_miner, GOM_TYPE_MINER)
+
+struct _GomOwncloudMinerPrivate {
+  GVolumeMonitor *monitor;
+};
+
+#define FILE_ATTRIBUTES \
+  G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE "," \
+  G_FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME "," \
+  G_FILE_ATTRIBUTE_STANDARD_NAME "," \
+  G_FILE_ATTRIBUTE_STANDARD_TYPE "," \
+  G_FILE_ATTRIBUTE_TIME_MODIFIED
+
+/* GVfs marks regular files on remote shares as UNKNOWN */
+#define FILE_TYPE_NETWORK G_FILE_TYPE_UNKNOWN
+
+typedef struct {
+  GError **error;
+  GMainLoop *loop;
+  GomAccountMinerJob *job;
+} SyncData;
+
+static gboolean
+account_miner_job_process_file (GomAccountMinerJob *job,
+                                GFile *file,
+                                GFileInfo *info,
+                                GFile *parent,
+                                GError **error)
+{
+  GChecksum *checksum = NULL;
+  GDateTime *modification_time;
+  GFileType type;
+  GTimeVal tv;
+  gboolean mtime_changed;
+  gboolean resource_exists;
+  const gchar *class;
+  const gchar *display_name;
+  const gchar *id;
+  const gchar *name;
+  gchar *identifier = NULL;
+  gchar *resource = NULL;
+  gchar *uri = NULL;
+  gint64 new_mtime;
+
+  type = g_file_info_get_file_type (info);
+  uri = g_file_get_uri (file);
+
+  checksum = g_checksum_new (G_CHECKSUM_MD5);
+  g_checksum_update (checksum, uri, -1);
+  id = g_checksum_get_string (checksum);
+  identifier = g_strdup_printf ("%sowncloud:%s", (type == G_FILE_TYPE_DIRECTORY ? "gd:collection:" : ""), 
id);
+  g_checksum_reset (checksum);
+
+  /* remove from the list of the previous resources */
+  g_hash_table_remove (job->previous_resources, identifier);
+
+  name = g_file_info_get_name (info);
+  if (type == FILE_TYPE_NETWORK)
+    class = gom_filename_to_rdf_type (name);
+  else
+    class = "nfo:DataContainer";
+
+  if (class == NULL)
+    goto out;
+
+  resource = gom_tracker_sparql_connection_ensure_resource
+    (job->connection,
+     job->cancellable, error,
+     &resource_exists,
+     job->datasource_urn, identifier,
+     "nfo:RemoteDataObject", class, NULL);
+
+  if (*error != NULL)
+    goto out;
+
+  gom_tracker_update_datasource (job->connection, job->datasource_urn,
+                                 resource_exists, identifier, resource,
+                                 job->cancellable, error);
+
+  if (*error != NULL)
+    goto out;
+
+  g_file_info_get_modification_time (info, &tv);
+  modification_time = g_date_time_new_from_timeval_local (&tv);
+  new_mtime = g_date_time_to_unix (modification_time);
+  mtime_changed = gom_tracker_update_mtime (job->connection, new_mtime,
+                                            resource_exists, identifier, resource,
+                                            job->cancellable, error);
+
+  if (*error != NULL)
+    goto out;
+
+  /* avoid updating the DB if the entry already exists and has not
+   * been modified since our last run.
+   */
+  if (!mtime_changed)
+    goto out;
+
+  /* the resource changed - just set all the properties again */
+  gom_tracker_sparql_connection_insert_or_replace_triple
+    (job->connection,
+     job->cancellable, error,
+     job->datasource_urn, resource,
+     "nie:url", uri);
+
+  if (*error != NULL)
+    goto out;
+
+  if (type == FILE_TYPE_NETWORK)
+    {
+      const gchar *mime;
+      const gchar *parent_id;
+      gchar *parent_identifier;
+      gchar *parent_resource_urn;
+      gchar *parent_uri;
+
+      if (parent != NULL)
+        {
+          parent_uri = g_file_get_uri (parent);
+          g_checksum_update (checksum, parent_uri, -1);
+          parent_id = g_checksum_get_string (checksum);
+          parent_identifier = g_strconcat ("gd:collection:owncloud:", parent_id, NULL);
+          parent_resource_urn = gom_tracker_sparql_connection_ensure_resource
+            (job->connection, job->cancellable, error,
+             NULL,
+             job->datasource_urn, parent_identifier,
+             "nfo:RemoteDataObject", "nfo:DataContainer", NULL);
+          g_checksum_reset (checksum);
+          g_free (parent_identifier);
+          g_free (parent_uri);
+
+          if (*error != NULL)
+            goto out;
+
+          gom_tracker_sparql_connection_insert_or_replace_triple
+            (job->connection,
+             job->cancellable, error,
+             job->datasource_urn, resource,
+             "nie:isPartOf", parent_resource_urn);
+          g_free (parent_resource_urn);
+
+          if (*error != NULL)
+            goto out;
+        }
+
+      mime = g_file_info_get_content_type (info);
+      if (mime != NULL)
+        {
+          gom_tracker_sparql_connection_insert_or_replace_triple
+            (job->connection,
+             job->cancellable, error,
+             job->datasource_urn, resource,
+             "nie:mimeType", mime);
+
+          if (*error != NULL)
+            goto out;
+        }
+    }
+
+  display_name = g_file_info_get_display_name (info);
+  gom_tracker_sparql_connection_insert_or_replace_triple
+    (job->connection,
+     job->cancellable, error,
+     job->datasource_urn, resource,
+     "nfo:fileName", display_name);
+
+  if (*error != NULL)
+    goto out;
+
+ out:
+  if (checksum != NULL)
+    g_checksum_free (checksum);
+  g_free (identifier);
+  g_free (resource);
+  g_free (uri);
+
+  if (*error != NULL)
+    return FALSE;
+
+  return TRUE;
+}
+
+static void
+account_miner_job_traverse_dir (GomAccountMinerJob *job,
+                                GFile *dir,
+                                gboolean is_root,
+                                GError **error)
+{
+  GError *local_error = NULL;
+  GFileEnumerator *enumerator;
+  GFileInfo *info;
+  gchar *dir_uri;
+
+  dir_uri = g_file_get_uri (dir);
+  enumerator = g_file_enumerate_children (dir,
+                                          FILE_ATTRIBUTES,
+                                          G_FILE_QUERY_INFO_NONE,
+                                          job->cancellable,
+                                          &local_error);
+  if (local_error != NULL)
+    goto out;
+
+  while ((info = g_file_enumerator_next_file (enumerator, job->cancellable, &local_error)) != NULL)
+    {
+      GFile *child;
+      GFileType type;
+      const gchar *name;
+      gchar *uri;
+
+      type = g_file_info_get_file_type (info);
+      name = g_file_info_get_name (info);
+      child = g_file_get_child (dir, name);
+
+      if (type == FILE_TYPE_NETWORK || type == G_FILE_TYPE_DIRECTORY)
+        {
+          account_miner_job_process_file (job, child, info, is_root ? NULL : dir, &local_error);
+          if (local_error != NULL)
+            {
+              uri = g_file_get_uri (child);
+              g_warning ("Unable to process %s: %s", uri, local_error->message);
+              g_free (uri);
+              g_clear_error (&local_error);
+            }
+        }
+
+      if (type == G_FILE_TYPE_DIRECTORY)
+        {
+          account_miner_job_traverse_dir (job, child, FALSE, &local_error);
+          if (local_error != NULL)
+            {
+              uri = g_file_get_uri (child);
+              g_warning ("Unable to traverse %s: %s", uri, local_error->message);
+              g_free (uri);
+              g_clear_error (&local_error);
+            }
+        }
+
+      g_object_unref (child);
+      g_object_unref (info);
+    }
+
+  if (local_error != NULL)
+    goto out;
+
+ out:
+  if (local_error != NULL)
+    g_propagate_error (error, local_error);
+
+  g_clear_object (&enumerator);
+  g_free (dir_uri);
+}
+
+static gboolean
+is_matching_volume (GVolume *volume, GoaObject *object)
+{
+  GoaAccount *account;
+  GoaFiles *files;
+  gboolean retval = FALSE;
+  const gchar *presentation_identity;
+  const gchar *uri;
+  gchar *name;
+  gchar *uuid;
+
+  account = goa_object_peek_account (object);
+  presentation_identity = goa_account_get_presentation_identity (account);
+
+  files = goa_object_peek_files (object);
+  uri = goa_files_get_uri (files);
+
+  name = g_volume_get_name (volume);
+  uuid = g_volume_get_uuid (volume);
+
+  if (g_strcmp0 (name, presentation_identity) == 0 && g_strcmp0 (uuid, uri) == 0)
+    retval = TRUE;
+
+  g_free (name);
+  g_free (uuid);
+  return retval;
+}
+
+static void
+volume_added_cb (GVolumeMonitor *monitor, GVolume *volume, gpointer user_data)
+{
+  SyncData *data = (SyncData *) user_data;
+
+  if (!is_matching_volume (volume, GOA_OBJECT (data->job->service)))
+    return;
+
+  g_object_set_data_full (data->job->service, "volume", g_object_ref (volume), g_object_unref);
+  g_main_loop_quit (data->loop);
+}
+
+static void
+volume_mount_cb (GObject *source_object, GAsyncResult *res, gpointer user_data)
+{
+  GVolume *volume = G_VOLUME (source_object);
+  SyncData *data = (SyncData *) user_data;
+
+  g_volume_mount_finish (volume, res, data->error);
+  g_main_loop_quit (data->loop);
+}
+
+static void
+query_owncloud (GomAccountMinerJob *job,
+                GError **error)
+{
+  GomOwncloudMiner *self = GOM_OWNCLOUD_MINER (job->miner);
+  GomOwncloudMinerPrivate *priv = self->priv;
+  GFile *root;
+  GList *l;
+  GList *volumes;
+  GMainContext *context;
+  GMount *mount;
+  GVolume *volume;
+  SyncData data;
+  gboolean found = FALSE;
+
+  data.job = job;
+  volumes = g_volume_monitor_get_volumes (priv->monitor);
+
+  /* find the volume corresponding to this GoaObject */
+  for (l = volumes; l != NULL && !found; l = l->next)
+    {
+      GVolume *volume = G_VOLUME (l->data);
+
+      if (is_matching_volume (volume, GOA_OBJECT (job->service)))
+        {
+          g_object_set_data_full (job->service, "volume", g_object_ref (volume), g_object_unref);
+          found = TRUE;
+        }
+    }
+
+  /* wait for GVfs to add the volume, if it hasn't already */
+  if (!found)
+    {
+      context = g_main_context_get_thread_default ();
+      data.loop = g_main_loop_new (context, FALSE);
+
+      g_signal_connect_object (priv->monitor, "volume-added", G_CALLBACK (volume_added_cb), &data, 0);
+      g_main_loop_run (data.loop);
+      g_main_loop_unref (data.loop);
+    }
+
+  /* mount the volume if needed */
+  volume = G_VOLUME (g_object_get_data (job->service, "volume"));
+  mount = g_volume_get_mount (volume);
+  if (mount == NULL)
+    {
+      data.error = error;
+
+      context = g_main_context_new ();
+      g_main_context_push_thread_default (context);
+      data.loop = g_main_loop_new (context, FALSE);
+
+      g_volume_mount (volume, G_MOUNT_MOUNT_NONE, NULL, job->cancellable, volume_mount_cb, &data);
+      g_main_loop_run (data.loop);
+
+      g_main_loop_unref (data.loop);
+      g_main_context_pop_thread_default (context);
+      g_main_context_unref (context);
+
+      if (*error != NULL)
+        return;
+
+      mount = g_volume_get_mount (volume);
+    }
+
+  root = g_mount_get_root (mount);
+  account_miner_job_traverse_dir (job, root, TRUE, error);
+
+  g_object_unref (root);
+  g_object_unref (mount);
+  g_list_free_full (volumes, g_object_unref);
+}
+
+static GObject *
+create_service (GomMiner *self,
+                GoaObject *object)
+{
+  return g_object_ref (object);
+}
+
+static void
+gom_owncloud_miner_dispose (GObject *object)
+{
+  GomOwncloudMiner *self = GOM_OWNCLOUD_MINER (object);
+
+  g_clear_object (&self->priv->monitor);
+
+  G_OBJECT_CLASS (gom_owncloud_miner_parent_class)->dispose (object);
+}
+
+static void
+gom_owncloud_miner_init (GomOwncloudMiner *self)
+{
+  self->priv = G_TYPE_INSTANCE_GET_PRIVATE (self, GOM_TYPE_OWNCLOUD_MINER, GomOwncloudMinerPrivate);
+  self->priv->monitor = g_volume_monitor_get ();
+}
+
+static void
+gom_owncloud_miner_class_init (GomOwncloudMinerClass *klass)
+{
+  GObjectClass *oclass = G_OBJECT_CLASS (klass);
+  GomMinerClass *miner_class = GOM_MINER_CLASS (klass);
+
+  oclass->dispose = gom_owncloud_miner_dispose;
+
+  miner_class->goa_provider_type = "owncloud";
+  miner_class->miner_identifier = MINER_IDENTIFIER;
+  miner_class->version = 1;
+
+  miner_class->create_service = create_service;
+  miner_class->query = query_owncloud;
+
+  g_type_class_add_private (klass, sizeof (GomOwncloudMinerPrivate));
+}
diff --git a/src/gom-owncloud-miner.h b/src/gom-owncloud-miner.h
new file mode 100644
index 0000000..cb607dd
--- /dev/null
+++ b/src/gom-owncloud-miner.h
@@ -0,0 +1,72 @@
+/*
+ * GNOME Online Miners - crawls through your online content
+ * Copyright (c) 2013 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * Author: Debarshi Ray <debarshir gnome org>
+ *
+ */
+
+#ifndef __GOM_OWNCLOUD_MINER_H__
+#define __GOM_OWNCLOUD_MINER_H__
+
+#include <gio/gio.h>
+#include "gom-miner.h"
+
+G_BEGIN_DECLS
+
+#define GOM_TYPE_OWNCLOUD_MINER gom_owncloud_miner_get_type()
+
+#define GOM_OWNCLOUD_MINER(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST ((obj), \
+   GOM_TYPE_OWNCLOUD_MINER, GomOwncloudMiner))
+
+#define GOM_OWNCLOUD_MINER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST ((klass), \
+   GOM_TYPE_OWNCLOUD_MINER, GomOwncloudMinerClass))
+
+#define GOM_IS_OWNCLOUD_MINER(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE ((obj), \
+   GOM_TYPE_OWNCLOUD_MINER))
+
+#define GOM_IS_OWNCLOUD_MINER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE ((klass), \
+   GOM_TYPE_OWNCLOUD_MINER))
+
+#define GOM_OWNCLOUD_MINER_GET_CLASS(obj) \
+  (G_TYPE_INSTANCE_GET_CLASS ((obj), \
+   GOM_TYPE_OWNCLOUD_MINER, GomOwncloudMinerClass))
+
+typedef struct _GomOwncloudMiner GomOwncloudMiner;
+typedef struct _GomOwncloudMinerClass GomOwncloudMinerClass;
+typedef struct _GomOwncloudMinerPrivate GomOwncloudMinerPrivate;
+
+struct _GomOwncloudMiner {
+  GomMiner parent;
+
+  GomOwncloudMinerPrivate *priv;
+};
+
+struct _GomOwncloudMinerClass {
+  GomMinerClass parent_class;
+};
+
+GType gom_owncloud_miner_get_type(void);
+
+G_END_DECLS
+
+#endif /* __GOM_OWNCLOUD_MINER_H__ */