GNOME.org
 

[gnome-online-miners/wip/flickr: 2/3] src: add a Flickr miner based on grilo

commit d864e0d76895b71af1ccedf88a22d77b6ae95167
Author: Marek Chalupa <mchalupa redhat com>
Date:   Fri Jun 21 14:50:40 2013 +0200

    src: add a Flickr miner based on grilo
    
    https://bugzilla.gnome.org/show_bug.cgi?id=700897

 configure.ac                |    2 +
 src/Makefile.am             |   27 +++
 src/gom-flickr-miner-main.c |   32 +++
 src/gom-flickr-miner.c      |  494 +++++++++++++++++++++++++++++++++++++++++++
 src/gom-flickr-miner.h      |   71 ++++++
 5 files changed, 626 insertions(+), 0 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 4e849f6..e0d8f46 100644
--- a/configure.ac
+++ b/configure.ac
@@ -22,6 +22,7 @@ AC_HEADER_STDC
 GDATA_MIN_VERSION=0.13.3
 GLIB_MIN_VERSION=2.35.1
 GOA_MIN_VERSION=3.2.0
+GRILO_MIN_VERSION=0.2.6
 ZAPOJIT_MIN_VERSION=0.0.2
 
 GNOME_COMMON_INIT
@@ -38,6 +39,7 @@ PKG_CHECK_MODULES(GIO, [gio-2.0])
 PKG_CHECK_MODULES(GOA, [goa-1.0 >= $GOA_MIN_VERSION])
 AC_DEFINE([GOA_API_IS_SUBJECT_TO_CHANGE], [], [We are aware that GOA's API can change])
 
+PKG_CHECK_MODULES(GRILO, [grilo-0.2 >= $GRILO_MIN_VERSION])
 PKG_CHECK_MODULES(TRACKER, [tracker-miner-0.16 tracker-sparql-0.16])
 PKG_CHECK_MODULES(ZAPOJIT, [zapojit-0.0 >= $ZAPOJIT_MIN_VERSION])
 
diff --git a/src/Makefile.am b/src/Makefile.am
index f6d3d63..24c4d42 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -31,10 +31,37 @@ libgom_1_0_la_LDFLAGS = \
     $(NULL)
 
 libexec_PROGRAMS = \
+    gom-flickr-miner \
     gom-gdata-miner \
     gom-zpj-miner \
     $(NULL)
 
+gom_flickr_miner_SOURCES = \
+    gom-flickr-miner-main.c \
+    gom-flickr-miner.c \
+    gom-flickr-miner.h \
+    $(NULL)
+
+gom_flickr_miner_CPPFLAGS = \
+    -DG_LOG_DOMAIN=\"Gom\" \
+    -DG_DISABLE_DEPRECATED \
+    -I$(top_srcdir)/src \
+    $(GIO_CFLAGS) \
+    $(GLIB_CFLAGS) \
+    $(GOA_CFLAGS) \
+    $(GRILO_CFLAGS) \
+    $(TRACKER_CFLAGS) \
+    $(NULL)
+
+gom_flickr_miner_LDADD = \
+    libgom-1.0.la  \
+    $(GIO_LIBS) \
+    $(GLIB_LIBS) \
+    $(GOA_LIBS) \
+    $(GRILO_LIBS) \
+    $(TRACKER_LIBS) \
+    $(NULL)
+
 gom_gdata_miner_SOURCES = \
     gom-gdata-miner-main.c \
     gom-gdata-miner.c \
diff --git a/src/gom-flickr-miner-main.c b/src/gom-flickr-miner-main.c
new file mode 100644
index 0000000..0415265
--- /dev/null
+++ b/src/gom-flickr-miner-main.c
@@ -0,0 +1,32 @@
+/*
+ * GNOME Online Miners - crawls through your online content
+ * Copyright (c) 2013 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * Author: Marek Chalupa <mchalupa redhat com>
+ */
+
+#include "config.h"
+
+#define INSIDE_MINER
+#define MINER_NAME "FLICKR"
+#define MINER_TYPE GOM_TYPE_FLICKR_MINER
+#define MINER_BUS_NAME "org.gnome.OnlineMiners.Flickr"
+#define MINER_OBJECT_PATH "/org/gnome/OnlineMiners/Flickr"
+
+#include "gom-flickr-miner.h"
+#include "gom-miner-main.c"
diff --git a/src/gom-flickr-miner.c b/src/gom-flickr-miner.c
new file mode 100644
index 0000000..f624d32
--- /dev/null
+++ b/src/gom-flickr-miner.c
@@ -0,0 +1,494 @@
+/*
+ * GNOME Online Miners - crawls through your online content
+ * Copyright (c) 2013 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * Author: Marek Chalupa <mchalupa redhat com>
+ */
+
+#include "config.h"
+
+#include <goa/goa.h>
+#include <grilo.h>
+
+#include "gom-flickr-miner.h"
+#include "gom-utils.h"
+
+#define MINER_IDENTIFIER "gd:flickr:miner:3c63f509-23e8-4283-8aed-154bb55ef07b"
+
+G_DEFINE_TYPE (GomFlickrMiner, gom_flickr_miner, GOM_TYPE_MINER)
+
+struct _GomFlickrMinerPrivate {
+  GQueue *boxes;
+};
+
+typedef enum {
+  OP_FETCH_ALL,
+  OP_CREATE_HIEARCHY
+} OpType;
+
+typedef struct {
+  GrlMedia  *media;
+  GrlMedia  *parent;
+} FlickrEntry;
+
+typedef struct {
+  FlickrEntry *parent_entry;
+  GMainLoop *loop;
+  GomAccountMinerJob *job;
+} SyncData;
+
+static void account_miner_job_browse_container (GomAccountMinerJob *job, FlickrEntry *entry);
+
+static FlickrEntry *
+create_entry (GrlMedia *media, GrlMedia *parent)
+{
+  FlickrEntry *entry;
+
+  entry = g_slice_new0 (FlickrEntry);
+
+  entry->media  = (media != NULL) ? g_object_ref (media) : NULL;
+  entry->parent = (parent != NULL) ? g_object_ref (parent) : NULL;
+
+  return entry;
+}
+
+static void
+free_entry (FlickrEntry *entry)
+{
+  g_clear_object (&entry->media);
+  g_clear_object (&entry->parent);
+  g_slice_free (FlickrEntry, entry);
+}
+
+static GrlOperationOptions *
+get_grl_options (GrlSource *source)
+{
+  GrlCaps *caps;
+  GrlOperationOptions *opts = NULL;
+
+  caps = grl_source_get_caps (source, GRL_OP_BROWSE);
+  opts = grl_operation_options_new (caps);
+
+  g_return_val_if_fail (opts != NULL, NULL);
+
+  grl_operation_options_set_flags (opts, GRL_RESOLVE_FAST_ONLY);
+
+  return opts;
+}
+
+static gboolean
+account_miner_job_process_entry (GomAccountMinerJob *job,
+                                 OpType op_type,
+                                 FlickrEntry *entry,
+                                 GError **error)
+{
+  GDateTime *created_time;
+  gchar *contact_resource;
+  gchar *resource = NULL;
+  gchar *date, *identifier;
+  const gchar *class = NULL, *id, *mime, *name;
+  gboolean resource_exists;
+
+  if (op_type == OP_CREATE_HIEARCHY && entry->parent == NULL)
+    return TRUE;
+
+  id = grl_media_get_id (entry->media);
+  identifier = g_strdup_printf ("%sflickr:%s",
+                                GRL_IS_MEDIA_BOX (entry->media) ?
+                                "photos:collection:" : "",
+                                id);
+
+  /* remove from the list of the previous resources */
+  g_hash_table_remove (job->previous_resources, identifier);
+
+  name = grl_media_get_title (entry->media);
+
+  if (GRL_IS_MEDIA_BOX (entry->media))
+    class = "nfo:DataContainer";
+  else
+    class = "nmm:Photo";
+
+  resource = gom_tracker_sparql_connection_ensure_resource
+    (job->connection,
+     job->cancellable, error,
+     &resource_exists,
+     job->datasource_urn, identifier,
+     "nfo:RemoteDataObject", class, NULL);
+
+  if (*error != NULL)
+    goto out;
+
+  gom_tracker_update_datasource (job->connection, job->datasource_urn,
+                                 resource_exists, identifier, resource,
+                                 job->cancellable, error);
+
+  if (*error != NULL)
+    goto out;
+
+  if (entry->parent != NULL)
+    {
+      gchar *parent_resource_urn, *parent_identifier;
+      const gchar *parent_id;
+
+      parent_identifier = g_strconcat ("photos:collection:flickr:",
+                                        grl_media_get_id (entry->parent) , NULL);
+      parent_resource_urn = gom_tracker_sparql_connection_ensure_resource
+        (job->connection, job->cancellable, error,
+         NULL,
+         job->datasource_urn, parent_identifier,
+         "nfo:RemoteDataObject", "nfo:DataContainer", NULL);
+      g_free (parent_identifier);
+
+      if (*error != NULL)
+        goto out;
+
+      gom_tracker_sparql_connection_insert_or_replace_triple
+        (job->connection,
+         job->cancellable, error,
+         job->datasource_urn, resource,
+         "nie:isPartOf", parent_resource_urn);
+      g_free (parent_resource_urn);
+
+      if (*error != NULL)
+        goto out;
+    }
+
+  if (op_type == OP_CREATE_HIEARCHY)
+    goto out;
+
+  gom_tracker_sparql_connection_insert_or_replace_triple
+    (job->connection,
+     job->cancellable, error,
+     job->datasource_urn, resource,
+     "nie:url", grl_media_get_url (entry->media));
+
+  if (*error != NULL)
+    goto out;
+
+  gom_tracker_sparql_connection_insert_or_replace_triple
+    (job->connection,
+     job->cancellable, error,
+     job->datasource_urn, resource,
+     "nie:description", grl_media_get_description (entry->media));
+
+  if (*error != NULL)
+    goto out;
+
+  gom_tracker_sparql_connection_insert_or_replace_triple
+    (job->connection,
+     job->cancellable, error,
+     job->datasource_urn, resource,
+     "nfo:fileName", name);
+
+  if (*error != NULL)
+    goto out;
+
+  /* FIXME: photos don't have a MIME type */
+  mime = grl_media_get_mime (entry->media);
+  if (mime == NULL && name != NULL)
+    mime = gom_filename_to_mime_type (name);
+  if (mime != NULL)
+    {
+      gom_tracker_sparql_connection_insert_or_replace_triple
+        (job->connection,
+         job->cancellable, error,
+         job->datasource_urn, resource,
+         "nie:mimeType", mime);
+
+      if (*error != NULL)
+        goto out;
+    }
+
+  contact_resource = gom_tracker_utils_ensure_contact_resource
+    (job->connection,
+     job->cancellable, error,
+     job->datasource_urn, grl_media_get_author (entry->media));
+
+  if (*error != NULL)
+    goto out;
+
+  gom_tracker_sparql_connection_insert_or_replace_triple
+    (job->connection,
+     job->cancellable, error,
+     job->datasource_urn, resource,
+     "nco:creator", contact_resource);
+  g_free (contact_resource);
+
+  if (*error != NULL)
+    goto out;
+
+  created_time = grl_media_get_creation_date (entry->media);
+  if (created_time != NULL)
+    {
+      date = gom_iso8601_from_timestamp (g_date_time_to_unix (created_time));
+      gom_tracker_sparql_connection_insert_or_replace_triple
+        (job->connection,
+         job->cancellable, error,
+         job->datasource_urn, resource,
+         "nie:contentCreated", date);
+      g_free (date);
+    }
+  if (*error != NULL)
+    goto out;
+
+ out:
+  g_free (resource);
+  g_free (identifier);
+
+  if (*error != NULL)
+    return FALSE;
+
+  return TRUE;
+}
+
+static void
+source_browse_cb (GrlSource *source,
+                  guint operation_id,
+                  GrlMedia *media,
+                  guint remaining,
+                  gpointer user_data,
+                  const GError *error)
+{
+  GError *local_error = NULL;
+  SyncData *data = (SyncData *) user_data;
+  GomFlickrMiner *self = GOM_FLICKR_MINER (data->job->miner);
+
+  if (error != NULL)
+    {
+      g_warning ("Unable to browse source %p: %s", source, error->message);
+      return;
+    }
+
+  if (media != NULL)
+    {
+      FlickrEntry *entry;
+
+      entry = create_entry (media, data->parent_entry->media);
+      account_miner_job_process_entry (data->job, OP_CREATE_HIEARCHY, entry, &local_error);
+      if (local_error != NULL)
+        {
+          g_warning ("Unable to process entry %p: %s", media, local_error->message);
+          g_error_free (local_error);
+        }
+
+      if (GRL_IS_MEDIA_BOX (media))
+        g_queue_push_tail (self->priv->boxes, entry);
+      else
+        free_entry (entry);
+    }
+
+  if (remaining == 0)
+    g_main_loop_quit (data->loop);
+}
+
+static void
+account_miner_job_browse_container (GomAccountMinerJob *job, FlickrEntry *entry)
+{
+  GMainContext *context;
+  GrlOperationOptions *opts;
+  const GList *keys;
+  SyncData data;
+
+  data.parent_entry = entry;
+  data.job = job;
+
+  context = g_main_context_new ();
+  g_main_context_push_thread_default (context);
+  data.loop = g_main_loop_new (context, FALSE);
+
+  keys = grl_source_supported_keys (GRL_SOURCE (data.job->service));
+  opts = get_grl_options (GRL_SOURCE (data.job->service));
+
+  grl_source_browse (GRL_SOURCE (job->service),
+                     entry->media,
+                     keys,
+                     opts,
+                     source_browse_cb,
+                     &data);
+  g_main_loop_run (data.loop);
+
+  g_object_unref (opts);
+  g_main_loop_unref (data.loop);
+  g_main_context_pop_thread_default (context);
+  g_main_context_unref (context);
+}
+
+static void
+source_search_cb (GrlSource *source,
+                  guint operation_id,
+                  GrlMedia *media,
+                  guint remaining,
+                  gpointer user_data,
+                  const GError *error)
+{
+  GError *local_error = NULL;
+  SyncData *data = (SyncData *) user_data;
+
+  if (error != NULL)
+    {
+      g_warning ("Unable to search source %p: %s", source, error->message);
+      return;
+    }
+
+  if (media != NULL)
+    {
+      FlickrEntry *entry;
+
+      entry = create_entry (media, NULL);
+      account_miner_job_process_entry (data->job, OP_FETCH_ALL, entry, &local_error);
+      if (local_error != NULL)
+        {
+          g_warning ("Unable to process entry %p: %s", media, local_error->message);
+          g_error_free (local_error);
+        }
+
+      free_entry (entry);
+    }
+
+  if (remaining == 0)
+    g_main_loop_quit (data->loop);
+}
+
+static void
+query_flickr (GomAccountMinerJob *job,
+              GError **error)
+{
+  GomFlickrMiner *self = GOM_FLICKR_MINER (job->miner);
+  GomFlickrMinerPrivate *priv = self->priv;
+  FlickrEntry *entry;
+  const GList *keys;
+  GMainContext *context;
+  GrlOperationOptions *opts;
+  SyncData data;
+
+  if (job->service == NULL)
+  {
+    /* FIXME: use proper #defines and enumerated types */
+    g_set_error (error,
+                 g_quark_from_static_string ("gom-error"),
+                 0,
+                 "Can not query without a source");
+    return;
+  }
+
+  /* grl_source_browse does not fetch photos that are not part of a
+   * set. So, use grl_source_search to fetch all photos and then allot
+   * each photo to any set that it might be a part of.
+   */
+
+  data.job = job;
+  context = g_main_context_new ();
+  g_main_context_push_thread_default (context);
+  data.loop = g_main_loop_new (context, FALSE);
+
+  keys = grl_source_supported_keys (GRL_SOURCE(job->service));
+  opts = get_grl_options (GRL_SOURCE(job->service));
+  grl_source_search (GRL_SOURCE (job->service), NULL, keys, opts, source_search_cb, &data);
+  g_main_loop_run (data.loop);
+
+  g_object_unref (opts);
+  g_main_loop_unref (data.loop);
+  g_main_context_pop_thread_default (context);
+  g_main_context_unref (context);
+
+  entry = create_entry (NULL, NULL);
+  account_miner_job_browse_container (job, entry);
+  free_entry (entry);
+
+  while (!g_queue_is_empty (priv->boxes))
+    {
+      entry = (FlickrEntry *) g_queue_pop_head (priv->boxes);
+      account_miner_job_browse_container (job, entry);
+      free_entry (entry);
+    }
+}
+
+static GObject *
+create_service (GomMiner *self,
+                GoaObject *object)
+{
+  GoaAccount *acc;
+  GrlRegistry *registry;
+  GrlSource *source = NULL;
+  gchar *source_id = NULL;
+
+  acc = goa_object_peek_account (object);
+  if (acc == NULL)
+    return NULL;
+
+  source_id = g_strdup_printf ("grl-flickr-%s", goa_account_get_id (acc));
+
+  registry = grl_registry_get_default ();
+
+  g_debug ("Looking for source %s", source_id);
+  source = grl_registry_lookup_source (registry, source_id);
+  g_free (source_id);
+  if (source == NULL)
+    return NULL;
+
+  /* freeing job calls unref upon this object */
+  g_object_ref (source);
+
+  return G_OBJECT (source);
+}
+
+static void
+gom_flickr_miner_finalize (GObject *object)
+{
+  GomFlickrMiner *self = GOM_FLICKR_MINER (object);
+
+  g_queue_free_full (self->priv->boxes, (GDestroyNotify) free_entry);
+
+  G_OBJECT_CLASS (gom_flickr_miner_parent_class)->finalize (object);
+}
+
+static void
+gom_flickr_miner_init (GomFlickrMiner *self)
+{
+  self->priv = G_TYPE_INSTANCE_GET_PRIVATE (self, GOM_TYPE_FLICKR_MINER, GomFlickrMinerPrivate);
+  self->priv->boxes = g_queue_new ();
+}
+
+static void
+gom_flickr_miner_class_init (GomFlickrMinerClass *klass)
+{
+  GObjectClass *oclass = G_OBJECT_CLASS (klass);
+  GomMinerClass *miner_class = GOM_MINER_CLASS (klass);
+  GrlRegistry *registry;
+  GError *error = NULL;
+
+  oclass->finalize = gom_flickr_miner_finalize;
+
+  miner_class->goa_provider_type = "flickr";
+  miner_class->miner_identifier = MINER_IDENTIFIER;
+  miner_class->version = 1;
+
+  miner_class->create_service = create_service;
+  miner_class->query = query_flickr;
+
+  grl_init (NULL, NULL);
+  registry = grl_registry_get_default ();
+
+  if (!grl_registry_load_plugin_by_id (registry, "grl-flickr", &error))
+    {
+      g_error ("%s", error->message);
+      g_error_free (error);
+    }
+
+  g_type_class_add_private (klass, sizeof (GomFlickrMinerPrivate));
+}
diff --git a/src/gom-flickr-miner.h b/src/gom-flickr-miner.h
new file mode 100644
index 0000000..7f452db
--- /dev/null
+++ b/src/gom-flickr-miner.h
@@ -0,0 +1,71 @@
+/*
+ * GNOME Online Miners - crawls through your online content
+ * Copyright (c) 2013 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * Author: Marek Chalupa <mchalupa redhat com>
+ */
+
+#ifndef __GOM_FLICKR_MINER_H__
+#define __GOM_FLICKR_MINER_H__
+
+#include <gio/gio.h>
+#include "gom-miner.h"
+
+G_BEGIN_DECLS
+
+#define GOM_TYPE_FLICKR_MINER gom_flickr_miner_get_type()
+
+#define GOM_FLICKR_MINER(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST ((obj), \
+   GOM_TYPE_FLICKR_MINER, GomFlickrMiner))
+
+#define GOM_FLICKR_MINER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST ((klass), \
+   GOM_TYPE_FLICKR_MINER, GomFlickrMinerClass))
+
+#define GOM_IS_FLICKR_MINER(obj) \
+  (G_TYPE_CHECK_INSTANCE_TYPE ((obj), \
+   GOM_TYPE_FLICKR_MINER))
+
+#define GOM_IS_FLICKR_MINER_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_TYPE ((klass), \
+   GOM_TYPE_FLICKR_MINER))
+
+#define GOM_FLICKR_MINER_GET_CLASS(obj) \
+  (G_TYPE_INSTANCE_GET_CLASS ((obj), \
+   GOM_TYPE_FLICKR_MINER, GomFlickrMinerClass))
+
+typedef struct _GomFlickrMiner GomFlickrMiner;
+typedef struct _GomFlickrMinerClass GomFlickrMinerClass;
+typedef struct _GomFlickrMinerPrivate GomFlickrMinerPrivate;
+
+struct _GomFlickrMiner {
+  GomMiner parent;
+
+  GomFlickrMinerPrivate *priv;
+};
+
+struct _GomFlickrMinerClass {
+  GomMinerClass parent_class;
+};
+
+GType gom_flickr_miner_get_type(void);
+
+G_END_DECLS
+
+#endif /* __GOM_FLICKR_MINER_H__ */
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]