GNOME.org
 

[tracker/miner-gdata: 1/2] Add GData miner

commit cbd3c6a6e773d5e13aa79b66b546f43b4cc4206d
Author: Adrien Bustany <abustany gnome org>
Date:   Fri Mar 5 20:53:59 2010 -0300

    Add GData miner
    
    So far the miner only exploits the PicasaWeb service

 configure.ac                                       |   32 +++
 src/Makefile.am                                    |    3 +
 src/tracker-miner-gdata/Makefile.am                |   65 +++++
 src/tracker-miner-gdata/query-queue.vala           |   56 ++++
 src/tracker-miner-gdata/tracker-miner-gdata.vala   |   69 +++++
 .../tracker-miner-picasaweb.vala                   |  286 ++++++++++++++++++++
 6 files changed, 511 insertions(+), 0 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 5d32778..ba9b78e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -158,6 +158,7 @@ LIBSTREAMANALYZER_REQUIRED=0.7.0
 GEE_REQUIRED=0.3
 ID3LIB_REQUIRED=3.8.3
 GNOME_KEYRING_REQUIRED=2.26
+LIBGDATA_REQUIRED=0.6
 
 # Library Checks
 PKG_CHECK_MODULES(GLIB2, [glib-2.0 >= $GLIB_REQUIRED])
@@ -792,6 +793,32 @@ fi
 
 AM_CONDITIONAL(HAVE_GNOME_KEYRING, test "x$have_gnome_keyring" = "xyes")
 
+##################################################################
+# GData miner
+##################################################################
+
+AC_ARG_ENABLE(miner_gdata,
+	      AS_HELP_STRING([--enable-miner-gdata],
+			     [enable GData miner [[default=auto]]]),,
+	      [enable_miner_gdata=auto])
+
+if test "x$enable_miner_gdata" != "xno"; then
+	PKG_CHECK_MODULES(MINER_GDATA,
+			  [ libgdata >= $LIBGDATA_REQUIRED ],
+			  [have_miner_gdata_deps=yes],
+			  [have_miner_gdata_deps=no])
+	AC_SUBST(MINER_GDATA_LIBS)
+	AC_SUBST(MINER_GDATA_CFLAGS)
+fi
+
+if test "x$enable_miner_gdata" = "xyes"; then
+   if test "x$have_miner_gdata_deps" != "xyes"; then
+      AC_MSG_ERROR([Couldn't find the required dependencies for the gdata miner: libgdata >= $LIBGDATA_REQUIRED.])
+   fi
+fi
+
+AM_CONDITIONAL(HAVE_MINER_GDATA, test "x$have_miner_gdata_deps" = "xyes")
+
 ####################################################################
 # Mail miners
 ####################################################################
@@ -1678,6 +1705,7 @@ AC_CONFIG_FILES([
 	src/tracker-control/Makefile
 	src/tracker-extract/Makefile
 	src/tracker-miner-fs/Makefile
+	src/tracker-miner-gdata/Makefile
 	src/tracker-preferences/Makefile
 	src/tracker-preferences/tracker-preferences.desktop.in
 	src/tracker-search-bar/Makefile
@@ -1795,6 +1823,10 @@ Plugins/Extensions:
 	Evolution plugin    (data-push):	$enable_evolution_miner
 	KMail plugin        (data-push):	$enable_kmail_miner
 
+Extra miners:
+
+	GData miner				$have_miner_gdata_deps
+
 Writeback:
 
 	MP3 writeback:                          $have_id3lib
diff --git a/src/Makefile.am b/src/Makefile.am
index 2f5ff33..f5a2ec4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -39,3 +39,6 @@ if HAVE_TRACKER_EXPLORER
 SUBDIRS += tracker-explorer
 endif
 
+if HAVE_MINER_GDATA
+SUBDIRS += tracker-miner-gdata
+endif
diff --git a/src/tracker-miner-gdata/Makefile.am b/src/tracker-miner-gdata/Makefile.am
new file mode 100644
index 0000000..4627cc3
--- /dev/null
+++ b/src/tracker-miner-gdata/Makefile.am
@@ -0,0 +1,65 @@
+include $(top_srcdir)/Makefile.decl
+
+INCLUDES =								\
+	-Wall								\
+	-DSHAREDIR=\""$(datadir)"\"					\
+	-DPKGLIBDIR=\""$(libdir)/tracker"\"				\
+	-DLOCALEDIR=\""$(localedir)"\" 					\
+	-DLIBEXEC_PATH=\""$(libexecdir)"\"				\
+	-DG_LOG_DOMAIN=\"Tracker\"					\
+	-DTRACKER_COMPILATION						\
+	-I$(top_srcdir)/src						\
+	$(WARN_CFLAGS)							\
+	$(GMODULE_CFLAGS)						\
+	$(PANGO_CFLAGS)							\
+	$(DBUS_CFLAGS)							\
+	$(MINER_GDATA_CFLAGS)						\
+	$(GCOV_CFLAGS)
+
+VALAFLAGS =								\
+	--pkg gio-2.0							\
+	--pkg posix							\
+	--pkg libgdata							\
+	--thread
+
+libexec_PROGRAMS = tracker-miner-gdata
+
+tracker_miner_gdata_VALASOURCES =					\
+	tracker-miner-gdata.vala					\
+	tracker-miner-picasaweb.vala					\
+	query-queue.vala						\
+	$(top_srcdir)/src/vapi/tracker-client-0.7.vapi
+
+tracker_miner_gdata_SOURCES =						\
+	$(tracker_miner_gdata_VALASOURCES:.vala=.c)
+
+tracker_miner_gdata_LDADD =						\
+	$(top_builddir)/src/libtracker-miner/libtracker-miner- TRACKER_API_VERSION@.la	\
+	$(top_builddir)/src/libtracker-client/libtracker-client- TRACKER_API_VERSION@.la	\
+	$(DBUS_LIBS)							\
+	$(GMODULE_LIBS)							\
+	$(GTHREAD_LIBS)							\
+	$(GIO_LIBS)							\
+	$(GCOV_LIBS)							\
+	$(GLIB2_LIBS)							\
+	$(MINER_GDATA_LIBS)							\
+	-lz								\
+	-lm
+
+vapi_sources =								\
+	$(top_srcdir)/src/libtracker-miner/tracker-miner- TRACKER_API_VERSION@.vapi
+
+tracker-miner-gdata.vala.stamp: $(tracker_miner_gdata_VALASOURCES) $(vapi_sources)
+	$(AM_V_GEN)$(VALAC) $(GCOV_VALAFLAGS) -C $(VALAFLAGS) $^
+	touch $@
+
+
+BUILT_SOURCES = tracker-miner-gdata.vala.stamp
+
+MAINTAINERCLEANFILES =							\
+	tracker-miner-gdata.vala.stamp				\
+	$(tracker_miner_gdata_VALASOURCES:.vala=.c)
+
+EXTRA_DIST =								\
+	$(tracker_miner_gdata_VALASOURCES)				\
+	tracker-miner-gdata.vala.stamp
diff --git a/src/tracker-miner-gdata/query-queue.vala b/src/tracker-miner-gdata/query-queue.vala
new file mode 100644
index 0000000..7afeeb7
--- /dev/null
+++ b/src/tracker-miner-gdata/query-queue.vala
@@ -0,0 +1,56 @@
+public class QueryQueue : GLib.Object {
+	/* Holds the pending sparql updates and monitors them */
+	private HashTable<uint, string> queue;
+	private uint cookie;
+
+	private Mutex flush_mutex;
+
+	private Tracker.Miner miner;
+
+	public QueryQueue (Tracker.Miner parent) {
+		miner = parent;
+
+		queue = new HashTable<uint, string> (direct_hash, direct_equal);
+		cookie = 0;
+
+		flush_mutex = new Mutex ();
+	}
+
+	public async void append (string query) {
+		uint current_cookie = cookie ++;
+		queue.insert (current_cookie, query);
+
+		try {
+			yield miner.execute_batch_update (query);
+		} catch (Error tracker_error) {
+			warning ("BatchUpdate query failed: %s", tracker_error.message);
+		}
+
+		queue.remove (current_cookie);
+	}
+
+	/* BLOCKING flush */
+	public void flush () {
+		if (!flush_mutex.trylock ()) {
+			message ("There's already a flush taking place");
+			return;
+		}
+
+		if (queue.size () > 0) {
+			MainLoop wait_loop;
+			try {
+				wait_loop = new MainLoop (null, false);
+				miner.commit (null, () => { wait_loop.quit (); });
+				wait_loop.run ();
+			} catch (Error tracker_error) {
+				warning ("Commit query failed: %s", tracker_error.message);
+			}
+		}
+
+		flush_mutex.unlock ();
+	}
+
+	public uint size () {
+		return queue.size ();
+	}
+}
diff --git a/src/tracker-miner-gdata/tracker-miner-gdata.vala b/src/tracker-miner-gdata/tracker-miner-gdata.vala
new file mode 100644
index 0000000..f9dd050
--- /dev/null
+++ b/src/tracker-miner-gdata/tracker-miner-gdata.vala
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+namespace Tracker {
+
+public class MinerGData {
+	private static MainLoop main_loop;
+
+	private static bool in_loop = false;
+	private static void signal_handler (int signo) {
+		if (in_loop) {
+			Posix.exit (Posix.EXIT_FAILURE);
+		}
+
+		switch (signo) {
+			case Posix.SIGINT:
+			case Posix.SIGTERM:
+				in_loop = true;
+				main_loop.quit ();
+				break;
+		}
+	}
+
+	private static void init_signals () {
+#if G_OS_WIN32
+#else
+		Posix.sigaction_t act = Posix.sigaction_t ();
+		Posix.sigset_t    empty_mask = Posix.sigset_t ();
+		Posix.sigemptyset (empty_mask);
+		act.sa_handler = signal_handler;
+		act.sa_mask    = empty_mask;
+		act.sa_flags   = 0;
+
+		Posix.sigaction (Posix.SIGTERM, act, null);
+		Posix.sigaction (Posix.SIGINT, act, null);
+#endif
+	}
+
+	public static void main (string[] args) {
+		Environment.set_application_name ("GData tracker miner");
+		MinerPicasaweb picasaweb_miner;
+		picasaweb_miner = Object.new (typeof (MinerPicasaweb)) as MinerPicasaweb;
+
+		init_signals ();
+
+		main_loop = new MainLoop (null, false);
+		main_loop.run ();
+
+		picasaweb_miner.shutdown ();
+	}
+}
+
+} // End namespace Tracker
diff --git a/src/tracker-miner-gdata/tracker-miner-picasaweb.vala b/src/tracker-miner-gdata/tracker-miner-picasaweb.vala
new file mode 100644
index 0000000..4bdaba4
--- /dev/null
+++ b/src/tracker-miner-gdata/tracker-miner-picasaweb.vala
@@ -0,0 +1,286 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+namespace Tracker {
+
+public class MinerPicasaweb : Tracker.MinerWeb {
+	private static const string API_KEY = "ABQIAAAAISU_TtyUrVg0N9RGhb8pTxRI3KVolA1NVmS0lLG3RQFyztcosBSHs5rHiQHsmL41EbX1SehKnf1rrw";
+	private static const string DATASOURCE_URN = "urn:682654ec-b59d-4e99-9b0f-7559fc9c5d42";
+	private static const string MINER_NAME = "PicasaWeb";
+	private static const string SERVICE_DESCRIPTION = "Tracker miner for PicasaWeb";
+	private static const uint   PULL_INTERVAL = 5*60; /* seconds */
+	private uint pull_timeout_handle;
+
+	private GData.PicasaWebService service = null;
+	private QueryQueue query_queue;
+
+	construct {
+		set ("name", MINER_NAME);
+		set ("association-status", MinerWebAssociationStatus.UNASSOCIATED);
+
+		service = new GData.PicasaWebService (API_KEY);
+
+		query_queue = new QueryQueue (this);
+
+		this.notify["association-status"].connect (association_status_changed);
+
+	}
+
+	private void association_status_changed (Object source, ParamSpec pspec) {
+		MinerWebAssociationStatus status;
+
+		get ("association-status", out status);
+
+		switch (status) {
+			case MinerWebAssociationStatus.ASSOCIATED:
+				if (pull_timeout_handle != 0)
+					return;
+
+				message ("Miner is now associated. Initiating periodic pull.");
+				Timeout.add_seconds (PULL_INTERVAL, pull_timeout_cb);
+				Idle.add ( () => { pull_timeout_cb (); return false; });
+				break;
+			case MinerWebAssociationStatus.UNASSOCIATED:
+				if (pull_timeout_handle == 0)
+					return;
+
+				Source.remove (pull_timeout_handle);
+				break;
+		}
+	}
+
+	private bool pull_timeout_cb () {
+		service.query_all_albums_async (null, service.get_username (), null, albums_cb, pull_finished_cb);
+		return true;
+	}
+
+	private void albums_cb (GData.Entry entry, uint entry_key, uint entry_count) {
+		GData.Feed feed_files;
+		GData.PicasaWebAlbum album = entry as GData.PicasaWebAlbum;
+		GData.PicasaWebFile current_file;
+		string current_file_url;
+		SparqlBuilder builder;
+		List<string> album_files_urls;
+		TimeVal tv = TimeVal ();
+
+		try {
+			feed_files = service.query_files (album, null, null, null);
+		} catch (Error service_error) {
+			warning ("Couldn't get files for album %s: %s", album.title, service_error.message);
+			return;
+		}
+
+		album_files_urls = new List<string> ();
+
+		foreach (GData.Entry current_entry in feed_files.get_entries ()) {
+			current_file = current_entry as GData.PicasaWebFile;
+			current_file_url = ((GData.MediaContent)current_file.get_contents ().first ().data).get_uri ();
+
+			album_files_urls.prepend (current_file_url);
+
+			builder = new SparqlBuilder.update ();
+
+			builder.insert_open (DATASOURCE_URN);
+
+			builder.subject ("_:picture");
+			builder.predicate ("a");
+			builder.object ("nmm:Photo");
+
+			builder.predicate ("a");
+			builder.object ("nfo:MediaFileListEntry");
+
+			builder.predicate ("a");
+			builder.object ("nfo:RemoteDataObject");
+
+			builder.predicate ("nie:url");
+			builder.object_string (current_file_url);
+
+			builder.predicate ("dc:title");
+			if (current_file.get_caption () != null) {
+				builder.object_string (current_file.get_caption ());
+			} else {
+				builder.object_string (current_file.get_title ());
+			}
+
+			current_file.get_timestamp (tv);
+			builder.predicate ("nie:contentCreated");
+			builder.object_string (tv.to_iso8601 ());
+
+			current_file.get_edited (tv);
+			builder.predicate ("nie:contentLastModified");
+			builder.object_string (tv.to_iso8601 ());
+
+			builder.predicate ("nfo:width");
+			builder.object_int64 ((int64)current_file.get_width ());
+
+			builder.predicate ("nfo:height");
+			builder.object_int64 ((int64)current_file.get_height ());
+
+			if (current_file.get_tags () != null) {
+				foreach (string tag_label in current_file.get_tags ().split (",")) {
+					builder.predicate ("nao:hasTag");
+					builder.object_blank_open ();
+					builder.predicate ("nao:prefLabel");
+					builder.object_string (tag_label);
+					builder.object_blank_close ();
+				}
+			}
+
+			builder.insert_close ();
+
+			query_queue.append (builder.get_result ());
+		}
+
+		builder = new SparqlBuilder.update ();
+
+		builder.insert_open (DATASOURCE_URN);
+
+		builder.subject ("_:album");
+		builder.predicate ("a");
+		builder.object ("nmm:ImageList");
+
+		builder.predicate ("dc:title");
+		builder.object_string (album.get_title ());
+
+		if (album.get_summary () != null) {
+			builder.predicate ("rdfs:comment");
+			builder.object_string (album.get_summary ());
+		}
+
+		if (album.get_tags () != null) {
+			foreach (string tag_label in album.get_tags ().split (",")) {
+				builder.predicate ("nao:hasTag");
+				builder.object_blank_open ();
+				builder.predicate ("nao:prefLabel");
+				builder.object_string (tag_label);
+				builder.object_blank_close ();
+			}
+		}
+
+		builder.predicate ("nfo:entryCounter");
+		builder.object_int64 ((int64)album.get_num_photos ());
+
+		album.get_timestamp (tv);
+		builder.predicate ("nie:contentCreated");
+		builder.object_string (tv.to_iso8601 ());
+
+		album.get_edited (tv);
+		builder.predicate ("nie:contentLastModified");
+		builder.object_string (tv.to_iso8601 ());
+
+		foreach (string current_file_url in album_files_urls) {
+			builder.predicate ("nfo:hasMediaFileListEntry");
+			builder.object_blank_open ();
+			builder.predicate ("a");
+			builder.object ("nmm:Photo");
+			builder.predicate ("a");
+			builder.object ("nfo:MediaFileListEntry");
+			builder.predicate ("a");
+			builder.object ("nfo:RemoteDataObject");
+			builder.predicate ("nie:url");
+			builder.object_string (current_file_url);
+			builder.object_blank_close ();
+		}
+
+		builder.insert_close ();
+
+		query_queue.append (builder.get_result ());
+	}
+
+	private void pull_finished_cb () {
+		query_queue.flush ();
+	}
+
+	public void shutdown () {
+		set ("association-status", MinerWebAssociationStatus.UNASSOCIATED);
+	}
+
+	public override void authenticate () throws MinerWebError {
+		PasswordProvider password_provider;
+		string username;
+		string password;
+		bool verified = false;
+
+		password_provider = PasswordProvider.get ();
+
+		set ("association-status", MinerWebAssociationStatus.UNASSOCIATED);
+
+		try {
+			password = password_provider.get_password (MINER_NAME, out username);
+		} catch (Error e) {
+			if (e is PasswordProviderError.NOTFOUND) {
+				throw new MinerWebError.NO_CREDENTIALS ("Miner is not associated");
+			}
+			throw new MinerWebError.KEYRING (e.message);
+		}
+
+		try {
+			verified = service.authenticate (username, password, null);
+		} catch (Error service_error) {
+			throw new MinerWebError.SERVICE (service_error.message);
+		}
+
+		if (!verified) {
+			throw new MinerWebError.WRONG_CREDENTIALS ("Wrong username and/or password");
+		}
+
+		message ("Authentication successful");
+		set ("association-status", MinerWebAssociationStatus.ASSOCIATED);
+	}
+
+	public override void dissociate () throws MinerWebError {
+		var password_provider = PasswordProvider.get ();
+
+		try {
+			password_provider.forget_password (MINER_NAME);
+		} catch (Error e) {
+			if (e is PasswordProviderError.SERVICE) {
+				throw new MinerWebError.KEYRING (e.message);
+			}
+
+			critical ("Internal error: %s", e.message);
+			return;
+		}
+
+		set ("association-status", MinerWebAssociationStatus.UNASSOCIATED);
+	}
+
+	public override void associate (HashTable<string, string> association_data) throws Tracker.MinerWebError {
+		assert (association_data.lookup ("username") != null && association_data.lookup ("password") != null);
+
+		var password_provider = PasswordProvider.get ();
+
+		try {
+			password_provider.store_password (MINER_NAME,
+			                                  SERVICE_DESCRIPTION,
+			                                  association_data.lookup ("username"),
+			                                  association_data.lookup ("password"));
+		} catch (Error e) {
+			if (e is PasswordProviderError.SERVICE) {
+				throw new MinerWebError.KEYRING (e.message);
+			}
+
+			critical ("Internal error: %s", e.message);
+			return;
+		}
+	}
+
+}
+
+} // End namespace Tracker
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]