[tracker/miner-flickr-review] Add Flickr miner



commit 4c61005a8007515fc276fa042e6901c54e76485f
Author: Adrien Bustany <abustany gnome org>
Date:   Wed Mar 10 09:44:56 2010 -0300

    Add Flickr miner
    
    This commit adds a Flickr miner, using the TrackerMinerWeb class. The Flickr
    miner can import your photos and albums, as well as their tags, from Flickr.

 configure.ac                                       |   32 +
 data/dbus/Makefile.am                              |    5 +
 ...rg.freedesktop.Tracker1.Miner.Flickr.service.in |    3 +
 data/icons/scalable/Makefile.am                    |   11 +-
 data/icons/scalable/tracker-miner-flickr.svg       |  206 +++++++
 data/miners/Makefile.am                            |    7 +
 data/miners/tracker-miner-flickr.desktop.in.in     |    8 +
 src/Makefile.am                                    |    4 +
 src/tracker-miner-flickr/Makefile.am               |   64 ++
 src/tracker-miner-flickr/query-queue.vala          |   75 +++
 src/tracker-miner-flickr/tracker-miner-flickr.vala |  607 ++++++++++++++++++++
 11 files changed, 1021 insertions(+), 1 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 5e0de56..d99b76a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -159,6 +159,7 @@ GEE_REQUIRED=0.3
 ID3LIB_REQUIRED=3.8.3
 GNOME_KEYRING_REQUIRED=2.26
 LIBGRSS_REQUIRED=0.3
+REST_REQUIRED=0.6
 
 # Library Checks
 PKG_CHECK_MODULES(GLIB2, [glib-2.0 >= $GLIB_REQUIRED])
@@ -819,6 +820,32 @@ fi
 
 AM_CONDITIONAL(HAVE_GNOME_KEYRING, test "x$have_gnome_keyring" = "xyes")
 
+##################################################################
+# Flickr miner
+##################################################################
+
+AC_ARG_ENABLE(miner_flickr,
+              AS_HELP_STRING([--miner-flickr],
+                             [enable Flickr miner [[default=auto]]]),,
+                             [enable_miner_flickr=auto])
+
+if test "x$enable_miner_flickr" != "xno"; then
+	PKG_CHECK_MODULES(MINER_FLICKR,
+	                  [ rest-0.6 >= $REST_REQUIRED ],
+	                  [have_miner_flickr_deps=yes],
+	                  [have_miner_flickr_deps=no])
+	AC_SUBST(MINER_FLICKR_LIBS)
+	AC_SUBST(MINER_FLICKR_CFLAGS)
+fi
+
+if test "x$enable_miner_flickr" = "xyes"; then
+	if test "x$have_miner_flickr_deps" != "xyes"; then
+		AC_MSG_ERROR([Couldn't find the required dependencies for the Flickr miner: rest-0.6 >= $REST_REQUIRED.])
+	fi
+fi
+
+AM_CONDITIONAL(HAVE_MINER_FLICKR, test "x$have_miner_flickr_deps" = "xyes")
+
 ####################################################################
 # Mail miners
 ####################################################################
@@ -1741,6 +1768,7 @@ AC_CONFIG_FILES([
 	src/tracker-extract/Makefile
 	src/tracker-miner-fs/Makefile
 	src/tracker-miner-rss/Makefile
+	src/tracker-miner-flickr/Makefile
 	src/tracker-preferences/Makefile
 	src/tracker-preferences/tracker-preferences.desktop.in
 	src/tracker-search-bar/Makefile
@@ -1866,6 +1894,10 @@ Plugins:
         Nautilus: (tagging widget)              $have_nautilus_extension
         Deskbar:                                $have_deskbar_applet
 
+Extra miners:
+
+	Flickr miner				$have_miner_flickr_deps
+
 Writeback:
 
 	MP3:                                    $have_id3lib
diff --git a/data/dbus/Makefile.am b/data/dbus/Makefile.am
index 0aab644..91e2505 100644
--- a/data/dbus/Makefile.am
+++ b/data/dbus/Makefile.am
@@ -22,6 +22,11 @@ service_in_files =						\
 	org.freedesktop.Tracker1.Miner.Applications.service.in	\
 	org.freedesktop.Tracker1.Miner.Files.service.in		\
 	org.freedesktop.Tracker1.Extract.service.in
+
+if HAVE_MINER_FLICKR
+service_in_files += org.freedesktop.Tracker1.Miner.Flickr.service.in
+endif
+
 service_DATA = $(service_in_files:.service.in=.service)
 
 %.service: %.service.in
diff --git a/data/dbus/org.freedesktop.Tracker1.Miner.Flickr.service.in b/data/dbus/org.freedesktop.Tracker1.Miner.Flickr.service.in
new file mode 100644
index 0000000..b06fe93
--- /dev/null
+++ b/data/dbus/org.freedesktop.Tracker1.Miner.Flickr.service.in
@@ -0,0 +1,3 @@
+[D-BUS Service]
+Name=org.freedesktop.Tracker1.Miner.Flickr
+Exec= libexecdir@/tracker-miner-flickr
diff --git a/data/icons/scalable/Makefile.am b/data/icons/scalable/Makefile.am
index 5583008..a8310e5 100644
--- a/data/icons/scalable/Makefile.am
+++ b/data/icons/scalable/Makefile.am
@@ -3,4 +3,13 @@ include $(top_srcdir)/Makefile.decl
 icondir = $(datadir)/icons/hicolor/scalable/apps
 icon_DATA = tracker.svg
 
-EXTRA_DIST = $(icon_DATA)
+minericonsdir = $(datadir)/tracker/icons
+minericons_DATA =
+
+if HAVE_MINER_FLICKR
+minericons_DATA += tracker-miner-flickr.svg
+endif
+
+EXTRA_DIST = 					\
+	$(icon_DATA)				\
+	$(minericons_DATA)
diff --git a/data/icons/scalable/tracker-miner-flickr.svg b/data/icons/scalable/tracker-miner-flickr.svg
new file mode 100644
index 0000000..bf5f830
--- /dev/null
+++ b/data/icons/scalable/tracker-miner-flickr.svg
@@ -0,0 +1,206 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/";
+   xmlns:cc="http://creativecommons.org/ns#";
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+   xmlns:svg="http://www.w3.org/2000/svg";
+   xmlns="http://www.w3.org/2000/svg";
+   xmlns:xlink="http://www.w3.org/1999/xlink";
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd";
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape";
+   width="48"
+   height="48"
+   id="svg2"
+   sodipodi:version="0.32"
+   inkscape:version="0.46"
+   version="1.0"
+   sodipodi:docname="flickr48.svg"
+   inkscape:output_extension="org.inkscape.output.svg.inkscape"
+   inkscape:export-filename="/home/cube/Dropbox/Inkscape-Files/flickr/flickr48.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs4">
+    <linearGradient
+       id="linearGradient6275">
+      <stop
+         id="stop6277"
+         offset="0"
+         style="stop-color:#f45d80;stop-opacity:1;" />
+      <stop
+         id="stop6279"
+         offset="1"
+         style="stop-color:#d40f3d;stop-opacity:1;" />
+    </linearGradient>
+    <linearGradient
+       id="linearGradient6261">
+      <stop
+         id="stop6263"
+         offset="0"
+         style="stop-color:#c5a0df;stop-opacity:1;" />
+      <stop
+         id="stop6265"
+         offset="1"
+         style="stop-color:#3e78b6;stop-opacity:1;" />
+    </linearGradient>
+    <inkscape:perspective
+       sodipodi:type="inkscape:persp3d"
+       inkscape:vp_x="0 : 526.18109 : 1"
+       inkscape:vp_y="0 : 1000 : 0"
+       inkscape:vp_z="744.09448 : 526.18109 : 1"
+       inkscape:persp3d-origin="372.04724 : 350.78739 : 1"
+       id="perspective10" />
+    <radialGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient6261"
+       id="radialGradient6259"
+       cx="7.3078551"
+       cy="18.809063"
+       fx="7.3078551"
+       fy="18.809063"
+       r="12.897959"
+       gradientUnits="userSpaceOnUse" />
+    <radialGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient6275"
+       id="radialGradient6273"
+       cx="6.4923115"
+       cy="17.852686"
+       fx="6.4923115"
+       fy="17.852686"
+       r="12.897959"
+       gradientUnits="userSpaceOnUse" />
+    <filter
+       inkscape:collect="always"
+       id="filter6287"
+       x="-0.13891452"
+       width="1.277829"
+       y="-0.83348718"
+       height="2.6669744">
+      <feGaussianBlur
+         inkscape:collect="always"
+         stdDeviation="1.0753131"
+         id="feGaussianBlur6289" />
+    </filter>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     gridtolerance="10000"
+     guidetolerance="10"
+     objecttolerance="10"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="8.1666667"
+     inkscape:cx="24"
+     inkscape:cy="24"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="true"
+     inkscape:window-width="1280"
+     inkscape:window-height="974"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:snap-nodes="false"
+     inkscape:snap-bbox="true">
+    <inkscape:grid
+       type="xygrid"
+       id="grid2383"
+       visible="true"
+       enabled="true" />
+  </sodipodi:namedview>
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage"; />
+        <dc:creator>
+          <cc:Agent>
+            <dc:title>Jakub Szypulka</dc:title>
+          </cc:Agent>
+        </dc:creator>
+        <dc:publisher>
+          <cc:Agent>
+            <dc:title>Jakub Szypulka</dc:title>
+          </cc:Agent>
+        </dc:publisher>
+        <dc:rights>
+          <cc:Agent>
+            <dc:title>Jakub Szypulka</dc:title>
+          </cc:Agent>
+        </dc:rights>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <path
+       sodipodi:type="arc"
+       style="opacity:1;fill:url(#radialGradient6259);fill-opacity:1;fill-rule:evenodd;stroke:#204a87;stroke-width:1.17254185999999990px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+       id="path5475"
+       sodipodi:cx="10.877922"
+       sodipodi:cy="23.407793"
+       sodipodi:rx="12.311688"
+       sodipodi:ry="12.311688"
+       d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+       transform="matrix(0.8528481,0,0,0.8528481,2.7227857,3.0367088)" />
+    <path
+       transform="matrix(0.8528481,0,0,0.8528481,25.722785,3.0367083)"
+       d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+       sodipodi:ry="12.311688"
+       sodipodi:rx="12.311688"
+       sodipodi:cy="23.407793"
+       sodipodi:cx="10.877922"
+       id="path5477"
+       style="opacity:1;fill:url(#radialGradient6273);fill-opacity:1;fill-rule:evenodd;stroke:#a40026;stroke-width:1.17254185999999994px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+       sodipodi:type="arc" />
+    <path
+       transform="matrix(0.7716245,0,0,0.7716245,3.6063293,4.9379737)"
+       d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+       sodipodi:ry="12.311688"
+       sodipodi:rx="12.311688"
+       sodipodi:cy="23.407793"
+       sodipodi:cx="10.877922"
+       id="path5479"
+       style="opacity:0.4;fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ffffff;stroke-width:1.29596722000000010px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+       sodipodi:type="arc" />
+    <path
+       sodipodi:type="arc"
+       style="opacity:0.4;fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ffffff;stroke-width:1.29596722px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;enable-background:accumulate"
+       id="path6251"
+       sodipodi:cx="10.877922"
+       sodipodi:cy="23.407793"
+       sodipodi:rx="12.311688"
+       sodipodi:ry="12.311688"
+       d="M 23.18961,23.407793 A 12.311688,12.311688 0 1 1 -1.4337664,23.407793 A 12.311688,12.311688 0 1 1 23.18961,23.407793 z"
+       transform="matrix(0.7716245,0,0,0.7716245,26.606329,4.9379737)" />
+    <path
+       sodipodi:type="arc"
+       style="opacity:0.3;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.90345997px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;filter:url(#filter6287);enable-background:accumulate"
+       id="path6281"
+       sodipodi:cx="14.449541"
+       sodipodi:cy="35.821102"
+       sodipodi:rx="9.288991"
+       sodipodi:ry="1.5481651"
+       d="M 23.738532,35.821102 A 9.288991,1.5481651 0 1 1 5.1605501,35.821102 A 9.288991,1.5481651 0 1 1 23.738532,35.821102 z"
+       transform="matrix(1.1659259,0,0,1.1190295,-4.6773701,-6.3524276)" />
+    <path
+       transform="matrix(1.2735802,0,0,1.1190295,16.427625,-6.3524276)"
+       d="M 23.738532,35.821102 A 9.288991,1.5481651 0 1 1 5.1605501,35.821102 A 9.288991,1.5481651 0 1 1 23.738532,35.821102 z"
+       sodipodi:ry="1.5481651"
+       sodipodi:rx="9.288991"
+       sodipodi:cy="35.821102"
+       sodipodi:cx="14.449541"
+       id="path6291"
+       style="opacity:0.3;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.90345997px;stroke-linecap:butt;stroke-linejoin:miter;marker:none;marker-start:none;marker-mid:none;marker-end:none;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;visibility:visible;display:inline;overflow:visible;filter:url(#filter6287);enable-background:accumulate"
+       sodipodi:type="arc" />
+  </g>
+</svg>
diff --git a/data/miners/Makefile.am b/data/miners/Makefile.am
index b435f45..8aa0a1b 100644
--- a/data/miners/Makefile.am
+++ b/data/miners/Makefile.am
@@ -18,7 +18,14 @@ if USING_MINER_RSS
 tracker_miners_DATA += 	tracker-miner-rss.desktop
 endif
 
+if HAVE_MINER_FLICKR
+tracker_miners_DATA += tracker-miner-flickr.desktop
+endif
+
 @INTLTOOL_DESKTOP_RULE@
 
+%.desktop.in: %.desktop.in.in
+	@sed -e "s|@datadir[ ]|$(datadir)|" $< > $@
+
 EXTRA_DIST = $(desktop_in_files)
 CLEANFILES = $(tracker_miners_DATA)
diff --git a/data/miners/tracker-miner-flickr.desktop.in.in b/data/miners/tracker-miner-flickr.desktop.in.in
new file mode 100644
index 0000000..a9f1ae9
--- /dev/null
+++ b/data/miners/tracker-miner-flickr.desktop.in.in
@@ -0,0 +1,8 @@
+[Desktop Entry]
+Encoding=UTF-8
+_Name=Flickr
+_Comment=Index your Flickr photo albums
+Icon= datadir@/tracker/icons/tracker-miner-flickr.svg
+DBusName=org.freedesktop.Tracker1.Miner.Flickr
+DBusPath=/org/freedesktop/Tracker1/Miner/Flickr
+AuthenticationMethod=Token
diff --git a/src/Makefile.am b/src/Makefile.am
index bbc1d09..00102c3 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -46,3 +46,7 @@ endif
 if USING_MINER_RSS
 SUBDIRS += tracker-miner-rss
 endif
+
+if HAVE_MINER_FLICKR
+SUBDIRS += tracker-miner-flickr
+endif
diff --git a/src/tracker-miner-flickr/Makefile.am b/src/tracker-miner-flickr/Makefile.am
new file mode 100644
index 0000000..54cabcc
--- /dev/null
+++ b/src/tracker-miner-flickr/Makefile.am
@@ -0,0 +1,64 @@
+include $(top_srcdir)/Makefile.decl
+
+INCLUDES =								\
+	-Wall								\
+	-DSHAREDIR=\""$(datadir)"\"					\
+	-DPKGLIBDIR=\""$(libdir)/tracker"\"				\
+	-DLOCALEDIR=\""$(localedir)"\" 					\
+	-DLIBEXEC_PATH=\""$(libexecdir)"\"				\
+	-DG_LOG_DOMAIN=\"Tracker\"					\
+	-DTRACKER_COMPILATION						\
+	-I$(top_srcdir)/src						\
+	$(WARN_CFLAGS)							\
+	$(GMODULE_CFLAGS)						\
+	$(PANGO_CFLAGS)							\
+	$(DBUS_CFLAGS)							\
+	$(MINER_FLICKR_CFLAGS)						\
+	$(GCOV_CFLAGS)
+
+VALAFLAGS =								\
+	--pkg gio-2.0							\
+	--pkg rest							\
+	--pkg posix							\
+	--thread
+
+libexec_PROGRAMS = tracker-miner-flickr
+
+tracker_miner_flickr_VALASOURCES =					\
+	tracker-miner-flickr.vala					\
+	query-queue.vala						\
+	$(top_srcdir)/src/libtracker-client/tracker-client-0.7.vapi
+
+tracker_miner_flickr_SOURCES =						\
+	$(tracker_miner_flickr_VALASOURCES:.vala=.c)
+
+tracker_miner_flickr_LDADD =						\
+	$(top_builddir)/src/libtracker-miner/libtracker-miner- TRACKER_API_VERSION@.la \
+	$(top_builddir)/src/libtracker-client/libtracker-client- TRACKER_API_VERSION@.la \
+	$(DBUS_LIBS)							\
+	$(GMODULE_LIBS)							\
+	$(GTHREAD_LIBS)							\
+	$(GIO_LIBS)							\
+	$(GCOV_LIBS)							\
+	$(GLIB2_LIBS)							\
+	$(MINER_FLICKR_LIBS)						\
+	-lz								\
+	-lm
+
+vapi_sources =								\
+	$(top_srcdir)/src/libtracker-miner/tracker-miner- TRACKER_API_VERSION@.vapi
+
+tracker-miner-flickr.vala.stamp: $(tracker_miner_flickr_VALASOURCES) $(vapi_sources)
+	$(AM_V_GEN)$(VALAC) $(GCOV_VALAFLAGS) -C $(VALAFLAGS) $^
+	touch $@
+
+
+BUILT_SOURCES = tracker-miner-flickr.vala.stamp
+
+MAINTAINERCLEANFILES =							\
+	tracker-miner-flickr.vala.stamp					\
+	$(tracker_miner_flickr_VALASOURCES:.vala=.c)
+
+EXTRA_DIST =								\
+	$(tracker_miner_flickr_VALASOURCES)				\
+	tracker-miner-flickr.vala.stamp
diff --git a/src/tracker-miner-flickr/query-queue.vala b/src/tracker-miner-flickr/query-queue.vala
new file mode 100644
index 0000000..b6104d1
--- /dev/null
+++ b/src/tracker-miner-flickr/query-queue.vala
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+public class QueryQueue : GLib.Object {
+	/* Holds the pending sparql updates and monitors them */
+	private HashTable<uint, string> queue;
+	private uint cookie;
+
+	private Mutex flush_mutex;
+
+	private Tracker.Miner miner;
+
+	public QueryQueue (Tracker.Miner parent) {
+		miner = parent;
+
+		queue = new HashTable<uint, string> (direct_hash, direct_equal);
+		cookie = 0;
+
+		flush_mutex = new Mutex ();
+	}
+
+	public async void append (string query) {
+		uint current_cookie = cookie ++;
+		queue.insert (current_cookie, query);
+
+		try {
+			yield miner.execute_batch_update (query);
+		} catch (Error tracker_error) {
+			warning ("BatchUpdate query failed: %s", tracker_error.message);
+		}
+
+		queue.remove (current_cookie);
+	}
+
+	/* BLOCKING flush */
+	public void flush () {
+		if (!flush_mutex.trylock ()) {
+			message ("There's already a flush taking place");
+			return;
+		}
+
+		if (queue.size () > 0) {
+			MainLoop wait_loop;
+			try {
+				wait_loop = new MainLoop (null, false);
+				miner.commit (null, () => { wait_loop.quit (); });
+				wait_loop.run ();
+			} catch (Error tracker_error) {
+				warning ("Commit query failed: %s", tracker_error.message);
+			}
+		}
+
+		flush_mutex.unlock ();
+	}
+
+	public uint size () {
+		return queue.size ();
+	}
+}
diff --git a/src/tracker-miner-flickr/tracker-miner-flickr.vala b/src/tracker-miner-flickr/tracker-miner-flickr.vala
new file mode 100644
index 0000000..acf5d5c
--- /dev/null
+++ b/src/tracker-miner-flickr/tracker-miner-flickr.vala
@@ -0,0 +1,607 @@
+/*
+ * Copyright (C) 2010, Adrien Bustany <abustany gnome org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+namespace Tracker {
+
+private errordomain RestCallError {
+	INVALID_RESPONSE, /* Malformed XML */
+	CALL_ERROR        /* Call failed */
+}
+
+public class MinerFlickr : Tracker.MinerWeb {
+	private static const string MINER_NAME = "Flickr";
+	private static const string MINER_DESCRIPTION = "Tracker miner for Flickr";
+	private static const string API_KEY = "7983269709fa3158c752e3e4d6b3b9e5";
+	private static const string SHARED_SECRET = "c0316d1cb4b15e2d";
+	private static const string DATASOURCE_URN = "urn:2208f9fc-3c5b-4e40-ade4-45a0d7b0cf6f";
+	private static const string FLICKR_AUTH_URL = "http://api.flickr.com/services/auth/";;
+	private static const string FLICKR_REST_URL = "http://api.flickr.com/services/rest/";;
+	private static const string FLICKR_PHOTOSET_URL = "http://www.flickr.com/photos/%s/sets/%s";;
+	private static const string FLICKR_PHOTO_URL = "http://farm%s.static.flickr.com/%s/%s_%s.jpg";;
+
+	/* Values taken from the EXIF spec */
+	private enum ExifTag {
+		CAMERA = 271,
+		FLASH = 37385,
+		FNUMBER = 33437,
+		FOCAL_LENGTH = 37386,
+		ISO_SPEED = 2,
+		METERING_MODE = 37383,
+		WHITE_BALANCE = 5
+	}
+
+	private enum ExifMeteringMode {
+		AVERAGE = 1,
+		CENTER_WEIGHTED_AVERAGE,
+		SPOT,
+		MULTISPOT,
+		PATTERN,
+		PARTIAL
+	}
+
+	private enum ExifWhiteBalance {
+		AUTO = 0,
+		MANUAL
+	}
+
+	private static const uint   PULL_INTERVAL = 5*60; /* seconds */
+	private uint pull_timeout_handle;
+
+	private QueryQueue query_queue;
+	private static MainLoop main_loop;
+
+	private Rest.Proxy rest;
+
+	/* Only used during association phase */
+	private string frob;
+	/* Used to sign calls */
+	private string auth_token;
+	/* Used to form some urls */
+	private string user_id;
+
+	construct {
+		set ("name", MINER_NAME);
+		set ("associated", false);
+		set ("status", "Idle");
+		set ("progress", 1.0);
+
+		rest = new Rest.Proxy (FLICKR_REST_URL, false);
+
+		query_queue = new QueryQueue (this);
+
+		this.notify["association"].connect (association_status_changed);
+	}
+
+	public void shutdown () {
+		set ("associated", false);
+	}
+
+	private void association_status_changed (Object source, ParamSpec pspec) {
+		bool associated;
+
+		get ("associated", out associated);
+
+		if (associated) {
+			if (pull_timeout_handle != 0)
+				return;
+
+			message ("Miner is now associated. Initiating periodic pull.");
+			pull_timeout_handle = Timeout.add_seconds (PULL_INTERVAL, pull_timeout_cb);
+			Idle.add ( () => { pull_timeout_cb (); return false; });
+		} else {
+			if (pull_timeout_handle == 0)
+				return;
+
+			Source.remove (pull_timeout_handle);
+		}
+	}
+
+	private bool pull_timeout_cb () {
+		init_pull ();
+		return true;
+	}
+
+	private async void init_pull () {
+		Rest.ProxyCall albums_call;
+		Rest.XmlNode photosets_node;
+
+		set ("status", "Refreshing photo albums");
+		set ("progress", 0.0);
+
+		Idle.add (init_pull.callback);
+		yield;
+
+
+		/* First get the list of albums */
+		albums_call = rest.new_call ();
+		albums_call.add_param ("method", "flickr.photosets.getList");
+
+		try {
+			photosets_node = run_call (albums_call);
+			insert_photosets (photosets_node);
+			query_queue.flush ();
+		} catch (Error call_error) {
+			warning ("Could not get photosets list: %s", call_error.message);
+		}
+
+		set ("status", "Idle");
+		set ("progress", 1.0);
+		message ("Pull finished");
+	}
+
+	private void insert_photosets (Rest.XmlNode root_node) {
+		Rest.XmlNode photoset_node;
+		Rest.XmlNode title_node;
+		Rest.XmlNode photos_node;
+		string photoset_url;
+		Rest.ProxyCall photos_call;
+		SparqlBuilder builder;
+		uint n_photosets;
+		uint indexed_photosets = 0;
+
+		photoset_node = root_node.find ("photoset");
+		n_photosets = root_node.children.size ();
+
+		while (photoset_node != null) {
+			photoset_url = FLICKR_PHOTOSET_URL.printf (user_id, photoset_node.get_attr ("id"));
+
+			message ("Getting photos for album %s", photoset_url);
+			builder = new SparqlBuilder.update ();
+			builder.insert_open (photoset_url);
+
+			builder.subject ("_:album");
+			builder.predicate ("a");
+			builder.object ("nfo:MediaList");
+			builder.predicate ("a");
+			builder.object ("nfo:RemoteDataObject");
+			builder.predicate ("nie:url");
+			builder.object_string (photoset_url);
+
+			title_node = photoset_node.find ("title");
+			if (title_node != null) {
+				builder.predicate ("dc:title");
+				builder.object_string (title_node.content);
+			}
+
+			builder.insert_close ();
+
+			query_queue.append (builder.get_result ());
+
+			set ("status", "Refresing album \"%s\"".printf (title_node.content));
+
+			try {
+				photos_call = rest.new_call ();
+				photos_call.add_params ("method", "flickr.photosets.getPhotos",
+				                        "photoset_id", photoset_node.get_attr ("id"),
+				                        "media", "photos",
+				                        "extras", "original_format");
+				photos_node = run_call (photos_call);
+				insert_photos (photos_node);
+			} catch (Error call_error) {
+				warning ("Could not list photos for photoset %s: %s", photoset_url, call_error.message);
+			}
+			photoset_node = photoset_node.next;
+
+			indexed_photosets ++;
+			set ("progress", (1.0*indexed_photosets)/n_photosets);
+		}
+	}
+
+	private void insert_photos (Rest.XmlNode root_node) {
+		Rest.XmlNode photoset_node;
+		string photoset_url;
+		Rest.XmlNode photo_node;
+		string photo_url;
+		SparqlBuilder builder;
+
+		photoset_node = root_node.find ("photoset");
+		if (photoset_node == null || photoset_node.get_attr ("id") == null) {
+			warning ("Malformed response for flickr.photosets.getPhotos");
+			return;
+		}
+
+		photoset_url = FLICKR_PHOTOSET_URL.printf (user_id, photoset_node.get_attr ("id"));
+		message ("Indexing photoset %s", photoset_url);
+
+		photo_node = root_node.find ("photo");
+
+		while (photo_node != null) {
+			photo_url = FLICKR_PHOTO_URL.printf (photo_node.get_attr ("farm"),
+			                                     photo_node.get_attr ("server"),
+			                                     photo_node.get_attr ("id"),
+			                                     photo_node.get_attr ("secret"));
+			builder = new SparqlBuilder.update ();
+			builder.insert_open (photo_url);
+
+			builder.subject ("_:photo");
+			builder.predicate ("a");
+			builder.object ("nmm:Photo");
+			builder.predicate ("a");
+			builder.object ("nfo:RemoteDataObject");
+			builder.predicate ("a");
+			builder.object ("nfo:MediaFileListEntry");
+
+			builder.predicate ("nie:url");
+			builder.object_string (photo_url);
+
+			insert_photo_info (photo_node, builder);
+			insert_exif_data (photo_node, builder);
+
+			builder.insert_close ();
+
+			query_queue.append (builder.get_result ());
+
+			photo_node = photo_node.next;
+		}
+	}
+
+	private void insert_photo_info (Rest.XmlNode photo_node, SparqlBuilder builder) {
+		var info_call = rest.new_call ();
+		Rest.XmlNode root_node;
+		Rest.XmlNode title_node;
+		Rest.XmlNode description_node;
+		Rest.XmlNode tag_node;
+
+		info_call.add_params ("method", "flickr.photos.getInfo",
+		                      "photo_id", photo_node.get_attr ("id"));
+
+		try {
+			root_node = run_call (info_call);
+		} catch (Error call_error) {
+			warning ("Couldn't get info for photo %s: %s", photo_node.get_attr ("id"), call_error.message);
+			return;
+		}
+
+		title_node = root_node.find ("title");
+		if (title_node != null && title_node.content != null) {
+			builder.predicate ("dc:title");
+			builder.object_string (title_node.content);
+		}
+
+		description_node = root_node.find ("description");
+		if (description_node != null && description_node.content != null) {
+			builder.predicate ("rdfs:comment");
+			builder.object_string (description_node.content);
+		}
+
+		tag_node = root_node.find ("tags").find ("tag");
+
+		while (tag_node != null) {
+			builder.predicate ("nao:hasTag");
+
+			builder.object_blank_open ();
+			builder.predicate ("a");
+			builder.object ("nao:Tag");
+			builder.predicate ("nao:prefLabel");
+			builder.object_string (tag_node.get_attr ("raw"));
+			builder.object_blank_close ();
+
+			tag_node = tag_node.next;
+		}
+	}
+
+	private void insert_exif_data (Rest.XmlNode photo_node, SparqlBuilder builder) {
+		var exif_call = rest.new_call ();
+		Rest.XmlNode root_node;
+		Rest.XmlNode exif_node;
+		string exif_value;
+
+		exif_call.add_params ("method", "flickr.photos.getExif",
+		                      "photo_id", photo_node.get_attr ("id"));
+
+		try {
+			root_node = run_call (exif_call);
+		} catch (Error call_error) {
+			warning ("Couldn't get EXIF data for photo %s: %s", photo_node.get_attr ("id"), call_error.message);
+			return;
+		}
+
+		exif_node = root_node.find ("exif");
+
+		while (exif_node != null) {
+			exif_value = exif_node.find ("raw").content;
+
+			switch (exif_node.get_attr ("tag").to_int ()) {
+				case ExifTag.CAMERA:
+					builder.predicate ("nmm:camera");
+					builder.object_string (exif_value);
+					break;
+				case ExifTag.FLASH:
+					builder.predicate ("nmm:flash");
+					builder.object (exif_value.to_int ()%2 == 1 ? "nmm:flash-on" : "nmm:flash-off");
+					break;
+				case ExifTag.FNUMBER:
+					builder.predicate ("nmm:fnumber");
+					builder.object_double (ratioToDouble (exif_value));
+					break;
+				case ExifTag.FOCAL_LENGTH:
+					builder.predicate ("nmm:focalLength");
+					builder.object_double (ratioToDouble (exif_value));
+					break;
+				case ExifTag.ISO_SPEED:
+					builder.predicate ("nmm:isoSpeed");
+					builder.object_int64 ((int64)exif_value.to_int ());
+					break;
+				case ExifTag.METERING_MODE:
+					builder.predicate ("nmm:meteringMode");
+					switch (exif_value.to_int ()) {
+						case ExifMeteringMode.AVERAGE:
+							builder.object ("nmm:meteringMode-average");
+							break;
+						case ExifMeteringMode.CENTER_WEIGHTED_AVERAGE:
+							builder.object ("nmm:meteringMode-center-weighted-average");
+							break;
+						case ExifMeteringMode.SPOT:
+							builder.object ("nmm:meteringMode-spot");
+							break;
+						case ExifMeteringMode.MULTISPOT:
+							builder.object ("nmm:meteringMode-multispot");
+							break;
+						case ExifMeteringMode.PATTERN:
+							builder.object ("nmm:meteringMode-pattern");
+							break;
+						case ExifMeteringMode.PARTIAL:
+							builder.object ("nmm:meteringMode-partial");
+							break;
+						default:
+							builder.object ("nmm:meteringMode-other");
+							break;
+					}
+					break;
+				case ExifTag.WHITE_BALANCE:
+					builder.predicate ("nmm:whiteBalance");
+					switch (exif_value.to_int ()) {
+						case ExifWhiteBalance.AUTO:
+							builder.object ("nmm:whiteBalance-auto");
+							break;
+						case ExifWhiteBalance.MANUAL:
+							builder.object ("nmm:whiteBalance-manual");
+							break;
+					}
+					break;
+				default:
+					break;
+			}
+			exif_node = exif_node.next;
+		}
+	}
+
+	private double ratioToDouble (string ratio) {
+		string[] tokens = ratio.split ("/");
+        if (tokens[1].to_int () == 0) {
+            critical ("fracToDouble : divide by zero while parsing ratio '%s'", ratio);
+            return 0;
+        }
+        return (tokens[0].to_int () * 1.0) / (tokens[1].to_int ());
+	}
+
+	private void sign_call (Rest.ProxyCall call) {
+		StringBuilder signature;
+		HashTable<string, string> parameters;
+		List<weak string> parameter_names;
+
+		call.add_param ("api_key", API_KEY);
+        if (auth_token != null)
+            call.add_param ("auth_token", auth_token);
+
+        signature = new StringBuilder (SHARED_SECRET);
+        parameters = call.get_params ();
+
+        parameter_names = parameters.get_keys ().copy ();
+        parameter_names.sort ((CompareFunc)strcmp);
+
+        foreach (string parameter in parameter_names) {
+            signature.append (parameter);
+			signature.append (parameters.lookup (parameter));
+        }
+
+        call.add_param ("api_sig", Checksum.compute_for_string (ChecksumType.MD5, signature.str));
+	}
+
+	private Rest.XmlNode? run_call (Rest.ProxyCall call) throws GLib.Error {
+		Rest.XmlParser parser;
+		Rest.XmlNode root_node;
+
+		sign_call (call);
+
+		try {
+			call.run (null);
+		} catch (Error call_error) {
+			throw call_error;
+		}
+
+		parser = new Rest.XmlParser ();
+		root_node = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
+		if (root_node == null || root_node.name != "rsp") {
+			throw new RestCallError.INVALID_RESPONSE ("Empty payload or root node not \"rsp\"");
+		}
+
+		return root_node;
+	}
+
+	public override HashTable<string, string> get_association_data () throws Tracker.MinerWebError {
+		var association_data = new HashTable<string, string> (str_hash, str_equal);
+		var frob_call = rest.new_call ();
+		Rest.XmlNode root_node;
+		Rest.XmlNode frob_node;
+		string api_signature;
+		string url;
+
+		frob_call.add_param ("method", "flickr.auth.getFrob");
+
+		try {
+			root_node = run_call (frob_call);
+		} catch (Error call_error) {
+			throw new MinerWebError.SERVICE ("Error while getting association data: %s", call_error.message);
+		}
+
+		frob_node = root_node.find ("frob");
+		if (frob_node == null || frob_node.content == null) {
+			throw new MinerWebError.SERVICE ("Malformed XML response while getting frob");
+		}
+
+		this.frob = frob_node.content;
+
+		api_signature = Checksum.compute_for_string (ChecksumType.MD5,
+		                                            SHARED_SECRET + "api_key" + API_KEY + "frob" + this.frob + "permsread");
+		url = FLICKR_AUTH_URL + "?api_key=" + API_KEY + "&perms=read&frob=" + this.frob + "&api_sig=" + api_signature;
+
+		association_data.insert ("url", url);
+
+		return association_data;
+	}
+
+	public override void associate (HashTable<string, string> association_data) throws Tracker.MinerWebError {
+		var password_provider = PasswordProvider.get ();
+		var token_call = rest.new_call ();
+		Rest.XmlNode root_node;
+		Rest.XmlNode token_node;
+		Rest.XmlNode user_node;
+
+		token_call.add_params ("method", "flickr.auth.getToken",
+		                       "frob", this.frob);
+
+		try {
+			root_node = run_call (token_call);
+		} catch (Error call_error) {
+			throw new MinerWebError.SERVICE ("Unable to get authentication token: %s", call_error.message);
+		}
+
+		token_node = root_node.find ("token");
+		user_node = root_node.find ("user");
+		if (token_node == null || token_node.content == null
+		 || user_node == null || user_node.get_attr ("username") == null) {
+			throw new MinerWebError.SERVICE ("Malformed XML response while getting token");
+		}
+
+		try {
+			password_provider.store_password (MINER_NAME,
+			                                  MINER_DESCRIPTION,
+			                                  user_node.get_attr ("username"),
+			                                  token_node.content);
+		} catch (Error e) {
+			if (e is PasswordProviderError.SERVICE) {
+				throw new MinerWebError.KEYRING (e.message);
+			}
+
+			critical ("Internal error: %s", e.message);
+			return;
+		}
+	}
+
+	public override void authenticate () throws MinerWebError {
+		PasswordProvider password_provider;
+		Rest.ProxyCall login_call;
+		Rest.XmlNode root_node;
+		Rest.XmlNode user_node;
+
+		password_provider = PasswordProvider.get ();
+
+		set ("associated", false);
+
+		try {
+			auth_token = password_provider.get_password (MINER_NAME, null);
+		} catch (Error e) {
+			if (e is PasswordProviderError.NOTFOUND) {
+				throw new MinerWebError.NO_CREDENTIALS ("Miner is not associated");
+			}
+			throw new MinerWebError.KEYRING (e.message);
+		}
+
+		login_call = rest.new_call ();
+		login_call.add_param ("method", "flickr.auth.checkToken");
+
+		try {
+			root_node = run_call (login_call);
+		} catch (Error call_error) {
+			throw new MinerWebError.SERVICE ("Cannot verify login: %s", call_error.message);
+		}
+
+		user_node = root_node.find ("user");
+		if (user_node == null || user_node.get_attr ("nsid") == null) {
+			throw new MinerWebError.WRONG_CREDENTIALS ("Stored authentication token is not valid");
+		}
+
+		user_id = user_node.get_attr ("nsid");
+
+		message ("Authentication successful");
+		set ("associated", true);
+	}
+
+	public override void dissociate () throws MinerWebError {
+		var password_provider = PasswordProvider.get ();
+
+		try {
+			password_provider.forget_password (MINER_NAME);
+		} catch (Error e) {
+			if (e is PasswordProviderError.SERVICE) {
+				throw new MinerWebError.KEYRING (e.message);
+			}
+
+			critical ("Internal error: %s", e.message);
+			return;
+		}
+
+		set ("associated", false);
+	}
+
+	private static bool in_loop = false;
+	private static void signal_handler (int signo) {
+		if (in_loop) {
+			Posix.exit (Posix.EXIT_FAILURE);
+		}
+
+		switch (signo) {
+			case Posix.SIGINT:
+			case Posix.SIGTERM:
+				in_loop = true;
+				main_loop.quit ();
+				break;
+		}
+	}
+
+	private static void init_signals () {
+#if G_OS_WIN32
+#else
+		Posix.sigaction_t act = Posix.sigaction_t ();
+		Posix.sigset_t    empty_mask = Posix.sigset_t ();
+		Posix.sigemptyset (empty_mask);
+		act.sa_handler = signal_handler;
+		act.sa_mask    = empty_mask;
+		act.sa_flags   = 0;
+
+		Posix.sigaction (Posix.SIGTERM, act, null);
+		Posix.sigaction (Posix.SIGINT, act, null);
+#endif
+	}
+
+	public static void main (string[] args) {
+		Environment.set_application_name ("Flickr tracker miner");
+		MinerFlickr flickr_miner = Object.new (typeof (MinerFlickr)) as MinerFlickr;
+
+		init_signals ();
+
+		main_loop = new MainLoop (null, false);
+		main_loop.run ();
+
+		flickr_miner.shutdown ();
+	}
+}
+
+} // End namespace Tracker



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]