GNOME.org
 

[tracker/miner-web: 6/17] Add Facebook miner, although not functionnal currently

commit 1e93b294972c04299e0ab9333eec490e50fa3d4f
Author: Adrien Bustany <madcat mymadcat com>
Date:   Thu Nov 5 00:02:28 2009 -0300

    Add Facebook miner, although not functionnal currently
    
    While the Facebook miner compiles, it cannot be used currently because it does
    not export the Tracker.MinerWeb interface on DBus. Therefore, one cannot
    associate it to a Facebook account, which makes it pretty useless.
    This miner is here mainly for review, so that we can integrate it quickly.

 configure.ac                                       |   33 +
 ....freedesktop.Tracker1.Miner.Facebook.service.in |    3 +
 data/miners/Makefile.am                            |    6 +-
 data/miners/tracker-miner-facebook.desktop.in      |    7 +
 src/Makefile.am                                    |    3 +-
 src/tracker-miner-facebook/Makefile.am             |   67 ++
 src/tracker-miner-facebook/facebook.vala           |  686 ++++++++++++++++++++
 7 files changed, 802 insertions(+), 3 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 9e763cd..de3df73 100644
--- a/configure.ac
+++ b/configure.ac
@@ -155,6 +155,7 @@ EDS_REQUIRED=2.25.5
 LIBSTREAMANALYZER_REQUIRED=0.7.0
 GEE_REQUIRED=0.3
 GNOME_KEYRING_REQUIRED=2.26
+LIBREST_REQUIRED=0.6.1
 
 # Library Checks
 PKG_CHECK_MODULES(GLIB2, [glib-2.0 >= $GLIB_REQUIRED])
@@ -728,6 +729,37 @@ fi
 
 AM_CONDITIONAL(HAVE_GNOME_KEYRING, test "x$have_gnome_keyring" = "xyes")
 
+##################################################################
+# Enable librest support?
+##################################################################
+
+AC_ARG_ENABLE(librest,
+	      AS_HELP_STRING([--enable-librest],
+			     [enable librest (necessary for web miners) [[default=auto]]]),,
+	      [enable_librest=auto])
+
+if test "x$enable_librest" != "xno"; then
+	PKG_CHECK_MODULES(LIBREST,
+					  [ rest-0.6 >= $LIBREST_REQUIRED ],
+					  [have_librest=yes],
+					  [have_librest=no])
+	AC_SUBST(LIBREST_LIBS)
+	AC_SUBST(LIBREST_CFLAGS)
+
+	if test "x$have_librest" = "xyes"; then
+		AC_DEFINE(HAVE_LIBREST, [], [Define if we have librest for web miners])
+	fi
+fi
+
+if test "x$enable_librest" = "xyes"; then
+   if test "x$have_librest" != "xyes"; then
+      AC_MSG_ERROR([Couldn't find librest >= $LIBREST_REQUIRED.])
+   fi
+fi
+
+AM_CONDITIONAL(HAVE_LIBREST, test "x$have_librest" = "xyes")
+
+
 ####################################################################
 # Push modules
 ####################################################################
@@ -1496,6 +1528,7 @@ AC_CONFIG_FILES([
 	src/tracker-control/Makefile
 	src/tracker-extract/Makefile
 	src/tracker-miner-fs/Makefile
+	src/tracker-miner-facebook/Makefile
 	src/tracker-preferences/Makefile
 	src/tracker-preferences/tracker-preferences.desktop.in
 	src/tracker-search-bar/Makefile
diff --git a/data/dbus/org.freedesktop.Tracker1.Miner.Facebook.service.in b/data/dbus/org.freedesktop.Tracker1.Miner.Facebook.service.in
new file mode 100644
index 0000000..cd01265
--- /dev/null
+++ b/data/dbus/org.freedesktop.Tracker1.Miner.Facebook.service.in
@@ -0,0 +1,3 @@
+[D-BUS Service]
+Name=org.freedesktop.Tracker1.Miner.Facebook
+Exec= libexecdir@/tracker-miner-facebook
diff --git a/data/miners/Makefile.am b/data/miners/Makefile.am
index cdb1db4..4ea7bd3 100644
--- a/data/miners/Makefile.am
+++ b/data/miners/Makefile.am
@@ -2,13 +2,15 @@ include $(top_srcdir)/Makefile.decl
 
 desktop_in_files = \
 	tracker-miner-applications.desktop.in \
-	tracker-miner-files.desktop.in
+	tracker-miner-files.desktop.in        \
+	tracker-miner-facebook.desktop.in
 
 tracker_minersdir = $(datadir)/tracker/miners
 
 tracker_miners_DATA = \
 	tracker-miner-applications.desktop \
-	tracker-miner-files.desktop
+	tracker-miner-files.desktop        \
+	tracker-miner-facebook.desktop
 
 @INTLTOOL_DESKTOP_RULE@
 
diff --git a/data/miners/tracker-miner-facebook.desktop.in b/data/miners/tracker-miner-facebook.desktop.in
new file mode 100644
index 0000000..0ecf121
--- /dev/null
+++ b/data/miners/tracker-miner-facebook.desktop.in
@@ -0,0 +1,7 @@
+[Desktop Entry]
+Encoding=UTF-8
+_Name=Facebook
+_Comment=Index your data on Facebook
+DBusName=org.freedesktop.Tracker1.Miner.Facebook
+DBusPath=/org/freedesktop/Tracker1/Miner/Facebook
+X-Tracker-Bridge-AuthScheme=Token
diff --git a/src/Makefile.am b/src/Makefile.am
index 69b27ad..03fd031 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -14,7 +14,8 @@ SUBDIRS = 					\
 	tracker-control				\
 	tracker-utils				\
 	tracker-extract				\
-	tracker-miner-fs
+	tracker-miner-fs        \
+	tracker-miner-facebook
 
 if HAVE_TRACKER_PREFERENCES
 SUBDIRS += tracker-preferences
diff --git a/src/tracker-miner-facebook/Makefile.am b/src/tracker-miner-facebook/Makefile.am
new file mode 100644
index 0000000..1f84821
--- /dev/null
+++ b/src/tracker-miner-facebook/Makefile.am
@@ -0,0 +1,67 @@
+include $(top_srcdir)/Makefile.decl
+
+AM_CPPFLAGS = \
+    -include $(CONFIG_HEADER)
+
+VALAINCLUDES= \
+	--vapidir $(top_srcdir)/src/libtracker-miner \
+	--pkg posix \
+	--pkg dbus-glib-1 \
+	--pkg tracker-miner-web-0.7 \
+	--pkg tracker-miner-0.7 \
+	--pkg rest \
+	--pkg uuid \
+	--thread
+
+libexec_PROGRAMS=tracker-miner-facebook
+
+tracker_miner_facebook_VALASOURCES= \
+	facebook.vala
+
+tracker_miner_facebook_SOURCES= \
+	$(tracker_miner_facebook_VALASOURCES:.vala=.c)
+
+tracker-miner-facebook.vala.stamp: $(tracker_miner_facebook_VALASOURCES)
+	$(VALAC) -C $(VALAINCLUDES) $(VALAFLAGS) $^
+	touch $@
+
+INCLUDES = \
+	-Wall								\
+	-DSHAREDIR=\""$(datadir)"\"					\
+	-DPKGLIBDIR=\""$(libdir)/tracker"\"				\
+	-DLOCALEDIR=\""$(localedir)"\" 					\
+	-DLIBEXEC_PATH=\""$(libexecdir)"\"				\
+	-DG_LOG_DOMAIN=\"Tracker\"					\
+	-DTRACKER_COMPILATION						\
+	-I$(top_srcdir)/src						\
+    $(WARN_CFLAGS)                          \
+    $(GMODULE_CFLAGS)                          \
+    $(GLIB2_CFLAGS)                         \
+    $(GOBJECT_CFLAGS)                       \
+    $(GTHREAD_CFLAGS)                       \
+    $(DBUS_CFLAGS)                          \
+	$(LIBREST_CFLAGS)                          \
+	$(UUID_CFLAGS)
+
+tracker_miner_facebook_LDADD= \
+	$(top_builddir)/src/libtracker-miner/libtracker-miner.la	\
+    $(GLIB2_LIBS)                         \
+    $(GOBJECT_LIBS)                       \
+    $(GTHREAD_LIBS)                       \
+    $(DBUS_LIBS)                          \
+	$(LIBREST_LIBS)                          \
+	$(UUID_LIBS)
+
+BUILT_SOURCES= \
+	tracker-miner-facebook.vala.stamp
+
+CLEANFILES= $(BUILT_SOURCES)
+
+EXTRA_DIST= \
+	tracker-miner-facebook.vala.stamp \
+	$(tracker_miner_facebook_SOURCES) \
+	$(tracker_miner_facebook_VALASOURCES)
+
+MAINTAINERCLEANFILES= \
+	$(tracker_miner_facebook_SOURCES) \
+	tracker-miner-facebook.vala.stamp
diff --git a/src/tracker-miner-facebook/facebook.vala b/src/tracker-miner-facebook/facebook.vala
new file mode 100644
index 0000000..0b1b7e3
--- /dev/null
+++ b/src/tracker-miner-facebook/facebook.vala
@@ -0,0 +1,686 @@
+using Tracker;
+using Rest;
+
+public class FacebookMiner : Tracker.Miner, Tracker.MinerWeb {
+	private const string SERVICE_NAME = "Facebook";
+	private const string SERVICE_DESCRIPTION = "Authentication token for Facebook miner";
+	private const string API_KEY = "a07366931355e51525938ade2d0df2fb";
+	private const string SHARED_SECRET = "dd34c9d53460953bfd3b5aa87c09b538";
+	private const string FACEBOOK_REST = "https://api.facebook.com/restserver.php";;
+	private const string REST_ERRORMSG = "Error during REST call : %s";
+
+	private const uint update_interval = 600; // in seconds
+
+	dynamic DBus.Object tracker;
+
+	private Proxy rest;
+	private string auth_token;
+	private string session = null;
+	private string secret = SHARED_SECRET;
+
+	private string username = null; // Is actually the numeric user ID
+
+	private uint timeout_handle = 0;
+
+	private MinerWeb.AssociationStatus _miner_status;
+	private MinerWeb.AssociationStatus miner_status {
+		get { return _miner_status; }
+		set {
+			if (_miner_status == miner_status)
+				return;
+
+			_miner_status = miner_status ;
+			AssociationStatusChanged (miner_status);
+
+			if (miner_status == MinerWeb.AssociationStatus.ASSOCIATED && timeout_handle != 0) {
+				timeout_handle = Timeout.add_seconds (update_interval, pull);
+			}
+		}
+	}
+
+	private bool _is_paused = false;
+	private bool is_paused {
+		get { return _is_paused; }
+		set {
+			if (is_paused == _is_paused)
+				return;
+
+			_is_paused = is_paused ;
+			if (is_paused) {
+			} else {
+			}
+		}
+	}
+
+	construct {
+		// Set name for Tracker.Miner
+		set ("name", SERVICE_NAME);
+		miner_status = MinerWeb.AssociationStatus.UNASSOCIATED;
+	}
+
+	public FacebookMiner ()
+	{
+		rest = new Proxy (FACEBOOK_REST, false);
+	}
+
+	// Tracker.Miner functions
+	public override void started ()
+	{
+		try {
+			Authenticate ();
+		} catch (Error e) {
+			critical ("Error while authenticating : %s", e.message);
+		}
+	}
+
+	public override void stopped ()
+	{
+	}
+
+	public override void paused ()
+	{
+	}
+
+	public override void resumed ()
+	{
+	}
+
+	public override void terminated ()
+	{
+	}
+
+	public override void error (GLib.Error error)
+	{
+		critical ("An error occured : %s", error.message);
+	}
+
+	public override void progress (string status, double progress)
+	{
+	}
+
+	// Tracker.MinerWeb
+	public HashTable<string, string> AssociationData () throws MinerWebError
+	{
+		var ret = new HashTable<string, string> (str_hash, str_equal);
+		ProxyCall c = rest.new_call ();
+		c.add_param ("method", "auth.createToken");
+		XmlNode node;
+		try {
+			node = runCall (c);
+		} catch (MinerWebError e) {
+			critical ("Error during REST call : %s", e.message);
+			throw e;
+		}
+
+		if (node.name != "auth_createToken_response") {
+#if DEBUG
+			critical (_("Got answer %s\n", c.get_payload ()));
+			critical (_("Couldn't get authentication token"));
+			//error (new MinerWebError.SERVICE (_("Couldn't get authentication token")));
+#endif
+			return ret;
+		}
+
+		auth_token = node.content;
+		string url = "http://www.facebook.com/login.php?api_key=%s&v=1.0&auth_token=%s".printf (API_KEY, auth_token);
+		ret.insert ("url", url);
+
+		ret.insert ("post_message", _("A last browser window will now open, which will allow you to grant Tracker permanent access to your Facebook account, as well as access to your stream (statuses of your friends etc.). You're not obliged to do so, but if you choose not to grant Tracker these permissions, you'll need to login again every 24 hours, and Tracker will only index your photo albums."));
+		ret.insert ("post_url", "http://www.facebook.com/connect/prompt_permissions.php?api_key=%s&v=1.0&next=http://www.facebook.com/connect/login_success.html&display=popup&ext_perm=read_stream,offline_access".printf (API_KEY));
+		return ret;
+	}
+
+	// This supposes we have a valid auth_token. Else, well, it'll just fail...
+	public void Associate (HashTable<string, string> data) throws MinerWebError
+	{
+		ProxyCall c = rest.new_call ();
+		c.add_param ("method", "auth.getSession");
+		c.add_param ("auth_token", auth_token);
+
+		{
+			XmlNode node = runCall (c);
+
+			if (node.name != "auth_getSession_response") {
+#if DEBUG
+				critical ("Got answer %s\n", c.get_payload ());
+				critical ("Couldn't get session token");
+#endif
+				throw new MinerWebError.SERVICE (_(REST_ERRORMSG), node.find ("error_msg").content);
+			}
+
+			username = node.find ("uid").content;
+			secret = node.find ("secret").content;
+			session = node.find ("session_key").content;
+		}
+
+		try {
+			PasswordProvider.password_provider.store (SERVICE_NAME, SERVICE_DESCRIPTION, session, secret);
+		} catch (Error e) {
+			critical ("Couldn't store credentials in the keyring : %s", e.message);
+			throw new MinerWebError.KEYRING (e.message);
+		}
+	}
+
+	public void Authenticate () throws MinerWebError
+	{
+		string secret;
+		try {
+			secret = PasswordProvider.password_provider.get (SERVICE_NAME, out session);
+		} catch (Error e) {
+			if (e is PasswordProviderError.NOTFOUND) {
+				miner_status = MinerWeb.AssociationStatus.UNASSOCIATED;
+				throw new MinerWebError.NO_CREDENTIALS (_("Association needed"));
+			} else {
+				critical ("Couldn't access the keyring : %s", e.message);
+				throw new MinerWebError.KEYRING (e.message);
+			}
+		}
+
+		ProxyCall c = rest.new_call ();
+		c.add_param ("method", "users.getLoggedInUser");
+		XmlNode node;
+		try {
+			node = runCall (c);
+		} catch (MinerWebError e) {
+			throw e;
+		}
+
+		if (node.name != "users_getLoggedInUser_response") {
+#if DEBUG
+			critical ("Couldn't get user info\nGot answer %s\n", c.get_payload ());
+#endif
+			if (node.find ("error_code").content == "102") { // Session key invalid or no longer valid
+				session = null;
+				secret = SHARED_SECRET;
+				throw new MinerWebError.TOKEN_EXPIRED (_("Please associate again"));
+			}
+		} else {
+			username = node.content;
+			miner_status = MinerWeb.AssociationStatus.ASSOCIATED;
+		}
+	}
+
+	public MinerWeb.AssociationStatus GetAssociationStatus ()
+	{
+		return miner_status;
+	}
+
+	public bool pull ()
+	{
+		// Only accept new work if we're idle
+		if (miner_status != MinerWeb.AssociationStatus.ASSOCIATED && timeout_handle != 0) {
+			timeout_handle = 0;
+			return false;
+		}
+
+
+		// Be smart, only pull what's necessary (we're smart, aren't we ?)
+		string photos_pull_from = "1980-01-01T00:00:00Z,";
+		string stream_pull_from = "1980-01-01T00:00:00Z,";
+
+		try {
+			// FIXME : REGEX is not really the right solution here
+			string[][] results = tracker.SparqlQuery ("select  ?date where { ?album a nfo:MediaList . ?album nie:isStoredAs ?uri . ?album nie:contentLastModified ?date . FILTER (REGEX(str(?uri), 'www.facebook.com')) } ORDER BY DESC(?date) LIMIT 1");
+			if (results.length > 0) {
+				photos_pull_from = results[0][0];
+			}
+
+			results = tracker.SparqlQuery ("select  ?date where { ?message a mfo:FeedMessage . ?message nie:isStoredAs ?uri . ?message nmo:receivedDate ?date . FILTER (REGEX(str(?uri), 'www.facebook.com')) } ORDER BY DESC(?date) LIMIT 1");
+			if (results.length > 0) {
+				stream_pull_from = results[0][0];
+			}
+
+			message ("Pulling photos starting from %s and stream starting from %s", photos_pull_from, stream_pull_from);
+		} catch (Error e) {
+			critical ("Error contacting Tracker : %s", e.message);
+			return true;
+		}
+
+		// Pull photos
+		var c = rest.new_call ();
+		c.add_params ("method", "fql.multiquery",
+					  "queries", """{
+									"connections" : "SELECT target_id FROM connection WHERE source_id='%s' AND is_following=1",
+									"friends" : "SELECT uid, name FROM user WHERE uid IN (SELECT target_id FROM #connections)",
+									"photos" : "SELECT pid, src_big, caption, aid, owner, created FROM photo WHERE aid IN (SELECT aid FROM album WHERE owner IN (SELECT target_id FROM #connections)) AND modified > '%2$s'",
+									"albums" : "SELECT aid, name, description, link, owner, modified FROM album WHERE owner IN (SELECT target_id FROM #connections) AND modified > '%2$s'"
+									} """.printf (username, timestamp_from_iso8601 (photos_pull_from)));
+		runCall_async (c, pull_photos_cb);
+
+		// Pull streams
+		c = rest.new_call ();
+		c.add_params ("method", "fql.multiquery",
+					  "queries", """{
+									"stream":"SELECT post_id, actor_id, target_id, message, attachment, permalink, created_time FROM stream WHERE source_id IN (SELECT target_id FROM connection WHERE source_id='%s' AND is_following=1) AND created_time > '%s'",
+									"actors":"SELECT uid, name, pic_square FROM user WHERE uid IN (SELECT actor_id FROM #stream)"
+									}""".printf (username, timestamp_from_iso8601 (stream_pull_from)));
+		runCall_async (c, pull_stream_cb);
+
+		return true;
+	}
+
+	private void pull_photos_cb (ProxyCall call, GLib.Error err, Object weak_object)
+	{
+		message ("Pulling pictures");
+		if (err != null) {
+			critical ("Error while pulling pictures : %s", err.message);
+			//error (new MinerWebError.SERVICE (err.message));
+		}
+
+		var parser = new XmlParser ();
+		XmlNode root = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
+
+		weak XmlNode connections_node = null;
+		weak XmlNode friends_node = null;
+		weak XmlNode albums_node = null;
+		weak XmlNode photos_node = null;
+
+		{ // Assign the results groups to the right variables
+			XmlNode current = root.find ("fql_result");
+
+			if (current == null) {
+				critical ("Error in request : \n%s", call.get_payload ());
+				//error (new MinerWebError.SERVICE (_(REST_ERRORMSG), root.find ("error_msg").content));
+				return;
+			}
+
+			while (current != null) {
+				switch (current.find ("name").content) {
+					case "connections":
+						connections_node = current;
+						break;
+					case "friends":
+						friends_node = current;
+						break;
+					case "albums":
+						albums_node = current;
+						break;
+					case "photos":
+						photos_node = current;
+						break;
+					default:
+						break;
+				}
+
+				current = current.next;
+			}
+		}
+
+		// Maps the friend's uid to their uri
+		var friend_urn = new HashTable<string, string> (str_hash, str_equal);
+		{ // List contacts
+			weak XmlNode current_friend = friends_node.find ("fql_result_set").find ("user");
+
+			while (current_friend != null) {
+				friend_urn.insert (current_friend.find ("uid").content, get_contact (current_friend.find ("name").content));
+
+				current_friend = current_friend.next;
+			}
+		}
+
+		// Maps the album's aid to their uri
+		var album_urn = new HashTable<string, string> (str_hash, str_equal);
+		{ // List albums
+			weak XmlNode current_album = albums_node.find ("fql_result_set").find ("album");
+
+			while (current_album != null) {
+				string uri = current_album.find ("link").content;
+				string urn = get_resource ("nfo:MediaList", uri);
+
+				album_urn.insert (current_album.find ("aid").content, urn);
+
+				try {
+					string name = current_album.find ("name").content;
+					if (name != null) {
+						message ("Will index album %s", name);
+						tracker.SparqlUpdate ("insert {<%s> rdfs:label \"%s\"}".printf (urn, escape_string (name)));
+					}
+
+					string comment = current_album.find ("description").content;
+					if (comment != null) {
+						tracker.SparqlUpdate ("insert {<%s> rdfs:comment \"%s\"}".printf (urn, escape_string (comment)));
+					}
+
+					tracker.SparqlUpdate ("insert {<%s> nie:contentLastModified '%s'}".printf (urn, timestamp_to_iso8601 (current_album.find ("modified").content)));
+
+					tracker.SparqlUpdate ("insert {<%s> nco:creator <%s>}".printf (urn, friend_urn.lookup (current_album.find ("owner").content)));
+				} catch (Error e) {
+					critical ("Error while contacting Trakcer : %s", e.message);
+					//error (new MinerWebError.TRACKER (e.message));
+					return;
+				}
+
+				current_album = current_album.next;
+			}
+		}
+
+		{ // And finally, list photos (remember, that's why whe're here for)
+			weak XmlNode current_photo = photos_node.find ("fql_result_set").find ("photo");
+
+			while (current_photo != null) {
+				string uri = current_photo.find ("src_big").content;
+				if (uri == null) {
+					uri = current_photo.find ("src").content;
+				}
+
+				if (uri == null) {
+					current_photo = current_photo.next;
+					continue;
+				}
+				string urn = get_resource ("nmm:Photo", uri);
+
+				message ("Indexing photo %s with urn %s", uri, urn);
+
+				try {
+					string caption = current_photo.find ("caption").content;
+					if (caption != null) {
+						tracker.SparqlUpdate ("insert {<%s> rdfs:label \"%s\"}".printf (urn, escape_string (caption)));
+					}
+
+					string date = timestamp_to_iso8601 (current_photo.find ("created").content);
+					tracker.SparqlUpdate ("insert {<%s> nie:contentCreated '%s'}".printf (urn, date));
+
+					if (album_urn.lookup (current_photo.find ("aid").content) != null) {
+						tracker.SparqlUpdate ("insert {<%s> a nfo:MediaFileListEntry}".printf (urn));
+						tracker.SparqlUpdate ("insert {<%s> nfo:mediaListEntry <%s>}".printf (album_urn.lookup (current_photo.find ("aid").content), urn));
+					} else {
+						warning ("Unknown album for picture %s", uri);
+					}
+
+					tracker.SparqlUpdate ("insert {<%s> nco:creator <%s>}".printf (urn, friend_urn.lookup (current_photo.find ("owner").content)));
+				} catch (Error e) {
+					critical ("Error while contacting Tracker : %s", e.message);
+					//error (new MinerWebError.TRACKER (e.message));
+					return;
+				}
+
+				current_photo = current_photo.next;
+			}
+		}
+
+		message ("Photos pulled");
+	}
+
+	private void pull_stream_cb (ProxyCall call, GLib.Error? err, Object weak_object)
+	{
+		if (err != null) {
+			critical ("Error while pulling pictures : %s", err.message);
+			//error (new MinerWebError.SERVICE (err.message));
+			return;
+		}
+
+		var parser = new XmlParser ();
+		XmlNode root = parser.parse_from_data (call.get_payload (), call.get_payload_length ());
+		XmlNode stream_result;
+		XmlNode actors_result;
+		{
+			XmlNode result_1 = root.find ("fql_result");
+			if (result_1 == null) {
+				critical ("Error in request : \n%s", call.get_payload ());
+				//error (new MinerWebError.SERVICE (_(REST_ERRORMSG), root.find ("error_msg").content));
+				return;
+			}
+
+			if (result_1.find ("name").content == "stream") {
+				stream_result = result_1;
+				actors_result = result_1.next;
+			} else {
+				stream_result = result_1.next;
+				actors_result = result_1;
+			}
+		}
+
+		// Maps Facebook ids to nco urns
+		var authors = new HashTable<string, string>(str_hash, str_equal);
+
+		XmlNode current = actors_result.find ("fql_result_set").find ("user");
+		while (current != null) {
+			string facebook_id = current.find ("uid").content;
+			var author = current.find ("name").content;
+			try {
+				var authorUid = get_contact (author);
+
+				var avatarUrl = current.find ("pic_square").content;
+				var avatarUid = get_resource ("nfo:Image", avatarUrl);
+				tracker.SparqlUpdate ("insert {<%s> nco:photo <%s>}".printf (authorUid, avatarUid));
+
+				authors.insert (facebook_id, authorUid);
+				break;
+			} catch (Error e) {
+				critical ("Error contacting Tracker : %s", e.message);
+				//error (new MinerWebError.TRACKER (e.message));
+			}
+			current = current.next;
+		}
+
+		// Pull the stream
+		current = stream_result.find ("fql_result_set").find ("stream_post");
+		while (current != null) {
+			try {
+				string uri = current.find ("permalink").content;
+				string urn = get_resource ("mfo:FeedMessage", uri);
+
+				tracker.SparqlUpdate ("insert {<%s> nmo:from <%s>}".printf (urn, authors.lookup (current.find ("actor_id").content)));
+
+				string date = timestamp_to_iso8601 (current.find ("created_time").content);
+				tracker.SparqlUpdate ("insert {<%s> nmo:receivedDate \"%s\"}".printf (urn, date));
+
+				if (current.find ("message").content != null) {
+					tracker.SparqlUpdate ("insert {<%s> nmo:plainTextMessageContent \"%s\"}".printf (urn, escape_string (current.find ("message").content)));
+				}
+
+				// Deal with any attachment we might have
+				// We are obliged to first list the pictures, then do a query to get their src_big attribute
+				if (current.find ("attachment").children.size () > 0) {
+					// Maps the photo pid to the nmo:Message urn
+					var pictures = new List<string> ();
+
+					XmlNode current_media = current.find ("stream_media");
+					while (current_media != null) {
+						switch (current_media.find ("type").content) {
+							case "photo":
+								pictures.append (current_media.find ("photo").find ("pid").content);
+								break;
+							default:
+								warning ("Media type %s not handled yet", current_media.find ("type").content);
+								break;
+						}
+						current_media = current_media.next;
+					}
+
+					if (pictures != null) {
+						weak List<string> current_pic = pictures.first ();
+						var query = new StringBuilder ("SELECT src_big FROM photo WHERE pid = '%s'".printf (current_pic.data));
+
+						while ((current_pic = current_pic.next) != null) {
+							query.append_printf (" OR pid = '%s'", current_pic.data);
+						}
+
+						var c = rest.new_call ();
+						c.add_params ("method", "fql.query",
+									  "query", query.str);
+
+						// here we can block, since we don't bother anyone
+						XmlNode photos_node;
+						try {
+							photos_node = runCall (c);
+						} catch (MinerWebError e) {
+							critical ("REST call failed!");
+							return;
+						}
+
+						XmlNode current_photo = photos_node.find ("photo");
+						while (current_photo != null) {
+							var picture_uri = current_photo.find ("src_big").content;
+							var enclosure_urn = get_resource ("nmm:Photo", picture_uri);
+
+							tracker.SparqlUpdate ("insert {<%s> a mfo:Enclosure}".printf (enclosure_urn));
+							tracker.SparqlUpdate ("insert {<%s> mfo:remoteLink <%s>}".printf (enclosure_urn, picture_uri));
+							tracker.SparqlUpdate ("insert {<%s> mfo:enclosureList <%s>}".printf (urn, enclosure_urn));
+
+							// The rest of the photo indexing will be done by pull_photos_cb
+
+							current_photo = current_photo.next;
+						}
+					}
+				}
+			} catch (Error e) {
+				critical ("Error while inserting data into Tracker : %s", e.message);
+				//error (new MinerWebError.TRACKER (e.message));
+			}
+			current = current.next;
+		}
+		message ("Stream pulled");
+	}
+
+	// Private functions
+
+	private XmlNode? runCall (ProxyCall c) throws MinerWebError
+	{
+		signCall (c);
+
+		try {
+			c.run (null);
+		} catch (Error e) {
+			critical ("Error in REST call : %s\n", e.message);
+			//error (new MinerWebError.SERVICE (_(REST_ERRORMSG), e.message));
+			throw new MinerWebError.SERVICE (e.message);
+		}
+
+		var parser = new XmlParser ();
+		weak XmlNode ret = parser.parse_from_data (c.get_payload (), c.get_payload_length ());
+		return ret;
+	}
+
+
+	private void runCall_async (ProxyCall c, ProxyCallAsyncCallback callback)
+	{
+		signCall (c);
+
+		try {
+			c.run_async (callback, this);
+		} catch (Error e) {
+			critical ("Error in REST call : %s\n", e.message);
+			//error (new MinerWebError.SERVICE (_(REST_ERRORMSG), e.message));
+			throw new MinerWebError.SERVICE (e.message);
+		}
+	}
+
+	// Add version, api_key, call_id and sig
+	private void signCall (ProxyCall c)
+	{
+		var time = TimeVal ();
+		var callid = "%ld".printf(1000000*(time.tv_sec) + time.tv_usec);
+		c.add_params ("v", "1.0",
+					  "api_key", API_KEY,
+					  "call_id", callid,
+					  null);
+
+		if (session != null)
+			c.add_param ("session_key", session);
+
+		string sig = "";
+		var par = c.get_params ();
+		List<weak string> keys = par.get_keys ().copy ();
+		keys.sort ((CompareFunc)strcmp);
+
+		for (int i = 0 ; i < keys.length () ; ++i) {
+			sig += keys.nth_data (i) + "=" + par.lookup (keys.nth_data (i));
+		}
+		sig += secret;
+
+		sig = Checksum.compute_for_string (ChecksumType.MD5, sig);
+
+		c.add_param ("sig", sig);
+	}
+
+	// tries to find a resource with the given url as a nie:isStoredAs property. If none is foud, creates one (return value is urn)
+	private string? get_resource (string klass, string stored_as)
+	{
+		string[][] results;
+		try {
+			results = tracker.SparqlQuery ("select ?r where { ?r a %s . ?r nie:isStoredAs \"%s\" }".printf (klass, stored_as));
+		} catch (Error e) {
+			critical ("Couldn't contact Tracker : %s", e.message);
+			//error (new MinerWebError.TRACKER (e.message));
+			return null;
+		}
+
+		switch (results.length) {
+			case 0:
+				string urn = "urn:uuid:%s".printf (uuid_generate_string ());
+				tracker.SparqlUpdate ("insert {<%s> a %s}".printf (urn, klass));
+				tracker.SparqlUpdate ("insert {<%s> a nfo:RemoteDataObject}".printf (stored_as));
+				tracker.SparqlUpdate ("insert {<%s> nie:isStoredAs <%s>}".printf (urn, stored_as));
+				return urn;
+			case 1:
+				return results[0][0];
+			default:
+				warning ("More than one object of type %s and with nie:isStoredAs '%s'. Returning the first one", klass, stored_as);
+				return results[0][0];
+		}
+	}
+
+	// tries to find a nco:Contact with the given nco:fullname. If none is foud, creates one (return value is urn)
+	private string? get_contact (string fullname)
+	{
+		string[][] results;
+		string escaped_fullname = escape_string (fullname);
+		try {
+			results = tracker.SparqlQuery ("select ?r where { ?r a nco:Contact . ?r nco:fullname \"%s\" }".printf (escaped_fullname));
+		} catch (Error e) {
+			critical ("Couldn't contact Tracker : %s", e.message);
+			//error (new MinerWebError.TRACKER (e.message));
+			return null;
+		}
+
+		switch (results.length) {
+			case 0:
+				string urn = "urn:uuid:%s".printf (uuid_generate_string ());
+				tracker.SparqlUpdate ("insert {<%s> a nco:Contact}".printf (urn));
+				tracker.SparqlUpdate ("insert {<%s> nco:fullname \"%s\"}".printf (urn, escaped_fullname));
+				return urn;
+			case 1:
+				return results[0][0];
+			default:
+				warning ("More than one nco:Contact with nco:fullname '%s'. Returning the first one", fullname);
+				return results[0][0];
+		}
+	}
+
+	private string escape_string(string str) {
+		return str.escape ("").replace ("'", "\\'").replace ("\"", "\\\"");
+	}
+
+
+	private string timestamp_to_iso8601 (string timestamp)
+	{
+		GLib.TimeVal timeval = GLib.TimeVal ();
+		timeval.tv_sec = timestamp.to_long ();
+		return timeval.to_iso8601 ();
+	}
+
+	private string? timestamp_from_iso8601 (string isodate)
+	{
+		GLib.TimeVal timeval = GLib.TimeVal ();
+		if (!timeval.from_iso8601 (isodate))
+			return null;
+
+		return "%ld".printf (timeval.tv_sec);
+	}
+
+}
+
+MainLoop loop;
+
+void main ()
+{
+	loop = new MainLoop (null, false);
+
+	Environment.set_application_name ("FacebookBrige");
+
+	var miner = new FacebookMiner ();
+	miner.start ();
+
+	loop.run ();
+}
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]