GNOME.org
 

[tracker/dbus-fd-experiment: 2/7] libtracker-client: Add Steroids support

commit 35abf1a3a8183917a79e5ef4f6503ee256c225e2
Author: Adrien Bustany <abustany gnome org>
Date:   Mon May 24 08:52:02 2010 -0400

    libtracker-client: Add Steroids support
    
    This commit add several function in libtracker-client to exploit the new
    Steroids interface.
    
    The main function is tracker_resources_sparql_query_iterate, which will
    return a TrackerResultIterator. This iterator can then be used to
    iterate over the results using the tracker_result_iterator_* functions.
    Note that iteration is not done on DB side, all results are first fetch
    into a buffer on client side. This is because keeping an iterator on
    server side would block access to other clients, SQLite not being MVCC.

 src/libtracker-client/tracker.c |  535 +++++++++++++++++++++++++++++++++++++++
 src/libtracker-client/tracker.h |   21 ++
 2 files changed, 556 insertions(+), 0 deletions(-)
---
diff --git a/src/libtracker-client/tracker.c b/src/libtracker-client/tracker.c
index 455fb86..784c3d9 100644
--- a/src/libtracker-client/tracker.c
+++ b/src/libtracker-client/tracker.c
@@ -27,6 +27,11 @@
 #include <dbus/dbus-glib-bindings.h>
 
 #include <libtracker-common/tracker-dbus.h>
+#include <tracker-store/tracker-steroids.h>
+
+#include <errno.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 #include "tracker.h"
 
@@ -92,6 +97,7 @@
  **/
 
 typedef struct {
+	DBusGConnection *connection;
 	DBusGProxy *proxy_statistics;
 	DBusGProxy *proxy_resources;
 
@@ -141,6 +147,26 @@ typedef struct {
 	guint id;
 } CallbackArray;
 
+struct TrackerResultIterator {
+#ifdef HAVE_DBUS_FD_PASSING
+	int fd;
+	int rc;
+	char *buffer;
+	int message_size;
+	int buffer_size;
+	int buffer_index;
+	char *large_row_buf;
+
+	guint  n_columns;
+	int   *offsets;
+	char  *data;
+	gboolean has_next;
+#else
+	GPtrArray *results;
+	gint current_row;
+#endif
+};
+
 #endif /* TRACKER_DISABLE_DEPRECATED */
 
 static gboolean is_service_available (void);
@@ -360,6 +386,8 @@ client_constructed (GObject *object)
 		return;
 	}
 
+	private->connection = connection;
+
 	private->proxy_statistics =
 		dbus_g_proxy_new_for_name (connection,
 		                           TRACKER_DBUS_SERVICE,
@@ -386,6 +414,12 @@ client_constructed (GObject *object)
 	private->is_constructed = TRUE;
 }
 
+GQuark
+tracker_client_error_quark (void)
+{
+	return g_quark_from_static_string (TRACKER_CLIENT_ERROR_DOMAIN);
+}
+
 static void
 callback_with_gptrarray (DBusGProxy *proxy,
                          GPtrArray  *OUT_result,
@@ -677,6 +711,133 @@ find_conversion (const char  *format,
 	return start;
 }
 
+#ifdef HAVE_DBUS_FD_PASSING
+static int
+buffer_read_int (char *buf)
+{
+	int result = 0;
+
+	result += (((unsigned char)*(buf++)));
+	result += (((unsigned char)*(buf++)) <<  8);
+	result += (((unsigned char)*(buf++)) << 16);
+	result += (((unsigned char)*(buf++)) << 24);
+
+	return result;
+}
+
+static int
+iterator_buffer_read_int (TrackerResultIterator *iterator)
+{
+	int result;
+
+	result = buffer_read_int (iterator->buffer + iterator->buffer_index);
+
+	iterator->buffer_index += sizeof (int);
+
+	return result;
+}
+
+static gboolean
+pipe_read (int fd, char *dst, int size)
+{
+	ssize_t readsofar = 0;
+	ssize_t ret;
+
+	while (readsofar < size) {
+		ret = read (fd,
+		            dst + readsofar,
+		            size - readsofar);
+		if (ret < 0) {
+			switch (errno) {
+			case EAGAIN:
+				break;
+			case EPIPE:
+				g_critical ("SIGPIPE in pipe_read");
+				return FALSE;
+			default:
+				g_critical ("write returned %d in pipe_read", errno);
+				return FALSE;
+			}
+		} else {
+			readsofar += ret;
+		}
+	}
+
+	return TRUE;
+}
+
+static int
+iterator_buffer_fill (TrackerResultIterator *iterator)
+{
+	int rc;
+	int n_columns;
+	int *offsets;
+	int row_size;
+	int rows = 0;
+	gboolean force_page_load = FALSE;
+
+	for (;;) {
+		iterator->message_size += TRACKER_STEROIDS_BUFFER_SIZE;
+
+		while (iterator->message_size > iterator->buffer_size) {
+			iterator->buffer = g_realloc (iterator->buffer, 2 * iterator->buffer_size);
+			iterator->buffer_size *= 2;
+		}
+
+		if (!pipe_read (iterator->fd,
+		                iterator->buffer + iterator->buffer_index,
+		                TRACKER_STEROIDS_BUFFER_SIZE)) {
+			return -1;
+		}
+
+		for (;;) {
+			if ((unsigned char) (*(iterator->buffer + iterator->buffer_index)) == TRACKER_STEROIDS_EOP ||
+			    force_page_load) {
+				force_page_load = FALSE;
+				break;
+			}
+
+			rc = iterator_buffer_read_int (iterator);
+
+			switch (rc) {
+				case TRACKER_STEROIDS_RC_LARGEROW:
+					row_size = iterator_buffer_read_int (iterator);
+
+					while (iterator->buffer_size - iterator->buffer_index < row_size) {
+						iterator->buffer = g_realloc (iterator->buffer, 2 * iterator->buffer_size);
+						iterator->buffer_size *= 2;
+					}
+
+					if (!pipe_read (iterator->fd,
+					                iterator->buffer + iterator->buffer_index + TRACKER_STEROIDS_BUFFER_SIZE - 2 * sizeof (int),
+					                row_size - TRACKER_STEROIDS_BUFFER_SIZE + 2 * sizeof (int))) {
+						return -1;
+					}
+
+					force_page_load = TRUE;
+
+
+					/* Fall through to normal row handling */
+				case TRACKER_STEROIDS_RC_ROW:
+					n_columns = iterator_buffer_read_int (iterator);
+					iterator->n_columns = n_columns;
+					offsets = (int *) (iterator->buffer + iterator->buffer_index);
+					iterator->buffer_index += sizeof (int) * n_columns;
+					iterator->buffer_index += offsets[iterator->n_columns - 1] + 1;
+					rows ++;
+					break;
+				default:
+					goto end_of_results;
+			}
+		}
+	}
+
+end_of_results:
+	return rows;
+}
+
+#endif
+
 /**
  * tracker_uri_vprintf_escaped:
  * @format: a standard printf() format string, but notice
@@ -1046,6 +1207,380 @@ tracker_resources_sparql_query (TrackerClient  *client,
 }
 
 /**
+ * tracker_resources_sparql_query_iterate:
+ * @client: a #TrackerClient.
+ * @query: a string representing SPARQL.
+ * @error: a #GError.
+ *
+ * Queries the database using SPARQL, and returns an iterator instead of an
+ * array with all the results inside.
+ *
+ * Using an iterator will lower the memory usage. Additionally, this function
+ * uses a pipe when available get the results from Tracker store, which is
+ * roughly two times faster than using DBus.
+ *
+ * This API call is completely synchronous so it may block.
+ *
+ * <example>
+ * <title>Using tracker_resources_sparql_query_iterate(<!-- -->)</title>
+ * An example of using tracker_resources_sparql_query_iterate() to list all
+ * albums by title and include their song count and song total length.
+ * <programlisting>
+ *  TrackerClient *client;
+ *  TrackerResultIterator *iterator;
+ *  GError *error = NULL;
+ *  const gchar *query;
+ *
+ *  /&ast; Create D-Bus connection with no warnings and maximum timeout. &ast;/
+ *  client = tracker_client_new (0, G_MAXINT);
+ *  query = "SELECT {"
+ *          "  ?album"
+ *          "  ?title"
+ *          "  COUNT(?song) AS songs"
+ *          "  SUM(?length) AS totallength"
+ *          "} WHERE {"
+ *          "  ?album a nmm:MusicAlbum ;"
+ *          "  nie:title ?title ."
+ *          "  ?song nmm:musicAlbum ?album ;"
+ *          "  nfo:duration ?length"
+ *          "} "
+ *          "GROUP BY (?album");
+ *
+ *  iterator = tracker_resources_sparql_query_iterate (client, query, &error);
+ *
+ *  if (error) {
+ *          g_warning ("Could not query Tracker, %s", error->message);
+ *          g_error_free (error);
+ *          g_object_unref (client);
+ *          return;
+ *  }
+ *
+ *  while (tracker_result_iterator_has_next (iterator)) {
+ *          tracker_result_iterator_next (iterator);
+ *
+ *          g_message ("Album: %s, Title: %s",
+ *                     tracker_result_iterator_value (iterator, 0),
+ *                     tracker_result_iterator_value (iterator, 1));
+ *  }
+ *
+ *  tracker_result_iterator_free (iterator);
+ *
+ * </programlisting>
+ * </example>
+ *
+ * Returns: A #TrackerResultIterator pointing before the first result row. This
+ * iterator must be disposed when done using tracker_result_iterator_free().
+ *
+ * Since: 0.9
+ **/
+TrackerResultIterator*
+tracker_resources_sparql_query_iterate (TrackerClient  *client,
+                                        const gchar    *query,
+                                        GError        **error)
+{
+#ifdef HAVE_DBUS_FD_PASSING
+	TrackerClientPrivate *private;
+	DBusConnection *connection;
+	DBusMessage *message;
+	DBusMessage *reply;
+	DBusPendingCall *call;
+	guint query_id;
+	DBusError dbus_error;
+	TrackerResultIterator *iterator;
+
+	g_return_val_if_fail (TRACKER_IS_CLIENT (client), NULL);
+	g_return_val_if_fail (query, NULL);
+
+	private = TRACKER_CLIENT_GET_PRIVATE (client);
+
+	connection = dbus_g_connection_get_connection (private->connection);
+
+	dbus_error_init (&dbus_error);
+
+	message = dbus_message_new_method_call (TRACKER_STEROIDS_SERVICE,
+	                                        TRACKER_STEROIDS_PATH,
+	                                        TRACKER_STEROIDS_INTERFACE,
+	                                        "PrepareQuery");
+	dbus_message_append_args (message,
+	                          DBUS_TYPE_STRING, &query,
+	                          DBUS_TYPE_INVALID);
+	reply = dbus_connection_send_with_reply_and_block (connection,
+	                                                   message,
+	                                                   -1,
+	                                                   &dbus_error);
+	dbus_message_unref (message);
+
+	if (!reply) {
+		dbus_set_g_error (error, &dbus_error);
+		return NULL;
+	}
+
+	iterator = g_slice_new0 (TrackerResultIterator);
+	iterator->buffer_size = TRACKER_STEROIDS_BUFFER_SIZE;
+
+	dbus_message_get_args (reply,
+	                       &dbus_error,
+	                       DBUS_TYPE_UNIX_FD, &iterator->fd,
+	                       DBUS_TYPE_UINT32,  &query_id,
+	                       DBUS_TYPE_INVALID);
+	dbus_message_unref (reply);
+
+	message = dbus_message_new_method_call (TRACKER_STEROIDS_SERVICE,
+	                                        TRACKER_STEROIDS_PATH,
+	                                        TRACKER_STEROIDS_INTERFACE,
+	                                        "Fetch");
+	dbus_message_append_args (message,
+	                          DBUS_TYPE_UINT32, &query_id,
+	                          DBUS_TYPE_INVALID);
+	dbus_connection_send_with_reply (connection,
+	                                 message,
+	                                 &call,
+	                                 -1);
+
+	if (!call) {
+		g_set_error (error,
+		             TRACKER_CLIENT_ERROR,
+		             TRACKER_CLIENT_ERROR_UNSUPPORTED,
+		             "FD passing unsupported or connection disconnected");
+		return NULL;
+	}
+
+	iterator->buffer = g_malloc (TRACKER_STEROIDS_BUFFER_SIZE);
+	if (iterator_buffer_fill (iterator) < 0) {
+		g_set_error (error,
+		             TRACKER_CLIENT_ERROR,
+		             TRACKER_CLIENT_ERROR_BROKEN_PIPE,
+		             "Couldn't get results from server");
+		tracker_result_iterator_free (iterator);
+		return NULL;
+	}
+	iterator->buffer_index = 0;
+	iterator->rc = iterator_buffer_read_int (iterator);
+
+	/* Reset the iterator internal state */
+	iterator->buffer_index = 0;
+
+	if (iterator->rc == TRACKER_STEROIDS_RC_ROW ||
+	    iterator->rc == TRACKER_STEROIDS_RC_LARGEROW) {
+		iterator->has_next = TRUE;
+	}
+
+	dbus_pending_call_block (call);
+
+	reply = dbus_pending_call_steal_reply (call);
+
+	g_assert (reply);
+
+	if (dbus_message_get_type (reply) == DBUS_MESSAGE_TYPE_ERROR) {
+		dbus_set_error_from_message (&dbus_error, reply);
+		dbus_set_g_error (error, &dbus_error);
+		dbus_pending_call_unref (call);
+		return NULL;
+	}
+
+	dbus_message_unref (reply);
+
+	dbus_pending_call_unref (call);
+
+	return iterator;
+#else
+	TrackerResultIterator *iterator;
+	GError *inner_error = NULL;
+
+	g_return_val_if_fail (TRACKER_IS_CLIENT (client), NULL);
+	g_return_val_if_fail (query, NULL);
+
+	iterator = g_slice_new0 (TrackerResultIterator);
+
+	iterator->results = tracker_resources_sparql_query (client, query, &inner_error);
+	iterator->current_row = -1;
+
+	if (inner_error) {
+		g_propagate_error (error, inner_error);
+		g_slice_free (TrackerResultIterator, iterator);
+		iterator = NULL;
+	}
+
+	return iterator;
+#endif
+}
+
+/**
+ * tracker_result_iterator_free:
+ * @iterator: A TrackerResultIterator
+ *
+ * Frees a TrackerResultIterator and its associated resources
+ *
+ * Since: 0.9
+ **/
+void
+tracker_result_iterator_free (TrackerResultIterator *iterator)
+{
+#ifndef HAVE_DBUS_FD_PASSING
+	g_ptr_array_foreach (iterator->results, (GFunc) g_free, NULL);
+	g_ptr_array_free (iterator->results, TRUE);
+#else
+	if (iterator->buffer) {
+		g_free (iterator->buffer);
+	}
+	g_slice_free (TrackerResultIterator, iterator);
+#endif
+}
+
+/**
+ * tracker_result_iterator_n_columns:
+ * @iterator: A TrackerResultIterator
+ *
+ * Returns: the number of columns in the row pointed by @iterator
+ *
+ * Since: 0.9
+ **/
+guint
+tracker_result_iterator_n_columns (TrackerResultIterator *iterator)
+{
+#ifdef HAVE_DBUS_FD_PASSING
+	g_return_val_if_fail (iterator, 0);
+
+	return iterator->n_columns;
+#else
+	GStrv row;
+	guint i = 0;
+
+	g_return_val_if_fail (iterator, 0);
+
+	if (!iterator->results->len) {
+		return 0;
+	}
+
+	row = g_ptr_array_index (iterator->results, 0);
+
+	while (row[i++]) {
+	}
+
+	return i - 1;
+#endif
+}
+
+/**
+ * tracker_result_iterator_has_next:
+ * @iterator: A TrackerResultIterator
+ *
+ * Checks if the iterator has more rows
+ *
+ * Returns: TRUE if there are more rows to fetch, FALSE else
+ *
+ * Since: 0.9
+ **/
+gboolean
+tracker_result_iterator_has_next (TrackerResultIterator *iterator)
+{
+#ifdef HAVE_DBUS_FD_PASSING
+	g_return_val_if_fail (iterator, FALSE);
+
+	return iterator->has_next;
+#else
+	g_return_val_if_fail (iterator, FALSE);
+
+	if (!iterator->results->len) {
+		return FALSE;
+	}
+
+	return (iterator->current_row < (gint)(iterator->results->len - 1));
+#endif
+}
+
+/**
+ * tracker_result_iterator_next:
+ * @iterator: A TrackerResultIterator
+ *
+ * Fetches the next results row.
+ *
+ * Since: 0.9
+ **/
+void
+tracker_result_iterator_next (TrackerResultIterator  *iterator)
+{
+#ifdef HAVE_DBUS_FD_PASSING
+	int nextrc;
+
+	iterator->rc = iterator_buffer_read_int (iterator);
+	switch (iterator->rc) {
+	case TRACKER_STEROIDS_RC_LARGEROW:
+		/* Skip row size int */
+		iterator_buffer_read_int (iterator);
+	case TRACKER_STEROIDS_RC_ROW:
+		iterator->n_columns = iterator_buffer_read_int (iterator);
+		iterator->offsets = (int *)(iterator->buffer + iterator->buffer_index);
+		iterator->buffer_index += iterator->n_columns * sizeof (int);
+		iterator->data = iterator->buffer + iterator->buffer_index;
+		iterator->buffer_index += iterator->offsets[iterator->n_columns - 1] + 1;
+
+		nextrc = buffer_read_int (iterator->buffer + iterator->buffer_index);
+		iterator->has_next = (nextrc == TRACKER_STEROIDS_RC_ROW || nextrc == TRACKER_STEROIDS_RC_LARGEROW);
+		break;
+	case TRACKER_STEROIDS_RC_DONE:
+		break;
+	default:
+		/* If an error happened, it has been reported by
+		 * tracker_resources_sparql_query_iterate */
+		break;
+	}
+#else
+	g_return_if_fail (iterator);
+
+	if (!iterator->results->len) {
+		return;
+	}
+
+	if (iterator->current_row < (gint)iterator->results->len) {
+		iterator->current_row++;
+	}
+#endif
+}
+
+/**
+ * tracker_result_iterator_value:
+ * @iterator: A TrackerResultIterator
+ *
+ * Get a column's value as a string
+ *
+ * Returns: the value of the column as a string. The returned string belongs to
+ * the iterator and should not be freed.
+ *
+ * Since: 0.9
+ **/
+const gchar *
+tracker_result_iterator_value (TrackerResultIterator *iterator,
+                               guint                  column)
+{
+#ifdef HAVE_DBUS_FD_PASSING
+	g_return_val_if_fail (iterator, NULL);
+	g_return_val_if_fail (column < iterator->n_columns, NULL);
+
+	if (column == 0) {
+		return iterator->data;
+	} else {
+		return iterator->data + iterator->offsets[column-1] + 1;
+	}
+#else
+	GStrv row;
+
+	g_return_val_if_fail (iterator, NULL);
+	g_return_val_if_fail (column < tracker_result_iterator_n_columns (iterator), NULL);
+
+	if (!iterator->results->len) {
+		return NULL;
+	}
+
+	g_return_val_if_fail (iterator->current_row < (gint)iterator->results->len, NULL);
+
+	row = g_ptr_array_index (iterator->results, iterator->current_row);
+
+	return row[column];
+#endif
+}
+
+/**
  * tracker_resources_sparql_update:
  * @client: a #TrackerClient.
  * @query: a string representing SPARQL.
diff --git a/src/libtracker-client/tracker.h b/src/libtracker-client/tracker.h
index 0069907..f6fd5f0 100644
--- a/src/libtracker-client/tracker.h
+++ b/src/libtracker-client/tracker.h
@@ -49,6 +49,8 @@ typedef struct {
 	GObjectClass parent;
 } TrackerClientClass;
 
+typedef struct TrackerResultIterator TrackerResultIterator;
+
 /**
  * TrackerClientFlags:
  * @TRACKER_CLIENT_ENABLE_WARNINGS: If supplied warnings will be
@@ -59,6 +61,14 @@ typedef enum {
 	TRACKER_CLIENT_ENABLE_WARNINGS      = 1 << 0
 } TrackerClientFlags;
 
+#define TRACKER_CLIENT_ERROR        tracker_client_error_quark ()
+#define TRACKER_CLIENT_ERROR_DOMAIN "TrackerClient"
+
+typedef enum {
+	TRACKER_CLIENT_ERROR_UNSUPPORTED,
+	TRACKER_CLIENT_ERROR_BROKEN_PIPE
+} TrackerClientError;
+
 /**
  * TrackerReplyGPtrArray:
  * @result: a #GPtrArray with the results of the query.
@@ -98,6 +108,7 @@ typedef void (*TrackerWritebackCallback) (const GHashTable *resources,
                                           gpointer          user_data);
 
 GType          tracker_client_get_type                     (void) G_GNUC_CONST;
+GQuark         tracker_client_error_quark                  (void);
 TrackerClient *tracker_client_new                          (TrackerClientFlags      flags,
                                                             gint                    timeout);
 
@@ -122,6 +133,16 @@ void           tracker_resources_load                      (TrackerClient
 GPtrArray *    tracker_resources_sparql_query              (TrackerClient          *client,
                                                             const gchar            *query,
                                                             GError                **error);
+TrackerResultIterator *
+               tracker_resources_sparql_query_iterate      (TrackerClient          *client,
+                                                            const gchar            *query,
+                                                            GError                **error);
+void           tracker_result_iterator_free                (TrackerResultIterator  *iterator);
+guint          tracker_result_iterator_n_columns           (TrackerResultIterator  *iterator);
+gboolean       tracker_result_iterator_has_next            (TrackerResultIterator  *iterator);
+void           tracker_result_iterator_next                (TrackerResultIterator  *iterator);
+const gchar *  tracker_result_iterator_value               (TrackerResultIterator  *iterator,
+                                                            guint                   column);
 void           tracker_resources_sparql_update             (TrackerClient          *client,
                                                             const gchar            *query,
                                                             GError                **error);
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]