[tracker/journal-rotation] Implemented backup() using tar, squash with next which will implement restore() too

[Philip Van Hoof <pvanhoof src gnome org>]

commit 45173d79fb4584b7cda0a1916bbb2bb30624fdb0
Author: Philip Van Hoof <philip codeminded be>
Date:   Fri May 7 13:31:04 2010 +0200

    Implemented backup() using tar, squash with next which will implement restore() too

 configure.ac                                      |    7 +
 src/libtracker-common/tracker-os-dependant-unix.c |    4 +-
 src/libtracker-data/tracker-data-backup.c         |  236 +++++++++++++++++++--
 src/libtracker-db/tracker-db-journal.c            |    9 +-
 src/libtracker-db/tracker-db-journal.h            |    1 +
 src/libtracker-db/tracker-db-manager.c            |    4 +-
 6 files changed, 233 insertions(+), 28 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 562ac15..fac9e20 100644
--- a/configure.ac
+++ b/configure.ac
@@ -227,6 +227,13 @@ fi
 
 AM_CONDITIONAL(HAVE_ENCA, test "$have_enca" = "yes")
 
+# We need tar for Backup and Restore support
+AC_PATH_PROG(TAR, tar, tar)
+if test -z $TAR; then
+   AC_MSG_ERROR([Could not find 'tar'])
+fi
+AC_DEFINE_UNQUOTED(TAR, "$TAR", [The tar program])
+
 AC_PATH_PROG(VALAC, valac, valac)
 AC_SUBST(VALAC)
 
diff --git a/src/libtracker-common/tracker-os-dependant-unix.c b/src/libtracker-common/tracker-os-dependant-unix.c
index 70dd389..0c8f5ea 100644
--- a/src/libtracker-common/tracker-os-dependant-unix.c
+++ b/src/libtracker-common/tracker-os-dependant-unix.c
@@ -90,8 +90,8 @@ tracker_spawn (gchar **argv,
 
 gboolean
 tracker_spawn_async_with_channels (const gchar **argv,
-                                   gint                  timeout,
-                                   GPid                 *pid,
+                                   gint          timeout,
+                                   GPid         *pid,
                                    GIOChannel  **stdin_channel,
                                    GIOChannel  **stdout_channel,
                                    GIOChannel  **stderr_channel)
diff --git a/src/libtracker-data/tracker-data-backup.c b/src/libtracker-data/tracker-data-backup.c
index 3c5c2f5..aef4bb4 100644
--- a/src/libtracker-data/tracker-data-backup.c
+++ b/src/libtracker-data/tracker-data-backup.c
@@ -25,6 +25,7 @@
 #include <libtracker-db/tracker-db-manager.h>
 #include <libtracker-db/tracker-db-journal.h>
 #include <libtracker-data/tracker-data-manager.h>
+#include <libtracker-common/tracker-os-dependant.h>
 
 #include "tracker-data-backup.h"
 
@@ -36,6 +37,17 @@ typedef struct {
 	GError *error;
 } BackupSaveInfo;
 
+typedef struct {
+	GPid pid;
+	guint stdout_watch_id;
+	guint stderr_watch_id;
+	GIOChannel *stdin_channel;
+	GIOChannel *stdout_channel;
+	GIOChannel *stderr_channel;
+	gpointer data;
+	GString *lines;
+} ProcessContext;
+
 GQuark
 tracker_data_backup_error_quark (void)
 {
@@ -63,22 +75,139 @@ free_backup_save_info (BackupSaveInfo *info)
 }
 
 static void
-on_journal_copied (GObject *source_object,
-                   GAsyncResult *res,
-                   gpointer user_data)
+on_journal_copied (BackupSaveInfo *info, GError *error)
 {
-	BackupSaveInfo *info = user_data;
-	GError *error = NULL;
-
-	g_file_copy_finish (info->journal, res, &error);
-
 	if (info->callback) {
 		info->callback (error, info->user_data);
 	}
 
 	free_backup_save_info (info);
+}
+
+
+
+static void
+process_context_destroy (ProcessContext *context, GError *error)
+{
+	on_journal_copied (context->data, error);
+
+	if (context->lines) {
+		g_string_free (context->lines, TRUE);
+	}
+
+	if (context->stdin_channel) {
+		g_io_channel_shutdown (context->stdin_channel, FALSE, NULL);
+		g_io_channel_unref (context->stdin_channel);
+		context->stdin_channel = NULL;
+	}
+
+	if (context->stdout_watch_id != 0) {
+		g_source_remove (context->stdout_watch_id);
+		context->stdout_watch_id = 0;
+	}
+
+	if (context->stdout_channel) {
+		g_io_channel_shutdown (context->stdout_channel, FALSE, NULL);
+		g_io_channel_unref (context->stdout_channel);
+		context->stdout_channel = NULL;
+	}
+
+	if (context->stderr_watch_id != 0) {
+		g_source_remove (context->stderr_watch_id);
+		context->stderr_watch_id = 0;
+	}
+
+	if (context->stderr_channel) {
+		g_io_channel_shutdown (context->stderr_channel, FALSE, NULL);
+		g_io_channel_unref (context->stderr_channel);
+		context->stderr_channel = NULL;
+	}
+
+	if (context->pid != 0) {
+		g_spawn_close_pid (context->pid);
+		context->pid = 0;
+	}
+
+	g_free (context);
+}
+
+static gboolean
+read_line_of_tar_output (GIOChannel  *channel,
+                         GIOCondition condition,
+                         gpointer     user_data)
+{
+	ProcessContext *context;
+
+	context = user_data;
+
+	if (condition & G_IO_ERR || condition & G_IO_HUP) {
+		return FALSE;
+	}
 
-	g_clear_error (&error);
+	/* TODO: progress support */
+	return TRUE;
+}
+
+static gboolean
+read_error_of_tar_output (GIOChannel  *channel,
+                          GIOCondition condition,
+                          gpointer     user_data)
+{
+	ProcessContext *context;
+	GIOStatus status;
+	gchar *line;
+
+	context = user_data;
+	status = G_IO_STATUS_NORMAL;
+
+	if (condition & G_IO_IN || condition & G_IO_PRI) {
+		do {
+			GError *error = NULL;
+
+			status = g_io_channel_read_line (channel, &line, NULL, NULL, &error);
+
+			if (status == G_IO_STATUS_NORMAL) {
+				if (context->lines == NULL)
+					context->lines = g_string_new (NULL);
+				g_string_append (context->lines, line);
+				g_free (line);
+			} else if (error) {
+				g_warning ("%s", error->message);
+				g_error_free (error);
+			}
+		} while (status == G_IO_STATUS_NORMAL);
+
+		if (status == G_IO_STATUS_EOF ||
+		    status == G_IO_STATUS_ERROR) {
+			return FALSE;
+		}
+	}
+
+	if (condition & G_IO_ERR || condition & G_IO_HUP) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static void
+process_context_child_watch_cb (GPid     pid,
+                                gint     status,
+                                gpointer user_data)
+{
+	ProcessContext *context;
+	GError *error = NULL;
+
+	g_debug ("Process '%d' exited with code %d", pid, status);
+
+	context = (ProcessContext *) user_data;
+
+	if (context->lines) {
+		g_set_error (&error, TRACKER_DATA_BACKUP_ERROR, 0, 
+		             "%s", context->lines->str);
+	}
+
+	process_context_destroy (context, error);
 }
 
 void
@@ -88,6 +217,16 @@ tracker_data_backup_save (GFile *destination,
                           GDestroyNotify destroy)
 {
 	BackupSaveInfo *info;
+	ProcessContext *context;
+	gchar **argv;
+	gchar *path, *directory;
+	GDir *journal_dir;
+	GFile *parent;
+	GIOChannel *stdin_channel, *stdout_channel, *stderr_channel;
+	GPid pid;
+	GPtrArray *files;
+	const gchar *f_name;
+	guint i;
 
 	info = g_new0 (BackupSaveInfo, 1);
 	info->destination = g_object_ref (destination);
@@ -96,16 +235,77 @@ tracker_data_backup_save (GFile *destination,
 	info->user_data = user_data;
 	info->destroy = destroy;
 
-	/* It's fine to copy this asynchronous: the journal replay code can or 
-	 * should cope with unfinished entries at the end of the file, while
-	 * restoring a backup made this way. */
+	parent = g_file_get_parent (info->journal);
+	directory = g_file_get_path (parent);
+	g_object_unref (parent);
+	path = g_file_get_path (destination);
+
+	journal_dir = g_dir_open (directory, 0, NULL);
+	f_name = g_dir_read_name (journal_dir);
+	files = g_ptr_array_new ();
+
+	while (f_name) {
+		if (f_name) {
+			if (!g_str_has_prefix (f_name, TRACKER_DB_JOURNAL_FILENAME ".")) {
+				f_name = g_dir_read_name (journal_dir);
+				continue;
+			}
+			g_ptr_array_add (files, g_strdup (f_name));
+		}
+		f_name = g_dir_read_name (journal_dir);
+	}
+
+	g_dir_close (journal_dir);
+
+	argv = g_new0 (gchar*, files->len + 7);
+
+	argv[0] = g_strdup (TAR);
+	argv[1] = g_strdup ("-zcf");
+	argv[2] = path;
+	argv[3] = g_strdup ("-C");
+	argv[4] = directory;
+	argv[5] = g_strdup (TRACKER_DB_JOURNAL_FILENAME);
+
+	for (i = 0; i < files->len; i++) {
+		argv[i+6] = g_ptr_array_index (files, i);
+	}
+
+	if (!tracker_spawn_async_with_channels ((const gchar **) argv,
+	                                        0, &pid,
+	                                        &stdin_channel,
+	                                        &stdout_channel,
+	                                        &stderr_channel)) {
+		GError *error = NULL;
+		g_set_error (&error, TRACKER_DATA_BACKUP_ERROR, 0, 
+		             "Error starting tar program");
+		on_journal_copied (info, error);
+		g_strfreev (argv);
+		g_error_free (error);
+		return;
+	}
+
+	context = g_new0 (ProcessContext, 1);
+	context->lines = NULL;
+	context->data = info;
+	context->pid = pid;
+	context->stdin_channel = stdin_channel;
+	context->stderr_channel = stderr_channel;
+	context->stdout_watch_id = g_io_add_watch (stdout_channel,
+	                                           G_IO_IN | G_IO_PRI | G_IO_ERR | G_IO_HUP,
+	                                           read_line_of_tar_output,
+	                                           context);
+	context->stderr_watch_id = g_io_add_watch (stderr_channel,
+	                                           G_IO_IN | G_IO_PRI | G_IO_ERR | G_IO_HUP,
+	                                           read_error_of_tar_output,
+	                                           context);
+
+	g_child_watch_add (context->pid, process_context_child_watch_cb, context);
+
+	g_strfreev (argv);
+
+	g_debug ("Process '%d' spawned for command:'%s %s %s'",
+	         pid, argv[0], argv[1], argv[2]);
 
-	g_file_copy_async (info->journal, info->destination,
-	                   G_FILE_COPY_OVERWRITE,
-	                   G_PRIORITY_HIGH,
-	                   NULL, NULL, NULL,
-	                   on_journal_copied,
-	                   info);
 }
 
 static gboolean
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index 8fc34f5..5db144d 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -43,7 +43,6 @@
 
 #include "tracker-db-journal.h"
 
-#define JOURNAL_FILENAME  "tracker-store.journal"
 #define MIN_BLOCK_SIZE    1024
 
 /*
@@ -291,7 +290,7 @@ tracker_db_journal_init (const gchar *filename,
 		writer.journal_filename = g_build_filename (g_get_user_data_dir (),
 		                                            "tracker",
 		                                            "data",
-		                                            JOURNAL_FILENAME,
+		                                            TRACKER_DB_JOURNAL_FILENAME,
 		                                            NULL);
 	}
 
@@ -655,7 +654,7 @@ tracker_db_journal_rotate (void)
 
 			if (f_name) {
 
-				if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+				if (!g_str_has_prefix (f_name, TRACKER_DB_JOURNAL_FILENAME ".")) {
 					f_name = g_dir_read_name (journal_dir);
 					continue;
 				}
@@ -666,7 +665,7 @@ tracker_db_journal_rotate (void)
 					cur = atoi (ptr);
 					max = MAX (cur, max);
 				}
-			}
+			} 
 
 			f_name = g_dir_read_name (journal_dir);
 		}
@@ -775,7 +774,7 @@ tracker_db_journal_reader_init (const gchar *filename)
 		filename_used = g_build_filename (g_get_user_data_dir (),
 		                                  "tracker",
 		                                  "data",
-		                                  JOURNAL_FILENAME,
+		                                  TRACKER_DB_JOURNAL_FILENAME,
 		                                  NULL);
 	}
 
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index 4ef0ca5..fefd515 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -29,6 +29,7 @@ G_BEGIN_DECLS
 
 #define TRACKER_DB_JOURNAL_ERROR_DOMAIN "TrackerDBJournal"
 #define TRACKER_DB_JOURNAL_ERROR        tracker_db_journal_error_quark()
+#define TRACKER_DB_JOURNAL_FILENAME     "tracker-store.journal"
 
 typedef enum {
 	TRACKER_DB_JOURNAL_START,
diff --git a/src/libtracker-db/tracker-db-manager.c b/src/libtracker-db/tracker-db-manager.c
index fcc8d82..39ced81 100644
--- a/src/libtracker-db/tracker-db-manager.c
+++ b/src/libtracker-db/tracker-db-manager.c
@@ -440,15 +440,13 @@ db_manager_remove_all (gboolean rm_journal)
 				gchar *fullpath;
 
 				if (f_name) {
-					if (!g_str_has_prefix (f_name, "tracker-store.journal.")) {
+					if (!g_str_has_prefix (f_name, TRACKER_DB_JOURNAL_FILENAME ".")) {
 						f_name = g_dir_read_name (journal_dir);
 						continue;
 					}
-					
 				}
 
 				fullpath = g_build_filename (directory, f_name, NULL);
-
 				file = g_file_new_for_path (fullpath);
 				g_file_delete (file, NULL, NULL);
 				g_object_unref (file);