[tracker/tracker-0.6] Fixes: NB#119230, tracker-extract: memory management shall be improved

[Martyn James Russell <mr src gnome org>]

commit 455ff69c90e2853c61b16501b2746a546a87d56c
Author: Martyn Russell <martyn imendio com>
Date:   Thu Jun 11 13:34:29 2009 +0100

    Fixes: NB#119230, tracker-extract: memory management shall be improved
    
    This is another stab at using setrlimit(). For now we set the memory
    limit to 80Mb as default and 512Mb for AMD. For some reason, anything
    less than ~450Mb doesn't even allow tracker-extract to dlopen the
    modules when we use GStreamer.
    
    Doing a quick test with tracker-stats before and after shows it is
    working on my AMD64 desktop and still indexing the same amount of
    content.
---
 src/libtracker-common/tracker-os-dependant-unix.c |  126 ++++++++++++++++-----
 src/tracker-extract/tracker-main.c                |    4 +-
 2 files changed, 100 insertions(+), 30 deletions(-)

diff --git a/src/libtracker-common/tracker-os-dependant-unix.c b/src/libtracker-common/tracker-os-dependant-unix.c
index cdbc282..304af4a 100644
--- a/src/libtracker-common/tracker-os-dependant-unix.c
+++ b/src/libtracker-common/tracker-os-dependant-unix.c
@@ -21,6 +21,8 @@
 
 #include "config.h"
 
+#include <string.h>
+#include <stdlib.h>
 #include <errno.h>
 #include <unistd.h>
 #include <sys/resource.h>
@@ -30,15 +32,21 @@
 #include "tracker-log.h"
 #include "tracker-os-dependant.h"
 
-#define MAX_MEM       128
-#define MAX_MEM_AMD64 512
-
-#define DISABLE_MEM_LIMITS
+/* Maximum here is a G_MAXLONG, so if you want to use > 2GB, you have
+ * to set MEM_LIMIT to RLIM_INFINITY
+ */
+#ifdef __x86_64__
+#define MEM_LIMIT 512 * 1024 * 1024
+#else
+#define MEM_LIMIT 80 * 1024 * 1024
+#endif
 
 #if defined(__OpenBSD__) && !defined(RLIMIT_AS)
-#define RLIMIT_AS      RLIMIT_DATA
+#define RLIMIT_AS RLIMIT_DATA
 #endif
 
+#undef DISABLE_MEM_LIMITS
+
 gboolean
 tracker_spawn (gchar **argv,
 	       gint    timeout,
@@ -212,43 +220,103 @@ tracker_create_permission_string (struct stat finfo)
 	return str;
 }
 
+static guint
+get_memory_total (void)
+{
+	GError      *error = NULL;
+	const gchar *filename;
+	gchar       *contents = NULL;
+	glong        total = 0;
+
+	filename = "/proc/meminfo";
+
+	if (!g_file_get_contents (filename,
+				  &contents,
+				  NULL,
+				  &error)) {
+		g_critical ("Couldn't get memory information:'%s', %s",
+			    filename,
+			    error ? error->message : "no error given");
+		g_clear_error (&error);
+	} else {
+		gchar *start, *end, *p;
+
+		start = "MemTotal:";
+		end = "kB";
+
+		p = strstr (contents, start);
+		if (p) {
+			p += strlen (start);
+			end = strstr (p, end);
+
+			if (end) {
+				*end = '\0';
+				total = 1024 * atol (p);
+			}
+		}
+
+		g_free (contents);
+	}
+
+	if (!total) {
+		/* Setting limit to an arbitary limit */
+		total = RLIM_INFINITY;
+	}
+
+	return total;
+}
+
 gboolean
 tracker_memory_setrlimits (void)
 {
 #ifndef DISABLE_MEM_LIMITS
 	struct rlimit rl;
-	gboolean      fail = FALSE;
+	glong         total;
+	glong         limit;
+
+	total = get_memory_total ();
+	limit = CLAMP (MEM_LIMIT, 0, total);
 
 	/* We want to limit the max virtual memory
 	 * most extractors use mmap() so only virtual memory can be
 	 * effectively limited.
 	 */
-#ifdef __x86_64__
-	/* Many extractors on AMD64 require 512M of virtual memory, so
-	 * we limit heap too.
-	 */
-	getrlimit (RLIMIT_AS, &rl);
-	rl.rlim_cur = MAX_MEM_AMD64 * 1024 * 1024;
-	fail |= setrlimit (RLIMIT_AS, &rl);
-
-	getrlimit (RLIMIT_DATA, &rl);
-	rl.rlim_cur = MAX_MEM * 1024 * 1024;
-	fail |= setrlimit (RLIMIT_DATA, &rl);
-#else  /* __x86_64__ */
-	/* On other architectures, 128M of virtual memory seems to be
-	 * enough.
-	 */
 	getrlimit (RLIMIT_AS, &rl);
-	rl.rlim_cur = MAX_MEM * 1024 * 1024;
-	fail |= setrlimit (RLIMIT_AS, &rl);
-#endif /* __x86_64__ */
+	rl.rlim_cur = limit;
+
+	if (setrlimit (RLIMIT_AS, &rl) == -1) {
+               const gchar *str = g_strerror (errno);
+
+               g_critical ("Could not set virtual memory limit with setrlimit(RLIMIT_AS), %s",
+			   str ? str : "no error given");
+
+               return FALSE;
+	} else {
+		getrlimit (RLIMIT_DATA, &rl);
+		rl.rlim_cur = limit;
+
+		if (setrlimit (RLIMIT_DATA, &rl) == -1) {
+			const gchar *str = g_strerror (errno);
 
-	if (fail) {
-		g_critical ("Error trying to set memory limit");
+			g_critical ("Could not set heap memory limit with setrlimit(RLIMIT_DATA), %s",
+				    str ? str : "no error given");
+
+			return FALSE;
+		} else {
+			gchar *str1, *str2;
+
+			str1 = g_format_size_for_display (total);
+			str2 = g_format_size_for_display (limit);
+
+			g_message ("Setting memory limitations: total is %s, virtual/heap set to %s",
+				   str1,
+				   str2);
+
+			g_free (str2);
+			g_free (str1);
+		}
 	}
+#endif /* DISABLE_MEM_LIMITS */
 
-	return !fail;
-#else  /* DISABLE_MEM_LIMITS */
 	return TRUE;
-#endif /* DISABLE_MEM_LIMITS */
 }
diff --git a/src/tracker-extract/tracker-main.c b/src/tracker-extract/tracker-main.c
index e143296..4f87545 100644
--- a/src/tracker-extract/tracker-main.c
+++ b/src/tracker-extract/tracker-main.c
@@ -185,8 +185,9 @@ signal_handler (int signo)
 	}
 
 	switch (signo) {
+	case SIGABRT:
 	case SIGALRM:
-		exit (EXIT_FAILURE);
+		_exit (EXIT_FAILURE);
 		break;
 	case SIGTERM:
 	case SIGINT:
@@ -219,6 +220,7 @@ initialize_signal_handler (void)
 	sigaction (SIGINT,  &act, NULL);
 	sigaction (SIGHUP,  &act, NULL);
 	sigaction (SIGALRM,  &act, NULL);
+	sigaction (SIGABRT, &act, NULL);
 #endif /* G_OS_WIN32 */
 }