[tracker] Fixes: NB#119230, tracker-extract: memory management shall be improved
- From: Philip Van Hoof <pvanhoof src gnome org>
- To: svn-commits-list gnome org
- Subject: [tracker] Fixes: NB#119230, tracker-extract: memory management shall be improved
- Date: Fri, 12 Jun 2009 04:23:10 -0400 (EDT)
commit 3f42a4390d48c6a4840a7bcfce74673ebfd23479
Author: Martyn Russell <martyn imendio com>
Date: Thu Jun 11 13:34:29 2009 +0100
Fixes: NB#119230, tracker-extract: memory management shall be improved
This is another stab at using setrlimit(). For now we set the memory
limit to 80Mb as default and 512Mb for AMD. For some reason, anything
less than ~450Mb doesn't even allow tracker-extract to dlopen the
modules when we use GStreamer.
Doing a quick test with tracker-stats before and after shows it is
working on my AMD64 desktop and still indexing the same amount of
content.
src/libtracker-common/tracker-os-dependant-unix.c | 126 ++++++++++++++++-----
src/tracker-extract/tracker-main.c | 4 +-
2 files changed, 100 insertions(+), 30 deletions(-)
---
diff --git a/src/libtracker-common/tracker-os-dependant-unix.c b/src/libtracker-common/tracker-os-dependant-unix.c
index cdbc282..304af4a 100644
--- a/src/libtracker-common/tracker-os-dependant-unix.c
+++ b/src/libtracker-common/tracker-os-dependant-unix.c
@@ -21,6 +21,8 @@
#include "config.h"
+#include <string.h>
+#include <stdlib.h>
#include <errno.h>
#include <unistd.h>
#include <sys/resource.h>
@@ -30,15 +32,21 @@
#include "tracker-log.h"
#include "tracker-os-dependant.h"
-#define MAX_MEM 128
-#define MAX_MEM_AMD64 512
-
-#define DISABLE_MEM_LIMITS
+/* Maximum here is a G_MAXLONG, so if you want to use > 2GB, you have
+ * to set MEM_LIMIT to RLIM_INFINITY
+ */
+#ifdef __x86_64__
+#define MEM_LIMIT 512 * 1024 * 1024
+#else
+#define MEM_LIMIT 80 * 1024 * 1024
+#endif
#if defined(__OpenBSD__) && !defined(RLIMIT_AS)
-#define RLIMIT_AS RLIMIT_DATA
+#define RLIMIT_AS RLIMIT_DATA
#endif
+#undef DISABLE_MEM_LIMITS
+
gboolean
tracker_spawn (gchar **argv,
gint timeout,
@@ -212,43 +220,103 @@ tracker_create_permission_string (struct stat finfo)
return str;
}
+static guint
+get_memory_total (void)
+{
+ GError *error = NULL;
+ const gchar *filename;
+ gchar *contents = NULL;
+ glong total = 0;
+
+ filename = "/proc/meminfo";
+
+ if (!g_file_get_contents (filename,
+ &contents,
+ NULL,
+ &error)) {
+ g_critical ("Couldn't get memory information:'%s', %s",
+ filename,
+ error ? error->message : "no error given");
+ g_clear_error (&error);
+ } else {
+ gchar *start, *end, *p;
+
+ start = "MemTotal:";
+ end = "kB";
+
+ p = strstr (contents, start);
+ if (p) {
+ p += strlen (start);
+ end = strstr (p, end);
+
+ if (end) {
+ *end = '\0';
+ total = 1024 * atol (p);
+ }
+ }
+
+ g_free (contents);
+ }
+
+ if (!total) {
+ /* Setting limit to an arbitary limit */
+ total = RLIM_INFINITY;
+ }
+
+ return total;
+}
+
gboolean
tracker_memory_setrlimits (void)
{
#ifndef DISABLE_MEM_LIMITS
struct rlimit rl;
- gboolean fail = FALSE;
+ glong total;
+ glong limit;
+
+ total = get_memory_total ();
+ limit = CLAMP (MEM_LIMIT, 0, total);
/* We want to limit the max virtual memory
* most extractors use mmap() so only virtual memory can be
* effectively limited.
*/
-#ifdef __x86_64__
- /* Many extractors on AMD64 require 512M of virtual memory, so
- * we limit heap too.
- */
- getrlimit (RLIMIT_AS, &rl);
- rl.rlim_cur = MAX_MEM_AMD64 * 1024 * 1024;
- fail |= setrlimit (RLIMIT_AS, &rl);
-
- getrlimit (RLIMIT_DATA, &rl);
- rl.rlim_cur = MAX_MEM * 1024 * 1024;
- fail |= setrlimit (RLIMIT_DATA, &rl);
-#else /* __x86_64__ */
- /* On other architectures, 128M of virtual memory seems to be
- * enough.
- */
getrlimit (RLIMIT_AS, &rl);
- rl.rlim_cur = MAX_MEM * 1024 * 1024;
- fail |= setrlimit (RLIMIT_AS, &rl);
-#endif /* __x86_64__ */
+ rl.rlim_cur = limit;
+
+ if (setrlimit (RLIMIT_AS, &rl) == -1) {
+ const gchar *str = g_strerror (errno);
+
+ g_critical ("Could not set virtual memory limit with setrlimit(RLIMIT_AS), %s",
+ str ? str : "no error given");
+
+ return FALSE;
+ } else {
+ getrlimit (RLIMIT_DATA, &rl);
+ rl.rlim_cur = limit;
+
+ if (setrlimit (RLIMIT_DATA, &rl) == -1) {
+ const gchar *str = g_strerror (errno);
- if (fail) {
- g_critical ("Error trying to set memory limit");
+ g_critical ("Could not set heap memory limit with setrlimit(RLIMIT_DATA), %s",
+ str ? str : "no error given");
+
+ return FALSE;
+ } else {
+ gchar *str1, *str2;
+
+ str1 = g_format_size_for_display (total);
+ str2 = g_format_size_for_display (limit);
+
+ g_message ("Setting memory limitations: total is %s, virtual/heap set to %s",
+ str1,
+ str2);
+
+ g_free (str2);
+ g_free (str1);
+ }
}
+#endif /* DISABLE_MEM_LIMITS */
- return !fail;
-#else /* DISABLE_MEM_LIMITS */
return TRUE;
-#endif /* DISABLE_MEM_LIMITS */
}
diff --git a/src/tracker-extract/tracker-main.c b/src/tracker-extract/tracker-main.c
index b01b1c9..3445e3b 100644
--- a/src/tracker-extract/tracker-main.c
+++ b/src/tracker-extract/tracker-main.c
@@ -197,8 +197,9 @@ signal_handler (int signo)
}
switch (signo) {
+ case SIGABRT:
case SIGALRM:
- exit (EXIT_FAILURE);
+ _exit (EXIT_FAILURE);
break;
case SIGTERM:
case SIGINT:
@@ -231,6 +232,7 @@ initialize_signal_handler (void)
sigaction (SIGINT, &act, NULL);
sigaction (SIGHUP, &act, NULL);
sigaction (SIGALRM, &act, NULL);
+ sigaction (SIGABRT, &act, NULL);
#endif /* G_OS_WIN32 */
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]