[sysprof/wip/chergert/mem-preload] memory: start on a memory profile
- From: Christian Hergert <chergert src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [sysprof/wip/chergert/mem-preload] memory: start on a memory profile
- Date: Sun, 2 Feb 2020 00:43:33 +0000 (UTC)
commit 2368a7b056a8100c285a25f3cf3c72ca41c4de95
Author: Christian Hergert <chergert redhat com>
Date: Fri Jan 31 15:26:40 2020 -0800
memory: start on a memory profile
The long term goal here is to get the stack from allocations as well as
get information about what is still on the heap. This can be useful when
tracking down fragmentation issues for long running modules or temporary
allocations which could be better served from secondary allocators.
src/libsysprof/meson.build | 22 +-
src/libsysprof/preload/meson.build | 1 +
src/libsysprof/preload/sysprof-memory-collector.c | 34 +++-
src/libsysprof/sysprof-memory-profile.c | 236 ++++++++++++++++++++++
src/libsysprof/sysprof-memory-profile.h | 43 ++++
src/libsysprof/sysprof.h | 1 +
src/tests/meson.build | 6 +
src/tests/show-page-usage.c | 196 ++++++++++++++++++
8 files changed, 523 insertions(+), 16 deletions(-)
---
diff --git a/src/libsysprof/meson.build b/src/libsysprof/meson.build
index bcb57db..459b093 100644
--- a/src/libsysprof/meson.build
+++ b/src/libsysprof/meson.build
@@ -16,6 +16,7 @@ libsysprof_public_sources = [
'sysprof-kernel-symbol.c',
'sysprof-kernel-symbol-resolver.c',
'sysprof-local-profiler.c',
+ 'sysprof-memory-profile.c',
'sysprof-netdev-source.c',
'sysprof-process-model.c',
'sysprof-process-model-item.c',
@@ -45,6 +46,7 @@ libsysprof_public_headers = [
'sysprof-kernel-symbol.h',
'sysprof-kernel-symbol-resolver.h',
'sysprof-local-profiler.h',
+ 'sysprof-memory-profile.h',
'sysprof-process-model.h',
'sysprof-process-model-item.h',
'sysprof-profile.h',
@@ -79,10 +81,22 @@ libsysprof_private_sources = [
libsysprof_public_sources += libsysprof_capture_sources
+librax = static_library('rax', ['rax.c'],
+ c_args: [ '-Wno-declaration-after-statement',
+ '-Wno-format-nonliteral',
+ '-Wno-shadow' ],
+)
+
+librax_dep = declare_dependency(
+ link_whole: librax,
+ include_directories: include_directories('.'),
+)
+
libsysprof_deps = [
gio_dep,
gio_unix_dep,
polkit_dep,
+ librax_dep,
]
if host_machine.system() == 'linux'
@@ -110,13 +124,6 @@ if host_machine.system() != 'darwin'
libsysprof_deps += [cxx.find_library('stdc++')]
endif
-librax = static_library('rax', ['rax.c'],
- c_args: [ '-Wno-declaration-after-statement',
- '-Wno-format-nonliteral',
- '-Wno-shadow' ],
- install: false
-)
-
libsysprof = shared_library(
'sysprof-@0@'.format(libsysprof_api_version),
libsysprof_public_sources + libsysprof_private_sources,
@@ -129,7 +136,6 @@ libsysprof = shared_library(
install: true,
install_dir: get_option('libdir'),
gnu_symbol_visibility: 'hidden',
- link_whole: [ librax ],
)
libsysprof_dep = declare_dependency(
diff --git a/src/libsysprof/preload/meson.build b/src/libsysprof/preload/meson.build
index 11d94fa..2d298ac 100644
--- a/src/libsysprof/preload/meson.build
+++ b/src/libsysprof/preload/meson.build
@@ -1,6 +1,7 @@
libsysprof_memory_preload_deps = [
libsysprof_capture_dep,
cc.find_library('dl', required: false),
+ dependency('libunwind-generic'),
]
libsysprof_memory_preload = shared_library('sysprof-memory-collector',
diff --git a/src/libsysprof/preload/sysprof-memory-collector.c
b/src/libsysprof/preload/sysprof-memory-collector.c
index d40714a..a0ae98f 100644
--- a/src/libsysprof/preload/sysprof-memory-collector.c
+++ b/src/libsysprof/preload/sysprof-memory-collector.c
@@ -7,6 +7,9 @@
#include <sys/types.h>
#include <sysprof-capture.h>
#include <unistd.h>
+#include <libunwind.h>
+
+#define CAPTURE_MAX_STACK_DEPTH 32
typedef void *(* RealMalloc) (size_t);
typedef void (* RealFree) (void *);
@@ -67,7 +70,7 @@ scratch_calloc (size_t nmemb,
*/
if (!hooked)
hook_memtable ();
-
+
size *= nmemb;
ret = &scratch.buf[scratch.off];
scratch.off += size;
@@ -140,16 +143,31 @@ static inline void
track_malloc (void *ptr,
size_t size)
{
+ SysprofCaptureAddress addrs[CAPTURE_MAX_STACK_DEPTH];
+ unw_context_t uc;
+ unw_cursor_t cursor;
+ unw_word_t ip;
+ guint n_addrs = 0;
+
if G_UNLIKELY (!writer)
return;
+ /* Get a stacktrace for the current allocation, but walk past our
+ * current function because we don't care about that stack frame.
+ */
+ unw_getcontext (&uc);
+ unw_init_local (&cursor, &uc);
+ if (unw_step (&cursor) > 0)
+ {
+ while (n_addrs < G_N_ELEMENTS (addrs) && unw_step (&cursor) > 0)
+ {
+ unw_get_reg (&cursor, UNW_REG_IP, &ip);
+ addrs[n_addrs++] = ip;
+ }
+ }
+
G_LOCK (writer);
- /* TODO: To make this really useful, we need to take a backtrace
- * of the current stack so that we can show the user allocations
- * within the application. However, for now we just want to get
- * the allocation information to draw fragmentation.
- */
sysprof_capture_writer_add_memory_alloc (writer,
SYSPROF_CAPTURE_CURRENT_TIME,
sched_getcpu (),
@@ -157,8 +175,8 @@ track_malloc (void *ptr,
gettid(),
GPOINTER_TO_SIZE (ptr),
size,
- NULL, /* TODO: Sample */
- 0);
+ addrs,
+ n_addrs);
G_UNLOCK (writer);
}
diff --git a/src/libsysprof/sysprof-memory-profile.c b/src/libsysprof/sysprof-memory-profile.c
new file mode 100644
index 0000000..5caf3df
--- /dev/null
+++ b/src/libsysprof/sysprof-memory-profile.c
@@ -0,0 +1,236 @@
+/* sysprof-memory-profile.c
+ *
+ * Copyright 2020 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#define G_LOG_DOMAIN "sysprof-memory-profile"
+
+#include "config.h"
+
+#include <sysprof-capture.h>
+
+#include "sysprof-memory-profile.h"
+
+#include "rax.h"
+
+struct _SysprofMemoryProfile
+{
+ GObject parent_instance;
+ SysprofCaptureReader *reader;
+ rax *rax;
+};
+
+typedef struct
+{
+ SysprofCaptureReader *reader;
+ rax *rax;
+} Generate;
+
+static void profile_iface_init (SysprofProfileInterface *iface);
+
+G_DEFINE_TYPE_WITH_CODE (SysprofMemoryProfile, sysprof_memory_profile, G_TYPE_OBJECT,
+ G_IMPLEMENT_INTERFACE (SYSPROF_TYPE_PROFILE, profile_iface_init))
+
+static void
+generate_free (Generate *g)
+{
+ g_clear_pointer (&g->reader, sysprof_capture_reader_unref);
+ g_clear_pointer (&g->rax, raxFree);
+ g_slice_free (Generate, g);
+}
+
+static void
+sysprof_memory_profile_finalize (GObject *object)
+{
+ SysprofMemoryProfile *self = (SysprofMemoryProfile *)object;
+
+ g_clear_pointer (&self->reader, sysprof_capture_reader_unref);
+ g_clear_pointer (&self->rax, raxFree);
+
+ G_OBJECT_CLASS (sysprof_memory_profile_parent_class)->finalize (object);
+}
+
+static void
+sysprof_memory_profile_class_init (SysprofMemoryProfileClass *klass)
+{
+ GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+ object_class->finalize = sysprof_memory_profile_finalize;
+}
+
+static void
+sysprof_memory_profile_init (SysprofMemoryProfile *self)
+{
+}
+
+SysprofProfile *
+sysprof_memory_profile_new (void)
+{
+ return g_object_new (SYSPROF_TYPE_MEMORY_PROFILE, NULL);
+}
+
+static void
+sysprof_memory_profile_set_reader (SysprofProfile *profile,
+ SysprofCaptureReader *reader)
+{
+ SysprofMemoryProfile *self = (SysprofMemoryProfile *)profile;
+
+ g_assert (SYSPROF_IS_MEMORY_PROFILE (self));
+ g_assert (reader != NULL);
+
+ if (reader != self->reader)
+ {
+ g_clear_pointer (&self->reader, sysprof_capture_reader_unref);
+ self->reader = sysprof_capture_reader_ref (reader);
+ }
+}
+
+static SysprofCaptureCursor *
+create_cursor (SysprofCaptureReader *reader)
+{
+ static SysprofCaptureFrameType types[] = {
+ SYSPROF_CAPTURE_FRAME_MEMORY_ALLOC,
+ SYSPROF_CAPTURE_FRAME_MEMORY_FREE,
+ };
+ SysprofCaptureCursor *cursor;
+ SysprofCaptureCondition *cond;
+
+ cond = sysprof_capture_condition_new_where_type_in (G_N_ELEMENTS (types), types);
+ cursor = sysprof_capture_cursor_new (reader);
+ sysprof_capture_cursor_add_condition (cursor, cond);
+
+ return cursor;
+}
+
+static gboolean
+cursor_foreach_cb (const SysprofCaptureFrame *frame,
+ gpointer user_data)
+{
+ Generate *g = user_data;
+
+ g_assert (frame != NULL);
+ g_assert (frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_ALLOC ||
+ frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_FREE);
+
+ if (frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_ALLOC)
+ {
+ const SysprofCaptureMemoryAlloc *ev = (const SysprofCaptureMemoryAlloc *)frame;
+
+ raxInsert (g->rax,
+ (guint8 *)&ev->alloc_addr,
+ sizeof ev->alloc_addr,
+ (gpointer)ev->alloc_size,
+ NULL);
+ }
+ else if (frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_FREE)
+ {
+ const SysprofCaptureMemoryFree *ev = (const SysprofCaptureMemoryFree *)frame;
+
+ raxRemove (g->rax,
+ (guint8 *)&ev->alloc_addr,
+ sizeof ev->alloc_addr,
+ NULL);
+ }
+
+ return TRUE;
+}
+
+static void
+sysprof_memory_profile_generate_worker (GTask *task,
+ gpointer source_object,
+ gpointer task_data,
+ GCancellable *cancellable)
+{
+ SysprofCaptureCursor *cursor;
+ Generate *g = task_data;
+
+ g_assert (G_IS_TASK (task));
+ g_assert (g != NULL);
+ g_assert (g->reader != NULL);
+ g_assert (!cancellable || G_IS_CANCELLABLE (cancellable));
+
+ cursor = create_cursor (g->reader);
+ sysprof_capture_cursor_foreach (cursor, cursor_foreach_cb, g);
+
+ g_task_return_boolean (task, TRUE);
+}
+
+static void
+sysprof_memory_profile_generate (SysprofProfile *profile,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
+{
+ SysprofMemoryProfile *self = (SysprofMemoryProfile *)profile;
+ g_autoptr(GTask) task = NULL;
+ Generate *g;
+
+ g_assert (SYSPROF_IS_MEMORY_PROFILE (self));
+ g_assert (!cancellable || G_IS_CANCELLABLE (cancellable));
+ g_assert (self->rax == NULL);
+
+ task = g_task_new (self, cancellable, callback, user_data);
+ g_task_set_source_tag (task, sysprof_memory_profile_generate);
+
+ if (self->reader == NULL)
+ {
+ g_task_return_new_error (task,
+ G_IO_ERROR,
+ G_IO_ERROR_NOT_INITIALIZED,
+ "No capture reader has been set");
+ return;
+ }
+
+ g = g_slice_new0 (Generate);
+ g->reader = sysprof_capture_reader_copy (self->reader);
+ g->rax = raxNew ();
+
+ g_task_set_task_data (task, g, (GDestroyNotify) generate_free);
+ g_task_run_in_thread (task, sysprof_memory_profile_generate_worker);
+}
+
+static gboolean
+sysprof_memory_profile_generate_finish (SysprofProfile *profile,
+ GAsyncResult *result,
+ GError **error)
+{
+ SysprofMemoryProfile *self = (SysprofMemoryProfile *)profile;
+ Generate *g;
+
+ g_assert (SYSPROF_IS_MEMORY_PROFILE (self));
+ g_assert (G_IS_TASK (result));
+
+ if ((g = g_task_get_task_data (G_TASK (result))))
+ self->rax = g_steal_pointer (&g->rax);
+
+ return g_task_propagate_boolean (G_TASK (result), error);
+}
+
+static void
+profile_iface_init (SysprofProfileInterface *iface)
+{
+ iface->set_reader = sysprof_memory_profile_set_reader;
+ iface->generate = sysprof_memory_profile_generate;
+ iface->generate_finish = sysprof_memory_profile_generate_finish;
+}
+
+gpointer
+sysprof_memory_profile_get_native (SysprofMemoryProfile *self)
+{
+ return self->rax;
+}
diff --git a/src/libsysprof/sysprof-memory-profile.h b/src/libsysprof/sysprof-memory-profile.h
new file mode 100644
index 0000000..0f9334f
--- /dev/null
+++ b/src/libsysprof/sysprof-memory-profile.h
@@ -0,0 +1,43 @@
+/* sysprof-memory-profile.h
+ *
+ * Copyright 2020 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#pragma once
+
+#if !defined (SYSPROF_INSIDE) && !defined (SYSPROF_COMPILATION)
+# error "Only <sysprof.h> can be included directly."
+#endif
+
+#include "sysprof-version-macros.h"
+
+#include "sysprof-profile.h"
+
+G_BEGIN_DECLS
+
+#define SYSPROF_TYPE_MEMORY_PROFILE (sysprof_memory_profile_get_type())
+
+SYSPROF_AVAILABLE_IN_ALL
+G_DECLARE_FINAL_TYPE (SysprofMemoryProfile, sysprof_memory_profile, SYSPROF, MEMORY_PROFILE, GObject)
+
+SYSPROF_AVAILABLE_IN_3_36
+SysprofProfile *sysprof_memory_profile_new (void);
+SYSPROF_AVAILABLE_IN_3_36
+gpointer sysprof_memory_profile_get_native (SysprofMemoryProfile *self);
+
+G_END_DECLS
diff --git a/src/libsysprof/sysprof.h b/src/libsysprof/sysprof.h
index da4b524..0485ec0 100644
--- a/src/libsysprof/sysprof.h
+++ b/src/libsysprof/sysprof.h
@@ -36,6 +36,7 @@ G_BEGIN_DECLS
# include "sysprof-kernel-symbol-resolver.h"
# include "sysprof-kernel-symbol.h"
# include "sysprof-local-profiler.h"
+# include "sysprof-memory-profile.h"
# include "sysprof-netdev-source.h"
# include "sysprof-process-model-item.h"
# include "sysprof-process-model.h"
diff --git a/src/tests/meson.build b/src/tests/meson.build
index c506d19..d063e34 100644
--- a/src/tests/meson.build
+++ b/src/tests/meson.build
@@ -64,6 +64,12 @@ test_resolvers = executable('test-resolvers',
dependencies: test_deps,
)
+show_page_usage = executable('show-page-usage',
+ [ 'show-page-usage.c' ],
+ c_args: test_cflags,
+ dependencies: test_deps + [ librax_dep,
+ dependency('cairo') ],
+)
if get_option('enable_gtk')
diff --git a/src/tests/show-page-usage.c b/src/tests/show-page-usage.c
new file mode 100644
index 0000000..2b4d4b7
--- /dev/null
+++ b/src/tests/show-page-usage.c
@@ -0,0 +1,196 @@
+/* show-page-usage.c
+ *
+ * Copyright 2020 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#include "config.h"
+
+#include <cairo.h>
+#include <stddef.h>
+#include <rax.h>
+#include <sysprof.h>
+
+static GMainLoop *main_loop;
+
+static gint
+u64_compare (gconstpointer a,
+ gconstpointer b)
+{
+ const guint64 *aptr = a;
+ const guint64 *bptr = b;
+
+ if (*aptr < *bptr)
+ return -1;
+ else if (*aptr > *bptr)
+ return 1;
+ else
+ return 0;
+}
+
+static void
+generate_cb (GObject *object,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ SysprofProfile *profile = (SysprofProfile *)object;
+ g_autoptr(GError) error = NULL;
+ GHashTable *seen;
+ GHashTableIter iter;
+ cairo_t *cr;
+ cairo_surface_t *surface;
+ GArray *ar;
+ raxIterator it;
+ rax *r;
+ gpointer k,v;
+
+ g_assert (SYSPROF_IS_MEMORY_PROFILE (profile));
+ g_assert (G_IS_ASYNC_RESULT (result));
+
+ if (!sysprof_profile_generate_finish (profile, result, &error))
+ {
+ g_printerr ("%s\n", error->message);
+ exit (EXIT_FAILURE);
+ }
+
+ r = sysprof_memory_profile_get_native (SYSPROF_MEMORY_PROFILE (profile));
+ seen = g_hash_table_new (NULL, NULL);
+
+ raxStart (&it, r);
+ raxSeek (&it, "^", NULL, 0);
+ while (raxNext (&it))
+ {
+ guint64 page;
+ guint64 addr;
+
+ memcpy (&addr, it.key, sizeof addr);
+ page = addr / 4096;
+
+ if (g_hash_table_contains (seen, GSIZE_TO_POINTER (page)))
+ continue;
+
+ g_hash_table_insert (seen, GSIZE_TO_POINTER (page), NULL);
+ }
+ raxStop (&it);
+
+ ar = g_array_sized_new (FALSE, FALSE, sizeof (guint64), g_hash_table_size (seen));
+
+ g_hash_table_iter_init (&iter, seen);
+ while (g_hash_table_iter_next (&iter, &k, &v))
+ {
+ guint64 key = GPOINTER_TO_SIZE (k);
+
+ g_array_append_val (ar, key);
+ }
+
+ g_array_sort (ar, u64_compare);
+
+ for (guint i = 0; i < ar->len; i++)
+ {
+ guint64 key = g_array_index (ar, guint64, i);
+
+ g_hash_table_insert (seen, GSIZE_TO_POINTER (key), GSIZE_TO_POINTER (i));
+ }
+
+ g_printerr ("We have %u pages to graph\n", ar->len);
+
+ surface = cairo_image_surface_create (CAIRO_FORMAT_RGB24, ar->len, (4096/16));
+ cr = cairo_create (surface);
+
+ cairo_set_line_width (cr, 1.0);
+ cairo_set_antialias (cr, CAIRO_ANTIALIAS_NONE);
+
+ cairo_set_source_rgb (cr, 1, 1, 1);
+ cairo_rectangle (cr, 0, 0, ar->len, (4096/16));
+ cairo_fill (cr);
+
+ cairo_set_source_rgb (cr, 0, 0, 0);
+
+ cairo_scale (cr, 1.0, 1.0/16.0);
+ cairo_translate (cr, .5, .5);
+
+ raxStart (&it, r);
+ raxSeek (&it, "^", NULL, 0);
+ while (raxNext (&it))
+ {
+ guint64 page;
+ guint64 addr;
+ guint64 size;
+ guint x;
+ guint y;
+
+ memcpy (&addr, it.key, sizeof addr);
+ page = addr / 4096;
+ size = GPOINTER_TO_SIZE (it.data);
+
+ x = GPOINTER_TO_UINT (g_hash_table_lookup (seen, GSIZE_TO_POINTER (page)));
+ y = addr % 4096;
+
+ /* TODO: Need size */
+
+ cairo_move_to (cr, x, y);
+ cairo_line_to (cr, x, y+size);
+ }
+ raxStop (&it);
+
+ cairo_stroke (cr);
+
+ cairo_surface_write_to_png (surface, "memory.png");
+
+ cairo_destroy (cr);
+ cairo_surface_destroy (surface);
+
+ g_array_unref (ar);
+ g_hash_table_unref (seen);
+
+ g_main_loop_quit (main_loop);
+}
+
+gint
+main (gint argc,
+ gchar *argv[])
+{
+ SysprofCaptureReader *reader;
+ const gchar *filename = argv[1];
+ g_autoptr(SysprofProfile) memprof = NULL;
+ g_autoptr(GError) error = NULL;
+
+ if (argc < 2)
+ {
+ g_printerr ("usage: %s FILENAME\n", argv[0]);
+ return EXIT_FAILURE;
+ }
+
+ main_loop = g_main_loop_new (NULL, FALSE);
+
+ if (!(reader = sysprof_capture_reader_new (filename, &error)))
+ {
+ g_printerr ("%s\n", error->message);
+ return EXIT_FAILURE;
+ }
+
+ memprof = sysprof_memory_profile_new ();
+ sysprof_profile_set_reader (memprof, reader);
+ sysprof_profile_generate (memprof, NULL, generate_cb, NULL);
+
+ g_main_loop_run (main_loop);
+ g_main_loop_unref (main_loop);
+
+ sysprof_capture_reader_unref (reader);
+
+ return EXIT_SUCCESS;
+}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
