GNOME.org
 

[sysprof/wip/chergert/mem-preload: 13/43] memory: start on a memory profile

commit 24c5d95babedaeaf78c1eed7f0bcedbd0f7e15be
Author: Christian Hergert <chergert redhat com>
Date:   Fri Jan 31 15:26:40 2020 -0800

    memory: start on a memory profile
    
    The long term goal here is to get the stack from allocations as well as
    get information about what is still on the heap. This can be useful when
    tracking down fragmentation issues for long running modules or temporary
    allocations which could be better served from secondary allocators.

 src/libsysprof/meson.build                        |  22 +-
 src/libsysprof/preload/meson.build                |   1 +
 src/libsysprof/preload/sysprof-memory-collector.c |  34 +++-
 src/libsysprof/sysprof-memory-profile.c           | 236 ++++++++++++++++++++++
 src/libsysprof/sysprof-memory-profile.h           |  43 ++++
 src/libsysprof/sysprof.h                          |   1 +
 src/tests/meson.build                             |   6 +
 src/tests/show-page-usage.c                       | 196 ++++++++++++++++++
 8 files changed, 523 insertions(+), 16 deletions(-)
---
diff --git a/src/libsysprof/meson.build b/src/libsysprof/meson.build
index bcb57db..459b093 100644
--- a/src/libsysprof/meson.build
+++ b/src/libsysprof/meson.build
@@ -16,6 +16,7 @@ libsysprof_public_sources = [
   'sysprof-kernel-symbol.c',
   'sysprof-kernel-symbol-resolver.c',
   'sysprof-local-profiler.c',
+  'sysprof-memory-profile.c',
   'sysprof-netdev-source.c',
   'sysprof-process-model.c',
   'sysprof-process-model-item.c',
@@ -45,6 +46,7 @@ libsysprof_public_headers = [
   'sysprof-kernel-symbol.h',
   'sysprof-kernel-symbol-resolver.h',
   'sysprof-local-profiler.h',
+  'sysprof-memory-profile.h',
   'sysprof-process-model.h',
   'sysprof-process-model-item.h',
   'sysprof-profile.h',
@@ -79,10 +81,22 @@ libsysprof_private_sources = [
 
 libsysprof_public_sources += libsysprof_capture_sources
 
+librax = static_library('rax', ['rax.c'],
+               c_args: [ '-Wno-declaration-after-statement',
+                         '-Wno-format-nonliteral',
+                         '-Wno-shadow' ],
+)
+
+librax_dep = declare_dependency(
+           link_whole: librax,
+  include_directories: include_directories('.'),
+)
+
 libsysprof_deps = [
   gio_dep,
   gio_unix_dep,
   polkit_dep,
+  librax_dep,
 ]
 
 if host_machine.system() == 'linux'
@@ -110,13 +124,6 @@ if host_machine.system() != 'darwin'
   libsysprof_deps += [cxx.find_library('stdc++')]
 endif
 
-librax = static_library('rax', ['rax.c'],
-   c_args: [ '-Wno-declaration-after-statement',
-             '-Wno-format-nonliteral',
-             '-Wno-shadow' ],
-  install: false
-)
-
 libsysprof = shared_library(
   'sysprof-@0@'.format(libsysprof_api_version),
   libsysprof_public_sources + libsysprof_private_sources,
@@ -129,7 +136,6 @@ libsysprof = shared_library(
                 install: true,
             install_dir: get_option('libdir'),
   gnu_symbol_visibility: 'hidden',
-             link_whole: [ librax ],
 )
 
 libsysprof_dep = declare_dependency(
diff --git a/src/libsysprof/preload/meson.build b/src/libsysprof/preload/meson.build
index 11d94fa..2d298ac 100644
--- a/src/libsysprof/preload/meson.build
+++ b/src/libsysprof/preload/meson.build
@@ -1,6 +1,7 @@
 libsysprof_memory_preload_deps = [
   libsysprof_capture_dep,
   cc.find_library('dl', required: false),
+  dependency('libunwind-generic'),
 ]
 
 libsysprof_memory_preload = shared_library('sysprof-memory-collector',
diff --git a/src/libsysprof/preload/sysprof-memory-collector.c 
b/src/libsysprof/preload/sysprof-memory-collector.c
index d40714a..a0ae98f 100644
--- a/src/libsysprof/preload/sysprof-memory-collector.c
+++ b/src/libsysprof/preload/sysprof-memory-collector.c
@@ -7,6 +7,9 @@
 #include <sys/types.h>
 #include <sysprof-capture.h>
 #include <unistd.h>
+#include <libunwind.h>
+
+#define CAPTURE_MAX_STACK_DEPTH 32
 
 typedef void *(* RealMalloc)        (size_t);
 typedef void  (* RealFree)          (void *);
@@ -67,7 +70,7 @@ scratch_calloc (size_t nmemb,
    */
   if (!hooked)
     hook_memtable ();
- 
+
   size *= nmemb;
   ret = &scratch.buf[scratch.off];
   scratch.off += size;
@@ -140,16 +143,31 @@ static inline void
 track_malloc (void   *ptr,
               size_t  size)
 {
+  SysprofCaptureAddress addrs[CAPTURE_MAX_STACK_DEPTH];
+  unw_context_t uc;
+  unw_cursor_t cursor;
+  unw_word_t ip;
+  guint n_addrs = 0;
+
   if G_UNLIKELY (!writer)
     return;
 
+  /* Get a stacktrace for the current allocation, but walk past our
+   * current function because we don't care about that stack frame.
+   */
+  unw_getcontext (&uc);
+  unw_init_local (&cursor, &uc);
+  if (unw_step (&cursor) > 0)
+    {
+      while (n_addrs < G_N_ELEMENTS (addrs) && unw_step (&cursor) > 0)
+        {
+          unw_get_reg (&cursor, UNW_REG_IP, &ip);
+          addrs[n_addrs++] = ip;
+        }
+    }
+
   G_LOCK (writer);
 
-  /* TODO: To make this really useful, we need to take a backtrace
-   * of the current stack so that we can show the user allocations
-   * within the application. However, for now we just want to get
-   * the allocation information to draw fragmentation.
-   */
   sysprof_capture_writer_add_memory_alloc (writer,
                                            SYSPROF_CAPTURE_CURRENT_TIME,
                                            sched_getcpu (),
@@ -157,8 +175,8 @@ track_malloc (void   *ptr,
                                            gettid(),
                                            GPOINTER_TO_SIZE (ptr),
                                            size,
-                                           NULL, /* TODO: Sample */
-                                           0);
+                                           addrs,
+                                           n_addrs);
   G_UNLOCK (writer);
 }
 
diff --git a/src/libsysprof/sysprof-memory-profile.c b/src/libsysprof/sysprof-memory-profile.c
new file mode 100644
index 0000000..5caf3df
--- /dev/null
+++ b/src/libsysprof/sysprof-memory-profile.c
@@ -0,0 +1,236 @@
+/* sysprof-memory-profile.c
+ *
+ * Copyright 2020 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#define G_LOG_DOMAIN "sysprof-memory-profile"
+
+#include "config.h"
+
+#include <sysprof-capture.h>
+
+#include "sysprof-memory-profile.h"
+
+#include "rax.h"
+
+struct _SysprofMemoryProfile
+{
+  GObject               parent_instance;
+  SysprofCaptureReader *reader;
+  rax *rax;
+};
+
+typedef struct
+{
+  SysprofCaptureReader *reader;
+  rax *rax;
+} Generate;
+
+static void profile_iface_init (SysprofProfileInterface *iface);
+
+G_DEFINE_TYPE_WITH_CODE (SysprofMemoryProfile, sysprof_memory_profile, G_TYPE_OBJECT,
+                         G_IMPLEMENT_INTERFACE (SYSPROF_TYPE_PROFILE, profile_iface_init))
+
+static void
+generate_free (Generate *g)
+{
+  g_clear_pointer (&g->reader, sysprof_capture_reader_unref);
+  g_clear_pointer (&g->rax, raxFree);
+  g_slice_free (Generate, g);
+}
+
+static void
+sysprof_memory_profile_finalize (GObject *object)
+{
+  SysprofMemoryProfile *self = (SysprofMemoryProfile *)object;
+
+  g_clear_pointer (&self->reader, sysprof_capture_reader_unref);
+  g_clear_pointer (&self->rax, raxFree);
+
+  G_OBJECT_CLASS (sysprof_memory_profile_parent_class)->finalize (object);
+}
+
+static void
+sysprof_memory_profile_class_init (SysprofMemoryProfileClass *klass)
+{
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+  object_class->finalize = sysprof_memory_profile_finalize;
+}
+
+static void
+sysprof_memory_profile_init (SysprofMemoryProfile *self)
+{
+}
+
+SysprofProfile *
+sysprof_memory_profile_new (void)
+{
+  return g_object_new (SYSPROF_TYPE_MEMORY_PROFILE, NULL);
+}
+
+static void
+sysprof_memory_profile_set_reader (SysprofProfile       *profile,
+                                   SysprofCaptureReader *reader)
+{
+  SysprofMemoryProfile *self = (SysprofMemoryProfile *)profile;
+
+  g_assert (SYSPROF_IS_MEMORY_PROFILE (self));
+  g_assert (reader != NULL);
+
+  if (reader != self->reader)
+    {
+      g_clear_pointer (&self->reader, sysprof_capture_reader_unref);
+      self->reader = sysprof_capture_reader_ref (reader);
+    }
+}
+
+static SysprofCaptureCursor *
+create_cursor (SysprofCaptureReader *reader)
+{
+  static SysprofCaptureFrameType types[] = {
+    SYSPROF_CAPTURE_FRAME_MEMORY_ALLOC,
+    SYSPROF_CAPTURE_FRAME_MEMORY_FREE,
+  };
+  SysprofCaptureCursor *cursor;
+  SysprofCaptureCondition *cond;
+
+  cond = sysprof_capture_condition_new_where_type_in (G_N_ELEMENTS (types), types);
+  cursor = sysprof_capture_cursor_new (reader);
+  sysprof_capture_cursor_add_condition (cursor, cond);
+
+  return cursor;
+}
+
+static gboolean
+cursor_foreach_cb (const SysprofCaptureFrame *frame,
+                   gpointer                   user_data)
+{
+  Generate *g = user_data;
+
+  g_assert (frame != NULL);
+  g_assert (frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_ALLOC ||
+            frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_FREE);
+
+  if (frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_ALLOC)
+    {
+      const SysprofCaptureMemoryAlloc *ev = (const SysprofCaptureMemoryAlloc *)frame;
+
+      raxInsert (g->rax,
+                 (guint8 *)&ev->alloc_addr,
+                 sizeof ev->alloc_addr,
+                 (gpointer)ev->alloc_size,
+                 NULL);
+    }
+  else if (frame->type == SYSPROF_CAPTURE_FRAME_MEMORY_FREE)
+    {
+      const SysprofCaptureMemoryFree *ev = (const SysprofCaptureMemoryFree *)frame;
+
+      raxRemove (g->rax,
+                 (guint8 *)&ev->alloc_addr,
+                 sizeof ev->alloc_addr,
+                 NULL);
+    }
+
+  return TRUE;
+}
+
+static void
+sysprof_memory_profile_generate_worker (GTask        *task,
+                                        gpointer      source_object,
+                                        gpointer      task_data,
+                                        GCancellable *cancellable)
+{
+  SysprofCaptureCursor *cursor;
+  Generate *g = task_data;
+
+  g_assert (G_IS_TASK (task));
+  g_assert (g != NULL);
+  g_assert (g->reader != NULL);
+  g_assert (!cancellable || G_IS_CANCELLABLE (cancellable));
+
+  cursor = create_cursor (g->reader);
+  sysprof_capture_cursor_foreach (cursor, cursor_foreach_cb, g);
+
+  g_task_return_boolean (task, TRUE);
+}
+
+static void
+sysprof_memory_profile_generate (SysprofProfile      *profile,
+                                 GCancellable        *cancellable,
+                                 GAsyncReadyCallback  callback,
+                                 gpointer             user_data)
+{
+  SysprofMemoryProfile *self = (SysprofMemoryProfile *)profile;
+  g_autoptr(GTask) task = NULL;
+  Generate *g;
+
+  g_assert (SYSPROF_IS_MEMORY_PROFILE (self));
+  g_assert (!cancellable || G_IS_CANCELLABLE (cancellable));
+  g_assert (self->rax == NULL);
+
+  task = g_task_new (self, cancellable, callback, user_data);
+  g_task_set_source_tag (task, sysprof_memory_profile_generate);
+
+  if (self->reader == NULL)
+    {
+      g_task_return_new_error (task,
+                               G_IO_ERROR,
+                               G_IO_ERROR_NOT_INITIALIZED,
+                               "No capture reader has been set");
+      return;
+    }
+
+  g = g_slice_new0 (Generate);
+  g->reader = sysprof_capture_reader_copy (self->reader);
+  g->rax = raxNew ();
+
+  g_task_set_task_data (task, g, (GDestroyNotify) generate_free);
+  g_task_run_in_thread (task, sysprof_memory_profile_generate_worker);
+}
+
+static gboolean
+sysprof_memory_profile_generate_finish (SysprofProfile  *profile,
+                                        GAsyncResult    *result,
+                                        GError         **error)
+{
+  SysprofMemoryProfile *self = (SysprofMemoryProfile *)profile;
+  Generate *g;
+
+  g_assert (SYSPROF_IS_MEMORY_PROFILE (self));
+  g_assert (G_IS_TASK (result));
+
+  if ((g = g_task_get_task_data (G_TASK (result))))
+    self->rax = g_steal_pointer (&g->rax);
+
+  return g_task_propagate_boolean (G_TASK (result), error);
+}
+
+static void
+profile_iface_init (SysprofProfileInterface *iface)
+{
+  iface->set_reader = sysprof_memory_profile_set_reader;
+  iface->generate = sysprof_memory_profile_generate;
+  iface->generate_finish = sysprof_memory_profile_generate_finish;
+}
+
+gpointer
+sysprof_memory_profile_get_native (SysprofMemoryProfile *self)
+{
+  return self->rax;
+}
diff --git a/src/libsysprof/sysprof-memory-profile.h b/src/libsysprof/sysprof-memory-profile.h
new file mode 100644
index 0000000..0f9334f
--- /dev/null
+++ b/src/libsysprof/sysprof-memory-profile.h
@@ -0,0 +1,43 @@
+/* sysprof-memory-profile.h
+ *
+ * Copyright 2020 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#pragma once
+
+#if !defined (SYSPROF_INSIDE) && !defined (SYSPROF_COMPILATION)
+# error "Only <sysprof.h> can be included directly."
+#endif
+
+#include "sysprof-version-macros.h"
+
+#include "sysprof-profile.h"
+
+G_BEGIN_DECLS
+
+#define SYSPROF_TYPE_MEMORY_PROFILE (sysprof_memory_profile_get_type())
+
+SYSPROF_AVAILABLE_IN_ALL
+G_DECLARE_FINAL_TYPE (SysprofMemoryProfile, sysprof_memory_profile, SYSPROF, MEMORY_PROFILE, GObject)
+
+SYSPROF_AVAILABLE_IN_3_36
+SysprofProfile *sysprof_memory_profile_new        (void);
+SYSPROF_AVAILABLE_IN_3_36
+gpointer        sysprof_memory_profile_get_native (SysprofMemoryProfile *self);
+
+G_END_DECLS
diff --git a/src/libsysprof/sysprof.h b/src/libsysprof/sysprof.h
index da4b524..0485ec0 100644
--- a/src/libsysprof/sysprof.h
+++ b/src/libsysprof/sysprof.h
@@ -36,6 +36,7 @@ G_BEGIN_DECLS
 # include "sysprof-kernel-symbol-resolver.h"
 # include "sysprof-kernel-symbol.h"
 # include "sysprof-local-profiler.h"
+# include "sysprof-memory-profile.h"
 # include "sysprof-netdev-source.h"
 # include "sysprof-process-model-item.h"
 # include "sysprof-process-model.h"
diff --git a/src/tests/meson.build b/src/tests/meson.build
index c506d19..d063e34 100644
--- a/src/tests/meson.build
+++ b/src/tests/meson.build
@@ -64,6 +64,12 @@ test_resolvers = executable('test-resolvers',
   dependencies: test_deps,
 )
 
+show_page_usage = executable('show-page-usage',
+  [ 'show-page-usage.c' ],
+        c_args: test_cflags,
+  dependencies: test_deps + [ librax_dep,
+                              dependency('cairo') ],
+)
 
 if get_option('enable_gtk')
 
diff --git a/src/tests/show-page-usage.c b/src/tests/show-page-usage.c
new file mode 100644
index 0000000..2b4d4b7
--- /dev/null
+++ b/src/tests/show-page-usage.c
@@ -0,0 +1,196 @@
+/* show-page-usage.c
+ *
+ * Copyright 2020 Christian Hergert <chergert redhat com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+#include "config.h"
+
+#include <cairo.h>
+#include <stddef.h>
+#include <rax.h>
+#include <sysprof.h>
+
+static GMainLoop *main_loop;
+
+static gint
+u64_compare (gconstpointer a,
+             gconstpointer b)
+{
+  const guint64 *aptr = a;
+  const guint64 *bptr = b;
+
+  if (*aptr < *bptr)
+    return -1;
+  else if (*aptr > *bptr)
+    return 1;
+  else
+    return 0;
+}
+
+static void
+generate_cb (GObject      *object,
+             GAsyncResult *result,
+             gpointer      user_data)
+{
+  SysprofProfile *profile = (SysprofProfile *)object;
+  g_autoptr(GError) error = NULL;
+  GHashTable *seen;
+  GHashTableIter iter;
+  cairo_t *cr;
+  cairo_surface_t *surface;
+  GArray *ar;
+  raxIterator it;
+  rax *r;
+  gpointer k,v;
+
+  g_assert (SYSPROF_IS_MEMORY_PROFILE (profile));
+  g_assert (G_IS_ASYNC_RESULT (result));
+
+  if (!sysprof_profile_generate_finish (profile, result, &error))
+    {
+      g_printerr ("%s\n", error->message);
+      exit (EXIT_FAILURE);
+    }
+
+  r = sysprof_memory_profile_get_native (SYSPROF_MEMORY_PROFILE (profile));
+  seen = g_hash_table_new (NULL, NULL);
+
+  raxStart (&it, r);
+  raxSeek (&it, "^", NULL, 0);
+  while (raxNext (&it))
+    {
+      guint64 page;
+      guint64 addr;
+
+      memcpy (&addr, it.key, sizeof addr);
+      page = addr / 4096;
+
+      if (g_hash_table_contains (seen, GSIZE_TO_POINTER (page)))
+        continue;
+
+      g_hash_table_insert (seen, GSIZE_TO_POINTER (page), NULL);
+    }
+  raxStop (&it);
+
+  ar = g_array_sized_new (FALSE, FALSE, sizeof (guint64), g_hash_table_size (seen));
+
+  g_hash_table_iter_init (&iter, seen);
+  while (g_hash_table_iter_next (&iter, &k, &v))
+    {
+      guint64 key = GPOINTER_TO_SIZE (k);
+
+      g_array_append_val (ar, key);
+    }
+
+  g_array_sort (ar, u64_compare);
+
+  for (guint i = 0; i < ar->len; i++)
+    {
+      guint64 key = g_array_index (ar, guint64, i);
+
+      g_hash_table_insert (seen, GSIZE_TO_POINTER (key), GSIZE_TO_POINTER (i));
+    }
+
+  g_printerr ("We have %u pages to graph\n", ar->len);
+
+  surface = cairo_image_surface_create (CAIRO_FORMAT_RGB24, ar->len, (4096/16));
+  cr = cairo_create (surface);
+
+  cairo_set_line_width (cr, 1.0);
+  cairo_set_antialias (cr, CAIRO_ANTIALIAS_NONE);
+
+  cairo_set_source_rgb (cr, 1, 1, 1);
+  cairo_rectangle (cr, 0, 0, ar->len, (4096/16));
+  cairo_fill (cr);
+
+  cairo_set_source_rgb (cr, 0, 0, 0);
+
+  cairo_scale (cr, 1.0, 1.0/16.0);
+  cairo_translate (cr, .5, .5);
+
+  raxStart (&it, r);
+  raxSeek (&it, "^", NULL, 0);
+  while (raxNext (&it))
+    {
+      guint64 page;
+      guint64 addr;
+      guint64 size;
+      guint x;
+      guint y;
+
+      memcpy (&addr, it.key, sizeof addr);
+      page = addr / 4096;
+      size = GPOINTER_TO_SIZE (it.data);
+
+      x = GPOINTER_TO_UINT (g_hash_table_lookup (seen, GSIZE_TO_POINTER (page)));
+      y = addr % 4096;
+
+      /* TODO: Need size */
+
+      cairo_move_to (cr, x, y);
+      cairo_line_to (cr, x, y+size);
+    }
+  raxStop (&it);
+
+  cairo_stroke (cr);
+
+  cairo_surface_write_to_png (surface, "memory.png");
+
+  cairo_destroy (cr);
+  cairo_surface_destroy (surface);
+
+  g_array_unref (ar);
+  g_hash_table_unref (seen);
+
+  g_main_loop_quit (main_loop);
+}
+
+gint
+main (gint   argc,
+      gchar *argv[])
+{
+  SysprofCaptureReader *reader;
+  const gchar *filename = argv[1];
+  g_autoptr(SysprofProfile) memprof = NULL;
+  g_autoptr(GError) error = NULL;
+
+  if (argc < 2)
+    {
+      g_printerr ("usage: %s FILENAME\n", argv[0]);
+      return EXIT_FAILURE;
+    }
+
+  main_loop = g_main_loop_new (NULL, FALSE);
+
+  if (!(reader = sysprof_capture_reader_new (filename, &error)))
+    {
+      g_printerr ("%s\n", error->message);
+      return EXIT_FAILURE;
+    }
+
+  memprof = sysprof_memory_profile_new ();
+  sysprof_profile_set_reader (memprof, reader);
+  sysprof_profile_generate (memprof, NULL, generate_cb, NULL);
+
+  g_main_loop_run (main_loop);
+  g_main_loop_unref (main_loop);
+
+  sysprof_capture_reader_unref (reader);
+
+  return EXIT_SUCCESS;
+}
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]