[sysprof] libsysprof-capture: use signed int for backtrace return

[Christian Hergert <chergert src gnome org>]

commit 7490a774aba6d3ea28514e113abefa71fff155be
Author: Christian Hergert <chergert redhat com>
Date:   Tue Feb 18 14:03:19 2020 -0800

    libsysprof-capture: use signed int for backtrace return
    
    This allows us to more safely subtract 1 from the unw_backtrace() to get
    the proper number of frames (and detect it in the collector).

 src/libsysprof-capture/sysprof-capture-writer.h   |  6 +++---
 src/libsysprof-capture/sysprof-collector.c        |  7 +++++--
 src/libsysprof/preload/sysprof-memory-collector.c | 15 ++++++++-------
 3 files changed, 16 insertions(+), 12 deletions(-)
---
diff --git a/src/libsysprof-capture/sysprof-capture-writer.h b/src/libsysprof-capture/sysprof-capture-writer.h
index 07120ba..6e49025 100644
--- a/src/libsysprof-capture/sysprof-capture-writer.h
+++ b/src/libsysprof-capture/sysprof-capture-writer.h
@@ -72,9 +72,9 @@ typedef struct _SysprofCaptureWriter SysprofCaptureWriter;
  *
  * Returns: the number of addresses filled in @addrs
  */
-typedef guint (*SysprofBacktraceFunc) (SysprofCaptureAddress *addrs,
-                                       guint                  n_addrs,
-                                       gpointer               user_data);
+typedef gint (*SysprofBacktraceFunc) (SysprofCaptureAddress *addrs,
+                                      guint                  n_addrs,
+                                      gpointer               user_data);
 
 SYSPROF_AVAILABLE_IN_ALL
 SysprofCaptureWriter *sysprof_capture_writer_new_from_env                    (gsize                          
    buffer_size);
diff --git a/src/libsysprof-capture/sysprof-collector.c b/src/libsysprof-capture/sysprof-collector.c
index 7379b6f..8f31159 100644
--- a/src/libsysprof-capture/sysprof-collector.c
+++ b/src/libsysprof-capture/sysprof-collector.c
@@ -383,6 +383,8 @@ sysprof_collector_allocate (SysprofCaptureAddress   alloc_addr,
 
     if ((ev = mapped_ring_buffer_allocate (collector->buffer, len)))
       {
+        gint n_addrs;
+
         /* First take a backtrace, so that backtrace_func() can overwrite
          * a little bit of data *BEFORE* ev->addrs as stratch space. This
          * is useful to allow using unw_backtrace() or backtrace() to skip
@@ -392,9 +394,9 @@ sysprof_collector_allocate (SysprofCaptureAddress   alloc_addr,
          * scratch space anyway.
          */
         if (backtrace_func)
-          ev->n_addrs = backtrace_func (ev->addrs, MAX_UNWIND_DEPTH, backtrace_data);
+          n_addrs = backtrace_func (ev->addrs, MAX_UNWIND_DEPTH, backtrace_data);
         else
-          ev->n_addrs = 0;
+          n_addrs = 0;
 
         ev->frame.len = sizeof *ev + sizeof (SysprofCaptureAddress) * ev->n_addrs;
         ev->frame.type = SYSPROF_CAPTURE_FRAME_ALLOCATION;
@@ -404,6 +406,7 @@ sysprof_collector_allocate (SysprofCaptureAddress   alloc_addr,
         ev->tid = collector->tid;
         ev->alloc_addr = alloc_addr;
         ev->alloc_size = alloc_size;
+        ev->n_addrs = CLAMP (n_addrs, 0, MAX_UNWIND_DEPTH);
 
         mapped_ring_buffer_advance (collector->buffer, ev->frame.len);
       }
diff --git a/src/libsysprof/preload/sysprof-memory-collector.c 
b/src/libsysprof/preload/sysprof-memory-collector.c
index a2ba8f3..557d5d3 100644
--- a/src/libsysprof/preload/sysprof-memory-collector.c
+++ b/src/libsysprof/preload/sysprof-memory-collector.c
@@ -70,7 +70,7 @@ collector_init_ctor (void)
   collector_ready = TRUE;
 }
 
-static guint
+static gint
 backtrace_func (SysprofCaptureAddress *addrs,
                 guint                  n_addrs,
                 gpointer               user_data)
@@ -82,7 +82,7 @@ backtrace_func (SysprofCaptureAddress *addrs,
    * and subtract an offset from addrs to avoid having to
    * copy frame pointers around.
    */
-  return unw_backtrace ((void **)addrs - 1, n_addrs);
+  return unw_backtrace ((void **)addrs - 1, n_addrs) - 1;
 # else
   static const gint skip = 1;
   void **stack = alloca (n_addrs * sizeof (gpointer));
@@ -94,13 +94,14 @@ backtrace_func (SysprofCaptureAddress *addrs,
 #elif defined(HAVE_EXECINFO_H)
 # if GLIB_SIZEOF_VOID_P == 8
   /* See note on unw_backtrace() */
-  return backtrace ((void **)addrs - 1, n_addrs);
+  return backtrace ((void **)addrs - 1, n_addrs) - 1;
 # else /* GLIB_SIZEOF_VOID_P != 8 */
+  static const gint skip = 1;
   void **stack = alloca (n_addrs * sizeof (gpointer));
-  guint n = backtrace (stack, n_addrs);
-  for (guint i = 0; i < n; i++)
-    addrs[i] = GPOINTER_TO_SIZE (stack[i]);
-  return n;
+  gint n = backtrace (stack, n_addrs);
+  for (guint i = skip; i < n; i++)
+    addrs[i-skip] = GPOINTER_TO_SIZE (stack[i]);
+  return MAX (0, n - skip);
 # endif /* GLIB_SIZEOF_VOID_P */
 #else
   return 0;