[gnome-remote-desktop] vnc-pipewire-stream: Fix usage of implicit DRM format modifiers



commit d9923317f0b60621f0aeb219f4d10612a2178e63
Author: Pascal Nowack <Pascal Nowack gmx de>
Date:   Sun Apr 24 17:57:43 2022 +0200

    vnc-pipewire-stream: Fix usage of implicit DRM format modifiers
    
    When using implicit modifiers, instead of explicit modifiers, then no
    modifiers should be submitted to the attributes of an EGL image.
    The EGL thread already takes care of the situation, when no modifiers
    are submitted.
    However, the RDP pipewire stream class does not and happily submits
    invalid modifiers, leading to undefined results.
    
    Fix this, by only submitting DRM format modifiers, when they are
    explicit, i.e. modifier != DRM_FORMAT_MOD_INVALID.
    
    See also: 
https://github.com/obsproject/obs-studio/pull/5221/commits/de46b491c2681c0cf4b2557a149f654a9c72d8a7

 src/grd-vnc-pipewire-stream.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
---
diff --git a/src/grd-vnc-pipewire-stream.c b/src/grd-vnc-pipewire-stream.c
index 874ba290..ffcbad2b 100644
--- a/src/grd-vnc-pipewire-stream.c
+++ b/src/grd-vnc-pipewire-stream.c
@@ -465,7 +465,7 @@ process_frame_data (GrdVncPipeWireStream *stream,
       int *fds;
       uint32_t *offsets;
       uint32_t *strides;
-      uint64_t *modifiers;
+      uint64_t *modifiers = NULL;
       uint32_t n_planes;
       unsigned int i;
       uint8_t *dst_data;
@@ -476,14 +476,16 @@ process_frame_data (GrdVncPipeWireStream *stream,
       fds = g_alloca (sizeof (int) * n_planes);
       offsets = g_alloca (sizeof (uint32_t) * n_planes);
       strides = g_alloca (sizeof (uint32_t) * n_planes);
-      modifiers = g_alloca (sizeof (uint64_t) * n_planes);
+      if (stream->spa_format.modifier != DRM_FORMAT_MOD_INVALID)
+        modifiers = g_alloca (sizeof (uint64_t) * n_planes);
 
       for (i = 0; i < n_planes; i++)
         {
           fds[i] = buffer->datas[i].fd;
           offsets[i] = buffer->datas[i].chunk->offset;
           strides[i] = buffer->datas[i].chunk->stride;
-          modifiers[i] = stream->spa_format.modifier;
+          if (modifiers)
+            modifiers[i] = stream->spa_format.modifier;
         }
       dst_data = g_malloc0 (height * dst_stride);
 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]