[gnome-remote-desktop] rdp-pipewire-stream: Add support for explicit DMA buffer modifiers

[Jonas Ådahl <jadahl src gnome org>]

commit aadb2b6d98b70d0da72044c7880bfca30defe631
Author: Jonas Ådahl <jadahl gmail com>
Date:   Fri Nov 26 11:59:45 2021 +0100

    rdp-pipewire-stream: Add support for explicit DMA buffer modifiers
    
    This is needed for proper DMA buffer PipeWire support if the producer is
    updated to require a modifier property for enabling DMA buffer
    allocation. It does not break compatibility with old producers, and as a
    bonus, explicit DMA buffer support will work once the producer supports
    it.

 src/grd-pipewire-utils.c      |   6 +-
 src/grd-rdp-pipewire-stream.c | 124 ++++++++++++++++++++++++++++++++++--------
 2 files changed, 104 insertions(+), 26 deletions(-)
---
diff --git a/src/grd-pipewire-utils.c b/src/grd-pipewire-utils.c
index 0e985e9..46634e6 100644
--- a/src/grd-pipewire-utils.c
+++ b/src/grd-pipewire-utils.c
@@ -73,8 +73,10 @@ grd_get_spa_format_details (enum spa_video_format  spa_format,
     {
       if (format_table[i].spa_format == spa_format)
         {
-          *drm_format = format_table[i].drm_format;
-          *bpp = format_table[i].bpp;
+          if (drm_format)
+            *drm_format = format_table[i].drm_format;
+          if (bpp)
+            *bpp = format_table[i].bpp;
           return;
         }
     }
diff --git a/src/grd-rdp-pipewire-stream.c b/src/grd-rdp-pipewire-stream.c
index d830d9b..a7024af 100644
--- a/src/grd-rdp-pipewire-stream.c
+++ b/src/grd-rdp-pipewire-stream.c
@@ -21,6 +21,7 @@
 
 #include "grd-rdp-pipewire-stream.h"
 
+#include <drm_fourcc.h>
 #include <linux/dma-buf.h>
 #include <pipewire/pipewire.h>
 #include <spa/param/props.h>
@@ -632,45 +633,120 @@ static const struct pw_stream_events stream_events = {
   .process = on_stream_process,
 };
 
+static void
+add_common_format_params (struct spa_pod_builder *pod_builder,
+                          enum spa_video_format   spa_format,
+                          uint32_t                refresh_rate)
+{
+  struct spa_rectangle min_rect;
+  struct spa_rectangle max_rect;
+  struct spa_fraction min_framerate;
+  struct spa_fraction max_framerate;
+
+  min_rect = SPA_RECTANGLE (1, 1);
+  max_rect = SPA_RECTANGLE (INT32_MAX, INT32_MAX);
+  min_framerate = SPA_FRACTION (1, 1);
+  max_framerate = SPA_FRACTION (refresh_rate, 1);
+
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_mediaType,
+                       SPA_POD_Id (SPA_MEDIA_TYPE_video), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_mediaSubtype,
+                       SPA_POD_Id (SPA_MEDIA_SUBTYPE_raw), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_format,
+                       SPA_POD_Id (spa_format), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_size,
+                       SPA_POD_CHOICE_RANGE_Rectangle (&min_rect,
+                                                       &min_rect,
+                                                       &max_rect), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_framerate,
+                       SPA_POD_Fraction (&SPA_FRACTION (0, 1)), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_maxFramerate,
+                       SPA_POD_CHOICE_RANGE_Fraction (&min_framerate,
+                                                      &min_framerate,
+                                                      &max_framerate), 0);
+}
+
 static gboolean
 connect_to_stream (GrdRdpPipeWireStream  *stream,
                    uint32_t               refresh_rate,
                    GError               **error)
 {
+  GrdSession *session = GRD_SESSION (stream->session_rdp);
+  GrdContext *context = grd_session_get_context (session);
   struct pw_stream *pipewire_stream;
   uint8_t params_buffer[1024];
   struct spa_pod_builder pod_builder;
-  struct spa_rectangle min_rect;
-  struct spa_rectangle max_rect;
-  struct spa_fraction min_framerate;
-  struct spa_fraction max_framerate;
   const struct spa_pod *params[2];
+  struct spa_pod_frame format_frame;
+  GrdEglThread *egl_thread;
+  enum spa_video_format spa_format = SPA_VIDEO_FORMAT_BGRx;
+  gboolean need_fallback_format = FALSE;
   int ret;
 
   pipewire_stream = pw_stream_new (stream->pipewire_core,
                                    "grd-rdp-pipewire-stream",
                                    NULL);
 
-  min_rect = SPA_RECTANGLE (1, 1);
-  max_rect = SPA_RECTANGLE (INT32_MAX, INT32_MAX);
-  min_framerate = SPA_FRACTION (1, 1);
-  max_framerate = SPA_FRACTION (refresh_rate, 1);
-
   pod_builder = SPA_POD_BUILDER_INIT (params_buffer, sizeof (params_buffer));
-  params[0] = spa_pod_builder_add_object (
-    &pod_builder,
-    SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
-    SPA_FORMAT_mediaType, SPA_POD_Id (SPA_MEDIA_TYPE_video),
-    SPA_FORMAT_mediaSubtype, SPA_POD_Id (SPA_MEDIA_SUBTYPE_raw),
-    SPA_FORMAT_VIDEO_format, SPA_POD_Id (SPA_VIDEO_FORMAT_BGRx),
-    SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle (&min_rect,
-                                                           &min_rect,
-                                                           &max_rect),
-    SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction (&SPA_FRACTION (0, 1)),
-    SPA_FORMAT_VIDEO_maxFramerate, SPA_POD_CHOICE_RANGE_Fraction (&min_framerate,
-                                                                  &min_framerate,
-                                                                  &max_framerate),
-    0);
+
+  spa_pod_builder_push_object (&pod_builder, &format_frame,
+                               SPA_TYPE_OBJECT_Format,
+                               SPA_PARAM_EnumFormat);
+  add_common_format_params (&pod_builder, spa_format, refresh_rate);
+
+  egl_thread = grd_context_get_egl_thread (context);
+  if (egl_thread)
+    {
+      uint32_t drm_format;
+      int n_modifiers;
+      g_autofree uint64_t *modifiers = NULL;
+
+      grd_get_spa_format_details (spa_format, &drm_format, NULL);
+      if (grd_egl_thread_get_modifiers_for_format (egl_thread, drm_format,
+                                                   &n_modifiers,
+                                                   &modifiers))
+        {
+          struct spa_pod_frame modifier_frame;
+          int i;
+
+          spa_pod_builder_prop (&pod_builder,
+                                SPA_FORMAT_VIDEO_modifier,
+                                (SPA_POD_PROP_FLAG_MANDATORY |
+                                 SPA_POD_PROP_FLAG_DONT_FIXATE));
+
+          spa_pod_builder_push_choice (&pod_builder, &modifier_frame,
+                                       SPA_CHOICE_Enum, 0);
+          spa_pod_builder_long (&pod_builder, modifiers[0]);
+
+          for (i = 0; i < n_modifiers; i++)
+            {
+              uint64_t modifier = modifiers[i];
+
+              spa_pod_builder_long (&pod_builder, modifier);
+            }
+          spa_pod_builder_long (&pod_builder, DRM_FORMAT_MOD_INVALID);
+          spa_pod_builder_pop (&pod_builder, &modifier_frame);
+
+          need_fallback_format = TRUE;
+        }
+    }
+
+  params[0] = spa_pod_builder_pop (&pod_builder, &format_frame);
+
+  if (need_fallback_format)
+    {
+      spa_pod_builder_push_object (&pod_builder, &format_frame,
+                                   SPA_TYPE_OBJECT_Format,
+                                   SPA_PARAM_EnumFormat);
+      add_common_format_params (&pod_builder, spa_format, refresh_rate);
+      params[1] = spa_pod_builder_pop (&pod_builder, &format_frame);
+    }
 
   stream->pipewire_stream = pipewire_stream;
 
@@ -684,7 +760,7 @@ connect_to_stream (GrdRdpPipeWireStream  *stream,
                            stream->src_node_id,
                            (PW_STREAM_FLAG_RT_PROCESS |
                             PW_STREAM_FLAG_AUTOCONNECT),
-                           params, 1);
+                           params, need_fallback_format ? 2 : 1);
   if (ret < 0)
     {
       g_set_error_literal (error, G_IO_ERROR, g_io_error_from_errno (-ret),