[gnome-remote-desktop] vnc-pipewire-stream: Add support for explicit DMA buffer modifiers

[Jonas Ådahl <jadahl src gnome org>]

commit 346af9a1fe0cf861595a5705355ccbb64f12fc4b
Author: Jonas Ådahl <jadahl gmail com>
Date:   Fri Nov 26 12:13:11 2021 +0100

    vnc-pipewire-stream: Add support for explicit DMA buffer modifiers
    
    This is needed for proper DMA buffer PipeWire support if the producer is
    updated to require a modifier property for enabling DMA buffer
    allocation. It does not break compatibility with old producers, and as a
    bonus, explicit DMA buffer support will work once the producer supports
    it.

 src/grd-vnc-pipewire-stream.c | 122 +++++++++++++++++++++++++++++++++---------
 1 file changed, 98 insertions(+), 24 deletions(-)
---
diff --git a/src/grd-vnc-pipewire-stream.c b/src/grd-vnc-pipewire-stream.c
index b79158d..5ed3ae9 100644
--- a/src/grd-vnc-pipewire-stream.c
+++ b/src/grd-vnc-pipewire-stream.c
@@ -620,44 +620,118 @@ static const struct pw_stream_events stream_events = {
   .process = on_stream_process,
 };
 
+static void
+add_common_format_params (struct spa_pod_builder *pod_builder,
+                          enum spa_video_format   spa_format)
+{
+  struct spa_rectangle min_rect;
+  struct spa_rectangle max_rect;
+  struct spa_fraction min_framerate;
+  struct spa_fraction max_framerate;
+
+  min_rect = SPA_RECTANGLE (1, 1);
+  max_rect = SPA_RECTANGLE (INT32_MAX, INT32_MAX);
+  min_framerate = SPA_FRACTION (1, 1);
+  max_framerate = SPA_FRACTION (30, 1);
+
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_mediaType,
+                       SPA_POD_Id (SPA_MEDIA_TYPE_video), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_mediaSubtype,
+                       SPA_POD_Id (SPA_MEDIA_SUBTYPE_raw), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_format,
+                       SPA_POD_Id (spa_format), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_size,
+                       SPA_POD_CHOICE_RANGE_Rectangle (&min_rect,
+                                                       &min_rect,
+                                                       &max_rect), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_framerate,
+                       SPA_POD_Fraction (&SPA_FRACTION (0, 1)), 0);
+  spa_pod_builder_add (pod_builder,
+                       SPA_FORMAT_VIDEO_maxFramerate,
+                       SPA_POD_CHOICE_RANGE_Fraction (&min_framerate,
+                                                      &min_framerate,
+                                                      &max_framerate), 0);
+}
+
 static gboolean
 connect_to_stream (GrdVncPipeWireStream  *stream,
                    GError               **error)
 {
+  GrdSession *session = GRD_SESSION (stream->session);
+  GrdContext *context = grd_session_get_context (session);
   struct pw_stream *pipewire_stream;
   uint8_t params_buffer[1024];
   struct spa_pod_builder pod_builder;
-  struct spa_rectangle min_rect;
-  struct spa_rectangle max_rect;
-  struct spa_fraction min_framerate;
-  struct spa_fraction max_framerate;
   const struct spa_pod *params[2];
   int ret;
+  enum spa_video_format spa_format = SPA_VIDEO_FORMAT_BGRx;
+  GrdEglThread *egl_thread;
+  struct spa_pod_frame format_frame;
+  gboolean need_fallback_format = FALSE;
 
   pipewire_stream = pw_stream_new (stream->pipewire_core,
                                    "grd-vnc-pipewire-stream",
                                    NULL);
 
-  min_rect = SPA_RECTANGLE (1, 1);
-  max_rect = SPA_RECTANGLE (INT32_MAX, INT32_MAX);
-  min_framerate = SPA_FRACTION (1, 1);
-  max_framerate = SPA_FRACTION (30, 1);
-
   pod_builder = SPA_POD_BUILDER_INIT (params_buffer, sizeof (params_buffer));
-  params[0] = spa_pod_builder_add_object (
-    &pod_builder,
-    SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
-    SPA_FORMAT_mediaType, SPA_POD_Id (SPA_MEDIA_TYPE_video),
-    SPA_FORMAT_mediaSubtype, SPA_POD_Id (SPA_MEDIA_SUBTYPE_raw),
-    SPA_FORMAT_VIDEO_format, SPA_POD_Id (SPA_VIDEO_FORMAT_BGRx),
-    SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle (&min_rect,
-                                                           &min_rect,
-                                                           &max_rect),
-    SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction (&SPA_FRACTION(0, 1)),
-    SPA_FORMAT_VIDEO_maxFramerate, SPA_POD_CHOICE_RANGE_Fraction (&min_framerate,
-                                                                  &min_framerate,
-                                                                  &max_framerate),
-    0);
+
+  spa_pod_builder_push_object (&pod_builder, &format_frame,
+                               SPA_TYPE_OBJECT_Format,
+                               SPA_PARAM_EnumFormat);
+  add_common_format_params (&pod_builder, spa_format);
+
+  egl_thread = grd_context_get_egl_thread (context);
+  if (egl_thread)
+    {
+      uint32_t drm_format;
+      int n_modifiers;
+      g_autofree uint64_t *modifiers = NULL;
+
+      grd_get_spa_format_details (spa_format, &drm_format, NULL);
+      if (grd_egl_thread_get_modifiers_for_format (egl_thread, drm_format,
+                                                   &n_modifiers,
+                                                   &modifiers))
+        {
+          struct spa_pod_frame modifier_frame;
+          int i;
+
+          spa_pod_builder_prop (&pod_builder,
+                                SPA_FORMAT_VIDEO_modifier,
+                                (SPA_POD_PROP_FLAG_MANDATORY |
+                                 SPA_POD_PROP_FLAG_DONT_FIXATE));
+
+          spa_pod_builder_push_choice (&pod_builder, &modifier_frame,
+                                       SPA_CHOICE_Enum, 0);
+          spa_pod_builder_long (&pod_builder, modifiers[0]);
+
+          for (i = 0; i < n_modifiers; i++)
+            {
+              uint64_t modifier = modifiers[i];
+
+              spa_pod_builder_long (&pod_builder, modifier);
+            }
+          spa_pod_builder_long (&pod_builder, DRM_FORMAT_MOD_INVALID);
+          spa_pod_builder_pop (&pod_builder, &modifier_frame);
+
+          need_fallback_format = TRUE;
+        }
+    }
+
+  params[0] = spa_pod_builder_pop (&pod_builder, &format_frame);
+
+  if (need_fallback_format)
+    {
+      spa_pod_builder_push_object (&pod_builder, &format_frame,
+                                   SPA_TYPE_OBJECT_Format,
+                                   SPA_PARAM_EnumFormat);
+      add_common_format_params (&pod_builder, spa_format);
+      params[1] = spa_pod_builder_pop (&pod_builder, &format_frame);
+    }
 
   stream->pipewire_stream = pipewire_stream;
 
@@ -671,7 +745,7 @@ connect_to_stream (GrdVncPipeWireStream  *stream,
                            stream->src_node_id,
                            (PW_STREAM_FLAG_RT_PROCESS |
                             PW_STREAM_FLAG_AUTOCONNECT),
-                           params, 1);
+                           params, need_fallback_format ? 2 : 1);
   if (ret < 0)
     {
       g_set_error_literal (error, G_IO_ERROR, g_io_error_from_errno (-ret),