[mutter] screen-cast: Disable DMA buffer based screen casting for QXL

[Robert Mader <rmader src gnome org>]

commit 986066ed407213f3c2ad79d8ddad3cd5bc1548e4
Author: Jonas Ådahl <jadahl gmail com>
Date:   Wed Jun 17 17:50:17 2020 +0200

    screen-cast: Disable DMA buffer based screen casting for QXL
    
    QXL doesn't support mmap():ing a DMA buffer allocated in mutter inside
    the PipeWire stream consumer process. To make screen casting work again
    on QXL, disable DMA buffer based screen casting for QXL.
    
    Eventually, it should be the client that renegotiates the supported
    buffer types, but until then we need this list.
    
    https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1318

 src/backends/meta-screen-cast.c           | 11 +++++++++
 src/backends/meta-screen-cast.h           |  2 ++
 src/backends/native/meta-backend-native.c | 40 +++++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+)
---
diff --git a/src/backends/meta-screen-cast.c b/src/backends/meta-screen-cast.c
index 499598042c..87e3881631 100644
--- a/src/backends/meta-screen-cast.c
+++ b/src/backends/meta-screen-cast.c
@@ -46,6 +46,8 @@ struct _MetaScreenCast
 
   MetaDbusSessionWatcher *session_watcher;
   MetaBackend *backend;
+
+  gboolean disable_dma_bufs;
 };
 
 static void
@@ -94,6 +96,12 @@ meta_screen_cast_get_backend (MetaScreenCast *screen_cast)
   return screen_cast->backend;
 }
 
+void
+meta_screen_cast_disable_dma_bufs (MetaScreenCast *screen_cast)
+{
+  screen_cast->disable_dma_bufs = TRUE;
+}
+
 CoglDmaBufHandle *
 meta_screen_cast_create_dma_buf_handle (MetaScreenCast *screen_cast,
                                         int             width,
@@ -107,6 +115,9 @@ meta_screen_cast_create_dma_buf_handle (MetaScreenCast *screen_cast,
   g_autoptr (GError) error = NULL;
   CoglDmaBufHandle *dmabuf_handle;
 
+  if (screen_cast->disable_dma_bufs)
+    return NULL;
+
   dmabuf_handle = cogl_renderer_create_dma_buf (cogl_renderer,
                                                 width, height,
                                                 &error);
diff --git a/src/backends/meta-screen-cast.h b/src/backends/meta-screen-cast.h
index bc1d3d29c1..e2ea5a5e46 100644
--- a/src/backends/meta-screen-cast.h
+++ b/src/backends/meta-screen-cast.h
@@ -56,6 +56,8 @@ GDBusConnection * meta_screen_cast_get_connection (MetaScreenCast *screen_cast);
 
 MetaBackend * meta_screen_cast_get_backend (MetaScreenCast *screen_cast);
 
+void meta_screen_cast_disable_dma_bufs (MetaScreenCast *screen_cast);
+
 CoglDmaBufHandle * meta_screen_cast_create_dma_buf_handle (MetaScreenCast *screen_cast,
                                                            int             width,
                                                            int             height);
diff --git a/src/backends/native/meta-backend-native.c b/src/backends/native/meta-backend-native.c
index 1808c6df22..cc7fc532d8 100644
--- a/src/backends/native/meta-backend-native.c
+++ b/src/backends/native/meta-backend-native.c
@@ -64,6 +64,10 @@
 #include "core/meta-border.h"
 #include "meta/main.h"
 
+#ifdef HAVE_REMOTE_DESKTOP
+#include "backends/meta-screen-cast.h"
+#endif
+
 struct _MetaBackendNative
 {
   MetaBackend parent;
@@ -338,6 +342,38 @@ meta_backend_native_create_clutter_backend (MetaBackend *backend)
   return g_object_new (META_TYPE_CLUTTER_BACKEND_NATIVE, NULL);
 }
 
+#ifdef HAVE_REMOTE_DESKTOP
+static void
+maybe_disable_screen_cast_dma_bufs (MetaBackendNative *native)
+{
+  MetaBackend *backend = META_BACKEND (native);
+  MetaRenderer *renderer = meta_backend_get_renderer (backend);
+  MetaRendererNative *renderer_native = META_RENDERER_NATIVE (renderer);
+  MetaGpuKms *primary_gpu;
+  MetaKmsDevice *kms_device;
+  const char *driver_name;
+  static const char *disable_dma_buf_drivers[] = {
+    "qxl",
+    NULL,
+  };
+
+  primary_gpu = meta_renderer_native_get_primary_gpu (renderer_native);
+  kms_device = meta_gpu_kms_get_kms_device (primary_gpu);
+  driver_name = meta_kms_device_get_driver_name (kms_device);
+
+  if (g_strv_contains (disable_dma_buf_drivers, driver_name))
+    {
+      MetaScreenCast *screen_cast = meta_backend_get_screen_cast (backend);
+
+      g_message ("The '%s' driver doesn't support DMA buffer screen sharing, disabling.",
+                 driver_name);
+
+      meta_screen_cast_disable_dma_bufs (screen_cast);
+      return;
+    }
+}
+#endif /* HAVE_REMOTE_DESKTOP */
+
 static void
 meta_backend_native_post_init (MetaBackend *backend)
 {
@@ -368,6 +404,10 @@ meta_backend_native_post_init (MetaBackend *backend)
         g_warning ("Failed to set RT scheduler: %m");
     }
 
+#ifdef HAVE_REMOTE_DESKTOP
+  maybe_disable_screen_cast_dma_bufs (META_BACKEND_NATIVE (backend));
+#endif
+
 #ifdef HAVE_WAYLAND
   meta_backend_init_wayland (backend);
 #endif