[gnome-remote-desktop] hwaccel-nvidia: Load extra CUDA functions for GL PBO mappings

[Jonas Ådahl <jadahl src gnome org>]

commit 4cf415aa6b842acb104a0280dac101cb413b199c
Author: Pascal Nowack <Pascal Nowack gmx de>
Date:   Tue Jan 4 15:43:07 2022 +0100

    hwaccel-nvidia: Load extra CUDA functions for GL PBO mappings
    
    The nv-codec-headers don't load all functions from the NVIDIA driver.
    Two of these functions are cuGraphicsGLRegisterBuffer() and
    cuGraphicsResourceGetMappedPointer().
    These two functions are required in order to be able to map pixel
    buffer objects in GL.
    As there is no obvious way to contribute to the nv-codec-headers and
    having a new version ready in Fedora for the CI image takes a lot of
    time, load these functions in gnome-remote-desktop manually.
    Since dlopen() is refcounted, this won't cause any problems.
    
    So, load these two extra functions in order to be able to map pixel
    buffer objects.

 src/grd-hwaccel-nvidia.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
---
diff --git a/src/grd-hwaccel-nvidia.c b/src/grd-hwaccel-nvidia.c
index fc3ff56e..90bda2e4 100644
--- a/src/grd-hwaccel-nvidia.c
+++ b/src/grd-hwaccel-nvidia.c
@@ -28,6 +28,19 @@
 
 #define MAX_CUDA_DEVICES_FOR_RETRIEVAL 32
 
+typedef CUresult CUDAAPI tcuGraphicsGLRegisterBuffer (CUgraphicsResource *resource,
+                                                      GLuint              buffer,
+                                                      unsigned int        flags);
+typedef CUresult CUDAAPI tcuGraphicsResourceGetMappedPointer_v2 (CUdeviceptr        *dev_ptr,
+                                                                 size_t             *size,
+                                                                 CUgraphicsResource  resource);
+
+typedef struct _ExtraCudaFunctions
+{
+  tcuGraphicsGLRegisterBuffer *cuGraphicsGLRegisterBuffer;
+  tcuGraphicsResourceGetMappedPointer_v2 *cuGraphicsResourceGetMappedPointer;
+} ExtraCudaFunctions;
+
 typedef struct _DevRetrievalData
 {
   GrdHwAccelNvidia *hwaccel_nvidia;
@@ -55,6 +68,9 @@ struct _GrdHwAccelNvidia
   NvencFunctions *nvenc_funcs;
   NV_ENCODE_API_FUNCTION_LIST nvenc_api;
 
+  void *cuda_lib;
+  ExtraCudaFunctions *extra_cuda_funcs;
+
   CUdevice cu_device;
   CUcontext cu_context;
   gboolean initialized;
@@ -424,6 +440,31 @@ grd_hwaccel_nvidia_avc420_encode_bgrx_frame (GrdHwAccelNvidia  *hwaccel_nvidia,
   return TRUE;
 }
 
+static gboolean
+load_extra_cuda_functions (GrdHwAccelNvidia *hwaccel_nvidia)
+{
+  ExtraCudaFunctions *extra_cuda_funcs;
+
+  hwaccel_nvidia->cuda_lib = dlopen ("libcuda.so.1", RTLD_LAZY);
+  if (!hwaccel_nvidia->cuda_lib)
+    return FALSE;
+
+  hwaccel_nvidia->extra_cuda_funcs = g_malloc0 (sizeof (ExtraCudaFunctions));
+
+  extra_cuda_funcs = hwaccel_nvidia->extra_cuda_funcs;
+  extra_cuda_funcs->cuGraphicsGLRegisterBuffer =
+    dlsym (hwaccel_nvidia->cuda_lib, "cuGraphicsGLRegisterBuffer");
+  if (!extra_cuda_funcs->cuGraphicsGLRegisterBuffer)
+    return FALSE;
+
+  extra_cuda_funcs->cuGraphicsResourceGetMappedPointer =
+    dlsym (hwaccel_nvidia->cuda_lib, "cuGraphicsResourceGetMappedPointer_v2");
+  if (!extra_cuda_funcs->cuGraphicsGLRegisterBuffer)
+    return FALSE;
+
+  return TRUE;
+}
+
 static gboolean
 get_cuda_devices_in_impl (gpointer user_data)
 {
@@ -497,6 +538,11 @@ grd_hwaccel_nvidia_new (GrdEglThread *egl_thread)
       g_debug ("[HWAccel.CUDA] Failed to load CUDA or NVENC library");
       return NULL;
     }
+  if (!load_extra_cuda_functions (hwaccel_nvidia))
+    {
+      g_warning ("[HWAccel.CUDA] Failed to load extra CUDA functions");
+      return NULL;
+    }
 
   cuda_funcs = hwaccel_nvidia->cuda_funcs;
   nvenc_funcs = hwaccel_nvidia->nvenc_funcs;
@@ -600,6 +646,9 @@ grd_hwaccel_nvidia_dispose (GObject *object)
   g_clear_pointer (&hwaccel_nvidia->cu_module_avc_utils,
                    hwaccel_nvidia->cuda_funcs->cuModuleUnload);
 
+  g_clear_pointer (&hwaccel_nvidia->cuda_lib, dlclose);
+  g_clear_pointer (&hwaccel_nvidia->extra_cuda_funcs, g_free);
+
   nvenc_free_functions (&hwaccel_nvidia->nvenc_funcs);
   cuda_free_functions (&hwaccel_nvidia->cuda_funcs);