[gnome-remote-desktop] rdp/rdpgfx: Forbid protocol resets without AVC support by the client

[Jonas Ådahl <jadahl src gnome org>]

commit bac239f6ca7c2a5ba4ff518436e5d87826f3ee2b
Author: Pascal Nowack <Pascal Nowack gmx de>
Date:   Thu Jan 13 10:54:48 2022 +0100

    rdp/rdpgfx: Forbid protocol resets without AVC support by the client
    
    Clients, that advertise capability sets with version
    RDPGFX_CAPVERSION_103 or later, can reset the graphics pipeline any
    time.
    This behaviour is only used by mstsc and also then only in some error
    cases.
    
    Since a reset of the graphics pipeline can imply, that the client won't
    advertise support for AVC any more, gnome-remote-desktop always needs
    to have a local copy of the frame data on the host, as encoding with
    the RFX Progressive codec can currently only be done with the CPU and
    with a protocol reset, the frame data might actually need to be
    downloaded to the host, which would add huge amount of additional
    complexity of the code paths.
    Since mstsc will not stop advertising support for AVC, a reset of the
    graphics pipeline without AVC support is unlikely to happen, but in
    theory it is possible, as other clients can theoretically make use of
    this protocol reset too.
    
    To be able to use full hardware acceleration, without any transfer of
    raw frame data between the Host and the GPU, protocol resets without
    AVC support on the client side need to be forbidden.
    
    So, do exactly that, to allow gnome-remote-desktop to avoid unnecessary
    frame data transfer between the Host and the GPU.

 src/grd-rdp-graphics-pipeline.c | 59 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
---
diff --git a/src/grd-rdp-graphics-pipeline.c b/src/grd-rdp-graphics-pipeline.c
index fa7d6ad5..270a99b0 100644
--- a/src/grd-rdp-graphics-pipeline.c
+++ b/src/grd-rdp-graphics-pipeline.c
@@ -68,6 +68,7 @@ struct _GrdRdpGraphicsPipeline
   RdpgfxServerContext *rdpgfx_context;
   HANDLE stop_event;
   gboolean channel_opened;
+  gboolean received_first_cap_sets;
   gboolean initialized;
   uint32_t initial_version;
 
@@ -998,6 +999,51 @@ cap_sets_contains_supported_version (RDPGFX_CAPSET *cap_sets,
   return FALSE;
 }
 
+static gboolean
+cap_sets_would_disable_avc (RDPGFX_CAPSET *cap_sets,
+                            uint16_t       n_cap_sets)
+{
+  size_t i;
+  uint16_t j;
+
+  for (i = 0; i < G_N_ELEMENTS (cap_list); ++i)
+    {
+      for (j = 0; j < n_cap_sets; ++j)
+        {
+          if (cap_sets[j].version == cap_list[i])
+            {
+              uint32_t flags = cap_sets[i].flags;
+
+              switch (cap_sets[j].version)
+                {
+                case RDPGFX_CAPVERSION_106:
+                case RDPGFX_CAPVERSION_105:
+                case RDPGFX_CAPVERSION_104:
+                case RDPGFX_CAPVERSION_103:
+                case RDPGFX_CAPVERSION_102:
+                case RDPGFX_CAPVERSION_101:
+                case RDPGFX_CAPVERSION_10:
+                  if (flags & RDPGFX_CAPS_FLAG_AVC_DISABLED)
+                    return TRUE;
+                  return FALSE;
+                case RDPGFX_CAPVERSION_81:
+                  if (!(flags & RDPGFX_CAPS_FLAG_AVC420_ENABLED))
+                    return TRUE;
+                  return FALSE;
+                case RDPGFX_CAPVERSION_8:
+                  return TRUE;
+                default:
+                  g_assert_not_reached ();
+                }
+            }
+        }
+    }
+
+  g_assert_not_reached ();
+
+  return TRUE;
+}
+
 static uint32_t
 rdpgfx_caps_advertise (RdpgfxServerContext             *rdpgfx_context,
                        const RDPGFX_CAPS_ADVERTISE_PDU *caps_advertise)
@@ -1030,6 +1076,19 @@ rdpgfx_caps_advertise (RdpgfxServerContext             *rdpgfx_context,
       return CHANNEL_RC_UNSUPPORTED_VERSION;
     }
 
+  if (graphics_pipeline->received_first_cap_sets &&
+      cap_sets_would_disable_avc (caps_advertise->capsSets,
+                                  caps_advertise->capsSetCount))
+    {
+      g_warning ("[RDP.RDPGFX] CapsAdvertise PDU would reset protocol with "
+                 "unsupported capability sets (disabling AVC)");
+      grd_session_rdp_notify_error (graphics_pipeline->session_rdp,
+                                    GRD_SESSION_RDP_ERROR_BAD_CAPS);
+
+      return CHANNEL_RC_ALREADY_INITIALIZED;
+    }
+  graphics_pipeline->received_first_cap_sets = TRUE;
+
   g_mutex_lock (&graphics_pipeline->caps_mutex);
   g_clear_pointer (&graphics_pipeline->cap_sets, g_free);