[gnome-remote-desktop] rdp-gfx-surface: Add handling to lower frame latency

[Jonas Ådahl <jadahl src gnome org>]

commit 7d510f134d3cd428464c4bbc31f72bcd9f59806b
Author: Pascal Nowack <Pascal Nowack gmx de>
Date:   Mon Jun 21 17:31:31 2021 +0200

    rdp-gfx-surface: Add handling to lower frame latency
    
    While the throttling mechanism can currently take care of higher and
    increasing RTT values, it can currently not take care of lowering the
    latency.
    For example: If an RTT value of 300ms is detected and the RDP client is
    slow with the decoding and displaying process, then
    gnome-remote-desktop adapts to the situation.
    However, if the RTT value suddenly drops to, for example, 150ms, then
    the frame content might still be delayed by 300ms, since the client
    might be too slow with the frame updates.
    
    To solve this situation, recalculate the activate threshold for the
    throttling mechanism, when a new frame is encoded or a frame is being
    acked.
    When the new activate threshold is lower than the current one, suspend
    the encoding until the client acks enough logical frames.
    After this, reevaluate the throttling situation.
    This ensures that gnome-remote-desktop uses the lowest possible
    activate threshold for the throttling mechanism to ensure an experience
    with the lowest possible latency, while still being able to handle high
    latency connections.

 src/grd-rdp-gfx-surface.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 59 insertions(+), 1 deletion(-)
---
diff --git a/src/grd-rdp-gfx-surface.c b/src/grd-rdp-gfx-surface.c
index e05f44c..74f08e1 100644
--- a/src/grd-rdp-gfx-surface.c
+++ b/src/grd-rdp-gfx-surface.c
@@ -33,6 +33,7 @@ typedef enum _ThrottlingState
 {
   THROTTLING_STATE_INACTIVE,
   THROTTLING_STATE_ACTIVE,
+  THROTTLING_STATE_ACTIVE_LOWERING_LATENCY,
 } ThrottlingState;
 
 struct _GrdRdpGfxSurface
@@ -111,6 +112,7 @@ grd_rdp_gfx_surface_unack_frame (GrdRdpGfxSurface *gfx_surface,
 {
   GrdRdpSurface *rdp_surface = gfx_surface->rdp_surface;
   GrdRdpGfxFrameLog *frame_log = gfx_surface->frame_log;
+  uint32_t current_activate_throttling_th;
   uint32_t n_unacked_frames;
   uint32_t enc_rate = 0;
   uint32_t ack_rate = 0;
@@ -133,7 +135,22 @@ grd_rdp_gfx_surface_unack_frame (GrdRdpGfxSurface *gfx_surface,
         }
       break;
     case THROTTLING_STATE_ACTIVE:
-      rdp_surface->encoding_suspended = enc_rate > ack_rate + 1;
+      current_activate_throttling_th = get_activate_throttling_th_from_rtt (
+        gfx_surface, gfx_surface->nw_auto_last_rtt_us);
+
+      if (current_activate_throttling_th < gfx_surface->activate_throttling_th)
+        {
+          gfx_surface->throttling_state = THROTTLING_STATE_ACTIVE_LOWERING_LATENCY;
+          rdp_surface->encoding_suspended = TRUE;
+        }
+      else
+        {
+          gfx_surface->activate_throttling_th = current_activate_throttling_th;
+          rdp_surface->encoding_suspended = enc_rate > ack_rate + 1;
+        }
+      break;
+    case THROTTLING_STATE_ACTIVE_LOWERING_LATENCY:
+      g_assert (rdp_surface->encoding_suspended);
       break;
     }
 }
@@ -146,6 +163,7 @@ grd_rdp_gfx_surface_ack_frame (GrdRdpGfxSurface *gfx_surface,
   GrdRdpSurface *rdp_surface = gfx_surface->rdp_surface;
   GrdRdpGfxFrameLog *frame_log = gfx_surface->frame_log;
   gboolean encoding_was_suspended;
+  uint32_t current_activate_throttling_th;
   uint32_t n_unacked_frames;
   uint32_t enc_rate = 0;
   uint32_t ack_rate = 0;
@@ -165,11 +183,44 @@ grd_rdp_gfx_surface_ack_frame (GrdRdpGfxSurface *gfx_surface,
         {
           gfx_surface->throttling_state = THROTTLING_STATE_INACTIVE;
           rdp_surface->encoding_suspended = FALSE;
+          break;
+        }
+
+      current_activate_throttling_th = get_activate_throttling_th_from_rtt (
+        gfx_surface, gfx_surface->nw_auto_last_rtt_us);
+      if (current_activate_throttling_th < gfx_surface->activate_throttling_th)
+        {
+          gfx_surface->throttling_state = THROTTLING_STATE_ACTIVE_LOWERING_LATENCY;
+          rdp_surface->encoding_suspended = TRUE;
         }
       else
         {
+          gfx_surface->activate_throttling_th = current_activate_throttling_th;
+          rdp_surface->encoding_suspended = enc_rate > ack_rate;
+        }
+      break;
+    case THROTTLING_STATE_ACTIVE_LOWERING_LATENCY:
+      current_activate_throttling_th = get_activate_throttling_th_from_rtt (
+        gfx_surface, gfx_surface->nw_auto_last_rtt_us);
+
+      if (n_unacked_frames < current_activate_throttling_th)
+        {
+          gfx_surface->throttling_state = THROTTLING_STATE_INACTIVE;
+          rdp_surface->encoding_suspended = FALSE;
+        }
+      else if (n_unacked_frames == current_activate_throttling_th)
+        {
+          gfx_surface->throttling_state = THROTTLING_STATE_ACTIVE;
           rdp_surface->encoding_suspended = enc_rate > ack_rate;
         }
+      else if (n_unacked_frames > current_activate_throttling_th)
+        {
+          g_assert (rdp_surface->encoding_suspended);
+        }
+      else
+        {
+          g_assert_not_reached ();
+        }
       break;
     }
 
@@ -207,6 +258,13 @@ reevaluate_encoding_suspension_state (GrdRdpGfxSurface *gfx_surface)
       g_assert (n_unacked_frames > gfx_surface->deactivate_throttling_th);
       g_assert (rdp_surface->encoding_suspended);
       break;
+    case THROTTLING_STATE_ACTIVE_LOWERING_LATENCY:
+      /*
+       * While the graphics pipeline rewrites the frame history, the RTT
+       * detection mechanism cannot submit a new round trip time.
+       */
+      g_assert_not_reached ();
+      break;
     }
 }