[mutter] gpu-kms: Don't add outputs without modes

[Jonas Ådahl <jadahl src gnome org>]

commit d092e913d6541ed801d1d17af66e6ea715790d5d
Author: Jonas Ådahl <jadahl gmail com>
Date:   Fri Nov 3 15:55:07 2017 +0800

    gpu-kms: Don't add outputs without modes
    
    There seems to be a kernel race when one disconnects an external
    monitor connected to a DisplayPort via a USB-C adapter. The race
    results in a connector being reported as connected, but without any
    modes supported.
    
    This had the side effect that we tried to set a preferred mode to
    the first listed mode, but as no modes were available, we instead tried
    to dereference the first element of a NULL array, causing a
    segmentation fault.
    
    Mitigate this by skipping adding output if no supported modes are
    advertised and the output doesn't support scaling, while moving the
    fallback path for calculating a preferred output mode to after possibly
    adding the common modes, to avoid the unvolentary NULL dereference.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=789501

 src/backends/native/meta-gpu-kms.c    | 14 ++++++++--
 src/backends/native/meta-output-kms.c | 51 ++++++++++++++++++++++-------------
 src/backends/native/meta-output-kms.h |  9 ++++---
 3 files changed, 50 insertions(+), 24 deletions(-)
---
diff --git a/src/backends/native/meta-gpu-kms.c b/src/backends/native/meta-gpu-kms.c
index 0ab345ba9..d8b512342 100644
--- a/src/backends/native/meta-gpu-kms.c
+++ b/src/backends/native/meta-gpu-kms.c
@@ -675,11 +675,21 @@ init_outputs (MetaGpuKms       *gpu_kms,
         {
           MetaOutput *output;
           MetaOutput *old_output;
+          GError *error = NULL;
 
           old_output = find_output_by_id (old_outputs, connector->connector_id);
           output = meta_create_kms_output (gpu_kms, connector, resources,
-                                           old_output);
-          outputs = g_list_prepend (outputs, output);
+                                           old_output,
+                                           &error);
+          if (!output)
+            {
+              g_warning ("Failed to create KMS output: %s", error->message);
+              g_error_free (error);
+            }
+          else
+            {
+              outputs = g_list_prepend (outputs, output);
+            }
         }
     }
 
diff --git a/src/backends/native/meta-output-kms.c b/src/backends/native/meta-output-kms.c
index 1b633b155..5668be51c 100644
--- a/src/backends/native/meta-output-kms.c
+++ b/src/backends/native/meta-output-kms.c
@@ -438,9 +438,10 @@ compare_modes (const void *one,
   return g_strcmp0 (b->name, a->name);
 }
 
-static void
-init_output_modes (MetaOutput *output,
-                   MetaGpuKms *gpu_kms)
+static gboolean
+init_output_modes (MetaOutput  *output,
+                   MetaGpuKms  *gpu_kms,
+                   GError     **error)
 {
   MetaOutputKms *output_kms = output->driver_private;
   unsigned int i;
@@ -460,15 +461,35 @@ init_output_modes (MetaOutput *output,
         output->preferred_mode = output->modes[i];
     }
 
+  /* FIXME: MSC feature bit? */
+  /* Presume that if the output supports scaling, then we have
+   * a panel fitter capable of adjusting any mode to suit.
+   */
+  if (output_kms->has_scaling)
+    add_common_modes (output, gpu_kms);
+
+  if (!output->modes)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No modes available");
+      return FALSE;
+    }
+
+  qsort (output->modes, output->n_modes,
+         sizeof (MetaCrtcMode *), compare_modes);
+
   if (!output->preferred_mode)
     output->preferred_mode = output->modes[0];
+
+  return TRUE;
 }
 
 MetaOutput *
-meta_create_kms_output (MetaGpuKms       *gpu_kms,
-                        drmModeConnector *connector,
-                        MetaKmsResources *resources,
-                        MetaOutput       *old_output)
+meta_create_kms_output (MetaGpuKms        *gpu_kms,
+                        drmModeConnector  *connector,
+                        MetaKmsResources  *resources,
+                        MetaOutput        *old_output,
+                        GError           **error)
 {
   MetaGpu *gpu = META_GPU (gpu_kms);
   MetaOutput *output;
@@ -527,17 +548,11 @@ meta_create_kms_output (MetaGpuKms       *gpu_kms,
       output->height_mm = connector->mmHeight;
     }
 
-  init_output_modes (output, gpu_kms);
-
-  /* FIXME: MSC feature bit? */
-  /* Presume that if the output supports scaling, then we have
-   * a panel fitter capable of adjusting any mode to suit.
-   */
-  if (output_kms->has_scaling)
-    add_common_modes (output, gpu_kms);
-
-  qsort (output->modes, output->n_modes,
-         sizeof (MetaCrtcMode *), compare_modes);
+  if (!init_output_modes (output, gpu_kms, error))
+    {
+      g_object_unref (output);
+      return NULL;
+    }
 
   output_kms->n_encoders = connector->count_encoders;
   output_kms->encoders = g_new0 (drmModeEncoderPtr, output_kms->n_encoders);
diff --git a/src/backends/native/meta-output-kms.h b/src/backends/native/meta-output-kms.h
index ca811b92b..cbca02ef5 100644
--- a/src/backends/native/meta-output-kms.h
+++ b/src/backends/native/meta-output-kms.h
@@ -35,9 +35,10 @@ gboolean meta_output_kms_can_clone (MetaOutput *output,
 
 GBytes * meta_output_kms_read_edid (MetaOutput *output);
 
-MetaOutput * meta_create_kms_output (MetaGpuKms       *gpu_kms,
-                                     drmModeConnector *connector,
-                                     MetaKmsResources *resources,
-                                     MetaOutput       *old_output);
+MetaOutput * meta_create_kms_output (MetaGpuKms        *gpu_kms,
+                                     drmModeConnector  *connector,
+                                     MetaKmsResources  *resources,
+                                     MetaOutput        *old_output,
+                                     GError           **error);
 
 #endif /* META_OUTPUT_KMS_H */