[totem/wip/hadess/screenshot: 4/7] gst: Import video conversion code from GStreamer

[Bastien Nocera <hadess src gnome org>]

commit ea760f8412b8b10ab122befe1ca2ad08c7f6fa6b
Author: Bastien Nocera <hadess hadess net>
Date:   Tue Mar 8 14:25:19 2022 +0100

    gst: Import video conversion code from GStreamer
    
    So we can modify it for our needs.
    
    From gst-plugins-base/gst-libs/gst/video/convertframe.c

 src/gst/totem-gst-pixbuf-helpers.c | 409 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 406 insertions(+), 3 deletions(-)
---
diff --git a/src/gst/totem-gst-pixbuf-helpers.c b/src/gst/totem-gst-pixbuf-helpers.c
index d234637ba..8420a41cf 100644
--- a/src/gst/totem-gst-pixbuf-helpers.c
+++ b/src/gst/totem-gst-pixbuf-helpers.c
@@ -2,8 +2,9 @@
  * Copyright (C) 2003-2007 the GStreamer project
  *      Julien Moutte <julien moutte net>
  *      Ronald Bultje <rbultje ronald bitfreak net>
+ * Copyright (C) 2007 Wim Taymans <wim taymans gmail com>
  * Copyright (C) 2005-2008 Tim-Philipp Müller <tim centricular net>
- * Copyright (C) 2009 Sebastian Dröge <sebastian droege collabora co uk>
+ * Copyright (C) 2009,2011 Sebastian Dröge <sebastian droege collabora co uk>
  * Copyright © 2009 Christian Persch
  *
  * This program is free software; you can redistribute it and/or modify
@@ -34,6 +35,7 @@
 
 #include <gst/tag/tag.h>
 #include <gst/video/video-format.h>
+#include <gst/video/gstvideometa.h>
 
 static void
 destroy_pixbuf (guchar *pix, gpointer data)
@@ -41,11 +43,405 @@ destroy_pixbuf (guchar *pix, gpointer data)
   gst_sample_unref (GST_SAMPLE (data));
 }
 
+static gboolean
+caps_are_raw (const GstCaps * caps)
+{
+  guint i, len;
+
+  len = gst_caps_get_size (caps);
+
+  for (i = 0; i < len; i++) {
+    GstStructure *st = gst_caps_get_structure (caps, i);
+    if (gst_structure_has_name (st, "video/x-raw"))
+      return TRUE;
+  }
+
+  return FALSE;
+}
+
+static gboolean
+create_element (const gchar * factory_name, GstElement ** element,
+    GError ** err)
+{
+  *element = gst_element_factory_make (factory_name, NULL);
+  if (*element)
+    return TRUE;
+
+  if (err && *err == NULL) {
+    *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
+        "cannot create element '%s' - please check your GStreamer installation",
+        factory_name);
+  }
+
+  return FALSE;
+}
+
+static GstElement *
+get_encoder (const GstCaps * caps, GError ** err)
+{
+  GList *encoders = NULL;
+  GList *filtered = NULL;
+  GstElementFactory *factory = NULL;
+  GstElement *encoder = NULL;
+
+  encoders =
+      gst_element_factory_list_get_elements (GST_ELEMENT_FACTORY_TYPE_ENCODER |
+      GST_ELEMENT_FACTORY_TYPE_MEDIA_IMAGE, GST_RANK_NONE);
+
+  if (encoders == NULL) {
+    *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
+        "Cannot find any image encoder");
+    goto fail;
+  }
+
+  GST_INFO ("got factory list %p", encoders);
+  gst_plugin_feature_list_debug (encoders);
+
+  filtered =
+      gst_element_factory_list_filter (encoders, caps, GST_PAD_SRC, FALSE);
+  GST_INFO ("got filtered list %p", filtered);
+
+  if (filtered == NULL) {
+    gchar *tmp = gst_caps_to_string (caps);
+    *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_MISSING_PLUGIN,
+        "Cannot find any image encoder for caps %s", tmp);
+    g_free (tmp);
+    goto fail;
+  }
+
+  gst_plugin_feature_list_debug (filtered);
+
+  factory = (GstElementFactory *) filtered->data;
+
+  GST_INFO ("got factory %p", factory);
+  encoder = gst_element_factory_create (factory, NULL);
+
+  GST_INFO ("created encoder element %p, %s", encoder,
+      GST_ELEMENT_NAME (encoder));
+
+fail:
+  if (encoders)
+    gst_plugin_feature_list_free (encoders);
+  if (filtered)
+    gst_plugin_feature_list_free (filtered);
+
+  return encoder;
+}
+
+static GstElement *
+build_convert_frame_pipeline (GstElement ** src_element,
+    GstElement ** sink_element, const GstCaps * from_caps,
+    GstVideoCropMeta * cmeta, const GstCaps * to_caps, GError ** err)
+{
+  GstElement *vcrop = NULL, *csp = NULL, *csp2 = NULL, *vscale = NULL;
+  GstElement *src = NULL, *sink = NULL, *encoder = NULL, *pipeline;
+  GstVideoInfo info;
+  GError *error = NULL;
+
+  if (cmeta) {
+    if (!create_element ("videocrop", &vcrop, &error)) {
+      g_error_free (error);
+      g_warning
+          ("build_convert_frame_pipeline: Buffer has crop metadata but videocrop element is not found. 
Cropping will be disabled");
+    } else {
+      if (!create_element ("videoconvert", &csp2, &error))
+        goto no_elements;
+    }
+  }
+
+  /* videoscale is here to correct for the pixel-aspect-ratio for us */
+  GST_DEBUG ("creating elements");
+  if (!create_element ("appsrc", &src, &error) ||
+      !create_element ("videoconvert", &csp, &error) ||
+      !create_element ("videoscale", &vscale, &error) ||
+      !create_element ("appsink", &sink, &error))
+    goto no_elements;
+
+  pipeline = gst_pipeline_new ("videoconvert-pipeline");
+  if (pipeline == NULL)
+    goto no_pipeline;
+
+  /* Add black borders if necessary to keep the DAR */
+  g_object_set (vscale, "add-borders", TRUE, NULL);
+
+  GST_DEBUG ("adding elements");
+  gst_bin_add_many (GST_BIN (pipeline), src, csp, vscale, sink, NULL);
+  if (vcrop)
+    gst_bin_add_many (GST_BIN (pipeline), vcrop, csp2, NULL);
+
+  /* set caps */
+  g_object_set (src, "caps", from_caps, NULL);
+  if (vcrop) {
+    gst_video_info_from_caps (&info, from_caps);
+    g_object_set (vcrop, "left", cmeta->x, NULL);
+    g_object_set (vcrop, "top", cmeta->y, NULL);
+    g_object_set (vcrop, "right", GST_VIDEO_INFO_WIDTH (&info) - cmeta->width,
+        NULL);
+    g_object_set (vcrop, "bottom",
+        GST_VIDEO_INFO_HEIGHT (&info) - cmeta->height, NULL);
+    GST_DEBUG ("crop meta [x,y,width,height]: %d %d %d %d", cmeta->x, cmeta->y,
+        cmeta->width, cmeta->height);
+  }
+  g_object_set (sink, "caps", to_caps, NULL);
+
+  /* FIXME: linking is still way too expensive, profile this properly */
+  if (vcrop) {
+    GST_DEBUG ("linking src->csp2");
+    if (!gst_element_link_pads (src, "src", csp2, "sink"))
+      goto link_failed;
+
+    GST_DEBUG ("linking csp2->vcrop");
+    if (!gst_element_link_pads (csp2, "src", vcrop, "sink"))
+      goto link_failed;
+
+    GST_DEBUG ("linking vcrop->csp");
+    if (!gst_element_link_pads (vcrop, "src", csp, "sink"))
+      goto link_failed;
+  } else {
+    GST_DEBUG ("linking src->csp");
+    if (!gst_element_link_pads (src, "src", csp, "sink"))
+      goto link_failed;
+  }
+
+  GST_DEBUG ("linking csp->vscale");
+  if (!gst_element_link_pads_full (csp, "src", vscale, "sink",
+          GST_PAD_LINK_CHECK_NOTHING))
+    goto link_failed;
+
+  if (caps_are_raw (to_caps)) {
+    GST_DEBUG ("linking vscale->sink");
+
+    if (!gst_element_link_pads_full (vscale, "src", sink, "sink",
+            GST_PAD_LINK_CHECK_NOTHING))
+      goto link_failed;
+  } else {
+    encoder = get_encoder (to_caps, &error);
+    if (!encoder)
+      goto no_encoder;
+    gst_bin_add (GST_BIN (pipeline), encoder);
+
+    GST_DEBUG ("linking vscale->encoder");
+    if (!gst_element_link (vscale, encoder))
+      goto link_failed;
+
+    GST_DEBUG ("linking encoder->sink");
+    if (!gst_element_link_pads (encoder, "src", sink, "sink"))
+      goto link_failed;
+  }
+
+  g_object_set (src, "emit-signals", TRUE, NULL);
+  g_object_set (sink, "emit-signals", TRUE, NULL);
+
+  *src_element = src;
+  *sink_element = sink;
+
+  return pipeline;
+  /* ERRORS */
+no_encoder:
+  {
+    gst_object_unref (pipeline);
+
+    GST_ERROR ("could not find an encoder for provided caps");
+    if (err)
+      *err = error;
+    else
+      g_error_free (error);
+
+    return NULL;
+  }
+no_elements:
+  {
+    if (src)
+      gst_object_unref (src);
+    if (vcrop)
+      gst_object_unref (vcrop);
+    if (csp)
+      gst_object_unref (csp);
+    if (csp2)
+      gst_object_unref (csp2);
+    if (vscale)
+      gst_object_unref (vscale);
+    if (sink)
+      gst_object_unref (sink);
+    GST_ERROR ("Could not convert video frame: %s", error->message);
+    if (err)
+      *err = error;
+    else
+      g_error_free (error);
+    return NULL;
+  }
+no_pipeline:
+  {
+    gst_object_unref (src);
+    if (vcrop)
+      gst_object_unref (vcrop);
+    gst_object_unref (csp);
+    if (csp2)
+      gst_object_unref (csp2);
+    gst_object_unref (vscale);
+    gst_object_unref (sink);
+
+    GST_ERROR ("Could not convert video frame: no pipeline (unknown error)");
+    if (err)
+      *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
+          "Could not convert video frame: no pipeline (unknown error)");
+    return NULL;
+  }
+link_failed:
+  {
+    gst_object_unref (pipeline);
+
+    GST_ERROR ("Could not convert video frame: failed to link elements");
+    if (err)
+      *err = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_NEGOTIATION,
+          "Could not convert video frame: failed to link elements");
+    return NULL;
+  }
+}
+
+/**
+ * totem_gst_video_convert_sample:
+ * @sample: a #GstSample
+ * @to_caps: the #GstCaps to convert to
+ * @timeout: the maximum amount of time allowed for the processing.
+ * @error: pointer to a #GError. Can be %NULL.
+ *
+ * Converts a raw video buffer into the specified output caps.
+ *
+ * The output caps can be any raw video formats or any image formats (jpeg, png, ...).
+ *
+ * The width, height and pixel-aspect-ratio can also be specified in the output caps.
+ *
+ * Returns: The converted #GstSample, or %NULL if an error happened (in which case @err
+ * will point to the #GError).
+ */
+static GstSample *
+totem_gst_video_convert_sample (GstSample * sample, const GstCaps * to_caps,
+    GstClockTime timeout, GError ** error)
+{
+  GstMessage *msg;
+  GstBuffer *buf;
+  GstSample *result = NULL;
+  GError *err = NULL;
+  GstBus *bus;
+  GstCaps *from_caps, *to_caps_copy = NULL;
+  GstFlowReturn ret;
+  GstElement *pipeline, *src, *sink;
+  guint i, n;
+
+  g_return_val_if_fail (sample != NULL, NULL);
+  g_return_val_if_fail (to_caps != NULL, NULL);
+
+  buf = gst_sample_get_buffer (sample);
+  g_return_val_if_fail (buf != NULL, NULL);
+
+  from_caps = gst_sample_get_caps (sample);
+  g_return_val_if_fail (from_caps != NULL, NULL);
+
+  to_caps_copy = gst_caps_new_empty ();
+  n = gst_caps_get_size (to_caps);
+  for (i = 0; i < n; i++) {
+    GstStructure *s = gst_caps_get_structure (to_caps, i);
+
+    s = gst_structure_copy (s);
+    gst_structure_remove_field (s, "framerate");
+    gst_caps_append_structure (to_caps_copy, s);
+  }
+
+  pipeline =
+      build_convert_frame_pipeline (&src, &sink, from_caps,
+      gst_buffer_get_video_crop_meta (buf), to_caps_copy, &err);
+  if (!pipeline)
+    goto no_pipeline;
+
+  /* now set the pipeline to the paused state, after we push the buffer into
+   * appsrc, this should preroll the converted buffer in appsink */
+  GST_DEBUG ("running conversion pipeline to caps %" GST_PTR_FORMAT,
+      to_caps_copy);
+  if (gst_element_set_state (pipeline,
+          GST_STATE_PAUSED) == GST_STATE_CHANGE_FAILURE)
+    goto state_change_failed;
+
+  /* feed buffer in appsrc */
+  GST_DEBUG ("feeding buffer %p, size %" G_GSIZE_FORMAT ", caps %"
+      GST_PTR_FORMAT, buf, gst_buffer_get_size (buf), from_caps);
+  g_signal_emit_by_name (src, "push-buffer", buf, &ret);
+
+  /* now see what happens. We either got an error somewhere or the pipeline
+   * prerolled */
+  bus = gst_element_get_bus (pipeline);
+  msg = gst_bus_timed_pop_filtered (bus,
+      timeout, GST_MESSAGE_ERROR | GST_MESSAGE_ASYNC_DONE);
+
+  if (msg) {
+    switch (GST_MESSAGE_TYPE (msg)) {
+      case GST_MESSAGE_ASYNC_DONE:
+      {
+        /* we're prerolled, get the frame from appsink */
+        g_signal_emit_by_name (sink, "pull-preroll", &result);
+
+        if (result) {
+          GST_DEBUG ("conversion successful: result = %p", result);
+        } else {
+          GST_ERROR ("prerolled but no result frame?!");
+        }
+        break;
+      }
+      case GST_MESSAGE_ERROR:{
+        gchar *dbg = NULL;
+
+        gst_message_parse_error (msg, &err, &dbg);
+        if (err) {
+          GST_ERROR ("Could not convert video frame: %s", err->message);
+          GST_DEBUG ("%s [debug: %s]", err->message, GST_STR_NULL (dbg));
+          if (error)
+            *error = err;
+          else
+            g_error_free (err);
+        }
+        g_free (dbg);
+        break;
+      }
+      default:{
+        g_return_val_if_reached (NULL);
+      }
+    }
+    gst_message_unref (msg);
+  } else {
+    GST_ERROR ("Could not convert video frame: timeout during conversion");
+    if (error)
+      *error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_FAILED,
+          "Could not convert video frame: timeout during conversion");
+  }
+
+  gst_element_set_state (pipeline, GST_STATE_NULL);
+  gst_object_unref (bus);
+  gst_object_unref (pipeline);
+  gst_caps_unref (to_caps_copy);
+
+  return result;
+
+  /* ERRORS */
+no_pipeline:
+state_change_failed:
+  {
+    gst_caps_unref (to_caps_copy);
+
+    if (error)
+      *error = err;
+    else
+      g_error_free (err);
+
+    return NULL;
+  }
+}
+
 GdkPixbuf *
 totem_gst_playbin_get_frame (GstElement *play, GError **error)
 {
   GstStructure *s;
   GstSample *sample = NULL;
+  GstSample *last_sample = NULL;
   GdkPixbuf *pixbuf = NULL;
   GstCaps *to_caps, *sample_caps;
   gint outwidth = 0;
@@ -70,12 +466,19 @@ totem_gst_playbin_get_frame (GstElement *play, GError **error)
       NULL);
 
   /* get frame */
-  g_signal_emit_by_name (play, "convert-sample", to_caps, &sample);
+  g_object_get (G_OBJECT (play), "sample", &last_sample, NULL);
+  if (!last_sample) {
+    g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                         "Failed to retrieve video frame");
+    return NULL;
+  }
+  sample = totem_gst_video_convert_sample (last_sample, to_caps, 25 * GST_SECOND, error);
+  gst_sample_unref (last_sample);
   gst_caps_unref (to_caps);
 
   if (!sample) {
     g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                         "Failed to retrieve or convert video frame");
+                         "Failed to convert video frame");
     return NULL;
   }