[cogl/wip/pixel-format-2101010: 3/5] Improve pixel format detection for fallback OpenGL rendering

[Robert Bragg <rbragg src gnome org>]

commit 85ae929a0d97717aec09e770ac6caa3d1116c453
Author: Damien Leone <dleone nvidia com>
Date:   Mon Nov 21 15:41:13 2011 -0800

    Improve pixel format detection for fallback OpenGL rendering
    
    The previous detection was based on bits per pixel only and would
    consider bpp >= 24 as X888 or 8888 24-bit color depth formats.
    
    This commit adds a utility function that returns a CoglPixelFormat
    according to channel masks and color depth. This helps to add support
    for more pixel formats.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=660188
    
    Reviewed-by: Robert Bragg <robert linux intel com>

 cogl/cogl-util.c                      |   47 +++++++++++++++++++++++++++++++
 cogl/cogl-util.h                      |   15 ++++++++++
 cogl/winsys/cogl-texture-pixmap-x11.c |   49 +++++---------------------------
 3 files changed, 70 insertions(+), 41 deletions(-)
---
diff --git a/cogl/cogl-util.c b/cogl/cogl-util.c
index 487a762..cc7f10d 100644
--- a/cogl/cogl-util.c
+++ b/cogl/cogl-util.c
@@ -25,6 +25,7 @@
 #include "config.h"
 #endif
 
+#include "cogl-internal.h"
 #include "cogl-util.h"
 
 /*
@@ -120,3 +121,49 @@ _cogl_util_popcount_table[256] =
   };
 
 #endif /* COGL_UTIL_HAVE_BUILTIN_POPCOUNTL */
+
+CoglPixelFormat
+_cogl_util_pixel_format_from_masks (unsigned long r_mask,
+                                    unsigned long g_mask,
+                                    unsigned long b_mask,
+                                    int depth, int bpp,
+                                    int byte_order)
+{
+#define MASK_FMT_888(a, b, c) ((a) == 0xff0000 && (b) == 0xff00 && (c) == 0xff)
+
+  CoglPixelFormat image_format;
+
+  /* Determine format from masks */
+  if ((depth == 24 || depth == 32) && MASK_FMT_888(r_mask, g_mask, b_mask))
+    image_format = COGL_PIXEL_FORMAT_RGB_888;
+  else
+    {
+      g_warning ("Could not find a matching pixel format for red mask=0x%lx," \
+                 "green mask=0x%lx, blue mask=0x%lx at depth=%d, bpp=%d " \
+                 "and byte order=%d\n", r_mask, g_mask, b_mask, depth, bpp,
+                 byte_order);
+
+      return COGL_PIXEL_FORMAT_ANY;
+    }
+
+  /* Check if there is an alpha channel */
+  if (_cogl_util_popcountl(r_mask | g_mask | b_mask) != depth || bpp != depth)
+    {
+      if (image_format & COGL_PIXEL_FORMAT_24)
+        image_format |= COGL_PIXEL_FORMAT_32 & ~COGL_PIXEL_FORMAT_24;
+
+      image_format |= COGL_A_BIT | COGL_PREMULT_BIT;
+    }
+
+  /* Check if the alpha channel comes first */
+  if (image_format & COGL_A_BIT &&
+      ((image_format & COGL_BGR_BIT) || (byte_order == MSBFirst)))
+      image_format |= COGL_AFIRST_BIT;
+
+  /* If the image is in little-endian then the order in memory is
+     reversed */
+  if (byte_order == LSBFirst)
+      image_format ^= COGL_BGR_BIT;
+
+  return image_format;
+}
diff --git a/cogl/cogl-util.h b/cogl/cogl-util.h
index 04e59d0..dfaa6a9 100644
--- a/cogl/cogl-util.h
+++ b/cogl/cogl-util.h
@@ -27,6 +27,7 @@
 #include <glib.h>
 #include <math.h>
 #include "cogl-defines.h"
+#include "cogl-types.h"
 
 #ifndef COGL_HAS_GLIB_SUPPORT
 #include <stdio.h>
@@ -190,4 +191,18 @@ _cogl_util_popcountl (unsigned long num)
   } while(0)
 #endif /* COGL_HAS_GLIB_SUPPORT */
 
+/* Match a CoglPixelFormat according to channel masks, color depth,
+ * bits per pixel and byte order. These information are provided by
+ * the Visual and XImage structures.
+ *
+ * If no specific pixel format could be found, COGL_PIXEL_FORMAT_ANY
+ * is returned.
+ */
+CoglPixelFormat
+_cogl_util_pixel_format_from_masks (unsigned long r_mask,
+                                    unsigned long g_mask,
+                                    unsigned long b_mask,
+                                    int depth, int bpp,
+                                    int byte_order);
+
 #endif /* __COGL_UTIL_H */
diff --git a/cogl/winsys/cogl-texture-pixmap-x11.c b/cogl/winsys/cogl-texture-pixmap-x11.c
index 33d74ba..5ba004e 100644
--- a/cogl/winsys/cogl-texture-pixmap-x11.c
+++ b/cogl/winsys/cogl-texture-pixmap-x11.c
@@ -477,12 +477,14 @@ static void
 _cogl_texture_pixmap_x11_update_image_texture (CoglTexturePixmapX11 *tex_pixmap)
 {
   Display *display;
+  Visual *visual;
   CoglPixelFormat image_format;
   XImage *image;
   int src_x, src_y;
   int x, y, width, height;
 
   display = cogl_xlib_get_display ();
+  visual = tex_pixmap->visual;
 
   /* If the damage region is empty then there's nothing to do */
   if (tex_pixmap->damage_rect.x2 == tex_pixmap->damage_rect.x1)
@@ -576,47 +578,12 @@ _cogl_texture_pixmap_x11_update_image_texture (CoglTexturePixmapX11 *tex_pixmap)
                     x, y);
     }
 
-  /* xlib doesn't appear to fill in image->{red,green,blue}_mask so
-     this just assumes that the image is stored as ARGB from most
-     significant byte to to least significant. If the format is little
-     endian that means the order will be BGRA in memory */
-
-  switch (image->bits_per_pixel)
-    {
-    default:
-    case 32:
-      {
-        /* If the pixmap is actually non-packed-pixel RGB format then
-           the texture would have been created in RGB_888 format so Cogl
-           will ignore the alpha channel and effectively pack it for
-           us */
-        image_format = COGL_PIXEL_FORMAT_RGBA_8888_PRE;
-
-        /* If the format is actually big endian then the alpha
-           component will come first */
-        if (image->byte_order == MSBFirst)
-          image_format |= COGL_AFIRST_BIT;
-      }
-      break;
-
-    case 24:
-      image_format = COGL_PIXEL_FORMAT_RGB_888;
-      break;
-
-    case 16:
-      /* FIXME: this should probably swap the orders around if the
-         endianness does not match */
-      image_format = COGL_PIXEL_FORMAT_RGB_565;
-      break;
-    }
-
-  if (image->bits_per_pixel != 16)
-    {
-      /* If the image is in little-endian then the order in memory is
-         reversed */
-      if (image->byte_order == LSBFirst)
-        image_format |= COGL_BGR_BIT;
-    }
+  image_format = _cogl_util_pixel_format_from_masks (visual->red_mask,
+                                                     visual->green_mask,
+                                                     visual->blue_mask,
+                                                     image->depth,
+                                                     image->bits_per_pixel,
+                                                     image->byte_order);
 
   cogl_texture_set_region (tex_pixmap->tex,
                            src_x, src_y,