Adds a _cogl_util_pixel_format_from_masks API

This adds a utility function for inferring a CoglPixelFormat from a
set of channel masks, a bits-per-pixel value, a pixel-depth value and
pixel byte order.

This plan is to use this to improve how we map X visuals to Cogl pixel
formats.

This patch was based on some ideas from Damien Leone <dleone@nvidia.com>

https://bugzilla.gnome.org/show_bug.cgi?id=660188

Reviewed-by: Neil Roberts <neil@linux.intel.com>

cogl/cogl-util.c

@@ -26,6 +26,7 @@
 #endif
 
 #include "cogl-util.h"
+#include "cogl-private.h"
 
 /*
  * cogl_util_next_p2:
@@ -120,3 +121,130 @@ _cogl_util_popcount_table[256] =
   };
 
 #endif /* COGL_UTIL_HAVE_BUILTIN_POPCOUNTL */
+
+/* tests/conform/test-bitmask.c tests some cogl internals and includes this
+ * file directly but since these functions depend on other internal Cogl
+ * symbols we hide them from test-bitmask.c
+ *
+ * XXX: maybe there's a better way for us to handle internal testing
+ * to avoid needing hacks like this.
+ */
+#ifndef _COGL_IN_TEST_BITMASK
+
+/* Given a set of red, green and blue component masks, a depth and
+ * bits per pixel this function tries to determine a corresponding
+ * CoglPixelFormat.
+ *
+ * The depth is measured in bits not including padding for un-used
+ * alpha. The bits per pixel (bpp) does include padding for un-used
+ * alpha.
+ *
+ * This function firstly aims to match formats with RGB ordered
+ * components and only considers alpha coming first, in the most
+ * significant bits. If the function fails to match then it recurses
+ * by either switching the r and b masks around to check for BGR
+ * ordered formats or it recurses with the masks shifted to check for
+ * formats where the alpha component is the least significant bits.
+ */
+static CoglPixelFormat
+_cogl_util_pixel_format_from_masks_real (unsigned long r_mask,
+                                         unsigned long g_mask,
+                                         unsigned long b_mask,
+                                         int depth, int bpp,
+                                         gboolean check_bgr,
+                                         gboolean check_afirst,
+                                         int recursion_depth)
+{
+  CoglPixelFormat image_format;
+
+  if (depth == 24 && bpp == 24 &&
+      r_mask == 0xff0000 && g_mask == 0xff00 && b_mask == 0xff)
+    {
+      return COGL_PIXEL_FORMAT_RGB_888;
+    }
+  else if ((depth == 24 || depth == 32) && bpp == 32 &&
+           r_mask == 0xff0000 && g_mask == 0xff00 && b_mask == 0xff)
+    {
+      return COGL_PIXEL_FORMAT_ARGB_8888_PRE;
+    }
+  else if (depth == 16 && bpp == 16 &&
+           r_mask == 0xf800 && g_mask == 0x7e0 && b_mask == 0x1f)
+    {
+      return COGL_PIXEL_FORMAT_RGB_565;
+    }
+
+  if (recursion_depth == 2)
+    return 0;
+
+  /* Check for BGR ordering if we didn't find a match */
+  if (check_bgr)
+    {
+      image_format =
+        _cogl_util_pixel_format_from_masks_real (b_mask, g_mask, r_mask,
+                                                 depth, bpp,
+                                                 FALSE,
+                                                 TRUE,
+                                                 recursion_depth + 1);
+      if (image_format)
+        return image_format ^ COGL_BGR_BIT;
+    }
+
+  /* Check for alpha in the least significant bits if we still
+   * haven't found a match... */
+  if (check_afirst && depth != bpp)
+    {
+      int shift = bpp - depth;
+
+      image_format =
+        _cogl_util_pixel_format_from_masks_real (r_mask >> shift,
+                                                 g_mask >> shift,
+                                                 b_mask >> shift,
+                                                 depth, bpp,
+                                                 TRUE,
+                                                 FALSE,
+                                                 recursion_depth + 1);
+      if (image_format)
+        return image_format ^ COGL_AFIRST_BIT;
+    }
+
+  return 0;
+}
+
+CoglPixelFormat
+_cogl_util_pixel_format_from_masks (unsigned long r_mask,
+                                    unsigned long g_mask,
+                                    unsigned long b_mask,
+                                    int depth, int bpp,
+                                    gboolean byte_order_is_lsb_first)
+{
+  CoglPixelFormat image_format =
+    _cogl_util_pixel_format_from_masks_real (r_mask, g_mask, b_mask,
+                                             depth, bpp,
+                                             TRUE,
+                                             TRUE,
+                                             0);
+
+  if (!image_format)
+    {
+      const char *byte_order[] = { "MSB first", "LSB first" };
+      g_warning ("Could not find a matching pixel format for red mask=0x%lx,"
+                 "green mask=0x%lx, blue mask=0x%lx at depth=%d, bpp=%d "
+                 "and byte order=%s\n", r_mask, g_mask, b_mask, depth, bpp,
+                 byte_order[!!byte_order_is_lsb_first]);
+      return 0;
+    }
+
+  /* If the image is in little-endian then the order in memory is
+     reversed */
+  if (byte_order_is_lsb_first &&
+      _cogl_pixel_format_is_endian_dependant (image_format))
+    {
+      image_format ^= COGL_BGR_BIT;
+      if (image_format & COGL_A_BIT)
+        image_format ^= COGL_AFIRST_BIT;
+    }
+
+  return image_format;
+}
+
+#endif /* _COGL_IN_TEST_BITMASK */

cogl/cogl-util.h

@@ -27,6 +27,7 @@
 #include <glib.h>
 #include <math.h>
 #include "cogl-defines.h"
+#include "cogl-types.h"
 
 #ifndef COGL_HAS_GLIB_SUPPORT
 #include <stdio.h>
@@ -190,4 +191,18 @@ _cogl_util_popcountl (unsigned long num)
   } while(0)
 #endif /* COGL_HAS_GLIB_SUPPORT */
 
+/* Match a CoglPixelFormat according to channel masks, color depth,
+ * bits per pixel and byte order. These information are provided by
+ * the Visual and XImage structures.
+ *
+ * If no specific pixel format could be found, COGL_PIXEL_FORMAT_ANY
+ * is returned.
+ */
+CoglPixelFormat
+_cogl_util_pixel_format_from_masks (unsigned long r_mask,
+                                    unsigned long g_mask,
+                                    unsigned long b_mask,
+                                    int depth, int bpp,
+                                    int byte_order);
+
 #endif /* __COGL_UTIL_H */

tests/conform/test-bitmask.c

@@ -11,6 +11,7 @@
 
 #include <cogl/cogl-bitmask.h>
 #include <cogl/cogl-bitmask.c>
+#define _COGL_IN_TEST_BITMASK
 #include <cogl/cogl-util.c>
 
 typedef struct