Fix checks for out-of-bounds coordinates and repeats

Add a return result from CoglTexture.transform_quad_coords_to_gl(),
so that we can properly determine the nature of repeats in
the face of GL_TEXTURE_RECTANGLE_ARB, where the returned
coordinates are not normalized.

The comment "We also work out whether any of the texture
coordinates are outside the range [0.0,1.0]. We need to do
this after calling transform_coords_to_gl in case the texture
backend is munging the coordinates (such as in the sub texture
backend)." is disregarded and removed, since it's actually
the virtual coordinates that determine whether we repeat,
not the GL coordinates.

Warnings about disregarded layers are used in all cases where
applicable, including for subtextures.

http://bugzilla.openedhand.com/show_bug.cgi?id=2016

Signed-off-by: Neil Roberts <neil@linux.intel.com>

clutter/cogl/cogl/cogl-atlas-texture.c

@@ -308,7 +308,7 @@ _cogl_atlas_texture_transform_coords_to_gl (CoglTexture *tex,
   _cogl_texture_transform_coords_to_gl (atlas_tex->sub_texture, s, t);
 }
 
-static gboolean
+static CoglTransformResult
 _cogl_atlas_texture_transform_quad_coords_to_gl (CoglTexture *tex,
                                                  float *coords)
 {

clutter/cogl/cogl/cogl-primitives.c

@@ -250,8 +250,7 @@ _cogl_multitexture_quad_single_primitive (const float *position,
       const float        *in_tex_coords;
       float              *out_tex_coords;
       float               default_tex_coords[4] = {0.0, 0.0, 1.0, 1.0};
-      gboolean           need_repeat = FALSE;
+      CoglTransformResult transform_result;
-      int               coord_num;
       GLenum              wrap_mode;
 
       tex_handle = cogl_material_layer_get_texture (layer);
@@ -272,22 +271,11 @@ _cogl_multitexture_quad_single_primitive (const float *position,
 
       memcpy (out_tex_coords, in_tex_coords, sizeof (GLfloat) * 4);
 
-      /* Convert the texture coordinates to GL. We also work out
+      /* Convert the texture coordinates to GL.
-         whether any of the texture coordinates are outside the range
+       */
-         [0.0,1.0]. We need to do this after calling
+      transform_result =
-         transform_coords_to_gl in case the texture backend is munging
+        _cogl_texture_transform_quad_coords_to_gl (tex_handle,
-         the coordinates (such as in the sub texture backend). This
+                                                   out_tex_coords);
-         should be safe to call because we know that the texture only
-         has one slice. */
-      if (!_cogl_texture_transform_quad_coords_to_gl (tex_handle,
-                                                      out_tex_coords))
-        /* If the backend can't support these coordinates then bail out */
-        return FALSE;
-      for (coord_num = 0; coord_num < 4; coord_num++)
-        if (out_tex_coords[coord_num] < 0.0f ||
-            out_tex_coords[coord_num] > 1.0f)
-          need_repeat = TRUE;
-
       /* If the texture has waste or we are using GL_TEXTURE_RECT we
        * can't handle texture repeating so we can't use the layer if
        * repeating is required.
@@ -295,7 +283,7 @@ _cogl_multitexture_quad_single_primitive (const float *position,
        * NB: We already know that no texture matrix is being used if the
        * texture doesn't support hardware repeat.
        */
-      if (!_cogl_texture_can_hardware_repeat (tex_handle) && need_repeat)
+      if (transform_result == COGL_TRANSFORM_SOFTWARE_REPEAT)
         {
           if (i == 0)
             {
@@ -336,7 +324,7 @@ _cogl_multitexture_quad_single_primitive (const float *position,
       /* If we're not repeating then we want to clamp the coords
          to the edge otherwise it can pull in edge pixels from the
          wrong side when scaled */
-      if (need_repeat)
+      if (transform_result == COGL_TRANSFORM_HARDWARE_REPEAT)
         wrap_mode = GL_REPEAT;
       else
         wrap_mode = GL_CLAMP_TO_EDGE;

clutter/cogl/cogl/cogl-sub-texture.c

@@ -337,7 +337,7 @@ _cogl_sub_texture_transform_coords_to_gl (CoglTexture *tex,
   _cogl_texture_transform_coords_to_gl (sub_tex->full_texture, s, t);
 }
 
-static gboolean
+static CoglTransformResult
 _cogl_sub_texture_transform_quad_coords_to_gl (CoglTexture *tex,
                                                float *coords)
 {
@@ -348,13 +348,12 @@ _cogl_sub_texture_transform_quad_coords_to_gl (CoglTexture *tex,
      cogl-primitives will resort to manual repeating */
   for (i = 0; i < 4; i++)
     if (coords[i] < 0.0f || coords[i] > 1.0f)
-      return FALSE;
+      return COGL_TRANSFORM_SOFTWARE_REPEAT;
 
   _cogl_sub_texture_map_quad (sub_tex, coords);
 
-  _cogl_texture_transform_quad_coords_to_gl (sub_tex->full_texture, coords);
+  return _cogl_texture_transform_quad_coords_to_gl (sub_tex->full_texture,
-
+                                                    coords);
-  return TRUE;
 }
 
 static gboolean

clutter/cogl/cogl/cogl-texture-2d-sliced.c

@@ -1290,17 +1290,33 @@ _cogl_texture_2d_sliced_transform_coords_to_gl (CoglTexture *tex,
 #endif
 }
 
-static gboolean
+static CoglTransformResult
 _cogl_texture_2d_sliced_transform_quad_coords_to_gl (CoglTexture *tex,
                                                      float *coords)
 {
+  gboolean need_repeat = FALSE;
+  int i;
+
+  /* This is a bit lazy - in the case where the quad lies entirely
+   * within a single slice we could avoid the fallback. But that
+   * could likely lead to visual inconsistency if the fallback involves
+   * dropping layers, so this might be the right thing to do anyways.
+   */
   if (_cogl_texture_2d_sliced_is_sliced (tex))
-    return FALSE;
+    return COGL_TRANSFORM_SOFTWARE_REPEAT;
+
+  for (i = 0; i < 4; i++)
+    if (coords[i] < 0.0f || coords[i] > 1.0f)
+      need_repeat = TRUE;
+
+  if (need_repeat && !_cogl_texture_2d_sliced_can_hardware_repeat (tex))
+    return COGL_TRANSFORM_SOFTWARE_REPEAT;
 
   _cogl_texture_2d_sliced_transform_coords_to_gl (tex, coords + 0, coords + 1);
   _cogl_texture_2d_sliced_transform_coords_to_gl (tex, coords + 2, coords + 3);
 
-  return TRUE;
+  return (need_repeat
+          ? COGL_TRANSFORM_HARDWARE_REPEAT : COGL_TRANSFORM_NO_REPEAT);
 }
 
 static gboolean

clutter/cogl/cogl/cogl-texture-2d.c

@@ -347,13 +347,22 @@ _cogl_texture_2d_transform_coords_to_gl (CoglTexture *tex,
      anything */
 }
 
-static gboolean
+static CoglTransformResult
 _cogl_texture_2d_transform_quad_coords_to_gl (CoglTexture *tex,
                                               float *coords)
 {
   /* The texture coordinates map directly so we don't need to do
-     anything */
+     anything other than check for repeats */
-  return TRUE;
+
+  gboolean need_repeat = FALSE;
+  int i;
+
+  for (i = 0; i < 4; i++)
+    if (coords[i] < 0.0f || coords[i] > 1.0f)
+      need_repeat = TRUE;
+
+  return (need_repeat ? COGL_TRANSFORM_HARDWARE_REPEAT
+          : COGL_TRANSFORM_NO_REPEAT);
 }
 
 static gboolean

clutter/cogl/cogl/cogl-texture-private.h

@@ -43,6 +43,18 @@ typedef void (*CoglTextureSliceCallback) (CoglHandle handle,
 typedef void (* CoglTextureManualRepeatCallback) (const float *coords,
                                                   void *user_data);
 
+/* Encodes three possibiloities result of transforming a quad */
+typedef enum {
+  /* quad doesn't cross the boundaries of a texture */
+  COGL_TRANSFORM_NO_REPEAT,
+  /* quad crosses boundaries, hardware wrap mode can handle */
+  COGL_TRANSFORM_HARDWARE_REPEAT,
+  /* quad crosses boundaries, needs software fallback;
+   * for a sliced texture, this might not actually involve
+   * repeating, just a quad crossing multiple slices */
+  COGL_TRANSFORM_SOFTWARE_REPEAT,
+} CoglTransformResult;
+
 struct _CoglTextureVtable
 {
   /* Virtual functions that must be implemented for a texture
@@ -83,7 +95,7 @@ struct _CoglTextureVtable
   void (* transform_coords_to_gl) (CoglTexture *tex,
                                    float *s,
                                    float *t);
-  gboolean (* transform_quad_coords_to_gl) (CoglTexture *tex,
+  CoglTransformResult (* transform_quad_coords_to_gl) (CoglTexture *tex,
 						       float *coords);
 
   gboolean (* get_gl_texture) (CoglTexture *tex,
@@ -128,7 +140,7 @@ void
 _cogl_texture_transform_coords_to_gl (CoglHandle handle,
                                       float *s,
                                       float *t);
-gboolean
+CoglTransformResult
 _cogl_texture_transform_quad_coords_to_gl (CoglHandle handle,
                                            float *coords);
 

clutter/cogl/cogl/cogl-texture.c

@@ -664,7 +664,7 @@ _cogl_texture_transform_coords_to_gl (CoglHandle handle,
   tex->vtable->transform_coords_to_gl (tex, s, t);
 }
 
-gboolean
+CoglTransformResult
 _cogl_texture_transform_quad_coords_to_gl (CoglHandle handle,
                                            float *coords)
 {