cogl-primitives: Check for repeating after the coordinate transform

In _cogl_multitexture_quad_single_primitive we use a wrap mode of
GL_CLAMP_TO_EDGE if the texture coordinates are all in the range [0,1]
or GL_REPEAT otherwise. This is to avoid pulling in pixels from either
side when using GL_LINEAR filter mode and rendering the entire
texture. Previously it was checking using the unconverted texture
coordinates. This is ok unless the texture backend is radically
transforming the texture coordinates, such as in the sub texture
backend where the coordinates may map to something completely
different. We now check whether the coordinates are in range after
converting them.

clutter/cogl/cogl/cogl-primitives.c

@@ -249,6 +249,9 @@ _cogl_multitexture_quad_single_primitive (float        x_1,
       const float       *in_tex_coords;
       float             *out_tex_coords;
       float              default_tex_coords[4] = {0.0, 0.0, 1.0, 1.0};
+      gboolean           need_repeat = FALSE;
+      gint               coord_num;
+      GLenum             wrap_mode;
 
       tex_handle = cogl_material_layer_get_texture (layer);
 
@@ -257,27 +260,41 @@ _cogl_multitexture_quad_single_primitive (float        x_1,
       if (tex_handle == COGL_INVALID_HANDLE)
         continue;
 
-      in_tex_coords = &user_tex_coords[i * 4];
+      /* If the user didn't supply texture coordinates for this layer
+         then use the default coords */
+      if (i >= user_tex_coords_len / 4)
+        in_tex_coords = default_tex_coords;
+      else
+        in_tex_coords = &user_tex_coords[i * 4];
+
       out_tex_coords = &final_tex_coords[i * 4];
 
+      memcpy (out_tex_coords, in_tex_coords, sizeof (GLfloat) * 4);
+
+      /* Convert the texture coordinates to GL. We also work out
+         whether any of the texture coordinates are outside the range
+         [0.0,1.0]. We need to do this after calling
+         transform_coords_to_gl in case the texture backend is munging
+         the coordinates (such as in the sub texture backend). This
+         should be safe to call because we know that the texture only
+         has one slice. */
+      for (coord_num = 0; coord_num < 2; coord_num++)
+        {
+          float *s = out_tex_coords + coord_num * 2;
+          float *t = s + 1;
+          _cogl_texture_transform_coords_to_gl (tex_handle, s, t);
+          if (*s < 0.0f || *s > 1.0f || *t < 0.0f || *t > 1.0f)
+            need_repeat = TRUE;
+        }
 
       /* If the texture has waste or we are using GL_TEXTURE_RECT we
-       * can't handle texture repeating so we check that the texture
+       * can't handle texture repeating so we can't use the layer if
-       * coords lie in the range [0,1].
+       * repeating is required.
-       *
-       * NB: We already know that the texture isn't sliced so we can assume
-       * that the default coords (0,0) and (1,1) would only reference a single
-       * GL texture.
        *
        * NB: We already know that no texture matrix is being used if the
        * texture doesn't support hardware repeat.
        */
-      if (!_cogl_texture_can_hardware_repeat (tex_handle)
+      if (!_cogl_texture_can_hardware_repeat (tex_handle) && need_repeat)
-           && i < user_tex_coords_len / 4
-           && (in_tex_coords[0] < 0 || in_tex_coords[0] > 1.0
-               || in_tex_coords[1] < 0 || in_tex_coords[1] > 1.0
-               || in_tex_coords[2] < 0 || in_tex_coords[2] > 1.0
-               || in_tex_coords[3] < 0 || in_tex_coords[3] > 1.0))
         {
           if (i == 0)
             {
@@ -315,45 +332,15 @@ _cogl_multitexture_quad_single_primitive (float        x_1,
             }
         }
 
-
+      /* If we're not repeating then we want to clamp the coords
-      /*
+         to the edge otherwise it can pull in edge pixels from the
-       * Setup the texture unit...
+         wrong side when scaled */
-       */
+      if (need_repeat)
-
+        wrap_mode = GL_REPEAT;
-      /* NB: The user might not have supplied texture coordinates for all
-       * layers... */
-      if (i < (user_tex_coords_len / 4))
-        {
-          GLenum wrap_mode;
-
-          /* If the texture coords are all in the range [0,1] then we want to
-             clamp the coords to the edge otherwise it can pull in edge pixels
-             from the wrong side when scaled */
-          if (in_tex_coords[0] >= 0 && in_tex_coords[0] <= 1.0
-              && in_tex_coords[1] >= 0 && in_tex_coords[1] <= 1.0
-              && in_tex_coords[2] >= 0 && in_tex_coords[2] <= 1.0
-              && in_tex_coords[3] >= 0 && in_tex_coords[3] <= 1.0)
-            wrap_mode = GL_CLAMP_TO_EDGE;
-          else
-            wrap_mode = GL_REPEAT;
-
-          memcpy (out_tex_coords, in_tex_coords, sizeof (GLfloat) * 4);
-
-          _cogl_texture_set_wrap_mode_parameter (tex_handle, wrap_mode);
-        }
       else
-        {
+        wrap_mode = GL_CLAMP_TO_EDGE;
-          memcpy (out_tex_coords, default_tex_coords, sizeof (GLfloat) * 4);
 
-          _cogl_texture_set_wrap_mode_parameter (tex_handle, GL_CLAMP_TO_EDGE);
+      _cogl_texture_set_wrap_mode_parameter (tex_handle, wrap_mode);
-        }
-
-      _cogl_texture_transform_coords_to_gl (tex_handle,
-                                            &out_tex_coords[0],
-                                            &out_tex_coords[1]);
-      _cogl_texture_transform_coords_to_gl (tex_handle,
-                                            &out_tex_coords[2],
-                                            &out_tex_coords[3]);
     }
 
   _cogl_journal_log_quad (x_1,