diff --git a/cogl/cogl-context.c b/cogl/cogl-context.c
index 38954caf6..25aaf1016 100644
--- a/cogl/cogl-context.c
+++ b/cogl/cogl-context.c
@@ -130,6 +130,7 @@ cogl_create_context (void)
   _context->legacy_fog_state.enabled = FALSE;
 
   _context->simple_material = cogl_material_new ();
+  _context->texture_material = cogl_material_new ();
   _context->arbfp_source_buffer = g_string_new ("");
   _context->source_stack = NULL;
 
@@ -268,6 +269,8 @@ _cogl_destroy_context (void)
 
   if (_context->simple_material)
     cogl_handle_unref (_context->simple_material);
+  if (_context->texture_material)
+    cogl_handle_unref (_context->texture_material);
 
   if (_context->journal)
     g_array_free (_context->journal, TRUE);
diff --git a/cogl/cogl-context.h b/cogl/cogl-context.h
index 50eef3556..134b21097 100644
--- a/cogl/cogl-context.h
+++ b/cogl/cogl-context.h
@@ -83,7 +83,8 @@ typedef struct
   CoglMaterialFogState legacy_fog_state;
 
   /* Materials */
-  CoglMaterial     *simple_material;
+  CoglMaterial     *simple_material; /* used for set_source_color */
+  CoglMaterial     *texture_material; /* used for set_source_texture */
   GString          *arbfp_source_buffer;
   GList            *source_stack;
 
diff --git a/cogl/cogl.c b/cogl/cogl.c
index 8392caf91..f740173ca 100644
--- a/cogl/cogl.c
+++ b/cogl/cogl.c
@@ -373,9 +373,6 @@ cogl_set_source_color (const CoglColor *color)
 
   _COGL_GET_CONTEXT (ctx, NO_RETVAL);
 
-  /* In case cogl_set_source_texture was previously used... */
-  cogl_material_remove_layer (ctx->simple_material, 0);
-
   premultiplied = *color;
   cogl_color_premultiply (&premultiplied);
   cogl_material_set_color (ctx->simple_material, &premultiplied);
@@ -1058,16 +1055,12 @@ cogl_set_source (CoglMaterial *material)
 void
 cogl_set_source_texture (CoglHandle texture_handle)
 {
-  CoglColor white;
-
   _COGL_GET_CONTEXT (ctx, NO_RETVAL);
 
   g_return_if_fail (texture_handle != NULL);
 
-  cogl_material_set_layer (ctx->simple_material, 0, texture_handle);
-  cogl_color_init_from_4ub (&white, 0xff, 0xff, 0xff, 0xff);
-  cogl_material_set_color (ctx->simple_material, &white);
-  cogl_set_source (ctx->simple_material);
+  cogl_material_set_layer (ctx->texture_material, 0, texture_handle);
+  cogl_set_source (ctx->texture_material);
 }
 
 void