diff --git a/clutter/cogl/cogl/cogl-material.c b/clutter/cogl/cogl/cogl-material.c
index b21f4311b..f4936bf43 100644
--- a/clutter/cogl/cogl/cogl-material.c
+++ b/clutter/cogl/cogl/cogl-material.c
@@ -4457,7 +4457,8 @@ _cogl_material_layer_free (CoglMaterialLayer *layer)
         }
     }
 
-  if (layer->differences & COGL_MATERIAL_LAYER_STATE_TEXTURE)
+  if (layer->differences & COGL_MATERIAL_LAYER_STATE_TEXTURE &&
+      layer->texture != COGL_INVALID_HANDLE)
     cogl_handle_unref (layer->texture);
 
   if (layer->differences & COGL_MATERIAL_LAYER_STATE_NEEDS_BIG_STATE)