diff --git a/common/cogl-material.c b/common/cogl-material.c
index 699ea0a10..2c7df33c8 100644
--- a/common/cogl-material.c
+++ b/common/cogl-material.c
@@ -644,7 +644,8 @@ cogl_material_set_layer_matrix (CoglHandle material_handle,
 static void
 _cogl_material_layer_free (CoglMaterialLayer *layer)
 {
-  cogl_handle_unref (layer->texture);
+  if (layer->texture != COGL_INVALID_HANDLE)
+    cogl_handle_unref (layer->texture);
   g_free (layer);
 }