diff --git a/cogl/cogl-material.c b/cogl/cogl-material.c
index 6ddcefb68..36716c339 100644
--- a/cogl/cogl-material.c
+++ b/cogl/cogl-material.c
@@ -1941,8 +1941,8 @@ _cogl_material_prune_empty_layer_difference (CoglMaterial *layers_authority,
   if (layer_parent->index == layer->index && layer_parent->owner == NULL)
     {
       cogl_handle_ref (layer_parent);
-      cogl_handle_unref (layer);
       link->data = layer->parent;
+      cogl_handle_unref (layer);
       recursively_free_layer_caches (layers_authority);
       return;
     }