MutterShapedWindow: drop all indirect references to textures

We don't get correct notifications for the ::cogl-texture property of
ClutterTexture in the case when we are unsetting the pixmap before calling
XFreePixmap. (This is because ClutterX11TexturePixmap is a hack on top
of ClutterTexture and we're a hack on top of that.) So we need to manually
clear everything out.

For consistency we also make sure that we drop all references to dead
textures:

 - When the shape changes
 - If the window pixmap texture changes without first being cleared
   (this is not expected to happen)

https://bugzilla.gnome.org/show_bug.cgi?id=627210

src/compositor/mutter-shaped-texture.c

@@ -166,6 +166,8 @@ mutter_shaped_texture_notify (GObject    *object,
       MutterShapedTexture *stex = (MutterShapedTexture *) object;
       MutterShapedTexturePrivate *priv = stex->priv;
 
+      mutter_shaped_texture_clear (stex);
+
       if (priv->create_mipmaps)
 	mutter_texture_tower_set_base_texture (priv->paint_tower,
 					       clutter_texture_get_cogl_texture (CLUTTER_TEXTURE (stex)));
@@ -190,6 +192,9 @@ mutter_shaped_texture_dirty_mask (MutterShapedTexture *stex)
 
       cogl_handle_unref (priv->mask_texture);
       priv->mask_texture = COGL_INVALID_HANDLE;
+
+      if (priv->material != COGL_INVALID_HANDLE)
+        cogl_material_set_layer (priv->material, 1, COGL_INVALID_HANDLE);
     }
 }
 
@@ -517,6 +522,40 @@ mutter_shaped_texture_set_create_mipmaps (MutterShapedTexture *stex,
     }
 }
 
+/* This is a workaround for deficiencies in the hack tower:
+ *
+ * When we call clutter_x11_texture_pixmap_set_pixmap(tp, None),
+ * ClutterX11TexturePixmap knows that it has to get rid of the old texture, but
+ * clutter_texture_set_cogl_texture(texture, COGL_INVALID_HANDLE) isn't allowed, so
+ * it grabs the material for the texture and manually sets the texture in it. This means
+ * that the "cogl-texture" property isn't notified, so we don't find out about it.
+ *
+ * And if we keep the CoglX11TexturePixmap around after the X pixmap is freed, then
+ * we'll trigger X errors when we actually try to free it.
+ *
+ * The only correct thing to do here is to change our code to derive
+ * from ClutterActor and get rid of the inheritance hack tower.  Once
+ * we want to depend on Clutter-1.4 (which has CoglTexturePixmapX11),
+ * that will be very easy to do.
+ */
+void
+mutter_shaped_texture_clear (MutterShapedTexture *stex)
+{
+  MutterShapedTexturePrivate *priv;
+
+  g_return_if_fail (MUTTER_IS_SHAPED_TEXTURE (stex));
+
+  priv = stex->priv;
+
+  mutter_texture_tower_set_base_texture (priv->paint_tower, COGL_INVALID_HANDLE);
+
+  if (priv->material != COGL_INVALID_HANDLE)
+    cogl_material_set_layer (priv->material, 0, COGL_INVALID_HANDLE);
+
+  if (priv->material_unshaped != COGL_INVALID_HANDLE)
+    cogl_material_set_layer (priv->material_unshaped, 0, COGL_INVALID_HANDLE);
+}
+
 void
 mutter_shaped_texture_clear_rectangles (MutterShapedTexture *stex)
 {

src/compositor/mutter-shaped-texture.h

@@ -89,6 +89,8 @@ ClutterActor *mutter_shaped_texture_new (void);
 void mutter_shaped_texture_set_create_mipmaps (MutterShapedTexture *stex,
 					       gboolean             create_mipmaps);
 
+void mutter_shaped_texture_clear (MutterShapedTexture *stex);
+
 void mutter_shaped_texture_clear_rectangles (MutterShapedTexture *stex);
 
 void mutter_shaped_texture_add_rectangle (MutterShapedTexture *stex,

src/compositor/mutter-window.c

@@ -1028,8 +1028,15 @@ mutter_window_detach (MutterWindow *self)
   if (!priv->back_pixmap)
     return;
 
+  /* Get rid of all references to the pixmap before freeing it; it's unclear whether
+   * you are supposed to be able to free a GLXPixmap after freeing the underlying
+   * pixmap, but it certainly doesn't work with current DRI/Mesa
+   */
   clutter_x11_texture_pixmap_set_pixmap (CLUTTER_X11_TEXTURE_PIXMAP (priv->actor),
                                          None);
+  mutter_shaped_texture_clear (MUTTER_SHAPED_TEXTURE (priv->actor));
+  cogl_flush();
+
   XFreePixmap (xdisplay, priv->back_pixmap);
   priv->back_pixmap = None;