MetaShapedTexture: Allow for a "cairo overlay"

A cairo overlay gives us a path to overlay the mask texture data, allowing for
complex, anti-aliasing effects on the frame shape.

https://bugzilla.gnome.org/show_bug.cgi?id=628195

src/compositor/meta-shaped-texture.c

@@ -66,6 +66,11 @@ struct _MetaShapedTexturePrivate
   cairo_region_t *clip_region;
   cairo_region_t *shape_region;
 
+  cairo_region_t *overlay_region;
+  cairo_path_t *overlay_path;
+
+  cairo_region_t *visible_pixels_region;
+
   guint mask_width, mask_height;
 
   guint create_mipmaps : 1;
@@ -97,6 +102,9 @@ meta_shaped_texture_init (MetaShapedTexture *self)
   priv = self->priv = META_SHAPED_TEXTURE_GET_PRIVATE (self);
 
   priv->shape_region = NULL;
+  priv->overlay_path = NULL;
+  priv->overlay_region = NULL;
+  priv->visible_pixels_region = NULL;
   priv->paint_tower = meta_texture_tower_new ();
   priv->mask_texture = COGL_INVALID_HANDLE;
   priv->create_mipmaps = TRUE;
@@ -127,6 +135,7 @@ meta_shaped_texture_dispose (GObject *object)
 
   meta_shaped_texture_set_shape_region (self, NULL);
   meta_shaped_texture_set_clip_region (self, NULL);
+  meta_shaped_texture_set_overlay_path (self, NULL, NULL);
 
   G_OBJECT_CLASS (meta_shaped_texture_parent_class)->dispose (object);
 }
@@ -161,16 +170,139 @@ meta_shaped_texture_dirty_mask (MetaShapedTexture *stex)
 {
   MetaShapedTexturePrivate *priv = stex->priv;
 
-  if (priv->mask_texture != COGL_INVALID_HANDLE)
+  if (priv->visible_pixels_region != NULL)
+    {
+      cairo_region_destroy (priv->visible_pixels_region);
+      priv->visible_pixels_region = NULL;
+
+      if (priv->mask_texture == COGL_INVALID_HANDLE)
         {
           cogl_handle_unref (priv->mask_texture);
           priv->mask_texture = COGL_INVALID_HANDLE;
+        }
 
       if (priv->material != COGL_INVALID_HANDLE)
         cogl_material_set_layer (priv->material, 1, COGL_INVALID_HANDLE);
     }
 }
 
+static void
+scan_visible_region (MetaShapedTexture *stex,
+                     guchar            *mask_data,
+                     int                stride)
+{
+  MetaShapedTexturePrivate *priv = stex->priv;
+  cairo_region_t *visible_pixels_region;
+  cairo_region_t *overlay_region;
+  int i, n_rects;
+
+  /* The visible pixels region contains all pixel values above 0.
+   * This is somewhat complicated when there's an overlay: we
+   * need to scan all regions potentially modified by it.
+   */
+
+  if (priv->visible_pixels_region)
+    cairo_region_destroy (priv->visible_pixels_region);
+
+  priv->visible_pixels_region = cairo_region_copy (priv->shape_region);
+
+  visible_pixels_region = priv->visible_pixels_region;
+  overlay_region = priv->overlay_region;
+
+  /* With no overlay region, the visible region is defined
+   * by the mask region, so we don't need to scan anything. */
+  if (overlay_region == NULL)
+      return;
+
+  /* Subtract all the rectangles in the overlay region so that we can
+   * scan all the pixels potentially added by the overlay path. */
+  cairo_region_subtract (visible_pixels_region, overlay_region);
+
+  n_rects = cairo_region_num_rectangles (overlay_region);
+
+  for (i = 0; i < n_rects; i++)
+    {
+      int x, y;
+      cairo_rectangle_int_t rect;
+
+      cairo_region_get_rectangle (overlay_region, i, &rect);
+
+      for (y = rect.y; y < (rect.y + rect.height); y++)
+        {
+          for (x = rect.x; x < (rect.x + rect.width); x++)
+            {
+              int w = x;
+              while (mask_data[y * stride + w] > 0 && w < (rect.x + rect.width))
+                w++;
+
+              if (w > 0)
+                {
+                  cairo_rectangle_int_t tmp;
+                  tmp.x = x;
+                  tmp.y = y;
+                  tmp.width = w - x;
+                  tmp.height = 1;
+                  cairo_region_union_rectangle (visible_pixels_region, &tmp);
+                  x = w;
+                }
+            }
+        }
+    }
+}
+
+static void
+install_overlay_path (MetaShapedTexture *stex,
+                      guchar            *mask_data,
+                      int                tex_width,
+                      int                tex_height,
+                      int                stride)
+{
+  MetaShapedTexturePrivate *priv = stex->priv;
+  int i, n_rects;
+  cairo_t *cr;
+  cairo_rectangle_int_t rect;
+  cairo_surface_t *surface;
+
+  if (priv->overlay_region == NULL)
+    return;
+
+  surface = cairo_image_surface_create_for_data (mask_data,
+                                                 CAIRO_FORMAT_A8,
+                                                 tex_width,
+                                                 tex_height,
+                                                 stride);
+
+  cr = cairo_create (surface);
+  cairo_set_operator (cr, CAIRO_OPERATOR_CLEAR);
+
+  n_rects = cairo_region_num_rectangles (priv->overlay_region);
+  for (i = 0; i < n_rects; i++)
+    {
+      cairo_region_get_rectangle (priv->overlay_region, i, &rect);
+      cairo_rectangle (cr, rect.x, rect.y, rect.width, rect.height);
+    }
+
+  cairo_fill_preserve (cr);
+  if (priv->overlay_path == NULL)
+    {
+      /* If we have an overlay region but not an overlay path, then we
+       * just need to clear the rectangles in the overlay region. */
+      goto out;
+    }
+
+  cairo_clip (cr);
+
+  cairo_set_operator (cr, CAIRO_OPERATOR_OVER);
+  cairo_set_source_rgba (cr, 1, 1, 1, 1);
+
+  cairo_append_path (cr, priv->overlay_path);
+  cairo_fill (cr);
+
+ out:
+  cairo_destroy (cr);
+  cairo_surface_destroy (surface);
+}
+
 static void
 meta_shaped_texture_ensure_mask (MetaShapedTexture *stex)
 {
@@ -193,7 +325,7 @@ meta_shaped_texture_ensure_mask (MetaShapedTexture *stex)
     meta_shaped_texture_dirty_mask (stex);
 
   /* If we don't have a mask texture yet then create one */
-  if (priv->mask_texture == COGL_INVALID_HANDLE)
+  if (priv->visible_pixels_region == NULL)
     {
       guchar *mask_data;
       int i;
@@ -201,6 +333,19 @@ meta_shaped_texture_ensure_mask (MetaShapedTexture *stex)
       int stride;
       GLenum paint_gl_target;
 
+      /* If we have no shape region and no (or an empty) overlay region, we
+       * don't need to create a full mask texture, so quit early. */
+      if (priv->shape_region == NULL &&
+          (priv->overlay_region == NULL ||
+           cairo_region_num_rectangles (priv->overlay_region) == 0))
+        {
+          /* With no mask, the visible region is just
+           * {0, 0, tex_width, tex_height}. */
+          cairo_rectangle_int_t rect = { 0, 0, tex_width, tex_height };
+          priv->visible_pixels_region = cairo_region_create_rectangle (&rect);
+          return;
+        }
+
       stride = cairo_format_stride_for_width (CAIRO_FORMAT_A8, tex_width);
 
       /* Create data for an empty image */
@@ -231,6 +376,9 @@ meta_shaped_texture_ensure_mask (MetaShapedTexture *stex)
             memset (p, 255, x2 - x1);
         }
 
+      install_overlay_path (stex, mask_data, tex_width, tex_height, stride);
+      scan_visible_region (stex, mask_data, stride);
+
       cogl_texture_get_gl_texture (paint_tex, NULL, &paint_gl_target);
 
 #ifdef GL_TEXTURE_RECTANGLE_ARB
@@ -562,6 +710,66 @@ meta_shaped_texture_set_shape_region (MetaShapedTexture *stex,
   clutter_actor_queue_redraw (CLUTTER_ACTOR (stex));
 }
 
+/**
+ * meta_shaped_texture_get_visible_pixels_region:
+ * @stex: a #MetaShapedTexture
+ *
+ * Return a region enclosing only visible pixels: those with
+ * alpha values above 0.
+ *
+ * Returns: a #cairo_region_t
+ */
+cairo_region_t *
+meta_shaped_texture_get_visible_pixels_region (MetaShapedTexture *stex)
+{
+  g_return_val_if_fail (META_IS_SHAPED_TEXTURE (stex), NULL);
+
+  meta_shaped_texture_ensure_mask (stex);
+  return stex->priv->visible_pixels_region;
+}
+
+/**
+ * meta_shaped_texture_set_overlay_path:
+ * @stex: a #MetaShapedTexture
+ * @overlay_region: A region containing the parts of the mask to overlay.
+ *   All rectangles in this region are wiped clear to full transparency,
+ *   and the overlay path is clipped to this region.
+ * @overlay_path (transfer full): This path will be painted onto the mask
+ *   texture with a fully opaque source. Due to the lack of refcounting
+ *   in #cairo_path_t, ownership of the path is assumed.
+ */
+void
+meta_shaped_texture_set_overlay_path (MetaShapedTexture *stex,
+                                      cairo_region_t    *overlay_region,
+                                      cairo_path_t      *overlay_path)
+{
+  MetaShapedTexturePrivate *priv;
+
+  g_return_if_fail (META_IS_SHAPED_TEXTURE (stex));
+
+  priv = stex->priv;
+
+  if (priv->overlay_region != NULL)
+    {
+      cairo_region_destroy (priv->overlay_region);
+      priv->overlay_region = NULL;
+    }
+
+  if (priv->overlay_path != NULL)
+    {
+      cairo_path_destroy (priv->overlay_path);
+      priv->overlay_path = NULL;
+    }
+
+  cairo_region_reference (overlay_region);
+  priv->overlay_region = overlay_region;
+
+  /* cairo_path_t does not have refcounting. */
+  priv->overlay_path = overlay_path;
+
+  meta_shaped_texture_dirty_mask (stex);
+}
+
 /**
  * meta_shaped_texture_set_clip_region:
  * @frame: a #TidyTextureframe

src/compositor/meta-shaped-texture.h

@@ -68,6 +68,12 @@ void meta_shaped_texture_clear (MetaShapedTexture *stex);
 void meta_shaped_texture_set_shape_region (MetaShapedTexture *stex,
                                            cairo_region_t    *region);
 
+cairo_region_t *meta_shaped_texture_get_visible_pixels_region (MetaShapedTexture *stex);
+
+void meta_shaped_texture_set_overlay_path (MetaShapedTexture *stex,
+                                           cairo_region_t    *overlay_region,
+                                           cairo_path_t      *overlay_path);
+
 /* Assumes ownership of clip_region */
 void meta_shaped_texture_set_clip_region (MetaShapedTexture *stex,
 					  cairo_region_t    *clip_region);