diff --git a/src/wayland/meta-wayland-buffer.c b/src/wayland/meta-wayland-buffer.c
index bb0e8d234..11db63211 100644
--- a/src/wayland/meta-wayland-buffer.c
+++ b/src/wayland/meta-wayland-buffer.c
@@ -51,32 +51,30 @@ void
 meta_wayland_buffer_unref (MetaWaylandBuffer *buffer)
 {
   buffer->ref_count--;
+
   if (buffer->ref_count == 0)
     {
-      g_clear_pointer (&buffer->texture, cogl_object_unref);
+      g_warn_if_fail (buffer->use_count == 0);
 
-      if (buffer->accessible)
-        meta_wayland_buffer_release_control (buffer);
+      g_clear_pointer (&buffer->texture, cogl_object_unref);
     }
 }
 
 void
-meta_wayland_buffer_take_control (MetaWaylandBuffer *buffer)
+meta_wayland_buffer_ref_use_count (MetaWaylandBuffer *buffer)
 {
-  if (buffer->accessible)
-    meta_fatal ("buffer control taken twice");
-
-  buffer->accessible = TRUE;
+  buffer->use_count++;
 }
 
 void
-meta_wayland_buffer_release_control (MetaWaylandBuffer *buffer)
+meta_wayland_buffer_unref_use_count (MetaWaylandBuffer *buffer)
 {
-  if (!buffer->accessible)
-    meta_fatal ("buffer released when not in control");
+  g_return_if_fail (buffer->use_count != 0);
 
-  wl_resource_queue_event (buffer->resource, WL_BUFFER_RELEASE);
-  buffer->accessible = FALSE;
+  buffer->use_count--;
+
+  if (buffer->use_count == 0)
+    wl_resource_queue_event (buffer->resource, WL_BUFFER_RELEASE);
 }
 
 MetaWaylandBuffer *
@@ -114,8 +112,7 @@ meta_wayland_buffer_ensure_texture (MetaWaylandBuffer *buffer)
   CoglTexture *texture;
   struct wl_shm_buffer *shm_buffer;
 
-  if (!buffer->accessible)
-    meta_warning ("attempted to process damage on uncommitted buffer");
+  g_return_val_if_fail (buffer->use_count != 0, NULL);
 
   if (buffer->texture)
     goto out;
@@ -150,8 +147,7 @@ meta_wayland_buffer_process_damage (MetaWaylandBuffer *buffer,
 {
   struct wl_shm_buffer *shm_buffer;
 
-  if (!buffer->accessible)
-    meta_warning ("attempted to process damage on uncommitted buffer");
+  g_return_if_fail (buffer->use_count != 0);
 
   shm_buffer = wl_shm_buffer_get (buffer->resource);
 
diff --git a/src/wayland/meta-wayland-buffer.h b/src/wayland/meta-wayland-buffer.h
index 9f6a5bf83..fda534bc1 100644
--- a/src/wayland/meta-wayland-buffer.h
+++ b/src/wayland/meta-wayland-buffer.h
@@ -39,15 +39,14 @@ struct _MetaWaylandBuffer
 
   CoglTexture *texture;
   uint32_t ref_count;
-
-  uint32_t accessible : 1;
+  uint32_t use_count;
 };
 
 MetaWaylandBuffer *     meta_wayland_buffer_from_resource       (struct wl_resource    *resource);
 void                    meta_wayland_buffer_ref                 (MetaWaylandBuffer     *buffer);
 void                    meta_wayland_buffer_unref               (MetaWaylandBuffer     *buffer);
-void                    meta_wayland_buffer_take_control        (MetaWaylandBuffer     *buffer);
-void                    meta_wayland_buffer_release_control     (MetaWaylandBuffer     *buffer);
+void                    meta_wayland_buffer_ref_use_count       (MetaWaylandBuffer     *buffer);
+void                    meta_wayland_buffer_unref_use_count     (MetaWaylandBuffer     *buffer);
 CoglTexture *           meta_wayland_buffer_ensure_texture      (MetaWaylandBuffer     *buffer);
 void                    meta_wayland_buffer_process_damage      (MetaWaylandBuffer     *buffer,
                                                                  cairo_region_t        *region);
diff --git a/src/wayland/meta-wayland-surface.c b/src/wayland/meta-wayland-surface.c
index fc884d9ef..e54da49e5 100644
--- a/src/wayland/meta-wayland-surface.c
+++ b/src/wayland/meta-wayland-surface.c
@@ -174,6 +174,25 @@ meta_wayland_surface_assign_role (MetaWaylandSurface *surface,
     }
 }
 
+static void
+surface_use_buffer (MetaWaylandSurface *surface)
+{
+  g_return_if_fail (!surface->using_buffer);
+
+  meta_wayland_buffer_ref_use_count (surface->buffer);
+  surface->using_buffer = TRUE;
+}
+
+static void
+surface_stop_using_buffer (MetaWaylandSurface *surface)
+{
+  if (!surface->using_buffer)
+    return;
+
+  meta_wayland_buffer_unref_use_count (surface->buffer);
+  surface->using_buffer = FALSE;
+}
+
 static void
 surface_set_buffer (MetaWaylandSurface *surface,
                     MetaWaylandBuffer  *buffer)
@@ -184,6 +203,8 @@ surface_set_buffer (MetaWaylandSurface *surface,
   if (surface->buffer)
     {
       wl_list_remove (&surface->buffer_destroy_listener.link);
+
+      surface_stop_using_buffer (surface);
       meta_wayland_buffer_unref (surface->buffer);
     }
 
@@ -657,11 +678,11 @@ apply_pending_state (MetaWaylandSurface      *surface,
 
       surface_set_buffer (surface, pending->buffer);
 
-      if (pending->buffer)
+      if (pending->buffer && !surface->using_buffer)
         {
           struct wl_shm_buffer *shm_buffer = wl_shm_buffer_get (pending->buffer->resource);
 
-          meta_wayland_buffer_take_control (pending->buffer);
+          surface_use_buffer (surface);
           CoglTexture *texture = meta_wayland_buffer_ensure_texture (pending->buffer);
           meta_surface_actor_wayland_set_texture (META_SURFACE_ACTOR_WAYLAND (surface->surface_actor), texture);
 
@@ -679,7 +700,7 @@ apply_pending_state (MetaWaylandSurface      *surface,
     surface_process_damage (surface, pending->damage);
 
   if (release_new_buffer)
-    meta_wayland_buffer_release_control (pending->buffer);
+    surface_stop_using_buffer (surface);
 
   surface->offset_x += pending->dx;
   surface->offset_y += pending->dy;
diff --git a/src/wayland/meta-wayland-surface.h b/src/wayland/meta-wayland-surface.h
index 7200bbfb7..4a2c24422 100644
--- a/src/wayland/meta-wayland-surface.h
+++ b/src/wayland/meta-wayland-surface.h
@@ -152,6 +152,7 @@ struct _MetaWaylandSurface
   MetaWaylandSurfaceRole *role;
   MetaWindow *window;
   MetaWaylandBuffer *buffer;
+  gboolean using_buffer;
   struct wl_listener buffer_destroy_listener;
   cairo_region_t *input_region;
   cairo_region_t *opaque_region;