diff --git a/src/compositor/meta-window-actor.c b/src/compositor/meta-window-actor.c
index 6df5a36d6..ab260fa21 100644
--- a/src/compositor/meta-window-actor.c
+++ b/src/compositor/meta-window-actor.c
@@ -407,10 +407,11 @@ meta_window_actor_constructed (GObject *object)
 
   if (!priv->surface)
     {
-      priv->surface = meta_surface_actor_new ();
-
       if (window->surface)
-        window->surface->surface_actor = priv->surface;
+        priv->surface = window->surface->surface_actor;
+      else
+        priv->surface = meta_surface_actor_new ();
+      g_object_ref_sink (priv->surface);
 
       clutter_actor_add_child (CLUTTER_ACTOR (self), CLUTTER_ACTOR (priv->surface));
 
@@ -420,15 +421,6 @@ meta_window_actor_constructed (GObject *object)
                           G_CALLBACK (surface_allocation_changed_notify),
                           self);
 
-      /*
-       * Since we are holding a pointer to this actor independently of the
-       * ClutterContainer internals, and provide a public API to access it,
-       * add a reference here, so that if someone is messing about with us
-       * via the container interface, we do not end up with a dangling pointer.
-       * We will release it in dispose().
-       */
-      g_object_ref (priv->surface);
-
       g_signal_connect_object (window, "notify::decorated",
                                G_CALLBACK (window_decorated_notify), self, 0);
       g_signal_connect_object (window, "notify::appears-focused",
diff --git a/src/wayland/meta-wayland-surface.c b/src/wayland/meta-wayland-surface.c
index 39dc2bbb7..b2c4c907d 100644
--- a/src/wayland/meta-wayland-surface.c
+++ b/src/wayland/meta-wayland-surface.c
@@ -450,6 +450,7 @@ meta_wayland_surface_create (MetaWaylandCompositor *compositor,
     surface_handle_pending_buffer_destroy;
   wl_list_init (&surface->pending.frame_callback_list);
 
+  surface->surface_actor = meta_surface_actor_new ();
   return surface;
 }