diff --git a/src/wayland/meta-wayland-data-device-primary-legacy.c b/src/wayland/meta-wayland-data-device-primary-legacy.c
index 26d432357..684247e04 100644
--- a/src/wayland/meta-wayland-data-device-primary-legacy.c
+++ b/src/wayland/meta-wayland-data-device-primary-legacy.c
@@ -172,7 +172,11 @@ primary_device_set_selection (struct wl_client   *client,
 
   if (wl_resource_get_client (resource) !=
       meta_wayland_keyboard_get_focus_client (seat->keyboard))
-    return;
+    {
+      if (source)
+        meta_wayland_data_source_cancel (source);
+      return;
+    }
 
   meta_wayland_data_device_primary_legacy_set_selection (data_device, source, serial);
 }
diff --git a/src/wayland/meta-wayland-data-device-primary.c b/src/wayland/meta-wayland-data-device-primary.c
index ef1c5d964..9592e8f77 100644
--- a/src/wayland/meta-wayland-data-device-primary.c
+++ b/src/wayland/meta-wayland-data-device-primary.c
@@ -172,7 +172,11 @@ primary_device_set_selection (struct wl_client   *client,
 
   if (wl_resource_get_client (resource) !=
       meta_wayland_keyboard_get_focus_client (seat->keyboard))
-    return;
+    {
+      if (source)
+        meta_wayland_data_source_cancel (source);
+      return;
+    }
 
   meta_wayland_data_device_primary_set_selection (data_device, source, serial);
 }
diff --git a/src/wayland/meta-wayland-data-device.c b/src/wayland/meta-wayland-data-device.c
index 3e54cfa3c..61b4435d4 100644
--- a/src/wayland/meta-wayland-data-device.c
+++ b/src/wayland/meta-wayland-data-device.c
@@ -938,7 +938,11 @@ data_device_set_selection (struct wl_client *client,
 
   if (wl_resource_get_client (resource) !=
       meta_wayland_keyboard_get_focus_client (seat->keyboard))
-    return;
+    {
+      if (source)
+        meta_wayland_data_source_cancel (source);
+      return;
+    }
 
   /* FIXME: Store serial and check against incoming serial here. */
   meta_wayland_data_device_set_selection (data_device, source, serial);