diff --git a/src/backends/native/meta-input-device-native.c b/src/backends/native/meta-input-device-native.c
index df7545ede..04b34c9fe 100644
--- a/src/backends/native/meta-input-device-native.c
+++ b/src/backends/native/meta-input-device-native.c
@@ -1592,3 +1592,9 @@ meta_input_device_native_get_coords_in_impl (MetaInputDeviceNative *device_nativ
   if (y)
     *y = device_native->pointer_y;
 }
+
+void
+meta_input_device_native_detach_libinput_in_impl (MetaInputDeviceNative *device_native)
+{
+  g_clear_pointer (&device_native->libinput_device, libinput_device_unref);
+}
diff --git a/src/backends/native/meta-input-device-native.h b/src/backends/native/meta-input-device-native.h
index 841abf1e5..88af07c43 100644
--- a/src/backends/native/meta-input-device-native.h
+++ b/src/backends/native/meta-input-device-native.h
@@ -157,5 +157,6 @@ void                     meta_input_device_native_get_coords_in_impl (MetaInputD
                                                                       float                 *y);
 gboolean                 meta_input_device_native_process_kbd_a11y_event_in_impl (ClutterInputDevice *device,
                                                                                   ClutterEvent       *event);
+void                     meta_input_device_native_detach_libinput_in_impl (MetaInputDeviceNative *device_native);
 
 #endif /* META_INPUT_DEVICE_NATIVE_H */
diff --git a/src/backends/native/meta-seat-impl.c b/src/backends/native/meta-seat-impl.c
index 09bef5e8b..904a9079f 100644
--- a/src/backends/native/meta-seat-impl.c
+++ b/src/backends/native/meta-seat-impl.c
@@ -1602,6 +1602,8 @@ evdev_remove_device (MetaSeatImpl          *seat_impl,
   if (seat_impl->repeat_source && seat_impl->repeat_device == device)
     meta_seat_impl_clear_repeat_source (seat_impl);
 
+  meta_input_device_native_detach_libinput_in_impl (device_native);
+
   g_object_unref (device);
 }
 
@@ -1633,9 +1635,9 @@ process_base_event (MetaSeatImpl          *seat_impl,
       device = libinput_device_get_user_data (libinput_device);
       device_event = clutter_event_new (CLUTTER_DEVICE_REMOVED);
       clutter_event_set_device (device_event, device);
+      meta_input_settings_remove_device (input_settings, device);
       evdev_remove_device (seat_impl,
                            META_INPUT_DEVICE_NATIVE (device));
-      meta_input_settings_remove_device (input_settings, device);
       break;
 
     default: