diff --git a/src/backends/meta-backend-private.h b/src/backends/meta-backend-private.h
index 416475bff..132646068 100644
--- a/src/backends/meta-backend-private.h
+++ b/src/backends/meta-backend-private.h
@@ -121,8 +121,8 @@ void meta_init_backend (GType backend_gtype);
 
 ClutterBackend * meta_backend_get_clutter_backend (MetaBackend *backend);
 
-MetaIdleMonitor * meta_backend_get_idle_monitor (MetaBackend *backend,
-                                                 int          device_id);
+MetaIdleMonitor * meta_backend_get_idle_monitor (MetaBackend        *backend,
+                                                 ClutterInputDevice *device);
 void meta_backend_foreach_device_monitor (MetaBackend *backend,
                                           GFunc        func,
                                           gpointer     user_data);
diff --git a/src/backends/meta-backend.c b/src/backends/meta-backend.c
index f5d915766..2d1519787 100644
--- a/src/backends/meta-backend.c
+++ b/src/backends/meta-backend.c
@@ -81,8 +81,6 @@
 #include "backends/native/meta-backend-native.h"
 #endif
 
-#define META_IDLE_MONITOR_CORE_DEVICE 0
-
 enum
 {
   KEYMAP_CHANGED,
@@ -311,44 +309,42 @@ meta_backend_foreach_device_monitor (MetaBackend *backend,
 }
 
 static MetaIdleMonitor *
-meta_backend_create_idle_monitor (MetaBackend *backend,
-                                  int          device_id)
+meta_backend_create_idle_monitor (MetaBackend        *backend,
+                                  ClutterInputDevice *device)
 {
   return g_object_new (META_TYPE_IDLE_MONITOR,
-                       "device-id", device_id,
+                       "device", device,
                        NULL);
 }
 
 static void
-create_device_monitor (MetaBackend *backend,
-                       int          device_id)
+create_device_monitor (MetaBackend        *backend,
+                       ClutterInputDevice *device)
 {
   MetaBackendPrivate *priv = meta_backend_get_instance_private (backend);
   MetaIdleMonitor *idle_monitor;
 
-  g_assert (g_hash_table_lookup (priv->device_monitors, &device_id) == NULL);
+  if (g_hash_table_contains (priv->device_monitors, device))
+    return;
 
-  idle_monitor = meta_backend_create_idle_monitor (backend, device_id);
-  g_hash_table_insert (priv->device_monitors, &idle_monitor->device_id, idle_monitor);
+  idle_monitor = meta_backend_create_idle_monitor (backend, device);
+  g_hash_table_insert (priv->device_monitors, device, idle_monitor);
 }
 
 static void
-destroy_device_monitor (MetaBackend *backend,
-                        int          device_id)
+destroy_device_monitor (MetaBackend        *backend,
+                        ClutterInputDevice *device)
 {
   MetaBackendPrivate *priv = meta_backend_get_instance_private (backend);
 
-  g_hash_table_remove (priv->device_monitors, &device_id);
+  g_hash_table_remove (priv->device_monitors, device);
 }
 
 static void
 meta_backend_monitor_device (MetaBackend        *backend,
                              ClutterInputDevice *device)
 {
-  int device_id;
-
-  device_id = clutter_input_device_get_device_id (device);
-  create_device_monitor (backend, device_id);
+  create_device_monitor (backend, device);
 }
 
 static void
@@ -357,9 +353,8 @@ on_device_added (ClutterSeat        *seat,
                  gpointer            user_data)
 {
   MetaBackend *backend = META_BACKEND (user_data);
-  int device_id = clutter_input_device_get_device_id (device);
 
-  create_device_monitor (backend, device_id);
+  create_device_monitor (backend, device);
 }
 
 static inline gboolean
@@ -428,9 +423,8 @@ on_device_removed (ClutterSeat        *seat,
 {
   MetaBackend *backend = META_BACKEND (user_data);
   MetaBackendPrivate *priv = meta_backend_get_instance_private (backend);
-  int device_id = clutter_input_device_get_device_id (device);
 
-  destroy_device_monitor (backend, device_id);
+  destroy_device_monitor (backend, device);
 
   /* If the device the user last interacted goes away, check again pointer
    * visibility.
@@ -467,7 +461,8 @@ create_device_monitors (MetaBackend *backend,
 {
   GList *l, *devices;
 
-  create_device_monitor (backend, META_IDLE_MONITOR_CORE_DEVICE);
+  create_device_monitor (backend, clutter_seat_get_pointer (seat));
+  create_device_monitor (backend, clutter_seat_get_keyboard (seat));
 
   devices = clutter_seat_list_devices (seat);
   for (l = devices; l; l = l->next)
@@ -525,8 +520,7 @@ meta_backend_real_post_init (MetaBackend *backend)
   priv->cursor_renderer = META_BACKEND_GET_CLASS (backend)->create_cursor_renderer (backend);
 
   priv->device_monitors =
-    g_hash_table_new_full (g_int_hash, g_int_equal,
-                           NULL, (GDestroyNotify) g_object_unref);
+    g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify) g_object_unref);
 
   create_device_monitors (backend, seat);
 
@@ -934,12 +928,12 @@ meta_backend_post_init (MetaBackend *backend)
  * meta_backend_get_idle_monitor: (skip)
  */
 MetaIdleMonitor *
-meta_backend_get_idle_monitor (MetaBackend *backend,
-                               int          device_id)
+meta_backend_get_idle_monitor (MetaBackend        *backend,
+                               ClutterInputDevice *device)
 {
   MetaBackendPrivate *priv = meta_backend_get_instance_private (backend);
 
-  return g_hash_table_lookup (priv->device_monitors, &device_id);
+  return g_hash_table_lookup (priv->device_monitors, device);
 }
 
 /**
diff --git a/src/backends/meta-idle-monitor-dbus.c b/src/backends/meta-idle-monitor-dbus.c
index 8fcc2f25c..f7433068e 100644
--- a/src/backends/meta-idle-monitor-dbus.c
+++ b/src/backends/meta-idle-monitor-dbus.c
@@ -220,7 +220,7 @@ on_device_added (ClutterSeat              *seat,
   char *path;
 
   device_id = clutter_input_device_get_device_id (device);
-  monitor = meta_idle_monitor_get_for_device (device_id);
+  monitor = meta_idle_monitor_get_for_device (device);
   path = g_strdup_printf ("/org/gnome/Mutter/IdleMonitor/Device%d", device_id);
 
   create_monitor_skeleton (manager, monitor, path);
diff --git a/src/backends/meta-idle-monitor-private.h b/src/backends/meta-idle-monitor-private.h
index cc08f8c8e..d498f13f9 100644
--- a/src/backends/meta-idle-monitor-private.h
+++ b/src/backends/meta-idle-monitor-private.h
@@ -45,7 +45,7 @@ struct _MetaIdleMonitor
   GDBusProxy *session_proxy;
   gboolean inhibited;
   GHashTable *watches;
-  int device_id;
+  ClutterInputDevice *device;
   guint64 last_event_time;
 };
 
diff --git a/src/backends/meta-idle-monitor.c b/src/backends/meta-idle-monitor.c
index 4304db142..08835a8f4 100644
--- a/src/backends/meta-idle-monitor.c
+++ b/src/backends/meta-idle-monitor.c
@@ -46,7 +46,7 @@ G_STATIC_ASSERT(sizeof(unsigned long) == sizeof(gpointer));
 enum
 {
   PROP_0,
-  PROP_DEVICE_ID,
+  PROP_DEVICE,
   PROP_LAST,
 };
 
@@ -99,8 +99,8 @@ meta_idle_monitor_get_property (GObject    *object,
 
   switch (prop_id)
     {
-    case PROP_DEVICE_ID:
-      g_value_set_int (value, monitor->device_id);
+    case PROP_DEVICE:
+      g_value_set_object (value, monitor->device);
       break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
@@ -117,8 +117,8 @@ meta_idle_monitor_set_property (GObject      *object,
   MetaIdleMonitor *monitor = META_IDLE_MONITOR (object);
   switch (prop_id)
     {
-    case PROP_DEVICE_ID:
-      monitor->device_id = g_value_get_int (value);
+    case PROP_DEVICE:
+      monitor->device = g_value_get_object (value);
       break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
@@ -136,17 +136,18 @@ meta_idle_monitor_class_init (MetaIdleMonitorClass *klass)
   object_class->set_property = meta_idle_monitor_set_property;
 
   /**
-   * MetaIdleMonitor:device_id:
+   * MetaIdleMonitor:device:
    *
    * The device to listen to idletime on.
    */
-  obj_props[PROP_DEVICE_ID] =
-    g_param_spec_int ("device-id",
-                      "Device ID",
-                      "The device to listen to idletime on",
-                      0, 255, 0,
-                      G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY);
-  g_object_class_install_property (object_class, PROP_DEVICE_ID, obj_props[PROP_DEVICE_ID]);
+  obj_props[PROP_DEVICE] =
+    g_param_spec_object ("device",
+                         "Device",
+                         "The device to listen to idletime on",
+                         CLUTTER_TYPE_INPUT_DEVICE,
+                         G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY);
+
+  g_object_class_install_property (object_class, PROP_DEVICE, obj_props[PROP_DEVICE]);
 }
 
 static void
@@ -276,22 +277,25 @@ MetaIdleMonitor *
 meta_idle_monitor_get_core (void)
 {
   MetaBackend *backend = meta_get_backend ();
-  return meta_backend_get_idle_monitor (backend, 0);
+  ClutterBackend *clutter_backend = meta_backend_get_clutter_backend (backend);
+  ClutterSeat *seat = clutter_backend_get_default_seat (clutter_backend);
+
+  return meta_backend_get_idle_monitor (backend, clutter_seat_get_pointer (seat));
 }
 
 /**
  * meta_idle_monitor_get_for_device:
- * @device_id: the device to get the idle time for.
+ * @device: the device to get the idle time for.
  *
  * Returns: (transfer none): a new #MetaIdleMonitor that tracks the
  * device-specific idletime for @device. To track server-global idletime
  * for all devices, use meta_idle_monitor_get_core().
  */
 MetaIdleMonitor *
-meta_idle_monitor_get_for_device (int device_id)
+meta_idle_monitor_get_for_device (ClutterInputDevice *device)
 {
   MetaBackend *backend = meta_get_backend ();
-  return meta_backend_get_idle_monitor (backend, device_id);
+  return meta_backend_get_idle_monitor (backend, device);
 }
 
 static guint32
diff --git a/src/backends/x11/meta-backend-x11.c b/src/backends/x11/meta-backend-x11.c
index b5d051adc..32c3b71b5 100644
--- a/src/backends/x11/meta-backend-x11.c
+++ b/src/backends/x11/meta-backend-x11.c
@@ -157,6 +157,9 @@ handle_alarm_notify (MetaBackend           *backend,
   MetaBackendX11Private *priv = meta_backend_x11_get_instance_private (x11);
   MetaIdleMonitor *idle_monitor;
   XSyncAlarmAttributes attr;
+  ClutterBackend *clutter_backend;
+  ClutterSeat *seat;
+  ClutterInputDevice *pointer;
 
   if (alarm_event->state != XSyncAlarmActive ||
       alarm_event->alarm != priv->user_active_alarm)
@@ -166,7 +169,10 @@ handle_alarm_notify (MetaBackend           *backend,
   XSyncChangeAlarm (priv->xdisplay, priv->user_active_alarm,
                     XSyncCAEvents, &attr);
 
-  idle_monitor = meta_backend_get_idle_monitor (backend, 0);
+  clutter_backend = meta_backend_get_clutter_backend (backend);
+  seat = clutter_backend_get_default_seat (clutter_backend);
+  pointer = clutter_seat_get_pointer (seat);
+  idle_monitor = meta_backend_get_idle_monitor (backend, pointer);
   meta_idle_monitor_reset_idletime (idle_monitor);
 }
 
diff --git a/src/core/events.c b/src/core/events.c
index 65a7792ee..6b57f4f5f 100644
--- a/src/core/events.c
+++ b/src/core/events.c
@@ -103,7 +103,6 @@ handle_idletime_for_event (const ClutterEvent *event)
 {
   ClutterInputDevice *device, *source_device;
   MetaIdleMonitor *core_monitor, *device_monitor;
-  int device_id;
 
   device = clutter_event_get_device (event);
   if (device == NULL)
@@ -118,10 +117,8 @@ handle_idletime_for_event (const ClutterEvent *event)
       event->type == CLUTTER_DELETE)
     return;
 
-  device_id = clutter_input_device_get_device_id (device);
-
   core_monitor = meta_idle_monitor_get_core ();
-  device_monitor = meta_idle_monitor_get_for_device (device_id);
+  device_monitor = meta_idle_monitor_get_for_device (device);
 
   meta_idle_monitor_reset_idletime (core_monitor);
   meta_idle_monitor_reset_idletime (device_monitor);
@@ -129,8 +126,7 @@ handle_idletime_for_event (const ClutterEvent *event)
   source_device = clutter_event_get_source_device (event);
   if (source_device != device)
     {
-      device_id = clutter_input_device_get_device_id (device);
-      device_monitor = meta_idle_monitor_get_for_device (device_id);
+      device_monitor = meta_idle_monitor_get_for_device (source_device);
       meta_idle_monitor_reset_idletime (device_monitor);
     }
 }
diff --git a/src/meta/meta-idle-monitor.h b/src/meta/meta-idle-monitor.h
index 6ef4eab00..51902a17f 100644
--- a/src/meta/meta-idle-monitor.h
+++ b/src/meta/meta-idle-monitor.h
@@ -44,7 +44,7 @@ META_EXPORT
 MetaIdleMonitor *meta_idle_monitor_get_core (void);
 
 META_EXPORT
-MetaIdleMonitor *meta_idle_monitor_get_for_device (int device_id);
+MetaIdleMonitor *meta_idle_monitor_get_for_device (ClutterInputDevice *device);
 
 META_EXPORT
 guint         meta_idle_monitor_add_idle_watch        (MetaIdleMonitor          *monitor,