diff --git a/src/shell-app.c b/src/shell-app.c
index d52e0c374..53d75ee15 100644
--- a/src/shell-app.c
+++ b/src/shell-app.c
@@ -30,9 +30,6 @@ typedef enum {
 typedef struct {
   guint refcount;
 
-  /* Last time the user interacted with any of this application's windows */
-  guint32 last_user_time;
-
   /* Signal connection to dirty window sort list on workspace changes */
   guint workspace_switch_id;
 
@@ -509,15 +506,6 @@ shell_app_activate_window (ShellApp     *app,
         window = most_recent_transient;
 
 
-      if (!shell_window_tracker_is_window_interesting (window))
-        {
-          /* We won't get notify::user-time signals for uninteresting windows,
-           * which means that an app's last_user_time won't get updated.
-           * Update it here instead.
-           */
-          app->running_state->last_user_time = timestamp;
-        }
-
       if (active != workspace)
         meta_workspace_activate_with_focus (workspace, window, timestamp);
       else
@@ -752,6 +740,23 @@ shell_app_is_on_workspace (ShellApp *app,
   return FALSE;
 }
 
+static int
+shell_app_get_last_user_time (ShellApp *app)
+{
+  GSList *iter;
+  int last_user_time;
+
+  last_user_time = 0;
+
+  if (app->running_state != NULL)
+    {
+      for (iter = app->running_state->windows; iter; iter = iter->next)
+        last_user_time = MAX (last_user_time, meta_window_get_user_time (iter->data));
+    }
+
+  return last_user_time;
+}
+
 /**
  * shell_app_compare:
  * @app:
@@ -791,7 +796,8 @@ shell_app_compare (ShellApp *app,
         return -1;
       else if (!app->running_state->windows && other->running_state->windows)
         return 1;
-      return other->running_state->last_user_time - app->running_state->last_user_time;
+
+      return shell_app_get_last_user_time (other) - shell_app_get_last_user_time (app);
     }
 
   return 0;
@@ -871,8 +877,6 @@ shell_app_on_user_time_changed (MetaWindow *window,
 {
   g_assert (app->running_state != NULL);
 
-  app->running_state->last_user_time = meta_window_get_user_time (window);
-
   /* Ideally we don't want to emit windows-changed if the sort order
    * isn't actually changing. This check catches most of those.
    */
@@ -903,8 +907,6 @@ void
 _shell_app_add_window (ShellApp        *app,
                        MetaWindow      *window)
 {
-  guint32 user_time;
-
   if (app->running_state && g_slist_find (app->running_state->windows, window))
     return;
 
@@ -918,10 +920,6 @@ _shell_app_add_window (ShellApp        *app,
   g_signal_connect (window, "unmanaged", G_CALLBACK(shell_app_on_unmanaged), app);
   g_signal_connect (window, "notify::user-time", G_CALLBACK(shell_app_on_user_time_changed), app);
 
-  user_time = meta_window_get_user_time (window);
-  if (user_time > app->running_state->last_user_time)
-    app->running_state->last_user_time = user_time;
-
   if (app->state != SHELL_APP_STATE_STARTING)
     shell_app_state_transition (app, SHELL_APP_STATE_RUNNING);