diff --git a/js/ui/focusCaretTracker.js b/js/ui/focusCaretTracker.js
index 5af8d37db..beaf2de0f 100644
--- a/js/ui/focusCaretTracker.js
+++ b/js/ui/focusCaretTracker.js
@@ -34,7 +34,11 @@ const FocusCaretTracker = new Lang.Class({
     _init: function() {
         Atspi.init();
         Atspi.set_timeout(250, 250);
+
         this._atspiListener = Atspi.EventListener.new(Lang.bind(this, this._onChanged));
+
+        this._focusListenerRegistered = false;
+        this._caretListenerRegistered = false;
     },
 
     _onChanged: function(event) {
@@ -45,21 +49,39 @@ const FocusCaretTracker = new Lang.Class({
     },
 
     registerFocusListener: function() {
-        return this._atspiListener.register(STATECHANGED + ':focused') &&
-               this._atspiListener.register(STATECHANGED + ':selected');
+        if (this._focusListenerRegistered)
+            return;
+
+        // Ignore the return value, we get an exception if they fail
+        // And they should never fail
+        this._atspiListener.register(STATECHANGED + ':focused');
+        this._atspiListener.register(STATECHANGED + ':selected');
+        this._focusListenerRegistered = true;
     },
 
     registerCaretListener: function() {
-        return this._atspiListener.register(CARETMOVED);
+        if (this._caretListenerRegistered)
+            return;
+
+        this._atspiListener.register(CARETMOVED);
+        this._caretListenerRegistered = true;
     },
 
     deregisterFocusListener: function() {
-        return this._atspiListener.deregister(STATECHANGED + ':focused') &&
-               this._atspiListener.deregister(STATECHANGED + ':selected');
+        if (!this._focusListenerRegistered)
+            return;
+
+        this._atspiListener.deregister(STATECHANGED + ':focused');
+        this._atspiListener.deregister(STATECHANGED + ':selected');
+        this._focusListenerRegistered = false;
     },
 
     deregisterCaretListener: function() {
-        return this._atspiListener.deregister(CARETMOVED);
+        if (!this._caretListenerRegistered)
+            return;
+
+        this._atspiListener.deregister(CARETMOVED);
+        this._caretListenerRegistered = false;
     }
 });
 Signals.addSignalMethods(FocusCaretTracker.prototype);
diff --git a/js/ui/magnifier.js b/js/ui/magnifier.js
index b02b1146c..f64ac3b99 100644
--- a/js/ui/magnifier.js
+++ b/js/ui/magnifier.js
@@ -766,6 +766,9 @@ const ZoomRegion = new Lang.Class({
         } else {
             this._destroyActors();
         }
+
+        this._syncCaretTracking();
+        this._syncFocusTracking();
     },
 
     /**
@@ -825,10 +828,7 @@ const ZoomRegion = new Lang.Class({
      */
     setFocusTrackingMode: function(mode) {
         this._focusTrackingMode = mode;
-        if (this._focusTrackingMode == GDesktopEnums.MagnifierFocusTrackingMode.NONE)
-            this._focusCaretTracker.deregisterFocusListener();
-        else
-            this._focusCaretTracker.registerFocusListener();
+        this._syncFocusTracking();
     },
 
     /**
@@ -837,10 +837,27 @@ const ZoomRegion = new Lang.Class({
      */
     setCaretTrackingMode: function(mode) {
         this._caretTrackingMode = mode;
-        if (this._caretTrackingMode == GDesktopEnums.MagnifierCaretTrackingMode.NONE)
-            this._focusCaretTracker.deregisterCaretListener();
+        this._syncCaretTracking();
+    },
+
+    _syncFocusTracking: function() {
+        let enabled = this._focusTrackingMode != GDesktopEnums.MagnifierFocusTrackingMode.NONE &&
+            this.isActive();
+
+        if (enabled)
+            this._focusCaretTracker.registerFocusListener();
         else
+            this._focusCaretTracker.deregisterFocusListener();
+    },
+
+    _syncCaretTracking: function() {
+        let enabled = this._caretTrackingMode != GDesktopEnums.MagnifierCaretTrackingMode.NONE &&
+            this.isActive();
+
+        if (enabled)
             this._focusCaretTracker.registerCaretListener();
+        else
+            this._focusCaretTracker.deregisterCaretListener();
     },
 
     /**