diff --git a/js/ui/keyboard.js b/js/ui/keyboard.js
index 8dc1b306f..c4ac72d1d 100644
--- a/js/ui/keyboard.js
+++ b/js/ui/keyboard.js
@@ -1470,7 +1470,12 @@ var Keyboard = class Keyboard {
     }
 
     _onKeyboardGroupsChanged(keyboard) {
-        this._groups = [];
+        let nonGroupActors = [this._emojiSelection.actor, this._keypad.actor];
+        this._aspectContainer.get_children().filter(c => !nonGroupActors.includes(c)).forEach(c => {
+            c.destroy();
+        });
+
+        this._groups = {};
         this._onGroupChanged();
     }