keymap/x11: Consider the out of range group action

Some keys, such as enter or backspace are only bound to a single group,
even if multiple groups are configured. Because the code was previously
only looking for keysyms in the same group as the current one, no
matching keycodes for these would be found if the current group is not
the first group. This was causing those keys to not work on the X11 OSK.

To fix this use the correct action to convert an out of range group for
that key according to its group_info field.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1955>

src/backends/x11/meta-keymap-x11.c

@@ -704,6 +704,42 @@ meta_keymap_x11_get_is_modifier (MetaKeymapX11 *keymap,
   return FALSE;
 }
 
+static gboolean
+matches_group (XkbDescRec *xkb,
+               uint32_t    keycode,
+               uint32_t    group,
+               uint32_t    target_group)
+{
+  unsigned char group_info = XkbKeyGroupInfo (xkb, keycode);
+  unsigned char action = XkbOutOfRangeGroupAction (group_info);
+  unsigned char num_groups = XkbNumGroups (group_info);
+
+  if (num_groups == 0)
+    return FALSE;
+
+  if (target_group >= num_groups)
+    {
+      switch (action)
+        {
+        case XkbRedirectIntoRange:
+          target_group = XkbOutOfRangeGroupNumber (group_info);
+          if (target_group >= num_groups)
+            target_group = 0;
+          break;
+
+        case XkbClampIntoRange:
+          target_group = num_groups - 1;
+          break;
+
+        default:
+          target_group %= num_groups;
+          break;
+        }
+    }
+
+  return group == target_group;
+}
+
 static gboolean
 meta_keymap_x11_get_entry_for_keyval (MetaKeymapX11     *keymap_x11,
                                       uint32_t           keyval,
@@ -734,7 +770,8 @@ meta_keymap_x11_get_entry_for_keyval (MetaKeymapX11     *keymap_x11,
             {
               g_assert (i == (group * max_shift_levels + level));
 
-              if (entry[i] == keyval && group == target_group)
+              if (entry[i] == keyval &&
+                  matches_group (xkb, keycode, group, target_group))
                 {
                   key->keycode = keycode;
                   key->group = group;