diff --git a/src/tests/meson.build b/src/tests/meson.build
index d6ef39dca..621d2778a 100644
--- a/src/tests/meson.build
+++ b/src/tests/meson.build
@@ -382,13 +382,6 @@ if have_native_tests
         wayland_test_utils,
       ],
     },
-    {
-      'name': 'xwayland',
-      'suite': 'wayland',
-      'sources': [
-        'xwayland-tests.c',
-      ],
-    },
     {
       'name': 'wayland-fullscreen',
       'suite': 'wayland',
@@ -398,6 +391,17 @@ if have_native_tests
       ],
     },
   ]
+  if have_xwayland
+    test_cases += [
+      {
+        'name': 'xwayland',
+        'suite': 'wayland',
+        'sources': [
+          'xwayland-tests.c',
+        ],
+      },
+    ]
+  endif
 
   foreach test_case: test_cases
     test_executable = executable('mutter-' + test_case['name'],
diff --git a/src/tests/meta-context-test.c b/src/tests/meta-context-test.c
index 95aa625ce..97dd44566 100644
--- a/src/tests/meta-context-test.c
+++ b/src/tests/meta-context-test.c
@@ -83,7 +83,9 @@ meta_context_test_configure (MetaContext   *context,
     meta_ensure_test_client_path (*argc, *argv);
 
   meta_wayland_override_display_name ("mutter-test-display");
+#ifdef HAVE_XWAYLAND
   meta_xwayland_override_display_number (512);
+#endif
 
   plugin_name = g_getenv ("MUTTER_TEST_PLUGIN_PATH");
   if (!plugin_name)