diff --git a/meson.build b/meson.build
index db8fb991a..573ddb773 100644
--- a/meson.build
+++ b/meson.build
@@ -106,8 +106,6 @@ mutter_installed_tests_libexecdir = join_paths(
 
 m_dep = cc.find_library('m', required: true)
 graphene_dep = dependency('graphene-gobject-1.0', version: graphene_req)
-gtk3_dep = dependency('gtk+-3.0', version: gtk3_req)
-gtk4_dep = dependency('gtk4', version: gtk4_req)
 gdk_pixbuf_dep = dependency('gdk-pixbuf-2.0')
 pango_dep = dependency('pango', version: pango_req)
 cairo_dep = dependency('cairo', version: cairo_req)
@@ -146,6 +144,9 @@ if not have_wayland and not have_x11
 endif
 
 if have_x11_client
+  gtk3_dep = dependency('gtk+-3.0', version: gtk3_req)
+  gtk4_dep = dependency('gtk4', version: gtk4_req)
+
   x11_dep = dependency('x11', version: x11_req)
   xcomposite_dep = dependency('xcomposite', version: xcomposite_req)
   xcursor_dep = dependency('xcursor')
diff --git a/src/compositor/plugins/meson.build b/src/compositor/plugins/meson.build
index 9dd06814d..fadb9fadc 100644
--- a/src/compositor/plugins/meson.build
+++ b/src/compositor/plugins/meson.build
@@ -10,7 +10,6 @@ default_plugin = shared_module('default',
   c_args: default_plugin_c_args,
   dependencies: [
     glib_dep,
-    gtk3_dep,
     json_glib_dep,
     gsettings_desktop_schemas_dep,
     libmutter_clutter_dep,
diff --git a/src/meson.build b/src/meson.build
index 411c3356a..1231d5544 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -17,7 +17,6 @@ mutter_pkg_deps = [
   gio_unix_dep,
   glib_dep,
   gsettings_desktop_schemas_dep,
-  gtk3_dep,
   pango_dep,
 ]
 
@@ -91,6 +90,12 @@ if have_introspection
   ]
 endif
 
+if have_x11_client
+  mutter_pkg_deps += [
+    gtk3_dep,
+  ]
+endif
+
 if have_x11
   mutter_pkg_deps += [
     xfixes_dep,
@@ -1150,8 +1155,6 @@ if have_introspection
     includes: [
       'GObject-2.0',
       'GDesktopEnums-3.0',
-      'Gdk-3.0',
-      'Gtk-3.0',
       'xlib-2.0',
       'xfixes-4.0',
       libmutter_cogl_gir[0],
@@ -1185,7 +1188,9 @@ pkg.generate(libmutter,
 )
 
 subdir('compositor/plugins')
-subdir('frames')
+if have_x11_client
+  subdir('frames')
+endif
 
 if have_core_tests
   subdir('tests')
diff --git a/src/x11/meta-x11-display.c b/src/x11/meta-x11-display.c
index 3d6cb04f9..1d94fc138 100644
--- a/src/x11/meta-x11-display.c
+++ b/src/x11/meta-x11-display.c
@@ -34,6 +34,7 @@
 #include "x11/meta-x11-display-private.h"
 
 #include <gdk/gdk.h>
+#include <gtk/gtk.h>
 #include <gdk/gdkx.h>
 #include <stdlib.h>
 #include <string.h>