Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

wayland support (#1910)

r33drichardsafe48d4decb5
diff --git a/.github/workflows/nix-wayland.yml b/.github/workflows/nix-wayland.yml
new file mode 100644
index 0000000000..08ea44ef15
--- /dev/null
+++ b/.github/workflows/nix-wayland.yml
@@ -0,0 +1,212 @@
+name: CUA Driver Native-Wayland TDD Suite
+
+# Native-Wayland reproduction of the cua-driver NixOS tests across five desktop
+# sessions (XFCE on labwc/sway, plus KDE and GNOME). These are a TDD
+# RED suite: the driver is X11-only today, so they are EXPECTED TO FAIL until
+# native Wayland support lands. The triggers mirror the X11 nix-build.yml
+# workflow (PRs/pushes touching nix/flake/driver, plus manual dispatch). These
+# jobs are BLOCKING (no continue-on-error): a red cell fails the PR, so the suite
+# can only go green once native Wayland support is implemented.
+# Artifacts (screenshots, GIFs, logs) are uploaded so you can see exactly what
+# each compositor does.
+on:
+ pull_request:
+ paths:
+ - 'nix/**'
+ - 'flake.nix'
+ - 'flake.lock'
+ - 'libs/cua-driver/rust/**'
+ - '.github/workflows/nix-wayland.yml'
+ push:
+ branches: [main]
+ paths:
+ - 'nix/**'
+ - 'flake.nix'
+ - 'flake.lock'
+ - 'libs/cua-driver/rust/**'
+ - '.github/workflows/nix-wayland.yml'
+ workflow_dispatch:
+
+permissions:
+ id-token: write
+ contents: read
+env:
+ AWS_REGION: us-west-2
+ NIX_CACHE_BUCKET: trycua-nix-cache
+ NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key
+
+jobs:
+ wayland-scenarios:
+ name: ${{ matrix.desktop }} / ${{ matrix.scenario }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ desktop: [xfce-labwc, xfce-sway, kde, gnome]
+ scenario:
+ - integration
+ - screenshot
+ - cursor-click-gif
+ - background-terminal-gif
+ - parallel-drag
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Configure AWS Credentials via OIDC
+ uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4
+ with:
+ role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache
+ aws-region: ${{ env.AWS_REGION }}
+
+ - name: Get Nix signing key
+ id: nix-key
+ run: |
+ SECRET=$(aws secretsmanager get-secret-value \
+ --secret-id "${{ env.NIX_CACHE_SECRET }}" \
+ --query 'SecretString' --output text)
+ SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key')
+ echo "::add-mask::$SECRET_KEY"
+ echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec"
+ chmod 600 "${{ runner.temp }}/signing-key.sec"
+ PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key')
+ echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT"
+
+ - name: Setup AWS credentials file for Nix
+ run: |
+ mkdir -p ~/.aws
+ printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \
+ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials
+ chmod 600 ~/.aws/credentials
+ sudo mkdir -p /root/.aws
+ sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \
+ '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials"
+ sudo chmod 600 /root/.aws/credentials
+
+ - name: Install Nix
+ uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+ with:
+ extra_nix_config: |
+ experimental-features = nix-command flakes
+ substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org
+ trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}
+ trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+
+ - name: Run cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }}
+ run: |
+ nix build ".#checks.x86_64-linux.cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }}" \
+ --print-build-logs --show-trace \
+ -o "result-${{ matrix.desktop }}-${{ matrix.scenario }}"
+
+ - name: Collect artifacts
+ if: always()
+ run: |
+ mkdir -p artifacts
+ find -L "result-${{ matrix.desktop }}-${{ matrix.scenario }}/" \
+ \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f \
+ -exec cp {} artifacts/ \; 2>/dev/null || true
+ ls -la artifacts/ 2>/dev/null || echo "No artifacts found"
+
+ - name: Upload artifacts
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }}
+ path: artifacts/*
+ if-no-files-found: warn
+
+ - name: Sign and upload to Nix cache
+ if: always()
+ run: |
+ nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all || true
+ nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L || true
+
+ - name: Cleanup signing key
+ if: always()
+ run: rm -f "${{ runner.temp }}/signing-key.sec"
+
+ wayland-background-gui:
+ name: ${{ matrix.desktop }} / background-gui ${{ matrix.app }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ desktop: [xfce-labwc, xfce-sway, kde, gnome]
+ app: [foot, gtk3-gedit, qt6-kcalc]
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+ - name: Configure AWS Credentials via OIDC
+ uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4
+ with:
+ role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache
+ aws-region: ${{ env.AWS_REGION }}
+
+ - name: Get Nix signing key
+ id: nix-key
+ run: |
+ SECRET=$(aws secretsmanager get-secret-value \
+ --secret-id "${{ env.NIX_CACHE_SECRET }}" \
+ --query 'SecretString' --output text)
+ SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key')
+ echo "::add-mask::$SECRET_KEY"
+ echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec"
+ chmod 600 "${{ runner.temp }}/signing-key.sec"
+ PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key')
+ echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT"
+
+ - name: Setup AWS credentials file for Nix
+ run: |
+ mkdir -p ~/.aws
+ printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \
+ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials
+ chmod 600 ~/.aws/credentials
+ sudo mkdir -p /root/.aws
+ sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \
+ '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials"
+ sudo chmod 600 /root/.aws/credentials
+
+ - name: Install Nix
+ uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+ with:
+ extra_nix_config: |
+ experimental-features = nix-command flakes
+ substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org
+ trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}
+ trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+
+ - name: Run cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }}
+ run: |
+ nix build ".#checks.x86_64-linux.cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }}" \
+ --print-build-logs --show-trace \
+ -o "result-${{ matrix.desktop }}-bg-gui-${{ matrix.app }}"
+
+ - name: Collect artifacts
+ if: always()
+ run: |
+ mkdir -p artifacts
+ find -L "result-${{ matrix.desktop }}-bg-gui-${{ matrix.app }}/" \
+ \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f \
+ -exec cp {} artifacts/ \; 2>/dev/null || true
+ ls -la artifacts/ 2>/dev/null || echo "No artifacts found"
+
+ - name: Upload artifacts
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }}
+ path: artifacts/*
+ if-no-files-found: warn
+
+ - name: Sign and upload to Nix cache
+ if: always()
+ run: |
+ nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all || true
+ nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L || true
+
+ - name: Cleanup signing key
+ if: always()
+ run: rm -f "${{ runner.temp }}/signing-key.sec"
diff --git a/flake.nix b/flake.nix
index dbf8b5ce4c..ce510e3ae5 100644
--- a/flake.nix
+++ b/flake.nix
@@ -156,6 +156,74 @@
"electron-logseq"
]
)
+ )
+ # Native-Wayland TDD matrix — reproduce the cua-driver scenarios on
+ # real, NATIVE Wayland sessions (XFCE on labwc/wayfire/sway, plus KDE
+ # and GNOME). Apps run as Wayland clients and the tests never set
+ # DISPLAY, so the X11-only driver cannot see them: this is a RED suite
+ # specifying native Wayland support. One check per (desktop × scenario)
+ # and per (desktop × background-GUI app). See
+ # nix/cua-driver/tests/wayland/README.md.
+ // pkgs.lib.optionalAttrs (system == "x86_64-linux") (
+ let
+ # NOTE: xfce-wayfire dropped — the wayfire package fails to
+ # build in the current nixpkgs pin (wf-config can't link
+ # -ldoctest), an upstream packaging bug unrelated to cua-driver.
+ # labwc + sway still cover XFCE-on-wlroots.
+ waylandDesktops = [
+ "xfce-labwc"
+ "xfce-sway"
+ "kde"
+ "gnome"
+ ];
+ waylandScenarios = {
+ integration = ./nix/cua-driver/tests/wayland/integration.nix;
+ screenshot = ./nix/cua-driver/tests/wayland/screenshot.nix;
+ cursor-click-gif = ./nix/cua-driver/tests/wayland/cursor-click-gif.nix;
+ background-terminal-gif = ./nix/cua-driver/tests/wayland/background-terminal-gif.nix;
+ parallel-drag = ./nix/cua-driver/tests/wayland/parallel-drag.nix;
+ };
+ waylandBgApps = [
+ "foot"
+ "gtk3-gedit"
+ "qt6-kcalc"
+ ];
+ waylandModule = {
+ imports = [ ./nix/cua-driver/module.nix ];
+ services.cua-driver.package = cuaDriverPackage;
+ };
+ scenarioChecks = pkgs.lib.listToAttrs (
+ pkgs.lib.concatMap (
+ desktop:
+ map (
+ scenario:
+ pkgs.lib.nameValuePair "cua-driver-wayland-${desktop}-${scenario}" (
+ import waylandScenarios.${scenario} {
+ inherit pkgs desktop;
+ inherit (pkgs) lib;
+ cuaDriverModule = waylandModule;
+ }
+ )
+ ) (builtins.attrNames waylandScenarios)
+ ) waylandDesktops
+ );
+ bgGuiChecks = pkgs.lib.listToAttrs (
+ pkgs.lib.concatMap (
+ desktop:
+ map (
+ app:
+ pkgs.lib.nameValuePair "cua-driver-wayland-${desktop}-background-gui-${app}" (
+ import ./nix/cua-driver/tests/wayland/background-gui.nix {
+ inherit pkgs desktop app;
+ inherit (pkgs) lib;
+ cuaDriverModule = waylandModule;
+ }
+ )
+ ) waylandBgApps
+ ) waylandDesktops
+ );
+ in
+ scenarioChecks // bgGuiChecks
);
}
)
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index 40909f85cf..5133676757 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -620,6 +620,12 @@ dependencies = [
"futures-util",
]
+[[package]]
+name = "downcast-rs"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
+
[[package]]
name = "embed-manifest"
version = "1.5.0"
@@ -1630,6 +1636,8 @@ dependencies = [
"tiny-skia",
"tokio",
"tracing",
+ "wayland-client",
+ "wayland-protocols-wlr",
"x11",
"x11rb",
]
@@ -2936,6 +2944,76 @@ dependencies = [
"semver",
]
+[[package]]
+name = "wayland-backend"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d"
+dependencies = [
+ "cc",
+ "downcast-rs",
+ "rustix",
+ "smallvec",
+ "wayland-sys",
+]
+
+[[package]]
+name = "wayland-client"
+version = "0.31.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144"
+dependencies = [
+ "bitflags 2.11.1",
+ "rustix",
+ "wayland-backend",
+ "wayland-scanner",
+]
+
+[[package]]
+name = "wayland-protocols"
+version = "0.32.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f"
+dependencies = [
+ "bitflags 2.11.1",
+ "wayland-backend",
+ "wayland-client",
+ "wayland-scanner",
+]
+
+[[package]]
+name = "wayland-protocols-wlr"
+version = "0.3.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234"
+dependencies = [
+ "bitflags 2.11.1",
+ "wayland-backend",
+ "wayland-client",
+ "wayland-protocols",
+ "wayland-scanner",
+]
+
+[[package]]
+name = "wayland-scanner"
+version = "0.31.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a"
+dependencies = [
+ "proc-macro2",
+ "quick-xml",
+ "quote",
+]
+
+[[package]]
+name = "wayland-sys"
+version = "0.31.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be"
+dependencies = [
+ "pkg-config",
+]
+
[[package]]
name = "webpki-roots"
version = "1.0.7"
diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
index 7d530b4874..77515608c1 100644
--- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
+++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
@@ -33,3 +33,8 @@ atspi = { version = "0.30", features = ["tokio", "zbus"] }
evdev = "0.12"
# Math-expression evaluator for parallel_mouse_drag `fn` paths (y = f(x)).
meval = "0.2"
+# Native Wayland backend (window enumeration via wlr-foreign-toplevel-management;
+# capture/input protocols added as those slices land). Pure-Rust client; used
+# when running under a Wayland compositor with no X11 (no DISPLAY).
+wayland-client = "0.31"
+wayland-protocols-wlr = { version = "0.3", features = ["client"] }
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
index 36fbcf15a9..8fb7dafa2e 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
@@ -37,7 +37,12 @@ pub mod capture;
#[cfg(target_os = "linux")]
pub mod atspi;
+#[cfg(target_os = "linux")]
+pub mod wayland;
+
pub fn register_tools() -> ToolRegistry {
+ #[cfg(target_os = "linux")]
+ wayland::ensure_nested_session();
tools::build_registry(false)
}
@@ -46,6 +51,8 @@ pub fn register_tools() -> ToolRegistry {
/// (pid + window_id required, JPEG @ 85%, text note pointing at pixel
/// tools). See `tools::impl_::ScreenshotCompatTool`.
pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig, compat: bool) -> ToolRegistry {
+ #[cfg(target_os = "linux")]
+ wayland::ensure_nested_session();
if cfg.enabled {
overlay::init(cfg.clone());
overlay::run_on_thread();
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index a89d397ab6..7e10040d7f 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -314,7 +314,7 @@ impl Tool for ListWindowsTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let filter_pid = args.opt_u64("pid").map(|v| v as u32);
- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(filter_pid)).await.unwrap_or_default();
+ let windows = tokio::task::spawn_blocking(move || crate::wayland::list_windows_dispatch(filter_pid)).await.unwrap_or_default();
let mut lines = vec![format!("Found {} windows:", windows.len())];
for w in &windows {
lines.push(format!(" [xid={}] pid={:?} \"{}\" {}x{}+{}+{}",
@@ -386,7 +386,7 @@ impl Tool for GetWindowStateTool {
Vec::new()
};
let screenshot = if do_shot {
- match crate::capture::screenshot_window_bytes(xid) {
+ match crate::wayland::screenshot_dispatch(xid) {
Ok(raw) => {
let orig_w = crate::capture::png_dimensions_pub(&raw).map(|(w, _)| w).unwrap_or(0);
let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;
@@ -943,6 +943,13 @@ impl Tool for ClickTool {
let (xi, yi) = (x as i32, y as i32);
let result = tokio::task::spawn_blocking(move || {
+ // Native Wayland: focus+raise the target toplevel (foreign-toplevel
+ // `activate`). Wayland hides cross-window geometry, so window-local
+ // x/y can't be mapped to a global pointer position; activating the
+ // window the caller targeted is the focus-based equivalent.
+ if crate::wayland::is_wayland() {
+ return crate::wayland::click(xid);
+ }
crate::input::send_click(xid, xi, yi, count, button)
}).await;
match result {
@@ -997,6 +1004,39 @@ impl Tool for TypeTextTool {
}
}
};
+
+ // EIS nested compositor: focus-FREE per-surface typing into window_id
+ // (the target need not be focused). Routed over the inject control socket.
+ if crate::wayland::is_inject_mode() {
+ let text_len = text.chars().count();
+ let text_w = text.clone();
+ let result =
+ tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w)).await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) (focus-free via EIS compositor)."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
+ // Native Wayland: keys go to the *focused* surface (no pid/window
+ // targeting in the protocol). Type via the virtual-keyboard tool; pair
+ // with a prior `click`/`activate` to focus the intended window.
+ if crate::wayland::is_wayland() {
+ let text_len = text.chars().count();
+ let text_w = text.clone();
+ let result =
+ tokio::task::spawn_blocking(move || crate::wayland::type_text(&text_w)).await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) (via Wayland virtual-keyboard)."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
// Pulse the agent cursor onto the field being typed into (when an
// element_index is supplied) so the viewer sees *where* typing happens.
if let Some(idx) = args.opt_u64("element_index") {
@@ -1124,6 +1164,31 @@ impl Tool for PressKeyTool {
}
}
};
+ // EIS nested compositor: focus-free named-key into window_id.
+ if crate::wayland::is_inject_mode() {
+ let key_w = key.clone();
+ let result =
+ tokio::task::spawn_blocking(move || crate::wayland::inject_press_key(xid, &key_w)).await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!("Pressed key '{key}' (focus-free via EIS compositor).")),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
+ // Native Wayland: send the key to the focused surface via virtual-keyboard.
+ if crate::wayland::is_wayland() {
+ let key_w = key.clone();
+ let result = tokio::task::spawn_blocking(move || crate::wayland::press_key(&key_w)).await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Pressed key '{key}' (via Wayland virtual-keyboard)."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
let key_for_task = key.clone();
let result = tokio::task::spawn_blocking(move || {
if mods.is_empty() && key_for_task.eq_ignore_ascii_case("enter") {
@@ -2066,6 +2131,55 @@ pub struct ParallelMouseDragTool {
}
static PMDRAG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
+/// EIS-compositor path for parallel_mouse_drag: build window-local drag paths
+/// and run them as concurrent multi-cursor injections over the control socket.
+/// Coordinates stay window-local (the compositor maps them per app_id), so no
+/// X11 geometry/MPX is needed — the X11 path's hard blocker on Wayland.
+async fn parallel_drag_inject(args: &Value) -> ToolResult {
+ let Some(items) = args.get("drags").and_then(|v| v.as_array()) else {
+ return ToolResult::error("drags[] is required.");
+ };
+ if items.len() < 2 {
+ return ToolResult::error("parallel_mouse_drag requires at least two drag items.");
+ }
+ let mut drags = Vec::with_capacity(items.len());
+ for (i, item) in items.iter().enumerate() {
+ let Some(xid) = item.get("window_id").and_then(|v| v.as_u64()) else {
+ return ToolResult::error("each drag item requires window_id.");
+ };
+ let local: Vec<(f64, f64)> = if let Some(pts) = item.get("path").and_then(|v| v.as_array()) {
+ pts.iter()
+ .filter_map(|p| {
+ let a = p.as_array()?;
+ Some((a.first()?.as_f64()?, a.get(1)?.as_f64()?))
+ })
+ .collect()
+ } else {
+ let g = |k: &str| item.get(k).and_then(|v| v.as_f64());
+ match (g("from_x"), g("from_y"), g("to_x"), g("to_y")) {
+ (Some(fx), Some(fy), Some(tx), Some(ty)) => vec![(fx, fy), (tx, ty)],
+ _ => return ToolResult::error("each drag item requires path[] or from_x/from_y/to_x/to_y."),
+ }
+ };
+ if local.len() < 2 {
+ return ToolResult::error("drag path needs at least 2 points.");
+ }
+ let steps = item.get("steps").and_then(|v| v.as_u64()).unwrap_or(60).clamp(1, 300) as usize;
+ let x_button = parse_mouse_button(item.get("button").and_then(|v| v.as_str()).unwrap_or("left")) as u32;
+ let app = match tokio::task::spawn_blocking(move || crate::wayland::app_id_for_window(xid)).await {
+ Ok(Some(a)) => a,
+ _ => return ToolResult::error(format!("no Wayland app_id for window {xid}")),
+ };
+ drags.push(crate::wayland::InjectDrag { app_id: app, idx: i, x_button, path: local, steps });
+ }
+ let n = drags.len();
+ match tokio::task::spawn_blocking(move || crate::wayland::inject_parallel_drags(&drags)).await {
+ Ok(Ok(())) => ToolResult::text(format!("Ran {n} concurrent drags (multi-cursor via EIS compositor).")),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ }
+}
+
#[async_trait]
impl Tool for ParallelMouseDragTool {
fn def(&self) -> &ToolDef {
@@ -2100,6 +2214,11 @@ impl Tool for ParallelMouseDragTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
+ // EIS nested compositor: run the drags as concurrent multi-cursor
+ // injections (window-local, no X11 MPX/geometry needed).
+ if crate::wayland::is_inject_mode() {
+ return parallel_drag_inject(&args).await;
+ }
match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await {
Ok(Ok(())) => {}
Ok(Err(e)) => return ToolResult::error(e.to_string()),
@@ -2675,14 +2794,19 @@ impl Tool for CheckPermissionsTool {
let atspi_ok = std::env::var("DBUS_SESSION_BUS_ADDRESS").is_ok()
|| std::path::Path::new("/run/user").exists();
+ let wayland_display = std::env::var("WAYLAND_DISPLAY").ok();
let status_text = format!(
- "X11 display: {}\nAT-SPI (D-Bus): {}\nXSendEvent injection: {}",
+ "X11 display: {}\nWayland: {}\nAT-SPI (D-Bus): {}\nXSendEvent injection: {}",
if x11_ok { "✅ connected" } else { "❌ DISPLAY not set or X11 unavailable" },
+ match &wayland_display {
+ Some(s) => format!("✅ native Wayland session (WAYLAND_DISPLAY={s})"),
+ None => "❌ not a Wayland session".to_string(),
+ },
if atspi_ok { "✅ D-Bus session available" } else { "⚠️ D-Bus session not detected" },
if x11_ok { "✅ available" } else { "❌ requires X11" }
);
ToolResult::text(status_text)
- .with_structured(json!({ "x11": x11_ok, "atspi": atspi_ok, "xsend_event": x11_ok }))
+ .with_structured(json!({ "x11": x11_ok, "wayland": wayland_display.is_some(), "atspi": atspi_ok, "xsend_event": x11_ok }))
}
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
new file mode 100644
index 0000000000..c5a2a8979e
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
@@ -0,0 +1,594 @@
+//! Native-Wayland backend.
+//!
+//! Used when running under a Wayland compositor with no X11 (WAYLAND_DISPLAY
+//! set, DISPLAY unset). The first slice enumerates toplevels via
+//! `zwlr_foreign_toplevel_manager_v1` (wlroots: labwc, sway, …) so
+//! `list_windows` works on native Wayland. Capture and input slices land next.
+
+use std::collections::HashMap;
+
+use wayland_client::{
+ event_created_child,
+ protocol::{wl_output::{self, WlOutput}, wl_pointer::ButtonState, wl_registry, wl_seat::WlSeat},
+ Connection, Dispatch, Proxy, QueueHandle,
+};
+use wayland_protocols_wlr::foreign_toplevel::v1::client::{
+ zwlr_foreign_toplevel_handle_v1::{self as ftl_handle, ZwlrForeignToplevelHandleV1},
+ zwlr_foreign_toplevel_manager_v1::{
+ self as ftl_manager, ZwlrForeignToplevelManagerV1, EVT_TOPLEVEL_OPCODE,
+ },
+};
+use wayland_protocols_wlr::virtual_pointer::v1::client::{
+ zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
+ zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
+};
+
+/// Linux evdev BTN_LEFT — the button code the virtual-pointer protocol expects.
+const BTN_LEFT: u32 = 0x110;
+
+use crate::x11::WindowInfo;
+
+/// True when we should drive Wayland rather than X11: a Wayland display is
+/// present and there is no X11 DISPLAY to fall back to.
+pub fn is_wayland() -> bool {
+ std::env::var_os("WAYLAND_DISPLAY").is_some() && std::env::var_os("DISPLAY").is_none()
+}
+
+fn wl_sockets(dir: &str) -> std::collections::HashSet<String> {
+ std::fs::read_dir(dir)
+ .into_iter()
+ .flatten()
+ .flatten()
+ .filter_map(|e| e.file_name().into_string().ok())
+ .filter(|n| n.strip_prefix("wayland-").is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit())))
+ .collect()
+}
+
+/// "Bring your own compositor": if `CUA_WAYLAND_NEST` is set, spawn a private
+/// **headless wlroots compositor** (labwc by default) and point this process —
+/// and therefore every app it launches (`launch_app`), every capture (`grim`),
+/// and all enumeration/injection — at it via `WAYLAND_DISPLAY`. This lets
+/// cua-driver automate apps in its **own** Wayland session on ANY host,
+/// including KDE (kwin) and GNOME (mutter) which expose no client protocols for
+/// this, without ever touching the host compositor or its focus. Idempotent.
+pub fn ensure_nested_session() {
+ use std::sync::OnceLock;
+ static DONE: OnceLock<()> = OnceLock::new();
+ if std::env::var_os("CUA_WAYLAND_NEST").is_none() {
+ return;
+ }
+ DONE.get_or_init(|| {
+ let xdg = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/run/user/0".into());
+ let comp = std::env::var("CUA_WAYLAND_NEST_COMPOSITOR").unwrap_or_else(|_| "labwc".into());
+ let before = wl_sockets(&xdg);
+ let spawned = std::process::Command::new(&comp)
+ .env("WLR_BACKENDS", "headless")
+ .env("WLR_RENDERER", "pixman")
+ .env("WLR_RENDERER_ALLOW_SOFTWARE", "1")
+ .env("WLR_LIBINPUT_NO_DEVICES", "1")
+ .env("WLR_HEADLESS_OUTPUTS", "1")
+ .env_remove("WAYLAND_DISPLAY") // headless: do not nest into the host compositor
+ .env_remove("DISPLAY")
+ .stdin(std::process::Stdio::null())
+ .stdout(std::process::Stdio::null())
+ .stderr(std::process::Stdio::null())
+ .spawn();
+ match spawned {
+ Ok(child) => {
+ std::mem::forget(child); // keep the compositor alive for our lifetime
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);
+ loop {
+ if let Some(sock) = wl_sockets(&xdg).difference(&before).min().cloned() {
+ std::env::set_var("WAYLAND_DISPLAY", &sock);
+ std::env::remove_var("DISPLAY");
+ // Publish the nested socket so external tools (e.g. a
+ // `grim` recorder) can target the same session we drive.
+ let _ = std::fs::write(format!("{xdg}/.cua-nested-display"), &sock);
+ tracing::info!("cua nested compositor '{comp}' up: WAYLAND_DISPLAY={sock}");
+ break;
+ }
+ if std::time::Instant::now() >= deadline {
+ tracing::error!("cua nested compositor '{comp}': no Wayland socket appeared");
+ break;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(200));
+ }
+ }
+ Err(e) => tracing::error!("cua nested compositor '{comp}' spawn failed: {e}"),
+ }
+ });
+}
+
+#[derive(Default)]
+struct Toplevel {
+ title: String,
+ app_id: String,
+ closed: bool,
+}
+
+#[derive(Default)]
+struct State {
+ manager: Option<ZwlrForeignToplevelManagerV1>,
+ toplevels: HashMap<u32, Toplevel>,
+ // Live handles + a seat, kept so `click` can `activate` a target toplevel by
+ // its window_id (foreign-toplevel protocol id) — the focus-based input model.
+ handles: HashMap<u32, ZwlrForeignToplevelHandleV1>,
+ seat: Option<WlSeat>,
+ // Virtual-pointer manager + output dimensions, so `click` can land a real
+ // button press at the output centre (over the just-activated window).
+ vptr_manager: Option<ZwlrVirtualPointerManagerV1>,
+ output_w: u32,
+ output_h: u32,
+}
+
+impl Dispatch<wl_registry::WlRegistry, ()> for State {
+ fn event(
+ state: &mut Self,
+ registry: &wl_registry::WlRegistry,
+ event: wl_registry::Event,
+ _: &(),
+ _: &Connection,
+ qh: &QueueHandle<Self>,
+ ) {
+ if let wl_registry::Event::Global { name, interface, version } = event {
+ if interface == ZwlrForeignToplevelManagerV1::interface().name {
+ let v = version.min(3);
+ state.manager = Some(registry.bind::<ZwlrForeignToplevelManagerV1, _, _>(name, v, qh, ()));
+ } else if interface == WlSeat::interface().name {
+ let v = version.min(7);
+ state.seat = Some(registry.bind::<WlSeat, _, _>(name, v, qh, ()));
+ } else if interface == ZwlrVirtualPointerManagerV1::interface().name {
+ state.vptr_manager =
+ Some(registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(name, version.min(2), qh, ()));
+ } else if interface == WlOutput::interface().name {
+ registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ());
+ }
+ }
+ }
+}
+
+impl Dispatch<WlSeat, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &WlSeat,
+ _: wayland_client::protocol::wl_seat::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ // Seat name/capabilities events are irrelevant here — we only need the
+ // seat object to pass to foreign-toplevel `activate`.
+ }
+}
+
+impl Dispatch<WlOutput, ()> for State {
+ fn event(
+ state: &mut Self,
+ _: &WlOutput,
+ event: wl_output::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ // Remember the output resolution so `click` can aim at its centre.
+ if let wl_output::Event::Mode { width, height, .. } = event {
+ state.output_w = width.max(0) as u32;
+ state.output_h = height.max(0) as u32;
+ }
+ }
+}
+
+impl Dispatch<ZwlrVirtualPointerManagerV1, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &ZwlrVirtualPointerManagerV1,
+ _: <ZwlrVirtualPointerManagerV1 as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+impl Dispatch<ZwlrVirtualPointerV1, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &ZwlrVirtualPointerV1,
+ _: <ZwlrVirtualPointerV1 as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+impl Dispatch<ZwlrForeignToplevelManagerV1, ()> for State {
+ fn event(
+ _state: &mut Self,
+ _: &ZwlrForeignToplevelManagerV1,
+ _event: ftl_manager::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ // The `toplevel` event creates a handle object (see event_created_child!);
+ // the handle's own events carry the title/app_id we collect below.
+ }
+
+ event_created_child!(State, ZwlrForeignToplevelManagerV1, [
+ EVT_TOPLEVEL_OPCODE => (ZwlrForeignToplevelHandleV1, ()),
+ ]);
+}
+
+impl Dispatch<ZwlrForeignToplevelHandleV1, ()> for State {
+ fn event(
+ state: &mut Self,
+ handle: &ZwlrForeignToplevelHandleV1,
+ event: ftl_handle::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ let id = handle.id().protocol_id();
+ state.handles.entry(id).or_insert_with(|| handle.clone());
+ let tl = state.toplevels.entry(id).or_default();
+ match event {
+ ftl_handle::Event::Title { title } => tl.title = title,
+ ftl_handle::Event::AppId { app_id } => tl.app_id = app_id,
+ ftl_handle::Event::Closed => tl.closed = true,
+ _ => {}
+ }
+ }
+}
+
+/// Enumerate native Wayland toplevels via wlr-foreign-toplevel-management.
+/// `xid` is the foreign-toplevel handle's protocol id (a stable per-session
+/// window id); pid is unknown (not exposed by the protocol); geometry is 0
+/// (the protocol does not surface position/size). app_id is folded into the
+/// title (`"<title> [<app_id>]"`) so callers matching on either still match.
+pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?; // registry globals -> bind manager
+ if state.manager.is_none() {
+ anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");
+ }
+ // Manager emits a `toplevel` per window; each handle then emits title/app_id
+ // and a `done`. A few roundtrips drain the initial enumeration.
+ for _ in 0..4 {
+ queue.roundtrip(&mut state)?;
+ }
+
+ let mut out = Vec::new();
+ for (id, tl) in &state.toplevels {
+ if tl.closed {
+ continue;
+ }
+ let title = if tl.app_id.is_empty() {
+ tl.title.clone()
+ } else {
+ format!("{} [{}]", tl.title, tl.app_id)
+ };
+ out.push(WindowInfo { xid: *id as u64, pid: None, title, x: 0, y: 0, width: 0, height: 0 });
+ }
+ Ok(out)
+}
+
+/// Capture the Wayland output as PNG bytes via `grim` (the wlroots screenshot
+/// tool — wlr-screencopy under the hood, works on labwc/sway). This mirrors the
+/// X11 backend shelling out to `import`/`xwd`. foreign-toplevel exposes no
+/// per-window geometry, so this captures the whole output; that is sufficient
+/// for `get_window_state`'s vision payload.
+pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> {
+ let out = std::process::Command::new("grim").args(["-t", "png", "-"]).output()?;
+ if !out.status.success() {
+ anyhow::bail!("grim failed: {}", String::from_utf8_lossy(&out.stderr));
+ }
+ if out.stdout.is_empty() {
+ anyhow::bail!("grim produced no output");
+ }
+ Ok(out.stdout)
+}
+
+/// Capture dispatcher: native Wayland (grim) when applicable, else X11.
+pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
+ if is_wayland() {
+ screenshot_bytes()
+ } else {
+ crate::capture::screenshot_window_bytes(xid)
+ }
+}
+
+/// Click a native Wayland toplevel identified by its `window_id` (the
+/// foreign-toplevel protocol id from `list_windows`). Wayland forbids a client
+/// from knowing another window's on-screen geometry, so we cannot map the
+/// caller's window-local x/y to a global pointer position. Instead, the
+/// focused-input model is two steps:
+/// 1. `activate` the target toplevel (foreign-toplevel) — focus+raise it, so
+/// on a stacking layout it fills the output.
+/// 2. land a real virtual-pointer button press at the output centre (now over
+/// the target). This both visually clicks and — crucially on sway, whose
+/// headless seat has no keyboard until a pointer interaction occurs — makes
+/// the compositor route the subsequent virtual-keyboard input to the
+/// target. (labwc routes it from `activate` alone; the centre click is
+/// harmless there since the activated window is centred under the cursor.)
+pub fn click(window_id: u64) -> anyhow::Result<()> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?; // bind manager + seat + vptr + outputs
+ if state.manager.is_none() {
+ anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");
+ }
+ for _ in 0..4 {
+ queue.roundtrip(&mut state)?; // drain toplevel/handle creation + output mode
+ }
+
+ let id = window_id as u32;
+ let handle = state
+ .handles
+ .get(&id)
+ .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {window_id}"))?
+ .clone();
+ let seat = state
+ .seat
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat to activate the window"))?;
+ handle.activate(&seat);
+ queue.roundtrip(&mut state)?; // flush activate so the target is focused/raised
+
+ // Land a button press at the output centre (over the now-focused window).
+ if let Some(mgr) = state.vptr_manager.clone() {
+ let (w, h) = (state.output_w.max(1), state.output_h.max(1));
+ let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());
+ vptr.motion_absolute(0, w / 2, h / 2, w, h);
+ vptr.frame();
+ vptr.button(0, BTN_LEFT, ButtonState::Pressed);
+ vptr.frame();
+ vptr.button(0, BTN_LEFT, ButtonState::Released);
+ vptr.frame();
+ queue.roundtrip(&mut state)?;
+ vptr.destroy();
+ queue.roundtrip(&mut state)?;
+ }
+ Ok(())
+}
+
+/// Type Unicode text into the focused Wayland surface via `wtype` (the
+/// virtual-keyboard tool — `zwp_virtual_keyboard_v1` under the hood; it builds
+/// the xkb keymap and resolves shift levels for us). This mirrors the X11
+/// backend's XSendEvent typing and the capture slice's shell-out to `grim`.
+/// foreign-toplevel exposes no pid and Wayland delivers keys to the *focused*
+/// surface, so this is window_id-free; pair it with `click`/`activate` to put
+/// the intended window in focus first.
+pub fn type_text(text: &str) -> anyhow::Result<()> {
+ if text.is_empty() {
+ return Ok(());
+ }
+ // Lead with a no-op Shift_L tap: on a freshly-focused window under a headless
+ // seat (notably sway), the compositor needs the first virtual-keyboard event
+ // to wire up keyboard routing, and that first key is dropped. Sacrificing a
+ // modifier tap (no character) absorbs the drop so the real text lands intact;
+ // it's harmless where routing is already live (labwc).
+ let out = std::process::Command::new("wtype")
+ .args(["-k", "Shift_L", "--"])
+ .arg(text)
+ .output()?;
+ if !out.status.success() {
+ anyhow::bail!("wtype failed: {}", String::from_utf8_lossy(&out.stderr));
+ }
+ Ok(())
+}
+
+/// Press a single named key into the focused Wayland surface via `wtype -k`.
+pub fn press_key(key: &str) -> anyhow::Result<()> {
+ let keysym = key_to_keysym(key);
+ let out = std::process::Command::new("wtype").args(["-k", &keysym]).output()?;
+ if !out.status.success() {
+ anyhow::bail!("wtype -k {keysym} failed: {}", String::from_utf8_lossy(&out.stderr));
+ }
+ Ok(())
+}
+
+/// Map cua key names to X keysym names that `wtype -k` understands. Unknown
+/// values pass through (single characters and valid keysym names work as-is).
+fn key_to_keysym(key: &str) -> String {
+ match key.to_lowercase().as_str() {
+ "enter" | "return" => "Return",
+ "tab" => "Tab",
+ "esc" | "escape" => "Escape",
+ "space" => "space",
+ "backspace" => "BackSpace",
+ "delete" | "del" => "Delete",
+ "up" => "Up",
+ "down" => "Down",
+ "left" => "Left",
+ "right" => "Right",
+ "home" => "Home",
+ "end" => "End",
+ "pageup" | "page_up" => "Prior",
+ "pagedown" | "page_down" => "Next",
+ _ => return key.to_string(),
+ }
+ .to_string()
+}
+
+// ── EIS nested-compositor injection ────────────────────────────────────────
+//
+// When cua-driver's nested compositor is `cua-compositor` (our patched wlroots,
+// see nix/cua-driver/compositor/), it exposes a line-protocol control socket at
+// $CUA_INJECT_SOCKET for what stock Wayland forbids: focus-FREE per-surface
+// keyboard injection and MULTI-cursor pointer injection, both routed to a target
+// window by its xdg app_id. These helpers speak that protocol.
+
+/// The control socket path, when running against the EIS nested compositor.
+pub fn inject_socket_path() -> Option<String> {
+ std::env::var("CUA_INJECT_SOCKET").ok().filter(|s| !s.is_empty())
+}
+
+/// True when input should be routed through the EIS compositor's control socket
+/// (focus-free / multi-cursor) rather than wtype / virtual-pointer.
+pub fn is_inject_mode() -> bool {
+ inject_socket_path().is_some()
+}
+
+fn inject_send(lines: &[String]) -> anyhow::Result<()> {
+ use std::io::Write;
+ use std::os::unix::net::UnixStream;
+ let path = inject_socket_path().ok_or_else(|| anyhow::anyhow!("CUA_INJECT_SOCKET not set"))?;
+ // The nested compositor may still be starting; retry the connect briefly.
+ let mut stream = None;
+ for _ in 0..60 {
+ match UnixStream::connect(&path) {
+ Ok(s) => { stream = Some(s); break; }
+ Err(_) => std::thread::sleep(std::time::Duration::from_millis(50)),
+ }
+ }
+ let mut s = stream.ok_or_else(|| anyhow::anyhow!("could not connect to inject socket {path}"))?;
+ let mut buf = String::new();
+ for l in lines {
+ buf.push_str(l);
+ buf.push('\n');
+ }
+ s.write_all(buf.as_bytes())?;
+ s.flush()?;
+ // Give the compositor a moment to process before the socket closes.
+ std::thread::sleep(std::time::Duration::from_millis(80));
+ Ok(())
+}
+
+/// Resolve a window_id (foreign-toplevel protocol id) to its xdg app_id by
+/// enumerating toplevels — the inject protocol addresses windows by app_id.
+pub fn app_id_for_window(window_id: u64) -> Option<String> {
+ let conn = Connection::connect_to_env().ok()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+ let mut state = State::default();
+ queue.roundtrip(&mut state).ok()?;
+ for _ in 0..4 {
+ queue.roundtrip(&mut state).ok()?;
+ }
+ state
+ .toplevels
+ .get(&(window_id as u32))
+ .map(|t| t.app_id.clone())
+ .filter(|s| !s.is_empty())
+}
+
+fn to_hex(s: &str) -> String {
+ s.bytes().map(|b| format!("{b:02x}")).collect()
+}
+
+/// Map a cua/X11 mouse-button number to its evdev (wl_pointer) button code.
+fn evdev_button(x_button: u32) -> u32 {
+ match x_button {
+ 3 => 0x111, // BTN_RIGHT
+ 2 => 0x112, // BTN_MIDDLE
+ _ => 0x110, // BTN_LEFT
+ }
+}
+
+/// Focus-free type into the window's surface (no focus change).
+pub fn inject_type_text(window_id: u64, text: &str) -> anyhow::Result<()> {
+ let app = app_id_for_window(window_id)
+ .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;
+ inject_send(&[format!("t {app} {}", to_hex(text))])
+}
+
+/// Focus-free named-key press into the window's surface.
+pub fn inject_press_key(window_id: u64, key: &str) -> anyhow::Result<()> {
+ let app = app_id_for_window(window_id)
+ .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;
+ inject_send(&[format!("k {app} {key}")])
+}
+
+/// A single pointer drag for `inject_parallel_drags`: window-local waypoints,
+/// driven by cursor `idx` so several run concurrently on one window.
+pub struct InjectDrag {
+ pub app_id: String,
+ pub idx: usize,
+ pub x_button: u32,
+ pub path: Vec<(f64, f64)>,
+ pub steps: usize,
+}
+
+fn resample(path: &[(f64, f64)], steps: usize) -> Vec<(f64, f64)> {
+ if path.len() < 2 || steps == 0 {
+ return path.to_vec();
+ }
+ let seglen: Vec<f64> = path
+ .windows(2)
+ .map(|w| ((w[1].0 - w[0].0).powi(2) + (w[1].1 - w[0].1).powi(2)).sqrt())
+ .collect();
+ let total: f64 = seglen.iter().sum();
+ if total == 0.0 {
+ return vec![path[0]; steps + 1];
+ }
+ let mut out = Vec::with_capacity(steps + 1);
+ for s in 0..=steps {
+ let target = total * (s as f64) / (steps as f64);
+ let mut acc = 0.0;
+ let mut pt = path[path.len() - 1];
+ for (i, &l) in seglen.iter().enumerate() {
+ if acc + l >= target || i == seglen.len() - 1 {
+ let f = if l > 0.0 { (target - acc) / l } else { 0.0 };
+ pt = (path[i].0 + (path[i + 1].0 - path[i].0) * f,
+ path[i].1 + (path[i + 1].1 - path[i].1) * f);
+ break;
+ }
+ acc += l;
+ }
+ out.push(pt);
+ }
+ out
+}
+
+/// Run N pointer drags concurrently on their target windows: each cursor presses
+/// at its start, glides through its (interleaved) waypoints, and releases. This
+/// is true multi-cursor — each `idx` is an independent cursor in the compositor.
+pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> {
+ if drags.is_empty() {
+ return Ok(());
+ }
+ let resampled: Vec<Vec<(f64, f64)>> =
+ drags.iter().map(|d| resample(&d.path, d.steps.max(1))).collect();
+ let max_steps = resampled.iter().map(|p| p.len()).max().unwrap_or(0);
+ let mut lines = Vec::new();
+ // Press each cursor at its start point.
+ for (d, pts) in drags.iter().zip(&resampled) {
+ let (x, y) = pts[0];
+ lines.push(format!("m {} {} {x:.1} {y:.1}", d.app_id, d.idx));
+ lines.push(format!("b {} {} {} 1", d.app_id, d.idx, evdev_button(d.x_button)));
+ }
+ // Glide all cursors together, one interleaved step at a time.
+ for s in 1..max_steps {
+ for (d, pts) in drags.iter().zip(&resampled) {
+ let (x, y) = pts[s.min(pts.len() - 1)];
+ lines.push(format!("m {} {} {x:.1} {y:.1}", d.app_id, d.idx));
+ }
+ }
+ // Release each cursor.
+ for (d, _) in drags.iter().zip(&resampled) {
+ lines.push(format!("b {} {} {} 0", d.app_id, d.idx, evdev_button(d.x_button)));
+ }
+ inject_send(&lines)
+}
+
+/// Window-enumeration dispatcher: native Wayland when applicable, else X11.
+pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> {
+ if is_wayland() {
+ match list_windows() {
+ Ok(ws) => return ws, // foreign-toplevel has no pid, so filter_pid can't apply
+ Err(e) => tracing::warn!("wayland list_windows failed, falling back to X11: {e}"),
+ }
+ }
+ crate::x11::list_windows(filter_pid)
+}
diff --git a/nix/cua-driver/compositor/cua_compositor_patch.py b/nix/cua-driver/compositor/cua_compositor_patch.py
new file mode 100644
index 0000000000..e6abdaabd5
--- /dev/null
+++ b/nix/cua-driver/compositor/cua_compositor_patch.py
@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+# Transforms wlroots' upstream `tinywl/tinywl.c` (0.19) into `cua-compositor.c`:
+# a minimal headless wlroots compositor that cua-driver spawns as its nested
+# session (CUA_WAYLAND_NEST_COMPOSITOR=cua-compositor) to perform what stock
+# Wayland forbids a client from doing —
+#
+# * focus-FREE per-surface KEYBOARD injection (type into an unfocused window),
+# * MULTI-cursor pointer injection (N independent cursors on one window),
+#
+# both routed to a target window by its xdg app_id, driven over a tiny line
+# protocol on a unix control socket ($CUA_INJECT_SOCKET). It also exposes
+# foreign-toplevel-management (so cua-driver's list_windows enumerates windows)
+# and screencopy (so grim captures the output) — the same protocols labwc gives
+# the non-nested cells — so the EIS cells keep working end to end.
+#
+# The injection primitives (cua_motion/cua_button/cua_kbd_*) deliver wl_pointer/
+# wl_keyboard straight to a target client's resources, bypassing seat focus —
+# routed by app_id (not positional device index) and transported over a plain
+# socket instead of libei/EIS, since cua owns both ends and the portal/libei
+# layer buys nothing here.
+#
+# Usage: cua_compositor_patch.py <tinywl.c in> <cua-compositor.c out>
+import sys, io
+
+inp = sys.argv[1] if len(sys.argv) > 1 else "tinywl.c"
+out = sys.argv[2] if len(sys.argv) > 2 else "cua-compositor.c"
+src = io.open(inp, encoding="utf-8").read()
+
+# memfd_create + SOCK_CLOEXEC etc. need _GNU_SOURCE before any system header.
+src = "#define _GNU_SOURCE\n" + src
+
+INCLUDES = r"""#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <time.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <linux/input-event-codes.h>
+#include <wayland-server-protocol.h>
+#include <xkbcommon/xkbcommon.h>
+#include <wlr/types/wlr_foreign_toplevel_management_v1.h>
+#include <wlr/types/wlr_screencopy_v1.h>
+#include <wlr/types/wlr_xdg_output_v1.h>
+
+#define CUA_MAXDEV 64
+/* Per-cursor / per-keyboard enter bookkeeping (idx = logical device). */
+struct cua_devstate { struct wlr_surface *entered; };
+static struct cua_devstate cua_ptr[CUA_MAXDEV];
+static struct cua_devstate cua_kbd_state[CUA_MAXDEV];
+static int g_keymap_fd = -1;
+static size_t g_keymap_size = 0;
+static xkb_mod_mask_t g_shift_mask = 1;
+struct cua_keyent { uint32_t keycode; int shift; int valid; };
+static struct cua_keyent g_chartab[128];
+static struct wlr_foreign_toplevel_manager_v1 *g_ftl_mgr = NULL;
+
+"""
+
+# A foreign-toplevel handle pointer on each toplevel (for list_windows).
+STRUCT_FIELD = (
+ "\tstruct wlr_xdg_toplevel *xdg_toplevel;\n"
+ "\tstruct wlr_foreign_toplevel_handle_v1 *ftl;\n"
+)
+
+FUNCS = r"""
+static uint32_t cua_now_ms(void) {
+ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (uint32_t)(ts.tv_sec * 1000 + ts.tv_nsec / 1000000);
+}
+static void cua_pframe(struct wl_resource *res) {
+ if (wl_resource_get_version(res) >= WL_POINTER_FRAME_SINCE_VERSION)
+ wl_pointer_send_frame(res);
+}
+/* Resolve a target window by its xdg app_id. */
+static struct tinywl_toplevel *cua_find_appid(struct tinywl_server *server, const char *app_id) {
+ struct tinywl_toplevel *t;
+ wl_list_for_each(t, &server->toplevels, link) {
+ const char *a = t->xdg_toplevel ? t->xdg_toplevel->app_id : NULL;
+ if (a && strcmp(a, app_id) == 0) return t;
+ }
+ return NULL;
+}
+static void cua_ptr_leave(struct wlr_seat *seat, struct wlr_surface *surf) {
+ if (!surf) return;
+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(seat, wl_resource_get_client(surf->resource));
+ if (!sc) return;
+ uint32_t serial = wlr_seat_client_next_serial(sc);
+ struct wl_resource *res;
+ wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_leave(res, serial, surf->resource); cua_pframe(res); }
+}
+/* Inject pointer motion from logical cursor `idx` into window `t` at window-
+ * local (x,y). enter/leave is tracked per idx, so several idx values can drive
+ * independent cursors against the same or different surfaces. Focus-free: we
+ * write straight to the client's wl_pointer resources, never touching the seat
+ * focus or any real cursor. */
+static void cua_motion(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, double x, double y) {
+ if (!t || idx < 0 || idx >= CUA_MAXDEV) return;
+ struct wlr_surface *surface = t->xdg_toplevel->base->surface;
+ struct wlr_box geo = t->xdg_toplevel->base->geometry;
+ wl_fixed_t sx = wl_fixed_from_double(geo.x + x), sy = wl_fixed_from_double(geo.y + y);
+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource));
+ if (!sc || wl_list_empty(&sc->pointers)) return;
+ struct wl_resource *res;
+ if (cua_ptr[idx].entered != surface) {
+ if (cua_ptr[idx].entered) cua_ptr_leave(server->seat, cua_ptr[idx].entered);
+ uint32_t es = wlr_seat_client_next_serial(sc);
+ wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_enter(res, es, surface->resource, sx, sy); cua_pframe(res); }
+ cua_ptr[idx].entered = surface;
+ }
+ uint32_t tm = cua_now_ms();
+ wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_motion(res, tm, sx, sy); cua_pframe(res); }
+}
+static void cua_button(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, uint32_t button, bool pressed) {
+ if (!t || idx < 0 || idx >= CUA_MAXDEV) return;
+ struct wlr_surface *surface = t->xdg_toplevel->base->surface;
+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource));
+ if (!sc || wl_list_empty(&sc->pointers)) return;
+ uint32_t tm = cua_now_ms(), bs = wlr_seat_client_next_serial(sc);
+ struct wl_resource *res;
+ wl_resource_for_each(res, &sc->pointers) {
+ wl_pointer_send_button(res, bs, tm, button, pressed ? WL_POINTER_BUTTON_STATE_PRESSED : WL_POINTER_BUTTON_STATE_RELEASED);
+ cua_pframe(res);
+ }
+}
+static void cua_init_keymap(void) {
+ struct xkb_context *ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS);
+ struct xkb_rule_names names = {0};
+ struct xkb_keymap *km = xkb_keymap_new_from_names(ctx, &names, XKB_KEYMAP_COMPILE_NO_FLAGS);
+ char *s = xkb_keymap_get_as_string(km, XKB_KEYMAP_FORMAT_TEXT_V1);
+ g_keymap_size = strlen(s) + 1;
+ g_keymap_fd = memfd_create("cua-keymap", MFD_CLOEXEC);
+ if (ftruncate(g_keymap_fd, g_keymap_size) == 0) {
+ void *p = mmap(NULL, g_keymap_size, PROT_READ | PROT_WRITE, MAP_SHARED, g_keymap_fd, 0);
+ memcpy(p, s, g_keymap_size); munmap(p, g_keymap_size);
+ }
+ free(s);
+ /* Build a US-layout char -> (evdev keycode, shift-level) table from the
+ * keymap so the `t` command can type arbitrary ASCII. xkb keycodes are
+ * evdev+8; wl_keyboard.key wants the evdev code. */
+ xkb_mod_index_t shift = xkb_keymap_mod_get_index(km, XKB_MOD_NAME_SHIFT);
+ if (shift != XKB_MOD_INVALID) g_shift_mask = (xkb_mod_mask_t)1 << shift;
+ for (xkb_keycode_t kc = 9; kc < 256; kc++) {
+ for (int lvl = 0; lvl < 2; lvl++) {
+ const xkb_keysym_t *syms;
+ int n = xkb_keymap_key_get_syms_by_level(km, kc, 0, lvl, &syms);
+ if (n != 1) continue;
+ uint32_t cp = xkb_keysym_to_utf32(syms[0]);
+ if (cp > 0 && cp < 128 && !g_chartab[cp].valid) {
+ g_chartab[cp].keycode = (uint32_t)kc - 8;
+ g_chartab[cp].shift = lvl;
+ g_chartab[cp].valid = 1;
+ }
+ }
+ }
+ /* Control characters used by the protocol map to dedicated keys. */
+ g_chartab['\n'] = (struct cua_keyent){ KEY_ENTER, 0, 1 };
+ g_chartab['\t'] = (struct cua_keyent){ KEY_TAB, 0, 1 };
+ xkb_keymap_unref(km); xkb_context_unref(ctx);
+ wlr_log(WLR_INFO, "[cua] xkb keymap + chartab ready (%zu bytes)", g_keymap_size);
+}
+/* Ensure the target client's keyboard has had keymap + enter sent once (focus
+ * free). idx 0 is the typing keyboard. */
+static struct wlr_seat_client *cua_kbd_enter(struct tinywl_server *server, struct tinywl_toplevel *t) {
+ struct wlr_surface *surface = t->xdg_toplevel->base->surface;
+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource));
+ if (!sc || wl_list_empty(&sc->keyboards)) return NULL;
+ if (cua_kbd_state[0].entered != surface) {
+ struct wl_resource *res; struct wl_array keys; wl_array_init(&keys);
+ wl_resource_for_each(res, &sc->keyboards) {
+ wl_keyboard_send_keymap(res, WL_KEYBOARD_KEYMAP_FORMAT_XKB_V1, g_keymap_fd, g_keymap_size);
+ wl_keyboard_send_enter(res, wlr_seat_client_next_serial(sc), surface->resource, &keys);
+ wl_keyboard_send_modifiers(res, wlr_seat_client_next_serial(sc), 0, 0, 0, 0);
+ }
+ wl_array_release(&keys);
+ cua_kbd_state[0].entered = surface;
+ }
+ return sc;
+}
+static void cua_kbd_mods(struct wlr_seat_client *sc, uint32_t depressed) {
+ struct wl_resource *res;
+ wl_resource_for_each(res, &sc->keyboards)
+ wl_keyboard_send_modifiers(res, wlr_seat_client_next_serial(sc), depressed, 0, 0, 0);
+}
+static void cua_kbd_key(struct wlr_seat_client *sc, uint32_t keycode, bool pressed) {
+ uint32_t tm = cua_now_ms();
+ struct wl_resource *res;
+ wl_resource_for_each(res, &sc->keyboards)
+ wl_keyboard_send_key(res, wlr_seat_client_next_serial(sc), tm, keycode,
+ pressed ? WL_KEYBOARD_KEY_STATE_PRESSED : WL_KEYBOARD_KEY_STATE_RELEASED);
+}
+static void cua_type_cp(struct tinywl_server *server, struct tinywl_toplevel *t, uint32_t cp) {
+ if (cp >= 128 || !g_chartab[cp].valid) return;
+ struct wlr_seat_client *sc = cua_kbd_enter(server, t);
+ if (!sc) return;
+ struct cua_keyent e = g_chartab[cp];
+ if (e.shift) cua_kbd_mods(sc, g_shift_mask);
+ cua_kbd_key(sc, e.keycode, true);
+ cua_kbd_key(sc, e.keycode, false);
+ if (e.shift) cua_kbd_mods(sc, 0);
+}
+/* Decode a hex-encoded ASCII string and type it focus-free into `t`. */
+static void cua_type_hex(struct tinywl_server *server, struct tinywl_toplevel *t, const char *hex) {
+ if (!t) return;
+ for (const char *p = hex; p[0] && p[1]; p += 2) {
+ int hi = (p[0] <= '9') ? p[0] - '0' : (p[0] | 0x20) - 'a' + 10;
+ int lo = (p[1] <= '9') ? p[1] - '0' : (p[1] | 0x20) - 'a' + 10;
+ cua_type_cp(server, t, (uint32_t)((hi << 4) | lo));
+ }
+}
+static void cua_key_named(struct tinywl_server *server, struct tinywl_toplevel *t, const char *name) {
+ if (!t) return;
+ uint32_t kc = 0;
+ if (!strcasecmp(name, "enter") || !strcasecmp(name, "return")) kc = KEY_ENTER;
+ else if (!strcasecmp(name, "tab")) kc = KEY_TAB;
+ else if (!strcasecmp(name, "escape") || !strcasecmp(name, "esc")) kc = KEY_ESC;
+ else if (!strcasecmp(name, "backspace")) kc = KEY_BACKSPACE;
+ else if (!strcasecmp(name, "space")) kc = KEY_SPACE;
+ else if (!strcasecmp(name, "up")) kc = KEY_UP;
+ else if (!strcasecmp(name, "down")) kc = KEY_DOWN;
+ else if (!strcasecmp(name, "left")) kc = KEY_LEFT;
+ else if (!strcasecmp(name, "right")) kc = KEY_RIGHT;
+ if (!kc) return;
+ struct wlr_seat_client *sc = cua_kbd_enter(server, t);
+ if (!sc) return;
+ cua_kbd_key(sc, kc, true);
+ cua_kbd_key(sc, kc, false);
+}
+/* ── control socket: one line per command, routed by app_id ───────────────── */
+static void cua_handle_cmd(struct tinywl_server *server, char *line) {
+ char cmd[8], app[128];
+ if (sscanf(line, "%7s", cmd) != 1) return;
+ if (!strcmp(cmd, "m")) {
+ int idx; double x, y;
+ if (sscanf(line, "m %127s %d %lf %lf", app, &idx, &x, &y) == 4)
+ cua_motion(server, cua_find_appid(server, app), idx, x, y);
+ } else if (!strcmp(cmd, "b")) {
+ int idx; unsigned btn, pr;
+ if (sscanf(line, "b %127s %d %u %u", app, &idx, &btn, &pr) == 4)
+ cua_button(server, cua_find_appid(server, app), idx, btn, pr != 0);
+ } else if (!strcmp(cmd, "t")) {
+ char hex[8192];
+ if (sscanf(line, "t %127s %8191s", app, hex) == 2)
+ cua_type_hex(server, cua_find_appid(server, app), hex);
+ } else if (!strcmp(cmd, "k")) {
+ char key[32];
+ if (sscanf(line, "k %127s %31s", app, key) == 2)
+ cua_key_named(server, cua_find_appid(server, app), key);
+ }
+}
+struct cua_conn { struct tinywl_server *server; struct wl_event_source *src; char buf[16384]; size_t len; };
+/* Tear a connection down once: remove its event source (else the loop fires it
+ * again on freed data -> double free), close the fd, free the state. */
+static int cua_conn_drop(struct cua_conn *c, int fd) {
+ if (c->src) wl_event_source_remove(c->src);
+ close(fd); free(c);
+ return 0;
+}
+static int cua_conn_readable(int fd, uint32_t mask, void *data) {
+ struct cua_conn *c = data;
+ if (mask & (WL_EVENT_HANGUP | WL_EVENT_ERROR)) return cua_conn_drop(c, fd);
+ ssize_t n = read(fd, c->buf + c->len, sizeof(c->buf) - c->len - 1);
+ if (n <= 0) return cua_conn_drop(c, fd);
+ c->len += (size_t)n; c->buf[c->len] = 0;
+ char *p = c->buf, *nl;
+ while ((nl = memchr(p, '\n', (size_t)(c->buf + c->len - p)))) {
+ *nl = 0; cua_handle_cmd(c->server, p); p = nl + 1;
+ }
+ size_t rem = (size_t)(c->buf + c->len - p);
+ memmove(c->buf, p, rem); c->len = rem;
+ wl_display_flush_clients(c->server->wl_display);
+ return 0;
+}
+static int cua_listen_cb(int fd, uint32_t mask, void *data) {
+ struct tinywl_server *server = data;
+ int cfd = accept(fd, NULL, NULL);
+ if (cfd < 0) return 0;
+ struct cua_conn *c = calloc(1, sizeof *c); c->server = server;
+ c->src = wl_event_loop_add_fd(wl_display_get_event_loop(server->wl_display), cfd,
+ WL_EVENT_READABLE, cua_conn_readable, c);
+ return 0;
+}
+static void cua_setup_control_socket(struct tinywl_server *server) {
+ const char *path = getenv("CUA_INJECT_SOCKET");
+ if (!path) { wlr_log(WLR_INFO, "[cua] no CUA_INJECT_SOCKET; injection disabled"); return; }
+ int fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);
+ if (fd < 0) { wlr_log(WLR_ERROR, "[cua] socket(): %s", strerror(errno)); return; }
+ struct sockaddr_un addr = {0}; addr.sun_family = AF_UNIX;
+ snprintf(addr.sun_path, sizeof addr.sun_path, "%s", path);
+ unlink(path);
+ if (bind(fd, (struct sockaddr *)&addr, sizeof addr) < 0 || listen(fd, 4) < 0) {
+ wlr_log(WLR_ERROR, "[cua] bind/listen %s: %s", path, strerror(errno)); close(fd); return;
+ }
+ cua_init_keymap();
+ wl_event_loop_add_fd(wl_display_get_event_loop(server->wl_display), fd,
+ WL_EVENT_READABLE, cua_listen_cb, server);
+ wlr_log(WLR_INFO, "[cua] injection control socket on %s", path);
+}
+
+"""
+
+def repl(s, old, new, label, count=1):
+ if s.count(old) != count:
+ sys.stderr.write("ANCHOR FAIL [%s]: found %d (want %d)\n" % (label, s.count(old), count))
+ sys.exit(1)
+ return s.replace(old, new, count)
+
+# 1) includes + globals + a foreign-toplevel handle field on the toplevel.
+src = repl(src, "struct tinywl_server {", INCLUDES + "struct tinywl_server {", "includes")
+src = repl(src,
+ "\tstruct wlr_xdg_toplevel *xdg_toplevel;\n",
+ STRUCT_FIELD, "ftl-field")
+
+# 2) injection helpers + control socket, just before main().
+src = repl(src, "int main(int argc, char *argv[]) {", FUNCS + "int main(int argc, char *argv[]) {", "funcs")
+
+# 3) On map: register a foreign-toplevel handle (title/app_id) for list_windows.
+src = repl(src,
+ "\twl_list_insert(&toplevel->server->toplevels, &toplevel->link);\n\n\tfocus_toplevel(toplevel);",
+ "\twl_list_insert(&toplevel->server->toplevels, &toplevel->link);\n"
+ "\tif (g_ftl_mgr) {\n"
+ "\t\ttoplevel->ftl = wlr_foreign_toplevel_handle_v1_create(g_ftl_mgr);\n"
+ "\t\tif (toplevel->xdg_toplevel->title)\n"
+ "\t\t\twlr_foreign_toplevel_handle_v1_set_title(toplevel->ftl, toplevel->xdg_toplevel->title);\n"
+ "\t\tif (toplevel->xdg_toplevel->app_id)\n"
+ "\t\t\twlr_foreign_toplevel_handle_v1_set_app_id(toplevel->ftl, toplevel->xdg_toplevel->app_id);\n"
+ "\t}\n\n\tfocus_toplevel(toplevel);",
+ "ftl-on-map")
+
+# 4) On unmap: drop the foreign-toplevel handle.
+src = repl(src,
+ "\twl_list_remove(&toplevel->link);\n}",
+ "\tif (toplevel->ftl) { wlr_foreign_toplevel_handle_v1_destroy(toplevel->ftl); toplevel->ftl = NULL; }\n"
+ "\twl_list_remove(&toplevel->link);\n}",
+ "ftl-on-unmap")
+
+# 5) On commit: keep title/app_id fresh (they often arrive after the map).
+src = repl(src,
+ "\t\twlr_xdg_toplevel_set_size(toplevel->xdg_toplevel, 0, 0);\n\t}",
+ "\t\twlr_xdg_toplevel_set_size(toplevel->xdg_toplevel, 0, 0);\n\t}\n"
+ "\tif (toplevel->ftl) {\n"
+ "\t\tif (toplevel->xdg_toplevel->title)\n"
+ "\t\t\twlr_foreign_toplevel_handle_v1_set_title(toplevel->ftl, toplevel->xdg_toplevel->title);\n"
+ "\t\tif (toplevel->xdg_toplevel->app_id)\n"
+ "\t\t\twlr_foreign_toplevel_handle_v1_set_app_id(toplevel->ftl, toplevel->xdg_toplevel->app_id);\n"
+ "\t}",
+ "ftl-on-commit")
+
+# 6) Headless backend has no preferred mode -> set a real custom mode so the
+# scene has a non-zero output for screencopy/grim.
+src = repl(src,
+ "\tstruct wlr_output_mode *mode = wlr_output_preferred_mode(wlr_output);\n\tif (mode != NULL) {\n\t\twlr_output_state_set_mode(&state, mode);\n\t}",
+ "\tstruct wlr_output_mode *mode = wlr_output_preferred_mode(wlr_output);\n\tif (mode != NULL) {\n\t\twlr_output_state_set_mode(&state, mode);\n\t} else {\n"
+ "\t\tint ow = getenv(\"CUA_OUTW\") ? atoi(getenv(\"CUA_OUTW\")) : 1280;\n"
+ "\t\tint oh = getenv(\"CUA_OUTH\") ? atoi(getenv(\"CUA_OUTH\")) : 1024;\n"
+ "\t\twlr_output_state_set_custom_mode(&state, ow, oh, 0);\n\t}",
+ "headless-custom-mode")
+
+# 7) Create the extra managers + the seat caps + the control socket in main.
+src = repl(src,
+ "\tserver.output_layout = wlr_output_layout_create(server.wl_display);",
+ "\tserver.output_layout = wlr_output_layout_create(server.wl_display);\n"
+ "\twlr_xdg_output_manager_v1_create(server.wl_display, server.output_layout);\n"
+ "\twlr_screencopy_manager_v1_create(server.wl_display);\n"
+ "\tg_ftl_mgr = wlr_foreign_toplevel_manager_v1_create(server.wl_display);",
+ "managers")
+src = repl(src,
+ 'server.seat = wlr_seat_create(server.wl_display, "seat0");',
+ 'server.seat = wlr_seat_create(server.wl_display, "seat0");\n'
+ "\twlr_seat_set_capabilities(server.seat, WL_SEAT_CAPABILITY_POINTER | WL_SEAT_CAPABILITY_KEYBOARD);",
+ "seat-caps")
+src = repl(src,
+ "\twl_display_run(server.wl_display);",
+ "\tcua_setup_control_socket(&server);\n\twl_display_run(server.wl_display);",
+ "control-socket-setup")
+
+io.open(out, "w", encoding="utf-8").write(src)
+sys.stderr.write("cua-compositor.c written (%d bytes)\n" % len(src))
diff --git a/nix/cua-driver/compositor/default.nix b/nix/cua-driver/compositor/default.nix
new file mode 100644
index 0000000000..9e7b6afea1
--- /dev/null
+++ b/nix/cua-driver/compositor/default.nix
@@ -0,0 +1,62 @@
+# cua-compositor: a minimal headless wlroots compositor that cua-driver spawns
+# as its nested session (CUA_WAYLAND_NEST_COMPOSITOR=cua-compositor) to perform
+# focus-free keyboard injection and multi-cursor pointer injection via a unix
+# control socket. Built from wlroots' own upstream tinywl example (pinned to the
+# same wlroots_0_19 the patch targets) transformed by cua_compositor_patch.py —
+# so it tracks the exact wlroots API in nixpkgs and needs no vendored C.
+{
+ stdenv,
+ lib,
+ pkg-config,
+ python3,
+ wayland,
+ wayland-scanner,
+ wayland-protocols,
+ wlroots_0_19,
+ libxkbcommon,
+ pixman,
+}:
+
+stdenv.mkDerivation {
+ pname = "cua-compositor";
+ version = "0.1.0";
+
+ src = lib.cleanSource ./.;
+
+ nativeBuildInputs = [
+ pkg-config
+ python3
+ wayland-scanner
+ ];
+ buildInputs = [
+ wayland
+ wayland-protocols
+ wlroots_0_19
+ libxkbcommon
+ pixman
+ ];
+
+ buildPhase = ''
+ runHook preBuild
+ cp ${wlroots_0_19.src}/tinywl/tinywl.c tinywl.c
+ python3 cua_compositor_patch.py tinywl.c cua-compositor.c
+ wp=${wayland-protocols}/share/wayland-protocols
+ wayland-scanner server-header "$wp/stable/xdg-shell/xdg-shell.xml" xdg-shell-protocol.h
+ wayland-scanner private-code "$wp/stable/xdg-shell/xdg-shell.xml" xdg-shell-protocol.c
+ $CC -DWLR_USE_UNSTABLE -I. cua-compositor.c xdg-shell-protocol.c -o cua-compositor \
+ $(pkg-config --cflags --libs wlroots-0.19 wayland-server xkbcommon pixman-1)
+ runHook postBuild
+ '';
+
+ installPhase = ''
+ runHook preInstall
+ install -Dm755 cua-compositor $out/bin/cua-compositor
+ runHook postInstall
+ '';
+
+ meta = {
+ description = "Headless wlroots compositor with focus-free + multi-cursor input injection for cua-driver";
+ platforms = lib.platforms.linux;
+ mainProgram = "cua-compositor";
+ };
+}
diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nix
index 50a41c973a..e17317e415 100644
--- a/nix/cua-driver/package.nix
+++ b/nix/cua-driver/package.nix
@@ -15,7 +15,7 @@
pkgs.rustPlatform.buildRustPackage {
pname = "cua-driver";
- version = "0.5.1";
+ version = "0.5.3";
inherit src;
@@ -25,7 +25,7 @@ pkgs.rustPlatform.buildRustPackage {
# gracefully via `cargo vendor`.
# Bumped when the dependency set changes (added `atspi`/zbus for native
# AT-SPI). If this mismatches, the nix build prints the expected value.
- cargoHash = "sha256-8e/8inqyEUJA3s9lp/YGU5ckDXV1PnUVTyArwAM1e5g=";
+ cargoHash = "sha256-ykVW0duMeoWho0qwDdu0k//cOY6HG+HsIBQF7iT0nZ0=";
# Build only the main binary crate. The workspace also contains
# platform-macos, platform-windows, cua-driver-uia, and focus-monitor-win
diff --git a/nix/cua-driver/tests/wayland/README.md b/nix/cua-driver/tests/wayland/README.md
new file mode 100644
index 0000000000..33a01cd6d9
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/README.md
@@ -0,0 +1,67 @@
+# cua-driver native-Wayland TDD suite
+
+A reproduction of the cua-driver NixOS tests on **native Wayland**, across five
+desktop sessions. These tests are a **TDD red suite**: the Linux backend is
+X11-only today, so they are **expected to fail** until native Wayland support is
+added. They exist to *specify* that work and to show, per compositor, exactly
+what survives.
+
+## Why they fail today
+
+cua-driver enumerates windows with X11 `_NET_CLIENT_LIST`, captures with
+`import`/`xwd`/XGetImage, and injects input with X11 `XSendEvent` (plus uinput +
+XI2 for the agent cursor / parallel drags). On native Wayland:
+
+- apps are **Wayland clients with no X11 window id**, so `list_windows` returns
+ nothing and `find_window` times out;
+- there is no X display to capture or `XSendEvent` into (the tests never set
+ `DISPLAY`, so there is no XWayland fallback either).
+
+To make a scenario green you implement the corresponding native-Wayland path in
+`libs/cua-driver/rust/crates/platform-linux` (e.g. a Wayland window enumerator,
+a `grim`/screencopy capture path, and a `wlr-virtual-pointer` /
+`virtual-keyboard` / `libei` input path), then re-run the matching check.
+
+## Layout
+
+- `session.nix` — brings up a chosen desktop headless and exposes the Wayland
+ socket. The compositor is the only per-desktop difference.
+- `driver-client.nix` — shared MCP client; window discovery goes **only** through
+ cua-driver's own `list_windows` (no xdotool/X11 cheat).
+- `record-wayland-gif.nix` — `grim`-based GIF recorder (wlroots only; no-op
+ elsewhere).
+- `integration.nix`, `screenshot.nix`, `cursor-click-gif.nix`,
+ `background-terminal-gif.nix`, `parallel-drag.nix`, `background-gui.nix` — the
+ scenarios, each parameterised by `desktop` (and `background-gui` also by `app`).
+
+## Matrix
+
+Desktops: `xfce-labwc`, `xfce-sway` (XFCE 4.20's Wayland-session compositors),
+`kde` (kwin_wayland), `gnome` (mutter headless). (`xfce-wayfire` was dropped —
+wayfire fails to build in the current nixpkgs pin, an upstream `wf-config`/
+`doctest` issue unrelated to cua-driver.)
+
+Checks (built on demand):
+
+```
+nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-<scenario>
+nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-background-gui-<app>
+```
+
+Scenarios: `integration`, `screenshot`, `cursor-click-gif`,
+`background-terminal-gif`, `parallel-drag`.
+Background-GUI apps: `foot`, `gtk3-gedit`, `qt6-kcalc`.
+
+`parallel-drag` is the hardest: it needs X11 MPX master pointers, for which there
+is no native Wayland equivalent — keep it red, or redesign around multi-seat /
+virtual-pointer protocols.
+
+## Running in CI
+
+The `.github/workflows/nix-wayland.yml` workflow runs the whole matrix on the
+same triggers as the X11 `nix-build.yml` workflow — PRs and `main` pushes that
+touch `nix/**`, `flake.{nix,lock}`, `libs/cua-driver/rust/**`, or the workflow
+itself, plus manual `workflow_dispatch`. The jobs are BLOCKING: a red cell fails
+the PR, so the suite only goes green once native Wayland support lands. Each job
+uploads its screenshots / GIFs / logs as artifacts so you can see what each
+compositor did.
diff --git a/nix/cua-driver/tests/wayland/background-gui.nix b/nix/cua-driver/tests/wayland/background-gui.nix
new file mode 100644
index 0000000000..0df054f28b
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/background-gui.nix
@@ -0,0 +1,159 @@
+# CUA Driver native-Wayland background-GUI skeleton (per desktop, per app) — TDD red.
+#
+# Native-Wayland analogue of the READ-ONLY skeleton class in
+# ../../tests/linux-background-gui.nix. Launches a real toolkit app as a NATIVE
+# Wayland client (no GDK_BACKEND/QT_QPA forcing), finds it via cua-driver
+# list_windows, and screenshots it via get_window_state (vision). Records a GIF
+# via grim. Fails today: the driver can neither enumerate nor capture native
+# Wayland surfaces — the spec for native Wayland read support across toolkits.
+#
+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-background-gui-<app>
+{
+ pkgs,
+ lib ? pkgs.lib,
+ cuaDriverModule,
+ desktop,
+ app,
+ ...
+}:
+
+let
+ session = import ./session.nix { inherit pkgs desktop; };
+ driverClient = import ./driver-client.nix { inherit pkgs; };
+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };
+
+ # Representative native-Wayland apps, one per major toolkit. `match` is the
+ # identity substring find_window looks for (title/app_id/app/class).
+ #
+ # NOTE: no per-app env prefix here. Apps are launched via cua-driver's
+ # `launch_app`, which execs the command's FIRST token as the program — an
+ # `ENV=val cmd` prefix would be (mis)read as the program name. The native
+ # Wayland backends (GDK_BACKEND/QT_QPA_PLATFORM) are exported by the session
+ # driver wrapper instead (session.nix appBackendEnv), so the launched toolkit
+ # apps inherit them. Do not reintroduce an `env` field on these.
+ apps = {
+ foot = {
+ packages = [ pkgs.foot ];
+ launch = "foot --app-id=cua-wayland-foot-app --title=cua-wayland-foot-app";
+ match = "cua-wayland-foot-app";
+ };
+ "gtk3-gedit" = {
+ packages = [ pkgs.gedit ];
+ launch = "gedit";
+ match = "gedit";
+ };
+ "qt6-kcalc" = {
+ packages = [ pkgs.kdePackages.kcalc ];
+ launch = "kcalc";
+ match = "kcalc";
+ };
+ };
+
+ cfg = apps.${app} or (throw "wayland/background-gui.nix: unknown app '${app}'");
+ outputGif = "/tmp/cua-driver-wayland-${desktop}-background-gui-${app}.gif";
+ outputPng = "/tmp/cua-driver-wayland-${desktop}-background-gui-${app}.png";
+
+ # GIF recorder env: native desktops point grim at the host socket; nested
+ # desktops leave WAYLAND_DISPLAY unset so the recorder self-resolves the
+ # driver's published nested socket ($XDG_RUNTIME_DIR/.cua-nested-display).
+ # No quotes around the command substitution: this string is interpolated into
+ # a python double-quoted string, and the socket path has no spaces. The inner
+ # `sh -c` re-parses and expands $(cat ...) itself.
+ recorderWlEnv =
+ if session.nested then
+ "env XDG_RUNTIME_DIR=/run/user/0"
+ else
+ "env XDG_RUNTIME_DIR=/run/user/0 WAYLAND_DISPLAY=$(cat /tmp/wl-display)";
+
+ testScriptPy = pkgs.writeText "wayland-bg-gui.py" ''
+ import base64, sys
+ sys.path.insert(0, "/tmp")
+ from driver_client import Driver
+
+ MATCH = "${cfg.match}"
+ OUTPNG = "${outputPng}"
+ LAUNCH = "${cfg.launch}"
+ d = Driver()
+ try:
+ d.initialize("nixos-wayland-bg-gui")
+ # Launch through cua-driver so the app lands in the session it owns
+ # (host compositor on native desktops; nested labwc on kde/gnome).
+ d.launch_app(LAUNCH)
+ pid, wid = d.find_window(MATCH, timeout=50)
+ print(f"app window pid={pid} window_id={wid}", flush=True)
+ resp = d.call("get_window_state", {
+ "pid": pid, "window_id": wid, "capture_mode": "vision"}, timeout=45)
+ saved = False
+ for item in resp.get("result", {}).get("content", []):
+ if item.get("type") == "image" and item.get("data"):
+ with open(OUTPNG, "wb") as f:
+ f.write(base64.b64decode(item["data"]))
+ saved = True
+ break
+ assert saved, f"no capture for native Wayland window: {resp}"
+ print("background gui read test complete", flush=True)
+ finally:
+ d.close()
+ '';
+in
+
+pkgs.testers.nixosTest {
+ name = "cua-driver-wayland-${desktop}-background-gui-${app}-test";
+ meta.maintainers = [ ];
+
+ nodes.machine =
+ { pkgs, ... }:
+ {
+ imports = [ cuaDriverModule ];
+ virtualisation = {
+ cores = 2;
+ memorySize = 4096;
+ };
+ services.cua-driver.enable = true;
+ boot.kernelModules = [ "uinput" ];
+ hardware.graphics.enable = true;
+ services.udev.extraRules = ''
+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"
+ '';
+ environment.systemPackages =
+ session.packages ++ cfg.packages ++ (with pkgs; [ python3 jq at-spi2-core ]);
+ };
+
+ testScript = ''
+ machine.start()
+ machine.wait_for_unit("multi-user.target")
+
+ with subtest("Bring up ${session.label}"):
+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")
+ try:
+ machine.wait_for_file("/tmp/wl-ready", timeout=120)
+ except Exception:
+ machine.log(machine.execute("cat /tmp/session.log || true")[1])
+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])
+ raise
+
+ with subtest("Launch (via cua-driver) + record GIF + read inactive app window"):
+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")
+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-bg-gui.py")
+ # Start the recorder first; it grabs frames while the driver launches
+ # and reads the app. (Native: host socket; nested: self-resolves.)
+ machine.execute(
+ "sh -lc '${recorderWlEnv} ${recordGifScript} /tmp/bg-gui-frames "
+ "${outputGif} /tmp/stop-bg-gui-recorder /tmp/rec-bg-gui.log 12 0.2 "
+ ">/dev/null 2>&1 & echo $! >/tmp/rec-bg-gui.pid'"
+ )
+ result = machine.execute(
+ "timeout 150 env CUA_DRIVER_BIN=${session.driverWrapper} "
+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-bg-gui.py 2>&1"
+ )[1]
+ machine.log(result)
+ # Stop + copy the GIF BEFORE asserting so even failing jobs upload one.
+ machine.execute("touch /tmp/stop-bg-gui-recorder")
+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-bg-gui.pid) 2>/dev/null || break; sleep 1; done'")
+ machine.execute("test -e ${outputGif} || : > ${outputGif}")
+ machine.copy_from_machine("${outputGif}", "")
+
+ with subtest("cua-driver read/captured the native Wayland app window"):
+ assert "background gui read test complete" in result, result
+ '';
+}
diff --git a/nix/cua-driver/tests/wayland/background-terminal-gif.nix b/nix/cua-driver/tests/wayland/background-terminal-gif.nix
new file mode 100644
index 0000000000..871d62d18b
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/background-terminal-gif.nix
@@ -0,0 +1,111 @@
+# CUA Driver native-Wayland background-terminal GIF test (per desktop).
+#
+# The hard one: type into an UNFOCUSED window. Stock Wayland delivers keyboard
+# only to the focused surface, so cua-driver nests its own cua-compositor (EIS
+# mode) which injects wl_keyboard straight into a target surface by app_id,
+# bypassing focus. A control terminal is launched AFTER the target so the target
+# is NOT focused; cua-driver types a command into the inactive target and we
+# verify it ran via a file side-effect. Records a GIF via grim.
+#
+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-background-terminal-gif
+{
+ pkgs,
+ lib ? pkgs.lib,
+ cuaDriverModule,
+ desktop,
+ ...
+}:
+
+let
+ session = import ./session.nix { inherit pkgs desktop; eis = true; };
+ driverClient = import ./driver-client.nix { inherit pkgs; };
+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };
+
+ testScriptPy = pkgs.writeText "wayland-background-terminal.py" ''
+ import sys, time
+ sys.path.insert(0, "/tmp")
+ from driver_client import Driver
+
+ d = Driver()
+ try:
+ d.initialize("nixos-wayland-bg-terminal")
+ # Launch the TARGET first, then the CONTROL — so control holds focus and
+ # the target is the inactive window we must type into focus-free.
+ d.launch_app("foot --app-id=cua-wayland-target --title=cua-wayland-target")
+ time.sleep(1.2)
+ d.launch_app("foot --app-id=cua-wayland-control --title=cua-wayland-control")
+ tpid, twid = d.find_window("cua-wayland-target", timeout=40)
+ cpid, cwid = d.find_window("cua-wayland-control", timeout=40)
+ print(f"target pid={tpid} window_id={twid} control pid={cpid} window_id={cwid}", flush=True)
+ d.call("set_agent_cursor_enabled", {"enabled": True})
+ # Focus-free type into the INACTIVE target while control stays focused.
+ d.call("type_text", {"pid": tpid, "window_id": twid,
+ "text": "echo hello | tee /tmp/background-hello.txt"})
+ d.call("press_key", {"pid": tpid, "window_id": twid, "key": "enter"})
+ time.sleep(1.8)
+ print("background terminal GIF test complete", flush=True)
+ finally:
+ d.close()
+ '';
+in
+
+pkgs.testers.nixosTest {
+ name = "cua-driver-wayland-${desktop}-background-terminal-gif-test";
+ meta.maintainers = [ ];
+
+ nodes.machine =
+ { pkgs, ... }:
+ {
+ imports = [ cuaDriverModule ];
+ virtualisation = {
+ cores = 2;
+ memorySize = 3072;
+ };
+ services.cua-driver.enable = true;
+ boot.kernelModules = [ "uinput" ];
+ hardware.graphics.enable = true;
+ services.udev.extraRules = ''
+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"
+ '';
+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);
+ };
+
+ testScript = ''
+ machine.start()
+ machine.wait_for_unit("multi-user.target")
+
+ with subtest("Bring up ${session.label}"):
+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")
+ try:
+ machine.wait_for_file("/tmp/wl-ready", timeout=120)
+ except Exception:
+ machine.log(machine.execute("cat /tmp/session.log || true")[1])
+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])
+ raise
+
+ with subtest("Record GIF + driver types focus-free into the INACTIVE Wayland terminal"):
+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")
+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-background-terminal.py")
+ # Recorder self-resolves the driver's nested cua-compositor socket.
+ machine.execute(
+ "sh -lc 'env XDG_RUNTIME_DIR=/run/user/0 ${recordGifScript} /tmp/background-frames "
+ "/tmp/cua-driver-wayland-${desktop}-background-terminal.gif /tmp/stop-background-recorder "
+ "/tmp/rec-background.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/rec-background.pid'"
+ )
+ result = machine.execute(
+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "
+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-background-terminal.py 2>&1"
+ )[1]
+ machine.log(result)
+ machine.execute("touch /tmp/stop-background-recorder")
+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-background.pid) 2>/dev/null || break; sleep 1; done'")
+
+ with subtest("Copy GIF out of the VM (best-effort)"):
+ machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-background-terminal.gif || : > /tmp/cua-driver-wayland-${desktop}-background-terminal.gif")
+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-background-terminal.gif", "")
+
+ with subtest("Driver typed into inactive terminal AND the command ran"):
+ assert "background terminal GIF test complete" in result, result
+ machine.wait_until_succeeds("grep -Fx 'hello' /tmp/background-hello.txt", timeout=20)
+ '';
+}
diff --git a/nix/cua-driver/tests/wayland/cursor-click-gif.nix b/nix/cua-driver/tests/wayland/cursor-click-gif.nix
new file mode 100644
index 0000000000..9918d9d9d6
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/cursor-click-gif.nix
@@ -0,0 +1,125 @@
+# CUA Driver native-Wayland cursor-click GIF test (per desktop).
+#
+# Launches two native Wayland terminals (foot) THROUGH cua-driver (launch_app) —
+# the second is the focused target — then clicks it (foreign-toplevel activate),
+# types a command via the Wayland virtual-keyboard (wtype), and verifies the
+# command RAN by checking a file it writes (display-server-agnostic proof).
+# Records a GIF of the composited output via grim. On wlroots desktops the driver
+# drives the host compositor; on kde/gnome it drives its own nested labwc.
+#
+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-cursor-click-gif
+{
+ pkgs,
+ lib ? pkgs.lib,
+ cuaDriverModule,
+ desktop,
+ ...
+}:
+
+let
+ session = import ./session.nix { inherit pkgs desktop; };
+ driverClient = import ./driver-client.nix { inherit pkgs; };
+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };
+
+ # GIF recorder env: native points grim at the host socket; nested self-resolves
+ # the driver's published nested socket ($XDG_RUNTIME_DIR/.cua-nested-display).
+ recorderWlEnv =
+ if session.nested then
+ "env XDG_RUNTIME_DIR=/run/user/0"
+ else
+ "env XDG_RUNTIME_DIR=/run/user/0 WAYLAND_DISPLAY=$(cat /tmp/wl-display)";
+
+ testScriptPy = pkgs.writeText "wayland-cursor-click.py" ''
+ import sys, time
+ sys.path.insert(0, "/tmp")
+ from driver_client import Driver
+
+ d = Driver()
+ try:
+ d.initialize("nixos-wayland-click")
+ # Launch a control terminal then the target, so the target is the
+ # focused (last-mapped) window the keystrokes will reach.
+ d.launch_app("foot --app-id=cua-wayland-control --title=cua-wayland-control")
+ time.sleep(1)
+ d.launch_app("foot --app-id=cua-wayland-target --title=cua-wayland-target")
+ pid, wid = d.find_window("cua-wayland-target", timeout=40)
+ print(f"target pid={pid} window_id={wid}", flush=True)
+ d.call("set_agent_cursor_enabled", {"enabled": True})
+ d.call("move_cursor", {"x": 1100.0, "y": 900.0})
+ time.sleep(0.6)
+ # click focuses+raises the target (foreign-toplevel activate); type +
+ # enter go to it via the Wayland virtual-keyboard.
+ d.call("click", {"pid": pid, "window_id": wid, "x": 120.0, "y": 120.0})
+ time.sleep(0.4)
+ d.call("type_text", {"pid": pid, "window_id": wid,
+ "text": "echo click-focus > /tmp/click-focus.txt"})
+ d.call("press_key", {"pid": pid, "window_id": wid, "key": "enter"})
+ time.sleep(1.5)
+ print("click GIF test complete", flush=True)
+ finally:
+ d.close()
+ '';
+in
+
+pkgs.testers.nixosTest {
+ name = "cua-driver-wayland-${desktop}-cursor-click-gif-test";
+ meta.maintainers = [ ];
+
+ nodes.machine =
+ { pkgs, ... }:
+ {
+ imports = [ cuaDriverModule ];
+ virtualisation = {
+ cores = 2;
+ memorySize = 3072;
+ };
+ services.cua-driver.enable = true;
+ boot.kernelModules = [ "uinput" ];
+ hardware.graphics.enable = true;
+ services.udev.extraRules = ''
+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"
+ '';
+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);
+ };
+
+ testScript = ''
+ machine.start()
+ machine.wait_for_unit("multi-user.target")
+ machine.succeed("modprobe uinput && test -e /dev/uinput")
+
+ with subtest("Bring up ${session.label}"):
+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")
+ try:
+ machine.wait_for_file("/tmp/wl-ready", timeout=120)
+ except Exception:
+ machine.log(machine.execute("cat /tmp/session.log || true")[1])
+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])
+ raise
+
+ with subtest("Record GIF + driver launches, clicks, and types into a native Wayland terminal"):
+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")
+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-cursor-click.py")
+ machine.execute(
+ "sh -lc '${recorderWlEnv} ${recordGifScript} /tmp/click-frames "
+ "/tmp/cua-driver-wayland-${desktop}-cursor-click.gif /tmp/stop-click-recorder "
+ "/tmp/rec-click.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/rec-click.pid'"
+ )
+ result = machine.execute(
+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "
+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-cursor-click.py 2>&1"
+ )[1]
+ machine.log(result)
+ machine.execute("touch /tmp/stop-click-recorder")
+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-click.pid) 2>/dev/null || break; sleep 1; done'")
+
+ with subtest("Copy GIF out of the VM (best-effort)"):
+ # Ensure the path exists so copy never errors before the real assertion;
+ # it may be empty on compositors where grim could not capture.
+ machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-cursor-click.gif || : > /tmp/cua-driver-wayland-${desktop}-cursor-click.gif")
+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-cursor-click.gif", "")
+
+ with subtest("Driver completed click+type AND the command ran in the Wayland terminal"):
+ assert "click GIF test complete" in result, result
+ machine.wait_until_succeeds("test -f /tmp/click-focus.txt", timeout=20)
+ '';
+}
diff --git a/nix/cua-driver/tests/wayland/driver-client.nix b/nix/cua-driver/tests/wayland/driver-client.nix
new file mode 100644
index 0000000000..5b20fe0c52
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/driver-client.nix
@@ -0,0 +1,138 @@
+# Shared MCP client used by the native-Wayland TDD tests.
+#
+# It is copied into the VM as /tmp/driver_client.py and imported by the small
+# per-test scripts. Crucially, window discovery goes ONLY through cua-driver's
+# own `list_windows` tool — there is no xdotool/X11 fallback — so on native
+# Wayland (where the driver currently enumerates nothing) `find_window` times
+# out and the test fails. That is the intended red state for TDD.
+{ pkgs }:
+
+pkgs.writeText "driver_client.py" ''
+ import json, os, subprocess, sys, threading, time
+
+ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")
+
+
+ class Driver:
+ def __init__(self):
+ self.proc = subprocess.Popen(
+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ env={**os.environ},
+ )
+ threading.Thread(target=self._drain, daemon=True).start()
+ self._id = 1
+
+ def _drain(self):
+ for line in self.proc.stderr:
+ sys.stderr.buffer.write(line)
+ sys.stderr.buffer.flush()
+
+ def _send(self, method, params=None, req_id=None):
+ msg = {"jsonrpc": "2.0", "method": method}
+ if params is not None:
+ msg["params"] = params
+ if req_id is not None:
+ msg["id"] = req_id
+ self.proc.stdin.write((json.dumps(msg) + "\n").encode())
+ self.proc.stdin.flush()
+
+ def _recv(self, timeout=45):
+ result = [None]
+
+ def reader():
+ result[0] = self.proc.stdout.readline()
+
+ th = threading.Thread(target=reader)
+ th.start()
+ th.join(timeout)
+ if th.is_alive():
+ raise TimeoutError("no response from driver within timeout")
+ line = result[0].decode().strip()
+ if not line:
+ raise RuntimeError("driver returned an empty response")
+ return json.loads(line)
+
+ def initialize(self, client="nixos-wayland"):
+ self._send("initialize", {
+ "protocolVersion": "2024-11-05", "capabilities": {},
+ "clientInfo": {"name": client, "version": "1.0.0"},
+ }, req_id=self._id)
+ resp = self._recv()
+ self._id += 1
+ assert "result" in resp, f"initialize failed: {resp}"
+ self._send("notifications/initialized", {})
+ time.sleep(0.3)
+ return resp
+
+ def call(self, name, args, timeout=60):
+ self._send("tools/call", {"name": name, "arguments": args}, req_id=self._id)
+ self._id += 1
+ resp = self._recv(timeout=timeout)
+ if resp.get("error"):
+ raise RuntimeError(f"{name} failed: {resp}")
+ if resp.get("result", {}).get("isError"):
+ raise RuntimeError(f"{name} returned isError: {resp}")
+ return resp
+
+ def launch_app(self, command):
+ """Launch an app THROUGH cua-driver so it lands in whatever Wayland
+ session the driver owns — the host compositor (native desktops) or the
+ driver's own nested labwc (kde/gnome). Returns the launch response."""
+ return self.call("launch_app", {"name": command})
+
+ def list_windows(self):
+ resp = self.call("list_windows", {})
+ result = resp.get("result", {})
+ # Preferred: MCP structuredContent = { "windows": [ {window_id, pid,
+ # title, ...}, ... ] }.
+ structured = result.get("structuredContent")
+ if isinstance(structured, dict) and isinstance(structured.get("windows"), list):
+ return structured["windows"]
+ if isinstance(structured, list):
+ return structured
+ # Fallback: a text item that happens to be JSON.
+ for item in result.get("content", []):
+ if item.get("type") == "text":
+ try:
+ data = json.loads(item.get("text", ""))
+ except Exception:
+ continue
+ if isinstance(data, list):
+ return data
+ if isinstance(data, dict) and "windows" in data:
+ return data["windows"]
+ return []
+
+ def find_window(self, title_substr, timeout=30, interval=1.0):
+ """Poll list_windows until a native Wayland toplevel whose title
+ contains `title_substr` appears. Raises on timeout — the red point on
+ a driver that cannot yet enumerate Wayland windows."""
+ deadline = time.time() + timeout
+ needle = title_substr.lower()
+ last = []
+ while time.time() < deadline:
+ last = self.list_windows()
+ for w in last:
+ # Match across whatever identity fields a Wayland-capable
+ # list_windows might surface (title today; app_id/app/class
+ # once native Wayland metadata is wired up).
+ hay = " ".join(
+ str(w.get(k, "")) for k in ("title", "app_id", "app", "class")
+ ).lower()
+ if needle in hay:
+ return int(w.get("pid") or 0), int(w["window_id"])
+ time.sleep(interval)
+ raise AssertionError(
+ f"cua-driver never enumerated a Wayland window titled ~{title_substr!r}; "
+ f"last list_windows() = {last}"
+ )
+
+ def close(self):
+ try:
+ self.proc.stdin.close()
+ self.proc.terminate()
+ self.proc.wait(timeout=5)
+ except Exception:
+ pass
+''
diff --git a/nix/cua-driver/tests/wayland/integration.nix b/nix/cua-driver/tests/wayland/integration.nix
new file mode 100644
index 0000000000..e249141167
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/integration.nix
@@ -0,0 +1,109 @@
+# CUA Driver native-Wayland integration test (per desktop).
+#
+# Brings up the desktop session, then launches a native Wayland terminal (foot)
+# THROUGH cua-driver (launch_app) and asserts cua-driver can ENUMERATE it via
+# list_windows. On native (wlroots) desktops the driver talks to the host
+# compositor; on kde/gnome it drives its own nested labwc (CUA_WAYLAND_NEST=1,
+# set by the session driver wrapper). Either way the app lands in the session
+# the driver owns.
+#
+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-integration
+{
+ pkgs,
+ lib ? pkgs.lib,
+ cuaDriverModule,
+ desktop,
+ ...
+}:
+
+let
+ session = import ./session.nix { inherit pkgs desktop; };
+ driverClient = import ./driver-client.nix { inherit pkgs; };
+
+ testScriptPy = pkgs.writeText "wayland-integration.py" ''
+ import sys, time
+ sys.path.insert(0, "/tmp")
+ from driver_client import Driver
+
+ d = Driver()
+ try:
+ d.initialize("nixos-wayland-integration")
+
+ # Handshake sanity: required tools are advertised.
+ d._send("tools/list", {}, req_id=999)
+ resp = d._recv()
+ names = [t["name"] for t in resp.get("result", {}).get("tools", [])]
+ for t in ("list_windows", "get_window_state", "click", "type_text", "launch_app"):
+ assert t in names, f"{t} not advertised: {names}"
+ print(f"tools advertised: {len(names)}", flush=True)
+
+ # Launch foot through cua-driver so it lands in the session the driver
+ # owns (host compositor on native desktops; nested labwc on kde/gnome).
+ d.launch_app("foot --app-id=cua-wayland-foot --title=cua-wayland-foot")
+
+ # list_windows must surface the foot terminal the driver just launched.
+ pid, wid = d.find_window("cua-wayland-foot", timeout=40)
+ print(f"RESOLVED native wayland window pid={pid} window_id={wid}", flush=True)
+ print("integration test complete", flush=True)
+ finally:
+ d.close()
+ '';
+in
+
+pkgs.testers.nixosTest {
+ name = "cua-driver-wayland-${desktop}-integration-test";
+ meta.maintainers = [ ];
+
+ nodes.machine =
+ { pkgs, ... }:
+ {
+ imports = [ cuaDriverModule ];
+ virtualisation = {
+ cores = 2;
+ memorySize = 3072;
+ };
+ services.cua-driver.enable = true;
+ boot.kernelModules = [ "uinput" ];
+ hardware.graphics.enable = true;
+ services.udev.extraRules = ''
+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"
+ '';
+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);
+ };
+
+ testScript = ''
+ machine.start()
+ machine.wait_for_unit("multi-user.target")
+
+ with subtest("Binary exists and lists tools"):
+ machine.succeed("cua-driver --help")
+ tools = machine.succeed("cua-driver list-tools")
+ for t in ("list_windows", "get_window_state", "click", "type_text"):
+ assert t in tools, f"{t} missing from list-tools"
+
+ with subtest("Bring up ${session.label}"):
+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")
+ try:
+ machine.wait_for_file("/tmp/wl-ready", timeout=120)
+ except Exception:
+ machine.log(machine.execute("cat /tmp/session.log || true")[1])
+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])
+ raise
+ machine.log("host WAYLAND_DISPLAY=" + machine.succeed("cat /tmp/wl-display").strip())
+
+ with subtest("doctor reports a Wayland status line"):
+ result = machine.succeed("timeout 30 ${session.driverWrapper} doctor 2>&1 || true")
+ machine.log(result)
+ assert "Wayland" in result, f"doctor did not report Wayland: {result}"
+
+ with subtest("cua-driver launches + enumerates a native Wayland window"):
+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")
+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-integration.py")
+ result = machine.succeed(
+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "
+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-integration.py 2>&1"
+ )
+ machine.log(result)
+ assert "integration test complete" in result, result
+ '';
+}
diff --git a/nix/cua-driver/tests/wayland/parallel-drag.nix b/nix/cua-driver/tests/wayland/parallel-drag.nix
new file mode 100644
index 0000000000..7d67ce50d0
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/parallel-drag.nix
@@ -0,0 +1,104 @@
+# CUA Driver native-Wayland parallel multi-cursor drag test (per desktop).
+#
+# Two simultaneous drag gestures on one window. Stock Wayland has a single seat
+# cursor and no multi-pointer protocol, so cua-driver nests its own
+# cua-compositor (EIS mode), which drives N independent logical cursors and
+# injects wl_pointer per cursor by app_id. parallel_mouse_drag routes through
+# the compositor's control socket (window-local coords, no X11 MPX). Records a
+# GIF via grim.
+#
+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-parallel-drag
+{
+ pkgs,
+ lib ? pkgs.lib,
+ cuaDriverModule,
+ desktop,
+ ...
+}:
+
+let
+ session = import ./session.nix { inherit pkgs desktop; eis = true; };
+ driverClient = import ./driver-client.nix { inherit pkgs; };
+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };
+
+ testScriptPy = pkgs.writeText "wayland-parallel-drag.py" ''
+ import sys, time
+ sys.path.insert(0, "/tmp")
+ from driver_client import Driver
+
+ d = Driver()
+ try:
+ d.initialize("nixos-wayland-parallel-drag")
+ d.launch_app("foot --app-id=cua-wayland-paint --title=cua-wayland-paint")
+ pid, wid = d.find_window("cua-wayland-paint", timeout=40)
+ print(f"target pid={pid} window_id={wid}", flush=True)
+ d.call("parallel_mouse_drag", {"drags": [
+ {"session": "agent-1", "window_id": wid, "from_x": 100.0, "from_y": 100.0, "to_x": 380.0, "to_y": 420.0, "duration_ms": 2000, "steps": 60},
+ {"session": "agent-2", "window_id": wid, "from_x": 700.0, "from_y": 100.0, "to_x": 420.0, "to_y": 420.0, "duration_ms": 2000, "steps": 60},
+ ]})
+ time.sleep(0.6)
+ print("parallel drag test complete", flush=True)
+ finally:
+ d.close()
+ '';
+in
+
+pkgs.testers.nixosTest {
+ name = "cua-driver-wayland-${desktop}-parallel-drag-test";
+ meta.maintainers = [ ];
+
+ nodes.machine =
+ { pkgs, ... }:
+ {
+ imports = [ cuaDriverModule ];
+ virtualisation = {
+ cores = 2;
+ memorySize = 3072;
+ };
+ services.cua-driver.enable = true;
+ boot.kernelModules = [ "uinput" ];
+ hardware.graphics.enable = true;
+ services.udev.extraRules = ''
+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"
+ '';
+ services.libinput.enable = true;
+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);
+ };
+
+ testScript = ''
+ machine.start()
+ machine.wait_for_unit("multi-user.target")
+
+ with subtest("Bring up ${session.label}"):
+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")
+ try:
+ machine.wait_for_file("/tmp/wl-ready", timeout=120)
+ except Exception:
+ machine.log(machine.execute("cat /tmp/session.log || true")[1])
+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])
+ raise
+
+ with subtest("Record GIF + pilot cua-driver through parallel_mouse_drag (multi-cursor)"):
+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")
+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-parallel-drag.py")
+ machine.execute(
+ "sh -lc 'env XDG_RUNTIME_DIR=/run/user/0 ${recordGifScript} /tmp/drag-frames "
+ "/tmp/cua-driver-wayland-${desktop}-parallel-drag.gif /tmp/stop-drag-recorder "
+ "/tmp/rec-drag.log 8 0.12 >/dev/null 2>&1 & echo $! >/tmp/rec-drag.pid'"
+ )
+ result = machine.execute(
+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "
+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-parallel-drag.py 2>&1"
+ )[1]
+ machine.log(result)
+ machine.execute("touch /tmp/stop-drag-recorder")
+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-drag.pid) 2>/dev/null || break; sleep 1; done'")
+
+ with subtest("Copy GIF out of the VM (best-effort)"):
+ machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-parallel-drag.gif || : > /tmp/cua-driver-wayland-${desktop}-parallel-drag.gif")
+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-parallel-drag.gif", "")
+
+ with subtest("parallel_mouse_drag completed on native Wayland (multi-cursor via EIS compositor)"):
+ assert "parallel drag test complete" in result, result
+ '';
+}
diff --git a/nix/cua-driver/tests/wayland/record-wayland-gif.nix b/nix/cua-driver/tests/wayland/record-wayland-gif.nix
new file mode 100644
index 0000000000..bc870404ba
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/record-wayland-gif.nix
@@ -0,0 +1,59 @@
+# Native-Wayland screen-recording helper for the cua-driver Wayland TDD tests.
+#
+# Captures the composited Wayland output with `grim` (wlr-screencopy) in a loop
+# and stitches the frames into an animated GIF. grim only works on wlroots
+# compositors (labwc/wayfire/sway); on kwin/mutter it simply produces no frames
+# and no GIF — which is fine, the GIF is a best-effort artifact and these tests
+# are expected to fail before/around it anyway.
+#
+# Usage (in a test's `let`):
+# recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };
+# then inside the testScript, start it in the background before driving and
+# `touch` the stop-file afterwards:
+# ${recordGifScript} <frames_dir> <output_gif> <stop_file> <log_file> \
+# <delay_cs> <interval>
+# The caller must export WAYLAND_DISPLAY and XDG_RUNTIME_DIR for grim.
+{ pkgs }:
+
+pkgs.writeShellScript "record-wayland-gif.sh" ''
+ set -u
+ frames_dir="$1"
+ output_gif="$2"
+ stop_file="$3"
+ log_file="$4"
+ delay_cs="$5"
+ interval="$6"
+
+ rm -f "$stop_file" "$output_gif" "$log_file"
+ rm -rf "$frames_dir"
+ mkdir -p "$frames_dir"
+
+ # In nested mode the caller can't know cua-driver's private socket ahead of
+ # time; the driver publishes it to $XDG_RUNTIME_DIR/.cua-nested-display once
+ # up. If WAYLAND_DISPLAY isn't already set, wait for and adopt that socket so
+ # the GIF captures the SAME session the driver drives.
+ if [ -z "''${WAYLAND_DISPLAY:-}" ]; then
+ dfile="''${XDG_RUNTIME_DIR:-/run/user/0}/.cua-nested-display"
+ w=0
+ while [ ! -s "$dfile" ] && [ "$w" -lt 60 ]; do sleep 0.5; w=$((w + 1)); done
+ if [ -s "$dfile" ]; then
+ WAYLAND_DISPLAY="$(cat "$dfile")"
+ export WAYLAND_DISPLAY
+ echo "recorder adopted nested WAYLAND_DISPLAY=$WAYLAND_DISPLAY" >>"$log_file"
+ fi
+ fi
+
+ max_frames=450
+ i=0
+ while [ ! -f "$stop_file" ] && [ "$i" -lt "$max_frames" ]; do
+ frame=$(printf "%s/frame-%04d.png" "$frames_dir" "$i")
+ timeout 10 ${pkgs.grim}/bin/grim "$frame" >>"$log_file" 2>&1 || true
+ i=$((i + 1))
+ sleep "$interval"
+ done
+
+ if ls "$frames_dir"/frame-*.png >/dev/null 2>&1; then
+ timeout 120 ${pkgs.imagemagick}/bin/convert -delay "$delay_cs" -loop 0 \
+ "$frames_dir"/frame-*.png "$output_gif" >>"$log_file" 2>&1 || true
+ fi
+''
diff --git a/nix/cua-driver/tests/wayland/screenshot.nix b/nix/cua-driver/tests/wayland/screenshot.nix
new file mode 100644
index 0000000000..7a3cd31317
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/screenshot.nix
@@ -0,0 +1,98 @@
+# CUA Driver native-Wayland screenshot test (per desktop) — TDD red.
+#
+# Launches a native Wayland terminal (foot), finds it via cua-driver
+# list_windows, and asserts get_window_state (capture_mode=vision) returns a
+# real PNG of that native Wayland surface. The driver captures via X11 today, so
+# both discovery and capture fail on native Wayland — the spec for native
+# Wayland window capture.
+#
+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-screenshot
+{
+ pkgs,
+ lib ? pkgs.lib,
+ cuaDriverModule,
+ desktop,
+ ...
+}:
+
+let
+ session = import ./session.nix { inherit pkgs desktop; };
+ driverClient = import ./driver-client.nix { inherit pkgs; };
+
+ testScriptPy = pkgs.writeText "wayland-screenshot.py" ''
+ import base64, sys
+ sys.path.insert(0, "/tmp")
+ from driver_client import Driver
+
+ d = Driver()
+ try:
+ d.initialize("nixos-wayland-screenshot")
+ d.launch_app("foot --app-id=cua-wayland-foot --title=cua-wayland-foot")
+ pid, wid = d.find_window("cua-wayland-foot", timeout=40)
+ print(f"window pid={pid} window_id={wid}", flush=True)
+ resp = d.call("get_window_state", {
+ "pid": pid, "window_id": wid, "capture_mode": "vision"}, timeout=30)
+ saved = False
+ for item in resp.get("result", {}).get("content", []):
+ if item.get("type") == "image" and item.get("data"):
+ data = base64.b64decode(item["data"])
+ assert len(data) > 0, "empty image payload"
+ with open("/tmp/cua-driver-wayland-${desktop}-screenshot.png", "wb") as f:
+ f.write(data)
+ saved = True
+ break
+ assert saved, f"no image returned for native Wayland window: {resp}"
+ print("screenshot test complete", flush=True)
+ finally:
+ d.close()
+ '';
+in
+
+pkgs.testers.nixosTest {
+ name = "cua-driver-wayland-${desktop}-screenshot-test";
+ meta.maintainers = [ ];
+
+ nodes.machine =
+ { pkgs, ... }:
+ {
+ imports = [ cuaDriverModule ];
+ virtualisation = {
+ cores = 2;
+ memorySize = 3072;
+ };
+ services.cua-driver.enable = true;
+ boot.kernelModules = [ "uinput" ];
+ hardware.graphics.enable = true;
+ services.udev.extraRules = ''
+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"
+ '';
+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);
+ };
+
+ testScript = ''
+ machine.start()
+ machine.wait_for_unit("multi-user.target")
+
+ with subtest("Bring up ${session.label}"):
+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")
+ try:
+ machine.wait_for_file("/tmp/wl-ready", timeout=120)
+ except Exception:
+ machine.log(machine.execute("cat /tmp/session.log || true")[1])
+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])
+ raise
+
+ with subtest("Launch + screenshot a native Wayland window via cua-driver"):
+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")
+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-screenshot.py")
+ result = machine.succeed(
+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "
+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-screenshot.py 2>&1"
+ )
+ machine.log(result)
+ assert "screenshot test complete" in result, result
+
+ with subtest("Extract screenshot"):
+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-screenshot.png", "")
+ '';
+}
diff --git a/nix/cua-driver/tests/wayland/session.nix b/nix/cua-driver/tests/wayland/session.nix
new file mode 100644
index 0000000000..72d5df3fe0
--- /dev/null
+++ b/nix/cua-driver/tests/wayland/session.nix
@@ -0,0 +1,243 @@
+# Shared Wayland desktop-session bring-up for the cua-driver Wayland suite.
+#
+# Two execution models, selected per desktop:
+#
+# * NATIVE (wlroots: labwc, sway) — apps run as real Wayland clients of the
+# HOST compositor, and the driver talks to that same compositor's wlr
+# protocols (foreign-toplevel + screencopy). The session `start` script
+# launches the compositor headless, waits for its socket, and publishes it
+# to /tmp/wl-display.
+#
+# * NESTED (kde: kwin, gnome: mutter) — kwin/mutter expose NO client protocols
+# for cross-window enumeration/capture, so the driver cannot drive them
+# directly. Instead cua-driver "brings its own compositor": with
+# CUA_WAYLAND_NEST=1 it spawns a private headless labwc and points
+# WAYLAND_DISPLAY at it, so every app launched via `launch_app` runs inside
+# that nested session where the wlr protocols DO work. The host kwin/mutter
+# is still booted (best-effort) to prove cua-driver coexists with a real
+# KDE/GNOME host, but the test never depends on it.
+#
+# Given a `desktop` this returns:
+#
+# { packages; start; label; nested; driverWrapper; }
+#
+# * packages — extra `environment.systemPackages` the session needs.
+# * start — a `writeShellScript` that brings the session up and
+# `touch`es /tmp/wl-ready, then blocks.
+# * label — human label for logs.
+# * nested — true for kde/gnome (driver hosts its own compositor).
+# * driverWrapper — a `writeShellScript` to use as CUA_DRIVER_BIN: it sets the
+# correct env (host WAYLAND_DISPLAY for native; the nest env
+# for nested) and execs `cua-driver "$@"`. Tests spawn the
+# driver through this so the python harness stays
+# desktop-agnostic.
+{
+ pkgs,
+ desktop,
+ # EIS mode: cua-driver nests its OWN cua-compositor (focus-free keyboard +
+ # multi-cursor injection over a control socket) instead of labwc, on ANY host.
+ # Used by the background-terminal-gif + parallel-drag cells.
+ eis ? false,
+}:
+
+let
+ inherit (pkgs) lib;
+ cuaCompositor = pkgs.callPackage ../../compositor { };
+
+ # Native-Wayland tooling only. No xorg.* / xdotool / xterm here — those are
+ # X11 and would let the driver cheat via XWayland. `foot` is a Wayland-native
+ # terminal; `grim` captures the wlroots output for the GIF artifacts.
+ commonPkgs = with pkgs; [
+ foot
+ grim
+ wtype # virtual-keyboard CLI: cua-driver shells out to it for type_text/press_key
+ dbus
+ procps
+ ];
+
+ # Minimal sway config: native Wayland only (XWayland off), stable output.
+ # `workspace_layout stacking` makes the focused window fill the output, so the
+ # driver's centre virtual-pointer click reliably lands on the activated target
+ # (sway needs a pointer interaction to route virtual-keyboard input on a
+ # headless seat with no physical keyboard).
+ swayConfig = pkgs.writeText "sway-config" ''
+ xwayland disable
+ output HEADLESS-1 resolution 1280x1024
+ workspace_layout stacking
+ exec_always foot --server
+ '';
+
+ desktops = {
+ "xfce-labwc" = {
+ label = "XFCE on labwc (native Wayland)";
+ packages = with pkgs; [ labwc ];
+ wlroots = true;
+ nested = false;
+ launch = "labwc >/tmp/compositor.log 2>&1 &";
+ };
+ # NOTE: xfce-wayfire intentionally omitted — wayfire fails to build in the
+ # current nixpkgs pin (wf-config can't link -ldoctest). labwc + sway cover
+ # XFCE-on-wlroots.
+ "xfce-sway" = {
+ label = "XFCE on sway (native Wayland)";
+ packages = with pkgs; [ sway ];
+ wlroots = true;
+ nested = false;
+ launch = "sway -c ${swayConfig} >/tmp/compositor.log 2>&1 &";
+ };
+ "kde" = {
+ label = "KDE Plasma kwin_wayland host + cua-driver nested labwc";
+ # labwc is what cua-driver nests; kwin is the host it coexists with.
+ packages = with pkgs; [ labwc kdePackages.kwin ];
+ wlroots = false;
+ nested = true;
+ # No --xwayland: keep the host session native-Wayland only.
+ launch = "kwin_wayland --virtual --width 1280 --height 1024 --no-lockscreen >/tmp/compositor.log 2>&1 &";
+ };
+ "gnome" = {
+ label = "GNOME Shell / mutter host + cua-driver nested labwc";
+ packages = with pkgs; [ labwc gnome-shell mutter ];
+ wlroots = false;
+ nested = true;
+ launch = "gnome-shell --wayland --headless --virtual-monitor 1280x1024 --unsafe-mode >/tmp/compositor.log 2>&1 &";
+ };
+ };
+
+ cfg = desktops.${desktop} or (throw "wayland/session.nix: unknown desktop '${desktop}'");
+
+ wlrootsEnv = lib.optionalString cfg.wlroots ''
+ export WLR_BACKENDS=headless
+ export WLR_RENDERER=pixman
+ export WLR_RENDERER_ALLOW_SOFTWARE=1
+ export WLR_LIBINPUT_NO_DEVICES=1
+ '';
+
+ # Native model: block until the HOST compositor's socket appears, publish it.
+ nativeStart = ''
+ dbus-run-session -- sh -c '
+ wlsock() { ls "$XDG_RUNTIME_DIR" 2>/dev/null | grep -E "^wayland-[0-9]+$" | head -1; }
+
+ ${cfg.launch}
+ compositor_pid=$!
+
+ n=0
+ while [ -z "$(wlsock)" ] && [ "$n" -lt 120 ]; do
+ if ! kill -0 "$compositor_pid" 2>/dev/null; then
+ echo "compositor exited early" >&2
+ break
+ fi
+ sleep 0.5
+ n=$((n + 1))
+ done
+
+ sock="$(wlsock)"
+ if [ -z "$sock" ]; then
+ echo "no Wayland display socket appeared" >&2
+ cat /tmp/compositor.log >&2 2>/dev/null || true
+ exit 1
+ fi
+ echo "$sock" > /tmp/wl-display
+
+ touch /tmp/wl-ready
+ exec sleep infinity
+ '
+ '';
+
+ # Nested model: boot the host compositor BEST-EFFORT (log whether it came up),
+ # but never block the test on it — cua-driver provides its own session. We
+ # record the host socket (if any) for logs; the driver wrapper ignores it.
+ nestedStart = ''
+ dbus-run-session -- sh -c '
+ wlsock() { ls "$XDG_RUNTIME_DIR" 2>/dev/null | grep -E "^wayland-[0-9]+$" | head -1; }
+
+ ${cfg.launch}
+
+ n=0
+ while [ -z "$(wlsock)" ] && [ "$n" -lt 40 ]; do
+ sleep 0.5
+ n=$((n + 1))
+ done
+ sock="$(wlsock)"
+ if [ -n "$sock" ]; then
+ echo "host compositor up: $sock" >&2
+ else
+ echo "host compositor did not come up; proceeding with nested session only" >&2
+ cat /tmp/compositor.log >&2 2>/dev/null || true
+ fi
+ echo "''${sock:-none}" > /tmp/wl-display
+
+ touch /tmp/wl-ready
+ exec sleep infinity
+ '
+ '';
+
+ start = pkgs.writeShellScript "wayland-session-start-${desktop}.sh" ''
+ set -u
+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH
+
+ export XDG_RUNTIME_DIR=/run/user/0
+ mkdir -p "$XDG_RUNTIME_DIR"
+ chmod 700 "$XDG_RUNTIME_DIR"
+
+ ${wlrootsEnv}
+ export QT_QPA_PLATFORM=wayland
+ export GDK_BACKEND=wayland
+ export LIBGL_ALWAYS_SOFTWARE=1
+
+ rm -f /tmp/wl-ready /tmp/wl-display "$XDG_RUNTIME_DIR/.cua-nested-display" "$XDG_RUNTIME_DIR/cua-inject.sock"
+
+ ${if (cfg.nested || eis) then nestedStart else nativeStart}
+ '';
+
+ # Driver launcher used as CUA_DRIVER_BIN. It owns the WAYLAND env so the python
+ # harness never has to. labwc/foot/grim are on PATH for the app/compositor
+ # processes cua-driver spawns.
+ # Toolkit apps cua-driver launches inherit its env, so force the NATIVE
+ # Wayland backends here (GTK auto-detects from WAYLAND_DISPLAY; Qt would
+ # otherwise default to xcb and fail with no X). Harmless for foot.
+ appBackendEnv = ''
+ export GDK_BACKEND=wayland
+ export QT_QPA_PLATFORM=wayland
+ export LIBGL_ALWAYS_SOFTWARE=1
+ '';
+
+ driverWrapper =
+ if eis then
+ # EIS mode: nest cua-compositor (focus-free + multi-cursor injection over
+ # the control socket) on ANY host. Apps launched via launch_app land in it.
+ pkgs.writeShellScript "cua-driver-eis-${desktop}" ''
+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages ++ [ cuaCompositor ])}:$PATH
+ export XDG_RUNTIME_DIR=/run/user/0
+ export CUA_WAYLAND_NEST=1
+ export CUA_WAYLAND_NEST_COMPOSITOR=${cuaCompositor}/bin/cua-compositor
+ export CUA_INJECT_SOCKET=/run/user/0/cua-inject.sock
+ ${appBackendEnv}
+ unset DISPLAY WAYLAND_DISPLAY
+ exec cua-driver "$@"
+ ''
+ else if cfg.nested then
+ pkgs.writeShellScript "cua-driver-nested-${desktop}" ''
+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH
+ export XDG_RUNTIME_DIR=/run/user/0
+ export CUA_WAYLAND_NEST=1
+ export CUA_WAYLAND_NEST_COMPOSITOR=labwc
+ ${appBackendEnv}
+ unset DISPLAY WAYLAND_DISPLAY
+ exec cua-driver "$@"
+ ''
+ else
+ pkgs.writeShellScript "cua-driver-native-${desktop}" ''
+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH
+ export XDG_RUNTIME_DIR=/run/user/0
+ export WAYLAND_DISPLAY="$(cat /tmp/wl-display)"
+ ${appBackendEnv}
+ unset DISPLAY
+ exec cua-driver "$@"
+ '';
+in
+{
+ label = if eis then "${cfg.label} + cua-compositor (EIS injection)" else cfg.label;
+ nested = cfg.nested || eis;
+ packages = commonPkgs ++ cfg.packages ++ lib.optional eis cuaCompositor;
+ inherit start driverWrapper;
+}