Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
wayland support (#1910)
r33drichardsafe48d4decb5
diff --git a/.github/workflows/nix-wayland.yml b/.github/workflows/nix-wayland.ymlnew file mode 100644index 0000000000..08ea44ef15--- /dev/null+++ b/.github/workflows/nix-wayland.yml@@ -0,0 +1,212 @@+name: CUA Driver Native-Wayland TDD Suite++# Native-Wayland reproduction of the cua-driver NixOS tests across five desktop+# sessions (XFCE on labwc/sway, plus KDE and GNOME). These are a TDD+# RED suite: the driver is X11-only today, so they are EXPECTED TO FAIL until+# native Wayland support lands. The triggers mirror the X11 nix-build.yml+# workflow (PRs/pushes touching nix/flake/driver, plus manual dispatch). These+# jobs are BLOCKING (no continue-on-error): a red cell fails the PR, so the suite+# can only go green once native Wayland support is implemented.+# Artifacts (screenshots, GIFs, logs) are uploaded so you can see exactly what+# each compositor does.+on:+ pull_request:+ paths:+ - 'nix/**'+ - 'flake.nix'+ - 'flake.lock'+ - 'libs/cua-driver/rust/**'+ - '.github/workflows/nix-wayland.yml'+ push:+ branches: [main]+ paths:+ - 'nix/**'+ - 'flake.nix'+ - 'flake.lock'+ - 'libs/cua-driver/rust/**'+ - '.github/workflows/nix-wayland.yml'+ workflow_dispatch:++permissions:+ id-token: write+ contents: read+env:+ AWS_REGION: us-west-2+ NIX_CACHE_BUCKET: trycua-nix-cache+ NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key++jobs:+ wayland-scenarios:+ name: ${{ matrix.desktop }} / ${{ matrix.scenario }}+ runs-on: ubuntu-latest+ timeout-minutes: 30+ strategy:+ fail-fast: false+ matrix:+ desktop: [xfce-labwc, xfce-sway, kde, gnome]+ scenario:+ - integration+ - screenshot+ - cursor-click-gif+ - background-terminal-gif+ - parallel-drag+ steps:+ - name: Checkout+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Configure AWS Credentials via OIDC+ uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4+ with:+ role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache+ aws-region: ${{ env.AWS_REGION }}++ - name: Get Nix signing key+ id: nix-key+ run: |+ SECRET=$(aws secretsmanager get-secret-value \+ --secret-id "${{ env.NIX_CACHE_SECRET }}" \+ --query 'SecretString' --output text)+ SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key')+ echo "::add-mask::$SECRET_KEY"+ echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec"+ chmod 600 "${{ runner.temp }}/signing-key.sec"+ PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key')+ echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT"++ - name: Setup AWS credentials file for Nix+ run: |+ mkdir -p ~/.aws+ printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \+ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials+ chmod 600 ~/.aws/credentials+ sudo mkdir -p /root/.aws+ sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \+ '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials"+ sudo chmod 600 /root/.aws/credentials++ - name: Install Nix+ uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30+ with:+ extra_nix_config: |+ experimental-features = nix-command flakes+ substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org+ trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}+ trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=++ - name: Run cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }}+ run: |+ nix build ".#checks.x86_64-linux.cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }}" \+ --print-build-logs --show-trace \+ -o "result-${{ matrix.desktop }}-${{ matrix.scenario }}"++ - name: Collect artifacts+ if: always()+ run: |+ mkdir -p artifacts+ find -L "result-${{ matrix.desktop }}-${{ matrix.scenario }}/" \+ \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f \+ -exec cp {} artifacts/ \; 2>/dev/null || true+ ls -la artifacts/ 2>/dev/null || echo "No artifacts found"++ - name: Upload artifacts+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }}+ path: artifacts/*+ if-no-files-found: warn++ - name: Sign and upload to Nix cache+ if: always()+ run: |+ nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all || true+ nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L || true++ - name: Cleanup signing key+ if: always()+ run: rm -f "${{ runner.temp }}/signing-key.sec"++ wayland-background-gui:+ name: ${{ matrix.desktop }} / background-gui ${{ matrix.app }}+ runs-on: ubuntu-latest+ timeout-minutes: 30+ strategy:+ fail-fast: false+ matrix:+ desktop: [xfce-labwc, xfce-sway, kde, gnome]+ app: [foot, gtk3-gedit, qt6-kcalc]+ steps:+ - name: Checkout+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Configure AWS Credentials via OIDC+ uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4+ with:+ role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache+ aws-region: ${{ env.AWS_REGION }}++ - name: Get Nix signing key+ id: nix-key+ run: |+ SECRET=$(aws secretsmanager get-secret-value \+ --secret-id "${{ env.NIX_CACHE_SECRET }}" \+ --query 'SecretString' --output text)+ SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key')+ echo "::add-mask::$SECRET_KEY"+ echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec"+ chmod 600 "${{ runner.temp }}/signing-key.sec"+ PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key')+ echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT"++ - name: Setup AWS credentials file for Nix+ run: |+ mkdir -p ~/.aws+ printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \+ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials+ chmod 600 ~/.aws/credentials+ sudo mkdir -p /root/.aws+ sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \+ '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials"+ sudo chmod 600 /root/.aws/credentials++ - name: Install Nix+ uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30+ with:+ extra_nix_config: |+ experimental-features = nix-command flakes+ substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org+ trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}+ trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=++ - name: Run cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }}+ run: |+ nix build ".#checks.x86_64-linux.cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }}" \+ --print-build-logs --show-trace \+ -o "result-${{ matrix.desktop }}-bg-gui-${{ matrix.app }}"++ - name: Collect artifacts+ if: always()+ run: |+ mkdir -p artifacts+ find -L "result-${{ matrix.desktop }}-bg-gui-${{ matrix.app }}/" \+ \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f \+ -exec cp {} artifacts/ \; 2>/dev/null || true+ ls -la artifacts/ 2>/dev/null || echo "No artifacts found"++ - name: Upload artifacts+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }}+ path: artifacts/*+ if-no-files-found: warn++ - name: Sign and upload to Nix cache+ if: always()+ run: |+ nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all || true+ nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L || true++ - name: Cleanup signing key+ if: always()+ run: rm -f "${{ runner.temp }}/signing-key.sec"diff --git a/flake.nix b/flake.nixindex dbf8b5ce4c..ce510e3ae5 100644--- a/flake.nix+++ b/flake.nix@@ -156,6 +156,74 @@"electron-logseq"])+ )+ # Native-Wayland TDD matrix — reproduce the cua-driver scenarios on+ # real, NATIVE Wayland sessions (XFCE on labwc/wayfire/sway, plus KDE+ # and GNOME). Apps run as Wayland clients and the tests never set+ # DISPLAY, so the X11-only driver cannot see them: this is a RED suite+ # specifying native Wayland support. One check per (desktop × scenario)+ # and per (desktop × background-GUI app). See+ # nix/cua-driver/tests/wayland/README.md.+ // pkgs.lib.optionalAttrs (system == "x86_64-linux") (+ let+ # NOTE: xfce-wayfire dropped — the wayfire package fails to+ # build in the current nixpkgs pin (wf-config can't link+ # -ldoctest), an upstream packaging bug unrelated to cua-driver.+ # labwc + sway still cover XFCE-on-wlroots.+ waylandDesktops = [+ "xfce-labwc"+ "xfce-sway"+ "kde"+ "gnome"+ ];+ waylandScenarios = {+ integration = ./nix/cua-driver/tests/wayland/integration.nix;+ screenshot = ./nix/cua-driver/tests/wayland/screenshot.nix;+ cursor-click-gif = ./nix/cua-driver/tests/wayland/cursor-click-gif.nix;+ background-terminal-gif = ./nix/cua-driver/tests/wayland/background-terminal-gif.nix;+ parallel-drag = ./nix/cua-driver/tests/wayland/parallel-drag.nix;+ };+ waylandBgApps = [+ "foot"+ "gtk3-gedit"+ "qt6-kcalc"+ ];+ waylandModule = {+ imports = [ ./nix/cua-driver/module.nix ];+ services.cua-driver.package = cuaDriverPackage;+ };+ scenarioChecks = pkgs.lib.listToAttrs (+ pkgs.lib.concatMap (+ desktop:+ map (+ scenario:+ pkgs.lib.nameValuePair "cua-driver-wayland-${desktop}-${scenario}" (+ import waylandScenarios.${scenario} {+ inherit pkgs desktop;+ inherit (pkgs) lib;+ cuaDriverModule = waylandModule;+ }+ )+ ) (builtins.attrNames waylandScenarios)+ ) waylandDesktops+ );+ bgGuiChecks = pkgs.lib.listToAttrs (+ pkgs.lib.concatMap (+ desktop:+ map (+ app:+ pkgs.lib.nameValuePair "cua-driver-wayland-${desktop}-background-gui-${app}" (+ import ./nix/cua-driver/tests/wayland/background-gui.nix {+ inherit pkgs desktop app;+ inherit (pkgs) lib;+ cuaDriverModule = waylandModule;+ }+ )+ ) waylandBgApps+ ) waylandDesktops+ );+ in+ scenarioChecks // bgGuiChecks);})diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lockindex 40909f85cf..5133676757 100644--- a/libs/cua-driver/rust/Cargo.lock+++ b/libs/cua-driver/rust/Cargo.lock@@ -620,6 +620,12 @@ dependencies = ["futures-util",]+[[package]]+name = "downcast-rs"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"+[[package]]name = "embed-manifest"version = "1.5.0"@@ -1630,6 +1636,8 @@ dependencies = ["tiny-skia","tokio","tracing",+ "wayland-client",+ "wayland-protocols-wlr","x11","x11rb",]@@ -2936,6 +2944,76 @@ dependencies = ["semver",]+[[package]]+name = "wayland-backend"+version = "0.3.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d"+dependencies = [+ "cc",+ "downcast-rs",+ "rustix",+ "smallvec",+ "wayland-sys",+]++[[package]]+name = "wayland-client"+version = "0.31.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144"+dependencies = [+ "bitflags 2.11.1",+ "rustix",+ "wayland-backend",+ "wayland-scanner",+]++[[package]]+name = "wayland-protocols"+version = "0.32.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f"+dependencies = [+ "bitflags 2.11.1",+ "wayland-backend",+ "wayland-client",+ "wayland-scanner",+]++[[package]]+name = "wayland-protocols-wlr"+version = "0.3.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234"+dependencies = [+ "bitflags 2.11.1",+ "wayland-backend",+ "wayland-client",+ "wayland-protocols",+ "wayland-scanner",+]++[[package]]+name = "wayland-scanner"+version = "0.31.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a"+dependencies = [+ "proc-macro2",+ "quick-xml",+ "quote",+]++[[package]]+name = "wayland-sys"+version = "0.31.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be"+dependencies = [+ "pkg-config",+]+[[package]]name = "webpki-roots"version = "1.0.7"diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.tomlindex 7d530b4874..77515608c1 100644--- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml+++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml@@ -33,3 +33,8 @@ atspi = { version = "0.30", features = ["tokio", "zbus"] }evdev = "0.12"# Math-expression evaluator for parallel_mouse_drag `fn` paths (y = f(x)).meval = "0.2"+# Native Wayland backend (window enumeration via wlr-foreign-toplevel-management;+# capture/input protocols added as those slices land). Pure-Rust client; used+# when running under a Wayland compositor with no X11 (no DISPLAY).+wayland-client = "0.31"+wayland-protocols-wlr = { version = "0.3", features = ["client"] }diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rsindex 36fbcf15a9..8fb7dafa2e 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs@@ -37,7 +37,12 @@ pub mod capture;#[cfg(target_os = "linux")]pub mod atspi;+#[cfg(target_os = "linux")]+pub mod wayland;+pub fn register_tools() -> ToolRegistry {+ #[cfg(target_os = "linux")]+ wayland::ensure_nested_session();tools::build_registry(false)}@@ -46,6 +51,8 @@ pub fn register_tools() -> ToolRegistry {/// (pid + window_id required, JPEG @ 85%, text note pointing at pixel/// tools). See `tools::impl_::ScreenshotCompatTool`.pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig, compat: bool) -> ToolRegistry {+ #[cfg(target_os = "linux")]+ wayland::ensure_nested_session();if cfg.enabled {overlay::init(cfg.clone());overlay::run_on_thread();diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex a89d397ab6..7e10040d7f 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -314,7 +314,7 @@ impl Tool for ListWindowsTool {async fn invoke(&self, args: Value) -> ToolResult {use cua_driver_core::tool_args::ArgsExt;let filter_pid = args.opt_u64("pid").map(|v| v as u32);- let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(filter_pid)).await.unwrap_or_default();+ let windows = tokio::task::spawn_blocking(move || crate::wayland::list_windows_dispatch(filter_pid)).await.unwrap_or_default();let mut lines = vec![format!("Found {} windows:", windows.len())];for w in &windows {lines.push(format!(" [xid={}] pid={:?} \"{}\" {}x{}+{}+{}",@@ -386,7 +386,7 @@ impl Tool for GetWindowStateTool {Vec::new()};let screenshot = if do_shot {- match crate::capture::screenshot_window_bytes(xid) {+ match crate::wayland::screenshot_dispatch(xid) {Ok(raw) => {let orig_w = crate::capture::png_dimensions_pub(&raw).map(|(w, _)| w).unwrap_or(0);let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;@@ -943,6 +943,13 @@ impl Tool for ClickTool {let (xi, yi) = (x as i32, y as i32);let result = tokio::task::spawn_blocking(move || {+ // Native Wayland: focus+raise the target toplevel (foreign-toplevel+ // `activate`). Wayland hides cross-window geometry, so window-local+ // x/y can't be mapped to a global pointer position; activating the+ // window the caller targeted is the focus-based equivalent.+ if crate::wayland::is_wayland() {+ return crate::wayland::click(xid);+ }crate::input::send_click(xid, xi, yi, count, button)}).await;match result {@@ -997,6 +1004,39 @@ impl Tool for TypeTextTool {}}};++ // EIS nested compositor: focus-FREE per-surface typing into window_id+ // (the target need not be focused). Routed over the inject control socket.+ if crate::wayland::is_inject_mode() {+ let text_len = text.chars().count();+ let text_w = text.clone();+ let result =+ tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w)).await;+ return match result {+ Ok(Ok(())) => ToolResult::text(format!(+ "Typed {text_len} character(s) (focus-free via EIS compositor)."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }++ // Native Wayland: keys go to the *focused* surface (no pid/window+ // targeting in the protocol). Type via the virtual-keyboard tool; pair+ // with a prior `click`/`activate` to focus the intended window.+ if crate::wayland::is_wayland() {+ let text_len = text.chars().count();+ let text_w = text.clone();+ let result =+ tokio::task::spawn_blocking(move || crate::wayland::type_text(&text_w)).await;+ return match result {+ Ok(Ok(())) => ToolResult::text(format!(+ "Typed {text_len} character(s) (via Wayland virtual-keyboard)."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }// Pulse the agent cursor onto the field being typed into (when an// element_index is supplied) so the viewer sees *where* typing happens.if let Some(idx) = args.opt_u64("element_index") {@@ -1124,6 +1164,31 @@ impl Tool for PressKeyTool {}}};+ // EIS nested compositor: focus-free named-key into window_id.+ if crate::wayland::is_inject_mode() {+ let key_w = key.clone();+ let result =+ tokio::task::spawn_blocking(move || crate::wayland::inject_press_key(xid, &key_w)).await;+ return match result {+ Ok(Ok(())) => ToolResult::text(format!("Pressed key '{key}' (focus-free via EIS compositor).")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }++ // Native Wayland: send the key to the focused surface via virtual-keyboard.+ if crate::wayland::is_wayland() {+ let key_w = key.clone();+ let result = tokio::task::spawn_blocking(move || crate::wayland::press_key(&key_w)).await;+ return match result {+ Ok(Ok(())) => ToolResult::text(format!(+ "Pressed key '{key}' (via Wayland virtual-keyboard)."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+let key_for_task = key.clone();let result = tokio::task::spawn_blocking(move || {if mods.is_empty() && key_for_task.eq_ignore_ascii_case("enter") {@@ -2066,6 +2131,55 @@ pub struct ParallelMouseDragTool {}static PMDRAG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();+/// EIS-compositor path for parallel_mouse_drag: build window-local drag paths+/// and run them as concurrent multi-cursor injections over the control socket.+/// Coordinates stay window-local (the compositor maps them per app_id), so no+/// X11 geometry/MPX is needed — the X11 path's hard blocker on Wayland.+async fn parallel_drag_inject(args: &Value) -> ToolResult {+ let Some(items) = args.get("drags").and_then(|v| v.as_array()) else {+ return ToolResult::error("drags[] is required.");+ };+ if items.len() < 2 {+ return ToolResult::error("parallel_mouse_drag requires at least two drag items.");+ }+ let mut drags = Vec::with_capacity(items.len());+ for (i, item) in items.iter().enumerate() {+ let Some(xid) = item.get("window_id").and_then(|v| v.as_u64()) else {+ return ToolResult::error("each drag item requires window_id.");+ };+ let local: Vec<(f64, f64)> = if let Some(pts) = item.get("path").and_then(|v| v.as_array()) {+ pts.iter()+ .filter_map(|p| {+ let a = p.as_array()?;+ Some((a.first()?.as_f64()?, a.get(1)?.as_f64()?))+ })+ .collect()+ } else {+ let g = |k: &str| item.get(k).and_then(|v| v.as_f64());+ match (g("from_x"), g("from_y"), g("to_x"), g("to_y")) {+ (Some(fx), Some(fy), Some(tx), Some(ty)) => vec![(fx, fy), (tx, ty)],+ _ => return ToolResult::error("each drag item requires path[] or from_x/from_y/to_x/to_y."),+ }+ };+ if local.len() < 2 {+ return ToolResult::error("drag path needs at least 2 points.");+ }+ let steps = item.get("steps").and_then(|v| v.as_u64()).unwrap_or(60).clamp(1, 300) as usize;+ let x_button = parse_mouse_button(item.get("button").and_then(|v| v.as_str()).unwrap_or("left")) as u32;+ let app = match tokio::task::spawn_blocking(move || crate::wayland::app_id_for_window(xid)).await {+ Ok(Some(a)) => a,+ _ => return ToolResult::error(format!("no Wayland app_id for window {xid}")),+ };+ drags.push(crate::wayland::InjectDrag { app_id: app, idx: i, x_button, path: local, steps });+ }+ let n = drags.len();+ match tokio::task::spawn_blocking(move || crate::wayland::inject_parallel_drags(&drags)).await {+ Ok(Ok(())) => ToolResult::text(format!("Ran {n} concurrent drags (multi-cursor via EIS compositor).")),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ }+}+#[async_trait]impl Tool for ParallelMouseDragTool {fn def(&self) -> &ToolDef {@@ -2100,6 +2214,11 @@ impl Tool for ParallelMouseDragTool {}async fn invoke(&self, args: Value) -> ToolResult {+ // EIS nested compositor: run the drags as concurrent multi-cursor+ // injections (window-local, no X11 MPX/geometry needed).+ if crate::wayland::is_inject_mode() {+ return parallel_drag_inject(&args).await;+ }match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await {Ok(Ok(())) => {}Ok(Err(e)) => return ToolResult::error(e.to_string()),@@ -2675,14 +2794,19 @@ impl Tool for CheckPermissionsTool {let atspi_ok = std::env::var("DBUS_SESSION_BUS_ADDRESS").is_ok()|| std::path::Path::new("/run/user").exists();+ let wayland_display = std::env::var("WAYLAND_DISPLAY").ok();let status_text = format!(- "X11 display: {}\nAT-SPI (D-Bus): {}\nXSendEvent injection: {}",+ "X11 display: {}\nWayland: {}\nAT-SPI (D-Bus): {}\nXSendEvent injection: {}",if x11_ok { "✅ connected" } else { "❌ DISPLAY not set or X11 unavailable" },+ match &wayland_display {+ Some(s) => format!("✅ native Wayland session (WAYLAND_DISPLAY={s})"),+ None => "❌ not a Wayland session".to_string(),+ },if atspi_ok { "✅ D-Bus session available" } else { "⚠️ D-Bus session not detected" },if x11_ok { "✅ available" } else { "❌ requires X11" });ToolResult::text(status_text)- .with_structured(json!({ "x11": x11_ok, "atspi": atspi_ok, "xsend_event": x11_ok }))+ .with_structured(json!({ "x11": x11_ok, "wayland": wayland_display.is_some(), "atspi": atspi_ok, "xsend_event": x11_ok }))}}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rsnew file mode 100644index 0000000000..c5a2a8979e--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs@@ -0,0 +1,594 @@+//! Native-Wayland backend.+//!+//! Used when running under a Wayland compositor with no X11 (WAYLAND_DISPLAY+//! set, DISPLAY unset). The first slice enumerates toplevels via+//! `zwlr_foreign_toplevel_manager_v1` (wlroots: labwc, sway, …) so+//! `list_windows` works on native Wayland. Capture and input slices land next.++use std::collections::HashMap;++use wayland_client::{+ event_created_child,+ protocol::{wl_output::{self, WlOutput}, wl_pointer::ButtonState, wl_registry, wl_seat::WlSeat},+ Connection, Dispatch, Proxy, QueueHandle,+};+use wayland_protocols_wlr::foreign_toplevel::v1::client::{+ zwlr_foreign_toplevel_handle_v1::{self as ftl_handle, ZwlrForeignToplevelHandleV1},+ zwlr_foreign_toplevel_manager_v1::{+ self as ftl_manager, ZwlrForeignToplevelManagerV1, EVT_TOPLEVEL_OPCODE,+ },+};+use wayland_protocols_wlr::virtual_pointer::v1::client::{+ zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,+ zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,+};++/// Linux evdev BTN_LEFT — the button code the virtual-pointer protocol expects.+const BTN_LEFT: u32 = 0x110;++use crate::x11::WindowInfo;++/// True when we should drive Wayland rather than X11: a Wayland display is+/// present and there is no X11 DISPLAY to fall back to.+pub fn is_wayland() -> bool {+ std::env::var_os("WAYLAND_DISPLAY").is_some() && std::env::var_os("DISPLAY").is_none()+}++fn wl_sockets(dir: &str) -> std::collections::HashSet<String> {+ std::fs::read_dir(dir)+ .into_iter()+ .flatten()+ .flatten()+ .filter_map(|e| e.file_name().into_string().ok())+ .filter(|n| n.strip_prefix("wayland-").is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit())))+ .collect()+}++/// "Bring your own compositor": if `CUA_WAYLAND_NEST` is set, spawn a private+/// **headless wlroots compositor** (labwc by default) and point this process —+/// and therefore every app it launches (`launch_app`), every capture (`grim`),+/// and all enumeration/injection — at it via `WAYLAND_DISPLAY`. This lets+/// cua-driver automate apps in its **own** Wayland session on ANY host,+/// including KDE (kwin) and GNOME (mutter) which expose no client protocols for+/// this, without ever touching the host compositor or its focus. Idempotent.+pub fn ensure_nested_session() {+ use std::sync::OnceLock;+ static DONE: OnceLock<()> = OnceLock::new();+ if std::env::var_os("CUA_WAYLAND_NEST").is_none() {+ return;+ }+ DONE.get_or_init(|| {+ let xdg = std::env::var("XDG_RUNTIME_DIR").unwrap_or_else(|_| "/run/user/0".into());+ let comp = std::env::var("CUA_WAYLAND_NEST_COMPOSITOR").unwrap_or_else(|_| "labwc".into());+ let before = wl_sockets(&xdg);+ let spawned = std::process::Command::new(&comp)+ .env("WLR_BACKENDS", "headless")+ .env("WLR_RENDERER", "pixman")+ .env("WLR_RENDERER_ALLOW_SOFTWARE", "1")+ .env("WLR_LIBINPUT_NO_DEVICES", "1")+ .env("WLR_HEADLESS_OUTPUTS", "1")+ .env_remove("WAYLAND_DISPLAY") // headless: do not nest into the host compositor+ .env_remove("DISPLAY")+ .stdin(std::process::Stdio::null())+ .stdout(std::process::Stdio::null())+ .stderr(std::process::Stdio::null())+ .spawn();+ match spawned {+ Ok(child) => {+ std::mem::forget(child); // keep the compositor alive for our lifetime+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);+ loop {+ if let Some(sock) = wl_sockets(&xdg).difference(&before).min().cloned() {+ std::env::set_var("WAYLAND_DISPLAY", &sock);+ std::env::remove_var("DISPLAY");+ // Publish the nested socket so external tools (e.g. a+ // `grim` recorder) can target the same session we drive.+ let _ = std::fs::write(format!("{xdg}/.cua-nested-display"), &sock);+ tracing::info!("cua nested compositor '{comp}' up: WAYLAND_DISPLAY={sock}");+ break;+ }+ if std::time::Instant::now() >= deadline {+ tracing::error!("cua nested compositor '{comp}': no Wayland socket appeared");+ break;+ }+ std::thread::sleep(std::time::Duration::from_millis(200));+ }+ }+ Err(e) => tracing::error!("cua nested compositor '{comp}' spawn failed: {e}"),+ }+ });+}++#[derive(Default)]+struct Toplevel {+ title: String,+ app_id: String,+ closed: bool,+}++#[derive(Default)]+struct State {+ manager: Option<ZwlrForeignToplevelManagerV1>,+ toplevels: HashMap<u32, Toplevel>,+ // Live handles + a seat, kept so `click` can `activate` a target toplevel by+ // its window_id (foreign-toplevel protocol id) — the focus-based input model.+ handles: HashMap<u32, ZwlrForeignToplevelHandleV1>,+ seat: Option<WlSeat>,+ // Virtual-pointer manager + output dimensions, so `click` can land a real+ // button press at the output centre (over the just-activated window).+ vptr_manager: Option<ZwlrVirtualPointerManagerV1>,+ output_w: u32,+ output_h: u32,+}++impl Dispatch<wl_registry::WlRegistry, ()> for State {+ fn event(+ state: &mut Self,+ registry: &wl_registry::WlRegistry,+ event: wl_registry::Event,+ _: &(),+ _: &Connection,+ qh: &QueueHandle<Self>,+ ) {+ if let wl_registry::Event::Global { name, interface, version } = event {+ if interface == ZwlrForeignToplevelManagerV1::interface().name {+ let v = version.min(3);+ state.manager = Some(registry.bind::<ZwlrForeignToplevelManagerV1, _, _>(name, v, qh, ()));+ } else if interface == WlSeat::interface().name {+ let v = version.min(7);+ state.seat = Some(registry.bind::<WlSeat, _, _>(name, v, qh, ()));+ } else if interface == ZwlrVirtualPointerManagerV1::interface().name {+ state.vptr_manager =+ Some(registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(name, version.min(2), qh, ()));+ } else if interface == WlOutput::interface().name {+ registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ());+ }+ }+ }+}++impl Dispatch<WlSeat, ()> for State {+ fn event(+ _: &mut Self,+ _: &WlSeat,+ _: wayland_client::protocol::wl_seat::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ // Seat name/capabilities events are irrelevant here — we only need the+ // seat object to pass to foreign-toplevel `activate`.+ }+}++impl Dispatch<WlOutput, ()> for State {+ fn event(+ state: &mut Self,+ _: &WlOutput,+ event: wl_output::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ // Remember the output resolution so `click` can aim at its centre.+ if let wl_output::Event::Mode { width, height, .. } = event {+ state.output_w = width.max(0) as u32;+ state.output_h = height.max(0) as u32;+ }+ }+}++impl Dispatch<ZwlrVirtualPointerManagerV1, ()> for State {+ fn event(+ _: &mut Self,+ _: &ZwlrVirtualPointerManagerV1,+ _: <ZwlrVirtualPointerManagerV1 as Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ }+}++impl Dispatch<ZwlrVirtualPointerV1, ()> for State {+ fn event(+ _: &mut Self,+ _: &ZwlrVirtualPointerV1,+ _: <ZwlrVirtualPointerV1 as Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ }+}++impl Dispatch<ZwlrForeignToplevelManagerV1, ()> for State {+ fn event(+ _state: &mut Self,+ _: &ZwlrForeignToplevelManagerV1,+ _event: ftl_manager::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ // The `toplevel` event creates a handle object (see event_created_child!);+ // the handle's own events carry the title/app_id we collect below.+ }++ event_created_child!(State, ZwlrForeignToplevelManagerV1, [+ EVT_TOPLEVEL_OPCODE => (ZwlrForeignToplevelHandleV1, ()),+ ]);+}++impl Dispatch<ZwlrForeignToplevelHandleV1, ()> for State {+ fn event(+ state: &mut Self,+ handle: &ZwlrForeignToplevelHandleV1,+ event: ftl_handle::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ let id = handle.id().protocol_id();+ state.handles.entry(id).or_insert_with(|| handle.clone());+ let tl = state.toplevels.entry(id).or_default();+ match event {+ ftl_handle::Event::Title { title } => tl.title = title,+ ftl_handle::Event::AppId { app_id } => tl.app_id = app_id,+ ftl_handle::Event::Closed => tl.closed = true,+ _ => {}+ }+ }+}++/// Enumerate native Wayland toplevels via wlr-foreign-toplevel-management.+/// `xid` is the foreign-toplevel handle's protocol id (a stable per-session+/// window id); pid is unknown (not exposed by the protocol); geometry is 0+/// (the protocol does not surface position/size). app_id is folded into the+/// title (`"<title> [<app_id>]"`) so callers matching on either still match.+pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {+ let conn = Connection::connect_to_env()?;+ let mut queue = conn.new_event_queue::<State>();+ let qh = queue.handle();+ conn.display().get_registry(&qh, ());++ let mut state = State::default();+ queue.roundtrip(&mut state)?; // registry globals -> bind manager+ if state.manager.is_none() {+ anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");+ }+ // Manager emits a `toplevel` per window; each handle then emits title/app_id+ // and a `done`. A few roundtrips drain the initial enumeration.+ for _ in 0..4 {+ queue.roundtrip(&mut state)?;+ }++ let mut out = Vec::new();+ for (id, tl) in &state.toplevels {+ if tl.closed {+ continue;+ }+ let title = if tl.app_id.is_empty() {+ tl.title.clone()+ } else {+ format!("{} [{}]", tl.title, tl.app_id)+ };+ out.push(WindowInfo { xid: *id as u64, pid: None, title, x: 0, y: 0, width: 0, height: 0 });+ }+ Ok(out)+}++/// Capture the Wayland output as PNG bytes via `grim` (the wlroots screenshot+/// tool — wlr-screencopy under the hood, works on labwc/sway). This mirrors the+/// X11 backend shelling out to `import`/`xwd`. foreign-toplevel exposes no+/// per-window geometry, so this captures the whole output; that is sufficient+/// for `get_window_state`'s vision payload.+pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> {+ let out = std::process::Command::new("grim").args(["-t", "png", "-"]).output()?;+ if !out.status.success() {+ anyhow::bail!("grim failed: {}", String::from_utf8_lossy(&out.stderr));+ }+ if out.stdout.is_empty() {+ anyhow::bail!("grim produced no output");+ }+ Ok(out.stdout)+}++/// Capture dispatcher: native Wayland (grim) when applicable, else X11.+pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {+ if is_wayland() {+ screenshot_bytes()+ } else {+ crate::capture::screenshot_window_bytes(xid)+ }+}++/// Click a native Wayland toplevel identified by its `window_id` (the+/// foreign-toplevel protocol id from `list_windows`). Wayland forbids a client+/// from knowing another window's on-screen geometry, so we cannot map the+/// caller's window-local x/y to a global pointer position. Instead, the+/// focused-input model is two steps:+/// 1. `activate` the target toplevel (foreign-toplevel) — focus+raise it, so+/// on a stacking layout it fills the output.+/// 2. land a real virtual-pointer button press at the output centre (now over+/// the target). This both visually clicks and — crucially on sway, whose+/// headless seat has no keyboard until a pointer interaction occurs — makes+/// the compositor route the subsequent virtual-keyboard input to the+/// target. (labwc routes it from `activate` alone; the centre click is+/// harmless there since the activated window is centred under the cursor.)+pub fn click(window_id: u64) -> anyhow::Result<()> {+ let conn = Connection::connect_to_env()?;+ let mut queue = conn.new_event_queue::<State>();+ let qh = queue.handle();+ conn.display().get_registry(&qh, ());++ let mut state = State::default();+ queue.roundtrip(&mut state)?; // bind manager + seat + vptr + outputs+ if state.manager.is_none() {+ anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");+ }+ for _ in 0..4 {+ queue.roundtrip(&mut state)?; // drain toplevel/handle creation + output mode+ }++ let id = window_id as u32;+ let handle = state+ .handles+ .get(&id)+ .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {window_id}"))?+ .clone();+ let seat = state+ .seat+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat to activate the window"))?;+ handle.activate(&seat);+ queue.roundtrip(&mut state)?; // flush activate so the target is focused/raised++ // Land a button press at the output centre (over the now-focused window).+ if let Some(mgr) = state.vptr_manager.clone() {+ let (w, h) = (state.output_w.max(1), state.output_h.max(1));+ let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());+ vptr.motion_absolute(0, w / 2, h / 2, w, h);+ vptr.frame();+ vptr.button(0, BTN_LEFT, ButtonState::Pressed);+ vptr.frame();+ vptr.button(0, BTN_LEFT, ButtonState::Released);+ vptr.frame();+ queue.roundtrip(&mut state)?;+ vptr.destroy();+ queue.roundtrip(&mut state)?;+ }+ Ok(())+}++/// Type Unicode text into the focused Wayland surface via `wtype` (the+/// virtual-keyboard tool — `zwp_virtual_keyboard_v1` under the hood; it builds+/// the xkb keymap and resolves shift levels for us). This mirrors the X11+/// backend's XSendEvent typing and the capture slice's shell-out to `grim`.+/// foreign-toplevel exposes no pid and Wayland delivers keys to the *focused*+/// surface, so this is window_id-free; pair it with `click`/`activate` to put+/// the intended window in focus first.+pub fn type_text(text: &str) -> anyhow::Result<()> {+ if text.is_empty() {+ return Ok(());+ }+ // Lead with a no-op Shift_L tap: on a freshly-focused window under a headless+ // seat (notably sway), the compositor needs the first virtual-keyboard event+ // to wire up keyboard routing, and that first key is dropped. Sacrificing a+ // modifier tap (no character) absorbs the drop so the real text lands intact;+ // it's harmless where routing is already live (labwc).+ let out = std::process::Command::new("wtype")+ .args(["-k", "Shift_L", "--"])+ .arg(text)+ .output()?;+ if !out.status.success() {+ anyhow::bail!("wtype failed: {}", String::from_utf8_lossy(&out.stderr));+ }+ Ok(())+}++/// Press a single named key into the focused Wayland surface via `wtype -k`.+pub fn press_key(key: &str) -> anyhow::Result<()> {+ let keysym = key_to_keysym(key);+ let out = std::process::Command::new("wtype").args(["-k", &keysym]).output()?;+ if !out.status.success() {+ anyhow::bail!("wtype -k {keysym} failed: {}", String::from_utf8_lossy(&out.stderr));+ }+ Ok(())+}++/// Map cua key names to X keysym names that `wtype -k` understands. Unknown+/// values pass through (single characters and valid keysym names work as-is).+fn key_to_keysym(key: &str) -> String {+ match key.to_lowercase().as_str() {+ "enter" | "return" => "Return",+ "tab" => "Tab",+ "esc" | "escape" => "Escape",+ "space" => "space",+ "backspace" => "BackSpace",+ "delete" | "del" => "Delete",+ "up" => "Up",+ "down" => "Down",+ "left" => "Left",+ "right" => "Right",+ "home" => "Home",+ "end" => "End",+ "pageup" | "page_up" => "Prior",+ "pagedown" | "page_down" => "Next",+ _ => return key.to_string(),+ }+ .to_string()+}++// ── EIS nested-compositor injection ────────────────────────────────────────+//+// When cua-driver's nested compositor is `cua-compositor` (our patched wlroots,+// see nix/cua-driver/compositor/), it exposes a line-protocol control socket at+// $CUA_INJECT_SOCKET for what stock Wayland forbids: focus-FREE per-surface+// keyboard injection and MULTI-cursor pointer injection, both routed to a target+// window by its xdg app_id. These helpers speak that protocol.++/// The control socket path, when running against the EIS nested compositor.+pub fn inject_socket_path() -> Option<String> {+ std::env::var("CUA_INJECT_SOCKET").ok().filter(|s| !s.is_empty())+}++/// True when input should be routed through the EIS compositor's control socket+/// (focus-free / multi-cursor) rather than wtype / virtual-pointer.+pub fn is_inject_mode() -> bool {+ inject_socket_path().is_some()+}++fn inject_send(lines: &[String]) -> anyhow::Result<()> {+ use std::io::Write;+ use std::os::unix::net::UnixStream;+ let path = inject_socket_path().ok_or_else(|| anyhow::anyhow!("CUA_INJECT_SOCKET not set"))?;+ // The nested compositor may still be starting; retry the connect briefly.+ let mut stream = None;+ for _ in 0..60 {+ match UnixStream::connect(&path) {+ Ok(s) => { stream = Some(s); break; }+ Err(_) => std::thread::sleep(std::time::Duration::from_millis(50)),+ }+ }+ let mut s = stream.ok_or_else(|| anyhow::anyhow!("could not connect to inject socket {path}"))?;+ let mut buf = String::new();+ for l in lines {+ buf.push_str(l);+ buf.push('\n');+ }+ s.write_all(buf.as_bytes())?;+ s.flush()?;+ // Give the compositor a moment to process before the socket closes.+ std::thread::sleep(std::time::Duration::from_millis(80));+ Ok(())+}++/// Resolve a window_id (foreign-toplevel protocol id) to its xdg app_id by+/// enumerating toplevels — the inject protocol addresses windows by app_id.+pub fn app_id_for_window(window_id: u64) -> Option<String> {+ let conn = Connection::connect_to_env().ok()?;+ let mut queue = conn.new_event_queue::<State>();+ let qh = queue.handle();+ conn.display().get_registry(&qh, ());+ let mut state = State::default();+ queue.roundtrip(&mut state).ok()?;+ for _ in 0..4 {+ queue.roundtrip(&mut state).ok()?;+ }+ state+ .toplevels+ .get(&(window_id as u32))+ .map(|t| t.app_id.clone())+ .filter(|s| !s.is_empty())+}++fn to_hex(s: &str) -> String {+ s.bytes().map(|b| format!("{b:02x}")).collect()+}++/// Map a cua/X11 mouse-button number to its evdev (wl_pointer) button code.+fn evdev_button(x_button: u32) -> u32 {+ match x_button {+ 3 => 0x111, // BTN_RIGHT+ 2 => 0x112, // BTN_MIDDLE+ _ => 0x110, // BTN_LEFT+ }+}++/// Focus-free type into the window's surface (no focus change).+pub fn inject_type_text(window_id: u64, text: &str) -> anyhow::Result<()> {+ let app = app_id_for_window(window_id)+ .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;+ inject_send(&[format!("t {app} {}", to_hex(text))])+}++/// Focus-free named-key press into the window's surface.+pub fn inject_press_key(window_id: u64, key: &str) -> anyhow::Result<()> {+ let app = app_id_for_window(window_id)+ .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;+ inject_send(&[format!("k {app} {key}")])+}++/// A single pointer drag for `inject_parallel_drags`: window-local waypoints,+/// driven by cursor `idx` so several run concurrently on one window.+pub struct InjectDrag {+ pub app_id: String,+ pub idx: usize,+ pub x_button: u32,+ pub path: Vec<(f64, f64)>,+ pub steps: usize,+}++fn resample(path: &[(f64, f64)], steps: usize) -> Vec<(f64, f64)> {+ if path.len() < 2 || steps == 0 {+ return path.to_vec();+ }+ let seglen: Vec<f64> = path+ .windows(2)+ .map(|w| ((w[1].0 - w[0].0).powi(2) + (w[1].1 - w[0].1).powi(2)).sqrt())+ .collect();+ let total: f64 = seglen.iter().sum();+ if total == 0.0 {+ return vec![path[0]; steps + 1];+ }+ let mut out = Vec::with_capacity(steps + 1);+ for s in 0..=steps {+ let target = total * (s as f64) / (steps as f64);+ let mut acc = 0.0;+ let mut pt = path[path.len() - 1];+ for (i, &l) in seglen.iter().enumerate() {+ if acc + l >= target || i == seglen.len() - 1 {+ let f = if l > 0.0 { (target - acc) / l } else { 0.0 };+ pt = (path[i].0 + (path[i + 1].0 - path[i].0) * f,+ path[i].1 + (path[i + 1].1 - path[i].1) * f);+ break;+ }+ acc += l;+ }+ out.push(pt);+ }+ out+}++/// Run N pointer drags concurrently on their target windows: each cursor presses+/// at its start, glides through its (interleaved) waypoints, and releases. This+/// is true multi-cursor — each `idx` is an independent cursor in the compositor.+pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> {+ if drags.is_empty() {+ return Ok(());+ }+ let resampled: Vec<Vec<(f64, f64)>> =+ drags.iter().map(|d| resample(&d.path, d.steps.max(1))).collect();+ let max_steps = resampled.iter().map(|p| p.len()).max().unwrap_or(0);+ let mut lines = Vec::new();+ // Press each cursor at its start point.+ for (d, pts) in drags.iter().zip(&resampled) {+ let (x, y) = pts[0];+ lines.push(format!("m {} {} {x:.1} {y:.1}", d.app_id, d.idx));+ lines.push(format!("b {} {} {} 1", d.app_id, d.idx, evdev_button(d.x_button)));+ }+ // Glide all cursors together, one interleaved step at a time.+ for s in 1..max_steps {+ for (d, pts) in drags.iter().zip(&resampled) {+ let (x, y) = pts[s.min(pts.len() - 1)];+ lines.push(format!("m {} {} {x:.1} {y:.1}", d.app_id, d.idx));+ }+ }+ // Release each cursor.+ for (d, _) in drags.iter().zip(&resampled) {+ lines.push(format!("b {} {} {} 0", d.app_id, d.idx, evdev_button(d.x_button)));+ }+ inject_send(&lines)+}++/// Window-enumeration dispatcher: native Wayland when applicable, else X11.+pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> {+ if is_wayland() {+ match list_windows() {+ Ok(ws) => return ws, // foreign-toplevel has no pid, so filter_pid can't apply+ Err(e) => tracing::warn!("wayland list_windows failed, falling back to X11: {e}"),+ }+ }+ crate::x11::list_windows(filter_pid)+}diff --git a/nix/cua-driver/compositor/cua_compositor_patch.py b/nix/cua-driver/compositor/cua_compositor_patch.pynew file mode 100644index 0000000000..e6abdaabd5--- /dev/null+++ b/nix/cua-driver/compositor/cua_compositor_patch.py@@ -0,0 +1,381 @@+#!/usr/bin/env python3+# Transforms wlroots' upstream `tinywl/tinywl.c` (0.19) into `cua-compositor.c`:+# a minimal headless wlroots compositor that cua-driver spawns as its nested+# session (CUA_WAYLAND_NEST_COMPOSITOR=cua-compositor) to perform what stock+# Wayland forbids a client from doing —+#+# * focus-FREE per-surface KEYBOARD injection (type into an unfocused window),+# * MULTI-cursor pointer injection (N independent cursors on one window),+#+# both routed to a target window by its xdg app_id, driven over a tiny line+# protocol on a unix control socket ($CUA_INJECT_SOCKET). It also exposes+# foreign-toplevel-management (so cua-driver's list_windows enumerates windows)+# and screencopy (so grim captures the output) — the same protocols labwc gives+# the non-nested cells — so the EIS cells keep working end to end.+#+# The injection primitives (cua_motion/cua_button/cua_kbd_*) deliver wl_pointer/+# wl_keyboard straight to a target client's resources, bypassing seat focus —+# routed by app_id (not positional device index) and transported over a plain+# socket instead of libei/EIS, since cua owns both ends and the portal/libei+# layer buys nothing here.+#+# Usage: cua_compositor_patch.py <tinywl.c in> <cua-compositor.c out>+import sys, io++inp = sys.argv[1] if len(sys.argv) > 1 else "tinywl.c"+out = sys.argv[2] if len(sys.argv) > 2 else "cua-compositor.c"+src = io.open(inp, encoding="utf-8").read()++# memfd_create + SOCK_CLOEXEC etc. need _GNU_SOURCE before any system header.+src = "#define _GNU_SOURCE\n" + src++INCLUDES = r"""#include <stdint.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <unistd.h>+#include <errno.h>+#include <time.h>+#include <sys/mman.h>+#include <sys/socket.h>+#include <sys/un.h>+#include <linux/input-event-codes.h>+#include <wayland-server-protocol.h>+#include <xkbcommon/xkbcommon.h>+#include <wlr/types/wlr_foreign_toplevel_management_v1.h>+#include <wlr/types/wlr_screencopy_v1.h>+#include <wlr/types/wlr_xdg_output_v1.h>++#define CUA_MAXDEV 64+/* Per-cursor / per-keyboard enter bookkeeping (idx = logical device). */+struct cua_devstate { struct wlr_surface *entered; };+static struct cua_devstate cua_ptr[CUA_MAXDEV];+static struct cua_devstate cua_kbd_state[CUA_MAXDEV];+static int g_keymap_fd = -1;+static size_t g_keymap_size = 0;+static xkb_mod_mask_t g_shift_mask = 1;+struct cua_keyent { uint32_t keycode; int shift; int valid; };+static struct cua_keyent g_chartab[128];+static struct wlr_foreign_toplevel_manager_v1 *g_ftl_mgr = NULL;++"""++# A foreign-toplevel handle pointer on each toplevel (for list_windows).+STRUCT_FIELD = (+ "\tstruct wlr_xdg_toplevel *xdg_toplevel;\n"+ "\tstruct wlr_foreign_toplevel_handle_v1 *ftl;\n"+)++FUNCS = r"""+static uint32_t cua_now_ms(void) {+ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);+ return (uint32_t)(ts.tv_sec * 1000 + ts.tv_nsec / 1000000);+}+static void cua_pframe(struct wl_resource *res) {+ if (wl_resource_get_version(res) >= WL_POINTER_FRAME_SINCE_VERSION)+ wl_pointer_send_frame(res);+}+/* Resolve a target window by its xdg app_id. */+static struct tinywl_toplevel *cua_find_appid(struct tinywl_server *server, const char *app_id) {+ struct tinywl_toplevel *t;+ wl_list_for_each(t, &server->toplevels, link) {+ const char *a = t->xdg_toplevel ? t->xdg_toplevel->app_id : NULL;+ if (a && strcmp(a, app_id) == 0) return t;+ }+ return NULL;+}+static void cua_ptr_leave(struct wlr_seat *seat, struct wlr_surface *surf) {+ if (!surf) return;+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(seat, wl_resource_get_client(surf->resource));+ if (!sc) return;+ uint32_t serial = wlr_seat_client_next_serial(sc);+ struct wl_resource *res;+ wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_leave(res, serial, surf->resource); cua_pframe(res); }+}+/* Inject pointer motion from logical cursor `idx` into window `t` at window-+ * local (x,y). enter/leave is tracked per idx, so several idx values can drive+ * independent cursors against the same or different surfaces. Focus-free: we+ * write straight to the client's wl_pointer resources, never touching the seat+ * focus or any real cursor. */+static void cua_motion(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, double x, double y) {+ if (!t || idx < 0 || idx >= CUA_MAXDEV) return;+ struct wlr_surface *surface = t->xdg_toplevel->base->surface;+ struct wlr_box geo = t->xdg_toplevel->base->geometry;+ wl_fixed_t sx = wl_fixed_from_double(geo.x + x), sy = wl_fixed_from_double(geo.y + y);+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource));+ if (!sc || wl_list_empty(&sc->pointers)) return;+ struct wl_resource *res;+ if (cua_ptr[idx].entered != surface) {+ if (cua_ptr[idx].entered) cua_ptr_leave(server->seat, cua_ptr[idx].entered);+ uint32_t es = wlr_seat_client_next_serial(sc);+ wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_enter(res, es, surface->resource, sx, sy); cua_pframe(res); }+ cua_ptr[idx].entered = surface;+ }+ uint32_t tm = cua_now_ms();+ wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_motion(res, tm, sx, sy); cua_pframe(res); }+}+static void cua_button(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, uint32_t button, bool pressed) {+ if (!t || idx < 0 || idx >= CUA_MAXDEV) return;+ struct wlr_surface *surface = t->xdg_toplevel->base->surface;+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource));+ if (!sc || wl_list_empty(&sc->pointers)) return;+ uint32_t tm = cua_now_ms(), bs = wlr_seat_client_next_serial(sc);+ struct wl_resource *res;+ wl_resource_for_each(res, &sc->pointers) {+ wl_pointer_send_button(res, bs, tm, button, pressed ? WL_POINTER_BUTTON_STATE_PRESSED : WL_POINTER_BUTTON_STATE_RELEASED);+ cua_pframe(res);+ }+}+static void cua_init_keymap(void) {+ struct xkb_context *ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS);+ struct xkb_rule_names names = {0};+ struct xkb_keymap *km = xkb_keymap_new_from_names(ctx, &names, XKB_KEYMAP_COMPILE_NO_FLAGS);+ char *s = xkb_keymap_get_as_string(km, XKB_KEYMAP_FORMAT_TEXT_V1);+ g_keymap_size = strlen(s) + 1;+ g_keymap_fd = memfd_create("cua-keymap", MFD_CLOEXEC);+ if (ftruncate(g_keymap_fd, g_keymap_size) == 0) {+ void *p = mmap(NULL, g_keymap_size, PROT_READ | PROT_WRITE, MAP_SHARED, g_keymap_fd, 0);+ memcpy(p, s, g_keymap_size); munmap(p, g_keymap_size);+ }+ free(s);+ /* Build a US-layout char -> (evdev keycode, shift-level) table from the+ * keymap so the `t` command can type arbitrary ASCII. xkb keycodes are+ * evdev+8; wl_keyboard.key wants the evdev code. */+ xkb_mod_index_t shift = xkb_keymap_mod_get_index(km, XKB_MOD_NAME_SHIFT);+ if (shift != XKB_MOD_INVALID) g_shift_mask = (xkb_mod_mask_t)1 << shift;+ for (xkb_keycode_t kc = 9; kc < 256; kc++) {+ for (int lvl = 0; lvl < 2; lvl++) {+ const xkb_keysym_t *syms;+ int n = xkb_keymap_key_get_syms_by_level(km, kc, 0, lvl, &syms);+ if (n != 1) continue;+ uint32_t cp = xkb_keysym_to_utf32(syms[0]);+ if (cp > 0 && cp < 128 && !g_chartab[cp].valid) {+ g_chartab[cp].keycode = (uint32_t)kc - 8;+ g_chartab[cp].shift = lvl;+ g_chartab[cp].valid = 1;+ }+ }+ }+ /* Control characters used by the protocol map to dedicated keys. */+ g_chartab['\n'] = (struct cua_keyent){ KEY_ENTER, 0, 1 };+ g_chartab['\t'] = (struct cua_keyent){ KEY_TAB, 0, 1 };+ xkb_keymap_unref(km); xkb_context_unref(ctx);+ wlr_log(WLR_INFO, "[cua] xkb keymap + chartab ready (%zu bytes)", g_keymap_size);+}+/* Ensure the target client's keyboard has had keymap + enter sent once (focus+ * free). idx 0 is the typing keyboard. */+static struct wlr_seat_client *cua_kbd_enter(struct tinywl_server *server, struct tinywl_toplevel *t) {+ struct wlr_surface *surface = t->xdg_toplevel->base->surface;+ struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource));+ if (!sc || wl_list_empty(&sc->keyboards)) return NULL;+ if (cua_kbd_state[0].entered != surface) {+ struct wl_resource *res; struct wl_array keys; wl_array_init(&keys);+ wl_resource_for_each(res, &sc->keyboards) {+ wl_keyboard_send_keymap(res, WL_KEYBOARD_KEYMAP_FORMAT_XKB_V1, g_keymap_fd, g_keymap_size);+ wl_keyboard_send_enter(res, wlr_seat_client_next_serial(sc), surface->resource, &keys);+ wl_keyboard_send_modifiers(res, wlr_seat_client_next_serial(sc), 0, 0, 0, 0);+ }+ wl_array_release(&keys);+ cua_kbd_state[0].entered = surface;+ }+ return sc;+}+static void cua_kbd_mods(struct wlr_seat_client *sc, uint32_t depressed) {+ struct wl_resource *res;+ wl_resource_for_each(res, &sc->keyboards)+ wl_keyboard_send_modifiers(res, wlr_seat_client_next_serial(sc), depressed, 0, 0, 0);+}+static void cua_kbd_key(struct wlr_seat_client *sc, uint32_t keycode, bool pressed) {+ uint32_t tm = cua_now_ms();+ struct wl_resource *res;+ wl_resource_for_each(res, &sc->keyboards)+ wl_keyboard_send_key(res, wlr_seat_client_next_serial(sc), tm, keycode,+ pressed ? WL_KEYBOARD_KEY_STATE_PRESSED : WL_KEYBOARD_KEY_STATE_RELEASED);+}+static void cua_type_cp(struct tinywl_server *server, struct tinywl_toplevel *t, uint32_t cp) {+ if (cp >= 128 || !g_chartab[cp].valid) return;+ struct wlr_seat_client *sc = cua_kbd_enter(server, t);+ if (!sc) return;+ struct cua_keyent e = g_chartab[cp];+ if (e.shift) cua_kbd_mods(sc, g_shift_mask);+ cua_kbd_key(sc, e.keycode, true);+ cua_kbd_key(sc, e.keycode, false);+ if (e.shift) cua_kbd_mods(sc, 0);+}+/* Decode a hex-encoded ASCII string and type it focus-free into `t`. */+static void cua_type_hex(struct tinywl_server *server, struct tinywl_toplevel *t, const char *hex) {+ if (!t) return;+ for (const char *p = hex; p[0] && p[1]; p += 2) {+ int hi = (p[0] <= '9') ? p[0] - '0' : (p[0] | 0x20) - 'a' + 10;+ int lo = (p[1] <= '9') ? p[1] - '0' : (p[1] | 0x20) - 'a' + 10;+ cua_type_cp(server, t, (uint32_t)((hi << 4) | lo));+ }+}+static void cua_key_named(struct tinywl_server *server, struct tinywl_toplevel *t, const char *name) {+ if (!t) return;+ uint32_t kc = 0;+ if (!strcasecmp(name, "enter") || !strcasecmp(name, "return")) kc = KEY_ENTER;+ else if (!strcasecmp(name, "tab")) kc = KEY_TAB;+ else if (!strcasecmp(name, "escape") || !strcasecmp(name, "esc")) kc = KEY_ESC;+ else if (!strcasecmp(name, "backspace")) kc = KEY_BACKSPACE;+ else if (!strcasecmp(name, "space")) kc = KEY_SPACE;+ else if (!strcasecmp(name, "up")) kc = KEY_UP;+ else if (!strcasecmp(name, "down")) kc = KEY_DOWN;+ else if (!strcasecmp(name, "left")) kc = KEY_LEFT;+ else if (!strcasecmp(name, "right")) kc = KEY_RIGHT;+ if (!kc) return;+ struct wlr_seat_client *sc = cua_kbd_enter(server, t);+ if (!sc) return;+ cua_kbd_key(sc, kc, true);+ cua_kbd_key(sc, kc, false);+}+/* ── control socket: one line per command, routed by app_id ───────────────── */+static void cua_handle_cmd(struct tinywl_server *server, char *line) {+ char cmd[8], app[128];+ if (sscanf(line, "%7s", cmd) != 1) return;+ if (!strcmp(cmd, "m")) {+ int idx; double x, y;+ if (sscanf(line, "m %127s %d %lf %lf", app, &idx, &x, &y) == 4)+ cua_motion(server, cua_find_appid(server, app), idx, x, y);+ } else if (!strcmp(cmd, "b")) {+ int idx; unsigned btn, pr;+ if (sscanf(line, "b %127s %d %u %u", app, &idx, &btn, &pr) == 4)+ cua_button(server, cua_find_appid(server, app), idx, btn, pr != 0);+ } else if (!strcmp(cmd, "t")) {+ char hex[8192];+ if (sscanf(line, "t %127s %8191s", app, hex) == 2)+ cua_type_hex(server, cua_find_appid(server, app), hex);+ } else if (!strcmp(cmd, "k")) {+ char key[32];+ if (sscanf(line, "k %127s %31s", app, key) == 2)+ cua_key_named(server, cua_find_appid(server, app), key);+ }+}+struct cua_conn { struct tinywl_server *server; struct wl_event_source *src; char buf[16384]; size_t len; };+/* Tear a connection down once: remove its event source (else the loop fires it+ * again on freed data -> double free), close the fd, free the state. */+static int cua_conn_drop(struct cua_conn *c, int fd) {+ if (c->src) wl_event_source_remove(c->src);+ close(fd); free(c);+ return 0;+}+static int cua_conn_readable(int fd, uint32_t mask, void *data) {+ struct cua_conn *c = data;+ if (mask & (WL_EVENT_HANGUP | WL_EVENT_ERROR)) return cua_conn_drop(c, fd);+ ssize_t n = read(fd, c->buf + c->len, sizeof(c->buf) - c->len - 1);+ if (n <= 0) return cua_conn_drop(c, fd);+ c->len += (size_t)n; c->buf[c->len] = 0;+ char *p = c->buf, *nl;+ while ((nl = memchr(p, '\n', (size_t)(c->buf + c->len - p)))) {+ *nl = 0; cua_handle_cmd(c->server, p); p = nl + 1;+ }+ size_t rem = (size_t)(c->buf + c->len - p);+ memmove(c->buf, p, rem); c->len = rem;+ wl_display_flush_clients(c->server->wl_display);+ return 0;+}+static int cua_listen_cb(int fd, uint32_t mask, void *data) {+ struct tinywl_server *server = data;+ int cfd = accept(fd, NULL, NULL);+ if (cfd < 0) return 0;+ struct cua_conn *c = calloc(1, sizeof *c); c->server = server;+ c->src = wl_event_loop_add_fd(wl_display_get_event_loop(server->wl_display), cfd,+ WL_EVENT_READABLE, cua_conn_readable, c);+ return 0;+}+static void cua_setup_control_socket(struct tinywl_server *server) {+ const char *path = getenv("CUA_INJECT_SOCKET");+ if (!path) { wlr_log(WLR_INFO, "[cua] no CUA_INJECT_SOCKET; injection disabled"); return; }+ int fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0);+ if (fd < 0) { wlr_log(WLR_ERROR, "[cua] socket(): %s", strerror(errno)); return; }+ struct sockaddr_un addr = {0}; addr.sun_family = AF_UNIX;+ snprintf(addr.sun_path, sizeof addr.sun_path, "%s", path);+ unlink(path);+ if (bind(fd, (struct sockaddr *)&addr, sizeof addr) < 0 || listen(fd, 4) < 0) {+ wlr_log(WLR_ERROR, "[cua] bind/listen %s: %s", path, strerror(errno)); close(fd); return;+ }+ cua_init_keymap();+ wl_event_loop_add_fd(wl_display_get_event_loop(server->wl_display), fd,+ WL_EVENT_READABLE, cua_listen_cb, server);+ wlr_log(WLR_INFO, "[cua] injection control socket on %s", path);+}++"""++def repl(s, old, new, label, count=1):+ if s.count(old) != count:+ sys.stderr.write("ANCHOR FAIL [%s]: found %d (want %d)\n" % (label, s.count(old), count))+ sys.exit(1)+ return s.replace(old, new, count)++# 1) includes + globals + a foreign-toplevel handle field on the toplevel.+src = repl(src, "struct tinywl_server {", INCLUDES + "struct tinywl_server {", "includes")+src = repl(src,+ "\tstruct wlr_xdg_toplevel *xdg_toplevel;\n",+ STRUCT_FIELD, "ftl-field")++# 2) injection helpers + control socket, just before main().+src = repl(src, "int main(int argc, char *argv[]) {", FUNCS + "int main(int argc, char *argv[]) {", "funcs")++# 3) On map: register a foreign-toplevel handle (title/app_id) for list_windows.+src = repl(src,+ "\twl_list_insert(&toplevel->server->toplevels, &toplevel->link);\n\n\tfocus_toplevel(toplevel);",+ "\twl_list_insert(&toplevel->server->toplevels, &toplevel->link);\n"+ "\tif (g_ftl_mgr) {\n"+ "\t\ttoplevel->ftl = wlr_foreign_toplevel_handle_v1_create(g_ftl_mgr);\n"+ "\t\tif (toplevel->xdg_toplevel->title)\n"+ "\t\t\twlr_foreign_toplevel_handle_v1_set_title(toplevel->ftl, toplevel->xdg_toplevel->title);\n"+ "\t\tif (toplevel->xdg_toplevel->app_id)\n"+ "\t\t\twlr_foreign_toplevel_handle_v1_set_app_id(toplevel->ftl, toplevel->xdg_toplevel->app_id);\n"+ "\t}\n\n\tfocus_toplevel(toplevel);",+ "ftl-on-map")++# 4) On unmap: drop the foreign-toplevel handle.+src = repl(src,+ "\twl_list_remove(&toplevel->link);\n}",+ "\tif (toplevel->ftl) { wlr_foreign_toplevel_handle_v1_destroy(toplevel->ftl); toplevel->ftl = NULL; }\n"+ "\twl_list_remove(&toplevel->link);\n}",+ "ftl-on-unmap")++# 5) On commit: keep title/app_id fresh (they often arrive after the map).+src = repl(src,+ "\t\twlr_xdg_toplevel_set_size(toplevel->xdg_toplevel, 0, 0);\n\t}",+ "\t\twlr_xdg_toplevel_set_size(toplevel->xdg_toplevel, 0, 0);\n\t}\n"+ "\tif (toplevel->ftl) {\n"+ "\t\tif (toplevel->xdg_toplevel->title)\n"+ "\t\t\twlr_foreign_toplevel_handle_v1_set_title(toplevel->ftl, toplevel->xdg_toplevel->title);\n"+ "\t\tif (toplevel->xdg_toplevel->app_id)\n"+ "\t\t\twlr_foreign_toplevel_handle_v1_set_app_id(toplevel->ftl, toplevel->xdg_toplevel->app_id);\n"+ "\t}",+ "ftl-on-commit")++# 6) Headless backend has no preferred mode -> set a real custom mode so the+# scene has a non-zero output for screencopy/grim.+src = repl(src,+ "\tstruct wlr_output_mode *mode = wlr_output_preferred_mode(wlr_output);\n\tif (mode != NULL) {\n\t\twlr_output_state_set_mode(&state, mode);\n\t}",+ "\tstruct wlr_output_mode *mode = wlr_output_preferred_mode(wlr_output);\n\tif (mode != NULL) {\n\t\twlr_output_state_set_mode(&state, mode);\n\t} else {\n"+ "\t\tint ow = getenv(\"CUA_OUTW\") ? atoi(getenv(\"CUA_OUTW\")) : 1280;\n"+ "\t\tint oh = getenv(\"CUA_OUTH\") ? atoi(getenv(\"CUA_OUTH\")) : 1024;\n"+ "\t\twlr_output_state_set_custom_mode(&state, ow, oh, 0);\n\t}",+ "headless-custom-mode")++# 7) Create the extra managers + the seat caps + the control socket in main.+src = repl(src,+ "\tserver.output_layout = wlr_output_layout_create(server.wl_display);",+ "\tserver.output_layout = wlr_output_layout_create(server.wl_display);\n"+ "\twlr_xdg_output_manager_v1_create(server.wl_display, server.output_layout);\n"+ "\twlr_screencopy_manager_v1_create(server.wl_display);\n"+ "\tg_ftl_mgr = wlr_foreign_toplevel_manager_v1_create(server.wl_display);",+ "managers")+src = repl(src,+ 'server.seat = wlr_seat_create(server.wl_display, "seat0");',+ 'server.seat = wlr_seat_create(server.wl_display, "seat0");\n'+ "\twlr_seat_set_capabilities(server.seat, WL_SEAT_CAPABILITY_POINTER | WL_SEAT_CAPABILITY_KEYBOARD);",+ "seat-caps")+src = repl(src,+ "\twl_display_run(server.wl_display);",+ "\tcua_setup_control_socket(&server);\n\twl_display_run(server.wl_display);",+ "control-socket-setup")++io.open(out, "w", encoding="utf-8").write(src)+sys.stderr.write("cua-compositor.c written (%d bytes)\n" % len(src))diff --git a/nix/cua-driver/compositor/default.nix b/nix/cua-driver/compositor/default.nixnew file mode 100644index 0000000000..9e7b6afea1--- /dev/null+++ b/nix/cua-driver/compositor/default.nix@@ -0,0 +1,62 @@+# cua-compositor: a minimal headless wlroots compositor that cua-driver spawns+# as its nested session (CUA_WAYLAND_NEST_COMPOSITOR=cua-compositor) to perform+# focus-free keyboard injection and multi-cursor pointer injection via a unix+# control socket. Built from wlroots' own upstream tinywl example (pinned to the+# same wlroots_0_19 the patch targets) transformed by cua_compositor_patch.py —+# so it tracks the exact wlroots API in nixpkgs and needs no vendored C.+{+ stdenv,+ lib,+ pkg-config,+ python3,+ wayland,+ wayland-scanner,+ wayland-protocols,+ wlroots_0_19,+ libxkbcommon,+ pixman,+}:++stdenv.mkDerivation {+ pname = "cua-compositor";+ version = "0.1.0";++ src = lib.cleanSource ./.;++ nativeBuildInputs = [+ pkg-config+ python3+ wayland-scanner+ ];+ buildInputs = [+ wayland+ wayland-protocols+ wlroots_0_19+ libxkbcommon+ pixman+ ];++ buildPhase = ''+ runHook preBuild+ cp ${wlroots_0_19.src}/tinywl/tinywl.c tinywl.c+ python3 cua_compositor_patch.py tinywl.c cua-compositor.c+ wp=${wayland-protocols}/share/wayland-protocols+ wayland-scanner server-header "$wp/stable/xdg-shell/xdg-shell.xml" xdg-shell-protocol.h+ wayland-scanner private-code "$wp/stable/xdg-shell/xdg-shell.xml" xdg-shell-protocol.c+ $CC -DWLR_USE_UNSTABLE -I. cua-compositor.c xdg-shell-protocol.c -o cua-compositor \+ $(pkg-config --cflags --libs wlroots-0.19 wayland-server xkbcommon pixman-1)+ runHook postBuild+ '';++ installPhase = ''+ runHook preInstall+ install -Dm755 cua-compositor $out/bin/cua-compositor+ runHook postInstall+ '';++ meta = {+ description = "Headless wlroots compositor with focus-free + multi-cursor input injection for cua-driver";+ platforms = lib.platforms.linux;+ mainProgram = "cua-compositor";+ };+}diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nixindex 50a41c973a..e17317e415 100644--- a/nix/cua-driver/package.nix+++ b/nix/cua-driver/package.nix@@ -15,7 +15,7 @@pkgs.rustPlatform.buildRustPackage {pname = "cua-driver";- version = "0.5.1";+ version = "0.5.3";inherit src;@@ -25,7 +25,7 @@ pkgs.rustPlatform.buildRustPackage {# gracefully via `cargo vendor`.# Bumped when the dependency set changes (added `atspi`/zbus for native# AT-SPI). If this mismatches, the nix build prints the expected value.- cargoHash = "sha256-8e/8inqyEUJA3s9lp/YGU5ckDXV1PnUVTyArwAM1e5g=";+ cargoHash = "sha256-ykVW0duMeoWho0qwDdu0k//cOY6HG+HsIBQF7iT0nZ0=";# Build only the main binary crate. The workspace also contains# platform-macos, platform-windows, cua-driver-uia, and focus-monitor-windiff --git a/nix/cua-driver/tests/wayland/README.md b/nix/cua-driver/tests/wayland/README.mdnew file mode 100644index 0000000000..33a01cd6d9--- /dev/null+++ b/nix/cua-driver/tests/wayland/README.md@@ -0,0 +1,67 @@+# cua-driver native-Wayland TDD suite++A reproduction of the cua-driver NixOS tests on **native Wayland**, across five+desktop sessions. These tests are a **TDD red suite**: the Linux backend is+X11-only today, so they are **expected to fail** until native Wayland support is+added. They exist to *specify* that work and to show, per compositor, exactly+what survives.++## Why they fail today++cua-driver enumerates windows with X11 `_NET_CLIENT_LIST`, captures with+`import`/`xwd`/XGetImage, and injects input with X11 `XSendEvent` (plus uinput ++XI2 for the agent cursor / parallel drags). On native Wayland:++- apps are **Wayland clients with no X11 window id**, so `list_windows` returns+ nothing and `find_window` times out;+- there is no X display to capture or `XSendEvent` into (the tests never set+ `DISPLAY`, so there is no XWayland fallback either).++To make a scenario green you implement the corresponding native-Wayland path in+`libs/cua-driver/rust/crates/platform-linux` (e.g. a Wayland window enumerator,+a `grim`/screencopy capture path, and a `wlr-virtual-pointer` /+`virtual-keyboard` / `libei` input path), then re-run the matching check.++## Layout++- `session.nix` — brings up a chosen desktop headless and exposes the Wayland+ socket. The compositor is the only per-desktop difference.+- `driver-client.nix` — shared MCP client; window discovery goes **only** through+ cua-driver's own `list_windows` (no xdotool/X11 cheat).+- `record-wayland-gif.nix` — `grim`-based GIF recorder (wlroots only; no-op+ elsewhere).+- `integration.nix`, `screenshot.nix`, `cursor-click-gif.nix`,+ `background-terminal-gif.nix`, `parallel-drag.nix`, `background-gui.nix` — the+ scenarios, each parameterised by `desktop` (and `background-gui` also by `app`).++## Matrix++Desktops: `xfce-labwc`, `xfce-sway` (XFCE 4.20's Wayland-session compositors),+`kde` (kwin_wayland), `gnome` (mutter headless). (`xfce-wayfire` was dropped —+wayfire fails to build in the current nixpkgs pin, an upstream `wf-config`/+`doctest` issue unrelated to cua-driver.)++Checks (built on demand):++```+nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-<scenario>+nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-background-gui-<app>+```++Scenarios: `integration`, `screenshot`, `cursor-click-gif`,+`background-terminal-gif`, `parallel-drag`.+Background-GUI apps: `foot`, `gtk3-gedit`, `qt6-kcalc`.++`parallel-drag` is the hardest: it needs X11 MPX master pointers, for which there+is no native Wayland equivalent — keep it red, or redesign around multi-seat /+virtual-pointer protocols.++## Running in CI++The `.github/workflows/nix-wayland.yml` workflow runs the whole matrix on the+same triggers as the X11 `nix-build.yml` workflow — PRs and `main` pushes that+touch `nix/**`, `flake.{nix,lock}`, `libs/cua-driver/rust/**`, or the workflow+itself, plus manual `workflow_dispatch`. The jobs are BLOCKING: a red cell fails+the PR, so the suite only goes green once native Wayland support lands. Each job+uploads its screenshots / GIFs / logs as artifacts so you can see what each+compositor did.diff --git a/nix/cua-driver/tests/wayland/background-gui.nix b/nix/cua-driver/tests/wayland/background-gui.nixnew file mode 100644index 0000000000..0df054f28b--- /dev/null+++ b/nix/cua-driver/tests/wayland/background-gui.nix@@ -0,0 +1,159 @@+# CUA Driver native-Wayland background-GUI skeleton (per desktop, per app) — TDD red.+#+# Native-Wayland analogue of the READ-ONLY skeleton class in+# ../../tests/linux-background-gui.nix. Launches a real toolkit app as a NATIVE+# Wayland client (no GDK_BACKEND/QT_QPA forcing), finds it via cua-driver+# list_windows, and screenshots it via get_window_state (vision). Records a GIF+# via grim. Fails today: the driver can neither enumerate nor capture native+# Wayland surfaces — the spec for native Wayland read support across toolkits.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-background-gui-<app>+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ desktop,+ app,+ ...+}:++let+ session = import ./session.nix { inherit pkgs desktop; };+ driverClient = import ./driver-client.nix { inherit pkgs; };+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };++ # Representative native-Wayland apps, one per major toolkit. `match` is the+ # identity substring find_window looks for (title/app_id/app/class).+ #+ # NOTE: no per-app env prefix here. Apps are launched via cua-driver's+ # `launch_app`, which execs the command's FIRST token as the program — an+ # `ENV=val cmd` prefix would be (mis)read as the program name. The native+ # Wayland backends (GDK_BACKEND/QT_QPA_PLATFORM) are exported by the session+ # driver wrapper instead (session.nix appBackendEnv), so the launched toolkit+ # apps inherit them. Do not reintroduce an `env` field on these.+ apps = {+ foot = {+ packages = [ pkgs.foot ];+ launch = "foot --app-id=cua-wayland-foot-app --title=cua-wayland-foot-app";+ match = "cua-wayland-foot-app";+ };+ "gtk3-gedit" = {+ packages = [ pkgs.gedit ];+ launch = "gedit";+ match = "gedit";+ };+ "qt6-kcalc" = {+ packages = [ pkgs.kdePackages.kcalc ];+ launch = "kcalc";+ match = "kcalc";+ };+ };++ cfg = apps.${app} or (throw "wayland/background-gui.nix: unknown app '${app}'");+ outputGif = "/tmp/cua-driver-wayland-${desktop}-background-gui-${app}.gif";+ outputPng = "/tmp/cua-driver-wayland-${desktop}-background-gui-${app}.png";++ # GIF recorder env: native desktops point grim at the host socket; nested+ # desktops leave WAYLAND_DISPLAY unset so the recorder self-resolves the+ # driver's published nested socket ($XDG_RUNTIME_DIR/.cua-nested-display).+ # No quotes around the command substitution: this string is interpolated into+ # a python double-quoted string, and the socket path has no spaces. The inner+ # `sh -c` re-parses and expands $(cat ...) itself.+ recorderWlEnv =+ if session.nested then+ "env XDG_RUNTIME_DIR=/run/user/0"+ else+ "env XDG_RUNTIME_DIR=/run/user/0 WAYLAND_DISPLAY=$(cat /tmp/wl-display)";++ testScriptPy = pkgs.writeText "wayland-bg-gui.py" ''+ import base64, sys+ sys.path.insert(0, "/tmp")+ from driver_client import Driver++ MATCH = "${cfg.match}"+ OUTPNG = "${outputPng}"+ LAUNCH = "${cfg.launch}"+ d = Driver()+ try:+ d.initialize("nixos-wayland-bg-gui")+ # Launch through cua-driver so the app lands in the session it owns+ # (host compositor on native desktops; nested labwc on kde/gnome).+ d.launch_app(LAUNCH)+ pid, wid = d.find_window(MATCH, timeout=50)+ print(f"app window pid={pid} window_id={wid}", flush=True)+ resp = d.call("get_window_state", {+ "pid": pid, "window_id": wid, "capture_mode": "vision"}, timeout=45)+ saved = False+ for item in resp.get("result", {}).get("content", []):+ if item.get("type") == "image" and item.get("data"):+ with open(OUTPNG, "wb") as f:+ f.write(base64.b64decode(item["data"]))+ saved = True+ break+ assert saved, f"no capture for native Wayland window: {resp}"+ print("background gui read test complete", flush=True)+ finally:+ d.close()+ '';+in++pkgs.testers.nixosTest {+ name = "cua-driver-wayland-${desktop}-background-gui-${app}-test";+ meta.maintainers = [ ];++ nodes.machine =+ { pkgs, ... }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 4096;+ };+ services.cua-driver.enable = true;+ boot.kernelModules = [ "uinput" ];+ hardware.graphics.enable = true;+ services.udev.extraRules = ''+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"+ '';+ environment.systemPackages =+ session.packages ++ cfg.packages ++ (with pkgs; [ python3 jq at-spi2-core ]);+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Bring up ${session.label}"):+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")+ try:+ machine.wait_for_file("/tmp/wl-ready", timeout=120)+ except Exception:+ machine.log(machine.execute("cat /tmp/session.log || true")[1])+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])+ raise++ with subtest("Launch (via cua-driver) + record GIF + read inactive app window"):+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-bg-gui.py")+ # Start the recorder first; it grabs frames while the driver launches+ # and reads the app. (Native: host socket; nested: self-resolves.)+ machine.execute(+ "sh -lc '${recorderWlEnv} ${recordGifScript} /tmp/bg-gui-frames "+ "${outputGif} /tmp/stop-bg-gui-recorder /tmp/rec-bg-gui.log 12 0.2 "+ ">/dev/null 2>&1 & echo $! >/tmp/rec-bg-gui.pid'"+ )+ result = machine.execute(+ "timeout 150 env CUA_DRIVER_BIN=${session.driverWrapper} "+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-bg-gui.py 2>&1"+ )[1]+ machine.log(result)+ # Stop + copy the GIF BEFORE asserting so even failing jobs upload one.+ machine.execute("touch /tmp/stop-bg-gui-recorder")+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-bg-gui.pid) 2>/dev/null || break; sleep 1; done'")+ machine.execute("test -e ${outputGif} || : > ${outputGif}")+ machine.copy_from_machine("${outputGif}", "")++ with subtest("cua-driver read/captured the native Wayland app window"):+ assert "background gui read test complete" in result, result+ '';+}diff --git a/nix/cua-driver/tests/wayland/background-terminal-gif.nix b/nix/cua-driver/tests/wayland/background-terminal-gif.nixnew file mode 100644index 0000000000..871d62d18b--- /dev/null+++ b/nix/cua-driver/tests/wayland/background-terminal-gif.nix@@ -0,0 +1,111 @@+# CUA Driver native-Wayland background-terminal GIF test (per desktop).+#+# The hard one: type into an UNFOCUSED window. Stock Wayland delivers keyboard+# only to the focused surface, so cua-driver nests its own cua-compositor (EIS+# mode) which injects wl_keyboard straight into a target surface by app_id,+# bypassing focus. A control terminal is launched AFTER the target so the target+# is NOT focused; cua-driver types a command into the inactive target and we+# verify it ran via a file side-effect. Records a GIF via grim.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-background-terminal-gif+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ desktop,+ ...+}:++let+ session = import ./session.nix { inherit pkgs desktop; eis = true; };+ driverClient = import ./driver-client.nix { inherit pkgs; };+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };++ testScriptPy = pkgs.writeText "wayland-background-terminal.py" ''+ import sys, time+ sys.path.insert(0, "/tmp")+ from driver_client import Driver++ d = Driver()+ try:+ d.initialize("nixos-wayland-bg-terminal")+ # Launch the TARGET first, then the CONTROL — so control holds focus and+ # the target is the inactive window we must type into focus-free.+ d.launch_app("foot --app-id=cua-wayland-target --title=cua-wayland-target")+ time.sleep(1.2)+ d.launch_app("foot --app-id=cua-wayland-control --title=cua-wayland-control")+ tpid, twid = d.find_window("cua-wayland-target", timeout=40)+ cpid, cwid = d.find_window("cua-wayland-control", timeout=40)+ print(f"target pid={tpid} window_id={twid} control pid={cpid} window_id={cwid}", flush=True)+ d.call("set_agent_cursor_enabled", {"enabled": True})+ # Focus-free type into the INACTIVE target while control stays focused.+ d.call("type_text", {"pid": tpid, "window_id": twid,+ "text": "echo hello | tee /tmp/background-hello.txt"})+ d.call("press_key", {"pid": tpid, "window_id": twid, "key": "enter"})+ time.sleep(1.8)+ print("background terminal GIF test complete", flush=True)+ finally:+ d.close()+ '';+in++pkgs.testers.nixosTest {+ name = "cua-driver-wayland-${desktop}-background-terminal-gif-test";+ meta.maintainers = [ ];++ nodes.machine =+ { pkgs, ... }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 3072;+ };+ services.cua-driver.enable = true;+ boot.kernelModules = [ "uinput" ];+ hardware.graphics.enable = true;+ services.udev.extraRules = ''+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"+ '';+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Bring up ${session.label}"):+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")+ try:+ machine.wait_for_file("/tmp/wl-ready", timeout=120)+ except Exception:+ machine.log(machine.execute("cat /tmp/session.log || true")[1])+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])+ raise++ with subtest("Record GIF + driver types focus-free into the INACTIVE Wayland terminal"):+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-background-terminal.py")+ # Recorder self-resolves the driver's nested cua-compositor socket.+ machine.execute(+ "sh -lc 'env XDG_RUNTIME_DIR=/run/user/0 ${recordGifScript} /tmp/background-frames "+ "/tmp/cua-driver-wayland-${desktop}-background-terminal.gif /tmp/stop-background-recorder "+ "/tmp/rec-background.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/rec-background.pid'"+ )+ result = machine.execute(+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-background-terminal.py 2>&1"+ )[1]+ machine.log(result)+ machine.execute("touch /tmp/stop-background-recorder")+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-background.pid) 2>/dev/null || break; sleep 1; done'")++ with subtest("Copy GIF out of the VM (best-effort)"):+ machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-background-terminal.gif || : > /tmp/cua-driver-wayland-${desktop}-background-terminal.gif")+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-background-terminal.gif", "")++ with subtest("Driver typed into inactive terminal AND the command ran"):+ assert "background terminal GIF test complete" in result, result+ machine.wait_until_succeeds("grep -Fx 'hello' /tmp/background-hello.txt", timeout=20)+ '';+}diff --git a/nix/cua-driver/tests/wayland/cursor-click-gif.nix b/nix/cua-driver/tests/wayland/cursor-click-gif.nixnew file mode 100644index 0000000000..9918d9d9d6--- /dev/null+++ b/nix/cua-driver/tests/wayland/cursor-click-gif.nix@@ -0,0 +1,125 @@+# CUA Driver native-Wayland cursor-click GIF test (per desktop).+#+# Launches two native Wayland terminals (foot) THROUGH cua-driver (launch_app) —+# the second is the focused target — then clicks it (foreign-toplevel activate),+# types a command via the Wayland virtual-keyboard (wtype), and verifies the+# command RAN by checking a file it writes (display-server-agnostic proof).+# Records a GIF of the composited output via grim. On wlroots desktops the driver+# drives the host compositor; on kde/gnome it drives its own nested labwc.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-cursor-click-gif+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ desktop,+ ...+}:++let+ session = import ./session.nix { inherit pkgs desktop; };+ driverClient = import ./driver-client.nix { inherit pkgs; };+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };++ # GIF recorder env: native points grim at the host socket; nested self-resolves+ # the driver's published nested socket ($XDG_RUNTIME_DIR/.cua-nested-display).+ recorderWlEnv =+ if session.nested then+ "env XDG_RUNTIME_DIR=/run/user/0"+ else+ "env XDG_RUNTIME_DIR=/run/user/0 WAYLAND_DISPLAY=$(cat /tmp/wl-display)";++ testScriptPy = pkgs.writeText "wayland-cursor-click.py" ''+ import sys, time+ sys.path.insert(0, "/tmp")+ from driver_client import Driver++ d = Driver()+ try:+ d.initialize("nixos-wayland-click")+ # Launch a control terminal then the target, so the target is the+ # focused (last-mapped) window the keystrokes will reach.+ d.launch_app("foot --app-id=cua-wayland-control --title=cua-wayland-control")+ time.sleep(1)+ d.launch_app("foot --app-id=cua-wayland-target --title=cua-wayland-target")+ pid, wid = d.find_window("cua-wayland-target", timeout=40)+ print(f"target pid={pid} window_id={wid}", flush=True)+ d.call("set_agent_cursor_enabled", {"enabled": True})+ d.call("move_cursor", {"x": 1100.0, "y": 900.0})+ time.sleep(0.6)+ # click focuses+raises the target (foreign-toplevel activate); type ++ # enter go to it via the Wayland virtual-keyboard.+ d.call("click", {"pid": pid, "window_id": wid, "x": 120.0, "y": 120.0})+ time.sleep(0.4)+ d.call("type_text", {"pid": pid, "window_id": wid,+ "text": "echo click-focus > /tmp/click-focus.txt"})+ d.call("press_key", {"pid": pid, "window_id": wid, "key": "enter"})+ time.sleep(1.5)+ print("click GIF test complete", flush=True)+ finally:+ d.close()+ '';+in++pkgs.testers.nixosTest {+ name = "cua-driver-wayland-${desktop}-cursor-click-gif-test";+ meta.maintainers = [ ];++ nodes.machine =+ { pkgs, ... }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 3072;+ };+ services.cua-driver.enable = true;+ boot.kernelModules = [ "uinput" ];+ hardware.graphics.enable = true;+ services.udev.extraRules = ''+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"+ '';+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")+ machine.succeed("modprobe uinput && test -e /dev/uinput")++ with subtest("Bring up ${session.label}"):+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")+ try:+ machine.wait_for_file("/tmp/wl-ready", timeout=120)+ except Exception:+ machine.log(machine.execute("cat /tmp/session.log || true")[1])+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])+ raise++ with subtest("Record GIF + driver launches, clicks, and types into a native Wayland terminal"):+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-cursor-click.py")+ machine.execute(+ "sh -lc '${recorderWlEnv} ${recordGifScript} /tmp/click-frames "+ "/tmp/cua-driver-wayland-${desktop}-cursor-click.gif /tmp/stop-click-recorder "+ "/tmp/rec-click.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/rec-click.pid'"+ )+ result = machine.execute(+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-cursor-click.py 2>&1"+ )[1]+ machine.log(result)+ machine.execute("touch /tmp/stop-click-recorder")+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-click.pid) 2>/dev/null || break; sleep 1; done'")++ with subtest("Copy GIF out of the VM (best-effort)"):+ # Ensure the path exists so copy never errors before the real assertion;+ # it may be empty on compositors where grim could not capture.+ machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-cursor-click.gif || : > /tmp/cua-driver-wayland-${desktop}-cursor-click.gif")+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-cursor-click.gif", "")++ with subtest("Driver completed click+type AND the command ran in the Wayland terminal"):+ assert "click GIF test complete" in result, result+ machine.wait_until_succeeds("test -f /tmp/click-focus.txt", timeout=20)+ '';+}diff --git a/nix/cua-driver/tests/wayland/driver-client.nix b/nix/cua-driver/tests/wayland/driver-client.nixnew file mode 100644index 0000000000..5b20fe0c52--- /dev/null+++ b/nix/cua-driver/tests/wayland/driver-client.nix@@ -0,0 +1,138 @@+# Shared MCP client used by the native-Wayland TDD tests.+#+# It is copied into the VM as /tmp/driver_client.py and imported by the small+# per-test scripts. Crucially, window discovery goes ONLY through cua-driver's+# own `list_windows` tool — there is no xdotool/X11 fallback — so on native+# Wayland (where the driver currently enumerates nothing) `find_window` times+# out and the test fails. That is the intended red state for TDD.+{ pkgs }:++pkgs.writeText "driver_client.py" ''+ import json, os, subprocess, sys, threading, time++ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")+++ class Driver:+ def __init__(self):+ self.proc = subprocess.Popen(+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ env={**os.environ},+ )+ threading.Thread(target=self._drain, daemon=True).start()+ self._id = 1++ def _drain(self):+ for line in self.proc.stderr:+ sys.stderr.buffer.write(line)+ sys.stderr.buffer.flush()++ def _send(self, method, params=None, req_id=None):+ msg = {"jsonrpc": "2.0", "method": method}+ if params is not None:+ msg["params"] = params+ if req_id is not None:+ msg["id"] = req_id+ self.proc.stdin.write((json.dumps(msg) + "\n").encode())+ self.proc.stdin.flush()++ def _recv(self, timeout=45):+ result = [None]++ def reader():+ result[0] = self.proc.stdout.readline()++ th = threading.Thread(target=reader)+ th.start()+ th.join(timeout)+ if th.is_alive():+ raise TimeoutError("no response from driver within timeout")+ line = result[0].decode().strip()+ if not line:+ raise RuntimeError("driver returned an empty response")+ return json.loads(line)++ def initialize(self, client="nixos-wayland"):+ self._send("initialize", {+ "protocolVersion": "2024-11-05", "capabilities": {},+ "clientInfo": {"name": client, "version": "1.0.0"},+ }, req_id=self._id)+ resp = self._recv()+ self._id += 1+ assert "result" in resp, f"initialize failed: {resp}"+ self._send("notifications/initialized", {})+ time.sleep(0.3)+ return resp++ def call(self, name, args, timeout=60):+ self._send("tools/call", {"name": name, "arguments": args}, req_id=self._id)+ self._id += 1+ resp = self._recv(timeout=timeout)+ if resp.get("error"):+ raise RuntimeError(f"{name} failed: {resp}")+ if resp.get("result", {}).get("isError"):+ raise RuntimeError(f"{name} returned isError: {resp}")+ return resp++ def launch_app(self, command):+ """Launch an app THROUGH cua-driver so it lands in whatever Wayland+ session the driver owns — the host compositor (native desktops) or the+ driver's own nested labwc (kde/gnome). Returns the launch response."""+ return self.call("launch_app", {"name": command})++ def list_windows(self):+ resp = self.call("list_windows", {})+ result = resp.get("result", {})+ # Preferred: MCP structuredContent = { "windows": [ {window_id, pid,+ # title, ...}, ... ] }.+ structured = result.get("structuredContent")+ if isinstance(structured, dict) and isinstance(structured.get("windows"), list):+ return structured["windows"]+ if isinstance(structured, list):+ return structured+ # Fallback: a text item that happens to be JSON.+ for item in result.get("content", []):+ if item.get("type") == "text":+ try:+ data = json.loads(item.get("text", ""))+ except Exception:+ continue+ if isinstance(data, list):+ return data+ if isinstance(data, dict) and "windows" in data:+ return data["windows"]+ return []++ def find_window(self, title_substr, timeout=30, interval=1.0):+ """Poll list_windows until a native Wayland toplevel whose title+ contains `title_substr` appears. Raises on timeout — the red point on+ a driver that cannot yet enumerate Wayland windows."""+ deadline = time.time() + timeout+ needle = title_substr.lower()+ last = []+ while time.time() < deadline:+ last = self.list_windows()+ for w in last:+ # Match across whatever identity fields a Wayland-capable+ # list_windows might surface (title today; app_id/app/class+ # once native Wayland metadata is wired up).+ hay = " ".join(+ str(w.get(k, "")) for k in ("title", "app_id", "app", "class")+ ).lower()+ if needle in hay:+ return int(w.get("pid") or 0), int(w["window_id"])+ time.sleep(interval)+ raise AssertionError(+ f"cua-driver never enumerated a Wayland window titled ~{title_substr!r}; "+ f"last list_windows() = {last}"+ )++ def close(self):+ try:+ self.proc.stdin.close()+ self.proc.terminate()+ self.proc.wait(timeout=5)+ except Exception:+ pass+''diff --git a/nix/cua-driver/tests/wayland/integration.nix b/nix/cua-driver/tests/wayland/integration.nixnew file mode 100644index 0000000000..e249141167--- /dev/null+++ b/nix/cua-driver/tests/wayland/integration.nix@@ -0,0 +1,109 @@+# CUA Driver native-Wayland integration test (per desktop).+#+# Brings up the desktop session, then launches a native Wayland terminal (foot)+# THROUGH cua-driver (launch_app) and asserts cua-driver can ENUMERATE it via+# list_windows. On native (wlroots) desktops the driver talks to the host+# compositor; on kde/gnome it drives its own nested labwc (CUA_WAYLAND_NEST=1,+# set by the session driver wrapper). Either way the app lands in the session+# the driver owns.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-integration+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ desktop,+ ...+}:++let+ session = import ./session.nix { inherit pkgs desktop; };+ driverClient = import ./driver-client.nix { inherit pkgs; };++ testScriptPy = pkgs.writeText "wayland-integration.py" ''+ import sys, time+ sys.path.insert(0, "/tmp")+ from driver_client import Driver++ d = Driver()+ try:+ d.initialize("nixos-wayland-integration")++ # Handshake sanity: required tools are advertised.+ d._send("tools/list", {}, req_id=999)+ resp = d._recv()+ names = [t["name"] for t in resp.get("result", {}).get("tools", [])]+ for t in ("list_windows", "get_window_state", "click", "type_text", "launch_app"):+ assert t in names, f"{t} not advertised: {names}"+ print(f"tools advertised: {len(names)}", flush=True)++ # Launch foot through cua-driver so it lands in the session the driver+ # owns (host compositor on native desktops; nested labwc on kde/gnome).+ d.launch_app("foot --app-id=cua-wayland-foot --title=cua-wayland-foot")++ # list_windows must surface the foot terminal the driver just launched.+ pid, wid = d.find_window("cua-wayland-foot", timeout=40)+ print(f"RESOLVED native wayland window pid={pid} window_id={wid}", flush=True)+ print("integration test complete", flush=True)+ finally:+ d.close()+ '';+in++pkgs.testers.nixosTest {+ name = "cua-driver-wayland-${desktop}-integration-test";+ meta.maintainers = [ ];++ nodes.machine =+ { pkgs, ... }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 3072;+ };+ services.cua-driver.enable = true;+ boot.kernelModules = [ "uinput" ];+ hardware.graphics.enable = true;+ services.udev.extraRules = ''+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"+ '';+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Binary exists and lists tools"):+ machine.succeed("cua-driver --help")+ tools = machine.succeed("cua-driver list-tools")+ for t in ("list_windows", "get_window_state", "click", "type_text"):+ assert t in tools, f"{t} missing from list-tools"++ with subtest("Bring up ${session.label}"):+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")+ try:+ machine.wait_for_file("/tmp/wl-ready", timeout=120)+ except Exception:+ machine.log(machine.execute("cat /tmp/session.log || true")[1])+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])+ raise+ machine.log("host WAYLAND_DISPLAY=" + machine.succeed("cat /tmp/wl-display").strip())++ with subtest("doctor reports a Wayland status line"):+ result = machine.succeed("timeout 30 ${session.driverWrapper} doctor 2>&1 || true")+ machine.log(result)+ assert "Wayland" in result, f"doctor did not report Wayland: {result}"++ with subtest("cua-driver launches + enumerates a native Wayland window"):+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-integration.py")+ result = machine.succeed(+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-integration.py 2>&1"+ )+ machine.log(result)+ assert "integration test complete" in result, result+ '';+}diff --git a/nix/cua-driver/tests/wayland/parallel-drag.nix b/nix/cua-driver/tests/wayland/parallel-drag.nixnew file mode 100644index 0000000000..7d67ce50d0--- /dev/null+++ b/nix/cua-driver/tests/wayland/parallel-drag.nix@@ -0,0 +1,104 @@+# CUA Driver native-Wayland parallel multi-cursor drag test (per desktop).+#+# Two simultaneous drag gestures on one window. Stock Wayland has a single seat+# cursor and no multi-pointer protocol, so cua-driver nests its own+# cua-compositor (EIS mode), which drives N independent logical cursors and+# injects wl_pointer per cursor by app_id. parallel_mouse_drag routes through+# the compositor's control socket (window-local coords, no X11 MPX). Records a+# GIF via grim.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-parallel-drag+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ desktop,+ ...+}:++let+ session = import ./session.nix { inherit pkgs desktop; eis = true; };+ driverClient = import ./driver-client.nix { inherit pkgs; };+ recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };++ testScriptPy = pkgs.writeText "wayland-parallel-drag.py" ''+ import sys, time+ sys.path.insert(0, "/tmp")+ from driver_client import Driver++ d = Driver()+ try:+ d.initialize("nixos-wayland-parallel-drag")+ d.launch_app("foot --app-id=cua-wayland-paint --title=cua-wayland-paint")+ pid, wid = d.find_window("cua-wayland-paint", timeout=40)+ print(f"target pid={pid} window_id={wid}", flush=True)+ d.call("parallel_mouse_drag", {"drags": [+ {"session": "agent-1", "window_id": wid, "from_x": 100.0, "from_y": 100.0, "to_x": 380.0, "to_y": 420.0, "duration_ms": 2000, "steps": 60},+ {"session": "agent-2", "window_id": wid, "from_x": 700.0, "from_y": 100.0, "to_x": 420.0, "to_y": 420.0, "duration_ms": 2000, "steps": 60},+ ]})+ time.sleep(0.6)+ print("parallel drag test complete", flush=True)+ finally:+ d.close()+ '';+in++pkgs.testers.nixosTest {+ name = "cua-driver-wayland-${desktop}-parallel-drag-test";+ meta.maintainers = [ ];++ nodes.machine =+ { pkgs, ... }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 3072;+ };+ services.cua-driver.enable = true;+ boot.kernelModules = [ "uinput" ];+ hardware.graphics.enable = true;+ services.udev.extraRules = ''+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"+ '';+ services.libinput.enable = true;+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Bring up ${session.label}"):+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")+ try:+ machine.wait_for_file("/tmp/wl-ready", timeout=120)+ except Exception:+ machine.log(machine.execute("cat /tmp/session.log || true")[1])+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])+ raise++ with subtest("Record GIF + pilot cua-driver through parallel_mouse_drag (multi-cursor)"):+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-parallel-drag.py")+ machine.execute(+ "sh -lc 'env XDG_RUNTIME_DIR=/run/user/0 ${recordGifScript} /tmp/drag-frames "+ "/tmp/cua-driver-wayland-${desktop}-parallel-drag.gif /tmp/stop-drag-recorder "+ "/tmp/rec-drag.log 8 0.12 >/dev/null 2>&1 & echo $! >/tmp/rec-drag.pid'"+ )+ result = machine.execute(+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-parallel-drag.py 2>&1"+ )[1]+ machine.log(result)+ machine.execute("touch /tmp/stop-drag-recorder")+ machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-drag.pid) 2>/dev/null || break; sleep 1; done'")++ with subtest("Copy GIF out of the VM (best-effort)"):+ machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-parallel-drag.gif || : > /tmp/cua-driver-wayland-${desktop}-parallel-drag.gif")+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-parallel-drag.gif", "")++ with subtest("parallel_mouse_drag completed on native Wayland (multi-cursor via EIS compositor)"):+ assert "parallel drag test complete" in result, result+ '';+}diff --git a/nix/cua-driver/tests/wayland/record-wayland-gif.nix b/nix/cua-driver/tests/wayland/record-wayland-gif.nixnew file mode 100644index 0000000000..bc870404ba--- /dev/null+++ b/nix/cua-driver/tests/wayland/record-wayland-gif.nix@@ -0,0 +1,59 @@+# Native-Wayland screen-recording helper for the cua-driver Wayland TDD tests.+#+# Captures the composited Wayland output with `grim` (wlr-screencopy) in a loop+# and stitches the frames into an animated GIF. grim only works on wlroots+# compositors (labwc/wayfire/sway); on kwin/mutter it simply produces no frames+# and no GIF — which is fine, the GIF is a best-effort artifact and these tests+# are expected to fail before/around it anyway.+#+# Usage (in a test's `let`):+# recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; };+# then inside the testScript, start it in the background before driving and+# `touch` the stop-file afterwards:+# ${recordGifScript} <frames_dir> <output_gif> <stop_file> <log_file> \+# <delay_cs> <interval>+# The caller must export WAYLAND_DISPLAY and XDG_RUNTIME_DIR for grim.+{ pkgs }:++pkgs.writeShellScript "record-wayland-gif.sh" ''+ set -u+ frames_dir="$1"+ output_gif="$2"+ stop_file="$3"+ log_file="$4"+ delay_cs="$5"+ interval="$6"++ rm -f "$stop_file" "$output_gif" "$log_file"+ rm -rf "$frames_dir"+ mkdir -p "$frames_dir"++ # In nested mode the caller can't know cua-driver's private socket ahead of+ # time; the driver publishes it to $XDG_RUNTIME_DIR/.cua-nested-display once+ # up. If WAYLAND_DISPLAY isn't already set, wait for and adopt that socket so+ # the GIF captures the SAME session the driver drives.+ if [ -z "''${WAYLAND_DISPLAY:-}" ]; then+ dfile="''${XDG_RUNTIME_DIR:-/run/user/0}/.cua-nested-display"+ w=0+ while [ ! -s "$dfile" ] && [ "$w" -lt 60 ]; do sleep 0.5; w=$((w + 1)); done+ if [ -s "$dfile" ]; then+ WAYLAND_DISPLAY="$(cat "$dfile")"+ export WAYLAND_DISPLAY+ echo "recorder adopted nested WAYLAND_DISPLAY=$WAYLAND_DISPLAY" >>"$log_file"+ fi+ fi++ max_frames=450+ i=0+ while [ ! -f "$stop_file" ] && [ "$i" -lt "$max_frames" ]; do+ frame=$(printf "%s/frame-%04d.png" "$frames_dir" "$i")+ timeout 10 ${pkgs.grim}/bin/grim "$frame" >>"$log_file" 2>&1 || true+ i=$((i + 1))+ sleep "$interval"+ done++ if ls "$frames_dir"/frame-*.png >/dev/null 2>&1; then+ timeout 120 ${pkgs.imagemagick}/bin/convert -delay "$delay_cs" -loop 0 \+ "$frames_dir"/frame-*.png "$output_gif" >>"$log_file" 2>&1 || true+ fi+''diff --git a/nix/cua-driver/tests/wayland/screenshot.nix b/nix/cua-driver/tests/wayland/screenshot.nixnew file mode 100644index 0000000000..7a3cd31317--- /dev/null+++ b/nix/cua-driver/tests/wayland/screenshot.nix@@ -0,0 +1,98 @@+# CUA Driver native-Wayland screenshot test (per desktop) — TDD red.+#+# Launches a native Wayland terminal (foot), finds it via cua-driver+# list_windows, and asserts get_window_state (capture_mode=vision) returns a+# real PNG of that native Wayland surface. The driver captures via X11 today, so+# both discovery and capture fail on native Wayland — the spec for native+# Wayland window capture.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-wayland-<desktop>-screenshot+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ desktop,+ ...+}:++let+ session = import ./session.nix { inherit pkgs desktop; };+ driverClient = import ./driver-client.nix { inherit pkgs; };++ testScriptPy = pkgs.writeText "wayland-screenshot.py" ''+ import base64, sys+ sys.path.insert(0, "/tmp")+ from driver_client import Driver++ d = Driver()+ try:+ d.initialize("nixos-wayland-screenshot")+ d.launch_app("foot --app-id=cua-wayland-foot --title=cua-wayland-foot")+ pid, wid = d.find_window("cua-wayland-foot", timeout=40)+ print(f"window pid={pid} window_id={wid}", flush=True)+ resp = d.call("get_window_state", {+ "pid": pid, "window_id": wid, "capture_mode": "vision"}, timeout=30)+ saved = False+ for item in resp.get("result", {}).get("content", []):+ if item.get("type") == "image" and item.get("data"):+ data = base64.b64decode(item["data"])+ assert len(data) > 0, "empty image payload"+ with open("/tmp/cua-driver-wayland-${desktop}-screenshot.png", "wb") as f:+ f.write(data)+ saved = True+ break+ assert saved, f"no image returned for native Wayland window: {resp}"+ print("screenshot test complete", flush=True)+ finally:+ d.close()+ '';+in++pkgs.testers.nixosTest {+ name = "cua-driver-wayland-${desktop}-screenshot-test";+ meta.maintainers = [ ];++ nodes.machine =+ { pkgs, ... }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 3072;+ };+ services.cua-driver.enable = true;+ boot.kernelModules = [ "uinput" ];+ hardware.graphics.enable = true;+ services.udev.extraRules = ''+ KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput"+ '';+ environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]);+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Bring up ${session.label}"):+ machine.execute("${session.start} >/tmp/session.log 2>&1 &")+ try:+ machine.wait_for_file("/tmp/wl-ready", timeout=120)+ except Exception:+ machine.log(machine.execute("cat /tmp/session.log || true")[1])+ machine.log(machine.execute("cat /tmp/compositor.log || true")[1])+ raise++ with subtest("Launch + screenshot a native Wayland window via cua-driver"):+ machine.copy_from_host("${driverClient}", "/tmp/driver_client.py")+ machine.copy_from_host("${testScriptPy}", "/tmp/wayland-screenshot.py")+ result = machine.succeed(+ "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} "+ "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-screenshot.py 2>&1"+ )+ machine.log(result)+ assert "screenshot test complete" in result, result++ with subtest("Extract screenshot"):+ machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-screenshot.png", "")+ '';+}diff --git a/nix/cua-driver/tests/wayland/session.nix b/nix/cua-driver/tests/wayland/session.nixnew file mode 100644index 0000000000..72d5df3fe0--- /dev/null+++ b/nix/cua-driver/tests/wayland/session.nix@@ -0,0 +1,243 @@+# Shared Wayland desktop-session bring-up for the cua-driver Wayland suite.+#+# Two execution models, selected per desktop:+#+# * NATIVE (wlroots: labwc, sway) — apps run as real Wayland clients of the+# HOST compositor, and the driver talks to that same compositor's wlr+# protocols (foreign-toplevel + screencopy). The session `start` script+# launches the compositor headless, waits for its socket, and publishes it+# to /tmp/wl-display.+#+# * NESTED (kde: kwin, gnome: mutter) — kwin/mutter expose NO client protocols+# for cross-window enumeration/capture, so the driver cannot drive them+# directly. Instead cua-driver "brings its own compositor": with+# CUA_WAYLAND_NEST=1 it spawns a private headless labwc and points+# WAYLAND_DISPLAY at it, so every app launched via `launch_app` runs inside+# that nested session where the wlr protocols DO work. The host kwin/mutter+# is still booted (best-effort) to prove cua-driver coexists with a real+# KDE/GNOME host, but the test never depends on it.+#+# Given a `desktop` this returns:+#+# { packages; start; label; nested; driverWrapper; }+#+# * packages — extra `environment.systemPackages` the session needs.+# * start — a `writeShellScript` that brings the session up and+# `touch`es /tmp/wl-ready, then blocks.+# * label — human label for logs.+# * nested — true for kde/gnome (driver hosts its own compositor).+# * driverWrapper — a `writeShellScript` to use as CUA_DRIVER_BIN: it sets the+# correct env (host WAYLAND_DISPLAY for native; the nest env+# for nested) and execs `cua-driver "$@"`. Tests spawn the+# driver through this so the python harness stays+# desktop-agnostic.+{+ pkgs,+ desktop,+ # EIS mode: cua-driver nests its OWN cua-compositor (focus-free keyboard ++ # multi-cursor injection over a control socket) instead of labwc, on ANY host.+ # Used by the background-terminal-gif + parallel-drag cells.+ eis ? false,+}:++let+ inherit (pkgs) lib;+ cuaCompositor = pkgs.callPackage ../../compositor { };++ # Native-Wayland tooling only. No xorg.* / xdotool / xterm here — those are+ # X11 and would let the driver cheat via XWayland. `foot` is a Wayland-native+ # terminal; `grim` captures the wlroots output for the GIF artifacts.+ commonPkgs = with pkgs; [+ foot+ grim+ wtype # virtual-keyboard CLI: cua-driver shells out to it for type_text/press_key+ dbus+ procps+ ];++ # Minimal sway config: native Wayland only (XWayland off), stable output.+ # `workspace_layout stacking` makes the focused window fill the output, so the+ # driver's centre virtual-pointer click reliably lands on the activated target+ # (sway needs a pointer interaction to route virtual-keyboard input on a+ # headless seat with no physical keyboard).+ swayConfig = pkgs.writeText "sway-config" ''+ xwayland disable+ output HEADLESS-1 resolution 1280x1024+ workspace_layout stacking+ exec_always foot --server+ '';++ desktops = {+ "xfce-labwc" = {+ label = "XFCE on labwc (native Wayland)";+ packages = with pkgs; [ labwc ];+ wlroots = true;+ nested = false;+ launch = "labwc >/tmp/compositor.log 2>&1 &";+ };+ # NOTE: xfce-wayfire intentionally omitted — wayfire fails to build in the+ # current nixpkgs pin (wf-config can't link -ldoctest). labwc + sway cover+ # XFCE-on-wlroots.+ "xfce-sway" = {+ label = "XFCE on sway (native Wayland)";+ packages = with pkgs; [ sway ];+ wlroots = true;+ nested = false;+ launch = "sway -c ${swayConfig} >/tmp/compositor.log 2>&1 &";+ };+ "kde" = {+ label = "KDE Plasma kwin_wayland host + cua-driver nested labwc";+ # labwc is what cua-driver nests; kwin is the host it coexists with.+ packages = with pkgs; [ labwc kdePackages.kwin ];+ wlroots = false;+ nested = true;+ # No --xwayland: keep the host session native-Wayland only.+ launch = "kwin_wayland --virtual --width 1280 --height 1024 --no-lockscreen >/tmp/compositor.log 2>&1 &";+ };+ "gnome" = {+ label = "GNOME Shell / mutter host + cua-driver nested labwc";+ packages = with pkgs; [ labwc gnome-shell mutter ];+ wlroots = false;+ nested = true;+ launch = "gnome-shell --wayland --headless --virtual-monitor 1280x1024 --unsafe-mode >/tmp/compositor.log 2>&1 &";+ };+ };++ cfg = desktops.${desktop} or (throw "wayland/session.nix: unknown desktop '${desktop}'");++ wlrootsEnv = lib.optionalString cfg.wlroots ''+ export WLR_BACKENDS=headless+ export WLR_RENDERER=pixman+ export WLR_RENDERER_ALLOW_SOFTWARE=1+ export WLR_LIBINPUT_NO_DEVICES=1+ '';++ # Native model: block until the HOST compositor's socket appears, publish it.+ nativeStart = ''+ dbus-run-session -- sh -c '+ wlsock() { ls "$XDG_RUNTIME_DIR" 2>/dev/null | grep -E "^wayland-[0-9]+$" | head -1; }++ ${cfg.launch}+ compositor_pid=$!++ n=0+ while [ -z "$(wlsock)" ] && [ "$n" -lt 120 ]; do+ if ! kill -0 "$compositor_pid" 2>/dev/null; then+ echo "compositor exited early" >&2+ break+ fi+ sleep 0.5+ n=$((n + 1))+ done++ sock="$(wlsock)"+ if [ -z "$sock" ]; then+ echo "no Wayland display socket appeared" >&2+ cat /tmp/compositor.log >&2 2>/dev/null || true+ exit 1+ fi+ echo "$sock" > /tmp/wl-display++ touch /tmp/wl-ready+ exec sleep infinity+ '+ '';++ # Nested model: boot the host compositor BEST-EFFORT (log whether it came up),+ # but never block the test on it — cua-driver provides its own session. We+ # record the host socket (if any) for logs; the driver wrapper ignores it.+ nestedStart = ''+ dbus-run-session -- sh -c '+ wlsock() { ls "$XDG_RUNTIME_DIR" 2>/dev/null | grep -E "^wayland-[0-9]+$" | head -1; }++ ${cfg.launch}++ n=0+ while [ -z "$(wlsock)" ] && [ "$n" -lt 40 ]; do+ sleep 0.5+ n=$((n + 1))+ done+ sock="$(wlsock)"+ if [ -n "$sock" ]; then+ echo "host compositor up: $sock" >&2+ else+ echo "host compositor did not come up; proceeding with nested session only" >&2+ cat /tmp/compositor.log >&2 2>/dev/null || true+ fi+ echo "''${sock:-none}" > /tmp/wl-display++ touch /tmp/wl-ready+ exec sleep infinity+ '+ '';++ start = pkgs.writeShellScript "wayland-session-start-${desktop}.sh" ''+ set -u+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH++ export XDG_RUNTIME_DIR=/run/user/0+ mkdir -p "$XDG_RUNTIME_DIR"+ chmod 700 "$XDG_RUNTIME_DIR"++ ${wlrootsEnv}+ export QT_QPA_PLATFORM=wayland+ export GDK_BACKEND=wayland+ export LIBGL_ALWAYS_SOFTWARE=1++ rm -f /tmp/wl-ready /tmp/wl-display "$XDG_RUNTIME_DIR/.cua-nested-display" "$XDG_RUNTIME_DIR/cua-inject.sock"++ ${if (cfg.nested || eis) then nestedStart else nativeStart}+ '';++ # Driver launcher used as CUA_DRIVER_BIN. It owns the WAYLAND env so the python+ # harness never has to. labwc/foot/grim are on PATH for the app/compositor+ # processes cua-driver spawns.+ # Toolkit apps cua-driver launches inherit its env, so force the NATIVE+ # Wayland backends here (GTK auto-detects from WAYLAND_DISPLAY; Qt would+ # otherwise default to xcb and fail with no X). Harmless for foot.+ appBackendEnv = ''+ export GDK_BACKEND=wayland+ export QT_QPA_PLATFORM=wayland+ export LIBGL_ALWAYS_SOFTWARE=1+ '';++ driverWrapper =+ if eis then+ # EIS mode: nest cua-compositor (focus-free + multi-cursor injection over+ # the control socket) on ANY host. Apps launched via launch_app land in it.+ pkgs.writeShellScript "cua-driver-eis-${desktop}" ''+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages ++ [ cuaCompositor ])}:$PATH+ export XDG_RUNTIME_DIR=/run/user/0+ export CUA_WAYLAND_NEST=1+ export CUA_WAYLAND_NEST_COMPOSITOR=${cuaCompositor}/bin/cua-compositor+ export CUA_INJECT_SOCKET=/run/user/0/cua-inject.sock+ ${appBackendEnv}+ unset DISPLAY WAYLAND_DISPLAY+ exec cua-driver "$@"+ ''+ else if cfg.nested then+ pkgs.writeShellScript "cua-driver-nested-${desktop}" ''+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH+ export XDG_RUNTIME_DIR=/run/user/0+ export CUA_WAYLAND_NEST=1+ export CUA_WAYLAND_NEST_COMPOSITOR=labwc+ ${appBackendEnv}+ unset DISPLAY WAYLAND_DISPLAY+ exec cua-driver "$@"+ ''+ else+ pkgs.writeShellScript "cua-driver-native-${desktop}" ''+ export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH+ export XDG_RUNTIME_DIR=/run/user/0+ export WAYLAND_DISPLAY="$(cat /tmp/wl-display)"+ ${appBackendEnv}+ unset DISPLAY+ exec cua-driver "$@"+ '';+in+{+ label = if eis then "${cfg.label} + cua-compositor (EIS injection)" else cfg.label;+ nested = cfg.nested || eis;+ packages = commonPkgs ++ cfg.packages ++ lib.optional eis cuaCompositor;+ inherit start driverWrapper;+}