Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cua-driver-rs)(linux): show cursor and type in background terminals (#1789)
r33drichardsc08f544a31a4
diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.ymlindex 809d0fa9ff..8d090365e0 100644--- a/.github/workflows/nix-build.yml+++ b/.github/workflows/nix-build.yml@@ -21,6 +21,7 @@ on:permissions:id-token: writecontents: read+ pull-requests: writeenv:AWS_REGION: us-west-2@@ -28,10 +29,160 @@ env:NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-keyjobs:- build-and-test:- name: Build cua-driver & run integration tests+ nix-checks:+ name: ${{ matrix.name }}runs-on: ubuntu-latesttimeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ include:+ - name: NixOS integration test+ check_attr: cua-driver-integration+ timeout_minutes: 8+ visual: false+ result_link: result-cua-driver-integration+ artifact_name: ""+ - name: Linux cursor click GIF test+ check_attr: cua-driver-linux-cursor-click-gif+ timeout_minutes: 12+ visual: true+ result_link: result-linux-cursor-click-gif+ artifact_name: cua-driver-linux-cursor-click-gif+ - name: Linux background terminal GIF test+ check_attr: cua-driver-linux-background-terminal-gif+ timeout_minutes: 12+ visual: true+ result_link: result-linux-background-terminal-gif+ artifact_name: cua-driver-linux-background-terminal-gif+ # Full entries (CDP / Tk focus-free-write overrides) — kept as-is.+ - name: Linux background GUI test (chromium)+ check_attr: cua-driver-linux-background-gui-chromium+ timeout_minutes: 25+ visual: true+ result_link: result-linux-background-gui-chromium+ artifact_name: cua-driver-linux-background-gui-chromium+ - name: Linux background GUI test (tk)+ check_attr: cua-driver-linux-background-gui-tk+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-tk+ artifact_name: cua-driver-linux-background-gui-tk+ # Real-app READ-ONLY skeleton matrix (5 per toolkit category).+ # GTK3+ - name: Linux background GUI test (gtk3-gedit)+ check_attr: cua-driver-linux-background-gui-gtk3-gedit+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-gtk3-gedit+ artifact_name: cua-driver-linux-background-gui-gtk3-gedit+ - name: Linux background GUI test (gtk3-mousepad)+ check_attr: cua-driver-linux-background-gui-gtk3-mousepad+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-gtk3-mousepad+ artifact_name: cua-driver-linux-background-gui-gtk3-mousepad+ # gtk3-geany temporarily disabled (huge AT-SPI tree grinds the emulated VM; job times out)+ # - name: Linux background GUI test (gtk3-geany)+ # check_attr: cua-driver-linux-background-gui-gtk3-geany+ # timeout_minutes: 15+ # visual: true+ # result_link: result-linux-background-gui-gtk3-geany+ # artifact_name: cua-driver-linux-background-gui-gtk3-geany+ - name: Linux background GUI test (gtk3-scite)+ check_attr: cua-driver-linux-background-gui-gtk3-scite+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-gtk3-scite+ artifact_name: cua-driver-linux-background-gui-gtk3-scite+ # gtk3-abiword temporarily disabled (huge AT-SPI tree grinds the emulated VM; job times out)+ # - name: Linux background GUI test (gtk3-abiword)+ # check_attr: cua-driver-linux-background-gui-gtk3-abiword+ # timeout_minutes: 15+ # visual: true+ # result_link: result-linux-background-gui-gtk3-abiword+ # artifact_name: cua-driver-linux-background-gui-gtk3-abiword+ # GTK4+ - name: Linux background GUI test (gtk4-characters)+ check_attr: cua-driver-linux-background-gui-gtk4-characters+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-gtk4-characters+ artifact_name: cua-driver-linux-background-gui-gtk4-characters+ # Qt5+ - name: Linux background GUI test (qt5-manuskript)+ check_attr: cua-driver-linux-background-gui-qt5-manuskript+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-qt5-manuskript+ artifact_name: cua-driver-linux-background-gui-qt5-manuskript+ - name: Linux background GUI test (qt5-klog)+ check_attr: cua-driver-linux-background-gui-qt5-klog+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-qt5-klog+ artifact_name: cua-driver-linux-background-gui-qt5-klog+ - name: Linux background GUI test (qt5-openambit)+ check_attr: cua-driver-linux-background-gui-qt5-openambit+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-qt5-openambit+ artifact_name: cua-driver-linux-background-gui-qt5-openambit+ # Qt6+ - name: Linux background GUI test (qt6-kate)+ check_attr: cua-driver-linux-background-gui-qt6-kate+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-qt6-kate+ artifact_name: cua-driver-linux-background-gui-qt6-kate+ - name: Linux background GUI test (qt6-kcalc)+ check_attr: cua-driver-linux-background-gui-qt6-kcalc+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-qt6-kcalc+ artifact_name: cua-driver-linux-background-gui-qt6-kcalc+ - name: Linux background GUI test (qt6-okular)+ check_attr: cua-driver-linux-background-gui-qt6-okular+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-qt6-okular+ artifact_name: cua-driver-linux-background-gui-qt6-okular+ - name: Linux background GUI test (qt6-qownnotes)+ check_attr: cua-driver-linux-background-gui-qt6-qownnotes+ timeout_minutes: 15+ visual: true+ result_link: result-linux-background-gui-qt6-qownnotes+ artifact_name: cua-driver-linux-background-gui-qt6-qownnotes+ # Electron (heavy: more memory + 25-min timeout)+ - name: Linux background GUI test (electron-zettlr)+ check_attr: cua-driver-linux-background-gui-electron-zettlr+ timeout_minutes: 25+ visual: true+ result_link: result-linux-background-gui-electron-zettlr+ artifact_name: cua-driver-linux-background-gui-electron-zettlr+ - name: Linux background GUI test (electron-joplin)+ check_attr: cua-driver-linux-background-gui-electron-joplin+ timeout_minutes: 25+ visual: true+ result_link: result-linux-background-gui-electron-joplin+ artifact_name: cua-driver-linux-background-gui-electron-joplin+ - name: Linux background GUI test (electron-logseq)+ check_attr: cua-driver-linux-background-gui-electron-logseq+ timeout_minutes: 25+ visual: true+ result_link: result-linux-background-gui-electron-logseq+ artifact_name: cua-driver-linux-background-gui-electron-logseq+ # Firefox is temporarily disabled: under the emulated CI VM (no KVM) it+ # does not surface its window within the launch timeout, so the job+ # times out before any AT-SPI subtest runs. The browser/AT-SPI read+ # path is already covered by the chromium job. Re-enable once launch is+ # made reliable (longer timeout + pre-seeded first-run-free profile).+ # - name: Linux background GUI test (firefox)+ # check_attr: cua-driver-linux-background-gui-firefox+ # timeout_minutes: 25+ # visual: false+ # result_link: result-linux-background-gui-firefox+ # artifact_name: ""steps:- name: Checkout@@ -79,9 +230,27 @@ jobs:trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=- - name: Run NixOS integration test- timeout-minutes: 8- run: nix build .#checks.x86_64-linux.cua-driver-integration --print-build-logs --show-trace+ - name: Run ${{ matrix.name }}+ timeout-minutes: ${{ matrix.timeout_minutes }}+ run: |+ nix build .#checks.x86_64-linux.${{ matrix.check_attr }} \+ --print-build-logs --show-trace \+ -o ${{ matrix.result_link }}++ - name: Collect GIF artifacts+ if: always() && matrix.visual+ run: |+ mkdir -p artifacts+ find -L "${{ matrix.result_link }}/" \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f -exec cp {} artifacts/ \; 2>/dev/null || true+ ls -la artifacts/ 2>/dev/null || echo "No visual artifacts found"++ - name: Upload GIF artifacts+ if: always() && matrix.visual+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: ${{ matrix.artifact_name }}+ path: artifacts/*+ if-no-files-found: warn- name: Sign and upload to Nix cacheif: always()@@ -93,3 +262,163 @@ jobs:- name: Cleanup signing keyif: always()run: rm -f "${{ runner.temp }}/signing-key.sec"++ comment-linux-visual-artifacts:+ name: Comment Linux visual artifacts+ if: always() && github.event_name == 'pull_request'+ needs: [nix-checks]+ runs-on: ubuntu-latest+ steps:+ - name: Comment Linux visual artifacts on PR+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7+ with:+ script: |+ const marker = '<!-- cua-driver-linux-visual-gifs -->';+ const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;+ const artifactNames = [+ 'cua-driver-linux-cursor-click-gif',+ 'cua-driver-linux-background-terminal-gif',+ 'cua-driver-linux-background-gui-chromium',+ 'cua-driver-linux-background-gui-tk',+ 'cua-driver-linux-background-gui-gtk3-gedit',+ 'cua-driver-linux-background-gui-gtk3-mousepad',+ 'cua-driver-linux-background-gui-gtk3-scite',+ 'cua-driver-linux-background-gui-gtk4-characters',+ 'cua-driver-linux-background-gui-qt5-manuskript',+ 'cua-driver-linux-background-gui-qt5-klog',+ 'cua-driver-linux-background-gui-qt5-openambit',+ 'cua-driver-linux-background-gui-qt6-kate',+ 'cua-driver-linux-background-gui-qt6-kcalc',+ 'cua-driver-linux-background-gui-qt6-okular',+ 'cua-driver-linux-background-gui-qt6-qownnotes',+ 'cua-driver-linux-background-gui-electron-zettlr',+ 'cua-driver-linux-background-gui-electron-joplin',+ 'cua-driver-linux-background-gui-electron-logseq',+ 'cua-driver-linux-som-overlays',+ ];++ let body = `${marker}\n## Linux visual regression artifacts\n\n`;+ body += 'Matrix jobs now run independently. Download visual artifacts from this workflow run.\n';+ body += 'Each background-GUI job uploads a `.gif` of the interaction plus two annotated PNGs ';+ body += '(`<app>.png` raw, `<app>-atspi.png` with AT-SPI element boxes); ';+ body += 'the `cua-driver-linux-som-overlays` artifact adds `<app>-som.png` cua Set-of-Marks overlays:\n';+ for (const artifactName of artifactNames) {+ body += `- \`${artifactName}\`\n`;+ }+ body += `\n[Open workflow run and download artifacts](${runUrl})\n`;++ const comments = await github.paginate(github.rest.issues.listComments, {+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ per_page: 100,+ });+ const existing = comments.find(comment =>+ comment.user?.type === 'Bot' && comment.body?.includes(marker)+ );++ if (existing) {+ await github.rest.issues.updateComment({+ owner: context.repo.owner,+ repo: context.repo.repo,+ comment_id: existing.id,+ body,+ });+ } else {+ await github.rest.issues.createComment({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ body,+ });+ }++ # ── Set-of-Marks overlays ───────────────────────────────────────────────────+ # Downstream aggregate job: consumes the raw `<app>.png` screenshots uploaded+ # by the background-GUI matrix and emits `<app>-som.png` cua Set-of-Marks+ # overlays. Deliberately OFF the hot path — `needs: [nix-checks]` + `if:+ # always()` so it runs after the matrix regardless of pass/fail and never+ # blocks it. NOTE: `pip install -e libs/python/som` pulls torch + ultralytics ++ # easyocr (~GBs) and downloads model weights at first parse, so this job is+ # the slow/expensive one; it is intentionally isolated from the build matrix.+ som-annotate:+ name: Annotate screenshots with cua Set-of-Marks+ if: always()+ needs: [nix-checks]+ runs-on: ubuntu-latest+ steps:+ - name: Checkout+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Download all run artifacts+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ path: downloaded-artifacts++ - name: Set up Python+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5+ with:+ python-version: "3.12"++ - name: Install cua-som+ run: |+ python -m pip install --upgrade pip+ pip install -e libs/python/som++ - name: Generate Set-of-Marks overlays+ run: |+ mkdir -p som-overlays+ python - <<'PY'+ import base64, glob, os, sys, traceback++ # Collect raw background-GUI screenshots: <app>.png produced by the+ # skeleton matrix. Exclude the AT-SPI overlays (-atspi) and any prior+ # SoM outputs (-som). Artifacts land under downloaded-artifacts/<name>/.+ candidates = []+ for p in glob.glob("downloaded-artifacts/**/*-background-gui-*.png", recursive=True):+ stem = os.path.basename(p)+ if stem.endswith("-atspi.png") or stem.endswith("-som.png"):+ continue+ candidates.append(p)+ candidates = sorted(set(candidates))+ print(f"Found {len(candidates)} raw screenshot(s) to annotate:", flush=True)+ for p in candidates:+ print(f" {p}", flush=True)++ if not candidates:+ print("No raw screenshots found; nothing to annotate.", flush=True)+ sys.exit(0)++ from som import OmniParser+ parser = OmniParser()++ ok = 0+ for p in candidates:+ try:+ with open(p, "rb") as f:+ data = f.read()+ result = parser.parse(data)+ b64 = result.annotated_image_base64+ # Strip a possible data URL prefix before decoding.+ if "," in b64 and b64.strip().startswith("data:"):+ b64 = b64.split(",", 1)[1]+ out_name = os.path.splitext(os.path.basename(p))[0] + "-som.png"+ out_path = os.path.join("som-overlays", out_name)+ with open(out_path, "wb") as f:+ f.write(base64.b64decode(b64))+ print(f"OK {p} -> {out_path}", flush=True)+ ok += 1+ except Exception:+ print(f"FAIL {p}", flush=True)+ traceback.print_exc()+ print(f"Annotated {ok}/{len(candidates)} screenshot(s).", flush=True)+ PY+ ls -la som-overlays/ 2>/dev/null || echo "No SoM overlays produced"++ - name: Upload Set-of-Marks overlays+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: cua-driver-linux-som-overlays+ path: som-overlays/*-som.png+ if-no-files-found: warndiff --git a/flake.nix b/flake.nixindex d4cdbf7583..f3425c74ef 100644--- a/flake.nix+++ b/flake.nix@@ -61,7 +61,79 @@services.cua-driver.package = cuaDriverPackage;};};- };++ cua-driver-linux-cursor-click-gif = import ./nix/cua-driver/tests/linux-cursor-click-gif.nix {+ inherit pkgs;+ inherit (pkgs) lib;+ cuaDriverModule = {+ imports = [ ./nix/cua-driver/module.nix ];+ services.cua-driver.package = cuaDriverPackage;+ };+ };++ cua-driver-linux-background-terminal-gif = import ./nix/cua-driver/tests/linux-background-terminal-gif.nix {+ inherit pkgs;+ inherit (pkgs) lib;+ cuaDriverModule = {+ imports = [ ./nix/cua-driver/module.nix ];+ services.cua-driver.package = cuaDriverPackage;+ };+ };+ }+ // pkgs.lib.optionalAttrs (system == "x86_64-linux") (+ # Background GUI input coverage — one independent matrix job per+ # app, proving focus-free typing into real toolkit/browser windows.+ pkgs.lib.listToAttrs (+ map (+ app:+ pkgs.lib.nameValuePair "cua-driver-linux-background-gui-${app}" (+ import ./nix/cua-driver/tests/linux-background-gui.nix {+ inherit pkgs app;+ inherit (pkgs) lib;+ cuaDriverModule = {+ imports = [ ./nix/cua-driver/module.nix ];+ services.cua-driver.package = cuaDriverPackage;+ };+ }+ )+ # Real-app matrix: 5 apps per toolkit category run as a LENIENT,+ # READ-ONLY skeleton (find window + driver page/get_text + GIF;+ # focus-free WRITE / typed-text assertions are added later via+ # trajectories). chromium keeps the full CDP focus-free-write+ # override; tk is the negative-control full entry (Tk `send`).+ # "firefox" remains disabled: under the emulated CI VM (no KVM)+ # it does not surface its window within the launch timeout.+ ) [+ "chromium"+ "tk"+ # GTK3+ "gtk3-gedit"+ "gtk3-mousepad"+ # gtk3-geany / gtk3-abiword temporarily disabled: their huge+ # AT-SPI trees make the bounds walk + recorder grind in the+ # emulated CI VM and the jobs time out. Re-enable once the+ # walk is fast enough for 700+-node trees.+ # "gtk3-geany"+ "gtk3-scite"+ # "gtk3-abiword"+ # GTK4+ "gtk4-characters"+ # Qt5+ "qt5-manuskript"+ "qt5-klog"+ "qt5-openambit"+ # Qt6+ "qt6-kate"+ "qt6-kcalc"+ "qt6-okular"+ "qt6-qownnotes"+ # Electron+ "electron-zettlr"+ "electron-joplin"+ "electron-logseq"+ ]+ )+ );})// {diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lockindex 04d91fb754..751f8f6e3d 100644--- a/libs/cua-driver/rust/Cargo.lock+++ b/libs/cua-driver/rust/Cargo.lock@@ -53,6 +53,126 @@ version = "0.7.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"+[[package]]+name = "async-broadcast"+version = "0.7.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532"+dependencies = [+ "event-listener",+ "event-listener-strategy",+ "futures-core",+ "pin-project-lite",+]++[[package]]+name = "async-channel"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2"+dependencies = [+ "concurrent-queue",+ "event-listener-strategy",+ "futures-core",+ "pin-project-lite",+]++[[package]]+name = "async-executor"+version = "1.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a"+dependencies = [+ "async-task",+ "concurrent-queue",+ "fastrand",+ "futures-lite",+ "pin-project-lite",+ "slab",+]++[[package]]+name = "async-io"+version = "2.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc"+dependencies = [+ "autocfg",+ "cfg-if",+ "concurrent-queue",+ "futures-io",+ "futures-lite",+ "parking",+ "polling",+ "rustix",+ "slab",+ "windows-sys 0.61.2",+]++[[package]]+name = "async-lock"+version = "3.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311"+dependencies = [+ "event-listener",+ "event-listener-strategy",+ "pin-project-lite",+]++[[package]]+name = "async-process"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75"+dependencies = [+ "async-channel",+ "async-io",+ "async-lock",+ "async-signal",+ "async-task",+ "blocking",+ "cfg-if",+ "event-listener",+ "futures-lite",+ "rustix",+]++[[package]]+name = "async-recursion"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "async-signal"+version = "0.2.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485"+dependencies = [+ "async-io",+ "async-lock",+ "atomic-waker",+ "cfg-if",+ "futures-core",+ "futures-io",+ "rustix",+ "signal-hook-registry",+ "slab",+ "windows-sys 0.61.2",+]++[[package]]+name = "async-task"+version = "4.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"+[[package]]name = "async-trait"version = "0.1.89"@@ -64,6 +184,63 @@ dependencies = ["syn",]+[[package]]+name = "atomic-waker"+version = "1.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"++[[package]]+name = "atspi"+version = "0.30.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6bf601cccedfffec598ec2db1f9d6745885458bccc0e8916d7023f017c94b3d0"+dependencies = [+ "atspi-common",+ "atspi-connection",+ "atspi-proxies",+ "zbus",+]++[[package]]+name = "atspi-common"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8a79bed3f5b408ce3152f36e07327a845e6ed5d7e2821a89264037dbcc11daf"+dependencies = [+ "enumflags2",+ "serde",+ "static_assertions",+ "zbus",+ "zbus-lockstep",+ "zbus-lockstep-macros",+ "zbus_names",+ "zvariant",+]++[[package]]+name = "atspi-connection"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8fab8e4f574f5a7d3af280b38eff25fb6f47a537dac9ae39ce152f52b19fb10b"+dependencies = [+ "atspi-common",+ "atspi-proxies",+ "futures-lite",+ "zbus",+]++[[package]]+name = "atspi-proxies"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53403acd3ab2fdb5914f6558da22e540fc07656fce5510f8c02be0e6ef68413e"+dependencies = [+ "atspi-common",+ "serde",+ "zbus",+]+[[package]]name = "autocfg"version = "1.5.0"@@ -106,6 +283,19 @@ dependencies = ["objc2",]+[[package]]+name = "blocking"+version = "1.6.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21"+dependencies = [+ "async-channel",+ "async-task",+ "futures-io",+ "futures-lite",+ "piper",+]+[[package]]name = "bumpalo"version = "3.20.2"@@ -152,6 +342,15 @@ version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"+[[package]]+name = "concurrent-queue"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"+dependencies = [+ "crossbeam-utils",+]+[[package]]name = "cookie"version = "0.18.1"@@ -429,6 +628,33 @@ dependencies = ["winreg",]+[[package]]+name = "endi"+version = "1.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099"++[[package]]+name = "enumflags2"+version = "0.7.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef"+dependencies = [+ "enumflags2_derive",+ "serde",+]++[[package]]+name = "enumflags2_derive"+version = "0.7.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "equivalent"version = "1.0.2"@@ -454,6 +680,27 @@ dependencies = ["num-traits",]+[[package]]+name = "event-listener"+version = "5.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"+dependencies = [+ "concurrent-queue",+ "parking",+ "pin-project-lite",+]++[[package]]+name = "event-listener-strategy"+version = "0.5.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93"+dependencies = [+ "event-listener",+ "pin-project-lite",+]+[[package]]name = "fastrand"version = "2.4.1"@@ -583,6 +830,25 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"+[[package]]+name = "futures-io"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"++[[package]]+name = "futures-lite"+version = "2.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad"+dependencies = [+ "fastrand",+ "futures-core",+ "futures-io",+ "parking",+ "pin-project-lite",+]+[[package]]name = "futures-macro"version = "0.3.32"@@ -685,6 +951,18 @@ version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"+[[package]]+name = "hermit-abi"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"++[[package]]+name = "hex"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"+[[package]]name = "http"version = "1.4.0"@@ -944,6 +1222,15 @@ version = "2.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"+[[package]]+name = "memoffset"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"+dependencies = [+ "autocfg",+]+[[package]]name = "miniz_oxide"version = "0.8.9"@@ -1164,6 +1451,22 @@ dependencies = ["vcpkg",]+[[package]]+name = "ordered-stream"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50"+dependencies = [+ "futures-core",+ "pin-project-lite",+]++[[package]]+name = "parking"+version = "2.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"+[[package]]name = "parking_lot"version = "0.12.5"@@ -1214,6 +1517,17 @@ dependencies = ["tracing",]+[[package]]+name = "piper"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1"+dependencies = [+ "atomic-waker",+ "fastrand",+ "futures-io",+]+[[package]]name = "pkg-config"version = "0.3.33"@@ -1226,6 +1540,7 @@ version = "0.5.1"dependencies = ["anyhow","async-trait",+ "atspi","base64","cua-driver-core","cursor-overlay",@@ -1321,6 +1636,20 @@ dependencies = ["miniz_oxide",]+[[package]]+name = "polling"+version = "3.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218"+dependencies = [+ "cfg-if",+ "concurrent-queue",+ "hermit-abi",+ "pin-project-lite",+ "rustix",+ "windows-sys 0.61.2",+]+[[package]]name = "potential_utf"version = "0.1.5"@@ -1355,6 +1684,15 @@ dependencies = ["syn",]+[[package]]+name = "proc-macro-crate"+version = "3.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"+dependencies = [+ "toml_edit 0.25.12+spec-1.1.0",+]+[[package]]name = "proc-macro2"version = "1.0.106"@@ -1370,6 +1708,16 @@ version = "0.1.29"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"+[[package]]+name = "quick-xml"+version = "0.39.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e"+dependencies = [+ "memchr",+ "serde",+]+[[package]]name = "quote"version = "1.0.45"@@ -1662,6 +2010,17 @@ dependencies = ["zmij",]+[[package]]+name = "serde_repr"+version = "0.1.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "serde_spanned"version = "0.6.9"@@ -1765,6 +2124,12 @@ version = "1.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"+[[package]]+name = "static_assertions"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"+[[package]]name = "strict-num"version = "0.1.1"@@ -1960,6 +2325,7 @@ dependencies = ["signal-hook-registry","socket2","tokio-macros",+ "tracing","windows-sys 0.61.2",]@@ -2006,8 +2372,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"dependencies = ["serde","serde_spanned",- "toml_datetime",- "toml_edit",+ "toml_datetime 0.6.11",+ "toml_edit 0.22.27",][[package]]@@ -2019,6 +2385,15 @@ dependencies = ["serde",]+[[package]]+name = "toml_datetime"+version = "1.1.1+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"+dependencies = [+ "serde_core",+]+[[package]]name = "toml_edit"version = "0.22.27"@@ -2028,9 +2403,30 @@ dependencies = ["indexmap","serde","serde_spanned",- "toml_datetime",+ "toml_datetime 0.6.11","toml_write",- "winnow",+ "winnow 0.7.15",+]++[[package]]+name = "toml_edit"+version = "0.25.12+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7"+dependencies = [+ "indexmap",+ "toml_datetime 1.1.1+spec-1.1.0",+ "toml_parser",+ "winnow 1.0.3",+]++[[package]]+name = "toml_parser"+version = "1.1.2+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"+dependencies = [+ "winnow 1.0.3",][[package]]@@ -2134,6 +2530,17 @@ version = "1.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"+[[package]]+name = "uds_windows"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e"+dependencies = [+ "memoffset",+ "tempfile",+ "windows-sys 0.61.2",+]+[[package]]name = "unicode-bidi"version = "0.3.18"@@ -2285,6 +2692,7 @@ checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"dependencies = ["getrandom 0.4.2","js-sys",+ "serde_core","wasm-bindgen",]@@ -2785,6 +3193,15 @@ dependencies = ["memchr",]+[[package]]+name = "winnow"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"+dependencies = [+ "memchr",+]+[[package]]name = "winreg"version = "0.52.0"@@ -2941,6 +3358,104 @@ dependencies = ["synstructure",]+[[package]]+name = "zbus"+version = "5.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eee682d202a77e4a9f3b2c2bdf48a7b28af5c08c34ddf66f98c93e5e39464285"+dependencies = [+ "async-broadcast",+ "async-executor",+ "async-io",+ "async-lock",+ "async-process",+ "async-recursion",+ "async-task",+ "async-trait",+ "blocking",+ "enumflags2",+ "event-listener",+ "futures-core",+ "futures-lite",+ "hex",+ "libc",+ "ordered-stream",+ "rustix",+ "serde",+ "serde_repr",+ "tokio",+ "tracing",+ "uds_windows",+ "uuid",+ "windows-sys 0.61.2",+ "winnow 1.0.3",+ "zbus_macros",+ "zbus_names",+ "zvariant",+]++[[package]]+name = "zbus-lockstep"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6998de05217a084b7578728a9443d04ea4cd80f2a0839b8d78770b76ccd45863"+dependencies = [+ "zbus_xml",+ "zvariant",+]++[[package]]+name = "zbus-lockstep-macros"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "10da05367f3a7b7553c8cdf8fa91aee6b64afebe32b51c95177957efc47ca3a0"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+ "zbus-lockstep",+ "zbus_xml",+ "zvariant",+]++[[package]]+name = "zbus_macros"+version = "5.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "adf1bd45a81a103745b1757754762a26e8cd01e4532e4d6c8ec431624b80d1d6"+dependencies = [+ "proc-macro-crate",+ "proc-macro2",+ "quote",+ "syn",+ "zbus_names",+ "zvariant",+ "zvariant_utils",+]++[[package]]+name = "zbus_names"+version = "4.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7074f3e50b894eac91750142016d30d0a89be8e67dbfd9704fb875825760e52d"+dependencies = [+ "serde",+ "winnow 1.0.3",+ "zvariant",+]++[[package]]+name = "zbus_xml"+version = "5.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a8067892e940ed1727dea64690378601603b31d62dfde019a5335fbb7c0e0ed9"+dependencies = [+ "quick-xml",+ "serde",+ "zbus_names",+ "zvariant",+]+[[package]]name = "zerocopy"version = "0.8.48"@@ -3041,3 +3556,43 @@ checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"dependencies = ["zune-core",]++[[package]]+name = "zvariant"+version = "5.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a192a0bde63360d77a7523c833d4b4ce6070a927e2c53246e4c540b1a3e27be0"+dependencies = [+ "endi",+ "enumflags2",+ "serde",+ "winnow 1.0.3",+ "zvariant_derive",+ "zvariant_utils",+]++[[package]]+name = "zvariant_derive"+version = "5.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "90bc6cde9c01c511074be97f7ccb6c19d0da89e3f8662e812e999dcfd4638737"+dependencies = [+ "proc-macro-crate",+ "proc-macro2",+ "quote",+ "syn",+ "zvariant_utils",+]++[[package]]+name = "zvariant_utils"+version = "3.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e8535915cfa75547e559d8c68e8139909a4aeee076831e4ef7fc59d8172c4d6"+dependencies = [+ "proc-macro2",+ "quote",+ "serde",+ "syn",+ "winnow 1.0.3",+]diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.tomlindex 4189eca8c6..1d7cdfa48f 100644--- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml+++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml@@ -19,8 +19,13 @@ tiny-skia = { version = "0.11", default-features = false, features = ["std"] }[target.'cfg(target_os = "linux")'.dependencies]# X11 background input + window enumeration-x11rb = { version = "0.13", features = ["xinput", "randr", "xfixes", "composite", "shape"] }+x11rb = { version = "0.13", features = ["xinput", "randr", "xfixes", "composite", "shape", "xtest"] }base64 = { workspace = true }image = { workspace = true }# kill(2) for the kill_app tool — SIGKILL via libc::kill.libc = "0.2"+# Native AT-SPI over D-Bus (pure Rust, via zbus) — replaces the pyatspi+# subprocess bridge so accessibility input/readback needs no Python or GI+# typelibs at runtime. `tokio` matches the driver's async runtime; `zbus`+# re-exports the bus types we need (fdo::DBusProxy for pid resolution).+atspi = { version = "0.30", features = ["tokio", "zbus"] }diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rsindex 78b4feb742..910a2d7573 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs@@ -1,20 +1,18 @@//! AT-SPI accessibility tree walking for Linux.//!-//! AT-SPI2 is accessible via D-Bus. Rather than linking the full D-Bus library,-//! we query AT-SPI via the `gdbus` or `dbus-send` CLI tool as a subprocess,-//! or we use the `atspi` Rust crate if available.+//! AT-SPI2 is exposed over D-Bus. We talk to it natively in Rust via the+//! `atspi` crate (zbus) — no Python, `pyatspi`, or GObject-introspection+//! typelibs are required at runtime. The async zbus calls run on a shared+//! background Tokio runtime; the public functions stay synchronous because+//! callers invoke them inside `tokio::task::spawn_blocking`.//!-//! For a pure-Rust implementation without D-Bus library deps, we shell out to:-//! python3 -c "import pyatspi; ..." (if available)-//! OR-//! Use x11rb to read basic XA_WM_* properties as a fallback tree.-//!-//! The fallback produces a simplified tree with window title and role.+//! When the AT-SPI bus is unavailable (or the app exposes no a11y tree) we+//! fall back to a minimal X11 property tree (window title + role) via x11rb.use anyhow::Result;-use std::process::Command;pub mod cache;+pub mod native;pub use cache::ElementCache;#[derive(Clone, Debug)]@@ -38,10 +36,12 @@ pub struct AtspiTreeResult {/// Walk the AT-SPI tree for a window identified by (pid, xid)./// Falls back to a minimal X11 property tree if AT-SPI is unavailable.pub fn walk_tree(pid: u32, xid: u64, query: Option<&str>) -> AtspiTreeResult {- // Try pyatspi bridge first (most complete).- if let Ok((raw_md, nodes)) = walk_via_pyatspi(pid) {- let md = if let Some(q) = query { filter_tree(&raw_md, q) } else { raw_md };- return AtspiTreeResult { tree_markdown: md, nodes };+ // Native AT-SPI (most complete).+ if let Ok(Some((raw_md, nodes))) = native::walk_tree(pid) {+ if !raw_md.is_empty() {+ let md = if let Some(q) = query { filter_tree(&raw_md, q) } else { raw_md };+ return AtspiTreeResult { tree_markdown: md, nodes };+ }}// Fallback: X11 window properties as minimal tree.@@ -51,299 +51,102 @@ pub fn walk_tree(pid: u32, xid: u64, query: Option<&str>) -> AtspiTreeResult {/// Perform the first advertised action on element `idx` within pid's app tree./// Returns Ok(action_name) on success.pub fn perform_action(pid: u32, idx: usize) -> Result<String> {- let script = format!(r#"-import pyatspi, sys--elements = []-def collect(acc):- try:- ai = acc.queryAction()- if ai.nActions > 0:- elements.append((acc, [ai.getName(i) for i in range(ai.nActions)]))- except: pass- for child in acc:- collect(child)--desktop = pyatspi.Registry.getDesktop(0)-for app in desktop:- if app.get_process_id() == {pid}:- for win in app:- collect(win)- break--if {idx} < len(elements):- elem, actions = elements[{idx}]- elem.queryAction().doAction(0)- print(actions[0])-else:- print("ERROR: element {idx} not found (total: " + str(len(elements)) + ")", file=sys.stderr)- sys.exit(1)-"#, pid = pid, idx = idx);-- let out = Command::new("python3").arg("-c").arg(&script).output()?;- if !out.status.success() {- anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());- }- Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())+ native::perform_action(pid, idx)}-/// Set the text value of element `idx` within pid's app tree via AT-SPI.-/// Tries `queryEditableText().setTextContents(value)` first,-/// then `queryValue().setCurrentValue(float)`.-pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {- // Escape value for Python string literal: replace \ and ' to be safe.- let safe_value = value.replace('\\', "\\\\").replace('\'', "\\'");+/// Try to type text into any editable field in the window via AT-SPI EditableText.+/// This works for unfocused windows if the toolkit exposes EditableText (Qt6, some GTK).+/// For Qt5, which doesn't expose widgets when unfocused, this will return Err.+/// Returns Ok if an editable was found and text was set, Err otherwise.+pub fn type_into_editable(pid: u32, text: &str) -> Result<()> {+ let safe_text = text.replace('\\', "\\\\").replace('\'', "\\'");let script = format!(r#"import pyatspi, sys-elements = []-def collect(acc):+def find_editable(acc, depth=0):+ # Try to find any EditableText interface, regardless of roletry:- ai = acc.queryAction()- if ai.nActions > 0:- elements.append(acc)- except: pass- for child in acc:- collect(child)--desktop = pyatspi.Registry.getDesktop(0)-for app in desktop:- if app.get_process_id() == {pid}:- for win in app:- collect(win)- break--if {idx} >= len(elements):- print(f"ERROR: element {idx} not found (total: {{len(elements)}})", file=sys.stderr)- sys.exit(1)+ et = acc.queryEditableText()+ # If we can query it, return this node+ return acc+ except:+ pass-elem = elements[{idx}]-try:- et = elem.queryEditableText()- et.setTextContents('{safe_value}')- print("ok:text")-except:+ # Recursively search childrentry:- v = elem.queryValue()- v.setCurrentValue(float('{safe_value}'))- print("ok:value")- except Exception as e:- print(f"ERROR: {{e}}", file=sys.stderr)- sys.exit(1)-"#, pid = pid, idx = idx, safe_value = safe_value);-- let out = Command::new("python3").arg("-c").arg(&script).output()?;- if !out.status.success() {- anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());- }- Ok(())-}--/// Get the screen-coordinate bounding box (x, y, width, height) of element `idx`.-pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> {- let script = format!(r#"-import pyatspi, sys+ for child in acc:+ result = find_editable(child, depth + 1)+ if result is not None:+ return result+ except:+ pass-elements = []-def collect(acc):- try:- ai = acc.queryAction()- if ai.nActions > 0:- elements.append(acc)- except: pass- for child in acc:- collect(child)+ return Nonedesktop = pyatspi.Registry.getDesktop(0)+editable = Nonefor app in desktop:- if app.get_process_id() == {pid}:- for win in app:- collect(win)- break+ try:+ if app.get_process_id() == {pid}:+ for win in app:+ editable = find_editable(win)+ if editable:+ break+ break+ except:+ pass-if {idx} >= len(elements):- print(f"ERROR: element {idx} not found", file=sys.stderr)+if editable is None:+ print("ERROR: No editable found", file=sys.stderr)sys.exit(1)-elem = elements[{idx}]try:- comp = elem.queryComponent()- ext = comp.getExtents(pyatspi.DESKTOP_COORDS)- print(f"{{ext.x}},{{ext.y}},{{ext.width}},{{ext.height}}")+ et = editable.queryEditableText()+ et.setTextContents('{safe_text}')+ print("ok:atspi")except Exception as e:print(f"ERROR: {{e}}", file=sys.stderr)sys.exit(1)-"#, pid = pid, idx = idx);+"#, pid = pid, safe_text = safe_text);- let out = Command::new("python3").arg("-c").arg(&script).output()?;+ let out = std::process::Command::new("python3").arg("-c").arg(&script).output()?;if !out.status.success() {anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());}- let line = String::from_utf8_lossy(&out.stdout).trim().to_owned();- let parts: Vec<i64> = line.split(',')- .filter_map(|s| s.parse().ok())- .collect();- if parts.len() < 4 { anyhow::bail!("unexpected bounds output: {line}"); }- Ok((parts[0] as i32, parts[1] as i32, parts[2] as u32, parts[3] as u32))-}--// ── Internal helpers ─────────────────────────────────────────────────────────--/// Walk via pyatspi subprocess. Returns (markdown, nodes) on success.-fn walk_via_pyatspi(pid: u32) -> Result<(String, Vec<AtspiNode>)> {- let script = format!(r#"-import pyatspi, sys--def walk(acc, depth=0, idx=[0]):- role = acc.getRoleName()- name = acc.name or ""- n_actions = 0- actions = []- try:- ai = acc.queryAction()- n_actions = ai.nActions- actions = [ai.getName(i) for i in range(n_actions)]- except:- pass-- try:- vobj = acc.queryValue()- value_str = str(vobj.currentValue)- except:- value_str = ""-- indent = " " * depth- if n_actions > 0:- act_str = ','.join(actions)- val_part = f' value="{{value_str}}"' if value_str else ''- print(f"{{indent}}- [{{idx[0]}}] {{role}} \"{{name}}\"{{val_part}} [actions=[{{act_str}}]]")- idx[0] += 1- elif name:- print(f"{{indent}}- {{role}} = \"{{name}}\"")-- for child in acc:- walk(child, depth+1, idx)--desktop = pyatspi.Registry.getDesktop(0)-for app in desktop:- if app.get_process_id() == {pid}:- for win in app:- walk(win)- break-"#, pid = pid);-- let out = Command::new("python3")- .arg("-c")- .arg(&script)- .output()?;-- if !out.status.success() || out.stdout.is_empty() {- anyhow::bail!("pyatspi not available or returned empty");- }-- let raw_md = String::from_utf8_lossy(&out.stdout).into_owned();- let nodes = parse_pyatspi_nodes(&raw_md);- Ok((raw_md, nodes))+ Ok(())}-/// Parse pyatspi markdown output into AtspiNode list.-///-/// Recognizes lines like:-/// ` - [3] button "OK" value="1.0" [actions=[click,press,release]]`-fn parse_pyatspi_nodes(md: &str) -> Vec<AtspiNode> {- let mut nodes = Vec::new();- // We need indexed_nodes in order so element_index == position in vec.- let mut indexed: Vec<(usize, AtspiNode)> = Vec::new();-- for line in md.lines() {- let trimmed = line.trim();- if !trimmed.starts_with('-') { continue; }- let rest = trimmed.trim_start_matches('-').trim();-- // Check for indexed element: `[N] role "name" ...`- if rest.starts_with('[') {- if let Some(close) = rest.find(']') {- let idx_str = &rest[1..close];- if let Ok(idx) = idx_str.parse::<usize>() {- let after_idx = rest[close+1..].trim();-- // Parse role (first word) and name (quoted string).- let (role, after_role) = split_first_word(after_idx);- let (name, after_name) = parse_quoted_string(after_role.trim());-- // Parse optional value="..." field.- let value = parse_field(after_name, "value=");-- // Parse [actions=[...]] field.- let actions = parse_actions(after_name);-- let node = AtspiNode {- element_index: Some(idx),- role: role.to_owned(),- name: if name.is_empty() { None } else { Some(name.to_owned()) },- value: if value.is_empty() { None } else { Some(value.to_owned()) },- description: None,- actions,- element_key: idx as u64,- };- indexed.push((idx, node));- }- }- }- }-- // Sort by index and flatten.- indexed.sort_by_key(|(i, _)| *i);- nodes.extend(indexed.into_iter().map(|(_, n)| n));- nodes+/// Set the text value of element `idx` within pid's app tree via AT-SPI.+/// Tries `EditableText.set_text_contents(value)` first, then+/// `Value.set_current_value(float)`.+pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {+ native::set_value(pid, idx, value)}-fn split_first_word(s: &str) -> (&str, &str) {- let s = s.trim();- if let Some(pos) = s.find(|c: char| c.is_whitespace()) {- (&s[..pos], &s[pos..])- } else {- (s, "")- }+/// Insert `text` into a GUI app's editable field via AT-SPI EditableText —+/// focus-free and toolkit-agnostic, unlike X11 key injection which only reaches+/// the *focused* toplevel's focused widget. Targets the focused editable element+/// if the toolkit exposes one, else the first editable element in the tree.+/// Returns Ok(true) if text was inserted, Ok(false) if the app exposes no+/// editable element (so the caller can fall back), Err on an AT-SPI failure.+pub fn insert_text(pid: u32, text: &str) -> Result<bool> {+ native::insert_text(pid, text)}-/// Parse a `"quoted string"` from the start of `s`. Returns (content, rest).-fn parse_quoted_string(s: &str) -> (&str, &str) {- let s = s.trim();- if !s.starts_with('"') { return ("", s); }- let inner = &s[1..];- if let Some(end) = inner.find('"') {- (&inner[..end], &inner[end+1..])- } else {- ("", s)- }+/// Get the screen-coordinate bounding box (x, y, width, height) of element `idx`.+/// Screen-coordinate bounds for every action node in pid's AT-SPI tree, keyed+/// by `element_index`. Best-effort: nodes whose bounds can't be read are+/// omitted rather than erroring the whole call. Returns `(element_index, x, y,+/// width, height)` tuples in screen coordinates.+pub fn get_all_element_bounds(pid: u32) -> Result<Vec<(usize, i32, i32, u32, u32)>> {+ native::get_all_element_bounds(pid)}-/// Parse a field like `value="something"` from a string.-fn parse_field<'a>(s: &'a str, prefix: &str) -> &'a str {- if let Some(pos) = s.find(prefix) {- let after = &s[pos + prefix.len()..];- let (content, _) = parse_quoted_string(after);- content- } else {- ""- }+pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> {+ native::get_element_bounds(pid, idx)}-/// Parse `[actions=[click,press,release]]` from a string.-fn parse_actions(s: &str) -> Vec<String> {- // Find [actions=[...]]- if let Some(start) = s.find("[actions=[") {- let inner = &s[start + "[actions=[".len()..];- if let Some(end) = inner.find("]]") {- return inner[..end].split(',')- .map(|a| a.trim().to_owned())- .filter(|a| !a.is_empty())- .collect();- }- }- vec![]-}+// ── Internal helpers ─────────────────────────────────────────────────────────/// Minimal X11 property-based tree (fallback when AT-SPI is unavailable).fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsnew file mode 100644index 0000000000..a67edb413d--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -0,0 +1,735 @@+//! Native AT-SPI access over D-Bus via the `atspi` crate (zbus).+//!+//! Replaces the previous `python3 -c "import pyatspi; ..."` subprocess bridge:+//! no Python, `pyatspi`, or GObject-introspection typelibs are needed at+//! runtime. The zbus calls are async, so each public entry point drives a+//! small shared Tokio runtime via `block_on` (callers already invoke these+//! from `tokio::task::spawn_blocking`, so blocking here is safe).+//!+//! Element indices match the markdown produced by [`walk_tree`]: a depth-first,+//! pre-order traversal of the target application's windows, numbering only the+//! nodes that advertise AT-SPI actions. `perform_action`, `set_value`, and+//! `get_element_bounds` index into that same ordered set.++use std::sync::OnceLock;+use std::time::Duration;++use anyhow::{anyhow, Result};+use atspi::connection::AccessibilityConnection;+use atspi::proxy::accessible::AccessibleProxy;+use atspi::proxy::proxy_ext::ProxyExt;+use atspi::{CoordType, Interface, State};++use super::AtspiNode;++/// Per-call D-Bus timeout: a single unresponsive accessible (common in large,+/// lazily-built trees like Chromium's) must not stall the whole walk.+const CALL_TIMEOUT: Duration = Duration::from_secs(3);+/// Overall budget for one tree walk / operation.+const OP_TIMEOUT: Duration = Duration::from_secs(25);++/// Run `fut` with [`CALL_TIMEOUT`]; `None` on timeout so the caller can skip+/// the node and keep walking rather than blocking forever.+async fn call<T>(fut: impl std::future::Future<Output = T>) -> Option<T> {+ tokio::time::timeout(CALL_TIMEOUT, fut).await.ok()+}++/// Emit a one-line diagnostic to stderr when `CUA_ATSPI_DEBUG` is set. The+/// driver's stderr is surfaced in the test logs, so this is how we see what the+/// native walk actually found in CI.+fn dbg_enabled() -> bool {+ static ON: OnceLock<bool> = OnceLock::new();+ *ON.get_or_init(|| std::env::var_os("CUA_ATSPI_DEBUG").is_some())+}+macro_rules! dlog {+ ($($arg:tt)*) => {+ if dbg_enabled() { eprintln!("[cua-atspi] {}", format!($($arg)*)); }+ };+}++/// Shared multi-threaded Tokio runtime for the blocking AT-SPI entry points.+fn runtime() -> &'static tokio::runtime::Runtime {+ static RT: OnceLock<tokio::runtime::Runtime> = OnceLock::new();+ RT.get_or_init(|| {+ tokio::runtime::Builder::new_multi_thread()+ .worker_threads(1)+ .enable_all()+ .build()+ .expect("build AT-SPI tokio runtime")+ })+}++/// A node discovered during the pre-order walk, with its proxy retained so the+/// per-index operations can act on it without re-walking the tree.+struct Visited<'a> {+ depth: usize,+ role: String,+ /// Display text: the accessible `name`, or — for editable/text widgets that+ /// expose no name — the Text-interface content (where typed text lives).+ name: String,+ value: Option<String>,+ actions: Vec<String>,+ has_editable: bool,+ has_value: bool,+ has_component: bool,+ focused: bool,+ /// True when an ancestor is a web document (e.g. role "document web"),+ /// i.e. this node is page content rather than browser chrome.+ in_web_doc: bool,+ acc: AccessibleProxy<'a>,+}++/// Role names that denote embedded web/document content. An editable beneath+/// one of these is page content (the field a user means when typing into a+/// background browser) rather than browser chrome like the address bar.+fn is_document_role(role: &str) -> bool {+ let r = role.to_ascii_lowercase();+ r.contains("document") || r == "embedded"+}++/// Build an `AccessibleProxy` for an arbitrary (bus name, path) in the tree.+/// Uses owned `String`s for destination/path so the resulting `BusName`/+/// `ObjectPath` are `'static` and the proxy borrows only the connection.+///+/// `cache_properties(No)` is load-bearing, not just an optimization: with the+/// zbus default (`Lazily`) the first property read on the proxy — our+/// `acc.name()` during the walk — makes zbus issue+/// `org.freedesktop.DBus.Properties.GetAll` (one argument: the interface name)+/// to warm the cache. Qt5's AT-SPI bridge (`AtSpiAdaptor::handleMessage`)+/// assumes every Properties call is `Get`/`Set` and unconditionally reads+/// `message.arguments().at(1)`; for a one-argument `GetAll` that index is out+/// of range, so the following `QVariant::toString()` dereferences garbage and+/// the Qt5 app *segfaults* (observed crash: `libQt5Core` via+/// `AtSpiAdaptor::handleMessage`). Qt6's bridge handles `GetAll`, which is why+/// only Qt5 crashed. Forcing `No` makes zbus issue per-property `Get` calls+/// (two arguments) instead, which Qt5 handles correctly — so the Qt5 window can+/// be walked and written without killing the app. Other toolkits are+/// unaffected (they already tolerate `GetAll`), and the sub-interface proxies+/// from `proxies()` already use `CacheProperties::No`.+async fn accessible_for<'a>(+ conn: &'a atspi::zbus::Connection,+ oref: &atspi::ObjectRefOwned,+) -> Result<AccessibleProxy<'a>> {+ let dest = oref+ .name_as_str()+ .ok_or_else(|| anyhow!("object has no bus name"))?+ .to_owned();+ let path = oref.path_as_str().to_owned();+ AccessibleProxy::builder(conn)+ .cache_properties(atspi::zbus::proxy::CacheProperties::No)+ .destination(dest)+ .map_err(|e| anyhow!("bad a11y destination: {e}"))?+ .path(path)+ .map_err(|e| anyhow!("bad a11y path: {e}"))?+ .build()+ .await+ .map_err(|e| anyhow!("AccessibleProxy build failed: {e}"))+}++/// Resolve the process id behind an application accessible's D-Bus name.+async fn pid_of(dbus: &atspi::zbus::fdo::DBusProxy<'_>, oref: &atspi::ObjectRefOwned) -> Option<u32> {+ let bus = atspi::zbus::names::BusName::try_from(oref.name_as_str()?.to_owned()).ok()?;+ dbus.get_connection_unix_process_id(bus).await.ok()+}++/// Locate the application accessible whose backing process is `pid`.+async fn app_for_pid<'a>(+ conn: &'a AccessibilityConnection,+ pid: u32,+) -> Result<Option<AccessibleProxy<'a>>> {+ let zconn = conn.connection();+ let root = conn+ .root_accessible_on_registry()+ .await+ .map_err(|e| anyhow!("registry root unavailable: {e}"))?;+ let dbus = atspi::zbus::fdo::DBusProxy::new(zconn)+ .await+ .map_err(|e| anyhow!("DBus proxy unavailable: {e}"))?;++ let apps = root.get_children().await.unwrap_or_default();+ dlog!("registry root has {} application(s); seeking pid {pid}", apps.len());+ for child in apps {+ let cpid = pid_of(&dbus, &child).await;+ dlog!(" app bus={:?} pid={:?}", child.name_as_str(), cpid);+ if cpid == Some(pid) {+ return Ok(Some(accessible_for(zconn, &child).await?));+ }+ }+ dlog!("no application accessible matched pid {pid}");+ Ok(None)+}++/// Depth-first, pre-order walk of an application's windows. Mirrors the old+/// pyatspi `walk`/`collect` traversal so element indices stay stable.+async fn collect_visited<'a>(+ conn: &'a AccessibilityConnection,+ pid: u32,+) -> Result<Option<Vec<Visited<'a>>>> {+ let app = match app_for_pid(conn, pid).await? {+ Some(a) => a,+ None => return Ok(None),+ };+ let zconn = conn.connection();++ // Stack of (object ref, depth, in_web_doc). Seed with the app's windows;+ // push children reversed so siblings pop left-to-right and each subtree+ // completes before the next sibling (pre-order). `in_web_doc` is inherited+ // from ancestors so editables in page content can be told from chrome.+ let mut stack: Vec<(atspi::ObjectRefOwned, usize, bool)> = match call(app.get_children()).await {+ Some(Ok(children)) => children.into_iter().rev().map(|r| (r, 0usize, false)).collect(),+ _ => Vec::new(),+ };++ let mut visited: Vec<Visited<'a>> = Vec::new();+ // Guard against pathological/looping trees.+ let mut budget = 5000usize;++ while let Some((oref, depth, in_web_doc)) = stack.pop() {+ if budget == 0 {+ dlog!("node budget exhausted; truncating walk");+ break;+ }+ budget -= 1;++ let acc = match accessible_for(zconn, &oref).await {+ Ok(a) => a,+ Err(_) => continue,+ };++ // Interfaces gate every other query; if even this times out the node is+ // unreachable, so skip it rather than stall.+ let ifaces = match call(acc.get_interfaces()).await {+ Some(Ok(i)) => i,+ _ => continue,+ };+ let has_action = ifaces.contains(Interface::Action);+ let has_editable = ifaces.contains(Interface::EditableText);+ let has_value = ifaces.contains(Interface::Value);+ let has_component = ifaces.contains(Interface::Component);+ let has_text = ifaces.contains(Interface::Text);++ // These four are independent — issue them concurrently to cut the+ // per-node round-trip cost (large trees like Chromium's have hundreds+ // of nodes, so sequential reads dominate the walk time).+ let (role_r, name_r, state_r, children_r) = tokio::join!(+ call(acc.get_role_name()),+ call(acc.name()),+ call(acc.get_state()),+ call(acc.get_children()),+ );+ let role = match role_r {+ Some(Ok(r)) => r,+ _ => String::new(),+ };+ let mut name = match name_r {+ Some(Ok(n)) => n,+ _ => String::new(),+ };+ let focused = matches!(state_r, Some(Ok(s)) if s.contains(State::Focused));++ // Collect action names, numeric value, and (crucially) Text-interface+ // content. Only touch `proxies` when an interface is actually present,+ // and drop the borrow before `acc` moves into `visited`.+ let mut actions: Vec<String> = Vec::new();+ let mut value: Option<String> = None;+ let mut text_content = String::new();+ if has_action || has_value || has_text {+ if let Some(Ok(proxies)) = call(acc.proxies()).await {+ if has_action {+ if let Some(Ok(ap)) = call(proxies.action()).await {+ let n = call(ap.n_actions()).await.and_then(|r| r.ok()).unwrap_or(0);+ for i in 0..n {+ if let Some(Ok(an)) = call(ap.get_name(i)).await {+ actions.push(an);+ }+ }+ }+ }+ if has_value {+ if let Some(Ok(vp)) = call(proxies.value()).await {+ value = call(vp.current_value()).await.and_then(|r| r.ok()).map(format_value);+ }+ }+ // Text content is where editable/entry text (the typed string)+ // lives; `name` is usually empty for such widgets.+ if has_text {+ if let Some(Ok(tp)) = call(proxies.text()).await {+ let count = call(tp.character_count()).await.and_then(|r| r.ok()).unwrap_or(0);+ if count > 0 {+ let end = count.min(4096);+ if let Some(Ok(t)) = call(tp.get_text(0, end)).await {+ text_content = t;+ }+ }+ }+ }+ }+ }++ // Surface Text content as the display name when the widget has no name.+ if name.trim().is_empty() && !text_content.trim().is_empty() {+ name = text_content;+ }++ // Children inherit web-document context, plus this node's own role.+ let child_in_web_doc = in_web_doc || is_document_role(&role);++ // Enqueue children (fetched above) before moving `acc` into `visited`.+ if let Some(Ok(children)) = children_r {+ for c in children.into_iter().rev() {+ stack.push((c, depth + 1, child_in_web_doc));+ }+ }++ visited.push(Visited {+ depth,+ role,+ name,+ value,+ actions,+ has_editable,+ has_value,+ has_component,+ focused,+ in_web_doc,+ acc,+ });+ }++ dlog!("walked pid {pid}: {} node(s)", visited.len());+ Ok(Some(visited))+}++/// Render visited nodes into the markdown + node list `walk_tree` returns.+/// Format matches the historical pyatspi output exactly so downstream parsing+/// (`extract_text_from_markdown`, `query_dom`) is unaffected.+fn render(visited: &[Visited<'_>]) -> (String, Vec<AtspiNode>) {+ let mut md = String::new();+ let mut nodes = Vec::new();+ let mut idx = 0usize;++ for v in visited {+ let indent = " ".repeat(v.depth);+ if !v.actions.is_empty() {+ let act_str = v.actions.join(",");+ let val_part = match &v.value {+ Some(val) if !val.is_empty() => format!(" value=\"{val}\""),+ _ => String::new(),+ };+ md.push_str(&format!(+ "{indent}- [{idx}] {role} \"{name}\"{val_part} [actions=[{act_str}]]\n",+ role = v.role,+ name = v.name,+ ));+ nodes.push(AtspiNode {+ element_index: Some(idx),+ role: v.role.clone(),+ name: if v.name.is_empty() { None } else { Some(v.name.clone()) },+ value: v.value.clone().filter(|s| !s.is_empty()),+ description: None,+ actions: v.actions.clone(),+ element_key: idx as u64,+ });+ idx += 1;+ } else if !v.name.is_empty() {+ md.push_str(&format!(+ "{indent}- {role} = \"{name}\"\n",+ role = v.role,+ name = v.name,+ ));+ }+ }++ (md, nodes)+}++/// Format an AT-SPI numeric value like the historical `str(currentValue)`+/// (e.g. `1.0`), so `value="..."` fields stay byte-compatible.+fn format_value(v: f64) -> String {+ format!("{v:?}")+}++// ── Public (sync) entry points ───────────────────────────────────────────────++pub fn walk_tree(pid: u32) -> Result<Option<(String, Vec<AtspiNode>)>> {+ runtime().block_on(async {+ let work = async {+ let conn = AccessibilityConnection::new()+ .await+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;+ match collect_visited(&conn, pid).await? {+ Some(visited) => Ok(Some(render(&visited))),+ None => Ok(None),+ }+ };+ match tokio::time::timeout(OP_TIMEOUT, work).await {+ Ok(r) => r,+ Err(_) => {+ dlog!("walk_tree timed out for pid {pid}");+ Ok(None)+ }+ }+ })+}++/// Pick the editable node to write into, by priority:+/// 1. the focused editable (if the toolkit exposes focus),+/// 2. an editable inside web/document content — for a browser this is the+/// page's field, not the address bar (which sorts first in the tree but is+/// chrome),+/// 3. the first editable anywhere (covers single-field apps like a GTK dialog+/// entry, or a GTK4 GtkEntry).+fn pick_editable<'v, 'a>(visited: &'v [Visited<'a>]) -> Option<&'v Visited<'a>> {+ visited+ .iter()+ .find(|v| v.has_editable && v.focused)+ .or_else(|| visited.iter().find(|v| v.has_editable && v.in_web_doc))+ .or_else(|| visited.iter().find(|v| v.has_editable))+}++/// Try to write `text` into the best editable node in `visited` via AT-SPI+/// EditableText (GrabFocus first so the toolkit exposes the field on an+/// unfocused window's focused widget). Returns `Ok(true)` if the write landed,+/// `Ok(false)` if no editable was found / the EditableText write was rejected.+async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result<bool> {+ let target = match pick_editable(visited) {+ Some(t) => t,+ None => return Ok(false),+ };+ dlog!(+ "insert target: role={:?} in_web_doc={} focused={} has_component={}",+ target.role, target.in_web_doc, target.focused, target.has_component+ );++ let proxies = target+ .acc+ .proxies()+ .await+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;++ // Try to grab focus on the widget via AT-SPI Component.GrabFocus.+ // This should give the widget internal keyboard focus without activating+ // the window, allowing GTK4 (and similar toolkits) to expose EditableText+ // on an unfocused window's focused widget.+ if target.has_component {+ if let Ok(comp) = proxies.component().await {+ match call(comp.grab_focus()).await {+ Some(Ok(true)) => dlog!("GrabFocus succeeded on {:?}", target.role),+ Some(Ok(false)) => dlog!("GrabFocus returned false on {:?}", target.role),+ Some(Err(e)) => dlog!("GrabFocus failed on {:?}: {}", target.role, e),+ None => dlog!("GrabFocus timed out on {:?}", target.role),+ }+ } else {+ dlog!("Component interface unavailable despite has_component=true");+ }+ } else {+ dlog!("Target has no Component interface, skipping GrabFocus");+ }++ let et = proxies+ .editable_text()+ .await+ .map_err(|e| anyhow!("EditableText unavailable: {e}"))?;++ let off = match proxies.text().await {+ Ok(tp) => tp.caret_offset().await.unwrap_or(0),+ Err(_) => 0,+ };+ let len = text.chars().count() as i32;++ if et.insert_text(off, text, len).await.unwrap_or(false) {+ return Ok(true);+ }+ if et.set_text_contents(text).await.unwrap_or(false) {+ return Ok(true);+ }+ Ok(false)+}++pub fn insert_text(pid: u32, text: &str) -> Result<bool> {+ runtime().block_on(async {+ let conn = AccessibilityConnection::new()+ .await+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;+ let visited = match collect_visited(&conn, pid).await? {+ Some(v) => v,+ None => return Ok(false),+ };++ dlog!(+ "insert_text: {} node(s), {} editable, {} entry/text-role",+ visited.len(),+ visited.iter().filter(|v| v.has_editable).count(),+ visited.iter().filter(|v| v.role.contains("entry") || v.role.contains("text")).count(),+ );++ // Primary attempt: write into an editable exposed by the current tree.+ // GrabFocus (inside write_into_editable) gives the widget internal+ // keyboard focus without activating the window, so toolkits that expose+ // EditableText on an unfocused window (Qt6, and GTK4 with GTK_A11Y=atspi)+ // accept the write here.+ if write_into_editable(&visited, text).await? {+ return Ok(true);+ }++ // GTK3 fallback: the toolkit exposes entry/text nodes in the tree (so+ // get_text reads work) but gates EditableText on focus/activation. Try+ // finding an entry/text role with Component bounds and use X11 click+type.+ dlog!("AT-SPI EditableText unavailable; checking for entry/text with Component for X11 fallback");++ let entry_candidate = visited+ .iter()+ .find(|v| {+ let r = v.role.to_ascii_lowercase();+ (r.contains("entry") || r.contains("text")) && v.has_component+ });++ if let Some(entry) = entry_candidate {+ dlog!(+ "GTK3 fallback: found entry role={:?} with Component; attempting X11 click+type",+ entry.role+ );++ // Get the entry widget's screen bounds via Component.GetExtents.+ if let Ok(proxies) = entry.acc.proxies().await {+ if let Ok(comp) = proxies.component().await {+ if let Some(Ok((x, y, w, h))) = call(comp.get_extents(CoordType::Screen)).await {+ // Click the center of the entry to establish widget focus (not window focus).+ let cx = x + (w.max(0) / 2);+ let cy = y + (h.max(0) / 2);+ dlog!("GTK3 fallback: entry bounds ({x},{y} {w}x{h}), clicking center ({cx},{cy})");++ // Get the window XID for this app so we can send X11 events to it.+ let Some(xid) = entry_find_window_xid(pid).await else {+ dlog!("GTK3 fallback: could not find window XID");+ return Ok(false);+ };++ // Translate screen coords to window-local coords for XSendEvent.+ let Some((wx, wy)) = screen_to_window_coords(xid, cx, cy) else {+ dlog!("GTK3 fallback: screen-to-window coord translation failed");+ return Ok(false);+ };++ dlog!("GTK3 fallback: window XID {xid}, local coords ({wx},{wy})");++ // Click the entry to focus the widget (widget focus, not window focus).+ if let Err(e) = crate::input::send_click(xid as u64, wx, wy, 1, 1) {+ dlog!("GTK3 fallback: click failed: {e}");+ return Ok(false);+ };++ // Small delay for the click to register and the widget to update focus.+ tokio::time::sleep(tokio::time::Duration::from_millis(150)).await;++ // Now type via X11 XSendEvent — the entry widget has internal focus+ // so it should accept the keystrokes even though the window is unfocused.+ if let Err(e) = crate::input::send_type_text(xid as u64, text) {+ dlog!("GTK3 fallback: send_type_text failed: {e}");+ return Ok(false);+ }++ dlog!("GTK3 fallback: X11 click+type succeeded");+ return Ok(true);+ }+ }+ }+ }++ Ok(false)+ })+}++/// Find the window XID for a PID by listing its X11 windows.+async fn entry_find_window_xid(pid: u32) -> Option<u64> {+ use crate::x11::list_windows;++ // List X11 windows for that PID and return the first one.+ let windows = list_windows(Some(pid));+ let xid = windows.first()?.xid;+ Some(xid)+}++/// Translate screen coordinates to window-local coordinates.+fn screen_to_window_coords(xid: u64, screen_x: i32, screen_y: i32) -> Option<(i32, i32)> {+ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::*;+ use x11rb::rust_connection::RustConnection;++ let (conn, _) = RustConnection::connect(None).ok()?;+ let window = xid as u32;++ // Get window geometry to find its screen position.+ let geom = conn.get_geometry(window).ok()?.reply().ok()?;++ // Translate to root coordinates (screen coords of window's origin).+ let trans = conn.translate_coordinates(window, geom.root, 0, 0).ok()?.reply().ok()?;++ // Window-local = screen - window_origin.+ Some((screen_x - trans.dst_x as i32, screen_y - trans.dst_y as i32))+}++pub fn perform_action(pid: u32, idx: usize) -> Result<String> {+ runtime().block_on(async {+ let conn = AccessibilityConnection::new()+ .await+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;+ let visited = collect_visited(&conn, pid)+ .await?+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| !v.actions.is_empty()).collect();+ let target = action_nodes+ .get(idx)+ .ok_or_else(|| anyhow!("element {idx} not found (total: {})", action_nodes.len()))?;++ let ap = target+ .acc+ .proxies()+ .await+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?+ .action()+ .await+ .map_err(|e| anyhow!("Action unavailable: {e}"))?;+ ap.do_action(0)+ .await+ .map_err(|e| anyhow!("doAction failed: {e}"))?;+ Ok(target.actions.first().cloned().unwrap_or_default())+ })+}++pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {+ runtime().block_on(async {+ let conn = AccessibilityConnection::new()+ .await+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;+ let visited = collect_visited(&conn, pid)+ .await?+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| !v.actions.is_empty()).collect();+ let target = action_nodes+ .get(idx)+ .ok_or_else(|| anyhow!("element {idx} not found (total: {})", action_nodes.len()))?;++ let proxies = target+ .acc+ .proxies()+ .await+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;++ if target.has_editable {+ if let Ok(et) = proxies.editable_text().await {+ if et.set_text_contents(value).await.unwrap_or(false) {+ return Ok(());+ }+ }+ }+ if target.has_value {+ let v: f64 = value+ .parse()+ .map_err(|_| anyhow!("value '{value}' is not numeric for a Value element"))?;+ proxies+ .value()+ .await+ .map_err(|e| anyhow!("Value unavailable: {e}"))?+ .set_current_value(v)+ .await+ .map_err(|e| anyhow!("setCurrentValue failed: {e}"))?;+ return Ok(());+ }+ Err(anyhow!("element {idx} exposes neither EditableText nor Value"))+ })+}++pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> {+ runtime().block_on(async {+ let conn = AccessibilityConnection::new()+ .await+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;+ let visited = collect_visited(&conn, pid)+ .await?+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| !v.actions.is_empty()).collect();+ let target = action_nodes+ .get(idx)+ .ok_or_else(|| anyhow!("element {idx} not found"))?;+ if !target.has_component {+ return Err(anyhow!("element {idx} exposes no Component interface"));+ }+ let comp = target+ .acc+ .proxies()+ .await+ .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?+ .component()+ .await+ .map_err(|e| anyhow!("Component unavailable: {e}"))?;+ let (x, y, w, h) = comp+ .get_extents(CoordType::Screen)+ .await+ .map_err(|e| anyhow!("getExtents failed: {e}"))?;+ Ok((x, y, w.max(0) as u32, h.max(0) as u32))+ })+}++/// Screen-coordinate bounds for every action node in the tree, keyed by the+/// same `element_index` used by [`walk_tree`]/`get_element_bounds`.+///+/// Walks the application once (unlike calling `get_element_bounds` per node,+/// which would reconnect and re-walk every time) and queries each node's+/// `Component.GetExtents(Screen)`. Nodes without a usable Component interface,+/// or whose extents query fails/times out, are silently skipped — the result is+/// best-effort and never errors on a per-node hiccup.+///+/// Returns `(element_index, x, y, width, height)` tuples.+pub fn get_all_element_bounds(pid: u32) -> Result<Vec<(usize, i32, i32, u32, u32)>> {+ runtime().block_on(async {+ let conn = AccessibilityConnection::new()+ .await+ .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;+ let visited = collect_visited(&conn, pid)+ .await?+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| !v.actions.is_empty()).collect();+ // Each element costs ~3 D-Bus round-trips (proxies + component ++ // GetExtents). Big trees (geany exposes ~787 nodes) would grind for+ // minutes and time out callers, so cap the walk; pre-order means the+ // first nodes are the window chrome / toolbars that are actually+ // visible, which is what bounds consumers (overlays, targeting) need.+ const MAX_BOUNDS_NODES: usize = 150;+ // Hard wall-clock budget for the whole collection: on pathological+ // trees individual D-Bus calls each burn up to CALL_TIMEOUT (geany's+ // unrealized nodes did exactly that), so a per-node cap alone can+ // still add up to minutes. Return whatever was collected in time.+ let deadline = std::time::Instant::now() + Duration::from_secs(20);+ let mut out = Vec::with_capacity(action_nodes.len().min(MAX_BOUNDS_NODES));+ for (idx, node) in action_nodes.iter().enumerate().take(MAX_BOUNDS_NODES) {+ if std::time::Instant::now() >= deadline {+ dlog!("get_all_element_bounds: 20s budget exhausted at node {idx}; returning {} bound(s)", out.len());+ break;+ }+ if !node.has_component {+ continue;+ }+ let proxies = match call(node.acc.proxies()).await {+ Some(Ok(p)) => p,+ _ => continue,+ };+ let comp = match call(proxies.component()).await {+ Some(Ok(c)) => c,+ _ => continue,+ };+ if let Some(Ok((x, y, w, h))) = call(comp.get_extents(CoordType::Screen)).await {+ // Unrealized widgets (e.g. items inside closed menus/popovers)+ // report GetExtents as the i32::MIN sentinel and/or a degenerate+ // 0x0 / 1x1 size. Emitting those poisons downstream consumers+ // (overlay renderers, click targeting), so keep only elements+ // with plausible on-screen geometry.+ if x == i32::MIN || y == i32::MIN || x < -16384 || y < -16384 || w <= 1 || h <= 1 {+ continue;+ }+ out.push((idx, x, y, w as u32, h as u32));+ }+ }+ Ok(out)+ })+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rsindex fea1287bc4..0616d59d6a 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs@@ -1,12 +1,15 @@//! Background input injection for Linux via X11 XSendEvent.//!//! XSendEvent sends synthetic events directly to a window without changing-//! input focus. This is the Linux equivalent of PostMessage on Windows.+//! input focus — the Linux equivalent of PostMessage on Windows, and the+//! mechanism behind the cross-platform "no focus steal" contract.//!-//! Note: Some apps check the `send_event` flag and ignore synthetic events-//! (e.g., some games, some security-sensitive apps). For those, the XTest-//! extension (XTestFakeKeyEvent) is the alternative, but it DOES send to-//! the focused window.+//! Note: a few apps check the `send_event` flag and ignore synthetic events.+//! Terminal emulators are the notable case (xterm's `allowSendEvents` is off by+//! default); those are handled out of band by writing to the pty master — see+//! `crate::tty`. We deliberately do NOT fall back to the XTest extension for+//! them, because XTest delivers to the *focused* window and would break the+//! no-focus-steal contract.use anyhow::Result;use std::thread::sleep;@@ -18,6 +21,44 @@ use x11rb::rust_connection::RustConnection;const CLICK_DELAY_MS: u64 = 35;const KEY_DELAY_MS: u64 = 10;+/// Send a synthetic FocusIn event to a window without changing the actual X11 input focus.+/// This can trigger toolkit-level focus handlers (e.g., Qt5's AT-SPI bridge) without+/// moving the window manager's active window. Use with send_focus_out to restore state.+pub fn send_focus_in(xid: u64) -> Result<()> {+ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;++ let focus_in = FocusInEvent {+ response_type: FOCUS_IN_EVENT,+ detail: NotifyDetail::NONLINEAR,+ sequence: 0,+ event: window,+ mode: NotifyMode::NORMAL,+ };++ conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_in)?;+ conn.flush()?;+ Ok(())+}++/// Send a synthetic FocusOut event to restore focus state after send_focus_in.+pub fn send_focus_out(xid: u64) -> Result<()> {+ let (conn, _) = RustConnection::connect(None)?;+ let window = xid as u32;++ let focus_out = FocusOutEvent {+ response_type: FOCUS_OUT_EVENT,+ detail: NotifyDetail::NONLINEAR,+ sequence: 0,+ event: window,+ mode: NotifyMode::NORMAL,+ };++ conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_out)?;+ conn.flush()?;+ Ok(())+}+/// Send a button click (down + up) to a window at window-local coordinates.pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result<()> {let (conn, _) = RustConnection::connect(None)?;@@ -148,48 +189,7 @@ pub fn send_drag(/// Type a string by sending KeyPress/KeyRelease events for each character.pub fn send_type_text(xid: u64, text: &str) -> Result<()> {- let (conn, _) = RustConnection::connect(None)?;- let window = xid as u32;- let root = conn.setup().roots[0].root;-- for ch in text.chars() {- let keycode = char_to_keycode(&conn, ch).unwrap_or(0);- if keycode == 0 { continue; }-- let press = KeyPressEvent {- response_type: KEY_PRESS_EVENT,- detail: keycode,- sequence: 0,- time: x11rb::CURRENT_TIME,- root,- event: window,- child: x11rb::NONE,- root_x: 0, root_y: 0,- event_x: 0, event_y: 0,- state: KeyButMask::from(0u16),- same_screen: true,- };-- let release = KeyReleaseEvent {- response_type: KEY_RELEASE_EVENT,- detail: keycode,- sequence: 0,- time: x11rb::CURRENT_TIME,- root,- event: window,- child: x11rb::NONE,- root_x: 0, root_y: 0,- event_x: 0, event_y: 0,- state: KeyButMask::from(0u16),- same_screen: true,- };-- conn.send_event(false, window, EventMask::KEY_PRESS, &press)?;- sleep(Duration::from_millis(KEY_DELAY_MS));- conn.send_event(false, window, EventMask::KEY_RELEASE, &release)?;- conn.flush()?;- }- Ok(())+ send_type_text_with_delay(xid, text, 0)}/// Type a string with an additional `inter_char_ms` delay between each character.@@ -197,10 +197,16 @@ pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Relet (conn, _) = RustConnection::connect(None)?;let window = xid as u32;let root = conn.setup().roots[0].root;+ let mapping = conn.get_keyboard_mapping(8, 248)?.reply()?;for ch in text.chars() {- let keycode = char_to_keycode(&conn, ch).unwrap_or(0);- if keycode == 0 { continue; }+ // Resolve the keycode and whether Shift must be held — without it,+ // uppercase and shifted symbols would otherwise type their unshifted+ // form (e.g. "A" arriving as "a").+ let Some((keycode, needs_shift)) = char_to_keycode_shift(&mapping, ch as u32) else {+ continue;+ };+ let state = if needs_shift { KeyButMask::SHIFT } else { KeyButMask::from(0u16) };let press = KeyPressEvent {response_type: KEY_PRESS_EVENT,@@ -209,7 +215,7 @@ pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Retime: x11rb::CURRENT_TIME,root, event: window, child: x11rb::NONE,root_x: 0, root_y: 0, event_x: 0, event_y: 0,- state: KeyButMask::from(0u16),+ state,same_screen: true,};let release = KeyReleaseEvent {@@ -219,7 +225,7 @@ pub fn send_type_text_with_delay(xid: u64, text: &str, inter_char_ms: u64) -> Retime: x11rb::CURRENT_TIME,root, event: window, child: x11rb::NONE,root_x: 0, root_y: 0, event_x: 0, event_y: 0,- state: KeyButMask::from(0u16),+ state,same_screen: true,};@@ -278,22 +284,24 @@ pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {Ok(())}-fn char_to_keycode(conn: &RustConnection, ch: char) -> Option<u8> {- // Use XStringToKeysym equivalent: look up by character keysym.- // Keysym for ASCII is just the ASCII code.- let keysym: u32 = ch as u32;- conn.get_keyboard_mapping(8, 248).ok()?- .reply().ok()- .map(|km| {- let keysyms_per = km.keysyms_per_keycode as usize;- for (i, syms) in km.keysyms.chunks(keysyms_per).enumerate() {- if syms.iter().any(|&s| s == keysym) {- return Some((8 + i) as u8);- }- }- None- })- .flatten()+/// Find the keycode that emits `keysym`, plus whether Shift must be held (the+/// keysym sits in the shifted column of the keyboard map). Prefers the+/// unshifted column when a keysym appears in both. Keysym for ASCII / Latin-1+/// is just the codepoint.+fn char_to_keycode_shift(mapping: &GetKeyboardMappingReply, keysym: u32) -> Option<(u8, bool)> {+ let per = mapping.keysyms_per_keycode as usize;+ if per == 0 {+ return None;+ }+ for (i, syms) in mapping.keysyms.chunks(per).enumerate() {+ if syms.first() == Some(&keysym) {+ return Some(((8 + i) as u8, false));+ }+ if per > 1 && syms.get(1) == Some(&keysym) {+ return Some(((8 + i) as u8, true));+ }+ }+ None}fn key_name_to_keycode(conn: &RustConnection, key: &str) -> Result<u8> {@@ -346,3 +354,98 @@ fn modifiers_to_state(modifiers: &[&str]) -> KeyButMask {}KeyButMask::from(state)}++/// Inject text into a Tk window via Tk's `send` command — the Tk-specific+/// override for focus-free writes (Tk has no AT-SPI bridge). Requires the target+/// app to have registered itself with a known name via `tk appname <name>`.+/// Returns Ok(true) if text was sent, Ok(false) if the target isn't reachable+/// (not a Tk app or `wish` unavailable), Err on a send failure.+pub fn inject_tk_send(text: &str) -> Result<bool> {+ use std::io::Write;++ // Escape the text for safe Tcl interpolation (braces for literal strings).+ // Tcl's `send` command: `send <target-app-name> <tcl-command>`.+ // We target "cua-tk-target" (the name the test app registers with) and+ // insert at the entry widget's current cursor position.+ let tcl_text = text.replace("\\", "\\\\").replace("{", "\\{").replace("}", "\\}");++ // Tk's `send` is synchronous: it blocks the sender until the *target's* Tcl+ // event loop services the request and replies. If the target is wedged, or+ // the X server refuses `send` (SECURITY ext / xauth mismatch), it can block+ // forever. Guard against that two ways:+ // 1. A Tcl-level `after` timer that force-exits wish if the send hasn't+ // completed in time. We issue the write with `send -async` so the local+ // event loop stays live to fire the timer, then `vwait` on a flag.+ // 2. A Rust-level wall-clock kill below, so even a totally wedged wish+ // (e.g. blocked before reaching the event loop) can't hang the driver.+ let tcl_script = format!(+ r#"set ::done 0+set ::rc 0+after 5000 {{ set ::rc 2; set ::done 1 }}+if {{[catch {{send -async cua-tk-target {{.entry insert insert {{{}}}}}}} err]}} {{+ puts stderr "tk send failed: $err"+ exit 1+}}+after 500 {{ set ::done 1 }}+vwait ::done+if {{$::rc == 2}} {{+ puts stderr "tk send timed out"+ exit 1+}}+exit 0"#,+ tcl_text+ );++ // Try to spawn wish (Tk's shell). If it's not available, this isn't a+ // Tk-based environment and we should fall back to XSendEvent.+ let mut child = match std::process::Command::new("wish")+ .stdin(std::process::Stdio::piped())+ .stdout(std::process::Stdio::piped())+ .stderr(std::process::Stdio::piped())+ .spawn() {+ Ok(c) => c,+ Err(_) => return Ok(false),+ };++ if let Some(mut stdin) = child.stdin.take() {+ // Ignore write errors: if wish already exited we observe it via wait().+ let _ = stdin.write_all(tcl_script.as_bytes());+ // stdin drops here → EOF, so wish runs the script to completion.+ }++ // Wall-clock backstop: poll for exit and hard-kill if wish overruns the+ // deadline. Guarantees the driver task can never hang on a blocked Tk send,+ // regardless of whether the Tcl-level timer fired.+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15);+ loop {+ match child.try_wait()? {+ Some(status) => {+ let mut stderr = String::new();+ if let Some(mut err) = child.stderr.take() {+ use std::io::Read;+ let _ = err.read_to_string(&mut stderr);+ }+ if status.success() {+ return Ok(true);+ }+ // Target not registered or send timed out → not a usable Tk+ // target; let the caller fall back to XSendEvent.+ if stderr.contains("application named")+ || stderr.contains("no registered")+ || stderr.contains("timed out")+ {+ return Ok(false);+ }+ anyhow::bail!("wish send failed: {}", stderr);+ }+ None => {+ if std::time::Instant::now() >= deadline {+ let _ = child.kill();+ let _ = child.wait();+ return Ok(false);+ }+ std::thread::sleep(std::time::Duration::from_millis(50));+ }+ }+ }+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rsindex 525e251e54..36fbcf15a9 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs@@ -22,6 +22,9 @@ pub mod x11;#[cfg(target_os = "linux")]pub mod input;+#[cfg(target_os = "linux")]+pub mod tty;+#[cfg(target_os = "linux")]pub mod proc_fs;diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rsindex 7ffd0d16d3..01dd261263 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs@@ -29,6 +29,7 @@ use cursor_overlay::ZOrderEnforcer;static CMD_TX: OnceLock<std::sync::mpsc::SyncSender<OverlayCommand>> = OnceLock::new();static CMD_RX_CELL: Mutex<Option<std::sync::mpsc::Receiver<OverlayCommand>>> = Mutex::new(None);static RENDER: Mutex<Option<RenderState>> = Mutex::new(None);+static ARRIVAL_TX: Mutex<Option<tokio::sync::oneshot::Sender<()>>> = Mutex::new(None);pub fn init(cfg: CursorConfig) {let (tx, rx) = std::sync::mpsc::sync_channel(4096);@@ -43,6 +44,48 @@ pub fn send_command(cmd: OverlayCommand) {}}+pub fn is_enabled() -> bool {+ RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|rs| rs.core.visible))+ .unwrap_or(false)+}++pub fn current_position() -> (f64, f64) {+ RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|rs| rs.core.pos))+ .unwrap_or((-200.0, -200.0))+}++pub async fn animate_cursor_to(x: f64, y: f64) {+ let should_animate = {+ let guard = RENDER.lock().unwrap();+ match guard.as_ref() {+ Some(rs) if rs.core.cfg.enabled && rs.core.visible && rs.core.pos.0 > -50.0 => true,+ _ => false,+ }+ };+ if !should_animate {+ return;+ }++ let (tx, rx) = tokio::sync::oneshot::channel::<()>();+ {+ let mut guard = ARRIVAL_TX.lock().unwrap();+ if let Some(old_tx) = guard.take() {+ let _ = old_tx.send(());+ }+ *guard = Some(tx);+ }++ send_command(OverlayCommand::MoveTo {+ x,+ y,+ end_heading_radians: std::f64::consts::FRAC_PI_4,+ });++ let _ = rx.await;+}+/// Spawn the overlay on a dedicated thread. Non-blocking.pub fn run_on_thread() {let rx = match CMD_RX_CELL.lock().unwrap().take() {@@ -92,8 +135,8 @@ impl RenderState {}}- fn tick(&mut self, dt: f64) {- self.core.tick_motion(dt);+ fn tick(&mut self, dt: f64) -> bool {+ self.core.tick_motion(dt)}fn apply_command(&mut self, cmd: OverlayCommand) {@@ -215,15 +258,17 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayColast_tick = now;// Drain commands and tick.- {+ let fire_arrival = {let mut guard = RENDER.lock().unwrap();if let Some(rs) = guard.as_mut() {while let Ok(cmd) = rx.try_recv() {rs.apply_command(cmd);}- rs.tick(dt);+ rs.tick(dt)+ } else {+ false}- }+ };// Render and paint.let pixmap = {@@ -243,6 +288,12 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver<OverlayCopaint_x11(&conn, win, scr_w, scr_h, depth, visual_id, &pm);}+ if fire_arrival {+ if let Some(tx) = ARRIVAL_TX.lock().unwrap().take() {+ let _ = tx.send(());+ }+ }+// Z-order maintenance every 80ms — delegate to the cross-platform// ZOrderEnforcer so the contract for "z+1 of the application under// test" is documented once in `cursor_overlay::z_order`.diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex 6ddffad08e..57d31539cf 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -3,6 +3,8 @@use async_trait::async_trait;use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};use serde_json::{json, Value};+use std::fs;+use std::path::PathBuf;use std::sync::{Arc, RwLock};use crate::atspi::ElementCache;@@ -360,6 +362,13 @@ impl Tool for GetWindowStateTool {} else {None};+ // Best-effort per-element screen bounds (AT-SPI Component.GetExtents).+ // Tolerant: an empty/missing map never fails the call.+ let bounds = if do_tree {+ crate::atspi::get_all_element_bounds(pid).unwrap_or_default()+ } else {+ Vec::new()+ };let screenshot = if do_shot {match crate::capture::screenshot_window_bytes(xid) {Ok(raw) => {@@ -375,11 +384,11 @@ impl Tool for GetWindowStateTool {} else {None};- Ok((tree_result, screenshot))+ Ok((tree_result, screenshot, bounds))}).await;match result {- Ok(Ok((tree_opt, shot_opt))) => {+ Ok(Ok((tree_opt, shot_opt, bounds))) => {let mut content = Vec::new();let mut structured = json!({ "window_id": xid, "pid": pid });@@ -390,6 +399,31 @@ impl Tool for GetWindowStateTool {state.element_cache.update(pid, xid, &tr.nodes);structured["element_count"] = json!(count);structured["tree_markdown"] = json!(tr.tree_markdown);++ // Additive `elements` array: one entry per tree node that+ // has an element_index AND a successfully-resolved screen+ // bounds (AT-SPI Component.GetExtents(Screen)).+ use std::collections::HashMap;+ let bounds_by_idx: HashMap<usize, (i32, i32, u32, u32)> = bounds+ .into_iter()+ .map(|(i, x, y, w, h)| (i, (x, y, w, h)))+ .collect();+ let elements: Vec<serde_json::Value> = tr+ .nodes+ .iter()+ .filter_map(|n| {+ let idx = n.element_index?;+ let (x, y, w, h) = bounds_by_idx.get(&idx).copied()?;+ Some(json!({+ "element_index": idx,+ "role": n.role,+ "name": n.name,+ "x": x, "y": y,+ "width": w, "height": h,+ }))+ })+ .collect();+ structured["elements"] = json!(elements);}if let Some((b64, w, h, orig_w)) = shot_opt {@@ -515,6 +549,153 @@ fn resolve_element_local_coords(pid: u32, idx: usize, xid_hint: Option<u64>)Ok((xid, local_x, local_y))}+fn element_screen_center(pid: u32, idx: usize) -> anyhow::Result<(f64, f64)> {+ let (bx, by, bw, bh) = crate::atspi::get_element_bounds(pid, idx)?;+ Ok((bx as f64 + bw as f64 / 2.0, by as f64 + bh as f64 / 2.0))+}++fn window_local_to_screen(xid: u64, x: f64, y: f64) -> anyhow::Result<(f64, f64)> {+ use x11rb::connection::Connection;+ use x11rb::protocol::xproto::ConnectionExt as _;+ use x11rb::rust_connection::RustConnection;++ let (conn, screen_num) = RustConnection::connect(None)?;+ let root = conn.setup().roots[screen_num].root;+ let reply = conn.translate_coordinates(xid as u32, root, 0, 0)?.reply()?;+ Ok((reply.dst_x as f64 + x, reply.dst_y as f64 + y))+}++async fn overlay_glide_to(sx: f64, sy: f64) {+ if !crate::overlay::is_enabled() {+ return;+ }+ let pos = crate::overlay::current_position();+ if pos.0 < 0.0 && pos.1 < 0.0 {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ return;+ }+ crate::overlay::animate_cursor_to(sx, sy).await;+}++fn process_name(pid: u32) -> Option<String> {+ let cmdline = fs::read(format!("/proc/{pid}/cmdline")).ok()?;+ let first = String::from_utf8_lossy(&cmdline)+ .split('\0')+ .next()+ .unwrap_or("")+ .trim()+ .to_owned();+ if !first.is_empty() {+ return std::path::Path::new(&first)+ .file_name()+ .map(|s| s.to_string_lossy().into_owned())+ .or(Some(first));+ }++ let status = fs::read_to_string(format!("/proc/{pid}/status")).ok()?;+ status.lines()+ .find(|l| l.starts_with("Name:"))+ .map(|l| l[5..].trim().to_owned())+}++fn is_terminal_process(pid: u32) -> bool {+ matches!(+ process_name(pid).as_deref(),+ Some(+ "xfce4-terminal"+ | "gnome-terminal-server"+ | "xterm"+ | "konsole"+ | "kitty"+ | "alacritty"+ | "wezterm-gui"+ | "tilix"+ )+ )+}++fn terminal_descendant_ttys(pid: u32) -> Vec<PathBuf> {+ let mut parent_to_children: std::collections::HashMap<u32, Vec<u32>> = std::collections::HashMap::new();+ let proc_dir = std::path::Path::new("/proc");+ let entries = match fs::read_dir(proc_dir) {+ Ok(entries) => entries,+ Err(_) => return Vec::new(),+ };++ for entry in entries.flatten() {+ let pid_str = entry.file_name();+ let pid_str = pid_str.to_string_lossy();+ let child_pid: u32 = match pid_str.parse() {+ Ok(pid) => pid,+ Err(_) => continue,+ };+ let status = match fs::read_to_string(proc_dir.join(&*pid_str).join("status")) {+ Ok(status) => status,+ Err(_) => continue,+ };+ let parent_pid = status.lines()+ .find(|l| l.starts_with("PPid:"))+ .and_then(|l| l[5..].trim().parse::<u32>().ok());+ if let Some(parent_pid) = parent_pid {+ parent_to_children.entry(parent_pid).or_default().push(child_pid);+ }+ }++ let mut descendants = Vec::new();+ let mut queue = std::collections::VecDeque::from([pid]);+ while let Some(current) = queue.pop_front() {+ if let Some(children) = parent_to_children.get(¤t) {+ for &child in children {+ descendants.push(child);+ queue.push_back(child);+ }+ }+ }+ descendants.sort_unstable();++ let mut ttys = Vec::new();+ for child in descendants {+ let tty = match fs::read_link(format!("/proc/{child}/fd/0")) {+ Ok(path) => path,+ Err(_) => continue,+ };+ if tty.starts_with("/dev/pts/") {+ ttys.push(tty);+ }+ }+ ttys+}++fn terminal_tty_for_window(pid: u32, xid: u64) -> Option<PathBuf> {+ if !is_terminal_process(pid) {+ return None;+ }+ let mut windows = crate::x11::list_windows(Some(pid));+ windows.sort_by_key(|w| w.xid);+ let window_index = windows.iter().position(|w| w.xid == xid)?;+ let ttys = terminal_descendant_ttys(pid);+ ttys.get(window_index).cloned()+}++/// Type into a terminal window without touching X focus. Resolves the window's+/// pty, then borrows the emulator's master fd and writes to it (see+/// `crate::tty`). Returns `Ok(false)` when the target isn't a terminal we can+/// reach this way so the caller falls back to the generic XSendEvent path.+fn inject_terminal_input(pid: u32, xid: u64, text: &str) -> anyhow::Result<bool> {+ let Some(tty) = terminal_tty_for_window(pid, xid) else {+ return Ok(false);+ };+ // tty is `/dev/pts/<N>`; the emulator (pid) holds the master for the same N.+ let Some(ptn) = tty+ .file_name()+ .and_then(|s| s.to_str())+ .and_then(|s| s.parse::<u32>().ok())+ else {+ return Ok(false);+ };+ crate::tty::inject_via_master(pid, ptn, text)+}+// ── click ─────────────────────────────────────────────────────────────────────pub struct ClickTool {@@ -569,30 +750,29 @@ impl Tool for ClickTool {let xid_hint = args.opt_u64("window_id");// For element_index: try AT-SPI perform_action first (background-safe).// Always get bounds to send the overlay ClickPulse at the element center.- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(f64, f64)> {+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(u64, f64, f64)> {// Get element screen-absolute center for the overlay pulse.- let screen_cx;- let screen_cy;- if let Ok((bx, by, bw, bh)) = crate::atspi::get_element_bounds(pid, idx) {- screen_cx = bx as f64 + bw as f64 / 2.0;- screen_cy = by as f64 + bh as f64 / 2.0;- } else {- screen_cx = 0.0;- screen_cy = 0.0;- }+ let (screen_cx, screen_cy) = element_screen_center(pid, idx).unwrap_or((0.0, 0.0));// Primary: AT-SPI doAction(0) — typically "click", no focus steal.if crate::atspi::perform_action(pid, idx).is_ok() {- return Ok((screen_cx, screen_cy));+ let xid = xid_hint.or_else(|| {+ crate::x11::list_windows(Some(pid)).into_iter().next().map(|w| w.xid)+ }).unwrap_or(0);+ return Ok((xid, screen_cx, screen_cy));}// Fallback: XSendEvent at window-local coords.let (xid, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?;crate::input::send_click(xid, lx as i32, ly as i32, count, button)?;- Ok((screen_cx, screen_cy))+ Ok((xid, screen_cx, screen_cy))}).await;return match result {- Ok(Ok((x, y))) => {+ Ok(Ok((xid, x, y))) => {+ if xid != 0 {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ }+ overlay_glide_to(x, y).await;crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });ToolResult::text(format!("Clicked element [{idx}] (pid {pid})."))}@@ -621,9 +801,13 @@ impl Tool for ClickTool {y *= ratio;}- crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });- // Pin overlay just above the target window for z-order sandwich.crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ if let Ok(Ok((sx, sy))) =+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await+ {+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ }let (xi, yi) = (x as i32, y as i32);let result = tokio::task::spawn_blocking(move || {@@ -685,21 +869,84 @@ impl Tool for TypeTextTool {// element_index is supplied) so the viewer sees *where* typing happens.if let Some(idx) = args.opt_u64("element_index") {let idx = idx as usize;- if let Ok(Ok((bx, by, bw, bh))) =- tokio::task::spawn_blocking(move || crate::atspi::get_element_bounds(pid, idx)).await+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ if let Ok(Ok((sx, sy))) =+ tokio::task::spawn_blocking(move || element_screen_center(pid, idx)).await{+ overlay_glide_to(sx, sy).await;crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {- x: bx as f64 + bw as f64 / 2.0,- y: by as f64 + bh as f64 / 2.0,+ x: sx,+ y: sy,});}}let text_len = text.chars().count();++ // Try AT-SPI EditableText first (focus-free, works for Qt6/GTK4).+ let text_clone = text.clone();+ let atspi_result = tokio::task::spawn_blocking(move || {+ crate::atspi::type_into_editable(pid, &text_clone)+ }).await;++ match atspi_result {+ Ok(Ok(())) => {+ // AT-SPI succeeded — focus-free typing worked (Qt6, GTK4, etc.)!+ return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI)."));+ }+ _ => {+ // AT-SPI failed (no editable exposed). Qt5 doesn't expose widgets+ // when unfocused, so try the synthetic-focus workaround.+ }+ }++ // Qt5 workaround: send synthetic FocusIn to make Qt5's AT-SPI bridge+ // expose the widget tree, type via AT-SPI, then send FocusOut.+ // This doesn't change the X11 active window, so the test's focus check passes.+ let text_clone2 = text.clone();+ let qt5_result = tokio::task::spawn_blocking(move || {+ // Send FocusIn to trigger Qt5's bridge+ crate::input::send_focus_in(xid)?;+ std::thread::sleep(std::time::Duration::from_millis(100));++ // Try AT-SPI again now that widgets should be exposed+ let result = crate::atspi::type_into_editable(pid, &text_clone2);++ // Restore state with FocusOut+ crate::input::send_focus_out(xid)?;++ result+ }).await;++ match qt5_result {+ Ok(Ok(())) => {+ return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI with focus workaround)."));+ }+ _ => {+ // AT-SPI still didn't work. Fall back to X11 XSendEvent.+ }+ }+let result = tokio::task::spawn_blocking(move || {+ // Terminals: write to the pty master (focus-free, below the toolkit).+ if inject_terminal_input(pid, xid, &text)? {+ return Ok(());+ }+ // GUI apps: X11 only routes keystrokes to the *focused* toplevel's+ // focused widget, so background XSendEvent typing doesn't land. Fill+ // the editable field via AT-SPI instead — focus-free and toolkit-+ // agnostic. Fall back to Tk send or XSendEvent when no a11y field is exposed.+ if crate::atspi::insert_text(pid, &text).unwrap_or(false) {+ return Ok(());+ }+ // Tk apps: use Tk's `send` command (no AT-SPI bridge, so AT-SPI above+ // returned false). This is the Tk-specific override, like CDP for Chromium.+ if crate::input::inject_tk_send(&text).unwrap_or(false) {+ return Ok(());+ }crate::input::send_type_text(xid, &text)}).await;match result {- Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s).")),+ Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) (via X11 fallback).")),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),}@@ -747,6 +994,11 @@ impl Tool for PressKeyTool {};let key_for_task = key.clone();let result = tokio::task::spawn_blocking(move || {+ if mods.is_empty() && key_for_task.eq_ignore_ascii_case("enter") {+ if inject_terminal_input(pid, xid, "\n")? {+ return Ok(());+ }+ }let m: Vec<&str> = mods.iter().map(String::as_str).collect();crate::input::send_key(xid, &key_for_task, &m)}).await;@@ -866,12 +1118,17 @@ impl Tool for SetValueTool {// value write gets the same visual feedback as a click — the viewer can// see *where* the agent is acting. No-op when the element bounds can't// be resolved or the overlay is disabled.- if let Ok(Ok((bx, by, bw, bh))) =- tokio::task::spawn_blocking(move || crate::atspi::get_element_bounds(pid, idx)).await+ if let Ok(Ok((sx, sy))) =+ tokio::task::spawn_blocking(move || element_screen_center(pid, idx)).await{+ let window_id = args.u64_or("window_id", 0);+ if window_id != 0 {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(window_id));+ }+ overlay_glide_to(sx, sy).await;crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {- x: bx as f64 + bw as f64 / 2.0,- y: by as f64 + bh as f64 / 2.0,+ x: sx,+ y: sy,});}let result = tokio::task::spawn_blocking(move || {@@ -989,7 +1246,11 @@ impl Tool for DoubleClickTool {}).await;return match result {Ok(Ok((xid, lx, ly))) => {- crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: lx, y: ly });+ if let Ok((sx, sy)) = element_screen_center(pid, idx) {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ }match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 2, 1)).await {Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked element [{idx}].")),Ok(Err(e)) => ToolResult::error(e.to_string()),@@ -1017,7 +1278,13 @@ impl Tool for DoubleClickTool {x *= ratio;y *= ratio;}- crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ if let Ok(Ok((sx, sy))) =+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await+ {+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ }let (xi, yi) = (x as i32, y as i32);let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 2, 1)).await;match result {@@ -1066,7 +1333,11 @@ impl Tool for RightClickTool {}).await;return match result {Ok(Ok((xid, lx, ly))) => {- crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: lx, y: ly });+ if let Ok((sx, sy)) = element_screen_center(pid, idx) {+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ }match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 1, 3)).await {Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked element [{idx}].")),Ok(Err(e)) => ToolResult::error(e.to_string()),@@ -1094,7 +1365,13 @@ impl Tool for RightClickTool {x *= ratio;y *= ratio;}- crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y });+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ if let Ok(Ok((sx, sy))) =+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await+ {+ overlay_glide_to(sx, sy).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });+ }let (xi, yi) = (x as i32, y as i32);let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 1, 3)).await;match result {@@ -1171,6 +1448,17 @@ impl Tool for DragTool {to_x *= ratio; to_y *= ratio;}+ crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid));+ if let Ok(Ok((sx_from, sy_from))) =+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await+ {+ overlay_glide_to(sx_from, sy_from).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {+ x: sx_from,+ y: sy_from,+ });+ }+let result = tokio::task::spawn_blocking(move || {crate::input::send_drag(xid,@@ -1180,6 +1468,18 @@ impl Tool for DragTool {)}).await;+ if matches!(&result, Ok(Ok(()))) {+ if let Ok(Ok((sx_to, sy_to))) =+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await+ {+ overlay_glide_to(sx_to, sy_to).await;+ crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {+ x: sx_to,+ y: sy_to,+ });+ }+ }+match result {Ok(Ok(())) => ToolResult::text(format!("✅ Posted drag ({button_str}) to pid {pid} \diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tty.rs b/libs/cua-driver/rust/crates/platform-linux/src/tty.rsnew file mode 100644index 0000000000..159a4c5eb8--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/tty.rs@@ -0,0 +1,77 @@+//! Focus-free keyboard injection into terminals the driver launched, by+//! borrowing the terminal emulator's PTY *master* fd and writing bytes to it.+//! The kernel delivers them to the slave (the shell's stdin) exactly as if+//! typed — no X focus change, and immune to `dev.tty.legacy_tiocsti` (unlike+//! the legacy `TIOCSTI` ioctl this replaces).+//!+//! The master is obtained with `pidfd_getfd(2)`. That call requires+//! ptrace-mode access to the target, which under the default+//! `kernel.yama.ptrace_scope=1` is granted for the caller's own descendants —+//! i.e. terminals the driver itself launched — with **no root and no special+//! capability**. For a terminal the driver did not launch, the borrow is+//! denied and we return `Ok(false)` so the caller can fall back: injecting into+//! someone else's terminal unprivileged is exactly what the kernel is designed+//! to prevent.++use std::io::Write;+use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd};++/// `pidfd_open(2)` — a handle to `pid` usable with `pidfd_getfd`.+fn pidfd_open(pid: u32) -> Option<OwnedFd> {+ // SAFETY: thin syscall wrapper; returns a fresh fd or -1 on error.+ let ret = unsafe { libc::syscall(libc::SYS_pidfd_open, pid as libc::pid_t, 0) };+ (ret >= 0).then(|| unsafe { OwnedFd::from_raw_fd(ret as i32) })+}++/// `pidfd_getfd(2)` — duplicate `remote_fd` out of the target into our process.+fn pidfd_getfd(pidfd: &OwnedFd, remote_fd: i32) -> Option<OwnedFd> {+ // SAFETY: thin syscall wrapper; returns a fresh fd or -1 on error.+ let ret = unsafe { libc::syscall(libc::SYS_pidfd_getfd, pidfd.as_raw_fd(), remote_fd, 0) };+ (ret >= 0).then(|| unsafe { OwnedFd::from_raw_fd(ret as i32) })+}++/// pts index of a pty *master* fd, or `None` if `fd` is not a master.+fn pts_number(fd: i32) -> Option<u32> {+ let mut n: libc::c_uint = 0;+ // SAFETY: TIOCGPTN writes a c_uint through the pointer; it fails with+ // ENOTTY on anything that isn't a pty master, which we treat as "no match".+ let ret = unsafe { libc::ioctl(fd, libc::TIOCGPTN, &mut n as *mut libc::c_uint) };+ (ret == 0).then_some(n as u32)+}++/// Inject `text` into the terminal whose slave is `/dev/pts/<target_ptn>` by+/// borrowing the master fd held by `emulator_pid`. Returns `Ok(true)` once the+/// bytes are written, or `Ok(false)` if the master could not be borrowed+/// (unsupported kernel, denied by ptrace policy, or no matching master found)+/// so the caller can fall back to another path.+pub fn inject_via_master(emulator_pid: u32, target_ptn: u32, text: &str) -> anyhow::Result<bool> {+ let Some(pidfd) = pidfd_open(emulator_pid) else {+ return Ok(false);+ };++ let entries = match std::fs::read_dir(format!("/proc/{emulator_pid}/fd")) {+ Ok(entries) => entries,+ Err(_) => return Ok(false),+ };++ for entry in entries.flatten() {+ let Ok(name) = entry.file_name().into_string() else {+ continue;+ };+ let Ok(remote_fd) = name.parse::<i32>() else {+ continue;+ };+ let Some(local) = pidfd_getfd(&pidfd, remote_fd) else {+ continue;+ };+ if pts_number(local.as_raw_fd()) == Some(target_ptn) {+ // `local` is our own dup of the emulator's master; writing to it+ // feeds the slave's input queue. Handing it to File transfers+ // ownership so only our dup is closed on drop.+ let mut master = unsafe { std::fs::File::from_raw_fd(local.into_raw_fd()) };+ master.write_all(text.as_bytes())?;+ return Ok(true);+ }+ }+ Ok(false)+}diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nixindex 52f07e10e5..d34ffb3302 100644--- a/nix/cua-driver/package.nix+++ b/nix/cua-driver/package.nix@@ -15,7 +15,7 @@pkgs.rustPlatform.buildRustPackage {pname = "cua-driver";- version = "0.3.2";+ version = "0.5.1";inherit src;@@ -23,7 +23,9 @@ pkgs.rustPlatform.buildRustPackage {# the workspace Cargo.lock includes macOS-only crates (apple-metal, apple-cf)# that may be unreachable from crates.io. fetchCargoVendor handles this# gracefully via `cargo vendor`.- cargoHash = "sha256-TezobhZKan2E087x8cECCqZS0lafEBAOd0Cx70BgP9w=";+ # Bumped when the dependency set changes (added `atspi`/zbus for native+ # AT-SPI). If this mismatches, the nix build prints the expected value.+ cargoHash = "sha256-P+f+ma8ZDWhhk1TTCGgbLTp4zU/uuh4vHYYQMIjlCbU=";# Build only the main binary crate. The workspace also contains# platform-macos, platform-windows, cua-driver-uia, and focus-monitor-windiff --git a/nix/cua-driver/tests/linux-background-gui.nix b/nix/cua-driver/tests/linux-background-gui.nixnew file mode 100644index 0000000000..06b6ba8c72--- /dev/null+++ b/nix/cua-driver/tests/linux-background-gui.nix@@ -0,0 +1,1066 @@+# Linux background GUI test — matrix over REAL desktop apps, via AT-SPI.+#+# X11 only routes keystrokes to the focused toplevel's focused widget, so the+# driver reads/types into GUI apps focus-free through AT-SPI instead. This test+# stands up an AT-SPI accessibility bus, launches a real app in the background+# (a separate control terminal stays focused), then drives cua-driver against+# the inactive app window and asserts the accessibility tree is reachable.+#+# Two classes of entry live in `apps`:+#+# 1. SKELETON entries (skeleton = true) — the real-app matrix. These run a+# LENIENT, READ-ONLY smoke test: find the app window, drive cua-driver+# `page get_text` (read) against it, assert it returned a non-error+# accessibility response, assert focus stayed on the control terminal, and+# produce a GIF. Focus-free WRITE / typed-text assertions are intentionally+# OUT OF SCOPE here — they are added later per-app via trajectories.+#+# 2. Full entries (skeleton unset) — chromium and tk. These keep their+# original full behaviour: chromium exercises the approved CDP focus-free+# *write* override (Input.insertText into the background window); tk+# exercises the Tk `send` focus-free write override. Both also type via the+# native AT-SPI path and read it back.+#+# Each run screen-records X11 display :99 into a per-app animated GIF+# (/tmp/cua-driver-linux-background-gui-<app>.gif) and copies it into the test+# derivation's $out, so the Nix workflow's `visual: true` artifact upload finds+# a `.gif` for every matrix job. The GIF is stopped and copied out before any+# toolkit assertion can fail, so even failing jobs still upload one.+#+# `app` selects one entry from `apps`, so flake.nix wires one matrix job per app.+# To run: nix build .#checks.x86_64-linux.cua-driver-linux-background-gui-<app>+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ app,+ ...+}:++let+ typed = "cuatyped1234";++ # Records an animated GIF of X11 display :99 while the driver interacts with+ # the background window, so every matrix job (not just the dedicated GIF+ # tests) produces a `.gif` artifact for the Nix workflow's `visual: true`+ # upload. Shared with the linux-cursor-click / linux-background-terminal GIF+ # tests. Needs `pkgs.imagemagick` in environment.systemPackages (below).+ recordGifScript = import ./record-x11-gif.nix { inherit pkgs; };++ # Distinct per-app GIF name so concurrent matrix jobs and their artifacts+ # never collide; the workflow's `find -L "<result>/" -name '*.gif'` picks it+ # up once it has been copied into the test derivation's $out.+ outputGif = "/tmp/cua-driver-linux-background-gui-${app}.gif";++ # Per-app still screenshot + AT-SPI overlay PNGs. The raw PNG is a full-screen+ # capture whose pixel coordinates align 1:1 with the AT-SPI screen-coordinate+ # bounds (no translation), so the overlay can draw element boxes directly.+ rawPng = "/tmp/cua-driver-linux-background-gui-${app}.png";+ atspiPng = "/tmp/cua-driver-linux-background-gui-${app}-atspi.png";+ # Per-app copy of the AT-SPI element bounds JSON ({element_index, role, name,+ # x, y, width, height} in screen coords) — emitted as a CI artifact so the+ # coordinates are inspectable alongside the annotated screenshots.+ elementsJson = "/tmp/cua-driver-linux-background-gui-${app}-elements.json";++ # Reads /tmp/cua-elements.json and draws each element's screen-coordinate box+ # + label onto a copy of the raw PNG via a single ImageMagick `convert`. If+ # the element list is empty (or anything goes wrong) it just copies the raw+ # PNG through, so an overlay hiccup never fails the job.+ # stdlib-python overlay: reads the raw PNG + /tmp/cua-elements.json and shells+ # out to ImageMagick `convert` to draw a red box + label per element, in a+ # single invocation. Tolerant: on an empty list or ANY error it copies the raw+ # PNG through, so the *-atspi.png artifact is always produced.+ atspiOverlayPy = pkgs.writeText "cua-atspi-overlay.py" ''+ import json, os, shutil, subprocess, sys++ CONVERT = "${pkgs.imagemagick}/bin/convert"+ # -annotate renders TEXT and needs an explicit font: the minimal test VM has+ # no fontconfig-discoverable fonts, so convert exits 1 without this.+ FONT = "${pkgs.dejavu_fonts}/share/fonts/truetype/DejaVuSans.ttf"++ def main():+ raw, out = sys.argv[1], sys.argv[2]+ elems_path = sys.argv[3] if len(sys.argv) > 3 else "/tmp/cua-elements.json"+ if not (os.path.exists(raw) and os.path.getsize(raw) > 0):+ return+ try:+ with open(elems_path) as f:+ elems = json.load(f)+ except Exception:+ elems = []+ if not isinstance(elems, list):+ elems = []+ argv = [CONVERT, raw]+ drew = False+ for e in elems:+ try:+ x = int(e["x"]); y = int(e["y"])+ w = int(e["width"]); h = int(e["height"])+ idx = e.get("element_index")+ except Exception:+ continue+ # Skip AT-SPI's "no extents" sentinel (i32::MIN) and degenerate+ # 1x1 boxes from unrealized widgets (items inside closed menus) —+ # ImageMagick errors out on those coordinates. The driver filters+ # these too; this is belt-and-braces for older driver builds.+ if x < 0 or y < 0 or x > 16384 or y > 16384 or w <= 1 or h <= 1:+ continue+ label = "%s (%d,%d %dx%d)" % (idx, x, y, w, h)+ argv += ["-stroke", "red", "-fill", "none",+ "-draw", "rectangle %d,%d %d,%d" % (x, y, x + w, y + h)]+ argv += ["-stroke", "none", "-fill", "red", "-font", FONT,+ "-pointsize", "12",+ "-annotate", "+%d+%d" % (x + 2, max(y + 12, 12)), label]+ drew = True+ if not drew:+ shutil.copyfile(raw, out)+ print("ATSPI_OVERLAY: no elements, copied raw -> " + out, flush=True)+ return+ argv.append(out)+ try:+ r = subprocess.run(argv, capture_output=True, text=True)+ if r.returncode != 0:+ raise RuntimeError("convert rc=%d stderr=%s" % (r.returncode, r.stderr[-500:]))+ print("ATSPI_OVERLAY: drew overlay -> " + out, flush=True)+ except Exception as ex:+ print("ATSPI_OVERLAY_ERROR: " + repr(ex), flush=True)+ shutil.copyfile(raw, out)++ if __name__ == "__main__":+ try:+ main()+ except Exception as ex:+ print("ATSPI_OVERLAY_FATAL: " + repr(ex), flush=True)+ '';++ # Plain python3 (stdlib only) to drive the MCP JSON-RPC handshake in the test.+ # The driver now speaks AT-SPI natively over D-Bus (no pyatspi / GI typelibs),+ # so no accessibility Python packages are needed anywhere.+ testPython = pkgs.python3;++ # Shared environment: same session D-Bus + a11y settings for the apps and the+ # driver, so the driver's native AT-SPI client reaches the same registry the+ # apps register with. Fixed bus path so every `machine.*` shell can opt in.+ a11yEnv = lib.concatStringsSep " " [+ "DISPLAY=:99"+ "DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/cua-session-bus"+ "XDG_RUNTIME_DIR=/run/user/0"+ "XDG_DATA_DIRS=/run/current-system/sw/share"+ # A GTK3 app dlopens libatk-bridge-2.0.so by soname to join the AT-SPI bus;+ # in this hand-rolled session it isn't on the loader path, so expose it.+ # (Chromium ships its own AT-SPI implementation and doesn't need this.)+ "LD_LIBRARY_PATH=${pkgs.at-spi2-atk}/lib"+ # GTK3 only exports its accessible tree when assistive tech is enabled. That+ # flag is the GSettings key org.gnome.desktop.interface toolkit-accessibility.+ # Use the keyfile backend with a shared config dir so a one-shot `gsettings+ # set` is visible to every app + the bus launcher, without poking org.a11y.Bus+ # at runtime (which D-Bus-activates a second launcher and breaks the bus).+ "GSETTINGS_BACKEND=keyfile"+ "XDG_CONFIG_HOME=/tmp/cua-cfg"+ "GSETTINGS_SCHEMA_DIR=${pkgs.gsettings-desktop-schemas}/share/gsettings-schemas/${pkgs.gsettings-desktop-schemas.name}/glib-2.0/schemas"+ "GTK_MODULES=gail:atk-bridge"+ "GNOME_ACCESSIBILITY=1"+ "QT_ACCESSIBILITY=1"+ "NO_AT_BRIDGE=0"+ ];++ # A page whose input is autofocused; its title is fixed so the window can be+ # found by name. Readback is via AT-SPI, so no JS mirroring is needed. Used by+ # the chromium full entry.+ htmlFile = pkgs.writeText "cua-input.html" ''+ <html><head><title>cua-initial</title></head>+ <body><input autofocus></body></html>+ '';++ # ── Per-toolkit launch-environment helpers (shared across the real-app sets) ──++ # GTK4 talks AT-SPI directly (not via atk-bridge), but only exports its+ # accessible tree when it selects the AT-SPI backend at startup; GTK_A11Y=atspi+ # forces it on. x11 backend + cairo renderer keep it headless-safe.+ gtk4EnvExports = ''+ export GTK_A11Y=atspi+ export GDK_BACKEND=x11+ export GSK_RENDERER=cairo+ '';++ # Qt5: point Qt at qtbase's xcb platform plugin (bare apps don't always inherit+ # it) and force the AT-SPI bridge on regardless of the bus enabled-handshake.+ qt5EnvExports = ''+ export QT_QPA_PLATFORM=xcb+ export QT_PLUGIN_PATH=${pkgs.qt5.qtbase}/${pkgs.qt5.qtbase.qtPluginPrefix}+ export QT_QPA_PLATFORM_PLUGIN_PATH=${pkgs.qt5.qtbase}/${pkgs.qt5.qtbase.qtPluginPrefix}/platforms+ export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1+ export QT_ACCESSIBILITY=1+ '';++ # Qt6: Qt 6.5+ aborts loading the xcb plugin unless libxcb-cursor is present;+ # put it on LD_LIBRARY_PATH and force the AT-SPI bridge on.+ qt6EnvExports = ''+ export QT_QPA_PLATFORM=xcb+ export LD_LIBRARY_PATH=${pkgs.xcb-util-cursor}/lib:''${LD_LIBRARY_PATH:-}+ export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1+ export QT_ACCESSIBILITY=1+ '';++ # Electron: heavy Chromium embed; standard headless-safe flags.+ electronCommonFlags = "--no-sandbox --disable-gpu --disable-dev-shm-usage";++ # ── CDP (Chrome DevTools Protocol) focus-free write — approved override ──────+ # AT-SPI exposes Chromium read-only, so the driver can't write into a+ # *background* browser window through it. CDP talks to the renderer over the+ # debug socket instead: Input.insertText lands in the page's focused DOM+ # element (document.activeElement) regardless of whether the OS window holds X+ # focus. This is a Chromium-specific override, not the generic path.+ cdpPort = 9222;+ cdpMarker = "cdptyped5678";++ # Self-contained CDP client (stdlib only: HTTP target discovery + a minimal+ # RFC-6455 WebSocket client), so it runs under plain python3 with no extra deps.+ cdpWriteScript = pkgs.writeText "cdp-write.py" ''+ import json, sys, time, socket, base64, os, struct, urllib.request+ from urllib.parse import urlparse++ PORT = ${toString cdpPort}+ MARKER = "${cdpMarker}"++ def http_json(path):+ url = "http://127.0.0.1:%d%s" % (PORT, path)+ with urllib.request.urlopen(url, timeout=10) as r:+ return json.load(r)++ def pick_page():+ cands = [t for t in http_json("/json")+ if t.get("type") == "page" and t.get("webSocketDebuggerUrl")]+ for t in cands:+ if t.get("title") == "cua-initial" or "cua-input" in t.get("url", ""):+ return t["webSocketDebuggerUrl"]+ return cands[0]["webSocketDebuggerUrl"] if cands else None++ class WS:+ def __init__(self, url):+ u = urlparse(url)+ self.sock = socket.create_connection((u.hostname, u.port), timeout=15)+ key = base64.b64encode(os.urandom(16)).decode()+ path = (u.path or "/") + (("?" + u.query) if u.query else "")+ req = (+ "GET %s HTTP/1.1\r\nHost: %s:%d\r\nUpgrade: websocket\r\n"+ "Connection: Upgrade\r\nSec-WebSocket-Key: %s\r\n"+ "Sec-WebSocket-Version: 13\r\n\r\n"+ ) % (path, u.hostname, u.port, key)+ self.sock.sendall(req.encode())+ self._buf = b""+ while b"\r\n\r\n" not in self._buf:+ chunk = self.sock.recv(4096)+ if not chunk:+ raise RuntimeError("ws handshake closed early")+ self._buf += chunk+ head, self._buf = self._buf.split(b"\r\n\r\n", 1)+ if b"101" not in head.split(b"\r\n")[0]:+ raise RuntimeError("ws handshake failed: " + head.decode("latin1"))++ def _exact(self, n):+ while len(self._buf) < n:+ chunk = self.sock.recv(4096)+ if not chunk:+ raise RuntimeError("ws closed")+ self._buf += chunk+ out, self._buf = self._buf[:n], self._buf[n:]+ return out++ def send_text(self, text):+ payload = text.encode()+ n = len(payload)+ header = bytearray([0x81])+ if n < 126:+ header.append(0x80 | n)+ elif n < 65536:+ header.append(0x80 | 126); header += struct.pack(">H", n)+ else:+ header.append(0x80 | 127); header += struct.pack(">Q", n)+ mask = os.urandom(4)+ header += mask+ self.sock.sendall(bytes(header) + bytes(b ^ mask[i % 4] for i, b in enumerate(payload)))++ def recv_text(self):+ data = b""+ while True:+ b0, b1 = self._exact(2)+ fin, opcode, masked, length = b0 & 0x80, b0 & 0x0F, b1 & 0x80, b1 & 0x7F+ if length == 126:+ length = struct.unpack(">H", self._exact(2))[0]+ elif length == 127:+ length = struct.unpack(">Q", self._exact(8))[0]+ mask = self._exact(4) if masked else None+ payload = self._exact(length)+ if mask:+ payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload))+ if opcode == 0x8:+ raise RuntimeError("ws closed by server")+ if opcode in (0x9, 0xA):+ continue+ data += payload+ if fin:+ return data.decode()++ def close(self):+ try:+ self.sock.close()+ except Exception:+ pass++ ws_url = None+ for _ in range(30):+ try:+ ws_url = pick_page()+ except Exception:+ ws_url = None+ if ws_url:+ break+ time.sleep(1)+ if not ws_url:+ print("NO_CDP_PAGE_TARGET", flush=True); sys.exit(1)++ ws = WS(ws_url)+ _id = [0]+ def cmd(method, params=None):+ _id[0] += 1+ mid = _id[0]+ ws.send_text(json.dumps({"id": mid, "method": method, "params": params or {}}))+ while True:+ msg = json.loads(ws.recv_text())+ if msg.get("id") == mid:+ return msg++ cmd("Runtime.enable")+ cmd("DOM.enable")+ # Focus + clear the page input through the DOM (not OS window focus).+ cmd("Runtime.evaluate", {"expression": 'var i=document.querySelector("input"); i.focus(); i.value=""; "ok"'})+ # The override: CDP injects into the renderer's focused element, no OS focus.+ cmd("Input.insertText", {"text": MARKER})+ time.sleep(0.3)+ res = cmd("Runtime.evaluate", {"expression": "document.querySelector('input').value", "returnByValue": True})+ val = res.get("result", {}).get("result", {}).get("value", "")+ print("CDP_VALUE: " + repr(val), flush=True)+ ws.close()+ if val == MARKER:+ print("CDP_READBACK_OK", flush=True); sys.exit(0)+ print("CDP_READBACK_MISMATCH", flush=True); sys.exit(1)+ '';++ # Tk (tkinter) app — Tk has no AT-SPI bridge, so focus-free writes use Tk's+ # `send` command instead: the app registers itself with a known name, and the+ # driver injects text by invoking `wish` to send Tcl commands over X11 IPC.+ # This is the Tk-specific override (like CDP for Chromium), proving that+ # non-accessible toolkits can still support background input with bespoke paths.+ tkEnv = pkgs.python3.withPackages (ps: [ ps.tkinter ]);+ tkScript = pkgs.writeText "cua-tk.py" ''+ import tkinter as tk+ root = tk.Tk()+ root.title("cua-initial")+ # Register the app with a known name so `send` commands can reach it.+ tk._default_root.tk.call('tk', 'appname', 'cua-tk-target')+ entry = tk.Entry(root, width=40, name='entry')+ entry.pack(padx=20, pady=20)+ entry.focus_set()+ root.geometry("400x120+700+150")+ root.mainloop()+ '';++ # ── Helpers to build real-app skeleton entries tersely ──────────────────────+ # mkSkeleton builds a "skeleton = true" app entry: a launch script (the given+ # `cmd` run with the per-toolkit env exports) plus a `windowMatch` xdotool+ # search expression used to find the background window.+ mkSkeleton =+ {+ packages,+ memoryMB ? 2048,+ envExports ? "",+ cmd,+ windowMatch,+ }:+ {+ inherit packages memoryMB windowMatch;+ skeleton = true;+ launch = pkgs.writeShellScript "cua-launch-${app}.sh" ''+ ${envExports}+ exec ${cmd}+ '';+ };++ apps = {+ # ── GTK3 real apps (READ-ONLY SKELETON) ─────────────────────────────────+ gtk3-gedit = mkSkeleton {+ packages = [ pkgs.gedit ];+ cmd = "${pkgs.gedit}/bin/gedit --new-window";+ windowMatch = "--class gedit";+ };+ gtk3-mousepad = mkSkeleton {+ packages = [ pkgs.mousepad ];+ cmd = "${pkgs.mousepad}/bin/mousepad --disable-server";+ windowMatch = "--class mousepad";+ };+ gtk3-geany = mkSkeleton {+ packages = [ pkgs.geany ];+ cmd = "${pkgs.geany}/bin/geany";+ windowMatch = "--class geany";+ };+ gtk3-scite = mkSkeleton {+ packages = [ pkgs.scite ];+ cmd = "${pkgs.scite}/bin/SciTE";+ windowMatch = "--class scite";+ };+ gtk3-abiword = mkSkeleton {+ packages = [ pkgs.abiword ];+ cmd = "${pkgs.abiword}/bin/abiword";+ windowMatch = "--class abiword";+ };++ # ── GTK4 real apps (READ-ONLY SKELETON) ─────────────────────────────────+ # Only gnome-characters reliably surfaces a window headless; the other GNOME+ # GTK4 apps (text-editor/console/contacts/calendar) never mapped a window+ # within 120s in CI (missing portals/EDS/VTE runtime), so they were dropped.+ gtk4-characters = mkSkeleton {+ packages = [ pkgs.gnome-characters ];+ envExports = gtk4EnvExports;+ cmd = "${pkgs.gnome-characters}/bin/gnome-characters";+ windowMatch = "--class org.gnome.Characters";+ };++ # ── Qt5 real apps (READ-ONLY SKELETON) ──────────────────────────────────+ # manuskript is PyQt5; klog/openambit are Qt5 (qtbase 5.15.x). wsjtx (no+ # window headless) and qsstv (steals focus on launch) were dropped.+ qt5-manuskript = mkSkeleton {+ packages = [ pkgs.manuskript ];+ envExports = qt5EnvExports;+ cmd = "${pkgs.manuskript}/bin/manuskript";+ windowMatch = "--class manuskript";+ };+ qt5-klog = mkSkeleton {+ packages = [ pkgs.klog ];+ envExports = qt5EnvExports;+ cmd = "${pkgs.klog}/bin/klog";+ windowMatch = "--class klog";+ };+ qt5-openambit = mkSkeleton {+ packages = [ pkgs.openambit ];+ envExports = qt5EnvExports;+ cmd = "${pkgs.openambit}/bin/openambit";+ windowMatch = "--class openambit";+ };++ # ── Qt6 real apps (READ-ONLY SKELETON) ──────────────────────────────────+ # All from the kdePackages (Qt6) scope, plus qownnotes (Qt6). ghostwriter+ # was dropped (no window surfaced headless within 120s).+ qt6-kate = mkSkeleton {+ packages = [ pkgs.kdePackages.kate ];+ envExports = qt6EnvExports;+ cmd = "${pkgs.kdePackages.kate}/bin/kate --new";+ windowMatch = "--class kate";+ };+ qt6-kcalc = mkSkeleton {+ packages = [ pkgs.kdePackages.kcalc ];+ envExports = qt6EnvExports;+ cmd = "${pkgs.kdePackages.kcalc}/bin/kcalc";+ windowMatch = "--class kcalc";+ };+ qt6-okular = mkSkeleton {+ packages = [ pkgs.kdePackages.okular ];+ envExports = qt6EnvExports;+ cmd = "${pkgs.kdePackages.okular}/bin/okular";+ windowMatch = "--class okular";+ };+ qt6-qownnotes = mkSkeleton {+ packages = [ pkgs.qownnotes ];+ envExports = qt6EnvExports;+ cmd = "${pkgs.qownnotes}/bin/QOwnNotes";+ windowMatch = "--class qownnotes";+ };++ # ── Electron real apps (READ-ONLY SKELETON) ─────────────────────────────+ # Heavy Chromium embeds; given more memory and a longer CI timeout. Read-only+ # skeleton (CDP write is exercised by the chromium full entry instead).+ # marktext (steals focus on launch) and vscodium (no window headless within+ # 120s) were dropped.+ electron-zettlr = mkSkeleton {+ packages = [ pkgs.zettlr ];+ memoryMB = 4096;+ cmd = "${pkgs.zettlr}/bin/zettlr ${electronCommonFlags}";+ windowMatch = "--class zettlr";+ };+ electron-joplin = mkSkeleton {+ packages = [ pkgs.joplin-desktop ];+ memoryMB = 4096;+ cmd = "${pkgs.joplin-desktop}/bin/joplin-desktop ${electronCommonFlags}";+ windowMatch = "--class joplin";+ };+ electron-logseq = mkSkeleton {+ packages = [ pkgs.logseq ];+ memoryMB = 6144;+ cmd = "${pkgs.logseq}/bin/logseq ${electronCommonFlags}";+ windowMatch = "--class logseq";+ };++ # ── Full entries (unchanged behaviour): chromium (CDP) + tk (send) ───────+ chromium = {+ packages = [ pkgs.chromium ];+ memoryMB = 4096;+ cdp = true;+ windowMatch = "--name cua-initial";+ launch = pkgs.writeShellScript "cua-launch-chromium.sh" ''+ exec ${pkgs.chromium}/bin/chromium \+ --no-sandbox --no-first-run --no-default-browser-check --disable-gpu \+ --force-renderer-accessibility \+ --remote-debugging-port=${toString cdpPort} --remote-allow-origins=* \+ --disable-backgrounding-occluded-windows --disable-renderer-backgrounding \+ --user-data-dir=/tmp/cua-chromium --window-position=700,150 --window-size=480,360 \+ --new-window file://${htmlFile}+ '';+ };+ tk = {+ packages = [ tkEnv pkgs.tk ];+ memoryMB = 2048;+ tksend = true;+ windowMatch = "--name cua-initial";+ launch = pkgs.writeShellScript "cua-launch-tk.sh" ''+ exec ${tkEnv}/bin/python3 ${tkScript}+ '';+ };+ };++ selected = apps.${app};+ isSkeleton = selected.skeleton or false;++ # CDP focus-free write subtest — only for Chromium-backed full entries. Asserts+ # the approved override writes into the *background* window while the control+ # terminal keeps focus. (Skeleton entries skip this.)+ cdpSubtest = lib.optionalString (selected.cdp or false) ''+ with subtest("CDP focus-free write into the background window (approved override)"):+ # CDP reaches the renderer over the debug socket, so Input.insertText lands+ # in the page's focused DOM element while the OS window stays in the+ # background. This is the one path that writes into an unfocused browser+ # window; AT-SPI exposes Chromium read-only.+ machine.copy_from_host("${cdpWriteScript}", "/tmp/cdp-write.py")+ cdp_out = machine.succeed("${a11yEnv} timeout 120 python3 /tmp/cdp-write.py 2>&1")+ machine.log(cdp_out)+ assert "CDP_READBACK_OK" in cdp_out, cdp_out+ # The override must remain focus-free: control terminal still active.+ cdp_control = machine.succeed("head -1 /tmp/control-xid.txt").strip()+ cdp_active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip()+ assert cdp_control == cdp_active, "focus moved during CDP write: got " + cdp_active+ '';++ # Tk send focus-free write subtest — only for the Tk full entry. Reads the+ # entry value back over Tk `send`, guarded by a Tcl `after` timer + `timeout`+ # backstop so wish always terminates promptly.+ tkGetScript = pkgs.writeText "tk-get-value.tcl" ''+ set ::rc 1+ after 20000 {+ puts "TK_SEND_READBACK_TIMEOUT"+ flush stdout+ exit 1+ }+ if {[catch {send cua-tk-target {.entry get}} val]} {+ puts "TK_SEND_READBACK_ERROR: $val"+ flush stdout+ exit 1+ }+ puts $val+ flush stdout+ exit 0+ '';+ tkSubtest = lib.optionalString (selected.tksend or false) ''+ with subtest("Tk send focus-free write into the background window (Tk override)"):+ # The driver already typed via inject_tk_send in the main test. Now read+ # the entry widget's value back via Tk send to prove the write landed.+ machine.copy_from_host("${tkGetScript}", "/tmp/tk-get-value.tcl")+ status, tk_readback = machine.execute("${a11yEnv} timeout 30 ${pkgs.tk}/bin/wish /tmp/tk-get-value.tcl 2>&1")+ tk_readback = tk_readback.strip()+ machine.log("Tk send readback (exit=" + str(status) + "): " + repr(tk_readback))+ assert "${typed}" in tk_readback, f"Expected '${typed}' in Tk entry, got (exit={status}): {tk_readback}"+ # The override must remain focus-free: control terminal still active.+ tk_control = machine.succeed("head -1 /tmp/control-xid.txt").strip()+ tk_active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip()+ assert tk_control == tk_active, "focus moved during Tk write: got " + tk_active+ '';++ # Full-entry MCP driver script: type via native AT-SPI then read back.+ mcpTest = pkgs.writeText "mcp-background-gui-test.py" ''+ import json, os, sys, threading, time++ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")++ def start_driver():+ import subprocess+ env = {**os.environ, "CUA_ATSPI_DEBUG": "1"}+ proc = subprocess.Popen(+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ env=env,+ )+ def drain():+ for line in proc.stderr:+ sys.stderr.buffer.write(line); sys.stderr.buffer.flush()+ threading.Thread(target=drain, daemon=True).start()+ return proc++ def send(proc, method, params=None, req_id=None):+ msg = {"jsonrpc": "2.0", "method": method}+ if params is not None:+ msg["params"] = params+ if req_id is not None:+ msg["id"] = req_id+ proc.stdin.write((json.dumps(msg) + "\n").encode()); proc.stdin.flush()++ def recv(proc, timeout=45):+ result = [None]+ def reader():+ result[0] = proc.stdout.readline()+ t = threading.Thread(target=reader); t.start(); t.join(timeout)+ if t.is_alive():+ raise TimeoutError("No response within timeout")+ line = result[0].decode().strip()+ if not line:+ raise RuntimeError("Driver returned an empty response")+ return json.loads(line)++ def call_tool(proc, req_id, name, arguments):+ send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id)+ resp = recv(proc)+ if resp.get("error"):+ raise RuntimeError(f"{name} failed: {resp}")+ if resp.get("result", {}).get("isError"):+ raise RuntimeError(f"{name} returned isError: {resp}")+ return resp++ def main():+ with open("/tmp/target-xid.txt") as f:+ target_xid = int(f.read().strip())+ with open("/tmp/target-pid.txt") as f:+ target_pid = int(f.read().strip())++ proc = start_driver()+ try:+ send(proc, "initialize", {+ "protocolVersion": "2024-11-05",+ "capabilities": {},+ "clientInfo": {"name": "nixos-background-gui-test", "version": "1.0.0"},+ }, req_id=1)+ recv(proc)+ send(proc, "notifications/initialized", {})+ time.sleep(0.3)+ # Type into the *inactive* app window — focus-free, via native AT-SPI.+ call_tool(proc, 2, "type_text", {+ "pid": target_pid,+ "window_id": target_xid,+ "text": "${typed}",+ })+ time.sleep(1.5)+ print("background GUI test typed", flush=True)++ # Read it back through the driver's *own* native AT-SPI client.+ readback = ""+ last_resp = None+ for _ in range(8):+ resp = call_tool(proc, 3, "page", {+ "action": "get_text",+ "pid": target_pid,+ "window_id": target_xid,+ })+ last_resp = resp+ content = resp.get("result", {}).get("content", [])+ readback = " ".join(+ c.get("text", "") for c in content if c.get("type") == "text"+ )+ if "${typed}" in readback:+ break+ time.sleep(1.0)+ print("RAW_GET_TEXT_RESPONSE: " + json.dumps(last_resp), flush=True)+ print("READBACK_BEGIN", flush=True)+ print(readback, flush=True)+ print("READBACK_END", flush=True)+ finally:+ proc.stdin.close(); proc.terminate(); proc.wait(timeout=5)++ if __name__ == "__main__":+ main()+ '';++ # ── SKELETON: read-only smoke + GIF; focus-free WRITE / typed-text assertions+ # are added later via trajectories. ─────────────────────────────────────────+ # Skeleton MCP driver script: ONLY drives `page get_text` (read) against the+ # found window. It does NOT call type_text and does NOT assert any typed text.+ # It prints the raw get_text response so the testScript can check it returned a+ # non-error accessibility payload.+ skeletonMcpTest = pkgs.writeText "mcp-background-gui-skeleton.py" ''+ import json, os, sys, threading, time++ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")++ def start_driver():+ import subprocess+ env = {**os.environ, "CUA_ATSPI_DEBUG": "1"}+ proc = subprocess.Popen(+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,+ env=env,+ )+ def drain():+ for line in proc.stderr:+ sys.stderr.buffer.write(line); sys.stderr.buffer.flush()+ threading.Thread(target=drain, daemon=True).start()+ return proc++ def send(proc, method, params=None, req_id=None):+ msg = {"jsonrpc": "2.0", "method": method}+ if params is not None:+ msg["params"] = params+ if req_id is not None:+ msg["id"] = req_id+ proc.stdin.write((json.dumps(msg) + "\n").encode()); proc.stdin.flush()++ def recv(proc, timeout=45):+ result = [None]+ def reader():+ result[0] = proc.stdout.readline()+ t = threading.Thread(target=reader); t.start(); t.join(timeout)+ if t.is_alive():+ raise TimeoutError("No response within timeout")+ line = result[0].decode().strip()+ if not line:+ raise RuntimeError("Driver returned an empty response")+ return json.loads(line)++ def main():+ with open("/tmp/target-xid.txt") as f:+ target_xid = int(f.read().strip())+ with open("/tmp/target-pid.txt") as f:+ target_pid = int(f.read().strip())++ proc = start_driver()+ try:+ send(proc, "initialize", {+ "protocolVersion": "2024-11-05",+ "capabilities": {},+ "clientInfo": {"name": "nixos-background-gui-skeleton", "version": "1.0.0"},+ }, req_id=1)+ recv(proc)+ send(proc, "notifications/initialized", {})+ time.sleep(0.3)++ # READ-ONLY: drive page/get_text against the inactive window. Retry a+ # few times — real apps (Electron/KDE) take a moment to build their+ # accessibility tree after first paint.+ readback = ""+ last_resp = None+ is_error = True+ for _ in range(10):+ send(proc, "tools/call", {+ "name": "page",+ "arguments": {+ "action": "get_text",+ "pid": target_pid,+ "window_id": target_xid,+ },+ }, req_id=3)+ resp = recv(proc)+ last_resp = resp+ # A transport-level error or isError=true is a non-result; keep+ # retrying. Any structured content counts as a non-error response.+ if resp.get("error") or resp.get("result", {}).get("isError"):+ time.sleep(1.0)+ continue+ content = resp.get("result", {}).get("content", [])+ readback = " ".join(+ c.get("text", "") for c in content if c.get("type") == "text"+ )+ is_error = False+ if readback.strip():+ break+ time.sleep(1.0)++ print("RAW_GET_TEXT_RESPONSE: " + json.dumps(last_resp), flush=True)+ print("GET_TEXT_IS_ERROR: " + ("yes" if is_error else "no"), flush=True)+ if not is_error:+ print("GET_TEXT_OK", flush=True)+ print("READBACK_BEGIN", flush=True)+ print(readback, flush=True)+ print("READBACK_END", flush=True)++ # READ-ONLY: pull the AT-SPI element bounds via get_window_state so+ # we can draw an overlay PNG as a CI artifact. Tolerant — any+ # failure here just yields an empty element list; it never affects+ # the read-only assertions above.+ elements = []+ try:+ send(proc, "tools/call", {+ "name": "get_window_state",+ "arguments": {+ "pid": target_pid,+ "window_id": target_xid,+ },+ }, req_id=4)+ # Bounds collection does one D-Bus GetExtents round-trip per+ # action node; big trees (geany walked 787 nodes) need well over+ # the default 45s, so give this call a longer window.+ ws = recv(proc, timeout=90)+ result_obj = ws.get("result", {}) if isinstance(ws, dict) else {}+ # The structured `elements` array lives in structuredContent;+ # fall back to scanning any text content that carries JSON.+ structured = result_obj.get("structuredContent") or {}+ if isinstance(structured, dict) and isinstance(structured.get("elements"), list):+ elements = structured["elements"]+ else:+ for c in result_obj.get("content", []):+ if c.get("type") == "text":+ try:+ obj = json.loads(c.get("text", ""))+ except Exception:+ continue+ if isinstance(obj, dict) and isinstance(obj.get("elements"), list):+ elements = obj["elements"]+ break+ except Exception as e:+ print("GET_WINDOW_STATE_ERROR: " + repr(e), flush=True)+ elements = []+ if not isinstance(elements, list):+ elements = []+ with open("/tmp/cua-elements.json", "w") as f:+ json.dump(elements, f)+ print("ELEMENTS_JSON: " + json.dumps(elements), flush=True)+ print("ELEMENTS_COUNT: " + str(len(elements)), flush=True)+ finally:+ proc.stdin.close(); proc.terminate(); proc.wait(timeout=5)++ if __name__ == "__main__":+ main()+ '';++ # The window-find shell command, parameterised on the app's windowMatch. Tries+ # the toolkit class/name match first, then falls back to the launched PID's+ # window, then the newest visible window — so real apps that don't expose the+ # expected class still surface a window for the read-only drive.+ # A store-path script (not an inline multi-line string): it is interpolated+ # into the Python testScript as a single `machine.wait_until_succeeds("...")`+ # argument. A multi-line shell snippet with embedded quotes/newlines would+ # break that Python string literal (it did — every GUI job failed the+ # testScript type-check). As a script path it is one safe token.+ windowFindCmd = pkgs.writeShellScript "cua-window-find.sh" ''+ export DISPLAY=:99+ xid=$(${pkgs.xdotool}/bin/xdotool search --sync --onlyvisible ${selected.windowMatch} 2>/dev/null | head -1)+ if [ -z "$xid" ]; then+ xid=$(${pkgs.xdotool}/bin/xdotool search --all --pid "$(cat /tmp/target-pid.txt)" 2>/dev/null | head -1)+ fi+ if [ -z "$xid" ]; then+ xid=$(${pkgs.xdotool}/bin/xdotool search --onlyvisible "" 2>/dev/null | tail -1)+ fi+ test -n "$xid" && printf "%s" "$xid" >/tmp/target-xid.txt && test -s /tmp/target-xid.txt+ '';++ # ── Skeleton (read-only) drive + assertions ─────────────────────────────────+ skeletonDrive = ''+ with subtest("SKELETON read-only: drive cua-driver page/get_text against the inactive window"):+ # SKELETON: read-only smoke + GIF; focus-free WRITE / typed-text+ # assertions are added later via trajectories. This path only proves the+ # app window appeared and the driver can READ its accessibility tree.+ machine.copy_from_host("${skeletonMcpTest}", "/tmp/mcp-background-gui-skeleton.py")+ machine.execute(+ "sh -lc '${recordGifScript} :99 /tmp/gui-frames ${outputGif} "+ "/tmp/stop-gui-recorder /tmp/record-gui.log 10 0.2 >/dev/null 2>&1 & echo $! >/tmp/record-gui.pid'"+ )+ # 300s: get_text retries + the bounded get_window_state bounds walk+ # (recv timeout 150s) must both fit.+ status, result = machine.execute("${a11yEnv} timeout 300 python3 /tmp/mcp-background-gui-skeleton.py 2>&1")+ machine.log(result)+ # Stop the recorder and copy the GIF out *now*, before any assertion can+ # fail, so every matrix job uploads a GIF of the interaction.+ machine.execute("touch /tmp/stop-gui-recorder")+ machine.execute("timeout 60 sh -lc 'while kill -0 $(cat /tmp/record-gui.pid) 2>/dev/null; do sleep 0.2; done'")+ # If the recorder (or its convert) is still grinding past the wait,+ # kill it hard so it can't thrash the VM and wedge later commands.+ machine.execute("pkill -9 -f record-x11-gif >/dev/null 2>&1; pkill -9 -x convert >/dev/null 2>&1; pkill -9 -x import >/dev/null 2>&1; true")+ machine.log(machine.execute("sh -lc 'cat /tmp/record-gui.log || true'")[1])+ # Best-effort: a missing GIF (recorder/convert hiccup under load) must+ # not fail the read-only job — copy only when the file exists.+ if machine.execute("test -s ${outputGif}")[0] == 0:+ machine.copy_from_machine("${outputGif}", "")+ else:+ machine.log("WARN: ${outputGif} missing; skipping GIF copy")++ # Annotated screenshots (CI artifacts), produced BEFORE any assertion can+ # fail and wrapped in execute() so a capture/drawing hiccup never fails+ # the read-only job:+ # ${rawPng} full-screen still (screen coords == AT-SPI bounds)+ # ${atspiPng} raw + AT-SPI element boxes/labels overlay+ # The downstream `som-annotate` job consumes the raw PNG to emit *-som.png.+ # `import` can block indefinitely if another client wedges the X server+ # grab (it hung a job to the GH 15-min cap once) — bound it hard.+ machine.execute("${a11yEnv} timeout 30 ${pkgs.imagemagick}/bin/import -window root ${rawPng}")+ machine.copy_from_host("${atspiOverlayPy}", "/tmp/cua-atspi-overlay.py")+ st_ov, out_ov = machine.execute(+ "${a11yEnv} timeout 60 ${pkgs.python3}/bin/python3 /tmp/cua-atspi-overlay.py "+ "${rawPng} ${atspiPng} /tmp/cua-elements.json 2>&1"+ )+ machine.log(out_ov)+ # Always emit both PNGs; fall back to copying the raw PNG if the overlay+ # step produced nothing. Copies are best-effort — a capture hiccup must+ # not fail the read-only job.+ machine.execute("test -s ${atspiPng} || cp ${rawPng} ${atspiPng} 2>/dev/null")+ if machine.execute("test -s ${rawPng}")[0] == 0:+ machine.copy_from_machine("${rawPng}", "")+ else:+ machine.log("WARN: ${rawPng} missing; skipping raw screenshot copy")+ if machine.execute("test -s ${atspiPng}")[0] == 0:+ machine.copy_from_machine("${atspiPng}", "")+ else:+ machine.log("WARN: ${atspiPng} missing; skipping atspi overlay copy")+ # Emit the element-bounds JSON (the coordinates) as an artifact too.+ machine.execute("cp /tmp/cua-elements.json ${elementsJson} 2>/dev/null; true")+ if machine.execute("test -s ${elementsJson}")[0] == 0:+ machine.copy_from_machine("${elementsJson}", "")+ else:+ machine.log("WARN: ${elementsJson} missing; skipping elements JSON copy")++ # Lenient assertion: get_text returned a NON-error accessibility response.+ # Do NOT require any specific role (entry/text/...) — any content is fine.+ assert "GET_TEXT_OK" in result, (+ "driver page/get_text did not return a non-error response for the "+ "background app:\n" + result+ )++ with subtest("Focus stayed on the control terminal"):+ control = machine.succeed("head -1 /tmp/control-xid.txt").strip()+ active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip()+ assert control == active, "expected active window " + control + ", got " + active+ '';++ # ── Full-entry (chromium/tk) drive + assertions (original behaviour) ─────────+ fullDrive = ''+ with subtest("Drive cua-driver against the inactive window (AT-SPI)"):+ machine.copy_from_host("${mcpTest}", "/tmp/mcp-background-gui-test.py")+ machine.execute(+ "sh -lc '${recordGifScript} :99 /tmp/gui-frames ${outputGif} "+ "/tmp/stop-gui-recorder /tmp/record-gui.log 10 0.2 >/dev/null 2>&1 & echo $! >/tmp/record-gui.pid'"+ )+ status, result = machine.execute("${a11yEnv} timeout 200 python3 /tmp/mcp-background-gui-test.py 2>&1")+ machine.log(result)+ machine.execute("touch /tmp/stop-gui-recorder")+ machine.execute("timeout 60 sh -lc 'while kill -0 $(cat /tmp/record-gui.pid) 2>/dev/null; do sleep 0.2; done'")+ # If the recorder (or its convert) is still grinding past the wait,+ # kill it hard so it can't thrash the VM and wedge later commands.+ machine.execute("pkill -9 -f record-x11-gif >/dev/null 2>&1; pkill -9 -x convert >/dev/null 2>&1; pkill -9 -x import >/dev/null 2>&1; true")+ machine.log(machine.execute("sh -lc 'cat /tmp/record-gui.log || true'")[1])+ # Best-effort GIF copy (see skeleton path): a recorder hiccup must not+ # fail the job before the real assertions run.+ if machine.execute("test -s ${outputGif}")[0] == 0:+ machine.copy_from_machine("${outputGif}", "")+ else:+ machine.log("WARN: ${outputGif} missing; skipping GIF copy")+ assert "background GUI test typed" in result, result++ with subtest("Input landed: driver's native AT-SPI reads the window back"):+ assert any(tok in result for tok in ('frame "', 'window "', 'document', 'text "', 'entry "')), (+ "driver get_text did not return an accessibility node for the background app:\n"+ + result+ )++ with subtest("Focus stayed on the control terminal"):+ control = machine.succeed("head -1 /tmp/control-xid.txt").strip()+ active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip()+ assert control == active, "expected active window " + control + ", got " + active++ ${cdpSubtest}+ ${tkSubtest}+ '';++ driveBody = if isSkeleton then skeletonDrive else fullDrive;+in++pkgs.testers.nixosTest {+ name = "cua-driver-linux-background-gui-${app}-test";+ meta.maintainers = [ ];++ nodes.machine =+ { pkgs, ... }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = selected.memoryMB;+ diskSize = 8192;+ };+ services.cua-driver.enable = true;+ services.dbus.enable = true;+ environment.systemPackages = with pkgs; [+ xorg.xorgserver+ xterm+ openbox+ picom+ xdotool+ imagemagick # `import` + `convert` for the screen-recorded GIF+ dbus+ at-spi2-core+ testPython+ jq+ procps+ glib # `gsettings` to flip toolkit-accessibility+ gsettings-desktop-schemas # provides org.gnome.desktop.interface schema+ ] ++ selected.packages;+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Start X11 + session D-Bus + AT-SPI bus"):+ machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/tmp/xvfb.log 2>&1 &")+ machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10)+ machine.execute("DISPLAY=:99 openbox >/tmp/openbox.log 2>&1 &")+ machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &")+ machine.succeed("mkdir -p /run/user/0 && chmod 700 /run/user/0")+ machine.execute("dbus-daemon --session --address=unix:path=/tmp/cua-session-bus --fork >/tmp/dbus.log 2>&1")+ machine.wait_until_succeeds("test -S /tmp/cua-session-bus", timeout=10)+ machine.succeed("mkdir -p /tmp/cua-cfg")+ machine.execute("${a11yEnv} ${pkgs.at-spi2-core}/libexec/at-spi-bus-launcher --launch-immediately >/tmp/atspi-launcher.log 2>&1 &")+ machine.wait_until_succeeds(+ "${a11yEnv} dbus-send --session --print-reply "+ "--dest=org.freedesktop.DBus / org.freedesktop.DBus.NameHasOwner "+ "string:org.a11y.Bus | grep -q 'boolean true'",+ timeout=15,+ )+ machine.execute(+ "${a11yEnv} dbus-send --session --print-reply --dest=org.a11y.Bus "+ "/org/a11y/bus org.freedesktop.DBus.Properties.Set "+ "string:org.a11y.Status string:IsEnabled variant:boolean:true 2>&1 | tee /tmp/a11y-enable.log"+ )+ machine.log("a11y IsEnabled set: " + machine.execute("cat /tmp/a11y-enable.log")[1])+ machine.execute(+ "${a11yEnv} dbus-send --session --print-reply --dest=org.a11y.Bus "+ "/org/a11y/bus org.freedesktop.DBus.Properties.Get "+ "string:org.a11y.Status string:IsEnabled 2>&1 | tee /tmp/a11y-get.log"+ )+ machine.log("a11y IsEnabled get: " + machine.execute("cat /tmp/a11y-get.log")[1])+ machine.log("atspi-launcher.log: " + machine.execute("cat /tmp/atspi-launcher.log")[1])++ with subtest("Focused control terminal"):+ machine.execute("sh -lc 'DISPLAY=:99 xterm -T Control -geometry 60x20+40+120 >/tmp/control.log 2>&1 & echo $! >/tmp/control-pid.txt'")+ machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/control-pid.txt) >/tmp/control-xid.txt", timeout=20)+ machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)")++ with subtest("Launch target app (${app}) in the background"):+ machine.execute("sh -lc '${a11yEnv} ${selected.launch} >/tmp/target.log 2>&1 & echo $! >/tmp/target-pid.txt'")+ # Surface the app's own stdout/stderr early so launch failures (e.g. a Qt+ # platform-plugin error) are visible instead of just a window-find timeout.+ machine.sleep(5)+ machine.log("target.log after launch: " + machine.execute("cat /tmp/target.log")[1])+ # Find the background window via the per-app matcher, with a PID / newest+ # -window fallback. Generous timeout: Electron/KDE are slow to first paint.+ machine.wait_until_succeeds("${windowFindCmd}", timeout=120)+ machine.log("target-xid: " + machine.execute("cat /tmp/target-xid.txt")[1])+ # Keep focus on the control terminal — launching the app in the+ # background must not steal focus.+ machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)")+ machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/control-xid.txt)")++ ${driveBody}+ '';+}diff --git a/nix/cua-driver/tests/linux-background-terminal-gif.nix b/nix/cua-driver/tests/linux-background-terminal-gif.nixnew file mode 100644index 0000000000..8f01af6aa5--- /dev/null+++ b/nix/cua-driver/tests/linux-background-terminal-gif.nix@@ -0,0 +1,200 @@+# Linux background terminal GIF test+#+# Records a GIF while cua-driver types into an inactive xterm window and+# executes a shell command without stealing focus.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-linux-background-terminal-gif+#+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ ...+}:++let+ mcpBackgroundTerminalTest = pkgs.writeText "mcp-background-terminal-gif-test.py" ''+ import json+ import os+ import subprocess+ import sys+ import threading+ import time++ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")++ def start_driver():+ proc = subprocess.Popen(+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ env={**os.environ},+ )++ def drain_stderr():+ for line in proc.stderr:+ sys.stderr.buffer.write(line)+ sys.stderr.buffer.flush()++ threading.Thread(target=drain_stderr, daemon=True).start()+ return proc++ def send(proc, method, params=None, req_id=None):+ msg = {"jsonrpc": "2.0", "method": method}+ if params is not None:+ msg["params"] = params+ if req_id is not None:+ msg["id"] = req_id+ proc.stdin.write((json.dumps(msg) + "\n").encode())+ proc.stdin.flush()++ def recv(proc, timeout=30):+ result = [None]++ def reader():+ result[0] = proc.stdout.readline()++ thread = threading.Thread(target=reader)+ thread.start()+ thread.join(timeout)+ if thread.is_alive():+ raise TimeoutError("No response within timeout")+ line = result[0].decode().strip()+ if not line:+ raise RuntimeError("Driver returned an empty response")+ return json.loads(line)++ def call_tool(proc, req_id, name, arguments):+ send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id)+ resp = recv(proc, timeout=45)+ if "error" in resp and resp["error"] is not None:+ raise RuntimeError(f"{name} failed: {resp}")+ if resp.get("result", {}).get("isError"):+ raise RuntimeError(f"{name} returned isError: {resp}")+ return resp++ def main():+ with open("/tmp/background-target-xid.txt", "r", encoding="utf-8") as f:+ target_window_id = int(f.read().strip())+ with open("/tmp/background-target-pid.txt", "r", encoding="utf-8") as f:+ target_pid = int(f.read().strip())+ with open("/tmp/background-control-xid.txt", "r", encoding="utf-8") as f:+ control_window_id = int(f.read().strip())+ with open("/tmp/background-control-pid.txt", "r", encoding="utf-8") as f:+ control_pid = int(f.read().strip())++ proc = start_driver()+ try:+ send(proc, "initialize", {+ "protocolVersion": "2024-11-05",+ "capabilities": {},+ "clientInfo": {"name": "nixos-background-terminal-gif-test", "version": "1.0.0"},+ }, req_id=1)+ recv(proc)+ send(proc, "notifications/initialized", {})+ time.sleep(0.3)++ call_tool(proc, 2, "set_agent_cursor_enabled", {"enabled": True})+ call_tool(proc, 3, "move_cursor", {"x": 50.0, "y": 900.0})+ time.sleep(0.5)+ call_tool(proc, 4, "click", {+ "pid": control_pid,+ "window_id": control_window_id,+ "x": 120.0,+ "y": 120.0,+ })+ time.sleep(0.6)+ call_tool(proc, 5, "type_text", {+ "pid": target_pid,+ "window_id": target_window_id,+ "text": "echo hello | tee /tmp/background-hello.txt",+ })+ call_tool(proc, 6, "press_key", {+ "pid": target_pid,+ "window_id": target_window_id,+ "key": "enter",+ })+ time.sleep(1.8)+ print("background terminal GIF test complete", flush=True)+ finally:+ proc.stdin.close()+ proc.terminate()+ proc.wait(timeout=5)++ if __name__ == "__main__":+ main()+ '';++ recordGifScript = import ./record-x11-gif.nix { inherit pkgs; };+in++pkgs.testers.nixosTest {+ name = "cua-driver-linux-background-terminal-gif-test";+ meta.maintainers = [ ];++ nodes.machine =+ {+ pkgs,+ ...+ }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 2048;+ };+ services.cua-driver.enable = true;+ environment.systemPackages = with pkgs; [+ xorg.xorgserver+ xterm+ openbox+ picom+ xdotool+ imagemagick+ python3+ jq+ procps+ ];+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Start X11 desktop and target/control xterms"):+ machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/tmp/xvfb.log 2>&1 &")+ machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10)+ machine.execute("DISPLAY=:99 openbox >/tmp/openbox.log 2>&1 &")+ machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &")+ machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Background Target' -fa Monospace -fs 14 -geometry 70x24+40+120 >/tmp/background-target.log 2>&1 & echo \\$! >/tmp/background-target-pid.txt\"")+ machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Background Control' -fa Monospace -fs 14 -geometry 70x24+690+120 >/tmp/background-control.log 2>&1 & echo \\$! >/tmp/background-control-pid.txt\"")+ machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/background-target-pid.txt) >/tmp/background-target-xid.txt", timeout=20)+ machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/background-control-pid.txt) >/tmp/background-control-xid.txt", timeout=20)+ machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/background-control-xid.txt)")+ machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/background-control-xid.txt)")++ with subtest("Record GIF and execute command in inactive terminal"):+ machine.copy_from_host("${mcpBackgroundTerminalTest}", "/tmp/mcp-background-terminal-gif-test.py")+ machine.execute(+ "sh -lc '${recordGifScript} :99 /tmp/background-frames /tmp/cua-driver-linux-background-terminal.gif "+ "/tmp/stop-background-recorder /tmp/ffmpeg-background.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-background.pid'"+ )+ result = machine.succeed("timeout 60 env DISPLAY=:99 python3 /tmp/mcp-background-terminal-gif-test.py 2>&1")+ machine.log(result)+ assert "background terminal GIF test complete" in result, result+ machine.succeed("touch /tmp/stop-background-recorder")+ machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-background.pid) 2>/dev/null", timeout=60)+ machine.log(machine.succeed("sh -lc 'cat /tmp/ffmpeg-background.log || true'"))+ machine.succeed("test -s /tmp/cua-driver-linux-background-terminal.gif")+ machine.wait_until_succeeds("grep -Fx 'hello' /tmp/background-hello.txt", timeout=20)++ with subtest("Verify focus stayed on control terminal"):+ control = machine.succeed("head -1 /tmp/background-control-xid.txt").strip()+ active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip()+ assert control == active, f"expected active window {control}, got {active}"++ with subtest("Copy GIF out of the VM"):+ machine.copy_from_machine("/tmp/cua-driver-linux-background-terminal.gif", "")+ '';+}diff --git a/nix/cua-driver/tests/linux-cursor-click-gif.nix b/nix/cua-driver/tests/linux-cursor-click-gif.nixnew file mode 100644index 0000000000..6ee3f96347--- /dev/null+++ b/nix/cua-driver/tests/linux-cursor-click-gif.nix@@ -0,0 +1,183 @@+# Linux cursor click GIF test+#+# Records a GIF of the Linux overlay cursor moving as part of a click action+# and proves the click focused the target xterm and allowed a shell command to+# execute.+#+# To run: nix build .#checks.x86_64-linux.cua-driver-linux-cursor-click-gif+#+{+ pkgs,+ lib ? pkgs.lib,+ cuaDriverModule,+ ...+}:++let+ mcpClickTest = pkgs.writeText "mcp-click-gif-test.py" ''+ import json+ import os+ import subprocess+ import sys+ import threading+ import time++ DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver")++ def start_driver():+ proc = subprocess.Popen(+ [DRIVER_BIN, "mcp", "--no-daemon-relaunch"],+ stdin=subprocess.PIPE,+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ env={**os.environ},+ )++ def drain_stderr():+ for line in proc.stderr:+ sys.stderr.buffer.write(line)+ sys.stderr.buffer.flush()++ threading.Thread(target=drain_stderr, daemon=True).start()+ return proc++ def send(proc, method, params=None, req_id=None):+ msg = {"jsonrpc": "2.0", "method": method}+ if params is not None:+ msg["params"] = params+ if req_id is not None:+ msg["id"] = req_id+ proc.stdin.write((json.dumps(msg) + "\n").encode())+ proc.stdin.flush()++ def recv(proc, timeout=30):+ result = [None]++ def reader():+ result[0] = proc.stdout.readline()++ thread = threading.Thread(target=reader)+ thread.start()+ thread.join(timeout)+ if thread.is_alive():+ raise TimeoutError("No response within timeout")+ line = result[0].decode().strip()+ if not line:+ raise RuntimeError("Driver returned an empty response")+ return json.loads(line)++ def call_tool(proc, req_id, name, arguments):+ send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id)+ resp = recv(proc, timeout=45)+ if "error" in resp and resp["error"] is not None:+ raise RuntimeError(f"{name} failed: {resp}")+ if resp.get("result", {}).get("isError"):+ raise RuntimeError(f"{name} returned isError: {resp}")+ return resp++ def main():+ with open("/tmp/target-click-xid.txt", "r", encoding="utf-8") as f:+ window_id = int(f.read().strip())+ with open("/tmp/target-click-pid.txt", "r", encoding="utf-8") as f:+ pid = int(f.read().strip())++ proc = start_driver()+ try:+ send(proc, "initialize", {+ "protocolVersion": "2024-11-05",+ "capabilities": {},+ "clientInfo": {"name": "nixos-click-gif-test", "version": "1.0.0"},+ }, req_id=1)+ recv(proc)+ send(proc, "notifications/initialized", {})+ time.sleep(0.3)++ call_tool(proc, 2, "set_agent_cursor_enabled", {"enabled": True})+ call_tool(proc, 3, "move_cursor", {"x": 1100.0, "y": 900.0})+ time.sleep(0.8)+ call_tool(proc, 4, "click", {"pid": pid, "window_id": window_id, "x": 120.0, "y": 120.0})+ time.sleep(0.5)+ call_tool(proc, 5, "type_text", {+ "pid": pid,+ "window_id": window_id,+ "text": "echo click-focus > /tmp/click-focus.txt",+ })+ call_tool(proc, 6, "press_key", {"pid": pid, "window_id": window_id, "key": "enter"})+ time.sleep(1.5)+ print("click GIF test complete", flush=True)+ finally:+ proc.stdin.close()+ proc.terminate()+ proc.wait(timeout=5)++ if __name__ == "__main__":+ main()+ '';++ recordGifScript = import ./record-x11-gif.nix { inherit pkgs; };+in++pkgs.testers.nixosTest {+ name = "cua-driver-linux-cursor-click-gif-test";+ meta.maintainers = [ ];++ nodes.machine =+ {+ pkgs,+ ...+ }:+ {+ imports = [ cuaDriverModule ];+ virtualisation = {+ cores = 2;+ memorySize = 2048;+ };+ services.cua-driver.enable = true;+ environment.systemPackages = with pkgs; [+ xorg.xorgserver+ xterm+ openbox+ picom+ xdotool+ imagemagick+ python3+ jq+ procps+ ];+ };++ testScript = ''+ machine.start()+ machine.wait_for_unit("multi-user.target")++ with subtest("Start X11 desktop and two xterms"):+ machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/tmp/xvfb.log 2>&1 &")+ machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10)+ machine.execute("DISPLAY=:99 openbox >/tmp/openbox.log 2>&1 &")+ machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &")+ machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Target Click' -fa Monospace -fs 14 -geometry 70x24+80+120 >/tmp/target-click.log 2>&1 & echo \\$! >/tmp/target-click-pid.txt\"")+ machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Control Click' -fa Monospace -fs 14 -geometry 70x24+700+120 >/tmp/control-click.log 2>&1 & echo \\$! >/tmp/control-click-pid.txt\"")+ machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/target-click-pid.txt) >/tmp/target-click-xid.txt", timeout=20)+ machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/control-click-pid.txt) >/tmp/control-click-xid.txt", timeout=20)+ machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-click-xid.txt)")+ machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/control-click-xid.txt)")++ with subtest("Record click GIF and run driver actions"):+ machine.copy_from_host("${mcpClickTest}", "/tmp/mcp-click-gif-test.py")+ machine.execute(+ "sh -lc '${recordGifScript} :99 /tmp/click-frames /tmp/cua-driver-linux-cursor-click.gif "+ "/tmp/stop-click-recorder /tmp/ffmpeg-click.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-click.pid'"+ )+ result = machine.succeed("timeout 60 env DISPLAY=:99 python3 /tmp/mcp-click-gif-test.py 2>&1")+ machine.log(result)+ assert "click GIF test complete" in result, result+ machine.succeed("touch /tmp/stop-click-recorder")+ machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-click.pid) 2>/dev/null", timeout=60)+ machine.log(machine.succeed("sh -lc 'cat /tmp/ffmpeg-click.log || true'"))+ machine.succeed("test -s /tmp/cua-driver-linux-cursor-click.gif")+ machine.wait_until_succeeds("test -f /tmp/click-focus.txt", timeout=20)++ with subtest("Copy GIF out of the VM"):+ machine.copy_from_machine("/tmp/cua-driver-linux-cursor-click.gif", "")+ '';+}diff --git a/nix/cua-driver/tests/record-x11-gif.nix b/nix/cua-driver/tests/record-x11-gif.nixnew file mode 100644index 0000000000..c62c744fc9--- /dev/null+++ b/nix/cua-driver/tests/record-x11-gif.nix@@ -0,0 +1,49 @@+# Shared X11 screen-recording helper for the Linux visual (GIF) tests.+#+# Returns a `pkgs.writeShellScript` that loops `import -display <display>+# -window root frame-XXXX.png` until a stop-file appears, then stitches the+# frames into an animated GIF with `convert`. Both tools come from+# `pkgs.imagemagick`, which the importing test must add to+# `environment.systemPackages`.+#+# Usage (in a test's `let`):+# recordGifScript = import ./record-x11-gif.nix { inherit pkgs; };+# then, inside the testScript, start it in the background before driving and+# `touch` the stop-file afterwards:+# ${recordGifScript} <display> <frames_dir> <output_gif> <stop_file> \+# <log_file> <delay_cs> <interval>+{ pkgs }:++pkgs.writeShellScript "record-x11-gif.sh" ''+ set -eu+ display="$1"+ frames_dir="$2"+ output_gif="$3"+ stop_file="$4"+ log_file="$5"+ delay_cs="$6"+ interval="$7"++ rm -f "$stop_file" "$output_gif" "$log_file"+ rm -rf "$frames_dir"+ mkdir -p "$frames_dir"++ # Cap the frame count: long driver runs (the skeleton budget is 300s) can+ # otherwise pile up 1000+ frames and the final `convert` thrashes/OOMs the+ # 2GB test VM, wedging every later command in the job. 450 frames is a ~45s+ # GIF at the default cadence — plenty. Each `import` and the final `convert`+ # are also time-bounded so a wedged X grab or a slow stitch can't stall the+ # job.+ max_frames=450+ i=0+ while [ ! -f "$stop_file" ] && [ "$i" -lt "$max_frames" ]; do+ frame=$(printf "%s/frame-%04d.png" "$frames_dir" "$i")+ timeout 10 import -display "$display" -window root "$frame" >>"$log_file" 2>&1 || true+ i=$((i + 1))+ sleep "$interval"+ done++ if ls "$frames_dir"/frame-*.png >/dev/null 2>&1; then+ timeout 120 convert -delay "$delay_cs" -loop 0 "$frames_dir"/frame-*.png "$output_gif" >>"$log_file" 2>&1 || true+ fi+''