Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): Linux X11 and Wayland convergence (#2182)

Francesco Bonacci51a8e2593fc8
diff --git a/.github/workflows/cd-rust-cua-driver.yml b/.github/workflows/cd-rust-cua-driver.yml
index 312dded99c..d4707e89e5 100644
--- a/.github/workflows/cd-rust-cua-driver.yml
+++ b/.github/workflows/cd-rust-cua-driver.yml
@@ -74,16 +74,14 @@ jobs:
- name: Install base tooling (container is bare)
run: |
apt-get update
- # The PipeWire / libei deps the native-Wayland portal path uses
- # are intentionally NOT installed here: bullseye ships PipeWire
- # 0.3.19 (too old for libspa-sys 0.8, which needs >= 0.3.40) and
- # has no libei-dev at all. They sit behind the `portal-libei`
- # Cargo feature (disabled by default) and are wired in via the
- # Nix build. The wlroots screencopy + virtual-pointer paths and
- # the X11 fallback work without them.
+ # PipeWire ScreenCast capture remains Nix/modern-build only because
+ # bullseye's 0.3.19 headers are too old for libspa-sys 0.8. Portal
+ # RemoteDesktop/libei input is pure Rust plus libxkbcommon and ships
+ # in these portable release binaries.
apt-get install -y --no-install-recommends \
git ca-certificates curl build-essential pkg-config \
- libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev
+ libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev \
+ libxkbcommon-dev
- uses: actions/checkout@v4
- name: Determine version
id: version
@@ -109,7 +107,7 @@ jobs:
rust-cua-driver-rs-linux-${{ matrix.arch }}-
- name: Build (release)
working-directory: libs/cua-driver/rust
- run: cargo build -p cua-driver --release --target ${{ matrix.target }}
+ run: cargo build -p cua-driver --release --features portal-input --target ${{ matrix.target }}
- name: Package
working-directory: libs/cua-driver/rust
run: |
diff --git a/.github/workflows/ci-cua-driver-interactive-linux.yml b/.github/workflows/ci-cua-driver-interactive-linux.yml
deleted file mode 100644
index 8e9e2a5cb0..0000000000
--- a/.github/workflows/ci-cua-driver-interactive-linux.yml
+++ /dev/null
@@ -1,128 +0,0 @@
-name: "CI: cua-driver interactive modality suite (Linux)"
-
-# Runs the cua-driver INTERACTIVE `#[ignore]` modality/harness tests on a real
-# (headless) Linux GUI session, so their BEHAVIOR — not just compilation — is
-# guarded on every cua-driver change. These tests need a display, the AT-SPI
-# accessibility bus, and the GTK3 harness app; the rest of CI only compiles them.
-#
-# What runs here:
-# - modality_capture_mode_test (ax→tree-only / vision→image-only / som→both)
-# - modality_desktop_scope_linux_test (window-less screen-absolute click via XTest + gate)
-#
-# How the GUI session is provided: Xvfb for the display, a per-job D-Bus session
-# bus (so AT-SPI's `org.a11y.Bus` can activate), at-spi2-core for the bridge, and
-# the GTK3/PyGObject runtime for the harness. The tests skip-with-note when the
-# a11y bus is unavailable rather than false-failing, so a vacuous run is visible
-# (no green-for-nothing surprise) without breaking the build.
-#
-# Companion to nix-build.yml / nix-wayland.yml (real desktops, broad app matrix).
-# This lane is narrow + fast and runs on free GitHub runners. Trigger: PRs that
-# touch the Rust driver or the harness, push to main, and manual dispatch.
-
-on:
- pull_request:
- paths:
- - "libs/cua-driver/rust/**"
- - "libs/cua-driver/tests/fixtures/**"
- - ".github/workflows/ci-cua-driver-interactive-linux.yml"
- push:
- branches: [main]
- paths:
- - "libs/cua-driver/rust/**"
- - "libs/cua-driver/tests/fixtures/**"
- - ".github/workflows/ci-cua-driver-interactive-linux.yml"
- workflow_dispatch:
-
-permissions:
- contents: read
-
-jobs:
- interactive-modality:
- name: Interactive modality suite (Xvfb + AT-SPI)
- runs-on: ubuntu-latest
- timeout-minutes: 30
-
- steps:
- - name: Checkout
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-
- - name: Install GUI + accessibility + harness runtime deps
- run: |
- sudo apt-get update
- sudo apt-get install -y --no-install-recommends \
- xvfb dbus-x11 at-spi2-core openbox \
- libgtk-3-0 gir1.2-gtk-3.0 python3-gi \
- build-essential libglib2.0-dev libgtk-3-dev \
- libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
- libayatana-appindicator3-dev librsvg2-dev \
- libxtst6 libxtst-dev libx11-dev libxext-dev \
- ffmpeg
-
- - name: Rust toolchain
- uses: dtolnay/rust-toolchain@stable
-
- - name: Cache cargo
- uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
- with:
- workspaces: "libs/cua-driver/rust -> target"
-
- - name: Build cua-driver (release — the test harness prefers release)
- working-directory: libs/cua-driver/rust
- run: cargo build --release -p cua-driver
-
- - name: Build the GTK3 harness app
- run: bash libs/cua-driver/tests/fixtures/build/linux.sh
-
- - name: AT-SPI reachability probe (diagnostic, non-fatal)
- run: |
- cat > /tmp/atspi_probe.py <<'PY'
- import gi
- gi.require_version("Atspi", "2.0")
- from gi.repository import Atspi
- print("[interactive-ci] AT-SPI reachable, desktop count:", Atspi.get_desktop_count())
- PY
-
- - name: Run the interactive modality suite under Xvfb + a session a11y bus
- working-directory: libs/cua-driver/rust
- env:
- NO_AT_BRIDGE: "0"
- GTK_A11Y: "1"
- run: |
- set -euo pipefail
- # Xvfb provides the display; dbus-run-session provides a session bus on
- # which AT-SPI's org.a11y.Bus auto-activates; GTK3's atk-bridge then
- # registers the harness on the accessibility tree.
- xvfb-run -a --server-args="-screen 0 1920x1080x24" \
- dbus-run-session -- bash -c '
- set -e
- # openbox manages/places the harness window for AT-SPI extents.
- openbox &
- sleep 2
- python3 /tmp/atspi_probe.py || echo "[interactive-ci] AT-SPI probe inconclusive — AX assertions will skip if the bus is unavailable"
-
- # capture_mode matrix (ax/vision/som): real assertions, headless-safe.
- cargo test -p cua-driver --test capture_contract_test \
- -- --ignored --nocapture --test-threads=1
-
- # desktop-scope: run the GATE here (the desktop_scope_disabled
- # contract). The *landing* assertion (counter advances on a
- # window-less click) needs a real display + WM to deliver the
- # screen-absolute XTest click — Xvfb does not faithfully deliver it,
- # so that test runs on the real-desktop lanes (the GNOME/Azure VM run
- # + the recorded artifact), not here. See nix-wayland.yml for the
- # real-session follow-up that can host the landing assertion too.
- cargo test -p cua-driver --test modality_desktop_scope_linux_test \
- window_scope_rejects_windowless_click \
- -- --ignored --nocapture --test-threads=1
- '
-
- - name: Upload any recordings / artifacts
- if: always()
- uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
- with:
- name: interactive-modality-artifacts
- path: |
- /tmp/*.mp4
- libs/cua-driver/rust/target/release/*.log
- if-no-files-found: ignore
- retention-days: 7
diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.yml
index e2d050aa78..40c2cfee03 100644
--- a/.github/workflows/ci-rust-linux.yml
+++ b/.github/workflows/ci-rust-linux.yml
@@ -48,7 +48,10 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
clang pkg-config libdbus-1-dev libpipewire-0.3-dev libspa-0.2-dev \
- libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev
+ libei-dev libxkbcommon-dev libx11-dev libxi-dev libxtst-dev libxext-dev
- name: Run Linux Rust tests
working-directory: libs/cua-driver/rust
run: cargo test -p cua-driver -p cua-driver-core -p cua-driver-testkit -p platform-linux --all-targets --locked
+ - name: Compile portable GNOME/KDE portal input
+ working-directory: libs/cua-driver/rust
+ run: cargo check -p cua-driver --features portal-input --locked
diff --git a/.github/workflows/e2e-rust-linux-wayland.yml b/.github/workflows/e2e-rust-linux-wayland.yml
index 1d80fea196..d03136df3d 100644
--- a/.github/workflows/e2e-rust-linux-wayland.yml
+++ b/.github/workflows/e2e-rust-linux-wayland.yml
@@ -1,10 +1,31 @@
-name: "E2E: Rust Linux Wayland interactive"
+name: "E2E: Rust Linux Wayland"
on:
workflow_dispatch:
inputs:
ref:
- description: "Optional commit, branch, or tag override"
+ description: "Optional full 40-character commit SHA; defaults to dispatch SHA"
+ required: false
+ default: ""
+ environment:
+ description: "Wayland environment to validate"
+ required: false
+ type: choice
+ default: sway
+ options:
+ - sway
+ lane:
+ description: "Diagnostic lane; canonical dispatches use all"
+ required: false
+ type: choice
+ default: all
+ options:
+ - all
+ - shared
+ - native
+ - capture
+ cell_filter:
+ description: "Optional shared-cell substring for targeted diagnostics"
required: false
default: ""
@@ -13,28 +34,73 @@ permissions:
actions: read
jobs:
+ source:
+ name: "Resolve exact source"
+ runs-on: ubuntu-latest
+ outputs:
+ sha: ${{ steps.resolve.outputs.sha }}
+ lanes: ${{ steps.resolve.outputs.lanes }}
+ steps:
+ - id: resolve
+ name: Validate source SHA
+ shell: bash
+ env:
+ REQUESTED_SHA: ${{ inputs.ref }}
+ DISPATCH_SHA: ${{ github.sha }}
+ run: |
+ sha="${REQUESTED_SHA:-$DISPATCH_SHA}"
+ if [[ ! "$sha" =~ ^[0-9a-fA-F]{40}$ ]]; then
+ echo "ref must be a full 40-character commit SHA" >&2
+ exit 2
+ fi
+ echo "sha=${sha,,}" >> "$GITHUB_OUTPUT"
+ case "${{ inputs.lane }}" in
+ all) echo 'lanes=["shared","native","capture"]' >> "$GITHUB_OUTPUT" ;;
+ shared|native|capture) printf 'lanes=["%s"]\n' "${{ inputs.lane }}" >> "$GITHUB_OUTPUT" ;;
+ *) echo "unsupported lane: ${{ inputs.lane }}" >&2; exit 2 ;;
+ esac
+
wayland:
- name: "Linux / native Wayland Rust matrix"
+ name: "Linux / ${{ inputs.environment }} / ${{ matrix.lane }}"
+ needs: source
runs-on: ubuntu-latest
timeout-minutes: 90
+ strategy:
+ fail-fast: false
+ matrix:
+ lane: ${{ fromJSON(needs.source.outputs.lanes) }}
+ env:
+ CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }}
+ CUA_E2E_INTERNAL_LANE: ${{ matrix.lane }}
+ CUA_E2E_CELL_FILTER: ${{ inputs.cell_filter }}
+ CUA_E2E_WAYLAND_SESSION: ${{ inputs.environment }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
- ref: ${{ inputs.ref || github.ref }}
- - name: Install Nix
- uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
- with:
- extra_nix_config: |
- experimental-features = nix-command flakes
+ ref: ${{ needs.source.outputs.sha }}
+ - uses: dtolnay/rust-toolchain@stable
+ - name: Install Sway and Rust build dependencies
+ shell: bash
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ sway wf-recorder grim wtype dbus dbus-x11 at-spi2-core libglib2.0-bin \
+ python3-gi gir1.2-gtk-3.0 libgtk-3-dev clang pkg-config \
+ libdbus-1-dev libpipewire-0.3-dev libspa-0.2-dev libei-dev \
+ libwayland-dev libxkbcommon-dev libx11-dev libxi-dev libxtst-dev \
+ libxext-dev libdrm-dev libgbm-dev libwebkit2gtk-4.1-dev \
+ libssl-dev libxdo-dev libayatana-appindicator3-dev librsvg2-dev \
+ ffmpeg jq
- name: Run complete native Wayland matrix
- run: >-
- nix develop .#cua-driver-wayland-e2e
- -c scripts/ci/linux/run-rust-e2e-wayland.sh
+ shell: bash
+ env:
+ CUA_AT_SPI_BUS_LAUNCHER: /usr/libexec/at-spi-bus-launcher
+ run: scripts/ci/linux/run-rust-e2e-wayland.sh
- name: Upload Wayland results
if: always()
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
with:
- name: rust-linux-wayland
+ name: rust-linux-wayland-${{ inputs.environment }}-${{ matrix.lane }}
path: artifacts/cua-driver/linux
if-no-files-found: ignore
compression-level: 0
@@ -42,36 +108,78 @@ jobs:
summary:
if: always()
- needs: [wayland]
- name: "Linux / native Wayland summary"
+ needs: [source, wayland]
+ name: "Linux / pure Wayland summary"
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
- ref: ${{ inputs.ref || github.ref }}
+ ref: ${{ needs.source.outputs.sha }}
- name: Download Wayland results
continue-on-error: true
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
with:
- name: rust-linux-wayland
- path: artifacts/rust-linux-wayland
+ pattern: rust-linux-wayland-${{ inputs.environment }}-*
+ path: artifacts
- name: Publish typed matrix summary
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
- summary="artifacts/rust-linux-wayland/summary.md"
- artifact_id=$(gh api \
- "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" \
- --jq '.artifacts[] | select(.name == "rust-linux-wayland") | .id' | head -n 1)
- if [[ -f "$summary" && -n "$artifact_id" ]]; then
- artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id"
- scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$GITHUB_STEP_SUMMARY"
- elif [[ -f "$summary" ]]; then
- cat "$summary" >> "$GITHUB_STEP_SUMMARY"
- else
- echo "# CUA Rust Linux Wayland E2E" >> "$GITHUB_STEP_SUMMARY"
- echo >> "$GITHUB_STEP_SUMMARY"
- echo "No typed matrix summary was produced; inspect the Wayland job log." >> "$GITHUB_STEP_SUMMARY"
+ output=matrix-summary.md
+ artifacts_json=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100")
+ {
+ echo "# CUA Rust Linux Wayland E2E: ${{ inputs.environment }}"
+ echo
+ echo "Shared, native, and capture lanes run independently on clean runners."
+ } > "$output"
+ found=0
+ while IFS= read -r summary; do
+ found=1
+ artifact=$(basename "$(dirname "$summary")")
+ artifact_id=$(jq -r --arg name "$artifact" \
+ '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1)
+ echo >> "$output"
+ echo "## $artifact" >> "$output"
+ if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then
+ artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id"
+ scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$output"
+ else
+ cat "$summary" >> "$output"
+ fi
+ done < <(find artifacts -type f -name summary.md -print | sort)
+ if [[ "$found" == 0 ]]; then
+ echo >> "$output"
+ echo "No typed lane summary was produced; inspect the lane logs." >> "$output"
fi
+ {
+ echo
+ echo "## Trajectory videos"
+ echo
+ echo "| Lane | Videos | Artifact |"
+ echo "| --- | ---: | --- |"
+ } >> "$output"
+ for lane in shared native capture; do
+ artifact="rust-linux-wayland-${{ inputs.environment }}-$lane"
+ recording_dir="artifacts/$artifact/recordings"
+ video_count=0
+ [[ -d "$recording_dir" ]] && video_count=$(find "$recording_dir" -type f -name recording.mp4 | wc -l | tr -d ' ')
+ artifact_id=$(jq -r --arg name "$artifact" \
+ '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1)
+ if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then
+ artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id"
+ link="[Open artifact]($artifact_url)"
+ else
+ link="Not produced"
+ fi
+ echo "| $lane | $video_count | $link |" >> "$output"
+ done
+ cat "$output" >> "$GITHUB_STEP_SUMMARY"
+ - name: Upload combined summary
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: rust-linux-wayland-matrix-summary
+ path: matrix-summary.md
+ if-no-files-found: error
diff --git a/.github/workflows/e2e-rust-linux.yml b/.github/workflows/e2e-rust-linux.yml
index 147462dd79..d86e4c5894 100644
--- a/.github/workflows/e2e-rust-linux.yml
+++ b/.github/workflows/e2e-rust-linux.yml
@@ -4,29 +4,62 @@ on:
workflow_dispatch:
inputs:
ref:
- description: "Commit, branch, or tag to test"
- required: true
- default: "main"
- suite:
- description: "Rust desktop suite"
- required: true
+ description: "Optional full 40-character commit SHA; defaults to dispatch SHA"
+ required: false
+ default: ""
+ lane:
+ description: "Diagnostic lane; canonical dispatches use all"
+ required: false
type: choice
- options: [shared, modality, all]
- default: shared
+ default: all
+ options:
+ - all
+ - shared
+ - native
+ - capture
+ cell_filter:
+ description: "Optional shared-cell substring for targeted diagnostics"
+ required: false
+ default: ""
permissions:
contents: read
+ actions: read
jobs:
+ source:
+ name: "Resolve exact source"
+ runs-on: ubuntu-latest
+ outputs:
+ sha: ${{ steps.resolve.outputs.sha }}
+ steps:
+ - id: resolve
+ name: Validate source SHA
+ shell: bash
+ env:
+ REQUESTED_SHA: ${{ inputs.ref }}
+ DISPATCH_SHA: ${{ github.sha }}
+ run: |
+ sha="${REQUESTED_SHA:-$DISPATCH_SHA}"
+ if [[ ! "$sha" =~ ^[0-9a-fA-F]{40}$ ]]; then
+ echo "ref must be a full 40-character commit SHA" >&2
+ exit 2
+ fi
+ echo "sha=${sha,,}" >> "$GITHUB_OUTPUT"
+
shared:
- if: inputs.suite == 'shared' || inputs.suite == 'all'
- name: "Linux / shared Electron"
+ if: inputs.lane == 'all' || inputs.lane == 'shared'
+ name: "Linux / shared Electron + Tauri"
+ needs: source
runs-on: ubuntu-latest
timeout-minutes: 60
+ env:
+ CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }}
+ CUA_E2E_CELL_FILTER: ${{ inputs.cell_filter }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
- ref: ${{ inputs.ref }}
+ ref: ${{ needs.source.outputs.sha }}
- uses: dtolnay/rust-toolchain@stable
- name: Install GUI and Rust build dependencies
run: |
@@ -38,10 +71,12 @@ jobs:
libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
libayatana-appindicator3-dev librsvg2-dev ffmpeg
- name: Run shared Rust behavior matrix
+ env:
+ CUA_E2E_INTERNAL_LANE: shared
run: |
xvfb-run -a --server-args="-screen 0 1920x1080x24" \
dbus-run-session -- bash -lc \
- "scripts/ci/linux/run-rust-e2e.sh --suite shared"
+ "openbox >/tmp/cua-openbox.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh"
- name: Upload shared results
if: always()
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
@@ -52,15 +87,18 @@ jobs:
compression-level: 0
retention-days: 14
- modality:
- if: inputs.suite == 'modality' || inputs.suite == 'all'
- name: "Linux / modality and desktop scope"
+ native:
+ if: inputs.lane == 'all' || inputs.lane == 'native'
+ name: "Linux / GTK3 native harness"
+ needs: source
runs-on: ubuntu-latest
timeout-minutes: 60
+ env:
+ CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
- ref: ${{ inputs.ref }}
+ ref: ${{ needs.source.outputs.sha }}
- uses: dtolnay/rust-toolchain@stable
- name: Install GUI and Rust build dependencies
run: |
@@ -71,16 +109,57 @@ jobs:
libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \
libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
libayatana-appindicator3-dev librsvg2-dev ffmpeg
- - name: Run modality Rust matrix
+ - name: Run GTK3 native Rust harness
+ env:
+ CUA_E2E_INTERNAL_LANE: native
run: |
xvfb-run -a --server-args="-screen 0 1920x1080x24" \
dbus-run-session -- bash -lc \
- "scripts/ci/linux/run-rust-e2e.sh --suite modality"
- - name: Upload modality results
+ "openbox >/tmp/cua-openbox.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh"
+ - name: Upload native results
if: always()
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
with:
- name: rust-linux-modality
+ name: rust-linux-native
+ path: artifacts/cua-driver/linux
+ if-no-files-found: ignore
+ compression-level: 0
+ retention-days: 14
+
+ capture:
+ if: inputs.lane == 'all' || inputs.lane == 'capture'
+ name: "Linux / capture and desktop scope"
+ needs: source
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ env:
+ CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }}
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ ref: ${{ needs.source.outputs.sha }}
+ - uses: dtolnay/rust-toolchain@stable
+ - name: Install GUI and Rust build dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ xvfb dbus-x11 at-spi2-core openbox python3-gi gir1.2-gtk-3.0 \
+ libgtk-3-dev clang pkg-config libdbus-1-dev libpipewire-0.3-dev \
+ libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \
+ libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \
+ libayatana-appindicator3-dev librsvg2-dev ffmpeg
+ - name: Run capture and desktop-scope contracts
+ env:
+ CUA_E2E_INTERNAL_LANE: capture
+ run: |
+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ dbus-run-session -- bash -lc \
+ "openbox >/tmp/cua-openbox.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh"
+ - name: Upload capture results
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: rust-linux-capture
path: artifacts/cua-driver/linux
if-no-files-found: ignore
compression-level: 0
@@ -88,31 +167,89 @@ jobs:
summary:
if: always()
- needs: [shared, modality]
+ needs: [source, shared, native, capture]
name: "Linux / matrix summary"
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ ref: ${{ needs.source.outputs.sha }}
- name: Download lane results
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
with:
path: artifacts
- name: Publish matrix summary
shell: bash
+ env:
+ GH_TOKEN: ${{ github.token }}
run: |
+ summary_path=matrix-summary.md
{
echo "# CUA Rust Linux E2E matrix"
echo
echo "The lane jobs above are independent; a failure in one lane does not hide the others."
echo
- } >> "$GITHUB_STEP_SUMMARY"
+ } > "$summary_path"
+ artifacts_json=$(gh api \
+ "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100")
found=0
while IFS= read -r summary; do
found=1
- echo "## $(basename "$(dirname "$summary")")" >> "$GITHUB_STEP_SUMMARY"
- cat "$summary" >> "$GITHUB_STEP_SUMMARY"
- echo >> "$GITHUB_STEP_SUMMARY"
+ artifact=$(basename "$(dirname "$summary")")
+ echo "## $artifact" >> "$summary_path"
+ artifact_id=$(jq -r --arg name "$artifact" \
+ '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1)
+ if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then
+ artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id"
+ scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$summary_path"
+ else
+ cat "$summary" >> "$summary_path"
+ fi
+ echo >> "$summary_path"
done < <(find artifacts -type f -name summary.md -print | sort)
if [[ "$found" == 0 ]]; then
- echo "No lane summary artifact was produced." >> "$GITHUB_STEP_SUMMARY"
+ echo "No lane summary artifact was produced." >> "$summary_path"
fi
+ {
+ echo
+ echo "## Trajectory videos"
+ echo
+ echo "Full-desktop MP4s are retained for 14 days inside each lane artifact."
+ echo
+ echo "| Lane | Videos | Artifact |"
+ echo "| --- | ---: | --- |"
+ } >> "$summary_path"
+ while IFS='|' read -r lane artifact; do
+ recording_dir="artifacts/$artifact/recordings"
+ if [[ -d "$recording_dir" ]]; then
+ video_count=$(find "$recording_dir" -type f -name recording.mp4 | wc -l | tr -d ' ')
+ else
+ video_count=0
+ fi
+ artifact_id=$(jq -r --arg name "$artifact" \
+ '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1)
+ if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then
+ artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id"
+ artifact_link="[Download $artifact]($artifact_url)"
+ else
+ artifact_link="Not produced in this dispatch"
+ fi
+ echo "| $lane | $video_count | $artifact_link |" >> "$summary_path"
+ done <<'EOF'
+ Shared Electron + Tauri|rust-linux-shared
+ GTK3 native|rust-linux-native
+ Capture + desktop scope|rust-linux-capture
+ EOF
+ {
+ echo
+ echo "Each evidence link opens its owning lane artifact; the row text is the exact \`recordings/<cell-label>-pid<pid>-<sequence>/recording.mp4\` path, with an adjacent \`trajectory.json\`."
+ } >> "$summary_path"
+ cat "$summary_path" >> "$GITHUB_STEP_SUMMARY"
+ - name: Upload matrix summary
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: rust-linux-matrix-summary
+ path: matrix-summary.md
+ if-no-files-found: error
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index fda7abab4f..9e0f6a1a31 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -1973,9 +1973,11 @@ dependencies = [
"tracing",
"tracing-subscriber",
"url",
+ "wayland-backend",
"wayland-client",
"wayland-protocols",
"wayland-protocols-wlr",
+ "wayland-scanner",
"x11",
"x11rb",
"xkbcommon",
diff --git a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md
index 24cd6906b1..ed0d3ef37c 100644
--- a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md
+++ b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md
@@ -67,8 +67,9 @@ keystroke / XSendEvent / XTest / foreground rungs are not read-back-confirmed
**`effect` / `escalation`** — alongside `verified`, action responses carry the
cross-platform `effect` (`confirmed` / `unverifiable` / `suspected_noop`) and,
when you should change rung, `escalation:{recommended, reason}`. See `SKILL.md`
-→ behavior matrix. The Linux-specific value of `recommended` is **`foreground`
-on Wayland** (background pixel can't target an unfocused window) and
+→ behavior matrix. On a standard Wayland compositor the Linux-specific value
+of `recommended` is **`foreground`** (raw background pixels cannot target an
+unfocused window); the opt-in nested compositor is a separate environment. Use
**`px` on X11** (an element px action — background pixel click — lands via
AT-SPI `do_action`-at-point off the screenshot already in the snapshot — the
matrix below).
@@ -101,8 +102,9 @@ Modality is chosen at **action time**, by how you address the target:
`get_window_state` returning `degraded:true` (empty AT-SPI walk) is the cue to
do an **element px action** off that same screenshot (X11) or escalate to
-`delivery_mode:"foreground"` (Wayland — background pixel can't target an
-unfocused window there).
+`delivery_mode:"foreground"` (standard Wayland: raw background pixels cannot
+target an unfocused window there). The nested compositor has its own
+experimental per-surface routes.
## Cross-platform schema residuals (Linux)
@@ -180,42 +182,37 @@ uinput, escalate to `delivery_mode:"foreground"`. (`type_text` in the
`background` rung is focus-dependent for non-editable widgets; that's the one
genuine background limitation, and `foreground` is the documented escalation.)
-## Wayland (opt-in — still preview)
-
-Native Wayland support is behind an opt-in flag and not yet at the same
-maturity as X11:
-
-- `CUA_DRIVER_RS_ENABLE_WAYLAND=1` enables the native-Wayland backend.
- On wlroots compositors (labwc/sway) it uses foreign-toplevel +
- screencopy; on GNOME-Mutter / KDE-KWin, which expose no client
- protocols for cross-window enumeration, cua-driver brings its own
- nested compositor (`CUA_WAYLAND_NEST=1`).
-- Without the flag, on a Wayland session the driver falls back to
- XWayland where available.
-- Screenshots work via `grim` / wlr-screencopy. **Video recording is
- not yet available on Wayland** — the recorder is `x11grab`, which is
- X11-only.
-- **`delivery_mode` on Wayland is constrained by the protocol**, honestly:
- input goes through libei + xdg-desktop-portal, which injects to the
- **compositor's input focus** — there is no per-window background targeting
- like X11/macOS/Windows. So `background` can't aim at a specific non-focused
- window, and `bring_to_front` has no standalone external activate (the
- compositor bundles activation into the virtual-pointer/click path) — it
- returns `bring_to_front_wayland_bundled` and points you at
- `delivery_mode:"foreground"` on the input call instead. The delivery
- contract also defines a structured `background_unavailable` error for the
- no-libei-backend case (built without `portal-libei` or a denied portal
- session); when input has no actuator the tools surface an error rather
- than silently succeeding.
-- **Wayland escalation is `foreground`, NOT `px`.** This is the
- one place the cross-platform escalation hint flips. Everywhere else
- (macOS, X11, most Windows surfaces) a background pixel click can
- land on an unfocused window, so `escalation.recommended` is `px`.
- On Wayland an unfocused window **cannot** be pixel-targeted in the
- background (libei injects only to compositor focus → `background_unavailable`),
- so the hint on action responses and on a `degraded` `get_window_state` is
- `foreground`: re-call the action with `delivery_mode:"foreground"`, don't
- expect an element px action off the screenshot to land in the background.
+## Wayland (opt-in)
+
+Set `CUA_DRIVER_RS_ENABLE_WAYLAND=1` to enable native Wayland support. The
+driver selects a backend from compositor capabilities:
+
+- Sway and other wlroots compositors use foreign-toplevel discovery,
+ wlr-screencopy, virtual pointer, and virtual keyboard protocols.
+- GNOME/Mutter uses the bundled WinRects Shell helper for target geometry and
+ activation, plus portal/libei for foreground raw input.
+- KDE/KWin uses AT-SPI and portal facilities where available. Target-specific
+ foreground activation remains experimental, so unsafe raw input refuses.
+- The optional `cua-compositor` is a separate nested session enabled
+ explicitly for controlled automation. GNOME and KDE never switch into it.
+
+Sway recording works through the wlroots recorder path and is exercised by the
+canonical harness runner. Portal-backed GNOME recording is still an evidence
+gap. Capture and recording availability therefore depend on the compositor,
+installed helpers, and portal grant.
+
+Standard Wayland has no general client protocol for raw input to an arbitrary
+occluded surface. Background AX actions can still deliver through AT-SPI, and
+a PX left click can deliver when hit-testing resolves to an actionable AT-SPI
+control. Other focus-bound background pointer and keyboard shapes return an
+exact `background_unavailable` result. They do not report success after a
+silent drop.
+
+Use `delivery_mode:"foreground"` for raw Wayland input. The driver activates
+the selected target through a verified compositor adapter before dispatch. If
+the compositor has no target-addressable activation or input backend, the call
+refuses before sending input. Reconstructing coordinates alone does not make
+raw background PX possible on a standard compositor.
## Quick triage
@@ -251,17 +248,13 @@ ask the user.
## What to expect
-| Intent | Status |
-|---|---|
-| Snapshot AT-SPI tree | ✅ GTK3/4, Qt5/6, wxWidgets, Electron (GTK4/Qt6 can be partial — re-snapshot) |
-| Pixel click | ✅ background `XSendEvent`, no focus steal, no pointer move |
-| Element-indexed click | ✅ AT-SPI `do_action` |
-| Type text | ✅ AT-SPI focus-free, with XTest / pty fallback for the focused widget |
-| Hotkey / `press_key` | ✅ |
-| Screenshot full-display | ✅ X11; ✅ Wayland via `grim` |
-| Screenshot per-window | ✅ X11 |
-| `launch_app` | ✅ env-scrubbed launch (no workspace steal) |
-| Recording (video) | ✅ X11 (`x11grab`); ⚠️ Wayland not yet supported (preview) |
+| Environment | Proven baseline | Main limits |
+|---|---|---|
+| X11/Openbox | AT-SPI trees and actions, foreground pointer and keyboard input, window and desktop capture, and video | Raw background delivery remains toolkit-specific; unsupported shapes refuse |
+| Sway/wlroots | AT-SPI, native discovery, full-display and cropped-window screencopy, foreground input, semantic background actions, and video | Raw background pointer and keyboard input remains focus-bound |
+| GNOME/Mutter | AT-SPI, WinRects geometry and activation, capture, and portal/libei foreground input | Requires the helper and portal grant; portal video parity remains open |
+| KDE/KWin | AT-SPI and generic discovery where exposed | Target-specific activation and behavioral coverage remain experimental |
+| Nested `cua-compositor` | Versioned direct per-surface input, native GTK 31/31, capture/scope 5/5, and partial Electron coverage | The complete shared matrix remains experimental; do not infer standard-Wayland support |
See `SKILL.md` for the cross-platform loop (snapshot-before-AND-after,
pixel-click contract, failure modes) and `RECORDING.md` for session
diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
index ecbbb9ec1f..f915a83249 100644
--- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
+++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
@@ -61,10 +61,11 @@ platform-linux = { path = "../platform-linux" }
[features]
default = []
-# Forwards `platform-linux/portal-libei` so the Nix build can flip the
-# GNOME/KDE portal stack (PipeWire ScreenCast + libei RemoteDesktop) on
-# from a single workspace-level flag while the cross-platform release CD
-# leaves it off. No-op on macOS/Windows builds.
+# GNOME/KDE portal input ships in ordinary Linux releases; PipeWire capture
+# remains a separate modern-desktop feature. The combined name stays as a
+# compatibility alias for Nix and downstream builds.
+portal-input = ["platform-linux/portal-input"]
+portal-capture = ["platform-linux/portal-capture"]
portal-libei = ["platform-linux/portal-libei"]
[target.'cfg(target_os = "windows")'.build-dependencies]
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs
index b44946646e..8c727cd327 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs
@@ -774,11 +774,11 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core::
{
cua_driver_core::recording::set_screenshot_fn(|window_id, pid| {
if let Some(xid) = window_id {
- platform_linux::capture::screenshot_window_bytes(xid).ok()
+ platform_linux::wayland::screenshot_dispatch(xid).ok()
} else if let Some(p) = pid {
- let wins = platform_linux::x11::list_windows(Some(p as u32));
+ let wins = platform_linux::wayland::list_windows_dispatch(Some(p as u32));
wins.first().and_then(|w| {
- platform_linux::capture::screenshot_window_bytes(w.xid).ok()
+ platform_linux::wayland::screenshot_dispatch(w.xid).ok()
})
} else {
platform_linux::capture::screenshot_display_bytes().ok()
@@ -787,9 +787,21 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core::
cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| {
platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()
});
- cua_driver_core::video::set_video_backend_factory(
- Box::new(cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory),
- );
+ cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
+ platform_linux::recording_hooks::app_state_json_for(window_id, pid)
+ });
+ cua_driver_core::recording::set_element_bounds_fn(|wid, pid, idx| {
+ platform_linux::recording_hooks::element_window_local_xy(wid, pid, idx)
+ });
+ if platform_linux::wayland::is_wayland() {
+ cua_driver_core::video::set_video_backend_factory(Box::new(
+ platform_linux::video_wayland::WfRecorderVideoBackendFactory,
+ ));
+ } else {
+ cua_driver_core::video::set_video_backend_factory(Box::new(
+ cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
+ ));
+ }
// SSH-driven Wayland+Xwayland sessions inherit DISPLAY but not
// XAUTHORITY; adopt the running X server's auth cookie so X11 tools
// don't all fail "Authorization required" (#1926). No-op when
@@ -806,6 +818,9 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core::
// so their AT-SPI trees are visible to get_window_state. Best-effort and
// idempotent; only on the serve path, not for short-lived CLI calls.
platform_linux::a11y::ensure_chromium_accessibility_enabled();
+ if let Err(error) = platform_linux::atspi::ensure_listener_active() {
+ tracing::warn!("could not activate the persistent AT-SPI listener: {error}");
+ }
{ let mut r = platform_linux::register_tools_with_cursor(cursor_cfg, compat); check_update_tool::register_into(&mut r); r }
}
#[cfg(not(any(target_os = "windows", target_os = "linux")))]
@@ -851,13 +866,19 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry {
}
#[cfg(target_os = "linux")]
{
+ platform_linux::xauth::ensure_xauthority_discovered();
+ platform_linux::session_bus::ensure_session_bus_discovered();
+ platform_linux::a11y::ensure_chromium_accessibility_enabled();
+ if let Err(error) = platform_linux::atspi::ensure_listener_active() {
+ tracing::warn!("could not activate the persistent AT-SPI listener: {error}");
+ }
cua_driver_core::recording::set_screenshot_fn(|window_id, pid| {
if let Some(xid) = window_id {
- platform_linux::capture::screenshot_window_bytes(xid).ok()
+ platform_linux::wayland::screenshot_dispatch(xid).ok()
} else if let Some(p) = pid {
- let wins = platform_linux::x11::list_windows(Some(p as u32));
+ let wins = platform_linux::wayland::list_windows_dispatch(Some(p as u32));
wins.first().and_then(|w| {
- platform_linux::capture::screenshot_window_bytes(w.xid).ok()
+ platform_linux::wayland::screenshot_dispatch(w.xid).ok()
})
} else {
platform_linux::capture::screenshot_display_bytes().ok()
@@ -866,9 +887,21 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry {
cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| {
platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok()
});
- cua_driver_core::video::set_video_backend_factory(
- Box::new(cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory),
- );
+ cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| {
+ platform_linux::recording_hooks::app_state_json_for(window_id, pid)
+ });
+ cua_driver_core::recording::set_element_bounds_fn(|wid, pid, idx| {
+ platform_linux::recording_hooks::element_window_local_xy(wid, pid, idx)
+ });
+ if platform_linux::wayland::is_wayland() {
+ cua_driver_core::video::set_video_backend_factory(Box::new(
+ platform_linux::video_wayland::WfRecorderVideoBackendFactory,
+ ));
+ } else {
+ cua_driver_core::video::set_video_backend_factory(Box::new(
+ cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory,
+ ));
+ }
{
let mut r = platform_linux::register_tools_with_cursor(
cursor_overlay::CursorConfig { enabled: false, ..Default::default() },
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
index f4f1236ac9..e1944140f6 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs
@@ -355,14 +355,51 @@ fn launch_host_with_evidence(spec: &HostSpec, scenario: &str, evidence: &mut Evi
}
fn snapshot(fixture: &mut Fixture) -> ToolResponse {
- fixture.driver.call(
+ let state = fixture.driver.call(
"get_window_state",
serde_json::json!({
"pid": fixture.pid as i64,
"window_id": fixture.wid,
"capture_mode": "ax"
}),
- )
+ );
+ refresh_wayland_window_geometry(fixture);
+ state
+}
+
+#[cfg(target_os = "linux")]
+fn refresh_wayland_window_geometry(fixture: &mut Fixture) {
+ if std::env::var_os("WAYLAND_DISPLAY").is_none() {
+ return;
+ }
+ // A compositor may place a newly mapped toplevel after its first
+ // list_windows appearance. Re-read the current compositor-owned geometry
+ // for every snapshot, matching the native GTK harness, instead of turning
+ // that launch-time placement into a stale PX coordinate origin.
+ let windows = fixture.driver.call(
+ "list_windows",
+ serde_json::json!({ "pid": fixture.pid as i64 }),
+ );
+ let Some(window) = windows.structured()["windows"]
+ .as_array()
+ .and_then(|items| {
+ items
+ .iter()
+ .find(|window| window["pid"].as_u64() == Some(fixture.pid as u64))
+ })
+ else {
+ return;
+ };
+ if let Some(window_id) = window["window_id"].as_u64() {
+ fixture.wid = window_id;
+ }
+ let bounds = &window["bounds"];
+ fixture.window_x = bounds["x"].as_f64().unwrap_or(fixture.window_x);
+ fixture.window_y = bounds["y"].as_f64().unwrap_or(fixture.window_y);
+}
+
+#[cfg(not(target_os = "linux"))]
+fn refresh_wayland_window_geometry(_fixture: &mut Fixture) {
}
fn window_origin(fixture: &Fixture, _state: &ToolResponse) -> (f64, f64) {
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs
new file mode 100644
index 0000000000..a7c887f167
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs
@@ -0,0 +1,277 @@
+//! Linux **desktop-scope** (vision/foreground) modality through the SAME
+//! cua-driver interface as Windows/macOS: `set_config capture_scope=desktop` +
+//! a window-less screen-absolute `click` (no `pid`/`window_id`/`list_windows`).
+//! The Linux actuator warps the pointer and injects a real button press via the
+//! XTest extension — the peer of the Windows `WindowFromPoint` + macOS
+//! global-HID desktop click. (XTest delivering to the under-pointer window is
+//! why the *background* paths use `XSendEvent`; for desktop scope that delivery
+//! is exactly what we want.)
+//!
+//! Grounds the click on the GTK3 harness increment button's screen-absolute
+//! `frame` (AT-SPI Component extents), asserts the counter advanced, plus the
+//! window-scope rejection gate.
+//!
+//! Linux config is global-only (no per-session override), so `set_config`
+//! capture_scope writes the on-disk default — the test resets it to `window`
+//! before asserting so a failure can't leave the sandbox in desktop scope.
+//!
+//! #[ignore] (needs an X11/Xwayland display + AT-SPI + the GTK3 harness). Run:
+//! cargo test -p cua-driver --test desktop_scope_linux_test -- --ignored --nocapture --test-threads=1
+
+#![cfg(target_os = "linux")]
+
+use std::panic::{self, AssertUnwindSafe};
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+use cua_driver_testkit::e2e::{
+ execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation,
+ OracleKind, Scope, Targeting,
+};
+use cua_driver_testkit::{harness_app, Driver, McpDriver};
+
+fn harness_exe() -> std::path::PathBuf {
+ std::env::var("HARNESS_GTK3_EXE")
+ .map(std::path::PathBuf::from)
+ .ok()
+ .filter(|p| p.exists())
+ .unwrap_or_else(|| harness_app("harness-gtk3", "CuaTestHarness.Gtk3"))
+}
+
+fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
+ let exe = harness_exe();
+ if !exe.exists() {
+ if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() {
+ panic!("required GTK3 harness is missing at {exe:?}");
+ }
+ eprintln!("[desktop-linux] GTK3 harness not built ({exe:?}) — run tests/fixtures/build/linux.sh; skipping");
+ return None;
+ }
+ let launched = driver.reaper().spawn(
+ Command::new(&exe)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null()),
+ );
+ if let Err(error) = launched {
+ if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() {
+ panic!("failed to launch required GTK3 harness {exe:?}: {error}");
+ }
+ eprintln!("[desktop-linux] GTK3 harness launch failed: {error}; skipping");
+ return None;
+ }
+ let deadline = Instant::now() + Duration::from_secs(14);
+ while Instant::now() < deadline {
+ let r = driver.call("list_windows", serde_json::json!({}));
+ if let Some(wins) = r.structured()["windows"].as_array() {
+ for w in wins {
+ if w["title"]
+ .as_str()
+ .unwrap_or("")
+ .contains("CuaTestHarness GTK3")
+ {
+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;
+ let wid = w["window_id"].as_u64().unwrap_or(0);
+ if pid != 0 && wid != 0 {
+ driver.reaper().track_pid(pid);
+ return Some((pid, wid));
+ }
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(400));
+ }
+ if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() {
+ panic!("required GTK3 harness window never appeared");
+ }
+ eprintln!("[desktop-linux] harness window never appeared — graphical session + AT-SPI available? skipping");
+ None
+}
+
+fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value {
+ driver
+ .call(
+ "get_window_state",
+ serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }),
+ )
+ .structured()
+ .clone()
+}
+
+/// Screen-absolute center (px) of the increment button from `elements[].frame`
+/// (AT-SPI Component extents are screen-absolute). The GTK3 harness sets the
+/// button's accessible NAME to `btn-increment`; match any element whose blob
+/// carries that aid and has a frame, so we're robust to the exact field name.
+fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
+ let els = snap["elements"].as_array()?;
+ let btn = els.iter().find(|e| {
+ serde_json::to_string(e)
+ .map(|s| s.contains("btn-increment"))
+ .unwrap_or(false)
+ && e.get("frame").map(|f| f.is_object()).unwrap_or(false)
+ })?;
+ let f = &btn["frame"];
+ let (x, y, w, h) = (
+ f["x"].as_f64()?,
+ f["y"].as_f64()?,
+ f["w"].as_f64()?,
+ f["h"].as_f64()?,
+ );
+ Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))
+}
+
+fn counter(snap: &serde_json::Value) -> Option<u64> {
+ let tree = snap["tree_markdown"].as_str()?;
+ let idx = tree.find("counter=")? + "counter=".len();
+ let digits: String = tree[idx..]
+ .chars()
+ .take_while(|c| c.is_ascii_digit())
+ .collect();
+ digits.parse().ok()
+}
+
+fn set_scope(driver: &mut McpDriver, scope: &str) {
+ let r = driver.call(
+ "set_config",
+ serde_json::json!({ "key": "capture_scope", "value": scope }),
+ );
+ assert!(
+ !r.is_error(),
+ "set_config capture_scope={scope} failed: {}",
+ r.text()
+ );
+}
+
+fn with_desktop_scope<R>(driver: &mut McpDriver, test: impl FnOnce(&mut McpDriver) -> R) -> R {
+ set_scope(driver, "desktop");
+ let result = panic::catch_unwind(AssertUnwindSafe(|| test(driver)));
+ set_scope(driver, "window");
+ match result {
+ Ok(value) => value,
+ Err(payload) => panic::resume_unwind(payload),
+ }
+}
+
+// ── tests ───────────────────────────────────────────────────────────────────────
+
+/// In desktop scope, a window-less screen-absolute click (no pid/window_id)
+/// lands on the increment button — its counter advances.
+#[test]
+#[ignore]
+fn desktop_scope_windowless_click_lands_on_control() {
+ let cell_id = "linux-gtk3-desktop-left-click-px-foreground";
+ let case = CaseSpec::delivered(
+ cell_id,
+ "gtk3",
+ "gtk3",
+ "left_click",
+ Targeting::Px,
+ Delivery::Foreground,
+ Scope::Desktop,
+ if std::env::var_os("CUA_INJECT_SOCKET").is_some() {
+ DriverRoute::LinuxCuaCompositorInject
+ } else {
+ DriverRoute::LinuxXTest
+ },
+ vec![OracleKind::FixtureState],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");
+ *evidence = recording_evidence(driver.recording_dir());
+ let (pid, wid) = launch(&mut driver).expect("required GTK3 harness did not launch");
+
+ // Settle for the AT-SPI tree to register the button + its extents.
+ let mut snap = ax_snapshot(&mut driver, pid, wid);
+ let mut center = increment_center(&snap);
+ let deadline = Instant::now() + Duration::from_secs(8);
+ while center.is_none() && Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(400));
+ snap = ax_snapshot(&mut driver, pid, wid);
+ center = increment_center(&snap);
+ }
+ let Some((cx, cy)) = center else {
+ panic!("increment button frame not found in required GTK3 AT-SPI tree");
+ };
+ let pre = counter(&snap).unwrap_or(0);
+ println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}");
+ let posture = driver.call(
+ "bring_to_front",
+ serde_json::json!({"pid": pid as i64, "window_id": wid}),
+ );
+ assert!(!posture.is_error(), "could not foreground GTK3 fixture: {}", posture.text());
+ std::thread::sleep(Duration::from_millis(300));
+ driver.start_behavior_recording();
+
+ // Retry the window-less desktop click until the counter advances. A
+ // freshly-mapped harness window may not yet be raised under the pointer on
+ // the first click (X11 window-raise timing differs across WMs — XFCE/Openbox
+ // lag GNOME), so the screen-absolute XTest click can miss the first attempt.
+ // Re-issuing the SAME click is safe: extra landed clicks only increment the
+ // counter further, and `post > pre` still holds. We assert the click was
+ // *dispatched as desktop scope* on the first attempt, and that it eventually
+ // *lands* within the budget.
+ let (post, first_text) = with_desktop_scope(&mut driver, |driver| {
+ let mut post = pre;
+ let mut first_text = String::new();
+ for attempt in 0..12 {
+ let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));
+ if attempt == 0 {
+ first_text = clicked.text().to_string();
+ assert!(
+ !clicked.is_error(),
+ "desktop-scope click errored: {}",
+ clicked.text()
+ );
+ }
+ std::thread::sleep(Duration::from_millis(500));
+ post = counter(&ax_snapshot(driver, pid, wid)).unwrap_or(pre);
+ if post > pre {
+ break;
+ }
+ }
+ (post, first_text)
+ });
+
+ assert!(
+ first_text.to_lowercase().contains("desktop scope"),
+ "click not reported as desktop-scope: {first_text}"
+ );
+ assert!(
+ post > pre,
+ "counter did not advance after window-less desktop clicks: pre={pre} post={post} \
+ (the harness window never became clickable at ({cx},{cy}) within the retry budget)"
+ );
+ Observation::delivered_with_fixture_state(Vec::new())
+ });
+}
+
+/// Negative gate: a window-less click under `capture_scope=window` is rejected.
+#[test]
+#[ignore]
+fn window_scope_rejects_windowless_click() {
+ let cell_id = "linux-window-scope-gate-px-not-applicable";
+ let case = CaseSpec::delivered(
+ cell_id,
+ "desktop",
+ "x11",
+ "window_scope_gate",
+ Targeting::Px,
+ Delivery::NotApplicable,
+ Scope::Window,
+ DriverRoute::Composite,
+ vec![OracleKind::Protocol],
+ );
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver");
+ *evidence = recording_evidence(driver.recording_dir());
+ set_scope(&mut driver, "window");
+ driver.start_behavior_recording();
+ let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100 }));
+ assert!(
+ r.is_error()
+ && r.structured()["code"].as_str() == Some("desktop_scope_disabled"),
+ "window-scope window-less click was NOT rejected: {}",
+ r.text()
+ );
+ Observation::delivered(vec![OracleKind::Protocol], Evidence::default())
+ });
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
index 9be13a143c..54ae8ba0db 100644
--- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs
@@ -1,255 +1,1090 @@
-//! Integration test against the CuaTestHarness.Gtk3 (PyGObject / GTK3) app —
-//! the Linux peer of `harness_appkit_test` / `harness_swiftui_test` (macOS) and
-//! `harness_wpf_test` / `harness_winui3_test` (Windows). Brings Linux to parity
-//! with the other platforms' controlled-harness coverage (alongside the real-app
-//! Nix GUI scenarios).
+//! Evidence-bearing native GTK3 harness catalog for Linux.
//!
-//! ## The Linux id contract: NAME, not id
-//! cua-driver's Linux `get_window_state` renders each element as
-//! `[idx] <role> "<name>"` from AT-SPI — there is NO `id=` field like Windows
-//! (UIA AutomationId) or macOS (AX identifier). So the harness sets each
-//! actionable control's AT-SPI **accessible name** to the scenario aid
-//! (`btn-increment`, …), and this test matches on name via
-//! `ax::element_index_containing` (substring), not `ax::has_id`.
+//! GTK exposes accessibility through ATK -> AT-SPI. The Linux snapshot has no
+//! AutomationId/AXIdentifier field, so actionable controls publish their
+//! scenario id as their accessible name and this test addresses them by name.
//!
-//! Requires a real Linux desktop with AT-SPI running + a display (Xwayland is
-//! fine — the launcher forces `GDK_BACKEND=x11`), GTK3 + PyGObject, and the app
-//! built via `tests/fixtures/build/linux.sh`. `#[ignore]`; run explicitly:
+//! Requires a Linux desktop, AT-SPI, GTK3, PyGObject, and the fixture built by
+//! `tests/fixtures/build/linux.sh`. The canonical lane runs this target with:
//! cargo test -p cua-driver --test harness_gtk3_test -- --ignored --nocapture --test-threads=1
#![cfg(target_os = "linux")]
use std::process::{Command, Stdio};
-use std::time::Duration;
+use std::time::{Duration, Instant};
-use cua_driver_testkit::{ax, harness_app, Driver, McpDriver};
+use cua_driver_testkit::e2e::{
+ execute_case, native_background_case, native_foreground_case, native_readonly_case,
+ recording_evidence, CaseSpec, Delivery, DisplayServer, DriverRoute, Evidence, Observation,
+ OracleKind, RefusalCode, Targeting,
+};
+use cua_driver_testkit::observer::TargetWindow;
+use cua_driver_testkit::sentinel::run_with_background_oracles;
+use cua_driver_testkit::{ax, harness_app, Driver, McpDriver, ToolResponse};
fn harness_exe() -> std::path::PathBuf {
- if let Ok(p) = std::env::var("HARNESS_GTK3_EXE") {
- let pb = std::path::PathBuf::from(p);
- if pb.exists() {
- return pb;
+ if let Ok(path) = std::env::var("HARNESS_GTK3_EXE") {
+ let path = std::path::PathBuf::from(path);
+ if path.exists() {
+ return path;
}
}
harness_app("harness-gtk3", "CuaTestHarness.Gtk3")
}
-/// Launch the GTK3 harness, tying its lifetime to the driver's reaper, and
-/// return (pid, main window_id). Returns None (skip) if the app isn't built.
-fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
+fn launch(driver: &mut McpDriver) -> (u32, u64) {
let exe = harness_exe();
- if !exe.exists() {
- eprintln!("GTK3 harness not built at {exe:?} — run tests/fixtures/build/linux.sh");
- return None;
- }
+ assert!(exe.exists(), "required GTK3 harness is missing: {exe:?}");
driver
.reaper()
.spawn(
Command::new(&exe)
- .stdout(Stdio::null())
- .stderr(Stdio::null()),
+ .stdout(Stdio::inherit())
+ .stderr(Stdio::inherit()),
)
- .ok()?;
- // GTK app cold-start + window map + AT-SPI registration.
- std::thread::sleep(Duration::from_millis(1500));
-
- // Resolve the harness window by title. find_window needs the pid; the
- // launcher execs python3 in-place so the spawned pid IS the GTK process.
- // We don't have that pid directly here (reaper owns the child), so discover
- // by title across list_windows.
- let deadline = std::time::Instant::now() + Duration::from_secs(12);
- while std::time::Instant::now() < deadline {
- let r = driver.call("list_windows", serde_json::json!({}));
- if let Some(wins) = r.structured()["windows"].as_array() {
- for w in wins {
- if w["title"]
+ .unwrap_or_else(|error| panic!("launch GTK3 harness {exe:?}: {error}"));
+
+ let deadline = Instant::now() + Duration::from_secs(12);
+ while Instant::now() < deadline {
+ let response = driver.call("list_windows", serde_json::json!({}));
+ if let Some(windows) = response.structured()["windows"].as_array() {
+ for window in windows {
+ if window["title"]
.as_str()
.unwrap_or("")
.contains("CuaTestHarness GTK3")
{
- let pid = w["pid"].as_u64().unwrap_or(0) as u32;
- let wid = w["window_id"].as_u64().unwrap_or(0);
- if pid != 0 && wid != 0 {
+ let pid = window["pid"].as_u64().unwrap_or(0) as u32;
+ let window_id = window["window_id"].as_u64().unwrap_or(0);
+ if pid != 0 && window_id != 0 {
driver.reaper().track_pid(pid);
- return Some((pid, wid));
+ std::thread::sleep(Duration::from_millis(500));
+ return (pid, window_id);
}
}
}
}
std::thread::sleep(Duration::from_millis(400));
}
- eprintln!("GTK3 harness window never appeared — is a graphical session + AT-SPI available?");
- None
+ panic!("required GTK3 harness window never appeared");
}
-fn snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> String {
- driver
- .call(
- "get_window_state",
- serde_json::json!({ "pid": pid as i64, "window_id": wid }),
- )
- .text()
- .to_string()
+fn snapshot(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolResponse {
+ driver.call(
+ "get_window_state",
+ serde_json::json!({ "pid": pid as i64, "window_id": window_id }),
+ )
}
-fn ax_empty(text: &str) -> bool {
- ax::looks_empty(text) || text.contains("D-Bus") || text.contains("AT-SPI")
+fn element_index(state: &ToolResponse, name: &str) -> u64 {
+ ax::element_index_containing(state.tree_text(), name).unwrap_or_else(|| {
+ panic!(
+ "{name:?} not found in GTK3 AT-SPI tree:\n{}",
+ state.tree_text()
+ )
+ })
}
-#[test]
-#[ignore]
-fn harness_gtk3_smoke() {
- let Some(mut driver) = McpDriver::spawn() else {
- return;
- };
- let Some((pid, wid)) = launch(&mut driver) else {
- return;
- };
- println!("gtk3 harness pid={pid} wid={wid}");
-
- let text = snapshot(&mut driver, pid, wid);
- if ax_empty(&text) {
- eprintln!("AT-SPI tree empty — accessibility bus unavailable; skipping element assertions");
- return;
- }
- println!("snapshot:\n{text}");
-
- // Actionable controls expose their aid as the AT-SPI accessible name.
- for name in [
- "btn-increment",
- "btn-reset",
- "txt-input",
- "btn-open-popover",
- "btn-exit",
- ] {
- assert!(
- text.contains(name),
- "missing control named {name:?} in GTK3 AT-SPI tree"
- );
- }
- // Marker label + counter label carry their text as the accessible name.
- assert!(
- text.contains("HARNESS_TEXT_MARKER_v1"),
- "text_body marker not in tree"
- );
+fn element_rect(
+ driver: &mut McpDriver,
+ pid: u32,
+ window_id: u64,
+ state: &ToolResponse,
+ name: &str,
+) -> (f64, f64, f64, f64) {
+ let index = element_index(state, name);
+ let elements = state.structured()["elements"]
+ .as_array()
+ .expect("PX targeting requires structured GTK3 elements");
+ let target = elements
+ .iter()
+ .find(|element| element["element_index"].as_u64() == Some(index))
+ .and_then(|element| element["frame"].as_object())
+ .unwrap_or_else(|| panic!("GTK3 element [{index}] {name:?} has no frame"));
+ let windows = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 }));
+ let window = windows.structured()["windows"]
+ .as_array()
+ .and_then(|windows| {
+ windows
+ .iter()
+ .find(|window| window["window_id"].as_u64() == Some(window_id))
+ })
+ .unwrap_or_else(|| panic!("GTK3 window {window_id} disappeared before PX targeting"));
+
+ let window_width = window["width"].as_f64().unwrap_or(0.0);
+ let window_height = window["height"].as_f64().unwrap_or(0.0);
+ let screenshot_width = state.structured()["screenshot_width"]
+ .as_f64()
+ .expect("PX targeting requires screenshot_width");
+ let screenshot_height = state.structured()["screenshot_height"]
+ .as_f64()
+ .expect("PX targeting requires screenshot_height");
assert!(
- text.contains("counter=0"),
- "initial counter label not in tree"
+ window_width > 0.0 && window_height > 0.0,
+ "GTK3 top-level frame has invalid geometry: {window:?}"
);
+ let scale_x = screenshot_width / window_width;
+ let scale_y = screenshot_height / window_height;
+ let x = (target["x"].as_f64().unwrap_or(0.0) - window["x"].as_f64().unwrap_or(0.0)) * scale_x;
+ let y = (target["y"].as_f64().unwrap_or(0.0) - window["y"].as_f64().unwrap_or(0.0)) * scale_y;
+ let width = target["w"].as_f64().unwrap_or(0.0) * scale_x;
+ let height = target["h"].as_f64().unwrap_or(0.0) * scale_y;
assert!(
- text.contains("popover_open=False"),
- "initial popover state not in tree"
+ width > 0.0
+ && height > 0.0
+ && x + width / 2.0 >= 0.0
+ && y + height / 2.0 >= 0.0
+ && x + width / 2.0 < screenshot_width
+ && y + height / 2.0 < screenshot_height,
+ "GTK3 PX target {name:?} is outside the capture: rect=({x},{y},{width},{height}) capture=({screenshot_width},{screenshot_height})"
);
+ (x, y, width, height)
+}
- println!("✅ harness_gtk3_smoke: all scenarios present in AT-SPI tree");
+fn wait_for_state(driver: &mut McpDriver, pid: u32, window_id: u64, expected: &str) -> String {
+ let deadline = Instant::now() + Duration::from_secs(4);
+ let mut last = String::new();
+ while Instant::now() < deadline {
+ last = snapshot(driver, pid, window_id).tree_text().to_owned();
+ if last.contains(expected) {
+ return last;
+ }
+ std::thread::sleep(Duration::from_millis(200));
+ }
+ let relevant = last
+ .lines()
+ .filter(|line| line.contains('=') || line.contains("MARKER"))
+ .collect::<Vec<_>>()
+ .join(" / ");
+ panic!("fixture state never contained {expected:?}; final state: {relevant}");
}
-#[test]
-#[ignore]
-fn harness_gtk3_counter_click() {
- let Some(mut driver) = McpDriver::spawn() else {
- return;
- };
- let Some((pid, wid)) = launch(&mut driver) else {
- return;
- };
+fn state_number(text: &str, key: &str) -> Option<u64> {
+ let start = text.find(key)? + key.len();
+ let digits = text[start..]
+ .chars()
+ .take_while(char::is_ascii_digit)
+ .collect::<String>();
+ digits.parse().ok()
+}
- let pre = snapshot(&mut driver, pid, wid);
- if ax_empty(&pre) {
- eprintln!("AT-SPI empty — skipping");
- return;
+fn wait_for_positive_state(driver: &mut McpDriver, pid: u32, window_id: u64, key: &str) -> String {
+ let deadline = Instant::now() + Duration::from_secs(4);
+ let mut last = String::new();
+ while Instant::now() < deadline {
+ last = snapshot(driver, pid, window_id).tree_text().to_owned();
+ if state_number(&last, key).is_some_and(|value| value > 0) {
+ return last;
+ }
+ std::thread::sleep(Duration::from_millis(200));
}
- let idx = match ax::element_index_containing(&pre, "btn-increment") {
- Some(i) => i,
- None => {
- eprintln!("btn-increment not found in tree — skipping");
+ panic!("fixture state {key:?} never became positive; final tree:\n{last}");
+}
+
+fn assert_popover_marker(driver: &mut McpDriver, pid: u32, main_window_id: u64) {
+ let deadline = Instant::now() + Duration::from_secs(4);
+ while Instant::now() < deadline {
+ if snapshot(driver, pid, main_window_id)
+ .tree_text()
+ .contains("POPOVER_MARKER_v1")
+ {
return;
}
- };
- let click = driver.call(
- "click",
- serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }),
- );
- println!("click [{idx}] btn-increment: {}", click.text());
- std::thread::sleep(Duration::from_millis(400));
+ let windows = driver.call("list_windows", serde_json::json!({}));
+ if let Some(windows) = windows.structured()["windows"].as_array() {
+ for window in windows {
+ let window_id = window["window_id"].as_u64().unwrap_or(0);
+ if window["pid"].as_u64() == Some(pid as u64)
+ && window_id != 0
+ && window_id != main_window_id
+ && snapshot(driver, pid, window_id)
+ .tree_text()
+ .contains("POPOVER_MARKER_v1")
+ {
+ return;
+ }
+ }
+ }
+ std::thread::sleep(Duration::from_millis(200));
+ }
+ panic!("GTK3 popover opened but POPOVER_MARKER_v1 was not exposed");
+}
+
+fn run_case(case: CaseSpec, test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation) {
+ let cell_id = case.cell_id.clone();
+ let delivery = case.delivery;
+ execute_case(case, |evidence| {
+ let mut driver = McpDriver::spawn_named(&cell_id).expect("start source-built Linux driver");
+ *evidence = recording_evidence(driver.recording_dir());
+ let (pid, window_id) = launch(&mut driver);
+ if delivery != Delivery::Background {
+ driver.start_behavior_recording();
+ }
+ test(pid, window_id, &mut driver)
+ });
+}
+
+#[derive(Clone, Copy, Debug)]
+enum Operation {
+ AxClick {
+ target: &'static str,
+ expected: &'static str,
+ },
+ AxTypeText {
+ target: &'static str,
+ text: &'static str,
+ expected: &'static str,
+ },
+ AxSetValue {
+ target: &'static str,
+ value: &'static str,
+ expected: &'static str,
+ },
+ PxClick {
+ target: &'static str,
+ button: &'static str,
+ count: u8,
+ expected: &'static str,
+ },
+ PxTypeText {
+ target: &'static str,
+ text: &'static str,
+ expected: &'static str,
+ },
+ PressKey {
+ key: &'static str,
+ expected: &'static str,
+ },
+ Hotkey {
+ expected: &'static str,
+ },
+ Scroll {
+ target: &'static str,
+ pixel: bool,
+ state_key: &'static str,
+ },
+ Drag {
+ target: &'static str,
+ state_key: &'static str,
+ },
+ Popover {
+ target: &'static str,
+ expected: &'static str,
+ },
+}
+
+#[derive(Clone, Copy, Debug)]
+struct CatalogRow {
+ action: &'static str,
+ targeting: Targeting,
+ delivery: Delivery,
+ route: DriverRoute,
+ operation: Operation,
+}
- let post = snapshot(&mut driver, pid, wid);
+fn assert_refusal_without_mutation(
+ response: &ToolResponse,
+ before: &str,
+ pid: u32,
+ window_id: u64,
+ driver: &mut McpDriver,
+) {
assert!(
- post.contains("counter=1"),
- "counter did not advance after clicking btn-increment. counter lines: {}",
- post.lines()
- .filter(|l| l.contains("counter="))
- .collect::<Vec<_>>()
- .join(" / ")
+ response.is_error(),
+ "expected background_unavailable refusal"
+ );
+ assert_eq!(
+ response.structured()["code"].as_str(),
+ Some("background_unavailable"),
+ "unexpected refusal: {}",
+ response.text()
+ );
+ std::thread::sleep(Duration::from_millis(200));
+ assert_eq!(
+ snapshot(driver, pid, window_id).tree_text(),
+ before,
+ "refused GTK3 action mutated fixture state"
);
- println!("✅ harness_gtk3_counter_click: counter advanced via AT-SPI click");
}
-#[test]
-#[ignore]
-fn harness_gtk3_popover() {
- let Some(mut driver) = McpDriver::spawn() else {
- return;
+fn invoke_operation(
+ row: CatalogRow,
+ pid: u32,
+ window_id: u64,
+ driver: &mut McpDriver,
+ expect_refusal: bool,
+) -> bool {
+ let mode = match row.delivery {
+ Delivery::Background => "background",
+ Delivery::Foreground => "foreground",
+ Delivery::NotApplicable => unreachable!("catalog operations require delivery"),
};
- let Some((pid, wid)) = launch(&mut driver) else {
- return;
- };
-
- let pre = snapshot(&mut driver, pid, wid);
- if ax_empty(&pre) {
- eprintln!("AT-SPI empty — skipping");
- return;
- }
+ let pre = snapshot(driver, pid, window_id);
assert!(
- pre.contains("popover_open=False"),
- "popover state not initially closed"
+ !ax::looks_empty(pre.tree_text()),
+ "required GTK3 AT-SPI tree is empty"
);
- let idx = match ax::element_index_containing(&pre, "btn-open-popover") {
- Some(i) => i,
- None => {
- eprintln!("btn-open-popover not found — skipping");
- return;
+ let (response, expected) = match row.operation {
+ Operation::AxClick { target, expected } => {
+ let index = element_index(&pre, target);
+ (
+ driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "element_index": index, "delivery_mode": mode
+ }),
+ ),
+ expected,
+ )
+ }
+ Operation::AxTypeText {
+ target,
+ text,
+ expected,
+ } => {
+ let index = element_index(&pre, target);
+ (
+ driver.call(
+ "type_text",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "element_index": index, "text": text, "delivery_mode": mode
+ }),
+ ),
+ expected,
+ )
+ }
+ Operation::AxSetValue {
+ target,
+ value,
+ expected,
+ } => {
+ let index = element_index(&pre, target);
+ (
+ driver.call(
+ "set_value",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "element_index": index, "value": value
+ }),
+ ),
+ expected,
+ )
+ }
+ Operation::PxClick {
+ target,
+ button,
+ count,
+ expected,
+ } => {
+ let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target);
+ let tool = if count == 2 {
+ "double_click"
+ } else if button == "right" {
+ "right_click"
+ } else {
+ "click"
+ };
+ let mut args = serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "x": x + width / 2.0, "y": y + height / 2.0,
+ "delivery_mode": mode
+ });
+ if tool == "click" {
+ args["button"] = serde_json::json!(button);
+ args["count"] = serde_json::json!(count);
+ }
+ (driver.call(tool, args), expected)
+ }
+ Operation::PxTypeText {
+ target,
+ text,
+ expected,
+ } => {
+ let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target);
+ (
+ driver.call(
+ "type_text",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "x": x + width / 2.0, "y": y + height / 2.0,
+ "text": text, "delivery_mode": mode
+ }),
+ ),
+ expected,
+ )
+ }
+ Operation::PressKey { key, expected } => (
+ driver.call(
+ "press_key",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "key": key, "delivery_mode": mode
+ }),
+ ),
+ expected,
+ ),
+ Operation::Hotkey { expected } => (
+ driver.call(
+ "hotkey",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "keys": ["ctrl", "shift", "k"], "delivery_mode": mode
+ }),
+ ),
+ expected,
+ ),
+ Operation::Scroll {
+ target,
+ pixel,
+ state_key,
+ } => {
+ let mut args = serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "direction": "down", "amount": 6, "delivery_mode": mode
+ });
+ if pixel {
+ let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target);
+ args["x"] = serde_json::json!(x + width / 2.0);
+ args["y"] = serde_json::json!(y + height / 2.0);
+ } else {
+ args["element_index"] = serde_json::json!(element_index(&pre, target));
+ }
+ let response = driver.call("scroll", args);
+ if expect_refusal {
+ assert_refusal_without_mutation(&response, pre.tree_text(), pid, window_id, driver);
+ return true;
+ }
+ assert!(
+ !response.is_error(),
+ "GTK3 scroll failed: {}",
+ response.text()
+ );
+ wait_for_positive_state(driver, pid, window_id, state_key);
+ return false;
+ }
+ Operation::Drag { target, state_key } => {
+ let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target);
+ let response = driver.call(
+ "drag",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "from_x": x + width * 0.05, "from_y": y + height / 2.0,
+ "to_x": x + width * 0.90, "to_y": y + height / 2.0,
+ "duration_ms": 500, "steps": 30, "delivery_mode": mode
+ }),
+ );
+ if expect_refusal {
+ assert_refusal_without_mutation(&response, pre.tree_text(), pid, window_id, driver);
+ return true;
+ }
+ assert!(
+ !response.is_error(),
+ "GTK3 drag failed: {}; raw={}",
+ response.text(),
+ response.raw
+ );
+ wait_for_positive_state(driver, pid, window_id, state_key);
+ return false;
+ }
+ Operation::Popover { target, expected } => {
+ let index = element_index(&pre, target);
+ let response = driver.call(
+ "click",
+ serde_json::json!({
+ "pid": pid as i64, "window_id": window_id,
+ "element_index": index, "delivery_mode": mode
+ }),
+ );
+ assert!(
+ !response.is_error(),
+ "GTK3 popover click failed: {}",
+ response.text()
+ );
+ wait_for_state(driver, pid, window_id, expected);
+ assert_popover_marker(driver, pid, window_id);
+ return false;
}
};
- let _ = driver.call(
- "click",
- serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }),
- );
- std::thread::sleep(Duration::from_millis(500));
- // GtkPopover may surface as a separate top-level; check the main window
- // first, then any new window of this pid.
- let post = snapshot(&mut driver, pid, wid);
+ if expect_refusal {
+ assert_refusal_without_mutation(&response, pre.tree_text(), pid, window_id, driver);
+ return true;
+ }
+
assert!(
- post.contains("popover_open=True"),
- "popover state did not flip open after click. Popover lines: {}",
- post.lines()
- .filter(|l| l.contains("popover_open="))
- .collect::<Vec<_>>()
- .join(" / ")
+ !response.is_error(),
+ "GTK3 {} {:?}/{:?} failed: {}",
+ row.action,
+ row.targeting,
+ row.delivery,
+ response.text()
);
- let mut found = post.contains("POPOVER_MARKER_v1");
- if !found {
- let r = driver.call("list_windows", serde_json::json!({}));
- if let Some(wins) = r.structured()["windows"].as_array() {
- for w in wins {
- if w["pid"].as_u64() == Some(pid as u64) {
- if let Some(other) = w["window_id"].as_u64() {
- if other != wid
- && snapshot(&mut driver, pid, other).contains("POPOVER_MARKER_v1")
- {
- found = true;
- break;
- }
- }
- }
+ wait_for_state(driver, pid, window_id, expected);
+ false
+}
+
+fn row_expects_refusal(row: CatalogRow) -> bool {
+ if row.delivery != Delivery::Background {
+ return false;
+ }
+ let inject_mode = std::env::var_os("CUA_INJECT_SOCKET").is_some();
+ if !inject_mode
+ && matches!(
+ row.operation,
+ Operation::PressKey { .. } | Operation::Hotkey { .. }
+ )
+ {
+ return true;
+ }
+ let focus_bound_pointer = matches!(
+ row.operation,
+ Operation::PxClick {
+ button: "right",
+ ..
+ } | Operation::PxClick { count: 2, .. }
+ | Operation::Scroll { pixel: true, .. }
+ | Operation::Drag { .. }
+ );
+ if DisplayServer::current() == DisplayServer::X11
+ && (focus_bound_pointer || matches!(row.operation, Operation::PxTypeText { .. }))
+ && !platform_linux::input::real_pointer_input_available()
+ {
+ return true;
+ }
+ DisplayServer::current() == DisplayServer::Wayland
+ && !inject_mode
+ && (focus_bound_pointer || matches!(row.operation, Operation::PxTypeText { .. }))
+}
+
+fn run_catalog_row(row: CatalogRow) {
+ let expect_refusal = row_expects_refusal(row);
+ let route = if DisplayServer::current() == DisplayServer::Wayland
+ && (row.targeting == Targeting::Px
+ || matches!(
+ row.operation,
+ Operation::PressKey { .. } | Operation::Hotkey { .. }
+ )) {
+ if std::env::var_os("CUA_INJECT_SOCKET").is_some() {
+ DriverRoute::LinuxCuaCompositorInject
+ } else {
+ DriverRoute::LinuxWaylandVirtualPointer
+ }
+ } else {
+ row.route
+ };
+ let case = match row.delivery {
+ Delivery::Background => native_background_case("gtk3", row.action, row.targeting, route),
+ Delivery::Foreground => native_foreground_case("gtk3", row.action, row.targeting, route),
+ Delivery::NotApplicable => unreachable!("catalog operations require delivery"),
+ };
+ let case = if expect_refusal {
+ case.expecting_refusal(vec![RefusalCode::BackgroundUnavailable])
+ } else {
+ case
+ };
+ run_case(case, |pid, window_id, driver| match row.delivery {
+ Delivery::Background => {
+ let mut refused = false;
+ let (_, passed) = run_with_background_oracles(
+ driver,
+ TargetWindow {
+ pid,
+ native_id: window_id,
+ },
+ |driver| {
+ refused = invoke_operation(row, pid, window_id, driver, expect_refusal);
+ },
+ )
+ .unwrap_or_else(|error| panic!("background desktop contract failed: {error}"));
+ if refused {
+ let mut passed = passed;
+ passed.push(OracleKind::FixtureState);
+ Observation::refused(
+ RefusalCode::BackgroundUnavailable,
+ passed,
+ "GTK3 focus-bound background input was refused",
+ Evidence::default(),
+ )
+ } else {
+ Observation::delivered_with_fixture_state(passed)
}
}
- }
- assert!(
- found,
- "popover body marker POPOVER_MARKER_v1 not found after open"
+ Delivery::Foreground => {
+ assert!(!invoke_operation(row, pid, window_id, driver, false));
+ Observation::delivered_with_fixture_state(Vec::new())
+ }
+ Delivery::NotApplicable => unreachable!(),
+ });
+}
+
+#[test]
+#[ignore]
+fn harness_gtk3_ax_tree() {
+ run_case(
+ native_readonly_case(
+ "gtk3",
+ "ax_tree",
+ Targeting::Ax,
+ DriverRoute::AxRead,
+ vec![OracleKind::AxState],
+ ),
+ |pid, window_id, driver| {
+ let state = snapshot(driver, pid, window_id);
+ let text = state.tree_text();
+ assert!(!ax::looks_empty(text), "required GTK3 AT-SPI tree is empty");
+ for name in [
+ "btn-increment",
+ "txt-input",
+ "btn-clicktarget",
+ "sld-value",
+ "chk-agree",
+ "scroll-tall",
+ "btn-open-popover",
+ "btn-exit",
+ ] {
+ assert!(text.contains(name), "missing GTK3 control named {name:?}");
+ }
+ for state in [
+ "HARNESS_TEXT_MARKER_v1",
+ "counter=0",
+ "mirror=",
+ "last_action=none",
+ "last_key=none",
+ "last_hotkey=none",
+ "slider_value=0",
+ "agreed=False",
+ "scroll_offset=0",
+ "popover_open=False",
+ ] {
+ assert!(text.contains(state), "missing GTK3 fixture state {state:?}");
+ }
+ Observation::delivered(vec![OracleKind::AxState], Evidence::default())
+ },
);
- println!("✅ harness_gtk3_popover: popover body enumerated after open");
}
+
+macro_rules! catalog_test {
+ ($name:ident, $row:expr) => {
+ #[test]
+ #[ignore]
+ fn $name() {
+ run_catalog_row($row);
+ }
+ };
+}
+
+catalog_test!(
+ harness_gtk3_left_click_ax_background,
+ CatalogRow {
+ action: "left_click",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::AxClick {
+ target: "btn-increment",
+ expected: "counter=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_left_click_ax_foreground,
+ CatalogRow {
+ action: "left_click",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::AxClick {
+ target: "btn-increment",
+ expected: "counter=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_left_click_px_background,
+ CatalogRow {
+ action: "left_click",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::PxClick {
+ target: "btn-clicktarget",
+ button: "left",
+ count: 1,
+ expected: "last_action=click clicks=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_left_click_px_foreground,
+ CatalogRow {
+ action: "left_click",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::PxClick {
+ target: "btn-clicktarget",
+ button: "left",
+ count: 1,
+ expected: "last_action=click clicks=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_right_click_px_background,
+ CatalogRow {
+ action: "right_click",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::PxClick {
+ target: "btn-clicktarget",
+ button: "right",
+ count: 1,
+ expected: "last_action=right_click clicks=0"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_right_click_px_foreground,
+ CatalogRow {
+ action: "right_click",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::PxClick {
+ target: "btn-clicktarget",
+ button: "right",
+ count: 1,
+ expected: "last_action=right_click clicks=0"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_double_click_px_background,
+ CatalogRow {
+ action: "double_click",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::PxClick {
+ target: "btn-clicktarget",
+ button: "left",
+ count: 2,
+ expected: "last_action=double_click clicks=2"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_double_click_px_foreground,
+ CatalogRow {
+ action: "double_click",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::PxClick {
+ target: "btn-clicktarget",
+ button: "left",
+ count: 2,
+ expected: "last_action=double_click clicks=2"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_type_text_ax_background,
+ CatalogRow {
+ action: "type_text",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxAtSpiValue,
+ operation: Operation::AxTypeText {
+ target: "txt-input",
+ text: "ax-typed",
+ expected: "mirror=ax-typed"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_type_text_ax_foreground,
+ CatalogRow {
+ action: "type_text",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxAtSpiValue,
+ operation: Operation::AxTypeText {
+ target: "txt-input",
+ text: "ax-typed",
+ expected: "mirror=ax-typed"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_type_text_px_background,
+ CatalogRow {
+ action: "type_text",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::PxTypeText {
+ target: "txt-input",
+ text: "px-typed",
+ expected: "mirror=px-typed"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_type_text_px_foreground,
+ CatalogRow {
+ action: "type_text",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::PxTypeText {
+ target: "txt-input",
+ text: "px-typed",
+ expected: "mirror=px-typed"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_set_value_ax_background,
+ CatalogRow {
+ action: "set_value",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxAtSpiValue,
+ operation: Operation::AxSetValue {
+ target: "txt-input",
+ value: "set-value",
+ expected: "mirror=set-value"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_set_value_ax_foreground,
+ CatalogRow {
+ action: "set_value",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxAtSpiValue,
+ operation: Operation::AxSetValue {
+ target: "txt-input",
+ value: "set-value",
+ expected: "mirror=set-value"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_check_ax_background,
+ CatalogRow {
+ action: "checkbox_toggle",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::AxClick {
+ target: "chk-agree",
+ expected: "agreed=True"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_check_ax_foreground,
+ CatalogRow {
+ action: "checkbox_toggle",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::AxClick {
+ target: "chk-agree",
+ expected: "agreed=True"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_slider_set_value_ax_background,
+ CatalogRow {
+ action: "slider_set_value",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxAtSpiValue,
+ operation: Operation::AxSetValue {
+ target: "sld-value",
+ value: "64",
+ expected: "slider_value=64"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_slider_set_value_ax_foreground,
+ CatalogRow {
+ action: "slider_set_value",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxAtSpiValue,
+ operation: Operation::AxSetValue {
+ target: "sld-value",
+ value: "64",
+ expected: "slider_value=64"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_press_key_px_background,
+ CatalogRow {
+ action: "press_key",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::PressKey {
+ key: "f5",
+ expected: "last_key=f5 key_presses=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_press_key_px_foreground,
+ CatalogRow {
+ action: "press_key",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::PressKey {
+ key: "f5",
+ expected: "last_key=f5 key_presses=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_hotkey_px_background,
+ CatalogRow {
+ action: "hotkey",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::Hotkey {
+ expected: "last_hotkey=ctrl+shift+k hotkeys=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_hotkey_px_foreground,
+ CatalogRow {
+ action: "hotkey",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::Hotkey {
+ expected: "last_hotkey=ctrl+shift+k hotkeys=1"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_scroll_ax_background,
+ CatalogRow {
+ action: "scroll",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::Scroll {
+ target: "scroll-tall-vertical",
+ pixel: false,
+ state_key: "scroll_offset="
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_scroll_ax_foreground,
+ CatalogRow {
+ action: "scroll",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::Scroll {
+ target: "scroll-tall-vertical",
+ pixel: false,
+ state_key: "scroll_offset="
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_scroll_px_background,
+ CatalogRow {
+ action: "scroll",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::Scroll {
+ target: "scroll-tall-viewport",
+ pixel: true,
+ state_key: "scroll_offset="
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_scroll_px_foreground,
+ CatalogRow {
+ action: "scroll",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::Scroll {
+ target: "scroll-tall-viewport",
+ pixel: true,
+ state_key: "scroll_offset="
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_drag_px_background,
+ CatalogRow {
+ action: "drag",
+ targeting: Targeting::Px,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxXSendEvent,
+ operation: Operation::Drag {
+ target: "sld-value",
+ state_key: "slider_value="
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_drag_px_foreground,
+ CatalogRow {
+ action: "drag",
+ targeting: Targeting::Px,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxXTest,
+ operation: Operation::Drag {
+ target: "sld-value",
+ state_key: "slider_value="
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_child_window_ax_background,
+ CatalogRow {
+ action: "child_window",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Background,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::Popover {
+ target: "btn-open-popover",
+ expected: "popover_open=True"
+ },
+ }
+);
+catalog_test!(
+ harness_gtk3_child_window_ax_foreground,
+ CatalogRow {
+ action: "child_window",
+ targeting: Targeting::Ax,
+ delivery: Delivery::Foreground,
+ route: DriverRoute::LinuxAtSpiAction,
+ operation: Operation::Popover {
+ target: "btn-open-popover",
+ expected: "popover_open=True"
+ },
+ }
+);
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs
deleted file mode 100644
index ec2c43cc39..0000000000
--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs
+++ /dev/null
@@ -1,186 +0,0 @@
-//! Linux **desktop-scope** (vision/foreground) modality through the SAME
-//! cua-driver interface as Windows/macOS: `set_config capture_scope=desktop` +
-//! a window-less screen-absolute `click` (no `pid`/`window_id`/`list_windows`).
-//! The Linux actuator warps the pointer and injects a real button press via the
-//! XTest extension — the peer of the Windows `WindowFromPoint` + macOS
-//! global-HID desktop click. (XTest delivering to the under-pointer window is
-//! why the *background* paths use `XSendEvent`; for desktop scope that delivery
-//! is exactly what we want.)
-//!
-//! Grounds the click on the GTK3 harness increment button's screen-absolute
-//! `frame` (AT-SPI Component extents), asserts the counter advanced, plus the
-//! window-scope rejection gate.
-//!
-//! Linux config is global-only (no per-session override), so `set_config`
-//! capture_scope writes the on-disk default — the test resets it to `window`
-//! before asserting so a failure can't leave the sandbox in desktop scope.
-//!
-//! #[ignore] (needs an X11/Xwayland display + AT-SPI + the GTK3 harness). Run:
-//! cargo test -p cua-driver --test modality_desktop_scope_linux_test -- --ignored --nocapture --test-threads=1
-
-#![cfg(target_os = "linux")]
-
-use std::process::{Command, Stdio};
-use std::time::{Duration, Instant};
-
-use cua_driver_testkit::{harness_app, Driver, McpDriver};
-
-fn harness_exe() -> std::path::PathBuf {
- std::env::var("HARNESS_GTK3_EXE")
- .map(std::path::PathBuf::from)
- .ok()
- .filter(|p| p.exists())
- .unwrap_or_else(|| harness_app("harness-gtk3", "CuaTestHarness.Gtk3"))
-}
-
-fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> {
- let exe = harness_exe();
- if !exe.exists() {
- eprintln!("[desktop-linux] GTK3 harness not built ({exe:?}) — run tests/fixtures/build/linux.sh; skipping");
- return None;
- }
- driver
- .reaper()
- .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null()))
- .ok()?;
- let deadline = Instant::now() + Duration::from_secs(14);
- while Instant::now() < deadline {
- let r = driver.call("list_windows", serde_json::json!({}));
- if let Some(wins) = r.structured()["windows"].as_array() {
- for w in wins {
- if w["title"].as_str().unwrap_or("").contains("CuaTestHarness GTK3") {
- let pid = w["pid"].as_u64().unwrap_or(0) as u32;
- let wid = w["window_id"].as_u64().unwrap_or(0);
- if pid != 0 && wid != 0 {
- driver.reaper().track_pid(pid);
- return Some((pid, wid));
- }
- }
- }
- }
- std::thread::sleep(Duration::from_millis(400));
- }
- eprintln!("[desktop-linux] harness window never appeared — graphical session + AT-SPI available? skipping");
- None
-}
-
-fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value {
- driver
- .call(
- "get_window_state",
- serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }),
- )
- .structured()
- .clone()
-}
-
-/// Screen-absolute center (px) of the increment button from `elements[].frame`
-/// (AT-SPI Component extents are screen-absolute). The GTK3 harness sets the
-/// button's accessible NAME to `btn-increment`; match any element whose blob
-/// carries that aid and has a frame, so we're robust to the exact field name.
-fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> {
- let els = snap["elements"].as_array()?;
- let btn = els.iter().find(|e| {
- serde_json::to_string(e).map(|s| s.contains("btn-increment")).unwrap_or(false)
- && e.get("frame").map(|f| f.is_object()).unwrap_or(false)
- })?;
- let f = &btn["frame"];
- let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?);
- Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64))
-}
-
-fn counter(snap: &serde_json::Value) -> Option<u64> {
- let tree = snap["tree_markdown"].as_str()?;
- let idx = tree.find("counter=")? + "counter=".len();
- let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect();
- digits.parse().ok()
-}
-
-fn set_scope(driver: &mut McpDriver, scope: &str) {
- let r = driver.call(
- "set_config",
- serde_json::json!({ "key": "capture_scope", "value": scope }),
- );
- assert!(!r.is_error(), "set_config capture_scope={scope} failed: {}", r.text());
-}
-
-// ── tests ───────────────────────────────────────────────────────────────────────
-
-/// In desktop scope, a window-less screen-absolute click (no pid/window_id)
-/// lands on the increment button — its counter advances.
-#[test]
-#[ignore]
-fn desktop_scope_windowless_click_lands_on_control() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- let Some((pid, wid)) = launch(&mut driver) else { return };
-
- // Settle for the AT-SPI tree to register the button + its extents.
- let mut snap = ax_snapshot(&mut driver, pid, wid);
- let mut center = increment_center(&snap);
- let deadline = Instant::now() + Duration::from_secs(8);
- while center.is_none() && Instant::now() < deadline {
- std::thread::sleep(Duration::from_millis(400));
- snap = ax_snapshot(&mut driver, pid, wid);
- center = increment_center(&snap);
- }
- let Some((cx, cy)) = center else {
- eprintln!("[desktop-linux] increment button frame not found (AT-SPI extents missing?) — skipping");
- return;
- };
- let pre = counter(&snap).unwrap_or(0);
- println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}");
-
- set_scope(&mut driver, "desktop");
-
- // Retry the window-less desktop click until the counter advances. A
- // freshly-mapped harness window may not yet be raised under the pointer on
- // the first click (X11 window-raise timing differs across WMs — XFCE/Openbox
- // lag GNOME), so the screen-absolute XTest click can miss the first attempt.
- // Re-issuing the SAME click is safe: extra landed clicks only increment the
- // counter further, and `post > pre` still holds. We assert the click was
- // *dispatched as desktop scope* on the first attempt, and that it eventually
- // *lands* within the budget.
- let mut post = pre;
- let mut first_text = String::new();
- for attempt in 0..12 {
- let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy }));
- if attempt == 0 {
- first_text = clicked.text().to_string();
- assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text());
- }
- std::thread::sleep(Duration::from_millis(500));
- post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre);
- if post > pre {
- break;
- }
- }
- // Reset scope so a later failure can't leave the box in desktop scope.
- set_scope(&mut driver, "window");
-
- assert!(
- first_text.to_lowercase().contains("desktop scope"),
- "click not reported as desktop-scope: {first_text}"
- );
- assert!(
- post > pre,
- "counter did not advance after window-less desktop clicks: pre={pre} post={post} \
- (the harness window never became clickable at ({cx},{cy}) within the retry budget)"
- );
- println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}");
-}
-
-/// Negative gate: a window-less click under `capture_scope=window` is rejected.
-#[test]
-#[ignore]
-fn window_scope_rejects_windowless_click() {
- let Some(mut driver) = McpDriver::spawn() else { return };
- set_scope(&mut driver, "window");
- let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100 }));
- let txt = r.text().to_lowercase();
- assert!(
- r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"),
- "window-scope window-less click was NOT rejected: {}",
- r.text()
- );
- println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated");
-}
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_linux_test.rs
deleted file mode 100644
index 26d71cf6fd..0000000000
--- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_linux_test.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-//! Modality axis: the Linux dispatch ladder (delivery_mode parity).
-//!
-//! Asserts the contract the Linux parity work added (X11 background/foreground
-//! + bring_to_front). Uses the binary's own `describe` (which computes each
-//! ToolDef schema locally) rather than a daemon round-trip, so it reflects the
-//! freshly-built binary and runs in CI without a display or daemon.
-
-#![cfg(target_os = "linux")]
-
-use std::path::PathBuf;
-use std::process::Command;
-
-fn driver_bin() -> PathBuf {
- // tests run with CARGO_MANIFEST_DIR = crates/cua-driver; the workspace
- // target/debug is two levels up.
- PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/debug/cua-driver")
-}
-
-/// `cua-driver describe <tool>` → the tool's advertised inputSchema as text.
-/// Returns None (test skips) when the binary isn't built.
-fn describe(tool: &str) -> Option<String> {
- let bin = driver_bin();
- if !bin.exists() {
- eprintln!("[dispatch-linux] {bin:?} not built — skipping");
- return None;
- }
- let out = Command::new(&bin).arg("describe").arg(tool).output().ok()?;
- Some(String::from_utf8_lossy(&out.stdout).into_owned())
-}
-
-/// Every Linux input tool advertises `delivery_mode` with the two-mode enum —
-/// the per-call background/foreground rung selector (parity with macOS/Windows).
-#[test]
-fn linux_input_tools_advertise_delivery_mode() {
- for tool in ["click", "type_text", "press_key", "hotkey", "double_click", "right_click", "scroll"] {
- let Some(schema) = describe(tool) else { return };
- assert!(
- schema.contains("delivery_mode"),
- "{tool} schema is missing delivery_mode:\n{schema}"
- );
- assert!(
- schema.contains("foreground") && schema.contains("background"),
- "{tool} delivery_mode enum should include background + foreground:\n{schema}"
- );
- }
-}
-
-/// `bring_to_front` is a real EWMH activation now (not the old Windows-only
-/// stub) — its description must reflect the X11 _NET_ACTIVE_WINDOW path.
-#[test]
-fn linux_bring_to_front_is_real_not_windows_only_stub() {
- let Some(schema) = describe("bring_to_front") else { return };
- assert!(
- !schema.contains("Windows-only"),
- "bring_to_front should be a real Linux EWMH activation now, not a Windows-only stub:\n{schema}"
- );
- assert!(
- schema.contains("_NET_ACTIVE_WINDOW") || schema.to_lowercase().contains("activate"),
- "bring_to_front description should mention the activation mechanism:\n{schema}"
- );
-}
diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
index f8fbedf8e4..debfc3cff4 100644
--- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
+++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
@@ -37,6 +37,8 @@ meval = "0.2"
# capture/input protocols added as those slices land). Pure-Rust client; used
# when running under a Wayland compositor with no X11 (no DISPLAY).
wayland-client = "0.31"
+wayland-backend = "0.3"
+wayland-scanner = "0.31"
wayland-protocols-wlr = { version = "0.3", features = ["client"] }
# Wayland staging protocols: ext-foreign-toplevel-list-v1,
# ext-image-capture-source-v1, ext-image-copy-capture-v1 (per-window
@@ -51,10 +53,8 @@ wayland-protocols = { version = "0.32", features = ["client", "staging"] }
# (org.freedesktop.portal.RemoteDesktop.ConnectToEIS). `tokio` shares the
# driver's async runtime.
#
-# `screencast` + `remote_desktop` ashpd features pull pipewire-rs (libspa-sys),
-# which needs `libpipewire-0.3 >= 0.3.40` headers at build time — newer than
-# Debian bullseye ships. They're moved behind the `portal-libei` feature so
-# the cross-platform CD (Debian 11, GLIBC_2.31 floor) can keep building.
+# ashpd's RemoteDesktop feature is pure Rust. Only our ScreenCast frame reader
+# pulls pipewire-rs/libspa, whose headers are newer than Debian bullseye.
ashpd = { version = "0.13", default-features = false, features = ["tokio", "screenshot"] }
# zbus is pulled transitively via ashpd + atspi but declared directly so
# the portal-probe path in wayland::portal_screenshot can talk to the
@@ -68,11 +68,11 @@ url = "2"
# frames from the node id returned by `open_pipe_wire_remote`. Pinned to
# 0.8 (last well-documented release; 0.10 had docs.rs build failures in
# mid-2026). Needs libpipewire-0.3 headers at build time — gated behind
-# the `portal-libei` feature.
+# the `portal-capture` feature.
pipewire = { version = "0.8", optional = true }
# Companion to pipewire 0.8 — needed to construct the SPA_PARAM_EnumFormat
# pod (SPA_VIDEO_FORMAT_BGRx at announced width/height) we hand to
-# `Stream::connect`. Same `portal-libei` gating as `pipewire`.
+# `Stream::connect`. Same `portal-capture` gating as `pipewire`.
libspa = { version = "0.8", optional = true }
# Pure-Rust libei/libeis bindings for the GNOME/KDE input path (handshake
# → seat → device → pointer_absolute/button/scroll/keyboard/text). API is
@@ -107,26 +107,27 @@ dirs = "5"
[features]
default = []
linux-example = []
-# Opt-in stack that wires the portal ScreenCast per-window capture path
-# (`wayland::portal_screencast`) and the libei input path
-# (`wayland::libei`). Pulls in `pipewire`, `libspa` (need
-# `libpipewire-0.3 >= 0.3.40` headers — newer than Debian bullseye) and
-# `reis` (pure-Rust libei) plus the ashpd RemoteDesktop+ScreenCast
-# features. Disabled in the cross-platform release CD (debian:11
-# container, GLIBC_2.31 floor) and enabled in the Nix build that
-# already has the modern PipeWire + libei from nixpkgs. The wlroots
-# screencopy + virtual-pointer paths and the X11 fallback work either
-# way — only GNOME/KDE-portal-specific tiers go dark when this is off.
-portal-libei = [
- "ashpd/screencast",
+# GNOME/KDE input is pure Rust plus libxkbcommon, which is available in the
+# Debian 11 release container. Ashpd's RemoteDesktop API exposes ScreenCast
+# stream metadata, so its lightweight screencast module is required here too.
+# The PipeWire frame reader remains isolated in portal-capture below.
+portal-input = [
"ashpd/remote_desktop",
- "dep:pipewire",
- "dep:libspa",
+ "ashpd/screencast",
"dep:reis",
"dep:calloop",
"dep:enumflags2",
"dep:xkbcommon",
]
+# PipeWire-backed per-window ScreenCast capture keeps its newer system-header
+# requirement and remains enabled by Nix/modern desktop builds.
+portal-capture = [
+ "ashpd/screencast",
+ "dep:pipewire",
+ "dep:libspa",
+]
+# Backward-compatible full desktop feature used by the Nix package.
+portal-libei = ["portal-input", "portal-capture"]
[dev-dependencies]
# Used by the `screenshot_cascade` example to surface the
@@ -135,11 +136,11 @@ portal-libei = [
tracing-subscriber = { workspace = true }
# Cargo example that exercises the libei input path directly. Lives under
-# the `portal-libei` feature gate so a stock `cargo build --examples`
+# the `portal-input` feature gate so a stock `cargo build --examples`
# doesn't try to compile against the (now-optional) reis/xkbcommon deps.
[[example]]
name = "libei_input"
-required-features = ["portal-libei"]
+required-features = ["portal-input"]
[[example]]
name = "screenshot_cascade"
diff --git a/libs/cua-driver/rust/crates/platform-linux/protocol/virtual-keyboard-unstable-v1.xml b/libs/cua-driver/rust/crates/platform-linux/protocol/virtual-keyboard-unstable-v1.xml
new file mode 100644
index 0000000000..6b0e1206cf
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/protocol/virtual-keyboard-unstable-v1.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<protocol name="virtual_keyboard_unstable_v1">
+ <copyright>
+ Copyright © 2008-2011 Kristian Høgsberg
+ Copyright © 2010-2013 Intel Corporation
+ Copyright © 2012-2013 Collabora, Ltd.
+ Copyright © 2018 Purism SPC
+
+ Permission is hereby granted, free of charge, to any person obtaining a
+ copy of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ and/or sell copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ DEALINGS IN THE SOFTWARE.
+ </copyright>
+
+ <interface name="zwp_virtual_keyboard_v1" version="1">
+ <description summary="virtual keyboard">
+ The virtual keyboard provides an application with requests which emulate
+ the behaviour of a physical keyboard.
+ </description>
+ <request name="keymap">
+ <arg name="format" type="uint" summary="keymap format"/>
+ <arg name="fd" type="fd" summary="keymap file descriptor"/>
+ <arg name="size" type="uint" summary="keymap size, in bytes"/>
+ </request>
+ <enum name="error">
+ <entry name="no_keymap" value="0" summary="No keymap was set"/>
+ </enum>
+ <request name="key">
+ <arg name="time" type="uint" summary="timestamp with millisecond granularity"/>
+ <arg name="key" type="uint" summary="key that produced the event"/>
+ <arg name="state" type="uint" summary="physical state of the key"/>
+ </request>
+ <request name="modifiers">
+ <arg name="mods_depressed" type="uint"/>
+ <arg name="mods_latched" type="uint"/>
+ <arg name="mods_locked" type="uint"/>
+ <arg name="group" type="uint"/>
+ </request>
+ <request name="destroy" type="destructor" since="1"/>
+ </interface>
+
+ <interface name="zwp_virtual_keyboard_manager_v1" version="1">
+ <description summary="virtual keyboard manager">
+ A virtual keyboard manager allows an application to provide keyboard
+ input events as if they came from a physical keyboard.
+ </description>
+ <enum name="error">
+ <entry name="unauthorized" value="0" summary="client not authorized"/>
+ </enum>
+ <request name="create_virtual_keyboard">
+ <arg name="seat" type="object" interface="wl_seat"/>
+ <arg name="id" type="new_id" interface="zwp_virtual_keyboard_v1"/>
+ </request>
+ </interface>
+</protocol>
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs b/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs
index 5a96af06d9..53532de47a 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs
@@ -12,14 +12,12 @@
//! to the driver. Electron apps shipped as AppImages behave identically; they
//! embed the same Chromium.
//!
-//! A real screen reader (Orca) turns the tree on simply by writing that status
-//! property when it starts. We do the same once, at daemon startup. Because the
-//! status object lives on the session-scoped accessibility-bus launcher rather
-//! than in our process, the flag is session-wide, outlives us, and takes effect
-//! retroactively on apps that are already running — no relaunch and no
-//! per-application command-line flag. GTK and Qt gate their own AT-SPI bridges
-//! on the companion `IsEnabled` property, so setting it warms those toolkits
-//! too.
+//! A real screen reader turns the Chromium signal on. Doing that ourselves is
+//! unsafe on GNOME: its settings daemon treats the signal as a user request and
+//! launches Orca. GNOME therefore gets only the generic `IsEnabled` signal by
+//! default. Other desktops retain the Chromium signal for compatibility, and a
+//! caller can choose either policy explicitly with
+//! `CUA_DRIVER_RS_A11Y_ADVERTISE_MODE`.
//!
//! Everything here is best-effort. A session without an accessibility bus (some
//! headless or minimal setups) just yields an error we log and ignore; enabling
@@ -42,6 +40,13 @@ const SCREEN_READER_ENABLED_PROPERTY: &str = "ScreenReaderEnabled";
/// Companion property GTK/Qt watch to load their AT-SPI bridges.
const ACCESSIBILITY_IS_ENABLED_PROPERTY: &str = "IsEnabled";
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum AdvertiseMode {
+ All,
+ IsEnabledOnly,
+ None,
+}
+
/// Advertise an assistive technology to the session exactly once per daemon
/// process, so Chromium/Electron (including Electron AppImages), GTK, and Qt
/// expose their accessibility trees to [`crate::atspi`]. Idempotent and
@@ -50,17 +55,24 @@ const ACCESSIBILITY_IS_ENABLED_PROPERTY: &str = "IsEnabled";
pub fn ensure_chromium_accessibility_enabled() {
static ADVERTISED: Once = Once::new();
ADVERTISED.call_once(|| {
- // Opt-out for the rare session that wants its accessibility status left
- // untouched (e.g. one already driven by a real screen reader the user
- // configured deliberately).
- if std::env::var_os("CUA_DRIVER_RS_DISABLE_A11Y_ADVERTISE").is_some() {
+ let mode = advertise_mode_from(
+ std::env::var_os("CUA_DRIVER_RS_DISABLE_A11Y_ADVERTISE").is_some(),
+ std::env::var("CUA_DRIVER_RS_A11Y_ADVERTISE_MODE")
+ .ok()
+ .as_deref(),
+ std::env::var("XDG_CURRENT_DESKTOP")
+ .or_else(|_| std::env::var("XDG_SESSION_DESKTOP"))
+ .or_else(|_| std::env::var("DESKTOP_SESSION"))
+ .ok()
+ .as_deref(),
+ );
+ if mode == AdvertiseMode::None {
tracing::debug!(
- "CUA_DRIVER_RS_DISABLE_A11Y_ADVERTISE set; leaving session \
- accessibility status untouched"
+ "accessibility advertisement disabled; leaving session status untouched"
);
return;
}
- if let Err(error) = advertise_screen_reader_to_session() {
+ if let Err(error) = advertise_accessibility_to_session(mode) {
tracing::debug!(
"skipped advertising accessibility to the session \
(Chromium/Electron trees may stay empty): {error:#}"
@@ -69,25 +81,25 @@ pub fn ensure_chromium_accessibility_enabled() {
});
}
-fn advertise_screen_reader_to_session() -> anyhow::Result<()> {
+fn advertise_accessibility_to_session(mode: AdvertiseMode) -> anyhow::Result<()> {
// The daemon's tokio runtime is already driving this thread when the tool
// registry is built, and `block_on` panics if called from within a runtime.
// Run the one-shot bus work on a dedicated OS thread that owns a small
// runtime of its own, then join it — no nesting, torn down once it returns.
std::thread::Builder::new()
.name("cua-a11y-advertise".into())
- .spawn(|| {
+ .spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
- runtime.block_on(advertise_screen_reader())
+ runtime.block_on(advertise_accessibility(mode))
})
.context("spawning the accessibility-advertise thread")?
.join()
.map_err(|_| anyhow!("accessibility-advertise thread panicked"))?
}
-async fn advertise_screen_reader() -> anyhow::Result<()> {
+async fn advertise_accessibility(mode: AdvertiseMode) -> anyhow::Result<()> {
let session_bus = zbus::Connection::session().await?;
let status = zbus::Proxy::new(
&session_bus,
@@ -100,7 +112,7 @@ async fn advertise_screen_reader() -> anyhow::Result<()> {
// Don't clobber a screen reader the user is already running: only write when
// a flag is currently false, so an active Orca session stays authoritative
// and we avoid emitting a redundant PropertiesChanged.
- if !is_flag_set(&status, SCREEN_READER_ENABLED_PROPERTY).await {
+ if mode == AdvertiseMode::All && !is_flag_set(&status, SCREEN_READER_ENABLED_PROPERTY).await {
status
.set_property(SCREEN_READER_ENABLED_PROPERTY, true)
.await?;
@@ -113,8 +125,93 @@ async fn advertise_screen_reader() -> anyhow::Result<()> {
Ok(())
}
+fn advertise_mode_from(
+ disabled: bool,
+ configured: Option<&str>,
+ desktop: Option<&str>,
+) -> AdvertiseMode {
+ if disabled {
+ return AdvertiseMode::None;
+ }
+ match configured
+ .map(str::trim)
+ .map(str::to_ascii_lowercase)
+ .as_deref()
+ {
+ Some("all") => AdvertiseMode::All,
+ Some("is_enabled_only") => AdvertiseMode::IsEnabledOnly,
+ Some("none") => AdvertiseMode::None,
+ Some(other) => {
+ tracing::warn!(
+ mode = other,
+ "unknown CUA_DRIVER_RS_A11Y_ADVERTISE_MODE; using desktop default"
+ );
+ desktop_default_mode(desktop)
+ }
+ None => desktop_default_mode(desktop),
+ }
+}
+
+fn desktop_default_mode(desktop: Option<&str>) -> AdvertiseMode {
+ let is_gnome = desktop.is_some_and(|desktop| {
+ desktop
+ .split([':', ';'])
+ .any(|part| part.trim().eq_ignore_ascii_case("gnome"))
+ });
+ if is_gnome {
+ AdvertiseMode::IsEnabledOnly
+ } else {
+ AdvertiseMode::All
+ }
+}
+
/// Read a boolean status property, treating an unreadable property as unset so
/// the caller falls through to writing it.
async fn is_flag_set(status: &zbus::Proxy<'_>, property: &str) -> bool {
status.get_property::<bool>(property).await.unwrap_or(false)
}
+
+#[cfg(test)]
+mod tests {
+ use super::{advertise_mode_from, AdvertiseMode};
+
+ #[test]
+ fn gnome_default_does_not_claim_a_screen_reader() {
+ assert_eq!(
+ advertise_mode_from(false, None, Some("ubuntu:GNOME")),
+ AdvertiseMode::IsEnabledOnly
+ );
+ }
+
+ #[test]
+ fn non_gnome_default_preserves_chromium_compatibility() {
+ assert_eq!(
+ advertise_mode_from(false, None, Some("KDE")),
+ AdvertiseMode::All
+ );
+ }
+
+ #[test]
+ fn explicit_mode_overrides_desktop_default() {
+ assert_eq!(
+ advertise_mode_from(false, Some("all"), Some("GNOME")),
+ AdvertiseMode::All
+ );
+ assert_eq!(
+ advertise_mode_from(false, Some("is_enabled_only"), Some("KDE")),
+ AdvertiseMode::IsEnabledOnly
+ );
+ assert_eq!(
+ advertise_mode_from(false, Some("none"), Some("KDE")),
+ AdvertiseMode::None
+ );
+ }
+
+ #[test]
+ fn legacy_disable_wins_over_explicit_mode() {
+ assert_eq!(
+ advertise_mode_from(true, Some("all"), Some("KDE")),
+ AdvertiseMode::None
+ );
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
index 6678910d0b..e83970012f 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
@@ -14,6 +14,7 @@ use anyhow::Result;
pub mod cache;
pub mod native;
pub use cache::ElementCache;
+pub use native::ensure_listener_active;
#[derive(Clone, Debug)]
pub struct AtspiNode {
@@ -23,7 +24,7 @@ pub struct AtspiNode {
pub value: Option<String>,
pub description: Option<String>,
pub actions: Vec<String>,
- /// For pyatspi path: element_key = element_index as u64.
+ /// For AT-SPI: element_key = element_index as u64.
/// For X11 fallback: element_key = xid.
pub element_key: u64,
/// Depth in the markdown tree (0 = top-level window child).
@@ -37,6 +38,7 @@ pub struct AtspiNode {
pub struct AtspiTreeResult {
pub tree_markdown: String,
pub nodes: Vec<AtspiNode>,
+ pub bounds: Vec<(usize, i32, i32, u32, u32)>,
}
/// Walk the AT-SPI tree for a window identified by (pid, xid).
@@ -65,7 +67,9 @@ pub fn walk_tree_bounded(
// after launch returns the real tree instead of an empty one. See #1927.
const MAX_ATTEMPTS: usize = 4;
for attempt in 0..MAX_ATTEMPTS {
- if let Ok(Some((raw_md, nodes))) = native::walk_tree_bounded(pid, max_elements, max_depth) {
+ if let Ok(Some((raw_md, nodes, bounds))) =
+ native::walk_tree_bounded(pid, xid, max_elements, max_depth)
+ {
// `nodes.len() <= 1` == only the root window resolved: the
// cold-registry symptom. Accept any real tree immediately; only
// keep waiting on the degenerate case, and accept it anyway on the
@@ -79,6 +83,7 @@ pub fn walk_tree_bounded(
return AtspiTreeResult {
tree_markdown: md,
nodes,
+ bounds,
};
}
}
@@ -146,68 +151,12 @@ pub fn perform_action_at_screen_point(
/// For Qt5, which doesn't expose widgets when unfocused, this will return Err.
/// Returns Ok if an editable was found and text was set, Err otherwise.
pub fn type_into_editable(pid: u32, text: &str) -> Result<()> {
- let safe_text = text.replace('\\', "\\\\").replace('\'', "\\'");
- let script = format!(
- r#"
-import pyatspi, sys
-
-def find_editable(acc, depth=0):
- # Try to find any EditableText interface, regardless of role
- try:
- et = acc.queryEditableText()
- # If we can query it, return this node
- return acc
- except:
- pass
-
- # Recursively search children
- try:
- for child in acc:
- result = find_editable(child, depth + 1)
- if result is not None:
- return result
- except:
- pass
-
- return None
-
-desktop = pyatspi.Registry.getDesktop(0)
-editable = None
-for app in desktop:
- try:
- if app.get_process_id() == {pid}:
- for win in app:
- editable = find_editable(win)
- if editable:
- break
- break
- except:
- pass
-
-if editable is None:
- print("ERROR: No editable found", file=sys.stderr)
- sys.exit(1)
-
-try:
- et = editable.queryEditableText()
- et.setTextContents('{safe_text}')
- print("ok:atspi")
-except Exception as e:
- print(f"ERROR: {{e}}", file=sys.stderr)
- sys.exit(1)
-"#,
- pid = pid,
- safe_text = safe_text
- );
-
- let out = std::process::Command::new("python3")
- .arg("-c")
- .arg(&script)
- .output()?;
- if !out.status.success() {
- anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());
- }
- Ok(())
+ native::type_into_editable(pid, text)
+}
+
+/// Type into the exact indexed editable from the caller's accessibility snapshot.
+pub fn type_into_editable_at(pid: u32, idx: usize, text: &str) -> Result<()> {
+ native::type_into_editable_at(pid, idx, text)
}
/// Set the text value of element `idx` within pid's app tree via AT-SPI.
@@ -235,15 +184,6 @@ pub fn focused_is_editable(pid: u32) -> Result<Option<bool>> {
native::focused_is_editable(pid)
}
-/// Get the screen-coordinate bounding box (x, y, width, height) of element `idx`.
-/// Screen-coordinate bounds for every action node in pid's AT-SPI tree, keyed
-/// by `element_index`. Best-effort: nodes whose bounds can't be read are
-/// omitted rather than erroring the whole call. Returns `(element_index, x, y,
-/// width, height)` tuples in screen coordinates.
-pub fn get_all_element_bounds(pid: u32, xid: u64) -> Result<Vec<(usize, i32, i32, u32, u32)>> {
- native::get_all_element_bounds(pid, xid)
-}
-
pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> {
native::get_element_bounds(pid, idx)
}
@@ -260,6 +200,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
return AtspiTreeResult {
tree_markdown: String::new(),
nodes: vec![],
+ bounds: vec![],
}
}
};
@@ -310,6 +251,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult {
AtspiTreeResult {
tree_markdown,
nodes,
+ bounds: vec![],
}
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index 5f2c6feab5..4a37af54aa 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -82,6 +82,43 @@ fn runtime() -> &'static tokio::runtime::Runtime {
})
}
+static SHARED_CONNECTION: tokio::sync::OnceCell<AccessibilityConnection> =
+ tokio::sync::OnceCell::const_new();
+
+/// Keep one AT-SPI connection and registry registration alive for the daemon
+/// lifetime. WebKitGTK only publishes its WebProcess accessibility subtree
+/// while the registry reports an interested listener.
+async fn shared_connection() -> Result<&'static AccessibilityConnection> {
+ SHARED_CONNECTION
+ .get_or_try_init(|| async {
+ let conn = AccessibilityConnection::new()
+ .await
+ .map_err(|error| anyhow!("AT-SPI connect failed: {error}"))?;
+ if let Err(error) = conn.add_registry_event::<atspi::ObjectEvents>().await {
+ dlog!("AT-SPI object-event registration failed: {error}");
+ }
+ Ok(conn)
+ })
+ .await
+}
+
+/// Establish the process-lifetime listener before accessibility-aware apps are
+/// launched. Idempotent; later calls reuse the same connection.
+pub fn ensure_listener_active() -> Result<()> {
+ let connect = || runtime().block_on(async { shared_connection().await.map(|_| ()) });
+ if tokio::runtime::Handle::try_current().is_ok() {
+ // The daemon builds its registry from its Tokio entry-point. Calling
+ // Runtime::block_on there panics even though this module owns a separate
+ // runtime, so initialize the AT-SPI connection on a plain thread and
+ // wait for it before accessibility-aware apps can launch.
+ std::thread::spawn(connect)
+ .join()
+ .map_err(|_| anyhow!("AT-SPI listener initialization thread panicked"))?
+ } else {
+ connect()
+ }
+}
+
/// A node discovered during the pre-order walk, with its proxy retained so the
/// per-index operations can act on it without re-walking the tree.
struct Visited<'a> {
@@ -110,6 +147,11 @@ fn is_document_role(role: &str) -> bool {
r.contains("document") || r == "embedded"
}
+fn is_web_process_bus(name: &str) -> bool {
+ let name = name.to_ascii_lowercase();
+ name.contains("webkit") || name.contains("webprocess")
+}
+
/// Build an `AccessibleProxy` for an arbitrary (bus name, path) in the tree.
/// Uses owned `String`s for destination/path so the resulting `BusName`/
/// `ObjectPath` are `'static` and the proxy borrows only the connection.
@@ -133,22 +175,24 @@ async fn accessible_for<'a>(
conn: &'a AccessibilityConnection,
oref: &RawObjectRef,
) -> Result<AccessibleProxy<'a>> {
- // Keep the atspi crate's peer-to-peer path for normal AT-SPI unique names.
- // Electron/Chromium uses this path for focused-child input delivery. The
- // raw string path below is only needed for WebKitGTK's well-known
- // WebProcess references, which cannot be represented as ObjectRef names.
+ // Keep the atspi crate's peer-to-peer path when this connection actually
+ // knows the peer. Late WebKit WebProcess children are not in the initial
+ // peer snapshot; object_as_accessible's bus fallback omits their destination
+ // and targets the Accessible interface name instead. Build an explicit bus
+ // proxy below for those late peers and for well-known references.
if oref.name.starts_with(':') {
let name = atspi::zbus::names::UniqueName::try_from(oref.name.clone())
.map_err(|e| anyhow!("bad a11y unique name: {e}"))?;
- let path = atspi::zbus::zvariant::ObjectPath::try_from(oref.path.clone())
- .map_err(|e| anyhow!("bad a11y path: {e}"))?;
- let object = atspi::ObjectRef::new_owned(name, path);
- // `object_as_accessible` chooses a P2P peer when the toolkit exposes
- // one and falls back to the shared accessibility bus otherwise.
- return conn
- .object_as_accessible(&object)
- .await
- .map_err(|e| anyhow!("AccessibleProxy build failed: {e}"));
+ let bus_name = atspi::zbus::names::BusName::Unique(name.as_ref());
+ if conn.get_peer(&bus_name).is_some() {
+ let path = atspi::zbus::zvariant::ObjectPath::try_from(oref.path.clone())
+ .map_err(|e| anyhow!("bad a11y path: {e}"))?;
+ let object = atspi::ObjectRef::new_owned(name, path);
+ return conn
+ .object_as_accessible(&object)
+ .await
+ .map_err(|e| anyhow!("AccessibleProxy build failed: {e}"));
+ }
}
AccessibleProxy::builder(conn.connection())
.cache_properties(atspi::zbus::proxy::CacheProperties::No)
@@ -336,7 +380,7 @@ async fn collect_visited_bounded<'a>(
// AT-SPI (most commonly because it holds a modal grab and isn't servicing
// D-Bus), every per-node `call()` burns the full CALL_TIMEOUT before being
// skipped, so the walk would otherwise grind for minutes. Callers that lack
- // their own OP_TIMEOUT (get_all_element_bounds, insert_text) relied on this
+ // their own OP_TIMEOUT (snapshot bounds, insert_text) relied on this
// never happening — bound it here so the walk returns partial within
// OP_TIMEOUT for every caller, instead of hanging get_window_state/type_text
// on modal dialogs (#1936).
@@ -348,7 +392,7 @@ async fn collect_visited_bounded<'a>(
// few seconds rather than ~25s.
let mut consecutive_timeouts = 0u32;
- while let Some((oref, depth, in_web_doc)) = stack.pop() {
+ while let Some((oref, depth, inherited_web_doc)) = stack.pop() {
if budget == 0 {
dlog!("node budget exhausted; truncating walk");
break;
@@ -358,13 +402,18 @@ async fn collect_visited_bounded<'a>(
break;
}
budget -= 1;
+ // WebKitGTK publishes its embedded page on a distinct WebProcess
+ // D-Bus peer and can expose blank role names for the entire subtree.
+ // The peer identity is therefore the reliable document boundary when
+ // role-based AT-SPI discovery cannot identify one.
+ let in_web_doc = inherited_web_doc || is_web_process_bus(&oref.name);
// accessible_for builds a proxy whose first use round-trips to the
// target app. On a modal-grabbed (AT-SPI-unresponsive) app this is the
// await that actually hangs, so it MUST carry the per-call timeout —
// otherwise the loop never returns to the deadline check at the top and
// the walk stalls past OP_TIMEOUT for callers without an outer guard
- // (get_all_element_bounds, insert_text). That was the residual #1936 hang.
+ // (snapshot bounds, insert_text). That was the residual #1936 hang.
let acc = match call(accessible_for(conn, &oref)).await {
Some(Ok(a)) => a,
Some(Err(error)) => {
@@ -611,24 +660,27 @@ fn format_value(v: f64) -> String {
/// Historically this was "the node advertises AT-SPI Actions" (buttons, menu
/// items, links). That silently dropped every **Value**-only widget — GTK
/// `GtkScale` sliders, scroll bars, spin buttons, progress bars expose the
-/// `Value` interface but NO `Action`, so they never got an `element_index` and
-/// were invisible to `get_window_state`/`set_value` even though the driver can
-/// drive them (`set_value` already handles `has_value`). We now also index any
-/// node carrying the Value interface so sliders and scroll regions surface as
-/// usable elements.
+/// `Value` interface but NO `Action`, while some text fields expose
+/// `EditableText` without either. Omitting those interfaces makes controls the
+/// driver can operate impossible to address by `element_index`.
///
/// This predicate is the single source of truth for the element-index space and
/// MUST be applied identically in `render` and in every `action_nodes` filter
-/// (`perform_action`, `set_value`, `get_element_bounds`, `get_all_element_bounds`);
+/// (`perform_action`, `set_value`, `get_element_bounds`, snapshot bounds);
/// any divergence would desync indices between the snapshot and the operations.
fn is_indexable(v: &Visited) -> bool {
- !v.actions.is_empty() || v.has_value
+ is_indexable_capabilities(!v.actions.is_empty(), v.has_editable, v.has_value)
+}
+
+fn is_indexable_capabilities(has_action: bool, has_editable: bool, has_value: bool) -> bool {
+ has_action || has_editable || has_value
}
// ── Public (sync) entry points ───────────────────────────────────────────────
pub fn walk_tree(pid: u32) -> Result<Option<(String, Vec<AtspiNode>)>> {
- walk_tree_bounded(pid, None, None)
+ walk_tree_bounded(pid, 0, None, None)
+ .map(|snapshot| snapshot.map(|(markdown, nodes, _)| (markdown, nodes)))
}
/// Walk the AT-SPI tree with caller-supplied node + depth caps.
@@ -636,26 +688,28 @@ pub fn walk_tree(pid: u32) -> Result<Option<(String, Vec<AtspiNode>)>> {
/// = None` keeps the historical unbounded depth. Issue #22865.
pub fn walk_tree_bounded(
pid: u32,
+ xid: u64,
max_elements: Option<usize>,
max_depth: Option<usize>,
-) -> Result<Option<(String, Vec<AtspiNode>)>> {
+) -> Result<Option<(String, Vec<AtspiNode>, Vec<(usize, i32, i32, u32, u32)>)>> {
runtime().block_on(async {
- let work = async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- match collect_visited_bounded(&conn, pid, max_elements, max_depth).await? {
- Some(visited) => Ok(Some(render(&visited))),
- None => Ok(None),
- }
+ let walk = async {
+ let conn = shared_connection().await?;
+ collect_visited_bounded(conn, pid, max_elements, max_depth).await
};
- match tokio::time::timeout(OP_TIMEOUT, work).await {
- Ok(r) => r,
+ let visited = match tokio::time::timeout(OP_TIMEOUT, walk).await {
+ Ok(result) => result?,
Err(_) => {
dlog!("walk_tree timed out for pid {pid}");
- Ok(None)
+ return Ok(None);
}
- }
+ };
+ let Some(visited) = visited else {
+ return Ok(None);
+ };
+ let (markdown, nodes) = render(&visited);
+ let bounds = element_bounds_for_visited(&visited, pid, xid).await;
+ Ok(Some((markdown, nodes, bounds)))
})
}
@@ -672,12 +726,19 @@ pub fn walk_tree_bounded(
/// dereference the xid against X11, so the synthetic value only needs to be
/// non-zero and to round-trip back from the caller.
pub fn list_windows(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo> {
+ if tokio::runtime::Handle::try_current().is_ok() {
+ return std::thread::spawn(move || list_windows_blocking(filter_pid))
+ .join()
+ .unwrap_or_default();
+ }
+ list_windows_blocking(filter_pid)
+}
+
+fn list_windows_blocking(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo> {
use crate::x11::WindowInfo;
runtime().block_on(async {
let work = async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
+ let conn = shared_connection().await?;
let zconn = conn.connection();
let root = match call(conn.root_accessible_on_registry()).await {
Some(Ok(r)) => r,
@@ -707,7 +768,7 @@ pub fn list_windows(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo> {
Some(app_ref) => app_ref,
None => continue,
};
- let app = match call(accessible_for(&conn, &app_ref)).await {
+ let app = match call(accessible_for(conn, &app_ref)).await {
Some(Ok(a)) => a,
_ => continue,
};
@@ -725,7 +786,7 @@ pub fn list_windows(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo> {
Some(frame_ref) => frame_ref,
None => continue,
};
- let frame = match call(accessible_for(&conn, &frame_ref)).await {
+ let frame = match call(accessible_for(conn, &frame_ref)).await {
Some(Ok(f)) => f,
_ => continue,
};
@@ -744,16 +805,44 @@ pub fn list_windows(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo> {
.and_then(|r| r.ok())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| app_name.clone());
+ let geometry = match call(frame.proxies()).await {
+ Some(Ok(proxies)) => match call(proxies.component()).await {
+ Some(Ok(component)) => call(component.get_extents(CoordType::Screen))
+ .await
+ .and_then(|result| result.ok()),
+ _ => None,
+ },
+ _ => None,
+ };
+ let (observed_x, observed_y, width, height) = geometry
+ .filter(|(_, _, width, height)| *width > 0 && *height > 0)
+ .map(|(x, y, width, height)| {
+ (x, y, width.max(0) as u32, height.max(0) as u32)
+ })
+ .unwrap_or((0, 0, 0, 0));
+ // On native Wayland, AT-SPI frame extents can be a stale
+ // toolkit default even when the compositor has already
+ // placed the window elsewhere. Prefer compositor metadata
+ // by pid/title so list_windows and later element geometry
+ // share the same screen origin.
+ let (x, y) = prefer_authoritative_wayland_origin(
+ authoritative_wayland_origin(cpid, 0, Some(&title)),
+ Some((observed_x, observed_y)),
+ )
+ .unwrap_or((observed_x, observed_y));
// Stable, non-zero, unique per (pid, frame ordinal).
let xid = (((cpid as u64) << 16) | (i as u64)).max(1);
out.push(WindowInfo {
xid,
pid: Some(cpid),
+ app_name: app_name.clone(),
title,
- x: 0,
- y: 0,
- width: 0,
- height: 0,
+ is_on_screen: width > 0 && height > 0,
+ z_index: None,
+ x,
+ y,
+ width,
+ height,
});
emitted += 1;
}
@@ -763,7 +852,10 @@ pub fn list_windows(filter_pid: Option<u32>) -> Vec<crate::x11::WindowInfo> {
out.push(WindowInfo {
xid: (cpid as u64).max(1),
pid: Some(cpid),
+ app_name: app_name.clone(),
title: app_name,
+ is_on_screen: true,
+ z_index: None,
x: 0,
y: 0,
width: 0,
@@ -803,14 +895,17 @@ fn pick_editable<'v, 'a>(visited: &'v [Visited<'a>]) -> Option<&'v Visited<'a>>
}
/// Try to write `text` into the best editable node in `visited` via AT-SPI
-/// EditableText (GrabFocus first so the toolkit exposes the field on an
-/// unfocused window's focused widget). Returns `Ok(true)` if the write landed,
+/// EditableText. Returns `Ok(true)` if the write landed,
/// `Ok(false)` if no editable was found / the EditableText write was rejected.
async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result<bool> {
let target = match pick_editable(visited) {
Some(t) => t,
None => return Ok(false),
};
+ write_into_editable_target(target, text).await
+}
+
+async fn write_into_editable_target(target: &Visited<'_>, text: &str) -> Result<bool> {
dlog!(
"insert target: role={:?} in_web_doc={} focused={} has_component={}",
target.role,
@@ -825,6 +920,13 @@ async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result<bool
.await
.map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
+ // A focus-free EditableText write is the strongest background route. Try it
+ // before GrabFocus: WebKitGTK can invalidate the original proxy when focus
+ // changes, and writing through that stale object then returns false.
+ if write_through_editable_proxies(&proxies, text).await? {
+ return Ok(true);
+ }
+
// Try to grab focus on the widget via AT-SPI Component.GrabFocus.
// This should give the widget internal keyboard focus without activating
// the window, allowing GTK4 (and similar toolkits) to expose EditableText
@@ -844,6 +946,13 @@ async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result<bool
dlog!("Target has no Component interface, skipping GrabFocus");
}
+ write_through_editable_proxies(&proxies, text).await
+}
+
+async fn write_through_editable_proxies(
+ proxies: &atspi::proxy::proxy_ext::Proxies<'_>,
+ text: &str,
+) -> Result<bool> {
let et = proxies
.editable_text()
.await
@@ -864,13 +973,76 @@ async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result<bool
Ok(false)
}
+/// Write into the best editable exposed by the current AT-SPI tree without
+/// falling through to synthetic X11 input.
+pub fn type_into_editable(pid: u32, text: &str) -> Result<()> {
+ bounded(
+ async {
+ let conn = shared_connection().await?;
+ let visited = collect_visited(conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ if write_into_editable(&visited, text).await? {
+ Ok(())
+ } else {
+ Err(anyhow!("no writable AT-SPI element found for pid {pid}"))
+ }
+ },
+ || Err(anyhow!("AT-SPI editable lookup timed out for pid {pid}")),
+ )
+}
+
+/// Write into the exact indexed editable exposed by the caller's snapshot.
+pub fn type_into_editable_at(pid: u32, idx: usize, text: &str) -> Result<()> {
+ bounded(
+ async {
+ let conn = shared_connection().await?;
+ let visited = collect_visited(conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let target = visited
+ .iter()
+ .filter(|node| is_indexable(node))
+ .nth(idx)
+ .ok_or_else(|| anyhow!("element {idx} not found (total: {})", visited.len()))?;
+ if write_into_editable_target(target, text).await? {
+ Ok(())
+ } else {
+ // GrabFocus can rebuild WebKitGTK's accessibility object. Walk
+ // the same index space again and retry only that exact element;
+ // never fall through to a different focused/first editable.
+ let refreshed = collect_visited(conn, pid)
+ .await?
+ .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let refreshed_target = refreshed
+ .iter()
+ .filter(|node| is_indexable(node))
+ .nth(idx)
+ .ok_or_else(|| {
+ anyhow!("element {idx} disappeared after AT-SPI focus refresh")
+ })?;
+ if write_into_editable_target(refreshed_target, text).await? {
+ Ok(())
+ } else {
+ Err(anyhow!(
+ "element {idx} is not writable through AT-SPI EditableText"
+ ))
+ }
+ }
+ },
+ || {
+ Err(anyhow!(
+ "AT-SPI editable write timed out for element {idx} in pid {pid}"
+ ))
+ },
+ )
+}
+
pub fn insert_text(pid: u32, text: &str) -> Result<bool> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
+ let conn = shared_connection().await?;
+ let visited = match collect_visited(conn, pid).await? {
Some(v) => v,
None => return Ok(false),
};
@@ -981,10 +1153,8 @@ pub fn insert_text(pid: u32, text: &str) -> Result<bool> {
pub fn focused_is_editable(pid: u32) -> Result<Option<bool>> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
+ let conn = shared_connection().await?;
+ let visited = match collect_visited(conn, pid).await? {
Some(v) => v,
None => return Ok(None),
};
@@ -1029,10 +1199,8 @@ fn screen_to_window_coords(xid: u64, screen_x: i32, screen_y: i32) -> Option<(i3
pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
+ let conn = shared_connection().await?;
+ let visited = collect_visited(conn, pid)
.await?
.ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
@@ -1084,10 +1252,8 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> {
pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> Result<()> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
+ let conn = shared_connection().await?;
+ let visited = collect_visited(conn, pid)
.await?
.ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
let target = visited
@@ -1100,46 +1266,86 @@ pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> R
.proxies()
.await
.map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
- let action = proxies
- .action()
- .await
- .map_err(|e| anyhow!("Action interface unavailable: {e}"))?;
let wanted = match direction {
"up" => ["scrollup", "scrollbackward"],
"left" => ["scrollleft", "scrollbackward"],
"right" => ["scrollright", "scrollforward"],
_ => ["scrolldown", "scrollforward"],
};
- let count = call(action.n_actions())
- .await
- .and_then(|result| result.ok())
- .unwrap_or(0);
let mut selected = None;
- for action_index in 0..count {
- if let Some(Ok(name)) = call(action.get_name(action_index)).await {
- let normalized: String = name
- .chars()
- .filter(|ch| ch.is_ascii_alphanumeric())
- .flat_map(|ch| ch.to_lowercase())
- .collect();
- if wanted.iter().any(|candidate| *candidate == normalized) {
- selected = Some(action_index);
- break;
+ let mut action_proxy = None;
+ if let Ok(action) = proxies.action().await {
+ let count = call(action.n_actions())
+ .await
+ .and_then(|result| result.ok())
+ .unwrap_or(0);
+ for action_index in 0..count {
+ if let Some(Ok(name)) = call(action.get_name(action_index)).await {
+ let normalized: String = name
+ .chars()
+ .filter(|ch| ch.is_ascii_alphanumeric())
+ .flat_map(|ch| ch.to_lowercase())
+ .collect();
+ if wanted.iter().any(|candidate| *candidate == normalized) {
+ selected = Some(action_index);
+ break;
+ }
}
}
+ action_proxy = Some(action);
}
- let action_index = selected.ok_or_else(|| {
- anyhow!("element {idx} exposes no directional scroll action for {direction}")
- })?;
- for _ in 0..amount.max(1) {
- match call(action.do_action(action_index)).await {
- Some(Ok(true)) => {}
- Some(Ok(false)) => return Err(anyhow!("scroll action returned false")),
- Some(Err(e)) => return Err(anyhow!("scroll action failed: {e}")),
- None => return Err(anyhow!("scroll action timed out")),
+
+ if let (Some(action), Some(action_index)) = (action_proxy, selected) {
+ for _ in 0..amount.max(1) {
+ match call(action.do_action(action_index)).await {
+ Some(Ok(true)) => {}
+ Some(Ok(false)) => return Err(anyhow!("scroll action returned false")),
+ Some(Err(e)) => return Err(anyhow!("scroll action failed: {e}")),
+ None => return Err(anyhow!("scroll action timed out")),
+ }
}
+ return Ok(());
+ }
+
+ if target.has_value {
+ let value = proxies
+ .value()
+ .await
+ .map_err(|e| anyhow!("Value interface unavailable: {e}"))?;
+ let current = call(value.current_value())
+ .await
+ .and_then(|result| result.ok())
+ .ok_or_else(|| anyhow!("scroll value lookup timed out"))?;
+ let minimum = call(value.minimum_value())
+ .await
+ .and_then(|result| result.ok())
+ .unwrap_or(current);
+ let maximum = call(value.maximum_value())
+ .await
+ .and_then(|result| result.ok())
+ .unwrap_or(current);
+ let increment = call(value.minimum_increment())
+ .await
+ .and_then(|result| result.ok())
+ .filter(|increment| *increment > 0.0)
+ .unwrap_or(1.0);
+ let sign = if matches!(direction, "up" | "left") {
+ -1.0
+ } else {
+ 1.0
+ };
+ let next =
+ (current + sign * increment * amount.max(1) as f64).clamp(minimum, maximum);
+ call(value.set_current_value(next))
+ .await
+ .and_then(|result| result.ok())
+ .ok_or_else(|| anyhow!("scroll value update timed out"))?;
+ return Ok(());
}
- Ok(())
+
+ Err(anyhow!(
+ "element {idx} exposes neither directional scroll actions nor Value"
+ ))
},
|| Err(anyhow!("scroll_element timed out for pid {pid}")),
)
@@ -1150,10 +1356,8 @@ pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> R
pub fn focus_element(pid: u32, idx: usize) -> Result<bool> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
+ let conn = shared_connection().await?;
+ let visited = collect_visited(conn, pid)
.await?
.ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
let target = visited
@@ -1200,13 +1404,14 @@ pub fn focus_element(pid: u32, idx: usize) -> Result<bool> {
pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Option<String>> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
+ let conn = shared_connection().await?;
+ let visited = match collect_visited(conn, pid).await? {
Some(v) => v,
None => return Ok(None),
};
+ let web_document_origin = web_document_origin_for_visited(&visited, pid)
+ .await
+ .unwrap_or((0, 0));
// Collect actionable nodes whose window-local bounds contain the point,
// then let `select_click_target` pick the innermost *real actuator* —
@@ -1231,7 +1436,19 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Optio
if w <= 0 || h <= 0 {
continue;
}
- frames.push((i, x, y, w as u32, h as u32, is_passive_role(&v.role)));
+ let (document_x, document_y) = if v.in_web_doc {
+ web_document_origin
+ } else {
+ (0, 0)
+ };
+ frames.push((
+ i,
+ x + document_x,
+ y + document_y,
+ w as u32,
+ h as u32,
+ is_passive_role(&v.role),
+ ));
}
let Some(idx) = select_click_target(&frames, win_x, win_y) else {
@@ -1266,12 +1483,12 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result<Optio
/// (a label inside the button) → a silent no-op that still returns `Some`
/// ("false success"). And on native Wayland there is no virtual-pointer click to
/// fall back to (Mutter drops synthetic pointer events). This routine instead
-/// reuses [`get_all_element_bounds`] — the SAME screen frames `get_window_state`
-/// exposes to the agent, reconstructed via the GNOME Shell helper on Wayland and
-/// `_GTK_FRAME_EXTENTS` on X11 — and actuates by `element_index`, the click path
-/// already verified working. So "click at pixel (x,y)" becomes "click the
-/// element the agent sees there", with no pointer injection and no reliance on
-/// `CoordType::Screen` (which GTK4 reports as (0,0)).
+/// uses the SAME screen-frame reconstruction that `get_window_state` exposes to
+/// the agent, via the GNOME Shell helper on Wayland and `_GTK_FRAME_EXTENTS` on
+/// X11, and actuates by `element_index`, the click path already verified
+/// working. So "click at pixel (x,y)" becomes "click the element the agent sees
+/// there", with no pointer injection and no reliance on `CoordType::Screen`
+/// (which GTK4 reports as (0,0)).
///
/// `screen_x`/`screen_y` are full-display screen pixels (what the vision
/// screenshot and `get_window_state` frames are in). Returns `Ok(Some(action))`
@@ -1285,20 +1502,21 @@ pub fn perform_action_at_screen_point(
) -> Result<Option<String>> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = match collect_visited(&conn, pid).await? {
+ let conn = shared_connection().await?;
+ let visited = match collect_visited(conn, pid).await? {
Some(v) => v,
None => return Ok(None),
};
+ let web_document_origin = web_document_origin_for_visited(&visited, pid)
+ .await
+ .unwrap_or((0, 0));
// Reconstruct each indexable element's SCREEN frame the same way
// get_window_state does: WINDOW-relative extents (GTK4 reports these
// correctly; Screen is (0,0)) plus the window's screen origin (the
// GNOME Shell helper on Wayland, _GTK_FRAME_EXTENTS on X11). When no
// offset resolves, fall back to CoordType::Screen (correct on Qt/GTK3).
- let offset = window_to_screen_offset(pid, xid);
+ let offset = window_to_screen_offset(pid, xid, None);
let coord = if offset.is_some() {
CoordType::Window
} else {
@@ -1327,10 +1545,15 @@ pub fn perform_action_at_screen_point(
if x == i32::MIN || y == i32::MIN || w <= 1 || h <= 1 {
continue;
}
+ let (document_x, document_y) = if node.in_web_doc {
+ web_document_origin
+ } else {
+ (0, 0)
+ };
frames.push((
idx,
- x + ox,
- y + oy,
+ x + ox + document_x,
+ y + oy + document_y,
w as u32,
h as u32,
is_passive_role(&node.role),
@@ -1403,10 +1626,8 @@ fn select_click_target(
pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
+ let conn = shared_connection().await?;
+ let visited = collect_visited(conn, pid)
.await?
.ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
@@ -1420,20 +1641,11 @@ pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {
.await
.map_err(|e| anyhow!("interface proxies unavailable: {e}"))?;
- // EditableText write. We don't gate on the cached `has_editable` flag:
- // GTK4 (and similar toolkits) only advertise the EditableText interface
- // on a widget once it holds keyboard focus, so the interface list
- // captured during the unfocused tree walk can be missing it even though
- // the element is a real editable text box. GrabFocus first (internal
- // widget focus, no window activation — same trick as `type_text`'s
- // EditableText path), then resolve the EditableText proxy live over
- // D-Bus and try to write. If the proxy genuinely isn't there the
- // `editable_text()` resolve fails and we fall through to Value below.
- if target.has_component {
- if let Ok(comp) = proxies.component().await {
- let _ = call(comp.grab_focus()).await;
- }
- }
+ // SetValue is a focus-free accessibility operation. Do not call
+ // Component.GrabFocus here: GTK may activate and raise the entire
+ // toplevel in response, violating the background contract. Toolkits
+ // that expose EditableText only while focused must return an honest
+ // unsupported error rather than changing desktop focus implicitly.
if let Ok(et) = proxies.editable_text().await {
// Replace whole contents (parity with the Windows/macOS set_value,
// which overwrite rather than insert at the caret).
@@ -1479,12 +1691,13 @@ pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> {
pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> {
bounded(
async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
+ let conn = shared_connection().await?;
+ let visited = collect_visited(conn, pid)
.await?
.ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
+ let web_document_origin = web_document_origin_for_visited(&visited, pid)
+ .await
+ .unwrap_or((0, 0));
let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
let target = action_nodes
.get(idx)
@@ -1503,13 +1716,23 @@ pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)>
// Prefer WINDOW coords + a deterministic screen offset — fixes GTK4,
// whose CoordType::Screen collapses every element to (0,0). Fall back to
// Screen on Wayland / when no X11 window resolves (offset is None).
- match window_to_screen_offset(pid, 0) {
+ match window_to_screen_offset(pid, 0, None) {
Some((ox, oy)) => {
let (x, y, w, h) = comp
.get_extents(CoordType::Window)
.await
.map_err(|e| anyhow!("getExtents failed: {e}"))?;
- Ok((x + ox, y + oy, w.max(0) as u32, h.max(0) as u32))
+ let (document_x, document_y) = if target.in_web_doc {
+ web_document_origin
+ } else {
+ (0, 0)
+ };
+ Ok((
+ x + ox + document_x,
+ y + oy + document_y,
+ w.max(0) as u32,
+ h.max(0) as u32,
+ ))
}
None => {
let (x, y, w, h) = comp
@@ -1611,7 +1834,7 @@ fn parse_gtk_frame_extents(vals: &[u32]) -> Option<(i32, i32)> {
/// path and the WINDOW reconstruction can never regress a toolkit that was
/// already correct. Also returns `None` on native Wayland (clients may not
/// query screen origins, by design) or when no X11 window resolves.
-fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> {
+fn window_to_screen_offset(pid: u32, xid: u64, title: Option<&str>) -> Option<(i32, i32)> {
if crate::wayland::is_wayland() {
// Native Wayland: clients can't query a window's screen origin, and
// AT-SPI CoordType::Screen collapses to (0,0) on Mutter. The bundled
@@ -1621,7 +1844,15 @@ fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> {
// this reconstructs real screen coords — the GNOME analogue of the X11
// `_GTK_FRAME_EXTENTS` path below. `None` (no extension) keeps the
// legacy Screen path (still (0,0), but no worse than before).
- return crate::wayland::shell_helper::window_origin_for_pid(pid);
+ let authoritative = authoritative_wayland_origin(pid, xid, title);
+ // AT-SPI discovery can only guess a native Wayland origin when the
+ // compositor exposes no geometry. Keep that observation as the final
+ // fallback so stale default placement cannot override Sway IPC or a
+ // shell helper's authoritative frame.
+ return prefer_authoritative_wayland_origin(
+ authoritative,
+ crate::wayland::observed_window_origin(pid),
+ );
}
// Resolve a usable window xid. `xid == 0` means "no hint" (get_element_bounds
// has no window context); fall back to this pid's first window — the same
@@ -1639,14 +1870,146 @@ fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> {
Some((ox + fl, oy + ft))
}
-/// Screen-coordinate bounds for every action node in the tree, keyed by the
-/// same `element_index` used by [`walk_tree`]/`get_element_bounds`.
-///
-/// Walks the application once (unlike calling `get_element_bounds` per node,
-/// which would reconnect and re-walk every time) and queries each node's
-/// `Component.GetExtents(Screen)`. Nodes without a usable Component interface,
-/// or whose extents query fails/times out, are silently skipped — the result is
-/// best-effort and never errors on a per-node hiccup.
+/// Resolve a native Wayland window origin from compositor-owned metadata.
+/// These sources are authoritative over AT-SPI's observed frame location,
+/// which can remain at a toolkit default after Sway places the real window.
+fn authoritative_wayland_origin(pid: u32, xid: u64, title: Option<&str>) -> Option<(i32, i32)> {
+ if !crate::wayland::is_wayland() {
+ return None;
+ }
+ crate::wayland::inject_accessibility_offset(pid)
+ .or_else(|| crate::wayland::sway_ipc::window_origin_for_pid(pid))
+ .or_else(|| {
+ (xid != 0)
+ .then(|| crate::wayland::sway_ipc::window_for_id(xid))
+ .flatten()
+ .map(|window| (window.x, window.y))
+ })
+ .or_else(|| {
+ (xid != 0)
+ .then(|| {
+ crate::wayland::sway_ipc::list_windows().and_then(|_| {
+ crate::wayland::window_geometry(xid)
+ .map(|(window_x, window_y, _, _)| (window_x, window_y))
+ })
+ })
+ .flatten()
+ })
+ .or_else(|| crate::wayland::shell_helper::window_origin_for_pid(pid))
+ .or_else(|| title.and_then(crate::wayland::sway_ipc::window_origin_for_title))
+}
+
+fn prefer_authoritative_wayland_origin(
+ authoritative: Option<(i32, i32)>,
+ observed: Option<(i32, i32)>,
+) -> Option<(i32, i32)> {
+ authoritative.or(observed)
+}
+
+fn combine_wayland_content_offsets(
+ compositor: Option<(i32, i32)>,
+ document: Option<(i32, i32)>,
+) -> Option<(i32, i32)> {
+ match (compositor, document) {
+ (Some((cx, cy)), Some((dx, dy))) => Some((cx + dx, cy + dy)),
+ (Some(offset), None) | (None, Some(offset)) => Some(offset),
+ (None, None) => None,
+ }
+}
+
+/// Offset of embedded web content inside a captured Wayland toplevel.
+/// Compositor decorations and toolkit document offsets are independent and
+/// therefore additive: choosing one or the other leaves WebKit controls one
+/// title bar away from the pixels shown to the caller.
+async fn web_document_origin_for_visited(visited: &[Visited<'_>], pid: u32) -> Option<(i32, i32)> {
+ if !crate::wayland::is_wayland() {
+ return None;
+ }
+ let sway_window = crate::wayland::sway_ipc::window_for_pid(pid);
+ let compositor = sway_window
+ .as_ref()
+ .map(|window| (window.content_x, window.content_y));
+ let document = visited
+ .iter()
+ .filter(|node| node.has_component)
+ .filter(|node| is_document_role(&node.role) || node.in_web_doc)
+ .min_by_key(|node| node.depth);
+ let document = if let Some(document) = document {
+ match call(document.acc.proxies()).await {
+ Some(Ok(proxies)) => match call(proxies.component()).await {
+ Some(Ok(component)) => match call(component.get_extents(CoordType::Window)).await {
+ Some(Ok((x, y, width, height))) if x >= 0 && y >= 0 => {
+ let inferred_top = match (compositor, sway_window.as_ref()) {
+ (Some((_, 0)), Some(window))
+ if width > 0
+ && height > 0
+ && (i64::from(window.width) - i64::from(width)).abs() <= 4
+ && i64::from(window.height) > i64::from(height) =>
+ {
+ (i64::from(window.height) - i64::from(height))
+ .min(i64::from(i32::MAX)) as i32
+ }
+ _ => 0,
+ };
+ Some((x, y.max(inferred_top)))
+ }
+ _ => None,
+ },
+ _ => None,
+ },
+ _ => None,
+ }
+ } else {
+ None
+ };
+ let combined = combine_wayland_content_offsets(compositor, document);
+ dlog!(
+ "Wayland web content offset: compositor={compositor:?} document={document:?} combined={combined:?}"
+ );
+ combined
+}
+
+fn screen_extent_rebase(
+ x11_origin: (i32, i32),
+ accessible_frame_origin: (i32, i32),
+) -> Option<(i32, i32)> {
+ // Chromium's broken "Screen" provider is rooted at the renderer-local
+ // origin. A legitimate screen provider may differ from the X11 client
+ // origin by title-bar/CSD extents; rebasing that small decoration delta
+ // would move otherwise-correct GTK coordinates off their controls.
+ if accessible_frame_origin.0.abs() <= 2 && accessible_frame_origin.1.abs() <= 2 {
+ Some((
+ x11_origin.0 - accessible_frame_origin.0,
+ x11_origin.1 - accessible_frame_origin.1,
+ ))
+ } else {
+ None
+ }
+}
+
+fn rebase_renderer_window_offset(
+ mut offset: (i32, i32),
+ frame_origin: Option<(i32, i32)>,
+) -> (i32, i32) {
+ if let Some((frame_x, frame_y)) = frame_origin {
+ // Chromium may expose a negative renderer-local frame origin. Rebase
+ // that shape, but keep positive content insets: its descendants are
+ // already relative to the content origin and subtracting the inset
+ // moves first-row controls above the captured Wayland window.
+ if frame_x < 0 {
+ offset.0 = offset.0.saturating_sub(frame_x);
+ }
+ if frame_y < 0 {
+ offset.1 = offset.1.saturating_sub(frame_y);
+ }
+ }
+ offset
+}
+
+/// Screen-coordinate bounds for the exact visited sequence rendered into the
+/// current snapshot. Nodes without a usable Component interface, or whose
+/// extents query fails/times out, are omitted rather than borrowing another
+/// live traversal's ordinal.
///
/// GTK4 caveat: GTK4's AT-SPI bridge returns `GetExtents(Screen)` as `(0,0)`
/// for every element (issue #1564 / the #1739 a11y rework), so a screen query
@@ -1657,95 +2020,243 @@ fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> {
/// offset is just the X11 origin and the result matches the old screen path.
///
/// Returns `(element_index, x, y, width, height)` tuples.
-pub fn get_all_element_bounds(pid: u32, xid: u64) -> Result<Vec<(usize, i32, i32, u32, u32)>> {
- bounded(
- async {
- let conn = AccessibilityConnection::new()
- .await
- .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?;
- let visited = collect_visited(&conn, pid)
- .await?
- .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?;
-
- // Query WINDOW-relative extents and add a deterministic screen offset
- // (X11 window origin + GTK4 CSD inset). This fixes GTK4 — whose
- // CoordType::Screen reports every element at (0,0) — by using the
- // distinct per-widget WINDOW coords instead. On Wayland / when no X11
- // window resolves, `offset` is None and we keep the legacy Screen path
- // so non-X11 behaviour is unchanged.
- let offset = window_to_screen_offset(pid, xid);
- let coord = if offset.is_some() {
- CoordType::Window
- } else {
- CoordType::Screen
+async fn element_bounds_for_visited(
+ visited: &[Visited<'_>],
+ pid: u32,
+ xid: u64,
+) -> Vec<(usize, i32, i32, u32, u32)> {
+ // Query WINDOW-relative extents and add a deterministic screen offset
+ // (X11 window origin + GTK4 CSD inset). This fixes GTK4 — whose
+ // CoordType::Screen reports every element at (0,0) — by using the
+ // distinct per-widget WINDOW coords instead. On Wayland / when no X11
+ // window resolves, `offset` is None and we keep the legacy Screen path
+ // so non-X11 behaviour is unchanged.
+ let window_title = visited.iter().find_map(|node| {
+ matches!(
+ node.role.to_ascii_lowercase().as_str(),
+ "frame" | "window" | "dialog" | "alert" | "file chooser"
+ )
+ .then_some(node.name.as_str())
+ });
+ let offset = window_to_screen_offset(pid, xid, window_title);
+ let coord = if offset.is_some() {
+ CoordType::Window
+ } else {
+ CoordType::Screen
+ };
+ // Chromium on X11 labels its component extents as Screen while
+ // returning coordinates relative to the renderer frame. Rebase
+ // those values by comparing the top-level accessible frame with
+ // the actual X11 window origin. Correct screen-coordinate providers
+ // produce a zero delta; Chromium's local (0,0) frame produces the
+ // required window-origin delta. GTK's explicit Window-coordinate
+ // path above remains authoritative when available.
+ let screen_rebase = if offset.is_none() && !crate::wayland::is_wayland() && xid != 0 {
+ let x11_origin = x11_window_origin(xid);
+ let frame = visited.iter().find(|node| {
+ node.has_component
+ && matches!(
+ node.role.to_ascii_lowercase().as_str(),
+ "frame" | "window" | "dialog" | "alert" | "file chooser"
+ )
+ });
+ if let (Some(origin), Some(frame)) = (x11_origin, frame) {
+ let accessible_origin = match call(frame.acc.proxies()).await {
+ Some(Ok(proxies)) => match call(proxies.component()).await {
+ Some(Ok(component)) => {
+ match call(component.get_extents(CoordType::Screen)).await {
+ Some(Ok((x, y, _, _))) => Some((x, y)),
+ _ => None,
+ }
+ }
+ _ => None,
+ },
+ _ => None,
};
- let (offset_x, offset_y) = offset.unwrap_or((0, 0));
- if let Some((ox, oy)) = offset {
- dlog!("element bounds: WINDOW coords + screen offset ({ox},{oy})");
+ accessible_origin.and_then(|frame_origin| screen_extent_rebase(origin, frame_origin))
+ } else {
+ None
+ }
+ } else {
+ None
+ };
+ // Renderer bridges can expose Window coordinates relative to an internal
+ // frame whose origin is not (0,0) (Chromium commonly reports a negative
+ // title-bar offset). Normalize that frame to the compositor window origin
+ // before adding the screen offset. Native GTK reports (0,0), so this is a
+ // no-op there.
+ let window_frame_origin = if offset.is_some() {
+ let frame = visited.iter().find(|node| {
+ node.has_component
+ && matches!(
+ node.role.to_ascii_lowercase().as_str(),
+ "frame" | "window" | "dialog" | "alert" | "file chooser"
+ )
+ });
+ if let Some(frame) = frame {
+ match call(frame.acc.proxies()).await {
+ Some(Ok(proxies)) => match call(proxies.component()).await {
+ Some(Ok(component)) => {
+ match call(component.get_extents(CoordType::Window)).await {
+ Some(Ok((x, y, _, _))) => Some((x, y)),
+ _ => None,
+ }
+ }
+ _ => None,
+ },
+ _ => None,
}
+ } else {
+ None
+ }
+ } else {
+ None
+ };
+ // Add compositor decorations and the embedded document origin for web
+ // descendants only. Electron commonly contributes zero for both; WebKitGTK
+ // under Sway needs the sum.
+ let web_document_origin = if offset.is_some() {
+ web_document_origin_for_visited(visited, pid).await
+ } else {
+ None
+ };
+ let (offset_x, offset_y) = rebase_renderer_window_offset(
+ offset.or(screen_rebase).unwrap_or((0, 0)),
+ window_frame_origin,
+ );
+ if let Some((ox, oy)) = offset {
+ dlog!("element bounds: WINDOW coords + screen offset ({ox},{oy})");
+ } else if let Some((ox, oy)) = screen_rebase {
+ dlog!("element bounds: SCREEN coords + X11 frame rebase ({ox},{oy})");
+ }
- let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
- // Each element costs ~3 D-Bus round-trips (proxies + component +
- // GetExtents). Big trees (geany exposes ~787 nodes) would grind for
- // minutes and time out callers, so cap the walk; pre-order means the
- // first nodes are the window chrome / toolbars that are actually
- // visible, which is what bounds consumers (overlays, targeting) need.
- const MAX_BOUNDS_NODES: usize = 150;
- // Hard wall-clock budget for the whole collection: on pathological
- // trees individual D-Bus calls each burn up to CALL_TIMEOUT (geany's
- // unrealized nodes did exactly that), so a per-node cap alone can
- // still add up to minutes. Return whatever was collected in time.
- let deadline = std::time::Instant::now() + Duration::from_secs(20);
- let mut out = Vec::with_capacity(action_nodes.len().min(MAX_BOUNDS_NODES));
- for (idx, node) in action_nodes.iter().enumerate().take(MAX_BOUNDS_NODES) {
- if std::time::Instant::now() >= deadline {
- dlog!("get_all_element_bounds: 20s budget exhausted at node {idx}; returning {} bound(s)", out.len());
- break;
- }
- if !node.has_component {
- continue;
- }
- let proxies = match call(node.acc.proxies()).await {
- Some(Ok(p)) => p,
- _ => continue,
- };
- let comp = match call(proxies.component()).await {
- Some(Ok(c)) => c,
- _ => continue,
- };
- if let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await {
- // Unrealized widgets (e.g. items inside closed menus/popovers)
- // report GetExtents as the i32::MIN sentinel and/or a degenerate
- // 0x0 / 1x1 size. Emitting those poisons downstream consumers
- // (overlay renderers, click targeting), so keep only elements
- // with plausible on-screen geometry. (Validate the raw extents,
- // before applying the screen offset, so the sentinel check still
- // catches unrealized widgets.)
- if x == i32::MIN
- || y == i32::MIN
- || x < -16384
- || y < -16384
- || w <= 1
- || h <= 1
- {
- continue;
- }
- out.push((idx, x + offset_x, y + offset_y, w as u32, h as u32));
- }
+ let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect();
+ // Hard wall-clock budget for the whole collection: on pathological
+ // trees individual D-Bus calls each burn up to CALL_TIMEOUT (geany's
+ // unrealized nodes did exactly that). Return whatever was collected
+ // in time, but do not impose an index-based node cap: a cap silently
+ // stripped frames from valid controls later in renderer trees and
+ // made PX targeting depend on DOM order.
+ let deadline = std::time::Instant::now() + Duration::from_secs(20);
+ let mut out = Vec::with_capacity(action_nodes.len());
+ for (idx, node) in action_nodes.iter().enumerate() {
+ if std::time::Instant::now() >= deadline {
+ dlog!(
+ "snapshot bounds: 20s budget exhausted at node {idx}; returning {} bound(s)",
+ out.len()
+ );
+ break;
+ }
+ if !node.has_component {
+ continue;
+ }
+ let proxies = match call(node.acc.proxies()).await {
+ Some(Ok(p)) => p,
+ _ => continue,
+ };
+ let comp = match call(proxies.component()).await {
+ Some(Ok(c)) => c,
+ _ => continue,
+ };
+ if let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await {
+ // Unrealized widgets (e.g. items inside closed menus/popovers)
+ // report GetExtents as the i32::MIN sentinel and/or a degenerate
+ // 0x0 / 1x1 size. Emitting those poisons downstream consumers
+ // (overlay renderers, click targeting), so keep only elements
+ // with plausible on-screen geometry. (Validate the raw extents,
+ // before applying the screen offset, so the sentinel check still
+ // catches unrealized widgets.)
+ if x == i32::MIN || y == i32::MIN || x < -16384 || y < -16384 || w <= 1 || h <= 1 {
+ continue;
}
- Ok(out)
- },
- || {
- dlog!("get_all_element_bounds timed out for pid {pid}; returning no bounds");
- Ok(Vec::new())
- },
- )
+ let (document_x, document_y) = if node.in_web_doc {
+ web_document_origin.unwrap_or((0, 0))
+ } else {
+ (0, 0)
+ };
+ out.push((
+ idx,
+ x + offset_x + document_x,
+ y + offset_y + document_y,
+ w as u32,
+ h as u32,
+ ));
+ }
+ }
+ out
}
#[cfg(test)]
mod coord_tests {
use super::parse_gtk_frame_extents;
- use super::{is_passive_role, select_click_target};
+ use super::{
+ combine_wayland_content_offsets, is_indexable_capabilities, is_passive_role,
+ is_web_process_bus, prefer_authoritative_wayland_origin, rebase_renderer_window_offset,
+ screen_extent_rebase, select_click_target,
+ };
+
+ #[test]
+ fn editable_only_nodes_are_addressable() {
+ assert!(is_indexable_capabilities(false, true, false));
+ assert!(is_indexable_capabilities(true, false, false));
+ assert!(is_indexable_capabilities(false, false, true));
+ assert!(!is_indexable_capabilities(false, false, false));
+ }
+
+ #[test]
+ fn screen_extents_are_rebased_from_accessible_frame_to_x11_origin() {
+ assert_eq!(screen_extent_rebase((604, 80), (0, 0)), Some((604, 80)));
+ assert_eq!(screen_extent_rebase((604, 100), (604, 80)), None);
+ assert_eq!(screen_extent_rebase((604, 80), (604, 80)), None);
+ }
+
+ #[test]
+ fn renderer_window_offset_only_rebases_negative_frame_origins() {
+ assert_eq!(
+ rebase_renderer_window_offset((100, 50), Some((-8, -29))),
+ (108, 79)
+ );
+ assert_eq!(
+ rebase_renderer_window_offset((100, 50), Some((0, 29))),
+ (100, 50)
+ );
+ }
+
+ #[test]
+ fn compositor_origin_wins_over_stale_accessibility_observation() {
+ assert_eq!(
+ prefer_authoritative_wayland_origin(Some((0, 0)), Some((120, 120))),
+ Some((0, 0))
+ );
+ assert_eq!(
+ prefer_authoritative_wayland_origin(None, Some((120, 120))),
+ Some((120, 120))
+ );
+ }
+
+ #[test]
+ fn wayland_compositor_and_document_offsets_are_additive() {
+ assert_eq!(
+ combine_wayland_content_offsets(Some((0, 47)), Some((0, 0))),
+ Some((0, 47))
+ );
+ assert_eq!(
+ combine_wayland_content_offsets(Some((2, 20)), Some((0, 47))),
+ Some((2, 67))
+ );
+ assert_eq!(
+ combine_wayland_content_offsets(None, Some((0, 47))),
+ Some((0, 47))
+ );
+ assert_eq!(combine_wayland_content_offsets(None, None), None);
+ }
+
+ #[test]
+ fn webkit_web_process_bus_marks_roleless_document_subtrees() {
+ assert!(is_web_process_bus("org.webkitgtk.WebProcess.1234"));
+ assert!(is_web_process_bus("org.example.WebProcess"));
+ assert!(!is_web_process_bus(":1.42"));
+ }
#[test]
fn click_target_prefers_button_over_its_inner_label() {
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs
index 4e60dcdef0..b7d21eb8c9 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs
@@ -304,10 +304,36 @@ async fn check_wayland_backend() -> CheckEntry {
);
}
};
+ let remote_desktop_portal_reachable = if portal_input_enabled() {
+ tokio::task::spawn_blocking(probe_portal_remote_desktop)
+ .await
+ .ok()
+ .and_then(|r| r.ok())
+ .unwrap_or(false)
+ } else {
+ false
+ };
+ classify_wayland_backend(
+ &snap,
+ portal_input_enabled(),
+ remote_desktop_portal_reachable,
+ target_activation_available(),
+ )
+}
+
+fn classify_wayland_backend(
+ snap: &WaylandManagers,
+ portal_libei_enabled: bool,
+ remote_desktop_portal_reachable: bool,
+ target_activation_available: bool,
+) -> CheckEntry {
let msg = format!(
- "foreign-toplevel={ftl}, screencopy={cap}, virtual-pointer={vp}, wl_shm={shm}",
+ "foreign-toplevel={ftl}, screencopy={cap}, ext-image-copy={ext_cap}, \
+ ext-output-source={ext_src}, virtual-pointer={vp}, wl_shm={shm}",
ftl = snap.foreign_toplevel,
cap = snap.screencopy,
+ ext_cap = snap.ext_image_copy_capture,
+ ext_src = snap.ext_output_image_capture_source,
vp = snap.virtual_pointer,
shm = snap.wl_shm,
);
@@ -317,15 +343,52 @@ async fn check_wayland_backend() -> CheckEntry {
format!("All wlroots manager globals advertised ({msg})."),
);
}
- // Input-injection backend check (#1982). A non-wlroots compositor
- // (KWin/Plasma, Mutter/GNOME) advertises no zwlr_virtual_pointer; on those
- // the ONLY working input path is libei via xdg-desktop-portal. If this
- // binary was built without `portal-libei` (the published tarball is — see
- // #1967), input injection has no backend and silently no-ops: the agent
- // cursor renders but clicks/keys are never delivered, while list_windows
- // and capture still work. Report that explicitly instead of the misleading
- // "input may fall back" partial-pass below.
- if !snap.virtual_pointer && !crate::wayland::PORTAL_LIBEI_ENABLED {
+ if !snap.virtual_pointer {
+ if remote_desktop_portal_reachable && target_activation_available {
+ return CheckEntry::pass(
+ NAME_WAYLAND_BACKEND,
+ format!(
+ "No wlroots virtual-pointer advertised ({msg}), but this \
+ portal/libei build can reach the RemoteDesktop portal \
+ (proxy reachability only — the full create_session → \
+ select_devices → start → connect_to_eis handshake is NOT \
+ exercised here, to avoid a consent prompt on every doctor \
+ run, so this is not a guarantee that injection succeeds). \
+ The compositor helper also provides verified target \
+ activation before focus-bound portal input."
+ ),
+ );
+ }
+ if remote_desktop_portal_reachable {
+ return CheckEntry::fail(
+ NAME_WAYLAND_BACKEND,
+ format!(
+ "The RemoteDesktop portal is reachable, but this compositor \
+ has no verified target-activation adapter ({msg}). Portal/libei \
+ input is global and would otherwise affect whichever window is \
+ focused, potentially the wrong application, so cua-driver \
+ refuses foreground dispatch."
+ ),
+ "On GNOME, install and enable the bundled WinRects Shell helper, then \
+ log out and back in. KDE foreground input remains unavailable until \
+ a target-addressable KWin activation adapter is installed; AX actions \
+ and exact background refusals remain usable.",
+ );
+ }
+ if portal_libei_enabled {
+ return CheckEntry::fail(
+ NAME_WAYLAND_BACKEND,
+ format!(
+ "No wlroots virtual-pointer advertised ({msg}) and the \
+ portal/libei RemoteDesktop backend is compiled in but not \
+ reachable on this session; clicks and key presses have no \
+ native Wayland input backend."
+ ),
+ "Ensure xdg-desktop-portal and a desktop backend such as \
+ xdg-desktop-portal-gnome or xdg-desktop-portal-kde are running \
+ on the session bus, or run under XWayland.",
+ );
+ }
return CheckEntry::fail(
NAME_WAYLAND_BACKEND,
format!(
@@ -336,31 +399,31 @@ async fn check_wayland_backend() -> CheckEntry {
screen capture are unaffected."
),
"Use the portal-enabled Linux build (compiled with --features \
- portal-libei) for input on KDE Plasma / GNOME, or a wlroots \
+ portal-input) for input on KDE Plasma / GNOME, or a wlroots \
compositor (sway, labwc, hyprland) where zwlr_virtual_pointer exists.",
);
}
- // Partial-pass: list_windows + capture both work, but virtual-pointer
- // input is missing. Require `wl_shm` here too — `check_screen_capture_capability`
- // gates on both `screencopy && wl_shm`, so excluding `wl_shm` from the
- // partial-pass verdict would let the matrices disagree on degenerate
- // compositors that omit it.
+ // Partial-pass: list_windows + capture both work, but some optional
+ // wlroots globals are absent. Require `wl_shm` here too —
+ // `check_screen_capture_capability` gates on both `screencopy && wl_shm`,
+ // so excluding `wl_shm` from the partial-pass verdict would let the
+ // matrices disagree on degenerate compositors that omit it.
if snap.foreign_toplevel && snap.screencopy && snap.wl_shm {
return CheckEntry::pass(
NAME_WAYLAND_BACKEND,
format!(
- "Core wlroots manager globals available; some optional globals missing ({msg}). \
- Input may fall back where virtual-pointer is absent."
+ "Core wlroots manager globals available; some optional globals missing ({msg})."
),
);
}
CheckEntry::fail(
NAME_WAYLAND_BACKEND,
format!(
- "Compositor does not advertise the wlroots manager globals cua-driver \
- needs ({msg})."
+ "Compositor does not advertise a complete native Wayland backend \
+ set ({msg})."
),
- "Use a wlroots-based compositor (sway, labwc, hyprland) or run under XWayland.",
+ "Use a wlroots-based compositor (sway, labwc, hyprland), a portal/libei \
+ build on GNOME/KDE, or run under XWayland.",
)
}
@@ -448,11 +511,34 @@ fn wayland_env_name() -> &'static str {
"CUA_DRIVER_RS_ENABLE_WAYLAND"
}
+#[cfg(target_os = "linux")]
+fn portal_input_enabled() -> bool {
+ crate::wayland::PORTAL_INPUT_ENABLED
+}
+
+#[cfg(not(target_os = "linux"))]
+fn portal_input_enabled() -> bool {
+ false
+}
+
+#[cfg(target_os = "linux")]
+fn target_activation_available() -> bool {
+ crate::wayland::shell_helper::list_windows(None).is_some()
+}
+
+#[cfg(not(target_os = "linux"))]
+fn target_activation_available() -> bool {
+ false
+}
+
+#[cfg(target_os = "linux")]
+type WaylandManagers = crate::wayland::WaylandManagers;
+
/// Snapshot of wlroots manager globals. The non-Linux stub returns an empty
/// snapshot so off-platform builds stay green; doctor short-circuits before
/// reaching it via [`is_wayland_session`].
#[cfg(target_os = "linux")]
-fn probe_wayland_managers() -> anyhow::Result<crate::wayland::WaylandManagers> {
+fn probe_wayland_managers() -> anyhow::Result<WaylandManagers> {
crate::wayland::probe_managers()
}
@@ -471,6 +557,45 @@ fn probe_portal_screenshot() -> anyhow::Result<bool> {
Ok(false)
}
+#[cfg(target_os = "linux")]
+fn probe_portal_remote_desktop() -> anyhow::Result<bool> {
+ #[cfg(feature = "portal-input")]
+ {
+ use ashpd::desktop::remote_desktop::RemoteDesktop;
+
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for RemoteDesktop probe: {e}"))?;
+
+ rt.block_on(async {
+ match RemoteDesktop::new().await {
+ Ok(_) => Ok(true),
+ Err(e) => {
+ let msg = format!("{e}");
+ if msg.contains("ServiceUnknown")
+ || msg.contains("NameHasNoOwner")
+ || msg.contains("NotFound")
+ {
+ Ok(false)
+ } else {
+ Err(anyhow::anyhow!("portal RemoteDesktop probe failed: {e}"))
+ }
+ }
+ }
+ })
+ }
+ #[cfg(not(feature = "portal-input"))]
+ {
+ Ok(false)
+ }
+}
+
+#[cfg(not(target_os = "linux"))]
+fn probe_portal_remote_desktop() -> anyhow::Result<bool> {
+ Ok(false)
+}
+
/// Stub of `wayland::WaylandManagers` so off-Linux builds compile. Always
/// reports nothing advertised — non-Linux code paths never call this.
#[cfg(not(target_os = "linux"))]
@@ -541,6 +666,60 @@ mod tests {
}
}
+ #[test]
+ fn wayland_backend_passes_on_non_wlroots_when_portal_libei_backend_is_reachable() {
+ let snap = WaylandManagers {
+ foreign_toplevel: false,
+ screencopy: false,
+ ext_image_copy_capture: false,
+ ext_output_image_capture_source: false,
+ virtual_pointer: false,
+ wl_shm: true,
+ };
+
+ let entry = classify_wayland_backend(&snap, true, true, true);
+
+ assert_eq!(entry.status, CheckStatus::Pass);
+ assert!(entry.message.contains("portal/libei"), "{}", entry.message);
+ assert!(entry.message.contains("RemoteDesktop"), "{}", entry.message);
+ }
+
+ #[test]
+ fn wayland_backend_fails_on_non_wlroots_when_portal_libei_backend_is_unreachable() {
+ let snap = WaylandManagers {
+ foreign_toplevel: false,
+ screencopy: false,
+ ext_image_copy_capture: false,
+ ext_output_image_capture_source: false,
+ virtual_pointer: false,
+ wl_shm: true,
+ };
+
+ let entry = classify_wayland_backend(&snap, true, false, false);
+
+ assert_eq!(entry.status, CheckStatus::Fail);
+ assert!(entry.message.contains("portal/libei"), "{}", entry.message);
+ assert!(entry.hint.as_deref().unwrap_or("").contains("xdg-desktop-portal"));
+ }
+
+ #[test]
+ fn wayland_backend_fails_closed_when_portal_input_cannot_target_a_window() {
+ let snap = WaylandManagers {
+ foreign_toplevel: false,
+ screencopy: false,
+ ext_image_copy_capture: false,
+ ext_output_image_capture_source: false,
+ virtual_pointer: false,
+ wl_shm: true,
+ };
+
+ let entry = classify_wayland_backend(&snap, true, true, false);
+
+ assert_eq!(entry.status, CheckStatus::Fail);
+ assert!(entry.message.contains("target-activation"), "{}", entry.message);
+ assert!(entry.message.contains("wrong application"), "{}", entry.message);
+ }
+
#[tokio::test]
async fn invoke_full_run_produces_linux_check_set() {
let provider = Arc::new(LinuxHealthProvider);
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
index 258c392cdb..33f6e26e67 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs
@@ -15,7 +15,7 @@
//! specific non-focused window the way X11/macOS/Windows can (this is a
//! platform constraint, reported honestly, like macOS pixel input being
//! driver-unverifiable). When no libei backend is available
-//! (`PORTAL_LIBEI_ENABLED == false`) the tool returns a structured
+//! (`PORTAL_INPUT_ENABLED == false`) the tool returns a structured
//! `background_unavailable` error so the caller can escalate to foreground.
//!
//! - `foreground` — activate the target first, inject, then restore the prior
@@ -92,31 +92,46 @@ pub fn delivery_mode_schema() -> Value {
/// Reason a `background` delivery cannot be performed on Wayland.
#[derive(Copy, Clone, Debug)]
pub enum BackgroundUnavailable {
- /// No libei backend (built without `portal-libei`, or the portal session
+ /// No libei backend (built without `portal-input`, or the portal session
/// was denied / unavailable). Input has no actuator at all.
NoLibeiBackend,
- /// X11/Chromium does not accept a key chord addressed to an unfocused
- /// renderer without briefly moving focus, which background delivery forbids.
- ChromiumHotkey,
+ /// X11/Chromium does not accept synthetic pointer or keyboard input
+ /// addressed to an occluded, unfocused renderer without briefly moving
+ /// focus, which background delivery forbids.
+ ChromiumInput,
+ /// The remaining backend can only inject into the globally focused widget.
+ FocusedInputOnly,
+ /// WebKitGTK rejects synthetic XSendEvent input and no real target-addressed
+ /// pointer backend is available in this session.
+ WebKitSyntheticInput,
}
impl BackgroundUnavailable {
fn code(self) -> &'static str {
match self {
Self::NoLibeiBackend => "background_unavailable",
- Self::ChromiumHotkey => "background_unavailable",
+ Self::ChromiumInput => "background_unavailable",
+ Self::FocusedInputOnly => "background_unavailable",
+ Self::WebKitSyntheticInput => "background_unavailable",
}
}
fn detail(self) -> &'static str {
match self {
Self::NoLibeiBackend => {
"no libei input backend on this Wayland compositor (built without \
- portal-libei, or the xdg-desktop-portal RemoteDesktop session was \
+ portal-input, or the xdg-desktop-portal RemoteDesktop session was \
unavailable/denied): synthetic input has no actuator"
}
- Self::ChromiumHotkey => {
- "Chromium/Electron does not accept a key chord addressed to an \
- unfocused renderer through X11 background injection"
+ Self::ChromiumInput => {
+ "Chromium/Electron does not accept pointer or keyboard input \
+ addressed to an occluded, unfocused renderer through X11 \
+ background injection"
+ }
+ Self::FocusedInputOnly => {
+ "the requested target has no focus-free input backend; the remaining XTest/X11 route can only deliver to the globally focused widget"
+ }
+ Self::WebKitSyntheticInput => {
+ "WebKitGTK rejects synthetic XSendEvent input and this session has no real target-addressed pointer backend"
}
}
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
index 6f5ced3118..063923db88 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
@@ -31,6 +31,7 @@ use x11rb::protocol::xproto::*;
use x11rb::rust_connection::RustConnection;
const CLICK_DELAY_MS: u64 = 35;
+const DOUBLE_CLICK_DELAY_MS: u64 = 50;
const KEY_DELAY_MS: u64 = 10;
#[derive(Clone, Debug)]
@@ -2018,9 +2019,17 @@ pub fn send_click_xtest_desktop(x: i32, y: i32, button: u8, count: usize) -> Res
// Absolute pointer warp (MotionNotify, detail=0 => absolute) so the button
// events that follow are delivered at (x, y).
conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?;
- for _ in 0..count.max(1) {
+ let count = count.max(1);
+ for click_index in 0..count {
conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?;
conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?;
+ if click_index + 1 < count {
+ // Chromium needs the first pair to reach the server before the
+ // second pair. A zero-gap batch produces two click events but no
+ // DOM dblclick event under Xvfb/Openbox.
+ conn.flush()?;
+ sleep(Duration::from_millis(DOUBLE_CLICK_DELAY_MS));
+ }
}
conn.flush()?;
// Round-trip so the server processes the warp+button events before this
@@ -2031,6 +2040,75 @@ pub fn send_click_xtest_desktop(x: i32, y: i32, button: u8, count: usize) -> Res
Ok(())
}
+/// Screen-absolute drag via XTest. The caller activates the target first; XTest
+/// then supplies one real press, interpolated pointer motion, and one release.
+/// This is the foreground counterpart to the window-addressed XSendEvent drag.
+pub fn send_drag_xtest_desktop(
+ from_x: i32,
+ from_y: i32,
+ to_x: i32,
+ to_y: i32,
+ button: u8,
+ duration_ms: u64,
+ steps: usize,
+) -> Result<()> {
+ use x11rb::protocol::xtest::ConnectionExt as _;
+ let (conn, screen_num) = connect_x11_for_input()?;
+ let root = conn.setup().roots[screen_num].root;
+ let steps = steps.max(1);
+ let delay = duration_ms / steps as u64;
+
+ conn.xtest_fake_input(
+ MOTION_NOTIFY_EVENT,
+ 0,
+ 0,
+ root,
+ from_x as i16,
+ from_y as i16,
+ 0,
+ )?;
+ conn.xtest_fake_input(
+ BUTTON_PRESS_EVENT,
+ button,
+ 0,
+ root,
+ from_x as i16,
+ from_y as i16,
+ 0,
+ )?;
+ conn.flush()?;
+ for step in 1..=steps {
+ let t = step as f64 / steps as f64;
+ let x = from_x as f64 + (to_x - from_x) as f64 * t;
+ let y = from_y as f64 + (to_y - from_y) as f64 * t;
+ conn.xtest_fake_input(
+ MOTION_NOTIFY_EVENT,
+ 0,
+ 0,
+ root,
+ x.round() as i16,
+ y.round() as i16,
+ 0,
+ )?;
+ conn.flush()?;
+ if delay > 0 {
+ sleep(Duration::from_millis(delay));
+ }
+ }
+ conn.xtest_fake_input(
+ BUTTON_RELEASE_EVENT,
+ button,
+ 0,
+ root,
+ to_x as i16,
+ to_y as i16,
+ 0,
+ )?;
+ conn.flush()?;
+ let _ = conn.get_input_focus()?.reply();
+ Ok(())
+}
+
/// Send a named key press to a window.
pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> {
send_key_to_target(xid, None, key, modifiers)
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
index f6b41cb766..c4ca6749b2 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs
@@ -41,6 +41,12 @@ pub mod atspi;
#[cfg(target_os = "linux")]
pub mod a11y;
+#[cfg(target_os = "linux")]
+pub mod recording_hooks;
+
+#[cfg(target_os = "linux")]
+pub mod video_wayland;
+
#[cfg(target_os = "linux")]
pub mod wayland;
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
index 80544f5157..696567f87e 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
@@ -5,7 +5,9 @@
//! from XComposite. The window covers the full display area.
//! - A background thread renders frames at ~60 Hz using tiny-skia and XShmPutImage
//! (or XPutImage fallback) with XRender ARGB compositing.
-//! - Mouse events pass through via `XShapeSelectInput(ShapeInput, empty-region)`.
+//! - XShape clips both input and visible pixels. The visible shape follows the
+//! rendered alpha mask so bare X11 window managers do not show a black
+//! full-screen ARGB window when no compositor is present.
//! - Z-ordering: `XRaiseWindow` every 80ms to stay above normal windows.
//! - Wayland: when WAYLAND_DISPLAY is set but DISPLAY is also available (XWayland),
//! the X11 path is used. Pure Wayland support is a TODO.
@@ -19,6 +21,7 @@
use std::collections::{HashMap, HashSet};
use std::sync::{Mutex, OnceLock};
+#[cfg(target_os = "linux")]
use std::time::{Duration, Instant};
#[cfg(target_os = "linux")]
@@ -140,22 +143,21 @@ pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) {
#[cfg(target_os = "linux")]
{
if crate::wayland::is_wayland() {
- let _ = crate::wayland::overlay::forward(&msg);
- // Non-wlroots compositors (GNOME Mutter / KDE) expose no
- // `zwlr_layer_shell_v1`, so the forward above is a no-op there. Drive
- // the agent cursor through the WinRects shell extension instead
- // (no-op if it isn't installed). Only the SINGLE positioning commands
- // are forwarded — never the interpolated `MoveTo` stream (the
- // extension does its own easing; the glide target is sent once from
- // `overlay_glide_to_for`).
- match &cmd {
- cursor_overlay::OverlayCommand::ClickPulse { x, y } => {
- crate::wayland::shell_helper::click_pulse(*x as i32, *y as i32);
- }
- cursor_overlay::OverlayCommand::SnapTo { x, y, .. } => {
- crate::wayland::shell_helper::move_cursor(*x as i32, *y as i32);
+ if crate::wayland::shell_helper::available() {
+ // GNOME has no layer-shell. Drive only the final positioning
+ // commands through the compositor helper; it performs its own
+ // easing and avoids starting a worker that must fail.
+ match &cmd {
+ cursor_overlay::OverlayCommand::ClickPulse { x, y } => {
+ crate::wayland::shell_helper::click_pulse(*x as i32, *y as i32);
+ }
+ cursor_overlay::OverlayCommand::SnapTo { x, y, .. } => {
+ crate::wayland::shell_helper::move_cursor(*x as i32, *y as i32);
+ }
+ _ => {}
}
- _ => {}
+ } else {
+ let _ = crate::wayland::overlay::forward(&msg);
}
}
}
@@ -196,6 +198,21 @@ pub fn current_position_for(key: &str) -> (f64, f64) {
.unwrap_or((-200.0, -200.0))
}
+pub fn current_motion_for(key: &str) -> cursor_overlay::MotionConfig {
+ RENDER
+ .lock()
+ .ok()
+ .and_then(|guard| {
+ guard.as_ref().and_then(|map| {
+ map.cursors
+ .get(key)
+ .or_else(|| map.cursors.get("default"))
+ .map(|state| state.core.motion.clone())
+ })
+ })
+ .unwrap_or_default()
+}
+
fn seed_start_if_sentinel(key: &CursorKey, target_x: f64, target_y: f64) -> bool {
const SEED_OFFSET: f64 = 140.0;
let mut guard = RENDER.lock().unwrap();
@@ -616,6 +633,7 @@ fn paint_x11(
_visual_id: u32,
pm: &tiny_skia::Pixmap,
) {
+ use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK, SO};
use x11rb::protocol::xproto::{ConnectionExt as XprotoConnectionExt, CreateGCAux, ImageFormat};
if pm.width() == 0 || pm.height() == 0 {
return;
@@ -632,15 +650,21 @@ fn paint_x11(
return;
}
- // Convert RGBA premult → BGRA premult for X11.
- let src = pm.data();
- let mut bgra: Vec<u8> = Vec::with_capacity(src.len());
- for chunk in src.chunks_exact(4) {
- bgra.push(chunk[2]); // B
- bgra.push(chunk[1]); // G
- bgra.push(chunk[0]); // R
- bgra.push(chunk[3]); // A
- }
+ let (bgra, visible_shape) = bgra_and_visible_shape(pm);
+
+ // A 32-bit ARGB window needs a compositor to blend transparent pixels.
+ // Without one, zero-alpha pixels display as opaque black. Clip the native
+ // window to the rendered non-zero alpha runs so the overlay remains usable
+ // under bare Openbox/i3/Xvfb sessions as well as composited desktops.
+ let _ = conn.shape_rectangles(
+ SO::SET,
+ SK::BOUNDING,
+ x11rb::protocol::xproto::ClipOrdering::UNSORTED,
+ win,
+ 0,
+ 0,
+ &visible_shape,
+ );
// XPutImage (ZPixmap).
let _ = conn.put_image(
@@ -660,5 +684,79 @@ fn paint_x11(
conn.flush().ok();
}
+#[cfg(target_os = "linux")]
+fn bgra_and_visible_shape(
+ pm: &tiny_skia::Pixmap,
+) -> (Vec<u8>, Vec<x11rb::protocol::xproto::Rectangle>) {
+ use x11rb::protocol::xproto::Rectangle;
+
+ let width = pm.width() as usize;
+ let height = pm.height() as usize;
+ let src = pm.data();
+ let mut bgra = Vec::with_capacity(src.len());
+ let mut rectangles = Vec::new();
+
+ for y in 0..height {
+ let mut run_start = None;
+ for x in 0..width {
+ let offset = (y * width + x) * 4;
+ let pixel = &src[offset..offset + 4];
+ bgra.extend_from_slice(&[pixel[2], pixel[1], pixel[0], pixel[3]]);
+
+ if pixel[3] != 0 {
+ run_start.get_or_insert(x);
+ } else if let Some(start) = run_start.take() {
+ rectangles.push(Rectangle {
+ x: start as i16,
+ y: y as i16,
+ width: (x - start) as u16,
+ height: 1,
+ });
+ }
+ }
+ if let Some(start) = run_start {
+ rectangles.push(Rectangle {
+ x: start as i16,
+ y: y as i16,
+ width: (width - start) as u16,
+ height: 1,
+ });
+ }
+ }
+
+ (bgra, rectangles)
+}
+
+#[cfg(all(test, target_os = "linux"))]
+mod tests {
+ use super::bgra_and_visible_shape;
+
+ #[test]
+ fn visible_shape_contains_only_nontransparent_runs() {
+ let mut pixmap = tiny_skia::Pixmap::new(4, 2).unwrap();
+ pixmap.data_mut().copy_from_slice(&[
+ 1, 2, 3, 0, 10, 20, 30, 255, 11, 21, 31, 128, 4, 5, 6, 0, 7, 8, 9, 64, 1, 1, 1, 0, 2,
+ 2, 2, 0, 12, 22, 32, 255,
+ ]);
+
+ let (bgra, rectangles) = bgra_and_visible_shape(&pixmap);
+
+ assert_eq!(&bgra[4..8], &[30, 20, 10, 255]);
+ assert_eq!(rectangles.len(), 3);
+ assert_eq!(
+ (rectangles[0].x, rectangles[0].y, rectangles[0].width),
+ (1, 0, 2)
+ );
+ assert_eq!(
+ (rectangles[1].x, rectangles[1].y, rectangles[1].width),
+ (0, 1, 1)
+ );
+ assert_eq!(
+ (rectangles[2].x, rectangles[2].y, rectangles[2].width),
+ (3, 1, 1)
+ );
+ }
+}
+
#[cfg(not(target_os = "linux"))]
fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver<OverlayMsg>) {}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
new file mode 100644
index 0000000000..e77960d3c8
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs
@@ -0,0 +1,95 @@
+//! Application-state snapshots used by trajectory recording on Linux.
+
+#[cfg(target_os = "linux")]
+pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
+ if tokio::runtime::Handle::try_current().is_ok() {
+ return std::thread::spawn(move || app_state_json_for_blocking(window_id, pid))
+ .join()
+ .ok()
+ .flatten();
+ }
+ app_state_json_for_blocking(window_id, pid)
+}
+
+#[cfg(target_os = "linux")]
+fn app_state_json_for_blocking(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
+ let pid = u32::try_from(pid?).ok()?;
+ let window_id = resolve_window_for_recording(pid, window_id)?.xid;
+ let result = crate::atspi::walk_tree(pid, window_id, None);
+ if result.nodes.is_empty() || result.tree_markdown.trim().is_empty() {
+ return None;
+ }
+ let element_count = result
+ .nodes
+ .iter()
+ .filter(|node| node.element_index.is_some())
+ .count();
+ let payload = serde_json::json!({
+ "pid": pid,
+ "window_id": window_id,
+ "element_count": element_count,
+ "tree_markdown": result.tree_markdown,
+ });
+ serde_json::to_vec_pretty(&payload).ok()
+}
+
+#[cfg(target_os = "linux")]
+pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> {
+ if tokio::runtime::Handle::try_current().is_ok() {
+ return std::thread::spawn(move || {
+ element_window_local_xy_blocking(window_id, pid, element_index)
+ })
+ .join()
+ .ok()
+ .flatten();
+ }
+ element_window_local_xy_blocking(window_id, pid, element_index)
+}
+
+#[cfg(target_os = "linux")]
+fn element_window_local_xy_blocking(
+ window_id: u64,
+ pid: i64,
+ element_index: u32,
+) -> Option<(f64, f64)> {
+ let pid = u32::try_from(pid).ok()?;
+ let (screen_x, screen_y, width, height) =
+ crate::atspi::get_element_bounds(pid, element_index as usize).ok()?;
+ let window = resolve_window_for_recording(pid, Some(window_id))?;
+ Some((
+ f64::from(screen_x - window.x) + f64::from(width) / 2.0,
+ f64::from(screen_y - window.y) + f64::from(height) / 2.0,
+ ))
+}
+
+#[cfg(target_os = "linux")]
+fn resolve_window_for_recording(
+ pid: u32,
+ window_id: Option<u64>,
+) -> Option<crate::x11::WindowInfo> {
+ let windows = crate::wayland::list_windows_dispatch(Some(pid));
+ if crate::wayland::is_wayland() {
+ // Foreign-toplevel protocol object ids are scoped to one Wayland
+ // connection. Recording hooks open a fresh connection, so re-resolve
+ // the target by pid instead of comparing an id from the action call.
+ windows.into_iter().next()
+ } else if let Some(window_id) = window_id {
+ windows.into_iter().find(|window| window.xid == window_id)
+ } else {
+ windows.into_iter().next()
+ }
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn app_state_json_for(_window_id: Option<u64>, _pid: Option<i64>) -> Option<Vec<u8>> {
+ None
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn element_window_local_xy(
+ _window_id: u64,
+ _pid: i64,
+ _element_index: u32,
+) -> Option<(f64, f64)> {
+ None
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 4a5b528569..7db83fc589 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -402,10 +402,17 @@ impl Tool for ListWindowsTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let filter_pid = args.opt_u64("pid").map(|v| v as u32);
- let windows =
+ let on_screen_only = args.bool_or("on_screen_only", false);
+ let mut windows =
tokio::task::spawn_blocking(move || crate::wayland::list_windows_dispatch(filter_pid))
.await
.unwrap_or_default();
+ if on_screen_only {
+ windows.retain(|window| window.is_on_screen);
+ }
+ if crate::wayland::is_wayland() {
+ crate::wayland::remember_observed_window_origins(&windows);
+ }
let mut lines = vec![format!("Found {} windows:", windows.len())];
for w in &windows {
lines.push(format!(
@@ -428,19 +435,16 @@ impl Tool for ListWindowsTool {
/// existing Linux callers don't break. Fully additive; no field removed,
/// no schema_version bump.
///
-/// The Linux `WindowInfo` struct (see `crate::x11::WindowInfo`) exposes
-/// neither an app name nor a visibility flag, so `app_name` is an empty
-/// string and `is_on_screen` defaults to `true` — the same best-effort
-/// default the Windows backend uses.
fn window_record_json(w: &crate::x11::WindowInfo) -> Value {
json!({
"window_id": w.xid,
"pid": w.pid,
- "app_name": "",
+ "app_name": w.app_name,
"title": w.title,
// Canonical cross-platform geometry (macOS/Windows parity).
"bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height },
- "is_on_screen": true,
+ "is_on_screen": w.is_on_screen,
+ "z_index": w.z_index,
// Legacy alias: flat fields kept inline for pre-existing callers.
"x": w.x, "y": w.y,
"width": w.width, "height": w.height,
@@ -456,7 +460,10 @@ mod list_windows_tests {
let w = crate::x11::WindowInfo {
xid: 42,
pid: Some(1234),
+ app_name: "example-app".to_owned(),
title: "Example".to_owned(),
+ is_on_screen: true,
+ z_index: Some(3),
x: 10,
y: 20,
width: 300,
@@ -480,11 +487,20 @@ mod list_windows_tests {
assert_eq!(rec["height"], json!(400));
// Cross-platform companions.
- assert_eq!(rec["app_name"], json!(""));
+ assert_eq!(rec["app_name"], json!("example-app"));
assert_eq!(rec["is_on_screen"], json!(true));
+ assert_eq!(rec["z_index"], json!(3));
assert_eq!(rec["window_id"], json!(42));
assert_eq!(rec["title"], json!("Example"));
}
+
+ #[test]
+ fn chromium_launch_detection_uses_executable_basename() {
+ assert!(chromium_family_program("/usr/bin/google-chrome-stable"));
+ assert!(chromium_family_program("CuaTestHarness.Electron"));
+ assert!(chromium_family_program("chromium-browser"));
+ assert!(!chromium_family_program("/usr/bin/gnome-text-editor"));
+ }
}
// ── get_window_state ─────────────────────────────────────────────────────────
@@ -604,9 +620,13 @@ impl Tool for GetWindowStateTool {
max_elements,
max_depth,
));
- // Best-effort per-element screen bounds (AT-SPI Component.GetExtents).
- // Tolerant: an empty/missing map never fails the call.
- let bounds = crate::atspi::get_all_element_bounds(pid, xid).unwrap_or_default();
+ // Bounds and element indices come from the same captured AT-SPI
+ // traversal. Joining two live walks by ordinal mis-associated
+ // Chromium controls when its lazy subtree changed between walks.
+ let bounds = tree_result
+ .as_ref()
+ .map(|tree| tree.bounds.clone())
+ .unwrap_or_default();
// Capture and DELIVER the screenshot alongside the tree by default — the
// grounding frame the agent cross-checks the tree against. With
// screenshot_out_file set, write to disk and surface the path instead
@@ -630,7 +650,11 @@ impl Tool for GetWindowStateTool {
Some((Some(B64.encode(&png)), None, w, h, original_w))
}
}
- Err(_) => None,
+ Err(error) => {
+ return Err(anyhow::anyhow!(
+ "window screenshot failed for window {xid}: {error}"
+ ));
+ }
}
} else {
None
@@ -851,7 +875,21 @@ impl Tool for LaunchAppTool {
let mut parts = cmd.split_whitespace();
let prog = parts.next().unwrap_or(cmd);
let rest: Vec<&str> = parts.collect();
- match std::process::Command::new(prog).args(&rest).spawn() {
+ let mut launch = std::process::Command::new(prog);
+ launch
+ .args(&rest)
+ // Enable accessibility for this child without toggling
+ // GNOME's global ScreenReaderEnabled setting (which can
+ // launch Orca). Native toolkits ignore these when they
+ // do not need them.
+ .env("ACCESSIBILITY_ENABLED", "1")
+ .env("NO_AT_BRIDGE", "0");
+ if chromium_family_program(prog)
+ && !rest.iter().any(|arg| *arg == "--force-renderer-accessibility")
+ {
+ launch.arg("--force-renderer-accessibility");
+ }
+ match launch.spawn() {
Ok(child) => {
let pid = child.id();
return Ok((
@@ -881,10 +919,15 @@ impl Tool for LaunchAppTool {
Ok(Ok((message, pid_opt, name))) => {
if let Some(pid) = pid_opt {
let windows = tokio::task::spawn_blocking(move || {
- crate::x11::list_windows(Some(pid))
- .iter()
- .map(window_record_json)
- .collect::<Vec<_>>()
+ let deadline = std::time::Instant::now()
+ + std::time::Duration::from_secs(3);
+ loop {
+ let windows = crate::wayland::list_windows_dispatch(Some(pid));
+ if !windows.is_empty() || std::time::Instant::now() >= deadline {
+ return windows.iter().map(window_record_json).collect::<Vec<_>>();
+ }
+ std::thread::sleep(std::time::Duration::from_millis(100));
+ }
})
.await
.unwrap_or_default();
@@ -913,14 +956,26 @@ impl Tool for LaunchAppTool {
}
}
+fn chromium_family_program(program: &str) -> bool {
+ let basename = std::path::Path::new(program)
+ .file_name()
+ .and_then(|name| name.to_str())
+ .unwrap_or(program)
+ .to_ascii_lowercase();
+ ["chrome", "chromium", "electron", "brave", "edge"]
+ .iter()
+ .any(|needle| basename.contains(needle))
+}
+
// ── shared helpers ────────────────────────────────────────────────────────────
-/// Resolve an AT-SPI element's center in window-local X11 coordinates.
+/// Resolve an AT-SPI element's center in window-local coordinates.
///
/// Returns `(xid, window_local_x, window_local_y)`.
-/// Looks up element bounds via pyatspi subprocess, finds the owning window
+/// Looks up element bounds via native AT-SPI, finds the owning window
/// (via `xid_hint` or the first window for `pid`), then converts screen-absolute
-/// → window-local coords via X11 translate_coordinates.
+/// → window-local coords via compositor metadata on Wayland or
+/// XTranslateCoordinates on X11.
fn resolve_element_local_coords(
pid: u32,
idx: usize,
@@ -932,6 +987,12 @@ fn resolve_element_local_coords(
let xid = if let Some(x) = xid_hint {
x
+ } else if crate::wayland::wayland_input_enabled() {
+ crate::wayland::list_windows_dispatch(Some(pid))
+ .into_iter()
+ .next()
+ .map(|window| window.xid)
+ .ok_or_else(|| anyhow::anyhow!("No Wayland windows for pid {pid}"))?
} else {
crate::x11::list_windows(Some(pid))
.into_iter()
@@ -940,6 +1001,20 @@ fn resolve_element_local_coords(
.ok_or_else(|| anyhow::anyhow!("No windows for pid {pid}"))?
};
+ if crate::wayland::wayland_input_enabled() {
+ let (window_x, window_y, window_width, window_height) =
+ crate::wayland::window_geometry(xid)
+ .ok_or_else(|| anyhow::anyhow!("No Wayland geometry for window {xid}"))?;
+ if window_width == 0 || window_height == 0 {
+ anyhow::bail!("Wayland window {xid} has no usable geometry");
+ }
+ return Ok((
+ xid,
+ screen_cx - window_x as f64,
+ screen_cy - window_y as f64,
+ ));
+ }
+
use x11rb::connection::Connection;
use x11rb::protocol::xproto::ConnectionExt as _;
use x11rb::rust_connection::RustConnection;
@@ -1150,6 +1225,159 @@ fn is_chromium_embedder(pid: u32) -> bool {
false
}
+fn is_webkitgtk_embedder(pid: u32) -> bool {
+ fn argv_is_webkit_helper(pid: u32) -> bool {
+ fs::read(format!("/proc/{pid}/cmdline"))
+ .map(|raw| {
+ let cmdline = String::from_utf8_lossy(&raw);
+ cmdline.contains("WebKitWebProcess") || cmdline.contains("WebKitNetworkProcess")
+ })
+ .unwrap_or(false)
+ }
+
+ let mut children: std::collections::HashMap<u32, Vec<u32>> = std::collections::HashMap::new();
+ let Ok(entries) = fs::read_dir("/proc") else {
+ return false;
+ };
+ for entry in entries.flatten() {
+ let Ok(child) = entry.file_name().to_string_lossy().parse::<u32>() else {
+ continue;
+ };
+ let Ok(status) = fs::read_to_string(format!("/proc/{child}/status")) else {
+ continue;
+ };
+ if let Some(ppid) = status
+ .lines()
+ .find(|line| line.starts_with("PPid:"))
+ .and_then(|line| line[5..].trim().parse::<u32>().ok())
+ {
+ children.entry(ppid).or_default().push(child);
+ }
+ }
+ let mut queue = std::collections::VecDeque::from([pid]);
+ let mut seen = std::collections::HashSet::new();
+ while let Some(current) = queue.pop_front() {
+ if !seen.insert(current) {
+ continue;
+ }
+ if argv_is_webkit_helper(current) {
+ return true;
+ }
+ if let Some(descendants) = children.get(&current) {
+ queue.extend(descendants.iter().copied());
+ }
+ }
+ false
+}
+
+fn maps_indicate_gtk(maps: &str) -> bool {
+ maps.contains("libgtk-3.so") || maps.contains("libgtk-4.so")
+}
+
+fn is_gtk_process(pid: u32) -> bool {
+ fs::read_to_string(format!("/proc/{pid}/maps"))
+ .map(|maps| maps_indicate_gtk(&maps))
+ .unwrap_or(false)
+}
+
+fn unavailable_webkit_background(
+ pid: u32,
+ delivery: crate::input::delivery::DeliveryMode,
+) -> Option<ToolResult> {
+ (!delivery.is_foreground()
+ && is_webkitgtk_embedder(pid)
+ && !crate::wayland::is_inject_mode()
+ && !crate::input::real_pointer_input_available())
+ .then(|| {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::WebKitSyntheticInput,
+ )
+ })
+}
+
+fn unavailable_webkit_keyboard_background(
+ pid: u32,
+ delivery: crate::input::delivery::DeliveryMode,
+) -> Option<ToolResult> {
+ (!delivery.is_foreground()
+ && is_webkitgtk_embedder(pid)
+ && !crate::wayland::is_inject_mode())
+ .then(|| {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ )
+ })
+}
+
+fn unavailable_gtk_keyboard_background(
+ pid: u32,
+ delivery: crate::input::delivery::DeliveryMode,
+) -> Option<ToolResult> {
+ (!delivery.is_foreground() && is_gtk_process(pid) && !crate::wayland::is_inject_mode()).then(|| {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ )
+ })
+}
+
+fn unavailable_gtk_pointer_background(
+ pid: u32,
+ delivery: crate::input::delivery::DeliveryMode,
+) -> Option<ToolResult> {
+ (!delivery.is_foreground()
+ && is_gtk_process(pid)
+ && !crate::wayland::is_inject_mode()
+ && !crate::input::real_pointer_input_available())
+ .then(|| {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ )
+ })
+}
+
+fn unavailable_wayland_focused_input_background(
+ delivery: crate::input::delivery::DeliveryMode,
+ focus_free_inject_supported: bool,
+) -> Option<ToolResult> {
+ (crate::wayland::wayland_input_enabled()
+ && !(focus_free_inject_supported && crate::wayland::is_inject_mode())
+ && !delivery.is_foreground())
+ .then(|| {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ )
+ })
+}
+
+/// Chromium's X11 renderer drops synthetic input sent to an occluded,
+/// unfocused toplevel. Returning success here would be a silent loss, so all
+/// input tools expose the same typed refusal and leave foreground activation
+/// as the explicit escalation.
+fn unavailable_chromium_background(
+ pid: u32,
+ delivery: crate::input::delivery::DeliveryMode,
+) -> Option<ToolResult> {
+ if chromium_background_must_refuse(
+ delivery.is_foreground(),
+ crate::wayland::is_inject_mode(),
+ is_chromium_embedder(pid),
+ ) {
+ Some(crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::ChromiumInput,
+ ))
+ } else {
+ None
+ }
+}
+
+fn chromium_background_must_refuse(
+ foreground: bool,
+ focus_free_inject_mode: bool,
+ chromium: bool,
+) -> bool {
+ chromium && !foreground && !focus_free_inject_mode
+}
+
/// Screen-absolute center of a window (top-left from translate_coordinates plus
/// half its geometry). Used to position the no-focus-steal scroll over the
/// window's content. Blocking — call inside spawn_blocking.
@@ -1552,17 +1780,21 @@ impl Tool for ClickTool {
// sees the cursor "click somewhere else."
overlay_glide_to_for(&cursor_id, sx as f64, sy as f64).await;
let r = tokio::task::spawn_blocking(move || {
- crate::input::send_click_xtest_desktop(sx, sy, button, n)
+ if crate::wayland::wayland_input_enabled() {
+ crate::wayland::click_desktop(sx, sy, n as u32, button)
+ } else {
+ crate::input::send_click_xtest_desktop(sx, sy, button, n)
+ }
})
.await;
return match r {
- // Screen-absolute XTEST click — never driver-verifiable (no
+ // Screen-absolute click — never driver-verifiable (no
// read-back); the caller confirms via screenshot.
Ok(Ok(())) => ToolResult::text(format!(
"✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope)."
))
.with_structured(
- json!({ "path": "xtest_desktop", "verified": false, "effect": "unverifiable" }),
+ json!({ "path": if crate::wayland::wayland_input_enabled() { "wayland_desktop" } else { "xtest_desktop" }, "verified": false, "effect": "unverifiable" }),
),
Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")),
Err(e) => ToolResult::error(format!("task error: {e}")),
@@ -1573,6 +1805,7 @@ impl Tool for ClickTool {
Ok(v) => v,
Err(e) => return e,
};
+ let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
let count = args.u64_or("count", 1) as usize;
// Surface 5: reject unknown buttons so a typo can't silently fall through
// to a left-click. Empty string keeps back-compat with old clients.
@@ -1613,7 +1846,7 @@ impl Tool for ClickTool {
};
let window_id_resolved: Option<u64> = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
- window_id.map(|v| v as u64)
+ window_id_arg.or_else(|| window_id.map(|v| v as u64))
}
cua_driver_core::element_token::ResolvedElement::None => window_id_arg,
};
@@ -1651,36 +1884,45 @@ impl Tool for ClickTool {
cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
);
- // Now perform the actual click: AT-SPI doAction(0) first (background-
- // safe, no focus steal), else XSendEvent at window-local coords. The
- // AT-SPI rung also reports whether the actuated element looked like a
- // silent no-op (passive role / no advertised action) — see
- // perform_action — so the response can flag `effect: "suspected_noop"`.
- let result =
- tokio::task::spawn_blocking(move || -> anyhow::Result<(&'static str, bool)> {
- if let Ok((_action, suspected_noop)) = crate::atspi::perform_action(pid, idx) {
- return Ok(("ax", suspected_noop));
- }
- let (xid2, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?;
- crate::input::send_click(xid2, lx as i32, ly as i32, count, button)?;
- Ok(("x11_pixel", false))
- })
- .await;
+ // Chromium can execute a genuine AT-SPI action without focus. Try
+ // that route before applying its background synthetic-input gate.
+ let ax_result =
+ tokio::task::spawn_blocking(move || crate::atspi::perform_action(pid, idx)).await;
+ if let Ok(Ok((_action, suspected_noop))) = ax_result {
+ let mut structured = json!({
+ "path": "ax",
+ "verified": false,
+ "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" },
+ });
+ if suspected_noop {
+ structured["escalation"] = non_ax_escalation();
+ }
+ return ToolResult::text(format!("Clicked element [{idx}] (pid {pid})."))
+ .with_structured(structured);
+ }
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+
+ // The AX route was unavailable. Fall back to a target-addressed
+ // X11 event for toolkits that accept it.
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+ let (xid2, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?;
+ crate::input::send_click(xid2, lx as i32, ly as i32, count, button)
+ })
+ .await;
return match result {
// An element click is never driver-verifiable (no read-back) —
// verified:false; the caller confirms via screenshot. `effect` is
// the richer signal: a passive/role-mismatched AT-SPI actuation is
// a likely no-op (→ cross to vision/pixel), otherwise the dispatch
// was fine but unconfirmable.
- Ok(Ok((path, suspected_noop))) => {
- let mut structured = json!({
- "path": path,
+ Ok(Ok(())) => {
+ let structured = json!({
+ "path": "x11_pixel",
"verified": false,
- "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" },
+ "effect": "unverifiable",
});
- if suspected_noop {
- structured["escalation"] = non_ax_escalation();
- }
ToolResult::text(format!("Clicked element [{idx}] (pid {pid})."))
.with_structured(structured)
}
@@ -1689,6 +1931,10 @@ impl Tool for ClickTool {
};
}
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+
// Coordinate-based path.
let xid = match args.opt_u64("window_id") {
Some(v) => v,
@@ -1719,13 +1965,20 @@ impl Tool for ClickTool {
cursor_id.clone(),
cursor_overlay::OverlayCommand::PinAbove(xid),
);
- // Resolve the screen point the cursor glides to. On native Wayland the
- // agent already passes screen coordinates (the vision screenshot and
- // `get_window_state` frames are screen-space, and `window_local_to_screen`
- // — an X11 `translate_coordinates` call — can't run with DISPLAY unset),
- // so use them directly. On X11 the coords are window-local; translate.
- let glide_target = if crate::wayland::is_wayland() {
- Some((x, y))
+ // Resolve the screen point the cursor glides to. Tool coordinates are
+ // always window-local screenshot pixels; native Wayland translates
+ // through compositor/AT-SPI geometry while X11 uses XTranslateCoordinates.
+ let wayland_output_point = if crate::wayland::wayland_input_enabled() {
+ Some(crate::wayland::window_local_to_output(
+ xid,
+ x.round() as i32,
+ y.round() as i32,
+ ))
+ } else {
+ None
+ };
+ let glide_target = if let Some((sx, sy)) = wayland_output_point {
+ Some((sx as f64, sy as f64))
} else {
tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y))
.await
@@ -1741,13 +1994,13 @@ impl Tool for ClickTool {
}
let (xi, yi) = (x as i32, y as i32);
+ let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi));
let cursor_id_for_task = cursor_id.clone();
// delivery_mode: background (default) = no-focus-steal injection;
// foreground = activate the target window (EWMH) first, then inject,
// then restore prior active. Mirrors macOS/Windows.
- let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> {
- if crate::wayland::is_wayland() {
+ if crate::wayland::wayland_input_enabled() {
// Vision/pixel click on native Wayland. Mutter drops synthetic
// virtual-pointer events (the `wayland::click` warp doesn't land),
// so for a plain left single click resolve the screen pixel to the
@@ -1755,21 +2008,29 @@ impl Tool for ClickTool {
// `element_index` — the coordinate-free path already verified
// working. (x,y) are screen coords here, matching the frames in
// `get_window_state`. Miss → fall through to the injection paths.
- if button == 1 && count == 1 {
+ if !delivery.is_foreground() && button == 1 && count == 1 {
if let Ok(Some(_)) =
- crate::atspi::perform_action_at_screen_point(pid, xid, xi, yi)
+ crate::atspi::perform_action_at_screen_point(
+ pid,
+ xid,
+ output_x,
+ output_y,
+ )
{
return Ok("wayland_atspi");
}
}
if crate::wayland::is_inject_mode() {
crate::wayland::inject_click(xid, x, y, count as u32, button)?;
- return Ok("wayland_libei");
+ return Ok("wayland_cua_compositor");
+ }
+ if !delivery.is_foreground() {
+ return Ok("background_unavailable");
}
// Native Wayland: focus+raise the target toplevel
// (foreign-toplevel `activate`), then drive `count` virtual-pointer
// button events. Wayland injection routes to the compositor focus.
- crate::wayland::click(xid, xi, yi, count as u32, button)?;
+ crate::wayland::click(xid, output_x, output_y, count as u32, button)?;
return Ok("wayland_activate");
}
// X11 injection. Tiered no-focus-steal delivery (background):
@@ -1819,6 +2080,11 @@ impl Tool for ClickTool {
"background"
};
match result {
+ Ok(Ok("background_unavailable")) => {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ )
+ }
// A pixel/coordinate click is never driver-verifiable (no read-back) —
// verified:false, effect:"unverifiable"; the caller confirms via
// screenshot. path reports the rung taken.
@@ -1882,6 +2148,43 @@ async fn focus_by_pixel(
Ok(())
}
+/// Establish widget-local focus inside a nested-compositor target without
+/// changing the compositor's focused toplevel. AX uses Component.GrabFocus;
+/// PX sends a private per-surface left click at the requested local point.
+async fn focus_nested_inject_target(
+ pid: u32,
+ window_id: u64,
+ element_index: Option<usize>,
+ pixel: Option<(f64, f64)>,
+) -> Result<(), ToolResult> {
+ if let Some(index) = element_index {
+ return match tokio::task::spawn_blocking(move || {
+ crate::atspi::focus_element(pid, index)
+ })
+ .await
+ {
+ Ok(Ok(true)) => Ok(()),
+ Ok(Ok(false)) => Err(ToolResult::error(format!(
+ "AT-SPI Component.GrabFocus returned false for element {index}"
+ ))),
+ Ok(Err(error)) => Err(ToolResult::error(error.to_string())),
+ Err(error) => Err(ToolResult::error(format!("Task error: {error}"))),
+ };
+ }
+ if let Some((x, y)) = pixel {
+ return match tokio::task::spawn_blocking(move || {
+ crate::wayland::inject_click(window_id, x, y, 1, 1)
+ })
+ .await
+ {
+ Ok(Ok(())) => Ok(()),
+ Ok(Err(error)) => Err(ToolResult::error(error.to_string())),
+ Err(error) => Err(ToolResult::error(format!("Task error: {error}"))),
+ };
+ }
+ Ok(())
+}
+
// ── type_text ─────────────────────────────────────────────────────────────────
pub struct TypeTextTool {
@@ -1944,7 +2247,7 @@ impl Tool for TypeTextTool {
} => (Some(*element_index), window_id.map(|v| v as u64)),
cua_driver_core::element_token::ResolvedElement::None => (None, None),
};
- let xid_opt = resolved_window_id.or_else(|| args.opt_u64("window_id"));
+ let xid_opt = args.opt_u64("window_id").or(resolved_window_id);
// Resolve XID: use window_id if given, else first window for pid.
let xid = match xid_opt {
@@ -1964,6 +2267,74 @@ impl Tool for TypeTextTool {
}
}
};
+ let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+ if resolved_elem_idx.is_none() {
+ if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) {
+ return refusal;
+ }
+ }
+
+ let px = args.get("x").and_then(|value| value.as_f64());
+ let py = args.get("y").and_then(|value| value.as_f64());
+ if px.is_some() != py.is_some() {
+ return ToolResult::error("Pass both x and y to type_text, or neither.");
+ }
+ if px.is_some() && resolved_elem_idx.is_some() {
+ return ToolResult::error(
+ "Pass either element_index (ax) or x,y (px) to type_text, not both.",
+ );
+ }
+
+ let text_len = text.chars().count();
+ // Native toolkit editables have a stronger focus-free route than raw
+ // compositor keyboard injection. Keep Chromium/WebKit on real key events
+ // because their accessibility bridges may echo a write that never reaches
+ // renderer-owned state.
+ if crate::wayland::is_inject_mode()
+ && resolved_elem_idx.is_some()
+ && !is_chromium_embedder(pid)
+ && !is_webkitgtk_embedder(pid)
+ {
+ let idx = resolved_elem_idx.expect("checked above");
+ let text_at = text.clone();
+ let targeted = tokio::task::spawn_blocking(move || {
+ crate::atspi::type_into_editable_at(pid, idx, &text_at)
+ })
+ .await;
+ if let Ok(Ok(())) = targeted {
+ return type_text_ax_confirm_result(pid, text_len, "via targeted AT-SPI");
+ }
+ }
+ // The private nested compositor can target the owning Wayland client
+ // directly. Establish widget-local focus first, without changing the
+ // compositor's focused toplevel, so keys reach the addressed control.
+ if crate::wayland::is_inject_mode() {
+ if let Err(error) = focus_nested_inject_target(
+ pid,
+ xid,
+ resolved_elem_idx,
+ px.zip(py),
+ )
+ .await
+ {
+ return error;
+ }
+ let text_w = text.clone();
+ let result =
+ tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w))
+ .await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) (focus-free via cua-compositor)."
+ ))
+ .with_structured(type_text_structured("key_events", text_len, false)),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
// ── px form: focus by pixel-click, then type into the now-focused element ──
// Pass x,y (no element_index/token) for an *element px action*: pixel-click
@@ -1971,16 +2342,10 @@ impl Tool for TypeTextTool {
// can't, then fall through to the focused-element type path below (it
// escalates AT-SPI → key events and lands once focused). Reuses ClickTool's
// exact coordinate translation + delivery_mode.
- if let (Some(cx), Some(cy)) = (
- args.get("x").and_then(|v| v.as_f64()),
- args.get("y").and_then(|v| v.as_f64()),
- ) {
- if resolved_elem_idx.is_some() {
- return ToolResult::error(
- "Pass either element_index (ax) or x,y (px) to type_text, not both.",
- );
+ if let (Some(cx), Some(cy)) = (px, py) {
+ if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) {
+ return refusal;
}
- let fg = crate::input::delivery::DeliveryMode::from_args(&args).is_foreground();
let from_zoom = args.bool_or("from_zoom", false);
if let Err(e) = focus_by_pixel(
&self.state,
@@ -1988,7 +2353,7 @@ impl Tool for TypeTextTool {
Some(xid),
cx,
cy,
- fg,
+ delivery.is_foreground(),
args.opt_str("session"),
from_zoom,
)
@@ -2000,36 +2365,92 @@ impl Tool for TypeTextTool {
// focused element via the background key / AT-SPI rung.
}
- // EIS nested compositor: focus-FREE per-surface typing into window_id
- // (the target need not be focused). Routed over the inject control socket.
- if crate::wayland::is_inject_mode() {
- let text_len = text.chars().count();
- let text_w = text.clone();
- let result =
- tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w))
- .await;
- return match result {
- Ok(Ok(())) => ToolResult::text(format!(
- "Typed {text_len} character(s) (focus-free via EIS compositor)."
- ))
- .with_structured(type_text_structured(
- "key_events",
- text_len,
- false,
- )),
- Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
- };
+ if resolved_elem_idx.is_some() {
+ if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) {
+ return refusal;
+ }
+ }
+
+ // Renderer EditableText writes can update the accessible value without
+ // emitting the DOM input event. For an explicit foreground request on
+ // native Wayland, focus the named field and send real keyboard input so
+ // Chromium/WebKit observe the same event sequence as a user.
+ if delivery.is_foreground()
+ && crate::wayland::wayland_input_enabled()
+ && (is_chromium_embedder(pid) || is_webkitgtk_embedder(pid))
+ {
+ if let Some(idx) = resolved_elem_idx {
+ let focused = tokio::task::spawn_blocking(move || {
+ crate::atspi::focus_element(pid, idx)
+ })
+ .await;
+ match focused {
+ Ok(Ok(true)) => {}
+ Ok(Ok(false)) => {
+ return ToolResult::error(format!(
+ "AT-SPI Component.GrabFocus returned false for element {idx}"
+ ))
+ }
+ Ok(Err(error)) => return ToolResult::error(error.to_string()),
+ Err(error) => return ToolResult::error(format!("Task error: {error}")),
+ }
+
+ let text_w = text.clone();
+ let result = tokio::task::spawn_blocking(move || {
+ crate::wayland::type_text(xid, &text_w)
+ })
+ .await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) (via Wayland virtual-keyboard)."
+ ))
+ .with_structured(type_text_structured(
+ "key_events",
+ text_len,
+ false,
+ )),
+ Ok(Err(error)) => ToolResult::error(error.to_string()),
+ Err(error) => ToolResult::error(format!("Task error: {error}")),
+ };
+ }
+ }
+
+ // AX addressing names one exact editable. Try this focus-free route
+ // before native Wayland keyboard injection, which can only target the
+ // compositor's globally focused surface.
+ if let Some(idx) = resolved_elem_idx {
+ let text_at = text.clone();
+ let targeted = tokio::task::spawn_blocking(move || {
+ crate::atspi::type_into_editable_at(pid, idx, &text_at)
+ })
+ .await;
+ match targeted {
+ Ok(Ok(())) => {
+ return type_text_ax_confirm_result(pid, text_len, "via targeted AT-SPI");
+ }
+ Ok(Err(_)) | Err(_)
+ if !delivery.is_foreground() && crate::wayland::wayland_input_enabled() =>
+ {
+ return crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ );
+ }
+ _ => {}
+ }
}
// Native Wayland: keys go to the *focused* surface (no pid/window
// targeting in the protocol). Type via the virtual-keyboard tool; pair
// with a prior `click`/`activate` to focus the intended window.
- if crate::wayland::is_wayland() {
- let text_len = text.chars().count();
+ if crate::wayland::wayland_input_enabled() {
+ if !delivery.is_foreground() {
+ return crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ );
+ }
let text_w = text.clone();
let result =
- tokio::task::spawn_blocking(move || crate::wayland::type_text(&text_w)).await;
+ tokio::task::spawn_blocking(move || crate::wayland::type_text(xid, &text_w)).await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Typed {text_len} character(s) (via Wayland virtual-keyboard)."
@@ -2060,25 +2481,34 @@ impl Tool for TypeTextTool {
if pid_is_terminal || wm_class_is_terminal {
let text_len = text.chars().count();
let text_t = text.clone();
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
+ let foreground = delivery.is_foreground();
+ let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> {
// pty-master injection is preferred — it skips the X event
// queue entirely. Falls through to XTest if the terminal
// isn't reachable that way (descendant pty unresolvable).
if inject_terminal_input(pid, xid, &text_t)? {
- return Ok(());
+ return Ok("pty");
+ }
+ if foreground {
+ crate::input::with_x11_foreground(xid, 80, || {
+ crate::input::send_type_text_xtest(&text_t)
+ })?;
+ Ok("key_events_fg")
+ } else {
+ Ok("background_unavailable")
}
- crate::input::send_type_text_xtest(&text_t)
})
.await;
return match result {
- Ok(Ok(())) => ToolResult::text(format!(
+ Ok(Ok("background_unavailable")) => {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ )
+ }
+ Ok(Ok(path)) => ToolResult::text(format!(
"Typed {text_len} character(s) (terminal emulator: pty/XTest key events)."
))
- .with_structured(type_text_structured(
- "key_events",
- text_len,
- false,
- )),
+ .with_structured(type_text_structured(path, text_len, false)),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
};
@@ -2098,7 +2528,46 @@ impl Tool for TypeTextTool {
});
}
}
- let text_len = text.chars().count();
+ // Foreground means the caller explicitly permits activation. Chromium
+ // and WebKitGTK can acknowledge an accessibility write without
+ // producing the renderer input event, so web embedders use real XTest
+ // key events. Native toolkits keep their verifiable AT-SPI path below.
+ if delivery.is_foreground() && (is_chromium_embedder(pid) || is_webkitgtk_embedder(pid)) {
+ if let Some(idx) = resolved_elem_idx {
+ let focused =
+ tokio::task::spawn_blocking(move || crate::atspi::focus_element(pid, idx))
+ .await;
+ match focused {
+ Ok(Ok(true)) => {}
+ Ok(Ok(false)) => {
+ return ToolResult::error(format!(
+ "AT-SPI Component.GrabFocus returned false for element {idx}"
+ ))
+ }
+ Ok(Err(e)) => return ToolResult::error(e.to_string()),
+ Err(e) => return ToolResult::error(format!("Task error: {e}")),
+ }
+ }
+ let text_f = text.clone();
+ let result = tokio::task::spawn_blocking(move || {
+ crate::input::with_x11_foreground(xid, 80, || {
+ crate::input::send_type_text_xtest(&text_f)
+ })
+ })
+ .await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Typed {text_len} character(s) (via X11, delivery_mode=foreground)."
+ ))
+ .with_structured(type_text_structured(
+ "key_events_fg",
+ text_len,
+ false,
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
// Prefer the focused widget — the element the user just clicked. If a
// NON-editable input holds keyboard focus (a spreadsheet cell, a
@@ -2115,6 +2584,11 @@ impl Tool for TypeTextTool {
.ok()
.flatten();
if focus_kind == Some(false) {
+ if !delivery.is_foreground() {
+ return crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ );
+ }
let text_f = text.clone();
let result = tokio::task::spawn_blocking(move || {
if inject_terminal_input(pid, xid, &text_f)? {
@@ -2124,7 +2598,9 @@ impl Tool for TypeTextTool {
// drop synthetic key events, so a spreadsheet cell / canvas would
// stay empty. The click that gave this widget focus already put it
// under the X input focus, so XTest-to-focus lands correctly.
- crate::input::send_type_text_xtest(&text_f)
+ crate::input::with_x11_foreground(xid, 80, || {
+ crate::input::send_type_text_xtest(&text_f)
+ })
})
.await;
return match result {
@@ -2200,19 +2676,11 @@ impl Tool for TypeTextTool {
// foreground = activate the window (EWMH), then synthesize REAL key
// events to it via XTest — the escalation when background didn't land
// (e.g. a GTK dialog whose widget ignores synthetic XSendEvent keys).
- let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> {
// Terminals: write to the pty master (focus-free, below the toolkit).
if inject_terminal_input(pid, xid, &text)? {
return Ok("key_events");
}
- if delivery.is_foreground() {
- // Activate target → XTest real keystrokes → restore prior active.
- return crate::input::with_x11_foreground(xid, 80, || {
- crate::input::send_type_text_xtest(&text)?;
- Ok("key_events_fg")
- });
- }
// GUI apps: X11 only routes keystrokes to the *focused* toplevel's
// focused widget, so background XSendEvent typing doesn't land. Fill
// the editable field via AT-SPI instead — focus-free and toolkit-
@@ -2225,8 +2693,14 @@ impl Tool for TypeTextTool {
if crate::input::inject_tk_send(&text).unwrap_or(false) {
return Ok("key_events");
}
- crate::input::send_type_text(xid, &text)?;
- Ok("key_events")
+ if delivery.is_foreground() {
+ crate::input::with_x11_foreground(xid, 80, || {
+ crate::input::send_type_text_xtest(&text)
+ })?;
+ Ok("key_events_fg")
+ } else {
+ Ok("background_unavailable")
+ }
})
.await;
let mode_label = if delivery.is_foreground() {
@@ -2248,6 +2722,11 @@ impl Tool for TypeTextTool {
text_len,
&format!("via X11, delivery_mode={mode_label}"),
),
+ Ok(Ok("background_unavailable")) => {
+ crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ )
+ }
Ok(Ok(path)) => ToolResult::text(format!(
"Typed {text_len} character(s) (via X11, delivery_mode={mode_label})."
))
@@ -2317,7 +2796,7 @@ impl Tool for PressKeyTool {
};
let xid_opt = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
- window_id.map(|v| v as u64).or(window_id_arg)
+ window_id_arg.or_else(|| window_id.map(|v| v as u64))
}
cua_driver_core::element_token::ResolvedElement::None => window_id_arg,
};
@@ -2344,6 +2823,59 @@ impl Tool for PressKeyTool {
// background path (the focus-click already handled fronting when fg). Pass x,y
// (no element_index) for Chromium/Electron surfaces the AX path can't focus.
let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+
+ if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_gtk_keyboard_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) {
+ return refusal;
+ }
+
+ let px = args.get("x").and_then(|value| value.as_f64());
+ let py = args.get("y").and_then(|value| value.as_f64());
+ if px.is_some() != py.is_some() {
+ return ToolResult::error("Pass both x and y to press_key, or neither.");
+ }
+ if px.is_some() && element_index_arg.is_some() {
+ return ToolResult::error(
+ "Pass either element_index (ax) or x,y (px) to press_key, not both.",
+ );
+ }
+
+ // Nested cua-compositor addresses the owning Wayland client directly.
+ // Preserve legacy modifiers by promoting the request to a chord.
+ if crate::wayland::is_inject_mode() {
+ if let Err(error) =
+ focus_nested_inject_target(pid, xid, element_index_arg, px.zip(py)).await
+ {
+ return error;
+ }
+ let result = if mods.is_empty() {
+ let key_w = key.clone();
+ tokio::task::spawn_blocking(move || {
+ crate::wayland::inject_press_key(xid, &key_w)
+ })
+ .await
+ } else {
+ let mut chord = mods.clone();
+ chord.push(key.clone());
+ tokio::task::spawn_blocking(move || crate::wayland::inject_hotkey(xid, &chord))
+ .await
+ };
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Pressed key '{key}' (focus-free via cua-compositor)."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
// An element-addressed keypress needs to establish the target's
// focus before the window-level X11 key event is sent. AT-SPI's
@@ -2368,14 +2900,7 @@ impl Tool for PressKeyTool {
}
let px_target = {
- let px = args.get("x").and_then(|v| v.as_f64());
- let py = args.get("y").and_then(|v| v.as_f64());
if let (Some(cx), Some(cy)) = (px, py) {
- if element_index_arg.is_some() {
- return ToolResult::error(
- "Pass either element_index (ax) or x,y (px) to press_key, not both.",
- );
- }
let from_zoom = args.bool_or("from_zoom", false);
if let Err(e) = focus_by_pixel(
&self.state,
@@ -2397,26 +2922,11 @@ impl Tool for PressKeyTool {
}
};
- // EIS nested compositor: focus-free named-key into window_id.
- if crate::wayland::is_inject_mode() {
- let key_w = key.clone();
- let result =
- tokio::task::spawn_blocking(move || crate::wayland::inject_press_key(xid, &key_w))
- .await;
- return match result {
- Ok(Ok(())) => ToolResult::text(format!(
- "Pressed key '{key}' (focus-free via EIS compositor)."
- )),
- Ok(Err(e)) => ToolResult::error(e.to_string()),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
- };
- }
-
// Native Wayland: send the key to the focused surface via virtual-keyboard.
- if crate::wayland::is_wayland() {
+ if crate::wayland::wayland_input_enabled() {
let key_w = key.clone();
let result =
- tokio::task::spawn_blocking(move || crate::wayland::press_key(&key_w)).await;
+ tokio::task::spawn_blocking(move || crate::wayland::press_key(xid, &key_w)).await;
return match result {
Ok(Ok(())) => ToolResult::text(format!(
"Pressed key '{key}' (via Wayland virtual-keyboard)."
@@ -2427,9 +2937,10 @@ impl Tool for PressKeyTool {
}
let key_for_task = key.clone();
- // px-focus already clicked (and fronted, when fg) the target → deliver via
- // the plain background path. Otherwise honor the requested delivery_mode.
- let deliver_fg = delivery.is_foreground() && px_target.is_none();
+ // Foreground delivery is one atomic activate-and-XTest transaction.
+ // A preceding PX click establishes internal widget focus, but that
+ // click restores the prior top-level before returning.
+ let deliver_fg = delivery.is_foreground();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
if mods.is_empty() && key_for_task.eq_ignore_ascii_case("enter") {
if inject_terminal_input(pid, xid, "\n")? {
@@ -2507,6 +3018,8 @@ impl Tool for HotkeyTool {
"window_id":{"type":"integer"},
"keys":{"type":"array","items":{"type":"string"},"minItems":2,
"description":"Modifier(s) + one non-modifier key, e.g. [\"ctrl\",\"c\"]."},
+ "element_index": cua_driver_core::tool_schema::element_index_schema(),
+ "element_token": cua_driver_core::tool_schema::element_token_schema(),
"x":{"type":"number","description":"Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Ctrl+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach."},
"y":{"type":"number","description":"Screenshot-pixel Y (see x)."},
"delivery_mode": crate::input::delivery::delivery_mode_schema()
@@ -2519,7 +3032,31 @@ impl Tool for HotkeyTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
let pid = args.u64_or("pid", 0) as u32;
- let xid_opt = args.opt_u64("window_id");
+ let window_id_arg = args.opt_u64("window_id");
+ let element_index_arg = args.opt_u64("element_index").map(|value| value as usize);
+ let resolved = match cua_driver_core::element_token::resolve_element_args(
+ pid as i32,
+ element_index_arg,
+ args.opt_str("element_token").as_deref(),
+ window_id_arg.map(|value| value as u32),
+ "hotkey",
+ ) {
+ Ok(resolved) => resolved,
+ Err(error) => return error,
+ };
+ let resolved_element_index = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element {
+ element_index,
+ ..
+ } => Some(*element_index),
+ cua_driver_core::element_token::ResolvedElement::None => None,
+ };
+ let xid_opt = match &resolved {
+ cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
+ window_id_arg.or_else(|| window_id.map(|value| value as u64))
+ }
+ cua_driver_core::element_token::ResolvedElement::None => window_id_arg,
+ };
// Resolve XID: use window_id if given, else first window for pid.
let xid = match xid_opt {
@@ -2566,18 +3103,100 @@ impl Tool for HotkeyTool {
let mods_for_wayland = mods.clone();
let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
- // X11 can address an unfocused native window with synthetic events, but
- // Chromium's renderer only processes modifier chords for its focused
- // input surface. Do not report the XSendEvent attempt as success: the
- // explicit foreground rung is the only truthful fallback unless an
- // EIS compositor can target the surface without focus.
- if !delivery.is_foreground()
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_gtk_keyboard_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) {
+ return refusal;
+ }
+
+ let px = args.get("x").and_then(|value| value.as_f64());
+ let py = args.get("y").and_then(|value| value.as_f64());
+ if px.is_some() != py.is_some() {
+ return ToolResult::error("Pass both x and y to hotkey, or neither.");
+ }
+ if px.is_some() && resolved_element_index.is_some() {
+ return ToolResult::error(
+ "Pass either element_index (ax) or x,y (px) to hotkey, not both.",
+ );
+ }
+
+ if crate::wayland::is_inject_mode() {
+ if let Err(error) =
+ focus_nested_inject_target(pid, xid, resolved_element_index, px.zip(py)).await
+ {
+ return error;
+ }
+ let mut chord = mods.clone();
+ chord.push(key.clone());
+ let result = tokio::task::spawn_blocking(move || {
+ crate::wayland::inject_hotkey(xid, &chord)
+ })
+ .await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Pressed hotkey '{key_display}' (focus-free via cua-compositor)."
+ )),
+ Ok(Err(error)) => ToolResult::error(error.to_string()),
+ Err(error) => ToolResult::error(format!("Task error: {error}")),
+ };
+ }
+
+ // Chromium renderer shortcuts need DOM focus established by a real
+ // pointer event. Component.GrabFocus alone can report success while
+ // the renderer still drops the virtual-keyboard chord.
+ if delivery.is_foreground()
+ && crate::wayland::wayland_input_enabled()
&& is_chromium_embedder(pid)
- && !crate::wayland::is_inject_mode()
{
- return crate::input::delivery::background_unavailable_error(
- crate::input::delivery::BackgroundUnavailable::ChromiumHotkey,
- );
+ if let Some(element_index) = resolved_element_index {
+ let coordinates = tokio::task::spawn_blocking(move || {
+ resolve_element_local_coords(pid, element_index, Some(xid))
+ })
+ .await;
+ let (_, x, y) = match coordinates {
+ Ok(Ok(value)) => value,
+ Ok(Err(error)) => return ToolResult::error(error.to_string()),
+ Err(error) => return ToolResult::error(format!("Task error: {error}")),
+ };
+ if let Err(error) = focus_by_pixel(
+ &self.state,
+ pid,
+ Some(xid),
+ x,
+ y,
+ true,
+ args.opt_str("session"),
+ false,
+ )
+ .await
+ {
+ return error;
+ }
+ }
+ }
+
+ if let Some(element_index) = resolved_element_index {
+ let focused = tokio::task::spawn_blocking(move || {
+ crate::atspi::focus_element(pid, element_index)
+ })
+ .await;
+ match focused {
+ Ok(Ok(true)) => {}
+ Ok(Ok(false)) => {
+ return ToolResult::error(format!(
+ "AT-SPI Component.GrabFocus returned false for element {element_index}"
+ ))
+ }
+ Ok(Err(error)) => return ToolResult::error(error.to_string()),
+ Err(error) => return ToolResult::error(format!("Task error: {error}")),
+ }
}
// ── px form: pixel-click to focus, then the combo acts on the focused field ──
@@ -2585,8 +3204,6 @@ impl Tool for HotkeyTool {
// delivery_mode; after it, deliver the combo via the plain background path
// (the focus-click already fronted when fg).
let px_target = {
- let px = args.get("x").and_then(|v| v.as_f64());
- let py = args.get("y").and_then(|v| v.as_f64());
if let (Some(cx), Some(cy)) = (px, py) {
let from_zoom = args.bool_or("from_zoom", false);
if let Err(e) = focus_by_pixel(
@@ -2608,16 +3225,16 @@ impl Tool for HotkeyTool {
None
}
};
- let deliver_fg = delivery.is_foreground() && px_target.is_none();
+ let deliver_fg = delivery.is_foreground();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- if crate::wayland::is_wayland() {
+ if crate::wayland::wayland_input_enabled() {
// Native Wayland: route the modifier combo through wtype's
// -M/-k/-m sequence — the closest equivalent to the X11
// state-mask path. window_id is irrelevant once focused.
let mut combo: Vec<String> = mods_for_wayland.clone();
combo.push(key_for_wayland.clone());
- return crate::wayland::hotkey(&combo);
+ return crate::wayland::hotkey(xid, &combo);
}
let m: Vec<&str> = mods.iter().map(String::as_str).collect();
// foreground: activate the target first, then inject the accelerator
@@ -2761,6 +3378,8 @@ impl Tool for ScrollTool {
"window_id":{"type":"integer"},
"element_index": cua_driver_core::tool_schema::element_index_schema(),
"element_token": cua_driver_core::tool_schema::element_token_schema(),
+ "x":{"type":"number","description":"Window-local screenshot-pixel X of the scroll target. Pass with y and without element_index."},
+ "y":{"type":"number","description":"Window-local screenshot-pixel Y of the scroll target. Pass with x and without element_index."},
"delivery_mode": crate::input::delivery::delivery_mode_schema()
},"additionalProperties":false
}),
@@ -2821,30 +3440,155 @@ impl Tool for ScrollTool {
};
let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
- if !delivery.is_foreground() {
- if let cua_driver_core::element_token::ResolvedElement::Element {
- element_index, ..
- } = &resolved
- {
- let idx = *element_index;
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+ if let cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } =
+ &resolved
+ {
+ let idx = *element_index;
+ // WebKitGTK acknowledges the AT-SPI scroll action without moving
+ // the DOM scroller. On native Wayland, use a real compositor wheel
+ // event at the resolved element instead of reporting a silent
+ // success. Chromium's AT-SPI scroll path remains effective.
+ if !(crate::wayland::wayland_input_enabled() && is_webkitgtk_embedder(pid)) {
let direction_for_ax = direction.clone();
let ax_result = tokio::task::spawn_blocking(move || {
crate::atspi::scroll_element(pid, idx, &direction_for_ax, amount)
})
.await;
if matches!(ax_result, Ok(Ok(()))) {
+ let mode = if delivery.is_foreground() {
+ "foreground"
+ } else {
+ "background"
+ };
return ToolResult::text(format!(
- "Scrolled {direction} {amount} ticks via AT-SPI (delivery_mode:background)."
+ "Scrolled {direction} {amount} ticks via AT-SPI (delivery_mode:{mode})."
))
.with_structured(json!({
"path": "atspi",
"verified": false,
- "delivery_mode": "background"
+ "delivery_mode": mode
}));
}
}
}
+ let pixel_target = match (
+ args.get("x").and_then(|value| value.as_f64()),
+ args.get("y").and_then(|value| value.as_f64()),
+ ) {
+ (Some(x), Some(y)) => Some((x, y)),
+ (None, None) => None,
+ _ => return ToolResult::error("Pass both x and y to pixel-target scroll."),
+ };
+ if pixel_target.is_some()
+ && matches!(
+ &resolved,
+ cua_driver_core::element_token::ResolvedElement::Element { .. }
+ )
+ {
+ return ToolResult::error(
+ "Pass either element_index (ax) or x,y (px) to scroll, not both.",
+ );
+ }
+
+ if crate::wayland::is_inject_mode() {
+ let Some((x, y)) = pixel_target else {
+ return crate::input::delivery::background_unavailable_error(
+ crate::input::delivery::BackgroundUnavailable::FocusedInputOnly,
+ );
+ };
+ let direction_for_inject = direction.clone();
+ let result = tokio::task::spawn_blocking(move || {
+ crate::wayland::inject_scroll(
+ xid,
+ x,
+ y,
+ &direction_for_inject,
+ amount as u32,
+ )
+ })
+ .await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Scrolled {direction} {amount} ticks (focus-free via cua-compositor)."
+ ))
+ .with_structured(json!({
+ "verified": false,
+ "delivery_mode": if delivery.is_foreground() { "foreground" } else { "background" },
+ "route": "cua_compositor_inject"
+ })),
+ Ok(Err(error)) => ToolResult::error(error.to_string()),
+ Err(error) => ToolResult::error(format!("Task error: {error}")),
+ };
+ }
+
+ if crate::wayland::wayland_input_enabled() {
+ if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, false) {
+ return refusal;
+ }
+ let direction_for_wayland = direction.clone();
+ let local_point = match (pixel_target, &resolved) {
+ (Some(point), _) => Some(point),
+ (
+ None,
+ cua_driver_core::element_token::ResolvedElement::Element {
+ element_index,
+ ..
+ },
+ ) => {
+ let idx = *element_index;
+ match tokio::task::spawn_blocking(move || {
+ resolve_element_local_coords(pid, idx, Some(xid))
+ })
+ .await
+ {
+ Ok(Ok((_resolved_xid, x, y))) => Some((x, y)),
+ Ok(Err(error)) => {
+ return ToolResult::error(format!(
+ "Could not resolve scroll element [{idx}] coordinates: {error}"
+ ))
+ }
+ Err(error) => {
+ return ToolResult::error(format!(
+ "Scroll element coordinate task failed: {error}"
+ ))
+ }
+ }
+ }
+ (None, cua_driver_core::element_token::ResolvedElement::None) => None,
+ };
+ let output_point = local_point.map(|(x, y)| {
+ crate::wayland::window_local_to_output(xid, x.round() as i32, y.round() as i32)
+ });
+ let result = tokio::task::spawn_blocking(move || {
+ crate::wayland::scroll_at(
+ xid,
+ output_point,
+ &direction_for_wayland,
+ amount as u32,
+ )
+ })
+ .await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Scrolled {direction} {amount} ticks (delivery_mode=foreground)."
+ ))
+ .with_structured(json!({ "verified": false, "delivery_mode": "foreground" })),
+ Ok(Err(error)) => ToolResult::error(error.to_string()),
+ Err(error) => ToolResult::error(format!("Task error: {error}")),
+ };
+ }
+
+ if let Some(refusal) = unavailable_webkit_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) {
+ return refusal;
+ }
+
// An element-addressed scroll must land over the element. The old
// fallback used (0, 0) in the window, which can report success while
// Chromium/GTK routes the wheel to the toplevel instead of the
@@ -2873,7 +3617,10 @@ impl Tool for ScrollTool {
}
}
}
- cua_driver_core::element_token::ResolvedElement::None => None,
+ cua_driver_core::element_token::ResolvedElement::None => pixel_target.map(|local| {
+ let screen = window_local_to_screen(xid, local.0, local.1).ok();
+ (local, screen)
+ }),
};
// X11 scroll buttons: 4=up, 5=down, 6=left, 7=right
@@ -2884,11 +3631,11 @@ impl Tool for ScrollTool {
"right" => 7,
_ => 5,
};
+ let cursor_id_for_task = cursor_id.clone();
let direction_for_wayland = direction.clone();
let amount_u32 = amount as u32;
- let cursor_id_for_task = cursor_id.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- if crate::wayland::is_wayland() {
+ if crate::wayland::wayland_input_enabled() {
return crate::wayland::scroll(xid, &direction_for_wayland, amount_u32);
}
// foreground: activate the window, then scroll, then restore — for
@@ -2932,7 +3679,22 @@ impl Tool for ScrollTool {
crate::input::send_click(xid, local_x as i32, local_y as i32, amount, button)
};
if delivery.is_foreground() {
- crate::input::with_x11_foreground(xid, 80, x11_scroll)
+ let point = element_point
+ .and_then(|(_, screen)| screen)
+ .or_else(|| {
+ window_screen_center(xid)
+ .ok()
+ .map(|(x, y)| (x as f64, y as f64))
+ })
+ .ok_or_else(|| anyhow::anyhow!("could not resolve foreground scroll point"))?;
+ crate::input::with_x11_foreground(xid, 80, || {
+ crate::input::send_click_xtest_desktop(
+ point.0 as i32,
+ point.1 as i32,
+ button,
+ amount,
+ )
+ })
} else {
x11_scroll()
}
@@ -2996,6 +3758,19 @@ impl Tool for DoubleClickTool {
Ok(v) => v,
Err(e) => return e,
};
+ let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_webkit_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) {
+ return refusal;
+ }
// Surface 6: element_token / element_index precedence.
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -3015,7 +3790,8 @@ impl Tool for DoubleClickTool {
};
let window_id_resolved: Option<u64> = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
- window_id.map(|v| v as u64)
+ args.opt_u64("window_id")
+ .or_else(|| window_id.map(|v| v as u64))
}
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
@@ -3042,10 +3818,27 @@ impl Tool for DoubleClickTool {
}
let lxi = lx as i32;
let lyi = ly as i32;
+ let wayland_point = crate::wayland::wayland_input_enabled()
+ .then(|| crate::wayland::window_local_to_output(xid, lxi, lyi));
let cursor_id_for_task = cursor_id.clone();
let click_result = tokio::task::spawn_blocking(move || {
- if crate::wayland::is_wayland() {
- return crate::wayland::click(xid, lxi, lyi, 2, 1);
+ if crate::wayland::is_inject_mode() {
+ return crate::wayland::inject_click(xid, lx, ly, 2, 1);
+ }
+ if crate::wayland::wayland_input_enabled() {
+ let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi));
+ return crate::wayland::click(xid, output_x, output_y, 2, 1);
+ }
+ if delivery.is_foreground() {
+ return crate::input::with_x11_foreground(xid, 80, || {
+ let (sx, sy) = window_local_to_screen(xid, lxi as f64, lyi as f64)?;
+ crate::input::send_click_xtest_desktop(
+ sx.round() as i32,
+ sy.round() as i32,
+ 1,
+ 2,
+ )
+ });
}
x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, lxi, lyi, 1, 2)
})
@@ -3090,13 +3883,17 @@ impl Tool for DoubleClickTool {
cursor_id.clone(),
cursor_overlay::OverlayCommand::PinAbove(xid),
);
- // Resolve the screen point the cursor glides to. On native Wayland the
- // agent already passes screen coordinates (the vision screenshot and
- // `get_window_state` frames are screen-space, and `window_local_to_screen`
- // — an X11 `translate_coordinates` call — can't run with DISPLAY unset),
- // so use them directly. On X11 the coords are window-local; translate.
- let glide_target = if crate::wayland::is_wayland() {
- Some((x, y))
+ let wayland_output_point = if crate::wayland::wayland_input_enabled() {
+ Some(crate::wayland::window_local_to_output(
+ xid,
+ x.round() as i32,
+ y.round() as i32,
+ ))
+ } else {
+ None
+ };
+ let glide_target = if let Some((sx, sy)) = wayland_output_point {
+ Some((sx as f64, sy as f64))
} else {
tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y))
.await
@@ -3112,10 +3909,13 @@ impl Tool for DoubleClickTool {
}
let (xi, yi) = (x as i32, y as i32);
let cursor_id_for_task = cursor_id.clone();
- let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- if crate::wayland::is_wayland() {
- return crate::wayland::click(xid, xi, yi, 2, 1);
+ if crate::wayland::is_inject_mode() {
+ return crate::wayland::inject_click(xid, x, y, 2, 1);
+ }
+ if crate::wayland::wayland_input_enabled() {
+ let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi));
+ return crate::wayland::click(xid, output_x, output_y, 2, 1);
}
if delivery.is_foreground() {
return crate::input::with_x11_foreground(xid, 80, || {
@@ -3191,6 +3991,19 @@ impl Tool for RightClickTool {
Ok(v) => v,
Err(e) => return e,
};
+ let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_webkit_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) {
+ return refusal;
+ }
// Surface 6: element_token / element_index precedence.
let resolved = match cua_driver_core::element_token::resolve_element_args(
pid as i32,
@@ -3210,7 +4023,8 @@ impl Tool for RightClickTool {
};
let window_id_resolved: Option<u64> = match &resolved {
cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => {
- window_id.map(|v| v as u64)
+ args.opt_u64("window_id")
+ .or_else(|| window_id.map(|v| v as u64))
}
cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"),
};
@@ -3237,10 +4051,27 @@ impl Tool for RightClickTool {
}
let lxi = lx as i32;
let lyi = ly as i32;
+ let wayland_point = crate::wayland::wayland_input_enabled()
+ .then(|| crate::wayland::window_local_to_output(xid, lxi, lyi));
let cursor_id_for_task = cursor_id.clone();
let click_result = tokio::task::spawn_blocking(move || {
- if crate::wayland::is_wayland() {
- return crate::wayland::click(xid, lxi, lyi, 1, 3);
+ if crate::wayland::is_inject_mode() {
+ return crate::wayland::inject_click(xid, lx, ly, 1, 3);
+ }
+ if crate::wayland::wayland_input_enabled() {
+ let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi));
+ return crate::wayland::click(xid, output_x, output_y, 1, 3);
+ }
+ if delivery.is_foreground() {
+ return crate::input::with_x11_foreground(xid, 80, || {
+ let (sx, sy) = window_local_to_screen(xid, lxi as f64, lyi as f64)?;
+ crate::input::send_click_xtest_desktop(
+ sx.round() as i32,
+ sy.round() as i32,
+ 3,
+ 1,
+ )
+ });
}
x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, lxi, lyi, 3, 1)
})
@@ -3285,13 +4116,17 @@ impl Tool for RightClickTool {
cursor_id.clone(),
cursor_overlay::OverlayCommand::PinAbove(xid),
);
- // Resolve the screen point the cursor glides to. On native Wayland the
- // agent already passes screen coordinates (the vision screenshot and
- // `get_window_state` frames are screen-space, and `window_local_to_screen`
- // — an X11 `translate_coordinates` call — can't run with DISPLAY unset),
- // so use them directly. On X11 the coords are window-local; translate.
- let glide_target = if crate::wayland::is_wayland() {
- Some((x, y))
+ let wayland_output_point = if crate::wayland::wayland_input_enabled() {
+ Some(crate::wayland::window_local_to_output(
+ xid,
+ x.round() as i32,
+ y.round() as i32,
+ ))
+ } else {
+ None
+ };
+ let glide_target = if let Some((sx, sy)) = wayland_output_point {
+ Some((sx as f64, sy as f64))
} else {
tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y))
.await
@@ -3307,10 +4142,13 @@ impl Tool for RightClickTool {
}
let (xi, yi) = (x as i32, y as i32);
let cursor_id_for_task = cursor_id.clone();
- let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- if crate::wayland::is_wayland() {
- return crate::wayland::click(xid, xi, yi, 1, 3);
+ if crate::wayland::is_inject_mode() {
+ return crate::wayland::inject_click(xid, x, y, 1, 3);
+ }
+ if crate::wayland::wayland_input_enabled() {
+ let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi));
+ return crate::wayland::click(xid, output_x, output_y, 1, 3);
}
if delivery.is_foreground() {
return crate::input::with_x11_foreground(xid, 80, || {
@@ -3376,7 +4214,8 @@ impl Tool for DragTool {
"steps":{"type":"integer","minimum":1,"maximum":200,"description":"Intermediate MotionNotify events. Default: 20."},
"modifier": cua_driver_core::tool_schema::modifier_schema(),
"button": cua_driver_core::tool_schema::button_schema(),
- "from_zoom":{"type":"boolean"}
+ "from_zoom":{"type":"boolean"},
+ "delivery_mode": crate::input::delivery::delivery_mode_schema()
},"additionalProperties":false}),
read_only: false, destructive: true, idempotent: false, open_world: true,
})
@@ -3391,6 +4230,19 @@ impl Tool for DragTool {
Some(v) => v,
None => return ToolResult::error("window_id is required on Linux."),
};
+ let delivery = crate::input::delivery::DeliveryMode::from_args(&args);
+ if let Some(refusal) = unavailable_chromium_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_webkit_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) {
+ return refusal;
+ }
+ if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) {
+ return refusal;
+ }
let coerce = |key: &str| -> Option<f64> {
args.opt_f64(key)
@@ -3446,9 +4298,29 @@ impl Tool for DragTool {
cursor_id.clone(),
cursor_overlay::OverlayCommand::PinAbove(xid),
);
- if let Ok(Ok((sx_from, sy_from))) =
- tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await
- {
+ let wayland_points = crate::wayland::wayland_input_enabled().then(|| {
+ (
+ crate::wayland::window_local_to_output(
+ xid,
+ from_x.round() as i32,
+ from_y.round() as i32,
+ ),
+ crate::wayland::window_local_to_output(
+ xid,
+ to_x.round() as i32,
+ to_y.round() as i32,
+ ),
+ )
+ });
+ let screen_from = if let Some((from, _)) = wayland_points {
+ Some((from.0 as f64, from.1 as f64))
+ } else {
+ tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y))
+ .await
+ .ok()
+ .and_then(|result| result.ok())
+ };
+ if let Some((sx_from, sy_from)) = screen_from {
overlay_glide_to_for(&cursor_id, sx_from, sy_from).await;
self.state
.cursor_registry
@@ -3468,16 +4340,31 @@ impl Tool for DragTool {
);
// Native Wayland: emit press + interpolated motion + release as one
- // virtual-pointer sequence (output-relative coords). Returns early so
- // we don't fall into the X11 XSendEvent loop below.
- if crate::wayland::is_wayland() {
- let (fxi, fyi) = (from_x.round() as i32, from_y.round() as i32);
- let (txi, tyi) = (to_x.round() as i32, to_y.round() as i32);
+ // virtual-pointer (wlroots) or libei (GNOME/KDE) sequence, output-relative
+ // coords. Returns early so we don't fall into the X11 XSendEvent loop below.
+ if crate::wayland::wayland_input_enabled() {
let steps_u32 = steps as u32;
- let drag_result = tokio::task::spawn_blocking(move || {
- crate::wayland::drag(xid, fxi, fyi, txi, tyi, steps_u32, button)
- })
- .await;
+ let drag_result = if crate::wayland::is_inject_mode() {
+ tokio::task::spawn_blocking(move || {
+ crate::wayland::inject_drag(
+ xid,
+ (from_x, from_y),
+ (to_x, to_y),
+ steps,
+ button as u32,
+ )
+ })
+ .await
+ } else {
+ let ((fxi, fyi), (txi, tyi)) = wayland_points.unwrap_or((
+ (from_x.round() as i32, from_y.round() as i32),
+ (to_x.round() as i32, to_y.round() as i32),
+ ));
+ tokio::task::spawn_blocking(move || {
+ crate::wayland::drag(xid, fxi, fyi, txi, tyi, steps_u32, button)
+ })
+ .await
+ };
crate::overlay::send_command_for(
cursor_id.clone(),
cursor_overlay::OverlayCommand::SetPressed(false),
@@ -3493,6 +4380,53 @@ impl Tool for DragTool {
};
}
+ if delivery.is_foreground() {
+ let screen_points = tokio::task::spawn_blocking(move || {
+ Ok::<_, anyhow::Error>((
+ window_local_to_screen(xid, from_x, from_y)?,
+ window_local_to_screen(xid, to_x, to_y)?,
+ ))
+ })
+ .await;
+ let ((screen_from_x, screen_from_y), (screen_to_x, screen_to_y)) = match screen_points {
+ Ok(Ok(points)) => points,
+ Ok(Err(e)) => return ToolResult::error(e.to_string()),
+ Err(e) => return ToolResult::error(format!("Task error: {e}")),
+ };
+ let drag_result = tokio::task::spawn_blocking(move || {
+ crate::input::with_x11_foreground(xid, 80, || {
+ crate::input::send_drag_xtest_desktop(
+ screen_from_x.round() as i32,
+ screen_from_y.round() as i32,
+ screen_to_x.round() as i32,
+ screen_to_y.round() as i32,
+ button,
+ duration_ms,
+ steps,
+ )
+ })
+ })
+ .await;
+ crate::overlay::send_command_for(
+ cursor_id.clone(),
+ cursor_overlay::OverlayCommand::SetPressed(false),
+ );
+ return match drag_result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Dragged ({button_str}) to pid {pid} from ({from_x:.0}, {from_y:.0}) \
+ to ({to_x:.0}, {to_y:.0}) in {duration_ms}ms / {steps} steps \
+ (delivery_mode=foreground)."
+ ))
+ .with_structured(json!({
+ "path": "x11_xtest_fg",
+ "verified": false,
+ "delivery_mode": "foreground"
+ })),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
let press_result = tokio::task::spawn_blocking(move || {
crate::input::send_button_down(
xid,
@@ -4097,7 +5031,7 @@ pub struct ParallelMouseDragTool {
}
static PMDRAG_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();
-/// EIS-compositor path for parallel_mouse_drag: build window-local drag paths
+/// cua-compositor path for parallel_mouse_drag: build window-local drag paths
/// and run them as concurrent multi-cursor injections over the control socket.
/// Coordinates stay window-local (the compositor maps them per app_id), so no
/// X11 geometry/MPX is needed — the X11 path's hard blocker on Wayland.
@@ -4145,11 +5079,14 @@ async fn parallel_drag_inject(args: &Value) -> ToolResult {
.and_then(|v| v.as_str())
.unwrap_or("left"),
) as u32;
- let app = match tokio::task::spawn_blocking(move || crate::wayland::app_id_for_window(xid))
- .await
+ let app = match tokio::task::spawn_blocking(move || {
+ crate::wayland::inject_target_for_window(xid)
+ })
+ .await
{
- Ok(Some(a)) => a,
- _ => return ToolResult::error(format!("no Wayland app_id for window {xid}")),
+ Ok(Ok(target)) => target,
+ Ok(Err(error)) => return ToolResult::error(error.to_string()),
+ Err(error) => return ToolResult::error(format!("Task error: {error}")),
};
drags.push(crate::wayland::InjectDrag {
app_id: app,
@@ -4162,7 +5099,7 @@ async fn parallel_drag_inject(args: &Value) -> ToolResult {
let n = drags.len();
match tokio::task::spawn_blocking(move || crate::wayland::inject_parallel_drags(&drags)).await {
Ok(Ok(())) => ToolResult::text(format!(
- "Ran {n} concurrent drags (multi-cursor via EIS compositor)."
+ "Ran {n} concurrent drags (multi-cursor via cua-compositor)."
)),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -4203,7 +5140,7 @@ impl Tool for ParallelMouseDragTool {
}
async fn invoke(&self, args: Value) -> ToolResult {
- // EIS nested compositor: run the drags as concurrent multi-cursor
+ // Nested cua-compositor: run the drags as concurrent multi-cursor
// injections (window-local, no X11 MPX/geometry needed).
if crate::wayland::is_inject_mode() {
return parallel_drag_inject(&args).await;
@@ -4715,7 +5652,7 @@ impl Tool for MoveCursorTool {
// Off-thread because the wayland-client roundtrip is blocking. Best-effort
// — overlay update + registry write already succeeded; surface a warning
// only if the warp itself failed.
- let real_warp_note = if crate::wayland::is_wayland() {
+ let real_warp_note = if crate::wayland::wayland_input_enabled() {
let xi = x.round() as i32;
let yi = y.round() as i32;
match tokio::task::spawn_blocking(move || {
@@ -4787,13 +5724,19 @@ impl Tool for SetAgentCursorMotionTool {
fn def(&self) -> &ToolDef {
CURSOR_DEF.get_or_init(|| ToolDef {
name: "set_agent_cursor_motion".into(),
- description: format!("Configure the visual appearance of an agent cursor instance.\n\n\
+ description: format!("Configure the visual appearance and motion curve of an agent cursor instance.\n\n\
- cursor_id: instance name (default='default')\n\
- cursor_icon: built-in ({}) or a path to a PNG/JPEG/SVG/ICO file; '' reverts to the default cursor\n\
- cursor_color: hex color e.g. '#00FFFF' or CSS name\n\
- cursor_label: short text shown near the cursor\n\
- cursor_size: dot radius in points (default=16)\n\
- - cursor_opacity: 0.0–1.0 (default=0.85)",
+ - cursor_opacity: 0.0–1.0 (default=0.85)\n\n\
+ Motion curve (Bezier):\n\
+ - arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\
+ - spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\
+ - glide_duration_ms: fixed flight duration per move [50,5000]; omit for speed-based\n\
+ - dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\
+ - idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000",
cursor_overlay::BuiltinShape::names_help()),
input_schema: json!({
"type":"object","properties":{
@@ -4803,13 +5746,30 @@ impl Tool for SetAgentCursorMotionTool {
"cursor_color":{"type":"string"},
"cursor_label":{"type":"string"},
"cursor_size":{"type":"number"},
- "cursor_opacity":{"type":"number"}
+ "cursor_opacity":{"type":"number"},
+ "start_handle":{"type":"number","description":"Start-handle fraction [0,1]. Default 0.3."},
+ "end_handle":{"type":"number","description":"End-handle fraction [0,1]. Default 0.3."},
+ "arc_size":{"type":"number","description":"Arc deflection as fraction of path length [0,1]. Default 0.25."},
+ "arc_flow":{"type":"number","description":"Asymmetry bias [-1,1]. Default 0.0."},
+ "spring":{"type":"number","description":"Settle damping [0.3,1.0]. Default 0.72."},
+ "glide_duration_ms":{"type":"number","minimum":50,"maximum":5000},
+ "dwell_after_click_ms":{"type":"number","minimum":0,"maximum":5000},
+ "idle_hide_ms":{"type":"number","minimum":0,"maximum":60000},
+ "turn_radius":{"type":"number","minimum":1,"maximum":1000}
},"additionalProperties":false
}),
read_only: false, destructive: false, idempotent: true, open_world: false,
})
}
async fn invoke(&self, args: Value) -> ToolResult {
+ fn num(value: Option<&Value>) -> Option<f64> {
+ value.and_then(|value| {
+ value
+ .as_f64()
+ .or_else(|| value.as_i64().map(|integer| integer as f64))
+ })
+ }
+
let cursor_id = resolve_cursor_key(&args);
// Resolve `cursor_icon` (built-in name or image path — same vocabulary as
// the CLI flags) to a shape override and dispatch it, so the overlay
@@ -4849,7 +5809,44 @@ impl Tool for SetAgentCursorMotionTool {
if let Some(cmd) = shape_cmd {
crate::overlay::send_command_for(cursor_id.clone(), cmd);
}
- ToolResult::text(format!("Cursor '{cursor_id}' config updated.")).with_structured(args)
+
+ let motion = crate::overlay::current_motion_for(&cursor_id).with_overrides(
+ num(args.get("start_handle")),
+ num(args.get("end_handle")),
+ num(args.get("arc_size")),
+ num(args.get("arc_flow")),
+ num(args.get("spring")),
+ num(args.get("glide_duration_ms")),
+ num(args.get("dwell_after_click_ms")),
+ num(args.get("idle_hide_ms")),
+ None,
+ num(args.get("turn_radius")),
+ );
+ crate::overlay::send_command_for(
+ cursor_id.clone(),
+ cursor_overlay::OverlayCommand::SetMotion(motion.clone()),
+ );
+
+ ToolResult::text(format!(
+ "Cursor '{cursor_id}' config updated. Motion: arc={:.2} spring={:.2} glide={}ms dwell={}ms idle={}ms",
+ motion.arc_size,
+ motion.spring,
+ motion.glide_duration_ms as u32,
+ motion.dwell_after_click_ms as u32,
+ motion.idle_hide_ms as u32,
+ ))
+ .with_structured(json!({
+ "cursor_id": cursor_id,
+ "start_handle": motion.start_handle,
+ "end_handle": motion.end_handle,
+ "arc_size": motion.arc_size,
+ "arc_flow": motion.arc_flow,
+ "spring": motion.spring,
+ "glide_duration_ms": motion.glide_duration_ms,
+ "dwell_after_click_ms": motion.dwell_after_click_ms,
+ "idle_hide_ms": motion.idle_hide_ms,
+ "turn_radius": motion.turn_radius,
+ }))
}
}
@@ -5597,14 +6594,14 @@ impl Tool for TypeTextCharsTool {
};
let text_len = text.chars().count();
let result = tokio::task::spawn_blocking(move || {
- if crate::wayland::is_wayland() {
+ if crate::wayland::wayland_input_enabled() {
// Per-char `wtype` loop with the requested delay — mirrors the
// X11 XSendEvent per-char path. Sleeping here is fine because
// we're inside spawn_blocking.
let mut buf = [0u8; 4];
for ch in text.chars() {
let s = ch.encode_utf8(&mut buf);
- crate::wayland::type_text(s)?;
+ crate::wayland::type_text(xid, s)?;
if delay_ms > 0 {
std::thread::sleep(std::time::Duration::from_millis(delay_ms));
}
@@ -5691,11 +6688,10 @@ impl Tool for BringToFrontTool {
proper timestamp handling to beat focus-stealing prevention) — call \
it before `delivery_mode:\"foreground\"` input to avoid a per-call \
flash, or to escalate when background injection didn't land. \
- Wayland: a standalone activate is NOT exposed — the compositor's \
- security model bundles activation into the virtual-pointer/click \
- path, so use `delivery_mode:\"foreground\"` on the input call \
- itself; this reports that constraint on Wayland rather than \
- faking it. Matches the macOS / Windows bring_to_front rung."
+ Wayland: activates through a target-addressable compositor adapter \
+ (wlroots foreign-toplevel or the GNOME Shell helper) and refuses \
+ when the compositor offers no safe adapter. Matches the macOS / \
+ Windows bring_to_front rung."
.into(),
input_schema: serde_json::json!({
"type":"object","required":["pid"],"properties":{
@@ -5709,33 +6705,41 @@ impl Tool for BringToFrontTool {
async fn invoke(&self, args: Value) -> ToolResult {
use cua_driver_core::tool_args::ArgsExt;
- // Wayland: no standalone external activate (compositor security model
- // bundles it into the vptr/click path). Report honestly; the agent
- // escalates via delivery_mode:"foreground" on the input call instead.
+ let pid = args.u64_or("pid", 0) as u32;
if crate::wayland::is_wayland() {
- return ToolResult::error(
- "bring_to_front: Wayland has no standalone window-activation API for \
- external clients — the compositor bundles activation into the \
- virtual-pointer/click path. Use delivery_mode:\"foreground\" on the \
- click/type_text call itself (it activates the target as part of the \
- injection)."
- .to_string(),
- )
- .with_structured(serde_json::json!({
- "code": "bring_to_front_wayland_bundled",
- "platform": "linux",
- "session": "wayland",
- "suggestion":
- "On Wayland, pass delivery_mode:\"foreground\" to click / type_text — \
- activation is performed as part of the injection.",
- }));
+ let window_id = match args.opt_u64("window_id") {
+ Some(window_id) => window_id,
+ None => match crate::wayland::list_windows_dispatch(Some(pid)).first() {
+ Some(window) => window.xid,
+ None => {
+ return ToolResult::error(format!(
+ "bring_to_front: no window_id given and no Wayland windows found for pid {pid}."
+ ))
+ }
+ },
+ };
+ let result = tokio::task::spawn_blocking(move || {
+ crate::wayland::activate_window_for_input_target(window_id, Some(pid))
+ })
+ .await;
+ return match result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "Brought Wayland window {window_id} to front."
+ ))
+ .with_structured(serde_json::json!({
+ "window_id": window_id,
+ "platform": "linux",
+ "session": "wayland",
+ })),
+ Ok(Err(error)) => ToolResult::error(error.to_string()),
+ Err(error) => ToolResult::error(format!("Task error: {error}")),
+ };
}
// X11: resolve the target xid (window_id, else first window for pid).
let xid = match args.opt_u64("window_id") {
Some(x) => x,
None => {
- let pid = args.u64_or("pid", 0) as u32;
let windows =
tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid)))
.await
@@ -5871,7 +6875,9 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
r.register(Box::new(ZoomTool {
state: state.clone(),
}));
- r.register(Box::new(TypeTextCharsTool));
+ // `type_text_chars` is a deprecated invoke-time alias for `type_text`.
+ // Keep it out of tools/list, matching the macOS and Windows registries.
+ let _: &TypeTextCharsTool = &TypeTextCharsTool;
// Cross-platform `page` tool definition lives in mcp-server; Linux plugs
// in its AT-SPI + CDP backend here.
r.register(Box::new(cua_driver_core::page::PageTool::new(Arc::new(
@@ -5884,7 +6890,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry {
#[cfg(test)]
mod click_button_schema_tests {
- use super::ClickTool;
+ use super::{chromium_background_must_refuse, maps_indicate_gtk, ClickTool};
use cua_driver_core::tool::Tool;
/// Surface 5: schema must advertise the three canonical button values and
@@ -5920,6 +6926,27 @@ mod click_button_schema_tests {
"description should call out wayland fallback"
);
}
+
+ #[test]
+ fn chromium_background_requires_focus_free_inject_mode() {
+ assert!(chromium_background_must_refuse(false, false, true));
+ assert!(!chromium_background_must_refuse(false, true, true));
+ assert!(!chromium_background_must_refuse(true, false, true));
+ assert!(!chromium_background_must_refuse(false, false, false));
+ }
+
+ #[test]
+ fn gtk_process_maps_are_detected_without_matching_unrelated_libraries() {
+ assert!(maps_indicate_gtk(
+ "7f00-7f01 r-xp /usr/lib/x86_64-linux-gnu/libgtk-3.so.0.2404.32"
+ ));
+ assert!(maps_indicate_gtk(
+ "7f00-7f01 r-xp /nix/store/hash-gtk4/lib/libgtk-4.so.1"
+ ));
+ assert!(!maps_indicate_gtk(
+ "7f00-7f01 r-xp /usr/lib/x86_64-linux-gnu/libgdk_pixbuf-2.0.so"
+ ));
+ }
}
#[cfg(test)]
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs
index 00d959dc93..5c820456fa 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs
@@ -2,7 +2,7 @@
//! Schemas match the macOS platform-macos implementations (cross-platform interface).
use async_trait::async_trait;
-use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};
+use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef}};
use serde_json::Value;
fn not_impl(name: &str) -> ToolResult {
@@ -235,7 +235,9 @@ pub fn build_registry() -> cua_driver_core::tool::ToolRegistry {
r.register(Box::new(SetConfigTool));
r.register(Box::new(GetAccessibilityTreeTool));
r.register(Box::new(ZoomTool));
- r.register(Box::new(TypeTextCharsTool));
+ // `type_text_chars` remains accepted as an invoke-time alias, but is not
+ // advertised as a separate tool on any platform.
+ let _: &TypeTextCharsTool = &TypeTextCharsTool;
r.register_recording_tools();
r.register_session_tools();
r
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/video_wayland.rs b/libs/cua-driver/rust/crates/platform-linux/src/video_wayland.rs
new file mode 100644
index 0000000000..291e7cb7fb
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/video_wayland.rs
@@ -0,0 +1,141 @@
+//! Native Wayland full-desktop video capture through `wf-recorder`.
+//!
+//! FFmpeg's Linux input backend is X11-only. On a native Wayland session,
+//! `wf-recorder` consumes the compositor's screencopy protocol and writes the
+//! same MP4 artifact shape expected by the shared recording session.
+
+use std::io::Read;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use cua_driver_core::video::{VideoBackend, VideoBackendFactory, VideoMetadata};
+
+pub struct WfRecorderVideoBackendFactory;
+
+impl VideoBackendFactory for WfRecorderVideoBackendFactory {
+ fn start(&self, output_path: &Path) -> anyhow::Result<Box<dyn VideoBackend>> {
+ WfRecorderVideoBackend::start(output_path)
+ .map(|backend| Box::new(backend) as Box<dyn VideoBackend>)
+ }
+}
+
+struct WfRecorderVideoBackend {
+ child: Child,
+ output_path: PathBuf,
+ started_at: Instant,
+ stderr_thread: Option<std::thread::JoinHandle<Vec<u8>>>,
+}
+
+impl WfRecorderVideoBackend {
+ fn start(output_path: &Path) -> anyhow::Result<Self> {
+ if std::env::var_os("WAYLAND_DISPLAY").is_none() {
+ anyhow::bail!("wf-recorder requires WAYLAND_DISPLAY");
+ }
+ let available = Command::new("wf-recorder")
+ .arg("--help")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()
+ .map(|status| status.success())
+ .unwrap_or(false);
+ if !available {
+ anyhow::bail!(
+ "wf-recorder not found on PATH. Install wf-recorder for native Wayland video."
+ );
+ }
+ if let Some(parent) = output_path.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+
+ let started_at = Instant::now();
+ let mut command = Command::new("wf-recorder");
+ command
+ .arg("-f")
+ .arg(output_path)
+ .args(["--no-damage", "-c", "libx264", "-x", "yuv420p"])
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::piped());
+ if let Some(output) = std::env::var_os("CUA_WAYLAND_RECORDING_OUTPUT") {
+ command.arg("-o").arg(output);
+ }
+ let mut child = command
+ .spawn()
+ .map_err(|error| anyhow::anyhow!("failed to start wf-recorder: {error}"))?;
+ let stderr_thread = child.stderr.take().map(|mut stderr| {
+ std::thread::spawn(move || {
+ let mut output = Vec::new();
+ let _ = stderr.read_to_end(&mut output);
+ if output.len() > 4096 {
+ let excess = output.len() - 4096;
+ output.drain(..excess);
+ }
+ output
+ })
+ });
+
+ let probe_deadline = Instant::now() + Duration::from_millis(1500);
+ while Instant::now() < probe_deadline {
+ if let Some(status) = child.try_wait()? {
+ let stderr = stderr_thread
+ .map(|worker| worker.join().unwrap_or_default())
+ .unwrap_or_default();
+ anyhow::bail!(
+ "wf-recorder exited during startup ({status}): {}",
+ String::from_utf8_lossy(&stderr)
+ );
+ }
+ std::thread::sleep(Duration::from_millis(100));
+ }
+
+ Ok(Self {
+ child,
+ output_path: output_path.to_path_buf(),
+ started_at,
+ stderr_thread,
+ })
+ }
+}
+
+impl VideoBackend for WfRecorderVideoBackend {
+ fn stop(mut self: Box<Self>) -> anyhow::Result<VideoMetadata> {
+ let elapsed = self.started_at.elapsed();
+ unsafe {
+ libc::kill(self.child.id() as i32, libc::SIGINT);
+ }
+ let deadline = Instant::now() + Duration::from_secs(10);
+ let finalized = loop {
+ if let Some(status) = self.child.try_wait()? {
+ break status.success();
+ }
+ if Instant::now() >= deadline {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ break false;
+ }
+ std::thread::sleep(Duration::from_millis(80));
+ };
+ let stderr = self
+ .stderr_thread
+ .take()
+ .and_then(|worker| worker.join().ok())
+ .unwrap_or_default();
+ let has_video = self
+ .output_path
+ .metadata()
+ .map(|metadata| metadata.len() > 0)
+ .unwrap_or(false);
+ if !finalized || !has_video {
+ anyhow::bail!(
+ "wf-recorder did not finalize a playable artifact: {}",
+ String::from_utf8_lossy(&stderr)
+ );
+ }
+ Ok(VideoMetadata {
+ path: self.output_path,
+ duration_ms: elapsed.as_millis() as u64,
+ finalized: true,
+ })
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs
new file mode 100644
index 0000000000..e5e6071a25
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs
@@ -0,0 +1,431 @@
+//! Generic staging `ext-foreign-toplevel-list-v1` window enumeration.
+
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
+
+use wayland_client::{
+ event_created_child, protocol::wl_registry, Connection, Dispatch, Proxy, QueueHandle,
+};
+use wayland_protocols::ext::foreign_toplevel_list::v1::client::{
+ ext_foreign_toplevel_handle_v1::{self as ext_handle, ExtForeignToplevelHandleV1},
+ ext_foreign_toplevel_list_v1::{
+ self as ext_list, ExtForeignToplevelListV1, EVT_TOPLEVEL_OPCODE,
+ },
+};
+
+use crate::x11::WindowInfo;
+
+/// Synthetic ext-toplevel IDs must survive the existing `window_id as u32`
+/// element-token paths. Keep them in a high, nonzero u32 namespace and reserve
+/// `u32::MAX` as an exhaustion sentinel rather than emitting it.
+const EXT_ID_NAMESPACE_START: u32 = 0xF000_0000;
+const EXT_ID_NAMESPACE_END: u32 = 0xFFFF_FFFE;
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+struct ToplevelRecord {
+ identifier: String,
+ title: String,
+ app_id: String,
+}
+
+#[derive(Default)]
+struct PendingRecord {
+ identifier: Option<String>,
+ title: Option<String>,
+ app_id: Option<String>,
+}
+
+#[derive(Default)]
+struct State {
+ manager: Option<ExtForeignToplevelListV1>,
+ pending: HashMap<u32, PendingRecord>,
+ toplevels: HashMap<u32, ToplevelRecord>,
+}
+
+impl Dispatch<wl_registry::WlRegistry, ()> for State {
+ fn event(
+ state: &mut Self,
+ registry: &wl_registry::WlRegistry,
+ event: wl_registry::Event,
+ _: &(),
+ _: &Connection,
+ qh: &QueueHandle<Self>,
+ ) {
+ if let wl_registry::Event::Global {
+ name,
+ interface,
+ version,
+ } = event
+ {
+ if interface == ExtForeignToplevelListV1::interface().name {
+ state.manager = Some(registry.bind::<ExtForeignToplevelListV1, _, _>(
+ name,
+ version.min(1),
+ qh,
+ (),
+ ));
+ }
+ }
+ }
+}
+
+impl Dispatch<ExtForeignToplevelListV1, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &ExtForeignToplevelListV1,
+ _: ext_list::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+
+ event_created_child!(State, ExtForeignToplevelListV1, [
+ EVT_TOPLEVEL_OPCODE => (ExtForeignToplevelHandleV1, ()),
+ ]);
+}
+
+impl Dispatch<ExtForeignToplevelHandleV1, ()> for State {
+ fn event(
+ state: &mut Self,
+ handle: &ExtForeignToplevelHandleV1,
+ event: ext_handle::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ let protocol_id = handle.id().protocol_id();
+ match event {
+ ext_handle::Event::Title { title } => {
+ state.pending.entry(protocol_id).or_default().title = Some(title);
+ }
+ ext_handle::Event::AppId { app_id } => {
+ state.pending.entry(protocol_id).or_default().app_id = Some(app_id);
+ }
+ ext_handle::Event::Identifier { identifier } => {
+ state.pending.entry(protocol_id).or_default().identifier = Some(identifier);
+ }
+ ext_handle::Event::Done => {
+ let pending = state.pending.remove(&protocol_id).unwrap_or_default();
+ let record = state.toplevels.entry(protocol_id).or_default();
+ if let Some(identifier) = pending.identifier {
+ record.identifier = identifier;
+ }
+ if let Some(title) = pending.title {
+ record.title = title;
+ }
+ if let Some(app_id) = pending.app_id {
+ record.app_id = app_id;
+ }
+ }
+ ext_handle::Event::Closed => {
+ state.pending.remove(&protocol_id);
+ state.toplevels.remove(&protocol_id);
+ handle.destroy();
+ }
+ _ => {}
+ }
+ }
+}
+
+struct IdRegistry {
+ by_identifier: HashMap<String, u32>,
+ by_id: HashMap<u32, String>,
+ next: Option<u32>,
+}
+
+impl Default for IdRegistry {
+ fn default() -> Self {
+ Self {
+ by_identifier: HashMap::new(),
+ by_id: HashMap::new(),
+ next: Some(EXT_ID_NAMESPACE_START),
+ }
+ }
+}
+
+impl IdRegistry {
+ fn id_for(&mut self, identifier: &str) -> anyhow::Result<u64> {
+ if let Some(id) = self.by_identifier.get(identifier) {
+ if self.by_id.get(id).is_some_and(|known| known == identifier) {
+ return Ok(u64::from(*id));
+ }
+ anyhow::bail!("ext toplevel id registry collision for identifier {identifier:?}");
+ }
+
+ let mut id = self
+ .next
+ .ok_or_else(|| anyhow::anyhow!("ext toplevel numeric id space exhausted"))?;
+ while self.by_id.contains_key(&id) {
+ id = id
+ .checked_add(1)
+ .filter(|candidate| *candidate <= EXT_ID_NAMESPACE_END)
+ .ok_or_else(|| anyhow::anyhow!("ext toplevel numeric id space exhausted"))?;
+ }
+ self.next = (id < EXT_ID_NAMESPACE_END).then_some(id + 1);
+ self.by_identifier.insert(identifier.to_owned(), id);
+ self.by_id.insert(id, identifier.to_owned());
+ Ok(u64::from(id))
+ }
+}
+
+fn id_registry() -> &'static Mutex<IdRegistry> {
+ static REGISTRY: OnceLock<Mutex<IdRegistry>> = OnceLock::new();
+ REGISTRY.get_or_init(|| Mutex::new(IdRegistry::default()))
+}
+
+fn stable_id(identifier: &str) -> anyhow::Result<u64> {
+ id_registry()
+ .lock()
+ .map_err(|_| anyhow::anyhow!("ext toplevel id registry lock poisoned"))?
+ .id_for(identifier)
+}
+
+/// Enumerate ext toplevels and enrich them with AT-SPI pid and geometry data.
+pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?;
+ if state.manager.is_none() {
+ anyhow::bail!("compositor does not expose ext_foreign_toplevel_list_v1");
+ }
+ for _ in 0..3 {
+ queue.roundtrip(&mut state)?;
+ }
+
+ let mut native = Vec::with_capacity(state.toplevels.len());
+ for record in state.toplevels.into_values() {
+ if record.identifier.is_empty() {
+ tracing::warn!("ignoring ext toplevel whose done batch had no identifier");
+ continue;
+ }
+ native.push((stable_id(&record.identifier)?, record));
+ }
+ native.sort_unstable_by_key(|(id, _)| *id);
+ Ok(merge_atspi_records(
+ native,
+ crate::atspi::list_windows(None),
+ ))
+}
+
+fn merge_atspi_records(
+ native: Vec<(u64, ToplevelRecord)>,
+ atspi: Vec<WindowInfo>,
+) -> Vec<WindowInfo> {
+ let mut claimed = vec![false; atspi.len()];
+ let mut out = Vec::with_capacity(native.len() + atspi.len());
+
+ for (id, record) in native {
+ let exact_title = atspi.iter().enumerate().find_map(|(index, window)| {
+ (!claimed[index] && !record.title.is_empty() && window.title == record.title)
+ .then_some(index)
+ });
+ let matching_atspi = exact_title.or_else(|| {
+ unique_match(&atspi, &claimed, |window| {
+ !record.app_id.is_empty() && window.app_name == record.app_id
+ })
+ });
+
+ let mut window = WindowInfo {
+ xid: id,
+ pid: None,
+ app_name: record.app_id,
+ title: record.title,
+ is_on_screen: true,
+ z_index: None,
+ x: 0,
+ y: 0,
+ width: 0,
+ height: 0,
+ };
+ if let Some(index) = matching_atspi {
+ claimed[index] = true;
+ let enrichment = &atspi[index];
+ window.pid = enrichment.pid;
+ window.is_on_screen = enrichment.is_on_screen;
+ window.z_index = enrichment.z_index;
+ window.x = enrichment.x;
+ window.y = enrichment.y;
+ window.width = enrichment.width;
+ window.height = enrichment.height;
+ if window.app_name.is_empty() {
+ window.app_name = enrichment.app_name.clone();
+ }
+ if window.title.is_empty() {
+ window.title = enrichment.title.clone();
+ }
+ }
+ out.push(window);
+ }
+
+ out.extend(
+ atspi
+ .into_iter()
+ .enumerate()
+ .filter_map(|(index, window)| (!claimed[index]).then_some(window)),
+ );
+ out
+}
+
+fn unique_match(
+ windows: &[WindowInfo],
+ claimed: &[bool],
+ predicate: impl Fn(&WindowInfo) -> bool,
+) -> Option<usize> {
+ let mut matches = windows
+ .iter()
+ .enumerate()
+ .filter(|(index, window)| !claimed[*index] && predicate(window));
+ let (index, _) = matches.next()?;
+ matches.next().is_none().then_some(index)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn record(identifier: &str, title: &str, app_id: &str) -> ToplevelRecord {
+ ToplevelRecord {
+ identifier: identifier.to_owned(),
+ title: title.to_owned(),
+ app_id: app_id.to_owned(),
+ }
+ }
+
+ fn atspi(xid: u64, pid: u32, title: &str, app_name: &str, x: i32) -> WindowInfo {
+ WindowInfo {
+ xid,
+ pid: Some(pid),
+ app_name: app_name.to_owned(),
+ title: title.to_owned(),
+ is_on_screen: true,
+ z_index: None,
+ x,
+ y: 20,
+ width: 800,
+ height: 600,
+ }
+ }
+
+ #[test]
+ fn id_registry_is_stable_and_bijective() {
+ let mut registry = IdRegistry::default();
+ let first = registry.id_for("opaque-a").unwrap();
+ let second = registry.id_for("opaque-b").unwrap();
+ assert_eq!(registry.id_for("opaque-a").unwrap(), first);
+ assert_ne!(first, second);
+ assert_eq!(
+ registry
+ .by_id
+ .get(&u32::try_from(first).unwrap())
+ .map(String::as_str),
+ Some("opaque-a")
+ );
+ assert_eq!(
+ registry
+ .by_id
+ .get(&u32::try_from(second).unwrap())
+ .map(String::as_str),
+ Some("opaque-b")
+ );
+ }
+
+ #[test]
+ fn merge_recovers_metadata_and_preserves_unmatched_records() {
+ let native = vec![
+ (
+ u64::from(EXT_ID_NAMESPACE_START),
+ record("a", "Editor", "org.editor"),
+ ),
+ (
+ u64::from(EXT_ID_NAMESPACE_START + 1),
+ record("b", "Other", "org.viewer"),
+ ),
+ (
+ u64::from(EXT_ID_NAMESPACE_START + 2),
+ record("c", "Native only", "org.native"),
+ ),
+ ];
+ let merged = merge_atspi_records(
+ native,
+ vec![
+ atspi(11, 101, "Editor", "unrelated-name", 10),
+ atspi(12, 202, "Different title", "org.viewer", 30),
+ atspi(13, 303, "AT-SPI only", "org.extra", 50),
+ ],
+ );
+
+ assert_eq!(merged.len(), 4);
+ assert_eq!(
+ (merged[0].xid, merged[0].pid, merged[0].x),
+ (u64::from(EXT_ID_NAMESPACE_START), Some(101), 10)
+ );
+ assert_eq!(merged[1].pid, Some(202));
+ assert_eq!(merged[2].pid, None);
+ assert_eq!((merged[3].xid, merged[3].pid), (13, Some(303)));
+ }
+
+ #[test]
+ fn merge_does_not_guess_ambiguous_app_ids() {
+ let merged = merge_atspi_records(
+ vec![(
+ u64::from(EXT_ID_NAMESPACE_START),
+ record("a", "", "org.same"),
+ )],
+ vec![
+ atspi(11, 101, "One", "org.same", 10),
+ atspi(12, 202, "Two", "org.same", 30),
+ ],
+ );
+ assert_eq!(merged.len(), 3);
+ assert_eq!(merged[0].pid, None);
+ assert_eq!(merged[1].pid, Some(101));
+ assert_eq!(merged[2].pid, Some(202));
+ }
+
+ #[test]
+ fn merge_pairs_repeated_exact_titles_one_to_one() {
+ let merged = merge_atspi_records(
+ vec![
+ (
+ u64::from(EXT_ID_NAMESPACE_START),
+ record("a", "Document", ""),
+ ),
+ (
+ u64::from(EXT_ID_NAMESPACE_START + 1),
+ record("b", "Document", ""),
+ ),
+ ],
+ vec![
+ atspi(11, 101, "Document", "first", 10),
+ atspi(12, 202, "Document", "second", 30),
+ ],
+ );
+ assert_eq!(merged.len(), 2);
+ assert_eq!(merged[0].pid, Some(101));
+ assert_eq!(merged[1].pid, Some(202));
+ }
+
+ #[test]
+ fn id_registry_skips_collisions_and_reports_exhaustion() {
+ let mut registry = IdRegistry::default();
+ registry
+ .by_id
+ .insert(EXT_ID_NAMESPACE_START, "already-used".to_owned());
+ assert_eq!(
+ registry.id_for("next").unwrap(),
+ u64::from(EXT_ID_NAMESPACE_START + 1)
+ );
+
+ registry.next = Some(EXT_ID_NAMESPACE_END);
+ assert_eq!(
+ registry.id_for("last").unwrap(),
+ u64::from(EXT_ID_NAMESPACE_END)
+ );
+ assert!(registry.id_for("exhausted").is_err());
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs
index 2265ba74cb..745c0732f4 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs
@@ -7,8 +7,8 @@
//! `zwlr_virtual_pointer_v1` / `zwlr_virtual_keyboard_v1` protocols.
//!
//! Sequence:
-//! 1. `ei::Context::connect_to_env()` — fast path when a $LIBEI_SOCKET is
-//! already exported (cua-compositor / inject mode).
+//! 1. `ei::Context::connect_to_env()` — fast path when a compositor or test
+//! environment already exports `$LIBEI_SOCKET`.
//! 2. Fallback: ashpd `RemoteDesktop::create_session` →
//! `select_devices(KEYBOARD|POINTER)` →
//! `start(session, parent)` (user consent dialog the first time) →
@@ -20,10 +20,10 @@
//! until the worker reports the request was flushed to the EIS
//! server.
//!
-//! Persistence: ashpd `PersistMode::Application` caches the consent grant
-//! for the lifetime of the requesting binary; restore_token written to
-//! `~/.config/cua-driver/libei.token` survives reboots (the worker
-//! reads it on startup).
+//! Persistence: ashpd `PersistMode::ExplicitlyRevoked` keeps the user's consent
+//! until they revoke it in desktop settings. The restore token is stored at
+//! `~/.config/cua-driver/libei-persistent.token` and reused across daemon
+//! restarts.
//!
//! Coordinates: ei_pointer_absolute uses LOGICAL PIXELS inside an
//! announced ei_device.Region — collected between device creation and
@@ -36,6 +36,7 @@ use std::sync::OnceLock;
use std::thread;
use crossbeam_channel::{bounded, Receiver, Sender};
+use xkbcommon::xkb;
/// Buttons the public API exposes. Mapped to evdev codes in the worker
/// thread so the libei surface only sees evdev integers.
@@ -56,9 +57,52 @@ impl Button {
}
}
+/// One evdev keyboard state transition in a [`key_sequence`] request.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum KeyTransition {
+ Press(u32),
+ Release(u32),
+}
+
+const MAX_KEY_SEQUENCE_TRANSITIONS: usize = 64;
+
+/// EIS frame timestamps are CLOCK_MONOTONIC microseconds, not per-command
+/// sequence numbers. Mutter discards transactions carrying the old 0,1,2...
+/// placeholders. Keep them strictly increasing even when several frames are
+/// emitted inside one microsecond.
+fn event_time_us() -> u64 {
+ use std::sync::atomic::{AtomicU64, Ordering};
+
+ static LAST: AtomicU64 = AtomicU64::new(0);
+ let mut ts = unsafe {
+ let mut value: libc::timespec = std::mem::zeroed();
+ if libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut value) == 0 {
+ (value.tv_sec as u64)
+ .saturating_mul(1_000_000)
+ .saturating_add((value.tv_nsec as u64) / 1_000)
+ } else {
+ 0
+ }
+ };
+ loop {
+ let previous = LAST.load(Ordering::Relaxed);
+ ts = ts.max(previous.saturating_add(1));
+ if LAST
+ .compare_exchange_weak(previous, ts, Ordering::Relaxed, Ordering::Relaxed)
+ .is_ok()
+ {
+ return ts;
+ }
+ }
+}
+
/// Commands the worker thread accepts. Each carries a reply channel so
/// the caller blocks until the EIS server has received the event.
enum Cmd {
+ WaitReady {
+ interface: &'static str,
+ reply: Sender<anyhow::Result<()>>,
+ },
Click {
x: f64,
y: f64,
@@ -83,6 +127,19 @@ enum Cmd {
keycode: u32,
reply: Sender<anyhow::Result<()>>,
},
+ KeySequence {
+ transitions: Vec<KeyTransition>,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ Drag {
+ from_x: f64,
+ from_y: f64,
+ to_x: f64,
+ to_y: f64,
+ steps: u32,
+ button: Button,
+ reply: Sender<anyhow::Result<()>>,
+ },
Shutdown,
}
@@ -92,9 +149,21 @@ fn tx() -> anyhow::Result<&'static Sender<Cmd>> {
TX.get().ok_or_else(|| anyhow::anyhow!("libei worker not started; call ensure_started() first"))
}
+fn wait_for_reply(rx: Receiver<anyhow::Result<()>>) -> anyhow::Result<()> {
+ rx.recv_timeout(std::time::Duration::from_secs(20))
+ .map_err(|error| match error {
+ crossbeam_channel::RecvTimeoutError::Timeout => anyhow::anyhow!(
+ "libei input backend did not become ready within 20s; the desktop portal may be waiting for Remote Desktop consent or its EIS session may be wedged"
+ ),
+ crossbeam_channel::RecvTimeoutError::Disconnected => {
+ anyhow::anyhow!("libei worker reply channel closed")
+ }
+ })?
+}
+
fn restore_token_path() -> Option<PathBuf> {
let base = dirs::config_dir()?;
- Some(base.join("cua-driver").join("libei.token"))
+ Some(base.join("cua-driver").join("libei-persistent.token"))
}
fn read_restore_token() -> Option<String> {
@@ -110,8 +179,21 @@ fn write_restore_token(token: &str) -> anyhow::Result<()> {
anyhow::anyhow!("failed to create {} for restore_token: {e}", parent.display())
})?;
}
- std::fs::write(&path, token)
- .map_err(|e| anyhow::anyhow!("failed to write libei restore_token to {}: {e}", path.display()))
+ use std::io::Write;
+ use std::os::unix::fs::OpenOptionsExt;
+
+ let mut file = std::fs::OpenOptions::new()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .mode(0o600)
+ .open(&path)
+ .map_err(|e| {
+ anyhow::anyhow!("failed to open libei restore_token at {}: {e}", path.display())
+ })?;
+ file.write_all(token.as_bytes()).map_err(|e| {
+ anyhow::anyhow!("failed to write libei restore_token to {}: {e}", path.display())
+ })
}
/// Spawn the libei worker thread (idempotent — safe to call from every
@@ -132,6 +214,34 @@ pub fn ensure_started() -> anyhow::Result<()> {
Ok(())
}
+fn wait_until_ready(interface: &'static str) -> anyhow::Result<()> {
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?
+ .send(Cmd::WaitReady {
+ interface,
+ reply: tx_r,
+ })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ wait_for_reply(rx_r)
+}
+
+/// Negotiate a resumed absolute-pointer device without emitting input.
+/// Callers can then restore target activation after a portal consent dialog.
+pub fn wait_pointer_ready() -> anyhow::Result<()> {
+ wait_until_ready("ei_pointer_absolute")
+}
+
+/// Negotiate a resumed scroll device without emitting input.
+pub fn wait_scroll_ready() -> anyhow::Result<()> {
+ wait_until_ready("ei_scroll")
+}
+
+/// Negotiate a resumed keyboard device without emitting input.
+pub fn wait_keyboard_ready() -> anyhow::Result<()> {
+ wait_until_ready("ei_keyboard")
+}
+
// ── Public API ───────────────────────────────────────────────────────────
/// Move the cursor to absolute (x, y) within the announced device region,
@@ -142,7 +252,7 @@ pub fn click(x: f64, y: f64, button: Button) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx()?.send(Cmd::Click { x, y, button, reply: tx_r })
.map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+ wait_for_reply(rx_r)
}
/// Move the cursor to absolute (x, y) inside the device region (no
@@ -152,7 +262,7 @@ pub fn move_absolute(x: f64, y: f64) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx()?.send(Cmd::MoveAbsolute { x, y, reply: tx_r })
.map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+ wait_for_reply(rx_r)
}
/// Scroll by (dx, dy) logical units. Positive y scrolls down.
@@ -161,7 +271,7 @@ pub fn scroll(dx: f64, dy: f64) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx()?.send(Cmd::Scroll { dx, dy, reply: tx_r })
.map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+ wait_for_reply(rx_r)
}
/// Inject a UTF-8 string via the `ei_text` interface (libei 1.6+).
@@ -171,7 +281,7 @@ pub fn type_text(text: &str) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx()?.send(Cmd::TypeText { text: text.to_string(), reply: tx_r })
.map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+ wait_for_reply(rx_r)
}
/// Press + release a key by evdev code (e.g. `KEY_ENTER` = 28). For
@@ -181,7 +291,74 @@ pub fn press_key(keycode: u32) -> anyhow::Result<()> {
let (tx_r, rx_r) = bounded(1);
tx()?.send(Cmd::PressKey { keycode, reply: tx_r })
.map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
- rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+ wait_for_reply(rx_r)
+}
+
+/// Submit an ordered sequence of evdev key press/release transitions.
+///
+/// The worker emits every transition in one emulation session and frames each
+/// transition separately, allowing callers to hold modifiers while pressing a
+/// key. Requests are capped to keep the synchronous worker responsive.
+pub fn key_sequence(transitions: &[KeyTransition]) -> anyhow::Result<()> {
+ validate_key_sequence(transitions)?;
+ if transitions.is_empty() {
+ return Ok(());
+ }
+
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?
+ .send(Cmd::KeySequence {
+ transitions: transitions.to_vec(),
+ reply: tx_r,
+ })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ wait_for_reply(rx_r)
+}
+
+fn validate_key_sequence(transitions: &[KeyTransition]) -> anyhow::Result<()> {
+ if transitions.len() > MAX_KEY_SEQUENCE_TRANSITIONS {
+ anyhow::bail!(
+ "libei key sequence has {} transitions; maximum is {}",
+ transitions.len(),
+ MAX_KEY_SEQUENCE_TRANSITIONS
+ );
+ }
+ Ok(())
+}
+
+fn emit_key_transitions(transitions: &[KeyTransition], mut emit: impl FnMut(KeyTransition, u64)) {
+ for (frame, transition) in transitions.iter().copied().enumerate() {
+ emit(transition, frame as u64);
+ }
+}
+
+/// Press `button` at (from_x, from_y), move through `steps` interpolated points
+/// to (to_x, to_y), then release — a genuine button-held drag (text selection /
+/// drag-and-drop / slider). Coordinates are absolute within the announced
+/// device region, like [`click`].
+pub fn drag(
+ from_x: f64,
+ from_y: f64,
+ to_x: f64,
+ to_y: f64,
+ steps: u32,
+ button: Button,
+) -> anyhow::Result<()> {
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?
+ .send(Cmd::Drag {
+ from_x,
+ from_y,
+ to_x,
+ to_y,
+ steps,
+ button,
+ reply: tx_r,
+ })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ wait_for_reply(rx_r)
}
/// Cleanly stop the worker thread.
@@ -197,9 +374,31 @@ pub fn shutdown() {
// the negotiated input region. It runs a calloop event loop that handles
// both the EIS protocol and the inbound command channel.
+/// Owns the resources that must outlive a single libei call so the portal
+/// RemoteDesktop + EIS session stays alive for the whole worker-thread lifetime.
+///
+/// GNOME/Mutter (and KDE) tie the RemoteDesktop session — and therefore the
+/// handed-off EIS fd — to the D-Bus connection that created it. The previous
+/// code dropped the ashpd proxy/session (and the tokio runtime backing their
+/// zbus connection) at the end of `open_eis_context`, before the calloop loop
+/// even started, so the session died immediately on those compositors (#2105).
+/// Holding these for the worker's lifetime keeps the connection — and the
+/// session — open. The direct `$LIBEI_SOCKET` fast path has no portal
+/// session, so it carries `None`.
+#[allow(dead_code)] // fields are keep-alive only; never read
+enum PortalKeepAlive {
+ None,
+ Portal {
+ _rt: tokio::runtime::Runtime,
+ _proxy: ashpd::desktop::remote_desktop::RemoteDesktop,
+ _session: ashpd::desktop::Session<ashpd::desktop::remote_desktop::RemoteDesktop>,
+ },
+}
+
fn worker(rx: Receiver<Cmd>) -> anyhow::Result<()> {
- // Phase 1 — acquire an EIS context.
- let context = open_eis_context()
+ // Phase 1 — acquire an EIS context (plus, on the portal path, a keep-alive
+ // that owns the ashpd RemoteDesktop session + its tokio runtime).
+ let (context, _portal_keepalive) = open_eis_context()
.map_err(|e| anyhow::anyhow!("failed to obtain EIS connection: {e}"))?;
// Phase 2 — handshake. The reis API requires we call context.handshake()
@@ -210,17 +409,19 @@ fn worker(rx: Receiver<Cmd>) -> anyhow::Result<()> {
// Phase 3 — run the calloop event loop. We use a calloop Generic source
// wrapping the ei::Context's underlying socket so the loop wakes up on
// EIS messages. The command channel is polled at the top of each loop
- // iteration.
+ // iteration. `_portal_keepalive` stays bound until this fn returns — i.e.
+ // across the entire loop — so the portal session is never dropped early (#2105).
run_calloop(context, rx)
}
-/// Open the EIS context. Fast path: $LIBEI_SOCKET (cua-compositor /
-/// inject mode). Fallback: ashpd portal RemoteDesktop handshake.
-fn open_eis_context() -> anyhow::Result<reis::ei::Context> {
+/// Open the EIS context. Fast path: `$LIBEI_SOCKET` supplied by the environment.
+/// Fallback: ashpd portal RemoteDesktop handshake.
+fn open_eis_context() -> anyhow::Result<(reis::ei::Context, PortalKeepAlive)> {
if let Some(ctx) = reis::ei::Context::connect_to_env()
.map_err(|e| anyhow::anyhow!("env ei socket open failed: {e}"))?
{
- return Ok(ctx);
+ // Direct $LIBEI_SOCKET fast path: no portal session to keep alive.
+ return Ok((ctx, PortalKeepAlive::None));
}
// Portal RemoteDesktop fallback.
@@ -233,12 +434,18 @@ fn open_eis_context() -> anyhow::Result<reis::ei::Context> {
use ashpd::enumflags2::BitFlags;
use std::os::unix::net::UnixStream;
- let rt = tokio::runtime::Builder::new_current_thread()
+ // A multi-threaded runtime (1 worker) so the zbus connection backing the
+ // RemoteDesktop session keeps being driven after this fn returns. The
+ // session must stay live for the whole worker lifetime (#2105); a
+ // current-thread runtime would freeze the connection the moment we stop
+ // calling `block_on`, and GNOME/KDE would tear the session down.
+ let rt = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(1)
.enable_all()
.build()
.map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?;
- let fd = rt.block_on(async {
+ let (fd, proxy, session) = rt.block_on(async {
let proxy = RemoteDesktop::new()
.await
.map_err(|e| anyhow::anyhow!("portal RemoteDesktop proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?;
@@ -249,7 +456,7 @@ fn open_eis_context() -> anyhow::Result<reis::ei::Context> {
let mut select_opts = SelectDevicesOptions::default()
.set_devices(BitFlags::<DeviceType>::from(DeviceType::Keyboard) | DeviceType::Pointer)
- .set_persist_mode(PersistMode::Application);
+ .set_persist_mode(PersistMode::ExplicitlyRevoked);
if let Some(tok) = read_restore_token() {
select_opts = select_opts.set_restore_token(Some(tok.as_str()));
}
@@ -270,9 +477,9 @@ fn open_eis_context() -> anyhow::Result<reis::ei::Context> {
// Best-effort persist of the restore_token. Without this, every
// process restart re-prompts the user for consent — with it, ashpd
- // 0.13's PersistMode::Application + the stored token together
+ // 0.13's explicitly-revoked mode and the stored token together
// skip the dialog for the lifetime of the persistence grant
- // (typically per login session on GNOME, indefinite on KDE).
+ // until the user explicitly revokes the grant in desktop settings.
if let Some(tok) = started.restore_token() {
let _ = write_restore_token(tok);
}
@@ -281,13 +488,13 @@ fn open_eis_context() -> anyhow::Result<reis::ei::Context> {
.connect_to_eis(&session, ConnectToEISOptions::default())
.await
.map_err(|e| anyhow::anyhow!("portal connect_to_eis failed: {e}"))?;
- anyhow::Ok(fd)
+ anyhow::Ok((fd, proxy, session))
})?;
let stream = UnixStream::from(fd);
let ctx = reis::ei::Context::new(stream)
.map_err(|e| anyhow::anyhow!("reis ei::Context::new failed: {e}"))?;
- Ok(ctx)
+ Ok((ctx, PortalKeepAlive::Portal { _rt: rt, _proxy: proxy, _session: session }))
}
// ── calloop dispatch ────────────────────────────────────────────────────
@@ -302,7 +509,7 @@ fn open_eis_context() -> anyhow::Result<reis::ei::Context> {
// Region selection picks the first announced ei_device::Region.
fn run_calloop(context: reis::ei::Context, rx: Receiver<Cmd>) -> anyhow::Result<()> {
- use calloop::{generic::Generic, EventLoop, Interest, Mode, PostAction};
+ use calloop::{generic::Generic, EventLoop, Interest, Mode};
let mut event_loop: EventLoop<EisState> = EventLoop::try_new()
.map_err(|e| anyhow::anyhow!("calloop EventLoop::try_new failed: {e}"))?;
@@ -318,13 +525,21 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver<Cmd>) -> anyhow::Result<
let mut state = EisState::default();
- // Command channel: drain at the top of each loop iteration with a
- // bounded timeout so the EIS source stays responsive.
+ // Inbound commands can arrive before the EIS handshake has negotiated a
+ // usable input device — seat → device → interface → Resumed is async and
+ // only advances while `dispatch` runs, and the portal consent-and-connect
+ // path (#2105) returns before the device is live. Running a command against
+ // a not-yet-negotiated device fails with "no EIS device negotiated yet". So
+ // queue commands and run each only once `input_ready()`; fail any that wait
+ // longer than READY_TIMEOUT (handle_command then replies with the natural
+ // "no device" error) so a caller blocked on the reply never hangs forever.
+ let mut pending: Vec<(Cmd, std::time::Instant)> = Vec::new();
+ const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
+
loop {
- // Process pending commands.
match rx.try_recv() {
Ok(Cmd::Shutdown) => break,
- Ok(cmd) => state.handle_command(cmd),
+ Ok(cmd) => pending.push((cmd, std::time::Instant::now())),
Err(crossbeam_channel::TryRecvError::Empty) => {}
Err(crossbeam_channel::TryRecvError::Disconnected) => break,
}
@@ -333,6 +548,21 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver<Cmd>) -> anyhow::Result<
.dispatch(Some(std::time::Duration::from_millis(20)), &mut state)
.map_err(|e| anyhow::anyhow!("calloop dispatch failed: {e}"))?;
+ // Run each queued command once the device IT needs is negotiated; time
+ // stale ones out. Per-command (not one global gate) so a keyboard-only
+ // session doesn't make keyboard commands wait 20s for a pointer device.
+ if !pending.is_empty() {
+ let mut still = Vec::with_capacity(pending.len());
+ for (cmd, queued) in pending.drain(..) {
+ if state.cmd_ready(&cmd) || queued.elapsed() >= READY_TIMEOUT {
+ state.handle_command(cmd);
+ } else {
+ still.push((cmd, queued));
+ }
+ }
+ pending = still;
+ }
+
// The handle_command path may have queued frame() requests; flush
// them once per loop iteration so the EIS server sees them.
if let Some(ctx) = state.context.as_ref() {
@@ -349,7 +579,21 @@ struct EisState {
seats: HashMap<reis::ei::Seat, SeatData>,
devices: HashMap<reis::ei::Device, DeviceData>,
sequence: u32,
+ /// The latest serial emitted by the EIS connection. Request serials are
+ /// connection-global (libei's `ei_get_serial`), while readiness is tracked
+ /// per device below.
last_serial: u32,
+ /// Last device lifecycle event. Mutter 46 may advertise a provisional
+ /// absolute pointer before seat binding, then a second live device. Waiting
+ /// briefly for this stream to settle avoids dispatching to the transient one.
+ last_device_event: Option<std::time::Instant>,
+ /// `char -> (evdev keycode, needs_shift)` built from the compositor's active
+ /// xkb keymap (delivered by the `ei_keyboard` device). Lets keycode typing
+ /// follow the user's real layout instead of assuming US. Empty until the
+ /// `Keymap` event arrives; typing then falls back to [`char_to_evdev`].
+ keymap_chars: HashMap<char, (u32, bool)>,
+ /// evdev keycode that produces `Shift_L` in the active keymap (else 42).
+ keymap_shift: Option<u32>,
}
#[derive(Default)]
@@ -388,6 +632,8 @@ struct DeviceData {
/// one Region.
regions: Vec<RegionRect>,
interfaces: HashMap<String, reis::Object>,
+ resumed: bool,
+ resumed_serial: Option<u32>,
}
impl EisState {
@@ -421,11 +667,20 @@ impl EisState {
handshake.handshake_version(1);
handshake.name("cua-driver");
handshake.context_type(reis::ei::handshake::ContextType::Sender);
+ // Advertise the interface set + versions that reis 0.7's own
+ // EIS client examples use against real compositors. The
+ // previous list advertised ei_device v2 and omitted
+ // ei_pingpong, which Mutter rejects — it closes the EIS
+ // connection immediately after our handshake `finish()`
+ // (observed: one Handshake event, then read() errors). reis's
+ // handshaker also requires ei_pingpong / ei_callback /
+ // ei_connection. See reis 0.7 examples/type-text.rs.
for (iface, ver) in [
("ei_callback", 1u32),
("ei_connection", 1),
+ ("ei_pingpong", 1),
("ei_seat", 1),
- ("ei_device", 2),
+ ("ei_device", 1),
("ei_pointer", 1),
("ei_pointer_absolute", 1),
("ei_button", 1),
@@ -438,11 +693,17 @@ impl EisState {
handshake.finish();
}
}
- Event::Connection(_conn, ev) => {
- if let reis::ei::connection::Event::Seat { seat } = ev {
+ Event::Connection(_conn, ev) => match ev {
+ reis::ei::connection::Event::Seat { seat } => {
self.seats.insert(seat, SeatData::default());
}
- }
+ // The EIS server pings to check liveness; failing to answer
+ // makes it drop the connection. Mirror reis's examples.
+ reis::ei::connection::Event::Ping { ping } => {
+ ping.done(0);
+ }
+ _ => {}
+ },
Event::Seat(seat, ev) => {
let data = self.seats.entry(seat.clone()).or_default();
match ev {
@@ -478,6 +739,12 @@ impl EisState {
}
}
Event::Device(device, ev) => {
+ self.last_device_event = Some(std::time::Instant::now());
+ if let reis::ei::device::Event::Destroyed { serial } = &ev {
+ self.last_serial = *serial;
+ self.devices.remove(&device);
+ return;
+ }
let data = self.devices.entry(device.clone()).or_default();
match ev {
reis::ei::device::Event::DeviceType { device_type } => {
@@ -499,89 +766,317 @@ impl EisState {
}
}
reis::ei::device::Event::Resumed { serial } => {
+ data.resumed = true;
+ data.resumed_serial = Some(serial);
+ self.last_serial = serial;
+ // A RemoteDesktop session is one emulation transaction,
+ // not one transaction per click. Mutter can neutralize a
+ // device stopped in the same batch as its frame. Match
+ // libei's reference client: start on Resumed and keep it
+ // active until the device is paused or disconnected.
+ self.sequence = self.sequence.wrapping_add(1);
+ device.start_emulating(serial, self.sequence);
+ }
+ reis::ei::device::Event::Paused { serial } => {
+ data.resumed = false;
+ data.resumed_serial = None;
self.last_serial = serial;
}
_ => {}
}
}
+ Event::Keyboard(
+ _kb,
+ reis::ei::keyboard::Event::Keymap {
+ keymap_type,
+ size,
+ keymap,
+ },
+ ) => {
+ if keymap_type == reis::ei::keyboard::KeymapType::Xkb {
+ self.load_xkb_keymap(keymap, size);
+ }
+ }
_ => {}
}
}
+ /// Compile the compositor's xkb keymap (from the `ei_keyboard` `Keymap` fd)
+ /// into a `char -> (evdev keycode, shift)` table + the Shift keycode, so
+ /// keycode typing follows the user's real layout instead of assuming US.
+ /// Best-effort: on any failure the table stays empty and typing falls back
+ /// to [`char_to_evdev`].
+ fn load_xkb_keymap(&mut self, fd: std::os::unix::io::OwnedFd, size: u32) {
+ use std::io::{Read, Seek, SeekFrom};
+ const XKB_EVDEV_OFFSET: u32 = 8;
+ const SHIFT_L: u32 = 0xffe1;
+
+ // Sanity-cap the size from the (trusted, but possibly buggy) compositor
+ // event: real xkb keymaps are a few KiB; refuse absurd values rather
+ // than attempt a multi-GB allocation.
+ if size == 0 || size as usize > 16 * 1024 * 1024 {
+ return;
+ }
+ let mut file = std::fs::File::from(fd);
+ let mut buf = vec![0u8; size as usize];
+ let _ = file.seek(SeekFrom::Start(0));
+ if file.read_exact(&mut buf).is_err() {
+ return;
+ }
+ // The keymap buffer is NUL-terminated text.
+ let text = match buf.iter().position(|&b| b == 0) {
+ Some(n) => String::from_utf8_lossy(&buf[..n]).into_owned(),
+ None => String::from_utf8_lossy(&buf).into_owned(),
+ };
+ let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS);
+ let keymap = match xkb::Keymap::new_from_string(
+ &ctx,
+ text,
+ xkb::KEYMAP_FORMAT_TEXT_V1,
+ xkb::KEYMAP_COMPILE_NO_FLAGS,
+ ) {
+ Some(k) => k,
+ None => return,
+ };
+
+ let mut chars: HashMap<char, (u32, bool)> = HashMap::new();
+ let mut shift = None;
+ for kc in keymap.min_keycode().raw()..=keymap.max_keycode().raw() {
+ let keycode = xkb::Keycode::new(kc);
+ let evdev = kc.wrapping_sub(XKB_EVDEV_OFFSET);
+ // Level 0 = unshifted, 1 = shifted; scan 0 first so the unshifted
+ // mapping wins when a char is reachable both ways.
+ for level in 0..=1u32 {
+ for sym in keymap.key_get_syms_by_level(keycode, 0, level) {
+ if shift.is_none() && sym.raw() == SHIFT_L {
+ shift = Some(evdev);
+ }
+ if let Some(ch) = sym.key_char() {
+ chars.entry(ch).or_insert((evdev, level == 1));
+ }
+ }
+ }
+ }
+ self.keymap_chars = chars;
+ self.keymap_shift = shift;
+ }
+
+ /// True once the device THIS command needs has been negotiated and
+ /// `Resumed`. Gating per-command (rather than on one global pointer check)
+ /// means a keyboard-only session doesn't stall keyboard commands waiting for
+ /// an absolute-pointer device that never arrives — and vice versa. Commands
+ /// run before their device exists fail with "no EIS device negotiated yet",
+ /// so `run_calloop` queues them until this holds (or READY_TIMEOUT elapses).
+ fn cmd_ready(&self, cmd: &Cmd) -> bool {
+ const DEVICE_QUIET_PERIOD: std::time::Duration =
+ std::time::Duration::from_millis(200);
+ if self
+ .last_device_event
+ .is_none_or(|last| last.elapsed() < DEVICE_QUIET_PERIOD)
+ {
+ return false;
+ }
+ match cmd {
+ Cmd::WaitReady { interface, .. } => self.device_with_interface(interface).is_some(),
+ Cmd::Click { .. } | Cmd::MoveAbsolute { .. } | Cmd::Drag { .. } => {
+ self.device_with_interface("ei_pointer_absolute").is_some()
+ }
+ Cmd::Scroll { .. } => self.device_with_interface("ei_scroll").is_some(),
+ Cmd::PressKey { .. } | Cmd::KeySequence { .. } => {
+ self.device_with_interface("ei_keyboard").is_some()
+ }
+ Cmd::TypeText { .. } => {
+ self.device_with_interface("ei_text").is_some()
+ || self.device_with_interface("ei_keyboard").is_some()
+ }
+ Cmd::Shutdown => true,
+ }
+ }
+
fn handle_command(&mut self, cmd: Cmd) {
let result = self.run_command(&cmd);
// Send the reply on whichever channel this command carries.
match cmd {
+ Cmd::WaitReady { reply, .. } => { let _ = reply.send(result); }
Cmd::Click { reply, .. } => { let _ = reply.send(result); }
Cmd::MoveAbsolute { reply, .. } => { let _ = reply.send(result); }
Cmd::Scroll { reply, .. } => { let _ = reply.send(result); }
Cmd::TypeText { reply, .. } => { let _ = reply.send(result); }
Cmd::PressKey { reply, .. } => { let _ = reply.send(result); }
+ Cmd::KeySequence { reply, .. } => { let _ = reply.send(result); }
+ Cmd::Drag { reply, .. } => { let _ = reply.send(result); }
Cmd::Shutdown => {}
}
}
fn run_command(&mut self, cmd: &Cmd) -> anyhow::Result<()> {
match cmd {
+ Cmd::WaitReady { interface, .. } => {
+ self.device_with_interface(interface).ok_or_else(|| {
+ anyhow::anyhow!("no resumed EIS {interface} device negotiated within 20s")
+ })?;
+ }
Cmd::Click { x, y, button, .. } => {
let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?;
- if let Some(ptr_abs) = device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device) {
- device.start_emulating(self.sequence, self.last_serial);
- self.sequence = self.sequence.wrapping_add(1);
- ptr_abs.motion_absolute(rel_x, rel_y);
- }
- if let Some(btn) = device_interface::<reis::ei::Button>(&self.devices, &device) {
- btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press);
- device.frame(self.last_serial, 0);
- btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released);
- device.frame(self.last_serial, 1);
+ let serial = self.last_serial;
+ let ptr_abs = require_device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device)?;
+ let btn = require_device_interface::<reis::ei::Button>(&self.devices, &device)?;
+
+ ptr_abs.motion_absolute(rel_x, rel_y);
+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press);
+ device.frame(serial, event_time_us());
+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released);
+ device.frame(serial, event_time_us());
+ }
+ Cmd::Drag {
+ from_x,
+ from_y,
+ to_x,
+ to_y,
+ steps,
+ button,
+ ..
+ } => {
+ let (device, from_rx, from_ry) = self.pointer_device_for(*from_x, *from_y)?;
+ let serial = self.last_serial;
+ let ptr_abs =
+ require_device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device)?;
+ let btn = require_device_interface::<reis::ei::Button>(&self.devices, &device)?;
+ // Map the drag end into the SAME device's region (a single EIS
+ // emulation session can't span devices). Fall back to the start
+ // region's offset when no region of this device contains the end
+ // (single-region device, or a point just off-screen) — this is
+ // correct only within one region, but avoids the cross-monitor
+ // mis-mapping of blindly reusing the start offset.
+ let (to_rx, to_ry) =
+ self.region_local_on(&device, *to_x, *to_y).unwrap_or_else(|| {
+ let off_x = *from_x as f32 - from_rx;
+ let off_y = *from_y as f32 - from_ry;
+ (*to_x as f32 - off_x, *to_y as f32 - off_y)
+ });
+
+ ptr_abs.motion_absolute(from_rx, from_ry);
+ device.frame(serial, event_time_us());
+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press);
+ device.frame(serial, event_time_us());
+ // Bound the interpolation: run_command executes synchronously in
+ // the worker loop, so a huge step count would block EIS event
+ // processing (incl. Ping) and balloon the unflushed request queue.
+ let n = (*steps).max(1).min(500);
+ for s in 1..=n {
+ let t = s as f32 / n as f32;
+ let ix = from_rx + (to_rx - from_rx) * t;
+ let iy = from_ry + (to_ry - from_ry) * t;
+ ptr_abs.motion_absolute(ix, iy);
+ device.frame(serial, event_time_us());
}
- device.stop_emulating(self.last_serial);
+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released);
+ device.frame(serial, event_time_us());
}
Cmd::MoveAbsolute { x, y, .. } => {
let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?;
- if let Some(ptr_abs) = device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device) {
- device.start_emulating(self.sequence, self.last_serial);
- self.sequence = self.sequence.wrapping_add(1);
- ptr_abs.motion_absolute(rel_x, rel_y);
- device.frame(self.last_serial, 0);
- device.stop_emulating(self.last_serial);
- }
+ let serial = self.last_serial;
+ let ptr_abs = require_device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device)?;
+
+ ptr_abs.motion_absolute(rel_x, rel_y);
+ device.frame(serial, event_time_us());
}
Cmd::Scroll { dx, dy, .. } => {
- let device = self.any_pointer_device()
- .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?;
- if let Some(scroll) = device_interface::<reis::ei::Scroll>(&self.devices, &device) {
- device.start_emulating(self.sequence, self.last_serial);
- self.sequence = self.sequence.wrapping_add(1);
- scroll.scroll(*dx as f32, *dy as f32);
- device.frame(self.last_serial, 0);
- device.stop_emulating(self.last_serial);
- }
+ let device = self.device_with_interface("ei_scroll")
+ .ok_or_else(|| anyhow::anyhow!("no EIS scroll device negotiated yet — wait for handshake"))?;
+ let serial = self.last_serial;
+ let scroll = require_device_interface::<reis::ei::Scroll>(&self.devices, &device)?;
+
+ scroll.scroll(*dx as f32, *dy as f32);
+ device.frame(serial, event_time_us());
}
Cmd::TypeText { text, .. } => {
- let device = self.any_pointer_device()
- .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?;
- if let Some(text_iface) = device_interface::<reis::ei::Text>(&self.devices, &device) {
- device.start_emulating(self.sequence, self.last_serial);
- self.sequence = self.sequence.wrapping_add(1);
+ // Prefer ei_text (libei 1.6+ — a UTF-8 string, layout-correct).
+ // Mutter does NOT advertise ei_text, so fall back to keycode
+ // typing via ei_keyboard: map each char to a US-layout evdev
+ // keycode + shift and emit press/release. Chars with no US-layout
+ // keycode are skipped (a documented limitation — full layout
+ // support needs an xkb keymap, like reis's type-text example).
+ if let Some(device) = self.device_with_interface("ei_text") {
+ let serial = self.last_serial;
+ let text_iface = require_device_interface::<reis::ei::Text>(&self.devices, &device)?;
text_iface.utf8(text);
- device.frame(self.last_serial, 0);
- device.stop_emulating(self.last_serial);
+ device.frame(serial, event_time_us());
+ } else {
+ use reis::ei::keyboard::KeyState;
+ let device = self.device_with_interface("ei_keyboard").ok_or_else(|| {
+ anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake")
+ })?;
+ let serial = self.last_serial;
+ let kb = require_device_interface::<reis::ei::Keyboard>(&self.devices, &device)?;
+ let shift_code = self.keymap_shift.unwrap_or(42);
+ let mut skipped = 0usize;
+ for ch in text.chars() {
+ // Prefer the compositor's actual keymap (layout-correct);
+ // fall back to the US-layout table if no keymap arrived.
+ let mapped = self
+ .keymap_chars
+ .get(&ch)
+ .copied()
+ .or_else(|| char_to_evdev(ch));
+ let Some((code, shift)) = mapped else {
+ // No keycode for this char in the keymap/US table
+ // (e.g. CJK/emoji via the keycode path). Skip it, but
+ // don't pretend the whole string was typed.
+ skipped += 1;
+ continue;
+ };
+ if shift {
+ kb.key(shift_code, KeyState::Press);
+ device.frame(serial, event_time_us());
+ }
+ kb.key(code, KeyState::Press);
+ device.frame(serial, event_time_us());
+ kb.key(code, KeyState::Released);
+ device.frame(serial, event_time_us());
+ if shift {
+ kb.key(shift_code, KeyState::Released);
+ device.frame(serial, event_time_us());
+ }
+ }
+ if skipped > 0 {
+ tracing::warn!(
+ "libei keycode typing skipped {skipped} char(s) with no \
+ keycode in the active keymap (e.g. non-Latin/emoji) — \
+ the ei_text path (libei 1.6+) would type these verbatim"
+ );
+ }
}
}
Cmd::PressKey { keycode, .. } => {
- let device = self.any_pointer_device()
- .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?;
- if let Some(kb) = device_interface::<reis::ei::Keyboard>(&self.devices, &device) {
- device.start_emulating(self.sequence, self.last_serial);
- self.sequence = self.sequence.wrapping_add(1);
- kb.key(*keycode, reis::ei::keyboard::KeyState::Press);
- device.frame(self.last_serial, 0);
- kb.key(*keycode, reis::ei::keyboard::KeyState::Released);
- device.frame(self.last_serial, 1);
- device.stop_emulating(self.last_serial);
- }
+ let device = self.device_with_interface("ei_keyboard")
+ .ok_or_else(|| anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake"))?;
+ let serial = self.last_serial;
+ let kb = require_device_interface::<reis::ei::Keyboard>(&self.devices, &device)?;
+
+ kb.key(*keycode, reis::ei::keyboard::KeyState::Press);
+ device.frame(serial, event_time_us());
+ kb.key(*keycode, reis::ei::keyboard::KeyState::Released);
+ device.frame(serial, event_time_us());
+ }
+ Cmd::KeySequence { transitions, .. } => {
+ let device = self.device_with_interface("ei_keyboard")
+ .ok_or_else(|| anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake"))?;
+ let serial = self.last_serial;
+ let kb = require_device_interface::<reis::ei::Keyboard>(&self.devices, &device)?;
+
+ emit_key_transitions(transitions, |transition, _frame| {
+ let (keycode, state) = match transition {
+ KeyTransition::Press(keycode) => {
+ (keycode, reis::ei::keyboard::KeyState::Press)
+ }
+ KeyTransition::Release(keycode) => {
+ (keycode, reis::ei::keyboard::KeyState::Released)
+ }
+ };
+ kb.key(keycode, state);
+ device.frame(serial, event_time_us());
+ });
}
Cmd::Shutdown => {}
}
@@ -603,12 +1098,12 @@ impl EisState {
/// back to the first pointer device with the raw coordinates, which
/// matches the pre-multi-monitor behaviour.
fn pointer_device_for(&self, x: f64, y: f64) -> anyhow::Result<(reis::ei::Device, f32, f32)> {
- let is_pointer = |data: &DeviceData| matches!(
- data.device_type,
- Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical)
- );
+ // Absolute motion needs the device that actually carries
+ // ei_pointer_absolute — Mutter/KWin split the relative and absolute
+ // pointers onto separate devices, and only one of them accepts an
+ // absolute warp.
for (device, data) in self.devices.iter() {
- if !is_pointer(data) {
+ if !data.resumed || !data.interfaces.contains_key("ei_pointer_absolute") {
continue;
}
for region in &data.regions {
@@ -619,20 +1114,140 @@ impl EisState {
}
}
}
- // Fallback: first pointer device, raw coords (single-region behaviour).
- let device = self.any_pointer_device()
- .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?;
+ // Fallback: first absolute-pointer device, raw coords (single-region).
+ let device = self.device_with_interface("ei_pointer_absolute")
+ .ok_or_else(|| anyhow::anyhow!("no EIS absolute-pointer device negotiated yet — wait for handshake"))?;
Ok((device, x as f32, y as f32))
}
- fn any_pointer_device(&self) -> Option<reis::ei::Device> {
- self.devices.iter()
- .find(|(_, data)| matches!(
- data.device_type,
- Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical)
- ))
+ /// Region-local coordinates for absolute `(x, y)` within one of `device`'s
+ /// announced regions, or `None` when no region of that device contains the
+ /// point. Used to map a drag endpoint into the same device the drag start
+ /// resolved to (a single EIS emulation session can't span devices).
+ fn region_local_on(&self, device: &reis::ei::Device, x: f64, y: f64) -> Option<(f32, f32)> {
+ let data = self.devices.get(device)?;
+ data.regions.iter().find(|r| r.contains(x, y)).map(|r| {
+ (x as f32 - r.offset_x, y as f32 - r.offset_y)
+ })
+ }
+
+ /// The negotiated device that exposes `iface` (e.g. `ei_pointer_absolute`,
+ /// `ei_scroll`, `ei_keyboard`, `ei_text`). Mutter/KWin announce one device
+ /// per capability group — a relative pointer, an absolute pointer, a
+ /// keyboard — so a command must pick the device carrying the interface it
+ /// needs, not just "the first virtual device" (which may be the relative
+ /// pointer that has no `ei_pointer_absolute`).
+ fn device_with_interface(&self, iface: &str) -> Option<reis::ei::Device> {
+ self.devices
+ .iter()
+ .filter(|(_, data)| {
+ data.resumed && data.interfaces.contains_key(iface)
+ })
+ .max_by_key(|(_, data)| data.resumed_serial.unwrap_or(0))
.map(|(d, _)| d.clone())
}
+
+}
+
+/// Map a character to a US-layout evdev keycode + whether Shift is needed.
+/// Used by the keyboard-typing fallback when the compositor's EIS exposes no
+/// `ei_text` interface (e.g. Mutter). Returns `None` for characters outside the
+/// US-ASCII printable set — those are skipped rather than mistyped. Full
+/// layout-correct typing would need an xkb keymap (see reis `type-text` example).
+fn char_to_evdev(c: char) -> Option<(u32, bool)> {
+ // Letter codes follow the QWERTY scancode layout (input-event-codes.h).
+ fn letter(c: char) -> u32 {
+ // KEY_A..KEY_Z follow the QWERTY scancode layout, not the alphabet.
+ match c {
+ 'a' => 30,
+ 'b' => 48,
+ 'c' => 46,
+ 'd' => 32,
+ 'e' => 18,
+ 'f' => 33,
+ 'g' => 34,
+ 'h' => 35,
+ 'i' => 23,
+ 'j' => 36,
+ 'k' => 37,
+ 'l' => 38,
+ 'm' => 50,
+ 'n' => 49,
+ 'o' => 24,
+ 'p' => 25,
+ 'q' => 16,
+ 'r' => 19,
+ 's' => 31,
+ 't' => 20,
+ 'u' => 22,
+ 'v' => 47,
+ 'w' => 17,
+ 'x' => 45,
+ 'y' => 21,
+ 'z' => 44,
+ _ => 0,
+ }
+ }
+ // (keycode, needs_shift). Digit/symbol codes and their shifted variants
+ // follow the US QWERTY row (input-event-codes.h).
+ Some(match c {
+ 'a'..='z' => (letter(c), false),
+ 'A'..='Z' => (letter(c.to_ascii_lowercase()), true),
+ '1' => (2, false),
+ '2' => (3, false),
+ '3' => (4, false),
+ '4' => (5, false),
+ '5' => (6, false),
+ '6' => (7, false),
+ '7' => (8, false),
+ '8' => (9, false),
+ '9' => (10, false),
+ '0' => (11, false),
+ '!' => (2, true),
+ '@' => (3, true),
+ '#' => (4, true),
+ '$' => (5, true),
+ '%' => (6, true),
+ '^' => (7, true),
+ '&' => (8, true),
+ '*' => (9, true),
+ '(' => (10, true),
+ ')' => (11, true),
+ ' ' => (57, false),
+ '\n' => (28, false),
+ '\t' => (15, false),
+ '-' => (12, false),
+ '_' => (12, true),
+ '=' => (13, false),
+ '+' => (13, true),
+ '[' => (26, false),
+ '{' => (26, true),
+ ']' => (27, false),
+ '}' => (27, true),
+ '\\' => (43, false),
+ '|' => (43, true),
+ ';' => (39, false),
+ ':' => (39, true),
+ '\'' => (40, false),
+ '"' => (40, true),
+ '`' => (41, false),
+ '~' => (41, true),
+ ',' => (51, false),
+ '<' => (51, true),
+ '.' => (52, false),
+ '>' => (52, true),
+ '/' => (53, false),
+ '?' => (53, true),
+ _ => return None,
+ })
+}
+
+fn require_device_interface<T: reis::Interface>(
+ devices: &HashMap<reis::ei::Device, DeviceData>,
+ device: &reis::ei::Device,
+) -> anyhow::Result<T> {
+ device_interface::<T>(devices, device)
+ .ok_or_else(|| anyhow::anyhow!("EIS device lacks required {} interface", T::NAME))
}
fn device_interface<T: reis::Interface>(
@@ -645,3 +1260,52 @@ fn device_interface<T: reis::Interface>(
.clone()
.downcast()
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn key_sequence_accepts_empty_and_maximum_length_requests() {
+ assert!(validate_key_sequence(&[]).is_ok());
+
+ let transitions = vec![KeyTransition::Press(29); MAX_KEY_SEQUENCE_TRANSITIONS];
+ assert!(validate_key_sequence(&transitions).is_ok());
+ }
+
+ #[test]
+ fn key_sequence_rejects_requests_over_the_worker_bound() {
+ let transitions = vec![KeyTransition::Release(29); MAX_KEY_SEQUENCE_TRANSITIONS + 1];
+ let error = validate_key_sequence(&transitions).unwrap_err();
+
+ assert_eq!(
+ error.to_string(),
+ "libei key sequence has 65 transitions; maximum is 64"
+ );
+ }
+
+ #[test]
+ fn key_sequence_preserves_transition_order_and_assigns_frames() {
+ let transitions = [
+ KeyTransition::Press(29),
+ KeyTransition::Press(46),
+ KeyTransition::Release(46),
+ KeyTransition::Release(29),
+ ];
+ let mut emitted = Vec::new();
+
+ emit_key_transitions(&transitions, |transition, frame| {
+ emitted.push((transition, frame));
+ });
+
+ assert_eq!(
+ emitted,
+ vec![
+ (KeyTransition::Press(29), 0),
+ (KeyTransition::Press(46), 1),
+ (KeyTransition::Release(46), 2),
+ (KeyTransition::Release(29), 3),
+ ]
+ );
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
index f214fa9e54..7aa572fc2f 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
@@ -1,8 +1,8 @@
//! Native-Wayland backend.
//!
-//! Used when running under a Wayland compositor with no X11 (WAYLAND_DISPLAY
-//! set, DISPLAY unset). Enumerates toplevels via
-//! `zwlr_foreign_toplevel_manager_v1`, captures per-output screenshots via
+//! Used when the experimental backend is enabled under a Wayland compositor.
+//! Enumerates toplevels via `zwlr_foreign_toplevel_manager_v1` or the generic
+//! staging `ext_foreign_toplevel_list_v1`, captures per-output screenshots via
//! `zwlr_screencopy_manager_v1` + `wl_shm` (native — `grim` remains a
//! fallback), and synthesises pointer / scroll / drag input via
//! `zwlr_virtual_pointer_v1`. Per-window image capture is deferred until
@@ -11,33 +11,25 @@
//! typed error on pure Wayland.
pub mod ext_screencopy;
+pub mod ext_toplevel;
pub mod overlay;
pub mod persistent_vptr;
pub mod portal_screenshot;
pub mod shell_helper;
-// `portal_screencast` (PipeWire per-window capture) and `libei` (GNOME/KDE
-// input via xdg-desktop-portal RemoteDesktop) need libpipewire-0.3 and reis
-// at build time, which the cross-platform release container (debian:11,
-// GLIBC_2.31 floor) can't satisfy without bumping the floor. They're behind
-// the `portal-libei` feature so the published binaries stay portable; the
-// Nix build (which already has modern PipeWire + libei from nixpkgs)
-// enables it. Wlroots screencopy + virtual-pointer remain unconditional.
-#[cfg(feature = "portal-libei")]
+pub mod sway_ipc;
+mod virtual_keyboard;
+// RemoteDesktop/libei input is portable and ships in release binaries.
+// PipeWire ScreenCast capture remains separately gated for modern/Nix builds.
+#[cfg(feature = "portal-input")]
pub mod libei;
-#[cfg(feature = "portal-libei")]
+#[cfg(feature = "portal-capture")]
pub mod portal_screencast;
-/// Whether this binary was compiled with the `portal-libei` feature — the
-/// xdg-desktop-portal RemoteDesktop + libei input path. It is the ONLY input
-/// backend that works on non-wlroots compositors (KWin/Plasma, Mutter/GNOME),
-/// which do not implement `zwlr_virtual_pointer_v1`. The published
-/// curl-pipe-bash tarball is built WITHOUT it (#1967 — debian:11 CD container
-/// lacks a new-enough PipeWire/libei), so on those compositors input injection
-/// has no backend and silently no-ops. Consulted by the doctor and the input
-/// dispatch so that failure is reported instead of hidden. See #1982.
-pub const PORTAL_LIBEI_ENABLED: bool = cfg!(feature = "portal-libei");
+/// Whether the GNOME/KDE RemoteDesktop + libei input backend is compiled in.
+pub const PORTAL_INPUT_ENABLED: bool = cfg!(feature = "portal-input");
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use std::sync::{Mutex, OnceLock};
use wayland_client::{
event_created_child,
@@ -96,15 +88,25 @@ pub fn wayland_enabled() -> bool {
}
}
-/// True when we should drive Wayland rather than X11: the experimental backend
-/// is opted in ([`wayland_enabled`]), a Wayland display is present, and there is
-/// no X11 DISPLAY to fall back to. Without the opt-in this returns false even on
-/// a pure-Wayland session, so the backend treats it as unsupported rather than
-/// silently engaging an incomplete code path.
+/// True when this is an opted-in Wayland desktop session.
+///
+/// GNOME and KDE export `DISPLAY` for XWayland even when the target and the
+/// desktop are native Wayland. Treating that compatibility variable as proof
+/// of an X11 session routed native windows, capture, video, geometry, and focus
+/// through invalid XIDs. Backend selection below is capability based; the mere
+/// presence of `DISPLAY` must not disable Wayland.
pub fn is_wayland() -> bool {
- wayland_enabled()
- && std::env::var_os("WAYLAND_DISPLAY").is_some()
- && std::env::var_os("DISPLAY").is_none()
+ wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some()
+}
+
+/// True when input tools should attempt the Wayland input path (wlroots
+/// virtual-pointer, falling back to libei/portal via [`with_libei_fallback`]).
+///
+/// Input-specific alias retained to make dispatch intent explicit. The
+/// wlroots-vs-portal decision is made from live compositor capabilities inside
+/// the `wayland::*` input functions, not from XWayland's `DISPLAY` variable.
+pub fn wayland_input_enabled() -> bool {
+ wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some()
}
/// Reason string when X11 input injection cannot possibly work, so callers
@@ -208,6 +210,99 @@ struct Toplevel {
closed: bool,
}
+#[derive(Clone, Debug)]
+struct ToplevelIdentity {
+ title: String,
+ app_id: String,
+}
+
+fn identity_registry() -> &'static Mutex<HashMap<u64, ToplevelIdentity>> {
+ static REGISTRY: OnceLock<Mutex<HashMap<u64, ToplevelIdentity>>> = OnceLock::new();
+ REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
+}
+
+fn observed_origin_registry() -> &'static Mutex<HashMap<u32, (i32, i32)>> {
+ static REGISTRY: OnceLock<Mutex<HashMap<u32, (i32, i32)>>> = OnceLock::new();
+ REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
+}
+
+pub fn remember_observed_window_origins(windows: &[WindowInfo]) {
+ if let Ok(mut registry) = observed_origin_registry().lock() {
+ for window in windows {
+ if let Some(pid) = window.pid {
+ // Generic foreign-toplevel and AT-SPI fallbacks use (0,0) when
+ // they do not know compositor geometry. Do not let that
+ // placeholder erase a previously observed real origin or
+ // prevent the caller from falling through to Sway/GNOME data.
+ if (window.x, window.y) != (0, 0) {
+ registry.insert(pid, (window.x, window.y));
+ }
+ }
+ }
+ }
+}
+
+pub fn observed_window_origin(pid: u32) -> Option<(i32, i32)> {
+ observed_origin_registry()
+ .lock()
+ .ok()
+ .and_then(|registry| registry.get(&pid).copied())
+}
+
+fn remember_identity(id: u64, toplevel: &Toplevel) {
+ if let Ok(mut registry) = identity_registry().lock() {
+ registry.insert(
+ id,
+ ToplevelIdentity {
+ title: toplevel.title.clone(),
+ app_id: toplevel.app_id.clone(),
+ },
+ );
+ }
+}
+
+fn identity_for(id: u64) -> Option<ToplevelIdentity> {
+ identity_registry()
+ .lock()
+ .ok()
+ .and_then(|registry| registry.get(&id).cloned())
+ .or_else(|| {
+ sway_ipc::window_for_id(id).map(|window| ToplevelIdentity {
+ title: window.title,
+ app_id: window.app_id,
+ })
+ })
+ .or_else(|| {
+ crate::atspi::list_windows(None)
+ .into_iter()
+ .find(|window| window.xid == id || u64::from(window.xid as u32) == id)
+ .map(|window| ToplevelIdentity {
+ title: window.title,
+ app_id: window.app_name,
+ })
+ })
+}
+
+fn matching_handle(state: &State, id: u64) -> Option<ZwlrForeignToplevelHandleV1> {
+ if let Some(identity) = identity_for(id) {
+ let by_title = state.toplevels.iter().find_map(|(protocol_id, toplevel)| {
+ (!identity.title.is_empty() && toplevel.title == identity.title)
+ .then(|| state.handles.get(protocol_id).cloned())
+ .flatten()
+ });
+ return by_title.or_else(|| {
+ state.toplevels.iter().find_map(|(protocol_id, toplevel)| {
+ (!identity.app_id.is_empty() && toplevel.app_id == identity.app_id)
+ .then(|| state.handles.get(protocol_id).cloned())
+ .flatten()
+ })
+ });
+ }
+
+ let protocol_id = u32::try_from(id).ok()?;
+ state.handles.get(&protocol_id).cloned()
+}
+
/// Per-capture in-flight state populated by the screencopy frame Dispatch.
#[derive(Default)]
struct CaptureState {
@@ -474,8 +569,9 @@ impl Dispatch<ZwlrForeignToplevelHandleV1, ()> for State {
}
/// Enumerate native Wayland toplevels via wlr-foreign-toplevel-management.
-/// `xid` is the foreign-toplevel handle's protocol id (a stable per-session
-/// window id); pid is unknown (not exposed by the protocol); geometry is 0
+/// `xid` begins as the foreign-toplevel handle's connection-scoped protocol id.
+/// The dispatcher replaces it with a stable compositor or AT-SPI identity when
+/// available. pid is unknown (not exposed by the protocol); geometry is 0
/// (the protocol does not surface position/size). app_id is folded into the
/// title (`"<title> [<app_id>]"`) so callers matching on either still match.
pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {
@@ -495,6 +591,8 @@ pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {
queue.roundtrip(&mut state)?;
}
+ let sway_windows = sway_ipc::list_windows().unwrap_or_default();
+ let mut used_sway_ids = HashSet::new();
let mut out = Vec::new();
for (id, tl) in &state.toplevels {
if tl.closed {
@@ -505,14 +603,36 @@ pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {
} else {
format!("{} [{}]", tl.title, tl.app_id)
};
+ let sway = sway_windows
+ .iter()
+ .find(|window| {
+ !used_sway_ids.contains(&window.id)
+ && !tl.title.is_empty()
+ && window.title == tl.title
+ })
+ .or_else(|| {
+ sway_windows.iter().find(|window| {
+ !used_sway_ids.contains(&window.id)
+ && !tl.app_id.is_empty()
+ && window.app_id == tl.app_id
+ })
+ });
+ let stable_id = sway.map(|window| window.id).unwrap_or(*id as u64);
+ if let Some(window) = sway {
+ used_sway_ids.insert(window.id);
+ }
+ remember_identity(stable_id, tl);
out.push(WindowInfo {
- xid: *id as u64,
- pid: None,
+ xid: stable_id,
+ pid: sway.map(|window| window.pid),
+ app_name: tl.app_id.clone(),
title,
- x: 0,
- y: 0,
- width: 0,
- height: 0,
+ is_on_screen: sway.map(|window| window.visible).unwrap_or(true),
+ z_index: None,
+ x: sway.map(|window| window.x).unwrap_or(0),
+ y: sway.map(|window| window.y).unwrap_or(0),
+ width: sway.map(|window| window.width).unwrap_or(0),
+ height: sway.map(|window| window.height).unwrap_or(0),
});
}
Ok(out)
@@ -756,12 +876,53 @@ pub(crate) unsafe fn borrowed_fd(fd: i32) -> std::os::fd::OwnedFd {
/// output-level path used by `get_window_state`'s vision payload.
pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
if is_wayland() {
- screenshot_bytes()
+ let bytes = screenshot_display_dispatch()?;
+ if let Some((x, y, width, height)) = window_geometry(xid) {
+ crop_png_to_rect(
+ &bytes,
+ x,
+ y,
+ width,
+ height,
+ &format!("Wayland window {xid}"),
+ )
+ } else {
+ Ok(bytes)
+ }
} else {
crate::capture::screenshot_window_bytes(xid)
}
}
+fn crop_png_to_rect(
+ output_png: &[u8],
+ rect_x: i32,
+ rect_y: i32,
+ rect_width: u32,
+ rect_height: u32,
+ label: &str,
+) -> anyhow::Result<Vec<u8>> {
+ let image = image::load_from_memory(output_png)?;
+ let image_width = image.width();
+ let image_height = image.height();
+ let x = rect_x.max(0) as u32;
+ let y = rect_y.max(0) as u32;
+ if x >= image_width || y >= image_height {
+ anyhow::bail!(
+ "{label} origin ({x},{y}) is outside captured output {image_width}x{image_height}"
+ );
+ }
+ let width = rect_width.min(image_width - x);
+ let height = rect_height.min(image_height - y);
+ if width == 0 || height == 0 {
+ anyhow::bail!("{label} has empty capture geometry");
+ }
+ let cropped = image.crop_imm(x, y, width, height);
+ let mut cursor = std::io::Cursor::new(Vec::new());
+ cropped.write_to(&mut cursor, image::ImageFormat::Png)?;
+ Ok(cursor.into_inner())
+}
+
/// Display-level capture dispatcher. Cascade:
/// 1. Native Wayland on wlroots: zwlr_screencopy_manager_v1 (fast, zero
/// consent).
@@ -771,7 +932,12 @@ pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
/// 3. X11: existing root-window path.
pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
if is_wayland() {
- // Tier 1: native wlroots screencopy (fast, zero consent).
+ // Tier 1: the opt-in GNOME compositor helper. It avoids probing
+ // wlroots-only protocols and captures the Shell stage without consent.
+ if let Some(bytes) = shell_helper::screenshot_display() {
+ return Ok(bytes);
+ }
+ // Tier 2: native wlroots screencopy (fast, zero consent).
match screenshot_bytes() {
Ok(bytes) => return Ok(bytes),
Err(e) => {
@@ -780,7 +946,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
);
}
}
- // Tier 2: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri,
+ // Tier 3: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri,
// hyprland, KDE 6.2+, GNOME 47+).
match ext_screencopy::screenshot_via_ext_copy() {
Ok(bytes) => return Ok(bytes),
@@ -790,7 +956,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
);
}
}
- // Tier 3: xdg-desktop-portal (GNOME, KDE, COSMIC fallback).
+ // Tier 4: xdg-desktop-portal (GNOME, KDE, COSMIC fallback).
match portal_screenshot::screenshot_via_portal() {
Ok(bytes) => return Ok(bytes),
Err(e) => {
@@ -813,6 +979,16 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
/// crop with.
pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
if is_wayland() {
+ if let Some((x, y, width, height)) = window_geometry(xid) {
+ return crop_png_to_rect(
+ &screenshot_display_dispatch()?,
+ x,
+ y,
+ width,
+ height,
+ &format!("Wayland window {xid}"),
+ );
+ }
anyhow::bail!(
"per-window screenshot is not yet supported on native Wayland — \
zwlr_screencopy_manager_v1 is output-only and ext-image-copy-capture-v1 \
@@ -825,6 +1001,50 @@ pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
// ── Input session helper ─────────────────────────────────────────────────────
+/// Sentinel substring carried by the `open_vptr_session` error when the
+/// compositor exposes no `zwlr_virtual_pointer_manager_v1` (KWin/Plasma,
+/// Mutter/GNOME). The input dispatch matches on this to decide whether the
+/// libei/portal fallback ([`libei`]) can recover the call. Kept as a string
+/// marker (rather than a typed error) so the existing `anyhow::Result`
+/// signatures of every input fn are unchanged. See #1982.
+pub const NO_VPTR_MARKER: &str = "no-zwlr-virtual-pointer";
+
+/// True when `err` is the "compositor has no wlroots virtual-pointer" failure
+/// from [`open_vptr_session`] — i.e. the point where a non-wlroots compositor
+/// needs the libei fallback rather than a hard error.
+fn is_no_vptr(err: &anyhow::Error) -> bool {
+ err.to_string().contains(NO_VPTR_MARKER)
+}
+
+/// Run the wlroots virtual-pointer closure `f`; if it fails specifically
+/// because the compositor exposes no `zwlr_virtual_pointer_manager_v1` and this
+/// binary carries the `portal-input` feature, run the libei `fallback` instead.
+/// Any other wlroots error (and the no-vptr error in a build without the
+/// feature) propagates unchanged. This is the single seam through which #1982's
+/// KDE/GNOME input recovery flows.
+fn with_libei_fallback<T>(
+ f: impl FnOnce() -> anyhow::Result<T>,
+ #[allow(unused_variables)] fallback: impl FnOnce() -> anyhow::Result<T>,
+) -> anyhow::Result<T> {
+ match f() {
+ Ok(v) => Ok(v),
+ Err(e) if is_no_vptr(&e) => {
+ #[cfg(feature = "portal-input")]
+ {
+ tracing::info!(
+ "wlroots virtual-pointer unavailable ({e}); falling back to libei/portal"
+ );
+ return fallback();
+ }
+ #[cfg(not(feature = "portal-input"))]
+ {
+ Err(e)
+ }
+ }
+ Err(e) => Err(e),
+ }
+}
+
/// Live virtual-pointer session: connection + queue + the bound objects every
/// pointer op (click, scroll, drag) needs. Returned by [`open_vptr_session`].
pub struct VptrSession {
@@ -843,7 +1063,7 @@ pub struct VptrSession {
/// client from knowing another window's on-screen geometry, so we drive every
/// pointer event in *output* coordinates and rely on the activated toplevel
/// covering the centre.
-pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<VptrSession> {
+pub fn open_vptr_session(activate_window_id: Option<u64>) -> anyhow::Result<VptrSession> {
let conn = Connection::connect_to_env()?;
let mut queue = conn.new_event_queue::<State>();
let qh = queue.handle();
@@ -851,38 +1071,53 @@ pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<Vptr
let mut state = State::default();
queue.roundtrip(&mut state)?;
- if state.manager.is_none() {
- anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");
- }
for _ in 0..4 {
queue.roundtrip(&mut state)?;
}
- let seat = state.seat.clone().ok_or_else(|| {
- anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input")
- })?;
+ // Evaluate the virtual-pointer / NO_VPTR_MARKER path FIRST: on compositors
+ // that expose neither zwlr_virtual_pointer nor zwlr_foreign_toplevel
+ // (KWin/Plasma, Mutter/GNOME) we must surface the marker so
+ // `with_libei_fallback` re-routes through libei/portal. Requiring
+ // foreign-toplevel up front would mask the marker and leave the libei
+ // fallback dead. See #1982.
let mgr = state.vptr_manager.clone().ok_or_else(|| {
- if PORTAL_LIBEI_ENABLED {
- anyhow::anyhow!("compositor does not expose zwlr_virtual_pointer_manager_v1")
+ if PORTAL_INPUT_ENABLED {
+ // The caller (via `with_libei_fallback`) recognises NO_VPTR_MARKER
+ // and re-routes the op through the libei/portal backend, which DOES
+ // reach KWin/Plasma and Mutter/GNOME. Keep the marker in the text.
+ anyhow::anyhow!(
+ "compositor does not expose zwlr_virtual_pointer_manager_v1 \
+ ({NO_VPTR_MARKER})"
+ )
} else {
// KWin/Plasma and Mutter/GNOME don't implement zwlr_virtual_pointer,
// and this build has no libei/portal fallback — so input has no
- // backend at all rather than silently no-op'ing. See #1982.
+ // backend at all rather than silently no-op'ing. The marker still
+ // lets the dispatch layer classify the failure uniformly. See #1982.
anyhow::anyhow!(
- "no input backend for this compositor: it exposes no \
- zwlr_virtual_pointer_manager_v1 and this build was compiled \
- without libei/portal support (#1982). Use the portal-enabled \
- Linux build for input on KDE Plasma / GNOME, or a wlroots \
- compositor (sway, labwc, hyprland)."
+ "no input backend for this compositor ({NO_VPTR_MARKER}): it \
+ exposes no zwlr_virtual_pointer_manager_v1 and this build was \
+ compiled without libei/portal support (#1982). Use the \
+ portal-enabled Linux build for input on KDE Plasma / GNOME, or \
+ a wlroots compositor (sway, labwc, hyprland)."
)
}
})?;
+ // foreign-toplevel is only needed to activate a specific window before
+ // synthesising input; require it only when a caller actually asks for that.
+ if activate_window_id.is_some() && state.manager.is_none() {
+ anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");
+ }
+
+ let seat = state
+ .seat
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input"))?;
+
if let Some(id) = activate_window_id {
- let handle = state
- .handles
- .get(&id)
- .cloned()
+ let handle = matching_handle(&state, id)
.ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {id}"))?;
handle.activate(&seat);
queue.roundtrip(&mut state)?;
@@ -901,6 +1136,106 @@ pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<Vptr
})
}
+/// Focus and raise a specific native Wayland toplevel before focus-bound
+/// keyboard or portal/libei input. wlroots exposes an activation request on its
+/// foreign-toplevel protocol; GNOME uses the bundled compositor helper. Other
+/// compositors must refuse until they provide an equally target-addressable
+/// adapter, because global injection without this gate can affect the wrong app.
+pub fn activate_window_for_input(window_id: u64) -> anyhow::Result<()> {
+ let pid = crate::atspi::list_windows(None)
+ .into_iter()
+ .find(|window| window.xid == window_id)
+ .and_then(|window| window.pid);
+ activate_window_for_input_target(window_id, pid)
+}
+
+/// Activate a Wayland target with an explicit process identity when available.
+/// The bundled compositor does not depend on connection-local Wayland object
+/// ids: its control protocol resolves the one mapped toplevel owned by `pid`.
+pub fn activate_window_for_input_target(
+ window_id: u64,
+ target_pid: Option<u32>,
+) -> anyhow::Result<()> {
+ if is_inject_mode() {
+ let pid = target_pid.ok_or_else(|| {
+ anyhow::anyhow!(
+ "foreground_unavailable: cua-compositor activation requires a verified target pid"
+ )
+ })?;
+ inject_send(&[format!("f {pid}")])?;
+ std::thread::sleep(std::time::Duration::from_millis(60));
+ return Ok(());
+ }
+
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?;
+ for _ in 0..4 {
+ queue.roundtrip(&mut state)?;
+ }
+
+ if let (Some(_), Some(seat), Some(handle)) = (
+ state.manager.as_ref(),
+ state.seat.clone(),
+ matching_handle(&state, window_id),
+ ) {
+ handle.activate(&seat);
+ queue.roundtrip(&mut state)?;
+ std::thread::sleep(std::time::Duration::from_millis(60));
+ return Ok(());
+ }
+
+ if shell_helper::activate_window(window_id) {
+ std::thread::sleep(std::time::Duration::from_millis(60));
+ return Ok(());
+ }
+
+ anyhow::bail!(
+ "foreground_unavailable: this Wayland compositor does not expose a verified, \
+ target-addressable activation adapter for window {window_id}; refusing global \
+ input because it could affect the wrong application"
+ )
+}
+
+/// Query the first `wl_output`'s pixel dimensions via a short Wayland
+/// roundtrip, independent of the virtual-pointer protocol. Used by the libei
+/// fallback (which never opens a `VptrSession`) to reproduce the vptr path's
+/// default-to-centre and clamp behaviour so both backends treat coordinates
+/// identically. Falls back to `(1, 1)` when no output reports a mode.
+#[cfg(feature = "portal-input")]
+fn output_dimensions() -> anyhow::Result<(u32, u32)> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?;
+ for _ in 0..4 {
+ queue.roundtrip(&mut state)?;
+ }
+ Ok((state.output_w.max(1), state.output_h.max(1)))
+}
+
+/// Reproduce the wlroots vptr path's coordinate handling for the libei
+/// fallback: `(0, 0)` defaults to the output centre, and any value is clamped
+/// to `[0, dim-1]`. Keeps `click(.., 0, 0, ..)` landing on centre rather than
+/// the top-left corner across both backends.
+#[cfg(feature = "portal-input")]
+fn normalize_click_xy(x: i32, y: i32, w: u32, h: u32) -> (i32, i32) {
+ let (px, py) = if x == 0 && y == 0 {
+ ((w / 2) as i32, (h / 2) as i32)
+ } else {
+ (x, y)
+ };
+ (
+ px.clamp(0, (w as i32).saturating_sub(1)),
+ py.clamp(0, (h as i32).saturating_sub(1)),
+ )
+}
+
/// Map a cua/X11 pointer button (1=left / 2=middle / 3=right) to its evdev
/// code, which is what `zwlr_virtual_pointer_v1::button` expects.
pub fn evdev_pointer_button(button: u8) -> u32 {
@@ -911,6 +1246,15 @@ pub fn evdev_pointer_button(button: u8) -> u32 {
}
}
+fn event_time_ms() -> u32 {
+ static START: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new();
+ START
+ .get_or_init(std::time::Instant::now)
+ .elapsed()
+ .as_millis()
+ .clamp(1, u32::MAX as u128) as u32
+}
+
/// Click a native Wayland toplevel identified by its `window_id` (the
/// foreign-toplevel protocol id from `list_windows`) at output-relative
/// `(x, y)`, with `button` (1/2/3 = left/middle/right) emitted `count` times.
@@ -919,7 +1263,41 @@ pub fn evdev_pointer_button(button: u8) -> u32 {
/// real coords. A short delay between iterations gives the compositor time
/// to discriminate single vs. double clicks.
pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> {
- let mut sess = open_vptr_session(Some(window_id as u32))?;
+ with_libei_fallback(
+ || click_vptr(Some(window_id), x, y, count, button),
+ || {
+ libei_wait_pointer_ready()?;
+ activate_window_for_input(window_id)?;
+ libei_click(x, y, count, button)
+ },
+ )
+}
+
+/// Click a desktop-absolute point without selecting or activating a toplevel.
+/// This is the Wayland peer of an XTest root-window click and is used only by
+/// the explicit desktop capture scope.
+pub fn click_desktop(x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> {
+ if is_inject_mode() {
+ let btn = evdev_button(button as u32);
+ return inject_send(&[format!("d {x} {y} {} {btn}", count.max(1))]);
+ }
+ with_libei_fallback(
+ || click_vptr(None, x, y, count, button),
+ || libei_click(x, y, count, button),
+ )
+}
+
+/// wlroots virtual-pointer implementation of [`click`]. Falls back to libei via
+/// [`with_libei_fallback`] when the compositor exposes no virtual-pointer.
+fn click_vptr(
+ window_id: Option<u64>,
+ x: i32,
+ y: i32,
+ count: u32,
+ button: u8,
+) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(window_id)?;
+ std::thread::sleep(std::time::Duration::from_millis(40));
let (w, h) = (sess.output_w, sess.output_h);
let (px, py) = if x == 0 && y == 0 {
((w / 2) as i32, (h / 2) as i32)
@@ -933,11 +1311,17 @@ pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow::
if i > 0 {
std::thread::sleep(std::time::Duration::from_millis(80));
}
- sess.vptr.motion_absolute(0, px, py, w, h);
+ sess.vptr.motion_absolute(event_time_ms(), px, py, w, h);
sess.vptr.frame();
- sess.vptr.button(0, btn, ButtonState::Pressed);
+ sess.queue.roundtrip(&mut sess.state)?;
+ std::thread::sleep(std::time::Duration::from_millis(15));
+ sess.vptr
+ .button(event_time_ms(), btn, ButtonState::Pressed);
sess.vptr.frame();
- sess.vptr.button(0, btn, ButtonState::Released);
+ sess.queue.roundtrip(&mut sess.state)?;
+ std::thread::sleep(std::time::Duration::from_millis(20));
+ sess.vptr
+ .button(event_time_ms(), btn, ButtonState::Released);
sess.vptr.frame();
sess.queue.roundtrip(&mut sess.state)?;
}
@@ -954,7 +1338,126 @@ pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow::
/// virtual-pointer protocol, mirroring how a real wheel notch decomposes. The
/// magnitude follows wl_pointer convention: ±10 (in wl_fixed = ×256) per tick.
pub fn scroll(window_id: u64, direction: &str, amount: u32) -> anyhow::Result<()> {
- let mut sess = open_vptr_session(Some(window_id as u32))?;
+ scroll_at(window_id, None, direction, amount)
+}
+
+/// Translate window-local screenshot coordinates into compositor output
+/// coordinates when the active compositor exposes the target geometry.
+pub fn window_local_to_output(window_id: u64, x: i32, y: i32) -> (i32, i32) {
+ window_geometry(window_id)
+ .map(|(window_x, window_y, _, _)| {
+ (window_x.saturating_add(x), window_y.saturating_add(y))
+ })
+ .unwrap_or((x, y))
+}
+
+/// Resolve geometry through stable title/app identity when a foreign-toplevel
+/// object ID came from an earlier Wayland connection. Protocol object IDs are
+/// connection-local, so direct equality is only a fast path.
+pub fn window_geometry(window_id: u64) -> Option<(i32, i32, u32, u32)> {
+ if let Some(window) = sway_ipc::window_for_id(window_id) {
+ return Some((window.x, window.y, window.width, window.height));
+ }
+
+ let identity = identity_for(window_id);
+ if let Some(identity) = identity.as_ref() {
+ if let Some(windows) = sway_ipc::list_windows() {
+ let title_matches = windows
+ .iter()
+ .filter(|window| !identity.title.is_empty() && window.title == identity.title)
+ .collect::<Vec<_>>();
+ if title_matches.len() == 1 {
+ let window = title_matches[0];
+ return Some((window.x, window.y, window.width, window.height));
+ }
+ let app_matches = windows
+ .iter()
+ .filter(|window| !identity.app_id.is_empty() && window.app_id == identity.app_id)
+ .collect::<Vec<_>>();
+ if app_matches.len() == 1 {
+ let window = app_matches[0];
+ return Some((window.x, window.y, window.width, window.height));
+ }
+ }
+ }
+
+ let windows = list_windows_dispatch(None);
+ if let Some(window) = windows
+ .iter()
+ .find(|window| window.xid == window_id && window.width > 0 && window.height > 0)
+ {
+ return Some((window.x, window.y, window.width, window.height));
+ }
+ let identity = identity?;
+ let title_matches = windows
+ .iter()
+ .filter(|window| {
+ window.width > 0
+ && window.height > 0
+ && !identity.title.is_empty()
+ && undecorated_native_title(window) == identity.title
+ })
+ .collect::<Vec<_>>();
+ if title_matches.len() == 1 {
+ let window = title_matches[0];
+ return Some((window.x, window.y, window.width, window.height));
+ }
+ let app_matches = windows
+ .iter()
+ .filter(|window| {
+ window.width > 0
+ && window.height > 0
+ && !identity.app_id.is_empty()
+ && window.app_name == identity.app_id
+ })
+ .collect::<Vec<_>>();
+ (app_matches.len() == 1).then(|| {
+ let window = app_matches[0];
+ (window.x, window.y, window.width, window.height)
+ })
+}
+
+/// Scroll after positioning the synthetic pointer over an output-relative
+/// target. Wayland routes wheel events to the surface beneath the pointer, so
+/// pixel-addressed scrolls must not inherit an unrelated cursor position.
+pub fn scroll_at(
+ window_id: u64,
+ point: Option<(i32, i32)>,
+ direction: &str,
+ amount: u32,
+) -> anyhow::Result<()> {
+ let direction = direction.to_string();
+ with_libei_fallback(
+ || scroll_vptr(window_id, point, &direction, amount),
+ || {
+ libei_wait_scroll_ready()?;
+ activate_window_for_input(window_id)?;
+ if let Some((x, y)) = point {
+ libei_move_absolute(x, y)?;
+ }
+ libei_scroll(&direction, amount)
+ },
+ )
+}
+
+/// wlroots virtual-pointer implementation of [`scroll`].
+fn scroll_vptr(
+ window_id: u64,
+ point: Option<(i32, i32)>,
+ direction: &str,
+ amount: u32,
+) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(Some(window_id))?;
+ if let Some((x, y)) = point {
+ let px = x.clamp(0, (sess.output_w as i32).saturating_sub(1)) as u32;
+ let py = y.clamp(0, (sess.output_h as i32).saturating_sub(1)) as u32;
+ sess.vptr
+ .motion_absolute(event_time_ms(), px, py, sess.output_w, sess.output_h);
+ sess.vptr.frame();
+ sess.queue.roundtrip(&mut sess.state)?;
+ record_synth_cursor(px as i32, py as i32);
+ std::thread::sleep(std::time::Duration::from_millis(15));
+ }
let (axis, sign): (Axis, i32) = match direction.to_ascii_lowercase().as_str() {
"up" => (Axis::VerticalScroll, -1),
"down" => (Axis::VerticalScroll, 1),
@@ -970,7 +1473,8 @@ pub fn scroll(window_id: u64, direction: &str, amount: u32) -> anyhow::Result<()
std::thread::sleep(std::time::Duration::from_millis(25));
}
sess.vptr.axis_source(AxisSource::Wheel);
- sess.vptr.axis_discrete(0, axis, value, sign);
+ sess.vptr
+ .axis_discrete(event_time_ms(), axis, value, sign);
sess.vptr.frame();
sess.queue.roundtrip(&mut sess.state)?;
}
@@ -1014,11 +1518,19 @@ pub fn last_synth_cursor_pos() -> Option<(i32, i32)> {
/// the compositor commits the warp before returning. Records the position in
/// the synthetic-cursor registry so `last_synth_cursor_pos` can report it.
pub fn move_cursor_absolute(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> {
- let mut sess = open_vptr_session(window_id.map(|w| w as u32))?;
+ with_libei_fallback(
+ || move_cursor_absolute_vptr(window_id, x, y),
+ || libei_move_absolute(x, y),
+ )
+}
+
+/// wlroots virtual-pointer implementation of [`move_cursor_absolute`].
+fn move_cursor_absolute_vptr(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(window_id)?;
let (w, h) = (sess.output_w, sess.output_h);
let px = x.clamp(0, (w as i32).saturating_sub(1)) as u32;
let py = y.clamp(0, (h as i32).saturating_sub(1)) as u32;
- sess.vptr.motion_absolute(0, px, py, w, h);
+ sess.vptr.motion_absolute(event_time_ms(), px, py, w, h);
sess.vptr.frame();
sess.queue.roundtrip(&mut sess.state)?;
record_synth_cursor(px as i32, py as i32);
@@ -1030,8 +1542,8 @@ pub fn move_cursor_absolute(window_id: Option<u64>, x: i32, y: i32) -> anyhow::R
/// Press-drag-release on a native Wayland toplevel. Emits one button press at
/// `(from_x, from_y)`, then `steps` interpolated motion events along the
/// straight segment to `(to_x, to_y)`, then a release. Coordinates are
-/// output-relative; window-local coords need the EIS inject socket
-/// (`CUA_INJECT_SOCKET`).
+/// output-relative; window-local coords need the nested cua-compositor
+/// injection socket (`CUA_INJECT_SOCKET`).
pub fn drag(
window_id: u64,
from_x: i32,
@@ -1041,7 +1553,29 @@ pub fn drag(
steps: u32,
button: u8,
) -> anyhow::Result<()> {
- let mut sess = open_vptr_session(Some(window_id as u32))?;
+ with_libei_fallback(
+ || drag_vptr(window_id, from_x, from_y, to_x, to_y, steps, button),
+ || {
+ libei_wait_pointer_ready()?;
+ activate_window_for_input(window_id)?;
+ libei_drag(from_x, from_y, to_x, to_y, steps, button)
+ },
+ )
+}
+
+/// wlroots virtual-pointer implementation of [`drag`].
+#[allow(clippy::too_many_arguments)]
+fn drag_vptr(
+ window_id: u64,
+ from_x: i32,
+ from_y: i32,
+ to_x: i32,
+ to_y: i32,
+ steps: u32,
+ button: u8,
+) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(Some(window_id))?;
+ std::thread::sleep(std::time::Duration::from_millis(40));
let (w, h) = (sess.output_w, sess.output_h);
let btn = evdev_pointer_button(button);
let clamp_xy = |x: i32, y: i32| -> (u32, u32) {
@@ -1051,9 +1585,12 @@ pub fn drag(
)
};
let (fx, fy) = clamp_xy(from_x, from_y);
- sess.vptr.motion_absolute(0, fx, fy, w, h);
+ sess.vptr.motion_absolute(event_time_ms(), fx, fy, w, h);
sess.vptr.frame();
- sess.vptr.button(0, btn, ButtonState::Pressed);
+ sess.queue.roundtrip(&mut sess.state)?;
+ std::thread::sleep(std::time::Duration::from_millis(15));
+ sess.vptr
+ .button(event_time_ms(), btn, ButtonState::Pressed);
sess.vptr.frame();
sess.queue.roundtrip(&mut sess.state)?;
let n = steps.max(1);
@@ -1062,15 +1599,18 @@ pub fn drag(
let ix = (from_x as f64 + (to_x - from_x) as f64 * t).round() as i32;
let iy = (from_y as f64 + (to_y - from_y) as f64 * t).round() as i32;
let (cx, cy) = clamp_xy(ix, iy);
- sess.vptr.motion_absolute(0, cx, cy, w, h);
+ sess.vptr
+ .motion_absolute(event_time_ms(), cx, cy, w, h);
sess.vptr.frame();
sess.queue.roundtrip(&mut sess.state)?;
std::thread::sleep(std::time::Duration::from_millis(8));
}
let (tx, ty) = clamp_xy(to_x, to_y);
- sess.vptr.motion_absolute(0, tx, ty, w, h);
+ sess.vptr.motion_absolute(event_time_ms(), tx, ty, w, h);
sess.vptr.frame();
- sess.vptr.button(0, btn, ButtonState::Released);
+ sess.queue.roundtrip(&mut sess.state)?;
+ sess.vptr
+ .button(event_time_ms(), btn, ButtonState::Released);
sess.vptr.frame();
// Sync the synthetic-cursor registry with the drag endpoint so a
// subsequent `get_cursor_position` reports where we left the pointer.
@@ -1088,38 +1628,58 @@ pub fn drag(
/// foreign-toplevel exposes no pid and Wayland delivers keys to the *focused*
/// surface, so this is window_id-free; pair it with `click`/`activate` to put
/// the intended window in focus first.
-pub fn type_text(text: &str) -> anyhow::Result<()> {
+pub fn type_text(window_id: u64, text: &str) -> anyhow::Result<()> {
if text.is_empty() {
return Ok(());
}
+ activate_window_for_input(window_id)?;
// Lead with a no-op Shift_L tap: on a freshly-focused window under a headless
// seat (notably sway), the compositor needs the first virtual-keyboard event
// to wire up keyboard routing, and that first key is dropped. Sacrificing a
// modifier tap (no character) absorbs the drop so the real text lands intact;
// it's harmless where routing is already live (labwc).
- let out = std::process::Command::new("wtype")
+ let result = std::process::Command::new("wtype")
.args(["-k", "Shift_L", "--"])
.arg(text)
- .output()?;
- if !out.status.success() {
- anyhow::bail!("wtype failed: {}", String::from_utf8_lossy(&out.stderr));
+ .output();
+ match result {
+ Ok(out) if out.status.success() => Ok(()),
+ // `wtype` relies on `zwp_virtual_keyboard_v1`, which KWin/Plasma and
+ // Mutter/GNOME don't implement (and the binary may be missing wtype
+ // entirely). On a portal-input build, route typing through libei's
+ // `ei_text` interface instead. See #1982.
+ other => with_wtype_libei_fallback(
+ || {
+ libei_wait_keyboard_ready()?;
+ activate_window_for_input(window_id)?;
+ libei_type_text(text)
+ },
+ other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()),
+ ),
}
- Ok(())
}
/// Press a single named key into the focused Wayland surface via `wtype -k`.
-pub fn press_key(key: &str) -> anyhow::Result<()> {
+pub fn press_key(window_id: u64, key: &str) -> anyhow::Result<()> {
+ activate_window_for_input(window_id)?;
let keysym = key_to_keysym(key);
- let out = std::process::Command::new("wtype")
- .args(["-k", &keysym])
- .output()?;
- if !out.status.success() {
- anyhow::bail!(
- "wtype -k {keysym} failed: {}",
- String::from_utf8_lossy(&out.stderr)
- );
+ // Keep the sacrificial modifier and requested key in one virtual-keyboard
+ // lifetime. Starting a second wtype process creates a fresh protocol object,
+ // causing headless seats to drop the requested key as their first event.
+ let result = std::process::Command::new("wtype")
+ .args(["-k", "Shift_L", "-k", &keysym])
+ .output();
+ match result {
+ Ok(out) if out.status.success() => Ok(()),
+ other => with_wtype_libei_fallback(
+ || {
+ libei_wait_keyboard_ready()?;
+ activate_window_for_input(window_id)?;
+ libei_press_key(key)
+ },
+ other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()),
+ ),
}
- Ok(())
}
/// Press a key combination (modifiers + final key) via `wtype`. Each modifier
@@ -1127,30 +1687,70 @@ pub fn press_key(key: &str) -> anyhow::Result<()> {
/// `wtype -M ctrl -M shift -k key -m shift -m ctrl`. Unknown values pass
/// straight to wtype's `-k` so single-character keys and X keysym names work
/// as-is. This is the Wayland equivalent of the X11 `send_key` modifier mask.
-pub fn hotkey(keys: &[String]) -> anyhow::Result<()> {
+pub fn hotkey(window_id: u64, keys: &[String]) -> anyhow::Result<()> {
+ activate_window_for_input(window_id)?;
let (mods, final_key) = partition_modifiers(keys)?;
+ if let Ok(()) = virtual_keyboard::hotkey(&mods, &final_key) {
+ return Ok(());
+ }
let keysym = key_to_keysym(&final_key);
- let mut args: Vec<String> = Vec::new();
- for m in &mods {
+ let args = wtype_hotkey_args(&mods, &keysym);
+ let result = std::process::Command::new("wtype").args(&args).output();
+ match result {
+ Ok(out) if out.status.success() => Ok(()),
+ other => {
+ let stderr = other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned());
+ #[cfg(feature = "portal-input")]
+ {
+ return with_wtype_libei_fallback(
+ || {
+ libei::wait_keyboard_ready()?;
+ activate_window_for_input(window_id)?;
+ libei_hotkey(&mods, &final_key)
+ },
+ stderr,
+ );
+ }
+ #[cfg(not(feature = "portal-input"))]
+ {
+ anyhow::bail!(
+ "wtype {} failed: {}",
+ args.join(" "),
+ stderr.unwrap_or_else(|_| "wtype unavailable".into())
+ );
+ }
+ }
+ }
+}
+
+fn wtype_hotkey_args(mods: &[String], keysym: &str) -> Vec<String> {
+ // Keep the same harmless first-event primer used by `press_key`. A fresh
+ // virtual-keyboard object on headless seats can drop its first event. Give
+ // wlroots one event cycle after the primer and modifier transitions;
+ // otherwise Chromium can miss a coalesced shortcut even though wtype exits
+ // successfully.
+ let mut args: Vec<String> = vec![
+ "-k".into(),
+ "Shift_L".into(),
+ "-s".into(),
+ "30".into(),
+ ];
+ for m in mods {
args.push("-M".into());
args.push(m.clone());
}
+ args.push("-s".into());
+ args.push("20".into());
args.push("-k".into());
- args.push(keysym.clone());
+ args.push(keysym.to_owned());
+ args.push("-s".into());
+ args.push("20".into());
// Release modifiers in reverse press order.
for m in mods.iter().rev() {
args.push("-m".into());
args.push(m.clone());
}
- let out = std::process::Command::new("wtype").args(&args).output()?;
- if !out.status.success() {
- anyhow::bail!(
- "wtype {} failed: {}",
- args.join(" "),
- String::from_utf8_lossy(&out.stderr)
- );
- }
- Ok(())
+ args
}
/// Split a `keys` array into wtype-compatible modifier names and a single
@@ -1199,29 +1799,478 @@ fn key_to_keysym(key: &str) -> String {
.to_string()
}
-// ── EIS nested-compositor injection ────────────────────────────────────────
+// ── libei / portal fallback adapters ───────────────────────────────────────
+//
+// These bridge the wlroots-shaped public input API (output-relative integer
+// coordinates, cua button codes, X-keysym key names) onto the libei worker
+// (`libei` module), which speaks logical device-region floats and evdev
+// codes. They are the recovery path for compositors with no
+// `zwlr_virtual_pointer_v1` (KWin/Plasma, Mutter/GNOME) — see #1982.
+//
+// In a build WITHOUT the `portal-input` feature the `libei` module does not
+// exist, so each adapter compiles to an error stub. The dispatch seams above
+// only ever CALL these inside `#[cfg(feature = "portal-input")]` branches, so
+// the stubs are dead in that build; they exist purely so the closures passed
+// to `with_libei_fallback` / `with_wtype_libei_fallback` type-check.
+
+/// libei recovery wrapper for the `wtype`-based typing/key functions: when the
+/// virtual-keyboard shell-out failed (`wtype_err`), try the libei `run` on a
+/// portal-input build, otherwise surface the original wtype failure.
+fn with_wtype_libei_fallback(
+ #[allow(unused_variables)] run: impl FnOnce() -> anyhow::Result<()>,
+ wtype_err: Result<String, std::io::Error>,
+) -> anyhow::Result<()> {
+ #[cfg(feature = "portal-input")]
+ {
+ match wtype_err {
+ Ok(stderr) => tracing::info!(
+ "wtype failed ({stderr}); falling back to libei/portal typing"
+ ),
+ Err(e) => tracing::info!(
+ "wtype unavailable ({e}); falling back to libei/portal typing"
+ ),
+ }
+ run()
+ }
+ #[cfg(not(feature = "portal-input"))]
+ {
+ let _ = run;
+ match wtype_err {
+ Ok(stderr) => anyhow::bail!("wtype failed: {stderr}"),
+ Err(e) => anyhow::bail!("wtype unavailable: {e}"),
+ }
+ }
+}
+
+// Stubs for the no-feature build: the dispatch seams never call these (the
+// libei branch in `with_libei_fallback` / `with_wtype_libei_fallback` is
+// `#[cfg]`-d out), but the closures still need them to exist to type-check.
+#[cfg(not(feature = "portal-input"))]
+fn libei_wait_pointer_ready() -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+fn libei_wait_scroll_ready() -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+fn libei_wait_keyboard_ready() -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+fn libei_click(_x: i32, _y: i32, _count: u32, _button: u8) -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+fn libei_scroll(_direction: &str, _amount: u32) -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+fn libei_move_absolute(_x: i32, _y: i32) -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+#[allow(clippy::too_many_arguments)]
+fn libei_drag(
+ _from_x: i32,
+ _from_y: i32,
+ _to_x: i32,
+ _to_y: i32,
+ _steps: u32,
+ _button: u8,
+) -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+fn libei_type_text(_text: &str) -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(not(feature = "portal-input"))]
+fn libei_press_key(_key: &str) -> anyhow::Result<()> {
+ unreachable!("libei fallback compiled out (no portal-input feature)")
+}
+#[cfg(feature = "portal-input")]
+fn cua_button_to_libei(button: u8) -> libei::Button {
+ match button {
+ 2 => libei::Button::Middle,
+ 3 => libei::Button::Right,
+ _ => libei::Button::Left,
+ }
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_wait_pointer_ready() -> anyhow::Result<()> {
+ libei::wait_pointer_ready()
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_wait_scroll_ready() -> anyhow::Result<()> {
+ libei::wait_scroll_ready()
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_wait_keyboard_ready() -> anyhow::Result<()> {
+ libei::wait_keyboard_ready()
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_click(x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> {
+ let btn = cua_button_to_libei(button);
+ let (w, h) = output_dimensions()?;
+ let (px, py) = normalize_click_xy(x, y, w, h);
+ libei::move_absolute(px as f64, py as f64)?;
+ for i in 0..count.max(1) {
+ if i > 0 {
+ std::thread::sleep(std::time::Duration::from_millis(80));
+ }
+ libei::click(px as f64, py as f64, btn)?;
+ }
+ record_synth_cursor(px, py);
+ Ok(())
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_scroll(direction: &str, amount: u32) -> anyhow::Result<()> {
+ // libei scroll is logical-unit deltas; mirror the wlroots ±10/tick step.
+ let (dx, dy): (f64, f64) = match direction.to_ascii_lowercase().as_str() {
+ "up" => (0.0, -10.0),
+ "down" => (0.0, 10.0),
+ "left" => (-10.0, 0.0),
+ "right" => (10.0, 0.0),
+ other => anyhow::bail!("unknown scroll direction: {other}"),
+ };
+ for i in 0..amount.max(1) {
+ if i > 0 {
+ std::thread::sleep(std::time::Duration::from_millis(25));
+ }
+ libei::scroll(dx, dy)?;
+ }
+ Ok(())
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_move_absolute(x: i32, y: i32) -> anyhow::Result<()> {
+ // Match `move_cursor_absolute_vptr`: clamp to output bounds (no
+ // default-to-centre — an explicit (0,0) move means the top-left corner).
+ let (w, h) = output_dimensions()?;
+ let px = x.clamp(0, (w as i32).saturating_sub(1));
+ let py = y.clamp(0, (h as i32).saturating_sub(1));
+ libei::move_absolute(px as f64, py as f64)?;
+ record_synth_cursor(px, py);
+ Ok(())
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_drag(
+ from_x: i32,
+ from_y: i32,
+ to_x: i32,
+ to_y: i32,
+ steps: u32,
+ button: u8,
+) -> anyhow::Result<()> {
+ // ei_button exposes separate Press/Released states, so the libei worker can
+ // hold the button across the interpolated motion — a genuine
+ // press→move→release drag. Clamp both endpoints to the output — but NOT via
+ // `normalize_click_xy`, whose (0,0)→centre convention (for coordinate-free
+ // clicks) is wrong here: a drag endpoint is always explicit and (0,0) is a
+ // valid top-left corner target.
+ let btn = cua_button_to_libei(button);
+ let (w, h) = output_dimensions()?;
+ let cx = |x: i32| x.clamp(0, (w as i32).saturating_sub(1));
+ let cy = |y: i32| y.clamp(0, (h as i32).saturating_sub(1));
+ libei::drag(
+ cx(from_x) as f64,
+ cy(from_y) as f64,
+ cx(to_x) as f64,
+ cy(to_y) as f64,
+ steps,
+ btn,
+ )?;
+ record_synth_cursor(cx(to_x), cy(to_y));
+ Ok(())
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_type_text(text: &str) -> anyhow::Result<()> {
+ if text.is_empty() {
+ return Ok(());
+ }
+ libei::type_text(text)
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_press_key(key: &str) -> anyhow::Result<()> {
+ let keycode = key_to_evdev(key)
+ .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}' (libei path)"))?;
+ libei::press_key(keycode)
+}
+
+#[cfg(feature = "portal-input")]
+fn libei_hotkey(mods: &[String], key: &str) -> anyhow::Result<()> {
+ use libei::KeyTransition::{Press, Release};
+
+ let mut modifier_codes = Vec::with_capacity(mods.len());
+ for modifier in mods {
+ modifier_codes.push(match modifier.as_str() {
+ "ctrl" => 29,
+ "shift" => 42,
+ "alt" => 56,
+ "logo" => 125,
+ other => anyhow::bail!("no evdev keycode mapping for modifier '{other}'"),
+ });
+ }
+ let keycode = key_to_evdev(key)
+ .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}' (libei path)"))?;
+ let mut transitions = Vec::with_capacity(modifier_codes.len() * 2 + 2);
+ transitions.extend(modifier_codes.iter().copied().map(Press));
+ transitions.push(Press(keycode));
+ transitions.push(Release(keycode));
+ transitions.extend(modifier_codes.iter().rev().copied().map(Release));
+ libei::key_sequence(&transitions)
+}
+
+/// Map cua key names to Linux evdev keycodes for the libei `press_key` path
+/// (libei emulates raw evdev, not X keysyms). Mirrors [`key_to_keysym`] but
+/// emits `linux/input-event-codes.h` values. Returns `None` for keys with no
+/// known mapping so the caller can fail loudly.
+fn key_to_evdev(key: &str) -> Option<u32> {
+ let code = match key.to_lowercase().as_str() {
+ "enter" | "return" => 28, // KEY_ENTER
+ "tab" => 15, // KEY_TAB
+ "esc" | "escape" => 1, // KEY_ESC
+ "space" => 57, // KEY_SPACE
+ "backspace" => 14, // KEY_BACKSPACE
+ "delete" | "del" => 111, // KEY_DELETE
+ "up" => 103, // KEY_UP
+ "down" => 108, // KEY_DOWN
+ "left" => 105, // KEY_LEFT
+ "right" => 106, // KEY_RIGHT
+ "home" => 102, // KEY_HOME
+ "end" => 107, // KEY_END
+ "pageup" | "page_up" => 104, // KEY_PAGEUP
+ "pagedown" | "page_down" => 109, // KEY_PAGEDOWN
+ // Letters a-z. evdev codes follow the QWERTY scancode layout, not the
+ // alphabet, so each is listed explicitly (linux/input-event-codes.h).
+ "a" => 30, // KEY_A
+ "b" => 48, // KEY_B
+ "c" => 46, // KEY_C
+ "d" => 32, // KEY_D
+ "e" => 18, // KEY_E
+ "f" => 33, // KEY_F
+ "g" => 34, // KEY_G
+ "h" => 35, // KEY_H
+ "i" => 23, // KEY_I
+ "j" => 36, // KEY_J
+ "k" => 37, // KEY_K
+ "l" => 38, // KEY_L
+ "m" => 50, // KEY_M
+ "n" => 49, // KEY_N
+ "o" => 24, // KEY_O
+ "p" => 25, // KEY_P
+ "q" => 16, // KEY_Q
+ "r" => 19, // KEY_R
+ "s" => 31, // KEY_S
+ "t" => 20, // KEY_T
+ "u" => 22, // KEY_U
+ "v" => 47, // KEY_V
+ "w" => 17, // KEY_W
+ "x" => 45, // KEY_X
+ "y" => 21, // KEY_Y
+ "z" => 44, // KEY_Z
+ // Digits. KEY_1=2 .. KEY_9=10, KEY_0=11 (input-event-codes.h).
+ "1" => 2, // KEY_1
+ "2" => 3, // KEY_2
+ "3" => 4, // KEY_3
+ "4" => 5, // KEY_4
+ "5" => 6, // KEY_5
+ "6" => 7, // KEY_6
+ "7" => 8, // KEY_7
+ "8" => 9, // KEY_8
+ "9" => 10, // KEY_9
+ "0" => 11, // KEY_0
+ // Function keys. KEY_F1=59 .. KEY_F10=68, then KEY_F11=87, KEY_F12=88.
+ "f1" => 59,
+ "f2" => 60,
+ "f3" => 61,
+ "f4" => 62,
+ "f5" => 63,
+ "f6" => 64,
+ "f7" => 65,
+ "f8" => 66,
+ "f9" => 67,
+ "f10" => 68,
+ "f11" => 87,
+ "f12" => 88,
+ _ => return None,
+ };
+ Some(code)
+}
+
+// ── Nested cua-compositor injection ────────────────────────────────────────
//
// When cua-driver's nested compositor is `cua-compositor` (our patched wlroots,
// see nix/cua-driver/compositor/), it exposes a line-protocol control socket at
// $CUA_INJECT_SOCKET for what stock Wayland forbids: focus-FREE per-surface
-// keyboard injection and MULTI-cursor pointer injection, both routed to a target
-// window by its xdg app_id. These helpers speak that protocol.
-
-/// The control socket path, when running against the EIS nested compositor.
+// keyboard injection and MULTI-cursor pointer injection, routed by stable PID
+// when available and xdg app_id only as a fallback. These helpers speak that
+// protocol.
+//
+// The protocol is a simple line-based v1 exchange with per-command
+// acknowledgement: the client sends `INJECT_PROTO_HELLO` and the compositor
+// echoes it (or replies `err ...`), then every command line is answered by
+// exactly one `ok` / `err <reason>` line. The client fails on a protocol
+// mismatch, a read timeout, an EOF, or any compositor error line — an
+// acknowledgement is transport evidence only, not proof the target changed.
+
+/// Version banner exchanged at connect time: the client sends this line and the
+/// compositor must echo it back verbatim to confirm both speak v1.
+const INJECT_PROTO_HELLO: &str = "cua-inject v1";
+
+/// The exact named keys the nested compositor's `k` command can emit — the
+/// whitelist in `cua_key_named` (cua_compositor_patch.py). Compared
+/// case-insensitively, matching the compositor's `strcasecmp`.
+const INJECT_NAMED_KEYS: &[&str] = &[
+ "enter", "return", "tab", "escape", "esc", "backspace", "space", "up", "down", "left", "right",
+ "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12",
+];
+
+/// The control socket path, when running against the nested cua-compositor.
pub fn inject_socket_path() -> Option<String> {
std::env::var("CUA_INJECT_SOCKET")
.ok()
.filter(|s| !s.is_empty())
}
-/// True when input should be routed through the EIS compositor's control socket
-/// (focus-free / multi-cursor) rather than wtype / virtual-pointer.
+/// True when input should be routed through the nested cua-compositor's control
+/// socket (focus-free / multi-cursor) rather than wtype / virtual-pointer.
pub fn is_inject_mode() -> bool {
inject_socket_path().is_some()
}
-fn inject_send(lines: &[String]) -> anyhow::Result<()> {
- use std::io::Write;
+/// Reject any character the nested compositor cannot type before it reaches the
+/// wire. The compositor's chartab (`cua_init_keymap`) only covers printable
+/// ASCII (`0x20..=0x7E`) plus newline and tab; anything else — Unicode, other
+/// control bytes — would be silently dropped, so fail loudly instead.
+fn validate_injectable_text(text: &str) -> anyhow::Result<()> {
+ for ch in text.chars() {
+ let ok = ch == '\n' || ch == '\t' || (ch.is_ascii() && !ch.is_ascii_control());
+ if !ok {
+ anyhow::bail!(
+ "cua-compositor cannot type {ch:?}: only printable ASCII plus newline and tab \
+ are supported in the v1 injection protocol"
+ );
+ }
+ }
+ Ok(())
+}
+
+/// Reject any key name outside the compositor's named-key whitelist before
+/// sending. Mirrors `cua_key_named`; unsupported names must fail here rather
+/// than being silently ignored by the compositor.
+fn validate_injectable_key(key: &str) -> anyhow::Result<()> {
+ let normalized = key.trim().to_ascii_lowercase();
+ if INJECT_NAMED_KEYS.contains(&normalized.as_str()) {
+ Ok(())
+ } else {
+ anyhow::bail!(
+ "cua-compositor does not support key {key:?}; supported keys: {}",
+ INJECT_NAMED_KEYS.join(", ")
+ );
+ }
+}
+
+fn validate_injectable_hotkey(keys: &[String]) -> anyhow::Result<(String, String)> {
+ let (key, modifiers) = keys
+ .split_last()
+ .ok_or_else(|| anyhow::anyhow!("cua-compositor hotkey requires a non-modifier key"))?;
+ let key = key.trim().to_ascii_lowercase();
+ if !(key.len() == 1 && key.is_ascii()) && !INJECT_NAMED_KEYS.contains(&key.as_str()) {
+ anyhow::bail!("cua-compositor does not support hotkey key {key:?}");
+ }
+ let mut normalized = Vec::with_capacity(modifiers.len());
+ for modifier in modifiers {
+ let modifier = modifier.trim().to_ascii_lowercase();
+ let canonical = match modifier.as_str() {
+ "ctrl" | "control" => "ctrl",
+ "shift" => "shift",
+ "alt" | "option" => "alt",
+ "meta" | "super" | "win" | "cmd" => "meta",
+ _ => anyhow::bail!("cua-compositor does not support modifier {modifier:?}"),
+ };
+ normalized.push(canonical);
+ }
+ if normalized.is_empty() {
+ anyhow::bail!("cua-compositor hotkey requires at least one modifier");
+ }
+ Ok((normalized.join(","), key))
+}
+
+/// Interpret the compositor's handshake reply. Accepts only the verbatim v1
+/// banner; a compositor `err ...` line or anything else is a protocol mismatch.
+fn parse_inject_hello(line: &str) -> anyhow::Result<()> {
+ let trimmed = line.trim();
+ if trimmed == INJECT_PROTO_HELLO {
+ Ok(())
+ } else if let Some(reason) = trimmed.strip_prefix("err") {
+ anyhow::bail!(
+ "cua-compositor rejected the v1 handshake:{}",
+ if reason.trim().is_empty() {
+ String::new()
+ } else {
+ format!(" {}", reason.trim())
+ }
+ )
+ } else {
+ anyhow::bail!(
+ "cua-compositor protocol mismatch: expected {INJECT_PROTO_HELLO:?}, got {trimmed:?}"
+ )
+ }
+}
+
+/// Interpret a single per-command acknowledgement line. `ok` succeeds; `err
+/// <reason>` and any unrecognised line fail.
+fn parse_inject_reply(line: &str) -> anyhow::Result<()> {
+ let trimmed = line.trim();
+ if trimmed == "ok" {
+ Ok(())
+ } else if let Some(reason) = trimmed.strip_prefix("err") {
+ let reason = reason.trim();
+ if reason.is_empty() {
+ anyhow::bail!("cua-compositor rejected the command");
+ }
+ anyhow::bail!("cua-compositor rejected the command: {reason}")
+ } else {
+ anyhow::bail!("unexpected cua-compositor response: {trimmed:?}")
+ }
+}
+
+/// Read one newline-terminated response line, mapping timeout and EOF to clear
+/// errors so the caller never blocks forever on an unresponsive compositor.
+fn read_inject_line(reader: &mut impl std::io::BufRead) -> anyhow::Result<String> {
+ let mut line = String::new();
+ let n = reader.read_line(&mut line).map_err(|e| {
+ if matches!(
+ e.kind(),
+ std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
+ ) {
+ anyhow::anyhow!("cua-compositor did not respond within the timeout")
+ } else {
+ anyhow::anyhow!("cua-compositor read failed: {e}")
+ }
+ })?;
+ if n == 0 {
+ anyhow::bail!("cua-compositor closed the connection before responding");
+ }
+ Ok(line)
+}
+
+/// Connect to the nested cua-compositor control socket, perform the v1
+/// handshake, then send each command line and require exactly one
+/// acknowledgement per command. Fails on protocol mismatch, timeout, EOF, or a
+/// compositor error line — the earlier fire-and-forget path hid all of these.
+fn inject_exchange(lines: &[String]) -> anyhow::Result<Vec<String>> {
+ use std::io::{BufReader, Write};
use std::os::unix::net::UnixStream;
let path = inject_socket_path().ok_or_else(|| anyhow::anyhow!("CUA_INJECT_SOCKET not set"))?;
// The nested compositor may still be starting; retry the connect briefly.
@@ -1235,36 +2284,80 @@ fn inject_send(lines: &[String]) -> anyhow::Result<()> {
Err(_) => std::thread::sleep(std::time::Duration::from_millis(50)),
}
}
- let mut s =
+ let stream =
stream.ok_or_else(|| anyhow::anyhow!("could not connect to inject socket {path}"))?;
- let mut buf = String::new();
+ stream.set_read_timeout(Some(std::time::Duration::from_secs(5)))?;
+ let mut writer = stream.try_clone()?;
+ let mut reader = BufReader::new(stream);
+
+ // v1 handshake: send our banner and require the compositor to echo it.
+ writeln!(writer, "{INJECT_PROTO_HELLO}")?;
+ writer.flush()?;
+ parse_inject_hello(&read_inject_line(&mut reader)?)?;
+
+ let mut replies = Vec::with_capacity(lines.len());
+ // One command per line; block on its response before the next.
for l in lines {
- buf.push_str(l);
- buf.push('\n');
+ writeln!(writer, "{l}")?;
+ writer.flush()?;
+ replies.push(read_inject_line(&mut reader)?);
+ }
+ Ok(replies)
+}
+
+fn inject_send(lines: &[String]) -> anyhow::Result<()> {
+ for reply in inject_exchange(lines)? {
+ parse_inject_reply(&reply)?;
}
- s.write_all(buf.as_bytes())?;
- s.flush()?;
- // Give the compositor a moment to process before the socket closes.
- std::thread::sleep(std::time::Duration::from_millis(80));
Ok(())
}
-/// Resolve a window_id (foreign-toplevel protocol id) to its xdg app_id by
-/// enumerating toplevels — the inject protocol addresses windows by app_id.
-pub fn app_id_for_window(window_id: u64) -> Option<String> {
- let conn = Connection::connect_to_env().ok()?;
- let mut queue = conn.new_event_queue::<State>();
- let qh = queue.handle();
- conn.display().get_registry(&qh, ());
- let mut state = State::default();
- queue.roundtrip(&mut state).ok()?;
- for _ in 0..4 {
- queue.roundtrip(&mut state).ok()?;
+fn parse_inject_geometry(line: &str) -> anyhow::Result<((i32, i32), (i32, i32))> {
+ let fields = line.split_whitespace().collect::<Vec<_>>();
+ if fields.len() == 5 && fields[0] == "geometry" {
+ return Ok((
+ (fields[1].parse()?, fields[2].parse()?),
+ (fields[3].parse()?, fields[4].parse()?),
+ ));
+ }
+ if let Some(reason) = line.trim().strip_prefix("err") {
+ anyhow::bail!("cua-compositor geometry query failed: {}", reason.trim());
+ }
+ anyhow::bail!("unexpected cua-compositor geometry response: {:?}", line.trim())
+}
+
+/// Return the offset that rebases native Wayland accessibility coordinates into
+/// the nested compositor's root-surface/output coordinate space.
+pub fn inject_accessibility_offset(pid: u32) -> Option<(i32, i32)> {
+ if !is_inject_mode() || pid == 0 {
+ return None;
}
- state
- .toplevels
- .get(&(window_id as u32))
- .map(|t| t.app_id.clone())
+ let replies = inject_exchange(&[format!("g {pid}")]).ok()?;
+ replies
+ .first()
+ .and_then(|line| parse_inject_geometry(line).ok())
+ .map(|geometry| geometry.0)
+}
+
+fn inject_window_origin(pid: u32) -> Option<(i32, i32)> {
+ if !is_inject_mode() || pid == 0 {
+ return None;
+ }
+ let replies = inject_exchange(&[format!("g {pid}")]).ok()?;
+ replies
+ .first()
+ .and_then(|line| parse_inject_geometry(line).ok())
+ .map(|geometry| geometry.1)
+}
+
+/// Resolve a window_id to its xdg app_id via the stable identity registry that
+/// [`list_windows`] populates (falling back to sway IPC / AT-SPI through
+/// [`identity_for`]) — the nested cua-compositor injection protocol addresses
+/// windows by app_id. Returns `None` when no identity is registered or the
+/// resolved app_id is empty, so callers can surface a clear error.
+pub fn app_id_for_window(window_id: u64) -> Option<String> {
+ identity_for(window_id)
+ .map(|identity| identity.app_id)
.filter(|s| !s.is_empty())
}
@@ -1295,25 +2388,77 @@ fn evdev_button(x_button: u32) -> u32 {
}
}
-/// Focus-free type into the window's surface (no focus change).
+/// Sentinel error when a window_id has no registered cua-compositor identity.
+fn no_app_id(window_id: u64) -> anyhow::Error {
+ anyhow::anyhow!(
+ "no known cua-compositor app_id for window {window_id}; call list_windows first so its \
+ Wayland identity is registered"
+ )
+}
+
+/// Resolve the strongest target token understood by the private nested
+/// compositor. AT-SPI window IDs are synthetic on Wayland, but its process ID
+/// is the same credential the compositor observes on the owning wl_client.
+/// Fall back to app_id for clients whose accessibility metadata has no PID.
+pub fn inject_target_for_window(window_id: u64) -> anyhow::Result<String> {
+ if let Some(pid) = crate::atspi::list_windows(None)
+ .into_iter()
+ .find(|window| window.xid == window_id)
+ .and_then(|window| window.pid)
+ {
+ return Ok(format!("pid:{pid}"));
+ }
+ app_id_for_window(window_id).ok_or_else(|| no_app_id(window_id))
+}
+
+/// Focus-free type into the window's surface (no focus change). Rejects any
+/// character the compositor cannot emit before touching the socket.
pub fn inject_type_text(window_id: u64, text: &str) -> anyhow::Result<()> {
- let app = app_id_for_window(window_id)
- .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;
+ validate_injectable_text(text)?;
+ let app = inject_target_for_window(window_id)?;
inject_send(&[format!("t {app} {}", to_hex(text))])
}
-/// Focus-free named-key press into the window's surface.
+/// Focus-free named-key press into the window's surface. Rejects any key
+/// outside the compositor's whitelist before touching the socket.
pub fn inject_press_key(window_id: u64, key: &str) -> anyhow::Result<()> {
- let app = app_id_for_window(window_id)
- .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;
- inject_send(&[format!("k {app} {key}")])
+ validate_injectable_key(key)?;
+ let app = inject_target_for_window(window_id)?;
+ inject_send(&[format!("k {app} {}", key.trim())])
+}
+
+/// Focus-free modifier chord into the target surface.
+pub fn inject_hotkey(window_id: u64, keys: &[String]) -> anyhow::Result<()> {
+ let (modifiers, key) = validate_injectable_hotkey(keys)?;
+ let app = inject_target_for_window(window_id)?;
+ inject_send(&[format!("h {app} {modifiers} {key}")])
+}
+
+/// Focus-free wheel/axis input at one target-local point.
+pub fn inject_scroll(
+ window_id: u64,
+ x: f64,
+ y: f64,
+ direction: &str,
+ amount: u32,
+) -> anyhow::Result<()> {
+ let app = inject_target_for_window(window_id)?;
+ let (axis, value) = match direction.to_ascii_lowercase().as_str() {
+ "up" => (0, -15.0),
+ "down" | "page" => (0, 15.0),
+ "left" => (1, -15.0),
+ "right" => (1, 15.0),
+ _ => anyhow::bail!("unsupported cua-compositor scroll direction {direction:?}"),
+ };
+ let mut lines = vec![format!("m {app} 0 {x:.1} {y:.1}")];
+ lines.extend((0..amount.max(1)).map(|_| format!("a {app} 0 {axis} {value:.1}")));
+ inject_send(&lines)
}
-/// Focus-free click into the window's surface via the nested EIS compositor.
+/// Focus-free click into the window's surface via the nested cua-compositor.
/// Coordinates are window-local, matching the rest of the inject protocol.
pub fn inject_click(window_id: u64, x: f64, y: f64, count: u32, button: u8) -> anyhow::Result<()> {
- let app = app_id_for_window(window_id)
- .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?;
+ let app = inject_target_for_window(window_id)?;
let btn = evdev_button(button as u32);
let n = count.max(1);
let mut lines = Vec::with_capacity((n as usize) * 4);
@@ -1417,28 +2562,117 @@ pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> {
inject_send(&lines)
}
-/// Window-enumeration dispatcher: native Wayland when applicable, else X11.
+/// Focus-free single drag using the same per-surface path as parallel drags.
+pub fn inject_drag(
+ window_id: u64,
+ from: (f64, f64),
+ to: (f64, f64),
+ steps: usize,
+ x_button: u32,
+) -> anyhow::Result<()> {
+ let app_id = inject_target_for_window(window_id)?;
+ inject_parallel_drags(&[InjectDrag {
+ app_id,
+ idx: 0,
+ x_button,
+ path: vec![from, to],
+ steps,
+ }])
+}
+
+fn wayland_atspi_windows(filter_pid: Option<u32>) -> Vec<WindowInfo> {
+ let mut windows = crate::atspi::list_windows(filter_pid);
+ // AT-SPI can retain a toolkit's default placement (commonly 120,120)
+ // after Sway has placed the real toplevel at another origin. Reconcile the
+ // fallback records with compositor-owned metadata before exposing them to
+ // callers; element bounds already use this same authoritative Sway tree.
+ for window in &mut windows {
+ let sway = window
+ .pid
+ .and_then(sway_ipc::window_for_pid)
+ .or_else(|| sway_ipc::window_for_title(&window.title))
+ .or_else(|| sway_ipc::window_for_app_id(&window.app_name));
+ if let Some(sway) = sway {
+ window.xid = sway.id;
+ window.x = sway.x;
+ window.y = sway.y;
+ window.width = sway.width;
+ window.height = sway.height;
+ window.is_on_screen = sway.visible && sway.width > 0 && sway.height > 0;
+ }
+ }
+ if is_inject_mode() {
+ for window in &mut windows {
+ if let Some(pid) = window.pid {
+ if let Some((window_x, window_y)) = inject_window_origin(pid) {
+ window.x = window_x;
+ window.y = window_y;
+ }
+ }
+ }
+ }
+ windows
+}
+
+/// Window-enumeration dispatcher: native Wayland when available, else X11.
pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> {
- if is_wayland() {
- // wlroots compositors expose zwlr_foreign_toplevel_management — use it
- // (it has no pid, so filter_pid can't apply there).
- match list_windows() {
- Ok(ws) if !ws.is_empty() => return ws,
+ if wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() {
+ // Prefer the richer wlroots protocol. The generic staging protocol is
+ // only consulted when wlroots yields no windows (including when its
+ // manager global is absent).
+ let native = match list_windows() {
+ Ok(ws) if !ws.is_empty() => Ok(enrich_native_windows(
+ ws,
+ wayland_atspi_windows(filter_pid),
+ is_inject_mode(),
+ )),
+ Ok(_) => ext_toplevel::list_windows(),
+ Err(wlr_error) => ext_toplevel::list_windows().map_err(|ext_error| {
+ anyhow::anyhow!(
+ "wlr provider failed ({wlr_error}); ext provider failed ({ext_error})"
+ )
+ }),
+ };
+ match native {
+ Ok(ws) if !ws.is_empty() => {
+ if let Some(pid) = filter_pid {
+ if let Some(filtered) = native_windows_for_pid(ws, pid) {
+ return filtered;
+ }
+ } else {
+ return ws;
+ }
+ // A compositor window without pid metadata cannot satisfy a
+ // pid-scoped request. Continue to the AT-SPI registry.
+ let ws = wayland_atspi_windows(filter_pid);
+ if !ws.is_empty() {
+ return ws;
+ }
+ }
Ok(_) => {
- // GNOME Mutter / KDE KWin don't implement foreign-toplevel, so the
- // list came back empty. Native Wayland apps have no X11 XID either,
- // so fall back to enumerating windows from the AT-SPI registry
- // (keyed by pid — the same tree get_window_state walks).
- let ws = crate::atspi::list_windows(filter_pid);
+ if let Some(ws) =
+ shell_helper::list_windows(filter_pid).filter(|ws| !ws.is_empty())
+ {
+ return ws;
+ }
+ let ws = wayland_atspi_windows(filter_pid);
if !ws.is_empty() {
return ws;
}
}
Err(e) => {
+ if let Some(ws) =
+ shell_helper::list_windows(filter_pid).filter(|ws| !ws.is_empty())
+ {
+ tracing::debug!(
+ "native Wayland protocols unavailable ({e}); using compositor helper"
+ );
+ return ws;
+ }
tracing::warn!(
- "wayland foreign-toplevel list_windows failed: {e}; trying AT-SPI registry"
+ "native Wayland list_windows failed: {e}; trying AT-SPI registry"
);
- let ws = crate::atspi::list_windows(filter_pid);
+ let ws = wayland_atspi_windows(filter_pid);
if !ws.is_empty() {
return ws;
}
@@ -1446,7 +2680,124 @@ pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> {
}
// Last resort under Wayland: an Xwayland app may still have an X11 XID.
}
- crate::x11::list_windows(filter_pid)
+ // If native enumeration and its AT-SPI fallback found nothing, X11 may still
+ // expose XWayland clients. Merge one final AT-SPI snapshot so native windows
+ // remain visible on hybrid sessions even when neither foreign-toplevel
+ // protocol is advertised (#1978). Gated on the native-Wayland opt-in.
+ //
+ // Caveats for the merged AT-SPI entries: they carry a synthetic (non-X11)
+ // xid and zero geometry (x/y/w/h = 0), like the existing wlroots AT-SPI
+ // fallback — so `bring_to_front` / `screenshot_window` / pixel translation
+ // against them error cleanly rather than acting (input on GNOME/KDE routes
+ // by pid + screen coords, not xid, so it's unaffected). Dedup is per-pid, so
+ // the rare app owning BOTH an XWayland window and a separate native-Wayland
+ // toplevel would list only the XWayland one.
+ let mut ws = crate::x11::list_windows(filter_pid);
+ if wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() {
+ let seen: std::collections::HashSet<u32> = ws.iter().filter_map(|w| w.pid).collect();
+ // A specific pid already resolved via X11 needs no AT-SPI walk (a full
+ // D-Bus enumeration of every registered app): it can only add duplicates.
+ let already_covered = filter_pid.map_or(false, |p| seen.contains(&p));
+ if !already_covered {
+ merge_atspi_windows(&mut ws, &seen, wayland_atspi_windows(filter_pid));
+ }
+ }
+ ws
+}
+
+fn merge_atspi_windows(
+ windows: &mut Vec<WindowInfo>,
+ x11_pids: &std::collections::HashSet<u32>,
+ atspi_windows: Vec<WindowInfo>,
+) {
+ for window in atspi_windows {
+ // XWayland apps appear in both lists; keep the X11 entry (real XID +
+ // geometry) and retain every native frame whose pid X11 did not expose.
+ if window.pid.is_none_or(|pid| !x11_pids.contains(&pid)) {
+ windows.push(window);
+ }
+ }
+}
+
+fn enrich_native_windows(
+ mut native: Vec<WindowInfo>,
+ atspi: Vec<WindowInfo>,
+ adopt_atspi_ids: bool,
+) -> Vec<WindowInfo> {
+ let mut claimed = std::collections::HashSet::new();
+ for window in &mut native {
+ if window.pid.is_some() {
+ continue;
+ }
+ let native_title = undecorated_native_title(window);
+ let title_match = atspi.iter().enumerate().find_map(|(index, candidate)| {
+ (!claimed.contains(&index)
+ && !native_title.is_empty()
+ && candidate.title == native_title)
+ .then_some(index)
+ });
+ let app_match = title_match.or_else(|| {
+ let matches = atspi
+ .iter()
+ .enumerate()
+ .filter(|(index, candidate)| {
+ !claimed.contains(index)
+ && !window.app_name.is_empty()
+ && candidate.app_name == window.app_name
+ })
+ .map(|(index, _)| index)
+ .collect::<Vec<_>>();
+ (matches.len() == 1).then(|| matches[0])
+ });
+ let Some(index) = app_match else { continue };
+ claimed.insert(index);
+ let candidate = &atspi[index];
+ window.pid = candidate.pid;
+ if adopt_atspi_ids {
+ let toplevel = Toplevel {
+ title: undecorated_native_title(window).to_owned(),
+ app_id: window.app_name.clone(),
+ closed: false,
+ };
+ window.xid = candidate.xid;
+ remember_identity(window.xid, &toplevel);
+ }
+ if window.width == 0 || window.height == 0 {
+ window.x = candidate.x;
+ window.y = candidate.y;
+ window.width = candidate.width;
+ window.height = candidate.height;
+ }
+ if adopt_atspi_ids {
+ if let Some(pid) = candidate.pid {
+ if let Some((window_x, window_y)) = inject_window_origin(pid) {
+ window.x = window_x;
+ window.y = window_y;
+ }
+ }
+ }
+ window.is_on_screen = candidate.is_on_screen;
+ }
+ native
+}
+
+fn undecorated_native_title(window: &WindowInfo) -> &str {
+ if window.app_name.is_empty() {
+ return &window.title;
+ }
+ let suffix = format!(" [{}]", window.app_name);
+ window.title.strip_suffix(&suffix).unwrap_or(&window.title)
+}
+
+/// Return native records only when they contain a real match for a pid-scoped
+/// request. Ext records whose AT-SPI merge left pid unknown must not suppress
+/// the later AT-SPI and X11 fallback providers.
+fn native_windows_for_pid(windows: Vec<WindowInfo>, pid: u32) -> Option<Vec<WindowInfo>> {
+ let matching: Vec<_> = windows
+ .into_iter()
+ .filter(|window| window.pid == Some(pid))
+ .collect();
+ (!matching.is_empty()).then_some(matching)
}
/// Snapshot of which wlroots manager globals the running compositor advertises.
@@ -1542,3 +2893,211 @@ impl Dispatch<wl_registry::WlRegistry, ()> for ExtProbeState {
// compatibility with earlier slice constants.
#[allow(dead_code)]
const _BTN_LEFT_ALIAS: u32 = BTN_LEFT;
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn window(xid: u64, pid: Option<u32>, title: &str) -> WindowInfo {
+ WindowInfo {
+ xid,
+ pid,
+ app_name: String::new(),
+ title: title.to_owned(),
+ is_on_screen: true,
+ z_index: None,
+ x: 0,
+ y: 0,
+ width: 0,
+ height: 0,
+ }
+ }
+
+ #[test]
+ fn atspi_merge_keeps_x11_geometry_owner_and_native_only_frames() {
+ let mut windows = vec![window(10, Some(100), "XWayland")];
+ let x11_pids = std::collections::HashSet::from([100]);
+ merge_atspi_windows(
+ &mut windows,
+ &x11_pids,
+ vec![
+ window(100 << 16, Some(100), "XWayland duplicate"),
+ window(200 << 16, Some(200), "Native Wayland"),
+ window(1, None, "Unknown native frame"),
+ ],
+ );
+ assert_eq!(windows.len(), 3);
+ assert_eq!(windows[0].xid, 10);
+ assert_eq!(windows[1].pid, Some(200));
+ assert_eq!(windows[2].pid, None);
+ }
+
+ #[test]
+ fn zero_geometry_does_not_replace_a_real_observed_origin() {
+ let pid = u32::MAX - 17;
+ let mut observed = window(1, Some(pid), "Observed");
+ observed.x = 120;
+ observed.y = 80;
+ remember_observed_window_origins(&[observed]);
+ assert_eq!(observed_window_origin(pid), Some((120, 80)));
+
+ remember_observed_window_origins(&[window(2, Some(pid), "Unknown")]);
+ assert_eq!(observed_window_origin(pid), Some((120, 80)));
+ }
+
+ #[test]
+ fn native_enrichment_matches_plain_atspi_title() {
+ let mut native = window(42, None, "CUA Fixture [cua-fixture]");
+ native.app_name = "cua-fixture".into();
+ let mut accessible = window(123 << 16, Some(123), "CUA Fixture");
+ accessible.x = 20;
+ accessible.y = 30;
+ accessible.width = 800;
+ accessible.height = 600;
+
+ let enriched = enrich_native_windows(vec![native], vec![accessible], false);
+
+ assert_eq!(enriched[0].xid, 42);
+ assert_eq!(enriched[0].pid, Some(123));
+ assert_eq!((enriched[0].x, enriched[0].y), (20, 30));
+ assert_eq!((enriched[0].width, enriched[0].height), (800, 600));
+ }
+
+ #[test]
+ fn unmatched_ext_windows_do_not_satisfy_pid_filter() {
+ let windows = vec![window(0xF000_0000, None, "Protocol-only")];
+ assert!(native_windows_for_pid(windows, 4242).is_none());
+ }
+
+ #[test]
+ fn native_title_match_recovers_pid_without_replacing_native_id() {
+ let native = vec![window(77, None, "CuaTestHarness")];
+ let mut accessible = window(123 << 16, Some(123), "CuaTestHarness");
+ accessible.x = 20;
+ accessible.y = 30;
+ accessible.width = 800;
+ accessible.height = 600;
+ let enriched = enrich_native_windows(native, vec![accessible], false);
+ assert_eq!(enriched[0].xid, 77);
+ assert_eq!(enriched[0].pid, Some(123));
+ assert_eq!(
+ (enriched[0].x, enriched[0].y, enriched[0].width, enriched[0].height),
+ (20, 30, 800, 600)
+ );
+ }
+
+ #[test]
+ fn nested_enrichment_adopts_stable_atspi_id() {
+ let native = vec![window(77, None, "CuaTestHarness")];
+ let accessible = window(123 << 16, Some(123), "CuaTestHarness");
+ let enriched = enrich_native_windows(native, vec![accessible], true);
+ assert_eq!(enriched[0].xid, 123 << 16);
+ assert_eq!(enriched[0].pid, Some(123));
+ assert_eq!(identity_for(enriched[0].xid).unwrap().title, "CuaTestHarness");
+ }
+
+ #[test]
+ fn sway_window_capture_is_cropped_to_compositor_geometry() {
+ let source = image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel(
+ 8,
+ 6,
+ image::Rgba([20, 40, 60, 255]),
+ ));
+ let mut encoded = std::io::Cursor::new(Vec::new());
+ source
+ .write_to(&mut encoded, image::ImageFormat::Png)
+ .expect("encode fixture PNG");
+ let cropped = crop_png_to_rect(encoded.get_ref(), 2, 1, 3, 4, "fixture")
+ .expect("crop fixture PNG");
+ let decoded = image::load_from_memory(&cropped).expect("decode cropped PNG");
+ assert_eq!((decoded.width(), decoded.height()), (3, 4));
+ }
+
+ #[test]
+ fn injectable_text_accepts_printable_ascii_newline_and_tab() {
+ validate_injectable_text("Hello, World! 123 @#$%\t\n").expect("printable ASCII is typable");
+ // The full printable ASCII span the compositor chartab covers.
+ let printable: String = (0x20u8..=0x7e).map(|b| b as char).collect();
+ validate_injectable_text(&printable).expect("every printable ASCII byte is typable");
+ }
+
+ #[test]
+ fn injectable_text_rejects_unicode_and_other_controls() {
+ for bad in ["café", "emoji 😀", "bell\u{07}", "null\u{00}", "delete\u{7f}", "cr\r"] {
+ assert!(
+ validate_injectable_text(bad).is_err(),
+ "{bad:?} must be rejected before it reaches the compositor"
+ );
+ }
+ }
+
+ #[test]
+ fn injectable_key_accepts_whitelist_case_insensitively() {
+ for good in ["enter", "Enter", "RETURN", "tab", "Escape", "esc", "space", "up", "Left", "f1", "F12"] {
+ validate_injectable_key(good).unwrap_or_else(|e| panic!("{good:?} should pass: {e}"));
+ }
+ }
+
+ #[test]
+ fn injectable_key_rejects_unsupported_names() {
+ for bad in ["f13", "ctrl", "a", "delete", "home", "pageup", ""] {
+ assert!(
+ validate_injectable_key(bad).is_err(),
+ "{bad:?} is not in the compositor whitelist"
+ );
+ }
+ }
+
+ #[test]
+ fn injectable_hotkey_normalizes_supported_chords() {
+ let keys = vec!["control".to_owned(), "SHIFT".to_owned(), "7".to_owned()];
+ assert_eq!(
+ validate_injectable_hotkey(&keys).expect("supported chord"),
+ ("ctrl,shift".to_owned(), "7".to_owned())
+ );
+ assert!(validate_injectable_hotkey(&["7".to_owned()]).is_err());
+ assert!(validate_injectable_hotkey(&["hyper".to_owned(), "k".to_owned()]).is_err());
+ }
+
+ #[test]
+ fn wtype_hotkey_keeps_primer_chord_and_releases_in_order() {
+ let args = wtype_hotkey_args(&["ctrl".into(), "shift".into()], "7");
+ assert_eq!(
+ args,
+ [
+ "-k", "Shift_L", "-s", "30", "-M", "ctrl", "-M", "shift", "-s", "20",
+ "-k", "7", "-s", "20", "-m", "shift", "-m", "ctrl",
+ ]
+ );
+ }
+
+ #[test]
+ fn hello_reply_parses_exact_banner_and_rejects_mismatch() {
+ parse_inject_hello("cua-inject v1\n").expect("verbatim banner is accepted");
+ parse_inject_hello(" cua-inject v1 ").expect("surrounding whitespace is tolerated");
+ assert!(parse_inject_hello("cua-inject v2").is_err());
+ assert!(parse_inject_hello("err unsupported-version").is_err());
+ assert!(parse_inject_hello("garbage").is_err());
+ }
+
+ #[test]
+ fn command_reply_parses_ok_and_surfaces_error_reason() {
+ parse_inject_reply("ok\n").expect("ok is success");
+ parse_inject_reply("ok").expect("ok without newline is success");
+ let err = parse_inject_reply("err ambiguous-app-id\n").unwrap_err();
+ assert!(err.to_string().contains("ambiguous-app-id"));
+ assert!(parse_inject_reply("err").is_err());
+ assert!(parse_inject_reply("maybe").is_err());
+ }
+
+ #[test]
+ fn geometry_reply_is_strict_and_signed() {
+ assert_eq!(
+ parse_inject_geometry("geometry -4 23 10 20\n").unwrap(),
+ ((-4, 23), (10, 20))
+ );
+ assert!(parse_inject_geometry("geometry 1").is_err());
+ assert!(parse_inject_geometry("err target-not-found").is_err());
+ assert!(parse_inject_geometry("ok").is_err());
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
index 4b9df956a9..9ee2718e12 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
@@ -147,7 +147,7 @@ fn handle_press(
// Keep the (out_w, out_h) but drop the queue + state at end of scope; the
// vptr itself remains alive (Wayland objects survive their original queue
// as long as the Connection is alive).
- let mut sess = open_vptr_session(Some(window_id as u32))?;
+ let mut sess = open_vptr_session(Some(window_id))?;
let (w, h) = (sess.output_w, sess.output_h);
let px = x.clamp(0, w as i32 - 1) as u32;
let py = y.clamp(0, h as i32 - 1) as u32;
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs
index c1f1afbc0a..887b4f8dac 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs
@@ -27,6 +27,15 @@ use ashpd::desktop::screenshot::Screenshot;
/// on the first call per session unless they've pre-approved the
/// requesting binary in their DE's privacy settings.
pub fn screenshot_via_portal() -> anyhow::Result<Vec<u8>> {
+ if tokio::runtime::Handle::try_current().is_ok() {
+ return std::thread::spawn(screenshot_via_portal_blocking)
+ .join()
+ .map_err(|_| anyhow::anyhow!("xdg-desktop-portal Screenshot worker panicked"))?;
+ }
+ screenshot_via_portal_blocking()
+}
+
+fn screenshot_via_portal_blocking() -> anyhow::Result<Vec<u8>> {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
index 076be30a47..4ec0524135 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs
@@ -23,11 +23,22 @@
use std::process::Command;
use std::time::Duration;
+use crate::x11::WindowInfo;
+
const DEST: &str = "org.cua.WinRects";
const PATH: &str = "/org/cua/WinRects";
const IFACE: &str = "org.cua.WinRects";
+pub fn available() -> bool {
+ static AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+ *AVAILABLE.get_or_init(|| gdbus_call("GetRects", &[]).is_some())
+}
+
fn gdbus_call(method: &str, args: &[String]) -> Option<String> {
+ gdbus_call_with_timeout(method, args, Duration::from_millis(800))
+}
+
+fn gdbus_call_with_timeout(method: &str, args: &[String], timeout: Duration) -> Option<String> {
let mut cmd = Command::new("gdbus");
cmd.arg("call")
.arg("--session")
@@ -47,34 +58,85 @@ fn gdbus_call(method: &str, args: &[String]) -> Option<String> {
.stderr(std::process::Stdio::null())
.spawn()
.ok()?;
- let out = wait_timeout(child, Duration::from_millis(800))?;
+ let out = wait_timeout(child, timeout)?;
if !out.status.success() {
return None;
}
Some(String::from_utf8_lossy(&out.stdout).into_owned())
}
+/// Capture the GNOME stage through the compositor helper.
+///
+/// Mutter does not expose wlroots screencopy protocols, and its one-shot
+/// Screenshot portal may reject an unregistered command-line process. The
+/// opt-in helper already runs inside Shell for geometry and activation, so it
+/// can use Shell's screenshot API without confusing a stable Wayland window id
+/// for an X11 drawable.
+pub fn screenshot_display() -> Option<Vec<u8>> {
+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
+
+ let raw = gdbus_call_with_timeout("Capture", &[], Duration::from_secs(5))?;
+ let start = raw.find('\'')? + 1;
+ let end = raw.rfind('\'')?;
+ if end <= start {
+ return None;
+ }
+ B64.decode(&raw[start..end]).ok()
+}
+
/// `Child::wait` with a deadline (no extra crates). Kills + reaps on timeout.
fn wait_timeout(mut child: std::process::Child, dur: Duration) -> Option<std::process::Output> {
+ use std::io::Read;
+
+ // Drain stdout while the child is running. Capture() returns a base64 PNG
+ // that readily exceeds a pipe's ~64 KiB capacity; waiting for exit before
+ // reading deadlocks the child on a full pipe and turns a healthy Shell
+ // response into a false timeout.
+ let stdout = child.stdout.take()?;
+ let reader = std::thread::spawn(move || {
+ let mut stdout = stdout;
+ let mut bytes = Vec::new();
+ stdout.read_to_end(&mut bytes).ok()?;
+ Some(bytes)
+ });
let deadline = std::time::Instant::now() + dur;
- loop {
+ let status = loop {
match child.try_wait() {
- Ok(Some(_)) => return child.wait_with_output().ok(),
+ Ok(Some(status)) => break status,
Ok(None) => {
if std::time::Instant::now() >= deadline {
let _ = child.kill();
- let _ = child.wait();
+ let status = child.wait().ok()?;
+ let _ = reader.join();
+ if !status.success() {
+ return None;
+ }
return None;
}
std::thread::sleep(Duration::from_millis(15));
}
- Err(_) => return None,
+ Err(_) => {
+ let _ = child.kill();
+ let _ = child.wait();
+ let _ = reader.join();
+ return None;
+ }
}
- }
+ };
+ let stdout = reader.join().ok().flatten()?;
+ Some(std::process::Output {
+ status,
+ stdout,
+ stderr: Vec::new(),
+ })
}
-/// Screen origin (x, y) of the window backing `pid`, from the extension's
-/// `GetRects`. `None` when the extension is unavailable or no window matches.
+/// Screen origin of the Wayland surface buffer backing `pid`.
+///
+/// GTK's AT-SPI `CoordType::Window` includes client-side shadow extents, while
+/// Mutter's frame rectangle excludes them. The buffer origin preserves those
+/// extents so accessibility frames line up with pixels. Older helpers omit the
+/// buffer fields and fall back to the frame origin.
pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> {
let raw = gdbus_call("GetRects", &[])?;
// gdbus prints a GVariant tuple like `('[{"pid":..,"x":..}]',)`. Pull the
@@ -86,14 +148,124 @@ pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> {
let arr: Vec<serde_json::Value> = serde_json::from_str(json).ok()?;
for w in &arr {
if w.get("pid").and_then(|p| p.as_u64()) == Some(pid as u64) {
- let x = w.get("x")?.as_i64()? as i32;
- let y = w.get("y")?.as_i64()? as i32;
+ let x = w
+ .get("buffer_x")
+ .and_then(serde_json::Value::as_i64)
+ .or_else(|| w.get("x").and_then(serde_json::Value::as_i64))?
+ as i32;
+ let y = w
+ .get("buffer_y")
+ .and_then(serde_json::Value::as_i64)
+ .or_else(|| w.get("y").and_then(serde_json::Value::as_i64))?
+ as i32;
return Some((x, y));
}
}
None
}
+/// Enumerate GNOME Shell toplevels when the compositor helper is available.
+///
+/// AT-SPI remains the source of accessibility elements, but it is a poor
+/// source of truth for desktop window discovery: one unresponsive application
+/// can exhaust the bounded registry walk and hide every healthy toplevel. The
+/// shell already owns the authoritative stacking list, geometry, visibility,
+/// title, and PID, so use that metadata directly for `list_windows`.
+pub fn list_windows(filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> {
+ let raw = gdbus_call("GetRects", &[])?;
+ parse_windows(&raw, filter_pid)
+}
+
+/// Ask GNOME Shell to focus and raise one stable-sequence window.
+///
+/// Returns `false` when the helper is absent, the id is unknown, or Shell did
+/// not confirm focus. Callers must not inject global libei input unless this
+/// returns true: portal input is focus-bound and otherwise targets whichever
+/// application the user happened to be using.
+pub fn activate_window(window_id: u64) -> bool {
+ let Ok(window_id) = u32::try_from(window_id) else {
+ return false;
+ };
+ let accepted = gdbus_call("Activate", &[window_id.to_string()])
+ .is_some_and(|output| output.trim_start().starts_with("(true,"));
+ if !accepted {
+ return false;
+ }
+ std::thread::sleep(Duration::from_millis(60));
+ window_is_focused(window_id)
+}
+
+fn window_is_focused(window_id: u32) -> bool {
+ let Some(raw) = gdbus_call("GetRects", &[]) else {
+ return false;
+ };
+ let (Some(start), Some(end)) = (raw.find('['), raw.rfind(']')) else {
+ return false;
+ };
+ serde_json::from_str::<Vec<serde_json::Value>>(&raw[start..=end])
+ .ok()
+ .and_then(|windows| {
+ windows.into_iter().find(|window| {
+ window.get("id").and_then(serde_json::Value::as_u64) == Some(window_id as u64)
+ })
+ })
+ .and_then(|window| window.get("focused").and_then(serde_json::Value::as_bool))
+ .unwrap_or(false)
+}
+
+fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> {
+ let start = raw.find('[')?;
+ let end = raw.rfind(']')?;
+ let windows: Vec<serde_json::Value> = serde_json::from_str(&raw[start..=end]).ok()?;
+
+ Some(
+ windows
+ .into_iter()
+ .filter_map(|window| {
+ let pid = u32::try_from(window.get("pid")?.as_u64()?).ok()?;
+ if filter_pid.is_some_and(|wanted| wanted != pid) {
+ return None;
+ }
+ let id = window.get("id")?.as_u64()?.max(1);
+ let x = i32::try_from(window.get("x")?.as_i64()?).ok()?;
+ let y = i32::try_from(window.get("y")?.as_i64()?).ok()?;
+ let width = u32::try_from(window.get("w")?.as_u64()?).ok()?;
+ let height = u32::try_from(window.get("h")?.as_u64()?).ok()?;
+ let visible = window
+ .get("visible")
+ .and_then(serde_json::Value::as_bool)
+ .unwrap_or(width > 0 && height > 0);
+ let minimized = window
+ .get("minimized")
+ .and_then(serde_json::Value::as_bool)
+ .unwrap_or(false);
+ let title = window
+ .get("title")
+ .and_then(serde_json::Value::as_str)
+ .unwrap_or_default()
+ .to_owned();
+ let z_index = window
+ .get("stacking")
+ .and_then(serde_json::Value::as_u64)
+ .and_then(|value| usize::try_from(value).ok());
+
+ Some(WindowInfo {
+ xid: id,
+ pid: Some(pid),
+ app_name: title.clone(),
+ title,
+ is_on_screen: visible && !minimized && width > 0 && height > 0,
+ z_index,
+ x,
+ y,
+ width,
+ height,
+ })
+ })
+ .collect(),
+ )
+}
+
/// Glide the agent cursor to screen `(x, y)`.
pub fn move_cursor(x: i32, y: i32) {
let _ = gdbus_call("MoveCursor", &[x.to_string(), y.to_string()]);
@@ -108,3 +280,30 @@ pub fn click_pulse(x: i32, y: i32) {
pub fn hide_cursor() {
let _ = gdbus_call("HideCursor", &[]);
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parses_and_filters_shell_windows() {
+ let raw = r#"('[{"id":46,"pid":6079,"title":"Sentinel's window","x":66,"y":32,"w":958,"h":736,"focused":true,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#;
+ let windows = parse_windows(raw, Some(6079)).expect("valid helper response");
+ assert_eq!(windows.len(), 1);
+ assert_eq!(windows[0].xid, 46);
+ assert_eq!(windows[0].pid, Some(6079));
+ assert_eq!(windows[0].title, "Sentinel's window");
+ assert_eq!((windows[0].x, windows[0].y), (66, 32));
+ assert_eq!((windows[0].width, windows[0].height), (958, 736));
+ assert!(windows[0].is_on_screen);
+ assert_eq!(windows[0].z_index, Some(2));
+ }
+
+ #[test]
+ fn marks_minimized_shell_windows_off_screen() {
+ let raw = r#"('[{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#;
+ let windows = parse_windows(raw, None).expect("valid helper response");
+ assert_eq!(windows.len(), 1);
+ assert!(!windows[0].is_on_screen);
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs
new file mode 100644
index 0000000000..4ee52148e5
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs
@@ -0,0 +1,264 @@
+//! Best-effort Sway/i3-compatible compositor metadata.
+//!
+//! The wlroots foreign-toplevel protocol exposes titles and app ids, but not
+//! process ids or geometry. Sway's IPC tree supplies those missing fields and
+//! a compositor-stable container id. Other compositors simply return no data.
+
+use std::process::{Command, Stdio};
+
+use serde::Deserialize;
+
+#[derive(Clone, Debug, Default, Deserialize)]
+struct Rect {
+ x: i32,
+ y: i32,
+ width: i32,
+ height: i32,
+}
+
+#[derive(Clone, Debug, Default, Deserialize)]
+struct Node {
+ id: u64,
+ #[serde(default)]
+ name: String,
+ #[serde(default)]
+ app_id: String,
+ pid: Option<u32>,
+ #[serde(default)]
+ rect: Rect,
+ #[serde(default)]
+ window_rect: Rect,
+ #[serde(default)]
+ deco_rect: Rect,
+ #[serde(default)]
+ focused: bool,
+ #[serde(default = "default_visible")]
+ visible: bool,
+ #[serde(default)]
+ fullscreen_mode: i32,
+ #[serde(default)]
+ nodes: Vec<Node>,
+ #[serde(default)]
+ floating_nodes: Vec<Node>,
+}
+
+fn default_visible() -> bool {
+ true
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Window {
+ pub id: u64,
+ pub pid: u32,
+ pub title: String,
+ pub app_id: String,
+ pub x: i32,
+ pub y: i32,
+ pub width: u32,
+ pub height: u32,
+ pub content_x: i32,
+ pub content_y: i32,
+ pub focused: bool,
+ pub visible: bool,
+ pub fullscreen: bool,
+}
+
+fn collect(node: &Node, windows: &mut Vec<Window>) {
+ if let Some(pid) = node.pid {
+ if !node.name.is_empty() || !node.app_id.is_empty() {
+ // Sway normally reports the client surface origin in
+ // `window_rect`. Some server-decorated Wayland clients instead
+ // leave that origin at zero and expose the title-bar inset only
+ // through `deco_rect`; use it as the content origin in that shape.
+ let inferred_top_inset = if node.window_rect.height > 0 {
+ node.rect
+ .height
+ .saturating_sub(node.window_rect.height)
+ .max(0)
+ } else {
+ 0
+ };
+ let content_y = if node.window_rect.y != 0 {
+ node.window_rect.y
+ } else {
+ node.deco_rect
+ .y
+ .saturating_add(node.deco_rect.height.max(0))
+ .max(inferred_top_inset)
+ };
+ windows.push(Window {
+ id: node.id,
+ pid,
+ title: node.name.clone(),
+ app_id: node.app_id.clone(),
+ x: node.rect.x,
+ y: node.rect.y,
+ width: node.rect.width.max(0) as u32,
+ height: node.rect.height.max(0) as u32,
+ content_x: node.window_rect.x,
+ content_y,
+ focused: node.focused,
+ visible: node.visible,
+ fullscreen: node.fullscreen_mode != 0,
+ });
+ }
+ }
+ for child in node.nodes.iter().chain(&node.floating_nodes) {
+ collect(child, windows);
+ }
+}
+
+fn parse_tree(bytes: &[u8]) -> Option<Vec<Window>> {
+ let root: Node = serde_json::from_slice(bytes).ok()?;
+ let mut windows = Vec::new();
+ collect(&root, &mut windows);
+ Some(windows)
+}
+
+pub fn list_windows() -> Option<Vec<Window>> {
+ if std::env::var_os("SWAYSOCK").is_none() {
+ return None;
+ }
+ let output = Command::new("swaymsg")
+ .args(["-r", "-t", "get_tree"])
+ .stdin(Stdio::null())
+ .stderr(Stdio::null())
+ .output()
+ .ok()?;
+ if !output.status.success() {
+ return None;
+ }
+ parse_tree(&output.stdout)
+}
+
+pub fn window_for_id(id: u64) -> Option<Window> {
+ list_windows()?.into_iter().find(|window| window.id == id)
+}
+
+pub fn window_for_pid(pid: u32) -> Option<Window> {
+ list_windows()?
+ .into_iter()
+ .filter(|window| window.pid == pid && window.width > 0 && window.height > 0)
+ .max_by_key(|window| {
+ (
+ window.focused,
+ window.visible,
+ u64::from(window.width) * u64::from(window.height),
+ )
+ })
+}
+
+pub fn window_for_title(title: &str) -> Option<Window> {
+ list_windows()?
+ .into_iter()
+ .filter(|window| {
+ window.width > 0
+ && window.height > 0
+ && (window.title == title
+ || (!window.title.is_empty() && title.starts_with(&window.title))
+ || (!title.is_empty() && window.title.starts_with(title)))
+ })
+ .max_by_key(|window| (window.focused, window.visible))
+}
+
+pub fn window_for_app_id(app_id: &str) -> Option<Window> {
+ if app_id.is_empty() {
+ return None;
+ }
+ list_windows()?
+ .into_iter()
+ .filter(|window| window.width > 0 && window.height > 0 && window.app_id == app_id)
+ .max_by_key(|window| (window.focused, window.visible))
+}
+
+pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> {
+ window_for_pid(pid).map(|window| (window.x, window.y))
+}
+
+/// Offset of the application content inside Sway's captured toplevel.
+/// Server-side decorations are part of `rect`/window screenshots but not of
+/// WebKitGTK's AT-SPI `CoordType::Window` descendants.
+pub fn window_content_offset_for_pid(pid: u32) -> Option<(i32, i32)> {
+ window_for_pid(pid).map(|window| (window.content_x, window.content_y))
+}
+
+pub fn window_origin_for_title(title: &str) -> Option<(i32, i32)> {
+ let window = window_for_title(title)?;
+ Some((window.x, window.y))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parses_nested_and_floating_windows() {
+ let tree = br#"{
+ "id": 1,
+ "nodes": [{
+ "id": 2,
+ "nodes": [{
+ "id": 10,
+ "name": "Editor",
+ "app_id": "org.example.Editor",
+ "pid": 123,
+ "rect": {"x": 20, "y": 30, "width": 800, "height": 600},
+ "window_rect": {"x": 0, "y": 47, "width": 800, "height": 553},
+ "focused": true,
+ "visible": true,
+ "fullscreen_mode": 1
+ }],
+ "floating_nodes": [{
+ "id": 11,
+ "name": "Dialog",
+ "pid": 124,
+ "rect": {"x": 100, "y": 120, "width": 300, "height": 200}
+ }]
+ }]
+ }"#;
+ let windows = parse_tree(tree).expect("parse Sway tree");
+ assert_eq!(windows.len(), 2);
+ assert_eq!(windows[0].pid, 123);
+ assert_eq!((windows[0].x, windows[0].y), (20, 30));
+ assert_eq!((windows[0].content_x, windows[0].content_y), (0, 47));
+ assert!(windows[0].focused);
+ assert!(windows[0].fullscreen);
+ assert_eq!(windows[1].title, "Dialog");
+ }
+
+ #[test]
+ fn decoration_height_fills_zero_window_content_origin() {
+ let tree = br#"{
+ "id": 1,
+ "nodes": [{
+ "id": 10,
+ "name": "Tauri",
+ "app_id": "cua-test-harness",
+ "pid": 123,
+ "rect": {"x": 0, "y": 0, "width": 940, "height": 780},
+ "window_rect": {"x": 0, "y": 0, "width": 940, "height": 733},
+ "deco_rect": {"x": 0, "y": 0, "width": 940, "height": 47}
+ }]
+ }"#;
+ let windows = parse_tree(tree).expect("parse decorated Sway tree");
+ assert_eq!((windows[0].content_x, windows[0].content_y), (0, 47));
+ }
+
+ #[test]
+ fn outer_client_height_delta_fills_missing_decoration_metadata() {
+ let tree = br#"{
+ "id": 1,
+ "nodes": [{
+ "id": 10,
+ "name": "Tauri",
+ "app_id": "cua-test-harness",
+ "pid": 123,
+ "rect": {"x": 0, "y": 0, "width": 940, "height": 780},
+ "window_rect": {"x": 0, "y": 0, "width": 940, "height": 733},
+ "deco_rect": {"x": 0, "y": 0, "width": 0, "height": 0}
+ }]
+ }"#;
+ let windows = parse_tree(tree).expect("parse Sway tree with implicit decoration");
+ assert_eq!((windows[0].content_x, windows[0].content_y), (0, 47));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/virtual_keyboard.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/virtual_keyboard.rs
new file mode 100644
index 0000000000..50e8b7ee0f
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/virtual_keyboard.rs
@@ -0,0 +1,246 @@
+//! Stable physical-key injection for wlroots virtual-keyboard compositors.
+
+use std::ffi::CString;
+use std::fs::File;
+use std::io::{Seek, SeekFrom, Write};
+use std::os::fd::{AsFd, FromRawFd};
+
+use wayland_client::protocol::{wl_registry, wl_seat::WlSeat};
+use wayland_client::{Connection, Dispatch, Proxy, QueueHandle};
+
+use self::protocol::zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1;
+use self::protocol::zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1;
+
+mod protocol {
+ use wayland_client;
+ use wayland_client::protocol::*;
+
+ pub mod __interfaces {
+ use wayland_client::protocol::__interfaces::*;
+ wayland_scanner::generate_interfaces!("protocol/virtual-keyboard-unstable-v1.xml");
+ }
+ use self::__interfaces::*;
+ wayland_scanner::generate_client_code!("protocol/virtual-keyboard-unstable-v1.xml");
+}
+
+const XKB_KEYMAP: &str = r#"xkb_keymap {
+ xkb_keycodes { include "evdev+aliases(qwerty)" };
+ xkb_types { include "complete" };
+ xkb_compatibility { include "complete" };
+ xkb_symbols { include "pc+us+inet(evdev)" };
+ xkb_geometry { include "pc(pc105)" };
+};
+"#;
+
+const KEY_RELEASED: u32 = 0;
+const KEY_PRESSED: u32 = 1;
+
+#[derive(Default)]
+struct State {
+ seat: Option<WlSeat>,
+ manager: Option<ZwpVirtualKeyboardManagerV1>,
+}
+
+impl Dispatch<wl_registry::WlRegistry, ()> for State {
+ fn event(
+ state: &mut Self,
+ registry: &wl_registry::WlRegistry,
+ event: wl_registry::Event,
+ _: &(),
+ _: &Connection,
+ qh: &QueueHandle<Self>,
+ ) {
+ if let wl_registry::Event::Global {
+ name,
+ interface,
+ version,
+ } = event
+ {
+ if interface == WlSeat::interface().name {
+ state.seat = Some(registry.bind(name, version.min(7), qh, ()));
+ } else if interface == ZwpVirtualKeyboardManagerV1::interface().name {
+ state.manager = Some(registry.bind(name, version.min(1), qh, ()));
+ }
+ }
+ }
+}
+
+impl Dispatch<WlSeat, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &WlSeat,
+ _: <WlSeat as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+impl Dispatch<ZwpVirtualKeyboardManagerV1, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &ZwpVirtualKeyboardManagerV1,
+ _: <ZwpVirtualKeyboardManagerV1 as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+impl Dispatch<ZwpVirtualKeyboardV1, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &ZwpVirtualKeyboardV1,
+ _: <ZwpVirtualKeyboardV1 as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+pub(super) fn hotkey(modifiers: &[String], key: &str) -> anyhow::Result<()> {
+ let transitions = hotkey_transitions(modifiers, key)?;
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?;
+ queue.roundtrip(&mut state)?;
+
+ let seat = state
+ .seat
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor exposes no wl_seat"))?;
+ let manager = state.manager.clone().ok_or_else(|| {
+ anyhow::anyhow!("compositor exposes no zwp_virtual_keyboard_manager_v1")
+ })?;
+ let keyboard = manager.create_virtual_keyboard(&seat, &qh, ());
+ let keymap = keymap_file()?;
+ keyboard.keymap(1, keymap.as_fd(), XKB_KEYMAP.len() as u32 + 1);
+ queue.roundtrip(&mut state)?;
+
+ // A harmless Shift tap absorbs the first-event loss seen on fresh
+ // headless wlroots seats without changing the requested chord.
+ send_key(&keyboard, &mut queue, &mut state, 42, KEY_PRESSED)?;
+ send_key(&keyboard, &mut queue, &mut state, 42, KEY_RELEASED)?;
+ for transition in transitions {
+ send_key(
+ &keyboard,
+ &mut queue,
+ &mut state,
+ transition.keycode,
+ if transition.pressed {
+ KEY_PRESSED
+ } else {
+ KEY_RELEASED
+ },
+ )?;
+ keyboard.modifiers(transition.modifier_mask, 0, 0, 0);
+ queue.roundtrip(&mut state)?;
+ }
+ keyboard.destroy();
+ queue.roundtrip(&mut state)?;
+ Ok(())
+}
+
+fn send_key(
+ keyboard: &ZwpVirtualKeyboardV1,
+ queue: &mut wayland_client::EventQueue<State>,
+ state: &mut State,
+ keycode: u32,
+ key_state: u32,
+) -> anyhow::Result<()> {
+ keyboard.key(super::event_time_ms(), keycode, key_state);
+ queue.roundtrip(state)?;
+ std::thread::sleep(std::time::Duration::from_millis(4));
+ Ok(())
+}
+
+fn keymap_file() -> anyhow::Result<File> {
+ let name = CString::new("cua-driver-keymap")?;
+ let fd = unsafe { libc::memfd_create(name.as_ptr(), libc::MFD_CLOEXEC) };
+ if fd < 0 {
+ return Err(std::io::Error::last_os_error().into());
+ }
+ let mut file = unsafe { File::from_raw_fd(fd) };
+ file.write_all(XKB_KEYMAP.as_bytes())?;
+ file.write_all(&[0])?;
+ file.seek(SeekFrom::Start(0))?;
+ Ok(file)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct KeyTransition {
+ keycode: u32,
+ pressed: bool,
+ modifier_mask: u32,
+}
+
+fn hotkey_transitions(modifiers: &[String], key: &str) -> anyhow::Result<Vec<KeyTransition>> {
+ let mut modifier_mask = 0;
+ let mut modifier_keys = Vec::with_capacity(modifiers.len());
+ for modifier in modifiers {
+ let (keycode, mask) = match modifier.as_str() {
+ "ctrl" => (29, 4),
+ "shift" => (42, 1),
+ "alt" => (56, 8),
+ "logo" => (125, 64),
+ other => anyhow::bail!("unsupported Wayland modifier '{other}'"),
+ };
+ modifier_keys.push((keycode, mask));
+ }
+ let keycode = super::key_to_evdev(key)
+ .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}'"))?;
+ let mut transitions = Vec::with_capacity(modifier_keys.len() * 2 + 2);
+ for (modifier_key, mask) in &modifier_keys {
+ modifier_mask |= mask;
+ transitions.push(KeyTransition {
+ keycode: *modifier_key,
+ pressed: true,
+ modifier_mask,
+ });
+ }
+ transitions.push(KeyTransition {
+ keycode,
+ pressed: true,
+ modifier_mask,
+ });
+ transitions.push(KeyTransition {
+ keycode,
+ pressed: false,
+ modifier_mask,
+ });
+ for (modifier_key, mask) in modifier_keys.iter().rev() {
+ modifier_mask &= !mask;
+ transitions.push(KeyTransition {
+ keycode: *modifier_key,
+ pressed: false,
+ modifier_mask,
+ });
+ }
+ Ok(transitions)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn chord_uses_physical_digit7_and_balanced_modifiers() {
+ assert_eq!(
+ hotkey_transitions(&["ctrl".into(), "shift".into()], "7").unwrap(),
+ [
+ KeyTransition { keycode: 29, pressed: true, modifier_mask: 4 },
+ KeyTransition { keycode: 42, pressed: true, modifier_mask: 5 },
+ KeyTransition { keycode: 8, pressed: true, modifier_mask: 5 },
+ KeyTransition { keycode: 8, pressed: false, modifier_mask: 5 },
+ KeyTransition { keycode: 42, pressed: false, modifier_mask: 4 },
+ KeyTransition { keycode: 29, pressed: false, modifier_mask: 0 },
+ ]
+ );
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
index 1dfa5f0985..d13de7967b 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs
@@ -13,7 +13,10 @@ pub struct WindowInfo {
/// X11 Window (XID) cast to u64.
pub xid: u64,
pub pid: Option<u32>,
+ pub app_name: String,
pub title: String,
+ pub is_on_screen: bool,
+ pub z_index: Option<usize>,
pub x: i32,
pub y: i32,
pub width: u32,
@@ -37,7 +40,7 @@ fn list_windows_inner(filter_pid: Option<u32>) -> Result<Vec<WindowInfo>> {
let windows = get_window_list(&conn, root)?;
let mut result = Vec::new();
- for xid in windows {
+ for (z_index, xid) in windows.into_iter().enumerate() {
let pid = get_window_pid(&conn, xid).ok().flatten();
if let Some(fp) = filter_pid {
if pid != Some(fp) { continue; }
@@ -45,6 +48,14 @@ fn list_windows_inner(filter_pid: Option<u32>) -> Result<Vec<WindowInfo>> {
let title = get_window_title(&conn, xid).unwrap_or_default();
if title.trim().is_empty() { continue; }
+ let app_name = get_window_class(&conn, xid)
+ .map(|(instance, class)| if class.is_empty() { instance } else { class })
+ .unwrap_or_default();
+ let is_on_screen = conn
+ .get_window_attributes(xid)
+ .ok()
+ .and_then(|cookie| cookie.reply().ok())
+ .is_some_and(|attributes| attributes.map_state == MapState::VIEWABLE);
let geom = conn.get_geometry(xid)?.reply().ok();
let (x, y, w, h) = if let Some(g) = geom {
@@ -56,7 +67,18 @@ fn list_windows_inner(filter_pid: Option<u32>) -> Result<Vec<WindowInfo>> {
(0, 0, 0, 0)
};
- result.push(WindowInfo { xid: xid as u64, pid, title, x, y, width: w, height: h });
+ result.push(WindowInfo {
+ xid: xid as u64,
+ pid,
+ app_name,
+ title,
+ is_on_screen,
+ z_index: Some(z_index),
+ x,
+ y,
+ width: w,
+ height: h,
+ });
}
Ok(result)
@@ -70,15 +92,36 @@ fn get_window_list(conn: &RustConnection, root: Window) -> Result<Vec<Window>> {
let windows: Vec<Window> = reply.value32()
.map(|iter| iter.collect())
.unwrap_or_default();
- if !windows.is_empty() {
+ if client_list_property(reply.type_, windows.as_slice()).is_some() {
return Ok(windows);
}
}
}
}
- // Fallback: query tree from root.
+
+ // No EWMH client-list property means there may be no window manager. In
+ // that case only expose mapped root children; unmapped Electron children
+ // can otherwise be reported before a late-starting WM reparents them.
let tree = conn.query_tree(root)?.reply()?;
- Ok(tree.children)
+ Ok(tree
+ .children
+ .into_iter()
+ .filter(|window| {
+ conn.get_window_attributes(*window)
+ .ok()
+ .and_then(|cookie| cookie.reply().ok())
+ .map(|attributes| fallback_window_is_listable(attributes.map_state))
+ .unwrap_or(false)
+ })
+ .collect())
+}
+
+fn client_list_property(property_type: Atom, windows: &[Window]) -> Option<&[Window]> {
+ (property_type != x11rb::NONE).then_some(windows)
+}
+
+fn fallback_window_is_listable(map_state: MapState) -> bool {
+ map_state == MapState::VIEWABLE
}
fn get_atom(conn: &RustConnection, name: &str) -> Result<Atom> {
@@ -119,6 +162,10 @@ fn get_window_title(conn: &RustConnection, window: Window) -> Result<String> {
/// WM_CLASS atom set, or the property could not be read.
pub fn wm_class_for_window(xid: u64) -> Option<(String, String)> {
let (conn, _) = RustConnection::connect(None).ok()?;
+ get_window_class(&conn, xid as u32)
+}
+
+fn get_window_class(conn: &RustConnection, xid: Window) -> Option<(String, String)> {
let reply = conn
.get_property(false, xid as u32, AtomEnum::WM_CLASS, AtomEnum::STRING, 0, 512)
.ok()?
@@ -133,3 +180,25 @@ pub fn wm_class_for_window(xid: u64) -> Option<(String, String)> {
}
Some((instance, class))
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn empty_present_client_list_does_not_fall_back_to_query_tree() {
+ assert_eq!(client_list_property(1, &[]), Some([].as_slice()));
+ }
+
+ #[test]
+ fn absent_client_list_allows_query_tree_fallback() {
+ assert_eq!(client_list_property(x11rb::NONE, &[]), None);
+ }
+
+ #[test]
+ fn query_tree_fallback_only_lists_viewable_windows() {
+ assert!(fallback_window_is_listable(MapState::VIEWABLE));
+ assert!(!fallback_window_is_listable(MapState::UNMAPPED));
+ assert!(!fallback_window_is_listable(MapState::UNVIEWABLE));
+ }
+}
diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/launcher-linux.sh b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/launcher-linux.sh
new file mode 100644
index 0000000000..6b49c008cd
--- /dev/null
+++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/launcher-linux.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# Keep Nix's glibc-bearing library path out of the prebuilt Electron child.
+unset LD_LIBRARY_PATH NIX_LD NIX_LD_LIBRARY_PATH
+export ACCESSIBILITY_ENABLED=1
+export NO_AT_BRIDGE=0
+exec "$(dirname "$0")/CuaTestHarness.Electron.bin" "$@"
diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
index 7b13e40765..d65e1689e0 100755
--- a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
@@ -36,17 +36,22 @@ def section(box, title):
class HarnessWindow(Gtk.Window):
def __init__(self):
super().__init__(title="CuaTestHarness GTK3")
- self.set_default_size(480, 760)
+ # Keep the nested scroll viewport visible on the canonical 1024x768
+ # desktop; the outer scroller still exposes controls below it.
+ self.set_default_size(560, 720)
self.counter = 0
self.clicks = 0
self._last_action = "none"
+ self._double_click_pending = False
self._menu_action = "none"
+ self.key_presses = 0
+ self.hotkeys = 0
# Top-level scroller so every control is reachable even on a short window.
scroller = Gtk.ScrolledWindow()
scroller.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC)
self.add(scroller)
- root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)
+ root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=4)
root.set_border_width(12)
scroller.add(root)
@@ -82,6 +87,12 @@ class HarnessWindow(Gtk.Window):
self.click_status = Gtk.Label(label="last_action=none clicks=0", xalign=0)
root.pack_start(self.click_status, False, False, 0)
+ # ── keyboard delivery ─────────────────────────────────────────────
+ self.key_status = Gtk.Label(label="last_key=none key_presses=0", xalign=0)
+ root.pack_start(self.key_status, False, False, 0)
+ self.hotkey_status = Gtk.Label(label="last_hotkey=none hotkeys=0", xalign=0)
+ root.pack_start(self.hotkey_status, False, False, 0)
+
# ── slider ────────────────────────────────────────────────────────
section(root, "slider")
adj = Gtk.Adjustment(value=0, lower=0, upper=100, step_increment=1, page_increment=10)
@@ -120,11 +131,15 @@ class HarnessWindow(Gtk.Window):
inner.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC)
inner.set_size_request(-1, 140)
tall = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=2)
+ scroll_target = aid(Gtk.Button(label="Scroll viewport target"), "scroll-tall-viewport")
+ scroll_target.set_can_focus(False)
+ tall.pack_start(scroll_target, False, False, 0)
tall.pack_start(Gtk.Label(label="SCROLL_TOP_MARKER_v1", xalign=0), False, False, 0)
for i in range(2, 41):
tall.pack_start(Gtk.Label(label=f"line {i:02d}", xalign=0), False, False, 0)
tall.pack_start(Gtk.Label(label="SCROLL_BOTTOM_MARKER_v1", xalign=0), False, False, 0)
inner.add(tall)
+ aid(inner.get_vscrollbar(), "scroll-tall-vertical")
root.pack_start(inner, False, False, 0)
self.scroll_inner = inner
self.scroll_status = Gtk.Label(label="scroll_offset=0", xalign=0)
@@ -136,6 +151,8 @@ class HarnessWindow(Gtk.Window):
open_pop = aid(Gtk.Button(label="Open Popover"), "btn-open-popover")
open_pop.connect("clicked", self.on_open_popover)
root.pack_start(open_pop, False, False, 0)
+ self.popover_status = Gtk.Label(label="popover_open=False", xalign=0)
+ root.pack_start(self.popover_status, False, False, 0)
self.popover = Gtk.Popover.new(open_pop)
self.popover.set_border_width(10)
self.popover.add(Gtk.Label(label="POPOVER_MARKER_v1"))
@@ -146,6 +163,7 @@ class HarnessWindow(Gtk.Window):
root.pack_start(ext, False, False, 0)
self.connect("destroy", Gtk.main_quit)
+ self.connect("key-press-event", self.on_key_press)
# ── handlers ──────────────────────────────────────────────────────────
def on_increment(self, *_):
@@ -161,16 +179,22 @@ class HarnessWindow(Gtk.Window):
def on_click_target(self, *_):
self.clicks += 1
- self._last_action = "click"
- self.click_status.set_text(f"last_action=click clicks={self.clicks}")
+ if self._double_click_pending:
+ self._double_click_pending = False
+ else:
+ self._last_action = "click"
+ self.click_status.set_text(f"last_action={self._last_action} clicks={self.clicks}")
def on_click_target_press(self, _w, ev):
if ev.type == Gdk.EventType.DOUBLE_BUTTON_PRESS:
+ self._double_click_pending = True
self._last_action = "double_click"
self.click_status.set_text(f"last_action=double_click clicks={self.clicks}")
elif ev.button == 3:
self._last_action = "right_click"
self.click_status.set_text(f"last_action=right_click clicks={self.clicks}")
+ elif ev.button == 1:
+ self._double_click_pending = False
def on_scale(self, s):
self.scale_status.set_text(f"slider_value={int(s.get_value())}")
@@ -186,11 +210,28 @@ class HarnessWindow(Gtk.Window):
self._menu_action = lbl
self.ctx_status.set_text(f"menu_action={lbl}")
+ def on_key_press(self, _w, ev):
+ key = (Gdk.keyval_name(ev.keyval) or "unknown").lower()
+ ctrl = bool(ev.state & Gdk.ModifierType.CONTROL_MASK)
+ shift = bool(ev.state & Gdk.ModifierType.SHIFT_MASK)
+ if ctrl and shift and key == "k":
+ self.hotkeys += 1
+ self.hotkey_status.set_text(
+ f"last_hotkey=ctrl+shift+k hotkeys={self.hotkeys}"
+ )
+ return True
+ if key == "f5":
+ self.key_presses += 1
+ self.key_status.set_text(f"last_key=f5 key_presses={self.key_presses}")
+ return True
+ return False
+
def on_scroll(self, adj):
self.scroll_status.set_text(f"scroll_offset={int(adj.get_value())}")
def on_open_popover(self, *_):
self.popover.show_all()
+ self.popover_status.set_text("popover_open=True")
def main():
diff --git a/libs/cua-driver/tests/fixtures/build/linux.sh b/libs/cua-driver/tests/fixtures/build/linux.sh
index b82c7e87a8..4632f70fdb 100755
--- a/libs/cua-driver/tests/fixtures/build/linux.sh
+++ b/libs/cua-driver/tests/fixtures/build/linux.sh
@@ -11,6 +11,7 @@
# Usage:
# ./linux.sh
# ./linux.sh --skip gtk3 # skip one target (gtk3|electron|tauri)
+# ./linux.sh --only electron,gtk3
# ./linux.sh --clean
set -euo pipefail
@@ -20,6 +21,7 @@ TEST_APPS_DIR="$(cd "$HARNESS_DIR/../../rust/test-apps" && pwd)"
STAGE="$TEST_APPS_DIR/harness-gtk3"
SRC="$HARNESS_DIR/apps/linux/gtk3"
SKIP="none"
+ONLY=",gtk3,electron,tauri,"
CLEAN=0
while [[ $# -gt 0 ]]; do
@@ -32,8 +34,12 @@ while [[ $# -gt 0 ]]; do
CLEAN=1
shift
;;
+ --only)
+ ONLY=",${2:-},"
+ shift 2
+ ;;
*)
- echo "Usage: $0 [--skip gtk3|electron|tauri] [--clean]" >&2
+ echo "Usage: $0 [--skip gtk3|electron|tauri] [--only comma-separated-targets] [--clean]" >&2
exit 2
;;
esac
@@ -43,33 +49,36 @@ if [[ "$CLEAN" == "1" ]]; then
rm -rf "$TEST_APPS_DIR/harness-gtk3" "$TEST_APPS_DIR/harness-electron" "$TEST_APPS_DIR/harness-tauri"
fi
-if [[ "$SKIP" != "gtk3" ]]; then
+if [[ "$SKIP" != "gtk3" && "$ONLY" == *",gtk3,"* ]]; then
rm -rf "$STAGE"
mkdir -p "$STAGE"
cp "$SRC/main.py" "$STAGE/main.py"
cat > "$STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER'
#!/usr/bin/env bash
-# Force the X11 backend so the window is enumerable via cua-driver's X11
-# list_windows (_NET_CLIENT_LIST) — under a Wayland session this routes through
-# Xwayland; on a pure-X11 session it's a no-op. AT-SPI works over D-Bus either way.
-export GDK_BACKEND=x11
+# X11 remains the default for the canonical Xvfb lane. A native Wayland lane
+# exports GDK_BACKEND=wayland before launching this repo-owned fixture.
+export GDK_BACKEND="${GDK_BACKEND:-x11}"
exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@"
LAUNCHER
chmod +x "$STAGE/CuaTestHarness.Gtk3"
echo "==> Staged GTK3 harness -> $STAGE/CuaTestHarness.Gtk3"
- if python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk" 2>/dev/null; then
+ gtk_probe=(python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk")
+ if command -v timeout >/dev/null 2>&1; then
+ gtk_probe=(timeout 10s "${gtk_probe[@]}")
+ fi
+ if "${gtk_probe[@]}" 2>/dev/null; then
echo "==> PyGObject/GTK3 present"
else
echo "WARNING: PyGObject/GTK3 not importable - install python3-gi gir1.2-gtk-3.0 at-spi2-core" >&2
fi
fi
-if [[ "$SKIP" != "electron" ]]; then
+if [[ "$SKIP" != "electron" && "$ONLY" == *",electron,"* ]]; then
"$HARNESS_DIR/apps/cross-platform/electron/build.sh"
fi
-if [[ "$SKIP" != "tauri" ]]; then
+if [[ "$SKIP" != "tauri" && "$ONLY" == *",tauri,"* ]]; then
"$HARNESS_DIR/apps/cross-platform/tauri/build.sh"
fi
diff --git a/libs/cua-driver/wayland-helper/README.md b/libs/cua-driver/wayland-helper/README.md
index e74921de5b..6bbb679cb2 100644
--- a/libs/cua-driver/wayland-helper/README.md
+++ b/libs/cua-driver/wayland-helper/README.md
@@ -1,16 +1,24 @@
# cua WinRects — GNOME Shell helper extension (Wayland)
-A ~40-line GNOME Shell extension that lets cua-driver get **pixel coordinates**
-and draw the **agent cursor** on GNOME Mutter Wayland — two things a normal
-Wayland client cannot do (no global coordinates; no `zwlr_layer_shell_v1`).
+A small GNOME Shell extension that lets cua-driver get **pixel coordinates**,
+activate an exact target window, capture the compositor stage, and draw the
+**agent cursor** on GNOME Mutter Wayland. A normal Wayland client cannot do
+these things globally.
It exposes `org.cua.WinRects` on the session bus:
-- `GetRects() -> json` — every window's `meta_window.get_frame_rect()` (screen
- geometry). cua-driver combines the window origin with AT-SPI
+- `GetRects() -> json` — every window's frame geometry and surface-buffer
+ origin. cua-driver combines the buffer origin with AT-SPI
`CoordType::Window` per-widget coords: `screen = origin + window_xy`. This is
the GNOME analogue of the X11 `_GTK_FRAME_EXTENTS` reconstruction (AT-SPI's
- `CoordType::Screen` is `(0,0)` for every widget on Mutter).
+ `CoordType::Screen` is `(0,0)` for every widget on Mutter). Keeping the frame
+ and buffer origins separate accounts for GTK client-side shadows.
+- `Activate(id) -> bool` — activate one Shell stable-sequence window and report
+ whether the request was accepted. cua-driver verifies focus through a second
+ `GetRects` snapshot before sending focus-bound portal/libei input, preventing
+ input from leaking into whichever application happened to be focused.
+- `Capture() -> png_base64` — capture the compositor stage through Shell's
+ screenshot API. cua-driver crops it with the same authoritative geometry.
- `MoveCursor(x,y)` / `ClickPulse(x,y)` / `HideCursor()` — render the agent
cursor as a Clutter actor on the compositor stage.
@@ -25,9 +33,14 @@ needed (unlike libei/RemoteDesktop).
gnome-extensions info winrects@cua # -> State: ACTIVE
```
-cua-driver auto-detects it at runtime (`wayland::shell_helper`); everything is
-best-effort, so the driver still runs (without screen coords / Wayland cursor)
-when the extension is absent. wlroots compositors (sway/labwc/KWin) don't need
-it — cua-driver uses `zwlr_layer_shell` + foreign-toplevel there.
+cua-driver auto-detects it at runtime (`wayland::shell_helper`). AX operations
+still work when it is absent, but pixel geometry, the Shell cursor, and safe
+foreground portal input are unavailable. cua-driver refuses focus-bound input
+instead of injecting into an unverified target.
-KDE Plasma Wayland (KWin) would need an equivalent KWin script; not yet provided.
+wlroots compositors such as Sway and labwc do not need it: cua-driver uses
+foreign-toplevel activation, virtual-pointer input, and layer-shell there.
+
+KDE Plasma Wayland needs an equivalent target-addressable KWin activation
+adapter; it is not yet provided. Portal reachability alone is insufficient
+because RemoteDesktop/libei input is global to the compositor focus.
diff --git a/libs/cua-driver/wayland-helper/winrects@cua/extension.js b/libs/cua-driver/wayland-helper/winrects@cua/extension.js
index 9668f5fdc3..817a44a9ab 100644
--- a/libs/cua-driver/wayland-helper/winrects@cua/extension.js
+++ b/libs/cua-driver/wayland-helper/winrects@cua/extension.js
@@ -1,12 +1,19 @@
import Gio from 'gi://Gio';
+import GLib from 'gi://GLib';
+import Shell from 'gi://Shell';
import St from 'gi://St';
import Clutter from 'gi://Clutter';
import Cairo from 'cairo';
import * as Main from 'resource:///org/gnome/shell/ui/main.js';
import {Extension} from 'resource:///org/gnome/shell/extensions/extension.js';
+Gio._promisify(Shell.Screenshot.prototype, 'screenshot_stage_to_content');
+Gio._promisify(Shell.Screenshot, 'composite_to_stream');
+
const IFACE = `<node><interface name="org.cua.WinRects">
<method name="GetRects"><arg type="s" direction="out" name="json"/></method>
+<method name="Capture"><arg type="s" direction="out" name="png_base64"/></method>
+<method name="Activate"><arg type="u" direction="in" name="id"/><arg type="b" direction="out" name="activated"/></method>
<method name="MoveCursor"><arg type="i" direction="in" name="x"/><arg type="i" direction="in" name="y"/></method>
<method name="ClickPulse"><arg type="i" direction="in" name="x"/><arg type="i" direction="in" name="y"/></method>
<method name="HideCursor"></method>
@@ -62,15 +69,80 @@ export default class WinRectsExtension extends Extension {
if (this._nameId) { Gio.bus_unown_name(this._nameId); this._nameId = 0; }
}
GetRects() {
+ const actors = global.get_window_actors();
+ const actorByWindow = new Map();
+ for (const actor of actors) {
+ if (actor.meta_window)
+ actorByWindow.set(actor.meta_window, actor);
+ }
+ const windows = global.display.sort_windows_by_stacking([...actorByWindow.keys()]);
+ const focusedWindow = global.display.focus_window;
const out = [];
- for (const a of global.get_window_actors()) {
- const w = a.meta_window;
- if (!w) continue;
+ for (let stacking = 0; stacking < windows.length; stacking++) {
+ const w = windows[stacking];
+ const actor = actorByWindow.get(w);
const r = w.get_frame_rect();
- out.push({pid: w.get_pid(), title: w.get_title(), x: r.x, y: r.y, w: r.width, h: r.height});
+ let buffer = r;
+ try {
+ buffer = w.get_buffer_rect();
+ } catch (_error) {
+ // Older Shell releases may not expose the buffer rectangle.
+ }
+ const minimized = Boolean(w.minimized);
+ out.push({
+ id: w.get_stable_sequence(),
+ pid: w.get_pid(),
+ title: w.get_title() ?? '',
+ x: r.x,
+ y: r.y,
+ w: r.width,
+ h: r.height,
+ buffer_x: buffer.x,
+ buffer_y: buffer.y,
+ focused: focusedWindow === w,
+ minimized,
+ visible: Boolean(actor?.visible) && !minimized,
+ stacking,
+ });
}
return JSON.stringify(out);
}
+ async CaptureAsync(_params, invocation) {
+ try {
+ const shooter = new Shell.Screenshot();
+ const [content, scale] = await shooter.screenshot_stage_to_content();
+ const stream = Gio.MemoryOutputStream.new_resizable();
+ await Shell.Screenshot.composite_to_stream(
+ content.get_texture(),
+ 0, 0, -1, -1,
+ scale,
+ null, 0, 0, 1,
+ stream
+ );
+ stream.close(null);
+ const encoded = GLib.base64_encode(stream.steal_as_bytes().get_data());
+ invocation.return_value(new GLib.Variant('(s)', [encoded]));
+ } catch (error) {
+ invocation.return_dbus_error('org.cua.WinRects.CaptureFailed', String(error));
+ }
+ }
+ ActivateAsync([id], invocation) {
+ const target = global.get_window_actors()
+ .map(actor => actor.meta_window)
+ .find(window => window?.get_stable_sequence() === id);
+ if (!target) {
+ invocation.return_value(new GLib.Variant('(b)', [false]));
+ return;
+ }
+ target.activate(global.get_current_time());
+ GLib.timeout_add(GLib.PRIORITY_DEFAULT, 100, () => {
+ invocation.return_value(new GLib.Variant(
+ '(b)',
+ [global.display.focus_window === target]
+ ));
+ return GLib.SOURCE_REMOVE;
+ });
+ }
MoveCursor(x, y) {
if (!this._cursor) return;
this._cursor.show();
diff --git a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json
index efc503bf18..89f708d423 100644
--- a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json
+++ b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json
@@ -1 +1 @@
-{"uuid":"winrects@cua","name":"WinRects","description":"Expose window frame rects over D-Bus for cua-driver coordinate reconstruction on Wayland.","shell-version":["46","45","47"],"version":1}
+{"uuid":"winrects@cua","name":"WinRects","description":"Expose window geometry, capture, cursor, and verified activation for cua-driver on Wayland.","shell-version":["45","46","47","48"],"version":3}
diff --git a/scripts/ci/linux/run-rust-e2e-desktop.sh b/scripts/ci/linux/run-rust-e2e-desktop.sh
new file mode 100755
index 0000000000..84c7ccc75c
--- /dev/null
+++ b/scripts/ci/linux/run-rust-e2e-desktop.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Validate the canonical Rust matrix in a representative maintainer desktop.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ENVIRONMENT="${1:-}"
+if [[ $# -gt 0 ]]; then
+ shift
+fi
+
+case "${ENVIRONMENT}" in
+ gnome)
+ [[ "${XDG_SESSION_TYPE:-}" == wayland ]] || {
+ echo "GNOME validation requires an active Wayland user session" >&2
+ exit 2
+ }
+ [[ "${XDG_CURRENT_DESKTOP:-}" == *GNOME* ]] || {
+ echo "XDG_CURRENT_DESKTOP does not identify GNOME: ${XDG_CURRENT_DESKTOP:-<unset>}" >&2
+ exit 2
+ }
+ gdbus call --session \
+ --dest org.cua.WinRects \
+ --object-path /org/cua/WinRects \
+ --method org.cua.WinRects.GetRects >/dev/null || {
+ echo "The GNOME WinRects helper is not available in this user session" >&2
+ exit 2
+ }
+ export CUA_E2E_COMPOSITOR=gnome-mutter
+ export CUA_E2E_INPUT_BACKENDS=atspi,libei-portal
+ export CUA_DRIVER_RS_ENABLE_WAYLAND=1
+ ;;
+ kde)
+ [[ "${XDG_SESSION_TYPE:-}" == wayland ]] || {
+ echo "KDE validation requires an active Wayland user session" >&2
+ exit 2
+ }
+ [[ "${XDG_CURRENT_DESKTOP:-}" == *KDE* ]] || {
+ echo "XDG_CURRENT_DESKTOP does not identify KDE: ${XDG_CURRENT_DESKTOP:-<unset>}" >&2
+ exit 2
+ }
+ kwin_version="$(kwin_wayland --version 2>/dev/null | sed -n 's/^kwin \([0-9][0-9]*\).*/\1/p')"
+ [[ "${kwin_version:-0}" -ge 6 ]] || {
+ echo "Representative KDE validation requires Plasma/KWin 6; found ${kwin_version:-unknown}" >&2
+ exit 2
+ }
+ export CUA_E2E_COMPOSITOR=kwin
+ export CUA_E2E_INPUT_BACKENDS=atspi,libei-portal
+ export CUA_DRIVER_RS_ENABLE_WAYLAND=1
+ ;;
+ xorg)
+ [[ -n "${DISPLAY:-}" && "${XDG_SESSION_TYPE:-x11}" != wayland ]] || {
+ echo "Real-Xorg validation requires DISPLAY in a non-Wayland session" >&2
+ exit 2
+ }
+ export CUA_E2E_COMPOSITOR=real-xorg
+ export CUA_E2E_INPUT_BACKENDS=atspi,xsend-event,xtest,mpx-uinput
+ ;;
+ *)
+ echo "Usage: run-rust-e2e-desktop.sh {gnome|kde|xorg} [--no-build]" >&2
+ exit 2
+ ;;
+esac
+
+if [[ "${CUA_E2E_HARNESS_FILTER:-}" == *tauri* && ! -e /dev/dri/renderD128 ]]; then
+ echo "Tauri/WebKitGTK validation requires a representative DRM render node" >&2
+ exit 2
+fi
+
+exec "${SCRIPT_DIR}/run-rust-e2e.sh" "$@"
diff --git a/scripts/ci/linux/run-rust-e2e-wayland.sh b/scripts/ci/linux/run-rust-e2e-wayland.sh
new file mode 100755
index 0000000000..c1e64e1e86
--- /dev/null
+++ b/scripts/ci/linux/run-rust-e2e-wayland.sh
@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+# Start one native, headless Sway session and run the complete Rust E2E matrix.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+RUNTIME_DIR="$(mktemp -d)"
+SWAY_CONFIG="$(mktemp)"
+SESSION_KIND="${CUA_E2E_WAYLAND_SESSION:-sway}"
+COMPOSITOR_LOG="${REPO_ROOT}/artifacts/cua-driver/linux/${SESSION_KIND}.log"
+ATSPI_LOG="${REPO_ROOT}/artifacts/cua-driver/linux/at-spi-bus.log"
+COMPOSITOR_PID=""
+DBUS_PID=""
+ATSPI_PID=""
+
+cleanup() {
+ if [[ -n "${COMPOSITOR_PID}" ]]; then
+ kill "${COMPOSITOR_PID}" 2>/dev/null || true
+ wait "${COMPOSITOR_PID}" 2>/dev/null || true
+ fi
+ if [[ -n "${DBUS_PID}" ]]; then
+ kill "${DBUS_PID}" 2>/dev/null || true
+ fi
+ if [[ -n "${ATSPI_PID}" ]]; then
+ kill "${ATSPI_PID}" 2>/dev/null || true
+ wait "${ATSPI_PID}" 2>/dev/null || true
+ fi
+ rm -f "${SWAY_CONFIG}"
+}
+trap cleanup EXIT
+
+mkdir -p "$(dirname "${COMPOSITOR_LOG}")"
+chmod 700 "${RUNTIME_DIR}"
+cat > "${SWAY_CONFIG}" <<'EOF'
+xwayland disable
+output HEADLESS-1 mode 1920x1080
+seat seat0 fallback true
+focus_follows_mouse no
+default_border none
+default_floating_border none
+for_window [title="^CuaTestHarness"] floating enable, resize set 940 780, move position 0 0
+for_window [title="CuaTestHarness Sentinel"] fullscreen enable
+EOF
+
+unset DISPLAY
+unset WAYLAND_DISPLAY
+export XDG_RUNTIME_DIR="${RUNTIME_DIR}"
+export XDG_SESSION_TYPE=wayland
+export XDG_CURRENT_DESKTOP=sway
+export XDG_SESSION_DESKTOP=sway
+export WLR_BACKENDS=headless
+export WLR_RENDERER=pixman
+export WLR_RENDERER_ALLOW_SOFTWARE=1
+export WLR_LIBINPUT_NO_DEVICES=1
+export WLR_HEADLESS_OUTPUTS=1
+export CUA_DRIVER_RS_ENABLE_WAYLAND=1
+if [[ "${SESSION_KIND}" == cua-compositor ]]; then
+ export CUA_E2E_COMPOSITOR=cua-compositor-nested
+ export CUA_E2E_INPUT_BACKENDS=atspi,cua-compositor-inject
+ export CUA_E2E_HARNESS_FILTER=electron
+ export CUA_INJECT_SOCKET="${XDG_RUNTIME_DIR}/cua-inject.sock"
+else
+ export CUA_E2E_COMPOSITOR=sway
+ export CUA_E2E_INPUT_BACKENDS=atspi,wlr-virtual-pointer
+fi
+export CUA_WAYLAND_RECORDING_OUTPUT=HEADLESS-1
+export ELECTRON_OZONE_PLATFORM_HINT=wayland
+export GDK_BACKEND=wayland
+export QT_QPA_PLATFORM=wayland
+# WebKitGTK's accelerated compositor requires a DRM render node. The canonical
+# hosted lane has none, so keep its best-effort software settings explicit;
+# native-Wayland WebKit coverage runs on the representative GNOME/KDE VMs.
+export WEBKIT_DISABLE_COMPOSITING_MODE=1
+export WEBKIT_DISABLE_DMABUF_RENDERER=1
+# This isolated CI session uses a private runtime directory and no user home
+# namespaces. Modern WebKitGTK ignores WEBKIT_FORCE_SANDBOX; use its explicit
+# test-only escape hatch so the WebProcess can publish its AT-SPI subtree.
+export WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS=1
+export NO_AT_BRIDGE=0
+export ACCESSIBILITY_ENABLED=1
+
+if [[ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ]]; then
+ dbus_daemon="$(command -v dbus-daemon)"
+ dbus_prefix="$(dirname "$(dirname "$(readlink -f "${dbus_daemon}")")")"
+ dbus_config="${dbus_prefix}/share/dbus-1/session.conf"
+ if [[ ! -f "${dbus_config}" ]]; then
+ echo "DBus session config is missing: ${dbus_config}" >&2
+ exit 1
+ fi
+ dbus_info="$(dbus-daemon --config-file="${dbus_config}" --fork --print-address=1 --print-pid=1)"
+ export DBUS_SESSION_BUS_ADDRESS="$(sed -n '1p' <<< "${dbus_info}")"
+ DBUS_PID="$(sed -n '2p' <<< "${dbus_info}")"
+fi
+
+# A private session bus does not activate the desktop accessibility stack by
+# itself. Start the repo-pinned AT-SPI launcher, then require org.a11y.Bus to
+# answer before any fixture or driver process inherits this session.
+ATSPI_LAUNCHER="${CUA_AT_SPI_BUS_LAUNCHER:-$(command -v at-spi-bus-launcher || true)}"
+if [[ ! -x "${ATSPI_LAUNCHER}" ]]; then
+ echo "AT-SPI bus launcher is unavailable: ${ATSPI_LAUNCHER:-<not found>}" >&2
+ exit 1
+fi
+"${ATSPI_LAUNCHER}" --launch-immediately > "${ATSPI_LOG}" 2>&1 &
+ATSPI_PID=$!
+deadline=$((SECONDS + 15))
+while ((SECONDS < deadline)); do
+ if ! kill -0 "${ATSPI_PID}" 2>/dev/null; then
+ echo "AT-SPI bus launcher exited before org.a11y.Bus became ready" >&2
+ cat "${ATSPI_LOG}" >&2
+ exit 1
+ fi
+ if gdbus call --session \
+ --dest org.a11y.Bus \
+ --object-path /org/a11y/bus \
+ --method org.a11y.Bus.GetAddress >/dev/null 2>&1; then
+ break
+ fi
+ sleep 0.2
+done
+if ! gdbus call --session \
+ --dest org.a11y.Bus \
+ --object-path /org/a11y/bus \
+ --method org.a11y.Bus.GetAddress >/dev/null 2>&1; then
+ echo "org.a11y.Bus did not become ready within 15 seconds" >&2
+ cat "${ATSPI_LOG}" >&2
+ exit 1
+fi
+
+# The bus launcher and the registry daemon are separate services. Resolve the
+# private accessibility bus and require the registry to activate on it before
+# any toolkit inherits this session.
+a11y_address="$(
+ gdbus call --session \
+ --dest org.a11y.Bus \
+ --object-path /org/a11y/bus \
+ --method org.a11y.Bus.GetAddress \
+ | sed -e "s/^('//" -e "s/',)$//"
+)"
+export AT_SPI_BUS_ADDRESS="${a11y_address}"
+if [[ -z "${a11y_address}" ]] || ! gdbus call \
+ --address "${a11y_address}" \
+ --dest org.a11y.atspi.Registry \
+ --object-path /org/a11y/atspi/accessible/root \
+ --method org.freedesktop.DBus.Properties.Get \
+ org.a11y.atspi.Accessible ChildCount >/dev/null 2>&1; then
+ echo "AT-SPI registry did not activate on the accessibility bus" >&2
+ cat "${ATSPI_LOG}" >&2
+ exit 1
+fi
+if ! gdbus call --session \
+ --dest org.a11y.Bus \
+ --object-path /org/a11y/bus \
+ --method org.freedesktop.DBus.Properties.Set \
+ org.a11y.Status IsEnabled '<true>' >/dev/null 2>&1; then
+ echo "Could not enable accessibility on org.a11y.Bus" >&2
+ cat "${ATSPI_LOG}" >&2
+ exit 1
+fi
+
+if [[ "${SESSION_KIND}" == cua-compositor ]]; then
+ command -v cua-compositor >/dev/null || {
+ echo "cua-compositor is required for the nested injection lane" >&2
+ exit 1
+ }
+ cua-compositor > "${COMPOSITOR_LOG}" 2>&1 &
+else
+ sway --unsupported-gpu --config "${SWAY_CONFIG}" > "${COMPOSITOR_LOG}" 2>&1 &
+fi
+COMPOSITOR_PID=$!
+
+deadline=$((SECONDS + 20))
+while ((SECONDS < deadline)); do
+ if ! kill -0 "${COMPOSITOR_PID}" 2>/dev/null; then
+ echo "${SESSION_KIND} exited before its Wayland socket became ready" >&2
+ cat "${COMPOSITOR_LOG}" >&2
+ exit 1
+ fi
+ socket="$(find "${XDG_RUNTIME_DIR}" -maxdepth 1 -type s -name 'wayland-*' -print -quit)"
+ if [[ -n "${socket}" ]]; then
+ export WAYLAND_DISPLAY="$(basename "${socket}")"
+ break
+ fi
+ sleep 0.2
+done
+
+if [[ -z "${WAYLAND_DISPLAY:-}" ]]; then
+ echo "${SESSION_KIND} did not create a Wayland socket within 20 seconds" >&2
+ cat "${COMPOSITOR_LOG}" >&2
+ exit 1
+fi
+
+if [[ "${SESSION_KIND}" == cua-compositor ]]; then
+ deadline=$((SECONDS + 10))
+ while [[ ! -S "${CUA_INJECT_SOCKET}" ]] && ((SECONDS < deadline)); do
+ sleep 0.2
+ done
+ if [[ ! -S "${CUA_INJECT_SOCKET}" ]]; then
+ echo "cua-compositor did not expose its injection socket" >&2
+ cat "${COMPOSITOR_LOG}" >&2
+ exit 1
+ fi
+else
+ export SWAYSOCK="$(find "${XDG_RUNTIME_DIR}" -maxdepth 1 -type s -name 'sway-ipc.*.sock' -print -quit)"
+ if [[ -z "${SWAYSOCK}" ]]; then
+ echo "Sway did not expose its IPC socket" >&2
+ cat "${COMPOSITOR_LOG}" >&2
+ exit 1
+ fi
+fi
+
+echo "Native Wayland E2E session: ${SESSION_KIND} on ${WAYLAND_DISPLAY}"
+set +e
+"${SCRIPT_DIR}/run-rust-e2e.sh" "$@"
+status=$?
+set -e
+if [[ "${status}" != 0 && "${SESSION_KIND}" == sway ]]; then
+ swaymsg -t get_tree > "${REPO_ROOT}/artifacts/cua-driver/linux/sway-tree.json" 2>/dev/null || true
+fi
+exit "${status}"
diff --git a/scripts/ci/linux/run-rust-e2e.sh b/scripts/ci/linux/run-rust-e2e.sh
index 12365c8431..e5e5e55975 100755
--- a/scripts/ci/linux/run-rust-e2e.sh
+++ b/scripts/ci/linux/run-rust-e2e.sh
@@ -8,21 +8,21 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
DRIVER_ROOT="${REPO_ROOT}/libs/cua-driver"
RUST_ROOT="${DRIVER_ROOT}/rust"
BUILD_FIXTURES=1
-SUITE="shared"
+SUITE="${CUA_E2E_INTERNAL_LANE:-all}"
usage() {
cat <<'EOF'
-Usage: run-rust-e2e.sh [--no-build] [--suite shared|modality|all]
+Usage: run-rust-e2e.sh [--no-build]
The caller must provide a real or virtual Linux desktop session. For a
headless session, wrap this command in xvfb-run and dbus-run-session.
+The contributor-facing command always runs the complete matrix.
EOF
}
while (($#)); do
case "$1" in
--no-build) BUILD_FIXTURES=0 ;;
- --suite) SUITE="${2:?missing suite}"; shift ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
esac
@@ -30,85 +30,150 @@ while (($#)); do
done
case "$SUITE" in
- shared|modality|all) ;;
- *) echo "unsupported suite: $SUITE" >&2; exit 2 ;;
+ shared|native|capture|all) ;;
+ *) echo "unsupported internal lane: $SUITE" >&2; exit 2 ;;
esac
-mkdir -p "${REPO_ROOT}/artifacts/cua-driver/linux"
-RECORDING_ROOT="${REPO_ROOT}/artifacts/cua-driver/linux/recordings"
+ARTIFACT_DIR="${REPO_ROOT}/artifacts/cua-driver/linux"
+mkdir -p "${ARTIFACT_DIR}"
+RECORDING_ROOT="${ARTIFACT_DIR}/recordings"
rm -rf "${RECORDING_ROOT}"
mkdir -p "${RECORDING_ROOT}"
-RESULTS_FILE="${REPO_ROOT}/artifacts/cua-driver/linux/results.jsonl"
-SUMMARY_FILE="${REPO_ROOT}/artifacts/cua-driver/linux/summary.md"
+DECLARATIONS_FILE="${ARTIFACT_DIR}/cases.jsonl"
+ENVIRONMENT_FILE="${ARTIFACT_DIR}/environment.jsonl"
+RESULTS_FILE="${ARTIFACT_DIR}/results.jsonl"
+SUMMARY_FILE="${ARTIFACT_DIR}/summary.md"
+: > "${DECLARATIONS_FILE}"
+: > "${ENVIRONMENT_FILE}"
: > "${RESULTS_FILE}"
-cat > "${SUMMARY_FILE}" <<'EOF'
-# CUA Rust Linux E2E matrix
-
-| Platform | Host/lane | Scenario | Status | Duration | Details |
-| --- | --- | --- | --- | --- | --- |
-EOF
+rm -f "${SUMMARY_FILE}"
+export CUA_E2E_DECLARATIONS_FILE="${DECLARATIONS_FILE}"
+export CUA_E2E_ENVIRONMENT_FILE="${ENVIRONMENT_FILE}"
export CUA_E2E_RESULTS_FILE="${RESULTS_FILE}"
-export CUA_E2E_SUMMARY_FILE="${SUMMARY_FILE}"
export CUA_E2E_RECORDINGS_ROOT="${RECORDING_ROOT}"
export CUA_TEST_WORKSPACE_ROOT="${RUST_ROOT}"
export CUA_TEST_DRIVER_BIN="${RUST_ROOT}/target/release/cua-driver"
export CUA_TEST_APPS_ROOT="${RUST_ROOT}/test-apps"
export CUA_TEST_REQUIRE_FIXTURES=1
export CUA_TEST_DRIVER_STDERR=1
+export RUST_BACKTRACE="${RUST_BACKTRACE:-1}"
+if [[ -z "${CUA_E2E_SOURCE_SHA:-}" && -f "${REPO_ROOT}/.cua-e2e-source-sha" ]]; then
+ export CUA_E2E_SOURCE_SHA="$(tr -d '[:space:]' < "${REPO_ROOT}/.cua-e2e-source-sha")"
+ export CUA_E2E_SOURCE_MARKER="${REPO_ROOT}/.cua-e2e-source-sha"
+fi
+if [[ -n "${WAYLAND_DISPLAY:-}" && -z "${DISPLAY:-}" ]]; then
+ export GDK_BACKEND="${GDK_BACKEND:-wayland}"
+ export CUA_E2E_COMPOSITOR="${CUA_E2E_COMPOSITOR:-wayland-unknown}"
+ export CUA_E2E_INPUT_BACKENDS="${CUA_E2E_INPUT_BACKENDS:-atspi}"
+else
+ export CUA_E2E_COMPOSITOR="${CUA_E2E_COMPOSITOR:-openbox-x11}"
+ export CUA_E2E_INPUT_BACKENDS="${CUA_E2E_INPUT_BACKENDS:-atspi,xsend-event,xtest}"
+fi
+if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then
+ export CUA_ATSPI_DEBUG=1
+fi
-command -v ffmpeg >/dev/null || { echo "ffmpeg is required for E2E trajectory videos" >&2; exit 1; }
+if [[ -n "${WAYLAND_DISPLAY:-}" && -z "${DISPLAY:-}" ]]; then
+ command -v wf-recorder >/dev/null || { echo "wf-recorder is required for native Wayland E2E videos" >&2; exit 1; }
+ command -v grim >/dev/null || { echo "grim is required for native Wayland capture fallback" >&2; exit 1; }
+ command -v wtype >/dev/null || { echo "wtype is required for native Wayland keyboard input" >&2; exit 1; }
+else
+ command -v ffmpeg >/dev/null || { echo "ffmpeg is required for X11 E2E trajectory videos" >&2; exit 1; }
+fi
command -v ffprobe >/dev/null || { echo "ffprobe is required for E2E trajectory validation" >&2; exit 1; }
+command -v jq >/dev/null || { echo "jq is required for E2E ownership validation" >&2; exit 1; }
if [[ "${BUILD_FIXTURES}" == 1 ]]; then
cargo build --release -p cua-driver --manifest-path "${RUST_ROOT}/Cargo.toml"
- bash "${DRIVER_ROOT}/tests/fixtures/build/linux.sh"
+ case "${SUITE}" in
+ shared) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri}" ;;
+ native|capture) FIXTURE_TARGETS="electron,gtk3" ;;
+ *) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3" ;;
+ esac
+ bash "${DRIVER_ROOT}/tests/fixtures/build/linux.sh" --only "${FIXTURE_TARGETS}"
fi
if [[ ! -x "${CUA_TEST_DRIVER_BIN}" ]]; then
echo "driver binary not found: ${CUA_TEST_DRIVER_BIN}" >&2
exit 1
fi
-if [[ ! -x "${CUA_TEST_APPS_ROOT}/harness-electron/CuaTestHarness.Electron" ]]; then
- echo "Electron fixture was not built: ${CUA_TEST_APPS_ROOT}/harness-electron" >&2
- exit 1
+required_fixtures=()
+required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-electron/CuaTestHarness.Electron")
+if [[ ("${SUITE}" == shared || "${SUITE}" == all) \
+ && ",${CUA_E2E_HARNESS_FILTER:-electron,tauri}," == *,tauri,* ]]; then
+ required_fixtures+=(
+ "${CUA_TEST_APPS_ROOT}/harness-tauri/CuaTestHarness.Tauri"
+ )
+fi
+if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
+ required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-gtk3/CuaTestHarness.Gtk3")
fi
+for fixture in "${required_fixtures[@]}"; do
+ if [[ ! -x "${fixture}" ]]; then
+ echo "Required fixture was not built: ${fixture}" >&2
+ exit 1
+ fi
+done
FAILURE_COUNT=0
+run_report() {
+ (cd "${RUST_ROOT}" && cargo run -p cua-driver-testkit --bin cua-e2e-report -- \
+ --declarations "${DECLARATIONS_FILE}" \
+ --environment "${ENVIRONMENT_FILE}" \
+ --results "${RESULTS_FILE}" \
+ --artifact-root "${ARTIFACT_DIR}" \
+ --require-video \
+ --output "${SUMMARY_FILE}")
+}
+
+echo "[PREFLIGHT] Linux desktop, fixture, AX, capture, and video"
+set +e
+(cd "${RUST_ROOT}" && cargo test -p cua-driver --test e2e_environment_preflight_test -- \
+ --ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1) \
+ 2>&1 | tee "${ARTIFACT_DIR}/environment-preflight.log"
+PREFLIGHT_EXIT=${PIPESTATUS[0]}
+set -e
+if [[ "${PREFLIGHT_EXIT}" != 0 ]]; then
+ set +e
+ run_report
+ set -e
+ echo "Linux E2E environment preflight failed" >&2
+ exit 1
+fi
+
run_test() {
local name="$1"
shift
echo "[RUN] ${name}"
set +e
- (cd "${RUST_ROOT}" && "$@") 2>&1 | tee "${REPO_ROOT}/artifacts/cua-driver/linux/${name}.log"
+ (cd "${RUST_ROOT}" && "$@") 2>&1 | tee "${ARTIFACT_DIR}/${name}.log"
local exit_code=${PIPESTATUS[0]}
set -e
- local status="PASS"
if [[ "${exit_code}" != 0 ]]; then
- status="FAIL"
FAILURE_COUNT=$((FAILURE_COUNT + 1))
fi
- local details="-"
- if [[ "${exit_code}" != 0 ]]; then
- details="exit code ${exit_code}"
- fi
- printf '{"schema":"cua-e2e-result/v1","platform":"linux","host":"lane","scenario":"%s","status":"%s","message":"%s"}\n' \
- "${name}" "${status}" "${details}" >> "${RESULTS_FILE}"
- printf '| Linux | lane | %s | %s | n/a | %s |\n' "${name}" "${status}" "${details}" >> "${SUMMARY_FILE}"
}
if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then
run_test shared-behavior-matrix \
cargo test -p cua-driver --test cross_platform_behavior_test -- \
+ --ignored --exact shared_web_action_matrix_is_state_verified \
+ --nocapture --test-threads=1
+fi
+
+if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then
+ run_test gtk3-native-harness \
+ cargo test -p cua-driver --test harness_gtk3_test -- \
--ignored --nocapture --test-threads=1
fi
-if [[ "${SUITE}" == modality || "${SUITE}" == all ]]; then
- run_test modality-capture \
- cargo test -p cua-driver --test modality_capture_mode_test -- \
+if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then
+ run_test capture-contract \
+ cargo test -p cua-driver --test capture_contract_test -- \
--ignored --nocapture --test-threads=1
- run_test modality-desktop-scope \
- cargo test -p cua-driver --test modality_desktop_scope_linux_test -- \
+ run_test desktop-scope \
+ cargo test -p cua-driver --test desktop_scope_linux_test -- \
--ignored --nocapture --test-threads=1
fi
@@ -124,6 +189,20 @@ while IFS= read -r -d '' video; do
fi
done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0)
+OWNED_VIDEOS="$(mktemp)"
+jq -r 'select(.evidence.video != null) | .evidence.video' "${RESULTS_FILE}" > "${OWNED_VIDEOS}"
+while IFS= read -r -d '' video; do
+ relative="${video#${ARTIFACT_DIR}/}"
+ if [[ "${relative}" == recordings/environment-preflight-*/recording.mp4 ]]; then
+ continue
+ fi
+ if ! grep -Fxq -- "${relative}" "${OWNED_VIDEOS}"; then
+ echo "[VIDEO FAIL] Orphan trajectory has no typed result row: ${relative}" >&2
+ FAILURE_COUNT=$((FAILURE_COUNT + 1))
+ fi
+done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0)
+rm -f "${OWNED_VIDEOS}"
+
while IFS= read -r -d '' recording_error; do
echo "[VIDEO FAIL] ${recording_error}" >&2
cat "${recording_error}" >&2
@@ -135,6 +214,15 @@ if [[ "${video_count}" == 0 ]]; then
FAILURE_COUNT=$((FAILURE_COUNT + 1))
fi
+set +e
+run_report
+REPORT_EXIT=$?
+set -e
+if [[ "${REPORT_EXIT}" != 0 ]]; then
+ echo "Linux E2E result validation failed" >&2
+ FAILURE_COUNT=$((FAILURE_COUNT + 1))
+fi
+
if [[ "${FAILURE_COUNT}" != 0 ]]; then
echo "Linux Rust e2e suite had ${FAILURE_COUNT} failing lane(s)" >&2
exit 1