Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cua-driver): run jev-use native tasks on Windows and Linux (#4290)

Francesco Bonacci07e4130e5031
diff --git a/.github/scripts/tests/test_ci_jev_use_workflow.py b/.github/scripts/tests/test_ci_jev_use_workflow.py
index c577389488..2136a2fa91 100644
--- a/.github/scripts/tests/test_ci_jev_use_workflow.py
+++ b/.github/scripts/tests/test_ci_jev_use_workflow.py
@@ -133,3 +133,56 @@ def test_native_harness_changes_trigger_the_workflow() -> None:
triggers = load()[True]
for event in ("pull_request", "push"):
assert "libs/cua-driver/tests/fixtures/apps/macos/appkit/**" in triggers[event]["paths"]
+
+
+def test_native_wpf_job_uses_released_driver_default_install_and_source_harness() -> None:
+ """RFC #4268 Phase 2: native tasks on the WPF harness (Windows UIA)."""
+ jobs = load()["jobs"]
+ assert jobs["native-wpf-windows"]["runs-on"].startswith("windows-")
+ native = job_text("native-wpf-windows")
+ assert "https://cua.ai/driver/install.ps1" in native
+ assert "cargo build" not in native
+ assert 'MIN_RELEASED_DRIVER_VERSION: "0.30.1"' in native
+ # The default install registers the elevated autostart; no opt-out flag.
+ assert '$task.Principal.RunLevel -ne "Highest"' in native
+ assert "-NoAutostart" not in native
+ assert "verify-user-session.ps1" in native
+ assert "tests\\fixtures\\build\\windows.ps1 -Targets wpf" in native
+ assert "verify_native.py --harness wpf --typescript" in native
+ assert "--live" not in native
+ assert 'check["verified"] is True' in native
+ assert "cua.jev_choice_request_v2" in native
+
+
+def test_native_gtk3_job_uses_released_driver_and_x11_stack() -> None:
+ """RFC #4268 Phase 2: native tasks on the GTK3 harness (Linux AT-SPI)."""
+ jobs = load()["jobs"]
+ assert jobs["native-gtk3-linux"]["runs-on"].startswith("ubuntu-")
+ native = job_text("native-gtk3-linux")
+ assert "https://cua.ai/driver/install.sh" in native
+ assert "cargo build" not in native
+ assert 'MIN_RELEASED_DRIVER_VERSION: "0.30.1"' in native
+ assert "tests/fixtures/build/linux.sh --only gtk3" in native
+ for piece in ("xvfb-run", "dbus-run-session", "openbox", "at-spi2-core", "python3-gi"):
+ assert piece in native
+ assert "verify_native.py --harness gtk3 --typescript" in native
+ # save-note is gated on the first Driver that reports named AT-SPI text
+ # values (#4291), and the audit still requires the other two tasks.
+ assert 'SAVE_NOTE_MIN_DRIVER_VERSION: "0.30.3"' in native
+ assert 'assert {"gtk3-counter", "gtk3-choose-size"} <= tasks' in native
+ assert "--live" not in native
+ assert 'check["verified"] is True' in native
+ assert "cua.jev_choice_request_v2" in native
+
+
+def test_windows_and_linux_harness_changes_trigger_the_workflow() -> None:
+ triggers = load()[True]
+ for event in ("pull_request", "push"):
+ paths = triggers[event]["paths"]
+ for path in (
+ "libs/cua-driver/tests/fixtures/apps/windows/wpf/**",
+ "libs/cua-driver/tests/fixtures/build/windows.ps1",
+ "libs/cua-driver/tests/fixtures/apps/linux/gtk3/**",
+ "libs/cua-driver/tests/fixtures/build/linux.sh",
+ ):
+ assert path in paths
diff --git a/.github/workflows/ci-jev-use.yml b/.github/workflows/ci-jev-use.yml
index f779455d56..cda70a5c73 100644
--- a/.github/workflows/ci-jev-use.yml
+++ b/.github/workflows/ci-jev-use.yml
@@ -6,6 +6,10 @@ on:
- "libs/cua-driver/examples/jev-use/**"
- "libs/cua-driver/tests/fixtures/apps/macos/appkit/**"
- "libs/cua-driver/tests/fixtures/build/macos.sh"
+ - "libs/cua-driver/tests/fixtures/apps/windows/wpf/**"
+ - "libs/cua-driver/tests/fixtures/build/windows.ps1"
+ - "libs/cua-driver/tests/fixtures/apps/linux/gtk3/**"
+ - "libs/cua-driver/tests/fixtures/build/linux.sh"
- "libs/cua-driver/docs/jev-use-validation.md"
- "docs/content/docs/how-to-guides/driver/jev-use.mdx"
- "docs/content/docs/how-to-guides/driver/meta.json"
@@ -20,6 +24,10 @@ on:
- "libs/cua-driver/examples/jev-use/**"
- "libs/cua-driver/tests/fixtures/apps/macos/appkit/**"
- "libs/cua-driver/tests/fixtures/build/macos.sh"
+ - "libs/cua-driver/tests/fixtures/apps/windows/wpf/**"
+ - "libs/cua-driver/tests/fixtures/build/windows.ps1"
+ - "libs/cua-driver/tests/fixtures/apps/linux/gtk3/**"
+ - "libs/cua-driver/tests/fixtures/build/linux.sh"
- "libs/cua-driver/docs/jev-use-validation.md"
- "docs/content/docs/how-to-guides/driver/jev-use.mdx"
- "docs/content/docs/how-to-guides/driver/meta.json"
@@ -650,3 +658,270 @@ jobs:
${{ env.JEV_USE_LOG_DIR }}/*.txt
if-no-files-found: error
retention-days: 14
+
+ # RFC #4268 Phase 2: the same native tasks on the WPF harness (Windows UIA),
+ # built from this checkout, driven by the released Driver from the default
+ # install.ps1 (which registers the RunLevel=Highest cua-driver-serve
+ # autostart; this administrator token also elevates the runners' in-process
+ # `cua-driver mcp` runtime), with the deterministic mock provider.
+ # verify_native.py launches a fresh harness in task mode per run and reads
+ # the harness's own state file as the independent oracle.
+ native-wpf-windows:
+ name: Windows native WPF mock E2E
+ runs-on: windows-latest
+ timeout-minutes: 45
+ env:
+ MIN_RELEASED_DRIVER_VERSION: "0.30.1"
+ defaults:
+ run:
+ shell: pwsh
+ working-directory: libs/cua-driver/examples/jev-use
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ persist-credentials: false
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ - uses: astral-sh/setup-uv@v6
+ with:
+ version: "0.8.15"
+ enable-cache: true
+ cache-dependency-glob: libs/cua-driver/examples/jev-use/uv.lock
+ - uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+ cache: npm
+ cache-dependency-path: libs/cua-driver/examples/jev-use/package-lock.json
+ - name: Verify the interactive desktop session
+ working-directory: ${{ github.workspace }}
+ run: .\scripts\ci\windows\verify-user-session.ps1
+ - name: Install the released Driver with the canonical installer defaults
+ run: |
+ & ([scriptblock]::Create((irm https://cua.ai/driver/install.ps1))) -NoPathUpdate
+ $driver = Join-Path $env:LOCALAPPDATA "Programs\Cua\cua-driver\bin\cua-driver.exe"
+ if (-not (Test-Path -LiteralPath $driver -PathType Leaf)) { throw "installer did not create $driver" }
+ $versionOutput = (& $driver --version | Out-String).Trim()
+ Write-Host $versionOutput
+ if ($versionOutput -notmatch '(\d+\.\d+\.\d+)') { throw "cannot parse the Driver version from: $versionOutput" }
+ $resolved = [version]$Matches[1]
+ if ($resolved -lt [version]$env:MIN_RELEASED_DRIVER_VERSION) {
+ throw "canonical installer resolved Cua Driver $resolved, older than $env:MIN_RELEASED_DRIVER_VERSION"
+ }
+ Write-Host "Released Cua Driver under test: $resolved"
+ $task = Get-ScheduledTask -TaskName "cua-driver-serve" -ErrorAction Stop
+ if ($task.Principal.RunLevel -ne "Highest") { throw "cua-driver-serve autostart is not RunLevel=Highest" }
+ "CUA_DRIVER_BIN=$driver" >> $env:GITHUB_ENV
+ - name: Build the WPF harness from source
+ working-directory: ${{ github.workspace }}
+ run: |
+ .\libs\cua-driver\tests\fixtures\build\windows.ps1 -Targets wpf
+ $exe = "libs\cua-driver\rust\test-apps\harness-wpf\CuaTestHarness.Wpf.exe"
+ if (-not (Test-Path -LiteralPath $exe -PathType Leaf)) { throw "WPF harness was not built: $exe" }
+ - name: Install locked example dependencies
+ run: |
+ uv sync --frozen
+ if ($LASTEXITCODE -ne 0) { throw "uv sync failed" }
+ npm.cmd ci --ignore-scripts
+ if ($LASTEXITCODE -ne 0) { throw "npm ci failed" }
+ - name: Run the native tasks with both mock agents
+ run: |
+ $proof = Join-Path $env:RUNNER_TEMP "jev-use-native-proof"
+ $capture = Join-Path $env:RUNNER_TEMP "jev-use-native-window-state"
+ uv run --frozen python verify_native.py --harness wpf --typescript --output-dir $proof --capture-dir $capture
+ if ($LASTEXITCODE -ne 0) { throw "native verification failed" }
+ - name: Audit redacted native evidence
+ run: |
+ @'
+ import json, pathlib, sys
+ root = pathlib.Path(sys.argv[1])
+ summary = json.loads((root / "summary.json").read_text(encoding="utf-8"))
+ assert summary["complete"] is True
+ assert summary["live_requested"] is False
+ assert summary["harness"] == "wpf"
+ tasks = {"wpf-counter", "wpf-save-note", "wpf-choose-size"}
+ assert {(c["language"], c["task"]) for c in summary["checks"]} == {
+ (language, task) for language in ("python", "typescript") for task in tasks
+ }
+ for check in summary["checks"]:
+ # verify_native.py read the harness's own state file; the
+ # runners' outcome events are not the oracle.
+ assert check["verified"] is True and check["runner_outcome"] == "verified", check
+ assert check["provider"] == "mock"
+ assert check["schemas"] == ["cua.jev_choice_request_v2"], check
+ assert check["sources"] == ["ax"], check
+ # set_value has no delivery mode; every click stayed in the background.
+ assert check["delivery_modes"] in (["background"], ["background", "none"]), check
+ assert check["visual_statuses"] == ["skipped"], check
+ counter = [c for c in summary["checks"] if c["task"] == "wpf-counter"]
+ # Reset (destructive) and Exit (close_unsaved) never reach the chooser;
+ # the title bar's System, Minimize, Maximize, and Close are window chrome.
+ assert all(c["risk_excluded"] == ["close_unsaved", "destructive"] for c in counter)
+ assert all(c["observed"]["counter"] == 3 for c in counter)
+ note = [c for c in summary["checks"] if c["task"] == "wpf-save-note"]
+ assert all(c["tools"] == ["click", "set_value"] for c in note)
+ forbidden = ("typesafe_api_key", "authorization", "screenshot_png_b64", "image_base64",
+ "element_token", "jev-use native note")
+ for path in root.iterdir():
+ if path.name == "summary.json" or path.suffix == ".jsonl":
+ text = path.read_text(encoding="utf-8").lower()
+ assert not any(value in text for value in forbidden), (path, [v for v in forbidden if v in text])
+ '@ | uv run --frozen python - (Join-Path $env:RUNNER_TEMP "jev-use-native-proof")
+ if ($LASTEXITCODE -ne 0) { throw "evidence audit failed" }
+ - name: Stop Driver and harness processes
+ if: always()
+ run: |
+ Get-Process cua-driver, CuaTestHarness.Wpf -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
+ - name: Upload redacted native evidence
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: jev-use-windows-native-wpf-mock
+ path: |
+ ${{ runner.temp }}/jev-use-native-proof/summary.json
+ ${{ runner.temp }}/jev-use-native-proof/*.jsonl
+ ${{ runner.temp }}/jev-use-native-window-state/*.json
+ if-no-files-found: error
+ retention-days: 14
+
+ # RFC #4268 Phase 2: the same native tasks on the GTK3 harness (Linux
+ # AT-SPI) on the supported X11 stack (Xvfb, openbox, session bus), driven by
+ # the released Driver from the canonical install.sh with the deterministic
+ # mock provider. The harness runs on the system python3 with PyGObject, so
+ # verify_native.py runs from the locked venv directly (not `uv run`, which
+ # would put the venv's python3 first on PATH).
+ native-gtk3-linux:
+ name: Linux native GTK3 mock E2E
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ env:
+ CUA_DRIVER_BIN: /home/runner/.local/bin/cua-driver
+ MIN_RELEASED_DRIVER_VERSION: "0.30.1"
+ # Released Drivers before this version report no value for a named
+ # AT-SPI text field (#4291), so save-note cannot observe its own write.
+ # Older Drivers run the other two tasks and record that limitation.
+ SAVE_NOTE_MIN_DRIVER_VERSION: "0.30.3"
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ persist-credentials: false
+ - uses: astral-sh/setup-uv@v6
+ with:
+ version: "0.8.15"
+ enable-cache: true
+ cache-dependency-glob: libs/cua-driver/examples/jev-use/uv.lock
+ - uses: actions/setup-node@v4
+ with:
+ node-version: "22"
+ cache: npm
+ cache-dependency-path: libs/cua-driver/examples/jev-use/package-lock.json
+ - name: Install Linux desktop dependencies
+ working-directory: ${{ github.workspace }}
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ xvfb dbus-x11 at-spi2-core openbox picom python3-gi gir1.2-gtk-3.0
+ - name: Install the released Driver with the canonical installer
+ working-directory: ${{ github.workspace }}
+ env:
+ CUA_DRIVER_NO_MODIFY_PATH: "1"
+ run: |
+ /bin/bash -c "$(curl -fsSL https://cua.ai/driver/install.sh)"
+ version_output="$("$CUA_DRIVER_BIN" --version)"
+ echo "$version_output"
+ resolved="$(grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' <<< "$version_output" | head -n 1)"
+ test -n "$resolved"
+ lowest="$(printf '%s\n%s\n' "$MIN_RELEASED_DRIVER_VERSION" "$resolved" | sort -V | head -n 1)"
+ if [ "$lowest" != "$MIN_RELEASED_DRIVER_VERSION" ]; then
+ echo "::error::canonical installer resolved Cua Driver $resolved, older than $MIN_RELEASED_DRIVER_VERSION"
+ exit 1
+ fi
+ echo "Released Cua Driver under test: $resolved"
+ echo "RESOLVED_DRIVER_VERSION=$resolved" >> "$GITHUB_ENV"
+ - name: Build the GTK3 harness from source
+ working-directory: ${{ github.workspace }}
+ run: |
+ bash libs/cua-driver/tests/fixtures/build/linux.sh --only gtk3
+ test -x libs/cua-driver/rust/test-apps/harness-gtk3/CuaTestHarness.Gtk3
+ python3 -c "import gi; gi.require_version('Gtk', '3.0'); from gi.repository import Gtk"
+ - name: Install locked example dependencies
+ working-directory: libs/cua-driver/examples/jev-use
+ run: |
+ uv sync --frozen
+ npm ci --ignore-scripts
+ - name: Run the native tasks with both mock agents on X11
+ working-directory: ${{ github.workspace }}
+ run: |
+ session_script="$RUNNER_TEMP/jev-use-native-x11-session.sh"
+ cat > "$session_script" <<'SH'
+ set -euo pipefail
+ openbox >/tmp/jev-use-native-openbox.log 2>&1 &
+ picom --backend xrender --config /dev/null >/tmp/jev-use-native-picom.log 2>&1 &
+ sleep 2
+ cd libs/cua-driver/examples/jev-use
+ .venv/bin/python verify_native.py --harness gtk3 --typescript "$@" \
+ --output-dir "$RUNNER_TEMP/jev-use-native-proof" \
+ --capture-dir "$RUNNER_TEMP/jev-use-native-window-state"
+ SH
+ tasks=()
+ lowest="$(printf '%s\n%s\n' "$SAVE_NOTE_MIN_DRIVER_VERSION" "$RESOLVED_DRIVER_VERSION" | sort -V | head -n 1)"
+ if [ "$lowest" != "$SAVE_NOTE_MIN_DRIVER_VERSION" ]; then
+ tasks=(--task counter --task choose-size)
+ echo "::notice::Cua Driver $RESOLVED_DRIVER_VERSION reports no value for named AT-SPI text fields (#4291); gtk3-save-note needs $SAVE_NOTE_MIN_DRIVER_VERSION or later"
+ echo "EXPECTED_GTK3_TASKS=gtk3-counter,gtk3-choose-size" >> "$GITHUB_ENV"
+ else
+ echo "EXPECTED_GTK3_TASKS=gtk3-counter,gtk3-save-note,gtk3-choose-size" >> "$GITHUB_ENV"
+ fi
+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ dbus-run-session -- bash "$session_script" "${tasks[@]}"
+ - name: Audit redacted native evidence
+ working-directory: libs/cua-driver/examples/jev-use
+ run: |
+ .venv/bin/python - "$RUNNER_TEMP/jev-use-native-proof" <<'PY'
+ import json, os, pathlib, sys
+ root = pathlib.Path(sys.argv[1])
+ summary = json.loads((root / "summary.json").read_text())
+ assert summary["complete"] is True
+ assert summary["live_requested"] is False
+ assert summary["harness"] == "gtk3"
+ tasks = set(os.environ["EXPECTED_GTK3_TASKS"].split(","))
+ assert {"gtk3-counter", "gtk3-choose-size"} <= tasks
+ assert {(c["language"], c["task"]) for c in summary["checks"]} == {
+ (language, task) for language in ("python", "typescript") for task in tasks
+ }
+ for check in summary["checks"]:
+ # verify_native.py read the harness's own state file; the
+ # runners' outcome events are not the oracle.
+ assert check["verified"] is True and check["runner_outcome"] == "verified", check
+ assert check["provider"] == "mock"
+ assert check["schemas"] == ["cua.jev_choice_request_v2"], check
+ assert check["sources"] == ["ax"], check
+ # set_value has no delivery mode; every click stayed in the background.
+ assert check["delivery_modes"] in (["background"], ["background", "none"]), check
+ assert check["visual_statuses"] == ["skipped"], check
+ counter = [c for c in summary["checks"] if c["task"] == "gtk3-counter"]
+ # Reset (destructive) and Exit (close_unsaved) never reach the chooser.
+ assert all(c["risk_excluded"] == ["close_unsaved", "destructive"] for c in counter)
+ assert all(c["observed"]["counter"] == 3 for c in counter)
+ note = [c for c in summary["checks"] if c["task"] == "gtk3-save-note"]
+ assert all(c["tools"] == ["click", "set_value"] for c in note)
+ forbidden = ("typesafe_api_key", "authorization", "screenshot_png_b64", "image_base64",
+ "element_token", "jev-use native note")
+ for path in root.iterdir():
+ if path.name == "summary.json" or path.suffix == ".jsonl":
+ text = path.read_text().lower()
+ assert not any(value in text for value in forbidden), (path, [v for v in forbidden if v in text])
+ PY
+ - name: Upload redacted native evidence
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: jev-use-linux-native-gtk3-mock
+ path: |
+ ${{ runner.temp }}/jev-use-native-proof/summary.json
+ ${{ runner.temp }}/jev-use-native-proof/*.jsonl
+ ${{ runner.temp }}/jev-use-native-window-state/*.json
+ if-no-files-found: error
+ retention-days: 14
diff --git a/libs/cua-driver/examples/jev-use/README.md b/libs/cua-driver/examples/jev-use/README.md
index 477a83dca0..79f790ad0f 100644
--- a/libs/cua-driver/examples/jev-use/README.md
+++ b/libs/cua-driver/examples/jev-use/README.md
@@ -93,18 +93,40 @@ task parameters. Actions are element-bound `click`, `set_value`, or
observation. Native steps send `cua.jev_choice_request_v2`, which adds a
per-candidate `source` and compact value-free `elements`.
-Three tasks drive the repository's AppKit harness, whose opt-in
-`CUA_APPKIT_TASK_STATE` file is the independent oracle: `appkit-counter` (set
-the counter to 3), `appkit-save-note` (set the Note field and save), and
-`appkit-choose-size` (select Large and check I agree). On macOS with Cua
-Driver 0.30.1 or later:
+The same three tasks run on three repository harnesses: `<harness>-counter`
+(set the counter to 3), `<harness>-save-note` (set the Note field and save),
+and `<harness>-choose-size` (select Large and check I agree). The harness is
+`appkit` (macOS AX), `wpf` (Windows UIA), or `gtk3` (Linux AT-SPI). In task
+mode, each harness shows the same labeled controls, so candidate IDs and the
+mock provider's choices are identical on every platform. Task mode is selected
+with `CUA_APPKIT_TASK_STATE`, `CUA_WPF_TASK_STATE`, or `CUA_GTK3_TASK_STATE`.
+WPF and GTK3 show a small dedicated task window. Each harness rewrites that
+state file on every change, and the file is the independent oracle. On
+Windows, the title bar's System menu, Minimize, Maximize, and Close buttons
+are window chrome and never become candidates. With Cua Driver 0.30.1 or
+later, run from this directory:
```bash
+# macOS
bash ../../tests/fixtures/build/macos.sh --only appkit
-uv run --frozen python verify_native.py --typescript --output-dir /tmp/jev-native-proof
+uv run --frozen python verify_native.py --harness appkit --typescript --output-dir /tmp/jev-native-proof
+# Linux (X11 session with AT-SPI; the harness runs on the system python3 with PyGObject)
+bash ../../tests/fixtures/build/linux.sh --only gtk3
+.venv/bin/python verify_native.py --harness gtk3 --typescript --output-dir /tmp/jev-native-proof
```
-Windows (UIA) and Linux (AT-SPI) tasks are the next RFC phase.
+```powershell
+# Windows
+..\..\tests\fixtures\build\windows.ps1 -Targets wpf
+uv run --frozen python verify_native.py --harness wpf --typescript --output-dir $env:TEMP\jev-native-proof
+```
+
+`--capture-dir` also records sanitized `get_window_state` fixtures like the
+ones under `fixtures/native/`. On Linux, Cua Driver 0.30.2 and earlier report
+no `value` for a named text field
+([#4291](https://github.com/trycua/cua/issues/4291)). With those Drivers,
+`gtk3-save-note` cannot observe its own write, so it keeps offering the write
+until the step budget runs out.
The MCP connection stays open across the entire loop. This preserves the
explicit named Cua Driver session and avoids rebuilding tool state for every
diff --git a/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-after-choose-size-v1.json b/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-after-choose-size-v1.json
new file mode 100644
index 0000000000..652538a8a6
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-after-choose-size-v1.json
@@ -0,0 +1,245 @@
+{
+ "_fixture": {
+ "sanitized": "Recorded by verify_native.py --capture-dir. Screenshot bytes, tree_markdown, local file paths, and Driver guidance notes removed.",
+ "source": "CuaTestHarness task window (after-choose-size), observed by get_window_state: Linux GTK3 harness (CUA_GTK3_TASK_STATE), released Cua Driver 0.30.2 on GitHub-hosted ubuntu-latest (Xvfb, openbox, AT-SPI), run 36407837325"
+ },
+ "app_name": "Main.py",
+ "bounds_complete": true,
+ "capture_id": "capture_9e1278484c414cb5a46aec37a1927f02_0000000000000000",
+ "coordinate_frame": "window",
+ "element_count": 9,
+ "elements": [
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 0,
+ "element_token": "s00000001:0",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 100,
+ "x": 736,
+ "y": 428
+ },
+ "label": "Increment",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 100,
+ "x": 16,
+ "y": 41
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 1,
+ "element_token": "s00000001:1",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 71,
+ "x": 844,
+ "y": 428
+ },
+ "label": "Reset",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 71,
+ "x": 124,
+ "y": 41
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 2,
+ "element_index": 2,
+ "element_token": "s00000001:2",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 448,
+ "x": 736,
+ "y": 470
+ },
+ "label": "I agree",
+ "role": "check box",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 448,
+ "x": 16,
+ "y": 83
+ },
+ "selected": true
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 3,
+ "element_token": "s00000001:3",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 65,
+ "x": 736,
+ "y": 499
+ },
+ "label": "Small",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 65,
+ "x": 16,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 4,
+ "element_token": "s00000001:4",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 81,
+ "x": 813,
+ "y": 499
+ },
+ "label": "Medium",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 81,
+ "x": 93,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 5,
+ "element_token": "s00000001:5",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 64,
+ "x": 906,
+ "y": 499
+ },
+ "label": "Large",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 64,
+ "x": 186,
+ "y": 112
+ },
+ "selected": true
+ },
+ {
+ "actions": [
+ "activate"
+ ],
+ "depth": 3,
+ "element_index": 6,
+ "element_token": "s00000001:6",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 242,
+ "x": 736,
+ "y": 528
+ },
+ "label": "Note",
+ "role": "text",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 242,
+ "x": 16,
+ "y": 141
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 7,
+ "element_token": "s00000001:7",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 99,
+ "x": 986,
+ "y": 528
+ },
+ "label": "Save note",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 99,
+ "x": 266,
+ "y": 141
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 2,
+ "element_index": 8,
+ "element_token": "s00000001:8",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 59,
+ "x": 736,
+ "y": 570
+ },
+ "label": "Exit",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 59,
+ "x": 16,
+ "y": 183
+ }
+ }
+ ],
+ "elements_complete": true,
+ "nodes_pending": 0,
+ "nodes_visited": 9,
+ "pid": 3685,
+ "returned_element_count": 9,
+ "screenshot_height": 320,
+ "screenshot_mime_type": "image/png",
+ "screenshot_width": 480,
+ "snapshot_id": "s00000001",
+ "timeout_ms": 1000,
+ "total_element_count": 9,
+ "truncated": false,
+ "walk_elapsed_ms": 19,
+ "window_bounds": {
+ "height": 320,
+ "width": 480,
+ "x": 720,
+ "y": 387
+ },
+ "window_id": 6291459,
+ "window_title": "CuaTestHarness GTK3 Tasks"
+}
diff --git a/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-after-save-note-v1.json b/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-after-save-note-v1.json
new file mode 100644
index 0000000000..2f3db85a41
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-after-save-note-v1.json
@@ -0,0 +1,246 @@
+{
+ "_fixture": {
+ "limitation": "Cua Driver 0.30.2 omits value for a named AT-SPI text field (#4291), so the Note field reports no value after set_value.",
+ "sanitized": "Recorded by verify_native.py --capture-dir. Screenshot bytes, tree_markdown, local file paths, and Driver guidance notes removed.",
+ "source": "CuaTestHarness task window (after-save-note), observed by get_window_state: Linux GTK3 harness (CUA_GTK3_TASK_STATE), released Cua Driver 0.30.2 on GitHub-hosted ubuntu-latest (Xvfb, openbox, AT-SPI), run 36407837325"
+ },
+ "app_name": "Main.py",
+ "bounds_complete": true,
+ "capture_id": "capture_74215aca4b424fbcbef4158ddef43c44_0000000000000000",
+ "coordinate_frame": "window",
+ "element_count": 9,
+ "elements": [
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 0,
+ "element_token": "s00000001:0",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 100,
+ "x": 736,
+ "y": 428
+ },
+ "label": "Increment",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 100,
+ "x": 16,
+ "y": 41
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 1,
+ "element_token": "s00000001:1",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 71,
+ "x": 844,
+ "y": 428
+ },
+ "label": "Reset",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 71,
+ "x": 124,
+ "y": 41
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 2,
+ "element_index": 2,
+ "element_token": "s00000001:2",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 448,
+ "x": 736,
+ "y": 470
+ },
+ "label": "I agree",
+ "role": "check box",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 448,
+ "x": 16,
+ "y": 83
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 3,
+ "element_token": "s00000001:3",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 65,
+ "x": 736,
+ "y": 499
+ },
+ "label": "Small",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 65,
+ "x": 16,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 4,
+ "element_token": "s00000001:4",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 81,
+ "x": 813,
+ "y": 499
+ },
+ "label": "Medium",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 81,
+ "x": 93,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 5,
+ "element_token": "s00000001:5",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 64,
+ "x": 906,
+ "y": 499
+ },
+ "label": "Large",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 64,
+ "x": 186,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "activate"
+ ],
+ "depth": 3,
+ "element_index": 6,
+ "element_token": "s00000001:6",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 242,
+ "x": 736,
+ "y": 528
+ },
+ "label": "Note",
+ "role": "text",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 242,
+ "x": 16,
+ "y": 141
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 7,
+ "element_token": "s00000001:7",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 99,
+ "x": 986,
+ "y": 528
+ },
+ "label": "Save note",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 99,
+ "x": 266,
+ "y": 141
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 2,
+ "element_index": 8,
+ "element_token": "s00000001:8",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 59,
+ "x": 736,
+ "y": 570
+ },
+ "label": "Exit",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 59,
+ "x": 16,
+ "y": 183
+ }
+ }
+ ],
+ "elements_complete": true,
+ "nodes_pending": 0,
+ "nodes_visited": 9,
+ "pid": 3961,
+ "returned_element_count": 9,
+ "screenshot_height": 320,
+ "screenshot_mime_type": "image/png",
+ "screenshot_width": 480,
+ "snapshot_id": "s00000001",
+ "timeout_ms": 1000,
+ "total_element_count": 9,
+ "truncated": false,
+ "walk_elapsed_ms": 18,
+ "window_bounds": {
+ "height": 320,
+ "width": 480,
+ "x": 720,
+ "y": 387
+ },
+ "window_id": 6291459,
+ "window_title": "CuaTestHarness GTK3 Tasks"
+}
diff --git a/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-initial-v1.json b/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-initial-v1.json
new file mode 100644
index 0000000000..62bdcff9f0
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/native/gtk3-window-state-initial-v1.json
@@ -0,0 +1,245 @@
+{
+ "_fixture": {
+ "sanitized": "Recorded by verify_native.py --capture-dir. Screenshot bytes, tree_markdown, local file paths, and Driver guidance notes removed.",
+ "source": "CuaTestHarness task window (initial), observed by get_window_state: Linux GTK3 harness (CUA_GTK3_TASK_STATE), released Cua Driver 0.30.2 on GitHub-hosted ubuntu-latest (Xvfb, openbox, AT-SPI), run 36407837325"
+ },
+ "app_name": "Main.py",
+ "bounds_complete": true,
+ "capture_id": "capture_4d841676244945c8afd21a666254f4e1_0000000000000000",
+ "coordinate_frame": "window",
+ "element_count": 9,
+ "elements": [
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 0,
+ "element_token": "s00000001:0",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 100,
+ "x": 736,
+ "y": 428
+ },
+ "label": "Increment",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 100,
+ "x": 16,
+ "y": 41
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 1,
+ "element_token": "s00000001:1",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 71,
+ "x": 844,
+ "y": 428
+ },
+ "label": "Reset",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 71,
+ "x": 124,
+ "y": 41
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 2,
+ "element_index": 2,
+ "element_token": "s00000001:2",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 448,
+ "x": 736,
+ "y": 470
+ },
+ "label": "I agree",
+ "role": "check box",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 448,
+ "x": 16,
+ "y": 83
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 3,
+ "element_token": "s00000001:3",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 65,
+ "x": 736,
+ "y": 499
+ },
+ "label": "Small",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 65,
+ "x": 16,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 4,
+ "element_token": "s00000001:4",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 81,
+ "x": 813,
+ "y": 499
+ },
+ "label": "Medium",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 81,
+ "x": 93,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 5,
+ "element_token": "s00000001:5",
+ "enabled": true,
+ "frame": {
+ "h": 21,
+ "w": 64,
+ "x": 906,
+ "y": 499
+ },
+ "label": "Large",
+ "role": "radio button",
+ "screenshot_frame": {
+ "h": 21,
+ "w": 64,
+ "x": 186,
+ "y": 112
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "activate"
+ ],
+ "depth": 3,
+ "element_index": 6,
+ "element_token": "s00000001:6",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 242,
+ "x": 736,
+ "y": 528
+ },
+ "label": "Note",
+ "role": "text",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 242,
+ "x": 16,
+ "y": 141
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 3,
+ "element_index": 7,
+ "element_token": "s00000001:7",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 99,
+ "x": 986,
+ "y": 528
+ },
+ "label": "Save note",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 99,
+ "x": 266,
+ "y": 141
+ }
+ },
+ {
+ "actions": [
+ "click"
+ ],
+ "depth": 2,
+ "element_index": 8,
+ "element_token": "s00000001:8",
+ "enabled": true,
+ "frame": {
+ "h": 34,
+ "w": 59,
+ "x": 736,
+ "y": 570
+ },
+ "label": "Exit",
+ "role": "push button",
+ "screenshot_frame": {
+ "h": 34,
+ "w": 59,
+ "x": 16,
+ "y": 183
+ }
+ }
+ ],
+ "elements_complete": true,
+ "nodes_pending": 0,
+ "nodes_visited": 9,
+ "pid": 3801,
+ "returned_element_count": 9,
+ "screenshot_height": 320,
+ "screenshot_mime_type": "image/png",
+ "screenshot_width": 480,
+ "snapshot_id": "s00000001",
+ "timeout_ms": 1000,
+ "total_element_count": 9,
+ "truncated": false,
+ "walk_elapsed_ms": 17,
+ "window_bounds": {
+ "height": 320,
+ "width": 480,
+ "x": 720,
+ "y": 387
+ },
+ "window_id": 6291459,
+ "window_title": "CuaTestHarness GTK3 Tasks"
+}
diff --git a/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-after-choose-size-v1.json b/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-after-choose-size-v1.json
new file mode 100644
index 0000000000..258eab4690
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-after-choose-size-v1.json
@@ -0,0 +1,366 @@
+{
+ "_fixture": {
+ "sanitized": "Recorded by verify_native.py --capture-dir. Screenshot bytes, tree_markdown, local file paths, and Driver guidance notes removed.",
+ "source": "CuaTestHarness task window (after-choose-size), observed by get_window_state: Windows WPF harness (CUA_WPF_TASK_STATE), released Cua Driver 0.30.2 on GitHub-hosted windows-latest, run 36407837325"
+ },
+ "app_name": "CuaTestHarness.Wpf.exe",
+ "capture_id": "capture_c3e1f7c2291445129574f107f1ccfeb1_0000000000000000",
+ "element_count": 14,
+ "elements": [
+ {
+ "actions": [
+ "set_value"
+ ],
+ "depth": 1,
+ "element_index": 0,
+ "element_token": "s00000001:0",
+ "enabled": true,
+ "frame": {
+ "h": 28,
+ "w": 448,
+ "x": 296,
+ "y": 183
+ },
+ "label": "CuaTestHarness WPF Tasks",
+ "role": "TitleBar",
+ "screenshot_frame": {
+ "h": 28,
+ "w": 448,
+ "x": 16,
+ "y": 2
+ },
+ "value": "CuaTestHarness WPF Tasks"
+ },
+ {
+ "actions": [
+ "invoke",
+ "expand"
+ ],
+ "depth": 3,
+ "element_index": 1,
+ "element_token": "s00000001:1",
+ "enabled": true,
+ "frame": {
+ "h": 22,
+ "w": 22,
+ "x": 280,
+ "y": 188
+ },
+ "label": "System",
+ "parent_index": 0,
+ "role": "MenuItem",
+ "screenshot_frame": {
+ "h": 22,
+ "w": 22,
+ "x": 0,
+ "y": 7
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 2,
+ "element_token": "s00000001:2",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 47,
+ "x": 605,
+ "y": 181
+ },
+ "label": "Minimize",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 47,
+ "x": 325,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 3,
+ "element_token": "s00000001:3",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 46,
+ "x": 652,
+ "y": 181
+ },
+ "label": "Maximize",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 46,
+ "x": 372,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 4,
+ "element_token": "s00000001:4",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 47,
+ "x": 698,
+ "y": 181
+ },
+ "label": "Close",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 47,
+ "x": 418,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 5,
+ "element_token": "s00000001:5",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 296,
+ "y": 251
+ },
+ "label": "Increment",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 16,
+ "y": 70
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 6,
+ "element_token": "s00000001:6",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 394,
+ "y": 251
+ },
+ "label": "Reset",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 114,
+ "y": 70
+ }
+ },
+ {
+ "actions": [
+ "toggle"
+ ],
+ "depth": 1,
+ "element_index": 7,
+ "element_token": "s00000001:7",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 432,
+ "x": 296,
+ "y": 279
+ },
+ "label": "I agree",
+ "role": "CheckBox",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 432,
+ "x": 16,
+ "y": 98
+ },
+ "selected": true
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 8,
+ "element_token": "s00000001:8",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 48,
+ "x": 296,
+ "y": 302
+ },
+ "label": "Small",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 48,
+ "x": 16,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 9,
+ "element_token": "s00000001:9",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 63,
+ "x": 356,
+ "y": 302
+ },
+ "label": "Medium",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 63,
+ "x": 76,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 10,
+ "element_token": "s00000001:10",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 48,
+ "x": 431,
+ "y": 302
+ },
+ "label": "Large",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 48,
+ "x": 151,
+ "y": 121
+ },
+ "selected": true
+ },
+ {
+ "actions": [
+ "set_value",
+ "text",
+ "scroll"
+ ],
+ "depth": 1,
+ "element_index": 11,
+ "element_token": "s00000001:11",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 240,
+ "x": 296,
+ "y": 325
+ },
+ "label": "Note",
+ "role": "Edit",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 240,
+ "x": 16,
+ "y": 144
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 12,
+ "element_token": "s00000001:12",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 536,
+ "y": 325
+ },
+ "label": "Save note",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 256,
+ "y": 144
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 13,
+ "element_token": "s00000001:13",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 296,
+ "y": 345
+ },
+ "label": "Exit",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 16,
+ "y": 164
+ }
+ }
+ ],
+ "elements_complete": false,
+ "nodes_pending": 0,
+ "nodes_visited": 25,
+ "pid": 7376,
+ "returned_element_count": 14,
+ "screenshot_height": 351,
+ "screenshot_mime_type": "image/png",
+ "screenshot_width": 464,
+ "snapshot_id": "s00000001",
+ "timeout_ms": 1000,
+ "total_element_count": 14,
+ "truncated": false,
+ "walk_elapsed_ms": 0,
+ "window_bounds": {
+ "height": 353,
+ "width": 466,
+ "x": 279,
+ "y": 180
+ },
+ "window_id": 131470,
+ "window_title": "CuaTestHarness WPF Tasks"
+}
diff --git a/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-after-save-note-v1.json b/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-after-save-note-v1.json
new file mode 100644
index 0000000000..fef24881c0
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-after-save-note-v1.json
@@ -0,0 +1,367 @@
+{
+ "_fixture": {
+ "sanitized": "Recorded by verify_native.py --capture-dir. Screenshot bytes, tree_markdown, local file paths, and Driver guidance notes removed.",
+ "source": "CuaTestHarness task window (after-save-note), observed by get_window_state: Windows WPF harness (CUA_WPF_TASK_STATE), released Cua Driver 0.30.2 on GitHub-hosted windows-latest, run 36407837325"
+ },
+ "app_name": "CuaTestHarness.Wpf.exe",
+ "capture_id": "capture_a2b17856111448d990cb1674a031fffa_0000000000000000",
+ "element_count": 14,
+ "elements": [
+ {
+ "actions": [
+ "set_value"
+ ],
+ "depth": 1,
+ "element_index": 0,
+ "element_token": "s00000001:0",
+ "enabled": true,
+ "frame": {
+ "h": 28,
+ "w": 448,
+ "x": 296,
+ "y": 183
+ },
+ "label": "CuaTestHarness WPF Tasks",
+ "role": "TitleBar",
+ "screenshot_frame": {
+ "h": 28,
+ "w": 448,
+ "x": 16,
+ "y": 2
+ },
+ "value": "CuaTestHarness WPF Tasks"
+ },
+ {
+ "actions": [
+ "invoke",
+ "expand"
+ ],
+ "depth": 3,
+ "element_index": 1,
+ "element_token": "s00000001:1",
+ "enabled": true,
+ "frame": {
+ "h": 22,
+ "w": 22,
+ "x": 280,
+ "y": 188
+ },
+ "label": "System",
+ "parent_index": 0,
+ "role": "MenuItem",
+ "screenshot_frame": {
+ "h": 22,
+ "w": 22,
+ "x": 0,
+ "y": 7
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 2,
+ "element_token": "s00000001:2",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 47,
+ "x": 605,
+ "y": 181
+ },
+ "label": "Minimize",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 47,
+ "x": 325,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 3,
+ "element_token": "s00000001:3",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 46,
+ "x": 652,
+ "y": 181
+ },
+ "label": "Maximize",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 46,
+ "x": 372,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 4,
+ "element_token": "s00000001:4",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 47,
+ "x": 698,
+ "y": 181
+ },
+ "label": "Close",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 47,
+ "x": 418,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 5,
+ "element_token": "s00000001:5",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 296,
+ "y": 251
+ },
+ "label": "Increment",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 16,
+ "y": 70
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 6,
+ "element_token": "s00000001:6",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 394,
+ "y": 251
+ },
+ "label": "Reset",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 114,
+ "y": 70
+ }
+ },
+ {
+ "actions": [
+ "toggle"
+ ],
+ "depth": 1,
+ "element_index": 7,
+ "element_token": "s00000001:7",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 432,
+ "x": 296,
+ "y": 279
+ },
+ "label": "I agree",
+ "role": "CheckBox",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 432,
+ "x": 16,
+ "y": 98
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 8,
+ "element_token": "s00000001:8",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 48,
+ "x": 296,
+ "y": 302
+ },
+ "label": "Small",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 48,
+ "x": 16,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 9,
+ "element_token": "s00000001:9",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 63,
+ "x": 356,
+ "y": 302
+ },
+ "label": "Medium",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 63,
+ "x": 76,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 10,
+ "element_token": "s00000001:10",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 48,
+ "x": 431,
+ "y": 302
+ },
+ "label": "Large",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 48,
+ "x": 151,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "set_value",
+ "text",
+ "scroll"
+ ],
+ "depth": 1,
+ "element_index": 11,
+ "element_token": "s00000001:11",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 240,
+ "x": 296,
+ "y": 325
+ },
+ "label": "Note",
+ "role": "Edit",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 240,
+ "x": 16,
+ "y": 144
+ },
+ "value": "jev-use native note"
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 12,
+ "element_token": "s00000001:12",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 536,
+ "y": 325
+ },
+ "label": "Save note",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 256,
+ "y": 144
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 13,
+ "element_token": "s00000001:13",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 296,
+ "y": 345
+ },
+ "label": "Exit",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 16,
+ "y": 164
+ }
+ }
+ ],
+ "elements_complete": false,
+ "nodes_pending": 0,
+ "nodes_visited": 25,
+ "pid": 7844,
+ "returned_element_count": 14,
+ "screenshot_height": 351,
+ "screenshot_mime_type": "image/png",
+ "screenshot_width": 464,
+ "snapshot_id": "s00000001",
+ "timeout_ms": 1000,
+ "total_element_count": 14,
+ "truncated": false,
+ "walk_elapsed_ms": 0,
+ "window_bounds": {
+ "height": 353,
+ "width": 466,
+ "x": 279,
+ "y": 180
+ },
+ "window_id": 262524,
+ "window_title": "CuaTestHarness WPF Tasks"
+}
diff --git a/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-initial-v1.json b/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-initial-v1.json
new file mode 100644
index 0000000000..d68fec310f
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/fixtures/native/wpf-window-state-initial-v1.json
@@ -0,0 +1,366 @@
+{
+ "_fixture": {
+ "sanitized": "Recorded by verify_native.py --capture-dir. Screenshot bytes, tree_markdown, local file paths, and Driver guidance notes removed.",
+ "source": "CuaTestHarness task window (initial), observed by get_window_state: Windows WPF harness (CUA_WPF_TASK_STATE), released Cua Driver 0.30.2 on GitHub-hosted windows-latest, run 36407837325"
+ },
+ "app_name": "CuaTestHarness.Wpf.exe",
+ "capture_id": "capture_6fde53aab7ec4f3882e226cbf5486fa4_0000000000000000",
+ "element_count": 14,
+ "elements": [
+ {
+ "actions": [
+ "set_value"
+ ],
+ "depth": 1,
+ "element_index": 0,
+ "element_token": "s00000001:0",
+ "enabled": true,
+ "frame": {
+ "h": 28,
+ "w": 448,
+ "x": 296,
+ "y": 183
+ },
+ "label": "CuaTestHarness WPF Tasks",
+ "role": "TitleBar",
+ "screenshot_frame": {
+ "h": 28,
+ "w": 448,
+ "x": 16,
+ "y": 2
+ },
+ "value": "CuaTestHarness WPF Tasks"
+ },
+ {
+ "actions": [
+ "invoke",
+ "expand"
+ ],
+ "depth": 3,
+ "element_index": 1,
+ "element_token": "s00000001:1",
+ "enabled": true,
+ "frame": {
+ "h": 22,
+ "w": 22,
+ "x": 280,
+ "y": 188
+ },
+ "label": "System",
+ "parent_index": 0,
+ "role": "MenuItem",
+ "screenshot_frame": {
+ "h": 22,
+ "w": 22,
+ "x": 0,
+ "y": 7
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 2,
+ "element_token": "s00000001:2",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 47,
+ "x": 605,
+ "y": 181
+ },
+ "label": "Minimize",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 47,
+ "x": 325,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 3,
+ "element_token": "s00000001:3",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 46,
+ "x": 652,
+ "y": 181
+ },
+ "label": "Maximize",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 46,
+ "x": 372,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 2,
+ "element_index": 4,
+ "element_token": "s00000001:4",
+ "enabled": true,
+ "frame": {
+ "h": 30,
+ "w": 47,
+ "x": 698,
+ "y": 181
+ },
+ "label": "Close",
+ "parent_index": 0,
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 30,
+ "w": 47,
+ "x": 418,
+ "y": 0
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 5,
+ "element_token": "s00000001:5",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 296,
+ "y": 251
+ },
+ "label": "Increment",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 16,
+ "y": 70
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 6,
+ "element_token": "s00000001:6",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 394,
+ "y": 251
+ },
+ "label": "Reset",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 114,
+ "y": 70
+ }
+ },
+ {
+ "actions": [
+ "toggle"
+ ],
+ "depth": 1,
+ "element_index": 7,
+ "element_token": "s00000001:7",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 432,
+ "x": 296,
+ "y": 279
+ },
+ "label": "I agree",
+ "role": "CheckBox",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 432,
+ "x": 16,
+ "y": 98
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 8,
+ "element_token": "s00000001:8",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 48,
+ "x": 296,
+ "y": 302
+ },
+ "label": "Small",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 48,
+ "x": 16,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 9,
+ "element_token": "s00000001:9",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 63,
+ "x": 356,
+ "y": 302
+ },
+ "label": "Medium",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 63,
+ "x": 76,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "select"
+ ],
+ "depth": 1,
+ "element_index": 10,
+ "element_token": "s00000001:10",
+ "enabled": true,
+ "frame": {
+ "h": 15,
+ "w": 48,
+ "x": 431,
+ "y": 302
+ },
+ "label": "Large",
+ "role": "RadioButton",
+ "screenshot_frame": {
+ "h": 15,
+ "w": 48,
+ "x": 151,
+ "y": 121
+ },
+ "selected": false
+ },
+ {
+ "actions": [
+ "set_value",
+ "text",
+ "scroll"
+ ],
+ "depth": 1,
+ "element_index": 11,
+ "element_token": "s00000001:11",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 240,
+ "x": 296,
+ "y": 325
+ },
+ "label": "Note",
+ "role": "Edit",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 240,
+ "x": 16,
+ "y": 144
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 12,
+ "element_token": "s00000001:12",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 536,
+ "y": 325
+ },
+ "label": "Save note",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 256,
+ "y": 144
+ }
+ },
+ {
+ "actions": [
+ "invoke"
+ ],
+ "depth": 1,
+ "element_index": 13,
+ "element_token": "s00000001:13",
+ "enabled": true,
+ "frame": {
+ "h": 20,
+ "w": 90,
+ "x": 296,
+ "y": 345
+ },
+ "label": "Exit",
+ "role": "Button",
+ "screenshot_frame": {
+ "h": 20,
+ "w": 90,
+ "x": 16,
+ "y": 164
+ }
+ }
+ ],
+ "elements_complete": false,
+ "nodes_pending": 0,
+ "nodes_visited": 25,
+ "pid": 6544,
+ "returned_element_count": 14,
+ "screenshot_height": 351,
+ "screenshot_mime_type": "image/png",
+ "screenshot_width": 464,
+ "snapshot_id": "s00000001",
+ "timeout_ms": 1000,
+ "total_element_count": 14,
+ "truncated": false,
+ "walk_elapsed_ms": 0,
+ "window_bounds": {
+ "height": 353,
+ "width": 466,
+ "x": 279,
+ "y": 180
+ },
+ "window_id": 328014,
+ "window_title": "CuaTestHarness WPF Tasks"
+}
diff --git a/libs/cua-driver/examples/jev-use/python/capture_window_state.py b/libs/cua-driver/examples/jev-use/python/capture_window_state.py
new file mode 100644
index 0000000000..ce4d51e1e1
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/capture_window_state.py
@@ -0,0 +1,78 @@
+"""Record a sanitized ``get_window_state`` fixture for the native unit tests.
+
+The native source's role table and element rules are tested against real
+Driver output from each harness (RFC #4268). This helper attaches to a running
+harness (``--pid``), finds its task window by title, takes one observation
+with the tree and screenshot together, exactly as ``run_native.py`` does, and
+writes the structured result without screenshot bytes, ``tree_markdown``, or
+local file paths. ``verify_native.py --capture-dir`` calls it around each run.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+
+from driver_env import driver_environment
+from run import Driver
+from run_native import find_window
+
+DROPPED_KEYS = frozenset({"tree_markdown", "screenshot_file_path", "screenshot_out_file"})
+MAX_STRING_CHARS = 4096
+
+
+def sanitize(payload: dict[str, Any], source: str) -> dict[str, Any]:
+ result: dict[str, Any] = {
+ "_fixture": {
+ "source": source,
+ "sanitized": "Screenshot bytes, tree_markdown, and local file paths removed.",
+ }
+ }
+ for key, value in payload.items():
+ if key in DROPPED_KEYS or (isinstance(value, str) and len(value) > MAX_STRING_CHARS):
+ continue
+ result[key] = value
+ return result
+
+
+async def capture(pid: int, title: str, source: str) -> dict[str, Any]:
+ params = StdioServerParameters(
+ command=os.getenv("CUA_DRIVER_BIN", "cua-driver"), args=["mcp"], env=driver_environment()
+ )
+ async with stdio_client(params) as (read, write):
+ async with ClientSession(read, write) as session:
+ await session.initialize()
+ driver = Driver(session, "jev-native-capture")
+ window = await find_window(driver, pid, title)
+ payload = await driver.call(
+ "get_window_state",
+ {
+ "pid": pid,
+ "window_id": int(window["window_id"]),
+ "include_accessibility_tree": True,
+ "include_screenshot": True,
+ },
+ )
+ return sanitize(payload, source)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--pid", type=int, required=True)
+ parser.add_argument("--title", required=True, help="exact task window title")
+ parser.add_argument("--source", required=True, help="fixture provenance, e.g. harness and Driver version")
+ parser.add_argument("--output", type=Path, required=True)
+ args = parser.parse_args()
+ state = asyncio.run(capture(args.pid, args.title, args.source))
+ args.output.write_text(json.dumps(state, indent=1, sort_keys=True) + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/libs/cua-driver/examples/jev-use/python/native.py b/libs/cua-driver/examples/jev-use/python/native.py
index 702afb74ed..be06daf588 100644
--- a/libs/cua-driver/examples/jev-use/python/native.py
+++ b/libs/cua-driver/examples/jev-use/python/native.py
@@ -12,7 +12,8 @@ call would publish a new snapshot and invalidate every token in the set.
Element rules (all must hold):
-1. the raw role maps to a role class (``native_roles.py``);
+1. the raw role maps to a role class (``native_roles.py``), and no actionable
+ ancestor is window chrome (the Windows title bar);
2. ``enabled`` is not ``false``;
3. ``frame`` exists, has positive size, and intersects ``window_bounds``;
4. the redacted label is non-empty, is not a copy of the element's own value,
@@ -32,7 +33,15 @@ import re
from dataclasses import dataclass
from typing import Any, Callable, Literal, Mapping
-from native_roles import ROLE_CLASS_ACTION, ActionKind, Platform, RoleClass, normalized_role, role_class
+from native_roles import (
+ ROLE_CLASS_ACTION,
+ ActionKind,
+ Platform,
+ RoleClass,
+ is_window_chrome,
+ normalized_role,
+ role_class,
+)
ID_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,63}\Z")
PARAMETER_NAME_PATTERN = re.compile(r"[a-z][a-z0-9_]{0,7}\Z")
@@ -214,7 +223,7 @@ class NativeControl:
ExclusionReason = Literal[
- "unknown_role", "disabled", "off_screen", "unlabeled", "web_content", "no_token"
+ "unknown_role", "window_chrome", "disabled", "off_screen", "unlabeled", "web_content", "no_token"
]
@@ -258,12 +267,25 @@ def eligible_controls(
parent = ancestor.get("parent_index")
return tuple(reversed(path))
+ def in_window_chrome(element: Mapping[str, Any]) -> bool:
+ seen: set[int] = set()
+ parent = element.get("parent_index")
+ while isinstance(parent, int) and parent in by_index and parent not in seen:
+ seen.add(parent)
+ if is_window_chrome(by_index[parent].get("role"), platform):
+ return True
+ parent = by_index[parent].get("parent_index")
+ return False
+
pending: list[tuple[Mapping[str, Any], RoleClass, str, tuple[tuple[str, str], ...]]] = []
for element in observation.elements:
klass = role_class(element.get("role"), platform)
if klass is None:
exclude("unknown_role")
continue
+ if in_window_chrome(element):
+ exclude("window_chrome")
+ continue
if element.get("enabled") is False:
exclude("disabled")
continue
diff --git a/libs/cua-driver/examples/jev-use/python/native_roles.py b/libs/cua-driver/examples/jev-use/python/native_roles.py
index c38223c006..315335b9f8 100644
--- a/libs/cua-driver/examples/jev-use/python/native_roles.py
+++ b/libs/cua-driver/examples/jev-use/python/native_roles.py
@@ -127,6 +127,23 @@ ROLE_TABLES: Mapping[Platform, Mapping[str, RoleClass]] = {
}
+# Window-chrome containers, by normalized role. Their actionable descendants
+# (Windows' title-bar System menu, Minimize, Maximize, and Close) belong to
+# the window manager, not the application, so they are never candidates. On
+# macOS the AppKit fixtures carry no such container, and on Linux the window
+# decorations are not part of the application's AT-SPI tree.
+WINDOW_CHROME_ROLES: Mapping[Platform, frozenset[str]] = {
+ "macos": frozenset(),
+ "windows": frozenset({normalized_role("TitleBar")}),
+ "linux": frozenset(),
+}
+
+
+def is_window_chrome(raw_role: object, platform: Platform) -> bool:
+ """Whether a raw Driver role is a window-chrome container on ``platform``."""
+ return isinstance(raw_role, str) and normalized_role(raw_role) in WINDOW_CHROME_ROLES[platform]
+
+
def role_class(raw_role: object, platform: Platform) -> RoleClass | None:
"""Return the role class for a raw Driver role, or ``None`` when excluded."""
if not isinstance(raw_role, str) or not raw_role:
diff --git a/libs/cua-driver/examples/jev-use/python/native_tasks.py b/libs/cua-driver/examples/jev-use/python/native_tasks.py
index 839b92bd36..f26b5e0518 100644
--- a/libs/cua-driver/examples/jev-use/python/native_tasks.py
+++ b/libs/cua-driver/examples/jev-use/python/native_tasks.py
@@ -7,10 +7,11 @@ a step budget, and an app-owned success oracle. It builds each step's closed
candidate set from the native accessibility source, plus visual regions only
under the fallback rule, and composes them in the fixed order page, ax, visual.
-The built-in tasks drive the repository's AppKit harness
-(``libs/cua-driver/tests/fixtures/apps/macos/appkit``) launched with
-``CUA_APPKIT_TASK_STATE``. Their oracle is the harness's own JSON state file,
-which the app rewrites on every change; it never depends on Driver output.
+The built-in tasks drive the repository's harness applications in task mode:
+AppKit (``CUA_APPKIT_TASK_STATE``), WPF (``CUA_WPF_TASK_STATE``), and GTK3
+(``CUA_GTK3_TASK_STATE``), under ``libs/cua-driver/tests/fixtures/apps``.
+Their oracle is the harness's own JSON state file, which the app rewrites on
+every change; it never depends on Driver output.
"""
from __future__ import annotations
@@ -476,15 +477,55 @@ def native_choice_request(
}
-# -- AppKit harness tasks ---------------------------------------------------
+# -- Harness tasks ------------------------------------------------------------
+#
+# The same three tasks run on every repository harness that has a task mode:
+# AppKit (macOS AX), WPF (Windows UIA), and GTK3 (Linux AT-SPI). Each harness
+# shows the same labeled controls in task mode (Increment, Reset, I agree,
+# Small/Medium/Large, Note, Save note, Exit) and rewrites the same app-owned
+# JSON state file, so the task semantics, candidate IDs, and mock choices are
+# identical across platforms. Only the window, the state schema, and the
+# platform role table differ.
-APPKIT_WINDOW_TITLE = "CuaTestHarness AppKit"
-APPKIT_BUNDLE_ID = "com.trycua.harness.appkit"
-APPKIT_STATE_SCHEMA = "cua.appkit_task_state_v1"
-APPKIT_STATE_ENV = "CUA_APPKIT_TASK_STATE"
+
+@dataclass(frozen=True)
+class HarnessSpec:
+ """A repository harness application that has a jev-use task mode."""
+
+ name: str
+ platform: str
+ window_title: str
+ state_schema: str
+ state_env: str
+ bundle_id: str | None = None
+ process_name: str | None = None
+
+
+HARNESSES: Mapping[str, HarnessSpec] = {
+ "appkit": HarnessSpec(
+ "appkit", "macos", "CuaTestHarness AppKit", "cua.appkit_task_state_v1",
+ "CUA_APPKIT_TASK_STATE", bundle_id="com.trycua.harness.appkit",
+ ),
+ # WPF and GTK3 show a dedicated task window in task mode: their ordinary
+ # main windows scroll, so most controls would be off screen (and excluded).
+ "wpf": HarnessSpec(
+ "wpf", "windows", "CuaTestHarness WPF Tasks", "cua.wpf_task_state_v1",
+ "CUA_WPF_TASK_STATE", process_name="CuaTestHarness.Wpf",
+ ),
+ "gtk3": HarnessSpec(
+ "gtk3", "linux", "CuaTestHarness GTK3 Tasks", "cua.gtk3_task_state_v1",
+ "CUA_GTK3_TASK_STATE", process_name="python3",
+ ),
+}
+
+APPKIT_WINDOW_TITLE = HARNESSES["appkit"].window_title
+APPKIT_BUNDLE_ID = HARNESSES["appkit"].bundle_id
+APPKIT_STATE_SCHEMA = HARNESSES["appkit"].state_schema
+APPKIT_STATE_ENV = HARNESSES["appkit"].state_env
COUNTER_TARGET = 3
DEFAULT_NOTE_TEXT = "jev-use native note"
TARGET_SIZE = "large"
+TASK_KINDS = ("counter", "save-note", "choose-size")
def _counter_check(state: Mapping[str, Any]) -> Check:
@@ -514,10 +555,23 @@ def _size_check(state: Mapping[str, Any]) -> Check:
return "pending"
-APPKIT_TASK_IDS = ("appkit-counter", "appkit-save-note", "appkit-choose-size")
+def harness_task_ids(harness: str) -> tuple[str, ...]:
+ return tuple(f"{harness}-{kind}" for kind in TASK_KINDS)
-def appkit_task(
+APPKIT_TASK_IDS = harness_task_ids("appkit")
+NATIVE_TASK_IDS = tuple(task for name in HARNESSES for task in harness_task_ids(name))
+
+
+def split_task_id(task_id: str) -> tuple[HarnessSpec, str]:
+ """Return the harness and task kind of a built-in task ID."""
+ harness, _, kind = task_id.partition("-")
+ if harness not in HARNESSES or kind not in TASK_KINDS:
+ raise ValueError(f"unknown native task: {task_id}")
+ return HARNESSES[harness], kind
+
+
+def native_task(
task_id: str,
state_path: Path,
*,
@@ -525,10 +579,13 @@ def appkit_task(
note_text: str = DEFAULT_NOTE_TEXT,
allow_foreground: bool = False,
) -> NativeTask:
- """Build one AppKit harness task bound to its app-owned state file."""
- oracle = AppStateOracle(Path(state_path), APPKIT_STATE_SCHEMA, pid)
- scope = WindowScope(APPKIT_WINDOW_TITLE, bundle_id=APPKIT_BUNDLE_ID)
- if task_id == "appkit-counter":
+ """Build one harness task bound to its app-owned state file."""
+ harness, kind = split_task_id(task_id)
+ oracle = AppStateOracle(Path(state_path), harness.state_schema, pid)
+ scope = WindowScope(
+ harness.window_title, bundle_id=harness.bundle_id, process_name=harness.process_name
+ )
+ if kind == "counter":
return NativeTask(
id=task_id,
goal=(
@@ -543,7 +600,7 @@ def appkit_task(
max_steps=COUNTER_TARGET + 3,
mock_preferences=("ax:button:increment",),
)
- if task_id == "appkit-save-note":
+ if kind == "save-note":
return NativeTask(
id=task_id,
goal="Enter the note text into the Note field, then save the note.",
@@ -556,16 +613,30 @@ def appkit_task(
max_steps=5,
mock_preferences=("ax:text_input:note:set:note", "ax:button:save-note"),
)
- if task_id == "appkit-choose-size":
- return NativeTask(
- id=task_id,
- goal="Choose the Large size option and check the I agree checkbox.",
- scope=scope,
- allowed_actions=frozenset({"select", "toggle"}),
- oracle=oracle,
- check=_size_check,
- allow_foreground=allow_foreground,
- max_steps=5,
- mock_preferences=("ax:radio:large", "ax:checkbox:i-agree"),
- )
- raise ValueError(f"unknown AppKit task: {task_id}")
+ return NativeTask(
+ id=task_id,
+ goal="Choose the Large size option and check the I agree checkbox.",
+ scope=scope,
+ allowed_actions=frozenset({"select", "toggle"}),
+ oracle=oracle,
+ check=_size_check,
+ allow_foreground=allow_foreground,
+ max_steps=5,
+ mock_preferences=("ax:radio:large", "ax:checkbox:i-agree"),
+ )
+
+
+def appkit_task(
+ task_id: str,
+ state_path: Path,
+ *,
+ pid: int | None = None,
+ note_text: str = DEFAULT_NOTE_TEXT,
+ allow_foreground: bool = False,
+) -> NativeTask:
+ """Build one AppKit harness task (kept for Phase 1 callers)."""
+ if task_id not in APPKIT_TASK_IDS:
+ raise ValueError(f"unknown AppKit task: {task_id}")
+ return native_task(
+ task_id, state_path, pid=pid, note_text=note_text, allow_foreground=allow_foreground
+ )
diff --git a/libs/cua-driver/examples/jev-use/python/run_native.py b/libs/cua-driver/examples/jev-use/python/run_native.py
index 23bf42042b..8dbdca5c9f 100644
--- a/libs/cua-driver/examples/jev-use/python/run_native.py
+++ b/libs/cua-driver/examples/jev-use/python/run_native.py
@@ -7,9 +7,10 @@ the closed candidate set, send a validated ``cua.jev_choice_request_v2`` to
the chooser, dispatch the one selected element-bound action, and read the
oracle again. The chooser only ever selects a supplied ID.
-Built-in tasks drive the AppKit harness launched with
-``CUA_APPKIT_TASK_STATE=<path>``; ``verify_native.py`` launches it, runs this
-runner, and checks the state file independently.
+Built-in tasks drive the AppKit, WPF, or GTK3 harness launched in task mode
+(``CUA_APPKIT_TASK_STATE``, ``CUA_WPF_TASK_STATE``, or ``CUA_GTK3_TASK_STATE``
+set to ``<path>``); ``verify_native.py`` launches it, runs this runner, and
+checks the state file independently.
"""
from __future__ import annotations
@@ -35,10 +36,10 @@ from jev_adapter import choose_mock_for_task
from native import NativeObservation, NativeObservationError
from native_roles import Platform
from native_tasks import (
- APPKIT_TASK_IDS,
+ NATIVE_TASK_IDS,
NativeTask,
OracleError,
- appkit_task,
+ native_task,
native_choice_request,
visual_fallback_reason,
)
@@ -350,11 +351,11 @@ async def run_task(args: argparse.Namespace, task: NativeTask) -> str:
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--task", choices=APPKIT_TASK_IDS, required=True)
+ parser.add_argument("--task", choices=NATIVE_TASK_IDS, required=True)
parser.add_argument("--provider", choices=("mock", "live"), default="mock")
parser.add_argument("--pid", type=int, required=True, help="the running harness process")
- parser.add_argument("--state-file", required=True, help="the harness CUA_APPKIT_TASK_STATE path")
- parser.add_argument("--note-text", help="note text for appkit-save-note")
+ parser.add_argument("--state-file", required=True, help="the harness task-state file path")
+ parser.add_argument("--note-text", help="note text for the save-note tasks")
parser.add_argument("--allow-foreground", action="store_true")
parser.add_argument("--platform", choices=("macos", "windows", "linux"))
parser.add_argument("--log", help="optional JSONL output path")
@@ -365,7 +366,7 @@ def task_from_args(args: argparse.Namespace) -> NativeTask:
options: dict[str, Any] = {"pid": args.pid, "allow_foreground": args.allow_foreground}
if args.note_text:
options["note_text"] = args.note_text
- return appkit_task(args.task, Path(args.state_file), **options)
+ return native_task(args.task, Path(args.state_file), **options)
def main() -> None:
diff --git a/libs/cua-driver/examples/jev-use/python/tests/test_native_platforms.py b/libs/cua-driver/examples/jev-use/python/tests/test_native_platforms.py
new file mode 100644
index 0000000000..bd232a0ee6
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/python/tests/test_native_platforms.py
@@ -0,0 +1,183 @@
+"""Native tasks on the WPF (Windows UIA) and GTK3 (Linux AT-SPI) harnesses (RFC #4268, Phase 2).
+
+The fixtures are real ``get_window_state`` results from each harness in task
+mode, recorded by ``verify_native.py --capture-dir`` in ``CI: jev-use``.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from choose_action import validate_request
+from jev_adapter import choose_mock_for_task
+from native import NativeObservation, eligible_controls
+from native_tasks import (
+ HARNESSES,
+ NATIVE_TASK_IDS,
+ appkit_task,
+ native_choice_request,
+ native_task,
+ split_task_id,
+)
+from sources import NativeAccessibilitySource
+from tasks import TaskSources
+
+FIXTURES = Path(__file__).resolve().parents[2] / "fixtures/native"
+
+
+def window_state(harness: str, name: str) -> dict:
+ path = FIXTURES / f"{harness}-window-state-{name}-v1.json"
+ return json.loads(path.read_text(encoding="utf-8"))
+
+
+def sources(task, harness: str, name: str) -> TaskSources:
+ payload = window_state(harness, name)
+ observation = NativeObservation.from_window_state(
+ payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
+ )
+ ax = NativeAccessibilitySource.from_observation(
+ observation, HARNESSES[harness].platform, redact=task.redact_text, text_method=task.text_method
+ )
+ return TaskSources(ax=ax)
+
+
+class HarnessRegistryTest(unittest.TestCase):
+ def test_every_harness_has_the_same_three_tasks(self) -> None:
+ self.assertEqual(
+ NATIVE_TASK_IDS,
+ tuple(f"{h}-{k}" for h in ("appkit", "wpf", "gtk3") for k in ("counter", "save-note", "choose-size")),
+ )
+ for task_id in NATIVE_TASK_IDS:
+ harness, kind = split_task_id(task_id)
+ task = native_task(task_id, Path("/tmp/none.json"), pid=7)
+ self.assertEqual(task.scope.window_title, harness.window_title)
+ self.assertEqual(task.oracle.schema, harness.state_schema)
+ self.assertEqual(task.oracle.expected_pid, 7)
+ # Same semantics everywhere: the mock takes the same IDs on every platform.
+ self.assertEqual(
+ task.mock_preferences,
+ native_task(f"appkit-{kind}", Path("/tmp/none.json")).mock_preferences,
+ )
+
+ def test_platforms_and_windows(self) -> None:
+ self.assertEqual({h.name: h.platform for h in HARNESSES.values()},
+ {"appkit": "macos", "wpf": "windows", "gtk3": "linux"})
+ self.assertEqual(HARNESSES["wpf"].window_title, "CuaTestHarness WPF Tasks")
+ self.assertEqual(HARNESSES["gtk3"].window_title, "CuaTestHarness GTK3 Tasks")
+
+ def test_unknown_tasks_are_rejected(self) -> None:
+ for task_id in ("wpf-reset", "winui3-counter", "counter", ""):
+ with self.assertRaises(ValueError):
+ split_task_id(task_id)
+ with self.assertRaises(ValueError):
+ appkit_task("wpf-counter", Path("/tmp/none.json"))
+
+
+class RecordedHarnessFixtureTest(unittest.TestCase):
+ """Role table and element rules against real WPF and GTK3 observations."""
+
+ HARNESSES = ("wpf", "gtk3")
+
+ def test_task_controls_map_to_the_same_candidate_ids(self) -> None:
+ for harness in self.HARNESSES:
+ with self.subTest(harness=harness):
+ note = native_task(f"{harness}-save-note", Path("/tmp/none.json"), note_text="secret note")
+ ids = [c.id for c in note.plan(sources(note, harness, "initial")).candidates]
+ self.assertEqual(ids[-2:], ["reobserve", "abstain"])
+ self.assertIn("ax:text_input:note:set:note", ids)
+ self.assertIn("ax:button:save-note", ids)
+ self.assertIn("ax:button:increment", ids)
+ # Reset (destructive) and Exit (close_unsaved) never reach the chooser.
+ self.assertNotIn("ax:button:reset", ids)
+ self.assertNotIn("ax:button:exit", ids)
+ size = native_task(f"{harness}-choose-size", Path("/tmp/none.json"))
+ size_ids = [c.id for c in size.plan(sources(size, harness, "initial")).candidates]
+ for expected in ("ax:radio:small", "ax:radio:medium", "ax:radio:large", "ax:checkbox:i-agree"):
+ self.assertIn(expected, size_ids)
+
+ def test_requests_are_valid_and_leak_no_secret_or_token(self) -> None:
+ for harness in self.HARNESSES:
+ with self.subTest(harness=harness):
+ task = native_task(f"{harness}-save-note", Path("/tmp/none.json"), note_text="secret note")
+ task_sources = sources(task, harness, "initial")
+ step = task.plan(task_sources)
+ setter = {c.id: c for c in step.candidates}["ax:text_input:note:set:note"]
+ self.assertEqual(setter.tool, "set_value")
+ self.assertEqual(setter.arguments["value"], "secret note")
+ request = native_choice_request(task, task_sources, step, [])
+ validate_request(request)
+ wire = json.dumps(request)
+ self.assertNotIn("secret note", wire)
+ self.assertNotIn("element_token", wire)
+ self.assertNotIn(f"{task_sources.ax.observation.snapshot_id}:", wire)
+
+ def test_selected_state_is_read_from_the_platform(self) -> None:
+ for harness in self.HARNESSES:
+ with self.subTest(harness=harness):
+ size = native_task(f"{harness}-choose-size", Path("/tmp/none.json"))
+ initial = sources(size, harness, "initial")
+ choice, _, _ = choose_mock_for_task(size, initial, size.candidates(initial), [])
+ self.assertEqual(choice, "ax:radio:large")
+ after = sources(size, harness, "after-choose-size")
+ controls = {c.id: c for c in after.ax.controls}
+ self.assertIs(controls["ax:radio:large"].selected, True)
+ self.assertIs(controls["ax:checkbox:i-agree"].selected, True)
+ ids = [c.id for c in size.candidates(after)]
+ self.assertNotIn("ax:radio:large", ids) # already selected
+ self.assertIn("ax:radio:small", ids)
+
+ def test_text_entry_value_is_never_a_label(self) -> None:
+ for harness in self.HARNESSES:
+ with self.subTest(harness=harness):
+ note = native_task(f"{harness}-save-note", Path("/tmp/none.json"))
+ after = sources(note, harness, "after-save-note")
+ field = next(c for c in after.ax.controls if c.role_class == "text_input")
+ self.assertEqual(field.label, "Note")
+ ids = [c.id for c in note.candidates(after)]
+ if harness == "gtk3":
+ # Known limitation of Cua Driver 0.30.2 (#4291): a named
+ # AT-SPI text field reports no value, so the written note
+ # is not observable and the write stays offered.
+ self.assertIn("limitation", window_state(harness, "after-save-note")["_fixture"])
+ self.assertIsNone(field.value)
+ self.assertIn("ax:text_input:note:set:note", ids)
+ continue
+ self.assertEqual(field.value, "jev-use native note")
+ self.assertNotIn("ax:text_input:note:set:note", ids) # already holds the parameter
+
+ def test_windows_title_bar_is_window_chrome(self) -> None:
+ payload = window_state("wpf", "initial")
+ observation = NativeObservation.from_window_state(
+ payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
+ )
+ native = eligible_controls(observation, "windows")
+ labels = [c.label for c in native.controls]
+ for chrome in ("System", "Minimize", "Maximize", "Close"):
+ self.assertNotIn(chrome, labels)
+ self.assertEqual(native.excluded.get("window_chrome"), 4)
+ self.assertEqual(
+ labels,
+ ["Increment", "Reset", "I agree", "Small", "Medium", "Large", "Note", "Save note", "Exit"],
+ )
+
+ def test_linux_roles_map_to_role_classes(self) -> None:
+ payload = window_state("gtk3", "initial")
+ observation = NativeObservation.from_window_state(
+ payload, expected_pid=payload["pid"], expected_window_id=payload["window_id"]
+ )
+ native = eligible_controls(observation, "linux")
+ self.assertEqual(native.excluded, {})
+ self.assertEqual(
+ [(c.role_class, c.label) for c in native.controls],
+ [("button", "Increment"), ("button", "Reset"), ("checkbox", "I agree"),
+ ("radio", "Small"), ("radio", "Medium"), ("radio", "Large"),
+ ("text_input", "Note"), ("button", "Save note"), ("button", "Exit")],
+ )
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/libs/cua-driver/examples/jev-use/typescript/native.ts b/libs/cua-driver/examples/jev-use/typescript/native.ts
index ca69ab690c..c98bbd1fa6 100644
--- a/libs/cua-driver/examples/jev-use/typescript/native.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/native.ts
@@ -10,6 +10,7 @@ import { createHash } from 'node:crypto';
import {
ROLE_CLASS_ACTION,
+ isWindowChrome,
normalizedRole,
roleClass,
type ActionKind,
@@ -217,6 +218,18 @@ export function eligibleControls(
return path.reverse();
};
+ const inWindowChrome = (item: WindowElement): boolean => {
+ const seen = new Set<number>();
+ let parent = item.parent_index;
+ while (Number.isInteger(parent) && byIndex.has(parent as number) && !seen.has(parent as number)) {
+ seen.add(parent as number);
+ const ancestor = byIndex.get(parent as number)!;
+ if (isWindowChrome(ancestor.role, platform)) return true;
+ parent = ancestor.parent_index;
+ }
+ return false;
+ };
+
const pending: { item: WindowElement; klass: RoleClass; label: string; path: [string, string][] }[] = [];
for (const item of observation.elements) {
const klass = roleClass(item.role, platform);
@@ -224,6 +237,10 @@ export function eligibleControls(
exclude('unknown_role');
continue;
}
+ if (inWindowChrome(item)) {
+ exclude('window_chrome');
+ continue;
+ }
if (item.enabled === false) {
exclude('disabled');
continue;
diff --git a/libs/cua-driver/examples/jev-use/typescript/native_platforms.test.ts b/libs/cua-driver/examples/jev-use/typescript/native_platforms.test.ts
new file mode 100644
index 0000000000..9e3997d96b
--- /dev/null
+++ b/libs/cua-driver/examples/jev-use/typescript/native_platforms.test.ts
@@ -0,0 +1,136 @@
+// Native tasks on the WPF (Windows UIA) and GTK3 (Linux AT-SPI) harnesses
+// (RFC #4268, Phase 2). Mirrors python/tests/test_native_platforms.py; the
+// fixtures are real get_window_state results recorded in CI: jev-use.
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import test from 'node:test';
+
+import { validateRequest } from './choose_action.js';
+import { chooseMockForTask } from './jev_adapter.js';
+import { eligibleControls, parseWindowState } from './native.js';
+import {
+ HARNESSES,
+ NATIVE_TASK_IDS,
+ appkitTask,
+ nativeChoiceRequest,
+ nativeTask,
+ splitTaskId,
+ type NativeTask,
+} from './native_tasks.js';
+import { NativeAccessibilitySource } from './sources.js';
+import type { TaskSources } from './tasks.js';
+
+const PLATFORM_HARNESSES = ['wpf', 'gtk3'] as const;
+
+function windowState(harness: string, name: string): any {
+ const url = new URL(`../fixtures/native/${harness}-window-state-${name}-v1.json`, import.meta.url);
+ return JSON.parse(readFileSync(url, 'utf8'));
+}
+const observe = (payload: any) => parseWindowState(payload, payload.pid, payload.window_id);
+
+function sources(task: NativeTask, harness: string, name: string): TaskSources {
+ return {
+ ax: NativeAccessibilitySource.fromObservation(observe(windowState(harness, name)), HARNESSES[harness].platform, {
+ redact: task.redactText,
+ textMethod: task.textMethod,
+ }),
+ visualPath: false,
+ foregroundIds: new Set(),
+ };
+}
+
+test('every harness has the same three tasks', () => {
+ assert.deepEqual(
+ NATIVE_TASK_IDS,
+ ['appkit', 'wpf', 'gtk3'].flatMap((h) => ['counter', 'save-note', 'choose-size'].map((k) => `${h}-${k}`))
+ );
+ for (const taskId of NATIVE_TASK_IDS) {
+ const [harness, kind] = splitTaskId(taskId);
+ const task = nativeTask(taskId, '/tmp/none.json', { pid: 7 });
+ assert.equal(task.scope.windowTitle, harness.windowTitle);
+ assert.equal(task.oracle.schema, harness.stateSchema);
+ assert.equal(task.oracle.expectedPid, 7);
+ assert.deepEqual(task.mockPreferences, nativeTask(`appkit-${kind}`, '/tmp/none.json').mockPreferences);
+ }
+ assert.equal(HARNESSES.wpf.windowTitle, 'CuaTestHarness WPF Tasks');
+ assert.equal(HARNESSES.gtk3.windowTitle, 'CuaTestHarness GTK3 Tasks');
+ for (const taskId of ['wpf-reset', 'winui3-counter', 'counter', '']) {
+ assert.throws(() => splitTaskId(taskId));
+ }
+ assert.throws(() => appkitTask('wpf-counter', '/tmp/none.json'));
+});
+
+test('recorded task controls map to the same candidate IDs', () => {
+ for (const harness of PLATFORM_HARNESSES) {
+ const note = nativeTask(`${harness}-save-note`, '/tmp/none.json', { noteText: 'secret note' });
+ const noteSources = sources(note, harness, 'initial');
+ const step = note.plan(noteSources);
+ const ids = step.candidates.map((c) => c.id);
+ assert.deepEqual(ids.slice(-2), ['reobserve', 'abstain'], harness);
+ for (const expected of ['ax:text_input:note:set:note', 'ax:button:save-note', 'ax:button:increment']) {
+ assert.ok(ids.includes(expected), `${harness} ${expected}`);
+ }
+ assert.ok(!ids.includes('ax:button:reset') && !ids.includes('ax:button:exit'), harness);
+ const setter = step.candidates.find((c) => c.id === 'ax:text_input:note:set:note')!;
+ assert.equal(setter.tool, 'set_value');
+ assert.equal(setter.arguments.value, 'secret note');
+ const request = nativeChoiceRequest(note, noteSources, step, []);
+ validateRequest(request);
+ const wire = JSON.stringify(request);
+ assert.ok(!wire.includes('secret note') && !wire.includes('element_token'), harness);
+ assert.ok(!wire.includes(`${noteSources.ax!.observation.snapshotId}:`), harness);
+
+ const size = nativeTask(`${harness}-choose-size`, '/tmp/none.json');
+ const initial = sources(size, harness, 'initial');
+ const sizeIds = size.candidates(initial).map((c) => c.id);
+ for (const expected of ['ax:radio:small', 'ax:radio:medium', 'ax:radio:large', 'ax:checkbox:i-agree']) {
+ assert.ok(sizeIds.includes(expected), `${harness} ${expected}`);
+ }
+ assert.equal(chooseMockForTask(size, initial, size.candidates(initial), []).choice, 'ax:radio:large');
+ const after = sources(size, harness, 'after-choose-size');
+ const controls = new Map(after.ax!.controls.map((c) => [c.id, c]));
+ assert.equal(controls.get('ax:radio:large')?.selected, true, harness);
+ assert.equal(controls.get('ax:checkbox:i-agree')?.selected, true, harness);
+ const afterIds = size.candidates(after).map((c) => c.id);
+ assert.ok(!afterIds.includes('ax:radio:large') && afterIds.includes('ax:radio:small'), harness);
+ }
+});
+
+test('a text entry value is never a label', () => {
+ for (const harness of PLATFORM_HARNESSES) {
+ const note = nativeTask(`${harness}-save-note`, '/tmp/none.json');
+ const after = sources(note, harness, 'after-save-note');
+ const field = after.ax!.controls.find((c) => c.roleClass === 'text_input')!;
+ assert.equal(field.label, 'Note');
+ const ids = note.candidates(after).map((c) => c.id);
+ if (harness === 'gtk3') {
+ // Known limitation of Cua Driver 0.30.2 (#4291): a named AT-SPI text
+ // field reports no value, so the written note is not observable.
+ assert.ok('limitation' in windowState(harness, 'after-save-note')._fixture);
+ assert.equal(field.value, undefined);
+ assert.ok(ids.includes('ax:text_input:note:set:note'));
+ continue;
+ }
+ assert.equal(field.value, 'jev-use native note');
+ assert.ok(!ids.includes('ax:text_input:note:set:note'));
+ }
+});
+
+test('the Windows title bar is window chrome and GTK3 roles map', () => {
+ const wpf = eligibleControls(observe(windowState('wpf', 'initial')), 'windows');
+ assert.equal(wpf.excluded.window_chrome, 4);
+ assert.deepEqual(
+ wpf.controls.map((c) => c.label),
+ ['Increment', 'Reset', 'I agree', 'Small', 'Medium', 'Large', 'Note', 'Save note', 'Exit']
+ );
+ const gtk3 = eligibleControls(observe(windowState('gtk3', 'initial')), 'linux');
+ assert.deepEqual(gtk3.excluded, {});
+ assert.deepEqual(
+ gtk3.controls.map((c) => [c.roleClass, c.label]),
+ [
+ ['button', 'Increment'], ['button', 'Reset'], ['checkbox', 'I agree'],
+ ['radio', 'Small'], ['radio', 'Medium'], ['radio', 'Large'],
+ ['text_input', 'Note'], ['button', 'Save note'], ['button', 'Exit'],
+ ]
+ );
+});
diff --git a/libs/cua-driver/examples/jev-use/typescript/native_roles.ts b/libs/cua-driver/examples/jev-use/typescript/native_roles.ts
index 114d528baa..24f3f84a95 100644
--- a/libs/cua-driver/examples/jev-use/typescript/native_roles.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/native_roles.ts
@@ -120,6 +120,20 @@ export const ROLE_TABLES: Readonly<Record<Platform, ReadonlyMap<string, RoleClas
};
/** Return the role class for a raw Driver role, or null when excluded. */
+// Window-chrome containers, by normalized role. Their actionable descendants
+// (Windows' title-bar System menu, Minimize, Maximize, and Close) belong to the
+// window manager, not the application, so they are never candidates. Mirrors
+// python/native_roles.py WINDOW_CHROME_ROLES.
+export const WINDOW_CHROME_ROLES: Readonly<Record<Platform, ReadonlySet<string>>> = {
+ macos: new Set(),
+ windows: new Set([normalizedRole('TitleBar')]),
+ linux: new Set(),
+};
+
+export function isWindowChrome(rawRole: unknown, platform: Platform): boolean {
+ return typeof rawRole === 'string' && WINDOW_CHROME_ROLES[platform].has(normalizedRole(rawRole));
+}
+
export function roleClass(rawRole: unknown, platform: Platform): RoleClass | null {
if (typeof rawRole !== 'string' || !rawRole) return null;
return ROLE_TABLES[platform].get(normalizedRole(rawRole)) ?? null;
diff --git a/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts b/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts
index 1239895940..f3c47b24b3 100644
--- a/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/native_tasks.ts
@@ -3,8 +3,9 @@
* python/native_tasks.py: a NativeTask declares goal, parameters (the only
* source of text), window scope, allowed action kinds, opt-in risks,
* foreground permission, step budget, and an app-owned oracle. The built-in
- * tasks drive the AppKit harness launched with CUA_APPKIT_TASK_STATE, whose
- * JSON state file is the oracle.
+ * tasks drive the AppKit, WPF, and GTK3 harnesses in task mode
+ * (CUA_APPKIT_TASK_STATE, CUA_WPF_TASK_STATE, CUA_GTK3_TASK_STATE), whose JSON
+ * state file is the oracle.
*/
import { readFile } from 'node:fs/promises';
@@ -427,25 +428,82 @@ export function nativeChoiceRequest(
};
}
-// AppKit harness tasks.
+// Harness tasks. The same three tasks run on every repository harness that has
+// a task mode: AppKit (macOS AX), WPF (Windows UIA), and GTK3 (Linux AT-SPI).
+// Each shows the same labeled controls and rewrites the same app-owned JSON
+// state file, so task semantics, candidate IDs, and mock choices are identical
+// across platforms. Only the window, the state schema, and the role table differ.
-export const APPKIT_WINDOW_TITLE = 'CuaTestHarness AppKit';
-export const APPKIT_BUNDLE_ID = 'com.trycua.harness.appkit';
-export const APPKIT_STATE_SCHEMA = 'cua.appkit_task_state_v1';
+export type HarnessSpec = Readonly<{
+ name: string;
+ platform: 'macos' | 'windows' | 'linux';
+ windowTitle: string;
+ stateSchema: string;
+ stateEnv: string;
+ bundleId?: string;
+ processName?: string;
+}>;
+
+export const HARNESSES: Readonly<Record<string, HarnessSpec>> = {
+ appkit: {
+ name: 'appkit',
+ platform: 'macos',
+ windowTitle: 'CuaTestHarness AppKit',
+ stateSchema: 'cua.appkit_task_state_v1',
+ stateEnv: 'CUA_APPKIT_TASK_STATE',
+ bundleId: 'com.trycua.harness.appkit',
+ },
+ // WPF and GTK3 show a dedicated task window in task mode: their ordinary
+ // main windows scroll, so most controls would be off screen (and excluded).
+ wpf: {
+ name: 'wpf',
+ platform: 'windows',
+ windowTitle: 'CuaTestHarness WPF Tasks',
+ stateSchema: 'cua.wpf_task_state_v1',
+ stateEnv: 'CUA_WPF_TASK_STATE',
+ processName: 'CuaTestHarness.Wpf',
+ },
+ gtk3: {
+ name: 'gtk3',
+ platform: 'linux',
+ windowTitle: 'CuaTestHarness GTK3 Tasks',
+ stateSchema: 'cua.gtk3_task_state_v1',
+ stateEnv: 'CUA_GTK3_TASK_STATE',
+ processName: 'python3',
+ },
+};
+
+export const APPKIT_WINDOW_TITLE = HARNESSES.appkit.windowTitle;
+export const APPKIT_BUNDLE_ID = HARNESSES.appkit.bundleId as string;
+export const APPKIT_STATE_SCHEMA = HARNESSES.appkit.stateSchema;
export const COUNTER_TARGET = 3;
export const DEFAULT_NOTE_TEXT = 'jev-use native note';
-export const APPKIT_TASK_IDS = ['appkit-counter', 'appkit-save-note', 'appkit-choose-size'] as const;
-export type AppKitTaskId = (typeof APPKIT_TASK_IDS)[number];
-
-export function appkitTask(
- taskId: string,
- statePath: string,
- options: { pid?: number; noteText?: string; allowForeground?: boolean } = {}
-): NativeTask {
- const oracle = new AppStateOracle(statePath, APPKIT_STATE_SCHEMA, options.pid);
- const scope: WindowScope = { windowTitle: APPKIT_WINDOW_TITLE, bundleId: APPKIT_BUNDLE_ID };
+export const TASK_KINDS = ['counter', 'save-note', 'choose-size'] as const;
+
+export const harnessTaskIds = (harness: string): string[] => TASK_KINDS.map((kind) => `${harness}-${kind}`);
+export const APPKIT_TASK_IDS = harnessTaskIds('appkit');
+export const NATIVE_TASK_IDS = Object.keys(HARNESSES).flatMap(harnessTaskIds);
+
+export function splitTaskId(taskId: string): [HarnessSpec, (typeof TASK_KINDS)[number]] {
+ const index = taskId.indexOf('-');
+ const harness = index > 0 ? HARNESSES[taskId.slice(0, index)] : undefined;
+ const kind = taskId.slice(index + 1) as (typeof TASK_KINDS)[number];
+ if (!harness || !TASK_KINDS.includes(kind)) throw new Error(`unknown native task: ${taskId}`);
+ return [harness, kind];
+}
+
+export type HarnessTaskOptions = { pid?: number; noteText?: string; allowForeground?: boolean };
+
+export function nativeTask(taskId: string, statePath: string, options: HarnessTaskOptions = {}): NativeTask {
+ const [harness, kind] = splitTaskId(taskId);
+ const oracle = new AppStateOracle(statePath, harness.stateSchema, options.pid);
+ const scope: WindowScope = {
+ windowTitle: harness.windowTitle,
+ ...(harness.bundleId ? { bundleId: harness.bundleId } : {}),
+ ...(harness.processName ? { processName: harness.processName } : {}),
+ };
const allowForeground = options.allowForeground ?? false;
- if (taskId === 'appkit-counter') {
+ if (kind === 'counter') {
return new NativeTask({
id: taskId,
goal:
@@ -464,7 +522,7 @@ export function appkitTask(
mockPreferences: ['ax:button:increment'],
});
}
- if (taskId === 'appkit-save-note') {
+ if (kind === 'save-note') {
const note = options.noteText ?? DEFAULT_NOTE_TEXT;
return new NativeTask({
id: taskId,
@@ -480,18 +538,21 @@ export function appkitTask(
mockPreferences: ['ax:text_input:note:set:note', 'ax:button:save-note'],
});
}
- if (taskId === 'appkit-choose-size') {
- return new NativeTask({
- id: taskId,
- goal: 'Choose the Large size option and check the I agree checkbox.',
- scope,
- allowedActions: new Set<ActionKind>(['select', 'toggle']),
- oracle,
- check: (state) => (state.size === 'large' && state.agreed === true ? 'verified' : 'pending'),
- allowForeground,
- maxSteps: 5,
- mockPreferences: ['ax:radio:large', 'ax:checkbox:i-agree'],
- });
- }
- throw new Error(`unknown AppKit task: ${taskId}`);
+ return new NativeTask({
+ id: taskId,
+ goal: 'Choose the Large size option and check the I agree checkbox.',
+ scope,
+ allowedActions: new Set<ActionKind>(['select', 'toggle']),
+ oracle,
+ check: (state) => (state.size === 'large' && state.agreed === true ? 'verified' : 'pending'),
+ allowForeground,
+ maxSteps: 5,
+ mockPreferences: ['ax:radio:large', 'ax:checkbox:i-agree'],
+ });
+}
+
+/** Build one AppKit harness task (kept for Phase 1 callers). */
+export function appkitTask(taskId: string, statePath: string, options: HarnessTaskOptions = {}): NativeTask {
+ if (!APPKIT_TASK_IDS.includes(taskId)) throw new Error(`unknown AppKit task: ${taskId}`);
+ return nativeTask(taskId, statePath, options);
}
diff --git a/libs/cua-driver/examples/jev-use/typescript/run_native.ts b/libs/cua-driver/examples/jev-use/typescript/run_native.ts
index 6372c1ec76..25a91cf670 100644
--- a/libs/cua-driver/examples/jev-use/typescript/run_native.ts
+++ b/libs/cua-driver/examples/jev-use/typescript/run_native.ts
@@ -21,8 +21,8 @@ import { chooseBoundedWithTypeSafe, chooseMockForTask } from './jev_adapter.js';
import { parseWindowState, type NativeObservation } from './native.js';
import type { Platform } from './native_roles.js';
import {
- APPKIT_TASK_IDS,
- appkitTask,
+ NATIVE_TASK_IDS,
+ nativeTask,
nativeChoiceRequest,
visualFallbackReason,
windowStateArguments,
@@ -66,8 +66,8 @@ export function parseArgs(argv: string[]): Arguments {
else if (value === '--log') result.log = argv[++index];
else throw new Error(`unknown argument: ${value}`);
}
- if (!result.task || !(APPKIT_TASK_IDS as readonly string[]).includes(result.task)) {
- throw new Error(`--task must be one of ${APPKIT_TASK_IDS.join(', ')}`);
+ if (!result.task || !NATIVE_TASK_IDS.includes(result.task)) {
+ throw new Error(`--task must be one of ${NATIVE_TASK_IDS.join(', ')}`);
}
if (result.provider !== 'mock' && result.provider !== 'live') throw new Error('--provider must be mock or live');
if (!Number.isInteger(result.pid) || (result.pid as number) <= 0) throw new Error('--pid is required');
@@ -343,7 +343,7 @@ export async function runTask(args: Arguments, task: NativeTask): Promise<Outcom
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const args = parseArgs(process.argv.slice(2));
- const task = appkitTask(args.task, args.stateFile, {
+ const task = nativeTask(args.task, args.stateFile, {
pid: args.pid,
noteText: args.noteText,
allowForeground: args.allowForeground,
diff --git a/libs/cua-driver/examples/jev-use/verify_native.py b/libs/cua-driver/examples/jev-use/verify_native.py
index 43ac91fd9b..8b07b26428 100644
--- a/libs/cua-driver/examples/jev-use/verify_native.py
+++ b/libs/cua-driver/examples/jev-use/verify_native.py
@@ -1,14 +1,18 @@
-"""Verify the native jev-use tasks against the macOS AppKit harness (RFC #4268).
-
-For every task and language this script launches a fresh AppKit harness with
-its own ``CUA_APPKIT_TASK_STATE`` file, runs the native runner against that
-exact process, then reads the harness state file itself and checks it against
-the task's expected end state. The runners' own outcome events are not the
-oracle. Evidence (redacted JSONL logs plus ``summary.json``) goes to a new
-``--output-dir``.
-
-Build the harness first:
-``libs/cua-driver/tests/fixtures/build/macos.sh --only appkit``.
+"""Verify the native jev-use tasks against a repository harness (RFC #4268).
+
+``--harness`` selects the AppKit (macOS), WPF (Windows), or GTK3 (Linux)
+harness. For every task and language this script launches a fresh harness in
+task mode with its own state file (``CUA_<HARNESS>_TASK_STATE``), runs the
+native runner against that exact process, then reads the harness state file
+itself and checks it against the task's expected end state. The runners' own
+outcome events are not the oracle. Evidence (redacted JSONL logs plus
+``summary.json``) goes to a new ``--output-dir``. ``--capture-dir`` also
+records sanitized ``get_window_state`` fixtures before and after each run.
+
+Build the harness first, from the repository root:
+``libs/cua-driver/tests/fixtures/build/macos.sh --only appkit``,
+``libs/cua-driver/tests/fixtures/build/windows.ps1 -Targets wpf``, or
+``libs/cua-driver/tests/fixtures/build/linux.sh --only gtk3``.
"""
from __future__ import annotations
@@ -22,24 +26,50 @@ import sys
import tempfile
import time
from pathlib import Path
+from typing import NamedTuple
BASE = Path(__file__).resolve().parent
REPO = BASE.parents[3]
-DEFAULT_APP = REPO / "libs/cua-driver/rust/test-apps/harness-appkit/CuaTestHarness.AppKit.app"
-STATE_SCHEMA = "cua.appkit_task_state_v1"
+TEST_APPS = REPO / "libs/cua-driver/rust/test-apps"
NOTE_TEXT = "jev-use native note"
-# The independent end-state check for each task, read from the app's own file.
+
+class Harness(NamedTuple):
+ app: Path
+ executable: str # relative to ``app`` (empty when ``app`` is the executable)
+ window_title: str
+ state_schema: str
+ state_env: str
+
+
+# Mirrors python/native_tasks.py HARNESSES; kept independent so the verifier
+# does not trust the code it verifies.
+HARNESSES = {
+ "appkit": Harness(
+ TEST_APPS / "harness-appkit/CuaTestHarness.AppKit.app", "Contents/MacOS/CuaTestHarness.AppKit",
+ "CuaTestHarness AppKit", "cua.appkit_task_state_v1", "CUA_APPKIT_TASK_STATE",
+ ),
+ "wpf": Harness(
+ TEST_APPS / "harness-wpf/CuaTestHarness.Wpf.exe", "",
+ "CuaTestHarness WPF Tasks", "cua.wpf_task_state_v1", "CUA_WPF_TASK_STATE",
+ ),
+ "gtk3": Harness(
+ TEST_APPS / "harness-gtk3/CuaTestHarness.Gtk3", "",
+ "CuaTestHarness GTK3 Tasks", "cua.gtk3_task_state_v1", "CUA_GTK3_TASK_STATE",
+ ),
+}
+
+# The independent end-state check for each task kind, read from the app's own file.
EXPECTED = {
- "appkit-counter": lambda state: state.get("counter") == 3,
- "appkit-save-note": lambda state: state.get("note_saved") == NOTE_TEXT,
- "appkit-choose-size": lambda state: state.get("size") == "large" and state.get("agreed") is True,
+ "counter": lambda state: state.get("counter") == 3,
+ "save-note": lambda state: state.get("note_saved") == NOTE_TEXT,
+ "choose-size": lambda state: state.get("size") == "large" and state.get("agreed") is True,
}
# The executable candidates the deterministic mock provider must have acted on.
MOCK_ACTIONS = {
- "appkit-counter": ["ax:button:increment"] * 3,
- "appkit-save-note": ["ax:text_input:note:set:note", "ax:button:save-note"],
- "appkit-choose-size": ["ax:radio:large", "ax:checkbox:i-agree"],
+ "counter": ["ax:button:increment"] * 3,
+ "save-note": ["ax:text_input:note:set:note", "ax:button:save-note"],
+ "choose-size": ["ax:radio:large", "ax:checkbox:i-agree"],
}
@@ -49,11 +79,11 @@ def runner_command(language: str) -> list[str]:
return ["node", "--import", "tsx", "typescript/run_native.ts"]
-def launch_harness(app: Path, state: Path) -> subprocess.Popen:
- executable = app / "Contents/MacOS/CuaTestHarness.AppKit"
+def launch_harness(harness: Harness, app: Path, state: Path) -> subprocess.Popen:
+ executable = app / harness.executable if harness.executable else app
if not executable.is_file():
- raise SystemExit(f"AppKit harness is missing at {executable}; run the fixture build")
- env = {**os.environ, "CUA_APPKIT_TASK_STATE": str(state)}
+ raise SystemExit(f"harness is missing at {executable}; run the fixture build")
+ env = {**os.environ, harness.state_env: str(state)}
process = subprocess.Popen(
[str(executable)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
@@ -66,15 +96,15 @@ def launch_harness(app: Path, state: Path) -> subprocess.Popen:
if initial.get("pid") == process.pid:
return process
if process.poll() is not None:
- raise RuntimeError("AppKit harness exited during launch")
+ raise RuntimeError("harness exited during launch")
time.sleep(0.1)
process.kill()
- raise RuntimeError("AppKit harness did not publish its initial task state")
+ raise RuntimeError("harness did not publish its initial task state")
-def read_state(state: Path, pid: int) -> dict:
+def read_state(harness: Harness, state: Path, pid: int) -> dict:
observed = json.loads(state.read_text(encoding="utf-8"))
- if observed.get("schema") != STATE_SCHEMA or observed.get("pid") != pid:
+ if observed.get("schema") != harness.state_schema or observed.get("pid") != pid:
raise RuntimeError("harness state file does not belong to the launched harness")
return observed
@@ -98,21 +128,43 @@ def summarize(events: list[dict]) -> dict:
}
-def verify(language: str, provider: str, task: str, app: Path, output: Path, work: Path) -> dict:
+def capture(harness_name: str, pid: int, output: Path, label: str) -> None:
+ """Record one sanitized get_window_state fixture; a failure is not fatal."""
+ harness = HARNESSES[harness_name]
+ completed = subprocess.run(
+ [sys.executable, "python/capture_window_state.py", "--pid", str(pid),
+ "--title", harness.window_title, "--output", str(output),
+ "--source", f"CuaTestHarness {harness_name} task mode ({label}), observed by get_window_state"],
+ cwd=BASE, check=False, timeout=120,
+ )
+ if completed.returncode != 0:
+ print(json.dumps({"event": "capture_failed", "label": label}), flush=True)
+
+
+def verify(
+ language: str, provider: str, harness_name: str, kind: str, app: Path, output: Path, work: Path,
+ capture_dir: Path | None = None,
+) -> dict:
+ harness = HARNESSES[harness_name]
+ task = f"{harness_name}-{kind}"
state = work / f"{language}-{provider}-{task}-state.json"
log = output / f"{language}-{provider}-{task}.jsonl"
- harness = launch_harness(app, state)
+ process = launch_harness(harness, app, state)
try:
- initial = read_state(state, harness.pid)
+ initial = read_state(harness, state, process.pid)
+ if capture_dir is not None:
+ capture(harness_name, process.pid, capture_dir / f"{language}-{task}-initial.json", "initial")
command = runner_command(language) + [
- "--task", task, "--provider", provider, "--pid", str(harness.pid),
+ "--task", task, "--provider", provider, "--pid", str(process.pid),
"--state-file", str(state), "--note-text", NOTE_TEXT, "--log", str(log),
]
completed = subprocess.run(command, cwd=BASE, check=False, timeout=300)
- observed = read_state(state, harness.pid)
+ observed = read_state(harness, state, process.pid)
+ if capture_dir is not None:
+ capture(harness_name, process.pid, capture_dir / f"{language}-{task}-after.json", "after the task")
finally:
- harness.kill()
- harness.wait(timeout=10)
+ process.kill()
+ process.wait(timeout=10)
events = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []
result = {
"language": language,
@@ -124,7 +176,7 @@ def verify(language: str, provider: str, task: str, app: Path, output: Path, wor
"observed": {key: observed.get(key) for key in ("counter", "agreed", "size", "note_saved")},
**summarize(events),
}
- result["verified"] = bool(EXPECTED[task](observed))
+ result["verified"] = bool(EXPECTED[kind](observed))
if result["observed"].get("note_saved") == NOTE_TEXT:
# The note text is a secret task parameter; never keep it in evidence.
result["observed"]["note_saved"] = "[note text]"
@@ -132,17 +184,19 @@ def verify(language: str, provider: str, task: str, app: Path, output: Path, wor
raise RuntimeError(f"runner did not report verified: {json.dumps(result)}")
if not result["verified"]:
raise RuntimeError(f"independent harness state does not match: {json.dumps(result)}")
- if provider == "mock" and result["actions"] != MOCK_ACTIONS[task]:
+ if provider == "mock" and result["actions"] != MOCK_ACTIONS[kind]:
raise RuntimeError(f"mock provider took unexpected actions: {json.dumps(result)}")
return result
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--app", type=Path, default=DEFAULT_APP, help="CuaTestHarness.AppKit.app")
+ parser.add_argument("--harness", choices=sorted(HARNESSES), default="appkit")
+ parser.add_argument("--app", type=Path, help="harness app or executable (default: the fixture build output)")
parser.add_argument("--typescript", action="store_true", help="also verify the TypeScript runner")
parser.add_argument("--live", action="store_true", help="also verify live Jev; needs a TypeSafe key")
- parser.add_argument("--task", action="append", choices=sorted(EXPECTED), help="limit to tasks")
+ parser.add_argument("--task", action="append", choices=sorted(EXPECTED), help="limit to task kinds")
+ parser.add_argument("--capture-dir", type=Path, help="also record sanitized window-state fixtures")
parser.add_argument("--output-dir", type=Path, required=True, help="new evidence directory")
args = parser.parse_args()
if args.live and not os.environ.get("TYPESAFE_API_KEY", "").strip():
@@ -151,9 +205,13 @@ def main() -> None:
os.environ["TYPESAFE_API_KEY"] = getpass.getpass("TypeSafe API key: ").strip()
output = args.output_dir.resolve()
output.mkdir(mode=0o700, parents=True, exist_ok=False)
+ app = args.app or HARNESSES[args.harness].app
+ capture_dir = args.capture_dir.resolve() if args.capture_dir else None
+ if capture_dir is not None:
+ capture_dir.mkdir(parents=True, exist_ok=True)
summary: dict = {
"complete": False,
- "harness": "appkit",
+ "harness": args.harness,
"live_requested": args.live,
"typescript_requested": args.typescript,
"checks": [],
@@ -162,8 +220,10 @@ def main() -> None:
with tempfile.TemporaryDirectory(prefix="jev-native-") as work:
for language in ["python", "typescript"] if args.typescript else ["python"]:
for provider in ["mock", "live"] if args.live else ["mock"]:
- for task in args.task or sorted(EXPECTED):
- result = verify(language, provider, task, args.app, output, Path(work))
+ for kind in args.task or sorted(EXPECTED):
+ result = verify(
+ language, provider, args.harness, kind, app, output, Path(work), capture_dir
+ )
summary["checks"].append(result)
print(json.dumps({"event": "independently_verified", **result}), flush=True)
summary["complete"] = True
diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
index 6ec9bfc269..03760112f4 100755
--- a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
+++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py
@@ -16,6 +16,9 @@
# modality recorder's effect verifier reads (counter=, mirror=, agreed=,
# slider_value=, last_action=, clicks=, menu_action=, scroll_offset=).
+import json
+import os
+
import gi
gi.require_version("Gtk", "3.0")
@@ -278,8 +281,119 @@ class HarnessWindow(Gtk.Window):
self.popover_status.set_text("popover_open=True")
+TASK_STATE_ENV = "CUA_GTK3_TASK_STATE"
+TASK_STATE_SCHEMA = "cua.gtk3_task_state_v1"
+
+
+class TaskWindow(Gtk.Window):
+ """Opt-in jev-use task window (RFC #4268), shown instead of HarnessWindow
+ when CUA_GTK3_TASK_STATE=<path> is set.
+
+ It carries the same labeled controls as the AppKit and WPF task modes
+ (Increment, Reset, I agree, Small/Medium/Large, Note, Save note, Exit) in a
+ small window where every control is on screen. Accessible names are the
+ visible labels here, not the aid-style names of HarnessWindow, so candidate
+ IDs match across platforms. Every change atomically rewrites an app-owned
+ JSON state file; the jev-use task oracle reads that file and never depends
+ on Cua Driver output. Ordinary launches never create this window.
+ """
+
+ def __init__(self, state_path):
+ super().__init__(title="CuaTestHarness GTK3 Tasks")
+ self.set_default_size(480, 320)
+ self.state_path = state_path
+ self.counter = 0
+ self.agreed = False
+ self.size = "none"
+ self.saved_note = None
+ self.sequence = 0
+
+ root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=8)
+ root.set_border_width(16)
+ self.add(root)
+ self.counter_label = Gtk.Label(label="counter=0", xalign=0)
+ root.pack_start(self.counter_label, False, False, 0)
+
+ counter_row = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=8)
+ counter_row.pack_start(self.button("Increment", self.on_increment), False, False, 0)
+ counter_row.pack_start(self.button("Reset", self.on_reset), False, False, 0)
+ root.pack_start(counter_row, False, False, 0)
+
+ agree = Gtk.CheckButton(label="I agree")
+ agree.connect("toggled", self.on_agree)
+ root.pack_start(agree, False, False, 0)
+
+ # A GTK radio group always has one active member. A hidden "none"
+ # member keeps Small/Medium/Large all unselected at launch.
+ size_row = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=12)
+ self.no_size = Gtk.RadioButton(label="none")
+ for title in ("Small", "Medium", "Large"):
+ radio = Gtk.RadioButton.new_with_label_from_widget(self.no_size, title)
+ radio.connect("toggled", self.on_size, title.lower())
+ size_row.pack_start(radio, False, False, 0)
+ root.pack_start(size_row, False, False, 0)
+
+ note_row = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=8)
+ # An accessible name and no placeholder text, like the other harnesses.
+ self.note = aid(Gtk.Entry(), "Note")
+ self.note.set_width_chars(28)
+ note_row.pack_start(self.note, False, False, 0)
+ note_row.pack_start(self.button("Save note", self.on_save_note), False, False, 0)
+ root.pack_start(note_row, False, False, 0)
+
+ root.pack_start(self.button("Exit", lambda *_: Gtk.main_quit()), False, False, 0)
+ self.connect("destroy", Gtk.main_quit)
+ self.publish()
+
+ @staticmethod
+ def button(label, handler):
+ widget = Gtk.Button(label=label)
+ widget.set_halign(Gtk.Align.START)
+ widget.connect("clicked", handler)
+ return widget
+
+ def on_increment(self, *_):
+ self.counter += 1
+ self.publish()
+
+ def on_reset(self, *_):
+ self.counter = 0
+ self.publish()
+
+ def on_agree(self, check):
+ self.agreed = check.get_active()
+ self.publish()
+
+ def on_size(self, radio, value):
+ if radio.get_active():
+ self.size = value
+ self.publish()
+
+ def on_save_note(self, *_):
+ self.saved_note = self.note.get_text()
+ self.publish()
+
+ def publish(self):
+ self.counter_label.set_text(f"counter={self.counter}")
+ self.sequence += 1
+ state = {
+ "schema": TASK_STATE_SCHEMA,
+ "pid": os.getpid(),
+ "seq": self.sequence,
+ "counter": self.counter,
+ "agreed": self.agreed,
+ "size": self.size,
+ "note_saved": self.saved_note,
+ }
+ temporary = f"{self.state_path}.{os.getpid()}.tmp"
+ with open(temporary, "w", encoding="utf-8") as stream:
+ json.dump(state, stream, sort_keys=True)
+ os.replace(temporary, self.state_path)
+
+
def main():
- win = HarnessWindow()
+ task_state = os.environ.get(TASK_STATE_ENV, "").strip()
+ win = TaskWindow(task_state) if task_state else HarnessWindow()
win.show_all()
Gtk.main()
diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml b/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml
index 3053577574..a27b324fb2 100644
--- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml
+++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml
@@ -1,6 +1,6 @@
<Application x:Class="CuaTestHarness.Wpf.App"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
- StartupUri="MainWindow.xaml">
+ Startup="OnStartup">
<Application.Resources/>
</Application>
diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml.cs b/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml.cs
index bca8942c2e..10f23e96ec 100644
--- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml.cs
+++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/App.xaml.cs
@@ -1,7 +1,19 @@
+using System;
using System.Windows;
namespace CuaTestHarness.Wpf;
public partial class App : Application
{
+ // Ordinary launches show MainWindow exactly as before (it was the
+ // StartupUri). CUA_WPF_TASK_STATE selects the opt-in jev-use task window.
+ private void OnStartup(object sender, StartupEventArgs e)
+ {
+ var taskState = Environment.GetEnvironmentVariable(TaskWindow.StateEnv);
+ Window window = string.IsNullOrWhiteSpace(taskState)
+ ? new MainWindow()
+ : new TaskWindow(taskState);
+ MainWindow = window;
+ window.Show();
+ }
}
diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/TaskWindow.cs b/libs/cua-driver/tests/fixtures/apps/windows/wpf/TaskWindow.cs
new file mode 100644
index 0000000000..8373d517da
--- /dev/null
+++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/TaskWindow.cs
@@ -0,0 +1,122 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Windows;
+using System.Windows.Automation;
+using System.Windows.Controls;
+
+namespace CuaTestHarness.Wpf;
+
+/// <summary>
+/// Opt-in jev-use task window (RFC #4268), shown instead of the main window
+/// when CUA_WPF_TASK_STATE=&lt;path&gt; is set. It carries the same labeled
+/// controls as the AppKit harness task mode (Increment, Reset, I agree,
+/// Small/Medium/Large, Note, Save note, Exit) in a small window where every
+/// control is on screen, and atomically rewrites an app-owned JSON state file
+/// on every change. The jev-use task oracle reads that file; it never depends
+/// on Cua Driver output. Ordinary launches never create this window.
+/// </summary>
+public sealed class TaskWindow : Window
+{
+ public const string StateEnv = "CUA_WPF_TASK_STATE";
+ private const string StateSchema = "cua.wpf_task_state_v1";
+
+ private readonly string _statePath;
+ private readonly TextBlock _counterLabel = new() { Text = "counter=0" };
+ private readonly TextBox _note = new() { Width = 240 };
+ private int _counter;
+ private bool _agreed;
+ private string _size = "none";
+ private string? _savedNote;
+ private int _sequence;
+
+ public TaskWindow(string statePath)
+ {
+ _statePath = statePath;
+ Title = "CuaTestHarness WPF Tasks";
+ AutomationProperties.SetAutomationId(this, "wnd-tasks");
+ Width = 480;
+ Height = 360;
+ WindowStartupLocation = WindowStartupLocation.CenterScreen;
+
+ var root = new StackPanel { Margin = new Thickness(16) };
+ root.Children.Add(_counterLabel);
+
+ var counterRow = Row();
+ counterRow.Children.Add(MakeButton("Increment", "btn-increment", () => { _counter++; }));
+ counterRow.Children.Add(MakeButton("Reset", "btn-reset", () => { _counter = 0; }));
+ root.Children.Add(counterRow);
+
+ var agree = new CheckBox { Content = "I agree", Margin = new Thickness(0, 8, 0, 0) };
+ AutomationProperties.SetAutomationId(agree, "chk-agree");
+ agree.Checked += (_, _) => { _agreed = true; Publish(); };
+ agree.Unchecked += (_, _) => { _agreed = false; Publish(); };
+ root.Children.Add(agree);
+
+ var sizeRow = Row();
+ foreach (var title in new[] { "Small", "Medium", "Large" })
+ {
+ var radio = new RadioButton { Content = title, GroupName = "size", Margin = new Thickness(0, 0, 12, 0) };
+ AutomationProperties.SetAutomationId(radio, $"rad-size-{title.ToLowerInvariant()}");
+ radio.Checked += (_, _) => { _size = title.ToLowerInvariant(); Publish(); };
+ sizeRow.Children.Add(radio);
+ }
+ root.Children.Add(sizeRow);
+
+ var noteRow = Row();
+ // A UIA Name and no placeholder: an unnamed Edit falls back to its
+ // value as its label, which jev-use treats as unlabeled.
+ AutomationProperties.SetName(_note, "Note");
+ AutomationProperties.SetAutomationId(_note, "txt-note");
+ noteRow.Children.Add(_note);
+ noteRow.Children.Add(MakeButton("Save note", "btn-save-note", () => { _savedNote = _note.Text; }));
+ root.Children.Add(noteRow);
+
+ root.Children.Add(MakeButton("Exit", "btn-exit", () => Application.Current.Shutdown(0)));
+ Content = root;
+ Publish();
+ }
+
+ private static StackPanel Row() =>
+ new() { Orientation = Orientation.Horizontal, Margin = new Thickness(0, 8, 0, 0) };
+
+ private Button MakeButton(string label, string automationId, Action onClick)
+ {
+ var button = new Button
+ {
+ Content = label,
+ MinWidth = 90,
+ Margin = new Thickness(0, 0, 8, 0),
+ HorizontalAlignment = HorizontalAlignment.Left,
+ };
+ AutomationProperties.SetAutomationId(button, automationId);
+ button.Click += (_, _) => { onClick(); Publish(); };
+ return button;
+ }
+
+ private void Publish()
+ {
+ _counterLabel.Text = $"counter={_counter}";
+ _sequence++;
+ var state = new SortedDictionary<string, object?>(StringComparer.Ordinal)
+ {
+ ["schema"] = StateSchema,
+ ["pid"] = Environment.ProcessId,
+ ["seq"] = _sequence,
+ ["counter"] = _counter,
+ ["agreed"] = _agreed,
+ ["size"] = _size,
+ ["note_saved"] = _savedNote,
+ };
+ try
+ {
+ var temporaryPath = $"{_statePath}.{Environment.ProcessId}.tmp";
+ File.WriteAllText(temporaryPath, System.Text.Json.JsonSerializer.Serialize(state));
+ File.Move(temporaryPath, _statePath, true);
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"WPF task state publish failed: {ex.Message}");
+ }
+ }
+}