Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(cua-driver): live smoke of the agent loop on Windows and Linux (#4769)

Francesco Bonacci4c3fad87c1d5
diff --git a/.github/workflows/smoke-cua-driver-live.yml b/.github/workflows/smoke-cua-driver-live.yml
new file mode 100644
index 0000000000..25ef0445e5
--- /dev/null
+++ b/.github/workflows/smoke-cua-driver-live.yml
@@ -0,0 +1,89 @@
+name: "Smoke: Cua Driver live agent loop"
+
+# Builds cua-driver from the checked-out commit and drives a real GUI window on
+# Windows and on Linux (Xvfb): the lean get_window_state default and
+# full_output, `since` diffs, act-by-element_token, run_actions, and set_value
+# on combo boxes. The checks live in scripts/ci/cua-driver-live-smoke/smoke.py.
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - ".github/workflows/smoke-cua-driver-live.yml"
+ - "scripts/ci/cua-driver-live-smoke/**"
+ - "libs/cua-driver/rust/crates/cua-driver-core/src/window_state_view.rs"
+ - "libs/cua-driver/rust/crates/cua-driver-core/src/batch_tools.rs"
+
+permissions:
+ contents: read
+
+env:
+ CUA_TELEMETRY: "0"
+ DO_NOT_TRACK: "1"
+ CUA_DRIVER_RS_TELEMETRY_ENABLED: "false"
+
+concurrency:
+ group: smoke-cua-driver-live-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ linux:
+ name: "Linux (Xvfb + GTK3) / live agent loop"
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - uses: dtolnay/rust-toolchain@4716b85f2fac3e324e64fa2810f6b5c3905760a5 # 1.97.1
+ - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
+ with:
+ workspaces: "libs/cua-driver/rust -> target"
+ key: live-smoke
+ - name: Install GUI and Rust build dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends \
+ xvfb dbus-x11 at-spi2-core openbox python3-gi gir1.2-gtk-3.0 \
+ clang pkg-config libdbus-1-dev libpipewire-0.3-dev libspa-0.2-dev \
+ libei-dev libwayland-dev libxkbcommon-dev libx11-dev libxi-dev \
+ libxtst-dev libxext-dev libxdo-dev libssl-dev
+ - name: Build cua-driver
+ run: cargo build --release --locked -p cua-driver --manifest-path libs/cua-driver/rust/Cargo.toml
+ - name: Run live smoke
+ run: |
+ xvfb-run -a --server-args="-screen 0 1920x1080x24" \
+ dbus-run-session -- bash -c \
+ "openbox >/tmp/openbox.log 2>&1 & sleep 2; python3 scripts/ci/cua-driver-live-smoke/smoke.py --driver libs/cua-driver/rust/target/release/cua-driver --out artifacts/live-smoke/linux"
+ - name: Upload smoke evidence
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: live-smoke-linux
+ path: artifacts/live-smoke/linux
+ if-no-files-found: ignore
+ retention-days: 14
+
+ windows:
+ name: "Windows (WinForms) / live agent loop"
+ runs-on: windows-latest
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ - uses: dtolnay/rust-toolchain@4716b85f2fac3e324e64fa2810f6b5c3905760a5 # 1.97.1
+ - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
+ with:
+ workspaces: "libs/cua-driver/rust -> target"
+ key: live-smoke
+ - name: Build cua-driver
+ shell: pwsh
+ run: cargo build --release --locked -p cua-driver --manifest-path libs/cua-driver/rust/Cargo.toml
+ - name: Run live smoke
+ shell: pwsh
+ run: python scripts/ci/cua-driver-live-smoke/smoke.py --driver libs/cua-driver/rust/target/release/cua-driver.exe --out artifacts/live-smoke/windows
+ - name: Upload smoke evidence
+ if: always()
+ uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+ with:
+ name: live-smoke-windows
+ path: artifacts/live-smoke/windows
+ if-no-files-found: ignore
+ retention-days: 14
diff --git a/scripts/ci/cua-driver-live-smoke/fixture_gtk3.py b/scripts/ci/cua-driver-live-smoke/fixture_gtk3.py
new file mode 100644
index 0000000000..43d8343205
--- /dev/null
+++ b/scripts/ci/cua-driver-live-smoke/fixture_gtk3.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""GTK3 window for the cua-driver live smoke: a text entry, two buttons, a
+dropdown combo box and an editable combo box, each with a label that echoes
+its state so a fresh snapshot can verify every action."""
+
+import gi
+
+gi.require_version("Gtk", "3.0")
+from gi.repository import Gtk # noqa: E402
+
+
+def named(widget, name):
+ widget.get_accessible().set_name(name)
+ return widget
+
+
+def main():
+ win = Gtk.Window(title="Cua Live Smoke")
+ win.set_default_size(420, 760)
+ win.connect("destroy", Gtk.main_quit)
+ box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=4)
+ box.set_border_width(8)
+ win.add(box)
+
+ entry = named(Gtk.Entry(), "Name field")
+ status = Gtk.Label(label="Status: idle")
+ apply_button = Gtk.Button(label="Apply")
+ apply_button.connect(
+ "clicked", lambda _b: status.set_text("Applied: " + entry.get_text())
+ )
+
+ count = Gtk.Label(label="Count: 0")
+ clicks = {"n": 0}
+
+ def increment(_button):
+ clicks["n"] += 1
+ count.set_text(f"Count: {clicks['n']}")
+
+ increment_button = Gtk.Button(label="Increment")
+ increment_button.connect("clicked", increment)
+
+ color = Gtk.Label(label="Color: Red")
+ color_combo = named(Gtk.ComboBoxText(), "Color")
+ for item in ("Red", "Green", "Blue"):
+ color_combo.append_text(item)
+ color_combo.set_active(0)
+ color_combo.connect(
+ "changed", lambda c: color.set_text("Color: " + (c.get_active_text() or ""))
+ )
+
+ size = Gtk.Label(label="Size: Small")
+ size_combo = named(Gtk.ComboBoxText.new_with_entry(), "Size")
+ for item in ("Small", "Medium", "Large"):
+ size_combo.append_text(item)
+ size_combo.set_active(0)
+ size_combo.get_child().get_accessible().set_name("Size entry")
+ size_combo.connect(
+ "changed", lambda c: size.set_text("Size: " + (c.get_active_text() or ""))
+ )
+
+ for widget in (
+ entry, apply_button, status, increment_button, count,
+ color_combo, color, size_combo, size,
+ ):
+ box.pack_start(widget, False, False, 0)
+ # Padding rows keep a one-row change small next to the whole tree, as in a
+ # real window, so `since` answers with a diff rather than a full read.
+ for row in range(1, 16):
+ box.pack_start(Gtk.Label(label=f"Row {row}"), False, False, 0)
+
+ win.show_all()
+ Gtk.main()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/ci/cua-driver-live-smoke/fixture_winforms.ps1 b/scripts/ci/cua-driver-live-smoke/fixture_winforms.ps1
new file mode 100644
index 0000000000..92fcdf857c
--- /dev/null
+++ b/scripts/ci/cua-driver-live-smoke/fixture_winforms.ps1
@@ -0,0 +1,74 @@
+# WinForms window for the cua-driver live smoke: a text box, two buttons, a
+# drop-down-list combo box and an editable combo box, each with a label that
+# echoes its state so a fresh snapshot can verify every action.
+Add-Type -AssemblyName System.Windows.Forms
+[System.Windows.Forms.Application]::EnableVisualStyles()
+
+$form = New-Object System.Windows.Forms.Form
+$form.Text = "Cua Live Smoke"
+$form.Width = 440
+$form.Height = 860
+$form.StartPosition = "Manual"
+$form.Location = New-Object System.Drawing.Point(40, 40)
+
+$panel = New-Object System.Windows.Forms.FlowLayoutPanel
+$panel.Dock = "Fill"
+$panel.FlowDirection = "TopDown"
+$panel.WrapContents = $false
+$form.Controls.Add($panel)
+
+function Add-Label([string]$text) {
+ $label = New-Object System.Windows.Forms.Label
+ $label.Text = $text
+ $label.AutoSize = $true
+ $panel.Controls.Add($label)
+ return $label
+}
+
+[void](Add-Label "Name field")
+$entry = New-Object System.Windows.Forms.TextBox
+$entry.Width = 300
+$entry.AccessibleName = "Name field"
+$panel.Controls.Add($entry)
+
+$apply = New-Object System.Windows.Forms.Button
+$apply.Text = "Apply"
+$panel.Controls.Add($apply)
+$status = Add-Label "Status: idle"
+$apply.Add_Click({ $status.Text = "Applied: " + $entry.Text })
+
+$increment = New-Object System.Windows.Forms.Button
+$increment.Text = "Increment"
+$panel.Controls.Add($increment)
+$count = Add-Label "Count: 0"
+$script:clicks = 0
+$increment.Add_Click({
+ $script:clicks += 1
+ $count.Text = "Count: $($script:clicks)"
+})
+
+[void](Add-Label "Color")
+$colorCombo = New-Object System.Windows.Forms.ComboBox
+$colorCombo.DropDownStyle = "DropDownList"
+$colorCombo.AccessibleName = "Color"
+[void]$colorCombo.Items.AddRange(@("Red", "Green", "Blue"))
+$colorCombo.SelectedIndex = 0
+$panel.Controls.Add($colorCombo)
+$color = Add-Label "Color: Red"
+$colorCombo.Add_SelectedIndexChanged({ $color.Text = "Color: " + $colorCombo.Text })
+
+[void](Add-Label "Size")
+$sizeCombo = New-Object System.Windows.Forms.ComboBox
+$sizeCombo.DropDownStyle = "DropDown"
+$sizeCombo.AccessibleName = "Size"
+[void]$sizeCombo.Items.AddRange(@("Small", "Medium", "Large"))
+$sizeCombo.SelectedIndex = 0
+$panel.Controls.Add($sizeCombo)
+$size = Add-Label "Size: Small"
+$sizeCombo.Add_TextChanged({ $size.Text = "Size: " + $sizeCombo.Text })
+
+# Padding rows keep a one-row change small next to the whole tree, as in a
+# real window, so `since` answers with a diff rather than a full read.
+foreach ($row in 1..15) { [void](Add-Label "Row $row") }
+
+[System.Windows.Forms.Application]::Run($form)
diff --git a/scripts/ci/cua-driver-live-smoke/smoke.py b/scripts/ci/cua-driver-live-smoke/smoke.py
new file mode 100644
index 0000000000..afb8dc767d
--- /dev/null
+++ b/scripts/ci/cua-driver-live-smoke/smoke.py
@@ -0,0 +1,565 @@
+#!/usr/bin/env python3
+"""Live smoke for the cua-driver agent loop on a real desktop.
+
+Starts a cua-driver daemon, talks MCP to it over `cua-driver mcp`, opens the
+fixture window (GTK3 on Linux, WinForms on Windows) and checks:
+
+ * get_window_state's default (lean) output and `full_output: true`,
+ * the act-by-element_token loop (snapshot, click a token, fresh snapshot),
+ * `since` (a diff against an earlier snapshot, and `no change`),
+ * run_actions (one batch with an end-of-batch observation),
+ * set_value on a drop-down combo box and on an editable combo box.
+
+The first five checks are required: each is `pass` or `fail`. set_value on
+combo boxes and the probes are best effort and never fail the job; each is
+`pass`, `not_possible` (the driver refused with an explicit error) or `bug`
+(the driver claimed success, or failed internally after acting, and the app's
+change handler never saw the value).
+
+Writes results.json, summary.md and the raw responses to --out, and appends
+summary.md to $GITHUB_STEP_SUMMARY when set. Exits 1 when any check fails.
+"""
+
+import argparse
+import json
+import os
+import queue
+import re
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+TITLE = "Cua Live Smoke"
+WINDOWS = sys.platform == "win32"
+
+# Row roles, matched against the role words before the quoted name. AT-SPI
+# names a GtkEntry `text` and a label `label`; UIA calls a text box `Edit`
+# and a label `Text`.
+BUTTON = r"(push )?button"
+ENTRY = r"edit" if WINDOWS else r"text|entry"
+COMBO = r"combo ?box"
+
+
+class Mcp:
+ """Line-delimited JSON-RPC over the stdio of `cua-driver mcp`."""
+
+ def __init__(self, driver, socket, env, log):
+ self.proc = subprocess.Popen(
+ [driver, "mcp", "--socket", socket],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=log,
+ env=env,
+ text=True,
+ encoding="utf-8",
+ )
+ self.lines = queue.Queue()
+ threading.Thread(target=self._read, daemon=True).start()
+ self.next_id = 1
+
+ def _read(self):
+ for line in self.proc.stdout:
+ self.lines.put(line)
+
+ def request(self, method, params, timeout=120):
+ rid = self.next_id
+ self.next_id += 1
+ self.proc.stdin.write(
+ json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params})
+ + "\n"
+ )
+ self.proc.stdin.flush()
+ deadline = time.monotonic() + timeout
+ while True:
+ left = deadline - time.monotonic()
+ if left <= 0:
+ raise TimeoutError(f"{method} timed out after {timeout}s")
+ try:
+ line = self.lines.get(timeout=left)
+ except queue.Empty:
+ continue
+ try:
+ message = json.loads(line)
+ except ValueError:
+ continue
+ if message.get("id") == rid:
+ return message
+
+ def close(self):
+ try:
+ self.proc.stdin.close()
+ self.proc.wait(timeout=10)
+ except Exception:
+ self.proc.kill()
+
+
+class Call:
+ def __init__(self, tool, args, message):
+ self.tool = tool
+ self.args = args
+ result = message.get("result") or {}
+ error = message.get("error")
+ self.is_error = bool(error) or bool(result.get("isError"))
+ self.structured = result.get("structuredContent") or {}
+ self.text = "\n".join(
+ part.get("text", "")
+ for part in result.get("content") or []
+ if part.get("type") == "text"
+ )
+ if error:
+ self.text = json.dumps(error)
+ self.size = len(self.text) + len(json.dumps(self.structured))
+
+ def first_line(self):
+ return (self.text.strip().splitlines() or [""])[0][:300]
+
+
+class Smoke:
+ def __init__(self, mcp, out):
+ self.mcp = mcp
+ self.out = out
+ self.results = []
+ self.calls = []
+ self.pid = None
+ self.window_id = None
+
+ def call(self, tool, args, timeout=120):
+ message = self.mcp.request(
+ "tools/call", {"name": tool, "arguments": args}, timeout=timeout
+ )
+ result = Call(tool, args, message)
+ self.calls.append(
+ {
+ "tool": tool,
+ "args": args,
+ "is_error": result.is_error,
+ "text": result.text[:4000],
+ "structured": result.structured,
+ }
+ )
+ return result
+
+ def record(self, check, status, evidence):
+ self.results.append({"check": check, "status": status, "evidence": evidence})
+ print(f"[{status.upper()}] {check}: {evidence}", flush=True)
+
+ def read(self, **extra):
+ args = {"pid": self.pid, "window_id": self.window_id}
+ args.update(extra)
+ return self.call("get_window_state", args)
+
+ def wait_for_text(self, needle, timeout=10):
+ """Fresh full reads until `needle` shows up; returns the last read."""
+ deadline = time.monotonic() + timeout
+ while True:
+ state = self.read()
+ if needle in tree(state):
+ return state
+ if time.monotonic() >= deadline:
+ return state
+ time.sleep(0.5)
+
+
+def tree(state):
+ return state.structured.get("tree_markdown") or state.text
+
+
+def rows(markdown):
+ for line in markdown.splitlines():
+ match = re.match(r"\s*-\s*\[(\d+)\]\s*(.*)", line)
+ if match:
+ yield int(match.group(1)), match.group(2)
+
+
+def find_row(markdown, role, label=None, after=None):
+ """Index of the first row whose role fully matches `role` (regex) and
+ whose text contains `label`, optionally only rows after index `after`."""
+ for index, text in rows(markdown):
+ if after is not None and index <= after:
+ continue
+ row_role = re.split(r'["\[]', text, maxsplit=1)[0].strip()
+ if not re.fullmatch(role, row_role, re.IGNORECASE):
+ continue
+ if label is None or label.lower() in text.lower():
+ return index
+ return None
+
+
+def first(*values):
+ return next((v for v in values if v is not None), None)
+
+
+def token(state, index):
+ return f"{state.structured['snapshot_id']}:{index}"
+
+
+def start_daemon(driver, socket, env, log):
+ daemon = subprocess.Popen(
+ [driver, "serve", "--socket", socket, "--no-permissions-gate", "--no-overlay"],
+ stdin=subprocess.DEVNULL,
+ stdout=log,
+ stderr=log,
+ env=env,
+ )
+ deadline = time.monotonic() + 90
+ while time.monotonic() < deadline:
+ if daemon.poll() is not None:
+ raise RuntimeError(f"daemon exited with {daemon.returncode} during startup")
+ probe = subprocess.run(
+ [driver, "--socket", socket, "call", "get_config", "{}"],
+ capture_output=True,
+ env=env,
+ )
+ if probe.returncode == 0:
+ return daemon
+ time.sleep(1)
+ raise RuntimeError("daemon did not answer get_config within 90 s")
+
+
+def start_fixture(log):
+ if WINDOWS:
+ command = [
+ "powershell.exe", "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass",
+ "-File", str(HERE / "fixture_winforms.ps1"),
+ ]
+ return subprocess.Popen(
+ command, stdout=log, stderr=log, creationflags=0x08000000 # CREATE_NO_WINDOW
+ )
+ return subprocess.Popen(
+ ["/usr/bin/python3", str(HERE / "fixture_gtk3.py")], stdout=log, stderr=log
+ )
+
+
+def find_window(smoke, pid):
+ deadline = time.monotonic() + 60
+ while time.monotonic() < deadline:
+ listed = smoke.call("list_windows", {"pid": pid})
+ for window in listed.structured.get("windows") or []:
+ if window.get("pid") == pid and TITLE in (window.get("title") or ""):
+ return window["window_id"]
+ time.sleep(1)
+ raise RuntimeError(f"no '{TITLE}' window for pid {pid} within 60 s")
+
+
+def run_checks(smoke):
+ # Wait until the accessibility tree is populated.
+ deadline = time.monotonic() + 60
+ while True:
+ lean = smoke.read()
+ if find_row(tree(lean), BUTTON, "Increment") is not None:
+ break
+ if time.monotonic() >= deadline:
+ raise RuntimeError("fixture tree never showed the Increment button:\n" + lean.text[:3000])
+ time.sleep(1)
+ print("---- lean tree ----\n" + tree(lean) + "\n-------------------", flush=True)
+
+ # 1. Lean default.
+ s = lean.structured
+ problems = []
+ if lean.is_error:
+ problems.append("call failed: " + lean.first_line())
+ if s.get("tree_format") != "markdown":
+ problems.append(f"tree_format={s.get('tree_format')!r}")
+ if "elements" in s:
+ problems.append("structured `elements` present")
+ if not s.get("snapshot_id"):
+ problems.append("no snapshot_id")
+ if "element_token for row [N]" not in lean.text:
+ problems.append("no element_token hint in the text block")
+ smoke.record(
+ "get_window_state default is lean",
+ "fail" if problems else "pass",
+ "; ".join(problems)
+ or f"tree_format=markdown, no `elements`, snapshot_id={s.get('snapshot_id')}, "
+ f"{lean.size} chars",
+ )
+
+ # 2. full_output: true.
+ full = smoke.read(full_output=True)
+ f = full.structured
+ elements = f.get("elements") or []
+ problems = []
+ if full.is_error:
+ problems.append("call failed: " + full.first_line())
+ if not elements:
+ problems.append("no `elements`")
+ elif not any(e.get("element_token") for e in elements):
+ problems.append("`elements` carry no element_token")
+ if not f.get("tree_markdown"):
+ problems.append("no tree_markdown")
+ if full.size <= lean.size:
+ problems.append(f"full ({full.size} chars) not larger than lean ({lean.size})")
+ smoke.record(
+ "get_window_state full_output:true returns the full tree",
+ "fail" if problems else "pass",
+ "; ".join(problems)
+ or f"{len(elements)} elements + tree_markdown, {full.size} chars vs lean {lean.size} "
+ f"({full.size / max(lean.size, 1):.1f}x)",
+ )
+
+ # 3. Act by element_token, verify with a fresh snapshot.
+ before = smoke.read()
+ increment = find_row(tree(before), BUTTON, "Increment")
+ click_args = {"pid": smoke.pid, "element_token": token(before, increment)}
+ click = smoke.call("click", click_args)
+ after = smoke.wait_for_text("Count: 1")
+ if click.is_error:
+ smoke.record("act by element_token (click) + fresh snapshot", "fail",
+ f"click {json.dumps(click_args)} -> {click.first_line()}")
+ elif "Count: 1" not in tree(after):
+ smoke.record("act by element_token (click) + fresh snapshot", "fail",
+ f"click ok ({click.first_line()}) but fresh snapshot lacks 'Count: 1'")
+ else:
+ smoke.record("act by element_token (click) + fresh snapshot", "pass",
+ f"click element_token={click_args['element_token']} -> fresh snapshot "
+ f"{after.structured.get('snapshot_id')} shows 'Count: 1'")
+
+ # 4. since: diff against the pre-click snapshot, then `no change`.
+ since_id = before.structured["snapshot_id"]
+ diff = smoke.read(since=since_id)
+ d = diff.structured
+ diff_text = d.get("tree_diff") or ""
+ problems = []
+ if diff.is_error:
+ problems.append("call failed: " + diff.first_line())
+ if d.get("since_status") != "diff":
+ problems.append(f"since_status={d.get('since_status')!r}")
+ if "Count: 1" not in diff_text:
+ problems.append("diff does not mention 'Count: 1'")
+ if "tree_markdown" in d:
+ problems.append("diff read still carries the full tree_markdown")
+ still = smoke.read(since=d.get("snapshot_id") or "")
+ if still.structured.get("since_status") != "no_change":
+ problems.append(
+ f"second read since={d.get('snapshot_id')} gave "
+ f"since_status={still.structured.get('since_status')!r}, expected 'no_change'"
+ )
+ smoke.record(
+ "since returns a diff, then no_change",
+ "fail" if problems else "pass",
+ "; ".join(problems)
+ or f"since={since_id}: diff_counts={d.get('diff_counts')}, {diff.size} chars vs "
+ f"full read {after.size}; repeat read: no_change",
+ )
+
+ # 5. run_actions: set a field, click Apply, click Increment, observe once.
+ state = smoke.read()
+ md = tree(state)
+ targets = {
+ "entry": first(find_row(md, ENTRY, "Name field"), find_row(md, ENTRY)),
+ "apply": find_row(md, BUTTON, "Apply"),
+ "increment": find_row(md, BUTTON, "Increment"),
+ }
+ if None in targets.values():
+ smoke.record("run_actions batch", "fail", f"fixture rows not found: {targets}")
+ else:
+ batch_args = {
+ "steps": [
+ {"tool": "set_value", "args": {"pid": smoke.pid, "element_token": token(state, targets["entry"]), "value": "Ada"}},
+ {"tool": "click", "args": {"pid": smoke.pid, "element_token": token(state, targets["apply"])}},
+ {"tool": "click", "args": {"pid": smoke.pid, "element_token": token(state, targets["increment"])}},
+ ],
+ "observe": {"pid": smoke.pid, "window_id": smoke.window_id},
+ }
+ batch = smoke.call("run_actions", batch_args)
+ b = batch.structured
+ observed = ((b.get("observation") or {}).get("state") or {}).get("tree_markdown") or ""
+ problems = []
+ if batch.is_error or not b.get("ok"):
+ problems.append(f"batch failed: {json.dumps(b.get('steps'))[:600]} {batch.first_line()}")
+ if b.get("executed") != 3:
+ problems.append(f"executed={b.get('executed')}")
+ if not (b.get("observation") or {}).get("ok"):
+ problems.append("no observation")
+ lagged = [n for n in ("Applied: Ada", "Count: 2") if n not in observed]
+ if lagged and not problems:
+ fresh = smoke.wait_for_text("Count: 2")
+ missing = [n for n in lagged if n not in tree(fresh)]
+ if missing:
+ problems.append(f"neither the observation nor a fresh read shows {missing}")
+ smoke.record(
+ "run_actions batch (set_value + 2 clicks + observe)",
+ "fail" if problems else "pass",
+ "; ".join(problems)
+ or "3/3 steps ok; "
+ + ("observation shows 'Applied: Ada' and 'Count: 2'" if not lagged
+ else f"observation lagged on {lagged}, fresh read confirms"),
+ )
+
+ # 6. set_value on combo boxes.
+ combo_check(smoke, "Color", "Blue", editable=False)
+ combo_check(smoke, "Size", "Large", editable=True)
+
+ # Probe (informational): element_token alone, no pid (#4739 on macOS).
+ state = smoke.read()
+ increment = find_row(tree(state), BUTTON, "Increment")
+ probe = smoke.call("click", {"element_token": token(state, increment)})
+ smoke.record(
+ "probe: click with element_token and no pid",
+ "not_possible" if probe.is_error else "pass",
+ probe.first_line(),
+ )
+
+ # Probe (informational, last because it may leave a menu open): the
+ # drop-down combo with delivery_mode foreground, which the background
+ # refusal on Linux recommends.
+ state = smoke.read()
+ combo = find_row(tree(state), COMBO, "Color")
+ status, evidence = set_value_attempt(
+ smoke, state, combo, "Color", "Green", delivery_mode="foreground"
+ )
+ smoke.record(
+ "probe: set_value on the drop-down combo box with delivery_mode foreground",
+ status,
+ evidence,
+ )
+
+
+def row_text(markdown, index):
+ return next((text for i, text in rows(markdown) if i == index), "")
+
+
+def set_value_attempt(smoke, state, index, name, value, **extra):
+ """set_value on row `index`, then a fresh read for '<name>: <value>'.
+ Returns (status, evidence): pass, not_possible (explicit refusal), or
+ bug (the call claimed success, or failed internally after acting, and the
+ app's change handler did not see the new value)."""
+ args = {"pid": smoke.pid, "element_token": token(state, index), "value": value}
+ args.update(extra)
+ result = smoke.call("set_value", args)
+ expected = f"{name}: {value}"
+ fresh = smoke.wait_for_text(expected, timeout=5 if result.is_error else 10)
+ if expected in tree(fresh):
+ return "pass", f"set_value on row [{index}] -> fresh snapshot shows '{expected}'"
+ control = row_text(tree(fresh), index)
+ if result.is_error and result.structured.get("code") != "action_outcome_mismatch":
+ return "not_possible", f"[{index}] {result.first_line()}"
+ claim = ("failed internally: " if result.is_error else "reported success: ") + result.first_line()
+ return "bug", (
+ f"set_value{' ' + json.dumps(extra) if extra else ''} on row [{index}] {claim}; "
+ f"the control now reads `{control}` but the app's change handler never ran "
+ f"(no '{expected}')"
+ )
+
+
+def combo_check(smoke, name, value, editable):
+ """set_value on a combo box is best effort ("where possible"): it records
+ pass, not_possible or bug and never fails the job."""
+ kind = "an editable combo box" if editable else "a drop-down combo box"
+ check = f"set_value on {kind} ({name} -> {value})"
+ state = smoke.read()
+ md = tree(state)
+ combo = find_row(md, COMBO, name)
+ if combo is None:
+ smoke.record(check, "fail", f"no combo row named {name!r} in the tree")
+ return
+ candidates = [combo]
+ if editable:
+ # GTK exposes the entry as a child row; UIA exposes an Edit child.
+ child = find_row(md, ENTRY, after=combo)
+ if child is not None:
+ candidates.append(child)
+ outcomes = []
+ for index in candidates:
+ status, evidence = set_value_attempt(smoke, state, index, name, value)
+ outcomes.append((status, evidence))
+ if status == "pass":
+ break
+ state = smoke.read()
+ status = next((s for s in ("pass", "bug", "not_possible") if any(o[0] == s for o in outcomes)))
+ smoke.record(check, status, "; ".join(e for st, e in outcomes if st == status))
+
+
+def write_reports(smoke, out, platform_name):
+ out.mkdir(parents=True, exist_ok=True)
+ (out / "results.json").write_text(json.dumps(smoke.results, indent=2), encoding="utf-8")
+ (out / "calls.json").write_text(json.dumps(smoke.calls, indent=2, default=str), encoding="utf-8")
+ lines = [
+ f"## cua-driver live smoke: {platform_name}",
+ "",
+ "`fail` fails the job. `bug` (a best-effort check where the driver claimed "
+ "success, or failed internally after acting, without the app seeing the "
+ "change) and `not_possible` (an explicit refusal) are recorded only.",
+ "",
+ "| Check | Result | Evidence |",
+ "| --- | --- | --- |",
+ ]
+ for r in smoke.results:
+ evidence = r["evidence"].replace("|", "\\|").replace("\n", " ")
+ lines.append(f"| {r['check']} | {r['status']} | {evidence} |")
+ if r["status"] == "bug" and os.environ.get("GITHUB_ACTIONS"):
+ print(f"::warning title=cua-driver bug: {r['check']}::{evidence}")
+ summary = "\n".join(lines) + "\n"
+ (out / "summary.md").write_text(summary, encoding="utf-8")
+ step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
+ if step_summary:
+ with open(step_summary, "a", encoding="utf-8") as handle:
+ handle.write(summary + "\n")
+ print(summary)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--driver", required=True, help="path to the cua-driver binary")
+ parser.add_argument("--out", required=True, help="artifact directory")
+ options = parser.parse_args()
+ # Tool text carries non-ASCII marks; a Windows console is cp1252.
+ sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+
+ out = Path(options.out)
+ out.mkdir(parents=True, exist_ok=True)
+ driver = str(Path(options.driver).resolve())
+ socket = r"\\.\pipe\cua-driver-live-smoke" if WINDOWS else str(out / "d.sock")
+ env = dict(os.environ)
+ env.update(
+ {
+ "CUA_DRIVER_RS_TELEMETRY_ENABLED": "false",
+ # Disposable CI desktop: no interactive approvals.
+ "CUA_DRIVER_PERMISSION_MODE": "unrestricted",
+ "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS": "1",
+ }
+ )
+ daemon_log = open(out / "daemon.log", "w", encoding="utf-8")
+ mcp_log = open(out / "mcp.log", "w", encoding="utf-8")
+ fixture_log = open(out / "fixture.log", "w", encoding="utf-8")
+
+ daemon = start_daemon(driver, socket, env, daemon_log)
+ fixture = None
+ mcp = Mcp(driver, socket, env, mcp_log)
+ smoke = Smoke(mcp, out)
+ platform_name = "Windows (WinForms)" if WINDOWS else "Linux Xvfb (GTK3)"
+ try:
+ init = mcp.request("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "live-smoke", "version": "1"}})
+ if "error" in init:
+ raise RuntimeError(f"initialize failed: {init['error']}")
+ mcp.proc.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n")
+ mcp.proc.stdin.flush()
+ version = smoke.call("get_config", {})
+ print("driver config:", version.text[:800], flush=True)
+
+ fixture = start_fixture(fixture_log)
+ smoke.pid = fixture.pid
+ smoke.window_id = find_window(smoke, fixture.pid)
+ print(f"fixture pid={smoke.pid} window_id={smoke.window_id}", flush=True)
+ run_checks(smoke)
+ except Exception as error: # noqa: BLE001 - every error becomes a failed row
+ smoke.record("smoke harness", "fail", f"{type(error).__name__}: {error}")
+ finally:
+ write_reports(smoke, out, platform_name)
+ mcp.close()
+ for proc in (fixture, daemon):
+ if proc is not None and proc.poll() is None:
+ proc.terminate()
+ try:
+ proc.wait(timeout=10)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+
+ return 1 if any(r["status"] == "fail" for r in smoke.results) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())