Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
test(computer): measure independently verified tasks (#36216)
Kelvin Amoaba596e5f52d635
diff --git a/.github/workflows/ci-closed-pr-caches.yml b/.github/workflows/ci-closed-pr-caches.ymlindex a2091211188..37dc522417f 100644--- a/.github/workflows/ci-closed-pr-caches.yml+++ b/.github/workflows/ci-closed-pr-caches.yml@@ -28,6 +28,7 @@ jobs:['ssh-hostile-hosts', 'ssh-hostile-hosts.yml'],['mobile', 'mobile.yml'],['computer-e2e', 'computer-e2e.yml'],+ ['computer-task-benchmark', 'computer-task-benchmark.yml'],['computer-macos-native', 'computer-macos-native.yml'],['wsl-orcad-e2e', 'wsl-orcad-e2e.yml'],['performance-contracts', 'performance-contracts.yml'],diff --git a/.github/workflows/computer-task-benchmark.yml b/.github/workflows/computer-task-benchmark.ymlnew file mode 100644index 00000000000..baec1d44e87--- /dev/null+++ b/.github/workflows/computer-task-benchmark.yml@@ -0,0 +1,158 @@+name: Computer-use task benchmark++on:+ pull_request:+ types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]+ paths:+ - 'config/benchmarks/computer-use/**'+ - 'native/computer-use-linux/**'+ - 'native/computer-use-windows/**'+ - '.github/workflows/computer-task-benchmark.yml'+ workflow_dispatch:++permissions:+ contents: read++concurrency:+ group: computer-task-benchmark-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: true++env:+ ORCA_BACKGROUND_LAUNCH: '1'+ CUA_TAG: cua-driver-rs-v0.38.0+ CUA_DRIVER_RS_TELEMETRY_ENABLED: '0'+ DO_NOT_TRACK: '1'+ CUA_DRIVER_RS_UPDATE_CHECK: '0'+ ELECTRON_VERSION: '43.8.1'+ BASELINE_SHA: 030c88deecf3adc212d22f57b085e4bd5c6e48e6++jobs:+ contract:+ if: github.event_name != 'pull_request' || github.event.pull_request.draft != true+ runs-on: ${{ matrix.os }}+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, windows-latest, macos-latest]+ steps:+ - uses: actions/checkout@v6+ with:+ persist-credentials: false+ - uses: actions/setup-python@v6+ with:+ python-version: '3.12'+ - run: python -m unittest discover -s config/benchmarks/computer-use -p 'test_*.py'++ linux:+ if: github.event_name != 'pull_request' || github.event.pull_request.draft != true+ runs-on: ubuntu-latest+ timeout-minutes: 40+ strategy:+ fail-fast: false+ matrix:+ target: [fixture, electron]+ env:+ BENCH_TARGET: ${{ matrix.target }}+ ORCA_COMPUTER_BENCH_ISOLATED: '1'+ ACCESSIBILITY_ENABLED: '1'+ steps:+ - uses: actions/checkout@v6+ with:+ persist-credentials: false+ ref: ${{ github.event.pull_request.head.sha || github.sha }}+ - name: Fetch immutable baseline+ run: git fetch --no-tags --depth=1 origin "$BASELINE_SHA"+ - name: Install isolated desktop and accessibility libraries+ run: |+ sudo apt-get update+ sudo apt-get install -y --no-install-recommends xvfb openbox dbus-x11 at-spi2-core python3-gi \+ gir1.2-atspi-2.0 gir1.2-gtk-3.0 python3-tk xdotool xclip ripgrep libnss3 libgbm1 libasound2t64+ - name: Download and verify pinned cua-driver+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ mkdir -p "$RUNNER_TEMP/cua-dl" "$RUNNER_TEMP/cua"+ cd "$RUNNER_TEMP/cua-dl"+ asset='cua-driver-rs-0.38.0-linux-x86_64-binary.tar.gz'+ gh release download "$CUA_TAG" --repo trycua/cua --pattern checksums.txt --pattern "$asset"+ rg " $asset\$" checksums.txt | sha256sum -c -+ tar -xzf "$asset" -C "$RUNNER_TEMP/cua"+ - name: Install Electron+ if: matrix.target == 'electron'+ run: |+ npm install --no-save --no-package-lock --prefix "$RUNNER_TEMP/electron" "electron@$ELECTRON_VERSION"+ node "$RUNNER_TEMP/electron/node_modules/electron/install.js"+ - name: Run task comparison on Xvfb+ run: |+ export CANDIDATE_SHA="$(git rev-parse HEAD)"+ xvfb-run -a --server-args='-screen 0 1280x1024x24' dbus-run-session -- bash -euc '+ openbox >"$RUNNER_TEMP/openbox.log" 2>&1 &+ /usr/bin/python3 config/benchmarks/computer-use/bench.py --target "$BENCH_TARGET" \+ --baseline "$BASELINE_SHA" --candidate "$CANDIDATE_SHA" \+ --cua "$RUNNER_TEMP/cua/cua-driver" \+ --electron "$RUNNER_TEMP/electron/node_modules/electron/dist/electron" \+ --out "out/task-linux-$BENCH_TARGET" --n 3 --budget 1800+ '+ - if: always()+ uses: actions/upload-artifact@v4+ with:+ name: computer-task-linux-${{ matrix.target }}+ path: out/task-linux-${{ matrix.target }}+ if-no-files-found: warn++ windows:+ if: github.event_name != 'pull_request' || github.event.pull_request.draft != true+ runs-on: windows-latest+ timeout-minutes: 40+ strategy:+ fail-fast: false+ matrix:+ target: [fixture, electron]+ env:+ BENCH_TARGET: ${{ matrix.target }}+ ORCA_COMPUTER_BENCH_ISOLATED: '1'+ steps:+ - uses: actions/checkout@v6+ with:+ persist-credentials: false+ ref: ${{ github.event.pull_request.head.sha || github.sha }}+ - uses: actions/setup-python@v6+ with:+ python-version: '3.12'+ - name: Fetch immutable baseline+ run: git fetch --no-tags --depth=1 origin $env:BASELINE_SHA+ - name: Download and verify pinned cua-driver+ env:+ GH_TOKEN: ${{ github.token }}+ shell: pwsh+ run: |+ $download = Join-Path $env:RUNNER_TEMP 'cua-dl'+ $destination = Join-Path $env:RUNNER_TEMP 'cua'+ New-Item -ItemType Directory -Force $download, $destination | Out-Null+ $asset = 'cua-driver-rs-0.38.0-windows-x86_64-binary.zip'+ gh release download $env:CUA_TAG --repo trycua/cua --pattern checksums.txt --pattern $asset --dir $download+ $matches = @(Select-String -Path (Join-Path $download 'checksums.txt') -Pattern " $([regex]::Escape($asset))$")+ if ($matches.Count -ne 1) { throw 'expected exactly one checksum' }+ $expected = $matches[0].Line.Split(' ')[0]+ $actual = (Get-FileHash (Join-Path $download $asset) -Algorithm SHA256).Hash.ToLower()+ if ($expected -ne $actual) { throw 'cua-driver checksum mismatch' }+ Expand-Archive (Join-Path $download $asset) -DestinationPath $destination+ - name: Install Electron+ if: matrix.target == 'electron'+ run: |+ npm install --no-save --no-package-lock --prefix "$env:RUNNER_TEMP\electron" "electron@$env:ELECTRON_VERSION"+ node "$env:RUNNER_TEMP\electron\node_modules\electron\install.js"+ - name: Run task comparison on disposable Windows desktop+ run: |+ $candidate = git rev-parse HEAD+ python config/benchmarks/computer-use/bench.py --target $env:BENCH_TARGET `+ --baseline $env:BASELINE_SHA --candidate $candidate `+ --cua "$env:RUNNER_TEMP\cua\cua-driver.exe" `+ --electron "$env:RUNNER_TEMP\electron\node_modules\electron\dist\electron.exe" `+ --out "out/task-windows-$env:BENCH_TARGET" --n 3 --budget 1800+ - if: always()+ uses: actions/upload-artifact@v4+ with:+ name: computer-task-windows-${{ matrix.target }}+ path: out/task-windows-${{ matrix.target }}+ if-no-files-found: warndiff --git a/config/benchmarks/computer-use/README.md b/config/benchmarks/computer-use/README.mdnew file mode 100644index 00000000000..f58895cb13d--- /dev/null+++ b/config/benchmarks/computer-use/README.md@@ -0,0 +1,78 @@+# Computer-use task benchmark++Compares Orca's direct Linux AT-SPI and Windows UI Automation providers with+cua-driver 0.38.0. This does not measure the CLI, sidecar, model, or complete agent+workflows. It does not install cua-driver in Orca.++Run the `Computer-use task benchmark` GitHub Actions workflow on a branch. Native+input runs only on its disposable Windows runner or Linux Xvfb desktop. The runner+refuses to run on the developer's desktop. Keep `ORCA_BACKGROUND_LAUNCH=1`.++The first matrix covers GTK/WinForms and Electron fixtures. Editor applications,+Wayland, macOS and additional Windows control frameworks need separate coverage.+Do not generalize these results to every application on a platform.++## Measurement contract++- Baseline and candidate runtimes come from full commit SHAs, copied before the+ run. Artifacts include source hashes, cua executable hash/version, image,+ seed, complete trial plan, responses, snapshots and independent fixture states.+- Each individual trial launches a fresh target, sentinel and provider. Process+ groups on Linux and jobs on Windows contain children, including timed-out calls.+ The provider is terminated and reaped before the terminal oracle read. No retry+ follows an uncertain mutation.+- Reset carries a unique nonce bound to the fixture instance and PID. The fixture+ acknowledges after applying text, selection, logical focus, counters, combo,+ scroll and menu state. Sentinel foreground and stacking are independently checked.+- Invalid preparation executes no provider operation, is excluded from the scored+ denominator, and makes the run incomplete with nonzero exit status. A missing or+ ambiguous control after valid setup is a scored coverage failure.+- Strict selectors require a unique matching role and label. References are fresh+ before the task and after every step. There is no label-only or role-only fallback.+- Completion requires exact independent final state, including unchanged decoy text+ and exactly one click/submit. Raw text is never cleaned. API errors, missing final+ observations and observed interference remain separate from task completion.+- The observer starts before provider initialization and initial selector snapshot,+ and stays active through all steps, reads, shutdown and settling. No sentinel+ refocusing happens during measurement. Idle and deliberate-steal controls must pass.+- Complete tasks run without simulated user typing. Separate click and scroll probes+ type `q` into the sentinel. Event counters preserve evidence even if agent input+ selects or deletes text. Observations sample roughly every 4 ms; they cannot prove+ the absence of shorter focus/pointer/clipboard changes. X11 clipboard observations+ detect ownership changes, not all changes made by the same owner.+- Engine order rotates across rounds from a recorded seed. Use a multiple of three+ rounds for balanced positions. Default/background and explicitly requested+ foreground profiles are separate trials. Foreground can intentionally interfere.+- Comparable per-step time includes the action and an additional full no-screenshot+ observation for every provider. Raw action time is diagnostic because Orca already+ returns a snapshot with mutations. Task time includes provider startup, discovery+ and all action/read pairs. Medians include completed tasks only; failed-attempt+ durations and sample counts are separate. Diagnostic serialization and disk I/O+ happen after timing. Final independent readback after shutdown is an upper+ bound on effect time, recorded separately.+- Fatal errors, invalid setup, failed observers, unobserved completion, budget+ exhaustion or missing planned cells leave JSON evidence and exit nonzero. A valid+ observed task failure remains a benchmark result, not an infrastructure failure.++Tasks cover explicit text targeting with another field logically focused, Unicode+entry plus submit, select-all replacement plus submit, and color choice plus one+click. Interference probes remain separate from task latency comparisons.++Explicit-target typing is a native capability probe. Orca's public CLI/RPC currently+accepts typing at the focused field and does not expose an element target for text.+Failure of this probe identifies a capability to consider adding; it does not prove+a violation of the current public typing contract. Click-then-type tasks exercise+the existing sequence, although this benchmark still bypasses public transports.++## Headless contract checks++```sh+ORCA_BACKGROUND_LAUNCH=1 python3 -m unittest discover -s config/benchmarks/computer-use -p 'test_*.py'+```++These checks protect the historical false-success cases and process-tree cleanup.+They do not import desktop observers or generate native input.++Transports, fixtures and observers were adapted from the historical+`bench/computer-cua` prototype at `68a68ccf0f16c0a7f952d55adfe21850247b8ee9`.+Historical checkouts and result artifacts remain unchanged.diff --git a/config/benchmarks/computer-use/bench.py b/config/benchmarks/computer-use/bench.pynew file mode 100644index 00000000000..dd1b814e2fb--- /dev/null+++ b/config/benchmarks/computer-use/bench.py@@ -0,0 +1,378 @@+#!/usr/bin/env python3+"""Compare direct native providers on disposable, independently observed task fixtures."""+import argparse+import hashlib+import json+import os+from pathlib import Path+import platform+import re+import subprocess+import sys+import time+import uuid++from process_scope import ProcessScope+from trial_contract import (DiscoveryError, MARKER, SELECTORS, TASKS, acknowledged, prepared,+ schedule, select_unique, summarize, verdict)++HERE = Path(__file__).resolve().parent+REPO = HERE.parents[2]+WIN = sys.platform == "win32"+++def read_json(path):+ try:+ return json.loads(Path(path).read_text(encoding="utf-8-sig"))+ except (OSError, ValueError):+ return None+++def write_json(path, value):+ temporary = Path(str(path) + ".tmp")+ temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")+ temporary.replace(path)+++def wait_for(predicate, timeout=10):+ end = time.monotonic() + timeout+ while time.monotonic() < end:+ value = predicate()+ if value:+ return value+ time.sleep(0.02)+ raise TimeoutError("independent setup/observation did not become ready")+++def pin_source(ref, output):+ if not re.fullmatch(r"[0-9a-f]{40}", ref):+ raise ValueError("source must be a full commit SHA")+ hashes = {}+ for relative in ("native/computer-use-linux/runtime.py", "native/computer-use-windows/runtime.ps1"):+ data = subprocess.check_output(["git", "show", f"{ref}:{relative}"], cwd=REPO)+ destination = output / relative+ destination.parent.mkdir(parents=True, exist_ok=True)+ destination.write_bytes(data)+ hashes[relative] = hashlib.sha256(data).hexdigest()+ return {"sha": ref, "files": hashes}+++class Trial:+ def __init__(self, args, directory, log):+ self.args, self.directory, self.log = args, directory, log+ self.instance = uuid.uuid4().hex+ directory.mkdir()+ self.targets = ProcessScope(str(directory))+ self.engines = ProcessScope(str(directory))+ self.sampler = self.typer = None+ self.state_path = directory / "state.json"+ self.sentinel_path = directory / "sentinel.json"+ self.nonce = None++ def spawn_target(self, command, **extra):+ env = {**os.environ, "BENCH_INSTANCE": self.instance, "BENCH_STATE_DIR": str(self.directory)}+ return self.targets.spawn(command, stdout=self.log, stderr=self.log, env=env, **extra)++ def window(self, pid, title):+ return wait_for(lambda: obs.find_window(pid=pid, title=title) if WIN+ else obs.find_client(pid, title), 30)++ def state(self):+ state = read_json(self.state_path)+ if state and state.get("instance") == self.instance and state.get("pid") == self.pid:+ return state+ return None++ def launch(self):+ if self.args.target == "electron":+ flags = ["--force-renderer-accessibility"] + ([] if WIN else ["--no-sandbox"])+ proc = self.spawn_target([self.args.electron, *flags, str(HERE / "electron/main.js")])+ title = "BenchElectron"+ elif WIN:+ proc = self.spawn_target(["powershell.exe", "-NoLogo", "-NoProfile", "-STA", "-File",+ str(HERE / "fixture_win.ps1"), "-StateDir", str(self.directory)], hidden=True)+ title = "BenchTarget"+ else:+ proc = self.spawn_target([sys.executable, str(HERE / "fixture_gtk.py"), str(self.directory)])+ title = "BenchTarget"+ self.pid = proc.pid+ self.target = self.window(proc.pid, title)+ obs.place(self.target, 60, 60, 760, 560)+ wait_for(self.state)+ proc = self.spawn_target([sys.executable, str(HERE / "sentinel.py"),+ str(self.sentinel_path), "880x660+20+20"], hidden=True)+ self.sentinel_pid = proc.pid+ self.sentinel = self.window(proc.pid, "BenchSentinel")+ wait_for(lambda: read_json(self.sentinel_path))+ # Tk's initial focus callback must finish before calibration or measurement.+ time.sleep(0.2)++ def front(self):+ if WIN:+ return obs.foreground() == self.sentinel and not obs.owned_above(self.sentinel, self.pid)+ display = obs.main_display()+ stacking = display.stacking()+ windows = display.windows_of(self.pid)+ return (display.active() == self.sentinel and display.focus() in display.descendants(self.sentinel)+ and self.sentinel in stacking and self.target in stacking and windows+ and all(stacking.index(window) < stacking.index(self.sentinel) for window in windows if window in stacking))++ def prepare(self, task):+ self.nonce = uuid.uuid4().hex+ write_json(self.directory / "cmd", {"instance": self.instance, "nonce": self.nonce, "task": task})+ wait_for(lambda: prepared(self.state() or {}, task, self.instance, self.pid, self.nonce), 5)+ obs.force_foreground(self.sentinel)+ wait_for(self.front, 5)+ if WIN:+ obs.u.SetCursorPos(2, 2)+ else:+ obs.xdotool("mousemove", 2, 2)+ wait_for(lambda: prepared(self.state() or {}, task, self.instance, self.pid, self.nonce), 5)++ def sample(self):+ self.sampler = obs.Sampler(self.sentinel, self.target, self.pid)+ self.sampler.start()++ def stop_sampling(self):+ if self.sampler:+ sampler, self.sampler = self.sampler, None+ result = sampler.stop()+ if result["n_samples"] < 3:+ raise RuntimeError("observer did not collect enough samples")+ return result+ return {}++ def calibrate(self):+ self.prepare("click_interference")+ self.sample()+ self.typer = obs.Typer()+ self.typer.start()+ time.sleep(0.65)+ self.typer.stop()+ sent = self.typer.sent+ self.typer = None+ if sent < 3:+ raise RuntimeError("simulated user input control failed")+ wait_for(lambda: (read_json(self.sentinel_path) or {}).get("q", 0) == sent, 2)+ idle = self.stop_sampling()+ if any(idle[key] for key in ("sentinel_lost_focus", "target_raised", "pointer_moved", "clipboard_changed")):+ raise RuntimeError("idle observer control failed")+ self.sample()+ obs.force_foreground(self.target)+ time.sleep(0.15)+ steal = self.stop_sampling()+ if not all(steal[key] for key in ("sentinel_lost_focus", "target_activated", "target_raised")):+ raise RuntimeError("intentional focus-steal observer control failed")+ return {"idle": idle, "steal": steal, "keys_sent": sent}++ def engine(self, name, sources):+ from providers import Cua, CuaEngine, OrcaEngine, OrcaLinux, OrcaWin, cua_sc+ if name == "cua":+ client = Cua(self.args.cua, str(self.directory / "cua-home"), self.log, self.engines)+ response = client.call("list_windows", {"pid": self.pid})+ if response.get("isError") or "_rpc_error" in response:+ raise RuntimeError("cua list_windows failed: " + str(response))+ windows = cua_sc(response).get("windows") or []+ matches = [window for window in windows if window.get("window_id") is not None+ and window.get("title") in ("BenchTarget", "BenchElectron")]+ if len(matches) != 1:+ raise DiscoveryError(f"cua list_windows matched {len(matches)} fixture windows")+ return CuaEngine(client, self.pid, matches[0]["window_id"])+ return OrcaEngine((OrcaWin if WIN else OrcaLinux)(str(sources / name), self.log, self.engines), self.pid)++ def run(self, cell, sources):+ row = {**cell, "prepared": False, "api_errors": [], "action_read_ms": [], "steps": []}+ state = None+ task_started = None+ snapshots, responses = [], []+ try:+ self.launch()+ self.prepare(cell["task"])+ row["initial_state"] = self.state()+ row["prepared"] = True+ self.sample()+ row["sentinel_before"] = read_json(self.sentinel_path)+ if cell["task"].endswith("_interference"):+ self.typer = obs.Typer()+ self.typer.start()+ task_started = time.perf_counter()+ engine = self.engine(cell["engine"], sources)+ acquire_started = time.perf_counter()+ elements = engine.snapshot()+ row["acquisition_ms"] = (time.perf_counter() - acquire_started) * 1000+ snapshots.append(engine.last_snapshot)+ for action, selector in TASKS[cell["task"]]:+ element = select_unique(elements, SELECTORS[selector])+ started = time.perf_counter()+ result = engine.act(action, element, cell["profile"] == "foreground", MARKER)+ row["steps"].append({"action": action, "result": result,+ "action_ms": (time.perf_counter() - started) * 1000})+ if result.get("error"):+ row["api_errors"].append(result)+ responses.append(engine.last_response)+ # Identical extra full no-screenshot read for all providers; reuse fresh refs next step.+ elements = engine.snapshot()+ row["action_read_ms"].append((time.perf_counter() - started) * 1000)+ snapshots.append(engine.last_snapshot)+ if result.get("error"):+ break+ except DiscoveryError as error:+ row["discovery_error"] = str(error)+ except Exception as error:+ row["api_errors"].append(repr(error))+ finally:+ if task_started is not None:+ row["attempt_ms"] = (time.perf_counter() - task_started) * 1000+ try:+ # No late operation can mutate the fixture after the terminal oracle starts.+ self.engines.close()+ if self.typer:+ self.typer.stop()+ row["keys_sent"] = self.typer.sent+ self.typer = None+ if row["prepared"]:+ if task_started is not None:+ row["elapsed_to_quiescent_ms"] = (time.perf_counter() - task_started) * 1000+ before = (self.state() or {}).get("seq", -1)+ time.sleep(0.4)+ def fresh_state():+ value = self.state()+ return value if value and value.get("seq", -1) > before and acknowledged(+ value, self.instance, self.pid, self.nonce) else None+ state = wait_for(fresh_state, 2)+ if task_started is not None:+ row["terminal_readback_ms"] = (time.perf_counter() - task_started) * 1000+ row["sentinel_after"] = read_json(self.sentinel_path)+ sentinel_before = row["sentinel_before"]+ sentinel_after = row["sentinel_after"]+ if not all(value and value.get("pid") == self.sentinel_pid+ for value in (sentinel_before, sentinel_after)):+ raise RuntimeError("missing independent sentinel counters")+ if cell["task"].endswith("_interference") and row.get("keys_sent", 0) == 0:+ row["input_observer_error"] = "no simulated user input was delivered"+ row["keys_received"] = sentinel_after.get("q", 0) - sentinel_before.get("q", 0)+ row["keys_lost"] = max(0, row.get("keys_sent", 0) - row["keys_received"])+ row["agent_text_in_sentinel"] = sentinel_after.get("other", 0) - sentinel_before.get("other", 0)+ row["target_q_characters"] = state["text"].count("q") + state["decoyText"].count("q")+ except Exception as error:+ row["terminal_observation_error"] = repr(error)+ try:+ row["observation"] = self.stop_sampling()+ except Exception as error:+ row["observer_error"] = repr(error)+ row["final_state"] = state+ row["verdict"] = verdict(row, state)+ for index, snapshot in enumerate(snapshots):+ write_json(self.directory / f"snapshot-{index}.json", snapshot)+ for index, response in enumerate(responses):+ write_json(self.directory / f"response-{index}.json", response)+ return row++ def close(self):+ if self.typer:+ self.typer.stop()+ try:+ self.engines.close()+ finally:+ try:+ self.stop_sampling()+ finally:+ self.targets.close()+++def markdown(meta, summary):+ lines = [f"## {meta['os']} / {meta['target']}", "",+ f"Run complete: {meta['complete']}. Cells: {meta['executed']}/{meta['planned']}. Seed: {meta['seed']}.", "",+ "| task | profile | engine | completed/scored | invalid | unobserved | API errors | task ms | action + read ms | focus lost | raised |",+ "|---|---|---|---|---|---|---|---|---|---|---|"]+ for row in summary:+ lines.append(f"| {row['task']} | {row['profile']} | {row['engine']} | {row['completed']}/{row['scored']} | "+ f"{row['invalid']} | {row['unobserved']} | {row['api_errors']} | {row['task_ms']} | "+ f"{row['action_read_ms']} | {row['sentinel_lost_focus']} | {row['target_raised']} |")+ if meta.get("fatal"):+ lines.extend(["", f"Incomplete: `{meta['fatal']}`"])+ return "\n".join(lines) + "\n"+++def main():+ if (os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true"+ or os.environ.get("ORCA_BACKGROUND_LAUNCH") != "1" or sys.platform not in ("linux", "win32")):+ raise SystemExit("Native input benchmark requires the isolated Linux/Windows GitHub Actions desktop")+ global obs+ if WIN:+ import obs_win as obs+ else:+ import obs_x11 as obs+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--target", choices=["fixture", "electron"], required=True)+ parser.add_argument("--cua", required=True)+ parser.add_argument("--electron")+ parser.add_argument("--out", type=Path, required=True)+ parser.add_argument("--baseline", default="030c88deecf3adc212d22f57b085e4bd5c6e48e6")+ parser.add_argument("--candidate", required=True)+ parser.add_argument("--n", type=int, default=3)+ parser.add_argument("--seed", type=int, default=417)+ parser.add_argument("--budget", type=int, default=1800)+ args = parser.parse_args()+ if args.n < 1 or (args.target == "electron" and not args.electron):+ parser.error("positive --n and an Electron executable for Electron targets are required")+ args.out = args.out.resolve()+ args.out.mkdir(parents=True, exist_ok=False)+ sources = args.out / "sources"+ cells = []+ for cell in schedule(["baseline", "candidate", "cua"], TASKS, args.n, args.seed):+ profiles = ["background", "foreground"] if not cell["task"].endswith("_interference") else ["background"]+ cells.extend({**cell, "profile": profile} for profile in profiles)+ meta = {"os": platform.platform(), "target": args.target, "seed": args.seed, "n": args.n,+ "python": sys.version, "image": os.environ.get("ImageVersion"), "planned": len(cells),+ "complete": False, "executed": 0, "sources": {}}+ rows = []+ started = time.monotonic()+ try:+ meta["sources"] = {name: pin_source(sha, sources / name)+ for name, sha in (("baseline", args.baseline), ("candidate", args.candidate))}+ meta["cua_sha256"] = hashlib.sha256(Path(args.cua).read_bytes()).hexdigest()+ meta["cua_version"] = subprocess.check_output([args.cua, "--version"], text=True, timeout=15).strip()+ if "0.38.0" not in meta["cua_version"]:+ raise RuntimeError("expected pinned cua-driver 0.38.0")+ with (args.out / "engines.log").open("ab") as log:+ control = Trial(args, args.out / "controls", log)+ try:+ control.launch()+ meta["calibration"] = control.calibrate()+ finally:+ control.close()+ for index, cell in enumerate(cells):+ if time.monotonic() - started >= args.budget:+ raise TimeoutError("budget exhausted; planned cells remain")+ trial = Trial(args, args.out / f"trial-{index:03d}", log)+ try:+ row = trial.run(cell, sources)+ rows.append(row)+ finally:+ trial.close()+ meta["executed"] = len(rows)+ write_json(args.out / "results.json", {"meta": meta, "rows": rows})+ print(f"{index + 1}/{len(cells)} {cell}: {row['verdict']}", flush=True)+ meta["complete"] = (len(rows) == len(cells) and all(row["prepared"] and not row.get("observer_error")+ and not row.get("terminal_observation_error") and not row.get("input_observer_error") and row["verdict"] != "unobserved" for row in rows))+ except Exception as error:+ import traceback+ traceback.print_exc()+ meta["fatal"] = repr(error)+ finally:+ meta["executed"] = len(rows)+ meta["elapsed_seconds"] = time.monotonic() - started+ summary = summarize(rows)+ write_json(args.out / "results.json", {"meta": meta, "summary": summary, "rows": rows, "plan": cells})+ report = markdown(meta, summary)+ (args.out / "summary.md").write_text(report, encoding="utf-8")+ print(report)+ if os.environ.get("GITHUB_STEP_SUMMARY"):+ with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream:+ stream.write(report)+ return 0 if meta["complete"] else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/config/benchmarks/computer-use/electron/main.js b/config/benchmarks/computer-use/electron/main.jsnew file mode 100644index 00000000000..5da2090d86f--- /dev/null+++ b/config/benchmarks/computer-use/electron/main.js@@ -0,0 +1,81 @@+// Disposable native-provider fixture, launched only on an isolated desktop.+const { app, BrowserWindow, ipcMain } = require('electron')+const fs = require('node:fs')+const path = require('node:path')+const dir = process.env.BENCH_STATE_DIR+const instance = process.env.BENCH_INSTANCE+process.on('uncaughtException', (error) => {+ console.error(error)+ app.exit(1)+})+if (process.env.ORCA_COMPUTER_BENCH_ISOLATED !== '1') {+ process.exit(1)+}+app.whenReady().then(() => {+ const w = new BrowserWindow({+ x: 60,+ y: 60,+ width: 760,+ height: 560,+ title: 'BenchElectron',+ webPreferences: { nodeIntegration: true, contextIsolation: false, backgroundThrottling: false }+ })+ ipcMain.on('st', (_event, state) => {+ const data = JSON.stringify({ ...state, instance, activeForm: w.isFocused(), pid: process.pid })+ try {+ fs.writeFileSync(path.join(dir, 'state.json.tmp'), data)+ fs.renameSync(path.join(dir, 'state.json.tmp'), path.join(dir, 'state.json'))+ } catch (error) {+ // Windows readers can briefly deny replacement; the next state tick retries.+ if (!['EACCES', 'EPERM', 'EBUSY'].includes(error.code)) {+ throw error+ }+ }+ })+ w.loadURL(+ `data:text/html,${encodeURIComponent(`<html><head><title>BenchElectron</title></head>+ <body style="font:14px sans-serif;height:4000px">+ <input id=field aria-label="field"><input id=decoy aria-label="decoy">+ <button id=button>Increment</button>+ <select id=color aria-label="Color"><option>Red</option><option>Green</option><option>Blue</option></select>+ <script>+ const { ipcRenderer } = require('electron');+ const field = document.getElementById('field'), decoy = document.getElementById('decoy');+ const color = document.getElementById('color');+ let count = 0, returns = 0, ack = null, seq = 0;+ document.getElementById('button').onclick = () => count++;+ field.addEventListener('keydown', event => { if (event.key === 'Enter') returns++ });+ function report() {+ ipcRenderer.send('st', { count, returns, ack, seq: ++seq, text: field.value, decoyText: decoy.value,+ top: window.scrollY, combo: color.value, selStart: field.selectionStart,+ selLen: field.selectionEnd - field.selectionStart, check: false, menuHits: 0, dropOpen: false,+ logicalFocus: document.activeElement === field ? 'field' : document.activeElement === decoy ? 'decoy' : 'other' });+ }+ ipcRenderer.on('cmd', (_event, cmd) => {+ count = returns = 0;+ field.value = cmd.task === 'replace_submit' ? 'hello' : '';+ field.setSelectionRange(field.value.length, field.value.length);+ decoy.value = 'untouched'; decoy.setSelectionRange(9, 9);+ color.selectedIndex = 0; color.blur(); window.scrollTo(0, 0);+ (cmd.task === 'targeted_type' ? decoy : field).focus();+ ack = cmd.nonce;+ report();+ });+ setInterval(report, 30);+ </script></body></html>`)}`+ )+ setInterval(() => {+ const cmdPath = path.join(dir, 'cmd')+ if (!fs.existsSync(cmdPath)) {+ return+ }+ const raw = fs.readFileSync(cmdPath, 'utf8').trim()+ if (raw) {+ const cmd = JSON.parse(raw)+ fs.unlinkSync(cmdPath)+ if (cmd.instance === instance) {+ w.webContents.send('cmd', cmd)+ }+ }+ }, 30)+})diff --git a/config/benchmarks/computer-use/fixture_gtk.py b/config/benchmarks/computer-use/fixture_gtk.pynew file mode 100644index 00000000000..978aad9fb0c--- /dev/null+++ b/config/benchmarks/computer-use/fixture_gtk.py@@ -0,0 +1,125 @@+#!/usr/bin/env python3+"""GTK3 target for the cua-vs-Orca background bench. Writes state.json every 30 ms; obeys cmd."""+import json+import os+import sys++if os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true":+ raise SystemExit("fixture requires isolated CI desktop")++import gi++gi.require_version("Gtk", "3.0")+from gi.repository import Gdk, GLib, Gtk # noqa: E402++state_dir = sys.argv[1]+STATE = os.path.join(state_dir, "state.json")+CMD = os.path.join(state_dir, "cmd")+instance = os.environ["BENCH_INSTANCE"]+ack = None+sequence = 0+s = {"count": 0, "returns": 0, "menuHits": 0, "last": None}+++def named(widget, name):+ widget.get_accessible().set_name(name)+ return widget+++Gtk.Settings.get_default().set_property("gtk-entry-select-on-focus", False)+win = Gtk.Window(title="BenchTarget")+win.set_default_size(760, 560)+win.move(60, 60)+win.connect("destroy", Gtk.main_quit)+accel = Gtk.AccelGroup()+win.add_accel_group(accel)+box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6)+win.add(box)++menubar = Gtk.MenuBar()+edit = Gtk.MenuItem(label="Edit")+sub = Gtk.Menu()+sub.set_accel_group(accel)+bump = Gtk.MenuItem(label="Bump")+bump.add_accelerator("activate", accel, ord("b"), Gdk.ModifierType.CONTROL_MASK, Gtk.AccelFlags.VISIBLE)+bump.connect("activate", lambda _w: s.__setitem__("menuHits", s["menuHits"] + 1))+sub.append(bump)+edit.set_submenu(sub)+menubar.append(edit)+box.pack_start(menubar, False, False, 0)++button = Gtk.Button(label="Increment")+button.connect("clicked", lambda _b: s.__setitem__("count", s["count"] + 1))+check = Gtk.CheckButton(label="Check me")+field = named(Gtk.Entry(), "field")+field.connect("activate", lambda _e: s.__setitem__("returns", s["returns"] + 1))+decoy = named(Gtk.Entry(), "decoy")+combo = named(Gtk.ComboBoxText(), "Color")+for item in ("Red", "Green", "Blue"):+ combo.append_text(item)+combo.set_active(0)+scroll = named(Gtk.ScrolledWindow(), "list")+scroll.set_size_request(300, 250)+rows = Gtk.ListBox()+for i in range(1, 301):+ rows.add(Gtk.Label(label=f"Row {i}"))+scroll.add(rows)+for widget in (button, check, field, decoy, combo):+ box.pack_start(widget, False, False, 0)+box.pack_start(scroll, True, True, 0)+++def command(c):+ global ack+ if c["instance"] != instance:+ return+ s["count"] = s["returns"] = s["menuHits"] = 0+ check.set_active(False)+ sub.popdown()+ combo.set_active(0)+ scroll.get_vadjustment().set_value(0)+ field.set_text("hello" if c["task"] == "replace_submit" else "")+ field.set_position(len(field.get_text()))+ field.select_region(len(field.get_text()), len(field.get_text()))+ decoy.set_text("untouched")+ decoy.set_position(9)+ decoy.select_region(9, 9)+ win.set_focus(decoy if c["task"] == "targeted_type" else field)+ ack = c["nonce"]+++def tick():+ global sequence+ sequence += 1+ try:+ with open(CMD) as f:+ c = f.read().strip()+ if c:+ open(CMD, "w").close()+ command(json.loads(c))+ except (OSError, ValueError):+ pass+ bounds = field.get_selection_bounds()+ sel = list(bounds) if bounds else [field.get_position(), field.get_position()]+ state = {+ "instance": instance, "ack": ack, "seq": sequence, "decoyText": decoy.get_text(),+ "logicalFocus": "field" if win.get_focus() is field else "decoy" if win.get_focus() is decoy else "other",+ "count": s["count"], "check": check.get_active(), "text": field.get_text(), "returns": s["returns"],+ "selStart": sel[0], "selLen": sel[1] - sel[0], "combo": combo.get_active_text(),+ "top": scroll.get_vadjustment().get_value(), "menuHits": s["menuHits"], "dropOpen": sub.get_visible(),+ "activeForm": win.is_active(), "fieldFocused": win.get_focus() is field, "pid": os.getpid(),+ }+ data = json.dumps(state)+ if data != s["last"]:+ tmp = STATE + ".tmp"+ with open(tmp, "w") as f:+ f.write(data)+ os.replace(tmp, STATE)+ s["last"] = data+ return True+++win.show_all()+win.set_focus(field)+GLib.timeout_add(30, tick)+Gtk.main()diff --git a/config/benchmarks/computer-use/fixture_win.ps1 b/config/benchmarks/computer-use/fixture_win.ps1new file mode 100644index 00000000000..f321fa5b6a6--- /dev/null+++ b/config/benchmarks/computer-use/fixture_win.ps1@@ -0,0 +1,124 @@+# WinForms target for the cua-vs-Orca background bench. Writes its state to state.json every 30 ms+# and acknowledges each JSON command after applying it.+param([string]$StateDir)+$ErrorActionPreference = "Stop"+if ($env:ORCA_COMPUTER_BENCH_ISOLATED -ne "1" -or $env:GITHUB_ACTIONS -ne "true") { throw "fixture requires isolated CI desktop" }+Add-Type -AssemblyName System.Windows.Forms+Add-Type -AssemblyName System.Drawing+[System.Windows.Forms.Application]::EnableVisualStyles()++$statePath = Join-Path $StateDir "state.json"+$cmdPath = Join-Path $StateDir "cmd"+$instance = $env:BENCH_INSTANCE+$script:ack = $null+$script:sequence = 0+$script:count = 0+$script:returns = 0+$script:menuHits = 0+$script:last = ""++$form = New-Object System.Windows.Forms.Form+$form.Text = "BenchTarget"+$form.StartPosition = "Manual"+$form.Location = New-Object System.Drawing.Point(60, 60)+$form.Size = New-Object System.Drawing.Size(760, 560)++$menu = New-Object System.Windows.Forms.MenuStrip+$edit = New-Object System.Windows.Forms.ToolStripMenuItem("Edit")+$bump = New-Object System.Windows.Forms.ToolStripMenuItem("Bump")+$bump.ShortcutKeys = [System.Windows.Forms.Keys]::Control -bor [System.Windows.Forms.Keys]::B+$bump.Add_Click({ $script:menuHits++ })+[void]$edit.DropDownItems.Add($bump)+[void]$menu.Items.Add($edit)+$form.MainMenuStrip = $menu+$form.Controls.Add($menu)++$button = New-Object System.Windows.Forms.Button+$button.Text = "Increment"+$button.Location = New-Object System.Drawing.Point(20, 40)+$button.Add_Click({ $script:count++ })+$form.Controls.Add($button)++$check = New-Object System.Windows.Forms.CheckBox+$check.Text = "Check me"+$check.Location = New-Object System.Drawing.Point(20, 80)+$form.Controls.Add($check)++$field = New-Object System.Windows.Forms.TextBox+$field.Name = "field"+$field.AccessibleName = "field"+$field.Location = New-Object System.Drawing.Point(20, 120)+$field.Width = 300+$field.Add_KeyDown({+ param($sender, $e)+ if ($e.KeyCode -eq [System.Windows.Forms.Keys]::Enter) { $script:returns++; $e.SuppressKeyPress = $true }+})+$form.Controls.Add($field)++$decoy = New-Object System.Windows.Forms.TextBox+$decoy.Name = "decoy"+$decoy.AccessibleName = "decoy"+$decoy.Location = New-Object System.Drawing.Point(350, 120)+$decoy.Width = 300+$form.Controls.Add($decoy)++$combo = New-Object System.Windows.Forms.ComboBox+$combo.DropDownStyle = "DropDownList"+$combo.AccessibleName = "Color"+[void]$combo.Items.AddRange(@("Red", "Green", "Blue"))+$combo.SelectedIndex = 0+$combo.Location = New-Object System.Drawing.Point(20, 160)+$form.Controls.Add($combo)++$list = New-Object System.Windows.Forms.ListBox+$list.AccessibleName = "list"+$list.Location = New-Object System.Drawing.Point(20, 200)+$list.Size = New-Object System.Drawing.Size(300, 250)+foreach ($i in 1..300) { [void]$list.Items.Add("Row $i") }+$form.Controls.Add($list)++function Invoke-BenchCommand($c) {+ if ($c.instance -ne $instance) { return }+ $script:count = 0; $script:returns = 0; $script:menuHits = 0+ $check.Checked = $false+ $edit.HideDropDown()+ $combo.SelectedIndex = 0+ $combo.DroppedDown = $false+ $list.TopIndex = 0+ $field.Text = if ($c.task -eq 'replace_submit') { 'hello' } else { '' }+ $field.SelectionStart = $field.Text.Length+ $field.SelectionLength = 0+ $decoy.Text = 'untouched'+ $decoy.SelectionStart = $decoy.Text.Length+ $decoy.SelectionLength = 0+ $form.ActiveControl = if ($c.task -eq 'targeted_type') { $decoy } else { $field }+ $script:ack = $c.nonce+}++$timer = New-Object System.Windows.Forms.Timer+$timer.Interval = 30+$timer.Add_Tick({+ $script:sequence+++ try {+ if (Test-Path $cmdPath) {+ $c = ([IO.File]::ReadAllText($cmdPath)).Trim()+ if ($c) { [IO.File]::WriteAllText($cmdPath, ""); Invoke-BenchCommand ($c | ConvertFrom-Json) }+ }+ } catch {}+ $state = [ordered]@{+ instance = $instance; ack = $script:ack; seq = $script:sequence+ decoyText = $decoy.Text+ logicalFocus = $(if ($form.ActiveControl -eq $field) { 'field' } elseif ($form.ActiveControl -eq $decoy) { 'decoy' } else { 'other' })+ count = $script:count; check = $check.Checked; text = $field.Text; returns = $script:returns+ selStart = $field.SelectionStart; selLen = $field.SelectionLength; combo = [string]$combo.SelectedItem+ top = $list.TopIndex; menuHits = $script:menuHits; dropOpen = ($edit.DropDown.Visible -or $combo.DroppedDown)+ activeForm = ($null -ne [System.Windows.Forms.Form]::ActiveForm); fieldFocused = $field.Focused+ pid = $PID+ }+ $json = $state | ConvertTo-Json -Compress+ if ($json -ne $script:last) {+ try { [IO.File]::WriteAllText($statePath, $json); $script:last = $json } catch {}+ }+})+$form.Add_Shown({ $form.ActiveControl = $field; $timer.Start() })+[System.Windows.Forms.Application]::Run($form)diff --git a/config/benchmarks/computer-use/obs_win.py b/config/benchmarks/computer-use/obs_win.pynew file mode 100644index 00000000000..17d9b9e579d--- /dev/null+++ b/config/benchmarks/computer-use/obs_win.py@@ -0,0 +1,309 @@+"""Windows host observation (foreground, z-order, cursor, clipboard) and the user's typer, via user32."""+import ctypes+import ctypes.wintypes as W+import threading+import time++try:+ ctypes.windll.shcore.SetProcessDpiAwareness(2)+except Exception:+ ctypes.windll.user32.SetProcessDPIAware()++u = ctypes.WinDLL("user32", use_last_error=True)+k32 = ctypes.WinDLL("kernel32", use_last_error=True)++GW_HWNDPREV = 3+WM_GETTEXT, WM_GETTEXTLENGTH, WM_SETTEXT = 0x000D, 0x000E, 0x000C+EM_GETSEL, EM_SETSEL, EM_GETFIRSTVISIBLELINE, EM_LINESCROLL = 0x00B0, 0x00B1, 0x00CE, 0x00B6+WM_VSCROLL, SB_TOP = 0x0115, 6+SMTO_ABORTIFHUNG = 0x2+SWP_NOACTIVATE, SWP_NOZORDER = 0x0010, 0x0004+SPI_GETFOREGROUNDLOCKTIMEOUT = 0x2000++ULONG_PTR = ctypes.c_size_t+++class KEYBDINPUT(ctypes.Structure):+ _fields_ = [("wVk", W.WORD), ("wScan", W.WORD), ("dwFlags", W.DWORD), ("time", W.DWORD), ("dwExtraInfo", ULONG_PTR)]+++class MOUSEINPUT(ctypes.Structure):+ _fields_ = [("dx", W.LONG), ("dy", W.LONG), ("mouseData", W.DWORD), ("dwFlags", W.DWORD), ("time", W.DWORD), ("dwExtraInfo", ULONG_PTR)]+++class _U(ctypes.Union):+ _fields_ = [("ki", KEYBDINPUT), ("mi", MOUSEINPUT)]+++class INPUT(ctypes.Structure):+ _fields_ = [("type", W.DWORD), ("u", _U)]+++WNDENUMPROC = ctypes.WINFUNCTYPE(W.BOOL, W.HWND, W.LPARAM)+u.EnumWindows.argtypes = [WNDENUMPROC, W.LPARAM]+u.GetWindowThreadProcessId.argtypes = [W.HWND, ctypes.POINTER(W.DWORD)]+u.GetWindowThreadProcessId.restype = W.DWORD+u.GetForegroundWindow.restype = W.HWND+u.GetWindow.restype = W.HWND+u.GetWindow.argtypes = [W.HWND, W.UINT]+u.FindWindowExW.restype = W.HWND+u.FindWindowExW.argtypes = [W.HWND, W.HWND, W.LPCWSTR, W.LPCWSTR]+u.SendMessageTimeoutW.argtypes = [W.HWND, W.UINT, W.WPARAM, W.LPARAM, W.UINT, W.UINT, ctypes.POINTER(ULONG_PTR)]+u.SendMessageTimeoutW.restype = W.LPARAM+u.SetWindowPos.argtypes = [W.HWND, W.HWND, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, W.UINT]+u.SendInput.argtypes = [W.UINT, ctypes.POINTER(INPUT), ctypes.c_int]+u.AttachThreadInput.argtypes = [W.DWORD, W.DWORD, W.BOOL]+u.SetForegroundWindow.argtypes = [W.HWND]+u.BringWindowToTop.argtypes = [W.HWND]+u.IsWindowVisible.argtypes = [W.HWND]+u.GetClassNameW.argtypes = [W.HWND, W.LPWSTR, ctypes.c_int]+u.GetWindowTextW.argtypes = [W.HWND, W.LPWSTR, ctypes.c_int]+u.GetWindowRect.argtypes = [W.HWND, ctypes.POINTER(W.RECT)]+++def hval(h):+ return int(h or 0)+++def pid_of(hwnd):+ p = W.DWORD(0)+ u.GetWindowThreadProcessId(hwnd, ctypes.byref(p))+ return p.value+++def title_of(hwnd):+ b = ctypes.create_unicode_buffer(512)+ u.GetWindowTextW(hwnd, b, 512)+ return b.value+++def class_of(hwnd):+ b = ctypes.create_unicode_buffer(256)+ u.GetClassNameW(hwnd, b, 256)+ return b.value+++def top_windows():+ out = []++ def cb(h, _l):+ out.append(hval(h))+ return True++ u.EnumWindows(WNDENUMPROC(cb), 0)+ return out+++def find_window(pid=None, title=None, cls=None, visible=True):+ for h in top_windows():+ if visible and not u.IsWindowVisible(h):+ continue+ if pid is not None and pid_of(h) != pid:+ continue+ if title is not None and title not in title_of(h):+ continue+ if cls is not None and class_of(h) != cls:+ continue+ return h+ return 0+++def describe(h):+ r = W.RECT()+ u.GetWindowRect(h, ctypes.byref(r))+ return {"hwnd": h, "pid": pid_of(h), "title": title_of(h), "class": class_of(h), "rect": [r.left, r.top, r.right, r.bottom]}+++def foreground():+ return hval(u.GetForegroundWindow())+++def cursor():+ p = W.POINT()+ u.GetCursorPos(ctypes.byref(p))+ return (p.x, p.y)+++def clip_seq():+ return u.GetClipboardSequenceNumber()+++def lock_timeout():+ v = W.DWORD(0)+ u.SystemParametersInfoW(SPI_GETFOREGROUNDLOCKTIMEOUT, 0, ctypes.byref(v), 0)+ return v.value+++def session_id():+ s = W.DWORD(0)+ k32.ProcessIdToSessionId(k32.GetCurrentProcessId(), ctypes.byref(s))+ return s.value+++def owned_above(sentinel, target_pid, limit=2000):+ """True when a visible top-level window of target_pid sits above the sentinel in z-order."""+ h = u.GetWindow(sentinel, GW_HWNDPREV)+ n = 0+ while h and n < limit:+ if u.IsWindowVisible(h) and pid_of(h) == target_pid:+ return True+ h = u.GetWindow(h, GW_HWNDPREV)+ n += 1+ return False+++def place(hwnd, x, y, w, h):+ u.SetWindowPos(hwnd, 0, x, y, w, h, SWP_NOACTIVATE | SWP_NOZORDER)+++def force_foreground(hwnd):+ """Bench setup only: re-front the sentinel between trials (AttachThreadInput, no keystrokes)."""+ if foreground() == hwnd:+ return True+ cur = k32.GetCurrentThreadId()+ fg = foreground()+ threads = {u.GetWindowThreadProcessId(fg, None) if fg else 0, u.GetWindowThreadProcessId(hwnd, None)} - {0, cur}+ attached = [t for t in threads if u.AttachThreadInput(cur, t, True)]+ u.BringWindowToTop(hwnd)+ u.SetForegroundWindow(hwnd)+ for t in attached:+ u.AttachThreadInput(cur, t, False)+ for _ in range(25):+ if foreground() == hwnd:+ return True+ time.sleep(0.02)+ return foreground() == hwnd+++def send(hwnd, msg, wp=0, lp=0, timeout=1000):+ res = ULONG_PTR(0)+ ok = u.SendMessageTimeoutW(hwnd, msg, wp, lp, SMTO_ABORTIFHUNG, timeout, ctypes.byref(res))+ return res.value if ok else None+++def edit_child(top):+ """Notepad's editor: classic Edit, or RichEditD2DPT in the WinUI Notepad."""+ for cls in ("Edit", "RichEditD2DPT"):+ h = hval(u.FindWindowExW(top, None, cls, None))+ if h:+ return h+ stack = [top]+ while stack:+ p = stack.pop()+ c = hval(u.FindWindowExW(p, None, None, None))+ while c:+ if class_of(c) in ("Edit", "RichEditD2DPT"):+ return c+ stack.append(c)+ c = hval(u.FindWindowExW(p, c, None, None))+ return 0+++def edit_text(h):+ n = send(h, WM_GETTEXTLENGTH) or 0+ buf = ctypes.create_unicode_buffer(n + 2)+ send(h, WM_GETTEXT, n + 1, ctypes.addressof(buf))+ return buf.value+++def edit_set(h, text, caret=None):+ buf = ctypes.create_unicode_buffer(text)+ send(h, WM_SETTEXT, 0, ctypes.addressof(buf))+ c = len(text) if caret is None else caret+ send(h, EM_SETSEL, c, c)+ send(h, WM_VSCROLL, SB_TOP, 0)+++def edit_sel(h):+ v = send(h, EM_GETSEL) or 0+ return (v & 0xFFFF, (v >> 16) & 0xFFFF)+++def edit_first_line(h):+ return send(h, EM_GETFIRSTVISIBLELINE) or 0+++class Typer:+ """Simulated user: one 'q' every 200 ms through SendInput (goes to whatever has keyboard focus)."""++ def __init__(self, period=0.2):+ self.period = period+ self.sent = 0+ self.run = False+ self.vk = 0x51+ self.scan = u.MapVirtualKeyW(self.vk, 0)++ def _key(self, up):+ i = INPUT(type=1)+ i.u.ki = KEYBDINPUT(self.vk, self.scan, 0x0002 if up else 0, 0, 0)+ return i++ def _loop(self):+ while self.run:+ arr = (INPUT * 2)(self._key(False), self._key(True))+ if u.SendInput(2, arr, ctypes.sizeof(INPUT)) == 2:+ self.sent += 1+ time.sleep(self.period)++ def start(self):+ self.run = True+ self.thread = threading.Thread(target=self._loop, daemon=True)+ self.thread.start()++ def stop(self):+ self.run = False+ self.thread.join()+ time.sleep(0.1)+++class Sampler:+ """Samples every ~4 ms: foreground window, target-above-sentinel, cursor, clipboard sequence."""++ def __init__(self, sentinel, target, target_pid, own_pids=()):+ self.s, self.t, self.tpid = sentinel, target, target_pid+ self.own = set(own_pids)++ def _sample(self):+ fg = foreground()+ return (time.perf_counter(), fg, pid_of(fg) if fg else 0, owned_above(self.s, self.tpid), cursor(), clip_seq())++ def start(self):+ self.base = self._sample()+ self.samples = [self.base]+ self.error = None+ self.run = True+ self.th = threading.Thread(target=self._loop, daemon=True)+ self.th.start()++ def _loop(self):+ try:+ while self.run:+ self.samples.append(self._sample())+ time.sleep(0.004)+ except Exception as error:+ self.error = error++ def stop(self):+ self.run = False+ self.th.join()+ if self.error:+ raise RuntimeError("observer sampling failed") from self.error+ self.samples.append(self._sample())+ b = self.base+ ss = self.samples+ lost = [x for x in ss if x[1] != self.s]+ return {+ "sentinel_lost_focus": bool(lost),+ "lost_ms": round(_dur(ss, lambda x: x[1] != self.s) * 1000, 1),+ "target_activated": any(x[2] == self.tpid for x in ss),+ "target_raised": any(x[3] for x in ss),+ "pointer_moved": any(abs(x[4][0] - b[4][0]) > 1 or abs(x[4][1] - b[4][1]) > 1 for x in ss),+ "pointer_end": list(ss[-1][4]), "pointer_start": list(b[4]),+ "clipboard_changed": any(x[5] != b[5] for x in ss),+ "fg_pids": sorted({x[2] for x in ss}),+ "n_samples": len(ss),+ }+++def _dur(ss, pred):+ return sum(b[0] - a[0] for a, b in zip(ss, ss[1:]) if pred(a))diff --git a/config/benchmarks/computer-use/obs_x11.py b/config/benchmarks/computer-use/obs_x11.pynew file mode 100644index 00000000000..2c4954017b8--- /dev/null+++ b/config/benchmarks/computer-use/obs_x11.py@@ -0,0 +1,263 @@+"""X11 host observation (active window, input focus, stacking, pointer, selections) and the user's typer."""+import ctypes+import ctypes.util+import subprocess+import threading+import time++X = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")+XT = ctypes.cdll.LoadLibrary(ctypes.util.find_library("Xtst") or "libXtst.so.6")+Window = ctypes.c_ulong+Atom = ctypes.c_ulong+P = ctypes.POINTER+vp = ctypes.c_void_p++X.XOpenDisplay.restype = vp+X.XOpenDisplay.argtypes = [ctypes.c_char_p]+X.XInitThreads.restype = ctypes.c_int+X.XDefaultRootWindow.restype = Window+X.XDefaultRootWindow.argtypes = [vp]+X.XInternAtom.restype = Atom+X.XInternAtom.argtypes = [vp, ctypes.c_char_p, ctypes.c_int]+X.XGetWindowProperty.argtypes = [vp, Window, Atom, ctypes.c_long, ctypes.c_long, ctypes.c_int, Atom,+ P(Atom), P(ctypes.c_int), P(ctypes.c_ulong), P(ctypes.c_ulong), P(vp)]+X.XFree.argtypes = [vp]+X.XGetInputFocus.argtypes = [vp, P(Window), P(ctypes.c_int)]+X.XQueryPointer.argtypes = [vp, Window, P(Window), P(Window), P(ctypes.c_int), P(ctypes.c_int),+ P(ctypes.c_int), P(ctypes.c_int), P(ctypes.c_uint)]+X.XGetSelectionOwner.restype = Window+X.XGetSelectionOwner.argtypes = [vp, Atom]+X.XQueryTree.argtypes = [vp, Window, P(Window), P(Window), P(P(Window)), P(ctypes.c_uint)]+X.XStringToKeysym.restype = ctypes.c_ulong+X.XStringToKeysym.argtypes = [ctypes.c_char_p]+X.XKeysymToKeycode.restype = ctypes.c_ubyte+X.XKeysymToKeycode.argtypes = [vp, ctypes.c_ulong]+X.XFlush.argtypes = [vp]+X.XCloseDisplay.argtypes = [vp]+XT.XTestFakeKeyEvent.argtypes = [vp, ctypes.c_uint, ctypes.c_int, ctypes.c_ulong]++X.XInitThreads()+ERRH = ctypes.CFUNCTYPE(ctypes.c_int, vp, vp)(lambda _d, _e: 0)+X.XSetErrorHandler(ERRH)+++class Display:+ def __init__(self):+ self.d = X.XOpenDisplay(None)+ if not self.d:+ raise RuntimeError("cannot open X display")+ self.root = X.XDefaultRootWindow(self.d)+ self.atoms = {}++ def atom(self, name):+ if name not in self.atoms:+ self.atoms[name] = X.XInternAtom(self.d, name.encode(), 0)+ return self.atoms[name]++ def prop_longs(self, win, name):+ at, fmt, n, after, data = Atom(), ctypes.c_int(), ctypes.c_ulong(), ctypes.c_ulong(), vp()+ rc = X.XGetWindowProperty(self.d, win, self.atom(name), 0, 4096, 0, 0, ctypes.byref(at), ctypes.byref(fmt),+ ctypes.byref(n), ctypes.byref(after), ctypes.byref(data))+ if rc != 0 or not data.value:+ return []+ try:+ if fmt.value != 32:+ return []+ arr = ctypes.cast(data, P(ctypes.c_ulong))+ return [arr[i] for i in range(n.value)]+ finally:+ X.XFree(data)++ def active(self):+ v = self.prop_longs(self.root, "_NET_ACTIVE_WINDOW")+ return v[0] if v else 0++ def stacking(self):+ return self.prop_longs(self.root, "_NET_CLIENT_LIST_STACKING")++ def clients(self):+ return self.prop_longs(self.root, "_NET_CLIENT_LIST")++ def pid_of(self, w):+ v = self.prop_longs(w, "_NET_WM_PID")+ return v[0] if v else 0++ def focus(self):+ w, r = Window(), ctypes.c_int()+ X.XGetInputFocus(self.d, ctypes.byref(w), ctypes.byref(r))+ return w.value++ def pointer(self):+ r, c = Window(), Window()+ rx, ry, wx, wy = ctypes.c_int(), ctypes.c_int(), ctypes.c_int(), ctypes.c_int()+ m = ctypes.c_uint()+ X.XQueryPointer(self.d, self.root, ctypes.byref(r), ctypes.byref(c), ctypes.byref(rx), ctypes.byref(ry),+ ctypes.byref(wx), ctypes.byref(wy), ctypes.byref(m))+ return (rx.value, ry.value)++ def owner(self, sel):+ return X.XGetSelectionOwner(self.d, self.atom(sel))++ def descendants(self, w):+ out, stack = {w}, [w]+ while stack:+ p = stack.pop()+ r, par, kids, n = Window(), Window(), P(Window)(), ctypes.c_uint()+ if not X.XQueryTree(self.d, p, ctypes.byref(r), ctypes.byref(par), ctypes.byref(kids), ctypes.byref(n)):+ continue+ for i in range(n.value):+ out.add(kids[i])+ stack.append(kids[i])+ if kids:+ X.XFree(kids)+ return out++ def windows_of(self, pid):+ return [w for w in self.clients() if self.pid_of(w) == pid]++ def close(self):+ if self.d:+ X.XCloseDisplay(self.d)+ self.d = None+++_MAIN = []+++def main_display():+ """One connection for the bench's main thread; Xvfb allows only ~256 clients."""+ if not _MAIN:+ _MAIN.append(Display())+ return _MAIN[0]+++def find_client(pid, title=None):+ """Managed client window of pid (by _NET_WM_PID), else by exact title (Tk may omit _NET_WM_PID)."""+ d = main_display()+ ws = d.windows_of(pid)+ if ws:+ return ws[0]+ if title:+ clients = set(d.clients())+ out = xdotool("search", "--name", f"^{title}$").stdout.split()+ for w in out:+ if int(w) in clients:+ return int(w)+ return 0+++def xdotool(*args):+ return subprocess.run(["xdotool", *map(str, args)], capture_output=True, text=True, timeout=10)+++def window_name(w):+ return xdotool("getwindowname", w).stdout.strip()+++def place(w, x, y, width, height):+ xdotool("windowsize", w, width, height)+ xdotool("windowmove", w, x, y)+++def force_foreground(w):+ """Bench setup only: re-front the sentinel between trials."""+ d = main_display()+ if d.active() == w:+ return True+ xdotool("windowactivate", "--sync", w)+ xdotool("windowraise", w)+ for _ in range(25):+ if d.active() == w:+ return True+ time.sleep(0.02)+ return d.active() == w+++class Typer:+ """Simulated user: one 'q' every 200 ms through XTest (goes to the X input focus)."""++ def __init__(self, period=0.2):+ self.period = period+ self.sent = 0+ self.run = False++ def _loop(self):+ d = Display()+ kc = X.XKeysymToKeycode(d.d, X.XStringToKeysym(b"q"))+ while self.run:+ XT.XTestFakeKeyEvent(d.d, kc, 1, 0)+ XT.XTestFakeKeyEvent(d.d, kc, 0, 0)+ X.XFlush(d.d)+ self.sent += 1+ time.sleep(self.period)+ d.close()++ def start(self):+ self.run = True+ self.thread = threading.Thread(target=self._loop, daemon=True)+ self.thread.start()++ def stop(self):+ self.run = False+ self.thread.join()+ time.sleep(0.1)+++class Sampler:+ """Samples every ~4 ms: active window, input focus, stacking, pointer, CLIPBOARD/PRIMARY owners."""++ def __init__(self, sentinel, target, target_pid, own_pids=()):+ self.s, self.t, self.tpid = sentinel, target, target_pid+ self.s_set = main_display().descendants(sentinel)++ def _sample(self, d):+ st = d.stacking()+ act = d.active()+ foc = d.focus()+ tw = set(d.windows_of(self.tpid)) or {self.t}+ raised = False+ if self.s in st:+ si = st.index(self.s)+ raised = any(st.index(w) > si for w in tw if w in st)+ return (time.perf_counter(), act, foc, raised, d.pointer(), d.owner("CLIPBOARD"), d.owner("PRIMARY"), act in tw)++ def start(self):+ self.d = main_display()+ self.base = self._sample(self.d)+ self.samples = [self.base]+ self.error = None+ self.run = True+ self.th = threading.Thread(target=self._loop, daemon=True)+ self.th.start()++ def _loop(self):+ d = Display()+ try:+ while self.run:+ self.samples.append(self._sample(d))+ time.sleep(0.004)+ except Exception as error:+ self.error = error+ finally:+ d.close()++ def stop(self):+ self.run = False+ self.th.join()+ if self.error:+ raise RuntimeError("observer sampling failed") from self.error+ self.samples.append(self._sample(self.d))+ b, ss = self.base, self.samples+ lost = lambda x: x[1] != self.s or (x[2] not in self.s_set)+ return {+ "sentinel_lost_focus": any(lost(x) for x in ss),+ "lost_ms": round(sum(q[0] - p[0] for p, q in zip(ss, ss[1:]) if lost(p)) * 1000, 1),+ "target_activated": any(x[7] for x in ss),+ "target_raised": any(x[3] for x in ss),+ "pointer_moved": any(abs(x[4][0] - b[4][0]) > 1 or abs(x[4][1] - b[4][1]) > 1 for x in ss),+ "pointer_end": list(ss[-1][4]), "pointer_start": list(b[4]),+ "clipboard_changed": any(x[5] != b[5] for x in ss),+ "primary_changed": any(x[6] != b[6] for x in ss),+ "focus_windows": sorted({hex(x[2]) for x in ss})[:6],+ "n_samples": len(ss),+ }diff --git a/config/benchmarks/computer-use/process_scope.py b/config/benchmarks/computer-use/process_scope.pynew file mode 100644index 00000000000..577a702766d--- /dev/null+++ b/config/benchmarks/computer-use/process_scope.py@@ -0,0 +1,103 @@+"""Contain each trial's process tree, including operations that time out."""+import os+import signal+import subprocess+import sys+import tempfile++WIN = sys.platform == "win32"++if WIN:+ import ctypes+ from ctypes import wintypes as W++ class BasicLimit(ctypes.Structure):+ _fields_ = [("processTime", ctypes.c_longlong), ("jobTime", ctypes.c_longlong),+ ("flags", W.DWORD), ("minWorkingSet", ctypes.c_size_t),+ ("maxWorkingSet", ctypes.c_size_t), ("activeProcesses", W.DWORD),+ ("affinity", ctypes.c_size_t), ("priority", W.DWORD), ("scheduling", W.DWORD)]++ class IoCounters(ctypes.Structure):+ _fields_ = [(name, ctypes.c_ulonglong) for name in+ ("readOps", "writeOps", "otherOps", "readBytes", "writeBytes", "otherBytes")]++ class ExtendedLimit(ctypes.Structure):+ _fields_ = [("basic", BasicLimit), ("io", IoCounters), ("processMemory", ctypes.c_size_t),+ ("jobMemory", ctypes.c_size_t), ("peakProcess", ctypes.c_size_t), ("peakJob", ctypes.c_size_t)]++ kernel = ctypes.WinDLL("kernel32", use_last_error=True)+ kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, W.LPCWSTR]+ kernel.CreateJobObjectW.restype = W.HANDLE+ kernel.SetInformationJobObject.argtypes = [W.HANDLE, ctypes.c_int, ctypes.c_void_p, W.DWORD]+ kernel.AssignProcessToJobObject.argtypes = [W.HANDLE, W.HANDLE]+ kernel.TerminateJobObject.argtypes = [W.HANDLE, W.UINT]+ kernel.CloseHandle.argtypes = [W.HANDLE]+ kernel.QueryInformationJobObject.argtypes = [W.HANDLE, ctypes.c_int, ctypes.c_void_p, W.DWORD, ctypes.c_void_p]+ nt = ctypes.WinDLL("ntdll")+ nt.NtResumeProcess.argtypes = [W.HANDLE]+ nt.NtResumeProcess.restype = ctypes.c_long++ class Accounting(ctypes.Structure):+ _fields_ = [("user", ctypes.c_longlong), ("kernel", ctypes.c_longlong),+ ("periodUser", ctypes.c_longlong), ("periodKernel", ctypes.c_longlong),+ ("faults", W.DWORD), ("total", W.DWORD), ("active", W.DWORD), ("terminated", W.DWORD)]+++class ProcessScope:+ def __init__(self, directory=None):+ self.directory = directory or tempfile.mkdtemp(prefix="orca-computer-trial-")+ self.procs = []+ self.job = None+ if WIN:+ self.job = kernel.CreateJobObjectW(None, None)+ limits = ExtendedLimit()+ limits.basic.flags = 0x2000 # Closing the job kills descendants too.+ if not self.job or not kernel.SetInformationJobObject(self.job, 9, ctypes.byref(limits), ctypes.sizeof(limits)):+ raise ctypes.WinError(ctypes.get_last_error())++ def spawn(self, args, hidden=False, **kwargs):+ if WIN:+ kwargs["creationflags"] = 0x4 | (0x08000000 if hidden else 0)+ else:+ kwargs["start_new_session"] = True+ proc = subprocess.Popen(args, **kwargs)+ self.procs.append(proc)+ if WIN:+ # Suspend until assigned so children cannot escape the trial job.+ if not kernel.AssignProcessToJobObject(self.job, int(proc._handle)):+ proc.kill()+ proc.wait()+ raise ctypes.WinError(ctypes.get_last_error())+ if nt.NtResumeProcess(int(proc._handle)) != 0:+ raise RuntimeError("failed to resume contained process")+ return proc++ def close(self):+ import time+ if WIN and self.job:+ if not kernel.TerminateJobObject(self.job, 1):+ raise ctypes.WinError(ctypes.get_last_error())+ end = time.monotonic() + 10+ while True:+ info = Accounting()+ if not kernel.QueryInformationJobObject(self.job, 1, ctypes.byref(info), ctypes.sizeof(info), None):+ raise ctypes.WinError(ctypes.get_last_error())+ if info.active == 0:+ break+ if time.monotonic() >= end:+ raise TimeoutError("trial job did not become quiescent")+ time.sleep(0.02)+ kernel.CloseHandle(self.job)+ self.job = None+ elif not WIN:+ for proc in self.procs:+ try:+ os.killpg(proc.pid, signal.SIGKILL)+ except ProcessLookupError:+ pass+ for proc in self.procs:+ proc.wait(timeout=10)+ for stream in (proc.stdin, proc.stdout):+ if stream:+ stream.close()+ self.procs.clear()diff --git a/config/benchmarks/computer-use/providers.py b/config/benchmarks/computer-use/providers.pynew file mode 100644index 00000000000..cc4544f8830--- /dev/null+++ b/config/benchmarks/computer-use/providers.py@@ -0,0 +1,275 @@+"""Direct native-provider transports, adapted from bench/computer-cua at 68a68ccf0f16."""+import json+import os+import queue+import re+import sys+import subprocess+import threading+import time++WIN = sys.platform == "win32"+++def now():+ return time.perf_counter() * 1000+++def md_rows(text, snapshot_id):+ out = []+ for line in (text or "").splitlines():+ match = re.match(r'\s*-\s*\[(\d+)\]\s*(.*)', line)+ if match:+ body = match.group(2)+ role = re.split(r'["\[(]', body, maxsplit=1)[0].strip()+ out.append({"role": role, "label": body[len(role):],+ "element_token": f"{snapshot_id}:{match.group(1)}"})+ return out+++class Cua:+ """cua-driver over MCP stdio, `mcp --direct` (the process owns the runtime on Windows/Linux)."""++ def __init__(self, exe, home, logf, processes):+ env = dict(os.environ)+ env.update({+ "CUA_DRIVER_RS_TELEMETRY_ENABLED": "0", "CUA_TELEMETRY_ENABLED": "0", "DO_NOT_TRACK": "1",+ "CUA_DRIVER_RS_UPDATE_CHECK": "0", "CUA_DRIVER_RS_HOME": home, "CUA_HOME": home,+ "CUA_DRIVER_TELEMETRY_HOME": home,+ # Disposable CI desktop, as in cua's own live-smoke workflow: no interactive approvals.+ "CUA_DRIVER_PERMISSION_MODE": "unrestricted", "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS": "1",+ })+ os.makedirs(home, exist_ok=True)+ t0 = now()+ self.p = processes.spawn([exe, "mcp", "--direct", "--no-overlay"], stdin=subprocess.PIPE,+ stdout=subprocess.PIPE, stderr=logf, env=env)+ self.pid = self.p.pid+ self.q = {}+ self.cv = threading.Condition()+ self.nid = 0+ threading.Thread(target=self._reader, daemon=True).start()+ init = self.rpc("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},+ "clientInfo": {"name": "orca-bench", "version": "0"}}, timeout=60)+ self._send({"jsonrpc": "2.0", "method": "notifications/initialized"})+ self.startup_ms = round(now() - t0, 1)+ self.server_info = (init or {}).get("serverInfo")++ def _reader(self):+ for line in self.p.stdout:+ line = line.strip()+ if not line:+ continue+ try:+ m = json.loads(line.decode("utf-8", "replace"))+ except ValueError:+ continue+ if "id" in m:+ with self.cv:+ self.q[m["id"]] = (m, len(line))+ self.cv.notify_all()++ def _send(self, obj):+ self.p.stdin.write((json.dumps(obj) + "\n").encode())+ self.p.stdin.flush()++ def rpc(self, method, params, timeout=30):+ with self.cv:+ self.nid += 1+ i = self.nid+ self._send({"jsonrpc": "2.0", "id": i, "method": method, "params": params})+ end = time.time() + timeout+ with self.cv:+ while i not in self.q:+ left = end - time.time()+ if left <= 0:+ raise TimeoutError(method)+ self.cv.wait(left)+ m, n = self.q.pop(i)+ self.last_raw = n+ if "error" in m:+ return {"_rpc_error": m["error"]}+ return m["result"]++ def call(self, tool, args, timeout=30):+ t0 = now()+ r = self.rpc("tools/call", {"name": tool, "arguments": args}, timeout)+ r["_ms"] = now() - t0+ r["_raw"] = self.last_raw+ return r++++def cua_text(r):+ return "\n".join(c.get("text", "") for c in r.get("content") or [] if c.get("type") == "text")+++def cua_sc(r):+ return r.get("structuredContent") or {}+++class OrcaWin:+ """runtime.ps1 -Serve over NDJSON, as DesktopScriptRuntimeHost runs it."""++ def __init__(self, root, logf, processes):+ script = os.path.join(root, "native", "computer-use-windows", "runtime.ps1")+ t0 = now()+ self.p = processes.spawn(["powershell.exe", "-NoLogo", "-NoProfile", "-NonInteractive",+ "-File", script, "-Serve"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=logf,+ hidden=True)+ self.pid = self.p.pid+ self.lines = queue.Queue()+ threading.Thread(target=self._reader, daemon=True).start()+ ready = self.lines.get(timeout=120)+ if '"ready"' not in ready:+ raise RuntimeError("runtime.ps1 did not announce readiness: " + ready[:200])+ self.startup_ms = round(now() - t0, 1)+ self.rid = 0++ def _reader(self):+ for line in self.p.stdout:+ self.lines.put(line.decode("utf-8", "replace").strip())++ def op(self, req, timeout=30):+ self.rid += 1+ req = {**req, "requestId": self.rid}+ t0 = now()+ self.p.stdin.write((json.dumps(req) + "\n").encode("utf-8"))+ self.p.stdin.flush()+ end = time.time() + timeout+ while True:+ line = self.lines.get(timeout=max(0.01, end - time.time()))+ try:+ m = json.loads(line)+ except ValueError:+ continue+ if m.get("requestId") == self.rid:+ return m, now() - t0, len(line)++++class OrcaLinux:+ """One subprocess per operation, matching the Linux production transport."""++ def __init__(self, root, logf, processes):+ self.script = os.path.join(root, "native", "computer-use-linux", "runtime.py")+ self.logf, self.processes = logf, processes+ self.path = os.path.join(processes.directory, "operation.json")++ def op(self, req, timeout=30):+ with open(self.path, "w", encoding="utf-8") as stream:+ json.dump(req, stream)+ started = now()+ proc = self.processes.spawn([sys.executable, self.script, self.path],+ stdout=-1, stderr=self.logf)+ raw, _ = proc.communicate(timeout=timeout)+ text = raw.decode("utf-8", "replace")+ try:+ result = json.loads(text)+ except ValueError:+ raise RuntimeError("invalid native response: " + text[:300])+ return result, now() - started, len(raw)++++class CuaEngine:+ name = "cua"++ def __init__(self, c, pid, wid):+ self.c, self.pid, self.wid = c, pid, wid+ self.elements = []++ def snapshot(self):+ r = self.c.call("get_window_state", {"pid": self.pid, "window_id": self.wid, "include_screenshot": False, "full_output": True})+ s = cua_sc(r)+ if r.get("isError") or "_rpc_error" in r:+ raise RuntimeError(cua_text(r) or str(r))+ self.elements = s.get("elements") or md_rows(s.get("tree_markdown") or cua_text(r), s.get("snapshot_id"))+ self.last_snapshot = r+ return self.elements++ def act(self, action, element, escalate, text=""):+ base = {"pid": self.pid, "window_id": self.wid}+ tok = element.get("element_token")+ if not tok:+ raise RuntimeError("selected cua element has no token")+ if action == "click":+ tool, args = "click", {**base, "element_token": tok}+ elif action == "type":+ tool, args = "type_text", {**base, "element_token": tok, "text": text}+ elif action == "return":+ tool, args = "press_key", {**base, "element_token": tok, "key": "return"}+ elif action == "ctrl_a":+ tool, args = "hotkey", {**base, "element_token": tok, "keys": ["ctrl", "a"]}+ elif action == "scroll":+ tool, args = "scroll", {**base, "element_token": tok, "direction": "down", "amount": 3}+ elif action == "choose":+ tool, args = "set_value", {**base, "element_token": tok, "value": "Blue"}+ if escalate and tool in ("click", "type_text", "press_key", "hotkey", "scroll"):+ args["delivery_mode"] = "foreground"+ if "element_token" in args and not args["element_token"]:+ return {"ms": 0, "reported": "no_element", "detail": "selector found no element", "error": True}+ r = self.c.call(tool, args)+ self.last_response = r+ if "_rpc_error" in r:+ return {"ms": r["_ms"], "reported": "rpc_error", "detail": json.dumps(r["_rpc_error"])[:300], "error": True}+ s = cua_sc(r)+ err = bool(r.get("isError"))+ code = s.get("code") or (s["error"].get("code") if isinstance(s.get("error"), dict) else None)+ effect = s.get("effect")+ reported = effect or (("error:" + str(code)) if err else "ok(no effect field)")+ if err and effect:+ reported = f"{effect}(isError)"+ return {"ms": r["_ms"], "reported": reported, "error": err, "route": s.get("route"),+ "delivery": (s.get("delivery") or {}).get("mode") if isinstance(s.get("delivery"), dict) else s.get("delivery"),+ "code": code, "detail": (s.get("summary") or cua_text(r))[:300]}++++class OrcaEngine:+ name = "orca"++ def __init__(self, o, pid):+ self.o, self.app = o, f"pid:{pid}"+ self.target = {}+ self.elements = []++ def snapshot(self):+ r, ms, n = self.o.op({"tool": "get_app_state", "app": self.app, "noScreenshot": True})+ if not r.get("ok"):+ raise RuntimeError("orca get_app_state: " + str(r.get("error"))[:300])+ snap = r["snapshot"]+ self.elements = snap.get("elements") or []+ # Mirror desktopActionWindowTarget: Windows targets by id, Linux by index.+ self.target = {"windowId": snap.get("windowId")} if snap.get("windowId") is not None else {"windowIndex": snap.get("windowIndex")}+ self.last_snapshot = snap+ return self.elements++ def _req(self, tool, element=None, escalate=False, **extra):+ req = {"tool": tool, "app": self.app, "element": element, "windowBounds": None, **self.target,+ "noScreenshot": True, "restoreWindow": bool(escalate)}+ req.update(extra)+ return req++ def act(self, action, el, escalate, text=""):+ if action == "click":+ req = self._req("click", el, escalate)+ elif action == "type":+ req = self._req("type_text", el, escalate, text=text)+ elif action == "return":+ req = self._req("press_key", el, escalate, key="return")+ elif action == "ctrl_a":+ req = self._req("hotkey", el, escalate, key="ctrl+a")+ elif action == "scroll":+ req = self._req("scroll", el, escalate, direction="down", pages=1)+ elif action == "choose":+ req = self._req("set_value", el, escalate, value="Blue")+ r, ms, _ = self.o.op(req)+ self.last_response = r+ return self._result(r, ms)++ def _result(self, r, ms):+ if not r.get("ok"):+ return {"ms": ms, "reported": "error", "error": True, "detail": str(r.get("error"))[:300]}+ a = r.get("action") or {}+ v = (a.get("verification") or {}).get("state")+ return {"ms": ms, "reported": v or "ok(no verification)", "error": False, "route": a.get("path"), "detail": json.dumps(a)[:300]}diff --git a/config/benchmarks/computer-use/sentinel.py b/config/benchmarks/computer-use/sentinel.pynew file mode 100644index 00000000000..bbf7027ab36--- /dev/null+++ b/config/benchmarks/computer-use/sentinel.py@@ -0,0 +1,57 @@+"""The "user": a frontmost Tk window that receives one 'q' every 200 ms from the bench typer.++Writes cumulative counts to <out>: q = 'q' keystrokes that reached it, other = any other+characters (agent input that leaked into the user's window).+"""+import json+import os+import sys++if os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true":+ raise SystemExit("fixture requires isolated CI desktop")+import tkinter as tk++out, geometry = sys.argv[1], sys.argv[2]+root = tk.Tk()+root.title("BenchSentinel")+root.geometry(geometry)+text = tk.Text(root, font=("Courier", 10))+text.pack(fill="both", expand=True)+base = {"q": 0, "other": 0}+ptr = {"clicks": 0, "wheels": 0}+root.bind_all("<ButtonPress>", lambda e: ptr.__setitem__("wheels" if e.num in (4, 5) else "clicks", ptr["wheels" if e.num in (4, 5) else "clicks"] + 1))+root.bind_all("<MouseWheel>", lambda e: ptr.__setitem__("wheels", ptr["wheels"] + 1))+last = [None]+++def key(event):+ if event.char == 'q':+ base['q'] += 1+ elif event.char:+ base['other'] += 1+++root.bind_all('<KeyPress>', key)+++def tick():+ try:+ focus = str(root.focus_get())+ except Exception:+ focus = "?"+ data = json.dumps({"q": base["q"], "other": base["other"], "focus": focus, "pid": os.getpid(), **ptr})+ if data != last[0]:+ tmp = out + ".tmp"+ try:+ with open(tmp, "w") as f:+ f.write(data)+ os.replace(tmp, out)+ last[0] = data+ except OSError:+ pass+ root.after(20, tick)+++root.after(100, lambda: (root.lift(), root.focus_force(), text.focus_set()))+root.after(20, tick)+root.mainloop()diff --git a/config/benchmarks/computer-use/test_trial_contract.py b/config/benchmarks/computer-use/test_trial_contract.pynew file mode 100644index 00000000000..096dca0b41e--- /dev/null+++ b/config/benchmarks/computer-use/test_trial_contract.py@@ -0,0 +1,111 @@+"""Regressions for false success claims and unsafe trial reuse in the original benchmark."""+from pathlib import Path+import sys+import tempfile+import time+import unittest++from process_scope import ProcessScope+from trial_contract import (DECOY, DiscoveryError, MARKER, TASKS, acknowledged, completed,+ initial_state, prepared, schedule, select_unique, summarize, verdict)+++class TaskContractTests(unittest.TestCase):+ def test_reset_requires_current_command_on_current_fixture(self):+ state = {**initial_state("targeted_type"), "instance": "new", "pid": 19, "ack": "current"}+ self.assertTrue(prepared(state, "targeted_type", "new", 19, "current"))+ for key, value in (("ack", "previous"), ("instance", "old"), ("pid", 20),+ ("logicalFocus", "field"), ("selLen", 1), ("top", 1), ("dropOpen", True)):+ with self.subTest(key=key):+ self.assertFalse(prepared({**state, key: value}, "targeted_type", "new", 19, "current"))+ self.assertFalse(acknowledged(state, "new", 19, "previous"))++ def test_selectors_never_substitute_another_control(self):+ selector = (r"button", r"\bincrement\b")+ desired = {"role": "button", "label": "Increment"}+ self.assertEqual(select_unique([{"role": "button", "label": "Delete"}, desired], selector), desired)+ for elements in ([{"role": "button", "label": "Delete"}],+ [{"role": "pane", "label": "Increment"}], [desired, dict(desired)], []):+ with self.subTest(elements=elements), self.assertRaises(DiscoveryError):+ select_unique(elements, selector)++ def test_completion_requires_exact_raw_text_and_unchanged_decoy(self):+ final = {**initial_state("targeted_type"), "text": MARKER}+ self.assertTrue(completed(final, "targeted_type"))+ for text in ("q" + MARKER, MARKER + "q", "Orca qcafé Ω", ""):+ with self.subTest(text=text):+ self.assertFalse(completed({**final, "text": text}, "targeted_type"))+ self.assertFalse(completed({**final, "decoyText": DECOY + MARKER}, "targeted_type"))++ def test_exactly_once_actions_reject_intermediate_and_duplicate_effects(self):+ for task, key in (("choose_and_click", "count"), ("type_submit", "returns"), ("replace_submit", "returns")):+ final = {**initial_state(task), key: 1}+ if task == "choose_and_click":+ final["combo"] = "Blue"+ else:+ final["text"] = MARKER+ self.assertTrue(completed(final, task))+ for value in (0, 2):+ with self.subTest(task=task, value=value):+ self.assertFalse(completed({**final, key: value}, task))++ def test_observed_completion_is_independent_of_api_errors(self):+ row = {"prepared": True, "task": "targeted_type", "api_errors": ["timeout"]}+ self.assertEqual(verdict(row, {**initial_state(row["task"]), "text": MARKER}), "completed")+ self.assertEqual(verdict(row, initial_state(row["task"])), "failed")+ self.assertEqual(verdict(row, None), "unobserved")+ self.assertEqual(verdict({**row, "prepared": False}, None), "invalid_setup")++ def test_invalid_setup_does_not_inflate_scored_completion(self):+ base = {"engine": "candidate", "task": "targeted_type", "profile": "background"}+ result = summarize([{**base, "verdict": "invalid_setup"},+ {**base, "verdict": "failed", "discovery_error": "no field"},+ {**base, "verdict": "unobserved"},+ {**base, "verdict": "completed", "api_errors": ["late error"]}])[0]+ self.assertEqual((result["planned"], result["scored"], result["invalid"], result["completed"]), (4, 3, 1, 1))+ self.assertEqual((result["discovery_failures"], result["unobserved"], result["api_errors"]), (1, 1, 1))++ def test_partial_failures_cannot_make_task_latency_look_fast(self):+ base = {"engine": "candidate", "task": "type_submit", "profile": "background"}+ result = summarize([{**base, "verdict": "completed", "attempt_ms": 400, "action_read_ms": [100, 150]},+ {**base, "verdict": "failed", "attempt_ms": 2, "action_read_ms": [1]},+ {**base, "verdict": "unobserved", "attempt_ms": 30000}])[0]+ self.assertEqual(result["task_ms"], 400)+ self.assertEqual(result["action_read_ms"], 125)+ self.assertEqual((result["timed_completions"], result["timed_failures"]), (1, 2))+ self.assertEqual(result["failed_attempt_ms"], 15001)++ def test_engine_order_is_reproducible_and_rotates_each_round(self):+ cells = schedule(["baseline", "candidate", "cua"], TASKS, 3, 417)+ self.assertEqual(cells, schedule(["baseline", "candidate", "cua"], TASKS, 3, 417))+ self.assertEqual(len(cells), len(TASKS) * 9)+ for task in TASKS:+ positions = [[cell["engine"] for cell in cells if cell["task"] == task and cell["round"] == index]+ for index in range(3)]+ for index in range(3):+ self.assertEqual({order[index] for order in positions}, {"baseline", "candidate", "cua"})++ def test_process_scope_stops_descendant_writes_before_returning(self):+ with tempfile.TemporaryDirectory() as directory:+ heartbeat = Path(directory) / "heartbeat"+ child = "import pathlib,time; p=pathlib.Path(" + repr(str(heartbeat)) + "); "+ child += "\nwhile True:\n p.write_text(str(time.monotonic())); time.sleep(0.02)"+ parent = "import subprocess,sys,time; subprocess.Popen([sys.executable,'-c'," + repr(child) + "]); time.sleep(60)"+ scope = ProcessScope(directory)+ try:+ proc = scope.spawn([sys.executable, "-c", parent], hidden=True)+ end = time.monotonic() + 10+ while not heartbeat.exists() and time.monotonic() < end:+ time.sleep(0.01)+ self.assertTrue(heartbeat.exists(), "descendant never started")+ scope.close()+ self.assertIsNotNone(proc.poll())+ final = heartbeat.read_bytes()+ time.sleep(0.12)+ self.assertEqual(heartbeat.read_bytes(), final)+ finally:+ scope.close()+++if __name__ == "__main__":+ unittest.main()diff --git a/config/benchmarks/computer-use/trial_contract.py b/config/benchmarks/computer-use/trial_contract.pynew file mode 100644index 00000000000..d7172625ec5--- /dev/null+++ b/config/benchmarks/computer-use/trial_contract.py@@ -0,0 +1,124 @@+"""Task definitions and independently observed benchmark verdicts. No desktop imports."""+import random+import re+import statistics++MARKER = "Orca café Ω"+DECOY = "untouched"+SELECTORS = {+ "field": (r"\b(edit|text|entry|textbox)\b", r"\bfield\b"),+ "button": (r"\b(button|push button)\b", r"\bincrement\b"),+ "color": (r"combo|popup|list ?box|menu ?button", r"\bcolor\b"),+ "scroll": (r"list|scroll|document|web", r"\blist\b|\bbenchelectron\b"),+}+TASKS = {+ "targeted_type": [("type", "field")],+ "type_submit": [("click", "field"), ("type", "field"), ("return", "field")],+ "replace_submit": [("click", "field"), ("ctrl_a", "field"), ("type", "field"), ("return", "field")],+ "choose_and_click": [("choose", "color"), ("click", "button")],+ "click_interference": [("click", "button")],+ "scroll_interference": [("scroll", "scroll")],+}+ROLE_KEYS = ("role", "control_type", "role_description", "localized_role", "subrole", "controlType", "localizedControlType")+LABEL_KEYS = ("label", "name", "title", "description", "automation_id", "automationId", "help", "placeholder")+++class DiscoveryError(Exception):+ pass+++def select_unique(elements, selector):+ role, label = selector+ matches = [element for element in elements+ if re.search(role, " ".join(str(element.get(key) or "") for key in ROLE_KEYS), re.I)+ and any(re.search(label, str(element.get(key) or ""), re.I) for key in LABEL_KEYS)]+ if len(matches) != 1:+ raise DiscoveryError(f"selector {selector!r} matched {len(matches)} elements")+ return matches[0]+++def initial_state(task):+ text = "hello" if task == "replace_submit" else ""+ return {"text": text, "decoyText": DECOY, "count": 0, "returns": 0, "combo": "Red", "top": 0,+ "check": False, "menuHits": 0, "dropOpen": False, "selStart": len(text), "selLen": 0,+ "logicalFocus": "decoy" if task == "targeted_type" else "field"}+++def acknowledged(state, instance, pid, nonce):+ return (state.get("instance") == instance and state.get("pid") == pid+ and state.get("ack") == nonce)+++def prepared(state, task, instance, pid, nonce):+ return acknowledged(state, instance, pid, nonce) and all(+ state.get(key) == value for key, value in initial_state(task).items())+++def completed(state, task):+ expected = initial_state(task)+ for key in ("selStart", "selLen", "logicalFocus"):+ del expected[key]+ if task in ("targeted_type", "type_submit", "replace_submit"):+ expected["text"] = MARKER+ if task in ("type_submit", "replace_submit"):+ expected["returns"] = 1+ if task in ("choose_and_click", "click_interference"):+ expected["count"] = 1+ if task == "choose_and_click":+ expected["combo"] = "Blue"+ if task == "scroll_interference":+ del expected["top"]+ if not isinstance(state.get("top"), (int, float)) or state["top"] <= 0:+ return False+ return all(state.get(key) == value for key, value in expected.items())+++def verdict(row, state):+ if not row.get("prepared"):+ return "invalid_setup"+ if state is None:+ return "unobserved"+ return "completed" if completed(state, row["task"]) else "failed"+++def schedule(engines, tasks, rounds, seed):+ rng = random.Random(seed)+ order = list(engines)+ rng.shuffle(order)+ cells = []+ for round_index in range(rounds):+ task_order = list(tasks)+ rng.shuffle(task_order)+ for task in task_order:+ for index in range(len(order)):+ cells.append({"engine": order[(index + round_index) % len(order)],+ "task": task, "round": round_index})+ return cells+++def summarize(rows):+ groups = {}+ for row in rows:+ groups.setdefault((row["engine"], row["task"], row["profile"]), []).append(row)+ summary = []+ for (engine, task, profile), group in sorted(groups.items()):+ valid = [row for row in group if row["verdict"] != "invalid_setup"]+ successful = [row for row in valid if row["verdict"] == "completed"]+ times = [row["attempt_ms"] for row in successful if row.get("attempt_ms") is not None]+ failures = [row["attempt_ms"] for row in valid if row["verdict"] != "completed" and row.get("attempt_ms") is not None]+ paired = [sample for row in successful for sample in row.get("action_read_ms", [])]+ summary.append({"engine": engine, "task": task, "profile": profile, "planned": len(group),+ "scored": len(valid), "invalid": len(group) - len(valid),+ "completed": sum(row["verdict"] == "completed" for row in valid),+ "unobserved": sum(row["verdict"] == "unobserved" for row in valid),+ "api_errors": sum(bool(row.get("api_errors")) for row in valid),+ **{key: sum(row.get(key, 0) for row in valid) for key in+ ("keys_sent", "keys_received", "keys_lost", "agent_text_in_sentinel", "target_q_characters")},+ "discovery_failures": sum(bool(row.get("discovery_error")) for row in valid),+ "task_ms": round(statistics.median(times), 1) if times else None,+ "failed_attempt_ms": round(statistics.median(failures), 1) if failures else None,+ "timed_completions": len(times), "timed_failures": len(failures),+ "action_read_ms": round(statistics.median(paired), 1) if paired else None,+ **{key: sum(bool(row.get("observation", {}).get(key)) for row in valid)+ for key in ("sentinel_lost_focus", "target_raised", "pointer_moved", "clipboard_changed")}})+ return summary