Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
fix(computer): verify Linux window restoration (#36234)
Kelvin Amoaba24cebf03b501
diff --git a/.github/workflows/computer-task-benchmark.yml b/.github/workflows/computer-task-benchmark.ymlindex baec1d44e87..f0adce6116b 100644--- a/.github/workflows/computer-task-benchmark.yml+++ b/.github/workflows/computer-task-benchmark.yml@@ -92,6 +92,12 @@ jobs:--cua "$RUNNER_TEMP/cua/cua-driver" \--electron "$RUNNER_TEMP/electron/node_modules/electron/dist/electron" \--out "out/task-linux-$BENCH_TARGET" --n 3 --budget 1800+ if [ "$BENCH_TARGET" = electron ]; then+ /usr/bin/python3 config/benchmarks/computer-use/linux_focus_probe.py \+ --electron "$RUNNER_TEMP/electron/node_modules/electron/dist/electron" \+ --baseline "$BASELINE_SHA" --candidate "$CANDIDATE_SHA" \+ --out "out/task-linux-$BENCH_TARGET/focus-probe"+ fi'- if: always()uses: actions/upload-artifact@v4diff --git a/config/benchmarks/computer-use/README.md b/config/benchmarks/computer-use/README.mdindex 1b7c3a7c6d1..7fb194bcf8a 100644--- a/config/benchmarks/computer-use/README.md+++ b/config/benchmarks/computer-use/README.md@@ -68,6 +68,20 @@ Failure of this probe identifies a capability to consider adding; it does not pra violation of the current public typing contract. Click-then-type tasks exercisethe existing sequence, although this benchmark still bypasses public transports.+The Linux Electron job also records restoration stages in `focus-probe/results.json`.+It records the AT-SPI focus acknowledgment, selected ACTIVE state, manual legacy PID+fallback, managed/native IDs, explicit singleton activation, minimized restoration+and the candidate's behavior with two same-PID windows. Managed windows include+minimized windows and exclude unmanaged helper windows.+The primary fixture publishes its native ID, checked against its PID before the probe.+Probe success establishes restoration behavior; actual text-and-submit completion+comes from the separate task results above.++AT-SPI ACTIVE is the runtime's focus authority. Global input can still race a later+focus change, and singleton accessibility/native counts do not establish an atomic+identity mapping if a window closes or the accessibility tree is stale. The probe+covers stable fixtures and detected ambiguity, not those concurrent lifecycle races.+## Headless contract checks```shdiff --git a/config/benchmarks/computer-use/electron/main.js b/config/benchmarks/computer-use/electron/main.jsindex a060995c952..fc501e1f423 100644--- a/config/benchmarks/computer-use/electron/main.js+++ b/config/benchmarks/computer-use/electron/main.js@@ -20,8 +20,16 @@ app.whenReady().then(() => {title: 'BenchElectron',webPreferences: { nodeIntegration: true, contextIsolation: false, backgroundThrottling: false }})+ const nativeWindowId =+ process.platform === 'linux' ? w.getNativeWindowHandle().readUInt32LE() : nullipcMain.on('st', (_event, state) => {- const data = JSON.stringify({ ...state, instance, activeForm: w.isFocused(), pid: process.pid })+ const data = JSON.stringify({+ ...state,+ instance,+ activeForm: w.isFocused(),+ pid: process.pid,+ nativeWindowId+ })try {fs.writeFileSync(path.join(dir, 'state.json.tmp'), data)fs.renameSync(path.join(dir, 'state.json.tmp'), path.join(dir, 'state.json'))@@ -67,6 +75,17 @@ app.whenReady().then(() => {setInterval(report, 30);</script></body></html>`)}`)+ if (process.env.BENCH_FOCUS_SIBLING === '1') {+ const sibling = new BrowserWindow({+ x: 400,+ y: 300,+ width: 300,+ height: 200,+ title: 'BenchSibling'+ })+ sibling.loadURL('data:text/html,<title>BenchSibling</title><input aria-label="sibling">')+ w.on('closed', () => sibling.close())+ }setInterval(() => {const cmdPath = path.join(dir, 'cmd')if (!fs.existsSync(cmdPath)) {diff --git a/config/benchmarks/computer-use/linux_focus_probe.py b/config/benchmarks/computer-use/linux_focus_probe.pynew file mode 100644index 00000000000..9a9ff1c32f9--- /dev/null+++ b/config/benchmarks/computer-use/linux_focus_probe.py@@ -0,0 +1,137 @@+#!/usr/bin/env python3+"""Record Linux restoration stages on a disposable Electron fixture."""+import argparse+import importlib.util+import os+from pathlib import Path+import sys+import time+++def load_runtime(path, name):+ spec = importlib.util.spec_from_file_location(name, path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ module.Atspi.set_timeout(800, 15000)+ return module+++def main():+ if (sys.platform != "linux" or os.environ.get("GITHUB_ACTIONS") != "true"+ or os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1"+ or os.environ.get("ORCA_BACKGROUND_LAUNCH") != "1"):+ raise SystemExit("focus probe requires isolated Linux CI")+ import bench+ import obs_x11 as obs+ bench.obs = obs+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--electron", required=True)+ parser.add_argument("--baseline", required=True)+ parser.add_argument("--candidate", required=True)+ parser.add_argument("--out", type=Path, required=True)+ args = parser.parse_args()+ args.target = "electron"+ args.out = args.out.resolve()+ args.out.mkdir(parents=True)+ sources = args.out / "sources"+ evidence = {"sources": {name: bench.pin_source(sha, sources / name) for name, sha in+ (("baseline", args.baseline), ("candidate", args.candidate))}, "trials": []}+ baseline = load_runtime(sources / "baseline/native/computer-use-linux/runtime.py", "focus_baseline")+ candidate = load_runtime(sources / "candidate/native/computer-use-linux/runtime.py", "focus_candidate")+ with (args.out / "probe.log").open("ab") as log:+ for mode in ("one-window", "minimized", "two-windows"):+ sibling = mode == "two-windows"+ os.environ["BENCH_FOCUS_SIBLING"] = "1" if sibling else "0"+ trial = bench.Trial(args, args.out / mode, log)+ row = {"mode": mode, "sibling": sibling}+ evidence["trials"].append(row)+ try:+ trial.launch()+ trial.target = trial.state()["nativeWindowId"]+ if not trial.target or obs.main_display().pid_of(trial.target) != trial.pid:+ raise AssertionError("primary fixture native identity was not confirmed")+ def prepare():+ trial.prepare("type_submit")+ if mode == "minimized":+ obs.xdotool("windowminimize", trial.target)+ bench.wait_for(lambda: (obs.main_display().prop_longs(trial.target, "WM_STATE") or [0])[0] == 3)+ prepare()+ app = bench.wait_for(lambda: baseline.attempt(lambda: baseline.find_app(f"pid:{trial.pid}")))+ if sibling:+ bench.wait_for(lambda: len(baseline.windows_for(app)) == 2)+ matches = [(i, w) for i, w in baseline.windows_for(app) if baseline.name_of(w) == "BenchElectron"]+ if len(matches) != 1:+ raise AssertionError("fixture accessibility window is missing or ambiguous")+ index, window = matches[0]+ row.update({"pid": trial.pid, "fixture_window": trial.target, "atspi_index": index,+ "atspi_windows": [i for i, _ in baseline.windows_for(app)],+ "native_ids": obs.xdotool("search", "--pid", trial.pid).stdout.split()})+ row["managed_ids"] = [value for value in row["native_ids"]+ if int(value) in obs.main_display().stacking()]++ def trace():+ return {"selected_active": baseline.has_state(window, baseline.Atspi.StateType.ACTIVE),+ "native_active": obs.main_display().active(), "fixture_state": trial.state()}++ row["before"] = trace()+ try:+ row["grab_focus"] = bool(baseline.Atspi.Component.grab_focus(window.get_component_iface()))+ except Exception as error:+ row["grab_focus_error"] = repr(error)+ row["after_grab"] = []+ for _ in range(11):+ row["after_grab"].append(trace())+ time.sleep(0.05)+ if not row["after_grab"][-1]["selected_active"]:+ try:+ result = obs.xdotool("search", "--pid", trial.pid, "windowactivate", "--sync")+ row["manual_legacy_fallback"] = {"status": result.returncode, "stderr": result.stderr}+ except Exception as error:+ row["manual_legacy_fallback"] = {"error": repr(error)}+ time.sleep(0.5)+ row["after_legacy"] = trace()+ prepare()+ ids = row["managed_ids"]+ if not sibling and len(ids) == 1:+ result = obs.xdotool("windowactivate", "--sync", ids[0])+ time.sleep(0.5)+ row["explicit_fallback"] = {"status": result.returncode, "after": trace()}+ prepare()+ row["candidate_before"] = trace()+ candidate_app = candidate.find_app(f"pid:{trial.pid}")+ _, candidate_window = candidate.choose_window(candidate_app, window_index=index)+ native_calls = []+ native_run = candidate.subprocess.run+ def observed_run(command, **kwargs):+ native_calls.append(command)+ return native_run(command, **kwargs)+ candidate.subprocess.run = observed_run+ try:+ candidate.restore_window(candidate_app, candidate_window)+ finally:+ candidate.subprocess.run = native_run+ row["candidate_native_calls"] = native_calls+ time.sleep(0.5)+ row["candidate_after"] = trace()+ if sibling:+ if any("windowactivate" in command for command in native_calls):+ raise AssertionError("ambiguous restoration used native activation")+ if row["candidate_after"]["native_active"] not in (trial.target, trial.sentinel):+ raise AssertionError("ambiguous restoration activated a sibling")+ if (row["candidate_after"]["native_active"] == trial.target+ and not row["candidate_after"]["selected_active"]):+ raise AssertionError("native and accessibility focus disagree")+ elif not row["candidate_after"]["selected_active"]:+ raise AssertionError("singleton restoration did not focus the selected window")+ row["passed"] = True+ except Exception as error:+ row["error"] = repr(error)+ finally:+ trial.close()+ bench.write_json(args.out / "results.json", evidence)+ return 0 if all(row.get("passed") for row in evidence["trials"]) else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.pyindex d20ab09efcb..5c16031182a 100644--- a/native/computer-use-linux/runtime.py+++ b/native/computer-use-linux/runtime.py@@ -35,6 +35,12 @@ except (ImportError, ValueError):GdkPixbuf = Nonefrom gi.repository import Atspi+try:+ gi.require_version("GdkX11", "3.0")+ from gi.repository import GdkX11+except (ImportError, ValueError):+ GdkX11 = None+MAX_NODES = 1200MAX_DEPTH = 64TEXT_LIMIT = 500@@ -182,19 +188,40 @@ def choose_window(app, window_id=None, window_index=None):def restore_window(app, window=None):- target = window if window is not None else app+ target = window if window is not None else choose_window(app)[1]+ if has_state(target, Atspi.StateType.ACTIVE):+ returncomponent = attempt(target.get_component_iface)- if component is not None and attempt(lambda: Atspi.Component.grab_focus(component), False):+ if component is not None:+ attempt(lambda: Atspi.Component.grab_focus(component), False)+ if wait_window_focused(target):+ return+ if os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland":+ return+ windows = windows_for(app)+ if len(windows) != 1 or windows[0][1] != target:returnpid = pid_of(app)- if not pid or not shutil.which("xdotool"):+ xdotool = shutil.which("xdotool")+ if not pid or not xdotool:+ return+ try:+ result = subprocess.run([xdotool, "search", "--pid", str(pid)],+ check=False, capture_output=True, text=True, timeout=2)+ ids = result.stdout.split()+ if result.returncode != 0 or any(not value.isdigit() or int(value) <= 0 for value in ids):+ return+ managed_ids = attempt(lambda: {str(GdkX11.X11Window.get_xid(item))+ for item in Gdk.Screen.get_default().get_window_stack()})+ if managed_ids is None or any(not value.isdigit() or int(value) <= 0 for value in managed_ids):+ return+ ids = [value for value in ids if value in managed_ids]+ if len(ids) != 1:+ return+ subprocess.run([xdotool, "windowactivate", "--sync", ids[0]], check=False,+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2)+ except (OSError, subprocess.TimeoutExpired):return- subprocess.run(- ["xdotool", "search", "--pid", str(pid), "windowactivate", "--sync"],- check=False,- stdout=subprocess.DEVNULL,- stderr=subprocess.DEVNULL,- )def wait_window_focused(window):diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.pyindex 3d0503d7e77..16c901f1f3c 100644--- a/native/computer-use-linux/runtime_render_test.py+++ b/native/computer-use-linux/runtime_render_test.py@@ -346,6 +346,86 @@ class RuntimeRenderTest(unittest.TestCase):self.assertGreaterEqual(fixture.clock, 0.5)self.assertTrue(restored)+ def test_restoration_requires_selected_window_focus_before_keyboard_input(self):+ restore_window = runtime.restore_window+ cases = (+ {"name": "already active", "initial": 71, "expected": True, "native": False},+ {"name": "delayed AT-SPI focus", "ack": False, "delay": 0.1, "expected": True, "native": False},+ {"name": "ineffective true acknowledgment", "ack": True, "expected": True, "native": True},+ {"name": "ineffective false acknowledgment", "ack": False, "expected": True, "native": True},+ {"name": "unmanaged helper", "ids": "70\n71\n", "managed": [71], "expected": True, "native": True},+ {"name": "minimized selected window", "minimized": True, "expected": True, "native": True},+ {"name": "two accessible windows", "windows": 2},+ {"name": "two native windows", "ids": "72\n71\n"},+ {"name": "missing managed stack", "managed": None},+ {"name": "failed managed window identity", "managed": [71, None]},+ {"name": "invalid managed window identity", "managed": [71, 0]},+ {"name": "Wayland", "session": "wayland"},+ {"name": "search timeout", "timeout": "search"},+ {"name": "activation timeout", "timeout": "windowactivate"},+ {"name": "activation refused", "refused": True},+ )+ for case in cases:+ with self.subTest(case=case["name"]), SnapshotFixture(FakeAccessible("frame", "Editor")) as fixture:+ active = [case.get("initial", 99)]+ activations, keys = [], []+ if case.get("windows") == 2:+ fixture.app.children.append(FakeAccessible("frame", "Sibling"))++ def state():+ if fixture.clock >= case.get("delay", float("inf")):+ active[0] = 71+ states = [] if case.get("minimized") else [FakeStateType.SHOWING]+ if active[0] == 71:+ states.append(FakeStateType.ACTIVE)+ return FakeStateSet(*states)++ def sleep(seconds):+ fixture.clock += seconds++ def activate(window_id):+ activations.append(window_id)+ if not case.get("refused"):+ active[0] = window_id++ def native(command, **_kwargs):+ stage = command[1]+ if case.get("timeout") == stage or (case.get("timeout") == "windowactivate" and "windowactivate" in command):+ raise runtime.subprocess.TimeoutExpired(command, 2)+ if stage == "search":+ ids = case.get("ids", "71\n")+ if "windowactivate" in command:+ activate(int(ids.split()[0]))+ return types.SimpleNamespace(returncode=0, stdout=ids)+ if stage == "windowactivate":+ activate(int(command[-1]))+ return types.SimpleNamespace(returncode=int(bool(case.get("refused"))), stdout="")+ raise AssertionError("unexpected native operation")++ fixture.root.get_state_set = state+ with patch.object(runtime, "restore_window", restore_window), \+ patch.object(runtime.time, "sleep", sleep), \+ patch.object(runtime.Atspi.Component, "grab_focus", lambda _: case.get("ack", False), create=True), \+ patch.object(runtime.Atspi, "KeySynthType", types.SimpleNamespace(STRING=1), create=True), \+ patch.object(runtime.Atspi, "generate_keyboard_event", lambda *_: keys.append(active[0]), create=True), \+ patch.object(runtime.Gdk, "Screen", types.SimpleNamespace(get_default=lambda: types.SimpleNamespace(+ get_window_stack=lambda: case.get("managed", [71, 72]))), create=True), \+ patch.object(runtime, "GdkX11", types.SimpleNamespace(X11Window=types.SimpleNamespace(get_xid=int)), create=True), \+ patch.object(runtime.shutil, "which", lambda _: "/usr/bin/xdotool"), \+ patch.object(runtime.subprocess, "run", native), \+ patch.dict(runtime.os.environ, {"XDG_SESSION_TYPE": case.get("session", "x11")}):+ request = {"tool": "press_key", "key": "x", "app": "Fixture", "windowIndex": 0,+ "restoreWindow": True, "noScreenshot": True}+ if case.get("expected"):+ self.assertTrue(runtime.run_operation(request)["ok"])+ else:+ with self.assertRaisesRegex(RuntimeError, "window_not_focused"):+ runtime.run_operation(request)+ self.assertEqual(keys, [71] if case.get("expected") else [])+ self.assertEqual(activations, [71] if case.get("native") or case.get("refused") else [])+ if not case.get("expected"):+ self.assertEqual(active[0], 99)+def test_text_entry_inserts_into_the_focused_field_of_a_background_window(self):for tool in ("type_text", "paste_text"):for value in ("there", "élan", "🚀 café"):