Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
fix(computer): skip activation for accessibility clicks (#36006)
Kelvin Amoaba3b83d3d2608c
diff --git a/config/scripts/verify-computer-native.mjs b/config/scripts/verify-computer-native.mjsindex 896558aa919..dcd4983de38 100644--- a/config/scripts/verify-computer-native.mjs+++ b/config/scripts/verify-computer-native.mjs@@ -231,7 +231,7 @@ function verifyNativeArgumentGuardrails() {}if (!linux.includes('def require_non_empty_string(value, name):') ||- !linux.includes('type_text(require_non_empty_string(operation.get("text"), "text"))') ||+ !linux.includes('text = require_non_empty_string(operation.get("text"), "text")') ||!linux.includes('press_key(require_non_empty_string(operation.get("key"), "key"))')) {failures.push(@@ -302,11 +302,6 @@ function verifyNativeArgumentGuardrails() {if (/restore_window\(app\)(?!:)/.test(linux)) {failures.push('Linux restore call sites must pass the selected window target')}- if (!linux.includes('even when the click uses an accessibility path')) {- failures.push(- 'Linux clicks must activate the target window before accessibility or pointer input'- )- }if (!linux.includes('compact_browser_tabs=is_browser_app(query, app)') ||!linux.includes('inactive browser tabs omitted')@@ -405,11 +400,6 @@ function verifyNativeArgumentGuardrails() {'Windows text/key actions must reject missing payloads instead of sending empty input')}- if (!windows.includes('even when UI Automation handles the click')) {- failures.push(- 'Windows clicks must activate the target window before UI Automation or pointer input'- )- }if (!windows.includes('Render-OrcaTree $root $windowFrame (Test-OrcaBrowserProcess $process)') ||!windows.includes('inactive browser tabs omitted')diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.pyindex ffac9b92cf5..d20ab09efcb 100644--- a/native/computer-use-linux/runtime.py+++ b/native/computer-use-linux/runtime.py@@ -197,21 +197,32 @@ def restore_window(app, window=None):)+def wait_window_focused(window):+ deadline = time.monotonic() + 0.5+ while time.monotonic() < deadline:+ if has_state(window, Atspi.StateType.ACTIVE):+ return True+ time.sleep(0.05)+ return has_state(window, Atspi.StateType.ACTIVE)++def require_keyboard_focus(window, operation):if has_state(window, Atspi.StateType.ACTIVE):returnif operation.get("restoreWindow"):- deadline = time.monotonic() + 0.5- while time.monotonic() < deadline:- if has_state(window, Atspi.StateType.ACTIVE):- return- time.sleep(0.05)- if has_state(window, Atspi.StateType.ACTIVE):+ if wait_window_focused(window):returnraise RuntimeError("window_not_focused: keyboard input requires the target window to be focused; restoreWindow was requested but the target window is still not focused; bring it forward manually or check desktop permissions")raise RuntimeError("window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window")+def restore_pointer_window(app, window):+ restore_window(app, window)+ if not wait_window_focused(window):+ raise RuntimeError("window_not_focused: window restoration did not focus the target window; no pointer input was sent")+ return screen_rect(window)++def app_matches(app, query):needle = str(query or "").strip().lower()if not needle:@@ -987,6 +998,56 @@ def type_text(value):Atspi.generate_keyboard_event(0, str(value), Atspi.KeySynthType.STRING)+def insert_focused_text(window, value):+ # Why: EditableText needs no window focus and still works on Wayland, where key synthesis is a no-op.+ node = first_descendant(window, lambda candidate: has_state(candidate, Atspi.StateType.FOCUSED))+ if node is None or not has_state(node, Atspi.StateType.EDITABLE):+ return None+ editable = attempt(node.get_editable_text_iface)+ text = text_iface(node)+ if editable is None or text is None:+ return None+ start = end = int(attempt(lambda: Atspi.Text.get_caret_offset(text), -1))+ selection_count = attempt(lambda: Atspi.Text.get_n_selections(text))+ if selection_count not in (0, 1):+ return None+ if selection_count == 1:+ selection = attempt(lambda: Atspi.Text.get_selection(text, 0))+ if selection is None or not 0 <= selection.start_offset < selection.end_offset:+ return None+ start, end = selection.start_offset, selection.end_offset+ if start < 0:+ return None+ original_count = attempt(lambda: Atspi.Text.get_character_count(text))+ if original_count is None or original_count < 0 or end > original_count:+ return None+ original = None+ if end > start:+ original = attempt(lambda: Atspi.Text.get_text(text, 0, original_count))+ if original is None or len(original) != original_count:+ return None+ inserted_count = original_count + len(value)+ if (not attempt(lambda: Atspi.EditableText.insert_text(editable, end, value, len(value.encode("utf-8"))), False)+ or attempt(lambda: Atspi.Text.get_character_count(text)) != inserted_count+ or attempt(lambda: Atspi.Text.get_text(text, end, end + len(value))) != value):+ raise RuntimeError("text_insertion_unconfirmed: text insertion did not confirm success; inspect state before retrying")+ # Why insert before deleting: an unsuccessful insert leaves the selected text intact.+ if end > start and (not attempt(lambda: Atspi.EditableText.delete_text(editable, start, end), False)+ or attempt(lambda: Atspi.Text.get_character_count(text)) != inserted_count - (end - start)+ or attempt(lambda: Atspi.Text.get_text(text, 0, inserted_count - (end - start)))+ != original[:start] + value + original[end:]):+ raise RuntimeError("text_replacement_incomplete: text was inserted but selection deletion did not confirm success; inspect state before retrying")+ caret = start + len(value)+ if attempt(lambda: Atspi.Text.get_caret_offset(text)) != caret:+ if (not attempt(lambda: Atspi.Text.set_caret_offset(text, caret), False)+ or attempt(lambda: Atspi.Text.get_caret_offset(text)) != caret):+ raise RuntimeError("text_caret_unconfirmed: text was inserted but caret movement did not confirm success; inspect state before retrying")+ action = {"path": "accessibility", "actionName": "insertText", "fallbackReason": None}+ if attempt(lambda: Atspi.Text.get_text(text, start, caret)) == value:+ action["verification"] = {"state": "verified", "property": "focusedText", "expected": value}+ return action++def paste_text(value):text = str(value)previous = read_clipboard()@@ -1089,7 +1150,7 @@ def run_operation(operation):if operation.get("restoreWindow"):restore_window(app, window)_, window = choose_window(app, operation.get("windowId"), operation.get("windowIndex"))- if tool in {"type_text", "press_key", "hotkey", "paste_text"}:+ if tool in {"press_key", "hotkey"}:require_keyboard_focus(window, operation)bounds = screen_rect(window)saved = operation.get("element")@@ -1099,9 +1160,6 @@ def run_operation(operation):action = Noneif tool == "click":- # Why: agents expect a click into a target app to make the next- # keyboard action safe, even when the click uses an accessibility path.- restore_window(app, window)preferred = preferred_action(node)click_count = (require_positive_integer(operation.get("click_count"), "click_count")@@ -1111,6 +1169,8 @@ def run_operation(operation):has_modifiers = bool(str(operation.get("modifiers") or "").strip())handled = not has_modifiers and operation.get("mouse_button", "left") == "left" and click_count <= 1 and perform_action(node, preferred)if not handled:+ # Why: pointer clicks are synthetic global input; AT-SPI actions need no focus.+ bounds = restore_pointer_window(app, window)click_at(*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node),operation.get("mouse_button", "left"),@@ -1132,21 +1192,25 @@ def run_operation(operation):elif tool == "scroll":# Why: pointer wheel events should land in the requested app window even# when another desktop window is currently foregrounded.- restore_window(app, window)+ bounds = restore_pointer_window(app, window)scroll_at(*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node), operation.get("direction"), operation.get("pages"))action = {"path": "synthetic", "actionName": "scroll", "fallbackReason": None}elif tool == "drag":# Why: pointer drags are synthetic global input, so activate the target# window before using cached coordinates from its accessibility tree.- restore_window(app, window)+ bounds = restore_pointer_window(app, window)drag_between(screen_point(bounds, operation.get("fromElement"), operation.get("from_x"), operation.get("from_y"), from_node),screen_point(bounds, operation.get("toElement"), operation.get("to_x"), operation.get("to_y"), to_node),)action = {"path": "synthetic", "actionName": "drag", "fallbackReason": None}elif tool == "type_text":- type_text(require_non_empty_string(operation.get("text"), "text"))- action = {"path": "synthetic", "actionName": "typeText", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}+ text = require_non_empty_string(operation.get("text"), "text")+ action = insert_focused_text(window, text)+ if action is None:+ require_keyboard_focus(window, operation)+ type_text(text)+ action = {"path": "synthetic", "actionName": "typeText", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}elif tool == "press_key":press_key(require_non_empty_string(operation.get("key"), "key"))action = {"path": "synthetic", "actionName": "pressKey", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}@@ -1154,8 +1218,12 @@ def run_operation(operation):hotkey(require_non_empty_string(operation.get("key"), "key"))action = {"path": "synthetic", "actionName": "hotkey", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}elif tool == "paste_text":- paste_text(require_non_empty_string(operation.get("text"), "text"))- action = {"path": "clipboard", "actionName": "paste", "fallbackReason": None, "verification": {"state": "unverified", "reason": "clipboard_paste"}}+ text = require_non_empty_string(operation.get("text"), "text")+ action = insert_focused_text(window, text)+ if action is None:+ require_keyboard_focus(window, operation)+ paste_text(text)+ action = {"path": "clipboard", "actionName": "paste", "fallbackReason": None, "verification": {"state": "unverified", "reason": "clipboard_paste"}}elif tool == "set_value":if not set_value(node, operation.get("value", "")):raise RuntimeError("element value is not settable")diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.pyindex 4574d2e7d12..3d0503d7e77 100644--- a/native/computer-use-linux/runtime_render_test.py+++ b/native/computer-use-linux/runtime_render_test.py@@ -14,6 +14,15 @@ class FakeStateType:ACTIVE = 3SHOWING = 4FOCUSED = 5+ EDITABLE = 6+++class FakeStateSet:+ def __init__(self, *states):+ self.states = set(states)++ def contains(self, state):+ return state in self.statesclass FakeText:@@ -25,12 +34,43 @@ class FakeText:def get_text(node, start, end):return node.value[start:end]+ @staticmethod+ def get_caret_offset(node):+ return node.caret++ @staticmethod+ def set_caret_offset(node, offset):+ node.caret = offset+ return True++ @staticmethod+ def get_n_selections(node):+ return 1 if node.selection else 0++ @staticmethod+ def get_selection(node, _):+ return types.SimpleNamespace(start_offset=node.selection[0], end_offset=node.selection[1])+++class FakeEditableText:+ @staticmethod+ def insert_text(node, position, text, length):+ inserted = text.encode("utf-8")[:length].decode("utf-8")+ node.value = node.value[:position] + inserted + node.value[position:]+ return True++ @staticmethod+ def delete_text(node, start, end):+ node.value = node.value[:start] + node.value[end:]+ node.selection = None+ return True+def load_runtime():gi = types.ModuleType("gi")repository = types.ModuleType("gi.repository")gi.require_version = lambda *_: None- repository.Atspi = types.SimpleNamespace(StateType=FakeStateType, Text=FakeText)+ repository.Atspi = types.SimpleNamespace(StateType=FakeStateType, Text=FakeText, EditableText=FakeEditableText)repository.Gdk = types.SimpleNamespace()repository.GdkPixbuf = types.SimpleNamespace()gi.repository = repository@@ -242,6 +282,215 @@ class RuntimeRenderTest(unittest.TestCase):runtime.run_operation({"tool": "click", "app": "Fixture"})self.assertEqual(effects, ["clicked"])+ def test_click_activates_the_window_only_for_pointer_fallback(self):+ button = FakeAccessible("button", "Continue")+ button.get_n_actions = lambda: 1+ button.get_action_name = lambda _: "click"+ button.do_action = lambda _: True+ label = FakeAccessible("label", "Status")+ for target, expected_path, expected_calls in (+ (button, "accessibility", []),+ (label, "synthetic", ["restore", "pointer"]),+ ):+ with self.subTest(path=expected_path), SnapshotFixture(+ FakeAccessible("frame", "Editor", children=[button, label])+ ) as fixture:+ calls = []+ def restore():+ calls.append("restore")+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)+ fixture.restore = restore+ snapshot = runtime.make_snapshot("Fixture", False)+ element = next(item for item in snapshot["elements"] if item["name"] == target.name)+ with patch.object(runtime, "click_at", lambda *_: calls.append("pointer")):+ result = runtime.run_operation({"tool": "click", "app": "Fixture", "element": element, "noScreenshot": True})+ self.assertEqual(result["action"]["path"], expected_path)+ self.assertEqual(calls, expected_calls)++ def test_pointer_input_waits_for_activation_and_uses_restored_geometry(self):+ operations = (+ {"tool": "click", "x": 5, "y": 6},+ {"tool": "scroll", "x": 5, "y": 6, "direction": "down", "pages": 1},+ {"tool": "drag", "from_x": 5, "from_y": 6, "to_x": 25, "to_y": 26},+ )+ for operation in operations:+ for activates in (False, True):+ with self.subTest(tool=operation["tool"], activates=activates), SnapshotFixture(+ FakeAccessible("frame", "Editor")+ ) as fixture:+ restored = []+ pointer_calls = []+ fixture.root.get_state_set = lambda: FakeStateSet(+ *([FakeStateType.ACTIVE] if activates and restored and fixture.clock >= 0.1 else [])+ )+ def restore():+ restored.append(True)+ fixture.bounds[fixture.root] = runtime.Rect(500, 200, 300, 400)+ fixture.restore = restore+ def sleep(seconds):+ fixture.clock += seconds+ with patch.object(runtime.time, "sleep", sleep), \+ patch.object(runtime, "click_at", lambda *args: pointer_calls.append(args[:2])), \+ patch.object(runtime, "scroll_at", lambda *args: pointer_calls.append(args[:2])), \+ patch.object(runtime, "drag_between", lambda start, end: pointer_calls.extend((start, end))):+ request = {**operation, "app": "Fixture", "noScreenshot": True, "restoreWindow": False}+ if activates:+ result = runtime.run_operation(request)+ self.assertTrue(result["ok"])+ self.assertEqual(pointer_calls, [(505, 206), (525, 226)] if operation["tool"] == "drag" else [(505, 206)])+ self.assertGreaterEqual(fixture.clock, 0.1)+ else:+ with self.assertRaisesRegex(RuntimeError, "window_not_focused"):+ runtime.run_operation(request)+ self.assertEqual(pointer_calls, [])+ self.assertGreaterEqual(fixture.clock, 0.5)+ self.assertTrue(restored)++ def test_text_entry_inserts_into_the_focused_field_of_a_background_window(self):+ for tool in ("type_text", "paste_text"):+ for value in ("there", "élan", "🚀 café"):+ for selection in ((6, 11), None):+ field = FakeAccessible("text", "Message", value="Hello world")+ field.caret, field.selection = 11, selection+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)+ field.get_editable_text_iface = lambda: field+ with self.subTest(tool=tool, value=value, selection=selection), SnapshotFixture(FakeAccessible("frame", "Editor", children=[field])) as fixture:+ calls = []+ fixture.restore = lambda: calls.append("restore")+ with patch.object(runtime, "type_text", lambda _: calls.append("keys")), \+ patch.object(runtime, "paste_text", lambda _: calls.append("clipboard")):+ result = runtime.run_operation({"tool": tool, "app": "Fixture", "text": value, "noScreenshot": True})+ self.assertEqual(calls, [])+ prefix = "Hello " if selection else "Hello world"+ self.assertEqual((field.value, field.caret), (prefix + value, len(prefix) + len(value)))+ self.assertEqual(result["action"]["path"], "accessibility")+ self.assertEqual(result["action"]["verification"]["state"], "verified")++ def test_text_entry_without_an_editable_focused_field_still_requires_window_focus(self):+ cases = [+ ("not-editable", 1, (6, 11)),+ ("count-error", TimeoutError("selection timed out"), (6, 11)),+ ("count-missing", None, (6, 11)),+ ("count-negative", -1, (6, 11)),+ ("multiple", 2, (6, 11)),+ ("range-error", 1, TimeoutError("range timed out")),+ ("range-missing", 1, None),+ ("range-empty", 1, (6, 6)),+ ("range-reversed", 1, (11, 6)),+ ("range-outside", 1, (6, 12)),+ ]+ for tool in ("type_text", "paste_text"):+ for active in (False, True):+ for name, count, selection in cases:+ field = FakeAccessible("text", "Message", value="Hello world")+ field.caret, field.selection = 11, (6, 11)+ states = [FakeStateType.FOCUSED] + ([] if name == "not-editable" else [FakeStateType.EDITABLE])+ field.get_state_set = lambda: FakeStateSet(*states)+ field.get_editable_text_iface = lambda: field+ selected = types.SimpleNamespace(start_offset=selection[0], end_offset=selection[1]) if isinstance(selection, tuple) else selection+ with self.subTest(tool=tool, active=active, case=name), SnapshotFixture(+ FakeAccessible("frame", "Editor", children=[field])+ ) as fixture, patch.object(runtime.Atspi.Text, "get_n_selections", return_value=count, side_effect=count if isinstance(count, Exception) else None), \+ patch.object(runtime.Atspi.Text, "get_selection", return_value=selected, side_effect=selected if isinstance(selected, Exception) else None), \+ patch.object(runtime.Atspi.EditableText, "insert_text", wraps=FakeEditableText.insert_text) as insert, \+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:+ if active:+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)+ result = runtime.run_operation({"tool": tool, "app": "Fixture", "text": "x", "noScreenshot": True})+ (keys if tool == "type_text" else paste).assert_called_once_with("x")+ self.assertEqual(result["action"]["verification"]["state"], "unverified")+ else:+ with self.assertRaisesRegex(RuntimeError, "window_not_focused"):+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "x", "noScreenshot": True})+ if not active or tool != "type_text":+ keys.assert_not_called()+ if not active or tool != "paste_text":+ paste.assert_not_called()+ insert.assert_not_called()+ self.assertEqual((field.value, field.selection), ("Hello world", (6, 11)))++ def test_partial_text_replacement_fails_without_typing_again(self):+ for tool in ("type_text", "paste_text"):+ for failure in (False, True, TimeoutError("provider timed out"), "wrong-range"):+ field = FakeAccessible("text", "Message", value="Hello world")+ field.caret, field.selection = 11, (6, 11)+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)+ field.get_editable_text_iface = lambda: field+ def delete(node, _start, _end):+ if failure == "wrong-range":+ node.value = node.value[5:]+ node.selection = None+ return True+ if isinstance(failure, Exception):+ raise failure+ return failure+ with self.subTest(tool=tool, failure=failure), SnapshotFixture(+ FakeAccessible("frame", "Editor", children=[field])+ ) as fixture, patch.object(runtime.Atspi.EditableText, "delete_text", delete), \+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)+ with self.assertRaisesRegex(RuntimeError, "text_replacement_incomplete"):+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "there", "noScreenshot": True})+ self.assertEqual(field.value, " worldthere" if failure == "wrong-range" else "Hello worldthere")+ keys.assert_not_called()+ paste.assert_not_called()++ def test_unconfirmed_text_insertion_does_not_fall_back_to_another_write(self):+ write = FakeEditableText.insert_text+ failures = {+ "false-after-write": "foobarbar",+ "timeout-after-write": "foobarbar",+ "no-op": "foobar",+ "truncated": "foobbar",+ "wrong-text": "foobazbar",+ "count-unavailable": "foobarbar",+ }+ for tool in ("type_text", "paste_text"):+ for failure, expected in failures.items():+ field = FakeAccessible("text", "Message", value="foobar")+ field.caret, field.selection = 6, (0, 3)+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)+ field.get_editable_text_iface = lambda: field+ def insert(node, position, text, length):+ if failure == "no-op":+ return True+ write(node, position, "baz" if failure == "wrong-text" else text, 1 if failure == "truncated" else length)+ if failure == "timeout-after-write":+ raise TimeoutError("reply lost after insertion")+ return failure != "false-after-write"+ def count(node):+ return None if failure == "count-unavailable" and node.value != "foobar" else len(node.value)+ with self.subTest(tool=tool, failure=failure), SnapshotFixture(+ FakeAccessible("frame", "Editor", children=[field])+ ) as fixture, patch.object(runtime.Atspi.EditableText, "insert_text", side_effect=insert), \+ patch.object(runtime.Atspi.Text, "get_character_count", side_effect=count), \+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)+ with self.assertRaisesRegex(RuntimeError, "text_insertion_unconfirmed"):+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "bar", "noScreenshot": True})+ self.assertEqual(field.value, expected)+ self.assertEqual(field.selection, (0, 3))+ keys.assert_not_called()+ paste.assert_not_called()++ def test_unconfirmed_caret_movement_does_not_report_text_entry_success(self):+ for tool in ("type_text", "paste_text"):+ for failure in (False, True, TimeoutError("provider timed out")):+ field = FakeAccessible("text", "Message", value="Hello world")+ field.caret, field.selection = 6, (6, 11)+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)+ field.get_editable_text_iface = lambda: field+ with self.subTest(tool=tool, failure=failure), SnapshotFixture(+ FakeAccessible("frame", "Editor", children=[field])+ ) as fixture, patch.object(runtime.Atspi.Text, "set_caret_offset", side_effect=failure if isinstance(failure, Exception) else None, return_value=failure), \+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)+ with self.assertRaisesRegex(RuntimeError, "text_caret_unconfirmed"):+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "there", "noScreenshot": True})+ self.assertEqual((field.value, field.caret), ("Hello there", 6))+ keys.assert_not_called()+ paste.assert_not_called()+def test_compacted_browser_records_keep_their_own_frames_and_indexes(self):tabs = [FakeAccessible("page tab", f"Tab {i}", value="1" if i == 9 else "") for i in range(10)]buttons = [FakeAccessible("button", f"Other {i}") for i in range(10)]diff --git a/native/computer-use-windows/runtime-pointer.test.ps1 b/native/computer-use-windows/runtime-pointer.test.ps1new file mode 100644index 00000000000..34d5fa81176--- /dev/null+++ b/native/computer-use-windows/runtime-pointer.test.ps1@@ -0,0 +1,117 @@+param([string]$RuntimePath = (Join-Path $PSScriptRoot "runtime.ps1"))++$ErrorActionPreference = "Stop"++function Assert-TestEqual($Actual, $Expected, [string]$Label) {+ if ($Actual -ne $Expected) { throw "$Label expected '$Expected', received '$Actual'" }+}++Add-Type -TypeDefinition @"+using System;+using System.Collections.Generic;++public static class OrcaDesktopWin32 {+ public static readonly List<string> Calls = new List<string>();+ public static bool AllowActivation;+ public static IntPtr Foreground;+ public static IntPtr Pending;+ public static int FocusReads;+ public static int WindowX;++ public static void Reset(bool allow) {+ Calls.Clear();+ AllowActivation = allow;+ Foreground = new IntPtr(202);+ Pending = IntPtr.Zero;+ FocusReads = 0;+ WindowX = 100;+ }+ public static bool ShowWindow(IntPtr hwnd, int command) {+ Calls.Add("restore");+ return true;+ }+ public static bool SetForegroundWindow(IntPtr hwnd) {+ Calls.Add("activate");+ Pending = hwnd;+ return AllowActivation;+ }+ public static IntPtr GetForegroundWindow() {+ FocusReads++;+ if (AllowActivation && FocusReads >= 2) {+ Foreground = Pending;+ WindowX = 500;+ }+ return Foreground;+ }+ public static bool SetCursorPos(int x, int y) {+ Calls.Add("move:" + x + "," + y);+ return true;+ }+ public static void mouse_event(uint flags, uint dx, uint dy, int data, UIntPtr extra) {+ Calls.Add("mouse");+ }+ public static void SendModifiedClick(byte[] modifiers, uint down, uint up) {+ Calls.Add("modified");+ }+}+"@++$tokens = $null+$errors = $null+$ast = [System.Management.Automation.Language.Parser]::ParseFile($RuntimePath, [ref]$tokens, [ref]$errors)+if ($errors.Count) { throw ($errors | Out-String) }+$functions = @(+ "Test-OrcaWindowFocused", "Wait-OrcaWindowFocused", "Assert-OrcaKeyboardFocus",+ "Assert-OrcaPointerFocus", "Restore-OrcaWindow", "Restore-OrcaPointerWindow", "Assert-OrcaWindowTarget",+ "Get-OrcaRequiredNumber", "Get-OrcaPositiveInteger", "Get-OrcaPositiveNumber",+ "Get-OrcaScreenPoint", "Get-OrcaElementScreenPoint", "Get-OrcaModifierVirtualKey",+ "Get-OrcaClickModifierVirtualKeys", "Send-OrcaMouseClick", "Send-OrcaDrag", "Invoke-OrcaOperation"+)+foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {+ if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }+}++$MouseEvents = @{ LeftDown = 2; LeftUp = 4; RightDown = 8; RightUp = 16; MiddleDown = 32; MiddleUp = 64; Wheel = 2048; HorizontalWheel = 4096 }+function Find-OrcaProcess { [pscustomobject]@{ MainWindowHandle = 101 } }+function Get-OrcaRootElement { $null }+function Get-OrcaWindowFrame { @{ x = [OrcaDesktopWin32]::WindowX; y = 200 } }+function Find-OrcaElement($RootElement, $Record) { $Record }+function Invoke-OrcaPrimaryAction($Element) { $null -ne $Element }+function New-OrcaSnapshot { [pscustomobject]@{} }++foreach ($case in @(+ @{ tool = "click"; x = 5; y = 6 },+ @{ tool = "click"; x = 5; y = 6; modifiers = "ctrl" },+ @{ tool = "scroll"; x = 5; y = 6; direction = "down"; pages = 1 },+ @{ tool = "drag"; from_x = 5; from_y = 6; to_x = 25; to_y = 26 }+)) {+ foreach ($activates in @($false, $true)) {+ [OrcaDesktopWin32]::Reset($activates)+ $operation = [pscustomobject]@{ app = "Fixture"; noScreenshot = $true; restoreWindow = $false }+ foreach ($key in $case.Keys) { $operation | Add-Member -NotePropertyName $key -NotePropertyValue $case[$key] }+ $denied = $false+ try {+ $result = Invoke-OrcaOperation $operation+ } catch {+ if ($_.Exception.Message -notlike "window_not_focused:*") { throw }+ $denied = $true+ }+ $inputCalls = @([OrcaDesktopWin32]::Calls | Where-Object { $_ -like "move:*" -or $_ -in @("mouse", "modified") })+ if ($activates) {+ Assert-TestEqual $denied $false "$($case.tool) successful activation"+ Assert-TestEqual $result.ok $true "$($case.tool) result"+ if ($inputCalls.Count -eq 0) { throw "$($case.tool) did not send pointer input after activation" }+ if ([OrcaDesktopWin32]::FocusReads -lt 2) { throw "$($case.tool) did not wait for activation" }+ Assert-TestEqual $inputCalls[0] "move:505,206" "$($case.tool) geometry after delayed activation"+ } else {+ Assert-TestEqual $denied $true "$($case.tool) denied activation"+ Assert-TestEqual $inputCalls.Count 0 "$($case.tool) denied activation sends no pointer or modifier input"+ }+ }+}++[OrcaDesktopWin32]::Reset($false)+$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })+Assert-TestEqual $result.action.path "accessibility" "pattern click path"+Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern click bypasses activation and global input"+Write-Output "windows-pointer-tests-ok"diff --git a/native/computer-use-windows/runtime-render.test.ps1 b/native/computer-use-windows/runtime-render.test.ps1index be8a9f6a666..fe47f7c89b2 100644--- a/native/computer-use-windows/runtime-render.test.ps1+++ b/native/computer-use-windows/runtime-render.test.ps1@@ -240,6 +240,9 @@ try {if ($keys -cne $case[1]) { throw "hotkey $($case[0]) expected '$($case[1])', received '$keys'" }}+ & (Join-Path $PSHOME "powershell.exe") -NoLogo -NoProfile -File (Join-Path $PSScriptRoot "runtime-pointer.test.ps1")+ if ($LASTEXITCODE -ne 0) { throw "Windows pointer tests failed" }+Write-Output "windows-snapshot-render-tests-ok"} finally {Remove-Item -LiteralPath $operationPath -Force -ErrorAction SilentlyContinuediff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1index ac97bd167e8..0582d14c9d0 100644--- a/native/computer-use-windows/runtime.ps1+++ b/native/computer-use-windows/runtime.ps1@@ -355,6 +355,17 @@ function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation) {throw "window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window"}+function Assert-OrcaPointerFocus([IntPtr]$WindowHandle) {+ if (Wait-OrcaWindowFocused $WindowHandle 500) { return }+ throw "window_not_focused: window activation did not focus the target window; no pointer input was sent"+}++function Restore-OrcaPointerWindow($Process, $RootElement) {+ Restore-OrcaWindow $Process+ Assert-OrcaPointerFocus ([IntPtr]$Process.MainWindowHandle)+ Get-OrcaWindowFrame $Process $RootElement+}+function Get-OrcaElementFrame($Element, $WindowFrame) {try {$bounds = $Element.Current.BoundingRectangle@@ -1004,8 +1015,7 @@ function Get-OrcaElementScreenPoint($Element) {$null}-function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count, [string]$Modifiers) {- [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)+function Send-OrcaMouseClick([int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count, [string]$Modifiers) {[void][OrcaDesktopWin32]::SetCursorPos($ScreenX, $ScreenY)$buttonName = if ([string]::IsNullOrWhiteSpace($Button)) { "left" } else { $Button.ToLowerInvariant() }switch ($buttonName) {@@ -1035,8 +1045,7 @@ function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY}}-function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) {- [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)+function Send-OrcaDrag($From, $To) {$startX = [int]$From.x$startY = [int]$From.y$endX = [int]$To.x@@ -1223,9 +1232,6 @@ function Invoke-OrcaOperation($Operation) {switch ($Operation.tool) {"click" {- # Why: agents expect a click into a target app to make the next- # keyboard action safe, even when UI Automation handles the click.- Restore-OrcaWindow $process$handledByPattern = $false$clickCount = Get-OrcaPositiveInteger $Operation.click_count "click_count"$hasModifiers = -not [string]::IsNullOrWhiteSpace([string]$Operation.modifiers)@@ -1233,9 +1239,11 @@ function Invoke-OrcaOperation($Operation) {$handledByPattern = Invoke-OrcaPrimaryAction $element}if (-not $handledByPattern) {+ # Why: mouse input lands on whichever window is on top at the point; UIA patterns need no focus.+ $windowFrame = Restore-OrcaPointerWindow $process $root$point = Get-OrcaElementScreenPoint $elementif ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }- Send-OrcaMouseClick $handle $point.x $point.y $Operation.mouse_button $clickCount $Operation.modifiers+ Send-OrcaMouseClick $point.x $point.y $Operation.mouse_button $clickCount $Operation.modifiers$action = [pscustomobject]@{ path = "synthetic"; actionName = $null; fallbackReason = "actionUnsupported" }} else {$action = [pscustomobject]@{ path = "accessibility"; actionName = "primaryAction"; fallbackReason = $null }@@ -1261,14 +1269,15 @@ function Invoke-OrcaOperation($Operation) {} elseif ($Operation.direction -ne "up") {throw "unsupported scroll direction: $($Operation.direction)"}+ $windowFrame = Restore-OrcaPointerWindow $process $root$point = Get-OrcaElementScreenPoint $elementif ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }- [void][OrcaDesktopWin32]::SetForegroundWindow($handle)[void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y)[OrcaDesktopWin32]::mouse_event($mouseEvent, 0, 0, $delta, [UIntPtr]::Zero)$action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null }}"drag" {+ $windowFrame = Restore-OrcaPointerWindow $process $root$from = Get-OrcaElementScreenPoint $fromElementif ($null -eq $from -and $null -ne $Operation.fromElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }if ($null -eq $from) {@@ -1285,7 +1294,7 @@ function Invoke-OrcaOperation($Operation) {y = $windowFrame.y + (Get-OrcaRequiredNumber $Operation.to_y "to_y")}}- Send-OrcaDrag $handle $from $to+ Send-OrcaDrag $from $to$action = [pscustomobject]@{ path = "synthetic"; actionName = "drag"; fallbackReason = $null }}"type_text" {diff --git a/tests/e2e/computer-linux.e2e.ts b/tests/e2e/computer-linux.e2e.tsindex 418b8377533..0342535a901 100644--- a/tests/e2e/computer-linux.e2e.ts+++ b/tests/e2e/computer-linux.e2e.ts@@ -12,6 +12,39 @@ import {const isLinux = process.platform === 'linux'const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'+async function focusGeditDocument(): Promise<void> {+ const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(+ (+ await runOrcaCli([+ 'computer',+ 'get-app-state',+ '--app',+ 'gedit',+ '--restore-window',+ '--no-screenshot',+ '--json'+ ])+ ).stdout+ )+ const textIndex = findRoleIndex(+ before.result.snapshot.treeText,+ /^\s*(\d+)\s+(text|text entry|entry|document|edit area)(?:\s|$)/im+ )+ expect(textIndex).toBeGreaterThanOrEqual(0)++ await runOrcaCli([+ 'computer',+ 'click',+ '--app',+ 'gedit',+ '--element-index',+ String(textIndex),+ '--restore-window',+ '--no-screenshot',+ '--json'+ ])+}+describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {beforeAll(async () => {await ensureOrcaRuntimeLaunched()@@ -33,39 +66,10 @@ describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {expect(envelope.result.screenshot?.path).toContain('orca-computer-use')})- test('click and type-text send synthetic input to the document', async () => {- const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(- (- await runOrcaCli([- 'computer',- 'get-app-state',- '--app',- 'gedit',- '--restore-window',- '--no-screenshot',- '--json'- ])- ).stdout- )- const textIndex = findRoleIndex(- before.result.snapshot.treeText,- /^\s*(\d+)\s+(text|text entry|entry|document|edit area)(?:\s|$)/im- )- expect(textIndex).toBeGreaterThanOrEqual(0)+ test('click and type-text insert into the focused document through AT-SPI', async () => {+ await focusGeditDocument()- await runOrcaCli([- 'computer',- 'click',- '--app',- 'gedit',- '--element-index',- String(textIndex),- '--restore-window',- '--no-screenshot',- '--json'- ])-- const marker = ` orca-linux-type-${Date.now()}`+ const marker = ` orca-linux-type-${Date.now()}-élan-🚀`const typed = parseJsonOutput<{ result: ComputerActionResult }>((await runOrcaCli([@@ -81,12 +85,14 @@ describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {])).stdout)- expect(typed.result.action?.path).toBe('synthetic')+ expect(typed.result.action?.path).toBe('accessibility')expect(typed.result.snapshot.treeText).toContain(marker.trim())})test('paste-text mutates the test-owned document', async () => {- const marker = `orca-linux-paste-${Date.now()}`+ await focusGeditDocument()++ const marker = `orca-linux-paste-${Date.now()}-café-🚀`const action = parseJsonOutput<{ result: ComputerActionResult }>((await runOrcaCli([@@ -102,10 +108,10 @@ describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {])).stdout)- expect(action.result.action?.path).toBe('clipboard')+ expect(action.result.action?.path).toBe('accessibility')expect(action.result.action?.verification).toMatchObject({- state: 'unverified',- reason: 'clipboard_paste'+ state: 'verified',+ property: 'focusedText'})const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(