Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

fix(computer): skip activation for accessibility clicks (#36006)

Kelvin Amoaba3b83d3d2608c
diff --git a/config/scripts/verify-computer-native.mjs b/config/scripts/verify-computer-native.mjs
index 896558aa919..dcd4983de38 100644
--- a/config/scripts/verify-computer-native.mjs
+++ b/config/scripts/verify-computer-native.mjs
@@ -231,7 +231,7 @@ function verifyNativeArgumentGuardrails() {
}
if (
!linux.includes('def require_non_empty_string(value, name):') ||
- !linux.includes('type_text(require_non_empty_string(operation.get("text"), "text"))') ||
+ !linux.includes('text = require_non_empty_string(operation.get("text"), "text")') ||
!linux.includes('press_key(require_non_empty_string(operation.get("key"), "key"))')
) {
failures.push(
@@ -302,11 +302,6 @@ function verifyNativeArgumentGuardrails() {
if (/restore_window\(app\)(?!:)/.test(linux)) {
failures.push('Linux restore call sites must pass the selected window target')
}
- if (!linux.includes('even when the click uses an accessibility path')) {
- failures.push(
- 'Linux clicks must activate the target window before accessibility or pointer input'
- )
- }
if (
!linux.includes('compact_browser_tabs=is_browser_app(query, app)') ||
!linux.includes('inactive browser tabs omitted')
@@ -405,11 +400,6 @@ function verifyNativeArgumentGuardrails() {
'Windows text/key actions must reject missing payloads instead of sending empty input'
)
}
- if (!windows.includes('even when UI Automation handles the click')) {
- failures.push(
- 'Windows clicks must activate the target window before UI Automation or pointer input'
- )
- }
if (
!windows.includes('Render-OrcaTree $root $windowFrame (Test-OrcaBrowserProcess $process)') ||
!windows.includes('inactive browser tabs omitted')
diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py
index ffac9b92cf5..d20ab09efcb 100644
--- a/native/computer-use-linux/runtime.py
+++ b/native/computer-use-linux/runtime.py
@@ -197,21 +197,32 @@ def restore_window(app, window=None):
)
+def wait_window_focused(window):
+ deadline = time.monotonic() + 0.5
+ while time.monotonic() < deadline:
+ if has_state(window, Atspi.StateType.ACTIVE):
+ return True
+ time.sleep(0.05)
+ return has_state(window, Atspi.StateType.ACTIVE)
+
+
def require_keyboard_focus(window, operation):
if has_state(window, Atspi.StateType.ACTIVE):
return
if operation.get("restoreWindow"):
- deadline = time.monotonic() + 0.5
- while time.monotonic() < deadline:
- if has_state(window, Atspi.StateType.ACTIVE):
- return
- time.sleep(0.05)
- if has_state(window, Atspi.StateType.ACTIVE):
+ if wait_window_focused(window):
return
raise RuntimeError("window_not_focused: keyboard input requires the target window to be focused; restoreWindow was requested but the target window is still not focused; bring it forward manually or check desktop permissions")
raise RuntimeError("window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window")
+def restore_pointer_window(app, window):
+ restore_window(app, window)
+ if not wait_window_focused(window):
+ raise RuntimeError("window_not_focused: window restoration did not focus the target window; no pointer input was sent")
+ return screen_rect(window)
+
+
def app_matches(app, query):
needle = str(query or "").strip().lower()
if not needle:
@@ -987,6 +998,56 @@ def type_text(value):
Atspi.generate_keyboard_event(0, str(value), Atspi.KeySynthType.STRING)
+def insert_focused_text(window, value):
+ # Why: EditableText needs no window focus and still works on Wayland, where key synthesis is a no-op.
+ node = first_descendant(window, lambda candidate: has_state(candidate, Atspi.StateType.FOCUSED))
+ if node is None or not has_state(node, Atspi.StateType.EDITABLE):
+ return None
+ editable = attempt(node.get_editable_text_iface)
+ text = text_iface(node)
+ if editable is None or text is None:
+ return None
+ start = end = int(attempt(lambda: Atspi.Text.get_caret_offset(text), -1))
+ selection_count = attempt(lambda: Atspi.Text.get_n_selections(text))
+ if selection_count not in (0, 1):
+ return None
+ if selection_count == 1:
+ selection = attempt(lambda: Atspi.Text.get_selection(text, 0))
+ if selection is None or not 0 <= selection.start_offset < selection.end_offset:
+ return None
+ start, end = selection.start_offset, selection.end_offset
+ if start < 0:
+ return None
+ original_count = attempt(lambda: Atspi.Text.get_character_count(text))
+ if original_count is None or original_count < 0 or end > original_count:
+ return None
+ original = None
+ if end > start:
+ original = attempt(lambda: Atspi.Text.get_text(text, 0, original_count))
+ if original is None or len(original) != original_count:
+ return None
+ inserted_count = original_count + len(value)
+ if (not attempt(lambda: Atspi.EditableText.insert_text(editable, end, value, len(value.encode("utf-8"))), False)
+ or attempt(lambda: Atspi.Text.get_character_count(text)) != inserted_count
+ or attempt(lambda: Atspi.Text.get_text(text, end, end + len(value))) != value):
+ raise RuntimeError("text_insertion_unconfirmed: text insertion did not confirm success; inspect state before retrying")
+ # Why insert before deleting: an unsuccessful insert leaves the selected text intact.
+ if end > start and (not attempt(lambda: Atspi.EditableText.delete_text(editable, start, end), False)
+ or attempt(lambda: Atspi.Text.get_character_count(text)) != inserted_count - (end - start)
+ or attempt(lambda: Atspi.Text.get_text(text, 0, inserted_count - (end - start)))
+ != original[:start] + value + original[end:]):
+ raise RuntimeError("text_replacement_incomplete: text was inserted but selection deletion did not confirm success; inspect state before retrying")
+ caret = start + len(value)
+ if attempt(lambda: Atspi.Text.get_caret_offset(text)) != caret:
+ if (not attempt(lambda: Atspi.Text.set_caret_offset(text, caret), False)
+ or attempt(lambda: Atspi.Text.get_caret_offset(text)) != caret):
+ raise RuntimeError("text_caret_unconfirmed: text was inserted but caret movement did not confirm success; inspect state before retrying")
+ action = {"path": "accessibility", "actionName": "insertText", "fallbackReason": None}
+ if attempt(lambda: Atspi.Text.get_text(text, start, caret)) == value:
+ action["verification"] = {"state": "verified", "property": "focusedText", "expected": value}
+ return action
+
+
def paste_text(value):
text = str(value)
previous = read_clipboard()
@@ -1089,7 +1150,7 @@ def run_operation(operation):
if operation.get("restoreWindow"):
restore_window(app, window)
_, window = choose_window(app, operation.get("windowId"), operation.get("windowIndex"))
- if tool in {"type_text", "press_key", "hotkey", "paste_text"}:
+ if tool in {"press_key", "hotkey"}:
require_keyboard_focus(window, operation)
bounds = screen_rect(window)
saved = operation.get("element")
@@ -1099,9 +1160,6 @@ def run_operation(operation):
action = None
if tool == "click":
- # Why: agents expect a click into a target app to make the next
- # keyboard action safe, even when the click uses an accessibility path.
- restore_window(app, window)
preferred = preferred_action(node)
click_count = (
require_positive_integer(operation.get("click_count"), "click_count")
@@ -1111,6 +1169,8 @@ def run_operation(operation):
has_modifiers = bool(str(operation.get("modifiers") or "").strip())
handled = not has_modifiers and operation.get("mouse_button", "left") == "left" and click_count <= 1 and perform_action(node, preferred)
if not handled:
+ # Why: pointer clicks are synthetic global input; AT-SPI actions need no focus.
+ bounds = restore_pointer_window(app, window)
click_at(
*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node),
operation.get("mouse_button", "left"),
@@ -1132,21 +1192,25 @@ def run_operation(operation):
elif tool == "scroll":
# Why: pointer wheel events should land in the requested app window even
# when another desktop window is currently foregrounded.
- restore_window(app, window)
+ bounds = restore_pointer_window(app, window)
scroll_at(*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node), operation.get("direction"), operation.get("pages"))
action = {"path": "synthetic", "actionName": "scroll", "fallbackReason": None}
elif tool == "drag":
# Why: pointer drags are synthetic global input, so activate the target
# window before using cached coordinates from its accessibility tree.
- restore_window(app, window)
+ bounds = restore_pointer_window(app, window)
drag_between(
screen_point(bounds, operation.get("fromElement"), operation.get("from_x"), operation.get("from_y"), from_node),
screen_point(bounds, operation.get("toElement"), operation.get("to_x"), operation.get("to_y"), to_node),
)
action = {"path": "synthetic", "actionName": "drag", "fallbackReason": None}
elif tool == "type_text":
- type_text(require_non_empty_string(operation.get("text"), "text"))
- action = {"path": "synthetic", "actionName": "typeText", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}
+ text = require_non_empty_string(operation.get("text"), "text")
+ action = insert_focused_text(window, text)
+ if action is None:
+ require_keyboard_focus(window, operation)
+ type_text(text)
+ action = {"path": "synthetic", "actionName": "typeText", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}
elif tool == "press_key":
press_key(require_non_empty_string(operation.get("key"), "key"))
action = {"path": "synthetic", "actionName": "pressKey", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}
@@ -1154,8 +1218,12 @@ def run_operation(operation):
hotkey(require_non_empty_string(operation.get("key"), "key"))
action = {"path": "synthetic", "actionName": "hotkey", "fallbackReason": None, "verification": {"state": "unverified", "reason": "synthetic_input"}}
elif tool == "paste_text":
- paste_text(require_non_empty_string(operation.get("text"), "text"))
- action = {"path": "clipboard", "actionName": "paste", "fallbackReason": None, "verification": {"state": "unverified", "reason": "clipboard_paste"}}
+ text = require_non_empty_string(operation.get("text"), "text")
+ action = insert_focused_text(window, text)
+ if action is None:
+ require_keyboard_focus(window, operation)
+ paste_text(text)
+ action = {"path": "clipboard", "actionName": "paste", "fallbackReason": None, "verification": {"state": "unverified", "reason": "clipboard_paste"}}
elif tool == "set_value":
if not set_value(node, operation.get("value", "")):
raise RuntimeError("element value is not settable")
diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py
index 4574d2e7d12..3d0503d7e77 100644
--- a/native/computer-use-linux/runtime_render_test.py
+++ b/native/computer-use-linux/runtime_render_test.py
@@ -14,6 +14,15 @@ class FakeStateType:
ACTIVE = 3
SHOWING = 4
FOCUSED = 5
+ EDITABLE = 6
+
+
+class FakeStateSet:
+ def __init__(self, *states):
+ self.states = set(states)
+
+ def contains(self, state):
+ return state in self.states
class FakeText:
@@ -25,12 +34,43 @@ class FakeText:
def get_text(node, start, end):
return node.value[start:end]
+ @staticmethod
+ def get_caret_offset(node):
+ return node.caret
+
+ @staticmethod
+ def set_caret_offset(node, offset):
+ node.caret = offset
+ return True
+
+ @staticmethod
+ def get_n_selections(node):
+ return 1 if node.selection else 0
+
+ @staticmethod
+ def get_selection(node, _):
+ return types.SimpleNamespace(start_offset=node.selection[0], end_offset=node.selection[1])
+
+
+class FakeEditableText:
+ @staticmethod
+ def insert_text(node, position, text, length):
+ inserted = text.encode("utf-8")[:length].decode("utf-8")
+ node.value = node.value[:position] + inserted + node.value[position:]
+ return True
+
+ @staticmethod
+ def delete_text(node, start, end):
+ node.value = node.value[:start] + node.value[end:]
+ node.selection = None
+ return True
+
def load_runtime():
gi = types.ModuleType("gi")
repository = types.ModuleType("gi.repository")
gi.require_version = lambda *_: None
- repository.Atspi = types.SimpleNamespace(StateType=FakeStateType, Text=FakeText)
+ repository.Atspi = types.SimpleNamespace(StateType=FakeStateType, Text=FakeText, EditableText=FakeEditableText)
repository.Gdk = types.SimpleNamespace()
repository.GdkPixbuf = types.SimpleNamespace()
gi.repository = repository
@@ -242,6 +282,215 @@ class RuntimeRenderTest(unittest.TestCase):
runtime.run_operation({"tool": "click", "app": "Fixture"})
self.assertEqual(effects, ["clicked"])
+ def test_click_activates_the_window_only_for_pointer_fallback(self):
+ button = FakeAccessible("button", "Continue")
+ button.get_n_actions = lambda: 1
+ button.get_action_name = lambda _: "click"
+ button.do_action = lambda _: True
+ label = FakeAccessible("label", "Status")
+ for target, expected_path, expected_calls in (
+ (button, "accessibility", []),
+ (label, "synthetic", ["restore", "pointer"]),
+ ):
+ with self.subTest(path=expected_path), SnapshotFixture(
+ FakeAccessible("frame", "Editor", children=[button, label])
+ ) as fixture:
+ calls = []
+ def restore():
+ calls.append("restore")
+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)
+ fixture.restore = restore
+ snapshot = runtime.make_snapshot("Fixture", False)
+ element = next(item for item in snapshot["elements"] if item["name"] == target.name)
+ with patch.object(runtime, "click_at", lambda *_: calls.append("pointer")):
+ result = runtime.run_operation({"tool": "click", "app": "Fixture", "element": element, "noScreenshot": True})
+ self.assertEqual(result["action"]["path"], expected_path)
+ self.assertEqual(calls, expected_calls)
+
+ def test_pointer_input_waits_for_activation_and_uses_restored_geometry(self):
+ operations = (
+ {"tool": "click", "x": 5, "y": 6},
+ {"tool": "scroll", "x": 5, "y": 6, "direction": "down", "pages": 1},
+ {"tool": "drag", "from_x": 5, "from_y": 6, "to_x": 25, "to_y": 26},
+ )
+ for operation in operations:
+ for activates in (False, True):
+ with self.subTest(tool=operation["tool"], activates=activates), SnapshotFixture(
+ FakeAccessible("frame", "Editor")
+ ) as fixture:
+ restored = []
+ pointer_calls = []
+ fixture.root.get_state_set = lambda: FakeStateSet(
+ *([FakeStateType.ACTIVE] if activates and restored and fixture.clock >= 0.1 else [])
+ )
+ def restore():
+ restored.append(True)
+ fixture.bounds[fixture.root] = runtime.Rect(500, 200, 300, 400)
+ fixture.restore = restore
+ def sleep(seconds):
+ fixture.clock += seconds
+ with patch.object(runtime.time, "sleep", sleep), \
+ patch.object(runtime, "click_at", lambda *args: pointer_calls.append(args[:2])), \
+ patch.object(runtime, "scroll_at", lambda *args: pointer_calls.append(args[:2])), \
+ patch.object(runtime, "drag_between", lambda start, end: pointer_calls.extend((start, end))):
+ request = {**operation, "app": "Fixture", "noScreenshot": True, "restoreWindow": False}
+ if activates:
+ result = runtime.run_operation(request)
+ self.assertTrue(result["ok"])
+ self.assertEqual(pointer_calls, [(505, 206), (525, 226)] if operation["tool"] == "drag" else [(505, 206)])
+ self.assertGreaterEqual(fixture.clock, 0.1)
+ else:
+ with self.assertRaisesRegex(RuntimeError, "window_not_focused"):
+ runtime.run_operation(request)
+ self.assertEqual(pointer_calls, [])
+ self.assertGreaterEqual(fixture.clock, 0.5)
+ self.assertTrue(restored)
+
+ def test_text_entry_inserts_into_the_focused_field_of_a_background_window(self):
+ for tool in ("type_text", "paste_text"):
+ for value in ("there", "élan", "🚀 café"):
+ for selection in ((6, 11), None):
+ field = FakeAccessible("text", "Message", value="Hello world")
+ field.caret, field.selection = 11, selection
+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)
+ field.get_editable_text_iface = lambda: field
+ with self.subTest(tool=tool, value=value, selection=selection), SnapshotFixture(FakeAccessible("frame", "Editor", children=[field])) as fixture:
+ calls = []
+ fixture.restore = lambda: calls.append("restore")
+ with patch.object(runtime, "type_text", lambda _: calls.append("keys")), \
+ patch.object(runtime, "paste_text", lambda _: calls.append("clipboard")):
+ result = runtime.run_operation({"tool": tool, "app": "Fixture", "text": value, "noScreenshot": True})
+ self.assertEqual(calls, [])
+ prefix = "Hello " if selection else "Hello world"
+ self.assertEqual((field.value, field.caret), (prefix + value, len(prefix) + len(value)))
+ self.assertEqual(result["action"]["path"], "accessibility")
+ self.assertEqual(result["action"]["verification"]["state"], "verified")
+
+ def test_text_entry_without_an_editable_focused_field_still_requires_window_focus(self):
+ cases = [
+ ("not-editable", 1, (6, 11)),
+ ("count-error", TimeoutError("selection timed out"), (6, 11)),
+ ("count-missing", None, (6, 11)),
+ ("count-negative", -1, (6, 11)),
+ ("multiple", 2, (6, 11)),
+ ("range-error", 1, TimeoutError("range timed out")),
+ ("range-missing", 1, None),
+ ("range-empty", 1, (6, 6)),
+ ("range-reversed", 1, (11, 6)),
+ ("range-outside", 1, (6, 12)),
+ ]
+ for tool in ("type_text", "paste_text"):
+ for active in (False, True):
+ for name, count, selection in cases:
+ field = FakeAccessible("text", "Message", value="Hello world")
+ field.caret, field.selection = 11, (6, 11)
+ states = [FakeStateType.FOCUSED] + ([] if name == "not-editable" else [FakeStateType.EDITABLE])
+ field.get_state_set = lambda: FakeStateSet(*states)
+ field.get_editable_text_iface = lambda: field
+ selected = types.SimpleNamespace(start_offset=selection[0], end_offset=selection[1]) if isinstance(selection, tuple) else selection
+ with self.subTest(tool=tool, active=active, case=name), SnapshotFixture(
+ FakeAccessible("frame", "Editor", children=[field])
+ ) as fixture, patch.object(runtime.Atspi.Text, "get_n_selections", return_value=count, side_effect=count if isinstance(count, Exception) else None), \
+ patch.object(runtime.Atspi.Text, "get_selection", return_value=selected, side_effect=selected if isinstance(selected, Exception) else None), \
+ patch.object(runtime.Atspi.EditableText, "insert_text", wraps=FakeEditableText.insert_text) as insert, \
+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:
+ if active:
+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)
+ result = runtime.run_operation({"tool": tool, "app": "Fixture", "text": "x", "noScreenshot": True})
+ (keys if tool == "type_text" else paste).assert_called_once_with("x")
+ self.assertEqual(result["action"]["verification"]["state"], "unverified")
+ else:
+ with self.assertRaisesRegex(RuntimeError, "window_not_focused"):
+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "x", "noScreenshot": True})
+ if not active or tool != "type_text":
+ keys.assert_not_called()
+ if not active or tool != "paste_text":
+ paste.assert_not_called()
+ insert.assert_not_called()
+ self.assertEqual((field.value, field.selection), ("Hello world", (6, 11)))
+
+ def test_partial_text_replacement_fails_without_typing_again(self):
+ for tool in ("type_text", "paste_text"):
+ for failure in (False, True, TimeoutError("provider timed out"), "wrong-range"):
+ field = FakeAccessible("text", "Message", value="Hello world")
+ field.caret, field.selection = 11, (6, 11)
+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)
+ field.get_editable_text_iface = lambda: field
+ def delete(node, _start, _end):
+ if failure == "wrong-range":
+ node.value = node.value[5:]
+ node.selection = None
+ return True
+ if isinstance(failure, Exception):
+ raise failure
+ return failure
+ with self.subTest(tool=tool, failure=failure), SnapshotFixture(
+ FakeAccessible("frame", "Editor", children=[field])
+ ) as fixture, patch.object(runtime.Atspi.EditableText, "delete_text", delete), \
+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:
+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)
+ with self.assertRaisesRegex(RuntimeError, "text_replacement_incomplete"):
+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "there", "noScreenshot": True})
+ self.assertEqual(field.value, " worldthere" if failure == "wrong-range" else "Hello worldthere")
+ keys.assert_not_called()
+ paste.assert_not_called()
+
+ def test_unconfirmed_text_insertion_does_not_fall_back_to_another_write(self):
+ write = FakeEditableText.insert_text
+ failures = {
+ "false-after-write": "foobarbar",
+ "timeout-after-write": "foobarbar",
+ "no-op": "foobar",
+ "truncated": "foobbar",
+ "wrong-text": "foobazbar",
+ "count-unavailable": "foobarbar",
+ }
+ for tool in ("type_text", "paste_text"):
+ for failure, expected in failures.items():
+ field = FakeAccessible("text", "Message", value="foobar")
+ field.caret, field.selection = 6, (0, 3)
+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)
+ field.get_editable_text_iface = lambda: field
+ def insert(node, position, text, length):
+ if failure == "no-op":
+ return True
+ write(node, position, "baz" if failure == "wrong-text" else text, 1 if failure == "truncated" else length)
+ if failure == "timeout-after-write":
+ raise TimeoutError("reply lost after insertion")
+ return failure != "false-after-write"
+ def count(node):
+ return None if failure == "count-unavailable" and node.value != "foobar" else len(node.value)
+ with self.subTest(tool=tool, failure=failure), SnapshotFixture(
+ FakeAccessible("frame", "Editor", children=[field])
+ ) as fixture, patch.object(runtime.Atspi.EditableText, "insert_text", side_effect=insert), \
+ patch.object(runtime.Atspi.Text, "get_character_count", side_effect=count), \
+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:
+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)
+ with self.assertRaisesRegex(RuntimeError, "text_insertion_unconfirmed"):
+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "bar", "noScreenshot": True})
+ self.assertEqual(field.value, expected)
+ self.assertEqual(field.selection, (0, 3))
+ keys.assert_not_called()
+ paste.assert_not_called()
+
+ def test_unconfirmed_caret_movement_does_not_report_text_entry_success(self):
+ for tool in ("type_text", "paste_text"):
+ for failure in (False, True, TimeoutError("provider timed out")):
+ field = FakeAccessible("text", "Message", value="Hello world")
+ field.caret, field.selection = 6, (6, 11)
+ field.get_state_set = lambda: FakeStateSet(FakeStateType.FOCUSED, FakeStateType.EDITABLE)
+ field.get_editable_text_iface = lambda: field
+ with self.subTest(tool=tool, failure=failure), SnapshotFixture(
+ FakeAccessible("frame", "Editor", children=[field])
+ ) as fixture, patch.object(runtime.Atspi.Text, "set_caret_offset", side_effect=failure if isinstance(failure, Exception) else None, return_value=failure), \
+ patch.object(runtime, "type_text") as keys, patch.object(runtime, "paste_text") as paste:
+ fixture.root.get_state_set = lambda: FakeStateSet(FakeStateType.ACTIVE)
+ with self.assertRaisesRegex(RuntimeError, "text_caret_unconfirmed"):
+ runtime.run_operation({"tool": tool, "app": "Fixture", "text": "there", "noScreenshot": True})
+ self.assertEqual((field.value, field.caret), ("Hello there", 6))
+ keys.assert_not_called()
+ paste.assert_not_called()
+
def test_compacted_browser_records_keep_their_own_frames_and_indexes(self):
tabs = [FakeAccessible("page tab", f"Tab {i}", value="1" if i == 9 else "") for i in range(10)]
buttons = [FakeAccessible("button", f"Other {i}") for i in range(10)]
diff --git a/native/computer-use-windows/runtime-pointer.test.ps1 b/native/computer-use-windows/runtime-pointer.test.ps1
new file mode 100644
index 00000000000..34d5fa81176
--- /dev/null
+++ b/native/computer-use-windows/runtime-pointer.test.ps1
@@ -0,0 +1,117 @@
+param([string]$RuntimePath = (Join-Path $PSScriptRoot "runtime.ps1"))
+
+$ErrorActionPreference = "Stop"
+
+function Assert-TestEqual($Actual, $Expected, [string]$Label) {
+ if ($Actual -ne $Expected) { throw "$Label expected '$Expected', received '$Actual'" }
+}
+
+Add-Type -TypeDefinition @"
+using System;
+using System.Collections.Generic;
+
+public static class OrcaDesktopWin32 {
+ public static readonly List<string> Calls = new List<string>();
+ public static bool AllowActivation;
+ public static IntPtr Foreground;
+ public static IntPtr Pending;
+ public static int FocusReads;
+ public static int WindowX;
+
+ public static void Reset(bool allow) {
+ Calls.Clear();
+ AllowActivation = allow;
+ Foreground = new IntPtr(202);
+ Pending = IntPtr.Zero;
+ FocusReads = 0;
+ WindowX = 100;
+ }
+ public static bool ShowWindow(IntPtr hwnd, int command) {
+ Calls.Add("restore");
+ return true;
+ }
+ public static bool SetForegroundWindow(IntPtr hwnd) {
+ Calls.Add("activate");
+ Pending = hwnd;
+ return AllowActivation;
+ }
+ public static IntPtr GetForegroundWindow() {
+ FocusReads++;
+ if (AllowActivation && FocusReads >= 2) {
+ Foreground = Pending;
+ WindowX = 500;
+ }
+ return Foreground;
+ }
+ public static bool SetCursorPos(int x, int y) {
+ Calls.Add("move:" + x + "," + y);
+ return true;
+ }
+ public static void mouse_event(uint flags, uint dx, uint dy, int data, UIntPtr extra) {
+ Calls.Add("mouse");
+ }
+ public static void SendModifiedClick(byte[] modifiers, uint down, uint up) {
+ Calls.Add("modified");
+ }
+}
+"@
+
+$tokens = $null
+$errors = $null
+$ast = [System.Management.Automation.Language.Parser]::ParseFile($RuntimePath, [ref]$tokens, [ref]$errors)
+if ($errors.Count) { throw ($errors | Out-String) }
+$functions = @(
+ "Test-OrcaWindowFocused", "Wait-OrcaWindowFocused", "Assert-OrcaKeyboardFocus",
+ "Assert-OrcaPointerFocus", "Restore-OrcaWindow", "Restore-OrcaPointerWindow", "Assert-OrcaWindowTarget",
+ "Get-OrcaRequiredNumber", "Get-OrcaPositiveInteger", "Get-OrcaPositiveNumber",
+ "Get-OrcaScreenPoint", "Get-OrcaElementScreenPoint", "Get-OrcaModifierVirtualKey",
+ "Get-OrcaClickModifierVirtualKeys", "Send-OrcaMouseClick", "Send-OrcaDrag", "Invoke-OrcaOperation"
+)
+foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {
+ if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }
+}
+
+$MouseEvents = @{ LeftDown = 2; LeftUp = 4; RightDown = 8; RightUp = 16; MiddleDown = 32; MiddleUp = 64; Wheel = 2048; HorizontalWheel = 4096 }
+function Find-OrcaProcess { [pscustomobject]@{ MainWindowHandle = 101 } }
+function Get-OrcaRootElement { $null }
+function Get-OrcaWindowFrame { @{ x = [OrcaDesktopWin32]::WindowX; y = 200 } }
+function Find-OrcaElement($RootElement, $Record) { $Record }
+function Invoke-OrcaPrimaryAction($Element) { $null -ne $Element }
+function New-OrcaSnapshot { [pscustomobject]@{} }
+
+foreach ($case in @(
+ @{ tool = "click"; x = 5; y = 6 },
+ @{ tool = "click"; x = 5; y = 6; modifiers = "ctrl" },
+ @{ tool = "scroll"; x = 5; y = 6; direction = "down"; pages = 1 },
+ @{ tool = "drag"; from_x = 5; from_y = 6; to_x = 25; to_y = 26 }
+)) {
+ foreach ($activates in @($false, $true)) {
+ [OrcaDesktopWin32]::Reset($activates)
+ $operation = [pscustomobject]@{ app = "Fixture"; noScreenshot = $true; restoreWindow = $false }
+ foreach ($key in $case.Keys) { $operation | Add-Member -NotePropertyName $key -NotePropertyValue $case[$key] }
+ $denied = $false
+ try {
+ $result = Invoke-OrcaOperation $operation
+ } catch {
+ if ($_.Exception.Message -notlike "window_not_focused:*") { throw }
+ $denied = $true
+ }
+ $inputCalls = @([OrcaDesktopWin32]::Calls | Where-Object { $_ -like "move:*" -or $_ -in @("mouse", "modified") })
+ if ($activates) {
+ Assert-TestEqual $denied $false "$($case.tool) successful activation"
+ Assert-TestEqual $result.ok $true "$($case.tool) result"
+ if ($inputCalls.Count -eq 0) { throw "$($case.tool) did not send pointer input after activation" }
+ if ([OrcaDesktopWin32]::FocusReads -lt 2) { throw "$($case.tool) did not wait for activation" }
+ Assert-TestEqual $inputCalls[0] "move:505,206" "$($case.tool) geometry after delayed activation"
+ } else {
+ Assert-TestEqual $denied $true "$($case.tool) denied activation"
+ Assert-TestEqual $inputCalls.Count 0 "$($case.tool) denied activation sends no pointer or modifier input"
+ }
+ }
+}
+
+[OrcaDesktopWin32]::Reset($false)
+$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })
+Assert-TestEqual $result.action.path "accessibility" "pattern click path"
+Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern click bypasses activation and global input"
+Write-Output "windows-pointer-tests-ok"
diff --git a/native/computer-use-windows/runtime-render.test.ps1 b/native/computer-use-windows/runtime-render.test.ps1
index be8a9f6a666..fe47f7c89b2 100644
--- a/native/computer-use-windows/runtime-render.test.ps1
+++ b/native/computer-use-windows/runtime-render.test.ps1
@@ -240,6 +240,9 @@ try {
if ($keys -cne $case[1]) { throw "hotkey $($case[0]) expected '$($case[1])', received '$keys'" }
}
+ & (Join-Path $PSHOME "powershell.exe") -NoLogo -NoProfile -File (Join-Path $PSScriptRoot "runtime-pointer.test.ps1")
+ if ($LASTEXITCODE -ne 0) { throw "Windows pointer tests failed" }
+
Write-Output "windows-snapshot-render-tests-ok"
} finally {
Remove-Item -LiteralPath $operationPath -Force -ErrorAction SilentlyContinue
diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1
index ac97bd167e8..0582d14c9d0 100644
--- a/native/computer-use-windows/runtime.ps1
+++ b/native/computer-use-windows/runtime.ps1
@@ -355,6 +355,17 @@ function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation) {
throw "window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window"
}
+function Assert-OrcaPointerFocus([IntPtr]$WindowHandle) {
+ if (Wait-OrcaWindowFocused $WindowHandle 500) { return }
+ throw "window_not_focused: window activation did not focus the target window; no pointer input was sent"
+}
+
+function Restore-OrcaPointerWindow($Process, $RootElement) {
+ Restore-OrcaWindow $Process
+ Assert-OrcaPointerFocus ([IntPtr]$Process.MainWindowHandle)
+ Get-OrcaWindowFrame $Process $RootElement
+}
+
function Get-OrcaElementFrame($Element, $WindowFrame) {
try {
$bounds = $Element.Current.BoundingRectangle
@@ -1004,8 +1015,7 @@ function Get-OrcaElementScreenPoint($Element) {
$null
}
-function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count, [string]$Modifiers) {
- [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
+function Send-OrcaMouseClick([int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count, [string]$Modifiers) {
[void][OrcaDesktopWin32]::SetCursorPos($ScreenX, $ScreenY)
$buttonName = if ([string]::IsNullOrWhiteSpace($Button)) { "left" } else { $Button.ToLowerInvariant() }
switch ($buttonName) {
@@ -1035,8 +1045,7 @@ function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY
}
}
-function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) {
- [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
+function Send-OrcaDrag($From, $To) {
$startX = [int]$From.x
$startY = [int]$From.y
$endX = [int]$To.x
@@ -1223,9 +1232,6 @@ function Invoke-OrcaOperation($Operation) {
switch ($Operation.tool) {
"click" {
- # Why: agents expect a click into a target app to make the next
- # keyboard action safe, even when UI Automation handles the click.
- Restore-OrcaWindow $process
$handledByPattern = $false
$clickCount = Get-OrcaPositiveInteger $Operation.click_count "click_count"
$hasModifiers = -not [string]::IsNullOrWhiteSpace([string]$Operation.modifiers)
@@ -1233,9 +1239,11 @@ function Invoke-OrcaOperation($Operation) {
$handledByPattern = Invoke-OrcaPrimaryAction $element
}
if (-not $handledByPattern) {
+ # Why: mouse input lands on whichever window is on top at the point; UIA patterns need no focus.
+ $windowFrame = Restore-OrcaPointerWindow $process $root
$point = Get-OrcaElementScreenPoint $element
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
- Send-OrcaMouseClick $handle $point.x $point.y $Operation.mouse_button $clickCount $Operation.modifiers
+ Send-OrcaMouseClick $point.x $point.y $Operation.mouse_button $clickCount $Operation.modifiers
$action = [pscustomobject]@{ path = "synthetic"; actionName = $null; fallbackReason = "actionUnsupported" }
} else {
$action = [pscustomobject]@{ path = "accessibility"; actionName = "primaryAction"; fallbackReason = $null }
@@ -1261,14 +1269,15 @@ function Invoke-OrcaOperation($Operation) {
} elseif ($Operation.direction -ne "up") {
throw "unsupported scroll direction: $($Operation.direction)"
}
+ $windowFrame = Restore-OrcaPointerWindow $process $root
$point = Get-OrcaElementScreenPoint $element
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
- [void][OrcaDesktopWin32]::SetForegroundWindow($handle)
[void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y)
[OrcaDesktopWin32]::mouse_event($mouseEvent, 0, 0, $delta, [UIntPtr]::Zero)
$action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null }
}
"drag" {
+ $windowFrame = Restore-OrcaPointerWindow $process $root
$from = Get-OrcaElementScreenPoint $fromElement
if ($null -eq $from -and $null -ne $Operation.fromElement) { throw "stale element frame; run get-app-state again and use a fresh element index" }
if ($null -eq $from) {
@@ -1285,7 +1294,7 @@ function Invoke-OrcaOperation($Operation) {
y = $windowFrame.y + (Get-OrcaRequiredNumber $Operation.to_y "to_y")
}
}
- Send-OrcaDrag $handle $from $to
+ Send-OrcaDrag $from $to
$action = [pscustomobject]@{ path = "synthetic"; actionName = "drag"; fallbackReason = $null }
}
"type_text" {
diff --git a/tests/e2e/computer-linux.e2e.ts b/tests/e2e/computer-linux.e2e.ts
index 418b8377533..0342535a901 100644
--- a/tests/e2e/computer-linux.e2e.ts
+++ b/tests/e2e/computer-linux.e2e.ts
@@ -12,6 +12,39 @@ import {
const isLinux = process.platform === 'linux'
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
+async function focusGeditDocument(): Promise<void> {
+ const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+ (
+ await runOrcaCli([
+ 'computer',
+ 'get-app-state',
+ '--app',
+ 'gedit',
+ '--restore-window',
+ '--no-screenshot',
+ '--json'
+ ])
+ ).stdout
+ )
+ const textIndex = findRoleIndex(
+ before.result.snapshot.treeText,
+ /^\s*(\d+)\s+(text|text entry|entry|document|edit area)(?:\s|$)/im
+ )
+ expect(textIndex).toBeGreaterThanOrEqual(0)
+
+ await runOrcaCli([
+ 'computer',
+ 'click',
+ '--app',
+ 'gedit',
+ '--element-index',
+ String(textIndex),
+ '--restore-window',
+ '--no-screenshot',
+ '--json'
+ ])
+}
+
describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {
beforeAll(async () => {
await ensureOrcaRuntimeLaunched()
@@ -33,39 +66,10 @@ describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {
expect(envelope.result.screenshot?.path).toContain('orca-computer-use')
})
- test('click and type-text send synthetic input to the document', async () => {
- const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
- (
- await runOrcaCli([
- 'computer',
- 'get-app-state',
- '--app',
- 'gedit',
- '--restore-window',
- '--no-screenshot',
- '--json'
- ])
- ).stdout
- )
- const textIndex = findRoleIndex(
- before.result.snapshot.treeText,
- /^\s*(\d+)\s+(text|text entry|entry|document|edit area)(?:\s|$)/im
- )
- expect(textIndex).toBeGreaterThanOrEqual(0)
+ test('click and type-text insert into the focused document through AT-SPI', async () => {
+ await focusGeditDocument()
- await runOrcaCli([
- 'computer',
- 'click',
- '--app',
- 'gedit',
- '--element-index',
- String(textIndex),
- '--restore-window',
- '--no-screenshot',
- '--json'
- ])
-
- const marker = ` orca-linux-type-${Date.now()}`
+ const marker = ` orca-linux-type-${Date.now()}-élan-🚀`
const typed = parseJsonOutput<{ result: ComputerActionResult }>(
(
await runOrcaCli([
@@ -81,12 +85,14 @@ describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {
])
).stdout
)
- expect(typed.result.action?.path).toBe('synthetic')
+ expect(typed.result.action?.path).toBe('accessibility')
expect(typed.result.snapshot.treeText).toContain(marker.trim())
})
test('paste-text mutates the test-owned document', async () => {
- const marker = `orca-linux-paste-${Date.now()}`
+ await focusGeditDocument()
+
+ const marker = `orca-linux-paste-${Date.now()}-café-🚀`
const action = parseJsonOutput<{ result: ComputerActionResult }>(
(
await runOrcaCli([
@@ -102,10 +108,10 @@ describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {
])
).stdout
)
- expect(action.result.action?.path).toBe('clipboard')
+ expect(action.result.action?.path).toBe('accessibility')
expect(action.result.action?.verification).toMatchObject({
- state: 'unverified',
- reason: 'clipboard_paste'
+ state: 'verified',
+ property: 'focusedText'
})
const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(