Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
import asyncio
from typing import Any
import pytest
from xagent.core.computer.cua_driver import CuaDriverError, CuaDriverResult
from xagent.core.computer.environment import (
ComputerFrameMismatchError,
ComputerTargetNotFoundError,
)
from xagent.core.computer.native_browser import NativeBrowserEnvironment
from xagent.core.computer.native_navigation import NativeBrowserNavigationResult
from xagent.core.computer.schema import (
COMPUTER_CONTROL_METADATA_KEY,
COMPUTER_FRAME_ID_METADATA_KEY,
COMPUTER_PERCEPTION_METADATA_KEY,
COMPUTER_SESSION_ID_METADATA_KEY,
ELEMENT_EXTRACTION_INCOMPLETE_KEY,
ComputerAction,
ComputerActionBatch,
ComputerActionType,
ComputerPerceptionMode,
ComputerTarget,
NormalizedPoint,
)
from xagent.core.context_ref import ContextReference
class FakeObservationStore:
async def save_screenshot(self, **kwargs: Any) -> ContextReference:
return ContextReference(
file_ref={
"file_id": "native-shot",
"filename": "native.png",
"mime_type": kwargs["mime_type"],
},
metadata={
COMPUTER_SESSION_ID_METADATA_KEY: kwargs["session_id"],
COMPUTER_FRAME_ID_METADATA_KEY: kwargs["frame_id"],
},
)
class FakeCuaDriver:
def __init__(
self,
*,
windows: list[dict[str, Any]] | None = None,
elements: list[dict[str, Any]] | None = None,
escalation: dict[str, Any] | None = None,
background_input: dict[str, Any] | None = None,
) -> None:
self.calls: list[tuple[str, dict[str, Any]]] = []
self.closed = False
self.windows = windows if windows is not None else self._default_windows()
self.elements = elements if elements is not None else self._default_elements()
self.escalation = escalation
self.background_input = background_input
async def call_tool(
self,
name: str,
arguments: dict[str, Any] | None = None,
) -> CuaDriverResult:
payload = dict(arguments or {})
self.calls.append((name, payload))
if name == "list_windows":
return CuaDriverResult(structured={"windows": self.windows})
if name == "get_window_state":
structured: dict[str, Any] = {
"window_id": payload["window_id"],
"pid": payload["pid"],
"url": "https://example.com/inbox",
"element_count": 2,
"screenshot_width": 1200,
"screenshot_height": 800,
"elements": self.elements,
}
if self.escalation is not None:
structured["escalation"] = self.escalation
if self.background_input is not None:
structured["background_input"] = self.background_input
return CuaDriverResult(
structured=structured,
image_bytes=b"native-png",
image_mime_type="image/png",
)
if name == "health_report":
return CuaDriverResult(structured={"schema_version": "1", "overall": "ok"})
return CuaDriverResult(
structured={"effect": "confirmed", "verified": True},
text=f"{name} completed",
)
async def close(self) -> None:
self.closed = True
@staticmethod
def _default_elements() -> list[dict[str, Any]]:
return [
{
"element_index": 4,
"element_token": "snapshot-1:4",
"role": "AXButton",
"label": "Continue",
"frame": {"x": 200, "y": 300, "w": 200, "h": 80},
},
{
"element_index": 5,
"element_token": "snapshot-1:5",
"role": "AXSecureTextField",
"label": "Password",
"value": "do-not-leak",
"frame": {"x": 300, "y": 450, "w": 300, "h": 50},
},
{
"element_index": 6,
"element_token": "snapshot-1:6",
"role": "AXTextField",
"label": "Address and search bar",
"value": "https://example.com/inbox",
"frame": {"x": 180, "y": 220, "w": 760, "h": 40},
},
]
@staticmethod
def _default_windows() -> list[dict[str, Any]]:
return [
{
"window_id": 10,
"pid": 100,
"app_name": "Google Chrome",
"title": "Background",
"bounds": {"x": 10, "y": 10, "width": 900, "height": 700},
"z_index": 1,
"is_on_screen": True,
"on_current_space": False,
},
{
"window_id": 20,
"pid": 200,
"app_name": "Google Chrome",
"title": "Inbox",
"bounds": {"x": 100, "y": 200, "width": 1000, "height": 800},
"z_index": 9,
"is_on_screen": True,
"on_current_space": True,
},
]
class FakeNativeBrowserNavigator:
def __init__(self, *, supported: bool) -> None:
self.supported = supported
self.calls: list[tuple[Any, str]] = []
def supports(self, target: Any) -> bool:
return self.supported
async def navigate(
self,
target: Any,
url: str,
) -> NativeBrowserNavigationResult:
self.calls.append((target, url))
return NativeBrowserNavigationResult(
route="fake_native_navigation",
browser_window_id=777,
actual_url=url,
)
def make_environment(
driver: FakeCuaDriver,
*,
perception_mode: ComputerPerceptionMode | str = ComputerPerceptionMode.AUTO,
native_navigator: FakeNativeBrowserNavigator | None = None,
) -> NativeBrowserEnvironment:
return NativeBrowserEnvironment(
session_id="task-1",
workspace=object(),
driver=driver,
observation_store=FakeObservationStore(), # type: ignore[arg-type]
perception_mode=perception_mode,
native_browser_navigator=(
native_navigator or FakeNativeBrowserNavigator(supported=False)
),
)
def batch(frame_id: str, action: ComputerAction) -> ComputerActionBatch:
return ComputerActionBatch(
session_id="task-1",
expected_frame_id=frame_id,
actions=[action],
)
def test_local_browser_requires_explicit_enablement(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("XAGENT_NATIVE_BROWSER_ENABLED", raising=False)
with pytest.raises(RuntimeError, match="Local browser access is disabled"):
NativeBrowserEnvironment(session_id="task-1", workspace=object())
@pytest.mark.asyncio
async def test_local_browser_binds_frontmost_window_and_redacts_password() -> None:
driver = FakeCuaDriver()
environment = make_environment(driver)
observation = await environment.observe()
assert observation.title == "Inbox"
assert observation.active_url == "https://example.com/inbox"
assert observation.viewport.width == 1200
assert observation.metadata["pid"] == 200
assert observation.metadata["window_id"] == 20
assert observation.environment.value == "desktop"
assert observation.metadata["computer_runtime_kind"] == "local_browser"
assert observation.metadata[COMPUTER_PERCEPTION_METADATA_KEY] == {
"mode": "auto",
"available": ["vision", "semantic"],
"semantic_source": "accessibility",
}
assert observation.metadata[COMPUTER_CONTROL_METADATA_KEY] == {
"transport": "native_accessibility",
"scope": "window",
"browser_debugging": False,
}
assert "available_windows" not in observation.metadata
assert "Background" not in observation.model_dump_json()
assert "move" not in observation.metadata["supported_actions"]
assert "navigate" not in observation.metadata["supported_actions"]
assert observation.metadata["unsupported_actions"]["navigate"] == (
"pid_keyboard_capability_unknown"
)
assert observation.elements[0].element_id == "snapshot-1:4"
assert observation.elements[1].label == "Sensitive input"
assert observation.elements[1].text is None
assert "do-not-leak" not in observation.model_dump_json()
assert [name for name, _payload in driver.calls[:3]] == [
"start_session",
"list_windows",
"get_window_state",
]
@pytest.mark.asyncio
async def test_local_browser_vision_mode_does_not_expose_semantic_targets() -> None:
driver = FakeCuaDriver()
environment = make_environment(
driver,
perception_mode=ComputerPerceptionMode.VISION,
)
observation = await environment.observe()
assert observation.elements == []
assert observation.metadata[COMPUTER_PERCEPTION_METADATA_KEY] == {
"mode": "vision",
"available": ["vision", "semantic"],
"semantic_source": "accessibility",
}
assert ELEMENT_EXTRACTION_INCOMPLETE_KEY not in observation.metadata
assert all(not name.startswith("browser_") for name, _ in driver.calls)
@pytest.mark.asyncio
async def test_local_browser_marks_ax_surfaces_from_parent_chain() -> None:
elements = [
{
"element_index": 0,
"element_token": "snapshot-1:0",
"role": "AXWindow",
"depth": 0,
"frame": {"x": 100, "y": 200, "w": 1000, "h": 800},
},
{
"element_index": 13,
"element_token": "snapshot-1:13",
"role": "AXButton",
"label": "Xuye",
"depth": 1,
"parent_index": 0,
"frame": {"x": 1060, "y": 220, "w": 20, "h": 20},
},
{
"element_index": 15,
"element_token": "snapshot-1:15",
"role": "AXWebArea",
"label": "Zhihu",
"depth": 1,
"parent_index": 0,
"frame": {"x": 100, "y": 260, "w": 1000, "h": 740},
},
{
"element_index": 16,
"element_token": "snapshot-1:16",
"role": "AXLink",
"label": "Qin Xuye",
"depth": 2,
"parent_index": 15,
"frame": {"x": 900, "y": 300, "w": 80, "h": 30},
},
{
"element_index": 20,
"element_token": "snapshot-1:20",
"role": "AXSheet",
"depth": 1,
"parent_index": 0,
"frame": {"x": 400, "y": 300, "w": 400, "h": 300},
},
{
"element_index": 21,
"element_token": "snapshot-1:21",
"role": "AXButton",
"label": "Confirm",
"depth": 2,
"parent_index": 20,
"frame": {"x": 650, "y": 520, "w": 100, "h": 40},
},
{
"element_index": 30,
"element_token": "snapshot-1:30",
"role": "AXButton",
"label": "Incomplete",
"depth": 2,
"parent_index": 999,
"frame": {"x": 200, "y": 300, "w": 100, "h": 40},
},
]
environment = make_environment(FakeCuaDriver(elements=elements))
observation = await environment.observe()
by_index = {
element.metadata["element_index"]: element for element in observation.elements
}
assert by_index[13].surface is not None
assert by_index[13].surface.value == "application_chrome"
assert by_index[13].metadata["surface"] == "application_chrome"
assert by_index[13].metadata["surface_root_index"] == 0
assert by_index[15].surface is not None
assert by_index[15].surface.value == "document"
assert by_index[15].metadata["surface"] == "document"
assert by_index[15].metadata["surface_root_index"] == 15
assert by_index[16].metadata["surface"] == "document"
assert by_index[16].metadata["surface_root_index"] == 15
assert by_index[20].metadata["surface"] == "overlay"
assert by_index[20].metadata["surface_root_index"] == 20
assert by_index[21].metadata["surface"] == "overlay"
assert by_index[21].metadata["surface_root_index"] == 20
assert by_index[30].metadata["surface"] == "unknown"
assert "surface_root_index" not in by_index[30].metadata
@pytest.mark.asyncio
async def test_native_browser_click_uses_bound_window_and_element_token(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def no_sleep(_seconds: float) -> None:
return None
monkeypatch.setattr("xagent.core.computer.native_browser.asyncio.sleep", no_sleep)
driver = FakeCuaDriver()
environment = make_environment(driver)
first = await environment.observe()
second = await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
),
)
)
name, payload = next(
(name, payload) for name, payload in driver.calls if name == "click"
)
assert name == "click"
assert payload["pid"] == 200
assert payload["window_id"] == 20
assert payload["element_token"] == "snapshot-1:4"
assert payload["delivery_mode"] == "background"
assert second.metadata["last_action_result"]["effect"] == "confirmed"
assert [name for name, _payload in driver.calls].count("list_windows") == 3
@pytest.mark.asyncio
async def test_local_browser_refuses_ambiguous_pid_keyboard_actions() -> None:
driver = FakeCuaDriver(
background_input={
"exact_window": {"pid": 200, "window_id": 20, "status": "matched"},
"routes": [
{"route": "accessibility", "status": "available"},
{"route": "window_pointer", "status": "available"},
{
"route": "pid_keyboard",
"status": "refused",
"reason": "same_pid_keyboard_ambiguity",
},
],
}
)
environment = make_environment(
driver,
native_navigator=FakeNativeBrowserNavigator(supported=True),
)
observation = await environment.observe()
assert observation.metadata["background_input"] == driver.background_input
assert "click" in observation.metadata["supported_actions"]
assert "type" not in observation.metadata["supported_actions"]
assert "replace_text" in observation.metadata["supported_actions"]
assert "keypress" not in observation.metadata["supported_actions"]
assert "navigate" in observation.metadata["supported_actions"]
assert observation.metadata["unsupported_actions"] == {
"type": "unscoped_keyboard_input_disabled",
"keypress": "unscoped_keyboard_input_disabled",
}
with pytest.raises(ValueError, match="unscoped_keyboard_input_disabled"):
await environment.execute(
batch(
observation.frame_id,
ComputerAction(type=ComputerActionType.TYPE, text="unsafe"),
)
)
@pytest.mark.asyncio
async def test_local_browser_refuses_keyboard_when_driver_capability_is_absent() -> (
None
):
observation = await make_environment(FakeCuaDriver()).observe()
assert "type" not in observation.metadata["supported_actions"]
assert observation.metadata["unsupported_actions"]["type"] == (
"unscoped_keyboard_input_disabled"
)
@pytest.mark.asyncio
async def test_local_browser_limits_keyboard_input_to_document_text_targets() -> None:
elements = [
{
"element_index": 0,
"element_token": "snapshot-1:0",
"role": "AXWindow",
"depth": 0,
"frame": {"x": 100, "y": 200, "w": 1000, "h": 800},
},
{
"element_index": 1,
"element_token": "snapshot-1:1",
"role": "AXTextField",
"label": "Address and search bar",
"depth": 1,
"parent_index": 0,
"frame": {"x": 180, "y": 220, "w": 760, "h": 40},
},
{
"element_index": 2,
"element_token": "snapshot-1:2",
"role": "AXWebArea",
"depth": 1,
"parent_index": 0,
"frame": {"x": 100, "y": 280, "w": 1000, "h": 720},
},
{
"element_index": 3,
"element_token": "snapshot-1:3",
"role": "AXTextField",
"label": "Search",
"depth": 2,
"parent_index": 2,
"frame": {"x": 300, "y": 350, "w": 400, "h": 40},
},
]
driver = FakeCuaDriver(
elements=elements,
background_input={
"routes": [{"route": "pid_keyboard", "status": "available"}],
},
)
environment = make_environment(driver)
observation = await environment.observe()
assert "replace_text" in observation.metadata["supported_actions"]
assert "type" not in observation.metadata["supported_actions"]
assert "keypress" not in observation.metadata["supported_actions"]
assert observation.metadata["unsupported_actions"]["type"] == (
"unscoped_keyboard_input_disabled"
)
after_document_edit = await environment.execute(
batch(
observation.frame_id,
ComputerAction(
type=ComputerActionType.REPLACE_TEXT,
target=ComputerTarget(element_id="snapshot-1:3"),
text="safe site search",
),
)
)
action_call_count = len(driver.calls)
with pytest.raises(ValueError, match="limited to document elements"):
await environment.execute(
batch(
after_document_edit.frame_id,
ComputerAction(
type=ComputerActionType.REPLACE_TEXT,
target=ComputerTarget(element_id="snapshot-1:1"),
text="file:///Users/admin/.ssh/id_rsa",
),
)
)
set_value = next(
payload
for name, payload in driver.calls[:action_call_count]
if name == "set_value"
)
assert set_value["element_token"] == "snapshot-1:3"
assert set_value["value"] == "safe site search"
assert all(
name not in {"hotkey", "type_text"}
for name, _ in driver.calls[:action_call_count]
)
assert len(driver.calls) == action_call_count + 1 # Bound-window revalidation only.
@pytest.mark.asyncio
async def test_local_browser_replace_text_does_not_use_pixel_occlusion() -> None:
elements = [
{
"element_index": 0,
"element_token": "snapshot-1:0",
"role": "AXWindow",
"depth": 0,
"frame": {"x": 100, "y": 200, "w": 1000, "h": 800},
},
{
"element_index": 1,
"element_token": "snapshot-1:1",
"role": "AXWebArea",
"depth": 1,
"parent_index": 0,
"frame": {"x": 100, "y": 280, "w": 1000, "h": 720},
},
{
"element_index": 2,
"element_token": "snapshot-1:2",
"role": "AXTextField",
"label": "Search",
"depth": 2,
"parent_index": 1,
"frame": {"x": 300, "y": 350, "w": 400, "h": 40},
},
]
driver = FakeCuaDriver(elements=elements)
driver.windows.append(
{
"window_id": 31,
"pid": 300,
"app_name": "Terminal",
"title": "Overlapping window",
"bounds": {"x": 100, "y": 200, "width": 1000, "height": 800},
"z_index": 10,
"is_on_screen": True,
"on_current_space": True,
}
)
environment = make_environment(driver)
observation = await environment.observe()
await environment.execute(
batch(
observation.frame_id,
ComputerAction(
type=ComputerActionType.REPLACE_TEXT,
target=ComputerTarget(element_id="snapshot-1:2"),
text="semantic input",
),
)
)
set_value = next(payload for name, payload in driver.calls if name == "set_value")
assert set_value["element_token"] == "snapshot-1:2"
assert set_value["value"] == "semantic input"
@pytest.mark.asyncio
async def test_local_browser_rejects_sensitive_document_text_targets() -> None:
elements = [
{
"element_index": 0,
"element_token": "snapshot-1:0",
"role": "AXWindow",
"depth": 0,
"frame": {"x": 100, "y": 200, "w": 1000, "h": 800},
},
{
"element_index": 1,
"element_token": "snapshot-1:1",
"role": "AXWebArea",
"depth": 1,
"parent_index": 0,
"frame": {"x": 100, "y": 280, "w": 1000, "h": 720},
},
{
"element_index": 2,
"element_token": "snapshot-1:2",
"role": "AXTextField",
"label": "一次性验证码 (OTP)",
"depth": 2,
"parent_index": 1,
"frame": {"x": 300, "y": 350, "w": 400, "h": 40},
},
]
driver = FakeCuaDriver(elements=elements)
environment = make_environment(driver)
observation = await environment.observe()
sensitive = next(
element
for element in observation.elements
if element.element_id == "snapshot-1:2"
)
assert sensitive.label == "Sensitive input"
assert sensitive.metadata["sensitive"] is True
with pytest.raises(ValueError, match="sensitive input"):
await environment.execute(
batch(
observation.frame_id,
ComputerAction(
type=ComputerActionType.REPLACE_TEXT,
target=ComputerTarget(element_id="snapshot-1:2"),
text="123456",
),
)
)
assert all(name != "set_value" for name, _ in driver.calls)
@pytest.mark.asyncio
async def test_local_browser_does_not_expose_snapshot_indices_without_tokens() -> None:
driver = FakeCuaDriver(
elements=[
{
"element_index": 4,
"role": "AXButton",
"label": "Stale target",
"frame": {"x": 200, "y": 300, "w": 200, "h": 80},
}
]
)
observation = await make_environment(driver).observe()
assert observation.elements == []
@pytest.mark.asyncio
async def test_native_browser_requires_driver_proof_for_foreground_delivery() -> None:
driver = FakeCuaDriver()
environment = make_environment(driver)
first = await environment.observe()
with pytest.raises(ValueError, match="escalation recommendation"):
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
metadata={"delivery_mode": "foreground"},
),
)
)
@pytest.mark.asyncio
async def test_native_browser_allows_driver_recommended_foreground_delivery(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def no_sleep(_seconds: float) -> None:
return None
monkeypatch.setattr("xagent.core.computer.native_browser.asyncio.sleep", no_sleep)
driver = FakeCuaDriver(
escalation={"recommended": "foreground", "reason": "background did not land"}
)
environment = make_environment(driver)
first = await environment.observe()
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
metadata={"delivery_mode": "foreground"},
),
)
)
click = next(payload for name, payload in driver.calls if name == "click")
assert click["delivery_mode"] == "foreground"
@pytest.mark.asyncio
async def test_local_browser_rejects_pixel_action_after_window_geometry_changes() -> (
None
):
driver = FakeCuaDriver()
environment = make_environment(driver)
first = await environment.observe()
assert first.metadata["coordinate_frame"] == {
"screenshot_width": 1200,
"screenshot_height": 800,
"screenshot_fresh": True,
"window_bounds": {
"x": 100.0,
"y": 200.0,
"width": 1000.0,
"height": 800.0,
},
}
driver.windows[1]["bounds"]["width"] = 900
with pytest.raises(ComputerFrameMismatchError, match="moved or resized"):
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(
point=NormalizedPoint(x=0.5, y=0.5),
),
),
)
)
assert all(name != "click" for name, _payload in driver.calls)
@pytest.mark.asyncio
async def test_local_browser_rejects_actions_after_bound_window_disappears() -> None:
driver = FakeCuaDriver()
environment = make_environment(driver)
first = await environment.observe()
driver.windows = [window for window in driver.windows if window["window_id"] != 20]
with pytest.raises(ComputerTargetNotFoundError, match="disappeared"):
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
),
)
)
assert all(name != "click" for name, _payload in driver.calls)
@pytest.mark.asyncio
async def test_local_browser_rejects_pixel_action_under_cross_app_occluding_window() -> (
None
):
driver = FakeCuaDriver()
driver.windows.append(
{
"window_id": 31,
"pid": 300,
"app_name": "Terminal",
"title": "Unrelated foreground window",
"bounds": {"x": 100, "y": 200, "width": 1000, "height": 800},
"z_index": 10,
"is_on_screen": True,
"on_current_space": True,
}
)
environment = NativeBrowserEnvironment(
session_id="task-1",
workspace=object(),
driver=driver,
observation_store=FakeObservationStore(), # type: ignore[arg-type]
target_pid=200,
target_window_id=20,
native_browser_navigator=FakeNativeBrowserNavigator(supported=False),
)
first = await environment.observe()
with pytest.raises(ComputerTargetNotFoundError, match="Pixel action is ambiguous"):
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(
point=NormalizedPoint(x=0.5, y=0.5),
),
),
)
)
assert all(name != "click" for name, _payload in driver.calls)
@pytest.mark.asyncio
async def test_local_browser_treats_unknown_overlapping_z_index_as_occluding() -> None:
driver = FakeCuaDriver()
driver.windows.append(
{
"window_id": 31,
"pid": 300,
"app_name": "Terminal",
"title": "Unknown stacking order",
"bounds": {"x": 100, "y": 200, "width": 1000, "height": 800},
"is_on_screen": True,
"on_current_space": True,
}
)
environment = NativeBrowserEnvironment(
session_id="task-1",
workspace=object(),
driver=driver,
observation_store=FakeObservationStore(), # type: ignore[arg-type]
target_pid=200,
target_window_id=20,
native_browser_navigator=FakeNativeBrowserNavigator(supported=False),
)
first = await environment.observe()
with pytest.raises(ComputerTargetNotFoundError, match="Pixel action is ambiguous"):
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(point=NormalizedPoint(x=0.5, y=0.5)),
),
)
)
assert all(name != "click" for name, _payload in driver.calls)
@pytest.mark.asyncio
async def test_local_browser_prioritizes_overlay_elements_beyond_context_limit() -> (
None
):
elements: list[dict[str, Any]] = [
{
"element_index": 0,
"element_token": "snapshot-1:0",
"role": "AXWindow",
"depth": 0,
"frame": {"x": 100, "y": 200, "w": 1000, "h": 800},
}
]
elements.extend(
{
"element_index": index,
"element_token": f"snapshot-1:{index}",
"role": "AXButton",
"label": f"Action {index}",
"depth": 1,
"parent_index": 0,
"enabled": True,
"frame": {
"x": 120 + (index % 20) * 30,
"y": 260 + (index % 15) * 25,
"w": 24,
"h": 20,
},
}
for index in range(1, 131)
)
elements.extend(
[
{
"element_index": 200,
"element_token": "snapshot-1:200",
"role": "AXMenu",
"depth": 1,
"parent_index": 0,
"frame": {"x": 800, "y": 260, "w": 200, "h": 180},
},
{
"element_index": 201,
"element_token": "snapshot-1:201",
"role": "AXMenuItem",
"label": "My profile",
"depth": 2,
"parent_index": 200,
"enabled": True,
"frame": {"x": 820, "y": 280, "w": 160, "h": 30},
},
]
)
driver = FakeCuaDriver(elements=elements)
environment = make_environment(driver)
observation = await environment.observe()
assert len(observation.elements) == 100
profile = next(
element for element in observation.elements if element.label == "My profile"
)
assert profile.surface is not None
assert profile.surface.value == "overlay"
state_call = next(
payload for name, payload in driver.calls if name == "get_window_state"
)
assert state_call["max_elements"] == 2_000
@pytest.mark.asyncio
async def test_local_browser_refuses_pixel_fallback_when_ax_owner_is_unprovable() -> (
None
):
class RefusingElementDriver(FakeCuaDriver):
async def call_tool(
self,
name: str,
arguments: dict[str, Any] | None = None,
) -> CuaDriverResult:
payload = dict(arguments or {})
if name == "click" and "element_token" in payload:
self.calls.append((name, payload))
raise CuaDriverError(
"Background input refused (element_outside_target_window)"
)
return await super().call_tool(name, arguments)
driver = RefusingElementDriver()
environment = make_environment(driver)
first = await environment.observe()
with pytest.raises(CuaDriverError, match="element_outside_target_window"):
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
),
)
)
clicks = [payload for name, payload in driver.calls if name == "click"]
assert len(clicks) == 1
assert clicks[0]["element_token"] == "snapshot-1:4"
assert "x" not in clicks[0]
assert "y" not in clicks[0]
@pytest.mark.asyncio
async def test_local_browser_keeps_fresh_semantics_when_transient_capture_is_unavailable() -> (
None
):
class TransientCaptureDriver(FakeCuaDriver):
async def call_tool(
self,
name: str,
arguments: dict[str, Any] | None = None,
) -> CuaDriverResult:
if name == "get_window_state" and any(
called_name == "click" for called_name, _payload in self.calls
):
payload = dict(arguments or {})
self.calls.append((name, payload))
return CuaDriverResult(
structured={
"window_id": payload["window_id"],
"pid": payload["pid"],
"element_count": 1,
"elements": [
{
"element_index": 20,
"element_token": "snapshot-2:20",
"role": "AXLink",
"label": "My profile",
"frame": {
"x": 800,
"y": 260,
"w": 120,
"h": 30,
},
}
],
"background_input": {
"observation": {"one_shot_capture": "unavailable"}
},
}
)
return await super().call_tool(name, arguments)
driver = TransientCaptureDriver()
environment = make_environment(driver)
first = await environment.observe()
second = await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
),
)
)
assert second.frame_id != first.frame_id
assert second.screenshot.file_id == first.screenshot.file_id
assert second.screenshot.metadata["reused_from_frame_id"] == first.frame_id
assert second.metadata["screenshot_fresh"] is False
assert second.metadata["computer_perception"]["available"] == ["semantic"]
assert [element.label for element in second.elements] == ["My profile"]
with pytest.raises(ComputerFrameMismatchError, match="fresh semantic elements"):
await environment.execute(
batch(
second.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(
point=NormalizedPoint(x=0.5, y=0.5),
),
),
)
)
@pytest.mark.asyncio
async def test_local_browser_waits_for_unverified_semantic_state_change(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def no_sleep(_seconds: float) -> None:
return None
monkeypatch.setattr("xagent.core.computer.native_browser.asyncio.sleep", no_sleep)
class DelayedSemanticDriver(FakeCuaDriver):
state_reads = 0
async def call_tool(
self,
name: str,
arguments: dict[str, Any] | None = None,
) -> CuaDriverResult:
if name == "click":
payload = dict(arguments or {})
self.calls.append((name, payload))
return CuaDriverResult(
structured={"effect": "unverifiable"},
text="AXPress posted",
)
if name == "get_window_state":
self.state_reads += 1
if self.state_reads >= 3:
self.elements = [
{
"element_index": 20,
"element_token": "snapshot-2:20",
"role": "AXLink",
"label": "My profile",
"frame": {"x": 250, "y": 360, "w": 120, "h": 30},
}
]
return await super().call_tool(name, arguments)
driver = DelayedSemanticDriver()
environment = make_environment(driver)
first = await environment.observe()
second = await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
),
)
)
assert driver.state_reads == 3
assert [element.label for element in second.elements] == ["My profile"]
assert second.metadata["last_action_result"]["effect"] == "unverifiable"
@pytest.mark.asyncio
async def test_local_browser_records_stall_without_authorizing_foreground(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def no_sleep(_seconds: float) -> None:
return None
monkeypatch.setattr("xagent.core.computer.native_browser.asyncio.sleep", no_sleep)
class UnchangedSemanticDriver(FakeCuaDriver):
async def call_tool(
self,
name: str,
arguments: dict[str, Any] | None = None,
) -> CuaDriverResult:
if name == "click":
payload = dict(arguments or {})
self.calls.append((name, payload))
return CuaDriverResult(
structured={"effect": "unverifiable"},
text="AXPress posted",
)
return await super().call_tool(name, arguments)
driver = UnchangedSemanticDriver()
environment = make_environment(driver)
first = await environment.observe()
stalled = await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
),
)
)
assert stalled.metadata["last_action_result"]["code"] == (
"no_observable_state_change_after_background_delivery"
)
assert "escalation" not in stalled.metadata["last_action_result"]
with pytest.raises(ValueError, match="current cua-driver escalation"):
await environment.execute(
batch(
stalled.frame_id,
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="snapshot-1:4"),
metadata={"delivery_mode": "foreground"},
),
)
)
@pytest.mark.asyncio
async def test_native_browser_defensively_rejects_incomplete_drag() -> None:
driver = FakeCuaDriver()
environment = make_environment(driver)
first = await environment.observe()
invalid_drag = ComputerAction.model_construct(
type=ComputerActionType.DRAG,
target=None,
url=None,
text=None,
keys=[],
delta_x=0,
delta_y=0,
start=None,
end=None,
duration_ms=0,
metadata={},
)
with pytest.raises(ValueError, match="drag requires start and end points"):
await environment.execute(batch(first.frame_id, invalid_drag))
@pytest.mark.asyncio
async def test_local_browser_navigates_with_native_browser_adapter() -> None:
driver = FakeCuaDriver()
navigator = FakeNativeBrowserNavigator(supported=True)
environment = NativeBrowserEnvironment(
session_id="task-1",
workspace=object(),
driver=driver,
observation_store=FakeObservationStore(), # type: ignore[arg-type]
native_browser_navigator=navigator,
)
first = await environment.observe()
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.NAVIGATE,
url="https://example.com/account",
),
)
)
assert len(navigator.calls) == 1
navigated_target, navigated_url = navigator.calls[0]
assert navigated_target.pid == 200
assert navigated_target.window_id == 20
assert navigated_url == "https://example.com/account"
assert environment.current_observation is not None
assert (
environment.current_observation.metadata["last_action_result"]["actual_url"]
== "https://example.com/account"
)
assert all(name not in {"set_value", "press_key"} for name, _ in driver.calls)
assert [name for name, _payload in driver.calls].count("get_window_state") == 2
await environment.close()
assert ("end_session", {"session": "task-1"}) in driver.calls
assert driver.closed is True
assert environment.closed is True
@pytest.mark.asyncio
async def test_local_browser_close_can_retry_after_cancellation() -> None:
class CancelOnceDriver(FakeCuaDriver):
close_calls = 0
async def close(self) -> None:
self.close_calls += 1
if self.close_calls == 1:
raise asyncio.CancelledError
self.closed = True
driver = CancelOnceDriver()
environment = make_environment(driver)
await environment.observe()
with pytest.raises(asyncio.CancelledError):
await environment.close()
await environment.close()
assert driver.close_calls == 2
assert driver.closed is True
@pytest.mark.asyncio
async def test_local_browser_falls_back_to_address_field_when_keyboard_is_safe() -> (
None
):
driver = FakeCuaDriver(
background_input={
"routes": [{"route": "pid_keyboard", "status": "available"}],
}
)
environment = NativeBrowserEnvironment(
session_id="task-1",
workspace=object(),
driver=driver,
observation_store=FakeObservationStore(), # type: ignore[arg-type]
native_browser_navigator=FakeNativeBrowserNavigator(supported=False),
)
first = await environment.observe()
await environment.execute(
batch(
first.frame_id,
ComputerAction(
type=ComputerActionType.NAVIGATE,
url="https://example.com/account",
),
)
)
set_value = next(payload for name, payload in driver.calls if name == "set_value")
assert set_value["element_token"] == "snapshot-1:6"
assert set_value["value"] == "https://example.com/account"
press_key = next(payload for name, payload in driver.calls if name == "press_key")
assert press_key["element_token"] == "snapshot-1:6"
assert press_key["key"] == "return"
assert press_key["delivery_mode"] == "background"
@pytest.mark.asyncio
async def test_local_browser_refuses_missing_or_hidden_window() -> None:
driver = FakeCuaDriver(windows=[])
with pytest.raises(RuntimeError, match="No visible Google Chrome window"):
await make_environment(driver).observe()
hidden = FakeCuaDriver(
windows=[
{
"window_id": 10,
"pid": 100,
"app_name": "Google Chrome",
"title": "Hidden",
"bounds": {"x": 10, "y": 10, "width": 900, "height": 700},
"z_index": 1,
"is_on_screen": False,
"on_current_space": False,
}
]
)
with pytest.raises(RuntimeError, match="No visible Google Chrome window"):
await make_environment(hidden).observe()
@pytest.mark.asyncio
async def test_local_browser_never_selects_a_non_browser_window() -> None:
driver = FakeCuaDriver(
windows=[
{
"window_id": 30,
"pid": 300,
"app_name": "Music",
"title": "Songs",
"bounds": {"x": 10, "y": 10, "width": 900, "height": 700},
"z_index": 20,
"is_on_screen": True,
"on_current_space": True,
}
]
)
with pytest.raises(RuntimeError, match="No visible Google Chrome window"):
await make_environment(driver).observe()
@pytest.mark.asyncio
async def test_local_browser_honors_exact_user_selected_window() -> None:
driver = FakeCuaDriver()
environment = NativeBrowserEnvironment(
session_id="task-1",
workspace=object(),
driver=driver,
observation_store=FakeObservationStore(), # type: ignore[arg-type]
target_pid=100,
target_window_id=10,
)
# Window 10 is on another Space and is therefore not a valid selected target.
with pytest.raises(RuntimeError, match="no longer visible"):
await environment.observe()
visible_driver = FakeCuaDriver()
visible_driver.windows[0]["on_current_space"] = True
selected = NativeBrowserEnvironment(
session_id="task-2",
workspace=object(),
driver=visible_driver,
observation_store=FakeObservationStore(), # type: ignore[arg-type]
target_pid=100,
target_window_id=10,
)
observation = await selected.observe()
assert observation.metadata["pid"] == 100
assert observation.metadata["window_id"] == 10