Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
from typing import Any
from unittest.mock import AsyncMock
import pytest
from xagent.core.computer.browser import (
_EDITABLE_ACTIVE_ELEMENT_SCRIPT,
_INTERACTIVE_ELEMENTS_SCRIPT,
BrowserComputerEnvironment,
)
from xagent.core.computer.environment import (
ComputerFrameMismatchError,
ComputerTargetNotFoundError,
)
from xagent.core.computer.schema import (
COMPUTER_FRAME_ID_METADATA_KEY,
COMPUTER_SESSION_ID_METADATA_KEY,
ComputerAction,
ComputerActionBatch,
ComputerActionType,
ComputerElementSurface,
ComputerTarget,
NormalizedPoint,
)
from xagent.core.context_ref import ContextReference
class FakeObservationStore:
def __init__(self) -> None:
self.calls: list[dict[str, Any]] = []
async def save_screenshot(self, **kwargs: Any) -> ContextReference:
self.calls.append(kwargs)
return ContextReference(
file_ref={
"file_id": f"image-{len(self.calls)}",
"filename": f"{kwargs['frame_id']}.png",
"mime_type": "image/png",
},
metadata={
COMPUTER_SESSION_ID_METADATA_KEY: kwargs["session_id"],
COMPUTER_FRAME_ID_METADATA_KEY: kwargs["frame_id"],
},
)
class FakeMouse:
def __init__(self) -> None:
self.calls: list[tuple[Any, ...]] = []
async def click(self, x: float, y: float) -> None:
self.calls.append(("click", x, y))
async def dblclick(self, x: float, y: float) -> None:
self.calls.append(("dblclick", x, y))
async def move(self, x: float, y: float, **kwargs: Any) -> None:
self.calls.append(("move", x, y, kwargs))
async def wheel(self, x: float, y: float) -> None:
self.calls.append(("wheel", x, y))
async def down(self) -> None:
self.calls.append(("down",))
async def up(self) -> None:
self.calls.append(("up",))
class FakeKeyboard:
def __init__(self) -> None:
self.calls: list[tuple[str, str]] = []
async def press(self, keys: str) -> None:
self.calls.append(("press", keys))
async def insert_text(self, text: str) -> None:
self.calls.append(("insert_text", text))
class FakePage:
def __init__(self) -> None:
self.viewport_size = {"width": 1280, "height": 720}
self.url = "about:blank"
self.frames: list[Any] = [object()]
self.mouse = FakeMouse()
self.keyboard = FakeKeyboard()
self.goto_calls: list[tuple[str, str, int]] = []
self.wait_calls: list[int] = []
self.active_element_editable = True
self.element_payload: Any = {
"elements": [
{
"bounds": {
"x": 0.1,
"y": 0.2,
"width": 0.2,
"height": 0.1,
},
"label": "Continue",
"role": "button",
"text": "Continue",
"metadata": {"tag": "button", "sensitive": False},
}
],
"truncated": False,
}
async def set_viewport_size(self, viewport: dict[str, int]) -> None:
self.viewport_size = viewport
async def screenshot(self, **kwargs: Any) -> bytes:
assert kwargs == {"full_page": False, "type": "png"}
return b"browser-png"
async def title(self) -> str:
return "Browser page"
async def evaluate(self, script: str, arg: Any = None) -> Any:
if "devicePixelRatio" in script:
return 2
if "querySelectorAll" in script:
assert arg == 100
if isinstance(self.element_payload, BaseException):
raise self.element_payload
return self.element_payload
if "currentDocument.activeElement" in script:
return self.active_element_editable
raise AssertionError(f"unexpected evaluate script: {script}")
async def goto(self, url: str, *, wait_until: str, timeout: int) -> None:
self.goto_calls.append((url, wait_until, timeout))
self.url = url
async def wait_for_timeout(self, duration_ms: int) -> None:
self.wait_calls.append(duration_ms)
class FakeSession:
def __init__(self, page: FakePage) -> None:
self.page = page
async def get_page(self) -> FakePage:
return self.page
class FakeManager:
def __init__(self, page: FakePage) -> None:
self.page = page
self.session = FakeSession(page)
self.calls: list[tuple[str, bool, str | None]] = []
self.closed: list[str] = []
async def get_or_create(
self,
session_id: str,
headless: bool,
locale: str | None = None,
workspace_root: str | None = None,
) -> FakeSession:
self.calls.append((session_id, headless, locale))
return self.session
async def close(self, session_id: str) -> None:
self.closed.append(session_id)
@pytest.fixture
def browser_environment(
monkeypatch: pytest.MonkeyPatch,
) -> tuple[
BrowserComputerEnvironment,
FakePage,
FakeObservationStore,
]:
# The fake page does not perform real DNS resolution; the production
# browser path validates every navigation target before calling goto().
monkeypatch.setattr(
"xagent.core.computer.browser._validate_browser_target",
AsyncMock(return_value=None),
)
page = FakePage()
store = FakeObservationStore()
environment = BrowserComputerEnvironment(
session_id="browser-1",
workspace=object(),
manager=FakeManager(page), # type: ignore[arg-type]
observation_store=store, # type: ignore[arg-type]
)
return environment, page, store
@pytest.mark.asyncio
async def test_browser_observation_captures_screenshot_and_dom_elements(
browser_environment,
) -> None:
environment, _page, store = browser_environment
observation = await environment.observe()
assert observation.active_url == "about:blank"
assert observation.title == "Browser page"
assert observation.viewport.model_dump() == {
"width": 1280,
"height": 720,
"device_pixel_ratio": 2.0,
}
assert observation.elements[0].element_id == "dom-1"
assert observation.elements[0].label == "Continue"
assert observation.elements[0].surface is ComputerElementSurface.DOCUMENT
assert store.calls[0]["image_bytes"] == b"browser-png"
assert observation.metadata["browser_runtime_kind"] == "ephemeral_playwright"
assert environment.current_observation == observation
assert environment.manager.calls == [ # type: ignore[attr-defined]
("browser-1:computer", True, None)
]
def test_browser_element_script_never_reads_input_values() -> None:
assert "node.value" not in _INTERACTIVE_ELEMENTS_SCRIPT
def test_editable_check_descends_into_same_origin_frames() -> None:
assert "node.contentDocument" in _EDITABLE_ACTIVE_ELEMENT_SCRIPT
@pytest.mark.asyncio
async def test_browser_actions_use_fresh_normalized_coordinates(
browser_environment,
) -> None:
environment, page, _store = browser_environment
first = await environment.observe()
clicked = await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=first.frame_id,
actions=[
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="dom-1"),
)
],
)
)
typed = await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=clicked.frame_id,
actions=[ComputerAction(type=ComputerActionType.TYPE, text="hello")],
)
)
scrolled = await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=typed.frame_id,
actions=[ComputerAction(type=ComputerActionType.SCROLL, delta_y=0.5)],
)
)
await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=scrolled.frame_id,
actions=[
ComputerAction(type=ComputerActionType.KEYPRESS, keys=["CTRL", "A"])
],
)
)
assert page.mouse.calls[0] == ("click", 256.0, 180.0)
assert page.mouse.calls[1] == ("wheel", 0, 360.0)
assert page.keyboard.calls == [
("insert_text", "hello"),
("press", "Control+A"),
]
@pytest.mark.asyncio
async def test_browser_point_click_uses_normalized_viewport(
browser_environment,
) -> None:
environment, page, _store = browser_environment
first = await environment.observe()
await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=first.frame_id,
actions=[
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(point=NormalizedPoint(x=0.5, y=0.5)),
)
],
)
)
assert page.mouse.calls[0] == ("click", 640.0, 360.0)
@pytest.mark.asyncio
async def test_browser_element_lookup_fails_with_clear_target_error(
browser_environment,
) -> None:
environment, _page, _store = browser_environment
await environment.observe()
action = ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(element_id="missing-element"),
)
with pytest.raises(ComputerTargetNotFoundError, match="missing-element"):
environment._target_pixels(action)
@pytest.mark.asyncio
async def test_browser_navigation_returns_new_observation(browser_environment) -> None:
environment, page, _store = browser_environment
first = await environment.observe()
observation = await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=first.frame_id,
actions=[
ComputerAction(
type=ComputerActionType.NAVIGATE,
url="https://example.com",
)
],
)
)
assert page.goto_calls == [("https://example.com", "domcontentloaded", 30_000)]
assert observation.active_url == "https://example.com"
@pytest.mark.asyncio
async def test_browser_omits_unsupported_active_url(browser_environment) -> None:
environment, page, _store = browser_environment
page.url = "data:text/html,secret"
observation = await environment.observe()
assert observation.active_url is None
assert observation.metadata["active_url_unavailable"] is True
@pytest.mark.asyncio
async def test_browser_recreated_session_requires_fresh_frame(
browser_environment,
) -> None:
environment, _page, _store = browser_environment
first = await environment.observe()
manager = environment.manager
replacement_page = FakePage()
manager.session = FakeSession(replacement_page) # type: ignore[attr-defined]
with pytest.raises(ComputerFrameMismatchError, match="session changed"):
await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=first.frame_id,
actions=[
ComputerAction(
type=ComputerActionType.CLICK,
target=ComputerTarget(point=NormalizedPoint(x=0.5, y=0.5)),
)
],
)
)
assert replacement_page.mouse.calls == []
assert environment.current_observation is None
refreshed = await environment.observe()
assert refreshed.frame_id != first.frame_id
@pytest.mark.asyncio
async def test_replace_text_rejects_non_editable_target(browser_environment) -> None:
environment, page, _store = browser_environment
first = await environment.observe()
page.active_element_editable = False
with pytest.raises(ValueError, match="not an editable element"):
await environment.execute(
ComputerActionBatch(
session_id="browser-1",
expected_frame_id=first.frame_id,
actions=[
ComputerAction(
type=ComputerActionType.REPLACE_TEXT,
target=ComputerTarget(point=NormalizedPoint(x=0.5, y=0.5)),
text="replacement",
)
],
)
)
assert page.keyboard.calls == []
def test_keypress_rejects_non_modifier_sequences() -> None:
assert BrowserComputerEnvironment._playwright_key_chord(["CTRL", "A"]) == (
"Control+A"
)
with pytest.raises(ValueError, match="one key chord"):
BrowserComputerEnvironment._playwright_key_chord(["ArrowDown", "Enter"])
@pytest.mark.asyncio
async def test_browser_reports_incomplete_or_failed_element_extraction(
browser_environment,
) -> None:
environment, page, _store = browser_environment
page.frames.append(object())
incomplete = await environment.observe()
assert incomplete.metadata["element_extraction_incomplete"] is True
page.element_payload = RuntimeError("detached document")
failed = await environment.observe()
assert failed.elements == []
assert failed.metadata["element_extraction_failed"] is True
@pytest.mark.asyncio
async def test_browser_environment_closes_its_session(browser_environment) -> None:
environment, _page, _store = browser_environment
await environment.close()
assert environment.manager.closed == [ # type: ignore[attr-defined]
"browser-1:computer"
]