Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
import pytest
from xagent.core.agent import ExecutionContext
from xagent.core.computer.environment import ComputerEnvironment
from xagent.core.computer.schema import (
COMPUTER_FRAME_ID_METADATA_KEY,
COMPUTER_SESSION_ID_METADATA_KEY,
ComputerActionBatch,
ComputerElement,
ComputerElementSource,
ComputerElementSurface,
ComputerEnvironmentType,
ComputerObservation,
ComputerPerceptionMode,
NormalizedRect,
Viewport,
)
from xagent.core.context_ref import CONTEXT_REFS_KEY, ContextReference
from xagent.core.tools.adapters.vibe.computer import ComputerTool, ComputerToolArgs
def make_observation(session_id: str, index: int) -> ComputerObservation:
frame_id = f"frame-{index}"
return ComputerObservation(
session_id=session_id,
frame_id=frame_id,
environment=ComputerEnvironmentType.BROWSER,
viewport=Viewport(width=1280, height=720),
screenshot=ContextReference(
file_ref={
"file_id": f"image-{index}",
"filename": f"{frame_id}.png",
"mime_type": "image/png",
"internal": True,
},
metadata={
COMPUTER_SESSION_ID_METADATA_KEY: session_id,
COMPUTER_FRAME_ID_METADATA_KEY: frame_id,
},
),
active_url="about:blank",
)
class FakeComputerEnvironment(ComputerEnvironment):
def __init__(self, session_id: str) -> None:
super().__init__(session_id)
self.observe_count = 0
self.executed: list[ComputerActionBatch] = []
self.close_count = 0
async def _observe(self) -> ComputerObservation:
self.observe_count += 1
return make_observation(self.session_id, self.observe_count)
async def _execute(self, batch: ComputerActionBatch) -> ComputerObservation:
self.executed.append(batch)
self.observe_count += 1
return make_observation(self.session_id, self.observe_count)
async def _close(self) -> None:
self.close_count += 1
class EnvironmentFactory:
def __init__(self) -> None:
self.environments: list[FakeComputerEnvironment] = []
self.calls: list[dict[str, Any]] = []
def __call__(self, **kwargs: Any) -> FakeComputerEnvironment:
self.calls.append(kwargs)
environment = FakeComputerEnvironment(kwargs["session_id"])
self.environments.append(environment)
return environment
class RestrictedComputerEnvironment(FakeComputerEnvironment):
async def _observe(self) -> ComputerObservation:
observation = await super()._observe()
return observation.model_copy(
update={
"metadata": {
"supported_actions": ["screenshot", "click"],
"unsupported_actions": {
"type": "same_pid_keyboard_ambiguity",
},
}
}
)
class RestrictedEnvironmentFactory(EnvironmentFactory):
def __call__(self, **kwargs: Any) -> RestrictedComputerEnvironment:
self.calls.append(kwargs)
environment = RestrictedComputerEnvironment(kwargs["session_id"])
self.environments.append(environment)
return environment
class StaleScreenshotEnvironment(FakeComputerEnvironment):
async def _observe(self) -> ComputerObservation:
observation = await super()._observe()
return observation.model_copy(update={"metadata": {"screenshot_fresh": False}})
class StaleScreenshotEnvironmentFactory(EnvironmentFactory):
def __call__(self, **kwargs: Any) -> StaleScreenshotEnvironment:
self.calls.append(kwargs)
environment = StaleScreenshotEnvironment(kwargs["session_id"])
self.environments.append(environment)
return environment
class SemanticComputerEnvironment(FakeComputerEnvironment):
async def _observe(self) -> ComputerObservation:
observation = await super()._observe()
return observation.model_copy(
update={
"elements": [
ComputerElement(
element_id="button-1",
source=ComputerElementSource.DOM,
surface=ComputerElementSurface.DOCUMENT,
bounds=NormalizedRect(
x=0.1,
y=0.1,
width=0.2,
height=0.1,
),
)
]
}
)
class SemanticEnvironmentFactory(EnvironmentFactory):
def __call__(self, **kwargs: Any) -> SemanticComputerEnvironment:
self.calls.append(kwargs)
environment = SemanticComputerEnvironment(kwargs["session_id"])
self.environments.append(environment)
return environment
class ArtifactWorkspace:
def __init__(self, root: Path) -> None:
self.workspace_dir = root
self.output_dir = root / "output"
self.output_dir.mkdir(parents=True)
self.source = root / "frame.png"
self.source.write_bytes(b"png-bytes")
self.registered: dict[str, str] = {}
def resolve_file_id(self, _file_id: str) -> Path:
return self.source
def get_file_id_from_path(self, file_path: str) -> str | None:
return self.registered.get(str(Path(file_path).resolve()))
def register_file(self, file_path: str) -> str:
resolved = str(Path(file_path).resolve())
file_id = f"public-{len(self.registered) + 1}"
self.registered[resolved] = file_id
return file_id
@pytest.mark.parametrize(
("mode", "description_fragment"),
[
(ComputerPerceptionMode.AUTO, "Never invent an element_id"),
(
ComputerPerceptionMode.VISION,
"Ignore semantic element IDs",
),
(ComputerPerceptionMode.SEMANTIC, "Prefer an exact element_id"),
],
)
def test_computer_tool_describes_explicit_perception_mode(
mode: ComputerPerceptionMode,
description_fragment: str,
) -> None:
tool = ComputerTool(perception_mode=mode)
assert f"Perception mode is {mode.value}" in tool.description
assert description_fragment in tool.description
def test_computer_tool_normalizes_bare_element_id_target() -> None:
parsed = ComputerToolArgs.model_validate(
{
"expected_frame_id": "frame-1",
"action": "click",
"target": "snapshot-1:4",
}
)
assert parsed.target is not None
assert parsed.target.element_id == "snapshot-1:4"
assert parsed.target.point is None
@pytest.mark.asyncio
async def test_computer_tool_executes_normalized_bare_element_id_target() -> None:
factory = SemanticEnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
initial = await tool.run_json_async({})
acted = await tool.run_json_async(
{
"expected_frame_id": initial["frame_id"],
"action": "click",
"target": "button-1",
}
)
assert acted["success"] is True
target = factory.environments[0].executed[0].actions[0].target
assert target is not None
assert target.element_id == "button-1"
@pytest.mark.asyncio
async def test_computer_tool_returns_environment_factory_failure() -> None:
def failing_factory(**_kwargs: Any) -> ComputerEnvironment:
raise RuntimeError("local browser authorization was revoked")
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=failing_factory,
)
result = await tool.run_json_async({})
assert result["success"] is False
assert result["session_id"] == "task-1"
assert "authorization was revoked" in result["error"]
assert tool._environments == {}
@pytest.mark.parametrize(
("serialized_target", "element_id", "point"),
[
('"snapshot-1:4"', "snapshot-1:4", None),
('{"element_id": "snapshot-1:4"}', "snapshot-1:4", None),
('{"point": {"x": 0.25, "y": 0.75}}', None, (0.25, 0.75)),
],
)
def test_computer_tool_normalizes_json_encoded_target(
serialized_target: str,
element_id: str | None,
point: tuple[float, float] | None,
) -> None:
parsed = ComputerToolArgs.model_validate(
{
"expected_frame_id": "frame-1",
"action": "click",
"target": serialized_target,
}
)
assert parsed.target is not None
assert parsed.target.element_id == element_id
assert (
None
if parsed.target.point is None
else (parsed.target.point.x, parsed.target.point.y)
) == point
@pytest.mark.parametrize("serialized_target", ['["snapshot-1:4"]', "null", "42"])
def test_computer_tool_rejects_json_encoded_non_target_values(
serialized_target: str,
) -> None:
with pytest.raises(ValueError):
ComputerToolArgs.model_validate(
{
"expected_frame_id": "frame-1",
"action": "click",
"target": serialized_target,
}
)
def test_computer_tool_rejects_explicit_null_legacy_actions() -> None:
with pytest.raises(ValueError, match="exactly one action object"):
ComputerToolArgs.model_validate({"actions": None})
def test_computer_tool_rejects_conflicting_nested_target() -> None:
with pytest.raises(ValueError, match="target conflicts"):
ComputerToolArgs.model_validate(
{
"target": "outer-1",
"actions": [
{
"type": "click",
"target": "inner-2",
}
],
}
)
def test_computer_tool_rejects_unknown_top_level_fields() -> None:
with pytest.raises(ValueError, match="extra_forbidden"):
ComputerToolArgs.model_validate({"unexpected": "value"})
@pytest.mark.asyncio
async def test_computer_tool_requires_initial_observation_then_expected_frame() -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
initial = await tool.run_json_async({})
assert initial["success"] is True
assert initial["session_id"] == "task-1"
assert initial["frame_id"] == "frame-1"
assert initial[CONTEXT_REFS_KEY][0]["file_ref"]["file_id"] == "image-1"
assert initial[CONTEXT_REFS_KEY][0]["file_ref"]["internal"] is True
assert "file_ref" not in initial
missing_frame = await tool.run_json_async(
{
"action": "click",
"target": {"point": {"x": 0.5, "y": 0.5}},
}
)
assert missing_frame["success"] is False
assert "expected_frame_id" in missing_frame["error"]
acted = await tool.run_json_async(
{
"expected_frame_id": "frame-1",
"action": "click",
"target": {"point": {"x": 0.5, "y": 0.5}},
}
)
assert acted["success"] is True
assert acted["frame_id"] == "frame-2"
assert factory.environments[0].executed[0].expected_frame_id == "frame-1"
@pytest.mark.asyncio
async def test_computer_screenshot_publishes_user_visible_artifact(tmp_path) -> None:
factory = EnvironmentFactory()
workspace = ArtifactWorkspace(tmp_path / "workspace")
tool = ComputerTool(
task_id="task-1",
workspace=workspace, # type: ignore[arg-type]
environment_factory=factory,
)
initial = await tool.run_json_async({})
captured = await tool.run_json_async({"action": "screenshot"})
assert initial["observation"]["screenshot"]["file_ref"]["internal"] is True
assert initial["delivery"] == "private_observation"
assert "not a user-visible image" in initial["message"]
assert captured["success"] is True
assert captured["delivery"] == "user_visible_artifact"
assert captured["file_ref"]["file_id"] == "public-1"
assert captured["file_ref"].get("internal") is not True
assert captured["inline_markdown"] == (
f"![{captured['file_ref']['filename']}](file:public-1)"
)
assert "rather than file_ref.markdown_link" in captured["message"]
assert Path(captured["file_ref"]["relative_path"]).parent.name == "output"
output_path = workspace.workspace_dir / captured["file_ref"]["relative_path"]
assert output_path.read_bytes() == b"png-bytes"
assert captured[CONTEXT_REFS_KEY][0]["file_ref"]["internal"] is True
@pytest.mark.asyncio
async def test_computer_screenshot_frames_unexpected_publish_failure(tmp_path) -> None:
factory = EnvironmentFactory()
workspace = ArtifactWorkspace(tmp_path / "workspace")
tool = ComputerTool(
task_id="task-1",
workspace=workspace, # type: ignore[arg-type]
environment_factory=factory,
)
initial = await tool.run_json_async({})
def fail_to_resolve(_file_id: str) -> Path:
raise ImportError("backend unavailable")
workspace.resolve_file_id = fail_to_resolve # type: ignore[method-assign]
captured = await tool.run_json_async({"action": "screenshot"})
assert captured["success"] is False
assert captured["frame_id"] != initial["frame_id"]
assert captured["error"] == (
"Could not publish computer screenshot: backend unavailable"
)
@pytest.mark.asyncio
async def test_computer_tool_does_not_attach_or_publish_reused_screenshot(
tmp_path,
) -> None:
factory = StaleScreenshotEnvironmentFactory()
workspace = ArtifactWorkspace(tmp_path / "workspace")
tool = ComputerTool(
task_id="task-1",
workspace=workspace, # type: ignore[arg-type]
environment_factory=factory,
)
initial = await tool.run_json_async({})
captured = await tool.run_json_async({"action": "screenshot"})
assert initial["success"] is True
assert initial[CONTEXT_REFS_KEY] == []
assert captured["success"] is False
assert "temporarily unavailable" in captured["error"]
@pytest.mark.asyncio
async def test_computer_tool_rejects_known_unsupported_action_without_losing_frame() -> (
None
):
factory = RestrictedEnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
initial = await tool.run_json_async({})
refused = await tool.run_json_async(
{
"expected_frame_id": initial["frame_id"],
"action": "type",
"text": "https://example.com",
}
)
assert refused["success"] is False
assert refused["frame_id"] == initial["frame_id"]
assert "same_pid_keyboard_ambiguity" in refused["error"]
assert "Do not retry" in refused["error"]
assert factory.environments[0].current_observation is not None
assert factory.environments[0].executed == []
@pytest.mark.asyncio
async def test_computer_tool_derives_session_from_task_and_step() -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
result = await tool.run_json_async(
{
"session_id": "invented-by-model",
"_xagent_step_id": "inspect page",
}
)
assert result["session_id"] == "task-1:inspect_page"
assert factory.calls[0]["session_id"] == "task-1:inspect_page"
@pytest.mark.asyncio
async def test_task_scoped_computer_tool_shares_one_environment_across_steps() -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
environment_scope="task",
)
first = await tool.run_json_async({"_xagent_step_id": "inspect"})
second = await tool.run_json_async({"_xagent_step_id": "act"})
assert first["session_id"] == "task-1"
assert second["session_id"] == "task-1"
assert len(factory.environments) == 1
assert factory.environments[0].observe_count == 2
@pytest.mark.asyncio
async def test_task_scoped_computer_tool_serializes_competing_step_actions() -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
environment_scope="task",
)
initial = await tool.run_json_async({"_xagent_step_id": "inspect"})
action = {
"expected_frame_id": initial["frame_id"],
"action": "click",
"target": {"point": {"x": 0.5, "y": 0.5}},
}
first, second = await asyncio.gather(
tool.run_json_async({**action, "_xagent_step_id": "first"}),
tool.run_json_async({**action, "_xagent_step_id": "second"}),
)
assert sorted((first["success"], second["success"])) == [False, True]
assert len(factory.environments) == 1
assert len(factory.environments[0].executed) == 1
failure = first if first["success"] is False else second
assert "current frame" in failure["error"]
@pytest.mark.asyncio
async def test_computer_tool_lifts_action_scoped_expected_frame_id() -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
await tool.run_json_async({})
result = await tool.run_json_async(
{
"actions": [
{
"type": "click",
"expected_frame_id": "frame-1",
"target": {"point": {"x": 0.5, "y": 0.5}},
}
]
}
)
assert result["success"] is True
assert factory.environments[0].executed[0].expected_frame_id == "frame-1"
@pytest.mark.asyncio
async def test_computer_tool_rejects_action_before_first_observation() -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
result = await tool.run_json_async(
{"actions": [{"type": "navigate", "url": "https://example.com"}]}
)
assert result["success"] is False
assert "observation" in result["error"]
assert factory.environments[0].current_observation is None
@pytest.mark.asyncio
async def test_computer_tool_returns_validation_error_with_current_frame() -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
await tool.run_json_async({})
result = await tool.run_json_async(
{"actions": [{"type": "click", "target": {"point": {"x": 2, "y": 0}}}]}
)
assert result["success"] is False
assert result["session_id"] == "task-1"
assert result["frame_id"] == "frame-1"
assert "Invalid computer action" in result["error"]
assert len(factory.environments) == 1
def test_computer_tool_accepts_only_one_action_per_observation() -> None:
with pytest.raises(ValueError, match="exactly one action object"):
ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=EnvironmentFactory(),
).args_type().model_validate(
{"actions": [{"type": "screenshot"}, {"type": "screenshot"}]}
)
def test_computer_tool_exposes_flat_action_schema_and_accepts_legacy_shape() -> None:
args_type = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=EnvironmentFactory(),
).args_type()
schema = args_type.model_json_schema()
assert "action" in schema["properties"]
assert "actions" not in schema["properties"]
parsed = args_type.model_validate(
{
"expected_frame_id": "frame-1",
"actions": [
{
"type": "click",
"target": {"point": {"x": 0.5, "y": 0.5}},
}
],
}
)
assert parsed.action.value == "click"
assert parsed.to_action().target is not None
@pytest.mark.asyncio
async def test_computer_tool_teardown_closes_created_environments(
monkeypatch,
) -> None:
factory = EnvironmentFactory()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=factory,
)
await tool.run_json_async({})
async def no_browser_sessions(_task_id: str | None = None) -> None:
return None
monkeypatch.setattr(tool, "_close_task_sessions", no_browser_sessions)
await tool.teardown()
assert factory.environments[0].close_count == 1
assert tool._environments == {}
def test_new_computer_observation_supersedes_stale_live_context() -> None:
context = ExecutionContext()
tool = ComputerTool(
task_id="task-1",
workspace=object(), # type: ignore[arg-type]
environment_factory=EnvironmentFactory(),
)
first = make_observation("task-1", 1)
second = make_observation("task-1", 2)
for observation in (first, second):
result = {
"success": True,
"frame_id": observation.frame_id,
"observation": observation.model_dump(mode="json"),
CONTEXT_REFS_KEY: [observation.screenshot.durable_dict()],
"_xagent_supersedes_scope": f"{tool.name}:task-1",
}
context.add_tool_result("computer", result)
old_message, current_message = context.messages
assert old_message.metadata["superseded"] is True
assert old_message.metadata["raw_result"] == {
"success": True,
"superseded": True,
}
assert old_message.context_refs
assert CONTEXT_REFS_KEY not in old_message.to_dict()
assert old_message.context_refs_token_estimate() == 0
assert "superseded by a later observation" in old_message.content
assert CONTEXT_REFS_KEY in current_message.to_dict()