Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(computer): derive SDK screenshot availability (#2929)

Call White40e7b52ba8cd
diff --git a/tests/test_computer_script_sdk.py b/tests/test_computer_script_sdk.py
index 37ed6b15e..9dd391529 100644
--- a/tests/test_computer_script_sdk.py
+++ b/tests/test_computer_script_sdk.py
@@ -77,18 +77,20 @@ def _fake_controller(
windows: list[dict] | None = None,
launch_pid: int | None = 41,
interaction_errors: dict[str, str] | None = None,
+ observation_screenshot_modes: list[str] | None = None,
) -> Iterator[tuple[ThreadingHTTPServer, list[dict]]]:
requests: list[dict] = []
observations = 0
clicks = 0
app_lookups = 0
connection_ids: set[int] = set()
+ driver_has_screenshot = False
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def do_POST(self) -> None: # noqa: N802
- nonlocal observations, clicks, app_lookups
+ nonlocal observations, clicks, app_lookups, driver_has_screenshot
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length))
connection_ids.add(id(self.connection))
@@ -301,13 +303,58 @@ def _fake_controller(
is False
):
elements = None
+ observation = {
+ "pid": 41,
+ "window_id": 9,
+ **({"elements": elements} if elements is not None else {}),
+ }
+ if observation_screenshot_modes is not None:
+ mode = observation_screenshot_modes[
+ min(observations - 1, len(observation_screenshot_modes) - 1)
+ ]
+ elif payload["arguments"].get("screenshot_out_file"):
+ mode = "file"
+ elif payload["arguments"].get("include_screenshot") is False:
+ mode = "tree"
+ else:
+ mode = "image"
+ if mode == "file":
+ observation["screenshot_file_path"] = payload["arguments"].get(
+ "screenshot_out_file", "/tmp/fixture-window.png"
+ )
+ elif mode == "invalid":
+ observation.update(
+ {
+ "screenshot_frame_valid": False,
+ "screenshot_width": 800,
+ "screenshot_height": 600,
+ }
+ )
+ driver_has_screenshot = mode in {"image", "file"}
+ body = _result(
+ observation,
+ image=mode in {"image", "invalid"},
+ )
+ elif tool in {
+ "click",
+ "double_click",
+ "right_click",
+ "drag",
+ "scroll",
+ "type_text",
+ "press_key",
+ "hotkey",
+ } and any(
+ field in payload["arguments"]
+ for field in ("x", "y", "from_x", "from_y", "to_x", "to_y")
+ ) and not driver_has_screenshot:
body = _result(
{
- "pid": 41,
- "window_id": 9,
- **({"elements": elements} if elements is not None else {}),
+ "code": "screenshot_context_missing",
+ "params": {},
+ "dispatched": False,
},
- image=True,
+ is_error=True,
)
elif tool == "click":
clicks += 1
@@ -400,6 +447,7 @@ def _run_node_script(
windows: list[dict] | None = None,
launch_pid: int | None = 41,
interaction_errors: dict[str, str] | None = None,
+ observation_screenshot_modes: list[str] | None = None,
) -> tuple[dict, ThreadingHTTPServer, list[dict]]:
if NODE is None:
pytest.skip("Node.js is unavailable")
@@ -430,6 +478,7 @@ def _run_node_script(
windows=windows,
launch_pid=launch_pid,
interaction_errors=interaction_errors,
+ observation_screenshot_modes=observation_screenshot_modes,
) as (server, requests):
request = {
"script_url": script.resolve().as_uri(),
@@ -884,6 +933,7 @@ import { computer } from "@avibe/computer";
export default async () => {
const mail = await computer.app({ bundleId: "com.apple.mail" });
const window = await mail.window({ labelContains: "New Message" });
+ await window.observe();
await window.element({ role: "button", label: "Missing" }).click();
return {};
};
@@ -2323,6 +2373,7 @@ import { computer } from "@avibe/computer";
export default async () => {
const mail = await computer.app({ bundleId: "com.apple.mail" });
const window = await mail.window({ labelContains: "New Message" });
+ await window.observe();
await window.element({ role: "button", label: "Send", index: 0 }).click();
};
""",
@@ -2336,7 +2387,7 @@ export default async () => {
try:
assert screenshot.read_bytes() == b"png-fixture"
assert error.details["step"] == "element.click"
- assert error.details["last_completed_step"] == "app.window"
+ assert error.details["last_completed_step"] == "window.observe"
assert error.details["ambiguous"] is True
assert any(
request["payload"].get("tool") == "click"
@@ -2810,3 +2861,126 @@ export default async () => {
("drag", None),
("drag", None),
]
+
+
+def test_sdk_tree_only_requested_capture_recaptures_before_pixel_input(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ await window.observe({ include_screenshot: true });
+ await window.drag({ from_x: 10, from_y: 20, to_x: 30, to_y: 40 });
+};
+""",
+ observation_screenshot_modes=["tree", "image"],
+ )
+
+ assert payload == {"ok": True, "result": None}
+ assert [tool for tool, _arguments in _tool_requests(requests) if tool in {"get_window_state", "drag"}] == [
+ "get_window_state",
+ "get_window_state",
+ "drag",
+ ]
+
+
+def test_sdk_invalid_screenshot_frame_is_not_reused_for_pixel_input(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ await window.observe({ include_screenshot: true });
+ await window.drag({ from_x: 10, from_y: 20, to_x: 30, to_y: 40 });
+};
+""",
+ observation_screenshot_modes=["invalid", "image"],
+ )
+
+ assert payload == {"ok": True, "result": None}
+ sequence = [
+ (tool, arguments.get("include_screenshot"))
+ for tool, arguments in _tool_requests(requests)
+ if tool in {"get_window_state", "drag"}
+ ]
+ assert sequence == [
+ ("get_window_state", True),
+ ("get_window_state", None),
+ ("drag", None),
+ ]
+
+
[email protected]("observation_mode", ["tree", "invalid"])
+def test_sdk_missing_screenshots_retry_once_per_pixel_action_and_stay_bounded(
+ tmp_path: Path,
+ observation_mode: str,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ await window.observe({ include_screenshot: true });
+ const errors = [];
+ for (const x of [10, 11]) {
+ try {
+ await window.drag({ from_x: x, from_y: 20, to_x: 30, to_y: 40 });
+ } catch (error) {
+ errors.push(error.code);
+ }
+ }
+ return { errors };
+};
+""",
+ observation_screenshot_modes=[observation_mode] * 3,
+ )
+
+ assert payload == {
+ "ok": True,
+ "result": {"errors": ["screenshot_context_missing", "screenshot_context_missing"]},
+ }
+ sequence = [tool for tool, _arguments in _tool_requests(requests) if tool in {"get_window_state", "drag"}]
+ assert sequence == [
+ "get_window_state",
+ "get_window_state",
+ "drag",
+ "get_window_state",
+ "drag",
+ ]
+
+
+def test_sdk_file_screenshot_output_is_reused_for_pixel_input(
+ tmp_path: Path,
+) -> None:
+ payload, _server, requests = _run_node_script(
+ tmp_path,
+ """
+import { computer } from "@avibe/computer";
+
+export default async () => {
+ const mail = await computer.app({ bundleId: "com.apple.mail" });
+ const window = await mail.window({ label: "New Message" });
+ await window.observe({ screenshot_out_file: "/tmp/fixture-window.png" });
+ await window.drag({ from_x: 10, from_y: 20, to_x: 30, to_y: 40 });
+ await window.drag({ from_x: 11, from_y: 21, to_x: 31, to_y: 41 });
+};
+""",
+ observation_screenshot_modes=["file"],
+ )
+
+ assert payload == {"ok": True, "result": None}
+ sequence = [tool for tool, _arguments in _tool_requests(requests) if tool in {"get_window_state", "drag"}]
+ assert sequence == ["get_window_state", "drag", "drag"]
diff --git a/vibe/computer_sdk/index.mjs b/vibe/computer_sdk/index.mjs
index 7659d12bd..fae5c3581 100644
--- a/vibe/computer_sdk/index.mjs
+++ b/vibe/computer_sdk/index.mjs
@@ -240,6 +240,32 @@ function structured(result) {
: {};
}
+function observationHasScreenshot(result) {
+ const payload = structured(result);
+ // The driver may return capture metadata alongside an explicitly invalid
+ // frame. That metadata does not authorize pixel coordinates.
+ if (payload.screenshot_frame_valid === false) {
+ return false;
+ }
+ if (
+ typeof payload.screenshot_file_path === "string" &&
+ payload.screenshot_file_path
+ ) {
+ return true;
+ }
+ return (
+ Array.isArray(result?.content) &&
+ result.content.some(
+ (item) =>
+ item &&
+ item.type === "image" &&
+ typeof item.data === "string" &&
+ item.data.length > 0 &&
+ (!item.mimeType || item.mimeType === "image/png"),
+ )
+ );
+}
+
function effectResult(result) {
const payload = sanitizePublic(structured(result));
const status = EFFECT_STATUSES.has(payload.effect)
@@ -822,15 +848,16 @@ async function observeWindow(handle, force, options = {}, deadline) {
if (!force && state.snapshot && state.snapshotKey === key) {
return state.snapshot;
}
- state.snapshot = await callHandleTool(
+ const observation = await callHandleTool(
handle,
"get_window_state",
windowArguments(handle, options),
deadline,
);
+ state.snapshot = observation;
state.snapshotKey = key;
- state.snapshotHasScreenshot = options.include_screenshot !== false;
- return state.snapshot;
+ state.snapshotHasScreenshot = observationHasScreenshot(observation);
+ return observation;
}
// Element resolution reads only the accessibility tree. Element actions act