Skip to content

Agent Zero AI framework

Fix canvas attachment for browser and documents

Alessandrobb2432693ee6
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index 039dc7ebe..a061c406a 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -92,15 +92,18 @@ class WsBrowser(WsHandler):
if existing:
existing.cancel()
viewer_id = str(data.get("viewer_id") or "")
+ snapshot = None
if runtime:
self._streams[stream_key] = asyncio.create_task(
self._stream_frames(sid, context_id, active_id, viewer_id)
)
+ snapshot = await self._snapshot_for_browser(runtime, active_id)
return {
"context_id": context_id,
"active_browser_context_id": context_id,
"active_browser_id": active_id,
+ "snapshot": snapshot,
"browsers": await self._all_browser_tabs(),
"all_browsers": True,
"viewer_id": viewer_id,
@@ -337,6 +340,17 @@ class WsBrowser(WsHandler):
return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
return None
+ async def _snapshot_for_browser(
+ self,
+ runtime: Any,
+ browser_id: int | str | None,
+ ) -> dict[str, Any] | None:
+ if not browser_id:
+ return None
+ with contextlib.suppress(Exception):
+ return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
+ return None
+
async def _all_browser_tabs(self) -> list[dict[str, Any]]:
browsers: list[dict[str, Any]] = []
for session in await list_runtime_sessions():
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index c6d9cbfcd..41c677024 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -973,6 +973,7 @@ const model = {
data.active_browser_id || requestedBrowserId || this.activeBrowserId || null,
data.active_browser_context_id || contextId,
);
+ this.applySnapshot(data.snapshot);
this.connected = true;
this.browserInstallExpected = false;
},
@@ -1016,11 +1017,9 @@ const model = {
this._surfaceSwitching = false;
},
});
- } else {
+ } else if (!data.state) {
this.cancelFrameRender();
- if (!data.state) {
- this.frameSrc = "";
- }
+ this.frameSrc = "";
}
if (!data.image && !data.state) {
if (!this.activeBrowserId) {
@@ -1113,7 +1112,9 @@ const model = {
const viewport = this.currentViewportSize() || this._lastViewport;
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
- return;
+ if (!this.shouldAcceptMismatchedFrame(dimensions)) {
+ return;
+ }
}
this.frameSrc = frameSrc;
this._lastFrameDimensions = dimensions;
@@ -1123,6 +1124,14 @@ const model = {
this.scheduleCanvasWidthNudgeAfterFirstFrame();
},
+ shouldAcceptMismatchedFrame(dimensions = null) {
+ return Boolean(
+ dimensions?.width
+ && dimensions?.height
+ && (!this.frameSrc || this._surfaceSwitching || this.isSwitchingBrowser())
+ );
+ },
+
scheduleCanvasWidthNudgeAfterFirstFrame() {
const surfaceSequence = this._surfaceOpenSequence;
if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
diff --git a/plugins/_desktop/api/desktop_session.py b/plugins/_desktop/api/desktop_session.py
index 33dd6b6b0..b1b92b19b 100644
--- a/plugins/_desktop/api/desktop_session.py
+++ b/plugins/_desktop/api/desktop_session.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from helpers.api import ApiHandler, Request
from plugins._desktop.helpers import desktop_session
-from plugins._office.helpers import document_store
+from plugins._office.helpers import document_store, markdown_sessions
from plugins._office.helpers import libreoffice
@@ -74,6 +74,8 @@ class DesktopSession(ApiHandler):
return {"ok": False, "error": str(exc)}
ext = str(doc.get("extension") or "").lower()
+ if ext == "md":
+ return self._open_markdown(doc, input, request)
if ext not in desktop_session.OFFICIAL_EXTENSIONS:
return {"ok": False, "error": f".{ext} documents do not use the Desktop surface."}
@@ -108,6 +110,26 @@ class DesktopSession(ApiHandler):
"mode": "edit",
}
+ def _open_markdown(self, doc: dict, input: dict, request: Request) -> dict:
+ mode = "edit" if str(input.get("mode") or "edit").lower() == "edit" else "view"
+ store_session = document_store.create_session(
+ doc["file_id"],
+ user_id=str(input.get("user_id") or "agent-zero-user"),
+ permission="write" if mode == "edit" else "read",
+ origin=self._origin(request),
+ )
+ try:
+ editor = markdown_sessions.get_manager().open(doc, sid="")
+ except ValueError as exc:
+ document_store.close_session(session_id=store_session["session_id"])
+ return {"ok": False, "error": str(exc)}
+ return {
+ **editor,
+ "store_session_id": store_session["session_id"],
+ "session_id": editor["session_id"],
+ "mode": mode,
+ }
+
def _save(self, input: dict) -> dict:
session_id = str(input.get("desktop_session_id") or input.get("session_id") or "").strip()
if not session_id:
diff --git a/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py b/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py
index a51f10ede..960484666 100644
--- a/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py
+++ b/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py
@@ -22,6 +22,8 @@ class DocumentResponseAffordance(Extension):
):
if not self.agent or response is None:
return
+ if document_affordance.is_subordinate_agent(self.agent):
+ return
if tool_name == "document_artifact":
if (response.additional or {}).get("file_id"):
diff --git a/plugins/_office/extensions/webui/lib/document-actions.js b/plugins/_office/extensions/webui/lib/document-actions.js
index f9abba1ba..a1a6dd708 100644
--- a/plugins/_office/extensions/webui/lib/document-actions.js
+++ b/plugins/_office/extensions/webui/lib/document-actions.js
@@ -2,9 +2,7 @@ import {
createActionButton,
copyToClipboard,
} from "/components/messages/action-buttons/simple-action-buttons.js";
-import { ensureModalOpen } from "/js/modals.js";
import { open as openSurface } from "/js/surfaces.js";
-import { store as officeStore } from "/plugins/_office/webui/office-store.js";
function basename(path = "") {
const value = String(path || "").split("?")[0].split("#")[0];
@@ -46,17 +44,7 @@ export async function openDocumentInDesktop(kvps = {}) {
}
export async function openDocumentArtifact(kvps = {}) {
- if (usesDesktop(kvps)) {
- await openDocumentInDesktop(kvps);
- return;
- }
- await ensureModalOpen("/plugins/_office/webui/main.html");
- await officeStore.openSession?.({
- path: kvps.path || "",
- file_id: kvps.file_id || "",
- refresh: true,
- source: "message-action",
- });
+ await openDocumentInDesktop(kvps);
}
function usesDesktop(doc = {}) {
diff --git a/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js b/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js
index 360233cda..6d0237e60 100644
--- a/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js
+++ b/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js
@@ -1,9 +1,7 @@
import { store as officeStore } from "/plugins/_office/webui/office-store.js";
-import { ensureModalOpen } from "/js/modals.js";
import { open as openSurface } from "/js/surfaces.js";
const SYNC_WINDOW_MS = 10 * 60 * 1000;
-const DESKTOP_DOCUMENT_EXTENSIONS = new Set(["odt", "ods", "odp", "docx", "xlsx", "pptx"]);
const syncedDocumentResults = new Set();
export default async function syncDocumentResultsIntoOpenOfficeModal(context) {
@@ -112,18 +110,7 @@ function isExplicitDocumentUiRequest(payload = {}) {
}
async function openDocumentUiFromResult(target = {}, payload = {}, document = {}) {
- if (isDesktopDocument(payload, document)) {
- await openSurface("desktop", {
- path: target.path || "",
- file_id: target.file_id || "",
- refresh: true,
- source: "tool-result-open",
- });
- return;
- }
-
- await ensureModalOpen("/plugins/_office/webui/main.html");
- await officeStore.openSession?.({
+ await openSurface("desktop", {
path: target.path || "",
file_id: target.file_id || "",
refresh: true,
@@ -131,10 +118,6 @@ async function openDocumentUiFromResult(target = {}, payload = {}, document = {}
});
}
-function isDesktopDocument(payload = {}, document = {}) {
- return DESKTOP_DOCUMENT_EXTENSIONS.has(documentExtension(payload, document));
-}
-
function documentExtension(payload = {}, document = {}) {
return String(
payload.format
diff --git a/plugins/_office/helpers/document_affordance.py b/plugins/_office/helpers/document_affordance.py
index b025ec23d..823332cbe 100644
--- a/plugins/_office/helpers/document_affordance.py
+++ b/plugins/_office/helpers/document_affordance.py
@@ -415,3 +415,32 @@ def format_created_response(basename: str, path: str) -> str:
f"Created **{basename}**.\n\n"
f"Path: `{path}`"
)
+
+
+def is_subordinate_agent(agent: Any) -> bool:
+ number = getattr(agent, "number", None)
+ if number is not None:
+ try:
+ return int(number) > 0
+ except (TypeError, ValueError):
+ pass
+
+ agent_name = str(getattr(agent, "agent_name", "") or "").strip().lower()
+ if agent_name.startswith("a") and agent_name[1:].isdigit():
+ return int(agent_name[1:]) > 0
+ if agent_name.isdigit():
+ return int(agent_name) > 0
+
+ get_data = getattr(agent, "get_data", None)
+ if callable(get_data):
+ try:
+ if get_data("_superior") is not None:
+ return True
+ except Exception:
+ pass
+
+ data = getattr(agent, "data", None)
+ if isinstance(data, dict) and data.get("_superior") is not None:
+ return True
+
+ return False
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index cc2c73269..7db446ac4 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1091,7 +1091,11 @@ def test_browser_viewer_uses_cdp_screencast_transport():
assert "this.frameState = data.state || null" not in browser_store
assert "function loadFrameDimensions(src)" in browser_store
assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
+ assert "shouldAcceptMismatchedFrame(dimensions = null)" in browser_store
assert "requestViewportSyncAfterRejectedFrame()" in browser_store
+ assert "this.applySnapshot(data.snapshot);" in browser_store
+ assert "else if (!data.state)" in browser_store
+ assert '"snapshot": snapshot' in ws_browser
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
assert '"frame_source": "state"' in ws_browser
assert '"frame_source"] = "screencast"' in ws_browser
@@ -1830,6 +1834,64 @@ async def test_browser_viewer_subscribe_can_create_blank_tab_when_requested(monk
await handler.on_disconnect("sid-create")
+async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):
+ calls = []
+
+ class FakeRuntime:
+ async def call(self, method, *args, **kwargs):
+ calls.append((method, args, kwargs))
+ if method == "list":
+ return {
+ "browsers": [{"id": 1, "context_id": "ctx", "currentUrl": "https://example.com/"}],
+ "last_interacted_browser_id": 1,
+ }
+ if method == "set_viewport":
+ return {"state": {"id": args[0], "currentUrl": "https://example.com/"}}
+ if method == "screenshot":
+ return {
+ "browser_id": args[0],
+ "mime": "image/jpeg",
+ "image": "jpeg-data",
+ "state": {"id": args[0], "context_id": "ctx", "currentUrl": "https://example.com/"},
+ }
+ raise AssertionError(method)
+
+ async def fake_get_runtime(context_id, create=True):
+ assert context_id == "ctx"
+ assert create is False
+ return FakeRuntime()
+
+ async def fake_all_browser_tabs():
+ return [{"id": 1, "context_id": "ctx", "currentUrl": "https://example.com/"}]
+
+ monkeypatch.setattr(ws_browser_module, "get_runtime", fake_get_runtime)
+ monkeypatch.setattr(ws_browser_module, "list_runtime_sessions", fake_all_browser_tabs)
+ monkeypatch.setattr(
+ ws_browser_module.AgentContext,
+ "get",
+ staticmethod(lambda context_id: SimpleNamespace(id=context_id)),
+ )
+
+ handler = ws_browser_module.WsBrowser(
+ SimpleNamespace(),
+ threading.RLock(),
+ manager=None,
+ )
+
+ result = await handler.process(
+ "browser_viewer_subscribe",
+ {"context_id": "ctx", "browser_id": 1, "viewport_width": 900, "viewport_height": 600},
+ "sid-snapshot",
+ )
+
+ assert result["active_browser_id"] == 1
+ assert result["snapshot"]["image"] == "jpeg-data"
+ assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls
+
+ await handler.on_disconnect("sid-snapshot")
+
+
@pytest.mark.anyio
async def test_browser_viewer_subscribe_without_runtime_does_not_create_runtime(monkeypatch):
async def fake_get_runtime(context_id, create=True):
diff --git a/tests/test_office_canvas_setup.py b/tests/test_office_canvas_setup.py
index d676a67f1..372228680 100644
--- a/tests/test_office_canvas_setup.py
+++ b/tests/test_office_canvas_setup.py
@@ -166,6 +166,9 @@ def test_desktop_plugin_owns_routes_runtime_surface_and_state_paths():
assert "virtual_desktop_routes.install_route_hooks()" in desktop_startup
assert 'action in {"open_document", "document"}' in desktop_api
+ assert "markdown_sessions" in desktop_api
+ assert 'if ext == "md":' in desktop_api
+ assert "return self._open_markdown(doc, input, request)" in desktop_api
assert '"status": desktop.get("status") or {}' in desktop_api
assert 'callJsonApi("/plugins/_desktop/desktop_session"' in desktop_store
assert 'callDesktop("open_document"' in desktop_store
@@ -233,7 +236,9 @@ def test_document_artifacts_only_open_desktop_from_explicit_document_ui_requests
assert "officeStore" in auto_open
assert "openDocumentInDesktop" in document_actions
assert "openDocumentArtifact" in document_actions
- assert "ensureModalOpen" in document_actions
+ assert "await openDocumentInDesktop(kvps);" in document_actions
+ assert 'ensureModalOpen("/plugins/_office/webui/main.html")' not in document_actions
+ assert 'ensureModalOpen("/plugins/_office/webui/main.html")' not in auto_open
assert "Open Document" in document_actions
assert 'openSurface("desktop"' in document_actions
assert "Edit in Writer" in document_actions
diff --git a/tests/test_office_document_affordance.py b/tests/test_office_document_affordance.py
index 24234d05d..20d511a92 100644
--- a/tests/test_office_document_affordance.py
+++ b/tests/test_office_document_affordance.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import sys
from pathlib import Path
+from types import SimpleNamespace
PROJECT_ROOT = Path(__file__).resolve().parents[1]
@@ -163,3 +164,11 @@ def test_created_response_does_not_claim_canvas_was_opened():
assert "Created **Project Brief.md**." in message
assert "opened" not in message.lower()
assert "Path: `/a0/usr/workdir/Project Brief.md`" in message
+
+
+def test_document_response_affordance_only_runs_for_primary_agent():
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(number=0, agent_name="A0")) is False
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(number=1, agent_name="A1")) is True
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(agent_name="A2")) is True
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(agent_name="0")) is False
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(data={"_superior": object()})) is True