Agent Zero AI framework
Fix canvas attachment for browser and documents
Alessandrobb2432693ee6
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.pyindex 039dc7ebe..a061c406a 100644--- a/plugins/_browser/api/ws_browser.py+++ b/plugins/_browser/api/ws_browser.py@@ -92,15 +92,18 @@ class WsBrowser(WsHandler):if existing:existing.cancel()viewer_id = str(data.get("viewer_id") or "")+ snapshot = Noneif runtime:self._streams[stream_key] = asyncio.create_task(self._stream_frames(sid, context_id, active_id, viewer_id))+ snapshot = await self._snapshot_for_browser(runtime, active_id)return {"context_id": context_id,"active_browser_context_id": context_id,"active_browser_id": active_id,+ "snapshot": snapshot,"browsers": await self._all_browser_tabs(),"all_browsers": True,"viewer_id": viewer_id,@@ -337,6 +340,17 @@ class WsBrowser(WsHandler):return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)return None+ async def _snapshot_for_browser(+ self,+ runtime: Any,+ browser_id: int | str | None,+ ) -> dict[str, Any] | None:+ if not browser_id:+ return None+ with contextlib.suppress(Exception):+ return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)+ return None+async def _all_browser_tabs(self) -> list[dict[str, Any]]:browsers: list[dict[str, Any]] = []for session in await list_runtime_sessions():diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex c6d9cbfcd..41c677024 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -973,6 +973,7 @@ const model = {data.active_browser_id || requestedBrowserId || this.activeBrowserId || null,data.active_browser_context_id || contextId,);+ this.applySnapshot(data.snapshot);this.connected = true;this.browserInstallExpected = false;},@@ -1016,11 +1017,9 @@ const model = {this._surfaceSwitching = false;},});- } else {+ } else if (!data.state) {this.cancelFrameRender();- if (!data.state) {- this.frameSrc = "";- }+ this.frameSrc = "";}if (!data.image && !data.state) {if (!this.activeBrowserId) {@@ -1113,7 +1112,9 @@ const model = {const viewport = this.currentViewportSize() || this._lastViewport;if (!this.frameMatchesViewport(dimensions, viewport)) {this.requestViewportSyncAfterRejectedFrame();- return;+ if (!this.shouldAcceptMismatchedFrame(dimensions)) {+ return;+ }}this.frameSrc = frameSrc;this._lastFrameDimensions = dimensions;@@ -1123,6 +1124,14 @@ const model = {this.scheduleCanvasWidthNudgeAfterFirstFrame();},+ shouldAcceptMismatchedFrame(dimensions = null) {+ return Boolean(+ dimensions?.width+ && dimensions?.height+ && (!this.frameSrc || this._surfaceSwitching || this.isSwitchingBrowser())+ );+ },+scheduleCanvasWidthNudgeAfterFirstFrame() {const surfaceSequence = this._surfaceOpenSequence;if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {diff --git a/plugins/_desktop/api/desktop_session.py b/plugins/_desktop/api/desktop_session.pyindex 33dd6b6b0..b1b92b19b 100644--- a/plugins/_desktop/api/desktop_session.py+++ b/plugins/_desktop/api/desktop_session.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom helpers.api import ApiHandler, Requestfrom plugins._desktop.helpers import desktop_session-from plugins._office.helpers import document_store+from plugins._office.helpers import document_store, markdown_sessionsfrom plugins._office.helpers import libreoffice@@ -74,6 +74,8 @@ class DesktopSession(ApiHandler):return {"ok": False, "error": str(exc)}ext = str(doc.get("extension") or "").lower()+ if ext == "md":+ return self._open_markdown(doc, input, request)if ext not in desktop_session.OFFICIAL_EXTENSIONS:return {"ok": False, "error": f".{ext} documents do not use the Desktop surface."}@@ -108,6 +110,26 @@ class DesktopSession(ApiHandler):"mode": "edit",}+ def _open_markdown(self, doc: dict, input: dict, request: Request) -> dict:+ mode = "edit" if str(input.get("mode") or "edit").lower() == "edit" else "view"+ store_session = document_store.create_session(+ doc["file_id"],+ user_id=str(input.get("user_id") or "agent-zero-user"),+ permission="write" if mode == "edit" else "read",+ origin=self._origin(request),+ )+ try:+ editor = markdown_sessions.get_manager().open(doc, sid="")+ except ValueError as exc:+ document_store.close_session(session_id=store_session["session_id"])+ return {"ok": False, "error": str(exc)}+ return {+ **editor,+ "store_session_id": store_session["session_id"],+ "session_id": editor["session_id"],+ "mode": mode,+ }+def _save(self, input: dict) -> dict:session_id = str(input.get("desktop_session_id") or input.get("session_id") or "").strip()if not session_id:diff --git a/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py b/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.pyindex a51f10ede..960484666 100644--- a/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py+++ b/plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py@@ -22,6 +22,8 @@ class DocumentResponseAffordance(Extension):):if not self.agent or response is None:return+ if document_affordance.is_subordinate_agent(self.agent):+ returnif tool_name == "document_artifact":if (response.additional or {}).get("file_id"):diff --git a/plugins/_office/extensions/webui/lib/document-actions.js b/plugins/_office/extensions/webui/lib/document-actions.jsindex f9abba1ba..a1a6dd708 100644--- a/plugins/_office/extensions/webui/lib/document-actions.js+++ b/plugins/_office/extensions/webui/lib/document-actions.js@@ -2,9 +2,7 @@ import {createActionButton,copyToClipboard,} from "/components/messages/action-buttons/simple-action-buttons.js";-import { ensureModalOpen } from "/js/modals.js";import { open as openSurface } from "/js/surfaces.js";-import { store as officeStore } from "/plugins/_office/webui/office-store.js";function basename(path = "") {const value = String(path || "").split("?")[0].split("#")[0];@@ -46,17 +44,7 @@ export async function openDocumentInDesktop(kvps = {}) {}export async function openDocumentArtifact(kvps = {}) {- if (usesDesktop(kvps)) {- await openDocumentInDesktop(kvps);- return;- }- await ensureModalOpen("/plugins/_office/webui/main.html");- await officeStore.openSession?.({- path: kvps.path || "",- file_id: kvps.file_id || "",- refresh: true,- source: "message-action",- });+ await openDocumentInDesktop(kvps);}function usesDesktop(doc = {}) {diff --git a/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js b/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.jsindex 360233cda..6d0237e60 100644--- a/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js+++ b/plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js@@ -1,9 +1,7 @@import { store as officeStore } from "/plugins/_office/webui/office-store.js";-import { ensureModalOpen } from "/js/modals.js";import { open as openSurface } from "/js/surfaces.js";const SYNC_WINDOW_MS = 10 * 60 * 1000;-const DESKTOP_DOCUMENT_EXTENSIONS = new Set(["odt", "ods", "odp", "docx", "xlsx", "pptx"]);const syncedDocumentResults = new Set();export default async function syncDocumentResultsIntoOpenOfficeModal(context) {@@ -112,18 +110,7 @@ function isExplicitDocumentUiRequest(payload = {}) {}async function openDocumentUiFromResult(target = {}, payload = {}, document = {}) {- if (isDesktopDocument(payload, document)) {- await openSurface("desktop", {- path: target.path || "",- file_id: target.file_id || "",- refresh: true,- source: "tool-result-open",- });- return;- }-- await ensureModalOpen("/plugins/_office/webui/main.html");- await officeStore.openSession?.({+ await openSurface("desktop", {path: target.path || "",file_id: target.file_id || "",refresh: true,@@ -131,10 +118,6 @@ async function openDocumentUiFromResult(target = {}, payload = {}, document = {}});}-function isDesktopDocument(payload = {}, document = {}) {- return DESKTOP_DOCUMENT_EXTENSIONS.has(documentExtension(payload, document));-}-function documentExtension(payload = {}, document = {}) {return String(payload.formatdiff --git a/plugins/_office/helpers/document_affordance.py b/plugins/_office/helpers/document_affordance.pyindex b025ec23d..823332cbe 100644--- a/plugins/_office/helpers/document_affordance.py+++ b/plugins/_office/helpers/document_affordance.py@@ -415,3 +415,32 @@ def format_created_response(basename: str, path: str) -> str:f"Created **{basename}**.\n\n"f"Path: `{path}`")+++def is_subordinate_agent(agent: Any) -> bool:+ number = getattr(agent, "number", None)+ if number is not None:+ try:+ return int(number) > 0+ except (TypeError, ValueError):+ pass++ agent_name = str(getattr(agent, "agent_name", "") or "").strip().lower()+ if agent_name.startswith("a") and agent_name[1:].isdigit():+ return int(agent_name[1:]) > 0+ if agent_name.isdigit():+ return int(agent_name) > 0++ get_data = getattr(agent, "get_data", None)+ if callable(get_data):+ try:+ if get_data("_superior") is not None:+ return True+ except Exception:+ pass++ data = getattr(agent, "data", None)+ if isinstance(data, dict) and data.get("_superior") is not None:+ return True++ return Falsediff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex cc2c73269..7db446ac4 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -1091,7 +1091,11 @@ def test_browser_viewer_uses_cdp_screencast_transport():assert "this.frameState = data.state || null" not in browser_storeassert "function loadFrameDimensions(src)" in browser_storeassert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store+ assert "shouldAcceptMismatchedFrame(dimensions = null)" in browser_storeassert "requestViewportSyncAfterRejectedFrame()" in browser_store+ assert "this.applySnapshot(data.snapshot);" in browser_store+ assert "else if (!data.state)" in browser_store+ assert '"snapshot": snapshot' in ws_browserassert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browserassert '"frame_source": "state"' in ws_browserassert '"frame_source"] = "screencast"' in ws_browser@@ -1830,6 +1834,64 @@ async def test_browser_viewer_subscribe_can_create_blank_tab_when_requested(monkawait handler.on_disconnect("sid-create")+async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):+ calls = []++ class FakeRuntime:+ async def call(self, method, *args, **kwargs):+ calls.append((method, args, kwargs))+ if method == "list":+ return {+ "browsers": [{"id": 1, "context_id": "ctx", "currentUrl": "https://example.com/"}],+ "last_interacted_browser_id": 1,+ }+ if method == "set_viewport":+ return {"state": {"id": args[0], "currentUrl": "https://example.com/"}}+ if method == "screenshot":+ return {+ "browser_id": args[0],+ "mime": "image/jpeg",+ "image": "jpeg-data",+ "state": {"id": args[0], "context_id": "ctx", "currentUrl": "https://example.com/"},+ }+ raise AssertionError(method)++ async def fake_get_runtime(context_id, create=True):+ assert context_id == "ctx"+ assert create is False+ return FakeRuntime()++ async def fake_all_browser_tabs():+ return [{"id": 1, "context_id": "ctx", "currentUrl": "https://example.com/"}]++ monkeypatch.setattr(ws_browser_module, "get_runtime", fake_get_runtime)+ monkeypatch.setattr(ws_browser_module, "list_runtime_sessions", fake_all_browser_tabs)+ monkeypatch.setattr(+ ws_browser_module.AgentContext,+ "get",+ staticmethod(lambda context_id: SimpleNamespace(id=context_id)),+ )++ handler = ws_browser_module.WsBrowser(+ SimpleNamespace(),+ threading.RLock(),+ manager=None,+ )++ result = await handler.process(+ "browser_viewer_subscribe",+ {"context_id": "ctx", "browser_id": 1, "viewport_width": 900, "viewport_height": 600},+ "sid-snapshot",+ )++ assert result["active_browser_id"] == 1+ assert result["snapshot"]["image"] == "jpeg-data"+ assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls++ await handler.on_disconnect("sid-snapshot")++@pytest.mark.anyioasync def test_browser_viewer_subscribe_without_runtime_does_not_create_runtime(monkeypatch):async def fake_get_runtime(context_id, create=True):diff --git a/tests/test_office_canvas_setup.py b/tests/test_office_canvas_setup.pyindex d676a67f1..372228680 100644--- a/tests/test_office_canvas_setup.py+++ b/tests/test_office_canvas_setup.py@@ -166,6 +166,9 @@ def test_desktop_plugin_owns_routes_runtime_surface_and_state_paths():assert "virtual_desktop_routes.install_route_hooks()" in desktop_startupassert 'action in {"open_document", "document"}' in desktop_api+ assert "markdown_sessions" in desktop_api+ assert 'if ext == "md":' in desktop_api+ assert "return self._open_markdown(doc, input, request)" in desktop_apiassert '"status": desktop.get("status") or {}' in desktop_apiassert 'callJsonApi("/plugins/_desktop/desktop_session"' in desktop_storeassert 'callDesktop("open_document"' in desktop_store@@ -233,7 +236,9 @@ def test_document_artifacts_only_open_desktop_from_explicit_document_ui_requestsassert "officeStore" in auto_openassert "openDocumentInDesktop" in document_actionsassert "openDocumentArtifact" in document_actions- assert "ensureModalOpen" in document_actions+ assert "await openDocumentInDesktop(kvps);" in document_actions+ assert 'ensureModalOpen("/plugins/_office/webui/main.html")' not in document_actions+ assert 'ensureModalOpen("/plugins/_office/webui/main.html")' not in auto_openassert "Open Document" in document_actionsassert 'openSurface("desktop"' in document_actionsassert "Edit in Writer" in document_actionsdiff --git a/tests/test_office_document_affordance.py b/tests/test_office_document_affordance.pyindex 24234d05d..20d511a92 100644--- a/tests/test_office_document_affordance.py+++ b/tests/test_office_document_affordance.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport sysfrom pathlib import Path+from types import SimpleNamespacePROJECT_ROOT = Path(__file__).resolve().parents[1]@@ -163,3 +164,11 @@ def test_created_response_does_not_claim_canvas_was_opened():assert "Created **Project Brief.md**." in messageassert "opened" not in message.lower()assert "Path: `/a0/usr/workdir/Project Brief.md`" in message+++def test_document_response_affordance_only_runs_for_primary_agent():+ assert document_affordance.is_subordinate_agent(SimpleNamespace(number=0, agent_name="A0")) is False+ assert document_affordance.is_subordinate_agent(SimpleNamespace(number=1, agent_name="A1")) is True+ assert document_affordance.is_subordinate_agent(SimpleNamespace(agent_name="A2")) is True+ assert document_affordance.is_subordinate_agent(SimpleNamespace(agent_name="0")) is False+ assert document_affordance.is_subordinate_agent(SimpleNamespace(data={"_superior": object()})) is True