Agent Zero AI framework
Add browser annotate mode
Alessandro4ff3244ce63e
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.pyindex b1df9146b..8430f7e7a 100644--- a/plugins/_browser/api/ws_browser.py+++ b/plugins/_browser/api/ws_browser.py@@ -43,6 +43,8 @@ class WsBrowser(WsHandler):return await self._command(data, sid)if event == "browser_viewer_input":return await self._input(data, sid)+ if event == "browser_viewer_annotation":+ return await self._annotation(data, sid)return WsResult.error(code="UNKNOWN_BROWSER_EVENT",@@ -215,6 +217,29 @@ class WsBrowser(WsHandler):else None,}+ async def _annotation(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:+ context_id = self._context_id(data)+ if not context_id:+ return self._error("MISSING_CONTEXT", "context_id is required", data)+ runtime = await get_runtime(context_id, create=False)+ if not runtime:+ return self._error("NO_BROWSER_RUNTIME", "No browser runtime exists for this context", data)++ browser_id = data.get("browser_id")+ viewer_id = str(data.get("viewer_id") or "")+ payload = data.get("payload") if isinstance(data.get("payload"), dict) else {}+ try:+ annotation = await runtime.call("annotation_target", browser_id, payload)+ except Exception as exc:+ return self._error("ANNOTATION_FAILED", str(exc), data)++ return {+ "annotation": annotation,+ "context_id": context_id,+ "browser_id": browser_id,+ "viewer_id": viewer_id,+ }+async def _snapshot_for_result(self,runtime: Any,diff --git a/plugins/_browser/assets/browser-page-content.js b/plugins/_browser/assets/browser-page-content.jsindex fec76e633..edf9d9403 100644--- a/plugins/_browser/assets/browser-page-content.js+++ b/plugins/_browser/assets/browser-page-content.js@@ -1,7 +1,7 @@(() => {const GLOBAL_KEY = "__spaceBrowserPageContent__";const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";- const VERSION = "6";+ const VERSION = "7";const BLOCK_TAGS = new Set(["ADDRESS","ARTICLE",@@ -2842,10 +2842,377 @@});}+ function cssEscape(value) {+ const rawValue = String(value || "");+ if (!rawValue) {+ return "";+ }++ if (typeof globalThis.CSS?.escape === "function") {+ return globalThis.CSS.escape(rawValue);+ }++ return rawValue.replace(/[^a-zA-Z0-9_-]/gu, (character) => `\\${character}`);+ }++ function getClassSummary(element) {+ try {+ return [...(element?.classList || [])]+ .map((className) => normalizeAttributeText(className))+ .filter(Boolean)+ .slice(0, 4)+ .join(" ");+ } catch {+ return "";+ }+ }++ function buildCssSelector(element) {+ if (!isElementNode(element)) {+ return "";+ }++ const id = normalizeAttributeText(element.getAttribute?.("id"));+ if (id) {+ return `#${cssEscape(id)}`;+ }++ const parts = [];+ let current = element;+ while (isElementNode(current) && current !== globalThis.document?.documentElement && parts.length < 6) {+ const tagName = getTagName(current).toLowerCase();+ if (!tagName) {+ break;+ }++ let part = tagName;+ const classes = getClassSummary(current)+ .split(/\s+/u)+ .filter(Boolean)+ .slice(0, 2);+ if (classes.length && !["body", "html"].includes(tagName)) {+ part += classes.map((className) => `.${cssEscape(className)}`).join("");+ }++ const parent = current.parentElement;+ if (parent) {+ const siblings = [...parent.children].filter((sibling) => getTagName(sibling) === getTagName(current));+ if (siblings.length > 1) {+ part += `:nth-of-type(${siblings.indexOf(current) + 1})`;+ }+ }++ parts.unshift(part);+ if (tagName === "body") {+ break;+ }+ current = parent;+ }++ return parts.join(" > ");+ }++ function sanitizeAnnotationDom(value) {+ return truncateText(+ String(value || "")+ .replace(/(<input\b(?=[^>]*\btype\s*=\s*(["'])?password\2?)[^>]*?)\s+value\s*=\s*(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")+ .replace(/\svalue\s*=\s*(["'])[\s\S]{0,600}?\1/giu, " value=\"[redacted]\"")+ .replace(/\sdata-space-browser-live-value\s*=\s*(["'])[\s\S]{0,600}?\1/giu, "")+ .replace(/\sdata-space-browser-selected-text\s*=\s*(["'])[\s\S]{0,600}?\1/giu, ""),+ 1200+ );+ }++ function summarizeAnnotationElement(element) {+ if (!isElementNode(element)) {+ return null;+ }++ const summaryData = collectReferenceSummaryData(element, {+ includeLabelQuotes: false,+ includeLinkUrls: true,+ includeSemanticTags: true,+ includeStateTags: true+ });+ const rawDom = serializeElementSnapshot(element);+ return {+ classes: getClassSummary(element),+ dom: sanitizeAnnotationDom(rawDom),+ id: normalizeAttributeText(element.getAttribute?.("id")),+ kind: summaryData.kind,+ name: normalizeAttributeText(element.getAttribute?.("name")),+ rect: getElementRectSafe(element),+ role: normalizeAttributeText(element.getAttribute?.("role")).toLowerCase(),+ selector: buildCssSelector(element),+ semanticTags: Array.isArray(summaryData.semanticTags) ? summaryData.semanticTags.slice(0, 4) : [],+ stateTags: Array.isArray(summaryData.state?.stateTags) ? summaryData.state.stateTags.slice(0, 8) : [],+ summary: truncateText(summaryData.summary || getLabelText(element, {+ includeAlt: true,+ includeDescendantImageAlt: true,+ includePlaceholder: true,+ includeText: true+ }), 240),+ tagName: getTagName(element)+ };+ }++ function annotationViewport() {+ return {+ height: Math.max(0, Number(globalThis.innerHeight || globalThis.document?.documentElement?.clientHeight || 0)),+ scrollX: Number(globalThis.scrollX || globalThis.pageXOffset || 0),+ scrollY: Number(globalThis.scrollY || globalThis.pageYOffset || 0),+ width: Math.max(0, Number(globalThis.innerWidth || globalThis.document?.documentElement?.clientWidth || 0))+ };+ }++ function normalizeAnnotationPoint(payload = {}, viewport = annotationViewport()) {+ const source = payload?.point && typeof payload.point === "object" ? payload.point : payload;+ const width = Math.max(1, Number(viewport.width || 1));+ const height = Math.max(1, Number(viewport.height || 1));+ return {+ x: Math.max(0, Math.min(width, Number(source?.x || 0))),+ y: Math.max(0, Math.min(height, Number(source?.y || 0)))+ };+ }++ function normalizeAnnotationRectPayload(payload = {}, viewport = annotationViewport()) {+ const source = payload?.rect && typeof payload.rect === "object" ? payload.rect : payload;+ const width = Math.max(1, Number(viewport.width || 1));+ const height = Math.max(1, Number(viewport.height || 1));+ const x = Math.max(0, Math.min(width, Number(source?.x || 0)));+ const y = Math.max(0, Math.min(height, Number(source?.y || 0)));+ return {+ height: Math.max(1, Math.min(height - y, Number(source?.height || source?.h || 1))),+ width: Math.max(1, Math.min(width - x, Number(source?.width || source?.w || 1))),+ x,+ y+ };+ }++ function intersectRects(leftRect, rightRect) {+ if (!leftRect || !rightRect) {+ return null;+ }++ const x = Math.max(Number(leftRect.x || 0), Number(rightRect.x || 0));+ const y = Math.max(Number(leftRect.y || 0), Number(rightRect.y || 0));+ const right = Math.min(+ Number(leftRect.x || 0) + Number(leftRect.width || 0),+ Number(rightRect.x || 0) + Number(rightRect.width || 0)+ );+ const bottom = Math.min(+ Number(leftRect.y || 0) + Number(leftRect.height || 0),+ Number(rightRect.y || 0) + Number(rightRect.height || 0)+ );+ const width = right - x;+ const height = bottom - y;+ if (width <= 0 || height <= 0) {+ return null;+ }+ return {+ area: width * height,+ height,+ width,+ x,+ y+ };+ }++ function deepElementFromPoint(x, y) {+ let element = null;+ try {+ element = globalThis.document?.elementFromPoint?.(x, y) || null;+ } catch {+ return null;+ }++ let guard = 0;+ while (isElementNode(element) && element.shadowRoot && guard < 8) {+ guard += 1;+ try {+ const nestedElement = element.shadowRoot.elementFromPoint?.(x, y);+ if (!nestedElement || nestedElement === element) {+ break;+ }+ element = nestedElement;+ } catch {+ break;+ }+ }++ return element;+ }++ function findAnnotationTarget(element) {+ if (!isElementNode(element)) {+ return null;+ }++ const selector = [+ "a[href]",+ "button",+ "input",+ "textarea",+ "select",+ "summary",+ "[role]",+ "img",+ "label",+ "form",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ "p",+ "li",+ "td",+ "th",+ "article",+ "section",+ "nav",+ "header",+ "main",+ "footer"+ ].join(",");+ const target = element.closest?.(selector) || element;+ return isElementNode(target) && !isHiddenElement(target) ? target : element;+ }++ function isMeaningfulAnnotationElement(element) {+ if (!isElementNode(element) || isHiddenElement(element)) {+ return false;+ }++ if (isInteractiveElement(element) || getTagName(element) === "IMG") {+ return true;+ }++ const tagName = getTagName(element);+ const role = normalizeAttributeText(element.getAttribute?.("role")).toLowerCase();+ return Boolean(+ role+ || /^H[1-6]$/u.test(tagName)+ || ["ARTICLE", "SECTION", "MAIN", "NAV", "HEADER", "FOOTER", "FORM", "LABEL", "P", "LI", "TD", "TH"].includes(tagName)+ );+ }++ function collectIntersectingAnnotationElements(rect) {+ const selector = [+ "a[href]",+ "button",+ "input",+ "textarea",+ "select",+ "summary",+ "[role]",+ "img",+ "label",+ "form",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ "p",+ "li",+ "td",+ "th",+ "article",+ "section",+ "main",+ "nav",+ "header",+ "footer"+ ].join(",");+ let candidates = [];+ try {+ candidates = [...(globalThis.document?.querySelectorAll?.(selector) || [])];+ } catch {+ candidates = [];+ }++ const seen = new Set();+ return candidates+ .map((element) => {+ if (!isMeaningfulAnnotationElement(element) || seen.has(element)) {+ return null;+ }+ seen.add(element);+ const elementRect = getElementRectSafe(element);+ const intersection = intersectRects(rect, elementRect);+ if (!intersection || intersection.area < 48) {+ return null;+ }+ return {+ element,+ elementArea: Math.max(1, Number(elementRect.width || 0) * Number(elementRect.height || 0)),+ intersection+ };+ })+ .filter(Boolean)+ .sort((left, right) => {+ if (right.intersection.area !== left.intersection.area) {+ return right.intersection.area - left.intersection.area;+ }+ return left.elementArea - right.elementArea;+ })+ .slice(0, 12)+ .map((entry) => summarizeAnnotationElement(entry.element))+ .filter(Boolean);+ }++ function annotate(payload = null) {+ const request = payload && typeof payload === "object" ? payload : {};+ const viewport = annotationViewport();+ const kind = request.kind === "area" || request.rect ? "area" : "element";++ if (kind === "area") {+ const rect = normalizeAnnotationRectPayload(request, viewport);+ const point = {+ x: rect.x + rect.width / 2,+ y: rect.y + rect.height / 2+ };+ const elements = collectIntersectingAnnotationElements(rect);+ const fallbackElement = findAnnotationTarget(deepElementFromPoint(point.x, point.y));+ const fallbackTarget = fallbackElement ? summarizeAnnotationElement(fallbackElement) : null;+ return {+ elements,+ kind,+ point,+ rect,+ status: elements.length || fallbackTarget ? "ok" : "empty",+ target: elements[0] || fallbackTarget,+ viewport+ };+ }++ const point = normalizeAnnotationPoint(request, viewport);+ const rawElement = deepElementFromPoint(point.x, point.y);+ const targetElement = findAnnotationTarget(rawElement);+ const target = targetElement ? summarizeAnnotationElement(targetElement) : null;+ return {+ kind,+ point,+ rect: target?.rect || {+ height: 1,+ width: 1,+ x: point.x,+ y: point.y+ },+ status: target ? "ok" : "empty",+ target,+ viewport+ };+ }+globalThis[GLOBAL_KEY] = {click(referenceId) {return activateElement(referenceId);},+ annotate,capture,clear() {state.captureId = 0;diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 036d1c715..db459de91 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -557,6 +557,22 @@ class _BrowserRuntimeCore:self.last_interacted_browser_id = resolved_idreturn result or {}+ async def annotation_target(+ self,+ browser_id: int | str | None,+ payload: dict[str, Any] | None = None,+ ) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ await self._ensure_content_helper(page)+ result = await page.evaluate(+ "(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)",+ payload or None,+ )+ self.last_interacted_browser_id = resolved_id+ return result or {}+async def evaluate(self, browser_id: int | str | None, script: str) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)@@ -918,7 +934,7 @@ class _BrowserRuntimeCore:async def _ensure_content_helper(self, page: Any) -> None:has_helper = await page.evaluate(- "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture)"+ "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate)")if has_helper:returndiff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.htmlindex e25808403..108b9c966 100644--- a/plugins/_browser/webui/browser-panel.html+++ b/plugins/_browser/webui/browser-panel.html@@ -11,7 +11,7 @@<div x-data><template x-if="$store.browserPage"><div class="browser-panel" x-create="$store.browserPage.onOpen($el, xAttrs($el) || {})" x-destroy="$store.browserPage.cleanup()"- @keydown.window="$store.browserPage.sendKey($event)">+ @keydown.window="$store.browserPage.handleKeydown($event)"><div class="browser-meta"><div class="browser-meta-top"><div class="browser-session-tabs" role="tablist" aria-label="Browser sessions">@@ -41,6 +41,15 @@</div><div class="browser-session-controls">+ <button type="button" class="btn browser-annotate-toggle" title="Annotate" aria-label="Annotate"+ :aria-pressed="$store.browserPage.annotating.toString()"+ :class="{ 'is-active': $store.browserPage.annotating }"+ :disabled="!$store.browserPage.canAnnotate() && !$store.browserPage.annotating"+ @click="$store.browserPage.toggleAnnotationMode()">+ <span class="material-symbols-outlined" aria-hidden="true"+ x-text="$store.browserPage.annotating ? 'rate_review' : 'edit_note'"></span>+ <span x-text="$store.browserPage.annotating ? 'Annotating' : 'Annotate'"></span>+ </button><div class="browser-extension-menu" @click.outside="$store.browserPage.closeExtensionsMenu()"@keydown.escape.window="$store.browserPage.closeExtensionsMenu()"><button type="button" class="btn btn-icon-action browser-extensions" title="Browser settings"@@ -161,12 +170,82 @@</div><div class="browser-stage" tabindex="0" @click="$el.focus()"- @wheel.prevent="$store.browserPage.sendWheel($event)">+ :class="{ 'is-annotating': $store.browserPage.annotating }"+ @wheel.prevent="$store.browserPage.handleStageWheel($event)"><template x-if="$store.browserPage.frameSrc"><img class="browser-frame" :src="$store.browserPage.frameSrc"@click="$store.browserPage.sendMouse('click', $event)"@mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" /></template>+ <template x-if="$store.browserPage.annotating && $store.browserPage.frameSrc">+ <div class="browser-annotation-layer"+ :class="{ 'is-busy': $store.browserPage.annotationBusy }"+ @pointerdown.stop.prevent="$store.browserPage.startAnnotationSelection($event)"+ @pointermove.stop.prevent="$store.browserPage.moveAnnotationSelection($event)"+ @pointerup.stop.prevent="$store.browserPage.finishAnnotationSelection($event)"+ @pointercancel.stop.prevent="$store.browserPage.cancelAnnotationSelection($event)">+ <template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">+ <div class="browser-annotation-box is-saved"+ :style="$store.browserPage.annotationBoxStyle(annotation.rect)">+ <span class="browser-annotation-number" x-text="annotation.index"></span>+ </div>+ </template>+ <template x-if="$store.browserPage.annotationDragRect">+ <div class="browser-annotation-box is-draft"+ :style="$store.browserPage.annotationBoxStyle($store.browserPage.annotationDragRect)">+ </div>+ </template>+ </div>+ </template>+ <template x-if="$store.browserPage.annotationDraft">+ <div class="browser-annotation-popover"+ :style="$store.browserPage.annotationPopoverStyle()"+ @click.stop @pointerdown.stop @keydown.stop>+ <div class="browser-annotation-popover-title">+ <span class="browser-annotation-number" x-text="$store.browserPage.nextAnnotationIndex()"></span>+ <span x-text="$store.browserPage.annotationDraftTitle()"></span>+ </div>+ <textarea x-model="$store.browserPage.annotationDraftText" placeholder="Comment"+ maxlength="1200"></textarea>+ <div class="browser-annotation-actions">+ <button type="button" class="btn btn-field"+ @click="$store.browserPage.cancelAnnotationDraft()">Cancel</button>+ <button type="button" class="btn btn-ok"+ :disabled="!String($store.browserPage.annotationDraftText || '').trim()"+ @click="$store.browserPage.addAnnotationComment()">Add</button>+ </div>+ </div>+ </template>+ <div class="browser-annotation-tray"+ x-show="$store.browserPage.visibleAnnotations().length"+ x-transition style="display: none;"+ @click.stop @pointerdown.stop @keydown.stop>+ <div class="browser-annotation-tray-header">+ <span>Annotations</span>+ <button type="button" class="browser-annotation-clear" title="Clear annotations"+ aria-label="Clear annotations" @click="$store.browserPage.clearVisibleAnnotations()">+ <span class="material-symbols-outlined">delete</span>+ </button>+ </div>+ <div class="browser-annotation-chips">+ <template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">+ <div class="browser-annotation-chip">+ <span class="browser-annotation-number" x-text="annotation.index"></span>+ <span class="browser-annotation-chip-text" x-text="annotation.comment"></span>+ <button type="button" title="Remove annotation" aria-label="Remove annotation"+ @click="$store.browserPage.removeAnnotationComment(annotation.id)">+ <span class="material-symbols-outlined">close</span>+ </button>+ </div>+ </template>+ </div>+ <div class="browser-annotation-tray-actions">+ <button type="button" class="btn btn-field"+ @click="$store.browserPage.draftAnnotationsToChat()">Draft to chat</button>+ <button type="button" class="btn btn-ok"+ @click="$store.browserPage.sendAnnotationsToChat()">Send now</button>+ </div>+ </div><template x-if="!$store.browserPage.frameSrc && !$store.browserPage.isBusy()"><div class="browser-empty"><span class="material-symbols-outlined">captive_portal</span>@@ -549,6 +628,34 @@padding-bottom: 1px;}+ .browser-annotate-toggle {+ display: inline-flex;+ align-items: center;+ justify-content: center;+ gap: 6px;+ min-width: 0;+ min-height: var(--browser-control-size);+ padding: 0 10px;+ border: 1px solid var(--browser-chrome-border);+ border-radius: var(--browser-control-radius);+ background: color-mix(in srgb, var(--color-background) 26%, transparent);+ color: color-mix(in srgb, var(--color-text) 74%, var(--color-primary) 26%);+ font-size: 0.82rem;+ font-weight: 650;+ white-space: nowrap;+ }++ .browser-annotate-toggle:hover:not(:disabled),+ .browser-annotate-toggle.is-active {+ border-color: color-mix(in srgb, var(--color-primary) 48%, var(--browser-chrome-border));+ background: color-mix(in srgb, var(--color-primary) 16%, var(--color-background));+ color: var(--color-text);+ }++ .browser-annotate-toggle .material-symbols-outlined {+ font-size: 1.05rem;+ }+.browser-session-controls .browser-extensions.is-active {color: #2e7d32;}@@ -813,6 +920,10 @@outline: none;}+ .browser-stage.is-annotating {+ cursor: crosshair;+ }+.browser-frame {display: block;position: absolute;@@ -827,6 +938,194 @@background: #fff;}+ .browser-annotation-layer {+ position: absolute;+ inset: 0;+ z-index: 12;+ cursor: crosshair;+ touch-action: none;+ background: rgba(37, 99, 235, 0.035);+ }++ .browser-annotation-layer.is-busy {+ cursor: progress;+ }++ .browser-annotation-box {+ position: absolute;+ box-sizing: border-box;+ min-width: 8px;+ min-height: 8px;+ border: 2px solid #3399ff;+ background: rgba(51, 153, 255, 0.16);+ box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.52), 0 8px 22px rgba(0, 0, 0, 0.18);+ pointer-events: none;+ }++ .browser-annotation-box.is-draft {+ border-style: dashed;+ background: rgba(51, 153, 255, 0.1);+ }++ .browser-annotation-number {+ display: inline-flex;+ align-items: center;+ justify-content: center;+ width: 22px;+ min-width: 22px;+ height: 22px;+ min-height: 22px;+ border-radius: 50%;+ background: #3399ff;+ color: #fff;+ font-size: 0.74rem;+ font-weight: 800;+ line-height: 1;+ box-shadow: 0 2px 8px rgba(0, 0, 0, 0.28);+ }++ .browser-annotation-box .browser-annotation-number {+ position: absolute;+ top: -12px;+ left: -12px;+ }++ .browser-annotation-popover,+ .browser-annotation-tray {+ position: absolute;+ z-index: 18;+ border: 1px solid color-mix(in srgb, var(--color-border) 76%, transparent);+ border-radius: 7px;+ background: color-mix(in srgb, var(--color-background) 96%, #000 4%);+ color: var(--color-text);+ box-shadow: 0 16px 38px rgba(0, 0, 0, 0.28);+ }++ .browser-annotation-popover {+ display: flex;+ flex-direction: column;+ gap: 8px;+ padding: 10px;+ }++ .browser-annotation-popover-title,+ .browser-annotation-tray-header {+ display: flex;+ align-items: center;+ gap: 8px;+ min-width: 0;+ font-size: 0.82rem;+ font-weight: 750;+ }++ .browser-annotation-popover textarea {+ width: 100%;+ min-height: 82px;+ max-height: 160px;+ resize: vertical;+ padding: 8px;+ border: 1px solid color-mix(in srgb, var(--color-border) 74%, transparent);+ border-radius: 6px;+ background: var(--color-input);+ color: var(--color-text);+ font: inherit;+ line-height: 1.35;+ }++ .browser-annotation-actions,+ .browser-annotation-tray-actions {+ display: flex;+ justify-content: flex-end;+ gap: 7px;+ }++ .browser-annotation-actions .btn,+ .browser-annotation-tray-actions .btn {+ min-height: 30px;+ padding: 0 10px;+ white-space: nowrap;+ }++ .browser-annotation-tray {+ right: 10px;+ bottom: 10px;+ display: flex;+ flex-direction: column;+ gap: 9px;+ width: min(360px, calc(100% - 20px));+ max-height: min(48%, 310px);+ padding: 10px;+ }++ .browser-annotation-tray-header {+ justify-content: space-between;+ }++ .browser-annotation-clear,+ .browser-annotation-chip button {+ display: inline-flex;+ align-items: center;+ justify-content: center;+ width: 24px;+ height: 24px;+ min-width: 24px;+ min-height: 24px;+ padding: 0;+ border: 0;+ border-radius: 6px;+ background: transparent;+ color: color-mix(in srgb, var(--color-text) 64%, transparent);+ cursor: pointer;+ }++ .browser-annotation-clear:hover,+ .browser-annotation-chip button:hover {+ background: color-mix(in srgb, var(--color-panel) 82%, transparent);+ color: var(--color-text);+ }++ .browser-annotation-clear .material-symbols-outlined,+ .browser-annotation-chip button .material-symbols-outlined {+ font-size: 16px;+ }++ .browser-annotation-chips {+ display: flex;+ flex-direction: column;+ gap: 6px;+ min-height: 0;+ overflow: auto;+ }++ .browser-annotation-chip {+ display: grid;+ grid-template-columns: auto minmax(0, 1fr) auto;+ align-items: center;+ gap: 7px;+ min-height: 32px;+ padding: 5px 6px;+ border: 1px solid color-mix(in srgb, var(--color-border) 54%, transparent);+ border-radius: 7px;+ background: color-mix(in srgb, var(--color-panel) 74%, transparent);+ }++ .browser-annotation-chip .browser-annotation-number {+ width: 20px;+ min-width: 20px;+ height: 20px;+ min-height: 20px;+ font-size: 0.68rem;+ }++ .browser-annotation-chip-text {+ min-width: 0;+ overflow: hidden;+ text-overflow: ellipsis;+ white-space: nowrap;+ font-size: 0.78rem;+ line-height: 1.25;+ }+.browser-empty {display: flex;align-items: center;@@ -928,6 +1227,16 @@height: var(--browser-control-size);}+ .browser-annotate-toggle {+ width: var(--browser-control-size);+ min-width: var(--browser-control-size);+ padding: 0;+ }++ .browser-annotate-toggle span:not(.material-symbols-outlined) {+ display: none;+ }+.browser-extension-dropdown {right: 0;left: auto;diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex 57252f179..1bc5bb337 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -13,6 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;const BROWSER_CONFIG_REFRESH_MS = 15000;const VIEWPORT_SYNC_DEBOUNCE_MS = 220;const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;+const ANNOTATION_DRAG_THRESHOLD = 6;+const ANNOTATION_MAX_COMMENTS = 24;+const ANNOTATION_DOM_LIMIT = 1200;function makeViewerToken() {return globalThis.crypto?.randomUUID?.()@@ -60,6 +63,13 @@ const model = {address: "",frameSrc: "",frameState: null,+ annotating: false,+ annotationComments: [],+ annotationDraft: null,+ annotationDraftText: "",+ annotationDragRect: null,+ annotationBusy: false,+ annotationError: "",connected: false,switchingBrowserId: null,commandInFlight: false,@@ -76,6 +86,8 @@ const model = {_viewportSyncTimer: null,_lastViewportKey: "",_lastViewport: null,+ _annotationPointer: null,+ _annotationSequence: 0,_mode: "",_surfaceMounted: false,_surfaceSwitching: false,@@ -613,6 +625,7 @@ const model = {async command(command, extra = {}) {this.error = "";+ this.annotationError = "";this.commandInFlight = true;const previousActiveBrowserId = this.activeBrowserId;try {@@ -642,13 +655,17 @@ const model = {this.frameState = null;this.frameSrc = "";}- if (result.state?.currentUrl || result.currentUrl) {- this.address = result.state?.currentUrl || result.currentUrl;- }- this.applySnapshot(data.snapshot);- const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;- if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {- await this.connectViewer({ browserId: this.activeBrowserId });+ if (result.state?.currentUrl || result.currentUrl) {+ this.address = result.state?.currentUrl || result.currentUrl;+ }+ this.applySnapshot(data.snapshot);+ if (["navigate", "back", "forward", "reload", "close"].includes(String(command || "").toLowerCase())) {+ this.clearAnnotationsForBrowser(previousActiveBrowserId);+ this.cancelAnnotationDraft();+ }+ const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;+ if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {+ await this.connectViewer({ browserId: this.activeBrowserId });}} catch (error) {this.error = error instanceof Error ? error.message : String(error);@@ -761,17 +778,22 @@ const model = {return null;},- applyActiveFrameState(nextState = null) {- if (!nextState) return;- const stateId = this.normalizeBrowserId(nextState.id);- if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {- return;+ applyActiveFrameState(nextState = null) {+ if (!nextState) return;+ const stateId = this.normalizeBrowserId(nextState.id);+ if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {+ return;}+ const previousUrl = String(this.frameState?.currentUrl || "");+ const nextUrl = String(nextState.currentUrl || "");this.frameState = nextState;+ if (previousUrl && nextUrl && previousUrl !== nextUrl) {+ this.cancelAnnotationDraft();+ }if (!this.addressFocused && nextState.currentUrl) {this.address = nextState.currentUrl;- }- },+ }+ },applySnapshot(snapshot = null) {if (!snapshot?.image) return;@@ -805,6 +827,7 @@ const model = {if (this.activeBrowserId !== previous) {this._lastViewportKey = "";this._lastViewport = null;+ this.cancelAnnotationDraft();}},@@ -849,6 +872,419 @@ const model = {};},+ handleKeydown(event) {+ const annotateShortcut = event?.key === "." && (event.metaKey || event.ctrlKey) && !event.altKey;+ if (annotateShortcut && this._surfaceMounted) {+ event.preventDefault();+ event.stopPropagation?.();+ this.toggleAnnotationMode();+ return;+ }++ if (this.annotating) {+ if (event?.key === "Escape") {+ event.preventDefault();+ if (this.annotationDraft || this.annotationDragRect) {+ this.cancelAnnotationDraft();+ } else {+ this.toggleAnnotationMode(false);+ }+ }+ return;+ }++ void this.sendKey(event);+ },++ handleStageWheel(event) {+ if (this.annotating) return;+ void this.sendWheel(event);+ },++ toggleAnnotationMode(force = null) {+ const nextValue = force === null ? !this.annotating : Boolean(force);+ if (nextValue && !this.canAnnotate()) return;++ this.annotating = nextValue;+ this.annotationError = "";+ this.closeExtensionsMenu();+ if (!nextValue) {+ this.cancelAnnotationDraft();+ this.annotationDragRect = null;+ this._annotationPointer = null;+ } else {+ this._stageElement?.focus?.({ preventScroll: true });+ }+ },++ canAnnotate() {+ return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());+ },++ activeAnnotationUrl() {+ return String(this.frameState?.currentUrl || this.address || "about:blank");+ },++ visibleAnnotations() {+ const browserId = this.normalizeBrowserId(this.activeBrowserId);+ const url = this.activeAnnotationUrl();+ return this.annotationComments.filter((annotation) => (+ this.sameBrowserId(annotation.browserId, browserId)+ && String(annotation.url || "") === url+ ));+ },++ nextAnnotationIndex() {+ return this.visibleAnnotations().length + 1;+ },++ clearVisibleAnnotations() {+ this.clearAnnotationsForBrowser(this.activeBrowserId, this.activeAnnotationUrl());+ },++ clearAnnotationsForBrowser(browserId, url = null) {+ const numericBrowserId = this.normalizeBrowserId(browserId);+ if (!numericBrowserId) return;+ this.annotationComments = this.annotationComments.filter((annotation) => {+ if (!this.sameBrowserId(annotation.browserId, numericBrowserId)) return true;+ return url ? String(annotation.url || "") !== String(url) : false;+ });+ },++ annotationBoxStyle(rect = {}) {+ const viewport = this.currentViewportSize() || this._lastViewport || {};+ const width = Math.max(1, Number(viewport.width || rect.width || 1));+ const height = Math.max(1, Number(viewport.height || rect.height || 1));+ const normalized = this.clampAnnotationRect(rect);+ return [+ `left: ${(normalized.x / width) * 100}%`,+ `top: ${(normalized.y / height) * 100}%`,+ `width: ${(Math.max(1, normalized.width) / width) * 100}%`,+ `height: ${(Math.max(1, normalized.height) / height) * 100}%`,+ ].join("; ");+ },++ annotationPopoverStyle() {+ const rect = this.annotationDraft?.rect || this.annotationDragRect || {};+ const viewport = this.currentViewportSize() || this._lastViewport || {};+ const width = Math.max(1, Number(viewport.width || 1));+ const height = Math.max(1, Number(viewport.height || 1));+ const popoverWidth = Math.min(320, Math.max(240, width - 20));+ const popoverHeight = 190;+ const nextLeft = Math.min(+ Math.max(10, Number(rect.x || 0) + Number(rect.width || 0) + 10),+ Math.max(10, width - popoverWidth - 10),+ );+ const nextTop = Math.min(+ Math.max(10, Number(rect.y || 0) + Number(rect.height || 0) + 10),+ Math.max(10, height - popoverHeight - 10),+ );+ return [+ `left: ${(nextLeft / width) * 100}%`,+ `top: ${(nextTop / height) * 100}%`,+ `width: min(${popoverWidth}px, calc(100% - 20px))`,+ ].join("; ");+ },++ annotationDraftTitle() {+ if (!this.annotationDraft) return "Annotation";+ return this.annotationDraft.kind === "area" ? "Area annotation" : "Element annotation";+ },++ stagePointForEvent(event) {+ const image = this._stageElement?.querySelector?.(".browser-frame") || null;+ return this.pointerCoordinatesFor(event, image);+ },++ normalizeAnnotationRect(start = {}, end = {}) {+ const x1 = Number(start.x || 0);+ const y1 = Number(start.y || 0);+ const x2 = Number(end.x || x1);+ const y2 = Number(end.y || y1);+ return this.clampAnnotationRect({+ x: Math.min(x1, x2),+ y: Math.min(y1, y2),+ width: Math.abs(x2 - x1),+ height: Math.abs(y2 - y1),+ });+ },++ clampAnnotationRect(rect = {}) {+ const viewport = this.currentViewportSize() || this._lastViewport || {};+ const viewportWidth = Math.max(1, Number(viewport.width || rect.x + rect.width || 1));+ const viewportHeight = Math.max(1, Number(viewport.height || rect.y + rect.height || 1));+ const x = Math.max(0, Math.min(viewportWidth, Number(rect.x || 0)));+ const y = Math.max(0, Math.min(viewportHeight, Number(rect.y || 0)));+ const width = Math.max(1, Math.min(viewportWidth - x, Number(rect.width || 1)));+ const height = Math.max(1, Math.min(viewportHeight - y, Number(rect.height || 1)));+ return {+ x: Math.round(x),+ y: Math.round(y),+ width: Math.round(width),+ height: Math.round(height),+ };+ },++ startAnnotationSelection(event) {+ if (!this.annotating || this.annotationBusy || !this.canAnnotate()) return;+ const point = this.stagePointForEvent(event);+ if (!point) return;+ this.cancelAnnotationDraft();+ this.annotationError = "";+ this._annotationPointer = {+ id: event.pointerId,+ start: point,+ last: point,+ };+ this.annotationDragRect = this.clampAnnotationRect({+ x: point.x,+ y: point.y,+ width: 1,+ height: 1,+ });+ event.currentTarget?.setPointerCapture?.(event.pointerId);+ },++ moveAnnotationSelection(event) {+ if (!this.annotating || !this._annotationPointer) return;+ if (event.pointerId !== this._annotationPointer.id) return;+ const point = this.stagePointForEvent(event);+ if (!point) return;+ this._annotationPointer.last = point;+ this.annotationDragRect = this.normalizeAnnotationRect(this._annotationPointer.start, point);+ },++ async finishAnnotationSelection(event) {+ if (!this.annotating || !this._annotationPointer) return;+ if (event.pointerId !== this._annotationPointer.id) return;+ const pointer = this._annotationPointer;+ this._annotationPointer = null;+ event.currentTarget?.releasePointerCapture?.(event.pointerId);+ const endPoint = this.stagePointForEvent(event) || pointer.last || pointer.start;+ const rect = this.normalizeAnnotationRect(pointer.start, endPoint);+ this.annotationDragRect = null;+ const isDrag = rect.width >= ANNOTATION_DRAG_THRESHOLD || rect.height >= ANNOTATION_DRAG_THRESHOLD;+ const point = {+ x: Math.round(endPoint.x),+ y: Math.round(endPoint.y),+ };+ const payload = {+ kind: isDrag ? "area" : "element",+ point,+ rect: isDrag ? rect : null,+ viewport: this.currentViewportSize(),+ url: this.activeAnnotationUrl(),+ title: this.activeTitle,+ };+ await this.createAnnotationDraft(payload, isDrag ? rect : {+ x: point.x - 10,+ y: point.y - 10,+ width: 20,+ height: 20,+ });+ },++ cancelAnnotationSelection(event = null) {+ if (event && this._annotationPointer?.id === event.pointerId) {+ event.currentTarget?.releasePointerCapture?.(event.pointerId);+ }+ this._annotationPointer = null;+ this.annotationDragRect = null;+ },++ cancelAnnotationDraft() {+ this.annotationDraft = null;+ this.annotationDraftText = "";+ this.annotationDragRect = null;+ },++ async createAnnotationDraft(payload, fallbackRect) {+ if (!this.activeBrowserId || !this.contextId) return;+ const sequence = this._annotationSequence + 1;+ const browserId = this.activeBrowserId;+ const url = this.activeAnnotationUrl();+ const title = this.activeTitle;+ this._annotationSequence = sequence;+ this.annotationBusy = true;+ this.annotationError = "";+ try {+ const response = await websocket.request(+ "browser_viewer_annotation",+ {+ context_id: this.contextId,+ browser_id: browserId,+ viewer_id: this._viewerToken,+ payload,+ },+ { timeoutMs: 10000 },+ );+ if (sequence !== this._annotationSequence) return;+ const data = firstOk(response);+ const metadata = data.annotation || {};+ this.annotationDraft = {+ id: makeViewerToken(),+ browserId,+ url,+ title,+ kind: metadata.kind || payload.kind,+ rect: this.annotationRectFromMetadata(metadata, fallbackRect),+ metadata,+ createdAt: Date.now(),+ };+ this.annotationDraftText = "";+ } catch (error) {+ this.annotationError = error instanceof Error ? error.message : String(error);+ this.error = this.annotationError;+ } finally {+ if (sequence === this._annotationSequence) {+ this.annotationBusy = false;+ }+ }+ },++ annotationRectFromMetadata(metadata = {}, fallbackRect = {}) {+ const targetRect = metadata?.target?.rect || metadata?.rect || null;+ return this.clampAnnotationRect(targetRect || fallbackRect);+ },++ addAnnotationComment() {+ const comment = String(this.annotationDraftText || "").trim();+ if (!this.annotationDraft || !comment) return;+ if (this.visibleAnnotations().length >= ANNOTATION_MAX_COMMENTS) {+ this.annotationError = `Keep each batch to ${ANNOTATION_MAX_COMMENTS} annotations or fewer.`;+ this.error = this.annotationError;+ return;+ }+ this.annotationComments = [+ ...this.annotationComments,+ {+ ...this.annotationDraft,+ comment,+ index: this.nextAnnotationIndex(),+ },+ ];+ this.cancelAnnotationDraft();+ },++ removeAnnotationComment(annotationId) {+ this.annotationComments = this.annotationComments.filter((annotation) => annotation.id !== annotationId);+ },++ annotationChipLabel(annotation) {+ const prefix = annotation?.kind === "area" ? "Area" : "Element";+ return `${prefix} ${annotation?.index || ""}`.trim();+ },++ formatAnnotationRect(rect = {}) {+ const normalized = this.clampAnnotationRect(rect);+ return `x=${normalized.x}, y=${normalized.y}, width=${normalized.width}, height=${normalized.height}`;+ },++ redactAnnotationText(value) {+ return String(value || "")+ .replace(/(<input\b(?=[^>]*\btype=(["'])?password\2?)[^>]*?)\svalue=(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")+ .replace(/\b(password|passcode|token|secret|value)=((["'])[\s\S]{1,240}?\3)/giu, "$1=\"[redacted]\"");+ },++ formatAnnotationMetadata(metadata = {}) {+ const lines = [];+ const target = metadata.target || {};+ const selector = target.selector || metadata.selector || "";+ const summary = target.summary || metadata.summary || "";+ const dom = this.redactAnnotationText(target.dom || metadata.dom || "").slice(0, ANNOTATION_DOM_LIMIT);++ if (selector) {+ lines.push(`Selector: ${selector}`);+ }+ if (target.tagName || target.role || target.id || target.name || target.classes) {+ lines.push([+ "Element:",+ target.tagName ? `<${String(target.tagName).toLowerCase()}>` : "",+ target.role ? `role=${target.role}` : "",+ target.id ? `id=${target.id}` : "",+ target.name ? `name=${target.name}` : "",+ target.classes ? `class=${target.classes}` : "",+ ].filter(Boolean).join(" "));+ }+ if (summary) {+ lines.push(`Summary: ${summary}`);+ }+ if (Array.isArray(metadata.elements) && metadata.elements.length) {+ lines.push("Intersecting elements:");+ metadata.elements.slice(0, 8).forEach((element, index) => {+ const elementLabel = [+ `${index + 1}.`,+ element.tagName ? `<${String(element.tagName).toLowerCase()}>` : "",+ element.selector || "",+ element.summary || "",+ ].filter(Boolean).join(" ");+ lines.push(elementLabel);+ });+ }+ if (dom) {+ lines.push(`DOM: ${dom}`);+ }+ return lines.join("\n");+ },++ buildAnnotationsPrompt() {+ const annotations = this.visibleAnnotations();+ if (!annotations.length) return "";+ const lines = [+ "Browser annotations",+ `Page title: ${this.activeTitle}`,+ `Page URL: ${this.activeAnnotationUrl()}`,+ `Browser id: ${this.activeBrowserId}`,+ "",+ ];+ annotations.forEach((annotation, index) => {+ lines.push(+ `Annotation ${index + 1}`,+ `Comment: ${annotation.comment}`,+ `Selection kind: ${annotation.kind}`,+ `Coordinates: ${this.formatAnnotationRect(annotation.rect)}`,+ );+ const metadata = this.formatAnnotationMetadata(annotation.metadata);+ if (metadata) {+ lines.push(metadata);+ }+ lines.push("");+ });+ return lines.join("\n").trim();+ },++ draftAnnotationsToChat() {+ const prompt = this.buildAnnotationsPrompt();+ if (!prompt) return;+ const existingMessage = String(chatInputStore.message || "").trim();+ chatInputStore.message = existingMessage ? `${existingMessage}\n\n${prompt}` : prompt;+ chatInputStore.adjustTextareaHeight?.();+ chatInputStore.focus?.();+ this.clearVisibleAnnotations();+ this.toggleAnnotationMode(false);+ },++ async sendAnnotationsToChat() {+ const prompt = this.buildAnnotationsPrompt();+ if (!prompt) return;+ chatInputStore.message = prompt;+ chatInputStore.adjustTextareaHeight?.();+ try {+ if (typeof chatInputStore.sendMessage === "function") {+ await chatInputStore.sendMessage();+ } else if (typeof globalThis.sendMessage === "function") {+ await globalThis.sendMessage();+ } else {+ chatInputStore.focus?.();+ return;+ }+ this.clearVisibleAnnotations();+ this.toggleAnnotationMode(false);+ } catch (error) {+ this.error = error instanceof Error ? error.message : String(error);+ }+ },+currentViewportSize() {const stage = this._stageElement;if (!stage) return null;@@ -910,6 +1346,7 @@ const model = {},async sendMouse(eventType, event) {+ if (this.annotating) return;if (!this.activeBrowserId || !event?.currentTarget) return;const pointer = this.pointerCoordinatesFor(event);if (!pointer) return;@@ -960,6 +1397,7 @@ const model = {},async sendKey(event) {+ if (this.annotating) return;if (!this.activeBrowserId) return;if (event.ctrlKey || event.metaKey || event.altKey) return;const editable = ["INPUT", "TEXTAREA", "SELECT"].includes(event.target?.tagName);@@ -982,6 +1420,11 @@ const model = {this._surfaceMounted = false;this._surfaceSwitching = false;this.commandInFlight = false;+ this.annotating = false;+ this.annotationBusy = false;+ this.annotationError = "";+ this.cancelAnnotationDraft();+ this.cancelAnnotationSelection();if (this.contextId) {try {await websocket.emit("browser_viewer_unsubscribe", { context_id: this.contextId });diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex f3c65b557..9e4c40c8e 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -2,7 +2,7 @@ import asyncioimport sysimport threadingfrom pathlib import Path-from types import SimpleNamespace+from types import ModuleType, SimpleNamespaceimport pytest@@ -11,6 +11,73 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]if str(PROJECT_ROOT) not in sys.path:sys.path.insert(0, str(PROJECT_ROOT))++class _TestAgentContext:+ @staticmethod+ def get(context_id):+ return None+++class _TestResponse(SimpleNamespace):+ def __init__(self, message="", break_loop=False, **kwargs):+ super().__init__(message=message, break_loop=break_loop, **kwargs)+++class _TestTool:+ def __init__(+ self,+ agent=None,+ name="",+ method=None,+ args=None,+ message="",+ loop_data=None,+ **kwargs,+ ):+ self.agent = agent+ self.name = name+ self.method = method+ self.args = args or {}+ self.message = message+ self.loop_data = loop_data+++class _TestWsHandler:+ def __init__(self, *args, **kwargs):+ self.emitted = []++ async def emit_to(self, sid, event, data, correlation_id=None):+ self.emitted.append((sid, event, data, correlation_id))+++class _TestWsResult(dict):+ @staticmethod+ def error(code="", message="", correlation_id=None):+ return _TestWsResult(+ {+ "ok": False,+ "code": code,+ "error": message,+ "correlation_id": correlation_id,+ }+ )+++sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))+sys.modules.setdefault("helpers.tool", SimpleNamespace(Response=_TestResponse, Tool=_TestTool))+sys.modules.setdefault("helpers.ws", SimpleNamespace(WsHandler=_TestWsHandler))+sys.modules.setdefault("helpers.ws_manager", SimpleNamespace(WsResult=_TestWsResult))+_model_config_stub = ModuleType("plugins._model_config.helpers.model_config")+_model_config_stub.get_presets = lambda: []+_model_config_stub.get_preset_by_name = lambda name: None+_model_config_stub.get_chat_model_config = lambda agent=None: {}+sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_config_stub)+++def anyio_backend():+ return "asyncio"+from plugins._browser.helpers.config import (build_browser_launch_config,get_browser_main_model_summary,@@ -63,6 +130,8 @@ def test_browser_config_normalizes_extension_paths(tmp_path):assert config == {"extension_paths": [str(extension_dir)],+ "default_homepage": "about:blank",+ "autofocus_active_page": True,"model_preset": "",}@@ -246,7 +315,7 @@ def test_browser_extension_manager_uses_modern_chrome_prodversion(monkeypatch):def test_browser_extension_menu_exposes_agent_and_url_paths():- html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(+ html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(encoding="utf-8")skill = PROJECT_ROOT / "skills" / "a0-browser-ext" / "SKILL.md"@@ -279,7 +348,7 @@ def test_browser_viewer_allows_slow_extension_startup():def test_browser_ui_spinners_have_browser_local_animation():- main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(encoding="utf-8")config_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "config.html").read_text(@@ -305,7 +374,7 @@ def test_browser_extension_settings_stay_user_facing():def test_browser_viewer_uses_tabs_for_session_switching():- main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(encoding="utf-8")browser_store = (@@ -329,7 +398,7 @@ def test_browser_viewer_uses_cdp_screencast_transport():ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(encoding="utf-8")- main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(encoding="utf-8")runtime = (@@ -339,12 +408,12 @@ def test_browser_viewer_uses_cdp_screencast_transport():PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(encoding="utf-8")- assert 'runtime.call("screenshot"' not in ws_browser+ assert 'runtime.call("screenshot"' in ws_browserassert "SCREENCAST_QUALITY = 92" in ws_browserassert "initial_viewport = self._viewport_from_data(data)" in ws_browserassert '"set_viewport"' in ws_browserassert "start_screencast" in ws_browser- assert "read_screencast_frame" in ws_browser+ assert "pop_screencast_frame" in ws_browserassert "stop_screencast" in ws_browserassert '"Page.startScreencast"' in runtimeassert '"Page.screencastFrame"' in runtime@@ -360,12 +429,54 @@ def test_browser_viewer_uses_cdp_screencast_transport():assert "viewport_height: initialViewport?.height" in browser_storeassert "this.frameState = data.state || null" not in browser_storeassert "overflow: hidden;" in main_html- assert "object-fit: fill;" not in main_html- assert "height: auto;" in main_html+ assert "object-fit: fill;" in main_htmlassert "image-rendering: auto;" in main_html+def test_browser_annotate_mode_ui_and_prompt_hooks():+ panel_html = (+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"+ ).read_text(encoding="utf-8")+ browser_store = (+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"+ ).read_text(encoding="utf-8")++ assert "Annotate" in panel_html+ assert "Annotating" in panel_html+ assert "browser-annotation-layer" in panel_html+ assert "browser-annotation-tray" in panel_html+ assert "Draft to chat" in panel_html+ assert "Send now" in panel_html+ assert "@pointerdown.stop.prevent=\"$store.browserPage.startAnnotationSelection($event)\"" in panel_html+ assert "@keydown.window=\"$store.browserPage.handleKeydown($event)\"" in panel_html+ assert "annotationComments: []" in browser_store+ assert '"browser_viewer_annotation"' in browser_store+ assert 'event?.key === "." && (event.metaKey || event.ctrlKey)' in browser_store+ assert "Browser annotations" in browser_store+ assert "Comment:" in browser_store+ assert "Coordinates:" in browser_store+ assert "Selector:" in browser_store+ assert "DOM:" in browser_store+ assert "value=\\\"[redacted]\\\"" in browser_store+++def test_browser_runtime_and_content_helper_expose_annotation_target():+ runtime = (+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"+ ).read_text(encoding="utf-8")+ helper = (+ PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"+ ).read_text(encoding="utf-8")++ assert "async def annotation_target" in runtime+ assert "globalThis.__spaceBrowserPageContent__.annotate(payload || null)" in runtime+ assert "function annotate(payload = null)" in helper+ assert "annotate," in helper+ assert "sanitizeAnnotationDom" in helper+ assert "password" in helper++async def test_browser_screencast_acknowledges_and_drops_stale_frames():first_image = ("/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"@@ -526,7 +637,7 @@ def test_browser_save_plugin_config_does_not_restart_runtimes_for_preset_only(moassert restarted == []async def test_browser_tool_dispatches_direct_actions(monkeypatch):calls = []@@ -558,7 +669,7 @@ async def test_browser_tool_dispatches_direct_actions(monkeypatch):assert calls == [("content", (1, None))]async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):class FakeRuntime:def __init__(self) -> None:@@ -608,7 +719,7 @@ async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):assert ("sid-1", "ctx") not in ws_browser_module.WsBrowser._streamsasync def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):calls = []@@ -642,11 +753,14 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):"sid-1",)- assert result == {"state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)}}+ assert result == {+ "state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)},+ "snapshot": None,+ }assert calls == [("set_viewport", (7, 1280, 720), {})]async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):calls = []@@ -682,10 +796,68 @@ async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):"sid-1",)- assert result == {"state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)}}+ assert result == {+ "state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)},+ "snapshot": None,+ }assert calls == [("wheel", (3, 320.0, 480.0, 0.0, 640.0), {})]+async def test_browser_viewer_annotation_dispatches_runtime(monkeypatch):+ calls = []++ class FakeRuntime:+ async def call(self, method, *args, **kwargs):+ calls.append((method, args, kwargs))+ return {+ "kind": "element",+ "point": {"x": 320, "y": 180},+ "target": {"tagName": "BUTTON", "selector": "#save"},+ }++ async def fake_get_runtime(context_id, create=True):+ assert context_id == "ctx"+ assert create is False+ return FakeRuntime()++ monkeypatch.setattr(ws_browser_module, "get_runtime", fake_get_runtime)++ handler = ws_browser_module.WsBrowser(+ SimpleNamespace(),+ threading.RLock(),+ manager=None,+ )++ payload = {+ "kind": "element",+ "point": {"x": 320, "y": 180},+ "viewport": {"width": 1280, "height": 720},+ }+ result = await handler.process(+ "browser_viewer_annotation",+ {+ "context_id": "ctx",+ "browser_id": 4,+ "viewer_id": "viewer-1",+ "payload": payload,+ },+ "sid-1",+ )++ assert result == {+ "annotation": {+ "kind": "element",+ "point": {"x": 320, "y": 180},+ "target": {"tagName": "BUTTON", "selector": "#save"},+ },+ "context_id": "ctx",+ "browser_id": 4,+ "viewer_id": "viewer-1",+ }+ assert calls == [("annotation_target", (4, payload), {})]++def test_browser_cleanup_extensions_follow_extensible_path_layout():extension = __import__("helpers.extension", fromlist=["_get_extension_classes"])remove_classes = extension._get_extension_classes( # type: ignore[attr-defined]diff --git a/tests/test_webui_extension_surfaces.py b/tests/test_webui_extension_surfaces.pyindex b0ca89f46..0536c4905 100644--- a/tests/test_webui_extension_surfaces.py+++ b/tests/test_webui_extension_surfaces.py@@ -5,6 +5,7 @@ import tempfileimport threadingfrom contextlib import contextmanagerfrom pathlib import Path+from types import SimpleNamespacefrom typing import Iteratorimport pytest@@ -14,6 +15,15 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]if str(PROJECT_ROOT) not in sys.path:sys.path.insert(0, str(PROJECT_ROOT))++class _TestAgentContext:+ @staticmethod+ def get(context_id):+ return None+++sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))+from api.load_webui_extensions import LoadWebuiExtensions@@ -69,6 +79,11 @@ def _new_handler() -> LoadWebuiExtensions:return LoadWebuiExtensions(app, threading.RLock())+def anyio_backend():+ return "asyncio"++def _assert_surface_anchor_in_template(surface: str, template_rel_path: str) -> None:template_path = PROJECT_ROOT / template_rel_pathtemplate_html = template_path.read_text(encoding="utf-8")@@ -118,7 +133,7 @@ def _temporary_probe_plugin(surface: str) -> Iterator[tuple[str, str]]:cache.clear("*(plugins)*")@pytest.mark.parametrize(("surface", "template_rel_path"),SURFACE_SCENARIOS,