Skip to content

Agent Zero AI framework

Add browser annotate mode

Alessandro4ff3244ce63e
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index b1df9146b..8430f7e7a 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -43,6 +43,8 @@ class WsBrowser(WsHandler):
return await self._command(data, sid)
if event == "browser_viewer_input":
return await self._input(data, sid)
+ if event == "browser_viewer_annotation":
+ return await self._annotation(data, sid)
return WsResult.error(
code="UNKNOWN_BROWSER_EVENT",
@@ -215,6 +217,29 @@ class WsBrowser(WsHandler):
else None,
}
+ async def _annotation(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
+ context_id = self._context_id(data)
+ if not context_id:
+ return self._error("MISSING_CONTEXT", "context_id is required", data)
+ runtime = await get_runtime(context_id, create=False)
+ if not runtime:
+ return self._error("NO_BROWSER_RUNTIME", "No browser runtime exists for this context", data)
+
+ browser_id = data.get("browser_id")
+ viewer_id = str(data.get("viewer_id") or "")
+ payload = data.get("payload") if isinstance(data.get("payload"), dict) else {}
+ try:
+ annotation = await runtime.call("annotation_target", browser_id, payload)
+ except Exception as exc:
+ return self._error("ANNOTATION_FAILED", str(exc), data)
+
+ return {
+ "annotation": annotation,
+ "context_id": context_id,
+ "browser_id": browser_id,
+ "viewer_id": viewer_id,
+ }
+
async def _snapshot_for_result(
self,
runtime: Any,
diff --git a/plugins/_browser/assets/browser-page-content.js b/plugins/_browser/assets/browser-page-content.js
index fec76e633..edf9d9403 100644
--- a/plugins/_browser/assets/browser-page-content.js
+++ b/plugins/_browser/assets/browser-page-content.js
@@ -1,7 +1,7 @@
(() => {
const GLOBAL_KEY = "__spaceBrowserPageContent__";
const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
- const VERSION = "6";
+ const VERSION = "7";
const BLOCK_TAGS = new Set([
"ADDRESS",
"ARTICLE",
@@ -2842,10 +2842,377 @@
});
}
+ function cssEscape(value) {
+ const rawValue = String(value || "");
+ if (!rawValue) {
+ return "";
+ }
+
+ if (typeof globalThis.CSS?.escape === "function") {
+ return globalThis.CSS.escape(rawValue);
+ }
+
+ return rawValue.replace(/[^a-zA-Z0-9_-]/gu, (character) => `\\${character}`);
+ }
+
+ function getClassSummary(element) {
+ try {
+ return [...(element?.classList || [])]
+ .map((className) => normalizeAttributeText(className))
+ .filter(Boolean)
+ .slice(0, 4)
+ .join(" ");
+ } catch {
+ return "";
+ }
+ }
+
+ function buildCssSelector(element) {
+ if (!isElementNode(element)) {
+ return "";
+ }
+
+ const id = normalizeAttributeText(element.getAttribute?.("id"));
+ if (id) {
+ return `#${cssEscape(id)}`;
+ }
+
+ const parts = [];
+ let current = element;
+ while (isElementNode(current) && current !== globalThis.document?.documentElement && parts.length < 6) {
+ const tagName = getTagName(current).toLowerCase();
+ if (!tagName) {
+ break;
+ }
+
+ let part = tagName;
+ const classes = getClassSummary(current)
+ .split(/\s+/u)
+ .filter(Boolean)
+ .slice(0, 2);
+ if (classes.length && !["body", "html"].includes(tagName)) {
+ part += classes.map((className) => `.${cssEscape(className)}`).join("");
+ }
+
+ const parent = current.parentElement;
+ if (parent) {
+ const siblings = [...parent.children].filter((sibling) => getTagName(sibling) === getTagName(current));
+ if (siblings.length > 1) {
+ part += `:nth-of-type(${siblings.indexOf(current) + 1})`;
+ }
+ }
+
+ parts.unshift(part);
+ if (tagName === "body") {
+ break;
+ }
+ current = parent;
+ }
+
+ return parts.join(" > ");
+ }
+
+ function sanitizeAnnotationDom(value) {
+ return truncateText(
+ String(value || "")
+ .replace(/(<input\b(?=[^>]*\btype\s*=\s*(["'])?password\2?)[^>]*?)\s+value\s*=\s*(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")
+ .replace(/\svalue\s*=\s*(["'])[\s\S]{0,600}?\1/giu, " value=\"[redacted]\"")
+ .replace(/\sdata-space-browser-live-value\s*=\s*(["'])[\s\S]{0,600}?\1/giu, "")
+ .replace(/\sdata-space-browser-selected-text\s*=\s*(["'])[\s\S]{0,600}?\1/giu, ""),
+ 1200
+ );
+ }
+
+ function summarizeAnnotationElement(element) {
+ if (!isElementNode(element)) {
+ return null;
+ }
+
+ const summaryData = collectReferenceSummaryData(element, {
+ includeLabelQuotes: false,
+ includeLinkUrls: true,
+ includeSemanticTags: true,
+ includeStateTags: true
+ });
+ const rawDom = serializeElementSnapshot(element);
+ return {
+ classes: getClassSummary(element),
+ dom: sanitizeAnnotationDom(rawDom),
+ id: normalizeAttributeText(element.getAttribute?.("id")),
+ kind: summaryData.kind,
+ name: normalizeAttributeText(element.getAttribute?.("name")),
+ rect: getElementRectSafe(element),
+ role: normalizeAttributeText(element.getAttribute?.("role")).toLowerCase(),
+ selector: buildCssSelector(element),
+ semanticTags: Array.isArray(summaryData.semanticTags) ? summaryData.semanticTags.slice(0, 4) : [],
+ stateTags: Array.isArray(summaryData.state?.stateTags) ? summaryData.state.stateTags.slice(0, 8) : [],
+ summary: truncateText(summaryData.summary || getLabelText(element, {
+ includeAlt: true,
+ includeDescendantImageAlt: true,
+ includePlaceholder: true,
+ includeText: true
+ }), 240),
+ tagName: getTagName(element)
+ };
+ }
+
+ function annotationViewport() {
+ return {
+ height: Math.max(0, Number(globalThis.innerHeight || globalThis.document?.documentElement?.clientHeight || 0)),
+ scrollX: Number(globalThis.scrollX || globalThis.pageXOffset || 0),
+ scrollY: Number(globalThis.scrollY || globalThis.pageYOffset || 0),
+ width: Math.max(0, Number(globalThis.innerWidth || globalThis.document?.documentElement?.clientWidth || 0))
+ };
+ }
+
+ function normalizeAnnotationPoint(payload = {}, viewport = annotationViewport()) {
+ const source = payload?.point && typeof payload.point === "object" ? payload.point : payload;
+ const width = Math.max(1, Number(viewport.width || 1));
+ const height = Math.max(1, Number(viewport.height || 1));
+ return {
+ x: Math.max(0, Math.min(width, Number(source?.x || 0))),
+ y: Math.max(0, Math.min(height, Number(source?.y || 0)))
+ };
+ }
+
+ function normalizeAnnotationRectPayload(payload = {}, viewport = annotationViewport()) {
+ const source = payload?.rect && typeof payload.rect === "object" ? payload.rect : payload;
+ const width = Math.max(1, Number(viewport.width || 1));
+ const height = Math.max(1, Number(viewport.height || 1));
+ const x = Math.max(0, Math.min(width, Number(source?.x || 0)));
+ const y = Math.max(0, Math.min(height, Number(source?.y || 0)));
+ return {
+ height: Math.max(1, Math.min(height - y, Number(source?.height || source?.h || 1))),
+ width: Math.max(1, Math.min(width - x, Number(source?.width || source?.w || 1))),
+ x,
+ y
+ };
+ }
+
+ function intersectRects(leftRect, rightRect) {
+ if (!leftRect || !rightRect) {
+ return null;
+ }
+
+ const x = Math.max(Number(leftRect.x || 0), Number(rightRect.x || 0));
+ const y = Math.max(Number(leftRect.y || 0), Number(rightRect.y || 0));
+ const right = Math.min(
+ Number(leftRect.x || 0) + Number(leftRect.width || 0),
+ Number(rightRect.x || 0) + Number(rightRect.width || 0)
+ );
+ const bottom = Math.min(
+ Number(leftRect.y || 0) + Number(leftRect.height || 0),
+ Number(rightRect.y || 0) + Number(rightRect.height || 0)
+ );
+ const width = right - x;
+ const height = bottom - y;
+ if (width <= 0 || height <= 0) {
+ return null;
+ }
+ return {
+ area: width * height,
+ height,
+ width,
+ x,
+ y
+ };
+ }
+
+ function deepElementFromPoint(x, y) {
+ let element = null;
+ try {
+ element = globalThis.document?.elementFromPoint?.(x, y) || null;
+ } catch {
+ return null;
+ }
+
+ let guard = 0;
+ while (isElementNode(element) && element.shadowRoot && guard < 8) {
+ guard += 1;
+ try {
+ const nestedElement = element.shadowRoot.elementFromPoint?.(x, y);
+ if (!nestedElement || nestedElement === element) {
+ break;
+ }
+ element = nestedElement;
+ } catch {
+ break;
+ }
+ }
+
+ return element;
+ }
+
+ function findAnnotationTarget(element) {
+ if (!isElementNode(element)) {
+ return null;
+ }
+
+ const selector = [
+ "a[href]",
+ "button",
+ "input",
+ "textarea",
+ "select",
+ "summary",
+ "[role]",
+ "img",
+ "label",
+ "form",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "p",
+ "li",
+ "td",
+ "th",
+ "article",
+ "section",
+ "nav",
+ "header",
+ "main",
+ "footer"
+ ].join(",");
+ const target = element.closest?.(selector) || element;
+ return isElementNode(target) && !isHiddenElement(target) ? target : element;
+ }
+
+ function isMeaningfulAnnotationElement(element) {
+ if (!isElementNode(element) || isHiddenElement(element)) {
+ return false;
+ }
+
+ if (isInteractiveElement(element) || getTagName(element) === "IMG") {
+ return true;
+ }
+
+ const tagName = getTagName(element);
+ const role = normalizeAttributeText(element.getAttribute?.("role")).toLowerCase();
+ return Boolean(
+ role
+ || /^H[1-6]$/u.test(tagName)
+ || ["ARTICLE", "SECTION", "MAIN", "NAV", "HEADER", "FOOTER", "FORM", "LABEL", "P", "LI", "TD", "TH"].includes(tagName)
+ );
+ }
+
+ function collectIntersectingAnnotationElements(rect) {
+ const selector = [
+ "a[href]",
+ "button",
+ "input",
+ "textarea",
+ "select",
+ "summary",
+ "[role]",
+ "img",
+ "label",
+ "form",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "p",
+ "li",
+ "td",
+ "th",
+ "article",
+ "section",
+ "main",
+ "nav",
+ "header",
+ "footer"
+ ].join(",");
+ let candidates = [];
+ try {
+ candidates = [...(globalThis.document?.querySelectorAll?.(selector) || [])];
+ } catch {
+ candidates = [];
+ }
+
+ const seen = new Set();
+ return candidates
+ .map((element) => {
+ if (!isMeaningfulAnnotationElement(element) || seen.has(element)) {
+ return null;
+ }
+ seen.add(element);
+ const elementRect = getElementRectSafe(element);
+ const intersection = intersectRects(rect, elementRect);
+ if (!intersection || intersection.area < 48) {
+ return null;
+ }
+ return {
+ element,
+ elementArea: Math.max(1, Number(elementRect.width || 0) * Number(elementRect.height || 0)),
+ intersection
+ };
+ })
+ .filter(Boolean)
+ .sort((left, right) => {
+ if (right.intersection.area !== left.intersection.area) {
+ return right.intersection.area - left.intersection.area;
+ }
+ return left.elementArea - right.elementArea;
+ })
+ .slice(0, 12)
+ .map((entry) => summarizeAnnotationElement(entry.element))
+ .filter(Boolean);
+ }
+
+ function annotate(payload = null) {
+ const request = payload && typeof payload === "object" ? payload : {};
+ const viewport = annotationViewport();
+ const kind = request.kind === "area" || request.rect ? "area" : "element";
+
+ if (kind === "area") {
+ const rect = normalizeAnnotationRectPayload(request, viewport);
+ const point = {
+ x: rect.x + rect.width / 2,
+ y: rect.y + rect.height / 2
+ };
+ const elements = collectIntersectingAnnotationElements(rect);
+ const fallbackElement = findAnnotationTarget(deepElementFromPoint(point.x, point.y));
+ const fallbackTarget = fallbackElement ? summarizeAnnotationElement(fallbackElement) : null;
+ return {
+ elements,
+ kind,
+ point,
+ rect,
+ status: elements.length || fallbackTarget ? "ok" : "empty",
+ target: elements[0] || fallbackTarget,
+ viewport
+ };
+ }
+
+ const point = normalizeAnnotationPoint(request, viewport);
+ const rawElement = deepElementFromPoint(point.x, point.y);
+ const targetElement = findAnnotationTarget(rawElement);
+ const target = targetElement ? summarizeAnnotationElement(targetElement) : null;
+ return {
+ kind,
+ point,
+ rect: target?.rect || {
+ height: 1,
+ width: 1,
+ x: point.x,
+ y: point.y
+ },
+ status: target ? "ok" : "empty",
+ target,
+ viewport
+ };
+ }
+
globalThis[GLOBAL_KEY] = {
click(referenceId) {
return activateElement(referenceId);
},
+ annotate,
capture,
clear() {
state.captureId = 0;
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 036d1c715..db459de91 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -557,6 +557,22 @@ class _BrowserRuntimeCore:
self.last_interacted_browser_id = resolved_id
return result or {}
+ async def annotation_target(
+ self,
+ browser_id: int | str | None,
+ payload: dict[str, Any] | None = None,
+ ) -> dict[str, Any]:
+ await self.ensure_started()
+ resolved_id = self._resolve_browser_id(browser_id)
+ page = self._page(resolved_id)
+ await self._ensure_content_helper(page)
+ result = await page.evaluate(
+ "(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)",
+ payload or None,
+ )
+ self.last_interacted_browser_id = resolved_id
+ return result or {}
+
async def evaluate(self, browser_id: int | str | None, script: str) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
@@ -918,7 +934,7 @@ class _BrowserRuntimeCore:
async def _ensure_content_helper(self, page: Any) -> None:
has_helper = await page.evaluate(
- "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture)"
+ "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate)"
)
if has_helper:
return
diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html
index e25808403..108b9c966 100644
--- a/plugins/_browser/webui/browser-panel.html
+++ b/plugins/_browser/webui/browser-panel.html
@@ -11,7 +11,7 @@
<div x-data>
<template x-if="$store.browserPage">
<div class="browser-panel" x-create="$store.browserPage.onOpen($el, xAttrs($el) || {})" x-destroy="$store.browserPage.cleanup()"
- @keydown.window="$store.browserPage.sendKey($event)">
+ @keydown.window="$store.browserPage.handleKeydown($event)">
<div class="browser-meta">
<div class="browser-meta-top">
<div class="browser-session-tabs" role="tablist" aria-label="Browser sessions">
@@ -41,6 +41,15 @@
</div>
<div class="browser-session-controls">
+ <button type="button" class="btn browser-annotate-toggle" title="Annotate" aria-label="Annotate"
+ :aria-pressed="$store.browserPage.annotating.toString()"
+ :class="{ 'is-active': $store.browserPage.annotating }"
+ :disabled="!$store.browserPage.canAnnotate() && !$store.browserPage.annotating"
+ @click="$store.browserPage.toggleAnnotationMode()">
+ <span class="material-symbols-outlined" aria-hidden="true"
+ x-text="$store.browserPage.annotating ? 'rate_review' : 'edit_note'"></span>
+ <span x-text="$store.browserPage.annotating ? 'Annotating' : 'Annotate'"></span>
+ </button>
<div class="browser-extension-menu" @click.outside="$store.browserPage.closeExtensionsMenu()"
@keydown.escape.window="$store.browserPage.closeExtensionsMenu()">
<button type="button" class="btn btn-icon-action browser-extensions" title="Browser settings"
@@ -161,12 +170,82 @@
</div>
<div class="browser-stage" tabindex="0" @click="$el.focus()"
- @wheel.prevent="$store.browserPage.sendWheel($event)">
+ :class="{ 'is-annotating': $store.browserPage.annotating }"
+ @wheel.prevent="$store.browserPage.handleStageWheel($event)">
<template x-if="$store.browserPage.frameSrc">
<img class="browser-frame" :src="$store.browserPage.frameSrc"
@click="$store.browserPage.sendMouse('click', $event)"
@mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
</template>
+ <template x-if="$store.browserPage.annotating && $store.browserPage.frameSrc">
+ <div class="browser-annotation-layer"
+ :class="{ 'is-busy': $store.browserPage.annotationBusy }"
+ @pointerdown.stop.prevent="$store.browserPage.startAnnotationSelection($event)"
+ @pointermove.stop.prevent="$store.browserPage.moveAnnotationSelection($event)"
+ @pointerup.stop.prevent="$store.browserPage.finishAnnotationSelection($event)"
+ @pointercancel.stop.prevent="$store.browserPage.cancelAnnotationSelection($event)">
+ <template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">
+ <div class="browser-annotation-box is-saved"
+ :style="$store.browserPage.annotationBoxStyle(annotation.rect)">
+ <span class="browser-annotation-number" x-text="annotation.index"></span>
+ </div>
+ </template>
+ <template x-if="$store.browserPage.annotationDragRect">
+ <div class="browser-annotation-box is-draft"
+ :style="$store.browserPage.annotationBoxStyle($store.browserPage.annotationDragRect)">
+ </div>
+ </template>
+ </div>
+ </template>
+ <template x-if="$store.browserPage.annotationDraft">
+ <div class="browser-annotation-popover"
+ :style="$store.browserPage.annotationPopoverStyle()"
+ @click.stop @pointerdown.stop @keydown.stop>
+ <div class="browser-annotation-popover-title">
+ <span class="browser-annotation-number" x-text="$store.browserPage.nextAnnotationIndex()"></span>
+ <span x-text="$store.browserPage.annotationDraftTitle()"></span>
+ </div>
+ <textarea x-model="$store.browserPage.annotationDraftText" placeholder="Comment"
+ maxlength="1200"></textarea>
+ <div class="browser-annotation-actions">
+ <button type="button" class="btn btn-field"
+ @click="$store.browserPage.cancelAnnotationDraft()">Cancel</button>
+ <button type="button" class="btn btn-ok"
+ :disabled="!String($store.browserPage.annotationDraftText || '').trim()"
+ @click="$store.browserPage.addAnnotationComment()">Add</button>
+ </div>
+ </div>
+ </template>
+ <div class="browser-annotation-tray"
+ x-show="$store.browserPage.visibleAnnotations().length"
+ x-transition style="display: none;"
+ @click.stop @pointerdown.stop @keydown.stop>
+ <div class="browser-annotation-tray-header">
+ <span>Annotations</span>
+ <button type="button" class="browser-annotation-clear" title="Clear annotations"
+ aria-label="Clear annotations" @click="$store.browserPage.clearVisibleAnnotations()">
+ <span class="material-symbols-outlined">delete</span>
+ </button>
+ </div>
+ <div class="browser-annotation-chips">
+ <template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">
+ <div class="browser-annotation-chip">
+ <span class="browser-annotation-number" x-text="annotation.index"></span>
+ <span class="browser-annotation-chip-text" x-text="annotation.comment"></span>
+ <button type="button" title="Remove annotation" aria-label="Remove annotation"
+ @click="$store.browserPage.removeAnnotationComment(annotation.id)">
+ <span class="material-symbols-outlined">close</span>
+ </button>
+ </div>
+ </template>
+ </div>
+ <div class="browser-annotation-tray-actions">
+ <button type="button" class="btn btn-field"
+ @click="$store.browserPage.draftAnnotationsToChat()">Draft to chat</button>
+ <button type="button" class="btn btn-ok"
+ @click="$store.browserPage.sendAnnotationsToChat()">Send now</button>
+ </div>
+ </div>
<template x-if="!$store.browserPage.frameSrc && !$store.browserPage.isBusy()">
<div class="browser-empty">
<span class="material-symbols-outlined">captive_portal</span>
@@ -549,6 +628,34 @@
padding-bottom: 1px;
}
+ .browser-annotate-toggle {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ gap: 6px;
+ min-width: 0;
+ min-height: var(--browser-control-size);
+ padding: 0 10px;
+ border: 1px solid var(--browser-chrome-border);
+ border-radius: var(--browser-control-radius);
+ background: color-mix(in srgb, var(--color-background) 26%, transparent);
+ color: color-mix(in srgb, var(--color-text) 74%, var(--color-primary) 26%);
+ font-size: 0.82rem;
+ font-weight: 650;
+ white-space: nowrap;
+ }
+
+ .browser-annotate-toggle:hover:not(:disabled),
+ .browser-annotate-toggle.is-active {
+ border-color: color-mix(in srgb, var(--color-primary) 48%, var(--browser-chrome-border));
+ background: color-mix(in srgb, var(--color-primary) 16%, var(--color-background));
+ color: var(--color-text);
+ }
+
+ .browser-annotate-toggle .material-symbols-outlined {
+ font-size: 1.05rem;
+ }
+
.browser-session-controls .browser-extensions.is-active {
color: #2e7d32;
}
@@ -813,6 +920,10 @@
outline: none;
}
+ .browser-stage.is-annotating {
+ cursor: crosshair;
+ }
+
.browser-frame {
display: block;
position: absolute;
@@ -827,6 +938,194 @@
background: #fff;
}
+ .browser-annotation-layer {
+ position: absolute;
+ inset: 0;
+ z-index: 12;
+ cursor: crosshair;
+ touch-action: none;
+ background: rgba(37, 99, 235, 0.035);
+ }
+
+ .browser-annotation-layer.is-busy {
+ cursor: progress;
+ }
+
+ .browser-annotation-box {
+ position: absolute;
+ box-sizing: border-box;
+ min-width: 8px;
+ min-height: 8px;
+ border: 2px solid #3399ff;
+ background: rgba(51, 153, 255, 0.16);
+ box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.52), 0 8px 22px rgba(0, 0, 0, 0.18);
+ pointer-events: none;
+ }
+
+ .browser-annotation-box.is-draft {
+ border-style: dashed;
+ background: rgba(51, 153, 255, 0.1);
+ }
+
+ .browser-annotation-number {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ width: 22px;
+ min-width: 22px;
+ height: 22px;
+ min-height: 22px;
+ border-radius: 50%;
+ background: #3399ff;
+ color: #fff;
+ font-size: 0.74rem;
+ font-weight: 800;
+ line-height: 1;
+ box-shadow: 0 2px 8px rgba(0, 0, 0, 0.28);
+ }
+
+ .browser-annotation-box .browser-annotation-number {
+ position: absolute;
+ top: -12px;
+ left: -12px;
+ }
+
+ .browser-annotation-popover,
+ .browser-annotation-tray {
+ position: absolute;
+ z-index: 18;
+ border: 1px solid color-mix(in srgb, var(--color-border) 76%, transparent);
+ border-radius: 7px;
+ background: color-mix(in srgb, var(--color-background) 96%, #000 4%);
+ color: var(--color-text);
+ box-shadow: 0 16px 38px rgba(0, 0, 0, 0.28);
+ }
+
+ .browser-annotation-popover {
+ display: flex;
+ flex-direction: column;
+ gap: 8px;
+ padding: 10px;
+ }
+
+ .browser-annotation-popover-title,
+ .browser-annotation-tray-header {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+ min-width: 0;
+ font-size: 0.82rem;
+ font-weight: 750;
+ }
+
+ .browser-annotation-popover textarea {
+ width: 100%;
+ min-height: 82px;
+ max-height: 160px;
+ resize: vertical;
+ padding: 8px;
+ border: 1px solid color-mix(in srgb, var(--color-border) 74%, transparent);
+ border-radius: 6px;
+ background: var(--color-input);
+ color: var(--color-text);
+ font: inherit;
+ line-height: 1.35;
+ }
+
+ .browser-annotation-actions,
+ .browser-annotation-tray-actions {
+ display: flex;
+ justify-content: flex-end;
+ gap: 7px;
+ }
+
+ .browser-annotation-actions .btn,
+ .browser-annotation-tray-actions .btn {
+ min-height: 30px;
+ padding: 0 10px;
+ white-space: nowrap;
+ }
+
+ .browser-annotation-tray {
+ right: 10px;
+ bottom: 10px;
+ display: flex;
+ flex-direction: column;
+ gap: 9px;
+ width: min(360px, calc(100% - 20px));
+ max-height: min(48%, 310px);
+ padding: 10px;
+ }
+
+ .browser-annotation-tray-header {
+ justify-content: space-between;
+ }
+
+ .browser-annotation-clear,
+ .browser-annotation-chip button {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ width: 24px;
+ height: 24px;
+ min-width: 24px;
+ min-height: 24px;
+ padding: 0;
+ border: 0;
+ border-radius: 6px;
+ background: transparent;
+ color: color-mix(in srgb, var(--color-text) 64%, transparent);
+ cursor: pointer;
+ }
+
+ .browser-annotation-clear:hover,
+ .browser-annotation-chip button:hover {
+ background: color-mix(in srgb, var(--color-panel) 82%, transparent);
+ color: var(--color-text);
+ }
+
+ .browser-annotation-clear .material-symbols-outlined,
+ .browser-annotation-chip button .material-symbols-outlined {
+ font-size: 16px;
+ }
+
+ .browser-annotation-chips {
+ display: flex;
+ flex-direction: column;
+ gap: 6px;
+ min-height: 0;
+ overflow: auto;
+ }
+
+ .browser-annotation-chip {
+ display: grid;
+ grid-template-columns: auto minmax(0, 1fr) auto;
+ align-items: center;
+ gap: 7px;
+ min-height: 32px;
+ padding: 5px 6px;
+ border: 1px solid color-mix(in srgb, var(--color-border) 54%, transparent);
+ border-radius: 7px;
+ background: color-mix(in srgb, var(--color-panel) 74%, transparent);
+ }
+
+ .browser-annotation-chip .browser-annotation-number {
+ width: 20px;
+ min-width: 20px;
+ height: 20px;
+ min-height: 20px;
+ font-size: 0.68rem;
+ }
+
+ .browser-annotation-chip-text {
+ min-width: 0;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ font-size: 0.78rem;
+ line-height: 1.25;
+ }
+
.browser-empty {
display: flex;
align-items: center;
@@ -928,6 +1227,16 @@
height: var(--browser-control-size);
}
+ .browser-annotate-toggle {
+ width: var(--browser-control-size);
+ min-width: var(--browser-control-size);
+ padding: 0;
+ }
+
+ .browser-annotate-toggle span:not(.material-symbols-outlined) {
+ display: none;
+ }
+
.browser-extension-dropdown {
right: 0;
left: auto;
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index 57252f179..1bc5bb337 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -13,6 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
const BROWSER_CONFIG_REFRESH_MS = 15000;
const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
+const ANNOTATION_DRAG_THRESHOLD = 6;
+const ANNOTATION_MAX_COMMENTS = 24;
+const ANNOTATION_DOM_LIMIT = 1200;
function makeViewerToken() {
return globalThis.crypto?.randomUUID?.()
@@ -60,6 +63,13 @@ const model = {
address: "",
frameSrc: "",
frameState: null,
+ annotating: false,
+ annotationComments: [],
+ annotationDraft: null,
+ annotationDraftText: "",
+ annotationDragRect: null,
+ annotationBusy: false,
+ annotationError: "",
connected: false,
switchingBrowserId: null,
commandInFlight: false,
@@ -76,6 +86,8 @@ const model = {
_viewportSyncTimer: null,
_lastViewportKey: "",
_lastViewport: null,
+ _annotationPointer: null,
+ _annotationSequence: 0,
_mode: "",
_surfaceMounted: false,
_surfaceSwitching: false,
@@ -613,6 +625,7 @@ const model = {
async command(command, extra = {}) {
this.error = "";
+ this.annotationError = "";
this.commandInFlight = true;
const previousActiveBrowserId = this.activeBrowserId;
try {
@@ -642,13 +655,17 @@ const model = {
this.frameState = null;
this.frameSrc = "";
}
- if (result.state?.currentUrl || result.currentUrl) {
- this.address = result.state?.currentUrl || result.currentUrl;
- }
- this.applySnapshot(data.snapshot);
- const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;
- if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
- await this.connectViewer({ browserId: this.activeBrowserId });
+ if (result.state?.currentUrl || result.currentUrl) {
+ this.address = result.state?.currentUrl || result.currentUrl;
+ }
+ this.applySnapshot(data.snapshot);
+ if (["navigate", "back", "forward", "reload", "close"].includes(String(command || "").toLowerCase())) {
+ this.clearAnnotationsForBrowser(previousActiveBrowserId);
+ this.cancelAnnotationDraft();
+ }
+ const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;
+ if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
+ await this.connectViewer({ browserId: this.activeBrowserId });
}
} catch (error) {
this.error = error instanceof Error ? error.message : String(error);
@@ -761,17 +778,22 @@ const model = {
return null;
},
- applyActiveFrameState(nextState = null) {
- if (!nextState) return;
- const stateId = this.normalizeBrowserId(nextState.id);
- if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {
- return;
+ applyActiveFrameState(nextState = null) {
+ if (!nextState) return;
+ const stateId = this.normalizeBrowserId(nextState.id);
+ if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {
+ return;
}
+ const previousUrl = String(this.frameState?.currentUrl || "");
+ const nextUrl = String(nextState.currentUrl || "");
this.frameState = nextState;
+ if (previousUrl && nextUrl && previousUrl !== nextUrl) {
+ this.cancelAnnotationDraft();
+ }
if (!this.addressFocused && nextState.currentUrl) {
this.address = nextState.currentUrl;
- }
- },
+ }
+ },
applySnapshot(snapshot = null) {
if (!snapshot?.image) return;
@@ -805,6 +827,7 @@ const model = {
if (this.activeBrowserId !== previous) {
this._lastViewportKey = "";
this._lastViewport = null;
+ this.cancelAnnotationDraft();
}
},
@@ -849,6 +872,419 @@ const model = {
};
},
+ handleKeydown(event) {
+ const annotateShortcut = event?.key === "." && (event.metaKey || event.ctrlKey) && !event.altKey;
+ if (annotateShortcut && this._surfaceMounted) {
+ event.preventDefault();
+ event.stopPropagation?.();
+ this.toggleAnnotationMode();
+ return;
+ }
+
+ if (this.annotating) {
+ if (event?.key === "Escape") {
+ event.preventDefault();
+ if (this.annotationDraft || this.annotationDragRect) {
+ this.cancelAnnotationDraft();
+ } else {
+ this.toggleAnnotationMode(false);
+ }
+ }
+ return;
+ }
+
+ void this.sendKey(event);
+ },
+
+ handleStageWheel(event) {
+ if (this.annotating) return;
+ void this.sendWheel(event);
+ },
+
+ toggleAnnotationMode(force = null) {
+ const nextValue = force === null ? !this.annotating : Boolean(force);
+ if (nextValue && !this.canAnnotate()) return;
+
+ this.annotating = nextValue;
+ this.annotationError = "";
+ this.closeExtensionsMenu();
+ if (!nextValue) {
+ this.cancelAnnotationDraft();
+ this.annotationDragRect = null;
+ this._annotationPointer = null;
+ } else {
+ this._stageElement?.focus?.({ preventScroll: true });
+ }
+ },
+
+ canAnnotate() {
+ return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());
+ },
+
+ activeAnnotationUrl() {
+ return String(this.frameState?.currentUrl || this.address || "about:blank");
+ },
+
+ visibleAnnotations() {
+ const browserId = this.normalizeBrowserId(this.activeBrowserId);
+ const url = this.activeAnnotationUrl();
+ return this.annotationComments.filter((annotation) => (
+ this.sameBrowserId(annotation.browserId, browserId)
+ && String(annotation.url || "") === url
+ ));
+ },
+
+ nextAnnotationIndex() {
+ return this.visibleAnnotations().length + 1;
+ },
+
+ clearVisibleAnnotations() {
+ this.clearAnnotationsForBrowser(this.activeBrowserId, this.activeAnnotationUrl());
+ },
+
+ clearAnnotationsForBrowser(browserId, url = null) {
+ const numericBrowserId = this.normalizeBrowserId(browserId);
+ if (!numericBrowserId) return;
+ this.annotationComments = this.annotationComments.filter((annotation) => {
+ if (!this.sameBrowserId(annotation.browserId, numericBrowserId)) return true;
+ return url ? String(annotation.url || "") !== String(url) : false;
+ });
+ },
+
+ annotationBoxStyle(rect = {}) {
+ const viewport = this.currentViewportSize() || this._lastViewport || {};
+ const width = Math.max(1, Number(viewport.width || rect.width || 1));
+ const height = Math.max(1, Number(viewport.height || rect.height || 1));
+ const normalized = this.clampAnnotationRect(rect);
+ return [
+ `left: ${(normalized.x / width) * 100}%`,
+ `top: ${(normalized.y / height) * 100}%`,
+ `width: ${(Math.max(1, normalized.width) / width) * 100}%`,
+ `height: ${(Math.max(1, normalized.height) / height) * 100}%`,
+ ].join("; ");
+ },
+
+ annotationPopoverStyle() {
+ const rect = this.annotationDraft?.rect || this.annotationDragRect || {};
+ const viewport = this.currentViewportSize() || this._lastViewport || {};
+ const width = Math.max(1, Number(viewport.width || 1));
+ const height = Math.max(1, Number(viewport.height || 1));
+ const popoverWidth = Math.min(320, Math.max(240, width - 20));
+ const popoverHeight = 190;
+ const nextLeft = Math.min(
+ Math.max(10, Number(rect.x || 0) + Number(rect.width || 0) + 10),
+ Math.max(10, width - popoverWidth - 10),
+ );
+ const nextTop = Math.min(
+ Math.max(10, Number(rect.y || 0) + Number(rect.height || 0) + 10),
+ Math.max(10, height - popoverHeight - 10),
+ );
+ return [
+ `left: ${(nextLeft / width) * 100}%`,
+ `top: ${(nextTop / height) * 100}%`,
+ `width: min(${popoverWidth}px, calc(100% - 20px))`,
+ ].join("; ");
+ },
+
+ annotationDraftTitle() {
+ if (!this.annotationDraft) return "Annotation";
+ return this.annotationDraft.kind === "area" ? "Area annotation" : "Element annotation";
+ },
+
+ stagePointForEvent(event) {
+ const image = this._stageElement?.querySelector?.(".browser-frame") || null;
+ return this.pointerCoordinatesFor(event, image);
+ },
+
+ normalizeAnnotationRect(start = {}, end = {}) {
+ const x1 = Number(start.x || 0);
+ const y1 = Number(start.y || 0);
+ const x2 = Number(end.x || x1);
+ const y2 = Number(end.y || y1);
+ return this.clampAnnotationRect({
+ x: Math.min(x1, x2),
+ y: Math.min(y1, y2),
+ width: Math.abs(x2 - x1),
+ height: Math.abs(y2 - y1),
+ });
+ },
+
+ clampAnnotationRect(rect = {}) {
+ const viewport = this.currentViewportSize() || this._lastViewport || {};
+ const viewportWidth = Math.max(1, Number(viewport.width || rect.x + rect.width || 1));
+ const viewportHeight = Math.max(1, Number(viewport.height || rect.y + rect.height || 1));
+ const x = Math.max(0, Math.min(viewportWidth, Number(rect.x || 0)));
+ const y = Math.max(0, Math.min(viewportHeight, Number(rect.y || 0)));
+ const width = Math.max(1, Math.min(viewportWidth - x, Number(rect.width || 1)));
+ const height = Math.max(1, Math.min(viewportHeight - y, Number(rect.height || 1)));
+ return {
+ x: Math.round(x),
+ y: Math.round(y),
+ width: Math.round(width),
+ height: Math.round(height),
+ };
+ },
+
+ startAnnotationSelection(event) {
+ if (!this.annotating || this.annotationBusy || !this.canAnnotate()) return;
+ const point = this.stagePointForEvent(event);
+ if (!point) return;
+ this.cancelAnnotationDraft();
+ this.annotationError = "";
+ this._annotationPointer = {
+ id: event.pointerId,
+ start: point,
+ last: point,
+ };
+ this.annotationDragRect = this.clampAnnotationRect({
+ x: point.x,
+ y: point.y,
+ width: 1,
+ height: 1,
+ });
+ event.currentTarget?.setPointerCapture?.(event.pointerId);
+ },
+
+ moveAnnotationSelection(event) {
+ if (!this.annotating || !this._annotationPointer) return;
+ if (event.pointerId !== this._annotationPointer.id) return;
+ const point = this.stagePointForEvent(event);
+ if (!point) return;
+ this._annotationPointer.last = point;
+ this.annotationDragRect = this.normalizeAnnotationRect(this._annotationPointer.start, point);
+ },
+
+ async finishAnnotationSelection(event) {
+ if (!this.annotating || !this._annotationPointer) return;
+ if (event.pointerId !== this._annotationPointer.id) return;
+ const pointer = this._annotationPointer;
+ this._annotationPointer = null;
+ event.currentTarget?.releasePointerCapture?.(event.pointerId);
+ const endPoint = this.stagePointForEvent(event) || pointer.last || pointer.start;
+ const rect = this.normalizeAnnotationRect(pointer.start, endPoint);
+ this.annotationDragRect = null;
+ const isDrag = rect.width >= ANNOTATION_DRAG_THRESHOLD || rect.height >= ANNOTATION_DRAG_THRESHOLD;
+ const point = {
+ x: Math.round(endPoint.x),
+ y: Math.round(endPoint.y),
+ };
+ const payload = {
+ kind: isDrag ? "area" : "element",
+ point,
+ rect: isDrag ? rect : null,
+ viewport: this.currentViewportSize(),
+ url: this.activeAnnotationUrl(),
+ title: this.activeTitle,
+ };
+ await this.createAnnotationDraft(payload, isDrag ? rect : {
+ x: point.x - 10,
+ y: point.y - 10,
+ width: 20,
+ height: 20,
+ });
+ },
+
+ cancelAnnotationSelection(event = null) {
+ if (event && this._annotationPointer?.id === event.pointerId) {
+ event.currentTarget?.releasePointerCapture?.(event.pointerId);
+ }
+ this._annotationPointer = null;
+ this.annotationDragRect = null;
+ },
+
+ cancelAnnotationDraft() {
+ this.annotationDraft = null;
+ this.annotationDraftText = "";
+ this.annotationDragRect = null;
+ },
+
+ async createAnnotationDraft(payload, fallbackRect) {
+ if (!this.activeBrowserId || !this.contextId) return;
+ const sequence = this._annotationSequence + 1;
+ const browserId = this.activeBrowserId;
+ const url = this.activeAnnotationUrl();
+ const title = this.activeTitle;
+ this._annotationSequence = sequence;
+ this.annotationBusy = true;
+ this.annotationError = "";
+ try {
+ const response = await websocket.request(
+ "browser_viewer_annotation",
+ {
+ context_id: this.contextId,
+ browser_id: browserId,
+ viewer_id: this._viewerToken,
+ payload,
+ },
+ { timeoutMs: 10000 },
+ );
+ if (sequence !== this._annotationSequence) return;
+ const data = firstOk(response);
+ const metadata = data.annotation || {};
+ this.annotationDraft = {
+ id: makeViewerToken(),
+ browserId,
+ url,
+ title,
+ kind: metadata.kind || payload.kind,
+ rect: this.annotationRectFromMetadata(metadata, fallbackRect),
+ metadata,
+ createdAt: Date.now(),
+ };
+ this.annotationDraftText = "";
+ } catch (error) {
+ this.annotationError = error instanceof Error ? error.message : String(error);
+ this.error = this.annotationError;
+ } finally {
+ if (sequence === this._annotationSequence) {
+ this.annotationBusy = false;
+ }
+ }
+ },
+
+ annotationRectFromMetadata(metadata = {}, fallbackRect = {}) {
+ const targetRect = metadata?.target?.rect || metadata?.rect || null;
+ return this.clampAnnotationRect(targetRect || fallbackRect);
+ },
+
+ addAnnotationComment() {
+ const comment = String(this.annotationDraftText || "").trim();
+ if (!this.annotationDraft || !comment) return;
+ if (this.visibleAnnotations().length >= ANNOTATION_MAX_COMMENTS) {
+ this.annotationError = `Keep each batch to ${ANNOTATION_MAX_COMMENTS} annotations or fewer.`;
+ this.error = this.annotationError;
+ return;
+ }
+ this.annotationComments = [
+ ...this.annotationComments,
+ {
+ ...this.annotationDraft,
+ comment,
+ index: this.nextAnnotationIndex(),
+ },
+ ];
+ this.cancelAnnotationDraft();
+ },
+
+ removeAnnotationComment(annotationId) {
+ this.annotationComments = this.annotationComments.filter((annotation) => annotation.id !== annotationId);
+ },
+
+ annotationChipLabel(annotation) {
+ const prefix = annotation?.kind === "area" ? "Area" : "Element";
+ return `${prefix} ${annotation?.index || ""}`.trim();
+ },
+
+ formatAnnotationRect(rect = {}) {
+ const normalized = this.clampAnnotationRect(rect);
+ return `x=${normalized.x}, y=${normalized.y}, width=${normalized.width}, height=${normalized.height}`;
+ },
+
+ redactAnnotationText(value) {
+ return String(value || "")
+ .replace(/(<input\b(?=[^>]*\btype=(["'])?password\2?)[^>]*?)\svalue=(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")
+ .replace(/\b(password|passcode|token|secret|value)=((["'])[\s\S]{1,240}?\3)/giu, "$1=\"[redacted]\"");
+ },
+
+ formatAnnotationMetadata(metadata = {}) {
+ const lines = [];
+ const target = metadata.target || {};
+ const selector = target.selector || metadata.selector || "";
+ const summary = target.summary || metadata.summary || "";
+ const dom = this.redactAnnotationText(target.dom || metadata.dom || "").slice(0, ANNOTATION_DOM_LIMIT);
+
+ if (selector) {
+ lines.push(`Selector: ${selector}`);
+ }
+ if (target.tagName || target.role || target.id || target.name || target.classes) {
+ lines.push([
+ "Element:",
+ target.tagName ? `<${String(target.tagName).toLowerCase()}>` : "",
+ target.role ? `role=${target.role}` : "",
+ target.id ? `id=${target.id}` : "",
+ target.name ? `name=${target.name}` : "",
+ target.classes ? `class=${target.classes}` : "",
+ ].filter(Boolean).join(" "));
+ }
+ if (summary) {
+ lines.push(`Summary: ${summary}`);
+ }
+ if (Array.isArray(metadata.elements) && metadata.elements.length) {
+ lines.push("Intersecting elements:");
+ metadata.elements.slice(0, 8).forEach((element, index) => {
+ const elementLabel = [
+ `${index + 1}.`,
+ element.tagName ? `<${String(element.tagName).toLowerCase()}>` : "",
+ element.selector || "",
+ element.summary || "",
+ ].filter(Boolean).join(" ");
+ lines.push(elementLabel);
+ });
+ }
+ if (dom) {
+ lines.push(`DOM: ${dom}`);
+ }
+ return lines.join("\n");
+ },
+
+ buildAnnotationsPrompt() {
+ const annotations = this.visibleAnnotations();
+ if (!annotations.length) return "";
+ const lines = [
+ "Browser annotations",
+ `Page title: ${this.activeTitle}`,
+ `Page URL: ${this.activeAnnotationUrl()}`,
+ `Browser id: ${this.activeBrowserId}`,
+ "",
+ ];
+ annotations.forEach((annotation, index) => {
+ lines.push(
+ `Annotation ${index + 1}`,
+ `Comment: ${annotation.comment}`,
+ `Selection kind: ${annotation.kind}`,
+ `Coordinates: ${this.formatAnnotationRect(annotation.rect)}`,
+ );
+ const metadata = this.formatAnnotationMetadata(annotation.metadata);
+ if (metadata) {
+ lines.push(metadata);
+ }
+ lines.push("");
+ });
+ return lines.join("\n").trim();
+ },
+
+ draftAnnotationsToChat() {
+ const prompt = this.buildAnnotationsPrompt();
+ if (!prompt) return;
+ const existingMessage = String(chatInputStore.message || "").trim();
+ chatInputStore.message = existingMessage ? `${existingMessage}\n\n${prompt}` : prompt;
+ chatInputStore.adjustTextareaHeight?.();
+ chatInputStore.focus?.();
+ this.clearVisibleAnnotations();
+ this.toggleAnnotationMode(false);
+ },
+
+ async sendAnnotationsToChat() {
+ const prompt = this.buildAnnotationsPrompt();
+ if (!prompt) return;
+ chatInputStore.message = prompt;
+ chatInputStore.adjustTextareaHeight?.();
+ try {
+ if (typeof chatInputStore.sendMessage === "function") {
+ await chatInputStore.sendMessage();
+ } else if (typeof globalThis.sendMessage === "function") {
+ await globalThis.sendMessage();
+ } else {
+ chatInputStore.focus?.();
+ return;
+ }
+ this.clearVisibleAnnotations();
+ this.toggleAnnotationMode(false);
+ } catch (error) {
+ this.error = error instanceof Error ? error.message : String(error);
+ }
+ },
+
currentViewportSize() {
const stage = this._stageElement;
if (!stage) return null;
@@ -910,6 +1346,7 @@ const model = {
},
async sendMouse(eventType, event) {
+ if (this.annotating) return;
if (!this.activeBrowserId || !event?.currentTarget) return;
const pointer = this.pointerCoordinatesFor(event);
if (!pointer) return;
@@ -960,6 +1397,7 @@ const model = {
},
async sendKey(event) {
+ if (this.annotating) return;
if (!this.activeBrowserId) return;
if (event.ctrlKey || event.metaKey || event.altKey) return;
const editable = ["INPUT", "TEXTAREA", "SELECT"].includes(event.target?.tagName);
@@ -982,6 +1420,11 @@ const model = {
this._surfaceMounted = false;
this._surfaceSwitching = false;
this.commandInFlight = false;
+ this.annotating = false;
+ this.annotationBusy = false;
+ this.annotationError = "";
+ this.cancelAnnotationDraft();
+ this.cancelAnnotationSelection();
if (this.contextId) {
try {
await websocket.emit("browser_viewer_unsubscribe", { context_id: this.contextId });
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index f3c65b557..9e4c40c8e 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -2,7 +2,7 @@ import asyncio
import sys
import threading
from pathlib import Path
-from types import SimpleNamespace
+from types import ModuleType, SimpleNamespace
import pytest
@@ -11,6 +11,73 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
+
+class _TestAgentContext:
+ @staticmethod
+ def get(context_id):
+ return None
+
+
+class _TestResponse(SimpleNamespace):
+ def __init__(self, message="", break_loop=False, **kwargs):
+ super().__init__(message=message, break_loop=break_loop, **kwargs)
+
+
+class _TestTool:
+ def __init__(
+ self,
+ agent=None,
+ name="",
+ method=None,
+ args=None,
+ message="",
+ loop_data=None,
+ **kwargs,
+ ):
+ self.agent = agent
+ self.name = name
+ self.method = method
+ self.args = args or {}
+ self.message = message
+ self.loop_data = loop_data
+
+
+class _TestWsHandler:
+ def __init__(self, *args, **kwargs):
+ self.emitted = []
+
+ async def emit_to(self, sid, event, data, correlation_id=None):
+ self.emitted.append((sid, event, data, correlation_id))
+
+
+class _TestWsResult(dict):
+ @staticmethod
+ def error(code="", message="", correlation_id=None):
+ return _TestWsResult(
+ {
+ "ok": False,
+ "code": code,
+ "error": message,
+ "correlation_id": correlation_id,
+ }
+ )
+
+
+sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))
+sys.modules.setdefault("helpers.tool", SimpleNamespace(Response=_TestResponse, Tool=_TestTool))
+sys.modules.setdefault("helpers.ws", SimpleNamespace(WsHandler=_TestWsHandler))
+sys.modules.setdefault("helpers.ws_manager", SimpleNamespace(WsResult=_TestWsResult))
+_model_config_stub = ModuleType("plugins._model_config.helpers.model_config")
+_model_config_stub.get_presets = lambda: []
+_model_config_stub.get_preset_by_name = lambda name: None
+_model_config_stub.get_chat_model_config = lambda agent=None: {}
+sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_config_stub)
+
+
+def anyio_backend():
+ return "asyncio"
+
from plugins._browser.helpers.config import (
build_browser_launch_config,
get_browser_main_model_summary,
@@ -63,6 +130,8 @@ def test_browser_config_normalizes_extension_paths(tmp_path):
assert config == {
"extension_paths": [str(extension_dir)],
+ "default_homepage": "about:blank",
+ "autofocus_active_page": True,
"model_preset": "",
}
@@ -246,7 +315,7 @@ def test_browser_extension_manager_uses_modern_chrome_prodversion(monkeypatch):
def test_browser_extension_menu_exposes_agent_and_url_paths():
- html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
+ html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
encoding="utf-8"
)
skill = PROJECT_ROOT / "skills" / "a0-browser-ext" / "SKILL.md"
@@ -279,7 +348,7 @@ def test_browser_viewer_allows_slow_extension_startup():
def test_browser_ui_spinners_have_browser_local_animation():
- main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
encoding="utf-8"
)
config_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "config.html").read_text(
@@ -305,7 +374,7 @@ def test_browser_extension_settings_stay_user_facing():
def test_browser_viewer_uses_tabs_for_session_switching():
- main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
encoding="utf-8"
)
browser_store = (
@@ -329,7 +398,7 @@ def test_browser_viewer_uses_cdp_screencast_transport():
ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
encoding="utf-8"
)
- main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
encoding="utf-8"
)
runtime = (
@@ -339,12 +408,12 @@ def test_browser_viewer_uses_cdp_screencast_transport():
PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
).read_text(encoding="utf-8")
- assert 'runtime.call("screenshot"' not in ws_browser
+ assert 'runtime.call("screenshot"' in ws_browser
assert "SCREENCAST_QUALITY = 92" in ws_browser
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
assert '"set_viewport"' in ws_browser
assert "start_screencast" in ws_browser
- assert "read_screencast_frame" in ws_browser
+ assert "pop_screencast_frame" in ws_browser
assert "stop_screencast" in ws_browser
assert '"Page.startScreencast"' in runtime
assert '"Page.screencastFrame"' in runtime
@@ -360,12 +429,54 @@ def test_browser_viewer_uses_cdp_screencast_transport():
assert "viewport_height: initialViewport?.height" in browser_store
assert "this.frameState = data.state || null" not in browser_store
assert "overflow: hidden;" in main_html
- assert "object-fit: fill;" not in main_html
- assert "height: auto;" in main_html
+ assert "object-fit: fill;" in main_html
assert "image-rendering: auto;" in main_html
+def test_browser_annotate_mode_ui_and_prompt_hooks():
+ panel_html = (
+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"
+ ).read_text(encoding="utf-8")
+ browser_store = (
+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
+ ).read_text(encoding="utf-8")
+
+ assert "Annotate" in panel_html
+ assert "Annotating" in panel_html
+ assert "browser-annotation-layer" in panel_html
+ assert "browser-annotation-tray" in panel_html
+ assert "Draft to chat" in panel_html
+ assert "Send now" in panel_html
+ assert "@pointerdown.stop.prevent=\"$store.browserPage.startAnnotationSelection($event)\"" in panel_html
+ assert "@keydown.window=\"$store.browserPage.handleKeydown($event)\"" in panel_html
+ assert "annotationComments: []" in browser_store
+ assert '"browser_viewer_annotation"' in browser_store
+ assert 'event?.key === "." && (event.metaKey || event.ctrlKey)' in browser_store
+ assert "Browser annotations" in browser_store
+ assert "Comment:" in browser_store
+ assert "Coordinates:" in browser_store
+ assert "Selector:" in browser_store
+ assert "DOM:" in browser_store
+ assert "value=\\\"[redacted]\\\"" in browser_store
+
+
+def test_browser_runtime_and_content_helper_expose_annotation_target():
+ runtime = (
+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
+ ).read_text(encoding="utf-8")
+ helper = (
+ PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"
+ ).read_text(encoding="utf-8")
+
+ assert "async def annotation_target" in runtime
+ assert "globalThis.__spaceBrowserPageContent__.annotate(payload || null)" in runtime
+ assert "function annotate(payload = null)" in helper
+ assert "annotate," in helper
+ assert "sanitizeAnnotationDom" in helper
+ assert "password" in helper
+
+
async def test_browser_screencast_acknowledges_and_drops_stale_frames():
first_image = (
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
@@ -526,7 +637,7 @@ def test_browser_save_plugin_config_does_not_restart_runtimes_for_preset_only(mo
assert restarted == []
async def test_browser_tool_dispatches_direct_actions(monkeypatch):
calls = []
@@ -558,7 +669,7 @@ async def test_browser_tool_dispatches_direct_actions(monkeypatch):
assert calls == [("content", (1, None))]
async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):
class FakeRuntime:
def __init__(self) -> None:
@@ -608,7 +719,7 @@ async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):
assert ("sid-1", "ctx") not in ws_browser_module.WsBrowser._streams
async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
calls = []
@@ -642,11 +753,14 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
"sid-1",
)
- assert result == {"state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)}}
+ assert result == {
+ "state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)},
+ "snapshot": None,
+ }
assert calls == [("set_viewport", (7, 1280, 720), {})]
async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):
calls = []
@@ -682,10 +796,68 @@ async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):
"sid-1",
)
- assert result == {"state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)}}
+ assert result == {
+ "state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)},
+ "snapshot": None,
+ }
assert calls == [("wheel", (3, 320.0, 480.0, 0.0, 640.0), {})]
+async def test_browser_viewer_annotation_dispatches_runtime(monkeypatch):
+ calls = []
+
+ class FakeRuntime:
+ async def call(self, method, *args, **kwargs):
+ calls.append((method, args, kwargs))
+ return {
+ "kind": "element",
+ "point": {"x": 320, "y": 180},
+ "target": {"tagName": "BUTTON", "selector": "#save"},
+ }
+
+ async def fake_get_runtime(context_id, create=True):
+ assert context_id == "ctx"
+ assert create is False
+ return FakeRuntime()
+
+ monkeypatch.setattr(ws_browser_module, "get_runtime", fake_get_runtime)
+
+ handler = ws_browser_module.WsBrowser(
+ SimpleNamespace(),
+ threading.RLock(),
+ manager=None,
+ )
+
+ payload = {
+ "kind": "element",
+ "point": {"x": 320, "y": 180},
+ "viewport": {"width": 1280, "height": 720},
+ }
+ result = await handler.process(
+ "browser_viewer_annotation",
+ {
+ "context_id": "ctx",
+ "browser_id": 4,
+ "viewer_id": "viewer-1",
+ "payload": payload,
+ },
+ "sid-1",
+ )
+
+ assert result == {
+ "annotation": {
+ "kind": "element",
+ "point": {"x": 320, "y": 180},
+ "target": {"tagName": "BUTTON", "selector": "#save"},
+ },
+ "context_id": "ctx",
+ "browser_id": 4,
+ "viewer_id": "viewer-1",
+ }
+ assert calls == [("annotation_target", (4, payload), {})]
+
+
def test_browser_cleanup_extensions_follow_extensible_path_layout():
extension = __import__("helpers.extension", fromlist=["_get_extension_classes"])
remove_classes = extension._get_extension_classes( # type: ignore[attr-defined]
diff --git a/tests/test_webui_extension_surfaces.py b/tests/test_webui_extension_surfaces.py
index b0ca89f46..0536c4905 100644
--- a/tests/test_webui_extension_surfaces.py
+++ b/tests/test_webui_extension_surfaces.py
@@ -5,6 +5,7 @@ import tempfile
import threading
from contextlib import contextmanager
from pathlib import Path
+from types import SimpleNamespace
from typing import Iterator
import pytest
@@ -14,6 +15,15 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
+
+class _TestAgentContext:
+ @staticmethod
+ def get(context_id):
+ return None
+
+
+sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))
+
from api.load_webui_extensions import LoadWebuiExtensions
@@ -69,6 +79,11 @@ def _new_handler() -> LoadWebuiExtensions:
return LoadWebuiExtensions(app, threading.RLock())
+def anyio_backend():
+ return "asyncio"
+
+
def _assert_surface_anchor_in_template(surface: str, template_rel_path: str) -> None:
template_path = PROJECT_ROOT / template_rel_path
template_html = template_path.read_text(encoding="utf-8")
@@ -118,7 +133,7 @@ def _temporary_probe_plugin(surface: str) -> Iterator[tuple[str, str]]:
cache.clear("*(plugins)*")
@pytest.mark.parametrize(
("surface", "template_rel_path"),
SURFACE_SCENARIOS,