Agent Zero AI framework
Add Browser v1 explicit screenshot and form actions
Alessandrod3c249cbdd2a
diff --git a/plugins/_browser/assets/browser-page-content.js b/plugins/_browser/assets/browser-page-content.jsindex b31b27e0a..5fd628bce 100644--- a/plugins/_browser/assets/browser-page-content.js+++ b/plugins/_browser/assets/browser-page-content.js@@ -1,7 +1,7 @@(() => {const GLOBAL_KEY = "__spaceBrowserPageContent__";const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";- const VERSION = "9";+ const VERSION = "11";const BLOCK_TAGS = new Set(["ADDRESS","ARTICLE",@@ -779,6 +779,44 @@return INTERACTIVE_ROLES.has(role) || hasInteractiveEventHandler(element);}+ function isFileInputElement(element) {+ return getTagName(element) === "INPUT"+ && String(element.getAttribute?.("type") || element.type || "").toLowerCase() === "file";+ }++ function getAssociatedLabelFileInput(labelElement) {+ if (getTagName(labelElement) !== "LABEL") {+ return null;+ }++ if (isFileInputElement(labelElement.control)) {+ return labelElement.control;+ }++ const descendantInput = labelElement.querySelector?.("input[type='file']");+ if (isFileInputElement(descendantInput)) {+ return descendantInput;+ }++ const forId = normalizeAttributeText(labelElement.getAttribute?.("for"));+ if (!forId) {+ return null;+ }++ return isFileInputElement(labelElement.ownerDocument?.getElementById?.(forId))+ ? labelElement.ownerDocument.getElementById(forId)+ : null;+ }++ function isFileInputLabel(element) {+ if (getTagName(element) !== "LABEL" || isHiddenElement(element)) {+ return false;+ }++ const input = getAssociatedLabelFileInput(element);+ return Boolean(input && isHiddenElement(input));+ }+function getComputedStyleSafe(element) {try {return globalThis.getComputedStyle?.(element) || null;@@ -1048,6 +1086,33 @@return readableText || normalizeText(element?.textContent || "");}+ function isLabelableControlForText(element) {+ return ["BUTTON", "INPUT", "METER", "OUTPUT", "PROGRESS", "SELECT", "TEXTAREA"].includes(getTagName(element));+ }++ function getLabelElementText(labelElement, controlElement = null) {+ const collect = (node) => {+ if (isTextNode(node)) {+ return node.textContent || "";+ }++ if (!isElementNode(node) || isHiddenElement(node)) {+ return "";+ }++ if (node !== labelElement && (node === controlElement || isLabelableControlForText(node))) {+ return "";+ }++ return getReadableChildNodes(node)+ .map((childNode) => collect(childNode))+ .filter(Boolean)+ .join(" ");+ };++ return normalizeText(collect(labelElement)) || getElementText(labelElement);+ }+function collectLabelCandidates(element, options = {}) {const includeAlt = options.includeAlt !== false;const includeDescendantImageAlt = options.includeDescendantImageAlt !== false;@@ -1058,7 +1123,7 @@try {if (Array.isArray(element?.labels) || typeof element?.labels?.forEach === "function") {element.labels.forEach((labelElement) => {- const text = getElementText(labelElement);+ const text = getLabelElementText(labelElement, element);if (text) {collectedLabels.push(text);}@@ -1193,6 +1258,10 @@return `input ${inputType || "text"}`;}+ if (tagName === "LABEL" && isFileInputLabel(element)) {+ return "file input label";+ }+if (String(element.getAttribute?.("contenteditable") || "").toLowerCase() === "true") {return "editable";}@@ -1493,7 +1562,47 @@}function isReferenceableElement(element) {- return isInteractiveElement(element) || getTagName(element) === "IMG";+ return isInteractiveElement(element) || getTagName(element) === "IMG" || isFileInputLabel(element);+ }++ function collectLabelControlElements(labelElement) {+ const controls = [];+ const seen = new Set();+ const addControl = (element) => {+ if (!isElementNode(element) || seen.has(element) || !isReferenceableElement(element)) {+ return;+ }++ seen.add(element);+ controls.push(element);+ };++ [+ "input",+ "textarea",+ "select",+ "button",+ "summary",+ "a[href]",+ "[role]",+ "[contenteditable='true']",+ "[contenteditable='']"+ ].forEach((selector) => {+ try {+ [...(labelElement.querySelectorAll?.(selector) || [])].forEach(addControl);+ } catch {+ // Ignore unsupported selectors in unusual DOMs.+ }+ });++ return controls;+ }++ function renderControlLabelReferences(labelElement, context) {+ return collectLabelControlElements(labelElement)+ .map((controlElement) => renderReference(controlElement, context))+ .filter(Boolean)+ .join("\n");}function renderInlineNode(node, context) {@@ -1517,7 +1626,7 @@const tagName = getTagName(node);if (tagName === "LABEL" && (node.getAttribute?.("for") || node.querySelector?.("input, textarea, select, button"))) {- return "";+ return renderControlLabelReferences(node, context);}if (tagName === "BR") {@@ -1693,7 +1802,7 @@const tagName = getTagName(element);if (tagName === "LABEL" && (element.getAttribute?.("for") || element.querySelector?.("input, textarea, select, button"))) {- return "";+ return renderControlLabelReferences(element, context);}if (/^H[1-6]$/u.test(tagName)) {@@ -2175,6 +2284,443 @@};}+ function pointFor(referenceId, offsets = {}) {+ const entry = requireReferenceEntry(referenceId, {+ actionLabel: "point",+ requireConnected: true+ });+ if (entry.helperBacked || !entry.element) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content cannot resolve point for helper-backed reference "${entry.referenceId}".`,+ {+ code: "browser_page_content_point_helper_backed"+ }+ );+ }++ const element = entry.element;+ scrollElementIntoView(element);+ const rect = getElementRectSafe(element);+ if (!rect || rect.width <= 0 || rect.height <= 0) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content reference "${entry.referenceId}" has no visible viewport box.`,+ {+ code: "browser_page_content_point_no_box"+ }+ );+ }++ const offsetX = Number(offsets?.offset_x ?? offsets?.offsetX ?? 0) || 0;+ const offsetY = Number(offsets?.offset_y ?? offsets?.offsetY ?? 0) || 0;+ const useOffsets = offsets?.useOffsets === true || offsetX !== 0 || offsetY !== 0;+ return {+ rect,+ selector: computeStableSelector(element),+ x: rect.x + (useOffsets ? offsetX : rect.width / 2),+ y: rect.y + (useOffsets ? offsetY : rect.height / 2)+ };+ }++ function normalizeActionValues(valueOrValues) {+ if (Array.isArray(valueOrValues)) {+ return valueOrValues.map((value) => String(value ?? ""));+ }++ if (valueOrValues === null || valueOrValues === undefined) {+ return [];+ }++ return [String(valueOrValues)];+ }++ function optionMatchesValue(option, value) {+ const normalizedValue = normalizeText(value);+ const candidates = [+ option?.value,+ option?.label,+ option?.textContent,+ option?.getAttribute?.("aria-label"),+ option?.getAttribute?.("data-value"),+ option?.getAttribute?.("id")+ ].map((candidate) => normalizeText(candidate));+ return candidates.some((candidate) => candidate === normalizedValue);+ }++ function findNativeSelectOption(selectElement, value) {+ const options = [...(selectElement.options || [])];+ return options.find((option) => optionMatchesValue(option, value)) || null;+ }++ function setNativeChecked(element, checked) {+ const descriptor = Object.getOwnPropertyDescriptor(globalThis.HTMLInputElement?.prototype || {}, "checked");+ if (typeof descriptor?.set === "function") {+ descriptor.set.call(element, Boolean(checked));+ } else {+ element.checked = Boolean(checked);+ }+ }++ async function selectNativeElement(entry, values) {+ const element = entry.element;+ const beforeSnapshot = captureActionEffectSnapshot(element);+ const requestedValues = values.length ? values : [""];+ const appliedValues = [];++ const {+ observedMutations+ } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {+ scrollElementIntoView(element);+ focusElement(element);++ if (element.multiple) {+ const matchedOptions = requestedValues.map((requestedValue) => {+ const option = findNativeSelectOption(element, requestedValue);+ if (!option) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content could not find select option "${requestedValue}".`,+ {+ code: "browser_page_content_select_option_not_found"+ }+ );+ }+ return option;+ });+ const matchedSet = new Set(matchedOptions);+ [...(element.options || [])].forEach((option) => {+ option.selected = matchedSet.has(option);+ });+ matchedOptions.forEach((option) => appliedValues.push(option.value));+ } else {+ const option = findNativeSelectOption(element, requestedValues[0]);+ if (!option) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content could not find select option "${requestedValues[0]}".`,+ {+ code: "browser_page_content_select_option_not_found"+ }+ );+ }+ appliedValues.push(setNativeValue(element, option.value));+ }++ dispatchDomEvent(element, "input", "InputEvent", {+ inputType: "insertReplacementText"+ });+ dispatchDomEvent(element, "change");+ return appliedValues.slice();+ });++ refreshReferenceEntry(entry);+ return buildActionResult(entry, {+ ...buildActionEffectResult(entry, beforeSnapshot, captureActionEffectSnapshot(element), observedMutations),+ values: appliedValues.slice()+ });+ }++ function ariaOptionMatchesValue(option, value) {+ const normalizedValue = normalizeText(value);+ const candidates = [+ option?.getAttribute?.("aria-label"),+ option?.getAttribute?.("data-value"),+ option?.getAttribute?.("value"),+ option?.getAttribute?.("id"),+ getElementText(option)+ ].map((candidate) => normalizeText(candidate));+ return candidates.some((candidate) => candidate === normalizedValue);+ }++ function visibleAriaOptions(root) {+ const scope = isElementNode(root) && String(root.getAttribute?.("role") || "").trim().toLowerCase() === "listbox"+ ? root+ : globalThis.document;+ try {+ return [...(scope.querySelectorAll?.("[role='option']") || [])]+ .filter((option) => isElementNode(option) && !isHiddenElement(option));+ } catch {+ return [];+ }+ }++ function findAriaOption(root, value) {+ const matches = visibleAriaOptions(root).filter((option) => ariaOptionMatchesValue(option, value));+ return matches.length === 1 ? matches[0] : null;+ }++ async function selectAriaElement(entry, values) {+ const element = entry.element;+ const role = String(element.getAttribute?.("role") || "").trim().toLowerCase();+ if (!["combobox", "listbox"].includes(role)) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content cannot select options on <${getTagName(element).toLowerCase()}>.`,+ {+ code: "browser_page_content_select_unsupported"+ }+ );+ }++ const beforeSnapshot = captureActionEffectSnapshot(element);+ const requestedValues = values.length ? values : [""];+ const appliedValues = [];+ const {+ observedMutations+ } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {+ scrollElementIntoView(element);+ focusElement(element);+ if (role === "combobox") {+ dispatchDomEvent(element, "mousedown", "MouseEvent", { button: 0 });+ if (typeof element.click === "function") {+ element.click();+ } else {+ dispatchDomEvent(element, "click", "MouseEvent", { button: 0 });+ }+ await delayMs(80);+ }++ for (const requestedValue of requestedValues) {+ const option = findAriaOption(element, requestedValue);+ if (!option) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content could not safely find one ARIA option "${requestedValue}".`,+ {+ code: "browser_page_content_aria_option_not_found"+ }+ );+ }+ scrollElementIntoView(option);+ dispatchDomEvent(option, "mousedown", "MouseEvent", { button: 0 });+ if (typeof option.click === "function") {+ option.click();+ } else {+ dispatchDomEvent(option, "click", "MouseEvent", { button: 0 });+ }+ appliedValues.push(requestedValue);+ await delayMs(40);+ }+ });++ refreshReferenceEntry(entry);+ return buildActionResult(entry, {+ ...buildActionEffectResult(entry, beforeSnapshot, captureActionEffectSnapshot(element), observedMutations),+ values: appliedValues.slice()+ });+ }++ async function selectReference(referenceId, valueOrValues) {+ const entry = requireReferenceEntry(referenceId, {+ actionLabel: "select"+ });+ if (entry.helperBacked) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content cannot select helper-backed reference "${entry.referenceId}".`,+ {+ code: "browser_page_content_select_helper_backed"+ }+ );+ }++ const element = entry.element;+ const values = normalizeActionValues(valueOrValues);+ if (getTagName(element) === "SELECT") {+ return selectNativeElement(entry, values);+ }++ return selectAriaElement(entry, values);+ }++ function checkedStateForElement(element) {+ const tagName = getTagName(element);+ const role = String(element.getAttribute?.("role") || "").trim().toLowerCase();+ if (tagName === "INPUT") {+ const inputType = String(element.getAttribute?.("type") || element.type || "").toLowerCase();+ if (["checkbox", "radio"].includes(inputType)) {+ return Boolean(element.checked);+ }+ }+ if (["checkbox", "radio", "switch", "menuitemcheckbox", "menuitemradio"].includes(role)) {+ return String(element.getAttribute?.("aria-checked") || "").trim().toLowerCase() === "true";+ }+ if (role === "button" && element.hasAttribute?.("aria-pressed")) {+ return String(element.getAttribute?.("aria-pressed") || "").trim().toLowerCase() === "true";+ }+ return null;+ }++ async function setCheckedReference(referenceId, checked = true) {+ const entry = requireReferenceEntry(referenceId, {+ actionLabel: "setChecked"+ });+ if (entry.helperBacked) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content cannot set helper-backed reference "${entry.referenceId}".`,+ {+ code: "browser_page_content_checked_helper_backed"+ }+ );+ }++ const element = entry.element;+ const beforeSnapshot = captureActionEffectSnapshot(element);+ const desiredChecked = Boolean(checked);+ const tagName = getTagName(element);+ const role = String(element.getAttribute?.("role") || "").trim().toLowerCase();+ const currentChecked = checkedStateForElement(element);+ if (currentChecked === null) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content cannot set checked state on <${tagName.toLowerCase()}>.`,+ {+ code: "browser_page_content_checked_unsupported"+ }+ );+ }++ const {+ observedMutations+ } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {+ scrollElementIntoView(element);+ focusElement(element);++ if (tagName === "INPUT") {+ setNativeChecked(element, desiredChecked);+ dispatchDomEvent(element, "input", "InputEvent", {+ inputType: "insertReplacementText"+ });+ dispatchDomEvent(element, "change");+ } else if (["checkbox", "radio", "switch", "menuitemcheckbox", "menuitemradio"].includes(role)) {+ if (currentChecked !== desiredChecked) {+ if (typeof element.click === "function") {+ element.click();+ } else {+ dispatchDomEvent(element, "click", "MouseEvent", {+ button: 0+ });+ }+ await delayMs(40);+ }+ if (checkedStateForElement(element) !== desiredChecked) {+ element.setAttribute("aria-checked", desiredChecked ? "true" : "false");+ dispatchDomEvent(element, "input", "InputEvent", {+ inputType: "insertReplacementText"+ });+ dispatchDomEvent(element, "change");+ }+ } else if (role === "button" && element.hasAttribute?.("aria-pressed")) {+ if (currentChecked !== desiredChecked) {+ if (typeof element.click === "function") {+ element.click();+ } else {+ dispatchDomEvent(element, "click", "MouseEvent", {+ button: 0+ });+ }+ await delayMs(40);+ }+ if (checkedStateForElement(element) !== desiredChecked) {+ element.setAttribute("aria-pressed", desiredChecked ? "true" : "false");+ }+ }+ });++ refreshReferenceEntry(entry);+ return buildActionResult(entry, {+ ...buildActionEffectResult(entry, beforeSnapshot, captureActionEffectSnapshot(element), observedMutations),+ checked: desiredChecked+ });+ }++ function resolveFileInputElement(referenceId) {+ const entry = requireReferenceEntry(referenceId, {+ actionLabel: "fileInput"+ });+ if (entry.helperBacked) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content cannot upload files through helper-backed reference "${entry.referenceId}".`,+ {+ code: "browser_page_content_file_input_helper_backed"+ }+ );+ }++ const element = entry.element;+ const isFileInput = (candidate) => {+ return getTagName(candidate) === "INPUT"+ && String(candidate.getAttribute?.("type") || candidate.type || "").toLowerCase() === "file";+ };++ if (isFileInput(element)) {+ return element;+ }++ if (getTagName(element) === "LABEL") {+ if (isFileInput(element.control)) {+ return element.control;+ }+ const labelledInput = element.querySelector?.("input[type='file']");+ if (isFileInput(labelledInput)) {+ return labelledInput;+ }+ const forId = normalizeAttributeText(element.getAttribute?.("for"));+ if (forId) {+ const byId = element.ownerDocument?.getElementById?.(forId);+ if (isFileInput(byId)) {+ return byId;+ }+ }+ }++ const descendantInput = element.querySelector?.("input[type='file']");+ if (isFileInput(descendantInput)) {+ return descendantInput;+ }++ const closestLabel = element.closest?.("label");+ if (closestLabel) {+ if (isFileInput(closestLabel.control)) {+ return closestLabel.control;+ }+ const labelledInput = closestLabel.querySelector?.("input[type='file']");+ if (isFileInput(labelledInput)) {+ return labelledInput;+ }+ }++ return null;+ }++ function fileInputFor(referenceId) {+ const input = resolveFileInputElement(referenceId);+ if (!input) {+ throw createNamedError(+ "BrowserPageContentActionError",+ `Browser page content reference "${normalizeReferenceId(referenceId)}" is not a file input or associated label.`,+ {+ code: "browser_page_content_file_input_not_found"+ }+ );+ }+ return {+ accept: normalizeAttributeText(input.getAttribute?.("accept")),+ multiple: Boolean(input.multiple),+ name: normalizeAttributeText(input.getAttribute?.("name")),+ selector: computeStableSelector(input),+ tagName: getTagName(input),+ type: String(input.getAttribute?.("type") || input.type || "").toLowerCase()+ };+ }++ function fileInputElementFor(referenceId) {+ return resolveFileInputElement(referenceId);+ }+function refreshReferenceEntry(entry) {if (!entry || entry.helperBacked || !entry.element) {return entry;@@ -3403,6 +3949,15 @@return typeAndSubmit(referenceId, value);},boundingBoxFor,+ fileInputElementFor,+ fileInputFor,+ pointFor,+ select(referenceId, valueOrValues) {+ return selectReference(referenceId, valueOrValues);+ },+ setChecked(referenceId, checked) {+ return setCheckedReference(referenceId, checked);+ },version: VERSION};})();diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 3b98d4277..6d443bf4b 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -636,6 +636,54 @@ class _BrowserRuntimeCore:)return normalized+ @staticmethod+ def _has_reference(reference_id: int | str | None) -> bool:+ return reference_id is not None and str(reference_id).strip() != ""++ def _screenshot_output_path(self, browser_id: int, path: str = "") -> tuple[Path, str, str]:+ raw_path = str(path or "").strip()+ if raw_path:+ output_path = Path(files.fix_dev_path(raw_path) if raw_path.startswith("/a0/") else raw_path)+ if not output_path.is_absolute():+ output_path = Path(files.get_abs_path(str(output_path)))+ suffix = output_path.suffix.lower()+ if suffix == ".png":+ return output_path, "png", "image/png"+ if suffix not in {".jpg", ".jpeg"}:+ output_path = output_path.with_suffix(".jpg") if not suffix else output_path.with_name(f"{output_path.name}.jpg")+ return output_path, "jpeg", "image/jpeg"++ timestamp = time.strftime("%Y%m%d-%H%M%S")+ millis = int((time.time() % 1) * 1000)+ output_path = self.screenshots_dir / f"browser-{int(browser_id)}-{timestamp}-{millis:03d}.jpg"+ return output_path, "jpeg", "image/jpeg"++ @staticmethod+ def _normalize_upload_paths(path: str = "", paths: list[str] | None = None) -> list[str]:+ raw_paths: list[str] = []+ if paths:+ if not isinstance(paths, list):+ raise ValueError("paths must be a list of file paths")+ raw_paths.extend(str(item or "").strip() for item in paths)+ if str(path or "").strip():+ raw_paths.append(str(path or "").strip())++ normalized_paths: list[str] = []+ for raw_path in raw_paths:+ if not raw_path:+ continue+ candidate = Path(files.fix_dev_path(raw_path) if raw_path.startswith("/a0/") else raw_path)+ if not candidate.is_absolute():+ candidate = Path(files.get_abs_path(str(candidate)))+ candidate = candidate.expanduser().resolve()+ if not candidate.is_file():+ raise FileNotFoundError(f"Upload file does not exist: {candidate}")+ normalized_paths.append(str(candidate))++ if not normalized_paths:+ raise ValueError("upload_file requires path or non-empty paths")+ return normalized_paths+@staticmethoddef _multi_group_key(call: dict[str, Any]) -> Any:value = call.get("browser_id")@@ -657,6 +705,10 @@ class _BrowserRuntimeCore:def downloads_dir(self) -> Path:return Path(files.get_abs_path("usr/downloads/browser"))+ @property+ def screenshots_dir(self) -> Path:+ return Path(files.get_abs_path("tmp/browser/screenshots", self.safe_context_id))+async def ensure_started(self) -> None:if self._context_is_alive():return@@ -915,6 +967,13 @@ class _BrowserRuntimeCore:bid = call.get("browser_id")if action == "open":return await self.open(call.get("url") or "")+ if action == "screenshot":+ return await self.screenshot_file(+ bid,+ quality=int(call.get("quality") or 80),+ full_page=bool(call.get("full_page")),+ path=call.get("path") or "",+ )if action == "list":return await self.list(include_content=bool(call.get("include_content")))if action == "state":@@ -986,6 +1045,118 @@ class _BrowserRuntimeCore:button=call.get("button") or "left",modifiers=self._normalize_modifiers(call.get("modifiers")),)+ if action == "hover":+ return await self.hover(+ bid,+ ref=call.get("ref"),+ x=float(call.get("x") or 0),+ y=float(call.get("y") or 0),+ offset_x=float(call.get("offset_x") or 0),+ offset_y=float(call.get("offset_y") or 0),+ )+ if action == "double_click":+ return await self.double_click(+ bid,+ ref=call.get("ref"),+ x=float(call.get("x") or 0),+ y=float(call.get("y") or 0),+ button=call.get("button") or "left",+ modifiers=self._normalize_modifiers(call.get("modifiers")),+ offset_x=float(call.get("offset_x") or 0),+ offset_y=float(call.get("offset_y") or 0),+ )+ if action == "right_click":+ return await self.right_click(+ bid,+ ref=call.get("ref"),+ x=float(call.get("x") or 0),+ y=float(call.get("y") or 0),+ modifiers=self._normalize_modifiers(call.get("modifiers")),+ offset_x=float(call.get("offset_x") or 0),+ offset_y=float(call.get("offset_y") or 0),+ )+ if action == "drag":+ return await self.drag(+ bid,+ ref=call.get("ref"),+ target_ref=call.get("target_ref"),+ x=float(call.get("x") or 0),+ y=float(call.get("y") or 0),+ to_x=float(call.get("to_x") or 0),+ to_y=float(call.get("to_y") or 0),+ offset_x=float(call.get("offset_x") or 0),+ offset_y=float(call.get("offset_y") or 0),+ target_offset_x=float(call.get("target_offset_x") or 0),+ target_offset_y=float(call.get("target_offset_y") or 0),+ )+ if action == "wheel":+ return await self.wheel(+ bid,+ float(call.get("x") or 0),+ float(call.get("y") or 0),+ float(call.get("delta_x") or 0),+ float(call.get("delta_y") or 0),+ )+ if action == "keyboard":+ return await self.keyboard(+ bid,+ key=str(call.get("key") or ""),+ text=str(call.get("text") or ""),+ )+ if action == "clipboard":+ clipboard_action = str(+ call.get("clipboard_action")+ or call.get("operation")+ or call.get("event_type")+ or ""+ ).strip().lower()+ return await self.clipboard(+ bid,+ action=clipboard_action,+ text=str(call.get("text") or ""),+ )+ if action in {"copy", "cut", "paste"}:+ return await self.clipboard(+ bid,+ action=action,+ text=str(call.get("text") or ""),+ )+ if action == "set_viewport":+ return await self.set_viewport(+ bid,+ int(call.get("width") or 0),+ int(call.get("height") or 0),+ )+ if action == "select_option":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("select_option requires ref")+ return await self.select_option(+ bid,+ ref,+ value=str(call.get("value") or ""),+ values=call.get("values"),+ )+ if action == "set_checked":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("set_checked requires ref")+ checked = call.get("checked")+ return await self.set_checked(+ bid,+ ref,+ checked=True if checked is None else bool(checked),+ )+ if action == "upload_file":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("upload_file requires ref")+ return await self.upload_file(+ bid,+ ref,+ path=call.get("path") or "",+ paths=call.get("paths"),+ )if action == "close":return await self.close_browser(bid)if action == "close_all":@@ -1342,6 +1513,43 @@ class _BrowserRuntimeCore:"state": await self._state(resolved_id),}+ async def screenshot_file(+ self,+ browser_id: int | str | None = None,+ *,+ quality: int = 80,+ full_page: bool = False,+ path: str = "",+ ) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ output_path, image_type, mime = self._screenshot_output_path(resolved_id, path)+ output_path.parent.mkdir(parents=True, exist_ok=True)+ clamped_quality = max(20, min(95, int(quality)))+ screenshot_kwargs: dict[str, Any] = {+ "path": str(output_path),+ "type": image_type,+ "full_page": bool(full_page),+ }+ if image_type == "jpeg":+ screenshot_kwargs["quality"] = clamped_quality+ await page.screenshot(**screenshot_kwargs)+ local_path = str(output_path)+ return {+ "browser_id": resolved_id,+ "path": local_path,+ "a0_path": files.normalize_a0_path(local_path),+ "mime": mime,+ "state": await self._state(resolved_id),+ "vision_load": {+ "tool_name": "vision_load",+ "tool_args": {+ "paths": [local_path],+ },+ },+ }+async def start_screencast(self,browser_id: int | str | None = None,@@ -1448,6 +1656,311 @@ class _BrowserRuntimeCore:def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:return _nudged_viewport(viewport)+ async def _point_for(+ self,+ page: Any,+ reference_id: int | str,+ *,+ offset_x: float = 0,+ offset_y: float = 0,+ ) -> dict[str, Any]:+ await self._ensure_content_helper(page)+ point = await page.evaluate(+ "(args) => globalThis.__spaceBrowserPageContent__.pointFor(args.ref, args.offsets)",+ {+ "ref": reference_id,+ "offsets": {+ "offset_x": float(offset_x),+ "offset_y": float(offset_y),+ "useOffsets": bool(offset_x or offset_y),+ },+ },+ )+ if not point or not isinstance(point, dict):+ raise ValueError(f"Could not resolve Browser ref {reference_id!r} to a viewport point")+ return point++ async def _input_point(+ self,+ page: Any,+ reference_id: int | str | None,+ *,+ x: float = 0,+ y: float = 0,+ offset_x: float = 0,+ offset_y: float = 0,+ ) -> dict[str, Any]:+ if self._has_reference(reference_id):+ return await self._point_for(+ page,+ reference_id,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ return {+ "x": float(x),+ "y": float(y),+ "rect": None,+ "selector": None,+ }++ async def hover(+ self,+ browser_id: int | str | None,+ ref: int | str | None = None,+ x: float = 0,+ y: float = 0,+ offset_x: float = 0,+ offset_y: float = 0,+ ) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ point = await self._input_point(+ page,+ ref,+ x=x,+ y=y,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ await page.mouse.move(float(point["x"]), float(point["y"]))+ self._maybe_promote(resolved_id)+ return {+ "action": {+ "point": point,+ "ref": ref if self._has_reference(ref) else None,+ },+ "state": await self._state(resolved_id),+ }++ async def double_click(+ self,+ browser_id: int | str | None,+ ref: int | str | None = None,+ x: float = 0,+ y: float = 0,+ button: str = "left",+ modifiers: list[str] | str | None = None,+ offset_x: float = 0,+ offset_y: float = 0,+ ) -> dict[str, Any]:+ modifiers = self._normalize_modifiers(modifiers)+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ point = await self._input_point(+ page,+ ref,+ x=x,+ y=y,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ pressed: list[str] = []+ try:+ if modifiers:+ for mod in modifiers:+ await page.keyboard.down(mod)+ pressed.append(mod)+ await page.mouse.dblclick(float(point["x"]), float(point["y"]), button=button or "left")+ finally:+ for mod in reversed(pressed):+ with contextlib.suppress(Exception):+ await page.keyboard.up(mod)+ await self._settle(page, short=True)+ self._maybe_promote(resolved_id)+ return {+ "action": {+ "button": button or "left",+ "modifiers": modifiers or [],+ "point": point,+ "ref": ref if self._has_reference(ref) else None,+ },+ "state": await self._state(resolved_id),+ }++ async def right_click(+ self,+ browser_id: int | str | None,+ ref: int | str | None = None,+ x: float = 0,+ y: float = 0,+ modifiers: list[str] | str | None = None,+ offset_x: float = 0,+ offset_y: float = 0,+ ) -> dict[str, Any]:+ modifiers = self._normalize_modifiers(modifiers)+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ point = await self._input_point(+ page,+ ref,+ x=x,+ y=y,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ pressed: list[str] = []+ try:+ if modifiers:+ for mod in modifiers:+ await page.keyboard.down(mod)+ pressed.append(mod)+ await page.mouse.click(float(point["x"]), float(point["y"]), button="right")+ finally:+ for mod in reversed(pressed):+ with contextlib.suppress(Exception):+ await page.keyboard.up(mod)+ await self._settle(page, short=True)+ self._maybe_promote(resolved_id)+ return {+ "action": {+ "button": "right",+ "modifiers": modifiers or [],+ "point": point,+ "ref": ref if self._has_reference(ref) else None,+ },+ "state": await self._state(resolved_id),+ }++ async def drag(+ self,+ browser_id: int | str | None,+ ref: int | str | None = None,+ target_ref: int | str | None = None,+ x: float = 0,+ y: float = 0,+ to_x: float = 0,+ to_y: float = 0,+ offset_x: float = 0,+ offset_y: float = 0,+ target_offset_x: float = 0,+ target_offset_y: float = 0,+ ) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ start_point = await self._input_point(+ page,+ ref,+ x=x,+ y=y,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ end_point = await self._input_point(+ page,+ target_ref,+ x=to_x,+ y=to_y,+ offset_x=target_offset_x,+ offset_y=target_offset_y,+ )+ await page.mouse.move(float(start_point["x"]), float(start_point["y"]))+ await page.mouse.down()+ await page.mouse.move(float(end_point["x"]), float(end_point["y"]), steps=12)+ await page.mouse.up()+ await self._settle(page, short=True)+ self._maybe_promote(resolved_id)+ return {+ "action": {+ "from": start_point,+ "ref": ref if self._has_reference(ref) else None,+ "target_ref": target_ref if self._has_reference(target_ref) else None,+ "to": end_point,+ },+ "state": await self._state(resolved_id),+ }++ async def select_option(+ self,+ browser_id: int | str | None,+ ref: int | str,+ value: str = "",+ values: list[str] | None = None,+ ) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ await self._ensure_content_helper(page)+ action = await page.evaluate(+ "(args) => globalThis.__spaceBrowserPageContent__.select(args.ref, args.values)",+ {+ "ref": ref,+ "values": values if values is not None else value,+ },+ )+ await self._settle(page, short=True)+ self._maybe_promote(resolved_id)+ return {"action": action or {}, "state": await self._state(resolved_id)}++ async def set_checked(+ self,+ browser_id: int | str | None,+ ref: int | str,+ checked: bool = True,+ ) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ await self._ensure_content_helper(page)+ action = await page.evaluate(+ "(args) => globalThis.__spaceBrowserPageContent__.setChecked(args.ref, args.checked)",+ {+ "ref": ref,+ "checked": bool(checked),+ },+ )+ await self._settle(page, short=True)+ self._maybe_promote(resolved_id)+ return {"action": action or {}, "state": await self._state(resolved_id)}++ async def upload_file(+ self,+ browser_id: int | str | None,+ ref: int | str,+ path: str = "",+ paths: list[str] | None = None,+ ) -> dict[str, Any]:+ upload_paths = self._normalize_upload_paths(path=path, paths=paths)+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ await self._ensure_content_helper(page)+ metadata = await page.evaluate(+ "(ref) => globalThis.__spaceBrowserPageContent__.fileInputFor(ref)",+ ref,+ )+ handle = None+ try:+ handle = await page.evaluate_handle(+ "(ref) => globalThis.__spaceBrowserPageContent__.fileInputElementFor(ref)",+ ref,+ )+ element = handle.as_element() if handle else None+ if element:+ await element.set_input_files(upload_paths)+ elif metadata and metadata.get("selector"):+ await page.set_input_files(metadata["selector"], upload_paths)+ else:+ raise ValueError(f"Browser ref {ref!r} does not resolve to a file input")+ finally:+ if handle:+ with contextlib.suppress(Exception):+ await handle.dispose()+ await self._settle(page, short=True)+ self._maybe_promote(resolved_id)+ return {+ "action": {+ "files": upload_paths,+ "input": metadata or {},+ "ref": ref,+ },+ "state": await self._state(resolved_id),+ }+async def mouse(self,browser_id: int | str | None,@@ -1747,7 +2260,7 @@ class _BrowserRuntimeCore:async def _ensure_content_helper(self, page: Any) -> None:has_helper = await page.evaluate(- "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate && globalThis.__spaceBrowserPageContent__?.boundingBoxFor)"+ "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate && globalThis.__spaceBrowserPageContent__?.boundingBoxFor && globalThis.__spaceBrowserPageContent__?.pointFor && globalThis.__spaceBrowserPageContent__?.select && globalThis.__spaceBrowserPageContent__?.setChecked && globalThis.__spaceBrowserPageContent__?.fileInputFor)")if has_helper:returndiff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.mdindex 17d14a489..4dd9c1566 100644--- a/plugins/_browser/prompts/agent.system.tool.browser.md+++ b/plugins/_browser/prompts/agent.system.tool.browser.md@@ -6,8 +6,10 @@ refs come from content as typed markers: [link 3], [button 6], [image 1], [inputBrowser tool actions must not open the right canvas automatically. Use the tool headlessly unless the user opens the Browser canvas or explicitly asks for a visible browser view; if the Browser canvas is already open, it may reflect the active page.-actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_all-common args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button calls+Browser does not automatically load screenshots or canvas images into model context. Screenshots are explicit only.++actions: open list state set_active navigate back forward reload content detail screenshot click hover double_click right_click drag type submit type_submit scroll evaluate key_chord mouse wheel keyboard clipboard set_viewport select_option set_checked upload_file multi close close_all+common args: action browser_id url ref target_ref text selector selectors script modifiers keys key include_content focus_popup event_type x y to_x to_y offset_x offset_y target_offset_x target_offset_y delta_x delta_y button quality full_page path paths value values checked width height callsworkflow:- open creates a new browser and returns id/state@@ -17,6 +19,34 @@ workflow:- navigate/back/forward/reload return fresh state- list shows open browsers; pass include_content: true for one-call bulk read+explicit vision workflow:+1. call browser with action: "screenshot"+2. call vision_load with the returned path+3. reason from the latest loaded screenshot, not an older screenshot++screenshot:+- saves a JPEG by default and returns path, a0_path, mime, state, and a ready vision_load tool_args object+- pass quality 20..95, full_page true/false, or path+- PNG is used only when path ends in .png+- no base64 image data is returned in the tool message++pointer and raw input:+- hover moves to a ref center or x/y viewport CSS pixels+- double_click and right_click accept ref or x/y; double_click accepts button and modifiers+- drag moves from ref or x/y to target_ref or to_x/to_y+- wheel scrolls at x/y with delta_x and delta_y+- keyboard presses key or types text into the active page+- clipboard is copy, cut, or paste; for browser:clipboard pass action: "paste" and optional text+- set_viewport resizes the page viewport with width and height+- coordinates are Chromium viewport CSS pixels and match screenshots/Browser canvas+- ref offsets are relative to the target element top-left; refs default to element center++forms:+- use select_option for native select and safely detectable ARIA listbox/combobox controls+- use set_checked for checkbox, radio, switch, and toggle-like refs+- use upload_file for file input refs or associated labels; verify file paths exist before upload+- for complex forms, load browser-forms first with skills_tool:load+modifier clicks:- click accepts modifiers like ["Control"], ["Shift"], ["Alt"], ["Meta"]- ctrl/meta-click opens link in new tab in background (Chrome rule)@@ -33,7 +63,7 @@ background work (do not steal focus):- open (new tab created)- explicit set_active action- action on the already-active tab- - chrome popup-focus rule (plain click on target=_blank → follow; ctrl-click → stay)+ - chrome popup-focus rule (plain click on target=_blank -> follow; ctrl-click -> stay)- to switch focus deliberately: {"action":"set_active","browser_id":N}key_chord:@@ -46,6 +76,7 @@ multi (parallel batch):- different browser_ids run in parallel; same browser_id runs in submit order- returns array of {"ok":true,"result":...} or {"ok":false,"error":"..."} matching input order- ideal for: scrape N tabs at once, fan-out reads, parallel evaluate+- new v1 actions such as screenshot, hover, wheel, keyboard, select_option, set_checked, and upload_file are accepted- avoid mutating same tab twice in one batch unless serial order is intendedexamples:@@ -73,9 +104,18 @@ examples:{"tool_name": "browser","tool_args": {- "action": "click",+ "action": "screenshot","browser_id": 1,- "ref": 3+ "quality": 80+ }+}+~~~++~~~json+{+ "tool_name": "vision_load",+ "tool_args": {+ "paths": ["/absolute/local/path.jpg"]}}~~~@@ -84,10 +124,10 @@ examples:{"tool_name": "browser","tool_args": {- "action": "click",+ "action": "select_option","browser_id": 1,- "ref": 3,- "modifiers": ["Control"]+ "ref": 8,+ "value": "Canada"}}~~~@@ -96,9 +136,10 @@ examples:{"tool_name": "browser","tool_args": {- "action": "key_chord",+ "action": "set_checked","browser_id": 1,- "keys": ["Control", "a"]+ "ref": 9,+ "checked": true}}~~~@@ -107,8 +148,10 @@ examples:{"tool_name": "browser","tool_args": {- "action": "list",- "include_content": true+ "action": "upload_file",+ "browser_id": 1,+ "ref": 10,+ "path": "/a0/usr/workdir/resume.pdf"}}~~~@@ -120,7 +163,7 @@ examples:"action": "multi","calls": [{"action": "content", "browser_id": 1},- {"action": "content", "browser_id": 2},+ {"action": "screenshot", "browser_id": 2},{"action": "evaluate", "browser_id": 3, "script": "document.title"}]}diff --git a/plugins/_browser/skills/browser-forms/SKILL.md b/plugins/_browser/skills/browser-forms/SKILL.mdnew file mode 100644index 000000000..7a71cf10a--- /dev/null+++ b/plugins/_browser/skills/browser-forms/SKILL.md@@ -0,0 +1,18 @@+---+name: browser-forms+description: Use for complex Agent Zero Browser form workflows involving selects, checkboxes, radios, file uploads, contenteditable fields, multi-step validation, or visually verified submission.+---++# Browser Forms++Use this skill for complex Browser form workflows where the page state may depend on selects, checkboxes, radios, file uploads, contenteditable fields, validation, or visual confirmation.++Start with `browser:content` to capture current refs, then use `browser:detail` on ambiguous fields before acting. Prefer ref-based form actions before coordinates.++Use `select_option`, `set_checked`, `upload_file`, `type`, `type_submit`, and `submit` for form interaction. Use coordinates only when no stable ref exists or the UI is intentionally canvas-like.++Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history.++Verify after submission with `browser:content`, `browser:state`, or another explicit `browser:screenshot` plus `vision_load`.++Do not guess file paths for upload. Verify that every path exists before calling `upload_file`.diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.pyindex 309428935..b53ed96ea 100644--- a/plugins/_browser/tools/browser.py+++ b/plugins/_browser/tools/browser.py@@ -14,22 +14,48 @@ class Browser(Tool):browser_id: int | str | None = None,url: str = "",ref: int | str | None = None,+ target_ref: int | str | None = None,text: str = "",selector: str = "",selectors: list[str] | None = None,script: str = "",modifiers: list[str] | str | None = None,keys: list[str] | None = None,+ key: str = "",include_content: bool = False,focus_popup: bool | None = None,event_type: str = "",x: float = 0.0,y: float = 0.0,+ to_x: float = 0.0,+ to_y: float = 0.0,+ offset_x: float = 0.0,+ offset_y: float = 0.0,+ target_offset_x: float = 0.0,+ target_offset_y: float = 0.0,+ delta_x: float = 0.0,+ delta_y: float = 0.0,button: str = "left",+ quality: int = 80,+ full_page: bool = False,+ path: str = "",+ paths: list[str] | None = None,+ value: str = "",+ values: list[str] | None = None,+ checked: bool | None = None,+ width: int = 0,+ height: int = 0,calls: list[dict[str, Any]] | None = None,**kwargs: Any,) -> Response:- action = str(action or self.method or "state").strip().lower().replace("-", "_")+ method_action = str(self.method or "").strip().lower().replace("-", "_")+ requested_action = str(action or "").strip().lower().replace("-", "_")+ clipboard_action = ""+ if method_action == "clipboard" and requested_action in {"copy", "cut", "paste"}:+ clipboard_action = requested_action+ action = "clipboard"+ else:+ action = str(action or self.method or "state").strip().lower().replace("-", "_")runtime = await get_runtime(self.agent.context.id)if isinstance(modifiers, str):@@ -40,6 +66,14 @@ class Browser(Tool):try:if action == "open":result = await runtime.call("open", url or "")+ elif action == "screenshot":+ result = await runtime.call(+ "screenshot_file",+ browser_id,+ quality=quality,+ full_page=full_page,+ path=path,+ )elif action == "list":result = await runtime.call("list", include_content=bool(include_content))elif action == "state":@@ -86,6 +120,115 @@ class Browser(Tool):if not keys:raise ValueError("key_chord requires non-empty 'keys' list")result = await runtime.call("key_chord", browser_id, list(keys))+ elif action == "hover":+ result = await runtime.call(+ "hover",+ browser_id,+ ref=ref,+ x=x,+ y=y,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ elif action == "double_click":+ result = await runtime.call(+ "double_click",+ browser_id,+ ref=ref,+ x=x,+ y=y,+ button=button or "left",+ modifiers=modifiers,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ elif action == "right_click":+ result = await runtime.call(+ "right_click",+ browser_id,+ ref=ref,+ x=x,+ y=y,+ modifiers=modifiers,+ offset_x=offset_x,+ offset_y=offset_y,+ )+ elif action == "drag":+ result = await runtime.call(+ "drag",+ browser_id,+ ref=ref,+ target_ref=target_ref,+ x=x,+ y=y,+ to_x=to_x,+ to_y=to_y,+ offset_x=offset_x,+ offset_y=offset_y,+ target_offset_x=target_offset_x,+ target_offset_y=target_offset_y,+ )+ elif action == "wheel":+ result = await runtime.call(+ "wheel",+ browser_id,+ x,+ y,+ delta_x,+ delta_y,+ )+ elif action == "keyboard":+ result = await runtime.call(+ "keyboard",+ browser_id,+ key=key,+ text=text,+ )+ elif action == "clipboard":+ normalized_clipboard_action = clipboard_action or str(+ kwargs.get("clipboard_action")+ or kwargs.get("operation")+ or event_type+ or ""+ ).strip().lower()+ result = await runtime.call(+ "clipboard",+ browser_id,+ action=normalized_clipboard_action,+ text=text,+ )+ elif action in {"copy", "cut", "paste"}:+ result = await runtime.call(+ "clipboard",+ browser_id,+ action=action,+ text=text,+ )+ elif action == "set_viewport":+ result = await runtime.call("set_viewport", browser_id, width, height)+ elif action == "select_option":+ result = await runtime.call(+ "select_option",+ browser_id,+ self._require_ref(ref),+ value=value,+ values=values,+ )+ elif action == "set_checked":+ result = await runtime.call(+ "set_checked",+ browser_id,+ self._require_ref(ref),+ checked=True if checked is None else bool(checked),+ )+ elif action == "upload_file":+ result = await runtime.call(+ "upload_file",+ browser_id,+ self._require_ref(ref),+ path=path,+ paths=paths,+ )elif action == "mouse":result = await runtime.call("mouse", browser_id, event_type or "click", x, y,diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex 4a4e464f3..1a619c70c 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -733,10 +733,32 @@ def test_browser_tool_does_not_auto_open_canvas_policy_is_documented():assert "must not open the right canvas automatically" in promptassert "Use the tool headlessly unless the user opens the Browser canvas" in promptassert "optional visible WebUI viewer" in prompt+ assert "screenshot" in prompt+ assert "vision_load" in prompt+ assert "select_option" in prompt+ assert "set_checked" in prompt+ assert "upload_file" in prompt+ assert "browser-forms" in prompt+ assert "does not automatically load screenshots" in promptassert "already open" in configassert "already-open Browser canvas" in config_html+def test_browser_forms_skill_is_plugin_owned_and_discoverable():+ skill_path = PROJECT_ROOT / "plugins" / "_browser" / "skills" / "browser-forms" / "SKILL.md"+ assert skill_path.exists()+ skill = skill_path.read_text(encoding="utf-8")+ assert skill.startswith("---\n")+ frontmatter = skill.split("---", 2)[1]+ assert "name: browser-forms" in frontmatter+ assert "description:" in frontmatter+ assert "select_option" in skill+ assert "set_checked" in skill+ assert "upload_file" in skill+ assert "browser:screenshot" in skill+ assert "vision_load" in skill++def test_browser_canvas_uses_plain_panel_without_debug_probe():panel_html = (PROJECT_ROOT@@ -1014,10 +1036,26 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():assert "function annotate(payload = null)" in helperassert "annotate," in helperassert "boundingBoxFor," in helper+ assert "pointFor," in helper+ assert "select(referenceId, valueOrValues)" in helper+ assert "setChecked(referenceId, checked)" in helper+ assert "fileInputFor," in helperassert "sanitizeAnnotationDom" in helperassert "password" in helper+def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():+ helper = (+ PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"+ ).read_text(encoding="utf-8")++ assert 'const VERSION = "11"' in helper+ assert "function renderControlLabelReferences" in helper+ assert "getLabelElementText(labelElement, element)" in helper+ assert "return renderControlLabelReferences(node, context);" in helper+ assert "return renderControlLabelReferences(element, context);" in helper++def test_browser_runtime_requires_current_content_helper_for_modifier_clicks():runtime = (PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"@@ -1345,6 +1383,164 @@ async def test_browser_tool_dispatches_direct_actions(monkeypatch):assert calls == [("content", (1, None))]+async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):+ calls = []++ class FakeRuntime:+ async def call(self, method, *args, **kwargs):+ calls.append((method, args, kwargs))+ return {"ok": True, "method": method, "args": args, "kwargs": kwargs}++ async def fake_get_runtime(context_id, create=True):+ assert context_id == "ctx"+ return FakeRuntime()++ monkeypatch.setattr(browser_tool_module, "get_runtime", fake_get_runtime)+ agent = SimpleNamespace(context=SimpleNamespace(id="ctx"))++ async def execute(**kwargs):+ tool = browser_tool_module.Browser(+ agent=agent,+ name="browser",+ method=kwargs.pop("_method", None),+ args={},+ message="",+ loop_data=None,+ )+ response = await tool.execute(**kwargs)+ assert response.break_loop is False++ await execute(action="screenshot", browser_id=1, quality=91, full_page=True, path="/tmp/a.jpg")+ await execute(action="hover", browser_id=1, ref=2, offset_x=3, offset_y=4)+ await execute(action="double_click", browser_id=1, x=10, y=20, button="left", modifiers=["Shift"])+ await execute(action="right_click", browser_id=1, ref=3, modifiers="Control")+ await execute(action="drag", browser_id=1, ref=4, target_ref=5, target_offset_x=6, target_offset_y=7)+ await execute(action="wheel", browser_id=1, x=8, y=9, delta_x=1, delta_y=2)+ await execute(action="keyboard", browser_id=1, key="Enter")+ await execute(_method="clipboard", action="paste", browser_id=1, text="hello")+ await execute(action="copy", browser_id=1)+ await execute(action="set_viewport", browser_id=1, width=1280, height=720)+ await execute(action="select_option", browser_id=1, ref=6, value="CA")+ await execute(action="set_checked", browser_id=1, ref=7, checked=False)+ await execute(action="upload_file", browser_id=1, ref=8, paths=["/tmp/a.txt"])++ assert calls == [+ ("screenshot_file", (1,), {"quality": 91, "full_page": True, "path": "/tmp/a.jpg"}),+ ("hover", (1,), {"ref": 2, "x": 0.0, "y": 0.0, "offset_x": 3, "offset_y": 4}),+ (+ "double_click",+ (1,),+ {+ "ref": None,+ "x": 10,+ "y": 20,+ "button": "left",+ "modifiers": ["Shift"],+ "offset_x": 0.0,+ "offset_y": 0.0,+ },+ ),+ (+ "right_click",+ (1,),+ {+ "ref": 3,+ "x": 0.0,+ "y": 0.0,+ "modifiers": ["Control"],+ "offset_x": 0.0,+ "offset_y": 0.0,+ },+ ),+ (+ "drag",+ (1,),+ {+ "ref": 4,+ "target_ref": 5,+ "x": 0.0,+ "y": 0.0,+ "to_x": 0.0,+ "to_y": 0.0,+ "offset_x": 0.0,+ "offset_y": 0.0,+ "target_offset_x": 6,+ "target_offset_y": 7,+ },+ ),+ ("wheel", (1, 8, 9, 1, 2), {}),+ ("keyboard", (1,), {"key": "Enter", "text": ""}),+ ("clipboard", (1,), {"action": "paste", "text": "hello"}),+ ("clipboard", (1,), {"action": "copy", "text": ""}),+ ("set_viewport", (1, 1280, 720), {}),+ ("select_option", (1, 6), {"value": "CA", "values": None}),+ ("set_checked", (1, 7), {"checked": False}),+ ("upload_file", (1, 8), {"path": "", "paths": ["/tmp/a.txt"]}),+ ]+++async def test_browser_multi_dispatch_accepts_v1_actions():+ calls = []+ core = _BrowserRuntimeCore("ctx")++ async def record(method):+ async def inner(*args, **kwargs):+ calls.append((method, args, kwargs))+ return {"method": method}+ return inner++ for method in (+ "screenshot_file",+ "hover",+ "double_click",+ "right_click",+ "drag",+ "wheel",+ "keyboard",+ "clipboard",+ "set_viewport",+ "select_option",+ "set_checked",+ "upload_file",+ ):+ setattr(core, method, await record(method))++ results = await core.multi(+ [+ {"action": "screenshot", "browser_id": 1, "quality": 5, "full_page": True},+ {"action": "hover", "browser_id": 1, "ref": 2},+ {"action": "double_click", "browser_id": 1, "x": 1, "y": 2},+ {"action": "right_click", "browser_id": 1, "ref": 3},+ {"action": "drag", "browser_id": 1, "ref": 4, "target_ref": 5},+ {"action": "wheel", "browser_id": 1, "delta_y": 100},+ {"action": "keyboard", "browser_id": 1, "key": "Enter"},+ {"action": "paste", "browser_id": 1, "text": "x"},+ {"action": "set_viewport", "browser_id": 1, "width": 640, "height": 480},+ {"action": "select_option", "browser_id": 1, "ref": 6, "values": ["a", "b"]},+ {"action": "set_checked", "browser_id": 1, "ref": 7, "checked": False},+ {"action": "upload_file", "browser_id": 1, "ref": 8, "path": "/tmp/file.txt"},+ ]+ )++ assert all(result["ok"] for result in results)+ assert [call[0] for call in calls] == [+ "screenshot_file",+ "hover",+ "double_click",+ "right_click",+ "drag",+ "wheel",+ "keyboard",+ "clipboard",+ "set_viewport",+ "select_option",+ "set_checked",+ "upload_file",+ ]++@pytest.mark.anyioasync def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):class FakeRuntime:@@ -1641,6 +1837,137 @@ async def test_browser_runtime_remounts_initial_changed_viewport():assert settled == [True]+async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch, tmp_path):+ screenshot_calls = []++ def fake_get_abs_path(*parts):+ return str(tmp_path.joinpath(*parts))++ def fake_normalize_a0_path(path):+ return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")++ monkeypatch.setattr(browser_runtime_module.files, "get_abs_path", fake_get_abs_path)+ monkeypatch.setattr(browser_runtime_module.files, "normalize_a0_path", fake_normalize_a0_path)++ class FakePage:+ url = "about:blank"+ viewport_size = {"width": 1024, "height": 768}++ async def screenshot(self, **kwargs):+ screenshot_calls.append(kwargs)+ Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)+ Path(kwargs["path"]).write_bytes(b"image-bytes")+ return b"image-bytes"++ async def title(self):+ return "Blank"++ async def evaluate(self, script, payload=None):+ return 1++ core = _BrowserRuntimeCore("ctx/id")+ core.context = object()+ core.pages[5] = browser_runtime_module.BrowserPage(id=5, page=FakePage())++ result = await core.screenshot_file(5, quality=500)++ path = Path(result["path"])+ assert path.exists()+ assert path.parent == tmp_path / "tmp" / "browser" / "screenshots" / "ctx_id"+ assert path.name.startswith("browser-5-")+ assert path.suffix == ".jpg"+ assert result["a0_path"].startswith("/a0/tmp/browser/screenshots/ctx_id/browser-5-")+ assert result["mime"] == "image/jpeg"+ assert result["vision_load"] == {+ "tool_name": "vision_load",+ "tool_args": {"paths": [result["path"]]},+ }+ assert "image" not in result+ assert screenshot_calls[-1]["type"] == "jpeg"+ assert screenshot_calls[-1]["quality"] == 95+ assert screenshot_calls[-1]["full_page"] is False++ png_path = tmp_path / "custom.png"+ png_result = await core.screenshot_file(5, quality=1, full_page=True, path=str(png_path))++ assert png_result["path"] == str(png_path)+ assert png_result["mime"] == "image/png"+ assert screenshot_calls[-1] == {+ "path": str(png_path),+ "type": "png",+ "full_page": True,+ }+++async def test_browser_runtime_ref_point_resolution_applies_offsets():+ eval_payloads = []+ moves = []++ class FakeMouse:+ async def move(self, x, y, **kwargs):+ moves.append((x, y, kwargs))++ class FakePage:+ url = "about:blank"++ def __init__(self):+ self.mouse = FakeMouse()++ async def evaluate(self, script, payload=None):+ eval_payloads.append((script, payload))+ if payload and "offsets" in payload:+ return {+ "x": 10 + payload["offsets"]["offset_x"],+ "y": 20 + payload["offsets"]["offset_y"],+ "rect": {"x": 10, "y": 20, "width": 100, "height": 40},+ "selector": "#target",+ }+ return 1++ async def title(self):+ return "Blank"++ core = _BrowserRuntimeCore("ctx")+ core.context = object()+ core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=FakePage())+ core._ensure_content_helper = lambda _page: asyncio.sleep(0)++ result = await core.hover(7, ref=4, offset_x=3, offset_y=5)++ assert moves == [(13.0, 25.0, {})]+ assert result["action"]["point"]["selector"] == "#target"+ assert eval_payloads[0][1] == {+ "ref": 4,+ "offsets": {+ "offset_x": 3.0,+ "offset_y": 5.0,+ "useOffsets": True,+ },+ }+++def test_browser_runtime_upload_path_normalization(monkeypatch, tmp_path):+ first = tmp_path / "first.txt"+ second = tmp_path / "second.txt"+ first.write_text("one", encoding="utf-8")+ second.write_text("two", encoding="utf-8")++ monkeypatch.setattr(+ browser_runtime_module.files,+ "get_abs_path",+ lambda *parts: str(tmp_path.joinpath(*parts)),+ )++ assert _BrowserRuntimeCore._normalize_upload_paths(path=str(first)) == [str(first)]+ assert _BrowserRuntimeCore._normalize_upload_paths(paths=["second.txt"]) == [str(second)]+ assert _BrowserRuntimeCore._normalize_upload_paths(path=str(first), paths=["second.txt"]) == [+ str(second),+ str(first),+ ]++@pytest.mark.anyioasync def test_browser_runtime_clipboard_paste_uses_dom_bridge():eval_payloads = []