Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

version up

Emre Sokullu229f37d4b490
diff --git a/package-lock.json b/package-lock.json
index 64b2a5213..e9dcd1f05 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "webbrain",
- "version": "40.0.6",
+ "version": "40.0.7",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "webbrain",
- "version": "40.0.6",
+ "version": "40.0.7",
"license": "GPL-3.0-or-later",
"devDependencies": {
"playwright": "^1.48.0",
diff --git a/package.json b/package.json
index e8476ace6..2c2596653 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "webbrain",
- "version": "40.0.6",
+ "version": "40.0.7",
"description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
"private": true,
"type": "module",
@@ -53,8 +53,8 @@
"benchmark:relevance": "node scripts/benchmark-offline-relevance.mjs --verbose",
"test:social-contract": "node --test test/social-publish-contract.mjs",
"test:social-contract:dom": "node test/social-publish-contract-dom.mjs",
- "test:runtime-lifecycle": "node --test test/agent-lifecycle.mjs test/runtime-lifecycle.mjs test/collection-read-progress.mjs test/network-lifecycle.mjs test/text-write-cancellation.mjs test/chat-steering.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/page-feedback-policy.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/media-download-agent-handler.mjs && node test/ui-content-lifecycle.mjs && node --test test/chat-steering-ui.mjs test/page-monitor-ui.mjs test/page-monitor-passive-progress.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs && node test/message-recipient-upload.mjs && node --test test/message-recipient-whatsapp.mjs",
- "test:page-feedback": "node --test test/page-feedback.mjs test/page-feedback-progress.mjs test/page-feedback-policy.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/media-download-agent-handler.mjs test/page-monitor-ui.mjs test/page-monitor-passive-progress.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs && node test/message-recipient-upload.mjs && node --test test/message-recipient-whatsapp.mjs",
+ "test:runtime-lifecycle": "node --test test/agent-lifecycle.mjs test/runtime-lifecycle.mjs test/collection-read-progress.mjs test/network-lifecycle.mjs test/text-write-cancellation.mjs test/trusted-field-preparation.mjs test/chat-steering.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/page-feedback-policy.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/media-download-agent-handler.mjs && node test/ui-content-lifecycle.mjs && node --test test/chat-steering-ui.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-passive-progress.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs && node test/message-recipient-upload.mjs && node --test test/message-recipient-whatsapp.mjs",
+ "test:page-feedback": "node --test test/trusted-field-preparation.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/page-feedback-policy.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/media-download-agent-handler.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-passive-progress.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs && node test/message-recipient-upload.mjs && node --test test/message-recipient-whatsapp.mjs",
"test:chat-steering": "node --test test/chat-steering.mjs test/chat-steering-ui.mjs",
"test:rich-text-dom": "node test/ui-content-lifecycle.mjs --dom",
"test:firefox-bidi": "node --test test/firefox-bidi.mjs test/firefox-full-page-screenshot.mjs",
diff --git a/src/chrome/ARCHITECTURE.md b/src/chrome/ARCHITECTURE.md
index 7221a2c32..224680d93 100644
--- a/src/chrome/ARCHITECTURE.md
+++ b/src/chrome/ARCHITECTURE.md
@@ -1,6 +1,6 @@
# WebBrain Chrome/Edge Extension — Architecture
-> Version 40.0.6 · Manifest V3 · Service Worker background
+> Version 40.0.7 · Manifest V3 · Service Worker background
## High-Level Overview
diff --git a/src/chrome/manifest.json b/src/chrome/manifest.json
index 02661678e..ba60af71c 100644
--- a/src/chrome/manifest.json
+++ b/src/chrome/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "WebBrain",
- "version": "40.0.6",
+ "version": "40.0.7",
"description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
"permissions": [
"sidePanel",
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 55a57f3cf..45b393ff9 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -481,6 +481,10 @@ const CONTENT_ACTION_TIMEOUT_MS = 60_000;
const CONTENT_ACTION_RESPONSE_GRACE_MS = 5_000;
const ASK_MODE_HANDOFF_TIMEOUT_MS = 5_000;
const CONTENT_ACTION_SIGNAL_DEADLINES = new WeakMap();
+// A completed focus click is a browser dispatch, but need not be a text write.
+// Only this module can attest that its exact ref stayed unchanged before any
+// trusted text command was attempted; page/model data cannot forge this proof.
+const TRUSTED_FIELD_PREPARATION_UNCHANGED = Symbol('trustedFieldPreparationUnchanged');
const EARLY_CDP_ACTION_TOOLS = new Set(['click', 'type_text', 'press_keys', 'hover', 'drag_drop', 'upload_file']);
const DONE_OUTCOMES = new Set(['success', 'partial', 'failed']);
const LOCAL_CANCELLATION_ASSISTANT_RE = /^\[?Stopped by user(?: before (?:the run started|executing requested tool calls))?\.?\]?$/;
@@ -27966,6 +27970,14 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
if (result.noop === true && result.success === true && result.noDispatch === true) {
return result;
}
+ const unchangedPreparation = result[TRUSTED_FIELD_PREPARATION_UNCHANGED];
+ if (unchangedPreparation?.tabId === tabId && unchangedPreparation.tool === name
+ && unchangedPreparation.refId === args.ref_id
+ && result.success === false && result.verified === false) {
+ const unchangedResult = { ...result };
+ delete unchangedResult[TRUSTED_FIELD_PREPARATION_UNCHANGED];
+ return unchangedResult;
+ }
const enrich = operation => this._readTextMutationEnrichment(operation, enrichmentSignal);
let target = this._textMutationTarget(tabId, name, args);
const replacesValue = this._textMutationReplacesValue(name, args);
@@ -42180,7 +42192,32 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
recoveryRequired: 'verify_or_restore_field',
};
let trustedDispatched = false;
+ let trustedTextAttempted = false;
+ let beforePreparation = null;
+ const digestTarget = { locatorType: 'ax', refId: args.ref_id };
+ const finishPreparationFailure = async failure => {
+ if (!trustedDispatched || trustedTextAttempted || !beforePreparation?.documentToken || abortSignal?.aborted) return failure;
+ const afterPreparation = await this._textMutationValueDigest(tabId, digestTarget);
+ if (abortSignal?.aborted || !afterPreparation?.documentToken
+ || beforePreparation.documentToken !== afterPreparation.documentToken
+ || beforePreparation.pageUrl !== afterPreparation.pageUrl
+ || beforePreparation.valueLength !== afterPreparation.valueLength
+ || beforePreparation.valueSha256 !== afterPreparation.valueSha256) return failure;
+ return {
+ ...failure,
+ [TRUSTED_FIELD_PREPARATION_UNCHANGED]: { tabId, tool: toolName, refId: args.ref_id },
+ textUnchanged: true,
+ outcomeUnknown: false,
+ mutationMayHaveOccurred: false,
+ repeatBlocked: false,
+ retryable: true,
+ recoveryRequired: 'fresh_tree',
+ error: `${failure.error} No trusted text input was sent, and exact readback proves the editor value is unchanged. Re-observe the page before trying the field again.`,
+ };
+ };
try {
+ this._throwIfAborted(abortSignal);
+ beforePreparation = await this._textMutationValueDigest(tabId, digestTarget);
this._throwIfAborted(abortSignal);
let prepared = await chrome.tabs.sendMessage(tabId, {
target: 'content',
@@ -42191,6 +42228,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
if (!prepared?.success) {
return {
...failed,
+ ...(prepared?.pageFeedbackPending ? { pageFeedbackPending: true, recoveryRequired: 'fresh_tree' } : {}),
error: `${response.error || 'Field verification failed'} Trusted retry could not focus the current ref: ${prepared?.error || 'unknown preparation failure'}`,
};
}
@@ -42224,12 +42262,13 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
});
this._throwIfAborted(abortSignal);
if (!reselected?.success) {
- return {
+ return await finishPreparationFailure({
...failed,
dispatched: true,
noDispatch: false,
+ ...(reselected?.pageFeedbackPending ? { pageFeedbackPending: true } : {}),
error: `${response.error || 'Field verification failed'} Trusted retry focused the editor but could not reselect it: ${reselected?.error || 'unknown preparation failure'}`,
- };
+ });
}
prepared = reselected;
}
@@ -42239,6 +42278,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
? (typeof args?.text === 'string' ? args.text : '')
: expected;
this._throwIfAborted(abortSignal);
+ // A lost response can precede a delayed applied write. After this
+ // boundary an unchanged sample cannot make the text command retry-safe.
+ trustedTextAttempted = true;
if (trustedText || appendingContentEditable) {
await cdpClient.sendCommand(tabId, 'Input.insertText', { text: trustedText });
} else {
@@ -42344,11 +42386,12 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
return recovered;
} catch (error) {
if (error?.code === 'content_action_timeout') throw error;
- return {
+ return await finishPreparationFailure({
...failed,
...(trustedDispatched ? { dispatched: true, noDispatch: false } : {}),
+ ...(error?.code === 'page_feedback_pending' ? { pageFeedbackPending: true } : {}),
error: `${response.error || 'Field verification failed'} Trusted Chrome retry failed: ${error?.message || String(error)}`,
- };
+ });
}
}
@@ -44330,7 +44373,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
const feedbackCalls = result?.toolCalls?.length ? result.toolCalls
: !this._containsProviderReplayState(result?.responseItems) ? this._tryParseToolCallsFromText(result?.content || '', allowedToolNames) : [];
- if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, { responseToolCalls: feedbackCalls }))
+ if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, {
+ responseToolCalls: feedbackCalls, responseWithoutTools: feedbackCalls.length === 0,
+ }))
|| this._applyPendingSteering(tabId, messages, onUpdate)) {
const recovery = this._pageFeedbackRecoveryResult(tabId, messages, onUpdate);
if (recovery) {
@@ -45505,7 +45550,9 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
const feedbackCalls = streamedToolCalls?.length ? streamedToolCalls
: !this._containsProviderReplayState(responseItems) ? this._tryParseToolCallsFromText(fullText || '', allowedToolNames) : [];
- if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, { responseToolCalls: feedbackCalls }))
+ if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, {
+ responseToolCalls: feedbackCalls, responseWithoutTools: feedbackCalls.length === 0,
+ }))
|| this._applyPendingSteering(tabId, messages, onUpdate)) {
const recovery = this._pageFeedbackRecoveryResult(tabId, messages, onUpdate);
if (recovery) {
diff --git a/src/chrome/src/agent/page-feedback-policy.js b/src/chrome/src/agent/page-feedback-policy.js
index 75c2615b1..76fdf0d74 100644
--- a/src/chrome/src/agent/page-feedback-policy.js
+++ b/src/chrome/src/agent/page-feedback-policy.js
@@ -96,7 +96,13 @@ export function pageFeedbackCallPolicy(name, args = {}, state = {}, currentPage
}
if (name === 'download_public_media' && (httpUrl(args.url) || httpUrl(state.url))) return { kind: 'independent' };
if (name === 'navigate' && httpUrl(args.url)) return { kind: 'navigate' };
- if (name === 'done' && (['partial', 'failure'].includes(args.outcome)
+ // Ask exposes a summary-only done schema. Its answer can come from completed
+ // detached research, so unrelated counters (or a failed AX refresh) do not
+ // require a new inference. The run mode is private pre-inference state.
+ const askSummary = state.runMode === 'ask'
+ && !Object.prototype.hasOwnProperty.call(args, 'outcome')
+ && typeof args.summary === 'string' && !!args.summary.trim();
+ if (name === 'done' && (askSummary || ['partial', 'failed'].includes(args.outcome)
|| (args.outcome === 'success' && currentPage?.success !== false))) return { kind: 'observe', terminal: true };
if (name === 'download_social_media' && !args.scroll && !args.all
&& [undefined, 'main', 'auto'].includes(args.mode)
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index b3837e376..7c5305101 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -474,7 +474,8 @@ export const pageFeedbackMethods = {
const frames = await apiFor().webNavigation?.getAllFrames?.({ tabId });
documentId = frames?.find(frame => frame.frameId === 0)?.documentId || documentId;
} catch {}
- const state = { token: run.token, url, documentId, page: run.latestPage,
+ const runMode = this._effectiveRunMode?.(tabId, null) || null;
+ const state = { token: run.token, url, documentId, page: run.latestPage, runMode,
frames: new Map([...run.frames].map(([id, frame]) => [id, { token: frame.token, id: frame.id }])),
steeringRevision: this._steeringRuns?.get(tabId)?.acceptedIds.size || 0, mediaBindings: {} };
// The downloader is lazy-loaded. Capture its private document/node/asset
@@ -577,6 +578,7 @@ export const pageFeedbackMethods = {
_pageFeedbackStateCurrent(tabId, run, state, url = run.url) {
return !!state && state.token === run.token && state.url === url
+ && (state.runMode === undefined || state.runMode === (this._effectiveRunMode?.(tabId, null) || null))
&& state.steeringRevision === (this._steeringRuns?.get(tabId)?.acceptedIds.size || 0)
&& [...state.frames].every(([id, frame]) => run.frames.get(id)?.token === frame.token
&& run.frames.get(id)?.id === frame.id);
@@ -697,7 +699,8 @@ export const pageFeedbackMethods = {
},
async _applyPendingPageFeedback(tabId, messages, onUpdate = () => {},
- { workflow = false, responseToolCalls = null, stage = responseToolCalls === null ? 'observation' : 'response' } = {}) {
+ { workflow = false, responseToolCalls = null, responseWithoutTools = false,
+ stage = responseToolCalls === null ? 'observation' : 'response' } = {}) {
if (!this._hasPendingPageFeedback(tabId) || this._checkAbort(tabId)) return false;
const started = Date.now();
await this._waitForPageFeedbackIdle(tabId);
@@ -730,8 +733,16 @@ export const pageFeedbackMethods = {
}
const sameDocument = documentMatches && this._pageFeedbackRuns.get(tabId) === run
&& this._pageFeedbackStateCurrent(tabId, run, state, url);
- let retained = sameDocument && canRetainPageFeedbackCalls(responseToolCalls, state, page, events)
- && (!pending.length || canRetainPageFeedbackCalls(responseToolCalls, state, page, pending, { pending: true }));
+ // A tool-free Ask response is read-only, including an empty response that
+ // must reach the provider's bounded delivery recovery. This does not mint
+ // completion evidence or retain any mutation, structured terminal or tool.
+ const readonlyAskResponse = responseWithoutTools === true && state.runMode === 'ask'
+ && Array.isArray(responseToolCalls) && responseToolCalls.length === 0;
+ const canRetain = (feedback, options) => readonlyAskResponse
+ ? isPassivePageFeedback(feedback)
+ : canRetainPageFeedbackCalls(responseToolCalls, state, page, feedback, options);
+ let retained = sameDocument && canRetain(events)
+ && (!pending.length || canRetain(pending, { pending: true }));
if (retained) for (const call of responseToolCalls) {
const args = JSON.parse(call.function.arguments);
if (pageFeedbackCallPolicy(call.function.name, args, state, page, events).kind === 'bound_target'
@@ -767,7 +778,7 @@ export const pageFeedbackMethods = {
}
// Capture/description can itself receive a hard intervention.
const finalPending = [...run.events.values()];
- const accepted = retained && (!finalPending.length || canRetainPageFeedbackCalls(responseToolCalls, state, page, finalPending, { pending: true }));
+ const accepted = retained && (!finalPending.length || canRetain(finalPending, { pending: true }));
if (Array.isArray(responseToolCalls) && !accepted) {
const recovery = accountFeedbackSupersession(run, { passive: sameDocument && isPassivePageFeedback(events)
&& (!finalPending.length || isPassivePageFeedback(finalPending)), stage,
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index c27986de0..2b19a5747 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -632,6 +632,47 @@
return context;
} catch { return null; }
}
+ function dialogTextIsCovered(owner, boundTextNodes, addNode) {
+ const stack = [...owner.childNodes];
+ let visited = 0;
+ while (stack.length) {
+ if (++visited > 256) return false;
+ const node = stack.pop();
+ if (node.nodeType === 3) {
+ // Direct owner text is already bound; generic nested text may name a
+ // destination that the control/heading footprint cannot establish.
+ if (node.data.trim() && node.parentElement !== owner) return false;
+ continue;
+ }
+ if (!(node instanceof Element) || ignored(node)) continue;
+ const style = getComputedStyle(node);
+ if (style.display === 'none' || style.opacity === '0' || style.contentVisibility === 'hidden') continue;
+ // A zero-size/display:contents wrapper can still expose destination
+ // text. Shadow subtrees have no complete text binding here.
+ if (node.shadowRoot) return false;
+ if (boundTextNodes.has(node)) continue;
+ if (node.matches(modelEntityIdentitySelector) || ['aria-label', 'aria-labelledby', 'aria-describedby', 'title', 'alt']
+ .some(attribute => node.getAttribute(attribute)?.trim())) return false;
+ if (visited + stack.length + node.childNodes.length > 256) return false;
+ stack.push(...node.childNodes);
+ }
+ // Raw textContent cannot detect which nested audience label is rendered.
+ // Bind descendant semantics for controls and referenced full-text labels,
+ // including label roots outside the dialog. Editable descendants keep
+ // their existing value continuation instead of acquiring extra bindings.
+ for (const root of boundTextNodes) {
+ const descendants = root.querySelectorAll('*');
+ if (root.shadowRoot || descendants.length > 256) return false;
+ for (const child of descendants) {
+ if (ignored(child)) continue;
+ if (child.shadowRoot || (child.matches(modelEntityIdentitySelector) && !boundTextNodes.has(child))) return false;
+ // An ARIA textbox can be a read-only destination display. Only real
+ // editable DOM hosts need the native value-continuation exemption.
+ if (!child.isContentEditable && !child.closest('input,textarea,select') && !addNode(child)) return false;
+ }
+ }
+ return true;
+ }
function actionFootprint(target) {
const structuralFile = target instanceof Element && target.tagName === 'INPUT' && target.type === 'file';
if (!(target instanceof Element) || target.ownerDocument !== document || !target.isConnected
@@ -670,7 +711,8 @@
}
if (!owner) { owner = fallback; unstructuredOwner = true; }
}
- const nodes = [], ancestors = [];
+ const nodes = [], ancestors = [], boundTextNodes = new Set();
+ let uncoveredLabel = false;
const addNode = (node, options = {}) => {
if (nodes.some(item => item.node === node)) return true;
if (nodes.length >= 256 || (node.tagName === 'INPUT' && node.type === 'file' && node.files?.length > 32)) return false;
@@ -678,9 +720,12 @@
if (!labels) return false;
nodes.push({ node, value: modelSemantic(node, options),
...(node.tagName === 'INPUT' && node.type === 'file' ? { files: [...node.files || []] } : {}) });
+ if (options.text) boundTextNodes.add(node);
for (const label of labels) {
if (nodes.length >= 256) return false;
- if (!nodes.some(item => item.node === label.node)) nodes.push(label);
+ if (!nodes.some(item => item.node === label.node)) {
+ nodes.push(label); boundTextNodes.add(label.node);
+ } else if (!boundTextNodes.has(label.node)) uncoveredLabel = true;
}
return true;
};
@@ -716,12 +761,13 @@
for (const control of controls) if (!addNode(control, { text: true, control: true })) return null;
} else if (owner) {
if (!addNode(owner, { directText: true })) return null;
- const belongs = node => {
+ const belongs = (node, allowNavigation = false) => {
const closest = node.closest(entityOwnerSelector);
const relatedForm = node.closest('form,[role="form"]');
return (!closest || closest === owner || closest.contains(owner))
&& (!relatedForm || relatedForm === form)
- && !node.closest('aside,nav,[role="navigation"],[role="complementary"]');
+ && !node.closest(allowNavigation === true ? 'aside,[role="complementary"]'
+ : 'aside,nav,[role="navigation"],[role="complementary"]');
};
const headings = [...owner.querySelectorAll('h1,h2,h3,h4,h5,h6,[role="heading"],[aria-current]')]
.filter(belongs);
@@ -733,9 +779,13 @@
meaningfulOwner ||= headings.length > 0 || identities.length > 0 || owner.matches(modelEntityIdentitySelector);
// Recipients and attachment/caption editors may live next to the form.
// Bind every local control, excluding a separate entity, form or sidebar.
+ const headingFreeDialog = !meaningfulOwner && owner.matches('dialog,[role="dialog"],[role="alertdialog"]');
for (const control of owner.querySelectorAll(modelControlSelector)) {
- if (belongs(control) && !addNode(control, { text: true, control: true })) return null;
+ if (belongs(control, headingFreeDialog) && !addNode(control, { text: true, control: true })) return null;
}
+ // A heading-free dialog can supply a boundary for a controls-only
+ // composer. Uncovered generic destination text keeps the strict fence.
+ meaningfulOwner ||= headingFreeDialog && dialogTextIsCovered(owner, boundTextNodes, addNode) && !uncoveredLabel;
}
// An arbitrary container/toolbar with no entity anchor cannot establish
// the recipient or item. Such targets retain strict legacy dispatch only.
diff --git a/src/chrome/src/ui/settings.js b/src/chrome/src/ui/settings.js
index df6835289..8aa5ba292 100644
--- a/src/chrome/src/ui/settings.js
+++ b/src/chrome/src/ui/settings.js
@@ -90,7 +90,7 @@ const SUBSCRIPTION_GUIDE_PRODUCTS = Object.freeze({
// Version shown in the subtitle. Kept here so it only needs one update per
// release; the subtitle string itself is translated.
-const EXT_VERSION = '40.0.6';
+const EXT_VERSION = '40.0.7';
const providersContainer = document.getElementById('providers');
const displaySettings = document.getElementById('display-settings');
diff --git a/src/firefox/ARCHITECTURE.md b/src/firefox/ARCHITECTURE.md
index a6e1ec6f5..d2c2bed78 100644
--- a/src/firefox/ARCHITECTURE.md
+++ b/src/firefox/ARCHITECTURE.md
@@ -1,6 +1,6 @@
# WebBrain Firefox Extension — Architecture
-> Version 40.0.6 · Manifest V2 · Background Page
+> Version 40.0.7 · Manifest V2 · Background Page
## How Firefox Differs from Chrome
diff --git a/src/firefox/manifest.json b/src/firefox/manifest.json
index 0255e4ced..2f85fddd8 100644
--- a/src/firefox/manifest.json
+++ b/src/firefox/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 2,
"name": "WebBrain",
- "version": "40.0.6",
+ "version": "40.0.7",
"description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.",
"permissions": [
"activeTab",
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index a45f22cce..b70cffbae 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -37245,7 +37245,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const feedbackCalls = result?.toolCalls?.length ? result.toolCalls
: !this._containsProviderReplayState(result?.responseItems) ? this._tryParseToolCallsFromText(result?.content || '', allowedToolNames) : [];
- if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, { responseToolCalls: feedbackCalls }))
+ if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, {
+ responseToolCalls: feedbackCalls, responseWithoutTools: feedbackCalls.length === 0,
+ }))
|| this._applyPendingSteering(tabId, messages, onUpdate)) {
const recovery = this._pageFeedbackRecoveryResult(tabId, messages, onUpdate);
if (recovery) {
@@ -38271,7 +38273,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const feedbackCalls = streamedToolCalls?.length ? streamedToolCalls
: !this._containsProviderReplayState(responseItems) ? this._tryParseToolCallsFromText(fullText || '', allowedToolNames) : [];
- if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, { responseToolCalls: feedbackCalls }))
+ if ((await this._applyPendingPageFeedback(tabId, messages, onUpdate, {
+ responseToolCalls: feedbackCalls, responseWithoutTools: feedbackCalls.length === 0,
+ }))
|| this._applyPendingSteering(tabId, messages, onUpdate)) {
const recovery = this._pageFeedbackRecoveryResult(tabId, messages, onUpdate);
if (recovery) {
diff --git a/src/firefox/src/agent/page-feedback-policy.js b/src/firefox/src/agent/page-feedback-policy.js
index 75c2615b1..76fdf0d74 100644
--- a/src/firefox/src/agent/page-feedback-policy.js
+++ b/src/firefox/src/agent/page-feedback-policy.js
@@ -96,7 +96,13 @@ export function pageFeedbackCallPolicy(name, args = {}, state = {}, currentPage
}
if (name === 'download_public_media' && (httpUrl(args.url) || httpUrl(state.url))) return { kind: 'independent' };
if (name === 'navigate' && httpUrl(args.url)) return { kind: 'navigate' };
- if (name === 'done' && (['partial', 'failure'].includes(args.outcome)
+ // Ask exposes a summary-only done schema. Its answer can come from completed
+ // detached research, so unrelated counters (or a failed AX refresh) do not
+ // require a new inference. The run mode is private pre-inference state.
+ const askSummary = state.runMode === 'ask'
+ && !Object.prototype.hasOwnProperty.call(args, 'outcome')
+ && typeof args.summary === 'string' && !!args.summary.trim();
+ if (name === 'done' && (askSummary || ['partial', 'failed'].includes(args.outcome)
|| (args.outcome === 'success' && currentPage?.success !== false))) return { kind: 'observe', terminal: true };
if (name === 'download_social_media' && !args.scroll && !args.all
&& [undefined, 'main', 'auto'].includes(args.mode)
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index b3837e376..7c5305101 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -474,7 +474,8 @@ export const pageFeedbackMethods = {
const frames = await apiFor().webNavigation?.getAllFrames?.({ tabId });
documentId = frames?.find(frame => frame.frameId === 0)?.documentId || documentId;
} catch {}
- const state = { token: run.token, url, documentId, page: run.latestPage,
+ const runMode = this._effectiveRunMode?.(tabId, null) || null;
+ const state = { token: run.token, url, documentId, page: run.latestPage, runMode,
frames: new Map([...run.frames].map(([id, frame]) => [id, { token: frame.token, id: frame.id }])),
steeringRevision: this._steeringRuns?.get(tabId)?.acceptedIds.size || 0, mediaBindings: {} };
// The downloader is lazy-loaded. Capture its private document/node/asset
@@ -577,6 +578,7 @@ export const pageFeedbackMethods = {
_pageFeedbackStateCurrent(tabId, run, state, url = run.url) {
return !!state && state.token === run.token && state.url === url
+ && (state.runMode === undefined || state.runMode === (this._effectiveRunMode?.(tabId, null) || null))
&& state.steeringRevision === (this._steeringRuns?.get(tabId)?.acceptedIds.size || 0)
&& [...state.frames].every(([id, frame]) => run.frames.get(id)?.token === frame.token
&& run.frames.get(id)?.id === frame.id);
@@ -697,7 +699,8 @@ export const pageFeedbackMethods = {
},
async _applyPendingPageFeedback(tabId, messages, onUpdate = () => {},
- { workflow = false, responseToolCalls = null, stage = responseToolCalls === null ? 'observation' : 'response' } = {}) {
+ { workflow = false, responseToolCalls = null, responseWithoutTools = false,
+ stage = responseToolCalls === null ? 'observation' : 'response' } = {}) {
if (!this._hasPendingPageFeedback(tabId) || this._checkAbort(tabId)) return false;
const started = Date.now();
await this._waitForPageFeedbackIdle(tabId);
@@ -730,8 +733,16 @@ export const pageFeedbackMethods = {
}
const sameDocument = documentMatches && this._pageFeedbackRuns.get(tabId) === run
&& this._pageFeedbackStateCurrent(tabId, run, state, url);
- let retained = sameDocument && canRetainPageFeedbackCalls(responseToolCalls, state, page, events)
- && (!pending.length || canRetainPageFeedbackCalls(responseToolCalls, state, page, pending, { pending: true }));
+ // A tool-free Ask response is read-only, including an empty response that
+ // must reach the provider's bounded delivery recovery. This does not mint
+ // completion evidence or retain any mutation, structured terminal or tool.
+ const readonlyAskResponse = responseWithoutTools === true && state.runMode === 'ask'
+ && Array.isArray(responseToolCalls) && responseToolCalls.length === 0;
+ const canRetain = (feedback, options) => readonlyAskResponse
+ ? isPassivePageFeedback(feedback)
+ : canRetainPageFeedbackCalls(responseToolCalls, state, page, feedback, options);
+ let retained = sameDocument && canRetain(events)
+ && (!pending.length || canRetain(pending, { pending: true }));
if (retained) for (const call of responseToolCalls) {
const args = JSON.parse(call.function.arguments);
if (pageFeedbackCallPolicy(call.function.name, args, state, page, events).kind === 'bound_target'
@@ -767,7 +778,7 @@ export const pageFeedbackMethods = {
}
// Capture/description can itself receive a hard intervention.
const finalPending = [...run.events.values()];
- const accepted = retained && (!finalPending.length || canRetainPageFeedbackCalls(responseToolCalls, state, page, finalPending, { pending: true }));
+ const accepted = retained && (!finalPending.length || canRetain(finalPending, { pending: true }));
if (Array.isArray(responseToolCalls) && !accepted) {
const recovery = accountFeedbackSupersession(run, { passive: sameDocument && isPassivePageFeedback(events)
&& (!finalPending.length || isPassivePageFeedback(finalPending)), stage,
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index c27986de0..2b19a5747 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -632,6 +632,47 @@
return context;
} catch { return null; }
}
+ function dialogTextIsCovered(owner, boundTextNodes, addNode) {
+ const stack = [...owner.childNodes];
+ let visited = 0;
+ while (stack.length) {
+ if (++visited > 256) return false;
+ const node = stack.pop();
+ if (node.nodeType === 3) {
+ // Direct owner text is already bound; generic nested text may name a
+ // destination that the control/heading footprint cannot establish.
+ if (node.data.trim() && node.parentElement !== owner) return false;
+ continue;
+ }
+ if (!(node instanceof Element) || ignored(node)) continue;
+ const style = getComputedStyle(node);
+ if (style.display === 'none' || style.opacity === '0' || style.contentVisibility === 'hidden') continue;
+ // A zero-size/display:contents wrapper can still expose destination
+ // text. Shadow subtrees have no complete text binding here.
+ if (node.shadowRoot) return false;
+ if (boundTextNodes.has(node)) continue;
+ if (node.matches(modelEntityIdentitySelector) || ['aria-label', 'aria-labelledby', 'aria-describedby', 'title', 'alt']
+ .some(attribute => node.getAttribute(attribute)?.trim())) return false;
+ if (visited + stack.length + node.childNodes.length > 256) return false;
+ stack.push(...node.childNodes);
+ }
+ // Raw textContent cannot detect which nested audience label is rendered.
+ // Bind descendant semantics for controls and referenced full-text labels,
+ // including label roots outside the dialog. Editable descendants keep
+ // their existing value continuation instead of acquiring extra bindings.
+ for (const root of boundTextNodes) {
+ const descendants = root.querySelectorAll('*');
+ if (root.shadowRoot || descendants.length > 256) return false;
+ for (const child of descendants) {
+ if (ignored(child)) continue;
+ if (child.shadowRoot || (child.matches(modelEntityIdentitySelector) && !boundTextNodes.has(child))) return false;
+ // An ARIA textbox can be a read-only destination display. Only real
+ // editable DOM hosts need the native value-continuation exemption.
+ if (!child.isContentEditable && !child.closest('input,textarea,select') && !addNode(child)) return false;
+ }
+ }
+ return true;
+ }
function actionFootprint(target) {
const structuralFile = target instanceof Element && target.tagName === 'INPUT' && target.type === 'file';
if (!(target instanceof Element) || target.ownerDocument !== document || !target.isConnected
@@ -670,7 +711,8 @@
}
if (!owner) { owner = fallback; unstructuredOwner = true; }
}
- const nodes = [], ancestors = [];
+ const nodes = [], ancestors = [], boundTextNodes = new Set();
+ let uncoveredLabel = false;
const addNode = (node, options = {}) => {
if (nodes.some(item => item.node === node)) return true;
if (nodes.length >= 256 || (node.tagName === 'INPUT' && node.type === 'file' && node.files?.length > 32)) return false;
@@ -678,9 +720,12 @@
if (!labels) return false;
nodes.push({ node, value: modelSemantic(node, options),
...(node.tagName === 'INPUT' && node.type === 'file' ? { files: [...node.files || []] } : {}) });
+ if (options.text) boundTextNodes.add(node);
for (const label of labels) {
if (nodes.length >= 256) return false;
- if (!nodes.some(item => item.node === label.node)) nodes.push(label);
+ if (!nodes.some(item => item.node === label.node)) {
+ nodes.push(label); boundTextNodes.add(label.node);
+ } else if (!boundTextNodes.has(label.node)) uncoveredLabel = true;
}
return true;
};
@@ -716,12 +761,13 @@
for (const control of controls) if (!addNode(control, { text: true, control: true })) return null;
} else if (owner) {
if (!addNode(owner, { directText: true })) return null;
- const belongs = node => {
+ const belongs = (node, allowNavigation = false) => {
const closest = node.closest(entityOwnerSelector);
const relatedForm = node.closest('form,[role="form"]');
return (!closest || closest === owner || closest.contains(owner))
&& (!relatedForm || relatedForm === form)
- && !node.closest('aside,nav,[role="navigation"],[role="complementary"]');
+ && !node.closest(allowNavigation === true ? 'aside,[role="complementary"]'
+ : 'aside,nav,[role="navigation"],[role="complementary"]');
};
const headings = [...owner.querySelectorAll('h1,h2,h3,h4,h5,h6,[role="heading"],[aria-current]')]
.filter(belongs);
@@ -733,9 +779,13 @@
meaningfulOwner ||= headings.length > 0 || identities.length > 0 || owner.matches(modelEntityIdentitySelector);
// Recipients and attachment/caption editors may live next to the form.
// Bind every local control, excluding a separate entity, form or sidebar.
+ const headingFreeDialog = !meaningfulOwner && owner.matches('dialog,[role="dialog"],[role="alertdialog"]');
for (const control of owner.querySelectorAll(modelControlSelector)) {
- if (belongs(control) && !addNode(control, { text: true, control: true })) return null;
+ if (belongs(control, headingFreeDialog) && !addNode(control, { text: true, control: true })) return null;
}
+ // A heading-free dialog can supply a boundary for a controls-only
+ // composer. Uncovered generic destination text keeps the strict fence.
+ meaningfulOwner ||= headingFreeDialog && dialogTextIsCovered(owner, boundTextNodes, addNode) && !uncoveredLabel;
}
// An arbitrary container/toolbar with no entity anchor cannot establish
// the recipient or item. Such targets retain strict legacy dispatch only.
diff --git a/src/firefox/src/ui/settings.js b/src/firefox/src/ui/settings.js
index ce8a92640..027d25c81 100644
--- a/src/firefox/src/ui/settings.js
+++ b/src/firefox/src/ui/settings.js
@@ -82,7 +82,7 @@ const SUBSCRIPTION_GUIDE_PRODUCTS = Object.freeze({
// Version shown in the subtitle. Kept here so it only needs one update per
// release; the subtitle string itself is translated.
-const EXT_VERSION = '40.0.6';
+const EXT_VERSION = '40.0.7';
const providersContainer = document.getElementById('providers');
const displaySettings = document.getElementById('display-settings');
diff --git a/test/page-feedback-policy.mjs b/test/page-feedback-policy.mjs
index 0adb06c34..054ffbe12 100644
--- a/test/page-feedback-policy.mjs
+++ b/test/page-feedback-policy.mjs
@@ -140,7 +140,7 @@ for (const build of ['chrome', 'firefox']) {
}
});
}
- for (const outcome of ['partial', 'failure']) {
+ for (const outcome of ['partial', 'failed']) {
test(`${build}: explicit ${outcome} completion survives counters without becoming success`, () => {
const candidate = call('done', { summary: 'The task remains incomplete.', outcome });
assert.equal(canRetainPageFeedbackCalls([candidate], state, { success: false }, passive, { pending: true }), true);
@@ -148,6 +148,32 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(canRetainPageFeedbackCalls([candidate], state, page, [{ kind: 'pointer', source: 'user', frameId: 0 }]), false);
});
}
+ test(`${build}: summary-only Ask completion survives counters and unavailable AX reads`, () => {
+ const candidate = call('done', { summary: 'Answer from completed detached research.' });
+ const ask = { ...state, runMode: 'ask' };
+ for (const currentPage of [page, { success: false }, null]) {
+ assert.equal(canRetainPageFeedbackCalls([candidate], ask, currentPage, passive, { pending: true }), true);
+ }
+ assert.equal(canRetainPageFeedbackCalls([candidate, call('research_url', { url: 'https://example.com' })], ask, page, passive), false);
+ for (const feedback of [
+ { kind: 'input', source: 'user', frameId: 0 },
+ { kind: 'navigation', source: 'page', frameId: 0 },
+ { kind: 'dom', source: 'unknown', frameId: 0 },
+ { kind: 'dom', source: 'page', frameId: 1 },
+ { kind: 'dom', source: 'page', frameId: 0, target: 'style' },
+ ]) assert.equal(canRetainPageFeedbackCalls([candidate], ask, page, [feedback]), false);
+ });
+ test(`${build}: action and unknown modes cannot borrow Ask's summary-only completion`, () => {
+ const args = { summary: 'Claimed answer', runMode: 'ask', mode: 'ask' };
+ for (const runMode of ['act', 'dev', null, undefined]) {
+ assert.equal(pageFeedbackCallPolicy('done', args, { ...state, runMode }, page).kind, 'unsafe');
+ }
+ for (const outcome of ['failure', '', null, 'invented']) {
+ assert.equal(pageFeedbackCallPolicy('done', { summary: 'Incomplete', outcome }, { ...state, runMode: 'ask' }, page).kind, 'unsafe');
+ }
+ assert.equal(pageFeedbackCallPolicy('done_json', { data: { answer: 'Structured' } }, { ...state, runMode: 'ask' }, page).kind, 'unsafe');
+ assert.equal(pageFeedbackCallPolicy('done', { summary: 'Claimed success', outcome: 'success' }, { ...state, runMode: 'ask' }, { success: false }).kind, 'unsafe');
+ });
test(`${build}: selectorless input needs a captured focus before becoming a binding candidate`, () => {
for (const name of ['type_text', 'press_keys']) {
assert.equal(pageFeedbackCallPolicy(name, {}, state, page).kind, 'unsafe');
diff --git a/test/page-feedback-progress.mjs b/test/page-feedback-progress.mjs
index 1530a8006..0d03a005c 100644
--- a/test/page-feedback-progress.mjs
+++ b/test/page-feedback-progress.mjs
@@ -85,6 +85,100 @@ for (const build of ['chrome', 'firefox']) {
const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
for (const streaming of [false, true]) {
+ // webbrain-traces-1791395643317.json: completed detached research followed
+ // by repeated valid Ask completions discarded because X counters changed.
+ for (const response of ['done', 'prose', 'empty']) for (const refreshFails of [false, true]) {
+ test(`${build}: ${streaming ? 'stream' : 'chat'} Ask ${response} reaches delivery under passive churn${refreshFails ? ' with failed AX refresh' : ''}`, async () => {
+ const tab = nextTab++, updates = [], dispatched = [];
+ const answer = 'Robot Ventures is an investment firm founded by Robert Leshner and Tarun Chitra.';
+ let agent, send, requests = 0, reads = 0;
+ const next = async () => {
+ assert.ok(++requests <= (response === 'empty' ? 3 : 2), 'Passive counters cannot consume the answer or provider recovery budget');
+ send({});
+ if (requests === 1) return call('research-founders', 'research_url', { url: 'https://example.com/robot-ventures' });
+ if (response === 'done') return call('researched-answer', 'done', { summary: answer });
+ return { content: response === 'prose' ? answer : '' };
+ };
+ agent = makeAgent(Agent, next);
+ agent.maxSteps = Infinity;
+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => {
+ send = bind(agent, tab);
+ return { role: 'user', content };
+ };
+ agent.executeTool = async (_tab, name, args) => {
+ if (name === 'get_accessibility_tree') {
+ reads++; send({});
+ return refreshFails ? { success: false, error: 'AX refresh unavailable' }
+ : { success: true, pageContent: `article "Unrelated timeline" [ref_1]\n text "${reads} views"` };
+ }
+ dispatched.push(name);
+ if (name === 'done') return { success: true, done: true, summary: args.summary };
+ assert.equal(name, 'research_url');
+ return { success: true, content: answer };
+ };
+ const options = { detachedRequestId: 'ask-feedback-run', askStreamingEnabled: true };
+ const update = (type, data) => updates.push({ type, data });
+ const result = streaming
+ ? await agent.processMessageStream(tab, 'Who founded Robot Ventures?', update, 'ask', options)
+ : await agent.processMessage(tab, 'Who founded Robot Ventures?', update, 'ask', [], options);
+ if (response === 'empty') {
+ assert.match(result, /empty|visible answer|visible text/i);
+ assert.equal(requests, 3, 'The provider receives exactly one recovery turn after its first empty response');
+ } else {
+ assert.equal(result, answer);
+ assert.equal(requests, 2);
+ }
+ assert.deepEqual(dispatched, response === 'done' ? ['research_url', 'done'] : ['research_url']);
+ assert.ok(reads >= requests, 'Every pending passive batch is still observed');
+ assert.equal(updates.some(update => update.type === 'warning' && /page kept changing/.test(update.data.message)), false);
+ assert.equal(agent._pageFeedbackRuns.size, 0);
+ });
+ }
+ for (const response of ['done', 'prose']) for (const change of ['human_input', 'unknown', 'navigation', 'style', 'frame', 'task_steering', 'document', 'mode_change']) {
+ test(`${build}: ${streaming ? 'stream' : 'chat'} Ask ${response} is reconsidered after ${change}`, async () => {
+ const tab = nextTab++, updates = [], delivered = [];
+ let agent, send, requests = 0;
+ const next = async () => {
+ if (++requests === 1) {
+ if (change === 'task_steering') {
+ assert.equal(agent.steerMessage(tab, 'Use the corrected question', { requestId: 'ask-guard-run', messageId: 'ask-correction' }).accepted, true);
+ } else if (change === 'navigation') {
+ agent._queuePageFeedback(tab, { kind: 'navigation', source: 'user', frameId: 0, before: PAGE_URL, after: PAGE_URL });
+ } else if (change === 'frame') {
+ agent._queuePageFeedback(tab, { kind: 'dom', source: 'page', frameId: 1, target: 'span#views' });
+ } else if (change === 'document') {
+ documents.set(tab, 'new-document'); send({});
+ } else if (change === 'mode_change') {
+ agent._runModeOverrides.set(tab, 'act'); send({});
+ } else send({ kind: change === 'human_input' ? 'input' : 'dom',
+ source: change === 'human_input' ? 'user' : change === 'unknown' ? 'unknown' : 'page',
+ ...(change === 'style' ? { target: 'style' } : {}) });
+ return response === 'done' ? call('stale-answer', 'done', { summary: 'Old answer' }) : { content: 'Old answer' };
+ }
+ assert.equal(requests, 2);
+ return { content: 'Fresh answer' };
+ };
+ agent = makeAgent(Agent, next);
+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => {
+ send = bind(agent, tab); return { role: 'user', content };
+ };
+ agent.executeTool = async (_tab, name, args) => {
+ if (name === 'get_accessibility_tree') return { success: true, pageContent: 'article "Fresh context" [ref_1]' };
+ delivered.push(args.summary); assert.fail(`Stale ${name} must not dispatch`);
+ };
+ const update = (type, data) => updates.push({ type, data });
+ const options = { detachedRequestId: 'ask-guard-run', askStreamingEnabled: true };
+ try {
+ const result = streaming
+ ? await agent.processMessageStream(tab, 'Read the current question', update, 'ask', options)
+ : await agent.processMessage(tab, 'Read the current question', update, 'ask', [], options);
+ assert.equal(result, 'Fresh answer');
+ assert.equal(requests, 2);
+ assert.deepEqual(delivered, []);
+ assert.equal(agent._pageFeedbackRuns.size, 0);
+ } finally { documents.delete(tab); }
+ });
+ }
test(`${build}: ${streaming ? 'stream' : 'chat'} same-URL document replacement supersedes a read before the monitor registers`, async () => {
const tab = nextTab++, updates = [], dispatched = [];
let agent, requests = 0;
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 917ec3afa..37d7104b4 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -2,8 +2,9 @@ import assert from 'node:assert/strict';
import fs from 'node:fs';
import { test } from 'node:test';
import { chromium, firefox } from 'playwright';
-import { CDPClient } from '../src/chrome/src/cdp/cdp-client.js';
-import { pageFeedbackMethods } from '../src/chrome/src/agent/page-feedback.js';
+import { CDPClient, cdpClient } from '../src/chrome/src/cdp/cdp-client.js';
+import { Agent } from '../src/chrome/src/agent/agent.js';
+import { installPageFeedback, pageFeedbackMethods } from '../src/chrome/src/agent/page-feedback.js';
import { BidiSession, createNativeActionMarker } from '../firefox-companion/session.mjs';
const read = (build, file) => fs.readFileSync(new URL(`../src/${build}/src/${file}`, import.meta.url), 'utf8');
@@ -134,6 +135,171 @@ test('Chrome AX preparation attributes actual scrolling without claiming field i
} finally { await browser.close(); }
});
+test('Chrome controlled-field trusted typing survives its focus-created formatting toolbar', async () => {
+ const savedChrome = globalThis.chrome, savedBrowser = globalThis.browser, tab = 919;
+ const savedCdpAttach = cdpClient.attach, savedCdpSend = cdpClient.sendCommand;
+ let changeToolbarAfterFocus = false;
+ class FieldHost {
+ static STATE_CHANGE_TOOLS = new Set(['set_field']);
+ isRunning() { return true; }
+ _checkAbort() { return false; }
+ _throwIfAborted() {}
+ _persist() {}
+ async _textMutationValueDigest(...args) { return Agent.prototype._textMutationValueDigest.call(this, ...args); }
+ _appOwnedUserMessage(content) { return { role: 'user', content }; }
+ async executeTool(tabId, name, args, _update, executionContext) {
+ executionContext._contentActionDispatchState.started = true;
+ const result = await Agent.prototype._maybeFallbackFieldWithCdpImpl.call(this, tabId, name, args,
+ { success: false, verified: false, dispatched: false, noDispatch: true, _expectedValue: args.text,
+ error: 'Contenteditable fields require trusted browser typing', trustedTypeRequired: true });
+ return result.textUnchanged === true
+ ? Agent.prototype._finalizeTextMutationResult.call(this, tabId, name, args, result)
+ : result;
+ }
+ }
+ installPageFeedback(FieldHost);
+ const host = new FieldHost();
+ let browser, page, session;
+ const commands = [], preparations = [];
+ const api = { runtime: {}, tabs: {
+ get: async () => ({ url: 'https://monitor.test/start' }),
+ sendMessage: async (_tab, message) => {
+ if (!page) return {};
+ if (message.action.startsWith('ax_') || message.action === 'field_value_digest') return page.evaluate(async message => {
+ window.msg = message;
+ let finish = () => {};
+ try {
+ finish = __wbPageMonitor.beginContentAction(message.action, message.params || {});
+ const result = await axHelpers[message.action]();
+ if (message.action === 'field_value_digest') {
+ result.documentToken = document.documentElement.getAttribute('data-webbrain-page-revision').split(':')[0];
+ result.refScopeUrl = location.href;
+ }
+ return result;
+ } catch (error) {
+ return { success: false, dispatched: error.dispatched === true, noDispatch: error.dispatched !== true,
+ error: `${message.action} failed: ${error.message}` };
+ } finally { finish(); }
+ }, message).then(result => { if (message.action === 'ax_prepare_field_for_trusted_type') preparations.push(result); return result; });
+ return page.evaluate(message => new Promise(resolve => messageListeners.forEach(fn => fn(message, {}, resolve))), message);
+ },
+ }, debugger: { sendCommand(_source, method, params, callback) {
+ commands.push({ method, params });
+ void (async () => {
+ const result = await session.send(method, params);
+ if (changeToolbarAfterFocus && method === 'Input.dispatchMouseEvent' && params.type === 'mouseReleased') {
+ changeToolbarAfterFocus = false;
+ await page.waitForTimeout(30);
+ await page.evaluate(() => document.getElementById('formatting-toolbar').setAttribute('aria-label', 'Text formatting open'));
+ }
+ return result;
+ })().then(callback, error => {
+ api.runtime.lastError = { message: error.message }; callback(); delete api.runtime.lastError;
+ });
+ } } };
+ globalThis.chrome = api; delete globalThis.browser;
+ try {
+ await host._beginPageFeedbackRun(tab, 'interactive');
+ ({ browser, page } = await fixture(chromium, 'chrome', { runToken: host._pageFeedbackRuns.get(tab).token }));
+ const documentToken = await page.evaluate(() => document.documentElement.getAttribute('data-webbrain-page-revision').split(':')[0]);
+ host.pageMonitorState({ tab: { id: tab }, frameId: 0 }, documentToken);
+ await page.exposeFunction('feedbackToHost', feedback => host.observePageFeedback({ tab: { id: tab }, frameId: 0 }, feedback));
+ await page.exposeFunction('monitorStateFromHost', token => host.pageMonitorState({ tab: { id: tab }, frameId: 0 }, token));
+ await page.evaluate(() => {
+ const original = chrome.runtime.sendMessage;
+ chrome.runtime.sendMessage = async message => {
+ if (message.action === 'get_page_monitor_state') return await monitorStateFromHost(message.documentToken);
+ const response = await original(message);
+ return message.action === 'page_feedback' ? await feedbackToHost(message.feedback) : response;
+ };
+ const editor = document.createElement('div');
+ editor.id = 'ref_trusted_editor'; editor.contentEditable = 'true'; editor.setAttribute('role', 'textbox');
+ editor.setAttribute('aria-label', 'Post text'); editor.style.cssText = 'width:500px;height:96px;margin:12px';
+ editor.innerHTML = '<div><br></div>';
+ const dialog = document.createElement('div'); dialog.id = 'ref_compose_dialog'; dialog.setAttribute('role', 'dialog');
+ dialog.setAttribute('aria-modal', 'true'); dialog.append(editor); document.body.prepend(dialog);
+ editor.addEventListener('focus', () => {
+ if (document.getElementById('formatting-toolbar')) return;
+ const toolbar = document.createElement('div'); toolbar.id = 'formatting-toolbar';
+ toolbar.setAttribute('role', 'toolbar'); toolbar.setAttribute('aria-label', 'Text formatting');
+ toolbar.style.cssText = 'position:fixed;left:600px;top:200px';
+ toolbar.innerHTML = '<button aria-label="Bold">B</button><button aria-label="Italic">I</button>';
+ document.body.append(toolbar);
+ });
+ window.__wb_ax_lookup = ref => document.getElementById(ref);
+ window.__wb_ax_ref = node => node.id;
+ window.__generateAccessibilityTree = () => ({ pageContent: 'dialog [ref_compose_dialog]\ntextbox "Post text" [ref_trusted_editor]',
+ viewport: { width: innerWidth, height: innerHeight } });
+ window._fieldMeta = el => ({ tag: 'div', contentEditable: true, ariaLabel: el.getAttribute('aria-label') });
+ window._editableTextValue = el => el.innerText.replace(/\n$/, '');
+ window.readProseMirrorText = () => null;
+ window._stableFieldSelector = el => '#' + el.id;
+ window._setFieldValueMatches = (actual, prefix, text, clear) => actual === (clear ? '' : prefix) + text;
+ });
+ const source = read('chrome', 'content/content.js');
+ const entries = ['ax_prepare_field_for_trusted_type', 'ax_verify_field_value', 'field_value_digest'].map(action => {
+ const start = source.indexOf(`'${action}': ${action === 'field_value_digest' ? 'async ' : ''}() => {`), end = source.indexOf('\n },', start);
+ assert.ok(start > 0 && end > start); return source.slice(start, end + 8);
+ });
+ await page.addScriptTag({ content: `window.axHelpers = { ${entries.join(',\n')} };` });
+ await page.waitForTimeout(250);
+ await page.evaluate(() => { feedback = []; });
+ host._pageFeedbackRuns.get(tab).events.clear();
+ const client = new CDPClient(); client.sessions.set(tab, { attached: true });
+ session = await page.context().newCDPSession(page);
+ cdpClient.attach = async () => ({ attached: true });
+ cdpClient.sendCommand = (...args) => client.sendCommand(...args);
+ await host._capturePageFeedbackModelState(tab, []);
+ assert.equal(host._pageFeedbackRuns.get(tab).modelState.actionBinding.focusedTargetAvailable, false);
+ const capturedBinding = await page.evaluate(runToken => {
+ deliver('page_monitor_capture_model', { runToken, includeTree: true }); return lastMonitorResponse;
+ }, host._pageFeedbackRuns.get(tab).token);
+ assert.ok(capturedBinding.targetCount > 0, 'The heading-free compose dialog must certify its editor');
+ const result = await host.executeTool(tab, 'set_field',
+ { ref_id: 'ref_trusted_editor', text: 'A verified post', clear: true });
+ assert.equal(result.success, true, JSON.stringify({ result, preparations, feedback: await page.evaluate(() => feedback), commands: commands.filter(item => item.method.startsWith('Input.')) }));
+ assert.equal(await page.locator('#ref_trusted_editor').innerText(), 'A verified post');
+ assert.equal(commands.filter(item => item.method === 'Input.insertText').length, 1);
+
+ await page.waitForTimeout(250);
+ host._pageFeedbackRuns.get(tab).events.clear();
+ await host._capturePageFeedbackModelState(tab, []);
+ commands.length = 0; preparations.length = 0; changeToolbarAfterFocus = true;
+ const interrupted = await host.executeTool(tab, 'set_field',
+ { ref_id: 'ref_trusted_editor', text: 'The next verified post', clear: true });
+ assert.equal(interrupted.success, false);
+ assert.match(interrupted.error, /could not reselect it.*Browser changed during action preparation/);
+ assert.equal(preparations.length, 2);
+ assert.equal(preparations[0].success, true);
+ assert.equal(preparations[1].success, false);
+ assert.equal(commands.some(item => item.method === 'Input.insertText'), false,
+ 'A toolbar change between trusted focus and selection must stop text dispatch');
+ assert.equal(interrupted.dispatched, true, 'The trusted focus click remains a real dispatched interaction');
+ assert.equal(interrupted.textUnchanged, true, 'Before/after real SHA-256 readback proves the editor unchanged');
+ assert.equal(interrupted.mutationMayHaveOccurred, false);
+ assert.equal(interrupted.repeatBlocked, false);
+ assert.equal(interrupted.recoveryRequired, 'fresh_tree');
+ assert.equal(host._uncertainTextMutations?.get(tab)?.size || 0, 0);
+ assert.equal(await page.locator('#ref_trusted_editor').innerText(), 'A verified post');
+
+ await page.waitForTimeout(250);
+ host._pageFeedbackRuns.get(tab).events.clear();
+ await host._capturePageFeedbackModelState(tab, []);
+ const retried = await host.executeTool(tab, 'set_field',
+ { ref_id: 'ref_trusted_editor', text: 'The next verified post', clear: true });
+ assert.equal(retried.success, true, JSON.stringify(retried));
+ assert.equal(await page.locator('#ref_trusted_editor').innerText(), 'The next verified post');
+ assert.equal(commands.filter(item => item.method === 'Input.insertText').length, 1);
+ } finally {
+ cdpClient.attach = savedCdpAttach; cdpClient.sendCommand = savedCdpSend;
+ if (session) await session.detach().catch(() => {});
+ if (host._pageFeedbackRuns?.has(tab)) host._finishPageFeedbackRun(tab);
+ if (savedChrome === undefined) delete globalThis.chrome; else globalThis.chrome = savedChrome;
+ if (savedBrowser === undefined) delete globalThis.browser; else globalThis.browser = savedBrowser;
+ if (browser) await browser.close();
+ }
+});
+
test('Chrome full-page capture marks temporary scrolling without suppressing a later page scroll', async () => {
const { browser, page } = await fixture(chromium, 'chrome');
const tabId = 88001;