Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix page feedback review findings
Barack Sokullu89f40cc55195
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex 3fcac5078..c7bd3f2a0 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -2,6 +2,47 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join, basename } from 'node:path';+const sipMask = (1n << 64n) - 1n;+const sipRotate = (value, bits) => ((value << BigInt(bits)) | (value >> BigInt(64 - bits))) & sipMask;+function pageActionSignature(secret, data) {+ if (!/^[a-f\d]{32}$/i.test(secret || '')) return '';+ const key = Buffer.from(secret, 'hex');+ const read64 = offset => {+ let value = 0n;+ for (let index = 0; index < 8; index++) value |= BigInt(key[offset + index]) << BigInt(index * 8);+ return value;+ };+ const k0 = read64(0), k1 = read64(8);+ let v0 = 0x736f6d6570736575n ^ k0, v1 = 0x646f72616e646f6dn ^ k1;+ let v2 = 0x6c7967656e657261n ^ k0, v3 = 0x7465646279746573n ^ k1;+ const round = () => {+ v0 = (v0 + v1) & sipMask; v1 = sipRotate(v1, 13); v1 ^= v0; v0 = sipRotate(v0, 32);+ v2 = (v2 + v3) & sipMask; v3 = sipRotate(v3, 16); v3 ^= v2;+ v0 = (v0 + v3) & sipMask; v3 = sipRotate(v3, 21); v3 ^= v0;+ v2 = (v2 + v1) & sipMask; v1 = sipRotate(v1, 17); v1 ^= v2; v2 = sipRotate(v2, 32);+ };+ const bytes = Buffer.from(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,+ data.navigationCandidate === true]));+ let offset = 0;+ while (offset + 8 <= bytes.length) {+ let block = 0n;+ for (let index = 0; index < 8; index++) block |= BigInt(bytes[offset + index]) << BigInt(index * 8);+ v3 ^= block; round(); round(); v0 ^= block; offset += 8;+ }+ let tail = BigInt(bytes.length) << 56n;+ for (let index = 0; offset + index < bytes.length; index++) tail |= BigInt(bytes[offset + index]) << BigInt(index * 8);+ v3 ^= tail; round(); round(); v0 ^= tail; v2 ^= 0xffn;+ round(); round(); round(); round();+ return (v0 ^ v1 ^ v2 ^ v3).toString(16).padStart(16, '0');+}++export function createNativeActionMarker(guard, kind, sequence) {+ const { nativeSecret, ...data } = guard || {};+ const marker = { ...data, kind, sequence };+ marker.signature = pageActionSignature(nativeSecret, marker);+ return JSON.stringify(marker);+}+function unpackRemoteObjectValue(value) {if (value?.type !== 'object' || !Array.isArray(value.value)) return value;return Object.fromEntries(value.value.flatMap(entry => {@@ -237,7 +278,7 @@ export class BidiSession {assertLive();const guard = payload.pageFeedbackGuard;if (!guard) return; // Compatibility with clients without page monitoring.- const marker = JSON.stringify({ ...guard, kind, sequence: ++nativeSequence });+ const marker = createNativeActionMarker(guard, kind, ++nativeSequence);const checked = await this.call(match, `(el, fence, marker, kind, action) => {if (!el.isConnected || document.documentElement.getAttribute('data-webbrain-page-revision') !== fence) return false;const target = kind === 'input' && action !== 'upload' ? el.getRootNode().activeElement || el : el;@@ -250,6 +291,18 @@ export class BidiSession {error.code = 'page_feedback_pending'; throw error;}assertLive();+ return marker;+ };+ const clearNativeDispatch = async marker => {+ if (!marker || !payload.pageFeedbackGuard) return;+ try {+ await this.call(match, `(el, marker, action) => {+ const kind = JSON.parse(marker).kind;+ const targets = kind === 'input' && action !== 'upload' ? [el, el.getRootNode().activeElement] : [el];+ for (const target of new Set(targets))+ if (target?.getAttribute('data-webbrain-native-action') === marker) target.removeAttribute('data-webbrain-native-action');+ }`, [{ type: 'string', value: marker }, { type: 'string', value: action }]);+ } catch { /* The document may have navigated or the monitor already consumed the marker. */ }};const point = action === 'click' && payload.point != null ? payload.point : null;if (point && (!Number.isInteger(point.x) || !Number.isInteger(point.y))) throw new Error('Invalid click coordinates');@@ -296,9 +349,11 @@ export class BidiSession {const path = join(dir, name === '.' || name === '..' ? 'attachment' : name);await writeFile(path, bytes, { mode: 0o600 });assertLive();- await markNativeDispatch('input');- dispatch.started = true;- await this.send('input.setFiles', { context: match.context, element: { sharedId: match.node.sharedId }, files: [path] });+ const marker = await markNativeDispatch('input');+ try {+ dispatch.started = true;+ await this.send('input.setFiles', { context: match.context, element: { sharedId: match.node.sharedId }, files: [path] });+ } finally { await clearNativeDispatch(marker); }const attached = await this.call(match, '(el, name, size) => el.isConnected && el.files?.length === 1 && el.files[0].name === name && el.files[0].size === size', [{ type: 'string', value: name }, { type: 'number', value: bytes.length }]);if (attached.result?.value !== true) return { success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: 'File input changed after attachment; inspect the page before retrying.' };return { success: true, dispatched: true, attachmentState: 'input_attached', file: name, size: bytes.length };@@ -327,15 +382,19 @@ export class BidiSession {// Each keydown needs its own expectation. Keep the modifier held while// renewing the fence, so Ctrl/Meta+A cannot look like a human edit.if (modifier) {- await markNativeDispatch('input');- dispatch.started = true;- await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard',- actions: [{ type: 'keyDown', value: modifier }] }] });+ const marker = await markNativeDispatch('input');+ try {+ dispatch.started = true;+ await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard',+ actions: [{ type: 'keyDown', value: modifier }] }] });+ } finally { await clearNativeDispatch(marker); }await assertFocus();}- await markNativeDispatch('input');- dispatch.started = true;- await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] });+ const marker = await markNativeDispatch('input');+ try {+ dispatch.started = true;+ await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] });+ } finally { await clearNativeDispatch(marker); }}finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }assertLive();@@ -347,9 +406,12 @@ export class BidiSession {...(action === 'click' ? [{ type: 'pointerDown', button: 0 }, { type: 'pointerUp', button: 0 }] : []),] };assertLive();- await markNativeDispatch('click');+ const marker = await markNativeDispatch('click');try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [source] }); }- finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }+ finally {+ await clearNativeDispatch(marker);+ await this.send('input.releaseActions', { context: match.context }).catch(() => {});+ }assertLive();if (checkable) {const observation = await this._waitForCheckableState(match, checkable.desiredChecked, 80);@@ -411,12 +473,15 @@ export class BidiSession {await assertFocus();// Enter is a submit shortcut on many editors, even with Shift. Insert a literal// newline through the editing command instead; never synthesize a submit key.- await markNativeDispatch('input');- dispatch.started = true;- const inserted = await this.call(match, `(el, text) => {- if (!el.isConnected || el.getRootNode().activeElement !== el) return false;- return document.execCommand('insertText', false, text);- }`, [{ type: 'string', value: char }]);+ const marker = await markNativeDispatch('input');+ let inserted;+ try {+ dispatch.started = true;+ inserted = await this.call(match, `(el, text) => {+ if (!el.isConnected || el.getRootNode().activeElement !== el) return false;+ return document.execCommand('insertText', false, text);+ }`, [{ type: 'string', value: char }]);+ } finally { await clearNativeDispatch(marker); }if (inserted.result?.value !== true) throw new Error('Editor rejected literal newline insertion');assertLive();} else await press(char);diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 0f058daea..391cae9e1 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -37807,12 +37807,17 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaioperationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,});}- targetDispatched = true;- try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}- el.dispatchEvent(new MouseEvent('mousedown', opts));- try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}- el.dispatchEvent(new MouseEvent('mouseup', opts));- el.click();+ const dispatch = () => {+ targetDispatched = true;+ try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}+ el.dispatchEvent(new MouseEvent('mousedown', opts));+ try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}+ el.dispatchEvent(new MouseEvent('mouseup', opts));+ el.click();+ };+ if (monitorGuard?.operationId && window.__wbPageMonitor?.withPreparedDispatch)+ window.__wbPageMonitor.withPreparedDispatch(monitorGuard.operationId, dispatch);+ else dispatch();return { ok: true, url: location.href, tag: el.tagName, text: (el.innerText || el.value || '').slice(0, 80), dispatched: true };} catch (e) {return { ok: false, url: location.href, dispatched: targetDispatched, error: e.message };diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 5af8580d2..ba6a5a7e0 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -2354,9 +2354,11 @@// the editor's undo stack. Its beforeinput behavior differs by browser,// so offer the page a cancellable gate BEFORE invoking the command.window.__wbPageMonitor?.beforeLocalDispatch();- const accepted = el.dispatchEvent(new InputEvent('beforeinput', {+ const beforeInputGate = () => el.dispatchEvent(new InputEvent('beforeinput', {bubbles: true, composed: true, cancelable: true, inputType, data,}));+ const accepted = window.__wbPageMonitor?.withLocalDispatch+ ? window.__wbPageMonitor.withLocalDispatch(beforeInputGate) : beforeInputGate();if (el.innerHTML !== htmlBefore || !el.isConnected) {window.__wbPageMonitor?.beforeLocalDispatch();dispatched = true;diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 18cf7e9fe..0715e26f3 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -15,19 +15,57 @@const listeners = [];const operations = new Map();const nativeTargets = new Set();+ const liveControls = new Set();let signatures = new WeakMap();+ let controlSignatures = new WeakMap();let textSignatures = new WeakMap();let popoverTurns = new WeakMap();let roots = new WeakSet();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;- let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, lastUserAt = 0;- let pointerHeld = false, composing = false, localOperation = null;+ let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;+ let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;let requestGeneration = 0, unreported = 0, pendingDOM = null;let lastViewport = '';let lastFeedbackDelivery = Promise.resolve();let agentTurn = null, userTurn = null, lastUserTarget = null;- let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();+ let matchedEvents = new WeakMap();+ const activePointers = new Set();const fenceAttribute = 'data-webbrain-page-revision';+ const nativeMarkerAttribute = 'data-webbrain-native-action';+ const nativeMarkerMask = (1n << 64n) - 1n;+ const nativeMarkerRotate = (value, bits) => ((value << BigInt(bits)) | (value >> BigInt(64 - bits))) & nativeMarkerMask;+ function nativeMarkerSignature(secret, data) {+ if (!/^[a-f\d]{32}$/i.test(secret || '')) return '';+ const key = Uint8Array.from(secret.match(/.{2}/g), byte => parseInt(byte, 16));+ const read64 = offset => {+ let value = 0n;+ for (let index = 0; index < 8; index++) value |= BigInt(key[offset + index]) << BigInt(index * 8);+ return value;+ };+ const k0 = read64(0), k1 = read64(8);+ const rotate = nativeMarkerRotate;+ let v0 = 0x736f6d6570736575n ^ k0, v1 = 0x646f72616e646f6dn ^ k1;+ let v2 = 0x6c7967656e657261n ^ k0, v3 = 0x7465646279746573n ^ k1;+ const round = () => {+ v0 = (v0 + v1) & nativeMarkerMask; v1 = rotate(v1, 13); v1 ^= v0; v0 = rotate(v0, 32);+ v2 = (v2 + v3) & nativeMarkerMask; v3 = rotate(v3, 16); v3 ^= v2;+ v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;+ v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);+ };+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,+ data.navigationCandidate === true]));+ let offset = 0;+ while (offset + 8 <= bytes.length) {+ let block = 0n;+ for (let index = 0; index < 8; index++) block |= BigInt(bytes[offset + index]) << BigInt(index * 8);+ v3 ^= block; round(); round(); v0 ^= block; offset += 8;+ }+ let tail = BigInt(bytes.length) << 56n;+ for (let index = 0; offset + index < bytes.length; index++) tail |= BigInt(bytes[offset + index]) << BigInt(index * 8);+ v3 ^= tail; round(); round(); v0 ^= tail; v2 ^= 0xffn;+ round(); round(); round(); round();+ return (v0 ^ v1 ^ v2 ^ v3).toString(16).padStart(16, '0');+ }const publishRevision = () => {const value = `${documentToken}:${revision}`;if (active && document.documentElement?.getAttribute(fenceAttribute) !== value) document.documentElement?.setAttribute(fenceAttribute, value);@@ -194,20 +232,32 @@return true;}function expected(kind, el, event) {- const marker = el?.getAttribute?.('data-webbrain-native-action');- if (marker && nativeMarks.get(el) !== marker) {- nativeMarks.set(el, marker); nativeTargets.add(el);+ const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))+ || (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);+ const marker = markerNode?.getAttribute(nativeMarkerAttribute);+ if (marker && event?.isTrusted === true) {try {const data = JSON.parse(marker);- if (data.documentToken === documentToken && operations.has(data.operationId) && ['input', 'click', 'scroll'].includes(data.kind)) {+ const op = operations.get(data.operationId);+ const markerTarget = op && (markerNode === op.target || markerNode === op.focusTarget);+ const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;+ const allowedEvent = data.kind === 'input'+ ? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)+ : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;+ const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;+ if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence+ && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {+ // The capability is bound to the isolated-world prepared node and consumed once.+ op.nativeSequence = data.sequence;+ markerNode.removeAttribute(nativeMarkerAttribute);dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });- operations.get(data.operationId).target = el;}} catch { /* Page markers are hints; malformed ones grant no expectation. */ }}if (event && matchedEvents.has(event)) return matchedEvents.get(event);prune();const match = op => {+ if (event && !event.isTrusted && !op.synchronous) return null;if (event) {if (op.eventTypes && !op.eventTypes.has(event.type)) return null;// One native dispatch owns one occurrence of each input phase. A later@@ -258,13 +308,49 @@while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;return el;}+ function controlValueFingerprint(el) {+ let controlValue = null;+ if ((el.tagName === 'TEXTAREA' || (el.tagName === 'INPUT'+ && !['password', 'file', 'hidden', 'checkbox', 'radio', 'button', 'submit', 'reset', 'image'].includes(el.type)))) {+ const value = el.value || '', samples = Math.min(value.length, 8192);+ let first = 0x811c9dc5, second = 0x9e3779b9;+ // Hash locally only; feedback contains a generic target and never the value or typed keys.+ for (let index = 0; index < samples; index++) {+ const code = value.charCodeAt(samples === value.length ? index : Math.floor(index * value.length / samples));+ first = Math.imul(first ^ code, 16777619) >>> 0;+ second = Math.imul(second ^ (code + index), 2246822519) >>> 0;+ }+ controlValue = [value.length, first, second];+ }+ return controlValue;+ }+ function controlState(el) {+ return JSON.stringify([el.getAttribute('value'), el.selected === true, el.defaultSelected === true,+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null, el.checked === true,+ el.disabled === true, el.readOnly === true, el.required === true, el.validity?.valid !== false,+ controlValueFingerprint(el)]);+ }+ function trackControl(el) {+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;+ liveControls.add(el);+ controlSignatures.set(el, controlState(el));+ }+ function refreshControl(el) {+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) {+ trackControl(el);+ controlSignatures.set(el, controlState(el));+ }+ signatures.set(el, signature(el));+ }function signature(el) {if (ignored(el)) return '';+ trackControl(el);const shown = visible(el);const rect = shown ? el.getBoundingClientRect() : {};const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3).map(textSignature).reduce(([size, hash], [length, part, power]) =>[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);+ const controlValue = controlValueFingerprint(el);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-pressed'),@@ -272,17 +358,33 @@el.selected === true, el.defaultSelected === true,Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,- el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),+ el.getAttribute('aria-disabled'), el.getAttribute('value'), controlValue, el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);}function seed(root, budget = { remaining: 600 }) {const nodes = root.querySelectorAll?.('*') || [];for (const el of nodes) {+ // Form-state tracking has its own cap, separate from bounded layout reads.+ trackControl(el);if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }// Root discovery is cheap and must not share the layout-measurement cap.if (el.shadowRoot && !ignored(el)) observeRoot(el.shadowRoot, budget);}}+ function sampleFormControls() {+ if (!active || !liveControls.size) return;+ const controls = [...liveControls];+ if (controlCursor >= controls.length) controlCursor = 0;+ const batchSize = Math.min(200, controls.length);+ const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);+ controlCursor = (controlCursor + batchSize) % controls.length;+ const records = [];+ for (const el of batch) {+ if (!el.isConnected) { liveControls.delete(el); continue; }+ if (!ignored(el)) records.push({ type: 'control', target: el });+ }+ if (records.length) onMutations(records);+ }function sampleDescendants(root, limit = 100) {const nodes = [];const stack = [];@@ -417,6 +519,13 @@target ||= targetName(el);break;}+ if (record.type === 'control') {+ const nextControl = controlState(el), previousControl = controlSignatures.get(el);+ controlSignatures.set(el, nextControl);+ signatures.set(el, signature(el));+ if (nextControl !== previousControl) noteChange(el, record.agentUserAt);+ continue;+ }let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));@@ -434,6 +543,7 @@}const next = signature(el), previous = signatures.get(el);signatures.set(el, next);+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) controlSignatures.set(el, controlState(el));if (!identityChanged && (next === previous || (!visible(el) && !previous?.startsWith('[true')))) continue;noteChange(el, record.agentUserAt);}@@ -479,6 +589,7 @@active = true; runToken = state.runToken; seq = 0; revision = 0; publishRevision();observer = new MutationObserver(onMutations);observeRoot(document);+ controlTimer = setInterval(sampleFormControls, 250);observeLayoutShifts();listen(document, 'webbrain-shadow-root-attached', event => {const path = event.composedPath();@@ -531,17 +642,21 @@listen(document, 'pointerdown', event => {const el = elementFor(event);if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;+ activePointers.add(event.pointerId);pointerHeld = true; interact('activity', el);});listen(document, 'pointermove', event => {- if (pointerHeld && event.isTrusted) {- if (!event.buttons) pointerHeld = false;+ if (activePointers.has(event.pointerId) && event.isTrusted) {+ if (!event.buttons) activePointers.delete(event.pointerId);+ pointerHeld = activePointers.size > 0;interact('activity', elementFor(event));}});for (const name of ['pointerup', 'pointercancel']) listen(document, name, event => {- if (!pointerHeld || !event.isTrusted) return;- pointerHeld = false; interact('activity', elementFor(event), { interacting: composing, completeGesture: true });+ if (!activePointers.has(event.pointerId) || !event.isTrusted) return;+ activePointers.delete(event.pointerId);+ pointerHeld = activePointers.size > 0;+ interact('activity', elementFor(event), { completeGesture: activePointers.size === 0 });});listen(document, 'click', event => {const el = elementFor(event);@@ -559,10 +674,14 @@});for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {const el = elementFor(event);- if (ignored(el) || expected('input', el, event)- || (el.matches?.('input[type="checkbox"],input[type="radio"],select,option') && expected('click', el, event))) return;+ const expectedInput = expected('input', el, event);+ const expectedClick = !expectedInput && el.matches?.('input[type="checkbox"],input[type="radio"],select,option')+ ? expected('click', el, event) : null;+ if (ignored(el)) return;+ if (expectedInput || expectedClick) { refreshControl(el); return; }if (event.isTrusted) interact('input', el);else { revision++; send({ kind: 'input', source: 'page', target: targetName(el) }); }+ refreshControl(el);});listen(document, 'compositionstart', event => {const el = elementFor(event);@@ -619,13 +738,13 @@if (Date.now() - lastUserAt < 1000 && !editable(document.activeElement)) interact('selection', document.activeElement || document.body);});listen(window, 'blur', () => {- if (pointerHeld || composing) { pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }+ if (pointerHeld || composing) { activePointers.clear(); pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }});listen(window, 'pagehide', () => stop());}function stop() {- active = false; runToken = ''; pointerHeld = false; composing = false;- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();+ active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {el.removeAttribute?.('data-webbrain-native-action');@@ -634,10 +753,10 @@nativeTargets.clear();observer?.disconnect(); observer = null;layoutObserver?.disconnect(); layoutObserver = null;- clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();+ clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;- roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap();+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();}async function requestState() {const generation = ++requestGeneration;@@ -665,6 +784,7 @@function dispatch(params) {if (!active || (params.runToken && params.runToken !== runToken)) return;const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };+ if (params.fenceOnly && !op.nativeSecret) op.nativeSecret = Array.from(crypto.getRandomValues(new Uint8Array(16)), byte => byte.toString(16).padStart(2, '0')).join('');op.blurTarget ??= document.activeElement;op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;@@ -690,6 +810,7 @@operations.set(op.operationId, op);}function activatePreparedDispatch(params = {}) {+ sampleFormControls();const op = operations.get(params.operationId);const layoutChanged = coordinatePreparationShifted(op);if (!active || !op || layoutChanged || domTimer || unreported@@ -704,6 +825,14 @@publishRevision();return { ready: true, operationId: op.operationId, revision };}+ function withPreparedDispatch(operationId, callback) {+ const op = operations.get(operationId);+ if (!op?.dispatched || typeof callback !== 'function') return callback?.();+ const previous = op.synchronous;+ op.synchronous = true;+ try { return callback(); }+ finally { op.synchronous = previous; }+ }const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll','hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js','ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);@@ -726,6 +855,7 @@}function beforeLocalDispatch({ preparation = false, kind, target } = {}) {if (!active || !localOperation) return;+ sampleFormControls();const layoutChanged = coordinatePreparationShifted(operations.get(localOperation.operationId));if (layoutChanged || domTimer || unreported || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {const error = new Error('Browser changed during action preparation. Re-observe before acting.');@@ -762,6 +892,7 @@} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }else if (msg.action === 'page_monitor_dispatch') {const params = msg.params || {};+ if (!params.release) sampleFormControls();const prepared = operations.get(params.operationId);const layoutChanged = !params.release && active && coordinatePreparationShifted(prepared);if (!params.release && active && (layoutChanged || domTimer || unreported || (prepared && prepared.preparedRevision !== revision)@@ -775,7 +906,9 @@}dispatch(params); publishRevision();respond({ ready: true, ...(active && params.runToken === runToken ? { guard: {- documentToken, revision, operationId: params.operationId, navigationCandidate: params.navigationCandidate !== false,+ documentToken, revision, operationId: params.operationId, runToken,+ navigationCandidate: params.navigationCandidate !== false,+ ...(operations.get(params.operationId)?.nativeSecret ? { nativeSecret: operations.get(params.operationId).nativeSecret } : {}),} } : {}) });}else if (msg.action === 'page_monitor_finish') {@@ -785,7 +918,7 @@}};api.runtime.onMessage.addListener(onMessage);- window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, registerDecoration,+ window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, withPreparedDispatch, registerDecoration,get active() { return active; },get disposed() { return disposed; },dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 84299ba9b..3b69c621a 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -34859,12 +34859,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout doperationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,});}- targetDispatched = true;- try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}- el.dispatchEvent(new MouseEvent('mousedown', opts));- try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}- el.dispatchEvent(new MouseEvent('mouseup', opts));- el.click();+ const dispatch = () => {+ targetDispatched = true;+ try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}+ el.dispatchEvent(new MouseEvent('mousedown', opts));+ try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}+ el.dispatchEvent(new MouseEvent('mouseup', opts));+ el.click();+ };+ if (monitorGuard?.operationId && window.__wbPageMonitor?.withPreparedDispatch)+ window.__wbPageMonitor.withPreparedDispatch(monitorGuard.operationId, dispatch);+ else dispatch();return { ok: true, url: location.href, tag: el.tagName, text: (el.innerText || el.value || '').slice(0, 80), dispatched: true };} catch (e) { return { ok: false, url: location.href, dispatched: targetDispatched, error: e.message }; }})()diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex b233b728f..8aad1e061 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -2670,9 +2670,11 @@// the editor's undo stack. Its beforeinput behavior differs by browser,// so offer the page a cancellable gate BEFORE invoking the command.window.__wbPageMonitor?.beforeLocalDispatch();- const accepted = el.dispatchEvent(new InputEvent('beforeinput', {+ const beforeInputGate = () => el.dispatchEvent(new InputEvent('beforeinput', {bubbles: true, composed: true, cancelable: true, inputType, data,}));+ const accepted = window.__wbPageMonitor?.withLocalDispatch+ ? window.__wbPageMonitor.withLocalDispatch(beforeInputGate) : beforeInputGate();if (el.innerHTML !== htmlBefore || !el.isConnected) {window.__wbPageMonitor?.beforeLocalDispatch();dispatched = true;diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 18cf7e9fe..0715e26f3 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -15,19 +15,57 @@const listeners = [];const operations = new Map();const nativeTargets = new Set();+ const liveControls = new Set();let signatures = new WeakMap();+ let controlSignatures = new WeakMap();let textSignatures = new WeakMap();let popoverTurns = new WeakMap();let roots = new WeakSet();let active = false, disposed = false, runToken = '', seq = 0, revision = 0;- let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, lastUserAt = 0;- let pointerHeld = false, composing = false, localOperation = null;+ let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;+ let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;let requestGeneration = 0, unreported = 0, pendingDOM = null;let lastViewport = '';let lastFeedbackDelivery = Promise.resolve();let agentTurn = null, userTurn = null, lastUserTarget = null;- let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();+ let matchedEvents = new WeakMap();+ const activePointers = new Set();const fenceAttribute = 'data-webbrain-page-revision';+ const nativeMarkerAttribute = 'data-webbrain-native-action';+ const nativeMarkerMask = (1n << 64n) - 1n;+ const nativeMarkerRotate = (value, bits) => ((value << BigInt(bits)) | (value >> BigInt(64 - bits))) & nativeMarkerMask;+ function nativeMarkerSignature(secret, data) {+ if (!/^[a-f\d]{32}$/i.test(secret || '')) return '';+ const key = Uint8Array.from(secret.match(/.{2}/g), byte => parseInt(byte, 16));+ const read64 = offset => {+ let value = 0n;+ for (let index = 0; index < 8; index++) value |= BigInt(key[offset + index]) << BigInt(index * 8);+ return value;+ };+ const k0 = read64(0), k1 = read64(8);+ const rotate = nativeMarkerRotate;+ let v0 = 0x736f6d6570736575n ^ k0, v1 = 0x646f72616e646f6dn ^ k1;+ let v2 = 0x6c7967656e657261n ^ k0, v3 = 0x7465646279746573n ^ k1;+ const round = () => {+ v0 = (v0 + v1) & nativeMarkerMask; v1 = rotate(v1, 13); v1 ^= v0; v0 = rotate(v0, 32);+ v2 = (v2 + v3) & nativeMarkerMask; v3 = rotate(v3, 16); v3 ^= v2;+ v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;+ v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);+ };+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,+ data.navigationCandidate === true]));+ let offset = 0;+ while (offset + 8 <= bytes.length) {+ let block = 0n;+ for (let index = 0; index < 8; index++) block |= BigInt(bytes[offset + index]) << BigInt(index * 8);+ v3 ^= block; round(); round(); v0 ^= block; offset += 8;+ }+ let tail = BigInt(bytes.length) << 56n;+ for (let index = 0; offset + index < bytes.length; index++) tail |= BigInt(bytes[offset + index]) << BigInt(index * 8);+ v3 ^= tail; round(); round(); v0 ^= tail; v2 ^= 0xffn;+ round(); round(); round(); round();+ return (v0 ^ v1 ^ v2 ^ v3).toString(16).padStart(16, '0');+ }const publishRevision = () => {const value = `${documentToken}:${revision}`;if (active && document.documentElement?.getAttribute(fenceAttribute) !== value) document.documentElement?.setAttribute(fenceAttribute, value);@@ -194,20 +232,32 @@return true;}function expected(kind, el, event) {- const marker = el?.getAttribute?.('data-webbrain-native-action');- if (marker && nativeMarks.get(el) !== marker) {- nativeMarks.set(el, marker); nativeTargets.add(el);+ const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))+ || (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);+ const marker = markerNode?.getAttribute(nativeMarkerAttribute);+ if (marker && event?.isTrusted === true) {try {const data = JSON.parse(marker);- if (data.documentToken === documentToken && operations.has(data.operationId) && ['input', 'click', 'scroll'].includes(data.kind)) {+ const op = operations.get(data.operationId);+ const markerTarget = op && (markerNode === op.target || markerNode === op.focusTarget);+ const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;+ const allowedEvent = data.kind === 'input'+ ? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)+ : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;+ const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;+ if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence+ && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {+ // The capability is bound to the isolated-world prepared node and consumed once.+ op.nativeSequence = data.sequence;+ markerNode.removeAttribute(nativeMarkerAttribute);dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });- operations.get(data.operationId).target = el;}} catch { /* Page markers are hints; malformed ones grant no expectation. */ }}if (event && matchedEvents.has(event)) return matchedEvents.get(event);prune();const match = op => {+ if (event && !event.isTrusted && !op.synchronous) return null;if (event) {if (op.eventTypes && !op.eventTypes.has(event.type)) return null;// One native dispatch owns one occurrence of each input phase. A later@@ -258,13 +308,49 @@while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;return el;}+ function controlValueFingerprint(el) {+ let controlValue = null;+ if ((el.tagName === 'TEXTAREA' || (el.tagName === 'INPUT'+ && !['password', 'file', 'hidden', 'checkbox', 'radio', 'button', 'submit', 'reset', 'image'].includes(el.type)))) {+ const value = el.value || '', samples = Math.min(value.length, 8192);+ let first = 0x811c9dc5, second = 0x9e3779b9;+ // Hash locally only; feedback contains a generic target and never the value or typed keys.+ for (let index = 0; index < samples; index++) {+ const code = value.charCodeAt(samples === value.length ? index : Math.floor(index * value.length / samples));+ first = Math.imul(first ^ code, 16777619) >>> 0;+ second = Math.imul(second ^ (code + index), 2246822519) >>> 0;+ }+ controlValue = [value.length, first, second];+ }+ return controlValue;+ }+ function controlState(el) {+ return JSON.stringify([el.getAttribute('value'), el.selected === true, el.defaultSelected === true,+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null, el.checked === true,+ el.disabled === true, el.readOnly === true, el.required === true, el.validity?.valid !== false,+ controlValueFingerprint(el)]);+ }+ function trackControl(el) {+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;+ liveControls.add(el);+ controlSignatures.set(el, controlState(el));+ }+ function refreshControl(el) {+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) {+ trackControl(el);+ controlSignatures.set(el, controlState(el));+ }+ signatures.set(el, signature(el));+ }function signature(el) {if (ignored(el)) return '';+ trackControl(el);const shown = visible(el);const rect = shown ? el.getBoundingClientRect() : {};const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3).map(textSignature).reduce(([size, hash], [length, part, power]) =>[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);+ const controlValue = controlValueFingerprint(el);return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),el.getAttribute('aria-pressed'),@@ -272,17 +358,33 @@el.selected === true, el.defaultSelected === true,Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,- el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),+ el.getAttribute('aria-disabled'), el.getAttribute('value'), controlValue, el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);}function seed(root, budget = { remaining: 600 }) {const nodes = root.querySelectorAll?.('*') || [];for (const el of nodes) {+ // Form-state tracking has its own cap, separate from bounded layout reads.+ trackControl(el);if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }// Root discovery is cheap and must not share the layout-measurement cap.if (el.shadowRoot && !ignored(el)) observeRoot(el.shadowRoot, budget);}}+ function sampleFormControls() {+ if (!active || !liveControls.size) return;+ const controls = [...liveControls];+ if (controlCursor >= controls.length) controlCursor = 0;+ const batchSize = Math.min(200, controls.length);+ const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);+ controlCursor = (controlCursor + batchSize) % controls.length;+ const records = [];+ for (const el of batch) {+ if (!el.isConnected) { liveControls.delete(el); continue; }+ if (!ignored(el)) records.push({ type: 'control', target: el });+ }+ if (records.length) onMutations(records);+ }function sampleDescendants(root, limit = 100) {const nodes = [];const stack = [];@@ -417,6 +519,13 @@target ||= targetName(el);break;}+ if (record.type === 'control') {+ const nextControl = controlState(el), previousControl = controlSignatures.get(el);+ controlSignatures.set(el, nextControl);+ signatures.set(el, signature(el));+ if (nextControl !== previousControl) noteChange(el, record.agentUserAt);+ continue;+ }let identityChanged = (record.type === 'popover' && record.stateChanged)|| (record.type === 'layout' && record.layoutChanged)|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));@@ -434,6 +543,7 @@}const next = signature(el), previous = signatures.get(el);signatures.set(el, next);+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) controlSignatures.set(el, controlState(el));if (!identityChanged && (next === previous || (!visible(el) && !previous?.startsWith('[true')))) continue;noteChange(el, record.agentUserAt);}@@ -479,6 +589,7 @@active = true; runToken = state.runToken; seq = 0; revision = 0; publishRevision();observer = new MutationObserver(onMutations);observeRoot(document);+ controlTimer = setInterval(sampleFormControls, 250);observeLayoutShifts();listen(document, 'webbrain-shadow-root-attached', event => {const path = event.composedPath();@@ -531,17 +642,21 @@listen(document, 'pointerdown', event => {const el = elementFor(event);if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;+ activePointers.add(event.pointerId);pointerHeld = true; interact('activity', el);});listen(document, 'pointermove', event => {- if (pointerHeld && event.isTrusted) {- if (!event.buttons) pointerHeld = false;+ if (activePointers.has(event.pointerId) && event.isTrusted) {+ if (!event.buttons) activePointers.delete(event.pointerId);+ pointerHeld = activePointers.size > 0;interact('activity', elementFor(event));}});for (const name of ['pointerup', 'pointercancel']) listen(document, name, event => {- if (!pointerHeld || !event.isTrusted) return;- pointerHeld = false; interact('activity', elementFor(event), { interacting: composing, completeGesture: true });+ if (!activePointers.has(event.pointerId) || !event.isTrusted) return;+ activePointers.delete(event.pointerId);+ pointerHeld = activePointers.size > 0;+ interact('activity', elementFor(event), { completeGesture: activePointers.size === 0 });});listen(document, 'click', event => {const el = elementFor(event);@@ -559,10 +674,14 @@});for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {const el = elementFor(event);- if (ignored(el) || expected('input', el, event)- || (el.matches?.('input[type="checkbox"],input[type="radio"],select,option') && expected('click', el, event))) return;+ const expectedInput = expected('input', el, event);+ const expectedClick = !expectedInput && el.matches?.('input[type="checkbox"],input[type="radio"],select,option')+ ? expected('click', el, event) : null;+ if (ignored(el)) return;+ if (expectedInput || expectedClick) { refreshControl(el); return; }if (event.isTrusted) interact('input', el);else { revision++; send({ kind: 'input', source: 'page', target: targetName(el) }); }+ refreshControl(el);});listen(document, 'compositionstart', event => {const el = elementFor(event);@@ -619,13 +738,13 @@if (Date.now() - lastUserAt < 1000 && !editable(document.activeElement)) interact('selection', document.activeElement || document.body);});listen(window, 'blur', () => {- if (pointerHeld || composing) { pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }+ if (pointerHeld || composing) { activePointers.clear(); pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }});listen(window, 'pagehide', () => stop());}function stop() {- active = false; runToken = ''; pointerHeld = false; composing = false;- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();+ active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {el.removeAttribute?.('data-webbrain-native-action');@@ -634,10 +753,10 @@nativeTargets.clear();observer?.disconnect(); observer = null;layoutObserver?.disconnect(); layoutObserver = null;- clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();+ clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();listeners.splice(0).forEach(remove => remove());operations.clear(); localOperation = null; agentTurn = null;- roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap();+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();}async function requestState() {const generation = ++requestGeneration;@@ -665,6 +784,7 @@function dispatch(params) {if (!active || (params.runToken && params.runToken !== runToken)) return;const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };+ if (params.fenceOnly && !op.nativeSecret) op.nativeSecret = Array.from(crypto.getRandomValues(new Uint8Array(16)), byte => byte.toString(16).padStart(2, '0')).join('');op.blurTarget ??= document.activeElement;op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;@@ -690,6 +810,7 @@operations.set(op.operationId, op);}function activatePreparedDispatch(params = {}) {+ sampleFormControls();const op = operations.get(params.operationId);const layoutChanged = coordinatePreparationShifted(op);if (!active || !op || layoutChanged || domTimer || unreported@@ -704,6 +825,14 @@publishRevision();return { ready: true, operationId: op.operationId, revision };}+ function withPreparedDispatch(operationId, callback) {+ const op = operations.get(operationId);+ if (!op?.dispatched || typeof callback !== 'function') return callback?.();+ const previous = op.synchronous;+ op.synchronous = true;+ try { return callback(); }+ finally { op.synchronous = previous; }+ }const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll','hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js','ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);@@ -726,6 +855,7 @@}function beforeLocalDispatch({ preparation = false, kind, target } = {}) {if (!active || !localOperation) return;+ sampleFormControls();const layoutChanged = coordinatePreparationShifted(operations.get(localOperation.operationId));if (layoutChanged || domTimer || unreported || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {const error = new Error('Browser changed during action preparation. Re-observe before acting.');@@ -762,6 +892,7 @@} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }else if (msg.action === 'page_monitor_dispatch') {const params = msg.params || {};+ if (!params.release) sampleFormControls();const prepared = operations.get(params.operationId);const layoutChanged = !params.release && active && coordinatePreparationShifted(prepared);if (!params.release && active && (layoutChanged || domTimer || unreported || (prepared && prepared.preparedRevision !== revision)@@ -775,7 +906,9 @@}dispatch(params); publishRevision();respond({ ready: true, ...(active && params.runToken === runToken ? { guard: {- documentToken, revision, operationId: params.operationId, navigationCandidate: params.navigationCandidate !== false,+ documentToken, revision, operationId: params.operationId, runToken,+ navigationCandidate: params.navigationCandidate !== false,+ ...(operations.get(params.operationId)?.nativeSecret ? { nativeSecret: operations.get(params.operationId).nativeSecret } : {}),} } : {}) });}else if (msg.action === 'page_monitor_finish') {@@ -785,7 +918,7 @@}};api.runtime.onMessage.addListener(onMessage);- window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, registerDecoration,+ window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, withPreparedDispatch, registerDecoration,get active() { return active; },get disposed() { return disposed; },dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 3d02cd870..09448a66b 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -4,7 +4,7 @@ import { test } from 'node:test';import { chromium, firefox } from 'playwright';import { CDPClient } from '../src/chrome/src/cdp/cdp-client.js';import { pageFeedbackMethods } from '../src/chrome/src/agent/page-feedback.js';-import { BidiSession } from '../firefox-companion/session.mjs';+import { BidiSession, createNativeActionMarker } from '../firefox-companion/session.mjs';const read = (build, file) => fs.readFileSync(new URL(`../src/${build}/src/${file}`, import.meta.url), 'utf8');const html = `<!doctype html><style>@@ -17,11 +17,21 @@ body { margin: 0; } button,input { margin: 12px; }<div id="moving">Animated</div><div style="height:3000px"></div><script>document.getElementById('agent').addEventListener('click', () => { document.getElementById('status').textContent = 'Agent changed this'; });</script>`;-async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false, omitEmptyFrameMonitor = false } = {}) {+async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false, omitEmptyFrameMonitor = false,+ capturePointerHandlers = false } = {}) {const browser = await engine.launch({ headless: true, ...(siteIsolation ? { args: ['--site-per-process'] } : {}) });const context = await browser.newContext();await context.route('https://monitor.test/**', route => route.fulfill({ contentType: 'text/html', body: html }));await context.addInitScript(token => { window.monitorRunToken = token; }, runToken);+ if (capturePointerHandlers) await context.addInitScript(() => {+ window.monitorPointerHandlers = {};+ const add = EventTarget.prototype.addEventListener;+ EventTarget.prototype.addEventListener = function(type, handler, options) {+ if (this === document && ['pointerdown', 'pointermove', 'pointerup', 'pointercancel'].includes(type)+ && options?.capture && typeof handler === 'function') (monitorPointerHandlers[type] ||= []).push(handler);+ return add.call(this, type, handler, options);+ };+ });await context.addInitScript(() => {window.monitorEnabled = true;window.feedback = [];@@ -505,7 +515,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {try {__wbPageMonitor.activatePreparedDispatch({ operationId: 'iframe-click-live', kind: 'click',element: document.getElementById('agent'), navigationCandidate: true });- document.getElementById('agent').click();+ __wbPageMonitor.withPreparedDispatch('iframe-click-live', () => document.getElementById('agent').click());return true;} catch { return false; }});@@ -870,8 +880,10 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {await page.evaluate(() => {feedback = []; deliver('page_monitor_prepare', { operationId: 'agent-tab', tool: 'press_keys', selector: '#human' });deliver('page_monitor_dispatch', { operationId: 'agent-tab', kind: 'input', selector: '#human', fenceOnly: true });- document.getElementById('human').setAttribute('data-webbrain-native-action', JSON.stringify({ ...lastMonitorResponse.guard, kind: 'input', sequence: 1 }));});+ const tabGuard = await page.evaluate(() => lastMonitorResponse.guard);+ await page.evaluate(marker => document.getElementById('human').setAttribute('data-webbrain-native-action', marker),+ createNativeActionMarker(tabGuard, 'input', 1));await page.keyboard.press('Tab');await page.waitForTimeout(80);assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,@@ -889,6 +901,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {assert.ok(input.some(event => event.kind === 'input' && event.source === 'user'));assert.equal(JSON.stringify(input).includes('private-field-value'), false);assert.equal(JSON.stringify(input).includes('password-never-recorded'), false);+ await page.waitForTimeout(1100);await page.evaluate(() => {feedback = [];@@ -908,6 +921,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {&& event.target === 'button#agent'), 'A later human click on the same target must survive attribution');await page.locator('#human').click();assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user' && event.target === 'button#human'));+ await page.waitForTimeout(1100);await page.evaluate(() => {feedback = [];@@ -920,6 +934,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {assert.equal(ownTyping.some(event => event.source !== 'agent'), false, JSON.stringify(ownTyping));await page.locator('#human').click();assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'));+ await page.waitForTimeout(1100);await page.evaluate(() => {feedback = [];deliver('page_monitor_prepare', { operationId: 'click-with-external-dom', tool: 'click', selector: '#agent' });@@ -932,6 +947,96 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: property-only form changes invalidate the prepared page without exposing values`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const dense = document.createElement('div');+ dense.innerHTML = `${'<span></span>'.repeat(650)}<input id="late-control">`;+ document.body.append(dense);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });+ await page.evaluate(() => { feedback = []; document.getElementById('late-control').value = 'late-private-value'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1500 });+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('late-private-value'), false,+ 'Controls beyond the layout-sampling budget still receive bounded state monitoring');++ await page.evaluate(() => {+ feedback = [];+ document.getElementById('field').value = 'private-property-only-value';+ document.getElementById('select').selectedIndex = 1;+ const check = document.createElement('input'); check.type = 'checkbox'; check.id = 'property-check';+ document.body.append(check); check.checked = true;+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1500 });+ const observed = await page.evaluate(() => feedback);+ assert.equal(JSON.stringify(observed).includes('private-property-only-value'), false);+ assert.ok(observed.some(event => event.kind === 'dom' && event.source === 'page'));++ const result = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'property-preflight', tool: 'click', selector: '#agent' });+ document.getElementById('field').value = 'changed-during-preparation';+ try { __wbPageMonitor.activatePreparedDispatch({ operationId: 'property-preflight', kind: 'click' }); return null; }+ catch (error) { return error.code; }+ });+ assert.equal(result, 'page_feedback_pending', 'The final dispatch check must sample property-only changes');+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('changed-during-preparation'), false);+ } finally { await browser.close(); }+ });++ test(`${build}: native markers reject replay, tampering and copied targets`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ const guard = await page.evaluate(() => {+ deliver('page_monitor_prepare', { operationId: 'native-target', tool: 'type_text', selector: '#field' });+ deliver('page_monitor_dispatch', { operationId: 'native-target', kind: 'input', selector: '#field', fenceOnly: true });+ feedback = []; return lastMonitorResponse.guard;+ });+ const copiedMarker = createNativeActionMarker(guard, 'input', 1);+ await page.evaluate(marker => document.getElementById('secret').setAttribute('data-webbrain-native-action', marker), copiedMarker);+ await page.locator('#secret').fill('target-must-stay-bound');+ let events = await page.evaluate(() => feedback);+ assert.ok(events.some(event => event.kind === 'input' && event.source === 'user' && event.target === 'input#secret'),+ 'A valid capability copied to another control cannot retarget the isolated operation');+ assert.equal(JSON.stringify(events).includes('target-must-stay-bound'), false);++ const marker = JSON.parse(createNativeActionMarker(guard, 'input', 1));+ marker.sequence = 999;+ await page.evaluate(value => {+ feedback = [];+ const field = document.getElementById('field');+ field.setAttribute('data-webbrain-native-action', JSON.stringify(value));+ field.dispatchEvent(new Event('input', { bubbles: true }));+ }, marker);+ events = await page.evaluate(() => feedback);+ assert.ok(events.some(event => event.kind === 'input' && event.source === 'page'),+ 'A modified sequence and script-generated event cannot be attributed to the agent');+ assert.equal(events.some(event => event.source === 'agent'), false);+ } finally { await browser.close(); }+ });++ test(`${build}: a multi-pointer gesture stays active until the last pointer is released`, async () => {+ const { browser, page } = await fixture(engine, build, { capturePointerHandlers: true });+ try {+ const states = await page.evaluate(() => {+ feedback = [];+ const target = document.getElementById('human');+ const emit = (type, pointerId, buttons) => {+ const event = { type, pointerId, buttons, isTrusted: true, target, composedPath: () => [target] };+ for (const listener of monitorPointerHandlers[type] || []) listener.call(document, event);+ };+ emit('pointerdown', 11, 1);+ emit('pointerdown', 12, 1);+ emit('pointerup', 11, 0);+ const afterFirstRelease = feedback.at(-1)?.interacting;+ emit('pointerup', 12, 0);+ return { afterFirstRelease, afterLastRelease: feedback.at(-1)?.interacting };+ });+ assert.deepEqual(states, { afterFirstRelease: true, afterLastRelease: false });+ } finally { await browser.close(); }+ });+test(`${build}: DOM, window/container scrolling, cleanup and restored monitoring work`, async () => {const { browser, page } = await fixture(engine, build);try {@@ -1305,12 +1410,15 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {feedback = []; return lastMonitorResponse.guard;});await page.locator('#field').focus();+ let replayedMarker;for (let i = 0; i < 3; i++) {- await page.evaluate(({ guard, i }) => document.getElementById('field').setAttribute('data-webbrain-native-action',- JSON.stringify({ ...guard, kind: 'input', sequence: i })), { guard, i });+ replayedMarker = createNativeActionMarker(guard, 'input', i + 1);+ await page.evaluate(marker => document.getElementById('field').setAttribute('data-webbrain-native-action', marker), replayedMarker);await page.keyboard.insertText('a');}- assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,+ JSON.stringify(await page.evaluate(() => feedback)));+ await page.evaluate(marker => document.getElementById('field').setAttribute('data-webbrain-native-action', marker), replayedMarker);await page.keyboard.insertText('human-private');assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'));assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('human-private'), false);@@ -1621,8 +1729,8 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {deliver('page_monitor_dispatch', { operationId: 'clear-chord', kind: 'input', selector: '#field', fenceOnly: true });feedback = []; return lastMonitorResponse.guard;});- const mark = sequence => page.evaluate(({ guard, sequence }) => document.getElementById('field')- .setAttribute('data-webbrain-native-action', JSON.stringify({ ...guard, kind: 'input', sequence })), { guard, sequence });+ const mark = sequence => page.evaluate(marker => document.getElementById('field')+ .setAttribute('data-webbrain-native-action', marker), createNativeActionMarker(guard, 'input', sequence));await mark(1); await page.keyboard.down('Control');await mark(2); await page.keyboard.press('a'); await page.keyboard.up('Control');await mark(3); await page.keyboard.press('Delete');