Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix page feedback review findings

Barack Sokullu89f40cc55195
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index 3fcac5078..c7bd3f2a0 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -2,6 +2,47 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, basename } from 'node:path';
+const sipMask = (1n << 64n) - 1n;
+const sipRotate = (value, bits) => ((value << BigInt(bits)) | (value >> BigInt(64 - bits))) & sipMask;
+function pageActionSignature(secret, data) {
+ if (!/^[a-f\d]{32}$/i.test(secret || '')) return '';
+ const key = Buffer.from(secret, 'hex');
+ const read64 = offset => {
+ let value = 0n;
+ for (let index = 0; index < 8; index++) value |= BigInt(key[offset + index]) << BigInt(index * 8);
+ return value;
+ };
+ const k0 = read64(0), k1 = read64(8);
+ let v0 = 0x736f6d6570736575n ^ k0, v1 = 0x646f72616e646f6dn ^ k1;
+ let v2 = 0x6c7967656e657261n ^ k0, v3 = 0x7465646279746573n ^ k1;
+ const round = () => {
+ v0 = (v0 + v1) & sipMask; v1 = sipRotate(v1, 13); v1 ^= v0; v0 = sipRotate(v0, 32);
+ v2 = (v2 + v3) & sipMask; v3 = sipRotate(v3, 16); v3 ^= v2;
+ v0 = (v0 + v3) & sipMask; v3 = sipRotate(v3, 21); v3 ^= v0;
+ v2 = (v2 + v1) & sipMask; v1 = sipRotate(v1, 17); v1 ^= v2; v2 = sipRotate(v2, 32);
+ };
+ const bytes = Buffer.from(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,
+ data.navigationCandidate === true]));
+ let offset = 0;
+ while (offset + 8 <= bytes.length) {
+ let block = 0n;
+ for (let index = 0; index < 8; index++) block |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ v3 ^= block; round(); round(); v0 ^= block; offset += 8;
+ }
+ let tail = BigInt(bytes.length) << 56n;
+ for (let index = 0; offset + index < bytes.length; index++) tail |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ v3 ^= tail; round(); round(); v0 ^= tail; v2 ^= 0xffn;
+ round(); round(); round(); round();
+ return (v0 ^ v1 ^ v2 ^ v3).toString(16).padStart(16, '0');
+}
+
+export function createNativeActionMarker(guard, kind, sequence) {
+ const { nativeSecret, ...data } = guard || {};
+ const marker = { ...data, kind, sequence };
+ marker.signature = pageActionSignature(nativeSecret, marker);
+ return JSON.stringify(marker);
+}
+
function unpackRemoteObjectValue(value) {
if (value?.type !== 'object' || !Array.isArray(value.value)) return value;
return Object.fromEntries(value.value.flatMap(entry => {
@@ -237,7 +278,7 @@ export class BidiSession {
assertLive();
const guard = payload.pageFeedbackGuard;
if (!guard) return; // Compatibility with clients without page monitoring.
- const marker = JSON.stringify({ ...guard, kind, sequence: ++nativeSequence });
+ const marker = createNativeActionMarker(guard, kind, ++nativeSequence);
const checked = await this.call(match, `(el, fence, marker, kind, action) => {
if (!el.isConnected || document.documentElement.getAttribute('data-webbrain-page-revision') !== fence) return false;
const target = kind === 'input' && action !== 'upload' ? el.getRootNode().activeElement || el : el;
@@ -250,6 +291,18 @@ export class BidiSession {
error.code = 'page_feedback_pending'; throw error;
}
assertLive();
+ return marker;
+ };
+ const clearNativeDispatch = async marker => {
+ if (!marker || !payload.pageFeedbackGuard) return;
+ try {
+ await this.call(match, `(el, marker, action) => {
+ const kind = JSON.parse(marker).kind;
+ const targets = kind === 'input' && action !== 'upload' ? [el, el.getRootNode().activeElement] : [el];
+ for (const target of new Set(targets))
+ if (target?.getAttribute('data-webbrain-native-action') === marker) target.removeAttribute('data-webbrain-native-action');
+ }`, [{ type: 'string', value: marker }, { type: 'string', value: action }]);
+ } catch { /* The document may have navigated or the monitor already consumed the marker. */ }
};
const point = action === 'click' && payload.point != null ? payload.point : null;
if (point && (!Number.isInteger(point.x) || !Number.isInteger(point.y))) throw new Error('Invalid click coordinates');
@@ -296,9 +349,11 @@ export class BidiSession {
const path = join(dir, name === '.' || name === '..' ? 'attachment' : name);
await writeFile(path, bytes, { mode: 0o600 });
assertLive();
- await markNativeDispatch('input');
- dispatch.started = true;
- await this.send('input.setFiles', { context: match.context, element: { sharedId: match.node.sharedId }, files: [path] });
+ const marker = await markNativeDispatch('input');
+ try {
+ dispatch.started = true;
+ await this.send('input.setFiles', { context: match.context, element: { sharedId: match.node.sharedId }, files: [path] });
+ } finally { await clearNativeDispatch(marker); }
const attached = await this.call(match, '(el, name, size) => el.isConnected && el.files?.length === 1 && el.files[0].name === name && el.files[0].size === size', [{ type: 'string', value: name }, { type: 'number', value: bytes.length }]);
if (attached.result?.value !== true) return { success: false, dispatched: true, outcomeUnknown: true, retryable: false, error: 'File input changed after attachment; inspect the page before retrying.' };
return { success: true, dispatched: true, attachmentState: 'input_attached', file: name, size: bytes.length };
@@ -327,15 +382,19 @@ export class BidiSession {
// Each keydown needs its own expectation. Keep the modifier held while
// renewing the fence, so Ctrl/Meta+A cannot look like a human edit.
if (modifier) {
- await markNativeDispatch('input');
- dispatch.started = true;
- await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard',
- actions: [{ type: 'keyDown', value: modifier }] }] });
+ const marker = await markNativeDispatch('input');
+ try {
+ dispatch.started = true;
+ await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard',
+ actions: [{ type: 'keyDown', value: modifier }] }] });
+ } finally { await clearNativeDispatch(marker); }
await assertFocus();
}
- await markNativeDispatch('input');
- dispatch.started = true;
- await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] });
+ const marker = await markNativeDispatch('input');
+ try {
+ dispatch.started = true;
+ await this.send('input.performActions', { context: match.context, actions: [{ type: 'key', id: 'webbrain-keyboard', actions }] });
+ } finally { await clearNativeDispatch(marker); }
}
finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }
assertLive();
@@ -347,9 +406,12 @@ export class BidiSession {
...(action === 'click' ? [{ type: 'pointerDown', button: 0 }, { type: 'pointerUp', button: 0 }] : []),
] };
assertLive();
- await markNativeDispatch('click');
+ const marker = await markNativeDispatch('click');
try { dispatch.started = true; await this.send('input.performActions', { context: match.context, actions: [source] }); }
- finally { await this.send('input.releaseActions', { context: match.context }).catch(() => {}); }
+ finally {
+ await clearNativeDispatch(marker);
+ await this.send('input.releaseActions', { context: match.context }).catch(() => {});
+ }
assertLive();
if (checkable) {
const observation = await this._waitForCheckableState(match, checkable.desiredChecked, 80);
@@ -411,12 +473,15 @@ export class BidiSession {
await assertFocus();
// Enter is a submit shortcut on many editors, even with Shift. Insert a literal
// newline through the editing command instead; never synthesize a submit key.
- await markNativeDispatch('input');
- dispatch.started = true;
- const inserted = await this.call(match, `(el, text) => {
- if (!el.isConnected || el.getRootNode().activeElement !== el) return false;
- return document.execCommand('insertText', false, text);
- }`, [{ type: 'string', value: char }]);
+ const marker = await markNativeDispatch('input');
+ let inserted;
+ try {
+ dispatch.started = true;
+ inserted = await this.call(match, `(el, text) => {
+ if (!el.isConnected || el.getRootNode().activeElement !== el) return false;
+ return document.execCommand('insertText', false, text);
+ }`, [{ type: 'string', value: char }]);
+ } finally { await clearNativeDispatch(marker); }
if (inserted.result?.value !== true) throw new Error('Editor rejected literal newline insertion');
assertLive();
} else await press(char);
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 0f058daea..391cae9e1 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -37807,12 +37807,17 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
operationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,
});
}
- targetDispatched = true;
- try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}
- el.dispatchEvent(new MouseEvent('mousedown', opts));
- try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}
- el.dispatchEvent(new MouseEvent('mouseup', opts));
- el.click();
+ const dispatch = () => {
+ targetDispatched = true;
+ try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}
+ el.dispatchEvent(new MouseEvent('mousedown', opts));
+ try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}
+ el.dispatchEvent(new MouseEvent('mouseup', opts));
+ el.click();
+ };
+ if (monitorGuard?.operationId && window.__wbPageMonitor?.withPreparedDispatch)
+ window.__wbPageMonitor.withPreparedDispatch(monitorGuard.operationId, dispatch);
+ else dispatch();
return { ok: true, url: location.href, tag: el.tagName, text: (el.innerText || el.value || '').slice(0, 80), dispatched: true };
} catch (e) {
return { ok: false, url: location.href, dispatched: targetDispatched, error: e.message };
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 5af8580d2..ba6a5a7e0 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -2354,9 +2354,11 @@
// the editor's undo stack. Its beforeinput behavior differs by browser,
// so offer the page a cancellable gate BEFORE invoking the command.
window.__wbPageMonitor?.beforeLocalDispatch();
- const accepted = el.dispatchEvent(new InputEvent('beforeinput', {
+ const beforeInputGate = () => el.dispatchEvent(new InputEvent('beforeinput', {
bubbles: true, composed: true, cancelable: true, inputType, data,
}));
+ const accepted = window.__wbPageMonitor?.withLocalDispatch
+ ? window.__wbPageMonitor.withLocalDispatch(beforeInputGate) : beforeInputGate();
if (el.innerHTML !== htmlBefore || !el.isConnected) {
window.__wbPageMonitor?.beforeLocalDispatch();
dispatched = true;
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 18cf7e9fe..0715e26f3 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -15,19 +15,57 @@
const listeners = [];
const operations = new Map();
const nativeTargets = new Set();
+ const liveControls = new Set();
let signatures = new WeakMap();
+ let controlSignatures = new WeakMap();
let textSignatures = new WeakMap();
let popoverTurns = new WeakMap();
let roots = new WeakSet();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
- let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, lastUserAt = 0;
- let pointerHeld = false, composing = false, localOperation = null;
+ let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;
+ let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
let lastViewport = '';
let lastFeedbackDelivery = Promise.resolve();
let agentTurn = null, userTurn = null, lastUserTarget = null;
- let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();
+ let matchedEvents = new WeakMap();
+ const activePointers = new Set();
const fenceAttribute = 'data-webbrain-page-revision';
+ const nativeMarkerAttribute = 'data-webbrain-native-action';
+ const nativeMarkerMask = (1n << 64n) - 1n;
+ const nativeMarkerRotate = (value, bits) => ((value << BigInt(bits)) | (value >> BigInt(64 - bits))) & nativeMarkerMask;
+ function nativeMarkerSignature(secret, data) {
+ if (!/^[a-f\d]{32}$/i.test(secret || '')) return '';
+ const key = Uint8Array.from(secret.match(/.{2}/g), byte => parseInt(byte, 16));
+ const read64 = offset => {
+ let value = 0n;
+ for (let index = 0; index < 8; index++) value |= BigInt(key[offset + index]) << BigInt(index * 8);
+ return value;
+ };
+ const k0 = read64(0), k1 = read64(8);
+ const rotate = nativeMarkerRotate;
+ let v0 = 0x736f6d6570736575n ^ k0, v1 = 0x646f72616e646f6dn ^ k1;
+ let v2 = 0x6c7967656e657261n ^ k0, v3 = 0x7465646279746573n ^ k1;
+ const round = () => {
+ v0 = (v0 + v1) & nativeMarkerMask; v1 = rotate(v1, 13); v1 ^= v0; v0 = rotate(v0, 32);
+ v2 = (v2 + v3) & nativeMarkerMask; v3 = rotate(v3, 16); v3 ^= v2;
+ v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;
+ v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);
+ };
+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,
+ data.navigationCandidate === true]));
+ let offset = 0;
+ while (offset + 8 <= bytes.length) {
+ let block = 0n;
+ for (let index = 0; index < 8; index++) block |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ v3 ^= block; round(); round(); v0 ^= block; offset += 8;
+ }
+ let tail = BigInt(bytes.length) << 56n;
+ for (let index = 0; offset + index < bytes.length; index++) tail |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ v3 ^= tail; round(); round(); v0 ^= tail; v2 ^= 0xffn;
+ round(); round(); round(); round();
+ return (v0 ^ v1 ^ v2 ^ v3).toString(16).padStart(16, '0');
+ }
const publishRevision = () => {
const value = `${documentToken}:${revision}`;
if (active && document.documentElement?.getAttribute(fenceAttribute) !== value) document.documentElement?.setAttribute(fenceAttribute, value);
@@ -194,20 +232,32 @@
return true;
}
function expected(kind, el, event) {
- const marker = el?.getAttribute?.('data-webbrain-native-action');
- if (marker && nativeMarks.get(el) !== marker) {
- nativeMarks.set(el, marker); nativeTargets.add(el);
+ const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))
+ || (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);
+ const marker = markerNode?.getAttribute(nativeMarkerAttribute);
+ if (marker && event?.isTrusted === true) {
try {
const data = JSON.parse(marker);
- if (data.documentToken === documentToken && operations.has(data.operationId) && ['input', 'click', 'scroll'].includes(data.kind)) {
+ const op = operations.get(data.operationId);
+ const markerTarget = op && (markerNode === op.target || markerNode === op.focusTarget);
+ const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;
+ const allowedEvent = data.kind === 'input'
+ ? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)
+ : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;
+ const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;
+ if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence
+ && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {
+ // The capability is bound to the isolated-world prepared node and consumed once.
+ op.nativeSequence = data.sequence;
+ markerNode.removeAttribute(nativeMarkerAttribute);
dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });
- operations.get(data.operationId).target = el;
}
} catch { /* Page markers are hints; malformed ones grant no expectation. */ }
}
if (event && matchedEvents.has(event)) return matchedEvents.get(event);
prune();
const match = op => {
+ if (event && !event.isTrusted && !op.synchronous) return null;
if (event) {
if (op.eventTypes && !op.eventTypes.has(event.type)) return null;
// One native dispatch owns one occurrence of each input phase. A later
@@ -258,13 +308,49 @@
while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;
return el;
}
+ function controlValueFingerprint(el) {
+ let controlValue = null;
+ if ((el.tagName === 'TEXTAREA' || (el.tagName === 'INPUT'
+ && !['password', 'file', 'hidden', 'checkbox', 'radio', 'button', 'submit', 'reset', 'image'].includes(el.type)))) {
+ const value = el.value || '', samples = Math.min(value.length, 8192);
+ let first = 0x811c9dc5, second = 0x9e3779b9;
+ // Hash locally only; feedback contains a generic target and never the value or typed keys.
+ for (let index = 0; index < samples; index++) {
+ const code = value.charCodeAt(samples === value.length ? index : Math.floor(index * value.length / samples));
+ first = Math.imul(first ^ code, 16777619) >>> 0;
+ second = Math.imul(second ^ (code + index), 2246822519) >>> 0;
+ }
+ controlValue = [value.length, first, second];
+ }
+ return controlValue;
+ }
+ function controlState(el) {
+ return JSON.stringify([el.getAttribute('value'), el.selected === true, el.defaultSelected === true,
+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null, el.checked === true,
+ el.disabled === true, el.readOnly === true, el.required === true, el.validity?.valid !== false,
+ controlValueFingerprint(el)]);
+ }
+ function trackControl(el) {
+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;
+ liveControls.add(el);
+ controlSignatures.set(el, controlState(el));
+ }
+ function refreshControl(el) {
+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) {
+ trackControl(el);
+ controlSignatures.set(el, controlState(el));
+ }
+ signatures.set(el, signature(el));
+ }
function signature(el) {
if (ignored(el)) return '';
+ trackControl(el);
const shown = visible(el);
const rect = shown ? el.getBoundingClientRect() : {};
const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3)
.map(textSignature).reduce(([size, hash], [length, part, power]) =>
[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);
+ const controlValue = controlValueFingerprint(el);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-pressed'),
@@ -272,17 +358,33 @@
el.selected === true, el.defaultSelected === true,
Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,
el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
- el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
+ el.getAttribute('aria-disabled'), el.getAttribute('value'), controlValue, el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
}
function seed(root, budget = { remaining: 600 }) {
const nodes = root.querySelectorAll?.('*') || [];
for (const el of nodes) {
+ // Form-state tracking has its own cap, separate from bounded layout reads.
+ trackControl(el);
if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }
// Root discovery is cheap and must not share the layout-measurement cap.
if (el.shadowRoot && !ignored(el)) observeRoot(el.shadowRoot, budget);
}
}
+ function sampleFormControls() {
+ if (!active || !liveControls.size) return;
+ const controls = [...liveControls];
+ if (controlCursor >= controls.length) controlCursor = 0;
+ const batchSize = Math.min(200, controls.length);
+ const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);
+ controlCursor = (controlCursor + batchSize) % controls.length;
+ const records = [];
+ for (const el of batch) {
+ if (!el.isConnected) { liveControls.delete(el); continue; }
+ if (!ignored(el)) records.push({ type: 'control', target: el });
+ }
+ if (records.length) onMutations(records);
+ }
function sampleDescendants(root, limit = 100) {
const nodes = [];
const stack = [];
@@ -417,6 +519,13 @@
target ||= targetName(el);
break;
}
+ if (record.type === 'control') {
+ const nextControl = controlState(el), previousControl = controlSignatures.get(el);
+ controlSignatures.set(el, nextControl);
+ signatures.set(el, signature(el));
+ if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
+ continue;
+ }
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
@@ -434,6 +543,7 @@
}
const next = signature(el), previous = signatures.get(el);
signatures.set(el, next);
+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) controlSignatures.set(el, controlState(el));
if (!identityChanged && (next === previous || (!visible(el) && !previous?.startsWith('[true')))) continue;
noteChange(el, record.agentUserAt);
}
@@ -479,6 +589,7 @@
active = true; runToken = state.runToken; seq = 0; revision = 0; publishRevision();
observer = new MutationObserver(onMutations);
observeRoot(document);
+ controlTimer = setInterval(sampleFormControls, 250);
observeLayoutShifts();
listen(document, 'webbrain-shadow-root-attached', event => {
const path = event.composedPath();
@@ -531,17 +642,21 @@
listen(document, 'pointerdown', event => {
const el = elementFor(event);
if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;
+ activePointers.add(event.pointerId);
pointerHeld = true; interact('activity', el);
});
listen(document, 'pointermove', event => {
- if (pointerHeld && event.isTrusted) {
- if (!event.buttons) pointerHeld = false;
+ if (activePointers.has(event.pointerId) && event.isTrusted) {
+ if (!event.buttons) activePointers.delete(event.pointerId);
+ pointerHeld = activePointers.size > 0;
interact('activity', elementFor(event));
}
});
for (const name of ['pointerup', 'pointercancel']) listen(document, name, event => {
- if (!pointerHeld || !event.isTrusted) return;
- pointerHeld = false; interact('activity', elementFor(event), { interacting: composing, completeGesture: true });
+ if (!activePointers.has(event.pointerId) || !event.isTrusted) return;
+ activePointers.delete(event.pointerId);
+ pointerHeld = activePointers.size > 0;
+ interact('activity', elementFor(event), { completeGesture: activePointers.size === 0 });
});
listen(document, 'click', event => {
const el = elementFor(event);
@@ -559,10 +674,14 @@
});
for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {
const el = elementFor(event);
- if (ignored(el) || expected('input', el, event)
- || (el.matches?.('input[type="checkbox"],input[type="radio"],select,option') && expected('click', el, event))) return;
+ const expectedInput = expected('input', el, event);
+ const expectedClick = !expectedInput && el.matches?.('input[type="checkbox"],input[type="radio"],select,option')
+ ? expected('click', el, event) : null;
+ if (ignored(el)) return;
+ if (expectedInput || expectedClick) { refreshControl(el); return; }
if (event.isTrusted) interact('input', el);
else { revision++; send({ kind: 'input', source: 'page', target: targetName(el) }); }
+ refreshControl(el);
});
listen(document, 'compositionstart', event => {
const el = elementFor(event);
@@ -619,13 +738,13 @@
if (Date.now() - lastUserAt < 1000 && !editable(document.activeElement)) interact('selection', document.activeElement || document.body);
});
listen(window, 'blur', () => {
- if (pointerHeld || composing) { pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }
+ if (pointerHeld || composing) { activePointers.clear(); pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }
});
listen(window, 'pagehide', () => stop());
}
function stop() {
- active = false; runToken = ''; pointerHeld = false; composing = false;
- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();
+ active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;
+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
el.removeAttribute?.('data-webbrain-native-action');
@@ -634,10 +753,10 @@
nativeTargets.clear();
observer?.disconnect(); observer = null;
layoutObserver?.disconnect(); layoutObserver = null;
- clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
+ clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
- roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap();
+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();
}
async function requestState() {
const generation = ++requestGeneration;
@@ -665,6 +784,7 @@
function dispatch(params) {
if (!active || (params.runToken && params.runToken !== runToken)) return;
const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };
+ if (params.fenceOnly && !op.nativeSecret) op.nativeSecret = Array.from(crypto.getRandomValues(new Uint8Array(16)), byte => byte.toString(16).padStart(2, '0')).join('');
op.blurTarget ??= document.activeElement;
op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);
if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;
@@ -690,6 +810,7 @@
operations.set(op.operationId, op);
}
function activatePreparedDispatch(params = {}) {
+ sampleFormControls();
const op = operations.get(params.operationId);
const layoutChanged = coordinatePreparationShifted(op);
if (!active || !op || layoutChanged || domTimer || unreported
@@ -704,6 +825,14 @@
publishRevision();
return { ready: true, operationId: op.operationId, revision };
}
+ function withPreparedDispatch(operationId, callback) {
+ const op = operations.get(operationId);
+ if (!op?.dispatched || typeof callback !== 'function') return callback?.();
+ const previous = op.synchronous;
+ op.synchronous = true;
+ try { return callback(); }
+ finally { op.synchronous = previous; }
+ }
const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',
'hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js',
'ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);
@@ -726,6 +855,7 @@
}
function beforeLocalDispatch({ preparation = false, kind, target } = {}) {
if (!active || !localOperation) return;
+ sampleFormControls();
const layoutChanged = coordinatePreparationShifted(operations.get(localOperation.operationId));
if (layoutChanged || domTimer || unreported || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
@@ -762,6 +892,7 @@
} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }
else if (msg.action === 'page_monitor_dispatch') {
const params = msg.params || {};
+ if (!params.release) sampleFormControls();
const prepared = operations.get(params.operationId);
const layoutChanged = !params.release && active && coordinatePreparationShifted(prepared);
if (!params.release && active && (layoutChanged || domTimer || unreported || (prepared && prepared.preparedRevision !== revision)
@@ -775,7 +906,9 @@
}
dispatch(params); publishRevision();
respond({ ready: true, ...(active && params.runToken === runToken ? { guard: {
- documentToken, revision, operationId: params.operationId, navigationCandidate: params.navigationCandidate !== false,
+ documentToken, revision, operationId: params.operationId, runToken,
+ navigationCandidate: params.navigationCandidate !== false,
+ ...(operations.get(params.operationId)?.nativeSecret ? { nativeSecret: operations.get(params.operationId).nativeSecret } : {}),
} } : {}) });
}
else if (msg.action === 'page_monitor_finish') {
@@ -785,7 +918,7 @@
}
};
api.runtime.onMessage.addListener(onMessage);
- window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, registerDecoration,
+ window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, withPreparedDispatch, registerDecoration,
get active() { return active; },
get disposed() { return disposed; },
dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 84299ba9b..3b69c621a 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -34859,12 +34859,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
operationId: monitorGuard.operationId, kind: 'click', element: el, navigationCandidate: true,
});
}
- targetDispatched = true;
- try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}
- el.dispatchEvent(new MouseEvent('mousedown', opts));
- try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}
- el.dispatchEvent(new MouseEvent('mouseup', opts));
- el.click();
+ const dispatch = () => {
+ targetDispatched = true;
+ try { el.dispatchEvent(new PointerEvent('pointerdown', opts)); } catch (e) {}
+ el.dispatchEvent(new MouseEvent('mousedown', opts));
+ try { el.dispatchEvent(new PointerEvent('pointerup', opts)); } catch (e) {}
+ el.dispatchEvent(new MouseEvent('mouseup', opts));
+ el.click();
+ };
+ if (monitorGuard?.operationId && window.__wbPageMonitor?.withPreparedDispatch)
+ window.__wbPageMonitor.withPreparedDispatch(monitorGuard.operationId, dispatch);
+ else dispatch();
return { ok: true, url: location.href, tag: el.tagName, text: (el.innerText || el.value || '').slice(0, 80), dispatched: true };
} catch (e) { return { ok: false, url: location.href, dispatched: targetDispatched, error: e.message }; }
})()
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index b233b728f..8aad1e061 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -2670,9 +2670,11 @@
// the editor's undo stack. Its beforeinput behavior differs by browser,
// so offer the page a cancellable gate BEFORE invoking the command.
window.__wbPageMonitor?.beforeLocalDispatch();
- const accepted = el.dispatchEvent(new InputEvent('beforeinput', {
+ const beforeInputGate = () => el.dispatchEvent(new InputEvent('beforeinput', {
bubbles: true, composed: true, cancelable: true, inputType, data,
}));
+ const accepted = window.__wbPageMonitor?.withLocalDispatch
+ ? window.__wbPageMonitor.withLocalDispatch(beforeInputGate) : beforeInputGate();
if (el.innerHTML !== htmlBefore || !el.isConnected) {
window.__wbPageMonitor?.beforeLocalDispatch();
dispatched = true;
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 18cf7e9fe..0715e26f3 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -15,19 +15,57 @@
const listeners = [];
const operations = new Map();
const nativeTargets = new Set();
+ const liveControls = new Set();
let signatures = new WeakMap();
+ let controlSignatures = new WeakMap();
let textSignatures = new WeakMap();
let popoverTurns = new WeakMap();
let roots = new WeakSet();
let active = false, disposed = false, runToken = '', seq = 0, revision = 0;
- let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, lastUserAt = 0;
- let pointerHeld = false, composing = false, localOperation = null;
+ let observer = null, layoutObserver = null, domTimer = null, scrollTimer = null, controlTimer = null, lastUserAt = 0;
+ let pointerHeld = false, composing = false, localOperation = null, controlCursor = 0;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
let lastViewport = '';
let lastFeedbackDelivery = Promise.resolve();
let agentTurn = null, userTurn = null, lastUserTarget = null;
- let matchedEvents = new WeakMap(), nativeMarks = new WeakMap();
+ let matchedEvents = new WeakMap();
+ const activePointers = new Set();
const fenceAttribute = 'data-webbrain-page-revision';
+ const nativeMarkerAttribute = 'data-webbrain-native-action';
+ const nativeMarkerMask = (1n << 64n) - 1n;
+ const nativeMarkerRotate = (value, bits) => ((value << BigInt(bits)) | (value >> BigInt(64 - bits))) & nativeMarkerMask;
+ function nativeMarkerSignature(secret, data) {
+ if (!/^[a-f\d]{32}$/i.test(secret || '')) return '';
+ const key = Uint8Array.from(secret.match(/.{2}/g), byte => parseInt(byte, 16));
+ const read64 = offset => {
+ let value = 0n;
+ for (let index = 0; index < 8; index++) value |= BigInt(key[offset + index]) << BigInt(index * 8);
+ return value;
+ };
+ const k0 = read64(0), k1 = read64(8);
+ const rotate = nativeMarkerRotate;
+ let v0 = 0x736f6d6570736575n ^ k0, v1 = 0x646f72616e646f6dn ^ k1;
+ let v2 = 0x6c7967656e657261n ^ k0, v3 = 0x7465646279746573n ^ k1;
+ const round = () => {
+ v0 = (v0 + v1) & nativeMarkerMask; v1 = rotate(v1, 13); v1 ^= v0; v0 = rotate(v0, 32);
+ v2 = (v2 + v3) & nativeMarkerMask; v3 = rotate(v3, 16); v3 ^= v2;
+ v0 = (v0 + v3) & nativeMarkerMask; v3 = rotate(v3, 21); v3 ^= v0;
+ v2 = (v2 + v1) & nativeMarkerMask; v1 = rotate(v1, 17); v1 ^= v2; v2 = rotate(v2, 32);
+ };
+ const bytes = new TextEncoder().encode(JSON.stringify([data.documentToken, data.runToken, data.operationId, data.kind, data.sequence,
+ data.navigationCandidate === true]));
+ let offset = 0;
+ while (offset + 8 <= bytes.length) {
+ let block = 0n;
+ for (let index = 0; index < 8; index++) block |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ v3 ^= block; round(); round(); v0 ^= block; offset += 8;
+ }
+ let tail = BigInt(bytes.length) << 56n;
+ for (let index = 0; offset + index < bytes.length; index++) tail |= BigInt(bytes[offset + index]) << BigInt(index * 8);
+ v3 ^= tail; round(); round(); v0 ^= tail; v2 ^= 0xffn;
+ round(); round(); round(); round();
+ return (v0 ^ v1 ^ v2 ^ v3).toString(16).padStart(16, '0');
+ }
const publishRevision = () => {
const value = `${documentToken}:${revision}`;
if (active && document.documentElement?.getAttribute(fenceAttribute) !== value) document.documentElement?.setAttribute(fenceAttribute, value);
@@ -194,20 +232,32 @@
return true;
}
function expected(kind, el, event) {
- const marker = el?.getAttribute?.('data-webbrain-native-action');
- if (marker && nativeMarks.get(el) !== marker) {
- nativeMarks.set(el, marker); nativeTargets.add(el);
+ const markerNode = event?.composedPath?.().find(node => node instanceof Element && node.hasAttribute(nativeMarkerAttribute))
+ || (el?.hasAttribute?.(nativeMarkerAttribute) ? el : null);
+ const marker = markerNode?.getAttribute(nativeMarkerAttribute);
+ if (marker && event?.isTrusted === true) {
try {
const data = JSON.parse(marker);
- if (data.documentToken === documentToken && operations.has(data.operationId) && ['input', 'click', 'scroll'].includes(data.kind)) {
+ const op = operations.get(data.operationId);
+ const markerTarget = op && (markerNode === op.target || markerNode === op.focusTarget);
+ const nextSequence = Number.isSafeInteger(data.sequence) && data.sequence === (op?.nativeSequence || 0) + 1;
+ const allowedEvent = data.kind === 'input'
+ ? ['keydown', 'beforeinput', 'input', 'change', 'compositionstart', 'compositionend'].includes(event.type)
+ : data.kind === 'click' ? ['pointerdown', 'mousedown', 'click'].includes(event.type) : false;
+ const signed = op && nativeMarkerSignature(op.nativeSecret, data) === data.signature;
+ if (data.documentToken === documentToken && data.runToken === runToken && op && markerTarget && nextSequence
+ && signed && allowedEvent && ['input', 'click'].includes(data.kind)) {
+ // The capability is bound to the isolated-world prepared node and consumed once.
+ op.nativeSequence = data.sequence;
+ markerNode.removeAttribute(nativeMarkerAttribute);
dispatch({ operationId: data.operationId, kind: data.kind, runToken, navigationCandidate: data.navigationCandidate === true });
- operations.get(data.operationId).target = el;
}
} catch { /* Page markers are hints; malformed ones grant no expectation. */ }
}
if (event && matchedEvents.has(event)) return matchedEvents.get(event);
prune();
const match = op => {
+ if (event && !event.isTrusted && !op.synchronous) return null;
if (event) {
if (op.eventTypes && !op.eventTypes.has(event.type)) return null;
// One native dispatch owns one occurrence of each input phase. A later
@@ -258,13 +308,49 @@
while (el?.shadowRoot?.activeElement) el = el.shadowRoot.activeElement;
return el;
}
+ function controlValueFingerprint(el) {
+ let controlValue = null;
+ if ((el.tagName === 'TEXTAREA' || (el.tagName === 'INPUT'
+ && !['password', 'file', 'hidden', 'checkbox', 'radio', 'button', 'submit', 'reset', 'image'].includes(el.type)))) {
+ const value = el.value || '', samples = Math.min(value.length, 8192);
+ let first = 0x811c9dc5, second = 0x9e3779b9;
+ // Hash locally only; feedback contains a generic target and never the value or typed keys.
+ for (let index = 0; index < samples; index++) {
+ const code = value.charCodeAt(samples === value.length ? index : Math.floor(index * value.length / samples));
+ first = Math.imul(first ^ code, 16777619) >>> 0;
+ second = Math.imul(second ^ (code + index), 2246822519) >>> 0;
+ }
+ controlValue = [value.length, first, second];
+ }
+ return controlValue;
+ }
+ function controlState(el) {
+ return JSON.stringify([el.getAttribute('value'), el.selected === true, el.defaultSelected === true,
+ Number.isInteger(el.selectedIndex) ? el.selectedIndex : null, el.checked === true,
+ el.disabled === true, el.readOnly === true, el.required === true, el.validity?.valid !== false,
+ controlValueFingerprint(el)]);
+ }
+ function trackControl(el) {
+ if (!/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName) || liveControls.has(el) || liveControls.size >= 600) return;
+ liveControls.add(el);
+ controlSignatures.set(el, controlState(el));
+ }
+ function refreshControl(el) {
+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) {
+ trackControl(el);
+ controlSignatures.set(el, controlState(el));
+ }
+ signatures.set(el, signature(el));
+ }
function signature(el) {
if (ignored(el)) return '';
+ trackControl(el);
const shown = visible(el);
const rect = shown ? el.getBoundingClientRect() : {};
const content = editable(el) ? '' : [...el.childNodes, ...(el.shadowRoot?.childNodes || [])].filter(node => node.nodeType === 3)
.map(textSignature).reduce(([size, hash], [length, part, power]) =>
[size + length, (Math.imul(hash, power) + part) >>> 0], [0, 0]);
+ const controlValue = controlValueFingerprint(el);
return JSON.stringify([shown, content, el.children.length + (el.shadowRoot?.children.length || 0), el.getAttribute('role'), el.getAttribute('aria-label'),
el.getAttribute('aria-expanded'), el.getAttribute('aria-selected'), el.getAttribute('aria-checked'),
el.getAttribute('aria-pressed'),
@@ -272,17 +358,33 @@
el.selected === true, el.defaultSelected === true,
Number.isInteger(el.selectedIndex) ? el.selectedIndex : null,
el.getAttribute('aria-hidden'), el.hasAttribute('open'), el.inert === true,
- el.getAttribute('aria-disabled'), el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
+ el.getAttribute('aria-disabled'), el.getAttribute('value'), controlValue, el.disabled === true, el.readOnly === true, el.checked === true, el.validity?.valid !== false, Math.round((rect.width || 0) / 8), Math.round((rect.height || 0) / 8),
Math.round(((rect.x || 0) + window.scrollX) / 8), Math.round(((rect.y || 0) + window.scrollY) / 8)]);
}
function seed(root, budget = { remaining: 600 }) {
const nodes = root.querySelectorAll?.('*') || [];
for (const el of nodes) {
+ // Form-state tracking has its own cap, separate from bounded layout reads.
+ trackControl(el);
if (budget.remaining > 0) { budget.remaining--; signatures.set(el, signature(el)); }
// Root discovery is cheap and must not share the layout-measurement cap.
if (el.shadowRoot && !ignored(el)) observeRoot(el.shadowRoot, budget);
}
}
+ function sampleFormControls() {
+ if (!active || !liveControls.size) return;
+ const controls = [...liveControls];
+ if (controlCursor >= controls.length) controlCursor = 0;
+ const batchSize = Math.min(200, controls.length);
+ const batch = Array.from({ length: batchSize }, (_, index) => controls[(controlCursor + index) % controls.length]);
+ controlCursor = (controlCursor + batchSize) % controls.length;
+ const records = [];
+ for (const el of batch) {
+ if (!el.isConnected) { liveControls.delete(el); continue; }
+ if (!ignored(el)) records.push({ type: 'control', target: el });
+ }
+ if (records.length) onMutations(records);
+ }
function sampleDescendants(root, limit = 100) {
const nodes = [];
const stack = [];
@@ -417,6 +519,13 @@
target ||= targetName(el);
break;
}
+ if (record.type === 'control') {
+ const nextControl = controlState(el), previousControl = controlSignatures.get(el);
+ controlSignatures.set(el, nextControl);
+ signatures.set(el, signature(el));
+ if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
+ continue;
+ }
let identityChanged = (record.type === 'popover' && record.stateChanged)
|| (record.type === 'layout' && record.layoutChanged)
|| (record.type === 'shadow' && (visible(el) || signatures.get(el)?.startsWith('[true')));
@@ -434,6 +543,7 @@
}
const next = signature(el), previous = signatures.get(el);
signatures.set(el, next);
+ if (/^(INPUT|TEXTAREA|SELECT|OPTION)$/.test(el.tagName)) controlSignatures.set(el, controlState(el));
if (!identityChanged && (next === previous || (!visible(el) && !previous?.startsWith('[true')))) continue;
noteChange(el, record.agentUserAt);
}
@@ -479,6 +589,7 @@
active = true; runToken = state.runToken; seq = 0; revision = 0; publishRevision();
observer = new MutationObserver(onMutations);
observeRoot(document);
+ controlTimer = setInterval(sampleFormControls, 250);
observeLayoutShifts();
listen(document, 'webbrain-shadow-root-attached', event => {
const path = event.composedPath();
@@ -531,17 +642,21 @@
listen(document, 'pointerdown', event => {
const el = elementFor(event);
if (!event.isTrusted || ignored(el) || expected('click', el, event)) return;
+ activePointers.add(event.pointerId);
pointerHeld = true; interact('activity', el);
});
listen(document, 'pointermove', event => {
- if (pointerHeld && event.isTrusted) {
- if (!event.buttons) pointerHeld = false;
+ if (activePointers.has(event.pointerId) && event.isTrusted) {
+ if (!event.buttons) activePointers.delete(event.pointerId);
+ pointerHeld = activePointers.size > 0;
interact('activity', elementFor(event));
}
});
for (const name of ['pointerup', 'pointercancel']) listen(document, name, event => {
- if (!pointerHeld || !event.isTrusted) return;
- pointerHeld = false; interact('activity', elementFor(event), { interacting: composing, completeGesture: true });
+ if (!activePointers.has(event.pointerId) || !event.isTrusted) return;
+ activePointers.delete(event.pointerId);
+ pointerHeld = activePointers.size > 0;
+ interact('activity', elementFor(event), { completeGesture: activePointers.size === 0 });
});
listen(document, 'click', event => {
const el = elementFor(event);
@@ -559,10 +674,14 @@
});
for (const name of ['beforeinput', 'input', 'change']) listen(document, name, event => {
const el = elementFor(event);
- if (ignored(el) || expected('input', el, event)
- || (el.matches?.('input[type="checkbox"],input[type="radio"],select,option') && expected('click', el, event))) return;
+ const expectedInput = expected('input', el, event);
+ const expectedClick = !expectedInput && el.matches?.('input[type="checkbox"],input[type="radio"],select,option')
+ ? expected('click', el, event) : null;
+ if (ignored(el)) return;
+ if (expectedInput || expectedClick) { refreshControl(el); return; }
if (event.isTrusted) interact('input', el);
else { revision++; send({ kind: 'input', source: 'page', target: targetName(el) }); }
+ refreshControl(el);
});
listen(document, 'compositionstart', event => {
const el = elementFor(event);
@@ -619,13 +738,13 @@
if (Date.now() - lastUserAt < 1000 && !editable(document.activeElement)) interact('selection', document.activeElement || document.body);
});
listen(window, 'blur', () => {
- if (pointerHeld || composing) { pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }
+ if (pointerHeld || composing) { activePointers.clear(); pointerHeld = false; composing = false; interact('activity', document.body, { interacting: false }); }
});
listen(window, 'pagehide', () => stop());
}
function stop() {
- active = false; runToken = ''; pointerHeld = false; composing = false;
- lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); nativeMarks = new WeakMap();
+ active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;
+ lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlCursor = 0;
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
el.removeAttribute?.('data-webbrain-native-action');
@@ -634,10 +753,10 @@
nativeTargets.clear();
observer?.disconnect(); observer = null;
layoutObserver?.disconnect(); layoutObserver = null;
- clearTimeout(domTimer); clearTimeout(scrollTimer); domTimer = null; scrollTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
+ clearTimeout(domTimer); clearTimeout(scrollTimer); clearInterval(controlTimer); domTimer = null; scrollTimer = null; controlTimer = null; unreported = 0; pendingDOM = null; lastFeedbackDelivery = Promise.resolve();
listeners.splice(0).forEach(remove => remove());
operations.clear(); localOperation = null; agentTurn = null;
- roots = new WeakSet(); signatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap();
+ roots = new WeakSet(); signatures = new WeakMap(); controlSignatures = new WeakMap(); textSignatures = new WeakMap(); popoverTurns = new WeakMap(); liveControls.clear();
}
async function requestState() {
const generation = ++requestGeneration;
@@ -665,6 +784,7 @@
function dispatch(params) {
if (!active || (params.runToken && params.runToken !== runToken)) return;
const op = operations.get(params.operationId) || { operationId: params.operationId, userAt: lastUserAt };
+ if (params.fenceOnly && !op.nativeSecret) op.nativeSecret = Array.from(crypto.getRandomValues(new Uint8Array(16)), byte => byte.toString(16).padStart(2, '0')).join('');
op.blurTarget ??= document.activeElement;
op.target = resolveTarget(params) || (params.kind === 'input' ? op.focusTarget || deepActiveElement() : op.target);
if (params.kind === 'input' && !op.focusTarget) op.focusTarget = op.target;
@@ -690,6 +810,7 @@
operations.set(op.operationId, op);
}
function activatePreparedDispatch(params = {}) {
+ sampleFormControls();
const op = operations.get(params.operationId);
const layoutChanged = coordinatePreparationShifted(op);
if (!active || !op || layoutChanged || domTimer || unreported
@@ -704,6 +825,14 @@
publishRevision();
return { ready: true, operationId: op.operationId, revision };
}
+ function withPreparedDispatch(operationId, callback) {
+ const op = operations.get(operationId);
+ if (!op?.dispatched || typeof callback !== 'function') return callback?.();
+ const previous = op.synchronous;
+ op.synchronous = true;
+ try { return callback(); }
+ finally { op.synchronous = previous; }
+ }
const localMutations = new Set(['click', 'click_ax', 'type', 'type_ax', 'set_field', 'set_checked', 'press_keys', 'scroll',
'hover', 'drag_drop', 'patch_element', 'revert_patch', 'highlight_element', 'execute_js',
'ax_prepare_field_for_trusted_type', 'ax_resolve_two_rects', 'ax_resolve_rect']);
@@ -726,6 +855,7 @@
}
function beforeLocalDispatch({ preparation = false, kind, target } = {}) {
if (!active || !localOperation) return;
+ sampleFormControls();
const layoutChanged = coordinatePreparationShifted(operations.get(localOperation.operationId));
if (layoutChanged || domTimer || unreported || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
@@ -762,6 +892,7 @@
} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }
else if (msg.action === 'page_monitor_dispatch') {
const params = msg.params || {};
+ if (!params.release) sampleFormControls();
const prepared = operations.get(params.operationId);
const layoutChanged = !params.release && active && coordinatePreparationShifted(prepared);
if (!params.release && active && (layoutChanged || domTimer || unreported || (prepared && prepared.preparedRevision !== revision)
@@ -775,7 +906,9 @@
}
dispatch(params); publishRevision();
respond({ ready: true, ...(active && params.runToken === runToken ? { guard: {
- documentToken, revision, operationId: params.operationId, navigationCandidate: params.navigationCandidate !== false,
+ documentToken, revision, operationId: params.operationId, runToken,
+ navigationCandidate: params.navigationCandidate !== false,
+ ...(operations.get(params.operationId)?.nativeSecret ? { nativeSecret: operations.get(params.operationId).nativeSecret } : {}),
} } : {}) });
}
else if (msg.action === 'page_monitor_finish') {
@@ -785,7 +918,7 @@
}
};
api.runtime.onMessage.addListener(onMessage);
- window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, registerDecoration,
+ window.__wbPageMonitor = { beginContentAction, beforeLocalDispatch, withLocalDispatch, dispatch, activatePreparedDispatch, withPreparedDispatch, registerDecoration,
get active() { return active; },
get disposed() { return disposed; },
dispose() { disposed = true; requestGeneration++; stop(); api.runtime.onMessage.removeListener?.(onMessage); } };
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 3d02cd870..09448a66b 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -4,7 +4,7 @@ import { test } from 'node:test';
import { chromium, firefox } from 'playwright';
import { CDPClient } from '../src/chrome/src/cdp/cdp-client.js';
import { pageFeedbackMethods } from '../src/chrome/src/agent/page-feedback.js';
-import { BidiSession } from '../firefox-companion/session.mjs';
+import { BidiSession, createNativeActionMarker } from '../firefox-companion/session.mjs';
const read = (build, file) => fs.readFileSync(new URL(`../src/${build}/src/${file}`, import.meta.url), 'utf8');
const html = `<!doctype html><style>
@@ -17,11 +17,21 @@ body { margin: 0; } button,input { margin: 12px; }
<div id="moving">Animated</div><div style="height:3000px"></div>
<script>document.getElementById('agent').addEventListener('click', () => { document.getElementById('status').textContent = 'Agent changed this'; });</script>`;
-async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false, omitEmptyFrameMonitor = false } = {}) {
+async function fixture(engine, build, { runToken = 'test-run', siteIsolation = false, omitEmptyFrameMonitor = false,
+ capturePointerHandlers = false } = {}) {
const browser = await engine.launch({ headless: true, ...(siteIsolation ? { args: ['--site-per-process'] } : {}) });
const context = await browser.newContext();
await context.route('https://monitor.test/**', route => route.fulfill({ contentType: 'text/html', body: html }));
await context.addInitScript(token => { window.monitorRunToken = token; }, runToken);
+ if (capturePointerHandlers) await context.addInitScript(() => {
+ window.monitorPointerHandlers = {};
+ const add = EventTarget.prototype.addEventListener;
+ EventTarget.prototype.addEventListener = function(type, handler, options) {
+ if (this === document && ['pointerdown', 'pointermove', 'pointerup', 'pointercancel'].includes(type)
+ && options?.capture && typeof handler === 'function') (monitorPointerHandlers[type] ||= []).push(handler);
+ return add.call(this, type, handler, options);
+ };
+ });
await context.addInitScript(() => {
window.monitorEnabled = true;
window.feedback = [];
@@ -505,7 +515,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
try {
__wbPageMonitor.activatePreparedDispatch({ operationId: 'iframe-click-live', kind: 'click',
element: document.getElementById('agent'), navigationCandidate: true });
- document.getElementById('agent').click();
+ __wbPageMonitor.withPreparedDispatch('iframe-click-live', () => document.getElementById('agent').click());
return true;
} catch { return false; }
});
@@ -870,8 +880,10 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
await page.evaluate(() => {
feedback = []; deliver('page_monitor_prepare', { operationId: 'agent-tab', tool: 'press_keys', selector: '#human' });
deliver('page_monitor_dispatch', { operationId: 'agent-tab', kind: 'input', selector: '#human', fenceOnly: true });
- document.getElementById('human').setAttribute('data-webbrain-native-action', JSON.stringify({ ...lastMonitorResponse.guard, kind: 'input', sequence: 1 }));
});
+ const tabGuard = await page.evaluate(() => lastMonitorResponse.guard);
+ await page.evaluate(marker => document.getElementById('human').setAttribute('data-webbrain-native-action', marker),
+ createNativeActionMarker(tabGuard, 'input', 1));
await page.keyboard.press('Tab');
await page.waitForTimeout(80);
assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,
@@ -889,6 +901,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
assert.ok(input.some(event => event.kind === 'input' && event.source === 'user'));
assert.equal(JSON.stringify(input).includes('private-field-value'), false);
assert.equal(JSON.stringify(input).includes('password-never-recorded'), false);
+ await page.waitForTimeout(1100);
await page.evaluate(() => {
feedback = [];
@@ -908,6 +921,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
&& event.target === 'button#agent'), 'A later human click on the same target must survive attribution');
await page.locator('#human').click();
assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user' && event.target === 'button#human'));
+ await page.waitForTimeout(1100);
await page.evaluate(() => {
feedback = [];
@@ -920,6 +934,7 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
assert.equal(ownTyping.some(event => event.source !== 'agent'), false, JSON.stringify(ownTyping));
await page.locator('#human').click();
assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'));
+ await page.waitForTimeout(1100);
await page.evaluate(() => {
feedback = [];
deliver('page_monitor_prepare', { operationId: 'click-with-external-dom', tool: 'click', selector: '#agent' });
@@ -932,6 +947,96 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: property-only form changes invalidate the prepared page without exposing values`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const dense = document.createElement('div');
+ dense.innerHTML = `${'<span></span>'.repeat(650)}<input id="late-control">`;
+ document.body.append(dense);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1000 });
+ await page.evaluate(() => { feedback = []; document.getElementById('late-control').value = 'late-private-value'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1500 });
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('late-private-value'), false,
+ 'Controls beyond the layout-sampling budget still receive bounded state monitoring');
+
+ await page.evaluate(() => {
+ feedback = [];
+ document.getElementById('field').value = 'private-property-only-value';
+ document.getElementById('select').selectedIndex = 1;
+ const check = document.createElement('input'); check.type = 'checkbox'; check.id = 'property-check';
+ document.body.append(check); check.checked = true;
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'), null, { timeout: 1500 });
+ const observed = await page.evaluate(() => feedback);
+ assert.equal(JSON.stringify(observed).includes('private-property-only-value'), false);
+ assert.ok(observed.some(event => event.kind === 'dom' && event.source === 'page'));
+
+ const result = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'property-preflight', tool: 'click', selector: '#agent' });
+ document.getElementById('field').value = 'changed-during-preparation';
+ try { __wbPageMonitor.activatePreparedDispatch({ operationId: 'property-preflight', kind: 'click' }); return null; }
+ catch (error) { return error.code; }
+ });
+ assert.equal(result, 'page_feedback_pending', 'The final dispatch check must sample property-only changes');
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('changed-during-preparation'), false);
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: native markers reject replay, tampering and copied targets`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ const guard = await page.evaluate(() => {
+ deliver('page_monitor_prepare', { operationId: 'native-target', tool: 'type_text', selector: '#field' });
+ deliver('page_monitor_dispatch', { operationId: 'native-target', kind: 'input', selector: '#field', fenceOnly: true });
+ feedback = []; return lastMonitorResponse.guard;
+ });
+ const copiedMarker = createNativeActionMarker(guard, 'input', 1);
+ await page.evaluate(marker => document.getElementById('secret').setAttribute('data-webbrain-native-action', marker), copiedMarker);
+ await page.locator('#secret').fill('target-must-stay-bound');
+ let events = await page.evaluate(() => feedback);
+ assert.ok(events.some(event => event.kind === 'input' && event.source === 'user' && event.target === 'input#secret'),
+ 'A valid capability copied to another control cannot retarget the isolated operation');
+ assert.equal(JSON.stringify(events).includes('target-must-stay-bound'), false);
+
+ const marker = JSON.parse(createNativeActionMarker(guard, 'input', 1));
+ marker.sequence = 999;
+ await page.evaluate(value => {
+ feedback = [];
+ const field = document.getElementById('field');
+ field.setAttribute('data-webbrain-native-action', JSON.stringify(value));
+ field.dispatchEvent(new Event('input', { bubbles: true }));
+ }, marker);
+ events = await page.evaluate(() => feedback);
+ assert.ok(events.some(event => event.kind === 'input' && event.source === 'page'),
+ 'A modified sequence and script-generated event cannot be attributed to the agent');
+ assert.equal(events.some(event => event.source === 'agent'), false);
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: a multi-pointer gesture stays active until the last pointer is released`, async () => {
+ const { browser, page } = await fixture(engine, build, { capturePointerHandlers: true });
+ try {
+ const states = await page.evaluate(() => {
+ feedback = [];
+ const target = document.getElementById('human');
+ const emit = (type, pointerId, buttons) => {
+ const event = { type, pointerId, buttons, isTrusted: true, target, composedPath: () => [target] };
+ for (const listener of monitorPointerHandlers[type] || []) listener.call(document, event);
+ };
+ emit('pointerdown', 11, 1);
+ emit('pointerdown', 12, 1);
+ emit('pointerup', 11, 0);
+ const afterFirstRelease = feedback.at(-1)?.interacting;
+ emit('pointerup', 12, 0);
+ return { afterFirstRelease, afterLastRelease: feedback.at(-1)?.interacting };
+ });
+ assert.deepEqual(states, { afterFirstRelease: true, afterLastRelease: false });
+ } finally { await browser.close(); }
+ });
+
test(`${build}: DOM, window/container scrolling, cleanup and restored monitoring work`, async () => {
const { browser, page } = await fixture(engine, build);
try {
@@ -1305,12 +1410,15 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
feedback = []; return lastMonitorResponse.guard;
});
await page.locator('#field').focus();
+ let replayedMarker;
for (let i = 0; i < 3; i++) {
- await page.evaluate(({ guard, i }) => document.getElementById('field').setAttribute('data-webbrain-native-action',
- JSON.stringify({ ...guard, kind: 'input', sequence: i })), { guard, i });
+ replayedMarker = createNativeActionMarker(guard, 'input', i + 1);
+ await page.evaluate(marker => document.getElementById('field').setAttribute('data-webbrain-native-action', marker), replayedMarker);
await page.keyboard.insertText('a');
}
- assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false);
+ assert.equal((await page.evaluate(() => feedback)).some(event => event.source !== 'agent'), false,
+ JSON.stringify(await page.evaluate(() => feedback)));
+ await page.evaluate(marker => document.getElementById('field').setAttribute('data-webbrain-native-action', marker), replayedMarker);
await page.keyboard.insertText('human-private');
assert.ok((await page.evaluate(() => feedback)).some(event => event.source === 'user'));
assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('human-private'), false);
@@ -1621,8 +1729,8 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
deliver('page_monitor_dispatch', { operationId: 'clear-chord', kind: 'input', selector: '#field', fenceOnly: true });
feedback = []; return lastMonitorResponse.guard;
});
- const mark = sequence => page.evaluate(({ guard, sequence }) => document.getElementById('field')
- .setAttribute('data-webbrain-native-action', JSON.stringify({ ...guard, kind: 'input', sequence })), { guard, sequence });
+ const mark = sequence => page.evaluate(marker => document.getElementById('field')
+ .setAttribute('data-webbrain-native-action', marker), createNativeActionMarker(guard, 'input', sequence));
await mark(1); await page.keyboard.down('Control');
await mark(2); await page.keyboard.press('a'); await page.keyboard.up('Control');
await mark(3); await page.keyboard.press('Delete');