Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix page feedback for GET forms and stylesheets

Barack Sokullu2ed18ce9dff1
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index a1e3d1750..9ca01b5ec 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -110,6 +110,13 @@ function safeAction(tool, args = {}) {
return { tool, ...target };
}
+function sameOriginPath(expected, actual) {
+ try {
+ const expectedUrl = new URL(expected), actualUrl = new URL(actual);
+ return expectedUrl.origin === actualUrl.origin && expectedUrl.pathname === actualUrl.pathname;
+ } catch { return false; }
+}
+
export const pageFeedbackMethods = {
async _beginPageFeedbackRun(tabId, kind) {
this._pageFeedbackRuns ??= new Map();
@@ -189,6 +196,7 @@ export const pageFeedbackMethods = {
} catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }
run.navigation = url ? { at: Date.now(), url, kind: 'click',
frameId: feedback.navigationTarget === '_top' ? 0 : frameId,
+ ...(feedback.navigationFormGet === true ? { formGet: true } : {}),
operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;
}
return { accepted: true };
@@ -234,9 +242,11 @@ export const pageFeedbackMethods = {
const redirect = qualifiers.some(q => /redirect$/.test(q));
const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect
&& !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);
+ const formGetDestination = type === 'committed' && details.transitionType === 'form_submit'
+ && navigation?.formGet === true && navigation.frameId === frameId && sameOriginPath(navigation.url, details.url);
const sameNavigation = navigation?.redirectChain
? redirect && !!navigation.documentId && navigation.documentId === details.documentId
- : (!!navigation?.url && navigation.url === details.url) || firstRedirect
+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect || formGetDestination
|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
diff --git a/src/chrome/src/content/agent-visual-indicator.js b/src/chrome/src/content/agent-visual-indicator.js
index 00f9a3570..c20aeb782 100644
--- a/src/chrome/src/content/agent-visual-indicator.js
+++ b/src/chrome/src/content/agent-visual-indicator.js
@@ -58,6 +58,7 @@
if (document.getElementById('webbrain-agent-styles')) return;
const style = document.createElement('style');
style.id = 'webbrain-agent-styles';
+ (window.__wbPageMonitorDecorations ??= new WeakSet()).add(style);
style.textContent = `
@keyframes webbrain-pulse {
0% {
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index df58ae9bf..a0d43bd2f 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -464,7 +464,7 @@
if (!root || roots.has(root) || !observer) return;
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
- attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby',
+ attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby', 'rel', 'media',
'aria-required', 'aria-readonly', 'contenteditable', 'tabindex', 'onclick', 'required',
'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget',
@@ -548,6 +548,29 @@
|| (root === document ? document.getElementById(previousFor) : null);
return visibleControl(priorControl);
}
+ function stylesheetNode(node) {
+ if (!(node instanceof Element)) return false;
+ return node.tagName === 'STYLE' || (node.tagName === 'LINK'
+ && (node.getAttribute('rel') || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet'));
+ }
+ function stylesheetMutation(record) {
+ const target = record.target.nodeType === 1 ? record.target : record.target.parentElement;
+ if (record.type === 'attributes') {
+ return stylesheetNode(target) || (target?.tagName === 'LINK' && record.attributeName === 'rel'
+ && (record.oldValue || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet')) ? [target] : [];
+ }
+ if (record.type === 'characterData') {
+ const style = stylesheetNode(target) ? target : stylesheetNode(target?.parentElement) ? target.parentElement : null;
+ return style ? [style] : [];
+ }
+ if (record.type !== 'childList') return [];
+ const stylesheets = stylesheetNode(target) ? [target] : [];
+ for (const node of [...record.addedNodes, ...record.removedNodes]) {
+ const style = stylesheetNode(node) ? node : node.nodeType === 3 && stylesheetNode(node.parentElement) ? node.parentElement : null;
+ if (style && !stylesheets.includes(style)) stylesheets.push(style);
+ }
+ return stylesheets;
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -592,6 +615,11 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
+ const changedStylesheets = stylesheetMutation(record);
+ if (changedStylesheets.length) {
+ if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);
+ continue;
+ }
const editableTextMutation = editable(el) && (record.type === 'characterData'
|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));
if (ignored(el)) continue;
@@ -750,17 +778,22 @@
const path = event.composedPath();
const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));
const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);
+ const form = submitter?.form;
+ const formMethod = submitter?.hasAttribute('formmethod') ? submitter.formMethod : form?.method;
+ const navigationFormGet = !link && !!form && String(formMethod || 'get').toLowerCase() === 'get';
const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')
|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();
let navigationUrl = '';
try {
- const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');
+ const rawUrl = link?.href || (form ? (submitter.hasAttribute('formaction') ? submitter.formAction : form.action) : '');
const destination = new URL(rawUrl, document.baseURI);
+ if (navigationFormGet) { destination.search = ''; destination.hash = ''; }
if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password
&& destination.href.length <= 2000) navigationUrl = destination.href;
} catch { /* Non-web and malformed targets are not navigation correlations. */ }
send({ kind: 'activity', source: 'agent', operation: 'click',
...(navigationUrl ? { navigationUrl } : {}),
+ ...(navigationUrl && navigationFormGet ? { navigationFormGet: true } : {}),
...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)
? { navigationTarget: '_top' } : {}) });
}
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index a1e3d1750..9ca01b5ec 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -110,6 +110,13 @@ function safeAction(tool, args = {}) {
return { tool, ...target };
}
+function sameOriginPath(expected, actual) {
+ try {
+ const expectedUrl = new URL(expected), actualUrl = new URL(actual);
+ return expectedUrl.origin === actualUrl.origin && expectedUrl.pathname === actualUrl.pathname;
+ } catch { return false; }
+}
+
export const pageFeedbackMethods = {
async _beginPageFeedbackRun(tabId, kind) {
this._pageFeedbackRuns ??= new Map();
@@ -189,6 +196,7 @@ export const pageFeedbackMethods = {
} catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }
run.navigation = url ? { at: Date.now(), url, kind: 'click',
frameId: feedback.navigationTarget === '_top' ? 0 : frameId,
+ ...(feedback.navigationFormGet === true ? { formGet: true } : {}),
operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;
}
return { accepted: true };
@@ -234,9 +242,11 @@ export const pageFeedbackMethods = {
const redirect = qualifiers.some(q => /redirect$/.test(q));
const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect
&& !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);
+ const formGetDestination = type === 'committed' && details.transitionType === 'form_submit'
+ && navigation?.formGet === true && navigation.frameId === frameId && sameOriginPath(navigation.url, details.url);
const sameNavigation = navigation?.redirectChain
? redirect && !!navigation.documentId && navigation.documentId === details.documentId
- : (!!navigation?.url && navigation.url === details.url) || firstRedirect
+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect || formGetDestination
|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));
const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000
&& (!qualifiers.includes('forward_back') || navigation.history)
diff --git a/src/firefox/src/content/agent-visual-indicator.js b/src/firefox/src/content/agent-visual-indicator.js
index 2281d5d46..dacd74186 100644
--- a/src/firefox/src/content/agent-visual-indicator.js
+++ b/src/firefox/src/content/agent-visual-indicator.js
@@ -58,6 +58,7 @@
if (document.getElementById('webbrain-agent-styles')) return;
const style = document.createElement('style');
style.id = 'webbrain-agent-styles';
+ (window.__wbPageMonitorDecorations ??= new WeakSet()).add(style);
style.textContent = `
@keyframes webbrain-pulse {
0% {
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index df58ae9bf..a0d43bd2f 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -464,7 +464,7 @@
if (!root || roots.has(root) || !observer) return;
roots.add(root);
observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,
- attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby',
+ attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby', 'rel', 'media',
'aria-required', 'aria-readonly', 'contenteditable', 'tabindex', 'onclick', 'required',
'aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled',
'type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget',
@@ -548,6 +548,29 @@
|| (root === document ? document.getElementById(previousFor) : null);
return visibleControl(priorControl);
}
+ function stylesheetNode(node) {
+ if (!(node instanceof Element)) return false;
+ return node.tagName === 'STYLE' || (node.tagName === 'LINK'
+ && (node.getAttribute('rel') || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet'));
+ }
+ function stylesheetMutation(record) {
+ const target = record.target.nodeType === 1 ? record.target : record.target.parentElement;
+ if (record.type === 'attributes') {
+ return stylesheetNode(target) || (target?.tagName === 'LINK' && record.attributeName === 'rel'
+ && (record.oldValue || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet')) ? [target] : [];
+ }
+ if (record.type === 'characterData') {
+ const style = stylesheetNode(target) ? target : stylesheetNode(target?.parentElement) ? target.parentElement : null;
+ return style ? [style] : [];
+ }
+ if (record.type !== 'childList') return [];
+ const stylesheets = stylesheetNode(target) ? [target] : [];
+ for (const node of [...record.addedNodes, ...record.removedNodes]) {
+ const style = stylesheetNode(node) ? node : node.nodeType === 3 && stylesheetNode(node.parentElement) ? node.parentElement : null;
+ if (style && !stylesheets.includes(style)) stylesheets.push(style);
+ }
+ return stylesheets;
+ }
function onMutations(records) {
if (!active) return;
let changed = false, source = 'page', target = '';
@@ -592,6 +615,11 @@
for (const record of changes) {
const el = record.target.nodeType === 1 ? record.target
: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;
+ const changedStylesheets = stylesheetMutation(record);
+ if (changedStylesheets.length) {
+ if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);
+ continue;
+ }
const editableTextMutation = editable(el) && (record.type === 'characterData'
|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));
if (ignored(el)) continue;
@@ -750,17 +778,22 @@
const path = event.composedPath();
const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));
const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);
+ const form = submitter?.form;
+ const formMethod = submitter?.hasAttribute('formmethod') ? submitter.formMethod : form?.method;
+ const navigationFormGet = !link && !!form && String(formMethod || 'get').toLowerCase() === 'get';
const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')
|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();
let navigationUrl = '';
try {
- const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');
+ const rawUrl = link?.href || (form ? (submitter.hasAttribute('formaction') ? submitter.formAction : form.action) : '');
const destination = new URL(rawUrl, document.baseURI);
+ if (navigationFormGet) { destination.search = ''; destination.hash = ''; }
if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password
&& destination.href.length <= 2000) navigationUrl = destination.href;
} catch { /* Non-web and malformed targets are not navigation correlations. */ }
send({ kind: 'activity', source: 'agent', operation: 'click',
...(navigationUrl ? { navigationUrl } : {}),
+ ...(navigationUrl && navigationFormGet ? { navigationFormGet: true } : {}),
...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)
? { navigationTarget: '_top' } : {}) });
}
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index 75434dcb8..6319d5bf3 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -260,7 +260,7 @@ for (const build of ['chrome', 'firefox']) {
const agent = setup(Agent), tab = nextTab++;
await agent._claimRunEntry(tab, 'interactive');
try {
- const binding = bind(agent, tab);
+ let binding = bind(agent, tab);
for (let i = 0; i < 100; i++) binding.send({ kind: 'dom', source: 'page', target: `button#${i}` });
assert.equal(agent._pageFeedbackRuns.get(tab).events.size, 32);
for (const [type, qualifiers] of [['history', []], ['fragment', []], ['committed', ['forward_back']]]) {
@@ -753,6 +753,31 @@ for (const build of ['chrome', 'firefox']) {
} finally { agent._releaseRunEntry(tab); }
});
+ test(`${build}: GET form submission correlates serialized query by its private base action`, async () => {
+ const agent = setup(Agent), tab = nextTab++;
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ await agent._beginPageFeedbackRun(tab, 'interactive');
+ let binding = bind(agent, tab);
+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/search',
+ navigationFormGet: true });
+ const armed = agent._pageFeedbackRuns.get(tab).navigation;
+ assert.equal(armed.url, 'https://example.com/search', 'The correlation marker contains only the action path');
+ assert.equal(armed.formGet, true);
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'form-result',
+ url: 'https://example.com/search?q=private-form-value', transitionType: 'form_submit' }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'A GET form submission can append its successful controls to the query');
+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null);
+
+ binding = bind(agent, tab, 0, 'form-result', 'form-result-token');
+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/search',
+ navigationFormGet: true });
+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'wrong-form-result',
+ url: 'https://example.com/other?q=value', transitionType: 'form_submit' }, 'committed');
+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'Form-specific correlation still requires the same origin and action path');
+ } finally { agent._finishPageFeedbackRun(tab); agent._releaseRunEntry(tab); }
+ });
+
test(`${build}: first redirect of an armed URL navigation stays agent-attributed`, async () => {
const agent = setup(Agent), tab = nextTab++;
await agent._claimRunEntry(tab, 'interactive');
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index d6e881465..32345079f 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -1979,6 +1979,62 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: GET submitter activity omits serialized form values from its navigation marker`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const form = document.createElement('form'); form.id = 'get-submit-form';
+ form.method = 'get'; form.action = '/search?fixed=private-action-value#result';
+ form.addEventListener('submit', event => event.preventDefault());
+ const input = document.createElement('input'); input.name = 'query'; input.value = 'private-form-value';
+ const button = document.createElement('button'); button.id = 'get-submit'; button.type = 'submit'; button.textContent = 'Search';
+ form.append(input, button); document.body.append(form);
+ });
+ await page.waitForTimeout(250);
+ await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'get-form-submit', tool: 'click', selector: '#get-submit' });
+ deliver('page_monitor_dispatch', { operationId: 'get-form-submit', kind: 'click', selector: '#get-submit' });
+ });
+ await page.locator('#get-submit').click();
+ await page.waitForTimeout(150);
+ const events = await page.evaluate(() => feedback);
+ const marker = events.find(event => event.navigationFormGet === true);
+ assert.ok(marker, JSON.stringify(events));
+ assert.equal(marker.navigationUrl, 'https://monitor.test/search');
+ assert.equal(JSON.stringify(events).includes('private-form-value'), false);
+ assert.equal(JSON.stringify(events).includes('private-action-value'), false);
+ } finally { await browser.close(); }
+ });
+
+ test(`${build}: stylesheet text changes invalidate prepared actions without exposing CSS`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const style = document.createElement('style'); style.id = 'page-stylesheet';
+ style.textContent = '#stylesheet-target { visibility: visible; }';
+ const target = document.createElement('button'); target.id = 'stylesheet-target'; target.textContent = 'Checkout';
+ document.head.append(style); document.body.append(target);
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));
+ await page.waitForTimeout(180);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'stylesheet-change', tool: 'click', selector: '#stylesheet-target' });
+ deliver('page_monitor_dispatch', { operationId: 'stylesheet-change', kind: 'click', selector: '#stylesheet-target' });
+ return lastMonitorResponse.guard;
+ });
+ await page.locator('#page-stylesheet').evaluate(style => { style.textContent = '#stylesheet-target { visibility: hidden; opacity: 0; }'; });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'html'));
+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'focus' }), cancelable: true,
+ })), guard);
+ assert.equal(accepted, false, 'A stylesheet change must stale a prepared action even without layout movement');
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('visibility: hidden'), false,
+ 'Stylesheet contents must not be copied into feedback');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: submitter form association changes invalidate a prepared click`, async () => {
const { browser, page } = await fixture(engine, build);
try {