Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix page feedback for GET forms and stylesheets
Barack Sokullu2ed18ce9dff1
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex a1e3d1750..9ca01b5ec 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -110,6 +110,13 @@ function safeAction(tool, args = {}) {return { tool, ...target };}+function sameOriginPath(expected, actual) {+ try {+ const expectedUrl = new URL(expected), actualUrl = new URL(actual);+ return expectedUrl.origin === actualUrl.origin && expectedUrl.pathname === actualUrl.pathname;+ } catch { return false; }+}+export const pageFeedbackMethods = {async _beginPageFeedbackRun(tabId, kind) {this._pageFeedbackRuns ??= new Map();@@ -189,6 +196,7 @@ export const pageFeedbackMethods = {} catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }run.navigation = url ? { at: Date.now(), url, kind: 'click',frameId: feedback.navigationTarget === '_top' ? 0 : frameId,+ ...(feedback.navigationFormGet === true ? { formGet: true } : {}),operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;}return { accepted: true };@@ -234,9 +242,11 @@ export const pageFeedbackMethods = {const redirect = qualifiers.some(q => /redirect$/.test(q));const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect&& !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);+ const formGetDestination = type === 'committed' && details.transitionType === 'form_submit'+ && navigation?.formGet === true && navigation.frameId === frameId && sameOriginPath(navigation.url, details.url);const sameNavigation = navigation?.redirectChain? redirect && !!navigation.documentId && navigation.documentId === details.documentId- : (!!navigation?.url && navigation.url === details.url) || firstRedirect+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect || formGetDestination|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)diff --git a/src/chrome/src/content/agent-visual-indicator.js b/src/chrome/src/content/agent-visual-indicator.jsindex 00f9a3570..c20aeb782 100644--- a/src/chrome/src/content/agent-visual-indicator.js+++ b/src/chrome/src/content/agent-visual-indicator.js@@ -58,6 +58,7 @@if (document.getElementById('webbrain-agent-styles')) return;const style = document.createElement('style');style.id = 'webbrain-agent-styles';+ (window.__wbPageMonitorDecorations ??= new WeakSet()).add(style);style.textContent = `@keyframes webbrain-pulse {0% {diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex df58ae9bf..a0d43bd2f 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -464,7 +464,7 @@if (!root || roots.has(root) || !observer) return;roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,- attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby',+ attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby', 'rel', 'media','aria-required', 'aria-readonly', 'contenteditable', 'tabindex', 'onclick', 'required','aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget',@@ -548,6 +548,29 @@|| (root === document ? document.getElementById(previousFor) : null);return visibleControl(priorControl);}+ function stylesheetNode(node) {+ if (!(node instanceof Element)) return false;+ return node.tagName === 'STYLE' || (node.tagName === 'LINK'+ && (node.getAttribute('rel') || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet'));+ }+ function stylesheetMutation(record) {+ const target = record.target.nodeType === 1 ? record.target : record.target.parentElement;+ if (record.type === 'attributes') {+ return stylesheetNode(target) || (target?.tagName === 'LINK' && record.attributeName === 'rel'+ && (record.oldValue || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet')) ? [target] : [];+ }+ if (record.type === 'characterData') {+ const style = stylesheetNode(target) ? target : stylesheetNode(target?.parentElement) ? target.parentElement : null;+ return style ? [style] : [];+ }+ if (record.type !== 'childList') return [];+ const stylesheets = stylesheetNode(target) ? [target] : [];+ for (const node of [...record.addedNodes, ...record.removedNodes]) {+ const style = stylesheetNode(node) ? node : node.nodeType === 3 && stylesheetNode(node.parentElement) ? node.parentElement : null;+ if (style && !stylesheets.includes(style)) stylesheets.push(style);+ }+ return stylesheets;+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -592,6 +615,11 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;+ const changedStylesheets = stylesheetMutation(record);+ if (changedStylesheets.length) {+ if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);+ continue;+ }const editableTextMutation = editable(el) && (record.type === 'characterData'|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));if (ignored(el)) continue;@@ -750,17 +778,22 @@const path = event.composedPath();const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);+ const form = submitter?.form;+ const formMethod = submitter?.hasAttribute('formmethod') ? submitter.formMethod : form?.method;+ const navigationFormGet = !link && !!form && String(formMethod || 'get').toLowerCase() === 'get';const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();let navigationUrl = '';try {- const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');+ const rawUrl = link?.href || (form ? (submitter.hasAttribute('formaction') ? submitter.formAction : form.action) : '');const destination = new URL(rawUrl, document.baseURI);+ if (navigationFormGet) { destination.search = ''; destination.hash = ''; }if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password&& destination.href.length <= 2000) navigationUrl = destination.href;} catch { /* Non-web and malformed targets are not navigation correlations. */ }send({ kind: 'activity', source: 'agent', operation: 'click',...(navigationUrl ? { navigationUrl } : {}),+ ...(navigationUrl && navigationFormGet ? { navigationFormGet: true } : {}),...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)? { navigationTarget: '_top' } : {}) });}diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex a1e3d1750..9ca01b5ec 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -110,6 +110,13 @@ function safeAction(tool, args = {}) {return { tool, ...target };}+function sameOriginPath(expected, actual) {+ try {+ const expectedUrl = new URL(expected), actualUrl = new URL(actual);+ return expectedUrl.origin === actualUrl.origin && expectedUrl.pathname === actualUrl.pathname;+ } catch { return false; }+}+export const pageFeedbackMethods = {async _beginPageFeedbackRun(tabId, kind) {this._pageFeedbackRuns ??= new Map();@@ -189,6 +196,7 @@ export const pageFeedbackMethods = {} catch { /* Clicks without a safe, concrete destination cannot correlate navigation. */ }run.navigation = url ? { at: Date.now(), url, kind: 'click',frameId: feedback.navigationTarget === '_top' ? 0 : frameId,+ ...(feedback.navigationFormGet === true ? { formGet: true } : {}),operationId: dispatchOwners.get(tabId)?.operationId || '' } : null;}return { accepted: true };@@ -234,9 +242,11 @@ export const pageFeedbackMethods = {const redirect = qualifiers.some(q => /redirect$/.test(q));const firstRedirect = type === 'committed' && !navigation?.redirectChain && redirect&& !!navigation?.url && ['click', 'navigate'].includes(navigation.kind);+ const formGetDestination = type === 'committed' && details.transitionType === 'form_submit'+ && navigation?.formGet === true && navigation.frameId === frameId && sameOriginPath(navigation.url, details.url);const sameNavigation = navigation?.redirectChain? redirect && !!navigation.documentId && navigation.documentId === details.documentId- : (!!navigation?.url && navigation.url === details.url) || firstRedirect+ : (!!navigation?.url && navigation.url === details.url) || firstRedirect || formGetDestination|| (navigation?.kind === 'navigate' && navigation.history && qualifiers.includes('forward_back'));const agentNavigation = !explicit && navigation && navigation.frameId === frameId && Date.now() - navigation.at < 10000&& (!qualifiers.includes('forward_back') || navigation.history)diff --git a/src/firefox/src/content/agent-visual-indicator.js b/src/firefox/src/content/agent-visual-indicator.jsindex 2281d5d46..dacd74186 100644--- a/src/firefox/src/content/agent-visual-indicator.js+++ b/src/firefox/src/content/agent-visual-indicator.js@@ -58,6 +58,7 @@if (document.getElementById('webbrain-agent-styles')) return;const style = document.createElement('style');style.id = 'webbrain-agent-styles';+ (window.__wbPageMonitorDecorations ??= new WeakSet()).add(style);style.textContent = `@keyframes webbrain-pulse {0% {diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex df58ae9bf..a0d43bd2f 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -464,7 +464,7 @@if (!root || roots.has(root) || !observer) return;roots.add(root);observer.observe(root, { childList: true, subtree: true, characterData: true, attributes: true, attributeOldValue: true,- attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby',+ attributeFilter: ['role', 'aria-label', 'id', 'for', 'form', 'name', 'placeholder', 'title', 'alt', 'aria-labelledby', 'rel', 'media','aria-required', 'aria-readonly', 'contenteditable', 'tabindex', 'onclick', 'required','aria-expanded', 'aria-selected', 'aria-checked', 'aria-pressed', 'aria-disabled','type', 'href', 'target', 'download', 'action', 'method', 'formaction', 'formmethod', 'formtarget',@@ -548,6 +548,29 @@|| (root === document ? document.getElementById(previousFor) : null);return visibleControl(priorControl);}+ function stylesheetNode(node) {+ if (!(node instanceof Element)) return false;+ return node.tagName === 'STYLE' || (node.tagName === 'LINK'+ && (node.getAttribute('rel') || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet'));+ }+ function stylesheetMutation(record) {+ const target = record.target.nodeType === 1 ? record.target : record.target.parentElement;+ if (record.type === 'attributes') {+ return stylesheetNode(target) || (target?.tagName === 'LINK' && record.attributeName === 'rel'+ && (record.oldValue || '').split(/\s+/).some(token => token.toLowerCase() === 'stylesheet')) ? [target] : [];+ }+ if (record.type === 'characterData') {+ const style = stylesheetNode(target) ? target : stylesheetNode(target?.parentElement) ? target.parentElement : null;+ return style ? [style] : [];+ }+ if (record.type !== 'childList') return [];+ const stylesheets = stylesheetNode(target) ? [target] : [];+ for (const node of [...record.addedNodes, ...record.removedNodes]) {+ const style = stylesheetNode(node) ? node : node.nodeType === 3 && stylesheetNode(node.parentElement) ? node.parentElement : null;+ if (style && !stylesheets.includes(style)) stylesheets.push(style);+ }+ return stylesheets;+ }function onMutations(records) {if (!active) return;let changed = false, source = 'page', target = '';@@ -592,6 +615,11 @@for (const record of changes) {const el = record.target.nodeType === 1 ? record.target: record.target.host || record.target.parentElement || record.target.getRootNode?.().host;+ const changedStylesheets = stylesheetMutation(record);+ if (changedStylesheets.length) {+ if (changedStylesheets.some(style => !decorations.has(style))) noteChange(document.documentElement, record.agentUserAt);+ continue;+ }const editableTextMutation = editable(el) && (record.type === 'characterData'|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));if (ignored(el)) continue;@@ -750,17 +778,22 @@const path = event.composedPath();const link = path.find(node => node instanceof Element && node.matches('a[href],area[href]'));const submitter = path.find(node => node instanceof Element && node.matches('button,input[type="submit"],input[type="image"]') && node.form);+ const form = submitter?.form;+ const formMethod = submitter?.hasAttribute('formmethod') ? submitter.formMethod : form?.method;+ const navigationFormGet = !link && !!form && String(formMethod || 'get').toLowerCase() === 'get';const navigationTarget = (link?.getAttribute('target') || el?.form?.getAttribute('target')|| (link && document.querySelector('base[target]')?.getAttribute('target')) || '').toLowerCase();let navigationUrl = '';try {- const rawUrl = link?.href || (submitter?.form ? submitter.formAction || submitter.form.action : '');+ const rawUrl = link?.href || (form ? (submitter.hasAttribute('formaction') ? submitter.formAction : form.action) : '');const destination = new URL(rawUrl, document.baseURI);+ if (navigationFormGet) { destination.search = ''; destination.hash = ''; }if (['http:', 'https:'].includes(destination.protocol) && !destination.username && !destination.password&& destination.href.length <= 2000) navigationUrl = destination.href;} catch { /* Non-web and malformed targets are not navigation correlations. */ }send({ kind: 'activity', source: 'agent', operation: 'click',...(navigationUrl ? { navigationUrl } : {}),+ ...(navigationUrl && navigationFormGet ? { navigationFormGet: true } : {}),...(navigationTarget === '_top' || (navigationTarget === '_parent' && window.parent === window.top)? { navigationTarget: '_top' } : {}) });}diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex 75434dcb8..6319d5bf3 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -260,7 +260,7 @@ for (const build of ['chrome', 'firefox']) {const agent = setup(Agent), tab = nextTab++;await agent._claimRunEntry(tab, 'interactive');try {- const binding = bind(agent, tab);+ let binding = bind(agent, tab);for (let i = 0; i < 100; i++) binding.send({ kind: 'dom', source: 'page', target: `button#${i}` });assert.equal(agent._pageFeedbackRuns.get(tab).events.size, 32);for (const [type, qualifiers] of [['history', []], ['fragment', []], ['committed', ['forward_back']]]) {@@ -753,6 +753,31 @@ for (const build of ['chrome', 'firefox']) {} finally { agent._releaseRunEntry(tab); }});+ test(`${build}: GET form submission correlates serialized query by its private base action`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ await agent._claimRunEntry(tab, 'interactive');+ try {+ await agent._beginPageFeedbackRun(tab, 'interactive');+ let binding = bind(agent, tab);+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/search',+ navigationFormGet: true });+ const armed = agent._pageFeedbackRuns.get(tab).navigation;+ assert.equal(armed.url, 'https://example.com/search', 'The correlation marker contains only the action path');+ assert.equal(armed.formGet, true);+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'form-result',+ url: 'https://example.com/search?q=private-form-value', transitionType: 'form_submit' }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), false, 'A GET form submission can append its successful controls to the query');+ assert.equal(agent._pageFeedbackRuns.get(tab).navigation, null);++ binding = bind(agent, tab, 0, 'form-result', 'form-result-token');+ binding.send({ kind: 'activity', source: 'agent', operation: 'click', navigationUrl: 'https://example.com/search',+ navigationFormGet: true });+ agent.observePageNavigation({ tabId: tab, frameId: 0, documentId: 'wrong-form-result',+ url: 'https://example.com/other?q=value', transitionType: 'form_submit' }, 'committed');+ assert.equal(agent._hasPendingPageFeedback(tab), true, 'Form-specific correlation still requires the same origin and action path');+ } finally { agent._finishPageFeedbackRun(tab); agent._releaseRunEntry(tab); }+ });+test(`${build}: first redirect of an armed URL navigation stays agent-attributed`, async () => {const agent = setup(Agent), tab = nextTab++;await agent._claimRunEntry(tab, 'interactive');diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex d6e881465..32345079f 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -1979,6 +1979,62 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: GET submitter activity omits serialized form values from its navigation marker`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const form = document.createElement('form'); form.id = 'get-submit-form';+ form.method = 'get'; form.action = '/search?fixed=private-action-value#result';+ form.addEventListener('submit', event => event.preventDefault());+ const input = document.createElement('input'); input.name = 'query'; input.value = 'private-form-value';+ const button = document.createElement('button'); button.id = 'get-submit'; button.type = 'submit'; button.textContent = 'Search';+ form.append(input, button); document.body.append(form);+ });+ await page.waitForTimeout(250);+ await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'get-form-submit', tool: 'click', selector: '#get-submit' });+ deliver('page_monitor_dispatch', { operationId: 'get-form-submit', kind: 'click', selector: '#get-submit' });+ });+ await page.locator('#get-submit').click();+ await page.waitForTimeout(150);+ const events = await page.evaluate(() => feedback);+ const marker = events.find(event => event.navigationFormGet === true);+ assert.ok(marker, JSON.stringify(events));+ assert.equal(marker.navigationUrl, 'https://monitor.test/search');+ assert.equal(JSON.stringify(events).includes('private-form-value'), false);+ assert.equal(JSON.stringify(events).includes('private-action-value'), false);+ } finally { await browser.close(); }+ });++ test(`${build}: stylesheet text changes invalidate prepared actions without exposing CSS`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const style = document.createElement('style'); style.id = 'page-stylesheet';+ style.textContent = '#stylesheet-target { visibility: visible; }';+ const target = document.createElement('button'); target.id = 'stylesheet-target'; target.textContent = 'Checkout';+ document.head.append(style); document.body.append(target);+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'));+ await page.waitForTimeout(180);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'stylesheet-change', tool: 'click', selector: '#stylesheet-target' });+ deliver('page_monitor_dispatch', { operationId: 'stylesheet-change', kind: 'click', selector: '#stylesheet-target' });+ return lastMonitorResponse.guard;+ });+ await page.locator('#page-stylesheet').evaluate(style => { style.textContent = '#stylesheet-target { visibility: hidden; opacity: 0; }'; });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom' && event.target === 'html'));+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'focus' }), cancelable: true,+ })), guard);+ assert.equal(accepted, false, 'A stylesheet change must stale a prepared action even without layout movement');+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('visibility: hidden'), false,+ 'Stylesheet contents must not be copied into feedback');+ } finally { await browser.close(); }+ });+test(`${build}: submitter form association changes invalidate a prepared click`, async () => {const { browser, page } = await fixture(engine, build);try {