Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): element-derived refresh locator for focused proofs
Emre Sokullu31b3bfcc3d44
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 45ba13f19..ccf185475 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -20324,6 +20324,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dvalueSha256: String(response.valueSha256).toLowerCase(),verified: response.verified === true,fieldMeta: response.fieldMeta || null,+ // Element-derived revalidation locator (unique, content-checked);+ // consumed for focused proofs whose focus moves before submit.+ stableSelector: typeof response.stableSelector === 'string' && response.stableSelector.trim()+ ? response.stableSelector.trim()+ : null,};} catch {return null;@@ -20405,10 +20410,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dambiguous: false,focusedProof: true,focusedKind: kind,- // Stable pre-submit revalidation locator (focus moves before submit,- // so {focused:true} would digest the wrong field). The edit-file page- // owns a single file editor; the commit-message input has a stable id.- refreshSelector: kind === 'editor' ? '[contenteditable="true"]' : '#commit-message-input',+ // Pre-submit revalidation locator, filled from the write-time live+ // digest's element-derived stableSelector when the replacement record+ // is minted. Stays null until then: refresh drops locator-less proofs+ // fail-closed instead of probing a hard-coded kind selector that may+ // resolve to a different element (e.g. an aria-labelled textarea+ // editor is not [contenteditable]).+ refreshSelector: null,};}@@ -20430,6 +20438,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreadbackLength: readback.valueLength,readbackSha256: readback.valueSha256,} : {}),+ // Bind the element-derived revalidation locator for focused proofs so+ // pre-submit refresh re-reads the verified element itself (uniqueness+ // already content-checked). Null when the element offered nothing+ // unique β refresh then drops the proof fail-closed.+ ...(target?.focusedProof === true ? {+ refreshSelector: (readback && typeof readback.stableSelector === 'string'+ && readback.stableSelector.trim()) || null,+ } : {}),verifiedAt: Date.now(),// Bind the proof to the task that verified it: a later run in the same// tab must verify its own writes instead of reusing a prior task's@@ -20448,15 +20464,20 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// Focused proofs are bound at write time to the then-focused field's// live digest, but focus moves before submit (editor β commit dialog β// commit-message input), so re-digesting via {focused:true} would- // compare the wrong field. Revalidate via the stable refreshSelector- // captured at mint instead; a mismatch or missing element deletes the- // proof fail-closed. The byte-exact raw-blob check post-commit stays as- // backstop for the file; the commit message has no post-commit check,- // so its stable-id revalidation is the gate.+ // compare the wrong field. Revalidate via the element-derived+ // refreshSelector bound at mint (uniqueness content-checked there). A+ // proof with no locator β or any mismatch, kind drift, or missing+ // element β is dropped fail-closed: probing a hard-coded kind selector+ // instead could re-read a different element (an aria-labelled textarea+ // editor is not [contenteditable]) and bless stale bytes.if (record?.focusedProof === true) {- const refreshSelector = typeof record.refreshSelector === 'string' && record.refreshSelector.trim()+ const refreshSelector = typeof record.refreshSelector === 'string'? record.refreshSelector.trim()- : (record.focusedKind === 'editor' ? '[contenteditable="true"]' : '#commit-message-input');+ : '';+ if (!refreshSelector) {+ records.delete(key);+ continue;+ }let current = null;try {current = await this._textMutationValueDigest(tabId, {@@ -31913,6 +31934,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dname: prepared.name,contentEditable: prepared.contentEditable === true,},+ // Full live metadata of the verified element: lets downstream+ // guards classify editors identified by accessible label rather+ // than contentEditable (e.g. textarea-backed editors).+ fieldMeta: verification.fieldMeta || null,...(warning ? { warning } : {}),};} catch (error) {diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex febd584d0..675126134 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -4039,6 +4039,51 @@} catch { return null; }}+ // Derive a document-unique selector for a verified field so the agent can+ // re-digest the SAME element later (e.g. pre-submit proof refresh after+ // focus moved on). Each candidate is accepted only when it resolves back+ // to exactly this element β never a hard-coded guess that could match a+ // different field. Returns null when nothing identifies it uniquely.+ function _stableFieldSelector(el) {+ try {+ if (!el || el.nodeType !== 1 || !el.isConnected) return null;+ const resolvesUniquelyToEl = (selector) => {+ try {+ if (!selector) return false;+ const matches = document.querySelectorAll(selector);+ return matches.length === 1 && matches[0] === el;+ } catch { return false; }+ };+ const quoted = (value) => String(value ?? '').replace(/\\/g, '\\\\').replace(/"/g, '\\"');+ const elId = el.id || null;+ if (elId) {+ let selector = null;+ try {+ selector = (window.CSS && typeof window.CSS.escape === 'function')+ ? `#${window.CSS.escape(elId)}`+ : `#${quoted(elId)}`;+ } catch { selector = null; }+ if (selector && resolvesUniquelyToEl(selector)) return selector;+ }+ const tag = String(el.tagName || '').toLowerCase();+ const name = typeof el.getAttribute === 'function' ? el.getAttribute('name') : null;+ if (tag && name) {+ const selector = `${tag}[name="${quoted(name)}"]`;+ if (resolvesUniquelyToEl(selector)) return selector;+ }+ const ariaLabel = typeof el.getAttribute === 'function' ? el.getAttribute('aria-label') : null;+ if (tag && ariaLabel) {+ const selector = `${tag}[aria-label="${quoted(ariaLabel)}"]`;+ if (resolvesUniquelyToEl(selector)) return selector;+ }+ if (el.isContentEditable) {+ const selector = '[contenteditable="true"]';+ if (resolvesUniquelyToEl(selector)) return selector;+ }+ return null;+ } catch { return null; }+ }+function _richTextToolbarContextForElement(el) {try {if (!el || !el.isConnected) return { toolbarContext: false, toolbarRegionRef: '', toolbarRegionKey: '' };@@ -4423,6 +4468,7 @@return {success: true,verified,+ fieldMeta: _fieldMeta(el),rect: {x: Math.round(rect.x),y: Math.round(rect.y),@@ -6625,6 +6671,7 @@valueLength: value.length,valueSha256,fieldMeta: _fieldMeta(el),+ stableSelector: _stableFieldSelector(el),};} catch (error) {return { success: false, error: error && error.message || String(error) };diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex df3a162b0..a83920fb9 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -17895,6 +17895,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dvalueSha256: String(response.valueSha256).toLowerCase(),verified: response.verified === true,fieldMeta: response.fieldMeta || null,+ // Element-derived revalidation locator (unique, content-checked);+ // consumed for focused proofs whose focus moves before submit.+ stableSelector: typeof response.stableSelector === 'string' && response.stableSelector.trim()+ ? response.stableSelector.trim()+ : null,};} catch {return null;@@ -17976,10 +17981,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dambiguous: false,focusedProof: true,focusedKind: kind,- // Stable pre-submit revalidation locator (focus moves before submit,- // so {focused:true} would digest the wrong field). The edit-file page- // owns a single file editor; the commit-message input has a stable id.- refreshSelector: kind === 'editor' ? '[contenteditable="true"]' : '#commit-message-input',+ // Pre-submit revalidation locator, filled from the write-time live+ // digest's element-derived stableSelector when the replacement record+ // is minted. Stays null until then: refresh drops locator-less proofs+ // fail-closed instead of probing a hard-coded kind selector that may+ // resolve to a different element (e.g. an aria-labelled textarea+ // editor is not [contenteditable]).+ refreshSelector: null,};}@@ -18001,6 +18009,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreadbackLength: readback.valueLength,readbackSha256: readback.valueSha256,} : {}),+ // Bind the element-derived revalidation locator for focused proofs so+ // pre-submit refresh re-reads the verified element itself (uniqueness+ // already content-checked). Null when the element offered nothing+ // unique β refresh then drops the proof fail-closed.+ ...(target?.focusedProof === true ? {+ refreshSelector: (readback && typeof readback.stableSelector === 'string'+ && readback.stableSelector.trim()) || null,+ } : {}),verifiedAt: Date.now(),// Bind the proof to the task that verified it: a later run in the same// tab must verify its own writes instead of reusing a prior task's@@ -18019,15 +18035,20 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// Focused proofs are bound at write time to the then-focused field's// live digest, but focus moves before submit (editor β commit dialog β// commit-message input), so re-digesting via {focused:true} would- // compare the wrong field. Revalidate via the stable refreshSelector- // captured at mint instead; a mismatch or missing element deletes the- // proof fail-closed. The byte-exact raw-blob check post-commit stays as- // backstop for the file; the commit message has no post-commit check,- // so its stable-id revalidation is the gate.+ // compare the wrong field. Revalidate via the element-derived+ // refreshSelector bound at mint (uniqueness content-checked there). A+ // proof with no locator β or any mismatch, kind drift, or missing+ // element β is dropped fail-closed: probing a hard-coded kind selector+ // instead could re-read a different element (an aria-labelled textarea+ // editor is not [contenteditable]) and bless stale bytes.if (record?.focusedProof === true) {- const refreshSelector = typeof record.refreshSelector === 'string' && record.refreshSelector.trim()+ const refreshSelector = typeof record.refreshSelector === 'string'? record.refreshSelector.trim()- : (record.focusedKind === 'editor' ? '[contenteditable="true"]' : '#commit-message-input');+ : '';+ if (!refreshSelector) {+ records.delete(key);+ continue;+ }let current = null;try {current = await this._textMutationValueDigest(tabId, {diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 29f945448..c6538fa93 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -3350,6 +3350,51 @@} catch { return null; }}+ // Derive a document-unique selector for a verified field so the agent can+ // re-digest the SAME element later (e.g. pre-submit proof refresh after+ // focus moved). Each candidate is accepted only when it resolves back+ // to exactly this element β never a hard-coded guess that could match a+ // different field. Returns null when nothing identifies it uniquely.+ function _stableFieldSelector(el) {+ try {+ if (!el || el.nodeType !== 1 || !el.isConnected) return null;+ const resolvesUniquelyToEl = (selector) => {+ try {+ if (!selector) return false;+ const matches = document.querySelectorAll(selector);+ return matches.length === 1 && matches[0] === el;+ } catch { return false; }+ };+ const quoted = (value) => String(value ?? '').replace(/\\/g, '\\\\').replace(/"/g, '\\"');+ const elId = el.id || null;+ if (elId) {+ let selector = null;+ try {+ selector = (window.CSS && typeof window.CSS.escape === 'function')+ ? `#${window.CSS.escape(elId)}`+ : `#${quoted(elId)}`;+ } catch { selector = null; }+ if (selector && resolvesUniquelyToEl(selector)) return selector;+ }+ const tag = String(el.tagName || '').toLowerCase();+ const name = typeof el.getAttribute === 'function' ? el.getAttribute('name') : null;+ if (tag && name) {+ const selector = `${tag}[name="${quoted(name)}"]`;+ if (resolvesUniquelyToEl(selector)) return selector;+ }+ const ariaLabel = typeof el.getAttribute === 'function' ? el.getAttribute('aria-label') : null;+ if (tag && ariaLabel) {+ const selector = `${tag}[aria-label="${quoted(ariaLabel)}"]`;+ if (resolvesUniquelyToEl(selector)) return selector;+ }+ if (el.isContentEditable) {+ const selector = '[contenteditable="true"]';+ if (resolvesUniquelyToEl(selector)) return selector;+ }+ return null;+ } catch { return null; }+ }+function _richTextToolbarContextForElement(el) {try {if (!el || !el.isConnected) return { toolbarContext: false, toolbarRegionRef: '', toolbarRegionKey: '' };@@ -5674,6 +5719,7 @@valueLength: value.length,valueSha256,fieldMeta: _fieldMeta(el),+ stableSelector: _stableFieldSelector(el),};} catch (error) {return { success: false, error: error && error.message || String(error) };diff --git a/test/run.js b/test/run.jsindex 3e1a1f197..551b4af8b 100644--- a/test/run.js+++ b/test/run.js@@ -75286,6 +75286,7 @@ test('focused editor and commit-message writes authorize a commit', async () =>valueLength: live.body.length,valueSha256: await agent._sha256Text(live.body),fieldMeta: editorMeta,+ stableSelector: '[contenteditable="true"]',documentToken: 'doc-focused', refScopeUrl: 'https://github.com/example/repo/edit/main/docs/plan.md',};}@@ -75295,6 +75296,7 @@ test('focused editor and commit-message writes authorize a commit', async () =>valueLength: live.msg.length,valueSha256: await agent._sha256Text(live.msg),fieldMeta: messageMeta,+ stableSelector: '#commit-message-input',documentToken: 'doc-focused', refScopeUrl: 'https://github.com/example/repo/edit/main/docs/plan.md',};}@@ -75305,6 +75307,7 @@ test('focused editor and commit-message writes authorize a commit', async () =>valueLength: live.body.length,valueSha256: await agent._sha256Text(live.body),fieldMeta: editorMeta,+ stableSelector: '[contenteditable="true"]',};}if (selector === '#commit-message-input') {@@ -75313,6 +75316,7 @@ test('focused editor and commit-message writes authorize a commit', async () =>valueLength: live.msg.length,valueSha256: await agent._sha256Text(live.msg),fieldMeta: messageMeta,+ stableSelector: '#commit-message-input',};}return { success: false, documentToken: 'doc-focused', refScopeUrl: 'https://github.com/example/repo/edit/main/docs/plan.md' };@@ -75424,6 +75428,127 @@ test('bound focused typing preserves contentEditable for commit proofs', () => {}});+test('focused proofs revalidate through element-derived locators', async () => {+ for (const rel of [+ 'src/chrome/src/content/content.js',+ 'src/firefox/src/content/content.js',+ ]) {+ const source = fs.readFileSync(path.join(ROOT, rel), 'utf8');+ assert.match(source, /function _stableFieldSelector\(el\)/,+ `${rel}: digest lost its element-derived stable selector`);+ assert.match(source, /resolvesUniquelyToEl/,+ `${rel}: stable selector lost its uniqueness check`);+ }+ const originalChrome = globalThis.chrome;+ const originalBrowser = globalThis.browser;+ try {+ const body = '# Aria document\n\nOne complete copy.\n';+ const commitMsg = 'Update docs/plan.md';+ for (const [label, AgentClass, resolveJob, tabId] of [+ ['chrome', AgentCh, resolveAdapterWorkflowJob, 9496],+ ['firefox', AgentFx, resolveAdapterWorkflowJobFx, 9497],+ ]) {+ const agent = new AgentClass({});+ const pageUrl = 'https://github.com/Example/Repo/edit/main/docs/plan.md';+ agent._planExecutionGuards.set(tabId, {+ enabled: true,+ siteWorkflow: resolveJob(pageUrl, 'edit-file-and-commit'),+ workflowMetadataRequirements: [{ field: 'commit_message', value: commitMsg }],+ workflowMetadataRequirementsResolved: true,+ });+ agent._taskTokens.set(tabId, 'task-aria');+ agent._lastAxScopes.set(tabId, { documentToken: 'doc-aria', pageUrl });+ // Textarea-backed editor: classified by accessible label, NOT+ // contentEditable. The live digest derives a stable locator for the+ // verified element itself.+ const ariaEditorMeta = {+ contentEditable: false, tag: 'textarea', id: 'file-editor',+ ariaLabelledByText: 'Editing file contents',+ };+ const live = { msg: commitMsg };+ const tabs = {+ async sendMessage(_tabId, message) {+ const expected = message.params?.expected;+ const selector = message.params?.selector;+ if (expected === body) {+ return { success: true, verified: true,+ valueLength: body.length, valueSha256: await agent._sha256Text(body),+ fieldMeta: ariaEditorMeta, stableSelector: '#file-editor',+ documentToken: 'doc-aria', refScopeUrl: pageUrl };+ }+ if (expected === commitMsg) {+ return { success: true, verified: true,+ valueLength: commitMsg.length, valueSha256: await agent._sha256Text(commitMsg),+ fieldMeta: { id: 'commit-message-input', name: 'commit-message-input' },+ stableSelector: '#commit-message-input',+ documentToken: 'doc-aria', refScopeUrl: pageUrl };+ }+ if (selector === '#file-editor') {+ return { success: true,+ valueLength: body.length, valueSha256: await agent._sha256Text(body),+ fieldMeta: ariaEditorMeta, stableSelector: '#file-editor' };+ }+ if (selector === '#commit-message-input') {+ return { success: true,+ valueLength: live.msg.length, valueSha256: await agent._sha256Text(live.msg),+ fieldMeta: { id: 'commit-message-input', name: 'commit-message-input' },+ stableSelector: '#commit-message-input' };+ }+ return { success: false, documentToken: 'doc-aria', refScopeUrl: pageUrl };+ },+ };+ globalThis.chrome = { tabs };+ globalThis.browser = { tabs };+ // Chrome bound-branch shape (prepared metadata + verified fieldMeta);+ // Firefox/content shape (full fieldMeta). Both must bind the editor+ // through the accessible label with an element-derived locator.+ const editorResult = label === 'chrome'+ ? { success: true, verified: true, method: 'cdp-insert-focused',+ focusedField: { tag: 'TEXTAREA', type: '', name: 'file-editor', contentEditable: false },+ fieldMeta: ariaEditorMeta }+ : { success: true, verified: true, method: 'contenteditable', fieldMeta: ariaEditorMeta };+ await agent._finalizeTextMutationResult(tabId, 'type_text', { text: body, clear: true }, editorResult);+ const editorRecord = [...(agent._verifiedTextReplacements.get(tabId)?.values() || [])]+ .find(record => record?.focusedKind === 'editor');+ assert.ok(editorRecord, `${label}: aria-labelled editor minted no focused proof`);+ assert.equal(editorRecord?.refreshSelector, '#file-editor',+ `${label}: proof kept a hard-coded locator instead of the derived one`);+ const messageResult = label === 'chrome'+ ? { success: true, verified: true, method: 'cdp-insert-focused',+ focusedField: { tag: 'INPUT', type: 'text', name: 'commit-message-input', contentEditable: false } }+ : { success: true, verified: true, fieldMeta: { id: 'commit-message-input', name: 'commit-message-input' } };+ await agent._finalizeTextMutationResult(tabId, 'type_text', { text: commitMsg, clear: true }, messageResult);+ assert.ok(agent._workflowSubmitBindingForAttempt(tabId, pageUrl, {})?.githubFileCommit,+ `${label}: element-derived proofs did not authorize a commit`);+ // Refresh re-reads the derived elements (not [contenteditable]).+ await agent._refreshGithubTextReplacementProofs(tabId, pageUrl);+ assert.ok(agent._workflowSubmitBindingForAttempt(tabId, pageUrl, {})?.githubFileCommit,+ `${label}: derived-locator refresh dropped a good proof`);+ // An edit to the real editor drops the proof fail-closed.+ live.msg = `${commitMsg} (amended)`;+ await agent._refreshGithubTextReplacementProofs(tabId, pageUrl);+ assert.equal(agent._workflowSubmitBindingForAttempt(tabId, pageUrl, {})?.githubFileCommit, undefined,+ `${label}: stale proof survived an edit to the derived element`);+ // No unique locator: the proof cannot be revalidated and must not+ // authorize, rather than probing a hard-coded selector.+ live.msg = commitMsg;+ await agent._finalizeTextMutationResult(tabId, 'type_text', { text: body, clear: true }, editorResult);+ await agent._finalizeTextMutationResult(tabId, 'type_text', { text: commitMsg, clear: true }, messageResult);+ const reRecord = [...(agent._verifiedTextReplacements.get(tabId)?.values() || [])]+ .find(record => record?.focusedKind === 'editor');+ reRecord.refreshSelector = null;+ await agent._refreshGithubTextReplacementProofs(tabId, pageUrl);+ assert.equal(agent._verifiedTextReplacements.get(tabId)?.get(reRecord.key), undefined,+ `${label}: locator-less proof survived refresh`);+ }+ } finally {+ if (originalChrome === undefined) delete globalThis.chrome;+ else globalThis.chrome = originalChrome;+ if (originalBrowser === undefined) delete globalThis.browser;+ else globalThis.browser = originalBrowser;+ }+});+test('focused field_value_digest and live scope cover selectorless writes', async () => {for (const rel of ['src/chrome/src/content/content.js',