Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): capture selector identity for uncertain writes
Emre Sokullua91c6d85f6d7
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 630c540bc..75085937b 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -21347,6 +21347,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst identity = await this._textMutationValueDigest(tabId, { locatorType: 'focused', ambiguous: false });if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;} catch { /* identity stays as the result metadata */ }+ } else if (!debtFieldMeta && target.locatorType === 'selector') {+ // CDP-backed selector failures omit field metadata, which would+ // leave the block-time distinctness escape without a baseline and+ // block the rest of a multi-field form until navigation. Capture it+ // live: the target element is right there.+ try {+ const identity = await this._textMutationValueDigest(tabId, target);+ if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;+ } catch { /* identity-less debts stay fully blocking */ }}debts.set(target.key, {...target,diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex cdbdc5fef..80b40a8b4 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -18915,6 +18915,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst identity = await this._textMutationValueDigest(tabId, { locatorType: 'focused', ambiguous: false });if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;} catch { /* identity stays as the result metadata */ }+ } else if (!debtFieldMeta && target.locatorType === 'selector') {+ // Selector failures can omit field metadata, which would leave the+ // block-time distinctness escape without a baseline and block the+ // rest of a multi-field form until navigation. Capture it live: the+ // target element is right there.+ try {+ const identity = await this._textMutationValueDigest(tabId, target);+ if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;+ } catch { /* identity-less debts stay fully blocking */ }}debts.set(target.key, {...target,diff --git a/test/run.js b/test/run.jsindex efc263798..410b26c7a 100644--- a/test/run.js+++ b/test/run.js@@ -77164,6 +77164,72 @@ test('minted proofs carry the live digest scope', async () => {}});+test('uncertain selector writes capture live metadata for distinctness', async () => {+ const originalChrome = globalThis.chrome;+ const originalBrowser = globalThis.browser;+ try {+ const pageUrl = 'https://example.test/form';+ const fieldMetaA = { id: 'field-a', name: 'field-a', labelText: 'First' };+ const fieldMetaB = { id: 'field-b', name: 'field-b', labelText: 'Second' };+ for (const [label, AgentClass, tabId] of [+ ['chrome', AgentCh, 9526],+ ['firefox', AgentFx, 9527],+ ]) {+ const agent = new AgentClass({});+ agent._lastAxScopes.set(tabId, { documentToken: 'doc-multi', pageUrl });+ const tabs = {+ captureMeta: false,+ async sendMessage(_tabId, message) {+ assert.equal(message.action, 'field_value_digest');+ const selector = message.params?.selector;+ // Record-time probes carry no expected text: live scope first,+ // identity capture second.+ if (selector === '#a' && message.params?.expected === undefined) {+ if (!tabs.captureMeta) {+ return { success: false, documentToken: 'doc-multi', refScopeUrl: pageUrl };+ }+ return { success: true, valueLength: 5, valueSha256: await agent._sha256Text('alpha'),+ fieldMeta: fieldMetaA, documentToken: 'doc-multi', refScopeUrl: pageUrl };+ }+ if (selector === '#b') {+ return { success: true, valueLength: 4, valueSha256: await agent._sha256Text('beta'), fieldMeta: fieldMetaB };+ }+ return { success: false, documentToken: 'doc-multi', refScopeUrl: pageUrl };+ },+ };+ globalThis.chrome = { tabs };+ globalThis.browser = { tabs };+ // CDP-shaped failure: dispatched, uncertain, and no field metadata.+ await agent._finalizeTextMutationResult(+ tabId, 'type_text', { selector: '#a', text: 'alpha', clear: true },+ { success: false, dispatched: true, verified: false },+ );+ assert.equal(agent._uncertainTextMutations.get(tabId)?.get('selector:doc-multi:#a')?.fieldMeta ?? null, null,+ `${label}: debt unexpectedly carries identity without a capture source`);+ // Same failure, but the live element answers the capture probe.+ tabs.captureMeta = true;+ await agent._finalizeTextMutationResult(+ tabId, 'type_text', { selector: '#a', text: 'alpha', clear: true },+ { success: false, dispatched: true, verified: false },+ );+ assert.deepEqual(agent._uncertainTextMutations.get(tabId)?.get('selector:doc-multi:#a')?.fieldMeta, fieldMetaA,+ `${label}: record-time capture missed the selector identity`);+ // The distinctness escape now fires for the other field.+ const allowed = await agent._uncertainTextMutationBlock(+ tabId, 'type_text', { selector: '#b', text: 'beta' },+ );+ assert.equal(allowed, null, `${label}: distinct selector field stayed blocked without debt metadata`);+ assert.equal(agent._uncertainTextMutations.get(tabId)?.size, 1,+ `${label}: distinct-field dispatch dropped the original debt`);+ }+ } finally {+ if (originalChrome === undefined) delete globalThis.chrome;+ else globalThis.chrome = originalChrome;+ if (originalBrowser === undefined) delete globalThis.browser;+ else globalThis.browser = originalBrowser;+ }+});+test('selector writes to proven-distinct fields bypass unrelated debt', async () => {const originalChrome = globalThis.chrome;const originalBrowser = globalThis.browser;