Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): exact focused readback recovery with field identity
Emre Sokulluda40abba7bf9
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 2a3d5a3ad..e2e1d610c 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -20642,6 +20642,33 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn differingIdentityCount >= 2;}+ // Positive same-field identity for focused elements: the debt's+ // record-time metadata and the retry's live digest metadata must agree on+ // a stable identity. Strong identifiers win outright; weaker label text+ // only counts when neither side offers an id or name and the tag and+ // editability also agree. Anything less stays blocked (fail-closed).+ _focusedFieldIdentityMatches(previousMeta, nextMeta) {+ if (!previousMeta || typeof previousMeta !== 'object'+ || !nextMeta || typeof nextMeta !== 'object') return false;+ const normalized = (meta, field) => String(meta?.[field] ?? '').trim().toLowerCase();+ const previousId = normalized(previousMeta, 'id');+ const nextId = normalized(nextMeta, 'id');+ if (previousId || nextId) return !!previousId && previousId === nextId;+ const previousName = normalized(previousMeta, 'name');+ const nextName = normalized(nextMeta, 'name');+ const sameTag = !!previousMeta.tag+ && normalized(previousMeta, 'tag') === normalized(nextMeta, 'tag');+ if (previousName || nextName) return !!previousName && previousName === nextName && sameTag;+ const sameLabel = ['ariaLabel', 'ariaLabelledByText', 'labelText', 'placeholder']+ .some(field => {+ const previous = normalized(previousMeta, field);+ const next = normalized(nextMeta, field);+ return !!previous && previous === next;+ });+ return sameLabel && sameTag+ && previousMeta.contentEditable === nextMeta.contentEditable;+ }+async _sha256Text(value) {try {const bytes = new TextEncoder().encode(String(value ?? ''));@@ -21016,6 +21043,53 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// could not be proven distinct), clearing that debt and minting proof for// the target would unguard a partial or duplicated write. Those cases// stay blocked until the document changes.+ // Focused same-field recovery: a selectorless retry carries no locator,+ // so the ambiguous debt/target pair can never satisfy the key-identity+ // branch below β yet the still-focused element is directly readable via+ // the focused digest path. When the debt captured focused identity at+ // record time and the live digest verifies the exact replacement text on+ // the provably same field, recover exactly like a selector readback. A+ // moved focus, a proven-different field, or missing identity all stay+ // blocked, and a proof is minted only for a classifiable editor or+ // commit-message kind.+ if (sameReplacement+ && name === 'type_text' && !args.selector && args.index == null+ && target.locatorType === 'focused' && debt.locatorType === 'focused'+ && debt.key === target.key) {+ let focusedReadback = null;+ try {+ focusedReadback = await this._textMutationValueDigest(+ tabId, { locatorType: 'focused', ambiguous: false }, text);+ } catch { /* an unavailable readback leaves the write blocked */ }+ if (focusedReadback?.verified === true+ && this._focusedFieldIdentityMatches(debt.fieldMeta, focusedReadback.fieldMeta)) {+ debts.delete(debt.key);+ if (debts.size === 0) this._uncertainTextMutations.delete(tabId);+ const normalizedFocusedMeta = this._normalizeFocusedFieldMeta(null, focusedReadback.fieldMeta);+ const focusedKind = this._focusedGithubFieldKind(normalizedFocusedMeta);+ if ((focusedKind === 'editor' || focusedKind === 'commit-message') && debt.replacesValue === true) {+ let verifiedReplacements = this._verifiedTextReplacements.get(tabId);+ if (!(verifiedReplacements instanceof Map)) {+ verifiedReplacements = new Map();+ this._verifiedTextReplacements.set(tabId, verifiedReplacements);+ }+ const effectiveTarget = this._focusedReplacementTarget(tabId, focusedKind);+ verifiedReplacements.set(effectiveTarget.key, await this._verifiedTextReplacementRecord(+ tabId, effectiveTarget, text, normalizedFocusedMeta,+ ));+ }+ return {+ success: true,+ verified: true,+ dispatched: false,+ noDispatch: true,+ recoveredUncertainMutation: true,+ method: 'exact-text-readback',+ expectedLength: text.length,+ expectedSha256,+ };+ }+ }if (sameReplacement && !target.ambiguous && debt.key === target.key) {let verified = false;let readback = null;@@ -21172,6 +21246,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!target.documentToken && !debtPageUrl) {try { debtPageUrl = String(await this._currentUrl(tabId) || ''); } catch { debtPageUrl = ''; }}+ // Focused identity capture: selectorless writes carry no locator, so an+ // identical retry could never prove same-field recovery. Capture the+ // still-focused element's live metadata now β focus hasn't moved since+ // the write just ran. Failure keeps the result metadata (if any), and+ // identity-less retries stay blocked.+ let debtFieldMeta = result.fieldMeta || null;+ if (target.locatorType === 'focused') {+ try {+ const identity = await this._textMutationValueDigest(tabId, { locatorType: 'focused', ambiguous: false });+ if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;+ } catch { /* identity stays as the result metadata */ }+ }debts.set(target.key, {...target,pageUrl: debtPageUrl,@@ -21180,7 +21266,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dexpectedLength: text.length,expectedSha256,expectedFp: this._workflowInventoryFingerprint(text),- fieldMeta: result.fieldMeta || null,+ fieldMeta: debtFieldMeta,recordedAt: Date.now(),});return {diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 929dc4577..95577c8e2 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -18213,6 +18213,33 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn differingIdentityCount >= 2;}+ // Positive same-field identity for focused elements: the debt's+ // record-time metadata and the retry's live digest metadata must agree on+ // a stable identity. Strong identifiers win outright; weaker label text+ // only counts when neither side offers an id or name and the tag and+ // editability also agree. Anything less stays blocked (fail-closed).+ _focusedFieldIdentityMatches(previousMeta, nextMeta) {+ if (!previousMeta || typeof previousMeta !== 'object'+ || !nextMeta || typeof nextMeta !== 'object') return false;+ const normalized = (meta, field) => String(meta?.[field] ?? '').trim().toLowerCase();+ const previousId = normalized(previousMeta, 'id');+ const nextId = normalized(nextMeta, 'id');+ if (previousId || nextId) return !!previousId && previousId === nextId;+ const previousName = normalized(previousMeta, 'name');+ const nextName = normalized(nextMeta, 'name');+ const sameTag = !!previousMeta.tag+ && normalized(previousMeta, 'tag') === normalized(nextMeta, 'tag');+ if (previousName || nextName) return !!previousName && previousName === nextName && sameTag;+ const sameLabel = ['ariaLabel', 'ariaLabelledByText', 'labelText', 'placeholder']+ .some(field => {+ const previous = normalized(previousMeta, field);+ const next = normalized(nextMeta, field);+ return !!previous && previous === next;+ });+ return sameLabel && sameTag+ && previousMeta.contentEditable === nextMeta.contentEditable;+ }+async _sha256Text(value) {try {const bytes = new TextEncoder().encode(String(value ?? ''));@@ -18587,6 +18614,53 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// could not be proven distinct), clearing that debt and minting proof for// the target would unguard a partial or duplicated write. Those cases// stay blocked until the document changes.+ // Focused same-field recovery: a selectorless retry carries no locator,+ // so the ambiguous debt/target pair can never satisfy the key-identity+ // branch below β yet the still-focused element is directly readable via+ // the focused digest path. When the debt captured focused identity at+ // record time and the live digest verifies the exact replacement text on+ // the provably same field, recover exactly like a selector readback. A+ // moved focus, a proven-different field, or missing identity all stay+ // blocked, and a proof is minted only for a classifiable editor or+ // commit-message kind.+ if (sameReplacement+ && name === 'type_text' && !args.selector && args.index == null+ && target.locatorType === 'focused' && debt.locatorType === 'focused'+ && debt.key === target.key) {+ let focusedReadback = null;+ try {+ focusedReadback = await this._textMutationValueDigest(+ tabId, { locatorType: 'focused', ambiguous: false }, text);+ } catch { /* an unavailable readback leaves the write blocked */ }+ if (focusedReadback?.verified === true+ && this._focusedFieldIdentityMatches(debt.fieldMeta, focusedReadback.fieldMeta)) {+ debts.delete(debt.key);+ if (debts.size === 0) this._uncertainTextMutations.delete(tabId);+ const normalizedFocusedMeta = this._normalizeFocusedFieldMeta(null, focusedReadback.fieldMeta);+ const focusedKind = this._focusedGithubFieldKind(normalizedFocusedMeta);+ if ((focusedKind === 'editor' || focusedKind === 'commit-message') && debt.replacesValue === true) {+ let verifiedReplacements = this._verifiedTextReplacements.get(tabId);+ if (!(verifiedReplacements instanceof Map)) {+ verifiedReplacements = new Map();+ this._verifiedTextReplacements.set(tabId, verifiedReplacements);+ }+ const effectiveTarget = this._focusedReplacementTarget(tabId, focusedKind);+ verifiedReplacements.set(effectiveTarget.key, await this._verifiedTextReplacementRecord(+ tabId, effectiveTarget, text, normalizedFocusedMeta,+ ));+ }+ return {+ success: true,+ verified: true,+ dispatched: false,+ noDispatch: true,+ recoveredUncertainMutation: true,+ method: 'exact-text-readback',+ expectedLength: text.length,+ expectedSha256,+ };+ }+ }if (sameReplacement && !target.ambiguous && debt.key === target.key&& (((name === 'set_field' || name === 'type_ax') && typeof args.ref_id === 'string')|| (name === 'type_text' && typeof args.selector === 'string' && args.selector.trim()))) {@@ -18748,6 +18822,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!target.documentToken && !debtPageUrl) {try { debtPageUrl = String(await this._currentUrl(tabId) || ''); } catch { debtPageUrl = ''; }}+ // Focused identity capture: selectorless writes carry no locator, so an+ // identical retry could never prove same-field recovery. Capture the+ // still-focused element's live metadata now β focus hasn't moved since+ // the write just ran. Failure keeps the result metadata (if any), and+ // identity-less retries stay blocked.+ let debtFieldMeta = result.fieldMeta || null;+ if (target.locatorType === 'focused') {+ try {+ const identity = await this._textMutationValueDigest(tabId, { locatorType: 'focused', ambiguous: false });+ if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;+ } catch { /* identity stays as the result metadata */ }+ }debts.set(target.key, {...target,pageUrl: debtPageUrl,@@ -18756,7 +18842,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dexpectedLength: text.length,expectedSha256,expectedFp: this._workflowInventoryFingerprint(text),- fieldMeta: result.fieldMeta || null,+ fieldMeta: debtFieldMeta,recordedAt: Date.now(),});return {diff --git a/test/run.js b/test/run.jsindex 4da19be21..60d38442d 100644--- a/test/run.js+++ b/test/run.js@@ -77230,6 +77230,130 @@ test('uncertain write after navigation records live scoped debt', async () => {}});+test('focused uncertain writes recover on exact same-field readback', async () => {+ const originalChrome = globalThis.chrome;+ const originalBrowser = globalThis.browser;+ try {+ const text = 'hello focused';+ const editorMeta = { tag: 'div', contentEditable: true, id: 'editor-1', ariaLabelledByText: 'Editing file contents' };+ for (const [label, AgentClass, tabId, controlTabId] of [+ ['chrome', AgentCh, 9518, 9520],+ ['firefox', AgentFx, 9519, 9521],+ ]) {+ const agent = new AgentClass({});+ agent._lastAxScopes.set(tabId, { documentToken: 'doc-focus', pageUrl: 'https://example.test/form' });+ // Commit-proof workflow so recovery mints a full readback record.+ agent._planExecutionGuards.set(tabId, {+ enabled: true,+ siteWorkflow: { adapterName: 'github', job: { id: 'edit-file-and-commit' } },+ });+ // Record phase: the uncertain write carries no metadata, but focus is+ // still on the indebted field, so the capture digest identifies it.+ // Retry phase: the same focused field verifies the exact text.+ const tabs = {+ async sendMessage(_tabId, message) {+ assert.equal(message.action, 'field_value_digest');+ if (message.params?.expected !== undefined && message.params.expected !== text) {+ return { success: false, documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };+ }+ return { success: true,+ ...(typeof message.params?.expected === 'string' ? { verified: true } : {}),+ valueLength: text.length,+ valueSha256: await agent._sha256Text(text),+ fieldMeta: { ...editorMeta },+ documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };+ },+ };+ globalThis.chrome = { tabs };+ globalThis.browser = { tabs };+ await agent._finalizeTextMutationResult(+ tabId, 'type_text', { text, clear: true },+ { success: false, dispatched: true, verified: false },+ );+ const debt = [...(agent._uncertainTextMutations.get(tabId)?.values() || [])][0];+ assert.deepEqual(debt?.fieldMeta, editorMeta,+ `${label}: record-time capture missed the focused identity`);+ const recovered = await agent._uncertainTextMutationBlock(+ tabId, 'type_text', { text, clear: true },+ );+ assert.equal(recovered?.success, true, `${label}: exact focused readback did not recover`);+ assert.equal(recovered?.recoveredUncertainMutation, true);+ assert.equal(recovered?.noDispatch, true);+ assert.equal(agent._uncertainTextMutations.has(tabId), false, `${label}: recovered focused debt retained`);+ const proof = [...(agent._verifiedTextReplacements.get(tabId)?.values() || [])]+ .find(record => record?.focusedKind === 'editor');+ assert.ok(proof?.readbackSha256, `${label}: recovery minted no editor proof`);+ // Control: record-time focus already lost (identity capture fails), so+ // even a later exact readback cannot prove the same field.+ const control = new AgentClass({});+ control._lastAxScopes.set(controlTabId, { documentToken: 'doc-focus', pageUrl: 'https://example.test/form' });+ const controlTabs = {+ async sendMessage(_tabId, message) {+ assert.equal(message.action, 'field_value_digest');+ // Probes without expected text (live-scope refresh, identity+ // capture): nothing focused yet.+ if (message.params?.expected === undefined) {+ return { success: false, documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };+ }+ if (message.params.expected !== text) {+ return { success: false, documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };+ }+ return { success: true, verified: true,+ valueLength: text.length,+ valueSha256: await control._sha256Text(text),+ fieldMeta: { ...editorMeta },+ documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };+ },+ };+ globalThis.chrome = { tabs: controlTabs };+ globalThis.browser = { tabs: controlTabs };+ await control._finalizeTextMutationResult(+ controlTabId, 'type_text', { text, clear: true },+ { success: false, dispatched: true, verified: false },+ );+ const blocked = await control._uncertainTextMutationBlock(+ controlTabId, 'type_text', { text, clear: true },+ );+ assert.equal(blocked?.repeatBlocked, true,+ `${label}: identity-less focused debt was cleared by an unattributed readback`);+ assert.equal(control._uncertainTextMutations.get(controlTabId)?.size, 1,+ `${label}: identity-less focused debt was not retained`);+ assert.equal(control._verifiedTextReplacements.has(controlTabId), false,+ `${label}: identity-less recovery minted proof for the wrong field`);+ }+ } finally {+ if (originalChrome === undefined) delete globalThis.chrome;+ else globalThis.chrome = originalChrome;+ if (originalBrowser === undefined) delete globalThis.browser;+ else globalThis.browser = originalBrowser;+ }+});++test('focused field identity matching is strict', () => {+ for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) {+ const agent = new AgentClass({});+ const base = { tag: 'div', contentEditable: true, id: 'editor-1', ariaLabelledByText: 'Editing file contents' };+ assert.equal(agent._focusedFieldIdentityMatches(base, { ...base }), true, `${label}: identical metadata rejected`);+ assert.equal(agent._focusedFieldIdentityMatches(+ { tag: 'input', name: 'q' }, { tag: 'input', name: 'q' }), true, `${label}: name+tag rejected`);+ assert.equal(agent._focusedFieldIdentityMatches(+ { tag: 'input', name: 'q' }, { tag: 'textarea', name: 'q' }), false, `${label}: cross-tag name matched`);+ assert.equal(agent._focusedFieldIdentityMatches(+ { tag: 'textarea', labelText: 'Notes' }, { tag: 'textarea', labelText: 'Notes', contentEditable: false }), false,+ `${label}: editability drift matched`);+ assert.equal(agent._focusedFieldIdentityMatches(+ { tag: 'textarea', labelText: 'Notes' }, { tag: 'textarea', labelText: 'Notes' }), true,+ `${label}: label+tag identity rejected`);+ assert.equal(agent._focusedFieldIdentityMatches(+ { id: 'a', labelText: 'X' }, { id: 'b', labelText: 'X' }), false,+ `${label}: shared label overrode differing ids`);+ assert.equal(agent._focusedFieldIdentityMatches(null, base), false, `${label}: null matched`);+ assert.equal(agent._focusedFieldIdentityMatches(base, null), false, `${label}: null matched`);+ assert.equal(agent._focusedFieldIdentityMatches(+ { tag: 'div' }, { tag: 'div' }), false, `${label}: identity-less metadata matched`);+ }+});+test('first live token bootstraps tokenless debt by URL', async () => {const originalChrome = globalThis.chrome;const originalBrowser = globalThis.browser;