Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(agent): exact focused readback recovery with field identity

Emre Sokulluda40abba7bf9
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 2a3d5a3ad..e2e1d610c 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -20642,6 +20642,33 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return differingIdentityCount >= 2;
}
+ // Positive same-field identity for focused elements: the debt's
+ // record-time metadata and the retry's live digest metadata must agree on
+ // a stable identity. Strong identifiers win outright; weaker label text
+ // only counts when neither side offers an id or name and the tag and
+ // editability also agree. Anything less stays blocked (fail-closed).
+ _focusedFieldIdentityMatches(previousMeta, nextMeta) {
+ if (!previousMeta || typeof previousMeta !== 'object'
+ || !nextMeta || typeof nextMeta !== 'object') return false;
+ const normalized = (meta, field) => String(meta?.[field] ?? '').trim().toLowerCase();
+ const previousId = normalized(previousMeta, 'id');
+ const nextId = normalized(nextMeta, 'id');
+ if (previousId || nextId) return !!previousId && previousId === nextId;
+ const previousName = normalized(previousMeta, 'name');
+ const nextName = normalized(nextMeta, 'name');
+ const sameTag = !!previousMeta.tag
+ && normalized(previousMeta, 'tag') === normalized(nextMeta, 'tag');
+ if (previousName || nextName) return !!previousName && previousName === nextName && sameTag;
+ const sameLabel = ['ariaLabel', 'ariaLabelledByText', 'labelText', 'placeholder']
+ .some(field => {
+ const previous = normalized(previousMeta, field);
+ const next = normalized(nextMeta, field);
+ return !!previous && previous === next;
+ });
+ return sameLabel && sameTag
+ && previousMeta.contentEditable === nextMeta.contentEditable;
+ }
+
async _sha256Text(value) {
try {
const bytes = new TextEncoder().encode(String(value ?? ''));
@@ -21016,6 +21043,53 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// could not be proven distinct), clearing that debt and minting proof for
// the target would unguard a partial or duplicated write. Those cases
// stay blocked until the document changes.
+ // Focused same-field recovery: a selectorless retry carries no locator,
+ // so the ambiguous debt/target pair can never satisfy the key-identity
+ // branch below β€” yet the still-focused element is directly readable via
+ // the focused digest path. When the debt captured focused identity at
+ // record time and the live digest verifies the exact replacement text on
+ // the provably same field, recover exactly like a selector readback. A
+ // moved focus, a proven-different field, or missing identity all stay
+ // blocked, and a proof is minted only for a classifiable editor or
+ // commit-message kind.
+ if (sameReplacement
+ && name === 'type_text' && !args.selector && args.index == null
+ && target.locatorType === 'focused' && debt.locatorType === 'focused'
+ && debt.key === target.key) {
+ let focusedReadback = null;
+ try {
+ focusedReadback = await this._textMutationValueDigest(
+ tabId, { locatorType: 'focused', ambiguous: false }, text);
+ } catch { /* an unavailable readback leaves the write blocked */ }
+ if (focusedReadback?.verified === true
+ && this._focusedFieldIdentityMatches(debt.fieldMeta, focusedReadback.fieldMeta)) {
+ debts.delete(debt.key);
+ if (debts.size === 0) this._uncertainTextMutations.delete(tabId);
+ const normalizedFocusedMeta = this._normalizeFocusedFieldMeta(null, focusedReadback.fieldMeta);
+ const focusedKind = this._focusedGithubFieldKind(normalizedFocusedMeta);
+ if ((focusedKind === 'editor' || focusedKind === 'commit-message') && debt.replacesValue === true) {
+ let verifiedReplacements = this._verifiedTextReplacements.get(tabId);
+ if (!(verifiedReplacements instanceof Map)) {
+ verifiedReplacements = new Map();
+ this._verifiedTextReplacements.set(tabId, verifiedReplacements);
+ }
+ const effectiveTarget = this._focusedReplacementTarget(tabId, focusedKind);
+ verifiedReplacements.set(effectiveTarget.key, await this._verifiedTextReplacementRecord(
+ tabId, effectiveTarget, text, normalizedFocusedMeta,
+ ));
+ }
+ return {
+ success: true,
+ verified: true,
+ dispatched: false,
+ noDispatch: true,
+ recoveredUncertainMutation: true,
+ method: 'exact-text-readback',
+ expectedLength: text.length,
+ expectedSha256,
+ };
+ }
+ }
if (sameReplacement && !target.ambiguous && debt.key === target.key) {
let verified = false;
let readback = null;
@@ -21172,6 +21246,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!target.documentToken && !debtPageUrl) {
try { debtPageUrl = String(await this._currentUrl(tabId) || ''); } catch { debtPageUrl = ''; }
}
+ // Focused identity capture: selectorless writes carry no locator, so an
+ // identical retry could never prove same-field recovery. Capture the
+ // still-focused element's live metadata now β€” focus hasn't moved since
+ // the write just ran. Failure keeps the result metadata (if any), and
+ // identity-less retries stay blocked.
+ let debtFieldMeta = result.fieldMeta || null;
+ if (target.locatorType === 'focused') {
+ try {
+ const identity = await this._textMutationValueDigest(tabId, { locatorType: 'focused', ambiguous: false });
+ if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;
+ } catch { /* identity stays as the result metadata */ }
+ }
debts.set(target.key, {
...target,
pageUrl: debtPageUrl,
@@ -21180,7 +21266,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
expectedLength: text.length,
expectedSha256,
expectedFp: this._workflowInventoryFingerprint(text),
- fieldMeta: result.fieldMeta || null,
+ fieldMeta: debtFieldMeta,
recordedAt: Date.now(),
});
return {
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 929dc4577..95577c8e2 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -18213,6 +18213,33 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return differingIdentityCount >= 2;
}
+ // Positive same-field identity for focused elements: the debt's
+ // record-time metadata and the retry's live digest metadata must agree on
+ // a stable identity. Strong identifiers win outright; weaker label text
+ // only counts when neither side offers an id or name and the tag and
+ // editability also agree. Anything less stays blocked (fail-closed).
+ _focusedFieldIdentityMatches(previousMeta, nextMeta) {
+ if (!previousMeta || typeof previousMeta !== 'object'
+ || !nextMeta || typeof nextMeta !== 'object') return false;
+ const normalized = (meta, field) => String(meta?.[field] ?? '').trim().toLowerCase();
+ const previousId = normalized(previousMeta, 'id');
+ const nextId = normalized(nextMeta, 'id');
+ if (previousId || nextId) return !!previousId && previousId === nextId;
+ const previousName = normalized(previousMeta, 'name');
+ const nextName = normalized(nextMeta, 'name');
+ const sameTag = !!previousMeta.tag
+ && normalized(previousMeta, 'tag') === normalized(nextMeta, 'tag');
+ if (previousName || nextName) return !!previousName && previousName === nextName && sameTag;
+ const sameLabel = ['ariaLabel', 'ariaLabelledByText', 'labelText', 'placeholder']
+ .some(field => {
+ const previous = normalized(previousMeta, field);
+ const next = normalized(nextMeta, field);
+ return !!previous && previous === next;
+ });
+ return sameLabel && sameTag
+ && previousMeta.contentEditable === nextMeta.contentEditable;
+ }
+
async _sha256Text(value) {
try {
const bytes = new TextEncoder().encode(String(value ?? ''));
@@ -18587,6 +18614,53 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// could not be proven distinct), clearing that debt and minting proof for
// the target would unguard a partial or duplicated write. Those cases
// stay blocked until the document changes.
+ // Focused same-field recovery: a selectorless retry carries no locator,
+ // so the ambiguous debt/target pair can never satisfy the key-identity
+ // branch below β€” yet the still-focused element is directly readable via
+ // the focused digest path. When the debt captured focused identity at
+ // record time and the live digest verifies the exact replacement text on
+ // the provably same field, recover exactly like a selector readback. A
+ // moved focus, a proven-different field, or missing identity all stay
+ // blocked, and a proof is minted only for a classifiable editor or
+ // commit-message kind.
+ if (sameReplacement
+ && name === 'type_text' && !args.selector && args.index == null
+ && target.locatorType === 'focused' && debt.locatorType === 'focused'
+ && debt.key === target.key) {
+ let focusedReadback = null;
+ try {
+ focusedReadback = await this._textMutationValueDigest(
+ tabId, { locatorType: 'focused', ambiguous: false }, text);
+ } catch { /* an unavailable readback leaves the write blocked */ }
+ if (focusedReadback?.verified === true
+ && this._focusedFieldIdentityMatches(debt.fieldMeta, focusedReadback.fieldMeta)) {
+ debts.delete(debt.key);
+ if (debts.size === 0) this._uncertainTextMutations.delete(tabId);
+ const normalizedFocusedMeta = this._normalizeFocusedFieldMeta(null, focusedReadback.fieldMeta);
+ const focusedKind = this._focusedGithubFieldKind(normalizedFocusedMeta);
+ if ((focusedKind === 'editor' || focusedKind === 'commit-message') && debt.replacesValue === true) {
+ let verifiedReplacements = this._verifiedTextReplacements.get(tabId);
+ if (!(verifiedReplacements instanceof Map)) {
+ verifiedReplacements = new Map();
+ this._verifiedTextReplacements.set(tabId, verifiedReplacements);
+ }
+ const effectiveTarget = this._focusedReplacementTarget(tabId, focusedKind);
+ verifiedReplacements.set(effectiveTarget.key, await this._verifiedTextReplacementRecord(
+ tabId, effectiveTarget, text, normalizedFocusedMeta,
+ ));
+ }
+ return {
+ success: true,
+ verified: true,
+ dispatched: false,
+ noDispatch: true,
+ recoveredUncertainMutation: true,
+ method: 'exact-text-readback',
+ expectedLength: text.length,
+ expectedSha256,
+ };
+ }
+ }
if (sameReplacement && !target.ambiguous && debt.key === target.key
&& (((name === 'set_field' || name === 'type_ax') && typeof args.ref_id === 'string')
|| (name === 'type_text' && typeof args.selector === 'string' && args.selector.trim()))) {
@@ -18748,6 +18822,18 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!target.documentToken && !debtPageUrl) {
try { debtPageUrl = String(await this._currentUrl(tabId) || ''); } catch { debtPageUrl = ''; }
}
+ // Focused identity capture: selectorless writes carry no locator, so an
+ // identical retry could never prove same-field recovery. Capture the
+ // still-focused element's live metadata now β€” focus hasn't moved since
+ // the write just ran. Failure keeps the result metadata (if any), and
+ // identity-less retries stay blocked.
+ let debtFieldMeta = result.fieldMeta || null;
+ if (target.locatorType === 'focused') {
+ try {
+ const identity = await this._textMutationValueDigest(tabId, { locatorType: 'focused', ambiguous: false });
+ if (identity?.fieldMeta) debtFieldMeta = identity.fieldMeta;
+ } catch { /* identity stays as the result metadata */ }
+ }
debts.set(target.key, {
...target,
pageUrl: debtPageUrl,
@@ -18756,7 +18842,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
expectedLength: text.length,
expectedSha256,
expectedFp: this._workflowInventoryFingerprint(text),
- fieldMeta: result.fieldMeta || null,
+ fieldMeta: debtFieldMeta,
recordedAt: Date.now(),
});
return {
diff --git a/test/run.js b/test/run.js
index 4da19be21..60d38442d 100644
--- a/test/run.js
+++ b/test/run.js
@@ -77230,6 +77230,130 @@ test('uncertain write after navigation records live scoped debt', async () => {
}
});
+test('focused uncertain writes recover on exact same-field readback', async () => {
+ const originalChrome = globalThis.chrome;
+ const originalBrowser = globalThis.browser;
+ try {
+ const text = 'hello focused';
+ const editorMeta = { tag: 'div', contentEditable: true, id: 'editor-1', ariaLabelledByText: 'Editing file contents' };
+ for (const [label, AgentClass, tabId, controlTabId] of [
+ ['chrome', AgentCh, 9518, 9520],
+ ['firefox', AgentFx, 9519, 9521],
+ ]) {
+ const agent = new AgentClass({});
+ agent._lastAxScopes.set(tabId, { documentToken: 'doc-focus', pageUrl: 'https://example.test/form' });
+ // Commit-proof workflow so recovery mints a full readback record.
+ agent._planExecutionGuards.set(tabId, {
+ enabled: true,
+ siteWorkflow: { adapterName: 'github', job: { id: 'edit-file-and-commit' } },
+ });
+ // Record phase: the uncertain write carries no metadata, but focus is
+ // still on the indebted field, so the capture digest identifies it.
+ // Retry phase: the same focused field verifies the exact text.
+ const tabs = {
+ async sendMessage(_tabId, message) {
+ assert.equal(message.action, 'field_value_digest');
+ if (message.params?.expected !== undefined && message.params.expected !== text) {
+ return { success: false, documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };
+ }
+ return { success: true,
+ ...(typeof message.params?.expected === 'string' ? { verified: true } : {}),
+ valueLength: text.length,
+ valueSha256: await agent._sha256Text(text),
+ fieldMeta: { ...editorMeta },
+ documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };
+ },
+ };
+ globalThis.chrome = { tabs };
+ globalThis.browser = { tabs };
+ await agent._finalizeTextMutationResult(
+ tabId, 'type_text', { text, clear: true },
+ { success: false, dispatched: true, verified: false },
+ );
+ const debt = [...(agent._uncertainTextMutations.get(tabId)?.values() || [])][0];
+ assert.deepEqual(debt?.fieldMeta, editorMeta,
+ `${label}: record-time capture missed the focused identity`);
+ const recovered = await agent._uncertainTextMutationBlock(
+ tabId, 'type_text', { text, clear: true },
+ );
+ assert.equal(recovered?.success, true, `${label}: exact focused readback did not recover`);
+ assert.equal(recovered?.recoveredUncertainMutation, true);
+ assert.equal(recovered?.noDispatch, true);
+ assert.equal(agent._uncertainTextMutations.has(tabId), false, `${label}: recovered focused debt retained`);
+ const proof = [...(agent._verifiedTextReplacements.get(tabId)?.values() || [])]
+ .find(record => record?.focusedKind === 'editor');
+ assert.ok(proof?.readbackSha256, `${label}: recovery minted no editor proof`);
+ // Control: record-time focus already lost (identity capture fails), so
+ // even a later exact readback cannot prove the same field.
+ const control = new AgentClass({});
+ control._lastAxScopes.set(controlTabId, { documentToken: 'doc-focus', pageUrl: 'https://example.test/form' });
+ const controlTabs = {
+ async sendMessage(_tabId, message) {
+ assert.equal(message.action, 'field_value_digest');
+ // Probes without expected text (live-scope refresh, identity
+ // capture): nothing focused yet.
+ if (message.params?.expected === undefined) {
+ return { success: false, documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };
+ }
+ if (message.params.expected !== text) {
+ return { success: false, documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };
+ }
+ return { success: true, verified: true,
+ valueLength: text.length,
+ valueSha256: await control._sha256Text(text),
+ fieldMeta: { ...editorMeta },
+ documentToken: 'doc-focus', refScopeUrl: 'https://example.test/form' };
+ },
+ };
+ globalThis.chrome = { tabs: controlTabs };
+ globalThis.browser = { tabs: controlTabs };
+ await control._finalizeTextMutationResult(
+ controlTabId, 'type_text', { text, clear: true },
+ { success: false, dispatched: true, verified: false },
+ );
+ const blocked = await control._uncertainTextMutationBlock(
+ controlTabId, 'type_text', { text, clear: true },
+ );
+ assert.equal(blocked?.repeatBlocked, true,
+ `${label}: identity-less focused debt was cleared by an unattributed readback`);
+ assert.equal(control._uncertainTextMutations.get(controlTabId)?.size, 1,
+ `${label}: identity-less focused debt was not retained`);
+ assert.equal(control._verifiedTextReplacements.has(controlTabId), false,
+ `${label}: identity-less recovery minted proof for the wrong field`);
+ }
+ } finally {
+ if (originalChrome === undefined) delete globalThis.chrome;
+ else globalThis.chrome = originalChrome;
+ if (originalBrowser === undefined) delete globalThis.browser;
+ else globalThis.browser = originalBrowser;
+ }
+});
+
+test('focused field identity matching is strict', () => {
+ for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) {
+ const agent = new AgentClass({});
+ const base = { tag: 'div', contentEditable: true, id: 'editor-1', ariaLabelledByText: 'Editing file contents' };
+ assert.equal(agent._focusedFieldIdentityMatches(base, { ...base }), true, `${label}: identical metadata rejected`);
+ assert.equal(agent._focusedFieldIdentityMatches(
+ { tag: 'input', name: 'q' }, { tag: 'input', name: 'q' }), true, `${label}: name+tag rejected`);
+ assert.equal(agent._focusedFieldIdentityMatches(
+ { tag: 'input', name: 'q' }, { tag: 'textarea', name: 'q' }), false, `${label}: cross-tag name matched`);
+ assert.equal(agent._focusedFieldIdentityMatches(
+ { tag: 'textarea', labelText: 'Notes' }, { tag: 'textarea', labelText: 'Notes', contentEditable: false }), false,
+ `${label}: editability drift matched`);
+ assert.equal(agent._focusedFieldIdentityMatches(
+ { tag: 'textarea', labelText: 'Notes' }, { tag: 'textarea', labelText: 'Notes' }), true,
+ `${label}: label+tag identity rejected`);
+ assert.equal(agent._focusedFieldIdentityMatches(
+ { id: 'a', labelText: 'X' }, { id: 'b', labelText: 'X' }), false,
+ `${label}: shared label overrode differing ids`);
+ assert.equal(agent._focusedFieldIdentityMatches(null, base), false, `${label}: null matched`);
+ assert.equal(agent._focusedFieldIdentityMatches(base, null), false, `${label}: null matched`);
+ assert.equal(agent._focusedFieldIdentityMatches(
+ { tag: 'div' }, { tag: 'div' }), false, `${label}: identity-less metadata matched`);
+ }
+});
+
test('first live token bootstraps tokenless debt by URL', async () => {
const originalChrome = globalThis.chrome;
const originalBrowser = globalThis.browser;