Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: a field read back as another value is no evidence of success
Barack Sokullu2bdd7e706e2c
diff --git a/docs/guards.md b/docs/guards.mdindex e78730841..cd6428a36 100644--- a/docs/guards.md+++ b/docs/guards.md@@ -33,7 +33,8 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):completion judge's "not done", an unauthorized mutation, missing, failedor unfinished progress rows, unread pages of a whole thread the userasked for, text that went into a formatting toolbar, an exact table that- can no longer be delivered) ends the run as `partial`, not `success`, whether it ended with `done`, a structured+ can no longer be delivered, a field read back as another value than was+ written to it) ends the run as `partial`, not `success`, whether it ended with `done`, a structuredCloud run's `done_json` or a plain answer. Missing evidence alone keepsthe model's outcome.- A write that went out but could not be read back exactly (GitHub's@@ -41,7 +42,9 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):nothing, arms no text-write debt and is not a loop failure. A write thatan exact reader reads back as a different value is reported the sameway, but it ends a batch: the next action is planned once the model has- seen it.+ seen it. It is no evidence that the task's action succeeded, and a+ success claim while the field still reads back that value is refused+ once, then accepted as `partial`.- A GitHub commit sent by an action that can also rewrite the fields(`set_field` with a submit, Enter, `execute_js`) goes through with a notewhen the verified file binding covers it: the committed file is checkeddiff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 11c233810..21946f1b7 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -504,6 +504,8 @@ function selectionScopedResponseLanguagePolicy(locale) {// read-coverage-incomplete: pages of a requested whole read never read// (an unconfirmed Gmail expansion is its undecided case).// done-rich-text-toolbar-obligation: text typed into a toolbar.+// text-write-readback-unverified: a field an exact reader read back as+// another value than was written to it (its decided case).// Left out, because their decided case can be wrong about a finished task// or says nothing about it:// completion-page-warning: the open dialog or form error may be unrelated.@@ -520,7 +522,7 @@ function selectionScopedResponseLanguagePolicy(locale) {// and a mismatched one both reach it as missing evidence.const OUTCOME_DOWNGRADING_GUARDS = new Set(['completion-verifier-not-done', 'plan-unauthorized-mutation-recorded','progress-success-claim-checks', 'progress-ledger-unresolved', 'read-coverage-incomplete',- 'done-rich-text-toolbar-obligation']);+ 'done-rich-text-toolbar-obligation', 'text-write-readback-unverified']);// What the user is told when a run ends with plain text after the "end with// done" note was given once (Strict checks off).@@ -2629,6 +2631,12 @@ export class Agent extends LoopDetector {lastAction: toolbarObligation,};}+ if (outcome === 'success' && skip?.has?.('text_write_refuted') !== true && this._refutedTextWriteRecord(tabId)) {+ return {+ reason: 'text_write_refuted',+ error: 'Success is blocked because a field written in this task reads back a different value than the text written to it. Check that field and write the requested value again, or report a partial outcome that says what the field holds.',+ };+ }const state = this.completionInvariants.get(tabId);const block = completionDoneBlock(state, name, args, { skip });if (block) return block;@@ -5399,6 +5407,10 @@ export class Agent extends LoopDetector {return { id: 'done-rich-text-toolbar-obligation', decided: true,caveat: 'The last text may have gone into a formatting toolbar instead of the editor.' };}+ if (reason === 'text_write_refuted') {+ return { id: 'text-write-readback-unverified', decided: true,+ caveat: 'A field still reads back a different value than the text written to it.' };+ }if (reason === 'iframe_form_verification_required') {return { id: 'done-iframe-form-unverified', decided: false,caveat: 'The form fields edited inside an embedded frame were not read back.' };@@ -5434,6 +5446,7 @@ export class Agent extends LoopDetector {// that already advised is accepted and the next one is checked.const guards = [];if (this._richTextToolbarGuard.hasPending(tabId)) guards.push(this._completionInvariantGuard(tabId, 'rich_text_toolbar_target_unresolved'));+ if (this._refutedTextWriteRecord(tabId)) guards.push(this._completionInvariantGuard(tabId, 'text_write_refuted'));if (state?.iframeFormVerificationDebt) guards.push(this._completionInvariantGuard(tabId, 'iframe_form_verification_required'));if (state?.verificationDebt) guards.push(this._completionInvariantGuard(tabId, 'verification_required'));if (guards.length) {@@ -34299,6 +34312,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* (any page when empty) that passes `match(record)` wins. Records written* by another task are ignored.*/+ // A text write of this task that an exact reader read back as another+ // value, not superseded by a verified write to the same field: WebBrain+ // knows that field does not hold what was written to it.+ _refutedTextWriteRecord(tabId) {+ return this._unverifiedTextWriteRecord(tabId, {+ match: record => record.reason === 'readback_mismatch' && record.readbackExact === true,+ });+ }+_unverifiedTextWriteRecord(tabId, { key = '', pageUrl = '', match = null } = {}) {const records = this._unverifiedTextWrites?.get(tabId);if (!(records instanceof Map) || records.size === 0) return null;@@ -37865,6 +37887,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d|| result.missingToolResponse|| result.outcomeUnknown) return false;if (result.error && result.success !== true) return false;+ // A text write an exact reader read back as another value went out, but+ // the field does not hold what was asked: no evidence of success.+ if (textWriteRefuted(result)) return false;return true;}diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 0104914f8..e31ae3129 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -339,6 +339,8 @@ function selectionScopedResponseLanguagePolicy(locale) {// read-coverage-incomplete: pages of a requested whole read never read// (an unconfirmed Gmail expansion is its undecided case).// done-rich-text-toolbar-obligation: text typed into a toolbar.+// text-write-readback-unverified: a field an exact reader read back as+// another value than was written to it (its decided case).// Left out, because their decided case can be wrong about a finished task// or says nothing about it:// completion-page-warning: the open dialog or form error may be unrelated.@@ -355,7 +357,7 @@ function selectionScopedResponseLanguagePolicy(locale) {// and a mismatched one both reach it as missing evidence.const OUTCOME_DOWNGRADING_GUARDS = new Set(['completion-verifier-not-done', 'plan-unauthorized-mutation-recorded','progress-success-claim-checks', 'progress-ledger-unresolved', 'read-coverage-incomplete',- 'done-rich-text-toolbar-obligation']);+ 'done-rich-text-toolbar-obligation', 'text-write-readback-unverified']);// What the user is told when a run ends with plain text after the "end with// done" note was given once (Strict checks off).@@ -2186,6 +2188,12 @@ export class Agent extends LoopDetector {lastAction: toolbarObligation,};}+ if (outcome === 'success' && skip?.has?.('text_write_refuted') !== true && this._refutedTextWriteRecord(tabId)) {+ return {+ reason: 'text_write_refuted',+ error: 'Success is blocked because a field written in this task reads back a different value than the text written to it. Check that field and write the requested value again, or report a partial outcome that says what the field holds.',+ };+ }const state = this.completionInvariants.get(tabId);const block = completionDoneBlock(state, name, args, { skip });if (block) return block;@@ -4951,6 +4959,10 @@ export class Agent extends LoopDetector {return { id: 'done-rich-text-toolbar-obligation', decided: true,caveat: 'The last text may have gone into a formatting toolbar instead of the editor.' };}+ if (reason === 'text_write_refuted') {+ return { id: 'text-write-readback-unverified', decided: true,+ caveat: 'A field still reads back a different value than the text written to it.' };+ }if (reason === 'iframe_form_verification_required') {return { id: 'done-iframe-form-unverified', decided: false,caveat: 'The form fields edited inside an embedded frame were not read back.' };@@ -4986,6 +4998,7 @@ export class Agent extends LoopDetector {// that already advised is accepted and the next one is checked.const guards = [];if (this._richTextToolbarGuard.hasPending(tabId)) guards.push(this._completionInvariantGuard(tabId, 'rich_text_toolbar_target_unresolved'));+ if (this._refutedTextWriteRecord(tabId)) guards.push(this._completionInvariantGuard(tabId, 'text_write_refuted'));if (state?.iframeFormVerificationDebt) guards.push(this._completionInvariantGuard(tabId, 'iframe_form_verification_required'));if (state?.verificationDebt) guards.push(this._completionInvariantGuard(tabId, 'verification_required'));if (guards.length) {@@ -31112,6 +31125,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* (any page when empty) that passes `match(record)` wins. Records written* by another task are ignored.*/+ // A text write of this task that an exact reader read back as another+ // value, not superseded by a verified write to the same field: WebBrain+ // knows that field does not hold what was written to it.+ _refutedTextWriteRecord(tabId) {+ return this._unverifiedTextWriteRecord(tabId, {+ match: record => record.reason === 'readback_mismatch' && record.readbackExact === true,+ });+ }+_unverifiedTextWriteRecord(tabId, { key = '', pageUrl = '', match = null } = {}) {const records = this._unverifiedTextWrites?.get(tabId);if (!(records instanceof Map) || records.size === 0) return null;@@ -34735,6 +34757,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d|| result.missingToolResponse|| result.outcomeUnknown) return false;if (result.error && result.success !== true) return false;+ // A text write an exact reader read back as another value went out, but+ // the field does not hold what was asked: no evidence of success.+ if (textWriteRefuted(result)) return false;return true;}diff --git a/test/guard-advisory-completion.mjs b/test/guard-advisory-completion.mjsindex ea519113b..0b539fcb0 100644--- a/test/guard-advisory-completion.mjs+++ b/test/guard-advisory-completion.mjs@@ -535,6 +535,64 @@ for (const build of ['chrome', 'firefox']) {assert.equal(second.endStatus, undefined, `${build}: missing evidence alone must not end the run as partial`);});+ test(`${build}: a success over a field read back as another value is refused once, then ends as partial`, async () => {+ // An exact reader read the field back as another value than was written+ // to it (an advised, refuted write): WebBrain knows the field is wrong.+ const refute = (h, extra = {}) => h.agent._unverifiedTextWrites.set(h.tab, new Map([['ax:doc-1:ref_title', {+ key: 'ax:doc-1:ref_title', locatorType: 'ax', refId: 'ref_title', selector: null, documentToken: 'doc-1', pageUrl: PAGE,+ tool: 'set_field', replacesValue: true, expectedLength: 13, expectedSha256: 'a', observedLength: 12, observedSha256: 'b',+ readbackExact: true, reason: 'readback_mismatch', textSent: true, fieldMeta: { tag: 'input' },+ taskToken: h.agent._taskTokens?.get(h.tab) || null, recordedAt: Date.now(), ...extra }]]));+ const done = ['done', { summary: 'Updated the title.', outcome: 'success' }];+ const h = batchAgent(Agent);+ refute(h);+ assert.equal((await h.batch([done])).action, 'continue', `${build}: the first success claim gets its note`);+ const refusal = h.agent.conversations.get(h.tab).filter(message => message.role === 'tool').map(message => JSON.parse(message.content)).at(-1);+ assert.equal(refusal.reason, 'text_write_refuted');+ const updates = [];+ const accepted = await h.batch([done], updates);+ assert.equal(accepted.action, 'return');+ assert.equal(accepted.value, `Updated the title.\n\n${CAVEAT}A field still reads back a different value than the text written to it.`);+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;+ assert.equal(doneResult?.outcome, 'partial', `${build}: a success over a known-wrong field was not downgraded`);+ assert.equal(accepted.endStatus, 'partial');++ // A plain final over it is refused once too, and its run ends as partial.+ const plain = batchAgent(Agent);+ refute(plain);+ assert.equal(plain.agent._completionPlainFinalRefuses(plain.tab), true);+ assert.equal(plain.agent._completionPlainFinalRefuses(plain.tab), false);+ assert.equal(plain.agent._completionOutcomeDowngraded(plain.tab), true);++ // Not refused: a rendered readback (only unconfirmed), an honest partial,+ // or another task's write.+ for (const [label, extra, args] of [['rendered', { readbackExact: false }, done[1]],+ ['partial', {}, { summary: 'The title field holds another value.', outcome: 'partial' }],+ ['another task', { taskToken: 'another-task' }, done[1]]]) {+ const other = batchAgent(Agent);+ refute(other, extra);+ const updatesOther = [];+ assert.equal((await other.batch([['done', args]], updatesOther)).action, 'return', `${build} ${label}`);+ const result = updatesOther.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;+ assert.equal(result?.outcome, args.outcome, `${build} ${label}`);+ }++ // A refuted write is no evidence that the task's action succeeded; an+ // unconfirmed one still is.+ const refutedResult = { success: true, dispatched: true, unverifiedWrite: true, verified: false, readbackExact: true,+ unverified: { reason: 'readback_mismatch', reader: 'exact' } };+ const unconfirmed = { success: true, dispatched: true, unverifiedWrite: true, readbackExact: false,+ unverified: { reason: 'readback_mismatch', reader: 'rendered' } };+ assert.equal(h.agent._isSuccessfulExecutionEvidence(refutedResult), false);+ assert.equal(h.agent._isSuccessfulExecutionEvidence(unconfirmed), true);+ const plan = { enabled: true, successfulTaskToolCalls: 0, successfulConsequentialToolCalls: 0, pendingDownloadIds: [] };+ h.agent._planExecutionGuards.set(h.tab, plan);+ h.agent._markPlanExecutionToolCall(h.tab, 'set_field', refutedResult, { consequential: true });+ assert.deepEqual([plan.successfulTaskToolCalls, plan.successfulConsequentialToolCalls], [0, 0]);+ h.agent._markPlanExecutionToolCall(h.tab, 'set_field', unconfirmed, { consequential: true });+ assert.deepEqual([plan.successfulTaskToolCalls, plan.successfulConsequentialToolCalls], [1, 1]);+ });+test(`${build}: a chat run's success accepted over a decided check ends with the chat partial contract`, async () => {const { chatRunEndedNotice, createChatSession } = await import(`../src/${build}/src/agent/chat-workflow.js`);for (const decided of [true, false]) {