Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard policy: a field read back as another value is no evidence of success

Barack Sokullu2bdd7e706e2c
diff --git a/docs/guards.md b/docs/guards.md
index e78730841..cd6428a36 100644
--- a/docs/guards.md
+++ b/docs/guards.md
@@ -33,7 +33,8 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):
completion judge's "not done", an unauthorized mutation, missing, failed
or unfinished progress rows, unread pages of a whole thread the user
asked for, text that went into a formatting toolbar, an exact table that
- can no longer be delivered) ends the run as `partial`, not `success`, whether it ended with `done`, a structured
+ can no longer be delivered, a field read back as another value than was
+ written to it) ends the run as `partial`, not `success`, whether it ended with `done`, a structured
Cloud run's `done_json` or a plain answer. Missing evidence alone keeps
the model's outcome.
- A write that went out but could not be read back exactly (GitHub's
@@ -41,7 +42,9 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):
nothing, arms no text-write debt and is not a loop failure. A write that
an exact reader reads back as a different value is reported the same
way, but it ends a batch: the next action is planned once the model has
- seen it.
+ seen it. It is no evidence that the task's action succeeded, and a
+ success claim while the field still reads back that value is refused
+ once, then accepted as `partial`.
- A GitHub commit sent by an action that can also rewrite the fields
(`set_field` with a submit, Enter, `execute_js`) goes through with a note
when the verified file binding covers it: the committed file is checked
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 11c233810..21946f1b7 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -504,6 +504,8 @@ function selectionScopedResponseLanguagePolicy(locale) {
// read-coverage-incomplete: pages of a requested whole read never read
// (an unconfirmed Gmail expansion is its undecided case).
// done-rich-text-toolbar-obligation: text typed into a toolbar.
+// text-write-readback-unverified: a field an exact reader read back as
+// another value than was written to it (its decided case).
// Left out, because their decided case can be wrong about a finished task
// or says nothing about it:
// completion-page-warning: the open dialog or form error may be unrelated.
@@ -520,7 +522,7 @@ function selectionScopedResponseLanguagePolicy(locale) {
// and a mismatched one both reach it as missing evidence.
const OUTCOME_DOWNGRADING_GUARDS = new Set(['completion-verifier-not-done', 'plan-unauthorized-mutation-recorded',
'progress-success-claim-checks', 'progress-ledger-unresolved', 'read-coverage-incomplete',
- 'done-rich-text-toolbar-obligation']);
+ 'done-rich-text-toolbar-obligation', 'text-write-readback-unverified']);
// What the user is told when a run ends with plain text after the "end with
// done" note was given once (Strict checks off).
@@ -2629,6 +2631,12 @@ export class Agent extends LoopDetector {
lastAction: toolbarObligation,
};
}
+ if (outcome === 'success' && skip?.has?.('text_write_refuted') !== true && this._refutedTextWriteRecord(tabId)) {
+ return {
+ reason: 'text_write_refuted',
+ error: 'Success is blocked because a field written in this task reads back a different value than the text written to it. Check that field and write the requested value again, or report a partial outcome that says what the field holds.',
+ };
+ }
const state = this.completionInvariants.get(tabId);
const block = completionDoneBlock(state, name, args, { skip });
if (block) return block;
@@ -5399,6 +5407,10 @@ export class Agent extends LoopDetector {
return { id: 'done-rich-text-toolbar-obligation', decided: true,
caveat: 'The last text may have gone into a formatting toolbar instead of the editor.' };
}
+ if (reason === 'text_write_refuted') {
+ return { id: 'text-write-readback-unverified', decided: true,
+ caveat: 'A field still reads back a different value than the text written to it.' };
+ }
if (reason === 'iframe_form_verification_required') {
return { id: 'done-iframe-form-unverified', decided: false,
caveat: 'The form fields edited inside an embedded frame were not read back.' };
@@ -5434,6 +5446,7 @@ export class Agent extends LoopDetector {
// that already advised is accepted and the next one is checked.
const guards = [];
if (this._richTextToolbarGuard.hasPending(tabId)) guards.push(this._completionInvariantGuard(tabId, 'rich_text_toolbar_target_unresolved'));
+ if (this._refutedTextWriteRecord(tabId)) guards.push(this._completionInvariantGuard(tabId, 'text_write_refuted'));
if (state?.iframeFormVerificationDebt) guards.push(this._completionInvariantGuard(tabId, 'iframe_form_verification_required'));
if (state?.verificationDebt) guards.push(this._completionInvariantGuard(tabId, 'verification_required'));
if (guards.length) {
@@ -34299,6 +34312,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* (any page when empty) that passes `match(record)` wins. Records written
* by another task are ignored.
*/
+ // A text write of this task that an exact reader read back as another
+ // value, not superseded by a verified write to the same field: WebBrain
+ // knows that field does not hold what was written to it.
+ _refutedTextWriteRecord(tabId) {
+ return this._unverifiedTextWriteRecord(tabId, {
+ match: record => record.reason === 'readback_mismatch' && record.readbackExact === true,
+ });
+ }
+
_unverifiedTextWriteRecord(tabId, { key = '', pageUrl = '', match = null } = {}) {
const records = this._unverifiedTextWrites?.get(tabId);
if (!(records instanceof Map) || records.size === 0) return null;
@@ -37865,6 +37887,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| result.missingToolResponse
|| result.outcomeUnknown) return false;
if (result.error && result.success !== true) return false;
+ // A text write an exact reader read back as another value went out, but
+ // the field does not hold what was asked: no evidence of success.
+ if (textWriteRefuted(result)) return false;
return true;
}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 0104914f8..e31ae3129 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -339,6 +339,8 @@ function selectionScopedResponseLanguagePolicy(locale) {
// read-coverage-incomplete: pages of a requested whole read never read
// (an unconfirmed Gmail expansion is its undecided case).
// done-rich-text-toolbar-obligation: text typed into a toolbar.
+// text-write-readback-unverified: a field an exact reader read back as
+// another value than was written to it (its decided case).
// Left out, because their decided case can be wrong about a finished task
// or says nothing about it:
// completion-page-warning: the open dialog or form error may be unrelated.
@@ -355,7 +357,7 @@ function selectionScopedResponseLanguagePolicy(locale) {
// and a mismatched one both reach it as missing evidence.
const OUTCOME_DOWNGRADING_GUARDS = new Set(['completion-verifier-not-done', 'plan-unauthorized-mutation-recorded',
'progress-success-claim-checks', 'progress-ledger-unresolved', 'read-coverage-incomplete',
- 'done-rich-text-toolbar-obligation']);
+ 'done-rich-text-toolbar-obligation', 'text-write-readback-unverified']);
// What the user is told when a run ends with plain text after the "end with
// done" note was given once (Strict checks off).
@@ -2186,6 +2188,12 @@ export class Agent extends LoopDetector {
lastAction: toolbarObligation,
};
}
+ if (outcome === 'success' && skip?.has?.('text_write_refuted') !== true && this._refutedTextWriteRecord(tabId)) {
+ return {
+ reason: 'text_write_refuted',
+ error: 'Success is blocked because a field written in this task reads back a different value than the text written to it. Check that field and write the requested value again, or report a partial outcome that says what the field holds.',
+ };
+ }
const state = this.completionInvariants.get(tabId);
const block = completionDoneBlock(state, name, args, { skip });
if (block) return block;
@@ -4951,6 +4959,10 @@ export class Agent extends LoopDetector {
return { id: 'done-rich-text-toolbar-obligation', decided: true,
caveat: 'The last text may have gone into a formatting toolbar instead of the editor.' };
}
+ if (reason === 'text_write_refuted') {
+ return { id: 'text-write-readback-unverified', decided: true,
+ caveat: 'A field still reads back a different value than the text written to it.' };
+ }
if (reason === 'iframe_form_verification_required') {
return { id: 'done-iframe-form-unverified', decided: false,
caveat: 'The form fields edited inside an embedded frame were not read back.' };
@@ -4986,6 +4998,7 @@ export class Agent extends LoopDetector {
// that already advised is accepted and the next one is checked.
const guards = [];
if (this._richTextToolbarGuard.hasPending(tabId)) guards.push(this._completionInvariantGuard(tabId, 'rich_text_toolbar_target_unresolved'));
+ if (this._refutedTextWriteRecord(tabId)) guards.push(this._completionInvariantGuard(tabId, 'text_write_refuted'));
if (state?.iframeFormVerificationDebt) guards.push(this._completionInvariantGuard(tabId, 'iframe_form_verification_required'));
if (state?.verificationDebt) guards.push(this._completionInvariantGuard(tabId, 'verification_required'));
if (guards.length) {
@@ -31112,6 +31125,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* (any page when empty) that passes `match(record)` wins. Records written
* by another task are ignored.
*/
+ // A text write of this task that an exact reader read back as another
+ // value, not superseded by a verified write to the same field: WebBrain
+ // knows that field does not hold what was written to it.
+ _refutedTextWriteRecord(tabId) {
+ return this._unverifiedTextWriteRecord(tabId, {
+ match: record => record.reason === 'readback_mismatch' && record.readbackExact === true,
+ });
+ }
+
_unverifiedTextWriteRecord(tabId, { key = '', pageUrl = '', match = null } = {}) {
const records = this._unverifiedTextWrites?.get(tabId);
if (!(records instanceof Map) || records.size === 0) return null;
@@ -34735,6 +34757,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| result.missingToolResponse
|| result.outcomeUnknown) return false;
if (result.error && result.success !== true) return false;
+ // A text write an exact reader read back as another value went out, but
+ // the field does not hold what was asked: no evidence of success.
+ if (textWriteRefuted(result)) return false;
return true;
}
diff --git a/test/guard-advisory-completion.mjs b/test/guard-advisory-completion.mjs
index ea519113b..0b539fcb0 100644
--- a/test/guard-advisory-completion.mjs
+++ b/test/guard-advisory-completion.mjs
@@ -535,6 +535,64 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(second.endStatus, undefined, `${build}: missing evidence alone must not end the run as partial`);
});
+ test(`${build}: a success over a field read back as another value is refused once, then ends as partial`, async () => {
+ // An exact reader read the field back as another value than was written
+ // to it (an advised, refuted write): WebBrain knows the field is wrong.
+ const refute = (h, extra = {}) => h.agent._unverifiedTextWrites.set(h.tab, new Map([['ax:doc-1:ref_title', {
+ key: 'ax:doc-1:ref_title', locatorType: 'ax', refId: 'ref_title', selector: null, documentToken: 'doc-1', pageUrl: PAGE,
+ tool: 'set_field', replacesValue: true, expectedLength: 13, expectedSha256: 'a', observedLength: 12, observedSha256: 'b',
+ readbackExact: true, reason: 'readback_mismatch', textSent: true, fieldMeta: { tag: 'input' },
+ taskToken: h.agent._taskTokens?.get(h.tab) || null, recordedAt: Date.now(), ...extra }]]));
+ const done = ['done', { summary: 'Updated the title.', outcome: 'success' }];
+ const h = batchAgent(Agent);
+ refute(h);
+ assert.equal((await h.batch([done])).action, 'continue', `${build}: the first success claim gets its note`);
+ const refusal = h.agent.conversations.get(h.tab).filter(message => message.role === 'tool').map(message => JSON.parse(message.content)).at(-1);
+ assert.equal(refusal.reason, 'text_write_refuted');
+ const updates = [];
+ const accepted = await h.batch([done], updates);
+ assert.equal(accepted.action, 'return');
+ assert.equal(accepted.value, `Updated the title.\n\n${CAVEAT}A field still reads back a different value than the text written to it.`);
+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;
+ assert.equal(doneResult?.outcome, 'partial', `${build}: a success over a known-wrong field was not downgraded`);
+ assert.equal(accepted.endStatus, 'partial');
+
+ // A plain final over it is refused once too, and its run ends as partial.
+ const plain = batchAgent(Agent);
+ refute(plain);
+ assert.equal(plain.agent._completionPlainFinalRefuses(plain.tab), true);
+ assert.equal(plain.agent._completionPlainFinalRefuses(plain.tab), false);
+ assert.equal(plain.agent._completionOutcomeDowngraded(plain.tab), true);
+
+ // Not refused: a rendered readback (only unconfirmed), an honest partial,
+ // or another task's write.
+ for (const [label, extra, args] of [['rendered', { readbackExact: false }, done[1]],
+ ['partial', {}, { summary: 'The title field holds another value.', outcome: 'partial' }],
+ ['another task', { taskToken: 'another-task' }, done[1]]]) {
+ const other = batchAgent(Agent);
+ refute(other, extra);
+ const updatesOther = [];
+ assert.equal((await other.batch([['done', args]], updatesOther)).action, 'return', `${build} ${label}`);
+ const result = updatesOther.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;
+ assert.equal(result?.outcome, args.outcome, `${build} ${label}`);
+ }
+
+ // A refuted write is no evidence that the task's action succeeded; an
+ // unconfirmed one still is.
+ const refutedResult = { success: true, dispatched: true, unverifiedWrite: true, verified: false, readbackExact: true,
+ unverified: { reason: 'readback_mismatch', reader: 'exact' } };
+ const unconfirmed = { success: true, dispatched: true, unverifiedWrite: true, readbackExact: false,
+ unverified: { reason: 'readback_mismatch', reader: 'rendered' } };
+ assert.equal(h.agent._isSuccessfulExecutionEvidence(refutedResult), false);
+ assert.equal(h.agent._isSuccessfulExecutionEvidence(unconfirmed), true);
+ const plan = { enabled: true, successfulTaskToolCalls: 0, successfulConsequentialToolCalls: 0, pendingDownloadIds: [] };
+ h.agent._planExecutionGuards.set(h.tab, plan);
+ h.agent._markPlanExecutionToolCall(h.tab, 'set_field', refutedResult, { consequential: true });
+ assert.deepEqual([plan.successfulTaskToolCalls, plan.successfulConsequentialToolCalls], [0, 0]);
+ h.agent._markPlanExecutionToolCall(h.tab, 'set_field', unconfirmed, { consequential: true });
+ assert.deepEqual([plan.successfulTaskToolCalls, plan.successfulConsequentialToolCalls], [1, 1]);
+ });
+
test(`${build}: a chat run's success accepted over a decided check ends with the chat partial contract`, async () => {
const { chatRunEndedNotice, createChatSession } = await import(`../src/${build}/src/agent/chat-workflow.js`);
for (const decided of [true, false]) {