Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard policy: a write read back exactly as a different value ends the batch

Barack Sokullucbb79481e998
diff --git a/docs/guards.md b/docs/guards.md
index 8a69d03ac..f3166689f 100644
--- a/docs/guards.md
+++ b/docs/guards.md
@@ -38,7 +38,10 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):
the model's outcome.
- A write that went out but could not be read back exactly (GitHub's
CodeMirror editor is the usual case) is an unverified success: it locks
- nothing, arms no text-write debt and is not a loop failure.
+ nothing, arms no text-write debt and is not a loop failure. A write that
+ an exact reader reads back as a different value is reported the same
+ way, but it ends a batch: the next action is planned once the model has
+ seen it.
- A quality guard whose `onCannotDecide` is `ask` asks the user when it
cannot verify a consequential step, such as a commit or a post. Where
that guard has no card, it refuses.
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index b90162a9b..4569b43ad 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -22,7 +22,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENT
import { rejectedCompletionRecovery } from './completion-recovery.js';
import { FailureRecoveryRegistry, modelFailureRecovery } from './failure-recovery.js';
import { validateToolArguments } from './tool-arguments.js';
-import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus } from './browser-batch.js';
+import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus, textWriteRefuted } from './browser-batch.js';
import { calculateExpression, calculationResultInText } from './calculator.js';
import { isLiteralCalculationTask, literalCalculationSummaryFormatMatches, isPassivePageFeedback } from './page-feedback-policy.js';
import { chatRecoveryDisplayText, terminalRecoveryFinalText } from './chat-recovery-text.js';
@@ -14318,10 +14318,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// A text write that went out but could not be read back exactly is an
// advisory result (success + unverifiedWrite + a note), not a reason to
// replan the rest of the batch. Failures, refusals and unknown outcomes
- // still interrupt; so does every unverified result under Strict checks.
+ // still interrupt; so does a write an exact reader read back as a
+ // different value, since the rest of the batch would build on a value
+ // WebBrain knows is wrong, and every unverified result under Strict checks.
_advisoryTextWriteContinuesBatch(toolName, toolResult) {
return ['set_field', 'type_ax', 'type_text'].includes(toolName)
&& toolResult?.success === true && toolResult.unverifiedWrite === true && !toolResult.error
+ && !textWriteRefuted(toolResult)
&& this._guardAdvises('browser-batch-fresh-turn-interrupt', { decided: true });
}
diff --git a/src/chrome/src/agent/browser-batch.js b/src/chrome/src/agent/browser-batch.js
index 925b8e148..c6674e37b 100644
--- a/src/chrome/src/agent/browser-batch.js
+++ b/src/chrome/src/agent/browser-batch.js
@@ -60,11 +60,21 @@ export function expandBrowserBatchCalls(calls, options) {
});
}
+// A text write an exact reader read back as a different value: WebBrain
+// knows the field is wrong, so the batch must not build on it. A rendered-
+// layout or unreadable readback (readbackExact false or absent) only leaves
+// the write unconfirmed.
+export function textWriteRefuted(result) {
+ return !!result && result.readbackExact === true
+ && (result.verified === false || result.unverified?.reason === 'readback_mismatch');
+}
+
// advisoryWritesContinue: a text write that went out but could not be read
// back exactly (success with unverifiedWrite, see the agent's text-write
// advisories) ran and does not stop the batch. Only that marker is exempt:
-// failures, refusals and unknown outcomes still do. The agent passes false
-// under Settings β†’ "Strict checks", which never produce that marker anyway.
+// failures, refusals, unknown outcomes and a write read back exactly as a
+// different value still do. The agent passes false under Settings β†’
+// "Strict checks", which never produce that marker anyway.
export function browserBatchResultStatus(message, { advisoryWritesContinue = true } = {}) {
if (!message) return { ran: false, reason: 'interrupted' };
let text = String(message.content || '');
@@ -78,7 +88,7 @@ export function browserBatchResultStatus(message, { advisoryWritesContinue = tru
const ran = !(result.skipped || result.denied || result.invalidToolArguments || result.invalidArguments
|| ((result.noDispatch === true || result.dispatched === false) && result.success !== true));
const advisoryWrite = advisoryWritesContinue === true && result.success === true
- && result.unverifiedWrite === true && !result.error;
+ && result.unverifiedWrite === true && !result.error && !textWriteRefuted(result);
const unverified = !advisoryWrite && (result.inconclusive || result.verified === false || result.remoteStateVerified === false);
const failed = !!(result.success === false || result.error || result.noProgress || result.ambiguous
|| result.outcomeUnknown || result.stale || unverified);
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 242e9d1a9..16ab5c3c2 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -21,7 +21,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENT
import { rejectedCompletionRecovery } from './completion-recovery.js';
import { FailureRecoveryRegistry, modelFailureRecovery } from './failure-recovery.js';
import { validateToolArguments } from './tool-arguments.js';
-import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus } from './browser-batch.js';
+import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus, textWriteRefuted } from './browser-batch.js';
import { calculateExpression, calculationResultInText } from './calculator.js';
import { isLiteralCalculationTask, literalCalculationSummaryFormatMatches } from './page-feedback-policy.js';
import { chatRecoveryDisplayText, terminalRecoveryFinalText } from './chat-recovery-text.js';
@@ -12678,10 +12678,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// A text write that went out but could not be read back exactly is an
// advisory result (success + unverifiedWrite + a note), not a reason to
// replan the rest of the batch. Failures, refusals and unknown outcomes
- // still interrupt; so does every unverified result under Strict checks.
+ // still interrupt; so does a write an exact reader read back as a
+ // different value, since the rest of the batch would build on a value
+ // WebBrain knows is wrong, and every unverified result under Strict checks.
_advisoryTextWriteContinuesBatch(toolName, toolResult) {
return ['set_field', 'type_ax', 'type_text'].includes(toolName)
&& toolResult?.success === true && toolResult.unverifiedWrite === true && !toolResult.error
+ && !textWriteRefuted(toolResult)
&& this._guardAdvises('browser-batch-fresh-turn-interrupt', { decided: true });
}
diff --git a/src/firefox/src/agent/browser-batch.js b/src/firefox/src/agent/browser-batch.js
index 925b8e148..c6674e37b 100644
--- a/src/firefox/src/agent/browser-batch.js
+++ b/src/firefox/src/agent/browser-batch.js
@@ -60,11 +60,21 @@ export function expandBrowserBatchCalls(calls, options) {
});
}
+// A text write an exact reader read back as a different value: WebBrain
+// knows the field is wrong, so the batch must not build on it. A rendered-
+// layout or unreadable readback (readbackExact false or absent) only leaves
+// the write unconfirmed.
+export function textWriteRefuted(result) {
+ return !!result && result.readbackExact === true
+ && (result.verified === false || result.unverified?.reason === 'readback_mismatch');
+}
+
// advisoryWritesContinue: a text write that went out but could not be read
// back exactly (success with unverifiedWrite, see the agent's text-write
// advisories) ran and does not stop the batch. Only that marker is exempt:
-// failures, refusals and unknown outcomes still do. The agent passes false
-// under Settings β†’ "Strict checks", which never produce that marker anyway.
+// failures, refusals, unknown outcomes and a write read back exactly as a
+// different value still do. The agent passes false under Settings β†’
+// "Strict checks", which never produce that marker anyway.
export function browserBatchResultStatus(message, { advisoryWritesContinue = true } = {}) {
if (!message) return { ran: false, reason: 'interrupted' };
let text = String(message.content || '');
@@ -78,7 +88,7 @@ export function browserBatchResultStatus(message, { advisoryWritesContinue = tru
const ran = !(result.skipped || result.denied || result.invalidToolArguments || result.invalidArguments
|| ((result.noDispatch === true || result.dispatched === false) && result.success !== true));
const advisoryWrite = advisoryWritesContinue === true && result.success === true
- && result.unverifiedWrite === true && !result.error;
+ && result.unverifiedWrite === true && !result.error && !textWriteRefuted(result);
const unverified = !advisoryWrite && (result.inconclusive || result.verified === false || result.remoteStateVerified === false);
const failed = !!(result.success === false || result.error || result.noProgress || result.ambiguous
|| result.outcomeUnknown || result.stale || unverified);
diff --git a/test/guard-advisory-writes.mjs b/test/guard-advisory-writes.mjs
index 92c4d71a4..bb25671a4 100644
--- a/test/guard-advisory-writes.mjs
+++ b/test/guard-advisory-writes.mjs
@@ -332,6 +332,34 @@ for (const { build, Agent, batch } of builds) {
assert.equal(strictStatus.reason, 'action_unverified');
});
+ test(`${build}: a write an exact reader reads back as a different value stops the rest of the batch`, async () => {
+ // A semantic (exact) reader read another value: WebBrain knows the field
+ // is wrong, so a batched submit must not run on it. The write itself is
+ // still an advised success that locks nothing.
+ const agent = setup(Agent, { write: () => renderedMismatch({
+ unverified: { reason: 'readback_mismatch', reader: 'semantic' }, readbackExact: true }) });
+ const refuted = await agent.executeTool(tabId, 'set_field', args);
+ assert.equal(refuted.success, true);
+ assert.equal(refuted.unverifiedWrite, true);
+ assert.equal(refuted.verified, false);
+ assert.equal(refuted.readbackExact, true);
+ assert.match(advisoryOf(refuted, 'text-write-readback-unverified')?.note || '', /reads back a different value/);
+ assert.equal(agent._uncertainTextMutations.has(tabId), false, 'no text-write debt');
+ const batchContext = { navigationChanged: false, sameNext: true, remainingObservations: false,
+ hadMutation: true, hasRemaining: true, submit: false };
+ assert.equal(agent._browserActionFreshTurnReason('full', 'set_field', refuted, batchContext), 'action_unverified');
+ assert.equal(batch.textWriteRefuted(refuted), true);
+ const status = batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(refuted) });
+ assert.equal(status.failed, true, 'the next batched action must not run on a refuted value');
+ assert.equal(status.reason, 'action_unverified');
+ // Rendered-layout and unreadable readbacks only leave the write unconfirmed.
+ for (const uncertain of [{ ...refuted, readbackExact: false }, { ...refuted, readbackExact: undefined, verified: undefined }]) {
+ assert.equal(batch.textWriteRefuted(uncertain), false);
+ assert.equal(agent._browserActionFreshTurnReason('full', 'set_field', uncertain, batchContext), '');
+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(uncertain) }).failed, false);
+ }
+ });
+
test(`${build}: content readback reports which reader produced the value`, async () => {
const h = contentHarness(build);
// A generic block editor reads back through innerText, the browser's