Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: a write read back exactly as a different value ends the batch
Barack Sokullucbb79481e998
diff --git a/docs/guards.md b/docs/guards.mdindex 8a69d03ac..f3166689f 100644--- a/docs/guards.md+++ b/docs/guards.md@@ -38,7 +38,10 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):the model's outcome.- A write that went out but could not be read back exactly (GitHub'sCodeMirror editor is the usual case) is an unverified success: it locks- nothing, arms no text-write debt and is not a loop failure.+ nothing, arms no text-write debt and is not a loop failure. A write that+ an exact reader reads back as a different value is reported the same+ way, but it ends a batch: the next action is planned once the model has+ seen it.- A quality guard whose `onCannotDecide` is `ask` asks the user when itcannot verify a consequential step, such as a commit or a post. Wherethat guard has no card, it refuses.diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex b90162a9b..4569b43ad 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -22,7 +22,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENTimport { rejectedCompletionRecovery } from './completion-recovery.js';import { FailureRecoveryRegistry, modelFailureRecovery } from './failure-recovery.js';import { validateToolArguments } from './tool-arguments.js';-import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus } from './browser-batch.js';+import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus, textWriteRefuted } from './browser-batch.js';import { calculateExpression, calculationResultInText } from './calculator.js';import { isLiteralCalculationTask, literalCalculationSummaryFormatMatches, isPassivePageFeedback } from './page-feedback-policy.js';import { chatRecoveryDisplayText, terminalRecoveryFinalText } from './chat-recovery-text.js';@@ -14318,10 +14318,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// A text write that went out but could not be read back exactly is an// advisory result (success + unverifiedWrite + a note), not a reason to// replan the rest of the batch. Failures, refusals and unknown outcomes- // still interrupt; so does every unverified result under Strict checks.+ // still interrupt; so does a write an exact reader read back as a+ // different value, since the rest of the batch would build on a value+ // WebBrain knows is wrong, and every unverified result under Strict checks._advisoryTextWriteContinuesBatch(toolName, toolResult) {return ['set_field', 'type_ax', 'type_text'].includes(toolName)&& toolResult?.success === true && toolResult.unverifiedWrite === true && !toolResult.error+ && !textWriteRefuted(toolResult)&& this._guardAdvises('browser-batch-fresh-turn-interrupt', { decided: true });}diff --git a/src/chrome/src/agent/browser-batch.js b/src/chrome/src/agent/browser-batch.jsindex 925b8e148..c6674e37b 100644--- a/src/chrome/src/agent/browser-batch.js+++ b/src/chrome/src/agent/browser-batch.js@@ -60,11 +60,21 @@ export function expandBrowserBatchCalls(calls, options) {});}+// A text write an exact reader read back as a different value: WebBrain+// knows the field is wrong, so the batch must not build on it. A rendered-+// layout or unreadable readback (readbackExact false or absent) only leaves+// the write unconfirmed.+export function textWriteRefuted(result) {+ return !!result && result.readbackExact === true+ && (result.verified === false || result.unverified?.reason === 'readback_mismatch');+}+// advisoryWritesContinue: a text write that went out but could not be read// back exactly (success with unverifiedWrite, see the agent's text-write// advisories) ran and does not stop the batch. Only that marker is exempt:-// failures, refusals and unknown outcomes still do. The agent passes false-// under Settings β "Strict checks", which never produce that marker anyway.+// failures, refusals, unknown outcomes and a write read back exactly as a+// different value still do. The agent passes false under Settings β+// "Strict checks", which never produce that marker anyway.export function browserBatchResultStatus(message, { advisoryWritesContinue = true } = {}) {if (!message) return { ran: false, reason: 'interrupted' };let text = String(message.content || '');@@ -78,7 +88,7 @@ export function browserBatchResultStatus(message, { advisoryWritesContinue = truconst ran = !(result.skipped || result.denied || result.invalidToolArguments || result.invalidArguments|| ((result.noDispatch === true || result.dispatched === false) && result.success !== true));const advisoryWrite = advisoryWritesContinue === true && result.success === true- && result.unverifiedWrite === true && !result.error;+ && result.unverifiedWrite === true && !result.error && !textWriteRefuted(result);const unverified = !advisoryWrite && (result.inconclusive || result.verified === false || result.remoteStateVerified === false);const failed = !!(result.success === false || result.error || result.noProgress || result.ambiguous|| result.outcomeUnknown || result.stale || unverified);diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 242e9d1a9..16ab5c3c2 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -21,7 +21,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENTimport { rejectedCompletionRecovery } from './completion-recovery.js';import { FailureRecoveryRegistry, modelFailureRecovery } from './failure-recovery.js';import { validateToolArguments } from './tool-arguments.js';-import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus } from './browser-batch.js';+import { BATCH_INTERMEDIATE_RESULT_CHARS, BATCH_TRAILING_OBSERVATION_TOOLS, expandBrowserBatchCalls, foldBrowserBatchMessages, digestBrowserBatch, browserBatchResultStatus, textWriteRefuted } from './browser-batch.js';import { calculateExpression, calculationResultInText } from './calculator.js';import { isLiteralCalculationTask, literalCalculationSummaryFormatMatches } from './page-feedback-policy.js';import { chatRecoveryDisplayText, terminalRecoveryFinalText } from './chat-recovery-text.js';@@ -12678,10 +12678,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// A text write that went out but could not be read back exactly is an// advisory result (success + unverifiedWrite + a note), not a reason to// replan the rest of the batch. Failures, refusals and unknown outcomes- // still interrupt; so does every unverified result under Strict checks.+ // still interrupt; so does a write an exact reader read back as a+ // different value, since the rest of the batch would build on a value+ // WebBrain knows is wrong, and every unverified result under Strict checks._advisoryTextWriteContinuesBatch(toolName, toolResult) {return ['set_field', 'type_ax', 'type_text'].includes(toolName)&& toolResult?.success === true && toolResult.unverifiedWrite === true && !toolResult.error+ && !textWriteRefuted(toolResult)&& this._guardAdvises('browser-batch-fresh-turn-interrupt', { decided: true });}diff --git a/src/firefox/src/agent/browser-batch.js b/src/firefox/src/agent/browser-batch.jsindex 925b8e148..c6674e37b 100644--- a/src/firefox/src/agent/browser-batch.js+++ b/src/firefox/src/agent/browser-batch.js@@ -60,11 +60,21 @@ export function expandBrowserBatchCalls(calls, options) {});}+// A text write an exact reader read back as a different value: WebBrain+// knows the field is wrong, so the batch must not build on it. A rendered-+// layout or unreadable readback (readbackExact false or absent) only leaves+// the write unconfirmed.+export function textWriteRefuted(result) {+ return !!result && result.readbackExact === true+ && (result.verified === false || result.unverified?.reason === 'readback_mismatch');+}+// advisoryWritesContinue: a text write that went out but could not be read// back exactly (success with unverifiedWrite, see the agent's text-write// advisories) ran and does not stop the batch. Only that marker is exempt:-// failures, refusals and unknown outcomes still do. The agent passes false-// under Settings β "Strict checks", which never produce that marker anyway.+// failures, refusals, unknown outcomes and a write read back exactly as a+// different value still do. The agent passes false under Settings β+// "Strict checks", which never produce that marker anyway.export function browserBatchResultStatus(message, { advisoryWritesContinue = true } = {}) {if (!message) return { ran: false, reason: 'interrupted' };let text = String(message.content || '');@@ -78,7 +88,7 @@ export function browserBatchResultStatus(message, { advisoryWritesContinue = truconst ran = !(result.skipped || result.denied || result.invalidToolArguments || result.invalidArguments|| ((result.noDispatch === true || result.dispatched === false) && result.success !== true));const advisoryWrite = advisoryWritesContinue === true && result.success === true- && result.unverifiedWrite === true && !result.error;+ && result.unverifiedWrite === true && !result.error && !textWriteRefuted(result);const unverified = !advisoryWrite && (result.inconclusive || result.verified === false || result.remoteStateVerified === false);const failed = !!(result.success === false || result.error || result.noProgress || result.ambiguous|| result.outcomeUnknown || result.stale || unverified);diff --git a/test/guard-advisory-writes.mjs b/test/guard-advisory-writes.mjsindex 92c4d71a4..bb25671a4 100644--- a/test/guard-advisory-writes.mjs+++ b/test/guard-advisory-writes.mjs@@ -332,6 +332,34 @@ for (const { build, Agent, batch } of builds) {assert.equal(strictStatus.reason, 'action_unverified');});+ test(`${build}: a write an exact reader reads back as a different value stops the rest of the batch`, async () => {+ // A semantic (exact) reader read another value: WebBrain knows the field+ // is wrong, so a batched submit must not run on it. The write itself is+ // still an advised success that locks nothing.+ const agent = setup(Agent, { write: () => renderedMismatch({+ unverified: { reason: 'readback_mismatch', reader: 'semantic' }, readbackExact: true }) });+ const refuted = await agent.executeTool(tabId, 'set_field', args);+ assert.equal(refuted.success, true);+ assert.equal(refuted.unverifiedWrite, true);+ assert.equal(refuted.verified, false);+ assert.equal(refuted.readbackExact, true);+ assert.match(advisoryOf(refuted, 'text-write-readback-unverified')?.note || '', /reads back a different value/);+ assert.equal(agent._uncertainTextMutations.has(tabId), false, 'no text-write debt');+ const batchContext = { navigationChanged: false, sameNext: true, remainingObservations: false,+ hadMutation: true, hasRemaining: true, submit: false };+ assert.equal(agent._browserActionFreshTurnReason('full', 'set_field', refuted, batchContext), 'action_unverified');+ assert.equal(batch.textWriteRefuted(refuted), true);+ const status = batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(refuted) });+ assert.equal(status.failed, true, 'the next batched action must not run on a refuted value');+ assert.equal(status.reason, 'action_unverified');+ // Rendered-layout and unreadable readbacks only leave the write unconfirmed.+ for (const uncertain of [{ ...refuted, readbackExact: false }, { ...refuted, readbackExact: undefined, verified: undefined }]) {+ assert.equal(batch.textWriteRefuted(uncertain), false);+ assert.equal(agent._browserActionFreshTurnReason('full', 'set_field', uncertain, batchContext), '');+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(uncertain) }).failed, false);+ }+ });+test(`${build}: content readback reports which reader produced the value`, async () => {const h = contentHarness(build);// A generic block editor reads back through innerText, the browser's