Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard policy: a success claimed before the calculator ran ends as partial

Barack Sokullub25d447371c1
diff --git a/docs/guards.md b/docs/guards.md
index cd6428a36..eaf4ac1ba 100644
--- a/docs/guards.md
+++ b/docs/guards.md
@@ -34,7 +34,8 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):
or unfinished progress rows, unread pages of a whole thread the user
asked for, text that went into a formatting toolbar, an exact table that
can no longer be delivered, a field read back as another value than was
- written to it) ends the run as `partial`, not `success`, whether it ended with `done`, a structured
+ written to it, an arithmetic result claimed before the calculator ran)
+ ends the run as `partial`, not `success`, whether it ended with `done`, a structured
Cloud run's `done_json` or a plain answer. Missing evidence alone keeps
the model's outcome.
- A write that went out but could not be read back exactly (GitHub's
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 21946f1b7..c4ec3b583 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -510,7 +510,9 @@ function selectionScopedResponseLanguagePolicy(locale) {
// or says nothing about it:
// completion-page-warning: the open dialog or form error may be unrelated.
// done-summary-meta-only, plan-done-summary-shape: the summary's wording.
-// pure-calculation-done-checks: how the summary writes the result.
+// pure-calculation-done-checks: how the summary writes the result. A
+// success claimed before any calculator attempt is downgraded on its
+// own (_guardRefusesOnce's `downgrade`).
// exact-table-done: a rowCount that disagrees with rows that may be complete.
// A done whose table can no longer be delivered is downgraded where it
// is accepted (_resolveExactTableDone).
@@ -14726,11 +14728,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this._planExecutionGuards.get(tabId)?.pureCalculationReceipt?.fraction);
const prematureCompletion = !policy.stale && name === 'done' && !completed && !policy.attempted;
if (!policy.stale && policy.names.includes(name) && matchesInput && !missingResult && !missingFormat && !prematureCompletion) return null;
+ // A success claimed before any calculator attempt: WebBrain knows its
+ // result was not computed. An honest partial or failed ending is not.
+ const prematureSuccess = prematureCompletion && String(args?.outcome || '').trim().toLowerCase() === 'success';
// A done these arithmetic checks refuse is advised once per run by
// default, then accepted with a caveat. A stale task authority is a
// host check and keeps refusing.
+ // A premature success is accepted, but not as a success: the run ends
+ // as partial.
if (!policy.stale && name === 'done' && !this._guardRefusesOnce(tabId, 'pure-calculation-done-checks', {
- decided: !prematureCompletion,
+ decided: !prematureCompletion || prematureSuccess,
+ downgrade: prematureSuccess,
caveat: prematureCompletion ? 'The result was not computed with the calculator.'
: missingResult ? 'The answer may not show the calculator\'s exact result.'
: 'The answer may not use the exact format you asked for.',
@@ -21026,7 +21034,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* the user. Returns true when this call should still refuse: the first
* time, or always when the guard blocks in the current mode.
*/
- _guardRefusesOnce(tabId, id, { decided = true, caveat = '' } = {}) {
+ // `downgrade` marks a decided case of a guard whose other decided cases
+ // keep the model's outcome (OUTCOME_DOWNGRADING_GUARDS lists whole guards).
+ _guardRefusesOnce(tabId, id, { decided = true, caveat = '', downgrade = false } = {}) {
if (this._guardOutcome(id, { decided }) !== 'advise') return true;
const state = this._guardRunState(tabId);
if (!state.advised.has(id)) {
@@ -21037,7 +21047,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (text) state.caveats.set(id, text);
// A guard that KNOWS the task is not done (not one that only lacked
// evidence) lets the completion end the run, but not as a success.
- if (decided === true && OUTCOME_DOWNGRADING_GUARDS.has(id)) (state.outcomeDowngrades ||= new Set()).add(id);
+ if (decided === true && (downgrade === true || OUTCOME_DOWNGRADING_GUARDS.has(id))) {
+ (state.outcomeDowngrades ||= new Set()).add(id);
+ }
return false;
}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index e31ae3129..0a5278d29 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -345,7 +345,9 @@ function selectionScopedResponseLanguagePolicy(locale) {
// or says nothing about it:
// completion-page-warning: the open dialog or form error may be unrelated.
// done-summary-meta-only, plan-done-summary-shape: the summary's wording.
-// pure-calculation-done-checks: how the summary writes the result.
+// pure-calculation-done-checks: how the summary writes the result. A
+// success claimed before any calculator attempt is downgraded on its
+// own (_guardRefusesOnce's `downgrade`).
// exact-table-done: a rowCount that disagrees with rows that may be complete.
// A done whose table can no longer be delivered is downgraded where it
// is accepted (_resolveExactTableDone).
@@ -13091,11 +13093,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this._planExecutionGuards.get(tabId)?.pureCalculationReceipt?.fraction);
const prematureCompletion = !policy.stale && name === 'done' && !completed && !policy.attempted;
if (!policy.stale && policy.names.includes(name) && matchesInput && !missingResult && !missingFormat && !prematureCompletion) return null;
+ // A success claimed before any calculator attempt: WebBrain knows its
+ // result was not computed. An honest partial or failed ending is not.
+ const prematureSuccess = prematureCompletion && String(args?.outcome || '').trim().toLowerCase() === 'success';
// A done these arithmetic checks refuse is advised once per run by
// default, then accepted with a caveat. A stale task authority is a
// host check and keeps refusing.
+ // A premature success is accepted, but not as a success: the run ends
+ // as partial.
if (!policy.stale && name === 'done' && !this._guardRefusesOnce(tabId, 'pure-calculation-done-checks', {
- decided: !prematureCompletion,
+ decided: !prematureCompletion || prematureSuccess,
+ downgrade: prematureSuccess,
caveat: prematureCompletion ? 'The result was not computed with the calculator.'
: missingResult ? 'The answer may not show the calculator\'s exact result.'
: 'The answer may not use the exact format you asked for.',
@@ -18189,7 +18197,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
* the user. Returns true when this call should still refuse: the first
* time, or always when the guard blocks in the current mode.
*/
- _guardRefusesOnce(tabId, id, { decided = true, caveat = '' } = {}) {
+ // `downgrade` marks a decided case of a guard whose other decided cases
+ // keep the model's outcome (OUTCOME_DOWNGRADING_GUARDS lists whole guards).
+ _guardRefusesOnce(tabId, id, { decided = true, caveat = '', downgrade = false } = {}) {
if (this._guardOutcome(id, { decided }) !== 'advise') return true;
const state = this._guardRunState(tabId);
if (!state.advised.has(id)) {
@@ -18200,7 +18210,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (text) state.caveats.set(id, text);
// A guard that KNOWS the task is not done (not one that only lacked
// evidence) lets the completion end the run, but not as a success.
- if (decided === true && OUTCOME_DOWNGRADING_GUARDS.has(id)) (state.outcomeDowngrades ||= new Set()).add(id);
+ if (decided === true && (downgrade === true || OUTCOME_DOWNGRADING_GUARDS.has(id))) {
+ (state.outcomeDowngrades ||= new Set()).add(id);
+ }
return false;
}
diff --git a/test/guard-advisory-completion.mjs b/test/guard-advisory-completion.mjs
index 0b539fcb0..59238ce66 100644
--- a/test/guard-advisory-completion.mjs
+++ b/test/guard-advisory-completion.mjs
@@ -958,6 +958,28 @@ for (const build of ['chrome', 'firefox']) {
}
});
+ test(`${build}: a success claimed before any calculator attempt is refused once, then ends as partial`, async () => {
+ for (const strict of [false, true]) {
+ const h = batchAgent(Agent, { strict });
+ h.agent._pureCalculationToolPolicy = () => ({ scope: { owner: 'owned', runToken: 'run', expression: '2+2' },
+ stale: false, attempted: false, names: ['calculate', 'done'] });
+ const done = ['done', { summary: '2+2 is 4.', outcome: 'success' }];
+ assert.equal((await h.batch([done])).action, 'continue', `${build}: premature done gets one note`);
+ const updates = [];
+ const second = await h.batch([done], updates);
+ if (strict) {
+ assert.equal(second.action, 'continue', `${build}: ${STRICT_NOTE}`);
+ assert.equal(h.executed.length, 0);
+ continue;
+ }
+ assert.equal(second.action, 'return');
+ assert.equal(second.value, `2+2 is 4.\n\n${CAVEAT}The result was not computed with the calculator.`);
+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;
+ assert.equal(doneResult?.outcome, 'partial', `${build}: an uncomputed success was not downgraded`);
+ assert.equal(second.endStatus, 'partial');
+ }
+ });
+
// ── Meta-only done summary (Ask) ───────────────────────────────────────
test(`${build}: a summary that only says an answer was given is advised once`, async () => {
await refusedOnceThenAccepted(Agent, {