Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: a success claimed before the calculator ran ends as partial
Barack Sokullub25d447371c1
diff --git a/docs/guards.md b/docs/guards.mdindex cd6428a36..eaf4ac1ba 100644--- a/docs/guards.md+++ b/docs/guards.md@@ -34,7 +34,8 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):or unfinished progress rows, unread pages of a whole thread the userasked for, text that went into a formatting toolbar, an exact table thatcan no longer be delivered, a field read back as another value than was- written to it) ends the run as `partial`, not `success`, whether it ended with `done`, a structured+ written to it, an arithmetic result claimed before the calculator ran)+ ends the run as `partial`, not `success`, whether it ended with `done`, a structuredCloud run's `done_json` or a plain answer. Missing evidence alone keepsthe model's outcome.- A write that went out but could not be read back exactly (GitHub'sdiff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 21946f1b7..c4ec3b583 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -510,7 +510,9 @@ function selectionScopedResponseLanguagePolicy(locale) {// or says nothing about it:// completion-page-warning: the open dialog or form error may be unrelated.// done-summary-meta-only, plan-done-summary-shape: the summary's wording.-// pure-calculation-done-checks: how the summary writes the result.+// pure-calculation-done-checks: how the summary writes the result. A+// success claimed before any calculator attempt is downgraded on its+// own (_guardRefusesOnce's `downgrade`).// exact-table-done: a rowCount that disagrees with rows that may be complete.// A done whose table can no longer be delivered is downgraded where it// is accepted (_resolveExactTableDone).@@ -14726,11 +14728,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis._planExecutionGuards.get(tabId)?.pureCalculationReceipt?.fraction);const prematureCompletion = !policy.stale && name === 'done' && !completed && !policy.attempted;if (!policy.stale && policy.names.includes(name) && matchesInput && !missingResult && !missingFormat && !prematureCompletion) return null;+ // A success claimed before any calculator attempt: WebBrain knows its+ // result was not computed. An honest partial or failed ending is not.+ const prematureSuccess = prematureCompletion && String(args?.outcome || '').trim().toLowerCase() === 'success';// A done these arithmetic checks refuse is advised once per run by// default, then accepted with a caveat. A stale task authority is a// host check and keeps refusing.+ // A premature success is accepted, but not as a success: the run ends+ // as partial.if (!policy.stale && name === 'done' && !this._guardRefusesOnce(tabId, 'pure-calculation-done-checks', {- decided: !prematureCompletion,+ decided: !prematureCompletion || prematureSuccess,+ downgrade: prematureSuccess,caveat: prematureCompletion ? 'The result was not computed with the calculator.': missingResult ? 'The answer may not show the calculator\'s exact result.': 'The answer may not use the exact format you asked for.',@@ -21026,7 +21034,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* the user. Returns true when this call should still refuse: the first* time, or always when the guard blocks in the current mode.*/- _guardRefusesOnce(tabId, id, { decided = true, caveat = '' } = {}) {+ // `downgrade` marks a decided case of a guard whose other decided cases+ // keep the model's outcome (OUTCOME_DOWNGRADING_GUARDS lists whole guards).+ _guardRefusesOnce(tabId, id, { decided = true, caveat = '', downgrade = false } = {}) {if (this._guardOutcome(id, { decided }) !== 'advise') return true;const state = this._guardRunState(tabId);if (!state.advised.has(id)) {@@ -21037,7 +21047,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (text) state.caveats.set(id, text);// A guard that KNOWS the task is not done (not one that only lacked// evidence) lets the completion end the run, but not as a success.- if (decided === true && OUTCOME_DOWNGRADING_GUARDS.has(id)) (state.outcomeDowngrades ||= new Set()).add(id);+ if (decided === true && (downgrade === true || OUTCOME_DOWNGRADING_GUARDS.has(id))) {+ (state.outcomeDowngrades ||= new Set()).add(id);+ }return false;}diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex e31ae3129..0a5278d29 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -345,7 +345,9 @@ function selectionScopedResponseLanguagePolicy(locale) {// or says nothing about it:// completion-page-warning: the open dialog or form error may be unrelated.// done-summary-meta-only, plan-done-summary-shape: the summary's wording.-// pure-calculation-done-checks: how the summary writes the result.+// pure-calculation-done-checks: how the summary writes the result. A+// success claimed before any calculator attempt is downgraded on its+// own (_guardRefusesOnce's `downgrade`).// exact-table-done: a rowCount that disagrees with rows that may be complete.// A done whose table can no longer be delivered is downgraded where it// is accepted (_resolveExactTableDone).@@ -13091,11 +13093,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis._planExecutionGuards.get(tabId)?.pureCalculationReceipt?.fraction);const prematureCompletion = !policy.stale && name === 'done' && !completed && !policy.attempted;if (!policy.stale && policy.names.includes(name) && matchesInput && !missingResult && !missingFormat && !prematureCompletion) return null;+ // A success claimed before any calculator attempt: WebBrain knows its+ // result was not computed. An honest partial or failed ending is not.+ const prematureSuccess = prematureCompletion && String(args?.outcome || '').trim().toLowerCase() === 'success';// A done these arithmetic checks refuse is advised once per run by// default, then accepted with a caveat. A stale task authority is a// host check and keeps refusing.+ // A premature success is accepted, but not as a success: the run ends+ // as partial.if (!policy.stale && name === 'done' && !this._guardRefusesOnce(tabId, 'pure-calculation-done-checks', {- decided: !prematureCompletion,+ decided: !prematureCompletion || prematureSuccess,+ downgrade: prematureSuccess,caveat: prematureCompletion ? 'The result was not computed with the calculator.': missingResult ? 'The answer may not show the calculator\'s exact result.': 'The answer may not use the exact format you asked for.',@@ -18189,7 +18197,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d* the user. Returns true when this call should still refuse: the first* time, or always when the guard blocks in the current mode.*/- _guardRefusesOnce(tabId, id, { decided = true, caveat = '' } = {}) {+ // `downgrade` marks a decided case of a guard whose other decided cases+ // keep the model's outcome (OUTCOME_DOWNGRADING_GUARDS lists whole guards).+ _guardRefusesOnce(tabId, id, { decided = true, caveat = '', downgrade = false } = {}) {if (this._guardOutcome(id, { decided }) !== 'advise') return true;const state = this._guardRunState(tabId);if (!state.advised.has(id)) {@@ -18200,7 +18210,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (text) state.caveats.set(id, text);// A guard that KNOWS the task is not done (not one that only lacked// evidence) lets the completion end the run, but not as a success.- if (decided === true && OUTCOME_DOWNGRADING_GUARDS.has(id)) (state.outcomeDowngrades ||= new Set()).add(id);+ if (decided === true && (downgrade === true || OUTCOME_DOWNGRADING_GUARDS.has(id))) {+ (state.outcomeDowngrades ||= new Set()).add(id);+ }return false;}diff --git a/test/guard-advisory-completion.mjs b/test/guard-advisory-completion.mjsindex 0b539fcb0..59238ce66 100644--- a/test/guard-advisory-completion.mjs+++ b/test/guard-advisory-completion.mjs@@ -958,6 +958,28 @@ for (const build of ['chrome', 'firefox']) {}});+ test(`${build}: a success claimed before any calculator attempt is refused once, then ends as partial`, async () => {+ for (const strict of [false, true]) {+ const h = batchAgent(Agent, { strict });+ h.agent._pureCalculationToolPolicy = () => ({ scope: { owner: 'owned', runToken: 'run', expression: '2+2' },+ stale: false, attempted: false, names: ['calculate', 'done'] });+ const done = ['done', { summary: '2+2 is 4.', outcome: 'success' }];+ assert.equal((await h.batch([done])).action, 'continue', `${build}: premature done gets one note`);+ const updates = [];+ const second = await h.batch([done], updates);+ if (strict) {+ assert.equal(second.action, 'continue', `${build}: ${STRICT_NOTE}`);+ assert.equal(h.executed.length, 0);+ continue;+ }+ assert.equal(second.action, 'return');+ assert.equal(second.value, `2+2 is 4.\n\n${CAVEAT}The result was not computed with the calculator.`);+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;+ assert.equal(doneResult?.outcome, 'partial', `${build}: an uncomputed success was not downgraded`);+ assert.equal(second.endStatus, 'partial');+ }+ });+// ββ Meta-only done summary (Ask) βββββββββββββββββββββββββββββββββββββββtest(`${build}: a summary that only says an answer was given is advised once`, async () => {await refusedOnceThenAccepted(Agent, {