Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: a done whose exact table cannot be delivered ends as partial
Barack Sokullu7248537e09a6
diff --git a/docs/guards.md b/docs/guards.mdindex 946d96c02..4638d7936 100644--- a/docs/guards.md+++ b/docs/guards.md@@ -32,8 +32,8 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):- A completion accepted over a guard that knows the claim is wrong (thecompletion judge's "not done", an unauthorized mutation, missing, failedor unfinished progress rows, unread pages of a whole thread the user- asked for, text that went into a formatting toolbar) ends the run as- `partial`, not `success`, whether it ended with `done`, a structured+ asked for, text that went into a formatting toolbar, an exact table that+ can no longer be delivered) ends the run as `partial`, not `success`, whether it ended with `done`, a structuredCloud run's `done_json` or a plain answer. Missing evidence alone keepsthe model's outcome.- A write that went out but could not be read back exactly (GitHub'sdiff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 100a966ac..d19833660 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -510,6 +510,8 @@ function selectionScopedResponseLanguagePolicy(locale) {// done-summary-meta-only, plan-done-summary-shape: the summary's wording.// pure-calculation-done-checks: how the summary writes the result.// exact-table-done: a rowCount that disagrees with rows that may be complete.+// A done whose table can no longer be delivered is downgraded where it+// is accepted (_resolveExactTableDone).// plain-final-done-protocol, plan-plain-final-protocol,// plain-final-unusable-output: how the answer ended, not the task.// incomplete-task-redirect: it only sees partial or failed endings.@@ -1411,13 +1413,16 @@ export class Agent extends LoopDetector {const record = state?.exactTableExports?.get(args.data_ref);const owner = this._exactTableOwner(tabId);if (!record || !record.owner.current() || !owner?.current() || record.runToken !== owner.runToken) {+ if (this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,+ caveat: 'The table reference had expired, so the exact table could not be delivered.' })) {+ return { success: false, blockedDone: true, exactTableUnverified: true,+ error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' };+ }// Accepted, it delivers only the model's own summary, never rows- // another task or run extracted.- return this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,- caveat: 'The table reference had expired, so the exact table could not be delivered.' })- ? { success: false, blockedDone: true, exactTableUnverified: true,- error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' }- : null;+ // another task or run extracted. The table the task promised is then+ // known not to be delivered, so the run ends as partial.+ this._downgradeCompletionOutcome(tabId, 'exact-table-done');+ return null;}onUpdate?.('tool_call', { name: 'extract_data', args: record.args, verification: true });const verified = await this.executeTool(tabId, 'extract_data', record.args, onUpdate,@@ -1431,8 +1436,10 @@ export class Agent extends LoopDetector {}// Accepted: the stored snapshot is delivered while its task still owns// it, unless Strict secret mode now applies (the snapshot predates it,- // and its re-projection may be what failed).+ // and its re-projection may be what failed). A table that is not+ // delivered ends the run as partial.if (owner.current() && this.strictSecretMode !== true) args.summary = record.summary;+ else this._downgradeCompletionOutcome(tabId, 'exact-table-done');return null;}args.summary = record.summary;@@ -5323,6 +5330,14 @@ export class Agent extends LoopDetector {return (this._guardRunState(tabId).outcomeDowngrades?.size || 0) > 0;}+ // A completion accepted although WebBrain knows its result is incomplete,+ // where the guard's other cases only lack evidence (exact-table-done: a+ // table that can no longer be delivered). It ends the run as partial, as+ // one accepted over an OUTCOME_DOWNGRADING_GUARDS check does.+ _downgradeCompletionOutcome(tabId, id) {+ (this._guardRunState(tabId).outcomeDowngrades ||= new Set()).add(id);+ }+// native-chat-receipt-not-durable for a plain final. A plain final in an// Act or Dev run is the model's own success claim, and without Strict// checks it can end a run where done would have run this check: a chat@@ -16701,8 +16716,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}if (!scheduledResume) await cancelPendingResumes();// done_json is always a success claim; its result carries no outcome.+ // So is an Ask answer's done, whose schema has no outcome.const claimsSuccess = toolResult.outcome === 'success' || (fnName === 'done_json'- && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined);+ && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined)+ || (fnName === 'done' && toolResult.done === true && toolResult.outcome == null+ && !this._isActionMode(this._effectiveRunMode(tabId)));let doneEndStatus = null;if (!scheduledResume && claimsSuccess && this._completionOutcomeDowngraded(tabId)) {// Accepted over a guard that knows the claim is wrong: end the run,diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex ec3241a07..4fdfccff8 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -345,6 +345,8 @@ function selectionScopedResponseLanguagePolicy(locale) {// done-summary-meta-only, plan-done-summary-shape: the summary's wording.// pure-calculation-done-checks: how the summary writes the result.// exact-table-done: a rowCount that disagrees with rows that may be complete.+// A done whose table can no longer be delivered is downgraded where it+// is accepted (_resolveExactTableDone).// plain-final-done-protocol, plan-plain-final-protocol: how the answer// ended, not the task.// incomplete-task-redirect: it only sees partial or failed endings.@@ -1043,13 +1045,16 @@ export class Agent extends LoopDetector {const record = state?.exactTableExports?.get(args.data_ref);const owner = this._exactTableOwner(tabId);if (!record || !record.owner.current() || !owner?.current() || record.runToken !== owner.runToken) {+ if (this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,+ caveat: 'The table reference had expired, so the exact table could not be delivered.' })) {+ return { success: false, blockedDone: true, exactTableUnverified: true,+ error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' };+ }// Accepted, it delivers only the model's own summary, never rows- // another task or run extracted.- return this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,- caveat: 'The table reference had expired, so the exact table could not be delivered.' })- ? { success: false, blockedDone: true, exactTableUnverified: true,- error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' }- : null;+ // another task or run extracted. The table the task promised is then+ // known not to be delivered, so the run ends as partial.+ this._downgradeCompletionOutcome(tabId, 'exact-table-done');+ return null;}onUpdate?.('tool_call', { name: 'extract_data', args: record.args, verification: true });const verified = await this.executeTool(tabId, 'extract_data', record.args, onUpdate,@@ -1063,8 +1068,10 @@ export class Agent extends LoopDetector {}// Accepted: the stored snapshot is delivered while its task still owns// it, unless Strict secret mode now applies (the snapshot predates it,- // and its re-projection may be what failed).+ // and its re-projection may be what failed). A table that is not+ // delivered ends the run as partial.if (owner.current() && this.strictSecretMode !== true) args.summary = record.summary;+ else this._downgradeCompletionOutcome(tabId, 'exact-table-done');return null;}args.summary = record.summary;@@ -4875,6 +4882,14 @@ export class Agent extends LoopDetector {return (this._guardRunState(tabId).outcomeDowngrades?.size || 0) > 0;}+ // A completion accepted although WebBrain knows its result is incomplete,+ // where the guard's other cases only lack evidence (exact-table-done: a+ // table that can no longer be delivered). It ends the run as partial, as+ // one accepted over an OUTCOME_DOWNGRADING_GUARDS check does.+ _downgradeCompletionOutcome(tabId, id) {+ (this._guardRunState(tabId).outcomeDowngrades ||= new Set()).add(id);+ }+// native-chat-receipt-not-durable for a plain final. A plain final in an// Act or Dev run is the model's own success claim, and without Strict// checks it can end a run where done would have run this check: a chat@@ -14968,8 +14983,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}if (!scheduledResume) await cancelPendingResumes();// done_json is always a success claim; its result carries no outcome.+ // So is an Ask answer's done, whose schema has no outcome.const claimsSuccess = toolResult.outcome === 'success' || (fnName === 'done_json'- && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined);+ && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined)+ || (fnName === 'done' && toolResult.done === true && toolResult.outcome == null+ && !this._isActionMode(this._effectiveRunMode(tabId)));let doneEndStatus = null;if (!scheduledResume && claimsSuccess && this._completionOutcomeDowngraded(tabId)) {// Accepted over a guard that knows the claim is wrong: end the run,diff --git a/test/guard-advisory-completion.mjs b/test/guard-advisory-completion.mjsindex 84a6a8bdf..0d7fe7eef 100644--- a/test/guard-advisory-completion.mjs+++ b/test/guard-advisory-completion.mjs@@ -718,6 +718,24 @@ for (const build of ['chrome', 'firefox']) {}});+ test(`${build}: an Ask answer accepted over unread pages of a requested whole read ends as partial`, async () => {+ // An Ask done has no outcome of its own; it claims the answer is complete.+ const h = batchAgent(Agent, { mode: 'ask', task: 'Summarize this whole conversation.' });+ h.agent._currentUrl = async () => 'https://outlook.live.com/mail/0/inbox/id/AQMkADAwATM0MDAAMS1i';+ await h.agent._beginReadCompleteness(h.tab, 'Summarize this whole conversation.', {});+ h.agent._armReadCompletenessFromPlan(h.tab, { request_kind: 'execute', read_scope: 'complete_thread' });+ h.agent._recordReadCompleteness(h.tab, 'get_accessibility_tree', { filter: 'all', maxDepth: 15 }, {+ pageContent: 'main\n listitem "First message"', page: 1, totalChars: 9000, hasMore: true, truncated: false, nextPage: 2 });+ const done = ['done', { summary: 'The thread settles the launch date.' }];+ assert.equal((await h.batch([done])).action, 'continue', `${build}: the first answer gets its coverage note`);+ const updates = [];+ const accepted = await h.batch([done], updates);+ assert.equal(accepted.action, 'return', `${build}: the second answer is accepted`);+ assert.equal(accepted.endStatus, 'partial', `${build}: the run must end as partial`);+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;+ assert.equal(doneResult?.outcome, 'partial', `${build}: an answer over unread pages was not downgraded`);+ });+// ββ Exact table export βββββββββββββββββββββββββββββββββββββββββββββββββtest(`${build}: table JSON without an exact extract, and an expired table reference, are advised once`, async () => {const json = JSON.stringify({ rowCount: 2, rows: [{ id: 'A' }, { id: 'B' }] });@@ -733,6 +751,40 @@ for (const build of ['chrome', 'firefox']) {'The table reference had expired, so the exact table could not be delivered.', `${build} table ref`);});+ test(`${build}: a done whose exact table can no longer be delivered ends as partial`, async () => {+ // The reference expired: the accepted done delivers only the model's+ // summary, never the promised rows, and WebBrain knows it.+ const askSchemas = new Map(getToolsForMode('ask', { tier: 'full' }).map(tool => [tool.function.name, tool.function.parameters]));+ const h = batchAgent(Agent, { mode: 'ask', realDone: true, schemas: askSchemas });+ h.agent.readCompletenessStates.set(h.tab, { runToken: 'owned-run', tablePageObserved: true });+ // An Ask answer's done has no outcome of its own; it claims the answer.+ const done = ['done', { summary: 'Full inventory.', data_ref: 'table_expired' }];+ assert.equal((await h.batch([done])).action, 'continue', `${build}: the expired reference is refused once`);+ const updates = [];+ const accepted = await h.batch([done], updates);+ assert.equal(accepted.action, 'return', `${build}: the second completion is accepted`);+ assert.equal(accepted.endStatus, 'partial', `${build}: the run must end as partial`);+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;+ assert.equal(doneResult?.outcome, 'partial', `${build}: an undelivered table was reported as a success`);+ // A stale snapshot that is still delivered keeps the model's outcome; one+ // that Strict secret mode withholds does not.+ for (const strictSecretMode of [false, true]) {+ const s = batchAgent(Agent, { mode: 'ask' });+ s.agent.strictSecretMode = strictSecretMode;+ const owner = { runToken: 'owned-run', current: () => true };+ const record = { dataRef: 'table_owned', runToken: 'owned-run', owner, args: { columns: ['id'] }, summary: '[{"id":"A"}]' };+ s.agent.readCompletenessStates.set(s.tab, { runToken: 'owned-run', exactTableExports: new Map([['table_owned', record]]) });+ s.agent._exactTableOwner = () => owner;+ s.agent.executeTool = async () => ({ success: false, error: 'The source table changed.' });+ const args = () => ({ summary: 'Full inventory.', data_ref: 'table_owned' });+ s.agent._beginCompletionGuardAttempt(s.tab);+ assert.equal((await s.agent._resolveExactTableDone(s.tab, args()))?.blockedDone, true);+ s.agent._beginCompletionGuardAttempt(s.tab);+ assert.equal(await s.agent._resolveExactTableDone(s.tab, args()), null);+ assert.equal(s.agent._completionOutcomeDowngraded(s.tab), strictSecretMode, `${build}: strictSecretMode=${strictSecretMode}`);+ }+ });+test(`${build}: an unverifiable table snapshot is delivered over advice, but not under Strict secret mode`, async () => {for (const strictSecretMode of [false, true]) {const h = batchAgent(Agent, { mode: 'ask' });