Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard policy: a done whose exact table cannot be delivered ends as partial

Barack Sokullu7248537e09a6
diff --git a/docs/guards.md b/docs/guards.md
index 946d96c02..4638d7936 100644
--- a/docs/guards.md
+++ b/docs/guards.md
@@ -32,8 +32,8 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):
- A completion accepted over a guard that knows the claim is wrong (the
completion judge's "not done", an unauthorized mutation, missing, failed
or unfinished progress rows, unread pages of a whole thread the user
- asked for, text that went into a formatting toolbar) ends the run as
- `partial`, not `success`, whether it ended with `done`, a structured
+ asked for, text that went into a formatting toolbar, an exact table that
+ can no longer be delivered) ends the run as `partial`, not `success`, whether it ended with `done`, a structured
Cloud run's `done_json` or a plain answer. Missing evidence alone keeps
the model's outcome.
- A write that went out but could not be read back exactly (GitHub's
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 100a966ac..d19833660 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -510,6 +510,8 @@ function selectionScopedResponseLanguagePolicy(locale) {
// done-summary-meta-only, plan-done-summary-shape: the summary's wording.
// pure-calculation-done-checks: how the summary writes the result.
// exact-table-done: a rowCount that disagrees with rows that may be complete.
+// A done whose table can no longer be delivered is downgraded where it
+// is accepted (_resolveExactTableDone).
// plain-final-done-protocol, plan-plain-final-protocol,
// plain-final-unusable-output: how the answer ended, not the task.
// incomplete-task-redirect: it only sees partial or failed endings.
@@ -1411,13 +1413,16 @@ export class Agent extends LoopDetector {
const record = state?.exactTableExports?.get(args.data_ref);
const owner = this._exactTableOwner(tabId);
if (!record || !record.owner.current() || !owner?.current() || record.runToken !== owner.runToken) {
+ if (this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,
+ caveat: 'The table reference had expired, so the exact table could not be delivered.' })) {
+ return { success: false, blockedDone: true, exactTableUnverified: true,
+ error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' };
+ }
// Accepted, it delivers only the model's own summary, never rows
- // another task or run extracted.
- return this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,
- caveat: 'The table reference had expired, so the exact table could not be delivered.' })
- ? { success: false, blockedDone: true, exactTableUnverified: true,
- error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' }
- : null;
+ // another task or run extracted. The table the task promised is then
+ // known not to be delivered, so the run ends as partial.
+ this._downgradeCompletionOutcome(tabId, 'exact-table-done');
+ return null;
}
onUpdate?.('tool_call', { name: 'extract_data', args: record.args, verification: true });
const verified = await this.executeTool(tabId, 'extract_data', record.args, onUpdate,
@@ -1431,8 +1436,10 @@ export class Agent extends LoopDetector {
}
// Accepted: the stored snapshot is delivered while its task still owns
// it, unless Strict secret mode now applies (the snapshot predates it,
- // and its re-projection may be what failed).
+ // and its re-projection may be what failed). A table that is not
+ // delivered ends the run as partial.
if (owner.current() && this.strictSecretMode !== true) args.summary = record.summary;
+ else this._downgradeCompletionOutcome(tabId, 'exact-table-done');
return null;
}
args.summary = record.summary;
@@ -5323,6 +5330,14 @@ export class Agent extends LoopDetector {
return (this._guardRunState(tabId).outcomeDowngrades?.size || 0) > 0;
}
+ // A completion accepted although WebBrain knows its result is incomplete,
+ // where the guard's other cases only lack evidence (exact-table-done: a
+ // table that can no longer be delivered). It ends the run as partial, as
+ // one accepted over an OUTCOME_DOWNGRADING_GUARDS check does.
+ _downgradeCompletionOutcome(tabId, id) {
+ (this._guardRunState(tabId).outcomeDowngrades ||= new Set()).add(id);
+ }
+
// native-chat-receipt-not-durable for a plain final. A plain final in an
// Act or Dev run is the model's own success claim, and without Strict
// checks it can end a run where done would have run this check: a chat
@@ -16701,8 +16716,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (!scheduledResume) await cancelPendingResumes();
// done_json is always a success claim; its result carries no outcome.
+ // So is an Ask answer's done, whose schema has no outcome.
const claimsSuccess = toolResult.outcome === 'success' || (fnName === 'done_json'
- && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined);
+ && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined)
+ || (fnName === 'done' && toolResult.done === true && toolResult.outcome == null
+ && !this._isActionMode(this._effectiveRunMode(tabId)));
let doneEndStatus = null;
if (!scheduledResume && claimsSuccess && this._completionOutcomeDowngraded(tabId)) {
// Accepted over a guard that knows the claim is wrong: end the run,
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index ec3241a07..4fdfccff8 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -345,6 +345,8 @@ function selectionScopedResponseLanguagePolicy(locale) {
// done-summary-meta-only, plan-done-summary-shape: the summary's wording.
// pure-calculation-done-checks: how the summary writes the result.
// exact-table-done: a rowCount that disagrees with rows that may be complete.
+// A done whose table can no longer be delivered is downgraded where it
+// is accepted (_resolveExactTableDone).
// plain-final-done-protocol, plan-plain-final-protocol: how the answer
// ended, not the task.
// incomplete-task-redirect: it only sees partial or failed endings.
@@ -1043,13 +1045,16 @@ export class Agent extends LoopDetector {
const record = state?.exactTableExports?.get(args.data_ref);
const owner = this._exactTableOwner(tabId);
if (!record || !record.owner.current() || !owner?.current() || record.runToken !== owner.runToken) {
+ if (this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,
+ caveat: 'The table reference had expired, so the exact table could not be delivered.' })) {
+ return { success: false, blockedDone: true, exactTableUnverified: true,
+ error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' };
+ }
// Accepted, it delivers only the model's own summary, never rows
- // another task or run extracted.
- return this._guardRefusesOnce(tabId, 'exact-table-done', { decided: false,
- caveat: 'The table reference had expired, so the exact table could not be delivered.' })
- ? { success: false, blockedDone: true, exactTableUnverified: true,
- error: 'Unknown, expired, or superseded table data_ref. Extract the table in this current task first.' }
- : null;
+ // another task or run extracted. The table the task promised is then
+ // known not to be delivered, so the run ends as partial.
+ this._downgradeCompletionOutcome(tabId, 'exact-table-done');
+ return null;
}
onUpdate?.('tool_call', { name: 'extract_data', args: record.args, verification: true });
const verified = await this.executeTool(tabId, 'extract_data', record.args, onUpdate,
@@ -1063,8 +1068,10 @@ export class Agent extends LoopDetector {
}
// Accepted: the stored snapshot is delivered while its task still owns
// it, unless Strict secret mode now applies (the snapshot predates it,
- // and its re-projection may be what failed).
+ // and its re-projection may be what failed). A table that is not
+ // delivered ends the run as partial.
if (owner.current() && this.strictSecretMode !== true) args.summary = record.summary;
+ else this._downgradeCompletionOutcome(tabId, 'exact-table-done');
return null;
}
args.summary = record.summary;
@@ -4875,6 +4882,14 @@ export class Agent extends LoopDetector {
return (this._guardRunState(tabId).outcomeDowngrades?.size || 0) > 0;
}
+ // A completion accepted although WebBrain knows its result is incomplete,
+ // where the guard's other cases only lack evidence (exact-table-done: a
+ // table that can no longer be delivered). It ends the run as partial, as
+ // one accepted over an OUTCOME_DOWNGRADING_GUARDS check does.
+ _downgradeCompletionOutcome(tabId, id) {
+ (this._guardRunState(tabId).outcomeDowngrades ||= new Set()).add(id);
+ }
+
// native-chat-receipt-not-durable for a plain final. A plain final in an
// Act or Dev run is the model's own success claim, and without Strict
// checks it can end a run where done would have run this check: a chat
@@ -14968,8 +14983,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
if (!scheduledResume) await cancelPendingResumes();
// done_json is always a success claim; its result carries no outcome.
+ // So is an Ask answer's done, whose schema has no outcome.
const claimsSuccess = toolResult.outcome === 'success' || (fnName === 'done_json'
- && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined);
+ && toolResult.doneJson === true && toolResult.cloudFailed !== true && toolResult.outcome === undefined)
+ || (fnName === 'done' && toolResult.done === true && toolResult.outcome == null
+ && !this._isActionMode(this._effectiveRunMode(tabId)));
let doneEndStatus = null;
if (!scheduledResume && claimsSuccess && this._completionOutcomeDowngraded(tabId)) {
// Accepted over a guard that knows the claim is wrong: end the run,
diff --git a/test/guard-advisory-completion.mjs b/test/guard-advisory-completion.mjs
index 84a6a8bdf..0d7fe7eef 100644
--- a/test/guard-advisory-completion.mjs
+++ b/test/guard-advisory-completion.mjs
@@ -718,6 +718,24 @@ for (const build of ['chrome', 'firefox']) {
}
});
+ test(`${build}: an Ask answer accepted over unread pages of a requested whole read ends as partial`, async () => {
+ // An Ask done has no outcome of its own; it claims the answer is complete.
+ const h = batchAgent(Agent, { mode: 'ask', task: 'Summarize this whole conversation.' });
+ h.agent._currentUrl = async () => 'https://outlook.live.com/mail/0/inbox/id/AQMkADAwATM0MDAAMS1i';
+ await h.agent._beginReadCompleteness(h.tab, 'Summarize this whole conversation.', {});
+ h.agent._armReadCompletenessFromPlan(h.tab, { request_kind: 'execute', read_scope: 'complete_thread' });
+ h.agent._recordReadCompleteness(h.tab, 'get_accessibility_tree', { filter: 'all', maxDepth: 15 }, {
+ pageContent: 'main\n listitem "First message"', page: 1, totalChars: 9000, hasMore: true, truncated: false, nextPage: 2 });
+ const done = ['done', { summary: 'The thread settles the launch date.' }];
+ assert.equal((await h.batch([done])).action, 'continue', `${build}: the first answer gets its coverage note`);
+ const updates = [];
+ const accepted = await h.batch([done], updates);
+ assert.equal(accepted.action, 'return', `${build}: the second answer is accepted`);
+ assert.equal(accepted.endStatus, 'partial', `${build}: the run must end as partial`);
+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;
+ assert.equal(doneResult?.outcome, 'partial', `${build}: an answer over unread pages was not downgraded`);
+ });
+
// ── Exact table export ─────────────────────────────────────────────────
test(`${build}: table JSON without an exact extract, and an expired table reference, are advised once`, async () => {
const json = JSON.stringify({ rowCount: 2, rows: [{ id: 'A' }, { id: 'B' }] });
@@ -733,6 +751,40 @@ for (const build of ['chrome', 'firefox']) {
'The table reference had expired, so the exact table could not be delivered.', `${build} table ref`);
});
+ test(`${build}: a done whose exact table can no longer be delivered ends as partial`, async () => {
+ // The reference expired: the accepted done delivers only the model's
+ // summary, never the promised rows, and WebBrain knows it.
+ const askSchemas = new Map(getToolsForMode('ask', { tier: 'full' }).map(tool => [tool.function.name, tool.function.parameters]));
+ const h = batchAgent(Agent, { mode: 'ask', realDone: true, schemas: askSchemas });
+ h.agent.readCompletenessStates.set(h.tab, { runToken: 'owned-run', tablePageObserved: true });
+ // An Ask answer's done has no outcome of its own; it claims the answer.
+ const done = ['done', { summary: 'Full inventory.', data_ref: 'table_expired' }];
+ assert.equal((await h.batch([done])).action, 'continue', `${build}: the expired reference is refused once`);
+ const updates = [];
+ const accepted = await h.batch([done], updates);
+ assert.equal(accepted.action, 'return', `${build}: the second completion is accepted`);
+ assert.equal(accepted.endStatus, 'partial', `${build}: the run must end as partial`);
+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;
+ assert.equal(doneResult?.outcome, 'partial', `${build}: an undelivered table was reported as a success`);
+ // A stale snapshot that is still delivered keeps the model's outcome; one
+ // that Strict secret mode withholds does not.
+ for (const strictSecretMode of [false, true]) {
+ const s = batchAgent(Agent, { mode: 'ask' });
+ s.agent.strictSecretMode = strictSecretMode;
+ const owner = { runToken: 'owned-run', current: () => true };
+ const record = { dataRef: 'table_owned', runToken: 'owned-run', owner, args: { columns: ['id'] }, summary: '[{"id":"A"}]' };
+ s.agent.readCompletenessStates.set(s.tab, { runToken: 'owned-run', exactTableExports: new Map([['table_owned', record]]) });
+ s.agent._exactTableOwner = () => owner;
+ s.agent.executeTool = async () => ({ success: false, error: 'The source table changed.' });
+ const args = () => ({ summary: 'Full inventory.', data_ref: 'table_owned' });
+ s.agent._beginCompletionGuardAttempt(s.tab);
+ assert.equal((await s.agent._resolveExactTableDone(s.tab, args()))?.blockedDone, true);
+ s.agent._beginCompletionGuardAttempt(s.tab);
+ assert.equal(await s.agent._resolveExactTableDone(s.tab, args()), null);
+ assert.equal(s.agent._completionOutcomeDowngraded(s.tab), strictSecretMode, `${build}: strictSecretMode=${strictSecretMode}`);
+ }
+ });
+
test(`${build}: an unverifiable table snapshot is delivered over advice, but not under Strict secret mode`, async () => {
for (const strictSecretMode of [false, true]) {
const h = batchAgent(Agent, { mode: 'ask' });