Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: unresolved progress rows end an accepted completion as partial
Barack Sokullub605dd4661d4
diff --git a/docs/guards.md b/docs/guards.mdindex 53dee0bcb..051b63d39 100644--- a/docs/guards.md+++ b/docs/guards.md@@ -27,10 +27,11 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):would refuse is accepted, and the final answer ends with one line,"WebBrain could not verify everything: β¦" (at most four items).- A completion accepted over a guard that knows the claim is wrong (the- completion judge's "not done", an unauthorized mutation, missing or- failed progress rows, text that went into a formatting toolbar) ends the- run as `partial`, not `success`. Missing evidence alone keeps the- model's outcome.+ completion judge's "not done", an unauthorized mutation, missing, failed+ or unfinished progress rows, unread pages of a whole thread the user+ asked for, text that went into a formatting toolbar) ends the run as+ `partial`, not `success`. Missing evidence alone keeps the model's+ outcome.- A write that went out but could not be read back exactly (GitHub'sCodeMirror editor is the usual case) is an unverified success: it locksnothing, arms no text-write debt and is not a loop failure.diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex e0b30a3ae..69d57a579 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -495,15 +495,33 @@ function selectionScopedResponseLanguagePolicy(locale) {};}-// What the user is told when a run ends with plain text after the "end with-// done" note was given once (Strict checks off).-// Completion guards whose decided case means the task is not done (a-// judge's sure verdict, a forbidden submission, failed or missing items,-// text typed into a toolbar). Accepted over their advice, a success claim-// ends the run as partial.+// Completion guards whose decided case means the task as claimed is not+// done. A success accepted over one of them ends the run as partial.+// completion-verifier-not-done: the completion judge's sure verdict.+// plan-unauthorized-mutation-recorded: a forbidden submit or blocked repair.+// progress-success-claim-checks: missing, skipped or failed items.+// progress-ledger-unresolved: items still pending or acted on.+// read-coverage-incomplete: pages of a requested whole read never read+// (an unconfirmed Gmail expansion is its undecided case).+// done-rich-text-toolbar-obligation: text typed into a toolbar.+// Left out, because their decided case can be wrong about a finished task+// or says nothing about it:+// completion-page-warning: the open dialog or form error may be unrelated.+// done-summary-meta-only, plan-done-summary-shape: the summary's wording.+// pure-calculation-done-checks: how the summary writes the result.+// exact-table-done: a rowCount that disagrees with rows that may be complete.+// plain-final-done-protocol, plan-plain-final-protocol,+// plain-final-unusable-output: how the answer ended, not the task.+// incomplete-task-redirect: it only sees partial or failed endings.+// done-invariant-shape-checks: a missing outcome already counts as partial.+// github-committed-file-verification: never decided; an unread raw blob+// and a mismatched one both reach it as missing evidence.const OUTCOME_DOWNGRADING_GUARDS = new Set(['completion-verifier-not-done', 'plan-unauthorized-mutation-recorded',- 'progress-success-claim-checks', 'done-rich-text-toolbar-obligation']);+ 'progress-success-claim-checks', 'progress-ledger-unresolved', 'read-coverage-incomplete',+ 'done-rich-text-toolbar-obligation']);+// What the user is told when a run ends with plain text after the "end with+// done" note was given once (Strict checks off).const PLAIN_FINAL_PROTOCOL_CAVEAT = 'The task ended with a plain answer instead of a checked completion.';function defaultPlanExecutionBlockNudge({diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 2f45819f6..52876acf7 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -330,15 +330,33 @@ function selectionScopedResponseLanguagePolicy(locale) {};}-// What the user is told when a run ends with plain text after the "end with-// done" note was given once (Strict checks off).-// Completion guards whose decided case means the task is not done (a-// judge's sure verdict, a forbidden submission, failed or missing items,-// text typed into a toolbar). Accepted over their advice, a success claim-// ends the run as partial.+// Completion guards whose decided case means the task as claimed is not+// done. A success accepted over one of them ends the run as partial.+// completion-verifier-not-done: the completion judge's sure verdict.+// plan-unauthorized-mutation-recorded: a forbidden submit or blocked repair.+// progress-success-claim-checks: missing, skipped or failed items.+// progress-ledger-unresolved: items still pending or acted on.+// read-coverage-incomplete: pages of a requested whole read never read+// (an unconfirmed Gmail expansion is its undecided case).+// done-rich-text-toolbar-obligation: text typed into a toolbar.+// Left out, because their decided case can be wrong about a finished task+// or says nothing about it:+// completion-page-warning: the open dialog or form error may be unrelated.+// done-summary-meta-only, plan-done-summary-shape: the summary's wording.+// pure-calculation-done-checks: how the summary writes the result.+// exact-table-done: a rowCount that disagrees with rows that may be complete.+// plain-final-done-protocol, plan-plain-final-protocol: how the answer+// ended, not the task.+// incomplete-task-redirect: it only sees partial or failed endings.+// done-invariant-shape-checks: a missing outcome already counts as partial.+// github-committed-file-verification: never decided; an unread raw blob+// and a mismatched one both reach it as missing evidence.const OUTCOME_DOWNGRADING_GUARDS = new Set(['completion-verifier-not-done', 'plan-unauthorized-mutation-recorded',- 'progress-success-claim-checks', 'done-rich-text-toolbar-obligation']);+ 'progress-success-claim-checks', 'progress-ledger-unresolved', 'read-coverage-incomplete',+ 'done-rich-text-toolbar-obligation']);+// What the user is told when a run ends with plain text after the "end with+// done" note was given once (Strict checks off).const PLAIN_FINAL_PROTOCOL_CAVEAT = 'The task ended with a plain answer instead of a checked completion.';function defaultPlanExecutionBlockNudge({diff --git a/test/guard-advisory-completion.mjs b/test/guard-advisory-completion.mjsindex 1fd3eaacf..f732ebd41 100644--- a/test/guard-advisory-completion.mjs+++ b/test/guard-advisory-completion.mjs@@ -274,6 +274,31 @@ for (const build of ['chrome', 'firefox']) {}});+ test(`${label}: a plain final accepted over unfinished progress rows ends the run as partial`, async () => {+ for (const status of ['pending', 'acted']) {+ const tab = nextTab++;+ let n = 0;+ const agent = loopAgent(Agent, async () => {+ n++;+ return { content: 'Processed both items.' };+ });+ agent._currentTaskLedgerRows = () => [{ id: 'item-1', label: 'First', status: 'processed' },+ { id: 'item-2', label: 'Second', status }];+ const runStatuses = [];+ let finished = null;+ const onUpdate = (type, data) => { if (type === 'run_status') runStatuses.push(data?.status); };+ const options = { detachedRequestId: `guard-${tab}`, onRunFinished: value => { finished = value; } };+ const result = streaming+ ? await agent.processMessageStream(tab, 'Process both items.', onUpdate, 'act', options)+ : await agent.processMessage(tab, 'Process both items.', onUpdate, 'act', [], options);+ assert.ok(n > 1, `${label} ${status}: the first plain final gets its progress note`);+ assert.ok(String(result).startsWith(`Processed both items.\n\n${CAVEAT}`), `${label} ${status}: caveat missing from ${JSON.stringify(result)}`);+ assert.match(result, /1 of 2 items in the task's progress list were not finished\./);+ assert.equal(finished, 'partial', `${label} ${status}: the run finished as a success with an item unfinished`);+ assert.deepEqual(runStatuses, ['partial'], `${label} ${status}: the scheduler did not hear the partial ending`);+ }+ });+test(`${label}: a plain final after a chat send saves the delivery receipt first`, async () => {// A plain final ends a run where done would have saved the receipt// (native-chat-receipt-not-durable), so it runs the same check.@@ -528,6 +553,22 @@ for (const build of ['chrome', 'firefox']) {'1 of 2 items in the task\'s progress list were not finished.', `${build} unresolved rows`);});+ test(`${build}: a success accepted over unfinished progress rows ends as partial`, async () => {+ for (const status of ['pending', 'acted']) {+ const h = batchAgent(Agent);+ h.agent._currentTaskLedgerRows = () => [{ id: 'item-1', label: 'First', status: 'processed' },+ { id: 'item-2', label: 'Second', status }];+ const done = ['done', { summary: 'Both items are done.', outcome: 'success' }];+ assert.equal((await h.batch([done])).action, 'continue', `${build} ${status}: the first success claim gets its note`);+ const updates = [];+ const accepted = await h.batch([done], updates);+ assert.equal(accepted.action, 'return', `${build} ${status}: the second success claim is accepted`);+ assert.ok(String(accepted.value).includes(`${CAVEAT}1 of 2 items in the task's progress list were not finished.`));+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;+ assert.equal(doneResult?.outcome, 'partial', `${build} ${status}: a success with an item unfinished was not downgraded`);+ }+ });+test(`${build}: a success claim over skipped or failed progress rows is advised once`, async () => {const rows = [{ id: 'item-1', label: 'First', status: 'processed' }, { id: 'item-2', label: 'Second', status: 'skipped' }];await refusedOnceThenAccepted(Agent, {@@ -536,7 +577,7 @@ for (const build of ['chrome', 'firefox']) {'Some items in the task\'s progress list are missing, incomplete, skipped or failed.', `${build} skipped rows`);});- // ββ Read coverage (Ask-mode Gmail) βββββββββββββββββββββββββββββββββββββ+ // ββ Read coverage ββββββββββββββββββββββββββββββββββββββββββββββββββββββtest(`${build}: Ask keeps the answer and adds the Gmail expansion limitation after one note`, async () => {for (const strict of [false, true]) {const h = batchAgent(Agent, { mode: 'ask', strict });@@ -566,6 +607,44 @@ for (const build of ['chrome', 'firefox']) {}});+ test(`${build}: a success accepted over unread pages of a requested whole read ends as partial`, async () => {+ // decided: page 2 of the thread was never read (the check knows).+ // Undecided: Act mode could not confirm Gmail's expansion.+ for (const decided of [true, false]) {+ const h = batchAgent(Agent, { task: 'Summarize this whole conversation.' });+ h.agent._currentUrl = async () => decided ? 'https://outlook.live.com/mail/0/inbox/id/AQMkADAwATM0MDAAMS1i'+ : 'https://mail.google.com/mail/u/0/#inbox/FMfc123';+ await h.agent._beginReadCompleteness(h.tab, 'Summarize this whole conversation.', {});+ h.agent._armReadCompletenessFromPlan(h.tab, { request_kind: 'execute', read_scope: 'complete_thread' });+ if (decided) {+ h.agent._recordReadCompleteness(h.tab, 'get_accessibility_tree', { filter: 'all', maxDepth: 15 }, {+ pageContent: 'main\n listitem "First message"', page: 1, totalChars: 9000, hasMore: true, truncated: false, nextPage: 2 });+ } else {+ const root = `ref_${build}_root`;+ h.agent._recordReadCompleteness(h.tab, 'get_accessibility_tree', { filter: 'visible', maxDepth: 12 }, {+ pageContent: 'main\n button "Expand all"', page: 1, hasMore: false, truncated: false,+ conversationRootRefId: root, conversationExpansionState: 'collapsed' });+ h.agent._recordReadCompleteness(h.tab, 'get_accessibility_tree', { filter: 'all', maxDepth: 15, ref_id: root }, {+ pageContent: 'main\n button "Expand all"\n listitem "Latest visible message"', page: 1, totalChars: 1200,+ hasMore: false, truncated: false, continuationArgs: null, conversationRootRefId: root,+ conversationExpansionState: 'collapsed', treeRevision: 'fnv1a64:4444444444444444' });+ }+ const state = h.agent.readCompletenessStates.get(h.tab);+ assert.equal(state?.required === true && state.complete === false, true, `${build}: the whole read must be required and incomplete`);+ const done = ['done', { summary: 'The thread settles the launch date.', outcome: 'success' }];+ assert.equal((await h.batch([done])).action, 'continue', `${build}: the first completion gets its coverage note`);+ const updates = [];+ const accepted = await h.batch([done], updates);+ assert.equal(accepted.action, 'return', `${build}: the second completion is accepted`);+ assert.match(String(accepted.value), decided ? /Not every part of the conversation was confirmed as read\./+ : /Some Gmail messages may have stayed collapsed/);+ const doneResult = updates.filter(update => update.type === 'tool_result' && update.data.name === 'done').at(-1)?.data.result;+ assert.equal(doneResult?.outcome, decided ? 'partial' : 'success', decided+ ? `${build}: a success over unread pages was not downgraded`+ : `${build}: an unconfirmed Gmail expansion alone must not downgrade the outcome`);+ }+ });+// ββ Exact table export βββββββββββββββββββββββββββββββββββββββββββββββββtest(`${build}: table JSON without an exact extract, and an expired table reference, are advised once`, async () => {const json = JSON.stringify({ rowCount: 2, rows: [{ id: 'A' }, { id: 'B' }] });