Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Scope a card refusal to its own action in incomplete-task recovery
Barack Sokullubd2cfbd50e2b
diff --git a/src/chrome/src/agent/incomplete-completion.js b/src/chrome/src/agent/incomplete-completion.jsindex f8877fbe9..dd8fc43d2 100644--- a/src/chrome/src/agent/incomplete-completion.js+++ b/src/chrome/src/agent/incomplete-completion.js@@ -39,15 +39,16 @@ function recordedResults(agent, tabId, run) {for (const key of ['error', 'attachmentState', 'previousUrl', 'currentUrl']) {if (typeof result[key] === 'string') entry[key] = text(result[key]);}+ // The user's own refusal on a WebBrain card (a deletion, a submit or a+ // permission), as structured data the continuation judge can rely on.+ if (result.deletionDeclined === true+ || (result.denied === true && (result.submitConfirmationRequired === true || result.permissionRequired === true))) {+ entry.userDeclined = true;+ }return entry;}).filter(Boolean);}-function userDeclined(entry) {- return entry.deletionDeclined === true- || (entry.denied === true && (entry.submitConfirmationRequired === true || entry.permissionRequired === true));-}-export async function incompleteCompletionBlock(agent, tabId, { outcome, summary, pageState, pageUrl = '', terminalOnly = false }) {const guard = agent._planExecutionGuards.get(tabId);const run = agent.completionInvariants.get(tabId);@@ -75,11 +76,10 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summarylet parentIdentity = identity;try { const composed = JSON.parse(identity); if (Array.isArray(composed)) parentIdentity = composed[0]; } catch {}if (pageState.documentIdentity && pageState.documentIdentity !== parentIdentity) return changed();+ // A refusal settles only the action it answered. The judge still decides+ // whether other requested work remains; it never reopens the refused one.const results = recordedResults(agent, tabId, run);- // The user's own refusal on a WebBrain card (a deletion, a submit or a- // permission) is the reason this run is incomplete. Reporting that is the- // honest end; recovery must not push the model to try again.- if (results.some(userDeclined)) return null;+ const declined = results.some(entry => entry.userDeclined === true);const request = agent._progressTaskAnchorText(tabId) || agent._latestTaskText(tabId) || '';const reading = await agent._semanticDecision(tabId, 'incomplete_task', { request, page: {candidate_summary_not_proof: text(summary),@@ -108,7 +108,8 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summaryrecovery: reading.choice,error: reading.choice === 'wait_then_check'? 'The authorized task is still in progress. Wait for the relevant completion control or state with wait_for_element, or use a bounded wait_for_stable followed by a fresh page/status read. Continue the authorized flow when ready. Do not re-upload or repeat a submission already dispatched. For a longer external wait, use schedule_resume only when available and appropriate; otherwise explain the concrete remaining blocker after checking.'- : 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',+ : (declined ? 'The user declined one action on WebBrain\'s card; that refusal is final. Do not retry it, ask again or reach it another way. ' : '')+ + 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',};} catch (error) {if (completionStopError(error, signal) || agent._isCostAllowanceError(error) || agent._checkAbort(tabId)) throw error;diff --git a/src/chrome/src/agent/semantic-decisions.js b/src/chrome/src/agent/semantic-decisions.jsindex 6380f1044..96a48c5a9 100644--- a/src/chrome/src/agent/semantic-decisions.js+++ b/src/chrome/src/agent/semantic-decisions.js@@ -12,7 +12,7 @@//// This file is shared byte for byte by the Chrome and Firefox builds.-export const SEMANTIC_DECISION_VERSION = 2;+export const SEMANTIC_DECISION_VERSION = 3;// A choice listed in `authorizing` lets an action go ahead without asking the// user, so it needs a surer answer than the others: a higher decision-model@@ -96,7 +96,11 @@ export const SEMANTIC_QUESTIONS = Object.freeze({+ 'the task is impossible. Judge in any language. Never invent source facts, credentials, permission, an '+ 'attestation or a user answer; never propose repeating an uncertain or already dispatched submission. '+ 'A bounded read or wait can reconcile such a dispatch. Choose continuation only when a concrete '- + 'reasonable step remains within the existing task and normal tool permissions.',+ + 'reasonable step remains within the existing task and normal tool permissions. A recorded result with '+ + '`userDeclined` is the user\'s final refusal of that one action on WebBrain\'s own card: never choose a '+ + 'continuation to retry it, ask again or reach it another way, and do not treat it as missing permission '+ + 'for the rest of the task. Choose continue_now only when other requested work, unrelated to the refused '+ + 'action, still remains; if the refused action was the only outstanding work, choose stop_required.',choices: Object.freeze({continue_now: 'A reasonable permitted inspection, source retrieval, correction or untried approach remains; the agent stopped before trying it.',wait_then_check: 'The requested flow is still loading, uploading, validating or otherwise running, and a bounded wait followed by a fresh observation can let it continue.',diff --git a/src/firefox/src/agent/incomplete-completion.js b/src/firefox/src/agent/incomplete-completion.jsindex f8877fbe9..dd8fc43d2 100644--- a/src/firefox/src/agent/incomplete-completion.js+++ b/src/firefox/src/agent/incomplete-completion.js@@ -39,15 +39,16 @@ function recordedResults(agent, tabId, run) {for (const key of ['error', 'attachmentState', 'previousUrl', 'currentUrl']) {if (typeof result[key] === 'string') entry[key] = text(result[key]);}+ // The user's own refusal on a WebBrain card (a deletion, a submit or a+ // permission), as structured data the continuation judge can rely on.+ if (result.deletionDeclined === true+ || (result.denied === true && (result.submitConfirmationRequired === true || result.permissionRequired === true))) {+ entry.userDeclined = true;+ }return entry;}).filter(Boolean);}-function userDeclined(entry) {- return entry.deletionDeclined === true- || (entry.denied === true && (entry.submitConfirmationRequired === true || entry.permissionRequired === true));-}-export async function incompleteCompletionBlock(agent, tabId, { outcome, summary, pageState, pageUrl = '', terminalOnly = false }) {const guard = agent._planExecutionGuards.get(tabId);const run = agent.completionInvariants.get(tabId);@@ -75,11 +76,10 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summarylet parentIdentity = identity;try { const composed = JSON.parse(identity); if (Array.isArray(composed)) parentIdentity = composed[0]; } catch {}if (pageState.documentIdentity && pageState.documentIdentity !== parentIdentity) return changed();+ // A refusal settles only the action it answered. The judge still decides+ // whether other requested work remains; it never reopens the refused one.const results = recordedResults(agent, tabId, run);- // The user's own refusal on a WebBrain card (a deletion, a submit or a- // permission) is the reason this run is incomplete. Reporting that is the- // honest end; recovery must not push the model to try again.- if (results.some(userDeclined)) return null;+ const declined = results.some(entry => entry.userDeclined === true);const request = agent._progressTaskAnchorText(tabId) || agent._latestTaskText(tabId) || '';const reading = await agent._semanticDecision(tabId, 'incomplete_task', { request, page: {candidate_summary_not_proof: text(summary),@@ -108,7 +108,8 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summaryrecovery: reading.choice,error: reading.choice === 'wait_then_check'? 'The authorized task is still in progress. Wait for the relevant completion control or state with wait_for_element, or use a bounded wait_for_stable followed by a fresh page/status read. Continue the authorized flow when ready. Do not re-upload or repeat a submission already dispatched. For a longer external wait, use schedule_resume only when available and appropriate; otherwise explain the concrete remaining blocker after checking.'- : 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',+ : (declined ? 'The user declined one action on WebBrain\'s card; that refusal is final. Do not retry it, ask again or reach it another way. ' : '')+ + 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',};} catch (error) {if (completionStopError(error, signal) || agent._isCostAllowanceError(error) || agent._checkAbort(tabId)) throw error;diff --git a/src/firefox/src/agent/semantic-decisions.js b/src/firefox/src/agent/semantic-decisions.jsindex 6380f1044..96a48c5a9 100644--- a/src/firefox/src/agent/semantic-decisions.js+++ b/src/firefox/src/agent/semantic-decisions.js@@ -12,7 +12,7 @@//// This file is shared byte for byte by the Chrome and Firefox builds.-export const SEMANTIC_DECISION_VERSION = 2;+export const SEMANTIC_DECISION_VERSION = 3;// A choice listed in `authorizing` lets an action go ahead without asking the// user, so it needs a surer answer than the others: a higher decision-model@@ -96,7 +96,11 @@ export const SEMANTIC_QUESTIONS = Object.freeze({+ 'the task is impossible. Judge in any language. Never invent source facts, credentials, permission, an '+ 'attestation or a user answer; never propose repeating an uncertain or already dispatched submission. '+ 'A bounded read or wait can reconcile such a dispatch. Choose continuation only when a concrete '- + 'reasonable step remains within the existing task and normal tool permissions.',+ + 'reasonable step remains within the existing task and normal tool permissions. A recorded result with '+ + '`userDeclined` is the user\'s final refusal of that one action on WebBrain\'s own card: never choose a '+ + 'continuation to retry it, ask again or reach it another way, and do not treat it as missing permission '+ + 'for the rest of the task. Choose continue_now only when other requested work, unrelated to the refused '+ + 'action, still remains; if the refused action was the only outstanding work, choose stop_required.',choices: Object.freeze({continue_now: 'A reasonable permitted inspection, source retrieval, correction or untried approach remains; the agent stopped before trying it.',wait_then_check: 'The requested flow is still loading, uploading, validating or otherwise running, and a bounded wait followed by a fresh observation can let it continue.',diff --git a/test/incomplete-completion-recovery.mjs b/test/incomplete-completion-recovery.mjsindex b4fde1f5a..64a2b2767 100644--- a/test/incomplete-completion-recovery.mjs+++ b/test/incomplete-completion-recovery.mjs@@ -130,6 +130,40 @@ for (const build of ['chrome', 'firefox']) {assert.equal((await h.check()).recovery, 'wait_then_check');});+ test(`${build}: a refusal on WebBrain's card settles that action only; other requested work can still resume`, async () => {+ for (const [declinedResult, otherWorkLeft] of [+ [{ success: false, noDispatch: true, deletionDeclined: true, error: 'The user chose not to delete this item.' }, true],+ [{ success: false, denied: true, submitConfirmationRequired: true, error: 'The user did not confirm submitting.' }, true],+ [{ success: false, denied: true, permissionRequired: true, error: 'The user did not allow it.' }, false],+ ]) {+ const h = unitFixture();+ h.agent.conversations.set(1, [+ { role: 'assistant', tool_calls: [{ id: 'refused', function: { name: 'click_ax' } }, { id: 'other', function: { name: 'set_field' } }] },+ { role: 'tool', tool_call_id: 'refused', content: JSON.stringify(declinedResult) },+ { role: 'tool', tool_call_id: 'other', content: JSON.stringify({ success: false, denied: true, error: 'Not a user refusal.' }) },+ ]);+ let seen;+ h.agent._semanticDecision = async (_tab, kind, context) => {+ assert.equal(kind, 'incomplete_task');+ seen = context.page.recorded_current_results;+ return { choice: otherWorkLeft ? 'continue_now' : 'stop_required' };+ };+ const block = await h.check();+ assert.deepEqual(seen.map(entry => entry.userDeclined === true), [true, false], 'Only the card refusal is marked as the user\'s');+ if (otherWorkLeft) {+ assert.equal(block.recovery, 'continue_now', 'Unfinished sibling work still resumes after one refusal');+ assert.match(block.error, /refusal is final\. Do not retry it, ask again or reach it another way/);+ } else {+ assert.equal(block, null, 'A refusal of the only outstanding work ends honestly');+ }+ }+ });++ test(`${build}: without a refusal the continuation text says nothing about one`, async () => {+ const h = unitFixture('continue_now');+ assert.doesNotMatch((await h.check()).error, /declined/);+ });+for (const streaming of [false, true]) for (const scenario of ['upload', 'missing-source']) {test(`${build}: ${streaming ? 'stream' : 'chat'} recovers ${scenario} through the real done and tool-batch paths`, async () => {const tab = nextTab++, updates = [], actions = [], offered = [];diff --git a/test/quality/unbound-deletion-mv3.mjs b/test/quality/unbound-deletion-mv3.mjsindex 8f16539f9..6d3db028f 100644--- a/test/quality/unbound-deletion-mv3.mjs+++ b/test/quality/unbound-deletion-mv3.mjs@@ -44,7 +44,7 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,await fs.mkdir(out, { recursive: true });let actionCalls = 0, judgeCalls = 0, blockedRemote = 0;const completionRecovery = { reads: 0 };- const deletes = [], providerRequests = [], authorizations = [], decisions = [], summaryReadings = [];+ const deletes = [], providerRequests = [], authorizations = [], decisions = [], summaryReadings = [], continuationReadings = [];const textOf = content => typeof content === 'string' ? content : Array.isArray(content)? content.filter(part => part?.type === 'text').map(part => part.text || '').join('\n') : '';const refFor = (input, name) => {@@ -86,9 +86,14 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,const choices = [...system.matchAll(/^- "([a-z_]+)":/gm)].map(match => match[1]);// A failed or partial Act result is read once before it is shown; it// gives the outcome here.+ // A partial end is checked once for remaining work; a refused deletion+ // must reach that judge as the user's structured refusal.+ const continuation = choices.includes('continue_now');if (choices.includes('mode_claim')) summaryReadings.push({ request, summary: page.summary });+ else if (continuation) continuationReadings.push({ request, declined: (page.recorded_current_results || []).filter(entry => entry.userDeclined === true).length });else decisions.push({ choices, request, page });const answer = choices.includes('mode_claim') ? { choice: 'result', evidence: '' }+ : continuation ? { choice: 'stop_required', evidence: '' }: !choices.includes('delete_this') ? { choice: '', evidence: '' }: cardAnswer ? { choice: 'unclear', evidence: '' }: page.item === rows[0] ? { choice: 'delete_this', evidence } : { choice: 'other_item', evidence: '' };@@ -206,7 +211,7 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,const toolResults = result.updates.filter(event => event.type === 'tool_result').map(event => ({ name: event.data?.name,blocked: event.data?.result?.blocked === true, reason: event.data?.result?.actionGuard?.reason || null, error: event.data?.result?.error || null }));await fs.writeFile(path.join(out, 'evidence.json'), JSON.stringify({ result, diagnostic, deletes, judgeCalls, authorizations,- decisions, summaryReadings, toolResults, prompts, cards, unexpected, blockedRemote }, (_key, value) => typeof value === 'string' && value.startsWith('data:image/') ? '[owned fixture screenshot omitted]' : value, 2));+ decisions, summaryReadings, continuationReadings, toolResults, prompts, cards, unexpected, blockedRemote }, (_key, value) => typeof value === 'string' && value.startsWith('data:image/') ? '[owned fixture screenshot omitted]' : value, 2));assert.deepEqual(unexpected, [], 'No other question was asked');assert.equal(authorizations.length, 0, 'A deletion is decided by the deletion question, not the publication check');// The request reaches the model as written, with the row and the row beside it.@@ -230,7 +235,10 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,assert.notEqual(partial?.outcome, 'success', JSON.stringify(partial));assert.deepEqual(summaryReadings, [{ request: task, summary: `The user chose not to remove the ${target}; nothing was removed.` }],'The partial result was read once, as written');- console.log(JSON.stringify({ passed: true, installedMV3: true, variant, cards: cards.length, deletes: deletes.length, blockedRemote }));+ assert.ok(continuationReadings.length <= 1 && continuationReadings.every(reading => reading.request === task && reading.declined === 1),+ `A remaining-work check sees the refusal as the user's structured decision: ${JSON.stringify(continuationReadings)}`);+ console.log(JSON.stringify({ passed: true, installedMV3: true, variant, cards: cards.length, deletes: deletes.length,+ continuationReadings: continuationReadings.length, blockedRemote }));return;}assert.deepEqual(deletes, [{ method: 'DELETE', path: '/api/cart/items/1' }], `Only the named row is removed, once: ${JSON.stringify(toolResults)}`);