Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Scope a card refusal to its own action in incomplete-task recovery

Barack Sokullubd2cfbd50e2b
diff --git a/src/chrome/src/agent/incomplete-completion.js b/src/chrome/src/agent/incomplete-completion.js
index f8877fbe9..dd8fc43d2 100644
--- a/src/chrome/src/agent/incomplete-completion.js
+++ b/src/chrome/src/agent/incomplete-completion.js
@@ -39,15 +39,16 @@ function recordedResults(agent, tabId, run) {
for (const key of ['error', 'attachmentState', 'previousUrl', 'currentUrl']) {
if (typeof result[key] === 'string') entry[key] = text(result[key]);
}
+ // The user's own refusal on a WebBrain card (a deletion, a submit or a
+ // permission), as structured data the continuation judge can rely on.
+ if (result.deletionDeclined === true
+ || (result.denied === true && (result.submitConfirmationRequired === true || result.permissionRequired === true))) {
+ entry.userDeclined = true;
+ }
return entry;
}).filter(Boolean);
}
-function userDeclined(entry) {
- return entry.deletionDeclined === true
- || (entry.denied === true && (entry.submitConfirmationRequired === true || entry.permissionRequired === true));
-}
-
export async function incompleteCompletionBlock(agent, tabId, { outcome, summary, pageState, pageUrl = '', terminalOnly = false }) {
const guard = agent._planExecutionGuards.get(tabId);
const run = agent.completionInvariants.get(tabId);
@@ -75,11 +76,10 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summary
let parentIdentity = identity;
try { const composed = JSON.parse(identity); if (Array.isArray(composed)) parentIdentity = composed[0]; } catch {}
if (pageState.documentIdentity && pageState.documentIdentity !== parentIdentity) return changed();
+ // A refusal settles only the action it answered. The judge still decides
+ // whether other requested work remains; it never reopens the refused one.
const results = recordedResults(agent, tabId, run);
- // The user's own refusal on a WebBrain card (a deletion, a submit or a
- // permission) is the reason this run is incomplete. Reporting that is the
- // honest end; recovery must not push the model to try again.
- if (results.some(userDeclined)) return null;
+ const declined = results.some(entry => entry.userDeclined === true);
const request = agent._progressTaskAnchorText(tabId) || agent._latestTaskText(tabId) || '';
const reading = await agent._semanticDecision(tabId, 'incomplete_task', { request, page: {
candidate_summary_not_proof: text(summary),
@@ -108,7 +108,8 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summary
recovery: reading.choice,
error: reading.choice === 'wait_then_check'
? 'The authorized task is still in progress. Wait for the relevant completion control or state with wait_for_element, or use a bounded wait_for_stable followed by a fresh page/status read. Continue the authorized flow when ready. Do not re-upload or repeat a submission already dispatched. For a longer external wait, use schedule_resume only when available and appropriate; otherwise explain the concrete remaining blocker after checking.'
- : 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',
+ : (declined ? 'The user declined one action on WebBrain\'s card; that refusal is final. Do not retry it, ask again or reach it another way. ' : '')
+ + 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',
};
} catch (error) {
if (completionStopError(error, signal) || agent._isCostAllowanceError(error) || agent._checkAbort(tabId)) throw error;
diff --git a/src/chrome/src/agent/semantic-decisions.js b/src/chrome/src/agent/semantic-decisions.js
index 6380f1044..96a48c5a9 100644
--- a/src/chrome/src/agent/semantic-decisions.js
+++ b/src/chrome/src/agent/semantic-decisions.js
@@ -12,7 +12,7 @@
//
// This file is shared byte for byte by the Chrome and Firefox builds.
-export const SEMANTIC_DECISION_VERSION = 2;
+export const SEMANTIC_DECISION_VERSION = 3;
// A choice listed in `authorizing` lets an action go ahead without asking the
// user, so it needs a surer answer than the others: a higher decision-model
@@ -96,7 +96,11 @@ export const SEMANTIC_QUESTIONS = Object.freeze({
+ 'the task is impossible. Judge in any language. Never invent source facts, credentials, permission, an '
+ 'attestation or a user answer; never propose repeating an uncertain or already dispatched submission. '
+ 'A bounded read or wait can reconcile such a dispatch. Choose continuation only when a concrete '
- + 'reasonable step remains within the existing task and normal tool permissions.',
+ + 'reasonable step remains within the existing task and normal tool permissions. A recorded result with '
+ + '`userDeclined` is the user\'s final refusal of that one action on WebBrain\'s own card: never choose a '
+ + 'continuation to retry it, ask again or reach it another way, and do not treat it as missing permission '
+ + 'for the rest of the task. Choose continue_now only when other requested work, unrelated to the refused '
+ + 'action, still remains; if the refused action was the only outstanding work, choose stop_required.',
choices: Object.freeze({
continue_now: 'A reasonable permitted inspection, source retrieval, correction or untried approach remains; the agent stopped before trying it.',
wait_then_check: 'The requested flow is still loading, uploading, validating or otherwise running, and a bounded wait followed by a fresh observation can let it continue.',
diff --git a/src/firefox/src/agent/incomplete-completion.js b/src/firefox/src/agent/incomplete-completion.js
index f8877fbe9..dd8fc43d2 100644
--- a/src/firefox/src/agent/incomplete-completion.js
+++ b/src/firefox/src/agent/incomplete-completion.js
@@ -39,15 +39,16 @@ function recordedResults(agent, tabId, run) {
for (const key of ['error', 'attachmentState', 'previousUrl', 'currentUrl']) {
if (typeof result[key] === 'string') entry[key] = text(result[key]);
}
+ // The user's own refusal on a WebBrain card (a deletion, a submit or a
+ // permission), as structured data the continuation judge can rely on.
+ if (result.deletionDeclined === true
+ || (result.denied === true && (result.submitConfirmationRequired === true || result.permissionRequired === true))) {
+ entry.userDeclined = true;
+ }
return entry;
}).filter(Boolean);
}
-function userDeclined(entry) {
- return entry.deletionDeclined === true
- || (entry.denied === true && (entry.submitConfirmationRequired === true || entry.permissionRequired === true));
-}
-
export async function incompleteCompletionBlock(agent, tabId, { outcome, summary, pageState, pageUrl = '', terminalOnly = false }) {
const guard = agent._planExecutionGuards.get(tabId);
const run = agent.completionInvariants.get(tabId);
@@ -75,11 +76,10 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summary
let parentIdentity = identity;
try { const composed = JSON.parse(identity); if (Array.isArray(composed)) parentIdentity = composed[0]; } catch {}
if (pageState.documentIdentity && pageState.documentIdentity !== parentIdentity) return changed();
+ // A refusal settles only the action it answered. The judge still decides
+ // whether other requested work remains; it never reopens the refused one.
const results = recordedResults(agent, tabId, run);
- // The user's own refusal on a WebBrain card (a deletion, a submit or a
- // permission) is the reason this run is incomplete. Reporting that is the
- // honest end; recovery must not push the model to try again.
- if (results.some(userDeclined)) return null;
+ const declined = results.some(entry => entry.userDeclined === true);
const request = agent._progressTaskAnchorText(tabId) || agent._latestTaskText(tabId) || '';
const reading = await agent._semanticDecision(tabId, 'incomplete_task', { request, page: {
candidate_summary_not_proof: text(summary),
@@ -108,7 +108,8 @@ export async function incompleteCompletionBlock(agent, tabId, { outcome, summary
recovery: reading.choice,
error: reading.choice === 'wait_then_check'
? 'The authorized task is still in progress. Wait for the relevant completion control or state with wait_for_element, or use a bounded wait_for_stable followed by a fresh page/status read. Continue the authorized flow when ready. Do not re-upload or repeat a submission already dispatched. For a longer external wait, use schedule_resume only when available and appropriate; otherwise explain the concrete remaining blocker after checking.'
- : 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',
+ : (declined ? 'The user declined one action on WebBrain\'s card; that refusal is final. Do not retry it, ask again or reach it another way. ' : '')
+ + 'The authorized task still has a reasonable next step. Inspect the current page and available source/history, retrieve the missing information when possible, and continue with the permitted tools. Use clarify for essential information only the user can provide. Do not repeat a consequential action already dispatched. Finish partial or failed only for a concrete blocker or after reasonable recovery has been tried.',
};
} catch (error) {
if (completionStopError(error, signal) || agent._isCostAllowanceError(error) || agent._checkAbort(tabId)) throw error;
diff --git a/src/firefox/src/agent/semantic-decisions.js b/src/firefox/src/agent/semantic-decisions.js
index 6380f1044..96a48c5a9 100644
--- a/src/firefox/src/agent/semantic-decisions.js
+++ b/src/firefox/src/agent/semantic-decisions.js
@@ -12,7 +12,7 @@
//
// This file is shared byte for byte by the Chrome and Firefox builds.
-export const SEMANTIC_DECISION_VERSION = 2;
+export const SEMANTIC_DECISION_VERSION = 3;
// A choice listed in `authorizing` lets an action go ahead without asking the
// user, so it needs a surer answer than the others: a higher decision-model
@@ -96,7 +96,11 @@ export const SEMANTIC_QUESTIONS = Object.freeze({
+ 'the task is impossible. Judge in any language. Never invent source facts, credentials, permission, an '
+ 'attestation or a user answer; never propose repeating an uncertain or already dispatched submission. '
+ 'A bounded read or wait can reconcile such a dispatch. Choose continuation only when a concrete '
- + 'reasonable step remains within the existing task and normal tool permissions.',
+ + 'reasonable step remains within the existing task and normal tool permissions. A recorded result with '
+ + '`userDeclined` is the user\'s final refusal of that one action on WebBrain\'s own card: never choose a '
+ + 'continuation to retry it, ask again or reach it another way, and do not treat it as missing permission '
+ + 'for the rest of the task. Choose continue_now only when other requested work, unrelated to the refused '
+ + 'action, still remains; if the refused action was the only outstanding work, choose stop_required.',
choices: Object.freeze({
continue_now: 'A reasonable permitted inspection, source retrieval, correction or untried approach remains; the agent stopped before trying it.',
wait_then_check: 'The requested flow is still loading, uploading, validating or otherwise running, and a bounded wait followed by a fresh observation can let it continue.',
diff --git a/test/incomplete-completion-recovery.mjs b/test/incomplete-completion-recovery.mjs
index b4fde1f5a..64a2b2767 100644
--- a/test/incomplete-completion-recovery.mjs
+++ b/test/incomplete-completion-recovery.mjs
@@ -130,6 +130,40 @@ for (const build of ['chrome', 'firefox']) {
assert.equal((await h.check()).recovery, 'wait_then_check');
});
+ test(`${build}: a refusal on WebBrain's card settles that action only; other requested work can still resume`, async () => {
+ for (const [declinedResult, otherWorkLeft] of [
+ [{ success: false, noDispatch: true, deletionDeclined: true, error: 'The user chose not to delete this item.' }, true],
+ [{ success: false, denied: true, submitConfirmationRequired: true, error: 'The user did not confirm submitting.' }, true],
+ [{ success: false, denied: true, permissionRequired: true, error: 'The user did not allow it.' }, false],
+ ]) {
+ const h = unitFixture();
+ h.agent.conversations.set(1, [
+ { role: 'assistant', tool_calls: [{ id: 'refused', function: { name: 'click_ax' } }, { id: 'other', function: { name: 'set_field' } }] },
+ { role: 'tool', tool_call_id: 'refused', content: JSON.stringify(declinedResult) },
+ { role: 'tool', tool_call_id: 'other', content: JSON.stringify({ success: false, denied: true, error: 'Not a user refusal.' }) },
+ ]);
+ let seen;
+ h.agent._semanticDecision = async (_tab, kind, context) => {
+ assert.equal(kind, 'incomplete_task');
+ seen = context.page.recorded_current_results;
+ return { choice: otherWorkLeft ? 'continue_now' : 'stop_required' };
+ };
+ const block = await h.check();
+ assert.deepEqual(seen.map(entry => entry.userDeclined === true), [true, false], 'Only the card refusal is marked as the user\'s');
+ if (otherWorkLeft) {
+ assert.equal(block.recovery, 'continue_now', 'Unfinished sibling work still resumes after one refusal');
+ assert.match(block.error, /refusal is final\. Do not retry it, ask again or reach it another way/);
+ } else {
+ assert.equal(block, null, 'A refusal of the only outstanding work ends honestly');
+ }
+ }
+ });
+
+ test(`${build}: without a refusal the continuation text says nothing about one`, async () => {
+ const h = unitFixture('continue_now');
+ assert.doesNotMatch((await h.check()).error, /declined/);
+ });
+
for (const streaming of [false, true]) for (const scenario of ['upload', 'missing-source']) {
test(`${build}: ${streaming ? 'stream' : 'chat'} recovers ${scenario} through the real done and tool-batch paths`, async () => {
const tab = nextTab++, updates = [], actions = [], offered = [];
diff --git a/test/quality/unbound-deletion-mv3.mjs b/test/quality/unbound-deletion-mv3.mjs
index 8f16539f9..6d3db028f 100644
--- a/test/quality/unbound-deletion-mv3.mjs
+++ b/test/quality/unbound-deletion-mv3.mjs
@@ -44,7 +44,7 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,
await fs.mkdir(out, { recursive: true });
let actionCalls = 0, judgeCalls = 0, blockedRemote = 0;
const completionRecovery = { reads: 0 };
- const deletes = [], providerRequests = [], authorizations = [], decisions = [], summaryReadings = [];
+ const deletes = [], providerRequests = [], authorizations = [], decisions = [], summaryReadings = [], continuationReadings = [];
const textOf = content => typeof content === 'string' ? content : Array.isArray(content)
? content.filter(part => part?.type === 'text').map(part => part.text || '').join('\n') : '';
const refFor = (input, name) => {
@@ -86,9 +86,14 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,
const choices = [...system.matchAll(/^- "([a-z_]+)":/gm)].map(match => match[1]);
// A failed or partial Act result is read once before it is shown; it
// gives the outcome here.
+ // A partial end is checked once for remaining work; a refused deletion
+ // must reach that judge as the user's structured refusal.
+ const continuation = choices.includes('continue_now');
if (choices.includes('mode_claim')) summaryReadings.push({ request, summary: page.summary });
+ else if (continuation) continuationReadings.push({ request, declined: (page.recorded_current_results || []).filter(entry => entry.userDeclined === true).length });
else decisions.push({ choices, request, page });
const answer = choices.includes('mode_claim') ? { choice: 'result', evidence: '' }
+ : continuation ? { choice: 'stop_required', evidence: '' }
: !choices.includes('delete_this') ? { choice: '', evidence: '' }
: cardAnswer ? { choice: 'unclear', evidence: '' }
: page.item === rows[0] ? { choice: 'delete_this', evidence } : { choice: 'other_item', evidence: '' };
@@ -206,7 +211,7 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,
const toolResults = result.updates.filter(event => event.type === 'tool_result').map(event => ({ name: event.data?.name,
blocked: event.data?.result?.blocked === true, reason: event.data?.result?.actionGuard?.reason || null, error: event.data?.result?.error || null }));
await fs.writeFile(path.join(out, 'evidence.json'), JSON.stringify({ result, diagnostic, deletes, judgeCalls, authorizations,
- decisions, summaryReadings, toolResults, prompts, cards, unexpected, blockedRemote }, (_key, value) => typeof value === 'string' && value.startsWith('data:image/') ? '[owned fixture screenshot omitted]' : value, 2));
+ decisions, summaryReadings, continuationReadings, toolResults, prompts, cards, unexpected, blockedRemote }, (_key, value) => typeof value === 'string' && value.startsWith('data:image/') ? '[owned fixture screenshot omitted]' : value, 2));
assert.deepEqual(unexpected, [], 'No other question was asked');
assert.equal(authorizations.length, 0, 'A deletion is decided by the deletion question, not the publication check');
// The request reaches the model as written, with the row and the row beside it.
@@ -230,7 +235,10 @@ async function runVariant(variant, confirmation, { task, evidence, target, kept,
assert.notEqual(partial?.outcome, 'success', JSON.stringify(partial));
assert.deepEqual(summaryReadings, [{ request: task, summary: `The user chose not to remove the ${target}; nothing was removed.` }],
'The partial result was read once, as written');
- console.log(JSON.stringify({ passed: true, installedMV3: true, variant, cards: cards.length, deletes: deletes.length, blockedRemote }));
+ assert.ok(continuationReadings.length <= 1 && continuationReadings.every(reading => reading.request === task && reading.declined === 1),
+ `A remaining-work check sees the refusal as the user's structured decision: ${JSON.stringify(continuationReadings)}`);
+ console.log(JSON.stringify({ passed: true, installedMV3: true, variant, cards: cards.length, deletes: deletes.length,
+ continuationReadings: continuationReadings.length, blockedRemote }));
return;
}
assert.deepEqual(deletes, [{ method: 'DELETE', path: '/api/cart/items/1' }], `Only the named row is removed, once: ${JSON.stringify(toolResults)}`);