Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Name the earlier submission when the model fails after later steps

Claudeb83e80de3ec9
diff --git a/docs/architecture.md b/docs/architecture.md
index 14ca398fc..087aebbd7 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -1017,6 +1017,15 @@ object is recognized by its shape, and one the model rates at zero confidence
is its refusal; a plain final message in an Act run gets one recovery whatever
it says. No word list reads a summary or a final message.
+When the model request fails after an Act run sent a submission, WebBrain reads
+the page itself and completes the run only while that submission is still the
+run's latest action (or a navigation bound to its result) and the plan names at
+most one deliverable; its answer claims only that submission. Otherwise the run
+stops as partial with Retry and Give up. Right after the submission the message
+says the action may have completed. After later steps, such as opening the next
+site of a two-site post, it names the site the earlier submission went to, so
+the next run does not take a later, unsent step for a possible duplicate.
+
A request that contains one arithmetic expression (found by structure: the
one span the calculator accepts, not digits inside a word or code) is read with
the `calculation_request` question when the planner gave it no page reads,
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index fc380d861..c886f0266 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -7768,11 +7768,43 @@ export class Agent extends LoopDetector {
return descriptor;
}
+ // Whether the run's latest action is the submission it sent, or a
+ // navigation bound to that submission's result (the committed resource, or
+ // the Mastodon profile that shows the post). Only then does the current page
+ // speak for that submission. After any other action (the next site of a
+ // two-site post, say) the run has moved on to a later step.
+ _completionSubmissionIsLatestAction(tabId) {
+ const submit = this._completionSubmitStates.get(tabId);
+ const lastAction = this.completionInvariants.get(tabId)?.lastAction;
+ if (submit?.dispatched !== true || !lastAction) return false;
+ const resultSequence = submit.semanticCommit?.verificationNavigation?.actionSequence
+ ?? submit.mastodonPublication?.verificationNavigation?.actionSequence
+ ?? submit.actionSequence;
+ return Number.isFinite(Number(resultSequence)) && Number(resultSequence) === Number(lastAction.sequence);
+ }
+
+ // The site the run's submission was sent on, for the failure message.
+ _completionSubmissionHost(tabId) {
+ const submit = this._completionSubmitStates.get(tabId);
+ for (const url of [submit?.originatingUrl, submit?.currentUrl]) {
+ try {
+ const host = new URL(String(url || '')).hostname;
+ if (host) return host;
+ } catch { /* not a page address */ }
+ }
+ return '';
+ }
+
async _recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions = {}) {
const submit = this._completionSubmitStates.get(tabId);
const invariant = this.completionInvariants.get(tabId);
if (submit?.dispatched !== true || !invariant?.lastAction || !this._isActionMode(this._effectiveRunMode(tabId))
|| this._checkAbort(tabId) || this._hasPendingSteering(tabId)) return null;
+ // WebBrain reads the current page to speak for the submission, so the
+ // submission must still be the run's latest step. And when the plan names
+ // several deliverables, one verified submission cannot finish the task.
+ if (!this._completionSubmissionIsLatestAction(tabId)
+ || (this._planExecutionGuards.get(tabId)?.deliverables || []).length > 1) return null;
const binding = this._failureRecoveryBinding(tabId);
const actionSequence = invariant.lastAction.sequence;
const runToken = invariant.runToken;
@@ -7840,11 +7872,16 @@ export class Agent extends LoopDetector {
if (runId) trace.recordError(runId, step, 'model', formatErrorMessage(error), this._traceErrorCodeFor(error));
const recovered = await this._recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions);
if (recovered) return recovered;
- const postAction = this._completionSubmitStates.get(tabId)?.dispatched === true;
- const details = modelFailureRecovery(error, { postAction });
+ const submitted = this._completionSubmitStates.get(tabId)?.dispatched === true;
+ // "The action may have completed" refers to the latest step. After later
+ // steps the message names the earlier submission's site instead, so a
+ // next run does not take an unsent later step for a possible duplicate.
+ const postAction = submitted && this._completionSubmissionIsLatestAction(tabId);
+ const details = modelFailureRecovery(error, { postAction,
+ earlierSubmission: submitted && !postAction ? { host: this._completionSubmissionHost(tabId) } : null });
this._offerFailureRecovery(tabId, details, onUpdate);
onUpdate?.('error', { message: details.message });
- return { content: details.message, status: postAction ? 'partial' : 'error' };
+ return { content: details.message, status: submitted ? 'partial' : 'error' };
}
_restoreFailureRecoveryStart(tabId, requestId, onUpdate = null, retry = this._failureRecovery.monitoringRetries.get(tabId)) {
diff --git a/src/chrome/src/agent/failure-recovery.js b/src/chrome/src/agent/failure-recovery.js
index 8b50277a2..0bc71fb30 100644
--- a/src/chrome/src/agent/failure-recovery.js
+++ b/src/chrome/src/agent/failure-recovery.js
@@ -42,13 +42,22 @@ export class FailureRecoveryRegistry {
clear(tabId) { this.offers.delete(tabId); this.monitoringRetries.delete(tabId); }
}
-export function modelFailureRecovery(error, { postAction = false, phase = 'model' } = {}) {
+// postAction: the request failed while checking the submission that is the
+// run's latest action. earlierSubmission ({ host }): the run sent a submission
+// and then went on to other actions, so the request failed on a later step.
+// That message says where the submission went instead of suggesting that the
+// latest step may have submitted something.
+export function modelFailureRecovery(error, { postAction = false, earlierSubmission = null, phase = 'model' } = {}) {
const code = String(error?.code || 'model_request_failed');
const stage = postAction ? 'completion_verification' : phase;
+ const failed = code === 'webbrain_cloud_upstream_error'
+ ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request.'
+ : 'Failed while requesting the model response.';
+ const host = String(earlierSubmission?.host || '').slice(0, 253);
const message = postAction
? 'Failed during final verification. The action may have completed. Review the current result before choosing Retry.'
- : code === 'webbrain_cloud_upstream_error'
- ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request. You can retry from the current state.'
- : 'Failed while requesting the model response. You can retry from the current state.';
+ : earlierSubmission
+ ? `${failed} A submission${host ? ` on ${host}` : ''} was sent earlier in this task, so review the current result before choosing Retry.`
+ : `${failed} You can retry from the current state.`;
return { stage, code, message, retryMode: 'resume', diagnostics: String(error?.message || error || '').slice(0,6000) };
}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index d993e493a..60e8e88dc 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -7309,11 +7309,43 @@ export class Agent extends LoopDetector {
return descriptor;
}
+ // Whether the run's latest action is the submission it sent, or a
+ // navigation bound to that submission's result (the committed resource, or
+ // the Mastodon profile that shows the post). Only then does the current page
+ // speak for that submission. After any other action (the next site of a
+ // two-site post, say) the run has moved on to a later step.
+ _completionSubmissionIsLatestAction(tabId) {
+ const submit = this._completionSubmitStates.get(tabId);
+ const lastAction = this.completionInvariants.get(tabId)?.lastAction;
+ if (submit?.dispatched !== true || !lastAction) return false;
+ const resultSequence = submit.semanticCommit?.verificationNavigation?.actionSequence
+ ?? submit.mastodonPublication?.verificationNavigation?.actionSequence
+ ?? submit.actionSequence;
+ return Number.isFinite(Number(resultSequence)) && Number(resultSequence) === Number(lastAction.sequence);
+ }
+
+ // The site the run's submission was sent on, for the failure message.
+ _completionSubmissionHost(tabId) {
+ const submit = this._completionSubmitStates.get(tabId);
+ for (const url of [submit?.originatingUrl, submit?.currentUrl]) {
+ try {
+ const host = new URL(String(url || '')).hostname;
+ if (host) return host;
+ } catch { /* not a page address */ }
+ }
+ return '';
+ }
+
async _recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions = {}) {
const submit = this._completionSubmitStates.get(tabId);
const invariant = this.completionInvariants.get(tabId);
if (submit?.dispatched !== true || !invariant?.lastAction || !this._isActionMode(this._effectiveRunMode(tabId))
|| this._checkAbort(tabId) || this._hasPendingSteering(tabId)) return null;
+ // WebBrain reads the current page to speak for the submission, so the
+ // submission must still be the run's latest step. And when the plan names
+ // several deliverables, one verified submission cannot finish the task.
+ if (!this._completionSubmissionIsLatestAction(tabId)
+ || (this._planExecutionGuards.get(tabId)?.deliverables || []).length > 1) return null;
const binding = this._failureRecoveryBinding(tabId);
const actionSequence = invariant.lastAction.sequence;
const runToken = invariant.runToken;
@@ -7381,11 +7413,16 @@ export class Agent extends LoopDetector {
if (runId) trace.recordError(runId, step, 'model', formatErrorMessage(error), this._traceErrorCodeFor(error));
const recovered = await this._recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions);
if (recovered) return recovered;
- const postAction = this._completionSubmitStates.get(tabId)?.dispatched === true;
- const details = modelFailureRecovery(error, { postAction });
+ const submitted = this._completionSubmitStates.get(tabId)?.dispatched === true;
+ // "The action may have completed" refers to the latest step. After later
+ // steps the message names the earlier submission's site instead, so a
+ // next run does not take an unsent later step for a possible duplicate.
+ const postAction = submitted && this._completionSubmissionIsLatestAction(tabId);
+ const details = modelFailureRecovery(error, { postAction,
+ earlierSubmission: submitted && !postAction ? { host: this._completionSubmissionHost(tabId) } : null });
this._offerFailureRecovery(tabId, details, onUpdate);
onUpdate?.('error', { message: details.message });
- return { content: details.message, status: postAction ? 'partial' : 'error' };
+ return { content: details.message, status: submitted ? 'partial' : 'error' };
}
_restoreFailureRecoveryStart(tabId, requestId, onUpdate = null, retry = this._failureRecovery.monitoringRetries.get(tabId)) {
diff --git a/src/firefox/src/agent/failure-recovery.js b/src/firefox/src/agent/failure-recovery.js
index 8b50277a2..0bc71fb30 100644
--- a/src/firefox/src/agent/failure-recovery.js
+++ b/src/firefox/src/agent/failure-recovery.js
@@ -42,13 +42,22 @@ export class FailureRecoveryRegistry {
clear(tabId) { this.offers.delete(tabId); this.monitoringRetries.delete(tabId); }
}
-export function modelFailureRecovery(error, { postAction = false, phase = 'model' } = {}) {
+// postAction: the request failed while checking the submission that is the
+// run's latest action. earlierSubmission ({ host }): the run sent a submission
+// and then went on to other actions, so the request failed on a later step.
+// That message says where the submission went instead of suggesting that the
+// latest step may have submitted something.
+export function modelFailureRecovery(error, { postAction = false, earlierSubmission = null, phase = 'model' } = {}) {
const code = String(error?.code || 'model_request_failed');
const stage = postAction ? 'completion_verification' : phase;
+ const failed = code === 'webbrain_cloud_upstream_error'
+ ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request.'
+ : 'Failed while requesting the model response.';
+ const host = String(earlierSubmission?.host || '').slice(0, 253);
const message = postAction
? 'Failed during final verification. The action may have completed. Review the current result before choosing Retry.'
- : code === 'webbrain_cloud_upstream_error'
- ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request. You can retry from the current state.'
- : 'Failed while requesting the model response. You can retry from the current state.';
+ : earlierSubmission
+ ? `${failed} A submission${host ? ` on ${host}` : ''} was sent earlier in this task, so review the current result before choosing Retry.`
+ : `${failed} You can retry from the current state.`;
return { stage, code, message, retryMode: 'resume', diagnostics: String(error?.message || error || '').slice(0,6000) };
}
diff --git a/test/failure-recovery.mjs b/test/failure-recovery.mjs
index 692361c21..36a276499 100644
--- a/test/failure-recovery.mjs
+++ b/test/failure-recovery.mjs
@@ -179,6 +179,15 @@ for (const browser of ['chrome', 'firefox']) {
const after = modelFailureRecovery(error, { postAction: true });
assert.match(after.message, /action may have completed/);
assert.match(after.message, /Review the current result before choosing Retry/);
+ const earlier = modelFailureRecovery(error, { earlierSubmission: { host: 'mastoturk.org' } });
+ assert.equal(earlier.message, 'Failed while contacting WebBrain Cloud. The model service could not finish this request. '
+ + 'A submission on mastoturk.org was sent earlier in this task, so review the current result before choosing Retry.');
+ assert.equal(earlier.stage, 'model');
+ assert.equal(modelFailureRecovery(new Error('timed out'), { earlierSubmission: { host: '' } }).message,
+ 'Failed while requesting the model response. A submission was sent earlier in this task, so review the current result before choosing Retry.');
+ // Without a submission the messages stay as they were.
+ assert.equal(model.message, 'Failed while contacting WebBrain Cloud. The model service could not finish this request. You can retry from the current state.');
+ assert.equal(modelFailureRecovery(new Error('timed out')).message, 'Failed while requesting the model response. You can retry from the current state.');
});
test(`${browser}: checker trace budgets survive metadata projection while raw validation content stays private`, () => {
@@ -409,13 +418,42 @@ for (const browser of ['chrome', 'firefox']) {
}
});
+ test(`${browser}: a model failure after the run moved to the next site names where the earlier post went`, async () => {
+ // QA K1 with the real recorders: post on Mastodon, read the result, open
+ // x.com for the X post, then the model request fails.
+ const agent = setup(), messages = agent.conversations.get(tabId), updates = [], operations = [];
+ const home = 'https://mastoturk.org/home';
+ agent.conversationModes.set(tabId, 'act');
+ agent._planExecutionGuards.set(tabId, { requiresSubmission: true, requiresStateChange: true });
+ agent._beginCompletionInvariant(tabId);
+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_publish' }, { success: true, dispatched: true });
+ agent._recordCompletionSubmitAttempt(tabId, { isSubmit: true }, 'click_ax', { ref_id: 'ref_publish' },
+ home, home, { success: true, dispatched: true }, 'document-1', 'document-1');
+ assert.equal(agent._completionSubmissionIsLatestAction(tabId), true);
+ agent._recordCompletionToolResult(tabId, 'get_accessibility_tree', {}, { success: true, pageContent: 'article "Hello"' });
+ assert.equal(agent._completionSubmissionIsLatestAction(tabId), true, 'Reading the result is not a later step');
+ agent._recordCompletionToolResult(tabId, 'navigate', { url: 'https://x.com' },
+ { success: true, dispatched: true, verified: true, url: 'https://x.com/', pageUrlChanged: true });
+ assert.equal(agent._completionSubmissionIsLatestAction(tabId), false);
+ agent._executeToolBatch = async (_tab, calls) => { operations.push(calls[0].function.name); return { action: 'continue' }; };
+ agent._currentUrl = async () => 'https://x.com/home';
+ const error = Object.assign(new Error('local proxy timed out'), { code: 'TRANSPORT' });
+ const result = await agent._finalModelFailure(tabId, messages, (type, data) => updates.push({ type, data }), {}, error, 7);
+ assert.deepEqual(operations, [], 'Nothing is read or completed on x.com in the Mastodon post\'s name');
+ assert.equal(result.status, 'partial');
+ assert.equal(result.content, 'Failed while requesting the model response. A submission on mastoturk.org was sent '
+ + 'earlier in this task, so review the current result before choosing Retry.');
+ assert.equal(updates.find(x => x.type === 'failure_recovery').data.stage, 'model');
+ });
+
test(`${browser}: reconciliation stops on exact-task, conversation or in-place run-token changes during its read`, async () => {
for (const change of ['exact-task', 'conversation', 'run-token']) {
const agent = setup(), operations = [];
agent.conversationModes.set(tabId, 'act');
agent._beginCompletionInvariant(tabId);
agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'submit' }, { success: true, dispatched: true });
- agent._completionSubmitStates.set(tabId, { dispatched: true });
+ agent._completionSubmitStates.set(tabId, { dispatched: true,
+ actionSequence: agent.completionInvariants.get(tabId).lastAction.sequence });
agent._completionDoneBlock = () => null;
agent._executeToolBatch = async (_tab, calls) => {
const name = calls[0].function.name;
diff --git a/test/post-submit-model-recovery.mjs b/test/post-submit-model-recovery.mjs
index 38a4e24d6..ea6b42e14 100644
--- a/test/post-submit-model-recovery.mjs
+++ b/test/post-submit-model-recovery.mjs
@@ -63,7 +63,8 @@ for (const browser of ['chrome', 'firefox']) {
agent._completionDoneBlock = () => agent.completionInvariants.get(tab)?.verificationDebt ? { reason: 'verification_required' } : null;
agent.completionInvariants.set(tab, { runToken: 'owned-run', sequence: 3, verificationDebt: true,
lastAction: { name: 'click_ax', sequence: 3, uncertain: true } });
- agent._completionSubmitStates.set(tab, { dispatched: true });
+ agent._completionSubmitStates.set(tab, { dispatched: true, actionSequence: 3,
+ originatingUrl: 'https://addons.mozilla.org/developers/addon/owned/versions/submit' });
agent._executeToolBatch = async (_tab, calls, transcript) => {
const call = calls[0], name = call.function.name, args = JSON.parse(call.function.arguments);
operations.push({ name, args });
@@ -129,6 +130,44 @@ for (const browser of ['chrome', 'firefox']) {
assert.equal(await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 4), null);
assert.deepEqual(f.operations, []);
});
+ test(`${browser}: a failure after later steps names the earlier submission, not the latest step`, async () => {
+ // QA K1: the Mastodon post went out, the run opened x.com for the X post,
+ // and then the model request failed. x.com cannot speak for the Mastodon
+ // post, and "the action may have completed" made the next run treat the
+ // unsent X post as a possible duplicate.
+ const f = fixture();
+ f.agent._completionSubmitStates.set(f.tab, { dispatched: true, actionSequence: 2,
+ originatingUrl: 'https://mastoturk.org/home', currentUrl: 'https://mastoturk.org/home' });
+ f.agent._currentUrl = async () => 'https://x.com/home';
+ assert.equal(await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 7), null);
+ const error = Object.assign(new Error('offscreen proxy timed out'), { code: 'TRANSPORT' });
+ const result = await f.agent._finalModelFailure(f.tab, f.messages, f.update, {}, error, 7);
+ assert.deepEqual(f.operations, []);
+ assert.equal(result.status, 'partial');
+ assert.equal(result.content, 'Failed while requesting the model response. A submission on mastoturk.org was sent '
+ + 'earlier in this task, so review the current result before choosing Retry.');
+ const offer = f.updates.find(x => x.type === 'failure_recovery').data;
+ assert.equal(offer.stage, 'model');
+ assert.equal(offer.retryMode, 'resume');
+ assert.equal(f.updates.some(x => x.type === 'warning'), false, 'No post-submit inspection notice');
+ });
+ test(`${browser}: a navigation bound to the submission's result still lets WebBrain check it`, async () => {
+ const f = fixture();
+ f.agent._completionSubmitStates.set(f.tab, { dispatched: true, actionSequence: 2,
+ mastodonPublication: { verificationNavigation: { url: 'https://mastoturk.org/@owned', actionSequence: 3, documentToken: '' } } });
+ const result = await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 4);
+ assert.equal(result.status, 'done');
+ assert.deepEqual(f.operations.map(x => x.name), ['get_accessibility_tree', 'done']);
+ });
+ test(`${browser}: one checked submission does not finish a task with several deliverables`, async () => {
+ const f = fixture();
+ f.agent._planExecutionGuards.set(f.tab, { deliverables: ['Post the text on Mastodon.', 'Post the same text on X.'] });
+ assert.equal(await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 4), null);
+ const result = await f.agent._finalModelFailure(f.tab, f.messages, f.update, {}, new Error('timed out'), 4);
+ assert.deepEqual(f.operations, []);
+ assert.equal(result.status, 'partial');
+ assert.match(result.content, /^Failed during final verification\. The action may have completed\./);
+ });
test(`${browser}: response generation failures and empty output offer a bound generic Retry`, async () => {
for (const result of ['failure', 'empty']) {
@@ -325,7 +364,8 @@ for (const browser of ['chrome', 'firefox']) {
state.pending = structuredClone(harness.jobs().find(job => job.id === state.sourceId).pendingToolCall);
const click = { success: true, dispatched: true, verified: false, inconclusive: true, observedEffects: ['network_request'] };
agent._recordCompletionToolResult(tabId, 'click_ax', {}, click);
- agent._completionSubmitStates.set(tabId, { dispatched: true });
+ agent._completionSubmitStates.set(tabId, { dispatched: true,
+ actionSequence: agent.completionInvariants.get(tabId).lastAction.sequence });
assert.equal(await runOptions.afterConsequentialTool({ name: 'click_ax', result: click, outcomeUnknown: true }), false);
try {
state.recovered = await agent._recoverPostActionModelFailure(tabId, messages, onUpdate, provider, 3, {