Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Name the earlier submission when the model fails after later steps
Claudeb83e80de3ec9
diff --git a/docs/architecture.md b/docs/architecture.mdindex 14ca398fc..087aebbd7 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -1017,6 +1017,15 @@ object is recognized by its shape, and one the model rates at zero confidenceis its refusal; a plain final message in an Act run gets one recovery whateverit says. No word list reads a summary or a final message.+When the model request fails after an Act run sent a submission, WebBrain reads+the page itself and completes the run only while that submission is still the+run's latest action (or a navigation bound to its result) and the plan names at+most one deliverable; its answer claims only that submission. Otherwise the run+stops as partial with Retry and Give up. Right after the submission the message+says the action may have completed. After later steps, such as opening the next+site of a two-site post, it names the site the earlier submission went to, so+the next run does not take a later, unsent step for a possible duplicate.+A request that contains one arithmetic expression (found by structure: theone span the calculator accepts, not digits inside a word or code) is read withthe `calculation_request` question when the planner gave it no page reads,diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex fc380d861..c886f0266 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -7768,11 +7768,43 @@ export class Agent extends LoopDetector {return descriptor;}+ // Whether the run's latest action is the submission it sent, or a+ // navigation bound to that submission's result (the committed resource, or+ // the Mastodon profile that shows the post). Only then does the current page+ // speak for that submission. After any other action (the next site of a+ // two-site post, say) the run has moved on to a later step.+ _completionSubmissionIsLatestAction(tabId) {+ const submit = this._completionSubmitStates.get(tabId);+ const lastAction = this.completionInvariants.get(tabId)?.lastAction;+ if (submit?.dispatched !== true || !lastAction) return false;+ const resultSequence = submit.semanticCommit?.verificationNavigation?.actionSequence+ ?? submit.mastodonPublication?.verificationNavigation?.actionSequence+ ?? submit.actionSequence;+ return Number.isFinite(Number(resultSequence)) && Number(resultSequence) === Number(lastAction.sequence);+ }++ // The site the run's submission was sent on, for the failure message.+ _completionSubmissionHost(tabId) {+ const submit = this._completionSubmitStates.get(tabId);+ for (const url of [submit?.originatingUrl, submit?.currentUrl]) {+ try {+ const host = new URL(String(url || '')).hostname;+ if (host) return host;+ } catch { /* not a page address */ }+ }+ return '';+ }+async _recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions = {}) {const submit = this._completionSubmitStates.get(tabId);const invariant = this.completionInvariants.get(tabId);if (submit?.dispatched !== true || !invariant?.lastAction || !this._isActionMode(this._effectiveRunMode(tabId))|| this._checkAbort(tabId) || this._hasPendingSteering(tabId)) return null;+ // WebBrain reads the current page to speak for the submission, so the+ // submission must still be the run's latest step. And when the plan names+ // several deliverables, one verified submission cannot finish the task.+ if (!this._completionSubmissionIsLatestAction(tabId)+ || (this._planExecutionGuards.get(tabId)?.deliverables || []).length > 1) return null;const binding = this._failureRecoveryBinding(tabId);const actionSequence = invariant.lastAction.sequence;const runToken = invariant.runToken;@@ -7840,11 +7872,16 @@ export class Agent extends LoopDetector {if (runId) trace.recordError(runId, step, 'model', formatErrorMessage(error), this._traceErrorCodeFor(error));const recovered = await this._recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions);if (recovered) return recovered;- const postAction = this._completionSubmitStates.get(tabId)?.dispatched === true;- const details = modelFailureRecovery(error, { postAction });+ const submitted = this._completionSubmitStates.get(tabId)?.dispatched === true;+ // "The action may have completed" refers to the latest step. After later+ // steps the message names the earlier submission's site instead, so a+ // next run does not take an unsent later step for a possible duplicate.+ const postAction = submitted && this._completionSubmissionIsLatestAction(tabId);+ const details = modelFailureRecovery(error, { postAction,+ earlierSubmission: submitted && !postAction ? { host: this._completionSubmissionHost(tabId) } : null });this._offerFailureRecovery(tabId, details, onUpdate);onUpdate?.('error', { message: details.message });- return { content: details.message, status: postAction ? 'partial' : 'error' };+ return { content: details.message, status: submitted ? 'partial' : 'error' };}_restoreFailureRecoveryStart(tabId, requestId, onUpdate = null, retry = this._failureRecovery.monitoringRetries.get(tabId)) {diff --git a/src/chrome/src/agent/failure-recovery.js b/src/chrome/src/agent/failure-recovery.jsindex 8b50277a2..0bc71fb30 100644--- a/src/chrome/src/agent/failure-recovery.js+++ b/src/chrome/src/agent/failure-recovery.js@@ -42,13 +42,22 @@ export class FailureRecoveryRegistry {clear(tabId) { this.offers.delete(tabId); this.monitoringRetries.delete(tabId); }}-export function modelFailureRecovery(error, { postAction = false, phase = 'model' } = {}) {+// postAction: the request failed while checking the submission that is the+// run's latest action. earlierSubmission ({ host }): the run sent a submission+// and then went on to other actions, so the request failed on a later step.+// That message says where the submission went instead of suggesting that the+// latest step may have submitted something.+export function modelFailureRecovery(error, { postAction = false, earlierSubmission = null, phase = 'model' } = {}) {const code = String(error?.code || 'model_request_failed');const stage = postAction ? 'completion_verification' : phase;+ const failed = code === 'webbrain_cloud_upstream_error'+ ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request.'+ : 'Failed while requesting the model response.';+ const host = String(earlierSubmission?.host || '').slice(0, 253);const message = postAction? 'Failed during final verification. The action may have completed. Review the current result before choosing Retry.'- : code === 'webbrain_cloud_upstream_error'- ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request. You can retry from the current state.'- : 'Failed while requesting the model response. You can retry from the current state.';+ : earlierSubmission+ ? `${failed} A submission${host ? ` on ${host}` : ''} was sent earlier in this task, so review the current result before choosing Retry.`+ : `${failed} You can retry from the current state.`;return { stage, code, message, retryMode: 'resume', diagnostics: String(error?.message || error || '').slice(0,6000) };}diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex d993e493a..60e8e88dc 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -7309,11 +7309,43 @@ export class Agent extends LoopDetector {return descriptor;}+ // Whether the run's latest action is the submission it sent, or a+ // navigation bound to that submission's result (the committed resource, or+ // the Mastodon profile that shows the post). Only then does the current page+ // speak for that submission. After any other action (the next site of a+ // two-site post, say) the run has moved on to a later step.+ _completionSubmissionIsLatestAction(tabId) {+ const submit = this._completionSubmitStates.get(tabId);+ const lastAction = this.completionInvariants.get(tabId)?.lastAction;+ if (submit?.dispatched !== true || !lastAction) return false;+ const resultSequence = submit.semanticCommit?.verificationNavigation?.actionSequence+ ?? submit.mastodonPublication?.verificationNavigation?.actionSequence+ ?? submit.actionSequence;+ return Number.isFinite(Number(resultSequence)) && Number(resultSequence) === Number(lastAction.sequence);+ }++ // The site the run's submission was sent on, for the failure message.+ _completionSubmissionHost(tabId) {+ const submit = this._completionSubmitStates.get(tabId);+ for (const url of [submit?.originatingUrl, submit?.currentUrl]) {+ try {+ const host = new URL(String(url || '')).hostname;+ if (host) return host;+ } catch { /* not a page address */ }+ }+ return '';+ }+async _recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions = {}) {const submit = this._completionSubmitStates.get(tabId);const invariant = this.completionInvariants.get(tabId);if (submit?.dispatched !== true || !invariant?.lastAction || !this._isActionMode(this._effectiveRunMode(tabId))|| this._checkAbort(tabId) || this._hasPendingSteering(tabId)) return null;+ // WebBrain reads the current page to speak for the submission, so the+ // submission must still be the run's latest step. And when the plan names+ // several deliverables, one verified submission cannot finish the task.+ if (!this._completionSubmissionIsLatestAction(tabId)+ || (this._planExecutionGuards.get(tabId)?.deliverables || []).length > 1) return null;const binding = this._failureRecoveryBinding(tabId);const actionSequence = invariant.lastAction.sequence;const runToken = invariant.runToken;@@ -7381,11 +7413,16 @@ export class Agent extends LoopDetector {if (runId) trace.recordError(runId, step, 'model', formatErrorMessage(error), this._traceErrorCodeFor(error));const recovered = await this._recoverPostActionModelFailure(tabId, messages, onUpdate, provider, step, runOptions);if (recovered) return recovered;- const postAction = this._completionSubmitStates.get(tabId)?.dispatched === true;- const details = modelFailureRecovery(error, { postAction });+ const submitted = this._completionSubmitStates.get(tabId)?.dispatched === true;+ // "The action may have completed" refers to the latest step. After later+ // steps the message names the earlier submission's site instead, so a+ // next run does not take an unsent later step for a possible duplicate.+ const postAction = submitted && this._completionSubmissionIsLatestAction(tabId);+ const details = modelFailureRecovery(error, { postAction,+ earlierSubmission: submitted && !postAction ? { host: this._completionSubmissionHost(tabId) } : null });this._offerFailureRecovery(tabId, details, onUpdate);onUpdate?.('error', { message: details.message });- return { content: details.message, status: postAction ? 'partial' : 'error' };+ return { content: details.message, status: submitted ? 'partial' : 'error' };}_restoreFailureRecoveryStart(tabId, requestId, onUpdate = null, retry = this._failureRecovery.monitoringRetries.get(tabId)) {diff --git a/src/firefox/src/agent/failure-recovery.js b/src/firefox/src/agent/failure-recovery.jsindex 8b50277a2..0bc71fb30 100644--- a/src/firefox/src/agent/failure-recovery.js+++ b/src/firefox/src/agent/failure-recovery.js@@ -42,13 +42,22 @@ export class FailureRecoveryRegistry {clear(tabId) { this.offers.delete(tabId); this.monitoringRetries.delete(tabId); }}-export function modelFailureRecovery(error, { postAction = false, phase = 'model' } = {}) {+// postAction: the request failed while checking the submission that is the+// run's latest action. earlierSubmission ({ host }): the run sent a submission+// and then went on to other actions, so the request failed on a later step.+// That message says where the submission went instead of suggesting that the+// latest step may have submitted something.+export function modelFailureRecovery(error, { postAction = false, earlierSubmission = null, phase = 'model' } = {}) {const code = String(error?.code || 'model_request_failed');const stage = postAction ? 'completion_verification' : phase;+ const failed = code === 'webbrain_cloud_upstream_error'+ ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request.'+ : 'Failed while requesting the model response.';+ const host = String(earlierSubmission?.host || '').slice(0, 253);const message = postAction? 'Failed during final verification. The action may have completed. Review the current result before choosing Retry.'- : code === 'webbrain_cloud_upstream_error'- ? 'Failed while contacting WebBrain Cloud. The model service could not finish this request. You can retry from the current state.'- : 'Failed while requesting the model response. You can retry from the current state.';+ : earlierSubmission+ ? `${failed} A submission${host ? ` on ${host}` : ''} was sent earlier in this task, so review the current result before choosing Retry.`+ : `${failed} You can retry from the current state.`;return { stage, code, message, retryMode: 'resume', diagnostics: String(error?.message || error || '').slice(0,6000) };}diff --git a/test/failure-recovery.mjs b/test/failure-recovery.mjsindex 692361c21..36a276499 100644--- a/test/failure-recovery.mjs+++ b/test/failure-recovery.mjs@@ -179,6 +179,15 @@ for (const browser of ['chrome', 'firefox']) {const after = modelFailureRecovery(error, { postAction: true });assert.match(after.message, /action may have completed/);assert.match(after.message, /Review the current result before choosing Retry/);+ const earlier = modelFailureRecovery(error, { earlierSubmission: { host: 'mastoturk.org' } });+ assert.equal(earlier.message, 'Failed while contacting WebBrain Cloud. The model service could not finish this request. '+ + 'A submission on mastoturk.org was sent earlier in this task, so review the current result before choosing Retry.');+ assert.equal(earlier.stage, 'model');+ assert.equal(modelFailureRecovery(new Error('timed out'), { earlierSubmission: { host: '' } }).message,+ 'Failed while requesting the model response. A submission was sent earlier in this task, so review the current result before choosing Retry.');+ // Without a submission the messages stay as they were.+ assert.equal(model.message, 'Failed while contacting WebBrain Cloud. The model service could not finish this request. You can retry from the current state.');+ assert.equal(modelFailureRecovery(new Error('timed out')).message, 'Failed while requesting the model response. You can retry from the current state.');});test(`${browser}: checker trace budgets survive metadata projection while raw validation content stays private`, () => {@@ -409,13 +418,42 @@ for (const browser of ['chrome', 'firefox']) {}});+ test(`${browser}: a model failure after the run moved to the next site names where the earlier post went`, async () => {+ // QA K1 with the real recorders: post on Mastodon, read the result, open+ // x.com for the X post, then the model request fails.+ const agent = setup(), messages = agent.conversations.get(tabId), updates = [], operations = [];+ const home = 'https://mastoturk.org/home';+ agent.conversationModes.set(tabId, 'act');+ agent._planExecutionGuards.set(tabId, { requiresSubmission: true, requiresStateChange: true });+ agent._beginCompletionInvariant(tabId);+ agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'ref_publish' }, { success: true, dispatched: true });+ agent._recordCompletionSubmitAttempt(tabId, { isSubmit: true }, 'click_ax', { ref_id: 'ref_publish' },+ home, home, { success: true, dispatched: true }, 'document-1', 'document-1');+ assert.equal(agent._completionSubmissionIsLatestAction(tabId), true);+ agent._recordCompletionToolResult(tabId, 'get_accessibility_tree', {}, { success: true, pageContent: 'article "Hello"' });+ assert.equal(agent._completionSubmissionIsLatestAction(tabId), true, 'Reading the result is not a later step');+ agent._recordCompletionToolResult(tabId, 'navigate', { url: 'https://x.com' },+ { success: true, dispatched: true, verified: true, url: 'https://x.com/', pageUrlChanged: true });+ assert.equal(agent._completionSubmissionIsLatestAction(tabId), false);+ agent._executeToolBatch = async (_tab, calls) => { operations.push(calls[0].function.name); return { action: 'continue' }; };+ agent._currentUrl = async () => 'https://x.com/home';+ const error = Object.assign(new Error('local proxy timed out'), { code: 'TRANSPORT' });+ const result = await agent._finalModelFailure(tabId, messages, (type, data) => updates.push({ type, data }), {}, error, 7);+ assert.deepEqual(operations, [], 'Nothing is read or completed on x.com in the Mastodon post\'s name');+ assert.equal(result.status, 'partial');+ assert.equal(result.content, 'Failed while requesting the model response. A submission on mastoturk.org was sent '+ + 'earlier in this task, so review the current result before choosing Retry.');+ assert.equal(updates.find(x => x.type === 'failure_recovery').data.stage, 'model');+ });+test(`${browser}: reconciliation stops on exact-task, conversation or in-place run-token changes during its read`, async () => {for (const change of ['exact-task', 'conversation', 'run-token']) {const agent = setup(), operations = [];agent.conversationModes.set(tabId, 'act');agent._beginCompletionInvariant(tabId);agent._recordCompletionToolResult(tabId, 'click_ax', { ref_id: 'submit' }, { success: true, dispatched: true });- agent._completionSubmitStates.set(tabId, { dispatched: true });+ agent._completionSubmitStates.set(tabId, { dispatched: true,+ actionSequence: agent.completionInvariants.get(tabId).lastAction.sequence });agent._completionDoneBlock = () => null;agent._executeToolBatch = async (_tab, calls) => {const name = calls[0].function.name;diff --git a/test/post-submit-model-recovery.mjs b/test/post-submit-model-recovery.mjsindex 38a4e24d6..ea6b42e14 100644--- a/test/post-submit-model-recovery.mjs+++ b/test/post-submit-model-recovery.mjs@@ -63,7 +63,8 @@ for (const browser of ['chrome', 'firefox']) {agent._completionDoneBlock = () => agent.completionInvariants.get(tab)?.verificationDebt ? { reason: 'verification_required' } : null;agent.completionInvariants.set(tab, { runToken: 'owned-run', sequence: 3, verificationDebt: true,lastAction: { name: 'click_ax', sequence: 3, uncertain: true } });- agent._completionSubmitStates.set(tab, { dispatched: true });+ agent._completionSubmitStates.set(tab, { dispatched: true, actionSequence: 3,+ originatingUrl: 'https://addons.mozilla.org/developers/addon/owned/versions/submit' });agent._executeToolBatch = async (_tab, calls, transcript) => {const call = calls[0], name = call.function.name, args = JSON.parse(call.function.arguments);operations.push({ name, args });@@ -129,6 +130,44 @@ for (const browser of ['chrome', 'firefox']) {assert.equal(await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 4), null);assert.deepEqual(f.operations, []);});+ test(`${browser}: a failure after later steps names the earlier submission, not the latest step`, async () => {+ // QA K1: the Mastodon post went out, the run opened x.com for the X post,+ // and then the model request failed. x.com cannot speak for the Mastodon+ // post, and "the action may have completed" made the next run treat the+ // unsent X post as a possible duplicate.+ const f = fixture();+ f.agent._completionSubmitStates.set(f.tab, { dispatched: true, actionSequence: 2,+ originatingUrl: 'https://mastoturk.org/home', currentUrl: 'https://mastoturk.org/home' });+ f.agent._currentUrl = async () => 'https://x.com/home';+ assert.equal(await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 7), null);+ const error = Object.assign(new Error('offscreen proxy timed out'), { code: 'TRANSPORT' });+ const result = await f.agent._finalModelFailure(f.tab, f.messages, f.update, {}, error, 7);+ assert.deepEqual(f.operations, []);+ assert.equal(result.status, 'partial');+ assert.equal(result.content, 'Failed while requesting the model response. A submission on mastoturk.org was sent '+ + 'earlier in this task, so review the current result before choosing Retry.');+ const offer = f.updates.find(x => x.type === 'failure_recovery').data;+ assert.equal(offer.stage, 'model');+ assert.equal(offer.retryMode, 'resume');+ assert.equal(f.updates.some(x => x.type === 'warning'), false, 'No post-submit inspection notice');+ });+ test(`${browser}: a navigation bound to the submission's result still lets WebBrain check it`, async () => {+ const f = fixture();+ f.agent._completionSubmitStates.set(f.tab, { dispatched: true, actionSequence: 2,+ mastodonPublication: { verificationNavigation: { url: 'https://mastoturk.org/@owned', actionSequence: 3, documentToken: '' } } });+ const result = await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 4);+ assert.equal(result.status, 'done');+ assert.deepEqual(f.operations.map(x => x.name), ['get_accessibility_tree', 'done']);+ });+ test(`${browser}: one checked submission does not finish a task with several deliverables`, async () => {+ const f = fixture();+ f.agent._planExecutionGuards.set(f.tab, { deliverables: ['Post the text on Mastodon.', 'Post the same text on X.'] });+ assert.equal(await f.agent._recoverPostActionModelFailure(f.tab, f.messages, f.update, {}, 4), null);+ const result = await f.agent._finalModelFailure(f.tab, f.messages, f.update, {}, new Error('timed out'), 4);+ assert.deepEqual(f.operations, []);+ assert.equal(result.status, 'partial');+ assert.match(result.content, /^Failed during final verification\. The action may have completed\./);+ });test(`${browser}: response generation failures and empty output offer a bound generic Retry`, async () => {for (const result of ['failure', 'empty']) {@@ -325,7 +364,8 @@ for (const browser of ['chrome', 'firefox']) {state.pending = structuredClone(harness.jobs().find(job => job.id === state.sourceId).pendingToolCall);const click = { success: true, dispatched: true, verified: false, inconclusive: true, observedEffects: ['network_request'] };agent._recordCompletionToolResult(tabId, 'click_ax', {}, click);- agent._completionSubmitStates.set(tabId, { dispatched: true });+ agent._completionSubmitStates.set(tabId, { dispatched: true,+ actionSequence: agent.completionInvariants.get(tabId).lastAction.sequence });assert.equal(await runOptions.afterConsequentialTool({ name: 'click_ax', result: click, outcomeUnknown: true }), false);try {state.recovered = await agent._recoverPostActionModelFailure(tabId, messages, onUpdate, provider, 3, {