Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Recover rate-limited browser inference without replaying tools

Barack Sokullu03ad91f33e51
diff --git a/package.json b/package.json
index f847e7148..35676a7e7 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
"test:captcha:bridge:ui": "node test/captcha-callback-bridge-ui.mjs",
"test:billing-notice": "node test/billing-payment-notice.mjs",
"test:provider-limits": "node test/provider-model-limits.mjs",
- "test:openrouter-capabilities": "node --test test/openrouter-model-capabilities.mjs test/muse-tool-choice.mjs",
+ "test:openrouter-capabilities": "node --test test/openrouter-model-capabilities.mjs test/muse-tool-choice.mjs test/model-retry.mjs",
"test:markdown": "node --test test/markdown-render.mjs",
"test:accessibility-tree-benchmark": "node --test test/llm/lib/accessibility-tree-formats.test.mjs && node test/llm/accessibility-tree-benchmark.mjs --no-exact-tokenizer --check",
"test:attachment-drop": "node test/attachment-drop.mjs",
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index c48f0833f..a1fad1f37 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -6,6 +6,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMo
import { validateToolArguments } from './tool-arguments.js';
import { isSessionQuotaError, serializeConversationForSession, SESSION_CONVERSATION_BUDGET_BYTES, SESSION_CONVERSATION_RETRY_BUDGET_BYTES } from './conversation-persistence.js';
import { formatErrorMessage } from '../error-format.js';
+import { retryModelCall } from '../providers/model-retry.js';
import { aggregateMessageCompletion } from '../message-info.js';
import { handleDoneJson } from './cloud-output.js';
import { applyReadPageWindow, fitReadPageWindowResult, isReadPageWindowResult } from './read-page-window.js';
@@ -43234,10 +43235,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
messages.push({ role: 'assistant', content: finalResponse });
break;
}
- // Retry once after a short delay for transient errors (rate limits, network).
- this._logDebug({ type: 'llm_error_retrying', step: steps, error: e.message });
- if (runId) await trace.recordLLMRetry(runId, steps, { delayMs: 2000, code: this._traceErrorCodeFor(e) });
- await new Promise(r => setTimeout(r, 2000));
+ // Retry inference on this step only; preserve already executed tools.
try {
const useTools2 = provider.supportsTools && tools.length > 0;
const chatOpts2 = {
@@ -43246,7 +43244,14 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
maxTokens: mainMaxTokens,
...(completionToolChoice ? { toolChoice: completionToolChoice } : {}),
};
- result = await chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' });
+ result = await retryModelCall(e, () => chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' }), {
+ isAborted: () => this._checkAbort(tabId),
+ onRetry: async ({ error, delayMs, attempt }) => {
+ this._logDebug({ type: 'llm_error_retrying', step: steps, error: error.message, delayMs, attempt });
+ if (runId) await trace.recordLLMRetry(runId, steps, { delayMs, code: this._traceErrorCodeFor(error) });
+ if (error?.httpStatus === 429) onUpdate('warning', { message: `The model provider is rate-limiting requests. Retrying this step in ${Math.ceil(delayMs / 1000)} seconds (${attempt}/3).` });
+ },
+ });
this._logDebug({ type: 'llm_response_after_retry', step: steps, content: result.content, toolCalls: result.toolCalls });
if (runId) trace.recordStepEnd(runId, steps, this._traceStepEndForResult(result, { retried: true }));
} catch (e2) {
diff --git a/src/chrome/src/providers/model-retry.js b/src/chrome/src/providers/model-retry.js
new file mode 100644
index 000000000..fbfc68053
--- /dev/null
+++ b/src/chrome/src/providers/model-retry.js
@@ -0,0 +1,45 @@
+export function retryAfterMs(value, now = Date.now()) {
+ if (typeof value !== 'string' || !value.trim()) return null;
+ const text = value.trim();
+ const ms = /^\d+(?:\.\d+)?$/.test(text) ? Number(text) * 1000 : Date.parse(text) - now;
+ return Number.isFinite(ms) ? Math.max(0, ms) : null;
+}
+
+function cancelled(isAborted) {
+ if (isAborted()) throw new DOMException('The operation was aborted', 'AbortError');
+}
+
+/** Recover a single inference turn; completed browser tools are never replayed. */
+export async function retryModelCall(initialError, call, {
+ isAborted = () => false,
+ onRetry = async () => {},
+ sleep = async ms => {
+ while (ms > 0) {
+ cancelled(isAborted);
+ const chunk = Math.min(ms, 250);
+ await new Promise(resolve => setTimeout(resolve, chunk));
+ ms -= chunk;
+ }
+ },
+} = {}) {
+ const rateLimited = initialError?.httpStatus === 429;
+ const delays = rateLimited ? [5000, 15000, 30000] : [2000];
+ let error = initialError, waited = 0;
+ for (let i = 0; i < delays.length; i++) {
+ cancelled(isAborted);
+ const hint = Number.isFinite(error?.retryAfterMs) ? Math.max(0, error.retryAfterMs) : 0;
+ const delayMs = Math.max(delays[i], hint);
+ // Never disregard a provider's longer cooldown or keep a run indefinitely.
+ if (waited + delayMs > 60000) throw error;
+ await onRetry({ error, delayMs, attempt: i + 1 });
+ await sleep(delayMs);
+ waited += delayMs;
+ cancelled(isAborted);
+ try { return await call(); }
+ catch (next) {
+ error = next;
+ if (!rateLimited || error?.httpStatus !== 429) throw error;
+ }
+ }
+ throw error;
+}
diff --git a/src/chrome/src/providers/openai.js b/src/chrome/src/providers/openai.js
index 276cba264..8ab18621b 100644
--- a/src/chrome/src/providers/openai.js
+++ b/src/chrome/src/providers/openai.js
@@ -1,4 +1,5 @@
import { BaseLLMProvider } from './base.js';
+import { retryAfterMs } from './model-retry.js';
import { fetchWithFallback } from './fetch-with-fallback.js';
import {
baseModelNameSniffedVision,
@@ -419,9 +420,10 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
return body;
}
- _httpError(status, body, prefix) {
+ _httpError(status, body, prefix, retryAfter = null) {
const error = new Error(`${prefix}: ${this._formatHttpError(status, body)}`);
error.httpStatus = status;
+ if (status === 429) error.retryAfterMs = retryAfterMs(retryAfter);
try {
const providerCode = JSON.parse(body || '{}')?.error?.code;
if (typeof providerCode === 'string' && providerCode) error.code = providerCode;
@@ -934,7 +936,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
if (!res.ok) {
let err = '';
try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }
- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);
+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));
}
let data;
try { data = await res.json(); } catch (error) {
@@ -962,7 +964,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
}
if (!res.ok) {
const err = await this._readErrorResponse(res, 1200, options);
- const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`);
+ const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));
streamError.isResponsesStreamError = true;
throw streamError;
}
@@ -1114,7 +1116,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
if (!res.ok) {
let err = '';
try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }
- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);
+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));
}
let data;
@@ -1158,7 +1160,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
if (!res.ok) {
const err = await this._readErrorResponse(res, 1200, options);
- throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`);
+ throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));
}
if (!res.body?.getReader) {
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 35f6ef9fd..675f2f79f 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -7,6 +7,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMo
import { validateToolArguments } from './tool-arguments.js';
import { isSessionQuotaError, serializeConversationForSession, SESSION_CONVERSATION_BUDGET_BYTES, SESSION_CONVERSATION_RETRY_BUDGET_BYTES } from './conversation-persistence.js';
import { formatErrorMessage } from '../error-format.js';
+import { retryModelCall } from '../providers/model-retry.js';
import { aggregateMessageCompletion } from '../message-info.js';
import { handleDoneJson } from './cloud-output.js';
import { applyReadPageWindow, fitReadPageWindowResult, isReadPageWindowResult } from './read-page-window.js';
@@ -36168,10 +36169,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
messages.push({ role: 'assistant', content: finalResponse });
break;
}
- // Retry once after a short delay for transient errors (rate limits, network).
- this._logDebug({ type: 'llm_error_retrying', step: steps, error: e.message });
- if (runId) await trace.recordLLMRetry(runId, steps, { delayMs: 2000, code: this._traceErrorCodeFor(e) });
- await new Promise(r => setTimeout(r, 2000));
+ // Retry inference on this step only; preserve already executed tools.
try {
const useTools2 = provider.supportsTools && tools.length > 0;
const chatOpts2 = {
@@ -36180,7 +36178,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
maxTokens: mainMaxTokens,
...(completionToolChoice ? { toolChoice: completionToolChoice } : {}),
};
- result = await chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' });
+ result = await retryModelCall(e, () => chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' }), {
+ isAborted: () => this._checkAbort(tabId),
+ onRetry: async ({ error, delayMs, attempt }) => {
+ this._logDebug({ type: 'llm_error_retrying', step: steps, error: error.message, delayMs, attempt });
+ if (runId) await trace.recordLLMRetry(runId, steps, { delayMs, code: this._traceErrorCodeFor(error) });
+ if (error?.httpStatus === 429) onUpdate('warning', { message: `The model provider is rate-limiting requests. Retrying this step in ${Math.ceil(delayMs / 1000)} seconds (${attempt}/3).` });
+ },
+ });
this._logDebug({ type: 'llm_response_after_retry', step: steps, content: result.content, toolCalls: result.toolCalls });
if (runId) trace.recordStepEnd(runId, steps, this._traceStepEndForResult(result, { retried: true }));
} catch (e2) {
diff --git a/src/firefox/src/providers/model-retry.js b/src/firefox/src/providers/model-retry.js
new file mode 100644
index 000000000..fbfc68053
--- /dev/null
+++ b/src/firefox/src/providers/model-retry.js
@@ -0,0 +1,45 @@
+export function retryAfterMs(value, now = Date.now()) {
+ if (typeof value !== 'string' || !value.trim()) return null;
+ const text = value.trim();
+ const ms = /^\d+(?:\.\d+)?$/.test(text) ? Number(text) * 1000 : Date.parse(text) - now;
+ return Number.isFinite(ms) ? Math.max(0, ms) : null;
+}
+
+function cancelled(isAborted) {
+ if (isAborted()) throw new DOMException('The operation was aborted', 'AbortError');
+}
+
+/** Recover a single inference turn; completed browser tools are never replayed. */
+export async function retryModelCall(initialError, call, {
+ isAborted = () => false,
+ onRetry = async () => {},
+ sleep = async ms => {
+ while (ms > 0) {
+ cancelled(isAborted);
+ const chunk = Math.min(ms, 250);
+ await new Promise(resolve => setTimeout(resolve, chunk));
+ ms -= chunk;
+ }
+ },
+} = {}) {
+ const rateLimited = initialError?.httpStatus === 429;
+ const delays = rateLimited ? [5000, 15000, 30000] : [2000];
+ let error = initialError, waited = 0;
+ for (let i = 0; i < delays.length; i++) {
+ cancelled(isAborted);
+ const hint = Number.isFinite(error?.retryAfterMs) ? Math.max(0, error.retryAfterMs) : 0;
+ const delayMs = Math.max(delays[i], hint);
+ // Never disregard a provider's longer cooldown or keep a run indefinitely.
+ if (waited + delayMs > 60000) throw error;
+ await onRetry({ error, delayMs, attempt: i + 1 });
+ await sleep(delayMs);
+ waited += delayMs;
+ cancelled(isAborted);
+ try { return await call(); }
+ catch (next) {
+ error = next;
+ if (!rateLimited || error?.httpStatus !== 429) throw error;
+ }
+ }
+ throw error;
+}
diff --git a/src/firefox/src/providers/openai.js b/src/firefox/src/providers/openai.js
index 47d03dfe0..167a4f7d0 100644
--- a/src/firefox/src/providers/openai.js
+++ b/src/firefox/src/providers/openai.js
@@ -1,4 +1,5 @@
import { BaseLLMProvider } from './base.js';
+import { retryAfterMs } from './model-retry.js';
import { fetchWithTimeout } from './fetch-timeout.js';
import {
baseModelNameSniffedVision,
@@ -436,9 +437,10 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
return body;
}
- _httpError(status, body, prefix) {
+ _httpError(status, body, prefix, retryAfter = null) {
const error = new Error(`${prefix}: ${this._formatHttpError(status, body)}`);
error.httpStatus = status;
+ if (status === 429) error.retryAfterMs = retryAfterMs(retryAfter);
try {
const providerCode = JSON.parse(body || '{}')?.error?.code;
if (typeof providerCode === 'string' && providerCode) error.code = providerCode;
@@ -951,7 +953,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
if (!res.ok) {
let err = '';
try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }
- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);
+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));
}
let data;
try { data = await res.json(); } catch (error) {
@@ -979,7 +981,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
}
if (!res.ok) {
const err = await this._readErrorResponse(res, 1200, options);
- const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`);
+ const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));
streamError.isResponsesStreamError = true;
throw streamError;
}
@@ -1131,7 +1133,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
if (!res.ok) {
let err = '';
try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }
- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);
+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));
}
let data;
@@ -1175,7 +1177,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
if (!res.ok) {
const err = await this._readErrorResponse(res, 1200, options);
- throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`);
+ throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));
}
if (!res.body?.getReader) {
diff --git a/test/model-retry.mjs b/test/model-retry.mjs
new file mode 100644
index 000000000..5deac7ee0
--- /dev/null
+++ b/test/model-retry.mjs
@@ -0,0 +1,52 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+for (const browser of ['chrome','firefox']) {
+ const {retryModelCall,retryAfterMs}=await import(`../src/${browser}/src/providers/model-retry.js`);
+ const {OpenAICompatibleProvider}=await import(`../src/${browser}/src/providers/openai.js`);
+ const limited=(hint=null)=>Object.assign(new Error('HTTP 429'),{httpStatus:429,retryAfterMs:hint});
+ test(`${browser}: retry inference with growing cooldowns without replaying tools`,async()=>{
+ const waits=[],notes=[],order=[];let inference=0,toolsAlreadyExecuted=1;
+ const answer=await retryModelCall(limited(),async()=>{order.push('inference');inference++;if(inference===1)throw limited();return {toolCalls:[]};},{sleep:async ms=>{order.push('wait');waits.push(ms);},onRetry:async note=>{await Promise.resolve();order.push('record');notes.push(note);}});
+ assert.deepEqual(answer,{toolCalls:[]});assert.deepEqual(waits,[5000,15000]);
+ assert.equal(inference,2);assert.equal(toolsAlreadyExecuted,1);assert.deepEqual(notes.map(n=>n.attempt),[1,2]);
+ assert.deepEqual(order,['record','wait','inference','record','wait','inference']);
+ });
+ test(`${browser}: persistent quota exhaustion stops after three recovery requests`,async()=>{
+ let calls=0;const waits=[];
+ await assert.rejects(retryModelCall(limited(),async()=>{calls++;throw limited();},{sleep:async ms=>waits.push(ms)}),/HTTP 429/);
+ assert.equal(calls,3);assert.deepEqual(waits,[5000,15000,30000]);
+ });
+ test(`${browser}: honor Retry-After and reject cooldowns beyond the recovery budget`,async()=>{
+ const now=Date.parse('2026-10-06T18:00:00Z');
+ assert.equal(retryAfterMs('10',now),10000);assert.equal(retryAfterMs('Tue, 06 Oct 2026 18:00:20 GMT',now),20000);
+ assert.equal(retryAfterMs('nonsense',now),null);assert.equal(retryAfterMs(null,now),null);
+ const waits=[];await retryModelCall(limited(12000),async()=> 'OK',{sleep:async ms=>waits.push(ms)});assert.deepEqual(waits,[12000]);
+ await assert.rejects(retryModelCall(limited(120000),async()=>assert.fail('Do not call before cooldown'),{sleep:async()=>assert.fail('Do not hold the browser indefinitely')}),/HTTP 429/);
+ });
+ test(`${browser}: cancellation during cooldown prevents another inference request`,async()=>{
+ let aborted=false,calls=0;
+ await assert.rejects(retryModelCall(limited(),async()=>{calls++;},{isAborted:()=>aborted,sleep:async()=>{aborted=true;}}),e=>e.name==='AbortError');
+ assert.equal(calls,0);
+ });
+ test(`${browser}: a different error stops rate-limit recovery; other errors retain one retry`,async()=>{
+ let calls=0;
+ await assert.rejects(retryModelCall(limited(),async()=>{calls++;throw Object.assign(new Error('model missing'),{httpStatus:404});},{sleep:async()=>{}}),/model missing/);assert.equal(calls,1);
+ const waits=[];calls=0;await assert.rejects(retryModelCall(new Error('network'),async()=>{calls++;throw new Error('still offline');},{sleep:async ms=>waits.push(ms)}),/still offline/);
+ assert.equal(calls,1);assert.deepEqual(waits,[2000]);
+ });
+ test(`${browser}: real provider HTTP responses carry cooldown metadata and preserve request/tool choice`,async()=>{
+ const native=globalThis.fetch,requests=[];
+ globalThis.fetch=async(_url,options)=>{
+ requests.push(JSON.parse(options.body));
+ return requests.length===1?new Response('upstream capacity limit',{status:429,headers:{'Retry-After':'10'}}):Response.json({choices:[{finish_reason:'stop',message:{content:'OK'}}]});
+ };
+ try{
+ const p=new OpenAICompatibleProvider({providerName:'webbrain_me',baseUrl:'https://openrouter.ai/api/v1',model:'inclusionai/ling-3.1-flash'});
+ const opts={tools:[{type:'function',function:{name:'read_page',parameters:{type:'object',properties:{}}}}],toolChoice:'required'};
+ let first;try{await p.chat([{role:'user',content:'Read this page'}],opts);}catch(e){first=e;}
+ assert.equal(first.httpStatus,429);assert.equal(first.retryAfterMs,10000);
+ const result=await retryModelCall(first,()=>p.chat([{role:'user',content:'Read this page'}],opts),{sleep:async()=>{}});
+ assert.equal(result.content,'OK');assert.equal(requests.length,2);assert.deepEqual(requests[0],requests[1]);assert.equal(requests[1].tool_choice,'required');
+ }finally{globalThis.fetch=native;}
+ });
+}
diff --git a/test/run.js b/test/run.js
index c020c9fb9..85bc71e75 100644
--- a/test/run.js
+++ b/test/run.js
@@ -16170,7 +16170,9 @@ test('agent trace instrumentation: turn/step boundaries, retries-before-wait, an
const streamedClose = streamingBody.indexOf('closeTraceStep(this._traceStepEndForResult');
const streamedToolBatch = streamingBody.indexOf('await this._executeToolBatch(');
assert.ok(streamedClose >= 0 && streamedClose < streamedToolBatch, `${browser}: streaming LLM step stays open through terminal tool exits`);
- assert.match(nonStreamingBody, /await trace\.recordLLMRetry\(runId, steps, \{ delayMs: 2000,[\s\S]*?await new Promise\(r => setTimeout\(r, 2000\)\)/, `${browser}: retry record is not durable before backoff`);
+ assert.match(nonStreamingBody, /retryModelCall\(e,[\s\S]*?onRetry: async[\s\S]*?await trace\.recordLLMRetry\(runId, steps, \{ delayMs,/, `${browser}: retry hook must durably record the actual backoff`);
+ const retrySource = fs.readFileSync(path.join(ROOT, `src/${browser}/src/providers/model-retry.js`), 'utf8');
+ assert.match(retrySource, /await onRetry\([\s\S]*?await sleep\(delayMs\)/, `${browser}: retry record is not durable before backoff`);
assert.match(nonStreamingBody, /await trace\.recordTurnEnd\([\s\S]*?await this\._endTraceRun/, `${browser}: non-streaming turn_end is not flushed before run close`);
assert.match(streamingBody, /await trace\.recordTurnEnd\([\s\S]*?await this\._endTraceRun/, `${browser}: streaming turn_end is not flushed before run close`);
const finishStart = streamingBody.indexOf('const finish = (response, status = _traceStatus) => {');