Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Recover rate-limited browser inference without replaying tools
Barack Sokullu03ad91f33e51
diff --git a/package.json b/package.jsonindex f847e7148..35676a7e7 100644--- a/package.json+++ b/package.json@@ -10,7 +10,7 @@"test:captcha:bridge:ui": "node test/captcha-callback-bridge-ui.mjs","test:billing-notice": "node test/billing-payment-notice.mjs","test:provider-limits": "node test/provider-model-limits.mjs",- "test:openrouter-capabilities": "node --test test/openrouter-model-capabilities.mjs test/muse-tool-choice.mjs",+ "test:openrouter-capabilities": "node --test test/openrouter-model-capabilities.mjs test/muse-tool-choice.mjs test/model-retry.mjs","test:markdown": "node --test test/markdown-render.mjs","test:accessibility-tree-benchmark": "node --test test/llm/lib/accessibility-tree-formats.test.mjs && node test/llm/accessibility-tree-benchmark.mjs --no-exact-tokenizer --check","test:attachment-drop": "node test/attachment-drop.mjs",diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex c48f0833f..a1fad1f37 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -6,6 +6,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMoimport { validateToolArguments } from './tool-arguments.js';import { isSessionQuotaError, serializeConversationForSession, SESSION_CONVERSATION_BUDGET_BYTES, SESSION_CONVERSATION_RETRY_BUDGET_BYTES } from './conversation-persistence.js';import { formatErrorMessage } from '../error-format.js';+import { retryModelCall } from '../providers/model-retry.js';import { aggregateMessageCompletion } from '../message-info.js';import { handleDoneJson } from './cloud-output.js';import { applyReadPageWindow, fitReadPageWindowResult, isReadPageWindowResult } from './read-page-window.js';@@ -43234,10 +43235,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaimessages.push({ role: 'assistant', content: finalResponse });break;}- // Retry once after a short delay for transient errors (rate limits, network).- this._logDebug({ type: 'llm_error_retrying', step: steps, error: e.message });- if (runId) await trace.recordLLMRetry(runId, steps, { delayMs: 2000, code: this._traceErrorCodeFor(e) });- await new Promise(r => setTimeout(r, 2000));+ // Retry inference on this step only; preserve already executed tools.try {const useTools2 = provider.supportsTools && tools.length > 0;const chatOpts2 = {@@ -43246,7 +43244,14 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaimaxTokens: mainMaxTokens,...(completionToolChoice ? { toolChoice: completionToolChoice } : {}),};- result = await chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' });+ result = await retryModelCall(e, () => chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' }), {+ isAborted: () => this._checkAbort(tabId),+ onRetry: async ({ error, delayMs, attempt }) => {+ this._logDebug({ type: 'llm_error_retrying', step: steps, error: error.message, delayMs, attempt });+ if (runId) await trace.recordLLMRetry(runId, steps, { delayMs, code: this._traceErrorCodeFor(error) });+ if (error?.httpStatus === 429) onUpdate('warning', { message: `The model provider is rate-limiting requests. Retrying this step in ${Math.ceil(delayMs / 1000)} seconds (${attempt}/3).` });+ },+ });this._logDebug({ type: 'llm_response_after_retry', step: steps, content: result.content, toolCalls: result.toolCalls });if (runId) trace.recordStepEnd(runId, steps, this._traceStepEndForResult(result, { retried: true }));} catch (e2) {diff --git a/src/chrome/src/providers/model-retry.js b/src/chrome/src/providers/model-retry.jsnew file mode 100644index 000000000..fbfc68053--- /dev/null+++ b/src/chrome/src/providers/model-retry.js@@ -0,0 +1,45 @@+export function retryAfterMs(value, now = Date.now()) {+ if (typeof value !== 'string' || !value.trim()) return null;+ const text = value.trim();+ const ms = /^\d+(?:\.\d+)?$/.test(text) ? Number(text) * 1000 : Date.parse(text) - now;+ return Number.isFinite(ms) ? Math.max(0, ms) : null;+}++function cancelled(isAborted) {+ if (isAborted()) throw new DOMException('The operation was aborted', 'AbortError');+}++/** Recover a single inference turn; completed browser tools are never replayed. */+export async function retryModelCall(initialError, call, {+ isAborted = () => false,+ onRetry = async () => {},+ sleep = async ms => {+ while (ms > 0) {+ cancelled(isAborted);+ const chunk = Math.min(ms, 250);+ await new Promise(resolve => setTimeout(resolve, chunk));+ ms -= chunk;+ }+ },+} = {}) {+ const rateLimited = initialError?.httpStatus === 429;+ const delays = rateLimited ? [5000, 15000, 30000] : [2000];+ let error = initialError, waited = 0;+ for (let i = 0; i < delays.length; i++) {+ cancelled(isAborted);+ const hint = Number.isFinite(error?.retryAfterMs) ? Math.max(0, error.retryAfterMs) : 0;+ const delayMs = Math.max(delays[i], hint);+ // Never disregard a provider's longer cooldown or keep a run indefinitely.+ if (waited + delayMs > 60000) throw error;+ await onRetry({ error, delayMs, attempt: i + 1 });+ await sleep(delayMs);+ waited += delayMs;+ cancelled(isAborted);+ try { return await call(); }+ catch (next) {+ error = next;+ if (!rateLimited || error?.httpStatus !== 429) throw error;+ }+ }+ throw error;+}diff --git a/src/chrome/src/providers/openai.js b/src/chrome/src/providers/openai.jsindex 276cba264..8ab18621b 100644--- a/src/chrome/src/providers/openai.js+++ b/src/chrome/src/providers/openai.js@@ -1,4 +1,5 @@import { BaseLLMProvider } from './base.js';+import { retryAfterMs } from './model-retry.js';import { fetchWithFallback } from './fetch-with-fallback.js';import {baseModelNameSniffedVision,@@ -419,9 +420,10 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {return body;}- _httpError(status, body, prefix) {+ _httpError(status, body, prefix, retryAfter = null) {const error = new Error(`${prefix}: ${this._formatHttpError(status, body)}`);error.httpStatus = status;+ if (status === 429) error.retryAfterMs = retryAfterMs(retryAfter);try {const providerCode = JSON.parse(body || '{}')?.error?.code;if (typeof providerCode === 'string' && providerCode) error.code = providerCode;@@ -934,7 +936,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {if (!res.ok) {let err = '';try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));}let data;try { data = await res.json(); } catch (error) {@@ -962,7 +964,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {}if (!res.ok) {const err = await this._readErrorResponse(res, 1200, options);- const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`);+ const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));streamError.isResponsesStreamError = true;throw streamError;}@@ -1114,7 +1116,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {if (!res.ok) {let err = '';try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));}let data;@@ -1158,7 +1160,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {if (!res.ok) {const err = await this._readErrorResponse(res, 1200, options);- throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`);+ throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));}if (!res.body?.getReader) {diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 35f6ef9fd..675f2f79f 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -7,6 +7,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMoimport { validateToolArguments } from './tool-arguments.js';import { isSessionQuotaError, serializeConversationForSession, SESSION_CONVERSATION_BUDGET_BYTES, SESSION_CONVERSATION_RETRY_BUDGET_BYTES } from './conversation-persistence.js';import { formatErrorMessage } from '../error-format.js';+import { retryModelCall } from '../providers/model-retry.js';import { aggregateMessageCompletion } from '../message-info.js';import { handleDoneJson } from './cloud-output.js';import { applyReadPageWindow, fitReadPageWindowResult, isReadPageWindowResult } from './read-page-window.js';@@ -36168,10 +36169,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dmessages.push({ role: 'assistant', content: finalResponse });break;}- // Retry once after a short delay for transient errors (rate limits, network).- this._logDebug({ type: 'llm_error_retrying', step: steps, error: e.message });- if (runId) await trace.recordLLMRetry(runId, steps, { delayMs: 2000, code: this._traceErrorCodeFor(e) });- await new Promise(r => setTimeout(r, 2000));+ // Retry inference on this step only; preserve already executed tools.try {const useTools2 = provider.supportsTools && tools.length > 0;const chatOpts2 = {@@ -36180,7 +36178,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dmaxTokens: mainMaxTokens,...(completionToolChoice ? { toolChoice: completionToolChoice } : {}),};- result = await chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' });+ result = await retryModelCall(e, () => chatMainTurn(this._pruneOldImages(modelMessagesForRun(), provider), chatOpts2, { tabId, generationName: 'main' }), {+ isAborted: () => this._checkAbort(tabId),+ onRetry: async ({ error, delayMs, attempt }) => {+ this._logDebug({ type: 'llm_error_retrying', step: steps, error: error.message, delayMs, attempt });+ if (runId) await trace.recordLLMRetry(runId, steps, { delayMs, code: this._traceErrorCodeFor(error) });+ if (error?.httpStatus === 429) onUpdate('warning', { message: `The model provider is rate-limiting requests. Retrying this step in ${Math.ceil(delayMs / 1000)} seconds (${attempt}/3).` });+ },+ });this._logDebug({ type: 'llm_response_after_retry', step: steps, content: result.content, toolCalls: result.toolCalls });if (runId) trace.recordStepEnd(runId, steps, this._traceStepEndForResult(result, { retried: true }));} catch (e2) {diff --git a/src/firefox/src/providers/model-retry.js b/src/firefox/src/providers/model-retry.jsnew file mode 100644index 000000000..fbfc68053--- /dev/null+++ b/src/firefox/src/providers/model-retry.js@@ -0,0 +1,45 @@+export function retryAfterMs(value, now = Date.now()) {+ if (typeof value !== 'string' || !value.trim()) return null;+ const text = value.trim();+ const ms = /^\d+(?:\.\d+)?$/.test(text) ? Number(text) * 1000 : Date.parse(text) - now;+ return Number.isFinite(ms) ? Math.max(0, ms) : null;+}++function cancelled(isAborted) {+ if (isAborted()) throw new DOMException('The operation was aborted', 'AbortError');+}++/** Recover a single inference turn; completed browser tools are never replayed. */+export async function retryModelCall(initialError, call, {+ isAborted = () => false,+ onRetry = async () => {},+ sleep = async ms => {+ while (ms > 0) {+ cancelled(isAborted);+ const chunk = Math.min(ms, 250);+ await new Promise(resolve => setTimeout(resolve, chunk));+ ms -= chunk;+ }+ },+} = {}) {+ const rateLimited = initialError?.httpStatus === 429;+ const delays = rateLimited ? [5000, 15000, 30000] : [2000];+ let error = initialError, waited = 0;+ for (let i = 0; i < delays.length; i++) {+ cancelled(isAborted);+ const hint = Number.isFinite(error?.retryAfterMs) ? Math.max(0, error.retryAfterMs) : 0;+ const delayMs = Math.max(delays[i], hint);+ // Never disregard a provider's longer cooldown or keep a run indefinitely.+ if (waited + delayMs > 60000) throw error;+ await onRetry({ error, delayMs, attempt: i + 1 });+ await sleep(delayMs);+ waited += delayMs;+ cancelled(isAborted);+ try { return await call(); }+ catch (next) {+ error = next;+ if (!rateLimited || error?.httpStatus !== 429) throw error;+ }+ }+ throw error;+}diff --git a/src/firefox/src/providers/openai.js b/src/firefox/src/providers/openai.jsindex 47d03dfe0..167a4f7d0 100644--- a/src/firefox/src/providers/openai.js+++ b/src/firefox/src/providers/openai.js@@ -1,4 +1,5 @@import { BaseLLMProvider } from './base.js';+import { retryAfterMs } from './model-retry.js';import { fetchWithTimeout } from './fetch-timeout.js';import {baseModelNameSniffedVision,@@ -436,9 +437,10 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {return body;}- _httpError(status, body, prefix) {+ _httpError(status, body, prefix, retryAfter = null) {const error = new Error(`${prefix}: ${this._formatHttpError(status, body)}`);error.httpStatus = status;+ if (status === 429) error.retryAfterMs = retryAfterMs(retryAfter);try {const providerCode = JSON.parse(body || '{}')?.error?.code;if (typeof providerCode === 'string' && providerCode) error.code = providerCode;@@ -951,7 +953,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {if (!res.ok) {let err = '';try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));}let data;try { data = await res.json(); } catch (error) {@@ -979,7 +981,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {}if (!res.ok) {const err = await this._readErrorResponse(res, 1200, options);- const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`);+ const streamError = this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));streamError.isResponsesStreamError = true;throw streamError;}@@ -1131,7 +1133,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {if (!res.ok) {let err = '';try { err = await this._readErrorResponse(res, 500, options); } catch (error) { this._rethrowAbortedChat(error, options); }- throw this._httpError(res.status, err, `${this.name} error ${res.status}`);+ throw this._httpError(res.status, err, `${this.name} error ${res.status}`, res.headers?.get?.('retry-after'));}let data;@@ -1175,7 +1177,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {if (!res.ok) {const err = await this._readErrorResponse(res, 1200, options);- throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`);+ throw this._httpError(res.status, err, `${this.name} stream error ${res.status}`, res.headers?.get?.('retry-after'));}if (!res.body?.getReader) {diff --git a/test/model-retry.mjs b/test/model-retry.mjsnew file mode 100644index 000000000..5deac7ee0--- /dev/null+++ b/test/model-retry.mjs@@ -0,0 +1,52 @@+import test from 'node:test';+import assert from 'node:assert/strict';+for (const browser of ['chrome','firefox']) {+ const {retryModelCall,retryAfterMs}=await import(`../src/${browser}/src/providers/model-retry.js`);+ const {OpenAICompatibleProvider}=await import(`../src/${browser}/src/providers/openai.js`);+ const limited=(hint=null)=>Object.assign(new Error('HTTP 429'),{httpStatus:429,retryAfterMs:hint});+ test(`${browser}: retry inference with growing cooldowns without replaying tools`,async()=>{+ const waits=[],notes=[],order=[];let inference=0,toolsAlreadyExecuted=1;+ const answer=await retryModelCall(limited(),async()=>{order.push('inference');inference++;if(inference===1)throw limited();return {toolCalls:[]};},{sleep:async ms=>{order.push('wait');waits.push(ms);},onRetry:async note=>{await Promise.resolve();order.push('record');notes.push(note);}});+ assert.deepEqual(answer,{toolCalls:[]});assert.deepEqual(waits,[5000,15000]);+ assert.equal(inference,2);assert.equal(toolsAlreadyExecuted,1);assert.deepEqual(notes.map(n=>n.attempt),[1,2]);+ assert.deepEqual(order,['record','wait','inference','record','wait','inference']);+ });+ test(`${browser}: persistent quota exhaustion stops after three recovery requests`,async()=>{+ let calls=0;const waits=[];+ await assert.rejects(retryModelCall(limited(),async()=>{calls++;throw limited();},{sleep:async ms=>waits.push(ms)}),/HTTP 429/);+ assert.equal(calls,3);assert.deepEqual(waits,[5000,15000,30000]);+ });+ test(`${browser}: honor Retry-After and reject cooldowns beyond the recovery budget`,async()=>{+ const now=Date.parse('2026-10-06T18:00:00Z');+ assert.equal(retryAfterMs('10',now),10000);assert.equal(retryAfterMs('Tue, 06 Oct 2026 18:00:20 GMT',now),20000);+ assert.equal(retryAfterMs('nonsense',now),null);assert.equal(retryAfterMs(null,now),null);+ const waits=[];await retryModelCall(limited(12000),async()=> 'OK',{sleep:async ms=>waits.push(ms)});assert.deepEqual(waits,[12000]);+ await assert.rejects(retryModelCall(limited(120000),async()=>assert.fail('Do not call before cooldown'),{sleep:async()=>assert.fail('Do not hold the browser indefinitely')}),/HTTP 429/);+ });+ test(`${browser}: cancellation during cooldown prevents another inference request`,async()=>{+ let aborted=false,calls=0;+ await assert.rejects(retryModelCall(limited(),async()=>{calls++;},{isAborted:()=>aborted,sleep:async()=>{aborted=true;}}),e=>e.name==='AbortError');+ assert.equal(calls,0);+ });+ test(`${browser}: a different error stops rate-limit recovery; other errors retain one retry`,async()=>{+ let calls=0;+ await assert.rejects(retryModelCall(limited(),async()=>{calls++;throw Object.assign(new Error('model missing'),{httpStatus:404});},{sleep:async()=>{}}),/model missing/);assert.equal(calls,1);+ const waits=[];calls=0;await assert.rejects(retryModelCall(new Error('network'),async()=>{calls++;throw new Error('still offline');},{sleep:async ms=>waits.push(ms)}),/still offline/);+ assert.equal(calls,1);assert.deepEqual(waits,[2000]);+ });+ test(`${browser}: real provider HTTP responses carry cooldown metadata and preserve request/tool choice`,async()=>{+ const native=globalThis.fetch,requests=[];+ globalThis.fetch=async(_url,options)=>{+ requests.push(JSON.parse(options.body));+ return requests.length===1?new Response('upstream capacity limit',{status:429,headers:{'Retry-After':'10'}}):Response.json({choices:[{finish_reason:'stop',message:{content:'OK'}}]});+ };+ try{+ const p=new OpenAICompatibleProvider({providerName:'webbrain_me',baseUrl:'https://openrouter.ai/api/v1',model:'inclusionai/ling-3.1-flash'});+ const opts={tools:[{type:'function',function:{name:'read_page',parameters:{type:'object',properties:{}}}}],toolChoice:'required'};+ let first;try{await p.chat([{role:'user',content:'Read this page'}],opts);}catch(e){first=e;}+ assert.equal(first.httpStatus,429);assert.equal(first.retryAfterMs,10000);+ const result=await retryModelCall(first,()=>p.chat([{role:'user',content:'Read this page'}],opts),{sleep:async()=>{}});+ assert.equal(result.content,'OK');assert.equal(requests.length,2);assert.deepEqual(requests[0],requests[1]);assert.equal(requests[1].tool_choice,'required');+ }finally{globalThis.fetch=native;}+ });+}diff --git a/test/run.js b/test/run.jsindex c020c9fb9..85bc71e75 100644--- a/test/run.js+++ b/test/run.js@@ -16170,7 +16170,9 @@ test('agent trace instrumentation: turn/step boundaries, retries-before-wait, anconst streamedClose = streamingBody.indexOf('closeTraceStep(this._traceStepEndForResult');const streamedToolBatch = streamingBody.indexOf('await this._executeToolBatch(');assert.ok(streamedClose >= 0 && streamedClose < streamedToolBatch, `${browser}: streaming LLM step stays open through terminal tool exits`);- assert.match(nonStreamingBody, /await trace\.recordLLMRetry\(runId, steps, \{ delayMs: 2000,[\s\S]*?await new Promise\(r => setTimeout\(r, 2000\)\)/, `${browser}: retry record is not durable before backoff`);+ assert.match(nonStreamingBody, /retryModelCall\(e,[\s\S]*?onRetry: async[\s\S]*?await trace\.recordLLMRetry\(runId, steps, \{ delayMs,/, `${browser}: retry hook must durably record the actual backoff`);+ const retrySource = fs.readFileSync(path.join(ROOT, `src/${browser}/src/providers/model-retry.js`), 'utf8');+ assert.match(retrySource, /await onRetry\([\s\S]*?await sleep\(delayMs\)/, `${browser}: retry record is not durable before backoff`);assert.match(nonStreamingBody, /await trace\.recordTurnEnd\([\s\S]*?await this\._endTraceRun/, `${browser}: non-streaming turn_end is not flushed before run close`);assert.match(streamingBody, /await trace\.recordTurnEnd\([\s\S]*?await this\._endTraceRun/, `${browser}: streaming turn_end is not flushed before run close`);const finishStart = streamingBody.indexOf('const finish = (response, status = _traceStatus) => {');