Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔁 fix: Retry Code API Rate Limits That Reject Before a Workspace Call Starts (#16453)

Danny Avilaf624ad08763e
diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js
index 6371b1c40f..a5c2bbdfe9 100644
--- a/api/server/services/Files/Code/process.js
+++ b/api/server/services/Files/Code/process.js
@@ -1094,6 +1094,7 @@ async function readWorkspaceFile({
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
+ codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
@@ -1147,6 +1148,7 @@ async function searchWorkspace({
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
+ codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
@@ -1200,6 +1202,7 @@ async function listWorkspaceFiles({
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
+ codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
@@ -1239,6 +1242,7 @@ async function writeWorkspaceFile({
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
+ codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
@@ -1278,6 +1282,7 @@ async function editWorkspaceFile({
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
+ codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
@@ -1316,6 +1321,7 @@ async function previewWorkspaceEdit({
return executeWorkspaceTool({
baseURL: codeApiBaseUrl,
maxQueueWaitMs,
+ codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js
index 875368196f..65175294c0 100644
--- a/api/server/services/Files/Code/process.spec.js
+++ b/api/server/services/Files/Code/process.spec.js
@@ -2033,6 +2033,28 @@ describe('Code Process', () => {
});
describe('readWorkspaceFile', () => {
+ it('forwards the configured rate-limit budget to the workspace transport', async () => {
+ const req = {
+ ...mockReq,
+ config: { ...mockReq.config, endpoints: { agents: { codeApiMaxRetryWaitMs: 0 } } },
+ };
+ mockExecuteWorkspaceTool.mockResolvedValueOnce({ operation: 'read_file' });
+
+ await readWorkspaceFile({
+ file_path: 'src/app.ts',
+ workspace_id: 'primary',
+ start_line: 1,
+ max_lines: 10,
+ codeApiBaseUrl: 'https://attached-code.example.com/v1',
+ executionProfile: 'stateful',
+ req,
+ });
+
+ expect(mockExecuteWorkspaceTool).toHaveBeenCalledWith(
+ expect.objectContaining({ codeApiMaxRetryWaitMs: 0 }),
+ );
+ });
+
it('forwards authenticated reads to the selected attached worker', async () => {
const controller = new AbortController();
const result = {
@@ -2083,6 +2105,7 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: undefined,
maxRequestTimeoutMs: 125_000,
deadlineAtMs: 160_000,
request: {
@@ -2144,6 +2167,7 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: undefined,
maxRequestTimeoutMs: undefined,
deadlineAtMs: undefined,
request: {
@@ -2204,6 +2228,7 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: undefined,
maxRequestTimeoutMs: undefined,
deadlineAtMs: undefined,
request: {
@@ -2266,6 +2291,7 @@ describe('Code Process', () => {
baseURL: 'https://attached-code.example.com/v1',
authHeaders: expect.any(Function),
maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: undefined,
maxRequestTimeoutMs: undefined,
deadlineAtMs: undefined,
request: {
diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js
index f215d60cda..b047447ec1 100644
--- a/api/server/services/ToolService.js
+++ b/api/server/services/ToolService.js
@@ -1578,6 +1578,7 @@ async function loadToolDefinitionsWrapper({
enabled: codeExecutionEnabled,
context: resolvedCodeExecutionContext,
principalId: JSON.stringify([getTenantId(), req.user.id]),
+ codeApiMaxRetryWaitMs: req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
getAuthHeaders: (workerId) => getCodeApiAuthHeaders(req, workerId),
}),
};
@@ -1781,6 +1782,7 @@ async function loadAgentTools({
enabled: codeExecutionEnabled,
context: codeExecutionContext,
principalId: JSON.stringify([getTenantId(), req.user.id]),
+ codeApiMaxRetryWaitMs: req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
getAuthHeaders: (workerId) => getCodeApiAuthHeaders(req, workerId),
});
const { loadedTools, toolContextMap, dynamicToolContextMap, primedCodeFiles } = await loadTools({
@@ -2315,6 +2317,7 @@ async function loadToolsForExecution({
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(
codeExecutionContext.codeEnvironmentConfigSchema,
),
+ codeApiMaxRetryWaitMs: req.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(
codeExecutionContext.codeEnvironmentConfigSchema,
),
diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js
index dd388f232d..9c4d57eefd 100644
--- a/api/server/services/__tests__/ToolService.spec.js
+++ b/api/server/services/__tests__/ToolService.spec.js
@@ -713,6 +713,43 @@ describe('ToolService - Action Capability Gating', () => {
});
});
+ it.each([true, false])(
+ 'passes the Code API retry limit to repository instructions (definitionsOnly=%s)',
+ async (definitionsOnly) => {
+ const capabilities = [
+ AgentCapabilities.tools,
+ AgentCapabilities.execute_code,
+ AgentCapabilities.stateful_code_sessions,
+ ];
+ const req = createMockReq(capabilities);
+ req.config.endpoints[EModelEndpoint.agents].codeApiMaxRetryWaitMs = 0;
+ mockGetEndpointsConfig.mockResolvedValue(createEndpointsConfig(capabilities));
+ mockResolveCodeExecutionContext.mockReturnValueOnce({
+ baseUrl: 'https://attached-code.example.com/v1',
+ codeSessionKey: 'attached-session',
+ executionProfile: 'stateful',
+ statefulSessions: true,
+ environmentType: 'attached',
+ environmentId: 'personal-machine',
+ });
+
+ const result = await loadAgentTools({
+ req,
+ res: {},
+ agent: {
+ id: 'attached-agent',
+ tools: [Tools.execute_code],
+ stateful_code_sessions: true,
+ },
+ definitionsOnly,
+ });
+
+ expect(result.repositoryInstructionSource).toEqual(
+ expect.objectContaining({ codeApiMaxRetryWaitMs: 0 }),
+ );
+ },
+ );
+
describe('isActionTool — cross-delimiter collision guard', () => {
it('should identify real action tools', () => {
expect(isActionTool(`get_weather${actionDelimiter}api_example_com`)).toBe(true);
@@ -2930,6 +2967,7 @@ describe('ToolService - Action Capability Gating', () => {
AgentCapabilities.stateful_code_sessions,
];
const req = createMockReq(capabilities);
+ req.config.endpoints[EModelEndpoint.agents].codeApiMaxRetryWaitMs = 0;
req.body = {
codeWorkspaces: [{ environmentId: 'personal-machine', workspaceId: 'project-a' }],
};
@@ -2970,6 +3008,7 @@ describe('ToolService - Action Capability Gating', () => {
gitIdentity: { name: 'LibreChat Agent', email: '[email protected]' },
maxTimeoutMs: 120000,
maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: 0,
});
expect(mockResolveCodeExecutionWorkspaceContext).toHaveBeenCalledWith(
expect.objectContaining({ requestedSelections: req.body.codeWorkspaces }),
diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts
index afb806d0f8..367fb4b7a6 100644
--- a/packages/api/src/agents/handlers.spec.ts
+++ b/packages/api/src/agents/handlers.spec.ts
@@ -5843,11 +5843,11 @@ describe('createToolExecuteHandler', () => {
it.each([
{ budget: 1200, elapsed: 0, remaining: 1200 },
{ budget: 1200, elapsed: 400, remaining: 800 },
- { budget: 1200, elapsed: 1200, remaining: null },
- { budget: 1200, elapsed: 1500, remaining: null },
+ { budget: 1200, elapsed: 1200, remaining: 0 },
+ { budget: 1200, elapsed: 1500, remaining: 0 },
{ budget: 0, elapsed: 400, remaining: 0 },
])(
- 'shares a protected edit retry horizon ($budget ms, preview $elapsed ms)',
+ 'shares a protected edit capacity retry horizon ($budget ms, preview $elapsed ms)',
async ({ budget, elapsed, remaining }) => {
const startedAt = Date.now();
let nowMs = startedAt;
@@ -5931,12 +5931,6 @@ describe('createToolExecuteHandler', () => {
deadlineAtMs: startedAt + 125_000,
}),
);
- if (remaining == null) {
- expect(result.status).toBe('error');
- expect(result.errorMessage).toContain('The file was not modified');
- expect(editWorkspaceFile).not.toHaveBeenCalled();
- return;
- }
expect(result.status).toBe('success');
expect(editWorkspaceFile).toHaveBeenCalledWith(
expect.objectContaining({
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index 7eb99e4e4d..a239557eb9 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -4111,17 +4111,10 @@ async function handleAttachedWorkspaceEditFileCall({
const filteredContent = filteredFileResult(tc, req, path.filePath, preview.content);
if (filteredContent != null) return filteredContent;
expectedBaseSha256 = preview.baseSha256;
- /** Zero disables retries, not the two operations required for a protected
- * edit. Positive horizons must not restart after a successful preview. */
+ /** A spent queue horizon disables capacity retries, not the required
+ * hash-guarded edit attempt or its independent rate-limit recovery. */
if (workspaceParams.maxQueueWaitMs > 0) {
- const remainingMs = queueDeadlineAt - Date.now();
- if (remainingMs <= 0) {
- return errorResult(
- tc,
- 'The workspace retry budget expired after preview. The file was not modified.',
- );
- }
- workspaceParams.maxQueueWaitMs = remainingMs;
+ workspaceParams.maxQueueWaitMs = Math.max(0, queueDeadlineAt - Date.now());
}
}
const result = await options.editWorkspaceFile({
diff --git a/packages/api/src/code/command.spec.ts b/packages/api/src/code/command.spec.ts
index 4ba032bbde..b4cc19e0c9 100644
--- a/packages/api/src/code/command.spec.ts
+++ b/packages/api/src/code/command.spec.ts
@@ -164,6 +164,24 @@ function commandResponse(overrides: Record<string, unknown> = {}): Response {
}
describe('createAttachedWorkspaceBashTool', () => {
+ test('honors a zero Code API rate-limit retry budget', async () => {
+ const fetchImpl: CodeBridgeFetch = jest.fn(
+ async () => new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }),
+ );
+ const bashTool = createAttachedWorkspaceBashTool({
+ baseUrl: 'https://code.example.com/v1',
+ authHeaders: () => ({}),
+ workspaceId: 'project-a',
+ codeApiMaxRetryWaitMs: 0,
+ fetchImpl,
+ });
+
+ await expect(bashTool.invoke({ command: 'pwd' })).rejects.toThrow(
+ 'The operation was not started',
+ );
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ });
+
test('dispatches commands to the resolved conversation workspace instance', async () => {
const fetchImpl: CodeBridgeFetch = jest.fn(async () => commandResponse());
const workspaceInstanceId = 'e'.repeat(64);
diff --git a/packages/api/src/code/command.ts b/packages/api/src/code/command.ts
index 2ec67f9885..da191a5ec9 100644
--- a/packages/api/src/code/command.ts
+++ b/packages/api/src/code/command.ts
@@ -302,6 +302,7 @@ export function createAttachedWorkspaceBashTool({
gitIdentity,
maxTimeoutMs = WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS,
maxQueueWaitMs,
+ codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
fetchImpl,
}: {
@@ -315,6 +316,7 @@ export function createAttachedWorkspaceBashTool({
maxTimeoutMs?: number;
/** Retry horizon across typed queue expirations, not an admission budget. */
maxQueueWaitMs?: number;
+ codeApiMaxRetryWaitMs?: number;
/** Verified total HTTP budget; omission keeps the legacy per-attempt timeout. */
maxRequestTimeoutMs?: number;
fetchImpl?: CodeBridgeFetch;
@@ -395,6 +397,7 @@ export function createAttachedWorkspaceBashTool({
signal,
fetchImpl,
...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }),
+ ...(codeApiMaxRetryWaitMs == null ? {} : { codeApiMaxRetryWaitMs }),
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
});
if (result.operation !== 'execute_command') {
diff --git a/packages/api/src/code/instructions.spec.ts b/packages/api/src/code/instructions.spec.ts
index f5b92a0e5e..e6ad0f7e5e 100644
--- a/packages/api/src/code/instructions.spec.ts
+++ b/packages/api/src/code/instructions.spec.ts
@@ -37,6 +37,72 @@ const response = () =>
);
describe('repository instruction loading', () => {
+ it('does not retry a typed 429 when the configured Code API wait budget is zero', async () => {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(response());
+
+ const result = await createRepositoryInstructionLoader()({
+ enabled: true,
+ context,
+ principalId: 'alice',
+ authHeaders: async () => ({}),
+ assertContent: jest.fn(),
+ codeApiMaxRetryWaitMs: 0,
+ fetchImpl,
+ });
+
+ expect(result).toBeUndefined();
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ });
+
+ it('refreshes instruction-read authorization for a typed 429 retry', async () => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(response());
+ let minted = 0;
+ const authHeaders = jest.fn(async () => ({ Authorization: `Bearer token-${++minted}` }));
+ const load = createRepositoryInstructionLoader();
+ const input = {
+ enabled: true,
+ context,
+ principalId: 'alice',
+ authHeaders,
+ fetchImpl,
+ codeApiMaxRetryWaitMs: 100,
+ timeoutMs: 500,
+ assertContent: jest.fn(),
+ };
+ const result = load(input);
+ await jest.advanceTimersByTimeAsync(100);
+ await expect(result).resolves.toContain('Use the project test command.');
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ expect(fetchImpl.mock.calls.map((call) => call[1]?.headers.Authorization)).toEqual([
+ 'Bearer token-1',
+ 'Bearer token-2',
+ ]);
+ await expect(load(input)).resolves.toContain('Use the project test command.');
+ expect(authHeaders).toHaveBeenCalledTimes(3);
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
it('bounds optional authorization waits and preserves explicit cancellation', async () => {
jest.useFakeTimers();
try {
diff --git a/packages/api/src/code/instructions.ts b/packages/api/src/code/instructions.ts
index 74db097616..ce100b2f2d 100644
--- a/packages/api/src/code/instructions.ts
+++ b/packages/api/src/code/instructions.ts
@@ -9,6 +9,7 @@ export interface RepositoryInstructionSource {
context: CodeExecutionContext;
principalId: string;
authHeaders: () => Promise<Record<string, string>>;
+ codeApiMaxRetryWaitMs?: number;
load: ReturnType<typeof createRepositoryInstructionLoader>;
}
@@ -54,6 +55,7 @@ export function createRepositoryInstructionLoader() {
signal,
fetchImpl,
assertContent,
+ codeApiMaxRetryWaitMs,
timeoutMs = 2000,
}: {
enabled: boolean;
@@ -64,6 +66,7 @@ export function createRepositoryInstructionLoader() {
signal?: AbortSignal;
fetchImpl?: CodeBridgeFetch;
assertContent: (content: string) => void;
+ codeApiMaxRetryWaitMs?: number;
timeoutMs?: number;
}): Promise<string | undefined> => {
const workspace = context.codeWorkspace;
@@ -93,13 +96,13 @@ export function createRepositoryInstructionLoader() {
const timer = setTimeout(() => deadline.abort(), budget);
const readSignal = signal ? AbortSignal.any([signal, deadline.signal]) : deadline.signal;
try {
- const headers = await abortable(authHeaders(), readSignal);
if (content === undefined) {
const result = await executeWorkspaceTool({
baseURL: context.baseUrl,
- authHeaders: headers,
+ authHeaders: () => abortable(authHeaders(), readSignal),
signal: readSignal,
fetchImpl,
+ codeApiMaxRetryWaitMs,
request: {
protocolVersion: 1,
operation: 'read_file',
@@ -121,6 +124,8 @@ export function createRepositoryInstructionLoader() {
content = result.content;
if (cache.size >= 64) cache.delete(cache.keys().next().value!);
cache.set(key, content);
+ } else {
+ await abortable(authHeaders(), readSignal);
}
} catch {
if (signal?.aborted) throw signal.reason;
diff --git a/packages/api/src/code/limits.ts b/packages/api/src/code/limits.ts
new file mode 100644
index 0000000000..4364674482
--- /dev/null
+++ b/packages/api/src/code/limits.ts
@@ -0,0 +1 @@
+export const CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS: number = 20_000;
diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts
index 149f5bd4da..fb48d98933 100644
--- a/packages/api/src/code/workspace.spec.ts
+++ b/packages/api/src/code/workspace.spec.ts
@@ -338,6 +338,514 @@ describe('workspace admission feedback', () => {
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
+ test('cancels a typed rate-limit wait without replaying the request', async () => {
+ jest.useFakeTimers();
+ try {
+ const controller = new AbortController();
+ const reason = new DOMException('Stopped', 'AbortError');
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '1' },
+ }),
+ )
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const request = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ signal: controller.signal,
+ fetchImpl,
+ maxQueueWaitMs: 0,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ });
+
+ await jest.advanceTimersByTimeAsync(1);
+ controller.abort(reason);
+ await expect(request).rejects.toBe(reason);
+ await jest.advanceTimersByTimeAsync(1_000);
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('retries a Code API rate limit, which rejects before the operation starts', async () => {
+ const rateLimited = () =>
+ new Response(
+ JSON.stringify({
+ error: 'rate_limited',
+ message: 'Too many CodeAPI execution requests. Please retry in 1 second.',
+ retry_after_seconds: 1,
+ }),
+ { status: 429, headers: { 'Retry-After': '0' } },
+ );
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(rateLimited())
+ .mockResolvedValueOnce(rateLimited())
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ protocolVersion: 1,
+ operation: 'edit_file',
+ workspaceId: 'primary',
+ path: 'src/app.ts',
+ replacements: 1,
+ bytesWritten: 24,
+ }),
+ { status: 200, headers: { 'Content-Type': 'application/json' } },
+ ),
+ );
+
+ await expect(
+ executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: {
+ protocolVersion: 1,
+ operation: 'edit_file',
+ workspaceId: 'primary',
+ path: 'src/app.ts',
+ edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
+ },
+ }),
+ ).resolves.toMatchObject({ operation: 'edit_file', replacements: 1 });
+
+ expect(fetchImpl).toHaveBeenCalledTimes(3);
+ const bodies = fetchImpl.mock.calls.map((call) => call[1]?.body);
+ expect(new Set(bodies).size).toBe(1);
+ });
+
+ test.each([
+ ['missing', undefined],
+ ['invalid', 'invalid'],
+ ['overflowing', '9'.repeat(400)],
+ ] as const)('uses the typed 429 body delay when Retry-After is %s', async (_case, retryAfter) => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 2 }), {
+ status: 429,
+ ...(retryAfter ? { headers: { 'Retry-After': retryAfter } } : {}),
+ }),
+ )
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ codeApiMaxRetryWaitMs: 3_000,
+ request: {
+ protocolVersion: 1,
+ operation: 'read_file',
+ workspaceId: 'primary',
+ path: 'x',
+ },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(1_999);
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(1);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test.each([-1, 300_001, 1.5, Number.NaN])(
+ 'rejects an invalid Code API retry limit of %p before transport',
+ async (codeApiMaxRetryWaitMs) => {
+ const fetchImpl = jest.fn();
+ await expect(
+ executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ codeApiMaxRetryWaitMs,
+ request: {
+ protocolVersion: 1,
+ operation: 'read_file',
+ workspaceId: 'primary',
+ path: 'x',
+ },
+ }),
+ ).rejects.toMatchObject({ reason: 'invalid' });
+ expect(fetchImpl).not.toHaveBeenCalled();
+ },
+ );
+
+ test.each([0, 150])('bounds typed 429 retries to the %i ms Code API budget', async (budget) => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest.fn().mockImplementation(
+ () =>
+ new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 0 }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ );
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ codeApiMaxRetryWaitMs: budget,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(budget);
+ await expect(result).resolves.toMatchObject({
+ upstreamStatus: 429,
+ message: expect.stringContaining('The operation was not started'),
+ });
+ expect(fetchImpl).toHaveBeenCalledTimes(budget === 0 ? 1 : 2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('does not wait when a 429 retry hint exceeds the Code API budget', async () => {
+ const fetchImpl = jest.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 30 }), {
+ status: 429,
+ }),
+ );
+ await expect(
+ executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }),
+ ).rejects.toMatchObject({ upstreamStatus: 429 });
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ });
+
+ test('keeps queue retries enabled when Code API rate-limit retries are disabled', async () => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
+ status: 503,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), { status: 429 }),
+ );
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ codeApiMaxRetryWaitMs: 0,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(100);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('does not charge queue waits to the Code API rate-limit budget', async () => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
+ status: 503,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ codeApiMaxRetryWaitMs: 100,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(200);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
+ expect(fetchImpl).toHaveBeenCalledTimes(3);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test.each([0, 50])(
+ 'retries typed 429s after a %i ms queue horizon without re-enabling capacity retries',
+ async (maxQueueWaitMs) => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ maxQueueWaitMs,
+ codeApiMaxRetryWaitMs: 100,
+ request: {
+ protocolVersion: 1,
+ operation: 'read_file',
+ workspaceId: 'primary',
+ path: 'x',
+ },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(100);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ },
+ );
+
+ test('continues 429 recovery after a queue retry but never extends its capacity horizon', async () => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
+ status: 503,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ maxQueueWaitMs: 150,
+ codeApiMaxRetryWaitMs: 100,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(200);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
+ expect(fetchImpl).toHaveBeenCalledTimes(3);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('never borrows the rate-limit horizon to retry a later capacity timeout', async () => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
+ status: 503,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ maxQueueWaitMs: 50,
+ codeApiMaxRetryWaitMs: 200,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(100);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 503 });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('retries after an authorized rate-limit wait despite timer jitter', async () => {
+ jest.useFakeTimers();
+ const startedAt = Date.now();
+ let currentTime = startedAt;
+ jest.spyOn(Date, 'now').mockImplementation(() => currentTime);
+ try {
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ )
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: 100,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(0);
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ currentTime += 101;
+ await jest.advanceTimersByTimeAsync(100);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('charges an overslept 429 wait before considering another retry', async () => {
+ jest.useFakeTimers();
+ const startedAt = Date.now();
+ let currentTime = startedAt;
+ jest.spyOn(Date, 'now').mockImplementation(() => currentTime);
+ try {
+ const rateLimited = () =>
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ });
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(rateLimited())
+ .mockResolvedValueOnce(rateLimited())
+ .mockResolvedValueOnce(new Response('{}', { status: 504 }));
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: 200,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(0);
+ currentTime += 150;
+ await jest.advanceTimersByTimeAsync(100);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('reserves the caller HTTP execution budget when capacity retries are disabled', async () => {
+ jest.useFakeTimers();
+ try {
+ const fetchImpl = jest.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: 'rate_limited' }), {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ );
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ maxQueueWaitMs: 0,
+ codeApiMaxRetryWaitMs: 1_000,
+ maxRequestTimeoutMs: 35_050,
+ request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'x' },
+ }).catch((error: WorkspaceToolHttpError) => error);
+
+ await jest.advanceTimersByTimeAsync(50);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 429 });
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('reports a rate limit as not started when its retry budget is disabled', async () => {
+ const fetchImpl = jest.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: 'rate_limited', retry_after_seconds: 1 }), {
+ status: 429,
+ headers: { 'Retry-After': '1' },
+ }),
+ );
+
+ const failure = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ codeApiMaxRetryWaitMs: 0,
+ request: {
+ protocolVersion: 1,
+ operation: 'execute_command',
+ workspaceId: 'primary',
+ command: 'echo test',
+ },
+ });
+
+ await expect(failure).rejects.toThrow('The operation was not started');
+ await expect(failure).rejects.toMatchObject({ upstreamStatus: 429 });
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ });
+
+ test('never retries a 429 that does not carry the typed rate-limit body', async () => {
+ const fetchImpl = jest.fn().mockResolvedValue(
+ new Response('<html>Too Many Requests</html>', {
+ status: 429,
+ headers: { 'Retry-After': '0' },
+ }),
+ );
+
+ await expect(
+ executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: {
+ protocolVersion: 1,
+ operation: 'edit_file',
+ workspaceId: 'primary',
+ path: 'src/app.ts',
+ edits: [{ oldText: 'const old = true;', newText: 'const ready = true;' }],
+ },
+ }),
+ ).rejects.toMatchObject({ upstreamStatus: 429 });
+
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ });
+
test('never retries an ambiguous capacity-looking failure', async () => {
const fetchImpl = jest.fn().mockResolvedValue(
new Response('<html>Gateway unavailable</html>', {
@@ -369,6 +877,10 @@ describe('workspace admission feedback', () => {
[503, '<html>Gateway unavailable</html>', false],
[503, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', true],
[503, 'null', false],
+ [503, '{"error":"rate_limited"}', false],
+ [429, '{"code":"WORKSPACE_QUEUE_TIMEOUT"}', false],
+ [429, '{"error":"rate_limited"}', true],
+ [429, '<html>Too Many Requests</html>', false],
] as const)(
'does not infer non-execution from an ambiguous response',
(status, body, truncated) => {
diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts
index c39d4002f9..7f43a2ea98 100644
--- a/packages/api/src/code/workspace.ts
+++ b/packages/api/src/code/workspace.ts
@@ -4,6 +4,7 @@ import {
CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS,
} from 'librechat-data-provider';
import type { CodeBridgeFetch } from './bridge';
+import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from './limits';
const WORKSPACE_TOOL_TIMEOUT_MS = 30_000;
const MAX_PATH_LENGTH = 4096;
@@ -274,13 +275,18 @@ export class WorkspaceToolHttpError extends Error {
public readonly upstreamBodyTruncated = false,
) {
let message = `Workspace tool request ${reason}`;
- if (
- reason === 'rejected' &&
- isWorkspaceAdmissionTimeout(upstreamStatus, upstreamBody, upstreamBodyTruncated)
- ) {
+ const admissionRejection =
+ reason === 'rejected'
+ ? getWorkspaceAdmissionRejection(upstreamStatus, upstreamBody, upstreamBodyTruncated)
+ : undefined;
+ if (admissionRejection === 'queue_timeout') {
message =
'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.';
}
+ if (admissionRejection === 'rate_limited') {
+ message =
+ 'The Code API request rate limit was reached. The operation was not started. Wait a few seconds before retrying, or make fewer concurrent workspace calls.';
+ }
if (reason === 'insufficient_time') {
message =
'Workspace execution cannot fit within the remaining HTTP budget. The operation was not started.';
@@ -295,15 +301,38 @@ export class WorkspaceToolHttpError extends Error {
}
}
-function isWorkspaceAdmissionTimeout(status?: number, body?: string, truncated = false): boolean {
- if (status !== 503 || !body || truncated || body.length > MAX_ERROR_BODY_BYTES) {
- return false;
+type WorkspaceAdmissionRejection = 'queue_timeout' | 'rate_limited';
+
+/**
+ * Rejections the Code API issues before an operation is assigned, so retrying cannot repeat a
+ * mutation: a queue deadline that expired before admission, or its rate limiter, which runs as
+ * middleware ahead of the workspace router. Anything else, including an unparsable body under
+ * either status, keeps an unknown outcome and is never retried.
+ */
+function getWorkspaceAdmissionRejection(
+ status?: number,
+ body?: string,
+ truncated = false,
+): WorkspaceAdmissionRejection | undefined {
+ if (
+ (status !== 503 && status !== 429) ||
+ !body ||
+ truncated ||
+ body.length > MAX_ERROR_BODY_BYTES
+ ) {
+ return undefined;
}
try {
- const parsed: { code?: string } | null = JSON.parse(body);
- return parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT';
+ const parsed: { code?: string; error?: string } | null = JSON.parse(body);
+ if (status === 503 && parsed?.code === 'WORKSPACE_QUEUE_TIMEOUT') {
+ return 'queue_timeout';
+ }
+ if (status === 429 && parsed?.error === 'rate_limited') {
+ return 'rate_limited';
+ }
+ return undefined;
} catch {
- return false;
+ return undefined;
}
}
@@ -326,9 +355,25 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi
});
}
-function workspaceAdmissionRetryDelay(value: string | null): number {
- if (value == null || !/^\d+$/.test(value)) return WORKSPACE_QUEUE_RETRY_DELAY_MS;
- return Math.max(100, Math.min(Number(value) * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
+function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number {
+ if (value != null && /^\d+$/.test(value)) {
+ const seconds = Number(value);
+ if (Number.isFinite(seconds)) {
+ return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
+ }
+ }
+ if (rateLimitBody) {
+ try {
+ const parsed: { retry_after_seconds?: number } | null = JSON.parse(rateLimitBody);
+ const seconds = parsed?.retry_after_seconds;
+ if (typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0) {
+ return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
+ }
+ } catch {
+ // Fall back when the delay hint is unusable.
+ }
+ }
+ return WORKSPACE_QUEUE_RETRY_DELAY_MS;
}
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
@@ -801,6 +846,7 @@ export async function executeWorkspaceTool({
signal,
fetchImpl = fetch,
maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS,
+ codeApiMaxRetryWaitMs = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS,
maxRequestTimeoutMs,
deadlineAtMs,
}: {
@@ -810,6 +856,8 @@ export async function executeWorkspaceTool({
signal?: AbortSignal;
fetchImpl?: CodeBridgeFetch;
maxQueueWaitMs?: number;
+ /** Maximum time waiting for Code API rate-limit admission, independent of queue retries. */
+ codeApiMaxRetryWaitMs?: number;
/** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */
maxRequestTimeoutMs?: number;
/** Optional earlier caller deadline; a signal alone has no remaining-time value. */
@@ -820,6 +868,9 @@ export async function executeWorkspaceTool({
!Number.isSafeInteger(maxQueueWaitMs) ||
maxQueueWaitMs < 0 ||
maxQueueWaitMs > WORKSPACE_QUEUE_MAX_WAIT_MS ||
+ !Number.isSafeInteger(codeApiMaxRetryWaitMs) ||
+ codeApiMaxRetryWaitMs < 0 ||
+ codeApiMaxRetryWaitMs > 300_000 ||
(maxRequestTimeoutMs !== undefined &&
(!Number.isSafeInteger(maxRequestTimeoutMs) ||
maxRequestTimeoutMs < 1 ||
@@ -836,9 +887,11 @@ export async function executeWorkspaceTool({
maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs,
);
const queueDeadlineAt = Date.now() + maxQueueWaitMs;
- const retryDeadlineAt = Math.min(queueDeadlineAt, callerDeadlineAt - completionReserveMs);
+ const callerRetryDeadlineAt = callerDeadlineAt - completionReserveMs;
const body = JSON.stringify(request);
let lastAdmissionRejection: WorkspaceToolHttpError | undefined;
+ let lastRetryDeadlineAt = Infinity;
+ let rateLimitWaitedMs = 0;
while (true) {
try {
signal?.throwIfAborted();
@@ -871,7 +924,7 @@ export async function executeWorkspaceTool({
if (timeoutSignal.aborted) {
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
}
- if (lastAdmissionRejection && Date.now() >= retryDeadlineAt) {
+ if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) {
throw lastAdmissionRejection;
}
const remainingMs = Math.min(
@@ -903,20 +956,35 @@ export async function executeWorkspaceTool({
const { body, truncated } = await readErrorBody(response, requestSignal);
signal?.throwIfAborted();
const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated);
+ const admission = getWorkspaceAdmissionRejection(response.status, body, truncated);
+ if (admission == null) throw rejection;
+ const retryDeadlineAt = Math.min(
+ admission === 'queue_timeout' ? queueDeadlineAt : Infinity,
+ callerRetryDeadlineAt,
+ );
+ if (Date.now() >= retryDeadlineAt) throw rejection;
+ const retryAfterMs = workspaceAdmissionRetryDelay(
+ response.headers.get('Retry-After'),
+ admission === 'rate_limited' ? body : undefined,
+ );
if (
- !isWorkspaceAdmissionTimeout(response.status, body, truncated) ||
- Date.now() >= queueDeadlineAt
+ admission === 'rate_limited' &&
+ retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs
) {
throw rejection;
}
- const delayMs = Math.min(
- workspaceAdmissionRetryDelay(response.headers.get('Retry-After')),
- retryDeadlineAt - Date.now(),
- );
+ const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now());
lastAdmissionRejection = rejection;
+ lastRetryDeadlineAt = retryDeadlineAt;
if (delayMs <= 0) throw rejection;
+ const waitStartedAt = Date.now();
await waitForWorkspaceAdmission(delayMs, signal);
- /** A clamped delay can land exactly on either deadline. Never open
+ if (admission === 'rate_limited') {
+ /** A timer may run late. Charge the actual wait, but honor the retry that
+ * was authorized before waiting; another 429 cannot exceed the balance. */
+ rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt);
+ }
+ /** A clamped delay can land exactly on this admission's retry deadline. Never open
* another admission window without the full execution reserve. */
if (Date.now() >= retryDeadlineAt) {
throw rejection;
diff --git a/packages/api/src/utils/code.ts b/packages/api/src/utils/code.ts
index 344c1d07ac..974fa4b2af 100644
--- a/packages/api/src/utils/code.ts
+++ b/packages/api/src/utils/code.ts
@@ -3,6 +3,7 @@ import https from 'https';
import { isAxiosError } from 'axios';
import { setTimeout as delay } from 'node:timers/promises';
import type { ServerRequest } from '~/types';
+import { CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS } from '~/code/limits';
/**
* Dedicated agents for code-server requests, preventing socket pool contamination.
@@ -44,7 +45,7 @@ export function getCodeApiRetryAfterMs(error: unknown): number | null {
/** Total time one operation may spend waiting out Code API rate limits.
* Keeps recovery bounded inside a live chat turn. */
-export const MAX_CODE_API_RATE_LIMIT_WAIT_MS = 20_000;
+export const MAX_CODE_API_RATE_LIMIT_WAIT_MS: number = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS;
export const CODE_API_UPLOAD_CONCURRENCY_DEFAULT = 3;
export interface CodeApiUploadRegistry {