Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🪀 fix: Separate Workspace Admission and Transport Budgets (#16728)

lia-by-librechat[bot]5674faaa01cb
diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js
index a50987755c..594e1aba38 100644
--- a/api/server/services/Files/Code/process.js
+++ b/api/server/services/Files/Code/process.js
@@ -1092,6 +1092,8 @@ async function readWorkspaceFile({
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
deadlineAtMs,
}) {
return executeWorkspaceTool({
@@ -1100,6 +1102,8 @@ async function readWorkspaceFile({
maxQueueWaitMs,
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
+ ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
+ ...(admission == null ? {} : { admission }),
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
@@ -1149,6 +1153,8 @@ async function searchWorkspace({
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
deadlineAtMs,
}) {
return executeWorkspaceTool({
@@ -1157,6 +1163,8 @@ async function searchWorkspace({
maxQueueWaitMs,
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
+ ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
+ ...(admission == null ? {} : { admission }),
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
@@ -1206,6 +1214,8 @@ async function listWorkspaceFiles({
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
deadlineAtMs,
}) {
return executeWorkspaceTool({
@@ -1214,6 +1224,8 @@ async function listWorkspaceFiles({
maxQueueWaitMs,
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
+ ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
+ ...(admission == null ? {} : { admission }),
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
@@ -1248,6 +1260,8 @@ async function writeWorkspaceFile({
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
deadlineAtMs,
}) {
return executeWorkspaceTool({
@@ -1256,6 +1270,8 @@ async function writeWorkspaceFile({
maxQueueWaitMs,
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
+ ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
+ ...(admission == null ? {} : { admission }),
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
@@ -1291,6 +1307,8 @@ async function editWorkspaceFile({
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
deadlineAtMs,
}) {
return executeWorkspaceTool({
@@ -1299,6 +1317,8 @@ async function editWorkspaceFile({
maxQueueWaitMs,
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
+ ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
+ ...(admission == null ? {} : { admission }),
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
@@ -1334,6 +1354,8 @@ async function previewWorkspaceEdit({
signal,
maxQueueWaitMs,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
deadlineAtMs,
}) {
return executeWorkspaceTool({
@@ -1342,6 +1364,8 @@ async function previewWorkspaceEdit({
maxQueueWaitMs,
codeApiMaxRetryWaitMs: req?.config?.endpoints?.agents?.codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
+ ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
+ ...(admission == null ? {} : { admission }),
deadlineAtMs,
/** Minted per admission attempt: a queued call outlives one token TTL. */
authHeaders: async () => ({
diff --git a/api/server/services/Files/Code/process.spec.js b/api/server/services/Files/Code/process.spec.js
index 4257741ae9..cce7a4ba29 100644
--- a/api/server/services/Files/Code/process.spec.js
+++ b/api/server/services/Files/Code/process.spec.js
@@ -2085,6 +2085,14 @@ describe('Code Process', () => {
signal: controller.signal,
maxQueueWaitMs: 0,
maxRequestTimeoutMs: 125_000,
+ maxRunTimeoutMs: 180_000,
+ admission: {
+ queueWaitMs: 60_000,
+ initialDelayMs: 1_000,
+ maxDelayMs: 30_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ },
deadlineAtMs: 160_000,
}),
).resolves.toBe(result);
@@ -2108,6 +2116,14 @@ describe('Code Process', () => {
maxQueueWaitMs: 0,
codeApiMaxRetryWaitMs: undefined,
maxRequestTimeoutMs: 125_000,
+ maxRunTimeoutMs: 180_000,
+ admission: {
+ queueWaitMs: 60_000,
+ initialDelayMs: 1_000,
+ maxDelayMs: 30_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ },
deadlineAtMs: 160_000,
request: {
protocolVersion: 1,
diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js
index 048cd0c4ea..bb7002a2c5 100644
--- a/api/server/services/ToolService.js
+++ b/api/server/services/ToolService.js
@@ -53,6 +53,7 @@ const {
createRepositoryInstructionLoader,
resolveAttachedWorkspaceCommandTimeoutMax,
resolveAttachedWorkspaceQueueWaitMs,
+ resolveAttachedWorkspaceAdmissionOptions,
resolveAttachedWorkspaceRequestTimeoutMs,
createContextProgrammaticBashTool,
resolveCodeExecutionContext,
@@ -2383,6 +2384,9 @@ async function loadToolsForExecution({
maxRequestTimeoutMs: resolveAttachedWorkspaceRequestTimeoutMs(
codeExecutionContext.codeEnvironmentConfigSchema,
),
+ ...resolveAttachedWorkspaceAdmissionOptions(
+ codeExecutionContext.codeEnvironmentConfigSchema,
+ ),
minCommandAdmissionMs:
codeExecutionContext.codeEnvironmentConfigSchema?.limits?.minCommandAdmissionMs,
})
diff --git a/api/server/services/__tests__/ToolService.spec.js b/api/server/services/__tests__/ToolService.spec.js
index 5516ce25f5..d851f4075a 100644
--- a/api/server/services/__tests__/ToolService.spec.js
+++ b/api/server/services/__tests__/ToolService.spec.js
@@ -3062,8 +3062,16 @@ describe('ToolService - Action Capability Gating', () => {
defaultCommandTimeoutMs: 60_000,
maxQueueWaitMs: 0,
maxRequestTimeoutMs: 90_000,
+ maxRunTimeoutMs: 180_000,
minCommandAdmissionMs: 15_000,
},
+ admission: {
+ queueWaitMs: 60_000,
+ initialDelayMs: 1_000,
+ maxDelayMs: 30_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ },
},
});
const toolRegistry = new Map([
@@ -3095,6 +3103,14 @@ describe('ToolService - Action Capability Gating', () => {
maxQueueWaitMs: 0,
codeApiMaxRetryWaitMs: 0,
maxRequestTimeoutMs: 90_000,
+ maxRunTimeoutMs: 180_000,
+ admission: {
+ queueWaitMs: 60_000,
+ initialDelayMs: 1_000,
+ maxDelayMs: 30_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ },
minCommandAdmissionMs: 15_000,
});
expect(mockResolveCodeExecutionWorkspaceContext).toHaveBeenCalledWith(
diff --git a/packages/api/src/agents/handlers.spec.ts b/packages/api/src/agents/handlers.spec.ts
index 0e4609741d..e6e0e83b25 100644
--- a/packages/api/src/agents/handlers.spec.ts
+++ b/packages/api/src/agents/handlers.spec.ts
@@ -7394,7 +7394,18 @@ describe('createToolExecuteHandler', () => {
statefulSessions: true,
environmentType: 'attached',
codeEnvironmentConfigSchema: {
- limits: { maxQueueWaitMs: budget, maxRequestTimeoutMs: 125_000 },
+ limits: {
+ maxQueueWaitMs: budget,
+ maxRequestTimeoutMs: 125_000,
+ maxRunTimeoutMs: 180_000,
+ },
+ admission: {
+ queueWaitMs: 60_000,
+ initialDelayMs: 1_000,
+ maxDelayMs: 30_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ },
},
bridgeWorkerId: 'user-worker',
},
@@ -7417,7 +7428,13 @@ describe('createToolExecuteHandler', () => {
expect.objectContaining({
maxQueueWaitMs: budget,
maxRequestTimeoutMs: 125_000,
- deadlineAtMs: startedAt + 125_000,
+ maxRunTimeoutMs: 180_000,
+ admission: expect.objectContaining({
+ queueWaitMs: 60_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ }),
+ deadlineAtMs: startedAt + 180_000,
}),
);
expect(result.status).toBe('success');
@@ -7426,7 +7443,13 @@ describe('createToolExecuteHandler', () => {
expected_base_sha256: 'b'.repeat(64),
maxQueueWaitMs: remaining,
maxRequestTimeoutMs: 125_000,
- deadlineAtMs: startedAt + 125_000,
+ maxRunTimeoutMs: 180_000,
+ admission: expect.objectContaining({
+ queueWaitMs: 60_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ }),
+ deadlineAtMs: startedAt + 180_000,
}),
);
},
@@ -9052,8 +9075,9 @@ describe('createToolExecuteHandler', () => {
async (name) => {
const message = await reject(name, legacyBody);
- expect(message).toContain('Invalid workspace path');
- expect(message.endsWith('"}"')).toBe(true);
+ expect(message).toContain('INVALID_PATH');
+ expect(message).not.toContain('Invalid workspace path');
+ expect(message).not.toContain('upstreamBody');
},
);
});
diff --git a/packages/api/src/agents/handlers.ts b/packages/api/src/agents/handlers.ts
index fb40c62e8e..e64f4d2b14 100644
--- a/packages/api/src/agents/handlers.ts
+++ b/packages/api/src/agents/handlers.ts
@@ -38,6 +38,7 @@ import type { CodeEnvRef, CodeWorkspaceOperation, PtcToolCallEvent } from 'libre
import type { StructuredToolInterface } from '@librechat/agents/langchain/tools';
import type { CodeEnvFile, CodeSessionContext } from '@librechat/agents';
import type {
+ WorkspaceAdmissionOptions,
WorkspaceEditResult,
WorkspaceTextEdit,
WorkspacePreviewEditResult,
@@ -106,6 +107,7 @@ import {
import {
resolveAttachedWorkspaceReadFileLines,
WorkspaceToolHttpError,
+ resolveAttachedWorkspaceAdmissionOptions,
WORKSPACE_EDIT_MAX_COUNT,
WORKSPACE_WRITE_MAX_BYTES,
} from '~/code/workspace';
@@ -605,6 +607,8 @@ export interface ToolExecuteOptions {
signal?: AbortSignal;
maxQueueWaitMs?: number;
maxRequestTimeoutMs?: number;
+ maxRunTimeoutMs?: number;
+ admission?: WorkspaceAdmissionOptions['admission'];
deadlineAtMs?: number;
}) => Promise<WorkspaceReadResult>;
/** Searches literal text within an attached worker's logical workspace. */
@@ -622,6 +626,8 @@ export interface ToolExecuteOptions {
signal?: AbortSignal;
maxQueueWaitMs?: number;
maxRequestTimeoutMs?: number;
+ maxRunTimeoutMs?: number;
+ admission?: WorkspaceAdmissionOptions['admission'];
deadlineAtMs?: number;
}) => Promise<WorkspaceSearchResult>;
/** Lists relative file paths within an attached worker's logical workspace. */
@@ -639,6 +645,8 @@ export interface ToolExecuteOptions {
signal?: AbortSignal;
maxQueueWaitMs?: number;
maxRequestTimeoutMs?: number;
+ maxRunTimeoutMs?: number;
+ admission?: WorkspaceAdmissionOptions['admission'];
deadlineAtMs?: number;
}) => Promise<WorkspaceListResult>;
/** Writes a UTF-8 file within an attached worker's logical workspace. */
@@ -656,6 +664,8 @@ export interface ToolExecuteOptions {
signal?: AbortSignal;
maxQueueWaitMs?: number;
maxRequestTimeoutMs?: number;
+ maxRunTimeoutMs?: number;
+ admission?: WorkspaceAdmissionOptions['admission'];
deadlineAtMs?: number;
}) => Promise<WorkspaceWriteResult>;
/** Previews exact replacements without mutating an attached worker workspace. */
@@ -674,6 +684,8 @@ export interface ToolExecuteOptions {
signal?: AbortSignal;
maxQueueWaitMs?: number;
maxRequestTimeoutMs?: number;
+ maxRunTimeoutMs?: number;
+ admission?: WorkspaceAdmissionOptions['admission'];
deadlineAtMs?: number;
}) => Promise<WorkspacePreviewEditResult>;
/** Applies exact replacements atomically within an attached worker workspace. */
@@ -693,6 +705,8 @@ export interface ToolExecuteOptions {
signal?: AbortSignal;
maxQueueWaitMs?: number;
maxRequestTimeoutMs?: number;
+ maxRunTimeoutMs?: number;
+ admission?: WorkspaceAdmissionOptions['admission'];
deadlineAtMs?: number;
}) => Promise<WorkspaceEditResult>;
/** Bounded reads return complete text; omitted maxBytes retains the legacy stdout path. */
@@ -3842,10 +3856,11 @@ function attachedWorkspaceAuthoringPath(
function attachedWorkspaceRequestLimits(codeExecutionContext: CodeExecutionContext): {
maxQueueWaitMs: number;
maxRequestTimeoutMs?: number;
-} {
+} & WorkspaceAdmissionOptions {
const config = codeExecutionContext.codeEnvironmentConfigSchema;
const maxRequestTimeoutMs = resolveAttachedWorkspaceRequestTimeoutMs(config);
return {
+ ...resolveAttachedWorkspaceAdmissionOptions(config),
maxQueueWaitMs: resolveAttachedWorkspaceQueueWaitMs(config),
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
};
@@ -3868,8 +3883,9 @@ function attachedWorkspaceMutationParams(
maxQueueWaitMs: number;
maxRequestTimeoutMs?: number;
deadlineAtMs?: number;
-} {
+} & WorkspaceAdmissionOptions {
const limits = attachedWorkspaceRequestLimits(codeExecutionContext);
+ const runTimeoutMs = limits.maxRunTimeoutMs ?? limits.maxRequestTimeoutMs;
return {
workspace_id: workspaceId,
...(codeExecutionContext.codeWorkspace?.workspaceInstanceId
@@ -3878,9 +3894,7 @@ function attachedWorkspaceMutationParams(
...(codeExecutionContext.codeWorkspace?.linkedWorktrees ? { linked_worktrees: true } : {}),
codeApiBaseUrl: codeExecutionContext.baseUrl,
...limits,
- ...(limits.maxRequestTimeoutMs == null
- ? {}
- : { deadlineAtMs: Date.now() + limits.maxRequestTimeoutMs }),
+ ...(runTimeoutMs == null ? {} : { deadlineAtMs: Date.now() + runTimeoutMs }),
executionProfile: codeExecutionContext.executionProfile,
...(codeExecutionContext.bridgeWorkerId
? { bridgeWorkerId: codeExecutionContext.bridgeWorkerId }
diff --git a/packages/api/src/code/command.spec.ts b/packages/api/src/code/command.spec.ts
index 39c1caacb3..5b55ff0a0d 100644
--- a/packages/api/src/code/command.spec.ts
+++ b/packages/api/src/code/command.spec.ts
@@ -1308,3 +1308,55 @@ describe('createAttachedWorkspaceBashTool', () => {
]);
});
});
+
+describe('attached command admission configuration', () => {
+ test('bounds the advertised command ceiling by the shorter overall run budget', () => {
+ expect(
+ resolveAttachedWorkspaceCommandTimeoutMax({
+ limits: {
+ maxCommandTimeoutMs: 100_000,
+ maxRequestTimeoutMs: 100_000,
+ maxRunTimeoutMs: 65_000,
+ },
+ }),
+ ).toBe(45_000);
+ });
+
+ test('passes separate run and retry policies through the actual tool invocation', async () => {
+ const admission = {
+ queueWaitMs: 60_000,
+ initialDelayMs: 1_000,
+ maxDelayMs: 30_000,
+ multiplier: 2,
+ jitterRatio: 0.2,
+ };
+ const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(
+ async () =>
+ new Response(
+ JSON.stringify({
+ protocolVersion: 1,
+ operation: 'execute_command',
+ workspaceId: 'primary',
+ exitCode: 0,
+ stdout: 'ready',
+ stderr: '',
+ timedOut: false,
+ truncated: false,
+ }),
+ ),
+ );
+ const bash = createAttachedWorkspaceBashTool({
+ baseUrl: 'https://code.example/v1',
+ authHeaders: () => ({}),
+ workspaceId: 'primary',
+ fetchImpl,
+ maxRequestTimeoutMs: 100_000,
+ maxRunTimeoutMs: 180_000,
+ admission,
+ });
+ await bash.invoke({ command: 'echo ready' });
+ expect(
+ new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'),
+ ).toBe('60000');
+ });
+});
diff --git a/packages/api/src/code/command.ts b/packages/api/src/code/command.ts
index cdf53ee132..72e6e8a405 100644
--- a/packages/api/src/code/command.ts
+++ b/packages/api/src/code/command.ts
@@ -14,7 +14,7 @@ import type {
import type { createBashProgrammaticToolCallingSchema } from '@librechat/agents';
import type { DynamicStructuredTool } from '@librechat/agents/langchain/tools';
import type { LCTool } from '@librechat/agents';
-import type { WorkspaceExecuteCommandResult } from './workspace';
+import type { WorkspaceAdmissionOptions, WorkspaceExecuteCommandResult } from './workspace';
import type { CodeExecutionContext } from '~/agents/execution';
import type { CodeBridgeFetch } from './bridge';
import {
@@ -173,7 +173,10 @@ export function resolveAttachedWorkspaceCommandTimeoutMax(
}
return fitCommandTimeoutMaxToBudget(
Math.min(requested, upstream),
- resolveAttachedWorkspaceRequestTimeoutMs(configSchema),
+ Math.min(
+ resolveAttachedWorkspaceRequestTimeoutMs(configSchema) ?? Infinity,
+ configSchema?.limits?.maxRunTimeoutMs ?? Infinity,
+ ),
configSchema?.limits?.minCommandAdmissionMs,
);
}
@@ -183,7 +186,7 @@ function fitCommandTimeoutMaxToBudget(
maxRequestTimeoutMs?: number,
minCommandAdmissionMs?: number,
): number {
- if (maxRequestTimeoutMs == null) return maxTimeoutMs;
+ if (maxRequestTimeoutMs == null || !Number.isFinite(maxRequestTimeoutMs)) return maxTimeoutMs;
return Math.min(
maxTimeoutMs,
fitWorkspaceCommandTimeoutToBudget(maxRequestTimeoutMs, minCommandAdmissionMs),
@@ -450,6 +453,8 @@ export function createAttachedWorkspaceBashTool({
maxQueueWaitMs,
codeApiMaxRetryWaitMs,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
minCommandAdmissionMs,
linkedWorktrees = false,
nativeSandbox = false,
@@ -477,10 +482,10 @@ export function createAttachedWorkspaceBashTool({
/** Minimum time for command admission inside an opted-in HTTP budget. */
minCommandAdmissionMs?: number;
fetchImpl?: CodeBridgeFetch;
-}): DynamicStructuredTool {
+} & WorkspaceAdmissionOptions): DynamicStructuredTool {
const effectiveMaxTimeoutMs = fitCommandTimeoutMaxToBudget(
normalizeAttachedWorkspaceCommandTimeoutMax(maxTimeoutMs),
- maxRequestTimeoutMs,
+ Math.min(maxRequestTimeoutMs ?? Infinity, maxRunTimeoutMs ?? Infinity),
minCommandAdmissionMs,
);
const effectiveDefaultTimeoutMs = resolveAttachedWorkspaceCommandTimeoutDefault(
@@ -570,6 +575,8 @@ export function createAttachedWorkspaceBashTool({
...(maxQueueWaitMs == null ? {} : { maxQueueWaitMs }),
...(codeApiMaxRetryWaitMs == null ? {} : { codeApiMaxRetryWaitMs }),
...(maxRequestTimeoutMs == null ? {} : { maxRequestTimeoutMs }),
+ ...(maxRunTimeoutMs == null ? {} : { maxRunTimeoutMs }),
+ ...(admission == null ? {} : { admission }),
});
if (result.operation !== 'execute_command') {
throw new Error('Attached workspace returned an unexpected command result.');
diff --git a/packages/api/src/code/workspace.spec.ts b/packages/api/src/code/workspace.spec.ts
index 11ed3cce2d..2abcf3e18a 100644
--- a/packages/api/src/code/workspace.spec.ts
+++ b/packages/api/src/code/workspace.spec.ts
@@ -1,3 +1,5 @@
+import { logger } from '@librechat/data-schemas';
+import { codeEnvironmentAdmissionSchema } from 'librechat-data-provider';
import type { WorkspaceToolRequest } from './workspace';
import type { CodeBridgeFetch } from './bridge';
import {
@@ -1137,7 +1139,9 @@ describe('executeWorkspaceTool', () => {
}),
).rejects.toMatchObject({ upstreamStatus: 504 });
expect(timeout).toHaveBeenCalledWith(60_000);
- expect(fetchImpl.mock.calls[0][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe('25000');
+ expect(
+ new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'),
+ ).toBe('25000');
});
test('subtracts credential acquisition from the allowance before dispatching', async () => {
@@ -1230,7 +1234,7 @@ describe('executeWorkspaceTool', () => {
expect(fetchImpl).toHaveBeenCalledTimes(1);
});
- test('keeps Retry-After capped at 30 seconds even when one admission may wait longer', async () => {
+ test('does not shorten a Retry-After hint to fit a shrinking total budget', async () => {
jest.useFakeTimers();
try {
const fetchImpl = jest
@@ -1254,13 +1258,8 @@ describe('executeWorkspaceTool', () => {
maxRequestTimeoutMs: 125_000,
fetchImpl,
}).catch((error: WorkspaceToolHttpError) => error);
- await jest.advanceTimersByTimeAsync(29_999);
+ await expect(result).resolves.toMatchObject({ upstreamStatus: 503 });
expect(fetchImpl).toHaveBeenCalledTimes(1);
- await jest.advanceTimersByTimeAsync(1);
- await expect(result).resolves.toMatchObject({ upstreamStatus: 504 });
- expect(fetchImpl.mock.calls[1][1].headers['X-LibreChat-Workspace-Queue-Wait-Ms']).toBe(
- '60000',
- );
} finally {
jest.useRealTimers();
}
@@ -2425,3 +2424,221 @@ describe('linked worktree lanes', () => {
).rejects.toMatchObject({ reason: 'invalid' });
});
});
+
+describe('workspace admission policy and independent budgets', () => {
+ const command: WorkspaceToolRequest = {
+ protocolVersion: 1,
+ operation: 'execute_command',
+ workspaceId: 'primary',
+ command: 'echo ready',
+ timeoutMs: 30_000,
+ };
+ const success = () =>
+ new Response(
+ JSON.stringify({
+ protocolVersion: 1,
+ operation: 'execute_command',
+ workspaceId: 'primary',
+ exitCode: 0,
+ stdout: 'ready',
+ stderr: '',
+ timedOut: false,
+ truncated: false,
+ }),
+ );
+ const queueTimeout = () =>
+ new Response(JSON.stringify({ code: 'WORKSPACE_QUEUE_TIMEOUT' }), {
+ status: 503,
+ headers: { 'Retry-After': '0' },
+ });
+
+ beforeEach(() => {
+ jest.useFakeTimers();
+ jest.setSystemTime(new Date('2026-10-04T00:00:00.000Z'));
+ });
+ afterEach(() => jest.useRealTimers());
+
+ test('keeps the full 30-second execution budget after 60 seconds waiting', async () => {
+ let attempts = 0;
+ const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(
+ async (_url, init) => {
+ if (init == null) throw new Error('Missing request init');
+ attempts++;
+ if (attempts === 1) {
+ await new Promise((resolve) => setTimeout(resolve, 30_000));
+ return queueTimeout();
+ }
+ expect(Number(new Headers(init.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'))).toBe(
+ 60_000,
+ );
+ await new Promise((resolve) => setTimeout(resolve, 59_900));
+ expect(init.signal?.aborted).toBe(false);
+ return success();
+ },
+ );
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: command,
+ maxRequestTimeoutMs: 100_000,
+ maxRunTimeoutMs: 180_000,
+ admission: codeEnvironmentAdmissionSchema.parse({ queueWaitMs: 60_000 }),
+ });
+ await jest.advanceTimersByTimeAsync(90_000);
+ await expect(result).resolves.toMatchObject({ stdout: 'ready' });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ expect(fetchImpl.mock.calls[1][1]?.body).toBe(fetchImpl.mock.calls[0][1]?.body);
+ });
+
+ test('bounds queue allowance by the transport reserve, not the larger run deadline', async () => {
+ const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(async () =>
+ success(),
+ );
+ await executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: command,
+ maxRequestTimeoutMs: 65_000,
+ maxRunTimeoutMs: 180_000,
+ admission: codeEnvironmentAdmissionSchema.parse({ queueWaitMs: 180_000 }),
+ });
+ expect(
+ new Headers(fetchImpl.mock.calls[0][1]?.headers).get('X-LibreChat-Workspace-Queue-Wait-Ms'),
+ ).toBe('25000');
+ });
+
+ test('honors an earlier caller deadline even when overall and transport budgets are larger', async () => {
+ const fetchImpl = jest.fn<ReturnType<CodeBridgeFetch>, Parameters<CodeBridgeFetch>>(async () =>
+ success(),
+ );
+ await expect(
+ executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: command,
+ maxRequestTimeoutMs: 100_000,
+ maxRunTimeoutMs: 180_000,
+ deadlineAtMs: Date.now() + 40_000,
+ }),
+ ).rejects.toMatchObject({ reason: 'insufficient_time' });
+ expect(fetchImpl).not.toHaveBeenCalled();
+ });
+
+ test('adds exponential jitter only to whitelisted rejections and caps local backoff', async () => {
+ jest.spyOn(Math, 'random').mockReturnValue(1 - Number.EPSILON);
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 }))
+ .mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 }))
+ .mockResolvedValueOnce(new Response('{"code":"WORKSPACE_QUEUE_TIMEOUT"}', { status: 503 }))
+ .mockResolvedValueOnce(success());
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: command,
+ admission: codeEnvironmentAdmissionSchema.parse({
+ initialDelayMs: 1_000,
+ maxDelayMs: 3_000,
+ multiplier: 2,
+ jitterRatio: 0.5,
+ }),
+ });
+ await jest.advanceTimersByTimeAsync(1_499);
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(1);
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ await jest.advanceTimersByTimeAsync(3_000);
+ expect(fetchImpl).toHaveBeenCalledTimes(3);
+ await jest.advanceTimersByTimeAsync(3_000);
+ await expect(result).resolves.toMatchObject({ stdout: 'ready' });
+ expect(fetchImpl).toHaveBeenCalledTimes(4);
+ });
+
+ test.each(['60', 'date', 'body'])(
+ 'does not retry before a 60-second %s hint with jitter',
+ async (hint) => {
+ jest.spyOn(Math, 'random').mockReturnValue(1 - Number.EPSILON);
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ error: 'rate_limited',
+ ...(hint === 'body' ? { retry_after_seconds: 60 } : {}),
+ }),
+ {
+ status: 429,
+ headers:
+ hint === 'body'
+ ? {}
+ : {
+ 'Retry-After':
+ hint === 'date' ? new Date(Date.now() + 60_000).toUTCString() : hint,
+ },
+ },
+ ),
+ )
+ .mockResolvedValueOnce(success());
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: command,
+ maxRequestTimeoutMs: 100_000,
+ maxRunTimeoutMs: 180_000,
+ codeApiMaxRetryWaitMs: 180_000,
+ admission: codeEnvironmentAdmissionSchema.parse({ jitterRatio: 0.5 }),
+ });
+ await jest.advanceTimersByTimeAsync(89_999);
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(1);
+ await expect(result).resolves.toMatchObject({ stdout: 'ready' });
+ expect(fetchImpl).toHaveBeenCalledTimes(2);
+ },
+ );
+
+ test('keeps upstream bodies diagnostic-only for every rejection kind', () => {
+ for (const status of [429, 503, 502, 409]) {
+ const body = '{"code":"WORKSPACE_QUEUE_TIMEOUT","error":"private upstream diagnostic"}';
+ const error = new WorkspaceToolHttpError('rejected', status, body);
+ expect(error.message).not.toContain('private upstream diagnostic');
+ expect(error.upstreamBody).toBe(body);
+ }
+ });
+
+ test('emits one aggregate outcome without calling HTTP latency queue wait', async () => {
+ const log = jest.spyOn(logger, 'debug');
+ const fetchImpl = jest
+ .fn()
+ .mockResolvedValueOnce(queueTimeout())
+ .mockResolvedValueOnce(success());
+ const result = executeWorkspaceTool({
+ baseURL: 'https://code.example/v1',
+ authHeaders: {},
+ fetchImpl,
+ request: command,
+ });
+ await jest.advanceTimersByTimeAsync(100);
+ await result;
+ expect(log).toHaveBeenCalledTimes(1);
+ expect(log).toHaveBeenCalledWith(
+ '[WorkspaceAdmission] outcome',
+ expect.objectContaining({
+ attempts: 2,
+ queueRejections: 1,
+ retryWaitMs: 100,
+ outcome: 'completed',
+ }),
+ );
+ expect(log).toHaveBeenCalledWith(
+ '[WorkspaceAdmission] outcome',
+ expect.not.objectContaining({
+ queueWaitMs: expect.any(Number),
+ }),
+ );
+ });
+});
diff --git a/packages/api/src/code/workspace.ts b/packages/api/src/code/workspace.ts
index 471ba4ed3b..135f683822 100644
--- a/packages/api/src/code/workspace.ts
+++ b/packages/api/src/code/workspace.ts
@@ -1,4 +1,7 @@
+import { trace } from '@opentelemetry/api';
+import { logger } from '@librechat/data-schemas';
import {
+ codeEnvironmentAdmissionSchema,
CODE_ENVIRONMENT_ADMISSION_MAX_MS,
CODE_ENVIRONMENT_COMMAND_TIMEOUT_DEFAULT_MS,
CODE_ENVIRONMENT_READ_FILE_DEFAULT_LINES,
@@ -32,12 +35,11 @@ const DEFAULT_COMMAND_OUTPUT_BYTES = 256 * 1024;
const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024;
const MAX_COMMAND_SIGNAL_LENGTH = 32;
const WORKSPACE_COMMAND_TRANSPORT_GRACE_MS = 5_000;
-/** Legacy admission window and Retry-After cap, independent of the retry horizon. */
+/** Legacy per-request admission window, independent of the retry horizon. */
const WORKSPACE_QUEUE_TIMEOUT_MS = 30_000;
const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms';
/** Compatibility export; the deployment schema owns the default and hard cap. */
export const WORKSPACE_QUEUE_MAX_WAIT_MS: number = CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS;
-const WORKSPACE_QUEUE_RETRY_DELAY_MS = 1_000;
const WORKSPACE_COMMAND_SETTLEMENT_GRACE_MS = 5_000;
/**
@@ -319,6 +321,7 @@ export class WorkspaceToolHttpError extends Error {
public readonly upstreamBody?: string,
public readonly upstreamBodyTruncated = false,
) {
+ const parsed = upstreamBodyTruncated ? undefined : parseErrorBody(upstreamBody);
let message = `Workspace tool request ${reason}`;
const admissionRejection =
reason === 'rejected'
@@ -339,11 +342,9 @@ export class WorkspaceToolHttpError extends Error {
super(
message +
(upstreamStatus == null ? '' : ` (upstreamStatus: ${upstreamStatus})`) +
- (upstreamBody ? `; upstreamBody: ${JSON.stringify(upstreamBody)}` : '') +
- (upstreamBodyTruncated ? ' [body truncated or incomplete]' : ''),
+ (parsed?.code == null ? '' : ` (upstreamCode: ${parsed.code})`),
);
this.name = 'WorkspaceToolHttpError';
- const parsed = upstreamBodyTruncated ? undefined : parseErrorBody(upstreamBody);
this.upstreamCode = parsed?.code;
this.editConflict =
reason === 'rejected' && upstreamStatus === 409 && parsed?.code === 'EDIT_CONFLICT'
@@ -425,25 +426,56 @@ function waitForWorkspaceAdmission(delayMs: number, signal?: AbortSignal): Promi
});
}
-function workspaceAdmissionRetryDelay(value: string | null, rateLimitBody?: string): number {
+const DEFAULT_ADMISSION_POLICY = codeEnvironmentAdmissionSchema.parse({});
+
+export interface WorkspaceAdmissionOptions {
+ maxRunTimeoutMs?: number;
+ admission?: CodeEnvironmentUserConfigSchema['admission'];
+}
+
+/** Keep omitted policy absent so existing adapters and total-budget semantics remain unchanged. */
+export function resolveAttachedWorkspaceAdmissionOptions(
+ config?: CodeEnvironmentUserConfigSchema,
+): WorkspaceAdmissionOptions {
+ return {
+ ...(config?.limits?.maxRunTimeoutMs == null
+ ? {}
+ : { maxRunTimeoutMs: config.limits.maxRunTimeoutMs }),
+ ...(config?.admission == null ? {} : { admission: config.admission }),
+ };
+}
+
+function workspaceAdmissionRetryDelay(
+ value: string | null,
+ body: string | undefined,
+ attempt: number,
+ policy: NonNullable<WorkspaceAdmissionOptions['admission']>,
+): number {
+ let hintMs: number | undefined;
if (value != null && /^\d+$/.test(value)) {
- const seconds = Number(value);
- if (Number.isFinite(seconds)) {
- return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
- }
+ hintMs = Number(value) * 1_000;
+ } else if (value != null) {
+ const date = Date.parse(value);
+ if (Number.isFinite(date)) hintMs = Math.max(0, date - Date.now());
}
- if (rateLimitBody) {
+ if (hintMs != null && !Number.isFinite(hintMs)) hintMs = undefined;
+ if (hintMs == null && body) {
try {
- const parsed: { retry_after_seconds?: number } | null = JSON.parse(rateLimitBody);
+ const parsed: { retry_after_seconds?: number } | null = JSON.parse(body);
const seconds = parsed?.retry_after_seconds;
if (typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0) {
- return Math.max(100, Math.min(seconds * 1_000, WORKSPACE_QUEUE_TIMEOUT_MS));
+ hintMs = seconds * 1_000;
}
} catch {
- // Fall back when the delay hint is unusable.
+ // The rejection whitelist already requires complete JSON.
}
}
- return WORKSPACE_QUEUE_RETRY_DELAY_MS;
+ const delayMs =
+ hintMs == null
+ ? Math.min(policy.maxDelayMs, policy.initialDelayMs * policy.multiplier ** (attempt - 1))
+ : Math.max(100, hintMs);
+ const jitteredMs = Math.ceil(delayMs + delayMs * policy.jitterRatio * Math.random());
+ return hintMs == null ? Math.min(policy.maxDelayMs, jitteredMs) : jitteredMs;
}
/** Keep a received HTTP status even if reading its diagnostic body fails or stalls. */
@@ -1109,6 +1141,8 @@ export async function executeWorkspaceTool({
maxQueueWaitMs = WORKSPACE_QUEUE_MAX_WAIT_MS,
codeApiMaxRetryWaitMs = CODE_API_RATE_LIMIT_WAIT_DEFAULT_MS,
maxRequestTimeoutMs,
+ maxRunTimeoutMs,
+ admission,
deadlineAtMs,
linkedWorktrees = false,
}: {
@@ -1120,14 +1154,18 @@ export async function executeWorkspaceTool({
maxQueueWaitMs?: number;
/** Maximum time waiting for Code API rate-limit admission, independent of queue retries. */
codeApiMaxRetryWaitMs?: number;
- /** End-to-end HTTP budget for this call. Omission keeps legacy per-attempt timeouts. */
+ /** Per-attempt transport ceiling; also the total budget unless maxRunTimeoutMs is set. */
maxRequestTimeoutMs?: number;
/** Optional earlier caller deadline; a signal alone has no remaining-time value. */
deadlineAtMs?: number;
/** The worker runs each `.worktrees/<name>` in its own lane; route matching requests there. */
linkedWorktrees?: boolean;
-}): Promise<WorkspaceToolResult> {
+} & WorkspaceAdmissionOptions): Promise<WorkspaceToolResult> {
+ const parsedPolicy = codeEnvironmentAdmissionSchema.safeParse(
+ admission ?? DEFAULT_ADMISSION_POLICY,
+ );
if (
+ !parsedPolicy.success ||
!isValidRequest(request) ||
!Number.isSafeInteger(maxQueueWaitMs) ||
maxQueueWaitMs < 0 ||
@@ -1139,141 +1177,210 @@ export async function executeWorkspaceTool({
(!Number.isSafeInteger(maxRequestTimeoutMs) ||
maxRequestTimeoutMs < 1 ||
maxRequestTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) ||
+ (maxRunTimeoutMs !== undefined &&
+ (!Number.isSafeInteger(maxRunTimeoutMs) ||
+ maxRunTimeoutMs < 1 ||
+ maxRunTimeoutMs > CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)) ||
(deadlineAtMs !== undefined && (!Number.isSafeInteger(deadlineAtMs) || deadlineAtMs < 1))
) {
throw new WorkspaceToolHttpError('invalid');
}
+ const policy = parsedPolicy.data;
const lane = linkedWorktrees === true ? toLinkedWorktreeRequest(request) : undefined;
const wireRequest: WorkspaceToolRequest = lane?.request ?? request;
const executionBudgetMs = getWorkspaceExecutionBudgetMs(wireRequest);
const completionReserveMs = executionBudgetMs + WORKSPACE_COMMAND_TRANSPORT_GRACE_MS;
const perAttemptTimeoutMs = maxRequestTimeoutMs ?? getWorkspaceToolTimeoutMs(wireRequest);
+ const startedAt = Date.now();
+ const runTimeoutMs = maxRunTimeoutMs ?? maxRequestTimeoutMs;
const callerDeadlineAt = Math.min(
deadlineAtMs ?? Infinity,
- maxRequestTimeoutMs == null ? Infinity : Date.now() + maxRequestTimeoutMs,
+ runTimeoutMs == null ? Infinity : startedAt + runTimeoutMs,
);
- const queueDeadlineAt = Date.now() + maxQueueWaitMs;
+ const queueDeadlineAt = startedAt + maxQueueWaitMs;
const callerRetryDeadlineAt = callerDeadlineAt - completionReserveMs;
const body = JSON.stringify(wireRequest);
let lastAdmissionRejection: WorkspaceToolHttpError | undefined;
let lastRetryDeadlineAt = Infinity;
let rateLimitWaitedMs = 0;
- while (true) {
- try {
- signal?.throwIfAborted();
- /** Admission and retries cannot consume the reserved execution or delivery time. */
- const attemptTimeoutMs = Math.floor(
- Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()),
- );
- if (attemptTimeoutMs <= completionReserveMs) {
- throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
- }
- const attemptStartedAt = Date.now();
- const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs);
- const requestSignal =
- signal != null && typeof AbortSignal.any === 'function'
- ? AbortSignal.any([signal, timeoutSignal])
- : timeoutSignal;
- let attemptHeaders: Record<string, string>;
+ let retryWaitMs = 0;
+ let attempts = 0;
+ let queueRejections = 0;
+ let rateLimitRejections = 0;
+ let outcome = 'failed';
+ const span = trace.getTracer('librechat.workspace').startSpan('workspace.tool', {
+ attributes: {
+ 'workspace.operation': wireRequest.operation,
+ 'workspace.id': wireRequest.workspaceId,
+ ...(lane == null ? {} : { 'workspace.lane': lane.worktree }),
+ 'workspace.execution_budget_ms': executionBudgetMs,
+ 'workspace.transport_budget_ms': perAttemptTimeoutMs,
+ },
+ });
+ try {
+ while (true) {
try {
- attemptHeaders =
- typeof authHeaders === 'function'
- ? await getWorkspaceAuthHeaders(authHeaders, requestSignal)
- : authHeaders;
- } catch (error) {
- if (timeoutSignal.aborted && signal?.aborted !== true) {
+ signal?.throwIfAborted();
+ /** Admission and retries cannot consume the reserved execution or delivery time. */
+ const attemptTimeoutMs = Math.floor(
+ Math.min(perAttemptTimeoutMs, callerDeadlineAt - Date.now()),
+ );
+ if (attemptTimeoutMs <= completionReserveMs) {
throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
}
- throw error;
- }
- signal?.throwIfAborted();
- if (timeoutSignal.aborted) {
- throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
- }
- if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) {
- throw lastAdmissionRejection;
- }
- const remainingMs = Math.min(
- attemptTimeoutMs - (Date.now() - attemptStartedAt),
- callerDeadlineAt - Date.now(),
- );
- const queueAllowanceMs = Math.min(
- CODE_ENVIRONMENT_ADMISSION_MAX_MS,
- Math.floor(remainingMs - completionReserveMs),
- );
- if (queueAllowanceMs < 1) {
- throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
- }
- const response = await fetchImpl(
- `${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`,
- {
- method: 'POST',
- headers: {
- ...attemptHeaders,
- 'Content-Type': 'application/json',
- [WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs),
- },
- body,
- redirect: 'error',
- signal: requestSignal,
- },
- );
- if (!response.ok) {
- const { body, truncated } = await readErrorBody(response, requestSignal);
+ const attemptStartedAt = Date.now();
+ const timeoutSignal = AbortSignal.timeout(attemptTimeoutMs);
+ const requestSignal =
+ signal != null && typeof AbortSignal.any === 'function'
+ ? AbortSignal.any([signal, timeoutSignal])
+ : timeoutSignal;
+ let attemptHeaders: Record<string, string>;
+ try {
+ attemptHeaders =
+ typeof authHeaders === 'function'
+ ? await getWorkspaceAuthHeaders(authHeaders, requestSignal)
+ : authHeaders;
+ } catch (error) {
+ if (timeoutSignal.aborted && signal?.aborted !== true) {
+ throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
+ }
+ throw error;
+ }
signal?.throwIfAborted();
- const rejection = new WorkspaceToolHttpError('rejected', response.status, body, truncated);
- const admission = getWorkspaceAdmissionRejection(response.status, body, truncated);
- if (admission == null) throw rejection;
- const retryDeadlineAt = Math.min(
- admission === 'queue_timeout' ? queueDeadlineAt : Infinity,
- callerRetryDeadlineAt,
+ if (timeoutSignal.aborted) {
+ throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
+ }
+ if (lastAdmissionRejection && Date.now() >= lastRetryDeadlineAt) {
+ throw lastAdmissionRejection;
+ }
+ const remainingMs = Math.min(
+ attemptTimeoutMs - (Date.now() - attemptStartedAt),
+ callerDeadlineAt - Date.now(),
+ );
+ const queueAllowanceMs = Math.min(
+ policy.queueWaitMs ?? CODE_ENVIRONMENT_ADMISSION_MAX_MS,
+ Math.floor(remainingMs - completionReserveMs),
);
- if (Date.now() >= retryDeadlineAt) throw rejection;
- const retryAfterMs = workspaceAdmissionRetryDelay(
- response.headers.get('Retry-After'),
- admission === 'rate_limited' ? body : undefined,
+ if (queueAllowanceMs < 1) {
+ throw lastAdmissionRejection ?? new WorkspaceToolHttpError('insufficient_time');
+ }
+ attempts++;
+ const response = await fetchImpl(
+ `${baseURL.trim().replace(/\/+$/, '')}/workspace-tools/execute`,
+ {
+ method: 'POST',
+ headers: {
+ ...attemptHeaders,
+ 'Content-Type': 'application/json',
+ [WORKSPACE_QUEUE_WAIT_HEADER]: String(queueAllowanceMs),
+ },
+ body,
+ redirect: 'error',
+ signal: requestSignal,
+ },
);
+ if (!response.ok) {
+ const { body, truncated } = await readErrorBody(response, requestSignal);
+ signal?.throwIfAborted();
+ const rejection = new WorkspaceToolHttpError(
+ 'rejected',
+ response.status,
+ body,
+ truncated,
+ );
+ const admission = getWorkspaceAdmissionRejection(response.status, body, truncated);
+ if (admission == null) throw rejection;
+ if (admission === 'queue_timeout') queueRejections++;
+ else rateLimitRejections++;
+ const rejectionCount =
+ admission === 'queue_timeout' ? queueRejections : rateLimitRejections;
+ if (rejectionCount === 1) {
+ span.addEvent('workspace.admission.rejected', { kind: admission });
+ }
+ const retryDeadlineAt = Math.min(
+ admission === 'queue_timeout' ? queueDeadlineAt : Infinity,
+ callerRetryDeadlineAt,
+ );
+ if (Date.now() >= retryDeadlineAt) throw rejection;
+ const retryAfterMs = workspaceAdmissionRetryDelay(
+ response.headers.get('Retry-After'),
+ admission === 'rate_limited' ? body : undefined,
+ rejectionCount,
+ policy,
+ );
+ if (
+ admission === 'rate_limited' &&
+ retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs
+ ) {
+ throw rejection;
+ }
+ // Never truncate a server hint into an early retry.
+ if (!Number.isFinite(retryAfterMs) || retryAfterMs >= retryDeadlineAt - Date.now()) {
+ throw rejection;
+ }
+ const delayMs = retryAfterMs;
+ lastAdmissionRejection = rejection;
+ lastRetryDeadlineAt = retryDeadlineAt;
+ if (delayMs <= 0) throw rejection;
+ const waitStartedAt = Date.now();
+ await waitForWorkspaceAdmission(delayMs, signal);
+ retryWaitMs += Math.max(delayMs, Date.now() - waitStartedAt);
+ if (admission === 'rate_limited') {
+ /** A timer may run late. Charge the actual wait, but honor the retry that
+ * was authorized before waiting; another 429 cannot exceed the balance. */
+ rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt);
+ }
+ /** Timer oversleep cannot consume the execution reserve. */
+ if (Date.now() >= retryDeadlineAt) {
+ throw rejection;
+ }
+ continue;
+ }
+ const result = await readBoundedJson(response, requestSignal);
+ if (!isValidResult(wireRequest, result)) {
+ throw new WorkspaceToolHttpError('invalid');
+ }
+ outcome = 'completed';
+ return lane ? fromLinkedWorktreeResult(result, lane.worktree) : result;
+ } catch (error) {
+ outcome = error instanceof WorkspaceToolHttpError ? error.reason : 'failed';
+ if (signal?.aborted === true) outcome = 'cancelled';
+ if (error instanceof WorkspaceToolHttpError) throw error;
if (
- admission === 'rate_limited' &&
- retryAfterMs > codeApiMaxRetryWaitMs - rateLimitWaitedMs
+ signal?.aborted === true &&
+ (error === signal.reason || (isRecord(error) && error.name === 'AbortError'))
) {
- throw rejection;
- }
- const delayMs = Math.min(retryAfterMs, retryDeadlineAt - Date.now());
- lastAdmissionRejection = rejection;
- lastRetryDeadlineAt = retryDeadlineAt;
- if (delayMs <= 0) throw rejection;
- const waitStartedAt = Date.now();
- await waitForWorkspaceAdmission(delayMs, signal);
- if (admission === 'rate_limited') {
- /** A timer may run late. Charge the actual wait, but honor the retry that
- * was authorized before waiting; another 429 cannot exceed the balance. */
- rateLimitWaitedMs += Math.max(retryAfterMs, Date.now() - waitStartedAt);
+ throw error;
}
- /** A clamped delay can land exactly on this admission's retry deadline. Never open
- * another admission window without the full execution reserve. */
- if (Date.now() >= retryDeadlineAt) {
- throw rejection;
+ if (isRecord(error) && error.name === 'TimeoutError') {
+ outcome = 'timeout';
+ throw new WorkspaceToolHttpError('timeout');
}
- continue;
- }
- const result = await readBoundedJson(response, requestSignal);
- if (!isValidResult(wireRequest, result)) {
- throw new WorkspaceToolHttpError('invalid');
- }
- return lane ? fromLinkedWorktreeResult(result, lane.worktree) : result;
- } catch (error) {
- if (error instanceof WorkspaceToolHttpError) throw error;
- if (
- signal?.aborted === true &&
- (error === signal.reason || (isRecord(error) && error.name === 'AbortError'))
- ) {
- throw error;
+ throw new WorkspaceToolHttpError('failed');
}
- if (isRecord(error) && error.name === 'TimeoutError') {
- throw new WorkspaceToolHttpError('timeout');
- }
- throw new WorkspaceToolHttpError('failed');
}
+ } finally {
+ const diagnostics = {
+ operation: wireRequest.operation,
+ workspaceId: wireRequest.workspaceId,
+ ...(lane == null ? {} : { lane: lane.worktree }),
+ outcome,
+ attempts,
+ queueRejections,
+ rateLimitRejections,
+ retryWaitMs,
+ rateLimitWaitedMs,
+ elapsedMs: Math.max(0, Date.now() - startedAt),
+ };
+ span.setAttributes({
+ 'workspace.outcome': outcome,
+ 'workspace.attempts': attempts,
+ 'workspace.queue_rejections': queueRejections,
+ 'workspace.rate_limit_rejections': rateLimitRejections,
+ 'workspace.client_retry_wait_ms': retryWaitMs,
+ 'workspace.rate_limit_wait_ms': rateLimitWaitedMs,
+ });
+ span.end();
+ logger.debug('[WorkspaceAdmission] outcome', diagnostics);
}
}
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index 4081c33ffa..addf042c7b 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -2518,3 +2518,48 @@ describe('subagent activity policy', () => {
).toBe(false);
});
});
+
+describe('workspace admission configuration', () => {
+ it('preserves absent defaults and resolves an explicit admission policy', () => {
+ expect(codeEnvironmentUserConfigSchema.parse({})).toEqual({});
+ expect(codeEnvironmentUserConfigSchema.parse({ admission: {} }).admission).toEqual({
+ initialDelayMs: 1_000,
+ maxDelayMs: 30_000,
+ multiplier: 1,
+ jitterRatio: 0,
+ });
+ expect(
+ codeEnvironmentUserConfigSchema.parse({
+ admission: { queueWaitMs: 180_000, multiplier: 2, jitterRatio: 0.2 },
+ limits: { maxRequestTimeoutMs: 220_000, maxRunTimeoutMs: 400_000 },
+ }),
+ ).toMatchObject({
+ admission: { queueWaitMs: 180_000, multiplier: 2, jitterRatio: 0.2 },
+ limits: { maxRequestTimeoutMs: 220_000, maxRunTimeoutMs: 400_000 },
+ });
+ });
+
+ it.each([
+ { queueWaitMs: 0 },
+ { queueWaitMs: 300_001 },
+ { initialDelayMs: 99 },
+ { initialDelayMs: 2_000, maxDelayMs: 1_000 },
+ { multiplier: 0 },
+ { jitterRatio: -0.1 },
+ { jitterRatio: 1.1 },
+ { maxDelayMs: Infinity },
+ { queueWaitMs: 1.5 },
+ { unexpected: true },
+ ])('rejects an invalid policy %j', (admission) => {
+ expect(codeEnvironmentUserConfigSchema.safeParse({ admission }).success).toBe(false);
+ });
+
+ it.each([0, 1, -1, 1.5, 20_000, 610_001, Infinity])(
+ 'rejects an invalid run deadline %s',
+ (maxRunTimeoutMs) => {
+ expect(
+ codeEnvironmentUserConfigSchema.safeParse({ limits: { maxRunTimeoutMs } }).success,
+ ).toBe(false);
+ },
+ );
+});
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 86be6771e6..bdd91bbffa 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -1281,6 +1281,22 @@ export const CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS =
CODE_ENVIRONMENT_COMMAND_TIMEOUT_HARD_MAX_MS +
CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS;
+export const codeEnvironmentAdmissionSchema = z
+ .object({
+ /** Optional per-request queue ceiling, bounded by transport and execution reserves. */
+ queueWaitMs: z.number().int().min(1).max(CODE_ENVIRONMENT_ADMISSION_MAX_MS).optional(),
+ initialDelayMs: z.number().int().min(100).max(30_000).optional().default(1_000),
+ maxDelayMs: z.number().int().min(100).max(300_000).optional().default(30_000),
+ multiplier: z.number().min(1).max(10).optional().default(1),
+ /** Additive jitter never advances a server's Retry-After hint. */
+ jitterRatio: z.number().min(0).max(1).optional().default(0),
+ })
+ .strict()
+ .refine((policy) => policy.maxDelayMs >= policy.initialDelayMs, {
+ path: ['maxDelayMs'],
+ message: 'Maximum retry delay must cover the initial delay',
+ });
+
/**
* Typed user-tunable surface for one attached code environment. Omitted fields
* remain fixed at LibreChat's safe baseline. Isolation, networking, mounts,
@@ -1295,6 +1311,7 @@ export const codeEnvironmentUserConfigSchema = z
})
.strict()
.optional(),
+ admission: codeEnvironmentAdmissionSchema.optional(),
limits: z
.object({
/** Foreground Bash timeout when the call omits timeoutMs. Omission keeps 30 seconds;
@@ -1331,19 +1348,25 @@ export const codeEnvironmentUserConfigSchema = z
.min(0)
.max(CODE_ENVIRONMENT_QUEUE_WAIT_DEFAULT_MS)
.optional(),
- /** Total HTTP budget for one workspace tool call, including retries,
- * execution, settlement, and delivery. Only set this after verifying
- * the shortest timeout on the actual Code API path and updating Code API
- * to honor per-request queue allowances. Omission keeps the 30-second
- * per-attempt admission budget. */
+ /** Transport ceiling after verifying the shortest timeout on the Code API path.
+ * Also the overall call budget unless maxRunTimeoutMs is set. Omission keeps
+ * the legacy per-attempt admission and execution budgets. */
maxRequestTimeoutMs: z
.number()
.int()
.min(1)
.max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)
.optional(),
+ /** Separate overall call deadline across credentials, backoff, and HTTP attempts.
+ * Omission preserves maxRequestTimeoutMs as the total budget. */
+ maxRunTimeoutMs: z
+ .number()
+ .int()
+ .min(1)
+ .max(CODE_ENVIRONMENT_REQUEST_TIMEOUT_HARD_MAX_MS)
+ .optional(),
/** Admission allowance before local dispatch overhead for a Bash command inside
- * maxRequestTimeoutMs. Omission reserves ten seconds; ignored without a total HTTP budget. */
+ * the transport/run budgets. Omission reserves ten seconds. */
minCommandAdmissionMs: z
.number()
.int()
@@ -1353,19 +1376,22 @@ export const codeEnvironmentUserConfigSchema = z
})
.strict()
.superRefine((limits, context) => {
+ const budgetMs = Math.min(
+ limits.maxRequestTimeoutMs ?? Infinity,
+ limits.maxRunTimeoutMs ?? Infinity,
+ );
if (
- limits.maxRequestTimeoutMs == null ||
- limits.maxRequestTimeoutMs >
- (limits.minCommandAdmissionMs ?? CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS) +
- CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS
+ budgetMs >
+ (limits.minCommandAdmissionMs ?? CODE_ENVIRONMENT_COMMAND_ADMISSION_DEFAULT_MS) +
+ CODE_ENVIRONMENT_COMMAND_BUDGET_GRACE_MS
) {
return;
}
+ const field =
+ limits.maxRunTimeoutMs === budgetMs ? 'maxRunTimeoutMs' : 'maxRequestTimeoutMs';
context.addIssue({
code: z.ZodIssueCode.custom,
- path: [
- limits.minCommandAdmissionMs == null ? 'maxRequestTimeoutMs' : 'minCommandAdmissionMs',
- ],
+ path: [limits.minCommandAdmissionMs == null ? field : 'minCommandAdmissionMs'],
message: 'Command admission and settlement reserves must leave time for execution',
});
})