Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧯 fix: Guard SSE Writes and Interval Cleanups Against Crashes (#17462)

Marco Beretta784582850afb
diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.js
index f977e017b7..8cc6dc5260 100644
--- a/api/cache/getLogStores.js
+++ b/api/cache/getLogStores.js
@@ -12,6 +12,7 @@ const {
registerShutdownTask,
auditCacheStores,
formatMemoryUsage,
+ createSingleFlight,
} = require('@librechat/api');
/** No-op store for cache namespaces that are deliberately disabled. */
@@ -110,7 +111,7 @@ async function clearExpiredFromCache(cache) {
return;
}
- const expiryTime = Date.now() - ttl;
+ const expiryTime = Date.now();
let cleared = 0;
// Get all keys first to avoid modification during iteration
@@ -124,7 +125,7 @@ async function clearExpiredFromCache(cache) {
}
const data = cache.opts.deserialize(raw);
- // Check if the entry is older than TTL
+ // Check if the entry has expired
if (data?.expires && data.expires <= expiryTime) {
const deleted = await cache.opts.store.delete(key);
if (!deleted) {
@@ -151,9 +152,7 @@ async function clearExpiredFromCache(cache) {
if (cleared > 0) {
cacheConfig.DEBUG_MEMORY_CACHE &&
- logger.info(
- `[Cache] Cleared ${cleared} entries older than ${ttl}ms from ${cache.opts.namespace}`,
- );
+ logger.info(`[Cache] Cleared ${cleared} expired entries from ${cache.opts.namespace}`);
}
}
@@ -177,9 +176,10 @@ if (!cacheConfig.USE_REDIS && !cacheConfig.CI) {
const cleanupIntervals = new Set();
// Clear expired entries every 30 seconds
- const cleanup = setInterval(() => {
- clearAllExpiredFromCache();
- }, Time.THIRTY_SECONDS);
+ const cleanup = setInterval(
+ createSingleFlight(clearAllExpiredFromCache, logger, '[Cache] Expired entry sweep'),
+ Time.THIRTY_SECONDS,
+ );
cleanupIntervals.add(cleanup);
diff --git a/api/cache/logViolation.js b/api/cache/logViolation.js
index fd2549c58d..153c56a99e 100644
--- a/api/cache/logViolation.js
+++ b/api/cache/logViolation.js
@@ -1,4 +1,4 @@
-const { isEnabled } = require('@librechat/api');
+const { isEnabled, incrementViolationCount, appendViolationLog } = require('@librechat/api');
const { ViolationTypes } = require('librechat-data-provider');
const getLogStores = require('./getLogStores');
const banViolation = require('./banViolation');
@@ -21,20 +21,20 @@ const logViolation = async (req, res, type, errorMessage, score = 1) => {
const violationLogs = getLogStores(type);
const key = isEnabled(process.env.USE_REDIS) ? `${type}:${userId}` : userId;
- const userViolations = (await violationLogs.get(key)) ?? 0;
- const violationCount = +userViolations + +score;
- await violationLogs.set(key, violationCount);
+ const { prevCount, count: violationCount } = await incrementViolationCount(
+ violationLogs,
+ key,
+ score,
+ );
errorMessage.user_id = userId;
- errorMessage.prev_count = userViolations;
+ errorMessage.prev_count = prevCount;
errorMessage.violation_count = violationCount;
errorMessage.date = new Date().toISOString();
await banViolation(req, res, errorMessage);
- const userLogs = (await logs.get(key)) ?? [];
- userLogs.push(errorMessage);
delete errorMessage.user_id;
- await logs.set(key, userLogs);
+ await appendViolationLog(logs, key, errorMessage, req.config?.violationLog?.maxEntries);
};
module.exports = logViolation;
diff --git a/api/server/controllers/agents/__tests__/callbacks.spec.js b/api/server/controllers/agents/__tests__/callbacks.spec.js
index b1334c2e6c..18f6101f04 100644
--- a/api/server/controllers/agents/__tests__/callbacks.spec.js
+++ b/api/server/controllers/agents/__tests__/callbacks.spec.js
@@ -6,6 +6,13 @@ jest.mock('nanoid', () => ({
}));
jest.mock('@librechat/api', () => ({
+ writeGuarded: (res, chunk) => {
+ if (res.writableEnded === true || res.destroyed === true) {
+ return false;
+ }
+ res.write(chunk);
+ return true;
+ },
sendEvent: jest.fn(),
writeAttachmentEvent: jest.fn(),
createOwnedToolEndHandler: jest.fn((callback) => ({ handle: callback })),
diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js
index 85502523d8..4145f43b1b 100644
--- a/api/server/controllers/agents/callbacks.js
+++ b/api/server/controllers/agents/callbacks.js
@@ -39,6 +39,7 @@ const {
collectToolCallIds,
createToolTimingAdapter,
stampCommandExecutor,
+ writeGuarded,
} = require('@librechat/api');
const { processFileCitations } = require('~/server/services/Files/Citations');
const { processCodeOutput, runPreviewFinalize } = require('~/server/services/Files/Code/process');
@@ -849,7 +850,7 @@ function writeAttachment(res, streamId, attachment, expectedCreatedAt) {
{ expectedCreatedAt },
);
} else {
- res.write(`event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);
+ writeGuarded(res, `event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);
}
}
diff --git a/api/server/routes/files/speech/tts.js b/api/server/routes/files/speech/tts.js
index 3c0c6d1eb7..0cccf35016 100644
--- a/api/server/routes/files/speech/tts.js
+++ b/api/server/routes/files/speech/tts.js
@@ -53,7 +53,7 @@ router.post('/', async (req, res) => {
logDebugMessage(req, 'stream audio already running');
return res.status(401).json({ error: 'Audio stream already running' });
}
- audioRunsCache.set(req.body.runId, true);
+ await audioRunsCache.set(req.body.runId, true);
await streamAudio(req, res);
logDebugMessage(req, 'end stream audio');
res.status(200).end();
diff --git a/api/server/services/Endpoints/agents/title.js b/api/server/services/Endpoints/agents/title.js
index 8649154555..620637f741 100644
--- a/api/server/services/Endpoints/agents/title.js
+++ b/api/server/services/Endpoints/agents/title.js
@@ -1,4 +1,4 @@
-const { isEnabled, publishConversationTitle } = require('@librechat/api');
+const { isEnabled, publishConversationTitle, raceWithTimeout } = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
const { CacheKeys } = require('librechat-data-provider');
const getLogStores = require('~/cache/getLogStores');
@@ -68,15 +68,7 @@ const addTitle = async (
const titleCache = getLogStores(CacheKeys.GEN_TITLE);
const key = `${req.user.id}-${convoId}`;
- /** @type {NodeJS.Timeout} */
- let timeoutId;
try {
- const timeoutPromise = new Promise((_, reject) => {
- timeoutId = setTimeout(() => reject(new Error('Title generation timeout')), 45000);
- }).catch((error) => {
- logger.error('Title error:', error);
- });
-
let titlePromise;
let abortController = new AbortController();
/** Propagate a request abort (Stop) to the title generation so a cancelled
@@ -89,7 +81,7 @@ const addTitle = async (
}
}
if (client && typeof client.titleConvo === 'function') {
- titlePromise = Promise.race([
+ titlePromise = raceWithTimeout(
client
.titleConvo({
text,
@@ -99,8 +91,10 @@ const addTitle = async (
.catch((error) => {
logger.error('Client title error:', error);
}),
- timeoutPromise,
- ]);
+ 45000,
+ logger,
+ 'Title generation',
+ );
} else {
return;
}
@@ -109,9 +103,6 @@ const addTitle = async (
if (!abortController.signal.aborted) {
abortController.abort();
}
- if (timeoutId) {
- clearTimeout(timeoutId);
- }
if (!generatedTitle) {
logger.debug(`[${key}] No title generated`);
diff --git a/api/server/services/Tools/search.js b/api/server/services/Tools/search.js
index e0ded1268b..1b6b7f399a 100644
--- a/api/server/services/Tools/search.js
+++ b/api/server/services/Tools/search.js
@@ -1,7 +1,7 @@
const { nanoid } = require('nanoid');
const { Tools } = require('librechat-data-provider');
const { logger } = require('@librechat/data-schemas');
-const { GenerationJobManager } = require('@librechat/api');
+const { GenerationJobManager, writeGuarded } = require('@librechat/api');
/**
* Helper to write attachment events either to res or to job emitter.
@@ -18,7 +18,7 @@ function writeAttachment(res, streamId, attachment, jobCreatedAt) {
{ expectedCreatedAt: jobCreatedAt },
);
} else {
- res.write(`event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);
+ writeGuarded(res, `event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);
}
}
diff --git a/api/server/services/Tools/search.spec.js b/api/server/services/Tools/search.spec.js
index adb864ab7b..faa9f07204 100644
--- a/api/server/services/Tools/search.spec.js
+++ b/api/server/services/Tools/search.spec.js
@@ -5,6 +5,13 @@ jest.mock('nanoid', () => ({
}));
jest.mock('@librechat/api', () => ({
+ writeGuarded: (res, chunk) => {
+ if (res.writableEnded === true || res.destroyed === true) {
+ return false;
+ }
+ res.write(chunk);
+ return true;
+ },
GenerationJobManager: {
emitChunk: (...args) => mockEmitChunk(...args),
},
diff --git a/api/server/services/initializeOAuthReconnectManager.js b/api/server/services/initializeOAuthReconnectManager.js
index f60fe58c7f..f3c1ddfb90 100644
--- a/api/server/services/initializeOAuthReconnectManager.js
+++ b/api/server/services/initializeOAuthReconnectManager.js
@@ -1,5 +1,8 @@
const { logger, getTenantId } = require('@librechat/data-schemas');
-const { prepareMCPAuthorizationMutation } = require('@librechat/api');
+const {
+ prepareMCPAuthorizationMutation,
+ createOAuthReconnectionTracker,
+} = require('@librechat/api');
const { CacheKeys } = require('librechat-data-provider');
const { createOAuthReconnectionManager, getFlowStateManager, getMCPManager } = require('~/config');
const { findToken, updateToken, createToken, deleteTokens } = require('~/models');
@@ -22,10 +25,11 @@ async function initializeOAuthReconnectManager() {
createToken,
deleteTokens,
};
+ const startupConfig = await getAppConfig({ baseOnly: true });
await createOAuthReconnectionManager(
flowManager,
tokenMethods,
- undefined,
+ createOAuthReconnectionTracker(startupConfig),
undefined,
async (scope) => {
const appConfig = await getAppConfig({ tenantId: getTenantId(), userId: scope.userId });
diff --git a/api/server/services/initializeOAuthReconnectManager.spec.js b/api/server/services/initializeOAuthReconnectManager.spec.js
index aae1cd837e..c25f6c6655 100644
--- a/api/server/services/initializeOAuthReconnectManager.spec.js
+++ b/api/server/services/initializeOAuthReconnectManager.spec.js
@@ -8,6 +8,7 @@ jest.mock('@librechat/data-schemas', () => ({
}));
jest.mock('@librechat/api', () => ({
+ createOAuthReconnectionTracker: (config) => ({ type: 'tracker', config }),
prepareMCPAuthorizationMutation: (...args) => mockPrepareMCPAuthorizationMutation(...args),
}));
@@ -60,7 +61,10 @@ describe('initializeOAuthReconnectManager', () => {
it('resolves the effective user policy when a reconnect refresh prepares its write', async () => {
await initializeOAuthReconnectManager();
- expect(mockGetAppConfig).not.toHaveBeenCalled();
+ /* Startup reads only the base config for the tracker; the per-user policy stays lazy. */
+ expect(mockGetAppConfig).toHaveBeenCalledTimes(1);
+ expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true });
+ expect(mockCreateOAuthReconnectionManager.mock.calls[0][2]).toMatchObject({ type: 'tracker' });
const prepareRefresh = mockCreateOAuthReconnectionManager.mock.calls[0][4];
const scope = { userId: 'user-1', serverName: 'github' };
await prepareRefresh(scope);
diff --git a/config/flush-cache.js b/config/flush-cache.js
index 05cf23c720..5ef5004e39 100644
--- a/config/flush-cache.js
+++ b/config/flush-cache.js
@@ -158,23 +158,22 @@ async function flushRedisCache(dryRun = false, verbose = false) {
if (dryRun) {
console.log('🔍 [DRY RUN] Would flush Redis cache');
try {
- let allKeys = [];
+ let totalKeys = 0;
+ let sampleKeys = [];
+ const targets = useCluster ? redis.nodes('master') : [redis];
if (useCluster) {
- const nodes = redis.nodes('master');
- console.log(` Cluster detected: ${nodes.length} master nodes`);
- for (const node of nodes) {
- const keys = await node.keys('*');
- allKeys = allKeys.concat(keys);
+ console.log(` Cluster detected: ${targets.length} master nodes`);
+ }
+ for (const node of targets) {
+ totalKeys += await node.dbsize();
+ if (verbose && sampleKeys.length < 10) {
+ const [, keys] = await node.scan('0', 'COUNT', 100);
+ sampleKeys = sampleKeys.concat(keys).slice(0, 10);
}
- } else {
- allKeys = await redis.keys('*');
}
- console.log(` Would delete ${allKeys.length} keys`);
- if (verbose && allKeys.length > 0) {
- console.log(
- ' Sample keys:',
- allKeys.slice(0, 10).join(', ') + (allKeys.length > 10 ? '...' : ''),
- );
+ console.log(` Would delete ${totalKeys} keys`);
+ if (verbose && totalKeys > 0) {
+ console.log(' Sample keys:', sampleKeys.join(', ') + (totalKeys > 10 ? '...' : ''));
}
} catch (error) {
console.log(' Could not fetch keys for preview:', error.message);
@@ -186,15 +185,9 @@ async function flushRedisCache(dryRun = false, verbose = false) {
// Get key count before flushing
let keyCount = 0;
try {
- if (useCluster) {
- const nodes = redis.nodes('master');
- for (const node of nodes) {
- const keys = await node.keys('*');
- keyCount += keys.length;
- }
- } else {
- const keys = await redis.keys('*');
- keyCount = keys.length;
+ const targets = useCluster ? redis.nodes('master') : [redis];
+ for (const node of targets) {
+ keyCount += await node.dbsize();
}
} catch (_error) {
// Continue with flush even if we can't count keys
diff --git a/e2e/specs/mock/scenarios/stream-disconnect.spec.ts b/e2e/specs/mock/scenarios/stream-disconnect.spec.ts
new file mode 100644
index 0000000000..32a9cd97ad
--- /dev/null
+++ b/e2e/specs/mock/scenarios/stream-disconnect.spec.ts
@@ -0,0 +1,123 @@
+import { expect, test } from '@playwright/test';
+import type { APIRequestContext, Page } from '@playwright/test';
+import { randomUUID } from 'node:crypto';
+import { getE2EUser } from '../../../setup/user';
+import { deleteConversations, deleteMessagesByConversation, withMongo } from '../db';
+import {
+ MOCK_ENDPOINTS,
+ NEW_CHAT_PATH,
+ messagesView,
+ selectMockEndpoint,
+ sendMessage,
+} from '../helpers';
+
+async function login(request: APIRequestContext) {
+ const { email, password } = getE2EUser();
+ const response = await request.post('/api/auth/login', { data: { email, password } });
+ expect(response.ok()).toBeTruthy();
+ const { token } = (await response.json()) as { token: string };
+ return token;
+}
+
+/** Starts a slow scripted reply and returns its conversation once the reply is streaming. */
+async function startSlowReply(page: Page, prompt: string) {
+ await page.goto(NEW_CHAT_PATH);
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+ const run = await sendMessage(page, prompt);
+ expect(run.ok()).toBeTruthy();
+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15_000 });
+
+ let conversationId = '';
+ await expect
+ .poll(
+ async () => {
+ const row = await withMongo((db) =>
+ db.collection('messages').findOne({ text: prompt, isCreatedByUser: true }),
+ );
+ conversationId = (row?.conversationId as string | undefined) ?? '';
+ return conversationId;
+ },
+ { timeout: 10_000 },
+ )
+ .not.toBe('');
+ return conversationId;
+}
+
+async function cleanup(conversationId: string) {
+ if (!conversationId) {
+ return;
+ }
+ await deleteMessagesByConversation([conversationId]);
+ await deleteConversations([conversationId]);
+}
+
+test.describe('generation streams and client disconnects', () => {
+ test('a running generation is listed as active for its owner @scenario:running-generation-listed-as-active', async ({
+ page,
+ playwright,
+ baseURL,
+ }) => {
+ test.setTimeout(120_000);
+ const api = await playwright.request.newContext({ baseURL });
+ let conversationId = '';
+ try {
+ const token = await login(api);
+ conversationId = await startSlowReply(
+ page,
+ `E2E_SLOW_REPLY:active-${randomUUID().slice(0, 8)}`,
+ );
+
+ const response = await api.get('/api/agents/chat/active', {
+ headers: { Authorization: `Bearer ${token}` },
+ });
+ expect(response.ok()).toBeTruthy();
+ const body = (await response.json()) as { activeJobIds: string[] };
+ expect(body.activeJobIds).toContain(conversationId);
+ } finally {
+ await api.dispose();
+ await cleanup(conversationId);
+ }
+ });
+
+ test('a client leaving mid-stream does not stop the server or lose the reply @scenario:client-disconnect-keeps-stream-and-server-alive', async ({
+ page,
+ playwright,
+ baseURL,
+ }) => {
+ test.setTimeout(120_000);
+ const api = await playwright.request.newContext({ baseURL });
+ let conversationId = '';
+ try {
+ const token = await login(api);
+ conversationId = await startSlowReply(
+ page,
+ `E2E_SLOW_REPLY:leave-${randomUUID().slice(0, 8)}`,
+ );
+
+ /* Dropping the page closes the SSE connection while chunks are still being written. */
+ await page.goto('about:blank');
+
+ const health = await api.get('/api/config', { failOnStatusCode: false });
+ expect(health.ok()).toBeTruthy();
+
+ await expect
+ .poll(
+ async () => {
+ const response = await api.get('/api/agents/chat/active', {
+ headers: { Authorization: `Bearer ${token}` },
+ });
+ const body = (await response.json()) as { activeJobIds: string[] };
+ return body.activeJobIds.includes(conversationId);
+ },
+ { timeout: 60_000 },
+ )
+ .toBe(false);
+
+ await page.goto(`/c/${conversationId}`);
+ await expect(messagesView(page).getByText('chunk-159')).toBeVisible({ timeout: 20_000 });
+ } finally {
+ await api.dispose();
+ await cleanup(conversationId);
+ }
+ });
+});
diff --git a/librechat.example.yaml b/librechat.example.yaml
index 7e747b02fb..85a878b3fe 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -557,6 +557,10 @@ registration:
# moderationMs: 10000 # the OpenAI moderation check before a chat turn
# imageFetchMs: 30000 # fetching a stored image to encode it for a provider
+# Newest violations kept per user in the general violation log.
+# violationLog:
+# maxEntries: 100
+
# rateLimits:
# # Per-user MCP App browser-route limits. Each limit uses a fixed one-minute window.
# mcpApps:
@@ -653,6 +657,7 @@ actions:
# # New MCP Apps require explicit opt-in. If omitted, legacy inline HTML remains enabled;
# # false disables both MCP Apps and legacy inline HTML.
# apps: true
+# oauthFailureRetentionMs: 86400000 # how long a failed OAuth reconnection is remembered
# allowedDomains:
# - 'host.docker.internal' # Docker host access (required for Docker setups)
# - 'localhost' # Local development
diff --git a/packages/api/src/agents/memory.ts b/packages/api/src/agents/memory.ts
index 6acc1a0a95..22a74b441f 100644
--- a/packages/api/src/agents/memory.ts
+++ b/packages/api/src/agents/memory.ts
@@ -36,7 +36,7 @@ import type { BaseMessage, ToolMessage } from '@librechat/agents/langchain/messa
import type { DynamicStructuredTool } from '@librechat/agents/langchain/tools';
import type { Response as ServerResponse } from 'express';
import type { ServerRequest, RunLLMConfig } from '~/types';
-import { resolveConfigHeaders, createSafeUser, getSafeErrorMetadata } from '~/utils';
+import { resolveConfigHeaders, createSafeUser, getSafeErrorMetadata, writeGuarded } from '~/utils';
import { contentFilterModelBoundBlockResponse } from '~/middleware/contentFilter';
import { extractMemoryContent } from '~/protection/adapters/submissions';
import { assertModelBoundContent } from '~/middleware/modelBoundContent';
@@ -1197,7 +1197,7 @@ async function handleMemoryArtifact({
{ expectedCreatedAt: jobCreatedAt },
);
} else {
- res.write(`event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);
+ writeGuarded(res, `event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);
}
return attachment;
}
diff --git a/packages/api/src/agents/openai/handlers.ts b/packages/api/src/agents/openai/handlers.ts
index 550c14b0f7..01c9b500dd 100644
--- a/packages/api/src/agents/openai/handlers.ts
+++ b/packages/api/src/agents/openai/handlers.ts
@@ -18,6 +18,7 @@ import type { UsageMetadata } from '~/stream/interfaces/IJobStore';
import type { ToolExecuteOptions } from '~/agents/handlers';
import { createToolExecuteHandler } from '~/agents/handlers';
import { aggregateCollectedUsage } from '../usage';
+import { writeGuarded } from '~/utils/events';
/**
* Create a chat completion chunk in OpenAI format
@@ -47,11 +48,13 @@ export function createChunk(
/** One synchronous output frame; HTTP connection and lifecycle stay with the caller. */
export interface OpenAIWriter {
write(frame: string): boolean | void;
+ writableEnded?: boolean;
+ destroyed?: boolean;
}
/** Write one Chat Completions frame through a caller-owned transport. */
export function writeSSE(writer: OpenAIWriter, data: ChatCompletionChunk | string): void {
- writer.write(`data: ${typeof data === 'string' ? data : JSON.stringify(data)}\n\n`);
+ writeGuarded(writer, `data: ${typeof data === 'string' ? data : JSON.stringify(data)}\n\n`);
}
/**
diff --git a/packages/api/src/agents/responses/handlers.ts b/packages/api/src/agents/responses/handlers.ts
index 44deca90ac..fc0977ad48 100644
--- a/packages/api/src/agents/responses/handlers.ts
+++ b/packages/api/src/agents/responses/handlers.ts
@@ -20,6 +20,7 @@ import type {
ResponseStatus,
Usage,
} from './types';
+import { writeGuarded } from '~/utils/events';
/* =============================================================================
* RESPONSE TRACKER
@@ -102,15 +103,15 @@ export function createResponseTracker(): ResponseTracker {
* The `event:` field matches the `type` in the data payload.
*/
export function writeEvent(res: ServerResponse, event: ResponseEvent): void {
- res.write(`event: ${event.type}\n`);
- res.write(`data: ${JSON.stringify(event)}\n\n`);
+ writeGuarded(res, `event: ${event.type}\n`);
+ writeGuarded(res, `data: ${JSON.stringify(event)}\n\n`);
}
/**
* Write the terminal [DONE] event
*/
export function writeDone(res: ServerResponse): void {
- res.write('data: [DONE]\n\n');
+ writeGuarded(res, 'data: [DONE]\n\n');
}
/* =============================================================================
diff --git a/packages/api/src/cache/index.ts b/packages/api/src/cache/index.ts
index 1ddba95f7a..effa58d4c1 100644
--- a/packages/api/src/cache/index.ts
+++ b/packages/api/src/cache/index.ts
@@ -9,3 +9,4 @@ export * from './redisUtils';
export * from './redisTelemetry';
export * from './audit';
export * from './lock';
+export * from './violation';
diff --git a/packages/api/src/cache/violation.spec.ts b/packages/api/src/cache/violation.spec.ts
new file mode 100644
index 0000000000..09b21d4b37
--- /dev/null
+++ b/packages/api/src/cache/violation.spec.ts
@@ -0,0 +1,48 @@
+import {
+ incrementViolationCount,
+ appendViolationLog,
+ MAX_VIOLATION_LOG_ENTRIES,
+} from './violation';
+
+const makeStore = <T>() => {
+ const map = new Map<string, T>();
+ return {
+ get: async (key: string) => {
+ const value = map.get(key);
+ await new Promise((resolve) => setTimeout(resolve, 1));
+ return value;
+ },
+ set: async (key: string, value: T) => {
+ map.set(key, value);
+ },
+ };
+};
+
+describe('violation helpers', () => {
+ it('counts concurrent violations without undercounting', async () => {
+ const store = makeStore<number | string>();
+ const results = await Promise.all(
+ Array.from({ length: 10 }, () => incrementViolationCount(store, 'u', 2)),
+ );
+ expect(Math.max(...results.map((r) => r.count))).toBe(20);
+ expect(results[0].prevCount).toBe(0);
+ });
+
+ it('keeps only the newest log entries', async () => {
+ const store = makeStore<number[]>();
+ for (let i = 0; i < MAX_VIOLATION_LOG_ENTRIES + 5; i++) {
+ await appendViolationLog(store, 'u', i);
+ }
+ const entries = (await store.get('u')) ?? [];
+ expect(entries).toHaveLength(MAX_VIOLATION_LOG_ENTRIES);
+ expect(entries[entries.length - 1]).toBe(MAX_VIOLATION_LOG_ENTRIES + 4);
+ });
+
+ it('honors a configured retention', async () => {
+ const store = makeStore<number[]>();
+ for (let i = 0; i < 10; i++) {
+ await appendViolationLog(store, 'u', i, 3);
+ }
+ expect(await store.get('u')).toEqual([7, 8, 9]);
+ });
+});
diff --git a/packages/api/src/cache/violation.ts b/packages/api/src/cache/violation.ts
new file mode 100644
index 0000000000..3f06f5caca
--- /dev/null
+++ b/packages/api/src/cache/violation.ts
@@ -0,0 +1,47 @@
+import { withKeyLock } from '~/utils/keylock';
+
+/** Minimal store surface (Keyv satisfies it). */
+export interface ViolationStore<T> {
+ get(key: string): Promise<T | undefined | null> | T | undefined | null;
+ set(key: string, value: T, ttl?: number): Promise<unknown> | unknown;
+}
+
+/**
+ * Default for the newest entries kept per user in the GENERAL violation log, overridable
+ * by `violationLog.maxEntries`. The log is write-only file storage that is rewritten whole
+ * on every append, so it must not grow forever.
+ */
+export const MAX_VIOLATION_LOG_ENTRIES = 100;
+
+/**
+ * Adds `score` to a user's counter. Keyv exposes no atomic increment (stored values are
+ * serialized envelopes, so a backend INCR cannot read them), so concurrent updates within
+ * this process are serialized per key instead; multi-replica races remain possible.
+ * @returns the counter before and after the update.
+ */
+export function incrementViolationCount(
+ store: ViolationStore<number | string>,
+ key: string,
+ score: number | string = 1,
+): Promise<{ prevCount: number; count: number }> {
+ return withKeyLock(`violation:count:${key}`, async () => {
+ const prevCount = +((await store.get(key)) ?? 0);
+ const count = prevCount + +score;
+ await store.set(key, count);
+ return { prevCount, count };
+ });
+}
+
+/** Appends an entry to a user's violation log, keeping only the newest entries. */
+export function appendViolationLog<T>(
+ store: ViolationStore<T[]>,
+ key: string,
+ entry: T,
+ maxEntries: number = MAX_VIOLATION_LOG_ENTRIES,
+): Promise<void> {
+ return withKeyLock(`violation:log:${key}`, async () => {
+ const entries = (await store.get(key)) ?? [];
+ entries.push(entry);
+ await store.set(key, entries.slice(-maxEntries));
+ });
+}
diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts
index 1da4b371ca..a8cbd98930 100644
--- a/packages/api/src/index.ts
+++ b/packages/api/src/index.ts
@@ -64,6 +64,7 @@ export * from './html';
/* OAuth */
export * from './oauth';
export * from './mcp/oauth/OAuthReconnectionManager';
+export * from './mcp/oauth/OAuthReconnectionTracker';
/* Crypto */
export * from './crypto';
/* Flow */
diff --git a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.ts b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.ts
index 206fe96ef1..e35ad5815d 100644
--- a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.ts
+++ b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.ts
@@ -1,4 +1,7 @@
-import { OAuthReconnectionTracker } from './OAuthReconnectionTracker';
+import {
+ OAuthReconnectionTracker,
+ createOAuthReconnectionTracker,
+} from './OAuthReconnectionTracker';
describe('OAuthReconnectTracker', () => {
let tracker: OAuthReconnectionTracker;
@@ -547,4 +550,49 @@ describe('OAuthReconnectTracker', () => {
expect(tracker.isStillReconnecting(anotherUserId, serverName)).toBe(true);
});
});
+
+ describe('failed entry retention', () => {
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ it('drops failed entries that were never retried once retention passes', () => {
+ jest.useFakeTimers();
+ tracker.setFailed(userId, serverName);
+ expect(tracker.getStats().usersWithFailedServers).toBe(1);
+
+ jest.advanceTimersByTime(25 * 60 * 60 * 1000);
+ tracker.setFailed('other-user', serverName);
+
+ expect(tracker.isFailed(userId, serverName)).toBe(false);
+ expect(tracker.getStats().usersWithFailedServers).toBe(1);
+ });
+
+ it('honors mcpSettings.oauthFailureRetentionMs', () => {
+ jest.useFakeTimers();
+ const short = createOAuthReconnectionTracker({
+ mcpSettings: { oauthFailureRetentionMs: 60_000 },
+ });
+ short.setFailed(userId, serverName);
+ jest.advanceTimersByTime(2 * 60_000);
+ short.setFailed('other-user', serverName);
+ expect(short.getStats().usersWithFailedServers).toBe(1);
+
+ const dflt = createOAuthReconnectionTracker(undefined);
+ dflt.setFailed(userId, serverName);
+ jest.advanceTimersByTime(2 * 60_000);
+ dflt.setFailed('other-user', serverName);
+ expect(dflt.getStats().usersWithFailedServers).toBe(2);
+ });
+
+ it('prunes promptly when retention is shorter than the prune interval', () => {
+ jest.useFakeTimers();
+ const tiny = new OAuthReconnectionTracker(1_000);
+ tiny.setFailed(userId, serverName);
+ tiny.setFailed('warm', serverName);
+ jest.advanceTimersByTime(2_000);
+ tiny.setFailed('other-user', serverName);
+ expect(tiny.getStats().usersWithFailedServers).toBe(1);
+ });
+ });
});
diff --git a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.ts b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.ts
index 504ea7d43a..0faba0fea6 100644
--- a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.ts
+++ b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.ts
@@ -5,7 +5,21 @@ interface FailedMeta {
const COOLDOWN_SCHEDULE_MS = [5 * 60 * 1000, 10 * 60 * 1000, 20 * 60 * 1000, 30 * 60 * 1000];
+/** Failed entries untouched this long are dropped so pairs that never retry do not accumulate. */
+export const DEFAULT_FAILED_RETENTION_MS: number = 24 * 60 * 60 * 1000;
+const FAILED_PRUNE_INTERVAL_MS = 60 * 1000;
+
+/** Builds the tracker with the `mcpSettings.oauthFailureRetentionMs` budget, else the default. */
+export function createOAuthReconnectionTracker(
+ appConfig?: { mcpSettings?: { oauthFailureRetentionMs?: number } | null } | null,
+): OAuthReconnectionTracker {
+ return new OAuthReconnectionTracker(appConfig?.mcpSettings?.oauthFailureRetentionMs);
+}
+
export class OAuthReconnectionTracker {
+ constructor(private readonly failedRetentionMs: number = DEFAULT_FAILED_RETENTION_MS) {}
+
+ private lastFailedPrune = 0;
private failedMeta: Map<string, Map<string, FailedMeta>> = new Map();
/** Map of userId -> Set of serverNames that are actively reconnecting */
private active: Map<string, Set<string>> = new Map();
@@ -63,7 +77,25 @@ export class OAuthReconnectionTracker {
return false;
}
+ private pruneFailed(now: number): void {
+ if (now - this.lastFailedPrune < Math.min(FAILED_PRUNE_INTERVAL_MS, this.failedRetentionMs)) {
+ return;
+ }
+ this.lastFailedPrune = now;
+ for (const [userId, userMap] of this.failedMeta) {
+ for (const [serverName, meta] of userMap) {
+ if (now - meta.lastFailedAt >= this.failedRetentionMs) {
+ userMap.delete(serverName);
+ }
+ }
+ if (userMap.size === 0) {
+ this.failedMeta.delete(userId);
+ }
+ }
+ }
+
public setFailed(userId: string, serverName: string): void {
+ this.pruneFailed(Date.now());
if (!this.failedMeta.has(userId)) {
this.failedMeta.set(userId, new Map());
}
diff --git a/packages/api/src/middleware/concurrency.ts b/packages/api/src/middleware/concurrency.ts
index ad745a827a..cf18dbd663 100644
--- a/packages/api/src/middleware/concurrency.ts
+++ b/packages/api/src/middleware/concurrency.ts
@@ -1,7 +1,7 @@
import { logger } from '@librechat/data-schemas';
import { CacheKeys, Time, ViolationTypes } from 'librechat-data-provider';
import { standardCache, cacheConfig, instrumentIORedisClient, ioredisClient } from '~/cache';
-import { isEnabled, math } from '~/utils';
+import { isEnabled, math, withKeyLock } from '~/utils';
const { USE_REDIS } = cacheConfig;
@@ -95,7 +95,7 @@ export interface ViolationInfo {
* the actual request processing lifecycle.
*
* When Redis is available, uses atomic INCR to prevent race conditions.
- * Falls back to non-atomic get/set for in-memory cache.
+ * Falls back to per-key serialized get/set for in-memory cache.
*
* @param userId - The user's ID
* @returns Object with `allowed` (boolean), `pendingRequests` (current count), and `limit`
@@ -151,21 +151,23 @@ export async function checkAndIncrementPendingRequest(
}
const key = buildMemoryKey(userId);
- const pendingRequests = +((await cache.get(key)) ?? 0);
+ return withKeyLock(`pending:${key}`, async () => {
+ const pendingRequests = +((await cache.get(key)) ?? 0);
+
+ if (pendingRequests >= limit) {
+ logger.debug(
+ `[concurrency] User ${userId} exceeded concurrent limit: ${pendingRequests}/${limit}`,
+ );
+ return { allowed: false, pendingRequests, limit };
+ }
- if (pendingRequests >= limit) {
+ await cache.set(key, pendingRequests + 1, Time.ONE_MINUTE);
logger.debug(
- `[concurrency] User ${userId} exceeded concurrent limit: ${pendingRequests}/${limit}`,
+ `[concurrency] User ${userId} incremented pending requests: ${pendingRequests + 1}/${limit}`,
);
- return { allowed: false, pendingRequests, limit };
- }
-
- await cache.set(key, pendingRequests + 1, Time.ONE_MINUTE);
- logger.debug(
- `[concurrency] User ${userId} incremented pending requests: ${pendingRequests + 1}/${limit}`,
- );
- return { allowed: true, pendingRequests: pendingRequests + 1, limit };
+ return { allowed: true, pendingRequests: pendingRequests + 1, limit };
+ });
}
/**
@@ -176,7 +178,7 @@ export async function checkAndIncrementPendingRequest(
* operation that should not interrupt the main flow if cache operations fail.
*
* When Redis is available, uses atomic DECR to prevent race conditions.
- * Falls back to non-atomic get/set for in-memory cache.
+ * Falls back to per-key serialized get/set for in-memory cache.
*
* @param userId - The user's ID
*/
@@ -215,15 +217,19 @@ export async function decrementPendingRequest(userId: string): Promise<void> {
}
const key = buildMemoryKey(userId);
- const currentReq = +((await cache.get(key)) ?? 0);
-
- if (currentReq >= 1) {
- await cache.set(key, currentReq - 1, Time.ONE_MINUTE);
- logger.debug(`[concurrency] User ${userId} decremented pending requests: ${currentReq - 1}`);
- } else {
- await cache.delete(key);
- logger.debug(`[concurrency] User ${userId} pending requests cleared (was ${currentReq})`);
- }
+ await withKeyLock(`pending:${key}`, async () => {
+ const currentReq = +((await cache.get(key)) ?? 0);
+
+ if (currentReq >= 1) {
+ await cache.set(key, currentReq - 1, Time.ONE_MINUTE);
+ logger.debug(
+ `[concurrency] User ${userId} decremented pending requests: ${currentReq - 1}`,
+ );
+ } else {
+ await cache.delete(key);
+ logger.debug(`[concurrency] User ${userId} pending requests cleared (was ${currentReq})`);
+ }
+ });
} catch (error) {
logger.error('[concurrency] Error decrementing pending request:', error);
}
diff --git a/packages/api/src/skills/handlers.ts b/packages/api/src/skills/handlers.ts
index af667db212..fe1f9db43c 100644
--- a/packages/api/src/skills/handlers.ts
+++ b/packages/api/src/skills/handlers.ts
@@ -716,6 +716,12 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): {
res.destroy();
}
});
+ /** Release the upstream when the client leaves before the body is flushed. */
+ res.once('close', () => {
+ if ('destroy' in stream && typeof stream.destroy === 'function') {
+ stream.destroy();
+ }
+ });
stream.pipe(res);
return;
}
diff --git a/packages/api/src/stream/__tests__/cleanupInterval.spec.ts b/packages/api/src/stream/__tests__/cleanupInterval.spec.ts
new file mode 100644
index 0000000000..b3c7826b6c
--- /dev/null
+++ b/packages/api/src/stream/__tests__/cleanupInterval.spec.ts
@@ -0,0 +1,48 @@
+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
+
+describe('InMemoryJobStore cleanup interval', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ it('reports a failed cleanup pass instead of leaving an unhandled rejection', async () => {
+ const store = new InMemoryJobStore();
+ const cleanup = jest.spyOn(store, 'cleanup').mockRejectedValue(new Error('cleanup failed'));
+ const unhandled = jest.fn();
+ process.on('unhandledRejection', unhandled);
+
+ await store.initialize();
+ await jest.advanceTimersByTimeAsync(60000);
+ await jest.advanceTimersByTimeAsync(60000);
+ await store.destroy();
+ process.off('unhandledRejection', unhandled);
+
+ expect(cleanup).toHaveBeenCalledTimes(2);
+ expect(unhandled).not.toHaveBeenCalled();
+ });
+
+ it('does not start a pass while the previous one is still running', async () => {
+ const store = new InMemoryJobStore();
+ let finish: (n: number) => void = () => undefined;
+ const cleanup = jest.spyOn(store, 'cleanup').mockImplementation(
+ () =>
+ new Promise<number>((resolve) => {
+ finish = resolve;
+ }),
+ );
+
+ await store.initialize();
+ await jest.advanceTimersByTimeAsync(180000);
+ expect(cleanup).toHaveBeenCalledTimes(1);
+
+ finish(0);
+ await jest.advanceTimersByTimeAsync(60000);
+ expect(cleanup).toHaveBeenCalledTimes(2);
+ finish(0);
+ await store.destroy();
+ });
+});
diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts
index a48c2253ea..8045c8e3d2 100644
--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts
+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts
@@ -155,6 +155,7 @@ export class InMemoryJobStore implements IJobStoreV2 {
private jobs = new Map<string, SerializableJobData>();
private contentState = new Map<string, ContentState>();
private cleanupInterval: NodeJS.Timeout | null = null;
+ private cleanupRunning = false;
private staleGenerationHandler?: (streamId: string, createdAt: number) => void;
setStaleGenerationHandler(handler?: (streamId: string, createdAt: number) => void): void {
@@ -241,7 +242,17 @@ export class InMemoryJobStore implements IJobStoreV2 {
}
this.cleanupInterval = setInterval(() => {
- this.cleanup();
+ if (this.cleanupRunning) {
+ return;
+ }
+ this.cleanupRunning = true;
+ this.cleanup()
+ .catch((err) => {
+ logger.error('[InMemoryJobStore] Cleanup error:', err);
+ })
+ .finally(() => {
+ this.cleanupRunning = false;
+ });
}, 60000);
if (this.cleanupInterval.unref) {
diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts
index dcf31dc71a..ff96a3b803 100644
--- a/packages/api/src/stream/implementations/RedisJobStore.ts
+++ b/packages/api/src/stream/implementations/RedisJobStore.ts
@@ -1911,6 +1911,7 @@ export class RedisJobStore implements IJobStoreV2 {
private redis: Redis | Cluster;
private cleanupInterval: NodeJS.Timeout | null = null;
+ private cleanupRunning = false;
private cleanupMembershipReady?: Promise<void>;
private ttl: typeof DEFAULT_TTL;
/** Coalescable chunk appends awaiting their window flush, per stream */
@@ -1970,9 +1971,17 @@ export class RedisJobStore implements IJobStoreV2 {
// Start periodic cleanup
this.cleanupInterval = setInterval(() => {
- this.cleanup().catch((err) => {
- logger.error('[RedisJobStore] Cleanup error:', err);
- });
+ if (this.cleanupRunning) {
+ return;
+ }
+ this.cleanupRunning = true;
+ this.cleanup()
+ .catch((err) => {
+ logger.error('[RedisJobStore] Cleanup error:', err);
+ })
+ .finally(() => {
+ this.cleanupRunning = false;
+ });
}, this.cleanupIntervalMs);
if (this.cleanupInterval.unref) {
@@ -3978,8 +3987,10 @@ export class RedisJobStore implements IJobStoreV2 {
const activeIds: string[] = [];
let healed = 0;
- for (const streamId of trackedIds) {
- const job = await this.getJob(streamId);
+ const trackedJobs = await Promise.all(trackedIds.map((streamId) => this.getJob(streamId)));
+
+ for (const [index, streamId] of trackedIds.entries()) {
+ const job = trackedJobs[index];
// Include running jobs and jobs paused for human review (e.g. tool approval).
// A pending-approval job still occupies the user's conversation slot — but
// only while its prompt is live: a past-`expiresAt` approval no longer
diff --git a/packages/api/src/utils/events.spec.ts b/packages/api/src/utils/events.spec.ts
new file mode 100644
index 0000000000..436db53a55
--- /dev/null
+++ b/packages/api/src/utils/events.spec.ts
@@ -0,0 +1,43 @@
+import { writeGuarded, sendEvent, handleError } from './events';
+
+type FakeRes = {
+ write: jest.Mock;
+ end: jest.Mock;
+ writableEnded?: boolean;
+ destroyed?: boolean;
+};
+
+const makeRes = (overrides: Partial<FakeRes> = {}): FakeRes => ({
+ write: jest.fn(),
+ end: jest.fn(),
+ ...overrides,
+});
+
+describe('writeGuarded', () => {
+ it('writes to a live response', () => {
+ const res = makeRes();
+ expect(writeGuarded(res, 'x')).toBe(true);
+ expect(res.write).toHaveBeenCalledWith('x');
+ });
+
+ it('skips an ended response', () => {
+ const res = makeRes({ writableEnded: true });
+ expect(writeGuarded(res, 'x')).toBe(false);
+ expect(res.write).not.toHaveBeenCalled();
+ });
+
+ it('skips a destroyed response', () => {
+ const res = makeRes({ destroyed: true });
+ expect(writeGuarded(res, 'x')).toBe(false);
+ expect(res.write).not.toHaveBeenCalled();
+ });
+});
+
+describe('sendEvent and handleError', () => {
+ it('does not write after the response ended', () => {
+ const res = makeRes({ writableEnded: true });
+ sendEvent(res as never, { event: 'x' } as never);
+ handleError(res as never, 'boom');
+ expect(res.write).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/api/src/utils/events.ts b/packages/api/src/utils/events.ts
index e084e631f5..7f87ab8108 100644
--- a/packages/api/src/utils/events.ts
+++ b/packages/api/src/utils/events.ts
@@ -1,6 +1,26 @@
import type { Response as ServerResponse } from 'express';
import type { ServerSentEvent } from '~/types';
+/** Minimal writable transport; an Express response satisfies it. */
+export interface GuardedWriter {
+ write(chunk: string): boolean | void;
+ writableEnded?: boolean;
+ destroyed?: boolean;
+}
+
+/**
+ * Writes a chunk unless the response has already ended or been destroyed
+ * (client disconnect), which would otherwise raise `ERR_STREAM_WRITE_AFTER_END`.
+ * @returns whether the chunk was handed to the transport.
+ */
+export function writeGuarded(res: GuardedWriter, chunk: string): boolean {
+ if (res.writableEnded === true || res.destroyed === true) {
+ return false;
+ }
+ res.write(chunk);
+ return true;
+}
+
/**
* Sends a Server-Sent Event to the client.
* Empty-string StreamEvent data is silently dropped.
@@ -9,7 +29,7 @@ export function sendEvent(res: ServerResponse, event: ServerSentEvent): void {
if ('data' in event && typeof event.data === 'string' && event.data.length === 0) {
return;
}
- res.write(`event: message\ndata: ${JSON.stringify(event)}\n\n`);
+ writeGuarded(res, `event: message\ndata: ${JSON.stringify(event)}\n\n`);
}
/**
@@ -18,6 +38,6 @@ export function sendEvent(res: ServerResponse, event: ServerSentEvent): void {
* @param message - The error message.
*/
export function handleError(res: ServerResponse, message: string): void {
- res.write(`event: error\ndata: ${JSON.stringify(message)}\n\n`);
+ writeGuarded(res, `event: error\ndata: ${JSON.stringify(message)}\n\n`);
res.end();
}
diff --git a/packages/api/src/utils/index.ts b/packages/api/src/utils/index.ts
index c920eaf193..9e2fc052cd 100644
--- a/packages/api/src/utils/index.ts
+++ b/packages/api/src/utils/index.ts
@@ -39,3 +39,4 @@ export * from './url';
export * from './message';
export * from './quotes';
export * from './tracing';
+export * from './keylock';
diff --git a/packages/api/src/utils/keylock.spec.ts b/packages/api/src/utils/keylock.spec.ts
new file mode 100644
index 0000000000..30bcb8862a
--- /dev/null
+++ b/packages/api/src/utils/keylock.spec.ts
@@ -0,0 +1,64 @@
+import { withKeyLock, createSingleFlight, raceWithTimeout } from './keylock';
+
+describe('withKeyLock', () => {
+ it('serializes read-modify-write cycles on one key without losing updates', async () => {
+ let value = 0;
+ const bump = () =>
+ withKeyLock('k', async () => {
+ const read = value;
+ await new Promise((resolve) => setTimeout(resolve, 1));
+ value = read + 1;
+ });
+ await Promise.all(Array.from({ length: 20 }, bump));
+ expect(value).toBe(20);
+ });
+
+ it('keeps running later tasks after one rejects', async () => {
+ await expect(withKeyLock('r', () => Promise.reject(new Error('x')))).rejects.toThrow('x');
+ await expect(withKeyLock('r', async () => 'ok')).resolves.toBe('ok');
+ });
+});
+
+describe('createSingleFlight', () => {
+ it('skips overlapping runs and reports rejections', async () => {
+ let release: () => void = () => undefined;
+ const task = jest.fn(
+ () =>
+ new Promise<void>((resolve, reject) => {
+ release = () => reject(new Error('fail'));
+ void resolve;
+ }),
+ );
+ const onError = jest.fn();
+ const tick = createSingleFlight(task, { error: onError }, '[Test] sweep');
+ tick();
+ tick();
+ expect(task).toHaveBeenCalledTimes(1);
+ release();
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(onError).toHaveBeenCalledTimes(1);
+ expect(onError).toHaveBeenCalledWith('[Test] sweep failed', expect.any(Error));
+ tick();
+ expect(task).toHaveBeenCalledTimes(2);
+ });
+});
+
+describe('raceWithTimeout', () => {
+ afterEach(() => jest.useRealTimers());
+
+ it('returns the work result and leaves no timer pending', async () => {
+ jest.useFakeTimers();
+ const log = { error: jest.fn() };
+ await expect(raceWithTimeout(Promise.resolve('ok'), 1000, log, '[T]')).resolves.toBe('ok');
+ expect(jest.getTimerCount()).toBe(0);
+ });
+
+ it('logs and resolves undefined when the work outlives the budget', async () => {
+ jest.useFakeTimers();
+ const log = { error: jest.fn() };
+ const result = raceWithTimeout(new Promise<string>(() => undefined), 1000, log, '[T]');
+ jest.advanceTimersByTime(1000);
+ await expect(result).resolves.toBeUndefined();
+ expect(log.error).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/api/src/utils/keylock.ts b/packages/api/src/utils/keylock.ts
new file mode 100644
index 0000000000..2207ed5e80
--- /dev/null
+++ b/packages/api/src/utils/keylock.ts
@@ -0,0 +1,66 @@
+const tails = new Map<string, Promise<unknown>>();
+
+/**
+ * Runs `fn` after every earlier task queued under the same `key` has settled, so a
+ * read-modify-write against a store without atomic operations cannot interleave
+ * with another one in this process. It does not coordinate separate processes.
+ */
+export function withKeyLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
+ const previous = tails.get(key) ?? Promise.resolve();
+ const run = previous.then(fn, fn);
+ const tail = run.catch(() => undefined);
+ tails.set(key, tail);
+ void tail.then(() => {
+ if (tails.get(key) === tail) {
+ tails.delete(key);
+ }
+ });
+ return run;
+}
+
+/**
+ * Wraps an async task for use in a timer so a slow run is never overlapped by the
+ * next tick and a rejection is reported instead of becoming an unhandled rejection.
+ */
+export function createSingleFlight(
+ task: () => Promise<unknown>,
+ log: { error: (message: string, error: unknown) => void },
+ label: string,
+): () => void {
+ let running = false;
+ return () => {
+ if (running) {
+ return;
+ }
+ running = true;
+ task()
+ .catch((error) => log.error(`${label} failed`, error))
+ .finally(() => {
+ running = false;
+ });
+ };
+}
+
+/**
+ * Resolves with `work`, or with `undefined` after `ms` (logging the timeout). The timer is
+ * always cleared, so a settled race never leaves it pending.
+ */
+export async function raceWithTimeout<T>(
+ work: Promise<T>,
+ ms: number,
+ log: { error: (message: string, error: unknown) => void },
+ label: string,
+): Promise<T | undefined> {
+ let timer: NodeJS.Timeout | undefined;
+ const timeout = new Promise<undefined>((resolve) => {
+ timer = setTimeout(() => {
+ log.error(`${label} error:`, new Error(`${label} timeout`));
+ resolve(undefined);
+ }, ms);
+ });
+ try {
+ return await Promise.race([work, timeout]);
+ } finally {
+ clearTimeout(timer);
+ }
+}
diff --git a/packages/api/src/utils/tokenizer.ts b/packages/api/src/utils/tokenizer.ts
index 83148ba86d..cd5782f244 100644
--- a/packages/api/src/utils/tokenizer.ts
+++ b/packages/api/src/utils/tokenizer.ts
@@ -62,7 +62,21 @@ class Tokenizer {
: await import('ai-tokenizer/encoding/o200k_base');
this.tokenizersCache[encoding] = new AiTokenizer(data);
})();
- return this.loadingPromises[encoding];
+ const loading = this.loadingPromises[encoding];
+ /** A failed load must not stay cached, or every later call replays the rejection. */
+ loading.catch(() => {
+ if (this.loadingPromises[encoding] === loading) {
+ delete this.loadingPromises[encoding];
+ }
+ });
+ return loading;
+ }
+
+ /** Fire-and-forget warm-up; a load failure is logged instead of surfacing as an unhandled rejection. */
+ private warmEncoding(encoding: EncodingName): void {
+ this.initEncoding(encoding).catch((error) => {
+ logger.error(`[Tokenizer] Failed to load encoding ${encoding}`, error);
+ });
}
/** Returns an initialized, deterministic budgeting counter that throws on tokenizer failure. */
@@ -85,7 +99,7 @@ class Tokenizer {
getTokenCount(text: string, encoding: EncodingName = 'o200k_base'): number {
const tokenizer = this.tokenizersCache[encoding];
if (!tokenizer) {
- this.initEncoding(encoding);
+ this.warmEncoding(encoding);
return estimateUnavailableTokenCount(text);
}
try {
@@ -115,7 +129,7 @@ class Tokenizer {
}
const tokenizer = this.tokenizersCache[encoding];
if (!tokenizer) {
- void this.initEncoding(encoding);
+ this.warmEncoding(encoding);
return undefined;
}
try {
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 809d6bae69..6288605599 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -181,6 +181,7 @@ export const DEPLOYMENT_ONLY_CONFIG_PATHS: Readonly<Record<string, DeploymentOnl
'endpoints.agents.backgroundTasks.completionResultBatchSize': 'startup',
'endpoints.agents.backgroundTasks.completionReceiptBatching': 'startup',
'endpoints.agents.toolApproval.hooks': 'operator',
+ 'mcpSettings.oauthFailureRetentionMs': 'startup',
'mcpSettings.catalogRecovery.maxStateEntries': 'startup',
'mcpSettings.catalogRecovery.maxDetachedDiscoveries': 'startup',
'mcpSettings.catalogRecovery.authorizationFenceRetryIntervalMs': 'startup',
@@ -2714,6 +2715,12 @@ export const outboundTimeoutsSchema = z.object({
imageFetchMs: outboundTimeoutMs,
});
+/** Retention for the write-only GENERAL violation log kept per user. */
+export const violationLogSchema = z.object({
+ /** Newest entries kept per user; older ones are dropped on the next append. */
+ maxEntries: z.number().int().positive().optional(),
+});
+
export type TOutboundTimeouts = z.infer<typeof outboundTimeoutsSchema>;
export const rateLimitSchema = z.object({
@@ -4525,6 +4532,14 @@ export const configSchema = z.object({
.object({
allowedDomains: z.array(z.string()).optional(),
allowedAddresses: allowedAddressesSchema,
+ /** How long a failed OAuth reconnection is remembered before its attempt count resets.
+ * Defaults to 24 hours. */
+ oauthFailureRetentionMs: z
+ .number()
+ .int()
+ .positive()
+ .max(30 * 24 * 60 * 60_000)
+ .optional(),
catalogRecovery: z
.object({
discoveryBackoffMs: z
@@ -4642,6 +4657,7 @@ export const configSchema = z.object({
})
.optional(),
timeouts: outboundTimeoutsSchema.optional(),
+ violationLog: violationLogSchema.optional(),
fileConfig: fileConfigSchema.optional(),
modelSpecs: specsConfigSchema.optional(),
filters: filtersConfigSchema.optional(),
diff --git a/packages/data-schemas/src/app/service.ts b/packages/data-schemas/src/app/service.ts
index 160c9cdd1e..67eb9d0191 100644
--- a/packages/data-schemas/src/app/service.ts
+++ b/packages/data-schemas/src/app/service.ts
@@ -222,6 +222,7 @@ export const AppService = async (params?: {
webSearch,
githubCompare: config.githubCompare,
timeouts: config.timeouts,
+ violationLog: config.violationLog,
mcpSettings,
mcpAppSandbox,
fileStrategy,
diff --git a/packages/data-schemas/src/types/app.ts b/packages/data-schemas/src/types/app.ts
index e74870d1d3..5b890c6bba 100644
--- a/packages/data-schemas/src/types/app.ts
+++ b/packages/data-schemas/src/types/app.ts
@@ -71,6 +71,8 @@ export interface AppConfig {
githubCompare?: TCustomConfig['githubCompare'];
/** Outbound call budgets; unset keys fall back to the server defaults */
timeouts?: TCustomConfig['timeouts'];
+ /** Violation log retention; unset keeps the server default */
+ violationLog?: TCustomConfig['violationLog'];
skillDeletion?: {
finalizationEnabled?: boolean;
sweepIntervalMs?: number;