Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧯 fix: Guard SSE Writes and Interval Cleanups Against Crashes (#17462)
Marco Beretta784582850afb
diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.jsindex f977e017b7..8cc6dc5260 100644--- a/api/cache/getLogStores.js+++ b/api/cache/getLogStores.js@@ -12,6 +12,7 @@ const {registerShutdownTask,auditCacheStores,formatMemoryUsage,+ createSingleFlight,} = require('@librechat/api');/** No-op store for cache namespaces that are deliberately disabled. */@@ -110,7 +111,7 @@ async function clearExpiredFromCache(cache) {return;}- const expiryTime = Date.now() - ttl;+ const expiryTime = Date.now();let cleared = 0;// Get all keys first to avoid modification during iteration@@ -124,7 +125,7 @@ async function clearExpiredFromCache(cache) {}const data = cache.opts.deserialize(raw);- // Check if the entry is older than TTL+ // Check if the entry has expiredif (data?.expires && data.expires <= expiryTime) {const deleted = await cache.opts.store.delete(key);if (!deleted) {@@ -151,9 +152,7 @@ async function clearExpiredFromCache(cache) {if (cleared > 0) {cacheConfig.DEBUG_MEMORY_CACHE &&- logger.info(- `[Cache] Cleared ${cleared} entries older than ${ttl}ms from ${cache.opts.namespace}`,- );+ logger.info(`[Cache] Cleared ${cleared} expired entries from ${cache.opts.namespace}`);}}@@ -177,9 +176,10 @@ if (!cacheConfig.USE_REDIS && !cacheConfig.CI) {const cleanupIntervals = new Set();// Clear expired entries every 30 seconds- const cleanup = setInterval(() => {- clearAllExpiredFromCache();- }, Time.THIRTY_SECONDS);+ const cleanup = setInterval(+ createSingleFlight(clearAllExpiredFromCache, logger, '[Cache] Expired entry sweep'),+ Time.THIRTY_SECONDS,+ );cleanupIntervals.add(cleanup);diff --git a/api/cache/logViolation.js b/api/cache/logViolation.jsindex fd2549c58d..153c56a99e 100644--- a/api/cache/logViolation.js+++ b/api/cache/logViolation.js@@ -1,4 +1,4 @@-const { isEnabled } = require('@librechat/api');+const { isEnabled, incrementViolationCount, appendViolationLog } = require('@librechat/api');const { ViolationTypes } = require('librechat-data-provider');const getLogStores = require('./getLogStores');const banViolation = require('./banViolation');@@ -21,20 +21,20 @@ const logViolation = async (req, res, type, errorMessage, score = 1) => {const violationLogs = getLogStores(type);const key = isEnabled(process.env.USE_REDIS) ? `${type}:${userId}` : userId;- const userViolations = (await violationLogs.get(key)) ?? 0;- const violationCount = +userViolations + +score;- await violationLogs.set(key, violationCount);+ const { prevCount, count: violationCount } = await incrementViolationCount(+ violationLogs,+ key,+ score,+ );errorMessage.user_id = userId;- errorMessage.prev_count = userViolations;+ errorMessage.prev_count = prevCount;errorMessage.violation_count = violationCount;errorMessage.date = new Date().toISOString();await banViolation(req, res, errorMessage);- const userLogs = (await logs.get(key)) ?? [];- userLogs.push(errorMessage);delete errorMessage.user_id;- await logs.set(key, userLogs);+ await appendViolationLog(logs, key, errorMessage, req.config?.violationLog?.maxEntries);};module.exports = logViolation;diff --git a/api/server/controllers/agents/__tests__/callbacks.spec.js b/api/server/controllers/agents/__tests__/callbacks.spec.jsindex b1334c2e6c..18f6101f04 100644--- a/api/server/controllers/agents/__tests__/callbacks.spec.js+++ b/api/server/controllers/agents/__tests__/callbacks.spec.js@@ -6,6 +6,13 @@ jest.mock('nanoid', () => ({}));jest.mock('@librechat/api', () => ({+ writeGuarded: (res, chunk) => {+ if (res.writableEnded === true || res.destroyed === true) {+ return false;+ }+ res.write(chunk);+ return true;+ },sendEvent: jest.fn(),writeAttachmentEvent: jest.fn(),createOwnedToolEndHandler: jest.fn((callback) => ({ handle: callback })),diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.jsindex 85502523d8..4145f43b1b 100644--- a/api/server/controllers/agents/callbacks.js+++ b/api/server/controllers/agents/callbacks.js@@ -39,6 +39,7 @@ const {collectToolCallIds,createToolTimingAdapter,stampCommandExecutor,+ writeGuarded,} = require('@librechat/api');const { processFileCitations } = require('~/server/services/Files/Citations');const { processCodeOutput, runPreviewFinalize } = require('~/server/services/Files/Code/process');@@ -849,7 +850,7 @@ function writeAttachment(res, streamId, attachment, expectedCreatedAt) {{ expectedCreatedAt },);} else {- res.write(`event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);+ writeGuarded(res, `event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);}}diff --git a/api/server/routes/files/speech/tts.js b/api/server/routes/files/speech/tts.jsindex 3c0c6d1eb7..0cccf35016 100644--- a/api/server/routes/files/speech/tts.js+++ b/api/server/routes/files/speech/tts.js@@ -53,7 +53,7 @@ router.post('/', async (req, res) => {logDebugMessage(req, 'stream audio already running');return res.status(401).json({ error: 'Audio stream already running' });}- audioRunsCache.set(req.body.runId, true);+ await audioRunsCache.set(req.body.runId, true);await streamAudio(req, res);logDebugMessage(req, 'end stream audio');res.status(200).end();diff --git a/api/server/services/Endpoints/agents/title.js b/api/server/services/Endpoints/agents/title.jsindex 8649154555..620637f741 100644--- a/api/server/services/Endpoints/agents/title.js+++ b/api/server/services/Endpoints/agents/title.js@@ -1,4 +1,4 @@-const { isEnabled, publishConversationTitle } = require('@librechat/api');+const { isEnabled, publishConversationTitle, raceWithTimeout } = require('@librechat/api');const { logger } = require('@librechat/data-schemas');const { CacheKeys } = require('librechat-data-provider');const getLogStores = require('~/cache/getLogStores');@@ -68,15 +68,7 @@ const addTitle = async (const titleCache = getLogStores(CacheKeys.GEN_TITLE);const key = `${req.user.id}-${convoId}`;- /** @type {NodeJS.Timeout} */- let timeoutId;try {- const timeoutPromise = new Promise((_, reject) => {- timeoutId = setTimeout(() => reject(new Error('Title generation timeout')), 45000);- }).catch((error) => {- logger.error('Title error:', error);- });-let titlePromise;let abortController = new AbortController();/** Propagate a request abort (Stop) to the title generation so a cancelled@@ -89,7 +81,7 @@ const addTitle = async (}}if (client && typeof client.titleConvo === 'function') {- titlePromise = Promise.race([+ titlePromise = raceWithTimeout(client.titleConvo({text,@@ -99,8 +91,10 @@ const addTitle = async (.catch((error) => {logger.error('Client title error:', error);}),- timeoutPromise,- ]);+ 45000,+ logger,+ 'Title generation',+ );} else {return;}@@ -109,9 +103,6 @@ const addTitle = async (if (!abortController.signal.aborted) {abortController.abort();}- if (timeoutId) {- clearTimeout(timeoutId);- }if (!generatedTitle) {logger.debug(`[${key}] No title generated`);diff --git a/api/server/services/Tools/search.js b/api/server/services/Tools/search.jsindex e0ded1268b..1b6b7f399a 100644--- a/api/server/services/Tools/search.js+++ b/api/server/services/Tools/search.js@@ -1,7 +1,7 @@const { nanoid } = require('nanoid');const { Tools } = require('librechat-data-provider');const { logger } = require('@librechat/data-schemas');-const { GenerationJobManager } = require('@librechat/api');+const { GenerationJobManager, writeGuarded } = require('@librechat/api');/*** Helper to write attachment events either to res or to job emitter.@@ -18,7 +18,7 @@ function writeAttachment(res, streamId, attachment, jobCreatedAt) {{ expectedCreatedAt: jobCreatedAt },);} else {- res.write(`event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);+ writeGuarded(res, `event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);}}diff --git a/api/server/services/Tools/search.spec.js b/api/server/services/Tools/search.spec.jsindex adb864ab7b..faa9f07204 100644--- a/api/server/services/Tools/search.spec.js+++ b/api/server/services/Tools/search.spec.js@@ -5,6 +5,13 @@ jest.mock('nanoid', () => ({}));jest.mock('@librechat/api', () => ({+ writeGuarded: (res, chunk) => {+ if (res.writableEnded === true || res.destroyed === true) {+ return false;+ }+ res.write(chunk);+ return true;+ },GenerationJobManager: {emitChunk: (...args) => mockEmitChunk(...args),},diff --git a/api/server/services/initializeOAuthReconnectManager.js b/api/server/services/initializeOAuthReconnectManager.jsindex f60fe58c7f..f3c1ddfb90 100644--- a/api/server/services/initializeOAuthReconnectManager.js+++ b/api/server/services/initializeOAuthReconnectManager.js@@ -1,5 +1,8 @@const { logger, getTenantId } = require('@librechat/data-schemas');-const { prepareMCPAuthorizationMutation } = require('@librechat/api');+const {+ prepareMCPAuthorizationMutation,+ createOAuthReconnectionTracker,+} = require('@librechat/api');const { CacheKeys } = require('librechat-data-provider');const { createOAuthReconnectionManager, getFlowStateManager, getMCPManager } = require('~/config');const { findToken, updateToken, createToken, deleteTokens } = require('~/models');@@ -22,10 +25,11 @@ async function initializeOAuthReconnectManager() {createToken,deleteTokens,};+ const startupConfig = await getAppConfig({ baseOnly: true });await createOAuthReconnectionManager(flowManager,tokenMethods,- undefined,+ createOAuthReconnectionTracker(startupConfig),undefined,async (scope) => {const appConfig = await getAppConfig({ tenantId: getTenantId(), userId: scope.userId });diff --git a/api/server/services/initializeOAuthReconnectManager.spec.js b/api/server/services/initializeOAuthReconnectManager.spec.jsindex aae1cd837e..c25f6c6655 100644--- a/api/server/services/initializeOAuthReconnectManager.spec.js+++ b/api/server/services/initializeOAuthReconnectManager.spec.js@@ -8,6 +8,7 @@ jest.mock('@librechat/data-schemas', () => ({}));jest.mock('@librechat/api', () => ({+ createOAuthReconnectionTracker: (config) => ({ type: 'tracker', config }),prepareMCPAuthorizationMutation: (...args) => mockPrepareMCPAuthorizationMutation(...args),}));@@ -60,7 +61,10 @@ describe('initializeOAuthReconnectManager', () => {it('resolves the effective user policy when a reconnect refresh prepares its write', async () => {await initializeOAuthReconnectManager();- expect(mockGetAppConfig).not.toHaveBeenCalled();+ /* Startup reads only the base config for the tracker; the per-user policy stays lazy. */+ expect(mockGetAppConfig).toHaveBeenCalledTimes(1);+ expect(mockGetAppConfig).toHaveBeenCalledWith({ baseOnly: true });+ expect(mockCreateOAuthReconnectionManager.mock.calls[0][2]).toMatchObject({ type: 'tracker' });const prepareRefresh = mockCreateOAuthReconnectionManager.mock.calls[0][4];const scope = { userId: 'user-1', serverName: 'github' };await prepareRefresh(scope);diff --git a/config/flush-cache.js b/config/flush-cache.jsindex 05cf23c720..5ef5004e39 100644--- a/config/flush-cache.js+++ b/config/flush-cache.js@@ -158,23 +158,22 @@ async function flushRedisCache(dryRun = false, verbose = false) {if (dryRun) {console.log('🔍 [DRY RUN] Would flush Redis cache');try {- let allKeys = [];+ let totalKeys = 0;+ let sampleKeys = [];+ const targets = useCluster ? redis.nodes('master') : [redis];if (useCluster) {- const nodes = redis.nodes('master');- console.log(` Cluster detected: ${nodes.length} master nodes`);- for (const node of nodes) {- const keys = await node.keys('*');- allKeys = allKeys.concat(keys);+ console.log(` Cluster detected: ${targets.length} master nodes`);+ }+ for (const node of targets) {+ totalKeys += await node.dbsize();+ if (verbose && sampleKeys.length < 10) {+ const [, keys] = await node.scan('0', 'COUNT', 100);+ sampleKeys = sampleKeys.concat(keys).slice(0, 10);}- } else {- allKeys = await redis.keys('*');}- console.log(` Would delete ${allKeys.length} keys`);- if (verbose && allKeys.length > 0) {- console.log(- ' Sample keys:',- allKeys.slice(0, 10).join(', ') + (allKeys.length > 10 ? '...' : ''),- );+ console.log(` Would delete ${totalKeys} keys`);+ if (verbose && totalKeys > 0) {+ console.log(' Sample keys:', sampleKeys.join(', ') + (totalKeys > 10 ? '...' : ''));}} catch (error) {console.log(' Could not fetch keys for preview:', error.message);@@ -186,15 +185,9 @@ async function flushRedisCache(dryRun = false, verbose = false) {// Get key count before flushinglet keyCount = 0;try {- if (useCluster) {- const nodes = redis.nodes('master');- for (const node of nodes) {- const keys = await node.keys('*');- keyCount += keys.length;- }- } else {- const keys = await redis.keys('*');- keyCount = keys.length;+ const targets = useCluster ? redis.nodes('master') : [redis];+ for (const node of targets) {+ keyCount += await node.dbsize();}} catch (_error) {// Continue with flush even if we can't count keysdiff --git a/e2e/specs/mock/scenarios/stream-disconnect.spec.ts b/e2e/specs/mock/scenarios/stream-disconnect.spec.tsnew file mode 100644index 0000000000..32a9cd97ad--- /dev/null+++ b/e2e/specs/mock/scenarios/stream-disconnect.spec.ts@@ -0,0 +1,123 @@+import { expect, test } from '@playwright/test';+import type { APIRequestContext, Page } from '@playwright/test';+import { randomUUID } from 'node:crypto';+import { getE2EUser } from '../../../setup/user';+import { deleteConversations, deleteMessagesByConversation, withMongo } from '../db';+import {+ MOCK_ENDPOINTS,+ NEW_CHAT_PATH,+ messagesView,+ selectMockEndpoint,+ sendMessage,+} from '../helpers';++async function login(request: APIRequestContext) {+ const { email, password } = getE2EUser();+ const response = await request.post('/api/auth/login', { data: { email, password } });+ expect(response.ok()).toBeTruthy();+ const { token } = (await response.json()) as { token: string };+ return token;+}++/** Starts a slow scripted reply and returns its conversation once the reply is streaming. */+async function startSlowReply(page: Page, prompt: string) {+ await page.goto(NEW_CHAT_PATH);+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);+ const run = await sendMessage(page, prompt);+ expect(run.ok()).toBeTruthy();+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15_000 });++ let conversationId = '';+ await expect+ .poll(+ async () => {+ const row = await withMongo((db) =>+ db.collection('messages').findOne({ text: prompt, isCreatedByUser: true }),+ );+ conversationId = (row?.conversationId as string | undefined) ?? '';+ return conversationId;+ },+ { timeout: 10_000 },+ )+ .not.toBe('');+ return conversationId;+}++async function cleanup(conversationId: string) {+ if (!conversationId) {+ return;+ }+ await deleteMessagesByConversation([conversationId]);+ await deleteConversations([conversationId]);+}++test.describe('generation streams and client disconnects', () => {+ test('a running generation is listed as active for its owner @scenario:running-generation-listed-as-active', async ({+ page,+ playwright,+ baseURL,+ }) => {+ test.setTimeout(120_000);+ const api = await playwright.request.newContext({ baseURL });+ let conversationId = '';+ try {+ const token = await login(api);+ conversationId = await startSlowReply(+ page,+ `E2E_SLOW_REPLY:active-${randomUUID().slice(0, 8)}`,+ );++ const response = await api.get('/api/agents/chat/active', {+ headers: { Authorization: `Bearer ${token}` },+ });+ expect(response.ok()).toBeTruthy();+ const body = (await response.json()) as { activeJobIds: string[] };+ expect(body.activeJobIds).toContain(conversationId);+ } finally {+ await api.dispose();+ await cleanup(conversationId);+ }+ });++ test('a client leaving mid-stream does not stop the server or lose the reply @scenario:client-disconnect-keeps-stream-and-server-alive', async ({+ page,+ playwright,+ baseURL,+ }) => {+ test.setTimeout(120_000);+ const api = await playwright.request.newContext({ baseURL });+ let conversationId = '';+ try {+ const token = await login(api);+ conversationId = await startSlowReply(+ page,+ `E2E_SLOW_REPLY:leave-${randomUUID().slice(0, 8)}`,+ );++ /* Dropping the page closes the SSE connection while chunks are still being written. */+ await page.goto('about:blank');++ const health = await api.get('/api/config', { failOnStatusCode: false });+ expect(health.ok()).toBeTruthy();++ await expect+ .poll(+ async () => {+ const response = await api.get('/api/agents/chat/active', {+ headers: { Authorization: `Bearer ${token}` },+ });+ const body = (await response.json()) as { activeJobIds: string[] };+ return body.activeJobIds.includes(conversationId);+ },+ { timeout: 60_000 },+ )+ .toBe(false);++ await page.goto(`/c/${conversationId}`);+ await expect(messagesView(page).getByText('chunk-159')).toBeVisible({ timeout: 20_000 });+ } finally {+ await api.dispose();+ await cleanup(conversationId);+ }+ });+});diff --git a/librechat.example.yaml b/librechat.example.yamlindex 7e747b02fb..85a878b3fe 100644--- a/librechat.example.yaml+++ b/librechat.example.yaml@@ -557,6 +557,10 @@ registration:# moderationMs: 10000 # the OpenAI moderation check before a chat turn# imageFetchMs: 30000 # fetching a stored image to encode it for a provider+# Newest violations kept per user in the general violation log.+# violationLog:+# maxEntries: 100+# rateLimits:# # Per-user MCP App browser-route limits. Each limit uses a fixed one-minute window.# mcpApps:@@ -653,6 +657,7 @@ actions:# # New MCP Apps require explicit opt-in. If omitted, legacy inline HTML remains enabled;# # false disables both MCP Apps and legacy inline HTML.# apps: true+# oauthFailureRetentionMs: 86400000 # how long a failed OAuth reconnection is remembered# allowedDomains:# - 'host.docker.internal' # Docker host access (required for Docker setups)# - 'localhost' # Local developmentdiff --git a/packages/api/src/agents/memory.ts b/packages/api/src/agents/memory.tsindex 6acc1a0a95..22a74b441f 100644--- a/packages/api/src/agents/memory.ts+++ b/packages/api/src/agents/memory.ts@@ -36,7 +36,7 @@ import type { BaseMessage, ToolMessage } from '@librechat/agents/langchain/messaimport type { DynamicStructuredTool } from '@librechat/agents/langchain/tools';import type { Response as ServerResponse } from 'express';import type { ServerRequest, RunLLMConfig } from '~/types';-import { resolveConfigHeaders, createSafeUser, getSafeErrorMetadata } from '~/utils';+import { resolveConfigHeaders, createSafeUser, getSafeErrorMetadata, writeGuarded } from '~/utils';import { contentFilterModelBoundBlockResponse } from '~/middleware/contentFilter';import { extractMemoryContent } from '~/protection/adapters/submissions';import { assertModelBoundContent } from '~/middleware/modelBoundContent';@@ -1197,7 +1197,7 @@ async function handleMemoryArtifact({{ expectedCreatedAt: jobCreatedAt },);} else {- res.write(`event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);+ writeGuarded(res, `event: attachment\ndata: ${JSON.stringify(attachment)}\n\n`);}return attachment;}diff --git a/packages/api/src/agents/openai/handlers.ts b/packages/api/src/agents/openai/handlers.tsindex 550c14b0f7..01c9b500dd 100644--- a/packages/api/src/agents/openai/handlers.ts+++ b/packages/api/src/agents/openai/handlers.ts@@ -18,6 +18,7 @@ import type { UsageMetadata } from '~/stream/interfaces/IJobStore';import type { ToolExecuteOptions } from '~/agents/handlers';import { createToolExecuteHandler } from '~/agents/handlers';import { aggregateCollectedUsage } from '../usage';+import { writeGuarded } from '~/utils/events';/*** Create a chat completion chunk in OpenAI format@@ -47,11 +48,13 @@ export function createChunk(/** One synchronous output frame; HTTP connection and lifecycle stay with the caller. */export interface OpenAIWriter {write(frame: string): boolean | void;+ writableEnded?: boolean;+ destroyed?: boolean;}/** Write one Chat Completions frame through a caller-owned transport. */export function writeSSE(writer: OpenAIWriter, data: ChatCompletionChunk | string): void {- writer.write(`data: ${typeof data === 'string' ? data : JSON.stringify(data)}\n\n`);+ writeGuarded(writer, `data: ${typeof data === 'string' ? data : JSON.stringify(data)}\n\n`);}/**diff --git a/packages/api/src/agents/responses/handlers.ts b/packages/api/src/agents/responses/handlers.tsindex 44deca90ac..fc0977ad48 100644--- a/packages/api/src/agents/responses/handlers.ts+++ b/packages/api/src/agents/responses/handlers.ts@@ -20,6 +20,7 @@ import type {ResponseStatus,Usage,} from './types';+import { writeGuarded } from '~/utils/events';/* =============================================================================* RESPONSE TRACKER@@ -102,15 +103,15 @@ export function createResponseTracker(): ResponseTracker {* The `event:` field matches the `type` in the data payload.*/export function writeEvent(res: ServerResponse, event: ResponseEvent): void {- res.write(`event: ${event.type}\n`);- res.write(`data: ${JSON.stringify(event)}\n\n`);+ writeGuarded(res, `event: ${event.type}\n`);+ writeGuarded(res, `data: ${JSON.stringify(event)}\n\n`);}/*** Write the terminal [DONE] event*/export function writeDone(res: ServerResponse): void {- res.write('data: [DONE]\n\n');+ writeGuarded(res, 'data: [DONE]\n\n');}/* =============================================================================diff --git a/packages/api/src/cache/index.ts b/packages/api/src/cache/index.tsindex 1ddba95f7a..effa58d4c1 100644--- a/packages/api/src/cache/index.ts+++ b/packages/api/src/cache/index.ts@@ -9,3 +9,4 @@ export * from './redisUtils';export * from './redisTelemetry';export * from './audit';export * from './lock';+export * from './violation';diff --git a/packages/api/src/cache/violation.spec.ts b/packages/api/src/cache/violation.spec.tsnew file mode 100644index 0000000000..09b21d4b37--- /dev/null+++ b/packages/api/src/cache/violation.spec.ts@@ -0,0 +1,48 @@+import {+ incrementViolationCount,+ appendViolationLog,+ MAX_VIOLATION_LOG_ENTRIES,+} from './violation';++const makeStore = <T>() => {+ const map = new Map<string, T>();+ return {+ get: async (key: string) => {+ const value = map.get(key);+ await new Promise((resolve) => setTimeout(resolve, 1));+ return value;+ },+ set: async (key: string, value: T) => {+ map.set(key, value);+ },+ };+};++describe('violation helpers', () => {+ it('counts concurrent violations without undercounting', async () => {+ const store = makeStore<number | string>();+ const results = await Promise.all(+ Array.from({ length: 10 }, () => incrementViolationCount(store, 'u', 2)),+ );+ expect(Math.max(...results.map((r) => r.count))).toBe(20);+ expect(results[0].prevCount).toBe(0);+ });++ it('keeps only the newest log entries', async () => {+ const store = makeStore<number[]>();+ for (let i = 0; i < MAX_VIOLATION_LOG_ENTRIES + 5; i++) {+ await appendViolationLog(store, 'u', i);+ }+ const entries = (await store.get('u')) ?? [];+ expect(entries).toHaveLength(MAX_VIOLATION_LOG_ENTRIES);+ expect(entries[entries.length - 1]).toBe(MAX_VIOLATION_LOG_ENTRIES + 4);+ });++ it('honors a configured retention', async () => {+ const store = makeStore<number[]>();+ for (let i = 0; i < 10; i++) {+ await appendViolationLog(store, 'u', i, 3);+ }+ expect(await store.get('u')).toEqual([7, 8, 9]);+ });+});diff --git a/packages/api/src/cache/violation.ts b/packages/api/src/cache/violation.tsnew file mode 100644index 0000000000..3f06f5caca--- /dev/null+++ b/packages/api/src/cache/violation.ts@@ -0,0 +1,47 @@+import { withKeyLock } from '~/utils/keylock';++/** Minimal store surface (Keyv satisfies it). */+export interface ViolationStore<T> {+ get(key: string): Promise<T | undefined | null> | T | undefined | null;+ set(key: string, value: T, ttl?: number): Promise<unknown> | unknown;+}++/**+ * Default for the newest entries kept per user in the GENERAL violation log, overridable+ * by `violationLog.maxEntries`. The log is write-only file storage that is rewritten whole+ * on every append, so it must not grow forever.+ */+export const MAX_VIOLATION_LOG_ENTRIES = 100;++/**+ * Adds `score` to a user's counter. Keyv exposes no atomic increment (stored values are+ * serialized envelopes, so a backend INCR cannot read them), so concurrent updates within+ * this process are serialized per key instead; multi-replica races remain possible.+ * @returns the counter before and after the update.+ */+export function incrementViolationCount(+ store: ViolationStore<number | string>,+ key: string,+ score: number | string = 1,+): Promise<{ prevCount: number; count: number }> {+ return withKeyLock(`violation:count:${key}`, async () => {+ const prevCount = +((await store.get(key)) ?? 0);+ const count = prevCount + +score;+ await store.set(key, count);+ return { prevCount, count };+ });+}++/** Appends an entry to a user's violation log, keeping only the newest entries. */+export function appendViolationLog<T>(+ store: ViolationStore<T[]>,+ key: string,+ entry: T,+ maxEntries: number = MAX_VIOLATION_LOG_ENTRIES,+): Promise<void> {+ return withKeyLock(`violation:log:${key}`, async () => {+ const entries = (await store.get(key)) ?? [];+ entries.push(entry);+ await store.set(key, entries.slice(-maxEntries));+ });+}diff --git a/packages/api/src/index.ts b/packages/api/src/index.tsindex 1da4b371ca..a8cbd98930 100644--- a/packages/api/src/index.ts+++ b/packages/api/src/index.ts@@ -64,6 +64,7 @@ export * from './html';/* OAuth */export * from './oauth';export * from './mcp/oauth/OAuthReconnectionManager';+export * from './mcp/oauth/OAuthReconnectionTracker';/* Crypto */export * from './crypto';/* Flow */diff --git a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.ts b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.tsindex 206fe96ef1..e35ad5815d 100644--- a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.ts+++ b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.test.ts@@ -1,4 +1,7 @@-import { OAuthReconnectionTracker } from './OAuthReconnectionTracker';+import {+ OAuthReconnectionTracker,+ createOAuthReconnectionTracker,+} from './OAuthReconnectionTracker';describe('OAuthReconnectTracker', () => {let tracker: OAuthReconnectionTracker;@@ -547,4 +550,49 @@ describe('OAuthReconnectTracker', () => {expect(tracker.isStillReconnecting(anotherUserId, serverName)).toBe(true);});});++ describe('failed entry retention', () => {+ afterEach(() => {+ jest.useRealTimers();+ });++ it('drops failed entries that were never retried once retention passes', () => {+ jest.useFakeTimers();+ tracker.setFailed(userId, serverName);+ expect(tracker.getStats().usersWithFailedServers).toBe(1);++ jest.advanceTimersByTime(25 * 60 * 60 * 1000);+ tracker.setFailed('other-user', serverName);++ expect(tracker.isFailed(userId, serverName)).toBe(false);+ expect(tracker.getStats().usersWithFailedServers).toBe(1);+ });++ it('honors mcpSettings.oauthFailureRetentionMs', () => {+ jest.useFakeTimers();+ const short = createOAuthReconnectionTracker({+ mcpSettings: { oauthFailureRetentionMs: 60_000 },+ });+ short.setFailed(userId, serverName);+ jest.advanceTimersByTime(2 * 60_000);+ short.setFailed('other-user', serverName);+ expect(short.getStats().usersWithFailedServers).toBe(1);++ const dflt = createOAuthReconnectionTracker(undefined);+ dflt.setFailed(userId, serverName);+ jest.advanceTimersByTime(2 * 60_000);+ dflt.setFailed('other-user', serverName);+ expect(dflt.getStats().usersWithFailedServers).toBe(2);+ });++ it('prunes promptly when retention is shorter than the prune interval', () => {+ jest.useFakeTimers();+ const tiny = new OAuthReconnectionTracker(1_000);+ tiny.setFailed(userId, serverName);+ tiny.setFailed('warm', serverName);+ jest.advanceTimersByTime(2_000);+ tiny.setFailed('other-user', serverName);+ expect(tiny.getStats().usersWithFailedServers).toBe(1);+ });+ });});diff --git a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.ts b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.tsindex 504ea7d43a..0faba0fea6 100644--- a/packages/api/src/mcp/oauth/OAuthReconnectionTracker.ts+++ b/packages/api/src/mcp/oauth/OAuthReconnectionTracker.ts@@ -5,7 +5,21 @@ interface FailedMeta {const COOLDOWN_SCHEDULE_MS = [5 * 60 * 1000, 10 * 60 * 1000, 20 * 60 * 1000, 30 * 60 * 1000];+/** Failed entries untouched this long are dropped so pairs that never retry do not accumulate. */+export const DEFAULT_FAILED_RETENTION_MS: number = 24 * 60 * 60 * 1000;+const FAILED_PRUNE_INTERVAL_MS = 60 * 1000;++/** Builds the tracker with the `mcpSettings.oauthFailureRetentionMs` budget, else the default. */+export function createOAuthReconnectionTracker(+ appConfig?: { mcpSettings?: { oauthFailureRetentionMs?: number } | null } | null,+): OAuthReconnectionTracker {+ return new OAuthReconnectionTracker(appConfig?.mcpSettings?.oauthFailureRetentionMs);+}+export class OAuthReconnectionTracker {+ constructor(private readonly failedRetentionMs: number = DEFAULT_FAILED_RETENTION_MS) {}++ private lastFailedPrune = 0;private failedMeta: Map<string, Map<string, FailedMeta>> = new Map();/** Map of userId -> Set of serverNames that are actively reconnecting */private active: Map<string, Set<string>> = new Map();@@ -63,7 +77,25 @@ export class OAuthReconnectionTracker {return false;}+ private pruneFailed(now: number): void {+ if (now - this.lastFailedPrune < Math.min(FAILED_PRUNE_INTERVAL_MS, this.failedRetentionMs)) {+ return;+ }+ this.lastFailedPrune = now;+ for (const [userId, userMap] of this.failedMeta) {+ for (const [serverName, meta] of userMap) {+ if (now - meta.lastFailedAt >= this.failedRetentionMs) {+ userMap.delete(serverName);+ }+ }+ if (userMap.size === 0) {+ this.failedMeta.delete(userId);+ }+ }+ }+public setFailed(userId: string, serverName: string): void {+ this.pruneFailed(Date.now());if (!this.failedMeta.has(userId)) {this.failedMeta.set(userId, new Map());}diff --git a/packages/api/src/middleware/concurrency.ts b/packages/api/src/middleware/concurrency.tsindex ad745a827a..cf18dbd663 100644--- a/packages/api/src/middleware/concurrency.ts+++ b/packages/api/src/middleware/concurrency.ts@@ -1,7 +1,7 @@import { logger } from '@librechat/data-schemas';import { CacheKeys, Time, ViolationTypes } from 'librechat-data-provider';import { standardCache, cacheConfig, instrumentIORedisClient, ioredisClient } from '~/cache';-import { isEnabled, math } from '~/utils';+import { isEnabled, math, withKeyLock } from '~/utils';const { USE_REDIS } = cacheConfig;@@ -95,7 +95,7 @@ export interface ViolationInfo {* the actual request processing lifecycle.** When Redis is available, uses atomic INCR to prevent race conditions.- * Falls back to non-atomic get/set for in-memory cache.+ * Falls back to per-key serialized get/set for in-memory cache.** @param userId - The user's ID* @returns Object with `allowed` (boolean), `pendingRequests` (current count), and `limit`@@ -151,21 +151,23 @@ export async function checkAndIncrementPendingRequest(}const key = buildMemoryKey(userId);- const pendingRequests = +((await cache.get(key)) ?? 0);+ return withKeyLock(`pending:${key}`, async () => {+ const pendingRequests = +((await cache.get(key)) ?? 0);++ if (pendingRequests >= limit) {+ logger.debug(+ `[concurrency] User ${userId} exceeded concurrent limit: ${pendingRequests}/${limit}`,+ );+ return { allowed: false, pendingRequests, limit };+ }- if (pendingRequests >= limit) {+ await cache.set(key, pendingRequests + 1, Time.ONE_MINUTE);logger.debug(- `[concurrency] User ${userId} exceeded concurrent limit: ${pendingRequests}/${limit}`,+ `[concurrency] User ${userId} incremented pending requests: ${pendingRequests + 1}/${limit}`,);- return { allowed: false, pendingRequests, limit };- }-- await cache.set(key, pendingRequests + 1, Time.ONE_MINUTE);- logger.debug(- `[concurrency] User ${userId} incremented pending requests: ${pendingRequests + 1}/${limit}`,- );- return { allowed: true, pendingRequests: pendingRequests + 1, limit };+ return { allowed: true, pendingRequests: pendingRequests + 1, limit };+ });}/**@@ -176,7 +178,7 @@ export async function checkAndIncrementPendingRequest(* operation that should not interrupt the main flow if cache operations fail.** When Redis is available, uses atomic DECR to prevent race conditions.- * Falls back to non-atomic get/set for in-memory cache.+ * Falls back to per-key serialized get/set for in-memory cache.** @param userId - The user's ID*/@@ -215,15 +217,19 @@ export async function decrementPendingRequest(userId: string): Promise<void> {}const key = buildMemoryKey(userId);- const currentReq = +((await cache.get(key)) ?? 0);-- if (currentReq >= 1) {- await cache.set(key, currentReq - 1, Time.ONE_MINUTE);- logger.debug(`[concurrency] User ${userId} decremented pending requests: ${currentReq - 1}`);- } else {- await cache.delete(key);- logger.debug(`[concurrency] User ${userId} pending requests cleared (was ${currentReq})`);- }+ await withKeyLock(`pending:${key}`, async () => {+ const currentReq = +((await cache.get(key)) ?? 0);++ if (currentReq >= 1) {+ await cache.set(key, currentReq - 1, Time.ONE_MINUTE);+ logger.debug(+ `[concurrency] User ${userId} decremented pending requests: ${currentReq - 1}`,+ );+ } else {+ await cache.delete(key);+ logger.debug(`[concurrency] User ${userId} pending requests cleared (was ${currentReq})`);+ }+ });} catch (error) {logger.error('[concurrency] Error decrementing pending request:', error);}diff --git a/packages/api/src/skills/handlers.ts b/packages/api/src/skills/handlers.tsindex af667db212..fe1f9db43c 100644--- a/packages/api/src/skills/handlers.ts+++ b/packages/api/src/skills/handlers.ts@@ -716,6 +716,12 @@ export function createSkillsHandlers(deps: SkillsHandlersDeps): {res.destroy();}});+ /** Release the upstream when the client leaves before the body is flushed. */+ res.once('close', () => {+ if ('destroy' in stream && typeof stream.destroy === 'function') {+ stream.destroy();+ }+ });stream.pipe(res);return;}diff --git a/packages/api/src/stream/__tests__/cleanupInterval.spec.ts b/packages/api/src/stream/__tests__/cleanupInterval.spec.tsnew file mode 100644index 0000000000..b3c7826b6c--- /dev/null+++ b/packages/api/src/stream/__tests__/cleanupInterval.spec.ts@@ -0,0 +1,48 @@+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';++describe('InMemoryJobStore cleanup interval', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ });++ afterEach(() => {+ jest.useRealTimers();+ });++ it('reports a failed cleanup pass instead of leaving an unhandled rejection', async () => {+ const store = new InMemoryJobStore();+ const cleanup = jest.spyOn(store, 'cleanup').mockRejectedValue(new Error('cleanup failed'));+ const unhandled = jest.fn();+ process.on('unhandledRejection', unhandled);++ await store.initialize();+ await jest.advanceTimersByTimeAsync(60000);+ await jest.advanceTimersByTimeAsync(60000);+ await store.destroy();+ process.off('unhandledRejection', unhandled);++ expect(cleanup).toHaveBeenCalledTimes(2);+ expect(unhandled).not.toHaveBeenCalled();+ });++ it('does not start a pass while the previous one is still running', async () => {+ const store = new InMemoryJobStore();+ let finish: (n: number) => void = () => undefined;+ const cleanup = jest.spyOn(store, 'cleanup').mockImplementation(+ () =>+ new Promise<number>((resolve) => {+ finish = resolve;+ }),+ );++ await store.initialize();+ await jest.advanceTimersByTimeAsync(180000);+ expect(cleanup).toHaveBeenCalledTimes(1);++ finish(0);+ await jest.advanceTimersByTimeAsync(60000);+ expect(cleanup).toHaveBeenCalledTimes(2);+ finish(0);+ await store.destroy();+ });+});diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.tsindex a48c2253ea..8045c8e3d2 100644--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts@@ -155,6 +155,7 @@ export class InMemoryJobStore implements IJobStoreV2 {private jobs = new Map<string, SerializableJobData>();private contentState = new Map<string, ContentState>();private cleanupInterval: NodeJS.Timeout | null = null;+ private cleanupRunning = false;private staleGenerationHandler?: (streamId: string, createdAt: number) => void;setStaleGenerationHandler(handler?: (streamId: string, createdAt: number) => void): void {@@ -241,7 +242,17 @@ export class InMemoryJobStore implements IJobStoreV2 {}this.cleanupInterval = setInterval(() => {- this.cleanup();+ if (this.cleanupRunning) {+ return;+ }+ this.cleanupRunning = true;+ this.cleanup()+ .catch((err) => {+ logger.error('[InMemoryJobStore] Cleanup error:', err);+ })+ .finally(() => {+ this.cleanupRunning = false;+ });}, 60000);if (this.cleanupInterval.unref) {diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.tsindex dcf31dc71a..ff96a3b803 100644--- a/packages/api/src/stream/implementations/RedisJobStore.ts+++ b/packages/api/src/stream/implementations/RedisJobStore.ts@@ -1911,6 +1911,7 @@ export class RedisJobStore implements IJobStoreV2 {private redis: Redis | Cluster;private cleanupInterval: NodeJS.Timeout | null = null;+ private cleanupRunning = false;private cleanupMembershipReady?: Promise<void>;private ttl: typeof DEFAULT_TTL;/** Coalescable chunk appends awaiting their window flush, per stream */@@ -1970,9 +1971,17 @@ export class RedisJobStore implements IJobStoreV2 {// Start periodic cleanupthis.cleanupInterval = setInterval(() => {- this.cleanup().catch((err) => {- logger.error('[RedisJobStore] Cleanup error:', err);- });+ if (this.cleanupRunning) {+ return;+ }+ this.cleanupRunning = true;+ this.cleanup()+ .catch((err) => {+ logger.error('[RedisJobStore] Cleanup error:', err);+ })+ .finally(() => {+ this.cleanupRunning = false;+ });}, this.cleanupIntervalMs);if (this.cleanupInterval.unref) {@@ -3978,8 +3987,10 @@ export class RedisJobStore implements IJobStoreV2 {const activeIds: string[] = [];let healed = 0;- for (const streamId of trackedIds) {- const job = await this.getJob(streamId);+ const trackedJobs = await Promise.all(trackedIds.map((streamId) => this.getJob(streamId)));++ for (const [index, streamId] of trackedIds.entries()) {+ const job = trackedJobs[index];// Include running jobs and jobs paused for human review (e.g. tool approval).// A pending-approval job still occupies the user's conversation slot — but// only while its prompt is live: a past-`expiresAt` approval no longerdiff --git a/packages/api/src/utils/events.spec.ts b/packages/api/src/utils/events.spec.tsnew file mode 100644index 0000000000..436db53a55--- /dev/null+++ b/packages/api/src/utils/events.spec.ts@@ -0,0 +1,43 @@+import { writeGuarded, sendEvent, handleError } from './events';++type FakeRes = {+ write: jest.Mock;+ end: jest.Mock;+ writableEnded?: boolean;+ destroyed?: boolean;+};++const makeRes = (overrides: Partial<FakeRes> = {}): FakeRes => ({+ write: jest.fn(),+ end: jest.fn(),+ ...overrides,+});++describe('writeGuarded', () => {+ it('writes to a live response', () => {+ const res = makeRes();+ expect(writeGuarded(res, 'x')).toBe(true);+ expect(res.write).toHaveBeenCalledWith('x');+ });++ it('skips an ended response', () => {+ const res = makeRes({ writableEnded: true });+ expect(writeGuarded(res, 'x')).toBe(false);+ expect(res.write).not.toHaveBeenCalled();+ });++ it('skips a destroyed response', () => {+ const res = makeRes({ destroyed: true });+ expect(writeGuarded(res, 'x')).toBe(false);+ expect(res.write).not.toHaveBeenCalled();+ });+});++describe('sendEvent and handleError', () => {+ it('does not write after the response ended', () => {+ const res = makeRes({ writableEnded: true });+ sendEvent(res as never, { event: 'x' } as never);+ handleError(res as never, 'boom');+ expect(res.write).not.toHaveBeenCalled();+ });+});diff --git a/packages/api/src/utils/events.ts b/packages/api/src/utils/events.tsindex e084e631f5..7f87ab8108 100644--- a/packages/api/src/utils/events.ts+++ b/packages/api/src/utils/events.ts@@ -1,6 +1,26 @@import type { Response as ServerResponse } from 'express';import type { ServerSentEvent } from '~/types';+/** Minimal writable transport; an Express response satisfies it. */+export interface GuardedWriter {+ write(chunk: string): boolean | void;+ writableEnded?: boolean;+ destroyed?: boolean;+}++/**+ * Writes a chunk unless the response has already ended or been destroyed+ * (client disconnect), which would otherwise raise `ERR_STREAM_WRITE_AFTER_END`.+ * @returns whether the chunk was handed to the transport.+ */+export function writeGuarded(res: GuardedWriter, chunk: string): boolean {+ if (res.writableEnded === true || res.destroyed === true) {+ return false;+ }+ res.write(chunk);+ return true;+}+/*** Sends a Server-Sent Event to the client.* Empty-string StreamEvent data is silently dropped.@@ -9,7 +29,7 @@ export function sendEvent(res: ServerResponse, event: ServerSentEvent): void {if ('data' in event && typeof event.data === 'string' && event.data.length === 0) {return;}- res.write(`event: message\ndata: ${JSON.stringify(event)}\n\n`);+ writeGuarded(res, `event: message\ndata: ${JSON.stringify(event)}\n\n`);}/**@@ -18,6 +38,6 @@ export function sendEvent(res: ServerResponse, event: ServerSentEvent): void {* @param message - The error message.*/export function handleError(res: ServerResponse, message: string): void {- res.write(`event: error\ndata: ${JSON.stringify(message)}\n\n`);+ writeGuarded(res, `event: error\ndata: ${JSON.stringify(message)}\n\n`);res.end();}diff --git a/packages/api/src/utils/index.ts b/packages/api/src/utils/index.tsindex c920eaf193..9e2fc052cd 100644--- a/packages/api/src/utils/index.ts+++ b/packages/api/src/utils/index.ts@@ -39,3 +39,4 @@ export * from './url';export * from './message';export * from './quotes';export * from './tracing';+export * from './keylock';diff --git a/packages/api/src/utils/keylock.spec.ts b/packages/api/src/utils/keylock.spec.tsnew file mode 100644index 0000000000..30bcb8862a--- /dev/null+++ b/packages/api/src/utils/keylock.spec.ts@@ -0,0 +1,64 @@+import { withKeyLock, createSingleFlight, raceWithTimeout } from './keylock';++describe('withKeyLock', () => {+ it('serializes read-modify-write cycles on one key without losing updates', async () => {+ let value = 0;+ const bump = () =>+ withKeyLock('k', async () => {+ const read = value;+ await new Promise((resolve) => setTimeout(resolve, 1));+ value = read + 1;+ });+ await Promise.all(Array.from({ length: 20 }, bump));+ expect(value).toBe(20);+ });++ it('keeps running later tasks after one rejects', async () => {+ await expect(withKeyLock('r', () => Promise.reject(new Error('x')))).rejects.toThrow('x');+ await expect(withKeyLock('r', async () => 'ok')).resolves.toBe('ok');+ });+});++describe('createSingleFlight', () => {+ it('skips overlapping runs and reports rejections', async () => {+ let release: () => void = () => undefined;+ const task = jest.fn(+ () =>+ new Promise<void>((resolve, reject) => {+ release = () => reject(new Error('fail'));+ void resolve;+ }),+ );+ const onError = jest.fn();+ const tick = createSingleFlight(task, { error: onError }, '[Test] sweep');+ tick();+ tick();+ expect(task).toHaveBeenCalledTimes(1);+ release();+ await new Promise((resolve) => setImmediate(resolve));+ expect(onError).toHaveBeenCalledTimes(1);+ expect(onError).toHaveBeenCalledWith('[Test] sweep failed', expect.any(Error));+ tick();+ expect(task).toHaveBeenCalledTimes(2);+ });+});++describe('raceWithTimeout', () => {+ afterEach(() => jest.useRealTimers());++ it('returns the work result and leaves no timer pending', async () => {+ jest.useFakeTimers();+ const log = { error: jest.fn() };+ await expect(raceWithTimeout(Promise.resolve('ok'), 1000, log, '[T]')).resolves.toBe('ok');+ expect(jest.getTimerCount()).toBe(0);+ });++ it('logs and resolves undefined when the work outlives the budget', async () => {+ jest.useFakeTimers();+ const log = { error: jest.fn() };+ const result = raceWithTimeout(new Promise<string>(() => undefined), 1000, log, '[T]');+ jest.advanceTimersByTime(1000);+ await expect(result).resolves.toBeUndefined();+ expect(log.error).toHaveBeenCalledTimes(1);+ });+});diff --git a/packages/api/src/utils/keylock.ts b/packages/api/src/utils/keylock.tsnew file mode 100644index 0000000000..2207ed5e80--- /dev/null+++ b/packages/api/src/utils/keylock.ts@@ -0,0 +1,66 @@+const tails = new Map<string, Promise<unknown>>();++/**+ * Runs `fn` after every earlier task queued under the same `key` has settled, so a+ * read-modify-write against a store without atomic operations cannot interleave+ * with another one in this process. It does not coordinate separate processes.+ */+export function withKeyLock<T>(key: string, fn: () => Promise<T>): Promise<T> {+ const previous = tails.get(key) ?? Promise.resolve();+ const run = previous.then(fn, fn);+ const tail = run.catch(() => undefined);+ tails.set(key, tail);+ void tail.then(() => {+ if (tails.get(key) === tail) {+ tails.delete(key);+ }+ });+ return run;+}++/**+ * Wraps an async task for use in a timer so a slow run is never overlapped by the+ * next tick and a rejection is reported instead of becoming an unhandled rejection.+ */+export function createSingleFlight(+ task: () => Promise<unknown>,+ log: { error: (message: string, error: unknown) => void },+ label: string,+): () => void {+ let running = false;+ return () => {+ if (running) {+ return;+ }+ running = true;+ task()+ .catch((error) => log.error(`${label} failed`, error))+ .finally(() => {+ running = false;+ });+ };+}++/**+ * Resolves with `work`, or with `undefined` after `ms` (logging the timeout). The timer is+ * always cleared, so a settled race never leaves it pending.+ */+export async function raceWithTimeout<T>(+ work: Promise<T>,+ ms: number,+ log: { error: (message: string, error: unknown) => void },+ label: string,+): Promise<T | undefined> {+ let timer: NodeJS.Timeout | undefined;+ const timeout = new Promise<undefined>((resolve) => {+ timer = setTimeout(() => {+ log.error(`${label} error:`, new Error(`${label} timeout`));+ resolve(undefined);+ }, ms);+ });+ try {+ return await Promise.race([work, timeout]);+ } finally {+ clearTimeout(timer);+ }+}diff --git a/packages/api/src/utils/tokenizer.ts b/packages/api/src/utils/tokenizer.tsindex 83148ba86d..cd5782f244 100644--- a/packages/api/src/utils/tokenizer.ts+++ b/packages/api/src/utils/tokenizer.ts@@ -62,7 +62,21 @@ class Tokenizer {: await import('ai-tokenizer/encoding/o200k_base');this.tokenizersCache[encoding] = new AiTokenizer(data);})();- return this.loadingPromises[encoding];+ const loading = this.loadingPromises[encoding];+ /** A failed load must not stay cached, or every later call replays the rejection. */+ loading.catch(() => {+ if (this.loadingPromises[encoding] === loading) {+ delete this.loadingPromises[encoding];+ }+ });+ return loading;+ }++ /** Fire-and-forget warm-up; a load failure is logged instead of surfacing as an unhandled rejection. */+ private warmEncoding(encoding: EncodingName): void {+ this.initEncoding(encoding).catch((error) => {+ logger.error(`[Tokenizer] Failed to load encoding ${encoding}`, error);+ });}/** Returns an initialized, deterministic budgeting counter that throws on tokenizer failure. */@@ -85,7 +99,7 @@ class Tokenizer {getTokenCount(text: string, encoding: EncodingName = 'o200k_base'): number {const tokenizer = this.tokenizersCache[encoding];if (!tokenizer) {- this.initEncoding(encoding);+ this.warmEncoding(encoding);return estimateUnavailableTokenCount(text);}try {@@ -115,7 +129,7 @@ class Tokenizer {}const tokenizer = this.tokenizersCache[encoding];if (!tokenizer) {- void this.initEncoding(encoding);+ this.warmEncoding(encoding);return undefined;}try {diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 809d6bae69..6288605599 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -181,6 +181,7 @@ export const DEPLOYMENT_ONLY_CONFIG_PATHS: Readonly<Record<string, DeploymentOnl'endpoints.agents.backgroundTasks.completionResultBatchSize': 'startup','endpoints.agents.backgroundTasks.completionReceiptBatching': 'startup','endpoints.agents.toolApproval.hooks': 'operator',+ 'mcpSettings.oauthFailureRetentionMs': 'startup','mcpSettings.catalogRecovery.maxStateEntries': 'startup','mcpSettings.catalogRecovery.maxDetachedDiscoveries': 'startup','mcpSettings.catalogRecovery.authorizationFenceRetryIntervalMs': 'startup',@@ -2714,6 +2715,12 @@ export const outboundTimeoutsSchema = z.object({imageFetchMs: outboundTimeoutMs,});+/** Retention for the write-only GENERAL violation log kept per user. */+export const violationLogSchema = z.object({+ /** Newest entries kept per user; older ones are dropped on the next append. */+ maxEntries: z.number().int().positive().optional(),+});+export type TOutboundTimeouts = z.infer<typeof outboundTimeoutsSchema>;export const rateLimitSchema = z.object({@@ -4525,6 +4532,14 @@ export const configSchema = z.object({.object({allowedDomains: z.array(z.string()).optional(),allowedAddresses: allowedAddressesSchema,+ /** How long a failed OAuth reconnection is remembered before its attempt count resets.+ * Defaults to 24 hours. */+ oauthFailureRetentionMs: z+ .number()+ .int()+ .positive()+ .max(30 * 24 * 60 * 60_000)+ .optional(),catalogRecovery: z.object({discoveryBackoffMs: z@@ -4642,6 +4657,7 @@ export const configSchema = z.object({}).optional(),timeouts: outboundTimeoutsSchema.optional(),+ violationLog: violationLogSchema.optional(),fileConfig: fileConfigSchema.optional(),modelSpecs: specsConfigSchema.optional(),filters: filtersConfigSchema.optional(),diff --git a/packages/data-schemas/src/app/service.ts b/packages/data-schemas/src/app/service.tsindex 160c9cdd1e..67eb9d0191 100644--- a/packages/data-schemas/src/app/service.ts+++ b/packages/data-schemas/src/app/service.ts@@ -222,6 +222,7 @@ export const AppService = async (params?: {webSearch,githubCompare: config.githubCompare,timeouts: config.timeouts,+ violationLog: config.violationLog,mcpSettings,mcpAppSandbox,fileStrategy,diff --git a/packages/data-schemas/src/types/app.ts b/packages/data-schemas/src/types/app.tsindex e74870d1d3..5b890c6bba 100644--- a/packages/data-schemas/src/types/app.ts+++ b/packages/data-schemas/src/types/app.ts@@ -71,6 +71,8 @@ export interface AppConfig {githubCompare?: TCustomConfig['githubCompare'];/** Outbound call budgets; unset keys fall back to the server defaults */timeouts?: TCustomConfig['timeouts'];+ /** Violation log retention; unset keeps the server default */+ violationLog?: TCustomConfig['violationLog'];skillDeletion?: {finalizationEnabled?: boolean;sweepIntervalMs?: number;