Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔕 fix: Log Lost Generation Predecessor Fences as Warnings (#16689)
Danny Avila2529b22a66cd
diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.jsindex 18a8828497..8955e99e7d 100644--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js@@ -82,7 +82,7 @@ jest.mock('@librechat/api', () => ({resolvePersistableCodeEnvironmentDecision: (...args) =>jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args),getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata,- getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText,+ logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,startAgentProjectContextResolution:jest.requireActual('@librechat/api').startAgentProjectContextResolution,assertChatProjectInstructions: jest.requireActual('@librechat/api').assertChatProjectInstructions,diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.jsindex c8dcd02f81..5211dd610c 100644--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js@@ -308,7 +308,7 @@ jest.mock('@librechat/api', () => ({resolvePersistableCodeEnvironmentDecision: (...args) =>jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args),getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata,- getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText,+ logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,resolveFailedTurnContent: jest.requireActual('@librechat/api').resolveFailedTurnContent,getFailedTurnTraceFields: (...args) => mockGetFailedTurnTraceFields(...args),startAgentProjectContextResolution:@@ -1385,6 +1385,65 @@ describe('ResumableAgentController resume metadata', () => {);});+ it('defers a continuation that loses the create fence to a sibling and logs a warning', async () => {+ mockGetMessages.mockResolvedValue([{ _id: 'persisted-parent' }]);+ const { JobPredecessorMismatchError } = jest.requireActual('@librechat/api');+ mockGenerationJobManager.createJob.mockRejectedValue(+ new JobPredecessorMismatchError({+ createdAt: 2000,+ active: true,+ verified: true,+ status: 'running',+ conversationId: 'conversation-123',+ }),+ );+ const req = {+ _isAgentTrigger: true,+ user: { id: 'user-123' },+ body: {+ text: 'A background tool task is waiting to complete.',+ messageId: 'wakeup-user-message',+ parentMessageId: 'persisted-response_',+ conversationId: 'conversation-123',+ clientRequestId: 'trigger_sibling_wakeup',+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },+ },+ config: {},+ };+ const res = createResumableResponse();+ const initializeClient = jest.fn();++ await AgentController(req, res, jest.fn(), initializeClient, null);++ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(+ 'conversation-123',+ 'user-123',+ 'conversation-123',+ expect.objectContaining({ rejectActivePredecessor: true }),+ );+ expect(res.set).toHaveBeenCalledWith('Retry-After', '1');+ expect(res.status).toHaveBeenCalledWith(409);+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'PARENT_NOT_READY' }));+ expect(mockLogger.warn).toHaveBeenCalledWith(+ '[ResumableAgentController] Generation predecessor changed before creation',+ {+ streamId: 'conversation-123',+ conversationId: 'conversation-123',+ continuation: true,+ active: true,+ currentCreatedAt: 2000,+ currentStatus: 'running',+ },+ );+ expect(+ mockLogger.error.mock.calls.some((call) =>+ String(call[0]).startsWith('[ResumableAgentController] Initialization error:'),+ ),+ ).toBe(false);+ expect(initializeClient).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();+ });+it('keeps request-scoped MCP connections until resumable initialization finishes', async () => {const conversationId = 'conversation-123';const disconnect = jest.fn().mockResolvedValue(undefined);@@ -3055,6 +3114,23 @@ describe('ResumableAgentController resume metadata', () => {expect.objectContaining(DEFAULT_OWNED_CLAIM),);expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');+ expect(mockLogger.warn).toHaveBeenCalledWith(+ '[ResumableAgentController] Generation predecessor changed before creation',+ {+ streamId: 'conversation-123',+ conversationId: 'conversation-123',+ continuation: false,+ active: true,+ expectedPredecessorCreatedAt: 1000,+ currentCreatedAt: 2000,+ currentStatus: 'running',+ },+ );+ expect(+ mockLogger.error.mock.calls.some((call) =>+ String(call[0]).startsWith('[ResumableAgentController] Initialization error:'),+ ),+ ).toBe(false);});it('returns a finite fail-closed mismatch when predecessor evidence expired', async () => {diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex f378f1ad83..9801525b40 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -39,7 +39,7 @@ const {getAttachmentTitleText,createMCPRuntimeRequestBody,resolveRunCodeWorkspaces,- getSafeErrorText,+ logGenerationStartFailure,isAgentEventRetentionActive,createAgentEventActorTurn,createAgentEventActorDetachedActionLifecycle,@@ -3483,7 +3483,12 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit);});} catch (error) {- logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`);+ logGenerationStartFailure(error, {+ streamId,+ conversationId,+ continuation: isTriggerContinuation,+ expectedPredecessorCreatedAt,+ });const initializationFailure = getInitializationFailure(error);const streamStarted = res.headersSent;try {diff --git a/packages/api/src/stream/__tests__/admission.spec.ts b/packages/api/src/stream/__tests__/admission.spec.tsnew file mode 100644index 0000000000..a39475f43c--- /dev/null+++ b/packages/api/src/stream/__tests__/admission.spec.ts@@ -0,0 +1,189 @@+import { logger } from '@librechat/data-schemas';+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';+import { logGenerationStartFailure } from '../admission';++const STARTED = { generationProtocolVersion: 2 as const };++function createContinuationJob(store: InMemoryJobStore, streamId: string) {+ return store.createJob(+ streamId,+ 'owner-1',+ streamId,+ undefined,+ STARTED,+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ 'continuation-create-attempt',+ undefined,+ true,+ );+}++function createConditionalJob(+ store: InMemoryJobStore,+ streamId: string,+ expectedPredecessorCreatedAt: number,+) {+ return store.createJob(+ streamId,+ 'owner-1',+ streamId,+ undefined,+ STARTED,+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ 'conditional-create-attempt',+ expectedPredecessorCreatedAt,+ );+}++async function rejectionOf(promise: Promise<unknown>): Promise<unknown> {+ try {+ await promise;+ } catch (error) {+ return error;+ }+ throw new Error('Expected the create to be rejected');+}++describe('logGenerationStartFailure', () => {+ let store: InMemoryJobStore;+ let warn: jest.SpyInstance;+ let error: jest.SpyInstance;++ beforeEach(() => {+ store = new InMemoryJobStore();+ warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);+ error = jest.spyOn(logger, 'error').mockImplementation(() => logger);+ });++ afterEach(async () => {+ jest.restoreAllMocks();+ await store.destroy();+ });++ it('warns when a sibling continuation already won the conversation', async () => {+ const streamId = 'sibling-wakeup-race';+ const winner = await createContinuationJob(store, streamId);+ const mismatch = await rejectionOf(createContinuationJob(store, streamId));++ logGenerationStartFailure(mismatch, {+ streamId,+ conversationId: streamId,+ continuation: true,+ });++ expect(error).not.toHaveBeenCalled();+ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn).toHaveBeenCalledWith(+ '[ResumableAgentController] Generation predecessor changed before creation',+ {+ streamId,+ conversationId: streamId,+ continuation: true,+ active: true,+ currentCreatedAt: winner.createdAt,+ currentStatus: 'running',+ },+ );+ });++ it('warns with the stale epoch when a queued send lost its fence to a settled turn', async () => {+ const streamId = 'stale-queued-send';+ const settled = await store.createJob(streamId, 'owner-1', streamId, undefined, STARTED);+ await store.transitionStatus(streamId, {+ from: 'running',+ to: 'complete',+ expectCreatedAt: settled.createdAt,+ });+ const expectedPredecessorCreatedAt = settled.createdAt - 1;+ const mismatch = await rejectionOf(+ createConditionalJob(store, streamId, expectedPredecessorCreatedAt),+ );++ logGenerationStartFailure(mismatch, {+ streamId,+ conversationId: streamId,+ continuation: false,+ expectedPredecessorCreatedAt,+ });++ expect(error).not.toHaveBeenCalled();+ expect(warn).toHaveBeenCalledWith(+ '[ResumableAgentController] Generation predecessor changed before creation',+ {+ streamId,+ conversationId: streamId,+ continuation: false,+ active: false,+ expectedPredecessorCreatedAt,+ currentCreatedAt: settled.createdAt,+ currentStatus: 'complete',+ },+ );+ });++ it('treats a code-only mismatch the same way the 409 response does', () => {+ const mismatch = Object.assign(new Error('predecessor changed'), {+ code: 'GENERATION_PREDECESSOR_MISMATCH',+ currentJob: { createdAt: 2000, status: 'requires_action', verified: false },+ });++ logGenerationStartFailure(mismatch, {+ streamId: 'code-only',+ conversationId: 'code-only',+ continuation: false,+ });++ expect(error).not.toHaveBeenCalled();+ expect(warn).toHaveBeenCalledWith(+ '[ResumableAgentController] Generation predecessor changed before creation',+ {+ streamId: 'code-only',+ conversationId: 'code-only',+ continuation: false,+ active: true,+ currentCreatedAt: 2000,+ currentStatus: 'requires_action',+ verified: false,+ },+ );+ });++ it('keeps genuine initialization failures at error level as redacted text', () => {+ const failure = Object.assign(new Error('Agent not found'), { code: 'AGENT_NOT_FOUND' });++ logGenerationStartFailure(failure, {+ streamId: 'genuine-failure',+ conversationId: 'genuine-failure',+ continuation: true,+ });++ expect(warn).not.toHaveBeenCalled();+ expect(error).toHaveBeenCalledTimes(1);+ const [message, ...metadata] = error.mock.calls[0];+ expect(metadata).toEqual([]);+ expect(message).toMatch(+ /^\[ResumableAgentController\] Initialization error: Error: Agent not found/,+ );+ });++ it('logs a non-error rejection at error level', () => {+ logGenerationStartFailure(undefined, {+ streamId: 'unknown-failure',+ conversationId: 'unknown-failure',+ continuation: false,+ });++ expect(warn).not.toHaveBeenCalled();+ expect(error).toHaveBeenCalledWith(+ '[ResumableAgentController] Initialization error: UnknownError',+ );+ });+});diff --git a/packages/api/src/stream/admission.ts b/packages/api/src/stream/admission.tsnew file mode 100644index 0000000000..18dcd81414--- /dev/null+++ b/packages/api/src/stream/admission.ts@@ -0,0 +1,92 @@+import { logger } from '@librechat/data-schemas';+import type { GenerationPredecessorState, JobStatus } from './interfaces/IJobStore';+import { JobPredecessorMismatchError } from './interfaces/IJobStore';+import { getSafeErrorText } from '../utils/errors';++const PREDECESSOR_MISMATCH_CODE: JobPredecessorMismatchError['code'] =+ 'GENERATION_PREDECESSOR_MISMATCH';++export interface GenerationStartFailureContext {+ streamId: string;+ conversationId: string;+ /** Automatic continuation (trigger delivery, wake-up, queued turn); an active+ * winner answers it with a deferrable `PARENT_NOT_READY`. */+ continuation: boolean;+ expectedPredecessorCreatedAt?: number;+}++type PredecessorMismatchState = Partial<GenerationPredecessorState>;++function isJobStatus(value: unknown): value is JobStatus {+ return (+ value === 'running' ||+ value === 'requires_action' ||+ value === 'complete' ||+ value === 'error' ||+ value === 'aborted'+ );+}++/** Matches by `code` as well as class, mirroring how the controller selects its+ * 409 response, so the log level and the response never disagree. */+function getPredecessorMismatchState(error: unknown): PredecessorMismatchState | undefined {+ if (error instanceof JobPredecessorMismatchError) {+ return error.currentJob;+ }+ if (error == null || typeof error !== 'object' || !('code' in error)) {+ return undefined;+ }+ if (error.code !== PREDECESSOR_MISMATCH_CODE) {+ return undefined;+ }+ const currentJob = 'currentJob' in error ? error.currentJob : undefined;+ if (currentJob == null || typeof currentJob !== 'object') {+ return {};+ }+ const createdAt = 'createdAt' in currentJob ? currentJob.createdAt : undefined;+ const active = 'active' in currentJob ? currentJob.active : undefined;+ const verified = 'verified' in currentJob ? currentJob.verified : undefined;+ const status = 'status' in currentJob ? currentJob.status : undefined;+ return {+ ...(typeof createdAt === 'number' && { createdAt }),+ ...(typeof active === 'boolean' && { active }),+ ...(typeof verified === 'boolean' && { verified }),+ ...(isJobStatus(status) && { status }),+ };+}++/**+ * Logs why a resumable generation failed to start.+ *+ * A predecessor-fence rejection is an expected admission outcome, not a fault:+ * the store refused before replacing anything, an automatic continuation defers+ * without consuming an attempt, and a client restores its queued turn. Sibling+ * background-tool wake-ups for one conversation race this fence by design, so it+ * logs a concise warning. Every other initialization failure stays an error.+ */+export function logGenerationStartFailure(+ error: unknown,+ context: GenerationStartFailureContext,+): void {+ const currentJob = getPredecessorMismatchState(error);+ if (currentJob === undefined) {+ logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`);+ return;+ }+ const active =+ typeof currentJob.active === 'boolean'+ ? currentJob.active+ : currentJob.status === 'running' || currentJob.status === 'requires_action';+ logger.warn('[ResumableAgentController] Generation predecessor changed before creation', {+ streamId: context.streamId,+ conversationId: context.conversationId,+ continuation: context.continuation,+ active,+ ...(context.expectedPredecessorCreatedAt != null && {+ expectedPredecessorCreatedAt: context.expectedPredecessorCreatedAt,+ }),+ ...(currentJob.createdAt != null && { currentCreatedAt: currentJob.createdAt }),+ ...(currentJob.status != null && { currentStatus: currentJob.status }),+ ...(currentJob.verified === false && { verified: false }),+ });+}diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.tsindex 504fafa586..b709ad21ef 100644--- a/packages/api/src/stream/index.ts+++ b/packages/api/src/stream/index.ts@@ -63,6 +63,9 @@ export {} from './SteerRecovery';export type { RecoveredSteerPayload } from './SteerRecovery';+export { logGenerationStartFailure } from './admission';+export type { GenerationStartFailureContext } from './admission';+export { createStreamServices } from './createStreamServices';export type { StreamServicesConfig, StreamServices } from './createStreamServices';export { filterPersistableAbortContent, hasPersistableAbortContent } from './abortContent';