Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔕 fix: Log Lost Generation Predecessor Fences as Warnings (#16689)

Danny Avila2529b22a66cd
diff --git a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
index 18a8828497..8955e99e7d 100644
--- a/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
+++ b/api/server/controllers/agents/__tests__/request.partialDisconnect.spec.js
@@ -82,7 +82,7 @@ jest.mock('@librechat/api', () => ({
resolvePersistableCodeEnvironmentDecision: (...args) =>
jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args),
getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata,
- getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText,
+ logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,
startAgentProjectContextResolution:
jest.requireActual('@librechat/api').startAgentProjectContextResolution,
assertChatProjectInstructions: jest.requireActual('@librechat/api').assertChatProjectInstructions,
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index c8dcd02f81..5211dd610c 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -308,7 +308,7 @@ jest.mock('@librechat/api', () => ({
resolvePersistableCodeEnvironmentDecision: (...args) =>
jest.requireActual('@librechat/api').resolvePersistableCodeEnvironmentDecision(...args),
getSafeErrorMetadata: jest.requireActual('@librechat/api').getSafeErrorMetadata,
- getSafeErrorText: jest.requireActual('@librechat/api').getSafeErrorText,
+ logGenerationStartFailure: jest.requireActual('@librechat/api').logGenerationStartFailure,
resolveFailedTurnContent: jest.requireActual('@librechat/api').resolveFailedTurnContent,
getFailedTurnTraceFields: (...args) => mockGetFailedTurnTraceFields(...args),
startAgentProjectContextResolution:
@@ -1385,6 +1385,65 @@ describe('ResumableAgentController resume metadata', () => {
);
});
+ it('defers a continuation that loses the create fence to a sibling and logs a warning', async () => {
+ mockGetMessages.mockResolvedValue([{ _id: 'persisted-parent' }]);
+ const { JobPredecessorMismatchError } = jest.requireActual('@librechat/api');
+ mockGenerationJobManager.createJob.mockRejectedValue(
+ new JobPredecessorMismatchError({
+ createdAt: 2000,
+ active: true,
+ verified: true,
+ status: 'running',
+ conversationId: 'conversation-123',
+ }),
+ );
+ const req = {
+ _isAgentTrigger: true,
+ user: { id: 'user-123' },
+ body: {
+ text: 'A background tool task is waiting to complete.',
+ messageId: 'wakeup-user-message',
+ parentMessageId: 'persisted-response_',
+ conversationId: 'conversation-123',
+ clientRequestId: 'trigger_sibling_wakeup',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+ const initializeClient = jest.fn();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'user-123',
+ 'conversation-123',
+ expect.objectContaining({ rejectActivePredecessor: true }),
+ );
+ expect(res.set).toHaveBeenCalledWith('Retry-After', '1');
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'PARENT_NOT_READY' }));
+ expect(mockLogger.warn).toHaveBeenCalledWith(
+ '[ResumableAgentController] Generation predecessor changed before creation',
+ {
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ continuation: true,
+ active: true,
+ currentCreatedAt: 2000,
+ currentStatus: 'running',
+ },
+ );
+ expect(
+ mockLogger.error.mock.calls.some((call) =>
+ String(call[0]).startsWith('[ResumableAgentController] Initialization error:'),
+ ),
+ ).toBe(false);
+ expect(initializeClient).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ });
+
it('keeps request-scoped MCP connections until resumable initialization finishes', async () => {
const conversationId = 'conversation-123';
const disconnect = jest.fn().mockResolvedValue(undefined);
@@ -3055,6 +3114,23 @@ describe('ResumableAgentController resume metadata', () => {
expect.objectContaining(DEFAULT_OWNED_CLAIM),
);
expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ expect(mockLogger.warn).toHaveBeenCalledWith(
+ '[ResumableAgentController] Generation predecessor changed before creation',
+ {
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ continuation: false,
+ active: true,
+ expectedPredecessorCreatedAt: 1000,
+ currentCreatedAt: 2000,
+ currentStatus: 'running',
+ },
+ );
+ expect(
+ mockLogger.error.mock.calls.some((call) =>
+ String(call[0]).startsWith('[ResumableAgentController] Initialization error:'),
+ ),
+ ).toBe(false);
});
it('returns a finite fail-closed mismatch when predecessor evidence expired', async () => {
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index f378f1ad83..9801525b40 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -39,7 +39,7 @@ const {
getAttachmentTitleText,
createMCPRuntimeRequestBody,
resolveRunCodeWorkspaces,
- getSafeErrorText,
+ logGenerationStartFailure,
isAgentEventRetentionActive,
createAgentEventActorTurn,
createAgentEventActorDetachedActionLifecycle,
@@ -3483,7 +3483,12 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
);
});
} catch (error) {
- logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`);
+ logGenerationStartFailure(error, {
+ streamId,
+ conversationId,
+ continuation: isTriggerContinuation,
+ expectedPredecessorCreatedAt,
+ });
const initializationFailure = getInitializationFailure(error);
const streamStarted = res.headersSent;
try {
diff --git a/packages/api/src/stream/__tests__/admission.spec.ts b/packages/api/src/stream/__tests__/admission.spec.ts
new file mode 100644
index 0000000000..a39475f43c
--- /dev/null
+++ b/packages/api/src/stream/__tests__/admission.spec.ts
@@ -0,0 +1,189 @@
+import { logger } from '@librechat/data-schemas';
+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
+import { logGenerationStartFailure } from '../admission';
+
+const STARTED = { generationProtocolVersion: 2 as const };
+
+function createContinuationJob(store: InMemoryJobStore, streamId: string) {
+ return store.createJob(
+ streamId,
+ 'owner-1',
+ streamId,
+ undefined,
+ STARTED,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'continuation-create-attempt',
+ undefined,
+ true,
+ );
+}
+
+function createConditionalJob(
+ store: InMemoryJobStore,
+ streamId: string,
+ expectedPredecessorCreatedAt: number,
+) {
+ return store.createJob(
+ streamId,
+ 'owner-1',
+ streamId,
+ undefined,
+ STARTED,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'conditional-create-attempt',
+ expectedPredecessorCreatedAt,
+ );
+}
+
+async function rejectionOf(promise: Promise<unknown>): Promise<unknown> {
+ try {
+ await promise;
+ } catch (error) {
+ return error;
+ }
+ throw new Error('Expected the create to be rejected');
+}
+
+describe('logGenerationStartFailure', () => {
+ let store: InMemoryJobStore;
+ let warn: jest.SpyInstance;
+ let error: jest.SpyInstance;
+
+ beforeEach(() => {
+ store = new InMemoryJobStore();
+ warn = jest.spyOn(logger, 'warn').mockImplementation(() => logger);
+ error = jest.spyOn(logger, 'error').mockImplementation(() => logger);
+ });
+
+ afterEach(async () => {
+ jest.restoreAllMocks();
+ await store.destroy();
+ });
+
+ it('warns when a sibling continuation already won the conversation', async () => {
+ const streamId = 'sibling-wakeup-race';
+ const winner = await createContinuationJob(store, streamId);
+ const mismatch = await rejectionOf(createContinuationJob(store, streamId));
+
+ logGenerationStartFailure(mismatch, {
+ streamId,
+ conversationId: streamId,
+ continuation: true,
+ });
+
+ expect(error).not.toHaveBeenCalled();
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(warn).toHaveBeenCalledWith(
+ '[ResumableAgentController] Generation predecessor changed before creation',
+ {
+ streamId,
+ conversationId: streamId,
+ continuation: true,
+ active: true,
+ currentCreatedAt: winner.createdAt,
+ currentStatus: 'running',
+ },
+ );
+ });
+
+ it('warns with the stale epoch when a queued send lost its fence to a settled turn', async () => {
+ const streamId = 'stale-queued-send';
+ const settled = await store.createJob(streamId, 'owner-1', streamId, undefined, STARTED);
+ await store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ expectCreatedAt: settled.createdAt,
+ });
+ const expectedPredecessorCreatedAt = settled.createdAt - 1;
+ const mismatch = await rejectionOf(
+ createConditionalJob(store, streamId, expectedPredecessorCreatedAt),
+ );
+
+ logGenerationStartFailure(mismatch, {
+ streamId,
+ conversationId: streamId,
+ continuation: false,
+ expectedPredecessorCreatedAt,
+ });
+
+ expect(error).not.toHaveBeenCalled();
+ expect(warn).toHaveBeenCalledWith(
+ '[ResumableAgentController] Generation predecessor changed before creation',
+ {
+ streamId,
+ conversationId: streamId,
+ continuation: false,
+ active: false,
+ expectedPredecessorCreatedAt,
+ currentCreatedAt: settled.createdAt,
+ currentStatus: 'complete',
+ },
+ );
+ });
+
+ it('treats a code-only mismatch the same way the 409 response does', () => {
+ const mismatch = Object.assign(new Error('predecessor changed'), {
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ currentJob: { createdAt: 2000, status: 'requires_action', verified: false },
+ });
+
+ logGenerationStartFailure(mismatch, {
+ streamId: 'code-only',
+ conversationId: 'code-only',
+ continuation: false,
+ });
+
+ expect(error).not.toHaveBeenCalled();
+ expect(warn).toHaveBeenCalledWith(
+ '[ResumableAgentController] Generation predecessor changed before creation',
+ {
+ streamId: 'code-only',
+ conversationId: 'code-only',
+ continuation: false,
+ active: true,
+ currentCreatedAt: 2000,
+ currentStatus: 'requires_action',
+ verified: false,
+ },
+ );
+ });
+
+ it('keeps genuine initialization failures at error level as redacted text', () => {
+ const failure = Object.assign(new Error('Agent not found'), { code: 'AGENT_NOT_FOUND' });
+
+ logGenerationStartFailure(failure, {
+ streamId: 'genuine-failure',
+ conversationId: 'genuine-failure',
+ continuation: true,
+ });
+
+ expect(warn).not.toHaveBeenCalled();
+ expect(error).toHaveBeenCalledTimes(1);
+ const [message, ...metadata] = error.mock.calls[0];
+ expect(metadata).toEqual([]);
+ expect(message).toMatch(
+ /^\[ResumableAgentController\] Initialization error: Error: Agent not found/,
+ );
+ });
+
+ it('logs a non-error rejection at error level', () => {
+ logGenerationStartFailure(undefined, {
+ streamId: 'unknown-failure',
+ conversationId: 'unknown-failure',
+ continuation: false,
+ });
+
+ expect(warn).not.toHaveBeenCalled();
+ expect(error).toHaveBeenCalledWith(
+ '[ResumableAgentController] Initialization error: UnknownError',
+ );
+ });
+});
diff --git a/packages/api/src/stream/admission.ts b/packages/api/src/stream/admission.ts
new file mode 100644
index 0000000000..18dcd81414
--- /dev/null
+++ b/packages/api/src/stream/admission.ts
@@ -0,0 +1,92 @@
+import { logger } from '@librechat/data-schemas';
+import type { GenerationPredecessorState, JobStatus } from './interfaces/IJobStore';
+import { JobPredecessorMismatchError } from './interfaces/IJobStore';
+import { getSafeErrorText } from '../utils/errors';
+
+const PREDECESSOR_MISMATCH_CODE: JobPredecessorMismatchError['code'] =
+ 'GENERATION_PREDECESSOR_MISMATCH';
+
+export interface GenerationStartFailureContext {
+ streamId: string;
+ conversationId: string;
+ /** Automatic continuation (trigger delivery, wake-up, queued turn); an active
+ * winner answers it with a deferrable `PARENT_NOT_READY`. */
+ continuation: boolean;
+ expectedPredecessorCreatedAt?: number;
+}
+
+type PredecessorMismatchState = Partial<GenerationPredecessorState>;
+
+function isJobStatus(value: unknown): value is JobStatus {
+ return (
+ value === 'running' ||
+ value === 'requires_action' ||
+ value === 'complete' ||
+ value === 'error' ||
+ value === 'aborted'
+ );
+}
+
+/** Matches by `code` as well as class, mirroring how the controller selects its
+ * 409 response, so the log level and the response never disagree. */
+function getPredecessorMismatchState(error: unknown): PredecessorMismatchState | undefined {
+ if (error instanceof JobPredecessorMismatchError) {
+ return error.currentJob;
+ }
+ if (error == null || typeof error !== 'object' || !('code' in error)) {
+ return undefined;
+ }
+ if (error.code !== PREDECESSOR_MISMATCH_CODE) {
+ return undefined;
+ }
+ const currentJob = 'currentJob' in error ? error.currentJob : undefined;
+ if (currentJob == null || typeof currentJob !== 'object') {
+ return {};
+ }
+ const createdAt = 'createdAt' in currentJob ? currentJob.createdAt : undefined;
+ const active = 'active' in currentJob ? currentJob.active : undefined;
+ const verified = 'verified' in currentJob ? currentJob.verified : undefined;
+ const status = 'status' in currentJob ? currentJob.status : undefined;
+ return {
+ ...(typeof createdAt === 'number' && { createdAt }),
+ ...(typeof active === 'boolean' && { active }),
+ ...(typeof verified === 'boolean' && { verified }),
+ ...(isJobStatus(status) && { status }),
+ };
+}
+
+/**
+ * Logs why a resumable generation failed to start.
+ *
+ * A predecessor-fence rejection is an expected admission outcome, not a fault:
+ * the store refused before replacing anything, an automatic continuation defers
+ * without consuming an attempt, and a client restores its queued turn. Sibling
+ * background-tool wake-ups for one conversation race this fence by design, so it
+ * logs a concise warning. Every other initialization failure stays an error.
+ */
+export function logGenerationStartFailure(
+ error: unknown,
+ context: GenerationStartFailureContext,
+): void {
+ const currentJob = getPredecessorMismatchState(error);
+ if (currentJob === undefined) {
+ logger.error(`[ResumableAgentController] Initialization error: ${getSafeErrorText(error)}`);
+ return;
+ }
+ const active =
+ typeof currentJob.active === 'boolean'
+ ? currentJob.active
+ : currentJob.status === 'running' || currentJob.status === 'requires_action';
+ logger.warn('[ResumableAgentController] Generation predecessor changed before creation', {
+ streamId: context.streamId,
+ conversationId: context.conversationId,
+ continuation: context.continuation,
+ active,
+ ...(context.expectedPredecessorCreatedAt != null && {
+ expectedPredecessorCreatedAt: context.expectedPredecessorCreatedAt,
+ }),
+ ...(currentJob.createdAt != null && { currentCreatedAt: currentJob.createdAt }),
+ ...(currentJob.status != null && { currentStatus: currentJob.status }),
+ ...(currentJob.verified === false && { verified: false }),
+ });
+}
diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts
index 504fafa586..b709ad21ef 100644
--- a/packages/api/src/stream/index.ts
+++ b/packages/api/src/stream/index.ts
@@ -63,6 +63,9 @@ export {
} from './SteerRecovery';
export type { RecoveredSteerPayload } from './SteerRecovery';
+export { logGenerationStartFailure } from './admission';
+export type { GenerationStartFailureContext } from './admission';
+
export { createStreamServices } from './createStreamServices';
export type { StreamServicesConfig, StreamServices } from './createStreamServices';
export { filterPersistableAbortContent, hasPersistableAbortContent } from './abortContent';