Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧞 fix: Recover OAuth Prompts on Initial Replica Attachment (#17455)

lia-by-librechat[bot]600825ac3ca2
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 48353dbdce..56de9a96ea 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -5336,6 +5336,10 @@ class GenerationJobManagerClass {
}
};
+ const needsDurableInitialReplay =
+ this._isRedis &&
+ !options?.skipBufferReplay &&
+ this.ownedJobs.get(streamId) !== runtime.createdAt;
const deferSequenceDelivery =
this._isRedis && !runtime.hasSubscriber && !options?.skipBufferReplay;
const transportSubscription = this.eventTransport.subscribe(
@@ -5370,8 +5374,13 @@ class GenerationJobManagerClass {
// sequence reaches this process. Hold sequenced pub/sub delivery until replay and
// sync establish the exact frontier, otherwise the new subscriber sees it twice.
deferSequenceDelivery,
+ captureSequenceFrontier: needsDurableInitialReplay,
},
);
+ const synchronizeReorderBuffer = (replayedNextSeq?: number): void | Promise<void> =>
+ needsDurableInitialReplay && transportSubscription.syncReorderBuffer
+ ? transportSubscription.syncReorderBuffer()
+ : this.eventTransport.syncReorderBuffer?.(streamId, replayedNextSeq);
runtime.localErrorHandlers.add(queueError);
if (!options?.skipBufferReplay) {
runtime.earlyReplayHandlers.add(queueChunk);
@@ -5442,14 +5451,20 @@ class GenerationJobManagerClass {
return subscriptionActive;
};
+ let initialReplayUnavailable = false;
const stillMatchesExpectedGeneration = async (): Promise<boolean> => {
- if (options?.expectedCreatedAt == null) {
+ if (options?.expectedCreatedAt == null && !needsDurableInitialReplay) {
return true;
}
const currentJob = await this.jobStore.getJob(streamId);
+ initialReplayUnavailable =
+ needsDurableInitialReplay &&
+ (currentJob?.durableEventCount ?? 0) > 0 &&
+ (currentJob?.status === 'running' || currentJob?.status === 'requires_action');
+ const expectedCreatedAt = options?.expectedCreatedAt ?? runtime.createdAt;
return (
- currentJob?.createdAt === options.expectedCreatedAt &&
- runtime.createdAt === options.expectedCreatedAt &&
+ currentJob?.createdAt === expectedCreatedAt &&
+ runtime.createdAt === expectedCreatedAt &&
this.runtimeState.get(streamId) === runtime
);
};
@@ -5474,6 +5489,14 @@ class GenerationJobManagerClass {
throw err;
}
+ if (initialReplayUnavailable) {
+ // Check after SUBSCRIBE so a prompt published during attachment cannot be missed.
+ // This replica has no local replay; resume reconstructs the durable prefix before live data.
+ queueError(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ activateDelivery();
+ return subscription;
+ }
+
const isFirst = this.eventTransport.isFirstSubscriber(streamId);
if (!runtime.hasSubscriber) {
@@ -5531,6 +5554,7 @@ class GenerationJobManagerClass {
jobData,
attachmentGeneration,
deferSequenceDelivery,
+ synchronizeReorderBuffer,
);
return null;
}
@@ -5613,7 +5637,7 @@ class GenerationJobManagerClass {
runtime.earlyEventBufferClosed = true;
}
try {
- const reorderSync = this.eventTransport.syncReorderBuffer?.(streamId, replayedNextSeq);
+ const reorderSync = synchronizeReorderBuffer(replayedNextSeq);
if (reorderSync) {
await waitWhileAttached(reorderSync);
}
@@ -5985,6 +6009,7 @@ class GenerationJobManagerClass {
jobData: SerializableJobData | null,
attachmentGeneration: number,
sequenceDeliveryDeferred: boolean,
+ synchronizeReorderBuffer: (replayedNextSeq?: number) => void | Promise<void>,
): void {
if (this.eventTransport.getSubscriberCount(streamId) === 0) {
return;
@@ -6047,7 +6072,7 @@ class GenerationJobManagerClass {
/** Same closure as the owning-subscriber bootstrap above. */
runtime.earlyEventBufferClosed = true;
}
- await this.eventTransport.syncReorderBuffer?.(streamId, replayedNextSeq);
+ await synchronizeReorderBuffer(replayedNextSeq);
}
})
.catch((err) => {
diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
index 3ce5548ff0..3c4e200318 100644
--- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
@@ -781,6 +781,10 @@ describe('RedisEventTransport', () => {
isRedis: true,
cleanupOnComplete: false,
});
+ mockPublisher.publish.mockImplementation(async (channel: string, payload: string) => {
+ getMessageHandler(mockSubscriber)(channel, payload);
+ return 1;
+ });
const onDone = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, onDone);
const channel = `stream:{${streamId}}:events`;
@@ -1936,6 +1940,10 @@ describe('RedisEventTransport', () => {
data: { delta: { content: [{ type: 'text', text: 'World' }] } },
};
const messageHandler = getMessageHandler(attachingSubscriber);
+ mockPublisher.publish.mockImplementation(async (channel: string, payload: string) => {
+ messageHandler(channel, payload);
+ return 1;
+ });
const originalSync = attachingTransport.syncReorderBuffer.bind(attachingTransport);
jest
.spyOn(attachingTransport, 'syncReorderBuffer')
diff --git a/packages/api/src/stream/__tests__/initialRecovery.stream_integration.spec.ts b/packages/api/src/stream/__tests__/initialRecovery.stream_integration.spec.ts
new file mode 100644
index 0000000000..17989c2538
--- /dev/null
+++ b/packages/api/src/stream/__tests__/initialRecovery.stream_integration.spec.ts
@@ -0,0 +1,227 @@
+import { randomUUID } from 'crypto';
+import type { ServerSentEvent, StreamSubscription } from '~/types';
+import type { RedisTestClient } from './helpers/redis';
+import {
+ buildMCPAuthStepId,
+ buildMCPAuthToolCall,
+ buildMCPAuthRunStepEvent,
+ buildMCPAuthRunStepDeltaEvent,
+} from '~/mcp/oauth/events';
+import {
+ GenerationJobManagerClass,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
+} from '../GenerationJobManager';
+import { clearRedisTestPrefix, createRedisTestClient } from './helpers/redis';
+import { RedisEventTransport } from '../implementations/RedisEventTransport';
+import { RedisJobStore } from '../implementations/RedisJobStore';
+
+jest.spyOn(console, 'log').mockImplementation();
+
+const prefix = `initial-recovery:${randomUUID()}:`;
+const managers: GenerationJobManagerClass[] = [];
+const subscribers: RedisTestClient[] = [];
+const subscriptions: StreamSubscription[] = [];
+let redis: RedisTestClient;
+
+function replica() {
+ const subscriber = createRedisTestClient(prefix);
+ subscribers.push(subscriber);
+ const jobStore = new RedisJobStore(redis);
+ const eventTransport = new RedisEventTransport(redis, subscriber);
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: true });
+ manager.initialize();
+ managers.push(manager);
+ return { manager, jobStore, eventTransport, subscriber };
+}
+
+async function prompt(manager: GenerationJobManagerClass, streamId: string) {
+ const stepId = buildMCPAuthStepId('server');
+ const toolCall = buildMCPAuthToolCall({ id: 'user:server:flow', serverName: 'server' });
+ const events = [
+ buildMCPAuthRunStepEvent({ stepId, toolCall }),
+ buildMCPAuthRunStepDeltaEvent({
+ stepId,
+ toolCall,
+ authURL: 'https://auth.example/authorize',
+ }),
+ ];
+ for (const event of events) await manager.emitChunk(streamId, event);
+ return events;
+}
+
+beforeAll(async () => {
+ redis = createRedisTestClient(prefix);
+ await redis.ping();
+});
+
+afterEach(async () => {
+ for (const subscription of subscriptions.splice(0)) subscription.unsubscribe();
+ await Promise.all(managers.splice(0).map((manager) => manager.destroy()));
+ for (const subscriber of subscribers.splice(0)) subscriber.disconnect();
+ await clearRedisTestPrefix(redis, prefix);
+});
+
+afterAll(async () => {
+ await redis.quit();
+});
+
+describe('initial cross-replica durable recovery', () => {
+ test.each([
+ { timing: 'before attach', fenced: true },
+ { timing: 'before attach', fenced: false },
+ { timing: 'during attach', fenced: true },
+ { timing: 'during attach', fenced: false },
+ ])(
+ 'reconnects for a prompt emitted $timing (fenced: $fenced), then resumes it',
+ async ({ timing, fenced }) => {
+ const owner = replica();
+ const viewer = replica();
+ const streamId = randomUUID();
+ const job = await owner.manager.createJob(streamId, 'user', streamId);
+ if (timing === 'before attach') {
+ await prompt(owner.manager, streamId);
+ } else {
+ const subscribe = viewer.eventTransport.subscribe.bind(viewer.eventTransport);
+ jest.spyOn(viewer.eventTransport, 'subscribe').mockImplementation((...args) => {
+ const subscription = subscribe(...args);
+ return {
+ ...subscription,
+ ready: subscription.ready!.then(async () => {
+ await prompt(owner.manager, streamId);
+ }),
+ };
+ });
+ }
+ const received: ServerSentEvent[] = [];
+ const onError = jest.fn();
+ const subscription = await viewer.manager.subscribe(
+ streamId,
+ (event) => received.push(event),
+ undefined,
+ onError,
+ fenced ? { expectedCreatedAt: job.createdAt } : undefined,
+ );
+ expect(subscription).not.toBeNull();
+ subscriptions.push(subscription!);
+ expect(onError).toHaveBeenCalledTimes(1);
+ expect(onError).toHaveBeenCalledWith(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ expect(received).toEqual([]);
+ expect(viewer.eventTransport.getSubscriberCount(streamId)).toBe(0);
+ expect((await viewer.jobStore.getJob(streamId))?.error).toBeUndefined();
+
+ jest.restoreAllMocks();
+ const resumed = await viewer.manager.subscribeWithResume(
+ streamId,
+ (event) => received.push(event),
+ undefined,
+ onError,
+ { expectedCreatedAt: job.createdAt },
+ );
+ expect(resumed.resumeState?.pendingOAuthPrompts).toEqual([
+ expect.objectContaining({
+ stepId: buildMCPAuthStepId('server'),
+ toolCallId: 'user:server:flow',
+ authURL: 'https://auth.example/authorize',
+ }),
+ ]);
+ expect(resumed.subscription).not.toBeNull();
+ subscriptions.push(resumed.subscription!);
+ resumed.subscription!.activate();
+ expect(onError).toHaveBeenCalledTimes(1);
+ },
+ );
+
+ test('preserves a prompt published after validation but before sequence synchronization', async () => {
+ const owner = replica();
+ const viewer = replica();
+ const streamId = randomUUID();
+ const job = await owner.manager.createJob(streamId, 'user', streamId);
+ const listeners = viewer.subscriber.listeners('message');
+ const held: Array<[string, string]> = [];
+ viewer.subscriber.removeAllListeners('message');
+ viewer.subscriber.on('message', (channel: string, message: string) => {
+ if (JSON.parse(message).type === 'chunk') {
+ held.push([channel, message]);
+ } else {
+ for (const listener of listeners) listener(channel, message);
+ }
+ });
+ const getJob = viewer.jobStore.getJob.bind(viewer.jobStore);
+ let emitted = false;
+ jest.spyOn(viewer.jobStore, 'getJob').mockImplementation(async (...args) => {
+ const snapshot = await getJob(...args);
+ if (!emitted && viewer.eventTransport.getSubscriberCount(streamId) > 0) {
+ emitted = true;
+ await prompt(owner.manager, streamId);
+ }
+ return snapshot;
+ });
+ const received: ServerSentEvent[] = [];
+ const onError = jest.fn();
+ const subscription = await viewer.manager.subscribe(
+ streamId,
+ (event) => received.push(event),
+ undefined,
+ onError,
+ { expectedCreatedAt: job.createdAt },
+ );
+ subscriptions.push(subscription!);
+ const deadline = Date.now() + 5_000;
+ while (held.length < 2 && Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ expect(held).toHaveLength(2);
+ for (const [channel, message] of held) {
+ for (const listener of listeners) listener(channel, message);
+ }
+ expect(received.map((event) => ('event' in event ? event.event : undefined))).toEqual([
+ 'on_run_step',
+ 'on_run_step_delta',
+ ]);
+ expect(onError).not.toHaveBeenCalled();
+ });
+
+ test('keeps owner-local OAuth replay on the initial connection', async () => {
+ const owner = replica();
+ const streamId = randomUUID();
+ const job = await owner.manager.createJob(streamId, 'user', streamId);
+ const events = await prompt(owner.manager, streamId);
+ const received: ServerSentEvent[] = [];
+ const onError = jest.fn();
+ const subscription = await owner.manager.subscribe(
+ streamId,
+ (event) => received.push(event),
+ undefined,
+ onError,
+ { expectedCreatedAt: job.createdAt },
+ );
+ subscriptions.push(subscription!);
+ expect(received).toEqual(events);
+ expect(onError).not.toHaveBeenCalled();
+ });
+
+ test('keeps an empty remote attachment live without forcing a reconnect', async () => {
+ const owner = replica();
+ const viewer = replica();
+ const streamId = randomUUID();
+ const job = await owner.manager.createJob(streamId, 'user', streamId);
+ const received: ServerSentEvent[] = [];
+ const onError = jest.fn();
+ const subscription = await viewer.manager.subscribe(
+ streamId,
+ (event) => received.push(event),
+ undefined,
+ onError,
+ { expectedCreatedAt: job.createdAt },
+ );
+ subscriptions.push(subscription!);
+ const events = await prompt(owner.manager, streamId);
+ const deadline = Date.now() + 5_000;
+ while (received.length < events.length && Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ expect(received).toEqual(events);
+ expect(onError).not.toHaveBeenCalled();
+ });
+});