Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🚀 refactor: Move the Resumable Generation Start Into the Chat Core (#17131)

Marco Beretta2f724fccffaa
diff --git a/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts b/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts
index 6a2c5538ba..0c3227d47d 100644
--- a/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts
+++ b/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts
@@ -1,3 +1,4 @@
+import { createElement } from 'react';
import { getDefaultStore } from 'jotai';
import { QueryClient } from '@tanstack/react-query';
import { renderHook, act, waitFor } from '@testing-library/react';
@@ -12,6 +13,7 @@ import {
} from 'librechat-data-provider';
import type { TMessage, TSubmission } from 'librechat-data-provider';
import type { Query, QueryKey } from '@tanstack/react-query';
+import type { ReactNode } from 'react';
import {
activeUsageResponseIdFamily,
liveTokensFamily,
@@ -315,6 +317,8 @@ import useResumableSSE, {
} from '~/hooks/SSE/useResumableSSE';
import useSSE from '~/hooks/SSE/useSSE';
import type { PendingSteer } from '@librechat/chat';
+import type { Transport } from '~/hooks/Chat/contract';
+import { ChatTransportContext, defaultChatTransport } from '~/Providers/ChatTransportContext';
import {
recoveryDispositionsFamily,
queuedMessagesByConvoId,
@@ -627,6 +631,25 @@ describe('useResumableSSE', () => {
});
});
+ it('calls a host transport start on the transport itself', async () => {
+ const receivers: unknown[] = [];
+ const hostTransport: Transport = {
+ ...defaultChatTransport,
+ start(this: Transport, startRequest, options) {
+ receivers.push(this);
+ return defaultChatTransport.start(startRequest, options);
+ },
+ };
+ const wrapper = ({ children }: { children: ReactNode }) =>
+ createElement(ChatTransportContext.Provider, { value: hostTransport }, children);
+ const { unmount } = renderHook(() => useResumableSSE(buildSubmission(), buildChatHelpers()), {
+ wrapper,
+ });
+ await waitFor(() => expect(receivers).toHaveLength(1));
+ expect(receivers[0]).toBe(hostTransport);
+ unmount();
+ });
+
it.each([useSSE, useResumableSSE])(
'%p retains authoritative legacy text response ownership for subsequent usage events',
async (useTransport) => {
diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.ts
index d850b124c2..aa8afd77c0 100644
--- a/client/src/hooks/SSE/useResumableSSE.ts
+++ b/client/src/hooks/SSE/useResumableSSE.ts
@@ -3,6 +3,7 @@ import { v4 } from 'uuid';
import { useSetRecoilState } from 'recoil';
import { useStore, useSetAtom } from 'jotai';
import { useQueryClient } from '@tanstack/react-query';
+import { Constants, QueryKeys, dataService } from 'librechat-data-provider';
import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chat';
import {
resolveDetachedRunEnd,
@@ -10,13 +11,6 @@ import {
canRestoreRecovery,
blockRecovery,
} from '@librechat/chat';
-import {
- Constants,
- QueryKeys,
- dataService,
- createPayload,
- removeNullishValues,
-} from 'librechat-data-provider';
import {
submissionByIndex,
liveAppliedSteerIdsAtom,
@@ -34,44 +28,27 @@ import {
} from '@librechat/chat/react';
import {
toStartGenerationError,
- waitForRetryDelay,
- getRecoverySteerId,
- getRetryAfterDelay,
- isServerNotReadyError,
getOptimisticMessages,
- isResumedStartResponse,
- isSettledStartResponse,
- isRetryableNetworkError,
- isReplacedStartResponse,
isInitialNewConversation,
getStreamStartFailureText,
keepLocalCodeApprovalMode,
hydrateSubmissionMessages,
- getStartGenerationStreamId,
buildOptimisticConversation,
- getGenerationProtocolVersion,
GENERATION_PROTOCOL_VERSION,
- getStartGenerationCreatedAt,
- isUnnegotiatedV2StartControl,
supportsGenerationProtocolV2,
- START_GENERATION_NETWORK_RETRIES,
- getStartGenerationConversationId,
- getPredecessorMismatchStartResponse,
- START_GENERATION_READINESS_TIMEOUT_MS,
ResumableRun,
documentVisibility,
attachResumableStream,
+ startGeneration,
} from '@librechat/chat';
import type {
ResumableHost,
ResumableCache,
SubscriptionTarget,
- StartGenerationResult,
GenerationProtocolVersion,
} from '@librechat/chat';
import type {
TMessage,
- TPayload,
TSubmission,
TConversation,
TPendingSteer,
@@ -435,117 +412,137 @@ export default function useResumableSSE(
* Follows same auth pattern as useSSE
* @param isResume - If true, adds ?resume=true to trigger sync event from server
*/
- const subscribeToStream = useCallback(
- (target: SubscriptionTarget) => {
- const host: ResumableHost = {
- logger,
- visibility: documentVisibility,
- fetchStatus: fetchStreamStatus,
- subscribe: (next) => subscribeToStream(next),
- cache: resumableCache,
- pane: {
- setIsSubmitting,
- setShowStopButton,
- setSubmission,
- setStreamId,
- setGenerationEpoch: updateActiveGenerationCreatedAt,
- setPendingAction: (conversationId, action) =>
- jotaiStore.set(pendingApprovalActionFamily(conversationId), action),
- clearComposerDrafts: (conversationId, options) =>
- clearComposerDrafts(runIndex, conversationId, options),
- publishRunEnd: setRunEnd,
- setAbortScroll,
- renewActiveRun: () => setActiveRunId(v4()),
- runEndTarget: resolveRunEndTarget,
- clearDrainAfterAbort,
- refreshBalance: () => {
- (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
- },
- updateConversation: isAddedRequest ? undefined : setConversation,
- resolveDetachedEnd: (conversationId, messages) => {
- const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));
- const resolvable =
- detachedRun != null &&
- !isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);
- return {
- resolvable,
- end:
- resolvable && messages != null
- ? resolveDetachedRunEnd(conversationId, detachedRun, messages)
- : null,
- };
- },
+ const buildHost = useCallback(
+ (): ResumableHost => ({
+ logger,
+ visibility: documentVisibility,
+ fetchStatus: fetchStreamStatus,
+ subscribe: (next) => attachResumableStream(run, buildHost(), next),
+ cache: resumableCache,
+ pane: {
+ setIsSubmitting,
+ setShowStopButton,
+ setSubmission,
+ setStreamId,
+ setGenerationEpoch: updateActiveGenerationCreatedAt,
+ setPendingAction: (conversationId, action) =>
+ jotaiStore.set(pendingApprovalActionFamily(conversationId), action),
+ clearComposerDrafts: (conversationId, options) =>
+ clearComposerDrafts(runIndex, conversationId, options),
+ publishRunEnd: setRunEnd,
+ setAbortScroll,
+ renewActiveRun: () => setActiveRunId(v4()),
+ runEndTarget: resolveRunEndTarget,
+ clearDrainAfterAbort,
+ refreshBalance: () => {
+ (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
},
- steers: {
- resolve: resolveSteerChip,
- update: updateSteerChips,
- markLiveApplied: (steerId) =>
- setLiveAppliedSteerIds((prev) => appendAppliedSteerIds(prev, [steerId])),
- seed: seedSteerChips,
- appliedIds: collectAppliedSteerIds,
- settleApplied: settleAppliedSteerParts,
- convertToQueued: convertSteersToQueued,
- convertLocalToQueued: convertLocalSteersToQueued,
+ updateConversation: isAddedRequest ? undefined : setConversation,
+ resolveDetachedEnd: (conversationId, messages) => {
+ const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));
+ const resolvable =
+ detachedRun != null &&
+ !isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);
+ return {
+ resolvable,
+ end:
+ resolvable && messages != null
+ ? resolveDetachedRunEnd(conversationId, detachedRun, messages)
+ : null,
+ };
},
- events: {
- stepHandler,
- finalHandler: (data, finalSubmission) =>
- finalHandler(data as TFinalResData, finalSubmission),
- createdHandler,
- attachmentHandler,
- titleHandler,
- contentHandler,
- messageHandler,
- prunePtcTraces,
- resetContentHandler,
- syncStepMessage,
- clearStepMaps,
- flushPendingDeltas,
- resetLive,
- showError: (text, errorSubmission) =>
- errorHandler({
- data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],
- submission: errorSubmission as EventSubmission,
- }),
- },
- messages: { get: getMessages, set: setMessages },
- usage: {
- bindResponse,
- contextHandler,
- usageHandler,
- tapStream,
- tapContent,
- finalizeUsage,
- backfillUsage,
- seedLiveFromContent: (content, usageSubmission) =>
- seedLive(countTrailingOutputChars(content), usageSubmission),
- },
- stream: transport.stream({ token }),
- exposeDebugControls: import.meta.env.DEV
- ? ({ killNetwork, closeClean }) => {
- // Debug hooks for testing reconnection vs clean close behavior (dev only)
- const debugWindow = window as Window & {
- __killNetwork?: () => void;
- __closeClean?: () => void;
- };
- debugWindow.__killNetwork = killNetwork;
- debugWindow.__closeClean = closeClean;
- }
- : undefined,
- parts: {
- applySteer: applySteerPart,
- findSteerTarget: findSteerMessageIndex,
- applyActivityLabel: applyActivityLabelPart,
- findActivityLabelTarget: findActivityLabelMessageIndex,
- offsetActivityPhaseBoundary,
- applyReasoningLabel,
- findReasoningLabelTarget: findReasoningLabelMessageIndex,
+ },
+ steers: {
+ resolve: resolveSteerChip,
+ isRecoveryHeld: (conversationId, recoverySteerId) =>
+ jotaiStore.get(recoveryDispositionsFamily(conversationId))[recoverySteerId] != null,
+ blockRecovery: (conversationId, recoverySteerId) =>
+ jotaiStore.set(recoveryDispositionsFamily(conversationId), (previous) =>
+ blockRecovery(previous, recoverySteerId),
+ ),
+ restoreQueued: restoreQueuedSubmission,
+ update: updateSteerChips,
+ markLiveApplied: (steerId) =>
+ setLiveAppliedSteerIds((prev) => appendAppliedSteerIds(prev, [steerId])),
+ seed: seedSteerChips,
+ appliedIds: collectAppliedSteerIds,
+ settleApplied: settleAppliedSteerParts,
+ convertToQueued: convertSteersToQueued,
+ convertLocalToQueued: convertLocalSteersToQueued,
+ },
+ events: {
+ stepHandler,
+ showStartFailure: (failedSubmission, errorData, reason) => {
+ let text: string | undefined;
+ if (reason === 'recovery_held') {
+ text = localize('com_ui_steer_recovery_held');
+ } else if (reopenTermsOnRefusal(queryClient, errorData)) {
+ text = localize('com_error_terms_not_accepted');
+ }
+ errorHandler({
+ data: getStreamStartFailureData(errorData, text),
+ submission: failedSubmission as EventSubmission,
+ });
},
- };
- attachResumableStream(run, host, target);
- },
+ finalHandler: (data, finalSubmission) =>
+ finalHandler(data as TFinalResData, finalSubmission),
+ createdHandler,
+ attachmentHandler,
+ titleHandler,
+ contentHandler,
+ messageHandler,
+ prunePtcTraces,
+ resetContentHandler,
+ syncStepMessage,
+ clearStepMaps,
+ flushPendingDeltas,
+ resetLive,
+ showError: (text, errorSubmission) =>
+ errorHandler({
+ data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],
+ submission: errorSubmission as EventSubmission,
+ }),
+ },
+ messages: { get: getMessages, set: setMessages },
+ usage: {
+ bindResponse,
+ contextHandler,
+ usageHandler,
+ tapStream,
+ tapContent,
+ finalizeUsage,
+ backfillUsage,
+ seedLiveFromContent: (content, usageSubmission) =>
+ seedLive(countTrailingOutputChars(content), usageSubmission),
+ },
+ stream: transport.stream({ token }),
+ start: (request, options) => transport.start(request, options),
+ exposeDebugControls: import.meta.env.DEV
+ ? ({ killNetwork, closeClean }) => {
+ // Debug hooks for testing reconnection vs clean close behavior (dev only)
+ const debugWindow = window as Window & {
+ __killNetwork?: () => void;
+ __closeClean?: () => void;
+ };
+ debugWindow.__killNetwork = killNetwork;
+ debugWindow.__closeClean = closeClean;
+ }
+ : undefined,
+ parts: {
+ applySteer: applySteerPart,
+ findSteerTarget: findSteerMessageIndex,
+ applyActivityLabel: applyActivityLabelPart,
+ findActivityLabelTarget: findActivityLabelMessageIndex,
+ offsetActivityPhaseBoundary,
+ applyReasoningLabel,
+ findReasoningLabelTarget: findReasoningLabelMessageIndex,
+ },
+ }),
[
run,
+ localize,
+ queryClient,
+ restoreQueuedSubmission,
resumableCache,
isAddedRequest,
setConversation,
@@ -597,260 +594,9 @@ export default function useResumableSSE(
],
);
- /**
- * Start generation (POST request that returns streamId)
- * Posts through the host transport, which owns auth refresh.
- * Retries transient network failures and startup readiness responses.
- * Readiness retries honor Retry-After until cleanup or the readiness window expires.
- */
- const startGeneration = useCallback(
- async (
- currentSubmission: TSubmission,
- signal?: AbortSignal,
- ): Promise<StartGenerationResult | null> => {
- const payloadData = createPayload(currentSubmission);
- let { payload } = payloadData;
- payload = removeNullishValues(payload) as TPayload;
-
- clearStepMaps();
-
- const url = payloadData.server;
-
- let lastError: unknown = null;
- let requestAttempts = 0;
- let networkAttempts = 0;
- let readinessAttempts = 0;
- const readinessDeadline = run.scheduler.now() + START_GENERATION_READINESS_TIMEOUT_MS;
-
- while (!signal?.aborted) {
- const recoverySteerId = getRecoverySteerId(currentSubmission);
- const conversationId = currentSubmission.conversation?.conversationId;
- if (
- recoverySteerId != null &&
- conversationId &&
- jotaiStore.get(recoveryDispositionsFamily(conversationId))[recoverySteerId] != null
- ) {
- restoreQueuedSubmission(currentSubmission);
- errorHandler({
- data: getStreamStartFailureData(localize('com_ui_steer_recovery_held')),
- submission: currentSubmission as EventSubmission,
- });
- setShowStopButton(false);
- setIsSubmitting(false);
- setSubmission(null);
- return null;
- }
- requestAttempts += 1;
- try {
- const data = await transport.start({ server: url, payload }, { signal });
- if (signal?.aborted) {
- return null;
- }
- if (isSettledStartResponse(data)) {
- const conversationId = getStartGenerationConversationId(data);
- if (conversationId) {
- logger.log('ResumableSSE', 'Generation already settled:', { conversationId });
- return {
- status: 'settled',
- conversationId,
- generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
- };
- }
- lastError = { response: { data } };
- break;
- }
- if (isReplacedStartResponse(data)) {
- const streamId = getStartGenerationStreamId(data);
- const conversationId = getStartGenerationConversationId(data);
- const generationCreatedAt = getStartGenerationCreatedAt(data);
- if (streamId && conversationId && generationCreatedAt != null) {
- return {
- status: 'replaced',
- streamId,
- conversationId,
- generationCreatedAt,
- generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
- };
- }
- lastError = { response: { data } };
- break;
- }
- if (isUnnegotiatedV2StartControl(data)) {
- /** Never reinterpret a v2-only control word as a legacy stream
- * response merely because it also carries a stream id. */
- lastError = { response: { data } };
- break;
- }
- const streamId = getStartGenerationStreamId(data);
- if (streamId) {
- const resumed = isResumedStartResponse(data);
- const generationCreatedAt = getStartGenerationCreatedAt(data);
- const generationProtocolVersion = getGenerationProtocolVersion(data);
- logger.log('ResumableSSE', 'Generation started:', {
- streamId,
- resumed,
- generationCreatedAt,
- generationProtocolVersion,
- });
- return {
- status: 'stream',
- streamId,
- resumed,
- generationProtocolVersion,
- ...(generationCreatedAt != null && { generationCreatedAt }),
- };
- }
-
- lastError = { response: { data } };
- break;
- } catch (error) {
- if (signal?.aborted) {
- return null;
- }
-
- const predecessorMismatch = getPredecessorMismatchStartResponse(error);
- if (predecessorMismatch != null) {
- /** Admission was atomically rejected before this queued turn
- * created anything. Put the exact row back immediately and let
- * the caller reconcile or hand off to the generation that won. */
- restoreQueuedSubmission(currentSubmission);
- logger.log('ResumableSSE', 'Queued generation lost predecessor fence', {
- expectedPredecessorCreatedAt: currentSubmission.expectedPredecessorCreatedAt,
- ...predecessorMismatch,
- });
- return {
- status: 'predecessor_mismatch',
- ...predecessorMismatch,
- generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
- };
- }
-
- lastError = error;
- const isNetworkError = isRetryableNetworkError(error);
- const isServerNotReady = isServerNotReadyError(error);
- const remainingReadinessMs = readinessDeadline - run.scheduler.now();
- const shouldRetryNetwork =
- isNetworkError && networkAttempts < START_GENERATION_NETWORK_RETRIES - 1;
- const shouldRetryServerNotReady = isServerNotReady && remainingReadinessMs > 0;
-
- if (shouldRetryNetwork || shouldRetryServerNotReady) {
- networkAttempts += isNetworkError ? 1 : 0;
- readinessAttempts += isServerNotReady ? 1 : 0;
- const fallbackDelay = Math.min(1000 * Math.pow(2, requestAttempts - 1), 8000);
- const retryDelay = isServerNotReady
- ? Math.min(getRetryAfterDelay(error, fallbackDelay), remainingReadinessMs)
- : fallbackDelay;
- const reason = isServerNotReady ? 'Server not ready' : 'Network error';
- const attempt = isServerNotReady ? readinessAttempts : networkAttempts;
- const limit = isServerNotReady
- ? `${Math.ceil(START_GENERATION_READINESS_TIMEOUT_MS / 1000)}s readiness window`
- : `${START_GENERATION_NETWORK_RETRIES}`;
- logger.log(
- 'ResumableSSE',
- `${reason} starting generation, retrying in ${retryDelay}ms (attempt ${attempt}/${limit})`,
- );
- const shouldContinue = await waitForRetryDelay(retryDelay, signal);
- if (!shouldContinue) {
- return null;
- }
- continue;
- }
-
- // Don't retry: either not a network error or max retries reached
- break;
- }
- }
-
- if (signal?.aborted) {
- return null;
- }
-
- logger.error('ResumableSSE', 'Error starting generation:', lastError);
-
- const startError = toStartGenerationError(lastError);
- const errorData = startError?.response?.data;
- const responseStatus = startError?.response?.status;
- if (responseStatus != null && responseStatus >= 400 && responseStatus < 500) {
- const recoverySteerId = getRecoverySteerId(currentSubmission);
- const conversationId = currentSubmission.conversation?.conversationId;
- const recoveryRejected =
- errorData != null &&
- typeof errorData === 'object' &&
- 'code' in errorData &&
- (errorData.code === 'RECOVERY_PAYLOAD_MISMATCH' ||
- errorData.code === 'INVALID_RECOVERY_REQUEST');
- if (recoveryRejected && recoverySteerId != null && conversationId) {
- jotaiStore.set(recoveryDispositionsFamily(conversationId), (previous) =>
- blockRecovery(previous, recoverySteerId),
- );
- }
- // The server rejected admission before exposing a generation. Restore
- // the exact queue row/position; ambiguous transport/5xx outcomes must
- // first reconcile durable state instead of risking a duplicate start.
- restoreQueuedSubmission(currentSubmission);
- } else {
- const recoverySteerId = getRecoverySteerId(currentSubmission);
- const recoveryConvoId = currentSubmission.conversation?.conversationId;
- if (recoverySteerId != null && recoveryConvoId) {
- try {
- const status = await fetchStreamStatus(recoveryConvoId);
- if (signal?.aborted) {
- return null;
- }
- const releasedSource = status.unrecoveredSteers?.find(
- (steer) => steer.steerId === recoverySteerId,
- );
- if (
- status.active === false &&
- supportsGenerationProtocolV2(status) &&
- releasedSource != null
- ) {
- convertSteersToQueued(recoveryConvoId, [releasedSource], {
- generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
- allowPreviouslyConvertedIds: [recoverySteerId],
- });
- }
- } catch (error) {
- if (signal?.aborted) {
- return null;
- }
- logger.warn('ResumableSSE', 'Could not recover source after start failure', {
- conversationId: recoveryConvoId,
- error,
- });
- }
- }
- }
- if (signal?.aborted) {
- return null;
- }
- const termsRefused = reopenTermsOnRefusal(queryClient, errorData);
- errorHandler({
- data: getStreamStartFailureData(
- errorData,
- termsRefused ? localize('com_error_terms_not_accepted') : undefined,
- ),
- submission: currentSubmission as EventSubmission,
- });
- setShowStopButton(false);
- setIsSubmitting(false);
- setSubmission(null);
- return null;
- },
- [
- run,
- clearStepMaps,
- convertSteersToQueued,
- errorHandler,
- jotaiStore,
- localize,
- queryClient,
- restoreQueuedSubmission,
- setIsSubmitting,
- setShowStopButton,
- setSubmission,
- transport,
- ],
+ const subscribeToStream = useCallback(
+ (target: SubscriptionTarget) => attachResumableStream(run, buildHost(), target),
+ [buildHost, run],
);
useEffect(() => {
@@ -952,7 +698,7 @@ export default function useResumableSSE(
} else {
// New generation: start and then subscribe
logger.log('ResumableSSE', 'Starting NEW generation');
- const startResult = await startGeneration(submission, signal);
+ const startResult = await startGeneration(run, buildHost(), submission, signal);
if (!isCurrentEffect()) {
return;
}
diff --git a/packages/chat/src/core/__fixtures__/resumable.ts b/packages/chat/src/core/__fixtures__/resumable.ts
index c87f7051e1..d164237cfb 100644
--- a/packages/chat/src/core/__fixtures__/resumable.ts
+++ b/packages/chat/src/core/__fixtures__/resumable.ts
@@ -110,6 +110,9 @@ export const createHost = (
resolve: jest.fn(() => []),
markLiveApplied: jest.fn(),
update: jest.fn(),
+ isRecoveryHeld: jest.fn(() => false),
+ blockRecovery: jest.fn(),
+ restoreQueued: jest.fn(),
seed: jest.fn(),
appliedIds: jest.fn(() => []),
settleApplied: jest.fn(),
@@ -131,6 +134,7 @@ export const createHost = (
flushPendingDeltas: jest.fn(),
resetLive: jest.fn(),
showError: jest.fn(),
+ showStartFailure: jest.fn(),
},
messages: {
get: jest.fn(() => messages),
@@ -158,6 +162,7 @@ export const createHost = (
seedLiveFromContent: jest.fn(),
},
stream: { send: jest.fn(), reconnectToStream: jest.fn(() => ({ closed: false })) },
+ start: jest.fn(),
};
return host;
};
diff --git a/packages/chat/src/core/__tests__/launch.spec.ts b/packages/chat/src/core/__tests__/launch.spec.ts
new file mode 100644
index 0000000000..5ed9214e0c
--- /dev/null
+++ b/packages/chat/src/core/__tests__/launch.spec.ts
@@ -0,0 +1,111 @@
+import type { TMessage, TSubmission } from 'librechat-data-provider';
+import { createHost, createScheduler } from '../__fixtures__/resumable';
+import { ResumableRun, startGeneration } from '../resumable';
+
+const submission = {
+ userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true, text: 'Hi' },
+ initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,
+ conversation: { conversationId: 'c-1', endpoint: 'agents' },
+ endpointOption: { endpoint: 'agents' },
+} as unknown as TSubmission;
+
+const httpError = (status: number, data: unknown) =>
+ Object.assign(new Error('failed'), { response: { status, data, headers: {} } });
+
+const setup = () => {
+ const host = createHost();
+ const run = new ResumableRun(createScheduler().scheduler);
+ return { host, run };
+};
+
+describe('startGeneration', () => {
+ it('resolves to the stream a start names', async () => {
+ const { host, run } = setup();
+ (host.start as jest.Mock).mockResolvedValue({
+ streamId: 's-1',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ await expect(startGeneration(run, host, submission)).resolves.toEqual({
+ status: 'stream',
+ streamId: 's-1',
+ resumed: false,
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ expect(host.events.clearStepMaps).toHaveBeenCalled();
+ });
+
+ it('reports a settled v2 start', async () => {
+ const { host, run } = setup();
+ (host.start as jest.Mock).mockResolvedValue({
+ status: 'settled',
+ conversationId: 'c-1',
+ generationProtocolVersion: 2,
+ });
+ await expect(startGeneration(run, host, submission)).resolves.toEqual({
+ status: 'settled',
+ conversationId: 'c-1',
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('restores the queued turn when the predecessor fence was lost', async () => {
+ const { host, run } = setup();
+ (host.start as jest.Mock).mockRejectedValue(
+ httpError(409, {
+ status: 'predecessor_mismatch',
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ streamId: 'c-1',
+ conversationId: 'c-1',
+ generationCreatedAt: 3000,
+ active: true,
+ }),
+ );
+ await expect(startGeneration(run, host, submission)).resolves.toMatchObject({
+ status: 'predecessor_mismatch',
+ generationCreatedAt: 3000,
+ });
+ expect(host.steers.restoreQueued).toHaveBeenCalledWith(submission);
+ });
+
+ it('writes the error card and idles the pane on a rejected start', async () => {
+ const { host, run } = setup();
+ const body = { message: 'forbidden' };
+ (host.start as jest.Mock).mockRejectedValue(httpError(403, body));
+ await expect(startGeneration(run, host, submission)).resolves.toBeNull();
+ expect(host.steers.restoreQueued).toHaveBeenCalledWith(submission);
+ expect(host.events.showStartFailure).toHaveBeenCalledWith(submission, body);
+ expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(false);
+ expect(host.pane.setSubmission).toHaveBeenCalledWith(null);
+ });
+
+ it('blocks a recovery the server refused', async () => {
+ const { host, run } = setup();
+ (host.start as jest.Mock).mockRejectedValue(
+ httpError(400, { code: 'RECOVERY_PAYLOAD_MISMATCH' }),
+ );
+ await startGeneration(run, host, { ...submission, recoverySteerId: 'steer-1' } as TSubmission);
+ expect(host.steers.blockRecovery).toHaveBeenCalledWith('c-1', 'steer-1');
+ });
+
+ it('refuses a recovery the user already dealt with', async () => {
+ const { host, run } = setup();
+ (host.steers.isRecoveryHeld as jest.Mock).mockReturnValue(true);
+ const recovery = { ...submission, recoverySteerId: 'steer-1' } as TSubmission;
+ await expect(startGeneration(run, host, recovery)).resolves.toBeNull();
+ expect(host.start).not.toHaveBeenCalled();
+ expect(host.events.showStartFailure).toHaveBeenCalledWith(recovery, undefined, 'recovery_held');
+ });
+
+ it('stops quietly once the pane abandons the start', async () => {
+ const { host, run } = setup();
+ const controller = new AbortController();
+ (host.start as jest.Mock).mockImplementation(async () => {
+ controller.abort();
+ throw Object.assign(new Error('offline'), { code: 'ERR_NETWORK' });
+ });
+ await expect(startGeneration(run, host, submission, controller.signal)).resolves.toBeNull();
+ expect(host.events.showStartFailure).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/chat/src/core/resumable/host.ts b/packages/chat/src/core/resumable/host.ts
index c1fa5df2bc..4397c8c4f3 100644
--- a/packages/chat/src/core/resumable/host.ts
+++ b/packages/chat/src/core/resumable/host.ts
@@ -6,6 +6,7 @@ import type {
TPendingSteer,
TAttachment,
ChatTransport,
+ ChatTransportRequest,
ChatTitleFrame,
ChatFinalFrame,
EventSubmission,
@@ -127,6 +128,12 @@ export interface ResumableSteers {
markLiveApplied: (steerId: string) => void;
/** Patches the chips the server re-labelled (preempt capability); never removes one. */
update: (event: TSteerUpdatedEvent) => void;
+ /** Whether the user has already dealt with the parked steer a recovery start replays. */
+ isRecoveryHeld: (conversationId: string, recoverySteerId: string) => boolean;
+ /** Refuses further recovery of a parked steer the server rejected. */
+ blockRecovery: (conversationId: string, recoverySteerId: string) => void;
+ /** Puts a queued turn whose start was refused back in its queue position. */
+ restoreQueued: (submission: TSubmission, expectedPredecessorCreatedAt?: number) => void;
/** Re-seeds the chips from the server's still-queued steers. */
seed: (
conversationId: string,
@@ -247,6 +254,13 @@ export interface ResumableEvents {
flushPendingDeltas: () => void;
/** Discards the live usage estimate for a run that stopped streaming. */
resetLive: (submission: TSubmission) => void;
+ /** Writes the error card for a start that failed, from the response body it failed with, or
+ * for a recovery start the user already dealt with. */
+ showStartFailure: (
+ submission: TSubmission,
+ errorData?: unknown,
+ reason?: 'recovery_held',
+ ) => void;
/** Writes the failed turn's error card from the text (or raw error value) the server sent. */
showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void;
}
@@ -308,6 +322,8 @@ export interface ResumableHost extends SubscriptionHost {
usage: ResumableUsage;
/** The stream side of the app's transport, bound to the signed-in user. */
stream: ChatTransport;
+ /** POSTs a turn to the resumable generation route; resolves to the start response. */
+ start: (request: ChatTransportRequest, options?: { signal?: AbortSignal }) => Promise<unknown>;
/** Receives the dev-only controls that simulate a dropped or a clean close. */
exposeDebugControls?: (controls: { killNetwork: () => void; closeClean: () => void }) => void;
}
diff --git a/packages/chat/src/core/resumable/index.ts b/packages/chat/src/core/resumable/index.ts
index 7c1b1a400c..db73151282 100644
--- a/packages/chat/src/core/resumable/index.ts
+++ b/packages/chat/src/core/resumable/index.ts
@@ -15,3 +15,4 @@ export * from './patches';
export * from './frames';
export * from './sync';
export * from './attach';
+export * from './launch';
diff --git a/packages/chat/src/core/resumable/launch.ts b/packages/chat/src/core/resumable/launch.ts
new file mode 100644
index 0000000000..e94b205b68
--- /dev/null
+++ b/packages/chat/src/core/resumable/launch.ts
@@ -0,0 +1,259 @@
+import { createPayload, removeNullishValues } from 'librechat-data-provider';
+import type { TPayload, TSubmission } from 'librechat-data-provider';
+import type { StartGenerationResult } from './start';
+import type { ResumableHost } from './host';
+import type { ResumableRun } from './run';
+import {
+ getRecoverySteerId,
+ getRetryAfterDelay,
+ waitForRetryDelay,
+ isServerNotReadyError,
+ isResumedStartResponse,
+ isSettledStartResponse,
+ toStartGenerationError,
+ isRetryableNetworkError,
+ isReplacedStartResponse,
+ getStartGenerationStreamId,
+ getStartGenerationCreatedAt,
+ isUnnegotiatedV2StartControl,
+ START_GENERATION_NETWORK_RETRIES,
+ getStartGenerationConversationId,
+ getPredecessorMismatchStartResponse,
+ START_GENERATION_READINESS_TIMEOUT_MS,
+} from './start';
+import {
+ GENERATION_PROTOCOL_VERSION,
+ getGenerationProtocolVersion,
+ supportsGenerationProtocolV2,
+} from '../protocol';
+
+/**
+ * Starts a generation (the POST that returns the stream to attach to) through the host
+ * transport, which owns auth refresh. Retries transient network failures and startup
+ * readiness responses; readiness retries honor Retry-After until `signal` aborts or the
+ * readiness window expires. Resolves to null when the start failed (its error card written)
+ * or was abandoned.
+ */
+export async function startGeneration(
+ run: ResumableRun,
+ host: ResumableHost,
+ currentSubmission: TSubmission,
+ signal?: AbortSignal,
+): Promise<StartGenerationResult | null> {
+ const payloadData = createPayload(currentSubmission);
+ let { payload } = payloadData;
+ payload = removeNullishValues(payload) as TPayload;
+
+ host.events.clearStepMaps();
+
+ const url = payloadData.server;
+
+ let lastError: unknown = null;
+ let requestAttempts = 0;
+ let networkAttempts = 0;
+ let readinessAttempts = 0;
+ const readinessDeadline = run.scheduler.now() + START_GENERATION_READINESS_TIMEOUT_MS;
+
+ while (!signal?.aborted) {
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ const conversationId = currentSubmission.conversation?.conversationId;
+ if (
+ recoverySteerId != null &&
+ conversationId &&
+ host.steers.isRecoveryHeld(conversationId, recoverySteerId)
+ ) {
+ host.steers.restoreQueued(currentSubmission);
+ host.events.showStartFailure(currentSubmission, undefined, 'recovery_held');
+ host.pane.setShowStopButton(false);
+ host.pane.setIsSubmitting(false);
+ host.pane.setSubmission(null);
+ return null;
+ }
+ requestAttempts += 1;
+ try {
+ const data = await host.start({ server: url, payload }, { signal });
+ if (signal?.aborted) {
+ return null;
+ }
+ if (isSettledStartResponse(data)) {
+ const conversationId = getStartGenerationConversationId(data);
+ if (conversationId) {
+ host.logger.log('ResumableSSE', 'Generation already settled:', { conversationId });
+ return {
+ status: 'settled',
+ conversationId,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ }
+ lastError = { response: { data } };
+ break;
+ }
+ if (isReplacedStartResponse(data)) {
+ const streamId = getStartGenerationStreamId(data);
+ const conversationId = getStartGenerationConversationId(data);
+ const generationCreatedAt = getStartGenerationCreatedAt(data);
+ if (streamId && conversationId && generationCreatedAt != null) {
+ return {
+ status: 'replaced',
+ streamId,
+ conversationId,
+ generationCreatedAt,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ }
+ lastError = { response: { data } };
+ break;
+ }
+ if (isUnnegotiatedV2StartControl(data)) {
+ /** Never reinterpret a v2-only control word as a legacy stream
+ * response merely because it also carries a stream id. */
+ lastError = { response: { data } };
+ break;
+ }
+ const streamId = getStartGenerationStreamId(data);
+ if (streamId) {
+ const resumed = isResumedStartResponse(data);
+ const generationCreatedAt = getStartGenerationCreatedAt(data);
+ const generationProtocolVersion = getGenerationProtocolVersion(data);
+ host.logger.log('ResumableSSE', 'Generation started:', {
+ streamId,
+ resumed,
+ generationCreatedAt,
+ generationProtocolVersion,
+ });
+ return {
+ status: 'stream',
+ streamId,
+ resumed,
+ generationProtocolVersion,
+ ...(generationCreatedAt != null && { generationCreatedAt }),
+ };
+ }
+
+ lastError = { response: { data } };
+ break;
+ } catch (error) {
+ if (signal?.aborted) {
+ return null;
+ }
+
+ const predecessorMismatch = getPredecessorMismatchStartResponse(error);
+ if (predecessorMismatch != null) {
+ /** Admission was atomically rejected before this queued turn
+ * created anything. Put the exact row back immediately and let
+ * the caller reconcile or hand off to the generation that won. */
+ host.steers.restoreQueued(currentSubmission);
+ host.logger.log('ResumableSSE', 'Queued generation lost predecessor fence', {
+ expectedPredecessorCreatedAt: currentSubmission.expectedPredecessorCreatedAt,
+ ...predecessorMismatch,
+ });
+ return {
+ status: 'predecessor_mismatch',
+ ...predecessorMismatch,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ }
+
+ lastError = error;
+ const isNetworkError = isRetryableNetworkError(error);
+ const isServerNotReady = isServerNotReadyError(error);
+ const remainingReadinessMs = readinessDeadline - run.scheduler.now();
+ const shouldRetryNetwork =
+ isNetworkError && networkAttempts < START_GENERATION_NETWORK_RETRIES - 1;
+ const shouldRetryServerNotReady = isServerNotReady && remainingReadinessMs > 0;
+
+ if (shouldRetryNetwork || shouldRetryServerNotReady) {
+ networkAttempts += isNetworkError ? 1 : 0;
+ readinessAttempts += isServerNotReady ? 1 : 0;
+ const fallbackDelay = Math.min(1000 * Math.pow(2, requestAttempts - 1), 8000);
+ const retryDelay = isServerNotReady
+ ? Math.min(getRetryAfterDelay(error, fallbackDelay), remainingReadinessMs)
+ : fallbackDelay;
+ const reason = isServerNotReady ? 'Server not ready' : 'Network error';
+ const attempt = isServerNotReady ? readinessAttempts : networkAttempts;
+ const limit = isServerNotReady
+ ? `${Math.ceil(START_GENERATION_READINESS_TIMEOUT_MS / 1000)}s readiness window`
+ : `${START_GENERATION_NETWORK_RETRIES}`;
+ host.logger.log(
+ 'ResumableSSE',
+ `${reason} starting generation, retrying in ${retryDelay}ms (attempt ${attempt}/${limit})`,
+ );
+ const shouldContinue = await waitForRetryDelay(retryDelay, signal);
+ if (!shouldContinue) {
+ return null;
+ }
+ continue;
+ }
+
+ // Don't retry: either not a network error or max retries reached
+ break;
+ }
+ }
+
+ if (signal?.aborted) {
+ return null;
+ }
+
+ host.logger.error('ResumableSSE', 'Error starting generation:', lastError);
+
+ const startError = toStartGenerationError(lastError);
+ const errorData = startError?.response?.data;
+ const responseStatus = startError?.response?.status;
+ if (responseStatus != null && responseStatus >= 400 && responseStatus < 500) {
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ const conversationId = currentSubmission.conversation?.conversationId;
+ const recoveryRejected =
+ errorData != null &&
+ typeof errorData === 'object' &&
+ 'code' in errorData &&
+ (errorData.code === 'RECOVERY_PAYLOAD_MISMATCH' ||
+ errorData.code === 'INVALID_RECOVERY_REQUEST');
+ if (recoveryRejected && recoverySteerId != null && conversationId) {
+ host.steers.blockRecovery(conversationId, recoverySteerId);
+ }
+ // The server rejected admission before exposing a generation. Restore
+ // the exact queue row/position; ambiguous transport/5xx outcomes must
+ // first reconcile durable state instead of risking a duplicate start.
+ host.steers.restoreQueued(currentSubmission);
+ } else {
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ const recoveryConvoId = currentSubmission.conversation?.conversationId;
+ if (recoverySteerId != null && recoveryConvoId) {
+ try {
+ const status = await host.fetchStatus(recoveryConvoId);
+ if (signal?.aborted) {
+ return null;
+ }
+ const releasedSource = status.unrecoveredSteers?.find(
+ (steer) => steer.steerId === recoverySteerId,
+ );
+ if (
+ status.active === false &&
+ supportsGenerationProtocolV2(status) &&
+ releasedSource != null
+ ) {
+ host.steers.convertToQueued(recoveryConvoId, [releasedSource], {
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ });
+ }
+ } catch (error) {
+ if (signal?.aborted) {
+ return null;
+ }
+ host.logger.warn('ResumableSSE', 'Could not recover source after start failure', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+ }
+ }
+ if (signal?.aborted) {
+ return null;
+ }
+ host.events.showStartFailure(currentSubmission, errorData);
+ host.pane.setShowStopButton(false);
+ host.pane.setIsSubmitting(false);
+ host.pane.setSubmission(null);
+ return null;
+}