Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🛟 refactor: Move Resumable Transport Failure Handling Into the Chat Core (#17120)

Marco Berettad4e52acdfbb2
diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.ts
index d9008f45fa..de0479ac22 100644
--- a/client/src/hooks/SSE/useResumableSSE.ts
+++ b/client/src/hooks/SSE/useResumableSSE.ts
@@ -8,7 +8,6 @@ import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chat
import {
Constants,
QueryKeys,
- ErrorTypes,
StepEvents,
StepTypes,
SteerEvents,
@@ -20,15 +19,12 @@ import {
UsageEvents,
createPayload,
ApprovalEvents,
- ViolationTypes,
removeNullishValues,
} from 'librechat-data-provider';
import {
- MAX_RETRIES,
getStepEventId,
toStartGenerationError,
isOAuthStepEvent,
- getReconnectDelay,
waitForRetryDelay,
applyPendingAction,
getRecoverySteerId,
@@ -48,7 +44,6 @@ import {
keepLocalCodeApprovalMode,
hydrateSubmissionMessages,
buildResumeEventSubmission,
- completedResponseMessageId,
getStartGenerationStreamId,
buildOptimisticConversation,
buildCreatedInitialResponse,
@@ -65,9 +60,9 @@ import {
ResumableRun,
documentVisibility,
ResumableSubscription,
- handoffToReplacement,
authorizeTerminalTeardown,
reconcileGenerationLifecycle,
+ handleTransportFailure,
clearPendingApprovalForTerminal,
clearAttachedGenerationCreatedAt,
} from '@librechat/chat';
@@ -288,6 +283,16 @@ export default function useResumableSSE(
queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(conversationId) }),
addActiveJob,
removeActiveJob,
+ fetchConversation: (conversationId) =>
+ fetchConvoSnapshot(queryClient, conversationId, () =>
+ dataService.getConversationById(conversationId),
+ ),
+ setConversation: (conversationId, conversation) =>
+ queryClient.setQueryData([QueryKeys.conversation, conversationId], conversation),
+ upsertConversation: (conversation) =>
+ upsertConvoInAllQueries(queryClient, conversation, true, 'snapshot'),
+ removeConversation: (conversationId) =>
+ removeConvoFromAllQueries(queryClient, conversationId),
}),
[addActiveJob, queryClient, removeActiveJob],
);
@@ -487,6 +492,23 @@ export default function useResumableSSE(
clearComposerDrafts(runIndex, conversationId, options),
publishRunEnd: setRunEnd,
clearDrainAfterAbort,
+ refreshBalance: () => {
+ (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
+ },
+ updateConversation: isAddedRequest ? undefined : setConversation,
+ resolveDetachedEnd: (conversationId, messages) => {
+ const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));
+ const resolvable =
+ detachedRun != null &&
+ !isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);
+ return {
+ resolvable,
+ end:
+ resolvable && messages != null
+ ? resolveDetachedRunEnd(conversationId, detachedRun, messages)
+ : null,
+ };
+ },
},
steers: {
seed: seedSteerChips,
@@ -495,7 +517,17 @@ export default function useResumableSSE(
convertToQueued: convertSteersToQueued,
convertLocalToQueued: convertLocalSteersToQueued,
},
- events: { clearStepMaps, flushPendingDeltas, resetLive },
+ events: {
+ clearStepMaps,
+ flushPendingDeltas,
+ resetLive,
+ showError: (text, errorSubmission) =>
+ errorHandler({
+ data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],
+ submission: errorSubmission as EventSubmission,
+ }),
+ },
+ messages: { get: getMessages, set: setMessages },
};
const sub = new ResumableSubscription(run, host, target);
const {
@@ -1587,644 +1619,6 @@ export default function useResumableSSE(
}
};
- /**
- * Error event handler - handles BOTH:
- * 1. HTTP-level errors (status present) - 404, 409, network failures (0)
- * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
- *
- * Order matters: check the status first since HTTP errors may also include data.
- * The transport has already retried a 401 with a refreshed token.
- */
- const handleTransportFailure = async ({
- status: responseCode,
- data,
- }: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>) => {
- if (!sub.isCurrent()) {
- return;
- }
-
- if (sub.finalReceived) {
- logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
- responseCode,
- hasData: data != null,
- });
- return;
- }
-
- (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
-
- if (responseCode === 409) {
- const replacementConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
- const handedOff = await handoffToReplacement(terminal, replacementConvoId);
- if (!sub.isCurrent()) {
- return;
- }
- if (handedOff) {
- return;
- }
- }
-
- // 404 → job completed & was cleaned up; messages are persisted in DB.
- // Refetch and reconcile them before recovering local steers: the final
- // response may contain an applied steer whose live event was missed.
- if (responseCode === 404) {
- const convoId = sub.submission.conversation?.conversationId;
- logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
- // Keep the UI non-idle while the status read decides whether this is
- // terminal cleanup or a handoff to a newer generation epoch.
- run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
- run.cancelSteerRetryFrames();
- sub.close();
- /** Terminal: drop any in-flight live estimate so the gauge doesn't
- * keep counting stale streamed output after the stream ends */
- resetLive(sub.liveSubmission());
- clearComposerDrafts(runIndex, convoId, {
- includeNewChatDraft:
- !convoId ||
- convoId === Constants.NEW_CONVO ||
- run.optimisticStreamIds.has(currentStreamId) ||
- isInitialNewConversation(sub.submission),
- });
- clearStepMaps();
- let persistedMessages: TMessage[] | undefined;
- if (convoId) {
- try {
- const messageQueryKey = [QueryKeys.messages, convoId] as const;
- await queryClient.invalidateQueries({
- queryKey: messageQueryKey,
- refetchType: 'none',
- });
- if (!sub.isCurrent()) {
- return;
- }
- const fetched = await queryClient.fetchQuery<TMessage[]>({
- queryKey: messageQueryKey,
- // The first stream can finish before CREATED mounts the saved-chat query.
- queryFn: () => fetchConversationMessages(convoId),
- });
- if (!sub.isCurrent()) {
- return;
- }
- if (Array.isArray(fetched)) {
- persistedMessages = fetched;
- }
- } catch (error) {
- if (!sub.isCurrent()) {
- return;
- }
- logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {
- conversationId: convoId,
- error,
- });
- }
- queryClient.removeQueries({ queryKey: streamStatusQueryKey(convoId) });
- }
- const recoveryConvoId = convoId ?? currentStreamId;
- const reconciledIds = new Set(
- persistedMessages ? collectAppliedSteerIds(persistedMessages) : [],
- );
- if (persistedMessages) {
- settleAppliedSteerParts(recoveryConvoId, persistedMessages);
- }
-
- // The status replay is owner-gated and non-destructive; creating a
- // recovered follow-up consumes its exact source. Prefer these
- // authoritative leftovers over guessing from local chips.
- let confirmedV2Terminal = false;
- try {
- const status = await fetchStreamStatus(recoveryConvoId);
- if (!sub.isCurrent()) {
- return;
- }
- if (
- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
- !supportsGenerationProtocolV2(status)
- ) {
- sub.scheduleReattach(1_000);
- return;
- }
- if (
- status.active === true &&
- status.createdAt != null &&
- (generationCreatedAt == null || status.createdAt !== generationCreatedAt)
- ) {
- const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);
- if (!sub.isCurrent()) {
- return;
- }
- if (handedOff) {
- return;
- }
- /** Legacy generations have no replacement-handoff contract.
- * Never attach this submission to a different epoch. */
- sub.scheduleReattach(1_000);
- return;
- }
- if (status.active === true) {
- /** A stream lookup can briefly lose its routing record while the
- * authoritative job status still proves the same generation is
- * alive. Preserve ownership and retry the attachment instead of
- * publishing a terminal run-end (which could auto-drain queued
- * text into a still-running generation). */
- const activeStreamId = status.streamId ?? currentStreamId;
- queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
- if (status.createdAt != null) {
- updateActiveGenerationCreatedAt(
- recoveryConvoId,
- status.createdAt,
- undefined,
- generationProtocolVersion,
- );
- }
- addActiveJob(activeStreamId);
- seedSteerChips(
- recoveryConvoId,
- (status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
- status.createdAt,
- generationProtocolVersion,
- );
- settleAppliedSteerParts(
- recoveryConvoId,
- status.resumeState?.aggregatedContent ?? status.aggregatedContent,
- );
- if (status.pendingAction ?? status.resumeState?.pendingAction) {
- applyPendingActionToMessages(
- (status.pendingAction ??
- status.resumeState?.pendingAction) as Agents.PendingAction,
- );
- }
- setIsSubmitting(true);
- setShowStopButton(status.createdAt != null);
- run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
- sub.scheduleReattach(1_000, activeStreamId, status.createdAt);
- return;
- }
- confirmedV2Terminal =
- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
- supportsGenerationProtocolV2(status);
- const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
- const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
- if (unrecovered.length > 0) {
- const recoverySteerId = getRecoverySteerId(sub.submission);
- convertSteersToQueued(recoveryConvoId, unrecovered, {
- generationProtocolVersion,
- ...(recoverySteerId != null && {
- allowPreviouslyConvertedIds: [recoverySteerId],
- }),
- });
- for (const steer of unrecovered) {
- reconciledIds.add(steer.steerId);
- if (steer.clientSteerId) {
- reconciledIds.add(steer.clientSteerId);
- }
- }
- }
- } catch (error) {
- if (!sub.isCurrent()) {
- return;
- }
- logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {
- conversationId: recoveryConvoId,
- error,
- });
- }
-
- removeActiveJob(currentStreamId);
- clearAttachedGenerationCreatedAt(terminal);
- clearPendingApprovalForTerminal(terminal, recoveryConvoId);
- if (
- !run.createdStreamIds.has(currentStreamId) &&
- run.optimisticStreamIds.has(currentStreamId)
- ) {
- // Both fresh and resumed subscriptions can miss every event of a fast turn.
- // A missing job proves neither that the conversation was saved nor that it failed.
- try {
- const persisted = await fetchConvoSnapshot(queryClient, recoveryConvoId, () =>
- dataService.getConversationById(recoveryConvoId),
- );
- if (!sub.isCurrent()) return;
- if (persisted?.conversationId === recoveryConvoId) {
- queryClient.setQueryData([QueryKeys.conversation, recoveryConvoId], persisted);
- upsertConvoInAllQueries(queryClient, persisted, true, 'snapshot');
- if (!isAddedRequest) {
- setConversation?.((current) => {
- if (
- current?.conversationId != null &&
- current.conversationId !== Constants.NEW_CONVO &&
- current.conversationId !== recoveryConvoId
- )
- return current;
- return keepLocalCodeApprovalMode(
- { ...current, ...persisted },
- current,
- current?.conversationId,
- );
- });
- }
- }
- } catch (error) {
- if (!sub.isCurrent()) return;
- if (toStartGenerationError(error)?.response?.status === 404) {
- removeConvoFromAllQueries(queryClient, currentStreamId);
- }
- }
- // An inconclusive read must not turn a saved chat into a phantom.
- invalidateConversationLists(queryClient);
- queryClient.invalidateQueries({ queryKey: [QueryKeys.pinnedConversations] });
- }
- if (persistedMessages) {
- setMessages(persistedMessages);
- }
- sub.retired = true;
- setIsSubmitting(false);
- setShowStopButton(false);
-
- // Only infer recovery from local accepted chips after the persisted
- // final was successfully inspected. If that fetch failed, their
- // applied-vs-leftover outcome is unknown and converting could send
- // already-applied words twice.
- if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
- convertLocalSteersToQueued(recoveryConvoId, {
- excludeSteerIds: reconciledIds,
- generationProtocolVersion,
- });
- }
- // The true outcome is unknown here (job record already cleaned up):
- // a non-'completed' outcome releases parked interrupt flags without
- // auto-sending queued messages the user may not want fired. A run this
- // pane left and came back to is the exception: the history just read
- // names its response by epoch, which is the outcome it would have had.
- // History read before the job finished may not hold that response yet, or
- // may hold only the unfinished snapshot saved when the pane left, so a
- // marker that read leaves unresolved or stopped outlives this end for the
- // next visit, which reads history after the job is known to be done.
- const detachedRun = jotaiStore.get(detachedRunByConvoId(recoveryConvoId));
- const detachedResolvable =
- detachedRun != null &&
- !isDetachedRunManual(jotaiStore.get, recoveryConvoId, detachedRun);
- const detachedEnd =
- detachedResolvable && persistedMessages != null
- ? resolveDetachedRunEnd(recoveryConvoId, detachedRun, persistedMessages)
- : null;
- setRunEnd(
- {
- conversationId: recoveryConvoId,
- outcome: 'aborted',
- endedAt: run.scheduler.now(),
- ...detachedEnd,
- startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),
- },
- {
- retainDetached:
- detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),
- },
- );
- setStreamId(null);
- run.releaseStream(currentStreamId);
- run.reconnectAttempt = 0;
- return;
- }
-
- /**
- * Server-sent error event (event: error with data) - no responseCode.
- * These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.
- * Only check the data if there's no HTTP status, since HTTP errors may also have body data.
- * Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,
- * not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).
- */
- if (responseCode == null && data != null) {
- sub.finalReceived = true;
- const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
- if (
- !(await authorizeTerminalTeardown(terminal, recoveryConvoId, 'error')) ||
- !sub.isCurrent()
- ) {
- return;
- }
- logger.log('ResumableSSE', 'Server-sent error event received:', data);
- run.cancelSteerRetryFrames();
- sub.close();
- /** FLUSH (not cancel): the error card below is built from the cache
- * tail, so queued tokens must land first, and a stale trailing
- * frame must never overwrite the error write. */
- flushPendingDeltas();
- removeActiveJob(currentStreamId);
- clearAttachedGenerationCreatedAt(terminal);
- clearPendingApprovalForTerminal(terminal, recoveryConvoId);
- resetLive(sub.liveSubmission());
- if (
- !run.createdStreamIds.has(currentStreamId) &&
- run.optimisticStreamIds.has(currentStreamId)
- ) {
- removeConvoFromAllQueries(queryClient, currentStreamId);
- }
-
- const errorSupportsV2 = supportsGenerationProtocolV2(data);
- const errorString =
- typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));
-
- // Check if it's a known error type (ViolationTypes or ErrorTypes)
- let isKnownError = false;
- try {
- const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;
- const errorType = parsed?.type ?? parsed?.code;
- if (errorType) {
- const violationValues = Object.values(ViolationTypes) as string[];
- const errorTypeValues = Object.values(ErrorTypes) as string[];
- isKnownError =
- violationValues.includes(errorType) || errorTypeValues.includes(errorType);
- }
- } catch {
- // Not JSON or parsing failed - treat as generic error
- }
-
- logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });
-
- // Display the error to user via errorHandler
- errorHandler({
- data: { text: errorString } as unknown as Parameters<typeof errorHandler>[0]['data'],
- submission: sub.submission as EventSubmission,
- });
-
- setIsSubmitting(false);
- setShowStopButton(false);
- const recoverySteerId = getRecoverySteerId(sub.submission);
- if (
- recoverySteerId != null &&
- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
- errorSupportsV2
- ) {
- try {
- /** The error was received on the exact generation-fenced stream.
- * Read status only now, after terminal was observed: a claim
- * started before this recovery could return a stale parked copy
- * after a successful commit and must never authorize redelivery. */
- const terminalStatus = await fetchStreamStatus(recoveryConvoId);
- if (!sub.isCurrent()) {
- return;
- }
- const releasedSource = terminalStatus.unrecoveredSteers?.find(
- (steer) => steer.steerId === recoverySteerId,
- );
- if (
- terminalStatus.active === false &&
- supportsGenerationProtocolV2(terminalStatus) &&
- releasedSource != null
- ) {
- convertSteersToQueued(recoveryConvoId, [releasedSource], {
- generationProtocolVersion,
- allowPreviouslyConvertedIds: [recoverySteerId],
- });
- }
- } catch (error) {
- if (!sub.isCurrent()) {
- return;
- }
- logger.warn('ResumableSSE', 'Could not recover source after generation error', {
- conversationId: recoveryConvoId,
- error,
- });
- }
- }
- // The error terminal's backstop parks acked leftovers server-side;
- // claim it now so a reload can't resurrect the chips converted here.
- if (generationProtocolVersion === 1 || errorSupportsV2) {
- convertLocalSteersToQueued(recoveryConvoId, {
- claimParked: true,
- generationProtocolVersion,
- });
- }
- setRunEnd({
- conversationId: recoveryConvoId,
- outcome: 'error',
- endedAt: run.scheduler.now(),
- generationCreatedAt,
- });
- setStreamId(null);
- run.releaseStream(currentStreamId);
- run.reconnectAttempt = 0;
- return;
- }
-
- // Network failure or unknown HTTP error - attempt reconnection with backoff
- logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {
- responseCode,
- hasData: data != null,
- });
-
- if (run.reconnectAttempt < MAX_RETRIES) {
- // Increment counter BEFORE close() so abort handler knows we're reconnecting
- run.reconnectAttempt++;
- const delay = getReconnectDelay(run.reconnectAttempt);
-
- logger.log(
- 'ResumableSSE',
- `Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,
- );
-
- sub.close();
-
- sub.scheduleReattach(delay);
-
- // Keep UI in "submitting" state during reconnection attempts
- // so user knows we're still trying (abort handler may have reset these)
- setIsSubmitting(true);
- setShowStopButton(generationCreatedAt != null);
- } else {
- logger.error('ResumableSSE', 'Max reconnect attempts reached');
- sub.close();
- flushPendingDeltas();
- const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
- let status: Awaited<ReturnType<typeof fetchStreamStatus>> | undefined;
- try {
- status = await fetchStreamStatus(recoveryConvoId);
- if (!sub.isCurrent()) {
- return;
- }
- } catch (error) {
- if (!sub.isCurrent()) {
- return;
- }
- logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {
- conversationId: recoveryConvoId,
- error,
- });
- }
-
- if (
- status != null &&
- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
- !supportsGenerationProtocolV2(status)
- ) {
- // A legacy responder cannot adjudicate a v2 job during rollout.
- status = undefined;
- }
-
- if (
- status?.active === true &&
- status.createdAt != null &&
- (generationCreatedAt == null || status.createdAt !== generationCreatedAt)
- ) {
- const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);
- if (!sub.isCurrent()) {
- return;
- }
- if (handedOff) {
- return;
- }
- run.reconnectAttempt = 0;
- sub.scheduleReattach(30_000);
- return;
- }
-
- if (status?.active === true) {
- // The transport gave up, not the generation. Keep the active-job
- // marker and accepted chips intact, then release this stale
- // submission so useResumeOnLoad can establish a fresh attachment.
- queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
- seedSteerChips(
- recoveryConvoId,
- (status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
- status.createdAt,
- generationProtocolVersion,
- );
- settleAppliedSteerParts(
- recoveryConvoId,
- status.resumeState?.aggregatedContent ?? status.aggregatedContent,
- );
- if (status.pendingAction ?? status.resumeState?.pendingAction) {
- applyPendingActionToMessages(
- (status.pendingAction ?? status.resumeState?.pendingAction) as Agents.PendingAction,
- );
- }
- setIsSubmitting(true);
- setShowStopButton(generationCreatedAt != null);
- run.reconnectAttempt = 0;
- sub.scheduleReattach(30_000);
- return;
- }
-
- if (status == null) {
- // An inconclusive status read is still not proof of termination.
- // Preserve server/client ownership and let the normal resume-on-load
- // status query retry rather than duplicating accepted words.
- queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(recoveryConvoId) });
- setIsSubmitting(true);
- setShowStopButton(generationCreatedAt != null);
- run.reconnectAttempt = 0;
- sub.scheduleReattach(30_000);
- return;
- }
-
- // The status endpoint proved the job terminal. Reconcile persisted
- // content before recovering leftovers, just like the 404 path.
- run.cancelSteerRetryFrames();
- let persistedMessages: TMessage[] | undefined;
- try {
- const messageQueryKey = [QueryKeys.messages, recoveryConvoId] as const;
- await queryClient.invalidateQueries({
- queryKey: messageQueryKey,
- refetchType: 'none',
- });
- if (!sub.isCurrent()) {
- return;
- }
- const fetched = await queryClient.fetchQuery<TMessage[]>({
- queryKey: messageQueryKey,
- });
- if (!sub.isCurrent()) {
- return;
- }
- if (Array.isArray(fetched)) {
- persistedMessages = fetched;
- }
- } catch (error) {
- if (!sub.isCurrent()) {
- return;
- }
- logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {
- conversationId: recoveryConvoId,
- error,
- });
- }
-
- const authoritativeValues = [
- ...(persistedMessages ?? []),
- ...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),
- ];
- const confirmedV2Terminal =
- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
- supportsGenerationProtocolV2(status);
- const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues));
- settleAppliedSteerParts(recoveryConvoId, authoritativeValues);
- const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
- const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
- if (unrecovered.length > 0) {
- const recoverySteerId = getRecoverySteerId(sub.submission);
- convertSteersToQueued(recoveryConvoId, unrecovered, {
- generationProtocolVersion,
- ...(recoverySteerId != null && {
- allowPreviouslyConvertedIds: [recoverySteerId],
- }),
- });
- for (const steer of unrecovered) {
- reconciledIds.add(steer.steerId);
- if (steer.clientSteerId) {
- reconciledIds.add(steer.clientSteerId);
- }
- }
- }
- if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
- convertLocalSteersToQueued(recoveryConvoId, {
- excludeSteerIds: reconciledIds,
- generationProtocolVersion,
- });
- }
-
- resetLive(sub.liveSubmission());
- removeActiveJob(currentStreamId);
- clearAttachedGenerationCreatedAt(terminal);
- clearPendingApprovalForTerminal(terminal, recoveryConvoId);
- if (
- !run.createdStreamIds.has(currentStreamId) &&
- run.optimisticStreamIds.has(currentStreamId)
- ) {
- removeConvoFromAllQueries(queryClient, currentStreamId);
- }
- sub.retired = true;
- setIsSubmitting(false);
- setShowStopButton(false);
- let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';
- if (status.status === 'complete' && persistedMessages != null) {
- recoveryOutcome = 'completed';
- } else if (status.status === 'aborted') {
- recoveryOutcome = 'aborted';
- }
- const recoveredResponseMessageId =
- recoveryOutcome === 'completed'
- ? completedResponseMessageId(
- persistedMessages,
- sub.userMessage?.messageId,
- status?.resumeState?.responseMessageId ??
- sub.submission.initialResponse?.messageId,
- )
- : undefined;
- setRunEnd({
- conversationId: recoveryConvoId,
- outcome: recoveryOutcome,
- endedAt: run.scheduler.now(),
- generationCreatedAt: status.createdAt ?? generationCreatedAt,
- ...(recoveredResponseMessageId != null && {
- responseMessageId: recoveredResponseMessageId,
- }),
- });
- setSubmission(null);
- setStreamId(null);
- run.releaseStream(currentStreamId);
- run.reconnectAttempt = 0;
- }
- };
-
/**
* The transport emits `abort` only for this hook's own closes. A cancel
* the user agent issued (a backgrounded or frozen mobile tab, while the
@@ -2277,7 +1671,7 @@ export default function useResumableSSE(
sub.connect(transport.stream({ token }), {
onOpen: handleOpen,
onFrame: (event) => void handleFrame(event),
- onFailure: (event) => void handleTransportFailure(event),
+ onFailure: (event) => void handleTransportFailure(terminal, event),
onClose: handleClose,
});
@@ -2291,7 +1685,7 @@ export default function useResumableSSE(
/** Simulate network drop - triggers error event → reconnection */
debugWindow.__killNetwork = () => {
logger.log('Debug', 'Simulating network drop...');
- void handleTransportFailure({});
+ void handleTransportFailure(terminal, {});
};
/** Simulate clean close (navigation away) - triggers abort event → no reconnection */
@@ -2332,7 +1726,6 @@ export default function useResumableSSE(
startupConfig?.balance?.enabled,
balanceQuery,
removeActiveJob,
- queryClient,
bindResponse,
contextHandler,
usageHandler,
@@ -2351,7 +1744,6 @@ export default function useResumableSSE(
settleAppliedSteerParts,
convertSteersToQueued,
convertLocalSteersToQueued,
- addActiveJob,
setSubmission,
updateActiveGenerationCreatedAt,
jotaiStore,
diff --git a/packages/chat/src/core/__tests__/failure.spec.ts b/packages/chat/src/core/__tests__/failure.spec.ts
new file mode 100644
index 0000000000..ee6d2a342c
--- /dev/null
+++ b/packages/chat/src/core/__tests__/failure.spec.ts
@@ -0,0 +1,218 @@
+import type { TMessage, TSubmission } from 'librechat-data-provider';
+import type {
+ ResumableHost,
+ TerminalContext,
+ ResumableScheduler,
+ StreamStatusResponse,
+} from '../resumable';
+import { ResumableRun, ResumableSubscription, handleTransportFailure } from '../resumable';
+
+/** Timers that run only when the test advances the clock. */
+const createScheduler = () => {
+ let now = 0;
+ const timers = new Map<number, { at: number; callback: () => void }>();
+ let nextId = 1;
+ const scheduler: ResumableScheduler = {
+ setTimeout: (callback, delay) => {
+ const id = nextId++;
+ timers.set(id, { at: now + delay, callback });
+ return id as unknown as ReturnType<typeof setTimeout>;
+ },
+ clearTimeout: (handle) => {
+ timers.delete(handle as unknown as number);
+ },
+ requestFrame: () => nextId++,
+ cancelFrame: () => undefined,
+ now: () => now,
+ };
+ const advance = (ms: number) => {
+ now += ms;
+ for (const [id, timer] of [...timers]) {
+ if (timer.at <= now) {
+ timers.delete(id);
+ timer.callback();
+ }
+ }
+ };
+ return { scheduler, advance };
+};
+
+const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [string]>) => {
+ const host: ResumableHost = {
+ logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },
+ visibility: { isVisible: () => true, subscribe: () => () => undefined },
+ fetchStatus,
+ subscribe: jest.fn(),
+ cache: {
+ invalidateMessages: jest.fn(async () => undefined),
+ fetchMessages: jest.fn(async () => [] as TMessage[]),
+ refreshConversation: jest.fn(async () => undefined),
+ refreshConversationLists: jest.fn(async () => undefined),
+ refreshPinnedConversations: jest.fn(async () => undefined),
+ setStreamStatus: jest.fn(),
+ removeStreamStatus: jest.fn(),
+ invalidateStreamStatus: jest.fn(),
+ addActiveJob: jest.fn(),
+ removeActiveJob: jest.fn(),
+ fetchConversation: jest.fn(async () => undefined),
+ setConversation: jest.fn(),
+ upsertConversation: jest.fn(),
+ removeConversation: jest.fn(),
+ },
+ pane: {
+ setIsSubmitting: jest.fn(),
+ setShowStopButton: jest.fn(),
+ setSubmission: jest.fn(),
+ setStreamId: jest.fn(),
+ setGenerationEpoch: jest.fn(() => true),
+ setPendingAction: jest.fn(),
+ clearComposerDrafts: jest.fn(),
+ publishRunEnd: jest.fn(),
+ clearDrainAfterAbort: jest.fn(),
+ refreshBalance: jest.fn(),
+ updateConversation: jest.fn(),
+ resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),
+ },
+ steers: {
+ seed: jest.fn(),
+ appliedIds: jest.fn(() => []),
+ settleApplied: jest.fn(),
+ convertToQueued: jest.fn(),
+ convertLocalToQueued: jest.fn(),
+ },
+ events: {
+ clearStepMaps: jest.fn(),
+ flushPendingDeltas: jest.fn(),
+ resetLive: jest.fn(),
+ showError: jest.fn(),
+ },
+ messages: { get: jest.fn(() => []), set: jest.fn() },
+ };
+ return host;
+};
+
+const submission = {
+ userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true } as TMessage,
+ initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,
+ conversation: { conversationId: 'c-1' },
+} as TSubmission;
+
+const setup = () => {
+ const clock = createScheduler();
+ const fetchStatus = jest.fn<Promise<StreamStatusResponse>, [string]>();
+ const host = createHost(fetchStatus);
+ const run = new ResumableRun(clock.scheduler);
+ run.submission = submission;
+ const sub = new ResumableSubscription(run, host, {
+ streamId: 'c-1',
+ submission,
+ isResume: false,
+ generationCreatedAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ sub.claim();
+ const ctx: TerminalContext = { sub, host, applyPendingAction: jest.fn() };
+ return { clock, fetchStatus, host, run, sub, ctx };
+};
+
+describe('handleTransportFailure', () => {
+ it('ignores a failure after the final frame', async () => {
+ const { ctx, host } = setup();
+ ctx.sub.finalReceived = true;
+ await handleTransportFailure(ctx, { status: 0 });
+ expect(host.pane.refreshBalance).not.toHaveBeenCalled();
+ });
+
+ it('backs off and re-attaches a dropped connection', async () => {
+ const { ctx, host, run, clock } = setup();
+ await handleTransportFailure(ctx, { status: 0 });
+ expect(run.reconnectAttempt).toBe(1);
+ expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(true);
+ expect(host.pane.setShowStopButton).toHaveBeenCalledWith(true);
+ clock.advance(999);
+ expect(host.subscribe).not.toHaveBeenCalled();
+ clock.advance(1);
+ expect(host.subscribe).toHaveBeenCalledWith(expect.objectContaining({ isResume: true }));
+ });
+
+ it('writes a served error once the status proves the run over', async () => {
+ const { ctx, host, fetchStatus } = setup();
+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
+ await handleTransportFailure(ctx, {
+ data: { message: 'quota exceeded', generationProtocolVersion: 2 },
+ });
+ expect(ctx.sub.finalReceived).toBe(true);
+ expect(host.events.flushPendingDeltas).toHaveBeenCalled();
+ expect(host.events.showError).toHaveBeenCalledWith('quota exceeded', ctx.sub.submission);
+ expect(host.steers.convertLocalToQueued).toHaveBeenCalledWith('c-1', {
+ claimParked: true,
+ generationProtocolVersion: 2,
+ });
+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith({
+ conversationId: 'c-1',
+ outcome: 'error',
+ endedAt: 0,
+ generationCreatedAt: 2000,
+ });
+ });
+
+ it('settles a missing stream from persisted history', async () => {
+ const { ctx, host, fetchStatus } = setup();
+ const persisted = [submission.userMessage];
+ (host.cache.fetchMessages as jest.Mock).mockResolvedValue(persisted);
+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
+ await handleTransportFailure(ctx, { status: 404 });
+ expect(host.cache.fetchMessages).toHaveBeenCalledWith('c-1', true);
+ expect(host.cache.removeStreamStatus).toHaveBeenCalledWith('c-1');
+ expect(host.messages.set).toHaveBeenCalledWith(persisted);
+ expect(ctx.sub.retired).toBe(true);
+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: 'c-1', outcome: 'aborted' }),
+ { retainDetached: false },
+ );
+ });
+
+ it('keeps a missing stream attached while its generation is still live', async () => {
+ const { ctx, host, fetchStatus, clock } = setup();
+ fetchStatus.mockResolvedValue({
+ active: true,
+ streamId: 'c-1',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ await handleTransportFailure(ctx, { status: 404 });
+ expect(host.pane.publishRunEnd).not.toHaveBeenCalled();
+ expect(host.steers.seed).toHaveBeenCalledWith('c-1', [], 2000, 2);
+ clock.advance(1000);
+ expect(host.subscribe).toHaveBeenCalledWith(
+ expect.objectContaining({ streamId: 'c-1', generationCreatedAt: 2000 }),
+ );
+ });
+
+ it('waits on a live generation once the retry ceiling is reached', async () => {
+ const { ctx, host, run, fetchStatus } = setup();
+ run.reconnectAttempt = 5;
+ fetchStatus.mockResolvedValue({ active: true, createdAt: 2000, generationProtocolVersion: 2 });
+ await handleTransportFailure(ctx, { status: 0 });
+ expect(host.cache.setStreamStatus).toHaveBeenCalled();
+ expect(run.reconnectAttempt).toBe(0);
+ expect(host.pane.publishRunEnd).not.toHaveBeenCalled();
+ });
+
+ it('settles a run the status proves terminal past the retry ceiling', async () => {
+ const { ctx, host, run, fetchStatus } = setup();
+ run.reconnectAttempt = 5;
+ fetchStatus.mockResolvedValue({
+ active: false,
+ status: 'aborted',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ await handleTransportFailure(ctx, { status: 0 });
+ expect(ctx.sub.retired).toBe(true);
+ expect(host.pane.setSubmission).toHaveBeenCalledWith(null);
+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ outcome: 'aborted', generationCreatedAt: 2000 }),
+ );
+ });
+});
diff --git a/packages/chat/src/core/__tests__/terminal.spec.ts b/packages/chat/src/core/__tests__/terminal.spec.ts
index eeaa056285..3b9b7d1b00 100644
--- a/packages/chat/src/core/__tests__/terminal.spec.ts
+++ b/packages/chat/src/core/__tests__/terminal.spec.ts
@@ -60,6 +60,10 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
invalidateStreamStatus: jest.fn(),
addActiveJob: jest.fn(),
removeActiveJob: jest.fn(),
+ fetchConversation: jest.fn(async () => undefined),
+ setConversation: jest.fn(),
+ upsertConversation: jest.fn(),
+ removeConversation: jest.fn(),
},
pane: {
setIsSubmitting: jest.fn(),
@@ -71,6 +75,9 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
clearComposerDrafts: jest.fn(),
publishRunEnd: jest.fn(),
clearDrainAfterAbort: jest.fn(),
+ refreshBalance: jest.fn(),
+ updateConversation: jest.fn(),
+ resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),
},
steers: {
seed: jest.fn(),
@@ -79,7 +86,13 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strin
convertToQueued: jest.fn(),
convertLocalToQueued: jest.fn(),
},
- events: { clearStepMaps: jest.fn(), flushPendingDeltas: jest.fn(), resetLive: jest.fn() },
+ events: {
+ clearStepMaps: jest.fn(),
+ flushPendingDeltas: jest.fn(),
+ resetLive: jest.fn(),
+ showError: jest.fn(),
+ },
+ messages: { get: jest.fn(() => []), set: jest.fn() },
};
return host;
};
diff --git a/packages/chat/src/core/resumable/ceiling.ts b/packages/chat/src/core/resumable/ceiling.ts
new file mode 100644
index 0000000000..94cbde19d7
--- /dev/null
+++ b/packages/chat/src/core/resumable/ceiling.ts
@@ -0,0 +1,190 @@
+import type { TMessage } from 'librechat-data-provider';
+import type { StreamStatusResponse } from './host';
+import type { TerminalContext } from './terminal';
+import {
+ restoreLiveStatus,
+ handoffToReplacement,
+ clearPendingApprovalForTerminal,
+ clearAttachedGenerationCreatedAt,
+} from './terminal';
+import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
+import { completedResponseMessageId } from './messages';
+import { getRecoverySteerId } from './start';
+
+/**
+ * The connection failed past the last retry. The transport gave up, not necessarily the
+ * generation: a status read decides whether to keep waiting on it (or hand it to a newer
+ * epoch), and only a status that proves it terminal settles the run from durable history.
+ */
+export async function settleAfterRetryCeiling(ctx: TerminalContext): Promise<void> {
+ const { sub, host } = ctx;
+ const { run, generationCreatedAt, generationProtocolVersion } = sub;
+ const currentStreamId = sub.streamId;
+ host.logger.error('ResumableSSE', 'Max reconnect attempts reached');
+ sub.close();
+ host.events.flushPendingDeltas();
+ const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
+ let status: StreamStatusResponse | undefined;
+ try {
+ status = await host.fetchStatus(recoveryConvoId);
+ if (!sub.isCurrent()) {
+ return;
+ }
+ } catch (error) {
+ if (!sub.isCurrent()) {
+ return;
+ }
+ host.logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+
+ if (
+ status != null &&
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ !supportsGenerationProtocolV2(status)
+ ) {
+ // A legacy responder cannot adjudicate a v2 job during rollout.
+ status = undefined;
+ }
+
+ if (
+ status?.active === true &&
+ status.createdAt != null &&
+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)
+ ) {
+ const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);
+ if (!sub.isCurrent()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ run.reconnectAttempt = 0;
+ sub.scheduleReattach(30_000);
+ return;
+ }
+
+ if (status?.active === true) {
+ // The transport gave up, not the generation. Keep the active-job
+ // marker and accepted chips intact, then release this stale
+ // submission so useResumeOnLoad can establish a fresh attachment.
+ host.cache.setStreamStatus(recoveryConvoId, status);
+ restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);
+ host.pane.setIsSubmitting(true);
+ host.pane.setShowStopButton(generationCreatedAt != null);
+ run.reconnectAttempt = 0;
+ sub.scheduleReattach(30_000);
+ return;
+ }
+
+ if (status == null) {
+ // An inconclusive status read is still not proof of termination.
+ // Preserve server/client ownership and let the normal resume-on-load
+ // status query retry rather than duplicating accepted words.
+ host.cache.invalidateStreamStatus(recoveryConvoId);
+ host.pane.setIsSubmitting(true);
+ host.pane.setShowStopButton(generationCreatedAt != null);
+ run.reconnectAttempt = 0;
+ sub.scheduleReattach(30_000);
+ return;
+ }
+
+ // The status endpoint proved the job terminal. Reconcile persisted
+ // content before recovering leftovers, just like the 404 path.
+ run.cancelSteerRetryFrames();
+ let persistedMessages: TMessage[] | undefined;
+ try {
+ await host.cache.invalidateMessages(recoveryConvoId, 'none');
+ if (!sub.isCurrent()) {
+ return;
+ }
+ const fetched = await host.cache.fetchMessages(recoveryConvoId);
+ if (!sub.isCurrent()) {
+ return;
+ }
+ if (Array.isArray(fetched)) {
+ persistedMessages = fetched;
+ }
+ } catch (error) {
+ if (!sub.isCurrent()) {
+ return;
+ }
+ host.logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+
+ const authoritativeValues = [
+ ...(persistedMessages ?? []),
+ ...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),
+ ];
+ const confirmedV2Terminal =
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ supportsGenerationProtocolV2(status);
+ const reconciledIds = new Set(host.steers.appliedIds(authoritativeValues));
+ host.steers.settleApplied(recoveryConvoId, authoritativeValues);
+ const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
+ const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
+ if (unrecovered.length > 0) {
+ const recoverySteerId = getRecoverySteerId(sub.submission);
+ host.steers.convertToQueued(recoveryConvoId, unrecovered, {
+ generationProtocolVersion,
+ ...(recoverySteerId != null && {
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ }),
+ });
+ for (const steer of unrecovered) {
+ reconciledIds.add(steer.steerId);
+ if (steer.clientSteerId) {
+ reconciledIds.add(steer.clientSteerId);
+ }
+ }
+ }
+ if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
+ host.steers.convertLocalToQueued(recoveryConvoId, {
+ excludeSteerIds: reconciledIds,
+ generationProtocolVersion,
+ });
+ }
+
+ host.events.resetLive(sub.liveSubmission());
+ host.cache.removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt(ctx);
+ clearPendingApprovalForTerminal(ctx, recoveryConvoId);
+ if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
+ host.cache.removeConversation(currentStreamId);
+ }
+ sub.retired = true;
+ host.pane.setIsSubmitting(false);
+ host.pane.setShowStopButton(false);
+ let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';
+ if (status.status === 'complete' && persistedMessages != null) {
+ recoveryOutcome = 'completed';
+ } else if (status.status === 'aborted') {
+ recoveryOutcome = 'aborted';
+ }
+ const recoveredResponseMessageId =
+ recoveryOutcome === 'completed'
+ ? completedResponseMessageId(
+ persistedMessages,
+ sub.userMessage?.messageId,
+ status?.resumeState?.responseMessageId ?? sub.submission.initialResponse?.messageId,
+ )
+ : undefined;
+ host.pane.publishRunEnd({
+ conversationId: recoveryConvoId,
+ outcome: recoveryOutcome,
+ endedAt: run.scheduler.now(),
+ generationCreatedAt: status.createdAt ?? generationCreatedAt,
+ ...(recoveredResponseMessageId != null && {
+ responseMessageId: recoveredResponseMessageId,
+ }),
+ });
+ host.pane.setSubmission(null);
+ host.pane.setStreamId(null);
+ run.releaseStream(currentStreamId);
+ run.reconnectAttempt = 0;
+}
diff --git a/packages/chat/src/core/resumable/failure.ts b/packages/chat/src/core/resumable/failure.ts
new file mode 100644
index 0000000000..c423df933d
--- /dev/null
+++ b/packages/chat/src/core/resumable/failure.ts
@@ -0,0 +1,209 @@
+import { ErrorTypes, ViolationTypes } from 'librechat-data-provider';
+import type { ChatEvent, ChatErrorData } from 'librechat-data-provider';
+import type { TerminalContext } from './terminal';
+import {
+ handoffToReplacement,
+ authorizeTerminalTeardown,
+ clearPendingApprovalForTerminal,
+ clearAttachedGenerationCreatedAt,
+} from './terminal';
+import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
+import { MAX_RETRIES, getReconnectDelay, getRecoverySteerId } from './start';
+import { settleAfterRetryCeiling } from './ceiling';
+import { recoverMissingStream } from './missing';
+
+/**
+ * Server-sent error event (event: error with data) - no responseCode.
+ * These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.
+ */
+async function handleServedError(
+ ctx: TerminalContext,
+ data: ChatErrorData | string,
+): Promise<void> {
+ const { sub, host } = ctx;
+ const { run, generationCreatedAt, generationProtocolVersion } = sub;
+ const currentStreamId = sub.streamId;
+ sub.finalReceived = true;
+ const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;
+ if (!(await authorizeTerminalTeardown(ctx, recoveryConvoId, 'error')) || !sub.isCurrent()) {
+ return;
+ }
+ host.logger.log('ResumableSSE', 'Server-sent error event received:', data);
+ run.cancelSteerRetryFrames();
+ sub.close();
+ /** FLUSH (not cancel): the error card below is built from the cache
+ * tail, so queued tokens must land first, and a stale trailing
+ * frame must never overwrite the error write. */
+ host.events.flushPendingDeltas();
+ host.cache.removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt(ctx);
+ clearPendingApprovalForTerminal(ctx, recoveryConvoId);
+ host.events.resetLive(sub.liveSubmission());
+ if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
+ host.cache.removeConversation(currentStreamId);
+ }
+
+ const errorSupportsV2 = supportsGenerationProtocolV2(data);
+ const errorString =
+ typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));
+
+ // Check if it's a known error type (ViolationTypes or ErrorTypes)
+ let isKnownError = false;
+ try {
+ const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;
+ const errorType = parsed?.type ?? parsed?.code;
+ if (errorType) {
+ const violationValues = Object.values(ViolationTypes) as string[];
+ const errorTypeValues = Object.values(ErrorTypes) as string[];
+ isKnownError = violationValues.includes(errorType) || errorTypeValues.includes(errorType);
+ }
+ } catch {
+ // Not JSON or parsing failed - treat as generic error
+ }
+
+ host.logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });
+
+ // Display the error to user via errorHandler
+ host.events.showError(errorString, sub.submission);
+
+ host.pane.setIsSubmitting(false);
+ host.pane.setShowStopButton(false);
+ const recoverySteerId = getRecoverySteerId(sub.submission);
+ if (
+ recoverySteerId != null &&
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ errorSupportsV2
+ ) {
+ try {
+ /** The error was received on the exact generation-fenced stream.
+ * Read status only now, after terminal was observed: a claim
+ * started before this recovery could return a stale parked copy
+ * after a successful commit and must never authorize redelivery. */
+ const terminalStatus = await host.fetchStatus(recoveryConvoId);
+ if (!sub.isCurrent()) {
+ return;
+ }
+ const releasedSource = terminalStatus.unrecoveredSteers?.find(
+ (steer) => steer.steerId === recoverySteerId,
+ );
+ if (
+ terminalStatus.active === false &&
+ supportsGenerationProtocolV2(terminalStatus) &&
+ releasedSource != null
+ ) {
+ host.steers.convertToQueued(recoveryConvoId, [releasedSource], {
+ generationProtocolVersion,
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ });
+ }
+ } catch (error) {
+ if (!sub.isCurrent()) {
+ return;
+ }
+ host.logger.warn('ResumableSSE', 'Could not recover source after generation error', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+ }
+ // The error terminal's backstop parks acked leftovers server-side;
+ // claim it now so a reload can't resurrect the chips converted here.
+ if (generationProtocolVersion === 1 || errorSupportsV2) {
+ host.steers.convertLocalToQueued(recoveryConvoId, {
+ claimParked: true,
+ generationProtocolVersion,
+ });
+ }
+ host.pane.publishRunEnd({
+ conversationId: recoveryConvoId,
+ outcome: 'error',
+ endedAt: run.scheduler.now(),
+ generationCreatedAt,
+ });
+ host.pane.setStreamId(null);
+ run.releaseStream(currentStreamId);
+ run.reconnectAttempt = 0;
+}
+
+/**
+ * Error event handler - handles BOTH:
+ * 1. HTTP-level errors (status present) - 404, 409, network failures (0)
+ * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
+ *
+ * Order matters: check the status first since HTTP errors may also include data.
+ * The transport has already retried a 401 with a refreshed token.
+ */
+export async function handleTransportFailure(
+ ctx: TerminalContext,
+ { status: responseCode, data }: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>,
+): Promise<void> {
+ const { sub, host } = ctx;
+ const { run, generationCreatedAt } = sub;
+ if (!sub.isCurrent()) {
+ return;
+ }
+
+ if (sub.finalReceived) {
+ host.logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
+ responseCode,
+ hasData: data != null,
+ });
+ return;
+ }
+
+ host.pane.refreshBalance();
+
+ if (responseCode === 409) {
+ const handedOff = await handoffToReplacement(ctx, sub.conversationId());
+ if (!sub.isCurrent()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ }
+
+ // 404 → job completed & was cleaned up; messages are persisted in DB.
+ if (responseCode === 404) {
+ await recoverMissingStream(ctx);
+ return;
+ }
+
+ /**
+ * Only check the data if there's no HTTP status, since HTTP errors may also have body data.
+ * Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,
+ * not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).
+ */
+ if (responseCode == null && data != null) {
+ await handleServedError(ctx, data);
+ return;
+ }
+
+ // Network failure or unknown HTTP error - attempt reconnection with backoff
+ host.logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {
+ responseCode,
+ hasData: data != null,
+ });
+
+ if (run.reconnectAttempt < MAX_RETRIES) {
+ // Increment counter BEFORE close() so abort handler knows we're reconnecting
+ run.reconnectAttempt++;
+ const delay = getReconnectDelay(run.reconnectAttempt);
+
+ host.logger.log(
+ 'ResumableSSE',
+ `Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,
+ );
+
+ sub.close();
+
+ sub.scheduleReattach(delay);
+
+ // Keep UI in "submitting" state during reconnection attempts
+ // so user knows we're still trying (abort handler may have reset these)
+ host.pane.setIsSubmitting(true);
+ host.pane.setShowStopButton(generationCreatedAt != null);
+ } else {
+ await settleAfterRetryCeiling(ctx);
+ }
+}
diff --git a/packages/chat/src/core/resumable/host.ts b/packages/chat/src/core/resumable/host.ts
index 299a00a9dc..6594ec4fcf 100644
--- a/packages/chat/src/core/resumable/host.ts
+++ b/packages/chat/src/core/resumable/host.ts
@@ -1,4 +1,10 @@
-import type { Agents, TSubmission, TPendingSteer } from 'librechat-data-provider';
+import type {
+ Agents,
+ TMessage,
+ TSubmission,
+ TConversation,
+ TPendingSteer,
+} from 'librechat-data-provider';
import type { GenerationProtocolVersion } from '../protocol';
import type { RunEnd } from '../steer/types';
@@ -139,6 +145,13 @@ export interface ResumableCache {
invalidateStreamStatus: (conversationId: string) => void;
addActiveJob: (streamId: string) => void;
removeActiveJob: (streamId: string) => void;
+ /** Reads the conversation's saved record; rejects with the HTTP failure, a 404 included. */
+ fetchConversation: (conversationId: string) => Promise<TConversation | undefined>;
+ setConversation: (conversationId: string, conversation: TConversation) => void;
+ /** Places the conversation's snapshot into every conversation list. */
+ upsertConversation: (conversation: TConversation) => void;
+ /** Drops the conversation from every conversation list. */
+ removeConversation: (conversationId: string) => void;
}
/** The pane's controls and run bookkeeping a resumable run drives. */
@@ -164,6 +177,18 @@ export interface ResumablePane {
publishRunEnd: (end: RunEnd, options?: { retainDetached?: boolean }) => void;
/** Consumes an armed "interrupt & send" for exactly this generation. */
clearDrainAfterAbort: (conversationId: string, generationCreatedAt?: number) => void;
+ /** Refreshes the token balance, when the deployment tracks one. */
+ refreshBalance: () => void;
+ /** Updates the pane's conversation; absent for a pane that only adds a comparison response. */
+ updateConversation?: (update: (current: TConversation | null) => TConversation | null) => void;
+ /**
+ * The end of a run this pane left mid-stream and is now reattached to, as `messages` record
+ * it. `resolvable` is false when no such run is remembered or the user stopped it.
+ */
+ resolveDetachedEnd: (
+ conversationId: string,
+ messages: TMessage[] | undefined,
+ ) => { resolvable: boolean; end: RunEnd | null };
}
/** What the event handlers that render the stream expose to the run. */
@@ -172,6 +197,14 @@ export interface ResumableEvents {
flushPendingDeltas: () => void;
/** Discards the live usage estimate for a run that stopped streaming. */
resetLive: (submission: TSubmission) => void;
+ /** Writes the failed turn's error card from the text (or raw error value) the server sent. */
+ showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void;
+}
+
+/** The pane's message list. */
+export interface ResumableMessages {
+ get: () => TMessage[] | undefined;
+ set: (messages: TMessage[]) => void;
}
/** Everything a resumable run needs from the app it runs in. */
@@ -180,4 +213,5 @@ export interface ResumableHost extends SubscriptionHost {
pane: ResumablePane;
steers: ResumableSteers;
events: ResumableEvents;
+ messages: ResumableMessages;
}
diff --git a/packages/chat/src/core/resumable/index.ts b/packages/chat/src/core/resumable/index.ts
index 3ba5c14d0a..d94f8f3c90 100644
--- a/packages/chat/src/core/resumable/index.ts
+++ b/packages/chat/src/core/resumable/index.ts
@@ -8,3 +8,6 @@ export * from './host';
export * from './subscription';
export * from './terminal';
export * from './reconcile';
+export * from './failure';
+export * from './missing';
+export * from './ceiling';
diff --git a/packages/chat/src/core/resumable/missing.ts b/packages/chat/src/core/resumable/missing.ts
new file mode 100644
index 0000000000..a6ee87261a
--- /dev/null
+++ b/packages/chat/src/core/resumable/missing.ts
@@ -0,0 +1,242 @@
+import { Constants } from 'librechat-data-provider';
+import type { TMessage } from 'librechat-data-provider';
+import type { TerminalContext } from './terminal';
+import {
+ restoreLiveStatus,
+ handoffToReplacement,
+ clearPendingApprovalForTerminal,
+ clearAttachedGenerationCreatedAt,
+} from './terminal';
+import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';
+import { isInitialNewConversation, keepLocalCodeApprovalMode } from './messages';
+import { getRecoverySteerId, toStartGenerationError } from './start';
+
+/**
+ * A 404 from the stream route: the job completed and was cleaned up, and its messages are
+ * persisted. Refetches and reconciles them before recovering local steers (the final response
+ * may contain an applied steer whose live event was missed), unless a status read shows the
+ * generation, or a newer one, still running.
+ */
+export async function recoverMissingStream(ctx: TerminalContext): Promise<void> {
+ const { sub, host } = ctx;
+ const { run, generationCreatedAt, generationProtocolVersion } = sub;
+ const currentStreamId = sub.streamId;
+ const convoId = sub.submission.conversation?.conversationId;
+ host.logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
+ // Keep the UI non-idle while the status read decides whether this is
+ // terminal cleanup or a handoff to a newer generation epoch.
+ run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
+ run.cancelSteerRetryFrames();
+ sub.close();
+ /** Terminal: drop any in-flight live estimate so the gauge doesn't
+ * keep counting stale streamed output after the stream ends */
+ host.events.resetLive(sub.liveSubmission());
+ host.pane.clearComposerDrafts(convoId, {
+ includeNewChatDraft:
+ !convoId ||
+ convoId === Constants.NEW_CONVO ||
+ run.optimisticStreamIds.has(currentStreamId) ||
+ isInitialNewConversation(sub.submission),
+ });
+ host.events.clearStepMaps();
+ let persistedMessages: TMessage[] | undefined;
+ if (convoId) {
+ try {
+ await host.cache.invalidateMessages(convoId, 'none');
+ if (!sub.isCurrent()) {
+ return;
+ }
+ // The first stream can finish before CREATED mounts the saved-chat query.
+ const fetched = await host.cache.fetchMessages(convoId, true);
+ if (!sub.isCurrent()) {
+ return;
+ }
+ if (Array.isArray(fetched)) {
+ persistedMessages = fetched;
+ }
+ } catch (error) {
+ if (!sub.isCurrent()) {
+ return;
+ }
+ host.logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {
+ conversationId: convoId,
+ error,
+ });
+ }
+ host.cache.removeStreamStatus(convoId);
+ }
+ const recoveryConvoId = convoId ?? currentStreamId;
+ const reconciledIds = new Set(persistedMessages ? host.steers.appliedIds(persistedMessages) : []);
+ if (persistedMessages) {
+ host.steers.settleApplied(recoveryConvoId, persistedMessages);
+ }
+
+ // The status replay is owner-gated and non-destructive; creating a
+ // recovered follow-up consumes its exact source. Prefer these
+ // authoritative leftovers over guessing from local chips.
+ let confirmedV2Terminal = false;
+ try {
+ const status = await host.fetchStatus(recoveryConvoId);
+ if (!sub.isCurrent()) {
+ return;
+ }
+ if (
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ !supportsGenerationProtocolV2(status)
+ ) {
+ sub.scheduleReattach(1_000);
+ return;
+ }
+ if (
+ status.active === true &&
+ status.createdAt != null &&
+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)
+ ) {
+ const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);
+ if (!sub.isCurrent()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ /** Legacy generations have no replacement-handoff contract.
+ * Never attach this submission to a different epoch. */
+ sub.scheduleReattach(1_000);
+ return;
+ }
+ if (status.active === true) {
+ /** A stream lookup can briefly lose its routing record while the
+ * authoritative job status still proves the same generation is
+ * alive. Preserve ownership and retry the attachment instead of
+ * publishing a terminal run-end (which could auto-drain queued
+ * text into a still-running generation). */
+ const activeStreamId = status.streamId ?? currentStreamId;
+ host.cache.setStreamStatus(recoveryConvoId, status);
+ if (status.createdAt != null) {
+ host.pane.setGenerationEpoch(
+ recoveryConvoId,
+ status.createdAt,
+ undefined,
+ generationProtocolVersion,
+ );
+ }
+ host.cache.addActiveJob(activeStreamId);
+ restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);
+ host.pane.setIsSubmitting(true);
+ host.pane.setShowStopButton(status.createdAt != null);
+ run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);
+ sub.scheduleReattach(1_000, activeStreamId, status.createdAt);
+ return;
+ }
+ confirmedV2Terminal =
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ supportsGenerationProtocolV2(status);
+ const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
+ const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
+ if (unrecovered.length > 0) {
+ const recoverySteerId = getRecoverySteerId(sub.submission);
+ host.steers.convertToQueued(recoveryConvoId, unrecovered, {
+ generationProtocolVersion,
+ ...(recoverySteerId != null && {
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ }),
+ });
+ for (const steer of unrecovered) {
+ reconciledIds.add(steer.steerId);
+ if (steer.clientSteerId) {
+ reconciledIds.add(steer.clientSteerId);
+ }
+ }
+ }
+ } catch (error) {
+ if (!sub.isCurrent()) {
+ return;
+ }
+ host.logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+
+ host.cache.removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt(ctx);
+ clearPendingApprovalForTerminal(ctx, recoveryConvoId);
+ if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {
+ // Both fresh and resumed subscriptions can miss every event of a fast turn.
+ // A missing job proves neither that the conversation was saved nor that it failed.
+ try {
+ const persisted = await host.cache.fetchConversation(recoveryConvoId);
+ if (!sub.isCurrent()) return;
+ if (persisted?.conversationId === recoveryConvoId) {
+ host.cache.setConversation(recoveryConvoId, persisted);
+ host.cache.upsertConversation(persisted);
+ host.pane.updateConversation?.((current) => {
+ if (
+ current?.conversationId != null &&
+ current.conversationId !== Constants.NEW_CONVO &&
+ current.conversationId !== recoveryConvoId
+ )
+ return current;
+ return keepLocalCodeApprovalMode(
+ { ...current, ...persisted },
+ current,
+ current?.conversationId,
+ );
+ });
+ }
+ } catch (error) {
+ if (!sub.isCurrent()) return;
+ if (toStartGenerationError(error)?.response?.status === 404) {
+ host.cache.removeConversation(currentStreamId);
+ }
+ }
+ // An inconclusive read must not turn a saved chat into a phantom.
+ void host.cache.refreshConversationLists();
+ void host.cache.refreshPinnedConversations();
+ }
+ if (persistedMessages) {
+ host.messages.set(persistedMessages);
+ }
+ sub.retired = true;
+ host.pane.setIsSubmitting(false);
+ host.pane.setShowStopButton(false);
+
+ // Only infer recovery from local accepted chips after the persisted
+ // final was successfully inspected. If that fetch failed, their
+ // applied-vs-leftover outcome is unknown and converting could send
+ // already-applied words twice.
+ if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
+ host.steers.convertLocalToQueued(recoveryConvoId, {
+ excludeSteerIds: reconciledIds,
+ generationProtocolVersion,
+ });
+ }
+ // The true outcome is unknown here (job record already cleaned up):
+ // a non-'completed' outcome releases parked interrupt flags without
+ // auto-sending queued messages the user may not want fired. A run this
+ // pane left and came back to is the exception: the history just read
+ // names its response by epoch, which is the outcome it would have had.
+ // History read before the job finished may not hold that response yet, or
+ // may hold only the unfinished snapshot saved when the pane left, so a
+ // marker that read leaves unresolved or stopped outlives this end for the
+ // next visit, which reads history after the job is known to be done.
+ const detached = host.pane.resolveDetachedEnd(recoveryConvoId, persistedMessages);
+ const detachedResolvable = detached.resolvable;
+ const detachedEnd = detached.end;
+ host.pane.publishRunEnd(
+ {
+ conversationId: recoveryConvoId,
+ outcome: 'aborted',
+ endedAt: run.scheduler.now(),
+ ...detachedEnd,
+ startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),
+ },
+ {
+ retainDetached:
+ detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),
+ },
+ );
+ host.pane.setStreamId(null);
+ run.releaseStream(currentStreamId);
+ run.reconnectAttempt = 0;
+}