Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🛟 refactor: Move Resumable Transport Failure Handling Into the Chat Core (#17120)
Marco Berettad4e52acdfbb2
diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.tsindex d9008f45fa..de0479ac22 100644--- a/client/src/hooks/SSE/useResumableSSE.ts+++ b/client/src/hooks/SSE/useResumableSSE.ts@@ -8,7 +8,6 @@ import { ABORT_SWEEP_STATUSES, selectLocalSteersForQueue } from '@librechat/chatimport {Constants,QueryKeys,- ErrorTypes,StepEvents,StepTypes,SteerEvents,@@ -20,15 +19,12 @@ import {UsageEvents,createPayload,ApprovalEvents,- ViolationTypes,removeNullishValues,} from 'librechat-data-provider';import {- MAX_RETRIES,getStepEventId,toStartGenerationError,isOAuthStepEvent,- getReconnectDelay,waitForRetryDelay,applyPendingAction,getRecoverySteerId,@@ -48,7 +44,6 @@ import {keepLocalCodeApprovalMode,hydrateSubmissionMessages,buildResumeEventSubmission,- completedResponseMessageId,getStartGenerationStreamId,buildOptimisticConversation,buildCreatedInitialResponse,@@ -65,9 +60,9 @@ import {ResumableRun,documentVisibility,ResumableSubscription,- handoffToReplacement,authorizeTerminalTeardown,reconcileGenerationLifecycle,+ handleTransportFailure,clearPendingApprovalForTerminal,clearAttachedGenerationCreatedAt,} from '@librechat/chat';@@ -288,6 +283,16 @@ export default function useResumableSSE(queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(conversationId) }),addActiveJob,removeActiveJob,+ fetchConversation: (conversationId) =>+ fetchConvoSnapshot(queryClient, conversationId, () =>+ dataService.getConversationById(conversationId),+ ),+ setConversation: (conversationId, conversation) =>+ queryClient.setQueryData([QueryKeys.conversation, conversationId], conversation),+ upsertConversation: (conversation) =>+ upsertConvoInAllQueries(queryClient, conversation, true, 'snapshot'),+ removeConversation: (conversationId) =>+ removeConvoFromAllQueries(queryClient, conversationId),}),[addActiveJob, queryClient, removeActiveJob],);@@ -487,6 +492,23 @@ export default function useResumableSSE(clearComposerDrafts(runIndex, conversationId, options),publishRunEnd: setRunEnd,clearDrainAfterAbort,+ refreshBalance: () => {+ (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();+ },+ updateConversation: isAddedRequest ? undefined : setConversation,+ resolveDetachedEnd: (conversationId, messages) => {+ const detachedRun = jotaiStore.get(detachedRunByConvoId(conversationId));+ const resolvable =+ detachedRun != null &&+ !isDetachedRunManual(jotaiStore.get, conversationId, detachedRun);+ return {+ resolvable,+ end:+ resolvable && messages != null+ ? resolveDetachedRunEnd(conversationId, detachedRun, messages)+ : null,+ };+ },},steers: {seed: seedSteerChips,@@ -495,7 +517,17 @@ export default function useResumableSSE(convertToQueued: convertSteersToQueued,convertLocalToQueued: convertLocalSteersToQueued,},- events: { clearStepMaps, flushPendingDeltas, resetLive },+ events: {+ clearStepMaps,+ flushPendingDeltas,+ resetLive,+ showError: (text, errorSubmission) =>+ errorHandler({+ data: { text } as unknown as Parameters<typeof errorHandler>[0]['data'],+ submission: errorSubmission as EventSubmission,+ }),+ },+ messages: { get: getMessages, set: setMessages },};const sub = new ResumableSubscription(run, host, target);const {@@ -1587,644 +1619,6 @@ export default function useResumableSSE(}};- /**- * Error event handler - handles BOTH:- * 1. HTTP-level errors (status present) - 404, 409, network failures (0)- * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes- *- * Order matters: check the status first since HTTP errors may also include data.- * The transport has already retried a 401 with a refreshed token.- */- const handleTransportFailure = async ({- status: responseCode,- data,- }: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>) => {- if (!sub.isCurrent()) {- return;- }-- if (sub.finalReceived) {- logger.log('ResumableSSE', 'Ignoring error after FINAL event', {- responseCode,- hasData: data != null,- });- return;- }-- (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();-- if (responseCode === 409) {- const replacementConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;- const handedOff = await handoffToReplacement(terminal, replacementConvoId);- if (!sub.isCurrent()) {- return;- }- if (handedOff) {- return;- }- }-- // 404 → job completed & was cleaned up; messages are persisted in DB.- // Refetch and reconcile them before recovering local steers: the final- // response may contain an applied steer whose live event was missed.- if (responseCode === 404) {- const convoId = sub.submission.conversation?.conversationId;- logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);- // Keep the UI non-idle while the status read decides whether this is- // terminal cleanup or a handoff to a newer generation epoch.- run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);- run.cancelSteerRetryFrames();- sub.close();- /** Terminal: drop any in-flight live estimate so the gauge doesn't- * keep counting stale streamed output after the stream ends */- resetLive(sub.liveSubmission());- clearComposerDrafts(runIndex, convoId, {- includeNewChatDraft:- !convoId ||- convoId === Constants.NEW_CONVO ||- run.optimisticStreamIds.has(currentStreamId) ||- isInitialNewConversation(sub.submission),- });- clearStepMaps();- let persistedMessages: TMessage[] | undefined;- if (convoId) {- try {- const messageQueryKey = [QueryKeys.messages, convoId] as const;- await queryClient.invalidateQueries({- queryKey: messageQueryKey,- refetchType: 'none',- });- if (!sub.isCurrent()) {- return;- }- const fetched = await queryClient.fetchQuery<TMessage[]>({- queryKey: messageQueryKey,- // The first stream can finish before CREATED mounts the saved-chat query.- queryFn: () => fetchConversationMessages(convoId),- });- if (!sub.isCurrent()) {- return;- }- if (Array.isArray(fetched)) {- persistedMessages = fetched;- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {- conversationId: convoId,- error,- });- }- queryClient.removeQueries({ queryKey: streamStatusQueryKey(convoId) });- }- const recoveryConvoId = convoId ?? currentStreamId;- const reconciledIds = new Set(- persistedMessages ? collectAppliedSteerIds(persistedMessages) : [],- );- if (persistedMessages) {- settleAppliedSteerParts(recoveryConvoId, persistedMessages);- }-- // The status replay is owner-gated and non-destructive; creating a- // recovered follow-up consumes its exact source. Prefer these- // authoritative leftovers over guessing from local chips.- let confirmedV2Terminal = false;- try {- const status = await fetchStreamStatus(recoveryConvoId);- if (!sub.isCurrent()) {- return;- }- if (- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&- !supportsGenerationProtocolV2(status)- ) {- sub.scheduleReattach(1_000);- return;- }- if (- status.active === true &&- status.createdAt != null &&- (generationCreatedAt == null || status.createdAt !== generationCreatedAt)- ) {- const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);- if (!sub.isCurrent()) {- return;- }- if (handedOff) {- return;- }- /** Legacy generations have no replacement-handoff contract.- * Never attach this submission to a different epoch. */- sub.scheduleReattach(1_000);- return;- }- if (status.active === true) {- /** A stream lookup can briefly lose its routing record while the- * authoritative job status still proves the same generation is- * alive. Preserve ownership and retry the attachment instead of- * publishing a terminal run-end (which could auto-drain queued- * text into a still-running generation). */- const activeStreamId = status.streamId ?? currentStreamId;- queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);- if (status.createdAt != null) {- updateActiveGenerationCreatedAt(- recoveryConvoId,- status.createdAt,- undefined,- generationProtocolVersion,- );- }- addActiveJob(activeStreamId);- seedSteerChips(- recoveryConvoId,- (status.resumeState?.pendingSteers ?? []) as TPendingSteer[],- status.createdAt,- generationProtocolVersion,- );- settleAppliedSteerParts(- recoveryConvoId,- status.resumeState?.aggregatedContent ?? status.aggregatedContent,- );- if (status.pendingAction ?? status.resumeState?.pendingAction) {- applyPendingActionToMessages(- (status.pendingAction ??- status.resumeState?.pendingAction) as Agents.PendingAction,- );- }- setIsSubmitting(true);- setShowStopButton(status.createdAt != null);- run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);- sub.scheduleReattach(1_000, activeStreamId, status.createdAt);- return;- }- confirmedV2Terminal =- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&- supportsGenerationProtocolV2(status);- const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;- const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];- if (unrecovered.length > 0) {- const recoverySteerId = getRecoverySteerId(sub.submission);- convertSteersToQueued(recoveryConvoId, unrecovered, {- generationProtocolVersion,- ...(recoverySteerId != null && {- allowPreviouslyConvertedIds: [recoverySteerId],- }),- });- for (const steer of unrecovered) {- reconciledIds.add(steer.steerId);- if (steer.clientSteerId) {- reconciledIds.add(steer.clientSteerId);- }- }- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {- conversationId: recoveryConvoId,- error,- });- }-- removeActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt(terminal);- clearPendingApprovalForTerminal(terminal, recoveryConvoId);- if (- !run.createdStreamIds.has(currentStreamId) &&- run.optimisticStreamIds.has(currentStreamId)- ) {- // Both fresh and resumed subscriptions can miss every event of a fast turn.- // A missing job proves neither that the conversation was saved nor that it failed.- try {- const persisted = await fetchConvoSnapshot(queryClient, recoveryConvoId, () =>- dataService.getConversationById(recoveryConvoId),- );- if (!sub.isCurrent()) return;- if (persisted?.conversationId === recoveryConvoId) {- queryClient.setQueryData([QueryKeys.conversation, recoveryConvoId], persisted);- upsertConvoInAllQueries(queryClient, persisted, true, 'snapshot');- if (!isAddedRequest) {- setConversation?.((current) => {- if (- current?.conversationId != null &&- current.conversationId !== Constants.NEW_CONVO &&- current.conversationId !== recoveryConvoId- )- return current;- return keepLocalCodeApprovalMode(- { ...current, ...persisted },- current,- current?.conversationId,- );- });- }- }- } catch (error) {- if (!sub.isCurrent()) return;- if (toStartGenerationError(error)?.response?.status === 404) {- removeConvoFromAllQueries(queryClient, currentStreamId);- }- }- // An inconclusive read must not turn a saved chat into a phantom.- invalidateConversationLists(queryClient);- queryClient.invalidateQueries({ queryKey: [QueryKeys.pinnedConversations] });- }- if (persistedMessages) {- setMessages(persistedMessages);- }- sub.retired = true;- setIsSubmitting(false);- setShowStopButton(false);-- // Only infer recovery from local accepted chips after the persisted- // final was successfully inspected. If that fetch failed, their- // applied-vs-leftover outcome is unknown and converting could send- // already-applied words twice.- if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {- convertLocalSteersToQueued(recoveryConvoId, {- excludeSteerIds: reconciledIds,- generationProtocolVersion,- });- }- // The true outcome is unknown here (job record already cleaned up):- // a non-'completed' outcome releases parked interrupt flags without- // auto-sending queued messages the user may not want fired. A run this- // pane left and came back to is the exception: the history just read- // names its response by epoch, which is the outcome it would have had.- // History read before the job finished may not hold that response yet, or- // may hold only the unfinished snapshot saved when the pane left, so a- // marker that read leaves unresolved or stopped outlives this end for the- // next visit, which reads history after the job is known to be done.- const detachedRun = jotaiStore.get(detachedRunByConvoId(recoveryConvoId));- const detachedResolvable =- detachedRun != null &&- !isDetachedRunManual(jotaiStore.get, recoveryConvoId, detachedRun);- const detachedEnd =- detachedResolvable && persistedMessages != null- ? resolveDetachedRunEnd(recoveryConvoId, detachedRun, persistedMessages)- : null;- setRunEnd(- {- conversationId: recoveryConvoId,- outcome: 'aborted',- endedAt: run.scheduler.now(),- ...detachedEnd,- startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),- },- {- retainDetached:- detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),- },- );- setStreamId(null);- run.releaseStream(currentStreamId);- run.reconnectAttempt = 0;- return;- }-- /**- * Server-sent error event (event: error with data) - no responseCode.- * These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.- * Only check the data if there's no HTTP status, since HTTP errors may also have body data.- * Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,- * not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).- */- if (responseCode == null && data != null) {- sub.finalReceived = true;- const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;- if (- !(await authorizeTerminalTeardown(terminal, recoveryConvoId, 'error')) ||- !sub.isCurrent()- ) {- return;- }- logger.log('ResumableSSE', 'Server-sent error event received:', data);- run.cancelSteerRetryFrames();- sub.close();- /** FLUSH (not cancel): the error card below is built from the cache- * tail, so queued tokens must land first, and a stale trailing- * frame must never overwrite the error write. */- flushPendingDeltas();- removeActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt(terminal);- clearPendingApprovalForTerminal(terminal, recoveryConvoId);- resetLive(sub.liveSubmission());- if (- !run.createdStreamIds.has(currentStreamId) &&- run.optimisticStreamIds.has(currentStreamId)- ) {- removeConvoFromAllQueries(queryClient, currentStreamId);- }-- const errorSupportsV2 = supportsGenerationProtocolV2(data);- const errorString =- typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));-- // Check if it's a known error type (ViolationTypes or ErrorTypes)- let isKnownError = false;- try {- const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;- const errorType = parsed?.type ?? parsed?.code;- if (errorType) {- const violationValues = Object.values(ViolationTypes) as string[];- const errorTypeValues = Object.values(ErrorTypes) as string[];- isKnownError =- violationValues.includes(errorType) || errorTypeValues.includes(errorType);- }- } catch {- // Not JSON or parsing failed - treat as generic error- }-- logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });-- // Display the error to user via errorHandler- errorHandler({- data: { text: errorString } as unknown as Parameters<typeof errorHandler>[0]['data'],- submission: sub.submission as EventSubmission,- });-- setIsSubmitting(false);- setShowStopButton(false);- const recoverySteerId = getRecoverySteerId(sub.submission);- if (- recoverySteerId != null &&- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&- errorSupportsV2- ) {- try {- /** The error was received on the exact generation-fenced stream.- * Read status only now, after terminal was observed: a claim- * started before this recovery could return a stale parked copy- * after a successful commit and must never authorize redelivery. */- const terminalStatus = await fetchStreamStatus(recoveryConvoId);- if (!sub.isCurrent()) {- return;- }- const releasedSource = terminalStatus.unrecoveredSteers?.find(- (steer) => steer.steerId === recoverySteerId,- );- if (- terminalStatus.active === false &&- supportsGenerationProtocolV2(terminalStatus) &&- releasedSource != null- ) {- convertSteersToQueued(recoveryConvoId, [releasedSource], {- generationProtocolVersion,- allowPreviouslyConvertedIds: [recoverySteerId],- });- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not recover source after generation error', {- conversationId: recoveryConvoId,- error,- });- }- }- // The error terminal's backstop parks acked leftovers server-side;- // claim it now so a reload can't resurrect the chips converted here.- if (generationProtocolVersion === 1 || errorSupportsV2) {- convertLocalSteersToQueued(recoveryConvoId, {- claimParked: true,- generationProtocolVersion,- });- }- setRunEnd({- conversationId: recoveryConvoId,- outcome: 'error',- endedAt: run.scheduler.now(),- generationCreatedAt,- });- setStreamId(null);- run.releaseStream(currentStreamId);- run.reconnectAttempt = 0;- return;- }-- // Network failure or unknown HTTP error - attempt reconnection with backoff- logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {- responseCode,- hasData: data != null,- });-- if (run.reconnectAttempt < MAX_RETRIES) {- // Increment counter BEFORE close() so abort handler knows we're reconnecting- run.reconnectAttempt++;- const delay = getReconnectDelay(run.reconnectAttempt);-- logger.log(- 'ResumableSSE',- `Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,- );-- sub.close();-- sub.scheduleReattach(delay);-- // Keep UI in "submitting" state during reconnection attempts- // so user knows we're still trying (abort handler may have reset these)- setIsSubmitting(true);- setShowStopButton(generationCreatedAt != null);- } else {- logger.error('ResumableSSE', 'Max reconnect attempts reached');- sub.close();- flushPendingDeltas();- const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;- let status: Awaited<ReturnType<typeof fetchStreamStatus>> | undefined;- try {- status = await fetchStreamStatus(recoveryConvoId);- if (!sub.isCurrent()) {- return;- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {- conversationId: recoveryConvoId,- error,- });- }-- if (- status != null &&- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&- !supportsGenerationProtocolV2(status)- ) {- // A legacy responder cannot adjudicate a v2 job during rollout.- status = undefined;- }-- if (- status?.active === true &&- status.createdAt != null &&- (generationCreatedAt == null || status.createdAt !== generationCreatedAt)- ) {- const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);- if (!sub.isCurrent()) {- return;- }- if (handedOff) {- return;- }- run.reconnectAttempt = 0;- sub.scheduleReattach(30_000);- return;- }-- if (status?.active === true) {- // The transport gave up, not the generation. Keep the active-job- // marker and accepted chips intact, then release this stale- // submission so useResumeOnLoad can establish a fresh attachment.- queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);- seedSteerChips(- recoveryConvoId,- (status.resumeState?.pendingSteers ?? []) as TPendingSteer[],- status.createdAt,- generationProtocolVersion,- );- settleAppliedSteerParts(- recoveryConvoId,- status.resumeState?.aggregatedContent ?? status.aggregatedContent,- );- if (status.pendingAction ?? status.resumeState?.pendingAction) {- applyPendingActionToMessages(- (status.pendingAction ?? status.resumeState?.pendingAction) as Agents.PendingAction,- );- }- setIsSubmitting(true);- setShowStopButton(generationCreatedAt != null);- run.reconnectAttempt = 0;- sub.scheduleReattach(30_000);- return;- }-- if (status == null) {- // An inconclusive status read is still not proof of termination.- // Preserve server/client ownership and let the normal resume-on-load- // status query retry rather than duplicating accepted words.- queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(recoveryConvoId) });- setIsSubmitting(true);- setShowStopButton(generationCreatedAt != null);- run.reconnectAttempt = 0;- sub.scheduleReattach(30_000);- return;- }-- // The status endpoint proved the job terminal. Reconcile persisted- // content before recovering leftovers, just like the 404 path.- run.cancelSteerRetryFrames();- let persistedMessages: TMessage[] | undefined;- try {- const messageQueryKey = [QueryKeys.messages, recoveryConvoId] as const;- await queryClient.invalidateQueries({- queryKey: messageQueryKey,- refetchType: 'none',- });- if (!sub.isCurrent()) {- return;- }- const fetched = await queryClient.fetchQuery<TMessage[]>({- queryKey: messageQueryKey,- });- if (!sub.isCurrent()) {- return;- }- if (Array.isArray(fetched)) {- persistedMessages = fetched;- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {- conversationId: recoveryConvoId,- error,- });- }-- const authoritativeValues = [- ...(persistedMessages ?? []),- ...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),- ];- const confirmedV2Terminal =- generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&- supportsGenerationProtocolV2(status);- const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues));- settleAppliedSteerParts(recoveryConvoId, authoritativeValues);- const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;- const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];- if (unrecovered.length > 0) {- const recoverySteerId = getRecoverySteerId(sub.submission);- convertSteersToQueued(recoveryConvoId, unrecovered, {- generationProtocolVersion,- ...(recoverySteerId != null && {- allowPreviouslyConvertedIds: [recoverySteerId],- }),- });- for (const steer of unrecovered) {- reconciledIds.add(steer.steerId);- if (steer.clientSteerId) {- reconciledIds.add(steer.clientSteerId);- }- }- }- if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {- convertLocalSteersToQueued(recoveryConvoId, {- excludeSteerIds: reconciledIds,- generationProtocolVersion,- });- }-- resetLive(sub.liveSubmission());- removeActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt(terminal);- clearPendingApprovalForTerminal(terminal, recoveryConvoId);- if (- !run.createdStreamIds.has(currentStreamId) &&- run.optimisticStreamIds.has(currentStreamId)- ) {- removeConvoFromAllQueries(queryClient, currentStreamId);- }- sub.retired = true;- setIsSubmitting(false);- setShowStopButton(false);- let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';- if (status.status === 'complete' && persistedMessages != null) {- recoveryOutcome = 'completed';- } else if (status.status === 'aborted') {- recoveryOutcome = 'aborted';- }- const recoveredResponseMessageId =- recoveryOutcome === 'completed'- ? completedResponseMessageId(- persistedMessages,- sub.userMessage?.messageId,- status?.resumeState?.responseMessageId ??- sub.submission.initialResponse?.messageId,- )- : undefined;- setRunEnd({- conversationId: recoveryConvoId,- outcome: recoveryOutcome,- endedAt: run.scheduler.now(),- generationCreatedAt: status.createdAt ?? generationCreatedAt,- ...(recoveredResponseMessageId != null && {- responseMessageId: recoveredResponseMessageId,- }),- });- setSubmission(null);- setStreamId(null);- run.releaseStream(currentStreamId);- run.reconnectAttempt = 0;- }- };-/*** The transport emits `abort` only for this hook's own closes. A cancel* the user agent issued (a backgrounded or frozen mobile tab, while the@@ -2277,7 +1671,7 @@ export default function useResumableSSE(sub.connect(transport.stream({ token }), {onOpen: handleOpen,onFrame: (event) => void handleFrame(event),- onFailure: (event) => void handleTransportFailure(event),+ onFailure: (event) => void handleTransportFailure(terminal, event),onClose: handleClose,});@@ -2291,7 +1685,7 @@ export default function useResumableSSE(/** Simulate network drop - triggers error event → reconnection */debugWindow.__killNetwork = () => {logger.log('Debug', 'Simulating network drop...');- void handleTransportFailure({});+ void handleTransportFailure(terminal, {});};/** Simulate clean close (navigation away) - triggers abort event → no reconnection */@@ -2332,7 +1726,6 @@ export default function useResumableSSE(startupConfig?.balance?.enabled,balanceQuery,removeActiveJob,- queryClient,bindResponse,contextHandler,usageHandler,@@ -2351,7 +1744,6 @@ export default function useResumableSSE(settleAppliedSteerParts,convertSteersToQueued,convertLocalSteersToQueued,- addActiveJob,setSubmission,updateActiveGenerationCreatedAt,jotaiStore,diff --git a/packages/chat/src/core/__tests__/failure.spec.ts b/packages/chat/src/core/__tests__/failure.spec.tsnew file mode 100644index 0000000000..ee6d2a342c--- /dev/null+++ b/packages/chat/src/core/__tests__/failure.spec.ts@@ -0,0 +1,218 @@+import type { TMessage, TSubmission } from 'librechat-data-provider';+import type {+ ResumableHost,+ TerminalContext,+ ResumableScheduler,+ StreamStatusResponse,+} from '../resumable';+import { ResumableRun, ResumableSubscription, handleTransportFailure } from '../resumable';++/** Timers that run only when the test advances the clock. */+const createScheduler = () => {+ let now = 0;+ const timers = new Map<number, { at: number; callback: () => void }>();+ let nextId = 1;+ const scheduler: ResumableScheduler = {+ setTimeout: (callback, delay) => {+ const id = nextId++;+ timers.set(id, { at: now + delay, callback });+ return id as unknown as ReturnType<typeof setTimeout>;+ },+ clearTimeout: (handle) => {+ timers.delete(handle as unknown as number);+ },+ requestFrame: () => nextId++,+ cancelFrame: () => undefined,+ now: () => now,+ };+ const advance = (ms: number) => {+ now += ms;+ for (const [id, timer] of [...timers]) {+ if (timer.at <= now) {+ timers.delete(id);+ timer.callback();+ }+ }+ };+ return { scheduler, advance };+};++const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [string]>) => {+ const host: ResumableHost = {+ logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },+ visibility: { isVisible: () => true, subscribe: () => () => undefined },+ fetchStatus,+ subscribe: jest.fn(),+ cache: {+ invalidateMessages: jest.fn(async () => undefined),+ fetchMessages: jest.fn(async () => [] as TMessage[]),+ refreshConversation: jest.fn(async () => undefined),+ refreshConversationLists: jest.fn(async () => undefined),+ refreshPinnedConversations: jest.fn(async () => undefined),+ setStreamStatus: jest.fn(),+ removeStreamStatus: jest.fn(),+ invalidateStreamStatus: jest.fn(),+ addActiveJob: jest.fn(),+ removeActiveJob: jest.fn(),+ fetchConversation: jest.fn(async () => undefined),+ setConversation: jest.fn(),+ upsertConversation: jest.fn(),+ removeConversation: jest.fn(),+ },+ pane: {+ setIsSubmitting: jest.fn(),+ setShowStopButton: jest.fn(),+ setSubmission: jest.fn(),+ setStreamId: jest.fn(),+ setGenerationEpoch: jest.fn(() => true),+ setPendingAction: jest.fn(),+ clearComposerDrafts: jest.fn(),+ publishRunEnd: jest.fn(),+ clearDrainAfterAbort: jest.fn(),+ refreshBalance: jest.fn(),+ updateConversation: jest.fn(),+ resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),+ },+ steers: {+ seed: jest.fn(),+ appliedIds: jest.fn(() => []),+ settleApplied: jest.fn(),+ convertToQueued: jest.fn(),+ convertLocalToQueued: jest.fn(),+ },+ events: {+ clearStepMaps: jest.fn(),+ flushPendingDeltas: jest.fn(),+ resetLive: jest.fn(),+ showError: jest.fn(),+ },+ messages: { get: jest.fn(() => []), set: jest.fn() },+ };+ return host;+};++const submission = {+ userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true } as TMessage,+ initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,+ conversation: { conversationId: 'c-1' },+} as TSubmission;++const setup = () => {+ const clock = createScheduler();+ const fetchStatus = jest.fn<Promise<StreamStatusResponse>, [string]>();+ const host = createHost(fetchStatus);+ const run = new ResumableRun(clock.scheduler);+ run.submission = submission;+ const sub = new ResumableSubscription(run, host, {+ streamId: 'c-1',+ submission,+ isResume: false,+ generationCreatedAt: 2000,+ generationProtocolVersion: 2,+ });+ sub.claim();+ const ctx: TerminalContext = { sub, host, applyPendingAction: jest.fn() };+ return { clock, fetchStatus, host, run, sub, ctx };+};++describe('handleTransportFailure', () => {+ it('ignores a failure after the final frame', async () => {+ const { ctx, host } = setup();+ ctx.sub.finalReceived = true;+ await handleTransportFailure(ctx, { status: 0 });+ expect(host.pane.refreshBalance).not.toHaveBeenCalled();+ });++ it('backs off and re-attaches a dropped connection', async () => {+ const { ctx, host, run, clock } = setup();+ await handleTransportFailure(ctx, { status: 0 });+ expect(run.reconnectAttempt).toBe(1);+ expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(true);+ expect(host.pane.setShowStopButton).toHaveBeenCalledWith(true);+ clock.advance(999);+ expect(host.subscribe).not.toHaveBeenCalled();+ clock.advance(1);+ expect(host.subscribe).toHaveBeenCalledWith(expect.objectContaining({ isResume: true }));+ });++ it('writes a served error once the status proves the run over', async () => {+ const { ctx, host, fetchStatus } = setup();+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });+ await handleTransportFailure(ctx, {+ data: { message: 'quota exceeded', generationProtocolVersion: 2 },+ });+ expect(ctx.sub.finalReceived).toBe(true);+ expect(host.events.flushPendingDeltas).toHaveBeenCalled();+ expect(host.events.showError).toHaveBeenCalledWith('quota exceeded', ctx.sub.submission);+ expect(host.steers.convertLocalToQueued).toHaveBeenCalledWith('c-1', {+ claimParked: true,+ generationProtocolVersion: 2,+ });+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith({+ conversationId: 'c-1',+ outcome: 'error',+ endedAt: 0,+ generationCreatedAt: 2000,+ });+ });++ it('settles a missing stream from persisted history', async () => {+ const { ctx, host, fetchStatus } = setup();+ const persisted = [submission.userMessage];+ (host.cache.fetchMessages as jest.Mock).mockResolvedValue(persisted);+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });+ await handleTransportFailure(ctx, { status: 404 });+ expect(host.cache.fetchMessages).toHaveBeenCalledWith('c-1', true);+ expect(host.cache.removeStreamStatus).toHaveBeenCalledWith('c-1');+ expect(host.messages.set).toHaveBeenCalledWith(persisted);+ expect(ctx.sub.retired).toBe(true);+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith(+ expect.objectContaining({ conversationId: 'c-1', outcome: 'aborted' }),+ { retainDetached: false },+ );+ });++ it('keeps a missing stream attached while its generation is still live', async () => {+ const { ctx, host, fetchStatus, clock } = setup();+ fetchStatus.mockResolvedValue({+ active: true,+ streamId: 'c-1',+ createdAt: 2000,+ generationProtocolVersion: 2,+ });+ await handleTransportFailure(ctx, { status: 404 });+ expect(host.pane.publishRunEnd).not.toHaveBeenCalled();+ expect(host.steers.seed).toHaveBeenCalledWith('c-1', [], 2000, 2);+ clock.advance(1000);+ expect(host.subscribe).toHaveBeenCalledWith(+ expect.objectContaining({ streamId: 'c-1', generationCreatedAt: 2000 }),+ );+ });++ it('waits on a live generation once the retry ceiling is reached', async () => {+ const { ctx, host, run, fetchStatus } = setup();+ run.reconnectAttempt = 5;+ fetchStatus.mockResolvedValue({ active: true, createdAt: 2000, generationProtocolVersion: 2 });+ await handleTransportFailure(ctx, { status: 0 });+ expect(host.cache.setStreamStatus).toHaveBeenCalled();+ expect(run.reconnectAttempt).toBe(0);+ expect(host.pane.publishRunEnd).not.toHaveBeenCalled();+ });++ it('settles a run the status proves terminal past the retry ceiling', async () => {+ const { ctx, host, run, fetchStatus } = setup();+ run.reconnectAttempt = 5;+ fetchStatus.mockResolvedValue({+ active: false,+ status: 'aborted',+ createdAt: 2000,+ generationProtocolVersion: 2,+ });+ await handleTransportFailure(ctx, { status: 0 });+ expect(ctx.sub.retired).toBe(true);+ expect(host.pane.setSubmission).toHaveBeenCalledWith(null);+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith(+ expect.objectContaining({ outcome: 'aborted', generationCreatedAt: 2000 }),+ );+ });+});diff --git a/packages/chat/src/core/__tests__/terminal.spec.ts b/packages/chat/src/core/__tests__/terminal.spec.tsindex eeaa056285..3b9b7d1b00 100644--- a/packages/chat/src/core/__tests__/terminal.spec.ts+++ b/packages/chat/src/core/__tests__/terminal.spec.ts@@ -60,6 +60,10 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strininvalidateStreamStatus: jest.fn(),addActiveJob: jest.fn(),removeActiveJob: jest.fn(),+ fetchConversation: jest.fn(async () => undefined),+ setConversation: jest.fn(),+ upsertConversation: jest.fn(),+ removeConversation: jest.fn(),},pane: {setIsSubmitting: jest.fn(),@@ -71,6 +75,9 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strinclearComposerDrafts: jest.fn(),publishRunEnd: jest.fn(),clearDrainAfterAbort: jest.fn(),+ refreshBalance: jest.fn(),+ updateConversation: jest.fn(),+ resolveDetachedEnd: jest.fn(() => ({ resolvable: false, end: null })),},steers: {seed: jest.fn(),@@ -79,7 +86,13 @@ const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [strinconvertToQueued: jest.fn(),convertLocalToQueued: jest.fn(),},- events: { clearStepMaps: jest.fn(), flushPendingDeltas: jest.fn(), resetLive: jest.fn() },+ events: {+ clearStepMaps: jest.fn(),+ flushPendingDeltas: jest.fn(),+ resetLive: jest.fn(),+ showError: jest.fn(),+ },+ messages: { get: jest.fn(() => []), set: jest.fn() },};return host;};diff --git a/packages/chat/src/core/resumable/ceiling.ts b/packages/chat/src/core/resumable/ceiling.tsnew file mode 100644index 0000000000..94cbde19d7--- /dev/null+++ b/packages/chat/src/core/resumable/ceiling.ts@@ -0,0 +1,190 @@+import type { TMessage } from 'librechat-data-provider';+import type { StreamStatusResponse } from './host';+import type { TerminalContext } from './terminal';+import {+ restoreLiveStatus,+ handoffToReplacement,+ clearPendingApprovalForTerminal,+ clearAttachedGenerationCreatedAt,+} from './terminal';+import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';+import { completedResponseMessageId } from './messages';+import { getRecoverySteerId } from './start';++/**+ * The connection failed past the last retry. The transport gave up, not necessarily the+ * generation: a status read decides whether to keep waiting on it (or hand it to a newer+ * epoch), and only a status that proves it terminal settles the run from durable history.+ */+export async function settleAfterRetryCeiling(ctx: TerminalContext): Promise<void> {+ const { sub, host } = ctx;+ const { run, generationCreatedAt, generationProtocolVersion } = sub;+ const currentStreamId = sub.streamId;+ host.logger.error('ResumableSSE', 'Max reconnect attempts reached');+ sub.close();+ host.events.flushPendingDeltas();+ const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;+ let status: StreamStatusResponse | undefined;+ try {+ status = await host.fetchStatus(recoveryConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {+ conversationId: recoveryConvoId,+ error,+ });+ }++ if (+ status != null &&+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&+ !supportsGenerationProtocolV2(status)+ ) {+ // A legacy responder cannot adjudicate a v2 job during rollout.+ status = undefined;+ }++ if (+ status?.active === true &&+ status.createdAt != null &&+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)+ ) {+ const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);+ if (!sub.isCurrent()) {+ return;+ }+ if (handedOff) {+ return;+ }+ run.reconnectAttempt = 0;+ sub.scheduleReattach(30_000);+ return;+ }++ if (status?.active === true) {+ // The transport gave up, not the generation. Keep the active-job+ // marker and accepted chips intact, then release this stale+ // submission so useResumeOnLoad can establish a fresh attachment.+ host.cache.setStreamStatus(recoveryConvoId, status);+ restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);+ host.pane.setIsSubmitting(true);+ host.pane.setShowStopButton(generationCreatedAt != null);+ run.reconnectAttempt = 0;+ sub.scheduleReattach(30_000);+ return;+ }++ if (status == null) {+ // An inconclusive status read is still not proof of termination.+ // Preserve server/client ownership and let the normal resume-on-load+ // status query retry rather than duplicating accepted words.+ host.cache.invalidateStreamStatus(recoveryConvoId);+ host.pane.setIsSubmitting(true);+ host.pane.setShowStopButton(generationCreatedAt != null);+ run.reconnectAttempt = 0;+ sub.scheduleReattach(30_000);+ return;+ }++ // The status endpoint proved the job terminal. Reconcile persisted+ // content before recovering leftovers, just like the 404 path.+ run.cancelSteerRetryFrames();+ let persistedMessages: TMessage[] | undefined;+ try {+ await host.cache.invalidateMessages(recoveryConvoId, 'none');+ if (!sub.isCurrent()) {+ return;+ }+ const fetched = await host.cache.fetchMessages(recoveryConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ if (Array.isArray(fetched)) {+ persistedMessages = fetched;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {+ conversationId: recoveryConvoId,+ error,+ });+ }++ const authoritativeValues = [+ ...(persistedMessages ?? []),+ ...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),+ ];+ const confirmedV2Terminal =+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&+ supportsGenerationProtocolV2(status);+ const reconciledIds = new Set(host.steers.appliedIds(authoritativeValues));+ host.steers.settleApplied(recoveryConvoId, authoritativeValues);+ const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;+ const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];+ if (unrecovered.length > 0) {+ const recoverySteerId = getRecoverySteerId(sub.submission);+ host.steers.convertToQueued(recoveryConvoId, unrecovered, {+ generationProtocolVersion,+ ...(recoverySteerId != null && {+ allowPreviouslyConvertedIds: [recoverySteerId],+ }),+ });+ for (const steer of unrecovered) {+ reconciledIds.add(steer.steerId);+ if (steer.clientSteerId) {+ reconciledIds.add(steer.clientSteerId);+ }+ }+ }+ if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {+ host.steers.convertLocalToQueued(recoveryConvoId, {+ excludeSteerIds: reconciledIds,+ generationProtocolVersion,+ });+ }++ host.events.resetLive(sub.liveSubmission());+ host.cache.removeActiveJob(currentStreamId);+ clearAttachedGenerationCreatedAt(ctx);+ clearPendingApprovalForTerminal(ctx, recoveryConvoId);+ if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {+ host.cache.removeConversation(currentStreamId);+ }+ sub.retired = true;+ host.pane.setIsSubmitting(false);+ host.pane.setShowStopButton(false);+ let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';+ if (status.status === 'complete' && persistedMessages != null) {+ recoveryOutcome = 'completed';+ } else if (status.status === 'aborted') {+ recoveryOutcome = 'aborted';+ }+ const recoveredResponseMessageId =+ recoveryOutcome === 'completed'+ ? completedResponseMessageId(+ persistedMessages,+ sub.userMessage?.messageId,+ status?.resumeState?.responseMessageId ?? sub.submission.initialResponse?.messageId,+ )+ : undefined;+ host.pane.publishRunEnd({+ conversationId: recoveryConvoId,+ outcome: recoveryOutcome,+ endedAt: run.scheduler.now(),+ generationCreatedAt: status.createdAt ?? generationCreatedAt,+ ...(recoveredResponseMessageId != null && {+ responseMessageId: recoveredResponseMessageId,+ }),+ });+ host.pane.setSubmission(null);+ host.pane.setStreamId(null);+ run.releaseStream(currentStreamId);+ run.reconnectAttempt = 0;+}diff --git a/packages/chat/src/core/resumable/failure.ts b/packages/chat/src/core/resumable/failure.tsnew file mode 100644index 0000000000..c423df933d--- /dev/null+++ b/packages/chat/src/core/resumable/failure.ts@@ -0,0 +1,209 @@+import { ErrorTypes, ViolationTypes } from 'librechat-data-provider';+import type { ChatEvent, ChatErrorData } from 'librechat-data-provider';+import type { TerminalContext } from './terminal';+import {+ handoffToReplacement,+ authorizeTerminalTeardown,+ clearPendingApprovalForTerminal,+ clearAttachedGenerationCreatedAt,+} from './terminal';+import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';+import { MAX_RETRIES, getReconnectDelay, getRecoverySteerId } from './start';+import { settleAfterRetryCeiling } from './ceiling';+import { recoverMissingStream } from './missing';++/**+ * Server-sent error event (event: error with data) - no responseCode.+ * These are known errors (ErrorTypes, ViolationTypes) that should be displayed to user.+ */+async function handleServedError(+ ctx: TerminalContext,+ data: ChatErrorData | string,+): Promise<void> {+ const { sub, host } = ctx;+ const { run, generationCreatedAt, generationProtocolVersion } = sub;+ const currentStreamId = sub.streamId;+ sub.finalReceived = true;+ const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;+ if (!(await authorizeTerminalTeardown(ctx, recoveryConvoId, 'error')) || !sub.isCurrent()) {+ return;+ }+ host.logger.log('ResumableSSE', 'Server-sent error event received:', data);+ run.cancelSteerRetryFrames();+ sub.close();+ /** FLUSH (not cancel): the error card below is built from the cache+ * tail, so queued tokens must land first, and a stale trailing+ * frame must never overwrite the error write. */+ host.events.flushPendingDeltas();+ host.cache.removeActiveJob(currentStreamId);+ clearAttachedGenerationCreatedAt(ctx);+ clearPendingApprovalForTerminal(ctx, recoveryConvoId);+ host.events.resetLive(sub.liveSubmission());+ if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {+ host.cache.removeConversation(currentStreamId);+ }++ const errorSupportsV2 = supportsGenerationProtocolV2(data);+ const errorString =+ typeof data === 'string' ? data : (data.error ?? data.message ?? JSON.stringify(data));++ // Check if it's a known error type (ViolationTypes or ErrorTypes)+ let isKnownError = false;+ try {+ const parsed = typeof errorString === 'string' ? JSON.parse(errorString) : errorString;+ const errorType = parsed?.type ?? parsed?.code;+ if (errorType) {+ const violationValues = Object.values(ViolationTypes) as string[];+ const errorTypeValues = Object.values(ErrorTypes) as string[];+ isKnownError = violationValues.includes(errorType) || errorTypeValues.includes(errorType);+ }+ } catch {+ // Not JSON or parsing failed - treat as generic error+ }++ host.logger.log('ResumableSSE', 'Error type check:', { isKnownError, errorString });++ // Display the error to user via errorHandler+ host.events.showError(errorString, sub.submission);++ host.pane.setIsSubmitting(false);+ host.pane.setShowStopButton(false);+ const recoverySteerId = getRecoverySteerId(sub.submission);+ if (+ recoverySteerId != null &&+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&+ errorSupportsV2+ ) {+ try {+ /** The error was received on the exact generation-fenced stream.+ * Read status only now, after terminal was observed: a claim+ * started before this recovery could return a stale parked copy+ * after a successful commit and must never authorize redelivery. */+ const terminalStatus = await host.fetchStatus(recoveryConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ const releasedSource = terminalStatus.unrecoveredSteers?.find(+ (steer) => steer.steerId === recoverySteerId,+ );+ if (+ terminalStatus.active === false &&+ supportsGenerationProtocolV2(terminalStatus) &&+ releasedSource != null+ ) {+ host.steers.convertToQueued(recoveryConvoId, [releasedSource], {+ generationProtocolVersion,+ allowPreviouslyConvertedIds: [recoverySteerId],+ });+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not recover source after generation error', {+ conversationId: recoveryConvoId,+ error,+ });+ }+ }+ // The error terminal's backstop parks acked leftovers server-side;+ // claim it now so a reload can't resurrect the chips converted here.+ if (generationProtocolVersion === 1 || errorSupportsV2) {+ host.steers.convertLocalToQueued(recoveryConvoId, {+ claimParked: true,+ generationProtocolVersion,+ });+ }+ host.pane.publishRunEnd({+ conversationId: recoveryConvoId,+ outcome: 'error',+ endedAt: run.scheduler.now(),+ generationCreatedAt,+ });+ host.pane.setStreamId(null);+ run.releaseStream(currentStreamId);+ run.reconnectAttempt = 0;+}++/**+ * Error event handler - handles BOTH:+ * 1. HTTP-level errors (status present) - 404, 409, network failures (0)+ * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes+ *+ * Order matters: check the status first since HTTP errors may also include data.+ * The transport has already retried a 401 with a refreshed token.+ */+export async function handleTransportFailure(+ ctx: TerminalContext,+ { status: responseCode, data }: Pick<Extract<ChatEvent, { type: 'error' }>, 'status' | 'data'>,+): Promise<void> {+ const { sub, host } = ctx;+ const { run, generationCreatedAt } = sub;+ if (!sub.isCurrent()) {+ return;+ }++ if (sub.finalReceived) {+ host.logger.log('ResumableSSE', 'Ignoring error after FINAL event', {+ responseCode,+ hasData: data != null,+ });+ return;+ }++ host.pane.refreshBalance();++ if (responseCode === 409) {+ const handedOff = await handoffToReplacement(ctx, sub.conversationId());+ if (!sub.isCurrent()) {+ return;+ }+ if (handedOff) {+ return;+ }+ }++ // 404 → job completed & was cleaned up; messages are persisted in DB.+ if (responseCode === 404) {+ await recoverMissingStream(ctx);+ return;+ }++ /**+ * Only check the data if there's no HTTP status, since HTTP errors may also have body data.+ * Note: responseCode === 0 means transport failure (connection dropped) - treat as network error,+ * not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).+ */+ if (responseCode == null && data != null) {+ await handleServedError(ctx, data);+ return;+ }++ // Network failure or unknown HTTP error - attempt reconnection with backoff+ host.logger.log('ResumableSSE', 'Stream error (network failure) - will attempt reconnect', {+ responseCode,+ hasData: data != null,+ });++ if (run.reconnectAttempt < MAX_RETRIES) {+ // Increment counter BEFORE close() so abort handler knows we're reconnecting+ run.reconnectAttempt++;+ const delay = getReconnectDelay(run.reconnectAttempt);++ host.logger.log(+ 'ResumableSSE',+ `Reconnecting in ${delay}ms (attempt ${run.reconnectAttempt}/${MAX_RETRIES})`,+ );++ sub.close();++ sub.scheduleReattach(delay);++ // Keep UI in "submitting" state during reconnection attempts+ // so user knows we're still trying (abort handler may have reset these)+ host.pane.setIsSubmitting(true);+ host.pane.setShowStopButton(generationCreatedAt != null);+ } else {+ await settleAfterRetryCeiling(ctx);+ }+}diff --git a/packages/chat/src/core/resumable/host.ts b/packages/chat/src/core/resumable/host.tsindex 299a00a9dc..6594ec4fcf 100644--- a/packages/chat/src/core/resumable/host.ts+++ b/packages/chat/src/core/resumable/host.ts@@ -1,4 +1,10 @@-import type { Agents, TSubmission, TPendingSteer } from 'librechat-data-provider';+import type {+ Agents,+ TMessage,+ TSubmission,+ TConversation,+ TPendingSteer,+} from 'librechat-data-provider';import type { GenerationProtocolVersion } from '../protocol';import type { RunEnd } from '../steer/types';@@ -139,6 +145,13 @@ export interface ResumableCache {invalidateStreamStatus: (conversationId: string) => void;addActiveJob: (streamId: string) => void;removeActiveJob: (streamId: string) => void;+ /** Reads the conversation's saved record; rejects with the HTTP failure, a 404 included. */+ fetchConversation: (conversationId: string) => Promise<TConversation | undefined>;+ setConversation: (conversationId: string, conversation: TConversation) => void;+ /** Places the conversation's snapshot into every conversation list. */+ upsertConversation: (conversation: TConversation) => void;+ /** Drops the conversation from every conversation list. */+ removeConversation: (conversationId: string) => void;}/** The pane's controls and run bookkeeping a resumable run drives. */@@ -164,6 +177,18 @@ export interface ResumablePane {publishRunEnd: (end: RunEnd, options?: { retainDetached?: boolean }) => void;/** Consumes an armed "interrupt & send" for exactly this generation. */clearDrainAfterAbort: (conversationId: string, generationCreatedAt?: number) => void;+ /** Refreshes the token balance, when the deployment tracks one. */+ refreshBalance: () => void;+ /** Updates the pane's conversation; absent for a pane that only adds a comparison response. */+ updateConversation?: (update: (current: TConversation | null) => TConversation | null) => void;+ /**+ * The end of a run this pane left mid-stream and is now reattached to, as `messages` record+ * it. `resolvable` is false when no such run is remembered or the user stopped it.+ */+ resolveDetachedEnd: (+ conversationId: string,+ messages: TMessage[] | undefined,+ ) => { resolvable: boolean; end: RunEnd | null };}/** What the event handlers that render the stream expose to the run. */@@ -172,6 +197,14 @@ export interface ResumableEvents {flushPendingDeltas: () => void;/** Discards the live usage estimate for a run that stopped streaming. */resetLive: (submission: TSubmission) => void;+ /** Writes the failed turn's error card from the text (or raw error value) the server sent. */+ showError: (text: string | boolean | { message?: string }, submission: TSubmission) => void;+}++/** The pane's message list. */+export interface ResumableMessages {+ get: () => TMessage[] | undefined;+ set: (messages: TMessage[]) => void;}/** Everything a resumable run needs from the app it runs in. */@@ -180,4 +213,5 @@ export interface ResumableHost extends SubscriptionHost {pane: ResumablePane;steers: ResumableSteers;events: ResumableEvents;+ messages: ResumableMessages;}diff --git a/packages/chat/src/core/resumable/index.ts b/packages/chat/src/core/resumable/index.tsindex 3ba5c14d0a..d94f8f3c90 100644--- a/packages/chat/src/core/resumable/index.ts+++ b/packages/chat/src/core/resumable/index.ts@@ -8,3 +8,6 @@ export * from './host';export * from './subscription';export * from './terminal';export * from './reconcile';+export * from './failure';+export * from './missing';+export * from './ceiling';diff --git a/packages/chat/src/core/resumable/missing.ts b/packages/chat/src/core/resumable/missing.tsnew file mode 100644index 0000000000..a6ee87261a--- /dev/null+++ b/packages/chat/src/core/resumable/missing.ts@@ -0,0 +1,242 @@+import { Constants } from 'librechat-data-provider';+import type { TMessage } from 'librechat-data-provider';+import type { TerminalContext } from './terminal';+import {+ restoreLiveStatus,+ handoffToReplacement,+ clearPendingApprovalForTerminal,+ clearAttachedGenerationCreatedAt,+} from './terminal';+import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';+import { isInitialNewConversation, keepLocalCodeApprovalMode } from './messages';+import { getRecoverySteerId, toStartGenerationError } from './start';++/**+ * A 404 from the stream route: the job completed and was cleaned up, and its messages are+ * persisted. Refetches and reconciles them before recovering local steers (the final response+ * may contain an applied steer whose live event was missed), unless a status read shows the+ * generation, or a newer one, still running.+ */+export async function recoverMissingStream(ctx: TerminalContext): Promise<void> {+ const { sub, host } = ctx;+ const { run, generationCreatedAt, generationProtocolVersion } = sub;+ const currentStreamId = sub.streamId;+ const convoId = sub.submission.conversation?.conversationId;+ host.logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);+ // Keep the UI non-idle while the status read decides whether this is+ // terminal cleanup or a handoff to a newer generation epoch.+ run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);+ run.cancelSteerRetryFrames();+ sub.close();+ /** Terminal: drop any in-flight live estimate so the gauge doesn't+ * keep counting stale streamed output after the stream ends */+ host.events.resetLive(sub.liveSubmission());+ host.pane.clearComposerDrafts(convoId, {+ includeNewChatDraft:+ !convoId ||+ convoId === Constants.NEW_CONVO ||+ run.optimisticStreamIds.has(currentStreamId) ||+ isInitialNewConversation(sub.submission),+ });+ host.events.clearStepMaps();+ let persistedMessages: TMessage[] | undefined;+ if (convoId) {+ try {+ await host.cache.invalidateMessages(convoId, 'none');+ if (!sub.isCurrent()) {+ return;+ }+ // The first stream can finish before CREATED mounts the saved-chat query.+ const fetched = await host.cache.fetchMessages(convoId, true);+ if (!sub.isCurrent()) {+ return;+ }+ if (Array.isArray(fetched)) {+ persistedMessages = fetched;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {+ conversationId: convoId,+ error,+ });+ }+ host.cache.removeStreamStatus(convoId);+ }+ const recoveryConvoId = convoId ?? currentStreamId;+ const reconciledIds = new Set(persistedMessages ? host.steers.appliedIds(persistedMessages) : []);+ if (persistedMessages) {+ host.steers.settleApplied(recoveryConvoId, persistedMessages);+ }++ // The status replay is owner-gated and non-destructive; creating a+ // recovered follow-up consumes its exact source. Prefer these+ // authoritative leftovers over guessing from local chips.+ let confirmedV2Terminal = false;+ try {+ const status = await host.fetchStatus(recoveryConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ if (+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&+ !supportsGenerationProtocolV2(status)+ ) {+ sub.scheduleReattach(1_000);+ return;+ }+ if (+ status.active === true &&+ status.createdAt != null &&+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)+ ) {+ const handedOff = await handoffToReplacement(ctx, recoveryConvoId, status);+ if (!sub.isCurrent()) {+ return;+ }+ if (handedOff) {+ return;+ }+ /** Legacy generations have no replacement-handoff contract.+ * Never attach this submission to a different epoch. */+ sub.scheduleReattach(1_000);+ return;+ }+ if (status.active === true) {+ /** A stream lookup can briefly lose its routing record while the+ * authoritative job status still proves the same generation is+ * alive. Preserve ownership and retry the attachment instead of+ * publishing a terminal run-end (which could auto-drain queued+ * text into a still-running generation). */+ const activeStreamId = status.streamId ?? currentStreamId;+ host.cache.setStreamStatus(recoveryConvoId, status);+ if (status.createdAt != null) {+ host.pane.setGenerationEpoch(+ recoveryConvoId,+ status.createdAt,+ undefined,+ generationProtocolVersion,+ );+ }+ host.cache.addActiveJob(activeStreamId);+ restoreLiveStatus(ctx, recoveryConvoId, status, status.createdAt, generationProtocolVersion);+ host.pane.setIsSubmitting(true);+ host.pane.setShowStopButton(status.createdAt != null);+ run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);+ sub.scheduleReattach(1_000, activeStreamId, status.createdAt);+ return;+ }+ confirmedV2Terminal =+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&+ supportsGenerationProtocolV2(status);+ const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;+ const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];+ if (unrecovered.length > 0) {+ const recoverySteerId = getRecoverySteerId(sub.submission);+ host.steers.convertToQueued(recoveryConvoId, unrecovered, {+ generationProtocolVersion,+ ...(recoverySteerId != null && {+ allowPreviouslyConvertedIds: [recoverySteerId],+ }),+ });+ for (const steer of unrecovered) {+ reconciledIds.add(steer.steerId);+ if (steer.clientSteerId) {+ reconciledIds.add(steer.clientSteerId);+ }+ }+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {+ conversationId: recoveryConvoId,+ error,+ });+ }++ host.cache.removeActiveJob(currentStreamId);+ clearAttachedGenerationCreatedAt(ctx);+ clearPendingApprovalForTerminal(ctx, recoveryConvoId);+ if (!run.createdStreamIds.has(currentStreamId) && run.optimisticStreamIds.has(currentStreamId)) {+ // Both fresh and resumed subscriptions can miss every event of a fast turn.+ // A missing job proves neither that the conversation was saved nor that it failed.+ try {+ const persisted = await host.cache.fetchConversation(recoveryConvoId);+ if (!sub.isCurrent()) return;+ if (persisted?.conversationId === recoveryConvoId) {+ host.cache.setConversation(recoveryConvoId, persisted);+ host.cache.upsertConversation(persisted);+ host.pane.updateConversation?.((current) => {+ if (+ current?.conversationId != null &&+ current.conversationId !== Constants.NEW_CONVO &&+ current.conversationId !== recoveryConvoId+ )+ return current;+ return keepLocalCodeApprovalMode(+ { ...current, ...persisted },+ current,+ current?.conversationId,+ );+ });+ }+ } catch (error) {+ if (!sub.isCurrent()) return;+ if (toStartGenerationError(error)?.response?.status === 404) {+ host.cache.removeConversation(currentStreamId);+ }+ }+ // An inconclusive read must not turn a saved chat into a phantom.+ void host.cache.refreshConversationLists();+ void host.cache.refreshPinnedConversations();+ }+ if (persistedMessages) {+ host.messages.set(persistedMessages);+ }+ sub.retired = true;+ host.pane.setIsSubmitting(false);+ host.pane.setShowStopButton(false);++ // Only infer recovery from local accepted chips after the persisted+ // final was successfully inspected. If that fetch failed, their+ // applied-vs-leftover outcome is unknown and converting could send+ // already-applied words twice.+ if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {+ host.steers.convertLocalToQueued(recoveryConvoId, {+ excludeSteerIds: reconciledIds,+ generationProtocolVersion,+ });+ }+ // The true outcome is unknown here (job record already cleaned up):+ // a non-'completed' outcome releases parked interrupt flags without+ // auto-sending queued messages the user may not want fired. A run this+ // pane left and came back to is the exception: the history just read+ // names its response by epoch, which is the outcome it would have had.+ // History read before the job finished may not hold that response yet, or+ // may hold only the unfinished snapshot saved when the pane left, so a+ // marker that read leaves unresolved or stopped outlives this end for the+ // next visit, which reads history after the job is known to be done.+ const detached = host.pane.resolveDetachedEnd(recoveryConvoId, persistedMessages);+ const detachedResolvable = detached.resolvable;+ const detachedEnd = detached.end;+ host.pane.publishRunEnd(+ {+ conversationId: recoveryConvoId,+ outcome: 'aborted',+ endedAt: run.scheduler.now(),+ ...detachedEnd,+ startedAsNewConvo: run.optimisticStreamIds.has(currentStreamId),+ },+ {+ retainDetached:+ detachedResolvable && (detachedEnd == null || detachedEnd.outcome === 'aborted'),+ },+ );+ host.pane.setStreamId(null);+ run.releaseStream(currentStreamId);+ run.reconnectAttempt = 0;+}