Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🏁 refactor: Move Resumable Terminal Handoff and Lifecycle Reconcile Into the Chat Core (#17119)
Marco Berettad9f62bbcbe6a
diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.tsindex cbcf05a2b1..d9008f45fa 100644--- a/client/src/hooks/SSE/useResumableSSE.ts+++ b/client/src/hooks/SSE/useResumableSSE.ts@@ -1,4 +1,4 @@-import { useEffect, useState, useRef, useCallback } from 'react';+import { useMemo, useEffect, useState, useRef, useCallback } from 'react';import { v4 } from 'uuid';import { useSetRecoilState } from 'recoil';import { useStore, useSetAtom } from 'jotai';@@ -65,6 +65,11 @@ import {ResumableRun,documentVisibility,ResumableSubscription,+ handoffToReplacement,+ authorizeTerminalTeardown,+ reconcileGenerationLifecycle,+ clearPendingApprovalForTerminal,+ clearAttachedGenerationCreatedAt,} from '@librechat/chat';import type {Agents,@@ -79,12 +84,14 @@ import type {TSteerUpdatedEvent,TActivityLabelEvent,TReasoningLabelEvent,- ChatFinalFrame,TAttachment,TTokenUsageEvent,TContextUsageEvent,} from 'librechat-data-provider';import type {+ ResumableHost,+ ResumableCache,+ TerminalContext,SubscriptionTarget,StartGenerationResult,GenerationProtocolVersion,@@ -250,6 +257,41 @@ export default function useResumableSSE([queryClient],);+ const resumableCache = useMemo<ResumableCache>(+ () => ({+ invalidateMessages: (conversationId, refetch) =>+ queryClient.invalidateQueries({+ queryKey: [QueryKeys.messages, conversationId],+ refetchType: refetch,+ }),+ fetchMessages: (conversationId, fallback) =>+ queryClient.fetchQuery<TMessage[]>({+ queryKey: [QueryKeys.messages, conversationId],+ ...(fallback === true && {+ // The first stream can finish before CREATED mounts the saved-chat query.+ queryFn: () => fetchConversationMessages(conversationId),+ }),+ }),+ refreshConversation: (conversationId) =>+ queryClient.invalidateQueries({+ queryKey: [QueryKeys.conversation, conversationId],+ refetchType: 'all',+ }),+ refreshConversationLists: () => invalidateConversationLists(queryClient),+ refreshPinnedConversations: () =>+ queryClient.invalidateQueries({ queryKey: [QueryKeys.pinnedConversations] }),+ setStreamStatus: (conversationId, status) =>+ queryClient.setQueryData(streamStatusQueryKey(conversationId), status),+ removeStreamStatus: (conversationId) =>+ queryClient.removeQueries({ queryKey: streamStatusQueryKey(conversationId) }),+ invalidateStreamStatus: (conversationId) =>+ queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(conversationId) }),+ addActiveJob,+ removeActiveJob,+ }),+ [addActiveJob, queryClient, removeActiveJob],+ );+const addOptimisticConversation = useCallback((conversationId: string, currentSubmission: TSubmission): TSubmission => {if (!isInitialNewConversation(currentSubmission)) {@@ -427,16 +469,35 @@ export default function useResumableSSE(if (target.signal?.aborted || run.submission !== target.submission) {return;}- const sub = new ResumableSubscription(- run,- {- logger,- visibility: documentVisibility,- fetchStatus: fetchStreamStatus,- subscribe: (next) => subscribeToStream(next),+ const host: ResumableHost = {+ logger,+ visibility: documentVisibility,+ fetchStatus: fetchStreamStatus,+ subscribe: (next) => subscribeToStream(next),+ cache: resumableCache,+ pane: {+ setIsSubmitting,+ setShowStopButton,+ setSubmission,+ setStreamId,+ setGenerationEpoch: updateActiveGenerationCreatedAt,+ setPendingAction: (conversationId, action) =>+ jotaiStore.set(pendingApprovalActionFamily(conversationId), action),+ clearComposerDrafts: (conversationId, options) =>+ clearComposerDrafts(runIndex, conversationId, options),+ publishRunEnd: setRunEnd,+ clearDrainAfterAbort,},- target,- );+ steers: {+ seed: seedSteerChips,+ appliedIds: collectAppliedSteerIds,+ settleApplied: settleAppliedSteerParts,+ convertToQueued: convertSteersToQueued,+ convertLocalToQueued: convertLocalSteersToQueued,+ },+ events: { clearStepMaps, flushPendingDeltas, resetLive },+ };+ const sub = new ResumableSubscription(run, host, target);const {streamId: currentStreamId,isResume,@@ -445,29 +506,6 @@ export default function useResumableSSE(startedAsNewConversation,} = sub;run.subscribedEpoch = generationCreatedAt ?? null;- /** Terminal handlers below are fenced by this subscription and its generation.- * Clear every key the same run may have occupied, including the temporary- * new-chat key, so aborts and resumes from another tab cannot leave a stale- * approval card beside an idle composer. */- const clearPendingApprovalForTerminal = (conversationId?: string | null) => {- const conversationIds = new Set<string>([- currentStreamId,- ...(conversationId ? [conversationId] : []),- ...(sub.submission.conversation?.conversationId- ? [sub.submission.conversation.conversationId]- : []),- ...(startedAsNewConversation ? [String(Constants.NEW_CONVO)] : []),- ]);- for (const id of conversationIds) {- jotaiStore.set(pendingApprovalActionFamily(id), null);- }- };- const clearAttachedGenerationCreatedAt = () => {- updateActiveGenerationCreatedAt(currentStreamId, null, generationCreatedAt);- if (startedAsNewConversation) {- updateActiveGenerationCreatedAt(Constants.NEW_CONVO, null, generationCreatedAt);- }- };if (generationCreatedAt != null) {updateActiveGenerationCreatedAt(currentStreamId,@@ -648,6 +686,12 @@ export default function useResumableSSE(}};+ const terminal: TerminalContext = {+ sub,+ host,+ applyPendingAction: (action) => applyPendingActionToMessages(action),+ };+/*** The identities this pane may be rendering the in-flight response under* before the first run step renames it to the server's pre-allocated id:@@ -959,7 +1003,7 @@ export default function useResumableSSE(// replacement epoch) instead of passing them to finalHandler.sub.finalReceived = true;run.cancelSteerRetryFrames();- void reconcileGenerationLifecycle(data);+ void reconcileGenerationLifecycle(terminal, data);return;}@@ -970,7 +1014,10 @@ export default function useResumableSSE(data.conversation?.conversationId ??sub.submission.conversation?.conversationId ??currentStreamId;- if (!(await authorizeTerminalTeardown(finalConvoId, 'FINAL')) || !sub.isCurrent()) {+ if (+ !(await authorizeTerminalTeardown(terminal, finalConvoId, 'FINAL')) ||+ !sub.isCurrent()+ ) {return;}run.cancelSteerRetryFrames();@@ -981,7 +1028,7 @@ export default function useResumableSSE(conversationId: data.conversation?.conversationId,hasResponseMessage: !!data.responseMessage,});- clearPendingApprovalForTerminal(finalConvoId);+ clearPendingApprovalForTerminal(terminal, finalConvoId);clearComposerDrafts(runIndex, sub.submission.conversation?.conversationId, {includeNewChatDraft:!sub.submission.conversation?.conversationId ||@@ -1069,7 +1116,7 @@ export default function useResumableSSE(clearStepMaps();// Optimistically remove from active jobsremoveActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt();+ clearAttachedGenerationCreatedAt(terminal);(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();sub.close();setStreamId(null);@@ -1540,410 +1587,6 @@ export default function useResumableSSE(}};- async function handoffToReplacement(- conversationId: string,- knownStatus?: Awaited<ReturnType<typeof fetchStreamStatus>>,- ): Promise<boolean> {- if (!sub.isCurrent() || generationProtocolVersion !== GENERATION_PROTOCOL_VERSION) {- return false;- }- let replacementStatus = knownStatus;- if (replacementStatus == null) {- try {- // Deliberately unfenced: this read discovers the newer generation- // after the epoch-fenced SSE endpoint rejects the stale attachment.- replacementStatus = await fetchStreamStatus(conversationId);- if (!sub.isCurrent()) {- return false;- }- } catch (error) {- if (!sub.isCurrent()) {- return false;- }- logger.warn('ResumableSSE', 'Could not inspect replacement generation', {- conversationId,- generationCreatedAt,- error,- });- return false;- }- }-- if (- !supportsGenerationProtocolV2(replacementStatus) ||- replacementStatus.active !== true ||- !replacementStatus.streamId ||- replacementStatus.createdAt == null ||- replacementStatus.createdAt === generationCreatedAt- ) {- return false;- }-- logger.log('ResumableSSE', 'Handing stale attachment to replacement generation', {- conversationId,- staleCreatedAt: generationCreatedAt,- replacementCreatedAt: replacementStatus.createdAt,- });- run.replacementHandoff = true;- run.cancelSteerRetryFrames();- sub.close();- run.clearReconnectTimeout();- queryClient.setQueryData(streamStatusQueryKey(conversationId), {- ...replacementStatus,- generationHandoff: true,- });- updateActiveGenerationCreatedAt(- conversationId,- replacementStatus.createdAt,- undefined,- GENERATION_PROTOCOL_VERSION,- );- if (startedAsNewConversation) {- updateActiveGenerationCreatedAt(- Constants.NEW_CONVO,- replacementStatus.createdAt,- undefined,- GENERATION_PROTOCOL_VERSION,- );- }- addActiveJob(replacementStatus.streamId);- seedSteerChips(- conversationId,- (replacementStatus.resumeState?.pendingSteers ?? []) as TPendingSteer[],- replacementStatus.createdAt,- GENERATION_PROTOCOL_VERSION,- );- settleAppliedSteerParts(- conversationId,- replacementStatus.resumeState?.aggregatedContent ?? replacementStatus.aggregatedContent,- );- if (replacementStatus.pendingAction ?? replacementStatus.resumeState?.pendingAction) {- applyPendingActionToMessages(- (replacementStatus.pendingAction ??- replacementStatus.resumeState?.pendingAction) as Agents.PendingAction,- );- }- resetLive(sub.liveSubmission());- clearStepMaps();- run.reconnectAttempt = 0;- run.submission = null;- setIsSubmitting(true);- setShowStopButton(true);- setSubmission(null);- return true;- }-- /**- * A terminal frame proves only what happened on this SSE attachment. A- * fresh status read is the authority for whether terminal UI teardown is- * safe: a newer epoch may already own the conversation, while the exact- * epoch can still be active despite a racing FINAL/error frame.- */- function retryFencedTerminalAttachment(reason: string): void {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Deferring terminal teardown and retrying fenced attachment', {- conversationId: sub.submission.conversation?.conversationId ?? currentStreamId,- generationCreatedAt,- reason,- });- run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);- sub.close();- if (run.reconnectTimeout) {- run.scheduler.clearTimeout(run.reconnectTimeout);- }- sub.scheduleReattach(250);- }-- async function authorizeTerminalTeardown(- conversationId: string,- terminalKind: 'FINAL' | 'error',- ): Promise<boolean> {- if (!sub.isCurrent()) {- return false;- }- if (generationProtocolVersion !== GENERATION_PROTOCOL_VERSION) {- return true;- }-- let status: Awaited<ReturnType<typeof fetchStreamStatus>>;- try {- status = await fetchStreamStatus(conversationId);- if (!sub.isCurrent()) {- return false;- }- } catch (error) {- if (!sub.isCurrent()) {- return false;- }- logger.warn('ResumableSSE', `Could not verify ${terminalKind} generation status`, {- conversationId,- generationCreatedAt,- error,- });- retryFencedTerminalAttachment(`${terminalKind.toLowerCase()} status unavailable`);- return false;- }-- if (!supportsGenerationProtocolV2(status)) {- retryFencedTerminalAttachment(`${terminalKind.toLowerCase()} status was unnegotiated`);- return false;- }-- if (status.active === true) {- const handedOff =- status.createdAt != null &&- generationCreatedAt != null &&- status.createdAt !== generationCreatedAt &&- (await handoffToReplacement(conversationId, status));- if (!sub.isCurrent()) {- return false;- }- if (handedOff) {- return false;- }- retryFencedTerminalAttachment(- status.createdAt == null || generationCreatedAt == null- ? `${terminalKind.toLowerCase()} status epoch was inconclusive`- : `${terminalKind.toLowerCase()} generation remains active`,- );- return false;- }-- if (status.active !== false) {- retryFencedTerminalAttachment(`${terminalKind.toLowerCase()} status was inconclusive`);- return false;- }-- return true;- }-- const reconcileGenerationLifecycle = async (- event: Pick<- ChatFinalFrame,- 'reconcileReason' | 'terminalStatus' | 'generationCreatedAt' | 'conversation'- >,- ): Promise<void> => {- if (!sub.isCurrent()) {- return;- }- const reconciliationConvoId =- event.conversation?.conversationId ??- sub.submission.conversation?.conversationId ??- currentStreamId;- if (- generationCreatedAt != null &&- event.generationCreatedAt != null &&- event.generationCreatedAt !== generationCreatedAt- ) {- return;- }-- if (event.reconcileReason === 'generation_replaced') {- const handedOff = await handoffToReplacement(reconciliationConvoId);- if (!sub.isCurrent()) {- return;- }- if (handedOff) {- return;- }- // Replacement visibility can lag its notification by a round trip.- // Retry the stale epoch; the fenced HTTP endpoint will return the- // dedicated replacement response as soon as the new job is visible.- run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);- sub.close();- sub.scheduleReattach(250);- return;- }-- run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);- sub.close();- clearStepMaps();- let persistedMessages: TMessage[] | undefined;- const messageQueryKey = [QueryKeys.messages, reconciliationConvoId] as const;- try {- /** Force a fetch through the already-registered messages query and- * use its returned value. Merely awaiting invalidateQueries and- * reading the cache can mistake stale pre-run history for a- * successful reconciliation because React Query normally swallows- * refetch errors. */- await queryClient.invalidateQueries({- queryKey: messageQueryKey,- refetchType: 'none',- });- if (!sub.isCurrent()) {- return;- }- const fetched = await queryClient.fetchQuery<TMessage[]>({- queryKey: messageQueryKey,- });- if (!sub.isCurrent()) {- return;- }- if (Array.isArray(fetched)) {- persistedMessages = fetched;- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not refetch synthesized terminal state', {- conversationId: reconciliationConvoId,- error,- });- }- try {- await queryClient.invalidateQueries({- queryKey: [QueryKeys.conversation, reconciliationConvoId],- refetchType: 'all',- });- if (!sub.isCurrent()) {- return;- }- await invalidateConversationLists(queryClient);- if (!sub.isCurrent()) {- return;- }- await queryClient.invalidateQueries({ queryKey: [QueryKeys.pinnedConversations] });- if (!sub.isCurrent()) {- return;- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not refresh synthesized conversation metadata', {- conversationId: reconciliationConvoId,- error,- });- }-- let status: Awaited<ReturnType<typeof fetchStreamStatus>> | undefined;- try {- status = await fetchStreamStatus(reconciliationConvoId);- if (!sub.isCurrent()) {- return;- }- } catch (error) {- if (!sub.isCurrent()) {- return;- }- logger.warn('ResumableSSE', 'Could not inspect synthesized terminal status', {- conversationId: reconciliationConvoId,- error,- });- }- if (status != null && !supportsGenerationProtocolV2(status)) {- /** A v2 control frame cannot be reconciled with a legacy status- * snapshot (for example during a rolling deploy). Preserve the- * attachment and retry the exact fenced generation. */- sub.scheduleReattach(250);- return;- }- if (- status?.active === true &&- status.createdAt != null &&- (generationCreatedAt == null || status.createdAt !== generationCreatedAt)- ) {- const handedOff = await handoffToReplacement(reconciliationConvoId, status);- if (!sub.isCurrent()) {- return;- }- if (handedOff) {- return;- }- }- if (- status?.active === true &&- (generationCreatedAt == null || status.createdAt === generationCreatedAt)- ) {- // Durable state still calls this exact epoch active. Prefer another- // fenced resume over terminalizing on a racing synthesized frame.- sub.scheduleReattach(250);- return;- }-- const authoritativeValues = [- ...(persistedMessages ?? []),- ...(status?.resumeState?.aggregatedContent ?? status?.aggregatedContent ?? []),- ];- const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues));- settleAppliedSteerParts(reconciliationConvoId, authoritativeValues);- const unrecovered = status?.unrecoveredSteers ?? [];- if (unrecovered.length > 0) {- const recoverySteerId = getRecoverySteerId(sub.submission);- convertSteersToQueued(reconciliationConvoId, unrecovered, {- ...(recoverySteerId != null && {- allowPreviouslyConvertedIds: [recoverySteerId],- }),- });- for (const steer of unrecovered) {- reconciledIds.add(steer.steerId);- if (steer.clientSteerId) {- reconciledIds.add(steer.clientSteerId);- }- }- }- if (persistedMessages) {- convertLocalSteersToQueued(reconciliationConvoId, {- excludeSteerIds: reconciledIds,- generationProtocolVersion,- });- }-- resetLive(sub.liveSubmission());- removeActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt();- clearPendingApprovalForTerminal(reconciliationConvoId);- clearComposerDrafts(runIndex, reconciliationConvoId, {- includeNewChatDraft:- !reconciliationConvoId ||- reconciliationConvoId === Constants.NEW_CONVO ||- run.optimisticStreamIds.has(currentStreamId) ||- isInitialNewConversation(sub.submission),- });- setIsSubmitting(false);- setShowStopButton(false);- if (event.reconcileReason === 'abort_persistence_failed') {- /** Interrupt & send arms an override that drains even an `aborted`- * run. This terminal frame explicitly says durable persistence is- * unknown, so consume that override before publishing runEnd; the- * queued words remain available for a deliberate retry. */- clearDrainAfterAbort(- sub.submission.conversation?.conversationId || String(Constants.NEW_CONVO),- generationCreatedAt,- );- }- let reconciliationOutcome: 'completed' | 'aborted' | 'error' = 'aborted';- if (event.terminalStatus === 'complete') {- reconciliationOutcome = persistedMessages != null ? 'completed' : 'error';- } else if (event.terminalStatus === 'error') {- reconciliationOutcome = 'error';- }- const reconciledResponseMessageId =- reconciliationOutcome === 'completed'- ? completedResponseMessageId(- persistedMessages,- sub.userMessage?.messageId,- status?.resumeState?.responseMessageId ?? sub.submission.initialResponse?.messageId,- )- : undefined;- setRunEnd({- conversationId: reconciliationConvoId,- outcome: reconciliationOutcome,- endedAt: run.scheduler.now(),- generationCreatedAt: status?.createdAt ?? generationCreatedAt,- ...(reconciledResponseMessageId != null && {- responseMessageId: reconciledResponseMessageId,- }),- });- setSubmission(null);- setStreamId(null);- run.releaseStream(currentStreamId);- run.reconnectAttempt = 0;- };-/*** Error event handler - handles BOTH:* 1. HTTP-level errors (status present) - 404, 409, network failures (0)@@ -1972,7 +1615,7 @@ export default function useResumableSSE(if (responseCode === 409) {const replacementConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;- const handedOff = await handoffToReplacement(replacementConvoId);+ const handedOff = await handoffToReplacement(terminal, replacementConvoId);if (!sub.isCurrent()) {return;}@@ -2065,7 +1708,7 @@ export default function useResumableSSE(status.createdAt != null &&(generationCreatedAt == null || status.createdAt !== generationCreatedAt)) {- const handedOff = await handoffToReplacement(recoveryConvoId, status);+ const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);if (!sub.isCurrent()) {return;}@@ -2147,8 +1790,8 @@ export default function useResumableSSE(}removeActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt();- clearPendingApprovalForTerminal(recoveryConvoId);+ clearAttachedGenerationCreatedAt(terminal);+ clearPendingApprovalForTerminal(terminal, recoveryConvoId);if (!run.createdStreamIds.has(currentStreamId) &&run.optimisticStreamIds.has(currentStreamId)@@ -2252,7 +1895,10 @@ export default function useResumableSSE(if (responseCode == null && data != null) {sub.finalReceived = true;const recoveryConvoId = sub.submission.conversation?.conversationId ?? currentStreamId;- if (!(await authorizeTerminalTeardown(recoveryConvoId, 'error')) || !sub.isCurrent()) {+ if (+ !(await authorizeTerminalTeardown(terminal, recoveryConvoId, 'error')) ||+ !sub.isCurrent()+ ) {return;}logger.log('ResumableSSE', 'Server-sent error event received:', data);@@ -2263,8 +1909,8 @@ export default function useResumableSSE(* frame must never overwrite the error write. */flushPendingDeltas();removeActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt();- clearPendingApprovalForTerminal(recoveryConvoId);+ clearAttachedGenerationCreatedAt(terminal);+ clearPendingApprovalForTerminal(terminal, recoveryConvoId);resetLive(sub.liveSubmission());if (!run.createdStreamIds.has(currentStreamId) &&@@ -2419,7 +2065,7 @@ export default function useResumableSSE(status.createdAt != null &&(generationCreatedAt == null || status.createdAt !== generationCreatedAt)) {- const handedOff = await handoffToReplacement(recoveryConvoId, status);+ const handedOff = await handoffToReplacement(terminal, recoveryConvoId, status);if (!sub.isCurrent()) {return;}@@ -2537,8 +2183,8 @@ export default function useResumableSSE(resetLive(sub.liveSubmission());removeActiveJob(currentStreamId);- clearAttachedGenerationCreatedAt();- clearPendingApprovalForTerminal(recoveryConvoId);+ clearAttachedGenerationCreatedAt(terminal);+ clearPendingApprovalForTerminal(terminal, recoveryConvoId);if (!run.createdStreamIds.has(currentStreamId) &&run.optimisticStreamIds.has(currentStreamId)@@ -2658,6 +2304,7 @@ export default function useResumableSSE(},[run,+ resumableCache,isAddedRequest,setConversation,runIndex,diff --git a/packages/chat/src/core/__tests__/terminal.spec.ts b/packages/chat/src/core/__tests__/terminal.spec.tsnew file mode 100644index 0000000000..eeaa056285--- /dev/null+++ b/packages/chat/src/core/__tests__/terminal.spec.ts@@ -0,0 +1,236 @@+import type { TMessage, TSubmission } from 'librechat-data-provider';+import type {+ ResumableHost,+ TerminalContext,+ ResumableScheduler,+ StreamStatusResponse,+} from '../resumable';+import {+ ResumableRun,+ handoffToReplacement,+ ResumableSubscription,+ authorizeTerminalTeardown,+ reconcileGenerationLifecycle,+} from '../resumable';++/** Timers that run only when the test advances the clock. */+const createScheduler = () => {+ let now = 0;+ const timers = new Map<number, { at: number; callback: () => void }>();+ let nextId = 1;+ const scheduler: ResumableScheduler = {+ setTimeout: (callback, delay) => {+ const id = nextId++;+ timers.set(id, { at: now + delay, callback });+ return id as unknown as ReturnType<typeof setTimeout>;+ },+ clearTimeout: (handle) => {+ timers.delete(handle as unknown as number);+ },+ requestFrame: () => nextId++,+ cancelFrame: () => undefined,+ now: () => now,+ };+ const advance = (ms: number) => {+ now += ms;+ for (const [id, timer] of [...timers]) {+ if (timer.at <= now) {+ timers.delete(id);+ timer.callback();+ }+ }+ };+ return { scheduler, advance };+};++const createHost = (fetchStatus: jest.Mock<Promise<StreamStatusResponse>, [string]>) => {+ const host: ResumableHost = {+ logger: { log: jest.fn(), warn: jest.fn(), error: jest.fn() },+ visibility: { isVisible: () => true, subscribe: () => () => undefined },+ fetchStatus,+ subscribe: jest.fn(),+ cache: {+ invalidateMessages: jest.fn(async () => undefined),+ fetchMessages: jest.fn(async () => [] as TMessage[]),+ refreshConversation: jest.fn(async () => undefined),+ refreshConversationLists: jest.fn(async () => undefined),+ refreshPinnedConversations: jest.fn(async () => undefined),+ setStreamStatus: jest.fn(),+ removeStreamStatus: jest.fn(),+ invalidateStreamStatus: jest.fn(),+ addActiveJob: jest.fn(),+ removeActiveJob: jest.fn(),+ },+ pane: {+ setIsSubmitting: jest.fn(),+ setShowStopButton: jest.fn(),+ setSubmission: jest.fn(),+ setStreamId: jest.fn(),+ setGenerationEpoch: jest.fn(() => true),+ setPendingAction: jest.fn(),+ clearComposerDrafts: jest.fn(),+ publishRunEnd: jest.fn(),+ clearDrainAfterAbort: jest.fn(),+ },+ steers: {+ seed: jest.fn(),+ appliedIds: jest.fn(() => []),+ settleApplied: jest.fn(),+ convertToQueued: jest.fn(),+ convertLocalToQueued: jest.fn(),+ },+ events: { clearStepMaps: jest.fn(), flushPendingDeltas: jest.fn(), resetLive: jest.fn() },+ };+ return host;+};++const submission = {+ userMessage: { messageId: 'u-1', conversationId: 'c-1', isCreatedByUser: true } as TMessage,+ initialResponse: { messageId: 'u-1_', conversationId: 'c-1' } as TMessage,+ conversation: { conversationId: 'c-1' },+} as TSubmission;++const setup = () => {+ const clock = createScheduler();+ const fetchStatus = jest.fn<Promise<StreamStatusResponse>, [string]>();+ const host = createHost(fetchStatus);+ const run = new ResumableRun(clock.scheduler);+ run.submission = submission;+ const sub = new ResumableSubscription(run, host, {+ streamId: 'c-1',+ submission,+ isResume: false,+ generationCreatedAt: 2000,+ generationProtocolVersion: 2,+ });+ sub.claim();+ const ctx: TerminalContext = { sub, host, applyPendingAction: jest.fn() };+ return { clock, fetchStatus, host, run, sub, ctx };+};++const replacement: StreamStatusResponse = {+ active: true,+ streamId: 'c-1',+ createdAt: 3000,+ generationProtocolVersion: 2,+ resumeState: { pendingSteers: [] } as unknown as StreamStatusResponse['resumeState'],+};++describe('authorizeTerminalTeardown', () => {+ it('allows teardown once the status proves the run inactive', async () => {+ const { ctx, fetchStatus } = setup();+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });+ await expect(authorizeTerminalTeardown(ctx, 'c-1', 'FINAL')).resolves.toBe(true);+ });++ it('defers teardown and re-attaches while the same epoch is still active', async () => {+ const { ctx, fetchStatus, clock, host } = setup();+ fetchStatus.mockResolvedValue({ active: true, createdAt: 2000, generationProtocolVersion: 2 });+ await expect(authorizeTerminalTeardown(ctx, 'c-1', 'error')).resolves.toBe(false);+ expect(ctx.sub.controller.signal.aborted).toBe(true);+ clock.advance(249);+ expect(host.subscribe).not.toHaveBeenCalled();+ clock.advance(1);+ expect(host.subscribe).toHaveBeenCalledWith(+ expect.objectContaining({ streamId: 'c-1', isResume: true, generationCreatedAt: 2000 }),+ );+ });++ it('hands a stale attachment to the newer epoch that owns the conversation', async () => {+ const { ctx, fetchStatus, host, run } = setup();+ fetchStatus.mockResolvedValue(replacement);+ await expect(authorizeTerminalTeardown(ctx, 'c-1', 'FINAL')).resolves.toBe(false);+ expect(run.replacementHandoff).toBe(true);+ expect(run.submission).toBeNull();+ expect(host.cache.setStreamStatus).toHaveBeenCalledWith('c-1', {+ ...replacement,+ generationHandoff: true,+ });+ expect(host.pane.setGenerationEpoch).toHaveBeenCalledWith('c-1', 3000, undefined, 2);+ expect(host.cache.addActiveJob).toHaveBeenCalledWith('c-1');+ expect(host.pane.setSubmission).toHaveBeenCalledWith(null);+ });++ it('skips the status read for a legacy generation', async () => {+ const { ctx, fetchStatus, run } = setup();+ const legacy = new ResumableSubscription(run, ctx.host, {+ streamId: 'c-1',+ submission,+ isResume: false,+ generationProtocolVersion: 1,+ });+ legacy.claim();+ await expect(authorizeTerminalTeardown({ ...ctx, sub: legacy }, 'c-1', 'FINAL')).resolves.toBe(+ true,+ );+ expect(fetchStatus).not.toHaveBeenCalled();+ });+});++describe('handoffToReplacement', () => {+ it('declines without a live, different v2 epoch', async () => {+ const { ctx } = setup();+ await expect(+ handoffToReplacement(ctx, 'c-1', { ...replacement, createdAt: 2000 }),+ ).resolves.toBe(false);+ await expect(+ handoffToReplacement(ctx, 'c-1', { ...replacement, generationProtocolVersion: 1 }),+ ).resolves.toBe(false);+ expect(ctx.sub.run.replacementHandoff).toBe(false);+ });+});++describe('reconcileGenerationLifecycle', () => {+ it('settles a completed run from durable history and publishes its end', async () => {+ const { ctx, fetchStatus, host, run } = setup();+ const persisted = [+ submission.userMessage,+ { messageId: 'r-1', parentMessageId: 'u-1', isCreatedByUser: false } as TMessage,+ ];+ (host.cache.fetchMessages as jest.Mock).mockResolvedValue(persisted);+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });+ await reconcileGenerationLifecycle(ctx, { terminalStatus: 'complete' });++ expect(host.cache.invalidateMessages).toHaveBeenCalledWith('c-1', 'none');+ expect(host.steers.convertLocalToQueued).toHaveBeenCalled();+ expect(host.pane.setIsSubmitting).toHaveBeenCalledWith(false);+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith({+ conversationId: 'c-1',+ outcome: 'completed',+ endedAt: 0,+ generationCreatedAt: 2000,+ responseMessageId: 'r-1',+ });+ expect(run.reconnectAttempt).toBe(0);+ });++ it('reports an error when history could not be read for a completed run', async () => {+ const { ctx, fetchStatus, host } = setup();+ (host.cache.fetchMessages as jest.Mock).mockRejectedValue(new Error('offline'));+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });+ await reconcileGenerationLifecycle(ctx, { terminalStatus: 'complete' });+ expect(host.steers.convertLocalToQueued).not.toHaveBeenCalled();+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith(+ expect.objectContaining({ outcome: 'error' }),+ );+ });++ it('ignores a frame for a different epoch', async () => {+ const { ctx, host } = setup();+ await reconcileGenerationLifecycle(ctx, { generationCreatedAt: 1, terminalStatus: 'error' });+ expect(host.cache.invalidateMessages).not.toHaveBeenCalled();+ });++ it('consumes an armed interrupt when abort persistence failed', async () => {+ const { ctx, fetchStatus, host } = setup();+ fetchStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });+ await reconcileGenerationLifecycle(ctx, {+ terminalStatus: 'aborted',+ reconcileReason: 'abort_persistence_failed',+ });+ expect(host.pane.clearDrainAfterAbort).toHaveBeenCalledWith('c-1', 2000);+ expect(host.pane.publishRunEnd).toHaveBeenCalledWith(+ expect.objectContaining({ outcome: 'aborted' }),+ );+ });+});diff --git a/packages/chat/src/core/resumable/host.ts b/packages/chat/src/core/resumable/host.tsindex 6527420307..299a00a9dc 100644--- a/packages/chat/src/core/resumable/host.ts+++ b/packages/chat/src/core/resumable/host.ts@@ -1,4 +1,6 @@-import type { Agents, TPendingSteer } from 'librechat-data-provider';+import type { Agents, TSubmission, TPendingSteer } from 'librechat-data-provider';+import type { GenerationProtocolVersion } from '../protocol';+import type { RunEnd } from '../steer/types';/** Where a resumable run reports what it is doing. The app passes its own logger. */export interface ResumableLogger {@@ -53,3 +55,129 @@ export interface StreamStatusResponse {/** Reads a conversation's generation status. */export type FetchStreamStatus = (conversationId: string) => Promise<StreamStatusResponse>;++/** The generation one subscription attaches to, and how. */+export interface SubscriptionTarget {+ streamId: string;+ submission: TSubmission;+ /** Asks the server to replay the run's state (`sync`) before live events. */+ isResume: boolean;+ generationCreatedAt?: number;+ generationProtocolVersion: GenerationProtocolVersion;+ /** The pane's lifecycle: once aborted, nothing this subscription does lands. */+ signal?: AbortSignal;+}++/** How a subscription reaches the stream and decides when to re-attach. */+export interface SubscriptionHost {+ logger: ResumableLogger;+ visibility: PageVisibility;+ fetchStatus: FetchStreamStatus;+ /** Attaches a fresh subscription to `target`; how a subscription re-attaches itself. */+ subscribe: (target: SubscriptionTarget) => void;+}++/** How steers the run never injected are turned into queued follow-ups. */+export interface SteerQueueOptions {+ /** Reconcile the parked server copy of live-delivered terminal steers. */+ claimParked?: boolean;+ /** Protocol selected for the source generation. */+ generationProtocolVersion?: GenerationProtocolVersion;+ /** Exact recovery sources a post-terminal status read proved were not consumed. */+ allowPreviouslyConvertedIds?: readonly string[];+ /** Local failures have no durable receipt to reclaim on a later send. */+ bindRecoverySource?: boolean;+ /** Leave the converted rows for an explicit send instead of the run-end drain. */+ needsExplicitSend?: boolean;+}++/** The pane's steer chips: what the composer shows for steers sent into the running turn. */+export interface ResumableSteers {+ /** Re-seeds the chips from the server's still-queued steers. */+ seed: (+ conversationId: string,+ steers: TPendingSteer[],+ generationCreatedAt?: number,+ generationProtocolVersion?: GenerationProtocolVersion,+ ) => void;+ /** The steer ids the given messages or content parts already hold as applied parts. */+ appliedIds: (values: unknown[] | undefined) => string[];+ /** Settles the chips whose steers the given content already holds. */+ settleApplied: (conversationId: string, values: unknown[] | undefined) => void;+ convertToQueued: (+ conversationId: string,+ steers: TPendingSteer[],+ options?: SteerQueueOptions,+ ) => void;+ /** Sweeps the pane's local chips into queued follow-ups. */+ convertLocalToQueued: (+ conversationId: string,+ options?: {+ claimParked?: boolean;+ statuses?: readonly ('sending' | 'pending' | 'failed')[];+ excludeSteerIds?: Iterable<string>;+ generationProtocolVersion?: GenerationProtocolVersion;+ },+ ) => void;+}++/** The app's query cache, as far as a resumable run reads and refreshes it. */+export interface ResumableCache {+ /** Marks the conversation's messages stale; `all` refetches them now, `none` leaves that to+ * the next read. */+ invalidateMessages: (conversationId: string, refetch: 'all' | 'none') => Promise<void>;+ /** Fetches the conversation's messages through its query. `fallback` reads them directly+ * when no query for the conversation is mounted yet. */+ fetchMessages: (conversationId: string, fallback?: boolean) => Promise<unknown>;+ /** Marks the conversation stale and refetches it. */+ refreshConversation: (conversationId: string) => Promise<void>;+ /** Refetches every conversation list. */+ refreshConversationLists: () => Promise<void>;+ refreshPinnedConversations: () => Promise<void>;+ setStreamStatus: (conversationId: string, status: StreamStatusResponse) => void;+ removeStreamStatus: (conversationId: string) => void;+ invalidateStreamStatus: (conversationId: string) => void;+ addActiveJob: (streamId: string) => void;+ removeActiveJob: (streamId: string) => void;+}++/** The pane's controls and run bookkeeping a resumable run drives. */+export interface ResumablePane {+ setIsSubmitting: (isSubmitting: boolean) => void;+ setShowStopButton: (show: boolean) => void;+ setSubmission: (submission: TSubmission | null) => void;+ setStreamId: (streamId: string | null) => void;+ /** Records the generation epoch a conversation's controls fence against; with `expected`,+ * only while it still holds that epoch. Returns whether it was written. */+ setGenerationEpoch: (+ conversationId: string,+ next: number | null,+ expected?: number,+ generationProtocolVersion?: GenerationProtocolVersion,+ ) => boolean;+ setPendingAction: (conversationId: string, action: Agents.PendingAction | null) => void;+ /** Drops the composer drafts the finished run consumed. */+ clearComposerDrafts: (+ conversationId: string | null | undefined,+ options: { includeNewChatDraft: boolean },+ ) => void;+ publishRunEnd: (end: RunEnd, options?: { retainDetached?: boolean }) => void;+ /** Consumes an armed "interrupt & send" for exactly this generation. */+ clearDrainAfterAbort: (conversationId: string, generationCreatedAt?: number) => void;+}++/** What the event handlers that render the stream expose to the run. */+export interface ResumableEvents {+ clearStepMaps: () => void;+ flushPendingDeltas: () => void;+ /** Discards the live usage estimate for a run that stopped streaming. */+ resetLive: (submission: TSubmission) => void;+}++/** Everything a resumable run needs from the app it runs in. */+export interface ResumableHost extends SubscriptionHost {+ cache: ResumableCache;+ pane: ResumablePane;+ steers: ResumableSteers;+ events: ResumableEvents;+}diff --git a/packages/chat/src/core/resumable/index.ts b/packages/chat/src/core/resumable/index.tsindex db6cbf6b09..3ba5c14d0a 100644--- a/packages/chat/src/core/resumable/index.ts+++ b/packages/chat/src/core/resumable/index.ts@@ -6,3 +6,5 @@ export * from './messages';export * from './run';export * from './host';export * from './subscription';+export * from './terminal';+export * from './reconcile';diff --git a/packages/chat/src/core/resumable/reconcile.ts b/packages/chat/src/core/resumable/reconcile.tsnew file mode 100644index 0000000000..a0ff8003fd--- /dev/null+++ b/packages/chat/src/core/resumable/reconcile.ts@@ -0,0 +1,240 @@+import { Constants } from 'librechat-data-provider';+import type { TMessage, ChatFinalFrame } from 'librechat-data-provider';+import type { StreamStatusResponse } from './host';+import type { TerminalContext } from './terminal';+import {+ handoffToReplacement,+ clearAttachedGenerationCreatedAt,+ clearPendingApprovalForTerminal,+} from './terminal';+import { completedResponseMessageId, isInitialNewConversation } from './messages';+import { supportsGenerationProtocolV2 } from '../protocol';+import { getRecoverySteerId } from './start';++/** A synthesized lifecycle frame: the run ended without an ordinary final payload. */+export type LifecycleFrame = Pick<+ ChatFinalFrame,+ 'reconcileReason' | 'terminalStatus' | 'generationCreatedAt' | 'conversation'+>;++/**+ * Settles a run the server reports through a synthesized lifecycle frame. Such frames carry no+ * response payload, so durable history is reconciled (or the attachment handed to the+ * replacement epoch) instead of passing them to the final handler.+ */+export async function reconcileGenerationLifecycle(+ ctx: TerminalContext,+ event: LifecycleFrame,+): Promise<void> {+ const { sub, host } = ctx;+ const { run, generationCreatedAt, generationProtocolVersion } = sub;+ const currentStreamId = sub.streamId;+ if (!sub.isCurrent()) {+ return;+ }+ const reconciliationConvoId =+ event.conversation?.conversationId ??+ sub.submission.conversation?.conversationId ??+ currentStreamId;+ if (+ generationCreatedAt != null &&+ event.generationCreatedAt != null &&+ event.generationCreatedAt !== generationCreatedAt+ ) {+ return;+ }++ if (event.reconcileReason === 'generation_replaced') {+ const handedOff = await handoffToReplacement(ctx, reconciliationConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ if (handedOff) {+ return;+ }+ // Replacement visibility can lag its notification by a round trip.+ // Retry the stale epoch; the fenced HTTP endpoint will return the+ // dedicated replacement response as soon as the new job is visible.+ run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);+ sub.close();+ sub.scheduleReattach(250);+ return;+ }++ run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);+ sub.close();+ host.events.clearStepMaps();+ let persistedMessages: TMessage[] | undefined;+ try {+ /** Force a fetch through the already-registered messages query and+ * use its returned value. Merely awaiting invalidateQueries and+ * reading the cache can mistake stale pre-run history for a+ * successful reconciliation because React Query normally swallows+ * refetch errors. */+ await host.cache.invalidateMessages(reconciliationConvoId, 'none');+ if (!sub.isCurrent()) {+ return;+ }+ const fetched = await host.cache.fetchMessages(reconciliationConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ if (Array.isArray(fetched)) {+ persistedMessages = fetched;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not refetch synthesized terminal state', {+ conversationId: reconciliationConvoId,+ error,+ });+ }+ try {+ await host.cache.refreshConversation(reconciliationConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ await host.cache.refreshConversationLists();+ if (!sub.isCurrent()) {+ return;+ }+ await host.cache.refreshPinnedConversations();+ if (!sub.isCurrent()) {+ return;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not refresh synthesized conversation metadata', {+ conversationId: reconciliationConvoId,+ error,+ });+ }++ let status: StreamStatusResponse | undefined;+ try {+ status = await host.fetchStatus(reconciliationConvoId);+ if (!sub.isCurrent()) {+ return;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Could not inspect synthesized terminal status', {+ conversationId: reconciliationConvoId,+ error,+ });+ }+ if (status != null && !supportsGenerationProtocolV2(status)) {+ /** A v2 control frame cannot be reconciled with a legacy status+ * snapshot (for example during a rolling deploy). Preserve the+ * attachment and retry the exact fenced generation. */+ sub.scheduleReattach(250);+ return;+ }+ if (+ status?.active === true &&+ status.createdAt != null &&+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)+ ) {+ const handedOff = await handoffToReplacement(ctx, reconciliationConvoId, status);+ if (!sub.isCurrent()) {+ return;+ }+ if (handedOff) {+ return;+ }+ }+ if (+ status?.active === true &&+ (generationCreatedAt == null || status.createdAt === generationCreatedAt)+ ) {+ // Durable state still calls this exact epoch active. Prefer another+ // fenced resume over terminalizing on a racing synthesized frame.+ sub.scheduleReattach(250);+ return;+ }++ const authoritativeValues = [+ ...(persistedMessages ?? []),+ ...(status?.resumeState?.aggregatedContent ?? status?.aggregatedContent ?? []),+ ];+ const reconciledIds = new Set(host.steers.appliedIds(authoritativeValues));+ host.steers.settleApplied(reconciliationConvoId, authoritativeValues);+ const unrecovered = status?.unrecoveredSteers ?? [];+ if (unrecovered.length > 0) {+ const recoverySteerId = getRecoverySteerId(sub.submission);+ host.steers.convertToQueued(reconciliationConvoId, unrecovered, {+ ...(recoverySteerId != null && {+ allowPreviouslyConvertedIds: [recoverySteerId],+ }),+ });+ for (const steer of unrecovered) {+ reconciledIds.add(steer.steerId);+ if (steer.clientSteerId) {+ reconciledIds.add(steer.clientSteerId);+ }+ }+ }+ if (persistedMessages) {+ host.steers.convertLocalToQueued(reconciliationConvoId, {+ excludeSteerIds: reconciledIds,+ generationProtocolVersion,+ });+ }++ host.events.resetLive(sub.liveSubmission());+ host.cache.removeActiveJob(currentStreamId);+ clearAttachedGenerationCreatedAt(ctx);+ clearPendingApprovalForTerminal(ctx, reconciliationConvoId);+ host.pane.clearComposerDrafts(reconciliationConvoId, {+ includeNewChatDraft:+ !reconciliationConvoId ||+ reconciliationConvoId === Constants.NEW_CONVO ||+ run.optimisticStreamIds.has(currentStreamId) ||+ isInitialNewConversation(sub.submission),+ });+ host.pane.setIsSubmitting(false);+ host.pane.setShowStopButton(false);+ if (event.reconcileReason === 'abort_persistence_failed') {+ /** Interrupt & send arms an override that drains even an `aborted`+ * run. This terminal frame explicitly says durable persistence is+ * unknown, so consume that override before publishing runEnd; the+ * queued words remain available for a deliberate retry. */+ host.pane.clearDrainAfterAbort(+ sub.submission.conversation?.conversationId || String(Constants.NEW_CONVO),+ generationCreatedAt,+ );+ }+ let reconciliationOutcome: 'completed' | 'aborted' | 'error' = 'aborted';+ if (event.terminalStatus === 'complete') {+ reconciliationOutcome = persistedMessages != null ? 'completed' : 'error';+ } else if (event.terminalStatus === 'error') {+ reconciliationOutcome = 'error';+ }+ const reconciledResponseMessageId =+ reconciliationOutcome === 'completed'+ ? completedResponseMessageId(+ persistedMessages,+ sub.userMessage?.messageId,+ status?.resumeState?.responseMessageId ?? sub.submission.initialResponse?.messageId,+ )+ : undefined;+ host.pane.publishRunEnd({+ conversationId: reconciliationConvoId,+ outcome: reconciliationOutcome,+ endedAt: run.scheduler.now(),+ generationCreatedAt: status?.createdAt ?? generationCreatedAt,+ ...(reconciledResponseMessageId != null && {+ responseMessageId: reconciledResponseMessageId,+ }),+ });+ host.pane.setSubmission(null);+ host.pane.setStreamId(null);+ run.releaseStream(currentStreamId);+ run.reconnectAttempt = 0;+}diff --git a/packages/chat/src/core/resumable/subscription.ts b/packages/chat/src/core/resumable/subscription.tsindex 039f493bdf..85f71e9fdb 100644--- a/packages/chat/src/core/resumable/subscription.ts+++ b/packages/chat/src/core/resumable/subscription.ts@@ -6,33 +6,12 @@ import type {ChatTransport,ChatStreamConnection,} from 'librechat-data-provider';-import type { ResumableLogger, PageVisibility, FetchStreamStatus } from './host';+import type { SubscriptionHost, SubscriptionTarget } from './host';import type { GenerationProtocolVersion } from '../protocol';import type { StepEvent } from '../reducer';import type { ResumableRun } from './run';import { GENERATION_PROTOCOL_VERSION, generationProtocolHeaders } from '../protocol';-/** The generation one subscription attaches to, and how. */-export interface SubscriptionTarget {- streamId: string;- submission: TSubmission;- /** Asks the server to replay the run's state (`sync`) before live events. */- isResume: boolean;- generationCreatedAt?: number;- generationProtocolVersion: GenerationProtocolVersion;- /** The pane's lifecycle: once aborted, nothing this subscription does lands. */- signal?: AbortSignal;-}--/** How a subscription reaches the stream and decides when to re-attach. */-export interface SubscriptionHost {- logger: ResumableLogger;- visibility: PageVisibility;- fetchStatus: FetchStreamStatus;- /** Attaches a fresh subscription to `target`; how a subscription re-attaches itself. */- subscribe: (target: SubscriptionTarget) => void;-}-/** What a subscription does with the events its connection reports. */export interface SubscriptionHandlers {onOpen: () => void;diff --git a/packages/chat/src/core/resumable/terminal.ts b/packages/chat/src/core/resumable/terminal.tsnew file mode 100644index 0000000000..9f56b0f52b--- /dev/null+++ b/packages/chat/src/core/resumable/terminal.ts@@ -0,0 +1,256 @@+import { Constants } from 'librechat-data-provider';+import type { Agents, TPendingSteer } from 'librechat-data-provider';+import type { ResumableHost, StreamStatusResponse } from './host';+import type { GenerationProtocolVersion } from '../protocol';+import type { ResumableSubscription } from './subscription';+import { GENERATION_PROTOCOL_VERSION, supportsGenerationProtocolV2 } from '../protocol';++/** One subscription's terminal handling: the attachment, the app around it, and how a pending+ * action lands on the response it streams. */+export interface TerminalContext {+ sub: ResumableSubscription;+ host: ResumableHost;+ applyPendingAction: (action: Agents.PendingAction) => void;+}++/**+ * Terminal handlers are fenced by this subscription and its generation. Clear every key the+ * same run may have occupied, including the temporary new-chat key, so aborts and resumes from+ * another tab cannot leave a stale approval card beside an idle composer.+ */+export function clearPendingApprovalForTerminal(+ { sub, host }: TerminalContext,+ conversationId?: string | null,+): void {+ const conversationIds = new Set<string>([+ sub.streamId,+ ...(conversationId ? [conversationId] : []),+ ...(sub.submission.conversation?.conversationId+ ? [sub.submission.conversation.conversationId]+ : []),+ ...(sub.startedAsNewConversation ? [String(Constants.NEW_CONVO)] : []),+ ]);+ for (const id of conversationIds) {+ host.pane.setPendingAction(id, null);+ }+}++/** Releases the epoch this subscription installed, unless a newer one already replaced it. */+export function clearAttachedGenerationCreatedAt({ sub, host }: TerminalContext): void {+ host.pane.setGenerationEpoch(sub.streamId, null, sub.generationCreatedAt);+ if (sub.startedAsNewConversation) {+ host.pane.setGenerationEpoch(Constants.NEW_CONVO, null, sub.generationCreatedAt);+ }+}++/** Seeds the steer chips, settled parts and pending action a live status reports. */+export function restoreLiveStatus(+ { host, applyPendingAction }: TerminalContext,+ conversationId: string,+ status: StreamStatusResponse,+ generationCreatedAt: number | undefined,+ generationProtocolVersion: GenerationProtocolVersion,+): void {+ host.steers.seed(+ conversationId,+ (status.resumeState?.pendingSteers ?? []) as TPendingSteer[],+ generationCreatedAt,+ generationProtocolVersion,+ );+ host.steers.settleApplied(+ conversationId,+ status.resumeState?.aggregatedContent ?? status.aggregatedContent,+ );+ if (status.pendingAction ?? status.resumeState?.pendingAction) {+ applyPendingAction(+ (status.pendingAction ?? status.resumeState?.pendingAction) as Agents.PendingAction,+ );+ }+}++/**+ * Hands this stale attachment to the newer generation now occupying the conversation, when a+ * status read proves one is live. The replacement's snapshot is cached for resume-on-load to+ * build an epoch-fenced submission from; this pane stays non-idle until it does.+ */+export async function handoffToReplacement(+ ctx: TerminalContext,+ conversationId: string,+ knownStatus?: StreamStatusResponse,+): Promise<boolean> {+ const { sub, host } = ctx;+ const { generationCreatedAt } = sub;+ if (!sub.isCurrent() || sub.generationProtocolVersion !== GENERATION_PROTOCOL_VERSION) {+ return false;+ }+ let replacementStatus = knownStatus;+ if (replacementStatus == null) {+ try {+ // Deliberately unfenced: this read discovers the newer generation+ // after the epoch-fenced SSE endpoint rejects the stale attachment.+ replacementStatus = await host.fetchStatus(conversationId);+ if (!sub.isCurrent()) {+ return false;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return false;+ }+ host.logger.warn('ResumableSSE', 'Could not inspect replacement generation', {+ conversationId,+ generationCreatedAt,+ error,+ });+ return false;+ }+ }++ if (+ !supportsGenerationProtocolV2(replacementStatus) ||+ replacementStatus.active !== true ||+ !replacementStatus.streamId ||+ replacementStatus.createdAt == null ||+ replacementStatus.createdAt === generationCreatedAt+ ) {+ return false;+ }++ host.logger.log('ResumableSSE', 'Handing stale attachment to replacement generation', {+ conversationId,+ staleCreatedAt: generationCreatedAt,+ replacementCreatedAt: replacementStatus.createdAt,+ });+ const { run } = sub;+ run.replacementHandoff = true;+ run.cancelSteerRetryFrames();+ sub.close();+ run.clearReconnectTimeout();+ host.cache.setStreamStatus(conversationId, {+ ...replacementStatus,+ generationHandoff: true,+ });+ host.pane.setGenerationEpoch(+ conversationId,+ replacementStatus.createdAt,+ undefined,+ GENERATION_PROTOCOL_VERSION,+ );+ if (sub.startedAsNewConversation) {+ host.pane.setGenerationEpoch(+ Constants.NEW_CONVO,+ replacementStatus.createdAt,+ undefined,+ GENERATION_PROTOCOL_VERSION,+ );+ }+ host.cache.addActiveJob(replacementStatus.streamId);+ restoreLiveStatus(+ ctx,+ conversationId,+ replacementStatus,+ replacementStatus.createdAt,+ GENERATION_PROTOCOL_VERSION,+ );+ host.events.resetLive(sub.liveSubmission());+ host.events.clearStepMaps();+ run.reconnectAttempt = 0;+ run.submission = null;+ host.pane.setIsSubmitting(true);+ host.pane.setShowStopButton(true);+ host.pane.setSubmission(null);+ return true;+}++/**+ * A terminal frame proves only what happened on this SSE attachment. A fresh status read is+ * the authority for whether terminal UI teardown is safe: a newer epoch may already own the+ * conversation, while the exact epoch can still be active despite a racing FINAL/error frame.+ */+export function retryFencedTerminalAttachment(+ { sub, host }: TerminalContext,+ reason: string,+): void {+ if (!sub.isCurrent()) {+ return;+ }+ host.logger.warn('ResumableSSE', 'Deferring terminal teardown and retrying fenced attachment', {+ conversationId: sub.conversationId(),+ generationCreatedAt: sub.generationCreatedAt,+ reason,+ });+ const { run } = sub;+ run.reconnectAttempt = Math.max(run.reconnectAttempt, 1);+ sub.close();+ if (run.reconnectTimeout) {+ run.scheduler.clearTimeout(run.reconnectTimeout);+ }+ sub.scheduleReattach(250);+}++/** Whether a terminal frame may tear the pane down, per a fresh status read. */+export async function authorizeTerminalTeardown(+ ctx: TerminalContext,+ conversationId: string,+ terminalKind: 'FINAL' | 'error',+): Promise<boolean> {+ const { sub, host } = ctx;+ const { generationCreatedAt } = sub;+ if (!sub.isCurrent()) {+ return false;+ }+ if (sub.generationProtocolVersion !== GENERATION_PROTOCOL_VERSION) {+ return true;+ }++ let status: StreamStatusResponse;+ try {+ status = await host.fetchStatus(conversationId);+ if (!sub.isCurrent()) {+ return false;+ }+ } catch (error) {+ if (!sub.isCurrent()) {+ return false;+ }+ host.logger.warn('ResumableSSE', `Could not verify ${terminalKind} generation status`, {+ conversationId,+ generationCreatedAt,+ error,+ });+ retryFencedTerminalAttachment(ctx, `${terminalKind.toLowerCase()} status unavailable`);+ return false;+ }++ if (!supportsGenerationProtocolV2(status)) {+ retryFencedTerminalAttachment(ctx, `${terminalKind.toLowerCase()} status was unnegotiated`);+ return false;+ }++ if (status.active === true) {+ const handedOff =+ status.createdAt != null &&+ generationCreatedAt != null &&+ status.createdAt !== generationCreatedAt &&+ (await handoffToReplacement(ctx, conversationId, status));+ if (!sub.isCurrent()) {+ return false;+ }+ if (handedOff) {+ return false;+ }+ retryFencedTerminalAttachment(+ ctx,+ status.createdAt == null || generationCreatedAt == null+ ? `${terminalKind.toLowerCase()} status epoch was inconclusive`+ : `${terminalKind.toLowerCase()} generation remains active`,+ );+ return false;+ }++ if (status.active !== false) {+ retryFencedTerminalAttachment(ctx, `${terminalKind.toLowerCase()} status was inconclusive`);+ return false;+ }++ return true;+}