Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🚦 refactor: Add the Core Chat Class With a Serial Job Executor and React Query State (#17030)

Marco Berettaf177bed175b4
diff --git a/e2e/specs/mock/scenarios/chat-package.spec.ts b/e2e/specs/mock/scenarios/chat-package.spec.ts
index 4d9217c137..8172c1954d 100644
--- a/e2e/specs/mock/scenarios/chat-package.spec.ts
+++ b/e2e/specs/mock/scenarios/chat-package.spec.ts
@@ -267,4 +267,74 @@ test.describe('the built @librechat/chat package', () => {
expect(Object.values(exports)).toEqual(Array(7).fill('function'));
}
});
+
+ test('a host runs a turn through the built chat core @scenario:a-host-runs-a-turn-through-the-built-chat-core', () => {
+ test.setTimeout(120_000);
+ const dist = resolve(repoRoot, 'packages/chat/dist');
+ expect(existsSync(dist), 'packages/chat/dist is missing: run npm run build').toBe(true);
+
+ /** The core `Chat` drives a turn over a substitute transport with every UI module blocked;
+ * the `/react` entry then backs a second chat with the React Query message cache. */
+ const probe = `
+ const Module = require('node:module');
+ const blocked = new Set(${JSON.stringify(UI_MODULES)});
+ const resolveFilename = Module._resolveFilename;
+ Module._resolveFilename = function (request, ...rest) {
+ if (blocked.has(request)) throw new Error('the core reached ' + request);
+ return resolveFilename.call(this, request, ...rest);
+ };
+ const { Chat } = require('@librechat/chat');
+ const events = [
+ { type: 'created', data: { created: true } },
+ { type: 'final', data: { final: true, responseMessage: { messageId: 'r1', text: 'Hello' } } },
+ ];
+ const stream = {
+ close() {},
+ async *[Symbol.asyncIterator]() { yield* events; },
+ };
+ let messages = [];
+ const statuses = [];
+ const state = {
+ error: undefined,
+ connection: 'idle',
+ get status() { return statuses[statuses.length - 1] || 'ready'; },
+ set status(value) { statuses.push(value); },
+ get messages() { return messages; },
+ replaceMessages(next) { messages = next; },
+ };
+ const chat = new Chat({
+ id: 'convo-1',
+ state,
+ transport: {
+ sendMessages: async () => ({ status: 'stream', stream }),
+ reconnectToStream: async () => null,
+ abort: async () => ({ success: true }),
+ },
+ prepareRequest: () => ({
+ payload: { endpoint: 'agents' },
+ messages: [{ messageId: 'r1', text: '' }],
+ }),
+ });
+ chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] }).then(() => {
+ Module._resolveFilename = resolveFilename;
+ const { QueryClient } = require('@tanstack/react-query');
+ const { ReactQueryChatState } = require('@librechat/chat/react');
+ const queryClient = new QueryClient();
+ const cached = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ cached.replaceMessages(messages);
+ process.stdout.write(JSON.stringify({
+ statuses,
+ text: messages[0].text,
+ cached: queryClient.getQueryData(['messages', 'convo-1']).length,
+ }));
+ });
+ `;
+ const loaded = run(process.execPath, ['-e', probe], { cwd: resolve(repoRoot, 'client') });
+ expect(loaded.status, loaded.output).toBe(0);
+ expect(JSON.parse(loaded.stdout)).toEqual({
+ statuses: ['submitted', 'streaming', 'ready'],
+ text: 'Hello',
+ cached: 1,
+ });
+ });
});
diff --git a/package-lock.json b/package-lock.json
index 6ded1e552f..bb2e070a45 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -41452,6 +41452,7 @@
"@babel/preset-react": "^7.22.15",
"@babel/preset-typescript": "^7.21.0",
"@librechat/client": "*",
+ "@tanstack/react-query": "^4.28.0",
"@testing-library/jest-dom": "^5.16.5",
"@testing-library/react": "^14.0.0",
"@types/jest": "^29.5.2",
@@ -41472,6 +41473,7 @@
},
"peerDependencies": {
"@librechat/client": "*",
+ "@tanstack/react-query": "^4.28.0",
"jotai": "^2.12.5",
"librechat-data-provider": "*",
"lucide": "^0.528.0",
diff --git a/packages/chat/package.json b/packages/chat/package.json
index f426752675..f714138d0d 100644
--- a/packages/chat/package.json
+++ b/packages/chat/package.json
@@ -59,6 +59,7 @@
},
"peerDependencies": {
"@librechat/client": "*",
+ "@tanstack/react-query": "^4.28.0",
"jotai": "^2.12.5",
"librechat-data-provider": "*",
"lucide": "^0.528.0",
@@ -74,6 +75,7 @@
"@babel/preset-react": "^7.22.15",
"@babel/preset-typescript": "^7.21.0",
"@librechat/client": "*",
+ "@tanstack/react-query": "^4.28.0",
"@testing-library/jest-dom": "^5.16.5",
"@testing-library/react": "^14.0.0",
"@types/jest": "^29.5.2",
diff --git a/packages/chat/src/core/__tests__/chat.spec.ts b/packages/chat/src/core/__tests__/chat.spec.ts
new file mode 100644
index 0000000000..ffc11c922c
--- /dev/null
+++ b/packages/chat/src/core/__tests__/chat.spec.ts
@@ -0,0 +1,1462 @@
+import { StepEvents } from 'librechat-data-provider';
+import type { Agents, TMessage, TPayload, ChatEvent } from 'librechat-data-provider';
+import type { ChatStream, ChatStatus, ChatTransport, ChatConnection } from '../types';
+import type { ChatState, ChatInit } from '../chat';
+import { Chat } from '../chat';
+
+/** A stream the test feeds event by event; closing it ends the iteration. */
+const createStream = (meta: { streamId?: string; generationCreatedAt?: number } = {}) => {
+ const queued: ChatEvent[] = [];
+ let wake: (() => void) | undefined;
+ let closed = false;
+ const stream: ChatStream & { push(event: ChatEvent): void; closed(): boolean } = {
+ ...meta,
+ push(event) {
+ queued.push(event);
+ wake?.();
+ },
+ close() {
+ closed = true;
+ wake?.();
+ },
+ closed: () => closed,
+ async *[Symbol.asyncIterator]() {
+ while (true) {
+ while (queued.length > 0) {
+ yield queued.shift() as ChatEvent;
+ }
+ if (closed) {
+ return;
+ }
+ await new Promise<void>((resolve) => {
+ wake = resolve;
+ });
+ }
+ },
+ };
+ return stream;
+};
+
+const createState = (initial: TMessage[] = []) => {
+ let messages = initial;
+ const state: ChatState & { statuses: ChatStatus[] } = {
+ statuses: [],
+ error: undefined,
+ connection: 'idle' as ChatConnection,
+ get status() {
+ return this.statuses[this.statuses.length - 1] ?? 'ready';
+ },
+ set status(status: ChatStatus) {
+ this.statuses.push(status);
+ },
+ get messages() {
+ return messages;
+ },
+ replaceMessages(next) {
+ messages = next;
+ },
+ };
+ return state;
+};
+
+const userMessage: TMessage = {
+ messageId: 'user-1',
+ conversationId: 'convo-1',
+ parentMessageId: null,
+ isCreatedByUser: true,
+ text: 'Hi',
+};
+
+const placeholder: TMessage = {
+ messageId: 'response-1',
+ conversationId: 'convo-1',
+ parentMessageId: 'user-1',
+ isCreatedByUser: false,
+ text: '',
+ content: [],
+};
+
+const payload = { endpoint: 'agents', text: 'Hi' } as TPayload;
+
+const stepEvent: ChatEvent = {
+ type: 'step',
+ data: { event: StepEvents.ON_MESSAGE_DELTA, data: { id: 'step-1', delta: { content: [] } } },
+};
+
+const finalEvent = (overrides: Partial<Extract<ChatEvent, { type: 'final' }>['data']> = {}) =>
+ ({
+ type: 'final',
+ data: {
+ final: true,
+ requestMessage: userMessage,
+ responseMessage: { ...placeholder, text: 'Hello' },
+ ...overrides,
+ },
+ }) as ChatEvent;
+
+type Transport = jest.Mocked<Required<ChatTransport>>;
+
+const setup = (
+ overrides: Partial<ChatInit> = {},
+ transportOverrides: Partial<ChatTransport> = {},
+) => {
+ const stream = createStream({ streamId: 'convo-1', generationCreatedAt: 100 });
+ const transport = {
+ sendMessages: jest.fn(async () => ({ status: 'stream' as const, stream })),
+ reconnectToStream: jest.fn(async () => null),
+ abort: jest.fn(async () => ({ success: true })),
+ resume: jest.fn(async () => ({
+ streamId: 'convo-1',
+ conversationId: 'convo-1',
+ status: 'resuming' as const,
+ })),
+ ...transportOverrides,
+ } as unknown as Transport;
+ const state = createState([]);
+ const onFinish = jest.fn();
+ const onError = jest.fn();
+ const onEvent = jest.fn();
+ const chat = new Chat({
+ id: 'convo-1',
+ state,
+ transport,
+ prepareRequest: jest.fn(() => ({ payload, messages: [userMessage, placeholder] })),
+ onFinish,
+ onError,
+ onEvent,
+ ...overrides,
+ });
+ return { chat, stream, transport, state, onFinish, onError, onEvent };
+};
+
+/** Lets every queued stream event and executor job run. */
+const flush = () => new Promise<void>((resolve) => setTimeout(resolve, 0));
+
+describe('Chat', () => {
+ it('walks submitted, streaming and ready, merging the final messages', async () => {
+ const { chat, stream, transport, state, onFinish, onEvent } = setup();
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ expect(chat.status).toBe('submitted');
+ expect(chat.connection).toBe('live');
+ expect(chat.messages).toEqual([userMessage, placeholder]);
+ expect(transport.sendMessages).toHaveBeenCalledWith(
+ expect.objectContaining({ trigger: 'submit-message', chatId: 'convo-1', payload }),
+ );
+
+ stream.push({ type: 'created', data: { message: userMessage, created: true } });
+ await flush();
+ expect(chat.status).toBe('streaming');
+
+ stream.push(finalEvent());
+ await sent;
+
+ expect(state.statuses).toEqual(['submitted', 'streaming', 'ready']);
+ expect(chat.connection).toBe('idle');
+ expect(chat.messages[1].text).toBe('Hello');
+ expect(stream.closed()).toBe(true);
+ expect(onEvent).toHaveBeenCalledTimes(2);
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isAbort: false, isError: false, isDisconnect: false }),
+ );
+ });
+
+ describe('a host listener that throws', () => {
+ let reported: jest.SpyInstance;
+ beforeEach(() => {
+ reported = jest.spyOn(console, 'error').mockImplementation(() => undefined);
+ });
+
+ it.each([
+ ['the submitted state', (chat: Chat) => chat.status === 'submitted'],
+ ['the live connection', (chat: Chat) => chat.connection === 'live'],
+ ['a streamed frame', (chat: Chat) => chat.status === 'streaming'],
+ ['the terminal state', (chat: Chat) => chat.status === 'ready'],
+ ])('on %s neither ends the turn nor hides it from the others', async (_label, when) => {
+ const { chat, stream, onFinish } = setup();
+ const failure = new Error('listener failed');
+ let thrown = false;
+ chat.subscribe(() => {
+ if (!thrown && when(chat)) {
+ thrown = true;
+ throw failure;
+ }
+ });
+ const statuses: string[] = [];
+ chat.subscribe(() => statuses.push(chat.status));
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(stepEvent);
+ await flush();
+ stream.push(finalEvent());
+ await sent;
+
+ expect(thrown).toBe(true);
+ expect(reported).toHaveBeenCalledWith(failure);
+ expect(statuses).toEqual(expect.arrayContaining(['submitted', 'streaming', 'ready']));
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: false, isAbort: false, isDisconnect: false }),
+ );
+ expect(chat.status).toBe('ready');
+ expect(chat.connection).toBe('idle');
+ });
+
+ it('on dispose still releases the chat from its state', async () => {
+ const { chat, state } = setup();
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ const listener = jest.fn(() => {
+ throw new Error('listener failed');
+ });
+ chat.subscribe(listener);
+
+ chat.dispose();
+ listener.mockClear();
+ state.replaceMessages([userMessage]);
+ await sent;
+
+ expect(chat.connection).toBe('idle');
+ expect(listener).not.toHaveBeenCalled();
+ });
+
+ it('in onError still tells the host the turn finished', async () => {
+ const onError = jest.fn(() => {
+ throw new Error('callback failed');
+ });
+ const { chat, stream, onFinish } = setup({ onError });
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({ type: 'error', status: 500, data: 'boom' } as ChatEvent);
+ await sent;
+
+ expect(onError).toHaveBeenCalled();
+ expect(onFinish).toHaveBeenCalledWith(expect.objectContaining({ isError: true }));
+ });
+
+ it('in onEvent still ends the turn by its terminal frame', async () => {
+ const onEvent = jest.fn(() => {
+ throw new Error('callback failed');
+ });
+ const { chat, stream, onFinish } = setup({ onEvent });
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(finalEvent());
+ await sent;
+
+ expect(onEvent).toHaveBeenCalled();
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: false, isDisconnect: false }),
+ );
+ });
+ });
+
+ describe('dispose', () => {
+ it('ends a turn that never reached the server and releases its send', async () => {
+ const { chat, transport } = setup({
+ prepareRequest: () => new Promise<never>(() => undefined),
+ });
+ const statuses: string[] = [];
+ chat.subscribe(() => statuses.push(chat.status));
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+
+ chat.dispose();
+
+ await expect(sent).resolves.toBeUndefined();
+ expect(chat.status).toBe('ready');
+ expect(statuses[statuses.length - 1]).toBe('ready');
+ expect(transport.sendMessages).not.toHaveBeenCalled();
+ });
+
+ it('undoes the optimistic rows of a turn disposed before it was sent', async () => {
+ const earlier: TMessage = { ...placeholder, messageId: 'earlier-1', text: 'Earlier' };
+ const state = createState([earlier]);
+ const { chat, transport } = setup({
+ state,
+ prepareRequest: jest.fn(() => ({
+ payload,
+ messages: (current: TMessage[]) => [...current, userMessage, placeholder],
+ })),
+ });
+ chat.subscribe(() => {
+ if (chat.messages.length > 1) {
+ chat.dispose();
+ }
+ });
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(transport.sendMessages).not.toHaveBeenCalled();
+ expect(state.messages.map((message) => message.messageId)).toEqual(['earlier-1']);
+ expect(chat.status).toBe('ready');
+ });
+
+ it('keeps a sent turn running on the server but releases its send', async () => {
+ const { chat, transport } = setup();
+ transport.sendMessages.mockImplementationOnce(() => new Promise<never>(() => undefined));
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+
+ chat.dispose();
+
+ await expect(sent).resolves.toBeUndefined();
+ expect(chat.status).toBe('submitted');
+ expect(chat.connection).toBe('idle');
+ });
+ });
+
+ it('streams a turn whose first frame is a pause', async () => {
+ const { chat, stream } = setup();
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ expect(chat.status).toBe('submitted');
+
+ stream.push({
+ type: 'pending_action',
+ data: {
+ actionId: 'action-1',
+ streamId: 'convo-1',
+ conversationId: 'convo-1',
+ createdAt: 1,
+ expiresAt: Date.now() + 60_000,
+ payload: { type: 'ask_user_question', question: { question: 'Which?' }, tool_call_id: 'a' },
+ } as Agents.PendingAction,
+ });
+ await flush();
+
+ expect(chat.status).toBe('streaming');
+ stream.push(finalEvent());
+ await sent;
+ });
+
+ it('refuses a settled start the server did not negotiate under protocol v2', async () => {
+ const { chat, onFinish } = setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'settled' as const,
+ conversationId: 'convo-1',
+ })),
+ },
+ );
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(chat.status).toBe('error');
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: true, isSettled: false }),
+ );
+ });
+
+ it('notifies subscribers of each change', async () => {
+ const { chat, stream } = setup();
+ const listener = jest.fn();
+ const unsubscribe = chat.subscribe(listener);
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ stream.push(finalEvent());
+ await sent;
+ const calls = listener.mock.calls.length;
+ unsubscribe();
+ await chat.regenerate().catch(() => undefined);
+
+ expect(calls).toBeGreaterThan(0);
+ expect(listener).toHaveBeenCalledTimes(calls);
+ });
+
+ it('applies events through the host reducer one at a time, in wire order', async () => {
+ const seen: string[] = [];
+ const { chat, stream } = setup({
+ applyEvent: (event, context) => {
+ seen.push(event.type);
+ if (event.type === 'step') {
+ context.write([...context.messages, { ...placeholder, messageId: 'extra' }]);
+ }
+ },
+ });
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(stepEvent);
+ stream.push(stepEvent);
+ stream.push(finalEvent());
+ await sent;
+
+ expect(seen).toEqual(['step', 'step', 'final']);
+ expect(chat.messages.map((message) => message.messageId)).toEqual([
+ 'user-1',
+ 'response-1',
+ 'extra',
+ 'extra',
+ ]);
+ });
+
+ it('reports an error event as an error until it is cleared', async () => {
+ const { chat, stream, onError, onFinish } = setup();
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({ type: 'error', data: { message: 'Rate limited' } });
+ await sent;
+
+ expect(chat.status).toBe('error');
+ expect(chat.error?.message).toBe('Rate limited');
+ expect(onError).toHaveBeenCalledWith(chat.error);
+ expect(onFinish).toHaveBeenCalledWith(expect.objectContaining({ isError: true }));
+
+ chat.clearError();
+ expect(chat.status).toBe('ready');
+ expect(chat.error).toBeUndefined();
+ });
+
+ it('reports a final event with an error terminal status as an error', async () => {
+ const { chat, stream } = setup();
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ stream.push(finalEvent({ terminalStatus: 'error', error: { message: 'Context too long' } }));
+ await sent;
+
+ expect(chat.status).toBe('error');
+ expect(chat.error?.message).toBe('Context too long');
+ });
+
+ it('marks a dropped connection and a stream that ends early as a disconnect', async () => {
+ const dropped = setup();
+ const sent = dropped.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ dropped.stream.push({ type: 'error', status: 0 });
+ await sent;
+ expect(dropped.onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: true, isDisconnect: true }),
+ );
+
+ const ended = setup();
+ const endedSend = ended.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ ended.stream.close();
+ await endedSend;
+ expect(ended.chat.status).toBe('error');
+ expect(ended.onFinish).toHaveBeenCalledWith(expect.objectContaining({ isDisconnect: true }));
+ });
+
+ it('reports a failed start as an error and frees the chat for the next turn', async () => {
+ const failure = new Error('start failed');
+ const { chat, transport } = setup(
+ {},
+ { sendMessages: jest.fn(async () => Promise.reject(failure)) },
+ );
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(chat.status).toBe('error');
+ expect(chat.error).toBe(failure);
+ transport.sendMessages.mockResolvedValueOnce({
+ status: 'settled',
+ conversationId: 'convo-1',
+ generationProtocolVersion: 2,
+ });
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Again' }] });
+ expect(chat.status).toBe('ready');
+ expect(chat.error).toBeUndefined();
+ });
+
+ it('refuses a second turn while one is in flight', async () => {
+ const { chat, stream } = setup();
+
+ const first = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await expect(chat.sendMessage({ parts: [{ type: 'text', text: 'Again' }] })).rejects.toThrow(
+ 'already in flight',
+ );
+ stream.push(finalEvent());
+ await first;
+
+ expect(chat.status).toBe('ready');
+ });
+
+ it('merges a prepared turn into the list as it stands when the turn is written', async () => {
+ const loaded: TMessage = { ...userMessage, messageId: 'loaded-1', text: 'Loaded' };
+ let prepared: (() => void) | undefined;
+ const { chat, state, stream } = setup({
+ prepareRequest: jest.fn(
+ () =>
+ new Promise((resolve) => {
+ prepared = () =>
+ resolve({
+ payload,
+ messages: (current: TMessage[]) => [...current, userMessage, placeholder],
+ });
+ }),
+ ),
+ });
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ state.replaceMessages([loaded]);
+ prepared?.();
+ await flush();
+
+ expect(state.messages.map((message) => message.messageId)).toEqual([
+ 'loaded-1',
+ 'user-1',
+ 'response-1',
+ ]);
+ stream.push(finalEvent());
+ await sent;
+ });
+
+ it('drops its stream on dispose without reporting an error', async () => {
+ const { chat, stream, onError, onFinish } = setup();
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(stepEvent);
+ await flush();
+ chat.dispose();
+ await sent;
+
+ expect(stream.closed()).toBe(true);
+ expect(onError).not.toHaveBeenCalled();
+ expect(onFinish).not.toHaveBeenCalled();
+ expect(chat.status).toBe('streaming');
+ });
+
+ it('settles a start the server already finished, and attaches to the generation replacing it', async () => {
+ const settled = setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'settled' as const,
+ conversationId: 'convo-1',
+ generationProtocolVersion: 2,
+ })),
+ },
+ );
+ await settled.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ expect(settled.chat.status).toBe('ready');
+
+ const stream = createStream();
+ const replaced = setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => stream),
+ },
+ );
+ const sent = replaced.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(finalEvent());
+ await sent;
+ expect(replaced.transport.reconnectToStream).toHaveBeenCalledWith(
+ expect.objectContaining({ streamId: 'stream-2', generationCreatedAt: 200, resume: true }),
+ );
+ expect(replaced.chat.status).toBe('ready');
+ });
+
+ it('lets a completion callback start the next turn', async () => {
+ let next: Promise<void> | undefined;
+ const { chat, stream, transport } = setup({
+ onFinish: () => {
+ next ??= chat.sendMessage({ parts: [{ type: 'text', text: 'Again' }] });
+ },
+ });
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(finalEvent());
+ await sent;
+ await next;
+
+ expect(transport.sendMessages).toHaveBeenCalledTimes(2);
+ });
+
+ it('keeps the next turn submitted when a subscriber starts it as the connection goes idle', async () => {
+ let next: Promise<void> | undefined;
+ const { chat, stream } = setup();
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ let started = false;
+ const unsubscribe = chat.subscribe(() => {
+ if (chat.connection === 'idle' && !started) {
+ started = true;
+ next = chat.sendMessage({ parts: [{ type: 'text', text: 'Again' }] });
+ }
+ });
+ stream.push(finalEvent());
+ await sent;
+
+ expect(next).toBeDefined();
+ expect(chat.status).toBe('submitted');
+ unsubscribe();
+ });
+
+ it.each(['final', 'abort'] as const)(
+ 'finishes on %s with the response the turn produced, not the tail of another branch',
+ async (end) => {
+ const otherBranch: TMessage = { ...placeholder, messageId: 'other-1', parentMessageId: 'x' };
+ const { chat, stream, onFinish } = setup({
+ prepareRequest: jest.fn(() => ({
+ payload,
+ messages: [userMessage, placeholder, otherBranch],
+ responseMessageId: 'response-1',
+ })),
+ });
+
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(end === 'final' ? finalEvent() : ({ type: 'abort', data: {} } as ChatEvent));
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ message: expect.objectContaining({ messageId: 'response-1' }) }),
+ );
+ },
+ );
+
+ it('refuses a replacement the server did not report under protocol v2', async () => {
+ const { chat, transport, onError } = setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ },
+ );
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(transport.reconnectToStream).not.toHaveBeenCalled();
+ expect(chat.status).toBe('error');
+ expect(onError).toHaveBeenCalled();
+ });
+
+ it('settles a final frame that asks for reconciliation instead of completing it', async () => {
+ const { chat, stream, onFinish } = setup();
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({
+ type: 'final',
+ data: {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'abort_persistence_failed',
+ terminalStatus: 'aborted',
+ },
+ } as ChatEvent);
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({
+ isAbort: true,
+ isSettled: true,
+ reconcileReason: 'abort_persistence_failed',
+ }),
+ );
+ });
+
+ it.each(['ERR_NETWORK', 'ERR_INTERNET_DISCONNECTED'])(
+ 'reports a start that never got a response (%s) as a disconnect',
+ async (code) => {
+ const failure = Object.assign(new Error('Network Error'), { code });
+ const { chat, onFinish } = setup(
+ {},
+ { sendMessages: jest.fn(async () => Promise.reject(failure)) },
+ );
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: true, isDisconnect: true }),
+ );
+ },
+ );
+
+ it('reports a start the server settled before attaching for the host to reconcile', async () => {
+ const { chat, onFinish } = setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'settled' as const,
+ conversationId: 'c-9',
+ generationProtocolVersion: 2,
+ })),
+ },
+ );
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isSettled: true, conversationId: 'c-9' }),
+ );
+ });
+
+ it('reports no message when the named response left the list', async () => {
+ const { chat, stream, state, onFinish } = setup({
+ prepareRequest: jest.fn(() => ({
+ payload,
+ messages: [userMessage, placeholder],
+ responseMessageId: 'response-1',
+ })),
+ });
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ state.replaceMessages([userMessage]);
+ stream.push({ type: 'abort', data: {} } as ChatEvent);
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(expect.objectContaining({ message: undefined }));
+ });
+
+ it('continues a response with the continue trigger', async () => {
+ const { chat, stream, transport } = setup();
+ const sent = chat.continue({ messageId: 'response-1' });
+ await flush();
+ stream.push(finalEvent());
+ await sent;
+
+ expect(transport.sendMessages).toHaveBeenCalledWith(
+ expect.objectContaining({ trigger: 'continue-message', messageId: 'response-1' }),
+ );
+ });
+
+ it('regenerates a given message', async () => {
+ const { stream, transport } = setup();
+ const prepareRequest = jest.fn(() => ({ payload }));
+ const regenerating = new Chat({
+ id: 'convo-1',
+ state: createState(),
+ transport,
+ prepareRequest,
+ });
+
+ const sent = regenerating.regenerate({ messageId: 'response-1' });
+ stream.push(finalEvent());
+ await sent;
+
+ expect(prepareRequest).toHaveBeenCalledWith(
+ expect.objectContaining({ trigger: 'regenerate-message', messageId: 'response-1' }),
+ );
+ expect(transport.sendMessages).toHaveBeenCalledWith(
+ expect.objectContaining({ trigger: 'regenerate-message', messageId: 'response-1' }),
+ );
+ expect(regenerating.status).toBe('ready');
+ });
+
+ describe('replacement responses', () => {
+ it('reports the response of a replacement that ended before any frame named it as unknown', async () => {
+ const replacing = () => {
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ const harness = setup(
+ {
+ prepareRequest: jest.fn(() => ({
+ payload,
+ messages: [userMessage, placeholder],
+ responseMessageId: placeholder.messageId,
+ })),
+ },
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => stream),
+ },
+ );
+ return { ...harness, stream };
+ };
+
+ const early = replacing();
+ const sent = early.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ early.stream.push({ type: 'abort', data: {} } as ChatEvent);
+ await sent;
+ expect(early.onFinish).toHaveBeenLastCalledWith(
+ expect.objectContaining({ isAbort: true, isResponseUnknown: true, message: undefined }),
+ );
+
+ const named = replacing();
+ const replacement: TMessage = { ...placeholder, messageId: 'response-2' };
+ const sentNamed = named.chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ named.state.replaceMessages([...named.chat.messages, replacement]);
+ named.stream.push({
+ type: 'sync',
+ data: { conversationId: 'convo-1', responseMessage: replacement },
+ } as ChatEvent);
+ named.stream.push({ type: 'abort', data: {} } as ChatEvent);
+ await sentNamed;
+ expect(named.onFinish).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ isResponseUnknown: false,
+ message: expect.objectContaining({ messageId: 'response-2' }),
+ }),
+ );
+ });
+
+ it('sends the request options of a turn to the generation that replaced it', async () => {
+ const stream = createStream();
+ const { chat, transport } = setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => stream),
+ },
+ );
+ const options = { headers: { 'x-tenant': 'acme' }, body: { spec: 'a' } };
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] }, options);
+ await flush();
+ stream.push(finalEvent());
+ await sent;
+
+ expect(transport.reconnectToStream).toHaveBeenCalledWith(
+ expect.objectContaining({ ...options, chatId: 'convo-1', streamId: 'stream-2' }),
+ );
+ });
+ });
+
+ describe('ids a turn reports', () => {
+ it('opens no replacement a listener disposed the chat from on its reconnecting notice', async () => {
+ const { chat, transport } = setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ },
+ );
+ chat.subscribe(() => {
+ if (chat.connection === 'reconnecting') {
+ chat.dispose();
+ }
+ });
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] }).catch(() => undefined);
+
+ expect(transport.reconnectToStream).not.toHaveBeenCalled();
+ });
+
+ it('stays idle when a listener disposes the chat on the rollback of a replaced start', async () => {
+ const { chat, transport, state } = setup(
+ {
+ prepareRequest: jest.fn(() => ({ payload, messages: [userMessage, placeholder] })),
+ },
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ },
+ );
+ let written = false;
+ chat.subscribe(() => {
+ written ||= chat.messages.length > 0;
+ if (written && chat.messages.length === 0) {
+ chat.dispose();
+ }
+ });
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(transport.reconnectToStream).not.toHaveBeenCalled();
+ expect(state.connection).toBe('idle');
+ });
+
+ it('stops a stream event when a listener disposes the chat on its write', async () => {
+ const { chat, stream, state, onEvent } = setup({
+ applyEvent: (event, context) => {
+ if (event.type === 'created') {
+ context.write([...context.messages, { ...placeholder, messageId: 'created-1' }]);
+ }
+ },
+ });
+ chat.subscribe(() => {
+ if (chat.messages.some((m) => m.messageId === 'created-1')) {
+ chat.dispose();
+ }
+ });
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({ type: 'created', data: { message: userMessage, created: true } });
+ await sent;
+
+ expect(state.statuses).not.toContain('streaming');
+ expect(onEvent).not.toHaveBeenCalled();
+ });
+
+ it('reports no response for a turn whose optimistic write failed', async () => {
+ const earlier: TMessage = { ...placeholder, messageId: 'earlier-1', text: 'Earlier' };
+ const { chat, onFinish, transport } = setup({
+ state: createState([userMessage, earlier]),
+ prepareRequest: jest.fn(() => ({
+ payload,
+ messages: () => {
+ throw new Error('Write failed');
+ },
+ })),
+ });
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] }).catch(() => undefined);
+
+ expect(transport.sendMessages).not.toHaveBeenCalled();
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: true, message: undefined }),
+ );
+ });
+
+ it('reports no response for a turn whose preparation failed', async () => {
+ const earlier: TMessage = { ...placeholder, messageId: 'earlier-1', text: 'Earlier' };
+ const { chat, onFinish } = setup({
+ state: createState([userMessage, earlier]),
+ prepareRequest: jest.fn(() => Promise.reject(new Error('Upload failed'))),
+ });
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] }).catch(() => undefined);
+
+ expect(onFinish).toHaveBeenCalledTimes(1);
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: true, message: undefined }),
+ );
+ });
+ });
+});
+
+describe('Chat attachments and reported conversations', () => {
+ const replacing = (stream: ReturnType<typeof createStream>) =>
+ setup(
+ {},
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => stream),
+ },
+ );
+
+ it.each([
+ [500, true],
+ [401, true],
+ [404, false],
+ [410, false],
+ ])(
+ 'reports an attachment that fails with HTTP %s as a disconnect: %s',
+ async (status, disconnect) => {
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ const { chat, onFinish } = replacing(stream);
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({ type: 'error', status, data: 'failed' } as ChatEvent);
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(expect.objectContaining({ isDisconnect: disconnect }));
+ },
+ );
+
+ it('reports an attachment failure the host reducer threw on as a disconnect', async () => {
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ const { chat, onFinish } = setup(
+ {
+ applyEvent: (event) => {
+ if (event.type === 'error') {
+ throw new Error('reducer failed');
+ }
+ },
+ },
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => stream),
+ },
+ );
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({ type: 'error', status: 500, data: 'failed' } as ChatEvent);
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isDisconnect: true, isSettled: false }),
+ );
+ });
+
+ it('reports a resumed start that fails over HTTP as a disconnect', async () => {
+ const stream = Object.assign(createStream({ streamId: 'convo-1', generationCreatedAt: 100 }), {
+ resumed: true,
+ });
+ const { chat, onFinish } = setup(
+ {},
+ { sendMessages: jest.fn(async () => ({ status: 'stream' as const, stream })) },
+ );
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({ type: 'error', status: 500, data: 'failed' } as ChatEvent);
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(expect.objectContaining({ isDisconnect: true }));
+ });
+
+ it('drops the optimistic rows of a start the server replaced before attaching', async () => {
+ const earlier: TMessage = { ...placeholder, messageId: 'earlier-1', text: 'Earlier' };
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ let attachedWith: string[] = [];
+ const { chat } = setup(
+ {
+ state: createState([earlier]),
+ prepareRequest: jest.fn(() => ({
+ payload,
+ messages: (current: TMessage[]) => [...current, userMessage, placeholder],
+ })),
+ },
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => {
+ attachedWith = chat.messages.map((message) => message.messageId);
+ return stream;
+ }),
+ },
+ );
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(finalEvent({ requestMessage: null, responseMessage: null }));
+ await sent;
+
+ expect(attachedWith).toEqual(['earlier-1']);
+ });
+
+ it.each([
+ [
+ 'restores a row it overwrote',
+ (current: TMessage[]) =>
+ current.map((m) => (m.messageId === 'earlier-2' ? { ...m, text: '' } : m)),
+ undefined,
+ ['earlier-1:Earlier', 'earlier-2:Answer'],
+ ],
+ [
+ 'restores a row it removed in place',
+ (current: TMessage[]) => current.filter((m) => m.messageId !== 'earlier-1'),
+ undefined,
+ ['earlier-1:Earlier', 'earlier-2:Answer'],
+ ],
+ [
+ 'restores a row it removed after the row before it, past a later prepend',
+ (current: TMessage[]) => current.filter((m) => m.messageId !== 'earlier-2'),
+ (current: TMessage[]) => [{ ...placeholder, messageId: 'x', text: 'X' }, ...current],
+ ['x:X', 'earlier-1:Earlier', 'earlier-2:Answer'],
+ ],
+ [
+ 'keeps a later write to a row it overwrote',
+ (current: TMessage[]) =>
+ current.map((m) => (m.messageId === 'earlier-2' ? { ...m, text: '' } : m)),
+ (current: TMessage[]) =>
+ current.map((m) => (m.messageId === 'earlier-2' ? { ...m, text: 'External' } : m)),
+ ['earlier-1:Earlier', 'earlier-2:External'],
+ ],
+ ])(
+ 'undoes the optimistic write of a replaced start: %s',
+ async (_label, optimistic, external, expected) => {
+ const earlier: TMessage[] = [
+ { ...placeholder, messageId: 'earlier-1', text: 'Earlier' },
+ { ...placeholder, messageId: 'earlier-2', text: 'Answer' },
+ ];
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ const state = createState(earlier);
+ let attachedWith: string[] = [];
+ let answer: (() => void) | undefined;
+ const { chat } = setup(
+ {
+ state,
+ prepareRequest: jest.fn(() => ({ payload, messages: optimistic })),
+ },
+ {
+ sendMessages: jest.fn(
+ () =>
+ new Promise((resolve) => {
+ answer = () =>
+ resolve({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ });
+ }),
+ ),
+ reconnectToStream: jest.fn(async () => {
+ attachedWith = chat.messages.map((m) => `${m.messageId}:${m.text}`);
+ return stream;
+ }),
+ },
+ );
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ if (external) {
+ state.replaceMessages(external(state.messages));
+ }
+ answer?.();
+ await flush();
+ stream.push(finalEvent({ requestMessage: null, responseMessage: null }));
+ await sent;
+
+ expect(attachedWith).toEqual(expected);
+ },
+ );
+
+ it.each([
+ ['over a plain state', false],
+ ['over a store that reports its own writes', true],
+ ])(
+ 'keeps a row a listener appended on the optimistic write when a replaced start rolls back, %s',
+ async (_label, viaStore) => {
+ const earlier: TMessage = { ...placeholder, messageId: 'earlier-1', text: 'Earlier' };
+ const external: TMessage = { ...placeholder, messageId: 'external-1', text: 'External' };
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ const state = createState([earlier]);
+ const storeListeners = new Set<() => void>();
+ if (viaStore) {
+ const replaceMessages = state.replaceMessages.bind(state);
+ state.replaceMessages = (messages: TMessage[]) => {
+ replaceMessages(messages);
+ storeListeners.forEach((listener) => listener());
+ };
+ state.subscribe = (listener: () => void) => {
+ storeListeners.add(listener);
+ return () => storeListeners.delete(listener);
+ };
+ }
+ let attachedWith: string[] = [];
+ const { chat } = setup(
+ {
+ state,
+ prepareRequest: jest.fn(() => ({
+ payload,
+ messages: (current: TMessage[]) => [...current, userMessage, placeholder],
+ })),
+ },
+ {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => {
+ attachedWith = chat.messages.map((m) => m.messageId);
+ return stream;
+ }),
+ },
+ );
+ const append = () => {
+ const current = state.messages;
+ if (
+ current.some((m) => m.messageId === placeholder.messageId) &&
+ !current.includes(external)
+ ) {
+ state.replaceMessages([...current, external]);
+ }
+ };
+ chat.subscribe(append);
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(finalEvent({ requestMessage: null, responseMessage: null }));
+ await sent;
+
+ expect(attachedWith).toEqual(['earlier-1', 'external-1']);
+ },
+ );
+
+ describe('optimistic rollback at the store boundary', () => {
+ const earlier: TMessage[] = [
+ { ...placeholder, messageId: 'earlier-1', text: 'Earlier' },
+ { ...placeholder, messageId: 'earlier-2', text: 'Answer' },
+ ];
+ const external: TMessage = { ...placeholder, messageId: 'external-1', text: 'External' };
+ /** Overwrites `earlier-2` and appends the turn, so the write both replaces and adds rows. */
+ const optimistic = (current: TMessage[]) => [
+ ...current.map((m) => (m.messageId === 'earlier-2' ? { ...m, text: '' } : m)),
+ userMessage,
+ placeholder,
+ ];
+ const replaced = {
+ status: 'replaced' as const,
+ generationProtocolVersion: 2,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ };
+
+ /**
+ * A store that tells its own subscribers of every write before `replaceMessages` returns and,
+ * when `shared`, commits each changed row as a copy, as a cache with structural sharing does.
+ */
+ const createStore = (shared: boolean) => {
+ const state = createState(earlier);
+ const listeners = new Set<() => void>();
+ const replace = state.replaceMessages.bind(state);
+ state.replaceMessages = (next: TMessage[]) => {
+ const held = new Map(state.messages.map((m) => [m.messageId, m]));
+ replace(shared ? next.map((m) => (held.get(m.messageId) === m ? m : { ...m })) : next);
+ [...listeners].forEach((listener) => listener());
+ };
+ state.subscribe = (listener: () => void) => {
+ listeners.add(listener);
+ return () => {
+ listeners.delete(listener);
+ };
+ };
+ return state;
+ };
+
+ /** A store subscriber that appends `external` once the turn's placeholder is listed. */
+ const appendOnWrite = (state: ChatState) => () => {
+ const current = state.messages;
+ if (
+ current.some((m) => m.messageId === placeholder.messageId) &&
+ !current.some((m) => m.messageId === external.messageId)
+ ) {
+ state.replaceMessages([...current, external]);
+ }
+ };
+
+ const rows = (messages: TMessage[]) => messages.map((m) => `${m.messageId}:${m.text}`);
+
+ it.each([
+ ['over a store that reports writes synchronously', false],
+ ['over a structurally shared store', true],
+ ])(
+ 'keeps a row a store subscriber appended inside the write when a replaced start rolls back, %s',
+ async (_label, shared) => {
+ const state = createStore(shared);
+ state.subscribe?.(appendOnWrite(state));
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ let attachedWith: string[] = [];
+ const { chat } = setup(
+ { state, prepareRequest: jest.fn(() => ({ payload, messages: optimistic })) },
+ {
+ sendMessages: jest.fn(async () => replaced),
+ reconnectToStream: jest.fn(async () => {
+ attachedWith = rows(chat.messages);
+ return stream;
+ }),
+ },
+ );
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(finalEvent({ requestMessage: null, responseMessage: null }));
+ await sent;
+
+ expect(attachedWith).toEqual([
+ 'earlier-1:Earlier',
+ 'earlier-2:Answer',
+ 'external-1:External',
+ ]);
+ },
+ );
+
+ it.each([
+ ['over a store that reports writes synchronously', false],
+ ['over a structurally shared store', true],
+ ])(
+ 'undoes the write of a turn a store subscriber disposed inside that write, %s',
+ async (_label, shared) => {
+ const state = createStore(shared);
+ const append = appendOnWrite(state);
+ const { chat, transport } = setup({
+ state,
+ prepareRequest: jest.fn(() => ({ payload, messages: optimistic })),
+ });
+ state.subscribe?.(() => {
+ if (state.messages.some((m) => m.messageId === placeholder.messageId)) {
+ append();
+ chat.dispose();
+ }
+ });
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(transport.sendMessages).not.toHaveBeenCalled();
+ expect(rows(state.messages)).toEqual([
+ 'earlier-1:Earlier',
+ 'earlier-2:Answer',
+ 'external-1:External',
+ ]);
+ expect(chat.status).toBe('ready');
+ },
+ );
+
+ it.each([
+ ['over a store that reports writes synchronously', false],
+ ['over a structurally shared store', true],
+ ])(
+ 'keeps the edits a store subscriber made to the turn rows inside the write when a replaced start rolls back, %s',
+ async (_label, shared) => {
+ const state = createStore(shared);
+ state.subscribe?.(() => {
+ const current = state.messages;
+ if (current.some((m) => m.messageId === placeholder.messageId && m.text === '')) {
+ state.replaceMessages(
+ current.map((m) =>
+ m.messageId === placeholder.messageId || m.messageId === 'earlier-2'
+ ? { ...m, text: 'Edited' }
+ : m,
+ ),
+ );
+ }
+ });
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ let attachedWith: string[] = [];
+ const { chat } = setup(
+ { state, prepareRequest: jest.fn(() => ({ payload, messages: optimistic })) },
+ {
+ sendMessages: jest.fn(async () => replaced),
+ reconnectToStream: jest.fn(async () => {
+ attachedWith = rows(chat.messages);
+ return stream;
+ }),
+ },
+ );
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push(finalEvent({ requestMessage: null, responseMessage: null }));
+ await sent;
+
+ expect(attachedWith).toEqual([
+ 'earlier-1:Earlier',
+ 'earlier-2:Edited',
+ 'response-1:Edited',
+ ]);
+ },
+ );
+
+ it('keeps a later write to a row the turn added when a structurally shared store rolls back', async () => {
+ const state = createStore(true);
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ let attachedWith: string[] = [];
+ let answer: (() => void) | undefined;
+ const { chat } = setup(
+ { state, prepareRequest: jest.fn(() => ({ payload, messages: optimistic })) },
+ {
+ sendMessages: jest.fn(
+ () =>
+ new Promise((resolve) => {
+ answer = () => resolve(replaced);
+ }),
+ ),
+ reconnectToStream: jest.fn(async () => {
+ attachedWith = rows(chat.messages);
+ return stream;
+ }),
+ },
+ );
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ state.replaceMessages(
+ state.messages.map((m) =>
+ m.messageId === userMessage.messageId ? { ...m, text: 'Edited' } : m,
+ ),
+ );
+ answer?.();
+ await flush();
+ stream.push(finalEvent({ requestMessage: null, responseMessage: null }));
+ await sent;
+
+ expect(attachedWith).toEqual(['earlier-1:Earlier', 'earlier-2:Answer', 'user-1:Edited']);
+ });
+ });
+
+ it('reports an error the generation itself sent on an attachment as terminal', async () => {
+ const stream = createStream({ streamId: 'stream-2', generationCreatedAt: 200 });
+ const { chat, onFinish } = replacing(stream);
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({ type: 'error', data: 'The run failed' } as ChatEvent);
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(expect.objectContaining({ isDisconnect: false }));
+ });
+
+ it('reports a stream that fails while it is read as a disconnect', async () => {
+ const failing = {
+ streamId: 'convo-1',
+ generationCreatedAt: 100,
+ close: jest.fn(),
+ async *[Symbol.asyncIterator]() {
+ yield stepEvent;
+ throw new Error('The body could not be read');
+ },
+ };
+ const { chat, onFinish } = setup(
+ {},
+ { sendMessages: jest.fn(async () => ({ status: 'stream' as const, stream: failing })) },
+ );
+
+ await chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+
+ expect(onFinish).toHaveBeenCalledWith(
+ expect.objectContaining({ isError: true, isDisconnect: true }),
+ );
+ });
+
+ it('reports the conversation a first turn was created under on its finish', async () => {
+ const { chat, stream, onFinish } = setup({
+ id: 'new',
+ prepareRequest: jest.fn(() => ({
+ payload: { ...payload, conversationId: null } as unknown as TPayload,
+ messages: [userMessage, placeholder],
+ })),
+ });
+ const sent = chat.sendMessage({ parts: [{ type: 'text', text: 'Hi' }] });
+ await flush();
+ stream.push({
+ type: 'created',
+ data: { message: { ...userMessage, conversationId: 'convo-9' }, created: true },
+ } as ChatEvent);
+ stream.push(
+ finalEvent({
+ requestMessage: { ...userMessage, conversationId: 'convo-9' },
+ responseMessage: { ...placeholder, conversationId: 'convo-9', text: 'Hello' },
+ }),
+ );
+ await sent;
+
+ expect(onFinish).toHaveBeenCalledWith(expect.objectContaining({ conversationId: 'convo-9' }));
+ });
+});
diff --git a/packages/chat/src/core/__tests__/executor.spec.ts b/packages/chat/src/core/__tests__/executor.spec.ts
new file mode 100644
index 0000000000..b1b758bc29
--- /dev/null
+++ b/packages/chat/src/core/__tests__/executor.spec.ts
@@ -0,0 +1,63 @@
+import { SerialJobExecutor } from '../executor';
+
+describe('SerialJobExecutor', () => {
+ it('runs jobs one at a time in the order they were queued', async () => {
+ const executor = new SerialJobExecutor();
+ const order: string[] = [];
+ let release: () => void = () => {};
+ const gate = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+
+ const first = executor.run(async () => {
+ order.push('first:start');
+ await gate;
+ order.push('first:end');
+ });
+ const second = executor.run(() => {
+ order.push('second');
+ });
+
+ await Promise.resolve();
+ expect(order).toEqual(['first:start']);
+ expect(executor.idle).toBe(false);
+
+ release();
+ await Promise.all([first, second]);
+ expect(order).toEqual(['first:start', 'first:end', 'second']);
+ expect(executor.idle).toBe(true);
+ });
+
+ it('rejects the failing job and keeps running the ones behind it', async () => {
+ const executor = new SerialJobExecutor();
+ const failure = new Error('write failed');
+ const ran: string[] = [];
+
+ const failed = executor.run(() => {
+ throw failure;
+ });
+ const next = executor.run(() => {
+ ran.push('next');
+ });
+
+ await expect(failed).rejects.toBe(failure);
+ await next;
+ expect(ran).toEqual(['next']);
+ });
+
+ it('runs a job queued from inside a running job after it', async () => {
+ const executor = new SerialJobExecutor();
+ const order: string[] = [];
+ let inner: Promise<void> | undefined;
+
+ await executor.run(() => {
+ inner = executor.run(() => {
+ order.push('inner');
+ });
+ order.push('outer');
+ });
+ await inner;
+
+ expect(order).toEqual(['outer', 'inner']);
+ });
+});
diff --git a/packages/chat/src/core/chat.ts b/packages/chat/src/core/chat.ts
new file mode 100644
index 0000000000..fa1ce18ec6
--- /dev/null
+++ b/packages/chat/src/core/chat.ts
@@ -0,0 +1,1050 @@
+import { Constants } from 'librechat-data-provider';
+import type {
+ TPayload,
+ TMessage,
+ ChatEvent,
+ UITextPart,
+ ChatFinalFrame,
+ ChatErrorData,
+ UIMessageMetadata,
+} from 'librechat-data-provider';
+import type {
+ ChatStream,
+ ChatStatus,
+ ChatTrigger,
+ ChatTransport,
+ ChatConnection,
+ ChatRequestOptions,
+ ChatSendResult,
+ ChatTransportError,
+} from './types';
+import { isRetryableNetworkError } from './resumable/start';
+import { SerialJobExecutor } from './executor';
+
+/**
+ * Where a chat keeps its messages, status and error. AI SDK: `ChatState`. The app backs it with
+ * the React Query message cache, so every reader of that cache sees what the chat writes.
+ */
+export interface ChatState {
+ status: ChatStatus;
+ error: Error | undefined;
+ connection: ChatConnection;
+ /** The conversation's stored messages: every branch, in stored order. */
+ readonly messages: TMessage[];
+ /** Replaces the whole list; tree writes are whole-list, as the message cache takes them. */
+ replaceMessages(messages: TMessage[]): void;
+ /** Reports writes made outside the chat, such as another hook writing the same cache. */
+ subscribe?(listener: () => void): () => void;
+}
+
+/**
+ * A user turn in the AI SDK's `sendMessage` shape: its text parts, plus the message it attaches
+ * under. File parts are left out because a turn takes its files from the composer.
+ */
+export type SendMessageInput = {
+ parts: UITextPart[];
+ metadata?: Partial<Pick<UIMessageMetadata, 'parentMessageId'>>;
+};
+
+/** What the host turns into a request body. */
+export type ChatPrepareRequest = {
+ trigger: ChatTrigger;
+ chatId: string;
+ messages: TMessage[];
+ /** The user turn, for `submit-message`. */
+ input?: SendMessageInput;
+ /** The message to regenerate, for `regenerate-message`. */
+ messageId?: string;
+};
+
+export type ChatPreparedRequest = {
+ payload: TPayload;
+ /**
+ * The list to show while the request is in flight: the user turn and its placeholder reply. A
+ * function receives the list current when it is written, so writes made while the request was
+ * prepared are kept; a list replaces it as prepared.
+ */
+ messages?: TMessage[] | ((current: TMessage[]) => TMessage[]);
+ /**
+ * The placeholder reply's id, which `onFinish` reports when the turn ends without a final frame.
+ * Without it, that is the list's last message.
+ */
+ responseMessageId?: string;
+};
+
+/** The message list as an event handler reads and writes it. */
+export type ChatEventContext = {
+ readonly messages: TMessage[];
+ write(messages: TMessage[]): void;
+};
+
+export type ChatFinishResult = {
+ /**
+ * The turn's response: the one its final frame named, else the prepared `responseMessageId`,
+ * else the tail of the list. `undefined` when a named response is no longer in the list.
+ */
+ message: TMessage | undefined;
+ messages: TMessage[];
+ isAbort: boolean;
+ isDisconnect: boolean;
+ isError: boolean;
+ /**
+ * The server finished the turn without this chat receiving its final frame, so `messages` are
+ * what the chat held; the host reloads the conversation to reconcile them.
+ */
+ isSettled: boolean;
+ /**
+ * No frame named this turn's response, as when a generation replacing it ended before its sync
+ * frame. `message` is then `undefined` rather than a guess at the list's tail.
+ */
+ isResponseUnknown: boolean;
+ /**
+ * The conversation the turn ran under: the one the server settled it under, else the one its
+ * frames or payload named, so a turn under `new` reports the conversation the server created.
+ */
+ conversationId?: string;
+ /** Why a settled turn's final frame carried no messages. */
+ reconcileReason?: ChatFinalFrame['reconcileReason'];
+};
+
+export type ChatInit = {
+ /** The conversation id, or `new` before the server created one. */
+ id: string;
+ state: ChatState;
+ transport: ChatTransport;
+ /**
+ * Builds the request body, and the optimistic messages, for a turn. AI SDK:
+ * `prepareSendMessagesRequest`.
+ */
+ prepareRequest(request: ChatPrepareRequest): ChatPreparedRequest | Promise<ChatPreparedRequest>;
+ /**
+ * Folds one stream event into the message list. Without it, a `final` event's request and
+ * response messages are merged into the list by id and every other event only moves the status.
+ */
+ applyEvent?(event: ChatEvent, context: ChatEventContext): void;
+ onError?(error: Error): void;
+ onFinish?(result: ChatFinishResult): void;
+ /** Every event, after it was applied: the side channels (usage, title, attachments, steering). */
+ onEvent?(event: ChatEvent): void;
+};
+
+/** A turn whose messages the chat could not take from the stream, which the host reloads. */
+type Settled = {
+ conversationId?: string;
+ reason?: ChatFinalFrame['reconcileReason'];
+};
+
+type Outcome = {
+ settled?: Settled;
+ /** The conversation the turn's frames named, reported when a settled outcome names none. */
+ conversationId?: string;
+} & (
+ | { kind: 'complete' }
+ | { kind: 'abort' }
+ | { kind: 'error'; error: Error; disconnect: boolean }
+);
+
+/** The response a turn produces, as the frames of its generation name it. */
+type Generation = {
+ /** The conversation the payload, the server's answer or a frame named. */
+ conversationId?: string;
+ /** The response this generation produces: the placeholder, then the one a frame names. */
+ responseId?: string;
+ /**
+ * The generation is not the one this chat prepared (a replacement) and no frame has named its
+ * response yet.
+ */
+ responseUnknown?: boolean;
+ /** A turn that ended before its preparation installed a response, so it has none to report. */
+ responseless?: boolean;
+};
+
+/**
+ * What one optimistic write did, by message id, so undoing it touches only those rows. A row the
+ * state holds that the write did not add, replace or remove, such as one a store subscriber
+ * appended while the write was committed, is never part of it.
+ */
+type OptimisticWrite = {
+ /** The list before the write, which places a removed row back among its neighbours. */
+ before: TMessage[];
+ /** Each row the write added or replaced, as the state committed it. */
+ committed: Map<string, TMessage>;
+ /** Each row the write replaced, as it was before. */
+ replaced: Map<string, TMessage>;
+ /** The ids of the rows the write removed. */
+ removed: Set<string>;
+};
+
+type ActiveRequest = {
+ controller: AbortController;
+ /** The optimistic write, undone when the server never got the turn or ran another in its place. */
+ optimistic?: OptimisticWrite;
+ stream?: ChatStream;
+ /** The generation this request runs or attaches to. */
+ generation: Generation;
+ /** Set while attaching to a generation the server answered the start with, before it attaches. */
+ attaching?: boolean;
+ /**
+ * The stream came from attaching to a running generation, whose HTTP failure ends only the
+ * attachment unless the server says the generation is gone.
+ */
+ attached?: boolean;
+ /** Set once the start was handed to the transport, so the server may be running it. */
+ sent?: boolean;
+ /** Resolves once the request settled and released the chat. */
+ finished: Promise<void>;
+ finish: () => void;
+};
+
+const createActiveRequest = (generation: Generation = {}): ActiveRequest => {
+ let finish = () => {};
+ const finished = new Promise<void>((resolve) => {
+ finish = resolve;
+ });
+ return { controller: new AbortController(), generation, finished, finish };
+};
+
+/** Settles with `value`, or rejects once `signal` aborts, so a disposal never waits on it. */
+const untilAborted = <T>(value: T | Promise<T>, signal: AbortSignal): Promise<T> =>
+ new Promise<T>((resolve, reject) => {
+ const onAbort = () => reject(signal.reason);
+ if (signal.aborted) {
+ Promise.resolve(value).catch(() => undefined);
+ onAbort();
+ return;
+ }
+ signal.addEventListener('abort', onAbort, { once: true });
+ Promise.resolve(value)
+ .then(resolve, reject)
+ .finally(() => signal.removeEventListener('abort', onAbort));
+ });
+
+/** Races opening a stream with cancellation, and hands `onLate` what opens after it. */
+const openUntilAborted = <T>(
+ opening: Promise<T>,
+ signal: AbortSignal,
+ onLate: (value: T) => void,
+): Promise<T> => {
+ void opening.then(
+ (value) => {
+ if (signal.aborted) {
+ onLate(value);
+ }
+ },
+ () => undefined,
+ );
+ return untilAborted(opening, signal);
+};
+
+/**
+ * Reports a host listener's failure the way the DOM reports a throwing event listener, so one
+ * faulty subscriber neither hides a change from the others nor interrupts the chat's transition.
+ */
+const reportListenerError = (error: unknown): void => {
+ const { reportError } = globalThis as { reportError?: (error: unknown) => void };
+ if (reportError) {
+ reportError(error);
+ return;
+ }
+ console.error(error);
+};
+
+const streamingEvents = new Set<ChatEvent['type']>(['created', 'sync', 'step', 'content', 'text']);
+
+const getErrorMessage = (data: ChatErrorData | null | undefined): string | undefined => {
+ if (typeof data === 'string') {
+ return data || undefined;
+ }
+ if (!data) {
+ return undefined;
+ }
+ const nested = typeof data.error === 'object' ? data.error?.message : undefined;
+ const reason = typeof data.error === 'string' ? data.error : undefined;
+ return data.message || data.text || reason || nested;
+};
+
+/**
+ * An error frame the attachment's HTTP response raised rather than the generation: the transport
+ * sets `status` only then, and a 404 or 410 says the generation itself is gone.
+ */
+const isAttachFailure = (event: Extract<ChatEvent, { type: 'error' }>): boolean =>
+ event.status != null && event.status !== 404 && event.status !== 410;
+
+/** The server answered that no such generation runs, rather than failing to answer. */
+const isGone = (error: ChatTransportError): boolean =>
+ error.code === 'NO_ACTIVE_RUN' ||
+ error.response?.status === 404 ||
+ error.response?.status === 410;
+
+/**
+ * A frame the chat failed to apply. A terminal frame still ended the generation, which the host
+ * reloads from what the server kept; any other leaves it running.
+ */
+const getApplyFailure = (event: ChatEvent, failure: unknown, attached: boolean): Outcome => {
+ const error = failure instanceof Error ? failure : new Error(String(failure));
+ const terminal =
+ event.type === 'final' ||
+ event.type === 'abort' ||
+ (event.type === 'error' && event.status !== 0 && !(attached && isAttachFailure(event)));
+ return terminal
+ ? { kind: 'error', error, disconnect: false, settled: {} }
+ : { kind: 'error', error, disconnect: true };
+};
+
+/** The ids a frame names for the conversation and the response its generation produces. */
+type FrameIdentity = { conversationId?: string | null; responseMessageId?: string };
+
+/**
+ * Reads the identity every frame type carries. A `created` frame's message is the user's, so it
+ * names only the conversation; an attachment may belong to an uploaded file, so the same holds.
+ */
+const getFrameIdentity = (event: ChatEvent): FrameIdentity => {
+ switch (event.type) {
+ case 'created':
+ return { conversationId: event.data.message?.conversationId };
+ case 'sync': {
+ const { resumeState } = event.data;
+ return {
+ conversationId: event.data.conversationId ?? resumeState?.conversationId,
+ responseMessageId: event.data.responseMessage?.messageId ?? resumeState?.responseMessageId,
+ };
+ }
+ case 'final':
+ return {
+ conversationId:
+ event.data.conversation?.conversationId ?? event.data.responseMessage?.conversationId,
+ responseMessageId: event.data.responseMessage?.messageId ?? undefined,
+ };
+ case 'pending_action':
+ case 'steer_applied':
+ case 'activity_label':
+ case 'reasoning_label':
+ return {
+ conversationId: event.data.conversationId,
+ responseMessageId: event.data.responseMessageId,
+ };
+ case 'steer_updated':
+ return { conversationId: event.data.conversationId };
+ case 'title':
+ return { conversationId: event.data.data?.conversationId };
+ case 'content':
+ return { conversationId: event.data.conversationId, responseMessageId: event.data.messageId };
+ case 'text':
+ return { responseMessageId: event.data.messageId };
+ case 'attachment':
+ return {
+ conversationId: 'conversationId' in event.data ? event.data.conversationId : undefined,
+ };
+ default:
+ return {};
+ }
+};
+
+const getFinalOutcome = (data: ChatFinalFrame): Outcome => {
+ /** A `reconcile` frame committed its terminal status without the messages, so it is settled. */
+ const settled: Settled | undefined = data.reconcile
+ ? {
+ conversationId: data.conversation?.conversationId ?? undefined,
+ reason: data.reconcileReason,
+ }
+ : undefined;
+ if (data.terminalStatus === 'error' || data.error) {
+ const message = data.error?.message ?? 'The response failed';
+ return { kind: 'error', error: new Error(message), disconnect: false, settled };
+ }
+ if (data.terminalStatus === 'aborted' || data.aborted) {
+ return { kind: 'abort', settled };
+ }
+ return { kind: 'complete', settled };
+};
+
+/**
+ * Whether a frame's message carries every field a stored row needs. A partial one only updates the
+ * row it names; appended as it is, it would stand in the list as a message missing required fields.
+ */
+const isCompleteMessage = (message: Partial<TMessage>): message is TMessage =>
+ typeof message.messageId === 'string' &&
+ typeof message.text === 'string' &&
+ typeof message.isCreatedByUser === 'boolean' &&
+ message.conversationId !== undefined &&
+ message.parentMessageId !== undefined;
+
+/** Merges `finals` onto `messages` by id, appending the complete ones the list does not hold. */
+const mergeById = (
+ messages: TMessage[],
+ finals: Array<Partial<TMessage> | null | undefined>,
+): TMessage[] | undefined => {
+ const known = finals.filter(
+ (message): message is Partial<TMessage> & { messageId: string } => !!message?.messageId,
+ );
+ if (known.length === 0) {
+ return undefined;
+ }
+ const next = [...messages];
+ for (const final of known) {
+ const index = next.findIndex((message) => message.messageId === final.messageId);
+ if (index === -1 && isCompleteMessage(final)) {
+ next.push(final);
+ } else if (index !== -1) {
+ next[index] = { ...next[index], ...final };
+ }
+ }
+ return next;
+};
+
+const mergeFinal = (data: ChatFinalFrame, context: ChatEventContext) => {
+ const next = mergeById(context.messages, [data.requestMessage, data.responseMessage]);
+ if (next) {
+ context.write(next);
+ }
+};
+
+const defaultApplyEvent = (event: ChatEvent, context: ChatEventContext) => {
+ if (event.type === 'final') {
+ mergeFinal(event.data, context);
+ }
+};
+
+/**
+ * Where a removed row goes back: after the nearest earlier row of `before` still listed, else
+ * before the nearest later one, so rows written around it since then keep their place.
+ */
+const getRestoreIndex = (before: TMessage[], index: number, next: TMessage[]): number => {
+ const positionOf = (message: TMessage) =>
+ next.findIndex((listed) => listed.messageId === message.messageId);
+ for (let i = index - 1; i >= 0; i--) {
+ const position = positionOf(before[i]);
+ if (position !== -1) {
+ return position + 1;
+ }
+ }
+ for (let i = index + 1; i < before.length; i++) {
+ const position = positionOf(before[i]);
+ if (position !== -1) {
+ return position;
+ }
+ }
+ return next.length;
+};
+
+/** Structural equality over JSON-shaped values, which tells a store's copy of a row from an edit. */
+const isSameValue = (a: unknown, b: unknown): boolean => {
+ if (a === b) {
+ return true;
+ }
+ if (typeof a !== 'object' || typeof b !== 'object' || a == null || b == null) {
+ return false;
+ }
+ if (Array.isArray(a) || Array.isArray(b)) {
+ return (
+ Array.isArray(a) &&
+ Array.isArray(b) &&
+ a.length === b.length &&
+ a.every((item, index) => isSameValue(item, b[index]))
+ );
+ }
+ const left = a as Record<string, unknown>;
+ const right = b as Record<string, unknown>;
+ const keys = Object.keys(left);
+ return (
+ keys.length === Object.keys(right).length &&
+ keys.every(
+ (key) =>
+ Object.prototype.hasOwnProperty.call(right, key) && isSameValue(left[key], right[key]),
+ )
+ );
+};
+
+/**
+ * Records the rows `written` changed against `before`, each added or replaced row under the
+ * identity the state `committed` it with, since a store may copy a row on the way in. A row a
+ * store subscriber edited inside the write is that subscriber's, and the record leaves it out.
+ */
+const recordOptimisticWrite = (
+ before: TMessage[],
+ written: TMessage[],
+ committed: TMessage[],
+): OptimisticWrite => {
+ const prior = new Map(before.map((message) => [message.messageId, message]));
+ const held = new Map(committed.map((message) => [message.messageId, message]));
+ const record: OptimisticWrite = {
+ before,
+ committed: new Map(),
+ replaced: new Map(),
+ removed: new Set(prior.keys()),
+ };
+ for (const message of written) {
+ const was = prior.get(message.messageId);
+ record.removed.delete(message.messageId);
+ const row = held.get(message.messageId);
+ if (was === message || row == null || !isSameValue(row, message)) {
+ continue;
+ }
+ record.committed.set(message.messageId, row);
+ if (was != null) {
+ record.replaced.set(message.messageId, was);
+ }
+ }
+ return record;
+};
+
+/**
+ * Undoes an optimistic write on the list as it stands now: rows it added go, rows it replaced or
+ * removed come back as they were, and any other row, or one of its rows someone else wrote since,
+ * stays as it is.
+ */
+const undoOptimisticWrite = (record: OptimisticWrite, current: TMessage[]): TMessage[] => {
+ const next: TMessage[] = [];
+ const kept = new Set<string>();
+ for (const message of current) {
+ if (record.committed.get(message.messageId) !== message) {
+ next.push(message);
+ kept.add(message.messageId);
+ continue;
+ }
+ const was = record.replaced.get(message.messageId);
+ if (was != null) {
+ next.push(was);
+ kept.add(message.messageId);
+ }
+ }
+ record.before.forEach((message, index) => {
+ if (record.removed.has(message.messageId) && !kept.has(message.messageId)) {
+ next.splice(getRestoreIndex(record.before, index, next), 0, message);
+ }
+ });
+ return next;
+};
+
+/**
+ * Closes the stream of a start that answered after the chat let it go. Only a disposal lets a
+ * start go, and a disposed chat leaves the server's generation running, as it does an attached one.
+ */
+const closeLateStart = (sent: ChatSendResult): void => {
+ if (sent.status === 'stream') {
+ sent.stream.close();
+ }
+};
+
+/**
+ * One conversation's turn lifecycle, framework free. AI SDK: `AbstractChat`.
+ *
+ * A turn is `submitted` while the request is in flight, `streaming` from its first content event,
+ * then `ready` on completion or abort and `error` on failure, until `clearError`. Every write to
+ * the message list (stream events and the optimistic turn) runs through one
+ * {@link SerialJobExecutor}, so they apply in the order they were made.
+ */
+export class Chat {
+ readonly id: string;
+ private readonly init: ChatInit;
+ private readonly executor = new SerialJobExecutor();
+ private readonly listeners = new Set<() => void>();
+ private releaseState?: () => void;
+ private active?: ActiveRequest;
+ private disposed = false;
+ /** Set while the chat writes the state, whose own report of that write the chat announces itself. */
+ private writing = false;
+ /**
+ * The conversation this chat runs under: its id, until the server names the one it created for
+ * a chat started under `new`. Every later turn addresses that one.
+ */
+ private conversationId: string;
+
+ constructor(init: ChatInit) {
+ this.id = init.id;
+ this.conversationId = init.id;
+ this.init = init;
+ }
+
+ get status(): ChatStatus {
+ return this.init.state.status;
+ }
+
+ get error(): Error | undefined {
+ return this.init.state.error;
+ }
+
+ get connection(): ChatConnection {
+ return this.init.state.connection;
+ }
+
+ get messages(): TMessage[] {
+ return this.init.state.messages;
+ }
+
+ /** Submits a user turn. Rejects while another turn of this chat is in flight. */
+ sendMessage(input: SendMessageInput, options?: ChatRequestOptions): Promise<void> {
+ return this.request({ trigger: 'submit-message', input }, options);
+ }
+
+ /** Regenerates the response to `messageId`, or the latest response. */
+ regenerate(options: { messageId?: string } & ChatRequestOptions = {}): Promise<void> {
+ const { messageId, ...requestOptions } = options;
+ return this.request({ trigger: 'regenerate-message', messageId }, requestOptions);
+ }
+
+ /** Continues the response `messageId` where it stopped. */
+ continue(options: { messageId: string } & ChatRequestOptions): Promise<void> {
+ const { messageId, ...requestOptions } = options;
+ return this.request({ trigger: 'continue-message', messageId }, requestOptions);
+ }
+
+ /** Returns a failed chat to `ready`. AI SDK: `clearError`. */
+ clearError(): void {
+ if (this.status !== 'error') {
+ return;
+ }
+ this.init.state.error = undefined;
+ this.setStatus('ready');
+ }
+
+ /** Notifies `listener` after every change to the messages, status or error. */
+ subscribe(listener: () => void): () => void {
+ this.listeners.add(listener);
+ if (!this.releaseState && this.init.state.subscribe) {
+ this.releaseState = this.init.state.subscribe(() => {
+ if (!this.writing) {
+ this.notify();
+ }
+ });
+ }
+ return () => {
+ this.listeners.delete(listener);
+ if (this.listeners.size === 0) {
+ this.releaseState?.();
+ this.releaseState = undefined;
+ }
+ };
+ }
+
+ /** Drops the local stream and every listener; the server's generation keeps running. */
+ dispose(): void {
+ if (this.disposed) {
+ return;
+ }
+ this.disposed = true;
+ const active = this.active;
+ this.active = undefined;
+ active?.controller.abort();
+ active?.stream?.close();
+ const { state } = this.init;
+ const unsent = active != null && !active.sent;
+ const rollback = unsent ? active.optimistic : undefined;
+ const status = unsent && state.status === 'submitted' ? 'ready' : state.status;
+ if (rollback || status !== state.status || state.connection !== 'idle') {
+ /**
+ * A turn that never reached the server ends here, its optimistic rows with it; a sent one
+ * keeps its status and rows, since the server keeps running it. Either way the chat holds
+ * no stream.
+ */
+ if (active && rollback) {
+ active.optimistic = undefined;
+ state.replaceMessages(undoOptimisticWrite(rollback, state.messages));
+ }
+ state.status = status;
+ state.connection = 'idle';
+ this.notify();
+ }
+ /** Whoever waits on the request, such as its send, is released with the chat. */
+ active?.finish();
+ this.listeners.clear();
+ this.releaseState?.();
+ this.releaseState = undefined;
+ }
+
+ private async request(
+ request: Omit<ChatPrepareRequest, 'chatId' | 'messages'>,
+ options: ChatRequestOptions = {},
+ ): Promise<void> {
+ if (this.disposed) {
+ throw new Error('The chat was disposed');
+ }
+ if (this.active) {
+ throw new Error('A turn is already in flight for this chat');
+ }
+ const active = createActiveRequest({ responseless: true });
+ this.active = active;
+ try {
+ this.init.state.error = undefined;
+ this.setStatus('submitted');
+ if (active.controller.signal.aborted) {
+ throw active.controller.signal.reason;
+ }
+ const prepared = await untilAborted(
+ this.init.prepareRequest({
+ ...request,
+ chatId: this.conversationId,
+ messages: this.messages,
+ }),
+ active.controller.signal,
+ );
+ active.generation.responseId = prepared.responseMessageId;
+ if (prepared.messages) {
+ const optimistic = prepared.messages;
+ await this.executor.run(() => {
+ if (!active.controller.signal.aborted) {
+ const before = this.messages;
+ const written = typeof optimistic === 'function' ? optimistic(before) : optimistic;
+ /** The rows as the state committed them, read before a chat subscriber writes more. */
+ this.write(written, (committed) => {
+ active.optimistic = recordOptimisticWrite(before, written, committed);
+ });
+ if (this.disposed && active.optimistic) {
+ /** A store subscriber disposed the chat inside the write, before it had a record. */
+ this.init.state.replaceMessages(
+ undoOptimisticWrite(active.optimistic, this.messages),
+ );
+ active.optimistic = undefined;
+ }
+ }
+ });
+ }
+ if (active.controller.signal.aborted) {
+ throw active.controller.signal.reason;
+ }
+ /** Only now is the turn's response in the list, so a failure before this reports none. */
+ active.generation.responseless = undefined;
+ active.generation.conversationId = prepared.payload.conversationId ?? undefined;
+ const { signal } = active.controller;
+ active.sent = true;
+ const result = await openUntilAborted(
+ this.init.transport.sendMessages({
+ ...options,
+ trigger: request.trigger,
+ chatId: this.conversationId,
+ messageId: request.messageId,
+ payload: prepared.payload,
+ abortSignal: signal,
+ }),
+ signal,
+ closeLateStart,
+ );
+ if (signal.aborted) {
+ closeLateStart(result);
+ throw signal.reason;
+ }
+ if (result.status === 'replaced' && result.generationProtocolVersion !== 2) {
+ /** Only protocol v2 vouches for a replacement; an older server reported a conflict. */
+ throw new Error('Another generation is running in this conversation');
+ }
+ if (result.status === 'settled' && result.generationProtocolVersion !== 2) {
+ /** Only protocol v2 vouches for a settled start, which completes the turn without a stream. */
+ throw new Error('The server answered with a settled start it did not negotiate');
+ }
+ if (result.status !== 'stream') {
+ this.learnConversation(result.conversationId);
+ }
+ let stream: ChatStream | null = null;
+ if (result.status === 'stream') {
+ stream = result.stream;
+ /** A start the server attached to a generation already running is an attachment too. */
+ active.attached = result.stream.resumed === true;
+ } else if (result.status === 'replaced') {
+ const rollback = active.optimistic;
+ if (rollback) {
+ /** The server ran a newer generation instead, so this turn's optimistic write is undone. */
+ await this.executor.run(() => {
+ this.write(undoOptimisticWrite(rollback, this.messages));
+ });
+ }
+ /** The replacement's response is not this turn's placeholder; its first frame names it. */
+ active.generation.responseId = undefined;
+ active.generation.responseUnknown = true;
+ active.generation.conversationId = result.conversationId;
+ active.attaching = true;
+ active.attached = true;
+ if (signal.aborted) {
+ throw signal.reason;
+ }
+ this.setConnection('reconnecting');
+ if (signal.aborted) {
+ throw signal.reason;
+ }
+ stream = await openUntilAborted(
+ this.init.transport.reconnectToStream({
+ ...options,
+ chatId: result.conversationId,
+ streamId: result.streamId,
+ generationCreatedAt: result.generationCreatedAt,
+ resume: true,
+ abortSignal: signal,
+ }),
+ signal,
+ (opened) => opened?.close(),
+ );
+ }
+ if (!stream) {
+ /** The server finished the turn, or the generation replacing it, before it attached. */
+ const conversationId = result.status === 'stream' ? undefined : result.conversationId;
+ this.settle({ kind: 'complete', settled: { conversationId } }, active);
+ return;
+ }
+ await this.consume(stream, active);
+ } catch (error) {
+ this.settle(this.toOutcome(error, active), active);
+ } finally {
+ this.release(active);
+ }
+ }
+
+ private async consume(stream: ChatStream, active: ActiveRequest): Promise<void> {
+ const { signal } = active.controller;
+ if (signal.aborted) {
+ /** The request was cancelled while the stream was still being opened. */
+ stream.close();
+ this.settle({ kind: 'abort' }, active);
+ return;
+ }
+ active.stream = stream;
+ active.attaching = undefined;
+ let outcome: Outcome | undefined;
+ /** The event this chat is applying, whose failure is the chat's, not the generation's. */
+ let handling: ChatEvent | undefined;
+ try {
+ this.setConnection('live');
+ for await (const event of stream) {
+ if (signal.aborted) {
+ break;
+ }
+ await this.executor.run(() => {
+ if (!signal.aborted) {
+ handling = event;
+ this.adopt(active.generation, getFrameIdentity(event));
+ outcome = this.apply(event, active);
+ handling = undefined;
+ }
+ });
+ if (outcome) {
+ break;
+ }
+ }
+ } catch (error) {
+ if (signal.aborted) {
+ outcome = this.toOutcome(error, active);
+ } else if (handling) {
+ outcome = getApplyFailure(handling, error, active.attached === true);
+ } else {
+ /** The stream failed before a terminal frame, as one that ends early does: a disconnect. */
+ const failure = error instanceof Error ? error : new Error(String(error));
+ outcome = { kind: 'error', error: failure, disconnect: true };
+ }
+ } finally {
+ stream.close();
+ }
+ if (!outcome && signal.aborted) {
+ outcome = { kind: 'abort' };
+ }
+ this.settle(
+ outcome ?? {
+ kind: 'error',
+ error: new Error('The stream ended before the response finished'),
+ disconnect: true,
+ },
+ active,
+ );
+ }
+
+ /** Applies one event; returns the turn's outcome when the event ends it. */
+ private apply(event: ChatEvent, active: ActiveRequest): Outcome | undefined {
+ const applyEvent = this.init.applyEvent ?? defaultApplyEvent;
+ applyEvent(event, {
+ messages: this.messages,
+ write: (messages) => this.write(messages),
+ });
+ /** A listener told of that write may have disposed the chat, which then owns no more state. */
+ if (this.disposed) {
+ return undefined;
+ }
+ let outcome: Outcome | undefined;
+ if (streamingEvents.has(event.type) || event.type === 'pending_action') {
+ /** A live pause is the generation running, however its first frame arrives. */
+ if (this.markStreaming()) {
+ this.notify();
+ }
+ } else if (event.type === 'final') {
+ outcome = getFinalOutcome(event.data);
+ } else if (event.type === 'abort') {
+ outcome = { kind: 'abort' };
+ } else if (event.type === 'error') {
+ const disconnect = event.status === 0 || (active.attached === true && isAttachFailure(event));
+ const message = getErrorMessage(event.data) ?? 'The response failed';
+ outcome = { kind: 'error', error: new Error(message), disconnect };
+ }
+ this.callHost(this.init.onEvent, event);
+ return outcome;
+ }
+
+ /**
+ * Moves to `streaming` on a frame of a live generation, which also ends whatever failure a
+ * dropped connection reported. Returns whether anything changed.
+ */
+ private markStreaming(): boolean {
+ const { state } = this.init;
+ if (state.status === 'streaming' && state.error === undefined) {
+ return false;
+ }
+ state.error = undefined;
+ state.status = 'streaming';
+ return true;
+ }
+
+ private toOutcome(error: unknown, active: ActiveRequest): Outcome {
+ if (active.controller.signal.aborted) {
+ return { kind: 'abort' };
+ }
+ const failure = error instanceof Error ? error : new Error(String(error));
+ /**
+ * A request that never got a response may still be running on the server, and so may a
+ * generation a failed attachment never reached, unless the server answered that it is gone.
+ */
+ const disconnect =
+ isRetryableNetworkError(failure) ||
+ (active.attaching === true && !isGone(failure as ChatTransportError));
+ return { kind: 'error', error: failure, disconnect };
+ }
+
+ private settle(outcome: Outcome, active: ActiveRequest): void {
+ if (this.active !== active) {
+ return;
+ }
+ /**
+ * Released and published as one change before anyone is told, so a subscriber or callback
+ * that starts the next turn is not overwritten by this one's terminal state.
+ */
+ this.active = undefined;
+ const { responseId, responseUnknown, responseless, conversationId } = active.generation;
+ const named = conversationId !== Constants.NEW_CONVO ? conversationId : undefined;
+ this.publish({ ...outcome, conversationId: named }, responseId, responseUnknown, responseless);
+ }
+
+ /**
+ * Adopts the ids a frame names: the conversation for the generation and, for a chat started
+ * under `new`, for the chat; and the response the generation produces, which makes it known.
+ */
+ private adopt(generation: Generation, identity: FrameIdentity): void {
+ const { conversationId, responseMessageId } = identity;
+ if (conversationId && conversationId !== Constants.NEW_CONVO) {
+ generation.conversationId = conversationId;
+ this.learnConversation(conversationId);
+ }
+ if (responseMessageId) {
+ generation.responseId = responseMessageId;
+ generation.responseUnknown = false;
+ }
+ }
+
+ /** Adopts the conversation the server created for a chat started under `new`; a real id stays. */
+ private learnConversation(conversationId: string | null | undefined): void {
+ if (
+ conversationId &&
+ conversationId !== Constants.NEW_CONVO &&
+ this.conversationId === Constants.NEW_CONVO
+ ) {
+ this.conversationId = conversationId;
+ }
+ }
+
+ /** Publishes a turn's terminal state, then tells the host how it ended. */
+ private publish(
+ outcome: Outcome,
+ responseId: string | undefined,
+ responseUnknown = false,
+ responseless = false,
+ ): void {
+ const { state } = this.init;
+ const status: ChatStatus = outcome.kind === 'error' ? 'error' : 'ready';
+ if (state.connection !== 'idle') {
+ state.connection = 'idle';
+ }
+ if (state.status !== status) {
+ state.status = status;
+ }
+ state.error = outcome.kind === 'error' ? outcome.error : undefined;
+ const messages = this.messages;
+ let message: TMessage | undefined;
+ if (responseId !== undefined) {
+ message = messages.find((stored) => stored.messageId === responseId);
+ } else if (!responseUnknown && !responseless) {
+ message = messages[messages.length - 1];
+ }
+ const { settled } = outcome;
+ /** A settled outcome names its conversation; otherwise the one the turn's frames named. */
+ const reportedConversation = settled?.conversationId ?? outcome.conversationId;
+ this.notify();
+ if (outcome.kind === 'error') {
+ this.callHost(this.init.onError, outcome.error);
+ }
+ this.callHost(this.init.onFinish, {
+ message,
+ messages,
+ isAbort: outcome.kind === 'abort',
+ isDisconnect: outcome.kind === 'error' && outcome.disconnect,
+ isError: outcome.kind === 'error',
+ isSettled: settled !== undefined,
+ isResponseUnknown: responseUnknown,
+ ...(reportedConversation !== undefined && { conversationId: reportedConversation }),
+ ...(settled?.reason !== undefined && { reconcileReason: settled.reason }),
+ });
+ }
+
+ private release(active: ActiveRequest): void {
+ if (this.active === active) {
+ this.active = undefined;
+ }
+ active.finish();
+ }
+
+ /**
+ * Every write below is the chat's own; a disposed chat owns none of the shared state. `commit`
+ * reads the list the state committed before anyone is told of it, including a store that
+ * reports its own writes.
+ */
+ private write(messages: TMessage[], commit?: (committed: TMessage[]) => void): void {
+ if (this.disposed) {
+ return;
+ }
+ this.writing = true;
+ try {
+ this.init.state.replaceMessages(messages);
+ } finally {
+ this.writing = false;
+ }
+ commit?.(this.messages);
+ this.notify();
+ }
+
+ private setStatus(status: ChatStatus): void {
+ if (this.disposed || this.init.state.status === status) {
+ return;
+ }
+ this.init.state.status = status;
+ this.notify();
+ }
+
+ private setConnection(connection: ChatConnection): void {
+ if (this.disposed || this.init.state.connection === connection) {
+ return;
+ }
+ this.init.state.connection = connection;
+ this.notify();
+ }
+
+ /** Runs a host callback; one that throws is reported and never interrupts the chat's transition. */
+ private callHost<T>(callback: ((value: T) => void) | undefined, value: T): void {
+ try {
+ callback?.(value);
+ } catch (error) {
+ reportListenerError(error);
+ }
+ }
+
+ private notify(): void {
+ for (const listener of this.listeners) {
+ try {
+ listener();
+ } catch (error) {
+ reportListenerError(error);
+ }
+ }
+ }
+}
diff --git a/packages/chat/src/core/executor.ts b/packages/chat/src/core/executor.ts
new file mode 100644
index 0000000000..51219bd069
--- /dev/null
+++ b/packages/chat/src/core/executor.ts
@@ -0,0 +1,46 @@
+type Job = () => Promise<void> | void;
+
+/**
+ * Runs jobs one at a time, in the order they were queued, so a stream frame, a tool update and a
+ * stop never interleave their reads and writes of the message list. AI SDK: `SerialJobExecutor`.
+ */
+export class SerialJobExecutor {
+ private queue: Array<() => Promise<void>> = [];
+ private running = false;
+
+ /** Queues `job`; resolves once it ran, or rejects with what it threw. Later jobs still run. */
+ run(job: Job): Promise<void> {
+ return new Promise<void>((resolve, reject) => {
+ this.queue.push(async () => {
+ try {
+ await job();
+ resolve();
+ } catch (error) {
+ reject(error);
+ }
+ });
+ void this.drain();
+ });
+ }
+
+ /** Whether no job is queued or running. */
+ get idle(): boolean {
+ return !this.running && this.queue.length === 0;
+ }
+
+ private async drain(): Promise<void> {
+ if (this.running) {
+ return;
+ }
+ this.running = true;
+ try {
+ let next = this.queue.shift();
+ while (next) {
+ await next();
+ next = this.queue.shift();
+ }
+ } finally {
+ this.running = false;
+ }
+ }
+}
diff --git a/packages/chat/src/core/index.ts b/packages/chat/src/core/index.ts
index 7695032540..01919b24ff 100644
--- a/packages/chat/src/core/index.ts
+++ b/packages/chat/src/core/index.ts
@@ -2,12 +2,14 @@ export * from './ask';
export * from './pending';
export * from './decision';
export * from './resumable';
+export * from './chat';
export * from './steps';
export * from './steer';
export * from './types';
export * from './files';
export * from './composer';
export * from './reducer';
+export * from './executor';
export * from './protocol';
export * from './transport';
export * from './usage';
diff --git a/packages/chat/src/core/types.ts b/packages/chat/src/core/types.ts
index 885f229f65..fd5805f894 100644
--- a/packages/chat/src/core/types.ts
+++ b/packages/chat/src/core/types.ts
@@ -268,13 +268,20 @@ export type ChatTransportError = Error & {
export interface ChatTransport {
/** Starts a turn, then attaches to its stream. Rejects with a {@link ChatTransportError}. */
sendMessages(request: ChatSendRequest): Promise<ChatSendResult>;
- /** Attaches to the running generation, or resolves `null` when nothing is running. */
+ /**
+ * Attaches to the running generation, or resolves `null` when nothing is running. A generation
+ * paused for a tool decision or an answer is still running.
+ */
reconnectToStream(request: ChatReconnectRequest): Promise<ChatStream | null>;
/**
* Stops the running generation; its stream then reports the abort, unless the result says it
* settled first. A request with `abortKey` resolves to a {@link ChatRunAbortResult}.
*/
abort(request: ChatAbortRequest): Promise<ChatAbortResult>;
+ /**
+ * Releases what the transport holds. Its creator calls it: one transport can serve several chats,
+ * so disposing a chat closes only that chat's stream.
+ */
close?(): void;
/** Reads the conversation's generation state; what a reload checks before reattaching. */
getStatus?(conversationId: string): Promise<ChatStreamStatus>;
diff --git a/packages/chat/src/react.ts b/packages/chat/src/react.ts
index 9b00210031..06419cdcc6 100644
--- a/packages/chat/src/react.ts
+++ b/packages/chat/src/react.ts
@@ -109,3 +109,5 @@ export {
shouldRollbackReveal,
buildRevealedMessage,
} from './react/steering/useQueuedTurnReveal';
+export { ReactQueryChatState } from './react/state';
+export type { ReactQueryChatStateOptions } from './react/state';
diff --git a/packages/chat/src/react/__tests__/state.spec.ts b/packages/chat/src/react/__tests__/state.spec.ts
new file mode 100644
index 0000000000..df7eabb1fc
--- /dev/null
+++ b/packages/chat/src/react/__tests__/state.spec.ts
@@ -0,0 +1,310 @@
+import { QueryKeys } from 'librechat-data-provider';
+import { QueryClient } from '@tanstack/react-query';
+import type { TMessage } from 'librechat-data-provider';
+import { ReactQueryChatState } from '../state';
+import { Chat } from '../../core';
+
+const message = (messageId: string): TMessage => ({
+ messageId,
+ conversationId: 'convo-1',
+ parentMessageId: null,
+ isCreatedByUser: true,
+ text: messageId,
+});
+
+const flushMicrotasks = () => Promise.resolve();
+
+describe('ReactQueryChatState', () => {
+ it('reads and writes the conversation entry of the message cache', () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ expect(state.messages).toEqual([]);
+ expect(state.version).toBe(0);
+
+ state.replaceMessages([message('a')]);
+
+ expect(queryClient.getQueryData([QueryKeys.messages, 'convo-1'])).toEqual([message('a')]);
+ expect(state.messages).toEqual([message('a')]);
+ expect(state.version).toBe(1);
+ });
+
+ it('reads back its writes and reports other writes under a host key hash', async () => {
+ const queryClient = new QueryClient({
+ defaultOptions: { queries: { queryKeyHashFn: (key) => `custom:${JSON.stringify(key)}` } },
+ });
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const listener = jest.fn();
+ const unsubscribe = state.subscribe(listener);
+
+ state.replaceMessages([message('a')]);
+ await flushMicrotasks();
+
+ expect(state.messages).toEqual([message('a')]);
+ expect(state.version).toBe(1);
+ expect(listener).not.toHaveBeenCalled();
+
+ queryClient.setQueryData([QueryKeys.messages, 'convo-1'], [message('b')]);
+ await flushMicrotasks();
+ expect(listener).toHaveBeenCalledTimes(1);
+ unsubscribe();
+ });
+
+ it('counts a write that keeps the same array', () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const list = [message('a')];
+ state.replaceMessages(list);
+
+ queryClient.setQueryData([QueryKeys.messages, 'convo-1'], list);
+
+ expect(state.messages).toBe(list);
+ expect(state.version).toBe(2);
+ });
+
+ it('reads and writes through the host when it supplies both', () => {
+ const queryClient = new QueryClient();
+ const stored = [message('a')];
+ const write = jest.fn();
+ const state = new ReactQueryChatState({
+ queryClient,
+ key: 'convo-1',
+ read: () => stored,
+ write,
+ });
+
+ state.replaceMessages([message('b')]);
+
+ expect(state.messages).toBe(stored);
+ expect(write).toHaveBeenCalledWith([message('b')]);
+ expect(queryClient.getQueryData([QueryKeys.messages, 'convo-1'])).toBeUndefined();
+ });
+
+ it('reports writes to a host store that bypass the cache', async () => {
+ const queryClient = new QueryClient();
+ let stored = [message('a')];
+ const hostListeners = new Set<() => void>();
+ const state = new ReactQueryChatState({
+ queryClient,
+ key: 'convo-1',
+ read: () => stored,
+ write: (next) => {
+ stored = next;
+ hostListeners.forEach((notify) => notify());
+ },
+ subscribe: (notify) => {
+ hostListeners.add(notify);
+ return () => hostListeners.delete(notify);
+ },
+ });
+ const listener = jest.fn();
+ const unsubscribe = state.subscribe(listener);
+
+ state.replaceMessages([message('b')]);
+ await flushMicrotasks();
+ expect(listener).not.toHaveBeenCalled();
+ expect(state.messages).toEqual([message('b')]);
+
+ stored = [message('c')];
+ hostListeners.forEach((notify) => notify());
+ await flushMicrotasks();
+ expect(listener).toHaveBeenCalledTimes(1);
+
+ unsubscribe();
+ expect(hostListeners.size).toBe(0);
+ });
+
+ it('reports a host store write made between reading the version and subscribing', async () => {
+ const queryClient = new QueryClient();
+ let stored = [message('a')];
+ const state = new ReactQueryChatState({
+ queryClient,
+ key: 'convo-1',
+ read: () => stored,
+ write: (next) => {
+ stored = next;
+ },
+ subscribe: () => () => undefined,
+ });
+ const rendered = state.version;
+ stored = [message('b')];
+ const listener = jest.fn();
+ const unsubscribe = state.subscribe(listener);
+ await flushMicrotasks();
+
+ expect(listener).toHaveBeenCalledTimes(1);
+ expect(state.version).not.toBe(rendered);
+ unsubscribe();
+ });
+
+ it('counts writes to a host store that bypass the cache, same array included', () => {
+ const queryClient = new QueryClient();
+ const stored = [message('a')];
+ const hostListeners = new Set<() => void>();
+ const state = new ReactQueryChatState({
+ queryClient,
+ key: 'convo-1',
+ read: () => stored,
+ write: () => undefined,
+ subscribe: (notify) => {
+ hostListeners.add(notify);
+ return () => hostListeners.delete(notify);
+ },
+ });
+ const unsubscribe = state.subscribe(() => undefined);
+ const versions = [state.version];
+
+ state.replaceMessages(stored);
+ versions.push(state.version);
+ stored[0] = { ...stored[0], text: 'changed in place' };
+ hostListeners.forEach((notify) => notify());
+ versions.push(state.version);
+
+ expect(new Set(versions).size).toBe(3);
+ unsubscribe();
+ });
+
+ it('keeps counting up when its cache entry is removed and created again', () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const key = [QueryKeys.messages, 'convo-1'];
+ queryClient.setQueryData(key, [message('a')]);
+ const before = state.version;
+
+ queryClient.removeQueries({ queryKey: key, exact: true });
+ queryClient.setQueryData(key, [message('b')]);
+
+ expect(state.version).toBeGreaterThan(before);
+ });
+
+ it('moves past the last version when an observer creates its entry with initial data', () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const key = [QueryKeys.messages, 'convo-1'];
+ const before = state.version;
+
+ queryClient.getQueryCache().build(queryClient, { queryKey: key, initialData: [message('a')] });
+
+ expect(state.messages).toEqual([message('a')]);
+ expect(state.version).toBeGreaterThan(before);
+ });
+
+ it('moves past the last version when its entry is reset and written again', () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const key = [QueryKeys.messages, 'convo-1'];
+ queryClient.setQueryData(key, [message('a')]);
+ const before = state.version;
+
+ queryClient.resetQueries({ queryKey: key, exact: true });
+ queryClient.setQueryData(key, [message('b')]);
+
+ expect(state.version).toBeGreaterThan(before);
+ });
+
+ it('moves past the last version when a reset keeps the same array and count', () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const key = [QueryKeys.messages, 'convo-1'];
+ const list = [message('a')];
+ queryClient.getQueryCache().build(queryClient, { queryKey: key, initialData: list });
+ const unsubscribe = state.subscribe(() => undefined);
+ queryClient.setQueryData(key, list);
+ const before = state.version;
+
+ queryClient.resetQueries({ queryKey: key, exact: true });
+ list[0] = { ...list[0], text: 'changed in place' };
+ queryClient.setQueryData(key, list);
+
+ expect(state.version).not.toBe(before);
+ unsubscribe();
+ });
+
+ it('reports a write that creates its cache entry once', async () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const listener = jest.fn();
+ state.subscribe(listener);
+
+ queryClient.setQueryData([QueryKeys.messages, 'convo-1'], [message('a')]);
+ await flushMicrotasks();
+
+ expect(listener).toHaveBeenCalledTimes(1);
+ });
+
+ it('reports writes to its own key only, after the writing call returns', async () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const listener = jest.fn();
+ const unsubscribe = state.subscribe(listener);
+
+ queryClient.setQueryData([QueryKeys.messages, 'convo-2'], [message('x')]);
+ queryClient.setQueryData([QueryKeys.messages, 'convo-1'], [message('a')]);
+ expect(listener).not.toHaveBeenCalled();
+
+ await flushMicrotasks();
+ expect(listener).toHaveBeenCalled();
+
+ listener.mockClear();
+ queryClient.setQueryData([QueryKeys.messages, 'convo-1'], [message('b')]);
+ unsubscribe();
+ await flushMicrotasks();
+ expect(listener).not.toHaveBeenCalled();
+ });
+
+ it('backs a chat, which hears writes other hooks make to the cache', async () => {
+ const queryClient = new QueryClient();
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const chat = new Chat({
+ id: 'convo-1',
+ state,
+ transport: {
+ sendMessages: jest.fn(),
+ reconnectToStream: jest.fn(),
+ abort: jest.fn(),
+ },
+ prepareRequest: jest.fn(),
+ });
+ const listener = jest.fn();
+ const unsubscribe = chat.subscribe(listener);
+
+ queryClient.setQueryData([QueryKeys.messages, 'convo-1'], [message('a')]);
+ await flushMicrotasks();
+
+ expect(listener).toHaveBeenCalled();
+ expect(chat.messages).toEqual([message('a')]);
+ expect(chat.status).toBe('ready');
+ unsubscribe();
+ });
+
+ it('restores a row the cache committed as a copy when a replaced start rolls back', async () => {
+ const queryClient = new QueryClient();
+ const response: TMessage = { ...message('response-1'), isCreatedByUser: false, text: 'old' };
+ queryClient.setQueryData([QueryKeys.messages, 'convo-1'], [message('a'), response]);
+ const state = new ReactQueryChatState({ queryClient, key: 'convo-1' });
+ const chat = new Chat({
+ id: 'convo-1',
+ state,
+ transport: {
+ sendMessages: jest.fn(async () => ({
+ status: 'replaced' as const,
+ generationProtocolVersion: 2 as const,
+ conversationId: 'convo-1',
+ streamId: 'stream-2',
+ generationCreatedAt: 200,
+ })),
+ reconnectToStream: jest.fn(async () => null),
+ abort: jest.fn(),
+ },
+ prepareRequest: jest.fn(() => ({
+ payload: {} as never,
+ responseMessageId: response.messageId,
+ messages: (list: TMessage[]) =>
+ list.map((row) => (row.messageId === response.messageId ? { ...row, text: '' } : row)),
+ })),
+ });
+
+ await chat.regenerate({ messageId: response.messageId });
+
+ expect(chat.messages).toEqual([message('a'), response]);
+ });
+});
diff --git a/packages/chat/src/react/state.ts b/packages/chat/src/react/state.ts
new file mode 100644
index 0000000000..1a91e3b5d0
--- /dev/null
+++ b/packages/chat/src/react/state.ts
@@ -0,0 +1,154 @@
+import { QueryKeys } from 'librechat-data-provider';
+import { hashQueryKey } from '@tanstack/react-query';
+import type { Query, QueryClient, QueryCacheNotifyEvent } from '@tanstack/react-query';
+import type { TMessage } from 'librechat-data-provider';
+import type { ChatState, ChatStatus, ChatConnection } from '../core';
+
+export type ReactQueryChatStateOptions = {
+ queryClient: QueryClient;
+ /** The conversation the messages are cached under, `[QueryKeys.messages, key]`. */
+ key: string;
+} & (
+ | { read?: undefined; write?: undefined; subscribe?: undefined }
+ | {
+ /** Reads the list instead of the key's cache entry, for a host that also reads other keys. */
+ read: () => TMessage[] | undefined;
+ /** Writes the list the same host store `read` reads, so a write is read back. */
+ write: (messages: TMessage[]) => void;
+ /**
+ * Reports writes to that store which do not reach the key's cache entry. Without it, only
+ * the key's cache entry is watched, which suits a host that writes the list there too.
+ */
+ subscribe?: (listener: () => void) => () => void;
+ }
+);
+
+/** Cache events that add, drop or rewrite a query's data. */
+const dataEvents = new Set<QueryCacheNotifyEvent['type']>(['added', 'removed', 'updated']);
+
+/**
+ * {@link ChatState} over the React Query message cache, so the chat and every reader of
+ * `[QueryKeys.messages, key]` share one list while those readers move onto the chat.
+ */
+export class ReactQueryChatState implements ChatState {
+ status: ChatStatus = 'ready';
+ error: Error | undefined = undefined;
+ connection: ChatConnection = 'idle';
+ readonly queryHash: string;
+ private readonly options: ReactQueryChatStateOptions;
+ /** Set while the chat's own write runs, whose change the chat already announced. */
+ private writing = false;
+ /** Writes to a host store that never reach the key's cache entry, which `version` also counts. */
+ private revision = 0;
+ /** The cache entry and the data `version` last counted, and what it handed out for them. */
+ private counted?: Query;
+ private countedData?: unknown;
+ private carried = 0;
+ private reported?: number;
+ /** Data events the cache reported for the key, and how many `version` last accounted for. */
+ private changes = 0;
+ private countedChanges = 0;
+ /** The host store's list when `version` was last read, which a later subscription checks. */
+ private countedRead?: TMessage[];
+
+ constructor(options: ReactQueryChatStateOptions) {
+ this.options = options;
+ const queryKey = [QueryKeys.messages, options.key];
+ /** The hash `setQueryData` files the key under, including a host's `queryKeyHashFn`. */
+ this.queryHash =
+ options.queryClient.defaultQueryOptions({ queryKey }).queryHash ?? hashQueryKey(queryKey);
+ }
+
+ get messages(): TMessage[] {
+ const { read, queryClient } = this.options;
+ if (read) {
+ return read() ?? [];
+ }
+ return (queryClient.getQueryCache().get(this.queryHash)?.state.data as TMessage[]) ?? [];
+ }
+
+ /**
+ * How many times the key's data was written. A stream frame can replace a message's content
+ * on the same array, so a snapshot keyed by this count still sees that write.
+ */
+ get version(): number {
+ const query = this.options.queryClient.getQueryCache().get(this.queryHash);
+ const data = query?.state.data;
+ let version = this.carried + (query?.state.dataUpdateCount ?? 0) + this.revision;
+ const changed =
+ query !== this.counted || data !== this.countedData || this.changes !== this.countedChanges;
+ if (changed && this.reported != null && version <= this.reported) {
+ /**
+ * The write count restarts when an entry is created with initial data, removed and created
+ * again, or reset, so any change the cache reported moves past the last version handed out.
+ */
+ this.carried += this.reported - version + 1;
+ version = this.reported + 1;
+ }
+ this.counted = query;
+ this.countedData = data;
+ this.countedRead = this.options.read?.();
+ this.countedChanges = this.changes;
+ this.reported = version;
+ return version;
+ }
+
+ replaceMessages(messages: TMessage[]): void {
+ const { write, queryClient, key } = this.options;
+ this.writing = true;
+ try {
+ if (write) {
+ write(messages);
+ this.revision += 1;
+ return;
+ }
+ queryClient.setQueryData<TMessage[]>([QueryKeys.messages, key], messages);
+ } finally {
+ this.writing = false;
+ }
+ }
+
+ /**
+ * Reports writes to the key's data made outside the chat; the chat announces its own. The cache
+ * notifies synchronously, including from inside another component's render when an observer
+ * mounts with initial data, so the listener runs on the next microtask instead.
+ */
+ subscribe(listener: () => void): () => void {
+ let active = true;
+ /** One write can raise several cache events, such as `added` then `updated`; one call covers them. */
+ let queued = false;
+ const deliver = () => {
+ if (this.writing || queued) {
+ return;
+ }
+ queued = true;
+ queueMicrotask(() => {
+ queued = false;
+ if (active) {
+ listener();
+ }
+ });
+ };
+ const unsubscribe = this.options.queryClient.getQueryCache().subscribe((event) => {
+ if (event.query.queryHash === this.queryHash && dataEvents.has(event.type)) {
+ this.changes += 1;
+ deliver();
+ }
+ });
+ const { read, subscribe } = this.options;
+ const unsubscribeHost = subscribe?.(() => {
+ this.revision += 1;
+ deliver();
+ });
+ if (unsubscribeHost && this.reported != null && read?.() !== this.countedRead) {
+ /** The host store changed after `version` was read and before this subscription heard it. */
+ this.revision += 1;
+ deliver();
+ }
+ return () => {
+ active = false;
+ unsubscribe();
+ unsubscribeHost?.();
+ };
+ }
+}