Skip to content

Build your personal knowledge base with Trilium Notes

fix: Enhance SSE heartbeat functionality in chat streaming. Added delays for chunk yielding and implemented heartbeat mechanism to prevent idle connection drops. Updated tests to verify heartbeat behavior during streaming.

ltomazetto03645b538479
diff --git a/apps/server/src/routes/api/llm_chat.spec.ts b/apps/server/src/routes/api/llm_chat.spec.ts
index 1f5669ed22..57660d352c 100644
--- a/apps/server/src/routes/api/llm_chat.spec.ts
+++ b/apps/server/src/routes/api/llm_chat.spec.ts
@@ -1,8 +1,10 @@
import type { Request, Response } from "express";
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const state = vi.hoisted(() => ({
chunks: [] as unknown[],
+ /** Milliseconds to wait before yielding chunk i. Missing entries wait 0. */
+ delaysMs: [] as number[],
/** The signal runChat was handed, so the disconnect wiring can be asserted. */
signal: undefined as AbortSignal | undefined
}));
@@ -12,11 +14,17 @@ const state = vi.hoisted(() => ({
vi.mock("@triliumnext/core/src/services/llm/chat.js", () => ({
async *runChat(_messages: unknown, _config: unknown, signal?: AbortSignal) {
state.signal = signal;
- for (const c of state.chunks) yield c;
+ for (const [index, chunk] of state.chunks.entries()) {
+ const wait = state.delaysMs[index] ?? 0;
+ if (wait > 0) {
+ await new Promise((resolve) => setTimeout(resolve, wait));
+ }
+ yield chunk;
+ }
}
}));
-import llmChatRoute from "./llm_chat.js";
+import llmChatRoute, { SSE_HEARTBEAT_FRAME, SSE_HEARTBEAT_MS } from "./llm_chat.js";
function fakeRes({ withFlush = false } = {}) {
const writes: string[] = [];
@@ -50,6 +58,7 @@ function req(body: unknown) { return { body } as unknown as Request; }
describe("streamChat", () => {
afterEach(() => {
state.chunks = [];
+ state.delaysMs = [];
state.signal = undefined;
});
@@ -95,4 +104,69 @@ describe("streamChat", () => {
r.listeners.close();
expect(signal?.aborted).toBe(true);
});
+
+ describe("SSE heartbeat", () => {
+ const chatReq = () => req({ messages: [{ role: "user", content: "hello" }] });
+
+ beforeEach(() => {
+ vi.useFakeTimers();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("writes a comment every 30s of silence, flushes it, and stops after the stream ends", async () => {
+ state.chunks = [{ type: "text", content: "Hi" }, { type: "done" }];
+ state.delaysMs = [SSE_HEARTBEAT_MS * 2 + 5_000];
+ const r = fakeRes({ withFlush: true });
+ const finished = llmChatRoute.streamChat(chatReq(), r.res);
+ await vi.advanceTimersByTimeAsync(0);
+
+ await vi.advanceTimersByTimeAsync(SSE_HEARTBEAT_MS);
+ expect(r.writes).toEqual([SSE_HEARTBEAT_FRAME]);
+ expect(r.flushes).toBe(1);
+
+ await vi.advanceTimersByTimeAsync(SSE_HEARTBEAT_MS);
+ expect(r.writes).toEqual([SSE_HEARTBEAT_FRAME, SSE_HEARTBEAT_FRAME]);
+
+ await vi.advanceTimersByTimeAsync(5_000);
+ await finished;
+
+ expect(r.writes).toEqual([
+ SSE_HEARTBEAT_FRAME,
+ SSE_HEARTBEAT_FRAME,
+ 'data: {"type":"text","content":"Hi"}\n\n',
+ 'data: {"type":"done"}\n\n'
+ ]);
+ expect(r.ended).toBe(true);
+
+ const writesWhenEnded = r.writes.length;
+ await vi.advanceTimersByTimeAsync(SSE_HEARTBEAT_MS * 2);
+ expect(r.writes.length).toBe(writesWhenEnded);
+ });
+
+ it("does not write a comment while chunks arrive more often than 30s", async () => {
+ state.chunks = [
+ { type: "text", content: "A" },
+ { type: "text", content: "B" },
+ { type: "done" }
+ ];
+ state.delaysMs = [10_000, 10_000, 10_000];
+ const r = fakeRes();
+ const finished = llmChatRoute.streamChat(chatReq(), r.res);
+ await vi.advanceTimersByTimeAsync(0);
+
+ await vi.advanceTimersByTimeAsync(10_000);
+ await vi.advanceTimersByTimeAsync(10_000);
+ await vi.advanceTimersByTimeAsync(10_000);
+ await finished;
+
+ expect(r.writes).toEqual([
+ 'data: {"type":"text","content":"A"}\n\n',
+ 'data: {"type":"text","content":"B"}\n\n',
+ 'data: {"type":"done"}\n\n'
+ ]);
+ });
+ });
});
diff --git a/apps/server/src/routes/api/llm_chat.ts b/apps/server/src/routes/api/llm_chat.ts
index 929c733b13..dc8d872bb8 100644
--- a/apps/server/src/routes/api/llm_chat.ts
+++ b/apps/server/src/routes/api/llm_chat.ts
@@ -7,6 +7,12 @@ interface ChatRequest {
config?: LlmProviderConfig;
}
+/** Silence after which `streamChat` writes an SSE comment so idle proxies keep the socket. */
+export const SSE_HEARTBEAT_MS = 30_000;
+
+/** SSE comment frame. EventSource and the client's `data:` parser ignore it. */
+export const SSE_HEARTBEAT_FRAME = ":\n\n";
+
/**
* SSE endpoint for streaming chat completions.
*
@@ -17,6 +23,9 @@ interface ChatRequest {
*
* On error:
* data: {"type":"error","error":"Error message"}
+ *
+ * nginx/ALB drop an idle response at 60s. After 30s without a chunk the handler
+ * writes an SSE comment (`:\n\n`) that the client ignores.
*/
async function streamChat(req: Request, res: Response) {
const { messages, config = {} } = req.body as ChatRequest;
@@ -44,23 +53,55 @@ async function streamChat(req: Request, res: Response) {
const abortController = new AbortController();
res.on("close", () => abortController.abort());
+ let stopped = false;
+ const writeFrame = (frame: string) => {
+ if (stopped) {
+ return;
+ }
+ res.write(frame);
+ if (typeof flushableRes.flush === "function") {
+ flushableRes.flush();
+ }
+ };
+
+ const heartbeat = startSseHeartbeat(() => writeFrame(SSE_HEARTBEAT_FRAME));
+
try {
// Imported here rather than at module scope so the chat pipeline and
// the provider SDKs land in lazy chunks (the same convention as
// getProviderModels in core's routes/api/llm.ts).
const { runChat } = await import("@triliumnext/core/src/services/llm/chat.js");
for await (const chunk of runChat(messages, config, abortController.signal)) {
- res.write(`data: ${JSON.stringify(chunk)}\n\n`);
- // Flush immediately to ensure real-time streaming
- if (typeof flushableRes.flush === "function") {
- flushableRes.flush();
- }
+ writeFrame(`data: ${JSON.stringify(chunk)}\n\n`);
+ heartbeat.reset();
}
} finally {
+ stopped = true;
+ heartbeat.stop();
res.end();
}
}
+function startSseHeartbeat(send: () => void) {
+ let timer: ReturnType<typeof setInterval> | undefined;
+ const arm = () => {
+ if (timer !== undefined) {
+ clearInterval(timer);
+ }
+ timer = setInterval(send, SSE_HEARTBEAT_MS);
+ };
+ arm();
+ return {
+ reset: arm,
+ stop() {
+ if (timer !== undefined) {
+ clearInterval(timer);
+ timer = undefined;
+ }
+ }
+ };
+}
+
export default {
streamChat
};