Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

perf(eve-channel): open session event stream alongside tail lookup and respect backpressure (#4547)

Alex Langenfeld13059ba5bf24
diff --git a/.changeset/session-stream-latency.md b/.changeset/session-stream-latency.md
new file mode 100644
index 000000000..ffa43a480
--- /dev/null
+++ b/.changeset/session-stream-latency.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Session stream requests now open the event stream while the tail index lookup is in flight, and the event stream reads its durable source only as fast as the client consumes it instead of buffering the whole history in memory.
diff --git a/packages/eve/src/eve-channel/request.stream.test.ts b/packages/eve/src/eve-channel/request.stream.test.ts
index 2326f5742..c7fd26653 100644
--- a/packages/eve/src/eve-channel/request.stream.test.ts
+++ b/packages/eve/src/eve-channel/request.stream.test.ts
@@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import type { Session } from "#channel/session.js";
import { createSessionStreamResponse } from "#eve-channel/request.js";
+import { parseNdjsonStream } from "#execution/ndjson-stream.js";
import {
EVE_STREAM_CONTROL_VERSION,
EVE_STREAM_CONTROL_VERSION_QUERY,
@@ -10,18 +11,37 @@ import {
const encoder = new TextEncoder();
-function stubSession(events: ReadableStream<unknown>): Session {
+function stubSession(events: ReadableStream<unknown>, tailIndex = -1): Session {
return {
id: "session_1",
async getEventStream() {
return events;
},
async getStreamTailIndex() {
- return -1;
+ return tailIndex;
},
} as Session;
}
+/** A durable byte source that never reaches EOF, like a parked run's stream. */
+function openByteSource(lines: number): {
+ readonly source: ReadableStream<Uint8Array>;
+ readonly cancelled: () => boolean;
+} {
+ let cancelled = false;
+ const source = new ReadableStream<Uint8Array>({
+ start(controller) {
+ for (let i = 0; i < lines; i += 1) {
+ controller.enqueue(encoder.encode(`${JSON.stringify({ sequence: i })}\n`));
+ }
+ },
+ cancel() {
+ cancelled = true;
+ },
+ });
+ return { cancelled: () => cancelled, source };
+}
+
function leasedRequest(): Request {
const url = new URL("https://eve.test/eve/v1/session/session_1/stream");
url.searchParams.set(EVE_STREAM_CONTROL_VERSION_QUERY, EVE_STREAM_CONTROL_VERSION);
@@ -123,4 +143,51 @@ describe("createSessionStreamResponse", () => {
await reader.cancel();
await pending;
});
+
+ it("cancels the durable source once a tail-indexed read reaches the tail", async () => {
+ const { cancelled, source } = openByteSource(3);
+ const url = "https://eve.test/eve/v1/session/session_1/stream?includeTailIndex=1";
+
+ const response = await createSessionStreamResponse(
+ new Request(url),
+ stubSession(
+ parseNdjsonStream(() => source),
+ 1,
+ ),
+ );
+
+ await expect(response.text()).resolves.toBe('\n{"sequence":0}\n{"sequence":1}\n');
+ await vi.waitFor(() => expect(cancelled()).toBe(true));
+ });
+
+ it("cancels the durable source when the client aborts", async () => {
+ const { cancelled, source } = openByteSource(1);
+ const abort = new AbortController();
+
+ const response = await createSessionStreamResponse(
+ new Request("https://eve.test/eve/v1/session/session_1/stream", { signal: abort.signal }),
+ stubSession(parseNdjsonStream(() => source)),
+ );
+ const reader = response.body!.getReader();
+ await reader.read();
+ await reader.read();
+ abort.abort();
+
+ await vi.waitFor(() => expect(cancelled()).toBe(true));
+ });
+
+ it("cancels the durable source when the response body is cancelled", async () => {
+ const { cancelled, source } = openByteSource(1);
+
+ const response = await createSessionStreamResponse(
+ new Request("https://eve.test/eve/v1/session/session_1/stream"),
+ stubSession(parseNdjsonStream(() => source)),
+ );
+ const reader = response.body!.getReader();
+ await reader.read();
+ await reader.read();
+ await reader.cancel();
+
+ await vi.waitFor(() => expect(cancelled()).toBe(true));
+ });
});
diff --git a/packages/eve/src/eve-channel/request.ts b/packages/eve/src/eve-channel/request.ts
index f17efe5ab..cded0cd65 100644
--- a/packages/eve/src/eve-channel/request.ts
+++ b/packages/eve/src/eve-channel/request.ts
@@ -300,11 +300,20 @@ export async function createSessionStreamResponse(
const includeTailIndex = parseIncludeTailIndex(request);
try {
- // The event stream opens its durable source lazily, so an unknown or
- // unreachable session would otherwise answer 200 and then fail mid-body.
- // Resolving the tail first surfaces that before any bytes are committed.
- const tailIndex = await session.getStreamTailIndex();
- const events = await session.getEventStream({ startIndex });
+ // An unknown or unreachable session would otherwise answer 200 and then
+ // fail mid-body, so the tail must resolve before any bytes are committed.
+ // The event stream opens alongside it to save a round trip.
+ const eventsPromise = session.getEventStream({ startIndex });
+ // Handled below; this keeps an early rejection from being reported as unhandled.
+ eventsPromise.catch(() => {});
+ let tailIndex: number;
+ try {
+ tailIndex = await session.getStreamTailIndex();
+ } catch (error) {
+ void eventsPromise.then((events) => events.cancel()).catch(() => {});
+ throw error;
+ }
+ const events = await eventsPromise;
const controlVersion =
new URL(request.url).searchParams.get(EVE_STREAM_CONTROL_VERSION_QUERY) ===
EVE_STREAM_CONTROL_VERSION
diff --git a/packages/eve/src/execution/ndjson-stream.test.ts b/packages/eve/src/execution/ndjson-stream.test.ts
index 15780fae0..341373866 100644
--- a/packages/eve/src/execution/ndjson-stream.test.ts
+++ b/packages/eve/src/execution/ndjson-stream.test.ts
@@ -109,6 +109,42 @@ describe("parseNdjsonStream", () => {
expect(cancelledReason).toBe("client-disconnect");
});
+ it("reads the source only as fast as the consumer pulls", async () => {
+ let sourcePulls = 0;
+ const source = new ReadableStream<Uint8Array>({
+ pull(controller) {
+ controller.enqueue(encoder.encode(`{"i":${sourcePulls}}\n`));
+ sourcePulls += 1;
+ if (sourcePulls === 100) controller.close();
+ },
+ });
+
+ const reader = parseNdjsonStream<{ i: number }>(() => source).getReader();
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ const pullsBeforeRead = sourcePulls;
+ await expect(reader.read()).resolves.toEqual({ done: false, value: { i: 0 } });
+ await new Promise((resolve) => setTimeout(resolve, 10));
+
+ expect(pullsBeforeRead).toBeLessThan(5);
+ expect(sourcePulls).toBeLessThan(5);
+ await reader.cancel();
+ });
+
+ it("cancels the source when a line fails to parse", async () => {
+ let cancelled = false;
+ const source = new ReadableStream<Uint8Array>({
+ start(controller) {
+ controller.enqueue(encoder.encode("not json\n"));
+ },
+ cancel() {
+ cancelled = true;
+ },
+ });
+
+ await expect(drain(parseNdjsonStream(() => source))).rejects.toThrow(SyntaxError);
+ expect(cancelled).toBe(true);
+ });
+
it("surfaces source errors to the consumer", async () => {
const source = new ReadableStream<Uint8Array>({
start(controller) {
diff --git a/packages/eve/src/execution/ndjson-stream.ts b/packages/eve/src/execution/ndjson-stream.ts
index d8ea2bad5..95812a8c8 100644
--- a/packages/eve/src/execution/ndjson-stream.ts
+++ b/packages/eve/src/execution/ndjson-stream.ts
@@ -2,9 +2,9 @@
* Transforms a byte stream of newline-delimited JSON (NDJSON) into a
* stream of parsed values.
*
- * The byte stream is produced lazily by `createByteStream`, so the
- * underlying source (a world-local run readable) is only opened when the
- * returned stream is consumed.
+ * The source is read on demand: each pull reads only until it can enqueue
+ * at least one value, so a slow consumer applies backpressure to the source
+ * instead of this stream buffering everything the source can deliver.
*
* Cancellation is forwarded to the source. When the returned stream is
* cancelled — e.g. an SSE client disconnects and the server cancels the
@@ -12,9 +12,9 @@
* runs that never reach EOF: a parked (`session.waiting`) durable run keeps
* its event stream open indefinitely, and the world-local streamer runs a
* filesystem poll until its reader is cancelled. Without forwarding the
- * cancel, the read loop below would block on `reader.read()` forever and
- * that poll would leak for the life of the process, degrading streaming
- * throughput for every other session.
+ * cancel, a pending `reader.read()` would block forever and that poll would
+ * leak for the life of the process, degrading streaming throughput for every
+ * other session.
*/
export function parseNdjsonStream<T>(
createByteStream: () => ReadableStream<Uint8Array>,
@@ -26,16 +26,33 @@ export function parseNdjsonStream<T>(
let cancelled = false;
return new ReadableStream<T>({
- async start(controller) {
+ start() {
reader = createByteStream().getReader();
+ },
+ async pull(controller) {
try {
+ // A chunk may hold no complete line, and the stream does not pull
+ // again until this pull enqueues, so keep reading until it does.
while (true) {
- const { value, done } = await reader.read();
+ const { value, done } = await reader!.read();
- if (done) break;
+ // A cancel resolves the pending read with `done`; bail before
+ // touching the controller, which the cancel has already closed.
+ if (cancelled) return;
+
+ if (done) {
+ buffer += decoder.decode();
+ const trailing = buffer.trim();
+ if (trailing.length > 0) {
+ controller.enqueue(parse(JSON.parse(trailing)));
+ }
+ controller.close();
+ return;
+ }
buffer += decoder.decode(value, { stream: true });
+ let enqueued = false;
for (
let newlineIndex = buffer.indexOf("\n");
newlineIndex !== -1;
@@ -46,24 +63,16 @@ export function parseNdjsonStream<T>(
if (line.length > 0) {
controller.enqueue(parse(JSON.parse(line)));
+ enqueued = true;
}
}
+ if (enqueued) return;
}
-
- // A cancel resolves the pending read with `done`; bail before
- // touching the controller, which the cancel has already closed.
- if (cancelled) return;
-
- buffer += decoder.decode();
- const trailing = buffer.trim();
- if (trailing.length > 0) {
- controller.enqueue(parse(JSON.parse(trailing)));
- }
- controller.close();
} catch (error) {
- if (!cancelled) controller.error(error);
- } finally {
- reader.releaseLock();
+ if (cancelled) return;
+ controller.error(error);
+ // An errored stream never forwards a later cancel, so release the source now.
+ await reader!.cancel(error).catch(() => {});
}
},
async cancel(reason) {
diff --git a/packages/eve/src/public/channels/eve.test.ts b/packages/eve/src/public/channels/eve.test.ts
index 5713afb8e..469473763 100644
--- a/packages/eve/src/public/channels/eve.test.ts
+++ b/packages/eve/src/public/channels/eve.test.ts
@@ -610,7 +610,20 @@ describe("eveChannel — stream cursor", () => {
expect(response.status).toBe(404);
expect(await response.json()).toEqual({ error: "Session not found.", ok: false });
- expect(handler.getEventStream).not.toHaveBeenCalled();
+ });
+
+ it("cancels the early-opened event stream when the tail lookup fails", async () => {
+ const handler = createEveStreamHandler({ auth: none() });
+ const cancelled = vi.fn();
+ handler.getEventStream.mockResolvedValueOnce(new ReadableStream({ cancel: cancelled }));
+ handler.getStreamTailIndex.mockRejectedValueOnce(
+ new WorkflowRunNotFoundError("test-session-id"),
+ );
+
+ const response = await handler.fetch("https://eve.test/eve/v1/session/test-session-id/stream");
+
+ expect(response.status).toBe(404);
+ await vi.waitFor(() => expect(cancelled).toHaveBeenCalledOnce());
});
it("returns 503 when the session lookup fails transiently", async () => {
@@ -621,7 +634,6 @@ describe("eveChannel — stream cursor", () => {
expect(response.status).toBe(503);
expect(await response.json()).toEqual({ error: "Session stream unavailable.", ok: false });
- expect(handler.getEventStream).not.toHaveBeenCalled();
});
it("reports the durable tail index when the request opts in", async () => {