Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

perf(server): bound orchestration replay and slow-client live buffers (#216)

Leo41e213a90bda
diff --git a/apps/server/src/channels/ChannelRuntime.test.ts b/apps/server/src/channels/ChannelRuntime.test.ts
index c503ffae4..19cbfd08f 100644
--- a/apps/server/src/channels/ChannelRuntime.test.ts
+++ b/apps/server/src/channels/ChannelRuntime.test.ts
@@ -440,6 +440,8 @@ function makeHarness(input: {
});
const engine = {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch,
streamDomainEvents: Stream.empty,
subscribeDomainEvents: Effect.succeed(Stream.empty),
diff --git a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts b/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts
index ee215bb04..67745c9c0 100644
--- a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts
+++ b/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts
@@ -85,6 +85,7 @@ describe("CheckpointDiffQuery.layer", () => {
Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),
getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),
getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.succeed(Option.none()),
@@ -197,6 +198,7 @@ describe("CheckpointDiffQuery.layer", () => {
Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),
getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),
getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.succeed(Option.none()),
@@ -284,6 +286,7 @@ describe("CheckpointDiffQuery.layer", () => {
Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),
getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),
getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.succeed(Option.none()),
@@ -356,6 +359,7 @@ describe("CheckpointDiffQuery.layer", () => {
Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),
getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),
getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.succeed(Option.none()),
@@ -413,6 +417,7 @@ describe("CheckpointDiffQuery.layer", () => {
Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),
getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),
getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.succeed(Option.none()),
diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
index 260ec3a4e..040a8ec60 100644
--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts
@@ -106,6 +106,8 @@ describe("OrchestrationEngine", () => {
return savedEvent;
}),
readFromSequence: () => Stream.empty,
+ readAggregateRange: () => Stream.die("unused aggregate replay"),
+ getAggregateReplayStats: () => Effect.die("unused aggregate replay stats"),
readAll: () =>
Stream.fail(
new PersistenceSqlError({
@@ -209,6 +211,7 @@ describe("OrchestrationEngine", () => {
getSnapshotSequence: () =>
Effect.succeed({ snapshotSequence: projectionSnapshot.snapshotSequence }),
getCounts: () => Effect.succeed({ projectCount: 1, threadCount: 1 }),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.succeed(Option.none()),
@@ -823,6 +826,12 @@ describe("OrchestrationEngine", () => {
readFromSequence(sequenceExclusive) {
return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive));
},
+ readAggregateRange() {
+ return Stream.die("unused aggregate replay");
+ },
+ getAggregateReplayStats() {
+ return Effect.die("unused aggregate replay stats");
+ },
readAll() {
return Stream.fromIterable(events);
},
@@ -1064,6 +1073,12 @@ describe("OrchestrationEngine", () => {
readFromSequence(sequenceExclusive) {
return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive));
},
+ readAggregateRange() {
+ return Stream.die("unused aggregate replay");
+ },
+ getAggregateReplayStats() {
+ return Effect.die("unused aggregate replay stats");
+ },
readAll() {
return Stream.fromIterable(events);
},
diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
index dc63abf1f..311e85a6a 100644
--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts
@@ -408,8 +408,24 @@ const makeOrchestrationEngine = Effect.gen(function* () {
Effect.annotateLogs({ sequence: commandReadModel.snapshotSequence }),
);
- const readEvents: OrchestrationEngineShape["readEvents"] = (fromSequenceExclusive, limit) =>
- eventStore.readFromSequence(fromSequenceExclusive, limit);
+ const readEvents: OrchestrationEngineShape["readEvents"] = (
+ fromSequenceExclusive,
+ limit,
+ toSequenceInclusive,
+ ) => eventStore.readFromSequence(fromSequenceExclusive, limit, toSequenceInclusive);
+
+ const readThreadEvents: OrchestrationEngineShape["readThreadEvents"] = ({ threadId, ...range }) =>
+ eventStore.readAggregateRange({ ...range, aggregateKind: "thread", aggregateId: threadId });
+
+ const getThreadReplayStats: OrchestrationEngineShape["getThreadReplayStats"] = ({
+ threadId,
+ ...range
+ }) =>
+ eventStore.getAggregateReplayStats({
+ ...range,
+ aggregateKind: "thread",
+ aggregateId: threadId,
+ });
const dispatch: OrchestrationEngineShape["dispatch"] = (command, options) =>
Effect.gen(function* () {
@@ -426,6 +442,8 @@ const makeOrchestrationEngine = Effect.gen(function* () {
return {
readEvents,
+ readThreadEvents,
+ getThreadReplayStats,
dispatch,
// Each access creates a fresh PubSub subscription so that multiple
// consumers (wsServer, ProviderRuntimeIngestion, CheckpointReactor, etc.)
diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts
index 5bbb4dd06..77b519f79 100644
--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts
+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts
@@ -1008,6 +1008,55 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {
}),
);
+ it.effect("measures replay payload bytes without decoding event bodies", () =>
+ Effect.gen(function* () {
+ const snapshotQuery = yield* ProjectionSnapshotQuery;
+ const sql = yield* SqlClient.SqlClient;
+
+ yield* sql`DELETE FROM orchestration_events`;
+ const rows = yield* sql<{ readonly sequence: number }>`
+ INSERT INTO orchestration_events (
+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,
+ command_id, causation_event_id, correlation_id, actor_kind, payload_json, metadata_json
+ )
+ VALUES
+ (
+ 'replay-event-1', 'thread', 'thread-replay', 1, 'thread.activity-appended',
+ '2026-03-01T00:00:00.000Z', NULL, NULL, NULL, 'provider',
+ json_object('output', printf('%.*c', 1000, 'x')), '{}'
+ ),
+ (
+ 'replay-event-2', 'thread', 'thread-replay', 2, 'thread.activity-appended',
+ '2026-03-01T00:00:01.000Z', NULL, NULL, NULL, 'provider',
+ json_object('output', printf('%.*c', 2000, 'x')), '{}'
+ ),
+ (
+ 'replay-event-3', 'thread', 'thread-replay', 3, 'thread.activity-appended',
+ '2026-03-01T00:00:02.000Z', NULL, NULL, NULL, 'provider',
+ json_object('output', printf('%.*c', 3000, 'x')), '{}'
+ ),
+ (
+ 'replay-event-4', 'thread', 'thread-replay', 4, 'thread.activity-appended',
+ '2026-03-01T00:00:03.000Z', NULL, NULL, NULL, 'provider',
+ json_object('output', '😀'), '{}'
+ )
+ RETURNING sequence
+ `;
+
+ // Bytes, not code points: the 4-byte emoji row is {"output":"😀"}, 17 bytes.
+ // Shared-layer AUTOINCREMENT does not reset after DELETE, so bound the
+ // query to the inserted sequences rather than assuming 1..4.
+ const stats = yield* snapshotQuery.getEventReplayStats({
+ fromSequenceExclusive: rows[0]!.sequence,
+ toSequenceInclusive: rows[3]!.sequence,
+ });
+ assert.deepStrictEqual(stats, {
+ eventCount: 3,
+ payloadBytes: 5043,
+ });
+ }),
+ );
+
it.effect("reads single-thread checkpoint context without hydrating unrelated threads", () =>
Effect.gen(function* () {
const snapshotQuery = yield* ProjectionSnapshotQuery;
diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
index 65f9b3023..7a823f745 100644
--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts
@@ -81,6 +81,7 @@ import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityRes
import { ORCHESTRATION_PROJECTOR_NAMES } from "./ProjectionPipeline.ts";
import {
ProjectionSnapshotQuery,
+ type ProjectionEventReplayStats,
type ProjectionFullThreadDiffContext,
type ProjectionSnapshotCounts,
type ProjectionThreadCheckpointContext,
@@ -192,6 +193,14 @@ const ProjectionCountsRowSchema = Schema.Struct({
projectCount: Schema.Number,
threadCount: Schema.Number,
});
+const EventReplayStatsInput = Schema.Struct({
+ fromSequenceExclusive: NonNegativeInt,
+ toSequenceInclusive: NonNegativeInt,
+});
+const EventReplayStatsRowSchema = Schema.Struct({
+ eventCount: Schema.Number,
+ payloadBytes: Schema.Number,
+});
const ProjectionThreadSearchRequest = Schema.Struct({
pattern: Schema.String,
limit: Schema.Int,
@@ -1009,6 +1018,20 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {
`,
});
+ const readEventReplayStats = SqlSchema.findOne({
+ Request: EventReplayStatsInput,
+ Result: EventReplayStatsRowSchema,
+ execute: ({ fromSequenceExclusive, toSequenceInclusive }) =>
+ sql`
+ SELECT
+ COUNT(*) AS "eventCount",
+ COALESCE(SUM(octet_length(payload_json)), 0) AS "payloadBytes"
+ FROM orchestration_events
+ WHERE sequence > ${fromSequenceExclusive}
+ AND sequence <= ${toSequenceInclusive}
+ `,
+ });
+
const searchActiveThreadRows = SqlSchema.findAll({
Request: ProjectionThreadSearchRequest,
Result: ProjectionThreadSearchRow,
@@ -2952,6 +2975,22 @@ pending_approval_requests AS (
),
);
+ const getEventReplayStats: ProjectionSnapshotQueryShape["getEventReplayStats"] = (input) =>
+ readEventReplayStats(input).pipe(
+ Effect.mapError(
+ toPersistenceSqlOrDecodeError(
+ "ProjectionSnapshotQuery.getEventReplayStats:query",
+ "ProjectionSnapshotQuery.getEventReplayStats:decodeRow",
+ ),
+ ),
+ Effect.map(
+ (row): ProjectionEventReplayStats => ({
+ eventCount: row.eventCount,
+ payloadBytes: row.payloadBytes,
+ }),
+ ),
+ );
+
const searchThreads: ProjectionSnapshotQueryShape["searchThreads"] = Effect.fn(
"ProjectionSnapshotQuery.searchThreads",
)(function* (input) {
@@ -3665,6 +3704,7 @@ pending_approval_requests AS (
searchThreads,
getSnapshotSequence,
getCounts,
+ getEventReplayStats,
getActiveProjectByWorkspaceRoot,
getOriginalProjectIdByWorkspaceRoot,
getProjectShellById,
diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts
index d6ff7f8d6..21282429e 100644
--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts
+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts
@@ -473,6 +473,8 @@ describe("ProviderCommandReactor", () => {
const engine = yield* OrchestrationEngineService;
return {
readEvents: engine.readEvents,
+ readThreadEvents: engine.readThreadEvents,
+ getThreadReplayStats: engine.getThreadReplayStats,
dispatch: (command) => {
if (command.type === "thread.title.regeneration.complete") {
titleRegenerationCompletionDispatchAttempts += 1;
diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts
index 16b296f02..659e00034 100644
--- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts
+++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts
@@ -69,6 +69,8 @@ describe("ThreadDeletionReactor.start", () => {
const engine: OrchestrationEngineService["Service"] = {
readEvents: () => Stream.die("unused"),
+ readThreadEvents: () => Stream.die("unused"),
+ getThreadReplayStats: () => Effect.die("unused"),
dispatch: () => Effect.die("unused"),
get streamDomainEvents() {
return Stream.die(
diff --git a/apps/server/src/orchestration/LiveStreamBudget.test.ts b/apps/server/src/orchestration/LiveStreamBudget.test.ts
new file mode 100644
index 000000000..d163a8317
--- /dev/null
+++ b/apps/server/src/orchestration/LiveStreamBudget.test.ts
@@ -0,0 +1,47 @@
+import { it } from "@effect/vitest";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Queue from "effect/Queue";
+import * as Stream from "effect/Stream";
+import { describe, expect } from "vite-plus/test";
+
+import { makeLiveStreamBudget, type RetainedLiveItem } from "./LiveStreamBudget.ts";
+
+describe("LiveStreamBudget", () => {
+ it.effect("closes the source without releasing a batch still waiting for an ACK", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const budget = yield* makeLiveStreamBudget({ maxItems: 3 });
+ const queue = yield* Queue.unbounded<RetainedLiveItem<{ text: string }>>();
+ const sourceClosed = yield* Deferred.make<void>();
+ const first = yield* budget.retain({ text: "first" });
+ const second = yield* budget.retain({ text: "second" });
+ const third = yield* budget.retain({ text: "third" });
+ yield* Queue.offerAll(queue, [first, second, third]);
+ yield* Effect.scoped(
+ Effect.gen(function* () {
+ const pull = yield* Stream.toPull(
+ budget.deliver(
+ Stream.fromQueue(queue).pipe(
+ Stream.rechunk(1),
+ Stream.ensuring(Deferred.succeed(sourceClosed, undefined)),
+ ),
+ ),
+ );
+ expect(yield* pull).toEqual([{ text: "first" }]);
+ // The other two items are in the source's pull state, not its queue.
+ expect(yield* Queue.size(queue)).toBe(0);
+ expect((yield* budget.usage).retainedItems).toBe(3);
+ const overflow = yield* budget.retain({ text: "fourth" }).pipe(Effect.result);
+ expect(overflow._tag).toBe("Failure");
+ // Do not resume the consumer. Its source scope must close now.
+ yield* Deferred.await(sourceClosed);
+ yield* budget.closed;
+ expect((yield* budget.usage).retainedItems).toBe(1);
+ }),
+ );
+ expect(yield* budget.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
+ }),
+ ),
+ );
+});
diff --git a/apps/server/src/orchestration/LiveStreamBudget.ts b/apps/server/src/orchestration/LiveStreamBudget.ts
new file mode 100644
index 000000000..3e3d31128
--- /dev/null
+++ b/apps/server/src/orchestration/LiveStreamBudget.ts
@@ -0,0 +1,196 @@
+import { OrchestrationGetSnapshotError } from "@t3tools/contracts";
+import * as Arr from "effect/Array";
+import * as Deferred from "effect/Deferred";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Scope from "effect/Scope";
+import * as Stream from "effect/Stream";
+
+export const LIVE_STREAM_MAX_ITEMS = 1_000;
+export const LIVE_STREAM_MAX_SERIALIZED_BYTES = 8 * 1024 * 1024;
+
+export interface RetainedLiveItem<A> {
+ readonly value: A;
+ readonly serializedBytes: number;
+}
+
+// Published events are immutable and shared across subscriptions. Measure each
+// object once without keeping the event or its serialized copy alive.
+const serializedSizes = new WeakMap<object, number>();
+
+function serializedSize(value: object): number {
+ const cached = serializedSizes.get(value);
+ if (cached !== undefined) {
+ return cached;
+ }
+ const bytes = Buffer.byteLength(JSON.stringify(value));
+ serializedSizes.set(value, bytes);
+ return bytes;
+}
+
+/** One budget covers one subscription and its delivery stream, including the batch waiting for an RPC ACK. */
+export const makeLiveStreamBudget = Effect.fn("makeLiveStreamBudget")(function* (limits?: {
+ readonly maxItems?: number;
+ readonly maxSerializedBytes?: number;
+}) {
+ const maxItems = limits?.maxItems ?? LIVE_STREAM_MAX_ITEMS;
+ const maxSerializedBytes = limits?.maxSerializedBytes ?? LIVE_STREAM_MAX_SERIALIZED_BYTES;
+ const failed = yield* Deferred.make<never, OrchestrationGetSnapshotError>();
+ const cleanupComplete = yield* Deferred.make<void>();
+ let failure: OrchestrationGetSnapshotError | undefined;
+ const retained = new Set<RetainedLiveItem<unknown>>();
+ let retainedSerializedBytes = 0;
+
+ const release = (items: Iterable<RetainedLiveItem<unknown>>) => {
+ for (const item of items) {
+ if (!retained.delete(item)) {
+ continue;
+ }
+ retainedSerializedBytes -= item.serializedBytes;
+ }
+ };
+ yield* Effect.addFinalizer(() =>
+ Effect.sync(() => release(retained)).pipe(
+ Effect.andThen(Deferred.succeed(cleanupComplete, undefined)),
+ ),
+ );
+
+ const check = Effect.suspend(() => (failure ? Effect.fail(failure) : Effect.void));
+
+ const overflow = Effect.fn("LiveStreamBudget.overflow")(function* (
+ nextItems: number,
+ nextSerializedBytes: number,
+ ) {
+ failure ??= new OrchestrationGetSnapshotError({
+ message: "The live event buffer is full. Resume from the last received sequence.",
+ });
+ yield* Deferred.fail(failed, failure);
+ yield* Effect.logWarning("orchestration live event buffer is full", {
+ retainedItems: retained.size,
+ retainedSerializedBytes,
+ nextItems,
+ nextSerializedBytes,
+ maxItems,
+ maxSerializedBytes,
+ });
+ return yield* failure;
+ });
+
+ const retain = <A extends object>(value: A, payload: object = value) =>
+ Effect.suspend(() => {
+ if (failure) {
+ return Effect.fail(failure);
+ }
+ const serializedBytes = serializedSize(payload);
+ const nextItems = retained.size + 1;
+ const nextSerializedBytes = retainedSerializedBytes + serializedBytes;
+ if (nextItems > maxItems || nextSerializedBytes > maxSerializedBytes) {
+ return overflow(nextItems, nextSerializedBytes);
+ }
+ const item = { value, serializedBytes };
+ retained.add(item);
+ retainedSerializedBytes = nextSerializedBytes;
+ return Effect.succeed(item);
+ });
+
+ // Replace one coalescing batch atomically. Both raw and projected payloads
+ // count against the same budget, and discarded updates release their charge.
+ const replace = <A extends object>(
+ previous: ReadonlyArray<RetainedLiveItem<unknown>>,
+ values: ReadonlyArray<A>,
+ payload: (value: A) => object = (value) => value,
+ ) =>
+ Effect.suspend(() => {
+ if (failure) {
+ return Effect.fail(failure);
+ }
+ const next = values.map((value) => ({
+ value,
+ serializedBytes: serializedSize(payload(value)),
+ }));
+ let nextItems = retained.size + next.length;
+ let nextSerializedBytes =
+ retainedSerializedBytes + next.reduce((sum, item) => sum + item.serializedBytes, 0);
+ for (const item of previous) {
+ if (retained.has(item)) {
+ nextItems -= 1;
+ nextSerializedBytes -= item.serializedBytes;
+ }
+ }
+ if (nextItems > maxItems || nextSerializedBytes > maxSerializedBytes) {
+ return overflow(nextItems, nextSerializedBytes);
+ }
+ release(previous);
+ for (const item of next) {
+ retained.add(item);
+ }
+ retainedSerializedBytes = nextSerializedBytes;
+ return Effect.succeed(next);
+ });
+
+ const deliver = <A, E, R>(stream: Stream.Stream<RetainedLiveItem<A>, E, R>) =>
+ Stream.fromPull(
+ Effect.gen(function* () {
+ yield* check;
+ const sourceScope = yield* Scope.fork(yield* Effect.scope);
+ const source = {
+ pull: yield* Stream.toPull(stream).pipe(Scope.provide(sourceScope)),
+ };
+ let inFlight: ReadonlyArray<RetainedLiveItem<A>> = [];
+ yield* Effect.addFinalizer(() =>
+ Effect.sync(() => {
+ release(inFlight);
+ inFlight = [];
+ source.pull = Effect.interrupt;
+ }),
+ );
+ yield* Deferred.await(failed).pipe(
+ Effect.catchTags({
+ OrchestrationGetSnapshotError: (error) =>
+ Effect.sync(() => {
+ // A grouped source can retain its own pending chunk. Close it
+ // and release the pull closure without waiting for an RPC ACK.
+ source.pull = Effect.interrupt;
+ }).pipe(
+ Effect.andThen(Scope.close(sourceScope, Exit.fail(error))),
+ Effect.andThen(
+ Effect.sync(() => {
+ const delivered = new Set<RetainedLiveItem<unknown>>(inFlight);
+ for (const item of retained) {
+ if (!delivered.has(item)) {
+ release([item]);
+ }
+ }
+ }),
+ ),
+ Effect.andThen(Deferred.succeed(cleanupComplete, undefined)),
+ ),
+ }),
+ Effect.forkScoped,
+ );
+ // @effect-diagnostics-next-line returnEffectInGen:off - Stream.fromPull needs the pull effect as its result.
+ return Effect.gen(function* () {
+ // RpcServer requests the next batch only after the client ACKs this
+ // one. Removing items from a queue alone does not mean delivery ended.
+ release(inFlight);
+ inFlight = [];
+ yield* check;
+ const items = yield* Effect.raceFirst(source.pull, Deferred.await(failed));
+ inFlight = items;
+ yield* check;
+ return Arr.map(items, (item) => item.value);
+ });
+ }),
+ );
+
+ return {
+ retain,
+ replace,
+ release,
+ deliver,
+ check,
+ failed: Deferred.await(failed),
+ closed: Deferred.await(cleanupComplete),
+ usage: Effect.sync(() => ({ retainedItems: retained.size, retainedSerializedBytes })),
+ };
+});
diff --git a/apps/server/src/orchestration/Services/OrchestrationEngine.ts b/apps/server/src/orchestration/Services/OrchestrationEngine.ts
index fdb22ce64..082a199af 100644
--- a/apps/server/src/orchestration/Services/OrchestrationEngine.ts
+++ b/apps/server/src/orchestration/Services/OrchestrationEngine.ts
@@ -15,6 +15,7 @@ import type {
OrchestrationClientOrigin,
OrchestrationCommand,
OrchestrationEvent,
+ ThreadId,
} from "@t3tools/contracts";
import * as Context from "effect/Context";
import type * as Effect from "effect/Effect";
@@ -23,6 +24,13 @@ import type * as Stream from "effect/Stream";
import type { OrchestrationDispatchError } from "../Errors.ts";
import type { OrchestrationEventStoreError } from "../../persistence/Errors.ts";
+import type { OrchestrationAggregateReplayStats } from "../../persistence/Services/OrchestrationEventStore.ts";
+
+export interface OrchestrationThreadReplayRange {
+ readonly threadId: ThreadId;
+ readonly fromSequenceExclusive: number;
+ readonly toSequenceInclusive: number;
+}
export interface OrchestrationDispatchActor {
readonly personId: AuthSessionId;
@@ -39,15 +47,26 @@ export interface OrchestrationEngineShape {
* @param fromSequenceExclusive - Sequence cursor (exclusive).
* @param limit - Maximum number of events to read. Defaults to the event
* store's page-bounded default; pass a higher value when the caller must
- * read every event after the cursor (e.g. per-thread catch-up that filters
- * a small subset out of a potentially larger global range).
+ * read a wider global range. Thread subscriptions use readThreadEvents.
+ * @param toSequenceInclusive - Optional captured upper sequence boundary.
* @returns Stream containing ordered events.
*/
readonly readEvents: (
fromSequenceExclusive: number,
limit?: number,
+ toSequenceInclusive?: number,
) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError, never>;
+ /** Read only this thread's events through a captured authoritative head. */
+ readonly readThreadEvents: (
+ input: OrchestrationThreadReplayRange & { readonly limit?: number },
+ ) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;
+
+ /** Measure a bounded thread replay without decoding its event bodies. */
+ readonly getThreadReplayStats: (
+ input: OrchestrationThreadReplayRange & { readonly maxEvents: number },
+ ) => Effect.Effect<OrchestrationAggregateReplayStats, OrchestrationEventStoreError>;
+
/**
* Dispatch a validated orchestration command.
*
diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts
index 30ba972f8..b185b59e0 100644
--- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts
+++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts
@@ -39,6 +39,11 @@ export interface ProjectionSnapshotCounts {
readonly threadCount: number;
}
+export interface ProjectionEventReplayStats {
+ readonly eventCount: number;
+ readonly payloadBytes: number;
+}
+
export interface ProjectionSnapshotSequence {
readonly snapshotSequence: number;
}
@@ -150,6 +155,14 @@ export interface ProjectionSnapshotQueryShape {
*/
readonly getCounts: () => Effect.Effect<ProjectionSnapshotCounts, ProjectionRepositoryError>;
+ /**
+ * Measure a persisted event range without decoding its payload bodies.
+ */
+ readonly getEventReplayStats: (input: {
+ readonly fromSequenceExclusive: number;
+ readonly toSequenceInclusive: number;
+ }) => Effect.Effect<ProjectionEventReplayStats, ProjectionRepositoryError>;
+
/**
* Read the active project for an exact workspace root match.
*/
diff --git a/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.ts b/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.ts
index 0a9915294..6b54fadf8 100644
--- a/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.ts
+++ b/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.ts
@@ -3,12 +3,14 @@ import {
MessageId,
ThreadId,
TurnId,
- type OrchestrationEvent,
+ OrchestrationEvent,
type OrchestrationThreadActivity,
} from "@t3tools/contracts";
import { it } from "@effect/vitest";
import * as Clock from "effect/Clock";
import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import { describe, expect } from "vite-plus/test";
@@ -19,6 +21,7 @@ import {
const threadId = ThreadId.make("thread-coalescer-test");
const turnId = TurnId.make("turn-coalescer-test");
+const encodeEvent = Schema.encodeSync(Schema.fromJsonString(OrchestrationEvent));
function makeToolActivity(
sequence: number,
@@ -61,7 +64,7 @@ function makeToolActivity(
};
}
-function makeMessage(sequence: number): OrchestrationEvent {
+function makeMessage(sequence: number, text = "Still working"): OrchestrationEvent {
return {
sequence,
eventId: EventId.make(`event-${sequence}`),
@@ -77,7 +80,7 @@ function makeMessage(sequence: number): OrchestrationEvent {
threadId,
messageId: MessageId.make(`message-${sequence}`),
role: "assistant",
- text: "Still working",
+ text,
turnId,
streaming: false,
createdAt: "2026-01-01T00:00:02.000Z",
@@ -134,15 +137,14 @@ describe("ThreadLiveEventCoalescer", () => {
const startedAt = yield* Clock.currentTimeMillis;
yield* Effect.forEach(
Array.from({ length: 10 }, (_, index) => index + 2),
- (sequence) =>
- coalescer.offerAndWait({ kind: "event", event: makeToolActivity(sequence) }),
+ (sequence) => coalescer.offer({ kind: "event", event: makeToolActivity(sequence) }),
{ discard: true },
);
- yield* coalescer.offerAndWait({ kind: "event", event: makeMessage(12) });
+ yield* coalescer.offer({ kind: "event", event: makeMessage(12) });
expect(yield* Clock.currentTimeMillis).toBe(startedAt);
expect(
- Array.from(yield* coalescer.takeAll).map((item) =>
+ (yield* coalescer.stream.pipe(Stream.take(2), Stream.runCollect)).map((item) =>
item.kind === "event" ? item.event.sequence : item.kind,
),
).toEqual([11, 12]);
@@ -155,17 +157,94 @@ describe("ThreadLiveEventCoalescer", () => {
Effect.gen(function* () {
const coalescer = yield* makeThreadLiveEventCoalescer({ coalesceWindow: "500 millis" });
const startedAt = yield* Clock.currentTimeMillis;
- yield* coalescer.offerAndWait({ kind: "event", event: makeToolActivity(2) });
- yield* coalescer.offerAndWait({ kind: "event", event: makeToolActivity(3) });
- yield* coalescer.offerAndWait({ kind: "synchronized" });
+ yield* coalescer.offer({ kind: "event", event: makeToolActivity(2) });
+ yield* coalescer.offer({ kind: "event", event: makeToolActivity(3) });
+ yield* coalescer.offer({ kind: "synchronized" });
expect(yield* Clock.currentTimeMillis).toBe(startedAt);
expect(
- Array.from(yield* coalescer.takeAll).map((item) =>
+ (yield* coalescer.stream.pipe(Stream.take(2), Stream.runCollect)).map((item) =>
item.kind === "event" ? item.event.sequence : item.kind,
),
).toEqual([3, "synchronized"]);
}),
).pipe(Effect.provide(TestClock.layer())),
);
+
+ it.effect("fails and clears pending updates when their serialized payload fills the budget", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const first = makeToolActivity(1);
+ const coalescer = yield* makeThreadLiveEventCoalescer({
+ coalesceWindow: "500 millis",
+ maxSerializedBytes: Buffer.byteLength(encodeEvent(first)),
+ });
+ yield* coalescer.offer({ kind: "event", event: first });
+ expect(yield* coalescer.usage).toEqual({
+ retainedItems: 1,
+ retainedSerializedBytes: Buffer.byteLength(encodeEvent(first)),
+ });
+
+ const overflow = yield* coalescer
+ .offer({ kind: "event", event: makeToolActivity(2) })
+ .pipe(Effect.result);
+ expect(overflow._tag).toBe("Failure");
+ yield* coalescer.closed;
+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
+ const marker = yield* coalescer.offer({ kind: "synchronized" }).pipe(Effect.result);
+ expect(marker._tag).toBe("Failure");
+ const delivered = yield* coalescer.stream.pipe(Stream.runCollect, Effect.result);
+ expect(delivered._tag).toBe("Failure");
+ }),
+ ),
+ );
+
+ it.effect("keeps the flush timer alive when an offer's shorter scope closes", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const coalescer = yield* makeThreadLiveEventCoalescer({ coalesceWindow: "50 millis" });
+ yield* Effect.scoped(coalescer.offer({ kind: "event", event: makeToolActivity(1) }));
+ yield* TestClock.adjust("50 millis");
+ const items = yield* coalescer.stream.pipe(Stream.take(1), Stream.runCollect);
+ expect(
+ items.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),
+ ).toEqual([1]);
+ }),
+ ).pipe(Effect.provide(TestClock.layer())),
+ );
+
+ it.effect("keeps an unacknowledged batch charged and clears later events on overflow", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const coalescer = yield* makeThreadLiveEventCoalescer({ maxItems: 3 });
+ const first = makeMessage(1, "é".repeat(1_024));
+ yield* coalescer.offer({ kind: "event", event: first });
+
+ yield* Effect.scoped(
+ Effect.gen(function* () {
+ const pull = yield* Stream.toPull(coalescer.stream);
+ const batch = yield* pull;
+ expect(
+ batch.map((item) => (item.kind === "event" ? item.event.sequence : null)),
+ ).toEqual([1]);
+ yield* coalescer.offer({ kind: "event", event: makeMessage(2) });
+ yield* coalescer.offer({ kind: "event", event: makeToolActivity(3) });
+ expect((yield* coalescer.usage).retainedItems).toBe(3);
+
+ const overflow = yield* coalescer
+ .offer({ kind: "event", event: makeToolActivity(4, { kind: "tool.completed" }) })
+ .pipe(Effect.result);
+ expect(overflow._tag).toBe("Failure");
+ // Do not pull or acknowledge the batch. Cleanup must still finish.
+ yield* coalescer.closed;
+ expect(yield* coalescer.usage).toEqual({
+ retainedItems: 1,
+ retainedSerializedBytes: Buffer.byteLength(encodeEvent(first)),
+ });
+ }),
+ );
+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });
+ }),
+ ),
+ );
});
diff --git a/apps/server/src/orchestration/ThreadLiveEventCoalescer.ts b/apps/server/src/orchestration/ThreadLiveEventCoalescer.ts
index f188deea8..c7f2155b4 100644
--- a/apps/server/src/orchestration/ThreadLiveEventCoalescer.ts
+++ b/apps/server/src/orchestration/ThreadLiveEventCoalescer.ts
@@ -1,4 +1,8 @@
-import type { OrchestrationEvent, OrchestrationThreadStreamItem } from "@t3tools/contracts";
+import type {
+ OrchestrationEvent,
+ OrchestrationGetSnapshotError,
+ OrchestrationThreadStreamItem,
+} from "@t3tools/contracts";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
@@ -9,6 +13,7 @@ import * as Semaphore from "effect/Semaphore";
import * as Stream from "effect/Stream";
import { projectActivityEvent } from "./ActivityPayloadProjection.ts";
+import { makeLiveStreamBudget, type RetainedLiveItem } from "./LiveStreamBudget.ts";
const COALESCE_WINDOW = Duration.millis(50);
const MAX_PENDING_UPDATES = 512;
@@ -89,17 +94,24 @@ export function coalesceLiveToolUpdatedEvents(
}
export const makeThreadLiveEventCoalescer = Effect.fn("makeThreadLiveEventCoalescer")(
- function* (options?: { readonly coalesceWindow?: Duration.Input }) {
- const output = yield* Queue.unbounded<OrchestrationThreadStreamItem>();
- const input = yield* Queue.unbounded<{
- readonly value: ThreadLiveInput;
- readonly processed?: Deferred.Deferred<void>;
- }>();
+ function* (options?: {
+ readonly coalesceWindow?: Duration.Input;
+ readonly maxItems?: number;
+ readonly maxSerializedBytes?: number;
+ }) {
+ const coalescerScope = yield* Effect.scope;
+ const budget = yield* makeLiveStreamBudget(options);
+ const cleanupComplete = yield* Deferred.make<void>();
+ const output = yield* Queue.unbounded<
+ RetainedLiveItem<OrchestrationThreadStreamItem>,
+ OrchestrationGetSnapshotError
+ >();
const mutex = yield* Semaphore.make(1);
const coalesceWindow = options?.coalesceWindow ?? COALESCE_WINDOW;
- let pendingUpdates: Array<OrchestrationEvent> = [];
+ let pendingUpdates: Array<RetainedLiveItem<OrchestrationEvent>> = [];
let windowGeneration = 0;
let windowFiber: Fiber.Fiber<void, never> | null = null;
+ let closed = false;
const cancelWindow = Effect.fn("ThreadLiveEventCoalescer.cancelWindow")(function* () {
const fiber = windowFiber;
@@ -110,22 +122,21 @@ export const makeThreadLiveEventCoalescer = Effect.fn("makeThreadLiveEventCoales
yield* Fiber.interrupt(fiber);
});
- const flushPending = Effect.fn("ThreadLiveEventCoalescer.flushPending")(function* (
- boundary?: OrchestrationEvent,
- ) {
- const events = boundary ? [...pendingUpdates, boundary] : pendingUpdates;
- pendingUpdates = [];
- if (events.length === 0) {
+ const flushPending = Effect.fn("ThreadLiveEventCoalescer.flushPending")(function* () {
+ if (pendingUpdates.length === 0) {
return;
}
- yield* Queue.offerAll(
- output,
- coalesceLiveToolUpdatedEvents(events).map((event) => ({
+ const items = yield* budget.replace(
+ pendingUpdates,
+ coalesceLiveToolUpdatedEvents(pendingUpdates.map((item) => item.value)).map((event) => ({
kind: "event" as const,
event: projectActivityEvent(event),
})),
+ (item) => item.event,
);
- });
+ pendingUpdates = [];
+ yield* Queue.offerAll(output, items);
+ }, Effect.uninterruptible);
const flushWindow = (generation: number) =>
Effect.sleep(coalesceWindow).pipe(
@@ -141,63 +152,89 @@ export const makeThreadLiveEventCoalescer = Effect.fn("makeThreadLiveEventCoales
}
}),
),
+ Effect.catchTags({ OrchestrationGetSnapshotError: () => Effect.void }),
);
- const process = Effect.fn("ThreadLiveEventCoalescer.process")(function* (
- input: ThreadLiveInput,
+ // Keep each source batch together so a synchronization marker cannot pass
+ // events already pulled from PubSub but still being coalesced.
+ const offerAll = Effect.fn("ThreadLiveEventCoalescer.offerAll")(function* (
+ inputs: ReadonlyArray<ThreadLiveInput>,
) {
yield* mutex.withPermits(1)(
- Effect.gen(function* () {
- if (input.kind === "event" && isToolUpdated(input.event)) {
- pendingUpdates.push(input.event);
- if (pendingUpdates.length === 1) {
- const generation = ++windowGeneration;
- windowFiber = yield* Effect.forkScoped(flushWindow(generation));
- }
- if (pendingUpdates.length >= MAX_PENDING_UPDATES) {
+ Effect.forEach(
+ inputs,
+ (input) =>
+ Effect.gen(function* () {
+ yield* budget.check;
+ if (input.kind === "event") {
+ yield* budget.retain(input.event).pipe(
+ Effect.tap((item) => Effect.sync(() => pendingUpdates.push(item))),
+ Effect.uninterruptible,
+ );
+ }
+ if (input.kind === "event" && isToolUpdated(input.event)) {
+ if (pendingUpdates.length === 1) {
+ const generation = ++windowGeneration;
+ windowFiber = yield* Effect.forkIn(flushWindow(generation), coalescerScope);
+ }
+ if (pendingUpdates.length >= MAX_PENDING_UPDATES) {
+ yield* cancelWindow();
+ windowGeneration += 1;
+ yield* flushPending();
+ }
+ return;
+ }
+
yield* cancelWindow();
windowGeneration += 1;
+ // A non-update event closes the run immediately. The coalescer keeps
+ // that boundary after the final update from the run.
yield* flushPending();
- }
+ if (input.kind === "synchronized") {
+ yield* budget.retain({ kind: "synchronized" as const }).pipe(
+ Effect.flatMap((marker) => Queue.offer(output, marker)),
+ Effect.uninterruptible,
+ );
+ }
+ }),
+ { discard: true },
+ ),
+ );
+ });
+
+ const close = (error?: OrchestrationGetSnapshotError) =>
+ mutex.withPermits(1)(
+ Effect.gen(function* () {
+ if (closed) {
return;
}
-
- yield* cancelWindow();
+ closed = true;
windowGeneration += 1;
- if (input.kind === "event") {
- yield* flushPending(input.event);
- } else {
- yield* flushPending();
- yield* Queue.offer(output, { kind: "synchronized" });
+ yield* cancelWindow();
+ budget.release(pendingUpdates);
+ pendingUpdates = [];
+ budget.release(yield* Queue.clear(output).pipe(Effect.orDie));
+ if (error) {
+ yield* Queue.fail(output, error);
}
+ yield* Queue.shutdown(output);
+ yield* Deferred.succeed(cleanupComplete, undefined);
}),
);
- });
- yield* Stream.fromQueue(input).pipe(
- Stream.runForEach(({ value, processed }) =>
- process(value).pipe(
- Effect.andThen(processed ? Deferred.succeed(processed, undefined) : Effect.void),
- ),
- ),
+ yield* Effect.addFinalizer(() => close());
+ yield* budget.failed.pipe(
+ Effect.catchTags({ OrchestrationGetSnapshotError: close }),
Effect.forkScoped,
);
- const offer = (value: ThreadLiveInput) => Queue.offer(input, { value }).pipe(Effect.asVoid);
-
- const offerAndWait = Effect.fn("ThreadLiveEventCoalescer.offerAndWait")(function* (
- value: ThreadLiveInput,
- ) {
- const processed = yield* Deferred.make<void>();
- yield* Queue.offer(input, { value, processed });
- yield* Deferred.await(processed);
- });
-
return {
- offer,
- offerAndWait,
- stream: Stream.fromQueue(output),
- takeAll: Queue.takeAll(output),
+ offer: (input: ThreadLiveInput) => offerAll([input]),
+ offerAll,
+ stream: budget.deliver(Stream.fromQueue(output)),
+ failed: budget.failed,
+ closed: Deferred.await(cleanupComplete),
+ usage: budget.usage,
} as const;
},
);
diff --git a/apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts b/apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts
index 2bac5de92..ea0e645b0 100644
--- a/apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts
+++ b/apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts
@@ -1,4 +1,13 @@
-import { CommandId, EventId, ProjectId } from "@t3tools/contracts";
+import * as NodeV8 from "node:v8";
+
+import {
+ CommandId,
+ EventId,
+ MessageId,
+ ProjectId,
+ ThreadId,
+ type OrchestrationEvent,
+} from "@t3tools/contracts";
import { assert, it } from "@effect/vitest";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
@@ -12,6 +21,31 @@ import { OrchestrationEventStoreLive } from "./OrchestrationEventStore.ts";
import { SqlitePersistenceMemory } from "./Sqlite.ts";
const isPersistenceDecodeError = Schema.is(PersistenceDecodeError);
+function messageEvent(threadId: ThreadId, id: string): Omit<OrchestrationEvent, "sequence"> {
+ const now = "2026-01-01T00:00:00.000Z";
+ return {
+ type: "thread.message-sent",
+ eventId: EventId.make(id),
+ aggregateKind: "thread",
+ aggregateId: threadId,
+ occurredAt: now,
+ commandId: null,
+ causationEventId: null,
+ correlationId: null,
+ metadata: {},
+ payload: {
+ threadId,
+ messageId: MessageId.make(id),
+ role: "assistant",
+ text: id,
+ turnId: null,
+ streaming: false,
+ createdAt: now,
+ updatedAt: now,
+ },
+ };
+}
+
const layer = it.layer(
OrchestrationEventStoreLive.pipe(Layer.provideMerge(SqlitePersistenceMemory)),
);
@@ -75,7 +109,7 @@ layer("OrchestrationEventStore", (it) => {
const sql = yield* SqlClient.SqlClient;
const now = "2026-01-01T00:00:00.000Z";
- yield* sql`
+ const invalidRows = yield* sql<{ readonly sequence: number }>`
INSERT INTO orchestration_events (
event_id,
aggregate_kind,
@@ -104,6 +138,7 @@ layer("OrchestrationEventStore", (it) => {
${"{"},
${"{}"}
)
+ RETURNING sequence
`;
const replayResult = yield* Effect.result(
@@ -118,6 +153,232 @@ layer("OrchestrationEventStore", (it) => {
),
);
}
+ const scopedResult = yield* eventStore
+ .readAggregateRange({
+ aggregateKind: "project",
+ aggregateId: "project-invalid-json",
+ fromSequenceExclusive: 0,
+ toSequenceInclusive: invalidRows[0]!.sequence,
+ })
+ .pipe(Stream.runCollect, Effect.result);
+ assert.equal(scopedResult._tag, "Failure");
+ if (scopedResult._tag === "Failure") {
+ assert.ok(isPersistenceDecodeError(scopedResult.failure));
+ assert.ok(
+ scopedResult.failure.operation.includes(
+ "OrchestrationEventStore.readAggregateRange:decodeRows",
+ ),
+ );
+ }
+ }),
+ );
+
+ it.effect("bounds global replay to a captured head across sequence gaps", () =>
+ Effect.gen(function* () {
+ const store = yield* OrchestrationEventStore;
+ const sql = yield* SqlClient.SqlClient;
+ const threadId = ThreadId.make("bounded-global-replay");
+ const first = yield* store.append(messageEvent(threadId, "bounded-first"));
+ const pruned = yield* store.append(messageEvent(threadId, "bounded-pruned"));
+ const last = yield* store.append(messageEvent(threadId, "bounded-last"));
+ yield* sql`DELETE FROM orchestration_events WHERE sequence = ${pruned.sequence}`;
+ yield* sql`
+ INSERT INTO orchestration_events (
+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,
+ actor_kind, payload_json, metadata_json
+ ) VALUES (
+ 'bounded-after-captured-head', 'thread', ${threadId}, 3, 'thread.message-sent',
+ '2026-01-01T00:00:00.000Z', 'provider', '{', '{}'
+ )
+ `;
+
+ const events = yield* store
+ .readFromSequence(first.sequence - 1, last.sequence - first.sequence + 1, last.sequence)
+ .pipe(Stream.runCollect);
+ assert.deepEqual(
+ events.map((event) => event.sequence),
+ [first.sequence, last.sequence],
+ );
+ }),
+ );
+
+ it.effect("reads one aggregate through the captured head across pruned global gaps", () =>
+ Effect.gen(function* () {
+ const store = yield* OrchestrationEventStore;
+ const sql = yield* SqlClient.SqlClient;
+ const threadId = ThreadId.make("shared-stream-id");
+ const first = yield* store.append(messageEvent(threadId, "scoped-first"));
+ const pruned = yield* store.append(
+ messageEvent(ThreadId.make("pruned-thread"), "pruned-event"),
+ );
+ const second = yield* store.append(messageEvent(threadId, "scoped-second"));
+ // The same stream ID in a different aggregate is not part of this thread.
+ // Its invalid JSON must never reach the event decoder.
+ yield* sql`
+ INSERT INTO orchestration_events (
+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,
+ actor_kind, payload_json, metadata_json
+ ) VALUES (
+ 'same-id-project', 'project', ${threadId}, 0, 'project.created',
+ '2026-01-01T00:00:00.000Z', 'server', '{', '{'
+ ), (
+ 'unrelated-invalid', 'thread', 'unrelated-invalid-thread', 0, 'thread.activity-appended',
+ '2026-01-01T00:00:00.000Z', 'server', '{', '{'
+ )
+ `;
+ const last = yield* store.append(messageEvent(threadId, "scoped-last"));
+ yield* sql`DELETE FROM orchestration_events WHERE sequence = ${pruned.sequence}`;
+ yield* store.append(messageEvent(threadId, "after-captured-head"));
+
+ const events = yield* store
+ .readAggregateRange({
+ aggregateKind: "thread",
+ aggregateId: threadId,
+ fromSequenceExclusive: first.sequence,
+ toSequenceInclusive: last.sequence,
+ limit: 100,
+ })
+ .pipe(Stream.runCollect);
+ assert.deepEqual(
+ events.map((event) => event.sequence),
+ [second.sequence, last.sequence],
+ );
+ }),
+ );
+
+ it.effect("bounds thread replay metadata and counts UTF-8 bytes without decoding payloads", () =>
+ Effect.gen(function* () {
+ const store = yield* OrchestrationEventStore;
+ const sql = yield* SqlClient.SqlClient;
+ const rows = yield* sql<{ readonly sequence: number }>`
+ INSERT INTO orchestration_events (
+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,
+ actor_kind, payload_json, metadata_json
+ ) VALUES
+ ('stats-1', 'thread', 'stats-thread', 0, 'thread.message-sent',
+ '2026-01-01T00:00:00.000Z', 'provider', '{"output":"😀"}', '{}'),
+ ('stats-unrelated', 'thread', 'another-thread', 0, 'thread.created',
+ '2026-01-01T00:00:00.000Z', 'provider', printf('%.*c', 10000, 'x'), '{}'),
+ ('stats-2', 'thread', 'stats-thread', 1, 'thread.activity-appended',
+ '2026-01-01T00:00:00.000Z', 'provider', '{', '{}'),
+ ('stats-other-kind', 'project', 'stats-thread', 0, 'project.deleted',
+ '2026-01-01T00:00:00.000Z', 'provider', printf('%.*c', 20000, 'x'), '{}'),
+ ('stats-3', 'thread', 'stats-thread', 2, 'thread.deleted',
+ '2026-01-01T00:00:00.000Z', 'provider', '{"output":"é"}', '{}'),
+ ('stats-4', 'thread', 'stats-thread', 3, 'thread.created',
+ '2026-01-01T00:00:00.000Z', 'provider', printf('%.*c', 2000, 'x'), '{}')
+ RETURNING sequence
+ `;
+ const range = {
+ aggregateKind: "thread" as const,
+ aggregateId: "stats-thread",
+ fromSequenceExclusive: 0,
+ toSequenceInclusive: rows.at(-1)!.sequence,
+ };
+ assert.deepEqual(yield* store.getAggregateReplayStats({ ...range, maxEvents: 2 }), {
+ eventCount: 3,
+ payloadBytes: 33,
+ hasCreateEvent: false,
+ });
+ assert.deepEqual(yield* store.getAggregateReplayStats({ ...range, maxEvents: 10 }), {
+ eventCount: 4,
+ payloadBytes: 2033,
+ hasCreateEvent: true,
+ });
+ assert.deepEqual(
+ yield* store.getAggregateReplayStats({
+ ...range,
+ toSequenceInclusive: rows[2]!.sequence,
+ maxEvents: 10,
+ }),
+ {
+ eventCount: 2,
+ payloadBytes: 18,
+ hasCreateEvent: false,
+ },
+ );
+ }),
+ );
+
+ it.effect("keeps later pages below the captured head when new events are appended", () =>
+ Effect.gen(function* () {
+ const store = yield* OrchestrationEventStore;
+ const threadId = ThreadId.make("paged-thread");
+ const persisted = yield* Effect.forEach(
+ Array.from({ length: 502 }, (_, index) => index),
+ (index) => store.append(messageEvent(threadId, `paged-${index}`)),
+ );
+ const head = persisted.at(-1)!.sequence;
+ let appendedDuringReplay = false;
+ const replayed = yield* store
+ .readAggregateRange({
+ aggregateKind: "thread",
+ aggregateId: threadId,
+ fromSequenceExclusive: 0,
+ toSequenceInclusive: head,
+ limit: 1_000,
+ })
+ .pipe(
+ Stream.tap(() => {
+ if (appendedDuringReplay) return Effect.void;
+ appendedDuringReplay = true;
+ return store.append(messageEvent(threadId, "appended-during-replay"));
+ }),
+ Stream.runCollect,
+ );
+ assert.deepEqual(
+ replayed.map((event) => event.sequence),
+ persisted.map((event) => event.sequence),
+ );
+ const limited = store.readFromSequence(persisted[0]!.sequence, 501.9);
+ for (let run = 0; run < 2; run++) {
+ assert.deepEqual(
+ (yield* Stream.runCollect(limited)).map((event) => event.sequence),
+ persisted.slice(1).map((event) => event.sequence),
+ );
+ }
+ assert.deepEqual(yield* Stream.runCollect(store.readFromSequence(0, -1)), []);
}),
);
});
+
+for (const reader of ["all", "aggregate"] as const) {
+ it.effect(`releases consumed pages during ${reader} replay`, () =>
+ Effect.gen(function* () {
+ const store = yield* OrchestrationEventStore;
+ const threadId = ThreadId.make(`retention-${reader}`);
+ yield* Effect.forEach(
+ Array.from({ length: 1_501 }, (_, index) => index),
+ (index) => store.append(messageEvent(threadId, `retention-${reader}-${index}`)),
+ { discard: true },
+ );
+ // oxlint-disable-next-line typescript/no-extraneous-class -- Identifies page markers for V8's heap query.
+ class ReplayPage {}
+ let count = 0;
+ const replay =
+ reader === "all"
+ ? store.readAll()
+ : store.readAggregateRange({
+ aggregateKind: "thread",
+ aggregateId: threadId,
+ fromSequenceExclusive: 0,
+ toSequenceInclusive: 1_501,
+ limit: 1_501,
+ });
+ yield* Stream.runForEach(replay, (event) =>
+ Effect.sync(() => {
+ assert.equal(event.sequence, count + 1);
+ if (count % 500 === 0) {
+ // Count live page markers after full GC, without timing or heap-size thresholds.
+ Object.assign(event, { replayPage: new ReplayPage() });
+ assert.isAtMost(NodeV8.queryObjects(ReplayPage, { format: "count" }), 1);
+ }
+ count++;
+ }),
+ );
+ assert.equal(count, 1_501);
+ }).pipe(
+ Effect.provide(OrchestrationEventStoreLive.pipe(Layer.provide(SqlitePersistenceMemory))),
+ ),
+ );
+}
diff --git a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts b/apps/server/src/persistence/Layers/OrchestrationEventStore.ts
index e6b9c1444..1dc5d8d9f 100644
--- a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts
+++ b/apps/server/src/persistence/Layers/OrchestrationEventStore.ts
@@ -97,8 +97,21 @@ const OrchestrationEventPersistedRowSchema = Schema.Struct({
const ReadFromSequenceRequestSchema = Schema.Struct({
sequenceExclusive: NonNegativeInt,
+ toSequenceInclusive: NonNegativeInt,
limit: Schema.Number,
});
+const AggregateReplayRequestSchema = Schema.Struct({
+ aggregateKind: OrchestrationAggregateKind,
+ aggregateId: Schema.String,
+ fromSequenceExclusive: NonNegativeInt,
+ toSequenceInclusive: NonNegativeInt,
+ limit: Schema.Number,
+});
+const AggregateReplayStatsRowSchema = Schema.Struct({
+ eventCount: Schema.Number,
+ payloadBytes: Schema.Number,
+ hasCreateEvent: Schema.Number,
+});
const DEFAULT_READ_FROM_SEQUENCE_LIMIT = 1_000;
const READ_PAGE_SIZE = 500;
@@ -211,11 +224,63 @@ const makeEventStore = Effect.gen(function* () {
metadata_json AS "metadata"
FROM orchestration_events
WHERE sequence > ${request.sequenceExclusive}
+ AND sequence <= ${request.toSequenceInclusive}
ORDER BY sequence ASC
LIMIT ${request.limit}
`,
});
+ const readAggregateEventRows = SqlSchema.findAll({
+ Request: AggregateReplayRequestSchema,
+ Result: OrchestrationEventPersistedRowSchema,
+ execute: (request) =>
+ sql`
+ SELECT
+ sequence,
+ event_id AS "eventId",
+ event_type AS "type",
+ aggregate_kind AS "aggregateKind",
+ stream_id AS "aggregateId",
+ occurred_at AS "occurredAt",
+ command_id AS "commandId",
+ causation_event_id AS "causationEventId",
+ correlation_id AS "correlationId",
+ payload_json AS "payload",
+ metadata_json AS "metadata"
+ FROM orchestration_events
+ WHERE aggregate_kind = ${request.aggregateKind}
+ AND stream_id = ${request.aggregateId}
+ AND sequence > ${request.fromSequenceExclusive}
+ AND sequence <= ${request.toSequenceInclusive}
+ ORDER BY sequence ASC
+ LIMIT ${request.limit}
+ `,
+ });
+
+ const readAggregateReplayStats = SqlSchema.findOne({
+ Request: AggregateReplayRequestSchema,
+ Result: AggregateReplayStatsRowSchema,
+ execute: (request) =>
+ sql`
+ SELECT
+ COUNT(*) AS "eventCount",
+ COALESCE(SUM(octet_length(payload_json)), 0) AS "payloadBytes",
+ COALESCE(MAX(event_type IN (
+ 'thread.created', 'project.created'
+ )), 0) AS "hasCreateEvent"
+ FROM (
+ SELECT payload_json, event_type
+ FROM orchestration_events
+ WHERE aggregate_kind = ${request.aggregateKind}
+ AND stream_id = ${request.aggregateId}
+ AND sequence > ${request.fromSequenceExclusive}
+ AND sequence <= ${request.toSequenceInclusive}
+ ORDER BY sequence ASC
+ LIMIT ${request.limit}
+ )
+ `,
+ });
+
const append: OrchestrationEventStoreShape["append"] = (event) =>
appendEventRow({
eventId: event.eventId,
@@ -246,18 +311,18 @@ const makeEventStore = Effect.gen(function* () {
const readFromSequence: OrchestrationEventStoreShape["readFromSequence"] = (
sequenceExclusive,
limit = DEFAULT_READ_FROM_SEQUENCE_LIMIT,
+ toSequenceInclusive = Number.MAX_SAFE_INTEGER,
) => {
const normalizedLimit = Math.max(0, Math.floor(limit));
- if (normalizedLimit === 0) {
+ if (normalizedLimit === 0 || sequenceExclusive >= toSequenceInclusive) {
return Stream.empty;
}
- const readPage = (
- cursor: number,
- remaining: number,
- ): Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError> =>
- Stream.fromEffect(
+ return Stream.paginate(
+ { cursor: sequenceExclusive, remaining: normalizedLimit },
+ ({ cursor, remaining }) =>
readEventRowsFromSequence({
sequenceExclusive: cursor,
+ toSequenceInclusive,
limit: Math.min(remaining, READ_PAGE_SIZE),
}).pipe(
Effect.mapError(
@@ -275,26 +340,81 @@ const makeEventStore = Effect.gen(function* () {
),
),
),
+ Effect.map((events) => {
+ const last = events.at(-1);
+ const nextRemaining = remaining - events.length;
+ return [
+ events,
+ last === undefined || nextRemaining <= 0
+ ? Option.none()
+ : Option.some({ cursor: last.sequence, remaining: nextRemaining }),
+ ] as const;
+ }),
),
- ).pipe(
- Stream.flatMap((events) => {
- if (events.length === 0) {
- return Stream.empty;
- }
- const nextRemaining = remaining - events.length;
- if (nextRemaining <= 0) {
- return Stream.fromIterable(events);
- }
- return Stream.concat(
- Stream.fromIterable(events),
- readPage(events[events.length - 1]!.sequence, nextRemaining),
- );
- }),
- );
+ );
+ };
- return readPage(sequenceExclusive, normalizedLimit);
+ const readAggregateRange: OrchestrationEventStoreShape["readAggregateRange"] = (input) => {
+ const limit = Math.max(0, Math.floor(input.limit ?? DEFAULT_READ_FROM_SEQUENCE_LIMIT));
+ if (limit === 0 || input.fromSequenceExclusive >= input.toSequenceInclusive) {
+ return Stream.empty;
+ }
+ return Stream.paginate(
+ { cursor: input.fromSequenceExclusive, remaining: limit },
+ ({ cursor, remaining }) =>
+ readAggregateEventRows({
+ ...input,
+ fromSequenceExclusive: cursor,
+ limit: Math.min(remaining, READ_PAGE_SIZE),
+ }).pipe(
+ Effect.mapError(
+ toPersistenceSqlOrDecodeError(
+ "OrchestrationEventStore.readAggregateRange:query",
+ "OrchestrationEventStore.readAggregateRange:decodeRows",
+ ),
+ ),
+ Effect.flatMap((rows) =>
+ Effect.forEach(rows, (row) =>
+ decodeEvent(row).pipe(
+ Effect.mapError(
+ toPersistenceDecodeError("OrchestrationEventStore.readAggregateRange:rowToEvent"),
+ ),
+ ),
+ ),
+ ),
+ Effect.map((events) => {
+ const last = events.at(-1);
+ const nextRemaining = remaining - events.length;
+ return [
+ events,
+ last === undefined ||
+ events.length < READ_PAGE_SIZE ||
+ nextRemaining === 0 ||
+ last.sequence >= input.toSequenceInclusive
+ ? Option.none()
+ : Option.some({ cursor: last.sequence, remaining: nextRemaining }),
+ ] as const;
+ }),
+ ),
+ );
};
+ const getAggregateReplayStats: OrchestrationEventStoreShape["getAggregateReplayStats"] = (
+ input,
+ ) =>
+ readAggregateReplayStats({
+ ...input,
+ limit: Math.max(0, Math.floor(input.maxEvents)) + 1,
+ }).pipe(
+ Effect.mapError(
+ toPersistenceSqlOrDecodeError(
+ "OrchestrationEventStore.getAggregateReplayStats:query",
+ "OrchestrationEventStore.getAggregateReplayStats:decodeRow",
+ ),
+ ),
+ Effect.map((row) => ({ ...row, hasCreateEvent: row.hasCreateEvent !== 0 })),
+ );
+
const findEventAfter = SqlSchema.findOneOption({
Request: HasEventAfterRequestSchema,
Result: Schema.Struct({ sequence: Schema.Number }),
@@ -324,6 +444,8 @@ const makeEventStore = Effect.gen(function* () {
return {
append,
readFromSequence,
+ readAggregateRange,
+ getAggregateReplayStats,
readAll: () => readFromSequence(0, Number.MAX_SAFE_INTEGER),
hasEventAfter,
} satisfies OrchestrationEventStoreShape;
diff --git a/apps/server/src/persistence/Services/OrchestrationEventStore.ts b/apps/server/src/persistence/Services/OrchestrationEventStore.ts
index 49370b2bb..1ef12acea 100644
--- a/apps/server/src/persistence/Services/OrchestrationEventStore.ts
+++ b/apps/server/src/persistence/Services/OrchestrationEventStore.ts
@@ -16,6 +16,20 @@ import type * as Stream from "effect/Stream";
import type { OrchestrationEventStoreError } from "../Errors.ts";
+export interface OrchestrationAggregateReplayRange {
+ readonly aggregateKind: OrchestrationEvent["aggregateKind"];
+ readonly aggregateId: string;
+ readonly fromSequenceExclusive: number;
+ readonly toSequenceInclusive: number;
+}
+
+export interface OrchestrationAggregateReplayStats {
+ readonly eventCount: number;
+ readonly payloadBytes: number;
+ /** A creation in this range does not prove that the aggregate still exists. */
+ readonly hasCreateEvent: boolean;
+}
+
/**
* OrchestrationEventStoreShape - Service API for orchestration event persistence.
*/
@@ -37,6 +51,7 @@ export interface OrchestrationEventStoreShape {
*
* @param sequenceExclusive - Sequence cursor (exclusive).
* @param limit - Maximum number of events to emit.
+ * @param toSequenceInclusive - Optional captured upper sequence boundary.
* @returns Stream containing ordered events.
*
* Reads in fixed-size pages and normalizes non-integer/negative limits.
@@ -44,8 +59,22 @@ export interface OrchestrationEventStoreShape {
readonly readFromSequence: (
sequenceExclusive: number,
limit?: number,
+ toSequenceInclusive?: number,
) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;
+ /** Read one aggregate through a captured global head, without decoding other streams. */
+ readonly readAggregateRange: (
+ input: OrchestrationAggregateReplayRange & { readonly limit?: number },
+ ) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;
+
+ /**
+ * Measure at most maxEvents + 1 rows without decoding payloads. The extra
+ * row tells the caller to use a snapshot instead of a truncated replay.
+ */
+ readonly getAggregateReplayStats: (
+ input: OrchestrationAggregateReplayRange & { readonly maxEvents: number },
+ ) => Effect.Effect<OrchestrationAggregateReplayStats, OrchestrationEventStoreError>;
+
/**
* Read all events from the beginning of the stream.
*
diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.ts
index 0d51e517c..123541f58 100644
--- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts
+++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts
@@ -32,6 +32,7 @@ const makeProjectionSnapshotQueryLayer = (project: OrchestrationProject) =>
getArchivedShellSnapshot: () => Effect.die("unused"),
getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 1 }),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: (workspaceRoot) =>
Effect.succeed(
diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts
index eb9c17ef1..1d75bbe22 100644
--- a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts
+++ b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts
@@ -192,6 +192,7 @@ describe("ProviderSessionReaper", () => {
getSnapshotSequence: () =>
Effect.succeed({ snapshotSequence: input.readModel.snapshotSequence }),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.die("unused"),
getProjectShellById: () => Effect.die("unused"),
diff --git a/apps/server/src/routines/RoutineDraftDispatcher.test.ts b/apps/server/src/routines/RoutineDraftDispatcher.test.ts
index 0af8b05be..4fb6dcb16 100644
--- a/apps/server/src/routines/RoutineDraftDispatcher.test.ts
+++ b/apps/server/src/routines/RoutineDraftDispatcher.test.ts
@@ -163,6 +163,7 @@ function deleteHarness(input: {
searchThreads: () => Effect.succeed({ matches: [] }),
getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 1 }),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getOriginalProjectIdByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.succeed(Option.none()),
@@ -185,6 +186,8 @@ function deleteHarness(input: {
return { sequence: commands.length };
}),
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
streamDomainEvents: Stream.empty,
subscribeDomainEvents: Effect.succeed(Stream.empty),
latestSequence: Effect.succeed(0),
diff --git a/apps/server/src/routines/RuntimeLive.test.ts b/apps/server/src/routines/RuntimeLive.test.ts
index e4bc3efd0..0d870e66f 100644
--- a/apps/server/src/routines/RuntimeLive.test.ts
+++ b/apps/server/src/routines/RuntimeLive.test.ts
@@ -129,6 +129,8 @@ const harness = (
Layer.succeed(OrchestrationEngineService, {
dispatch: () => Effect.die("unused"),
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
streamDomainEvents: domainEvents,
subscribeDomainEvents: Effect.succeed(domainEvents),
latestSequence: Effect.succeed(0),
diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts
index fbb3bd710..475d45b74 100644
--- a/apps/server/src/server.test.ts
+++ b/apps/server/src/server.test.ts
@@ -21,6 +21,7 @@ import {
OrchestrationThreadDetailSnapshot,
type OrchestrationThreadActivity,
type OrchestrationThreadStreamItem,
+ type OrchestrationShellStreamItem,
type OrchestrationThreadShell,
TerminalNotRunningError,
type OrchestrationCommand,
@@ -116,6 +117,8 @@ import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionRe
import { OrchestrationListenerCallbackError } from "./orchestration/Errors.ts";
import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";
import { SqlitePersistenceMemory } from "./persistence/Layers/Sqlite.ts";
+import { OrchestrationEventStoreLive } from "./persistence/Layers/OrchestrationEventStore.ts";
+import { OrchestrationEventStore } from "./persistence/Services/OrchestrationEventStore.ts";
import * as ProjectionBots from "./persistence/Services/ProjectionBots.ts";
import * as ProjectionGroups from "./persistence/Services/ProjectionGroups.ts";
import { PersistenceSqlError } from "./persistence/Errors.ts";
@@ -200,11 +203,12 @@ const defaultModelSelection = {
const makeLiveToolActivityEvent = (
sequence: number,
+ kind: OrchestrationThreadActivity["kind"] = "tool.updated",
): Extract<OrchestrationEvent, { type: "thread.activity-appended" }> => {
const activity: OrchestrationThreadActivity = {
id: EventId.make(`activity-${sequence}`),
tone: "tool",
- kind: "tool.updated",
+ kind,
summary: "Editing app.ts",
payload: {
itemType: "file_change",
@@ -853,6 +857,13 @@ const buildAppUnderTest = (options?: {
Layer.mergeAll(
Layer.mock(OrchestrationEngine.OrchestrationEngineService)({
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () =>
+ Effect.succeed({
+ eventCount: 0,
+ payloadBytes: 0,
+ hasCreateEvent: false,
+ }),
dispatch: () => Effect.succeed({ sequence: 0 }),
streamDomainEvents: Stream.empty,
subscribeDomainEvents: Effect.succeed(Stream.empty),
@@ -958,6 +969,11 @@ const buildAppUnderTest = (options?: {
getThreadDetailById: () => Effect.succeed(Option.none()),
getThreadDetailSnapshot: () => Effect.succeed(Option.none()),
getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),
+ getEventReplayStats: ({ fromSequenceExclusive, toSequenceInclusive }) =>
+ Effect.succeed({
+ eventCount: Math.max(0, toSequenceInclusive - fromSequenceExclusive),
+ payloadBytes: 0,
+ }),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()),
getThreadCheckpointContext: () => Effect.succeed(Option.none()),
@@ -1142,6 +1158,32 @@ const withWsRpcClient = <A, E, R>(
f: (client: WsRpcClient) => Effect.Effect<A, E, R>,
) => makeWsRpcClient.pipe(Effect.flatMap(f), Effect.provide(wsRpcProtocolLayer(wsUrl)));
+const withFirstWsAckHeld = (
+ wsUrl: string,
+ held: Deferred.Deferred<void>,
+ release: Deferred.Deferred<void>,
+) => {
+ let holdNextAck = true;
+ return Layer.effect(RpcClient.Protocol)(
+ Effect.map(RpcClient.Protocol, (protocol) =>
+ RpcClient.Protocol.of({
+ ...protocol,
+ send: (clientId, request, transferables) => {
+ const send = protocol.send(clientId, request, transferables);
+ if (request._tag !== "Ack" || !holdNextAck) {
+ return send;
+ }
+ holdNextAck = false;
+ return Deferred.succeed(held, undefined).pipe(
+ Effect.andThen(Deferred.await(release)),
+ Effect.andThen(send),
+ );
+ },
+ }),
+ ),
+ ).pipe(Layer.provide(wsRpcProtocolLayer(wsUrl)));
+};
+
const appendSessionCookieToWsUrl = (url: string, sessionCookieHeader: string) => {
const isAbsoluteUrl = /^[a-zA-Z][a-zA-Z\d+.-]*:/.test(url);
const next = new URL(url, "http://localhost");
@@ -4830,6 +4872,75 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
);
+ it.effect("subscribeShell keeps sparse replay below the captured head", () =>
+ Effect.gen(function* () {
+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();
+ const now = "2026-01-01T00:00:00.000Z";
+ const replayProjectId = ProjectId.make("project-replayed");
+ const liveProjectId = ProjectId.make("project-live");
+ const projectDeleted = (sequence: number, projectId: ProjectId): OrchestrationEvent => ({
+ sequence,
+ eventId: EventId.make(`event-project-deleted-${sequence}`),
+ aggregateKind: "project",
+ aggregateId: projectId,
+ occurredAt: now,
+ commandId: null,
+ causationEventId: null,
+ correlationId: null,
+ metadata: {},
+ type: "project.deleted",
+ payload: { projectId, deletedAt: now },
+ });
+ const replayed = projectDeleted(1, replayProjectId);
+ const newer = projectDeleted(6, liveProjectId);
+ let replayHead: number | undefined;
+
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.succeed(5),
+ streamDomainEvents: Stream.fromPubSub(liveEvents),
+ readEvents: (_afterSequence, _limit, toSequenceInclusive) => {
+ replayHead = toSequenceInclusive;
+ return Stream.fromEffect(PubSub.publish(liveEvents, newer)).pipe(
+ Stream.flatMap(() =>
+ Stream.fromIterable(
+ [replayed, newer].filter(
+ (event) =>
+ toSequenceInclusive === undefined || event.sequence <= toSequenceInclusive,
+ ),
+ ),
+ ),
+ );
+ },
+ },
+ projectionSnapshotQuery: {
+ getEventReplayStats: () => Effect.succeed({ eventCount: 1, payloadBytes: 100 }),
+ },
+ },
+ });
+
+ const wsUrl = yield* getWsServerUrl("/ws");
+ const items = yield* Effect.scoped(
+ withWsRpcClient(wsUrl, (client) =>
+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({
+ afterSequence: 0,
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ ),
+ ),
+ );
+
+ assert.deepEqual(
+ items.map((item) => ("sequence" in item ? item.sequence : item.kind)),
+ [1, 6, "synchronized"],
+ );
+ assert.equal(replayHead, 5);
+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),
+ );
+
it.effect("maps delegation lifecycle events into shell upserts", () =>
Effect.gen(function* () {
const now = "2026-08-31T00:00:00.000Z";
@@ -5049,7 +5160,6 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
projectionSnapshotQuery: {
getThreadDetailSnapshot: () =>
Effect.gen(function* () {
- yield* Effect.sleep("25 millis");
yield* PubSub.publish(liveEvents, messageEvent);
return Option.some({ snapshotSequence: 1, thread });
}),
@@ -5062,14 +5172,19 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
withWsRpcClient(wsUrl, (client) =>
client[ORCHESTRATION_WS_METHODS.subscribeThread]({
threadId: defaultThreadId,
- }).pipe(Stream.take(2), Stream.runCollect),
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ ),
),
- ).pipe(Effect.timeout("2 seconds"));
+ );
assert.equal(items[0]?.kind, "snapshot");
assert.equal(items[1]?.kind, "event");
assert.equal(items[1]?.kind === "event" ? items[1].event.sequence : null, 2);
- }).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive),
+ assert.equal(items[2]?.kind, "synchronized");
+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),
);
it.effect("coalesces buffered live tool updates before websocket delivery", () =>
@@ -5113,50 +5228,57 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
}).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive),
);
- it.effect("subscribeThread sends a fresh snapshot instead of replaying a large gap", () =>
- Effect.gen(function* () {
- let readEventsCalls = 0;
- const thread = makeDefaultOrchestrationReadModel().threads[0]!;
+ it.effect(
+ "subscribeThread sends a fresh snapshot when its event count exceeds the replay limit",
+ () =>
+ Effect.gen(function* () {
+ let readEventsCalls = 0;
+ const thread = makeDefaultOrchestrationReadModel().threads[0]!;
- yield* buildAppUnderTest({
- layers: {
- orchestrationEngine: {
- // Head is far ahead of the client's afterSequence (gap > 1000).
- latestSequence: Effect.succeed(100_000),
- readEvents: () =>
- Stream.sync(() => {
- readEventsCalls += 1;
- return {} as OrchestrationEvent;
- }),
- },
- projectionSnapshotQuery: {
- getThreadDetailSnapshot: () =>
- Effect.succeed(Option.some({ snapshotSequence: 100_000, thread })),
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.succeed(100_000),
+ getThreadReplayStats: () =>
+ Effect.succeed({
+ eventCount: 1_001,
+ payloadBytes: 1_000,
+ hasCreateEvent: false,
+ }),
+ readThreadEvents: () =>
+ Stream.sync(() => {
+ readEventsCalls += 1;
+ return {} as OrchestrationEvent;
+ }),
+ },
+ projectionSnapshotQuery: {
+ getThreadDetailSnapshot: () =>
+ Effect.succeed(Option.some({ snapshotSequence: 100_000, thread })),
+ },
},
- },
- });
+ });
- const wsUrl = yield* getWsServerUrl("/ws");
- const items = yield* Effect.scoped(
- withWsRpcClient(wsUrl, (client) =>
- client[ORCHESTRATION_WS_METHODS.subscribeThread]({
- threadId: defaultThreadId,
- afterSequence: 5,
- requestCompletionMarker: true,
- }).pipe(Stream.take(2), Stream.runCollect),
- ),
- );
+ const wsUrl = yield* getWsServerUrl("/ws");
+ const items = yield* Effect.scoped(
+ withWsRpcClient(wsUrl, (client) =>
+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({
+ threadId: defaultThreadId,
+ afterSequence: 5,
+ requestCompletionMarker: true,
+ }).pipe(Stream.take(2), Stream.runCollect),
+ ),
+ );
- const [first, second] = Array.from(items);
- // Large gap => fresh thread snapshot, and the global replay never starts.
- assert.equal(first?.kind, "snapshot");
- if (first?.kind === "snapshot") {
- assert.equal(first.snapshot.thread.id, defaultThreadId);
- assert.equal(first.snapshot.snapshotSequence, 100_000);
- }
- assert.equal(second?.kind, "synchronized");
- assert.equal(readEventsCalls, 0);
- }).pipe(Effect.provide(NodeHttpServer.layerTest)),
+ const [first, second] = Array.from(items);
+ // Never truncate a thread's replay at the event limit.
+ assert.equal(first?.kind, "snapshot");
+ if (first?.kind === "snapshot") {
+ assert.equal(first.snapshot.thread.id, defaultThreadId);
+ assert.equal(first.snapshot.snapshotSequence, 100_000);
+ }
+ assert.equal(second?.kind, "synchronized");
+ assert.equal(readEventsCalls, 0);
+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),
);
it.effect("subscribeThread replaces a cursor ahead of the authoritative head", () =>
@@ -5168,7 +5290,9 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
layers: {
orchestrationEngine: {
latestSequence: Effect.succeed(5),
- readEvents: () =>
+ getThreadReplayStats: () =>
+ Effect.die("An invalid cursor must not start a replay query"),
+ readThreadEvents: () =>
Stream.sync(() => {
readEventsCalls += 1;
return {} as OrchestrationEvent;
@@ -5198,7 +5322,9 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
it.effect("subscribeThread bounds catch-up replay to the captured head", () =>
Effect.gen(function* () {
- let replayLimit: number | undefined;
+ let replayHead: number | undefined;
+ let headSequence = 50;
+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();
const now = "2026-01-01T00:00:00.000Z";
const messageEvent = {
sequence: 3,
@@ -5226,10 +5352,26 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
yield* buildAppUnderTest({
layers: {
orchestrationEngine: {
- latestSequence: Effect.succeed(50),
- readEvents: (_afterSequence, limit) => {
- replayLimit = limit;
- return Stream.make(messageEvent);
+ latestSequence: Effect.sync(() => headSequence),
+ streamDomainEvents: Stream.fromPubSub(liveEvents),
+ getThreadReplayStats: () =>
+ Effect.sync(() => {
+ headSequence = 100;
+ return { eventCount: 1, payloadBytes: 100, hasCreateEvent: false };
+ }),
+ readThreadEvents: ({ toSequenceInclusive }) => {
+ replayHead = toSequenceInclusive;
+ return Stream.fromEffect(
+ PubSub.publish(liveEvents, {
+ ...messageEvent,
+ sequence: 51,
+ eventId: EventId.make("event-live-after-head"),
+ payload: {
+ ...messageEvent.payload,
+ messageId: MessageId.make("message-live-after-head"),
+ },
+ }),
+ ).pipe(Stream.flatMap(() => Stream.make(messageEvent)));
},
},
},
@@ -5242,20 +5384,590 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
threadId: defaultThreadId,
afterSequence: 0,
requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ ),
+ ),
+ );
+
+ assert.deepEqual(
+ items.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),
+ [3, 51, "synchronized"],
+ );
+ assert.equal(replayHead, 50);
+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),
+ );
+
+ it.effect(
+ "stops an overflowing thread producer without an ACK and replays the missing events",
+ () =>
+ Effect.gen(function* () {
+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();
+ const attached = yield* Deferred.make<void>();
+ const detached = yield* Deferred.make<void>();
+ const ackHeld = yield* Deferred.make<void>();
+ const releaseAck = yield* Deferred.make<void>();
+ const firstApplied = yield* Deferred.make<void>();
+ const replayStarted = yield* Deferred.make<void>();
+ const replayCalls: Array<{ afterSequence: number; headSequence: number }> = [];
+ let headSequence = 0;
+ let snapshotCalls = 0;
+ const message = (sequence: number, text: string) =>
+ ({
+ sequence,
+ eventId: EventId.make(`slow-thread-${sequence}`),
+ aggregateKind: "thread",
+ aggregateId: defaultThreadId,
+ occurredAt: "2026-01-01T00:00:01.000Z",
+ commandId: null,
+ causationEventId: null,
+ correlationId: null,
+ metadata: {},
+ type: "thread.message-sent",
+ payload: {
+ threadId: defaultThreadId,
+ messageId: MessageId.make(`slow-message-${sequence}`),
+ role: "assistant",
+ text,
+ turnId: TurnId.make("turn-edit"),
+ streaming: false,
+ createdAt: "2026-01-01T00:00:01.000Z",
+ updatedAt: "2026-01-01T00:00:01.000Z",
+ },
+ }) satisfies OrchestrationEvent;
+ const events = [
+ message(1, "a".repeat(4 * 1024 * 1024)),
+ message(2, "b".repeat(4 * 1024 * 1024)),
+ makeLiveToolActivityEvent(3, "tool.completed"),
+ message(4, "Finished"),
+ ] as const;
+
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.sync(() => headSequence),
+ streamDomainEvents: Stream.unwrap(
+ Effect.gen(function* () {
+ const subscription = yield* PubSub.subscribe(liveEvents);
+ yield* Deferred.succeed(attached, undefined);
+ return Stream.fromSubscription(subscription);
+ }),
+ ).pipe(Stream.ensuring(Deferred.succeed(detached, undefined))),
+ readThreadEvents: ({ fromSequenceExclusive, toSequenceInclusive }) => {
+ replayCalls.push({
+ afterSequence: fromSequenceExclusive,
+ headSequence: toSequenceInclusive,
+ });
+ const range = events.filter(
+ (event) =>
+ event.sequence > fromSequenceExclusive && event.sequence <= toSequenceInclusive,
+ );
+ return Stream.concat(
+ Stream.fromEffect(Deferred.succeed(replayStarted, undefined)).pipe(Stream.drain),
+ Stream.fromIterable(range),
+ );
+ },
+ getThreadReplayStats: ({ fromSequenceExclusive, toSequenceInclusive }) => {
+ const range = events.filter(
+ (event) =>
+ event.sequence > fromSequenceExclusive && event.sequence <= toSequenceInclusive,
+ );
+ return Effect.succeed({
+ eventCount: range.length,
+ payloadBytes: range.reduce(
+ (bytes, event) => bytes + Buffer.byteLength(jsonRequestBody(event.payload)),
+ 0,
+ ),
+ hasCreateEvent: false,
+ });
+ },
+ },
+ projectionSnapshotQuery: {
+ getThreadDetailSnapshot: () =>
+ Effect.sync(() => {
+ snapshotCalls += 1;
+ return Option.none();
+ }),
+ },
+ },
+ });
+
+ const wsUrl = yield* getWsServerUrl("/ws");
+ yield* makeWsRpcClient.pipe(
+ Effect.flatMap((client) =>
+ Effect.gen(function* () {
+ let cursor = 0;
+ const received: number[] = [];
+ const attempt = yield* client[ORCHESTRATION_WS_METHODS.subscribeThread]({
+ threadId: defaultThreadId,
+ afterSequence: cursor,
+ }).pipe(
+ Stream.tap((item) => {
+ if (item.kind !== "event") return Effect.void;
+ cursor = item.event.sequence;
+ received.push(cursor);
+ return Deferred.succeed(firstApplied, undefined);
+ }),
+ Stream.runDrain,
+ Effect.result,
+ Effect.forkScoped,
+ );
+ yield* Deferred.await(attached);
+ yield* Deferred.await(replayStarted);
+ headSequence = 1;
+ yield* PubSub.publish(liveEvents, events[0]!);
+ yield* Deferred.await(ackHeld);
+ yield* Deferred.await(firstApplied);
+ headSequence = 4;
+ yield* PubSub.publishAll(liveEvents, events.slice(1));
+
+ // This must finish while the server is still waiting for the first
+ // batch's ACK. A failed output queue alone would leave PubSub live.
+ yield* Deferred.await(detached);
+ assert.equal(yield* PubSub.size(liveEvents), 0);
+ assert.deepEqual(received, [1]);
+ yield* Deferred.succeed(releaseAck, undefined);
+ const result = yield* Fiber.join(attempt);
+ assertTrue(result._tag === "Failure");
+ assert.equal(result.failure._tag, "OrchestrationGetSnapshotError");
+
+ const recovered = yield* client[ORCHESTRATION_WS_METHODS.subscribeThread]({
+ threadId: defaultThreadId,
+ afterSequence: cursor,
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ );
+ assert.deepEqual(
+ recovered.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),
+ [2, 3, 4, "synchronized"],
+ );
+ const first = recovered[0];
+ assertTrue(first?.kind === "event" && first.event.type === "thread.message-sent");
+ assert.equal(first.event.payload.text, events[1]!.payload.text);
+ const completed = recovered[1];
+ assertTrue(
+ completed?.kind === "event" && completed.event.type === "thread.activity-appended",
+ );
+ assert.equal(completed.event.payload.activity.kind, "tool.completed");
+ assert.deepEqual(replayCalls, [
+ { afterSequence: 0, headSequence: 0 },
+ { afterSequence: 1, headSequence: 4 },
+ ]);
+ assert.equal(snapshotCalls, 0);
+ }),
+ ),
+ Effect.provide(withFirstWsAckHeld(wsUrl, ackHeld, releaseAck)),
+ );
+ }).pipe(Effect.scoped, Effect.provide(NodeHttpServer.layerTest)),
+ );
+
+ it.effect("stops an overflowing shell producer without an ACK and recovers deleted entries", () =>
+ Effect.gen(function* () {
+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();
+ const detached = yield* Deferred.make<void>();
+ const ackHeld = yield* Deferred.make<void>();
+ const releaseAck = yield* Deferred.make<void>();
+ let headSequence = 1;
+ let snapshotCalls = 0;
+ let replayCalls = 0;
+ const thread = makeDefaultOrchestrationThreadShell();
+ const project = makeDefaultOrchestrationReadModel().projects[0]!;
+ const events: OrchestrationEvent[] = Array.from({ length: 1_001 }, (_, index) =>
+ makeLiveToolActivityEvent(index + 2, "tool.completed"),
+ );
+ events.push(
+ {
+ ...events[0]!,
+ sequence: 1_003,
+ type: "thread.archived",
+ payload: {
+ threadId: defaultThreadId,
+ archivedAt: "2026-01-01T00:00:02.000Z",
+ updatedAt: "2026-01-01T00:00:02.000Z",
+ },
+ },
+ {
+ ...events[0]!,
+ sequence: 1_004,
+ aggregateKind: "project",
+ aggregateId: defaultProjectId,
+ type: "project.deleted",
+ payload: { projectId: defaultProjectId, deletedAt: "2026-01-01T00:00:02.000Z" },
+ },
+ );
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.sync(() => headSequence),
+ streamDomainEvents: Stream.fromPubSub(liveEvents).pipe(
+ Stream.ensuring(Deferred.succeed(detached, undefined)),
+ ),
+ readEvents: () => {
+ replayCalls += 1;
+ return Stream.empty;
+ },
+ },
+ projectionSnapshotQuery: {
+ getShellSnapshot: () =>
+ Effect.sync(() => {
+ snapshotCalls += 1;
+ return {
+ snapshotSequence: headSequence,
+ bots: [],
+ groups: [],
+ delegations: [],
+ projects: headSequence === 1 ? [project] : [],
+ threads: headSequence === 1 ? [thread] : [],
+ updatedAt: "2026-01-01T00:00:02.000Z",
+ };
+ }),
+ },
+ },
+ });
+ const wsUrl = yield* getWsServerUrl("/ws");
+ yield* makeWsRpcClient.pipe(
+ Effect.flatMap((client) =>
+ Effect.gen(function* () {
+ const received: OrchestrationShellStreamItem[] = [];
+ const attempt = yield* client[ORCHESTRATION_WS_METHODS.subscribeShell]({}).pipe(
+ Stream.tap((item) => Effect.sync(() => received.push(item))),
+ Stream.runDrain,
+ Effect.result,
+ Effect.forkScoped,
+ );
+ yield* Deferred.await(ackHeld);
+ headSequence = 1_004;
+ yield* PubSub.publishAll(liveEvents, events);
+ yield* Deferred.await(detached);
+ assert.equal(yield* PubSub.size(liveEvents), 0);
+ yield* Deferred.succeed(releaseAck, undefined);
+ const result = yield* Fiber.join(attempt);
+ assertTrue(result._tag === "Failure");
+ assert.equal(result.failure._tag, "OrchestrationGetSnapshotError");
+ assert.equal(received.length, 1);
+ const initial = received[0];
+ assertTrue(initial?.kind === "snapshot");
+ assert.equal(initial.snapshot.threads.length, 1);
+
+ const recovered = yield* client[ORCHESTRATION_WS_METHODS.subscribeShell]({
+ afterSequence: initial.snapshot.snapshotSequence,
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ );
+ const snapshot = recovered[0];
+ assertTrue(snapshot?.kind === "snapshot");
+ assert.equal(snapshot.snapshot.snapshotSequence, 1_004);
+ assert.deepEqual(snapshot.snapshot.projects, []);
+ assert.deepEqual(snapshot.snapshot.threads, []);
+ assert.deepEqual(recovered[1], { kind: "synchronized" });
+ assert.equal(snapshotCalls, 2);
+ assert.equal(replayCalls, 0);
+ }),
+ ),
+ Effect.provide(withFirstWsAckHeld(wsUrl, ackHeld, releaseAck)),
+ );
+ }).pipe(Effect.scoped, Effect.provide(NodeHttpServer.layerTest)),
+ );
+
+ it.effect("subscriptions snapshot instead of decoding an oversized replay range", () =>
+ Effect.gen(function* () {
+ let readEventsCalls = 0;
+ let replayStatsCalls = 0;
+ const thread = makeDefaultOrchestrationReadModel().threads[0]!;
+ const shell = makeDefaultOrchestrationThreadShell({ id: thread.id });
+
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.succeed(5),
+ getThreadReplayStats: () =>
+ Effect.sync(() => {
+ replayStatsCalls += 1;
+ return {
+ eventCount: 5,
+ payloadBytes: 8 * 1024 * 1024 + 1,
+ hasCreateEvent: false,
+ };
+ }),
+ readThreadEvents: () => {
+ readEventsCalls += 1;
+ return Stream.empty;
+ },
+ readEvents: () =>
+ Stream.sync(() => {
+ readEventsCalls += 1;
+ return {} as OrchestrationEvent;
+ }),
+ },
+ projectionSnapshotQuery: {
+ getEventReplayStats: () =>
+ Effect.sync(() => {
+ replayStatsCalls += 1;
+ return { eventCount: 5, payloadBytes: 8 * 1024 * 1024 + 1 };
+ }),
+ getThreadDetailSnapshot: () =>
+ Effect.succeed(Option.some({ snapshotSequence: 5, thread })),
+ getShellSnapshot: () =>
+ Effect.succeed({
+ snapshotSequence: 5,
+ bots: [],
+ groups: [],
+ delegations: [],
+ projects: [],
+ threads: [shell],
+ updatedAt: "2026-01-01T00:00:00.000Z",
+ }),
+ },
+ },
+ });
+
+ const wsUrl = yield* getWsServerUrl("/ws");
+ const threadItems = yield* Effect.scoped(
+ withWsRpcClient(wsUrl, (client) =>
+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({
+ threadId: thread.id,
+ afterSequence: 0,
+ requestCompletionMarker: true,
+ }).pipe(Stream.take(2), Stream.runCollect),
+ ),
+ );
+ const shellItems = yield* Effect.scoped(
+ withWsRpcClient(wsUrl, (client) =>
+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({
+ afterSequence: 0,
+ requestCompletionMarker: true,
}).pipe(Stream.take(2), Stream.runCollect),
),
);
- const [first, second] = Array.from(items);
- assert.equal(first?.kind, "event");
- assert.equal(first?.kind === "event" ? first.event.sequence : null, 3);
- assert.equal(second?.kind, "synchronized");
- // The replay is bounded to the head captured before the read, not
- // Number.MAX_SAFE_INTEGER.
- assert.equal(replayLimit, 50);
+ assert.equal(threadItems[0]?.kind, "snapshot");
+ assert.equal(threadItems[1]?.kind, "synchronized");
+ assert.equal(shellItems[0]?.kind, "snapshot");
+ assert.equal(shellItems[1]?.kind, "synchronized");
+ assert.equal(replayStatsCalls, 2);
+ assert.equal(readEventsCalls, 0);
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
);
+ it.effect("subscribeThread replays a small thread range across a large global gap", () =>
+ Effect.gen(function* () {
+ const event = makeLiveToolActivityEvent(99_999, "tool.completed");
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.succeed(100_000),
+ getThreadReplayStats: () =>
+ Effect.succeed({
+ eventCount: 1,
+ payloadBytes: Buffer.byteLength(jsonRequestBody(event.payload)),
+ hasCreateEvent: false,
+ }),
+ readThreadEvents: () => Stream.make(event),
+ readEvents: () => Stream.die("Thread replay must not read the global log"),
+ },
+ projectionSnapshotQuery: {
+ getEventReplayStats: () => Effect.die("Thread replay must not measure the global log"),
+ getThreadDetailSnapshot: () =>
+ Effect.die("Unrelated activity must not force a snapshot"),
+ },
+ },
+ });
+ const wsUrl = yield* getWsServerUrl("/ws");
+ const items = yield* Effect.scoped(
+ withWsRpcClient(wsUrl, (client) =>
+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({
+ threadId: defaultThreadId,
+ afterSequence: 5,
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ ),
+ ),
+ );
+ assert.deepEqual(
+ items.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),
+ [99_999, "synchronized"],
+ );
+ const first = items[0];
+ assertTrue(first?.kind === "event" && first.event.type === "thread.activity-appended");
+ assert.equal(first.event.payload.activity.kind, "tool.completed");
+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),
+ );
+
+ it.effect("subscribeThread resets cached history when its ID is created again", () =>
+ Effect.gen(function* () {
+ const thread = {
+ ...makeDefaultOrchestrationReadModel().threads[0]!,
+ title: "Recreated thread",
+ };
+ let requestedTurnLimit: number | undefined;
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.succeed(5),
+ getThreadReplayStats: () =>
+ Effect.succeed({
+ eventCount: 3,
+ payloadBytes: 1_000,
+ hasCreateEvent: true,
+ }),
+ readThreadEvents: () => Stream.die("A recreated thread must not reuse cached history"),
+ },
+ projectionSnapshotQuery: {
+ getThreadDetailSnapshot: (_threadId, options) => {
+ requestedTurnLimit = options?.turnLimit;
+ return Effect.succeed(Option.some({ snapshotSequence: 5, thread }));
+ },
+ },
+ },
+ });
+ const wsUrl = yield* getWsServerUrl("/ws");
+ const items = yield* Effect.scoped(
+ withWsRpcClient(wsUrl, (client) =>
+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({
+ threadId: defaultThreadId,
+ afterSequence: 2,
+ turnLimit: 1,
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ ),
+ ),
+ );
+ const first = items[0];
+ assertTrue(first?.kind === "snapshot");
+ assert.equal(first.snapshot.thread.title, "Recreated thread");
+ assert.equal(first.snapshot.snapshotSequence, 5);
+ assert.equal(requestedTurnLimit, 1);
+ assert.deepEqual(items[1], { kind: "synchronized" });
+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),
+ );
+
+ for (const { createBeforeDelete, oversized } of [
+ { createBeforeDelete: false, oversized: false },
+ { createBeforeDelete: true, oversized: false },
+ { createBeforeDelete: true, oversized: true },
+ ]) {
+ it.effect(
+ oversized
+ ? "keeps the missing-snapshot error when an absent thread exceeds the replay limit"
+ : `synchronizes an absent thread and removes its shell after ${createBeforeDelete ? "creation and deletion" : "deletion"}`,
+ () =>
+ Effect.gen(function* () {
+ const store = yield* OrchestrationEventStore;
+ const base = makeLiveToolActivityEvent(0, "tool.completed");
+ if (createBeforeDelete) {
+ const thread = makeDefaultOrchestrationReadModel().threads[0]!;
+ yield* store.append({
+ ...base,
+ eventId: EventId.make(`create-before-final-delete-${oversized}`),
+ type: "thread.created",
+ payload: {
+ threadId: defaultThreadId,
+ projectId: thread.projectId,
+ title: thread.title,
+ modelSelection: thread.modelSelection,
+ runtimeMode: thread.runtimeMode,
+ interactionMode: thread.interactionMode,
+ branch: thread.branch,
+ worktreePath: thread.worktreePath,
+ createdAt: thread.createdAt,
+ updatedAt: thread.updatedAt,
+ },
+ });
+ }
+ if (oversized) {
+ yield* Effect.forEach(
+ Array.from({ length: 1_000 }, (_, index) => index + 1),
+ (sequence) => store.append(makeLiveToolActivityEvent(sequence, "tool.completed")),
+ { discard: true },
+ );
+ }
+ const deleted = yield* store.append({
+ ...base,
+ eventId: EventId.make(`deleted-replay-${createBeforeDelete}-${oversized}`),
+ type: "thread.deleted",
+ payload: { threadId: defaultThreadId, deletedAt: base.occurredAt },
+ });
+ yield* buildAppUnderTest({
+ layers: {
+ orchestrationEngine: {
+ latestSequence: Effect.succeed(deleted.sequence),
+ getThreadReplayStats: ({ threadId, ...range }) =>
+ store.getAggregateReplayStats({
+ ...range,
+ aggregateKind: "thread",
+ aggregateId: threadId,
+ }),
+ readThreadEvents: ({ threadId, ...range }) =>
+ store.readAggregateRange({
+ ...range,
+ aggregateKind: "thread",
+ aggregateId: threadId,
+ }),
+ readEvents: store.readFromSequence,
+ },
+ projectionSnapshotQuery: {
+ getThreadDetailSnapshot: () => Effect.succeed(Option.none()),
+ },
+ },
+ });
+ const wsUrl = yield* getWsServerUrl("/ws");
+ yield* Effect.scoped(
+ withWsRpcClient(wsUrl, (client) =>
+ Effect.gen(function* () {
+ const threadResult = yield* client[ORCHESTRATION_WS_METHODS.subscribeThread]({
+ threadId: defaultThreadId,
+ afterSequence: 0,
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ Effect.result,
+ );
+ if (oversized) {
+ assertTrue(threadResult._tag === "Failure");
+ assert.equal(threadResult.failure._tag, "OrchestrationGetSnapshotError");
+ assert.equal(
+ threadResult.failure.message,
+ `Chat ${defaultThreadId} was not found`,
+ );
+ return;
+ }
+ assertTrue(threadResult._tag === "Success");
+ assert.deepEqual(threadResult.success, [{ kind: "synchronized" }]);
+ const shellItems = yield* client[ORCHESTRATION_WS_METHODS.subscribeShell]({
+ afterSequence: 0,
+ requestCompletionMarker: true,
+ }).pipe(
+ Stream.takeUntil((item) => item.kind === "synchronized"),
+ Stream.runCollect,
+ );
+ assert.deepEqual(shellItems, [
+ { kind: "thread-removed", sequence: deleted.sequence, threadId: defaultThreadId },
+ { kind: "synchronized" },
+ ]);
+ }),
+ ),
+ );
+ }).pipe(
+ Effect.provide(
+ Layer.mergeAll(
+ OrchestrationEventStoreLive.pipe(Layer.provideMerge(SqlitePersistenceMemory)),
+ NodeHttpServer.layerTest,
+ ),
+ ),
+ ),
+ );
+ }
+
it.effect("subscribeShell sends a fresh snapshot instead of replaying a large gap", () =>
Effect.gen(function* () {
let readEventsCalls = 0;
diff --git a/apps/server/src/serverRuntimeStartup.reconcile.test.ts b/apps/server/src/serverRuntimeStartup.reconcile.test.ts
index 19d8cf6d6..a38929449 100644
--- a/apps/server/src/serverRuntimeStartup.reconcile.test.ts
+++ b/apps/server/src/serverRuntimeStartup.reconcile.test.ts
@@ -83,6 +83,8 @@ const runReconciliation = (input: {
Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, input.directory),
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch: input.dispatch,
streamDomainEvents: Stream.empty,
subscribeDomainEvents: Effect.succeed(Stream.empty),
@@ -289,6 +291,8 @@ it.effect("does not fail startup when the live provider session inventory cannot
}),
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch: () => Effect.die("unused"),
streamDomainEvents: Stream.empty,
subscribeDomainEvents: Effect.succeed(Stream.empty),
diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.ts
index 35d6045e7..6a2915058 100644
--- a/apps/server/src/serverRuntimeStartup.test.ts
+++ b/apps/server/src/serverRuntimeStartup.test.ts
@@ -102,6 +102,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threa
getArchivedShellSnapshot: () => Effect.die("unused"),
getSnapshotSequence: () => Effect.die("unused"),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () =>
Effect.succeed(
@@ -129,6 +130,8 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threa
}),
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch: (command) =>
Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(
Effect.as({ sequence: 1 }),
@@ -163,6 +166,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets creates a project and thread when
getArchivedShellSnapshot: () => Effect.die("unused"),
getSnapshotSequence: () => Effect.die("unused"),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.die("unused"),
@@ -178,6 +182,8 @@ it.effect("resolveAutoBootstrapWelcomeTargets creates a project and thread when
}),
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch: (command) =>
Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(
Effect.as({ sequence: 1 }),
@@ -212,6 +218,7 @@ it.effect(
getArchivedShellSnapshot: () => Effect.die("unused"),
getSnapshotSequence: () => Effect.die("unused"),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.die("unused"),
@@ -227,6 +234,8 @@ it.effect(
}),
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch: (command) =>
Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(
Effect.as({ sequence: 1 }),
@@ -262,6 +271,7 @@ it.effect(
getArchivedShellSnapshot: () => Effect.die("unused"),
getSnapshotSequence: () => Effect.die("unused"),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.die("unused"),
getProjectShellById: () => Effect.die("unused"),
@@ -277,6 +287,8 @@ it.effect(
}),
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch: (command) =>
Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(
Effect.as({ sequence: 1 }),
@@ -316,6 +328,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation fa
getArchivedShellSnapshot: () => Effect.die("unused"),
getSnapshotSequence: () => Effect.die("unused"),
getCounts: () => Effect.die("unused"),
+ getEventReplayStats: () => Effect.die("unused"),
getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),
getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),
getProjectShellById: () => Effect.die("unused"),
@@ -331,6 +344,8 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation fa
}),
Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
readEvents: () => Stream.empty,
+ readThreadEvents: () => Stream.empty,
+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),
dispatch: (command) =>
Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(
Effect.as({ sequence: 1 }),
diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts
index ca8904cc9..44650b9c4 100644
--- a/apps/server/src/ws.ts
+++ b/apps/server/src/ws.ts
@@ -103,6 +103,7 @@ import {
projectThreadDetailSnapshot,
} from "./orchestration/ActivityPayloadProjection.ts";
import { makeThreadLiveEventCoalescer } from "./orchestration/ThreadLiveEventCoalescer.ts";
+import { makeLiveStreamBudget, type RetainedLiveItem } from "./orchestration/LiveStreamBudget.ts";
import {
applyAuthenticatedCommandActor,
applyKnownGroupPerson,
@@ -418,12 +419,13 @@ const PROVIDER_STATUS_DEBOUNCE_MS = 200;
// Matches the event store's default page size (DEFAULT_READ_FROM_SEQUENCE_LIMIT).
const SHELL_RESUME_MAX_GAP = 1_000;
-// Same bound for thread resume. The replay reads the *global* event range and
-// filters per-thread afterwards, so a stale cursor far behind the head would
-// otherwise decode every intervening event's payload — reconnects with cursors
-// hundreds of thousands of events behind have OOM-killed servers on large
-// databases. Past this gap the client is reset with a fresh thread snapshot.
-const THREAD_RESUME_MAX_GAP = 1_000;
+// Thread replay counts only this thread's rows. Busy or pruned unrelated
+// streams must not force a full thread snapshot.
+const THREAD_RESUME_MAX_EVENTS = 1_000;
+// Row count alone does not bound replay memory: a few events with large tool
+// payloads can decode to gigabytes. Before replaying, sum the serialized
+// payload bytes of the range in SQL and reset with a snapshot past this budget.
+const ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES = 8 * 1024 * 1024;
function toAuthAccessStreamEvent(
change: PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange,
@@ -508,6 +510,42 @@ const makeWsRpcLayer = (
const routineRepository = yield* RoutineRepository;
const routineRuntime = yield* RoutineRuntime;
const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;
+ const canReplayPersistedRange = Effect.fnUntraced(function* (
+ afterSequence: number,
+ headSequence: number,
+ maxGap: number,
+ ) {
+ const replayGap = headSequence - afterSequence;
+ if (replayGap < 0 || replayGap > maxGap) {
+ return false;
+ }
+ const stats = yield* projectionSnapshotQuery
+ .getEventReplayStats({
+ fromSequenceExclusive: afterSequence,
+ toSequenceInclusive: headSequence,
+ })
+ .pipe(
+ Effect.mapError(
+ (cause) =>
+ new OrchestrationGetSnapshotError({
+ message: "Failed to measure orchestration replay range",
+ cause,
+ }),
+ ),
+ );
+ if (stats.payloadBytes > ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES) {
+ yield* Effect.logDebug("orchestration replay replaced by snapshot", {
+ afterSequence,
+ headSequence,
+ replayGap,
+ eventCount: stats.eventCount,
+ payloadBytes: stats.payloadBytes,
+ payloadBudgetBytes: ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES,
+ });
+ return false;
+ }
+ return true;
+ });
const threadDeletionReactor = yield* ThreadDeletionReactor;
// Every command dispatched on this connection carries the connecting
// client's origin, including server-generated bootstrap sub-commands:
@@ -1261,15 +1299,6 @@ const makeWsRpcLayer = (
return output;
});
- const coalesceShellLiveStream = <E, R>(
- stream: Stream.Stream<ShellLiveInput, E, R>,
- ): Stream.Stream<OrchestrationShellStreamItem, E, R> =>
- stream.pipe(
- Stream.groupedWithin(SHELL_COALESCE_MAX_CHUNK, SHELL_COALESCE_WINDOW),
- Stream.mapEffect(coalesceShellLiveInputs),
- Stream.flatMap((items) => Stream.fromIterable(items)),
- );
-
const dispatchBootstrapTurnStart = (
command: Extract<OrchestrationCommand, { type: "thread.turn.start" }>,
): Effect.Effect<{ readonly sequence: number }, OrchestrationDispatchCommandError> =>
@@ -1875,16 +1904,55 @@ const makeWsRpcLayer = (
// sequence but the live subscription is not attached yet). Every
// path below emits from this same buffered live tail. Overlapping
// events are deduped by sequence on the client.
- const liveBuffer = yield* Queue.unbounded<ShellLiveInput>();
+ const liveBudget = yield* makeLiveStreamBudget();
+ const liveBuffer = yield* Queue.unbounded<
+ RetainedLiveItem<ShellLiveInput>,
+ OrchestrationGetSnapshotError
+ >();
+ let liveBufferClosed = false;
+ const closeLiveBuffer = (error?: OrchestrationGetSnapshotError) =>
+ Effect.gen(function* () {
+ if (liveBufferClosed) {
+ return;
+ }
+ liveBufferClosed = true;
+ liveBudget.release(yield* Queue.clear(liveBuffer).pipe(Effect.orDie));
+ if (error) {
+ yield* Queue.fail(liveBuffer, error);
+ }
+ yield* Queue.shutdown(liveBuffer);
+ });
+ yield* Effect.addFinalizer(() => closeLiveBuffer());
+ yield* liveBudget.failed.pipe(
+ Effect.catchTags({ OrchestrationGetSnapshotError: closeLiveBuffer }),
+ Effect.forkScoped,
+ );
yield* Effect.forkScoped(
orchestrationEngine.streamDomainEvents.pipe(
Stream.runForEach((event) =>
- Queue.offer(liveBuffer, { kind: "event" as const, event }),
+ liveBudget.retain({ kind: "event" as const, event }, event).pipe(
+ Effect.flatMap((item) => Queue.offer(liveBuffer, item)),
+ Effect.uninterruptible,
+ ),
),
+ // Stop the PubSub consumer even if RPC delivery is waiting
+ // for an ACK and never pulls the failed buffer again.
+ Effect.raceFirst(liveBudget.failed),
+ Effect.catchTags({ OrchestrationGetSnapshotError: () => Effect.void }),
),
{ startImmediately: true },
);
- const bufferedLiveStream = coalesceShellLiveStream(Stream.fromQueue(liveBuffer));
+ const coalesceRetainedInputs = (
+ items: ReadonlyArray<RetainedLiveItem<ShellLiveInput>>,
+ ) =>
+ coalesceShellLiveInputs(items.map((item) => item.value)).pipe(
+ Effect.flatMap((output) => liveBudget.replace(items, output)),
+ );
+ const bufferedLiveStream = Stream.fromQueue(liveBuffer).pipe(
+ Stream.groupedWithin(SHELL_COALESCE_MAX_CHUNK, SHELL_COALESCE_WINDOW),
+ Stream.mapEffect(coalesceRetainedInputs),
+ Stream.flatMap((items) => Stream.fromIterable(items)),
+ );
const loadSnapshot = Effect.all({
snapshot: projectionSnapshotQuery.getShellSnapshot(),
@@ -1922,18 +1990,21 @@ const makeWsRpcLayer = (
// Offer the completion marker into the same queue as live events.
// Anything buffered while snapshot/replay work was in flight is
// therefore delivered before the client is told it is synchronized.
- const synchronizedThenLive =
+ const synchronizedThenLive = liveBudget.deliver(
input.requestCompletionMarker === true
? Stream.concat(
Stream.fromEffect(
- Queue.offer(liveBuffer, { kind: "synchronized" as const }).pipe(
+ liveBudget.retain({ kind: "synchronized" as const }).pipe(
+ Effect.flatMap((item) => Queue.offer(liveBuffer, item)),
+ Effect.uninterruptible,
Effect.andThen(Queue.takeAll(liveBuffer)),
- Effect.flatMap(coalesceShellLiveInputs),
+ Effect.flatMap(coalesceRetainedInputs),
),
).pipe(Stream.flatMap((items) => Stream.fromIterable(items))),
bufferedLiveStream,
)
- : bufferedLiveStream;
+ : bufferedLiveStream,
+ );
// When the client already holds a shell snapshot (cached, or loaded
// over HTTP) it passes that snapshot's sequence, and we resume by
@@ -1951,7 +2022,13 @@ const makeWsRpcLayer = (
// is also invalid, so reset it with a snapshot. Send the snapshot
// followed by the buffered live tail, exactly as the
// no-afterSequence path does.
- if (replayGap < 0 || replayGap > SHELL_RESUME_MAX_GAP) {
+ if (
+ !(yield* canReplayPersistedRange(
+ afterSequence,
+ headSequence,
+ SHELL_RESUME_MAX_GAP,
+ ))
+ ) {
const snapshot = yield* loadSnapshot;
return Stream.concat(
Stream.make({ kind: "snapshot" as const, snapshot }),
@@ -1963,7 +2040,7 @@ const makeWsRpcLayer = (
// are already covered by the live subscription, so this bound
// cannot chase a moving event-store head or grow the live
// buffer indefinitely while waiting for an empty page.
- orchestrationEngine.readEvents(afterSequence, replayGap),
+ orchestrationEngine.readEvents(afterSequence, replayGap, headSequence),
).pipe(
Stream.mapError(
(cause) =>
@@ -2033,8 +2110,16 @@ const makeWsRpcLayer = (
// Attach live delivery before reading either replay or snapshot state.
// Otherwise an event published while the snapshot is loading is lost.
const liveBuffer = yield* makeThreadLiveEventCoalescer();
- yield* Effect.forkScoped(liveStream.pipe(Stream.runForEach(liveBuffer.offer)));
+ yield* Effect.forkScoped(
+ liveStream.pipe(
+ Stream.runForEachArray(liveBuffer.offerAll),
+ Effect.raceFirst(liveBuffer.failed),
+ Effect.catchTags({ OrchestrationGetSnapshotError: () => Effect.void }),
+ ),
+ { startImmediately: true },
+ );
const bufferedLiveStream = liveBuffer.stream;
+ let replayOnMissingSnapshot: typeof bufferedLiveStream | undefined;
// When the client already loaded the snapshot over HTTP it passes
// that snapshot's sequence, and we resume the live subscription by
@@ -2049,22 +2134,41 @@ const makeWsRpcLayer = (
// catch-up followed by the buffered/ongoing live events. Overlapping
// events are deduped by sequence on the client.
//
- // The replay is bounded to the projection head captured below. The
- // catch-up range is normally tiny (a fresh HTTP snapshot sequence),
- // but a stale cached cursor can sit hundreds of thousands of global
- // events behind — replaying that decodes every intervening event
- // (including every other thread's tool payloads) only to discard
- // almost all of them, which has OOM-killed servers on large
- // databases. A truncated replay would silently drop this thread's
- // events, so past the gap cap we reset the client with a fresh
- // thread snapshot instead, exactly like subscribeShell above.
+ // Measure only this thread's rows. Global sequence gaps can
+ // contain unrelated or pruned streams. Keep an explicit upper
+ // bound so events after the captured head stay in the live tail.
if (input.afterSequence !== undefined) {
const afterSequence = input.afterSequence;
const headSequence = yield* orchestrationEngine.latestSequence;
- const replayGap = headSequence - afterSequence;
- if (replayGap >= 0 && replayGap <= THREAD_RESUME_MAX_GAP) {
+ const range = {
+ threadId: input.threadId,
+ fromSequenceExclusive: afterSequence,
+ toSequenceInclusive: headSequence,
+ };
+ const replayStats =
+ afterSequence > headSequence
+ ? null
+ : yield* orchestrationEngine
+ .getThreadReplayStats({
+ ...range,
+ maxEvents: THREAD_RESUME_MAX_EVENTS,
+ })
+ .pipe(
+ Effect.mapError(
+ (cause) =>
+ new OrchestrationGetSnapshotError({
+ message: `Failed to measure chat ${input.threadId} replay range`,
+ cause,
+ }),
+ ),
+ );
+ if (
+ replayStats !== null &&
+ replayStats.eventCount <= THREAD_RESUME_MAX_EVENTS &&
+ replayStats.payloadBytes <= ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES
+ ) {
const catchUpStream = orchestrationEngine
- .readEvents(afterSequence, replayGap)
+ .readThreadEvents({ ...range, limit: THREAD_RESUME_MAX_EVENTS })
.pipe(
Stream.filter(isThisThreadDetailEvent),
Stream.map((event) => ({
@@ -2081,20 +2185,20 @@ const makeWsRpcLayer = (
);
const afterCatchUp =
input.requestCompletionMarker === true
- ? Stream.concat(
- Stream.fromEffect(
- liveBuffer
- .offerAndWait({ kind: "synchronized" as const })
- .pipe(Effect.andThen(liveBuffer.takeAll)),
- ).pipe(Stream.flatMap((items) => Stream.fromIterable(items))),
- bufferedLiveStream,
+ ? Stream.unwrap(
+ liveBuffer
+ .offer({ kind: "synchronized" as const })
+ .pipe(Effect.as(bufferedLiveStream)),
)
: bufferedLiveStream;
- return Stream.concat(catchUpStream, afterCatchUp);
+ const replay = Stream.concat(catchUpStream, afterCatchUp);
+ if (!replayStats.hasCreateEvent) {
+ return replay;
+ }
+ replayOnMissingSnapshot = replay;
}
- // Gap too large (or cursor ahead of authoritative state): fall
- // through to the snapshot path so the client converges from a
- // fresh thread detail instead of an unbounded replay.
+ // A recreated thread needs a fresh snapshot if it still exists.
+ // Oversized replays and invalid cursors also use the snapshot path.
}
const snapshot = yield* projectionSnapshotQuery
@@ -2117,6 +2221,12 @@ const makeWsRpcLayer = (
);
if (Option.isNone(snapshot)) {
+ // The recreated thread can already be deleted. Preserve the
+ // bounded replay and shell removal instead of retrying a
+ // snapshot that cannot exist. Oversized ranges still fail.
+ if (replayOnMissingSnapshot !== undefined) {
+ return replayOnMissingSnapshot;
+ }
return yield* new OrchestrationGetSnapshotError({
message: `Chat ${input.threadId} was not found`,
cause: input.threadId,
@@ -2125,13 +2235,10 @@ const makeWsRpcLayer = (
const afterSnapshot =
input.requestCompletionMarker === true
- ? Stream.concat(
- Stream.fromEffect(
- liveBuffer
- .offerAndWait({ kind: "synchronized" as const })
- .pipe(Effect.andThen(liveBuffer.takeAll)),
- ).pipe(Stream.flatMap((items) => Stream.fromIterable(items))),
- bufferedLiveStream,
+ ? Stream.unwrap(
+ liveBuffer
+ .offer({ kind: "synchronized" as const })
+ .pipe(Effect.as(bufferedLiveStream)),
)
: bufferedLiveStream;
return Stream.concat(