Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
perf(server): bound orchestration replay and slow-client live buffers (#216)
Leo41e213a90bda
diff --git a/apps/server/src/channels/ChannelRuntime.test.ts b/apps/server/src/channels/ChannelRuntime.test.tsindex c503ffae4..19cbfd08f 100644--- a/apps/server/src/channels/ChannelRuntime.test.ts+++ b/apps/server/src/channels/ChannelRuntime.test.ts@@ -440,6 +440,8 @@ function makeHarness(input: {});const engine = {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch,streamDomainEvents: Stream.empty,subscribeDomainEvents: Effect.succeed(Stream.empty),diff --git a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts b/apps/server/src/checkpointing/CheckpointDiffQuery.test.tsindex ee215bb04..67745c9c0 100644--- a/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts+++ b/apps/server/src/checkpointing/CheckpointDiffQuery.test.ts@@ -85,6 +85,7 @@ describe("CheckpointDiffQuery.layer", () => {Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.succeed(Option.none()),@@ -197,6 +198,7 @@ describe("CheckpointDiffQuery.layer", () => {Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.succeed(Option.none()),@@ -284,6 +286,7 @@ describe("CheckpointDiffQuery.layer", () => {Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.succeed(Option.none()),@@ -356,6 +359,7 @@ describe("CheckpointDiffQuery.layer", () => {Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.succeed(Option.none()),@@ -413,6 +417,7 @@ describe("CheckpointDiffQuery.layer", () => {Effect.die("CheckpointDiffQuery should not request archived shell snapshots"),getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 0 }),getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.succeed(Option.none()),diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.tsindex 260ec3a4e..040a8ec60 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts@@ -106,6 +106,8 @@ describe("OrchestrationEngine", () => {return savedEvent;}),readFromSequence: () => Stream.empty,+ readAggregateRange: () => Stream.die("unused aggregate replay"),+ getAggregateReplayStats: () => Effect.die("unused aggregate replay stats"),readAll: () =>Stream.fail(new PersistenceSqlError({@@ -209,6 +211,7 @@ describe("OrchestrationEngine", () => {getSnapshotSequence: () =>Effect.succeed({ snapshotSequence: projectionSnapshot.snapshotSequence }),getCounts: () => Effect.succeed({ projectCount: 1, threadCount: 1 }),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.succeed(Option.none()),@@ -823,6 +826,12 @@ describe("OrchestrationEngine", () => {readFromSequence(sequenceExclusive) {return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive));},+ readAggregateRange() {+ return Stream.die("unused aggregate replay");+ },+ getAggregateReplayStats() {+ return Effect.die("unused aggregate replay stats");+ },readAll() {return Stream.fromIterable(events);},@@ -1064,6 +1073,12 @@ describe("OrchestrationEngine", () => {readFromSequence(sequenceExclusive) {return Stream.fromIterable(events.filter((event) => event.sequence > sequenceExclusive));},+ readAggregateRange() {+ return Stream.die("unused aggregate replay");+ },+ getAggregateReplayStats() {+ return Effect.die("unused aggregate replay stats");+ },readAll() {return Stream.fromIterable(events);},diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.tsindex dc63abf1f..311e85a6a 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts@@ -408,8 +408,24 @@ const makeOrchestrationEngine = Effect.gen(function* () {Effect.annotateLogs({ sequence: commandReadModel.snapshotSequence }),);- const readEvents: OrchestrationEngineShape["readEvents"] = (fromSequenceExclusive, limit) =>- eventStore.readFromSequence(fromSequenceExclusive, limit);+ const readEvents: OrchestrationEngineShape["readEvents"] = (+ fromSequenceExclusive,+ limit,+ toSequenceInclusive,+ ) => eventStore.readFromSequence(fromSequenceExclusive, limit, toSequenceInclusive);++ const readThreadEvents: OrchestrationEngineShape["readThreadEvents"] = ({ threadId, ...range }) =>+ eventStore.readAggregateRange({ ...range, aggregateKind: "thread", aggregateId: threadId });++ const getThreadReplayStats: OrchestrationEngineShape["getThreadReplayStats"] = ({+ threadId,+ ...range+ }) =>+ eventStore.getAggregateReplayStats({+ ...range,+ aggregateKind: "thread",+ aggregateId: threadId,+ });const dispatch: OrchestrationEngineShape["dispatch"] = (command, options) =>Effect.gen(function* () {@@ -426,6 +442,8 @@ const makeOrchestrationEngine = Effect.gen(function* () {return {readEvents,+ readThreadEvents,+ getThreadReplayStats,dispatch,// Each access creates a fresh PubSub subscription so that multiple// consumers (wsServer, ProviderRuntimeIngestion, CheckpointReactor, etc.)diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.tsindex 5bbb4dd06..77b519f79 100644--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts@@ -1008,6 +1008,55 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => {}),);+ it.effect("measures replay payload bytes without decoding event bodies", () =>+ Effect.gen(function* () {+ const snapshotQuery = yield* ProjectionSnapshotQuery;+ const sql = yield* SqlClient.SqlClient;++ yield* sql`DELETE FROM orchestration_events`;+ const rows = yield* sql<{ readonly sequence: number }>`+ INSERT INTO orchestration_events (+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,+ command_id, causation_event_id, correlation_id, actor_kind, payload_json, metadata_json+ )+ VALUES+ (+ 'replay-event-1', 'thread', 'thread-replay', 1, 'thread.activity-appended',+ '2026-03-01T00:00:00.000Z', NULL, NULL, NULL, 'provider',+ json_object('output', printf('%.*c', 1000, 'x')), '{}'+ ),+ (+ 'replay-event-2', 'thread', 'thread-replay', 2, 'thread.activity-appended',+ '2026-03-01T00:00:01.000Z', NULL, NULL, NULL, 'provider',+ json_object('output', printf('%.*c', 2000, 'x')), '{}'+ ),+ (+ 'replay-event-3', 'thread', 'thread-replay', 3, 'thread.activity-appended',+ '2026-03-01T00:00:02.000Z', NULL, NULL, NULL, 'provider',+ json_object('output', printf('%.*c', 3000, 'x')), '{}'+ ),+ (+ 'replay-event-4', 'thread', 'thread-replay', 4, 'thread.activity-appended',+ '2026-03-01T00:00:03.000Z', NULL, NULL, NULL, 'provider',+ json_object('output', '😀'), '{}'+ )+ RETURNING sequence+ `;++ // Bytes, not code points: the 4-byte emoji row is {"output":"😀"}, 17 bytes.+ // Shared-layer AUTOINCREMENT does not reset after DELETE, so bound the+ // query to the inserted sequences rather than assuming 1..4.+ const stats = yield* snapshotQuery.getEventReplayStats({+ fromSequenceExclusive: rows[0]!.sequence,+ toSequenceInclusive: rows[3]!.sequence,+ });+ assert.deepStrictEqual(stats, {+ eventCount: 3,+ payloadBytes: 5043,+ });+ }),+ );+it.effect("reads single-thread checkpoint context without hydrating unrelated threads", () =>Effect.gen(function* () {const snapshotQuery = yield* ProjectionSnapshotQuery;diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.tsindex 65f9b3023..7a823f745 100644--- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts+++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts@@ -81,6 +81,7 @@ import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityResimport { ORCHESTRATION_PROJECTOR_NAMES } from "./ProjectionPipeline.ts";import {ProjectionSnapshotQuery,+ type ProjectionEventReplayStats,type ProjectionFullThreadDiffContext,type ProjectionSnapshotCounts,type ProjectionThreadCheckpointContext,@@ -192,6 +193,14 @@ const ProjectionCountsRowSchema = Schema.Struct({projectCount: Schema.Number,threadCount: Schema.Number,});+const EventReplayStatsInput = Schema.Struct({+ fromSequenceExclusive: NonNegativeInt,+ toSequenceInclusive: NonNegativeInt,+});+const EventReplayStatsRowSchema = Schema.Struct({+ eventCount: Schema.Number,+ payloadBytes: Schema.Number,+});const ProjectionThreadSearchRequest = Schema.Struct({pattern: Schema.String,limit: Schema.Int,@@ -1009,6 +1018,20 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () {`,});+ const readEventReplayStats = SqlSchema.findOne({+ Request: EventReplayStatsInput,+ Result: EventReplayStatsRowSchema,+ execute: ({ fromSequenceExclusive, toSequenceInclusive }) =>+ sql`+ SELECT+ COUNT(*) AS "eventCount",+ COALESCE(SUM(octet_length(payload_json)), 0) AS "payloadBytes"+ FROM orchestration_events+ WHERE sequence > ${fromSequenceExclusive}+ AND sequence <= ${toSequenceInclusive}+ `,+ });+const searchActiveThreadRows = SqlSchema.findAll({Request: ProjectionThreadSearchRequest,Result: ProjectionThreadSearchRow,@@ -2952,6 +2975,22 @@ pending_approval_requests AS (),);+ const getEventReplayStats: ProjectionSnapshotQueryShape["getEventReplayStats"] = (input) =>+ readEventReplayStats(input).pipe(+ Effect.mapError(+ toPersistenceSqlOrDecodeError(+ "ProjectionSnapshotQuery.getEventReplayStats:query",+ "ProjectionSnapshotQuery.getEventReplayStats:decodeRow",+ ),+ ),+ Effect.map(+ (row): ProjectionEventReplayStats => ({+ eventCount: row.eventCount,+ payloadBytes: row.payloadBytes,+ }),+ ),+ );+const searchThreads: ProjectionSnapshotQueryShape["searchThreads"] = Effect.fn("ProjectionSnapshotQuery.searchThreads",)(function* (input) {@@ -3665,6 +3704,7 @@ pending_approval_requests AS (searchThreads,getSnapshotSequence,getCounts,+ getEventReplayStats,getActiveProjectByWorkspaceRoot,getOriginalProjectIdByWorkspaceRoot,getProjectShellById,diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.tsindex d6ff7f8d6..21282429e 100644--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts@@ -473,6 +473,8 @@ describe("ProviderCommandReactor", () => {const engine = yield* OrchestrationEngineService;return {readEvents: engine.readEvents,+ readThreadEvents: engine.readThreadEvents,+ getThreadReplayStats: engine.getThreadReplayStats,dispatch: (command) => {if (command.type === "thread.title.regeneration.complete") {titleRegenerationCompletionDispatchAttempts += 1;diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.tsindex 16b296f02..659e00034 100644--- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts+++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts@@ -69,6 +69,8 @@ describe("ThreadDeletionReactor.start", () => {const engine: OrchestrationEngineService["Service"] = {readEvents: () => Stream.die("unused"),+ readThreadEvents: () => Stream.die("unused"),+ getThreadReplayStats: () => Effect.die("unused"),dispatch: () => Effect.die("unused"),get streamDomainEvents() {return Stream.die(diff --git a/apps/server/src/orchestration/LiveStreamBudget.test.ts b/apps/server/src/orchestration/LiveStreamBudget.test.tsnew file mode 100644index 000000000..d163a8317--- /dev/null+++ b/apps/server/src/orchestration/LiveStreamBudget.test.ts@@ -0,0 +1,47 @@+import { it } from "@effect/vitest";+import * as Deferred from "effect/Deferred";+import * as Effect from "effect/Effect";+import * as Queue from "effect/Queue";+import * as Stream from "effect/Stream";+import { describe, expect } from "vite-plus/test";++import { makeLiveStreamBudget, type RetainedLiveItem } from "./LiveStreamBudget.ts";++describe("LiveStreamBudget", () => {+ it.effect("closes the source without releasing a batch still waiting for an ACK", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const budget = yield* makeLiveStreamBudget({ maxItems: 3 });+ const queue = yield* Queue.unbounded<RetainedLiveItem<{ text: string }>>();+ const sourceClosed = yield* Deferred.make<void>();+ const first = yield* budget.retain({ text: "first" });+ const second = yield* budget.retain({ text: "second" });+ const third = yield* budget.retain({ text: "third" });+ yield* Queue.offerAll(queue, [first, second, third]);+ yield* Effect.scoped(+ Effect.gen(function* () {+ const pull = yield* Stream.toPull(+ budget.deliver(+ Stream.fromQueue(queue).pipe(+ Stream.rechunk(1),+ Stream.ensuring(Deferred.succeed(sourceClosed, undefined)),+ ),+ ),+ );+ expect(yield* pull).toEqual([{ text: "first" }]);+ // The other two items are in the source's pull state, not its queue.+ expect(yield* Queue.size(queue)).toBe(0);+ expect((yield* budget.usage).retainedItems).toBe(3);+ const overflow = yield* budget.retain({ text: "fourth" }).pipe(Effect.result);+ expect(overflow._tag).toBe("Failure");+ // Do not resume the consumer. Its source scope must close now.+ yield* Deferred.await(sourceClosed);+ yield* budget.closed;+ expect((yield* budget.usage).retainedItems).toBe(1);+ }),+ );+ expect(yield* budget.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });+ }),+ ),+ );+});diff --git a/apps/server/src/orchestration/LiveStreamBudget.ts b/apps/server/src/orchestration/LiveStreamBudget.tsnew file mode 100644index 000000000..3e3d31128--- /dev/null+++ b/apps/server/src/orchestration/LiveStreamBudget.ts@@ -0,0 +1,196 @@+import { OrchestrationGetSnapshotError } from "@t3tools/contracts";+import * as Arr from "effect/Array";+import * as Deferred from "effect/Deferred";+import * as Effect from "effect/Effect";+import * as Exit from "effect/Exit";+import * as Scope from "effect/Scope";+import * as Stream from "effect/Stream";++export const LIVE_STREAM_MAX_ITEMS = 1_000;+export const LIVE_STREAM_MAX_SERIALIZED_BYTES = 8 * 1024 * 1024;++export interface RetainedLiveItem<A> {+ readonly value: A;+ readonly serializedBytes: number;+}++// Published events are immutable and shared across subscriptions. Measure each+// object once without keeping the event or its serialized copy alive.+const serializedSizes = new WeakMap<object, number>();++function serializedSize(value: object): number {+ const cached = serializedSizes.get(value);+ if (cached !== undefined) {+ return cached;+ }+ const bytes = Buffer.byteLength(JSON.stringify(value));+ serializedSizes.set(value, bytes);+ return bytes;+}++/** One budget covers one subscription and its delivery stream, including the batch waiting for an RPC ACK. */+export const makeLiveStreamBudget = Effect.fn("makeLiveStreamBudget")(function* (limits?: {+ readonly maxItems?: number;+ readonly maxSerializedBytes?: number;+}) {+ const maxItems = limits?.maxItems ?? LIVE_STREAM_MAX_ITEMS;+ const maxSerializedBytes = limits?.maxSerializedBytes ?? LIVE_STREAM_MAX_SERIALIZED_BYTES;+ const failed = yield* Deferred.make<never, OrchestrationGetSnapshotError>();+ const cleanupComplete = yield* Deferred.make<void>();+ let failure: OrchestrationGetSnapshotError | undefined;+ const retained = new Set<RetainedLiveItem<unknown>>();+ let retainedSerializedBytes = 0;++ const release = (items: Iterable<RetainedLiveItem<unknown>>) => {+ for (const item of items) {+ if (!retained.delete(item)) {+ continue;+ }+ retainedSerializedBytes -= item.serializedBytes;+ }+ };+ yield* Effect.addFinalizer(() =>+ Effect.sync(() => release(retained)).pipe(+ Effect.andThen(Deferred.succeed(cleanupComplete, undefined)),+ ),+ );++ const check = Effect.suspend(() => (failure ? Effect.fail(failure) : Effect.void));++ const overflow = Effect.fn("LiveStreamBudget.overflow")(function* (+ nextItems: number,+ nextSerializedBytes: number,+ ) {+ failure ??= new OrchestrationGetSnapshotError({+ message: "The live event buffer is full. Resume from the last received sequence.",+ });+ yield* Deferred.fail(failed, failure);+ yield* Effect.logWarning("orchestration live event buffer is full", {+ retainedItems: retained.size,+ retainedSerializedBytes,+ nextItems,+ nextSerializedBytes,+ maxItems,+ maxSerializedBytes,+ });+ return yield* failure;+ });++ const retain = <A extends object>(value: A, payload: object = value) =>+ Effect.suspend(() => {+ if (failure) {+ return Effect.fail(failure);+ }+ const serializedBytes = serializedSize(payload);+ const nextItems = retained.size + 1;+ const nextSerializedBytes = retainedSerializedBytes + serializedBytes;+ if (nextItems > maxItems || nextSerializedBytes > maxSerializedBytes) {+ return overflow(nextItems, nextSerializedBytes);+ }+ const item = { value, serializedBytes };+ retained.add(item);+ retainedSerializedBytes = nextSerializedBytes;+ return Effect.succeed(item);+ });++ // Replace one coalescing batch atomically. Both raw and projected payloads+ // count against the same budget, and discarded updates release their charge.+ const replace = <A extends object>(+ previous: ReadonlyArray<RetainedLiveItem<unknown>>,+ values: ReadonlyArray<A>,+ payload: (value: A) => object = (value) => value,+ ) =>+ Effect.suspend(() => {+ if (failure) {+ return Effect.fail(failure);+ }+ const next = values.map((value) => ({+ value,+ serializedBytes: serializedSize(payload(value)),+ }));+ let nextItems = retained.size + next.length;+ let nextSerializedBytes =+ retainedSerializedBytes + next.reduce((sum, item) => sum + item.serializedBytes, 0);+ for (const item of previous) {+ if (retained.has(item)) {+ nextItems -= 1;+ nextSerializedBytes -= item.serializedBytes;+ }+ }+ if (nextItems > maxItems || nextSerializedBytes > maxSerializedBytes) {+ return overflow(nextItems, nextSerializedBytes);+ }+ release(previous);+ for (const item of next) {+ retained.add(item);+ }+ retainedSerializedBytes = nextSerializedBytes;+ return Effect.succeed(next);+ });++ const deliver = <A, E, R>(stream: Stream.Stream<RetainedLiveItem<A>, E, R>) =>+ Stream.fromPull(+ Effect.gen(function* () {+ yield* check;+ const sourceScope = yield* Scope.fork(yield* Effect.scope);+ const source = {+ pull: yield* Stream.toPull(stream).pipe(Scope.provide(sourceScope)),+ };+ let inFlight: ReadonlyArray<RetainedLiveItem<A>> = [];+ yield* Effect.addFinalizer(() =>+ Effect.sync(() => {+ release(inFlight);+ inFlight = [];+ source.pull = Effect.interrupt;+ }),+ );+ yield* Deferred.await(failed).pipe(+ Effect.catchTags({+ OrchestrationGetSnapshotError: (error) =>+ Effect.sync(() => {+ // A grouped source can retain its own pending chunk. Close it+ // and release the pull closure without waiting for an RPC ACK.+ source.pull = Effect.interrupt;+ }).pipe(+ Effect.andThen(Scope.close(sourceScope, Exit.fail(error))),+ Effect.andThen(+ Effect.sync(() => {+ const delivered = new Set<RetainedLiveItem<unknown>>(inFlight);+ for (const item of retained) {+ if (!delivered.has(item)) {+ release([item]);+ }+ }+ }),+ ),+ Effect.andThen(Deferred.succeed(cleanupComplete, undefined)),+ ),+ }),+ Effect.forkScoped,+ );+ // @effect-diagnostics-next-line returnEffectInGen:off - Stream.fromPull needs the pull effect as its result.+ return Effect.gen(function* () {+ // RpcServer requests the next batch only after the client ACKs this+ // one. Removing items from a queue alone does not mean delivery ended.+ release(inFlight);+ inFlight = [];+ yield* check;+ const items = yield* Effect.raceFirst(source.pull, Deferred.await(failed));+ inFlight = items;+ yield* check;+ return Arr.map(items, (item) => item.value);+ });+ }),+ );++ return {+ retain,+ replace,+ release,+ deliver,+ check,+ failed: Deferred.await(failed),+ closed: Deferred.await(cleanupComplete),+ usage: Effect.sync(() => ({ retainedItems: retained.size, retainedSerializedBytes })),+ };+});diff --git a/apps/server/src/orchestration/Services/OrchestrationEngine.ts b/apps/server/src/orchestration/Services/OrchestrationEngine.tsindex fdb22ce64..082a199af 100644--- a/apps/server/src/orchestration/Services/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Services/OrchestrationEngine.ts@@ -15,6 +15,7 @@ import type {OrchestrationClientOrigin,OrchestrationCommand,OrchestrationEvent,+ ThreadId,} from "@t3tools/contracts";import * as Context from "effect/Context";import type * as Effect from "effect/Effect";@@ -23,6 +24,13 @@ import type * as Stream from "effect/Stream";import type { OrchestrationDispatchError } from "../Errors.ts";import type { OrchestrationEventStoreError } from "../../persistence/Errors.ts";+import type { OrchestrationAggregateReplayStats } from "../../persistence/Services/OrchestrationEventStore.ts";++export interface OrchestrationThreadReplayRange {+ readonly threadId: ThreadId;+ readonly fromSequenceExclusive: number;+ readonly toSequenceInclusive: number;+}export interface OrchestrationDispatchActor {readonly personId: AuthSessionId;@@ -39,15 +47,26 @@ export interface OrchestrationEngineShape {* @param fromSequenceExclusive - Sequence cursor (exclusive).* @param limit - Maximum number of events to read. Defaults to the event* store's page-bounded default; pass a higher value when the caller must- * read every event after the cursor (e.g. per-thread catch-up that filters- * a small subset out of a potentially larger global range).+ * read a wider global range. Thread subscriptions use readThreadEvents.+ * @param toSequenceInclusive - Optional captured upper sequence boundary.* @returns Stream containing ordered events.*/readonly readEvents: (fromSequenceExclusive: number,limit?: number,+ toSequenceInclusive?: number,) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError, never>;+ /** Read only this thread's events through a captured authoritative head. */+ readonly readThreadEvents: (+ input: OrchestrationThreadReplayRange & { readonly limit?: number },+ ) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;++ /** Measure a bounded thread replay without decoding its event bodies. */+ readonly getThreadReplayStats: (+ input: OrchestrationThreadReplayRange & { readonly maxEvents: number },+ ) => Effect.Effect<OrchestrationAggregateReplayStats, OrchestrationEventStoreError>;+/*** Dispatch a validated orchestration command.*diff --git a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.tsindex 30ba972f8..b185b59e0 100644--- a/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts+++ b/apps/server/src/orchestration/Services/ProjectionSnapshotQuery.ts@@ -39,6 +39,11 @@ export interface ProjectionSnapshotCounts {readonly threadCount: number;}+export interface ProjectionEventReplayStats {+ readonly eventCount: number;+ readonly payloadBytes: number;+}+export interface ProjectionSnapshotSequence {readonly snapshotSequence: number;}@@ -150,6 +155,14 @@ export interface ProjectionSnapshotQueryShape {*/readonly getCounts: () => Effect.Effect<ProjectionSnapshotCounts, ProjectionRepositoryError>;+ /**+ * Measure a persisted event range without decoding its payload bodies.+ */+ readonly getEventReplayStats: (input: {+ readonly fromSequenceExclusive: number;+ readonly toSequenceInclusive: number;+ }) => Effect.Effect<ProjectionEventReplayStats, ProjectionRepositoryError>;+/*** Read the active project for an exact workspace root match.*/diff --git a/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.ts b/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.tsindex 0a9915294..6b54fadf8 100644--- a/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.ts+++ b/apps/server/src/orchestration/ThreadLiveEventCoalescer.test.ts@@ -3,12 +3,14 @@ import {MessageId,ThreadId,TurnId,- type OrchestrationEvent,+ OrchestrationEvent,type OrchestrationThreadActivity,} from "@t3tools/contracts";import { it } from "@effect/vitest";import * as Clock from "effect/Clock";import * as Effect from "effect/Effect";+import * as Schema from "effect/Schema";+import * as Stream from "effect/Stream";import * as TestClock from "effect/testing/TestClock";import { describe, expect } from "vite-plus/test";@@ -19,6 +21,7 @@ import {const threadId = ThreadId.make("thread-coalescer-test");const turnId = TurnId.make("turn-coalescer-test");+const encodeEvent = Schema.encodeSync(Schema.fromJsonString(OrchestrationEvent));function makeToolActivity(sequence: number,@@ -61,7 +64,7 @@ function makeToolActivity(};}-function makeMessage(sequence: number): OrchestrationEvent {+function makeMessage(sequence: number, text = "Still working"): OrchestrationEvent {return {sequence,eventId: EventId.make(`event-${sequence}`),@@ -77,7 +80,7 @@ function makeMessage(sequence: number): OrchestrationEvent {threadId,messageId: MessageId.make(`message-${sequence}`),role: "assistant",- text: "Still working",+ text,turnId,streaming: false,createdAt: "2026-01-01T00:00:02.000Z",@@ -134,15 +137,14 @@ describe("ThreadLiveEventCoalescer", () => {const startedAt = yield* Clock.currentTimeMillis;yield* Effect.forEach(Array.from({ length: 10 }, (_, index) => index + 2),- (sequence) =>- coalescer.offerAndWait({ kind: "event", event: makeToolActivity(sequence) }),+ (sequence) => coalescer.offer({ kind: "event", event: makeToolActivity(sequence) }),{ discard: true },);- yield* coalescer.offerAndWait({ kind: "event", event: makeMessage(12) });+ yield* coalescer.offer({ kind: "event", event: makeMessage(12) });expect(yield* Clock.currentTimeMillis).toBe(startedAt);expect(- Array.from(yield* coalescer.takeAll).map((item) =>+ (yield* coalescer.stream.pipe(Stream.take(2), Stream.runCollect)).map((item) =>item.kind === "event" ? item.event.sequence : item.kind,),).toEqual([11, 12]);@@ -155,17 +157,94 @@ describe("ThreadLiveEventCoalescer", () => {Effect.gen(function* () {const coalescer = yield* makeThreadLiveEventCoalescer({ coalesceWindow: "500 millis" });const startedAt = yield* Clock.currentTimeMillis;- yield* coalescer.offerAndWait({ kind: "event", event: makeToolActivity(2) });- yield* coalescer.offerAndWait({ kind: "event", event: makeToolActivity(3) });- yield* coalescer.offerAndWait({ kind: "synchronized" });+ yield* coalescer.offer({ kind: "event", event: makeToolActivity(2) });+ yield* coalescer.offer({ kind: "event", event: makeToolActivity(3) });+ yield* coalescer.offer({ kind: "synchronized" });expect(yield* Clock.currentTimeMillis).toBe(startedAt);expect(- Array.from(yield* coalescer.takeAll).map((item) =>+ (yield* coalescer.stream.pipe(Stream.take(2), Stream.runCollect)).map((item) =>item.kind === "event" ? item.event.sequence : item.kind,),).toEqual([3, "synchronized"]);}),).pipe(Effect.provide(TestClock.layer())),);++ it.effect("fails and clears pending updates when their serialized payload fills the budget", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const first = makeToolActivity(1);+ const coalescer = yield* makeThreadLiveEventCoalescer({+ coalesceWindow: "500 millis",+ maxSerializedBytes: Buffer.byteLength(encodeEvent(first)),+ });+ yield* coalescer.offer({ kind: "event", event: first });+ expect(yield* coalescer.usage).toEqual({+ retainedItems: 1,+ retainedSerializedBytes: Buffer.byteLength(encodeEvent(first)),+ });++ const overflow = yield* coalescer+ .offer({ kind: "event", event: makeToolActivity(2) })+ .pipe(Effect.result);+ expect(overflow._tag).toBe("Failure");+ yield* coalescer.closed;+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });+ const marker = yield* coalescer.offer({ kind: "synchronized" }).pipe(Effect.result);+ expect(marker._tag).toBe("Failure");+ const delivered = yield* coalescer.stream.pipe(Stream.runCollect, Effect.result);+ expect(delivered._tag).toBe("Failure");+ }),+ ),+ );++ it.effect("keeps the flush timer alive when an offer's shorter scope closes", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const coalescer = yield* makeThreadLiveEventCoalescer({ coalesceWindow: "50 millis" });+ yield* Effect.scoped(coalescer.offer({ kind: "event", event: makeToolActivity(1) }));+ yield* TestClock.adjust("50 millis");+ const items = yield* coalescer.stream.pipe(Stream.take(1), Stream.runCollect);+ expect(+ items.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),+ ).toEqual([1]);+ }),+ ).pipe(Effect.provide(TestClock.layer())),+ );++ it.effect("keeps an unacknowledged batch charged and clears later events on overflow", () =>+ Effect.scoped(+ Effect.gen(function* () {+ const coalescer = yield* makeThreadLiveEventCoalescer({ maxItems: 3 });+ const first = makeMessage(1, "é".repeat(1_024));+ yield* coalescer.offer({ kind: "event", event: first });++ yield* Effect.scoped(+ Effect.gen(function* () {+ const pull = yield* Stream.toPull(coalescer.stream);+ const batch = yield* pull;+ expect(+ batch.map((item) => (item.kind === "event" ? item.event.sequence : null)),+ ).toEqual([1]);+ yield* coalescer.offer({ kind: "event", event: makeMessage(2) });+ yield* coalescer.offer({ kind: "event", event: makeToolActivity(3) });+ expect((yield* coalescer.usage).retainedItems).toBe(3);++ const overflow = yield* coalescer+ .offer({ kind: "event", event: makeToolActivity(4, { kind: "tool.completed" }) })+ .pipe(Effect.result);+ expect(overflow._tag).toBe("Failure");+ // Do not pull or acknowledge the batch. Cleanup must still finish.+ yield* coalescer.closed;+ expect(yield* coalescer.usage).toEqual({+ retainedItems: 1,+ retainedSerializedBytes: Buffer.byteLength(encodeEvent(first)),+ });+ }),+ );+ expect(yield* coalescer.usage).toEqual({ retainedItems: 0, retainedSerializedBytes: 0 });+ }),+ ),+ );});diff --git a/apps/server/src/orchestration/ThreadLiveEventCoalescer.ts b/apps/server/src/orchestration/ThreadLiveEventCoalescer.tsindex f188deea8..c7f2155b4 100644--- a/apps/server/src/orchestration/ThreadLiveEventCoalescer.ts+++ b/apps/server/src/orchestration/ThreadLiveEventCoalescer.ts@@ -1,4 +1,8 @@-import type { OrchestrationEvent, OrchestrationThreadStreamItem } from "@t3tools/contracts";+import type {+ OrchestrationEvent,+ OrchestrationGetSnapshotError,+ OrchestrationThreadStreamItem,+} from "@t3tools/contracts";import * as Deferred from "effect/Deferred";import * as Duration from "effect/Duration";import * as Effect from "effect/Effect";@@ -9,6 +13,7 @@ import * as Semaphore from "effect/Semaphore";import * as Stream from "effect/Stream";import { projectActivityEvent } from "./ActivityPayloadProjection.ts";+import { makeLiveStreamBudget, type RetainedLiveItem } from "./LiveStreamBudget.ts";const COALESCE_WINDOW = Duration.millis(50);const MAX_PENDING_UPDATES = 512;@@ -89,17 +94,24 @@ export function coalesceLiveToolUpdatedEvents(}export const makeThreadLiveEventCoalescer = Effect.fn("makeThreadLiveEventCoalescer")(- function* (options?: { readonly coalesceWindow?: Duration.Input }) {- const output = yield* Queue.unbounded<OrchestrationThreadStreamItem>();- const input = yield* Queue.unbounded<{- readonly value: ThreadLiveInput;- readonly processed?: Deferred.Deferred<void>;- }>();+ function* (options?: {+ readonly coalesceWindow?: Duration.Input;+ readonly maxItems?: number;+ readonly maxSerializedBytes?: number;+ }) {+ const coalescerScope = yield* Effect.scope;+ const budget = yield* makeLiveStreamBudget(options);+ const cleanupComplete = yield* Deferred.make<void>();+ const output = yield* Queue.unbounded<+ RetainedLiveItem<OrchestrationThreadStreamItem>,+ OrchestrationGetSnapshotError+ >();const mutex = yield* Semaphore.make(1);const coalesceWindow = options?.coalesceWindow ?? COALESCE_WINDOW;- let pendingUpdates: Array<OrchestrationEvent> = [];+ let pendingUpdates: Array<RetainedLiveItem<OrchestrationEvent>> = [];let windowGeneration = 0;let windowFiber: Fiber.Fiber<void, never> | null = null;+ let closed = false;const cancelWindow = Effect.fn("ThreadLiveEventCoalescer.cancelWindow")(function* () {const fiber = windowFiber;@@ -110,22 +122,21 @@ export const makeThreadLiveEventCoalescer = Effect.fn("makeThreadLiveEventCoalesyield* Fiber.interrupt(fiber);});- const flushPending = Effect.fn("ThreadLiveEventCoalescer.flushPending")(function* (- boundary?: OrchestrationEvent,- ) {- const events = boundary ? [...pendingUpdates, boundary] : pendingUpdates;- pendingUpdates = [];- if (events.length === 0) {+ const flushPending = Effect.fn("ThreadLiveEventCoalescer.flushPending")(function* () {+ if (pendingUpdates.length === 0) {return;}- yield* Queue.offerAll(- output,- coalesceLiveToolUpdatedEvents(events).map((event) => ({+ const items = yield* budget.replace(+ pendingUpdates,+ coalesceLiveToolUpdatedEvents(pendingUpdates.map((item) => item.value)).map((event) => ({kind: "event" as const,event: projectActivityEvent(event),})),+ (item) => item.event,);- });+ pendingUpdates = [];+ yield* Queue.offerAll(output, items);+ }, Effect.uninterruptible);const flushWindow = (generation: number) =>Effect.sleep(coalesceWindow).pipe(@@ -141,63 +152,89 @@ export const makeThreadLiveEventCoalescer = Effect.fn("makeThreadLiveEventCoales}}),),+ Effect.catchTags({ OrchestrationGetSnapshotError: () => Effect.void }),);- const process = Effect.fn("ThreadLiveEventCoalescer.process")(function* (- input: ThreadLiveInput,+ // Keep each source batch together so a synchronization marker cannot pass+ // events already pulled from PubSub but still being coalesced.+ const offerAll = Effect.fn("ThreadLiveEventCoalescer.offerAll")(function* (+ inputs: ReadonlyArray<ThreadLiveInput>,) {yield* mutex.withPermits(1)(- Effect.gen(function* () {- if (input.kind === "event" && isToolUpdated(input.event)) {- pendingUpdates.push(input.event);- if (pendingUpdates.length === 1) {- const generation = ++windowGeneration;- windowFiber = yield* Effect.forkScoped(flushWindow(generation));- }- if (pendingUpdates.length >= MAX_PENDING_UPDATES) {+ Effect.forEach(+ inputs,+ (input) =>+ Effect.gen(function* () {+ yield* budget.check;+ if (input.kind === "event") {+ yield* budget.retain(input.event).pipe(+ Effect.tap((item) => Effect.sync(() => pendingUpdates.push(item))),+ Effect.uninterruptible,+ );+ }+ if (input.kind === "event" && isToolUpdated(input.event)) {+ if (pendingUpdates.length === 1) {+ const generation = ++windowGeneration;+ windowFiber = yield* Effect.forkIn(flushWindow(generation), coalescerScope);+ }+ if (pendingUpdates.length >= MAX_PENDING_UPDATES) {+ yield* cancelWindow();+ windowGeneration += 1;+ yield* flushPending();+ }+ return;+ }+yield* cancelWindow();windowGeneration += 1;+ // A non-update event closes the run immediately. The coalescer keeps+ // that boundary after the final update from the run.yield* flushPending();- }+ if (input.kind === "synchronized") {+ yield* budget.retain({ kind: "synchronized" as const }).pipe(+ Effect.flatMap((marker) => Queue.offer(output, marker)),+ Effect.uninterruptible,+ );+ }+ }),+ { discard: true },+ ),+ );+ });++ const close = (error?: OrchestrationGetSnapshotError) =>+ mutex.withPermits(1)(+ Effect.gen(function* () {+ if (closed) {return;}-- yield* cancelWindow();+ closed = true;windowGeneration += 1;- if (input.kind === "event") {- yield* flushPending(input.event);- } else {- yield* flushPending();- yield* Queue.offer(output, { kind: "synchronized" });+ yield* cancelWindow();+ budget.release(pendingUpdates);+ pendingUpdates = [];+ budget.release(yield* Queue.clear(output).pipe(Effect.orDie));+ if (error) {+ yield* Queue.fail(output, error);}+ yield* Queue.shutdown(output);+ yield* Deferred.succeed(cleanupComplete, undefined);}),);- });- yield* Stream.fromQueue(input).pipe(- Stream.runForEach(({ value, processed }) =>- process(value).pipe(- Effect.andThen(processed ? Deferred.succeed(processed, undefined) : Effect.void),- ),- ),+ yield* Effect.addFinalizer(() => close());+ yield* budget.failed.pipe(+ Effect.catchTags({ OrchestrationGetSnapshotError: close }),Effect.forkScoped,);- const offer = (value: ThreadLiveInput) => Queue.offer(input, { value }).pipe(Effect.asVoid);-- const offerAndWait = Effect.fn("ThreadLiveEventCoalescer.offerAndWait")(function* (- value: ThreadLiveInput,- ) {- const processed = yield* Deferred.make<void>();- yield* Queue.offer(input, { value, processed });- yield* Deferred.await(processed);- });-return {- offer,- offerAndWait,- stream: Stream.fromQueue(output),- takeAll: Queue.takeAll(output),+ offer: (input: ThreadLiveInput) => offerAll([input]),+ offerAll,+ stream: budget.deliver(Stream.fromQueue(output)),+ failed: budget.failed,+ closed: Deferred.await(cleanupComplete),+ usage: budget.usage,} as const;},);diff --git a/apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts b/apps/server/src/persistence/Layers/OrchestrationEventStore.test.tsindex 2bac5de92..ea0e645b0 100644--- a/apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts+++ b/apps/server/src/persistence/Layers/OrchestrationEventStore.test.ts@@ -1,4 +1,13 @@-import { CommandId, EventId, ProjectId } from "@t3tools/contracts";+import * as NodeV8 from "node:v8";++import {+ CommandId,+ EventId,+ MessageId,+ ProjectId,+ ThreadId,+ type OrchestrationEvent,+} from "@t3tools/contracts";import { assert, it } from "@effect/vitest";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";@@ -12,6 +21,31 @@ import { OrchestrationEventStoreLive } from "./OrchestrationEventStore.ts";import { SqlitePersistenceMemory } from "./Sqlite.ts";const isPersistenceDecodeError = Schema.is(PersistenceDecodeError);+function messageEvent(threadId: ThreadId, id: string): Omit<OrchestrationEvent, "sequence"> {+ const now = "2026-01-01T00:00:00.000Z";+ return {+ type: "thread.message-sent",+ eventId: EventId.make(id),+ aggregateKind: "thread",+ aggregateId: threadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ payload: {+ threadId,+ messageId: MessageId.make(id),+ role: "assistant",+ text: id,+ turnId: null,+ streaming: false,+ createdAt: now,+ updatedAt: now,+ },+ };+}+const layer = it.layer(OrchestrationEventStoreLive.pipe(Layer.provideMerge(SqlitePersistenceMemory)),);@@ -75,7 +109,7 @@ layer("OrchestrationEventStore", (it) => {const sql = yield* SqlClient.SqlClient;const now = "2026-01-01T00:00:00.000Z";- yield* sql`+ const invalidRows = yield* sql<{ readonly sequence: number }>`INSERT INTO orchestration_events (event_id,aggregate_kind,@@ -104,6 +138,7 @@ layer("OrchestrationEventStore", (it) => {${"{"},${"{}"})+ RETURNING sequence`;const replayResult = yield* Effect.result(@@ -118,6 +153,232 @@ layer("OrchestrationEventStore", (it) => {),);}+ const scopedResult = yield* eventStore+ .readAggregateRange({+ aggregateKind: "project",+ aggregateId: "project-invalid-json",+ fromSequenceExclusive: 0,+ toSequenceInclusive: invalidRows[0]!.sequence,+ })+ .pipe(Stream.runCollect, Effect.result);+ assert.equal(scopedResult._tag, "Failure");+ if (scopedResult._tag === "Failure") {+ assert.ok(isPersistenceDecodeError(scopedResult.failure));+ assert.ok(+ scopedResult.failure.operation.includes(+ "OrchestrationEventStore.readAggregateRange:decodeRows",+ ),+ );+ }+ }),+ );++ it.effect("bounds global replay to a captured head across sequence gaps", () =>+ Effect.gen(function* () {+ const store = yield* OrchestrationEventStore;+ const sql = yield* SqlClient.SqlClient;+ const threadId = ThreadId.make("bounded-global-replay");+ const first = yield* store.append(messageEvent(threadId, "bounded-first"));+ const pruned = yield* store.append(messageEvent(threadId, "bounded-pruned"));+ const last = yield* store.append(messageEvent(threadId, "bounded-last"));+ yield* sql`DELETE FROM orchestration_events WHERE sequence = ${pruned.sequence}`;+ yield* sql`+ INSERT INTO orchestration_events (+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,+ actor_kind, payload_json, metadata_json+ ) VALUES (+ 'bounded-after-captured-head', 'thread', ${threadId}, 3, 'thread.message-sent',+ '2026-01-01T00:00:00.000Z', 'provider', '{', '{}'+ )+ `;++ const events = yield* store+ .readFromSequence(first.sequence - 1, last.sequence - first.sequence + 1, last.sequence)+ .pipe(Stream.runCollect);+ assert.deepEqual(+ events.map((event) => event.sequence),+ [first.sequence, last.sequence],+ );+ }),+ );++ it.effect("reads one aggregate through the captured head across pruned global gaps", () =>+ Effect.gen(function* () {+ const store = yield* OrchestrationEventStore;+ const sql = yield* SqlClient.SqlClient;+ const threadId = ThreadId.make("shared-stream-id");+ const first = yield* store.append(messageEvent(threadId, "scoped-first"));+ const pruned = yield* store.append(+ messageEvent(ThreadId.make("pruned-thread"), "pruned-event"),+ );+ const second = yield* store.append(messageEvent(threadId, "scoped-second"));+ // The same stream ID in a different aggregate is not part of this thread.+ // Its invalid JSON must never reach the event decoder.+ yield* sql`+ INSERT INTO orchestration_events (+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,+ actor_kind, payload_json, metadata_json+ ) VALUES (+ 'same-id-project', 'project', ${threadId}, 0, 'project.created',+ '2026-01-01T00:00:00.000Z', 'server', '{', '{'+ ), (+ 'unrelated-invalid', 'thread', 'unrelated-invalid-thread', 0, 'thread.activity-appended',+ '2026-01-01T00:00:00.000Z', 'server', '{', '{'+ )+ `;+ const last = yield* store.append(messageEvent(threadId, "scoped-last"));+ yield* sql`DELETE FROM orchestration_events WHERE sequence = ${pruned.sequence}`;+ yield* store.append(messageEvent(threadId, "after-captured-head"));++ const events = yield* store+ .readAggregateRange({+ aggregateKind: "thread",+ aggregateId: threadId,+ fromSequenceExclusive: first.sequence,+ toSequenceInclusive: last.sequence,+ limit: 100,+ })+ .pipe(Stream.runCollect);+ assert.deepEqual(+ events.map((event) => event.sequence),+ [second.sequence, last.sequence],+ );+ }),+ );++ it.effect("bounds thread replay metadata and counts UTF-8 bytes without decoding payloads", () =>+ Effect.gen(function* () {+ const store = yield* OrchestrationEventStore;+ const sql = yield* SqlClient.SqlClient;+ const rows = yield* sql<{ readonly sequence: number }>`+ INSERT INTO orchestration_events (+ event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at,+ actor_kind, payload_json, metadata_json+ ) VALUES+ ('stats-1', 'thread', 'stats-thread', 0, 'thread.message-sent',+ '2026-01-01T00:00:00.000Z', 'provider', '{"output":"😀"}', '{}'),+ ('stats-unrelated', 'thread', 'another-thread', 0, 'thread.created',+ '2026-01-01T00:00:00.000Z', 'provider', printf('%.*c', 10000, 'x'), '{}'),+ ('stats-2', 'thread', 'stats-thread', 1, 'thread.activity-appended',+ '2026-01-01T00:00:00.000Z', 'provider', '{', '{}'),+ ('stats-other-kind', 'project', 'stats-thread', 0, 'project.deleted',+ '2026-01-01T00:00:00.000Z', 'provider', printf('%.*c', 20000, 'x'), '{}'),+ ('stats-3', 'thread', 'stats-thread', 2, 'thread.deleted',+ '2026-01-01T00:00:00.000Z', 'provider', '{"output":"é"}', '{}'),+ ('stats-4', 'thread', 'stats-thread', 3, 'thread.created',+ '2026-01-01T00:00:00.000Z', 'provider', printf('%.*c', 2000, 'x'), '{}')+ RETURNING sequence+ `;+ const range = {+ aggregateKind: "thread" as const,+ aggregateId: "stats-thread",+ fromSequenceExclusive: 0,+ toSequenceInclusive: rows.at(-1)!.sequence,+ };+ assert.deepEqual(yield* store.getAggregateReplayStats({ ...range, maxEvents: 2 }), {+ eventCount: 3,+ payloadBytes: 33,+ hasCreateEvent: false,+ });+ assert.deepEqual(yield* store.getAggregateReplayStats({ ...range, maxEvents: 10 }), {+ eventCount: 4,+ payloadBytes: 2033,+ hasCreateEvent: true,+ });+ assert.deepEqual(+ yield* store.getAggregateReplayStats({+ ...range,+ toSequenceInclusive: rows[2]!.sequence,+ maxEvents: 10,+ }),+ {+ eventCount: 2,+ payloadBytes: 18,+ hasCreateEvent: false,+ },+ );+ }),+ );++ it.effect("keeps later pages below the captured head when new events are appended", () =>+ Effect.gen(function* () {+ const store = yield* OrchestrationEventStore;+ const threadId = ThreadId.make("paged-thread");+ const persisted = yield* Effect.forEach(+ Array.from({ length: 502 }, (_, index) => index),+ (index) => store.append(messageEvent(threadId, `paged-${index}`)),+ );+ const head = persisted.at(-1)!.sequence;+ let appendedDuringReplay = false;+ const replayed = yield* store+ .readAggregateRange({+ aggregateKind: "thread",+ aggregateId: threadId,+ fromSequenceExclusive: 0,+ toSequenceInclusive: head,+ limit: 1_000,+ })+ .pipe(+ Stream.tap(() => {+ if (appendedDuringReplay) return Effect.void;+ appendedDuringReplay = true;+ return store.append(messageEvent(threadId, "appended-during-replay"));+ }),+ Stream.runCollect,+ );+ assert.deepEqual(+ replayed.map((event) => event.sequence),+ persisted.map((event) => event.sequence),+ );+ const limited = store.readFromSequence(persisted[0]!.sequence, 501.9);+ for (let run = 0; run < 2; run++) {+ assert.deepEqual(+ (yield* Stream.runCollect(limited)).map((event) => event.sequence),+ persisted.slice(1).map((event) => event.sequence),+ );+ }+ assert.deepEqual(yield* Stream.runCollect(store.readFromSequence(0, -1)), []);}),);});++for (const reader of ["all", "aggregate"] as const) {+ it.effect(`releases consumed pages during ${reader} replay`, () =>+ Effect.gen(function* () {+ const store = yield* OrchestrationEventStore;+ const threadId = ThreadId.make(`retention-${reader}`);+ yield* Effect.forEach(+ Array.from({ length: 1_501 }, (_, index) => index),+ (index) => store.append(messageEvent(threadId, `retention-${reader}-${index}`)),+ { discard: true },+ );+ // oxlint-disable-next-line typescript/no-extraneous-class -- Identifies page markers for V8's heap query.+ class ReplayPage {}+ let count = 0;+ const replay =+ reader === "all"+ ? store.readAll()+ : store.readAggregateRange({+ aggregateKind: "thread",+ aggregateId: threadId,+ fromSequenceExclusive: 0,+ toSequenceInclusive: 1_501,+ limit: 1_501,+ });+ yield* Stream.runForEach(replay, (event) =>+ Effect.sync(() => {+ assert.equal(event.sequence, count + 1);+ if (count % 500 === 0) {+ // Count live page markers after full GC, without timing or heap-size thresholds.+ Object.assign(event, { replayPage: new ReplayPage() });+ assert.isAtMost(NodeV8.queryObjects(ReplayPage, { format: "count" }), 1);+ }+ count++;+ }),+ );+ assert.equal(count, 1_501);+ }).pipe(+ Effect.provide(OrchestrationEventStoreLive.pipe(Layer.provide(SqlitePersistenceMemory))),+ ),+ );+}diff --git a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts b/apps/server/src/persistence/Layers/OrchestrationEventStore.tsindex e6b9c1444..1dc5d8d9f 100644--- a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts+++ b/apps/server/src/persistence/Layers/OrchestrationEventStore.ts@@ -97,8 +97,21 @@ const OrchestrationEventPersistedRowSchema = Schema.Struct({const ReadFromSequenceRequestSchema = Schema.Struct({sequenceExclusive: NonNegativeInt,+ toSequenceInclusive: NonNegativeInt,limit: Schema.Number,});+const AggregateReplayRequestSchema = Schema.Struct({+ aggregateKind: OrchestrationAggregateKind,+ aggregateId: Schema.String,+ fromSequenceExclusive: NonNegativeInt,+ toSequenceInclusive: NonNegativeInt,+ limit: Schema.Number,+});+const AggregateReplayStatsRowSchema = Schema.Struct({+ eventCount: Schema.Number,+ payloadBytes: Schema.Number,+ hasCreateEvent: Schema.Number,+});const DEFAULT_READ_FROM_SEQUENCE_LIMIT = 1_000;const READ_PAGE_SIZE = 500;@@ -211,11 +224,63 @@ const makeEventStore = Effect.gen(function* () {metadata_json AS "metadata"FROM orchestration_eventsWHERE sequence > ${request.sequenceExclusive}+ AND sequence <= ${request.toSequenceInclusive}ORDER BY sequence ASCLIMIT ${request.limit}`,});+ const readAggregateEventRows = SqlSchema.findAll({+ Request: AggregateReplayRequestSchema,+ Result: OrchestrationEventPersistedRowSchema,+ execute: (request) =>+ sql`+ SELECT+ sequence,+ event_id AS "eventId",+ event_type AS "type",+ aggregate_kind AS "aggregateKind",+ stream_id AS "aggregateId",+ occurred_at AS "occurredAt",+ command_id AS "commandId",+ causation_event_id AS "causationEventId",+ correlation_id AS "correlationId",+ payload_json AS "payload",+ metadata_json AS "metadata"+ FROM orchestration_events+ WHERE aggregate_kind = ${request.aggregateKind}+ AND stream_id = ${request.aggregateId}+ AND sequence > ${request.fromSequenceExclusive}+ AND sequence <= ${request.toSequenceInclusive}+ ORDER BY sequence ASC+ LIMIT ${request.limit}+ `,+ });++ const readAggregateReplayStats = SqlSchema.findOne({+ Request: AggregateReplayRequestSchema,+ Result: AggregateReplayStatsRowSchema,+ execute: (request) =>+ sql`+ SELECT+ COUNT(*) AS "eventCount",+ COALESCE(SUM(octet_length(payload_json)), 0) AS "payloadBytes",+ COALESCE(MAX(event_type IN (+ 'thread.created', 'project.created'+ )), 0) AS "hasCreateEvent"+ FROM (+ SELECT payload_json, event_type+ FROM orchestration_events+ WHERE aggregate_kind = ${request.aggregateKind}+ AND stream_id = ${request.aggregateId}+ AND sequence > ${request.fromSequenceExclusive}+ AND sequence <= ${request.toSequenceInclusive}+ ORDER BY sequence ASC+ LIMIT ${request.limit}+ )+ `,+ });+const append: OrchestrationEventStoreShape["append"] = (event) =>appendEventRow({eventId: event.eventId,@@ -246,18 +311,18 @@ const makeEventStore = Effect.gen(function* () {const readFromSequence: OrchestrationEventStoreShape["readFromSequence"] = (sequenceExclusive,limit = DEFAULT_READ_FROM_SEQUENCE_LIMIT,+ toSequenceInclusive = Number.MAX_SAFE_INTEGER,) => {const normalizedLimit = Math.max(0, Math.floor(limit));- if (normalizedLimit === 0) {+ if (normalizedLimit === 0 || sequenceExclusive >= toSequenceInclusive) {return Stream.empty;}- const readPage = (- cursor: number,- remaining: number,- ): Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError> =>- Stream.fromEffect(+ return Stream.paginate(+ { cursor: sequenceExclusive, remaining: normalizedLimit },+ ({ cursor, remaining }) =>readEventRowsFromSequence({sequenceExclusive: cursor,+ toSequenceInclusive,limit: Math.min(remaining, READ_PAGE_SIZE),}).pipe(Effect.mapError(@@ -275,26 +340,81 @@ const makeEventStore = Effect.gen(function* () {),),),+ Effect.map((events) => {+ const last = events.at(-1);+ const nextRemaining = remaining - events.length;+ return [+ events,+ last === undefined || nextRemaining <= 0+ ? Option.none()+ : Option.some({ cursor: last.sequence, remaining: nextRemaining }),+ ] as const;+ }),),- ).pipe(- Stream.flatMap((events) => {- if (events.length === 0) {- return Stream.empty;- }- const nextRemaining = remaining - events.length;- if (nextRemaining <= 0) {- return Stream.fromIterable(events);- }- return Stream.concat(- Stream.fromIterable(events),- readPage(events[events.length - 1]!.sequence, nextRemaining),- );- }),- );+ );+ };- return readPage(sequenceExclusive, normalizedLimit);+ const readAggregateRange: OrchestrationEventStoreShape["readAggregateRange"] = (input) => {+ const limit = Math.max(0, Math.floor(input.limit ?? DEFAULT_READ_FROM_SEQUENCE_LIMIT));+ if (limit === 0 || input.fromSequenceExclusive >= input.toSequenceInclusive) {+ return Stream.empty;+ }+ return Stream.paginate(+ { cursor: input.fromSequenceExclusive, remaining: limit },+ ({ cursor, remaining }) =>+ readAggregateEventRows({+ ...input,+ fromSequenceExclusive: cursor,+ limit: Math.min(remaining, READ_PAGE_SIZE),+ }).pipe(+ Effect.mapError(+ toPersistenceSqlOrDecodeError(+ "OrchestrationEventStore.readAggregateRange:query",+ "OrchestrationEventStore.readAggregateRange:decodeRows",+ ),+ ),+ Effect.flatMap((rows) =>+ Effect.forEach(rows, (row) =>+ decodeEvent(row).pipe(+ Effect.mapError(+ toPersistenceDecodeError("OrchestrationEventStore.readAggregateRange:rowToEvent"),+ ),+ ),+ ),+ ),+ Effect.map((events) => {+ const last = events.at(-1);+ const nextRemaining = remaining - events.length;+ return [+ events,+ last === undefined ||+ events.length < READ_PAGE_SIZE ||+ nextRemaining === 0 ||+ last.sequence >= input.toSequenceInclusive+ ? Option.none()+ : Option.some({ cursor: last.sequence, remaining: nextRemaining }),+ ] as const;+ }),+ ),+ );};+ const getAggregateReplayStats: OrchestrationEventStoreShape["getAggregateReplayStats"] = (+ input,+ ) =>+ readAggregateReplayStats({+ ...input,+ limit: Math.max(0, Math.floor(input.maxEvents)) + 1,+ }).pipe(+ Effect.mapError(+ toPersistenceSqlOrDecodeError(+ "OrchestrationEventStore.getAggregateReplayStats:query",+ "OrchestrationEventStore.getAggregateReplayStats:decodeRow",+ ),+ ),+ Effect.map((row) => ({ ...row, hasCreateEvent: row.hasCreateEvent !== 0 })),+ );+const findEventAfter = SqlSchema.findOneOption({Request: HasEventAfterRequestSchema,Result: Schema.Struct({ sequence: Schema.Number }),@@ -324,6 +444,8 @@ const makeEventStore = Effect.gen(function* () {return {append,readFromSequence,+ readAggregateRange,+ getAggregateReplayStats,readAll: () => readFromSequence(0, Number.MAX_SAFE_INTEGER),hasEventAfter,} satisfies OrchestrationEventStoreShape;diff --git a/apps/server/src/persistence/Services/OrchestrationEventStore.ts b/apps/server/src/persistence/Services/OrchestrationEventStore.tsindex 49370b2bb..1ef12acea 100644--- a/apps/server/src/persistence/Services/OrchestrationEventStore.ts+++ b/apps/server/src/persistence/Services/OrchestrationEventStore.ts@@ -16,6 +16,20 @@ import type * as Stream from "effect/Stream";import type { OrchestrationEventStoreError } from "../Errors.ts";+export interface OrchestrationAggregateReplayRange {+ readonly aggregateKind: OrchestrationEvent["aggregateKind"];+ readonly aggregateId: string;+ readonly fromSequenceExclusive: number;+ readonly toSequenceInclusive: number;+}++export interface OrchestrationAggregateReplayStats {+ readonly eventCount: number;+ readonly payloadBytes: number;+ /** A creation in this range does not prove that the aggregate still exists. */+ readonly hasCreateEvent: boolean;+}+/*** OrchestrationEventStoreShape - Service API for orchestration event persistence.*/@@ -37,6 +51,7 @@ export interface OrchestrationEventStoreShape {** @param sequenceExclusive - Sequence cursor (exclusive).* @param limit - Maximum number of events to emit.+ * @param toSequenceInclusive - Optional captured upper sequence boundary.* @returns Stream containing ordered events.** Reads in fixed-size pages and normalizes non-integer/negative limits.@@ -44,8 +59,22 @@ export interface OrchestrationEventStoreShape {readonly readFromSequence: (sequenceExclusive: number,limit?: number,+ toSequenceInclusive?: number,) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;+ /** Read one aggregate through a captured global head, without decoding other streams. */+ readonly readAggregateRange: (+ input: OrchestrationAggregateReplayRange & { readonly limit?: number },+ ) => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;++ /**+ * Measure at most maxEvents + 1 rows without decoding payloads. The extra+ * row tells the caller to use a snapshot instead of a truncated replay.+ */+ readonly getAggregateReplayStats: (+ input: OrchestrationAggregateReplayRange & { readonly maxEvents: number },+ ) => Effect.Effect<OrchestrationAggregateReplayStats, OrchestrationEventStoreError>;+/*** Read all events from the beginning of the stream.*diff --git a/apps/server/src/project/ProjectSetupScriptRunner.test.ts b/apps/server/src/project/ProjectSetupScriptRunner.test.tsindex 0d51e517c..123541f58 100644--- a/apps/server/src/project/ProjectSetupScriptRunner.test.ts+++ b/apps/server/src/project/ProjectSetupScriptRunner.test.ts@@ -32,6 +32,7 @@ const makeProjectionSnapshotQueryLayer = (project: OrchestrationProject) =>getArchivedShellSnapshot: () => Effect.die("unused"),getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 1 }),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: (workspaceRoot) =>Effect.succeed(diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.test.tsindex eb9c17ef1..1d75bbe22 100644--- a/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts+++ b/apps/server/src/provider/Layers/ProviderSessionReaper.test.ts@@ -192,6 +192,7 @@ describe("ProviderSessionReaper", () => {getSnapshotSequence: () =>Effect.succeed({ snapshotSequence: input.readModel.snapshotSequence }),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.die("unused"),getProjectShellById: () => Effect.die("unused"),diff --git a/apps/server/src/routines/RoutineDraftDispatcher.test.ts b/apps/server/src/routines/RoutineDraftDispatcher.test.tsindex 0af8b05be..4fb6dcb16 100644--- a/apps/server/src/routines/RoutineDraftDispatcher.test.ts+++ b/apps/server/src/routines/RoutineDraftDispatcher.test.ts@@ -163,6 +163,7 @@ function deleteHarness(input: {searchThreads: () => Effect.succeed({ matches: [] }),getSnapshotSequence: () => Effect.succeed({ snapshotSequence: 1 }),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getOriginalProjectIdByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.succeed(Option.none()),@@ -185,6 +186,8 @@ function deleteHarness(input: {return { sequence: commands.length };}),readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),streamDomainEvents: Stream.empty,subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),diff --git a/apps/server/src/routines/RuntimeLive.test.ts b/apps/server/src/routines/RuntimeLive.test.tsindex e4bc3efd0..0d870e66f 100644--- a/apps/server/src/routines/RuntimeLive.test.ts+++ b/apps/server/src/routines/RuntimeLive.test.ts@@ -129,6 +129,8 @@ const harness = (Layer.succeed(OrchestrationEngineService, {dispatch: () => Effect.die("unused"),readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),streamDomainEvents: domainEvents,subscribeDomainEvents: Effect.succeed(domainEvents),latestSequence: Effect.succeed(0),diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.tsindex fbb3bd710..475d45b74 100644--- a/apps/server/src/server.test.ts+++ b/apps/server/src/server.test.ts@@ -21,6 +21,7 @@ import {OrchestrationThreadDetailSnapshot,type OrchestrationThreadActivity,type OrchestrationThreadStreamItem,+ type OrchestrationShellStreamItem,type OrchestrationThreadShell,TerminalNotRunningError,type OrchestrationCommand,@@ -116,6 +117,8 @@ import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionReimport { OrchestrationListenerCallbackError } from "./orchestration/Errors.ts";import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";import { SqlitePersistenceMemory } from "./persistence/Layers/Sqlite.ts";+import { OrchestrationEventStoreLive } from "./persistence/Layers/OrchestrationEventStore.ts";+import { OrchestrationEventStore } from "./persistence/Services/OrchestrationEventStore.ts";import * as ProjectionBots from "./persistence/Services/ProjectionBots.ts";import * as ProjectionGroups from "./persistence/Services/ProjectionGroups.ts";import { PersistenceSqlError } from "./persistence/Errors.ts";@@ -200,11 +203,12 @@ const defaultModelSelection = {const makeLiveToolActivityEvent = (sequence: number,+ kind: OrchestrationThreadActivity["kind"] = "tool.updated",): Extract<OrchestrationEvent, { type: "thread.activity-appended" }> => {const activity: OrchestrationThreadActivity = {id: EventId.make(`activity-${sequence}`),tone: "tool",- kind: "tool.updated",+ kind,summary: "Editing app.ts",payload: {itemType: "file_change",@@ -853,6 +857,13 @@ const buildAppUnderTest = (options?: {Layer.mergeAll(Layer.mock(OrchestrationEngine.OrchestrationEngineService)({readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () =>+ Effect.succeed({+ eventCount: 0,+ payloadBytes: 0,+ hasCreateEvent: false,+ }),dispatch: () => Effect.succeed({ sequence: 0 }),streamDomainEvents: Stream.empty,subscribeDomainEvents: Effect.succeed(Stream.empty),@@ -958,6 +969,11 @@ const buildAppUnderTest = (options?: {getThreadDetailById: () => Effect.succeed(Option.none()),getThreadDetailSnapshot: () => Effect.succeed(Option.none()),getCounts: () => Effect.succeed({ projectCount: 0, threadCount: 0 }),+ getEventReplayStats: ({ fromSequenceExclusive, toSequenceInclusive }) =>+ Effect.succeed({+ eventCount: Math.max(0, toSequenceInclusive - fromSequenceExclusive),+ payloadBytes: 0,+ }),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getFirstActiveThreadIdByProjectId: () => Effect.succeed(Option.none()),getThreadCheckpointContext: () => Effect.succeed(Option.none()),@@ -1142,6 +1158,32 @@ const withWsRpcClient = <A, E, R>(f: (client: WsRpcClient) => Effect.Effect<A, E, R>,) => makeWsRpcClient.pipe(Effect.flatMap(f), Effect.provide(wsRpcProtocolLayer(wsUrl)));+const withFirstWsAckHeld = (+ wsUrl: string,+ held: Deferred.Deferred<void>,+ release: Deferred.Deferred<void>,+) => {+ let holdNextAck = true;+ return Layer.effect(RpcClient.Protocol)(+ Effect.map(RpcClient.Protocol, (protocol) =>+ RpcClient.Protocol.of({+ ...protocol,+ send: (clientId, request, transferables) => {+ const send = protocol.send(clientId, request, transferables);+ if (request._tag !== "Ack" || !holdNextAck) {+ return send;+ }+ holdNextAck = false;+ return Deferred.succeed(held, undefined).pipe(+ Effect.andThen(Deferred.await(release)),+ Effect.andThen(send),+ );+ },+ }),+ ),+ ).pipe(Layer.provide(wsRpcProtocolLayer(wsUrl)));+};+const appendSessionCookieToWsUrl = (url: string, sessionCookieHeader: string) => {const isAbsoluteUrl = /^[a-zA-Z][a-zA-Z\d+.-]*:/.test(url);const next = new URL(url, "http://localhost");@@ -4830,6 +4872,75 @@ it.layer(NodeServices.layer)("server router seam", (it) => {}).pipe(Effect.provide(NodeHttpServer.layerTest)),);+ it.effect("subscribeShell keeps sparse replay below the captured head", () =>+ Effect.gen(function* () {+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();+ const now = "2026-01-01T00:00:00.000Z";+ const replayProjectId = ProjectId.make("project-replayed");+ const liveProjectId = ProjectId.make("project-live");+ const projectDeleted = (sequence: number, projectId: ProjectId): OrchestrationEvent => ({+ sequence,+ eventId: EventId.make(`event-project-deleted-${sequence}`),+ aggregateKind: "project",+ aggregateId: projectId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "project.deleted",+ payload: { projectId, deletedAt: now },+ });+ const replayed = projectDeleted(1, replayProjectId);+ const newer = projectDeleted(6, liveProjectId);+ let replayHead: number | undefined;++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(5),+ streamDomainEvents: Stream.fromPubSub(liveEvents),+ readEvents: (_afterSequence, _limit, toSequenceInclusive) => {+ replayHead = toSequenceInclusive;+ return Stream.fromEffect(PubSub.publish(liveEvents, newer)).pipe(+ Stream.flatMap(() =>+ Stream.fromIterable(+ [replayed, newer].filter(+ (event) =>+ toSequenceInclusive === undefined || event.sequence <= toSequenceInclusive,+ ),+ ),+ ),+ );+ },+ },+ projectionSnapshotQuery: {+ getEventReplayStats: () => Effect.succeed({ eventCount: 1, payloadBytes: 100 }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({+ afterSequence: 0,+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ ),+ ),+ );++ assert.deepEqual(+ items.map((item) => ("sequence" in item ? item.sequence : item.kind)),+ [1, 6, "synchronized"],+ );+ assert.equal(replayHead, 5);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );+it.effect("maps delegation lifecycle events into shell upserts", () =>Effect.gen(function* () {const now = "2026-08-31T00:00:00.000Z";@@ -5049,7 +5160,6 @@ it.layer(NodeServices.layer)("server router seam", (it) => {projectionSnapshotQuery: {getThreadDetailSnapshot: () =>Effect.gen(function* () {- yield* Effect.sleep("25 millis");yield* PubSub.publish(liveEvents, messageEvent);return Option.some({ snapshotSequence: 1, thread });}),@@ -5062,14 +5172,19 @@ it.layer(NodeServices.layer)("server router seam", (it) => {withWsRpcClient(wsUrl, (client) =>client[ORCHESTRATION_WS_METHODS.subscribeThread]({threadId: defaultThreadId,- }).pipe(Stream.take(2), Stream.runCollect),+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ ),),- ).pipe(Effect.timeout("2 seconds"));+ );assert.equal(items[0]?.kind, "snapshot");assert.equal(items[1]?.kind, "event");assert.equal(items[1]?.kind === "event" ? items[1].event.sequence : null, 2);- }).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive),+ assert.equal(items[2]?.kind, "synchronized");+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),);it.effect("coalesces buffered live tool updates before websocket delivery", () =>@@ -5113,50 +5228,57 @@ it.layer(NodeServices.layer)("server router seam", (it) => {}).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive),);- it.effect("subscribeThread sends a fresh snapshot instead of replaying a large gap", () =>- Effect.gen(function* () {- let readEventsCalls = 0;- const thread = makeDefaultOrchestrationReadModel().threads[0]!;+ it.effect(+ "subscribeThread sends a fresh snapshot when its event count exceeds the replay limit",+ () =>+ Effect.gen(function* () {+ let readEventsCalls = 0;+ const thread = makeDefaultOrchestrationReadModel().threads[0]!;- yield* buildAppUnderTest({- layers: {- orchestrationEngine: {- // Head is far ahead of the client's afterSequence (gap > 1000).- latestSequence: Effect.succeed(100_000),- readEvents: () =>- Stream.sync(() => {- readEventsCalls += 1;- return {} as OrchestrationEvent;- }),- },- projectionSnapshotQuery: {- getThreadDetailSnapshot: () =>- Effect.succeed(Option.some({ snapshotSequence: 100_000, thread })),+ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(100_000),+ getThreadReplayStats: () =>+ Effect.succeed({+ eventCount: 1_001,+ payloadBytes: 1_000,+ hasCreateEvent: false,+ }),+ readThreadEvents: () =>+ Stream.sync(() => {+ readEventsCalls += 1;+ return {} as OrchestrationEvent;+ }),+ },+ projectionSnapshotQuery: {+ getThreadDetailSnapshot: () =>+ Effect.succeed(Option.some({ snapshotSequence: 100_000, thread })),+ },},- },- });+ });- const wsUrl = yield* getWsServerUrl("/ws");- const items = yield* Effect.scoped(- withWsRpcClient(wsUrl, (client) =>- client[ORCHESTRATION_WS_METHODS.subscribeThread]({- threadId: defaultThreadId,- afterSequence: 5,- requestCompletionMarker: true,- }).pipe(Stream.take(2), Stream.runCollect),- ),- );+ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({+ threadId: defaultThreadId,+ afterSequence: 5,+ requestCompletionMarker: true,+ }).pipe(Stream.take(2), Stream.runCollect),+ ),+ );- const [first, second] = Array.from(items);- // Large gap => fresh thread snapshot, and the global replay never starts.- assert.equal(first?.kind, "snapshot");- if (first?.kind === "snapshot") {- assert.equal(first.snapshot.thread.id, defaultThreadId);- assert.equal(first.snapshot.snapshotSequence, 100_000);- }- assert.equal(second?.kind, "synchronized");- assert.equal(readEventsCalls, 0);- }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ const [first, second] = Array.from(items);+ // Never truncate a thread's replay at the event limit.+ assert.equal(first?.kind, "snapshot");+ if (first?.kind === "snapshot") {+ assert.equal(first.snapshot.thread.id, defaultThreadId);+ assert.equal(first.snapshot.snapshotSequence, 100_000);+ }+ assert.equal(second?.kind, "synchronized");+ assert.equal(readEventsCalls, 0);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),);it.effect("subscribeThread replaces a cursor ahead of the authoritative head", () =>@@ -5168,7 +5290,9 @@ it.layer(NodeServices.layer)("server router seam", (it) => {layers: {orchestrationEngine: {latestSequence: Effect.succeed(5),- readEvents: () =>+ getThreadReplayStats: () =>+ Effect.die("An invalid cursor must not start a replay query"),+ readThreadEvents: () =>Stream.sync(() => {readEventsCalls += 1;return {} as OrchestrationEvent;@@ -5198,7 +5322,9 @@ it.layer(NodeServices.layer)("server router seam", (it) => {it.effect("subscribeThread bounds catch-up replay to the captured head", () =>Effect.gen(function* () {- let replayLimit: number | undefined;+ let replayHead: number | undefined;+ let headSequence = 50;+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();const now = "2026-01-01T00:00:00.000Z";const messageEvent = {sequence: 3,@@ -5226,10 +5352,26 @@ it.layer(NodeServices.layer)("server router seam", (it) => {yield* buildAppUnderTest({layers: {orchestrationEngine: {- latestSequence: Effect.succeed(50),- readEvents: (_afterSequence, limit) => {- replayLimit = limit;- return Stream.make(messageEvent);+ latestSequence: Effect.sync(() => headSequence),+ streamDomainEvents: Stream.fromPubSub(liveEvents),+ getThreadReplayStats: () =>+ Effect.sync(() => {+ headSequence = 100;+ return { eventCount: 1, payloadBytes: 100, hasCreateEvent: false };+ }),+ readThreadEvents: ({ toSequenceInclusive }) => {+ replayHead = toSequenceInclusive;+ return Stream.fromEffect(+ PubSub.publish(liveEvents, {+ ...messageEvent,+ sequence: 51,+ eventId: EventId.make("event-live-after-head"),+ payload: {+ ...messageEvent.payload,+ messageId: MessageId.make("message-live-after-head"),+ },+ }),+ ).pipe(Stream.flatMap(() => Stream.make(messageEvent)));},},},@@ -5242,20 +5384,590 @@ it.layer(NodeServices.layer)("server router seam", (it) => {threadId: defaultThreadId,afterSequence: 0,requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ ),+ ),+ );++ assert.deepEqual(+ items.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),+ [3, 51, "synchronized"],+ );+ assert.equal(replayHead, 50);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect(+ "stops an overflowing thread producer without an ACK and replays the missing events",+ () =>+ Effect.gen(function* () {+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();+ const attached = yield* Deferred.make<void>();+ const detached = yield* Deferred.make<void>();+ const ackHeld = yield* Deferred.make<void>();+ const releaseAck = yield* Deferred.make<void>();+ const firstApplied = yield* Deferred.make<void>();+ const replayStarted = yield* Deferred.make<void>();+ const replayCalls: Array<{ afterSequence: number; headSequence: number }> = [];+ let headSequence = 0;+ let snapshotCalls = 0;+ const message = (sequence: number, text: string) =>+ ({+ sequence,+ eventId: EventId.make(`slow-thread-${sequence}`),+ aggregateKind: "thread",+ aggregateId: defaultThreadId,+ occurredAt: "2026-01-01T00:00:01.000Z",+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.message-sent",+ payload: {+ threadId: defaultThreadId,+ messageId: MessageId.make(`slow-message-${sequence}`),+ role: "assistant",+ text,+ turnId: TurnId.make("turn-edit"),+ streaming: false,+ createdAt: "2026-01-01T00:00:01.000Z",+ updatedAt: "2026-01-01T00:00:01.000Z",+ },+ }) satisfies OrchestrationEvent;+ const events = [+ message(1, "a".repeat(4 * 1024 * 1024)),+ message(2, "b".repeat(4 * 1024 * 1024)),+ makeLiveToolActivityEvent(3, "tool.completed"),+ message(4, "Finished"),+ ] as const;++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.sync(() => headSequence),+ streamDomainEvents: Stream.unwrap(+ Effect.gen(function* () {+ const subscription = yield* PubSub.subscribe(liveEvents);+ yield* Deferred.succeed(attached, undefined);+ return Stream.fromSubscription(subscription);+ }),+ ).pipe(Stream.ensuring(Deferred.succeed(detached, undefined))),+ readThreadEvents: ({ fromSequenceExclusive, toSequenceInclusive }) => {+ replayCalls.push({+ afterSequence: fromSequenceExclusive,+ headSequence: toSequenceInclusive,+ });+ const range = events.filter(+ (event) =>+ event.sequence > fromSequenceExclusive && event.sequence <= toSequenceInclusive,+ );+ return Stream.concat(+ Stream.fromEffect(Deferred.succeed(replayStarted, undefined)).pipe(Stream.drain),+ Stream.fromIterable(range),+ );+ },+ getThreadReplayStats: ({ fromSequenceExclusive, toSequenceInclusive }) => {+ const range = events.filter(+ (event) =>+ event.sequence > fromSequenceExclusive && event.sequence <= toSequenceInclusive,+ );+ return Effect.succeed({+ eventCount: range.length,+ payloadBytes: range.reduce(+ (bytes, event) => bytes + Buffer.byteLength(jsonRequestBody(event.payload)),+ 0,+ ),+ hasCreateEvent: false,+ });+ },+ },+ projectionSnapshotQuery: {+ getThreadDetailSnapshot: () =>+ Effect.sync(() => {+ snapshotCalls += 1;+ return Option.none();+ }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ yield* makeWsRpcClient.pipe(+ Effect.flatMap((client) =>+ Effect.gen(function* () {+ let cursor = 0;+ const received: number[] = [];+ const attempt = yield* client[ORCHESTRATION_WS_METHODS.subscribeThread]({+ threadId: defaultThreadId,+ afterSequence: cursor,+ }).pipe(+ Stream.tap((item) => {+ if (item.kind !== "event") return Effect.void;+ cursor = item.event.sequence;+ received.push(cursor);+ return Deferred.succeed(firstApplied, undefined);+ }),+ Stream.runDrain,+ Effect.result,+ Effect.forkScoped,+ );+ yield* Deferred.await(attached);+ yield* Deferred.await(replayStarted);+ headSequence = 1;+ yield* PubSub.publish(liveEvents, events[0]!);+ yield* Deferred.await(ackHeld);+ yield* Deferred.await(firstApplied);+ headSequence = 4;+ yield* PubSub.publishAll(liveEvents, events.slice(1));++ // This must finish while the server is still waiting for the first+ // batch's ACK. A failed output queue alone would leave PubSub live.+ yield* Deferred.await(detached);+ assert.equal(yield* PubSub.size(liveEvents), 0);+ assert.deepEqual(received, [1]);+ yield* Deferred.succeed(releaseAck, undefined);+ const result = yield* Fiber.join(attempt);+ assertTrue(result._tag === "Failure");+ assert.equal(result.failure._tag, "OrchestrationGetSnapshotError");++ const recovered = yield* client[ORCHESTRATION_WS_METHODS.subscribeThread]({+ threadId: defaultThreadId,+ afterSequence: cursor,+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ );+ assert.deepEqual(+ recovered.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),+ [2, 3, 4, "synchronized"],+ );+ const first = recovered[0];+ assertTrue(first?.kind === "event" && first.event.type === "thread.message-sent");+ assert.equal(first.event.payload.text, events[1]!.payload.text);+ const completed = recovered[1];+ assertTrue(+ completed?.kind === "event" && completed.event.type === "thread.activity-appended",+ );+ assert.equal(completed.event.payload.activity.kind, "tool.completed");+ assert.deepEqual(replayCalls, [+ { afterSequence: 0, headSequence: 0 },+ { afterSequence: 1, headSequence: 4 },+ ]);+ assert.equal(snapshotCalls, 0);+ }),+ ),+ Effect.provide(withFirstWsAckHeld(wsUrl, ackHeld, releaseAck)),+ );+ }).pipe(Effect.scoped, Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("stops an overflowing shell producer without an ACK and recovers deleted entries", () =>+ Effect.gen(function* () {+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();+ const detached = yield* Deferred.make<void>();+ const ackHeld = yield* Deferred.make<void>();+ const releaseAck = yield* Deferred.make<void>();+ let headSequence = 1;+ let snapshotCalls = 0;+ let replayCalls = 0;+ const thread = makeDefaultOrchestrationThreadShell();+ const project = makeDefaultOrchestrationReadModel().projects[0]!;+ const events: OrchestrationEvent[] = Array.from({ length: 1_001 }, (_, index) =>+ makeLiveToolActivityEvent(index + 2, "tool.completed"),+ );+ events.push(+ {+ ...events[0]!,+ sequence: 1_003,+ type: "thread.archived",+ payload: {+ threadId: defaultThreadId,+ archivedAt: "2026-01-01T00:00:02.000Z",+ updatedAt: "2026-01-01T00:00:02.000Z",+ },+ },+ {+ ...events[0]!,+ sequence: 1_004,+ aggregateKind: "project",+ aggregateId: defaultProjectId,+ type: "project.deleted",+ payload: { projectId: defaultProjectId, deletedAt: "2026-01-01T00:00:02.000Z" },+ },+ );+ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.sync(() => headSequence),+ streamDomainEvents: Stream.fromPubSub(liveEvents).pipe(+ Stream.ensuring(Deferred.succeed(detached, undefined)),+ ),+ readEvents: () => {+ replayCalls += 1;+ return Stream.empty;+ },+ },+ projectionSnapshotQuery: {+ getShellSnapshot: () =>+ Effect.sync(() => {+ snapshotCalls += 1;+ return {+ snapshotSequence: headSequence,+ bots: [],+ groups: [],+ delegations: [],+ projects: headSequence === 1 ? [project] : [],+ threads: headSequence === 1 ? [thread] : [],+ updatedAt: "2026-01-01T00:00:02.000Z",+ };+ }),+ },+ },+ });+ const wsUrl = yield* getWsServerUrl("/ws");+ yield* makeWsRpcClient.pipe(+ Effect.flatMap((client) =>+ Effect.gen(function* () {+ const received: OrchestrationShellStreamItem[] = [];+ const attempt = yield* client[ORCHESTRATION_WS_METHODS.subscribeShell]({}).pipe(+ Stream.tap((item) => Effect.sync(() => received.push(item))),+ Stream.runDrain,+ Effect.result,+ Effect.forkScoped,+ );+ yield* Deferred.await(ackHeld);+ headSequence = 1_004;+ yield* PubSub.publishAll(liveEvents, events);+ yield* Deferred.await(detached);+ assert.equal(yield* PubSub.size(liveEvents), 0);+ yield* Deferred.succeed(releaseAck, undefined);+ const result = yield* Fiber.join(attempt);+ assertTrue(result._tag === "Failure");+ assert.equal(result.failure._tag, "OrchestrationGetSnapshotError");+ assert.equal(received.length, 1);+ const initial = received[0];+ assertTrue(initial?.kind === "snapshot");+ assert.equal(initial.snapshot.threads.length, 1);++ const recovered = yield* client[ORCHESTRATION_WS_METHODS.subscribeShell]({+ afterSequence: initial.snapshot.snapshotSequence,+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ );+ const snapshot = recovered[0];+ assertTrue(snapshot?.kind === "snapshot");+ assert.equal(snapshot.snapshot.snapshotSequence, 1_004);+ assert.deepEqual(snapshot.snapshot.projects, []);+ assert.deepEqual(snapshot.snapshot.threads, []);+ assert.deepEqual(recovered[1], { kind: "synchronized" });+ assert.equal(snapshotCalls, 2);+ assert.equal(replayCalls, 0);+ }),+ ),+ Effect.provide(withFirstWsAckHeld(wsUrl, ackHeld, releaseAck)),+ );+ }).pipe(Effect.scoped, Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("subscriptions snapshot instead of decoding an oversized replay range", () =>+ Effect.gen(function* () {+ let readEventsCalls = 0;+ let replayStatsCalls = 0;+ const thread = makeDefaultOrchestrationReadModel().threads[0]!;+ const shell = makeDefaultOrchestrationThreadShell({ id: thread.id });++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(5),+ getThreadReplayStats: () =>+ Effect.sync(() => {+ replayStatsCalls += 1;+ return {+ eventCount: 5,+ payloadBytes: 8 * 1024 * 1024 + 1,+ hasCreateEvent: false,+ };+ }),+ readThreadEvents: () => {+ readEventsCalls += 1;+ return Stream.empty;+ },+ readEvents: () =>+ Stream.sync(() => {+ readEventsCalls += 1;+ return {} as OrchestrationEvent;+ }),+ },+ projectionSnapshotQuery: {+ getEventReplayStats: () =>+ Effect.sync(() => {+ replayStatsCalls += 1;+ return { eventCount: 5, payloadBytes: 8 * 1024 * 1024 + 1 };+ }),+ getThreadDetailSnapshot: () =>+ Effect.succeed(Option.some({ snapshotSequence: 5, thread })),+ getShellSnapshot: () =>+ Effect.succeed({+ snapshotSequence: 5,+ bots: [],+ groups: [],+ delegations: [],+ projects: [],+ threads: [shell],+ updatedAt: "2026-01-01T00:00:00.000Z",+ }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const threadItems = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({+ threadId: thread.id,+ afterSequence: 0,+ requestCompletionMarker: true,+ }).pipe(Stream.take(2), Stream.runCollect),+ ),+ );+ const shellItems = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({+ afterSequence: 0,+ requestCompletionMarker: true,}).pipe(Stream.take(2), Stream.runCollect),),);- const [first, second] = Array.from(items);- assert.equal(first?.kind, "event");- assert.equal(first?.kind === "event" ? first.event.sequence : null, 3);- assert.equal(second?.kind, "synchronized");- // The replay is bounded to the head captured before the read, not- // Number.MAX_SAFE_INTEGER.- assert.equal(replayLimit, 50);+ assert.equal(threadItems[0]?.kind, "snapshot");+ assert.equal(threadItems[1]?.kind, "synchronized");+ assert.equal(shellItems[0]?.kind, "snapshot");+ assert.equal(shellItems[1]?.kind, "synchronized");+ assert.equal(replayStatsCalls, 2);+ assert.equal(readEventsCalls, 0);}).pipe(Effect.provide(NodeHttpServer.layerTest)),);+ it.effect("subscribeThread replays a small thread range across a large global gap", () =>+ Effect.gen(function* () {+ const event = makeLiveToolActivityEvent(99_999, "tool.completed");+ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(100_000),+ getThreadReplayStats: () =>+ Effect.succeed({+ eventCount: 1,+ payloadBytes: Buffer.byteLength(jsonRequestBody(event.payload)),+ hasCreateEvent: false,+ }),+ readThreadEvents: () => Stream.make(event),+ readEvents: () => Stream.die("Thread replay must not read the global log"),+ },+ projectionSnapshotQuery: {+ getEventReplayStats: () => Effect.die("Thread replay must not measure the global log"),+ getThreadDetailSnapshot: () =>+ Effect.die("Unrelated activity must not force a snapshot"),+ },+ },+ });+ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({+ threadId: defaultThreadId,+ afterSequence: 5,+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ ),+ ),+ );+ assert.deepEqual(+ items.map((item) => (item.kind === "event" ? item.event.sequence : item.kind)),+ [99_999, "synchronized"],+ );+ const first = items[0];+ assertTrue(first?.kind === "event" && first.event.type === "thread.activity-appended");+ assert.equal(first.event.payload.activity.kind, "tool.completed");+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("subscribeThread resets cached history when its ID is created again", () =>+ Effect.gen(function* () {+ const thread = {+ ...makeDefaultOrchestrationReadModel().threads[0]!,+ title: "Recreated thread",+ };+ let requestedTurnLimit: number | undefined;+ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(5),+ getThreadReplayStats: () =>+ Effect.succeed({+ eventCount: 3,+ payloadBytes: 1_000,+ hasCreateEvent: true,+ }),+ readThreadEvents: () => Stream.die("A recreated thread must not reuse cached history"),+ },+ projectionSnapshotQuery: {+ getThreadDetailSnapshot: (_threadId, options) => {+ requestedTurnLimit = options?.turnLimit;+ return Effect.succeed(Option.some({ snapshotSequence: 5, thread }));+ },+ },+ },+ });+ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeThread]({+ threadId: defaultThreadId,+ afterSequence: 2,+ turnLimit: 1,+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ ),+ ),+ );+ const first = items[0];+ assertTrue(first?.kind === "snapshot");+ assert.equal(first.snapshot.thread.title, "Recreated thread");+ assert.equal(first.snapshot.snapshotSequence, 5);+ assert.equal(requestedTurnLimit, 1);+ assert.deepEqual(items[1], { kind: "synchronized" });+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ for (const { createBeforeDelete, oversized } of [+ { createBeforeDelete: false, oversized: false },+ { createBeforeDelete: true, oversized: false },+ { createBeforeDelete: true, oversized: true },+ ]) {+ it.effect(+ oversized+ ? "keeps the missing-snapshot error when an absent thread exceeds the replay limit"+ : `synchronizes an absent thread and removes its shell after ${createBeforeDelete ? "creation and deletion" : "deletion"}`,+ () =>+ Effect.gen(function* () {+ const store = yield* OrchestrationEventStore;+ const base = makeLiveToolActivityEvent(0, "tool.completed");+ if (createBeforeDelete) {+ const thread = makeDefaultOrchestrationReadModel().threads[0]!;+ yield* store.append({+ ...base,+ eventId: EventId.make(`create-before-final-delete-${oversized}`),+ type: "thread.created",+ payload: {+ threadId: defaultThreadId,+ projectId: thread.projectId,+ title: thread.title,+ modelSelection: thread.modelSelection,+ runtimeMode: thread.runtimeMode,+ interactionMode: thread.interactionMode,+ branch: thread.branch,+ worktreePath: thread.worktreePath,+ createdAt: thread.createdAt,+ updatedAt: thread.updatedAt,+ },+ });+ }+ if (oversized) {+ yield* Effect.forEach(+ Array.from({ length: 1_000 }, (_, index) => index + 1),+ (sequence) => store.append(makeLiveToolActivityEvent(sequence, "tool.completed")),+ { discard: true },+ );+ }+ const deleted = yield* store.append({+ ...base,+ eventId: EventId.make(`deleted-replay-${createBeforeDelete}-${oversized}`),+ type: "thread.deleted",+ payload: { threadId: defaultThreadId, deletedAt: base.occurredAt },+ });+ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(deleted.sequence),+ getThreadReplayStats: ({ threadId, ...range }) =>+ store.getAggregateReplayStats({+ ...range,+ aggregateKind: "thread",+ aggregateId: threadId,+ }),+ readThreadEvents: ({ threadId, ...range }) =>+ store.readAggregateRange({+ ...range,+ aggregateKind: "thread",+ aggregateId: threadId,+ }),+ readEvents: store.readFromSequence,+ },+ projectionSnapshotQuery: {+ getThreadDetailSnapshot: () => Effect.succeed(Option.none()),+ },+ },+ });+ const wsUrl = yield* getWsServerUrl("/ws");+ yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ Effect.gen(function* () {+ const threadResult = yield* client[ORCHESTRATION_WS_METHODS.subscribeThread]({+ threadId: defaultThreadId,+ afterSequence: 0,+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ Effect.result,+ );+ if (oversized) {+ assertTrue(threadResult._tag === "Failure");+ assert.equal(threadResult.failure._tag, "OrchestrationGetSnapshotError");+ assert.equal(+ threadResult.failure.message,+ `Chat ${defaultThreadId} was not found`,+ );+ return;+ }+ assertTrue(threadResult._tag === "Success");+ assert.deepEqual(threadResult.success, [{ kind: "synchronized" }]);+ const shellItems = yield* client[ORCHESTRATION_WS_METHODS.subscribeShell]({+ afterSequence: 0,+ requestCompletionMarker: true,+ }).pipe(+ Stream.takeUntil((item) => item.kind === "synchronized"),+ Stream.runCollect,+ );+ assert.deepEqual(shellItems, [+ { kind: "thread-removed", sequence: deleted.sequence, threadId: defaultThreadId },+ { kind: "synchronized" },+ ]);+ }),+ ),+ );+ }).pipe(+ Effect.provide(+ Layer.mergeAll(+ OrchestrationEventStoreLive.pipe(Layer.provideMerge(SqlitePersistenceMemory)),+ NodeHttpServer.layerTest,+ ),+ ),+ ),+ );+ }+it.effect("subscribeShell sends a fresh snapshot instead of replaying a large gap", () =>Effect.gen(function* () {let readEventsCalls = 0;diff --git a/apps/server/src/serverRuntimeStartup.reconcile.test.ts b/apps/server/src/serverRuntimeStartup.reconcile.test.tsindex 19d8cf6d6..a38929449 100644--- a/apps/server/src/serverRuntimeStartup.reconcile.test.ts+++ b/apps/server/src/serverRuntimeStartup.reconcile.test.ts@@ -83,6 +83,8 @@ const runReconciliation = (input: {Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, input.directory),Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch: input.dispatch,streamDomainEvents: Stream.empty,subscribeDomainEvents: Effect.succeed(Stream.empty),@@ -289,6 +291,8 @@ it.effect("does not fail startup when the live provider session inventory cannot}),Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch: () => Effect.die("unused"),streamDomainEvents: Stream.empty,subscribeDomainEvents: Effect.succeed(Stream.empty),diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.tsindex 35d6045e7..6a2915058 100644--- a/apps/server/src/serverRuntimeStartup.test.ts+++ b/apps/server/src/serverRuntimeStartup.test.ts@@ -102,6 +102,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threagetArchivedShellSnapshot: () => Effect.die("unused"),getSnapshotSequence: () => Effect.die("unused"),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () =>Effect.succeed(@@ -129,6 +130,8 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threa}),Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch: (command) =>Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(Effect.as({ sequence: 1 }),@@ -163,6 +166,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets creates a project and thread whengetArchivedShellSnapshot: () => Effect.die("unused"),getSnapshotSequence: () => Effect.die("unused"),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.die("unused"),@@ -178,6 +182,8 @@ it.effect("resolveAutoBootstrapWelcomeTargets creates a project and thread when}),Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch: (command) =>Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(Effect.as({ sequence: 1 }),@@ -212,6 +218,7 @@ it.effect(getArchivedShellSnapshot: () => Effect.die("unused"),getSnapshotSequence: () => Effect.die("unused"),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.die("unused"),@@ -227,6 +234,8 @@ it.effect(}),Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch: (command) =>Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(Effect.as({ sequence: 1 }),@@ -262,6 +271,7 @@ it.effect(getArchivedShellSnapshot: () => Effect.die("unused"),getSnapshotSequence: () => Effect.die("unused"),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.die("unused"),getProjectShellById: () => Effect.die("unused"),@@ -277,6 +287,8 @@ it.effect(}),Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch: (command) =>Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(Effect.as({ sequence: 1 }),@@ -316,6 +328,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation fagetArchivedShellSnapshot: () => Effect.die("unused"),getSnapshotSequence: () => Effect.die("unused"),getCounts: () => Effect.die("unused"),+ getEventReplayStats: () => Effect.die("unused"),getOriginalProjectIdByWorkspaceRoot: () => Effect.die("unused"),getActiveProjectByWorkspaceRoot: () => Effect.succeed(Option.none()),getProjectShellById: () => Effect.die("unused"),@@ -331,6 +344,8 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation fa}),Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {readEvents: () => Stream.empty,+ readThreadEvents: () => Stream.empty,+ getThreadReplayStats: () => Effect.die("unused thread replay stats"),dispatch: (command) =>Ref.update(dispatchCalls, (calls) => [...calls, command.type]).pipe(Effect.as({ sequence: 1 }),diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.tsindex ca8904cc9..44650b9c4 100644--- a/apps/server/src/ws.ts+++ b/apps/server/src/ws.ts@@ -103,6 +103,7 @@ import {projectThreadDetailSnapshot,} from "./orchestration/ActivityPayloadProjection.ts";import { makeThreadLiveEventCoalescer } from "./orchestration/ThreadLiveEventCoalescer.ts";+import { makeLiveStreamBudget, type RetainedLiveItem } from "./orchestration/LiveStreamBudget.ts";import {applyAuthenticatedCommandActor,applyKnownGroupPerson,@@ -418,12 +419,13 @@ const PROVIDER_STATUS_DEBOUNCE_MS = 200;// Matches the event store's default page size (DEFAULT_READ_FROM_SEQUENCE_LIMIT).const SHELL_RESUME_MAX_GAP = 1_000;-// Same bound for thread resume. The replay reads the *global* event range and-// filters per-thread afterwards, so a stale cursor far behind the head would-// otherwise decode every intervening event's payload — reconnects with cursors-// hundreds of thousands of events behind have OOM-killed servers on large-// databases. Past this gap the client is reset with a fresh thread snapshot.-const THREAD_RESUME_MAX_GAP = 1_000;+// Thread replay counts only this thread's rows. Busy or pruned unrelated+// streams must not force a full thread snapshot.+const THREAD_RESUME_MAX_EVENTS = 1_000;+// Row count alone does not bound replay memory: a few events with large tool+// payloads can decode to gigabytes. Before replaying, sum the serialized+// payload bytes of the range in SQL and reset with a snapshot past this budget.+const ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES = 8 * 1024 * 1024;function toAuthAccessStreamEvent(change: PairingGrantStore.BootstrapCredentialChange | SessionStore.SessionCredentialChange,@@ -508,6 +510,42 @@ const makeWsRpcLayer = (const routineRepository = yield* RoutineRepository;const routineRuntime = yield* RoutineRuntime;const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;+ const canReplayPersistedRange = Effect.fnUntraced(function* (+ afterSequence: number,+ headSequence: number,+ maxGap: number,+ ) {+ const replayGap = headSequence - afterSequence;+ if (replayGap < 0 || replayGap > maxGap) {+ return false;+ }+ const stats = yield* projectionSnapshotQuery+ .getEventReplayStats({+ fromSequenceExclusive: afterSequence,+ toSequenceInclusive: headSequence,+ })+ .pipe(+ Effect.mapError(+ (cause) =>+ new OrchestrationGetSnapshotError({+ message: "Failed to measure orchestration replay range",+ cause,+ }),+ ),+ );+ if (stats.payloadBytes > ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES) {+ yield* Effect.logDebug("orchestration replay replaced by snapshot", {+ afterSequence,+ headSequence,+ replayGap,+ eventCount: stats.eventCount,+ payloadBytes: stats.payloadBytes,+ payloadBudgetBytes: ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES,+ });+ return false;+ }+ return true;+ });const threadDeletionReactor = yield* ThreadDeletionReactor;// Every command dispatched on this connection carries the connecting// client's origin, including server-generated bootstrap sub-commands:@@ -1261,15 +1299,6 @@ const makeWsRpcLayer = (return output;});- const coalesceShellLiveStream = <E, R>(- stream: Stream.Stream<ShellLiveInput, E, R>,- ): Stream.Stream<OrchestrationShellStreamItem, E, R> =>- stream.pipe(- Stream.groupedWithin(SHELL_COALESCE_MAX_CHUNK, SHELL_COALESCE_WINDOW),- Stream.mapEffect(coalesceShellLiveInputs),- Stream.flatMap((items) => Stream.fromIterable(items)),- );-const dispatchBootstrapTurnStart = (command: Extract<OrchestrationCommand, { type: "thread.turn.start" }>,): Effect.Effect<{ readonly sequence: number }, OrchestrationDispatchCommandError> =>@@ -1875,16 +1904,55 @@ const makeWsRpcLayer = (// sequence but the live subscription is not attached yet). Every// path below emits from this same buffered live tail. Overlapping// events are deduped by sequence on the client.- const liveBuffer = yield* Queue.unbounded<ShellLiveInput>();+ const liveBudget = yield* makeLiveStreamBudget();+ const liveBuffer = yield* Queue.unbounded<+ RetainedLiveItem<ShellLiveInput>,+ OrchestrationGetSnapshotError+ >();+ let liveBufferClosed = false;+ const closeLiveBuffer = (error?: OrchestrationGetSnapshotError) =>+ Effect.gen(function* () {+ if (liveBufferClosed) {+ return;+ }+ liveBufferClosed = true;+ liveBudget.release(yield* Queue.clear(liveBuffer).pipe(Effect.orDie));+ if (error) {+ yield* Queue.fail(liveBuffer, error);+ }+ yield* Queue.shutdown(liveBuffer);+ });+ yield* Effect.addFinalizer(() => closeLiveBuffer());+ yield* liveBudget.failed.pipe(+ Effect.catchTags({ OrchestrationGetSnapshotError: closeLiveBuffer }),+ Effect.forkScoped,+ );yield* Effect.forkScoped(orchestrationEngine.streamDomainEvents.pipe(Stream.runForEach((event) =>- Queue.offer(liveBuffer, { kind: "event" as const, event }),+ liveBudget.retain({ kind: "event" as const, event }, event).pipe(+ Effect.flatMap((item) => Queue.offer(liveBuffer, item)),+ Effect.uninterruptible,+ ),),+ // Stop the PubSub consumer even if RPC delivery is waiting+ // for an ACK and never pulls the failed buffer again.+ Effect.raceFirst(liveBudget.failed),+ Effect.catchTags({ OrchestrationGetSnapshotError: () => Effect.void }),),{ startImmediately: true },);- const bufferedLiveStream = coalesceShellLiveStream(Stream.fromQueue(liveBuffer));+ const coalesceRetainedInputs = (+ items: ReadonlyArray<RetainedLiveItem<ShellLiveInput>>,+ ) =>+ coalesceShellLiveInputs(items.map((item) => item.value)).pipe(+ Effect.flatMap((output) => liveBudget.replace(items, output)),+ );+ const bufferedLiveStream = Stream.fromQueue(liveBuffer).pipe(+ Stream.groupedWithin(SHELL_COALESCE_MAX_CHUNK, SHELL_COALESCE_WINDOW),+ Stream.mapEffect(coalesceRetainedInputs),+ Stream.flatMap((items) => Stream.fromIterable(items)),+ );const loadSnapshot = Effect.all({snapshot: projectionSnapshotQuery.getShellSnapshot(),@@ -1922,18 +1990,21 @@ const makeWsRpcLayer = (// Offer the completion marker into the same queue as live events.// Anything buffered while snapshot/replay work was in flight is// therefore delivered before the client is told it is synchronized.- const synchronizedThenLive =+ const synchronizedThenLive = liveBudget.deliver(input.requestCompletionMarker === true? Stream.concat(Stream.fromEffect(- Queue.offer(liveBuffer, { kind: "synchronized" as const }).pipe(+ liveBudget.retain({ kind: "synchronized" as const }).pipe(+ Effect.flatMap((item) => Queue.offer(liveBuffer, item)),+ Effect.uninterruptible,Effect.andThen(Queue.takeAll(liveBuffer)),- Effect.flatMap(coalesceShellLiveInputs),+ Effect.flatMap(coalesceRetainedInputs),),).pipe(Stream.flatMap((items) => Stream.fromIterable(items))),bufferedLiveStream,)- : bufferedLiveStream;+ : bufferedLiveStream,+ );// When the client already holds a shell snapshot (cached, or loaded// over HTTP) it passes that snapshot's sequence, and we resume by@@ -1951,7 +2022,13 @@ const makeWsRpcLayer = (// is also invalid, so reset it with a snapshot. Send the snapshot// followed by the buffered live tail, exactly as the// no-afterSequence path does.- if (replayGap < 0 || replayGap > SHELL_RESUME_MAX_GAP) {+ if (+ !(yield* canReplayPersistedRange(+ afterSequence,+ headSequence,+ SHELL_RESUME_MAX_GAP,+ ))+ ) {const snapshot = yield* loadSnapshot;return Stream.concat(Stream.make({ kind: "snapshot" as const, snapshot }),@@ -1963,7 +2040,7 @@ const makeWsRpcLayer = (// are already covered by the live subscription, so this bound// cannot chase a moving event-store head or grow the live// buffer indefinitely while waiting for an empty page.- orchestrationEngine.readEvents(afterSequence, replayGap),+ orchestrationEngine.readEvents(afterSequence, replayGap, headSequence),).pipe(Stream.mapError((cause) =>@@ -2033,8 +2110,16 @@ const makeWsRpcLayer = (// Attach live delivery before reading either replay or snapshot state.// Otherwise an event published while the snapshot is loading is lost.const liveBuffer = yield* makeThreadLiveEventCoalescer();- yield* Effect.forkScoped(liveStream.pipe(Stream.runForEach(liveBuffer.offer)));+ yield* Effect.forkScoped(+ liveStream.pipe(+ Stream.runForEachArray(liveBuffer.offerAll),+ Effect.raceFirst(liveBuffer.failed),+ Effect.catchTags({ OrchestrationGetSnapshotError: () => Effect.void }),+ ),+ { startImmediately: true },+ );const bufferedLiveStream = liveBuffer.stream;+ let replayOnMissingSnapshot: typeof bufferedLiveStream | undefined;// When the client already loaded the snapshot over HTTP it passes// that snapshot's sequence, and we resume the live subscription by@@ -2049,22 +2134,41 @@ const makeWsRpcLayer = (// catch-up followed by the buffered/ongoing live events. Overlapping// events are deduped by sequence on the client.//- // The replay is bounded to the projection head captured below. The- // catch-up range is normally tiny (a fresh HTTP snapshot sequence),- // but a stale cached cursor can sit hundreds of thousands of global- // events behind — replaying that decodes every intervening event- // (including every other thread's tool payloads) only to discard- // almost all of them, which has OOM-killed servers on large- // databases. A truncated replay would silently drop this thread's- // events, so past the gap cap we reset the client with a fresh- // thread snapshot instead, exactly like subscribeShell above.+ // Measure only this thread's rows. Global sequence gaps can+ // contain unrelated or pruned streams. Keep an explicit upper+ // bound so events after the captured head stay in the live tail.if (input.afterSequence !== undefined) {const afterSequence = input.afterSequence;const headSequence = yield* orchestrationEngine.latestSequence;- const replayGap = headSequence - afterSequence;- if (replayGap >= 0 && replayGap <= THREAD_RESUME_MAX_GAP) {+ const range = {+ threadId: input.threadId,+ fromSequenceExclusive: afterSequence,+ toSequenceInclusive: headSequence,+ };+ const replayStats =+ afterSequence > headSequence+ ? null+ : yield* orchestrationEngine+ .getThreadReplayStats({+ ...range,+ maxEvents: THREAD_RESUME_MAX_EVENTS,+ })+ .pipe(+ Effect.mapError(+ (cause) =>+ new OrchestrationGetSnapshotError({+ message: `Failed to measure chat ${input.threadId} replay range`,+ cause,+ }),+ ),+ );+ if (+ replayStats !== null &&+ replayStats.eventCount <= THREAD_RESUME_MAX_EVENTS &&+ replayStats.payloadBytes <= ORCHESTRATION_REPLAY_PAYLOAD_BUDGET_BYTES+ ) {const catchUpStream = orchestrationEngine- .readEvents(afterSequence, replayGap)+ .readThreadEvents({ ...range, limit: THREAD_RESUME_MAX_EVENTS }).pipe(Stream.filter(isThisThreadDetailEvent),Stream.map((event) => ({@@ -2081,20 +2185,20 @@ const makeWsRpcLayer = ();const afterCatchUp =input.requestCompletionMarker === true- ? Stream.concat(- Stream.fromEffect(- liveBuffer- .offerAndWait({ kind: "synchronized" as const })- .pipe(Effect.andThen(liveBuffer.takeAll)),- ).pipe(Stream.flatMap((items) => Stream.fromIterable(items))),- bufferedLiveStream,+ ? Stream.unwrap(+ liveBuffer+ .offer({ kind: "synchronized" as const })+ .pipe(Effect.as(bufferedLiveStream)),): bufferedLiveStream;- return Stream.concat(catchUpStream, afterCatchUp);+ const replay = Stream.concat(catchUpStream, afterCatchUp);+ if (!replayStats.hasCreateEvent) {+ return replay;+ }+ replayOnMissingSnapshot = replay;}- // Gap too large (or cursor ahead of authoritative state): fall- // through to the snapshot path so the client converges from a- // fresh thread detail instead of an unbounded replay.+ // A recreated thread needs a fresh snapshot if it still exists.+ // Oversized replays and invalid cursors also use the snapshot path.}const snapshot = yield* projectionSnapshotQuery@@ -2117,6 +2221,12 @@ const makeWsRpcLayer = ();if (Option.isNone(snapshot)) {+ // The recreated thread can already be deleted. Preserve the+ // bounded replay and shell removal instead of retrying a+ // snapshot that cannot exist. Oversized ranges still fail.+ if (replayOnMissingSnapshot !== undefined) {+ return replayOnMissingSnapshot;+ }return yield* new OrchestrationGetSnapshotError({message: `Chat ${input.threadId} was not found`,cause: input.threadId,@@ -2125,13 +2235,10 @@ const makeWsRpcLayer = (const afterSnapshot =input.requestCompletionMarker === true- ? Stream.concat(- Stream.fromEffect(- liveBuffer- .offerAndWait({ kind: "synchronized" as const })- .pipe(Effect.andThen(liveBuffer.takeAll)),- ).pipe(Stream.flatMap((items) => Stream.fromIterable(items))),- bufferedLiveStream,+ ? Stream.unwrap(+ liveBuffer+ .offer({ kind: "synchronized" as const })+ .pipe(Effect.as(bufferedLiveStream)),): bufferedLiveStream;return Stream.concat(