Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
perf(orchestration): speed up new-chat propagation and offline catch-up (#4177)
Rusiru Sadathanadb4b2d8a0f23
diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.tsindex 405103450..b00c00e0d 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts@@ -218,6 +218,7 @@ describe("OrchestrationEngine", () => {const runtime = ManagedRuntime.make(layer);const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService));+ expect(await runtime.runPromise(engine.latestSequence)).toBe(7);const result = await runtime.runPromise(engine.dispatch({type: "thread.meta.update",@@ -228,6 +229,7 @@ describe("OrchestrationEngine", () => {);expect(result.sequence).toBe(8);+ expect(await runtime.runPromise(engine.latestSequence)).toBe(8);expect(fullSnapshotReadCount).toBe(0);await runtime.dispose();diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.tsindex 9598d1c3e..19184915a 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts@@ -329,6 +329,11 @@ const makeOrchestrationEngine = Effect.gen(function* () {get streamDomainEvents(): OrchestrationEngineShape["streamDomainEvents"] {return Stream.fromPubSub(eventPubSub);},+ // The command read model's snapshotSequence tracks the latest committed+ // event sequence (updated on the worker fiber). A plain property read is a+ // consistent, committed value — reassignment of `commandReadModel` is+ // atomic on the single-threaded event loop.+ latestSequence: Effect.sync(() => commandReadModel.snapshotSequence),} satisfies OrchestrationEngineShape;});diff --git a/apps/server/src/orchestration/Services/OrchestrationEngine.ts b/apps/server/src/orchestration/Services/OrchestrationEngine.tsindex dc1b5fd30..f8bcfd76a 100644--- a/apps/server/src/orchestration/Services/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Services/OrchestrationEngine.ts@@ -56,6 +56,13 @@ export interface OrchestrationEngineShape {* This is a hot runtime stream (new events only), not a historical replay.*/readonly streamDomainEvents: Stream.Stream<OrchestrationEvent>;++ /**+ * The latest sequence reflected in the engine's authoritative command read+ * model (0 if none). Used to gauge how far behind a resuming client is before+ * choosing between an incremental replay and a fresh projected snapshot.+ */+ readonly latestSequence: Effect.Effect<number, never, never>;}/**diff --git a/apps/server/src/relay/AgentAwarenessRelay.test.ts b/apps/server/src/relay/AgentAwarenessRelay.test.tsindex 9ca17dbc2..0c261c8f2 100644--- a/apps/server/src/relay/AgentAwarenessRelay.test.ts+++ b/apps/server/src/relay/AgentAwarenessRelay.test.ts@@ -470,6 +470,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {readEvents: () => Stream.empty,dispatch: () => Effect.succeed({ sequence: 1 }),streamDomainEvents: Stream.fromQueue(events),+ latestSequence: Effect.succeed(0),} satisfies OrchestrationEngineShape;const snapshotQuery = {@@ -659,6 +660,7 @@ describe.sequential("signRelayAgentActivityPublishProof", () => {readEvents: () => Stream.empty,dispatch: () => Effect.succeed({ sequence: 1 }),streamDomainEvents: Stream.fromQueue(events),+ latestSequence: Effect.succeed(0),} satisfies OrchestrationEngineShape),Layer.succeed(ProjectionSnapshotQuery, {getShellSnapshot: () =>diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.tsindex a0f41cbf0..6122c4981 100644--- a/apps/server/src/server.test.ts+++ b/apps/server/src/server.test.ts@@ -46,6 +46,7 @@ import * as DateTime from "effect/DateTime";import * as Duration from "effect/Duration";import * as Effect from "effect/Effect";import * as FileSystem from "effect/FileSystem";+import * as Fiber from "effect/Fiber";import * as Layer from "effect/Layer";import * as ManagedRuntime from "effect/ManagedRuntime";import * as Option from "effect/Option";@@ -677,6 +678,7 @@ const buildAppUnderTest = (options?: {readEvents: () => Stream.empty,dispatch: () => Effect.succeed({ sequence: 0 }),streamDomainEvents: Stream.empty,+ latestSequence: Effect.succeed(0),...options?.layers?.orchestrationEngine,}),),@@ -5688,7 +5690,6 @@ it.layer(NodeServices.layer)("server router seam", (it) => {projectionSnapshotQuery: {getShellSnapshot: () =>Effect.gen(function* () {- yield* Effect.sleep("25 millis");yield* PubSub.publish(liveEvents, deletedEvent);return {snapshotSequence: 1,@@ -5774,6 +5775,473 @@ it.layer(NodeServices.layer)("server router seam", (it) => {}).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive),);+ it.effect("subscribeShell sends a fresh snapshot instead of replaying a large gap", () =>+ Effect.gen(function* () {+ let readEventsCalls = 0;+ const snapshotThreadId = ThreadId.make("thread-from-snapshot");+ const now = "2026-01-01T00:00:00.000Z";++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ // Head is far ahead of the client's afterSequence (gap > 1000).+ latestSequence: Effect.succeed(100_000),+ readEvents: () =>+ Stream.sync(() => {+ readEventsCalls += 1;+ return {+ sequence: 1,+ eventId: EventId.make("event-should-not-be-read"),+ aggregateKind: "thread",+ aggregateId: snapshotThreadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.created",+ payload: {} as never,+ } satisfies OrchestrationEvent;+ }),+ },+ projectionSnapshotQuery: {+ getShellSnapshot: () =>+ Effect.succeed({+ snapshotSequence: 100_000,+ projects: [],+ threads: [makeDefaultOrchestrationThreadShell({ id: snapshotThreadId })],+ updatedAt: now,+ }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({+ afterSequence: 5,+ requestCompletionMarker: true,+ }).pipe(Stream.take(2), Stream.runCollect),+ ),+ );++ const [first, second] = Array.from(items);+ // Large gap => fresh snapshot, and the unbounded replay is never started.+ assert.equal(first?.kind, "snapshot");+ if (first?.kind === "snapshot") {+ assert.equal(first.snapshot.threads[0]?.id, snapshotThreadId);+ }+ assert.equal(second?.kind, "synchronized");+ assert.equal(readEventsCalls, 0);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("subscribeShell replaces a cursor ahead of the authoritative head", () =>+ Effect.gen(function* () {+ let readEventsCalls = 0;++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(5),+ readEvents: () =>+ Stream.sync(() => {+ readEventsCalls += 1;+ return {} as OrchestrationEvent;+ }),+ },+ projectionSnapshotQuery: {+ getShellSnapshot: () =>+ Effect.succeed({+ snapshotSequence: 5,+ projects: [],+ threads: [],+ updatedAt: "2026-01-01T00:00:00.000Z",+ }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const first = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({ afterSequence: 10 }).pipe(+ Stream.runHead,+ ),+ ),+ );++ assert.equal(Option.getOrThrow(first).kind, "snapshot");+ assert.equal(readEventsCalls, 0);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("subscribeShell coalesces a per-thread burst without stalling other threads", () =>+ Effect.gen(function* () {+ const busyThreadId = ThreadId.make("thread-busy");+ const newThreadId = ThreadId.make("thread-new");+ const now = "2026-01-01T00:00:00.000Z";+ const shellFetches: Array<string> = [];+ let replayLimit: number | undefined;++ const messageEvent = (sequence: number): OrchestrationEvent =>+ ({+ sequence,+ eventId: EventId.make(`event-${sequence}`),+ aggregateKind: "thread",+ aggregateId: busyThreadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.message-sent",+ payload: {} as never,+ }) satisfies OrchestrationEvent;++ const createdEvent: OrchestrationEvent = {+ sequence: 50,+ eventId: EventId.make("event-created"),+ aggregateKind: "thread",+ aggregateId: newThreadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.created",+ payload: {} as never,+ };++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(50),+ // A burst of message-sent deltas for the busy thread, plus one+ // thread.created for a different thread, all within one batch.+ readEvents: (_afterSequence, limit) => {+ replayLimit = limit;+ return Stream.fromIterable([+ ...Array.from({ length: 20 }, (_unused, index) => messageEvent(index + 1)),+ createdEvent,+ ]);+ },+ },+ projectionSnapshotQuery: {+ getThreadShellById: (threadId) =>+ Effect.sync(() => {+ shellFetches.push(threadId);+ return Option.some(makeDefaultOrchestrationThreadShell({ id: threadId }));+ }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({+ afterSequence: 0,+ requestCompletionMarker: true,+ }).pipe(Stream.take(3), Stream.runCollect),+ ),+ );++ const collected = Array.from(items);+ const upsertedIds = collected.flatMap((item) =>+ item.kind === "thread-upserted" ? [item.thread.id] : [],+ );+ // Both threads surface, and the busy thread's 20-event burst collapses to+ // a single shell refetch (not 20). The new thread is not stuck behind it.+ assert.include(upsertedIds, busyThreadId);+ assert.include(upsertedIds, newThreadId);+ assert.equal(collected[2]?.kind, "synchronized");+ assert.equal(shellFetches.filter((id) => id === busyThreadId).length, 1);+ assert.equal(replayLimit, 50);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("subscribeShell coalesces live bursts after the synchronization marker", () =>+ Effect.gen(function* () {+ const busyThreadId = ThreadId.make("thread-live-busy");+ const newThreadId = ThreadId.make("thread-live-new");+ const now = "2026-01-01T00:00:00.000Z";+ const liveEvents = yield* PubSub.unbounded<OrchestrationEvent>();+ const synchronized = yield* Deferred.make<void>();+ const shellFetches: Array<string> = [];+ const observedLiveThreadIds = new Set<string>();++ const messageEvent = (sequence: number): OrchestrationEvent =>+ ({+ sequence,+ eventId: EventId.make(`event-live-${sequence}`),+ aggregateKind: "thread",+ aggregateId: busyThreadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.message-sent",+ payload: {} as never,+ }) satisfies OrchestrationEvent;++ const createdEvent: OrchestrationEvent = {+ sequence: 50,+ eventId: EventId.make("event-live-created"),+ aggregateKind: "thread",+ aggregateId: newThreadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.created",+ payload: {} as never,+ };++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ streamDomainEvents: Stream.fromPubSub(liveEvents),+ },+ projectionSnapshotQuery: {+ getThreadShellById: (threadId) =>+ Effect.sync(() => {+ shellFetches.push(threadId);+ return Option.some(makeDefaultOrchestrationThreadShell({ id: threadId }));+ }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ Effect.gen(function* () {+ const itemsFiber = yield* withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({+ requestCompletionMarker: true,+ }).pipe(+ Stream.tap((item) =>+ item.kind === "synchronized"+ ? Deferred.succeed(synchronized, undefined).pipe(Effect.ignore)+ : Effect.void,+ ),+ Stream.takeUntil((item) => {+ if (item.kind === "thread-upserted") {+ observedLiveThreadIds.add(item.thread.id);+ }+ return (+ observedLiveThreadIds.has(busyThreadId) && observedLiveThreadIds.has(newThreadId)+ );+ }),+ Stream.runCollect,+ ),+ ).pipe(Effect.forkScoped);++ yield* Deferred.await(synchronized);+ for (const event of [+ ...Array.from({ length: 20 }, (_unused, index) => messageEvent(index + 1)),+ createdEvent,+ ]) {+ yield* PubSub.publish(liveEvents, event);+ }++ return yield* Fiber.join(itemsFiber);+ }),+ ).pipe(Effect.timeout("2 seconds"));++ assert.equal(items[0]?.kind, "snapshot");+ assert.equal(items[1]?.kind, "synchronized");+ const liveUpsertedIds = Array.from(items)+ .slice(2)+ .flatMap((item) => (item.kind === "thread-upserted" ? [item.thread.id] : []));+ assert.include(liveUpsertedIds, busyThreadId);+ assert.include(liveUpsertedIds, newThreadId);+ assert.isBelow(shellFetches.filter((id) => id === busyThreadId).length, 20);+ }).pipe(Effect.provide(NodeHttpServer.layerTest), TestClock.withLive),+ );++ it.effect("subscribeShell coalescing still emits a removal for a deleted thread", () =>+ Effect.gen(function* () {+ const goneThreadId = ThreadId.make("thread-gone");+ const now = "2026-01-01T00:00:00.000Z";++ const makeThreadEvent = (+ sequence: number,+ type: "thread.deleted" | "thread.message-sent",+ ): OrchestrationEvent =>+ ({+ sequence,+ eventId: EventId.make(`event-${sequence}`),+ aggregateKind: "thread",+ aggregateId: goneThreadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type,+ payload: type === "thread.deleted" ? { threadId: goneThreadId, deletedAt: now } : {},+ }) as OrchestrationEvent;++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(2),+ // A thread.deleted followed, within the same coalescing window, by a+ // later refetchable event for the same thread. The later event wins+ // coalescing; its shell refetch returns none (the row is gone), which+ // must still surface a removal rather than be swallowed.+ readEvents: () =>+ Stream.fromIterable([+ makeThreadEvent(1, "thread.deleted"),+ makeThreadEvent(2, "thread.message-sent"),+ ]),+ },+ projectionSnapshotQuery: {+ getThreadShellById: () => Effect.succeed(Option.none()),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({ afterSequence: 0 }).pipe(+ Stream.take(1),+ Stream.runCollect,+ ),+ ),+ );++ const [first] = Array.from(items);+ assert.equal(first?.kind, "thread-removed");+ assert.equal(first?.kind === "thread-removed" ? first.threadId : null, goneThreadId);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("subscribeShell retries a transient shell projection refetch failure", () =>+ Effect.gen(function* () {+ const threadId = ThreadId.make("thread-transient-refetch");+ const now = "2026-01-01T00:00:00.000Z";+ let attempts = 0;++ const event: OrchestrationEvent = {+ sequence: 1,+ eventId: EventId.make("event-transient-refetch"),+ aggregateKind: "thread",+ aggregateId: threadId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.message-sent",+ payload: {} as never,+ };++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(1),+ readEvents: () => Stream.make(event),+ },+ projectionSnapshotQuery: {+ getThreadShellById: () =>+ Effect.suspend(() => {+ attempts += 1;+ return attempts === 1+ ? Effect.fail(+ new PersistenceSqlError({+ operation: "test.shell-refetch",+ detail: "transient failure",+ }),+ )+ : Effect.succeed(+ Option.some(makeDefaultOrchestrationThreadShell({ id: threadId })),+ );+ }),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({ afterSequence: 0 }).pipe(+ Stream.take(1),+ Stream.runCollect,+ ),+ ),+ );++ const [first] = Array.from(items);+ assert.equal(first?.kind, "thread-upserted");+ assert.equal(first?.kind === "thread-upserted" ? first.thread.id : null, threadId);+ assert.equal(attempts, 2);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );++ it.effect("subscribeShell coalescing still removes a project after a trailing update", () =>+ Effect.gen(function* () {+ const projectId = ProjectId.make("project-gone");+ const now = "2026-01-01T00:00:00.000Z";++ const makeProjectEvent = (+ sequence: number,+ type: "project.deleted" | "project.meta-updated",+ ): OrchestrationEvent =>+ ({+ sequence,+ eventId: EventId.make(`event-project-${sequence}`),+ aggregateKind: "project",+ aggregateId: projectId,+ occurredAt: now,+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type,+ payload:+ type === "project.deleted"+ ? { projectId, deletedAt: now }+ : { projectId, title: "Still deleted", updatedAt: now },+ }) as OrchestrationEvent;++ yield* buildAppUnderTest({+ layers: {+ orchestrationEngine: {+ latestSequence: Effect.succeed(2),+ readEvents: () =>+ Stream.fromIterable([+ makeProjectEvent(1, "project.deleted"),+ makeProjectEvent(2, "project.meta-updated"),+ ]),+ },+ projectionSnapshotQuery: {+ getProjectShellById: () => Effect.succeed(Option.none()),+ },+ },+ });++ const wsUrl = yield* getWsServerUrl("/ws");+ const items = yield* Effect.scoped(+ withWsRpcClient(wsUrl, (client) =>+ client[ORCHESTRATION_WS_METHODS.subscribeShell]({ afterSequence: 0 }).pipe(+ Stream.take(1),+ Stream.runCollect,+ ),+ ),+ );++ const [first] = Array.from(items);+ assert.equal(first?.kind, "project-removed");+ assert.equal(first?.kind === "project-removed" ? first.projectId : null, projectId);+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),+ );+it.effect("enriches replayed project events with repository identity metadata", () =>Effect.gen(function* () {const repositoryIdentity = {diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.tsindex 1a331bc71..b8102bda9 100644--- a/apps/server/src/serverRuntimeStartup.test.ts+++ b/apps/server/src/serverRuntimeStartup.test.ts@@ -168,6 +168,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threaEffect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provide(NodeServices.layer),);@@ -211,6 +212,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets creates a project and thread whenEffect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provide(NodeServices.layer),);@@ -260,6 +262,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation faEffect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provideService(Crypto.Crypto, {...crypto,diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.tsindex 41c29fc33..08b1770a0 100644--- a/apps/server/src/ws.ts+++ b/apps/server/src/ws.ts@@ -36,6 +36,7 @@ import {OrchestrationGetSnapshotError,OrchestrationGetTurnDiffError,ORCHESTRATION_WS_METHODS,+ type ProjectId,type ProjectEntriesFailure,type ProjectFileFailure,type ProjectFileOperation,@@ -276,6 +277,13 @@ function isThreadDetailEvent(event: OrchestrationEvent): event is Extract<const PROVIDER_STATUS_DEBOUNCE_MS = 200;+// When a resuming client's cursor is more than this many events behind the+// current head, skip the per-event catch-up replay and send a fresh shell+// snapshot instead. Replaying each intervening event costs a shell refetch;+// past this gap a single O(active-threads) snapshot is cheaper and bounded.+// Matches the event store's default page size (DEFAULT_READ_FROM_SEQUENCE_LIMIT).+const SHELL_RESUME_MAX_GAP = 1_000;+const RPC_REQUIRED_SCOPE = new Map<string, AuthEnvironmentScope>([[ORCHESTRATION_WS_METHODS.dispatchCommand, AuthOrchestrationOperateScope],[ORCHESTRATION_WS_METHODS.getTurnDiff, AuthOrchestrationReadScope],@@ -621,16 +629,7 @@ const makeWsRpcLayer = (switch (event.type) {case "project.created":case "project.meta-updated":- return projectionSnapshotQuery.getProjectShellById(event.payload.projectId).pipe(- Effect.map((project) =>- Option.map(project, (nextProject) => ({- kind: "project-upserted" as const,- sequence: event.sequence,- project: nextProject,- })),- ),- Effect.orElseSucceed(() => Option.none()),- );+ return projectUpsertOrRemove(event.payload.projectId, event.sequence);case "project.deleted":return Effect.succeed(Option.some({@@ -649,35 +648,192 @@ const makeWsRpcLayer = (}),);case "thread.unarchived":- return projectionSnapshotQuery.getThreadShellById(event.payload.threadId).pipe(- Effect.map((thread) =>- Option.map(thread, (nextThread) => ({- kind: "thread-upserted" as const,- sequence: event.sequence,- thread: nextThread,- })),- ),- Effect.orElseSucceed(() => Option.none()),- );+ return threadUpsertOrRemove(event.payload.threadId, event.sequence);default:if (event.aggregateKind !== "thread") {return Effect.succeed(Option.none());}- return projectionSnapshotQuery- .getThreadShellById(ThreadId.make(event.aggregateId))- .pipe(- Effect.map((thread) =>- Option.map(thread, (nextThread) => ({- kind: "thread-upserted" as const,- sequence: event.sequence,- thread: nextThread,- })),- ),- Effect.orElseSucceed(() => Option.none()),- );+ return threadUpsertOrRemove(ThreadId.make(event.aggregateId), event.sequence);}};+ // Coalescing makes each projection read represent every event for that+ // aggregate in the current window. Retry a typed persistence failure once+ // so a brief read failure cannot strand the shell at its previous state.+ // If both attempts fail, log and drop the stream item; treating an error as+ // a missing row would incorrectly remove a still-active aggregate.+ const retryShellProjectionRead = <A, E>(+ aggregateKind: "project" | "thread",+ aggregateId: string,+ read: Effect.Effect<A, E>,+ ): Effect.Effect<Option.Option<A>, never, never> =>+ read.pipe(+ Effect.retry({ times: 1 }),+ Effect.map(Option.some),+ Effect.tapError((error) =>+ Effect.logWarning("orchestration shell projection refetch failed", {+ aggregateKind,+ aggregateId,+ error,+ }),+ ),+ Effect.orElseSucceed(() => Option.none()),+ );++ const projectUpsertOrRemove = (+ projectId: ProjectId,+ sequence: number,+ ): Effect.Effect<Option.Option<OrchestrationShellStreamEvent>, never, never> =>+ retryShellProjectionRead(+ "project",+ projectId,+ projectionSnapshotQuery.getProjectShellById(projectId),+ ).pipe(+ Effect.map(+ Option.flatMap((project) =>+ Option.match(project, {+ onNone: () =>+ Option.some<OrchestrationShellStreamEvent>({+ kind: "project-removed" as const,+ sequence,+ projectId,+ }),+ onSome: (nextProject) =>+ Option.some<OrchestrationShellStreamEvent>({+ kind: "project-upserted" as const,+ sequence,+ project: nextProject,+ }),+ }),+ ),+ ),+ );++ // Refetch a thread's shell and emit an upsert if it is still active, or a+ // `thread-removed` if the projection has no active row for it. Emitting a+ // removal on a `none` (rather than dropping the event) is what keeps+ // coalescing correct: when a burst collapses a `thread.deleted`/`archived`+ // into a later refetchable event for the same thread, the refetch returns+ // `none` for the now-inactive row and this still tells the sidebar to drop+ // it. A `thread-removed` the client does not have is a harmless no-op. The+ // projection commits in the same transaction before the event publishes,+ // so a `none` reliably means the thread is deleted or archived, not+ // not-yet-persisted.+ const threadUpsertOrRemove = (+ threadId: ThreadId,+ sequence: number,+ ): Effect.Effect<Option.Option<OrchestrationShellStreamEvent>, never, never> =>+ retryShellProjectionRead(+ "thread",+ threadId,+ projectionSnapshotQuery.getThreadShellById(threadId),+ ).pipe(+ Effect.map(+ Option.flatMap((thread) =>+ Option.match(thread, {+ onNone: () =>+ Option.some<OrchestrationShellStreamEvent>({+ kind: "thread-removed" as const,+ sequence,+ threadId,+ }),+ onSome: (nextThread) =>+ Option.some<OrchestrationShellStreamEvent>({+ kind: "thread-upserted" as const,+ sequence,+ thread: nextThread,+ }),+ }),+ ),+ ),+ );++ // Turn a batch of domain events into shell stream items, coalescing by+ // aggregate first. `toShellStreamEvent` re-reads the *current* projected+ // shell for an aggregate, so within a batch only the latest event per+ // aggregate matters: a burst of streaming `thread.message-sent` deltas for+ // one thread collapses into a single shell refetch, and an unrelated+ // `thread.created` in the same batch is never stuck behind those DB reads.+ //+ // Input events arrive in ascending sequence; we keep the last (highest+ // sequence) event per aggregate, then re-sort ascending before emitting so+ // the client — which applies shell items strictly by increasing sequence+ // and drops any `sequence <= snapshotSequence` — never skips a coalesced+ // item. The refetch runs with bounded concurrency (order-preserving).+ const SHELL_REFETCH_CONCURRENCY = 8;+ const coalesceShellEvents = (+ events: ReadonlyArray<OrchestrationEvent>,+ ): Effect.Effect<ReadonlyArray<OrchestrationShellStreamEvent>, never, never> =>+ Effect.gen(function* () {+ if (events.length === 0) {+ return [];+ }+ const latestByAggregate = new Map<string, OrchestrationEvent>();+ for (const event of events) {+ latestByAggregate.set(`${event.aggregateKind}:${event.aggregateId}`, event);+ }+ const survivors = Array.from(latestByAggregate.values()).sort(+ (left, right) => left.sequence - right.sequence,+ );+ const shellEvents = yield* Effect.forEach(survivors, toShellStreamEvent, {+ concurrency: SHELL_REFETCH_CONCURRENCY,+ });+ return shellEvents.flatMap((option) => (Option.isSome(option) ? [option.value] : []));+ });++ // Small time/size window over which to coalesce shell events. The window+ // bounds the worst-case added latency for a brand-new thread to appear in+ // the sidebar (imperceptible), while collapsing high-frequency streaming+ // traffic so it can't serialize the shell stream behind per-event DB reads.+ const SHELL_COALESCE_WINDOW = Duration.millis(50);+ const SHELL_COALESCE_MAX_CHUNK = 512;+ const coalesceShellStream = <E, R>(+ stream: Stream.Stream<OrchestrationEvent, E, R>,+ ): Stream.Stream<OrchestrationShellStreamEvent, E, R> =>+ stream.pipe(+ Stream.groupedWithin(SHELL_COALESCE_MAX_CHUNK, SHELL_COALESCE_WINDOW),+ Stream.mapEffect(coalesceShellEvents),+ Stream.flatMap((items) => Stream.fromIterable(items)),+ );++ type ShellLiveInput =+ | { readonly kind: "event"; readonly event: OrchestrationEvent }+ | { readonly kind: "synchronized" };++ // A completion marker is queued alongside raw live events so it cannot+ // overtake an event still waiting in the coalescing window. Split each+ // batch at markers and coalesce only the event segments on either side.+ const coalesceShellLiveInputs = (+ inputs: ReadonlyArray<ShellLiveInput>,+ ): Effect.Effect<ReadonlyArray<OrchestrationShellStreamItem>, never, never> =>+ Effect.gen(function* () {+ const output: Array<OrchestrationShellStreamItem> = [];+ let pendingEvents: Array<OrchestrationEvent> = [];++ for (const input of inputs) {+ if (input.kind === "event") {+ pendingEvents.push(input.event);+ continue;+ }++ output.push(...(yield* coalesceShellEvents(pendingEvents)));+ pendingEvents = [];+ output.push({ kind: "synchronized" });+ }++ output.push(...(yield* coalesceShellEvents(pendingEvents)));+ return output;+ });++ const coalesceShellLiveStream = <E, R>(+ stream: Stream.Stream<ShellLiveInput, E, R>,+ ): Stream.Stream<OrchestrationShellStreamItem, E, R> =>+ stream.pipe(+ Stream.groupedWithin(SHELL_COALESCE_MAX_CHUNK, SHELL_COALESCE_WINDOW),+ Stream.mapEffect(coalesceShellLiveInputs),+ Stream.flatMap((items) => Stream.fromIterable(items)),+ );+const dispatchBootstrapTurnStart = (command: Extract<OrchestrationCommand, { type: "thread.turn.start" }>,): Effect.Effect<{ readonly sequence: number }, OrchestrationDispatchCommandError> =>@@ -1068,68 +1224,29 @@ const makeWsRpcLayer = (observeRpcStreamEffect(ORCHESTRATION_WS_METHODS.subscribeShell,Effect.gen(function* () {- const liveStream = orchestrationEngine.streamDomainEvents.pipe(- Stream.mapEffect(toShellStreamEvent),- Stream.flatMap((event) =>- Option.isSome(event) ? Stream.succeed(event.value) : Stream.empty,+ // Coalesce the live shell stream per aggregate over a small window+ // so bursts of high-frequency events (streaming message deltas,+ // activity appends) collapse into a single shell refetch and never+ // serialize a brand-new thread's `thread.created` behind hundreds+ // of per-event DB reads. See coalesceShellStream.+ // Attach live delivery into a scope-bound buffer BEFORE loading any+ // snapshot or draining catch-up, otherwise an event published while+ // the snapshot query is in flight is lost (it is past the snapshot's+ // sequence but the live subscription is not attached yet). Every+ // path below emits from this same buffered live tail. Overlapping+ // events are deduped by sequence on the client.+ const liveBuffer = yield* Queue.unbounded<ShellLiveInput>();+ yield* Effect.forkScoped(+ orchestrationEngine.streamDomainEvents.pipe(+ Stream.runForEach((event) =>+ Queue.offer(liveBuffer, { kind: "event" as const, event }),+ ),),+ { startImmediately: true },);+ const bufferedLiveStream = coalesceShellLiveStream(Stream.fromQueue(liveBuffer));- // When the client already holds a shell snapshot (cached, or loaded- // over HTTP) it passes that snapshot's sequence, and we resume by- // replaying shell events after it instead of re-sending the whole- // projects/threads list over the socket. As in the thread path, the- // live subscription is attached (into a scope-bound buffer) before- // draining the catch-up replay so no event published during the- // replay window is lost; overlapping events are deduped by sequence- // on the client. The full range is read (not the store's default- // page limit) since the shell filter runs after reading.- if (input.afterSequence !== undefined) {- const afterSequence = input.afterSequence;- return Stream.unwrap(- Effect.gen(function* () {- const liveBuffer = yield* Queue.unbounded<OrchestrationShellStreamItem>();- yield* Effect.forkScoped(- liveStream.pipe(Stream.runForEach((item) => Queue.offer(liveBuffer, item))),- );- const catchUpStream = orchestrationEngine- .readEvents(afterSequence, Number.MAX_SAFE_INTEGER)- .pipe(- Stream.mapEffect(toShellStreamEvent),- Stream.flatMap((event) =>- Option.isSome(event) ? Stream.succeed(event.value) : Stream.empty,- ),- Stream.mapError(- (cause) =>- new OrchestrationGetSnapshotError({- message: "Failed to replay orchestration shell events",- cause,- }),- ),- );- const afterCatchUp =- input.requestCompletionMarker === true- ? Stream.concat(- Stream.fromEffect(- Queue.offer(liveBuffer, { kind: "synchronized" as const }),- ).pipe(Stream.drain),- Stream.fromQueue(liveBuffer),- )- : Stream.fromQueue(liveBuffer);- return Stream.concat(catchUpStream, afterCatchUp);- }),- );- }-- // The full-snapshot fallback needs the same replay-window safety- // as the resume path: subscribe before loading the projection so- // events published while the snapshot is read are buffered.- const liveBuffer = yield* Queue.unbounded<OrchestrationShellStreamItem>();- yield* Effect.forkScoped(- liveStream.pipe(Stream.runForEach((item) => Queue.offer(liveBuffer, item))),- );- const bufferedLiveStream = Stream.fromQueue(liveBuffer);- const snapshot = yield* projectionSnapshotQuery.getShellSnapshot().pipe(+ const loadSnapshot = projectionSnapshotQuery.getShellSnapshot().pipe(Effect.tapError((cause) =>Effect.logError("orchestration shell snapshot load failed", { cause }),),@@ -1142,21 +1259,70 @@ const makeWsRpcLayer = (),);- const afterSnapshot =+ // Offer the completion marker into the same queue as live events.+ // Anything buffered while snapshot/replay work was in flight is+ // therefore delivered before the client is told it is synchronized.+ const synchronizedThenLive =input.requestCompletionMarker === true? Stream.concat(Stream.fromEffect(- Queue.offer(liveBuffer, { kind: "synchronized" as const }),- ).pipe(Stream.drain),+ Queue.offer(liveBuffer, { kind: "synchronized" as const }).pipe(+ Effect.andThen(Queue.takeAll(liveBuffer)),+ Effect.flatMap(coalesceShellLiveInputs),+ ),+ ).pipe(Stream.flatMap((items) => Stream.fromIterable(items))),bufferedLiveStream,): bufferedLiveStream;++ // When the client already holds a shell snapshot (cached, or loaded+ // over HTTP) it passes that snapshot's sequence, and we resume by+ // replaying shell events after it instead of re-sending the whole+ // projects/threads list over the socket. If the client is too far+ // behind, we fall back to a fresh snapshot instead of an unbounded+ // replay (see below).+ if (input.afterSequence !== undefined) {+ const afterSequence = input.afterSequence;+ const headSequence = yield* orchestrationEngine.latestSequence;+ const replayGap = headSequence - afterSequence;+ // Gap too large: replaying every intervening event (each a shell+ // refetch) is far more expensive than a single O(active-threads)+ // snapshot. A cursor ahead of this engine's authoritative state+ // is also invalid, so reset it with a snapshot. Send the snapshot+ // followed by the buffered live tail, exactly as the+ // no-afterSequence path does.+ if (replayGap < 0 || replayGap > SHELL_RESUME_MAX_GAP) {+ const snapshot = yield* loadSnapshot;+ return Stream.concat(+ Stream.make({ kind: "snapshot" as const, snapshot }),+ synchronizedThenLive,+ );+ }+ const catchUpStream = coalesceShellStream(+ // Replay only through the head captured above. Newer events+ // are already covered by the live subscription, so this bound+ // cannot chase a moving event-store head or grow the live+ // buffer indefinitely while waiting for an empty page.+ orchestrationEngine.readEvents(afterSequence, replayGap),+ ).pipe(+ Stream.mapError(+ (cause) =>+ new OrchestrationGetSnapshotError({+ message: "Failed to replay orchestration shell events",+ cause,+ }),+ ),+ );+ return Stream.concat(catchUpStream, synchronizedThenLive);+ }++ const snapshot = yield* loadSnapshot;return Stream.concat(Stream.make({kind: "snapshot" as const,snapshot,}),- afterSnapshot,+ synchronizedThenLive,);}),{ "rpc.aggregate": "orchestration" },