Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
fix(server): keep attachments until commit and allow first-send retry (#229)
Leoed2e58c82bca
diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.tsindex 6b8cb5f55..3fe25593d 100644--- a/apps/server/integration/OrchestrationEngineHarness.integration.ts+++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts@@ -390,6 +390,7 @@ export const makeOrchestrationIntegrationHarness = (Layer.succeed(ThreadDeletionReactor, {start: () => Effect.void,drain: Effect.void,+ drainThrough: () => Effect.void,}),),);diff --git a/apps/server/src/channels/ChannelRuntime.test.ts b/apps/server/src/channels/ChannelRuntime.test.tsindex 2254a80ab..c503ffae4 100644--- a/apps/server/src/channels/ChannelRuntime.test.ts+++ b/apps/server/src/channels/ChannelRuntime.test.ts@@ -442,6 +442,7 @@ function makeHarness(input: {readEvents: () => Stream.empty,dispatch,streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.sync(() => sequence),} satisfies OrchestrationEngineShape;const dependencies: ChannelRuntimeDependencies = {diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.tsindex 3b1ac0d10..260ec3a4e 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.test.ts@@ -113,6 +113,7 @@ describe("OrchestrationEngine", () => {detail: "historical replay should not be used during bootstrap",}),),+ hasEventAfter: () => Effect.succeed(false),};const projectionSnapshot = {@@ -226,6 +227,7 @@ describe("OrchestrationEngine", () => {Layer.succeed(OrchestrationProjectionPipeline, {bootstrap: Effect.void,projectEvent: () => Effect.void,+ projectEventDeferred: () => Effect.succeed(Effect.void),} satisfies OrchestrationProjectionPipelineShape),),Layer.provide(Layer.succeed(OrchestrationEventStore, eventStore)),@@ -824,6 +826,9 @@ describe("OrchestrationEngine", () => {readAll() {return Stream.fromIterable(events);},+ hasEventAfter() {+ return Effect.succeed(false);+ },};const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {@@ -935,6 +940,8 @@ describe("OrchestrationEngine", () => {}return Effect.void;},+ projectEventDeferred: (event) =>+ flakyProjectionPipeline.projectEvent(event).pipe(Effect.as(Effect.void)),};const runtime = ManagedRuntime.make(@@ -1060,6 +1067,9 @@ describe("OrchestrationEngine", () => {readAll() {return Stream.fromIterable(events);},+ hasEventAfter() {+ return Effect.succeed(false);+ },};let shouldFailProjection = true;@@ -1080,6 +1090,8 @@ describe("OrchestrationEngine", () => {}return Effect.void;},+ projectEventDeferred: (event) =>+ flakyProjectionPipeline.projectEvent(event).pipe(Effect.as(Effect.void)),};const runtime = ManagedRuntime.make(diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.tsindex f7f7cd32c..dc63abf1f 100644--- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts@@ -263,12 +263,14 @@ const makeOrchestrationEngine = Effect.gen(function* () {.withTransaction(Effect.gen(function* () {const committedEvents: OrchestrationEvent[] = [];+ const attachmentCleanups: Effect.Effect<void>[] = [];let nextCommandReadModel = commandReadModel;for (const nextEvent of eventBases) {const savedEvent = yield* eventStore.append(nextEvent);nextCommandReadModel = yield* projectEvent(nextCommandReadModel, savedEvent);- yield* projectionPipeline.projectEvent(savedEvent);+ const cleanup = yield* projectionPipeline.projectEventDeferred(savedEvent);+ attachmentCleanups.push(cleanup);committedEvents.push(savedEvent);}@@ -292,6 +294,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {return {committedEvents,+ attachmentCleanups,lastSequence: lastSavedEvent.sequence,nextCommandReadModel,} as const;@@ -306,6 +309,9 @@ const makeOrchestrationEngine = Effect.gen(function* () {);commandReadModel = committedCommand.nextCommandReadModel;+ for (const cleanup of committedCommand.attachmentCleanups) {+ yield* cleanup;+ }for (const [index, event] of committedCommand.committedEvents.entries()) {yield* PubSub.publish(eventPubSub, event);if (index === 0) {@@ -427,6 +433,7 @@ const makeOrchestrationEngine = Effect.gen(function* () {get streamDomainEvents(): OrchestrationEngineShape["streamDomainEvents"] {return Stream.fromPubSub(eventPubSub);},+ subscribeDomainEvents: PubSub.subscribe(eventPubSub).pipe(Effect.map(Stream.fromSubscription)),// The command read model's snapshotSequence tracks the latest committed// event sequence (updated on the worker fiber). A plain property read is a// consistent, committed value — reassignment of `commandReadModel` isdiff --git a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.tsindex 13524ae85..2b4eeaeeb 100644--- a/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts+++ b/apps/server/src/orchestration/Layers/OrchestrationReactor.test.ts@@ -61,6 +61,7 @@ describe("OrchestrationReactor", () => {return Effect.void;},drain: Effect.void,+ drainThrough: () => Effect.void,}),),),diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.tsindex 52351e644..3a8ef58a9 100644--- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts+++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts@@ -1245,7 +1245,18 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti}case "thread.deleted": {- attachmentSideEffects.deletedThreadIds.add(event.payload.threadId);+ // A draft retry can re-create this id later in the log. During+ // replay the attachment files on disk already belong to that later+ // incarnation, so only an unsuperseded deletion removes them.+ const recreatedLater = yield* eventStore.hasEventAfter({+ aggregateKind: "thread",+ aggregateId: event.payload.threadId,+ type: "thread.created",+ sequenceExclusive: event.sequence,+ });+ if (!recreatedLater) {+ attachmentSideEffects.deletedThreadIds.add(event.payload.threadId);+ }const existingRow = yield* projectionThreadRepository.getById({threadId: event.payload.threadId,});@@ -1397,6 +1408,15 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti"applyThreadMessagesProjection",)(function* (event, attachmentSideEffects) {switch (event.type) {+ // A draft retry re-creates a soft-deleted thread id. Every projector+ // drops its own rows for the old incarnation here so replay from any+ // per-projector cursor rebuilds the new thread without stale history.+ case "thread.created":+ yield* projectionThreadMessageRepository.deleteByThreadId({+ threadId: event.payload.threadId,+ });+ return;+case "thread.message-sent": {if (event.payload.streaming) {const attachments =@@ -1529,6 +1549,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti"applyThreadProposedPlansProjection",)(function* (event, _attachmentSideEffects) {switch (event.type) {+ case "thread.created":+ yield* projectionThreadProposedPlanRepository.deleteByThreadId({+ threadId: event.payload.threadId,+ });+ return;+case "thread.proposed-plan-upserted":yield* projectionThreadProposedPlanRepository.upsert({planId: event.payload.proposedPlan.id,@@ -1580,6 +1606,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti"applyThreadActivitiesProjection",)(function* (event, _attachmentSideEffects) {switch (event.type) {+ case "thread.created":+ yield* projectionThreadActivityRepository.deleteByThreadId({+ threadId: event.payload.threadId,+ });+ return;+case "thread.activity-appended":yield* projectionThreadActivityRepository.upsert({activityId: event.payload.activity.id,@@ -1631,6 +1663,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecticonst applyThreadSessionsProjection: ProjectorDefinition["apply"] = Effect.fn("applyThreadSessionsProjection",)(function* (event, _attachmentSideEffects) {+ if (event.type === "thread.created") {+ yield* projectionThreadSessionRepository.deleteByThreadId({+ threadId: event.payload.threadId,+ });+ return;+ }if (event.type !== "thread.session-set") {return;}@@ -1651,6 +1689,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti"applyThreadTurnsProjection",)(function* (event, _attachmentSideEffects) {switch (event.type) {+ case "thread.created":+ yield* projectionTurnRepository.deleteByThreadId({+ threadId: event.payload.threadId,+ });+ return;+case "thread.turn-start-requested": {yield* projectionTurnRepository.replacePendingTurnStart({threadId: event.payload.threadId,@@ -2003,6 +2047,12 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti"applyPendingApprovalsProjection",)(function* (event, _attachmentSideEffects) {switch (event.type) {+ case "thread.created":+ yield* projectionPendingApprovalRepository.deleteByThreadId({+ threadId: event.payload.threadId,+ });+ return;+case "thread.activity-appended": {const requestId =extractActivityRequestId(event.payload.activity.payload) ??@@ -2132,6 +2182,70 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti},];+ const applyAttachmentSideEffects = Effect.fn("applyAttachmentSideEffects")(+ function* (event: OrchestrationEvent, sideEffects: AttachmentSideEffects) {+ if (+ sideEffects.deletedThreadIds.size === 0 &&+ sideEffects.prunedThreadRelativePaths.size === 0+ ) {+ return;+ }++ const deletedThreadIds = new Set<string>();+ for (const threadId of sideEffects.deletedThreadIds) {+ const recreatedLater = yield* eventStore.hasEventAfter({+ aggregateKind: "thread",+ aggregateId: ThreadId.make(threadId),+ type: "thread.created",+ sequenceExclusive: event.sequence,+ });+ if (!recreatedLater) {+ deletedThreadIds.add(threadId);+ }+ }++ // Later events in the same transaction can add attachment references.+ const prunedThreadRelativePaths = new Map<string, Set<string>>();+ for (const threadId of sideEffects.prunedThreadRelativePaths.keys()) {+ const messages = yield* projectionThreadMessageRepository.listByThreadId({+ threadId: ThreadId.make(threadId),+ });+ prunedThreadRelativePaths.set(+ threadId,+ collectThreadAttachmentRelativePaths(threadId, messages),+ );+ }++ yield* runAttachmentSideEffects({ deletedThreadIds, prunedThreadRelativePaths });+ },+ Effect.provideService(FileSystem.FileSystem, fileSystem),+ Effect.provideService(Path.Path, path),+ Effect.provideService(ServerConfig, serverConfig),+ (effect, event) =>+ effect.pipe(+ Effect.catch((cause) =>+ Effect.logWarning("failed to apply projected attachment side-effects", {+ sequence: event.sequence,+ eventType: event.type,+ cause,+ }),+ ),+ ),+ );++ const applyProjectorForEvent = Effect.fn("applyProjectorForEvent")(function* (+ projector: ProjectorDefinition,+ event: OrchestrationEvent,+ attachmentSideEffects: AttachmentSideEffects,+ ) {+ yield* projector.apply(event, attachmentSideEffects);+ yield* projectionStateRepository.upsert({+ projector: projector.name,+ lastAppliedSequence: event.sequence,+ updatedAt: event.occurredAt,+ });+ });+const runProjectorForEvent = Effect.fn("runProjectorForEvent")(function* (projector: ProjectorDefinition,event: OrchestrationEvent,@@ -2141,28 +2255,8 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjectiprunedThreadRelativePaths: new Map<string, Set<string>>(),};- yield* sql.withTransaction(- projector.apply(event, attachmentSideEffects).pipe(- Effect.flatMap(() =>- projectionStateRepository.upsert({- projector: projector.name,- lastAppliedSequence: event.sequence,- updatedAt: event.occurredAt,- }),- ),- ),- );-- yield* runAttachmentSideEffects(attachmentSideEffects).pipe(- Effect.catch((cause) =>- Effect.logWarning("failed to apply projected attachment side-effects", {- projector: projector.name,- sequence: event.sequence,- eventType: event.type,- cause,- }),- ),- );+ yield* sql.withTransaction(applyProjectorForEvent(projector, event, attachmentSideEffects));+ yield* applyAttachmentSideEffects(event, attachmentSideEffects);});const bootstrapProjector = (projector: ProjectorDefinition) =>@@ -2182,19 +2276,39 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti),);- const projectEvent: OrchestrationProjectionPipelineShape["projectEvent"] = (event) =>- Effect.forEach(projectors, (projector) => runProjectorForEvent(projector, event), {- concurrency: 1,- }).pipe(+ const projectEventDeferred: OrchestrationProjectionPipelineShape["projectEventDeferred"] =+ Effect.fn("projectEventDeferred")(+ function* (event) {+ const attachmentSideEffects: AttachmentSideEffects = {+ deletedThreadIds: new Set<string>(),+ prunedThreadRelativePaths: new Map<string, Set<string>>(),+ };+ yield* sql.withTransaction(+ Effect.forEach(+ projectors,+ (projector) => applyProjectorForEvent(projector, event, attachmentSideEffects),+ { concurrency: 1, discard: true },+ ),+ );+ // Return the cleanup effect so the caller runs it after the outer transaction commits.+ // @effect-diagnostics-next-line returnEffectInGen:off+ return applyAttachmentSideEffects(event, attachmentSideEffects);+ },Effect.provideService(FileSystem.FileSystem, fileSystem),Effect.provideService(Path.Path, path),Effect.provideService(ServerConfig, serverConfig),- Effect.asVoid,Effect.catchTag("SqlError", (sqlError) =>Effect.fail(toPersistenceSqlError("ProjectionPipeline.projectEvent:query")(sqlError)),),);+ const projectEvent: OrchestrationProjectionPipelineShape["projectEvent"] = Effect.fn(+ "projectEvent",+ )(function* (event) {+ const cleanup = yield* projectEventDeferred(event);+ yield* cleanup;+ });+const bootstrap: OrchestrationProjectionPipelineShape["bootstrap"] = Effect.forEach(projectors,bootstrapProjector,@@ -2217,6 +2331,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjectireturn {bootstrap,projectEvent,+ projectEventDeferred,} satisfies OrchestrationProjectionPipelineShape;},);diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.tsindex 8865121ed..d6ff7f8d6 100644--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts@@ -488,6 +488,7 @@ describe("ProviderCommandReactor", () => {get streamDomainEvents() {return engine.streamDomainEvents;},+ subscribeDomainEvents: engine.subscribeDomainEvents,latestSequence: engine.latestSequence,} satisfies OrchestrationEngineService["Service"];}),diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.tsindex bf6d4aa1d..b63bdeac3 100644--- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts+++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts@@ -2052,7 +2052,9 @@ const make = Effect.gen(function* () {}});- yield* forkParked(Stream.runForEach(orchestrationEngine.streamDomainEvents, processEvent));+ // Subscribe before returning, even while event handling waits for server activation.+ const domainEvents = yield* orchestrationEngine.subscribeDomainEvents;+ yield* forkParked(Stream.runForEach(domainEvents, processEvent));// The domain event stream is hot, so work pending before this reactor// starts cannot be resumed. Correlated completions only clear the requestdiff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.tsindex 34b1b995a..16b296f02 100644--- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts+++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.test.ts@@ -1,10 +1,23 @@-import { ThreadId } from "@t3tools/contracts";+import { it as effectIt } from "@effect/vitest";+import { EventId, ThreadId, type OrchestrationEvent } from "@t3tools/contracts";import * as Cause from "effect/Cause";+import * as Deferred from "effect/Deferred";import * as Effect from "effect/Effect";import * as Exit from "effect/Exit";+import * as Layer from "effect/Layer";+import * as PubSub from "effect/PubSub";+import * as Stream from "effect/Stream";import { describe, expect, it } from "vite-plus/test";-import { logCleanupCauseUnlessInterrupted } from "./ThreadDeletionReactor.ts";+import { AgentController } from "../../provider/Services/AgentController.ts";+import { ServerActivation } from "../../serverActivation.ts";+import * as TerminalManager from "../../terminal/Manager.ts";+import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts";+import { ThreadDeletionReactor } from "../Services/ThreadDeletionReactor.ts";+import {+ logCleanupCauseUnlessInterrupted,+ ThreadDeletionReactorLive,+} from "./ThreadDeletionReactor.ts";describe("logCleanupCauseUnlessInterrupted", () => {const threadId = ThreadId.make("thread-deletion-reactor-test");@@ -36,3 +49,104 @@ describe("logCleanupCauseUnlessInterrupted", () => {}});});++describe("ThreadDeletionReactor.start", () => {+ effectIt.effect(+ "queues delete/recreate published while parked and drains cleanup after activation",+ () =>+ Effect.gen(function* () {+ const threadId = ThreadId.make("thread-deletion-parked-recreate");+ const stoppedThreadIds: ThreadId[] = [];+ const closedThreadIds: ThreadId[] = [];+ const activation = yield* Deferred.make<void>();+ const eventPubSub = yield* PubSub.unbounded<OrchestrationEvent>();+ let latestSequence = 0;++ const publish = (event: OrchestrationEvent) =>+ Effect.sync(() => {+ latestSequence = event.sequence;+ }).pipe(Effect.andThen(PubSub.publish(eventPubSub, event)));++ const engine: OrchestrationEngineService["Service"] = {+ readEvents: () => Stream.die("unused"),+ dispatch: () => Effect.die("unused"),+ get streamDomainEvents() {+ return Stream.die(+ "ThreadDeletionReactor.start should subscribe before parking",+ ) as Stream.Stream<OrchestrationEvent>;+ },+ subscribeDomainEvents: PubSub.subscribe(eventPubSub).pipe(+ Effect.map(Stream.fromSubscription),+ ),+ latestSequence: Effect.sync(() => latestSequence),+ };++ yield* Effect.gen(function* () {+ const reactor = yield* ThreadDeletionReactor;+ yield* reactor.start();++ yield* publish({+ sequence: 1,+ eventId: EventId.make("event-parked-deleted"),+ aggregateKind: "thread",+ aggregateId: threadId,+ occurredAt: "2026-01-01T00:00:01.000Z",+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.deleted",+ payload: {+ threadId,+ deletedAt: "2026-01-01T00:00:01.000Z",+ },+ });+ yield* publish({+ sequence: 2,+ eventId: EventId.make("event-parked-created"),+ aggregateKind: "thread",+ aggregateId: threadId,+ occurredAt: "2026-01-01T00:00:02.000Z",+ commandId: null,+ causationEventId: null,+ correlationId: null,+ metadata: {},+ type: "thread.created",+ payload: {} as never,+ });++ expect(stoppedThreadIds).toEqual([]);+ expect(closedThreadIds).toEqual([]);++ yield* Deferred.succeed(activation, undefined);+ yield* reactor.drainThrough(2);++ expect(stoppedThreadIds).toEqual([threadId]);+ expect(closedThreadIds).toEqual([threadId]);+ }).pipe(+ Effect.provideService(ServerActivation, Deferred.await(activation)),+ Effect.provide(+ ThreadDeletionReactorLive.pipe(+ Layer.provide(Layer.succeed(OrchestrationEngineService, engine)),+ Layer.provide(+ Layer.mock(AgentController)({+ stopSession: (input) =>+ Effect.sync(() => {+ stoppedThreadIds.push(input.threadId);+ }),+ }),+ ),+ Layer.provide(+ Layer.mock(TerminalManager.TerminalManager)({+ close: (input) =>+ Effect.sync(() => {+ closedThreadIds.push(ThreadId.make(input.threadId));+ }),+ }),+ ),+ ),+ ),+ );+ }).pipe(Effect.scoped),+ );+});diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.tsindex c4015c259..8a2cf6973 100644--- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts+++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts@@ -4,6 +4,7 @@ import * as Cause from "effect/Cause";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as Stream from "effect/Stream";+import * as SubscriptionRef from "effect/SubscriptionRef";import { AgentController } from "../../provider/Services/AgentController.ts";import * as TerminalManager from "../../terminal/Manager.ts";@@ -80,20 +81,44 @@ const make = Effect.gen(function* () {const worker = yield* makeDrainableWorker(processThreadDeletedSafely);+ // Highest event sequence the subscriber has handed to the worker. Waiting+ // through a successful thread.created sequence covers every deletion that+ // was ahead of that create in the engine queue; the worker drain then covers+ // the in-flight cleanup.+ const seenSequence = yield* SubscriptionRef.make(0);+ const noteSeen = (sequence: number) =>+ SubscriptionRef.update(seenSequence, (seen) => Math.max(seen, sequence));+const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () {+ // Subscribe before returning, even while event handling waits for server+ // activation. A hot stream started after forkParked would miss a delete+ // and recreate that land in that window, and onStart would then advance+ // the watermark to the engine head without queuing that cleanup.+ const domainEvents = yield* orchestrationEngine.subscribeDomainEvents;+ yield* orchestrationEngine.latestSequence.pipe(Effect.flatMap(noteSeen));yield* forkParked(- Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => {- if (event.type !== "thread.deleted") {- return Effect.void;- }- return worker.enqueue(event);- }),+ Stream.runForEach(domainEvents, (event) =>+ (event.type === "thread.deleted" ? worker.enqueue(event) : Effect.void).pipe(+ Effect.andThen(noteSeen(event.sequence)),+ ),+ ),+ );+ });++ const drainThrough: ThreadDeletionReactorShape["drainThrough"] = Effect.fn(+ "ThreadDeletionReactor.drainThrough",+ )(function* (target) {+ yield* SubscriptionRef.changes(seenSequence).pipe(+ Stream.filter((seen) => seen >= target),+ Stream.runHead,);+ yield* worker.drain;});return {start,drain: worker.drain,+ drainThrough,} satisfies ThreadDeletionReactorShape;});diff --git a/apps/server/src/orchestration/Services/OrchestrationEngine.ts b/apps/server/src/orchestration/Services/OrchestrationEngine.tsindex 8515f77ca..fdb22ce64 100644--- a/apps/server/src/orchestration/Services/OrchestrationEngine.ts+++ b/apps/server/src/orchestration/Services/OrchestrationEngine.ts@@ -18,6 +18,7 @@ import type {} from "@t3tools/contracts";import * as Context from "effect/Context";import type * as Effect from "effect/Effect";+import type * as Scope from "effect/Scope";import type * as Stream from "effect/Stream";import type { OrchestrationDispatchError } from "../Errors.ts";@@ -73,6 +74,16 @@ export interface OrchestrationEngineShape {*/readonly streamDomainEvents: Stream.Stream<OrchestrationEvent>;+ /**+ * Acquire a domain-event subscription before starting a consumer.+ * The subscription is ready when this effect returns and closes with the scope.+ */+ readonly subscribeDomainEvents: Effect.Effect<+ Stream.Stream<OrchestrationEvent>,+ never,+ Scope.Scope+ >;+/*** The latest sequence reflected in the engine's authoritative command read* model (0 if none). Used to gauge how far behind a resuming client is beforediff --git a/apps/server/src/orchestration/Services/ProjectionPipeline.ts b/apps/server/src/orchestration/Services/ProjectionPipeline.tsindex 466502e56..167f2bcf3 100644--- a/apps/server/src/orchestration/Services/ProjectionPipeline.ts+++ b/apps/server/src/orchestration/Services/ProjectionPipeline.ts@@ -26,11 +26,20 @@ export interface OrchestrationProjectionPipelineShape {/*** Project a single orchestration event into projection repositories.*- * Projectors are executed sequentially to preserve deterministic ordering.+ * Projectors run sequentially in one transaction. Attachment cleanup runs+ * after that transaction commits.*/readonly projectEvent: (event: OrchestrationEvent,) => Effect.Effect<void, ProjectionRepositoryError>;++ /**+ * Project an event inside a caller's transaction and return its attachment+ * cleanup. Run the returned effect only after the outer transaction commits.+ */+ readonly projectEventDeferred: (+ event: OrchestrationEvent,+ ) => Effect.Effect<Effect.Effect<void>, ProjectionRepositoryError>;}/**diff --git a/apps/server/src/orchestration/Services/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Services/ThreadDeletionReactor.tsindex fd8fcf703..ffa53f876 100644--- a/apps/server/src/orchestration/Services/ThreadDeletionReactor.ts+++ b/apps/server/src/orchestration/Services/ThreadDeletionReactor.ts@@ -27,6 +27,14 @@ export interface ThreadDeletionReactorShape {* Intended for test use to replace timing-sensitive sleeps.*/readonly drain: Effect.Effect<void>;++ /**+ * Resolves once every thread.deleted at or before the supplied event+ * sequence has been handed to the worker and the worker is empty and idle.+ * A successful thread.create sequence is the fence callers use before the+ * new incarnation can own runtime resources.+ */+ readonly drainThrough: (sequence: number) => Effect.Effect<void>;}/**diff --git a/apps/server/src/orchestration/commandInvariants.test.ts b/apps/server/src/orchestration/commandInvariants.test.tsindex 153a81bfd..15391b1bf 100644--- a/apps/server/src/orchestration/commandInvariants.test.ts+++ b/apps/server/src/orchestration/commandInvariants.test.ts@@ -202,4 +202,39 @@ describe("commandInvariants", () => {),).rejects.toThrow("already exists");});++ it("allows recreating a soft-deleted thread id", async () => {+ const deletedReadModel: OrchestrationReadModel = {+ ...readModel,+ threads: [+ {+ ...readModel.threads[0]!,+ deletedAt: now,+ },+ ],+ };++ await Effect.runPromise(+ requireThreadAbsent({+ readModel: deletedReadModel,+ command: {+ type: "thread.create",+ commandId: CommandId.make("cmd-retry-create"),+ threadId: ThreadId.make("thread-1"),+ projectId: ProjectId.make("project-a"),+ title: "retry",+ modelSelection: {+ instanceId: ProviderInstanceId.make("codex"),+ model: "gpt-5-codex",+ },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ branch: null,+ worktreePath: null,+ createdAt: now,+ },+ threadId: ThreadId.make("thread-1"),+ }),+ );+ });});diff --git a/apps/server/src/orchestration/commandInvariants.ts b/apps/server/src/orchestration/commandInvariants.tsindex 040f72483..7b697e0ed 100644--- a/apps/server/src/orchestration/commandInvariants.ts+++ b/apps/server/src/orchestration/commandInvariants.ts@@ -451,7 +451,11 @@ export function requireThreadAbsent(input: {readonly command: OrchestrationCommand;readonly threadId: ThreadId;}): Effect.Effect<void, OrchestrationCommandInvariantError> {- if (!findThreadById(input.readModel, input.threadId)) {+ // Thread deletion is a soft delete and a draft keeps its client-minted id+ // across retries, so only a live row blocks creation. Projectors reset the+ // thread's rows when the id is created again.+ const existing = findThreadById(input.readModel, input.threadId);+ if (existing === undefined || existing.deletedAt !== null) {return Effect.void;}return Effect.fail(diff --git a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts b/apps/server/src/persistence/Layers/OrchestrationEventStore.tsindex 230007b80..e6b9c1444 100644--- a/apps/server/src/persistence/Layers/OrchestrationEventStore.ts+++ b/apps/server/src/persistence/Layers/OrchestrationEventStore.ts@@ -22,6 +22,7 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";import * as SqlSchema from "effect/unstable/sql/SqlSchema";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";+import * as Option from "effect/Option";import * as Schema from "effect/Schema";import * as Stream from "effect/Stream";@@ -63,6 +64,13 @@ const AppendEventRequestSchema = Schema.Struct({metadataJson: EventMetadataFromJsonString,});+const HasEventAfterRequestSchema = Schema.Struct({+ aggregateKind: Schema.String,+ aggregateId: Schema.String,+ type: Schema.String,+ sequenceExclusive: NonNegativeInt,+});+const OrchestrationEventPersistedRowSchema = Schema.Struct({sequence: NonNegativeInt,eventId: EventId,@@ -287,10 +295,37 @@ const makeEventStore = Effect.gen(function* () {return readPage(sequenceExclusive, normalizedLimit);};+ const findEventAfter = SqlSchema.findOneOption({+ Request: HasEventAfterRequestSchema,+ Result: Schema.Struct({ sequence: Schema.Number }),+ execute: (request) =>+ sql`+ SELECT sequence+ FROM orchestration_events+ WHERE aggregate_kind = ${request.aggregateKind}+ AND stream_id = ${request.aggregateId}+ AND event_type = ${request.type}+ AND sequence > ${request.sequenceExclusive}+ LIMIT 1+ `,+ });++ const hasEventAfter: OrchestrationEventStoreShape["hasEventAfter"] = (input) =>+ findEventAfter(input).pipe(+ Effect.map(Option.isSome),+ Effect.mapError(+ toPersistenceSqlOrDecodeError(+ "OrchestrationEventStore.hasEventAfter:query",+ "OrchestrationEventStore.hasEventAfter:decodeRow",+ ),+ ),+ );+return {append,readFromSequence,readAll: () => readFromSequence(0, Number.MAX_SAFE_INTEGER),+ hasEventAfter,} satisfies OrchestrationEventStoreShape;});diff --git a/apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts b/apps/server/src/persistence/Layers/ProjectionPendingApprovals.tsindex 617410dd1..d5631cb0a 100644--- a/apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts+++ b/apps/server/src/persistence/Layers/ProjectionPendingApprovals.ts@@ -106,6 +106,15 @@ const makeProjectionPendingApprovalRepository = Effect.gen(function* () {`,});+ const deleteProjectionPendingApprovalRowsByThread = SqlSchema.void({+ Request: ListProjectionPendingApprovalsInput,+ execute: ({ threadId }) =>+ sql`+ DELETE FROM projection_pending_approvals+ WHERE thread_id = ${threadId}+ `,+ });+const upsert: ProjectionPendingApprovalRepositoryShape["upsert"] = (row) =>upsertProjectionPendingApprovalRow(row).pipe(Effect.mapError(toPersistenceSqlError("ProjectionPendingApprovalRepository.upsert:query")),@@ -143,12 +152,20 @@ const makeProjectionPendingApprovalRepository = Effect.gen(function* () {),);+ const deleteByThreadId: ProjectionPendingApprovalRepositoryShape["deleteByThreadId"] = (input) =>+ deleteProjectionPendingApprovalRowsByThread(input).pipe(+ Effect.mapError(+ toPersistenceSqlError("ProjectionPendingApprovalRepository.deleteByThreadId:query"),+ ),+ );+return {upsert,listByThreadId,countPendingByThreadId,getByRequestId,deleteByRequestId,+ deleteByThreadId,} satisfies ProjectionPendingApprovalRepositoryShape;});diff --git a/apps/server/src/persistence/Services/OrchestrationEventStore.ts b/apps/server/src/persistence/Services/OrchestrationEventStore.tsindex e1835bf09..49370b2bb 100644--- a/apps/server/src/persistence/Services/OrchestrationEventStore.ts+++ b/apps/server/src/persistence/Services/OrchestrationEventStore.ts@@ -52,6 +52,19 @@ export interface OrchestrationEventStoreShape {* @returns Stream containing all stored events.*/readonly readAll: () => Stream.Stream<OrchestrationEvent, OrchestrationEventStoreError>;++ /**+ * Check whether an aggregate has an event of the given type after a sequence.+ *+ * Used during replay to tell whether a later event supersedes the one being+ * applied, without streaming the rest of the log.+ */+ readonly hasEventAfter: (input: {+ readonly aggregateKind: OrchestrationEvent["aggregateKind"];+ readonly aggregateId: string;+ readonly type: OrchestrationEvent["type"];+ readonly sequenceExclusive: number;+ }) => Effect.Effect<boolean, OrchestrationEventStoreError>;}/**diff --git a/apps/server/src/persistence/Services/ProjectionPendingApprovals.ts b/apps/server/src/persistence/Services/ProjectionPendingApprovals.tsindex 197914821..f4cebe92a 100644--- a/apps/server/src/persistence/Services/ProjectionPendingApprovals.ts+++ b/apps/server/src/persistence/Services/ProjectionPendingApprovals.ts@@ -87,6 +87,13 @@ export interface ProjectionPendingApprovalRepositoryShape {readonly deleteByRequestId: (input: DeleteProjectionPendingApprovalInput,) => Effect.Effect<void, ProjectionRepositoryError>;++ /**+ * Delete every pending approval row for a thread.+ */+ readonly deleteByThreadId: (+ input: ListProjectionPendingApprovalsInput,+ ) => Effect.Effect<void, ProjectionRepositoryError>;}/**diff --git a/apps/server/src/routines/RoutineDraftDispatcher.test.ts b/apps/server/src/routines/RoutineDraftDispatcher.test.tsindex 28d78f622..0af8b05be 100644--- a/apps/server/src/routines/RoutineDraftDispatcher.test.ts+++ b/apps/server/src/routines/RoutineDraftDispatcher.test.ts@@ -186,6 +186,7 @@ function deleteHarness(input: {}),readEvents: () => Stream.empty,streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),}),),diff --git a/apps/server/src/routines/RuntimeLive.test.ts b/apps/server/src/routines/RuntimeLive.test.tsindex f1281a295..e4bc3efd0 100644--- a/apps/server/src/routines/RuntimeLive.test.ts+++ b/apps/server/src/routines/RuntimeLive.test.ts@@ -130,6 +130,7 @@ const harness = (dispatch: () => Effect.die("unused"),readEvents: () => Stream.empty,streamDomainEvents: domainEvents,+ subscribeDomainEvents: Effect.succeed(domainEvents),latestSequence: Effect.succeed(0),}),),diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.tsindex 4cc4eebf2..fbb3bd710 100644--- a/apps/server/src/server.test.ts+++ b/apps/server/src/server.test.ts@@ -112,6 +112,7 @@ import * as Keybindings from "./keybindings.ts";import * as ExternalLauncher from "./process/externalLauncher.ts";import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";+import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionReactor.ts";import { OrchestrationListenerCallbackError } from "./orchestration/Errors.ts";import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";import { SqlitePersistenceMemory } from "./persistence/Layers/Sqlite.ts";@@ -473,6 +474,7 @@ const buildAppUnderTest = (options?: {>;terminalManager?: Partial<TerminalManager.TerminalManager["Service"]>;orchestrationEngine?: Partial<OrchestrationEngine.OrchestrationEngineService["Service"]>;+ threadDeletionReactor?: Partial<ThreadDeletionReactor["Service"]>;routineRepository?: Partial<RoutineRepositoryShape>;routineRuntime?: Partial<RoutineRuntimeShape>;projectionSnapshotQuery?: Partial<ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"]>;@@ -853,9 +855,16 @@ const buildAppUnderTest = (options?: {readEvents: () => Stream.empty,dispatch: () => Effect.succeed({ sequence: 0 }),streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),...options?.layers?.orchestrationEngine,}),+ Layer.mock(ThreadDeletionReactor)({+ start: () => Effect.void,+ drain: Effect.void,+ drainThrough: () => Effect.void,+ ...options?.layers?.threadDeletionReactor,+ }),Layer.succeed(RoutineRepository,RoutineRepository.of({@@ -4969,6 +4978,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {layers: {orchestrationEngine: {streamDomainEvents: Stream.fromPubSub(liveEvents),+ subscribeDomainEvents: Effect.succeed(Stream.fromPubSub(liveEvents)),},projectionSnapshotQuery: {getShellSnapshot: () =>@@ -5034,6 +5044,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {layers: {orchestrationEngine: {streamDomainEvents: Stream.fromPubSub(liveEvents),+ subscribeDomainEvents: Effect.succeed(Stream.fromPubSub(liveEvents)),},projectionSnapshotQuery: {getThreadDetailSnapshot: () =>@@ -5070,6 +5081,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {layers: {orchestrationEngine: {streamDomainEvents: Stream.fromPubSub(liveEvents),+ subscribeDomainEvents: Effect.succeed(Stream.fromPubSub(liveEvents)),},projectionSnapshotQuery: {getThreadDetailSnapshot: () =>@@ -5482,6 +5494,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => {layers: {orchestrationEngine: {streamDomainEvents: Stream.fromPubSub(liveEvents),+ subscribeDomainEvents: Effect.succeed(Stream.fromPubSub(liveEvents)),},projectionSnapshotQuery: {getThreadShellById: (threadId) =>diff --git a/apps/server/src/serverRuntimeStartup.reconcile.test.ts b/apps/server/src/serverRuntimeStartup.reconcile.test.tsindex 228d64f7d..19d8cf6d6 100644--- a/apps/server/src/serverRuntimeStartup.reconcile.test.ts+++ b/apps/server/src/serverRuntimeStartup.reconcile.test.ts@@ -85,6 +85,7 @@ const runReconciliation = (input: {readEvents: () => Stream.empty,dispatch: input.dispatch,streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),}),Effect.provide(NodeServices.layer),@@ -290,6 +291,7 @@ it.effect("does not fail startup when the live provider session inventory cannotreadEvents: () => Stream.empty,dispatch: () => Effect.die("unused"),streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),}),Effect.provide(NodeServices.layer),diff --git a/apps/server/src/serverRuntimeStartup.test.ts b/apps/server/src/serverRuntimeStartup.test.tsindex e96dcbb6b..35d6045e7 100644--- a/apps/server/src/serverRuntimeStartup.test.ts+++ b/apps/server/src/serverRuntimeStartup.test.ts@@ -134,6 +134,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets returns existing project and threaEffect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provide(NodeServices.layer),@@ -182,6 +183,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets creates a project and thread whenEffect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provide(NodeServices.layer),@@ -230,6 +232,7 @@ it.effect(Effect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provide(NodeServices.layer),@@ -279,6 +282,7 @@ it.effect(Effect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provide(NodeServices.layer),@@ -332,6 +336,7 @@ it.effect("resolveAutoBootstrapWelcomeTargets preserves typed UUID generation faEffect.as({ sequence: 1 }),),streamDomainEvents: Stream.empty,+ subscribeDomainEvents: Effect.succeed(Stream.empty),latestSequence: Effect.succeed(0),} satisfies OrchestrationEngine.OrchestrationEngineService["Service"]),Effect.provideService(Crypto.Crypto, {diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.tsindex 34fb0d77f..ca8904cc9 100644--- a/apps/server/src/ws.ts+++ b/apps/server/src/ws.ts@@ -115,6 +115,7 @@ import {import { decideCommandSequence } from "./orchestration/decider.ts";import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";+import { ThreadDeletionReactor } from "./orchestration/Services/ThreadDeletionReactor.ts";import * as ProjectionBots from "./persistence/Services/ProjectionBots.ts";import * as ProjectionGroups from "./persistence/Services/ProjectionGroups.ts";import { EntityMemoryRepository } from "./memory/Services/EntityMemoryRepository.ts";@@ -507,6 +508,7 @@ const makeWsRpcLayer = (const routineRepository = yield* RoutineRepository;const routineRuntime = yield* RoutineRuntime;const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;+ const threadDeletionReactor = yield* ThreadDeletionReactor;// Every command dispatched on this connection carries the connecting// client's origin, including server-generated bootstrap sub-commands:// the client's request caused them.@@ -1410,7 +1412,7 @@ const makeWsRpcLayer = (const bootstrapProgram = Effect.gen(function* () {if (bootstrap?.createThread) {- yield* dispatchFromClient({+ const created = yield* dispatchFromClient({type: "thread.create",commandId: yield* serverCommandId("bootstrap-thread-create"),threadId: command.threadId,@@ -1425,6 +1427,11 @@ const makeWsRpcLayer = (worktreePath: bootstrap.createThread.worktreePath,createdAt: bootstrap.createThread.createdAt,});+ // The successful create is a fence in the engine command queue:+ // every delete for the prior incarnation committed before it.+ // Drain through that event before setup or turn start can own+ // terminals and provider sessions under the reused thread id.+ yield* threadDeletionReactor.drainThrough(created.sequence);createdThread = true;}@@ -1512,6 +1519,14 @@ const makeWsRpcLayer = (normalizedCommand.type === "thread.turn.start" && normalizedCommand.bootstrap? dispatchBootstrapTurnStart(normalizedCommand): dispatchFromClient(normalizedCommand).pipe(+ Effect.tap(({ sequence }) =>+ // Returning from thread.create is the handoff point at which+ // clients may start resources for the new incarnation. Use+ // its event sequence as the exact deletion-cleanup fence.+ normalizedCommand.type === "thread.create"+ ? threadDeletionReactor.drainThrough(sequence)+ : Effect.void,+ ),Effect.mapError((cause) =>toDispatchCommandError(cause, "Failed to dispatch orchestration command"),),diff --git a/docs/internals/overview.md b/docs/internals/overview.mdindex 51bea3d65..a7f01a7fc 100644--- a/docs/internals/overview.md+++ b/docs/internals/overview.md@@ -73,7 +73,8 @@ processing is totally ordered. For each envelope `processEnvelope`:3. inside one SQL transaction, appends events to the event store, applies them to the in-memory readmodel via [`projector.ts`][projector], projects them into persisted tables, and writes theaccepted receipt;-4. after commit, swaps in the new read model and publishes committed events to subscribers.+4. after commit, swaps in the new read model, cleans up attachments, and publishes committed events+ to subscribers. Attachment cleanup failures are logged and do not reject committed commands.Because persistence and projection share a transaction, the read model cannot durably disagree withthe event log. On dispatch failure the engine rereads persisted events past the starting sequence and