Skip to content
TypeScript
153 KB
Raw Blame
import * as NodeSqlite from "node:sqlite";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
import {
  CheckpointId,
  CheckpointRef,
  CheckpointScopeId,
  CommandId,
  ContextTransferId,
  EventId,
  MessageId,
  type ModelSelection,
  NodeId,
  type OrchestrationV2AppThread,
  type OrchestrationV2DomainEvent,
  type OrchestrationV2Run,
  type OrchestrationV2ThreadProjection,
  type OrchestrationV2TurnItem,
  ProjectId,
  ProviderDriverKind,
  ProviderInstanceId,
  ProviderSessionId,
  ProviderThreadId,
  RunAttemptId,
  RunId,
  ThreadId,
  TurnItemId,
} from "@t3tools/contracts";
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import * as ConfigProvider from "effect/ConfigProvider";
import * as Queue from "effect/Queue";
import * as Arr from "effect/Array";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import * as Exit from "effect/Exit";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Ref from "effect/Ref";
import * as Scheduler from "effect/Scheduler";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import * as Tracer from "effect/Tracer";
import * as SqlClient from "effect/sql/SqlClient";
import * as Statement from "effect/sql/Statement";

import { vi } from "vite-plus/test";

import * as ServerConfig from "../config.ts";
import * as DatabaseStorage from "../persistence/DatabaseStorage.ts";
import * as ServerSettings from "../serverSettings.ts";
import { LIVE_STREAM_MAX_ITEMS, LiveStreamBufferError } from "./LiveStreamBudget.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
import * as EffectOutbox from "./EffectOutbox.ts";
import * as EffectWorker from "./EffectWorker.ts";
import * as EventSink from "./EventSink.ts";
import * as EventStore from "./EventStore.ts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
import * as ProjectionReplay from "./testkit/ProjectionReplay.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as ProjectStore from "./ProjectStore.ts";
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";

const isLiveStreamBufferError = Schema.is(LiveStreamBufferError);
const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));

const layerDatabase = SqlitePersistence.layerMemory;
const layerEventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(layerDatabase));
const layerProjectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(layerDatabase));
const layerStoresProvided = Layer.mergeAll(
  layerDatabase,
  layerEventStoreProvided,
  layerProjectionStoreProvided,
);
const layerEventSinkProvided = EventSink.layer.pipe(Layer.provide(layerStoresProvided));
const layerEffectOutboxProvided = EffectOutbox.layer.pipe(Layer.provide(layerDatabase));
const layerCommandReceiptStoreProvided = CommandReceiptStore.layer.pipe(
  Layer.provide(layerDatabase),
);
const layerIsolatedOutbox = Layer.fresh(EffectOutbox.layer.pipe(Layer.provideMerge(layerDatabase)));
const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe(
  Layer.provide(layerDatabase),
);
const layerProjectionReplayProvided = ProjectionReplay.layer.pipe(
  Layer.provide(layerStoresProvided),
);
const layerTest = Layer.mergeAll(
  layerStoresProvided,
  layerEventSinkProvided,
  layerEffectOutboxProvided,
  layerCommandReceiptStoreProvided,
  IdAllocator.layer,
  layerProjectionMaintenanceProvided,
  layerProjectionReplayProvided,
);

const providerInstanceId = ProviderInstanceId.make("codex");
const providerDriver = ProviderDriverKind.make("codex");
const modelSelection = {
  instanceId: providerInstanceId,
  model: "gpt-5.4",
} satisfies ModelSelection;

function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread {
  return {
    createdBy: "user",
    creationSource: "web",
    id: threadId,
    projectId: ProjectId.make(`project:${threadId}`),
    title: `Thread ${threadId}`,
    providerInstanceId,
    modelSelection,
    runtimeMode: "full-access",
    interactionMode: "default",
    branch: null,
    worktreePath: null,
    activeProviderThreadId: null,
    lineage: {
      parentThreadId: null,
      relationshipToParent: null,
      rootThreadId: threadId,
    },
    forkedFrom: null,
    createdAt: now,
    updatedAt: now,
    archivedAt: null,
    settledOverride: null,
    settledAt: null,
    lastVisitedAt: null,
    deletedAt: null,
  };
}

function threadCreatedEvent(input: {
  readonly id: string;
  readonly thread: OrchestrationV2AppThread;
  readonly now: DateTime.Utc;
}): OrchestrationV2DomainEvent {
  return {
    id: EventId.make(input.id),
    type: "thread.created",
    threadId: input.thread.id,
    providerInstanceId,
    occurredAt: input.now,
    payload: input.thread,
  };
}

const seedCompaction = Effect.fnUntraced(function* () {
  const sql = yield* SqlClient.SqlClient;
  yield* sql`INSERT INTO orchestration_v2_legacy_imports (
    thread_id, source_updated_at, shell_imported_at, transcript_imported_at, imported_message_count
  ) VALUES ('imported', '2026-10-17', '2026-10-17', '2026-10-17', 0)`;
  yield* sql`WITH RECURSIVE history(n) AS (
    SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 151
  ) INSERT INTO orchestration_events (
    event_id, aggregate_kind, stream_id, stream_version, event_type,
    occurred_at, actor_kind, payload_json, metadata_json, application_event_version
  ) SELECT 'event-' || n, 'thread', 'thread', n,
    CASE WHEN n IN (1, 51, 101) THEN 'message.updated' ELSE 'thread.visited' END,
    '2026-10-17', 'server', '{"id":"message"}', '{}', 2 FROM history`;
  // A pass a day ago marked the whole log; projections cover it.
  yield* sql`INSERT INTO orchestration_v2_projection_metadata (
    projection_name, schema_version, last_sequence, updated_at
  ) VALUES
    ('event-log-prune-mark', 2, 151, '1969-12-30T00:00:00.000Z'),
    ('thread-projections', 2, 151, '1969-12-30T00:00:00.000Z')
  ON CONFLICT(projection_name) DO UPDATE SET last_sequence = excluded.last_sequence`;
  yield* sql`WITH RECURSIVE history(n) AS (
    SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 100
  ) INSERT INTO orchestration_command_receipts (
    rowid, command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
  ) SELECT n * 2, 'command-' || n, 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy' FROM history`;
});

it.effect("discovers on the reader worker and deletes on the writer", () =>
  Effect.gen(function* () {
    const fs = yield* FileSystem.FileSystem;
    const path = yield* Path.Path;
    const filename = path.join(yield* fs.makeTempDirectoryScoped(), "fixture.sqlite");
    yield* Effect.gen(function* () {
      const sql = yield* SqlClient.SqlClient;
      yield* seedCompaction();
      // Establish the reader before recording main-thread statements.
      yield* sql`SELECT 1`.pipe(NodeSqliteClient.readOnly);
      const reads: string[] = [];
      const writes: string[] = [];
      const all = NodeSqlite.StatementSync.prototype.all;
      const run = NodeSqlite.StatementSync.prototype.run;
      yield* Effect.acquireRelease(
        Effect.sync(() => {
          const readSpy = vi
            .spyOn(NodeSqlite.StatementSync.prototype, "all")
            .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
              reads.push(this.sourceSQL);
              return all.apply(this, params);
            });
          const writeSpy = vi
            .spyOn(NodeSqlite.StatementSync.prototype, "run")
            .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
              writes.push(this.sourceSQL);
              return run.apply(this, params);
            });
          return { readSpy, writeSpy };
        }),
        ({ readSpy, writeSpy }) =>
          Effect.sync(() => {
            readSpy.mockRestore();
            writeSpy.mockRestore();
          }),
      );
      const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
      const result = yield* maintenance.pruneEventStore();
      assert.equal(result.deletedEventCount, 150);
      assert.equal(result.deletedReceiptCount, 100);
      // Only the in-transaction check for rows appended during the pass reads on the writer.
      assert.deepEqual(
        reads.filter((query) => /\bSELECT\b/.test(query) && !query.includes("AS prompt_sequence")),
        [],
      );
      assert(writes.some((query) => query.includes("DELETE FROM orchestration_events")));
      assert(writes.some((query) => query.includes("DELETE FROM orchestration_command_receipts")));
    }).pipe(
      Effect.provide(
        ProjectionMaintenance.layer.pipe(
          Layer.provideMerge(SqlitePersistence.layerFromPath(filename)),
        ),
      ),
    );
  }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("paces scheduled compaction while preserving retention state and fixed scan bounds", () =>
  Effect.gen(function* () {
    const sql = yield* SqlClient.SqlClient;
    const clock = yield* Clock.Clock;
    yield* seedCompaction();
    const completed = yield* Deferred.make<void>();
    const pauseStarted = yield* Deferred.make<void>();
    let activeNanos = 0n;
    let eventPages = 0;
    const pauses: number[] = [];
    const compactionSpans: Tracer.Span[] = [];
    const tracer = Tracer.make({
      span(options) {
        const span = new Tracer.NativeSpan(options);
        if (span.name === "ProjectionMaintenance.pruneEventStore") compactionSpans.push(span);
        const end = span.end.bind(span);
        span.end = (time, exit) => {
          end(time, exit);
          const query = span.attributes.get("db.query.text");
          if (typeof query !== "string" || !query.includes("LIMIT")) return;
          if (query.includes("FROM orchestration_events AS event")) {
            eventPages++;
          } else if (!query.includes("FROM orchestration_command_receipts AS receipt")) return;
          // Model a page exceeding the active budget, without wall-clock sleeps.
          activeNanos += 30_000_000n;
        };
        return span;
      },
    });
    yield* Effect.gen(function* () {
      yield* DatabaseStorage.DatabaseStorage;
      yield* TestClock.adjust("1 minute");
      yield* Effect.raceFirst(Deferred.await(pauseStarted), Deferred.await(completed));
      yield* TestClock.adjust("269 millis");
      assert.equal(eventPages, 1);
      assert.deepEqual(pauses, [270]);
      assert.deepEqual(
        compactionSpans.map((span) => span.status._tag),
        ["Started"],
      );
      // Foreground commits must not extend the compactor's initial scan bounds.
      yield* sql`INSERT INTO orchestration_events (
          event_id, aggregate_kind, stream_id, stream_version, event_type,
          occurred_at, actor_kind, payload_json, metadata_json, application_event_version
        ) VALUES ('after-start', 'thread', 'thread', 152, 'thread.visited', '2026-10-17', 'server', '{}', '{}', 2)`;
      yield* sql`INSERT INTO orchestration_command_receipts (
          command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
        ) VALUES ('after-start', 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy')`;
      yield* TestClock.adjust("10 seconds");
      yield* Deferred.await(completed);
    }).pipe(
      Effect.provide(
        DatabaseStorage.layer.pipe(
          Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "compaction-" })),
          Layer.provide(NodeServices.layer),
        ),
      ),
      Effect.withTracer(tracer),
      Effect.provideService(Clock.Clock, {
        ...clock,
        monotonicTimeNanos: Effect.sync(() => activeNanos),
        sleep: (duration) => {
          const millis = Duration.toMillis(duration);
          if (millis < 60_000) {
            pauses.push(millis);
            Deferred.doneUnsafe(pauseStarted, Effect.void);
          } else if (millis === 6 * 60 * 60_000) {
            Deferred.doneUnsafe(completed, Effect.void);
          }
          return clock.sleep(duration);
        },
      }),
    );
    assert.deepEqual(pauses, Array(8).fill(270));
    assert.deepEqual(
      compactionSpans.map((span) => span.status._tag),
      ["Ended"],
    );
    // 151 stopped being the stream head when 152 committed mid-pass; 152 is past the bound.
    assert.deepEqual(yield* sql`SELECT sequence FROM orchestration_events ORDER BY sequence`, [
      { sequence: 152 },
    ]);
    assert.deepEqual(yield* sql`SELECT command_id FROM orchestration_command_receipts`, [
      { command_id: "after-start" },
    ]);
  }).pipe(Effect.scoped, Effect.provide(layerTest)),
);

it.effect(
  "commits an ordered provider batch atomically and publishes only after the outer commit",
  () =>
    Effect.gen(function* () {
      const sink = yield* EventSink.EventSinkV2;
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const store = yield* EventStore.EventStoreV2;
      const now = yield* DateTime.now;
      const thread = makeThread(ThreadId.make("thread:single-stream-batch"), now);
      yield* sink.write({ events: [threadCreatedEvent({ id: "batch-created", thread, now })] });
      const message = (id: string): OrchestrationV2DomainEvent => ({
        id: EventId.make(id),
        type: "message.updated",
        threadId: thread.id,
        occurredAt: now,
        payload: {
          id: MessageId.make("batch-message"),
          threadId: thread.id,
          runId: null,
          nodeId: null,
          createdBy: "agent",
          creationSource: "provider",
          role: "assistant",
          text: id,
          streaming: true,
          attachments: [],
          createdAt: now,
          updatedAt: now,
        },
      });
      const highWater = yield* sink.latestSequence();
      const callerError = { _tag: "CallerError" } as const;
      assert.strictEqual(
        yield* sink.withProviderBatch(Effect.fail(callerError)).pipe(Effect.flip),
        callerError,
      );
      const failed = yield* sink
        .withProviderBatch(
          Effect.gen(function* () {
            yield* sink.write({ events: [message("batch-duplicate")] });
            yield* sink.write({ events: [message("batch-duplicate")] });
          }),
        )
        .pipe(Effect.exit);
      assert.isTrue(Exit.isFailure(failed));
      assert.equal(yield* sink.latestSequence(), highWater);
      assert.isEmpty((yield* projections.getThreadRecords(thread.id, ["messages"])).messages);

      const firstPublished = yield* Deferred.make<void>();
      const writesApplied = yield* Deferred.make<void>();
      const release = yield* Deferred.make<void>();
      const live = yield* sink.stream({ afterSequence: highWater }).pipe(
        Stream.tap(() => Deferred.succeed(firstPublished, undefined)),
        Stream.take(2),
        Stream.runCollect,
        Effect.forkScoped({ startImmediately: true }),
      );
      const batch = yield* sink
        .withProviderBatch(
          Effect.gen(function* () {
            yield* sink.write({ events: [message("batch-first")] });
            yield* sink.write({ events: [message("batch-second")] });
            assert.equal(
              (yield* projections.getThreadRecords(thread.id, ["messages"])).messages[0]?.text,
              "batch-second",
            );
            yield* Deferred.succeed(writesApplied, undefined);
            yield* Deferred.await(release);
          }),
        )
        .pipe(Effect.forkScoped({ startImmediately: true }));
      yield* Deferred.await(writesApplied);
      yield* TestClock.adjust(0);
      assert.isFalse(yield* Deferred.isDone(firstPublished));
      assert.isUndefined(yield* Effect.sync(() => batch.pollUnsafe()));
      yield* Deferred.succeed(release, undefined);
      yield* Fiber.join(batch);
      const published = yield* Fiber.join(live);
      assert.deepEqual(
        published.map((stored) => stored.event.id),
        ["batch-first", "batch-second"],
      );
      assert.deepEqual(
        yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect),
        published,
      );
    }).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped),
);

it.effect.each(
  (["none", "duplicate", "after-write"] as const).flatMap((failure) =>
    [1, 10].map((streams) => ({ failure, streams })),
  ),
)(
  "shares the writer and isolates failure=$failure across $streams independent consumers",
  ({ failure, streams }) => {
    const rollback = failure !== "none";
    const failedIndex = streams - 1;
    const jobError = { _tag: "InvalidUpdate" } as const;
    const writes: number[] = [];
    const tracer = Tracer.make({
      span(options) {
        const span = new Tracer.NativeSpan(options);
        const end = span.end.bind(span);
        span.end = (time, exit) => {
          end(time, exit);
          if (span.name === "orchestrationV2.EventSink.providerBatch" && Exit.isSuccess(exit)) {
            writes.push(Number(span.attributes.get("orchestration_v2.batch_writes")));
          }
        };
        return span;
      },
    });
    return Effect.gen(function* () {
      const sink = yield* EventSink.EventSinkV2;
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const threads = Array.from({ length: streams }, (_, index) =>
        makeThread(ThreadId.make(`shared:${index}`), now),
      );
      yield* sink.write({
        events: threads.map((thread, index) =>
          threadCreatedEvent({ id: `created:${index}`, thread, now }),
        ),
      });
      const highWater = yield* sink.latestSequence();
      const store = yield* EventStore.EventStoreV2;
      const publications = yield* Queue.unbounded<EventId>();
      yield* sink.stream({ afterSequence: highWater }).pipe(
        Stream.runForEach((stored) => Queue.offer(publications, stored.event.id)),
        Effect.forkScoped({ startImmediately: true }),
      );
      const pulled = yield* Deferred.make<void>();
      let arrivals = 0;
      const inputs = yield* Effect.forEach(threads, () =>
        Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>(),
      );
      const consumers = yield* Effect.forEach(inputs, (input, index) =>
        sink
          .consumeProviderEvents(
            Stream.fromQueue(input).pipe(
              Stream.tap(() =>
                ++arrivals === streams ? Deferred.succeed(pulled, undefined) : Effect.void,
              ),
            ),
            (raw) =>
              raw.type === "message.updated"
                ? sink
                    .write({
                      events: [
                        {
                          id: EventId.make(
                            failure === "duplicate" && index === failedIndex
                              ? "created:0"
                              : `shared-event:${index}`,
                          ),
                          threadId: raw.message.threadId,
                          occurredAt: now,
                          type: "message.updated",
                          payload: raw.message,
                        },
                      ],
                    })
                    .pipe(
                      Effect.andThen(
                        failure === "after-write" && index === failedIndex
                          ? Effect.fail(jobError)
                          : Effect.void,
                      ),
                    )
                : Effect.void,
            Effect.succeed(false),
            Effect.succeed(true),
          )
          .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })),
      );
      for (let index = 0; index < threads.length; index++) {
        yield* Queue.offer(inputs[index]!, {
          type: "message.updated",
          driver: ProviderDriverKind.make("codex"),
          message: {
            id: MessageId.make(`message:${index}`),
            threadId: threads[index]!.id,
            runId: null,
            nodeId: null,
            createdBy: "agent",
            creationSource: "provider",
            role: "assistant",
            text: `stream:${index}`,
            streaming: true,
            attachments: [],
            createdAt: now,
            updatedAt: now,
          },
        });
      }
      yield* Deferred.await(pulled);
      yield* TestClock.adjust(0);
      yield* sink.write({ events: [] });
      if (rollback) {
        const exit = yield* Fiber.join(consumers[failedIndex]!);
        assert.isTrue(Exit.isFailure(exit));
        if (failure === "after-write" && Exit.isFailure(exit))
          assert.strictEqual(Cause.squash(exit.cause), jobError);
      }
      yield* TestClock.adjust(0);
      const replay = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect);
      assert.deepEqual(
        replay.length === 0 ? [] : yield* Queue.takeN(publications, replay.length),
        replay.map((stored) => stored.event.id),
      );
      assert.equal(yield* Queue.size(publications), 0);
      assert.deepEqual(writes, rollback && streams === 1 ? [] : [streams - (rollback ? 1 : 0)]);
      assert.equal(yield* sink.latestSequence(), highWater + streams - (rollback ? 1 : 0));
      for (let index = 0; index < threads.length; index++) {
        assert.lengthOf(
          (yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages,
          rollback && index === failedIndex ? 0 : 1,
        );
      }
    }).pipe(
      Effect.scoped,
      Effect.provide(Layer.fresh(layerTest)),
      Effect.withTracer(tracer),
      Effect.withTracerEnabled(true),
      Effect.provideService(
        ConfigProvider.ConfigProvider,
        ConfigProvider.fromUnknown({ T3CODE_PROVIDER_COMMIT_BATCHING: true }),
      ),
    );
  },
);

it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
  let commits = 0;
  const tracer = Tracer.make({
    span(options) {
      const span = new Tracer.NativeSpan(options);
      const end = span.end.bind(span);
      span.end = (time, exit) => {
        end(time, exit);
        if (span.name === "COMMIT") commits++;
      };
      return span;
    },
  });
  return Effect.gen(function* () {
    const sink = yield* EventSink.EventSinkV2;
    const events = yield* EventStore.EventStoreV2;
    const projections = yield* ProjectionStore.ProjectionStoreV2;
    const sql = yield* SqlClient.SqlClient;
    const outbox = yield* EffectOutbox.EffectOutboxV2;
    const now = yield* DateTime.now;
    const threads = Array.from({ length: 65 }, (_, i) =>
      ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
    );
    const sharedSession = {
      id: ProviderSessionId.make("shutdown-shared"),
      driver: providerDriver,
      providerInstanceId,
      status: "ready" as const,
      cwd: process.cwd(),
      model: modelSelection.model,
      capabilities: CodexProviderCapabilitiesV2,
      createdAt: now,
      updatedAt: now,
      lastError: null,
    };
    yield* sink.write({
      events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
        threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
        ...(i < 2
          ? [
              {
                id: EventId.make(`shutdown-session:${i}`),
                type: "provider-session.attached" as const,
                threadId,
                occurredAt: now,
                payload: sharedSession,
              },
            ]
          : []),
        {
          id: EventId.make(`shutdown-provider:${i}`),
          type: "provider-thread.updated",
          threadId,
          occurredAt: now,
          payload: {
            id: ProviderThreadId.make(`shutdown-provider:${i}`),
            driver: providerDriver,
            providerInstanceId,
            providerSessionId: i < 2 ? sharedSession.id : null,
            appThreadId: threadId,
            ownerNodeId: null,
            nativeThreadRef: null,
            nativeConversationHeadRef: null,
            status: i === 1 || i === 32 ? "idle" : "active",
            firstRunOrdinal: null,
            lastRunOrdinal: null,
            handoffIds: [],
            forkedFrom: null,
            createdAt: now,
            updatedAt: now,
          },
        },
      ]),
    });
    yield* outbox.enqueue(
      threads.map((threadId, i) => ({
        id: `shutdown-effect:${i}`,
        commandId: CommandId.make(`shutdown-command:${i}`),
        threadId,
        request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
      })),
    );
    let eventlessNotifications = 0;
    let armed = fail;
    const recovery = yield* ProviderRuntimeRecovery.make.pipe(
      Effect.provide(ServerSettings.layerTest()),
      Effect.provideService(EffectOutbox.EffectOutboxV2, {
        ...outbox,
        signalCancellations: (ids) =>
          Effect.gen(function* () {
            assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
            eventlessNotifications += ids.length;
            yield* outbox.signalCancellations(ids);
          }),
      }),
      Effect.provideService(
        EffectWorker.OrchestrationEffectWorkerV2,
        EffectWorker.OrchestrationEffectWorkerV2.of({
          awaitWork: Effect.void,
          runRecoveryOnce: Effect.succeed(false),
          runOnce: Effect.succeed(false),
          nextClaimableAt: Effect.succeedNone,
          drain: () => Effect.succeed(0),
        }),
      ),
      Effect.provideService(EventSink.EventSinkV2, {
        ...sink,
        commitCommand: (input) =>
          armed && input.threadId === threads[33]
            ? Effect.die("injected shutdown failure")
            : sink.commitCommand(input),
      }),
    );
    commits = 0;
    const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
    assert.equal(Exit.isFailure(result), fail);
    assert.equal(commits, fail ? 1 : 3);
    const receipts =
      yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
    assert.deepEqual(
      receipts.map((r) => r.aggregate_id),
      threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
    );
    assert.deepEqual(
      yield* projections.getRecoveryThreadIds("runtime"),
      fail ? threads.slice(32) : [],
    );
    assert.equal(eventlessNotifications, fail ? 1 : 2);
    for (const [i] of threads.entries()) {
      const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
      assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
    }
    assert.equal(
      (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
        (e) =>
          e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
      ).length,
      1,
    );
    armed = false;
    yield* recovery.reconcile("shutdown");
    assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
    const highWater = yield* sink.latestSequence();
    yield* recovery.reconcile("shutdown");
    assert.equal(yield* sink.latestSequence(), highWater);
  }).pipe(
    Effect.provide(Layer.fresh(layerTest)),
    Effect.withTracer(tracer),
    Effect.withTracerEnabled(true),
    Effect.provideService(Statement.SpanPropagationEnabled, true),
  );
});

it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
  it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
    Effect.scoped(
      Effect.gen(function* () {
        const sink = yield* EventSink.EventSinkV2;
        const store = yield* EventStore.EventStoreV2;
        const projections = yield* ProjectionStore.ProjectionStoreV2;
        const now = yield* DateTime.now;
        const thread = makeThread(ThreadId.make("thread:large-stream-body"), now);
        const output = {
          content: [{ type: "text", text: "first line\n" + "x".repeat(9 * 1024 * 1024) }],
        };
        const tool: OrchestrationV2TurnItem = {
          id: TurnItemId.make("tool:large-stream-body"),
          type: "dynamic_tool",
          threadId: thread.id,
          runId: null,
          nodeId: null,
          providerThreadId: null,
          providerTurnId: null,
          nativeItemRef: null,
          parentItemId: null,
          ordinal: 1,
          status: "running",
          title: "Large tool",
          toolName: "mcp__large_tool",
          input: { query: "details" },
          output,
          startedAt: now,
          completedAt: null,
          updatedAt: now,
        };
        const [created] = yield* sink.write({
          events: [
            threadCreatedEvent({ id: "event:large-stream-body:created", thread, now }),
            {
              id: EventId.make("event:large-stream-body:replay"),
              type: "turn-item.updated",
              threadId: thread.id,
              occurredAt: now,
              payload: tool,
            },
          ],
        });
        const subscribe = () =>
          Stream.toPull(
            sink.stream({
              threadId: thread.id,
              afterSequence: created!.sequence,
              bounded: true,
            }),
          );
        const pull = yield* subscribe();
        const other = yield* subscribe();
        const replay = yield* pull;
        assert.lengthOf(replay, 1);
        assert.lengthOf(yield* other, 1);
        assert.equal(replay[0]!.event.type, "turn-item.updated");
        assert.isBelow(Buffer.byteLength(yield* encodeJson(replay)), 4_000);
        const stringify = yield* Effect.acquireRelease(
          Effect.sync(() => vi.spyOn(JSON, "stringify")),
          (spy) => Effect.sync(() => spy.mockRestore()),
        );
        const [completed] = yield* sink.write({
          events: [
            {
              id: EventId.make("event:large-stream-body:live"),
              type: "turn-item.updated",
              threadId: thread.id,
              occurredAt: now,
              payload: { ...tool, status: "completed", completedAt: now },
            },
          ],
        });
        const live = yield* pull;
        const otherLive = yield* other;
        assert.deepEqual(
          live.map((stored) => stored.sequence),
          [completed!.sequence],
        );
        // Both subscribers get one shared projection, so the live budget sizes it once.
        assert.strictEqual(otherLive[0], live[0]);
        assert.lengthOf(
          stringify.mock.calls.filter(([value]) => value === live[0] || value === otherLive[0]),
          1,
        );
        assert.isBelow(Buffer.byteLength(yield* encodeJson(live)), 4_000);
        for (const stored of [...replay, ...live]) {
          assert.equal(stored.event.type, "turn-item.updated");
          if (stored.event.type !== "turn-item.updated") return;
          assert.equal(stored.event.payload.type, "dynamic_tool");
          if (stored.event.payload.type !== "dynamic_tool") return;
          assert.notProperty(stored.event.payload, "output");
        }
        const persisted = yield* store.read({ threadId: thread.id }).pipe(Stream.runCollect);
        const fullEvent = persisted.find(
          (stored) => stored.sequence === completed!.sequence,
        )!.event;
        assert.equal(fullEvent.type, "turn-item.updated");
        if (fullEvent.type !== "turn-item.updated") return;
        assert.equal(fullEvent.payload.type, "dynamic_tool");
        if (fullEvent.payload.type !== "dynamic_tool") return;
        assert.deepEqual(fullEvent.payload.output, output);
        const detail = (yield* projections.getThreadProjection(thread.id)).turnItems.find(
          (item) => item.id === tool.id,
        )!;
        assert.equal(detail.type, "dynamic_tool");
        if (detail.type === "dynamic_tool") assert.deepEqual(detail.output, output);
      }),
    ),
  );

  it.effect.each(["high-water", "replay"] as const)(
    "bounds live events while the V2 %s query is blocked",
    (phase) =>
      Effect.scoped(
        Effect.gen(function* () {
          const sink = yield* EventSink.EventSinkV2;
          const now = yield* DateTime.now;
          const thread = makeThread(ThreadId.make(`thread:blocked-${phase}`), now);
          const [created] = yield* sink.write({
            events: [threadCreatedEvent({ id: `event:blocked-${phase}:created`, thread, now })],
          });
          const readStarted = yield* Deferred.make<void>();
          const readClosed = yield* Deferred.make<void>();
          const blockRead: Statement.Transformer = (statement) => {
            const [query] = statement.compile();
            if (
              query.includes("FROM orchestration_events") &&
              query.includes("MAX(sequence)") === (phase === "high-water")
            ) {
              return Deferred.succeed(readStarted, undefined).pipe(
                Effect.andThen(Effect.never),
                Effect.ensuring(Deferred.succeed(readClosed, undefined)),
                Effect.as(statement),
              );
            }
            return Effect.succeed(statement);
          };
          const reader = yield* sink
            .stream({ threadId: thread.id, afterSequence: created!.sequence, bounded: true })
            .pipe(
              Stream.provideService(Statement.CurrentTransformer, blockRead),
              Stream.runDrain,
              Effect.result,
              Effect.forkScoped,
            );
          yield* Deferred.await(readStarted);
          yield* sink.write({
            events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
              id: EventId.make(`event:blocked-${phase}:${index}`),
              type: "thread.metadata-updated" as const,
              threadId: thread.id,
              occurredAt: now,
              payload: { ...thread, title: `Updated ${index}` },
            })),
          });
          yield* Deferred.await(readClosed);
          const result = yield* Fiber.join(reader);
          assert.equal(result._tag, "Failure");
          if (result._tag === "Failure") {
            assert.equal(result.failure._tag, "EventSinkStreamError");
            assert.isTrue(isLiveStreamBufferError(result.failure.cause));
          }
        }),
      ),
  );

  it.effect(
    "keeps internal streams subscribed while replay is blocked beyond the RPC buffer cap",
    () =>
      Effect.scoped(
        Effect.gen(function* () {
          const sink = yield* EventSink.EventSinkV2;
          const now = yield* DateTime.now;
          const thread = makeThread(ThreadId.make("thread:internal-stream-burst"), now);
          const [created] = yield* sink.write({
            events: [
              threadCreatedEvent({ id: "event:internal-stream-burst:created", thread, now }),
            ],
          });
          const readStarted = yield* Deferred.make<void>();
          const releaseRead = yield* Deferred.make<void>();
          const blockHighWater: Statement.Transformer = (statement) => {
            const [query] = statement.compile();
            return query.includes("MAX(sequence)") && query.includes("FROM orchestration_events")
              ? Deferred.succeed(readStarted, undefined).pipe(
                  Effect.andThen(Deferred.await(releaseRead)),
                  Effect.as(statement),
                )
              : Effect.succeed(statement);
          };
          const eventCount = LIVE_STREAM_MAX_ITEMS + 1;
          const reader = yield* sink
            .stream({ threadId: thread.id, afterSequence: created!.sequence })
            .pipe(
              Stream.provideService(Statement.CurrentTransformer, blockHighWater),
              Stream.take(eventCount),
              Stream.runCollect,
              Effect.forkScoped,
            );
          yield* Deferred.await(readStarted);
          const written = yield* sink.write({
            events: Array.from({ length: eventCount }, (_, index) => ({
              id: EventId.make(`event:internal-stream-burst:${index}`),
              type: "thread.metadata-updated" as const,
              threadId: thread.id,
              occurredAt: now,
              payload: { ...thread, title: `Updated ${index}` },
            })),
          });
          yield* Deferred.succeed(releaseRead, undefined);
          assert.deepEqual(
            (yield* Fiber.join(reader)).map((event) => event.sequence),
            written.map((event) => event.sequence),
          );
        }),
      ),
  );

  it.effect("filters worker replay and live queues without losing matching events", () =>
    Effect.scoped(
      Effect.gen(function* () {
        const sink = yield* EventSink.EventSinkV2;
        const sql = yield* SqlClient.SqlClient;
        const now = yield* DateTime.now;
        const thread = makeThread(ThreadId.make("thread:filtered-worker"), now);
        const other = makeThread(ThreadId.make("thread:filtered-worker-other"), now);
        const run: OrchestrationV2Run = {
          id: RunId.make("run:filtered-worker"),
          threadId: thread.id,
          ordinal: 1,
          providerInstanceId,
          modelSelection,
          providerThreadId: null,
          userMessageId: MessageId.make("message:filtered-worker"),
          rootNodeId: null,
          activeAttemptId: null,
          status: "completed",
          queuePosition: null,
          requestedAt: now,
          startedAt: now,
          completedAt: now,
          checkpointId: null,
          contextHandoffId: null,
        };
        const runEvent = (id: string, payload: OrchestrationV2Run): OrchestrationV2DomainEvent => ({
          id: EventId.make(id),
          type: "run.updated",
          threadId: payload.threadId,
          runId: payload.id,
          occurredAt: now,
          payload,
        });
        const history = yield* sink.write({
          events: [
            threadCreatedEvent({ id: "event:filtered-worker:created", thread, now }),
            threadCreatedEvent({ id: "event:filtered-worker:other", thread: other, now }),
            runEvent("event:filtered-worker:history", run),
            runEvent("event:filtered-worker:other-history", {
              ...run,
              id: RunId.make("run:filtered-worker-other"),
              threadId: other.id,
            }),
          ],
        });
        const original = yield* sql<{ readonly payload_json: string }>`
          SELECT payload_json FROM orchestration_events
          WHERE sequence = ${history[0]!.sequence}
        `;
        yield* Effect.acquireRelease(
          sql`
            UPDATE orchestration_events SET payload_json = 'unreadable unrelated payload'
            WHERE sequence = ${history[0]!.sequence}
          `,
          () =>
            sql`
              UPDATE orchestration_events SET payload_json = ${original[0]!.payload_json}
              WHERE sequence = ${history[0]!.sequence}
            `.pipe(Effect.orDie),
        );
        const pull = yield* Stream.toPull(
          sink.stream({ threadId: thread.id, eventType: "run.updated" }),
        );
        assert.deepEqual(
          (yield* pull).map((stored) => stored.sequence),
          [history[2]!.sequence],
        );
        yield* sink.write({
          events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
            id: EventId.make(`event:filtered-worker:output:${index}`),
            type: "thread.metadata-updated" as const,
            threadId: thread.id,
            occurredAt: now,
            payload: { ...thread, title: `Output ${index}` },
          })),
        });
        const live = yield* sink.write({
          events: [
            runEvent("event:filtered-worker:live:1", { ...run, status: "interrupted" }),
            runEvent("event:filtered-worker:live:2", { ...run, status: "failed" }),
          ],
        });
        assert.deepEqual(
          (yield* pull).map((stored) => stored.sequence),
          live.map((stored) => stored.sequence),
        );
      }),
    ),
  );

  it.effect("paginates catch-up beyond the event-store read limit", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-large-catch-up");
      const thread = makeThread(threadId, now);
      const eventCount = 1_005;
      const events: Array<OrchestrationV2DomainEvent> = [
        threadCreatedEvent({ id: "event:foundation-catch-up:0", thread, now }),
        ...Array.from({ length: eventCount - 1 }, (_, index) => ({
          id: EventId.make(`event:foundation-catch-up:${index + 1}`),
          type: "thread.metadata-updated" as const,
          threadId,
          providerInstanceId,
          occurredAt: now,
          payload: {
            ...thread,
            title: `Catch-up update ${index + 1}`,
          },
        })),
      ];

      yield* eventSink.write({ events });
      const replayed = yield* eventSink.stream({ afterSequence: 0 }).pipe(
        Stream.take(eventCount),
        Stream.runCollect,
        Effect.map((events) => Array.from(events)),
      );

      assert.lengthOf(replayed, eventCount);
      assert.deepEqual(
        replayed.map((stored) => stored.sequence),
        Array.from({ length: eventCount }, (_, index) => index + 1),
      );
    }),
  );

  it.effect("does not lose or duplicate events while transitioning from catch-up to live", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-stream-race");
      const thread = makeThread(threadId, now);
      const created = yield* eventSink.write({
        events: [threadCreatedEvent({ id: "event:foundation-stream-race:0", thread, now })],
      });

      let afterSequence = created[0]!.sequence;
      // oxlint-disable-next-line t3code/no-polling-loop-in-tests -- Fixed-count writes exercise the catch-up/live handoff; each iteration awaits its event rather than polling.
      for (let index = 1; index <= 32; index += 1) {
        const nextEvent = {
          id: EventId.make(`event:foundation-stream-race:${index}`),
          type: "thread.metadata-updated" as const,
          threadId,
          providerInstanceId,
          occurredAt: now,
          payload: { ...thread, title: `Race update ${index}` },
        } satisfies OrchestrationV2DomainEvent;
        const reader = yield* eventSink
          .stream({ threadId, afterSequence })
          .pipe(Stream.runHead, Effect.forkChild);
        yield* Effect.yieldNow;
        const written = yield* eventSink.write({ events: [nextEvent] });
        const received = yield* Fiber.join(reader);
        if (Option.isNone(received)) {
          return yield* Effect.die("The event stream ended before delivering the live event.");
        }
        assert.equal(received.value.sequence, written[0]?.sequence);
        assert.equal(received.value.event.id, nextEvent.id);
        afterSequence = received.value.sequence;
      }
    }),
  );

  it.effect("replays shared provider-session payloads across every bound thread", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const maintenance = yield* ProjectionReplay.ProjectionReplay;
      const now = yield* DateTime.now;
      const firstThreadId = ThreadId.make("thread:foundation-shared-session:first");
      const secondThreadId = ThreadId.make("thread:foundation-shared-session:second");
      const providerSessionId = ProviderSessionId.make("provider-session:foundation:shared");
      const firstSession = {
        id: providerSessionId,
        driver: providerDriver,
        providerInstanceId,
        status: "ready" as const,
        cwd: "/workspace/first",
        model: modelSelection.model,
        capabilities: CodexProviderCapabilitiesV2,
        createdAt: now,
        updatedAt: now,
        lastError: null,
      };
      const secondSession = { ...firstSession, cwd: "/workspace/second" };

      yield* eventSink.write({
        events: [
          threadCreatedEvent({
            id: "event:foundation-shared-session:first-thread",
            thread: makeThread(firstThreadId, now),
            now,
          }),
          {
            id: EventId.make("event:foundation-shared-session:first-attachment"),
            type: "provider-session.attached",
            threadId: firstThreadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: firstSession,
          },
          threadCreatedEvent({
            id: "event:foundation-shared-session:second-thread",
            thread: makeThread(secondThreadId, now),
            now,
          }),
          {
            id: EventId.make("event:foundation-shared-session:second-attachment"),
            type: "provider-session.attached",
            threadId: secondThreadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: secondSession,
          },
        ],
      });

      assert.equal(
        (yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions[0]?.cwd,
        secondSession.cwd,
      );
      assert.isTrue((yield* maintenance.verify).valid);
      assert.isTrue((yield* maintenance.rebuild).valid);
    }),
  );

  it.effect("prunes covered history, imported v1 events, and legacy receipts", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
      const sql = yield* SqlClient.SqlClient;
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const nowIso = DateTime.formatIso(now);
      const threadId = ThreadId.make("thread:foundation-compact");
      const thread = makeThread(threadId, now);
      const messageId = MessageId.make("message:foundation-compact");
      const threadStateEvent = (
        suffix: string,
        type: "thread.visited" | "thread.metadata-updated" | "thread.tags-updated",
      ): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type,
        threadId,
        providerInstanceId,
        occurredAt: now,
        payload: { ...thread, lastVisitedAt: now },
      });
      const messageEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type: "message.updated",
        threadId,
        occurredAt: now,
        payload: {
          createdBy: "user",
          creationSource: "web",
          id: messageId,
          threadId,
          runId: null,
          nodeId: null,
          role: "user",
          text,
          attachments: [],
          streaming: false,
          createdAt: now,
          updatedAt: now,
        },
      });
      const nodeId = NodeId.make("node:foundation-compact");
      const nodeEvent = (
        suffix: string,
        status: "running" | "completed",
      ): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type: "node.updated",
        threadId,
        occurredAt: now,
        payload: {
          id: nodeId,
          threadId,
          runId: null,
          parentNodeId: null,
          rootNodeId: nodeId,
          kind: "assistant_message",
          status,
          countsForRun: false,
          providerThreadId: null,
          providerTurnId: null,
          nativeItemRef: null,
          runtimeRequestId: null,
          checkpointScopeId: null,
          startedAt: now,
          completedAt: status === "completed" ? now : null,
        },
      });
      const itemEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type: "turn-item.updated",
        threadId,
        occurredAt: now,
        payload: {
          id: TurnItemId.make("turn-item:foundation-compact"),
          threadId,
          runId: null,
          nodeId: null,
          providerThreadId: null,
          providerTurnId: null,
          nativeItemRef: null,
          parentItemId: null,
          ordinal: 1,
          status: "completed",
          title: null,
          startedAt: now,
          completedAt: now,
          updatedAt: now,
          type: "assistant_message",
          messageId,
          text,
          streaming: false,
        },
      });

      yield* eventSink.write({
        events: [
          threadCreatedEvent({ id: "event:foundation-compact:create", thread, now }),
          threadStateEvent("meta", "thread.metadata-updated"),
          threadStateEvent("visit-1", "thread.visited"),
          messageEvent("message-1", "streaming"),
          nodeEvent("node-1", "running"),
          itemEvent("item-1", "streaming"),
          itemEvent("item-middle", "still streaming"),
          ...Array.from({ length: 501 }, (_, index) =>
            threadStateEvent(`history-${index}`, "thread.visited"),
          ),
          threadStateEvent("tags-1", "thread.tags-updated"),
          threadStateEvent("visit-2", "thread.visited"),
          messageEvent("message-2", "final"),
          nodeEvent("node-2", "completed"),
          itemEvent("item-2", "final"),
          // Older clients skip tag events, so these must not remove the rename above.
          threadStateEvent("tags-2", "thread.tags-updated"),
        ],
      });
      const beforeCompaction = yield* projections.getThreadProjection(threadId);

      const importedV1ThreadId = "thread:foundation-compact-v1-imported";
      const pendingV1ThreadId = "thread:foundation-compact-v1-pending";
      const insertV1Event = (threadIdValue: string, version: number) => sql`
        INSERT INTO orchestration_events (
          event_id, aggregate_kind, stream_id, stream_version, event_type,
          occurred_at, actor_kind, payload_json, metadata_json, application_event_version
        )
        VALUES (
          ${`event:v1:${threadIdValue}:${version}`}, 'thread', ${threadIdValue}, ${version},
          'thread.message-appended', ${nowIso}, 'user', '{}', '{}', 1
        )
      `;
      yield* insertV1Event(importedV1ThreadId, 1);
      yield* insertV1Event(importedV1ThreadId, 2);
      yield* insertV1Event(pendingV1ThreadId, 1);
      yield* sql`
        INSERT INTO orchestration_v2_legacy_imports (
          thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
          imported_message_count, last_error
        )
        VALUES
          (${importedV1ThreadId}, ${nowIso}, ${nowIso}, ${nowIso}, 2, NULL),
          (${pendingV1ThreadId}, ${nowIso}, ${nowIso}, NULL, 0, NULL)
        ON CONFLICT(thread_id) DO NOTHING
      `;
      yield* sql`
        INSERT INTO orchestration_command_receipts (
          command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
        )
        VALUES
          ('command:foundation-compact:legacy', 'thread', ${importedV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy'),
          ('command:foundation-compact:pending', 'thread', ${pendingV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy')
        ON CONFLICT(command_id) DO NOTHING
      `;

      // The first pass only marks the head and drops imported legacy rows.
      const marked = yield* maintenance.pruneEventStore();
      yield* TestClock.adjust("1 day");
      const summary = yield* maintenance.pruneEventStore();
      assert.isAtLeast(summary.deletedEventCount, 504);
      assert.equal(marked.deletedReceiptCount + summary.deletedReceiptCount, 1);

      const remaining = yield* sql<{ readonly event_id: string }>`
        SELECT event_id
        FROM orchestration_events
        WHERE aggregate_kind = 'thread'
          AND stream_id = ${threadId}
        ORDER BY sequence ASC
      `;
      assert.deepEqual(
        remaining.map((row) => row.event_id),
        ["event:foundation-compact:create", "event:foundation-compact:tags-2"],
      );

      const remainingV1 = yield* sql<{ readonly stream_id: string }>`
        SELECT stream_id
        FROM orchestration_events
        WHERE application_event_version = 1
          AND stream_id IN (${importedV1ThreadId}, ${pendingV1ThreadId})
      `;
      assert.deepEqual(
        remainingV1.map((row) => row.stream_id),
        [pendingV1ThreadId],
      );

      const remainingReceipts = yield* sql<{ readonly command_id: string }>`
        SELECT command_id
        FROM orchestration_command_receipts
        WHERE command_id IN ('command:foundation-compact:legacy', 'command:foundation-compact:pending')
      `;
      assert.deepEqual(
        remainingReceipts.map((row) => row.command_id),
        ["command:foundation-compact:pending"],
      );

      assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeCompaction);
    }),
  );

  it.effect("verifies and rebuilds projections with cross-thread subagent relations", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const maintenance = yield* ProjectionReplay.ProjectionReplay;
      const sql = yield* SqlClient.SqlClient;
      const now = yield* DateTime.now;
      const parentThreadId = ThreadId.make("thread:foundation-cross-thread:parent");
      const childThreadId = ThreadId.make("thread:foundation-cross-thread:child");
      const childProviderThreadId = ProviderThreadId.make(
        "provider-thread:foundation-cross-thread:child",
      );
      const subagentId = NodeId.make("subagent:foundation-cross-thread");
      const spawnTransferId = ContextTransferId.make("transfer:foundation-cross-thread:spawn");
      const resultTransferId = ContextTransferId.make("transfer:foundation-cross-thread:result");
      const parentThread = makeThread(parentThreadId, now);
      const childThread = {
        ...makeThread(childThreadId, now),
        createdBy: "agent" as const,
        creationSource: "provider" as const,
        lineage: {
          parentThreadId,
          relationshipToParent: "subagent" as const,
          rootThreadId: parentThreadId,
        },
      };

      yield* eventSink.write({
        events: [
          threadCreatedEvent({
            id: "event:foundation-cross-thread:parent",
            thread: parentThread,
            now,
          }),
          threadCreatedEvent({
            id: "event:foundation-cross-thread:child",
            thread: childThread,
            now,
          }),
          {
            id: EventId.make("event:foundation-cross-thread:subagent"),
            type: "subagent.updated",
            threadId: parentThreadId,
            nodeId: subagentId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: subagentId,
              threadId: parentThreadId,
              runId: null,
              parentNodeId: NodeId.make("node:foundation-cross-thread:parent"),
              origin: "app_owned",
              createdBy: "agent",
              driver: providerDriver,
              providerInstanceId,
              providerThreadId: childProviderThreadId,
              childThreadId,
              nativeTaskRef: null,
              prompt: "Inspect the child flow",
              title: "Cross-thread child",
              model: modelSelection.model,
              status: "completed",
              result: "done",
              startedAt: now,
              completedAt: now,
              updatedAt: now,
            },
          },
          {
            id: EventId.make("event:foundation-cross-thread:spawn-transfer"),
            type: "context-transfer.created",
            threadId: childThreadId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: spawnTransferId,
              type: "subagent_spawn",
              sourceThreadId: parentThreadId,
              targetThreadId: childThreadId,
              sourcePoint: { threadId: parentThreadId },
              basePoint: null,
              sourceProviderInstanceId: providerInstanceId,
              targetProviderInstanceId: providerInstanceId,
              targetRunId: null,
              status: "consumed",
              resolution: null,
              createdBy: "agent",
              error: null,
              createdAt: now,
              updatedAt: now,
              consumedAt: now,
            },
          },
          {
            id: EventId.make("event:foundation-cross-thread:provider-thread"),
            type: "provider-thread.updated",
            threadId: parentThreadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: childProviderThreadId,
              driver: providerDriver,
              providerInstanceId,
              providerSessionId: null,
              appThreadId: childThreadId,
              ownerNodeId: null,
              nativeThreadRef: null,
              nativeConversationHeadRef: null,
              status: "idle",
              firstRunOrdinal: 1,
              lastRunOrdinal: 1,
              handoffIds: [],
              forkedFrom: null,
              createdAt: now,
              updatedAt: now,
            },
          },
          {
            id: EventId.make("event:foundation-cross-thread:result-transfer"),
            type: "context-transfer.created",
            threadId: parentThreadId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: resultTransferId,
              type: "subagent_result",
              sourceThreadId: childThreadId,
              targetThreadId: parentThreadId,
              sourcePoint: { threadId: childThreadId },
              basePoint: null,
              sourceProviderInstanceId: providerInstanceId,
              targetProviderInstanceId: providerInstanceId,
              targetRunId: null,
              status: "consumed",
              resolution: null,
              createdBy: "system",
              error: null,
              createdAt: now,
              updatedAt: now,
              consumedAt: now,
            },
          },
        ],
      });

      const assertCrossThreadProjection = Effect.gen(function* () {
        const parent = yield* projectionStore.getThreadProjection(parentThreadId);
        const child = yield* projectionStore.getThreadProjection(childThreadId);
        const expectedTransferIds = [spawnTransferId, resultTransferId].toSorted();
        assert.deepEqual(
          parent.contextTransfers.map((transfer) => transfer.id).toSorted(),
          expectedTransferIds,
        );
        assert.deepEqual(
          child.contextTransfers.map((transfer) => transfer.id).toSorted(),
          expectedTransferIds,
        );
        assert.deepEqual(
          parent.providerThreads.map((providerThread) => providerThread.id),
          [childProviderThreadId],
        );
        assert.equal(child.thread.activeProviderThreadId, childProviderThreadId);
      });

      yield* assertCrossThreadProjection;
      assert.isTrue((yield* maintenance.verify).valid);
      yield* sql`
        UPDATE orchestration_v2_projection_provider_threads SET payload_json = '{}'
        WHERE provider_thread_id = ${childProviderThreadId}
      `;
      assert.deepEqual(
        new Set((yield* maintenance.verify).unreadableThreadIds),
        new Set([parentThreadId, childThreadId]),
      );
      assert.isTrue((yield* maintenance.rebuild).valid);
      yield* assertCrossThreadProjection;
    }),
  );

  it.effect(
    "rolls back events, projections, receipts, and effects after a projection failure",
    () =>
      Effect.gen(function* () {
        const eventSink = yield* EventSink.EventSinkV2;
        const eventStore = yield* EventStore.EventStoreV2;
        const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
        const outbox = yield* EffectOutbox.EffectOutboxV2;
        const sql = yield* SqlClient.SqlClient;
        const now = yield* DateTime.now;
        const commandId = CommandId.make("command:foundation-atomic-failure");
        const threadId = ThreadId.make("thread:foundation-atomic-failure");
        const scopeId = CheckpointScopeId.make("scope:foundation-atomic-failure");
        const checkpoint = (index: number) => ({
          id: CheckpointId.make(`checkpoint:foundation-atomic-failure:${index}`),
          threadId,
          scopeId,
          runId: null,
          nodeId: NodeId.make("node:foundation-atomic-failure"),
          parentCheckpointId: null,
          ordinalWithinScope: 1,
          appRunOrdinal: null,
          ref: CheckpointRef.make(`checkpoint-ref:foundation-atomic-failure:${index}`),
          status: "ready" as const,
          files: [],
          capturedAt: now,
        });
        const events = [1, 2].map(
          (index) =>
            ({
              id: EventId.make(`event:foundation-atomic-failure:${index}`),
              type: "checkpoint.captured",
              threadId,
              occurredAt: now,
              payload: checkpoint(index),
            }) satisfies OrchestrationV2DomainEvent,
        );

        const exit = yield* Effect.exit(
          eventSink.commitCommand({
            commandId,
            threadId,
            commandType: "checkpoint.atomicity-test",
            acceptedAt: now,
            events,
            effects: [
              {
                id: "effect:foundation-atomic-failure",
                commandId,
                threadId,
                request: {
                  type: "provider-turn.start",
                  runId: RunId.make("run:foundation-atomic-failure"),
                },
              },
            ],
          }),
        );
        assert.equal(exit._tag, "Failure");
        assert.isTrue(Option.isNone(yield* receipts.getByCommandId(commandId)));
        assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
        assert.deepEqual(
          yield* eventStore.readByCommandId({ commandId }).pipe(
            Stream.runCollect,
            Effect.map((events) => Array.from(events)),
          ),
          [],
        );
        const checkpointRows = yield* sql<{ readonly count: number }>`
        SELECT COUNT(*) AS count
        FROM orchestration_v2_projection_checkpoints
        WHERE thread_id = ${threadId}
      `;
        assert.equal(checkpointRows[0]?.count, 0);
      }),
  );

  it.effect("replays every command event across bounded persistence pages", () =>
    Effect.gen(function* () {
      const sql = yield* SqlClient.SqlClient;
      const eventStore = yield* EventStore.EventStoreV2;
      const eventSink = yield* EventSink.EventSinkV2;
      const now = yield* DateTime.now;
      const occurredAt = DateTime.formatIso(now);
      const commandId = CommandId.make("command:paged-command-replay");
      const exactPageCommandId = CommandId.make("command:paged-command-replay:exact");
      const emptyCommandId = CommandId.make("command:paged-command-replay:empty");
      const threadId = "thread:paged-command-replay";

      const eventRow = (
        ordinal: number,
        input: {
          readonly commandId?: string | null;
          readonly aggregateKind?: "project" | "thread";
          readonly streamId?: string;
          readonly version?: number;
        } = {},
      ) => ({
        event_id: `event:paged-command-replay:${ordinal}`,
        aggregate_kind: input.aggregateKind ?? "thread",
        stream_id: input.streamId ?? threadId,
        stream_version: ordinal,
        event_type: "provider-session.detached",
        occurred_at: occurredAt,
        command_id: input.commandId ?? null,
        causation_event_id: null,
        correlation_id: null,
        actor_kind: "server",
        payload_json: JSON.stringify({
          providerSessionId: `session:paged-command-replay:${ordinal}`,
          detachedAt: occurredAt,
        }),
        metadata_json: "{}",
        application_event_version: input.version ?? 2,
      });

      const matchingCount = 1_001;
      const rows: Array<ReturnType<typeof eventRow>> = [];
      let ordinal = 0;
      for (let index = 0; index < matchingCount; index += 1) {
        rows.push(eventRow(++ordinal, { commandId }));
        if (index % 2 === 0) {
          rows.push(eventRow(++ordinal, { commandId: "command:unrelated" }));
        }
        if (index % 5 === 0) {
          rows.push(eventRow(++ordinal));
          rows.push(eventRow(++ordinal, { commandId, version: 1 }));
          rows.push(
            eventRow(++ordinal, {
              commandId,
              aggregateKind: "project",
              streamId: `project:paged-command-replay:${index}`,
            }),
          );
        }
      }
      for (let index = 0; index < 500; index += 1) {
        rows.push(eventRow(++ordinal, { commandId: exactPageCommandId }));
      }
      const inserted = yield* Effect.forEach(
        Arr.chunksOf(rows, 400),
        (chunk) =>
          sql<{
            readonly sequence: number;
            readonly command_id: string | null;
            readonly aggregate_kind: string;
            readonly application_event_version: number;
          }>`
            INSERT INTO orchestration_events ${sql.insert(chunk)}
            RETURNING sequence, command_id, aggregate_kind, application_event_version
          `,
        { concurrency: 1 },
      ).pipe(Effect.map((chunks) => chunks.flat()));
      const expectedSequences = inserted
        .filter(
          (row) =>
            row.command_id === commandId &&
            row.aggregate_kind === "thread" &&
            row.application_event_version === 2,
        )
        .map((row) => row.sequence);
      assert.lengthOf(expectedSequences, matchingCount);

      yield* sql`
        INSERT INTO orchestration_command_receipts (
          command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
        )
        VALUES (
          ${commandId}, 'thread', ${threadId}, ${occurredAt},
          ${expectedSequences.at(-1)!}, 'accepted', 'thread.create'
        )
      `;

      const pageLimits: Array<number> = [];
      const recordReads: Statement.Transformer = (statement) => {
        const [query, params] = statement.compile();
        if (query.includes("FROM orchestration_events") && query.includes("command_id")) {
          const limit = params.at(-1);
          if (typeof limit === "number") {
            pageLimits.push(limit);
          }
        }
        return Effect.succeed(statement);
      };
      const collectByCommandId = (id: CommandId) =>
        eventStore.readByCommandId({ commandId: id }).pipe(
          Stream.provideService(Statement.CurrentTransformer, recordReads),
          Stream.runCollect,
          Effect.map((events) => Array.from(events)),
        );

      const replayed = yield* collectByCommandId(commandId);
      assert.deepEqual(
        replayed.map((stored) => stored.sequence),
        expectedSequences,
      );
      assert.deepEqual(pageLimits, [500, 500, 500]);

      pageLimits.length = 0;
      const exactPage = yield* collectByCommandId(exactPageCommandId);
      assert.lengthOf(exactPage, 500);
      assert.deepEqual(pageLimits, [500, 500]);

      pageLimits.length = 0;
      const empty = yield* collectByCommandId(emptyCommandId);
      assert.lengthOf(empty, 0);
      assert.deepEqual(pageLimits, [500]);

      pageLimits.length = 0;
      const retried = yield* eventSink
        .commitCommand({
          commandId,
          threadId: ThreadId.make(threadId),
          commandType: "thread.create",
          acceptedAt: now,
          events: [
            threadCreatedEvent({
              id: "event:paged-command-replay:retry",
              thread: makeThread(ThreadId.make(threadId), now),
              now,
            }),
          ],
          effects: [],
        })
        .pipe(Effect.provideService(Statement.CurrentTransformer, recordReads));
      assert.isFalse(retried.committed);
      assert.equal(retried.receipt.resultSequence, expectedSequences.at(-1));
      assert.deepEqual(
        retried.storedEvents.map((stored) => stored.sequence),
        expectedSequences,
      );
      assert.deepEqual(pageLimits, [500, 500, 500]);
    }),
  );

  it.effect("keeps one durable effect across command retries and executes it after recovery", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const now = yield* DateTime.now;
      const commandId = CommandId.make("command:foundation-effect-recovery");
      const threadId = ThreadId.make("thread:foundation-effect-recovery");
      const thread = makeThread(threadId, now);
      const event = threadCreatedEvent({
        id: "event:foundation-effect-recovery",
        thread,
        now,
      });
      const effect = {
        id: "effect:foundation-effect-recovery",
        commandId,
        threadId,
        request: {
          type: "provider-turn.start" as const,
          runId: RunId.make("run:foundation-effect-recovery"),
        },
      };

      const first = yield* eventSink.commitCommand({
        commandId,
        threadId,
        commandType: "foundation.effect-recovery",
        acceptedAt: now,
        events: [event],
        effects: [effect],
      });
      const retry = yield* eventSink.commitCommand({
        commandId,
        threadId,
        commandType: "foundation.effect-recovery",
        acceptedAt: now,
        events: [event],
        effects: [effect],
      });

      assert.isTrue(first.committed);
      assert.isFalse(retry.committed);
      assert.equal(retry.receipt.resultSequence, first.receipt.resultSequence);
      assert.lengthOf(retry.storedEvents, 1);
      assert.lengthOf(yield* outbox.listByCommandId(commandId), 1);

      const executionCount = yield* Ref.make(0);
      const layerExecutor = Layer.succeed(
        EffectWorker.OrchestrationEffectExecutorV2,
        EffectWorker.OrchestrationEffectExecutorV2.of({
          execute: () => Ref.update(executionCount, (count) => count + 1),
        }),
      );
      const layerWorker = EffectWorker.layerWithOptions({ workerId: "recovery-worker" }).pipe(
        Layer.provide(
          Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
        ),
      );
      yield* Effect.gen(function* () {
        const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
        assert.isTrue(yield* worker.runOnce);
        assert.isFalse(yield* worker.runOnce);
      }).pipe(Effect.provide(layerWorker));

      assert.equal(yield* Ref.get(executionCount), 1);
      const storedEffect = yield* outbox.get(effect.id);
      assert.isTrue(Option.isSome(storedEffect));
      if (Option.isSome(storedEffect)) {
        assert.equal(storedEffect.value.status, "succeeded");
      }
    }),
  );

  it.effect("does not publish a stale provider start after an interrupt wins", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-stale-provider-start");
      const runId = RunId.make("run:foundation-stale-provider-start");
      const attemptId = RunAttemptId.make("run-attempt:foundation-stale-provider-start");
      const thread = makeThread(threadId, now);
      const startingRun: OrchestrationV2Run = {
        id: runId,
        threadId,
        ordinal: 1,
        providerInstanceId,
        modelSelection,
        providerThreadId: null,
        userMessageId: MessageId.make("message:foundation-stale-provider-start"),
        rootNodeId: null,
        activeAttemptId: attemptId,
        status: "starting",
        queuePosition: null,
        requestedAt: now,
        startedAt: null,
        completedAt: null,
        checkpointId: null,
        contextHandoffId: null,
      };
      yield* eventSink.write({
        events: [
          threadCreatedEvent({
            id: "event:foundation-stale-provider-start:thread",
            thread,
            now,
          }),
          {
            id: EventId.make("event:foundation-stale-provider-start:run"),
            type: "run.created",
            threadId,
            runId,
            providerInstanceId,
            occurredAt: now,
            payload: startingRun,
          },
        ],
      });

      const captureEffect = {
        id: "effect:foundation-current-capture",
        commandId: CommandId.make("command:foundation-current-capture"),
        threadId,
        request: {
          type: "checkpoint.capture" as const,
          runId,
          scopeId: CheckpointScopeId.make("scope:foundation-current-capture"),
        },
      };
      assert.isTrue(
        (yield* eventSink.writeIfRunCurrent({
          threadId,
          runId,
          activeAttemptId: attemptId,
          expectedStatus: "starting",
          events: [],
          effects: [captureEffect],
        })).committed,
      );
      assert.isTrue(Option.isSome(yield* outbox.get(captureEffect.id)));
      yield* outbox.awaitAvailable;
      const staleCaptureEffect = { ...captureEffect, id: "effect:foundation-stale-capture" };

      const reachedPrecommitGap = yield* Deferred.make<void>();
      const releaseStaleStart = yield* Deferred.make<void>();
      const providerStartCount = yield* Ref.make(0);
      const staleStartFiber = yield* Effect.gen(function* () {
        yield* Deferred.succeed(reachedPrecommitGap, undefined);
        yield* Deferred.await(releaseStaleStart);
        const result = yield* eventSink.writeIfRunCurrent({
          threadId,
          runId,
          activeAttemptId: attemptId,
          expectedStatus: "starting",
          effects: [staleCaptureEffect],
          events: [
            {
              id: EventId.make("event:foundation-stale-provider-start:running"),
              type: "run.updated",
              threadId,
              runId,
              providerInstanceId,
              occurredAt: now,
              payload: { ...startingRun, status: "running", startedAt: now },
            },
          ],
        });
        if (result.committed) {
          yield* Ref.update(providerStartCount, (count) => count + 1);
        }
        return result;
      }).pipe(Effect.forkChild);

      yield* Deferred.await(reachedPrecommitGap);
      const interruptedAt = yield* DateTime.now;
      yield* eventSink.write({
        events: [
          {
            id: EventId.make("event:foundation-stale-provider-start:cancelled"),
            type: "run.updated",
            threadId,
            runId,
            providerInstanceId,
            occurredAt: interruptedAt,
            payload: {
              ...startingRun,
              status: "cancelled",
              completedAt: interruptedAt,
            },
          },
        ],
      });
      yield* Deferred.succeed(releaseStaleStart, undefined);

      const staleResult = yield* Fiber.join(staleStartFiber);
      assert.isFalse(staleResult.committed);
      assert.isTrue(Option.isNone(yield* outbox.get(staleCaptureEffect.id)));
      assert.deepEqual(staleResult.storedEvents, []);
      assert.equal(yield* Ref.get(providerStartCount), 0);
      const projection = yield* projectionStore.getThreadProjection(threadId);
      assert.equal(projection.runs[0]?.status, "cancelled");
    }).pipe(Effect.provide(Layer.fresh(layerTest))),
  );

  it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-provider-thread-owner");
      const runId = RunId.make("run:foundation-provider-thread-owner");
      const attemptId = RunAttemptId.make("attempt:foundation-provider-thread-owner");
      const replacementAttemptId = RunAttemptId.make(
        "attempt:foundation-provider-thread-owner:replacement",
      );
      const rootNodeId = NodeId.make("node:foundation-provider-thread-owner");
      const providerThreadId = ProviderThreadId.make(
        "provider-thread:foundation-provider-thread-owner",
      );
      const thread = makeThread(threadId, now);
      const run: OrchestrationV2Run = {
        id: runId,
        threadId,
        ordinal: 1,
        providerInstanceId,
        modelSelection,
        providerThreadId,
        userMessageId: MessageId.make("message:foundation-provider-thread-owner"),
        rootNodeId,
        activeAttemptId: attemptId,
        status: "completed",
        queuePosition: null,
        requestedAt: now,
        startedAt: now,
        completedAt: now,
        checkpointId: null,
        contextHandoffId: null,
      };
      const baseProviderThread = {
        id: providerThreadId,
        driver: providerDriver,
        providerInstanceId,
        providerSessionId: null,
        appThreadId: threadId,
        ownerNodeId: null,
        nativeThreadRef: null,
        nativeConversationHeadRef: null,
        status: "idle" as const,
        firstRunOrdinal: 1,
        lastRunOrdinal: 1,
        handoffIds: [] as const,
        forkedFrom: null,
        pendingBackgroundTasks: [
          { taskId: "bg-owner", description: "sleep 20", kind: "command" as const },
        ],
        createdAt: now,
        updatedAt: now,
      };

      yield* eventSink.write({
        events: [
          threadCreatedEvent({
            id: "event:foundation-provider-thread-owner:thread",
            thread,
            now,
          }),
          {
            id: EventId.make("event:foundation-provider-thread-owner:run"),
            type: "run.created",
            threadId,
            runId,
            nodeId: rootNodeId,
            providerInstanceId,
            occurredAt: now,
            payload: run,
          },
          {
            id: EventId.make("event:foundation-provider-thread-owner:provider-thread"),
            type: "provider-thread.updated",
            threadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: baseProviderThread,
          },
        ],
      });

      const ownedClear = yield* eventSink.writeIfProviderThreadOwner({
        providerThreadId,
        runId,
        activeAttemptId: attemptId,
        expectedLastRunOrdinal: 1,
        events: [
          {
            id: EventId.make("event:foundation-provider-thread-owner:clear"),
            type: "provider-thread.updated",
            threadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: {
              ...baseProviderThread,
              pendingBackgroundTasks: [],
              updatedAt: now,
            },
          },
        ],
      });
      assert.isTrue(ownedClear.committed);
      assert.equal(ownedClear.storedEvents.length, 1);

      const afterReplacement = yield* DateTime.now;
      yield* eventSink.write({
        events: [
          {
            id: EventId.make("event:foundation-provider-thread-owner:replacement-attempt"),
            type: "run.updated",
            threadId,
            runId,
            nodeId: rootNodeId,
            providerInstanceId,
            occurredAt: afterReplacement,
            payload: {
              ...run,
              activeAttemptId: replacementAttemptId,
              status: "running",
            },
          },
        ],
      });

      const supersededAttemptWrite = yield* eventSink.writeIfProviderThreadOwner({
        providerThreadId,
        runId,
        activeAttemptId: attemptId,
        expectedLastRunOrdinal: 1,
        events: [
          {
            id: EventId.make("event:foundation-provider-thread-owner:superseded-attempt"),
            type: "provider-thread.updated",
            threadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: afterReplacement,
            payload: {
              ...baseProviderThread,
              status: "active",
              pendingBackgroundTasks: [
                {
                  taskId: "bg-superseded",
                  description: "should not land",
                  kind: "command" as const,
                },
              ],
              updatedAt: afterReplacement,
            },
          },
        ],
      });
      assert.isFalse(supersededAttemptWrite.committed);
      assert.deepEqual(supersededAttemptWrite.storedEvents, []);

      yield* eventSink.write({
        events: [
          {
            id: EventId.make("event:foundation-provider-thread-owner:replacement-completed"),
            type: "run.updated",
            threadId,
            runId,
            nodeId: rootNodeId,
            providerInstanceId,
            occurredAt: afterReplacement,
            payload: {
              ...run,
              activeAttemptId: replacementAttemptId,
            },
          },
          {
            id: EventId.make("event:foundation-provider-thread-owner:newer-run"),
            type: "provider-thread.updated",
            threadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: afterReplacement,
            payload: {
              ...baseProviderThread,
              lastRunOrdinal: 2,
              status: "active",
              pendingBackgroundTasks: [],
              updatedAt: afterReplacement,
            },
          },
        ],
      });

      const staleOrdinalWrite = yield* eventSink.writeIfProviderThreadOwner({
        providerThreadId,
        runId,
        activeAttemptId: replacementAttemptId,
        expectedLastRunOrdinal: 1,
        events: [
          {
            id: EventId.make("event:foundation-provider-thread-owner:stale-ordinal"),
            type: "provider-thread.updated",
            threadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: afterReplacement,
            payload: baseProviderThread,
          },
        ],
      });
      assert.isFalse(staleOrdinalWrite.committed);
      assert.deepEqual(staleOrdinalWrite.storedEvents, []);

      const projection = yield* projectionStore.getThreadProjection(threadId);
      const providerThread = projection.providerThreads.find(
        (candidate) => candidate.id === providerThreadId,
      );
      assert.isDefined(providerThread);
      assert.equal(providerThread?.lastRunOrdinal, 2);
      assert.equal(providerThread?.status, "active");
      assert.deepEqual(providerThread?.pendingBackgroundTasks ?? [], []);
    }),
  );

  it.effect("interrupts a running process-bound effect when it is cancelled", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const commandId = CommandId.make("command:foundation-cancel-running-effect");
      const threadId = ThreadId.make("thread:foundation-cancel-running-effect");
      const effectId = "effect:foundation-cancel-running-effect";
      const started = yield* Deferred.make<void>();
      const interrupted = yield* Deferred.make<void>();
      yield* outbox.enqueue([
        {
          id: effectId,
          commandId,
          threadId,
          request: {
            type: "provider-turn.start",
            runId: RunId.make("run:foundation-cancel-running-effect"),
          },
        },
      ]);

      const layerExecutor = Layer.succeed(
        EffectWorker.OrchestrationEffectExecutorV2,
        EffectWorker.OrchestrationEffectExecutorV2.of({
          execute: () =>
            Deferred.succeed(started, undefined).pipe(
              Effect.andThen(Effect.never),
              Effect.onInterrupt(() =>
                Deferred.succeed(interrupted, undefined).pipe(Effect.ignore),
              ),
            ),
        }),
      );
      const layerWorker = EffectWorker.layerWithOptions({
        workerId: "cancellation-worker",
      }).pipe(
        Layer.provide(
          Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
        ),
      );

      yield* Effect.gen(function* () {
        const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
        const workerFiber = yield* worker.runOnce.pipe(Effect.forkChild);
        yield* Deferred.await(started);
        const cancelledEffectIds = yield* outbox.cancelUnsettled({
          threadId,
          effectTypes: ["provider-turn.start"],
          reason: "The owning run was interrupted.",
        });
        assert.deepEqual(cancelledEffectIds, [effectId]);
        yield* outbox.signalCancellations(cancelledEffectIds);
        assert.isTrue(yield* Fiber.join(workerFiber));
        yield* Deferred.await(interrupted);
      }).pipe(Effect.provide(layerWorker));

      const cancelled = yield* outbox.get(effectId);
      assert.isTrue(Option.isSome(cancelled));
      if (Option.isSome(cancelled)) assert.equal(cancelled.value.status, "cancelled");
    }),
  );

  it.effect("does not resolve awaitSettled waiters until the cancel is signalled", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const commandId = CommandId.make("command:foundation-settle-on-cancel");
      const threadId = ThreadId.make("thread:foundation-settle-on-cancel");
      const effectId = "effect:foundation-settle-on-cancel";
      yield* outbox.enqueue([
        {
          id: effectId,
          commandId,
          threadId,
          request: {
            type: "provider-turn.start",
            runId: RunId.make("run:foundation-settle-on-cancel"),
          },
        },
      ]);

      const waiterRegistered = yield* Deferred.make<void>();
      const waiterFiber = yield* Deferred.succeed(waiterRegistered, undefined).pipe(
        Effect.andThen(outbox.awaitSettled(effectId)),
        Effect.forkChild,
      );
      yield* Deferred.await(waiterRegistered);

      // Padding enqueues keep the waiter running so its row read lands before
      // the cancel: registration is synchronous and the read dispatches first.
      // A stronger milestone would need a test-only seam in the service, which
      // the codebase avoids.
      const paddingCommandId = CommandId.make("command:foundation-settle-on-cancel:padding");
      yield* Effect.forEach(
        Array.from({ length: 100 }, (_, index) => index),
        (index) =>
          outbox.enqueue([
            {
              id: `effect:foundation-settle-on-cancel:padding-${index}`,
              commandId: paddingCommandId,
              threadId: ThreadId.make("thread:foundation-settle-on-cancel:padding"),
              request: { type: "terminal.cleanup" as const },
            },
          ]),
        { discard: true },
      );

      const cancelledEffectIds = yield* outbox.cancelUnsettled({
        threadId,
        effectTypes: ["provider-turn.start"],
        reason: "Test settle on cancel.",
      });
      assert.deepEqual(cancelledEffectIds, [effectId]);
      // The cancel runs inside the commit transaction; waiters resolve only via
      // the post-commit signalCancellations, never from the cancel itself.
      yield* Effect.yieldNow;
      assert.isUndefined(waiterFiber.pollUnsafe());
      yield* outbox.signalCancellations(cancelledEffectIds);
      yield* Fiber.join(waiterFiber);
    }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))),
  );

  it.effect("does not start an effect that was cancelled during claim registration", () =>
    Effect.gen(function* () {
      const effectId = "effect:foundation-cancelled-during-claim";
      const threadId = ThreadId.make("thread:foundation-cancelled-during-claim");
      const commandId = CommandId.make("command:foundation-cancelled-during-claim");
      const now = DateTime.formatIso(yield* DateTime.now);
      const claimedEffect = {
        id: effectId,
        commandId,
        threadId,
        request: { type: "terminal.cleanup" as const },
        status: "running" as const,
        attemptCount: 1,
        availableAt: now,
        leaseOwner: "claim-cancellation-worker",
        createdAt: now,
        updatedAt: now,
        completedAt: null,
        lastError: null,
      };
      const executionCount = yield* Ref.make(0);
      const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
        claimNext: () => Effect.succeedSome(claimedEffect),
        get: () =>
          Effect.succeedSome({
            ...claimedEffect,
            status: "cancelled" as const,
            leaseOwner: null,
            completedAt: now,
          }),
        clearCancellation: () => Effect.void,
        awaitCancellation: () => Effect.never,
      });
      const layerExecutor = Layer.succeed(
        EffectWorker.OrchestrationEffectExecutorV2,
        EffectWorker.OrchestrationEffectExecutorV2.of({
          execute: () => Ref.update(executionCount, (count) => count + 1),
        }),
      );
      const layerWorker = EffectWorker.layerWithOptions({
        workerId: "claim-cancellation-worker",
      }).pipe(Layer.provide(Layer.merge(layerOutbox, layerExecutor)));

      assert.isTrue(
        yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
          Effect.flatMap((worker) => worker.runOnce),
          Effect.provide(layerWorker),
        ),
      );
      assert.equal(yield* Ref.get(executionCount), 0);
    }),
  );

  it.effect("allows only one worker to claim an available effect", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const commandId = CommandId.make("command:foundation-exclusive-claim");
      yield* outbox.enqueue([
        {
          id: "effect:foundation-exclusive-claim",
          commandId,
          threadId: ThreadId.make("thread:foundation-exclusive-claim"),
          request: {
            type: "provider-turn.start",
            runId: RunId.make("run:foundation-exclusive-claim"),
          },
        },
      ]);

      const claims = yield* Effect.all(
        [outbox.claimNext({ workerId: "worker-a" }), outbox.claimNext({ workerId: "worker-b" })],
        { concurrency: "unbounded" },
      );
      assert.equal(claims.filter(Option.isSome).length, 1);
      assert.equal(claims.filter(Option.isNone).length, 1);
      const claimedByA = claims[0];
      const claimedByB = claims[1];
      if (Option.isSome(claimedByA)) {
        yield* outbox.succeed({ effectId: claimedByA.value.id, workerId: "worker-a" });
      }
      if (Option.isSome(claimedByB)) {
        yield* outbox.succeed({ effectId: claimedByB.value.id, workerId: "worker-b" });
      }
    }),
  );

  it.effect("wakes a settle waiter when its effect finishes", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const effectId = "effect:foundation-await-settled";
      yield* outbox.enqueue([
        {
          id: effectId,
          commandId: CommandId.make("command:foundation-await-settled"),
          threadId: ThreadId.make("thread:foundation-await-settled"),
          request: {
            type: "provider-runtime.continue",
            sourceRunId: RunId.make("run:foundation-await-settled"),
          },
        },
      ]);
      const waiter = yield* outbox.awaitSettled(effectId).pipe(Effect.forkChild);
      const claimed = yield* outbox.claimNext({
        workerId: "settle-worker",
      });
      assert.isTrue(Option.isSome(claimed));
      yield* Effect.yieldNow;
      assert.isUndefined(waiter.pollUnsafe());

      yield* outbox.succeed({ effectId, workerId: "settle-worker" });
      yield* Fiber.join(waiter);
      yield* outbox.awaitSettled(effectId);
    }),
  );

  it.effect("runs title generation beside critical work while serializing each lane", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const commandId = CommandId.make("command:foundation-title-effect-lane");
      const threadId = ThreadId.make("thread:foundation-title-effect-lane");
      const titleEffectId = "effect:foundation-title-effect-lane:a-title";
      const providerEffectId = "effect:foundation-title-effect-lane:b-provider";
      const nextTitleEffectId = "effect:foundation-title-effect-lane:c-title";
      const nextCriticalEffectId = "effect:foundation-title-effect-lane:d-critical";
      yield* outbox.enqueue([
        {
          id: titleEffectId,
          commandId,
          threadId,
          request: {
            type: "thread-title.generate",
            kind: {
              type: "initial",
              messageId: MessageId.make("message:foundation-title-effect-lane"),
            },
          },
        },
        {
          id: providerEffectId,
          commandId,
          threadId,
          request: {
            type: "provider-turn.start",
            runId: RunId.make("run:foundation-title-effect-lane"),
          },
        },
        {
          id: nextTitleEffectId,
          commandId,
          threadId,
          request: { type: "thread-title.generate", kind: { type: "regenerate" } },
        },
        {
          id: nextCriticalEffectId,
          commandId,
          threadId,
          request: { type: "terminal.cleanup" },
        },
      ]);

      const title = yield* outbox.claimNext({
        workerId: "title-lane-worker",
      });
      assert.isTrue(Option.isSome(title));
      if (Option.isNone(title)) return;
      assert.equal(title.value.id, titleEffectId);

      const provider = yield* outbox.claimNext({
        workerId: "critical-lane-worker",
      });
      assert.isTrue(Option.isSome(provider));
      if (Option.isNone(provider)) return;
      assert.equal(provider.value.id, providerEffectId);

      assert.isTrue(
        Option.isNone(
          yield* outbox.claimNext({
            workerId: "blocked-lanes-worker",
          }),
        ),
      );

      yield* outbox.succeed({
        effectId: provider.value.id,
        workerId: "critical-lane-worker",
      });
      const nextCritical = yield* outbox.claimNext({
        workerId: "critical-lane-worker",
      });
      assert.isTrue(Option.isSome(nextCritical));
      if (Option.isNone(nextCritical)) return;
      assert.equal(nextCritical.value.id, nextCriticalEffectId);
      yield* outbox.succeed({
        effectId: nextCritical.value.id,
        workerId: "critical-lane-worker",
      });

      yield* outbox.succeed({
        effectId: title.value.id,
        workerId: "title-lane-worker",
      });
      const nextTitle = yield* outbox.claimNext({
        workerId: "title-lane-worker",
      });
      assert.isTrue(Option.isSome(nextTitle));
      if (Option.isSome(nextTitle)) {
        assert.equal(nextTitle.value.id, nextTitleEffectId);
        yield* outbox.succeed({
          effectId: nextTitle.value.id,
          workerId: "title-lane-worker",
        });
      }
    }),
  );

  it.effect("ignores deadlines blocked by a running effect on the same thread", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const now = yield* DateTime.now;
      const future = DateTime.add(now, { seconds: 10 });
      const commandId = CommandId.make("command:foundation-next-claimable");
      const blockedThreadId = ThreadId.make("thread:foundation-next-claimable:blocked");
      yield* outbox.enqueue([
        {
          id: "effect:foundation-next-claimable:a1",
          commandId,
          threadId: blockedThreadId,
          request: { type: "terminal.cleanup" },
        },
        {
          id: "effect:foundation-next-claimable:a2",
          commandId,
          threadId: blockedThreadId,
          request: { type: "terminal.cleanup" },
        },
        {
          id: "effect:foundation-next-claimable:b1",
          commandId,
          threadId: ThreadId.make("thread:foundation-next-claimable:future"),
          request: { type: "terminal.cleanup" },
          availableAt: future,
        },
      ]);

      const claimed = yield* outbox.claimNext({
        workerId: "next-claimable-worker",
      });
      assert.isTrue(Option.isSome(claimed));
      if (Option.isNone(claimed)) return;
      assert.equal(claimed.value.id, "effect:foundation-next-claimable:a1");

      const whileBlocked = yield* outbox.nextClaimableAt;
      assert.isTrue(Option.isSome(whileBlocked));
      if (Option.isSome(whileBlocked)) {
        assert.equal(DateTime.toEpochMillis(whileBlocked.value), DateTime.toEpochMillis(future));
      }

      yield* outbox.succeed({
        effectId: claimed.value.id,
        workerId: "next-claimable-worker",
      });
      const afterCompletion = yield* outbox.nextClaimableAt;
      assert.isTrue(Option.isSome(afterCompletion));
      if (Option.isSome(afterCompletion)) {
        assert.isAtMost(DateTime.toEpochMillis(afterCompletion.value), DateTime.toEpochMillis(now));
      }

      const unblocked = yield* outbox.claimNext({
        workerId: "next-claimable-worker",
      });
      assert.isTrue(Option.isSome(unblocked));
      if (Option.isSome(unblocked)) {
        assert.equal(unblocked.value.id, "effect:foundation-next-claimable:a2");
        yield* outbox.succeed({
          effectId: unblocked.value.id,
          workerId: "next-claimable-worker",
        });
      }
      yield* outbox.cancelUnsettled({
        threadId: ThreadId.make("thread:foundation-next-claimable:future"),
        effectTypes: ["terminal.cleanup"],
        reason: "Test cleanup.",
      });
    }),
  );

  it.effect("prunes settled effects past retention in batches and keeps the rest", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const sql = yield* SqlClient.SqlClient;
      const now = yield* DateTime.now;
      const insert = (
        prefix: string,
        count: number,
        status: EffectOutbox.OrchestrationEffectStatusV2,
        completedAgo: Duration.Duration | null,
      ) => {
        const completedAt =
          completedAgo === null
            ? null
            : DateTime.formatIso(DateTime.subtractDuration(now, completedAgo));
        const createdAt = DateTime.formatIso(now);
        return sql`
          WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${count})
          INSERT INTO orchestration_v2_effect_outbox (
            effect_id, command_id, thread_id, effect_type, payload_json, status,
            available_at, created_at, updated_at, completed_at
          )
          SELECT ${prefix} || i, 'command:prune', 'thread:prune', 'terminal.cleanup',
            '{"type":"terminal.cleanup"}', ${status}, ${createdAt}, ${createdAt}, ${createdAt},
            ${completedAt}
          FROM n
        `;
      };
      const old = Duration.sum(EffectOutbox.SETTLED_EFFECT_RETENTION, Duration.minutes(1));
      const recent = Duration.subtract(EffectOutbox.SETTLED_EFFECT_RETENTION, Duration.minutes(1));
      yield* insert("succeeded-old:", 1_201, "succeeded", old);
      yield* insert("cancelled-old:", 2, "cancelled", old);
      yield* insert("succeeded-recent:", 1, "succeeded", recent);
      yield* insert("failed-old:", 1, "failed", old);
      yield* insert("pending:", 1, "pending", null);
      yield* insert("running:", 1, "running", null);

      assert.equal(yield* outbox.pruneSettled, 1_203);

      const remaining = yield* sql<{ readonly effect_id: string }>`
        SELECT effect_id FROM orchestration_v2_effect_outbox ORDER BY effect_id
      `;
      assert.deepEqual(
        remaining.map((row) => row.effect_id),
        ["failed-old:1", "pending:1", "running:1", "succeeded-recent:1"],
      );
      assert.equal(yield* outbox.pruneSettled, 0);
    }).pipe(Effect.provide(layerIsolatedOutbox)),
  );

  it.effect("wakes claimers when cancellation unblocks same-thread work", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const commandId = CommandId.make("command:foundation-cancellation-wakeup");
      const threadId = ThreadId.make("thread:foundation-cancellation-wakeup");
      yield* outbox.enqueue([
        {
          id: "effect:foundation-cancellation-wakeup:a-running",
          commandId,
          threadId,
          request: {
            type: "provider-turn.start",
            runId: RunId.make("run:foundation-cancellation-wakeup"),
          },
        },
        {
          id: "effect:foundation-cancellation-wakeup:b-pending",
          commandId,
          threadId,
          request: { type: "terminal.cleanup" },
        },
      ]);

      const running = yield* outbox.claimNext({
        workerId: "cancellation-wakeup-worker",
      });
      assert.isTrue(Option.isSome(running));
      if (Option.isNone(running)) return;
      assert.equal(running.value.request.type, "provider-turn.start");

      const cancelledEffectIds = yield* outbox.cancelUnsettled({
        threadId,
        effectTypes: ["provider-turn.start"],
        reason: "Test cancellation wakeup.",
      });
      yield* outbox.signalCancellations(cancelledEffectIds);

      const wake = yield* outbox.awaitAvailable.pipe(Effect.forkChild);
      yield* Effect.yieldNow;
      assert.isDefined(wake.pollUnsafe());

      const unblocked = yield* outbox.claimNext({
        workerId: "cancellation-wakeup-worker",
      });
      assert.isTrue(Option.isSome(unblocked));
      if (Option.isSome(unblocked)) {
        assert.equal(unblocked.value.request.type, "terminal.cleanup");
        yield* outbox.succeed({
          effectId: unblocked.value.id,
          workerId: "cancellation-wakeup-worker",
        });
      }
    }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))),
  );

  it.effect("keeps later thread effects behind an earlier effect waiting to retry", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const workerId = "retry-order-worker";
      const commandId = CommandId.make("command:foundation-retry-order");
      const threadId = ThreadId.make("thread:foundation-retry-order");
      yield* outbox.enqueue([
        {
          id: "effect:foundation-retry-order:z-rollback",
          commandId,
          threadId,
          request: {
            type: "provider-thread.rollback",
            providerThreadId: ProviderThreadId.make("provider-thread:foundation-retry-order"),
            checkpointId: CheckpointId.make("checkpoint:foundation-retry-order"),
            scopeId: CheckpointScopeId.make("scope:foundation-retry-order"),
          },
        },
      ]);
      const rollback = yield* outbox.claimNext({ workerId });
      assert.isTrue(Option.isSome(rollback));
      if (Option.isNone(rollback)) return;
      yield* outbox.retry({
        effectId: rollback.value.id,
        workerId,
        error: "rollback failed once",
        delayMs: 60_000,
      });

      yield* outbox.enqueue([
        {
          id: "effect:foundation-retry-order:a-start",
          commandId: CommandId.make("command:foundation-retry-order:start"),
          threadId,
          request: { type: "provider-turn.start", runId: RunId.make("run:foundation-retry-order") },
        },
        {
          id: "effect:foundation-retry-order:b-title",
          commandId: CommandId.make("command:foundation-retry-order:title"),
          threadId,
          request: { type: "thread-title.generate", kind: { type: "regenerate" } },
        },
      ]);
      const title = yield* outbox.claimNext({ workerId });
      assert.equal(Option.getOrUndefined(title)?.id, "effect:foundation-retry-order:b-title");
      const blocked = yield* outbox.claimNext({ workerId });
      assert.isTrue(Option.isNone(blocked));
      const nextClaimable = yield* outbox.nextClaimableAt;
      assert.isTrue(Option.isSome(nextClaimable));
      if (Option.isSome(nextClaimable)) {
        assert.equal(
          DateTime.formatIso(nextClaimable.value),
          (yield* outbox.get(rollback.value.id)).pipe(Option.getOrThrow).availableAt,
        );
      }

      yield* outbox.cancelUnsettled({
        threadId,
        effectTypes: ["provider-thread.rollback"],
        reason: "Test cleanup.",
      });
      const unblocked = yield* outbox.claimNext({ workerId });
      assert.equal(Option.getOrUndefined(unblocked)?.id, "effect:foundation-retry-order:a-start");
      yield* outbox.succeed({ effectId: "effect:foundation-retry-order:a-start", workerId });
      yield* outbox.succeed({ effectId: "effect:foundation-retry-order:b-title", workerId });
    }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))),
  );

  it.effect("runs distinct threads concurrently while serializing effects within a thread", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const commandId = CommandId.make("command:foundation-concurrent-effects");
      const threadA = ThreadId.make("thread:foundation-concurrent-effects:a");
      const threadB = ThreadId.make("thread:foundation-concurrent-effects:b");
      const effectA1 = "effect:foundation-concurrent-effects:a1";
      const effectA2 = "effect:foundation-concurrent-effects:a2";
      const effectB1 = "effect:foundation-concurrent-effects:b1";
      const startedA1 = yield* Deferred.make<void>();
      const startedA2 = yield* Deferred.make<void>();
      const startedB1 = yield* Deferred.make<void>();
      const releaseA1 = yield* Deferred.make<void>();
      const releaseA2 = yield* Deferred.make<void>();
      const releaseB1 = yield* Deferred.make<void>();
      const settled = yield* Deferred.make<void>();
      const completed = yield* Ref.make(0);
      const gates = new Map([
        [effectA1, { started: startedA1, release: releaseA1 }],
        [effectA2, { started: startedA2, release: releaseA2 }],
        [effectB1, { started: startedB1, release: releaseB1 }],
      ]);
      const layerExecutor = Layer.succeed(
        EffectWorker.OrchestrationEffectExecutorV2,
        EffectWorker.OrchestrationEffectExecutorV2.of({
          execute: (effect) => {
            const gate = gates.get(effect.id);
            if (gate === undefined) return Effect.die(`Missing gate for ${effect.id}`);
            return Deferred.succeed(gate.started, undefined).pipe(
              Effect.andThen(Deferred.await(gate.release)),
            );
          },
        }),
      );
      const layerWorker = EffectWorker.layerWithOptions({
        workerId: "concurrency-worker",
      }).pipe(
        Layer.provide(
          Layer.merge(
            Layer.succeed(EffectOutbox.EffectOutboxV2, {
              ...outbox,
              succeed: (input) =>
                outbox
                  .succeed(input)
                  .pipe(
                    Effect.tap(() =>
                      Ref.updateAndGet(completed, (count) => count + 1).pipe(
                        Effect.flatMap((count) =>
                          count === 3 ? Deferred.succeed(settled, undefined) : Effect.void,
                        ),
                      ),
                    ),
                  ),
            }),
            layerExecutor,
          ),
        ),
      );

      yield* Effect.gen(function* () {
        yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
        yield* Effect.yieldNow;
        yield* Effect.yieldNow;
        yield* outbox.enqueue([
          {
            id: effectA1,
            commandId,
            threadId: threadA,
            request: {
              type: "provider-turn.start",
              runId: RunId.make("run:foundation-concurrent-effects:a1"),
            },
          },
          {
            id: effectA2,
            commandId,
            threadId: threadA,
            request: {
              type: "provider-turn.start",
              runId: RunId.make("run:foundation-concurrent-effects:a2"),
            },
          },
          {
            id: effectB1,
            commandId,
            threadId: threadB,
            request: {
              type: "provider-turn.start",
              runId: RunId.make("run:foundation-concurrent-effects:b1"),
            },
          },
        ]);
        yield* outbox.notifyAvailable(3);

        yield* Effect.all([Deferred.await(startedA1), Deferred.await(startedB1)]);
        assert.isFalse(yield* Deferred.isDone(startedA2));

        yield* Deferred.succeed(releaseA1, undefined);
        yield* Deferred.succeed(releaseB1, undefined);
        yield* Deferred.await(startedA2);
        yield* Deferred.succeed(releaseA2, undefined);
        yield* Deferred.await(settled);
        assert.isTrue(
          (yield* outbox.listByCommandId(commandId)).every(
            (effect) => effect.status === "succeeded",
          ),
        );
      }).pipe(Effect.provide(layerWorker), Effect.scoped);
    }),
  );

  it.effect("does not reclaim a running effect after its process-local lease expires", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const sql = yield* SqlClient.SqlClient;
      const commandId = CommandId.make("command:foundation-no-live-reclaim");
      const threadId = ThreadId.make("thread:foundation-no-live-reclaim");
      const firstEffectId = "effect:foundation-no-live-reclaim:first";
      const secondEffectId = "effect:foundation-no-live-reclaim:second";
      const firstStarted = yield* Deferred.make<void>();
      const releaseFirst = yield* Deferred.make<void>();
      const executions = yield* Ref.make<ReadonlyArray<string>>([]);
      yield* outbox.enqueue([
        {
          id: firstEffectId,
          commandId,
          threadId,
          request: { type: "terminal.cleanup" },
        },
        {
          id: secondEffectId,
          commandId,
          threadId,
          request: { type: "terminal.cleanup" },
        },
      ]);

      const layerExecutor = Layer.succeed(
        EffectWorker.OrchestrationEffectExecutorV2,
        EffectWorker.OrchestrationEffectExecutorV2.of({
          execute: (effect) =>
            Ref.update(executions, (current) => [...current, effect.id]).pipe(
              Effect.andThen(
                effect.id === firstEffectId
                  ? Deferred.succeed(firstStarted, undefined).pipe(
                      Effect.andThen(Deferred.await(releaseFirst)),
                    )
                  : Effect.void,
              ),
            ),
        }),
      );
      const layerWorker = EffectWorker.layerWithOptions({
        workerId: "no-live-reclaim-worker",
      }).pipe(
        Layer.provide(
          Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
        ),
      );

      yield* Effect.gen(function* () {
        const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
        const firstFiber = yield* worker.runOnce.pipe(Effect.forkChild);
        yield* Deferred.await(firstStarted);
        yield* sql`
          UPDATE orchestration_v2_effect_outbox
          SET lease_expires_at = '1970-01-01T00:00:00.000Z'
          WHERE effect_id = ${firstEffectId}
        `;

        assert.isFalse(yield* worker.runOnce);
        assert.deepEqual(yield* Ref.get(executions), [firstEffectId]);

        yield* Deferred.succeed(releaseFirst, undefined);
        assert.isTrue(yield* Fiber.join(firstFiber));
        assert.isTrue(yield* worker.runOnce);
        assert.deepEqual(yield* Ref.get(executions), [firstEffectId, secondEffectId]);
      }).pipe(Effect.provide(layerWorker));
    }),
  );

  it.effect.each([{ replayed: false }, { replayed: true }] as const)(
    "starts later thread work behind a delegated-task stop that never returns (replayed: $replayed)",
    ({ replayed }) =>
      Effect.gen(function* () {
        const outbox = yield* EffectOutbox.EffectOutboxV2;
        const key = `foundation-hung-delegated-stop:${replayed}`;
        const threadId = ThreadId.make(`thread:${key}`);
        const stopEffectId = `effect:${key}:stop`;
        const stopStarted = yield* Deferred.make<void>();
        const turnStarted = yield* Deferred.make<void>();
        yield* outbox.enqueue([
          {
            id: stopEffectId,
            commandId: CommandId.make(`command:${key}:stop`),
            threadId,
            request: { type: "delegated-tasks.stop" },
          },
        ]);
        if (replayed) {
          assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));
          assert.equal((yield* outbox.reconcileAfterProcessLoss).requeued, 1);
        }

        const layerExecutor = Layer.succeed(
          EffectWorker.OrchestrationEffectExecutorV2,
          EffectWorker.OrchestrationEffectExecutorV2.of({
            execute: (effect) =>
              effect.request.type === "delegated-tasks.stop"
                ? Deferred.succeed(stopStarted, undefined).pipe(Effect.andThen(Effect.never))
                : Deferred.succeed(turnStarted, undefined),
          }),
        );
        const layerWorker = EffectWorker.layerWithOptions({
          workerId: "hung-delegated-stop-worker",
        }).pipe(
          Layer.provide(
            Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
          ),
        );

        yield* Effect.gen(function* () {
          yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
          yield* Deferred.await(stopStarted);
          yield* outbox.enqueue([
            {
              id: `effect:${key}:turn`,
              commandId: CommandId.make(`command:${key}:turn`),
              threadId,
              request: { type: "provider-turn.start", runId: RunId.make(`run:${key}`) },
            },
          ]);
          yield* outbox.notifyAvailable();

          const turn = yield* Deferred.await(turnStarted).pipe(
            Effect.timeoutOption("1 hour"),
            Effect.forkChild,
          );
          yield* TestClock.adjust("1 hour");
          assert.isTrue(
            Option.isSome(yield* Fiber.join(turn)),
            "provider-turn.start never ran behind the hung delegated-tasks.stop",
          );
          const stop = yield* outbox.get(stopEffectId);
          assert.isTrue(Option.isSome(stop));
          if (Option.isSome(stop)) {
            assert.equal(stop.value.status, "failed");
            assert.include(stop.value.lastError ?? "", "Timed out");
          }
        }).pipe(
          Effect.provide(layerWorker),
          Effect.scoped,
          Effect.ensuring(
            outbox
              .cancelUnsettled({
                threadId,
                effectTypes: ["delegated-tasks.stop", "provider-turn.start"],
                reason: "fixture complete",
              })
              .pipe(Effect.orDie),
          ),
        );
      }),
  );

  it.effect(
    "persists shutdown continuation intent through the real event sink without domain events",
    () =>
      Effect.gen(function* () {
        const outbox = yield* EffectOutbox.EffectOutboxV2;
        const now = yield* DateTime.now;
        const threadId = ThreadId.make("thread:shutdown-prepare");
        const runId = RunId.make("run:shutdown-prepare");
        const providerThreadId = ProviderThreadId.make("provider-thread:shutdown-prepare");
        const sessionId = ProviderSessionId.make("session:shutdown-prepare");
        const attemptId = RunAttemptId.make("attempt:shutdown-prepare");
        const projection = {
          thread: makeThread(threadId, now),
          runs: [
            {
              id: runId,
              ordinal: 1,
              status: "running",
              providerInstanceId,
              providerThreadId,
              activeAttemptId: attemptId,
            },
          ],
          providerThreads: [
            {
              id: providerThreadId,
              appThreadId: threadId,
              ownerNodeId: null,
              driver: "codex",
              providerInstanceId,
              providerSessionId: sessionId,
              status: "active",
              nativeThreadRef: {
                driver: "codex",
                nativeId: "saved-native-thread",
                strength: "strong",
              },
            },
          ],
          providerSessions: [
            { id: sessionId, driver: "codex", providerInstanceId, status: "running" },
          ],
          providerTurns: [{ providerThreadId, runAttemptId: attemptId, status: "running" }],
          runtimeRequests: [],
          turnItems: [],
        } as unknown as OrchestrationV2ThreadProjection;
        const recovery = yield* ProviderRuntimeRecovery.make.pipe(
          Effect.provide(
            Layer.mergeAll(
              ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
              Layer.mock(ProjectionStore.ProjectionStoreV2)({
                getRecoveryThreadIds: () => Effect.succeed([threadId]),
                getRuntimeRecoveryProjection: () => Effect.succeed(projection),
              }),
              Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
            ),
          ),
        );
        yield* recovery.prepareForShutdown;
        yield* recovery.prepareForShutdown;
        const effects = yield* outbox.listByCommandId(
          CommandId.make(`command:restart-prepare:${runId}`),
        );
        assert.lengthOf(effects, 1);
        assert.equal(effects[0]?.status, "pending");
        assert.deepEqual(effects[0]?.request, {
          type: "provider-runtime.continue",
          sourceRunId: runId,
        });
        yield* outbox.cancelUnsettled({
          threadId,
          effectTypes: ["provider-runtime.continue"],
          reason: "fixture complete",
        });
      }),
  );

  it.effect("leaves restart continuation pending until normal worker claims are enabled", () =>
    Effect.gen(function* () {
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const threadId = ThreadId.make("thread:activation-restart");
      const commandId = CommandId.make("command:activation-restart");
      yield* outbox.enqueue([
        {
          id: "effect:activation-a-restart",
          commandId,
          threadId,
          request: { type: "provider-runtime.continue", sourceRunId: RunId.make("run:activation") },
        },
        {
          id: "effect:activation-b-cleanup",
          commandId,
          threadId,
          request: { type: "terminal.cleanup" },
        },
      ]);
      const cleanup = yield* outbox.claimNext({
        workerId: "recovery",
        excludeRestartContinuations: true,
      });
      assert.isTrue(Option.isSome(cleanup));
      if (Option.isSome(cleanup)) {
        assert.equal(cleanup.value.request.type, "terminal.cleanup");
        yield* outbox.succeed({ effectId: cleanup.value.id, workerId: "recovery" });
      }
      assert.isTrue(
        Option.isNone(
          yield* outbox.claimNext({
            workerId: "recovery",
            excludeRestartContinuations: true,
          }),
        ),
      );
      const pending = yield* outbox.get("effect:activation-a-restart");
      assert.isTrue(Option.isSome(pending));
      if (Option.isSome(pending)) assert.equal(pending.value.status, "pending");
      const resumed = yield* outbox.claimNext({ workerId: "activated" });
      assert.isTrue(Option.isSome(resumed));
      if (Option.isSome(resumed)) {
        assert.equal(resumed.value.request.type, "provider-runtime.continue");
        yield* outbox.succeed({ effectId: resumed.value.id, workerId: "activated" });
      }
    }),
  );

  it.effect.each([
    { type: "terminal.cleanup" },
    { type: "provider-runtime.continue", sourceRunId: RunId.make("run:restart-replay") },
  ] as const)(
    "retires live provider effects and requeues $type after process loss",
    (replayRequest) =>
      Effect.gen(function* () {
        const outbox = yield* EffectOutbox.EffectOutboxV2;
        const commandId = CommandId.make(
          `command:foundation-reclaim-running:${replayRequest.type}`,
        );
        yield* outbox.enqueue([
          {
            id: `effect:a-foundation-cancel-provider-turn:${replayRequest.type}`,
            commandId,
            threadId: ThreadId.make(`thread:foundation-reclaim-running:${replayRequest.type}`),
            request: {
              type: "provider-turn.start",
              runId: RunId.make(`run:foundation-reclaim-running:${replayRequest.type}`),
            },
          },
          {
            id: `effect:b-foundation-requeue-cleanup:${replayRequest.type}`,
            commandId,
            threadId: ThreadId.make(`thread:foundation-reclaim-cleanup:${replayRequest.type}`),
            request: replayRequest,
          },
        ]);
        assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));
        assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));
        assert.deepEqual(yield* outbox.reconcileAfterProcessLoss, {
          cancelled: 1,
          requeued: 1,
        });
        const cancelled = yield* outbox.get(
          `effect:a-foundation-cancel-provider-turn:${replayRequest.type}`,
        );
        assert.isTrue(Option.isSome(cancelled));
        if (Option.isSome(cancelled)) assert.equal(cancelled.value.status, "cancelled");

        const reclaimed = yield* outbox.claimNext({
          workerId: "recovery-worker",
        });
        assert.isTrue(Option.isSome(reclaimed));
        if (Option.isSome(reclaimed)) {
          assert.equal(reclaimed.value.request.type, replayRequest.type);
          assert.equal(reclaimed.value.attemptCount, 2);
          yield* outbox.succeed({ effectId: reclaimed.value.id, workerId: "recovery-worker" });
        }
      }),
  );

  it.effect("atomically cancels stale runs and their process-bound effects", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const outbox = yield* EffectOutbox.EffectOutboxV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-process-loss");
      const runId = RunId.make("run:foundation-process-loss");
      const commandId = CommandId.make("command:foundation-process-loss");
      const thread = makeThread(threadId, now);
      yield* eventSink.commitCommand({
        commandId,
        threadId,
        commandType: "foundation.process-loss",
        acceptedAt: now,
        events: [
          threadCreatedEvent({ id: "event:foundation-process-loss:thread", thread, now }),
          {
            id: EventId.make("event:foundation-process-loss:run"),
            type: "run.created",
            threadId,
            runId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: runId,
              threadId,
              ordinal: 1,
              providerInstanceId,
              modelSelection,
              providerThreadId: null,
              userMessageId: MessageId.make("message:foundation-process-loss"),
              rootNodeId: null,
              activeAttemptId: null,
              status: "starting",
              queuePosition: null,
              requestedAt: now,
              startedAt: null,
              completedAt: null,
              checkpointId: null,
              contextHandoffId: null,
            },
          },
        ],
        effects: [
          {
            id: "effect:foundation-process-loss",
            commandId,
            threadId,
            request: { type: "provider-turn.start", runId },
          },
        ],
      });
      assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));

      const recovery = yield* ProviderRuntimeRecovery.make.pipe(
        Effect.provide(ServerSettings.layerTest()),
        Effect.provideService(
          EffectWorker.OrchestrationEffectWorkerV2,
          EffectWorker.OrchestrationEffectWorkerV2.of({
            awaitWork: Effect.void,
            runRecoveryOnce: Effect.succeed(false),
            runOnce: Effect.succeed(false),
            nextClaimableAt: Effect.succeedNone,
            drain: () => Effect.succeed(0),
          }),
        ),
      );
      const first = yield* recovery.recover;
      assert.equal(first.terminalizedRuns, 1);
      assert.equal(first.retiredEffects, 1);
      const projection = yield* projectionStore.getThreadProjection(threadId);
      assert.equal(projection.runs[0]?.status, "cancelled");
      const effect = yield* outbox.get("effect:foundation-process-loss");
      assert.isTrue(Option.isSome(effect));
      if (Option.isSome(effect)) assert.equal(effect.value.status, "cancelled");

      const second = yield* recovery.recover;
      assert.equal(second.terminalizedRuns, 0);
      assert.equal(second.retiredEffects, 0);
    }),
  );

  it.effect("settles a native subagent's child thread when its provider process is gone", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const parentId = ThreadId.make("thread:foundation-native-subagent-parent");
      const childId = ThreadId.make("thread:foundation-native-subagent-child");
      const runId = RunId.make("run:foundation-native-subagent");
      const subagentId = NodeId.make("node:foundation-native-subagent");
      const childRootId = NodeId.make("node:foundation-native-subagent-child-root");
      const parent = makeThread(parentId, now);
      const child: OrchestrationV2AppThread = {
        ...makeThread(childId, now),
        createdBy: "agent",
        creationSource: "provider",
        lineage: {
          parentThreadId: parentId,
          relationshipToParent: "subagent",
          rootThreadId: parentId,
        },
        forkedFrom: { type: "node", nodeId: subagentId },
      };
      const node = (input: {
        readonly id: NodeId;
        readonly threadId: ThreadId;
        readonly runId: RunId | null;
        readonly kind: "root_turn" | "subagent";
      }) => ({
        ...input,
        parentNodeId: null,
        rootNodeId: input.id,
        status: "running" as const,
        countsForRun: false,
        providerThreadId: null,
        providerTurnId: null,
        nativeItemRef: null,
        runtimeRequestId: null,
        checkpointScopeId: null,
        startedAt: now,
        completedAt: null,
      });
      yield* eventSink.commitCommand({
        commandId: CommandId.make("command:foundation-native-subagent"),
        threadId: parentId,
        commandType: "foundation.native-subagent",
        acceptedAt: now,
        events: [
          threadCreatedEvent({
            id: "event:foundation-native-subagent:parent",
            thread: parent,
            now,
          }),
          threadCreatedEvent({ id: "event:foundation-native-subagent:child", thread: child, now }),
          {
            id: EventId.make("event:foundation-native-subagent:run"),
            type: "run.created",
            threadId: parentId,
            runId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: runId,
              threadId: parentId,
              ordinal: 1,
              providerInstanceId,
              modelSelection,
              providerThreadId: null,
              userMessageId: MessageId.make("message:foundation-native-subagent"),
              rootNodeId: null,
              activeAttemptId: null,
              status: "completed",
              queuePosition: null,
              requestedAt: now,
              startedAt: now,
              completedAt: now,
              checkpointId: null,
              contextHandoffId: null,
            },
          },
          {
            id: EventId.make("event:foundation-native-subagent:subagent-node"),
            type: "node.updated",
            threadId: parentId,
            runId,
            nodeId: subagentId,
            occurredAt: now,
            payload: node({ id: subagentId, threadId: parentId, runId, kind: "subagent" }),
          },
          {
            id: EventId.make("event:foundation-native-subagent:child-root"),
            type: "node.updated",
            threadId: childId,
            nodeId: childRootId,
            occurredAt: now,
            payload: node({ id: childRootId, threadId: childId, runId: null, kind: "root_turn" }),
          },
          {
            id: EventId.make("event:foundation-native-subagent:child-progress"),
            type: "turn-item.updated",
            threadId: childId,
            nodeId: childRootId,
            occurredAt: now,
            payload: {
              id: TurnItemId.make("item:foundation-native-subagent:progress"),
              threadId: childId,
              runId: null,
              nodeId: childRootId,
              providerThreadId: null,
              providerTurnId: null,
              nativeItemRef: null,
              parentItemId: null,
              ordinal: 101,
              type: "reasoning",
              status: "running",
              title: "Thinking",
              startedAt: now,
              completedAt: null,
              updatedAt: now,
              text: "Checking the diff.",
              streaming: true,
            },
          },
          {
            id: EventId.make("event:foundation-native-subagent:subagent"),
            type: "subagent.updated",
            threadId: parentId,
            runId,
            nodeId: subagentId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: subagentId,
              threadId: parentId,
              runId,
              parentNodeId: subagentId,
              origin: "provider_native",
              createdBy: "agent",
              driver: providerDriver,
              providerInstanceId,
              providerThreadId: null,
              childThreadId: childId,
              nativeTaskRef: null,
              prompt: "Audit the adapters",
              title: null,
              model: null,
              status: "running",
              result: null,
              startedAt: now,
              completedAt: null,
              updatedAt: now,
            },
          },
          {
            id: EventId.make("event:foundation-native-subagent:item"),
            type: "turn-item.updated",
            threadId: parentId,
            runId,
            nodeId: subagentId,
            occurredAt: now,
            payload: {
              id: TurnItemId.make("item:foundation-native-subagent"),
              threadId: parentId,
              runId,
              nodeId: subagentId,
              providerThreadId: null,
              providerTurnId: null,
              nativeItemRef: null,
              parentItemId: null,
              ordinal: 1,
              type: "subagent",
              status: "interrupted",
              title: null,
              startedAt: now,
              completedAt: null,
              updatedAt: now,
              subagentId,
              origin: "provider_native",
              driver: providerDriver,
              providerInstanceId,
              childThreadId: childId,
              prompt: "Audit the adapters",
              result: null,
            },
          },
        ],
        effects: [],
      });

      const recovery = yield* ProviderRuntimeRecovery.make.pipe(
        Effect.provide(ServerSettings.layerTest()),
        Effect.provideService(
          EffectWorker.OrchestrationEffectWorkerV2,
          EffectWorker.OrchestrationEffectWorkerV2.of({
            awaitWork: Effect.void,
            runRecoveryOnce: Effect.succeed(false),
            runOnce: Effect.succeed(false),
            nextClaimableAt: Effect.succeedNone,
            drain: () => Effect.succeed(0),
          }),
        ),
      );
      assert.include(yield* projectionStore.getRecoveryThreadIds("runtime"), childId);
      yield* recovery.recover;

      const parentProjection = yield* projectionStore.getThreadProjection(parentId);
      assert.equal(parentProjection.subagents[0]?.status, "cancelled");
      assert.equal(
        parentProjection.nodes.find((node) => node.id === subagentId)?.status,
        "cancelled",
      );
      const childProjection = yield* projectionStore.getThreadProjection(childId);
      const childRoot = childProjection.nodes.find((candidate) => candidate.id === childRootId);
      assert.equal(childRoot?.status, "cancelled");
      assert.isNotNull(childRoot?.completedAt ?? null);
      const progress = childProjection.turnItems.find((item) => item.type === "reasoning");
      assert.equal(progress?.status, "cancelled");
      assert.isFalse(progress?.type === "reasoning" && progress.streaming);
      assert.isNotNull(progress?.completedAt ?? null);
      assert.notInclude(yield* projectionStore.getRecoveryThreadIds("runtime"), childId);
    }),
  );

  it.effect("allocates collision-free positions beyond 100 items and rebuilds equivalently", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const maintenance = yield* ProjectionReplay.ProjectionReplay;
      const sql = yield* SqlClient.SqlClient;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-many-items");
      const runId = RunId.make("run:foundation-many-items");
      const providerThreadId = ProviderThreadId.make("provider-thread:foundation-many-items");
      const thread = makeThread(threadId, now);
      const providerThreadEvent = {
        id: EventId.make("event:foundation-many-items:provider-thread"),
        type: "provider-thread.updated" as const,
        threadId,
        providerInstanceId,
        occurredAt: now,
        payload: {
          id: providerThreadId,
          driver: providerDriver,
          providerInstanceId,
          providerSessionId: null,
          appThreadId: threadId,
          ownerNodeId: null,
          nativeThreadRef: null,
          nativeConversationHeadRef: null,
          status: "active" as const,
          firstRunOrdinal: 1,
          lastRunOrdinal: 1,
          handoffIds: [],
          forkedFrom: null,
          createdAt: now,
          updatedAt: now,
        },
      } satisfies OrchestrationV2DomainEvent;
      const runEvent = {
        id: EventId.make("event:foundation-many-items:run"),
        type: "run.created" as const,
        threadId,
        runId,
        providerInstanceId,
        occurredAt: now,
        payload: {
          id: runId,
          threadId,
          ordinal: 1,
          providerInstanceId,
          modelSelection,
          providerThreadId: null,
          userMessageId: MessageId.make("message:foundation-many-items"),
          rootNodeId: null,
          activeAttemptId: null,
          status: "completed" as const,
          queuePosition: null,
          requestedAt: now,
          startedAt: now,
          completedAt: now,
          checkpointId: null,
          contextHandoffId: null,
        },
      } satisfies OrchestrationV2DomainEvent;
      const items = Array.from({ length: 151 }, (_, index) => ({
        id: TurnItemId.make(`turn-item:foundation-many-items:${index}`),
        threadId,
        runId,
        nodeId: null,
        providerThreadId: null,
        providerTurnId: null,
        nativeItemRef: null,
        parentItemId: null,
        ordinal: index % 3,
        status: "completed" as const,
        title: null,
        startedAt: now,
        completedAt: now,
        updatedAt: now,
        type: "dynamic_tool" as const,
        toolName: `tool-${index}`,
        input: { index },
        output: { completed: true },
      }));
      const itemEvents = items.map(
        (item, index) =>
          ({
            id: EventId.make(`event:foundation-many-items:item:${index}`),
            type: "turn-item.updated",
            threadId,
            runId,
            providerInstanceId,
            occurredAt: now,
            payload: item,
          }) satisfies OrchestrationV2DomainEvent,
      );

      yield* eventSink.write({
        events: [
          threadCreatedEvent({ id: "event:foundation-many-items:thread", thread, now }),
          providerThreadEvent,
          runEvent,
          ...itemEvents,
        ],
      });
      const beforeUpdate = yield* projectionStore.getThreadProjection(threadId);
      assert.equal(beforeUpdate.thread.activeProviderThreadId, providerThreadId);
      const ordinals = beforeUpdate.turnItems.map((item) => item.ordinal);
      assert.lengthOf(ordinals, 151);
      assert.equal(new Set(ordinals).size, 151);
      assert.isTrue(ordinals.every((ordinal) => ordinal > 1_000_000));
      assert.isTrue(
        ordinals.every((ordinal, index) => index === 0 || ordinal > ordinals[index - 1]!),
      );

      yield* eventSink.write({
        events: [
          {
            id: EventId.make("event:foundation-many-items:update"),
            type: "turn-item.updated",
            threadId,
            runId,
            providerInstanceId,
            occurredAt: now,
            payload: { ...items[0]!, ordinal: 99_999_999, title: "Updated" },
          },
        ],
      });
      const afterUpdate = yield* projectionStore.getThreadProjection(threadId);
      assert.equal(afterUpdate.turnItems[0]?.ordinal, ordinals[0]);
      assert.equal((yield* maintenance.verify).valid, true);

      yield* sql`
        UPDATE orchestration_v2_projection_turn_items
        SET payload_json = '{}'
        WHERE turn_item_id = ${items[75]!.id}
      `;
      const broken = yield* maintenance.verify;
      assert.isFalse(broken.valid);
      assert.deepEqual(broken.unreadableThreadIds, [threadId]);

      const rebuilt = yield* maintenance.rebuild;
      assert.isTrue(rebuilt.valid);
      assert.lengthOf((yield* projectionStore.getThreadProjection(threadId)).turnItems, 151);
    }),
  );
});

it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)(
  "publishes live events in commit order across concurrent writers with %s",
  (mode) =>
    Effect.gen(function* () {
      const firstCommitted = yield* Deferred.make<void>();
      const releaseFirst = yield* Deferred.make<void>();
      const layerPausingOutbox = Layer.effect(
        EffectOutbox.EffectOutboxV2,
        Effect.gen(function* () {
          const delegate = yield* EffectOutbox.EffectOutboxV2;
          return EffectOutbox.EffectOutboxV2.of({
            ...delegate,
            notifyAvailable: (count) =>
              Deferred.succeed(firstCommitted, undefined).pipe(
                Effect.andThen(Deferred.await(releaseFirst)),
                Effect.andThen(delegate.notifyAvailable(count)),
              ),
          });
        }),
      ).pipe(Layer.provide(layerEffectOutboxProvided));
      const layerEventSink = EventSink.layerFromStores.pipe(
        Layer.provide(
          Layer.mergeAll(
            layerStoresProvided,
            layerPausingOutbox,
            layerCommandReceiptStoreProvided,
            ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
            TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
          ),
        ),
      );

      yield* Effect.gen(function* () {
        const eventSink = yield* EventSink.EventSinkV2;
        const now = yield* DateTime.now;
        const first = makeThread(ThreadId.make("thread:foundation-publish-order:first"), now);
        const second = makeThread(ThreadId.make("thread:foundation-publish-order:second"), now);
        const published = yield* eventSink
          .stream({ afterSequence: yield* eventSink.latestSequence() })
          .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped({ startImmediately: true }));

        const firstWrite = yield* eventSink
          .writeWithEffects({
            events: [
              threadCreatedEvent({
                id: "event:foundation-publish-order:first",
                thread: first,
                now,
              }),
            ],
            effects: [
              {
                id: "effect:foundation-publish-order:first",
                commandId: CommandId.make("command:foundation-publish-order:first"),
                threadId: first.id,
                request: { type: "terminal.cleanup" },
              },
            ],
          })
          .pipe(
            (write) => (mode === "direct" ? write : eventSink.withProviderBatch(write)),
            Effect.forkScoped,
          );
        yield* Deferred.await(firstCommitted);
        const secondWrite = yield* eventSink
          .write({
            events: [
              threadCreatedEvent({
                id: "event:foundation-publish-order:second",
                thread: second,
                now,
              }),
            ],
          })
          .pipe(
            Effect.provideService(Scheduler.MaxOpsBeforeYield, Number.POSITIVE_INFINITY),
            Effect.forkScoped,
          );
        yield* Effect.yieldNow;
        const interrupted =
          mode === "cancel-provider-batch"
            ? yield* Fiber.interrupt(firstWrite).pipe(Effect.forkScoped({ startImmediately: true }))
            : undefined;
        yield* Deferred.succeed(releaseFirst, undefined);
        if (interrupted !== undefined) yield* Fiber.join(interrupted);
        else yield* Fiber.join(firstWrite);
        yield* Fiber.join(secondWrite);

        const sequences = Array.from(yield* Fiber.join(published), (stored) => stored.sequence);
        assert.deepEqual(
          sequences,
          [...sequences].sort((left, right) => left - right),
        );
      }).pipe(Effect.provide(layerEventSink));
    }).pipe(Effect.provide(layerDatabase)),
);

it.effect.each(["events", "receipts"] as const)(
  "allows writes while compaction waits for the reader to discover obsolete %s",
  (kind) =>
    Effect.gen(function* () {
      const fs = yield* FileSystem.FileSystem;
      const path = yield* Path.Path;
      const directory = yield* fs.makeTempDirectoryScoped({ prefix: "compaction-reader-" });
      const database = SqlitePersistence.layerFromPath(path.join(directory, "statev2.sqlite"));
      const maintenanceLayer = ProjectionMaintenance.layer.pipe(Layer.provideMerge(database));
      yield* Effect.gen(function* () {
        const sql = yield* SqlClient.SqlClient;
        const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
        const now = DateTime.formatIso(yield* DateTime.now);
        yield* sql`
          INSERT INTO orchestration_v2_legacy_imports (
            thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
            imported_message_count, last_error
          ) VALUES ('imported', ${now}, ${now}, ${now}, 1, NULL)
        `;
        for (const threadId of ["imported", "pending"]) {
          if (kind === "events") {
            yield* sql`
              INSERT INTO orchestration_events (
                event_id, aggregate_kind, stream_id, stream_version, event_type,
                occurred_at, actor_kind, payload_json, metadata_json, application_event_version
              ) VALUES (
                ${threadId}, 'thread', ${threadId}, 1, 'thread.message-appended',
                ${now}, 'user', '{}', '{}', 1
              )
            `;
          } else {
            yield* sql`
              INSERT INTO orchestration_command_receipts (
                command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence,
                status, command_type
              ) VALUES (${threadId}, 'thread', ${threadId}, ${now}, 1, 'accepted', 'legacy')
            `;
          }
        }
        yield* sql`CREATE TABLE concurrent_writes (value TEXT)`;
        const readerHeld = yield* Deferred.make<void>();
        const releaseReader = yield* Deferred.make<void>();
        const discoveryStarted = yield* Deferred.make<Tracer.Span>();
        const observeDiscovery: Statement.Transformer = (statement, _sql, _fiber, span) => {
          const [query] = statement.compile();
          const pageQuery =
            kind === "events" ? /^\s*SELECT\s+event\.sequence/ : /^\s*SELECT\s+receipt\.rowid/;
          return pageQuery.test(query)
            ? Deferred.succeed(discoveryStarted, span).pipe(
                Effect.andThen(Deferred.await(readerHeld)),
                Effect.as(statement),
              )
            : Effect.succeed(statement);
        };
        const compacted = yield* Deferred.make<void>();
        const compaction = yield* maintenance.pruneEventStore().pipe(
          Effect.provideService(Statement.CurrentTransformer, observeDiscovery),
          Effect.provideService(Statement.SpanPropagationEnabled, true),
          Effect.tap(() => Deferred.succeed(compacted, undefined)),
          Effect.forkChild,
        );
        const discoverySpan = yield* Deferred.await(discoveryStarted);
        // Let the initial bounds read finish before holding the reader for this page.
        const reader = yield* sql
          .withTransaction(
            Deferred.succeed(readerHeld, undefined).pipe(
              Effect.andThen(Deferred.await(releaseReader)),
            ),
          )
          .pipe(NodeSqliteClient.readOnly, Effect.forkChild);
        yield* Deferred.await(readerHeld);
        yield* sql`INSERT INTO concurrent_writes VALUES ('committed')`;
        assert.isFalse(yield* Deferred.isDone(compacted));
        assert.deepEqual(yield* sql`SELECT value FROM concurrent_writes`, [{ value: "committed" }]);
        if (kind === "events") {
          yield* sql`
            INSERT INTO orchestration_events (
              event_id, aggregate_kind, stream_id, stream_version, event_type,
              occurred_at, actor_kind, payload_json, metadata_json, application_event_version
            ) VALUES (
              'after-bound', 'thread', 'imported', 2, 'thread.message-appended',
              ${now}, 'user', '{}', '{}', 1
            )
          `;
        } else {
          yield* sql`
            INSERT INTO orchestration_command_receipts (
              command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence,
              status, command_type
            ) VALUES ('after-bound', 'thread', 'imported', ${now}, 1, 'accepted', 'legacy')
          `;
        }
        yield* Deferred.succeed(releaseReader, undefined);
        yield* Fiber.join(reader);
        const result = yield* Fiber.join(compaction);
        assert.equal(discoverySpan.attributes.get("db.connection"), "reader");
        assert.equal(result.deletedEventCount, kind === "events" ? 1 : 0);
        assert.equal(result.deletedReceiptCount, kind === "receipts" ? 1 : 0);
        const retained =
          kind === "events"
            ? yield* sql`SELECT event_id AS id FROM orchestration_events ORDER BY sequence`
            : yield* sql`SELECT command_id AS id FROM orchestration_command_receipts ORDER BY rowid`;
        assert.deepEqual(retained, [{ id: "pending" }, { id: "after-bound" }]);
      }).pipe(Effect.provide(maintenanceLayer));
    }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("routes projection node lookups to the reader connection", () =>
  Effect.gen(function* () {
    const fs = yield* FileSystem.FileSystem;
    const path = yield* Path.Path;
    const directory = yield* fs.makeTempDirectoryScoped({ prefix: "projection-reader-" });
    const database = SqlitePersistence.layerFromPath(path.join(directory, "statev2.sqlite"));
    const projectionLayer = ProjectionStore.layer.pipe(Layer.provideMerge(database));
    yield* Effect.gen(function* () {
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const nodeId = NodeId.make("node:reader-routing");
      const node = {
        id: nodeId,
        threadId: ThreadId.make("thread:reader-routing"),
        runId: null,
        parentNodeId: null,
        rootNodeId: nodeId,
        kind: "subagent" as const,
        status: "running" as const,
        countsForRun: false,
        providerThreadId: null,
        providerTurnId: null,
        nativeItemRef: null,
        runtimeRequestId: null,
        checkpointScopeId: null,
        startedAt: now,
        completedAt: null,
      };
      const spans: Array<Tracer.Span> = [];
      const observe: Statement.Transformer = (statement, _sql, _fiber, span) => {
        const [query] = statement.compile();
        if (/orchestration_v2_projection_nodes WHERE node_id/.test(query)) {
          spans.push(span);
        }
        return Effect.succeed(statement);
      };
      yield* Effect.gen(function* () {
        assert.isFalse(yield* projections.isNodeStored(node));
      }).pipe(
        Effect.provideService(Statement.CurrentTransformer, observe),
        Effect.provideService(Statement.SpanPropagationEnabled, true),
      );
      assert.equal(spans.length, 1);
      assert.isTrue(
        spans.every((span) => span.attributes.get("db.connection") === "reader"),
        JSON.stringify(spans.map((span) => span.attributes.get("db.connection"))),
      );
    }).pipe(Effect.provide(projectionLayer));
  }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("loads a shell snapshot while the writer connection is occupied", () =>
  Effect.gen(function* () {
    const fs = yield* FileSystem.FileSystem;
    const path = yield* Path.Path;
    const directory = yield* fs.makeTempDirectoryScoped({ prefix: "shell-reader-" });
    const database = SqlitePersistence.layerFromPath(path.join(directory, "statev2.sqlite"));
    const projectionLayer = ProjectionStore.layer.pipe(Layer.provideMerge(database));
    yield* Effect.gen(function* () {
      const sql = yield* SqlClient.SqlClient;
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const held = yield* Deferred.make<void>();
      const release = yield* Deferred.make<void>();
      const writer = yield* sql
        .withTransaction(
          sql`SELECT 1`.pipe(
            Effect.andThen(Deferred.succeed(held, undefined)),
            Effect.andThen(Deferred.await(release)),
          ),
        )
        .pipe(Effect.forkChild);
      yield* Deferred.await(held);
      const snapshot = yield* projections.getShellSnapshot();
      yield* Deferred.succeed(release, undefined);
      yield* Fiber.join(writer);
      assert.deepEqual(snapshot.threads, []);
    }).pipe(Effect.provide(projectionLayer));
  }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect.each(["cancel transaction", "fail transaction", "cancel publication"] as const)(
  "preserves event consistency and allows later writes after %s",
  (scenario) =>
    Effect.gen(function* () {
      const paused = yield* Deferred.make<void>();
      const release = yield* Deferred.make<void>();
      const committed = scenario === "cancel publication";
      const pause = Deferred.succeed(paused, undefined).pipe(
        Effect.andThen(Deferred.await(release)),
      );
      const layerPausingOutbox = Layer.effect(
        EffectOutbox.EffectOutboxV2,
        Effect.gen(function* () {
          const delegate = yield* EffectOutbox.EffectOutboxV2;
          return EffectOutbox.EffectOutboxV2.of({
            ...delegate,
            enqueue: (effects) =>
              delegate
                .enqueue(effects)
                .pipe(
                  Effect.andThen(!committed && effects.length > 0 ? pause : Effect.void),
                  Effect.andThen(
                    scenario === "fail transaction" && effects.length > 0
                      ? Effect.die("Injected transaction failure")
                      : Effect.void,
                  ),
                ),
            notifyAvailable: (count) =>
              (committed ? pause : Effect.void).pipe(
                Effect.andThen(delegate.notifyAvailable(count)),
              ),
          });
        }),
      ).pipe(Layer.provide(layerEffectOutboxProvided));
      const layerEventSink = EventSink.layerFromStores.pipe(
        Layer.provide(
          Layer.mergeAll(
            layerStoresProvided,
            layerPausingOutbox,
            layerCommandReceiptStoreProvided,
            ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
            TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
          ),
        ),
      );

      yield* Effect.gen(function* () {
        const eventSink = yield* EventSink.EventSinkV2;
        const projections = yield* ProjectionStore.ProjectionStoreV2;
        const outbox = yield* EffectOutbox.EffectOutboxV2;
        const now = yield* DateTime.now;
        const first = makeThread(ThreadId.make("thread:interrupted-publish:first"), now);
        const marker = makeThread(ThreadId.make("thread:interrupted-publish:marker"), now);
        const published = yield* eventSink
          .stream({ afterSequence: yield* eventSink.latestSequence() })
          .pipe(
            Stream.takeUntil((stored) => stored.event.threadId === marker.id),
            Stream.runCollect,
            Effect.forkScoped({ startImmediately: true }),
          );
        const writer = yield* eventSink
          .writeWithEffects({
            events: [
              threadCreatedEvent({ id: "event:interrupted-publish:first", thread: first, now }),
            ],
            effects: [
              {
                id: "effect:interrupted-publish:first",
                commandId: CommandId.make("command:interrupted-publish:first"),
                threadId: first.id,
                request: { type: "terminal.cleanup" },
              },
            ],
          })
          .pipe(Effect.forkScoped);
        yield* Deferred.await(paused);
        if (scenario !== "fail transaction") writer.interruptUnsafe();
        if (scenario !== "cancel transaction") yield* Deferred.succeed(release, undefined);
        assert.equal((yield* Fiber.await(writer))._tag, "Failure");
        assert.equal(
          (yield* projections.getThreadShell(first.id))?.id ?? null,
          committed ? first.id : null,
        );
        assert.equal(
          Option.isSome(yield* outbox.get("effect:interrupted-publish:first")),
          committed,
        );
        yield* eventSink.write({
          events: [
            threadCreatedEvent({ id: "event:interrupted-publish:marker", thread: marker, now }),
          ],
        });
        assert.deepEqual(
          Array.from(yield* Fiber.join(published), (stored) => stored.event.threadId),
          committed ? [first.id, marker.id] : [marker.id],
        );
      }).pipe(
        Effect.provide(
          Layer.mergeAll(layerEventSink, layerStoresProvided, layerEffectOutboxProvided),
        ),
      );
    }).pipe(Effect.provide(layerDatabase)),
);