Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

fix(server): reconcile orphaned provider sessions (#7719)

t3-code[bot]0929907ff9ae
diff --git a/.macroscope/check-run-agents/effect-service-conventions.md b/.macroscope/check-run-agents/effect-service-conventions.md
index 542e9028d..b76d56d45 100644
--- a/.macroscope/check-run-agents/effect-service-conventions.md
+++ b/.macroscope/check-run-agents/effect-service-conventions.md
@@ -68,6 +68,7 @@ Review changed TypeScript and directly affected call sites for the conventions b
- Export direct schema predicates such as `export const isFoo = Schema.is(Foo)`. Flag a private `Schema.is` constant wrapped by a redundant function with the same signature.
- Do not introduce a large `switch` or lookup table in an error's `message` getter to model failures that deserve separate error classes.
- Catch statically known tagged failures with `Effect.catchTags({ ... })`, including when handling only one tag. Do not use `catchIf` with a schema predicate merely to recover one or more known `_tag` variants, and do not use `catchTag`. `Effect.catch` is appropriate when the entire error channel is intentionally handled; `catchIf` remains appropriate for genuinely structural predicates such as inspecting an underlying platform error code.
+- For startup reconciliation that repairs multiple independent entities, preserve interruption rather than reducing it to a warning. Retry a transient per-entity repair before readiness, then isolate a persistent failure so one bad entity cannot abort global startup or prevent later entities from being repaired. Require tests for both the retry-success path and persistent-failure continuation.
- Do not add a helper whose only behavior is `(...args) => new SomeError({ ...args })`, including curried aliases used once with `mapError`. Construct the error at the failure boundary so its attributes and cause remain visible. Keep a mapper only when it performs real normalization, passes through existing domain errors, or adds reusable context/control flow.
- When a reusable error-to-error translation clearly belongs to the target error type, prefer a descriptive static factory on that error class over a detached production-side switch. Do not force a static method for one-off inline mappings.
diff --git a/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts b/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts
new file mode 100644
index 000000000..2a351cd6b
--- /dev/null
+++ b/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts
@@ -0,0 +1,358 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import {
+ CommandId,
+ DEFAULT_PROVIDER_INTERACTION_MODE,
+ EnvironmentId,
+ MessageId,
+ ProjectId,
+ ProviderDriverKind,
+ ProviderInstanceId,
+ ThreadId,
+} from "@t3tools/contracts";
+import { assert, it } from "@effect/vitest";
+import * as DateTime from "effect/DateTime";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Stream from "effect/Stream";
+import * as SqlClient from "effect/unstable/sql/SqlClient";
+import { HttpServer } from "effect/unstable/http";
+
+import * as EnvironmentAuth from "../src/auth/EnvironmentAuth.ts";
+import * as ServiceLauncherClient from "../src/cloud/serviceLauncherClient.ts";
+import * as ServerConfig from "../src/config.ts";
+import * as ServerEnvironment from "../src/environment/ServerEnvironment.ts";
+import * as Keybindings from "../src/keybindings.ts";
+import { OrchestrationLayerLive } from "../src/orchestration/runtimeLayer.ts";
+import * as OrchestrationEngine from "../src/orchestration/Services/OrchestrationEngine.ts";
+import * as OrchestrationReactor from "../src/orchestration/Services/OrchestrationReactor.ts";
+import * as ProjectionSnapshotQuery from "../src/orchestration/Services/ProjectionSnapshotQuery.ts";
+import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts";
+import * as ProviderSessionRuntime from "../src/persistence/ProviderSessionRuntime.ts";
+import * as ExternalLauncher from "../src/process/externalLauncher.ts";
+import { ProviderSessionDirectoryLive } from "../src/provider/Layers/ProviderSessionDirectory.ts";
+import * as ProviderService from "../src/provider/Services/ProviderService.ts";
+import * as ProviderSessionDirectory from "../src/provider/Services/ProviderSessionDirectory.ts";
+import * as ProviderSessionReaper from "../src/provider/Services/ProviderSessionReaper.ts";
+import * as RepositoryIdentityResolver from "../src/project/RepositoryIdentityResolver.ts";
+import * as ServerLifecycleEvents from "../src/serverLifecycleEvents.ts";
+import * as ServerRuntimeStartup from "../src/serverRuntimeStartup.ts";
+import * as ServerSettings from "../src/serverSettings.ts";
+import * as AnalyticsService from "../src/telemetry/AnalyticsService.ts";
+
+const providerInstanceId = ProviderInstanceId.make("codex");
+const projectId = ProjectId.make("project-startup-orphan");
+const threadId = ThreadId.make("thread-startup-orphan");
+const stoppedBindingThreadId = ThreadId.make("thread-startup-orphan-stopped-binding");
+const resumeCursor = { schemaVersion: 1, sessionId: "provider-session-before-restart" };
+const stoppedBindingResumeCursor = {
+ schemaVersion: 1,
+ sessionId: "provider-session-stopped-before-restart",
+};
+
+const makePersistedRuntimeLayer = (dbPath: string) => {
+ const persistence = makeSqlitePersistenceLive(dbPath);
+ const orchestration = OrchestrationLayerLive.pipe(
+ Layer.provideMerge(RepositoryIdentityResolver.layer),
+ Layer.provideMerge(persistence),
+ );
+ const directory = ProviderSessionDirectoryLive.pipe(
+ Layer.provide(ProviderSessionRuntime.layer),
+ Layer.provide(persistence),
+ );
+ return Layer.mergeAll(orchestration, directory);
+};
+
+const startupDependencies = Layer.mergeAll(
+ Layer.mock(Keybindings.Keybindings)({
+ start: Effect.void,
+ }),
+ ServerSettings.layerTest(),
+ Layer.succeed(OrchestrationReactor.OrchestrationReactor, {
+ start: () => Effect.void,
+ }),
+ Layer.succeed(ProviderSessionReaper.ProviderSessionReaper, {
+ start: () => Effect.void,
+ }),
+ ServerLifecycleEvents.layer,
+ Layer.succeed(ServerEnvironment.ServerEnvironment, {
+ getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-startup-orphan")),
+ getDescriptor: Effect.succeed({
+ environmentId: EnvironmentId.make("environment-startup-orphan"),
+ label: "Startup orphan test",
+ version: "test",
+ platform: { os: "linux", arch: "x64" },
+ capabilities: {},
+ } as never),
+ }),
+ Layer.mock(EnvironmentAuth.EnvironmentAuth)({
+ issueStartupPairingUrl: (baseUrl: string) => Effect.succeed(`${baseUrl}/pair`),
+ }),
+ Layer.mock(ExternalLauncher.ExternalLauncher)({
+ launchBrowser: () => Effect.void,
+ }),
+ Layer.succeed(ServiceLauncherClient.ServiceLauncherClient, {
+ managed: false,
+ requestUpdate: () => Effect.die("unused"),
+ prepareTrial: Effect.sync(() => undefined),
+ }),
+ Layer.succeed(
+ HttpServer.HttpServer,
+ HttpServer.HttpServer.of({
+ address: { _tag: "TcpAddress", hostname: "127.0.0.1", port: 3773 },
+ serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"],
+ }),
+ ),
+ AnalyticsService.layerTest,
+ Layer.succeed(ProviderService.ProviderService, {
+ startSession: () => Effect.die("unused"),
+ sendTurn: () => Effect.die("unused"),
+ interruptTurn: () => Effect.die("unused"),
+ respondToRequest: () => Effect.die("unused"),
+ respondToUserInput: () => Effect.die("unused"),
+ stopSession: () => Effect.die("unused"),
+ listSessions: () => Effect.succeed([]),
+ getCapabilities: () => Effect.die("unused"),
+ getInstanceInfo: () => Effect.die("unused"),
+ rollbackConversation: () => Effect.die("unused"),
+ streamEvents: Stream.empty,
+ }),
+);
+
+it.effect(
+ "recovers a persisted starting session before opening the command gate after restart",
+ () =>
+ Effect.gen(function* () {
+ const config = yield* ServerConfig.ServerConfig;
+ const firstRuntime = makePersistedRuntimeLayer(config.dbPath);
+ const now = yield* DateTime.now;
+ const createdAt = DateTime.formatIso(now);
+
+ yield* Effect.gen(function* () {
+ const engine = yield* OrchestrationEngine.OrchestrationEngineService;
+ const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
+
+ yield* engine.dispatch({
+ type: "project.create",
+ commandId: CommandId.make("command-create-project"),
+ projectId,
+ title: "Startup orphan project",
+ workspaceRoot: "/tmp/startup-orphan-project",
+ defaultModelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
+ createdAt,
+ });
+ yield* engine.dispatch({
+ type: "thread.create",
+ commandId: CommandId.make("command-create-thread"),
+ threadId,
+ projectId,
+ title: "Startup orphan thread",
+ modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "full-access",
+ branch: null,
+ worktreePath: null,
+ createdAt,
+ });
+ yield* engine.dispatch({
+ type: "thread.turn.start",
+ commandId: CommandId.make("command-start-pending-turn"),
+ threadId,
+ message: {
+ messageId: MessageId.make("message-pending-before-restart"),
+ role: "user",
+ text: "Persist this queued turn before restart",
+ attachments: [],
+ },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "full-access",
+ createdAt,
+ });
+ yield* engine.dispatch({
+ type: "thread.session.set",
+ commandId: CommandId.make("command-mark-session-starting"),
+ threadId,
+ session: {
+ threadId,
+ status: "starting",
+ providerName: "codex",
+ providerInstanceId,
+ runtimeMode: "full-access",
+ activeTurnId: null,
+ lastError: null,
+ updatedAt: createdAt,
+ },
+ createdAt,
+ });
+ yield* directory.upsert({
+ threadId,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId,
+ status: "running",
+ resumeCursor,
+ runtimePayload: { activeTurnId: null, unrelated: "preserve-me" },
+ runtimeMode: "full-access",
+ });
+ yield* engine.dispatch({
+ type: "thread.create",
+ commandId: CommandId.make("command-create-stopped-binding-thread"),
+ threadId: stoppedBindingThreadId,
+ projectId,
+ title: "Startup orphan with stopped binding",
+ modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "full-access",
+ branch: null,
+ worktreePath: null,
+ createdAt,
+ });
+ yield* engine.dispatch({
+ type: "thread.turn.start",
+ commandId: CommandId.make("command-start-stopped-binding-pending-turn"),
+ threadId: stoppedBindingThreadId,
+ message: {
+ messageId: MessageId.make("message-stopped-binding-pending-before-restart"),
+ role: "user",
+ text: "Persist another queued turn before restart",
+ attachments: [],
+ },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "full-access",
+ createdAt,
+ });
+ yield* engine.dispatch({
+ type: "thread.session.set",
+ commandId: CommandId.make("command-mark-stopped-binding-session-starting"),
+ threadId: stoppedBindingThreadId,
+ session: {
+ threadId: stoppedBindingThreadId,
+ status: "starting",
+ providerName: "codex",
+ providerInstanceId,
+ runtimeMode: "full-access",
+ activeTurnId: null,
+ lastError: null,
+ updatedAt: createdAt,
+ },
+ createdAt,
+ });
+ yield* directory.upsert({
+ threadId: stoppedBindingThreadId,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId,
+ status: "stopped",
+ resumeCursor: stoppedBindingResumeCursor,
+ runtimePayload: { activeTurnId: "stale", unrelated: "also-preserve-me" },
+ runtimeMode: "full-access",
+ });
+ }).pipe(Effect.provide(firstRuntime));
+
+ const secondRuntime = makePersistedRuntimeLayer(config.dbPath);
+ const startupLayer = ServerRuntimeStartup.layer.pipe(
+ Layer.provideMerge(secondRuntime),
+ Layer.provideMerge(startupDependencies),
+ );
+
+ const result = yield* Effect.gen(function* () {
+ const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup;
+ const engine = yield* OrchestrationEngine.OrchestrationEngineService;
+ const query = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
+ const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
+ const sql = yield* SqlClient.SqlClient;
+
+ yield* startup.markHttpListening;
+ yield* startup.awaitCommandReady;
+
+ const restartedThread = Option.getOrThrow(yield* query.getThreadDetailById(threadId));
+ const restartedStoppedBindingThread = Option.getOrThrow(
+ yield* query.getThreadDetailById(stoppedBindingThreadId),
+ );
+ const pendingRows = yield* sql<{ readonly threadId: string }>`
+ SELECT thread_id AS "threadId"
+ FROM projection_turns
+ WHERE thread_id IN (${threadId}, ${stoppedBindingThreadId})
+ AND turn_id IS NULL
+ AND state = 'pending'
+ `;
+ const settleExit = yield* Effect.exit(
+ engine.dispatch({
+ type: "thread.settle",
+ commandId: CommandId.make("command-settle-after-restart"),
+ threadId,
+ }),
+ );
+ const snoozeExit = yield* Effect.exit(
+ engine.dispatch({
+ type: "thread.snooze",
+ commandId: CommandId.make("command-snooze-after-restart"),
+ threadId,
+ snoozedUntil: DateTime.formatIso(DateTime.add(now, { hours: 1 })),
+ }),
+ );
+ const newTurnExit = yield* Effect.exit(
+ engine.dispatch({
+ type: "thread.turn.start",
+ commandId: CommandId.make("command-new-turn-after-restart"),
+ threadId,
+ message: {
+ messageId: MessageId.make("message-new-turn-after-restart"),
+ role: "user",
+ text: "Continue immediately after restart",
+ attachments: [],
+ },
+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,
+ runtimeMode: "full-access",
+ createdAt,
+ }),
+ );
+ const binding = Option.getOrThrow(yield* directory.getBinding(threadId));
+ const stoppedBinding = Option.getOrThrow(
+ yield* directory.getBinding(stoppedBindingThreadId),
+ );
+
+ return {
+ sessionStatus: restartedThread.session?.status,
+ activeTurnId: restartedThread.session?.activeTurnId,
+ latestTurn: restartedThread.latestTurn,
+ pendingTurnCount: pendingRows.length,
+ settleSucceeded: Exit.isSuccess(settleExit),
+ snoozeSucceeded: Exit.isSuccess(snoozeExit),
+ newTurnSucceeded: Exit.isSuccess(newTurnExit),
+ bindingStatus: binding.status,
+ resumeCursor: binding.resumeCursor,
+ runtimePayload: binding.runtimePayload,
+ stoppedBindingSessionStatus: restartedStoppedBindingThread.session?.status,
+ stoppedBindingStatus: stoppedBinding.status,
+ stoppedBindingResumeCursor: stoppedBinding.resumeCursor,
+ stoppedBindingRuntimePayload: stoppedBinding.runtimePayload,
+ };
+ }).pipe(Effect.provide(startupLayer));
+
+ assert.deepStrictEqual(result, {
+ sessionStatus: "error",
+ activeTurnId: null,
+ latestTurn: null,
+ pendingTurnCount: 0,
+ settleSucceeded: true,
+ snoozeSucceeded: true,
+ newTurnSucceeded: true,
+ bindingStatus: "stopped",
+ resumeCursor,
+ runtimePayload: { activeTurnId: null, unrelated: "preserve-me" },
+ stoppedBindingSessionStatus: "error",
+ stoppedBindingStatus: "stopped",
+ stoppedBindingResumeCursor,
+ stoppedBindingRuntimePayload: {
+ activeTurnId: null,
+ unrelated: "also-preserve-me",
+ },
+ });
+ }).pipe(
+ Effect.provide(
+ ServerConfig.layerTest(process.cwd(), {
+ prefix: "t3-orphaned-provider-session-startup-",
+ }).pipe(Layer.provideMerge(NodeServices.layer)),
+ ),
+ ),
+);
diff --git a/apps/server/src/serverRuntimeStartup.reconcile.test.ts b/apps/server/src/serverRuntimeStartup.reconcile.test.ts
new file mode 100644
index 000000000..60cb8d61b
--- /dev/null
+++ b/apps/server/src/serverRuntimeStartup.reconcile.test.ts
@@ -0,0 +1,296 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import {
+ type OrchestrationCommand,
+ ProviderDriverKind,
+ ProviderInstanceId,
+ ThreadId,
+ TurnId,
+} from "@t3tools/contracts";
+import { assert, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import * as Stream from "effect/Stream";
+
+import { OrchestrationCommandInvariantError } from "./orchestration/Errors.ts";
+import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";
+import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";
+import { ProviderSessionDirectoryPersistenceError } from "./provider/Errors.ts";
+import * as ProviderService from "./provider/Services/ProviderService.ts";
+import * as ProviderSessionDirectory from "./provider/Services/ProviderSessionDirectory.ts";
+import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts";
+
+const providerInstanceId = ProviderInstanceId.make("codex");
+const updatedAt = "2026-08-20T12:00:00.000Z";
+
+const makeThread = (
+ id: string,
+ status: "starting" | "running" | "ready" | "stopped" | "error",
+ activeTurnId: TurnId | null = null,
+ archivedAt: string | null = null,
+) => ({
+ id: ThreadId.make(id),
+ archivedAt,
+ deletedAt: null,
+ session: {
+ threadId: ThreadId.make(id),
+ status,
+ providerName: "codex" as const,
+ providerInstanceId,
+ runtimeMode: "full-access" as const,
+ activeTurnId,
+ lastError: null,
+ updatedAt,
+ },
+});
+
+const makeProviderService = (liveThreadIds: ReadonlyArray<ThreadId> = []) =>
+ ({
+ startSession: () => Effect.die("unused"),
+ sendTurn: () => Effect.die("unused"),
+ interruptTurn: () => Effect.die("unused"),
+ respondToRequest: () => Effect.die("unused"),
+ respondToUserInput: () => Effect.die("unused"),
+ stopSession: () => Effect.die("unused"),
+ listSessions: () => Effect.succeed(liveThreadIds.map((threadId) => ({ threadId }) as never)),
+ getCapabilities: () => Effect.die("unused"),
+ getInstanceInfo: () => Effect.die("unused"),
+ rollbackConversation: () => Effect.die("unused"),
+ streamEvents: Stream.empty,
+ }) satisfies ProviderService.ProviderService["Service"];
+
+const queryWithThreads = (threads: ReadonlyArray<ReturnType<typeof makeThread>>) =>
+ ({
+ getCommandReadModel: () => Effect.succeed({ threads } as never),
+ }) as unknown as ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"];
+
+const runReconciliation = (input: {
+ readonly threads: ReadonlyArray<ReturnType<typeof makeThread>>;
+ readonly liveThreadIds?: ReadonlyArray<ThreadId>;
+ readonly directory: ProviderSessionDirectory.ProviderSessionDirectory["Service"];
+ readonly dispatch: OrchestrationEngine.OrchestrationEngineService["Service"]["dispatch"];
+}) =>
+ ServerRuntimeStartup.reconcileProviderSessions.pipe(
+ Effect.provideService(
+ ProjectionSnapshotQuery.ProjectionSnapshotQuery,
+ queryWithThreads(input.threads),
+ ),
+ Effect.provideService(
+ ProviderService.ProviderService,
+ makeProviderService(input.liveThreadIds),
+ ),
+ Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, input.directory),
+ Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
+ readEvents: () => Stream.empty,
+ dispatch: input.dispatch,
+ streamDomainEvents: Stream.empty,
+ latestSequence: Effect.succeed(0),
+ }),
+ Effect.provide(NodeServices.layer),
+ );
+
+it.effect("reconciles multiple active and archived orphans but skips live sessions", () => {
+ const starting = makeThread("thread-starting", "starting");
+ const running = makeThread("thread-running", "running", TurnId.make("turn-running"));
+ const staleActiveTurn = makeThread(
+ "thread-stale-active-turn",
+ "ready",
+ TurnId.make("turn-stale-active"),
+ );
+ const archived = makeThread(
+ "thread-archived",
+ "running",
+ TurnId.make("turn-archived"),
+ updatedAt,
+ );
+ const live = makeThread("thread-live", "running", TurnId.make("turn-live"));
+ const settled = makeThread("thread-ready", "ready");
+ const dispatched: OrchestrationCommand[] = [];
+ const bindingReads: ThreadId[] = [];
+ const upserts: ProviderSessionDirectory.ProviderRuntimeBinding[] = [];
+
+ return runReconciliation({
+ threads: [starting, running, staleActiveTurn, archived, live, settled],
+ liveThreadIds: [live.id],
+ directory: {
+ getBinding: (candidate) =>
+ Effect.sync(() => bindingReads.push(candidate)).pipe(
+ Effect.as(
+ Option.some({
+ threadId: candidate,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId,
+ status: "running" as const,
+ resumeCursor: { cursor: candidate },
+ runtimePayload: { activeTurnId: "stale", unrelated: candidate },
+ }),
+ ),
+ ),
+ upsert: (binding) => Effect.sync(() => upserts.push(binding)),
+ getProvider: () => Effect.die("unused"),
+ listThreadIds: () => Effect.die("unused"),
+ listBindings: () => Effect.die("unused"),
+ },
+ dispatch: (command) =>
+ Effect.sync(() => dispatched.push(command)).pipe(Effect.as({ sequence: dispatched.length })),
+ }).pipe(
+ Effect.tap(() =>
+ Effect.sync(() => {
+ const orphanIds = [starting.id, running.id, staleActiveTurn.id, archived.id];
+ assert.deepStrictEqual(bindingReads, orphanIds);
+ assert.deepStrictEqual(
+ dispatched.map((command) => command.type === "thread.session.set" && command.threadId),
+ orphanIds,
+ );
+ assert.deepStrictEqual(
+ dispatched.map((command) =>
+ command.type === "thread.session.set"
+ ? {
+ status: command.session.status,
+ activeTurnId: command.session.activeTurnId,
+ }
+ : null,
+ ),
+ orphanIds.map(() => ({ status: "error" as const, activeTurnId: null })),
+ );
+ assert.equal(upserts.length, orphanIds.length);
+ for (const binding of upserts) {
+ assert.equal(binding.status, "stopped");
+ assert.deepStrictEqual(binding.runtimePayload, { activeTurnId: null });
+ assert.deepStrictEqual(binding.resumeCursor, { cursor: binding.threadId });
+ }
+ }),
+ ),
+ );
+});
+
+it.effect(
+ "settles projections when directory bindings are absent, corrupt, or fail to upsert",
+ () => {
+ const absent = makeThread("thread-binding-absent", "starting");
+ const corrupt = makeThread("thread-binding-corrupt", "running");
+ const upsertFailure = makeThread("thread-binding-upsert-failure", "running");
+ const dispatched: OrchestrationCommand[] = [];
+ const corruptFailure = new ProviderSessionDirectoryPersistenceError({
+ operation: "ProviderSessionDirectory.getBinding",
+ detail: "corrupt persisted binding",
+ });
+ const writeFailure = new ProviderSessionDirectoryPersistenceError({
+ operation: "ProviderSessionDirectory.upsert",
+ detail: "failed binding write",
+ });
+
+ return runReconciliation({
+ threads: [absent, corrupt, upsertFailure],
+ directory: {
+ getBinding: (candidate) =>
+ candidate === absent.id
+ ? Effect.succeed(Option.none())
+ : candidate === corrupt.id
+ ? Effect.fail(corruptFailure)
+ : Effect.succeed(
+ Option.some({
+ threadId: candidate,
+ provider: ProviderDriverKind.make("codex"),
+ providerInstanceId,
+ }),
+ ),
+ upsert: () => Effect.fail(writeFailure),
+ getProvider: () => Effect.die("unused"),
+ listThreadIds: () => Effect.die("unused"),
+ listBindings: () => Effect.die("unused"),
+ },
+ dispatch: (command) =>
+ Effect.sync(() => dispatched.push(command)).pipe(
+ Effect.as({ sequence: dispatched.length }),
+ ),
+ }).pipe(
+ Effect.tap(() =>
+ Effect.sync(() => {
+ assert.deepStrictEqual(
+ dispatched.map((command) => command.type === "thread.session.set" && command.threadId),
+ [absent.id, corrupt.id, upsertFailure.id],
+ );
+ }),
+ ),
+ );
+ },
+);
+
+it.effect("retries failed projections and continues after a persistent failure", () => {
+ const transient = makeThread("thread-dispatch-transient-failure", "running");
+ const persistent = makeThread("thread-dispatch-persistent-failure", "running");
+ const later = makeThread("thread-dispatch-success", "running");
+ const attempted: ThreadId[] = [];
+ let transientAttempts = 0;
+ const failure = new OrchestrationCommandInvariantError({
+ commandType: "thread.session.set",
+ detail: "simulated startup reconciliation failure",
+ });
+
+ return runReconciliation({
+ threads: [transient, persistent, later],
+ directory: {
+ getBinding: () => Effect.succeed(Option.none()),
+ upsert: () => Effect.void,
+ getProvider: () => Effect.die("unused"),
+ listThreadIds: () => Effect.die("unused"),
+ listBindings: () => Effect.die("unused"),
+ },
+ dispatch: (command) => {
+ if (command.type !== "thread.session.set") {
+ return Effect.die("unexpected command");
+ }
+ attempted.push(command.threadId);
+ if (command.threadId === transient.id && transientAttempts++ === 0) {
+ return Effect.fail(failure);
+ }
+ return command.threadId === persistent.id
+ ? Effect.fail(failure)
+ : Effect.succeed({ sequence: attempted.length });
+ },
+ }).pipe(
+ Effect.tap(() =>
+ Effect.sync(() =>
+ assert.deepStrictEqual(attempted, [
+ transient.id,
+ transient.id,
+ persistent.id,
+ persistent.id,
+ later.id,
+ ]),
+ ),
+ ),
+ );
+});
+
+it.effect("does not fail startup when the live provider session inventory cannot be read", () => {
+ let queried = false;
+ return ServerRuntimeStartup.reconcileProviderSessions.pipe(
+ Effect.provideService(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {
+ getCommandReadModel: () =>
+ Effect.sync(() => {
+ queried = true;
+ return { threads: [] } as never;
+ }),
+ } as unknown as ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"]),
+ Effect.provideService(ProviderService.ProviderService, {
+ ...makeProviderService(),
+ listSessions: () => Effect.die("provider inventory unavailable"),
+ }),
+ Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, {
+ getBinding: () => Effect.die("unused"),
+ upsert: () => Effect.die("unused"),
+ getProvider: () => Effect.die("unused"),
+ listThreadIds: () => Effect.die("unused"),
+ listBindings: () => Effect.die("unused"),
+ }),
+ Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {
+ readEvents: () => Stream.empty,
+ dispatch: () => Effect.die("unused"),
+ streamDomainEvents: Stream.empty,
+ latestSequence: Effect.succeed(0),
+ }),
+ Effect.provide(NodeServices.layer),
+ Effect.tap(() => Effect.sync(() => assert.equal(queried, false))),
+ );
+});
diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts
index 5db2b7555..6bea823b1 100644
--- a/apps/server/src/serverRuntimeStartup.ts
+++ b/apps/server/src/serverRuntimeStartup.ts
@@ -7,6 +7,7 @@ import {
ProviderInstanceId,
ThreadId,
} from "@t3tools/contracts";
+import * as Cause from "effect/Cause";
import * as Console from "effect/Console";
import * as Context from "effect/Context";
import * as Crypto from "effect/Crypto";
@@ -33,6 +34,8 @@ import * as ServerSettings from "./serverSettings.ts";
import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
+import * as ProviderService from "./provider/Services/ProviderService.ts";
+import * as ProviderSessionDirectory from "./provider/Services/ProviderSessionDirectory.ts";
import * as ProviderSessionReaper from "./provider/Services/ProviderSessionReaper.ts";
import { forkParked } from "./serverActivation.ts";
import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts";
@@ -290,6 +293,89 @@ const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>)
Effect.withSpan(`server.startup.${phase}`),
);
+const ORPHANED_PROVIDER_SESSION_ERROR =
+ "Provider session did not survive a server restart. Send a new message to continue.";
+
+export const reconcileProviderSessions = Effect.gen(function* () {
+ const crypto = yield* Crypto.Crypto;
+ const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;
+ const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;
+ const providerService = yield* ProviderService.ProviderService;
+ const query = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
+
+ const liveThreadIds = new Set(
+ (yield* providerService.listSessions()).map((session) => session.threadId),
+ );
+ const { threads } = yield* query.getCommandReadModel();
+ const orphanedThreads = threads.filter(
+ (thread) =>
+ thread.session !== null &&
+ (thread.session.status === "starting" ||
+ thread.session.status === "running" ||
+ thread.session.activeTurnId !== null) &&
+ !liveThreadIds.has(thread.id),
+ );
+
+ for (const thread of orphanedThreads) {
+ const session = thread.session;
+ if (session === null) {
+ continue;
+ }
+ yield* Effect.gen(function* () {
+ const binding = yield* directory.getBinding(thread.id);
+ if (Option.isSome(binding)) {
+ yield* directory.upsert({
+ ...binding.value,
+ status: "stopped",
+ runtimePayload: { activeTurnId: null },
+ });
+ }
+ }).pipe(
+ Effect.catchCause((cause) =>
+ Cause.hasInterrupts(cause)
+ ? Effect.failCause(cause)
+ : Effect.logWarning("failed to reconcile orphaned provider session directory binding", {
+ threadId: thread.id,
+ cause,
+ }),
+ ),
+ );
+
+ yield* Effect.gen(function* () {
+ const reconciledAt = DateTime.formatIso(yield* DateTime.now);
+ yield* orchestrationEngine.dispatch({
+ type: "thread.session.set",
+ commandId: CommandId.make(yield* crypto.randomUUIDv4),
+ threadId: thread.id,
+ session: {
+ ...session,
+ status: "error",
+ activeTurnId: null,
+ lastError: ORPHANED_PROVIDER_SESSION_ERROR,
+ updatedAt: reconciledAt,
+ },
+ createdAt: reconciledAt,
+ });
+ }).pipe(
+ Effect.retry({ times: 1 }),
+ Effect.catchCause((cause) =>
+ Cause.hasInterrupts(cause)
+ ? Effect.failCause(cause)
+ : Effect.logWarning("failed to settle orphaned provider session projection", {
+ threadId: thread.id,
+ cause,
+ }),
+ ),
+ );
+ }
+}).pipe(
+ Effect.catchCause((cause) =>
+ Cause.hasInterrupts(cause)
+ ? Effect.failCause(cause)
+ : Effect.logWarning("provider session startup reconciliation failed", { cause }),
+ ),
+);
+
interface StartupOptions {
readonly activate?: Effect.Effect<void>;
readonly awaitAuxiliaryParked?: Effect.Effect<void>;
@@ -354,6 +440,8 @@ export const make = (options?: StartupOptions) =>
}),
);
+ yield* runStartupPhase("provider-sessions.reconcile", reconcileProviderSessions);
+
const welcomeBase = yield* resolveWelcomeBase;
const environment = yield* serverEnvironment.getDescriptor;
yield* Effect.logDebug("startup phase: preparing welcome payload");