Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
fix(server): reconcile orphaned provider sessions (#7719)
t3-code[bot]0929907ff9ae
diff --git a/.macroscope/check-run-agents/effect-service-conventions.md b/.macroscope/check-run-agents/effect-service-conventions.mdindex 542e9028d..b76d56d45 100644--- a/.macroscope/check-run-agents/effect-service-conventions.md+++ b/.macroscope/check-run-agents/effect-service-conventions.md@@ -68,6 +68,7 @@ Review changed TypeScript and directly affected call sites for the conventions b- Export direct schema predicates such as `export const isFoo = Schema.is(Foo)`. Flag a private `Schema.is` constant wrapped by a redundant function with the same signature.- Do not introduce a large `switch` or lookup table in an error's `message` getter to model failures that deserve separate error classes.- Catch statically known tagged failures with `Effect.catchTags({ ... })`, including when handling only one tag. Do not use `catchIf` with a schema predicate merely to recover one or more known `_tag` variants, and do not use `catchTag`. `Effect.catch` is appropriate when the entire error channel is intentionally handled; `catchIf` remains appropriate for genuinely structural predicates such as inspecting an underlying platform error code.+- For startup reconciliation that repairs multiple independent entities, preserve interruption rather than reducing it to a warning. Retry a transient per-entity repair before readiness, then isolate a persistent failure so one bad entity cannot abort global startup or prevent later entities from being repaired. Require tests for both the retry-success path and persistent-failure continuation.- Do not add a helper whose only behavior is `(...args) => new SomeError({ ...args })`, including curried aliases used once with `mapError`. Construct the error at the failure boundary so its attributes and cause remain visible. Keep a mapper only when it performs real normalization, passes through existing domain errors, or adds reusable context/control flow.- When a reusable error-to-error translation clearly belongs to the target error type, prefer a descriptive static factory on that error class over a detached production-side switch. Do not force a static method for one-off inline mappings.diff --git a/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts b/apps/server/integration/orphanedProviderSessionStartup.integration.test.tsnew file mode 100644index 000000000..2a351cd6b--- /dev/null+++ b/apps/server/integration/orphanedProviderSessionStartup.integration.test.ts@@ -0,0 +1,358 @@+import * as NodeServices from "@effect/platform-node/NodeServices";+import {+ CommandId,+ DEFAULT_PROVIDER_INTERACTION_MODE,+ EnvironmentId,+ MessageId,+ ProjectId,+ ProviderDriverKind,+ ProviderInstanceId,+ ThreadId,+} from "@t3tools/contracts";+import { assert, it } from "@effect/vitest";+import * as DateTime from "effect/DateTime";+import * as Effect from "effect/Effect";+import * as Exit from "effect/Exit";+import * as Layer from "effect/Layer";+import * as Option from "effect/Option";+import * as Stream from "effect/Stream";+import * as SqlClient from "effect/unstable/sql/SqlClient";+import { HttpServer } from "effect/unstable/http";++import * as EnvironmentAuth from "../src/auth/EnvironmentAuth.ts";+import * as ServiceLauncherClient from "../src/cloud/serviceLauncherClient.ts";+import * as ServerConfig from "../src/config.ts";+import * as ServerEnvironment from "../src/environment/ServerEnvironment.ts";+import * as Keybindings from "../src/keybindings.ts";+import { OrchestrationLayerLive } from "../src/orchestration/runtimeLayer.ts";+import * as OrchestrationEngine from "../src/orchestration/Services/OrchestrationEngine.ts";+import * as OrchestrationReactor from "../src/orchestration/Services/OrchestrationReactor.ts";+import * as ProjectionSnapshotQuery from "../src/orchestration/Services/ProjectionSnapshotQuery.ts";+import { makeSqlitePersistenceLive } from "../src/persistence/Layers/Sqlite.ts";+import * as ProviderSessionRuntime from "../src/persistence/ProviderSessionRuntime.ts";+import * as ExternalLauncher from "../src/process/externalLauncher.ts";+import { ProviderSessionDirectoryLive } from "../src/provider/Layers/ProviderSessionDirectory.ts";+import * as ProviderService from "../src/provider/Services/ProviderService.ts";+import * as ProviderSessionDirectory from "../src/provider/Services/ProviderSessionDirectory.ts";+import * as ProviderSessionReaper from "../src/provider/Services/ProviderSessionReaper.ts";+import * as RepositoryIdentityResolver from "../src/project/RepositoryIdentityResolver.ts";+import * as ServerLifecycleEvents from "../src/serverLifecycleEvents.ts";+import * as ServerRuntimeStartup from "../src/serverRuntimeStartup.ts";+import * as ServerSettings from "../src/serverSettings.ts";+import * as AnalyticsService from "../src/telemetry/AnalyticsService.ts";++const providerInstanceId = ProviderInstanceId.make("codex");+const projectId = ProjectId.make("project-startup-orphan");+const threadId = ThreadId.make("thread-startup-orphan");+const stoppedBindingThreadId = ThreadId.make("thread-startup-orphan-stopped-binding");+const resumeCursor = { schemaVersion: 1, sessionId: "provider-session-before-restart" };+const stoppedBindingResumeCursor = {+ schemaVersion: 1,+ sessionId: "provider-session-stopped-before-restart",+};++const makePersistedRuntimeLayer = (dbPath: string) => {+ const persistence = makeSqlitePersistenceLive(dbPath);+ const orchestration = OrchestrationLayerLive.pipe(+ Layer.provideMerge(RepositoryIdentityResolver.layer),+ Layer.provideMerge(persistence),+ );+ const directory = ProviderSessionDirectoryLive.pipe(+ Layer.provide(ProviderSessionRuntime.layer),+ Layer.provide(persistence),+ );+ return Layer.mergeAll(orchestration, directory);+};++const startupDependencies = Layer.mergeAll(+ Layer.mock(Keybindings.Keybindings)({+ start: Effect.void,+ }),+ ServerSettings.layerTest(),+ Layer.succeed(OrchestrationReactor.OrchestrationReactor, {+ start: () => Effect.void,+ }),+ Layer.succeed(ProviderSessionReaper.ProviderSessionReaper, {+ start: () => Effect.void,+ }),+ ServerLifecycleEvents.layer,+ Layer.succeed(ServerEnvironment.ServerEnvironment, {+ getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-startup-orphan")),+ getDescriptor: Effect.succeed({+ environmentId: EnvironmentId.make("environment-startup-orphan"),+ label: "Startup orphan test",+ version: "test",+ platform: { os: "linux", arch: "x64" },+ capabilities: {},+ } as never),+ }),+ Layer.mock(EnvironmentAuth.EnvironmentAuth)({+ issueStartupPairingUrl: (baseUrl: string) => Effect.succeed(`${baseUrl}/pair`),+ }),+ Layer.mock(ExternalLauncher.ExternalLauncher)({+ launchBrowser: () => Effect.void,+ }),+ Layer.succeed(ServiceLauncherClient.ServiceLauncherClient, {+ managed: false,+ requestUpdate: () => Effect.die("unused"),+ prepareTrial: Effect.sync(() => undefined),+ }),+ Layer.succeed(+ HttpServer.HttpServer,+ HttpServer.HttpServer.of({+ address: { _tag: "TcpAddress", hostname: "127.0.0.1", port: 3773 },+ serve: (() => Effect.void) as HttpServer.HttpServer["Service"]["serve"],+ }),+ ),+ AnalyticsService.layerTest,+ Layer.succeed(ProviderService.ProviderService, {+ startSession: () => Effect.die("unused"),+ sendTurn: () => Effect.die("unused"),+ interruptTurn: () => Effect.die("unused"),+ respondToRequest: () => Effect.die("unused"),+ respondToUserInput: () => Effect.die("unused"),+ stopSession: () => Effect.die("unused"),+ listSessions: () => Effect.succeed([]),+ getCapabilities: () => Effect.die("unused"),+ getInstanceInfo: () => Effect.die("unused"),+ rollbackConversation: () => Effect.die("unused"),+ streamEvents: Stream.empty,+ }),+);++it.effect(+ "recovers a persisted starting session before opening the command gate after restart",+ () =>+ Effect.gen(function* () {+ const config = yield* ServerConfig.ServerConfig;+ const firstRuntime = makePersistedRuntimeLayer(config.dbPath);+ const now = yield* DateTime.now;+ const createdAt = DateTime.formatIso(now);++ yield* Effect.gen(function* () {+ const engine = yield* OrchestrationEngine.OrchestrationEngineService;+ const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;++ yield* engine.dispatch({+ type: "project.create",+ commandId: CommandId.make("command-create-project"),+ projectId,+ title: "Startup orphan project",+ workspaceRoot: "/tmp/startup-orphan-project",+ defaultModelSelection: { instanceId: providerInstanceId, model: "gpt-5" },+ createdAt,+ });+ yield* engine.dispatch({+ type: "thread.create",+ commandId: CommandId.make("command-create-thread"),+ threadId,+ projectId,+ title: "Startup orphan thread",+ modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ branch: null,+ worktreePath: null,+ createdAt,+ });+ yield* engine.dispatch({+ type: "thread.turn.start",+ commandId: CommandId.make("command-start-pending-turn"),+ threadId,+ message: {+ messageId: MessageId.make("message-pending-before-restart"),+ role: "user",+ text: "Persist this queued turn before restart",+ attachments: [],+ },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ createdAt,+ });+ yield* engine.dispatch({+ type: "thread.session.set",+ commandId: CommandId.make("command-mark-session-starting"),+ threadId,+ session: {+ threadId,+ status: "starting",+ providerName: "codex",+ providerInstanceId,+ runtimeMode: "full-access",+ activeTurnId: null,+ lastError: null,+ updatedAt: createdAt,+ },+ createdAt,+ });+ yield* directory.upsert({+ threadId,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId,+ status: "running",+ resumeCursor,+ runtimePayload: { activeTurnId: null, unrelated: "preserve-me" },+ runtimeMode: "full-access",+ });+ yield* engine.dispatch({+ type: "thread.create",+ commandId: CommandId.make("command-create-stopped-binding-thread"),+ threadId: stoppedBindingThreadId,+ projectId,+ title: "Startup orphan with stopped binding",+ modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ branch: null,+ worktreePath: null,+ createdAt,+ });+ yield* engine.dispatch({+ type: "thread.turn.start",+ commandId: CommandId.make("command-start-stopped-binding-pending-turn"),+ threadId: stoppedBindingThreadId,+ message: {+ messageId: MessageId.make("message-stopped-binding-pending-before-restart"),+ role: "user",+ text: "Persist another queued turn before restart",+ attachments: [],+ },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ createdAt,+ });+ yield* engine.dispatch({+ type: "thread.session.set",+ commandId: CommandId.make("command-mark-stopped-binding-session-starting"),+ threadId: stoppedBindingThreadId,+ session: {+ threadId: stoppedBindingThreadId,+ status: "starting",+ providerName: "codex",+ providerInstanceId,+ runtimeMode: "full-access",+ activeTurnId: null,+ lastError: null,+ updatedAt: createdAt,+ },+ createdAt,+ });+ yield* directory.upsert({+ threadId: stoppedBindingThreadId,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId,+ status: "stopped",+ resumeCursor: stoppedBindingResumeCursor,+ runtimePayload: { activeTurnId: "stale", unrelated: "also-preserve-me" },+ runtimeMode: "full-access",+ });+ }).pipe(Effect.provide(firstRuntime));++ const secondRuntime = makePersistedRuntimeLayer(config.dbPath);+ const startupLayer = ServerRuntimeStartup.layer.pipe(+ Layer.provideMerge(secondRuntime),+ Layer.provideMerge(startupDependencies),+ );++ const result = yield* Effect.gen(function* () {+ const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup;+ const engine = yield* OrchestrationEngine.OrchestrationEngineService;+ const query = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;+ const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;+ const sql = yield* SqlClient.SqlClient;++ yield* startup.markHttpListening;+ yield* startup.awaitCommandReady;++ const restartedThread = Option.getOrThrow(yield* query.getThreadDetailById(threadId));+ const restartedStoppedBindingThread = Option.getOrThrow(+ yield* query.getThreadDetailById(stoppedBindingThreadId),+ );+ const pendingRows = yield* sql<{ readonly threadId: string }>`+ SELECT thread_id AS "threadId"+ FROM projection_turns+ WHERE thread_id IN (${threadId}, ${stoppedBindingThreadId})+ AND turn_id IS NULL+ AND state = 'pending'+ `;+ const settleExit = yield* Effect.exit(+ engine.dispatch({+ type: "thread.settle",+ commandId: CommandId.make("command-settle-after-restart"),+ threadId,+ }),+ );+ const snoozeExit = yield* Effect.exit(+ engine.dispatch({+ type: "thread.snooze",+ commandId: CommandId.make("command-snooze-after-restart"),+ threadId,+ snoozedUntil: DateTime.formatIso(DateTime.add(now, { hours: 1 })),+ }),+ );+ const newTurnExit = yield* Effect.exit(+ engine.dispatch({+ type: "thread.turn.start",+ commandId: CommandId.make("command-new-turn-after-restart"),+ threadId,+ message: {+ messageId: MessageId.make("message-new-turn-after-restart"),+ role: "user",+ text: "Continue immediately after restart",+ attachments: [],+ },+ interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE,+ runtimeMode: "full-access",+ createdAt,+ }),+ );+ const binding = Option.getOrThrow(yield* directory.getBinding(threadId));+ const stoppedBinding = Option.getOrThrow(+ yield* directory.getBinding(stoppedBindingThreadId),+ );++ return {+ sessionStatus: restartedThread.session?.status,+ activeTurnId: restartedThread.session?.activeTurnId,+ latestTurn: restartedThread.latestTurn,+ pendingTurnCount: pendingRows.length,+ settleSucceeded: Exit.isSuccess(settleExit),+ snoozeSucceeded: Exit.isSuccess(snoozeExit),+ newTurnSucceeded: Exit.isSuccess(newTurnExit),+ bindingStatus: binding.status,+ resumeCursor: binding.resumeCursor,+ runtimePayload: binding.runtimePayload,+ stoppedBindingSessionStatus: restartedStoppedBindingThread.session?.status,+ stoppedBindingStatus: stoppedBinding.status,+ stoppedBindingResumeCursor: stoppedBinding.resumeCursor,+ stoppedBindingRuntimePayload: stoppedBinding.runtimePayload,+ };+ }).pipe(Effect.provide(startupLayer));++ assert.deepStrictEqual(result, {+ sessionStatus: "error",+ activeTurnId: null,+ latestTurn: null,+ pendingTurnCount: 0,+ settleSucceeded: true,+ snoozeSucceeded: true,+ newTurnSucceeded: true,+ bindingStatus: "stopped",+ resumeCursor,+ runtimePayload: { activeTurnId: null, unrelated: "preserve-me" },+ stoppedBindingSessionStatus: "error",+ stoppedBindingStatus: "stopped",+ stoppedBindingResumeCursor,+ stoppedBindingRuntimePayload: {+ activeTurnId: null,+ unrelated: "also-preserve-me",+ },+ });+ }).pipe(+ Effect.provide(+ ServerConfig.layerTest(process.cwd(), {+ prefix: "t3-orphaned-provider-session-startup-",+ }).pipe(Layer.provideMerge(NodeServices.layer)),+ ),+ ),+);diff --git a/apps/server/src/serverRuntimeStartup.reconcile.test.ts b/apps/server/src/serverRuntimeStartup.reconcile.test.tsnew file mode 100644index 000000000..60cb8d61b--- /dev/null+++ b/apps/server/src/serverRuntimeStartup.reconcile.test.ts@@ -0,0 +1,296 @@+import * as NodeServices from "@effect/platform-node/NodeServices";+import {+ type OrchestrationCommand,+ ProviderDriverKind,+ ProviderInstanceId,+ ThreadId,+ TurnId,+} from "@t3tools/contracts";+import { assert, it } from "@effect/vitest";+import * as Effect from "effect/Effect";+import * as Option from "effect/Option";+import * as Stream from "effect/Stream";++import { OrchestrationCommandInvariantError } from "./orchestration/Errors.ts";+import * as OrchestrationEngine from "./orchestration/Services/OrchestrationEngine.ts";+import * as ProjectionSnapshotQuery from "./orchestration/Services/ProjectionSnapshotQuery.ts";+import { ProviderSessionDirectoryPersistenceError } from "./provider/Errors.ts";+import * as ProviderService from "./provider/Services/ProviderService.ts";+import * as ProviderSessionDirectory from "./provider/Services/ProviderSessionDirectory.ts";+import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts";++const providerInstanceId = ProviderInstanceId.make("codex");+const updatedAt = "2026-08-20T12:00:00.000Z";++const makeThread = (+ id: string,+ status: "starting" | "running" | "ready" | "stopped" | "error",+ activeTurnId: TurnId | null = null,+ archivedAt: string | null = null,+) => ({+ id: ThreadId.make(id),+ archivedAt,+ deletedAt: null,+ session: {+ threadId: ThreadId.make(id),+ status,+ providerName: "codex" as const,+ providerInstanceId,+ runtimeMode: "full-access" as const,+ activeTurnId,+ lastError: null,+ updatedAt,+ },+});++const makeProviderService = (liveThreadIds: ReadonlyArray<ThreadId> = []) =>+ ({+ startSession: () => Effect.die("unused"),+ sendTurn: () => Effect.die("unused"),+ interruptTurn: () => Effect.die("unused"),+ respondToRequest: () => Effect.die("unused"),+ respondToUserInput: () => Effect.die("unused"),+ stopSession: () => Effect.die("unused"),+ listSessions: () => Effect.succeed(liveThreadIds.map((threadId) => ({ threadId }) as never)),+ getCapabilities: () => Effect.die("unused"),+ getInstanceInfo: () => Effect.die("unused"),+ rollbackConversation: () => Effect.die("unused"),+ streamEvents: Stream.empty,+ }) satisfies ProviderService.ProviderService["Service"];++const queryWithThreads = (threads: ReadonlyArray<ReturnType<typeof makeThread>>) =>+ ({+ getCommandReadModel: () => Effect.succeed({ threads } as never),+ }) as unknown as ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"];++const runReconciliation = (input: {+ readonly threads: ReadonlyArray<ReturnType<typeof makeThread>>;+ readonly liveThreadIds?: ReadonlyArray<ThreadId>;+ readonly directory: ProviderSessionDirectory.ProviderSessionDirectory["Service"];+ readonly dispatch: OrchestrationEngine.OrchestrationEngineService["Service"]["dispatch"];+}) =>+ ServerRuntimeStartup.reconcileProviderSessions.pipe(+ Effect.provideService(+ ProjectionSnapshotQuery.ProjectionSnapshotQuery,+ queryWithThreads(input.threads),+ ),+ Effect.provideService(+ ProviderService.ProviderService,+ makeProviderService(input.liveThreadIds),+ ),+ Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, input.directory),+ Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {+ readEvents: () => Stream.empty,+ dispatch: input.dispatch,+ streamDomainEvents: Stream.empty,+ latestSequence: Effect.succeed(0),+ }),+ Effect.provide(NodeServices.layer),+ );++it.effect("reconciles multiple active and archived orphans but skips live sessions", () => {+ const starting = makeThread("thread-starting", "starting");+ const running = makeThread("thread-running", "running", TurnId.make("turn-running"));+ const staleActiveTurn = makeThread(+ "thread-stale-active-turn",+ "ready",+ TurnId.make("turn-stale-active"),+ );+ const archived = makeThread(+ "thread-archived",+ "running",+ TurnId.make("turn-archived"),+ updatedAt,+ );+ const live = makeThread("thread-live", "running", TurnId.make("turn-live"));+ const settled = makeThread("thread-ready", "ready");+ const dispatched: OrchestrationCommand[] = [];+ const bindingReads: ThreadId[] = [];+ const upserts: ProviderSessionDirectory.ProviderRuntimeBinding[] = [];++ return runReconciliation({+ threads: [starting, running, staleActiveTurn, archived, live, settled],+ liveThreadIds: [live.id],+ directory: {+ getBinding: (candidate) =>+ Effect.sync(() => bindingReads.push(candidate)).pipe(+ Effect.as(+ Option.some({+ threadId: candidate,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId,+ status: "running" as const,+ resumeCursor: { cursor: candidate },+ runtimePayload: { activeTurnId: "stale", unrelated: candidate },+ }),+ ),+ ),+ upsert: (binding) => Effect.sync(() => upserts.push(binding)),+ getProvider: () => Effect.die("unused"),+ listThreadIds: () => Effect.die("unused"),+ listBindings: () => Effect.die("unused"),+ },+ dispatch: (command) =>+ Effect.sync(() => dispatched.push(command)).pipe(Effect.as({ sequence: dispatched.length })),+ }).pipe(+ Effect.tap(() =>+ Effect.sync(() => {+ const orphanIds = [starting.id, running.id, staleActiveTurn.id, archived.id];+ assert.deepStrictEqual(bindingReads, orphanIds);+ assert.deepStrictEqual(+ dispatched.map((command) => command.type === "thread.session.set" && command.threadId),+ orphanIds,+ );+ assert.deepStrictEqual(+ dispatched.map((command) =>+ command.type === "thread.session.set"+ ? {+ status: command.session.status,+ activeTurnId: command.session.activeTurnId,+ }+ : null,+ ),+ orphanIds.map(() => ({ status: "error" as const, activeTurnId: null })),+ );+ assert.equal(upserts.length, orphanIds.length);+ for (const binding of upserts) {+ assert.equal(binding.status, "stopped");+ assert.deepStrictEqual(binding.runtimePayload, { activeTurnId: null });+ assert.deepStrictEqual(binding.resumeCursor, { cursor: binding.threadId });+ }+ }),+ ),+ );+});++it.effect(+ "settles projections when directory bindings are absent, corrupt, or fail to upsert",+ () => {+ const absent = makeThread("thread-binding-absent", "starting");+ const corrupt = makeThread("thread-binding-corrupt", "running");+ const upsertFailure = makeThread("thread-binding-upsert-failure", "running");+ const dispatched: OrchestrationCommand[] = [];+ const corruptFailure = new ProviderSessionDirectoryPersistenceError({+ operation: "ProviderSessionDirectory.getBinding",+ detail: "corrupt persisted binding",+ });+ const writeFailure = new ProviderSessionDirectoryPersistenceError({+ operation: "ProviderSessionDirectory.upsert",+ detail: "failed binding write",+ });++ return runReconciliation({+ threads: [absent, corrupt, upsertFailure],+ directory: {+ getBinding: (candidate) =>+ candidate === absent.id+ ? Effect.succeed(Option.none())+ : candidate === corrupt.id+ ? Effect.fail(corruptFailure)+ : Effect.succeed(+ Option.some({+ threadId: candidate,+ provider: ProviderDriverKind.make("codex"),+ providerInstanceId,+ }),+ ),+ upsert: () => Effect.fail(writeFailure),+ getProvider: () => Effect.die("unused"),+ listThreadIds: () => Effect.die("unused"),+ listBindings: () => Effect.die("unused"),+ },+ dispatch: (command) =>+ Effect.sync(() => dispatched.push(command)).pipe(+ Effect.as({ sequence: dispatched.length }),+ ),+ }).pipe(+ Effect.tap(() =>+ Effect.sync(() => {+ assert.deepStrictEqual(+ dispatched.map((command) => command.type === "thread.session.set" && command.threadId),+ [absent.id, corrupt.id, upsertFailure.id],+ );+ }),+ ),+ );+ },+);++it.effect("retries failed projections and continues after a persistent failure", () => {+ const transient = makeThread("thread-dispatch-transient-failure", "running");+ const persistent = makeThread("thread-dispatch-persistent-failure", "running");+ const later = makeThread("thread-dispatch-success", "running");+ const attempted: ThreadId[] = [];+ let transientAttempts = 0;+ const failure = new OrchestrationCommandInvariantError({+ commandType: "thread.session.set",+ detail: "simulated startup reconciliation failure",+ });++ return runReconciliation({+ threads: [transient, persistent, later],+ directory: {+ getBinding: () => Effect.succeed(Option.none()),+ upsert: () => Effect.void,+ getProvider: () => Effect.die("unused"),+ listThreadIds: () => Effect.die("unused"),+ listBindings: () => Effect.die("unused"),+ },+ dispatch: (command) => {+ if (command.type !== "thread.session.set") {+ return Effect.die("unexpected command");+ }+ attempted.push(command.threadId);+ if (command.threadId === transient.id && transientAttempts++ === 0) {+ return Effect.fail(failure);+ }+ return command.threadId === persistent.id+ ? Effect.fail(failure)+ : Effect.succeed({ sequence: attempted.length });+ },+ }).pipe(+ Effect.tap(() =>+ Effect.sync(() =>+ assert.deepStrictEqual(attempted, [+ transient.id,+ transient.id,+ persistent.id,+ persistent.id,+ later.id,+ ]),+ ),+ ),+ );+});++it.effect("does not fail startup when the live provider session inventory cannot be read", () => {+ let queried = false;+ return ServerRuntimeStartup.reconcileProviderSessions.pipe(+ Effect.provideService(ProjectionSnapshotQuery.ProjectionSnapshotQuery, {+ getCommandReadModel: () =>+ Effect.sync(() => {+ queried = true;+ return { threads: [] } as never;+ }),+ } as unknown as ProjectionSnapshotQuery.ProjectionSnapshotQuery["Service"]),+ Effect.provideService(ProviderService.ProviderService, {+ ...makeProviderService(),+ listSessions: () => Effect.die("provider inventory unavailable"),+ }),+ Effect.provideService(ProviderSessionDirectory.ProviderSessionDirectory, {+ getBinding: () => Effect.die("unused"),+ upsert: () => Effect.die("unused"),+ getProvider: () => Effect.die("unused"),+ listThreadIds: () => Effect.die("unused"),+ listBindings: () => Effect.die("unused"),+ }),+ Effect.provideService(OrchestrationEngine.OrchestrationEngineService, {+ readEvents: () => Stream.empty,+ dispatch: () => Effect.die("unused"),+ streamDomainEvents: Stream.empty,+ latestSequence: Effect.succeed(0),+ }),+ Effect.provide(NodeServices.layer),+ Effect.tap(() => Effect.sync(() => assert.equal(queried, false))),+ );+});diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.tsindex 5db2b7555..6bea823b1 100644--- a/apps/server/src/serverRuntimeStartup.ts+++ b/apps/server/src/serverRuntimeStartup.ts@@ -7,6 +7,7 @@ import {ProviderInstanceId,ThreadId,} from "@t3tools/contracts";+import * as Cause from "effect/Cause";import * as Console from "effect/Console";import * as Context from "effect/Context";import * as Crypto from "effect/Crypto";@@ -33,6 +34,8 @@ import * as ServerSettings from "./serverSettings.ts";import * as AnalyticsService from "./telemetry/AnalyticsService.ts";import * as ServerEnvironment from "./environment/ServerEnvironment.ts";import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";+import * as ProviderService from "./provider/Services/ProviderService.ts";+import * as ProviderSessionDirectory from "./provider/Services/ProviderSessionDirectory.ts";import * as ProviderSessionReaper from "./provider/Services/ProviderSessionReaper.ts";import { forkParked } from "./serverActivation.ts";import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts";@@ -290,6 +293,89 @@ const runStartupPhase = <A, E, R>(phase: string, effect: Effect.Effect<A, E, R>)Effect.withSpan(`server.startup.${phase}`),);+const ORPHANED_PROVIDER_SESSION_ERROR =+ "Provider session did not survive a server restart. Send a new message to continue.";++export const reconcileProviderSessions = Effect.gen(function* () {+ const crypto = yield* Crypto.Crypto;+ const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory;+ const orchestrationEngine = yield* OrchestrationEngine.OrchestrationEngineService;+ const providerService = yield* ProviderService.ProviderService;+ const query = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;++ const liveThreadIds = new Set(+ (yield* providerService.listSessions()).map((session) => session.threadId),+ );+ const { threads } = yield* query.getCommandReadModel();+ const orphanedThreads = threads.filter(+ (thread) =>+ thread.session !== null &&+ (thread.session.status === "starting" ||+ thread.session.status === "running" ||+ thread.session.activeTurnId !== null) &&+ !liveThreadIds.has(thread.id),+ );++ for (const thread of orphanedThreads) {+ const session = thread.session;+ if (session === null) {+ continue;+ }+ yield* Effect.gen(function* () {+ const binding = yield* directory.getBinding(thread.id);+ if (Option.isSome(binding)) {+ yield* directory.upsert({+ ...binding.value,+ status: "stopped",+ runtimePayload: { activeTurnId: null },+ });+ }+ }).pipe(+ Effect.catchCause((cause) =>+ Cause.hasInterrupts(cause)+ ? Effect.failCause(cause)+ : Effect.logWarning("failed to reconcile orphaned provider session directory binding", {+ threadId: thread.id,+ cause,+ }),+ ),+ );++ yield* Effect.gen(function* () {+ const reconciledAt = DateTime.formatIso(yield* DateTime.now);+ yield* orchestrationEngine.dispatch({+ type: "thread.session.set",+ commandId: CommandId.make(yield* crypto.randomUUIDv4),+ threadId: thread.id,+ session: {+ ...session,+ status: "error",+ activeTurnId: null,+ lastError: ORPHANED_PROVIDER_SESSION_ERROR,+ updatedAt: reconciledAt,+ },+ createdAt: reconciledAt,+ });+ }).pipe(+ Effect.retry({ times: 1 }),+ Effect.catchCause((cause) =>+ Cause.hasInterrupts(cause)+ ? Effect.failCause(cause)+ : Effect.logWarning("failed to settle orphaned provider session projection", {+ threadId: thread.id,+ cause,+ }),+ ),+ );+ }+}).pipe(+ Effect.catchCause((cause) =>+ Cause.hasInterrupts(cause)+ ? Effect.failCause(cause)+ : Effect.logWarning("provider session startup reconciliation failed", { cause }),+ ),+);+interface StartupOptions {readonly activate?: Effect.Effect<void>;readonly awaitAuxiliaryParked?: Effect.Effect<void>;@@ -354,6 +440,8 @@ export const make = (options?: StartupOptions) =>}),);+ yield* runStartupPhase("provider-sessions.reconcile", reconcileProviderSessions);+const welcomeBase = yield* resolveWelcomeBase;const environment = yield* serverEnvironment.getDescriptor;yield* Effect.logDebug("startup phase: preparing welcome payload");