Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
fix(server): allow remote updates with database migrations (#5374)
Theo Browne7b38fb5c6b0f
diff --git a/apps/server/src/cli/connect.test.ts b/apps/server/src/cli/connect.test.tsindex cc52a50fe..1e0c88c24 100644--- a/apps/server/src/cli/connect.test.ts+++ b/apps/server/src/cli/connect.test.ts@@ -10,6 +10,7 @@ import * as Option from "effect/Option";import * as References from "effect/References";import * as Terminal from "effect/Terminal";+import * as BootService from "../cloud/bootService.ts";import {acquireRelayClientForLink,formatHeadlessAuthorizationPrompt,@@ -63,6 +64,15 @@ it.effect("treats cancelling optional background setup as a successful skip", ()}),);+it.effect("keeps a successful connection when a remote service update is pending", () =>+ Effect.gen(function* () {+ const result = yield* recoverServiceOnboardingOffer(+ Effect.fail(new BootService.BootServiceUpdatePendingError()),+ );+ assert.isFalse(result);+ }),+);+it.effect("does not install the relay client when the user declines the managed download", () =>Effect.gen(function* () {let installCalls = 0;diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.tsindex bd846eeee..d55b270f1 100644--- a/apps/server/src/cli/service.ts+++ b/apps/server/src/cli/service.ts@@ -185,6 +185,8 @@ export const recoverServiceOnboardingOffer = <R>(Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),BootServiceInstallError: (error) =>Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),+ BootServiceUpdatePendingError: (error) =>+ Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)),}),);diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.tsindex 9af69eb17..a9ae3f49f 100644--- a/apps/server/src/cloud/bootService.test.ts+++ b/apps/server/src/cloud/bootService.test.ts@@ -15,7 +15,11 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawneimport * as ProcessRunner from "../processRunner.ts";import * as BootService from "./bootService.ts";import { pinnedRuntimePaths } from "./pinnedRuntime.ts";-import { parseServiceState } from "./serviceProtocol.ts";+import {+ parseServiceState,+ SERVICE_LAUNCHER_PROTOCOL,+ serviceStateHasPendingUpdate,+} from "./serviceProtocol.ts";it("keeps systemd pinned to the stable launcher rather than a versioned server", () => {const unit = BootService.renderBootServiceUnit({@@ -97,15 +101,24 @@ it.layer(NodeServices.layer)("boot service install", (it) => {const plan = yield* service.install;expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,activeVersion: "1.2.3",});expect(yield* fs.readFileString(plan.launcherPath)).toBe("export {};\n");expect((yield* service.status).current).toBe(true);- yield* fs.writeFileString(- statePath,- '{"protocol":1,"activeVersion":"1.2.3","update":{"id":"u","fromVersion":"1.2.3","targetVersion":"1.2.4","status":"pending"}}',- );+ // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document.+ const pendingState = JSON.stringify({+ protocol: SERVICE_LAUNCHER_PROTOCOL,+ activeVersion: "1.2.3",+ update: {+ id: "u",+ fromVersion: "1.2.3",+ targetVersion: "1.2.4",+ dbPath: "/tmp/state.sqlite",+ status: "pending",+ },+ });+ yield* fs.writeFileString(statePath, pendingState);expect((yield* service.status).current).toBe(false);expect(yield* service.uninstall).toBe(true);expect((yield* service.status).installed).toBe(false);@@ -141,6 +154,33 @@ it.layer(NodeServices.layer)("boot service install", (it) => {}),);+ it.effect("restarts without overwriting a pending remote update", () =>+ Effect.gen(function* () {+ const { service, fs, statePath, commands } = yield* makeHarness();+ yield* service.install;+ // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document.+ const pendingState = JSON.stringify({+ protocol: SERVICE_LAUNCHER_PROTOCOL - 1,+ activeVersion: "1.2.3",+ update: {+ id: "remote-update",+ fromVersion: "1.2.3",+ targetVersion: "1.2.4",+ status: "pending",+ },+ });+ yield* fs.writeFileString(statePath, pendingState);+ commands.length = 0;++ expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUpdatePendingError");+ expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true);+ expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([+ "systemctl --user stop t3code.service",+ "systemctl --user restart t3code.service",+ ]);+ }),+ );+it.effect("fails closed off Linux", () =>Effect.gen(function* () {const { service } = yield* makeHarness("darwin");diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.tsindex 9a8481b11..ec110cb8b 100644--- a/apps/server/src/cloud/bootService.ts+++ b/apps/server/src/cloud/bootService.ts@@ -21,6 +21,7 @@ import {SERVICE_LAUNCHER_PROTOCOL,SERVICE_STATE_FILE,parseServiceState,+ serviceStateHasPendingUpdate,type ServiceState,} from "./serviceProtocol.ts";@@ -110,10 +111,20 @@ export class BootServiceInstallError extends Schema.TaggedErrorClass<BootService}}+export class BootServiceUpdatePendingError extends Schema.TaggedErrorClass<BootServiceUpdatePendingError>()(+ "BootServiceUpdatePendingError",+ {},+) {+ override get message(): string {+ return "A remote server update is still pending. Wait for it to finish, then retry.";+ }+}+export type BootServiceError =| BootServiceUnsupportedError| BootServiceCommandError- | BootServiceInstallError;+ | BootServiceInstallError+ | BootServiceUpdatePendingError;export interface BootServiceStatus {readonly supported: boolean;@@ -288,6 +299,15 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {}yield* Effect.gen(function* () {+ if (installed) {+ const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option);+ if (+ Option.isSome(previousStateText) &&+ serviceStateHasPendingUpdate(previousStateText.value)+ ) {+ return yield* new BootServiceUpdatePendingError();+ }+ }yield* fs.makeDirectory(unitDir, { recursive: true }).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.tsindex 6fe1d5a4a..276ee0377 100644--- a/apps/server/src/cloud/selfUpdate.test.ts+++ b/apps/server/src/cloud/selfUpdate.test.ts@@ -11,6 +11,7 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawneimport * as ServerConfig from "../config.ts";import * as ProcessRunner from "../processRunner.ts";import * as ServiceLauncherClient from "./serviceLauncherClient.ts";+import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";import * as ServerSelfUpdate from "./selfUpdate.ts";interface HarnessOptions {@@ -50,7 +51,11 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (const result =options.preflight === "blocked"? { status: "blocked", version: "1.1.0", reason: "local update required" }- : { status: "ready", version: "1.1.0", launcherProtocol: 1 };+ : {+ status: "ready",+ version: "1.1.0",+ launcherProtocol: SERVICE_LAUNCHER_PROTOCOL,+ };return {// @effect-diagnostics-next-line preferSchemaOverJson:off - fake child-process stdout.stdout: JSON.stringify(result),@@ -64,7 +69,6 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* (});const launcher = ServiceLauncherClient.ServiceLauncherClient.of({managed: options.managed ?? true,- trial: false,requestUpdate:options.requestUpdate ??(() =>diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.tsindex 58bb84117..015fd557d 100644--- a/apps/server/src/cloud/selfUpdate.ts+++ b/apps/server/src/cloud/selfUpdate.ts@@ -170,7 +170,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () {yield* reportProgress("installing");const updateId = yield* launcher- .requestUpdate({ targetVersion })+ .requestUpdate({ targetVersion, dbPath: serverConfig.dbPath }).pipe(Effect.mapError((error) =>failWith(diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.tsindex 6b9e926ff..eab093541 100644--- a/apps/server/src/cloud/serviceLauncherClient.test.ts+++ b/apps/server/src/cloud/serviceLauncherClient.test.ts@@ -5,6 +5,7 @@ import * as Fiber from "effect/Fiber";import {SERVICE_LAUNCHER_CONTEXT_ENV,+ SERVICE_LAUNCHER_PROTOCOL,type ServiceLauncherChildMessage,type ServiceLauncherParentMessage,} from "./serviceProtocol.ts";@@ -53,10 +54,11 @@ it.effect("waits for the launcher to durably commit the trial update ID", () =>id: "update-1",fromVersion: "1.0.0",targetVersion: "1.1.0",+ dbPath: "/tmp/state.sqlite",status: "pending" as const,};const host = new FakeLauncherProcess({- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,childVersion: "1.1.0",update: pending,});@@ -79,13 +81,14 @@ it.effect("waits for the launcher to durably commit the trial update ID", () =>it.effect("returns the launcher-generated ID only after update acceptance", () =>Effect.gen(function* () {const host = new FakeLauncherProcess({- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,childVersion: "1.0.0",});const client = yield* makeClient(host, "1.0.0");- const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), {- startImmediately: true,- });+ const requested = yield* Effect.forkChild(+ client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }),+ { startImmediately: true },+ );yield* Effect.yieldNow;host.emit({type: "update-accepted",@@ -97,11 +100,15 @@ it.effect("returns the launcher-generated ID only after update acceptance", () =it.effect("preserves a launcher rejection as a distinct error", () =>Effect.gen(function* () {- const host = new FakeLauncherProcess({ protocol: 1, childVersion: "1.0.0" });- const client = yield* makeClient(host, "1.0.0");- const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), {- startImmediately: true,+ const host = new FakeLauncherProcess({+ protocol: SERVICE_LAUNCHER_PROTOCOL,+ childVersion: "1.0.0",});+ const client = yield* makeClient(host, "1.0.0");+ const requested = yield* Effect.forkChild(+ client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }),+ { startImmediately: true },+ );yield* Effect.yieldNow;host.emit({ type: "update-rejected", reason: "requires local update" });expect(yield* Fiber.join(requested).pipe(Effect.flip)).toMatchObject({@@ -115,12 +122,13 @@ it.effect("preserves a launcher rejection as a distinct error", () =>it.effect("rejects contradictory trial context instead of leaving activation closed", () =>Effect.gen(function* () {const host = new FakeLauncherProcess({- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,childVersion: "1.1.0",update: {id: "update-1",fromVersion: "1.0.0",targetVersion: "1.2.0",+ dbPath: "/tmp/state.sqlite",status: "pending",},});diff --git a/apps/server/src/cloud/serviceLauncherClient.ts b/apps/server/src/cloud/serviceLauncherClient.tsindex 760642c29..970c72cee 100644--- a/apps/server/src/cloud/serviceLauncherClient.ts+++ b/apps/server/src/cloud/serviceLauncherClient.ts@@ -100,9 +100,9 @@ export class ServiceLauncherClient extends Context.Service<ServiceLauncherClient,{readonly managed: boolean;- readonly trial: boolean;readonly requestUpdate: (input: {readonly targetVersion: string;+ readonly dbPath: string;}) => Effect.Effect<string, ServiceLauncherClientError | ServiceLauncherRejectedError>;readonly prepareTrial: Effect.Effect<ServerSelfUpdateOutcome | undefined,@@ -137,8 +137,8 @@ const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup"export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_client.resolve_mode")(function* () {- const { context, managed } = yield* resolveStartup();- return { managed, trial: context?.update?.status === "pending" };+ const { managed } = yield* resolveStartup();+ return { managed };},);@@ -199,7 +199,7 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o}),);- const requestUpdate = (input: { readonly targetVersion: string }) =>+ const requestUpdate = (input: { readonly targetVersion: string; readonly dbPath: string }) =>exchange({ type: "request-update", ...input },(reply) => reply.type === "update-accepted" || reply.type === "update-rejected",@@ -233,14 +233,18 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (oif (reply.type !== "committed") {return Effect.die("service launcher returned an impossible prepared response");}- return Effect.succeed({ ...pending, status: "committed" as const });+ return Effect.succeed({+ id: pending.id,+ fromVersion: pending.fromVersion,+ targetVersion: pending.targetVersion,+ status: "committed" as const,+ });}),): Effect.succeed(outcome);return ServiceLauncherClient.of({managed,- trial: pending !== undefined,requestUpdate,prepareTrial,});diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.tsindex d2ce6db8d..2eb2e0201 100644--- a/apps/server/src/cloud/servicePreflight.test.ts+++ b/apps/server/src/cloud/servicePreflight.test.ts@@ -1,47 +1,26 @@-// @effect-diagnostics nodeBuiltinImport:off-import * as NodeServices from "@effect/platform-node/NodeServices";import { expect, it } from "@effect/vitest";-import * as Effect from "effect/Effect";-import * as FileSystem from "effect/FileSystem";-import * as Path from "effect/Path";-import * as NodeSqlite from "node:sqlite";-import { migrationManifest } from "../persistence/Migrations.ts";import { runServicePreflight } from "./servicePreflight.ts";+import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";-it.layer(NodeServices.layer)("service update preflight", (it) => {- it.effect("requires exact migration-manifest equality without mutating the database", () =>- Effect.gen(function* () {- const fs = yield* FileSystem.FileSystem;- const path = yield* Path.Path;- const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-preflight-test-" });- const databasePath = path.join(root, "state.sqlite");- const database = new NodeSqlite.DatabaseSync(databasePath);- database.exec("CREATE TABLE effect_sql_migrations (migration_id INTEGER, name TEXT)");- const insert = database.prepare(- "INSERT INTO effect_sql_migrations (migration_id, name) VALUES (?, ?)",- );- for (const [id, name] of migrationManifest) insert.run(id, name);- database.close();-- expect(runServicePreflight({ databasePath, launcherProtocol: 1, version: "1.2.3" })).toEqual({- status: "ready",- version: "1.2.3",- launcherProtocol: 1,- });+it("requires the database-snapshot launcher protocol", () => {+ expect(+ runServicePreflight({+ databasePath: "/missing/state.sqlite",+ launcherProtocol: SERVICE_LAUNCHER_PROTOCOL - 1,+ version: "1.2.3",+ }),+ ).toMatchObject({ status: "blocked", version: "1.2.3" });- const changed = new NodeSqlite.DatabaseSync(databasePath);- changed.exec("DELETE FROM effect_sql_migrations WHERE migration_id = 35");- changed.close();- const blocked = runServicePreflight({- databasePath,- launcherProtocol: 1,- version: "1.2.3",- });- expect(blocked.status).toBe("blocked");- if (blocked.status === "blocked") {- expect(blocked.reason).toContain("npx [email protected] service update");- }+ expect(+ runServicePreflight({+ databasePath: "/missing/state.sqlite",+ launcherProtocol: SERVICE_LAUNCHER_PROTOCOL,+ version: "1.2.3",}),- );+ ).toEqual({+ status: "ready",+ version: "1.2.3",+ launcherProtocol: SERVICE_LAUNCHER_PROTOCOL,+ });});diff --git a/apps/server/src/cloud/servicePreflight.ts b/apps/server/src/cloud/servicePreflight.tsindex 1843e1638..ee0f972ba 100644--- a/apps/server/src/cloud/servicePreflight.ts+++ b/apps/server/src/cloud/servicePreflight.ts@@ -1,7 +1,4 @@-import * as NodeSqlite from "node:sqlite";-import packageJson from "../../package.json" with { type: "json" };-import { migrationManifest } from "../persistence/Migrations.ts";import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts";export type ServicePreflightResult =@@ -16,20 +13,8 @@ export type ServicePreflightResult =readonly reason: string;};-const localUpdateReason = (version: string) =>- `This version includes a database update and cannot be installed remotely. Run \`npx t3@${version} service update\` on the server machine.`;--const isMigrationRow = (- value: unknown,-): value is { readonly migration_id: number; readonly name: string } =>- typeof value === "object" &&- value !== null &&- "migration_id" in value &&- typeof value.migration_id === "number" &&- "name" in value &&- typeof value.name === "string";-export function runServicePreflight(input: {+ /** Older servers always pass this flag when invoking a staged preflight. */readonly databasePath: string;readonly launcherProtocol: number;readonly version?: string;@@ -44,28 +29,6 @@ export function runServicePreflight(input: {};}- try {- const database = new NodeSqlite.DatabaseSync(input.databasePath, { readOnly: true });- try {- const rows: ReadonlyArray<unknown> = database- .prepare("SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id")- .all();- const exact =- rows.length === migrationManifest.length &&- rows.every((row, index) => {- const expected = migrationManifest[index];- return (- isMigrationRow(row) && row.migration_id === expected?.[0] && row.name === expected?.[1]- );- });- if (!exact) return { status: "blocked", version, reason: localUpdateReason(version) };- } finally {- database.close();- }- } catch {- return { status: "blocked", version, reason: localUpdateReason(version) };- }-return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL };}diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.tsindex 921bc1447..ebc5d15d5 100644--- a/apps/server/src/cloud/serviceProtocol.ts+++ b/apps/server/src/cloud/serviceProtocol.ts@@ -1,6 +1,7 @@import type { ServerSelfUpdateOutcome } from "@t3tools/contracts";-export const SERVICE_LAUNCHER_PROTOCOL = 1 as const;+/** Protocol 2 snapshots SQLite before trials so migrations can be rolled back safely. */+export const SERVICE_LAUNCHER_PROTOCOL = 2 as const;export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";export const SERVICE_STATE_FILE = "service-state.json";@@ -9,6 +10,7 @@ export interface PendingServiceUpdate {readonly id: string;readonly fromVersion: string;readonly targetVersion: string;+ readonly dbPath: string;readonly status: "pending";}@@ -31,6 +33,7 @@ export type ServiceLauncherChildMessage =| {readonly type: "request-update";readonly targetVersion: string;+ readonly dbPath: string;}| {readonly type: "prepared";@@ -78,7 +81,9 @@ export function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undefreturn undefined;}if (status === "pending") {- return { id, fromVersion, targetVersion, status };+ return typeof value.dbPath === "string" && value.dbPath.trim() !== ""+ ? { id, fromVersion, targetVersion, dbPath: value.dbPath, status }+ : undefined;}if ((status === "committed" || status === "rolled-back" || status === "failed") &&@@ -165,6 +170,16 @@ export function parseServiceState(value: string): ServiceState | undefined {}}+/** Detects an in-flight update across launcher protocol versions before replacing its state. */+export function serviceStateHasPendingUpdate(value: string): boolean {+ try {+ const parsed: unknown = JSON.parse(value);+ return isRecord(parsed) && isRecord(parsed.update) && parsed.update.status === "pending";+ } catch {+ return false;+ }+}+export function decodeServiceLauncherContext(value: string): ServiceLauncherContext | undefined {let parsed: unknown;try {@@ -202,8 +217,12 @@ export function decodeServiceLauncherChildMessage(value: unknown,): ServiceLauncherChildMessage | undefined {if (!isRecord(value)) return undefined;- if (value.type === "request-update" && typeof value.targetVersion === "string") {- return { type: value.type, targetVersion: value.targetVersion };+ if (+ value.type === "request-update" &&+ typeof value.targetVersion === "string" &&+ typeof value.dbPath === "string"+ ) {+ return { type: value.type, targetVersion: value.targetVersion, dbPath: value.dbPath };}return value.type === "prepared" && typeof value.updateId === "string"? { type: value.type, updateId: value.updateId }diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.tsindex 3cfdb5c5c..d1e002501 100644--- a/apps/server/src/persistence/Layers/Sqlite.ts+++ b/apps/server/src/persistence/Layers/Sqlite.ts@@ -7,7 +7,6 @@ import type { SqlError } from "effect/unstable/sql/SqlError";import { runMigrations } from "../Migrations.ts";import { ServerConfig } from "../../config.ts";-import * as ServiceLauncherClient from "../../cloud/serviceLauncherClient.ts";type RuntimeSqliteLayerConfig = {readonly filename: string;@@ -31,28 +30,24 @@ const makeRuntimeSqliteLayer = Effect.fn("makeRuntimeSqliteLayer")(function* (return clientModule.layer(config);}, Layer.unwrap);-const setup = (trial: boolean) =>- Layer.effectDiscard(- Effect.gen(function* () {- const sql = yield* SqlClient.SqlClient;- yield* sql`PRAGMA foreign_keys = ON;`;- if (!trial) {- yield* sql`PRAGMA journal_mode = WAL;`;- yield* runMigrations();- }- }),- );+const setup = Layer.effectDiscard(+ Effect.gen(function* () {+ const sql = yield* SqlClient.SqlClient;+ yield* sql`PRAGMA foreign_keys = ON;`;+ yield* sql`PRAGMA journal_mode = WAL;`;+ yield* runMigrations();+ }),+);export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(function* (dbPath: string,- options?: { readonly trial?: boolean },) {const fs = yield* FileSystem.FileSystem;const path = yield* Path.Path;yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });return Layer.provideMerge(- setup(options?.trial === true),+ setup,makeRuntimeSqliteLayer({filename: dbPath,spanAttributes: {@@ -64,14 +59,13 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(}, Layer.unwrap);export const SqlitePersistenceMemory = Layer.provideMerge(- setup(false),+ setup,makeRuntimeSqliteLayer({ filename: ":memory:" }),);export const layerConfig = Layer.unwrap(Effect.gen(function* () {const { dbPath } = yield* ServerConfig;- const launcher = yield* ServiceLauncherClient.resolveServiceLauncherMode();- return makeSqlitePersistenceLive(dbPath, { trial: launcher.trial });+ return makeSqlitePersistenceLive(dbPath);}),);diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.tsindex 21f3618d5..4562c6a6d 100644--- a/apps/server/src/serviceLauncher.test.ts+++ b/apps/server/src/serviceLauncher.test.ts@@ -9,6 +9,7 @@ import {compareExactServiceVersions,decodeServiceState,isExactServiceVersion,+ SERVICE_LAUNCHER_PROTOCOL,} from "./cloud/serviceProtocol.ts";it("accepts only exact semantic versions", () => {@@ -33,12 +34,13 @@ it("orders exact semantic versions without treating build metadata as precedenceit("rejects contradictory service state", () => {assert.isUndefined(decodeServiceState({- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,activeVersion: "0.0.31",update: {id: "update-1",fromVersion: "0.0.30",targetVersion: "0.0.32",+ dbPath: "/tmp/state.sqlite",status: "pending",},}),@@ -46,12 +48,26 @@ it("rejects contradictory service state", () => {assert.isUndefined(decodeServiceState({- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,+ activeVersion: "1.0.0",+ update: {+ id: "update-3",+ fromVersion: "1.0.0",+ targetVersion: "1.1.0",+ status: "pending",+ },+ }),+ );++ assert.isUndefined(+ decodeServiceState({+ protocol: SERVICE_LAUNCHER_PROTOCOL,activeVersion: "1.0.0",update: {id: "update-2",fromVersion: "1.0.0",targetVersion: "0.9.0",+ dbPath: "/tmp/state.sqlite",status: "pending",},}),@@ -66,7 +82,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-test-" });const statePath = path.join(root, "runtime", "service-state.json");const state = {- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,activeVersion: "0.0.31",} as const;@@ -88,7 +104,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n");yield* Effect.promise(() =>writeServiceState(statePath, {- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,activeVersion: "1.0.0",}),);@@ -106,6 +122,11 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {const path = yield* Path.Path;const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });const statePath = path.join(root, "runtime", "service-state.json");+ const databasePath = path.join(root, "userdata", "state.sqlite");+ yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });+ yield* fs.writeFileString(databasePath, "before trial");+ // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source.+ const encodedDatabasePath = JSON.stringify(databasePath);const childSource = `const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);if (context.update?.status === "pending") {@@ -114,7 +135,7 @@ if (context.update?.status === "pending") {if (message.type === "committed") process.exit(0);});} else if (context.update === undefined) {- process.send({ type: "request-update", targetVersion: "1.1.0" });+ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });setInterval(() => {}, 1_000);} else {process.exit(0);@@ -129,7 +150,7 @@ if (context.update?.status === "pending") {}yield* Effect.promise(() =>writeServiceState(statePath, {- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,activeVersion: "1.0.0",}),);@@ -154,12 +175,17 @@ if (context.update?.status === "pending") {const path = yield* Path.Path;const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" });const statePath = path.join(root, "runtime", "service-state.json");+ const databasePath = path.join(root, "userdata", "state.sqlite");+ yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });+ yield* fs.writeFileString(databasePath, "before trial");+ // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source.+ const encodedDatabasePath = JSON.stringify(databasePath);const childSource = `const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);if (context.update?.status === "pending") {process.send({ type: "prepared", updateId: "wrong-update" });} else if (context.update === undefined) {- process.send({ type: "request-update", targetVersion: "1.1.0" });+ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });setInterval(() => {}, 1_000);} else {process.exit(0);@@ -174,7 +200,7 @@ if (context.update?.status === "pending") {}yield* Effect.promise(() =>writeServiceState(statePath, {- protocol: 1,+ protocol: SERVICE_LAUNCHER_PROTOCOL,activeVersion: "1.0.0",}),);@@ -196,4 +222,65 @@ if (context.update?.status === "pending") {);}),);++ it.effect("restores the database when a migrating trial exits", () =>+ Effect.gen(function* () {+ const fs = yield* FileSystem.FileSystem;+ const path = yield* Path.Path;+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" });+ const statePath = path.join(root, "runtime", "service-state.json");+ const databasePath = path.join(root, "userdata", "state.sqlite");+ const original = "database before migration";+ yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });+ yield* fs.writeFileString(databasePath, original);+ // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source.+ const encodedDatabasePath = JSON.stringify(databasePath);+ const childSource = `+import { writeFileSync } from "node:fs";+const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);+if (context.update?.status === "pending") {+ writeFileSync(context.update.dbPath, "database after migration");+ writeFileSync(context.update.dbPath + "-wal", "trial wal");+ writeFileSync(context.update.dbPath + "-shm", "trial shm");+ process.exit(1);+} else if (context.update === undefined) {+ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });+ setInterval(() => {}, 1_000);+} else {+ process.exit(0);+}+`;+ for (const version of ["1.0.0", "1.1.0"]) {+ const versionDir = path.join(root, "runtime", "versions", version);+ const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs");+ yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true });+ yield* fs.writeFileString(entryPath, childSource);+ yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`);+ }+ yield* Effect.promise(() =>+ writeServiceState(statePath, {+ protocol: SERVICE_LAUNCHER_PROTOCOL,+ activeVersion: "1.0.0",+ }),+ );++ const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));+ yield* Effect.promise(() =>+ launcher.run().then(+ () => Promise.reject(new Error("launcher unexpectedly completed")),+ () => Promise.resolve(),+ ),+ );++ const state = yield* Effect.promise(() => readServiceState(statePath));+ assert.equal(state.activeVersion, "1.0.0");+ assert.equal(state.update?.status, "rolled-back");+ assert.equal(yield* fs.readFileString(databasePath), original);+ assert.isFalse(yield* fs.exists(`${databasePath}-wal`));+ assert.isFalse(yield* fs.exists(`${databasePath}-shm`));+ const updateId = state.update?.id;+ assert.isDefined(updateId);+ assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId)));+ }),+ );});diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.tsindex d7ca33578..211c0138b 100644--- a/apps/server/src/serviceLauncher.ts+++ b/apps/server/src/serviceLauncher.ts@@ -48,6 +48,121 @@ const runtimePaths = (baseDir: string, version: string) => {};};+/** SQLite persists across the main file plus its WAL and shared-memory sidecars. */+const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;+const RESTORE_MARKER = ".restore-pending";++const databaseBackupDir = (baseDir: string, updateId: string) =>+ NodePath.join(baseDir, "runtime", "db-backup", updateId);++const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) =>+ NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`);++async function pathExists(target: string): Promise<boolean> {+ try {+ await NodeFSP.access(target);+ return true;+ } catch (cause) {+ if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false;+ throw cause;+ }+}++async function syncFile(filePath: string): Promise<void> {+ const handle = await NodeFSP.open(filePath, "r");+ try {+ await handle.sync();+ } finally {+ await handle.close();+ }+}++async function syncDirectory(directory: string): Promise<void> {+ const handle = await NodeFSP.open(directory, "r");+ try {+ await handle.sync();+ } finally {+ await handle.close();+ }+}++/**+ * Snapshots the database once per update before the first trial. A completed+ * backup is never overwritten because a restarted launcher may be looking at+ * database writes from an earlier attempt by the same trial.+ */+async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {+ const backupDir = databaseBackupDir(baseDir, pending.id);+ if (await pathExists(backupDir)) return;++ const stagingDir = `${backupDir}.staging`;+ await NodeFSP.rm(stagingDir, { recursive: true, force: true });+ await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });+ try {+ for (const suffix of DB_FILE_SUFFIXES) {+ const source = `${pending.dbPath}${suffix}`;+ if (suffix !== "" && !(await pathExists(source))) continue;+ const destination = databaseBackupFile(stagingDir, suffix);+ await NodeFSP.copyFile(source, destination);+ await syncFile(destination);+ }+ await NodeFSP.rename(stagingDir, backupDir);+ await syncDirectory(NodePath.dirname(backupDir));+ } catch (cause) {+ await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);+ throw cause;+ }+}++const restoreMarkerPath = (baseDir: string, updateId: string) =>+ NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER);++const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) =>+ pathExists(restoreMarkerPath(baseDir, pending.id));++/** Mark rollback before changing live files so launcher recovery cannot boot a partial restore. */+async function markDatabaseRestorePending(backupDir: string): Promise<void> {+ const markerPath = NodePath.join(backupDir, RESTORE_MARKER);+ if (!(await pathExists(markerPath))) {+ const handle = await NodeFSP.open(markerPath, "wx", 0o600);+ try {+ await handle.sync();+ } finally {+ await handle.close();+ }+ await syncDirectory(backupDir);+ }+}++/** Restore is retryable after any process crash while the backup directory remains. */+async function restoreDatabaseBackup(+ baseDir: string,+ pending: PendingServiceUpdate,+): Promise<void> {+ const backupDir = databaseBackupDir(baseDir, pending.id);+ if (!(await pathExists(backupDir))) return;++ await markDatabaseRestorePending(backupDir);+ for (const suffix of DB_FILE_SUFFIXES) {+ const target = `${pending.dbPath}${suffix}`;+ const source = databaseBackupFile(backupDir, suffix);+ if (await pathExists(source)) {+ await NodeFSP.copyFile(source, target);+ await syncFile(target);+ } else {+ await NodeFSP.rm(target, { force: true });+ }+ }+ await syncDirectory(NodePath.dirname(pending.dbPath));+}++async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {+ const backupDir = databaseBackupDir(baseDir, updateId);+ if (!(await pathExists(backupDir))) return;+ await NodeFSP.rm(backupDir, { recursive: true, force: true });+ await syncDirectory(NodePath.dirname(backupDir));+}+export async function readServiceState(filePath: string): Promise<ServiceState> {const contents = await NodeFSP.readFile(filePath, "utf8");const state = parseServiceState(contents);@@ -217,9 +332,16 @@ export class Launcher {async #recover(): Promise<void> {const update = this.#state.update;if (update?.status !== "pending") {+ if (update !== undefined) {+ await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);+ }await this.#startChild(this.#state.activeVersion, "active", update);return;}+ if (await databaseRestorePending(this.#baseDir, update)) {+ await this.#returnToPrevious(update, "failed", "rollback-interrupted");+ return;+ }if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {await this.#returnToPrevious(update, "failed", "target-runtime-missing");return;@@ -228,6 +350,13 @@ export class Launcher {}async #startTrial(pending: PendingServiceUpdate): Promise<void> {+ // The previous child is dead here, so all three SQLite files are quiescent.+ try {+ await backupDatabaseOnce(this.#baseDir, pending);+ } catch {+ await this.#returnToPrevious(pending, "failed", "db-backup-failed");+ return;+ }try {await this.#startChild(pending.targetVersion, "trial", pending);} catch {@@ -322,6 +451,10 @@ export class Launcher {await reject("Remote updates must select a newer server version.");return;}+ if (!NodePath.isAbsolute(message.dbPath)) {+ await reject("The requested database path is not absolute.");+ return;+ }if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {await reject("The requested target runtime is missing or incomplete.");return;@@ -331,6 +464,7 @@ export class Launcher {id: NodeCrypto.randomUUID(),fromVersion: child.version,targetVersion: message.targetVersion,+ dbPath: message.dbPath,status: "pending",};const next: ServiceState = { ...this.#state, update: pending };@@ -375,6 +509,7 @@ export class Launcher {await writeServiceState(this.#statePath, next);this.#state = next;child.role = "active";+ await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);await sendMessage(child.process, { type: "committed", updateId: committed.id });}@@ -423,6 +558,11 @@ export class Launcher {reason: string,child?: ManagedChild,): Promise<void> {+ if (child !== undefined) {+ this.#child = null;+ await terminateChild(child.process);+ }+ await restoreDatabaseBackup(this.#baseDir, pending);const outcome = terminalUpdate({ pending, status, reason });const next: ServiceState = {...this.#state,@@ -431,10 +571,7 @@ export class Launcher {};await writeServiceState(this.#statePath, next);this.#state = next;- if (child !== undefined) {- this.#child = null;- await terminateChild(child.process);- }+ await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);await this.#startChild(next.activeVersion, "active", outcome);}}diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.mdindex f0a0034c3..c2ba77f6f 100644--- a/docs/internals/server-updates.md+++ b/docs/internals/server-updates.md@@ -29,39 +29,49 @@ Every write uses same-directory replacement plus file and directory fsync.## Remote Update1. The active server installs `t3@<target>` into a unique staging directory.-2. The target runs `__service-preflight` against the active database in read-only mode.+2. The target runs `__service-preflight` and verifies that the stable launcher supports its update+ protocol.3. The staging directory is renamed to its immutable version path only after preflight succeeds.4. The active child sends `request-update`. The launcher validates the child and target, writespending state, generates the update ID, then replies `update-accepted`.5. After a short response-flush grace period, the launcher stops the active child.-6. The launcher starts the target as a trial and gives it the pending update over IPC.-7. The trial acquires dependencies, binds HTTP, starts every long-running root fiber, and verifies- that each root is parked at the activation gate. It does not run migrations.-8. The trial sends `prepared`. The launcher durably commits B before replying `committed`.-9. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready- with the terminal update outcome.+6. With SQLite quiescent, the launcher snapshots the database, WAL, and shared-memory files.+7. The launcher starts the target as a trial and gives it the pending update over IPC.+8. The trial runs migrations, acquires dependencies, binds HTTP, starts every long-running root+ fiber, and verifies that each root is parked at the activation gate.+9. The trial sends `prepared`. The launcher durably commits B, deletes the snapshot, then replies+ `committed`.+10. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready+ with the terminal update outcome.Post-commit startup does not call service `start`, `initialize`, `connect`, `load`, or `acquire`operations. It only opens prepared gates and publishes prepared lifecycle state.The launcher serializes child exits, IPC messages, and timers. A trial must report prepared within-120 seconds. If the trial exits or times out before prepared, the launcher records rollback and-starts A. After commit, B is active and normal systemd restart policy applies.+120 seconds. If the trial exits or times out before prepared, the launcher stops it, restores the+snapshot, records rollback, and starts A. A durable restore marker makes an interrupted restore+resume before either version can boot. After commit, B is active and normal systemd restart policy+applies.-## Migration Boundary+## Database Rollback-Remote update preflight requires exact equality between the database's applied `(id, name)` rows-and the target release's migration manifest. A target with a missing, additional, renamed, or-unknown migration is blocked. Remote updates never migrate or downgrade a database.+The launcher snapshots `state.sqlite`, `state.sqlite-wal`, and `state.sqlite-shm` after the old+server stops and before the trial starts. This makes trial migrations and writes reversible without+requiring down migrations. The snapshot is retained across launcher restarts and is removed only+after commit or after both restore and the terminal rollback state are durable.-This deliberately means any release containing a migration requires a local service update:+The protocol version is part of the safety boundary. A target that requires database snapshots is+blocked when the installed launcher is too old. Upgrade the launcher once with:```shnpx t3@<version> service update```The local command stops the unit, selects the new launcher and exact runtime, then restarts the-service. Its normal server startup may run migrations.+service. Later releases, including releases with migrations, can use the remote trial path.++Snapshots briefly require enough free disk for another copy of the SQLite files. Attachments and+other files under the state directory are outside this rollback boundary.## Client Correlationdiff --git a/docs/user/background-service.md b/docs/user/background-service.mdindex 0ab5b2013..299e3b641 100644--- a/docs/user/background-service.md+++ b/docs/user/background-service.md@@ -30,10 +30,13 @@ npx t3@latest service uninstall```Updating restarts T3 Code briefly. Let active agent work and terminal commands finish first.+If a remote update is already in progress, wait for it to finish before retrying a local update.The systemd unit runs a small stable launcher. Exact T3 Code versions are installed separately, so-a failed remote candidate can return to the previous version without rewriting the unit. Releases-that change the database must be installed with the local `service update` command above.+a failed remote candidate can return to the previous version without rewriting the unit. The+launcher snapshots the database before a remote candidate starts, so database updates roll back+with the server version. An older launcher may require one local `service update` before this is+available.## Using It with T3 Connectdiff --git a/docs/user/updating.md b/docs/user/updating.mdindex a0cc0e5d1..ea2ff644c 100644--- a/docs/user/updating.md+++ b/docs/user/updating.md@@ -31,9 +31,9 @@ The update does not remove saved threads, settings, or project files.The available action depends on how that server was started. T3 Code does not update connectedservers silently in the background.-If the requested version includes a database update, remote installation stops before restart and-asks you to run the exact `npx t3@<version> service update` command on the server machine. This is-an intentional rollback-safety boundary.+An older background-service launcher may ask you to run the exact+`npx t3@<version> service update` command on the server machine. That one local update installs the+rollback support needed for later remote updates, including versions that change the database.After selecting **Update server**, the warning becomes a three-step progress rail:**Download**, **Install**, and **Resume**. The same progress appears in the conversation and in