Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

fix(server): release the tunnel when an explicit stop interrupts an update

Theo Brownef9e8236899f2
diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts
index 8aa800a33..d3cd9c8a4 100644
--- a/apps/server/src/cloud/http.test.ts
+++ b/apps/server/src/cloud/http.test.ts
@@ -1,6 +1,7 @@
import * as NodeServices from "@effect/platform-node/NodeServices";
import { describe, expect, it } from "@effect/vitest";
import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
@@ -22,6 +23,7 @@ import { writeServiceState } from "../serviceLauncher.ts";
import {
SERVICE_LAUNCHER_PROTOCOL,
SERVICE_STATE_FILE,
+ SERVICE_STOP_MARKER_FILE,
type ServiceUpdateRecord,
} from "./serviceProtocol.ts";
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
@@ -451,6 +453,35 @@ describe("releaseManagedTunnelOnShutdown", () => {
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
});
+ it.effect("still releases a pending update when the launcher is stopping", () => {
+ // `t3 service uninstall` or `systemctl stop` during the pending window:
+ // the launcher writes its stop marker before signalling the child, so no
+ // replacement server is coming and the tunnel must not be kept.
+ const { store, values } = makeMemorySecretStore(managedLinkSecrets);
+ const applyConfigCalls: Array<unknown> = [];
+ const requests: Array<HttpClientRequest.HttpClientRequest> = [];
+
+ return Effect.gen(function* () {
+ yield* writeLauncherState({
+ id: "update-1",
+ fromVersion: "0.0.30",
+ targetVersion: "0.0.31",
+ dbPath: "/tmp/state.sqlite",
+ status: "pending",
+ });
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const config = yield* ServerConfigModule.ServerConfig;
+ yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), "");
+
+ const released = yield* releaseManagedTunnelOnShutdown();
+
+ expect(released).toBe(true);
+ expect(requests).toHaveLength(1);
+ expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
+ }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
+ });
+
it.effect("still releases when the recorded update already settled", () => {
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
const applyConfigCalls: Array<unknown> = [];
diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts
index e51ec6f3c..3dc12f41c 100644
--- a/apps/server/src/cloud/http.ts
+++ b/apps/server/src/cloud/http.ts
@@ -61,7 +61,11 @@ import { requireEnvironmentScope } from "../auth/http.ts";
import * as ServerConfig from "../config.ts";
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
-import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts";
+import {
+ SERVICE_STATE_FILE,
+ SERVICE_STOP_MARKER_FILE,
+ serviceStateHasPendingUpdate,
+} from "./serviceProtocol.ts";
import {
CLOUD_ENDPOINT_RUNTIME_CONFIG,
CLOUD_LINKED_USER_ID,
@@ -635,14 +639,27 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD
// straight from disk: the launcher owns that file, and this runs while the
// server is tearing down. Unreadable or unparseable state means no pending
// update — the release then proceeds as before.
-const pendingServiceUpdateExists = Effect.gen(function* () {
+//
+// A pending update alone is not proof a replacement server is coming: an
+// explicit launcher stop (`t3 service uninstall`, `systemctl stop`) during
+// the pending window also tears this server down, with nothing starting
+// afterwards. The launcher marks that case with a stop-marker file just
+// before it signals the child, so pending + no marker is the update handoff.
+const pendingUpdateHandoffExists = Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
+ const runtimeDir = path.join(config.baseDir, "runtime");
const stateText = yield* fs
- .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE))
+ .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
.pipe(Effect.option);
- return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
+ if (Option.isNone(stateText) || !serviceStateHasPendingUpdate(stateText.value)) {
+ return false;
+ }
+ const stopping = yield* fs
+ .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
+ .pipe(Effect.orElseSucceed(() => false));
+ return !stopping;
});
// Cloudflare bills per provisioned tunnel, so an environment that goes offline
@@ -676,7 +693,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
// boot respawns the connector from the stored config and is reachable as
// soon as it connects, and the reconcile confirms the still-live tunnel
// without replacing it.
- if (yield* pendingServiceUpdateExists) {
+ if (yield* pendingUpdateHandoffExists) {
yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
return false;
}
diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts
index ebc5d15d5..0faf88948 100644
--- a/apps/server/src/cloud/serviceProtocol.ts
+++ b/apps/server/src/cloud/serviceProtocol.ts
@@ -5,6 +5,10 @@ export const SERVICE_LAUNCHER_PROTOCOL = 2 as const;
export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";
export const SERVICE_STATE_FILE = "service-state.json";
+/** Written by the launcher just before an explicit stop kills its child, so
+ the child can tell "the service is going away" from "the launcher is about
+ to start my replacement" while a pending update is recorded. */
+export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
export interface PendingServiceUpdate {
readonly id: string;
diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts
index 4562c6a6d..90c76b85b 100644
--- a/apps/server/src/serviceLauncher.test.ts
+++ b/apps/server/src/serviceLauncher.test.ts
@@ -1,3 +1,6 @@
+// @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock.
+import * as NodeFS from "node:fs";
+
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
import * as Effect from "effect/Effect";
@@ -10,6 +13,7 @@ import {
decodeServiceState,
isExactServiceVersion,
SERVICE_LAUNCHER_PROTOCOL,
+ SERVICE_STOP_MARKER_FILE,
} from "./cloud/serviceProtocol.ts";
it("accepts only exact semantic versions", () => {
@@ -113,6 +117,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
const running = launcher.run();
yield* Effect.promise(() => launcher.stop("SIGTERM"));
yield* Effect.promise(() => running);
+ // An explicit stop leaves the marker that tells a child shutting down
+ // mid-update that no replacement server is coming.
+ assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE)));
+
+ // The next launcher start clears the stale marker so a later update
+ // handoff keeps its tunnel again. Recovery runs before any stop
+ // transition, so polling until the marker disappears observes it; the
+ // poll uses real timers because it.effect runs under the test clock.
+ const restarted = new Launcher(
+ root,
+ yield* Effect.promise(() => readServiceState(statePath)),
+ );
+ const runningAgain = restarted.run();
+ const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE);
+ yield* Effect.promise(async () => {
+ while (NodeFS.existsSync(markerPath)) {
+ // @effect-diagnostics-next-line globalTimers:off - real delay under the test clock.
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ }
+ });
+ yield* Effect.promise(() => restarted.stop("SIGTERM"));
+ yield* Effect.promise(() => runningAgain);
}),
);
diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts
index 211c0138b..b704dc4c0 100644
--- a/apps/server/src/serviceLauncher.ts
+++ b/apps/server/src/serviceLauncher.ts
@@ -24,6 +24,7 @@ import {
SERVICE_LAUNCHER_CONTEXT_ENV,
SERVICE_LAUNCHER_PROTOCOL,
SERVICE_STATE_FILE,
+ SERVICE_STOP_MARKER_FILE,
} from "./cloud/serviceProtocol.ts";
const HANDOFF_DELAY_MS = 2_000;
@@ -257,6 +258,9 @@ async function terminateChild(
}
}
+const stopMarkerPath = (baseDir: string) =>
+ NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
+
export class Launcher {
readonly #baseDir: string;
readonly #statePath: string;
@@ -315,6 +319,15 @@ export class Launcher {
this.#stopping = true;
this.#clearTimer();
this.#enqueue(async () => {
+ // An explicit stop is the service going away, not an update handoff.
+ // The marker lets the child's shutdown release the managed tunnel even
+ // while an update is recorded as pending; the next launcher start
+ // clears it. Written before the child is signalled so the child sees
+ // it during teardown. Best-effort: if the write fails the child errs
+ // toward keeping the tunnel, and the next link or unlink reconciles it.
+ await NodeFSP.writeFile(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }).catch(
+ () => undefined,
+ );
const child = this.#child?.process;
this.#child = null;
if (child !== undefined) await terminateChild(child, signal);
@@ -330,6 +343,10 @@ export class Launcher {
}
async #recover(): Promise<void> {
+ // A fresh launcher means servers are running again: any stop marker from
+ // a previous explicit stop is stale and must not make a future update
+ // handoff release its tunnel.
+ await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
const update = this.#state.update;
if (update?.status !== "pending") {
if (update !== undefined) {