Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

fix(server): keep the managed tunnel across remote update restarts

Theo Browne808d6853555c
diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts
index c779c13aa..897912bc6 100644
--- a/apps/server/src/cloud/http.test.ts
+++ b/apps/server/src/cloud/http.test.ts
@@ -1,7 +1,9 @@
import * as NodeServices from "@effect/platform-node/NodeServices";
import { describe, expect, it } from "@effect/vitest";
import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
import * as Option from "effect/Option";
+import * as Path from "effect/Path";
import * as PlatformError from "effect/PlatformError";
import * as Tracer from "effect/Tracer";
import {
@@ -15,6 +17,7 @@ import { EnvironmentId } from "@t3tools/contracts";
import { RelayClientTracer } from "@t3tools/shared/relayTracing";
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import * as ServerConfigModule from "../config.ts";
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts";
import * as CliTokenManager from "./CliTokenManager.ts";
@@ -256,6 +259,21 @@ describe("releaseManagedTunnelOnShutdown", () => {
readonly respond?: () => Response;
}
+ // Writes the launcher's durable state file into this test's baseDir; the
+ // release reads it to detect an in-flight update handoff.
+ const writeServiceState = (update: Record<string, unknown>) =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const config = yield* ServerConfigModule.ServerConfig;
+ const statePath = path.join(config.baseDir, "runtime", "service-state.json");
+ yield* fs.makeDirectory(path.dirname(statePath), { recursive: true });
+ yield* fs.writeFileString(
+ statePath,
+ JSON.stringify({ version: 1, activeVersion: "0.0.30", update }),
+ );
+ });
+
const provideReleaseHarness =
(harness: ReleaseHarness) =>
<A, E, R>(effect: Effect.Effect<A, E, R>) =>
@@ -306,6 +324,11 @@ describe("releaseManagedTunnelOnShutdown", () => {
}),
),
),
+ // The release consults the launcher state file under the configured
+ // baseDir, so every harness run gets a scoped temp baseDir.
+ Effect.provide(ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" })),
+ Effect.provide(NodeServices.layer),
+ Effect.scoped,
);
// The persisted state of a CLI-managed link whose tunnel is releasable.
@@ -390,6 +413,54 @@ describe("releaseManagedTunnelOnShutdown", () => {
}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
});
+ it.effect("keeps the tunnel when shutdown hands off to a pending update", () => {
+ const { store, values } = makeMemorySecretStore(managedLinkSecrets);
+ const applyConfigCalls: Array<unknown> = [];
+ const requests: Array<HttpClientRequest.HttpClientRequest> = [];
+
+ return Effect.gen(function* () {
+ yield* writeServiceState({
+ id: "update-1",
+ fromVersion: "0.0.30",
+ targetVersion: "0.0.31",
+ dbPath: "/tmp/state.sqlite",
+ status: "pending",
+ });
+
+ const released = yield* releaseManagedTunnelOnShutdown();
+
+ // The launcher restarts a server immediately, so the tunnel is not
+ // orphaned; keeping it avoids the hostname route re-propagation that
+ // dominates update downtime. The stored config must survive so the
+ // next boot respawns the connector against the same tunnel.
+ expect(released).toBe(false);
+ expect(applyConfigCalls).toEqual([]);
+ expect(requests).toEqual([]);
+ expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true);
+ }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
+ });
+
+ it.effect("still releases when the recorded update already settled", () => {
+ const { store, values } = makeMemorySecretStore(managedLinkSecrets);
+ const applyConfigCalls: Array<unknown> = [];
+ const requests: Array<HttpClientRequest.HttpClientRequest> = [];
+
+ return Effect.gen(function* () {
+ yield* writeServiceState({
+ id: "update-1",
+ fromVersion: "0.0.30",
+ targetVersion: "0.0.31",
+ status: "committed",
+ });
+
+ const released = yield* releaseManagedTunnelOnShutdown();
+
+ expect(released).toBe(true);
+ expect(requests).toHaveLength(1);
+ expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
+ }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
+ });
+
it.effect("keeps a runtime config that a fast restart replaced mid-release", () => {
const { store, values } = makeMemorySecretStore(managedLinkSecrets);
const applyConfigCalls: Array<unknown> = [];
diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts
index eaf31a37f..e51ec6f3c 100644
--- a/apps/server/src/cloud/http.ts
+++ b/apps/server/src/cloud/http.ts
@@ -46,7 +46,9 @@ import * as DateTime from "effect/DateTime";
import * as Crypto from "effect/Crypto";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
import * as Option from "effect/Option";
+import * as Path from "effect/Path";
import * as Schema from "effect/Schema";
import * as HttpEffect from "effect/unstable/http/HttpEffect";
import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
@@ -56,8 +58,10 @@ import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
import { requireEnvironmentScope } from "../auth/http.ts";
+import * as ServerConfig from "../config.ts";
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
+import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts";
import {
CLOUD_ENDPOINT_RUNTIME_CONFIG,
CLOUD_LINKED_USER_ID,
@@ -627,6 +631,20 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD
},
);
+// The launcher records an in-flight update in its durable state file. Read
+// straight from disk: the launcher owns that file, and this runs while the
+// server is tearing down. Unreadable or unparseable state means no pending
+// update — the release then proceeds as before.
+const pendingServiceUpdateExists = Effect.gen(function* () {
+ const config = yield* ServerConfig.ServerConfig;
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const stateText = yield* fs
+ .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE))
+ .pipe(Effect.option);
+ return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
+});
+
// Cloudflare bills per provisioned tunnel, so an environment that goes offline
// must not leave its tunnel behind. Releasing deletes only the tunnel — the
// relay keeps the link and its hostname reservation, and the next startup's
@@ -649,6 +667,19 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
return false;
}
+ // A shutdown that hands off to a pending remote update is not the
+ // environment going offline: the launcher immediately brings a server back
+ // (the new version, or the old one after a rollback). Deleting the tunnel
+ // here forces that server to provision a replacement UUID, and the public
+ // hostname's route to the new tunnel takes 1-2 minutes to propagate — the
+ // dominant cost of an update restart. Keep the tunnel instead: the next
+ // boot respawns the connector from the stored config and is reachable as
+ // soon as it connects, and the reconcile confirms the still-live tunnel
+ // without replacing it.
+ if (yield* pendingServiceUpdateExists) {
+ yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
+ return false;
+ }
const token = yield* dependencies.cliTokenManager.getExisting;
if (Option.isNone(token)) {
return false;