TypeScript
fix(server): restart T3 Connect's tunnel client when Cloudflare drops it (#24946)
shivamc192c439c15f
diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.tsindex 8213288dee9..3e34bf0be28 100644--- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts+++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts@@ -206,6 +206,7 @@ describe("releaseManagedTunnelOnShutdown", () => {}),recoveryRequests: Stream.empty,requestRecovery: () => Effect.void,+ restartConnector: () => Effect.succeed(false),tunnelConnected: Stream.empty,inspectConnector: Effect.succeed(null),readinessChanged: Stream.empty,diff --git a/apps/server/src/cloud/ConnectReachability.test.ts b/apps/server/src/cloud/ConnectReachability.test.tsindex b6a73ab5dcd..b279e331f3d 100644--- a/apps/server/src/cloud/ConnectReachability.test.ts+++ b/apps/server/src/cloud/ConnectReachability.test.ts@@ -1,14 +1,32 @@// @effect-diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors.import * as NodeFS from "node:fs";+import * as NodeNet from "node:net";import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";import { describe, expect, it } from "@effect/vitest";+import * as Clock from "effect/Clock";+import * as Deferred from "effect/Deferred";+import * as Duration from "effect/Duration";+import * as Effect from "effect/Effect";+import * as Fiber from "effect/Fiber";+import * as Layer from "effect/Layer";+import * as Queue from "effect/Queue";+import * as Stream from "effect/Stream";+import * as TestClock from "effect/testing/TestClock";+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";+import { ChildProcessSpawner } from "effect/process";+import { vi } from "vite-plus/test";+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";+import * as ConnectReachability from "./ConnectReachability.ts";import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";+import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";++vi.mock("node:net", { spy: true });const checkedAt = "2026-10-19T00:00:00.000Z";-const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" };-const notConnected = { readyConnections: 0, hostname: null, protocol: null };+const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };+const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({managedTunnelConfigured: true,connector: connected,@@ -22,6 +40,8 @@ describe("classifyReachability", () => {it.each([[{ kind: "status", status: 502 }, "origin_unreachable"],[{ kind: "status", status: 530 }, "no_connector"],+ [{ kind: "status", status: 530, cloudflareError: "1033" }, "no_connector"],+ [{ kind: "status", status: 530, cloudflareError: "1016" }, "unexpected_response"],[{ kind: "network_error" }, "timeout"],] as const)("maps endpoint result %o to %s", (endpoint, cause) => {expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject(@@ -40,7 +60,12 @@ describe("classifyReachability", () => {expect(classifyReachability(facts({- connector: { readyConnections: null, hostname: "env.example.test", protocol: null },+ connector: {+ pid: 1,+ readyConnections: null,+ hostname: "env.example.test",+ protocol: null,+ },endpoint: { kind: "status", status: 502 },edgeReachable: false,}),@@ -51,7 +76,12 @@ describe("classifyReachability", () => {expect(classifyReachability(facts({- connector: { readyConnections: null, hostname: "env.example.test", protocol: null },+ connector: {+ pid: 1,+ readyConnections: null,+ hostname: "env.example.test",+ protocol: null,+ },endpoint: { kind: "dns_failed" },edgeReachable: false,}),@@ -74,6 +104,280 @@ describe("classifyReachability", () => {});});+interface ServedResponse {+ readonly status: number;+ readonly body: string | ReadableStream<Uint8Array> | { readonly cloudflareError: number };+}++// What Cloudflare serves for its 530 page: HTML unless the client accepts JSON.+const cloudflareErrorPage = (request: HttpClientRequest.HttpClientRequest, code: number) =>+ request.headers.accept === "application/json"+ ? JSON.stringify({ status: 530, error_code: code, error_name: "tunnel_error" })+ : `<!doctype html><h1><span>Error</span> <span>${code}</span></h1>`;++const reachabilityLayer = (+ respond: Effect.Effect<ServedResponse>,+ restartConnector: (pid: number, stillDead: Effect.Effect<boolean>) => Effect.Effect<boolean>,+) =>+ ConnectReachability.layer.pipe(+ Layer.provide(+ Layer.mergeAll(+ Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({+ inspectConnector: Effect.succeed(connected),+ readinessChanged: Stream.never,+ restartConnector,+ }),+ Layer.mock(ServerSecretStore.ServerSecretStore)({+ get: () => Effect.succeedSome(new Uint8Array()),+ }),+ Layer.succeed(+ HttpClient.HttpClient,+ HttpClient.make((request) =>+ respond.pipe(+ Effect.map(({ status, body }) =>+ HttpClientResponse.fromWeb(+ request,+ new Response(+ typeof body === "object" && "cloudflareError" in body+ ? cloudflareErrorPage(request, body.cloudflareError)+ : body,+ { status },+ ),+ ),+ ),+ ),+ ),+ ),+ Layer.succeed(+ ChildProcessSpawner.ChildProcessSpawner,+ ChildProcessSpawner.make(() => Effect.die("no tailscale")),+ ),+ ),+ ),+ );++const mockEdgeConnect = () =>+ vi.spyOn(NodeNet, "connect").mockImplementation(() => {+ const socket = new NodeNet.Socket();+ queueMicrotask(() => socket.emit("connect"));+ return socket;+ });++describe("ConnectReachability stale connector recovery", () => {+ it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>+ Effect.gen(function* () {+ using _connect = mockEdgeConnect();+ const restarts = yield* Queue.unbounded<number>();+ // Cloudflare answers 530 while cloudflared still reports four ready connections.+ let status = 530;+ let body = { cloudflareError: 1033 };+ const layer = reachabilityLayer(+ Effect.sync(() => ({ status, body })),+ (pid, stillDead) =>+ stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),+ );+ // The scheduled checks run in the layer's scope, so the assertions run inside it.+ yield* Effect.gen(function* () {+ const reachability = yield* ConnectReachability.ConnectReachability;++ // The first sighting schedules its own confirming check two minutes later.+ expect(yield* reachability.check).toMatchObject({ cause: "no_connector" });+ yield* TestClock.adjust(Duration.minutes(1));+ expect(yield* Queue.size(restarts)).toBe(0);+ yield* TestClock.adjust(Duration.minutes(1));+ expect(yield* Queue.take(restarts)).toBe(connected.pid);++ // A restart that did not help doubles the wait.+ yield* reachability.check;+ yield* TestClock.adjust(Duration.minutes(2));+ expect(yield* Queue.size(restarts)).toBe(0);+ yield* TestClock.adjust(Duration.minutes(2));+ expect(yield* Queue.take(restarts)).toBe(connected.pid);++ // Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone.+ status = 200;+ expect(yield* reachability.check).toMatchObject({ status: "reachable" });+ status = 530;+ body = { cloudflareError: 1016 };+ expect(yield* reachability.check).toMatchObject({ cause: "unexpected_response" });+ yield* TestClock.adjust(Duration.minutes(2));+ expect(yield* Queue.size(restarts)).toBe(0);++ body = { cloudflareError: 1033 };+ yield* reachability.check;+ yield* TestClock.adjust(Duration.minutes(2));+ expect(yield* Queue.take(restarts)).toBe(connected.pid);+ }).pipe(Effect.provide(layer));+ }),+ );++ it.effect("starts a new streak after a healthy check that a newer check overtook", () =>+ Effect.gen(function* () {+ using _connect = mockEdgeConnect();+ const restartTimes = yield* Queue.unbounded<number>();+ const healthyRequested = yield* Deferred.make<void>();+ const healthyAnswer = yield* Deferred.make<void>();+ let status = 530;+ const layer = reachabilityLayer(+ Effect.gen(function* () {+ const served = status;+ if (served === 200) {+ yield* Deferred.succeed(healthyRequested, undefined);+ yield* Deferred.await(healthyAnswer);+ }+ return { status: served, body: { cloudflareError: 1033 } };+ }),+ (_pid, stillDead) =>+ stillDead.pipe(+ Effect.tap((dead) =>+ dead+ ? Clock.currentTimeMillis.pipe(+ Effect.flatMap((now) => Queue.offer(restartTimes, now)),+ )+ : Effect.void,+ ),+ ),+ );+ yield* Effect.gen(function* () {+ const reachability = yield* ConnectReachability.ConnectReachability;+ const startedAt = yield* Clock.currentTimeMillis;+ yield* reachability.check;+ yield* TestClock.adjust(Duration.minutes(1));++ // The healthy answer arrives after a newer check already reported the tunnel dead.+ status = 200;+ const healthy = yield* Effect.forkChild(reachability.check);+ yield* Deferred.await(healthyRequested);+ status = 530;+ yield* reachability.check;+ yield* Deferred.succeed(healthyAnswer, undefined);+ expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });++ // The grace period starts again at the next dead check, a minute after the first one.+ yield* reachability.check;+ yield* TestClock.adjust(Duration.minutes(2));+ expect((yield* Queue.take(restartTimes)) - startedAt).toBe(+ Duration.toMillis(Duration.minutes(3)),+ );+ }).pipe(Effect.provide(layer));+ }),+ );++ it.effect("rejects a restart when an overtaken healthy check finishes while it waits", () =>+ Effect.gen(function* () {+ using _connect = mockEdgeConnect();+ const healthyRequested = yield* Deferred.make<void>();+ const healthyAnswer = yield* Deferred.make<void>();+ const restartRequested = yield* Deferred.make<void>();+ const releaseRestart = yield* Deferred.make<void>();+ const guardChecks = yield* Queue.unbounded<boolean>();+ let status = 530;+ const layer = reachabilityLayer(+ Effect.gen(function* () {+ const served = status;+ if (served === 200) {+ yield* Deferred.succeed(healthyRequested, undefined);+ yield* Deferred.await(healthyAnswer);+ }+ return { status: served, body: { cloudflareError: 1033 } };+ }),+ (_pid, stillDead) =>+ Effect.gen(function* () {+ yield* Deferred.succeed(restartRequested, undefined);+ yield* Deferred.await(releaseRestart);+ const accepted = yield* stillDead;+ yield* Queue.offer(guardChecks, accepted);+ return accepted;+ }),+ );+ yield* Effect.gen(function* () {+ const reachability = yield* ConnectReachability.ConnectReachability;+ yield* reachability.check;+ yield* TestClock.adjust(Duration.seconds(119));++ status = 200;+ const healthy = yield* Effect.forkChild(reachability.check);+ yield* Deferred.await(healthyRequested);+ status = 530;+ yield* TestClock.adjust(Duration.seconds(1));+ yield* Deferred.await(restartRequested);++ yield* Deferred.succeed(healthyAnswer, undefined);+ expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });+ yield* Deferred.succeed(releaseRestart, undefined);+ expect(yield* Queue.take(guardChecks)).toBe(false);++ yield* reachability.check;+ yield* TestClock.adjust(Duration.minutes(1));+ expect(yield* Queue.size(guardChecks)).toBe(0);+ yield* TestClock.adjust(Duration.minutes(1));+ expect(yield* Queue.take(guardChecks)).toBe(true);+ }).pipe(Effect.provide(layer));+ }),+ );++ it.effect("drops the confirming check of a streak that already ended", () =>+ Effect.gen(function* () {+ using _connect = mockEdgeConnect();+ const restarts = yield* Queue.unbounded<number>();+ let status = 530;+ let requests = 0;+ const layer = reachabilityLayer(+ Effect.sync(() => {+ requests += 1;+ return { status, body: { cloudflareError: 1033 } };+ }),+ (pid, stillDead) =>+ stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),+ );+ yield* Effect.gen(function* () {+ const reachability = yield* ConnectReachability.ConnectReachability;+ yield* reachability.check;+ yield* TestClock.adjust(Duration.minutes(1));+ status = 200;+ yield* reachability.check;+ status = 530;+ yield* reachability.check;++ const before = requests;+ yield* TestClock.adjust(Duration.minutes(1));+ expect(requests).toBe(before);+ yield* TestClock.adjust(Duration.minutes(1));+ expect(yield* Queue.take(restarts)).toBe(connected.pid);+ }).pipe(Effect.provide(layer));+ }),+ );++ it.effect("keeps a 530 whose error body never finishes", () =>+ Effect.gen(function* () {+ using _connect = mockEdgeConnect();+ let markBodyRead = () => {};+ const bodyRead = new Promise<void>((resolve) => {+ markBodyRead = resolve;+ });+ const layer = reachabilityLayer(+ Effect.sync(() => ({+ status: 530,+ body: new ReadableStream({+ pull: () => {+ markBodyRead();+ return new Promise<void>(() => {});+ },+ }),+ })),+ () => Effect.succeed(false),+ );+ yield* Effect.gen(function* () {+ const reachability = yield* ConnectReachability.ConnectReachability;+ const check = yield* Effect.forkChild(reachability.check);+ yield* Effect.promise(() => bodyRead);+ yield* TestClock.adjust(Duration.seconds(2));+ expect(yield* Fiber.join(check)).toMatchObject({ cause: "no_connector" });+ }).pipe(Effect.provide(layer));+ }),+ );+});+describe("T3 Connect troubleshooting links", () => {it("each section names a heading in the user guide", () => {const guide = NodeFS.readFileSync(diff --git a/apps/server/src/cloud/ConnectReachability.ts b/apps/server/src/cloud/ConnectReachability.tsindex acb581778d9..aa969767329 100644--- a/apps/server/src/cloud/ConnectReachability.ts+++ b/apps/server/src/cloud/ConnectReachability.ts@@ -6,6 +6,7 @@ import {formatConnectReachable,} from "@t3tools/contracts";import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale";+import * as Clock from "effect/Clock";import * as Context from "effect/Context";import * as DateTime from "effect/DateTime";import * as Duration from "effect/Duration";@@ -13,7 +14,9 @@ import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as Option from "effect/Option";import * as Ref from "effect/Ref";+import * as Scope from "effect/Scope";import * as Stream from "effect/Stream";+import * as SynchronizedRef from "effect/SynchronizedRef";import { HttpClient } from "effect/http";import { ChildProcessSpawner } from "effect/process";@@ -27,13 +30,19 @@ const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.cconst TUNNEL_EDGE_PORT = 7844;const EDGE_CONNECT_TIMEOUT = Duration.seconds(5);const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10);+const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2);// cloudflared registers several connections within a few seconds; they are one change.const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3);// Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors).const RECHECK_INTERVAL = Duration.minutes(5);+// cloudflared can keep reporting ready connections the edge already dropped, so its+// own readiness watch never restarts it. A 530 can also be brief, so restart only once it+// lasts, and wait longer after each restart that did not help.+const DEAD_CONNECTOR_RESTART_AFTER = Duration.minutes(2);+const DEAD_CONNECTOR_RESTART_AFTER_MAX = Duration.hours(1);export type EndpointProbe =- | { readonly kind: "status"; readonly status: number }+ | { readonly kind: "status"; readonly status: number; readonly cloudflareError?: string }| { readonly kind: "dns_failed" | "timeout" | "network_error" };/** Everything one check learned. `null` means that step did not run. */@@ -122,8 +131,10 @@ export function classifyReachability("Wait a minute. If it stays, check the network and the server log.",);}- // Cloudflare answers 530 (error 1033) when the tunnel has no live connector.- if (status === 530) {+ const cloudflareError = endpoint.kind === "status" ? endpoint.cloudflareError : undefined;+ // Cloudflare answers 530 with error 1033 when the tunnel has no live connector. Other+ // codes, such as 1016 for a broken DNS record, are not fixed by restarting cloudflared.+ if (status === 530 && (cloudflareError === undefined || cloudflareError === "1033")) {return fail("no_connector","Cloudflare has no live connection from this machine for the tunnel.",@@ -131,9 +142,10 @@ export function classifyReachability();}if (status !== null) {+ const code = cloudflareError === undefined ? "" : ` (Cloudflare error ${cloudflareError})`;return fail("unexpected_response",- `The T3 Connect address answered with HTTP ${status}.`,+ `The T3 Connect address answered with HTTP ${status}${code}.`,restart,);}@@ -207,24 +219,57 @@ const make = Effect.gen(function* () {Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),);const checkCounter = yield* Ref.make(0);+ // Checks update this as they finish, even when a newer check already did. A dead streak+ // or a restart backoff that began before a healthy check no longer counts.+ const lastHealthyOrder = yield* Ref.make(0);+ const deadConnectorRef = yield* SynchronizedRef.make<{+ readonly streak: {+ readonly pid: number;+ readonly since: number;+ readonly order: number;+ } | null;+ readonly restartAfterMillis: number;+ readonly restartOrder: number;+ }>({+ streak: null,+ restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),+ restartOrder: 0,+ });const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;+ const scope = yield* Scope.Scope;const readExitNode = readTailscaleExitNode.pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),Effect.orElseSucceed(() => null),);- const probeEndpoint = (hostname: string) =>- httpClient.get(`https://${hostname}/.well-known/t3/environment`).pipe(- Effect.map((response): EndpointProbe => ({ kind: "status", status: response.status })),- Effect.scoped,- Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT),- Effect.map(Option.getOrElse((): EndpointProbe => ({ kind: "timeout" }))),- Effect.catch((error) =>- Effect.succeed<EndpointProbe>({- kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",- }),- ),- );+ const probeEndpoint = Effect.fn("ConnectReachability.probeEndpoint")(+ function* (hostname: string) {+ const response = yield* httpClient+ .get(`https://${hostname}/.well-known/t3/environment`, { acceptJson: true })+ .pipe(Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT));+ if (Option.isNone(response)) return { kind: "timeout" } satisfies EndpointProbe;+ const { status } = response.value;+ if (status !== 530) return { kind: "status", status } satisfies EndpointProbe;+ // Cloudflare's error page is HTML unless the client accepts JSON, which carries+ // `"error_code":1033`. Clients that send no User-Agent get `error code: 1033`.+ const body = yield* response.value.text.pipe(+ Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT),+ Effect.orElseSucceed(() => Option.none<string>()),+ );+ const code = /error[_ ]code"?:\s*(\d{4})/i.exec(Option.getOrElse(body, () => ""))?.[1];+ return (+ code === undefined+ ? { kind: "status", status }+ : { kind: "status", status, cloudflareError: code }+ ) satisfies EndpointProbe;+ },+ Effect.scoped,+ Effect.catch((error) =>+ Effect.succeed<EndpointProbe>({+ kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",+ }),+ ),+ );const gatherFacts = Effect.gen(function* () {const managedTunnelConfigured = Option.isSome(@@ -257,16 +302,90 @@ const make = Effect.gen(function* () {return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };});- const check = Effect.gen(function* () {+ const isLatest = (order: number) =>+ Ref.get(latestRef).pipe(+ Effect.map((latest) => Option.isNone(latest) || latest.value.order <= order),+ );++ // Runs inside the check span; the runtime restart has its own. Returns the streak a dead+ // connector just started and when to confirm it.+ const restartDeadConnector = Effect.fnUntraced(function* (+ facts: ReachabilityFacts,+ result: EnvironmentConnectReachability,+ order: number,+ ) {+ const now = yield* Clock.currentTimeMillis;+ const { connector } = facts;+ const dead =+ result.status === "unreachable" &&+ result.cause === "no_connector" &&+ connector !== null &&+ (connector.readyConnections ?? 0) > 0+ ? connector+ : null;+ return yield* SynchronizedRef.modifyEffect(deadConnectorRef, (current) =>+ Effect.gen(function* () {+ const healthy = yield* Ref.get(lastHealthyOrder);+ const state = {+ streak: current.streak && current.streak.order > healthy ? current.streak : null,+ restartAfterMillis:+ current.restartOrder > healthy+ ? current.restartAfterMillis+ : Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),+ restartOrder: current.restartOrder,+ };+ if (!(yield* isLatest(order))) return [null, state] as const;+ if (dead === null) return [null, { ...state, streak: null }] as const;+ if (state.streak?.pid !== dead.pid) {+ const started = { ...state, streak: { pid: dead.pid, since: now, order } };+ return [{ order, after: Duration.millis(state.restartAfterMillis) }, started] as const;+ }+ if (now - state.streak.since < state.restartAfterMillis) return [null, state] as const;+ // Checks can invalidate the streak while this request waits for the runtime's lock.+ const streakOrder = state.streak.order;+ const stillDead = Effect.gen(function* () {+ if (!(yield* isLatest(order))) return false;+ return (yield* Ref.get(lastHealthyOrder)) < streakOrder;+ });+ if (!(yield* runtime.restartConnector(dead.pid, stillDead))) {+ return [null, state] as const;+ }+ const restarted = {+ streak: null,+ restartAfterMillis: Math.min(+ state.restartAfterMillis * 2,+ Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),+ ),+ restartOrder: order,+ };+ return [null, restarted] as const;+ }),+ );+ });++ const check: Effect.Effect<EnvironmentConnectReachability> = Effect.gen(function* () {const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1);const facts = yield* gatherFacts;const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now));+ if (result.status === "reachable") {+ yield* Ref.update(lastHealthyOrder, (healthy) => Math.max(healthy, order));+ }const previous = yield* Ref.modify(latestRef, (current) =>Option.isSome(current) && current.value.order > order? [null, current]: [current, Option.some({ order, result })],);if (previous === null) return result;+ const started = yield* restartDeadConnector(facts, result, order);+ // Nothing else would look again until the next periodic check.+ if (started !== null) {+ const confirm = SynchronizedRef.get(deadConnectorRef).pipe(+ Effect.flatMap(({ streak }) =>+ streak?.order === started.order ? Effect.suspend(() => check) : Effect.void,+ ),+ );+ yield* Effect.sleep(started.after).pipe(Effect.andThen(confirm), Effect.forkIn(scope));+ }const changed = Option.match(previous, {onNone: () => true,onSome: ({ result: last }) =>diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.tsindex ad7761f7c45..5cad290678f 100644--- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts+++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts@@ -413,6 +413,28 @@ describe("CloudManagedEndpointRuntime", () => {return { spawner, spawned, killed, exits, started, emit, observe };});+ it.effect("restarts only the inspected connector and supervises its replacement", () =>+ Effect.gen(function* () {+ const fixture = yield* makeTransportSpawner();+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);+ yield* runtime.applyConfig(tunnelConfig);+ yield* fixture.observe(runtime, 0);+ expect(yield* runtime.inspectConnector).toMatchObject({ pid: 800 });+ expect(yield* runtime.restartConnector(799, Effect.succeed(true))).toBe(false);+ // A newer check saw the tunnel recover while this one waited for the lock.+ expect(yield* runtime.restartConnector(800, Effect.succeed(false))).toBe(false);+ expect(fixture.killed).toEqual([]);+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(true);+ expect(yield* Queue.take(fixture.started)).toBe(0);+ expect(yield* Queue.take(fixture.started)).toBe(1);+ yield* fixture.observe(runtime, 1);+ expect(fixture.killed).toContain(800);+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);+ expect(yield* runtime.inspectConnector).toMatchObject({ pid: 801 });+ }),+ );+it.effect("preserves an explicit transport choice despite repeated QUIC stalls", () =>Effect.gen(function* () {const fixture = yield* makeTransportSpawner({ transport: "quic" });@@ -557,6 +579,39 @@ describe("CloudManagedEndpointRuntime", () => {}),);+ it.effect("decides a restart only once it holds the reconcile lock", () =>+ Effect.gen(function* () {+ const checking = yield* Deferred.make<void>();+ const releaseCheck = yield* Deferred.make<void>();+ const fixture = yield* makeTransportSpawner({+ isRunning: Deferred.succeed(checking, undefined).pipe(+ Effect.andThen(Deferred.await(releaseCheck)),+ Effect.as(true),+ ),+ });+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);+ yield* runtime.applyConfig(tunnelConfig);+ yield* fixture.observe(runtime, 0);++ // Hold reconciliation in a process probe so the restart waits for the lock.+ const probe = yield* runtime.applyConfig(tunnelConfig).pipe(Effect.forkChild);+ yield* Deferred.await(checking);+ let stillDead = true;+ const restart = yield* runtime+ .restartConnector(+ 800,+ Effect.sync(() => stillDead),+ )+ .pipe(Effect.forkChild({ startImmediately: true }));+ stillDead = false;+ yield* Deferred.succeed(releaseCheck, undefined);+ yield* Fiber.join(probe);++ expect(yield* Fiber.join(restart)).toBe(false);+ expect(fixture.killed).toEqual([]);+ }),+ );+it.effect("keeps redacted connector logs bounded even when rotation fails", () =>Effect.gen(function* () {const fs = yield* FileSystem.FileSystem;diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.tsindex ca069978761..def1a8d112c 100644--- a/apps/server/src/cloud/ManagedEndpointRuntime.ts+++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts@@ -53,6 +53,7 @@ export type CloudManagedEndpointRuntimeStatus =/** What the running cloudflared reports about itself. `null` fields are not known yet. */export interface ConnectorFacts {+ readonly pid: number;readonly readyConnections: number | null;/** The public tunnel hostname from the remote config cloudflared receives once it connects. */readonly hostname: string | null;@@ -71,6 +72,14 @@ export class CloudManagedEndpointRuntime extends Context.Service<readonly readinessChanged: Stream.Stream<void>;readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;+ /**+ * Restarts the connector if it still runs as `pid` and `stillDead` holds once it has+ * the lock. Returns whether it sent the restart.+ */+ readonly restartConnector: (+ pid: number,+ stillDead: Effect.Effect<boolean>,+ ) => Effect.Effect<boolean>;readonly tunnelConnected: Stream.Stream<void>;readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;}@@ -822,6 +831,23 @@ export const make = Effect.gen(function* () {withRelayClientLog,);+ const restartConnector = Effect.fn("CloudManagedEndpointRuntime.restartConnector")(function* (+ pid: number,+ stillDead: Effect.Effect<boolean>,+ ) {+ const active = yield* Ref.get(activeRef);+ if (!active || Number(active.child.pid) !== pid || !(yield* stillDead)) return false;+ yield* Effect.logWarning(+ "Cloudflare has no live connection from the relay client; restarting it",+ { pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName },+ );+ // The supervisor starts the replacement with its usual crash-loop backoff.+ return yield* active.child.kill().pipe(+ Effect.as(true),+ Effect.orElseSucceed(() => false),+ );+ }, reconcileSemaphore.withPermits(1));+const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>httpClient.get(url).pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),@@ -837,9 +863,10 @@ export const make = Effect.gen(function* () {if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) {return null;}+ const pid = Number(active.child.pid);const protocol = yield* Ref.get(active.protocol);if (!(yield* Deferred.isDone(active.metricsAddress))) {- return { readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;+ return { pid, readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;}const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;const [ready, config] = yield* Effect.all(@@ -850,6 +877,7 @@ export const make = Effect.gen(function* () {{ concurrency: 2 },);return {+ pid,readyConnections: ready?.readyConnections ?? null,hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null,protocol,@@ -864,6 +892,7 @@ export const make = Effect.gen(function* () {requestRecovery: Effect.fn("CloudManagedEndpointRuntime.requestRecovery")(function (config) {return Queue.offer(recoveryRequests, config).pipe(Effect.asVoid);}),+ restartConnector,tunnelConnected: Stream.fromQueue(tunnelConnections),withLinkStateLock: linkStateSemaphore.withPermits(1),});diff --git a/apps/server/src/guest/GuestShareService.test.ts b/apps/server/src/guest/GuestShareService.test.tsindex cfd8ca1363f..71402224787 100644--- a/apps/server/src/guest/GuestShareService.test.ts+++ b/apps/server/src/guest/GuestShareService.test.ts@@ -80,6 +80,7 @@ const layerOrchestration = Layer.mergeAll(const hostname = Ref.makeUnsafe<string | null>("env.t3.test");const layerConnect = Layer.mock(CloudManagedEndpointRuntime)({inspectConnector: Effect.map(Ref.get(hostname), (value) => ({+ pid: 1,readyConnections: 1,hostname: value,protocol: "http2",