Skip to content
TypeScript

fix(server): restart T3 Connect's tunnel client when Cloudflare drops it (#24946)

shivamc192c439c15f
diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts
index 8213288dee9..3e34bf0be28 100644
--- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts
+++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts
@@ -206,6 +206,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
}),
recoveryRequests: Stream.empty,
requestRecovery: () => Effect.void,
+ restartConnector: () => Effect.succeed(false),
tunnelConnected: Stream.empty,
inspectConnector: Effect.succeed(null),
readinessChanged: Stream.empty,
diff --git a/apps/server/src/cloud/ConnectReachability.test.ts b/apps/server/src/cloud/ConnectReachability.test.ts
index b6a73ab5dcd..b279e331f3d 100644
--- a/apps/server/src/cloud/ConnectReachability.test.ts
+++ b/apps/server/src/cloud/ConnectReachability.test.ts
@@ -1,14 +1,32 @@
// @effect-diagnostics nodeBuiltinImport:off - reads the user guide to check its anchors.
import * as NodeFS from "node:fs";
+import * as NodeNet from "node:net";
import { T3_CONNECT_TROUBLESHOOTING_SECTIONS } from "@t3tools/contracts";
import { describe, expect, it } from "@effect/vitest";
+import * as Clock from "effect/Clock";
+import * as Deferred from "effect/Deferred";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as Layer from "effect/Layer";
+import * as Queue from "effect/Queue";
+import * as Stream from "effect/Stream";
+import * as TestClock from "effect/testing/TestClock";
+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
+import { ChildProcessSpawner } from "effect/process";
+import { vi } from "vite-plus/test";
+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import * as ConnectReachability from "./ConnectReachability.ts";
import { classifyReachability, type ReachabilityFacts } from "./ConnectReachability.ts";
+import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
+
+vi.mock("node:net", { spy: true });
const checkedAt = "2026-10-19T00:00:00.000Z";
-const connected = { readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
-const notConnected = { readyConnections: 0, hostname: null, protocol: null };
+const connected = { pid: 1, readyConnections: 4, hostname: "env.example.test", protocol: "quic" };
+const notConnected = { pid: 1, readyConnections: 0, hostname: null, protocol: null };
const facts = (overrides: Partial<ReachabilityFacts>): ReachabilityFacts => ({
managedTunnelConfigured: true,
connector: connected,
@@ -22,6 +40,8 @@ describe("classifyReachability", () => {
it.each([
[{ kind: "status", status: 502 }, "origin_unreachable"],
[{ kind: "status", status: 530 }, "no_connector"],
+ [{ kind: "status", status: 530, cloudflareError: "1033" }, "no_connector"],
+ [{ kind: "status", status: 530, cloudflareError: "1016" }, "unexpected_response"],
[{ kind: "network_error" }, "timeout"],
] as const)("maps endpoint result %o to %s", (endpoint, cause) => {
expect(classifyReachability(facts({ endpoint, edgeReachable: true }), checkedAt)).toMatchObject(
@@ -40,7 +60,12 @@ describe("classifyReachability", () => {
expect(
classifyReachability(
facts({
- connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
+ connector: {
+ pid: 1,
+ readyConnections: null,
+ hostname: "env.example.test",
+ protocol: null,
+ },
endpoint: { kind: "status", status: 502 },
edgeReachable: false,
}),
@@ -51,7 +76,12 @@ describe("classifyReachability", () => {
expect(
classifyReachability(
facts({
- connector: { readyConnections: null, hostname: "env.example.test", protocol: null },
+ connector: {
+ pid: 1,
+ readyConnections: null,
+ hostname: "env.example.test",
+ protocol: null,
+ },
endpoint: { kind: "dns_failed" },
edgeReachable: false,
}),
@@ -74,6 +104,280 @@ describe("classifyReachability", () => {
});
});
+interface ServedResponse {
+ readonly status: number;
+ readonly body: string | ReadableStream<Uint8Array> | { readonly cloudflareError: number };
+}
+
+// What Cloudflare serves for its 530 page: HTML unless the client accepts JSON.
+const cloudflareErrorPage = (request: HttpClientRequest.HttpClientRequest, code: number) =>
+ request.headers.accept === "application/json"
+ ? JSON.stringify({ status: 530, error_code: code, error_name: "tunnel_error" })
+ : `<!doctype html><h1><span>Error</span> <span>${code}</span></h1>`;
+
+const reachabilityLayer = (
+ respond: Effect.Effect<ServedResponse>,
+ restartConnector: (pid: number, stillDead: Effect.Effect<boolean>) => Effect.Effect<boolean>,
+) =>
+ ConnectReachability.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ Layer.mock(ManagedEndpointRuntime.CloudManagedEndpointRuntime)({
+ inspectConnector: Effect.succeed(connected),
+ readinessChanged: Stream.never,
+ restartConnector,
+ }),
+ Layer.mock(ServerSecretStore.ServerSecretStore)({
+ get: () => Effect.succeedSome(new Uint8Array()),
+ }),
+ Layer.succeed(
+ HttpClient.HttpClient,
+ HttpClient.make((request) =>
+ respond.pipe(
+ Effect.map(({ status, body }) =>
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(
+ typeof body === "object" && "cloudflareError" in body
+ ? cloudflareErrorPage(request, body.cloudflareError)
+ : body,
+ { status },
+ ),
+ ),
+ ),
+ ),
+ ),
+ ),
+ Layer.succeed(
+ ChildProcessSpawner.ChildProcessSpawner,
+ ChildProcessSpawner.make(() => Effect.die("no tailscale")),
+ ),
+ ),
+ ),
+ );
+
+const mockEdgeConnect = () =>
+ vi.spyOn(NodeNet, "connect").mockImplementation(() => {
+ const socket = new NodeNet.Socket();
+ queueMicrotask(() => socket.emit("connect"));
+ return socket;
+ });
+
+describe("ConnectReachability stale connector recovery", () => {
+ it.effect("restarts cloudflared when Cloudflare has no connector but it reports ready", () =>
+ Effect.gen(function* () {
+ using _connect = mockEdgeConnect();
+ const restarts = yield* Queue.unbounded<number>();
+ // Cloudflare answers 530 while cloudflared still reports four ready connections.
+ let status = 530;
+ let body = { cloudflareError: 1033 };
+ const layer = reachabilityLayer(
+ Effect.sync(() => ({ status, body })),
+ (pid, stillDead) =>
+ stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
+ );
+ // The scheduled checks run in the layer's scope, so the assertions run inside it.
+ yield* Effect.gen(function* () {
+ const reachability = yield* ConnectReachability.ConnectReachability;
+
+ // The first sighting schedules its own confirming check two minutes later.
+ expect(yield* reachability.check).toMatchObject({ cause: "no_connector" });
+ yield* TestClock.adjust(Duration.minutes(1));
+ expect(yield* Queue.size(restarts)).toBe(0);
+ yield* TestClock.adjust(Duration.minutes(1));
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
+
+ // A restart that did not help doubles the wait.
+ yield* reachability.check;
+ yield* TestClock.adjust(Duration.minutes(2));
+ expect(yield* Queue.size(restarts)).toBe(0);
+ yield* TestClock.adjust(Duration.minutes(2));
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
+
+ // Reaching the endpoint resets the wait, and a 530 for a broken DNS record is left alone.
+ status = 200;
+ expect(yield* reachability.check).toMatchObject({ status: "reachable" });
+ status = 530;
+ body = { cloudflareError: 1016 };
+ expect(yield* reachability.check).toMatchObject({ cause: "unexpected_response" });
+ yield* TestClock.adjust(Duration.minutes(2));
+ expect(yield* Queue.size(restarts)).toBe(0);
+
+ body = { cloudflareError: 1033 };
+ yield* reachability.check;
+ yield* TestClock.adjust(Duration.minutes(2));
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
+ }).pipe(Effect.provide(layer));
+ }),
+ );
+
+ it.effect("starts a new streak after a healthy check that a newer check overtook", () =>
+ Effect.gen(function* () {
+ using _connect = mockEdgeConnect();
+ const restartTimes = yield* Queue.unbounded<number>();
+ const healthyRequested = yield* Deferred.make<void>();
+ const healthyAnswer = yield* Deferred.make<void>();
+ let status = 530;
+ const layer = reachabilityLayer(
+ Effect.gen(function* () {
+ const served = status;
+ if (served === 200) {
+ yield* Deferred.succeed(healthyRequested, undefined);
+ yield* Deferred.await(healthyAnswer);
+ }
+ return { status: served, body: { cloudflareError: 1033 } };
+ }),
+ (_pid, stillDead) =>
+ stillDead.pipe(
+ Effect.tap((dead) =>
+ dead
+ ? Clock.currentTimeMillis.pipe(
+ Effect.flatMap((now) => Queue.offer(restartTimes, now)),
+ )
+ : Effect.void,
+ ),
+ ),
+ );
+ yield* Effect.gen(function* () {
+ const reachability = yield* ConnectReachability.ConnectReachability;
+ const startedAt = yield* Clock.currentTimeMillis;
+ yield* reachability.check;
+ yield* TestClock.adjust(Duration.minutes(1));
+
+ // The healthy answer arrives after a newer check already reported the tunnel dead.
+ status = 200;
+ const healthy = yield* Effect.forkChild(reachability.check);
+ yield* Deferred.await(healthyRequested);
+ status = 530;
+ yield* reachability.check;
+ yield* Deferred.succeed(healthyAnswer, undefined);
+ expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
+
+ // The grace period starts again at the next dead check, a minute after the first one.
+ yield* reachability.check;
+ yield* TestClock.adjust(Duration.minutes(2));
+ expect((yield* Queue.take(restartTimes)) - startedAt).toBe(
+ Duration.toMillis(Duration.minutes(3)),
+ );
+ }).pipe(Effect.provide(layer));
+ }),
+ );
+
+ it.effect("rejects a restart when an overtaken healthy check finishes while it waits", () =>
+ Effect.gen(function* () {
+ using _connect = mockEdgeConnect();
+ const healthyRequested = yield* Deferred.make<void>();
+ const healthyAnswer = yield* Deferred.make<void>();
+ const restartRequested = yield* Deferred.make<void>();
+ const releaseRestart = yield* Deferred.make<void>();
+ const guardChecks = yield* Queue.unbounded<boolean>();
+ let status = 530;
+ const layer = reachabilityLayer(
+ Effect.gen(function* () {
+ const served = status;
+ if (served === 200) {
+ yield* Deferred.succeed(healthyRequested, undefined);
+ yield* Deferred.await(healthyAnswer);
+ }
+ return { status: served, body: { cloudflareError: 1033 } };
+ }),
+ (_pid, stillDead) =>
+ Effect.gen(function* () {
+ yield* Deferred.succeed(restartRequested, undefined);
+ yield* Deferred.await(releaseRestart);
+ const accepted = yield* stillDead;
+ yield* Queue.offer(guardChecks, accepted);
+ return accepted;
+ }),
+ );
+ yield* Effect.gen(function* () {
+ const reachability = yield* ConnectReachability.ConnectReachability;
+ yield* reachability.check;
+ yield* TestClock.adjust(Duration.seconds(119));
+
+ status = 200;
+ const healthy = yield* Effect.forkChild(reachability.check);
+ yield* Deferred.await(healthyRequested);
+ status = 530;
+ yield* TestClock.adjust(Duration.seconds(1));
+ yield* Deferred.await(restartRequested);
+
+ yield* Deferred.succeed(healthyAnswer, undefined);
+ expect(yield* Fiber.join(healthy)).toMatchObject({ status: "reachable" });
+ yield* Deferred.succeed(releaseRestart, undefined);
+ expect(yield* Queue.take(guardChecks)).toBe(false);
+
+ yield* reachability.check;
+ yield* TestClock.adjust(Duration.minutes(1));
+ expect(yield* Queue.size(guardChecks)).toBe(0);
+ yield* TestClock.adjust(Duration.minutes(1));
+ expect(yield* Queue.take(guardChecks)).toBe(true);
+ }).pipe(Effect.provide(layer));
+ }),
+ );
+
+ it.effect("drops the confirming check of a streak that already ended", () =>
+ Effect.gen(function* () {
+ using _connect = mockEdgeConnect();
+ const restarts = yield* Queue.unbounded<number>();
+ let status = 530;
+ let requests = 0;
+ const layer = reachabilityLayer(
+ Effect.sync(() => {
+ requests += 1;
+ return { status, body: { cloudflareError: 1033 } };
+ }),
+ (pid, stillDead) =>
+ stillDead.pipe(Effect.tap((dead) => (dead ? Queue.offer(restarts, pid) : Effect.void))),
+ );
+ yield* Effect.gen(function* () {
+ const reachability = yield* ConnectReachability.ConnectReachability;
+ yield* reachability.check;
+ yield* TestClock.adjust(Duration.minutes(1));
+ status = 200;
+ yield* reachability.check;
+ status = 530;
+ yield* reachability.check;
+
+ const before = requests;
+ yield* TestClock.adjust(Duration.minutes(1));
+ expect(requests).toBe(before);
+ yield* TestClock.adjust(Duration.minutes(1));
+ expect(yield* Queue.take(restarts)).toBe(connected.pid);
+ }).pipe(Effect.provide(layer));
+ }),
+ );
+
+ it.effect("keeps a 530 whose error body never finishes", () =>
+ Effect.gen(function* () {
+ using _connect = mockEdgeConnect();
+ let markBodyRead = () => {};
+ const bodyRead = new Promise<void>((resolve) => {
+ markBodyRead = resolve;
+ });
+ const layer = reachabilityLayer(
+ Effect.sync(() => ({
+ status: 530,
+ body: new ReadableStream({
+ pull: () => {
+ markBodyRead();
+ return new Promise<void>(() => {});
+ },
+ }),
+ })),
+ () => Effect.succeed(false),
+ );
+ yield* Effect.gen(function* () {
+ const reachability = yield* ConnectReachability.ConnectReachability;
+ const check = yield* Effect.forkChild(reachability.check);
+ yield* Effect.promise(() => bodyRead);
+ yield* TestClock.adjust(Duration.seconds(2));
+ expect(yield* Fiber.join(check)).toMatchObject({ cause: "no_connector" });
+ }).pipe(Effect.provide(layer));
+ }),
+ );
+});
+
describe("T3 Connect troubleshooting links", () => {
it("each section names a heading in the user guide", () => {
const guide = NodeFS.readFileSync(
diff --git a/apps/server/src/cloud/ConnectReachability.ts b/apps/server/src/cloud/ConnectReachability.ts
index acb581778d9..aa969767329 100644
--- a/apps/server/src/cloud/ConnectReachability.ts
+++ b/apps/server/src/cloud/ConnectReachability.ts
@@ -6,6 +6,7 @@ import {
formatConnectReachable,
} from "@t3tools/contracts";
import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale";
+import * as Clock from "effect/Clock";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Duration from "effect/Duration";
@@ -13,7 +14,9 @@ import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Ref from "effect/Ref";
+import * as Scope from "effect/Scope";
import * as Stream from "effect/Stream";
+import * as SynchronizedRef from "effect/SynchronizedRef";
import { HttpClient } from "effect/http";
import { ChildProcessSpawner } from "effect/process";
@@ -27,13 +30,19 @@ const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.c
const TUNNEL_EDGE_PORT = 7844;
const EDGE_CONNECT_TIMEOUT = Duration.seconds(5);
const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10);
+const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2);
// cloudflared registers several connections within a few seconds; they are one change.
const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3);
// Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors).
const RECHECK_INTERVAL = Duration.minutes(5);
+// cloudflared can keep reporting ready connections the edge already dropped, so its
+// own readiness watch never restarts it. A 530 can also be brief, so restart only once it
+// lasts, and wait longer after each restart that did not help.
+const DEAD_CONNECTOR_RESTART_AFTER = Duration.minutes(2);
+const DEAD_CONNECTOR_RESTART_AFTER_MAX = Duration.hours(1);
export type EndpointProbe =
- | { readonly kind: "status"; readonly status: number }
+ | { readonly kind: "status"; readonly status: number; readonly cloudflareError?: string }
| { readonly kind: "dns_failed" | "timeout" | "network_error" };
/** Everything one check learned. `null` means that step did not run. */
@@ -122,8 +131,10 @@ export function classifyReachability(
"Wait a minute. If it stays, check the network and the server log.",
);
}
- // Cloudflare answers 530 (error 1033) when the tunnel has no live connector.
- if (status === 530) {
+ const cloudflareError = endpoint.kind === "status" ? endpoint.cloudflareError : undefined;
+ // Cloudflare answers 530 with error 1033 when the tunnel has no live connector. Other
+ // codes, such as 1016 for a broken DNS record, are not fixed by restarting cloudflared.
+ if (status === 530 && (cloudflareError === undefined || cloudflareError === "1033")) {
return fail(
"no_connector",
"Cloudflare has no live connection from this machine for the tunnel.",
@@ -131,9 +142,10 @@ export function classifyReachability(
);
}
if (status !== null) {
+ const code = cloudflareError === undefined ? "" : ` (Cloudflare error ${cloudflareError})`;
return fail(
"unexpected_response",
- `The T3 Connect address answered with HTTP ${status}.`,
+ `The T3 Connect address answered with HTTP ${status}${code}.`,
restart,
);
}
@@ -207,24 +219,57 @@ const make = Effect.gen(function* () {
Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),
);
const checkCounter = yield* Ref.make(0);
+ // Checks update this as they finish, even when a newer check already did. A dead streak
+ // or a restart backoff that began before a healthy check no longer counts.
+ const lastHealthyOrder = yield* Ref.make(0);
+ const deadConnectorRef = yield* SynchronizedRef.make<{
+ readonly streak: {
+ readonly pid: number;
+ readonly since: number;
+ readonly order: number;
+ } | null;
+ readonly restartAfterMillis: number;
+ readonly restartOrder: number;
+ }>({
+ streak: null,
+ restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
+ restartOrder: 0,
+ });
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+ const scope = yield* Scope.Scope;
const readExitNode = readTailscaleExitNode.pipe(
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
Effect.orElseSucceed(() => null),
);
- const probeEndpoint = (hostname: string) =>
- httpClient.get(`https://${hostname}/.well-known/t3/environment`).pipe(
- Effect.map((response): EndpointProbe => ({ kind: "status", status: response.status })),
- Effect.scoped,
- Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT),
- Effect.map(Option.getOrElse((): EndpointProbe => ({ kind: "timeout" }))),
- Effect.catch((error) =>
- Effect.succeed<EndpointProbe>({
- kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",
- }),
- ),
- );
+ const probeEndpoint = Effect.fn("ConnectReachability.probeEndpoint")(
+ function* (hostname: string) {
+ const response = yield* httpClient
+ .get(`https://${hostname}/.well-known/t3/environment`, { acceptJson: true })
+ .pipe(Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT));
+ if (Option.isNone(response)) return { kind: "timeout" } satisfies EndpointProbe;
+ const { status } = response.value;
+ if (status !== 530) return { kind: "status", status } satisfies EndpointProbe;
+ // Cloudflare's error page is HTML unless the client accepts JSON, which carries
+ // `"error_code":1033`. Clients that send no User-Agent get `error code: 1033`.
+ const body = yield* response.value.text.pipe(
+ Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT),
+ Effect.orElseSucceed(() => Option.none<string>()),
+ );
+ const code = /error[_ ]code"?:\s*(\d{4})/i.exec(Option.getOrElse(body, () => ""))?.[1];
+ return (
+ code === undefined
+ ? { kind: "status", status }
+ : { kind: "status", status, cloudflareError: code }
+ ) satisfies EndpointProbe;
+ },
+ Effect.scoped,
+ Effect.catch((error) =>
+ Effect.succeed<EndpointProbe>({
+ kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",
+ }),
+ ),
+ );
const gatherFacts = Effect.gen(function* () {
const managedTunnelConfigured = Option.isSome(
@@ -257,16 +302,90 @@ const make = Effect.gen(function* () {
return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };
});
- const check = Effect.gen(function* () {
+ const isLatest = (order: number) =>
+ Ref.get(latestRef).pipe(
+ Effect.map((latest) => Option.isNone(latest) || latest.value.order <= order),
+ );
+
+ // Runs inside the check span; the runtime restart has its own. Returns the streak a dead
+ // connector just started and when to confirm it.
+ const restartDeadConnector = Effect.fnUntraced(function* (
+ facts: ReachabilityFacts,
+ result: EnvironmentConnectReachability,
+ order: number,
+ ) {
+ const now = yield* Clock.currentTimeMillis;
+ const { connector } = facts;
+ const dead =
+ result.status === "unreachable" &&
+ result.cause === "no_connector" &&
+ connector !== null &&
+ (connector.readyConnections ?? 0) > 0
+ ? connector
+ : null;
+ return yield* SynchronizedRef.modifyEffect(deadConnectorRef, (current) =>
+ Effect.gen(function* () {
+ const healthy = yield* Ref.get(lastHealthyOrder);
+ const state = {
+ streak: current.streak && current.streak.order > healthy ? current.streak : null,
+ restartAfterMillis:
+ current.restartOrder > healthy
+ ? current.restartAfterMillis
+ : Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
+ restartOrder: current.restartOrder,
+ };
+ if (!(yield* isLatest(order))) return [null, state] as const;
+ if (dead === null) return [null, { ...state, streak: null }] as const;
+ if (state.streak?.pid !== dead.pid) {
+ const started = { ...state, streak: { pid: dead.pid, since: now, order } };
+ return [{ order, after: Duration.millis(state.restartAfterMillis) }, started] as const;
+ }
+ if (now - state.streak.since < state.restartAfterMillis) return [null, state] as const;
+ // Checks can invalidate the streak while this request waits for the runtime's lock.
+ const streakOrder = state.streak.order;
+ const stillDead = Effect.gen(function* () {
+ if (!(yield* isLatest(order))) return false;
+ return (yield* Ref.get(lastHealthyOrder)) < streakOrder;
+ });
+ if (!(yield* runtime.restartConnector(dead.pid, stillDead))) {
+ return [null, state] as const;
+ }
+ const restarted = {
+ streak: null,
+ restartAfterMillis: Math.min(
+ state.restartAfterMillis * 2,
+ Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
+ ),
+ restartOrder: order,
+ };
+ return [null, restarted] as const;
+ }),
+ );
+ });
+
+ const check: Effect.Effect<EnvironmentConnectReachability> = Effect.gen(function* () {
const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1);
const facts = yield* gatherFacts;
const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now));
+ if (result.status === "reachable") {
+ yield* Ref.update(lastHealthyOrder, (healthy) => Math.max(healthy, order));
+ }
const previous = yield* Ref.modify(latestRef, (current) =>
Option.isSome(current) && current.value.order > order
? [null, current]
: [current, Option.some({ order, result })],
);
if (previous === null) return result;
+ const started = yield* restartDeadConnector(facts, result, order);
+ // Nothing else would look again until the next periodic check.
+ if (started !== null) {
+ const confirm = SynchronizedRef.get(deadConnectorRef).pipe(
+ Effect.flatMap(({ streak }) =>
+ streak?.order === started.order ? Effect.suspend(() => check) : Effect.void,
+ ),
+ );
+ yield* Effect.sleep(started.after).pipe(Effect.andThen(confirm), Effect.forkIn(scope));
+ }
const changed = Option.match(previous, {
onNone: () => true,
onSome: ({ result: last }) =>
diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts
index ad7761f7c45..5cad290678f 100644
--- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts
+++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts
@@ -413,6 +413,28 @@ describe("CloudManagedEndpointRuntime", () => {
return { spawner, spawned, killed, exits, started, emit, observe };
});
+ it.effect("restarts only the inspected connector and supervises its replacement", () =>
+ Effect.gen(function* () {
+ const fixture = yield* makeTransportSpawner();
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);
+ yield* runtime.applyConfig(tunnelConfig);
+ yield* fixture.observe(runtime, 0);
+ expect(yield* runtime.inspectConnector).toMatchObject({ pid: 800 });
+ expect(yield* runtime.restartConnector(799, Effect.succeed(true))).toBe(false);
+ // A newer check saw the tunnel recover while this one waited for the lock.
+ expect(yield* runtime.restartConnector(800, Effect.succeed(false))).toBe(false);
+ expect(fixture.killed).toEqual([]);
+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(true);
+ expect(yield* Queue.take(fixture.started)).toBe(0);
+ expect(yield* Queue.take(fixture.started)).toBe(1);
+ yield* fixture.observe(runtime, 1);
+ expect(fixture.killed).toContain(800);
+ expect(yield* runtime.restartConnector(800, Effect.succeed(true))).toBe(false);
+ expect(yield* runtime.inspectConnector).toMatchObject({ pid: 801 });
+ }),
+ );
+
it.effect("preserves an explicit transport choice despite repeated QUIC stalls", () =>
Effect.gen(function* () {
const fixture = yield* makeTransportSpawner({ transport: "quic" });
@@ -557,6 +579,39 @@ describe("CloudManagedEndpointRuntime", () => {
}),
);
+ it.effect("decides a restart only once it holds the reconcile lock", () =>
+ Effect.gen(function* () {
+ const checking = yield* Deferred.make<void>();
+ const releaseCheck = yield* Deferred.make<void>();
+ const fixture = yield* makeTransportSpawner({
+ isRunning: Deferred.succeed(checking, undefined).pipe(
+ Effect.andThen(Deferred.await(releaseCheck)),
+ Effect.as(true),
+ ),
+ });
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
+ yield* runtime.applyConfig(tunnelConfig);
+ yield* fixture.observe(runtime, 0);
+
+ // Hold reconciliation in a process probe so the restart waits for the lock.
+ const probe = yield* runtime.applyConfig(tunnelConfig).pipe(Effect.forkChild);
+ yield* Deferred.await(checking);
+ let stillDead = true;
+ const restart = yield* runtime
+ .restartConnector(
+ 800,
+ Effect.sync(() => stillDead),
+ )
+ .pipe(Effect.forkChild({ startImmediately: true }));
+ stillDead = false;
+ yield* Deferred.succeed(releaseCheck, undefined);
+ yield* Fiber.join(probe);
+
+ expect(yield* Fiber.join(restart)).toBe(false);
+ expect(fixture.killed).toEqual([]);
+ }),
+ );
+
it.effect("keeps redacted connector logs bounded even when rotation fails", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts
index ca069978761..def1a8d112c 100644
--- a/apps/server/src/cloud/ManagedEndpointRuntime.ts
+++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts
@@ -53,6 +53,7 @@ export type CloudManagedEndpointRuntimeStatus =
/** What the running cloudflared reports about itself. `null` fields are not known yet. */
export interface ConnectorFacts {
+ readonly pid: number;
readonly readyConnections: number | null;
/** The public tunnel hostname from the remote config cloudflared receives once it connects. */
readonly hostname: string | null;
@@ -71,6 +72,14 @@ export class CloudManagedEndpointRuntime extends Context.Service<
readonly readinessChanged: Stream.Stream<void>;
readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
+ /**
+ * Restarts the connector if it still runs as `pid` and `stillDead` holds once it has
+ * the lock. Returns whether it sent the restart.
+ */
+ readonly restartConnector: (
+ pid: number,
+ stillDead: Effect.Effect<boolean>,
+ ) => Effect.Effect<boolean>;
readonly tunnelConnected: Stream.Stream<void>;
readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
}
@@ -822,6 +831,23 @@ export const make = Effect.gen(function* () {
withRelayClientLog,
);
+ const restartConnector = Effect.fn("CloudManagedEndpointRuntime.restartConnector")(function* (
+ pid: number,
+ stillDead: Effect.Effect<boolean>,
+ ) {
+ const active = yield* Ref.get(activeRef);
+ if (!active || Number(active.child.pid) !== pid || !(yield* stillDead)) return false;
+ yield* Effect.logWarning(
+ "Cloudflare has no live connection from the relay client; restarting it",
+ { pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName },
+ );
+ // The supervisor starts the replacement with its usual crash-loop backoff.
+ return yield* active.child.kill().pipe(
+ Effect.as(true),
+ Effect.orElseSucceed(() => false),
+ );
+ }, reconcileSemaphore.withPermits(1));
+
const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>
httpClient.get(url).pipe(
Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),
@@ -837,9 +863,10 @@ export const make = Effect.gen(function* () {
if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) {
return null;
}
+ const pid = Number(active.child.pid);
const protocol = yield* Ref.get(active.protocol);
if (!(yield* Deferred.isDone(active.metricsAddress))) {
- return { readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
+ return { pid, readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
}
const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;
const [ready, config] = yield* Effect.all(
@@ -850,6 +877,7 @@ export const make = Effect.gen(function* () {
{ concurrency: 2 },
);
return {
+ pid,
readyConnections: ready?.readyConnections ?? null,
hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null,
protocol,
@@ -864,6 +892,7 @@ export const make = Effect.gen(function* () {
requestRecovery: Effect.fn("CloudManagedEndpointRuntime.requestRecovery")(function (config) {
return Queue.offer(recoveryRequests, config).pipe(Effect.asVoid);
}),
+ restartConnector,
tunnelConnected: Stream.fromQueue(tunnelConnections),
withLinkStateLock: linkStateSemaphore.withPermits(1),
});
diff --git a/apps/server/src/guest/GuestShareService.test.ts b/apps/server/src/guest/GuestShareService.test.ts
index cfd8ca1363f..71402224787 100644
--- a/apps/server/src/guest/GuestShareService.test.ts
+++ b/apps/server/src/guest/GuestShareService.test.ts
@@ -80,6 +80,7 @@ const layerOrchestration = Layer.mergeAll(
const hostname = Ref.makeUnsafe<string | null>("env.t3.test");
const layerConnect = Layer.mock(CloudManagedEndpointRuntime)({
inspectConnector: Effect.map(Ref.get(hostname), (value) => ({
+ pid: 1,
readyConnections: 1,
hostname: value,
protocol: "http2",