Skip to content
TypeScript

fix(relay): T3 Connect repairs tunnel ingress that drifted from a matching origin (#27961)

Julius Marminge0c863d63ec82
diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts
index ea3e8d3e2b2..3e360d8f621 100644
--- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts
+++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts
@@ -4,6 +4,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
import { describe, expect, it } from "@effect/vitest";
import * as Alchemy from "alchemy";
import * as Cloudflare from "@alchemy.run/cloudflare";
+import * as ZeroTrust from "@distilled.cloud/cloudflare/zero-trust";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
@@ -36,10 +37,25 @@ const config = RelayConfiguration.RelayConfiguration.of({
});
interface TunnelCall {
- readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete";
+ readonly operation:
+ | "get"
+ | "list"
+ | "create"
+ | "putConfiguration"
+ | "getConfiguration"
+ | "getToken"
+ | "delete";
readonly input: unknown;
}
+interface TunnelConfiguration {
+ readonly ingress: ReadonlyArray<{
+ readonly hostname?: string;
+ readonly path?: string;
+ readonly service: string;
+ }>;
+}
+
interface DnsCall {
readonly operation: "listRecords" | "createRecord" | "updateRecord" | "deleteRecord";
readonly input: unknown;
@@ -88,6 +104,11 @@ function makeTunnelClient(calls: TunnelCall[] = []) {
Effect.sync(() => {
calls.push({ operation: "putConfiguration", input: { tunnelId, tunnelConfig } });
}),
+ getConfiguration: (tunnelId) =>
+ Effect.sync(() => {
+ calls.push({ operation: "getConfiguration", input: tunnelId });
+ return { ingress: [] };
+ }),
getToken: (tunnelId) =>
Effect.sync(() => {
calls.push({ operation: "getToken", input: tunnelId });
@@ -100,7 +121,10 @@ function makeTunnelClient(calls: TunnelCall[] = []) {
});
}
-function makePersistentTunnelClient(calls: TunnelCall[] = []) {
+function makePersistentTunnelClient(
+ calls: TunnelCall[] = [],
+ configuration: { current: TunnelConfiguration | null } = { current: null },
+) {
let tunnel: { readonly id: string; readonly name: string } | null = null;
return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
get: (tunnelId) =>
@@ -134,6 +158,12 @@ function makePersistentTunnelClient(calls: TunnelCall[] = []) {
putConfiguration: (tunnelId, tunnelConfig) =>
Effect.sync(() => {
calls.push({ operation: "putConfiguration", input: { tunnelId, tunnelConfig } });
+ configuration.current = tunnelConfig;
+ }),
+ getConfiguration: (tunnelId: string) =>
+ Effect.sync(() => {
+ calls.push({ operation: "getConfiguration", input: tunnelId });
+ return configuration.current ?? { ingress: [] };
}),
getToken: (tunnelId) =>
Effect.sync(() => {
@@ -787,6 +817,44 @@ describe("ManagedEndpointProvider", () => {
});
});
+ it.effect("reads a tunnel without remote configuration as having no ingress", () => {
+ const tunnelClient = {
+ getConfiguration: () =>
+ Effect.fail(
+ new ZeroTrust.TunnelConfigurationNotFound({
+ code: 1055,
+ message: "Tunnel configuration not found",
+ }),
+ ),
+ } as unknown as Cloudflare.Tunnel.ReadWriteTunnelClient;
+ const layer = ManagedEndpointProvider.layerCloudflareBindings(
+ tunnelClient,
+ {} as Cloudflare.DNS.ReadWriteDnsClient,
+ {} as Alchemy.BaseRuntimeContext,
+ ).pipe(
+ Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
+ Layer.provide(RelayConfiguration.layer(config)),
+ Layer.provide(
+ Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()),
+ ),
+ Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())),
+ Layer.provide(
+ Layer.succeed(
+ CloudflareApiPause.CloudflareApiPause,
+ CloudflareApiPause.CloudflareApiPause.of({
+ pausedUntilMillis: Effect.succeed(0),
+ pauseUntil: () => Effect.void,
+ }),
+ ),
+ ),
+ );
+
+ return Effect.gen(function* () {
+ const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
+ expect(yield* tunnels.getConfiguration("tunnel-id")).toEqual({ ingress: [] });
+ }).pipe(Effect.provide(layer));
+ });
+
it.effect("does not hide non-not-found checkpoint update failures", () => {
const dnsCalls: DnsCall[] = [];
const failure = new ManagedEndpointProvider.ManagedEndpointDnsClientError({
@@ -1139,6 +1207,72 @@ describe("ManagedEndpointProvider", () => {
}).pipe(Effect.provide(layer));
});
+ it.effect("restores Cloudflare ingress that drifted from the recorded origin", () => {
+ const tunnelCalls: TunnelCall[] = [];
+ const configuration: { current: TunnelConfiguration | null } = { current: null };
+ const layer = layerProvider(makePersistentTunnelClient(tunnelCalls, configuration));
+
+ return Effect.gen(function* () {
+ const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
+ const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
+ const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
+ const provisioned = yield* provider.provision({ ...key, origin });
+ const reconcile = provider.reconcileOrigin({
+ ...key,
+ tunnelId: provisioned.runtime.tunnelId!,
+ origin,
+ endpoint: provisioned.endpoint,
+ });
+ tunnelCalls.length = 0;
+
+ expect(yield* reconcile).toBe("ready");
+ expect(tunnelCalls.map((call) => call.operation)).toEqual(["get", "getConfiguration"]);
+
+ configuration.current = {
+ ingress: [
+ { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:4711" },
+ { service: "http_status:404" },
+ ],
+ };
+ expect(yield* reconcile).toBe("ready");
+ expect(configuration.current).toEqual({
+ ingress: [
+ { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:3773" },
+ { service: "http_status:404" },
+ ],
+ });
+
+ configuration.current = {
+ ingress: [
+ { hostname: "*.t3code.test", service: "http://127.0.0.1:4711" },
+ { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:3773" },
+ { service: "http_status:404" },
+ ],
+ };
+ expect(yield* reconcile).toBe("ready");
+ expect(configuration.current.ingress[0]).toEqual({
+ hostname: expectedManagedHostname("env_ABC"),
+ service: "http://127.0.0.1:3773",
+ });
+
+ configuration.current = {
+ ingress: [
+ {
+ hostname: expectedManagedHostname("env_ABC"),
+ path: "/api",
+ service: "http://127.0.0.1:3773",
+ },
+ { service: "http_status:404" },
+ ],
+ };
+ expect(yield* reconcile).toBe("ready");
+ expect(configuration.current.ingress[0]).toEqual({
+ hostname: expectedManagedHostname("env_ABC"),
+ service: "http://127.0.0.1:3773",
+ });
+ }).pipe(Effect.provide(layer));
+ });
+
it.effect("rejects an active endpoint that does not match the allocation hostname", () => {
const layer = layerProvider(makePersistentTunnelClient());
diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts
index 0f8a99ceb44..188ad0e7b71 100644
--- a/infra/relay/src/environments/ManagedEndpointProvider.ts
+++ b/infra/relay/src/environments/ManagedEndpointProvider.ts
@@ -216,6 +216,7 @@ const ManagedEndpointTunnelClientOperation = Schema.Literals([
"list",
"create",
"put-configuration",
+ "get-configuration",
"get-token",
"delete",
]);
@@ -265,6 +266,16 @@ export class ManagedEndpointTunnelClient extends Context.Service<
}>;
},
) => Effect.Effect<unknown, ManagedEndpointTunnelClientError>;
+ readonly getConfiguration: (tunnelId: string) => Effect.Effect<
+ {
+ readonly ingress: ReadonlyArray<{
+ readonly hostname?: string | null;
+ readonly path?: string | null;
+ readonly service: string;
+ }>;
+ },
+ ManagedEndpointTunnelClientError
+ >;
readonly getToken: (
tunnelId: string,
) => Effect.Effect<string, ManagedEndpointTunnelClientError>;
@@ -756,32 +767,53 @@ const make = Effect.gen(function* () {
hostname: allocation.hostname,
});
}
+ const verifyTunnel = <A>(read: Effect.Effect<A, ManagedEndpointTunnelClientError>) =>
+ read.pipe(
+ Effect.asSome,
+ Effect.catchTags({
+ ManagedEndpointTunnelClientError: (cause) =>
+ isManagedEndpointNotFound(cause.cause)
+ ? Effect.succeedNone
+ : Effect.fail(
+ new ManagedEndpointProvisioningFailed({
+ userId: input.userId,
+ environmentId: input.environmentId,
+ stage: "verify-tunnel",
+ tunnelId: input.tunnelId,
+ cause,
+ }),
+ ),
+ }),
+ );
// Released records retain tunnel ids; verify Cloudflare still has the tunnel before reconnecting.
- const recorded = yield* tunnels.get(input.tunnelId).pipe(
- Effect.asSome,
- Effect.catchTags({
- ManagedEndpointTunnelClientError: (cause) =>
- isManagedEndpointNotFound(cause.cause)
- ? Effect.succeedNone
- : Effect.fail(
- new ManagedEndpointProvisioningFailed({
- userId: input.userId,
- environmentId: input.environmentId,
- stage: "verify-tunnel",
- tunnelId: input.tunnelId,
- cause,
- }),
- ),
- }),
- );
+ const recorded = yield* verifyTunnel(tunnels.get(input.tunnelId));
if (Option.isNone(recorded) || recorded.value.deletedAt) {
return "recovery_required";
}
+ const originService = formatOriginService(input.origin);
if (
allocation.origin?.localHttpHost === input.origin.localHttpHost &&
allocation.origin.localHttpPort === input.origin.localHttpPort
) {
- return "ready";
+ // Live ingress can drift from the recorded origin; only a matching rule skips the rewrite.
+ const configuration = yield* verifyTunnel(tunnels.getConfiguration(input.tunnelId));
+ if (Option.isNone(configuration)) {
+ return "recovery_required";
+ }
+ const hostname = normalizeHostname(allocation.hostname);
+ // Cloudflare routes by the first matching rule, so an earlier wildcard or catch-all wins.
+ const rule = configuration.value.ingress.find((entry) => {
+ if (entry.hostname == null) return true;
+ const candidate = normalizeHostname(entry.hostname);
+ return (
+ candidate === hostname ||
+ (candidate.startsWith("*.") && hostname.endsWith(candidate.slice(1)))
+ );
+ });
+ // A path-restricted rule leaves other paths to later rules, so it cannot prove routing.
+ if (rule?.service === originService && !rule.path) {
+ return "ready";
+ }
}
const updated = yield* allocations
@@ -795,10 +827,7 @@ const make = Effect.gen(function* () {
tunnels
.putConfiguration(input.tunnelId, {
ingress: [
- {
- hostname: allocation.hostname,
- service: formatOriginService(input.origin),
- },
+ { hostname: allocation.hostname, service: originService },
{ service: "http_status:404" },
],
})
@@ -1606,6 +1635,22 @@ export const layerCloudflareBindings = (
}),
),
),
+ getConfiguration: (tunnelId) =>
+ gate(tunnelClient.getConfiguration(tunnelId)).pipe(
+ Effect.map(({ config }) => ({ ingress: config?.ingress ?? [] })),
+ // Cloudflare reports a tunnel without remote configuration as code 1055, not 404.
+ Effect.catchTags({
+ TunnelConfigurationNotFound: () => Effect.succeed({ ingress: [] }),
+ }),
+ Effect.mapError(
+ (cause) =>
+ new ManagedEndpointTunnelClientError({
+ operation: "get-configuration",
+ tunnelId,
+ cause,
+ }),
+ ),
+ ),
getToken: (tunnelId) =>
gate(tunnelClient.getToken(tunnelId)).pipe(
Effect.mapError(
diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts
index d2a9bb7f6c3..5ce24e6b7e5 100644
--- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts
+++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts
@@ -140,6 +140,7 @@ function harness(input?: {
}),
create: () => Effect.die("unused"),
putConfiguration: () => Effect.die("unused"),
+ getConfiguration: () => Effect.die("unused"),
getToken: () => Effect.die("unused"),
delete: (tunnelId) =>
input?.failAllDeletes === true ||