Multiplayer agent harness for work.
porter: serve published apps in-cluster by default (#1679)
Jack Decker2fdc93c2c4ff
diff --git a/docs/porter.md b/docs/porter.mdindex 2f85cb8f1..9b7360471 100644--- a/docs/porter.md+++ b/docs/porter.md@@ -19,10 +19,10 @@ PORTER_DEPLOY_CLUSTER_ID=<cluster id>PORTER_SANDBOX_IMAGE=ghcr.io/porter-dev/qm-sandbox:latestPORTER_DEPLOY_RUNNER_IMAGE=ghcr.io/porter-dev/qm-app-runner:latest# PORTER_DEPLOY_URL= # only when the API host is not https://dashboard.porter.run-# PORTER_DEPLOY_APPS_DOMAIN= # optional; the cluster names apps itself (see below)-# PORTER_DEPLOY_VISIBILITY=public # public Porter ingress serves apps to ANYONE with the- # URL, bypassing qm's sign-in gate — leave private and- # use /d/<app>/ or DEPLOY_APPS_DOMAIN unless that is the intent+# PORTER_DEPLOY_APPS_DOMAIN= # optional; puts apps on sandbox ingress (see below)+# PORTER_DEPLOY_VISIBILITY=internal # default: no ingress host, core reaches apps in-cluster;+ # private/public put apps on sandbox ingress, and public+ # serves them to ANYONE with the URL, bypassing qm's gate```## Onboarding checklist@@ -195,7 +195,15 @@ skips the gate. `PORTER_DEPLOY_APPS_DOMAIN` is the separate, ungated mechanism:registers each app's domain on Porter's own sandbox ingress and is deliberately NOTdefaulted from `DEPLOY_APPS_DOMAIN`.-Published apps get their address from the cluster, not from qm. Declare the sandbox load+By default published apps get no sandbox ingress host at all. Core reaches each app at the+cluster-internal address the cluster reports, so `/d/<app>/` and `DEPLOY_APPS_DOMAIN` work,+no Porter hostname skips qm's gate, and the cluster needs no sandbox ingress. This needs core+running in the same cluster as the sandboxes and sandbox-api 0.1.58 or newer. Apps published+before an upgrade keep their ingress host until they are redeployed. The rest of this section+applies only when `PORTER_DEPLOY_VISIBILITY` is `private` or `public`, or+`PORTER_DEPLOY_APPS_DOMAIN` is set (which implies `private`).++With ingress, published apps get their address from the cluster, not from qm. Declare the sandbox loadbalancer with a root domain at cluster creation and **Porter names every published appitself**: `<app>.<root domain>`, served on a wildcard Let's Encrypt certificate that Porteralso obtains. `PORTER_DEPLOY_APPS_DOMAIN` is then only for choosing a different name — leave@@ -333,8 +341,8 @@ PORTER_DEPLOY_APPS_DOMAIN=apps.example.com # optional; overrides the assigned```Set it only to choose the name yourself; the wildcard record it relies on is the one Porter-already created. Deployments are **private** by default, matching the other providers;-`PORTER_DEPLOY_VISIBILITY=public` opts a deployment's domain into public ingress. Note the+already created. `PORTER_DEPLOY_VISIBILITY=private` serves apps on the private sandbox+ingress, and `public` opts them into public ingress. Note thecluster-assigned hostname is Porter ingress only — qm's sign-on gate, request-access flow,and live editing need `DEPLOY_APPS_DOMAIN` (a domain you control, not a shared platformdomain like `onporter.run`).@@ -386,7 +394,7 @@ grep for that line rather than assuming the container's own metadata block is in| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) || `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement (see the constraints below) || `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |-| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private |+| `PORTER_DEPLOY_VISIBILITY` | `internal` (default) serves apps in-cluster only; `private`/`public` put them on sandbox ingress || `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |To QA a branch against a real Porter cluster before deploying it, the dev instance takesdiff --git a/scripts/dev/lib/sandbox.ts b/scripts/dev/lib/sandbox.tsindex bf10c1c14..9d4843274 100644--- a/scripts/dev/lib/sandbox.ts+++ b/scripts/dev/lib/sandbox.ts@@ -212,6 +212,7 @@ export async function resolveSandbox(opts: {PORTER_DEPLOY_PROJECT_ID: projectId,PORTER_DEPLOY_CLUSTER_ID: clusterId,PORTER_SANDBOX_NAME_PREFIX: opts.baseEnv.PORTER_SANDBOX_NAME_PREFIX || "qmdev",+ PORTER_DEPLOY_VISIBILITY: opts.baseEnv.PORTER_DEPLOY_VISIBILITY || "private",};if (opts.baseEnv.PORTER_DEPLOY_URL) env.PORTER_DEPLOY_URL = opts.baseEnv.PORTER_DEPLOY_URL;if (opts.baseEnv.PORTER_SANDBOX_IMAGE) env.PORTER_SANDBOX_IMAGE = opts.baseEnv.PORTER_SANDBOX_IMAGE;diff --git a/src/config.ts b/src/config.tsindex 2f9a17544..e4b7e4043 100644--- a/src/config.ts+++ b/src/config.ts@@ -553,7 +553,7 @@ interface PorterDeployEnv {baseUrl?: string;runnerImage?: string;appsDomain?: string;- visibility?: "public" | "private";+ visibility?: "public" | "private" | "internal";namePrefix?: string;ttlSec?: number;}@@ -574,9 +574,9 @@ const porterLocatorPresent = (env: NodeJS.ProcessEnv): boolean =>function porterDeployVisibilityStrict(value: string | undefined): PorterDeployEnv["visibility"] {if (value === undefined || value.trim() === "") return undefined;const visibility = value.trim();- if (visibility === "public" || visibility === "private") return visibility;+ if (visibility === "public" || visibility === "private" || visibility === "internal") return visibility;throw new Error(- `PORTER_DEPLOY_VISIBILITY=${JSON.stringify(value)} is not recognized — use public or private, or unset it.`,+ `PORTER_DEPLOY_VISIBILITY=${JSON.stringify(value)} is not recognized — use public, private, or internal, or unset it.`,);}@@ -584,6 +584,11 @@ function porterDeployEnv(env: NodeJS.ProcessEnv): PorterDeployEnv {const token = env.PORTER_DEPLOY_API_TOKEN;const baseUrl = porterApiBaseUrl(env);const visibility = porterDeployVisibilityStrict(env.PORTER_DEPLOY_VISIBILITY);+ if (visibility === "internal" && env.PORTER_DEPLOY_APPS_DOMAIN) {+ throw new Error(+ "PORTER_DEPLOY_VISIBILITY=internal serves apps inside the cluster only, so it takes no PORTER_DEPLOY_APPS_DOMAIN — unset one of them.",+ );+ }const ttlSec = numEnvStrict("PORTER_DEPLOY_TTL_SEC", env.PORTER_DEPLOY_TTL_SEC);const runnerImage = env.PORTER_DEPLOY_RUNNER_IMAGE ?? env.PORTER_SANDBOX_IMAGE;return {@@ -1235,7 +1240,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {}if (env.DEPLOY_PROVIDER === "porter" && !env.PORTER_DEPLOY_APPS_DOMAIN && !env.DEPLOY_APPS_DOMAIN) {console.warn(- "[config] DEPLOY_PROVIDER=porter without an apps domain — published apps use hostnames assigned by the cluster and are reachable signed-in at /d/<app>/; set DEPLOY_APPS_DOMAIN to a domain you control to serve each app on its own subdomain.",+ "[config] DEPLOY_PROVIDER=porter without an apps domain — published apps are reachable signed-in at /d/<app>/ only; set DEPLOY_APPS_DOMAIN to a domain you control to serve each app on its own subdomain.",);}for (const [selected, label] of [diff --git a/src/deploy/porter-deploy-provider.ts b/src/deploy/porter-deploy-provider.tsindex b2325be40..0eed98fde 100644--- a/src/deploy/porter-deploy-provider.ts+++ b/src/deploy/porter-deploy-provider.ts@@ -43,6 +43,7 @@ export interface StoredPorterDeployBody {sandboxId: string;name: string;host: string;+ port?: number;createdAtMs: number;}@@ -51,7 +52,7 @@ export interface PorterDeployProviderOptions {token?: string;baseUrl?: string;runnerImage?: string;- visibility?: "public" | "private";+ visibility?: "public" | "private" | "internal";namePrefix?: string;ttlSec?: number;appPort?: number;@@ -65,7 +66,7 @@ export interface PorterDeployProviderOptions {export function createPorterDeployProvider(opts: PorterDeployProviderOptions): DeployProvider {const appsDomain = opts.appsDomain;const image = opts.runnerImage ?? DEFAULT_RUNNER_IMAGE;- const visibility = opts.visibility ?? "private";+ const visibility = opts.visibility ?? (appsDomain ? "private" : "internal");const prefix = opts.namePrefix ?? "qm";const appPort = opts.appPort ?? APP_PORT_DEFAULT;const readyWindowSec = opts.readyWindowSec ?? APP_READY_WINDOW_SEC_DEFAULT;@@ -88,13 +89,11 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): Dconst baseName = (d: Deployment): string => `${prefix}-app-${d.id.slice(0, 12).toLowerCase()}`;const volumeName = (d: Deployment): string => `${baseName(d)}-data`;const domainOf = (d: Deployment): string | undefined =>- appsDomain ? `${porterDnsLabel(d.name ?? d.id)}.${appsDomain}` : undefined;- const endpointOf = (host: string): DeployEndpoint => ({- host,- port: ENDPOINT_PORT,- tls: true,- publicUrl: `https://${host}/`,- });+ appsDomain && visibility !== "internal" ? `${porterDnsLabel(d.name ?? d.id)}.${appsDomain}` : undefined;+ const endpointOf = (body: { host: string; port?: number }): DeployEndpoint =>+ body.port+ ? { host: body.host, port: body.port }+ : { host: body.host, port: ENDPOINT_PORT, tls: true, publicUrl: `https://${body.host}/` };const serialized = <T>(d: Deployment, fn: () => Promise<T>): Promise<T> =>queue(d.id, () => advisoryLock.withLock(`porter-deploy:${d.id}`, fn));@@ -137,6 +136,28 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D});}+ async function addressOf(d: Deployment, sb: PorterSandboxLike): Promise<{ host: string; port?: number }> {+ const status = await sb.refresh();+ if (visibility === "internal") {+ const address = status.internal_address ?? "";+ const split = address.lastIndexOf(":");+ const port = Number(address.slice(split + 1));+ if (split <= 0 || !Number.isInteger(port) || port <= 0) {+ throw new Error(+ `porter deploy ${d.id}: the cluster reported no internal address for the app (${JSON.stringify(address)}) — internal visibility needs sandbox-api 0.1.58 or newer`,+ );+ }+ return { host: address.slice(0, split), port };+ }+ const host = status.host || domainOf(d);+ if (!host) {+ throw new Error(+ `porter deploy ${d.id}: the cluster named no host for the app — its sandbox ingress has no root domain, so set PORTER_DEPLOY_APPS_DOMAIN to a domain whose wildcard resolves to that ingress (docs/porter.md)`,+ );+ }+ return { host };+ }+async function liveStored(d: Deployment): Promise<StoredPorterDeployBody | null> {const stored = await store.get(d.id);if (!stored) return null;@@ -167,11 +188,14 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): Dtags: { [KIND_TAG]: "app", [DEPLOY_TAG]: d.id },env: appEnv(version),volume_mounts: { [DATA_DIR]: volumeId },- networking: [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],+ networking:+ visibility === "internal"+ ? [{ port: appPort, internal: true }]+ : [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],...(opts.ttlSec ? { ttl_seconds: opts.ttlSec } : {}),}).catch((e) => {- if (errMessage(e).includes("sandbox ingress")) {+ if (visibility !== "internal" && errMessage(e).includes("sandbox ingress")) {throw new Error(`porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster (docs/porter.md), then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,{ cause: e },@@ -181,16 +205,11 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D});try {await waitPorterRunning(name, sb);- const host = (await sb.refresh()).host || domainOf(d);- if (!host) {- throw new Error(- `porter deploy ${d.id}: the cluster named no host for the app — its sandbox ingress has no root domain, so set PORTER_DEPLOY_APPS_DOMAIN to a domain whose wildcard resolves to that ingress (docs/porter.md)`,- );- }+ const address = await addressOf(d, sb);await materialize(sb.id, version);await startApp(sb.id, version);- await store.put(d.id, { deploymentId: d.id, sandboxId: sb.id, name, host, createdAtMs: Date.now() });- return endpointOf(host);+ await store.put(d.id, { deploymentId: d.id, sandboxId: sb.id, name, ...address, createdAtMs: Date.now() });+ return endpointOf(address);} catch (e) {await sb.terminate().catch((err) => swallow("porter-deploy: abandon failed body", err));throw e;@@ -205,7 +224,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): DresolveCache.delete(d.id);return null;}- const endpoint = endpointOf(stored.host);+ const endpoint = endpointOf(stored);if (resolveCacheMs > 0) resolveCache.set(d.id, endpoint);return endpoint;},diff --git a/src/sandbox/porter-client.ts b/src/sandbox/porter-client.tsindex bf4a6709f..f3587a1bd 100644--- a/src/sandbox/porter-client.ts+++ b/src/sandbox/porter-client.ts@@ -19,7 +19,7 @@ export interface PorterSandboxLike {readonly id: string;readonly phase: string | null;readonly tags: Record<string, string> | null;- refresh(): Promise<{ name: string; host?: string }>;+ refresh(): Promise<{ name: string; host?: string; internal_address?: string }>;terminate(): Promise<void>;}@@ -31,7 +31,11 @@ export interface PorterSandboxSpec {env?: Record<string, string>;volume_mounts?: Record<string, string>;egress?: { allowed_destinations: string[] };- networking?: Array<{ port: number; domains?: Array<{ domain?: string; visibility?: "public" | "private" }> }>;+ networking?: Array<{+ port: number;+ internal?: boolean;+ domains?: Array<{ domain?: string; visibility?: "public" | "private" }>;+ }>;ttl_seconds?: number;}diff --git a/test/config.test.ts b/test/config.test.tsindex 4c54e5177..8ee32d3cd 100644--- a/test/config.test.ts+++ b/test/config.test.ts@@ -682,6 +682,21 @@ test("DEPLOY_PROVIDER=porter refuses to boot without a cluster and tolerates a m);});+test("PORTER_DEPLOY_VISIBILITY=internal is accepted and refuses an apps domain", () => {+ const cluster = {+ DEPLOY_PROVIDER: "porter",+ PORTER_DEPLOY_API_TOKEN: "tok",+ PORTER_DEPLOY_PROJECT_ID: "7",+ PORTER_DEPLOY_CLUSTER_ID: "9",+ PORTER_DEPLOY_VISIBILITY: "internal",+ };+ assert.equal(loadConfig(cluster).porterDeploy.visibility, "internal");+ assert.throws(+ () => loadConfig({ ...cluster, PORTER_DEPLOY_APPS_DOMAIN: "apps.example.com" }),+ /PORTER_DEPLOY_APPS_DOMAIN/,+ );+});+test("SANDBOX_BACKEND=porter locates the API and shares the deploy provider's token", () => {assert.throws(() => loadConfig({ SANDBOX_BACKEND: "porter", PORTER_DEPLOY_API_TOKEN: "tok" }),diff --git a/test/porter-deploy-provider.test.ts b/test/porter-deploy-provider.test.tsindex a4697e93c..0e21634d5 100644--- a/test/porter-deploy-provider.test.ts+++ b/test/porter-deploy-provider.test.ts@@ -122,6 +122,23 @@ test("explicit public visibility opts the domain out of the private default", as]);});+test("without an apps domain the app defaults to its cluster-internal address", async () => {+ const bare = createPorterDeployProvider({+ namePrefix: "qmt",+ appPort,+ readyWindowSec: 10,+ client: fake.client,+ store,+ });+ const d = deployment("dep-1c", "hidden");+ const endpoint = await bare.apply(d, version({ "server.js": SERVER }, "node server.js"));+ assert.deepEqual(endpoint, { host: "127.0.0.1", port: appPort });+ assert.deepEqual(fake.bodies()[0]!.networking, [{ port: appPort, internal: true }]);+ assert.equal(fake.bodies()[0]!.host, "");+ assert.deepEqual(await bare.resolveEndpoint!(d, version({}, "true")), endpoint);+ assert.match(await (await fetch(`http://${endpoint.host}:${endpoint.port}/`)).text(), /^hello from .*-app v\?$/);+});+test("redeploy keeps the domain and the /data volume and retires the old body", async () => {const d = deployment("dep-2", "keeper");await provider.apply(d, version({ "server.js": SERVER }, "node server.js", { env: { APP_VERSION: "1" } }));@@ -249,8 +266,9 @@ test("a body that vanished between list and terminate does not break destroy orassert.match(await fetchText("/"), /^hello from/);});-test("without an apps domain the cluster names the host itself", async () => {+test("without an apps domain a private app gets the host the cluster names", async () => {const bare = createPorterDeployProvider({+ visibility: "private",namePrefix: "qmt",appPort,readyWindowSec: 10,@@ -269,6 +287,7 @@ test("a cluster that names no host fails the deploy and leaves no body behind",fake.cleanup();fake = installFakePorter({ assignHost: false });const bare = createPorterDeployProvider({+ visibility: "private",namePrefix: "qmt",appPort,readyWindowSec: 10,diff --git a/test/support/fake-porter.ts b/test/support/fake-porter.tsindex 48ee0937c..987e848eb 100644--- a/test/support/fake-porter.ts+++ b/test/support/fake-porter.ts@@ -124,7 +124,9 @@ export function installFakePorter(opts: FakePorterOptions = {}): FakePorter {},async refresh() {known = true;- return { name, host: observe(bodies.get(name))?.host ?? "" };+ const cur = observe(bodies.get(name));+ const internal = cur?.networking?.[0]?.internal ? cur.networking[0].port : undefined;+ return { name, host: cur?.host ?? "", ...(internal ? { internal_address: `127.0.0.1:${internal}` } : {}) };},async terminate() {const cur = bodies.get(name);@@ -158,7 +160,7 @@ export function installFakePorter(opts: FakePorterOptions = {}): FakePorter {mkdirSync(app, { recursive: true });const exposed = spec.networking?.[0];const named = opts.assignHost === false ? undefined : `${spec.name}.fake.test`;- const host = exposed ? (exposed.domains?.[0]?.domain ?? named ?? "") : "";+ const host = exposed && !exposed.internal ? (exposed.domains?.[0]?.domain ?? named ?? "") : "";bodies.set(spec.name!, {id: `sb-${++bodySeq}`,phase: "running",