Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

porter: serve published apps in-cluster by default (#1679)

Jack Decker2fdc93c2c4ff
diff --git a/docs/porter.md b/docs/porter.md
index 2f85cb8f1..9b7360471 100644
--- a/docs/porter.md
+++ b/docs/porter.md
@@ -19,10 +19,10 @@ PORTER_DEPLOY_CLUSTER_ID=<cluster id>
PORTER_SANDBOX_IMAGE=ghcr.io/porter-dev/qm-sandbox:latest
PORTER_DEPLOY_RUNNER_IMAGE=ghcr.io/porter-dev/qm-app-runner:latest
# PORTER_DEPLOY_URL= # only when the API host is not https://dashboard.porter.run
-# PORTER_DEPLOY_APPS_DOMAIN= # optional; the cluster names apps itself (see below)
-# PORTER_DEPLOY_VISIBILITY=public # public Porter ingress serves apps to ANYONE with the
- # URL, bypassing qm's sign-in gate — leave private and
- # use /d/<app>/ or DEPLOY_APPS_DOMAIN unless that is the intent
+# PORTER_DEPLOY_APPS_DOMAIN= # optional; puts apps on sandbox ingress (see below)
+# PORTER_DEPLOY_VISIBILITY=internal # default: no ingress host, core reaches apps in-cluster;
+ # private/public put apps on sandbox ingress, and public
+ # serves them to ANYONE with the URL, bypassing qm's gate
```
## Onboarding checklist
@@ -195,7 +195,15 @@ skips the gate. `PORTER_DEPLOY_APPS_DOMAIN` is the separate, ungated mechanism:
registers each app's domain on Porter's own sandbox ingress and is deliberately NOT
defaulted from `DEPLOY_APPS_DOMAIN`.
-Published apps get their address from the cluster, not from qm. Declare the sandbox load
+By default published apps get no sandbox ingress host at all. Core reaches each app at the
+cluster-internal address the cluster reports, so `/d/<app>/` and `DEPLOY_APPS_DOMAIN` work,
+no Porter hostname skips qm's gate, and the cluster needs no sandbox ingress. This needs core
+running in the same cluster as the sandboxes and sandbox-api 0.1.58 or newer. Apps published
+before an upgrade keep their ingress host until they are redeployed. The rest of this section
+applies only when `PORTER_DEPLOY_VISIBILITY` is `private` or `public`, or
+`PORTER_DEPLOY_APPS_DOMAIN` is set (which implies `private`).
+
+With ingress, published apps get their address from the cluster, not from qm. Declare the sandbox load
balancer with a root domain at cluster creation and **Porter names every published app
itself**: `<app>.<root domain>`, served on a wildcard Let's Encrypt certificate that Porter
also obtains. `PORTER_DEPLOY_APPS_DOMAIN` is then only for choosing a different name — leave
@@ -333,8 +341,8 @@ PORTER_DEPLOY_APPS_DOMAIN=apps.example.com # optional; overrides the assigned
```
Set it only to choose the name yourself; the wildcard record it relies on is the one Porter
-already created. Deployments are **private** by default, matching the other providers;
-`PORTER_DEPLOY_VISIBILITY=public` opts a deployment's domain into public ingress. Note the
+already created. `PORTER_DEPLOY_VISIBILITY=private` serves apps on the private sandbox
+ingress, and `public` opts them into public ingress. Note the
cluster-assigned hostname is Porter ingress only — qm's sign-on gate, request-access flow,
and live editing need `DEPLOY_APPS_DOMAIN` (a domain you control, not a shared platform
domain like `onporter.run`).
@@ -386,7 +394,7 @@ grep for that line rather than assuming the container's own metadata block is in
| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) |
| `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement (see the constraints below) |
| `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |
-| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private |
+| `PORTER_DEPLOY_VISIBILITY` | `internal` (default) serves apps in-cluster only; `private`/`public` put them on sandbox ingress |
| `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |
To QA a branch against a real Porter cluster before deploying it, the dev instance takes
diff --git a/scripts/dev/lib/sandbox.ts b/scripts/dev/lib/sandbox.ts
index bf10c1c14..9d4843274 100644
--- a/scripts/dev/lib/sandbox.ts
+++ b/scripts/dev/lib/sandbox.ts
@@ -212,6 +212,7 @@ export async function resolveSandbox(opts: {
PORTER_DEPLOY_PROJECT_ID: projectId,
PORTER_DEPLOY_CLUSTER_ID: clusterId,
PORTER_SANDBOX_NAME_PREFIX: opts.baseEnv.PORTER_SANDBOX_NAME_PREFIX || "qmdev",
+ PORTER_DEPLOY_VISIBILITY: opts.baseEnv.PORTER_DEPLOY_VISIBILITY || "private",
};
if (opts.baseEnv.PORTER_DEPLOY_URL) env.PORTER_DEPLOY_URL = opts.baseEnv.PORTER_DEPLOY_URL;
if (opts.baseEnv.PORTER_SANDBOX_IMAGE) env.PORTER_SANDBOX_IMAGE = opts.baseEnv.PORTER_SANDBOX_IMAGE;
diff --git a/src/config.ts b/src/config.ts
index 2f9a17544..e4b7e4043 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -553,7 +553,7 @@ interface PorterDeployEnv {
baseUrl?: string;
runnerImage?: string;
appsDomain?: string;
- visibility?: "public" | "private";
+ visibility?: "public" | "private" | "internal";
namePrefix?: string;
ttlSec?: number;
}
@@ -574,9 +574,9 @@ const porterLocatorPresent = (env: NodeJS.ProcessEnv): boolean =>
function porterDeployVisibilityStrict(value: string | undefined): PorterDeployEnv["visibility"] {
if (value === undefined || value.trim() === "") return undefined;
const visibility = value.trim();
- if (visibility === "public" || visibility === "private") return visibility;
+ if (visibility === "public" || visibility === "private" || visibility === "internal") return visibility;
throw new Error(
- `PORTER_DEPLOY_VISIBILITY=${JSON.stringify(value)} is not recognized — use public or private, or unset it.`,
+ `PORTER_DEPLOY_VISIBILITY=${JSON.stringify(value)} is not recognized — use public, private, or internal, or unset it.`,
);
}
@@ -584,6 +584,11 @@ function porterDeployEnv(env: NodeJS.ProcessEnv): PorterDeployEnv {
const token = env.PORTER_DEPLOY_API_TOKEN;
const baseUrl = porterApiBaseUrl(env);
const visibility = porterDeployVisibilityStrict(env.PORTER_DEPLOY_VISIBILITY);
+ if (visibility === "internal" && env.PORTER_DEPLOY_APPS_DOMAIN) {
+ throw new Error(
+ "PORTER_DEPLOY_VISIBILITY=internal serves apps inside the cluster only, so it takes no PORTER_DEPLOY_APPS_DOMAIN — unset one of them.",
+ );
+ }
const ttlSec = numEnvStrict("PORTER_DEPLOY_TTL_SEC", env.PORTER_DEPLOY_TTL_SEC);
const runnerImage = env.PORTER_DEPLOY_RUNNER_IMAGE ?? env.PORTER_SANDBOX_IMAGE;
return {
@@ -1235,7 +1240,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
}
if (env.DEPLOY_PROVIDER === "porter" && !env.PORTER_DEPLOY_APPS_DOMAIN && !env.DEPLOY_APPS_DOMAIN) {
console.warn(
- "[config] DEPLOY_PROVIDER=porter without an apps domain — published apps use hostnames assigned by the cluster and are reachable signed-in at /d/<app>/; set DEPLOY_APPS_DOMAIN to a domain you control to serve each app on its own subdomain.",
+ "[config] DEPLOY_PROVIDER=porter without an apps domain — published apps are reachable signed-in at /d/<app>/ only; set DEPLOY_APPS_DOMAIN to a domain you control to serve each app on its own subdomain.",
);
}
for (const [selected, label] of [
diff --git a/src/deploy/porter-deploy-provider.ts b/src/deploy/porter-deploy-provider.ts
index b2325be40..0eed98fde 100644
--- a/src/deploy/porter-deploy-provider.ts
+++ b/src/deploy/porter-deploy-provider.ts
@@ -43,6 +43,7 @@ export interface StoredPorterDeployBody {
sandboxId: string;
name: string;
host: string;
+ port?: number;
createdAtMs: number;
}
@@ -51,7 +52,7 @@ export interface PorterDeployProviderOptions {
token?: string;
baseUrl?: string;
runnerImage?: string;
- visibility?: "public" | "private";
+ visibility?: "public" | "private" | "internal";
namePrefix?: string;
ttlSec?: number;
appPort?: number;
@@ -65,7 +66,7 @@ export interface PorterDeployProviderOptions {
export function createPorterDeployProvider(opts: PorterDeployProviderOptions): DeployProvider {
const appsDomain = opts.appsDomain;
const image = opts.runnerImage ?? DEFAULT_RUNNER_IMAGE;
- const visibility = opts.visibility ?? "private";
+ const visibility = opts.visibility ?? (appsDomain ? "private" : "internal");
const prefix = opts.namePrefix ?? "qm";
const appPort = opts.appPort ?? APP_PORT_DEFAULT;
const readyWindowSec = opts.readyWindowSec ?? APP_READY_WINDOW_SEC_DEFAULT;
@@ -88,13 +89,11 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
const baseName = (d: Deployment): string => `${prefix}-app-${d.id.slice(0, 12).toLowerCase()}`;
const volumeName = (d: Deployment): string => `${baseName(d)}-data`;
const domainOf = (d: Deployment): string | undefined =>
- appsDomain ? `${porterDnsLabel(d.name ?? d.id)}.${appsDomain}` : undefined;
- const endpointOf = (host: string): DeployEndpoint => ({
- host,
- port: ENDPOINT_PORT,
- tls: true,
- publicUrl: `https://${host}/`,
- });
+ appsDomain && visibility !== "internal" ? `${porterDnsLabel(d.name ?? d.id)}.${appsDomain}` : undefined;
+ const endpointOf = (body: { host: string; port?: number }): DeployEndpoint =>
+ body.port
+ ? { host: body.host, port: body.port }
+ : { host: body.host, port: ENDPOINT_PORT, tls: true, publicUrl: `https://${body.host}/` };
const serialized = <T>(d: Deployment, fn: () => Promise<T>): Promise<T> =>
queue(d.id, () => advisoryLock.withLock(`porter-deploy:${d.id}`, fn));
@@ -137,6 +136,28 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
});
}
+ async function addressOf(d: Deployment, sb: PorterSandboxLike): Promise<{ host: string; port?: number }> {
+ const status = await sb.refresh();
+ if (visibility === "internal") {
+ const address = status.internal_address ?? "";
+ const split = address.lastIndexOf(":");
+ const port = Number(address.slice(split + 1));
+ if (split <= 0 || !Number.isInteger(port) || port <= 0) {
+ throw new Error(
+ `porter deploy ${d.id}: the cluster reported no internal address for the app (${JSON.stringify(address)}) — internal visibility needs sandbox-api 0.1.58 or newer`,
+ );
+ }
+ return { host: address.slice(0, split), port };
+ }
+ const host = status.host || domainOf(d);
+ if (!host) {
+ throw new Error(
+ `porter deploy ${d.id}: the cluster named no host for the app — its sandbox ingress has no root domain, so set PORTER_DEPLOY_APPS_DOMAIN to a domain whose wildcard resolves to that ingress (docs/porter.md)`,
+ );
+ }
+ return { host };
+ }
+
async function liveStored(d: Deployment): Promise<StoredPorterDeployBody | null> {
const stored = await store.get(d.id);
if (!stored) return null;
@@ -167,11 +188,14 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
tags: { [KIND_TAG]: "app", [DEPLOY_TAG]: d.id },
env: appEnv(version),
volume_mounts: { [DATA_DIR]: volumeId },
- networking: [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],
+ networking:
+ visibility === "internal"
+ ? [{ port: appPort, internal: true }]
+ : [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],
...(opts.ttlSec ? { ttl_seconds: opts.ttlSec } : {}),
})
.catch((e) => {
- if (errMessage(e).includes("sandbox ingress")) {
+ if (visibility !== "internal" && errMessage(e).includes("sandbox ingress")) {
throw new Error(
`porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster (docs/porter.md), then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,
{ cause: e },
@@ -181,16 +205,11 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
});
try {
await waitPorterRunning(name, sb);
- const host = (await sb.refresh()).host || domainOf(d);
- if (!host) {
- throw new Error(
- `porter deploy ${d.id}: the cluster named no host for the app — its sandbox ingress has no root domain, so set PORTER_DEPLOY_APPS_DOMAIN to a domain whose wildcard resolves to that ingress (docs/porter.md)`,
- );
- }
+ const address = await addressOf(d, sb);
await materialize(sb.id, version);
await startApp(sb.id, version);
- await store.put(d.id, { deploymentId: d.id, sandboxId: sb.id, name, host, createdAtMs: Date.now() });
- return endpointOf(host);
+ await store.put(d.id, { deploymentId: d.id, sandboxId: sb.id, name, ...address, createdAtMs: Date.now() });
+ return endpointOf(address);
} catch (e) {
await sb.terminate().catch((err) => swallow("porter-deploy: abandon failed body", err));
throw e;
@@ -205,7 +224,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
resolveCache.delete(d.id);
return null;
}
- const endpoint = endpointOf(stored.host);
+ const endpoint = endpointOf(stored);
if (resolveCacheMs > 0) resolveCache.set(d.id, endpoint);
return endpoint;
},
diff --git a/src/sandbox/porter-client.ts b/src/sandbox/porter-client.ts
index bf4a6709f..f3587a1bd 100644
--- a/src/sandbox/porter-client.ts
+++ b/src/sandbox/porter-client.ts
@@ -19,7 +19,7 @@ export interface PorterSandboxLike {
readonly id: string;
readonly phase: string | null;
readonly tags: Record<string, string> | null;
- refresh(): Promise<{ name: string; host?: string }>;
+ refresh(): Promise<{ name: string; host?: string; internal_address?: string }>;
terminate(): Promise<void>;
}
@@ -31,7 +31,11 @@ export interface PorterSandboxSpec {
env?: Record<string, string>;
volume_mounts?: Record<string, string>;
egress?: { allowed_destinations: string[] };
- networking?: Array<{ port: number; domains?: Array<{ domain?: string; visibility?: "public" | "private" }> }>;
+ networking?: Array<{
+ port: number;
+ internal?: boolean;
+ domains?: Array<{ domain?: string; visibility?: "public" | "private" }>;
+ }>;
ttl_seconds?: number;
}
diff --git a/test/config.test.ts b/test/config.test.ts
index 4c54e5177..8ee32d3cd 100644
--- a/test/config.test.ts
+++ b/test/config.test.ts
@@ -682,6 +682,21 @@ test("DEPLOY_PROVIDER=porter refuses to boot without a cluster and tolerates a m
);
});
+test("PORTER_DEPLOY_VISIBILITY=internal is accepted and refuses an apps domain", () => {
+ const cluster = {
+ DEPLOY_PROVIDER: "porter",
+ PORTER_DEPLOY_API_TOKEN: "tok",
+ PORTER_DEPLOY_PROJECT_ID: "7",
+ PORTER_DEPLOY_CLUSTER_ID: "9",
+ PORTER_DEPLOY_VISIBILITY: "internal",
+ };
+ assert.equal(loadConfig(cluster).porterDeploy.visibility, "internal");
+ assert.throws(
+ () => loadConfig({ ...cluster, PORTER_DEPLOY_APPS_DOMAIN: "apps.example.com" }),
+ /PORTER_DEPLOY_APPS_DOMAIN/,
+ );
+});
+
test("SANDBOX_BACKEND=porter locates the API and shares the deploy provider's token", () => {
assert.throws(
() => loadConfig({ SANDBOX_BACKEND: "porter", PORTER_DEPLOY_API_TOKEN: "tok" }),
diff --git a/test/porter-deploy-provider.test.ts b/test/porter-deploy-provider.test.ts
index a4697e93c..0e21634d5 100644
--- a/test/porter-deploy-provider.test.ts
+++ b/test/porter-deploy-provider.test.ts
@@ -122,6 +122,23 @@ test("explicit public visibility opts the domain out of the private default", as
]);
});
+test("without an apps domain the app defaults to its cluster-internal address", async () => {
+ const bare = createPorterDeployProvider({
+ namePrefix: "qmt",
+ appPort,
+ readyWindowSec: 10,
+ client: fake.client,
+ store,
+ });
+ const d = deployment("dep-1c", "hidden");
+ const endpoint = await bare.apply(d, version({ "server.js": SERVER }, "node server.js"));
+ assert.deepEqual(endpoint, { host: "127.0.0.1", port: appPort });
+ assert.deepEqual(fake.bodies()[0]!.networking, [{ port: appPort, internal: true }]);
+ assert.equal(fake.bodies()[0]!.host, "");
+ assert.deepEqual(await bare.resolveEndpoint!(d, version({}, "true")), endpoint);
+ assert.match(await (await fetch(`http://${endpoint.host}:${endpoint.port}/`)).text(), /^hello from .*-app v\?$/);
+});
+
test("redeploy keeps the domain and the /data volume and retires the old body", async () => {
const d = deployment("dep-2", "keeper");
await provider.apply(d, version({ "server.js": SERVER }, "node server.js", { env: { APP_VERSION: "1" } }));
@@ -249,8 +266,9 @@ test("a body that vanished between list and terminate does not break destroy or
assert.match(await fetchText("/"), /^hello from/);
});
-test("without an apps domain the cluster names the host itself", async () => {
+test("without an apps domain a private app gets the host the cluster names", async () => {
const bare = createPorterDeployProvider({
+ visibility: "private",
namePrefix: "qmt",
appPort,
readyWindowSec: 10,
@@ -269,6 +287,7 @@ test("a cluster that names no host fails the deploy and leaves no body behind",
fake.cleanup();
fake = installFakePorter({ assignHost: false });
const bare = createPorterDeployProvider({
+ visibility: "private",
namePrefix: "qmt",
appPort,
readyWindowSec: 10,
diff --git a/test/support/fake-porter.ts b/test/support/fake-porter.ts
index 48ee0937c..987e848eb 100644
--- a/test/support/fake-porter.ts
+++ b/test/support/fake-porter.ts
@@ -124,7 +124,9 @@ export function installFakePorter(opts: FakePorterOptions = {}): FakePorter {
},
async refresh() {
known = true;
- return { name, host: observe(bodies.get(name))?.host ?? "" };
+ const cur = observe(bodies.get(name));
+ const internal = cur?.networking?.[0]?.internal ? cur.networking[0].port : undefined;
+ return { name, host: cur?.host ?? "", ...(internal ? { internal_address: `127.0.0.1:${internal}` } : {}) };
},
async terminate() {
const cur = bodies.get(name);
@@ -158,7 +160,7 @@ export function installFakePorter(opts: FakePorterOptions = {}): FakePorter {
mkdirSync(app, { recursive: true });
const exposed = spec.networking?.[0];
const named = opts.assignHost === false ? undefined : `${spec.name}.fake.test`;
- const host = exposed ? (exposed.domains?.[0]?.domain ?? named ?? "") : "";
+ const host = exposed && !exposed.internal ? (exposed.domains?.[0]?.domain ?? named ?? "") : "";
bodies.set(spec.name!, {
id: `sb-${++bodySeq}`,
phase: "running",