Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fix the Helm path and Porter's egress refusal from a live install

Josh Francef7b038e7388b
diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml
index c5c33ec44..ebbb72b35 100644
--- a/.github/workflows/release-package.yml
+++ b/.github/workflows/release-package.yml
@@ -27,6 +27,8 @@ jobs:
dockerfile: deploy/portal/Dockerfile
- name: auth
dockerfile: deploy/auth/Dockerfile
+ - name: egress-proxy
+ dockerfile: deploy/egress-proxy/Dockerfile
- name: sandbox-base
dockerfile: fly/Dockerfile
build-args: INSTALL_BROWSER_ENGINE=1
diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml
index 5edacf37b..96e0928ce 100644
--- a/deploy/helm/Chart.yaml
+++ b/deploy/helm/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
name: qm
description: Deploys the qm surfaces (core, auth, web-ui, admin, portal, egress-proxy) built from deploy/*/Dockerfile.
type: application
-version: 0.2.6
+version: 0.2.7
appVersion: "1.0.5"
diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml
index d97f00b3e..4d44e48d4 100644
--- a/deploy/helm/templates/deployment.yaml
+++ b/deploy/helm/templates/deployment.yaml
@@ -43,11 +43,15 @@ spec:
{{- if ne $name "core" }}
{{- $_ := set $wired "CORE_API_URL" (printf "http://%s:%v" (printf $svcHost "core") (index $root.Values.services "core").port) }}
{{- end }}
+ {{- $authEnabled := (index $root.Values.services "auth").enabled }}
+ {{- $authUpstream := printf "http://%s:%v" (printf $svcHost "auth") (index $root.Values.services "auth").port }}
+ {{- $base := trimSuffix "/" ($root.Values.publicUrl | default "") }}
+ {{- $issuer := printf "%s/idp" $base }}
{{- if eq $name "portal" }}
{{- $_ := set $wired "WEB_UI_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "web-ui") (index $root.Values.services "web-ui").port) }}
{{- $_ := set $wired "ADMIN_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "admin") (index $root.Values.services "admin").port) }}
- {{- if (index $root.Values.services "auth").enabled }}
- {{- $_ := set $wired "AUTH_BROKER_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "auth") (index $root.Values.services "auth").port) }}
+ {{- if $authEnabled }}
+ {{- $_ := set $wired "AUTH_BROKER_UPSTREAM" $authUpstream }}
{{- $_ := set $wired "AUTH_BROKER_PREFIX" "/idp" }}
{{- end }}
{{- end }}
@@ -56,7 +60,25 @@ spec:
{{- $_ := set $wired "AGENT_API_URL" $root.Values.publicUrl }}
{{- $_ := set $wired "PUBLIC_WEB_URL" $root.Values.publicUrl }}
{{- end }}
- {{- if eq $name "portal" }}{{- $_ := set $wired "PORTAL_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}
+ {{- if eq $name "portal" }}
+ {{- $_ := set $wired "PORTAL_PUBLIC_URL" $root.Values.publicUrl }}
+ {{- if $authEnabled }}
+ {{- $_ := set $wired "OIDC_ISSUER" $issuer }}
+ {{- $_ := set $wired "OIDC_AUTH_ENDPOINT" (printf "%s/authorize" $issuer) }}
+ {{- $_ := set $wired "OIDC_TOKEN_ENDPOINT" (printf "%s/token" $authUpstream) }}
+ {{- $_ := set $wired "OIDC_USERINFO_ENDPOINT" (printf "%s/userinfo" $authUpstream) }}
+ {{- $_ := set $wired "OIDC_JWKS_URI" (printf "%s/.well-known/jwks.json" $authUpstream) }}
+ {{- $_ := set $wired "OIDC_SCOPES" "openid email" }}
+ {{- $_ := set $wired "OIDC_PRINCIPAL_CLAIM" "email" }}
+ {{- with $root.Values.env.AUTH_ALLOWED_EMAIL_DOMAIN }}
+ {{- $_ := set $wired "OIDC_ALLOWED_EMAIL_DOMAIN" . }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- if and (eq $name "auth") $authEnabled }}
+ {{- $_ := set $wired "AUTH_ISSUER" $issuer }}
+ {{- $_ := set $wired "AUTH_REDIRECT_URI" (printf "%s/auth/callback" $base) }}
+ {{- end }}
{{- if eq $name "web-ui" }}{{- $_ := set $wired "WEB_UI_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}
{{- end }}
{{- if or $wired $root.Values.env $svc.env }}
diff --git a/deploy/helm/templates/secret.yaml b/deploy/helm/templates/secret.yaml
index 1aed6f35b..0bb9ad04d 100644
--- a/deploy/helm/templates/secret.yaml
+++ b/deploy/helm/templates/secret.yaml
@@ -18,4 +18,7 @@ stringData:
{{- if and (not .Values.secretEnv.OIDC_CLIENT_SECRET) .Values.secretEnv.AUTH_CLIENT_SECRET }}
OIDC_CLIENT_SECRET: {{ .Values.secretEnv.AUTH_CLIENT_SECRET | quote }}
{{- end }}
+ {{- if and (not .Values.secretEnv.OIDC_ALLOWED_EMAILS) .Values.secretEnv.AUTH_ALLOWED_EMAILS }}
+ OIDC_ALLOWED_EMAILS: {{ .Values.secretEnv.AUTH_ALLOWED_EMAILS | quote }}
+ {{- end }}
{{- end }}
diff --git a/docs/porter.md b/docs/porter.md
index 2bf96a791..5ef2871ab 100644
--- a/docs/porter.md
+++ b/docs/porter.md
@@ -80,7 +80,21 @@ entry ... no encryption`.
manage the release themselves; it expects the images published at
`ghcr.io/yc-software/qm/<service>` and an ingress you configure. Pin `image.tag` to a
commit that actually carries the Porter backend — the published tags are commit SHAs, and
-one from before Porter support landed rejects `SANDBOX_BACKEND=porter` at startup.
+one from before Porter support landed rejects `SANDBOX_BACKEND=porter` at startup. Set
+`publicUrl` and the chart derives the broker wiring (`AUTH_ISSUER`, `AUTH_REDIRECT_URI`
+and the portal's `OIDC_*` endpoints) the way `cli/src/services.ts` does for the other
+targets; what it cannot invent is the identity allow-list, so production still refuses to
+boot until `AUTH_ALLOWED_EMAILS`/`AUTH_ALLOWED_EMAIL_DOMAIN` is set, and the auth broker
+needs a real mail transport (`RESEND_API_KEY` or the `SMTP_*` set) because there is no
+console transport. `SANDBOX_BACKEND` has no default and core refuses to start in
+production without it.
+
+`porter kubectl --print-kubeconfig` is enough to read a Porter cluster but not to install
+into one: the kubeconfig it prints authenticates as `porter-readonly-sa-customer`, which
+holds `get`/`list`/`watch` and nothing else, so `helm install` fails on the first write. On
+EKS, grant yourself real access through AWS instead — `aws eks update-cluster-config
+--access-config authenticationMode=API_AND_CONFIG_MAP`, then an access entry for your own
+principal with `AmazonEKSClusterAdminPolicy`, then `aws eks update-kubeconfig`.
## Giving published apps stable hostnames
@@ -89,12 +103,51 @@ Apps the agent deploys are reachable only if the cluster gives them an address.
front, before anything is created.
The address comes from Porter's **sandbox ingress**, which is separate from the ingress
-that serves Porter apps and is enabled per cluster from the dashboard. A cluster without
-it rejects every publish before DNS is ever consulted, with
-`HTTP 400: validation error: visibility: no private sandbox ingress is configured on this
-cluster` (`no public sandbox ingress` when `PORTER_DEPLOY_VISIBILITY=public`). Turn
-sandbox ingress on first; the wildcard record below is what gives the resulting apps
-stable names.
+that serves Porter apps and is configured per cluster. A cluster without it rejects every
+publish before DNS is ever consulted, with `HTTP 400: validation error: visibility: no
+private sandbox ingress is configured on this cluster` (`no public sandbox ingress` when
+`PORTER_DEPLOY_VISIBILITY=public`). Turn sandbox ingress on first; the wildcard record
+below is what gives the resulting apps stable names.
+
+The dashboard has a toggle for it, but it is also a plain field on the cluster contract,
+so it can be turned on headlessly. `GET /api/projects/<project>/contracts` returns the
+revisions for the project, newest last, each with a `base64_contract`; decode the current
+one, add the sandbox load balancer, and `POST` the whole contract back to
+`/api/projects/<project>/contract` (singular, raw protojson — not base64, not wrapped):
+
+```json
+{
+ "kind": "LOAD_BALANCER_KIND_NLB",
+ "owner": "LOAD_BALANCER_OWNER_SANDBOX",
+ "networkAccess": "NETWORK_ACCESS_PUBLIC",
+ "dnsProviderConfig": { "provider": "DNS_PROVIDER_TYPE_ROUTE53", "route53Domain": "" },
+ "rootDomains": ["apps.example.com"],
+ "waf": { "enabled": false }
+}
+```
+
+appended to `cluster.additionalLoadBalancers`, alongside `cluster.eksKind.sandboxesEnabled:
+true`. Route53 authenticates through the cluster's own pod identity and needs no token;
+`DNS_PROVIDER_TYPE_CLOUDFLARE` needs one stored first at `POST
+/api/v2/projects/<project>/clusters/<cluster>/dns/credentials`. The same route creates a
+cluster in the first place — omit `cluster.clusterId` and Porter provisions a new one.
+Once the revision reconciles, `GET
+/api/v2/projects/<project>/clusters/<cluster>/load-balancers` returns the new
+`owner: "sandbox"` entry whose `address.value` is what the wildcard record points at.
+
+Routing and TLS arrive separately, and only the first is reliable. The load balancer and
+the ingress rules come up within a couple of minutes and published apps answer on the
+wildcard domain straight away — but the wildcard certificate is a `Certificate` in the
+`porter-sandbox` namespace naming a `letsencrypt-sandbox-<visibility>-route53-prod`
+ClusterIssuer, and on a cluster whose Porter IAM came from the standard CloudFormation
+template that issuer is never created: the template grants no Route53 actions to any
+`porter-*` role, and cert-manager gets neither an IAM role annotation nor an EKS pod
+identity association, so the DNS-01 challenge has no credentials. Until that is sorted out
+the ingress serves nginx's self-signed _Kubernetes Ingress Controller Fake Certificate_,
+so every client — including the agent's own probe of the app it just published, which is
+why it reports a gateway error it cannot explain — fails certificate verification against
+an app that is otherwise serving correctly. Check with `kubectl get certificate -n
+porter-sandbox` before believing a publish is broken.
Point a wildcard DNS record at the cluster's ingress load balancer and name it:
@@ -106,17 +159,50 @@ Each deployment then publishes at `<name>.apps.example.com`. Deployments are **p
by default, matching the other providers; `PORTER_DEPLOY_VISIBILITY=public` opts a
deployment's domain into public ingress.
+## Forcing sandbox egress through the proxy
+
+`PORTER_SANDBOX_EGRESS_PROXY_URL` makes core pass `egress.allowed_destinations: [<proxy
+host>]` when it creates a sandbox, and inject `HTTPS_PROXY`/`HTTP_PROXY` pointing at the
+proxy with a capability token as the password. Two cluster-side facts decide whether that
+works at all, and both fail in ways that do not name the real cause:
+
+- **The cluster must have egress restriction turned on.** Without it Porter rejects the
+ create outright — `HTTP 400: validation error: egress: egress restriction is not
+available on this cluster` — so the agent ends up with no computer rather than an
+ unrestricted one. It is a per-cluster setting on the `sandbox-api` system application:
+ `PATCH /api/v2/projects/<project>/clusters/<cluster>/system-applications/sandbox-api`
+ with `{"sandbox_config":{"egress_enabled":true}}`, then `POST
+.../trigger-system-application-reconcile` with `{"dry_run":false,
+"system_application_name":"sandbox-api"}`. `GET` on the same path reads it back, and
+ turning it on installs Cilium, which enforces the allowlist. A project API token is
+ refused here with `PERMISSION_DENIED` even though it may create clusters — this call
+ needs a Porter account with admin rights on the project.
+- **The proxy has to be reachable from outside the cluster.** Porter attaches a
+ NetworkPolicy to every sandbox that permits DNS plus `0.0.0.0/0` _except_ `10.0.0.0/8`,
+ `172.16.0.0/12`, `192.168.0.0/16`, `169.254.0.0/16` and `100.64.0.0/10` — every RFC1918
+ range, which is where both the pod and service CIDRs live. A sandbox therefore cannot
+ reach an in-cluster Service, so a `*.svc.cluster.local` proxy URL times out with no
+ error worth reading. Give the proxy an internet-reachable address (a `LoadBalancer`
+ Service in front of it is enough) and point `PORTER_SANDBOX_EGRESS_PROXY_URL` there.
+ This is also why `deploy/helm/`'s `egress-proxy` service cannot serve Porter sandboxes
+ as-is: it is a ClusterIP worker.
+
+The proxy itself runs fine unprivileged, but `deploy/egress-proxy` wants `NET_ADMIN` to
+install the iptables rule that blocks the cloud metadata endpoint. Without the capability
+it logs `could not install metadata firewall ... NET_ADMIN missing?` and keeps serving, so
+grep for that line rather than assuming the container's own metadata block is in place.
+
## Other knobs
-| Variable | Meaning |
-| -------------------------------------------------- | ---------------------------------------------------------------------------------- |
-| `PORTER_DEPLOY_URL` | Porter API host, when it isn't `https://dashboard.porter.run` |
-| `PORTER_SANDBOX_IMAGE` | Image agent computers boot from |
-| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) |
-| `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement |
-| `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |
-| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private |
-| `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |
+| Variable | Meaning |
+| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- |
+| `PORTER_DEPLOY_URL` | Porter API host, when it isn't `https://dashboard.porter.run` |
+| `PORTER_SANDBOX_IMAGE` | Image agent computers boot from |
+| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) |
+| `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement (see the constraints below) |
+| `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |
+| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private |
+| `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |
To QA a branch against a real Porter cluster before deploying it, the dev instance takes
the same backend:
diff --git a/eslint.config.mjs b/eslint.config.mjs
index 4567fd9c3..cc40069e9 100644
--- a/eslint.config.mjs
+++ b/eslint.config.mjs
@@ -8,6 +8,7 @@ export default tseslint.config(
"**/node_modules/",
"**/dist/",
"**/dist-web/",
+ "data/",
"deploy/layers/",
"docs/",
"plugins/web-ui/public/",
diff --git a/scripts/deploy-helm.sh b/scripts/deploy-helm.sh
index e7f629255..e8bc1ee8f 100755
--- a/scripts/deploy-helm.sh
+++ b/scripts/deploy-helm.sh
@@ -15,7 +15,7 @@ CHART_NS="${REPO%/*}"
SERVICES=(core auth web-ui admin portal egress-proxy)
for svc in "${SERVICES[@]}"; do
- image="$REPO-$svc:$TAG"
+ image="$REPO/$svc:$TAG"
echo ">> building $image from deploy/$svc/Dockerfile"
docker build \
--platform "$PLATFORM" \
diff --git a/src/deploy/porter-deploy-provider.ts b/src/deploy/porter-deploy-provider.ts
index 80a117d82..86fd5d3e9 100644
--- a/src/deploy/porter-deploy-provider.ts
+++ b/src/deploy/porter-deploy-provider.ts
@@ -156,7 +156,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
serialized(d, async () => {
if (!appsDomain) {
throw new Error(
- `porter deploy ${d.id}: PORTER_DEPLOY_APPS_DOMAIN is not set, so the app would have no address — enable sandbox ingress on the cluster in the Porter dashboard, point a wildcard DNS record at it, and set PORTER_DEPLOY_APPS_DOMAIN`,
+ `porter deploy ${d.id}: PORTER_DEPLOY_APPS_DOMAIN is not set, so the app would have no address — enable sandbox ingress on the cluster, point a wildcard DNS record at it, and set PORTER_DEPLOY_APPS_DOMAIN (docs/porter.md)`,
);
}
resolveCache.delete(d.id);
@@ -178,7 +178,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
.catch((e) => {
if (errMessage(e).includes("sandbox ingress")) {
throw new Error(
- `porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster in the Porter dashboard, then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,
+ `porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster (docs/porter.md), then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,
{ cause: e },
);
}
diff --git a/src/sandbox/porter-sandbox.ts b/src/sandbox/porter-sandbox.ts
index fbf298eef..b8a3dc3f3 100644
--- a/src/sandbox/porter-sandbox.ts
+++ b/src/sandbox/porter-sandbox.ts
@@ -18,7 +18,12 @@ import {
type PorterSandboxLike,
} from "./porter-client.ts";
import { shq } from "../util/shell.ts";
-import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, proxyExportPrefix } from "./sandbox-env.ts";
+import {
+ nonInteractiveShellPrefix,
+ DROPPED_PROXY_ENV,
+ forceThroughProxyEnv,
+ proxyExportPrefix,
+} from "./sandbox-env.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
import { materializeRoLayers } from "./ro-layers.ts";
import {
@@ -138,15 +143,25 @@ export function createPorterSandbox(workspace: WorkspaceStore, opts: PorterSandb
kind: "scope" | "scratch" = "scope",
): Promise<BodyEntry> {
const name = bodyName(slug);
- const sb = await client.sandboxes.create({
- image,
- name,
- command: ["sleep", "infinity"],
- tags: { [SCOPE_TAG]: slug, [EGRESS_TAG]: egressMode, [KIND_TAG]: kind },
- ...(volume ? { volume_mounts: { [volume.mountPath]: volume.id } } : {}),
- ...(egressMode === "proxy" && egressProxyHost ? { egress: { allowed_destinations: [egressProxyHost] } } : {}),
- ttl_seconds: ttlSec,
- });
+ const sb = await client.sandboxes
+ .create({
+ image,
+ name,
+ command: ["sleep", "infinity"],
+ tags: { [SCOPE_TAG]: slug, [EGRESS_TAG]: egressMode, [KIND_TAG]: kind },
+ ...(volume ? { volume_mounts: { [volume.mountPath]: volume.id } } : {}),
+ ...(egressMode === "proxy" && egressProxyHost ? { egress: { allowed_destinations: [egressProxyHost] } } : {}),
+ ttl_seconds: ttlSec,
+ })
+ .catch((e) => {
+ if (errMessage(e).includes("egress restriction is not available")) {
+ throw new Error(
+ `porter sandbox ${name}: PORTER_SANDBOX_EGRESS_PROXY_URL is set but this cluster has egress restriction turned off, so Porter refuses to create the body at all — enable it on the cluster's sandbox-api system application or unset the proxy URL (docs/porter.md) (${errMessage(e)})`,
+ { cause: e },
+ );
+ }
+ throw e;
+ });
try {
await waitPorterRunning(name, sb);
} catch (e) {
@@ -229,20 +244,6 @@ export function createPorterSandbox(workspace: WorkspaceStore, opts: PorterSandb
const { execRaw, writeAbsBytes, readAbsBytes } = createPorterExec(client, async (id) => (await refFor(id)).sb.id);
- function proxyEnv(token: string): Record<string, string> {
- const u = new URL(opts.egressProxyUrl!);
- const url = `${u.protocol}//x:${token}@${u.host}`;
- const noProxy = "localhost,127.0.0.1,::1";
- return {
- HTTPS_PROXY: url,
- HTTP_PROXY: url,
- NO_PROXY: noProxy,
- https_proxy: url,
- http_proxy: url,
- no_proxy: noProxy,
- };
- }
-
const profile: AgentComputerProfile = {
backend: "porter",
writablePersistence: "resident_disk",
@@ -342,7 +343,10 @@ export function createPorterSandbox(workspace: WorkspaceStore, opts: PorterSandb
const turnEnv = Object.fromEntries(
Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),
);
- const env = { ...turnEnv, ...(forceEgress ? proxyEnv(provOpts!.egressToken!) : {}) };
+ const env = {
+ ...turnEnv,
+ ...(forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!) : {}),
+ };
const handle: SandboxHandle = {
id: name,
rootDir: workspaceDir,