Multiplayer agent harness for work.
Fix the Helm path and Porter's egress refusal from a live install
Josh Francef7b038e7388b
diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.ymlindex c5c33ec44..ebbb72b35 100644--- a/.github/workflows/release-package.yml+++ b/.github/workflows/release-package.yml@@ -27,6 +27,8 @@ jobs:dockerfile: deploy/portal/Dockerfile- name: authdockerfile: deploy/auth/Dockerfile+ - name: egress-proxy+ dockerfile: deploy/egress-proxy/Dockerfile- name: sandbox-basedockerfile: fly/Dockerfilebuild-args: INSTALL_BROWSER_ENGINE=1diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yamlindex 5edacf37b..96e0928ce 100644--- a/deploy/helm/Chart.yaml+++ b/deploy/helm/Chart.yaml@@ -2,5 +2,5 @@ apiVersion: v2name: qmdescription: Deploys the qm surfaces (core, auth, web-ui, admin, portal, egress-proxy) built from deploy/*/Dockerfile.type: application-version: 0.2.6+version: 0.2.7appVersion: "1.0.5"diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yamlindex d97f00b3e..4d44e48d4 100644--- a/deploy/helm/templates/deployment.yaml+++ b/deploy/helm/templates/deployment.yaml@@ -43,11 +43,15 @@ spec:{{- if ne $name "core" }}{{- $_ := set $wired "CORE_API_URL" (printf "http://%s:%v" (printf $svcHost "core") (index $root.Values.services "core").port) }}{{- end }}+ {{- $authEnabled := (index $root.Values.services "auth").enabled }}+ {{- $authUpstream := printf "http://%s:%v" (printf $svcHost "auth") (index $root.Values.services "auth").port }}+ {{- $base := trimSuffix "/" ($root.Values.publicUrl | default "") }}+ {{- $issuer := printf "%s/idp" $base }}{{- if eq $name "portal" }}{{- $_ := set $wired "WEB_UI_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "web-ui") (index $root.Values.services "web-ui").port) }}{{- $_ := set $wired "ADMIN_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "admin") (index $root.Values.services "admin").port) }}- {{- if (index $root.Values.services "auth").enabled }}- {{- $_ := set $wired "AUTH_BROKER_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "auth") (index $root.Values.services "auth").port) }}+ {{- if $authEnabled }}+ {{- $_ := set $wired "AUTH_BROKER_UPSTREAM" $authUpstream }}{{- $_ := set $wired "AUTH_BROKER_PREFIX" "/idp" }}{{- end }}{{- end }}@@ -56,7 +60,25 @@ spec:{{- $_ := set $wired "AGENT_API_URL" $root.Values.publicUrl }}{{- $_ := set $wired "PUBLIC_WEB_URL" $root.Values.publicUrl }}{{- end }}- {{- if eq $name "portal" }}{{- $_ := set $wired "PORTAL_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}+ {{- if eq $name "portal" }}+ {{- $_ := set $wired "PORTAL_PUBLIC_URL" $root.Values.publicUrl }}+ {{- if $authEnabled }}+ {{- $_ := set $wired "OIDC_ISSUER" $issuer }}+ {{- $_ := set $wired "OIDC_AUTH_ENDPOINT" (printf "%s/authorize" $issuer) }}+ {{- $_ := set $wired "OIDC_TOKEN_ENDPOINT" (printf "%s/token" $authUpstream) }}+ {{- $_ := set $wired "OIDC_USERINFO_ENDPOINT" (printf "%s/userinfo" $authUpstream) }}+ {{- $_ := set $wired "OIDC_JWKS_URI" (printf "%s/.well-known/jwks.json" $authUpstream) }}+ {{- $_ := set $wired "OIDC_SCOPES" "openid email" }}+ {{- $_ := set $wired "OIDC_PRINCIPAL_CLAIM" "email" }}+ {{- with $root.Values.env.AUTH_ALLOWED_EMAIL_DOMAIN }}+ {{- $_ := set $wired "OIDC_ALLOWED_EMAIL_DOMAIN" . }}+ {{- end }}+ {{- end }}+ {{- end }}+ {{- if and (eq $name "auth") $authEnabled }}+ {{- $_ := set $wired "AUTH_ISSUER" $issuer }}+ {{- $_ := set $wired "AUTH_REDIRECT_URI" (printf "%s/auth/callback" $base) }}+ {{- end }}{{- if eq $name "web-ui" }}{{- $_ := set $wired "WEB_UI_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}{{- end }}{{- if or $wired $root.Values.env $svc.env }}diff --git a/deploy/helm/templates/secret.yaml b/deploy/helm/templates/secret.yamlindex 1aed6f35b..0bb9ad04d 100644--- a/deploy/helm/templates/secret.yaml+++ b/deploy/helm/templates/secret.yaml@@ -18,4 +18,7 @@ stringData:{{- if and (not .Values.secretEnv.OIDC_CLIENT_SECRET) .Values.secretEnv.AUTH_CLIENT_SECRET }}OIDC_CLIENT_SECRET: {{ .Values.secretEnv.AUTH_CLIENT_SECRET | quote }}{{- end }}+ {{- if and (not .Values.secretEnv.OIDC_ALLOWED_EMAILS) .Values.secretEnv.AUTH_ALLOWED_EMAILS }}+ OIDC_ALLOWED_EMAILS: {{ .Values.secretEnv.AUTH_ALLOWED_EMAILS | quote }}+ {{- end }}{{- end }}diff --git a/docs/porter.md b/docs/porter.mdindex 2bf96a791..5ef2871ab 100644--- a/docs/porter.md+++ b/docs/porter.md@@ -80,7 +80,21 @@ entry ... no encryption`.manage the release themselves; it expects the images published at`ghcr.io/yc-software/qm/<service>` and an ingress you configure. Pin `image.tag` to acommit that actually carries the Porter backend — the published tags are commit SHAs, and-one from before Porter support landed rejects `SANDBOX_BACKEND=porter` at startup.+one from before Porter support landed rejects `SANDBOX_BACKEND=porter` at startup. Set+`publicUrl` and the chart derives the broker wiring (`AUTH_ISSUER`, `AUTH_REDIRECT_URI`+and the portal's `OIDC_*` endpoints) the way `cli/src/services.ts` does for the other+targets; what it cannot invent is the identity allow-list, so production still refuses to+boot until `AUTH_ALLOWED_EMAILS`/`AUTH_ALLOWED_EMAIL_DOMAIN` is set, and the auth broker+needs a real mail transport (`RESEND_API_KEY` or the `SMTP_*` set) because there is no+console transport. `SANDBOX_BACKEND` has no default and core refuses to start in+production without it.++`porter kubectl --print-kubeconfig` is enough to read a Porter cluster but not to install+into one: the kubeconfig it prints authenticates as `porter-readonly-sa-customer`, which+holds `get`/`list`/`watch` and nothing else, so `helm install` fails on the first write. On+EKS, grant yourself real access through AWS instead — `aws eks update-cluster-config+--access-config authenticationMode=API_AND_CONFIG_MAP`, then an access entry for your own+principal with `AmazonEKSClusterAdminPolicy`, then `aws eks update-kubeconfig`.## Giving published apps stable hostnames@@ -89,12 +103,51 @@ Apps the agent deploys are reachable only if the cluster gives them an address.front, before anything is created.The address comes from Porter's **sandbox ingress**, which is separate from the ingress-that serves Porter apps and is enabled per cluster from the dashboard. A cluster without-it rejects every publish before DNS is ever consulted, with-`HTTP 400: validation error: visibility: no private sandbox ingress is configured on this-cluster` (`no public sandbox ingress` when `PORTER_DEPLOY_VISIBILITY=public`). Turn-sandbox ingress on first; the wildcard record below is what gives the resulting apps-stable names.+that serves Porter apps and is configured per cluster. A cluster without it rejects every+publish before DNS is ever consulted, with `HTTP 400: validation error: visibility: no+private sandbox ingress is configured on this cluster` (`no public sandbox ingress` when+`PORTER_DEPLOY_VISIBILITY=public`). Turn sandbox ingress on first; the wildcard record+below is what gives the resulting apps stable names.++The dashboard has a toggle for it, but it is also a plain field on the cluster contract,+so it can be turned on headlessly. `GET /api/projects/<project>/contracts` returns the+revisions for the project, newest last, each with a `base64_contract`; decode the current+one, add the sandbox load balancer, and `POST` the whole contract back to+`/api/projects/<project>/contract` (singular, raw protojson — not base64, not wrapped):++```json+{+ "kind": "LOAD_BALANCER_KIND_NLB",+ "owner": "LOAD_BALANCER_OWNER_SANDBOX",+ "networkAccess": "NETWORK_ACCESS_PUBLIC",+ "dnsProviderConfig": { "provider": "DNS_PROVIDER_TYPE_ROUTE53", "route53Domain": "" },+ "rootDomains": ["apps.example.com"],+ "waf": { "enabled": false }+}+```++appended to `cluster.additionalLoadBalancers`, alongside `cluster.eksKind.sandboxesEnabled:+true`. Route53 authenticates through the cluster's own pod identity and needs no token;+`DNS_PROVIDER_TYPE_CLOUDFLARE` needs one stored first at `POST+/api/v2/projects/<project>/clusters/<cluster>/dns/credentials`. The same route creates a+cluster in the first place — omit `cluster.clusterId` and Porter provisions a new one.+Once the revision reconciles, `GET+/api/v2/projects/<project>/clusters/<cluster>/load-balancers` returns the new+`owner: "sandbox"` entry whose `address.value` is what the wildcard record points at.++Routing and TLS arrive separately, and only the first is reliable. The load balancer and+the ingress rules come up within a couple of minutes and published apps answer on the+wildcard domain straight away — but the wildcard certificate is a `Certificate` in the+`porter-sandbox` namespace naming a `letsencrypt-sandbox-<visibility>-route53-prod`+ClusterIssuer, and on a cluster whose Porter IAM came from the standard CloudFormation+template that issuer is never created: the template grants no Route53 actions to any+`porter-*` role, and cert-manager gets neither an IAM role annotation nor an EKS pod+identity association, so the DNS-01 challenge has no credentials. Until that is sorted out+the ingress serves nginx's self-signed _Kubernetes Ingress Controller Fake Certificate_,+so every client — including the agent's own probe of the app it just published, which is+why it reports a gateway error it cannot explain — fails certificate verification against+an app that is otherwise serving correctly. Check with `kubectl get certificate -n+porter-sandbox` before believing a publish is broken.Point a wildcard DNS record at the cluster's ingress load balancer and name it:@@ -106,17 +159,50 @@ Each deployment then publishes at `<name>.apps.example.com`. Deployments are **pby default, matching the other providers; `PORTER_DEPLOY_VISIBILITY=public` opts adeployment's domain into public ingress.+## Forcing sandbox egress through the proxy++`PORTER_SANDBOX_EGRESS_PROXY_URL` makes core pass `egress.allowed_destinations: [<proxy+host>]` when it creates a sandbox, and inject `HTTPS_PROXY`/`HTTP_PROXY` pointing at the+proxy with a capability token as the password. Two cluster-side facts decide whether that+works at all, and both fail in ways that do not name the real cause:++- **The cluster must have egress restriction turned on.** Without it Porter rejects the+ create outright — `HTTP 400: validation error: egress: egress restriction is not+available on this cluster` — so the agent ends up with no computer rather than an+ unrestricted one. It is a per-cluster setting on the `sandbox-api` system application:+ `PATCH /api/v2/projects/<project>/clusters/<cluster>/system-applications/sandbox-api`+ with `{"sandbox_config":{"egress_enabled":true}}`, then `POST+.../trigger-system-application-reconcile` with `{"dry_run":false,+"system_application_name":"sandbox-api"}`. `GET` on the same path reads it back, and+ turning it on installs Cilium, which enforces the allowlist. A project API token is+ refused here with `PERMISSION_DENIED` even though it may create clusters — this call+ needs a Porter account with admin rights on the project.+- **The proxy has to be reachable from outside the cluster.** Porter attaches a+ NetworkPolicy to every sandbox that permits DNS plus `0.0.0.0/0` _except_ `10.0.0.0/8`,+ `172.16.0.0/12`, `192.168.0.0/16`, `169.254.0.0/16` and `100.64.0.0/10` — every RFC1918+ range, which is where both the pod and service CIDRs live. A sandbox therefore cannot+ reach an in-cluster Service, so a `*.svc.cluster.local` proxy URL times out with no+ error worth reading. Give the proxy an internet-reachable address (a `LoadBalancer`+ Service in front of it is enough) and point `PORTER_SANDBOX_EGRESS_PROXY_URL` there.+ This is also why `deploy/helm/`'s `egress-proxy` service cannot serve Porter sandboxes+ as-is: it is a ClusterIP worker.++The proxy itself runs fine unprivileged, but `deploy/egress-proxy` wants `NET_ADMIN` to+install the iptables rule that blocks the cloud metadata endpoint. Without the capability+it logs `could not install metadata firewall ... NET_ADMIN missing?` and keeps serving, so+grep for that line rather than assuming the container's own metadata block is in place.+## Other knobs-| Variable | Meaning |-| -------------------------------------------------- | ---------------------------------------------------------------------------------- |-| `PORTER_DEPLOY_URL` | Porter API host, when it isn't `https://dashboard.porter.run` |-| `PORTER_SANDBOX_IMAGE` | Image agent computers boot from |-| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) |-| `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement |-| `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |-| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private |-| `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |+| Variable | Meaning |+| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- |+| `PORTER_DEPLOY_URL` | Porter API host, when it isn't `https://dashboard.porter.run` |+| `PORTER_SANDBOX_IMAGE` | Image agent computers boot from |+| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) |+| `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement (see the constraints below) |+| `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |+| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private |+| `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |To QA a branch against a real Porter cluster before deploying it, the dev instance takesthe same backend:diff --git a/eslint.config.mjs b/eslint.config.mjsindex 4567fd9c3..cc40069e9 100644--- a/eslint.config.mjs+++ b/eslint.config.mjs@@ -8,6 +8,7 @@ export default tseslint.config("**/node_modules/","**/dist/","**/dist-web/",+ "data/","deploy/layers/","docs/","plugins/web-ui/public/",diff --git a/scripts/deploy-helm.sh b/scripts/deploy-helm.shindex e7f629255..e8bc1ee8f 100755--- a/scripts/deploy-helm.sh+++ b/scripts/deploy-helm.sh@@ -15,7 +15,7 @@ CHART_NS="${REPO%/*}"SERVICES=(core auth web-ui admin portal egress-proxy)for svc in "${SERVICES[@]}"; do- image="$REPO-$svc:$TAG"+ image="$REPO/$svc:$TAG"echo ">> building $image from deploy/$svc/Dockerfile"docker build \--platform "$PLATFORM" \diff --git a/src/deploy/porter-deploy-provider.ts b/src/deploy/porter-deploy-provider.tsindex 80a117d82..86fd5d3e9 100644--- a/src/deploy/porter-deploy-provider.ts+++ b/src/deploy/porter-deploy-provider.ts@@ -156,7 +156,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): Dserialized(d, async () => {if (!appsDomain) {throw new Error(- `porter deploy ${d.id}: PORTER_DEPLOY_APPS_DOMAIN is not set, so the app would have no address — enable sandbox ingress on the cluster in the Porter dashboard, point a wildcard DNS record at it, and set PORTER_DEPLOY_APPS_DOMAIN`,+ `porter deploy ${d.id}: PORTER_DEPLOY_APPS_DOMAIN is not set, so the app would have no address — enable sandbox ingress on the cluster, point a wildcard DNS record at it, and set PORTER_DEPLOY_APPS_DOMAIN (docs/porter.md)`,);}resolveCache.delete(d.id);@@ -178,7 +178,7 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D.catch((e) => {if (errMessage(e).includes("sandbox ingress")) {throw new Error(- `porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster in the Porter dashboard, then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,+ `porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster (docs/porter.md), then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,{ cause: e },);}diff --git a/src/sandbox/porter-sandbox.ts b/src/sandbox/porter-sandbox.tsindex fbf298eef..b8a3dc3f3 100644--- a/src/sandbox/porter-sandbox.ts+++ b/src/sandbox/porter-sandbox.ts@@ -18,7 +18,12 @@ import {type PorterSandboxLike,} from "./porter-client.ts";import { shq } from "../util/shell.ts";-import { nonInteractiveShellPrefix, DROPPED_PROXY_ENV, proxyExportPrefix } from "./sandbox-env.ts";+import {+ nonInteractiveShellPrefix,+ DROPPED_PROXY_ENV,+ forceThroughProxyEnv,+ proxyExportPrefix,+} from "./sandbox-env.ts";import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";import { materializeRoLayers } from "./ro-layers.ts";import {@@ -138,15 +143,25 @@ export function createPorterSandbox(workspace: WorkspaceStore, opts: PorterSandbkind: "scope" | "scratch" = "scope",): Promise<BodyEntry> {const name = bodyName(slug);- const sb = await client.sandboxes.create({- image,- name,- command: ["sleep", "infinity"],- tags: { [SCOPE_TAG]: slug, [EGRESS_TAG]: egressMode, [KIND_TAG]: kind },- ...(volume ? { volume_mounts: { [volume.mountPath]: volume.id } } : {}),- ...(egressMode === "proxy" && egressProxyHost ? { egress: { allowed_destinations: [egressProxyHost] } } : {}),- ttl_seconds: ttlSec,- });+ const sb = await client.sandboxes+ .create({+ image,+ name,+ command: ["sleep", "infinity"],+ tags: { [SCOPE_TAG]: slug, [EGRESS_TAG]: egressMode, [KIND_TAG]: kind },+ ...(volume ? { volume_mounts: { [volume.mountPath]: volume.id } } : {}),+ ...(egressMode === "proxy" && egressProxyHost ? { egress: { allowed_destinations: [egressProxyHost] } } : {}),+ ttl_seconds: ttlSec,+ })+ .catch((e) => {+ if (errMessage(e).includes("egress restriction is not available")) {+ throw new Error(+ `porter sandbox ${name}: PORTER_SANDBOX_EGRESS_PROXY_URL is set but this cluster has egress restriction turned off, so Porter refuses to create the body at all — enable it on the cluster's sandbox-api system application or unset the proxy URL (docs/porter.md) (${errMessage(e)})`,+ { cause: e },+ );+ }+ throw e;+ });try {await waitPorterRunning(name, sb);} catch (e) {@@ -229,20 +244,6 @@ export function createPorterSandbox(workspace: WorkspaceStore, opts: PorterSandbconst { execRaw, writeAbsBytes, readAbsBytes } = createPorterExec(client, async (id) => (await refFor(id)).sb.id);- function proxyEnv(token: string): Record<string, string> {- const u = new URL(opts.egressProxyUrl!);- const url = `${u.protocol}//x:${token}@${u.host}`;- const noProxy = "localhost,127.0.0.1,::1";- return {- HTTPS_PROXY: url,- HTTP_PROXY: url,- NO_PROXY: noProxy,- https_proxy: url,- http_proxy: url,- no_proxy: noProxy,- };- }-const profile: AgentComputerProfile = {backend: "porter",writablePersistence: "resident_disk",@@ -342,7 +343,10 @@ export function createPorterSandbox(workspace: WorkspaceStore, opts: PorterSandbconst turnEnv = Object.fromEntries(Object.entries(provOpts?.env ?? {}).filter(([k]) => !DROPPED_PROXY_ENV.has(k)),);- const env = { ...turnEnv, ...(forceEgress ? proxyEnv(provOpts!.egressToken!) : {}) };+ const env = {+ ...turnEnv,+ ...(forceEgress ? forceThroughProxyEnv(opts.egressProxyUrl!, provOpts!.egressToken!) : {}),+ };const handle: SandboxHandle = {id: name,rootDir: workspaceDir,