Multiplayer agent harness for work.
Smooth the Porter hosting path from a live end-to-end deploy
Josh Francee6862f23fc3a
diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yamlindex d9f06381d..d97f00b3e 100644--- a/deploy/helm/templates/deployment.yaml+++ b/deploy/helm/templates/deployment.yaml@@ -29,7 +29,7 @@ spec:{{- end }}containers:- name: {{ $name }}- image: "{{ $root.Values.image.repository }}-{{ $svc.image }}:{{ $svc.tag | default $root.Values.image.tag | default $root.Chart.AppVersion }}"+ image: "{{ $root.Values.image.repository }}{{ $root.Values.image.separator | default "/" }}{{ $svc.image }}:{{ $svc.tag | default $root.Values.image.tag | default $root.Chart.AppVersion }}"imagePullPolicy: {{ $root.Values.image.pullPolicy }}{{- with $svc.command }}command:@@ -39,9 +39,25 @@ spec:- name: {{ ternary "http" "proxy" (eq $svc.type "web") }}containerPort: {{ $svc.port }}{{- $wired := dict }}+ {{- $svcHost := printf "%s-%%s.%s.svc.cluster.local" (include "qm.fullname" $root) $root.Release.Namespace }}+ {{- if ne $name "core" }}+ {{- $_ := set $wired "CORE_API_URL" (printf "http://%s:%v" (printf $svcHost "core") (index $root.Values.services "core").port) }}+ {{- end }}+ {{- if eq $name "portal" }}+ {{- $_ := set $wired "WEB_UI_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "web-ui") (index $root.Values.services "web-ui").port) }}+ {{- $_ := set $wired "ADMIN_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "admin") (index $root.Values.services "admin").port) }}+ {{- if (index $root.Values.services "auth").enabled }}+ {{- $_ := set $wired "AUTH_BROKER_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "auth") (index $root.Values.services "auth").port) }}+ {{- $_ := set $wired "AUTH_BROKER_PREFIX" "/idp" }}+ {{- end }}+ {{- end }}{{- if $root.Values.publicUrl }}- {{- if eq $name "core" }}{{- $_ := set $wired "AGENT_API_URL" $root.Values.publicUrl }}{{- end }}+ {{- if eq $name "core" }}+ {{- $_ := set $wired "AGENT_API_URL" $root.Values.publicUrl }}+ {{- $_ := set $wired "PUBLIC_WEB_URL" $root.Values.publicUrl }}+ {{- end }}{{- if eq $name "portal" }}{{- $_ := set $wired "PORTAL_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}+ {{- if eq $name "web-ui" }}{{- $_ := set $wired "WEB_UI_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}{{- end }}{{- if or $wired $root.Values.env $svc.env }}env:diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yamlindex 118ec357c..87cf7292f 100644--- a/deploy/helm/values.yaml+++ b/deploy/helm/values.yaml@@ -1,5 +1,6 @@image:- repository: ghcr.io/porter-dev/qm+ repository: ghcr.io/yc-software/qm+ separator: "/"tag: ""pullPolicy: IfNotPresentdiff --git a/docs/porter.md b/docs/porter.mdindex 8e9f794b9..b27bde389 100644--- a/docs/porter.md+++ b/docs/porter.md@@ -17,27 +17,72 @@ PORTER_DEPLOY_CLUSTER_ID=<cluster id>## Hosting the qm surfaces-`porter/porter.yaml` declares the six services (core, auth, web-ui, admin, portal,-egress-proxy) as Porter apps built from `deploy/*/Dockerfile`:+`porter/apps/` declares the six services (core, auth, web-ui, admin, portal,+egress-proxy) as one Porter app per file, built from `deploy/*/Dockerfile` — `porter+apply` takes exactly one app per invocation and silently ignores extra YAML documents,+which is why they are separate files:```bash-porter apply -f porter/porter.yaml+for f in porter/apps/*.yaml; do porter apply -f "$f"; done```-Porter builds the images, runs the services, and assigns each web service a hostname, so-this path needs no DNS record, no TLS certificate, and no ingress controller. Set the-service secrets from the Porter dashboard or `porter apply`'s env-group support, and use-the portal's assigned hostname as `PUBLIC_WEB_URL`.+Porter runs the services and assigns each **public** web service an `onporter.run`+hostname with a Let's Encrypt certificate, so this path needs no DNS record, no TLS+certificate, and no ingress controller. Only the portal is meant to be Internet-facing;+auth, web-ui, and admin carry `private: true` so Porter creates no ingress for them and+they are reachable only at their in-cluster address.++Two things `porter apply` will not do for you:++- **Building on an Apple Silicon machine fails.** `build.method: docker` builds locally+ for `linux/amd64` and the legacy builder cannot cross-compile: the apply dies with+ `image ... does not provide the specified platform (linux/amd64)`. Build and push with+ `docker buildx build --platform linux/amd64 --push`, then point the app at the pushed+ image with an `image: {repository, tag}` block, or use `porter apply --remote`.+- **Env groups created by `porter env create` are not visible to `porter apply`.** Both+ the `envGroups:` key and `--attach-env-groups` fail with `internal: unable to find+latest environment with provided name`, because `porter env create` makes a+ project-scoped group and apply resolves cluster-scoped ones. Put non-secret wiring in+ the app's `env:` block and pass secrets with `--secrets KEY=value` (note that pflag+ splits those on commas, so a JSON value like `AUTH_SIGNING_JWK` has to go in `env:`).++Nothing generates the inter-service wiring for you — `cli/src/services.ts` has `fly` and+`docker` targets but no Porter one — so set it by hand on each app:++| Service | Wiring |+| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| all but core | `CORE_API_URL=http://qm-core-core.<namespace>.svc.cluster.local:8080`, `CORE_ORG_ID`, `CORE_SIGNING_SECRET`, `PORTAL_IDENTITY_SECRET` |+| core | `ORG_ID`, `MODEL_PROVIDER`, `HARNESS`, `SANDBOX_BACKEND`, `PUBLIC_WEB_URL`/`WEB_UI_PUBLIC_URL` (the portal's hostname), `CAPABILITY_SECRET`, `CONNECTOR_SECRET_KEY`, `SKILL_SIGNING_SECRET`, and `DATABASE_URL` when `SESSION_STORE`/`RUN_STORE` are `postgres` |+| portal | `PORTAL_PUBLIC_URL`, `PORTAL_SESSION_SECRET`, `WEB_UI_UPSTREAM`, `ADMIN_UPSTREAM`, `AUTH_BROKER_UPSTREAM`, `AUTH_BROKER_PREFIX=/idp`, and the `OIDC_*` set from `brokerWiring` in `cli/src/services.ts` |+| auth | `AUTH_ISSUER=<portal>/idp`, `AUTH_CLIENT_ID`, `AUTH_CLIENT_SECRET`, `AUTH_REDIRECT_URI=<portal>/auth/callback`, `AUTH_TOKEN_SECRET`, `AUTH_SIGNING_JWK` (a P-256 private JWK), and a `RESEND_API_KEY` or SMTP credentials — without a mail transport nobody can sign in |++The portal's hostname is only knowable after its first apply, and core, portal, and auth+all need it, so expect to apply twice: once to mint the hostname, once to wire it in.++Porter's own Postgres datastores are created from the dashboard. An RDS instance in the+cluster's VPC works too, but Postgres 17 defaults to `rds.force_ssl=1`, so the URL needs+`?sslmode=no-verify` (or a CA pinned with `DATABASE_CA_CERT`) or core dies on `no pg_hba.conf+entry ... no encryption`.`deploy/helm/` carries the same topology as a Helm chart for operators who would rathermanage the release themselves; it expects the images published at-`ghcr.io/yc-software/qm/<service>` and an ingress you configure.+`ghcr.io/yc-software/qm/<service>` and an ingress you configure. Pin `image.tag` to a+commit that actually carries the Porter backend — the published tags are commit SHAs, and+one from before Porter support landed rejects `SANDBOX_BACKEND=porter` at startup.## Giving published apps stable hostnamesApps the agent deploys are reachable only if the cluster gives them an address. Withoutone, `apply` fails with `the cluster assigned no hostname`, and core warns at startup.+The address comes from Porter's **sandbox ingress**, which is separate from the ingress+that serves Porter apps and is enabled per cluster from the dashboard. A cluster without+it rejects every publish before DNS is ever consulted, with+`HTTP 400: validation error: visibility: no private sandbox ingress is configured on this+cluster` (`no public sandbox ingress` when `PORTER_DEPLOY_VISIBILITY=public`). Turn+sandbox ingress on first; the wildcard record below is what gives the resulting apps+stable names.+Point a wildcard DNS record at the cluster's ingress load balancer and name it:```bash@@ -57,6 +102,7 @@ deployment's domain into public ingress.| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) || `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement || `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |+| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private || `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |To QA a branch against a real Porter cluster before deploying it, the dev instance takesdiff --git a/porter/apps/admin.yaml b/porter/apps/admin.yamlnew file mode 100644index 000000000..d6b47b594--- /dev/null+++ b/porter/apps/admin.yaml@@ -0,0 +1,15 @@+version: v2+name: qm-admin+build:+ method: docker+ context: .+ dockerfile: ./deploy/admin/Dockerfile+services:+ - name: admin+ type: web+ private: true+ run: node src/index.ts+ instances: 1+ cpuCores: 0.25+ ramMegabytes: 512+ port: 8080diff --git a/porter/apps/auth.yaml b/porter/apps/auth.yamlnew file mode 100644index 000000000..485f213de--- /dev/null+++ b/porter/apps/auth.yaml@@ -0,0 +1,15 @@+version: v2+name: qm-auth+build:+ method: docker+ context: .+ dockerfile: ./deploy/auth/Dockerfile+services:+ - name: auth+ type: web+ private: true+ run: node src/index.ts+ instances: 1+ cpuCores: 0.25+ ramMegabytes: 512+ port: 8080diff --git a/porter/apps/core.yaml b/porter/apps/core.yamlnew file mode 100644index 000000000..508240a9a--- /dev/null+++ b/porter/apps/core.yaml@@ -0,0 +1,17 @@+version: v2+name: qm-core+build:+ method: docker+ context: .+ dockerfile: ./deploy/core/Dockerfile+services:+ - name: core+ type: web+ run: node src/index.ts+ instances: 1+ cpuCores: 1+ ramMegabytes: 2048+ port: 8080+ healthCheck:+ enabled: true+ httpPath: /healthzdiff --git a/porter/apps/egress-proxy.yaml b/porter/apps/egress-proxy.yamlnew file mode 100644index 000000000..106be2f5b--- /dev/null+++ b/porter/apps/egress-proxy.yaml@@ -0,0 +1,13 @@+version: v2+name: qm-egress-proxy+build:+ method: docker+ context: .+ dockerfile: ./deploy/egress-proxy/Dockerfile+services:+ - name: egress-proxy+ type: worker+ run: /app/start.sh+ instances: 1+ cpuCores: 0.25+ ramMegabytes: 512diff --git a/porter/apps/portal.yaml b/porter/apps/portal.yamlnew file mode 100644index 000000000..e4064f6a6--- /dev/null+++ b/porter/apps/portal.yaml@@ -0,0 +1,14 @@+version: v2+name: qm-portal+build:+ method: docker+ context: .+ dockerfile: ./deploy/portal/Dockerfile+services:+ - name: portal+ type: web+ run: node src/index.ts+ instances: 1+ cpuCores: 0.25+ ramMegabytes: 512+ port: 8080diff --git a/porter/apps/web-ui.yaml b/porter/apps/web-ui.yamlnew file mode 100644index 000000000..b1cc2a3ee--- /dev/null+++ b/porter/apps/web-ui.yaml@@ -0,0 +1,15 @@+version: v2+name: qm-web-ui+build:+ method: docker+ context: .+ dockerfile: ./deploy/web-ui/Dockerfile+services:+ - name: web-ui+ type: web+ private: true+ run: node server/index.ts+ instances: 1+ cpuCores: 0.25+ ramMegabytes: 512+ port: 8080diff --git a/porter/porter.yaml b/porter/porter.yamldeleted file mode 100644index 0cea3f90d..000000000--- a/porter/porter.yaml+++ /dev/null@@ -1,91 +0,0 @@-version: v2-name: qm-core-build:- method: docker- context: .- dockerfile: ./deploy/core/Dockerfile-services:- - name: core- type: web- run: node src/index.ts- instances: 1- cpuCores: 1- ramMegabytes: 2048- port: 8080- healthCheck:- enabled: true- httpPath: /healthz-----version: v2-name: qm-auth-build:- method: docker- context: .- dockerfile: ./deploy/auth/Dockerfile-services:- - name: auth- type: web- run: node src/index.ts- instances: 1- cpuCores: 0.25- ramMegabytes: 512- port: 8080-----version: v2-name: qm-web-ui-build:- method: docker- context: .- dockerfile: ./deploy/web-ui/Dockerfile-services:- - name: web-ui- type: web- run: node server/index.ts- instances: 1- cpuCores: 0.25- ramMegabytes: 512- port: 8080-----version: v2-name: qm-admin-build:- method: docker- context: .- dockerfile: ./deploy/admin/Dockerfile-services:- - name: admin- type: web- run: node src/index.ts- instances: 1- cpuCores: 0.25- ramMegabytes: 512- port: 8080-----version: v2-name: qm-portal-build:- method: docker- context: .- dockerfile: ./deploy/portal/Dockerfile-services:- - name: portal- type: web- run: node src/index.ts- instances: 1- cpuCores: 0.25- ramMegabytes: 512- port: 8080-----version: v2-name: qm-egress-proxy-build:- method: docker- context: .- dockerfile: ./deploy/egress-proxy/Dockerfile-services:- - name: egress-proxy- type: worker- run: /app/start.sh- instances: 1- cpuCores: 0.25- ramMegabytes: 512diff --git a/src/deploy/porter-deploy-provider.ts b/src/deploy/porter-deploy-provider.tsindex ccef57fcd..990037e8a 100644--- a/src/deploy/porter-deploy-provider.ts+++ b/src/deploy/porter-deploy-provider.ts@@ -20,7 +20,7 @@ import { createMemoryMap, type DurableMap } from "../persistence/durable-map.ts"import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";import { createKeyedQueue } from "../util/async.ts";import { shq } from "../util/shell.ts";-import { swallow } from "../util/errors.ts";+import { errMessage, swallow } from "../util/errors.ts";const APP_DIR = "/app";const HOME_DIR = "/root";@@ -159,16 +159,26 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): Dawait retireBodies(d, true);await store.delete(d.id).catch((e) => swallow("porter-deploy: clear stale pointer", e));const name = `${baseName(d)}-${randomUUID().slice(0, 5)}`;- const sb = await client.sandboxes.create({- image,- name,- command: ["sleep", "infinity"],- tags: { [KIND_TAG]: "app", [DEPLOY_TAG]: d.id },- env: appEnv(version),- volume_mounts: { [DATA_DIR]: volumeId },- networking: [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],- ...(opts.ttlSec ? { ttl_seconds: opts.ttlSec } : {}),- });+ const sb = await client.sandboxes+ .create({+ image,+ name,+ command: ["sleep", "infinity"],+ tags: { [KIND_TAG]: "app", [DEPLOY_TAG]: d.id },+ env: appEnv(version),+ volume_mounts: { [DATA_DIR]: volumeId },+ networking: [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],+ ...(opts.ttlSec ? { ttl_seconds: opts.ttlSec } : {}),+ })+ .catch((e) => {+ if (errMessage(e).includes("sandbox ingress")) {+ throw new Error(+ `porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster in the Porter dashboard, then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,+ { cause: e },+ );+ }+ throw e;+ });try {await waitPorterRunning(name, sb);const host = (await sb.refresh()).host || domainOf(d);