Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Smooth the Porter hosting path from a live end-to-end deploy

Josh Francee6862f23fc3a
diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml
index d9f06381d..d97f00b3e 100644
--- a/deploy/helm/templates/deployment.yaml
+++ b/deploy/helm/templates/deployment.yaml
@@ -29,7 +29,7 @@ spec:
{{- end }}
containers:
- name: {{ $name }}
- image: "{{ $root.Values.image.repository }}-{{ $svc.image }}:{{ $svc.tag | default $root.Values.image.tag | default $root.Chart.AppVersion }}"
+ image: "{{ $root.Values.image.repository }}{{ $root.Values.image.separator | default "/" }}{{ $svc.image }}:{{ $svc.tag | default $root.Values.image.tag | default $root.Chart.AppVersion }}"
imagePullPolicy: {{ $root.Values.image.pullPolicy }}
{{- with $svc.command }}
command:
@@ -39,9 +39,25 @@ spec:
- name: {{ ternary "http" "proxy" (eq $svc.type "web") }}
containerPort: {{ $svc.port }}
{{- $wired := dict }}
+ {{- $svcHost := printf "%s-%%s.%s.svc.cluster.local" (include "qm.fullname" $root) $root.Release.Namespace }}
+ {{- if ne $name "core" }}
+ {{- $_ := set $wired "CORE_API_URL" (printf "http://%s:%v" (printf $svcHost "core") (index $root.Values.services "core").port) }}
+ {{- end }}
+ {{- if eq $name "portal" }}
+ {{- $_ := set $wired "WEB_UI_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "web-ui") (index $root.Values.services "web-ui").port) }}
+ {{- $_ := set $wired "ADMIN_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "admin") (index $root.Values.services "admin").port) }}
+ {{- if (index $root.Values.services "auth").enabled }}
+ {{- $_ := set $wired "AUTH_BROKER_UPSTREAM" (printf "http://%s:%v" (printf $svcHost "auth") (index $root.Values.services "auth").port) }}
+ {{- $_ := set $wired "AUTH_BROKER_PREFIX" "/idp" }}
+ {{- end }}
+ {{- end }}
{{- if $root.Values.publicUrl }}
- {{- if eq $name "core" }}{{- $_ := set $wired "AGENT_API_URL" $root.Values.publicUrl }}{{- end }}
+ {{- if eq $name "core" }}
+ {{- $_ := set $wired "AGENT_API_URL" $root.Values.publicUrl }}
+ {{- $_ := set $wired "PUBLIC_WEB_URL" $root.Values.publicUrl }}
+ {{- end }}
{{- if eq $name "portal" }}{{- $_ := set $wired "PORTAL_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}
+ {{- if eq $name "web-ui" }}{{- $_ := set $wired "WEB_UI_PUBLIC_URL" $root.Values.publicUrl }}{{- end }}
{{- end }}
{{- if or $wired $root.Values.env $svc.env }}
env:
diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml
index 118ec357c..87cf7292f 100644
--- a/deploy/helm/values.yaml
+++ b/deploy/helm/values.yaml
@@ -1,5 +1,6 @@
image:
- repository: ghcr.io/porter-dev/qm
+ repository: ghcr.io/yc-software/qm
+ separator: "/"
tag: ""
pullPolicy: IfNotPresent
diff --git a/docs/porter.md b/docs/porter.md
index 8e9f794b9..b27bde389 100644
--- a/docs/porter.md
+++ b/docs/porter.md
@@ -17,27 +17,72 @@ PORTER_DEPLOY_CLUSTER_ID=<cluster id>
## Hosting the qm surfaces
-`porter/porter.yaml` declares the six services (core, auth, web-ui, admin, portal,
-egress-proxy) as Porter apps built from `deploy/*/Dockerfile`:
+`porter/apps/` declares the six services (core, auth, web-ui, admin, portal,
+egress-proxy) as one Porter app per file, built from `deploy/*/Dockerfile` — `porter
+apply` takes exactly one app per invocation and silently ignores extra YAML documents,
+which is why they are separate files:
```bash
-porter apply -f porter/porter.yaml
+for f in porter/apps/*.yaml; do porter apply -f "$f"; done
```
-Porter builds the images, runs the services, and assigns each web service a hostname, so
-this path needs no DNS record, no TLS certificate, and no ingress controller. Set the
-service secrets from the Porter dashboard or `porter apply`'s env-group support, and use
-the portal's assigned hostname as `PUBLIC_WEB_URL`.
+Porter runs the services and assigns each **public** web service an `onporter.run`
+hostname with a Let's Encrypt certificate, so this path needs no DNS record, no TLS
+certificate, and no ingress controller. Only the portal is meant to be Internet-facing;
+auth, web-ui, and admin carry `private: true` so Porter creates no ingress for them and
+they are reachable only at their in-cluster address.
+
+Two things `porter apply` will not do for you:
+
+- **Building on an Apple Silicon machine fails.** `build.method: docker` builds locally
+ for `linux/amd64` and the legacy builder cannot cross-compile: the apply dies with
+ `image ... does not provide the specified platform (linux/amd64)`. Build and push with
+ `docker buildx build --platform linux/amd64 --push`, then point the app at the pushed
+ image with an `image: {repository, tag}` block, or use `porter apply --remote`.
+- **Env groups created by `porter env create` are not visible to `porter apply`.** Both
+ the `envGroups:` key and `--attach-env-groups` fail with `internal: unable to find
+latest environment with provided name`, because `porter env create` makes a
+ project-scoped group and apply resolves cluster-scoped ones. Put non-secret wiring in
+ the app's `env:` block and pass secrets with `--secrets KEY=value` (note that pflag
+ splits those on commas, so a JSON value like `AUTH_SIGNING_JWK` has to go in `env:`).
+
+Nothing generates the inter-service wiring for you — `cli/src/services.ts` has `fly` and
+`docker` targets but no Porter one — so set it by hand on each app:
+
+| Service | Wiring |
+| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| all but core | `CORE_API_URL=http://qm-core-core.<namespace>.svc.cluster.local:8080`, `CORE_ORG_ID`, `CORE_SIGNING_SECRET`, `PORTAL_IDENTITY_SECRET` |
+| core | `ORG_ID`, `MODEL_PROVIDER`, `HARNESS`, `SANDBOX_BACKEND`, `PUBLIC_WEB_URL`/`WEB_UI_PUBLIC_URL` (the portal's hostname), `CAPABILITY_SECRET`, `CONNECTOR_SECRET_KEY`, `SKILL_SIGNING_SECRET`, and `DATABASE_URL` when `SESSION_STORE`/`RUN_STORE` are `postgres` |
+| portal | `PORTAL_PUBLIC_URL`, `PORTAL_SESSION_SECRET`, `WEB_UI_UPSTREAM`, `ADMIN_UPSTREAM`, `AUTH_BROKER_UPSTREAM`, `AUTH_BROKER_PREFIX=/idp`, and the `OIDC_*` set from `brokerWiring` in `cli/src/services.ts` |
+| auth | `AUTH_ISSUER=<portal>/idp`, `AUTH_CLIENT_ID`, `AUTH_CLIENT_SECRET`, `AUTH_REDIRECT_URI=<portal>/auth/callback`, `AUTH_TOKEN_SECRET`, `AUTH_SIGNING_JWK` (a P-256 private JWK), and a `RESEND_API_KEY` or SMTP credentials — without a mail transport nobody can sign in |
+
+The portal's hostname is only knowable after its first apply, and core, portal, and auth
+all need it, so expect to apply twice: once to mint the hostname, once to wire it in.
+
+Porter's own Postgres datastores are created from the dashboard. An RDS instance in the
+cluster's VPC works too, but Postgres 17 defaults to `rds.force_ssl=1`, so the URL needs
+`?sslmode=no-verify` (or a CA pinned with `DATABASE_CA_CERT`) or core dies on `no pg_hba.conf
+entry ... no encryption`.
`deploy/helm/` carries the same topology as a Helm chart for operators who would rather
manage the release themselves; it expects the images published at
-`ghcr.io/yc-software/qm/<service>` and an ingress you configure.
+`ghcr.io/yc-software/qm/<service>` and an ingress you configure. Pin `image.tag` to a
+commit that actually carries the Porter backend — the published tags are commit SHAs, and
+one from before Porter support landed rejects `SANDBOX_BACKEND=porter` at startup.
## Giving published apps stable hostnames
Apps the agent deploys are reachable only if the cluster gives them an address. Without
one, `apply` fails with `the cluster assigned no hostname`, and core warns at startup.
+The address comes from Porter's **sandbox ingress**, which is separate from the ingress
+that serves Porter apps and is enabled per cluster from the dashboard. A cluster without
+it rejects every publish before DNS is ever consulted, with
+`HTTP 400: validation error: visibility: no private sandbox ingress is configured on this
+cluster` (`no public sandbox ingress` when `PORTER_DEPLOY_VISIBILITY=public`). Turn
+sandbox ingress on first; the wildcard record below is what gives the resulting apps
+stable names.
+
Point a wildcard DNS record at the cluster's ingress load balancer and name it:
```bash
@@ -57,6 +102,7 @@ deployment's domain into public ingress.
| `PORTER_DEPLOY_RUNNER_IMAGE` | Image published apps boot from (defaults to the sandbox image) |
| `PORTER_SANDBOX_EGRESS_PROXY_URL` | Forces sandbox traffic through the egress proxy; unset means no egress enforcement |
| `PORTER_SANDBOX_NAME_PREFIX` | Prefix for sandbox and app names on the cluster |
+| `PORTER_DEPLOY_VISIBILITY` | `public` puts a published app on public ingress; default is private |
| `PORTER_DEPLOY_TTL_SEC` / `PORTER_SANDBOX_TTL_SEC` | Reap bodies after this long |
To QA a branch against a real Porter cluster before deploying it, the dev instance takes
diff --git a/porter/apps/admin.yaml b/porter/apps/admin.yaml
new file mode 100644
index 000000000..d6b47b594
--- /dev/null
+++ b/porter/apps/admin.yaml
@@ -0,0 +1,15 @@
+version: v2
+name: qm-admin
+build:
+ method: docker
+ context: .
+ dockerfile: ./deploy/admin/Dockerfile
+services:
+ - name: admin
+ type: web
+ private: true
+ run: node src/index.ts
+ instances: 1
+ cpuCores: 0.25
+ ramMegabytes: 512
+ port: 8080
diff --git a/porter/apps/auth.yaml b/porter/apps/auth.yaml
new file mode 100644
index 000000000..485f213de
--- /dev/null
+++ b/porter/apps/auth.yaml
@@ -0,0 +1,15 @@
+version: v2
+name: qm-auth
+build:
+ method: docker
+ context: .
+ dockerfile: ./deploy/auth/Dockerfile
+services:
+ - name: auth
+ type: web
+ private: true
+ run: node src/index.ts
+ instances: 1
+ cpuCores: 0.25
+ ramMegabytes: 512
+ port: 8080
diff --git a/porter/apps/core.yaml b/porter/apps/core.yaml
new file mode 100644
index 000000000..508240a9a
--- /dev/null
+++ b/porter/apps/core.yaml
@@ -0,0 +1,17 @@
+version: v2
+name: qm-core
+build:
+ method: docker
+ context: .
+ dockerfile: ./deploy/core/Dockerfile
+services:
+ - name: core
+ type: web
+ run: node src/index.ts
+ instances: 1
+ cpuCores: 1
+ ramMegabytes: 2048
+ port: 8080
+ healthCheck:
+ enabled: true
+ httpPath: /healthz
diff --git a/porter/apps/egress-proxy.yaml b/porter/apps/egress-proxy.yaml
new file mode 100644
index 000000000..106be2f5b
--- /dev/null
+++ b/porter/apps/egress-proxy.yaml
@@ -0,0 +1,13 @@
+version: v2
+name: qm-egress-proxy
+build:
+ method: docker
+ context: .
+ dockerfile: ./deploy/egress-proxy/Dockerfile
+services:
+ - name: egress-proxy
+ type: worker
+ run: /app/start.sh
+ instances: 1
+ cpuCores: 0.25
+ ramMegabytes: 512
diff --git a/porter/apps/portal.yaml b/porter/apps/portal.yaml
new file mode 100644
index 000000000..e4064f6a6
--- /dev/null
+++ b/porter/apps/portal.yaml
@@ -0,0 +1,14 @@
+version: v2
+name: qm-portal
+build:
+ method: docker
+ context: .
+ dockerfile: ./deploy/portal/Dockerfile
+services:
+ - name: portal
+ type: web
+ run: node src/index.ts
+ instances: 1
+ cpuCores: 0.25
+ ramMegabytes: 512
+ port: 8080
diff --git a/porter/apps/web-ui.yaml b/porter/apps/web-ui.yaml
new file mode 100644
index 000000000..b1cc2a3ee
--- /dev/null
+++ b/porter/apps/web-ui.yaml
@@ -0,0 +1,15 @@
+version: v2
+name: qm-web-ui
+build:
+ method: docker
+ context: .
+ dockerfile: ./deploy/web-ui/Dockerfile
+services:
+ - name: web-ui
+ type: web
+ private: true
+ run: node server/index.ts
+ instances: 1
+ cpuCores: 0.25
+ ramMegabytes: 512
+ port: 8080
diff --git a/porter/porter.yaml b/porter/porter.yaml
deleted file mode 100644
index 0cea3f90d..000000000
--- a/porter/porter.yaml
+++ /dev/null
@@ -1,91 +0,0 @@
-version: v2
-name: qm-core
-build:
- method: docker
- context: .
- dockerfile: ./deploy/core/Dockerfile
-services:
- - name: core
- type: web
- run: node src/index.ts
- instances: 1
- cpuCores: 1
- ramMegabytes: 2048
- port: 8080
- healthCheck:
- enabled: true
- httpPath: /healthz
----
-version: v2
-name: qm-auth
-build:
- method: docker
- context: .
- dockerfile: ./deploy/auth/Dockerfile
-services:
- - name: auth
- type: web
- run: node src/index.ts
- instances: 1
- cpuCores: 0.25
- ramMegabytes: 512
- port: 8080
----
-version: v2
-name: qm-web-ui
-build:
- method: docker
- context: .
- dockerfile: ./deploy/web-ui/Dockerfile
-services:
- - name: web-ui
- type: web
- run: node server/index.ts
- instances: 1
- cpuCores: 0.25
- ramMegabytes: 512
- port: 8080
----
-version: v2
-name: qm-admin
-build:
- method: docker
- context: .
- dockerfile: ./deploy/admin/Dockerfile
-services:
- - name: admin
- type: web
- run: node src/index.ts
- instances: 1
- cpuCores: 0.25
- ramMegabytes: 512
- port: 8080
----
-version: v2
-name: qm-portal
-build:
- method: docker
- context: .
- dockerfile: ./deploy/portal/Dockerfile
-services:
- - name: portal
- type: web
- run: node src/index.ts
- instances: 1
- cpuCores: 0.25
- ramMegabytes: 512
- port: 8080
----
-version: v2
-name: qm-egress-proxy
-build:
- method: docker
- context: .
- dockerfile: ./deploy/egress-proxy/Dockerfile
-services:
- - name: egress-proxy
- type: worker
- run: /app/start.sh
- instances: 1
- cpuCores: 0.25
- ramMegabytes: 512
diff --git a/src/deploy/porter-deploy-provider.ts b/src/deploy/porter-deploy-provider.ts
index ccef57fcd..990037e8a 100644
--- a/src/deploy/porter-deploy-provider.ts
+++ b/src/deploy/porter-deploy-provider.ts
@@ -20,7 +20,7 @@ import { createMemoryMap, type DurableMap } from "../persistence/durable-map.ts"
import { createNoopAdvisoryLock, type AdvisoryLock } from "../persistence/advisory-lock.ts";
import { createKeyedQueue } from "../util/async.ts";
import { shq } from "../util/shell.ts";
-import { swallow } from "../util/errors.ts";
+import { errMessage, swallow } from "../util/errors.ts";
const APP_DIR = "/app";
const HOME_DIR = "/root";
@@ -159,16 +159,26 @@ export function createPorterDeployProvider(opts: PorterDeployProviderOptions): D
await retireBodies(d, true);
await store.delete(d.id).catch((e) => swallow("porter-deploy: clear stale pointer", e));
const name = `${baseName(d)}-${randomUUID().slice(0, 5)}`;
- const sb = await client.sandboxes.create({
- image,
- name,
- command: ["sleep", "infinity"],
- tags: { [KIND_TAG]: "app", [DEPLOY_TAG]: d.id },
- env: appEnv(version),
- volume_mounts: { [DATA_DIR]: volumeId },
- networking: [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],
- ...(opts.ttlSec ? { ttl_seconds: opts.ttlSec } : {}),
- });
+ const sb = await client.sandboxes
+ .create({
+ image,
+ name,
+ command: ["sleep", "infinity"],
+ tags: { [KIND_TAG]: "app", [DEPLOY_TAG]: d.id },
+ env: appEnv(version),
+ volume_mounts: { [DATA_DIR]: volumeId },
+ networking: [{ port: appPort, ...(domainOf(d) ? { domains: [{ domain: domainOf(d), visibility }] } : {}) }],
+ ...(opts.ttlSec ? { ttl_seconds: opts.ttlSec } : {}),
+ })
+ .catch((e) => {
+ if (errMessage(e).includes("sandbox ingress")) {
+ throw new Error(
+ `porter deploy ${d.id}: the cluster refused the app's ${visibility} domain because sandbox ingress is not enabled — turn it on for this cluster in the Porter dashboard, then point PORTER_DEPLOY_APPS_DOMAIN's wildcard DNS record at it (${errMessage(e)})`,
+ { cause: e },
+ );
+ }
+ throw e;
+ });
try {
await waitPorterRunning(name, sb);
const host = (await sb.refresh()).host || domainOf(d);