Multiplayer agent harness for work.
fix: make Helm storage and readiness explicit (#1660)
Joshua France8adee4b06a59
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.ymlindex b63bbef42..2aacac471 100644--- a/.github/workflows/cicd.yml+++ b/.github/workflows/cicd.yml@@ -48,6 +48,14 @@ jobs:run: npm ci- name: Build connector SDKrun: npm run build:connector-sdk+ - name: Install Helm for chart tests+ run: |+ cd "$(mktemp -d)"+ curl -fsSLO https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz+ curl -fsSLO https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz.sha256sum+ sha256sum -c helm-v3.19.0-linux-amd64.tar.gz.sha256sum+ tar -xzf helm-v3.19.0-linux-amd64.tar.gz+ sudo install linux-amd64/helm /usr/local/bin/helm- name: Verify root test shard planif: matrix.shard == 1run: npm run test:root:shard:checkdiff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yamlindex 96e0928ce..cddc528cd 100644--- a/deploy/helm/Chart.yaml+++ b/deploy/helm/Chart.yaml@@ -1,6 +1,5 @@apiVersion: v2name: qm-description: Deploys the qm surfaces (core, auth, web-ui, admin, portal, egress-proxy) built from deploy/*/Dockerfile.+description: Deploys qm core, web-ui (with admin), portal (with auth), and egress-proxy.type: application-version: 0.2.7-appVersion: "1.0.5"+version: 0.2.8diff --git a/deploy/helm/README.md b/deploy/helm/README.mdnew file mode 100644index 000000000..147410a04--- /dev/null+++ b/deploy/helm/README.md@@ -0,0 +1,151 @@+# QM Helm chart++This chart runs core, web-ui (including `/admin`), portal (including `/idp` auth),+and the optional egress proxy. `services.auth` and `services.admin` configure+embedded modules, not separate Deployments or Services.++## Images++There is deliberately no default image tag. Build all enabled services from the+same checked-out QM revision that supplies this chart, or independently verify a+release's images include the combined topology and core `/readyz` endpoint. Old+standalone auth/admin images are not compatible. The chart version is **not** an+application image version.++For example, from the repository root, with a registry you own and are signed into:++```bash+export REGISTRY=registry.example.com/your-project/qm+export TAG=$(git rev-parse HEAD)+for service in core web-ui portal egress-proxy; do+ docker buildx build --platform linux/amd64 --push \+ --build-arg GIT_SHA="$TAG" \+ -f "deploy/$service/Dockerfile" -t "$REGISTRY/$service:$TAG" .+done+cat > images.yaml <<EOF_IMAGES+image:+ repository: $REGISTRY+ tag: "$TAG"+EOF_IMAGES+```++Use the architecture of your cluster. A commit tag identifies the build only if+that checkout is clean; use a unique tag when testing uncommitted changes. For+immutable deployments, set each enabled service's `imageRef` to its full registry+reference, such as `registry.example.com/your-project/qm/core@sha256:<digest>`.+`imageRef` takes precedence over repository/tag composition. Otherwise+`services.<name>.tag` overrides `image.tag`. Configure `imagePullSecrets` for a+private registry. Neither this chart nor these examples publish an official release.++## Cluster setup++Bring these before installing:++- Kubernetes, Helm 3, a reachable durable Postgres database, and backups.+- A supported sandbox backend and its credentials. Kubernetes hosting does not+ configure sandbox provisioning or app hosting automatically.+- DNS and an ingress controller. For TLS, provision the referenced certificate or+ install cert-manager with the configured ClusterIssuer.+- The signing/encryption secrets, portal session secret, first administrator grant,+ sign-in allowlist, auth client/signing configuration and email transport described+ in [the deployment reference](../../docs/porter.md) and+ [`src/deployment/secret-schema.ts`](../../src/deployment/secret-schema.ts).++Store credentials in a Kubernetes Secret and use `envFrom`, or supply `secretEnv`+through a protected values file. Helm stores `secretEnv` values in its release+history. Never commit these values. With `envFrom`, explicitly provide both+`AUTH_ALLOWED_EMAILS` and `OIDC_ALLOWED_EMAILS` (or both corresponding domain+variables); Helm cannot inspect externally supplied Secret values. When `envFrom`+is present, the chart does not derive OIDC allowlist environment entries from+`env` or `services.auth.env`, since those entries would override a potentially+narrower policy in an external Secret. Set OIDC explicitly in values or the external+Secret, including when mixing auth values with `envFrom`. Without `envFrom`, auth+allowlists in `env` or `services.auth.env` propagate unless explicit OIDC values+in `env` or `secretEnv` override them. Auth allowlists in `secretEnv` are aliased+inside the chart-managed Secret; later `envFrom` sources retain their precedence.++Only portal should normally have public ingress. `ingress.service` accepts an+enabled `portal`, `core` or `web-ui`, not embedded modules or the TCP egress proxy.+Exposing core or web-ui is an operator choice, not required for the portal setup.++A minimal **Kubernetes configuration fragment**, not a complete installation:++```yaml+publicUrl: https://qm.example.com+imagePullSecrets:+ - name: registry-credentials+envFrom:+ - secretRef:+ name: qm-runtime+ingress:+ enabled: true+ service: portal+ hosts: [qm.example.com]+ className: nginx+ clusterIssuer: letsencrypt-prod+services:+ core:+ persistence:+ enabled: true+ storageClass: null+ size: 10Gi+```++Confirm that storage class exists in your cluster. After supplying runtime config+and `images.yaml`, render and inspect before applying:++```bash+helm lint deploy/helm -f images.yaml -f cluster.yaml+helm template qm deploy/helm -f images.yaml -f cluster.yaml+helm upgrade --install qm deploy/helm --namespace qm --create-namespace \+ -f images.yaml -f cluster.yaml --wait --timeout 10m+```++Validate the complete installation: test your database, ingress/TLS,+sign-in, real turns, sandbox provider and backup/restore in your target cluster.++## Core data and rollout behavior++Persistence is **opt-in**. Without it, core's local files (including uploaded+artifacts) are lost when its pod is replaced, even with Postgres configured.+`services.core.persistence.enabled=true` mounts a ReadWriteOnce PVC at `/data`,+sets `DATA_DIR=/data` and uses group 1000 for the Node image's volume permissions.+Storage drivers must support `fsGroup`, or an existing volume must already be+writable by UID/GID 1000.++Persistent core supports zero or one replica, with `Recreate` deployment strategy+so upgrades do not mount the same data concurrently. Expect downtime on upgrades;+shared-storage multi-replica operation is not supported. `replicas: 0` scales down+without discarding the claim.++The chart-created `<fullname>-core-data` claim has `helm.sh/resource-policy: keep`;+it is retained on uninstall or when persistence is removed. It is not a backup.+Record the actual claim name before uninstalling. Reinstall with+`services.core.persistence.existingClaim=<name>` and persistence enabled to reuse+it, rather than trying to create the retained claim again. Existing claims are+never created or deleted by this chart. Delete retained data manually only when+you intend to discard it. A null `storageClass` selects the cluster default; `""`+requests no storage class. Existing-claim size and class are managed externally.++`services.<name>.port` controls the process, Service and probes; conflicting+`PORT` values are rejected. The egress-proxy image's Envoy listener is fixed at+48080, so other proxy ports are rejected rather than rendered as broken Services.++Core liveness uses `healthPath: /healthz`; readiness uses+`readinessPath: /readyz`. Its startup probe allows five minutes for initialization+before liveness begins. Override `startupProbe` fields as needed, or set it to null+to disable. Helm merges probe maps, so set `startupProbe.httpGet: null` when+switching to an `exec` or `tcpSocket` handler. When overriding endpoint paths,+update the startup probe too. Database unavailability should remove core from+endpoints, not restart it through liveness.++## Regression tests++Install Helm 3 on PATH (or set `HELM_BIN`) and run:++```bash+node --test test/helm-chart.test.ts+```++These tests render real manifests and parse them with Helm's YAML parser. Missing+Helm is a local skip and a CI failure. They do not replace live Kubernetes tests.diff --git a/deploy/helm/templates/_helpers.tpl b/deploy/helm/templates/_helpers.tplindex f00de7d1f..2fc38ae57 100644--- a/deploy/helm/templates/_helpers.tpl+++ b/deploy/helm/templates/_helpers.tpl@@ -44,3 +44,15 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}{{- default "default" .Values.serviceAccount.name -}}{{- end -}}{{- end -}}++{{- define "qm.image" -}}+{{- if .service.imageRef -}}+{{- .service.imageRef -}}+{{- else -}}+{{- $tag := .service.tag | default .root.Values.image.tag -}}+{{- if not $tag -}}+{{- fail (printf "Set services.%s.imageRef (full tag or digest reference) or image.tag to images built from this checkout; see deploy/helm/README.md" .name) -}}+{{- end -}}+{{- printf "%s%s%s:%s" .root.Values.image.repository (.root.Values.image.separator | default "/") .service.image $tag -}}+{{- end -}}+{{- end -}}diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yamlindex fd098b535..44e9c699e 100644--- a/deploy/helm/templates/deployment.yaml+++ b/deploy/helm/templates/deployment.yaml@@ -15,7 +15,15 @@ metadata:{{- include "qm.labels" $root | nindent 4 }}app.kubernetes.io/component: {{ $name }}spec:- replicas: {{ $svc.replicas | default 1 }}+ replicas: {{ $svc.replicas }}+ {{- if and (eq $name "core") $svc.persistence.enabled }}+ {{- if gt (int $svc.replicas) 1 }}+ {{- fail "core persistence supports at most one replica; shared-storage multi-replica operation is not supported" }}+ {{- end }}+ strategy:+ type: Recreate+ rollingUpdate: null+ {{- end }}selector:matchLabels:{{- include "qm.selectorLabels" $root | nindent 6 }}@@ -33,9 +41,23 @@ spec:{{- toYaml . | nindent 8 }}{{- end }}serviceAccountName: {{ include "qm.serviceAccountName" $root }}+ {{- if and (eq $name "core") $svc.persistence.enabled }}+ securityContext:+ fsGroup: 1000+ fsGroupChangePolicy: OnRootMismatch+ volumes:+ - name: data+ persistentVolumeClaim:+ claimName: {{ $svc.persistence.existingClaim | default (printf "%s-core-data" (include "qm.fullname" $root)) | quote }}+ {{- end }}containers:- name: {{ $name }}- image: "{{ $root.Values.image.repository }}{{ $root.Values.image.separator | default "/" }}{{ $svc.image }}:{{ $svc.tag | default $root.Values.image.tag | default $root.Chart.AppVersion }}"+ image: {{ include "qm.image" (dict "root" $root "name" $name "service" $svc) | quote }}+ {{- if and (eq $name "core") $svc.persistence.enabled }}+ volumeMounts:+ - name: data+ mountPath: /data+ {{- end }}imagePullPolicy: {{ $root.Values.image.pullPolicy }}{{- with $svc.command }}command:@@ -77,9 +99,6 @@ spec:{{- $_ := set $wired "OIDC_JWKS_URI" (printf "%s/.well-known/jwks.json" $authUpstream) }}{{- $_ := set $wired "OIDC_SCOPES" "openid email" }}{{- $_ := set $wired "OIDC_PRINCIPAL_CLAIM" "email" }}- {{- with $root.Values.env.AUTH_ALLOWED_EMAIL_DOMAIN }}- {{- $_ := set $wired "OIDC_ALLOWED_EMAIL_DOMAIN" . }}- {{- end }}{{- end }}{{- end }}{{- if and (eq $name "portal") $authEnabled }}@@ -109,6 +128,31 @@ spec:{{- end }}{{- end }}{{- $env := mergeOverwrite (deepCopy $wired) (deepCopy ($root.Values.env | default dict)) $componentEnv }}+ {{- if eq $name "egress-proxy" }}+ {{- if ne (int $svc.port) 48080 }}+ {{- fail "services.egress-proxy.port must be 48080: the Envoy listener is fixed in the image" }}+ {{- end }}+ {{- else }}+ {{- if and (hasKey $env "PORT") (ne (toString (index $env "PORT")) (toString $svc.port)) }}+ {{- fail (printf "%s PORT conflicts with services.%s.port; configure the port there" $name $name) }}+ {{- end }}+ {{- $_ := set $env "PORT" (toString $svc.port) }}+ {{- end }}+ {{- if and (eq $name "core") $svc.persistence.enabled }}+ {{- if and (hasKey $env "DATA_DIR") (ne (index $env "DATA_DIR") "/data") }}+ {{- fail "core persistence requires DATA_DIR=/data" }}+ {{- end }}+ {{- $_ := set $env "DATA_DIR" "/data" }}+ {{- end }}+ {{- if and (eq $name "portal") $authEnabled (not $root.Values.envFrom) }}+ {{- range $authKey, $oidcKey := dict "AUTH_ALLOWED_EMAILS" "OIDC_ALLOWED_EMAILS" "AUTH_ALLOWED_EMAIL_DOMAIN" "OIDC_ALLOWED_EMAIL_DOMAIN" }}+ {{- if and (not (hasKey $env $oidcKey)) (not (index $root.Values.secretEnv $oidcKey)) }}+ {{- if hasKey $env $authKey }}+ {{- $_ := set $env $oidcKey (index $env $authKey) }}+ {{- end }}+ {{- end }}+ {{- end }}+ {{- end }}{{- if $env }}env:{{- range $k, $v := $env }}@@ -126,10 +170,15 @@ spec:{{- toYaml . | nindent 12 }}{{- end }}{{- end }}+ {{- if $svc.startupProbe }}+ startupProbe:+ {{- toYaml $svc.startupProbe | nindent 12 }}+ {{- end }}{{- if $svc.healthPath }}readinessProbe:+ timeoutSeconds: 2httpGet:- path: {{ $svc.healthPath }}+ path: {{ $svc.readinessPath | default $svc.healthPath }}port: {{ $svc.port }}livenessProbe:httpGet:diff --git a/deploy/helm/templates/ingress.yaml b/deploy/helm/templates/ingress.yamlindex e6eca2bf9..19c195c49 100644--- a/deploy/helm/templates/ingress.yaml+++ b/deploy/helm/templates/ingress.yaml@@ -1,6 +1,12 @@{{- if .Values.ingress.enabled }}{{- $svcKey := .Values.ingress.service }}{{- $svc := index .Values.services $svcKey }}+{{- if or (not $svc) (not (has $svcKey (list "portal" "core" "web-ui"))) }}+{{- fail "ingress.service must name an enabled HTTP service: portal, core, or web-ui (auth and admin are embedded)" }}+{{- end }}+{{- if not $svc.enabled }}+{{- fail (printf "ingress.service %s is disabled" $svcKey) }}+{{- end }}{{- $svcName := printf "%s-%s" (include "qm.fullname" .) $svcKey }}apiVersion: networking.k8s.io/v1kind: Ingressdiff --git a/deploy/helm/templates/pvc.yaml b/deploy/helm/templates/pvc.yamlnew file mode 100644index 000000000..f5b4938e5--- /dev/null+++ b/deploy/helm/templates/pvc.yaml@@ -0,0 +1,20 @@+{{- $core := .Values.services.core }}+{{- if and $core.enabled $core.persistence.enabled (not $core.persistence.existingClaim) }}+apiVersion: v1+kind: PersistentVolumeClaim+metadata:+ name: {{ include "qm.fullname" . }}-core-data+ labels:+ {{- include "qm.labels" . | nindent 4 }}+ annotations:+ helm.sh/resource-policy: keep+spec:+ accessModes:+ - ReadWriteOnce+ {{- if ne $core.persistence.storageClass nil }}+ storageClassName: {{ $core.persistence.storageClass | quote }}+ {{- end }}+ resources:+ requests:+ storage: {{ $core.persistence.size | quote }}+{{- end }}diff --git a/deploy/helm/templates/secret.yaml b/deploy/helm/templates/secret.yamlindex 0bb9ad04d..764325c13 100644--- a/deploy/helm/templates/secret.yaml+++ b/deploy/helm/templates/secret.yaml@@ -18,6 +18,9 @@ stringData:{{- if and (not .Values.secretEnv.OIDC_CLIENT_SECRET) .Values.secretEnv.AUTH_CLIENT_SECRET }}OIDC_CLIENT_SECRET: {{ .Values.secretEnv.AUTH_CLIENT_SECRET | quote }}{{- end }}+ {{- if and (not .Values.secretEnv.OIDC_ALLOWED_EMAIL_DOMAIN) .Values.secretEnv.AUTH_ALLOWED_EMAIL_DOMAIN }}+ OIDC_ALLOWED_EMAIL_DOMAIN: {{ .Values.secretEnv.AUTH_ALLOWED_EMAIL_DOMAIN | quote }}+ {{- end }}{{- if and (not .Values.secretEnv.OIDC_ALLOWED_EMAILS) .Values.secretEnv.AUTH_ALLOWED_EMAILS }}OIDC_ALLOWED_EMAILS: {{ .Values.secretEnv.AUTH_ALLOWED_EMAILS | quote }}{{- end }}diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yamlindex 31b7f1495..ae696eec6 100644--- a/deploy/helm/values.yaml+++ b/deploy/helm/values.yaml@@ -70,6 +70,19 @@ services:replicas: 1command: ["node", "src/index.ts"]healthPath: /healthz+ readinessPath: /readyz+ startupProbe:+ httpGet:+ path: /readyz+ port: http+ periodSeconds: 5+ timeoutSeconds: 2+ failureThreshold: 60+ persistence:+ enabled: false+ existingClaim: ""+ size: 10Gi+ storageClass: nullresources:requests:cpu: 500mdiff --git a/package.json b/package.jsonindex a2d269c9f..f0c993594 100644--- a/package.json+++ b/package.json@@ -46,7 +46,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node --test --test-concurrency=1 test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-migration-startup.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",+ "test:pg": "node --test --test-concurrency=1 test/readiness.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-migration-startup.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/src/api/deps.ts b/src/api/deps.tsindex fca749234..fa29ed789 100644--- a/src/api/deps.ts+++ b/src/api/deps.ts@@ -80,6 +80,7 @@ import type { SlackAccountLink, ComposioReturn } from "./routes/composio.ts";export interface ServerDeps {externalSlackPolicies?: ExternalSlackPolicies;+ checkReadiness?: (signal: AbortSignal) => Promise<void>;slackAccounts?: DurableMap<SlackAccountLink>;composioReturns?: DurableMap<ComposioReturn>;composioFetch?: typeof fetch;diff --git a/src/api/routes/index.ts b/src/api/routes/index.tsindex 56af1e4ee..c70aab347 100644--- a/src/api/routes/index.ts+++ b/src/api/routes/index.ts@@ -43,6 +43,29 @@ import { swarmRoutes } from "./swarms.ts";export const rawRoutes: ReadonlyArray<Route<BaseCtx>> = [{ method: "GET", path: "/healthz", auth: "public", handle: ({ res }) => sendJson(res, 200, { ok: true }) },+ {+ method: "GET",+ path: "/readyz",+ auth: "public",+ handle: async ({ res, deps }) => {+ const controller = new AbortController();+ let timer: ReturnType<typeof setTimeout> | undefined;+ const deadline = new Promise<never>((_, reject) => {+ timer = setTimeout(() => {+ controller.abort();+ reject(new Error("Readiness deadline exceeded"));+ }, 1000);+ });+ try {+ await Promise.race([Promise.resolve().then(() => deps.checkReadiness?.(controller.signal)), deadline]);+ sendJson(res, 200, { ok: true });+ } catch {+ sendJson(res, 503, { ok: false });+ } finally {+ clearTimeout(timer);+ }+ },+ },...slackEventRawRoutes,{match: (m, p) => (m === "GET" || m === "POST") && p.startsWith(GIT_HTTP_BROKER_PREFIX),diff --git a/src/api/server.ts b/src/api/server.tsindex f29f500b9..1465f3d40 100644--- a/src/api/server.ts+++ b/src/api/server.ts@@ -519,7 +519,10 @@ function buildServer(app: App, deps: ServerOptions, allowUnsignedSourceAuth: boo};const requestNames = new WeakMap<IncomingMessage, string>();const server = createHttpServer((req, res) => {- const finishTiming = req.url === "/healthz" ? undefined : startTiming("http.server", `${req.method ?? "GET"} /*`);+ const finishTiming =+ req.url === "/healthz" || req.url === "/readyz"+ ? undefined+ : startTiming("http.server", `${req.method ?? "GET"} /*`);if (finishTiming)res.once("close", () =>finishTiming({diff --git a/src/wiring.ts b/src/wiring.tsindex 49e35defb..d2ec9d711 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -460,6 +460,7 @@ export function stopWithBackstop(}export interface BuiltApp {+ checkReadiness: (signal: AbortSignal) => Promise<void>;backgroundOwnership?: { store: BackgroundOwnershipStore; instanceId: string; deploymentId: string };suggestedActivityMaintenance: Sweeper;suggestedActivities?: ReturnType<typeof createSuggestedActivityService>;@@ -2732,6 +2733,9 @@ export function buildApp(return {app,+ checkReadiness: async (signal) => {+ await pgArtifactMap?.pool.q("SELECT 1", [], { signal });+ },...(screenSecurity ? { screenSecurity } : {}),deploymentLayer,deploymentLayerStore,@@ -2844,6 +2848,7 @@ export function serverDeps(return {externalSlackPolicies: config.externalSlackPolicies,production: config.production,+ checkReadiness: built.checkReadiness,...(built.backgroundOwnership? {backgroundOwnership: built.backgroundOwnership,diff --git a/test/helm-chart.test.ts b/test/helm-chart.test.tsnew file mode 100644index 000000000..34bf1d130--- /dev/null+++ b/test/helm-chart.test.ts@@ -0,0 +1,260 @@+import assert from "node:assert/strict";+import { spawnSync } from "node:child_process";+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import { test } from "node:test";++const chart = fileURLToPath(new URL("../deploy/helm", import.meta.url));+const helm = process.env.HELM_BIN || "helm";+const helmAvailable = spawnSync(helm, ["version", "--short"]).status === 0;+if (process.env.CI) assert.ok(helmAvailable, "Install Helm 3 before running the chart regression tests in CI");+const options = { skip: !helmAvailable && "Helm 3 is required" };++type Resource = {+ kind: string;+ stringData?: Record<string, string>;+ metadata: { name: string; annotations?: Record<string, string> };+ spec: {+ replicas: number;+ strategy?: { type: string; rollingUpdate?: null };+ accessModes?: string[];+ storageClassName?: string;+ resources?: { requests: { storage: string } };+ ports?: { port: number; targetPort: number }[];+ rules?: { http: { paths: { backend: { service: { name: string; port: { number: number } } } }[] } }[];+ template: {+ spec: {+ securityContext?: { fsGroup: number };+ volumes?: { name: string; persistentVolumeClaim: { claimName: string } }[];+ containers: {+ image: string;+ env: { name: string; value: string }[];+ ports: { containerPort: number; name: string }[];+ volumeMounts?: { name: string; mountPath: string }[];+ readinessProbe: { httpGet: { path: string; port: number } };+ livenessProbe: { httpGet: { path: string; port: number } };+ startupProbe?: { httpGet: { path: string; port: string }; failureThreshold: number };+ }[];+ };+ };+ };+};++function render(values: object = {}, explicitImages = true): Resource[] {+ const dir = mkdtempSync(join(tmpdir(), "qm-helm-test-"));+ try {+ const valuesFile = join(dir, "values.json");+ writeFileSync(valuesFile, JSON.stringify(values));+ const result = spawnSync(+ helm,+ ["template", "test", chart, ...(explicitImages ? ["--set", "image.tag=fixture"] : []), "-f", valuesFile],+ { encoding: "utf8" },+ );+ assert.equal(result.status, 0, result.stderr);+ // Use Helm's YAML parser rather than require a second YAML implementation.+ writeFileSync(join(dir, "Chart.yaml"), "apiVersion: v2\nname: parser\nversion: 0.1.0\n");+ writeFileSync(join(dir, "rendered.yaml"), result.stdout);+ mkdirSync(join(dir, "templates"));+ writeFileSync(+ join(dir, "templates", "parse.yaml"),+ '{{- range splitList "\\n---\\n" (.Files.Get "rendered.yaml") }}\n---\n{{ . | fromYaml | toJson }}\n{{- end }}\n',+ );+ const parsed = spawnSync(helm, ["template", "parser", dir], { encoding: "utf8" });+ assert.equal(parsed.status, 0, parsed.stderr);+ return parsed.stdout+ .split("\n")+ .filter((line) => line.startsWith("{"))+ .map((line) => JSON.parse(line) as Resource);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+}++function deployment(resources: Resource[], name = "core") {+ const resource = resources.find((r) => r.kind === "Deployment" && r.metadata.name === `test-qm-${name}`);+ assert.ok(resource, `missing ${name} deployment`);+ return resource;+}++function container(resources: Resource[], name = "core") {+ const result = deployment(resources, name).spec.template.spec.containers[0];+ assert.ok(result, `missing ${name} container`);+ return result;+}++function env(resources: Resource[], name = "portal") {+ return {+ ...resources.find((r) => r.kind === "Secret" && r.metadata.name === "test-qm-env")?.stringData,+ ...Object.fromEntries((container(resources, name).env ?? []).map((e) => [e.name, e.value])),+ };+}++test("Helm requires an explicit compatible image and supports full digest references", options, () => {+ assert.throws(() => render({}, false), /imageRef.*image.tag/);+ const ref = `registry.example.com/qm/core@sha256:${"a".repeat(64)}`;+ const resources = render({ services: { core: { imageRef: ref }, portal: { tag: "portal-fixture" } } });+ assert.equal(container(resources).image, ref);+ assert.equal(container(resources, "portal").image, "ghcr.io/yc-software/qm/portal:portal-fixture");+ assert.equal(resources.filter((r) => r.kind === "Deployment").length, 4);+ const byReference = render(+ {+ services: Object.fromEntries(+ ["core", "portal", "web-ui", "egress-proxy"].map((name) => [name, { imageRef: ref }]),+ ),+ },+ false,+ );+ assert.ok(+ byReference+ .filter((r) => r.kind === "Deployment")+ .every((r) => container(byReference, r.metadata.name.replace("test-qm-", "")).image === ref),+ );+});++test("Helm mounts a retained core PVC with node group permissions and no overlapping pods", options, () => {+ const resources = render({+ services: { core: { persistence: { enabled: true, size: "20Gi", storageClass: "block" } } },+ });+ const core = deployment(resources);+ const pvc = resources.find((r) => r.kind === "PersistentVolumeClaim");+ assert.ok(pvc);+ assert.equal(pvc.metadata.annotations?.["helm.sh/resource-policy"], "keep");+ assert.deepEqual(pvc.spec.accessModes, ["ReadWriteOnce"]);+ assert.equal(pvc.spec.storageClassName, "block");+ assert.equal(pvc.spec.resources?.requests.storage, "20Gi");+ assert.equal(core.spec.strategy?.type, "Recreate");+ assert.equal(core.spec.strategy?.rollingUpdate, null);+ assert.equal(core.spec.template.spec.securityContext?.fsGroup, 1000);+ assert.equal(core.spec.template.spec.volumes?.[0]?.persistentVolumeClaim.claimName, pvc.metadata.name);+ assert.deepEqual(container(resources).volumeMounts, [{ name: "data", mountPath: "/data" }]);+ assert.equal(env(resources, "core").DATA_DIR, "/data");+ assert.throws(+ () => render({ services: { core: { replicas: 2, persistence: { enabled: true } } } }),+ /at most one replica/,+ );+ assert.throws(+ () => render({ services: { core: { env: { DATA_DIR: "/wrong" }, persistence: { enabled: true } } } }),+ /DATA_DIR=\/data/,+ );+});++test("Helm supports existing claims, default or empty storage class, and opting out", options, () => {+ const resources = render({ services: { core: { persistence: { enabled: true, existingClaim: "restored-data" } } } });+ assert.ok(!resources.some((r) => r.kind === "PersistentVolumeClaim"));+ assert.equal(deployment(resources).spec.template.spec.volumes?.[0]?.persistentVolumeClaim.claimName, "restored-data");+ for (const storageClass of [null, ""]) {+ const pvc = render({ services: { core: { persistence: { enabled: true, storageClass } } } }).find(+ (r) => r.kind === "PersistentVolumeClaim",+ );+ assert.ok(pvc);+ assert.equal(pvc.spec.storageClassName, storageClass === null ? undefined : "");+ }+ const disabled = render();+ assert.ok(!disabled.some((r) => r.kind === "PersistentVolumeClaim"));+ assert.equal(deployment(disabled).spec.template.spec.volumes, undefined);+});++test("Helm preserves zero replicas including a stopped persistent core", options, () => {+ const resources = render({+ services: { core: { replicas: 0, persistence: { enabled: true } }, portal: { replicas: 0 } },+ });+ assert.equal(deployment(resources).spec.replicas, 0);+ assert.equal(deployment(resources, "portal").spec.replicas, 0);+});++test("Helm keeps process ports, probes, Services and upstreams consistent", options, () => {+ const resources = render({ services: { core: { port: 9090 }, portal: { port: 9091 }, "web-ui": { port: 9092 } } });+ for (const [name, port] of [+ ["core", 9090],+ ["portal", 9091],+ ["web-ui", 9092],+ ] as const) {+ assert.equal(env(resources, name).PORT, String(port));+ assert.equal(container(resources, name).ports[0]?.containerPort, port);+ const service = resources.find((r) => r.kind === "Service" && r.metadata.name === `test-qm-${name}`);+ assert.equal(service?.spec.ports?.[0]?.targetPort, port);+ assert.equal(service?.spec.ports?.[0]?.port, port);+ }+ assert.equal(container(resources).readinessProbe.httpGet.port, 9090);+ assert.equal(env(resources).CORE_API_URL, "http://test-qm-core.default.svc.cluster.local:9090");+ assert.equal(env(resources).WEB_UI_UPSTREAM, "http://test-qm-web-ui.default.svc.cluster.local:9092");+ assert.throws(() => render({ services: { core: { env: { PORT: "9999" } } } }), /PORT conflicts/);+ assert.throws(() => render({ services: { "egress-proxy": { port: 49000 } } }), /Envoy listener is fixed/);+});++test("Helm separates core startup/readiness from liveness and allows probe overrides", options, () => {+ const core = container(render());+ assert.equal(core.readinessProbe.httpGet.path, "/readyz");+ assert.equal(core.livenessProbe.httpGet.path, "/healthz");+ assert.equal(core.startupProbe?.httpGet.path, "/readyz");+ assert.equal(core.startupProbe?.httpGet.port, "http");+ assert.equal(core.startupProbe?.failureThreshold, 60);+ const custom = container(render({ services: { core: { readinessPath: "/custom-ready", startupProbe: null } } }));+ assert.equal(custom.readinessProbe.httpGet.path, "/custom-ready");+ assert.equal(custom.livenessProbe.httpGet.path, "/healthz");+ assert.equal(custom.startupProbe, undefined);+});++test("Helm ingress rejects embedded, missing, worker and disabled targets", options, () => {+ for (const service of ["auth", "admin", "egress-proxy", "missing"]) {+ assert.throws(() => render({ ingress: { enabled: true, service } }), /ingress.service must name/);+ }+ assert.throws(+ () => render({ ingress: { enabled: true }, services: { portal: { enabled: false }, auth: { enabled: false } } }),+ /portal is disabled/,+ );+ const ingress = render({ ingress: { enabled: true, hosts: ["qm.example.com"] } }).find((r) => r.kind === "Ingress");+ assert.equal(ingress?.spec.rules?.[0]?.http.paths[0]?.backend.service.name, "test-qm-portal");+});++test("Helm propagates the effective embedded auth allowlist without overriding portal policy", options, () => {+ const auth = { AUTH_ALLOWED_EMAILS: "[email protected]", AUTH_ALLOWED_EMAIL_DOMAIN: "example.com" };+ for (const values of [{ env: auth }, { services: { auth: { env: auth } } }, { secretEnv: auth }]) {+ const portal = env(render(values));+ assert.equal(portal.OIDC_ALLOWED_EMAILS, auth.AUTH_ALLOWED_EMAILS);+ assert.equal(portal.OIDC_ALLOWED_EMAIL_DOMAIN, auth.AUTH_ALLOWED_EMAIL_DOMAIN);+ }+ const secretBacked = render({ secretEnv: auth });+ assert.ok(!container(secretBacked, "portal").env.some((e) => e.name === "OIDC_ALLOWED_EMAILS"));+ const values = { env: { AUTH_ALLOWED_EMAILS: "[email protected]" }, services: { auth: { env: auth } } };+ assert.equal(env(render(values)).OIDC_ALLOWED_EMAILS, auth.AUTH_ALLOWED_EMAILS);+ const explicit = { OIDC_ALLOWED_EMAILS: "[email protected]", OIDC_ALLOWED_EMAIL_DOMAIN: "example.org" };+ for (const overrides of [+ { env: explicit },+ { secretEnv: explicit },+ { services: { auth: { env: auth }, portal: { env: explicit } } },+ ]) {+ const portal = env(render({ services: { auth: { env: auth } }, ...overrides }));+ assert.equal(portal.OIDC_ALLOWED_EMAILS, explicit.OIDC_ALLOWED_EMAILS);+ assert.equal(portal.OIDC_ALLOWED_EMAIL_DOMAIN, explicit.OIDC_ALLOWED_EMAIL_DOMAIN);+ }+ assert.equal(env(render({ services: { auth: { enabled: false } }, env: auth })).OIDC_ALLOWED_EMAILS, undefined);+});++test("Helm does not shadow an external OIDC allowlist with a derived value", options, () => {+ const envFrom = [{ secretRef: { name: "narrow-oidc-policy" } }];+ for (const [authKey, oidcKey, broad, narrow] of [+ ["AUTH_ALLOWED_EMAILS", "OIDC_ALLOWED_EMAILS", "[email protected],[email protected]", "[email protected]"],+ ["AUTH_ALLOWED_EMAIL_DOMAIN", "OIDC_ALLOWED_EMAIL_DOMAIN", "example.com", "staff.example.com"],+ ] as const) {+ for (const source of [{ env: { [authKey]: broad } }, { services: { auth: { env: { [authKey]: broad } } } }]) {+ assert.ok(!container(render({ ...source, envFrom }), "portal").env.some((entry) => entry.name === oidcKey));+ assert.equal(env(render({ ...source, envFrom, secretEnv: { [oidcKey]: narrow } }))[oidcKey], narrow);+ assert.equal(+ env(+ render({+ ...source,+ envFrom,+ services: { auth: { env: { [authKey]: broad } }, portal: { env: { [oidcKey]: narrow } } },+ }),+ )[oidcKey],+ narrow,+ );+ }+ assert.equal(env(render({ envFrom, env: { [authKey]: broad, [oidcKey]: narrow } }))[oidcKey], narrow);+ }+ const externalOnly = container(render({ envFrom }), "portal").env;+ assert.ok(!externalOnly.some((entry) => entry.name.startsWith("OIDC_ALLOWED_")));+});diff --git a/test/readiness.test.ts b/test/readiness.test.tsnew file mode 100644index 000000000..06c0984fc--- /dev/null+++ b/test/readiness.test.ts@@ -0,0 +1,172 @@+import "./support/auto-fake-sprites.ts";+import assert from "node:assert/strict";+import { once } from "node:events";+import { createServer as createTcpServer, type AddressInfo, type Socket } from "node:net";+import { test } from "node:test";+import { createServer } from "../src/api/server.ts";+import { buildApp, serverDeps } from "../src/wiring.ts";+import { createPgPool } from "../src/persistence/pg-pool.ts";+import { testConfig } from "./support/test-config.ts";++async function fixture(checkReadiness?: (signal: AbortSignal) => Promise<void>) {+ const config = testConfig({ signingSecret: "readiness-test-source-secret".repeat(3) });+ const built = buildApp(config);+ const server = createServer(built.app, {+ ...serverDeps(config, built),+ ...(checkReadiness ? { checkReadiness } : {}),+ requireSignedPortalIdentity: true,+ portalIdentitySecret: "readiness-test-portal-secret".repeat(3),+ });+ server.listen(0, "127.0.0.1");+ await once(server, "listening");+ const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+ return {+ get: (path: string) => fetch(`${url}${path}`),+ close: () => new Promise<void>((resolve) => server.close(() => resolve())),+ };+}++test("readiness is public and ready without a configured database", async () => {+ const srv = await fixture();+ try {+ const response = await srv.get("/readyz");+ assert.equal(response.status, 200);+ assert.deepEqual(await response.json(), { ok: true });+ assert.equal((await srv.get("/healthz")).status, 200);+ } finally {+ await srv.close();+ }+});++test("readiness detects dependency loss and recovery without changing liveness or leaking errors", async () => {+ let available = true;+ let probes = 0;+ const srv = await fixture(async () => {+ probes++;+ if (!available) throw new Error("postgres://synthetic:[email protected]/unavailable");+ });+ try {+ assert.equal((await srv.get("/readyz")).status, 200);+ available = false;+ const response = await srv.get("/readyz");+ assert.equal(response.status, 503);+ assert.deepEqual(await response.json(), { ok: false });+ assert.equal((await srv.get("/healthz")).status, 200);+ assert.equal(probes, 2, "liveness must not touch the database");+ available = true;+ assert.equal((await srv.get("/readyz")).status, 200);+ assert.equal(probes, 3, "recovery must be checked, not cached");+ } finally {+ await srv.close();+ }+});++test("a stalled dependency is aborted within the readiness deadline while liveness stays responsive", async () => {+ let signal: AbortSignal | undefined;+ let recover = false;+ const srv = await fixture(async (s) => {+ signal = s;+ if (!recover) await new Promise(() => {});+ });+ try {+ const start = performance.now();+ const pending = srv.get("/readyz");+ const live = await srv.get("/healthz");+ assert.equal(live.status, 200);+ assert.ok(performance.now() - start < 500, "liveness must not wait for readiness");+ const response = await pending;+ assert.equal(response.status, 503);+ assert.ok(performance.now() - start < 2000, "readiness must return within two seconds");+ assert.deepEqual(await response.json(), { ok: false });+ assert.equal(signal?.aborted, true);+ recover = true;+ assert.equal((await srv.get("/readyz")).status, 200);+ } finally {+ await srv.close();+ }+});++test("readiness bounds a blackholed Postgres handshake and cleans up the pending connection", async () => {+ const sockets = new Set<Socket>();+ const blackhole = createTcpServer((socket) => sockets.add(socket));+ blackhole.listen(0, "127.0.0.1");+ await once(blackhole, "listening");+ const pg = createPgPool(`postgres://test:[email protected]:${(blackhole.address() as AddressInfo).port}/test`);+ const srv = await fixture(async (signal) => {+ await pg.q("SELECT 1", [], { signal });+ });+ try {+ const start = performance.now();+ const response = await srv.get("/readyz");+ assert.equal(response.status, 503);+ assert.ok(performance.now() - start < 2000);+ assert.deepEqual(await response.json(), { ok: false });+ assert.equal((await srv.get("/healthz")).status, 200);+ for (const socket of sockets) socket.destroy();+ const pool = await pg.pool();+ for (let i = 0; i < 50 && pool.totalCount; i++) await new Promise((resolve) => setTimeout(resolve, 10));+ assert.equal(pool.totalCount, 0);+ assert.equal(pool.waitingCount, 0);+ } finally {+ for (const socket of sockets) socket.destroy();+ await srv.close();+ await pg.close();+ await new Promise<void>((resolve) => blackhole.close(() => resolve()));+ }+});++test(+ "a stalled Postgres query discards its client and the next readiness probe recovers",+ {+ skip: !process.env.DATABASE_URL && "requires Postgres",+ },+ async () => {+ const pg = createPgPool(process.env.DATABASE_URL!);+ let slow = true;+ const srv = await fixture(async (signal) => {+ await pg.q(slow ? "SELECT pg_sleep(30)" : "SELECT 1", [], { signal });+ });+ try {+ const response = await srv.get("/readyz");+ assert.equal(response.status, 503);+ const pool = await pg.pool();+ assert.equal(pool.totalCount, 0, "a cancelled query must not retain a checked-out client");+ slow = false;+ assert.equal((await srv.get("/readyz")).status, 200);+ assert.equal(pool.totalCount, 1);+ assert.equal(pool.idleCount, 1);+ } finally {+ await srv.close();+ await pg.close();+ }+ },+);++test(+ "pool saturation fails readiness and a late acquired client is released before recovery",+ {+ skip: !process.env.DATABASE_URL && "requires Postgres",+ },+ async () => {+ const pg = createPgPool(process.env.DATABASE_URL!);+ const srv = await fixture(async (signal) => {+ await pg.q("SELECT 1", [], { signal });+ });+ const pool = await pg.pool();+ const held = await Promise.all(Array.from({ length: pool.options.max! }, () => pool.connect()));+ try {+ const start = performance.now();+ assert.equal((await srv.get("/readyz")).status, 503);+ assert.ok(performance.now() - start < 2000);+ assert.equal((await srv.get("/healthz")).status, 200);+ for (const client of held.splice(0)) client.release();+ assert.equal((await srv.get("/readyz")).status, 200);+ assert.equal(pool.waitingCount, 0);+ assert.equal(pool.idleCount, pool.totalCount, "no late acquisition may leak a client");+ } finally {+ for (const client of held) client.release();+ await srv.close();+ await pg.close();+ }+ },+);diff --git a/test/route-auth-conformance.test.ts b/test/route-auth-conformance.test.tsindex c23a3a9be..872759ea1 100644--- a/test/route-auth-conformance.test.ts+++ b/test/route-auth-conformance.test.ts@@ -73,6 +73,7 @@ test("every pinned dedicated-audience route actually resolves in the table", ()test("raw routes keep their declared auth contracts (they self-enforce, so the declaration is the pin)", () => {const pins: Array<[string, string, RouteAuth]> = [["GET", "/healthz", "public"],+ ["GET", "/readyz", "public"],["GET", "/v1/credentials/git/gitlab/acme/repo.git/info/refs", { aud: CREDENTIAL_BROKER_AUD }],["POST", "/v1/credentials/git/gitlab/acme/repo.git/git-upload-pack", { aud: CREDENTIAL_BROKER_AUD }],];