Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
feat(release): build once, pin the chart to its images by digest
Petr Baucha3eb52a48f8d
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.ymlindex 66330e248..e9854e050 100644--- a/.github/workflows/pr-ci.yml+++ b/.github/workflows/pr-ci.yml@@ -73,6 +73,7 @@ jobs:gateway: ${{ steps.filter.outputs.gateway }}artifacts: ${{ steps.filter.outputs.artifacts }}migrations: ${{ steps.filter.outputs.migrations }}+ chart: ${{ steps.filter.outputs.chart }}steps:- name: Checkoutuses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4@@ -92,6 +93,7 @@ jobs:- 'core/**'# Tested from core/tests, so a change to it alone still runs them.- 'scripts/check_migration_heads_for_open_prs.py'+ - 'scripts/pin_chart_images.py'console:- *shared- 'console/**'@@ -110,6 +112,13 @@ jobs:# Spans every tree, so it gets its own filter rather than riding# any one of them. Includes the files whose versions the registry is# checked against, so a release bump cannot land without the check.+ # The switch Helm chart and what the release does to it before+ # publishing (pins the image digests into values.yaml).+ chart:+ - *shared+ - 'deploy/remote/helm/**'+ - 'scripts/pin_chart_images.py'+ - '.github/workflows/switch-release.yml'artifacts:- *shared- 'artifacts.yaml'@@ -243,6 +252,36 @@ jobs:- name: Registry, generated modules and declared versions agreerun: uv run --project core python scripts/gen_artifacts.py --check+ chart:+ name: Helm chart+ # The release pins every first-party image in the chart by digest before+ # publishing. This runs the same pin with placeholder digests and fails if+ # any first-party image still renders by tag, so an image value the pinner+ # does not know about fails here rather than shipping unpinned.+ needs: changes+ if: ${{ needs.changes.outputs.chart == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}+ runs-on: ubuntu-latest+ steps:+ - name: Checkout+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4++ - name: Set up Helm+ uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1++ - name: Lint+ run: helm lint deploy/remote/helm/switch++ - name: Every first-party image is pinned by digest once pinned+ run: |+ zero="sha256:$(printf '0%.0s' $(seq 64))"+ python3 scripts/pin_chart_images.py pin \+ --values deploy/remote/helm/switch/values.yaml \+ --registry ghcr.io/render-check --version 0.0.0-check \+ --digest "switch-core=$zero" --digest "gateway=$zero" --digest "setup=$zero"+ helm template switch deploy/remote/helm/switch \+ -f deploy/remote/helm/render-check-values.yaml \+ | python3 scripts/pin_chart_images.py verify --registry ghcr.io/render-check+gateway:name: Gateway UIneeds: changesdiff --git a/.github/workflows/switch-release.yml b/.github/workflows/switch-release.ymlindex 0e071e8d3..bfb5e68cd 100644--- a/.github/workflows/switch-release.yml+++ b/.github/workflows/switch-release.yml@@ -18,6 +18,20 @@ name: switch release# `--version`. pyproject's version is stamped on the runner (PEP 440 form,# `0.30.0rc1`) so the running server reports the RC it is; nothing is committed.#+# Dev builds: every push to `main` publishes the stack as+# `<pyproject version>-dev.<run number>.g<short sha>` — never `latest`, and a+# semver pre-release Helm skips unless asked for by exact version — so the dev+# environment runs what is on main from published artifacts rather than building+# from source at deploy time. The version identifies the commit; it says nothing+# about ordering against releases. No compose artifact: Switch Console bundles+# that from releases only.+#+# Build once, pin by digest: the chart is packaged only after the images are+# pushed, with each image's registry digest written into its values.yaml+# (scripts/pin_chart_images.py). Pinning the chart therefore pins the exact+# images, and what runs is reproducible from that one pin. The run summary and+# the `release-pins` artifact carry every digest, chart included.+## Artifacts go to GHCR by default, and the packages are public alongside the# repo. REGISTRY / IMAGE_NAMESPACE are the only things to change to retarget# another registry (e.g. ECR) — a config flip, not a rewrite.@@ -26,6 +40,12 @@ on:push:tags:- 'switch-v*'+ # Every push, with no path filter: the images copy files from outside+ # core/ and gateway/ (a console fixture, core/README.md), so a filter+ # narrow enough to save builds is one that will someday skip a change+ # that mattered.+ branches:+ - mainworkflow_dispatch:env:@@ -45,6 +65,7 @@ jobs:python_version: ${{ steps.version.outputs.python_version }}push: ${{ steps.version.outputs.push }}latest: ${{ steps.version.outputs.latest }}+ channel: ${{ steps.version.outputs.channel }}compose_speaks: ${{ steps.contract.outputs.speaks }}compose_accepts: ${{ steps.contract.outputs.accepts }}steps:@@ -57,8 +78,9 @@ jobs:- name: Resolve versionid: version# On a release tag: version is the tag minus `switch-v`, and we push and- # move `latest`. On an RC tag: push under the RC version only. On a- # dispatch run: use pyproject's version and DON'T push (build-only).+ # move `latest`. On an RC tag: push under the RC version only. On a push+ # to main: push under a dev version only. On a dispatch run: use+ # pyproject's version and DON'T push (build-only).run: |pkg_version="$(python3 -c "import tomllib; print(tomllib.load(open('core/pyproject.toml','rb'))['project']['version'])")"if [[ "$GITHUB_REF" == refs/tags/* ]]; then@@ -92,17 +114,29 @@ jobs:exit 1fipush=true+ channel=release+ [ "$latest" = true ] || channel=rc+ elif [ "$GITHUB_EVENT_NAME" = push ] && [ "$GITHUB_REF" = refs/heads/main ]; then+ # The `g` keeps the identifier alphanumeric: an all-digit short sha+ # with a leading zero is not valid semver.+ tag_version="${pkg_version}-dev.${GITHUB_RUN_NUMBER}.g${GITHUB_SHA::7}"+ python_version="${pkg_version}.dev${GITHUB_RUN_NUMBER}"+ push=true+ latest=false+ channel=develsetag_version="$pkg_version"python_version="$pkg_version"push=falselatest=false+ channel=dry-runfi{echo "version=$tag_version"echo "python_version=$python_version"echo "push=$push"echo "latest=$latest"+ echo "channel=$channel"} | tee -a "$GITHUB_OUTPUT"# The published compose artifact declares its stack-compose range twice:@@ -147,9 +181,10 @@ jobs:- name: Checkoutuses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0- - name: Stamp the release candidate version+ - name: Stamp the pre-release version# The server reports its version from the installed package metadata,- # which comes from pyproject. Written here for an RC, never committed.+ # which comes from pyproject. Written here for an RC or a dev build,+ # never committed.if: ${{ needs.prep.outputs.python_version != needs.prep.outputs.version }}env:PYTHON_VERSION: ${{ needs.prep.outputs.python_version }}@@ -181,6 +216,7 @@ jobs:password: ${{ secrets.GITHUB_TOKEN }}- name: Build and push+ id: builduses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2with:context: ${{ matrix.context }}@@ -197,13 +233,42 @@ jobs:cache-from: type=gha,scope=${{ matrix.image }}cache-to: type=gha,scope=${{ matrix.image }},mode=max+ # The digest of the multi-arch index, which is what a pod pulls by. The+ # chart job writes it into values.yaml, so it has to leave this job.+ - name: Record digest+ if: ${{ needs.prep.outputs.push == 'true' }}+ env:+ DIGEST: ${{ steps.build.outputs.digest }}+ run: |+ if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then+ echo "::error::build-push-action reported no usable digest for ${{ matrix.image }}: '$DIGEST'"+ exit 1+ fi+ mkdir -p "$RUNNER_TEMP/digests"+ printf '%s' "$DIGEST" > "$RUNNER_TEMP/digests/${{ matrix.image }}"++ - name: Upload digest+ if: ${{ needs.prep.outputs.push == 'true' }}+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ with:+ name: digest-${{ matrix.image }}+ path: ${{ runner.temp }}/digests/${{ matrix.image }}+ if-no-files-found: error+ retention-days: 7+chart:name: Package & publish Helm chart- needs: prep+ # After the images: the chart carries their digests, so it cannot be+ # packaged before they exist, and a chart is never published pointing at an+ # image that failed to push.+ needs: [prep, images]runs-on: ubuntu-latestpermissions:contents: readpackages: write+ env:+ VERSION: ${{ needs.prep.outputs.version }}+ PUSH: ${{ needs.prep.outputs.push }}steps:- name: Checkoutuses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0@@ -211,6 +276,44 @@ jobs:- name: Set up Helmuses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1+ - name: Download image digests+ if: ${{ needs.prep.outputs.push == 'true' }}+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0+ with:+ pattern: digest-*+ path: ${{ runner.temp }}/digests+ merge-multiple: true++ - name: Log in to GHCR+ if: ${{ needs.prep.outputs.push == 'true' }}+ uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0+ with:+ registry: ${{ env.REGISTRY }}+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Pin image digests into the chart+ # A dry run pushed no images, so it pins placeholder digests: the pin+ # and the check below still run, and nothing is published.+ run: |+ args=()+ for image in switch-core gateway setup; do+ if [ "$PUSH" = true ]; then+ digest="$(cat "$RUNNER_TEMP/digests/$image")"+ # The registry must serve the exact reference the chart will name.+ docker buildx imagetools inspect "$REGISTRY/$IMAGE_NAMESPACE/$image@$digest" >/dev/null+ else+ digest="sha256:$(printf '0%.0s' $(seq 64))"+ fi+ args+=(--digest "$image=$digest")+ done+ python3 scripts/pin_chart_images.py pin \+ --values deploy/remote/helm/switch/values.yaml \+ --registry "$REGISTRY/$IMAGE_NAMESPACE" \+ --version "$VERSION" \+ "${args[@]}"+ git diff -- deploy/remote/helm/switch/values.yaml+- name: Lint chartrun: helm lint deploy/remote/helm/switch@@ -219,10 +322,17 @@ jobs:# a single tag pins the whole stack.run: |helm package deploy/remote/helm/switch \- --version "${{ needs.prep.outputs.version }}" \- --app-version "${{ needs.prep.outputs.version }}" \+ --version "$VERSION" \+ --app-version "$VERSION" \--destination "$RUNNER_TEMP/chart"+ - name: Check the packaged chart is digest-pinned+ # Renders the .tgz that is about to be pushed, not the source tree.+ run: |+ helm template switch "$RUNNER_TEMP/chart/switch-$VERSION.tgz" \+ -f deploy/remote/helm/render-check-values.yaml \+ | python3 scripts/pin_chart_images.py verify --registry "$REGISTRY/$IMAGE_NAMESPACE"+- name: Log in to GHCR (Helm OCI)if: ${{ needs.prep.outputs.push == 'true' }}run: |@@ -230,14 +340,69 @@ jobs:helm registry login "$REGISTRY" -u "${{ github.actor }}" --password-stdin- name: Push chart+ id: pushif: ${{ needs.prep.outputs.push == 'true' }}run: |- helm push "$RUNNER_TEMP/chart/switch-${{ needs.prep.outputs.version }}.tgz" \- "oci://$REGISTRY/$IMAGE_NAMESPACE/charts"+ helm push "$RUNNER_TEMP/chart/switch-$VERSION.tgz" \+ "oci://$REGISTRY/$IMAGE_NAMESPACE/charts" 2>&1 | tee "$RUNNER_TEMP/helm-push.log"+ digest="$(sed -n 's/^Digest: *//p' "$RUNNER_TEMP/helm-push.log")"+ if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then+ echo "::error::helm push printed no chart digest; got '$digest'"+ exit 1+ fi+ echo "digest=$digest" >> "$GITHUB_OUTPUT"++ - name: Record the pins+ # The run summary is for a person; release-pins.json is for whatever+ # promotes this build next (an environment's pins file), so it never has+ # to be copied out of a log by hand.+ if: ${{ needs.prep.outputs.push == 'true' }}+ env:+ CHART_DIGEST: ${{ steps.push.outputs.digest }}+ CHANNEL: ${{ needs.prep.outputs.channel }}+ run: |+ python3 - <<'PY' > "$RUNNER_TEMP/release-pins.json"+ import json, os, pathlib+ ns = f"{os.environ['REGISTRY']}/{os.environ['IMAGE_NAMESPACE']}"+ version = os.environ["VERSION"]+ digests = pathlib.Path(os.environ["RUNNER_TEMP"], "digests")+ print(json.dumps({+ "version": version,+ "channel": os.environ["CHANNEL"],+ "commit": os.environ["GITHUB_SHA"],+ "chart": {"ref": f"oci://{ns}/charts/switch", "version": version, "digest": os.environ["CHART_DIGEST"]},+ "images": {+ name: {"ref": f"{ns}/{name}", "digest": (digests / name).read_text().strip()}+ for name in ("switch-core", "gateway", "setup")+ },+ }, indent=2))+ PY+ {+ echo "## switch $VERSION ($CHANNEL)"+ echo+ echo "| Artifact | Pinned reference |"+ echo "| --- | --- |"+ echo "| chart | \`oci://$REGISTRY/$IMAGE_NAMESPACE/charts/switch:$VERSION@$CHART_DIGEST\` |"+ for image in switch-core gateway setup; do+ echo "| $image | \`$REGISTRY/$IMAGE_NAMESPACE/$image@$(cat "$RUNNER_TEMP/digests/$image")\` |"+ done+ } >> "$GITHUB_STEP_SUMMARY"+ cat "$RUNNER_TEMP/release-pins.json"++ - name: Upload the pins+ if: ${{ needs.prep.outputs.push == 'true' }}+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ with:+ name: release-pins+ path: ${{ runner.temp }}/release-pins.json+ if-no-files-found: errorcompose:name: Publish standalone compose (OCI artifact)needs: prep+ # Switch Console bundles the compose file from releases; a dev build of it+ # has no consumer.+ if: ${{ needs.prep.outputs.channel != 'dev' }}runs-on: ubuntu-latestpermissions:contents: readdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 370a3a54b..9bbe3f857 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -87,6 +87,26 @@ version of their own to them without also giving them a release of their own.pods twice. Rendering fails when the secret is missing or shorter than 32characters. Remove variables set by hand before the first upgrade that rendersthem (see the chart README).+- **`global.preStopSleepSeconds`** (default `5`): switch-core, gateway and+ Mattermost wait that long in a `preStop` hook before SIGTERM, so a roll stops+ answering with 503s while endpoints and load balancers catch up. Uses the+ native `sleep` action (Kubernetes 1.30+); `0` renders no hook.++#### Changed+- **The published Helm chart pins its images by digest.** The release writes+ each image's registry digest into the chart's values before packaging, so+ `switchCore.image`, `gateway.image` and `setup.image` default to+ `<image>:<version>@sha256:…` with `imagePullPolicy: IfNotPresent`, and+ `global.imageRegistry` defaults to the registry the release published to.+ Pinning the chart version now pins the exact images; a deployment no longer+ needs to set the three image values, and setting them by tag opts back out of+ the pin. Values that set `global.imageRegistry` to a mirror keep working when+ the mirror copies images byte for byte (digests survive a copy). A values+ default change: check any values file that relied on the old defaults.+- **Every merge to main publishes a dev build** of the images and chart, under+ `<version>-dev.<run>.g<sha>`: never `latest`, and skipped by Helm unless asked+ for by exact version. Each run records every digest in its summary and in a+ `release-pins` artifact.#### Changed- **Agent management moved behind the `agent_management` feature flag.** The olddiff --git a/RELEASING.md b/RELEASING.mdindex ff9e31f97..81a12620d 100644--- a/RELEASING.md+++ b/RELEASING.md@@ -77,9 +77,12 @@ peer still on it, and it can never be raised past what is running in the field.- builds multi-arch (`linux/amd64`, `linux/arm64`) images for `switch-core`,`gateway`, and `setup` and pushes them to the container registry tagged`<version>` and `latest`;- - packages the Helm chart at the same version and pushes it as an OCI- artifact to the same registry.- `workflow_dispatch` runs the build without pushing, for verification.+ - once the images are pushed, writes each image's digest into the chart's+ `values.yaml`, packages the chart at the same version, checks the packaged+ chart renders every first-party image by digest, and pushes it as an OCI+ artifact to the same registry (see [Pinned by digest](#pinned-by-digest)).+ `workflow_dispatch` runs the build without pushing, for verification. It+ pins placeholder digests, so the pin and the check run too.You can also trigger `workflow_dispatch` manually from the Actions tab to test abuild without creating a release.@@ -104,6 +107,49 @@ git push origin switch-v0.30.0-rc.1into `core/pyproject.toml` on the runner, so the running server's version sayswhich candidate it is. Nothing is committed.+### Dev builds++Every push to `main` publishes the images and chart under a dev version,+`<pyproject version>-dev.<run number>.g<short sha>` (e.g.+`0.29.0-dev.512.g7f0c803`), so the dev environment runs main from published,+digest-pinned artifacts instead of building from source at deploy time.++- **Never `latest`**, and a semver pre-release, so `helm install` skips it unless+ given that exact `--version`. No compose artifact: nothing consumes one.+- **The version names a commit, not a place in the release order.** Read the+ short sha; do not compare a dev version with a release.+- **The server reports it** as `<pyproject version>.dev<run number>` (PEP 440),+ stamped on the runner like an RC.+- **Not gated on PR CI.** It builds whatever reached main; what an environment+ runs is decided by its pin, not by the build existing.++### Pinned by digest++The chart and the images are built together and published as one unit. The+release writes each image's registry digest into the chart before packaging:++```yaml+global:+ imageRegistry: "ghcr.io/<owner>"+switchCore:+ image: switch-core:<version>@sha256:…+ imagePullPolicy: IfNotPresent+```++and the same for `gateway` and `setup`. So pinning the chart pins the exact+images: a deployment sets no image values, and what runs is reproducible from+the chart pin alone. The tag in front of the digest is for people; the runtime+pulls by digest. Setting an image value by tag on install opts out of the pin.+A mirror set with `global.imageRegistry` keeps the pin as long as it copies+images byte for byte, which registry-to-registry copies do.++Every publishing run lists all four digests (the chart's own included) in its+run summary, and uploads them as a `release-pins` artifact (`release-pins.json`)+for whatever promotes the build next. `scripts/pin_chart_images.py` does the+pinning and the check; PR CI runs both with placeholder digests, so a new+first-party image value the pinner does not know about fails a pull request+rather than shipping by tag.+## Switch Console desktop app release (separate)The desktop app (`console/`) releases on its own tag, `switch-console-v<version>`, via@@ -213,10 +259,14 @@ Consuming the published artifacts:# imagesdocker pull ghcr.io/<owner>/switch-core:<version>-# chart+# chart (its images are pinned inside it by digest)helm install switch oci://ghcr.io/<owner>/charts/switch --version <version> \-f my-values.yaml+# or pin the chart itself by digest, from the run summary+helm install switch oci://ghcr.io/<owner>/charts/switch@sha256:<chart digest> \+ -f my-values.yaml+# standalone compose (OCI artifact) — pull the file, then run itoras pull ghcr.io/<owner>/standalone-compose:<version>SWITCH_VERSION=<version> docker compose -f standalone-docker-compose.yml up -ddiff --git a/core/tests/test_pin_chart_images.py b/core/tests/test_pin_chart_images.pynew file mode 100644index 000000000..949a7f32f--- /dev/null+++ b/core/tests/test_pin_chart_images.py@@ -0,0 +1,151 @@+"""Tests for the release step that pins the Helm chart to its images by digest.++A published chart that names its images by tag runs whatever the tag points at+on the day, so the pinner must either pin every first-party image or fail the+release. Silently pinning nothing, or the wrong line, is the failure to rule out.+"""++from __future__ import annotations++import importlib.util+import sys+from pathlib import Path+from types import ModuleType++import pytest+import yaml++REPO_ROOT = Path(__file__).resolve().parents[2]+VALUES = REPO_ROOT / "deploy/remote/helm/switch/values.yaml"++DIGESTS = {+ "switch-core": "sha256:" + "a" * 64,+ "gateway": "sha256:" + "b" * 64,+ "setup": "sha256:" + "c" * 64,+}+++def _load() -> ModuleType:+ path = REPO_ROOT / "scripts" / "pin_chart_images.py"+ spec = importlib.util.spec_from_file_location("pin_chart_images", path)+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++pin_chart_images = _load()+++def test_pins_every_first_party_image_in_the_real_values_file() -> None:+ original = VALUES.read_text()+ pinned = yaml.safe_load(+ pin_chart_images.pin(original, "ghcr.io/acme", "1.2.3", DIGESTS)+ )++ assert pinned["global"]["imageRegistry"] == "ghcr.io/acme"+ assert (+ pinned["switchCore"]["image"] == f"switch-core:1.2.3@{DIGESTS['switch-core']}"+ )+ assert pinned["gateway"]["image"] == f"gateway:1.2.3@{DIGESTS['gateway']}"+ assert pinned["setup"]["image"] == f"setup:1.2.3@{DIGESTS['setup']}"+ for key in ("switchCore", "gateway", "setup"):+ assert pinned[key]["imagePullPolicy"] == "IfNotPresent"+++def test_changes_only_the_pinned_lines() -> None:+ original = VALUES.read_text()+ pinned = pin_chart_images.pin(original, "ghcr.io/acme", "1.2.3", DIGESTS)++ changed = [+ (a, b)+ for a, b in zip(original.splitlines(), pinned.splitlines(), strict=True)+ if a != b+ ]+ assert len(changed) == 7 # the registry, then an image and a pull policy per image+ assert (+ yaml.safe_load(original)["mattermost"] == yaml.safe_load(pinned)["mattermost"]+ )+++ "digests",+ [+ {k: v for k, v in DIGESTS.items() if k != "setup"},+ {**DIGESTS, "sidecar": "sha256:" + "d" * 64},+ {**DIGESTS, "gateway": "latest"},+ {**DIGESTS, "gateway": "sha256:" + "A" * 64},+ ],+ ids=["missing", "unknown", "not-a-digest", "uppercase"],+)+def test_refuses_digests_that_do_not_cover_exactly_the_images(+ digests: dict[str, str],+) -> None:+ with pytest.raises(pin_chart_images.PinError):+ pin_chart_images.pin(VALUES.read_text(), "ghcr.io/acme", "1.2.3", digests)+++def test_refuses_a_values_file_whose_shape_moved() -> None:+ values = VALUES.read_text().replace("\n image: gateway:", "\n img: gateway:")+ with pytest.raises(pin_chart_images.PinError, match="gateway.image"):+ pin_chart_images.pin(values, "ghcr.io/acme", "1.2.3", DIGESTS)+++def test_does_not_pin_a_nested_image_key() -> None:+ values = "global:\n imageRegistry: ''\nswitchCore:\n sidecar:\n image: x\n"+ with pytest.raises(pin_chart_images.PinError, match="switchCore.image"):+ pin_chart_images.pin(values, "r", "1", DIGESTS)+++def _pod(*images: str, init: tuple[str, ...] = ()) -> str:+ return yaml.safe_dump(+ {+ "kind": "Deployment",+ "spec": {+ "template": {+ "spec": {+ "initContainers": [+ {"name": f"i{n}", "image": i} for n, i in enumerate(init)+ ],+ "containers": [+ {"name": f"c{n}", "image": i} for n, i in enumerate(images)+ ],+ }+ }+ },+ }+ )+++def _pinned(name: str) -> str:+ return f"ghcr.io/acme/{name}:1.2.3@{DIGESTS[name]}"+++def test_verify_accepts_a_fully_pinned_render() -> None:+ rendered = "---\n".join(+ [+ _pod(_pinned("switch-core"), init=(_pinned("switch-core"), "busybox:1.36")),+ _pod(_pinned("gateway")),+ _pod(_pinned("setup"), "postgres:16-alpine"),+ ]+ )+ assert len(pin_chart_images.verify(rendered, "ghcr.io/acme")) == 3+++def test_verify_rejects_a_first_party_image_by_tag() -> None:+ rendered = "---\n".join(+ [+ _pod(_pinned("switch-core"), init=("ghcr.io/acme/switch-core:1.2.3",)),+ _pod(_pinned("gateway")),+ _pod(_pinned("setup")),+ ]+ )+ with pytest.raises(pin_chart_images.PinError, match="not pinned"):+ pin_chart_images.verify(rendered, "ghcr.io/acme")+++def test_verify_rejects_a_render_missing_an_image() -> None:+ rendered = "---\n".join([_pod(_pinned("switch-core")), _pod(_pinned("gateway"))])+ with pytest.raises(pin_chart_images.PinError, match="setup"):+ pin_chart_images.verify(rendered, "ghcr.io/acme")diff --git a/deploy/remote/helm/render-check-values.yaml b/deploy/remote/helm/render-check-values.yamlnew file mode 100644index 000000000..eedf55d0d--- /dev/null+++ b/deploy/remote/helm/render-check-values.yaml@@ -0,0 +1,9 @@+# Placeholder secrets that let `helm template` render the switch chart in CI,+# where the only question is what the manifests reference (see+# scripts/pin_chart_images.py). Never deploy with these.+secrets:+ postgresPassword: render-check+ agentRegistrationToken: render-check+ secretKeys: "render-check:0000000000000000000000000000000000000000"+ gatewayAdminEmail: [email protected]+ gatewayAdminPassword: render-checkdiff --git a/deploy/remote/helm/switch/templates/_helpers.tpl b/deploy/remote/helm/switch/templates/_helpers.tplindex 47e2d22f9..634dd2a8f 100644--- a/deploy/remote/helm/switch/templates/_helpers.tpl+++ b/deploy/remote/helm/switch/templates/_helpers.tpl@@ -434,6 +434,22 @@ than something you discover hours later at Graph subscription time.{{- end -}}{{- end }}+{{/*+preStop drain for a container behind a Service: hold the old pod for a few+seconds after it is marked terminating, so endpoint and load-balancer+deregistration land before the process gets SIGTERM. The native `sleep` action+needs no shell, which the distroless images do not have.+Usage: {{- include "switch.preStopDrain" . | nindent 10 }}+*/}}+{{- define "switch.preStopDrain" -}}+{{- with .Values.global.preStopSleepSeconds -}}+lifecycle:+ preStop:+ sleep:+ seconds: {{ . }}+{{- end }}+{{- end }}+{{/*Prepend the global image registry if set.Usage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.switchCore.image) }}diff --git a/deploy/remote/helm/switch/templates/gateway/deployment.yaml b/deploy/remote/helm/switch/templates/gateway/deployment.yamlindex 78048cbf4..80e0a3124 100644--- a/deploy/remote/helm/switch/templates/gateway/deployment.yaml+++ b/deploy/remote/helm/switch/templates/gateway/deployment.yaml@@ -32,6 +32,7 @@ spec:- name: nginx-configmountPath: /etc/nginx/conf.d/default.confsubPath: default.conf+ {{- include "switch.preStopDrain" . | nindent 10 }}readinessProbe:httpGet:path: /diff --git a/deploy/remote/helm/switch/templates/mattermost/deployment.yaml b/deploy/remote/helm/switch/templates/mattermost/deployment.yamlindex 3e42f2cff..637d03301 100644--- a/deploy/remote/helm/switch/templates/mattermost/deployment.yaml+++ b/deploy/remote/helm/switch/templates/mattermost/deployment.yaml@@ -87,6 +87,7 @@ spec:- name: MM_SERVICESETTINGS_ALLOWEDUNTRUSTEDINTERNALCONNECTIONSvalue: "{{ $core }} {{ $core }}.{{ .Release.Namespace }} {{ $core }}.{{ .Release.Namespace }}.svc.cluster.local"{{- end }}+ {{- include "switch.preStopDrain" . | nindent 10 }}readinessProbe:httpGet:path: /api/v4/system/pingdiff --git a/deploy/remote/helm/switch/templates/switch-core/deployment.yaml b/deploy/remote/helm/switch/templates/switch-core/deployment.yamlindex c637f546f..a5c6b1d33 100644--- a/deploy/remote/helm/switch/templates/switch-core/deployment.yaml+++ b/deploy/remote/helm/switch/templates/switch-core/deployment.yaml@@ -82,6 +82,7 @@ spec:value: /etc/switch-github/github.json{{- end }}{{- include "switch.coreEnv" (merge (dict "omitOwnerCredentials" true) .) | nindent 12 }}+ {{- include "switch.preStopDrain" . | nindent 10 }}# Readiness checks the database; liveness deliberately does not.# There is one replica, so a failing readiness probe empties the# Service rather than shifting traffic — which is right while thediff --git a/deploy/remote/helm/switch/values.yaml b/deploy/remote/helm/switch/values.yamlindex e78e2a90f..0406cc0f6 100644--- a/deploy/remote/helm/switch/values.yaml+++ b/deploy/remote/helm/switch/values.yaml@@ -10,6 +10,12 @@ global:# private registry (e.g. a private GHCR namespace). Applied to every pod. Leave# empty for public images.imagePullSecrets: []+ # Seconds each pod behind a Service (switch-core, gateway, Mattermost) waits in+ # a preStop hook before it is sent SIGTERM, so the endpoint and any load+ # balancer stop routing to it first and a roll does not answer requests with+ # 503s. Uses the native `sleep` lifecycle action (Kubernetes 1.30+); set 0 to+ # render no hook on an older cluster.+ preStopSleepSeconds: 5# ── Ingress ─────────────────────────────────────────────────────────────────# Generic Ingress for the web surface. Bring your own ingress controllerdiff --git a/scripts/pin_chart_images.py b/scripts/pin_chart_images.pynew file mode 100644index 000000000..85d1f4488--- /dev/null+++ b/scripts/pin_chart_images.py@@ -0,0 +1,201 @@+#!/usr/bin/env python3+"""Pin the Helm chart to the exact images it was built with, and check that it is.++The release builds the switch-core, gateway and setup images and then packages+the chart. Left alone, the chart's values name those images by a mutable tag,+so every deployment re-states which images go with which chart and gets+whatever the tag points at on the day. This writes each image's registry digest+into the chart's `values.yaml` before packaging, so pinning the chart version+pins the images with it, and what runs is reproducible from that one pin.++Two subcommands:++* `pin` rewrites `values.yaml` in place: `global.imageRegistry`, and for each+ first-party image `<name>:<version>@sha256:…` with pull policy+ `IfNotPresent` (a digest cannot change under the tag, so `Always` only costs a+ registry round trip). The edit is line-based so the file's comments, which+ are the chart's documentation under `helm show values`, survive.+* `verify` reads `helm template` output and fails unless every first-party+ image is digest-pinned and all of them are present. The release runs it on+ the chart it is about to push; PR CI runs it on a chart pinned with+ placeholder digests, so a new image value the pinner does not know about+ fails a pull request instead of shipping by tag.++Usage:+ python scripts/pin_chart_images.py pin --values <values.yaml> \\+ --registry ghcr.io/<owner> --version <version> \\+ --digest switch-core=sha256:… --digest gateway=sha256:… --digest setup=sha256:…+ helm template x <chart> | python scripts/pin_chart_images.py verify --registry ghcr.io/<owner>+"""++from __future__ import annotations++import argparse+import re+import sys+from collections.abc import Iterator+from pathlib import Path+from typing import Any++import yaml++# Image name as published -> the top-level values key whose `image` it fills.+IMAGES: dict[str, str] = {+ "switch-core": "switchCore",+ "gateway": "gateway",+ "setup": "setup",+}++DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")+TOP_LEVEL_KEY = re.compile(r"^([A-Za-z0-9_]+):\s*(#.*)?$")+++class PinError(Exception):+ pass+++def _set_child(lines: list[str], parent: str, child: str, value: str) -> None:+ """Set `parent.child` to `value`, where `parent` is a top-level mapping key.++ Exactly one `child` line at two-space indent must sit inside `parent`'s+ block; anything else means the values file changed shape under the pinner,+ which must fail rather than pin the wrong line or nothing at all.+ """+ start = None+ for i, line in enumerate(lines):+ m = TOP_LEVEL_KEY.match(line)+ if m and m.group(1) == parent:+ if start is not None:+ raise PinError(f"top-level key {parent!r} appears more than once")+ start = i+ if start is None:+ raise PinError(f"no top-level key {parent!r}")++ end = len(lines)+ for i in range(start + 1, len(lines)):+ line = lines[i]+ if line and not line[0].isspace() and not line.startswith("#"):+ end = i+ break++ pattern = re.compile(rf"^ {re.escape(child)}:(\s.*)?$")+ hits = [i for i in range(start + 1, end) if pattern.match(lines[i].rstrip("\n"))]+ if len(hits) != 1:+ raise PinError(f"expected one {parent}.{child} line, found {len(hits)}")+ newline = "\n" if lines[hits[0]].endswith("\n") else ""+ lines[hits[0]] = f" {child}: {value}{newline}"+++def pin(values_text: str, registry: str, version: str, digests: dict[str, str]) -> str:+ missing = sorted(set(IMAGES) - set(digests))+ unknown = sorted(set(digests) - set(IMAGES))+ if missing or unknown:+ raise PinError(+ f"digests must cover exactly {sorted(IMAGES)}; missing {missing}, unknown {unknown}"+ )+ for name, digest in digests.items():+ if not DIGEST.match(digest):+ raise PinError(f"{name}: {digest!r} is not a sha256 digest")+ if not registry or registry.endswith("/"):+ raise PinError(+ f"registry {registry!r} must be non-empty with no trailing slash"+ )+ if not version:+ raise PinError("version must be non-empty")++ lines = values_text.splitlines(keepends=True)+ _set_child(lines, "global", "imageRegistry", f'"{registry}"')+ for name, key in IMAGES.items():+ _set_child(lines, key, "image", f"{name}:{version}@{digests[name]}")+ _set_child(lines, key, "imagePullPolicy", "IfNotPresent")+ pinned = "".join(lines)++ loaded = yaml.safe_load(pinned)+ if loaded["global"]["imageRegistry"] != registry:+ raise PinError("pinned values do not read back the registry")+ for name, key in IMAGES.items():+ if loaded[key]["image"] != f"{name}:{version}@{digests[name]}":+ raise PinError(f"pinned values do not read back {key}.image")+ return pinned+++def _images(node: Any) -> Iterator[str]:+ if isinstance(node, dict):+ for key in ("containers", "initContainers"):+ for container in node.get(key) or []:+ if isinstance(container, dict) and "image" in container:+ yield container["image"]+ for value in node.values():+ yield from _images(value)+ elif isinstance(node, list):+ for item in node:+ yield from _images(item)+++def verify(rendered: str, registry: str) -> list[str]:+ """Return every first-party image reference, or raise if any is unpinned."""+ refs = sorted(+ {ref for doc in yaml.safe_load_all(rendered) if doc for ref in _images(doc)}+ )+ prefix = f"{registry}/"+ ours = [ref for ref in refs if ref.startswith(prefix)]+ unpinned = [ref for ref in ours if not re.search(r"@sha256:[0-9a-f]{64}$", ref)]+ if unpinned:+ raise PinError(f"first-party images not pinned by digest: {unpinned}")+ seen = {ref[len(prefix) :].split("@")[0].split(":")[0] for ref in ours}+ absent = sorted(set(IMAGES) - seen)+ if absent:+ raise PinError(+ f"first-party images missing from the rendered chart: {absent}; rendered images: {refs}"+ )+ return ours+++def _parse_digests(pairs: list[str]) -> dict[str, str]:+ digests: dict[str, str] = {}+ for pair in pairs:+ name, sep, digest = pair.partition("=")+ if not sep:+ raise PinError(f"--digest {pair!r} must be <image>=sha256:<hex>")+ if name in digests:+ raise PinError(f"--digest {name} given twice")+ digests[name] = digest+ return digests+++def main(argv: list[str]) -> int:+ parser = argparse.ArgumentParser(+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter+ )+ sub = parser.add_subparsers(dest="command", required=True)++ p_pin = sub.add_parser("pin", help="write image digests into values.yaml")+ p_pin.add_argument("--values", required=True, type=Path)+ p_pin.add_argument("--registry", required=True)+ p_pin.add_argument("--version", required=True)+ p_pin.add_argument("--digest", required=True, action="append", default=[])++ p_verify = sub.add_parser(+ "verify", help="check rendered manifests (stdin) are digest-pinned"+ )+ p_verify.add_argument("--registry", required=True)++ args = parser.parse_args(argv)+ try:+ if args.command == "pin":+ text = args.values.read_text()+ args.values.write_text(+ pin(text, args.registry, args.version, _parse_digests(args.digest))+ )+ print(f"pinned {args.values}")+ else:+ for ref in verify(sys.stdin.read(), args.registry):+ print(ref)+ except PinError as exc:+ print(f"error: {exc}", file=sys.stderr)+ return 1+ return 0+++if __name__ == "__main__":+ sys.exit(main(sys.argv[1:]))