Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Address review: registry-aware refs, digest-aware pull policy, dev namespace
Petr Bauchd56ef39bce6b
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.ymlindex e9854e050..3b03b4ba2 100644--- a/.github/workflows/pr-ci.yml+++ b/.github/workflows/pr-ci.yml@@ -94,6 +94,7 @@ jobs:# Tested from core/tests, so a change to it alone still runs them.- 'scripts/check_migration_heads_for_open_prs.py'- 'scripts/pin_chart_images.py'+ - 'scripts/prune_dev_packages.py'console:- *shared- 'console/**'@@ -109,9 +110,6 @@ jobs:# actually find a new head — not on every backend-touching push.migrations:- 'core/switch_core/migrations/versions/**'- # Spans every tree, so it gets its own filter rather than riding- # any one of them. Includes the files whose versions the registry is- # checked against, so a release bump cannot land without the check.# The switch Helm chart and what the release does to it before# publishing (pins the image digests into values.yaml).chart:@@ -119,6 +117,9 @@ jobs:- 'deploy/remote/helm/**'- 'scripts/pin_chart_images.py'- '.github/workflows/switch-release.yml'+ # Spans every tree, so it gets its own filter rather than riding+ # any one of them. Includes the files whose versions the registry is+ # checked against, so a release bump cannot land without the check.artifacts:- *shared- 'artifacts.yaml'@@ -272,15 +273,18 @@ jobs:run: helm lint deploy/remote/helm/switch- name: Every first-party image is pinned by digest once pinned+ # Rendered as configured by default and with the optional workloads on,+ # so an image behind a feature flag is checked too.run: |- zero="sha256:$(printf '0%.0s' $(seq 64))"python3 scripts/pin_chart_images.py pin \--values deploy/remote/helm/switch/values.yaml \- --registry ghcr.io/render-check --version 0.0.0-check \- --digest "switch-core=$zero" --digest "gateway=$zero" --digest "setup=$zero"- helm template switch deploy/remote/helm/switch \- -f deploy/remote/helm/render-check-values.yaml \- | python3 scripts/pin_chart_images.py verify --registry ghcr.io/render-check+ --registry ghcr.io/render-check --version 0.0.0-check --placeholder-digests+ for overlay in "" "-f deploy/remote/helm/render-check-optional-values.yaml"; do+ # shellcheck disable=SC2086 # $overlay is deliberately split into two words+ helm template switch deploy/remote/helm/switch \+ -f deploy/remote/helm/render-check-values.yaml $overlay \+ | python3 scripts/pin_chart_images.py verify --registry ghcr.io/render-check+ donegateway:name: Gateway UIdiff --git a/.github/workflows/switch-dev-cleanup.yml b/.github/workflows/switch-dev-cleanup.ymlnew file mode 100644index 000000000..5a276d1a4--- /dev/null+++ b/.github/workflows/switch-dev-cleanup.yml@@ -0,0 +1,43 @@+name: switch dev cleanup++# Prunes old dev builds — the images and chart every push to main publishes+# under ghcr.io/<owner>/dev/ (see switch-release.yml) — so the dev packages do+# not grow without bound. Only the dev/ packages are touched; releases and RCs+# live in packages of their own. A build is deleted once it is older than 30+# days and outside the 20 newest, so the dev environment's pin always has+# something recent to point at. See scripts/prune_dev_packages.py.++on:+ schedule:+ - cron: '17 4 * * *'+ workflow_dispatch:+ inputs:+ dry_run:+ description: List what would be deleted, delete nothing+ type: boolean+ default: true++permissions:+ contents: read+ packages: write++concurrency:+ group: switch-dev-cleanup+ cancel-in-progress: false++jobs:+ prune:+ name: Prune dev builds+ runs-on: ubuntu-latest+ steps:+ - name: Checkout+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0++ - name: Prune+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}+ run: |+ args=(--owner "${{ github.repository_owner }}" --max-age-days 30 --keep 20)+ [ "$DRY_RUN" != true ] || args+=(--dry-run)+ python3 scripts/prune_dev_packages.py "${args[@]}"diff --git a/.github/workflows/switch-release.yml b/.github/workflows/switch-release.ymlindex bfb5e68cd..4032c3a84 100644--- a/.github/workflows/switch-release.yml+++ b/.github/workflows/switch-release.yml@@ -19,18 +19,24 @@ name: switch release# `0.30.0rc1`) so the running server reports the RC it is; nothing is committed.## Dev builds: every push to `main` publishes the stack as-# `<pyproject version>-dev.<run number>.g<short sha>` — never `latest`, and a-# semver pre-release Helm skips unless asked for by exact version — so the dev-# environment runs what is on main from published artifacts rather than building-# from source at deploy time. The version identifies the commit; it says nothing-# about ordering against releases. No compose artifact: Switch Console bundles-# that from releases only.+# `<next patch>-dev.<run number>.g<short sha>` under a separate `dev/` namespace+# (ghcr.io/<owner>/dev/…) — never `latest`, never mixed into the release+# packages' tag lists, and a semver pre-release Helm skips unless asked for by+# exact version — so the dev environment runs what is on main from published+# artifacts rather than building from source at deploy time. "Next patch" is one+# above the newest release tag on main, so a dev build sorts after the release it+# contains. A newer push cancels an in-flight dev build; switch-dev-cleanup.yml+# prunes old ones. No compose artifact: Switch Console bundles that from+# releases only.## Build once, pin by digest: the chart is packaged only after the images are# pushed, with each image's registry digest written into its values.yaml# (scripts/pin_chart_images.py). Pinning the chart therefore pins the exact# images, and what runs is reproducible from that one pin. The run summary and-# the `release-pins` artifact carry every digest, chart included.+# the `release-pins` artifact carry every digest, chart included. A version is+# published once: a run (or re-run) whose chart version already exists in the+# registry fails before building, since a rebuilt chart would point the same+# version at different digests.## Artifacts go to GHCR by default, and the packages are public alongside the# repo. REGISTRY / IMAGE_NAMESPACE are the only things to change to retarget@@ -48,10 +54,19 @@ on:- mainworkflow_dispatch:+# Pushes to main share a group and the newest wins: an in-flight dev build of an+# older commit is superseded, so a busy merge day does not queue parallel+# emulated arm64 builds. Tags and dispatch runs each get their own group and are+# never cancelled — a cancelled release would be a half-published one.+concurrency:+ group: switch-release-${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') && 'main' || github.run_id }}+ cancel-in-progress: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}+env:REGISTRY: ghcr.io# GHCR namespace = the repo owner (lowercase). Images land at# ghcr.io/<owner>/<image>; the Helm chart at ghcr.io/<owner>/charts/switch.+ # Dev builds go one level down, under <owner>/dev (prep's `namespace`).IMAGE_NAMESPACE: ${{ github.repository_owner }}jobs:@@ -60,7 +75,11 @@ jobs:runs-on: ubuntu-latestpermissions:contents: read+ # To ask the registry whether this version is already published.+ packages: readoutputs:+ images: ${{ steps.images.outputs.images }}+ namespace: ${{ steps.version.outputs.namespace }}version: ${{ steps.version.outputs.version }}python_version: ${{ steps.version.outputs.python_version }}push: ${{ steps.version.outputs.push }}@@ -117,10 +136,20 @@ jobs:channel=release[ "$latest" = true ] || channel=rcelif [ "$GITHUB_EVENT_NAME" = push ] && [ "$GITHUB_REF" = refs/heads/main ]; then+ # One patch above the newest release on main, so the build sorts+ # after the release it contains (a `-dev`/`.dev` version sorts+ # before its own base) and before any later release or RC.+ released="$(git tag --merged HEAD --list 'switch-v*' | sed 's/^switch-v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n1)"+ if [ -z "$released" ]; then+ echo "::error::No switch-v<X.Y.Z> release tag is reachable from main; cannot place a dev version."+ exit 1+ fi+ IFS=. read -r major minor patch <<<"$released"+ base="$major.$minor.$((patch + 1))"# The `g` keeps the identifier alphanumeric: an all-digit short sha# with a leading zero is not valid semver.- tag_version="${pkg_version}-dev.${GITHUB_RUN_NUMBER}.g${GITHUB_SHA::7}"- python_version="${pkg_version}.dev${GITHUB_RUN_NUMBER}"+ tag_version="${base}-dev.${GITHUB_RUN_NUMBER}.g${GITHUB_SHA::7}"+ python_version="${base}.dev${GITHUB_RUN_NUMBER}"push=truelatest=falsechannel=dev@@ -131,7 +160,10 @@ jobs:latest=falsechannel=dry-runfi+ namespace="$IMAGE_NAMESPACE"+ [ "$channel" != dev ] || namespace="$IMAGE_NAMESPACE/dev"{+ echo "namespace=$namespace"echo "version=$tag_version"echo "python_version=$python_version"echo "push=$push"@@ -139,6 +171,41 @@ jobs:echo "channel=$channel"} | tee -a "$GITHUB_OUTPUT"+ - name: List the images+ # scripts/pin_chart_images.py is the one list of first-party images; the+ # build matrix, the pin and the recorded pins all come from it.+ id: images+ run: echo "images=$(python3 scripts/pin_chart_images.py images)" | tee -a "$GITHUB_OUTPUT"++ - name: Set up Helm+ if: ${{ steps.version.outputs.push == 'true' }}+ uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1++ - name: Refuse to republish a version+ # A re-run rebuilds the images (builds are not bit-for-bit reproducible)+ # and would push the chart again under the same version, pointing at the+ # new digests, so a version pin would no longer mean what it meant.+ # Checked before anything is built.+ if: ${{ steps.version.outputs.push == 'true' }}+ env:+ NAMESPACE: ${{ steps.version.outputs.namespace }}+ VERSION: ${{ steps.version.outputs.version }}+ run: |+ echo "${{ secrets.GITHUB_TOKEN }}" | \+ helm registry login "$REGISTRY" -u "${{ github.actor }}" --password-stdin+ chart="oci://$REGISTRY/$NAMESPACE/charts/switch"+ if out="$(helm show chart "$chart" --version "$VERSION" 2>&1)"; then+ echo "::error::$chart:$VERSION is already published. A version is published once; cut a new one."+ exit 1+ fi+ # Only a plain "not found" means unpublished; an auth or network+ # error must not read as permission to push.+ if ! grep -q ': not found$' <<<"$out"; then+ echo "::error::Could not tell whether $chart:$VERSION exists: $out"+ exit 1+ fi+ echo "$chart:$VERSION is not published yet: $out"+# The published compose artifact declares its stack-compose range twice:# in the file (for whoever has already downloaded it) and as an OCI# annotation here — which a consumer can read off the registry WITHOUT@@ -228,8 +295,8 @@ jobs:SWITCH_GIT_COMMIT=${{ github.sha }}SWITCH_GIT_REPOSITORY_URL=${{ github.server_url }}/${{ github.repository }}tags: |- ${{ env.REGISTRY }}/${{ env.IMAGE_NAMESPACE }}/${{ matrix.image }}:${{ needs.prep.outputs.version }}- ${{ needs.prep.outputs.latest == 'true' && format('{0}/{1}/{2}:latest', env.REGISTRY, env.IMAGE_NAMESPACE, matrix.image) || '' }}+ ${{ env.REGISTRY }}/${{ needs.prep.outputs.namespace }}/${{ matrix.image }}:${{ needs.prep.outputs.version }}+ ${{ needs.prep.outputs.latest == 'true' && format('{0}/{1}/{2}:latest', env.REGISTRY, needs.prep.outputs.namespace, matrix.image) || '' }}cache-from: type=gha,scope=${{ matrix.image }}cache-to: type=gha,scope=${{ matrix.image }},mode=max@@ -268,7 +335,7 @@ jobs:packages: writeenv:VERSION: ${{ needs.prep.outputs.version }}- PUSH: ${{ needs.prep.outputs.push }}+ NAMESPACE: ${{ needs.prep.outputs.namespace }}steps:- name: Checkoutuses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0@@ -292,26 +359,27 @@ jobs:username: ${{ github.actor }}password: ${{ secrets.GITHUB_TOKEN }}+ - name: Check the registry serves every pinned image+ if: ${{ needs.prep.outputs.push == 'true' }}+ run: |+ for file in "$RUNNER_TEMP"/digests/*; do+ docker buildx imagetools inspect "$REGISTRY/$NAMESPACE/$(basename "$file")@$(cat "$file")" >/dev/null+ done+- name: Pin image digests into the chart# A dry run pushed no images, so it pins placeholder digests: the pin# and the check below still run, and nothing is published.run: |- args=()- for image in switch-core gateway setup; do- if [ "$PUSH" = true ]; then- digest="$(cat "$RUNNER_TEMP/digests/$image")"- # The registry must serve the exact reference the chart will name.- docker buildx imagetools inspect "$REGISTRY/$IMAGE_NAMESPACE/$image@$digest" >/dev/null- else- digest="sha256:$(printf '0%.0s' $(seq 64))"- fi- args+=(--digest "$image=$digest")- done+ if [ "${{ needs.prep.outputs.push }}" = true ]; then+ source=(--digests-dir "$RUNNER_TEMP/digests")+ else+ source=(--placeholder-digests)+ fipython3 scripts/pin_chart_images.py pin \--values deploy/remote/helm/switch/values.yaml \- --registry "$REGISTRY/$IMAGE_NAMESPACE" \+ --registry "$REGISTRY/$NAMESPACE" \--version "$VERSION" \- "${args[@]}"+ "${source[@]}"git diff -- deploy/remote/helm/switch/values.yaml- name: Lint chart@@ -327,11 +395,15 @@ jobs:--destination "$RUNNER_TEMP/chart"- name: Check the packaged chart is digest-pinned- # Renders the .tgz that is about to be pushed, not the source tree.+ # Renders the .tgz that is about to be pushed, not the source tree:+ # once as configured by default, once with the optional workloads on.run: |- helm template switch "$RUNNER_TEMP/chart/switch-$VERSION.tgz" \- -f deploy/remote/helm/render-check-values.yaml \- | python3 scripts/pin_chart_images.py verify --registry "$REGISTRY/$IMAGE_NAMESPACE"+ for overlay in "" "-f deploy/remote/helm/render-check-optional-values.yaml"; do+ # shellcheck disable=SC2086 # $overlay is deliberately split into two words+ helm template switch "$RUNNER_TEMP/chart/switch-$VERSION.tgz" \+ -f deploy/remote/helm/render-check-values.yaml $overlay \+ | python3 scripts/pin_chart_images.py verify --registry "$REGISTRY/$NAMESPACE"+ done- name: Log in to GHCR (Helm OCI)if: ${{ needs.prep.outputs.push == 'true' }}@@ -343,8 +415,19 @@ jobs:id: pushif: ${{ needs.prep.outputs.push == 'true' }}run: |- helm push "$RUNNER_TEMP/chart/switch-$VERSION.tgz" \- "oci://$REGISTRY/$IMAGE_NAMESPACE/charts" 2>&1 | tee "$RUNNER_TEMP/helm-push.log"+ # prep refused a published version, but a re-run of this job alone+ # skips prep, so ask again right before pushing.+ chart="oci://$REGISTRY/$NAMESPACE/charts/switch"+ if out="$(helm show chart "$chart" --version "$VERSION" 2>&1)"; then+ echo "::error::$chart:$VERSION is already published; refusing to overwrite it."+ exit 1+ fi+ if ! grep -q ': not found$' <<<"$out"; then+ echo "::error::Could not tell whether $chart:$VERSION exists: $out"+ exit 1+ fi+ helm push "$RUNNER_TEMP/chart/switch-$VERSION.tgz" "oci://$REGISTRY/$NAMESPACE/charts" 2>&1 \+ | tee "$RUNNER_TEMP/helm-push.log"digest="$(sed -n 's/^Digest: *//p' "$RUNNER_TEMP/helm-push.log")"if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; thenecho "::error::helm push printed no chart digest; got '$digest'"@@ -357,37 +440,16 @@ jobs:# promotes this build next (an environment's pins file), so it never has# to be copied out of a log by hand.if: ${{ needs.prep.outputs.push == 'true' }}- env:- CHART_DIGEST: ${{ steps.push.outputs.digest }}- CHANNEL: ${{ needs.prep.outputs.channel }}run: |- python3 - <<'PY' > "$RUNNER_TEMP/release-pins.json"- import json, os, pathlib- ns = f"{os.environ['REGISTRY']}/{os.environ['IMAGE_NAMESPACE']}"- version = os.environ["VERSION"]- digests = pathlib.Path(os.environ["RUNNER_TEMP"], "digests")- print(json.dumps({- "version": version,- "channel": os.environ["CHANNEL"],- "commit": os.environ["GITHUB_SHA"],- "chart": {"ref": f"oci://{ns}/charts/switch", "version": version, "digest": os.environ["CHART_DIGEST"]},- "images": {- name: {"ref": f"{ns}/{name}", "digest": (digests / name).read_text().strip()}- for name in ("switch-core", "gateway", "setup")- },- }, indent=2))- PY- {- echo "## switch $VERSION ($CHANNEL)"- echo- echo "| Artifact | Pinned reference |"- echo "| --- | --- |"- echo "| chart | \`oci://$REGISTRY/$IMAGE_NAMESPACE/charts/switch:$VERSION@$CHART_DIGEST\` |"- for image in switch-core gateway setup; do- echo "| $image | \`$REGISTRY/$IMAGE_NAMESPACE/$image@$(cat "$RUNNER_TEMP/digests/$image")\` |"- done- } >> "$GITHUB_STEP_SUMMARY"- cat "$RUNNER_TEMP/release-pins.json"+ python3 scripts/pin_chart_images.py record \+ --digests-dir "$RUNNER_TEMP/digests" \+ --registry "$REGISTRY/$NAMESPACE" \+ --version "$VERSION" \+ --channel "${{ needs.prep.outputs.channel }}" \+ --commit "$GITHUB_SHA" \+ --chart-digest "${{ steps.push.outputs.digest }}" \+ --json "$RUNNER_TEMP/release-pins.json" \+ --summary "$GITHUB_STEP_SUMMARY"- name: Upload the pinsif: ${{ needs.prep.outputs.push == 'true' }}diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9bbe3f857..6602b2f20 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -87,26 +87,34 @@ version of their own to them without also giving them a release of their own.pods twice. Rendering fails when the secret is missing or shorter than 32characters. Remove variables set by hand before the first upgrade that rendersthem (see the chart README).-- **`global.preStopSleepSeconds`** (default `5`): switch-core, gateway and- Mattermost wait that long in a `preStop` hook before SIGTERM, so a roll stops- answering with 503s while endpoints and load balancers catch up. Uses the- native `sleep` action (Kubernetes 1.30+); `0` renders no hook.+- **`global.preStopSleepSeconds`** (default `5`): the gateway waits that long+ in a `preStop` hook before SIGTERM, so a roll stops sending it requests+ before it shuts down while its replacement is already serving. Rendered only+ on Kubernetes 1.30+ (the native `sleep` action); `0` turns it off.+ switch-core and Mattermost roll with `Recreate`, so they get no hook.#### Changed- **The published Helm chart pins its images by digest.** The release writeseach image's registry digest into the chart's values before packaging, so`switchCore.image`, `gateway.image` and `setup.image` default to- `<image>:<version>@sha256:…` with `imagePullPolicy: IfNotPresent`, and- `global.imageRegistry` defaults to the registry the release published to.- Pinning the chart version now pins the exact images; a deployment no longer- needs to set the three image values, and setting them by tag opts back out of- the pin. Values that set `global.imageRegistry` to a mirror keep working when- the mirror copies images byte for byte (digests survive a copy). A values- default change: check any values file that relied on the old defaults.+ `<image>:<version>@sha256:…`, and `global.imageRegistry` defaults to the+ registry the release published to. Pinning the chart version now pins the+ exact images; a deployment no longer needs to set the three image values.+- **Breaking for some values files: `global.imageRegistry` now has a default.**+ An image value that names a registry (`ghcr.io/acme/switch-core:1.0`,+ `localhost/…`, `host:5000/…`) is used as written, as before. One that names a+ Docker Hub namespace without a registry (`acme/switch-core:1.0`) now gets the+ default registry prepended; write it as `docker.io/acme/switch-core:1.0`.+- **`imagePullPolicy` for switch-core, gateway and setup defaults to `""`**:+ `IfNotPresent` for a digest-pinned image, `Always` for anything else, so an+ install that opts out of the pin with a mutable tag keeps pulling it. An+ explicit policy still wins.- **Every merge to main publishes a dev build** of the images and chart, under- `<version>-dev.<run>.g<sha>`: never `latest`, and skipped by Helm unless asked- for by exact version. Each run records every digest in its summary and in a- `release-pins` artifact.+ `ghcr.io/<owner>/dev/` and `<next patch>-dev.<run>.g<sha>`: never `latest`,+ apart from the release packages, and pruned after 30 days beyond the newest+ 20. Each publishing run records every digest in its summary and in a+ `release-pins` artifact, and a version already in the registry is never+ published again.#### Changed- **Agent management moved behind the `agent_management` feature flag.** The olddiff --git a/RELEASING.md b/RELEASING.mdindex 81a12620d..f1b89e090 100644--- a/RELEASING.md+++ b/RELEASING.md@@ -109,17 +109,24 @@ git push origin switch-v0.30.0-rc.1### Dev builds-Every push to `main` publishes the images and chart under a dev version,-`<pyproject version>-dev.<run number>.g<short sha>` (e.g.-`0.29.0-dev.512.g7f0c803`), so the dev environment runs main from published,-digest-pinned artifacts instead of building from source at deploy time.--- **Never `latest`**, and a semver pre-release, so `helm install` skips it unless- given that exact `--version`. No compose artifact: nothing consumes one.-- **The version names a commit, not a place in the release order.** Read the- short sha; do not compare a dev version with a release.-- **The server reports it** as `<pyproject version>.dev<run number>` (PEP 440),- stamped on the runner like an RC.+Every push to `main` publishes the images and chart as a dev build, so the dev+environment runs main from published, digest-pinned artifacts instead of+building from source at deploy time.++- **Separate packages.** Dev builds go to `ghcr.io/<owner>/dev/` (`dev/switch-core`,+ `dev/gateway`, `dev/setup`, `dev/charts/switch`), so the release packages'+ tag lists hold releases and RCs only. Nothing is ever tagged `latest`.+- **Version** `<next patch>-dev.<run number>.g<short sha>`, e.g.+ `0.29.1-dev.512.g7f0c803` after `switch-v0.29.0`: one patch above the newest+ release tag on main, so a dev build sorts after the release it contains and+ before the next release or RC. The server reports `0.29.1.dev512` (PEP 440),+ stamped on the runner like an RC. A semver pre-release, so `helm install`+ skips it unless given that exact `--version`. No compose artifact.+- **The newest push wins.** Pushes to main share a concurrency group, so a dev+ build still running when the next merge lands is cancelled.+- **Pruned.** `switch-dev-cleanup.yml` runs daily and deletes dev builds older+ than 30 days that are not among the 20 newest (`scripts/prune_dev_packages.py`;+ dispatch it with `dry_run` to see what it would delete).- **Not gated on PR CI.** It builds whatever reached main; what an environmentruns is decided by its pin, not by the build existing.@@ -139,16 +146,26 @@ switchCore:and the same for `gateway` and `setup`. So pinning the chart pins the exactimages: a deployment sets no image values, and what runs is reproducible fromthe chart pin alone. The tag in front of the digest is for people; the runtime-pulls by digest. Setting an image value by tag on install opts out of the pin.-A mirror set with `global.imageRegistry` keeps the pin as long as it copies+pulls by digest, and the chart pulls a digest-pinned image `IfNotPresent` (any+other reference `Always`). Setting an image value on install opts out of the+pin. A mirror set with `global.imageRegistry` keeps the pin as long as it copiesimages byte for byte, which registry-to-registry copies do.+**A version is published once.** Before building, the release asks the registry+whether the chart version exists and fails if it does (and asks again right+before pushing the chart). Builds are not bit-for-bit reproducible, so a re-run+would point the same version at different digests. Cut a new version instead.+Every publishing run lists all four digests (the chart's own included) in itsrun summary, and uploads them as a `release-pins` artifact (`release-pins.json`)-for whatever promotes the build next. `scripts/pin_chart_images.py` does the-pinning and the check; PR CI runs both with placeholder digests, so a new-first-party image value the pinner does not know about fails a pull request-rather than shipping by tag.+for whatever promotes the build next. `scripts/pin_chart_images.py` holds the one+list of first-party images (the build matrix comes from it), does the pinning,+and checks a rendered chart: every first-party image, recognised by repository+name whatever registry it names, must render from the pinning registry and by+digest. PR CI runs the pin with placeholder digests and the check on two+renders (defaults, and the optional workloads on), so an image the pinner does+not know about fails a pull request rather than shipping by tag. To add an+image, add it to `IMAGES` there.## Switch Console desktop app release (separate)diff --git a/core/tests/test_pin_chart_images.py b/core/tests/test_pin_chart_images.pyindex 949a7f32f..afd3e10c4 100644--- a/core/tests/test_pin_chart_images.py+++ b/core/tests/test_pin_chart_images.py@@ -8,7 +8,9 @@ release. Silently pinning nothing, or the wrong line, is the failure to rule outfrom __future__ import annotationsimport importlib.util+import jsonimport sys+from datetime import UTC, datetime, timedeltafrom pathlib import Pathfrom types import ModuleType@@ -25,9 +27,9 @@ DIGESTS = {}-def _load() -> ModuleType:- path = REPO_ROOT / "scripts" / "pin_chart_images.py"- spec = importlib.util.spec_from_file_location("pin_chart_images", path)+def _load(name: str) -> ModuleType:+ path = REPO_ROOT / "scripts" / f"{name}.py"+ spec = importlib.util.spec_from_file_location(name, path)assert spec is not None and spec.loader is not Nonemodule = importlib.util.module_from_spec(spec)sys.modules[spec.name] = module@@ -35,13 +37,20 @@ def _load() -> ModuleType:return module-pin_chart_images = _load()+pin_chart_images = _load("pin_chart_images")+prune_dev_packages = _load("prune_dev_packages")+++def test_the_image_table_matches_the_dockerfiles_and_values() -> None:+ values = yaml.safe_load(VALUES.read_text())+ for image in pin_chart_images.IMAGES.values():+ assert (REPO_ROOT / image.dockerfile).is_file()+ assert "image" in values[image.values_key]def test_pins_every_first_party_image_in_the_real_values_file() -> None:- original = VALUES.read_text()pinned = yaml.safe_load(- pin_chart_images.pin(original, "ghcr.io/acme", "1.2.3", DIGESTS)+ pin_chart_images.pin(VALUES.read_text(), "ghcr.io/acme", "1.2.3", DIGESTS))assert pinned["global"]["imageRegistry"] == "ghcr.io/acme"@@ -50,8 +59,6 @@ def test_pins_every_first_party_image_in_the_real_values_file() -> None:)assert pinned["gateway"]["image"] == f"gateway:1.2.3@{DIGESTS['gateway']}"assert pinned["setup"]["image"] == f"setup:1.2.3@{DIGESTS['setup']}"- for key in ("switchCore", "gateway", "setup"):- assert pinned[key]["imagePullPolicy"] == "IfNotPresent"def test_changes_only_the_pinned_lines() -> None:@@ -63,10 +70,9 @@ def test_changes_only_the_pinned_lines() -> None:for a, b in zip(original.splitlines(), pinned.splitlines(), strict=True)if a != b]- assert len(changed) == 7 # the registry, then an image and a pull policy per image- assert (- yaml.safe_load(original)["mattermost"] == yaml.safe_load(pinned)["mattermost"]- )+ assert len(changed) == 4 # the registry, then one image line per image+ # Pull policy is left to the chart, which derives it from the digest.+ assert yaml.safe_load(pinned)["switchCore"]["imagePullPolicy"] == ""@pytest.mark.parametrize(@@ -86,6 +92,18 @@ def test_refuses_digests_that_do_not_cover_exactly_the_images(pin_chart_images.pin(VALUES.read_text(), "ghcr.io/acme", "1.2.3", digests)+def test_reads_digests_from_a_directory_and_refuses_a_stray_file(+ tmp_path: Path,+) -> None:+ for name, digest in DIGESTS.items():+ (tmp_path / name).write_text(digest + "\n")+ assert pin_chart_images.read_digests(tmp_path) == DIGESTS++ (tmp_path / "sidecar").write_text("sha256:" + "d" * 64)+ with pytest.raises(pin_chart_images.PinError, match="sidecar"):+ pin_chart_images.read_digests(tmp_path)++def test_refuses_a_values_file_whose_shape_moved() -> None:values = VALUES.read_text().replace("\n image: gateway:", "\n img: gateway:")with pytest.raises(pin_chart_images.PinError, match="gateway.image"):@@ -122,30 +140,104 @@ def _pinned(name: str) -> str:return f"ghcr.io/acme/{name}:1.2.3@{DIGESTS[name]}"-def test_verify_accepts_a_fully_pinned_render() -> None:- rendered = "---\n".join(+def _render(*extra: str) -> str:+ return "---\n".join([_pod(_pinned("switch-core"), init=(_pinned("switch-core"), "busybox:1.36")),_pod(_pinned("gateway")),_pod(_pinned("setup"), "postgres:16-alpine"),+ *(_pod(ref) for ref in extra),])- assert len(pin_chart_images.verify(rendered, "ghcr.io/acme")) == 3-def test_verify_rejects_a_first_party_image_by_tag() -> None:- rendered = "---\n".join(- [- _pod(_pinned("switch-core"), init=("ghcr.io/acme/switch-core:1.2.3",)),- _pod(_pinned("gateway")),- _pod(_pinned("setup")),- ]- )- with pytest.raises(pin_chart_images.PinError, match="not pinned"):- pin_chart_images.verify(rendered, "ghcr.io/acme")+def test_verify_accepts_a_fully_pinned_render() -> None:+ assert len(pin_chart_images.verify(_render(), "ghcr.io/acme")) == 3+++ ("extra", "problem"),+ [+ ("ghcr.io/acme/switch-core:1.2.3", "not pinned by digest"),+ ("switch-core:latest", "not from ghcr.io/acme"),+ ("ghcr.io/other/gateway:1.2.3@" + DIGESTS["gateway"], "not from ghcr.io/acme"),+ ("registry.example:5000/setup", "not pinned by digest"),+ (+ "ghcr.io/acme/sidecar:1.2.3@" + "sha256:" + "d" * 64,+ "not a known first-party image",+ ),+ ],+ ids=[+ "by-tag",+ "bypasses-registry",+ "other-registry",+ "port-in-host",+ "unknown-image",+ ],+)+def test_verify_rejects_a_stray_first_party_reference(extra: str, problem: str) -> None:+ with pytest.raises(pin_chart_images.PinError, match=problem):+ pin_chart_images.verify(_render(extra), "ghcr.io/acme")def test_verify_rejects_a_render_missing_an_image() -> None:rendered = "---\n".join([_pod(_pinned("switch-core")), _pod(_pinned("gateway"))])with pytest.raises(pin_chart_images.PinError, match="setup"):pin_chart_images.verify(rendered, "ghcr.io/acme")+++def test_record_lists_every_image_and_the_chart() -> None:+ chart = "sha256:" + "e" * 64+ pins, summary = pin_chart_images.record(+ DIGESTS, "ghcr.io/acme/dev", "1.2.4-dev.7.gabc1234", "dev", "abc", chart+ )++ loaded = json.loads(pins)+ assert loaded["chart"] == {+ "ref": "oci://ghcr.io/acme/dev/charts/switch",+ "version": "1.2.4-dev.7.gabc1234",+ "digest": chart,+ }+ assert {+ name: entry["digest"] for name, entry in loaded["images"].items()+ } == DIGESTS+ for name, digest in DIGESTS.items():+ assert f"ghcr.io/acme/dev/{name}@{digest}" in summary+++def _version(created: datetime, *tags: str) -> dict:+ return {+ "id": id(created),+ "name": "x",+ "created_at": created.isoformat(),+ "metadata": {"container": {"tags": list(tags)}},+ }+++NOW = datetime(2026, 10, 9, tzinfo=UTC)+++def test_prune_keeps_everything_young_or_among_the_newest() -> None:+ builds = [_version(NOW - timedelta(days=40 + n), f"b{n}") for n in range(5)]+ assert prune_dev_packages.doomed(builds, NOW, timedelta(days=30), keep=5) == []+++def test_prune_deletes_old_builds_beyond_the_kept_ones_with_their_platform_manifests() -> (+ None+):+ builds = []+ for n in range(4):+ index = NOW - timedelta(days=40 + n)+ builds.append(_version(index, f"b{n}"))+ builds.append(+ _version(index - timedelta(seconds=30))+ ) # its untagged platform manifest++ doomed = prune_dev_packages.doomed(builds, NOW, timedelta(days=30), keep=2)++ kept = [v for v in builds if v not in doomed]+ assert {tag for v in kept for tag in v["metadata"]["container"]["tags"]} == {+ "b0",+ "b1",+ }+ assert len(kept) == 4 # two indexes and both of their platform manifestsdiff --git a/deploy/remote/helm/render-check-optional-values.yaml b/deploy/remote/helm/render-check-optional-values.yamlnew file mode 100644index 000000000..b853f64be--- /dev/null+++ b/deploy/remote/helm/render-check-optional-values.yaml@@ -0,0 +1,12 @@+# Layered over render-check-values.yaml for a second render with the chart's+# optional workloads switched on, so an image reference behind a feature flag+# is checked too (see scripts/pin_chart_images.py). Never deploy with these.+mattermost:+ enabled: true+switchCore:+ migrations:+ preUpgradeJob: true+ teamsBridge:+ enabled: true+ ingress:+ mode: externaldiff --git a/deploy/remote/helm/switch/templates/_helpers.tpl b/deploy/remote/helm/switch/templates/_helpers.tplindex 634dd2a8f..29a3c90a0 100644--- a/deploy/remote/helm/switch/templates/_helpers.tpl+++ b/deploy/remote/helm/switch/templates/_helpers.tpl@@ -435,33 +435,59 @@ than something you discover hours later at Graph subscription time.{{- end }}{{/*-preStop drain for a container behind a Service: hold the old pod for a few-seconds after it is marked terminating, so endpoint and load-balancer-deregistration land before the process gets SIGTERM. The native `sleep` action-needs no shell, which the distroless images do not have.+preStop drain for a pod that rolls with a surge (a new pod is Ready before the+old one stops): hold the old pod for a few seconds after it is marked+terminating, so endpoint and load-balancer deregistration land before the+process gets SIGTERM. Not for a Recreate Deployment, where nothing else is+serving and the wait only lengthens the outage. The native `sleep` action needs+no shell (the images are distroless) but needs Kubernetes 1.30+, so nothing+renders on an older cluster.Usage: {{- include "switch.preStopDrain" . | nindent 10 }}*/}}{{- define "switch.preStopDrain" -}}-{{- with .Values.global.preStopSleepSeconds -}}+{{- if and .Values.global.preStopSleepSeconds (semverCompare ">=1.30-0" .Capabilities.KubeVersion.Version) -}}lifecycle:preStop:sleep:- seconds: {{ . }}+ seconds: {{ .Values.global.preStopSleepSeconds }}{{- end }}{{- end }}{{/*-Prepend the global image registry if set.+Prepend the global image registry if set, unless the reference already names+a registry: a first path component with a "." or ":" in it, or "localhost",+which is how a container runtime tells a registry host from a repository+namespace. So `ghcr.io/acme/switch-core:1.0` is used as written whatever+global.imageRegistry says.Usage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.switchCore.image) }}*/}}{{- define "switch.image" -}}-{{- if .global.imageRegistry -}}+{{- $parts := splitList "/" .image -}}+{{- $host := first $parts -}}+{{- $hasRegistry := and (gt (len $parts) 1) (or (contains "." $host) (contains ":" $host) (eq $host "localhost")) -}}+{{- if and .global.imageRegistry (not $hasRegistry) -}}{{- printf "%s/%s" .global.imageRegistry .image }}{{- else -}}{{- .image }}{{- end -}}{{- end }}+{{/*+Pull policy for a first-party image. An explicit policy wins. Otherwise a+digest-pinned reference cannot change, so IfNotPresent; anything else may be a+mutable tag, so Always, or nodes would keep running a stale cached image.+Usage: {{ include "switch.imagePullPolicy" (dict "policy" .Values.switchCore.imagePullPolicy "image" .Values.switchCore.image) }}+*/}}+{{- define "switch.imagePullPolicy" -}}+{{- if .policy -}}+{{- .policy -}}+{{- else if contains "@sha256:" .image -}}+IfNotPresent+{{- else -}}+Always+{{- end -}}+{{- end }}+{{/*imagePullSecrets block from .Values.global.imagePullSecrets (a list of Secretnames) — for pulling images from a private registry. Renders nothing when thediff --git a/deploy/remote/helm/switch/templates/gateway/deployment.yaml b/deploy/remote/helm/switch/templates/gateway/deployment.yamlindex 80e0a3124..90b37df63 100644--- a/deploy/remote/helm/switch/templates/gateway/deployment.yaml+++ b/deploy/remote/helm/switch/templates/gateway/deployment.yaml@@ -24,7 +24,7 @@ spec:containers:- name: gatewayimage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.gateway.image) }}- imagePullPolicy: {{ .Values.gateway.imagePullPolicy }}+ imagePullPolicy: {{ include "switch.imagePullPolicy" (dict "policy" .Values.gateway.imagePullPolicy "image" .Values.gateway.image) }}ports:- containerPort: 3000protocol: TCPdiff --git a/deploy/remote/helm/switch/templates/mattermost/deployment.yaml b/deploy/remote/helm/switch/templates/mattermost/deployment.yamlindex 637d03301..3e42f2cff 100644--- a/deploy/remote/helm/switch/templates/mattermost/deployment.yaml+++ b/deploy/remote/helm/switch/templates/mattermost/deployment.yaml@@ -87,7 +87,6 @@ spec:- name: MM_SERVICESETTINGS_ALLOWEDUNTRUSTEDINTERNALCONNECTIONSvalue: "{{ $core }} {{ $core }}.{{ .Release.Namespace }} {{ $core }}.{{ .Release.Namespace }}.svc.cluster.local"{{- end }}- {{- include "switch.preStopDrain" . | nindent 10 }}readinessProbe:httpGet:path: /api/v4/system/pingdiff --git a/deploy/remote/helm/switch/templates/setup/job.yaml b/deploy/remote/helm/switch/templates/setup/job.yamlindex 892455d56..e87db67c4 100644--- a/deploy/remote/helm/switch/templates/setup/job.yaml+++ b/deploy/remote/helm/switch/templates/setup/job.yaml@@ -27,7 +27,7 @@ spec:containers:- name: setupimage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.setup.image) }}- imagePullPolicy: {{ .Values.setup.imagePullPolicy }}+ imagePullPolicy: {{ include "switch.imagePullPolicy" (dict "policy" .Values.setup.imagePullPolicy "image" .Values.setup.image) }}env:- name: SWITCH_URLvalue: "http://{{ include "switch.switchCoreHost" . }}:{{ .Values.switchCore.service.port }}"diff --git a/deploy/remote/helm/switch/templates/switch-core/deployment.yaml b/deploy/remote/helm/switch/templates/switch-core/deployment.yamlindex a5c6b1d33..091ef32fe 100644--- a/deploy/remote/helm/switch/templates/switch-core/deployment.yaml+++ b/deploy/remote/helm/switch/templates/switch-core/deployment.yaml@@ -39,7 +39,7 @@ spec:# finds nothing pending.- name: migrateimage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.switchCore.image) }}- imagePullPolicy: {{ .Values.switchCore.imagePullPolicy }}+ imagePullPolicy: {{ include "switch.imagePullPolicy" (dict "policy" .Values.switchCore.imagePullPolicy "image" .Values.switchCore.image) }}command: ["switch-migrate"]env:{{- include "switch.coreEnv" . | nindent 12 }}@@ -52,7 +52,7 @@ spec:containers:- name: switch-coreimage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.switchCore.image) }}- imagePullPolicy: {{ .Values.switchCore.imagePullPolicy }}+ imagePullPolicy: {{ include "switch.imagePullPolicy" (dict "policy" .Values.switchCore.imagePullPolicy "image" .Values.switchCore.image) }}ports:- containerPort: 8000protocol: TCP@@ -82,7 +82,6 @@ spec:value: /etc/switch-github/github.json{{- end }}{{- include "switch.coreEnv" (merge (dict "omitOwnerCredentials" true) .) | nindent 12 }}- {{- include "switch.preStopDrain" . | nindent 10 }}# Readiness checks the database; liveness deliberately does not.# There is one replica, so a failing readiness probe empties the# Service rather than shifting traffic — which is right while thediff --git a/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml b/deploy/remote/helm/switch/templates/switch-core/migrate-job.yamlindex c71a00f66..4ccd59e1b 100644--- a/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml+++ b/deploy/remote/helm/switch/templates/switch-core/migrate-job.yaml@@ -47,7 +47,7 @@ spec:containers:- name: migrateimage: {{ include "switch.image" (dict "global" .Values.global "image" .Values.switchCore.image) }}- imagePullPolicy: {{ .Values.switchCore.imagePullPolicy }}+ imagePullPolicy: {{ include "switch.imagePullPolicy" (dict "policy" .Values.switchCore.imagePullPolicy "image" .Values.switchCore.image) }}command: ["switch-migrate"]env:{{- include "switch.coreEnv" $ctx | nindent 12 }}diff --git a/deploy/remote/helm/switch/values.yaml b/deploy/remote/helm/switch/values.yamlindex 0406cc0f6..e59728457 100644--- a/deploy/remote/helm/switch/values.yaml+++ b/deploy/remote/helm/switch/values.yaml@@ -10,11 +10,12 @@ global:# private registry (e.g. a private GHCR namespace). Applied to every pod. Leave# empty for public images.imagePullSecrets: []- # Seconds each pod behind a Service (switch-core, gateway, Mattermost) waits in- # a preStop hook before it is sent SIGTERM, so the endpoint and any load- # balancer stop routing to it first and a roll does not answer requests with- # 503s. Uses the native `sleep` lifecycle action (Kubernetes 1.30+); set 0 to- # render no hook on an older cluster.+ # Seconds the gateway pod waits in a preStop hook before SIGTERM on a roll, so+ # the endpoint and any load balancer stop routing to it while its replacement+ # is already serving, and the roll answers no requests with 502/503s. Only+ # rendered on Kubernetes 1.30+ (the native `sleep` lifecycle action); 0 turns+ # it off. switch-core and Mattermost roll with Recreate, where there is no+ # replacement to drain to, so they get no hook.preStopSleepSeconds: 5# ── Ingress ─────────────────────────────────────────────────────────────────@@ -188,7 +189,9 @@ switchCore:hostedDiskRetentionDays: 7hostedIdleStopMinutes: 30image: switch-core:latest- imagePullPolicy: Always+ # Empty: IfNotPresent when the image is pinned by digest (as in a published+ # chart), Always otherwise. Set it to override.+ imagePullPolicy: ""logLevel: INFO# Logging. "text" is for reading in a terminal; "json" is what a log pipeline@@ -588,7 +591,9 @@ switchCore:# ── Gateway (React + nginx) ──────────────────────────────────────────────────gateway:image: gateway:latest- imagePullPolicy: Always+ # Empty: IfNotPresent when the image is pinned by digest (as in a published+ # chart), Always otherwise. Set it to override.+ imagePullPolicy: ""replicaCount: 1service:type: ClusterIP@@ -654,7 +659,9 @@ mattermost:# ── Setup Job ─────────────────────────────────────────────────────────────────setup:image: setup:latest- imagePullPolicy: Always+ # Empty: IfNotPresent when the image is pinned by digest (as in a published+ # chart), Always otherwise. Set it to override.+ imagePullPolicy: ""backoffLimit: 5# ── PostgreSQL ────────────────────────────────────────────────────────────────diff --git a/scripts/pin_chart_images.py b/scripts/pin_chart_images.pyindex 85d1f4488..a65d5a868 100644--- a/scripts/pin_chart_images.py+++ b/scripts/pin_chart_images.py@@ -8,45 +8,71 @@ whatever the tag points at on the day. This writes each image's registry digestinto the chart's `values.yaml` before packaging, so pinning the chart versionpins the images with it, and what runs is reproducible from that one pin.-Two subcommands:+`IMAGES` below is the only list of first-party images. The release workflow+builds its image matrix from `images`, and every other subcommand refuses a set+of digests that is not exactly that list, so adding an image here is the whole+change and forgetting one anywhere fails loudly.+Subcommands:++* `images` prints the image matrix as JSON, for the release workflow.* `pin` rewrites `values.yaml` in place: `global.imageRegistry`, and for each- first-party image `<name>:<version>@sha256:…` with pull policy- `IfNotPresent` (a digest cannot change under the tag, so `Always` only costs a- registry round trip). The edit is line-based so the file's comments, which- are the chart's documentation under `helm show values`, survive.+ first-party image `<name>:<version>@sha256:…` (the chart pulls a digest-pinned+ image `IfNotPresent` on its own). The edit is line-based so the file's+ comments, which are the chart's documentation under `helm show values`,+ survive.* `verify` reads `helm template` output and fails unless every first-party- image is digest-pinned and all of them are present. The release runs it on- the chart it is about to push; PR CI runs it on a chart pinned with- placeholder digests, so a new image value the pinner does not know about- fails a pull request instead of shipping by tag.+ image — recognised by repository name, whatever registry it names — renders+ from the pinning registry and by digest, all of them are present, and the+ pinning registry serves no image this script does not know.+* `record` writes the pins (images and chart) as JSON for whatever promotes the+ build next, and as a Markdown table for the run summary.Usage:+ python scripts/pin_chart_images.py imagespython scripts/pin_chart_images.py pin --values <values.yaml> \\- --registry ghcr.io/<owner> --version <version> \\- --digest switch-core=sha256:… --digest gateway=sha256:… --digest setup=sha256:…+ --registry ghcr.io/<owner> --version <version> --digests-dir <dir>+ python scripts/pin_chart_images.py pin ... --placeholder-digestshelm template x <chart> | python scripts/pin_chart_images.py verify --registry ghcr.io/<owner>+ python scripts/pin_chart_images.py record --digests-dir <dir> --registry ghcr.io/<owner> \\+ --version <version> --channel <channel> --commit <sha> --chart-digest sha256:… \\+ --json <out.json> --summary <out.md>++`--digests-dir` holds one file per image, named by the image, containing its+`sha256:` digest: what the release's image jobs upload."""from __future__ import annotationsimport argparse+import jsonimport reimport sysfrom collections.abc import Iterator+from dataclasses import dataclassfrom pathlib import Pathfrom typing import Anyimport yaml-# Image name as published -> the top-level values key whose `image` it fills.-IMAGES: dict[str, str] = {- "switch-core": "switchCore",- "gateway": "gateway",- "setup": "setup",++@dataclass(frozen=True)+class Image:+ values_key: str+ dockerfile: str+++# Image name as published -> where it is built and which values key it fills.+IMAGES: dict[str, Image] = {+ "switch-core": Image(+ "switchCore", "deploy/shared_resources/images/Dockerfile.switch"+ ),+ "gateway": Image("gateway", "deploy/shared_resources/images/Dockerfile.gateway"),+ "setup": Image("setup", "deploy/shared_resources/images/Dockerfile.setup"),}DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")+PLACEHOLDER_DIGEST = "sha256:" + "0" * 64TOP_LEVEL_KEY = re.compile(r"^([A-Za-z0-9_]+):\s*(#.*)?$")@@ -86,7 +112,7 @@ def _set_child(lines: list[str], parent: str, child: str, value: str) -> None:lines[hits[0]] = f" {child}: {value}{newline}"-def pin(values_text: str, registry: str, version: str, digests: dict[str, str]) -> str:+def _check_digests(digests: dict[str, str]) -> None:missing = sorted(set(IMAGES) - set(digests))unknown = sorted(set(digests) - set(IMAGES))if missing or unknown:@@ -96,26 +122,47 @@ def pin(values_text: str, registry: str, version: str, digests: dict[str, str])for name, digest in digests.items():if not DIGEST.match(digest):raise PinError(f"{name}: {digest!r} is not a sha256 digest")+++def _check_registry(registry: str) -> None:if not registry or registry.endswith("/"):raise PinError(f"registry {registry!r} must be non-empty with no trailing slash")+++def read_digests(directory: Path) -> dict[str, str]:+ if not directory.is_dir():+ raise PinError(f"digests directory {directory} does not exist")+ digests = {+ path.name: path.read_text().strip()+ for path in directory.iterdir()+ if path.is_file()+ }+ _check_digests(digests)+ return digests+++def pin(values_text: str, registry: str, version: str, digests: dict[str, str]) -> str:+ _check_digests(digests)+ _check_registry(registry)if not version:raise PinError("version must be non-empty")lines = values_text.splitlines(keepends=True)_set_child(lines, "global", "imageRegistry", f'"{registry}"')- for name, key in IMAGES.items():- _set_child(lines, key, "image", f"{name}:{version}@{digests[name]}")- _set_child(lines, key, "imagePullPolicy", "IfNotPresent")+ for name, image in IMAGES.items():+ _set_child(+ lines, image.values_key, "image", f"{name}:{version}@{digests[name]}"+ )pinned = "".join(lines)loaded = yaml.safe_load(pinned)if loaded["global"]["imageRegistry"] != registry:raise PinError("pinned values do not read back the registry")- for name, key in IMAGES.items():- if loaded[key]["image"] != f"{name}:{version}@{digests[name]}":- raise PinError(f"pinned values do not read back {key}.image")+ for name, image in IMAGES.items():+ if loaded[image.values_key]["image"] != f"{name}:{version}@{digests[name]}":+ raise PinError(f"pinned values do not read back {image.values_key}.image")return pinned@@ -132,35 +179,89 @@ def _images(node: Any) -> Iterator[str]:yield from _images(item)+def _repository(ref: str) -> str:+ """`reg/ns/name:tag@sha256:…` -> `reg/ns/name`."""+ ref = ref.split("@", 1)[0]+ slash = ref.rfind("/")+ colon = ref.rfind(":")+ return ref[:colon] if colon > slash else ref++def verify(rendered: str, registry: str) -> list[str]:"""Return every first-party image reference, or raise if any is unpinned."""+ _check_registry(registry)refs = sorted({ref for doc in yaml.safe_load_all(rendered) if doc for ref in _images(doc)})prefix = f"{registry}/"- ours = [ref for ref in refs if ref.startswith(prefix)]- unpinned = [ref for ref in ours if not re.search(r"@sha256:[0-9a-f]{64}$", ref)]- if unpinned:- raise PinError(f"first-party images not pinned by digest: {unpinned}")- seen = {ref[len(prefix) :].split("@")[0].split(":")[0] for ref in ours}+ problems: list[str] = []+ ours: list[str] = []+ seen: set[str] = set()+ for ref in refs:+ repository = _repository(ref)+ name = repository.rsplit("/", 1)[-1]+ if name in IMAGES:+ ours.append(ref)+ seen.add(name)+ if repository != f"{registry}/{name}":+ problems.append(f"{ref}: not from {registry}")+ if not re.search(r"@sha256:[0-9a-f]{64}$", ref):+ problems.append(f"{ref}: not pinned by digest")+ elif ref.startswith(prefix):+ problems.append(+ f"{ref}: from {registry} but not a known first-party image; add it to IMAGES"+ )absent = sorted(set(IMAGES) - seen)if absent:+ problems.append(f"missing from the rendered chart: {absent}")+ if problems:raise PinError(- f"first-party images missing from the rendered chart: {absent}; rendered images: {refs}"+ "chart is not pinned:\n "+ + "\n ".join(problems)+ + f"\nrendered images: {refs}")return ours-def _parse_digests(pairs: list[str]) -> dict[str, str]:- digests: dict[str, str] = {}- for pair in pairs:- name, sep, digest = pair.partition("=")- if not sep:- raise PinError(f"--digest {pair!r} must be <image>=sha256:<hex>")- if name in digests:- raise PinError(f"--digest {name} given twice")- digests[name] = digest- return digests+def record(+ digests: dict[str, str],+ registry: str,+ version: str,+ channel: str,+ commit: str,+ chart_digest: str,+) -> tuple[str, str]:+ _check_digests(digests)+ _check_registry(registry)+ if not DIGEST.match(chart_digest):+ raise PinError(f"chart: {chart_digest!r} is not a sha256 digest")+ pins = {+ "version": version,+ "channel": channel,+ "commit": commit,+ "chart": {+ "ref": f"oci://{registry}/charts/switch",+ "version": version,+ "digest": chart_digest,+ },+ "images": {+ name: {"ref": f"{registry}/{name}", "digest": digests[name]}+ for name in IMAGES+ },+ }+ rows = [f"| chart | `oci://{registry}/charts/switch:{version}@{chart_digest}` |"]+ rows += [f"| {name} | `{registry}/{name}@{digests[name]}` |" for name in IMAGES]+ summary = "\n".join(+ [+ f"## switch {version} ({channel})",+ "",+ "| Artifact | Pinned reference |",+ "| --- | --- |",+ *rows,+ "",+ ]+ )+ return json.dumps(pins, indent=2) + "\n", summarydef main(argv: list[str]) -> int:@@ -169,28 +270,74 @@ def main(argv: list[str]) -> int:)sub = parser.add_subparsers(dest="command", required=True)+ sub.add_parser("images", help="print the image build matrix as JSON")+p_pin = sub.add_parser("pin", help="write image digests into values.yaml")p_pin.add_argument("--values", required=True, type=Path)p_pin.add_argument("--registry", required=True)p_pin.add_argument("--version", required=True)- p_pin.add_argument("--digest", required=True, action="append", default=[])+ source = p_pin.add_mutually_exclusive_group(required=True)+ source.add_argument("--digests-dir", type=Path)+ source.add_argument(+ "--placeholder-digests",+ action="store_true",+ help="for a dry run that pushed nothing",+ )p_verify = sub.add_parser("verify", help="check rendered manifests (stdin) are digest-pinned")p_verify.add_argument("--registry", required=True)+ p_record = sub.add_parser(+ "record", help="write the pins as JSON and a Markdown summary"+ )+ p_record.add_argument("--digests-dir", required=True, type=Path)+ p_record.add_argument("--registry", required=True)+ p_record.add_argument("--version", required=True)+ p_record.add_argument("--channel", required=True)+ p_record.add_argument("--commit", required=True)+ p_record.add_argument("--chart-digest", required=True)+ p_record.add_argument("--json", required=True, type=Path)+ p_record.add_argument("--summary", required=True, type=Path)+args = parser.parse_args(argv)try:- if args.command == "pin":- text = args.values.read_text()+ if args.command == "images":+ print(+ json.dumps(+ [+ {"image": name, "dockerfile": image.dockerfile}+ for name, image in IMAGES.items()+ ]+ )+ )+ elif args.command == "pin":+ digests = (+ dict.fromkeys(IMAGES, PLACEHOLDER_DIGEST)+ if args.placeholder_digests+ else read_digests(args.digests_dir)+ )args.values.write_text(- pin(text, args.registry, args.version, _parse_digests(args.digest))+ pin(args.values.read_text(), args.registry, args.version, digests))print(f"pinned {args.values}")- else:+ elif args.command == "verify":for ref in verify(sys.stdin.read(), args.registry):print(ref)+ else:+ pins, summary = record(+ read_digests(args.digests_dir),+ args.registry,+ args.version,+ args.channel,+ args.commit,+ args.chart_digest,+ )+ args.json.write_text(pins)+ with args.summary.open("a") as out:+ out.write(summary)+ print(pins, end="")except PinError as exc:print(f"error: {exc}", file=sys.stderr)return 1diff --git a/scripts/prune_dev_packages.py b/scripts/prune_dev_packages.pynew file mode 100644index 000000000..97a7a17c4--- /dev/null+++ b/scripts/prune_dev_packages.py@@ -0,0 +1,119 @@+#!/usr/bin/env python3+"""Delete old dev builds from GHCR.++Every push to main publishes the images and chart under `<owner>/dev/…`+(see .github/workflows/switch-release.yml). Those packages hold nothing but dev+builds, so pruning them cannot touch a release. A version is deleted when it is+older than `--max-age-days` AND older than the `--keep` newest tagged builds,+so a quiet month does not empty the package and leave the dev environment+pinned to a digest that no longer exists.++A multi-arch image is an index plus one untagged manifest per platform, pushed+moments before it. The cutoff is set an hour before the oldest kept build, so a+kept index keeps its platform manifests; untagged manifests older than that+belong to builds being deleted anyway.++Needs `gh` authenticated with a token that may delete the packages' versions+(the publishing repository's GITHUB_TOKEN with `packages: write`).++Usage:+ python scripts/prune_dev_packages.py --owner <org> --max-age-days 30 --keep 20 [--dry-run]+"""++from __future__ import annotations++import argparse+import importlib.util+import json+import subprocess+import sys+from datetime import UTC, datetime, timedelta+from pathlib import Path+from urllib.parse import quote+++def _first_party_images() -> list[str]:+ path = Path(__file__).with_name("pin_chart_images.py")+ spec = importlib.util.spec_from_file_location("pin_chart_images", path)+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return list(module.IMAGES)+++def _gh(*args: str) -> str:+ return subprocess.run(+ ["gh", "api", *args], check=True, capture_output=True, text=True+ ).stdout+++def _versions(owner: str, package: str) -> list[dict]:+ pages = _gh(+ "--paginate",+ "--slurp",+ f"/orgs/{owner}/packages/container/{quote(package, safe='')}/versions",+ )+ return [version for page in json.loads(pages) for version in page]+++def _created(version: dict) -> datetime:+ return datetime.fromisoformat(version["created_at"].replace("Z", "+00:00"))+++def doomed(+ versions: list[dict], now: datetime, max_age: timedelta, keep: int+) -> list[dict]:+ tagged = sorted(+ (v for v in versions if v.get("metadata", {}).get("container", {}).get("tags")),+ key=_created,+ reverse=True,+ )+ if len(tagged) <= keep:+ return []+ cutoff = min(now - max_age, _created(tagged[keep - 1]) - timedelta(hours=1))+ return [v for v in versions if _created(v) < cutoff]+++def main(argv: list[str]) -> int:+ parser = argparse.ArgumentParser(+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter+ )+ parser.add_argument("--owner", required=True)+ parser.add_argument("--max-age-days", type=int, required=True)+ parser.add_argument("--keep", type=int, required=True)+ parser.add_argument("--dry-run", action="store_true")+ args = parser.parse_args(argv)+ if args.keep < 1:+ parser.error("--keep must be at least 1")++ now = datetime.now(UTC)+ packages = [f"dev/{name}" for name in _first_party_images()] + ["dev/charts/switch"]+ for package in packages:+ try:+ versions = _versions(args.owner, package)+ except subprocess.CalledProcessError as exc:+ if "404" in exc.stderr or "Not Found" in exc.stderr:+ print(f"{package}: not published yet")+ continue+ raise+ delete = doomed(versions, now, timedelta(days=args.max_age_days), args.keep)+ print(f"{package}: {len(versions)} versions, deleting {len(delete)}")+ for version in delete:+ tags = version.get("metadata", {}).get("container", {}).get("tags") or [+ "<untagged>"+ ]+ print(+ f" {'would delete' if args.dry_run else 'delete'} {version['name']} {','.join(tags)} {version['created_at']}"+ )+ if not args.dry_run:+ _gh(+ "--method",+ "DELETE",+ f"/orgs/{args.owner}/packages/container/{quote(package, safe='')}/versions/{version['id']}",+ )+ return 0+++if __name__ == "__main__":+ sys.exit(main(sys.argv[1:]))