Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
ci(release): publish the hosted controller image and chart, pinned by digest
Petr Bauch26ef9aae31f2
diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.ymlindex 3b03b4ba2..2259a1539 100644--- a/.github/workflows/pr-ci.yml+++ b/.github/workflows/pr-ci.yml@@ -110,11 +110,14 @@ jobs:# actually find a new head — not on every backend-touching push.migrations:- 'core/switch_core/migrations/versions/**'- # The switch Helm chart and what the release does to it before- # publishing (pins the image digests into values.yaml).+ # The published Helm charts (switch, and the hosted controller's)+ # and what the release does to them before publishing (pins the+ # image digests into values.yaml).chart:- *shared- 'deploy/remote/helm/**'+ - 'deploy/hosted/chart/**'+ - 'deploy/hosted/render-check-values.yaml'- 'scripts/pin_chart_images.py'- '.github/workflows/switch-release.yml'# Spans every tree, so it gets its own filter rather than riding@@ -269,21 +272,26 @@ jobs:- name: Set up Helmuses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1- - name: Lint- run: helm lint deploy/remote/helm/switch-- name: Every first-party image is pinned by digest once pinned- # Rendered as configured by default and with the optional workloads on,- # so an image behind a feature flag is checked too.+ # The release's own pin, with placeholder digests, then each of the+ # chart's check renders (defaults, and the optional workloads on), so an+ # image behind a feature flag is checked too.run: |- python3 scripts/pin_chart_images.py pin \- --values deploy/remote/helm/switch/values.yaml \- --registry ghcr.io/render-check --version 0.0.0-check --placeholder-digests- for overlay in "" "-f deploy/remote/helm/render-check-optional-values.yaml"; do- # shellcheck disable=SC2086 # $overlay is deliberately split into two words- helm template switch deploy/remote/helm/switch \- -f deploy/remote/helm/render-check-values.yaml $overlay \- | python3 scripts/pin_chart_images.py verify --registry ghcr.io/render-check+ python3 scripts/pin_chart_images.py charts > "$RUNNER_TEMP/charts.json"+ for chart in $(jq -r '.[].chart' "$RUNNER_TEMP/charts.json"); do+ path="$(jq -r --arg c "$chart" '.[] | select(.chart == $c) | .path' "$RUNNER_TEMP/charts.json")"+ python3 scripts/pin_chart_images.py pin --chart "$chart" \+ --registry ghcr.io/render-check --version 0.0.0-check --placeholder-digests+ renders="$(jq -c --arg c "$chart" '.[] | select(.chart == $c) | .renders[]' "$RUNNER_TEMP/charts.json")"+ while read -r files; do+ args=()+ for f in $(jq -r '.[]' <<<"$files"); do args+=(-f "$f"); done+ # Linted with the check values: a chart with a values schema+ # does not lint without the values it requires.+ helm lint "$path" "${args[@]}"+ helm template switch "$path" "${args[@]}" \+ | python3 scripts/pin_chart_images.py verify --chart "$chart" --registry ghcr.io/render-check+ done <<<"$renders"donegateway:diff --git a/.github/workflows/switch-release.yml b/.github/workflows/switch-release.ymlindex 4032c3a84..72729dd05 100644--- a/.github/workflows/switch-release.yml+++ b/.github/workflows/switch-release.yml@@ -182,10 +182,7 @@ jobs:uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1- name: Refuse to republish a version- # A re-run rebuilds the images (builds are not bit-for-bit reproducible)- # and would push the chart again under the same version, pointing at the- # new digests, so a version pin would no longer mean what it meant.- # Checked before anything is built.+ # Checked before anything is built; see scripts/refuse_published_chart.sh.if: ${{ steps.version.outputs.push == 'true' }}env:NAMESPACE: ${{ steps.version.outputs.namespace }}@@ -193,18 +190,9 @@ jobs:run: |echo "${{ secrets.GITHUB_TOKEN }}" | \helm registry login "$REGISTRY" -u "${{ github.actor }}" --password-stdin- chart="oci://$REGISTRY/$NAMESPACE/charts/switch"- if out="$(helm show chart "$chart" --version "$VERSION" 2>&1)"; then- echo "::error::$chart:$VERSION is already published. A version is published once; cut a new one."- exit 1- fi- # Only a plain "not found" means unpublished; an auth or network- # error must not read as permission to push.- if ! grep -q ': not found$' <<<"$out"; then- echo "::error::Could not tell whether $chart:$VERSION exists: $out"- exit 1- fi- echo "$chart:$VERSION is not published yet: $out"+ for chart in $(python3 scripts/pin_chart_images.py charts | jq -r '.[].chart'); do+ scripts/refuse_published_chart.sh "oci://$REGISTRY/$NAMESPACE/charts/$chart" "$VERSION"+ done# The published compose artifact declares its stack-compose range twice:# in the file (for whoever has already downloaded it) and as an OCI@@ -324,10 +312,11 @@ jobs:retention-days: 7chart:- name: Package & publish Helm chart- # After the images: the chart carries their digests, so it cannot be+ name: Package & publish Helm charts+ # After the images: each chart carries its images' digests, so it cannot be# packaged before they exist, and a chart is never published pointing at an- # image that failed to push.+ # image that failed to push. The charts come from+ # scripts/pin_chart_images.py (`charts`), like the image matrix.needs: [prep, images]runs-on: ubuntu-latestpermissions:@@ -339,10 +328,17 @@ jobs:steps:- name: Checkoutuses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0+ with:+ # To ask, before moving the dev `main` tag, whether this is still main's head.+ fetch-depth: 0- name: Set up Helmuses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1+ - name: Set up ORAS+ if: ${{ needs.prep.outputs.channel == 'dev' }}+ uses: oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6 # v1.2.4+- name: Download image digestsif: ${{ needs.prep.outputs.push == 'true' }}uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0@@ -366,43 +362,34 @@ jobs:docker buildx imagetools inspect "$REGISTRY/$NAMESPACE/$(basename "$file")@$(cat "$file")" >/dev/nulldone- - name: Pin image digests into the chart+ - name: Pin, lint, package and check each chart# A dry run pushed no images, so it pins placeholder digests: the pin- # and the check below still run, and nothing is published.+ # and the checks still run, and nothing is published. The check renders+ # the .tgz that is about to be pushed, not the source tree.run: |if [ "${{ needs.prep.outputs.push }}" = true ]; thensource=(--digests-dir "$RUNNER_TEMP/digests")elsesource=(--placeholder-digests)fi- python3 scripts/pin_chart_images.py pin \- --values deploy/remote/helm/switch/values.yaml \- --registry "$REGISTRY/$NAMESPACE" \- --version "$VERSION" \- "${source[@]}"- git diff -- deploy/remote/helm/switch/values.yaml-- - name: Lint chart- run: helm lint deploy/remote/helm/switch-- - name: Package chart- # Stamp both the chart version and appVersion with the release version so- # a single tag pins the whole stack.- run: |- helm package deploy/remote/helm/switch \- --version "$VERSION" \- --app-version "$VERSION" \- --destination "$RUNNER_TEMP/chart"-- - name: Check the packaged chart is digest-pinned- # Renders the .tgz that is about to be pushed, not the source tree:- # once as configured by default, once with the optional workloads on.- run: |- for overlay in "" "-f deploy/remote/helm/render-check-optional-values.yaml"; do- # shellcheck disable=SC2086 # $overlay is deliberately split into two words- helm template switch "$RUNNER_TEMP/chart/switch-$VERSION.tgz" \- -f deploy/remote/helm/render-check-values.yaml $overlay \- | python3 scripts/pin_chart_images.py verify --registry "$REGISTRY/$NAMESPACE"+ python3 scripts/pin_chart_images.py charts > "$RUNNER_TEMP/charts.json"+ for chart in $(jq -r '.[].chart' "$RUNNER_TEMP/charts.json"); do+ path="$(jq -r --arg c "$chart" '.[] | select(.chart == $c) | .path' "$RUNNER_TEMP/charts.json")"+ python3 scripts/pin_chart_images.py pin --chart "$chart" \+ --registry "$REGISTRY/$NAMESPACE" --version "$VERSION" "${source[@]}"+ git diff -- "$path/values.yaml"+ helm package "$path" --version "$VERSION" --app-version "$VERSION" \+ --destination "$RUNNER_TEMP/chart"+ renders="$(jq -c --arg c "$chart" '.[] | select(.chart == $c) | .renders[]' "$RUNNER_TEMP/charts.json")"+ while read -r files; do+ args=()+ for f in $(jq -r '.[]' <<<"$files"); do args+=(-f "$f"); done+ # Linted with the check values: a chart with a values schema+ # does not lint without the values it requires.+ helm lint "$path" "${args[@]}"+ helm template switch "$RUNNER_TEMP/chart/$chart-$VERSION.tgz" "${args[@]}" \+ | python3 scripts/pin_chart_images.py verify --chart "$chart" --registry "$REGISTRY/$NAMESPACE"+ done <<<"$renders"done- name: Log in to GHCR (Helm OCI)@@ -411,29 +398,25 @@ jobs:echo "${{ secrets.GITHUB_TOKEN }}" | \helm registry login "$REGISTRY" -u "${{ github.actor }}" --password-stdin- - name: Push chart+ - name: Push the chartsid: pushif: ${{ needs.prep.outputs.push == 'true' }}run: |- # prep refused a published version, but a re-run of this job alone- # skips prep, so ask again right before pushing.- chart="oci://$REGISTRY/$NAMESPACE/charts/switch"- if out="$(helm show chart "$chart" --version "$VERSION" 2>&1)"; then- echo "::error::$chart:$VERSION is already published; refusing to overwrite it."- exit 1- fi- if ! grep -q ': not found$' <<<"$out"; then- echo "::error::Could not tell whether $chart:$VERSION exists: $out"- exit 1- fi- helm push "$RUNNER_TEMP/chart/switch-$VERSION.tgz" "oci://$REGISTRY/$NAMESPACE/charts" 2>&1 \- | tee "$RUNNER_TEMP/helm-push.log"- digest="$(sed -n 's/^Digest: *//p' "$RUNNER_TEMP/helm-push.log")"- if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then- echo "::error::helm push printed no chart digest; got '$digest'"- exit 1- fi- echo "digest=$digest" >> "$GITHUB_OUTPUT"+ flags=""+ for chart in $(jq -r '.[].chart' "$RUNNER_TEMP/charts.json"); do+ # prep refused a published version, but a re-run of this job alone+ # skips prep, so ask again right before pushing.+ scripts/refuse_published_chart.sh "oci://$REGISTRY/$NAMESPACE/charts/$chart" "$VERSION"+ helm push "$RUNNER_TEMP/chart/$chart-$VERSION.tgz" "oci://$REGISTRY/$NAMESPACE/charts" 2>&1 \+ | tee "$RUNNER_TEMP/helm-push.log"+ digest="$(sed -n 's/^Digest: *//p' "$RUNNER_TEMP/helm-push.log")"+ if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then+ echo "::error::helm push of $chart printed no chart digest; got '$digest'"+ exit 1+ fi+ flags="$flags --chart-digest $chart=$digest"+ done+ echo "chart_digest_flags=$flags" >> "$GITHUB_OUTPUT"- name: Record the pins# The run summary is for a person; release-pins.json is for whatever@@ -441,13 +424,14 @@ jobs:# to be copied out of a log by hand.if: ${{ needs.prep.outputs.push == 'true' }}run: |+ # shellcheck disable=SC2086 # the flags are deliberately split into wordspython3 scripts/pin_chart_images.py record \--digests-dir "$RUNNER_TEMP/digests" \--registry "$REGISTRY/$NAMESPACE" \--version "$VERSION" \--channel "${{ needs.prep.outputs.channel }}" \--commit "$GITHUB_SHA" \- --chart-digest "${{ steps.push.outputs.digest }}" \+ ${{ steps.push.outputs.chart_digest_flags }} \--json "$RUNNER_TEMP/release-pins.json" \--summary "$GITHUB_STEP_SUMMARY"@@ -459,6 +443,24 @@ jobs:path: ${{ runner.temp }}/release-pins.jsonif-no-files-found: error+ - name: Move the dev `main` tag+ # The dev environment follows main: its ArgoCD Application tracks the+ # `main` tag of the dev charts, so moving it is the deploy. Only the+ # build of main's current head moves it; an older build finishing late+ # leaves it alone, and the newer build moves it when it finishes.+ if: ${{ needs.prep.outputs.channel == 'dev' }}+ run: |+ head="$(git ls-remote origin refs/heads/main | cut -f1)"+ if [ "$head" != "$GITHUB_SHA" ]; then+ echo "main is at $head now, not $GITHUB_SHA; leaving the main tag to that build."+ exit 0+ fi+ echo "${{ secrets.GITHUB_TOKEN }}" | \+ oras login "$REGISTRY" -u "${{ github.actor }}" --password-stdin+ for chart in $(jq -r '.[].chart' "$RUNNER_TEMP/charts.json"); do+ oras tag "$REGISTRY/$NAMESPACE/charts/$chart:$VERSION" main+ done+compose:name: Publish standalone compose (OCI artifact)needs: prepdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6602b2f20..e27caf9ed 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -115,6 +115,11 @@ version of their own to them without also giving them a release of their own.20. Each publishing run records every digest in its summary and in a`release-pins` artifact, and a version already in the registry is neverpublished again.+- **The hosted controller is published with every release and dev build**: its+ image as `switch-hosted-controller` and its chart as+ `charts/switch-hosted-controller`, versioned with switch-core and pinned to+ its image by digest (`image.repository`, `image.digest`). The dev charts also+ carry a `main` tag that follows the newest build of main.#### Changed- **Agent management moved behind the `agent_management` feature flag.** The olddiff --git a/RELEASING.md b/RELEASING.mdindex f1b89e090..42ac94e68 100644--- a/RELEASING.md+++ b/RELEASING.md@@ -114,8 +114,14 @@ environment runs main from published, digest-pinned artifacts instead ofbuilding from source at deploy time.- **Separate packages.** Dev builds go to `ghcr.io/<owner>/dev/` (`dev/switch-core`,- `dev/gateway`, `dev/setup`, `dev/charts/switch`), so the release packages'- tag lists hold releases and RCs only. Nothing is ever tagged `latest`.+ `dev/gateway`, `dev/setup`, `dev/switch-hosted-controller`, `dev/charts/switch`,+ `dev/charts/switch-hosted-controller`), so the release packages' tag lists+ hold releases and RCs only. Nothing is ever tagged `latest`.+- **The dev charts carry a moving `main` tag.** Once both charts are pushed,+ the run moves their `main` tag to its build, but only while the commit it+ built is still the tip of main, so a slower run cannot move it backwards. An+ environment that follows main (development, through ArgoCD) tracks that tag+ and resolves it to a digest; every other environment pins a version.- **Version** `<next patch>-dev.<run number>.g<short sha>`, e.g.`0.29.1-dev.512.g7f0c803` after `switch-v0.29.0`: one patch above the newestrelease tag on main, so a dev build sorts after the release it contains and@@ -143,7 +149,9 @@ switchCore:imagePullPolicy: IfNotPresent```-and the same for `gateway` and `setup`. So pinning the chart pins the exact+and the same for `gateway` and `setup`. The hosted controller chart+(`charts/switch-hosted-controller`) is pinned the same way, through its+`image.repository` and `image.digest`. So pinning a chart pins the exactimages: a deployment sets no image values, and what runs is reproducible fromthe chart pin alone. The tag in front of the digest is for people; the runtimepulls by digest, and the chart pulls a digest-pinned image `IfNotPresent` (any@@ -156,16 +164,18 @@ whether the chart version exists and fails if it does (and asks again rightbefore pushing the chart). Builds are not bit-for-bit reproducible, so a re-runwould point the same version at different digests. Cut a new version instead.-Every publishing run lists all four digests (the chart's own included) in its+Every publishing run lists every digest (the charts' own included) in itsrun summary, and uploads them as a `release-pins` artifact (`release-pins.json`)for whatever promotes the build next. `scripts/pin_chart_images.py` holds the one-list of first-party images (the build matrix comes from it), does the pinning,-and checks a rendered chart: every first-party image, recognised by repository-name whatever registry it names, must render from the pinning registry and by-digest. PR CI runs the pin with placeholder digests and the check on two-renders (defaults, and the optional workloads on), so an image the pinner does-not know about fails a pull request rather than shipping by tag. To add an-image, add it to `IMAGES` there.+list of charts and first-party images (the build matrix comes from it), does+the pinning, and checks each rendered chart: every image of that chart,+recognised by repository name whatever registry it names, must render from the+pinning registry and by digest, and no first-party image of another chart may+appear. PR CI runs the pin with placeholder digests and the check on each+chart's renders (for the switch chart, defaults and the optional workloads+on), so an image the pinner does not know about fails a pull request rather+than shipping by tag. To add an image, add it to `IMAGES` there; to add a+chart, to `CHARTS`.## Switch Console desktop app release (separate)@@ -276,7 +286,8 @@ Consuming the published artifacts:# imagesdocker pull ghcr.io/<owner>/switch-core:<version>-# chart (its images are pinned inside it by digest)+# chart (its images are pinned inside it by digest); the hosted controller's+# chart is charts/switch-hosted-controllerhelm install switch oci://ghcr.io/<owner>/charts/switch --version <version> \-f my-values.yamldiff --git a/artifacts.yaml b/artifacts.yamlindex 2a2fe580c..9e0b839bf 100644--- a/artifacts.yaml+++ b/artifacts.yaml@@ -131,6 +131,20 @@ artifacts:version_from: switch-corepublished_as: charts/switch+ hosted-controller:+ description: >-+ The hosted controller image, which provisions Switch cloud machines for+ switch-core.+ version_from: switch-core+ published_as: switch-hosted-controller++ hosted-controller-chart:+ description: >-+ The hosted controller's Kubernetes chart, pinned to its image by digest+ like the switch chart.+ version_from: switch-core+ published_as: charts/switch-hosted-controller+compose:description: >-The standalone compose file, published as an OCI artifact. Unlike thediff --git a/console/packages/shared/src/artifacts.ts b/console/packages/shared/src/artifacts.tsindex a49088489..e5db475e5 100644--- a/console/packages/shared/src/artifacts.ts+++ b/console/packages/shared/src/artifacts.ts@@ -28,6 +28,8 @@ export const ARTIFACT_VERSIONS = {setup: '0.29.0','hosted-controller': '0.29.0','helm-chart': '0.29.0',+ 'hosted-controller': '0.29.0',+ 'hosted-controller-chart': '0.29.0',compose: '0.29.0',} as const satisfies Record<string, string>;diff --git a/console/packages/switch-agent-runtime/src/artifacts.ts b/console/packages/switch-agent-runtime/src/artifacts.tsindex a49088489..e5db475e5 100644--- a/console/packages/switch-agent-runtime/src/artifacts.ts+++ b/console/packages/switch-agent-runtime/src/artifacts.ts@@ -28,6 +28,8 @@ export const ARTIFACT_VERSIONS = {setup: '0.29.0','hosted-controller': '0.29.0','helm-chart': '0.29.0',+ 'hosted-controller': '0.29.0',+ 'hosted-controller-chart': '0.29.0',compose: '0.29.0',} as const satisfies Record<string, string>;diff --git a/core/switch_core/artifacts.py b/core/switch_core/artifacts.pyindex 29638acde..3b6a8ea5c 100644--- a/core/switch_core/artifacts.py+++ b/core/switch_core/artifacts.py@@ -28,6 +28,8 @@ ARTIFACT_VERSIONS: Final[dict[str, str]] = {"setup": "0.29.0","hosted-controller": "0.29.0","helm-chart": "0.29.0",+ "hosted-controller": "0.29.0",+ "hosted-controller-chart": "0.29.0","compose": "0.29.0",}diff --git a/core/tests/test_pin_chart_images.py b/core/tests/test_pin_chart_images.pyindex afd3e10c4..617ceb968 100644--- a/core/tests/test_pin_chart_images.py+++ b/core/tests/test_pin_chart_images.py@@ -20,11 +20,15 @@ import yamlREPO_ROOT = Path(__file__).resolve().parents[2]VALUES = REPO_ROOT / "deploy/remote/helm/switch/values.yaml"+HOSTED_VALUES = REPO_ROOT / "deploy/hosted/chart/values.yaml"+DIGESTS = {"switch-core": "sha256:" + "a" * 64,"gateway": "sha256:" + "b" * 64,"setup": "sha256:" + "c" * 64,+ "switch-hosted-controller": "sha256:" + "f" * 64,}+SWITCH_IMAGES = ("switch-core", "gateway", "setup")def _load(name: str) -> ModuleType:@@ -42,15 +46,60 @@ prune_dev_packages = _load("prune_dev_packages")def test_the_image_table_matches_the_dockerfiles_and_values() -> None:- values = yaml.safe_load(VALUES.read_text())- for image in pin_chart_images.IMAGES.values():- assert (REPO_ROOT / image.dockerfile).is_file()- assert "image" in values[image.values_key]+ for name, image in pin_chart_images.IMAGES.items():+ chart = pin_chart_images.CHARTS[image.chart]+ values = yaml.safe_load((REPO_ROOT / chart.path / "values.yaml").read_text())+ assert (REPO_ROOT / image.dockerfile).is_file(), name+ assert (REPO_ROOT / image.context).is_dir(), name+ assert image.values_key in values, name+ assert set(pin_chart_images.images_of("switch")) == set(SWITCH_IMAGES)+++def test_every_chart_has_its_render_check_values() -> None:+ for chart in pin_chart_images.CHARTS.values():+ assert (REPO_ROOT / chart.path / "Chart.yaml").is_file()+ for render in chart.renders:+ for values in render:+ assert (REPO_ROOT / values).is_file(), values+++def test_pins_the_hosted_controller_chart_by_repository_and_digest() -> None:+ original = HOSTED_VALUES.read_text()+ pinned = pin_chart_images.pin(+ "switch-hosted-controller", original, "ghcr.io/acme", "1.2.3", DIGESTS+ )++ assert yaml.safe_load(pinned)["image"] == {+ "repository": "ghcr.io/acme/switch-hosted-controller",+ "digest": DIGESTS["switch-hosted-controller"],+ }+ changed = [+ a+ for a, b in zip(original.splitlines(), pinned.splitlines(), strict=True)+ if a != b+ ]+ assert len(changed) == 2+++def test_verify_scopes_each_chart_to_its_own_images() -> None:+ hosted = _pod(_pinned("switch-hosted-controller"))+ assert pin_chart_images.verify(+ "switch-hosted-controller", hosted, "ghcr.io/acme"+ ) == [_pinned("switch-hosted-controller")]++ with pytest.raises(+ pin_chart_images.PinError, match="not an image of the switch chart"+ ):+ pin_chart_images.verify(+ "switch", _render(_pinned("switch-hosted-controller")), "ghcr.io/acme"+ )def test_pins_every_first_party_image_in_the_real_values_file() -> None:pinned = yaml.safe_load(- pin_chart_images.pin(VALUES.read_text(), "ghcr.io/acme", "1.2.3", DIGESTS)+ pin_chart_images.pin(+ "switch", VALUES.read_text(), "ghcr.io/acme", "1.2.3", DIGESTS+ ))assert pinned["global"]["imageRegistry"] == "ghcr.io/acme"@@ -63,7 +112,7 @@ def test_pins_every_first_party_image_in_the_real_values_file() -> None:def test_changes_only_the_pinned_lines() -> None:original = VALUES.read_text()- pinned = pin_chart_images.pin(original, "ghcr.io/acme", "1.2.3", DIGESTS)+ pinned = pin_chart_images.pin("switch", original, "ghcr.io/acme", "1.2.3", DIGESTS)changed = [(a, b)@@ -89,7 +138,9 @@ def test_refuses_digests_that_do_not_cover_exactly_the_images(digests: dict[str, str],) -> None:with pytest.raises(pin_chart_images.PinError):- pin_chart_images.pin(VALUES.read_text(), "ghcr.io/acme", "1.2.3", digests)+ pin_chart_images.pin(+ "switch", VALUES.read_text(), "ghcr.io/acme", "1.2.3", digests+ )def test_reads_digests_from_a_directory_and_refuses_a_stray_file(@@ -107,13 +158,13 @@ def test_reads_digests_from_a_directory_and_refuses_a_stray_file(def test_refuses_a_values_file_whose_shape_moved() -> None:values = VALUES.read_text().replace("\n image: gateway:", "\n img: gateway:")with pytest.raises(pin_chart_images.PinError, match="gateway.image"):- pin_chart_images.pin(values, "ghcr.io/acme", "1.2.3", DIGESTS)+ pin_chart_images.pin("switch", values, "ghcr.io/acme", "1.2.3", DIGESTS)def test_does_not_pin_a_nested_image_key() -> None:values = "global:\n imageRegistry: ''\nswitchCore:\n sidecar:\n image: x\n"with pytest.raises(pin_chart_images.PinError, match="switchCore.image"):- pin_chart_images.pin(values, "r", "1", DIGESTS)+ pin_chart_images.pin("switch", values, "r", "1", DIGESTS)def _pod(*images: str, init: tuple[str, ...] = ()) -> str:@@ -152,7 +203,7 @@ def _render(*extra: str) -> str:def test_verify_accepts_a_fully_pinned_render() -> None:- assert len(pin_chart_images.verify(_render(), "ghcr.io/acme")) == 3+ assert len(pin_chart_images.verify("switch", _render(), "ghcr.io/acme")) == 3@pytest.mark.parametrize(@@ -164,7 +215,7 @@ def test_verify_accepts_a_fully_pinned_render() -> None:("registry.example:5000/setup", "not pinned by digest"),("ghcr.io/acme/sidecar:1.2.3@" + "sha256:" + "d" * 64,- "not a known first-party image",+ "not an image of the switch chart",),],ids=[@@ -177,26 +228,32 @@ def test_verify_accepts_a_fully_pinned_render() -> None:)def test_verify_rejects_a_stray_first_party_reference(extra: str, problem: str) -> None:with pytest.raises(pin_chart_images.PinError, match=problem):- pin_chart_images.verify(_render(extra), "ghcr.io/acme")+ pin_chart_images.verify("switch", _render(extra), "ghcr.io/acme")def test_verify_rejects_a_render_missing_an_image() -> None:rendered = "---\n".join([_pod(_pinned("switch-core")), _pod(_pinned("gateway"))])with pytest.raises(pin_chart_images.PinError, match="setup"):- pin_chart_images.verify(rendered, "ghcr.io/acme")+ pin_chart_images.verify("switch", rendered, "ghcr.io/acme")-def test_record_lists_every_image_and_the_chart() -> None:- chart = "sha256:" + "e" * 64+def test_record_lists_every_image_and_chart() -> None:+ charts = {+ "switch": "sha256:" + "e" * 64,+ "switch-hosted-controller": "sha256:" + "d" * 64,+ }pins, summary = pin_chart_images.record(- DIGESTS, "ghcr.io/acme/dev", "1.2.4-dev.7.gabc1234", "dev", "abc", chart+ DIGESTS, "ghcr.io/acme/dev", "1.2.4-dev.7.gabc1234", "dev", "abc", charts)loaded = json.loads(pins)- assert loaded["chart"] == {- "ref": "oci://ghcr.io/acme/dev/charts/switch",- "version": "1.2.4-dev.7.gabc1234",- "digest": chart,+ assert loaded["charts"] == {+ name: {+ "ref": f"oci://ghcr.io/acme/dev/charts/{name}",+ "version": "1.2.4-dev.7.gabc1234",+ "digest": digest,+ }+ for name, digest in charts.items()}assert {name: entry["digest"] for name, entry in loaded["images"].items()@@ -205,6 +262,22 @@ def test_record_lists_every_image_and_the_chart() -> None:assert f"ghcr.io/acme/dev/{name}@{digest}" in summary+def test_record_refuses_a_missing_chart_digest() -> None:+ with pytest.raises(pin_chart_images.PinError, match="chart digests"):+ pin_chart_images.record(+ DIGESTS, "r", "1", "dev", "abc", {"switch": "sha256:" + "e" * 64}+ )+++def test_parses_chart_digest_flags_and_refuses_a_repeat() -> None:+ digest = "sha256:" + "e" * 64+ assert pin_chart_images._parse_pairs([f"switch={digest}"]) == {"switch": digest}+ with pytest.raises(pin_chart_images.PinError):+ pin_chart_images._parse_pairs([f"switch={digest}", f"switch={digest}"])+ with pytest.raises(pin_chart_images.PinError):+ pin_chart_images._parse_pairs([digest])++def _version(created: datetime, *tags: str) -> dict:return {"id": id(created),diff --git a/deploy/hosted/chart/Chart.yaml b/deploy/hosted/chart/Chart.yamlindex aaad0ab93..5e8665f23 100644--- a/deploy/hosted/chart/Chart.yaml+++ b/deploy/hosted/chart/Chart.yaml@@ -1,5 +1,9 @@apiVersion: v2name: switch-hosted-controllerdescription: Single-controller lifecycle service for Switch cloud machines on EC2-version: 0.1.4+# Placeholder, like the switch chart's: the switch-release workflow stamps the+# switch-core release version at package time and publishes this chart beside+# the switch chart, with the controller image's digest written into its values.+version: 0.0.0-dev+appVersion: "0.0.0-dev"type: applicationdiff --git a/deploy/hosted/render-check-values.yaml b/deploy/hosted/render-check-values.yamlnew file mode 100644index 000000000..d0821d911--- /dev/null+++ b/deploy/hosted/render-check-values.yaml@@ -0,0 +1,19 @@+# Placeholder values that let `helm template` render the hosted controller chart+# in CI, where the only question is which image it references (see+# scripts/pin_chart_images.py). Never deploy with these.+serviceAccount:+ roleArn: arn:aws:iam::000000000000:role/render-check+controllerConfig:+ installation_id: render-check+ region: eu-west-3+ availability_zone: eu-west-3a+ subnet_id: subnet-00000000+ security_group_ids: [sg-00000000]+ image_id: ami-00000000+ root_device_name: /dev/xvda+ allowed_instance_types: [t3.large]+ max_machines: 1+ root_volume_gib: 30+ data_volume_gib: 30+ instance_profile_arn: arn:aws:iam::000000000000:instance-profile/render-check+ poll_interval_seconds: 30diff --git a/scripts/pin_chart_images.py b/scripts/pin_chart_images.pyindex a65d5a868..be20c5e45 100644--- a/scripts/pin_chart_images.py+++ b/scripts/pin_chart_images.py@@ -1,41 +1,52 @@#!/usr/bin/env python3-"""Pin the Helm chart to the exact images it was built with, and check that it is.+"""Pin the Helm charts to the exact images they were built with, and check that they are.-The release builds the switch-core, gateway and setup images and then packages-the chart. Left alone, the chart's values name those images by a mutable tag,-so every deployment re-states which images go with which chart and gets-whatever the tag points at on the day. This writes each image's registry digest-into the chart's `values.yaml` before packaging, so pinning the chart version-pins the images with it, and what runs is reproducible from that one pin.+The release builds the images and then packages the charts. Left alone, a+chart's values name its images by a mutable tag (or not at all), so every+deployment re-states which images go with which chart and gets whatever the tag+points at on the day. This writes each image's registry digest into its chart's+`values.yaml` before packaging, so pinning a chart version pins its images with+it, and what runs is reproducible from that one pin.-`IMAGES` below is the only list of first-party images. The release workflow-builds its image matrix from `images`, and every other subcommand refuses a set-of digests that is not exactly that list, so adding an image here is the whole-change and forgetting one anywhere fails loudly.+Two charts are published:++* `switch` (deploy/remote/helm/switch): switch-core, gateway and setup.+* `switch-hosted-controller` (deploy/hosted/chart): the controller that runs+ Switch cloud machines, and its one image.++`IMAGES` and `CHARTS` below are the only lists. The release workflow builds its+image matrix from `images` and packages the charts `charts` names, and every+other subcommand refuses a set of digests that is not exactly `IMAGES`, so+adding an image or a chart here is the whole change and forgetting one anywhere+fails loudly.Subcommands:-* `images` prints the image matrix as JSON, for the release workflow.-* `pin` rewrites `values.yaml` in place: `global.imageRegistry`, and for each- first-party image `<name>:<version>@sha256:…` (the chart pulls a digest-pinned- image `IfNotPresent` on its own). The edit is line-based so the file's- comments, which are the chart's documentation under `helm show values`,- survive.-* `verify` reads `helm template` output and fails unless every first-party- image — recognised by repository name, whatever registry it names — renders- from the pinning registry and by digest, all of them are present, and the- pinning registry serves no image this script does not know.-* `record` writes the pins (images and chart) as JSON for whatever promotes the+* `images` prints the image build matrix as JSON; `charts` prints the charts.+* `pin --chart <name>` rewrites that chart's `values.yaml` in place. The switch+ chart gets `global.imageRegistry` and `<name>:<version>@sha256:…` per image+ (it pulls a digest-pinned image `IfNotPresent` on its own); the hosted+ controller chart gets `image.repository` and `image.digest`. The edit is+ line-based so the file's comments, which are the chart's documentation under+ `helm show values`, survive.+* `verify --chart <name>` reads `helm template` output and fails unless every+ image of that chart — recognised by repository name, whatever registry it+ names — renders from the pinning registry and by digest, all of them are+ present, and the pinning registry serves no image the chart should not have.+* `record` writes the pins (images and charts) as JSON for whatever promotes thebuild next, and as a Markdown table for the run summary.Usage:python scripts/pin_chart_images.py images- python scripts/pin_chart_images.py pin --values <values.yaml> \\- --registry ghcr.io/<owner> --version <version> --digests-dir <dir>+ python scripts/pin_chart_images.py charts+ python scripts/pin_chart_images.py pin --chart switch --registry ghcr.io/<owner> \\+ --version <version> --digests-dir <dir>python scripts/pin_chart_images.py pin ... --placeholder-digests- helm template x <chart> | python scripts/pin_chart_images.py verify --registry ghcr.io/<owner>+ helm template x <chart> | python scripts/pin_chart_images.py verify --chart switch \\+ --registry ghcr.io/<owner>python scripts/pin_chart_images.py record --digests-dir <dir> --registry ghcr.io/<owner> \\- --version <version> --channel <channel> --commit <sha> --chart-digest sha256:… \\+ --version <version> --channel <channel> --commit <sha> \\+ --chart-digest switch=sha256:… --chart-digest switch-hosted-controller=sha256:… \\--json <out.json> --summary <out.md>`--digests-dir` holds one file per image, named by the image, containing its@@ -58,19 +69,65 @@ import yaml@dataclass(frozen=True)class Image:+ chart: str+ # Top-level values key holding the image: `<key>.image` in the switch+ # chart, `<key>.repository` / `<key>.digest` in the hosted controller's.values_key: strdockerfile: str+ context: str-# Image name as published -> where it is built and which values key it fills.+@dataclass(frozen=True)+class Chart:+ path: str+ # Each inner list is one `helm template` render (its -f files) that+ # `verify` checks: placeholder secrets, and the optional workloads on.+ renders: tuple[tuple[str, ...], ...]+++CHARTS: dict[str, Chart] = {+ "switch": Chart(+ "deploy/remote/helm/switch",+ (+ ("deploy/remote/helm/render-check-values.yaml",),+ (+ "deploy/remote/helm/render-check-values.yaml",+ "deploy/remote/helm/render-check-optional-values.yaml",+ ),+ ),+ ),+ "switch-hosted-controller": Chart(+ "deploy/hosted/chart", (("deploy/hosted/render-check-values.yaml",),)+ ),+}++# Image name as published -> its chart, which values key it fills, and how it+# is built.IMAGES: dict[str, Image] = {"switch-core": Image(- "switchCore", "deploy/shared_resources/images/Dockerfile.switch"+ "switch", "switchCore", "deploy/shared_resources/images/Dockerfile.switch", "."+ ),+ "gateway": Image(+ "switch", "gateway", "deploy/shared_resources/images/Dockerfile.gateway", "."+ ),+ "setup": Image(+ "switch", "setup", "deploy/shared_resources/images/Dockerfile.setup", "."+ ),+ "switch-hosted-controller": Image(+ "switch-hosted-controller",+ "image",+ "deploy/hosted/controller/Dockerfile",+ "deploy/hosted/controller",),- "gateway": Image("gateway", "deploy/shared_resources/images/Dockerfile.gateway"),- "setup": Image("setup", "deploy/shared_resources/images/Dockerfile.setup"),}++def images_of(chart: str) -> dict[str, Image]:+ if chart not in CHARTS:+ raise PinError(f"unknown chart {chart!r}; known: {sorted(CHARTS)}")+ return {name: image for name, image in IMAGES.items() if image.chart == chart}++DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")PLACEHOLDER_DIGEST = "sha256:" + "0" * 64TOP_LEVEL_KEY = re.compile(r"^([A-Za-z0-9_]+):\s*(#.*)?$")@@ -143,26 +200,33 @@ def read_digests(directory: Path) -> dict[str, str]:return digests-def pin(values_text: str, registry: str, version: str, digests: dict[str, str]) -> str:+def pin(+ chart: str, values_text: str, registry: str, version: str, digests: dict[str, str]+) -> str:_check_digests(digests)_check_registry(registry)if not version:raise PinError("version must be non-empty")+ images = images_of(chart)lines = values_text.splitlines(keepends=True)- _set_child(lines, "global", "imageRegistry", f'"{registry}"')- for name, image in IMAGES.items():- _set_child(- lines, image.values_key, "image", f"{name}:{version}@{digests[name]}"- )+ expected: dict[tuple[str, str], str] = {}+ if chart == "switch":+ expected[("global", "imageRegistry")] = registry+ for name, image in images.items():+ expected[(image.values_key, "image")] = f"{name}:{version}@{digests[name]}"+ else:+ for name, image in images.items():+ expected[(image.values_key, "repository")] = f"{registry}/{name}"+ expected[(image.values_key, "digest")] = digests[name]+ for (parent, child), value in expected.items():+ _set_child(lines, parent, child, f'"{value}"')pinned = "".join(lines)loaded = yaml.safe_load(pinned)- if loaded["global"]["imageRegistry"] != registry:- raise PinError("pinned values do not read back the registry")- for name, image in IMAGES.items():- if loaded[image.values_key]["image"] != f"{name}:{version}@{digests[name]}":- raise PinError(f"pinned values do not read back {image.values_key}.image")+ for (parent, child), value in expected.items():+ if loaded[parent][child] != value:+ raise PinError(f"pinned values do not read back {parent}.{child}")return pinned@@ -187,9 +251,10 @@ def _repository(ref: str) -> str:return ref[:colon] if colon > slash else ref-def verify(rendered: str, registry: str) -> list[str]:- """Return every first-party image reference, or raise if any is unpinned."""+def verify(chart: str, rendered: str, registry: str) -> list[str]:+ """Return the chart's image references, or raise if any is unpinned."""_check_registry(registry)+ images = images_of(chart)refs = sorted({ref for doc in yaml.safe_load_all(rendered) if doc for ref in _images(doc)})@@ -200,18 +265,18 @@ def verify(rendered: str, registry: str) -> list[str]:for ref in refs:repository = _repository(ref)name = repository.rsplit("/", 1)[-1]- if name in IMAGES:+ if name in images:ours.append(ref)seen.add(name)if repository != f"{registry}/{name}":problems.append(f"{ref}: not from {registry}")if not re.search(r"@sha256:[0-9a-f]{64}$", ref):problems.append(f"{ref}: not pinned by digest")- elif ref.startswith(prefix):+ elif ref.startswith(prefix) or name in IMAGES:problems.append(- f"{ref}: from {registry} but not a known first-party image; add it to IMAGES"+ f"{ref}: not an image of the {chart} chart; add it to IMAGES if it should be")- absent = sorted(set(IMAGES) - seen)+ absent = sorted(set(images) - seen)if absent:problems.append(f"missing from the rendered chart: {absent}")if problems:@@ -229,27 +294,38 @@ def record(version: str,channel: str,commit: str,- chart_digest: str,+ chart_digests: dict[str, str],) -> tuple[str, str]:_check_digests(digests)_check_registry(registry)- if not DIGEST.match(chart_digest):- raise PinError(f"chart: {chart_digest!r} is not a sha256 digest")+ if set(chart_digests) != set(CHARTS):+ raise PinError(+ f"chart digests must cover exactly {sorted(CHARTS)}; got {sorted(chart_digests)}"+ )+ for name, digest in chart_digests.items():+ if not DIGEST.match(digest):+ raise PinError(f"chart {name}: {digest!r} is not a sha256 digest")pins = {"version": version,"channel": channel,"commit": commit,- "chart": {- "ref": f"oci://{registry}/charts/switch",- "version": version,- "digest": chart_digest,+ "charts": {+ name: {+ "ref": f"oci://{registry}/charts/{name}",+ "version": version,+ "digest": chart_digests[name],+ }+ for name in CHARTS},"images": {name: {"ref": f"{registry}/{name}", "digest": digests[name]}for name in IMAGES},}- rows = [f"| chart | `oci://{registry}/charts/switch:{version}@{chart_digest}` |"]+ rows = [+ f"| chart {name} | `oci://{registry}/charts/{name}:{version}@{chart_digests[name]}` |"+ for name in CHARTS+ ]rows += [f"| {name} | `{registry}/{name}@{digests[name]}` |" for name in IMAGES]summary = "\n".join([@@ -264,6 +340,16 @@ def record(return json.dumps(pins, indent=2) + "\n", summary+def _parse_pairs(pairs: list[str]) -> dict[str, str]:+ parsed: dict[str, str] = {}+ for pair in pairs:+ name, sep, value = pair.partition("=")+ if not sep or name in parsed:+ raise PinError(f"{pair!r}: expected <chart>=sha256:…, each chart once")+ parsed[name] = value+ return parsed++def main(argv: list[str]) -> int:parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter@@ -271,9 +357,10 @@ def main(argv: list[str]) -> int:sub = parser.add_subparsers(dest="command", required=True)sub.add_parser("images", help="print the image build matrix as JSON")+ sub.add_parser("charts", help="print the published charts as JSON")- p_pin = sub.add_parser("pin", help="write image digests into values.yaml")- p_pin.add_argument("--values", required=True, type=Path)+ p_pin = sub.add_parser("pin", help="write image digests into a chart's values.yaml")+ p_pin.add_argument("--chart", required=True, choices=sorted(CHARTS))p_pin.add_argument("--registry", required=True)p_pin.add_argument("--version", required=True)source = p_pin.add_mutually_exclusive_group(required=True)@@ -287,6 +374,7 @@ def main(argv: list[str]) -> int:p_verify = sub.add_parser("verify", help="check rendered manifests (stdin) are digest-pinned")+ p_verify.add_argument("--chart", required=True, choices=sorted(CHARTS))p_verify.add_argument("--registry", required=True)p_record = sub.add_parser(@@ -297,7 +385,9 @@ def main(argv: list[str]) -> int:p_record.add_argument("--version", required=True)p_record.add_argument("--channel", required=True)p_record.add_argument("--commit", required=True)- p_record.add_argument("--chart-digest", required=True)+ p_record.add_argument(+ "--chart-digest", required=True, action="append", help="<chart>=sha256:…"+ )p_record.add_argument("--json", required=True, type=Path)p_record.add_argument("--summary", required=True, type=Path)@@ -307,23 +397,39 @@ def main(argv: list[str]) -> int:print(json.dumps([- {"image": name, "dockerfile": image.dockerfile}+ {+ "image": name,+ "dockerfile": image.dockerfile,+ "context": image.context,+ }for name, image in IMAGES.items()]))+ elif args.command == "charts":+ print(+ json.dumps(+ [+ {"chart": name, "path": chart.path, "renders": chart.renders}+ for name, chart in CHARTS.items()+ ]+ )+ )elif args.command == "pin":digests = (dict.fromkeys(IMAGES, PLACEHOLDER_DIGEST)if args.placeholder_digestselse read_digests(args.digests_dir))- args.values.write_text(- pin(args.values.read_text(), args.registry, args.version, digests)+ values = Path(CHARTS[args.chart].path) / "values.yaml"+ values.write_text(+ pin(+ args.chart, values.read_text(), args.registry, args.version, digests+ ))- print(f"pinned {args.values}")+ print(f"pinned {values}")elif args.command == "verify":- for ref in verify(sys.stdin.read(), args.registry):+ for ref in verify(args.chart, sys.stdin.read(), args.registry):print(ref)else:pins, summary = record(@@ -332,7 +438,7 @@ def main(argv: list[str]) -> int:args.version,args.channel,args.commit,- args.chart_digest,+ _parse_pairs(args.chart_digest),)args.json.write_text(pins)with args.summary.open("a") as out:diff --git a/scripts/prune_dev_packages.py b/scripts/prune_dev_packages.pyindex 97a7a17c4..a27cb050e 100644--- a/scripts/prune_dev_packages.py+++ b/scripts/prune_dev_packages.py@@ -32,14 +32,14 @@ from pathlib import Pathfrom urllib.parse import quote-def _first_party_images() -> list[str]:+def _published() -> tuple[list[str], list[str]]:path = Path(__file__).with_name("pin_chart_images.py")spec = importlib.util.spec_from_file_location("pin_chart_images", path)assert spec is not None and spec.loader is not Nonemodule = importlib.util.module_from_spec(spec)sys.modules[spec.name] = modulespec.loader.exec_module(module)- return list(module.IMAGES)+ return list(module.IMAGES), list(module.CHARTS)def _gh(*args: str) -> str:@@ -88,7 +88,10 @@ def main(argv: list[str]) -> int:parser.error("--keep must be at least 1")now = datetime.now(UTC)- packages = [f"dev/{name}" for name in _first_party_images()] + ["dev/charts/switch"]+ images, charts = _published()+ packages = [f"dev/{name}" for name in images] + [+ f"dev/charts/{name}" for name in charts+ ]for package in packages:try:versions = _versions(args.owner, package)diff --git a/scripts/refuse_published_chart.sh b/scripts/refuse_published_chart.shnew file mode 100755index 000000000..d0d693450--- /dev/null+++ b/scripts/refuse_published_chart.sh@@ -0,0 +1,25 @@+#!/usr/bin/env bash+# Fail unless a chart version is NOT yet in the registry.+#+# A version is published once: a re-run of the release rebuilds the images+# (builds are not bit-for-bit reproducible) and would push the chart again+# under the same version, pointing at new digests, so a version pin would no+# longer mean what it meant. Only helm's plain "not found" counts as+# unpublished; an auth or network error fails rather than reading as+# permission to push. Needs `helm registry login` done first.+#+# Usage: refuse_published_chart.sh oci://<registry>/<namespace>/charts/<name> <version>+set -euo pipefail++chart="${1:?chart reference required}"+version="${2:?version required}"++if out="$(helm show chart "$chart" --version "$version" 2>&1)"; then+ echo "::error::$chart:$version is already published. A version is published once; cut a new one."+ exit 1+fi+if ! grep -q ': not found$' <<<"$out"; then+ echo "::error::Could not tell whether $chart:$version exists: $out"+ exit 1+fi+echo "$chart:$version is not published yet"