Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Retry registry reads when verifying release container images
rcourtmanb09ba017eb57
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex aa9c536c47..4d29c00034 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -5010,6 +5010,15 @@ beta). That read is retried up to five times with backoff and accepts only acomplete listing of valid pages, so a failed or truncated read can never bemistaken for "no existing release" and stage a duplicate draft. Exhausting theattempts fails the job with an explicit error.+The release container image verification reads registry digests with `docker buildx+imagetools inspect` and checks provenance with `gh attestation verify`, both+against Docker Hub and GHCR. Those reads are idempotent and routinely meet a 5xx+from the public registries (a Docker Hub 502 during a blob copy failed the first+beta's release), so each is retried up to four times with doubling backoff from+10 seconds. Only the successful attempt's output reaches the step output, so a+half-printed failed attempt cannot corrupt the verified digests. A moved tag or an+invalid attestation fails identically on every attempt and is delayed by the+backoff, never masked.As of 2026-10-11, the governed release floor is Go `1.26.9`. It supersedes`1.26.8`, whose standard library is reachable through twelve vulnerable Pulsecall paths reported by `govulncheck` after the 2026-10-08 advisories, includingdiff --git a/scripts/release_control/verify_release_container_images_test.py b/scripts/release_control/verify_release_container_images_test.pyindex bf11a1694e..c3b8587a98 100644--- a/scripts/release_control/verify_release_container_images_test.py+++ b/scripts/release_control/verify_release_container_images_test.py@@ -26,6 +26,9 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):gh_version: str = "2.97.0",tag: str = "v6.4.1",source_sha: str = SOURCE_SHA,+ gh_fail_first: int = 0,+ docker_fail_first: int = 0,+ attempts: str = "4",) -> tuple[subprocess.CompletedProcess[str], str]:with tempfile.TemporaryDirectory() as temp:temp_path = Path(temp)@@ -53,6 +56,14 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):textwrap.dedent("""\#!/bin/sh+ # A transient registry fault prints a half answer and fails, as a dropped+ # connection does; the first DOCKER_FAIL_FIRST calls behave that way.+ n=$(cat "$DOCKER_COUNT" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$DOCKER_COUNT"+ if [ "$n" -le "$DOCKER_FAIL_FIRST" ]; then+ printf '{"digest":"sha256:bbbb'+ echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2+ exit 1+ fireference="$4"digest=$(awk -v ref="$reference" '$1 == ref { print $2 }' "$DIGEST_FILE")[ -n "$digest" ] || exit 1@@ -70,6 +81,11 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):exit 0fiprintf '%s\n' "$*" >> "$GH_LOG"+ n=$(wc -l < "$GH_LOG")+ if [ "$n" -le "$GH_FAIL_FIRST" ]; then+ echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2+ exit 1+ fiexit "$GH_EXIT""""),@@ -86,6 +102,11 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):"GH_LOG": str(gh_log),"GH_EXIT": str(gh_exit),"GH_VERSION": gh_version,+ "GH_FAIL_FIRST": str(gh_fail_first),+ "DOCKER_FAIL_FIRST": str(docker_fail_first),+ "DOCKER_COUNT": str(temp_path / "docker.count"),+ "VERIFY_REGISTRY_ATTEMPTS": attempts,+ "VERIFY_REGISTRY_WAIT_SECONDS": "0",})result = subprocess.run(@@ -147,10 +168,51 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):self.assertEqual(calls, "")def test_rejects_an_unverifiable_attestation(self) -> None:+ # A real failure is identical on every attempt: it is retried, then it still fails.result, calls = self.run_verifier(gh_exit=1)self.assertNotEqual(result.returncode, 0)- self.assertEqual(len(calls.splitlines()), 1)+ self.assertEqual(len(calls.splitlines()), 4)+ self.assertIn("registry read failed after 4 attempts", result.stderr)+ self.assertEqual(result.stdout, "")++ def test_retries_transient_attestation_reads_and_emits_only_the_proof(self) -> None:+ result, calls = self.run_verifier(gh_fail_first=2)++ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(+ result.stdout.splitlines(),+ [f"server_digest={DIGEST}", f"control_plane_digest={DIGEST}"],+ )+ # Two failed reads, then every one of the four subjects verified.+ self.assertEqual(len(calls.splitlines()), 6)+ self.assertEqual(result.stderr.count("retrying in 0s"), 2)++ def test_a_failed_digest_read_cannot_leak_partial_output_into_the_proof(self) -> None:+ result, _ = self.run_verifier(docker_fail_first=3)++ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(+ result.stdout.splitlines(),+ [f"server_digest={DIGEST}", f"control_plane_digest={DIGEST}"],+ )+ self.assertNotIn("bbbb\n", result.stdout.replace(DIGEST, ""))+ self.assertEqual(result.stderr.count("registry read failed on attempt"), 3)++ def test_gives_up_after_the_bounded_number_of_digest_attempts(self) -> None:+ result, calls = self.run_verifier(docker_fail_first=99, attempts="3")++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("registry read failed after 3 attempts", result.stderr)+ self.assertEqual(result.stdout, "")+ self.assertEqual(calls, "")++ def test_rejects_unbounded_retry_settings(self) -> None:+ for attempts in ("0", "100", "x", "-1", "2.5"):+ with self.subTest(attempts=attempts):+ result, calls = self.run_verifier(attempts=attempts)+ self.assertEqual(result.returncode, 2, result.stderr)+ self.assertEqual(calls, "")def test_rejects_an_unsafe_github_cli_before_registry_calls(self) -> None:result, calls = self.run_verifier(gh_version="2.96.1")diff --git a/scripts/verify-release-container-images.sh b/scripts/verify-release-container-images.shindex 06d5be03a5..ab20d98e3a 100755--- a/scripts/verify-release-container-images.sh+++ b/scripts/verify-release-container-images.sh@@ -42,10 +42,46 @@ doneSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)""${SCRIPT_DIR}/require-safe-gh-attestation.sh"+# Registry and attestation reads are idempotent and cross the public internet, where a+# Docker Hub or GHCR 5xx is routine: one 502 during a blob copy failed the first beta's+# release. Retry a bounded number of times. A moved tag or an invalid attestation fails+# identically on every attempt, so a real failure is delayed by the backoff, never masked.+# This script's stdout is the step output, so each attempt's stdout is captured and only+# the successful attempt's is emitted; a half-printed failed attempt cannot leak into it.+REGISTRY_ATTEMPTS="${VERIFY_REGISTRY_ATTEMPTS:-4}"+REGISTRY_WAIT_SECONDS="${VERIFY_REGISTRY_WAIT_SECONDS:-10}"+if [[ ! "$REGISTRY_ATTEMPTS" =~ ^[1-9][0-9]?$ || ! "$REGISTRY_WAIT_SECONDS" =~ ^[0-9]{1,3}$ ]]; then+ echo "VERIFY_REGISTRY_ATTEMPTS and VERIFY_REGISTRY_WAIT_SECONDS must be small whole numbers." >&2+ exit 2+fi++with_registry_retry() {+ local attempt=1+ local wait_seconds="$REGISTRY_WAIT_SECONDS"+ local output++ while true; do+ if output="$("$@")"; then+ printf '%s\n' "$output"+ return 0+ fi++ if [ "$attempt" -ge "$REGISTRY_ATTEMPTS" ]; then+ echo "registry read failed after ${REGISTRY_ATTEMPTS} attempts: $1 $2 $3" >&2+ return 1+ fi++ echo "registry read failed on attempt ${attempt}/${REGISTRY_ATTEMPTS}; retrying in ${wait_seconds}s: $1 $2 $3" >&2+ sleep "$wait_seconds"+ attempt=$((attempt + 1))+ wait_seconds=$((wait_seconds * 2))+ done+}+resolve_digest() {local reference="$1"local manifest- manifest="$(docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"+ manifest="$(with_registry_retry docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"jq -er '.digest | select(type == "string" and test("^sha256:[0-9a-f]{64}$"))' <<<"$manifest"}@@ -73,7 +109,7 @@ verify_image() {fifor subject in "$docker_name" "$ghcr_name"; do- gh attestation verify "oci://${subject}@${docker_tag_digest}" \+ with_registry_retry gh attestation verify "oci://${subject}@${docker_tag_digest}" \--repo "$REPOSITORY" \--bundle-from-oci \--signer-workflow "$SIGNER_WORKFLOW" \