Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Retry registry reads when verifying release container images

rcourtmanb09ba017eb57
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index aa9c536c47..4d29c00034 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -5010,6 +5010,15 @@ beta). That read is retried up to five times with backoff and accepts only a
complete listing of valid pages, so a failed or truncated read can never be
mistaken for "no existing release" and stage a duplicate draft. Exhausting the
attempts fails the job with an explicit error.
+The release container image verification reads registry digests with `docker buildx
+imagetools inspect` and checks provenance with `gh attestation verify`, both
+against Docker Hub and GHCR. Those reads are idempotent and routinely meet a 5xx
+from the public registries (a Docker Hub 502 during a blob copy failed the first
+beta's release), so each is retried up to four times with doubling backoff from
+10 seconds. Only the successful attempt's output reaches the step output, so a
+half-printed failed attempt cannot corrupt the verified digests. A moved tag or an
+invalid attestation fails identically on every attempt and is delayed by the
+backoff, never masked.
As of 2026-10-11, the governed release floor is Go `1.26.9`. It supersedes
`1.26.8`, whose standard library is reachable through twelve vulnerable Pulse
call paths reported by `govulncheck` after the 2026-10-08 advisories, including
diff --git a/scripts/release_control/verify_release_container_images_test.py b/scripts/release_control/verify_release_container_images_test.py
index bf11a1694e..c3b8587a98 100644
--- a/scripts/release_control/verify_release_container_images_test.py
+++ b/scripts/release_control/verify_release_container_images_test.py
@@ -26,6 +26,9 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
gh_version: str = "2.97.0",
tag: str = "v6.4.1",
source_sha: str = SOURCE_SHA,
+ gh_fail_first: int = 0,
+ docker_fail_first: int = 0,
+ attempts: str = "4",
) -> tuple[subprocess.CompletedProcess[str], str]:
with tempfile.TemporaryDirectory() as temp:
temp_path = Path(temp)
@@ -53,6 +56,14 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
textwrap.dedent(
"""\
#!/bin/sh
+ # A transient registry fault prints a half answer and fails, as a dropped
+ # connection does; the first DOCKER_FAIL_FIRST calls behave that way.
+ n=$(cat "$DOCKER_COUNT" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$DOCKER_COUNT"
+ if [ "$n" -le "$DOCKER_FAIL_FIRST" ]; then
+ printf '{"digest":"sha256:bbbb'
+ echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2
+ exit 1
+ fi
reference="$4"
digest=$(awk -v ref="$reference" '$1 == ref { print $2 }' "$DIGEST_FILE")
[ -n "$digest" ] || exit 1
@@ -70,6 +81,11 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
exit 0
fi
printf '%s\n' "$*" >> "$GH_LOG"
+ n=$(wc -l < "$GH_LOG")
+ if [ "$n" -le "$GH_FAIL_FIRST" ]; then
+ echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2
+ exit 1
+ fi
exit "$GH_EXIT"
"""
),
@@ -86,6 +102,11 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
"GH_LOG": str(gh_log),
"GH_EXIT": str(gh_exit),
"GH_VERSION": gh_version,
+ "GH_FAIL_FIRST": str(gh_fail_first),
+ "DOCKER_FAIL_FIRST": str(docker_fail_first),
+ "DOCKER_COUNT": str(temp_path / "docker.count"),
+ "VERIFY_REGISTRY_ATTEMPTS": attempts,
+ "VERIFY_REGISTRY_WAIT_SECONDS": "0",
}
)
result = subprocess.run(
@@ -147,10 +168,51 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
self.assertEqual(calls, "")
def test_rejects_an_unverifiable_attestation(self) -> None:
+ # A real failure is identical on every attempt: it is retried, then it still fails.
result, calls = self.run_verifier(gh_exit=1)
self.assertNotEqual(result.returncode, 0)
- self.assertEqual(len(calls.splitlines()), 1)
+ self.assertEqual(len(calls.splitlines()), 4)
+ self.assertIn("registry read failed after 4 attempts", result.stderr)
+ self.assertEqual(result.stdout, "")
+
+ def test_retries_transient_attestation_reads_and_emits_only_the_proof(self) -> None:
+ result, calls = self.run_verifier(gh_fail_first=2)
+
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(
+ result.stdout.splitlines(),
+ [f"server_digest={DIGEST}", f"control_plane_digest={DIGEST}"],
+ )
+ # Two failed reads, then every one of the four subjects verified.
+ self.assertEqual(len(calls.splitlines()), 6)
+ self.assertEqual(result.stderr.count("retrying in 0s"), 2)
+
+ def test_a_failed_digest_read_cannot_leak_partial_output_into_the_proof(self) -> None:
+ result, _ = self.run_verifier(docker_fail_first=3)
+
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(
+ result.stdout.splitlines(),
+ [f"server_digest={DIGEST}", f"control_plane_digest={DIGEST}"],
+ )
+ self.assertNotIn("bbbb\n", result.stdout.replace(DIGEST, ""))
+ self.assertEqual(result.stderr.count("registry read failed on attempt"), 3)
+
+ def test_gives_up_after_the_bounded_number_of_digest_attempts(self) -> None:
+ result, calls = self.run_verifier(docker_fail_first=99, attempts="3")
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("registry read failed after 3 attempts", result.stderr)
+ self.assertEqual(result.stdout, "")
+ self.assertEqual(calls, "")
+
+ def test_rejects_unbounded_retry_settings(self) -> None:
+ for attempts in ("0", "100", "x", "-1", "2.5"):
+ with self.subTest(attempts=attempts):
+ result, calls = self.run_verifier(attempts=attempts)
+ self.assertEqual(result.returncode, 2, result.stderr)
+ self.assertEqual(calls, "")
def test_rejects_an_unsafe_github_cli_before_registry_calls(self) -> None:
result, calls = self.run_verifier(gh_version="2.96.1")
diff --git a/scripts/verify-release-container-images.sh b/scripts/verify-release-container-images.sh
index 06d5be03a5..ab20d98e3a 100755
--- a/scripts/verify-release-container-images.sh
+++ b/scripts/verify-release-container-images.sh
@@ -42,10 +42,46 @@ done
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
"${SCRIPT_DIR}/require-safe-gh-attestation.sh"
+# Registry and attestation reads are idempotent and cross the public internet, where a
+# Docker Hub or GHCR 5xx is routine: one 502 during a blob copy failed the first beta's
+# release. Retry a bounded number of times. A moved tag or an invalid attestation fails
+# identically on every attempt, so a real failure is delayed by the backoff, never masked.
+# This script's stdout is the step output, so each attempt's stdout is captured and only
+# the successful attempt's is emitted; a half-printed failed attempt cannot leak into it.
+REGISTRY_ATTEMPTS="${VERIFY_REGISTRY_ATTEMPTS:-4}"
+REGISTRY_WAIT_SECONDS="${VERIFY_REGISTRY_WAIT_SECONDS:-10}"
+if [[ ! "$REGISTRY_ATTEMPTS" =~ ^[1-9][0-9]?$ || ! "$REGISTRY_WAIT_SECONDS" =~ ^[0-9]{1,3}$ ]]; then
+ echo "VERIFY_REGISTRY_ATTEMPTS and VERIFY_REGISTRY_WAIT_SECONDS must be small whole numbers." >&2
+ exit 2
+fi
+
+with_registry_retry() {
+ local attempt=1
+ local wait_seconds="$REGISTRY_WAIT_SECONDS"
+ local output
+
+ while true; do
+ if output="$("$@")"; then
+ printf '%s\n' "$output"
+ return 0
+ fi
+
+ if [ "$attempt" -ge "$REGISTRY_ATTEMPTS" ]; then
+ echo "registry read failed after ${REGISTRY_ATTEMPTS} attempts: $1 $2 $3" >&2
+ return 1
+ fi
+
+ echo "registry read failed on attempt ${attempt}/${REGISTRY_ATTEMPTS}; retrying in ${wait_seconds}s: $1 $2 $3" >&2
+ sleep "$wait_seconds"
+ attempt=$((attempt + 1))
+ wait_seconds=$((wait_seconds * 2))
+ done
+}
+
resolve_digest() {
local reference="$1"
local manifest
- manifest="$(docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"
+ manifest="$(with_registry_retry docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"
jq -er '.digest | select(type == "string" and test("^sha256:[0-9a-f]{64}$"))' <<<"$manifest"
}
@@ -73,7 +109,7 @@ verify_image() {
fi
for subject in "$docker_name" "$ghcr_name"; do
- gh attestation verify "oci://${subject}@${docker_tag_digest}" \
+ with_registry_retry gh attestation verify "oci://${subject}@${docker_tag_digest}" \
--repo "$REPOSITORY" \
--bundle-from-oci \
--signer-workflow "$SIGNER_WORKFLOW" \