Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stop container continuity reads at explicit access refusals
pulse-triage[bot]665b18acadaf
diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.ymlindex 8f336582bd..755fc12ac6 100644--- a/.github/workflows/security-scan.yml+++ b/.github/workflows/security-scan.yml@@ -214,7 +214,9 @@ jobs:- name: Verify stable container discovery aliasesid: aliases- if: ${{ always() && steps.release.outcome == 'success' && steps.frontier.outcome == 'success' && steps.activation.outcome == 'success' && github.event.schedule != '17 */6 * * *' }}+ # Do not follow a refused exact-image read with another registry path.+ # A failed exact check remains failed; aliases are recorded as skipped.+ if: ${{ always() && steps.release.outcome == 'success' && steps.frontier.outcome == 'success' && steps.activation.outcome == 'success' && steps.containers.outcome == 'success' && github.event.schedule != '17 */6 * * *' }}env:TAG: ${{ steps.release.outputs.tag }}EXPECTED_SERVER_DIGEST: ${{ steps.activation.outputs.server_image_digest }}diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex f56261ac3a..d763d07a1f 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -389,6 +389,14 @@ func TestSecurityScanRevalidatesLatestStableDelivery(t *testing.T) {}}}+ aliases := workflowStepBlock(t, workflowJobBlock(t, workflow, "release-continuity"), "Verify stable container discovery aliases")+ if !strings.Contains(aliases, `steps.containers.outcome == 'success'`) {+ t.Fatal("alias reads must not follow a failed or refused exact-container read")+ }+ helm := workflowStepBlock(t, workflowJobBlock(t, workflow, "release-continuity"), "Verify exact-version Helm identity")+ if strings.Contains(helm, "steps.containers.outcome") {+ t.Fatal("an unrelated Helm check must remain independent of container failures")+ }}func TestProPackagingBuildsFrontendEmbedWithoutTransferringBundle(t *testing.T) {diff --git a/scripts/release_control/verify_release_container_images_test.py b/scripts/release_control/verify_release_container_images_test.pyindex c3b8587a98..0f13619902 100644--- a/scripts/release_control/verify_release_container_images_test.py+++ b/scripts/release_control/verify_release_container_images_test.py@@ -28,13 +28,18 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):source_sha: str = SOURCE_SHA,gh_fail_first: int = 0,docker_fail_first: int = 0,+ docker_refusal: str = "",+ refusal_message: str = "insufficient_scope: authorization failed",attempts: str = "4",- ) -> tuple[subprocess.CompletedProcess[str], str]:+ ) -> tuple[subprocess.CompletedProcess[str], str, list[str]]:with tempfile.TemporaryDirectory() as temp:temp_path = Path(temp)bin_path = temp_path / "bin"bin_path.mkdir()gh_log = temp_path / "gh.log"+ docker_log = temp_path / "docker.log"+ scratch = temp_path / "scratch"+ scratch.mkdir()digest_file = temp_path / "digests"values = {"docker.io/rcourtman/pulse:v6.4.1": DIGEST,@@ -56,6 +61,13 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):textwrap.dedent("""\#!/bin/sh+ reference="$4"+ printf '%s\n' "$reference" >> "$DOCKER_LOG"+ if [ "$reference" = "$DOCKER_REFUSAL" ]; then+ printf '{"digest":"partial-refused-output'+ printf '%s\n' "$REFUSAL_MESSAGE" >&2+ exit 29+ fi# A transient registry fault prints a half answer and fails, as a dropped# connection does; the first DOCKER_FAIL_FIRST calls behave that way.n=$(cat "$DOCKER_COUNT" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$DOCKER_COUNT"@@ -64,7 +76,6 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2exit 1fi- reference="$4"digest=$(awk -v ref="$reference" '$1 == ref { print $2 }' "$DIGEST_FILE")[ -n "$digest" ] || exit 1printf '{"digest":"%s"}\n' "$digest"@@ -86,6 +97,10 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2exit 1fi+ if [ "$GH_EXIT" -eq 29 ]; then+ printf 'partial-refused-output'+ printf '%s\n' "$REFUSAL_MESSAGE" >&2+ fiexit "$GH_EXIT""""),@@ -105,6 +120,10 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):"GH_FAIL_FIRST": str(gh_fail_first),"DOCKER_FAIL_FIRST": str(docker_fail_first),"DOCKER_COUNT": str(temp_path / "docker.count"),+ "DOCKER_LOG": str(docker_log),+ "DOCKER_REFUSAL": docker_refusal,+ "REFUSAL_MESSAGE": refusal_message,+ "TMPDIR": str(scratch),"VERIFY_REGISTRY_ATTEMPTS": attempts,"VERIFY_REGISTRY_WAIT_SECONDS": "0",}@@ -117,16 +136,23 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):capture_output=True,check=False,)- return result, gh_log.read_text(encoding="utf-8") if gh_log.exists() else ""+ self.assertEqual(list(scratch.iterdir()), [], "registry stderr must be cleaned up")+ return (+ result,+ gh_log.read_text(encoding="utf-8") if gh_log.exists() else "",+ docker_log.read_text(encoding="utf-8").splitlines() if docker_log.exists() else [],+ )def test_emits_digest_proof_after_verifying_both_registries(self) -> None:- result, calls = self.run_verifier()+ result, calls, docker_calls = self.run_verifier()self.assertEqual(result.returncode, 0, result.stderr)self.assertEqual(result.stdout.splitlines(),[f"server_digest={DIGEST}", f"control_plane_digest={DIGEST}"],)+ self.assertEqual(len(docker_calls), 8)+ self.assertEqual(len(set(docker_calls)), 8)subjects = {f"oci://docker.io/rcourtman/pulse@{DIGEST}",f"oci://ghcr.io/rcourtman/pulse@{DIGEST}",@@ -159,7 +185,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):def test_rejects_a_moved_exact_version_tag_before_attestation(self) -> None:changed = "sha256:" + "c" * 64- result, calls = self.run_verifier(+ result, calls, _ = self.run_verifier(overrides={"ghcr.io/rcourtman/pulse:6.4.1": changed})@@ -169,7 +195,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):def test_rejects_an_unverifiable_attestation(self) -> None:# A real failure is identical on every attempt: it is retried, then it still fails.- result, calls = self.run_verifier(gh_exit=1)+ result, calls, _ = self.run_verifier(gh_exit=1)self.assertNotEqual(result.returncode, 0)self.assertEqual(len(calls.splitlines()), 4)@@ -177,7 +203,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):self.assertEqual(result.stdout, "")def test_retries_transient_attestation_reads_and_emits_only_the_proof(self) -> None:- result, calls = self.run_verifier(gh_fail_first=2)+ result, calls, _ = self.run_verifier(gh_fail_first=2)self.assertEqual(result.returncode, 0, result.stderr)self.assertEqual(@@ -189,7 +215,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):self.assertEqual(result.stderr.count("retrying in 0s"), 2)def test_a_failed_digest_read_cannot_leak_partial_output_into_the_proof(self) -> None:- result, _ = self.run_verifier(docker_fail_first=3)+ result, _, _ = self.run_verifier(docker_fail_first=3)self.assertEqual(result.returncode, 0, result.stderr)self.assertEqual(@@ -200,7 +226,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):self.assertEqual(result.stderr.count("registry read failed on attempt"), 3)def test_gives_up_after_the_bounded_number_of_digest_attempts(self) -> None:- result, calls = self.run_verifier(docker_fail_first=99, attempts="3")+ result, calls, _ = self.run_verifier(docker_fail_first=99, attempts="3")self.assertNotEqual(result.returncode, 0)self.assertIn("registry read failed after 3 attempts", result.stderr)@@ -210,24 +236,64 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):def test_rejects_unbounded_retry_settings(self) -> None:for attempts in ("0", "100", "x", "-1", "2.5"):with self.subTest(attempts=attempts):- result, calls = self.run_verifier(attempts=attempts)+ result, calls, _ = self.run_verifier(attempts=attempts)self.assertEqual(result.returncode, 2, result.stderr)self.assertEqual(calls, "")def test_rejects_an_unsafe_github_cli_before_registry_calls(self) -> None:- result, calls = self.run_verifier(gh_version="2.96.1")+ result, calls, _ = self.run_verifier(gh_version="2.96.1")self.assertNotEqual(result.returncode, 0)self.assertIn("too old for release attestation policy enforcement", result.stderr)self.assertEqual(calls, "")def test_rejects_invalid_release_identity_without_registry_calls(self) -> None:- result, calls = self.run_verifier(source_sha="main")+ result, calls, _ = self.run_verifier(source_sha="main")self.assertNotEqual(result.returncode, 0)self.assertIn("Invalid release source SHA", result.stderr)self.assertEqual(calls, "")+ def test_stops_at_first_refused_digest_without_another_registry_or_image(self) -> None:+ # The control-plane denial is the observed scheduled failure: success+ # for the server must not turn its refusal into a whole-image pass.+ for message in (+ "ERROR: pull access denied, repository does not exist or may require authorization",+ "insufficient_scope: authorization failed",+ "denied: requested access to the resource is denied",+ "unauthorized: authentication required",+ "FORBIDDEN",+ "unexpected status from HEAD request: 403 Forbidden",+ "unexpected status from GET request: 401",+ ):+ for image, expected_reads in (("pulse", 1), ("pulse-control-plane", 5)):+ with self.subTest(message=message, image=image):+ reference = f"docker.io/rcourtman/{image}:v6.4.1"+ result, calls, docker_calls = self.run_verifier(+ docker_refusal=reference, refusal_message=message+ )+ self.assertEqual(result.returncode, 29, result.stderr)+ self.assertEqual(len(docker_calls), expected_reads)+ self.assertEqual(docker_calls[-1], reference)+ self.assertIn(f"registry access refused for {reference}; not retrying", result.stderr)+ self.assertNotIn("retrying in", result.stderr)+ self.assertNotIn("partial-refused-output", result.stdout)+ self.assertNotIn("control_plane_digest=", result.stdout)+ self.assertEqual(len(calls.splitlines()), 0 if image == "pulse" else 2)++ def test_stops_at_first_refused_attestation_without_retry_or_next_subject(self) -> None:+ result, calls, docker_calls = self.run_verifier(gh_exit=29)++ self.assertEqual(result.returncode, 29, result.stderr)+ self.assertEqual(len(docker_calls), 4)+ self.assertEqual(len(calls.splitlines()), 1)+ self.assertIn(+ f"registry access refused for oci://docker.io/rcourtman/pulse@{DIGEST}; not retrying",+ result.stderr,+ )+ self.assertNotIn("retrying in", result.stderr)+ self.assertEqual(result.stdout, "")+if __name__ == "__main__":unittest.main()diff --git a/scripts/release_control/verify_stable_container_aliases_test.py b/scripts/release_control/verify_stable_container_aliases_test.pyindex 8fab1e4a3f..17b11e6d49 100644--- a/scripts/release_control/verify_stable_container_aliases_test.py+++ b/scripts/release_control/verify_stable_container_aliases_test.py@@ -23,6 +23,7 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):overrides: dict[str, str] | None = None,tag: str = "v6.4.1",owner: str = "rcourtman",+ refused_reference: str = "",) -> tuple[subprocess.CompletedProcess[str], list[str]]:with tempfile.TemporaryDirectory() as temp:temp_path = Path(temp)@@ -50,6 +51,11 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):#!/bin/shreference="$4"printf '%s\n' "$reference" >> "$CALL_LOG"+ if [ "$reference" = "$REFUSED_REFERENCE" ]; then+ printf '{"digest":"partial-refused-output'+ echo 'insufficient_scope: authorization failed' >&2+ exit 29+ fidigest=$(awk -v ref="$reference" '$1 == ref { print $2 }' "$DIGEST_FILE")[ -n "$digest" ] || exit 1printf '{"digest":"%s"}\n' "$digest"@@ -65,6 +71,7 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):"PATH": f"{bin_path}:{env['PATH']}","DIGEST_FILE": str(digest_file),"CALL_LOG": str(call_log),+ "REFUSED_REFERENCE": refused_reference,})result = subprocess.run(@@ -108,6 +115,17 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):self.assertIn("Invalid stable release tag", result.stderr)self.assertEqual(calls, [])+ def test_refused_alias_names_exact_target_and_stops_without_fallback(self) -> None:+ reference = "docker.io/rcourtman/pulse-control-plane:6.4"+ result, calls = self.run_verifier(refused_reference=reference)++ self.assertEqual(result.returncode, 29, result.stderr)+ self.assertEqual(len(calls), 7)+ self.assertEqual(calls[-1], reference)+ self.assertIn(f"Registry read failed for {reference}; not retrying", result.stderr)+ self.assertNotIn("partial-refused-output", result.stdout)+ self.assertEqual(result.stdout, "")+def test_rejects_invalid_digest_without_registry_calls(self) -> None:result = subprocess.run([str(SCRIPT), "v6.4.2", "latest", CONTROL_PLANE_DIGEST],diff --git a/scripts/verify-release-container-images.sh b/scripts/verify-release-container-images.shindex ab20d98e3a..a2d2e02c9b 100755--- a/scripts/verify-release-container-images.sh+++ b/scripts/verify-release-container-images.sh@@ -44,8 +44,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"# Registry and attestation reads are idempotent and cross the public internet, where a# Docker Hub or GHCR 5xx is routine: one 502 during a blob copy failed the first beta's-# release. Retry a bounded number of times. A moved tag or an invalid attestation fails-# identically on every attempt, so a real failure is delayed by the backoff, never masked.+# release. Retry a bounded number of times. A moved tag or an invalid attestation+# still fails closed. Explicit access refusals are not transient faults and must+# never trigger another attempt.# This script's stdout is the step output, so each attempt's stdout is captured and only# the successful attempt's is emitted; a half-printed failed attempt cannot leak into it.REGISTRY_ATTEMPTS="${VERIFY_REGISTRY_ATTEMPTS:-4}"@@ -55,33 +56,48 @@ if [[ ! "$REGISTRY_ATTEMPTS" =~ ^[1-9][0-9]?$ || ! "$REGISTRY_WAIT_SECONDS" =~ ^exit 2fi-with_registry_retry() {+with_registry_retry() (+ local reference="$1"+ shiftlocal attempt=1local wait_seconds="$REGISTRY_WAIT_SECONDS"- local output+ local output status error_file+ error_file="$(mktemp)" || return+ trap 'rm -- "$error_file"' EXITwhile true; do- if output="$("$@")"; then+ if output="$("$@" 2>"$error_file")"; then+ cat "$error_file" >&2printf '%s\n' "$output"return 0+ else+ status=$?+ fi+ cat "$error_file" >&2++ if LC_ALL=C grep -Eiq \+ 'pull access denied|requested access to the resource is denied|insufficient_scope|authorization failed|authentication required|unauthorized|forbidden|unexpected status[^[:cntrl:]]*(401|403)' \+ "$error_file"; then+ echo "registry access refused for ${reference}; not retrying." >&2+ return "$status"fiif [ "$attempt" -ge "$REGISTRY_ATTEMPTS" ]; then- echo "registry read failed after ${REGISTRY_ATTEMPTS} attempts: $1 $2 $3" >&2- return 1+ echo "registry read failed after ${REGISTRY_ATTEMPTS} attempts: ${reference}" >&2+ return "$status"fi- echo "registry read failed on attempt ${attempt}/${REGISTRY_ATTEMPTS}; retrying in ${wait_seconds}s: $1 $2 $3" >&2+ echo "registry read failed on attempt ${attempt}/${REGISTRY_ATTEMPTS}; retrying in ${wait_seconds}s: ${reference}" >&2sleep "$wait_seconds"attempt=$((attempt + 1))wait_seconds=$((wait_seconds * 2))done-}+)resolve_digest() {local reference="$1"local manifest- manifest="$(with_registry_retry docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"+ manifest="$(with_registry_retry "$reference" docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')" || return "$?"jq -er '.digest | select(type == "string" and test("^sha256:[0-9a-f]{64}$"))' <<<"$manifest"}@@ -109,7 +125,7 @@ verify_image() {fifor subject in "$docker_name" "$ghcr_name"; do- with_registry_retry gh attestation verify "oci://${subject}@${docker_tag_digest}" \+ with_registry_retry "oci://${subject}@${docker_tag_digest}" gh attestation verify "oci://${subject}@${docker_tag_digest}" \--repo "$REPOSITORY" \--bundle-from-oci \--signer-workflow "$SIGNER_WORKFLOW" \diff --git a/scripts/verify-stable-container-aliases.sh b/scripts/verify-stable-container-aliases.shindex 96ed4a53c8..b34e117d92 100755--- a/scripts/verify-stable-container-aliases.sh+++ b/scripts/verify-stable-container-aliases.sh@@ -48,7 +48,13 @@ ALIASES=("${MAJOR}.${MINOR}" "${MAJOR}" latest)resolve_digest() {local reference="$1"local manifest- manifest="$(docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"+ if manifest="$(docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"; then+ :+ else+ local status=$?+ echo "Registry read failed for ${reference}; not retrying." >&2+ return "$status"+ fijq -er '.digest | select(type == "string" and test("^sha256:[0-9a-f]{64}$"))' <<<"$manifest"}