Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Stop container continuity reads at explicit access refusals

pulse-triage[bot]665b18acadaf
diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml
index 8f336582bd..755fc12ac6 100644
--- a/.github/workflows/security-scan.yml
+++ b/.github/workflows/security-scan.yml
@@ -214,7 +214,9 @@ jobs:
- name: Verify stable container discovery aliases
id: aliases
- if: ${{ always() && steps.release.outcome == 'success' && steps.frontier.outcome == 'success' && steps.activation.outcome == 'success' && github.event.schedule != '17 */6 * * *' }}
+ # Do not follow a refused exact-image read with another registry path.
+ # A failed exact check remains failed; aliases are recorded as skipped.
+ if: ${{ always() && steps.release.outcome == 'success' && steps.frontier.outcome == 'success' && steps.activation.outcome == 'success' && steps.containers.outcome == 'success' && github.event.schedule != '17 */6 * * *' }}
env:
TAG: ${{ steps.release.outputs.tag }}
EXPECTED_SERVER_DIGEST: ${{ steps.activation.outputs.server_image_digest }}
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index f56261ac3a..d763d07a1f 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -389,6 +389,14 @@ func TestSecurityScanRevalidatesLatestStableDelivery(t *testing.T) {
}
}
}
+ aliases := workflowStepBlock(t, workflowJobBlock(t, workflow, "release-continuity"), "Verify stable container discovery aliases")
+ if !strings.Contains(aliases, `steps.containers.outcome == 'success'`) {
+ t.Fatal("alias reads must not follow a failed or refused exact-container read")
+ }
+ helm := workflowStepBlock(t, workflowJobBlock(t, workflow, "release-continuity"), "Verify exact-version Helm identity")
+ if strings.Contains(helm, "steps.containers.outcome") {
+ t.Fatal("an unrelated Helm check must remain independent of container failures")
+ }
}
func TestProPackagingBuildsFrontendEmbedWithoutTransferringBundle(t *testing.T) {
diff --git a/scripts/release_control/verify_release_container_images_test.py b/scripts/release_control/verify_release_container_images_test.py
index c3b8587a98..0f13619902 100644
--- a/scripts/release_control/verify_release_container_images_test.py
+++ b/scripts/release_control/verify_release_container_images_test.py
@@ -28,13 +28,18 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
source_sha: str = SOURCE_SHA,
gh_fail_first: int = 0,
docker_fail_first: int = 0,
+ docker_refusal: str = "",
+ refusal_message: str = "insufficient_scope: authorization failed",
attempts: str = "4",
- ) -> tuple[subprocess.CompletedProcess[str], str]:
+ ) -> tuple[subprocess.CompletedProcess[str], str, list[str]]:
with tempfile.TemporaryDirectory() as temp:
temp_path = Path(temp)
bin_path = temp_path / "bin"
bin_path.mkdir()
gh_log = temp_path / "gh.log"
+ docker_log = temp_path / "docker.log"
+ scratch = temp_path / "scratch"
+ scratch.mkdir()
digest_file = temp_path / "digests"
values = {
"docker.io/rcourtman/pulse:v6.4.1": DIGEST,
@@ -56,6 +61,13 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
textwrap.dedent(
"""\
#!/bin/sh
+ reference="$4"
+ printf '%s\n' "$reference" >> "$DOCKER_LOG"
+ if [ "$reference" = "$DOCKER_REFUSAL" ]; then
+ printf '{"digest":"partial-refused-output'
+ printf '%s\n' "$REFUSAL_MESSAGE" >&2
+ exit 29
+ fi
# A transient registry fault prints a half answer and fails, as a dropped
# connection does; the first DOCKER_FAIL_FIRST calls behave that way.
n=$(cat "$DOCKER_COUNT" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$DOCKER_COUNT"
@@ -64,7 +76,6 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2
exit 1
fi
- reference="$4"
digest=$(awk -v ref="$reference" '$1 == ref { print $2 }' "$DIGEST_FILE")
[ -n "$digest" ] || exit 1
printf '{"digest":"%s"}\n' "$digest"
@@ -86,6 +97,10 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
echo "ERROR: failed to copy: unexpected status 502 Bad Gateway" >&2
exit 1
fi
+ if [ "$GH_EXIT" -eq 29 ]; then
+ printf 'partial-refused-output'
+ printf '%s\n' "$REFUSAL_MESSAGE" >&2
+ fi
exit "$GH_EXIT"
"""
),
@@ -105,6 +120,10 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
"GH_FAIL_FIRST": str(gh_fail_first),
"DOCKER_FAIL_FIRST": str(docker_fail_first),
"DOCKER_COUNT": str(temp_path / "docker.count"),
+ "DOCKER_LOG": str(docker_log),
+ "DOCKER_REFUSAL": docker_refusal,
+ "REFUSAL_MESSAGE": refusal_message,
+ "TMPDIR": str(scratch),
"VERIFY_REGISTRY_ATTEMPTS": attempts,
"VERIFY_REGISTRY_WAIT_SECONDS": "0",
}
@@ -117,16 +136,23 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
capture_output=True,
check=False,
)
- return result, gh_log.read_text(encoding="utf-8") if gh_log.exists() else ""
+ self.assertEqual(list(scratch.iterdir()), [], "registry stderr must be cleaned up")
+ return (
+ result,
+ gh_log.read_text(encoding="utf-8") if gh_log.exists() else "",
+ docker_log.read_text(encoding="utf-8").splitlines() if docker_log.exists() else [],
+ )
def test_emits_digest_proof_after_verifying_both_registries(self) -> None:
- result, calls = self.run_verifier()
+ result, calls, docker_calls = self.run_verifier()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(
result.stdout.splitlines(),
[f"server_digest={DIGEST}", f"control_plane_digest={DIGEST}"],
)
+ self.assertEqual(len(docker_calls), 8)
+ self.assertEqual(len(set(docker_calls)), 8)
subjects = {
f"oci://docker.io/rcourtman/pulse@{DIGEST}",
f"oci://ghcr.io/rcourtman/pulse@{DIGEST}",
@@ -159,7 +185,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
def test_rejects_a_moved_exact_version_tag_before_attestation(self) -> None:
changed = "sha256:" + "c" * 64
- result, calls = self.run_verifier(
+ result, calls, _ = self.run_verifier(
overrides={"ghcr.io/rcourtman/pulse:6.4.1": changed}
)
@@ -169,7 +195,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
def test_rejects_an_unverifiable_attestation(self) -> None:
# A real failure is identical on every attempt: it is retried, then it still fails.
- result, calls = self.run_verifier(gh_exit=1)
+ result, calls, _ = self.run_verifier(gh_exit=1)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(len(calls.splitlines()), 4)
@@ -177,7 +203,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
self.assertEqual(result.stdout, "")
def test_retries_transient_attestation_reads_and_emits_only_the_proof(self) -> None:
- result, calls = self.run_verifier(gh_fail_first=2)
+ result, calls, _ = self.run_verifier(gh_fail_first=2)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(
@@ -189,7 +215,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
self.assertEqual(result.stderr.count("retrying in 0s"), 2)
def test_a_failed_digest_read_cannot_leak_partial_output_into_the_proof(self) -> None:
- result, _ = self.run_verifier(docker_fail_first=3)
+ result, _, _ = self.run_verifier(docker_fail_first=3)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(
@@ -200,7 +226,7 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
self.assertEqual(result.stderr.count("registry read failed on attempt"), 3)
def test_gives_up_after_the_bounded_number_of_digest_attempts(self) -> None:
- result, calls = self.run_verifier(docker_fail_first=99, attempts="3")
+ result, calls, _ = self.run_verifier(docker_fail_first=99, attempts="3")
self.assertNotEqual(result.returncode, 0)
self.assertIn("registry read failed after 3 attempts", result.stderr)
@@ -210,24 +236,64 @@ class VerifyReleaseContainerImagesTests(unittest.TestCase):
def test_rejects_unbounded_retry_settings(self) -> None:
for attempts in ("0", "100", "x", "-1", "2.5"):
with self.subTest(attempts=attempts):
- result, calls = self.run_verifier(attempts=attempts)
+ result, calls, _ = self.run_verifier(attempts=attempts)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertEqual(calls, "")
def test_rejects_an_unsafe_github_cli_before_registry_calls(self) -> None:
- result, calls = self.run_verifier(gh_version="2.96.1")
+ result, calls, _ = self.run_verifier(gh_version="2.96.1")
self.assertNotEqual(result.returncode, 0)
self.assertIn("too old for release attestation policy enforcement", result.stderr)
self.assertEqual(calls, "")
def test_rejects_invalid_release_identity_without_registry_calls(self) -> None:
- result, calls = self.run_verifier(source_sha="main")
+ result, calls, _ = self.run_verifier(source_sha="main")
self.assertNotEqual(result.returncode, 0)
self.assertIn("Invalid release source SHA", result.stderr)
self.assertEqual(calls, "")
+ def test_stops_at_first_refused_digest_without_another_registry_or_image(self) -> None:
+ # The control-plane denial is the observed scheduled failure: success
+ # for the server must not turn its refusal into a whole-image pass.
+ for message in (
+ "ERROR: pull access denied, repository does not exist or may require authorization",
+ "insufficient_scope: authorization failed",
+ "denied: requested access to the resource is denied",
+ "unauthorized: authentication required",
+ "FORBIDDEN",
+ "unexpected status from HEAD request: 403 Forbidden",
+ "unexpected status from GET request: 401",
+ ):
+ for image, expected_reads in (("pulse", 1), ("pulse-control-plane", 5)):
+ with self.subTest(message=message, image=image):
+ reference = f"docker.io/rcourtman/{image}:v6.4.1"
+ result, calls, docker_calls = self.run_verifier(
+ docker_refusal=reference, refusal_message=message
+ )
+ self.assertEqual(result.returncode, 29, result.stderr)
+ self.assertEqual(len(docker_calls), expected_reads)
+ self.assertEqual(docker_calls[-1], reference)
+ self.assertIn(f"registry access refused for {reference}; not retrying", result.stderr)
+ self.assertNotIn("retrying in", result.stderr)
+ self.assertNotIn("partial-refused-output", result.stdout)
+ self.assertNotIn("control_plane_digest=", result.stdout)
+ self.assertEqual(len(calls.splitlines()), 0 if image == "pulse" else 2)
+
+ def test_stops_at_first_refused_attestation_without_retry_or_next_subject(self) -> None:
+ result, calls, docker_calls = self.run_verifier(gh_exit=29)
+
+ self.assertEqual(result.returncode, 29, result.stderr)
+ self.assertEqual(len(docker_calls), 4)
+ self.assertEqual(len(calls.splitlines()), 1)
+ self.assertIn(
+ f"registry access refused for oci://docker.io/rcourtman/pulse@{DIGEST}; not retrying",
+ result.stderr,
+ )
+ self.assertNotIn("retrying in", result.stderr)
+ self.assertEqual(result.stdout, "")
+
if __name__ == "__main__":
unittest.main()
diff --git a/scripts/release_control/verify_stable_container_aliases_test.py b/scripts/release_control/verify_stable_container_aliases_test.py
index 8fab1e4a3f..17b11e6d49 100644
--- a/scripts/release_control/verify_stable_container_aliases_test.py
+++ b/scripts/release_control/verify_stable_container_aliases_test.py
@@ -23,6 +23,7 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):
overrides: dict[str, str] | None = None,
tag: str = "v6.4.1",
owner: str = "rcourtman",
+ refused_reference: str = "",
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
with tempfile.TemporaryDirectory() as temp:
temp_path = Path(temp)
@@ -50,6 +51,11 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):
#!/bin/sh
reference="$4"
printf '%s\n' "$reference" >> "$CALL_LOG"
+ if [ "$reference" = "$REFUSED_REFERENCE" ]; then
+ printf '{"digest":"partial-refused-output'
+ echo 'insufficient_scope: authorization failed' >&2
+ exit 29
+ fi
digest=$(awk -v ref="$reference" '$1 == ref { print $2 }' "$DIGEST_FILE")
[ -n "$digest" ] || exit 1
printf '{"digest":"%s"}\n' "$digest"
@@ -65,6 +71,7 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):
"PATH": f"{bin_path}:{env['PATH']}",
"DIGEST_FILE": str(digest_file),
"CALL_LOG": str(call_log),
+ "REFUSED_REFERENCE": refused_reference,
}
)
result = subprocess.run(
@@ -108,6 +115,17 @@ class VerifyStableContainerAliasesTests(unittest.TestCase):
self.assertIn("Invalid stable release tag", result.stderr)
self.assertEqual(calls, [])
+ def test_refused_alias_names_exact_target_and_stops_without_fallback(self) -> None:
+ reference = "docker.io/rcourtman/pulse-control-plane:6.4"
+ result, calls = self.run_verifier(refused_reference=reference)
+
+ self.assertEqual(result.returncode, 29, result.stderr)
+ self.assertEqual(len(calls), 7)
+ self.assertEqual(calls[-1], reference)
+ self.assertIn(f"Registry read failed for {reference}; not retrying", result.stderr)
+ self.assertNotIn("partial-refused-output", result.stdout)
+ self.assertEqual(result.stdout, "")
+
def test_rejects_invalid_digest_without_registry_calls(self) -> None:
result = subprocess.run(
[str(SCRIPT), "v6.4.2", "latest", CONTROL_PLANE_DIGEST],
diff --git a/scripts/verify-release-container-images.sh b/scripts/verify-release-container-images.sh
index ab20d98e3a..a2d2e02c9b 100755
--- a/scripts/verify-release-container-images.sh
+++ b/scripts/verify-release-container-images.sh
@@ -44,8 +44,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Registry and attestation reads are idempotent and cross the public internet, where a
# Docker Hub or GHCR 5xx is routine: one 502 during a blob copy failed the first beta's
-# release. Retry a bounded number of times. A moved tag or an invalid attestation fails
-# identically on every attempt, so a real failure is delayed by the backoff, never masked.
+# release. Retry a bounded number of times. A moved tag or an invalid attestation
+# still fails closed. Explicit access refusals are not transient faults and must
+# never trigger another attempt.
# This script's stdout is the step output, so each attempt's stdout is captured and only
# the successful attempt's is emitted; a half-printed failed attempt cannot leak into it.
REGISTRY_ATTEMPTS="${VERIFY_REGISTRY_ATTEMPTS:-4}"
@@ -55,33 +56,48 @@ if [[ ! "$REGISTRY_ATTEMPTS" =~ ^[1-9][0-9]?$ || ! "$REGISTRY_WAIT_SECONDS" =~ ^
exit 2
fi
-with_registry_retry() {
+with_registry_retry() (
+ local reference="$1"
+ shift
local attempt=1
local wait_seconds="$REGISTRY_WAIT_SECONDS"
- local output
+ local output status error_file
+ error_file="$(mktemp)" || return
+ trap 'rm -- "$error_file"' EXIT
while true; do
- if output="$("$@")"; then
+ if output="$("$@" 2>"$error_file")"; then
+ cat "$error_file" >&2
printf '%s\n' "$output"
return 0
+ else
+ status=$?
+ fi
+ cat "$error_file" >&2
+
+ if LC_ALL=C grep -Eiq \
+ 'pull access denied|requested access to the resource is denied|insufficient_scope|authorization failed|authentication required|unauthorized|forbidden|unexpected status[^[:cntrl:]]*(401|403)' \
+ "$error_file"; then
+ echo "registry access refused for ${reference}; not retrying." >&2
+ return "$status"
fi
if [ "$attempt" -ge "$REGISTRY_ATTEMPTS" ]; then
- echo "registry read failed after ${REGISTRY_ATTEMPTS} attempts: $1 $2 $3" >&2
- return 1
+ echo "registry read failed after ${REGISTRY_ATTEMPTS} attempts: ${reference}" >&2
+ return "$status"
fi
- echo "registry read failed on attempt ${attempt}/${REGISTRY_ATTEMPTS}; retrying in ${wait_seconds}s: $1 $2 $3" >&2
+ echo "registry read failed on attempt ${attempt}/${REGISTRY_ATTEMPTS}; retrying in ${wait_seconds}s: ${reference}" >&2
sleep "$wait_seconds"
attempt=$((attempt + 1))
wait_seconds=$((wait_seconds * 2))
done
-}
+)
resolve_digest() {
local reference="$1"
local manifest
- manifest="$(with_registry_retry docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"
+ manifest="$(with_registry_retry "$reference" docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')" || return "$?"
jq -er '.digest | select(type == "string" and test("^sha256:[0-9a-f]{64}$"))' <<<"$manifest"
}
@@ -109,7 +125,7 @@ verify_image() {
fi
for subject in "$docker_name" "$ghcr_name"; do
- with_registry_retry gh attestation verify "oci://${subject}@${docker_tag_digest}" \
+ with_registry_retry "oci://${subject}@${docker_tag_digest}" gh attestation verify "oci://${subject}@${docker_tag_digest}" \
--repo "$REPOSITORY" \
--bundle-from-oci \
--signer-workflow "$SIGNER_WORKFLOW" \
diff --git a/scripts/verify-stable-container-aliases.sh b/scripts/verify-stable-container-aliases.sh
index 96ed4a53c8..b34e117d92 100755
--- a/scripts/verify-stable-container-aliases.sh
+++ b/scripts/verify-stable-container-aliases.sh
@@ -48,7 +48,13 @@ ALIASES=("${MAJOR}.${MINOR}" "${MAJOR}" latest)
resolve_digest() {
local reference="$1"
local manifest
- manifest="$(docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"
+ if manifest="$(docker buildx imagetools inspect "$reference" --format '{{json .Manifest}}')"; then
+ :
+ else
+ local status=$?
+ echo "Registry read failed for ${reference}; not retrying." >&2
+ return "$status"
+ fi
jq -er '.digest | select(type == "string" and test("^sha256:[0-9a-f]{64}$"))' <<<"$manifest"
}