Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Fail closed on missing backend CI admission

pulse-triage[bot]37ac098542c9
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index c050f54f6c..08b20d0140 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -471,17 +471,23 @@ jobs:
steps:
- name: Require compiled backend tests
env:
+ CODE_CHANGED: ${{ needs.changes.outputs.code }}
COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}
COMPILE_RESULT: ${{ needs.test-compile.result }}
run: |
- if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then
- echo "::error::Backend test compilation did not pass (result: ${COMPILE_RESULT:-unknown}); no runtime tests started."
- exit 1
- fi
+ # Only explicit, consistent classifier/compiler results admit tests
+ # or a deliberate skip. Missing outputs must not look like false.
+ case "$CODE_CHANGED:$COMPILE_REQUIRED:$COMPILE_RESULT" in
+ true:true:success|true:false:skipped|false:false:skipped) ;;
+ *)
+ echo "::error::Backend CI admission is incomplete or inconsistent. Inspect Detect changed areas and Compile backend race tests; no runtime tests started."
+ exit 1
+ ;;
+ esac
- - name: Report documentation-only change
- if: needs.changes.outputs.code != 'true'
- run: echo "No backend code changed. Tests are not required for this shard."
+ - name: Report inapplicable backend runtime
+ if: needs.changes.outputs.code == 'false'
+ run: echo "Application runtime checks are not applicable to this classified change; no backend tests ran in this shard."
- name: Checkout repository
if: needs.changes.outputs.code == 'true'
@@ -578,17 +584,23 @@ jobs:
steps:
- name: Require compiled backend tests
env:
+ CODE_CHANGED: ${{ needs.changes.outputs.code }}
COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}
COMPILE_RESULT: ${{ needs.test-compile.result }}
run: |
- if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then
- echo "::error::Backend test compilation did not pass (result: ${COMPILE_RESULT:-unknown}); no runtime tests started."
- exit 1
- fi
+ # Only explicit, consistent classifier/compiler results admit tests
+ # or a deliberate skip. Missing outputs must not look like false.
+ case "$CODE_CHANGED:$COMPILE_REQUIRED:$COMPILE_RESULT" in
+ true:true:success|true:false:skipped|false:false:skipped) ;;
+ *)
+ echo "::error::Backend CI admission is incomplete or inconsistent. Inspect Detect changed areas and Compile backend race tests; no runtime tests started."
+ exit 1
+ ;;
+ esac
- - name: Report documentation-only change
- if: needs.changes.outputs.code != 'true'
- run: echo "No backend code changed. Tests are not required for this shard."
+ - name: Report inapplicable backend runtime
+ if: needs.changes.outputs.code == 'false'
+ run: echo "Application runtime checks are not applicable to this classified change; no backend tests ran in this shard."
- name: Checkout repository
if: needs.changes.outputs.code == 'true'
@@ -700,20 +712,30 @@ jobs:
# internal/api shard succeeded (a skipped or cancelled shard fails it too).
backend-api-verdict:
name: Backend tests (api)
- needs: backend-api
+ needs: [changes, backend-api]
if: always()
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- name: Require every internal/api shard
env:
+ CHANGE_RESULT: ${{ needs.changes.result }}
+ CODE_CHANGED: ${{ needs.changes.outputs.code }}
API_SHARDS_RESULT: ${{ needs.backend-api.result }}
run: |
+ if [ "$CHANGE_RESULT" != success ]; then
+ echo "::error::Changed-area classification did not pass; backend runtime cannot be verified."
+ exit 1
+ fi
if [ "$API_SHARDS_RESULT" != success ]; then
echo "::error::internal/api shards did not all pass (result: ${API_SHARDS_RESULT})"
exit 1
fi
- echo "Every internal/api shard passed."
+ case "$CODE_CHANGED" in
+ true) echo "Every internal/api runtime shard passed." ;;
+ false) echo "Application runtime checks are not applicable to this classified change; no internal/api tests ran." ;;
+ *) echo "::error::Missing or invalid changed-area classification; backend runtime cannot be verified."; exit 1 ;;
+ esac
script-smoke:
name: Script smoke tests (shard ${{ matrix.shard }}/2)
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 59d5f766b0..34ec6da634 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -8736,18 +8736,30 @@ full compilation. Frontend-only changes do not pay for this Go-only preflight
checks still run. `scripts/go-test-compile-required.sh` consumes the complete
file list without an early-close/SIGPIPE path.
-A failed, cancelled, skipped or unknown compile result when compilation is required must
-fail the unchanged required backend check names, not let dependency skipping
-report success. Documentation-only changes still expand those named checks and
-accept the deliberately skipped compiler, as do changes outside Go compilation. Frontend and independent secret-scan
-work do not wait for compilation; their existing audit rules remain unchanged.
+Both backend matrices admit only three explicit classifier/compiler tuples:
+code=true/compile=true/success, code=true/compile=false/skipped, or
+code=false/compile=false/skipped. Missing, invalid or inconsistent outputs and
+failed, cancelled, skipped or unknown required compilation fail the unchanged
+required backend check names before checkout/runtime work. Missing output must
+not be interpreted as an intentional false flag. Documentation-, receipt- and
+triage-only changes still expand those named checks and accept the deliberately
+skipped compiler. Frontend and independent secret-scan work do not wait for
+compilation; their existing audit rules remain unchanged.
+
+The API aggregate also requires successful classification and every expanded
+shard. It reports runtime success only for code=true; code=false explicitly
+reports that no API tests ran because runtime checks are inapplicable. An
+unknown classifier or skipped/failed/cancelled shard never reports success.
`scripts/tests/test_go_test_compilation.py` exercises the actual compiler on
inert multi-package fixtures, rejects a test-only removed API and a normal-source
compile defect, and proves init/TestMain/test/example code is never executed.
-It executes both matrix prerequisite scripts against successful and adverse
-results and the documentation-only skip. `TestCITestCompilationPreservesChecks`
-retains the required names, full race execution and separate dependency audit.
+It executes both actual matrix prerequisite scripts and the API aggregate over
+successful, missing, invalid, inconsistent, failed, cancelled and deliberately
+skipped results, including the distinct no-tests message.
+`TestCITestCompilationPreservesChecks` executes these admission/verdict controls
+as well as retaining the required names, full race execution and separate
+dependency audit.
These proofs establish CI source admission, not hosted queue-time relief.
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index 2e382c931a..f1acf951bd 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -4273,8 +4273,10 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {
verdict := workflowJobBlock(t, workflow, "backend-api-verdict")
for _, required := range []string{
"name: Backend tests (api)\n",
- "needs: backend-api",
+ "needs: [changes, backend-api]",
"if: always()",
+ "CHANGE_RESULT: ${{ needs.changes.result }}",
+ "CODE_CHANGED: ${{ needs.changes.outputs.code }}",
"API_SHARDS_RESULT: ${{ needs.backend-api.result }}",
`if [ "$API_SHARDS_RESULT" != success ]; then`,
} {
@@ -5646,6 +5648,14 @@ func TestCITestCompilationPreservesChecks(t *testing.T) {
if strings.Count(workflow, "name: Require compiled backend tests") != 2 {
t.Fatal("both required backend matrices must report compilation failures")
}
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_go_test_compilation.py"), "-v",
+ "GoTestCompilationTest.test_required_checks_admit_only_explicit_consistent_results",
+ "GoTestCompilationTest.test_actual_api_verdict_rejects_unknown_results_and_distinguishes_no_tests")
+ if out, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("required backend admission and truthful verdict controls: %v\n%s", err, out)
+ }
for _, path := range []string{
repoFile("scripts", "compile-go-tests.sh"),
repoFile("scripts", "tests", "test_go_test_compilation.py"),
diff --git a/scripts/tests/test_go_test_compilation.py b/scripts/tests/test_go_test_compilation.py
index b8af39061e..853e8222ec 100644
--- a/scripts/tests/test_go_test_compilation.py
+++ b/scripts/tests/test_go_test_compilation.py
@@ -130,25 +130,37 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle
# Frontend work does not wait for the Go-only gate.
self.assertIn("needs: changes", job(source, "frontend"))
- def test_required_checks_fail_on_bad_compile_and_pass_docs_only_skip(self):
+ def test_required_checks_admit_only_explicit_consistent_results(self):
source = WORKFLOW.read_text()
for name in ("backend", "backend-api"):
section = job(source, name)
self.assertIn("needs: [changes, test-compile]", section)
self.assertIn("if: ${{ !cancelled() && needs.changes.result == 'success' }}", section)
+ self.assertIn("CODE_CHANGED: ${{ needs.changes.outputs.code }}", section)
+ self.assertIn("COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}", section)
self.assertIn("COMPILE_RESULT: ${{ needs.test-compile.result }}", section)
self.assertLess(section.index("Require compiled backend tests"), section.index("Checkout repository"))
script = prerequisite_script(section)
- for code in ("true", "false"):
- for result in ("success", "failure", "cancelled", "skipped", ""):
- with self.subTest(job=name, code=code, result=result):
- observed = subprocess.run(
- ["bash", "-euo", "pipefail", "-c", script],
- env={**os.environ, "COMPILE_REQUIRED": code, "COMPILE_RESULT": result},
- capture_output=True, text=True,
- )
- expected = 0 if code == "false" or result == "success" else 1
- self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)
+ admitted = {("true", "true", "success"), ("true", "false", "skipped"),
+ ("false", "false", "skipped")}
+ for code in ("true", "false", "", "unknown", "true\nfalse"):
+ for required in ("true", "false", "", "unknown"):
+ for result in ("success", "failure", "cancelled", "skipped", ""):
+ with self.subTest(job=name, code=code, required=required, result=result):
+ observed = subprocess.run(
+ ["bash", "-euo", "pipefail", "-c", script],
+ env={**os.environ, "CODE_CHANGED": code,
+ "COMPILE_REQUIRED": required, "COMPILE_RESULT": result},
+ capture_output=True, text=True, timeout=5,
+ )
+ expected = 0 if (code, required, result) in admitted else 1
+ self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)
+ if expected:
+ self.assertIn("no runtime tests started", observed.stdout)
+ self.assertIn("Detect changed areas", observed.stdout)
+ self.assertIn("- name: Report inapplicable backend runtime\n"
+ " if: needs.changes.outputs.code == 'false'", section)
+ self.assertIn("no backend tests ran in this shard", section)
# Compile-only success cannot replace these full race/vet runs.
self.assertIn("go test -race -timeout 50m", section)
self.assertIn("shard: [rest-0, rest-1]", job(source, "backend"))
@@ -156,6 +168,36 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle
self.assertIn("name: Backend tests (api)", job(source, "backend-api-verdict"))
self.assertIn("if: always()", job(source, "backend-api-verdict"))
+ def test_actual_api_verdict_rejects_unknown_results_and_distinguishes_no_tests(self):
+ section = job(WORKFLOW.read_text(), "backend-api-verdict")
+ self.assertIn("needs: [changes, backend-api]", section)
+ for binding in ("CHANGE_RESULT: ${{ needs.changes.result }}",
+ "CODE_CHANGED: ${{ needs.changes.outputs.code }}",
+ "API_SHARDS_RESULT: ${{ needs.backend-api.result }}"):
+ self.assertIn(binding, section)
+ script = section.split(" run: |\n", 1)[1]
+ script = "\n".join(line[10:] for line in script.splitlines())
+ for change in ("success", "failure", "cancelled", "skipped", ""):
+ for code in ("true", "false", "", "unknown"):
+ for shards in ("success", "failure", "cancelled", "skipped", ""):
+ with self.subTest(change=change, code=code, shards=shards):
+ observed = subprocess.run(
+ ["bash", "-euo", "pipefail", "-c", script],
+ env={**os.environ, "CHANGE_RESULT": change,
+ "CODE_CHANGED": code, "API_SHARDS_RESULT": shards},
+ capture_output=True, text=True, timeout=5,
+ )
+ expected = 0 if change == shards == "success" and code in ("true", "false") else 1
+ self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)
+ if expected == 0:
+ if code == "true":
+ self.assertIn("Every internal/api runtime shard passed", observed.stdout)
+ self.assertNotIn("not applicable", observed.stdout)
+ else:
+ self.assertIn("not applicable", observed.stdout)
+ self.assertIn("no internal/api tests ran", observed.stdout)
+ self.assertNotIn("passed", observed.stdout)
+
if __name__ == "__main__":
unittest.main()