Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Split slow backend suites without weakening required checks
pulse-triage[bot]4c8225fbe7a7
diff --git a/.github/scripts/run-go-package-slice.py b/.github/scripts/run-go-package-slice.pynew file mode 100644index 0000000000..fd0d3a54c7--- /dev/null+++ b/.github/scripts/run-go-package-slice.py@@ -0,0 +1,96 @@+#!/usr/bin/env python3+"""Run one exhaustive, order-preserving slice of a slow backend test package.++The two fixed packages have outgrown whole-package CI balancing. Source supplies+the test inventory; no timing file or maintained test-name allowlist defines+coverage. The existing JSON recorder checks every selected root's completion,+including explicit skips, and preserves failures and unfinished output.+"""++from __future__ import annotations++import argparse+import json+from pathlib import Path+import re+import subprocess+import sys+++ROOT = Path(__file__).resolve().parents[2]+MODULE = "github.com/rcourtman/pulse-go-rewrite/"+PACKAGES = ("internal/monitoring", "scripts/installtests")+RECORDER = ROOT / ".github/scripts/record-internal-api-test-seconds.py"+++def source_tests(output: str) -> list[str]:+ names = [line for line in output.splitlines()+ if line.startswith(("Test", "Fuzz", "Example")) and line.isidentifier()]+ if not names or len(set(names)) != len(names):+ raise ValueError("source listing is empty or repeats test names")+ return names+++def select(names: list[str], index: int) -> list[str]:+ if index not in (0, 1) or len(names) < 2:+ raise ValueError("two nonempty slices and an index of 0 or 1 are required")+ return names[len(names) * index // 2:len(names) * (index + 1) // 2]+++def test_filter(names: list[str], selected: list[str]) -> list[str]:+ chosen = set(selected)+ others = [name for name in names if name not in chosen]+ pattern = lambda items: "^(" + "|".join(re.escape(name) for name in items) + ")$"+ run, skip = pattern(selected), pattern(others)+ flag, value = ("-skip", skip) if len(skip) < len(run) else ("-run", run)+ if len(value.encode()) > 120000:+ raise ValueError("slice filter exceeds the 120000-byte argument bound")+ # Never combine -run with -skip: Go recompiles the latter per test then.+ return [flag, value]+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--package", choices=PACKAGES, required=True)+ parser.add_argument("--index", type=int, choices=(0, 1), required=True)+ parser.add_argument("--output", type=Path, required=True)+ args = parser.parse_args()+ args.output.mkdir(parents=True, exist_ok=True)+ package = "./" + args.package+ listing = subprocess.run(["go", "test", "-race", "-list", ".", package],+ cwd=ROOT, capture_output=True, text=True)+ if listing.returncode:+ sys.stdout.write(listing.stdout)+ sys.stderr.write(listing.stderr)+ return listing.returncode+ try:+ names = source_tests(listing.stdout)+ selected = select(names, args.index)+ filters = test_filter(names, selected)+ except ValueError as error:+ print(f"Backend slice selection failed: {error}", file=sys.stderr)+ return 2+ (args.output / "listed.tests").write_text("\n".join(names) + "\n")+ selection = args.output / "selected.tests"+ selection.write_text("\n".join(selected) + "\n")+ command = ["go", "test", "-race", "-count=1", "-timeout", "50m", "-json",+ *filters, package]+ print(f"{args.package} slice {args.index}/2: {len(selected)} of {len(names)} roots "+ f"({selected[0]} .. {selected[-1]})", flush=True)+ with subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE,+ stderr=subprocess.STDOUT) as process:+ recorded = subprocess.run(+ [sys.executable, str(RECORDER), str(args.output / "timings.txt"),+ str(selection), MODULE + args.package], stdin=process.stdout, cwd=ROOT)+ process.stdout.close()+ test_exit = process.wait()+ receipt = {"package": MODULE + args.package, "index": args.index, "count": 2,+ "listed_roots": len(names), "selected_roots": len(selected),+ "test_exit": test_exit, "recorder_exit": recorded.returncode}+ (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")+ print(json.dumps(receipt), flush=True)+ return 0 if test_exit == recorded.returncode == 0 else 1+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.ymlindex cfdf870bd5..932b8789b1 100644--- a/.github/workflows/build-and-test.yml+++ b/.github/workflows/build-and-test.yml@@ -451,8 +451,8 @@ jobs:- name: Compile every Go race-test binary without executing itrun: bash scripts/compile-go-tests.sh- backend:- name: Backend tests (${{ matrix.shard }})+ backend-packages:+ name: Backend package tests (${{ matrix.shard }})needs: [changes, test-compile]# Expand the unchanged required checks even when compilation fails. A# skipped dependency must not turn a broken source into a green check.@@ -465,7 +465,8 @@ jobs:fail-fast: falsematrix:# internal/api has its own sharded job below. Measured weights balance- # every other whole package across these two unchanged required checks.+ # the remaining whole packages across these two workers. The two slow+ # packages have exhaustive source-derived slices in backend-slow.shard: [rest-0, rest-1]steps:@@ -524,7 +525,7 @@ jobs:rest-1) index=1 ;;*) echo "unknown shard ${{ matrix.shard }}" >&2; exit 1 ;;esac- pkgs=$(go list ./... | grep -v '/internal/api$' \+ pkgs=$(go list ./... | grep -vE '/(internal/api|internal/monitoring|scripts/installtests)$' \| python3 .github/scripts/select-go-package-shard.py \.github/scripts/go-package-test-seconds.txt 2 "$index")if [ -z "$pkgs" ]; then@@ -534,6 +535,98 @@ jobs:printf 'Packages in this shard:\n%s\n' "$pkgs"go test -race -timeout 50m $pkgs+ # These packages now take 10–12 minutes each under -race. Whole-package+ # weights cannot shorten that serial floor. Preserve each package's test+ # order in two source-derived slices; the recorder requires every selected+ # root exactly once, not just a package-level pass.+ backend-slow:+ name: Backend slices (${{ matrix.suite }}-${{ matrix.index }}/2)+ needs: [changes, test-compile]+ if: ${{ !cancelled() && needs.changes.result == 'success' && needs.changes.outputs.code == 'true' }}+ runs-on: ubuntu-24.04+ timeout-minutes: 45+ strategy:+ fail-fast: false+ matrix:+ suite: [monitoring, installtests]+ index: [0, 1]+ include:+ - suite: monitoring+ package: internal/monitoring+ - suite: installtests+ package: scripts/installtests+ steps:+ - name: Require compiled backend tests+ env:+ COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}+ COMPILE_RESULT: ${{ needs.test-compile.result }}+ run: |+ if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then+ echo "::error::Backend test compilation did not pass; no sliced tests started."+ exit 1+ fi++ - name: Checkout repository+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - name: Set up Go+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0+ with:+ go-version-file: go.mod+ cache: true++ - name: Provide frontend embed stub+ run: bash scripts/ensure_test_assets.sh++ - name: Run source-derived backend slice+ env:+ PULSE_DATA_DIR: /tmp/pulse-test-data+ TEST_PACKAGE: ${{ matrix.package }}+ TEST_INDEX: ${{ matrix.index }}+ run: >-+ python3 .github/scripts/run-go-package-slice.py+ --package "$TEST_PACKAGE" --index "$TEST_INDEX"+ --output "$RUNNER_TEMP/backend-slice"++ - name: Retain slice coverage and failure observations+ if: always()+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2+ with:+ name: backend-slice-${{ matrix.suite }}-${{ matrix.index }}+ path: ${{ runner.temp }}/backend-slice/+ if-no-files-found: warn+ retention-days: 7++ # Branch protection keeps its exact two names. Neither a skipped/failed+ # worker nor a partial matrix can manufacture a successful required check.+ backend:+ name: Backend tests (${{ matrix.shard }})+ needs: [changes, backend-packages, backend-slow]+ if: always()+ runs-on: ubuntu-24.04+ timeout-minutes: 5+ strategy:+ matrix:+ shard: [rest-0, rest-1]+ steps:+ - name: Require exhaustive backend coverage+ env:+ CHANGE_RESULT: ${{ needs.changes.result }}+ CODE_REQUIRED: ${{ needs.changes.outputs.code }}+ PACKAGES_RESULT: ${{ needs.backend-packages.result }}+ SLICES_RESULT: ${{ needs.backend-slow.result }}+ run: |+ set -euo pipefail+ test "$CHANGE_RESULT" = success+ test "$PACKAGES_RESULT" = success+ case "$CODE_REQUIRED:$SLICES_RESULT" in+ true:success) echo "Every whole-package worker and all four source-derived slices passed." ;;+ false:skipped) echo "Runtime tests not applicable to this source range." ;;+ *) echo "::error::Backend slice coverage unavailable or failed ($CODE_REQUIRED:$SLICES_RESULT)."; exit 1 ;;+ esac+# internal/api alone took ~27 of the ~31 minutes of the old single# `Backend tests (api)` job under -race, which made it the critical path of# every pull request. Its top-level tests are split across these shardsdiff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 240f63afad..a998c26710 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -1318,10 +1318,31 @@ caches only its last compiled pattern, so `-run .` next to `-skip` recompiledthe long skip pattern for every test and subtest and roughly doubled thatshard. The required check name`Backend tests (api)` belongs to a verdict job that passes only when every-shard succeeded, and `Backend tests (rest-0)` and `Backend tests (rest-1)` keep-their names and package split. All shards still expand for documentation-only-changes so no required check is left pending or reads skipped shards as a-pass. Go test steps are not skipped for frontend-only changes, because Go tests+shard succeeded. The required `Backend tests (rest-0)` and `Backend tests+(rest-1)` names now belong to exhaustive verdicts: both ordinary package+workers and all four slow-package slices must succeed on code changes.+The ordinary workers exclude exactly `internal/api`, `internal/monitoring`+and `scripts/installtests`; API coverage remains in its unchanged five shards.+`.github/scripts/run-go-package-slice.py` runs each of the latter two slow+packages in two contiguous, source-listed halves, retaining declaration order,+`-race`, uncached execution, the 50-minute test deadline and the existing JSON+recorder's exact selected-root/package checks. It chooses only one of `-run`+and `-skip`, bounded to 120,000 bytes, and retains the full inventory, selection,+terminal exits, timings and bounded failure/skip observations as CI artifacts.+No maintained test-name list or timing estimate defines coverage. A new root+belongs to exactly one half; failed listing, empty selection, partial execution+or a producer/recorder failure cannot pass the slice or either required verdict.++Three complete passing main runs on 27 October measured monitoring at+524–719 seconds and installer tests at 605–725 seconds. Whole-package weights+cannot shorten that serial floor. Splitting those suites is a CI scheduling+change, not reduced source coverage or release/native acceptance. Contiguous+halves may expose existing package-state coupling; actual split-suite proof and+required CI, not a prediction, establish whether the partition is valid.+Documentation/receipt-only changes still produce the required verdict names:+ordinary workers report runtime tests inapplicable, slow slices do not execute,+and verdicts accept only that exact expected combination. Missing, failed,+cancelled or unexpected worker states fail closed. Go test steps are not skipped for frontend-only changes, because Go testsread frontend sources (the `internal/api` contract tests, the`internal/unifiedresources` code standards walk of `frontend-modern/src`, andthe `internal/telemetry` repository-wide wording scan). Core E2E runs its@@ -2069,6 +2090,7 @@ Windows lifecycle controls or macOS installer and logging checks on native CI.13a. `.github/workflows/compile-release-payload.yml`13b. `.github/workflows/qualify-secure-runtime-release.yml`14. `.github/workflows/build-and-test.yml`+14a. `.github/scripts/run-go-package-slice.py`14. `.github/workflows/create-release.yml`14. `.github/workflows/deploy-demo-server.yml`15. `.github/workflows/helm-pages.yml`diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 154f4f1ee1..e39c54818c 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -4650,6 +4650,7 @@".github/scripts/recover-demo-runtime.sh",".github/scripts/refresh-internal-api-test-seconds.py",".github/scripts/resolve-demo-runtime-profile.sh",+ ".github/scripts/run-go-package-slice.py",".github/scripts/select-internal-api-shard.sh",".github/scripts/setup-demo-ssh.sh",".github/scripts/tests/demo_runtime_native.py",@@ -5290,7 +5291,8 @@".github/scripts/select-internal-api-shard.sh",".github/workflows/build-and-test.yml","frontend-modern/package-lock.json",- "frontend-modern/package.json"+ "frontend-modern/package.json",+ ".github/scripts/run-go-package-slice.py"],"allow_same_subsystem_tests": false,"test_prefixes": [],diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex bd1f0f6968..f062e8055a 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -4165,11 +4165,11 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {// Branch protection requires these exact check names. The rest shards stay// matrix entries and internal/api keeps its name on the verdict job.- backend := workflowJobBlock(t, workflow, "backend")+ backend := workflowJobBlock(t, workflow, "backend-packages")for _, required := range []string{- "name: Backend tests (${{ matrix.shard }})",+ "name: Backend package tests (${{ matrix.shard }})","shard: [rest-0, rest-1]",- "grep -v '/internal/api$'",+ "grep -vE '/(internal/api|internal/monitoring|scripts/installtests)$'","python3 .github/scripts/select-go-package-shard.py",`.github/scripts/go-package-test-seconds.txt 2 "$index")`,"set -euo pipefail",@@ -4180,6 +4180,37 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {}}+ // The old protected names now require both ordinary workers and every+ // source-derived slice. A successful ordinary package cannot conceal an+ // omitted, failed or skipped slow-package matrix.+ restVerdict := workflowJobBlock(t, workflow, "backend")+ for _, required := range []string{+ "name: Backend tests (${{ matrix.shard }})",+ "shard: [rest-0, rest-1]",+ "needs: [changes, backend-packages, backend-slow]",+ "if: always()",+ "PACKAGES_RESULT: ${{ needs.backend-packages.result }}",+ "SLICES_RESULT: ${{ needs.backend-slow.result }}",+ "true:success)", "false:skipped)",+ } {+ if !strings.Contains(restVerdict, required) {+ t.Fatalf("required rest verdict missing %q", required)+ }+ }+ slow := workflowJobBlock(t, workflow, "backend-slow")+ for _, required := range []string{+ "needs: [changes, test-compile]", "fail-fast: false",+ "suite: [monitoring, installtests]", "index: [0, 1]",+ "package: internal/monitoring", "package: scripts/installtests",+ "python3 .github/scripts/run-go-package-slice.py",+ `--package "$TEST_PACKAGE" --index "$TEST_INDEX"`,+ "PULSE_DATA_DIR: /tmp/pulse-test-data",+ } {+ if !strings.Contains(slow, required) {+ t.Fatalf("source-derived backend slices missing %q", required)+ }+ }+shards := workflowJobBlock(t, workflow, "backend-api")indexes := regexp.MustCompile(`(?m)^ index: \[([0-9, ]+)\]$`).FindStringSubmatch(shards)count := regexp.MustCompile(`(?m)^ API_SHARD_COUNT: ([0-9]+)$`).FindStringSubmatch(shards)@@ -5616,8 +5647,8 @@ func TestCITestCompilationPreservesChecks(t *testing.T) {t.Fatalf("compile admission removed independent CI requirement: %s", required)}}- if strings.Count(workflow, "name: Require compiled backend tests") != 2 {- t.Fatal("both required backend matrices must report compilation failures")+ if strings.Count(workflow, "name: Require compiled backend tests") != 3 {+ t.Fatal("ordinary, sliced and API backend workers must report compilation failures")}for _, path := range []string{repoFile("scripts", "compile-go-tests.sh"),diff --git a/scripts/tests/test_go_package_slices.py b/scripts/tests/test_go_package_slices.pynew file mode 100644index 0000000000..9c1ffefdb1--- /dev/null+++ b/scripts/tests/test_go_package_slices.py@@ -0,0 +1,174 @@+"""Exercise the actual sliced Go runner, recorder and required-check verdict."""++import importlib.util+import json+import os+from pathlib import Path+import re+import subprocess+import tempfile+import unittest+++ROOT = Path(__file__).resolve().parents[2]+RUNNER = ROOT / ".github/scripts/run-go-package-slice.py"+WORKFLOW = ROOT / ".github/workflows/build-and-test.yml"+spec = importlib.util.spec_from_file_location("package_slices", RUNNER)+runner = importlib.util.module_from_spec(spec)+spec.loader.exec_module(runner)+++def job(name):+ return re.search(rf"^ {name}:\n(.*?)(?=^ [\w-]+:\n|\Z)",+ WORKFLOW.read_text(), re.M | re.S)[1]+++class SliceTest(unittest.TestCase):+ def test_slices_are_contiguous_disjoint_exhaustive_and_keep_source_order(self):+ for count in (2, 3, 17, 10000):+ names = [f"TestZ{i}" for i in range(count)]+ pieces = [runner.select(names, i) for i in range(2)]+ self.assertTrue(all(pieces))+ self.assertEqual(pieces[0] + pieces[1], names)+ for selected in pieces:+ flag, pattern = runner.test_filter(names, selected)+ actual = [n for n in names if bool(re.fullmatch(pattern, n)) == (flag == "-run")]+ self.assertEqual(actual, selected)+ for names, index in (([], 0), (["TestOne"], 0), (["TestA", "TestB"], 2)):+ with self.assertRaises(ValueError):+ runner.select(names, index)++ def test_shorter_filter_never_combines_run_and_skip_and_bounds_actual_bytes(self):+ names = ["TestA", "Test" + "Long" * 20]+ self.assertEqual(runner.test_filter(names, [names[1]])[0], "-skip")+ with self.assertRaises(ValueError):+ runner.test_filter(["Test" + "A" * 120000, "Test" + "B" * 120000],+ ["Test" + "A" * 120000])++ def test_listing_rejects_empty_and_duplicate_names_but_ignores_go_summary(self):+ self.assertEqual(runner.source_tests("TestB\nFuzzA\nExampleZ\nok package 0.1s\n"),+ ["TestB", "FuzzA", "ExampleZ"])+ for output in ("ok package 0.1s\n", "TestA\nTestA\n"):+ with self.assertRaises(ValueError):+ runner.source_tests(output)+++class ExecutedRunnerTest(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name)+ go = self.root / "go"+ go.write_text('''#!/usr/bin/env python3+import json, os, re, sys+from pathlib import Path+args = sys.argv[1:]+with Path(os.environ['CALLS']).open('a') as f: f.write(json.dumps(args)+'\\n')+names = ['TestZ', 'TestA', 'FuzzB', 'ExampleC', 'TestLast']+if '-list' in args:+ if os.environ.get('LISTING_EXIT'):+ print('original listing failure', file=sys.stderr)+ sys.exit(int(os.environ['LISTING_EXIT']))+ print('\\n'.join(names)); print('ok package 0.01s'); sys.exit(0)+flag = '-run' if '-run' in args else '-skip'+pattern = args[args.index(flag)+1]+names = [n for n in names if bool(re.fullmatch(pattern,n)) == (flag == '-run')]+pkg = 'github.com/rcourtman/pulse-go-rewrite/'+args[-1][2:]+mode = os.environ.get('MODE','')+if mode == 'wrong-package': pkg = 'example.invalid/other'+if mode == 'extra': names.append('TestUnexpected')+if mode == 'missing': names.pop()+if mode == 'duplicate': names.append(names[0])+def emit(action, **kw): print(json.dumps({'Action':action,'Package':pkg,**kw}))+emit('start')+for name in names:+ emit('run',Test=name)+ if mode != 'unfinished':+ emit('skip' if mode == 'skip' else ('fail' if mode == 'failed-root' else 'pass'),+ Test=name,Elapsed=0.01)+if mode != 'truncated': emit('pass',Elapsed=0.02)+sys.exit(int(os.environ.get('TEST_EXIT','0')))+''')+ go.chmod(0o700)++ def execute(self, package="internal/monitoring", index=0, **overrides):+ target = self.root / "result"+ env = {**os.environ, "PATH": str(self.root) + ":" + os.environ["PATH"],+ "CALLS": str(self.root / "calls"), **overrides}+ result = subprocess.run(+ ["python3", str(RUNNER), "--package", package, "--index", str(index),+ "--output", str(target)], env=env, capture_output=True, text=True, timeout=20)+ return result, target++ def test_actual_command_runs_all_fixed_package_slices_through_recorder(self):+ for package in runner.PACKAGES:+ selected = []+ for index in range(2):+ result, target = self.execute(package, index)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ receipt = json.loads((target / "receipt.json").read_text())+ self.assertEqual((receipt['test_exit'], receipt['recorder_exit']), (0, 0))+ self.assertEqual(receipt['package'], runner.MODULE + package)+ selected += (target / "selected.tests").read_text().splitlines()+ actual = json.loads((self.root / "calls").read_text().splitlines()[-1])+ self.assertIn("-race", actual)+ self.assertIn("-count=1", actual)+ self.assertIn("50m", actual)+ self.assertEqual(sum(x in actual for x in ("-run", "-skip")), 1)+ self.assertEqual(selected, ['TestZ', 'TestA', 'FuzzB', 'ExampleC', 'TestLast'])++ def test_complete_explicit_skips_stay_observed_not_missing_execution(self):+ result, target = self.execute(MODE="skip")+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+ sidecar = json.loads((target / "timings.txt.failures.json").read_text())+ self.assertEqual(sidecar['skipped_top_level_tests']['names'], ['TestA', 'TestZ'])++ def test_actual_pipeline_cannot_hide_go_recorder_or_incomplete_coverage_failure(self):+ for options in ({"TEST_EXIT": "9"}, {"LISTING_EXIT": "7"},+ *({"MODE": mode} for mode in ('wrong-package', 'extra', 'missing',+ 'duplicate', 'unfinished', 'failed-root', 'truncated'))):+ with self.subTest(options=options):+ result, _ = self.execute(**options)+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)+ result, _ = self.execute(LISTING_EXIT="7")+ self.assertIn('original listing failure', result.stderr)+++class WorkflowTest(unittest.TestCase):+ def test_fixed_matrix_and_whole_package_exclusion_match_the_runner(self):+ slow = job('backend-slow')+ self.assertIn('suite: [monitoring, installtests]', slow)+ self.assertIn('index: [0, 1]', slow)+ self.assertEqual(re.findall(r' package: (\S+)', slow), list(runner.PACKAGES))+ for required in ('fail-fast: false', 'needs: [changes, test-compile]',+ 'needs.changes.outputs.code == \'true\'', 'persist-credentials: false',+ 'if: always()', 'run-go-package-slice.py', '--package "$TEST_PACKAGE"',+ '--index "$TEST_INDEX"', 'backend-slice-${{ matrix.suite }}-${{ matrix.index }}'):+ self.assertIn(required, slow)+ self.assertNotIn('continue-on-error', slow)+ self.assertIn("grep -vE '/(internal/api|internal/monitoring|scripts/installtests)$'",+ job('backend-packages'))+ self.assertIn('shard: [rest-0, rest-1]', job('backend'))+ self.assertIn('name: Backend tests (${{ matrix.shard }})', job('backend'))+ self.assertIn('needs: [changes, backend-packages, backend-slow]', job('backend'))+ self.assertIn('if: always()', job('backend'))++ def test_actual_required_verdict_rejects_every_unavailable_or_incomplete_matrix(self):+ verdict = job('backend').split(' run: |\n', 1)[1]+ script = '\n'.join(line[10:] for line in verdict.splitlines()+ if line.startswith(' ') or not line.strip())+ for code, slices in (('true', 'success'), ('false', 'skipped')):+ env = {**os.environ, 'CHANGE_RESULT': 'success', 'CODE_REQUIRED': code,+ 'PACKAGES_RESULT': 'success', 'SLICES_RESULT': slices}+ result = subprocess.run(['bash', '-c', script], env=env, capture_output=True)+ self.assertEqual(result.returncode, 0, result.stderr)+ for field in ('CHANGE_RESULT', 'PACKAGES_RESULT', 'SLICES_RESULT', 'CODE_REQUIRED'):+ for value in ('', 'failure', 'cancelled', 'unknown', 'true' if field != 'CODE_REQUIRED' else 'other'):+ with self.subTest(code=code, field=field, value=value):+ result = subprocess.run(['bash', '-c', script], env={**env, field: value},+ capture_output=True)+ self.assertNotEqual(result.returncode, 0)+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/tests/test_go_test_compilation.py b/scripts/tests/test_go_test_compilation.pyindex b8af39061e..91e76379ed 100644--- a/scripts/tests/test_go_test_compilation.py+++ b/scripts/tests/test_go_test_compilation.py@@ -132,10 +132,13 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAledef test_required_checks_fail_on_bad_compile_and_pass_docs_only_skip(self):source = WORKFLOW.read_text()- for name in ("backend", "backend-api"):+ for name in ("backend-packages", "backend-slow", "backend-api"):section = job(source, name)self.assertIn("needs: [changes, test-compile]", section)- self.assertIn("if: ${{ !cancelled() && needs.changes.result == 'success' }}", section)+ condition = "if: ${{ !cancelled() && needs.changes.result == 'success'"+ if name == "backend-slow":+ condition += " && needs.changes.outputs.code == 'true'"+ self.assertIn(condition + " }}", section)self.assertIn("COMPILE_RESULT: ${{ needs.test-compile.result }}", section)self.assertLess(section.index("Require compiled backend tests"), section.index("Checkout repository"))script = prerequisite_script(section)@@ -150,7 +153,12 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAleexpected = 0 if code == "false" or result == "success" else 1self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)# Compile-only success cannot replace these full race/vet runs.- self.assertIn("go test -race -timeout 50m", section)+ if name == "backend-slow":+ self.assertIn("run-go-package-slice.py", section)+ self.assertIn('"-race", "-count=1", "-timeout", "50m", "-json"',+ (ROOT / ".github/scripts/run-go-package-slice.py").read_text())+ else:+ self.assertIn("go test -race -timeout 50m", section)self.assertIn("shard: [rest-0, rest-1]", job(source, "backend"))self.assertIn("index: [0, 1, 2, 3, 4]", job(source, "backend-api"))self.assertIn("name: Backend tests (api)", job(source, "backend-api-verdict"))