Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Split slow backend suites without weakening required checks

pulse-triage[bot]4c8225fbe7a7
diff --git a/.github/scripts/run-go-package-slice.py b/.github/scripts/run-go-package-slice.py
new file mode 100644
index 0000000000..fd0d3a54c7
--- /dev/null
+++ b/.github/scripts/run-go-package-slice.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Run one exhaustive, order-preserving slice of a slow backend test package.
+
+The two fixed packages have outgrown whole-package CI balancing. Source supplies
+the test inventory; no timing file or maintained test-name allowlist defines
+coverage. The existing JSON recorder checks every selected root's completion,
+including explicit skips, and preserves failures and unfinished output.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+
+ROOT = Path(__file__).resolve().parents[2]
+MODULE = "github.com/rcourtman/pulse-go-rewrite/"
+PACKAGES = ("internal/monitoring", "scripts/installtests")
+RECORDER = ROOT / ".github/scripts/record-internal-api-test-seconds.py"
+
+
+def source_tests(output: str) -> list[str]:
+ names = [line for line in output.splitlines()
+ if line.startswith(("Test", "Fuzz", "Example")) and line.isidentifier()]
+ if not names or len(set(names)) != len(names):
+ raise ValueError("source listing is empty or repeats test names")
+ return names
+
+
+def select(names: list[str], index: int) -> list[str]:
+ if index not in (0, 1) or len(names) < 2:
+ raise ValueError("two nonempty slices and an index of 0 or 1 are required")
+ return names[len(names) * index // 2:len(names) * (index + 1) // 2]
+
+
+def test_filter(names: list[str], selected: list[str]) -> list[str]:
+ chosen = set(selected)
+ others = [name for name in names if name not in chosen]
+ pattern = lambda items: "^(" + "|".join(re.escape(name) for name in items) + ")$"
+ run, skip = pattern(selected), pattern(others)
+ flag, value = ("-skip", skip) if len(skip) < len(run) else ("-run", run)
+ if len(value.encode()) > 120000:
+ raise ValueError("slice filter exceeds the 120000-byte argument bound")
+ # Never combine -run with -skip: Go recompiles the latter per test then.
+ return [flag, value]
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--package", choices=PACKAGES, required=True)
+ parser.add_argument("--index", type=int, choices=(0, 1), required=True)
+ parser.add_argument("--output", type=Path, required=True)
+ args = parser.parse_args()
+ args.output.mkdir(parents=True, exist_ok=True)
+ package = "./" + args.package
+ listing = subprocess.run(["go", "test", "-race", "-list", ".", package],
+ cwd=ROOT, capture_output=True, text=True)
+ if listing.returncode:
+ sys.stdout.write(listing.stdout)
+ sys.stderr.write(listing.stderr)
+ return listing.returncode
+ try:
+ names = source_tests(listing.stdout)
+ selected = select(names, args.index)
+ filters = test_filter(names, selected)
+ except ValueError as error:
+ print(f"Backend slice selection failed: {error}", file=sys.stderr)
+ return 2
+ (args.output / "listed.tests").write_text("\n".join(names) + "\n")
+ selection = args.output / "selected.tests"
+ selection.write_text("\n".join(selected) + "\n")
+ command = ["go", "test", "-race", "-count=1", "-timeout", "50m", "-json",
+ *filters, package]
+ print(f"{args.package} slice {args.index}/2: {len(selected)} of {len(names)} roots "
+ f"({selected[0]} .. {selected[-1]})", flush=True)
+ with subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT) as process:
+ recorded = subprocess.run(
+ [sys.executable, str(RECORDER), str(args.output / "timings.txt"),
+ str(selection), MODULE + args.package], stdin=process.stdout, cwd=ROOT)
+ process.stdout.close()
+ test_exit = process.wait()
+ receipt = {"package": MODULE + args.package, "index": args.index, "count": 2,
+ "listed_roots": len(names), "selected_roots": len(selected),
+ "test_exit": test_exit, "recorder_exit": recorded.returncode}
+ (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n")
+ print(json.dumps(receipt), flush=True)
+ return 0 if test_exit == recorded.returncode == 0 else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index cfdf870bd5..932b8789b1 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -451,8 +451,8 @@ jobs:
- name: Compile every Go race-test binary without executing it
run: bash scripts/compile-go-tests.sh
- backend:
- name: Backend tests (${{ matrix.shard }})
+ backend-packages:
+ name: Backend package tests (${{ matrix.shard }})
needs: [changes, test-compile]
# Expand the unchanged required checks even when compilation fails. A
# skipped dependency must not turn a broken source into a green check.
@@ -465,7 +465,8 @@ jobs:
fail-fast: false
matrix:
# internal/api has its own sharded job below. Measured weights balance
- # every other whole package across these two unchanged required checks.
+ # the remaining whole packages across these two workers. The two slow
+ # packages have exhaustive source-derived slices in backend-slow.
shard: [rest-0, rest-1]
steps:
@@ -524,7 +525,7 @@ jobs:
rest-1) index=1 ;;
*) echo "unknown shard ${{ matrix.shard }}" >&2; exit 1 ;;
esac
- pkgs=$(go list ./... | grep -v '/internal/api$' \
+ pkgs=$(go list ./... | grep -vE '/(internal/api|internal/monitoring|scripts/installtests)$' \
| python3 .github/scripts/select-go-package-shard.py \
.github/scripts/go-package-test-seconds.txt 2 "$index")
if [ -z "$pkgs" ]; then
@@ -534,6 +535,98 @@ jobs:
printf 'Packages in this shard:\n%s\n' "$pkgs"
go test -race -timeout 50m $pkgs
+ # These packages now take 10–12 minutes each under -race. Whole-package
+ # weights cannot shorten that serial floor. Preserve each package's test
+ # order in two source-derived slices; the recorder requires every selected
+ # root exactly once, not just a package-level pass.
+ backend-slow:
+ name: Backend slices (${{ matrix.suite }}-${{ matrix.index }}/2)
+ needs: [changes, test-compile]
+ if: ${{ !cancelled() && needs.changes.result == 'success' && needs.changes.outputs.code == 'true' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 45
+ strategy:
+ fail-fast: false
+ matrix:
+ suite: [monitoring, installtests]
+ index: [0, 1]
+ include:
+ - suite: monitoring
+ package: internal/monitoring
+ - suite: installtests
+ package: scripts/installtests
+ steps:
+ - name: Require compiled backend tests
+ env:
+ COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}
+ COMPILE_RESULT: ${{ needs.test-compile.result }}
+ run: |
+ if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then
+ echo "::error::Backend test compilation did not pass; no sliced tests started."
+ exit 1
+ fi
+
+ - name: Checkout repository
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Set up Go
+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+ with:
+ go-version-file: go.mod
+ cache: true
+
+ - name: Provide frontend embed stub
+ run: bash scripts/ensure_test_assets.sh
+
+ - name: Run source-derived backend slice
+ env:
+ PULSE_DATA_DIR: /tmp/pulse-test-data
+ TEST_PACKAGE: ${{ matrix.package }}
+ TEST_INDEX: ${{ matrix.index }}
+ run: >-
+ python3 .github/scripts/run-go-package-slice.py
+ --package "$TEST_PACKAGE" --index "$TEST_INDEX"
+ --output "$RUNNER_TEMP/backend-slice"
+
+ - name: Retain slice coverage and failure observations
+ if: always()
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: backend-slice-${{ matrix.suite }}-${{ matrix.index }}
+ path: ${{ runner.temp }}/backend-slice/
+ if-no-files-found: warn
+ retention-days: 7
+
+ # Branch protection keeps its exact two names. Neither a skipped/failed
+ # worker nor a partial matrix can manufacture a successful required check.
+ backend:
+ name: Backend tests (${{ matrix.shard }})
+ needs: [changes, backend-packages, backend-slow]
+ if: always()
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ strategy:
+ matrix:
+ shard: [rest-0, rest-1]
+ steps:
+ - name: Require exhaustive backend coverage
+ env:
+ CHANGE_RESULT: ${{ needs.changes.result }}
+ CODE_REQUIRED: ${{ needs.changes.outputs.code }}
+ PACKAGES_RESULT: ${{ needs.backend-packages.result }}
+ SLICES_RESULT: ${{ needs.backend-slow.result }}
+ run: |
+ set -euo pipefail
+ test "$CHANGE_RESULT" = success
+ test "$PACKAGES_RESULT" = success
+ case "$CODE_REQUIRED:$SLICES_RESULT" in
+ true:success) echo "Every whole-package worker and all four source-derived slices passed." ;;
+ false:skipped) echo "Runtime tests not applicable to this source range." ;;
+ *) echo "::error::Backend slice coverage unavailable or failed ($CODE_REQUIRED:$SLICES_RESULT)."; exit 1 ;;
+ esac
+
# internal/api alone took ~27 of the ~31 minutes of the old single
# `Backend tests (api)` job under -race, which made it the critical path of
# every pull request. Its top-level tests are split across these shards
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 240f63afad..a998c26710 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -1318,10 +1318,31 @@ caches only its last compiled pattern, so `-run .` next to `-skip` recompiled
the long skip pattern for every test and subtest and roughly doubled that
shard. The required check name
`Backend tests (api)` belongs to a verdict job that passes only when every
-shard succeeded, and `Backend tests (rest-0)` and `Backend tests (rest-1)` keep
-their names and package split. All shards still expand for documentation-only
-changes so no required check is left pending or reads skipped shards as a
-pass. Go test steps are not skipped for frontend-only changes, because Go tests
+shard succeeded. The required `Backend tests (rest-0)` and `Backend tests
+(rest-1)` names now belong to exhaustive verdicts: both ordinary package
+workers and all four slow-package slices must succeed on code changes.
+The ordinary workers exclude exactly `internal/api`, `internal/monitoring`
+and `scripts/installtests`; API coverage remains in its unchanged five shards.
+`.github/scripts/run-go-package-slice.py` runs each of the latter two slow
+packages in two contiguous, source-listed halves, retaining declaration order,
+`-race`, uncached execution, the 50-minute test deadline and the existing JSON
+recorder's exact selected-root/package checks. It chooses only one of `-run`
+and `-skip`, bounded to 120,000 bytes, and retains the full inventory, selection,
+terminal exits, timings and bounded failure/skip observations as CI artifacts.
+No maintained test-name list or timing estimate defines coverage. A new root
+belongs to exactly one half; failed listing, empty selection, partial execution
+or a producer/recorder failure cannot pass the slice or either required verdict.
+
+Three complete passing main runs on 27 October measured monitoring at
+524–719 seconds and installer tests at 605–725 seconds. Whole-package weights
+cannot shorten that serial floor. Splitting those suites is a CI scheduling
+change, not reduced source coverage or release/native acceptance. Contiguous
+halves may expose existing package-state coupling; actual split-suite proof and
+required CI, not a prediction, establish whether the partition is valid.
+Documentation/receipt-only changes still produce the required verdict names:
+ordinary workers report runtime tests inapplicable, slow slices do not execute,
+and verdicts accept only that exact expected combination. Missing, failed,
+cancelled or unexpected worker states fail closed. Go test steps are not skipped for frontend-only changes, because Go tests
read frontend sources (the `internal/api` contract tests, the
`internal/unifiedresources` code standards walk of `frontend-modern/src`, and
the `internal/telemetry` repository-wide wording scan). Core E2E runs its
@@ -2069,6 +2090,7 @@ Windows lifecycle controls or macOS installer and logging checks on native CI.
13a. `.github/workflows/compile-release-payload.yml`
13b. `.github/workflows/qualify-secure-runtime-release.yml`
14. `.github/workflows/build-and-test.yml`
+14a. `.github/scripts/run-go-package-slice.py`
14. `.github/workflows/create-release.yml`
14. `.github/workflows/deploy-demo-server.yml`
15. `.github/workflows/helm-pages.yml`
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 154f4f1ee1..e39c54818c 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -4650,6 +4650,7 @@
".github/scripts/recover-demo-runtime.sh",
".github/scripts/refresh-internal-api-test-seconds.py",
".github/scripts/resolve-demo-runtime-profile.sh",
+ ".github/scripts/run-go-package-slice.py",
".github/scripts/select-internal-api-shard.sh",
".github/scripts/setup-demo-ssh.sh",
".github/scripts/tests/demo_runtime_native.py",
@@ -5290,7 +5291,8 @@
".github/scripts/select-internal-api-shard.sh",
".github/workflows/build-and-test.yml",
"frontend-modern/package-lock.json",
- "frontend-modern/package.json"
+ "frontend-modern/package.json",
+ ".github/scripts/run-go-package-slice.py"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index bd1f0f6968..f062e8055a 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -4165,11 +4165,11 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {
// Branch protection requires these exact check names. The rest shards stay
// matrix entries and internal/api keeps its name on the verdict job.
- backend := workflowJobBlock(t, workflow, "backend")
+ backend := workflowJobBlock(t, workflow, "backend-packages")
for _, required := range []string{
- "name: Backend tests (${{ matrix.shard }})",
+ "name: Backend package tests (${{ matrix.shard }})",
"shard: [rest-0, rest-1]",
- "grep -v '/internal/api$'",
+ "grep -vE '/(internal/api|internal/monitoring|scripts/installtests)$'",
"python3 .github/scripts/select-go-package-shard.py",
`.github/scripts/go-package-test-seconds.txt 2 "$index")`,
"set -euo pipefail",
@@ -4180,6 +4180,37 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {
}
}
+ // The old protected names now require both ordinary workers and every
+ // source-derived slice. A successful ordinary package cannot conceal an
+ // omitted, failed or skipped slow-package matrix.
+ restVerdict := workflowJobBlock(t, workflow, "backend")
+ for _, required := range []string{
+ "name: Backend tests (${{ matrix.shard }})",
+ "shard: [rest-0, rest-1]",
+ "needs: [changes, backend-packages, backend-slow]",
+ "if: always()",
+ "PACKAGES_RESULT: ${{ needs.backend-packages.result }}",
+ "SLICES_RESULT: ${{ needs.backend-slow.result }}",
+ "true:success)", "false:skipped)",
+ } {
+ if !strings.Contains(restVerdict, required) {
+ t.Fatalf("required rest verdict missing %q", required)
+ }
+ }
+ slow := workflowJobBlock(t, workflow, "backend-slow")
+ for _, required := range []string{
+ "needs: [changes, test-compile]", "fail-fast: false",
+ "suite: [monitoring, installtests]", "index: [0, 1]",
+ "package: internal/monitoring", "package: scripts/installtests",
+ "python3 .github/scripts/run-go-package-slice.py",
+ `--package "$TEST_PACKAGE" --index "$TEST_INDEX"`,
+ "PULSE_DATA_DIR: /tmp/pulse-test-data",
+ } {
+ if !strings.Contains(slow, required) {
+ t.Fatalf("source-derived backend slices missing %q", required)
+ }
+ }
+
shards := workflowJobBlock(t, workflow, "backend-api")
indexes := regexp.MustCompile(`(?m)^ index: \[([0-9, ]+)\]$`).FindStringSubmatch(shards)
count := regexp.MustCompile(`(?m)^ API_SHARD_COUNT: ([0-9]+)$`).FindStringSubmatch(shards)
@@ -5616,8 +5647,8 @@ func TestCITestCompilationPreservesChecks(t *testing.T) {
t.Fatalf("compile admission removed independent CI requirement: %s", required)
}
}
- if strings.Count(workflow, "name: Require compiled backend tests") != 2 {
- t.Fatal("both required backend matrices must report compilation failures")
+ if strings.Count(workflow, "name: Require compiled backend tests") != 3 {
+ t.Fatal("ordinary, sliced and API backend workers must report compilation failures")
}
for _, path := range []string{
repoFile("scripts", "compile-go-tests.sh"),
diff --git a/scripts/tests/test_go_package_slices.py b/scripts/tests/test_go_package_slices.py
new file mode 100644
index 0000000000..9c1ffefdb1
--- /dev/null
+++ b/scripts/tests/test_go_package_slices.py
@@ -0,0 +1,174 @@
+"""Exercise the actual sliced Go runner, recorder and required-check verdict."""
+
+import importlib.util
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+RUNNER = ROOT / ".github/scripts/run-go-package-slice.py"
+WORKFLOW = ROOT / ".github/workflows/build-and-test.yml"
+spec = importlib.util.spec_from_file_location("package_slices", RUNNER)
+runner = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(runner)
+
+
+def job(name):
+ return re.search(rf"^ {name}:\n(.*?)(?=^ [\w-]+:\n|\Z)",
+ WORKFLOW.read_text(), re.M | re.S)[1]
+
+
+class SliceTest(unittest.TestCase):
+ def test_slices_are_contiguous_disjoint_exhaustive_and_keep_source_order(self):
+ for count in (2, 3, 17, 10000):
+ names = [f"TestZ{i}" for i in range(count)]
+ pieces = [runner.select(names, i) for i in range(2)]
+ self.assertTrue(all(pieces))
+ self.assertEqual(pieces[0] + pieces[1], names)
+ for selected in pieces:
+ flag, pattern = runner.test_filter(names, selected)
+ actual = [n for n in names if bool(re.fullmatch(pattern, n)) == (flag == "-run")]
+ self.assertEqual(actual, selected)
+ for names, index in (([], 0), (["TestOne"], 0), (["TestA", "TestB"], 2)):
+ with self.assertRaises(ValueError):
+ runner.select(names, index)
+
+ def test_shorter_filter_never_combines_run_and_skip_and_bounds_actual_bytes(self):
+ names = ["TestA", "Test" + "Long" * 20]
+ self.assertEqual(runner.test_filter(names, [names[1]])[0], "-skip")
+ with self.assertRaises(ValueError):
+ runner.test_filter(["Test" + "A" * 120000, "Test" + "B" * 120000],
+ ["Test" + "A" * 120000])
+
+ def test_listing_rejects_empty_and_duplicate_names_but_ignores_go_summary(self):
+ self.assertEqual(runner.source_tests("TestB\nFuzzA\nExampleZ\nok package 0.1s\n"),
+ ["TestB", "FuzzA", "ExampleZ"])
+ for output in ("ok package 0.1s\n", "TestA\nTestA\n"):
+ with self.assertRaises(ValueError):
+ runner.source_tests(output)
+
+
+class ExecutedRunnerTest(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name)
+ go = self.root / "go"
+ go.write_text('''#!/usr/bin/env python3
+import json, os, re, sys
+from pathlib import Path
+args = sys.argv[1:]
+with Path(os.environ['CALLS']).open('a') as f: f.write(json.dumps(args)+'\\n')
+names = ['TestZ', 'TestA', 'FuzzB', 'ExampleC', 'TestLast']
+if '-list' in args:
+ if os.environ.get('LISTING_EXIT'):
+ print('original listing failure', file=sys.stderr)
+ sys.exit(int(os.environ['LISTING_EXIT']))
+ print('\\n'.join(names)); print('ok package 0.01s'); sys.exit(0)
+flag = '-run' if '-run' in args else '-skip'
+pattern = args[args.index(flag)+1]
+names = [n for n in names if bool(re.fullmatch(pattern,n)) == (flag == '-run')]
+pkg = 'github.com/rcourtman/pulse-go-rewrite/'+args[-1][2:]
+mode = os.environ.get('MODE','')
+if mode == 'wrong-package': pkg = 'example.invalid/other'
+if mode == 'extra': names.append('TestUnexpected')
+if mode == 'missing': names.pop()
+if mode == 'duplicate': names.append(names[0])
+def emit(action, **kw): print(json.dumps({'Action':action,'Package':pkg,**kw}))
+emit('start')
+for name in names:
+ emit('run',Test=name)
+ if mode != 'unfinished':
+ emit('skip' if mode == 'skip' else ('fail' if mode == 'failed-root' else 'pass'),
+ Test=name,Elapsed=0.01)
+if mode != 'truncated': emit('pass',Elapsed=0.02)
+sys.exit(int(os.environ.get('TEST_EXIT','0')))
+''')
+ go.chmod(0o700)
+
+ def execute(self, package="internal/monitoring", index=0, **overrides):
+ target = self.root / "result"
+ env = {**os.environ, "PATH": str(self.root) + ":" + os.environ["PATH"],
+ "CALLS": str(self.root / "calls"), **overrides}
+ result = subprocess.run(
+ ["python3", str(RUNNER), "--package", package, "--index", str(index),
+ "--output", str(target)], env=env, capture_output=True, text=True, timeout=20)
+ return result, target
+
+ def test_actual_command_runs_all_fixed_package_slices_through_recorder(self):
+ for package in runner.PACKAGES:
+ selected = []
+ for index in range(2):
+ result, target = self.execute(package, index)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ receipt = json.loads((target / "receipt.json").read_text())
+ self.assertEqual((receipt['test_exit'], receipt['recorder_exit']), (0, 0))
+ self.assertEqual(receipt['package'], runner.MODULE + package)
+ selected += (target / "selected.tests").read_text().splitlines()
+ actual = json.loads((self.root / "calls").read_text().splitlines()[-1])
+ self.assertIn("-race", actual)
+ self.assertIn("-count=1", actual)
+ self.assertIn("50m", actual)
+ self.assertEqual(sum(x in actual for x in ("-run", "-skip")), 1)
+ self.assertEqual(selected, ['TestZ', 'TestA', 'FuzzB', 'ExampleC', 'TestLast'])
+
+ def test_complete_explicit_skips_stay_observed_not_missing_execution(self):
+ result, target = self.execute(MODE="skip")
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ sidecar = json.loads((target / "timings.txt.failures.json").read_text())
+ self.assertEqual(sidecar['skipped_top_level_tests']['names'], ['TestA', 'TestZ'])
+
+ def test_actual_pipeline_cannot_hide_go_recorder_or_incomplete_coverage_failure(self):
+ for options in ({"TEST_EXIT": "9"}, {"LISTING_EXIT": "7"},
+ *({"MODE": mode} for mode in ('wrong-package', 'extra', 'missing',
+ 'duplicate', 'unfinished', 'failed-root', 'truncated'))):
+ with self.subTest(options=options):
+ result, _ = self.execute(**options)
+ self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
+ result, _ = self.execute(LISTING_EXIT="7")
+ self.assertIn('original listing failure', result.stderr)
+
+
+class WorkflowTest(unittest.TestCase):
+ def test_fixed_matrix_and_whole_package_exclusion_match_the_runner(self):
+ slow = job('backend-slow')
+ self.assertIn('suite: [monitoring, installtests]', slow)
+ self.assertIn('index: [0, 1]', slow)
+ self.assertEqual(re.findall(r' package: (\S+)', slow), list(runner.PACKAGES))
+ for required in ('fail-fast: false', 'needs: [changes, test-compile]',
+ 'needs.changes.outputs.code == \'true\'', 'persist-credentials: false',
+ 'if: always()', 'run-go-package-slice.py', '--package "$TEST_PACKAGE"',
+ '--index "$TEST_INDEX"', 'backend-slice-${{ matrix.suite }}-${{ matrix.index }}'):
+ self.assertIn(required, slow)
+ self.assertNotIn('continue-on-error', slow)
+ self.assertIn("grep -vE '/(internal/api|internal/monitoring|scripts/installtests)$'",
+ job('backend-packages'))
+ self.assertIn('shard: [rest-0, rest-1]', job('backend'))
+ self.assertIn('name: Backend tests (${{ matrix.shard }})', job('backend'))
+ self.assertIn('needs: [changes, backend-packages, backend-slow]', job('backend'))
+ self.assertIn('if: always()', job('backend'))
+
+ def test_actual_required_verdict_rejects_every_unavailable_or_incomplete_matrix(self):
+ verdict = job('backend').split(' run: |\n', 1)[1]
+ script = '\n'.join(line[10:] for line in verdict.splitlines()
+ if line.startswith(' ') or not line.strip())
+ for code, slices in (('true', 'success'), ('false', 'skipped')):
+ env = {**os.environ, 'CHANGE_RESULT': 'success', 'CODE_REQUIRED': code,
+ 'PACKAGES_RESULT': 'success', 'SLICES_RESULT': slices}
+ result = subprocess.run(['bash', '-c', script], env=env, capture_output=True)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ for field in ('CHANGE_RESULT', 'PACKAGES_RESULT', 'SLICES_RESULT', 'CODE_REQUIRED'):
+ for value in ('', 'failure', 'cancelled', 'unknown', 'true' if field != 'CODE_REQUIRED' else 'other'):
+ with self.subTest(code=code, field=field, value=value):
+ result = subprocess.run(['bash', '-c', script], env={**env, field: value},
+ capture_output=True)
+ self.assertNotEqual(result.returncode, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/tests/test_go_test_compilation.py b/scripts/tests/test_go_test_compilation.py
index b8af39061e..91e76379ed 100644
--- a/scripts/tests/test_go_test_compilation.py
+++ b/scripts/tests/test_go_test_compilation.py
@@ -132,10 +132,13 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle
def test_required_checks_fail_on_bad_compile_and_pass_docs_only_skip(self):
source = WORKFLOW.read_text()
- for name in ("backend", "backend-api"):
+ for name in ("backend-packages", "backend-slow", "backend-api"):
section = job(source, name)
self.assertIn("needs: [changes, test-compile]", section)
- self.assertIn("if: ${{ !cancelled() && needs.changes.result == 'success' }}", section)
+ condition = "if: ${{ !cancelled() && needs.changes.result == 'success'"
+ if name == "backend-slow":
+ condition += " && needs.changes.outputs.code == 'true'"
+ self.assertIn(condition + " }}", section)
self.assertIn("COMPILE_RESULT: ${{ needs.test-compile.result }}", section)
self.assertLess(section.index("Require compiled backend tests"), section.index("Checkout repository"))
script = prerequisite_script(section)
@@ -150,7 +153,12 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle
expected = 0 if code == "false" or result == "success" else 1
self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)
# Compile-only success cannot replace these full race/vet runs.
- self.assertIn("go test -race -timeout 50m", section)
+ if name == "backend-slow":
+ self.assertIn("run-go-package-slice.py", section)
+ self.assertIn('"-race", "-count=1", "-timeout", "50m", "-json"',
+ (ROOT / ".github/scripts/run-go-package-slice.py").read_text())
+ else:
+ self.assertIn("go test -race -timeout 50m", section)
self.assertIn("shard: [rest-0, rest-1]", job(source, "backend"))
self.assertIn("index: [0, 1, 2, 3, 4]", job(source, "backend-api"))
self.assertIn("name: Backend tests (api)", job(source, "backend-api-verdict"))