Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Require complete source-selected API shard execution evidence
pulse-triage[bot]f2b5080bcc7c
diff --git a/.github/scripts/record-internal-api-test-seconds.py b/.github/scripts/record-internal-api-test-seconds.pyindex 369bf5593f..b5e8c66ada 100755--- a/.github/scripts/record-internal-api-test-seconds.py+++ b/.github/scripts/record-internal-api-test-seconds.py@@ -1,7 +1,8 @@#!/usr/bin/env python3"""Print `go test -json` like plain `go test` and record per-test seconds.-Usage: go test -json ... | record-internal-api-test-seconds.py <seconds-file>+Usage: go test -json ... | record-internal-api-test-seconds.py <seconds-file> \+ [<selected-tests-file> <expected-package>]`go test -json` reports every test the way `-v` does, which for internal/apimeans thousands of lines per shard. This filter keeps the log as readable as@@ -23,6 +24,13 @@ existing timing-artifact upload makes it readable as soon as the shard ends,even while another workflow job is still running. It is an observation index,not a passing verdict: absent package completion and omitted names are explicit.+The recorder also requires one completed package and complete, non-repeated+top-level test executions. CI supplies its source-derived selected test list+and package identity: every selected test must run and finish exactly once,+with no other test or package substituted. Individual test skips are explicit+completed executions, not missing coverage. Empty, malformed and truncated+streams fail closed; their original observations remain in the sidecar.+The exit status is non-zero when any test or package failed, but the callermust still run with `set -o pipefail` so a `go test` failure that produced noJSON (for example a vet or build error) fails the step.@@ -31,6 +39,7 @@ JSON (for example a vet or build error) fails the step.from __future__ import annotationsimport json+from pathlib import Pathimport reimport sysfrom collections import defaultdict@@ -52,10 +61,30 @@ def bounded_identifiers(names: set[str]) -> dict[str, object]:def main() -> int:- if len(sys.argv) != 2:- print(f"usage: {sys.argv[0]} <seconds-file>", file=sys.stderr)+ if len(sys.argv) not in (2, 4):+ print(f"usage: {sys.argv[0]} <seconds-file> [<selected-tests-file> <expected-package>]",+ file=sys.stderr)return 2+ expected = None+ expected_package = None+ if len(sys.argv) == 4:+ try:+ names = Path(sys.argv[2]).read_text(encoding="utf-8").splitlines()+ except (OSError, UnicodeError):+ print("API shard selection is unavailable; no coverage verdict", file=sys.stderr)+ return 2+ # These are source-level identifiers from go test -list, never+ # parameterised subtest labels. Valid Unicode Go names are not subject+ # to the narrower ASCII export allowlist used by the bounded sidecar.+ if (not names or len(set(names)) != len(names) or+ any(not name.isidentifier() or not name.startswith(("Test", "Fuzz", "Example"))+ for name in names) or not sys.argv[3]):+ print("API shard selection is invalid; no coverage verdict", file=sys.stderr)+ return 2+ expected = set(names)+ expected_package = sys.argv[3]+held: dict[str, list[str]] = defaultdict(list)running: set[str] = set()seconds: dict[str, float] = {}@@ -65,6 +94,11 @@ def main() -> int:package_terminal_action = Nonepackage_terminal_count = 0non_json_lines = 0+ packages: set[str] = set()+ started: set[str] = set()+ completed: set[str] = set()+ invalid_events = 0+ unexpected_package_events = 0out = sys.stdoutfor raw in sys.stdin:@@ -79,9 +113,41 @@ def main() -> int:continueaction = event.get("Action")- test = event.get("Test") or ""+ test = event.get("Test", "")+ text = event.get("Output", "")+ # Recent Go versions send compiler/vet diagnostics with ImportPath,+ # not Package. Keep those real diagnostics visible without confusing+ # a dependency's build event with the selected package's test result.+ if action in ("build-output", "build-fail") and isinstance(text, str):+ out.write(text)+ out.flush()+ if action == "build-fail":+ failed = True+ if package_terminal_count:+ invalid_events += 1+ continue++ package = event.get("Package")+ if not isinstance(package, str) or not package:+ invalid_events += 1+ else:+ packages.add(package)+ if expected_package is not None and package != expected_package:+ unexpected_package_events += 1++ # No new event may follow this single package's terminal verdict.+ # Keep the stream's observations rather than overwriting that history.+ if package_terminal_count:+ invalid_events += 1++ if (action not in ("start", "run", "pause", "cont", "pass", "bench", "fail", "output", "skip") or+ not isinstance(test, str) or not isinstance(text, str)):+ invalid_events += 1+ if isinstance(text, str):+ out.write(text)+ out.flush()+ continuetop = test.split("/", 1)[0]- text = event.get("Output") or ""if not top:# Package level: build output, `ok`/`FAIL` summary, panics and@@ -99,13 +165,19 @@ def main() -> int:continueif action == "run" and test == top:+ if top in started:+ invalid_events += 1+ started.add(top)running.add(top)if text:held[top].append(text)if test != top or action not in ("pass", "fail", "skip"):continue+ if top not in running or top in completed:+ invalid_events += 1running.discard(top)+ completed.add(top)lines = held.pop(top, [])if action == "fail":failed = True@@ -122,6 +194,17 @@ def main() -> int:out.write("".join(held.get(top, [])))out.flush()+ missing = expected - completed if expected is not None else set()+ unexpected = (started | completed) - expected if expected is not None else set()+ if (package_terminal_action != "pass" or package_terminal_count != 1 or+ len(packages) != 1 or not completed or started != completed or+ non_json_lines or invalid_events or unexpected_package_events or missing or unexpected):+ failed = True+ # Fixed diagnostic only: the sidecar retains bounded source identities+ # and counts, not package names or private event payloads.+ print("API shard execution evidence is incomplete or inconsistent; no passing verdict",+ file=sys.stderr)+with open(sys.argv[1], "w", encoding="utf-8") as handle:if package_seconds is not None:handle.write(f"# package-seconds {package_seconds:.2f}\n")@@ -140,6 +223,13 @@ def main() -> int:"package_terminal_action": package_terminal_action,"package_terminal_count": package_terminal_count,"non_json_line_count": non_json_lines,+ "invalid_event_count": invalid_events,+ "package_identity_count": len(packages),+ "unexpected_package_event_count": unexpected_package_events,+ "selected_test_count": len(expected) if expected is not None else None,+ "completed_top_level_test_count": len(completed),+ "missing_selected_tests": bounded_identifiers(missing),+ "unexpected_top_level_tests": bounded_identifiers(unexpected),}with open(sys.argv[1] + ".failures.json", "w", encoding="utf-8") as handle:json.dump(index, handle, indent=2, sort_keys=True)diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.ymlindex a5edf45cd4..a5f34c70ee 100644--- a/.github/workflows/build-and-test.yml+++ b/.github/workflows/build-and-test.yml@@ -515,8 +515,14 @@ jobs:echo "internal/api shard ${API_SHARD_INDEX} of ${API_SHARD_COUNT} runs $(printf '%s\n' "$selected" | wc -l | tr -d ' ') of ${total} tests (${selected%%$'\n'*} .. ${selected##*$'\n'})"timings="$RUNNER_TEMP/internal-api-test-seconds"mkdir -p "$timings"+ # Bind the observed runtime stream to this exact source-derived+ # selection. A package pass alone cannot prove a shard ran its tests.+ # The non-.txt suffix keeps selection names out of timing refreshes.+ printf '%s\n' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"+ package=$(go list ./internal/api)go test -race -timeout 50m -json "${filter[@]}" ./internal/api \- | python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt"+ | python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt" \+ "$timings/api-${API_SHARD_INDEX}.selected" "$package"- name: Upload internal/api test timingsif: always() && needs.changes.outputs.code == 'true'diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex 2d5ffc46d3..8a9771e941 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -223,10 +223,24 @@ at 256 ASCII source names of at most 256 characters, with observed/omitted countmissing package completion remains explicit. The artifact's run/source bindingmust be reconciled before using the index. Empty lists alone are not a pass,and this does not replace terminal logs, diagnose failures, grant a retry or-change test selection, exit status, race/vet coverage or required check names.+change test selection, race/vet coverage or required check names.`test_api_failure_index.py` exercises failure/unfinished/package-only/unknown,privacy and bounds controls; the existing Go shard contract verifies the indexbeside unchanged timing output.+The recorder itself must now fail incomplete or inconsistent execution evidence,+even when the producer exits zero: it requires exactly one passing package+completion, no malformed events, and non-repeated complete top-level executions.+Each CI shard supplies its exact source-derived selection in a `.selected` file+and its `go list` package identity. Every selected test must start and finish+exactly once, without another test or package substituting for it. Explicit+per-test skips remain completed observations; a skipped package is not a passed+shard. Missing/unexpected source names use the same bounded, payload-free index+rules above. The selection is retained in the existing artifact but excluded+from timing refreshes by its non-`.txt` suffix. Real credential-free Go race JSON+controls cover pass, skip, fuzz seeds and examples, ordinary failure, a zero-exit+producer omitting selected tests, and a truncated package completion. This+strengthens CI evidence admission, not installed or native acceptance; cached+Go events remain subject to the existing Go cache semantics.`.github/scripts/refresh-internal-api-test-seconds.py --run <id>` downloadsthose artifacts with `gh run download`, takes each test's median across thegiven runs, and rewrites the weights file with `DEFAULT_WEIGHT` set to the meandiff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.goindex 4ffc3de0fd..58f2d8bd2b 100644--- a/scripts/installtests/build_release_assets_test.go+++ b/scripts/installtests/build_release_assets_test.go@@ -4071,6 +4071,9 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {"set -euo pipefail",`go test -race -timeout 50m -json "${filter[@]}" ./internal/api \| python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt"`,+ `printf '%s\n' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"`,+ `package=$(go list ./internal/api)`,+ `"$timings/api-${API_SHARD_INDEX}.selected" "$package"`,"if: always() && needs.changes.outputs.code == 'true'","name: internal-api-test-seconds-${{ matrix.index }}","path: ${{ runner.temp }}/internal-api-test-seconds/",@@ -4121,9 +4124,13 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {t.Helper()recorder := repoFile(".github", "scripts", "record-internal-api-test-seconds.py")- record := func(events string) (string, string, map[string]any, error) {+ record := func(events string, selected []string) (string, string, map[string]any, error) {out := filepath.Join(t.TempDir(), "seconds.txt")- cmd := exec.Command("python3", recorder, out)+ selection := out + ".selected"+ if err := os.WriteFile(selection, []byte(strings.Join(selected, "\n")+"\n"), 0600); err != nil {+ t.Fatal(err)+ }+ cmd := exec.Command("python3", recorder, out, selection, "example/internal/api")cmd.Stdin = strings.NewReader(events)printed, err := cmd.Output()written, readErr := os.ReadFile(out)@@ -4148,7 +4155,7 @@ func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {`{"Action":"output",` + pkg + `,"Output":"ok \texample/internal/api\t2.000s\n"}`,`{"Action":"pass",` + pkg + `,"Elapsed":2}`,}, "\n") + "\n"- printed, written, index, err := record(passing)+ printed, written, index, err := record(passing, []string{"TestQuiet"})if err != nil {t.Fatalf("recorder must pass a passing run: %v", err)}@@ -4172,7 +4179,7 @@ func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {`{"Action":"run",` + pkg + `,"Test":"TestHung"}`,`{"Action":"output",` + pkg + `,"Test":"TestHung","Output":"panic: test timed out\n"}`,}, "\n") + "\n"- printed, written, index, err = record(failing)+ printed, written, index, err = record(failing, []string{"TestBroken", "TestHung"})if err == nil {t.Fatal("recorder must exit non-zero when a test fails or never finishes")}@@ -4197,6 +4204,18 @@ func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {if index["package_terminal_action"] != nil {t.Fatalf("unfinished stream must not manufacture package completion: %v", index)}+ // A successful producer alone is not coverage. Require the observed+ // package completion and every source-selected test, not only emitted names.+ for _, events := range []string{"", `{"Action":"pass",` + pkg + `}`, passing} {+ _, _, index, err = record(events, []string{"TestQuiet", "TestMissing"})+ if err == nil {+ t.Fatal("recorder accepted missing selected tests or package completion")+ }+ missing := index["missing_selected_tests"].(map[string]any)+ if missing["observed_count"].(float64) < 1 {+ t.Fatalf("recorder did not retain missing coverage: %v", index)+ }+ }}// assertInternalAPIShardSelectionExhaustive runs the shard selector the waydiff --git a/scripts/tests/test_api_failure_index.py b/scripts/tests/test_api_failure_index.pyindex 0c4a81b288..07cfc6ceb0 100644--- a/scripts/tests/test_api_failure_index.py+++ b/scripts/tests/test_api_failure_index.py@@ -2,7 +2,9 @@"""Shard artifacts expose bounded source identities, never assertion payloads."""import json+import osfrom pathlib import Path+import shutilimport subprocessimport sysimport tempfile@@ -25,10 +27,15 @@ def event(action, test=None, output=None, elapsed=None):class APIFailureIndexTest(unittest.TestCase):- def record(self, stream):+ def record(self, stream, selected=None, package='example/internal/api'):with tempfile.TemporaryDirectory() as directory:seconds = Path(directory) / 'api-2.txt'- result = subprocess.run([sys.executable, str(RECORDER), str(seconds)],+ command = [sys.executable, str(RECORDER), str(seconds)]+ if selected is not None:+ selection = Path(directory) / 'api-2.selected'+ selection.write_text('\n'.join(selected) + '\n')+ command.extend([str(selection), package])+ result = subprocess.run(command,input=stream, text=True, capture_output=True,timeout=10)raw = Path(str(seconds) + '.failures.json').read_text()@@ -88,8 +95,9 @@ class APIFailureIndexTest(unittest.TestCase):for stream, count in [('', 0), ('private non-json line\n[]\n', 2)]:with self.subTest(stream=bool(stream)):result, _, index, raw = self.record(stream)- # The upstream Go exit still belongs to pipefail, not this index.- self.assertEqual(result.returncode, 0)+ # Unknown completion cannot become a passing workflow verdict,+ # even if a producer happens to return zero.+ self.assertEqual(result.returncode, 1)self.assertEqual(index['non_json_line_count'], count)self.assertIsNone(index['package_terminal_action'])self.assertNotIn('private', raw)@@ -139,8 +147,10 @@ class APIFailureIndexTest(unittest.TestCase):with tempfile.TemporaryDirectory() as directory:root = Path(directory)seconds = root / 'api-2.txt'+ (root / 'api-2.selected').write_text('TestTimed\nTestSelectionIsNotATiming\n')result = subprocess.run([sys.executable, str(RECORDER), str(seconds)],- input=event('pass', 'TestTimed', elapsed=1.25) ++ input=event('run', 'TestTimed') ++ event('pass', 'TestTimed', elapsed=1.25) +event('pass', elapsed=2),text=True, capture_output=True, timeout=10)self.assertEqual(result.returncode, 0, result.stderr)@@ -160,10 +170,12 @@ class APIFailureIndexTest(unittest.TestCase):['bash', '-c', 'set -o pipefail; (printf "compile diagnostic\\n"; exit 17) | ''python3 "$1" "$2"', 'recorder-proof', str(RECORDER), str(seconds)],text=True, capture_output=True, timeout=10)- self.assertEqual(result.returncode, 17)+ # Both producer and recorder fail; pipefail returns the last+ # failing stage rather than manufacturing success.+ self.assertEqual(result.returncode, 1)index = json.loads(Path(str(seconds) + '.failures.json').read_text())self.assertIsNone(index['package_terminal_action'])- self.assertEqual(index['recorder_exit_code'], 0)+ self.assertEqual(index['recorder_exit_code'], 1)self.assertEqual(index['non_json_line_count'], 1)def test_existing_upload_collects_sidecar_on_a_failed_shard(self):@@ -175,6 +187,182 @@ class APIFailureIndexTest(unittest.TestCase):self.assertIn('name: internal-api-test-seconds-${{ matrix.index }}', job)self.assertIn('set -euo pipefail', job)self.assertNotIn('continue-on-error:', job)+ self.assertIn('printf \'%s\\n\' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"', job)+ self.assertIn('package=$(go list ./internal/api)', job)+ self.assertIn('"$timings/api-${API_SHARD_INDEX}.selected" "$package"', job)++ def test_selected_pass_and_explicit_skip_complete_coverage(self):+ stream = (event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2) ++ event('run', 'TestNative') + event('skip', 'TestNative', elapsed=0) ++ event('pass', elapsed=0.3))+ result, _, index, _ = self.record(stream, ['TestGood', 'TestNative'])+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(index['selected_test_count'], 2)+ self.assertEqual(index['completed_top_level_test_count'], 2)+ self.assertEqual(index['missing_selected_tests']['observed_count'], 0)+ self.assertEqual(index['unexpected_top_level_tests']['observed_count'], 0)++ def test_successful_producer_cannot_substitute_incomplete_execution(self):+ good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)+ for stream in ['', event('pass'), good, 'malformed\n' + good + event('pass'),+ good + event('pass') + event('pass'),+ event('pass', 'TestGood') + event('pass'),+ good + good + event('pass'), good + event('skip')]:+ with self.subTest(stream=stream):+ result, _, index, _ = self.record(stream, ['TestGood'])+ self.assertEqual(result.returncode, 1)+ self.assertEqual(index['recorder_exit_code'], 1)+ self.assertIn('no passing verdict', result.stderr)++ def test_selected_tests_must_not_be_dropped_or_replaced(self):+ good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)+ result, _, index, _ = self.record(good + event('pass'), ['TestGood', 'TestMissing'])+ self.assertEqual(result.returncode, 1)+ self.assertEqual(index['missing_selected_tests']['names'], ['TestMissing'])+ other = event('run', 'TestOther') + event('pass', 'TestOther', elapsed=0.2)+ result, _, index, _ = self.record(other + event('pass'), ['TestGood'])+ self.assertEqual(result.returncode, 1)+ self.assertEqual(index['missing_selected_tests']['names'], ['TestGood'])+ self.assertEqual(index['unexpected_top_level_tests']['names'], ['TestOther'])++ def test_unfinished_selected_test_retains_both_coverage_and_original_log(self):+ stream = event('run', 'TestHung') + event('output', 'TestHung/private', 'private log\n')+ result, _, index, raw = self.record(stream, ['TestHung'])+ self.assertEqual(result.returncode, 1)+ self.assertIn('private log', result.stdout)+ self.assertEqual(index['unfinished_top_level_tests']['names'], ['TestHung'])+ self.assertEqual(index['missing_selected_tests']['names'], ['TestHung'])+ self.assertNotIn('private', raw)++ def test_wrong_or_multiple_packages_cannot_satisfy_selection(self):+ good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')+ for stream, package in [(good, 'other/private-package'),+ (good.replace('example/internal/api', 'other/private-package', 1),+ 'example/internal/api')]:+ result, _, index, raw = self.record(stream, ['TestGood'], package)+ self.assertEqual(result.returncode, 1)+ self.assertGreater(index['unexpected_package_event_count'], 0)+ self.assertNotIn('private-package', raw)++ def test_invalid_event_shapes_are_non_passing_not_recorder_crashes(self):+ for changed in [{'Action': 'invented'}, {'Package': None}, {'Test': []}, {'Output': 1}]:+ value = {'Action': 'output', 'Package': 'example/internal/api'}+ value.update(changed)+ stream = (json.dumps(value) + '\n' + event('run', 'TestGood') ++ event('pass', 'TestGood') + event('pass'))+ result, _, index, _ = self.record(stream, ['TestGood'])+ self.assertEqual(result.returncode, 1)+ self.assertGreater(index['invalid_event_count'], 0)+ self.assertNotIn('Traceback', result.stderr)++ def test_go_build_diagnostics_remain_visible_without_test_package_identity(self):+ good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')+ for action, code in [('build-output', 0), ('build-fail', 1)]:+ build = json.dumps({'Action': action, 'ImportPath': 'private/dependency',+ 'Output': 'retained build diagnostic\n'}) + '\n'+ result, _, index, raw = self.record(build + good, ['TestGood'])+ self.assertEqual(result.returncode, code)+ self.assertIn('retained build diagnostic', result.stdout)+ self.assertEqual(index['package_identity_count'], 1)+ self.assertNotIn('private/dependency', raw)+ self.assertNotIn('diagnostic', raw)++ def test_coverage_names_stay_bounded_without_rejecting_unicode_source(self):+ missing = [f'TestMissing{i:04d}' for i in range(301)]+ missing.extend(['TestÜnicode', 'Test' + 'A' * 257])+ stream = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')+ result, _, index, raw = self.record(stream, ['TestGood'] + missing)+ self.assertEqual(result.returncode, 1)+ observation = index['missing_selected_tests']+ self.assertEqual(observation['observed_count'], 303)+ self.assertEqual(len(observation['names']), 256)+ self.assertEqual(observation['omitted_count'], 47)+ self.assertNotIn('Ünicode', raw)+ result, _, _, _ = self.record(+ event('run', 'TestÜnicode') + event('pass', 'TestÜnicode') + event('pass'),+ ['TestÜnicode'])+ self.assertEqual(result.returncode, 0)++ def test_invalid_selection_is_rejected_before_observation(self):+ with tempfile.TemporaryDirectory() as directory:+ seconds = Path(directory) / 'api.txt'+ selected = Path(directory) / 'api.selected'+ for names in ['', 'TestA\nTestA\n', 'TestA/subtest\n', 'TestBad-name\n']:+ selected.write_text(names)+ result = subprocess.run([sys.executable, str(RECORDER), str(seconds),+ str(selected), 'example/internal/api'],+ input='', text=True, capture_output=True, timeout=10)+ self.assertEqual(result.returncode, 2)+ self.assertFalse(seconds.exists())+ selected.unlink()+ result = subprocess.run([sys.executable, str(RECORDER), str(seconds),+ str(selected), 'example/internal/api'],+ input='', text=True, capture_output=True, timeout=10)+ self.assertEqual(result.returncode, 2)+ self.assertIn('unavailable', result.stderr)++ @unittest.skipUnless(shutil.which('go'), 'real Go JSON requires the declared compiler')+ def test_real_race_json_pass_skip_fuzz_example_failure_and_missing_coverage(self):+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ package = 'example.invalid/shardfixture'+ (root / 'go.mod').write_text(f'module {package}\n\ngo 1.24\n')+ (root / 'fixture_test.go').write_text('''package shardfixture+import ("fmt"; "testing")+type Fixture struct{}+func TestGood(t *testing.T) { t.Log("passing fixture output") }+func TestSkip(t *testing.T) { t.Skip("explicit native fixture exclusion") }+func TestFailure(t *testing.T) { t.Fatal("retained fixture failure") }+func FuzzSeed(f *testing.F) { f.Add(1); f.Fuzz(func(t *testing.T, n int) {}) }+func ExampleFixture() { fmt.Println("fixture")+// Output: fixture+}+''')+ env = {**os.environ, 'GOWORK': 'off', 'GOPROXY': 'off', 'GOTOOLCHAIN': 'local'}+ listing = subprocess.run(['go', 'test', '-race', '-list', '.', '.'],+ cwd=root, env=env, text=True, capture_output=True, timeout=120)+ self.assertEqual(listing.returncode, 0, listing.stderr)+ names = [name for name in listing.stdout.splitlines()+ if name.startswith(('Test', 'Fuzz', 'Example'))]+ selected = [name for name in names if name != 'TestFailure']+ self.assertEqual(set(selected), {'TestGood', 'TestSkip', 'FuzzSeed', 'ExampleFixture'})+ command = ['go', 'test', '-race', '-count=1', '-timeout', '30s', '-json']+ # Exercise the same shorter-side -skip path used by large CI shards.+ passing = subprocess.run(command + ['-skip', '^TestFailure$', '.'],+ cwd=root, env=env, text=True, capture_output=True, timeout=120)+ self.assertEqual(passing.returncode, 0, passing.stderr)+ result, _, index, _ = self.record(passing.stdout, selected, package)+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(index['completed_top_level_test_count'], 4)+ self.assertNotIn('passing fixture output', result.stdout)++ # Real Go exits zero while selecting fewer tests than the source+ # list expects. Neither pipefail nor a package pass detects that.+ omitted = subprocess.run(command + ['-run', '^TestGood$', '.'],+ cwd=root, env=env, text=True, capture_output=True, timeout=120)+ self.assertEqual(omitted.returncode, 0, omitted.stderr)+ result, _, index, _ = self.record(omitted.stdout, selected, package)+ self.assertEqual(result.returncode, 1)+ self.assertEqual(index['missing_selected_tests']['observed_count'], 3)++ # Completed tests without the final package event are not a pass.+ events = [json.loads(line) for line in passing.stdout.splitlines()]+ truncated = ''.join(json.dumps(value) + '\n' for value in events+ if not (value.get('Action') == 'pass' and 'Test' not in value))+ result, _, index, _ = self.record(truncated, selected, package)+ self.assertEqual(result.returncode, 1)+ self.assertIsNone(index['package_terminal_action'])++ failing = subprocess.run(command + ['-run', '^TestFailure$', '.'],+ cwd=root, env=env, text=True, capture_output=True, timeout=120)+ self.assertEqual(failing.returncode, 1, failing.stderr)+ result, _, index, _ = self.record(failing.stdout, ['TestFailure'], package)+ self.assertEqual(result.returncode, 1)+ self.assertIn('retained fixture failure', result.stdout)+ self.assertEqual(index['failed_top_level_tests']['names'], ['TestFailure'])+ self.assertEqual(index['missing_selected_tests']['observed_count'], 0)+ print('Real Go race JSON: complete pass/skip/fuzz/example accepted; '+ 'producer-zero omission, truncated completion and actual failure rejected.')if __name__ == '__main__':