Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Require complete source-selected API shard execution evidence

pulse-triage[bot]f2b5080bcc7c
diff --git a/.github/scripts/record-internal-api-test-seconds.py b/.github/scripts/record-internal-api-test-seconds.py
index 369bf5593f..b5e8c66ada 100755
--- a/.github/scripts/record-internal-api-test-seconds.py
+++ b/.github/scripts/record-internal-api-test-seconds.py
@@ -1,7 +1,8 @@
#!/usr/bin/env python3
"""Print `go test -json` like plain `go test` and record per-test seconds.
-Usage: go test -json ... | record-internal-api-test-seconds.py <seconds-file>
+Usage: go test -json ... | record-internal-api-test-seconds.py <seconds-file> \
+ [<selected-tests-file> <expected-package>]
`go test -json` reports every test the way `-v` does, which for internal/api
means thousands of lines per shard. This filter keeps the log as readable as
@@ -23,6 +24,13 @@ existing timing-artifact upload makes it readable as soon as the shard ends,
even while another workflow job is still running. It is an observation index,
not a passing verdict: absent package completion and omitted names are explicit.
+The recorder also requires one completed package and complete, non-repeated
+top-level test executions. CI supplies its source-derived selected test list
+and package identity: every selected test must run and finish exactly once,
+with no other test or package substituted. Individual test skips are explicit
+completed executions, not missing coverage. Empty, malformed and truncated
+streams fail closed; their original observations remain in the sidecar.
+
The exit status is non-zero when any test or package failed, but the caller
must still run with `set -o pipefail` so a `go test` failure that produced no
JSON (for example a vet or build error) fails the step.
@@ -31,6 +39,7 @@ JSON (for example a vet or build error) fails the step.
from __future__ import annotations
import json
+from pathlib import Path
import re
import sys
from collections import defaultdict
@@ -52,10 +61,30 @@ def bounded_identifiers(names: set[str]) -> dict[str, object]:
def main() -> int:
- if len(sys.argv) != 2:
- print(f"usage: {sys.argv[0]} <seconds-file>", file=sys.stderr)
+ if len(sys.argv) not in (2, 4):
+ print(f"usage: {sys.argv[0]} <seconds-file> [<selected-tests-file> <expected-package>]",
+ file=sys.stderr)
return 2
+ expected = None
+ expected_package = None
+ if len(sys.argv) == 4:
+ try:
+ names = Path(sys.argv[2]).read_text(encoding="utf-8").splitlines()
+ except (OSError, UnicodeError):
+ print("API shard selection is unavailable; no coverage verdict", file=sys.stderr)
+ return 2
+ # These are source-level identifiers from go test -list, never
+ # parameterised subtest labels. Valid Unicode Go names are not subject
+ # to the narrower ASCII export allowlist used by the bounded sidecar.
+ if (not names or len(set(names)) != len(names) or
+ any(not name.isidentifier() or not name.startswith(("Test", "Fuzz", "Example"))
+ for name in names) or not sys.argv[3]):
+ print("API shard selection is invalid; no coverage verdict", file=sys.stderr)
+ return 2
+ expected = set(names)
+ expected_package = sys.argv[3]
+
held: dict[str, list[str]] = defaultdict(list)
running: set[str] = set()
seconds: dict[str, float] = {}
@@ -65,6 +94,11 @@ def main() -> int:
package_terminal_action = None
package_terminal_count = 0
non_json_lines = 0
+ packages: set[str] = set()
+ started: set[str] = set()
+ completed: set[str] = set()
+ invalid_events = 0
+ unexpected_package_events = 0
out = sys.stdout
for raw in sys.stdin:
@@ -79,9 +113,41 @@ def main() -> int:
continue
action = event.get("Action")
- test = event.get("Test") or ""
+ test = event.get("Test", "")
+ text = event.get("Output", "")
+ # Recent Go versions send compiler/vet diagnostics with ImportPath,
+ # not Package. Keep those real diagnostics visible without confusing
+ # a dependency's build event with the selected package's test result.
+ if action in ("build-output", "build-fail") and isinstance(text, str):
+ out.write(text)
+ out.flush()
+ if action == "build-fail":
+ failed = True
+ if package_terminal_count:
+ invalid_events += 1
+ continue
+
+ package = event.get("Package")
+ if not isinstance(package, str) or not package:
+ invalid_events += 1
+ else:
+ packages.add(package)
+ if expected_package is not None and package != expected_package:
+ unexpected_package_events += 1
+
+ # No new event may follow this single package's terminal verdict.
+ # Keep the stream's observations rather than overwriting that history.
+ if package_terminal_count:
+ invalid_events += 1
+
+ if (action not in ("start", "run", "pause", "cont", "pass", "bench", "fail", "output", "skip") or
+ not isinstance(test, str) or not isinstance(text, str)):
+ invalid_events += 1
+ if isinstance(text, str):
+ out.write(text)
+ out.flush()
+ continue
top = test.split("/", 1)[0]
- text = event.get("Output") or ""
if not top:
# Package level: build output, `ok`/`FAIL` summary, panics and
@@ -99,13 +165,19 @@ def main() -> int:
continue
if action == "run" and test == top:
+ if top in started:
+ invalid_events += 1
+ started.add(top)
running.add(top)
if text:
held[top].append(text)
if test != top or action not in ("pass", "fail", "skip"):
continue
+ if top not in running or top in completed:
+ invalid_events += 1
running.discard(top)
+ completed.add(top)
lines = held.pop(top, [])
if action == "fail":
failed = True
@@ -122,6 +194,17 @@ def main() -> int:
out.write("".join(held.get(top, [])))
out.flush()
+ missing = expected - completed if expected is not None else set()
+ unexpected = (started | completed) - expected if expected is not None else set()
+ if (package_terminal_action != "pass" or package_terminal_count != 1 or
+ len(packages) != 1 or not completed or started != completed or
+ non_json_lines or invalid_events or unexpected_package_events or missing or unexpected):
+ failed = True
+ # Fixed diagnostic only: the sidecar retains bounded source identities
+ # and counts, not package names or private event payloads.
+ print("API shard execution evidence is incomplete or inconsistent; no passing verdict",
+ file=sys.stderr)
+
with open(sys.argv[1], "w", encoding="utf-8") as handle:
if package_seconds is not None:
handle.write(f"# package-seconds {package_seconds:.2f}\n")
@@ -140,6 +223,13 @@ def main() -> int:
"package_terminal_action": package_terminal_action,
"package_terminal_count": package_terminal_count,
"non_json_line_count": non_json_lines,
+ "invalid_event_count": invalid_events,
+ "package_identity_count": len(packages),
+ "unexpected_package_event_count": unexpected_package_events,
+ "selected_test_count": len(expected) if expected is not None else None,
+ "completed_top_level_test_count": len(completed),
+ "missing_selected_tests": bounded_identifiers(missing),
+ "unexpected_top_level_tests": bounded_identifiers(unexpected),
}
with open(sys.argv[1] + ".failures.json", "w", encoding="utf-8") as handle:
json.dump(index, handle, indent=2, sort_keys=True)
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index a5edf45cd4..a5f34c70ee 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -515,8 +515,14 @@ jobs:
echo "internal/api shard ${API_SHARD_INDEX} of ${API_SHARD_COUNT} runs $(printf '%s\n' "$selected" | wc -l | tr -d ' ') of ${total} tests (${selected%%$'\n'*} .. ${selected##*$'\n'})"
timings="$RUNNER_TEMP/internal-api-test-seconds"
mkdir -p "$timings"
+ # Bind the observed runtime stream to this exact source-derived
+ # selection. A package pass alone cannot prove a shard ran its tests.
+ # The non-.txt suffix keeps selection names out of timing refreshes.
+ printf '%s\n' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"
+ package=$(go list ./internal/api)
go test -race -timeout 50m -json "${filter[@]}" ./internal/api \
- | python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt"
+ | python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt" \
+ "$timings/api-${API_SHARD_INDEX}.selected" "$package"
- name: Upload internal/api test timings
if: always() && needs.changes.outputs.code == 'true'
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 2d5ffc46d3..8a9771e941 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -223,10 +223,24 @@ at 256 ASCII source names of at most 256 characters, with observed/omitted count
missing package completion remains explicit. The artifact's run/source binding
must be reconciled before using the index. Empty lists alone are not a pass,
and this does not replace terminal logs, diagnose failures, grant a retry or
-change test selection, exit status, race/vet coverage or required check names.
+change test selection, race/vet coverage or required check names.
`test_api_failure_index.py` exercises failure/unfinished/package-only/unknown,
privacy and bounds controls; the existing Go shard contract verifies the index
beside unchanged timing output.
+The recorder itself must now fail incomplete or inconsistent execution evidence,
+even when the producer exits zero: it requires exactly one passing package
+completion, no malformed events, and non-repeated complete top-level executions.
+Each CI shard supplies its exact source-derived selection in a `.selected` file
+and its `go list` package identity. Every selected test must start and finish
+exactly once, without another test or package substituting for it. Explicit
+per-test skips remain completed observations; a skipped package is not a passed
+shard. Missing/unexpected source names use the same bounded, payload-free index
+rules above. The selection is retained in the existing artifact but excluded
+from timing refreshes by its non-`.txt` suffix. Real credential-free Go race JSON
+controls cover pass, skip, fuzz seeds and examples, ordinary failure, a zero-exit
+producer omitting selected tests, and a truncated package completion. This
+strengthens CI evidence admission, not installed or native acceptance; cached
+Go events remain subject to the existing Go cache semantics.
`.github/scripts/refresh-internal-api-test-seconds.py --run <id>` downloads
those artifacts with `gh run download`, takes each test's median across the
given runs, and rewrites the weights file with `DEFAULT_WEIGHT` set to the mean
diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go
index 4ffc3de0fd..58f2d8bd2b 100644
--- a/scripts/installtests/build_release_assets_test.go
+++ b/scripts/installtests/build_release_assets_test.go
@@ -4071,6 +4071,9 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {
"set -euo pipefail",
`go test -race -timeout 50m -json "${filter[@]}" ./internal/api \
| python3 .github/scripts/record-internal-api-test-seconds.py "$timings/api-${API_SHARD_INDEX}.txt"`,
+ `printf '%s\n' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"`,
+ `package=$(go list ./internal/api)`,
+ `"$timings/api-${API_SHARD_INDEX}.selected" "$package"`,
"if: always() && needs.changes.outputs.code == 'true'",
"name: internal-api-test-seconds-${{ matrix.index }}",
"path: ${{ runner.temp }}/internal-api-test-seconds/",
@@ -4121,9 +4124,13 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {
func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {
t.Helper()
recorder := repoFile(".github", "scripts", "record-internal-api-test-seconds.py")
- record := func(events string) (string, string, map[string]any, error) {
+ record := func(events string, selected []string) (string, string, map[string]any, error) {
out := filepath.Join(t.TempDir(), "seconds.txt")
- cmd := exec.Command("python3", recorder, out)
+ selection := out + ".selected"
+ if err := os.WriteFile(selection, []byte(strings.Join(selected, "\n")+"\n"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ cmd := exec.Command("python3", recorder, out, selection, "example/internal/api")
cmd.Stdin = strings.NewReader(events)
printed, err := cmd.Output()
written, readErr := os.ReadFile(out)
@@ -4148,7 +4155,7 @@ func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {
`{"Action":"output",` + pkg + `,"Output":"ok \texample/internal/api\t2.000s\n"}`,
`{"Action":"pass",` + pkg + `,"Elapsed":2}`,
}, "\n") + "\n"
- printed, written, index, err := record(passing)
+ printed, written, index, err := record(passing, []string{"TestQuiet"})
if err != nil {
t.Fatalf("recorder must pass a passing run: %v", err)
}
@@ -4172,7 +4179,7 @@ func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {
`{"Action":"run",` + pkg + `,"Test":"TestHung"}`,
`{"Action":"output",` + pkg + `,"Test":"TestHung","Output":"panic: test timed out\n"}`,
}, "\n") + "\n"
- printed, written, index, err = record(failing)
+ printed, written, index, err = record(failing, []string{"TestBroken", "TestHung"})
if err == nil {
t.Fatal("recorder must exit non-zero when a test fails or never finishes")
}
@@ -4197,6 +4204,18 @@ func assertInternalAPITimingRecorderKeepsFailuresVisible(t *testing.T) {
if index["package_terminal_action"] != nil {
t.Fatalf("unfinished stream must not manufacture package completion: %v", index)
}
+ // A successful producer alone is not coverage. Require the observed
+ // package completion and every source-selected test, not only emitted names.
+ for _, events := range []string{"", `{"Action":"pass",` + pkg + `}`, passing} {
+ _, _, index, err = record(events, []string{"TestQuiet", "TestMissing"})
+ if err == nil {
+ t.Fatal("recorder accepted missing selected tests or package completion")
+ }
+ missing := index["missing_selected_tests"].(map[string]any)
+ if missing["observed_count"].(float64) < 1 {
+ t.Fatalf("recorder did not retain missing coverage: %v", index)
+ }
+ }
}
// assertInternalAPIShardSelectionExhaustive runs the shard selector the way
diff --git a/scripts/tests/test_api_failure_index.py b/scripts/tests/test_api_failure_index.py
index 0c4a81b288..07cfc6ceb0 100644
--- a/scripts/tests/test_api_failure_index.py
+++ b/scripts/tests/test_api_failure_index.py
@@ -2,7 +2,9 @@
"""Shard artifacts expose bounded source identities, never assertion payloads."""
import json
+import os
from pathlib import Path
+import shutil
import subprocess
import sys
import tempfile
@@ -25,10 +27,15 @@ def event(action, test=None, output=None, elapsed=None):
class APIFailureIndexTest(unittest.TestCase):
- def record(self, stream):
+ def record(self, stream, selected=None, package='example/internal/api'):
with tempfile.TemporaryDirectory() as directory:
seconds = Path(directory) / 'api-2.txt'
- result = subprocess.run([sys.executable, str(RECORDER), str(seconds)],
+ command = [sys.executable, str(RECORDER), str(seconds)]
+ if selected is not None:
+ selection = Path(directory) / 'api-2.selected'
+ selection.write_text('\n'.join(selected) + '\n')
+ command.extend([str(selection), package])
+ result = subprocess.run(command,
input=stream, text=True, capture_output=True,
timeout=10)
raw = Path(str(seconds) + '.failures.json').read_text()
@@ -88,8 +95,9 @@ class APIFailureIndexTest(unittest.TestCase):
for stream, count in [('', 0), ('private non-json line\n[]\n', 2)]:
with self.subTest(stream=bool(stream)):
result, _, index, raw = self.record(stream)
- # The upstream Go exit still belongs to pipefail, not this index.
- self.assertEqual(result.returncode, 0)
+ # Unknown completion cannot become a passing workflow verdict,
+ # even if a producer happens to return zero.
+ self.assertEqual(result.returncode, 1)
self.assertEqual(index['non_json_line_count'], count)
self.assertIsNone(index['package_terminal_action'])
self.assertNotIn('private', raw)
@@ -139,8 +147,10 @@ class APIFailureIndexTest(unittest.TestCase):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
seconds = root / 'api-2.txt'
+ (root / 'api-2.selected').write_text('TestTimed\nTestSelectionIsNotATiming\n')
result = subprocess.run([sys.executable, str(RECORDER), str(seconds)],
- input=event('pass', 'TestTimed', elapsed=1.25) +
+ input=event('run', 'TestTimed') +
+ event('pass', 'TestTimed', elapsed=1.25) +
event('pass', elapsed=2),
text=True, capture_output=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
@@ -160,10 +170,12 @@ class APIFailureIndexTest(unittest.TestCase):
['bash', '-c', 'set -o pipefail; (printf "compile diagnostic\\n"; exit 17) | '
'python3 "$1" "$2"', 'recorder-proof', str(RECORDER), str(seconds)],
text=True, capture_output=True, timeout=10)
- self.assertEqual(result.returncode, 17)
+ # Both producer and recorder fail; pipefail returns the last
+ # failing stage rather than manufacturing success.
+ self.assertEqual(result.returncode, 1)
index = json.loads(Path(str(seconds) + '.failures.json').read_text())
self.assertIsNone(index['package_terminal_action'])
- self.assertEqual(index['recorder_exit_code'], 0)
+ self.assertEqual(index['recorder_exit_code'], 1)
self.assertEqual(index['non_json_line_count'], 1)
def test_existing_upload_collects_sidecar_on_a_failed_shard(self):
@@ -175,6 +187,182 @@ class APIFailureIndexTest(unittest.TestCase):
self.assertIn('name: internal-api-test-seconds-${{ matrix.index }}', job)
self.assertIn('set -euo pipefail', job)
self.assertNotIn('continue-on-error:', job)
+ self.assertIn('printf \'%s\\n\' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"', job)
+ self.assertIn('package=$(go list ./internal/api)', job)
+ self.assertIn('"$timings/api-${API_SHARD_INDEX}.selected" "$package"', job)
+
+ def test_selected_pass_and_explicit_skip_complete_coverage(self):
+ stream = (event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2) +
+ event('run', 'TestNative') + event('skip', 'TestNative', elapsed=0) +
+ event('pass', elapsed=0.3))
+ result, _, index, _ = self.record(stream, ['TestGood', 'TestNative'])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(index['selected_test_count'], 2)
+ self.assertEqual(index['completed_top_level_test_count'], 2)
+ self.assertEqual(index['missing_selected_tests']['observed_count'], 0)
+ self.assertEqual(index['unexpected_top_level_tests']['observed_count'], 0)
+
+ def test_successful_producer_cannot_substitute_incomplete_execution(self):
+ good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)
+ for stream in ['', event('pass'), good, 'malformed\n' + good + event('pass'),
+ good + event('pass') + event('pass'),
+ event('pass', 'TestGood') + event('pass'),
+ good + good + event('pass'), good + event('skip')]:
+ with self.subTest(stream=stream):
+ result, _, index, _ = self.record(stream, ['TestGood'])
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(index['recorder_exit_code'], 1)
+ self.assertIn('no passing verdict', result.stderr)
+
+ def test_selected_tests_must_not_be_dropped_or_replaced(self):
+ good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)
+ result, _, index, _ = self.record(good + event('pass'), ['TestGood', 'TestMissing'])
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(index['missing_selected_tests']['names'], ['TestMissing'])
+ other = event('run', 'TestOther') + event('pass', 'TestOther', elapsed=0.2)
+ result, _, index, _ = self.record(other + event('pass'), ['TestGood'])
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(index['missing_selected_tests']['names'], ['TestGood'])
+ self.assertEqual(index['unexpected_top_level_tests']['names'], ['TestOther'])
+
+ def test_unfinished_selected_test_retains_both_coverage_and_original_log(self):
+ stream = event('run', 'TestHung') + event('output', 'TestHung/private', 'private log\n')
+ result, _, index, raw = self.record(stream, ['TestHung'])
+ self.assertEqual(result.returncode, 1)
+ self.assertIn('private log', result.stdout)
+ self.assertEqual(index['unfinished_top_level_tests']['names'], ['TestHung'])
+ self.assertEqual(index['missing_selected_tests']['names'], ['TestHung'])
+ self.assertNotIn('private', raw)
+
+ def test_wrong_or_multiple_packages_cannot_satisfy_selection(self):
+ good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
+ for stream, package in [(good, 'other/private-package'),
+ (good.replace('example/internal/api', 'other/private-package', 1),
+ 'example/internal/api')]:
+ result, _, index, raw = self.record(stream, ['TestGood'], package)
+ self.assertEqual(result.returncode, 1)
+ self.assertGreater(index['unexpected_package_event_count'], 0)
+ self.assertNotIn('private-package', raw)
+
+ def test_invalid_event_shapes_are_non_passing_not_recorder_crashes(self):
+ for changed in [{'Action': 'invented'}, {'Package': None}, {'Test': []}, {'Output': 1}]:
+ value = {'Action': 'output', 'Package': 'example/internal/api'}
+ value.update(changed)
+ stream = (json.dumps(value) + '\n' + event('run', 'TestGood') +
+ event('pass', 'TestGood') + event('pass'))
+ result, _, index, _ = self.record(stream, ['TestGood'])
+ self.assertEqual(result.returncode, 1)
+ self.assertGreater(index['invalid_event_count'], 0)
+ self.assertNotIn('Traceback', result.stderr)
+
+ def test_go_build_diagnostics_remain_visible_without_test_package_identity(self):
+ good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
+ for action, code in [('build-output', 0), ('build-fail', 1)]:
+ build = json.dumps({'Action': action, 'ImportPath': 'private/dependency',
+ 'Output': 'retained build diagnostic\n'}) + '\n'
+ result, _, index, raw = self.record(build + good, ['TestGood'])
+ self.assertEqual(result.returncode, code)
+ self.assertIn('retained build diagnostic', result.stdout)
+ self.assertEqual(index['package_identity_count'], 1)
+ self.assertNotIn('private/dependency', raw)
+ self.assertNotIn('diagnostic', raw)
+
+ def test_coverage_names_stay_bounded_without_rejecting_unicode_source(self):
+ missing = [f'TestMissing{i:04d}' for i in range(301)]
+ missing.extend(['TestÜnicode', 'Test' + 'A' * 257])
+ stream = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
+ result, _, index, raw = self.record(stream, ['TestGood'] + missing)
+ self.assertEqual(result.returncode, 1)
+ observation = index['missing_selected_tests']
+ self.assertEqual(observation['observed_count'], 303)
+ self.assertEqual(len(observation['names']), 256)
+ self.assertEqual(observation['omitted_count'], 47)
+ self.assertNotIn('Ünicode', raw)
+ result, _, _, _ = self.record(
+ event('run', 'TestÜnicode') + event('pass', 'TestÜnicode') + event('pass'),
+ ['TestÜnicode'])
+ self.assertEqual(result.returncode, 0)
+
+ def test_invalid_selection_is_rejected_before_observation(self):
+ with tempfile.TemporaryDirectory() as directory:
+ seconds = Path(directory) / 'api.txt'
+ selected = Path(directory) / 'api.selected'
+ for names in ['', 'TestA\nTestA\n', 'TestA/subtest\n', 'TestBad-name\n']:
+ selected.write_text(names)
+ result = subprocess.run([sys.executable, str(RECORDER), str(seconds),
+ str(selected), 'example/internal/api'],
+ input='', text=True, capture_output=True, timeout=10)
+ self.assertEqual(result.returncode, 2)
+ self.assertFalse(seconds.exists())
+ selected.unlink()
+ result = subprocess.run([sys.executable, str(RECORDER), str(seconds),
+ str(selected), 'example/internal/api'],
+ input='', text=True, capture_output=True, timeout=10)
+ self.assertEqual(result.returncode, 2)
+ self.assertIn('unavailable', result.stderr)
+
+ @unittest.skipUnless(shutil.which('go'), 'real Go JSON requires the declared compiler')
+ def test_real_race_json_pass_skip_fuzz_example_failure_and_missing_coverage(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ package = 'example.invalid/shardfixture'
+ (root / 'go.mod').write_text(f'module {package}\n\ngo 1.24\n')
+ (root / 'fixture_test.go').write_text('''package shardfixture
+import ("fmt"; "testing")
+type Fixture struct{}
+func TestGood(t *testing.T) { t.Log("passing fixture output") }
+func TestSkip(t *testing.T) { t.Skip("explicit native fixture exclusion") }
+func TestFailure(t *testing.T) { t.Fatal("retained fixture failure") }
+func FuzzSeed(f *testing.F) { f.Add(1); f.Fuzz(func(t *testing.T, n int) {}) }
+func ExampleFixture() { fmt.Println("fixture")
+// Output: fixture
+}
+''')
+ env = {**os.environ, 'GOWORK': 'off', 'GOPROXY': 'off', 'GOTOOLCHAIN': 'local'}
+ listing = subprocess.run(['go', 'test', '-race', '-list', '.', '.'],
+ cwd=root, env=env, text=True, capture_output=True, timeout=120)
+ self.assertEqual(listing.returncode, 0, listing.stderr)
+ names = [name for name in listing.stdout.splitlines()
+ if name.startswith(('Test', 'Fuzz', 'Example'))]
+ selected = [name for name in names if name != 'TestFailure']
+ self.assertEqual(set(selected), {'TestGood', 'TestSkip', 'FuzzSeed', 'ExampleFixture'})
+ command = ['go', 'test', '-race', '-count=1', '-timeout', '30s', '-json']
+ # Exercise the same shorter-side -skip path used by large CI shards.
+ passing = subprocess.run(command + ['-skip', '^TestFailure$', '.'],
+ cwd=root, env=env, text=True, capture_output=True, timeout=120)
+ self.assertEqual(passing.returncode, 0, passing.stderr)
+ result, _, index, _ = self.record(passing.stdout, selected, package)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(index['completed_top_level_test_count'], 4)
+ self.assertNotIn('passing fixture output', result.stdout)
+
+ # Real Go exits zero while selecting fewer tests than the source
+ # list expects. Neither pipefail nor a package pass detects that.
+ omitted = subprocess.run(command + ['-run', '^TestGood$', '.'],
+ cwd=root, env=env, text=True, capture_output=True, timeout=120)
+ self.assertEqual(omitted.returncode, 0, omitted.stderr)
+ result, _, index, _ = self.record(omitted.stdout, selected, package)
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(index['missing_selected_tests']['observed_count'], 3)
+
+ # Completed tests without the final package event are not a pass.
+ events = [json.loads(line) for line in passing.stdout.splitlines()]
+ truncated = ''.join(json.dumps(value) + '\n' for value in events
+ if not (value.get('Action') == 'pass' and 'Test' not in value))
+ result, _, index, _ = self.record(truncated, selected, package)
+ self.assertEqual(result.returncode, 1)
+ self.assertIsNone(index['package_terminal_action'])
+
+ failing = subprocess.run(command + ['-run', '^TestFailure$', '.'],
+ cwd=root, env=env, text=True, capture_output=True, timeout=120)
+ self.assertEqual(failing.returncode, 1, failing.stderr)
+ result, _, index, _ = self.record(failing.stdout, ['TestFailure'], package)
+ self.assertEqual(result.returncode, 1)
+ self.assertIn('retained fixture failure', result.stdout)
+ self.assertEqual(index['failed_top_level_tests']['names'], ['TestFailure'])
+ self.assertEqual(index['missing_selected_tests']['observed_count'], 0)
+ print('Real Go race JSON: complete pass/skip/fuzz/example accepted; '
+ 'producer-zero omission, truncated completion and actual failure rejected.')
if __name__ == '__main__':