Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Retain bounded API shard skip observations

pulse-triage[bot]93459d762e17
diff --git a/.github/scripts/record-internal-api-test-seconds.py b/.github/scripts/record-internal-api-test-seconds.py
index b5e8c66ada..a640e9b934 100755
--- a/.github/scripts/record-internal-api-test-seconds.py
+++ b/.github/scripts/record-internal-api-test-seconds.py
@@ -28,8 +28,11 @@ The recorder also requires one completed package and complete, non-repeated
top-level test executions. CI supplies its source-derived selected test list
and package identity: every selected test must run and finish exactly once,
with no other test or package substituted. Individual test skips are explicit
-completed executions, not missing coverage. Empty, malformed and truncated
-streams fail closed; their original observations remain in the sidecar.
+completed executions, not missing coverage. The sidecar distinguishes fully
+skipped top-level tests from parents with skipped subtests, exporting only
+bounded source identifiers and aggregate child-event counts, never skip reasons
+or subtest labels. Empty, malformed and truncated streams fail closed; their
+original observations remain in the sidecar.
The exit status is non-zero when any test or package failed, but the caller
must still run with `set -o pipefail` so a `go test` failure that produced no
@@ -91,6 +94,9 @@ def main() -> int:
package_seconds = None
failed = False
failed_tests: set[str] = set()
+ skipped_tests: set[str] = set()
+ parents_with_skipped_subtests: set[str] = set()
+ skipped_subtest_events = 0
package_terminal_action = None
package_terminal_count = 0
non_json_lines = 0
@@ -164,6 +170,17 @@ def main() -> int:
package_seconds = float(event["Elapsed"])
continue
+ # Retain explicit skip observations even if the package later fails
+ # or never finishes. A passing parent can contain unavailable native
+ # cases, so top-level completion alone is not exercised-case evidence.
+ # Count child events without ever retaining their parameterised labels.
+ if action == "skip":
+ if test == top:
+ skipped_tests.add(top)
+ else:
+ parents_with_skipped_subtests.add(top)
+ skipped_subtest_events += 1
+
if action == "run" and test == top:
if top in started:
invalid_events += 1
@@ -218,6 +235,9 @@ def main() -> int:
"recorder_exit_code": 1 if failed else 0,
"failed_top_level_tests": bounded_identifiers(failed_tests),
"unfinished_top_level_tests": bounded_identifiers(running),
+ "skipped_top_level_tests": bounded_identifiers(skipped_tests),
+ "top_level_tests_with_skipped_subtests": bounded_identifiers(parents_with_skipped_subtests),
+ "skipped_subtest_event_count": skipped_subtest_events,
# Only the final package action matters; retain contrary earlier
# actions as a count instead of exporting an unbounded event stream.
"package_terminal_action": package_terminal_action,
diff --git a/scripts/tests/test_api_failure_index.py b/scripts/tests/test_api_failure_index.py
index 07cfc6ceb0..e3db3b3263 100644
--- a/scripts/tests/test_api_failure_index.py
+++ b/scripts/tests/test_api_failure_index.py
@@ -201,6 +201,69 @@ class APIFailureIndexTest(unittest.TestCase):
self.assertEqual(index['completed_top_level_test_count'], 2)
self.assertEqual(index['missing_selected_tests']['observed_count'], 0)
self.assertEqual(index['unexpected_top_level_tests']['observed_count'], 0)
+ self.assertIn('skipped_top_level_tests', index)
+ self.assertEqual(index['skipped_top_level_tests'],
+ {'names': ['TestNative'], 'observed_count': 1, 'omitted_count': 0})
+ self.assertEqual(index['top_level_tests_with_skipped_subtests']['observed_count'], 0)
+ self.assertEqual(index['skipped_subtest_event_count'], 0)
+
+ def test_skipped_subtests_do_not_disappear_under_a_passing_parent(self):
+ private = '[email protected]/secret-case'
+ stream = (event('run', 'TestPartial') +
+ event('run', 'TestPartial/' + private) +
+ event('output', 'TestPartial/' + private, 'private skip reason\n') +
+ event('skip', 'TestPartial/' + private) +
+ event('run', 'TestPartial/another-private-case') +
+ event('skip', 'TestPartial/another-private-case') +
+ event('pass', 'TestPartial', elapsed=0.5) + event('pass', elapsed=0.6))
+ result, seconds, index, raw = self.record(stream, ['TestPartial'])
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, '')
+ self.assertEqual(seconds, '# package-seconds 0.60\nTestPartial 0.50\n')
+ self.assertIn('top_level_tests_with_skipped_subtests', index)
+ self.assertEqual(index['top_level_tests_with_skipped_subtests'],
+ {'names': ['TestPartial'], 'observed_count': 1, 'omitted_count': 0})
+ self.assertEqual(index['skipped_top_level_tests']['observed_count'], 0)
+ self.assertEqual(index['skipped_subtest_event_count'], 2)
+ for hidden in [private, 'another-private-case', 'private skip reason']:
+ self.assertNotIn(hidden, raw)
+
+ def test_skip_observations_survive_failure_and_missing_completion(self):
+ partial = (event('run', 'TestPartial') +
+ event('skip', 'TestPartial/private-case') +
+ event('run', 'TestUnavailable') + event('skip', 'TestUnavailable'))
+ for terminal in ['', event('fail', 'TestPartial') + event('fail')]:
+ with self.subTest(completed=bool(terminal)):
+ result, _, index, raw = self.record(
+ partial + terminal, ['TestPartial', 'TestUnavailable'])
+ self.assertEqual(result.returncode, 1)
+ self.assertIn('skipped_top_level_tests', index)
+ self.assertEqual(index['skipped_top_level_tests']['names'], ['TestUnavailable'])
+ self.assertEqual(index['top_level_tests_with_skipped_subtests']['names'],
+ ['TestPartial'])
+ self.assertEqual(index['skipped_subtest_event_count'], 1)
+ self.assertNotIn('private-case', raw)
+ field = 'failed_top_level_tests' if terminal else 'unfinished_top_level_tests'
+ self.assertEqual(index[field]['names'], ['TestPartial'])
+
+ def test_skip_identifiers_are_bounded_and_private_labels_are_not_exported(self):
+ names = [f'TestSkip{i:04d}' for i in range(300, -1, -1)]
+ names += ['TestÜnicode', 'Test' + 'A' * 257, 'not-a-source-test']
+ stream = ''.join(event('run', name) + event('skip', name + '/private-child') +
+ event('skip', name) for name in names) + event('pass')
+ result, _, index, raw = self.record(stream)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ for field in ['skipped_top_level_tests', 'top_level_tests_with_skipped_subtests']:
+ self.assertIn(field, index)
+ observed = index[field]
+ self.assertEqual(observed['observed_count'], 304)
+ self.assertEqual(len(observed['names']), 256)
+ self.assertEqual(observed['omitted_count'], 48)
+ self.assertEqual(observed['names'], sorted(set(observed['names'])))
+ self.assertEqual(index['skipped_subtest_event_count'], 304)
+ for hidden in ['private-child', 'Ünicode', 'not-a-source-test', 'A' * 257]:
+ self.assertNotIn(hidden, raw)
+ self.assertLess(len(raw.encode()), 150_000)
def test_successful_producer_cannot_substitute_incomplete_execution(self):
good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)
@@ -312,6 +375,10 @@ import ("fmt"; "testing")
type Fixture struct{}
func TestGood(t *testing.T) { t.Log("passing fixture output") }
func TestSkip(t *testing.T) { t.Skip("explicit native fixture exclusion") }
+func TestPartial(t *testing.T) {
+ t.Run("working", func(t *testing.T) {})
+ t.Run("private-child", func(t *testing.T) { t.Skip("private native fixture reason") })
+}
func TestFailure(t *testing.T) { t.Fatal("retained fixture failure") }
func FuzzSeed(f *testing.F) { f.Add(1); f.Fuzz(func(t *testing.T, n int) {}) }
func ExampleFixture() { fmt.Println("fixture")
@@ -325,7 +392,7 @@ func ExampleFixture() { fmt.Println("fixture")
names = [name for name in listing.stdout.splitlines()
if name.startswith(('Test', 'Fuzz', 'Example'))]
selected = [name for name in names if name != 'TestFailure']
- self.assertEqual(set(selected), {'TestGood', 'TestSkip', 'FuzzSeed', 'ExampleFixture'})
+ self.assertEqual(set(selected), {'TestGood', 'TestSkip', 'TestPartial', 'FuzzSeed', 'ExampleFixture'})
command = ['go', 'test', '-race', '-count=1', '-timeout', '30s', '-json']
# Exercise the same shorter-side -skip path used by large CI shards.
passing = subprocess.run(command + ['-skip', '^TestFailure$', '.'],
@@ -333,7 +400,13 @@ func ExampleFixture() { fmt.Println("fixture")
self.assertEqual(passing.returncode, 0, passing.stderr)
result, _, index, _ = self.record(passing.stdout, selected, package)
self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(index['completed_top_level_test_count'], 4)
+ self.assertEqual(index['completed_top_level_test_count'], 5)
+ self.assertIn('skipped_top_level_tests', index)
+ self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkip'])
+ self.assertEqual(index['top_level_tests_with_skipped_subtests']['names'], ['TestPartial'])
+ self.assertEqual(index['skipped_subtest_event_count'], 1)
+ self.assertNotIn('private-child', json.dumps(index))
+ self.assertNotIn('private native fixture reason', json.dumps(index))
self.assertNotIn('passing fixture output', result.stdout)
# Real Go exits zero while selecting fewer tests than the source
@@ -343,7 +416,7 @@ func ExampleFixture() { fmt.Println("fixture")
self.assertEqual(omitted.returncode, 0, omitted.stderr)
result, _, index, _ = self.record(omitted.stdout, selected, package)
self.assertEqual(result.returncode, 1)
- self.assertEqual(index['missing_selected_tests']['observed_count'], 3)
+ self.assertEqual(index['missing_selected_tests']['observed_count'], 4)
# Completed tests without the final package event are not a pass.
events = [json.loads(line) for line in passing.stdout.splitlines()]