Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): executed-case floor, FULL tier for engine crates, required support
Richie McIlroy3146aa3dd50b
diff --git a/.github/actions/engine-verify-setup/action.yml b/.github/actions/engine-verify-setup/action.ymlindex 24295974c..b14586b33 100644--- a/.github/actions/engine-verify-setup/action.yml+++ b/.github/actions/engine-verify-setup/action.yml@@ -48,7 +48,7 @@ runs:- name: cargo-nextestuses: taiki-e/install-action@v2with:- tool: [email protected]+ tool: [email protected]# One cache entry per OS per day (saved by main, nightly and the CI dev# branches), restored by PRs. Unlike swatinem/rust-cache it includes the@@ -129,12 +129,13 @@ runs:run: echo "CAP_VERIFY_FIXTURE_DIR=${{ runner.temp }}/verify-fixtures" >> "$GITHUB_ENV"# Fixtures are generated by code (including the real-pipeline Studio and- # Instant recordings, ~1 min of real-time capture); the key is the whole- # verify crate source plus the native deps. Exact key only: a fixture made+ # Instant recordings, ~1 min of real-time capture); the key is every+ # crate's source and build script, the vendored crates (wgpu-hal, tao),+ # the workspace manifest, lockfile and toolchain, plus the native deps. Exact key only: a fixture made# by an older generator is never reused.- name: Restore fixturesid: fixturesuses: actions/cache/restore@v4with:path: ${{ runner.temp }}/verify-fixtures- key: verify-fixtures-v1-${{ runner.os }}-${{ runner.environment }}-${{ inputs.tier }}-${{ hashFiles('crates/**/src/**', 'crates/**/Cargo.toml', 'Cargo.lock', 'scripts/setup.js') }}+ key: verify-fixtures-v1-${{ runner.os }}-${{ runner.environment }}-${{ inputs.tier }}-${{ hashFiles('crates/**/src/**', 'crates/**/build.rs', 'crates/**/Cargo.toml', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'vendor/**', 'scripts/setup.js') }}diff --git a/.github/workflows/engine-verify-os.yml b/.github/workflows/engine-verify-os.ymlindex 1bb925b9d..95da06162 100644--- a/.github/workflows/engine-verify-os.yml+++ b/.github/workflows/engine-verify-os.yml@@ -57,6 +57,11 @@ env:CAP_VERIFY_NEXTEST_PROFILE: ciCAP_VERIFY_LOCKED: "1"CAP_VERIFY_CARGO_CONFIG: ${{ inputs.cargo-config }}+ # A case this runner cannot run fails unless scripts/verify/suites.json+ # allows it as unsupported on this platform (with the reason).+ CAP_VERIFY_REQUIRE_SUPPORTED: "1"+ # Machine class for host-scoped expectations (crates/verify-expect hosts).+ CAP_VERIFY_HOST: ${{ startsWith(inputs.runner, 'blacksmith-') && 'blacksmith' || 'github-hosted' }}jobs:build:@@ -159,6 +164,13 @@ jobs:target/ci-verify/key: ${{ steps.setup.outputs.build-cache-key }}+ # The retry and flaky reruns read test names back from Python (CRLF on+ # Windows): check that such a name selects exactly its test.+ - name: Rerun selection self-check+ if: ${{ !inputs.archive }}+ shell: bash+ run: scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --check-rerun-selection+- name: Download archiveif: inputs.archiveuses: actions/download-artifact@v4diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.ymlindex bbce3c29d..ece6d8f89 100644--- a/.github/workflows/engine-verify.yml+++ b/.github/workflows/engine-verify.yml@@ -4,8 +4,10 @@ name: Engine verify# and captions engines (crates/verify, crates/verify-expect, the decoder model# fuzzer in cap-rendering). See crates/verify/README.md.#-# Tiers: FAST on every PR push; FULL with the `verify-full` label-# (engine-verify-label.yml), in the merge queue and nightly. Only suites whose+# Tiers: FULL for every pull request that changes an engine crate+# (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the+# `verify-full` label (engine-verify-label.yml), in the merge queue and+# nightly; FAST for other pull requests. Only suites whose# code a change can reach run (scripts/verify/plan.py maps changed paths to# suites through the workspace dependency graph), and only on the platforms# where they test platform code. The "Engine verify" job is the one to require.@@ -71,6 +73,7 @@ jobs:run: ${{ steps.plan.outputs.run }}tier: ${{ steps.escalated.outputs.tier }}escalate: ${{ steps.risk.outputs.escalate }}+ engine: ${{ steps.risk.outputs.engine }}matrix: ${{ steps.plan.outputs.matrix }}save-cache: ${{ steps.tier.outputs.save-cache }}steps:@@ -162,13 +165,14 @@ jobs:env:TIER: ${{ steps.tier.outputs.tier }}ESCALATE: ${{ steps.risk.outputs.escalate }}+ ENGINE: ${{ steps.risk.outputs.engine }}EVENT: ${{ github.event_name }}BASE: ${{ inputs.base }}run: |tier="$TIER"- if [[ "$ESCALATE" == true && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; then+ if [[ ( "$ESCALATE" == true || "$ENGINE" == true ) && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; thentier=full- echo "High-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY"+ echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY"fiecho "tier=$tier" >> "$GITHUB_OUTPUT"@@ -265,6 +269,7 @@ jobs:RUN: ${{ needs.plan.outputs.run }}VERIFY: ${{ needs.verify.result }}TIER: ${{ needs.plan.outputs.tier }}+ MATRIX: ${{ needs.plan.outputs.matrix }}run: |if [[ "$PLAN" != success ]]; thenecho "planning failed" >&2@@ -287,4 +292,14 @@ jobs:echo "engine verification $VERIFY ($TIER tier): see the platform jobs" >&2exit 1fi+ # Executed-case floor: every planned suite ran at least its known+ # number of cases on every platform, and every named test ran.+ floor=0+ python3 scripts/verify/floor.py --matrix "$MATRIX" --tier "$TIER" "$RUNNER_TEMP/reports" > "$RUNNER_TEMP/floor.txt" || floor=$?+ cat "$RUNNER_TEMP/floor.txt"+ { echo "### Executed cases"; echo '```text'; cat "$RUNNER_TEMP/floor.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY"+ if [[ $floor != 0 ]]; then+ echo "::error::fewer cases or tests ran than planned (see Executed cases)"+ exit 1+ fiecho "engine verification passed ($TIER tier)"diff --git a/scripts/verify/floor.py b/scripts/verify/floor.pynew file mode 100644index 000000000..498fb5d54--- /dev/null+++ b/scripts/verify/floor.py@@ -0,0 +1,130 @@+#!/usr/bin/env python3+"""Executed-case floor for an engine-verify run.++ floor.py --matrix JSON --tier fast|full REPORTS_DIR+ floor.py --recording --runners JSON --tier fast|full REPORTS_DIR++REPORTS_DIR holds the downloaded report artifacts+(verify-report-<platform>-<tier>-<shard>/). For every platform and suite the+plan ran, the distinct case ids whose id starts with the suite's `cases.prefix`+(scripts/verify/suites.json) must reach the floor for this tier and platform,+and a suite with an exact `test` name must appear in the run's junit. A suite+that silently shrinks (a renamed test, a skip glob or catalog edit that drops+cases, a shard that never ran) fails the gate instead of passing vacuously.+With --recording, REPORTS_DIR holds the real-capture artifacts+(verify-recording-<runner>/verify-report): every runner must have recorded at+least suites.json's `recording.cases` floor of distinct real-capture cases.+Prints one line per check; exit status 1 when any check fails.+"""++import argparse+import glob+import json+import os+import sys+import xml.etree.ElementTree as ET++ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".."))+++def platform_reports(reports, platform, tier):+ return sorted(glob.glob(os.path.join(reports, f"verify-report-{platform}-{tier}-*")))+++def case_ids(dirs):+ ids = set()+ for d in dirs:+ for path in glob.glob(os.path.join(d, "**", "*.jsonl"), recursive=True):+ rel = os.path.relpath(path, d).replace(os.sep, "/")+ # Retry and flaky-rerun records repeat cases of the main pass.+ if rel.startswith("retry/") or ".rerun" in rel:+ continue+ with open(path, encoding="utf-8") as f:+ for line in f:+ try:+ record = json.loads(line)+ except json.JSONDecodeError:+ continue+ if record.get("case"):+ ids.add(record["case"])+ return ids+++def junit_tests(dirs):+ names = set()+ for d in dirs:+ path = os.path.join(d, "junit.xml")+ if not os.path.exists(path):+ continue+ for case in ET.parse(path).getroot().iter("testcase"):+ names.add(case.get("name", ""))+ return names+++def check(config, matrix, tier, reports):+ lines, failed = [], 0+ suites = {s["name"]: s for s in config["suites"]}+ for entry in matrix.get("include", []):+ platform = entry["platform"]+ dirs = platform_reports(reports, platform, tier)+ expected_shards = int(entry.get("shards", 1))+ if len(dirs) < expected_shards:+ failed += 1+ lines.append(f"FAIL {platform}: {len(dirs)} of {expected_shards} shard reports present")+ ids = case_ids(dirs)+ tests = junit_tests(dirs)+ for name in [n for n in entry["suites"].split(",") if n]:+ suite = suites[name]+ floor = suite.get("cases", {}).get(tier, {}).get(platform)+ if floor is not None:+ prefix = suite["cases"]["prefix"]+ ran = sum(1 for i in ids if i.startswith(prefix))+ ok = ran >= floor+ failed += not ok+ lines.append(+ f"{'ok ' if ok else 'FAIL'} {platform} {name}: {ran} cases ran (floor {floor}, prefix {prefix})"+ )+ if suite.get("test"):+ ok = suite["test"] in tests+ failed += not ok+ lines.append(+ f"{'ok ' if ok else 'FAIL'} {platform} {name}: test {suite['test']} {'ran' if ok else 'is missing from the junit report'}"+ )+ return lines, failed+++def check_recording(config, runners, tier, reports):+ lines, failed = [], 0+ floor = config.get("recording", {}).get("cases", {}).get(tier, 0)+ for runner in runners:+ d = os.path.join(reports, f"verify-recording-{runner}")+ ids = {+ i for i in case_ids([d])+ if i.startswith("recording/") and "(self-check" not in i+ }+ ok = os.path.isdir(d) and len(ids) >= floor+ failed += not ok+ lines.append(f"{'ok ' if ok else 'FAIL'} {runner}: {len(ids)} real-capture cases recorded (floor {floor})")+ return lines, failed+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument("--matrix")+ parser.add_argument("--recording", action="store_true")+ parser.add_argument("--runners")+ parser.add_argument("--tier", required=True)+ parser.add_argument("reports")+ args = parser.parse_args()+ with open(os.path.join(ROOT, "scripts", "verify", "suites.json")) as f:+ config = json.load(f)+ if args.recording:+ lines, failed = check_recording(config, json.loads(args.runners), args.tier, args.reports)+ else:+ lines, failed = check(config, json.loads(args.matrix), args.tier, args.reports)+ print("\n".join(lines))+ sys.exit(1 if failed else 0)+++if __name__ == "__main__":+ main()diff --git a/scripts/verify/plan.py b/scripts/verify/plan.pyindex 104487cd2..91618fa9d 100755--- a/scripts/verify/plan.py+++ b/scripts/verify/plan.py@@ -9,6 +9,9 @@Maps changed paths to suites through the workspace dependency graph(`cargo metadata`) and prints GitHub step outputs: run, reason,suites, matrix.+ plan.py recording [--files FILE | --all]+ run=, reason= for the real-capture workflow: whether a change reaches+ the recorder's or the stimulus's dependency closure or the harness.plan.py risk --files F [--summary FILE] [--output FILE]changed files -> high-risk regions and guards (scripts/verify/guards.toml):escalate=, regions=, guards= lines plus ::warning annotations.@@ -35,6 +38,33 @@ def load_guards():return tomllib.load(f)+# Engine crates: a pull request that changes any of them runs the FULL tier+# (with Blacksmith runners there is no queue to save).+ENGINE_CRATES = [+ "crates/rendering/**",+ "crates/editor/**",+ "crates/export/**",+ "crates/recording/**",+ "crates/enc-*/**",+ "crates/audio/**",+ "crates/video-decode/**",+ "crates/project/**",+ "crates/scap-*/**",+ "crates/camera*/**",+ "crates/media/**",+ "crates/media-info/**",+ "crates/frame-converter/**",+ "crates/gpu-converters/**",+ "crates/cursor-capture/**",+ "crates/timestamp/**",+ "crates/cap-muxer/**",+]+++def engine_files(files):+ return sorted({p for p in files for pattern in ENGINE_CRATES if match(p, pattern)})++def risk(files, guards=None):"""(regions touched, guards whose files are touched) for changed files."""guards = guards or load_guards()@@ -81,7 +111,10 @@ def load_config():def match(path, pattern):if pattern.endswith("/**"):- return path.startswith(pattern[:-2])+ prefix = pattern[:-2]+ if any(c in prefix for c in "*?["):+ return fnmatch.fnmatchcase(path, prefix + "*")+ return path.startswith(prefix)return fnmatch.fnmatchcase(path, pattern)@@ -154,11 +187,15 @@ def cmd_select(args):def cmd_env(args):"""Shell `export` lines for the env of the selected suites (suites.json- `env`: {"KEY": "value"} or {"KEY": {"<platform>": "value"}}). Variables- already set in the environment win."""+ `env`: {"KEY": "value"} or {"KEY": {"<platform>": "value"}}) and the+ platform's unsupported-case allowlist. Variables already set in the+ environment win."""import shlexconfig = load_config()names = [n for n in (args.suites or "").split(",") if n]+ allow = ",".join(config.get("unsupported", {}).get(args.platform, {}).keys())+ if allow and "CAP_VERIFY_UNSUPPORTED_ALLOW" not in os.environ:+ print(f"export CAP_VERIFY_UNSUPPORTED_ALLOW={shlex.quote(allow)}")for suite in select(config, args.tier, args.platform, names or None, args.local):for key, value in suite.get("env", {}).items():if isinstance(value, dict):@@ -311,6 +348,56 @@ def cmd_changes(args):print(f"{key}={value}")+# Real capture (crates/verify/tests/recording.rs): the recorder and the+# stimulus, their whole dependency closure, and the harness itself.+RECORDING = {+ "roots": ["cap-recording", "cap-verify-stimulus"],+ "own": [+ "crates/verify/tests/recording.rs",+ "crates/verify/src/**",+ "crates/verify/Cargo.toml",+ "crates/verify/expectations.toml",+ "crates/verify-expect/**",+ "crates/verify-stimulus/**",+ ".github/workflows/verify-recording.yml",+ ],+}+++def recording_affected(config, files, metadata):+ """(run, reason) for the real-capture workflow."""+ for path in files:+ for pattern in config["always"] + RECORDING["own"]:+ if match(path, pattern):+ return True, f"{path} affects real capture"+ dirs, deps = workspace_graph(metadata)+ reach = set().union(*(closure(r, deps) for r in RECORDING["roots"]))+ for path in files:+ # Other suites' test files share the cap-verify package but not the+ # recording test binary.+ if path.startswith("crates/verify/tests/"):+ continue+ package = owning_package(path, dirs)+ if package in reach:+ return True, f"{path} ({package}) is in the recorder's dependency closure"+ return False, "no recording code changed"+++def cmd_recording(args):+ config = load_config()+ if args.all:+ run, reason = True, "full run"+ else:+ with open(args.files) as f:+ files = [line.strip() for line in f if line.strip()]+ if len(files) >= args.max_files:+ run, reason = True, f"{len(files)} changed files (list may be truncated)"+ else:+ run, reason = recording_affected(config, files, load_metadata(args.metadata))+ print(f"run={'true' if run else 'false'}")+ print(f"reason={reason}")++def cmd_risk(args):with open(args.files) as f:files = [line.strip() for line in f if line.strip()]@@ -321,8 +408,12 @@ def cmd_risk(args):f.write(md + "\n")for note in notes:print(note)+ engine = engine_files(files)+ if engine:+ print(f"::notice title=Engine crates changed::FULL tier for {', '.join(engine[:6])}{' ...' if len(engine) > 6 else ''}")outputs = [f"escalate={'true' if regions or touched else 'false'}",+ f"engine={'true' if engine else 'false'}",f"regions={','.join(name for name, _ in regions)}",f"guards={','.join(g['id'] for g in touched)}",]@@ -374,6 +465,27 @@ def cmd_test(_args):ok = got == expectedfailed += not okprint(f"{'ok ' if ok else 'BAD'} risk {files} -> {got}")+ for files, expected in [+ (["crates/rendering/src/lib.rs"], True),+ (["crates/enc-ffmpeg/src/lib.rs"], True),+ (["crates/camera-ffmpeg/src/lib.rs"], True),+ (["apps/desktop-gpui/src/main.rs", "crates/verify/src/oracle.rs"], False),+ ]:+ ok = bool(engine_files(files)) == expected+ failed += not ok+ print(f"{'ok ' if ok else 'BAD'} engine {files} -> {bool(engine_files(files))}")+ for files, expected in [+ (["crates/frame-converter/src/lib.rs"], True),+ (["crates/recording/src/studio_recording.rs"], True),+ (["crates/verify/tests/recording.rs"], True),+ (["vendor/wgpu-hal/src/lib.rs"], True),+ (["Cargo.toml"], True),+ (["crates/verify/tests/export.rs", "apps/web/app/page.tsx"], False),+ ]:+ got = recording_affected(config, files, metadata)[0]+ ok = got == expected+ failed += not ok+ print(f"{'ok ' if ok else 'BAD'} recording {files} -> {got}")rendering_only = affected(config, ["crates/rendering/src/decoder/model/mod.rs"], metadata, "fast")[0]if "decoder-model-ffmpeg" not in rendering_only:failed += 1@@ -404,6 +516,12 @@ def main():p.add_argument("--max-files", type=int, default=3000)p.add_argument("--suites")p.set_defaults(func=cmd_changes)+ p = sub.add_parser("recording")+ p.add_argument("--files")+ p.add_argument("--all", action="store_true")+ p.add_argument("--metadata")+ p.add_argument("--max-files", type=int, default=3000)+ p.set_defaults(func=cmd_recording)p = sub.add_parser("risk")p.add_argument("--files", required=True)p.add_argument("--summary", help="append the markdown report to this file")diff --git a/scripts/verify/suites.json b/scripts/verify/suites.jsonindex bbf24b582..b1e11d671 100644--- a/scripts/verify/suites.json+++ b/scripts/verify/suites.json@@ -1,5 +1,5 @@{- "_doc": "Engine verification suites. Read by scripts/verify/plan.py (CI planning, change detection) and scripts/verify/run.sh (local and CI runs). Fields: cargo = the cargo target selection that builds the suite; test = optional exact test name inside that binary; platforms = where each tier runs it (platform-independent logic runs on linux only); roots = workspace packages whose dependency closure decides whether a change can affect the suite; own = files that only affect this suite (a change confined to them selects just this suite). skip = case-id globs left out of a tier on a platform (CAP_VERIFY_SKIP), to keep the FAST tier within its time budget; the FULL tier runs them. Exception: hosted Windows runners have no D3D11 video device, so every Windows case decodes with FFmpeg (the code Linux runs) and renders on the WARP software rasterizer; there the 120 s jitter-camera export and playback and the 3000-seek storms took 6 to more than 15 minutes each (killed at the 15 minute per-test limit in 4 of 5 full runs), so FULL leaves them to Linux and macOS.",+ "_doc": "Engine verification suites. Read by scripts/verify/plan.py (CI planning, change detection) and scripts/verify/run.sh (local and CI runs). Fields: cargo = the cargo target selection that builds the suite; test = optional exact test name inside that binary; platforms = where each tier runs it (platform-independent logic runs on linux only); roots = workspace packages whose dependency closure decides whether a change can affect the suite; own = files that only affect this suite (a change confined to them selects just this suite). skip = case-id globs left out of a tier on a platform (CAP_VERIFY_SKIP), to keep the FAST tier within its time budget; the FULL tier runs them. Exception: hosted Windows runners have no D3D11 video device, so every Windows case decodes with FFmpeg (the code Linux runs) and renders on the WARP software rasterizer; there the 120 s jitter-camera export and playback and the 3000-seek storms took 6 to more than 15 minutes each (killed at the 15 minute per-test limit in 4 of 5 full runs), so FULL leaves them to Linux and macOS. cases = the executed-case floor per tier and platform: the gate fails when fewer distinct cases whose id starts with prefix ran across all shards (scripts/verify/floor.py). unsupported = per-platform case globs that may report unsupported (with the reason); CI sets CAP_VERIFY_REQUIRE_SUPPORTED=1, so any other unsupported case fails.","suites": [{"name": "verify-self",@@ -27,16 +27,24 @@"skip": {"fast": {"macos": ["export/*/ffmpeg", "export/backward-hold/*/mp430/*"],- "windows": [- "export/*/mp460/*",- "export/backward-hold/*",- "export/jitter-camera-*",- "export/*/uhd-*"- ]+ "windows": ["export/*/mp460/*", "export/backward-hold/*", "export/jitter-camera-*", "export/*/uhd-*"]},"full": {"windows": ["export/jitter-camera-120s/*"]}+ },+ "cases": {+ "prefix": "export/",+ "fast": {+ "linux": 42,+ "macos": 41,+ "windows": 43+ },+ "full": {+ "linux": 43,+ "macos": 69,+ "windows": 67+ }}},{@@ -57,6 +65,18 @@"full": {"windows": ["playback/jitter-camera-120s/*"]}+ },+ "cases": {+ "prefix": "playback/",+ "fast": {+ "linux": 19,+ "macos": 19+ },+ "full": {+ "linux": 19,+ "macos": 38,+ "windows": 36+ }}},{@@ -77,6 +97,18 @@"full": {"windows": ["seek/*/storm*"]}+ },+ "cases": {+ "prefix": "seek/",+ "fast": {+ "linux": 11,+ "macos": 11+ },+ "full": {+ "linux": 11,+ "macos": 22,+ "windows": 16+ }}},{@@ -100,6 +132,19 @@"windows": ["audio/*/mp4-plain30", "audio/*/gif30", "audio/*/mov30"],"macos": ["audio/*/mp4-plain30", "audio/*/gif30"]}+ },+ "cases": {+ "prefix": "audio/",+ "fast": {+ "linux": 56,+ "macos": 40,+ "windows": 39+ },+ "full": {+ "linux": 58,+ "macos": 58,+ "windows": 58+ }}},{@@ -112,7 +157,16 @@"full": ["linux"]},"roots": ["cap-verify"],- "own": ["crates/verify/tests/editing.rs", "crates/verify/tests/edits/**"]+ "own": ["crates/verify/tests/editing.rs", "crates/verify/tests/edits/**"],+ "cases": {+ "prefix": "editing/",+ "fast": {+ "linux": 22+ },+ "full": {+ "linux": 78+ }+ }},{"name": "captions",@@ -124,7 +178,16 @@"full": ["linux"]},"roots": ["cap-verify"],- "own": ["crates/verify/tests/captions.rs"]+ "own": ["crates/verify/tests/captions.rs"],+ "cases": {+ "prefix": "captions/",+ "fast": {+ "linux": 19+ },+ "full": {+ "linux": 19+ }+ }},{"name": "decoder-model-ffmpeg",@@ -137,7 +200,16 @@"full": ["linux"]},"roots": ["cap-rendering"],- "own": []+ "own": [],+ "cases": {+ "prefix": "decoder-model/ffmpeg/",+ "fast": {+ "linux": 29+ },+ "full": {+ "linux": 29+ }+ }},{"name": "decoder-model-avf",@@ -150,7 +222,16 @@"full": ["macos"]},"roots": ["cap-rendering"],- "own": []+ "own": [],+ "cases": {+ "prefix": "decoder-model/avf/",+ "fast": {+ "macos": 31+ },+ "full": {+ "macos": 31+ }+ }},{"name": "native-rendering",@@ -162,10 +243,16 @@"full": ["macos"]},"roots": ["cap-rendering"],- "own": [- "crates/rendering/tests/iosurface-source-lifetime.rs",- "crates/rendering/tests/expectations.toml"- ]+ "own": ["crates/rendering/tests/iosurface-source-lifetime.rs", "crates/rendering/tests/expectations.toml"],+ "cases": {+ "prefix": "native/",+ "fast": {+ "macos": 1+ },+ "full": {+ "macos": 1+ }+ }}],"always": [@@ -208,5 +295,17 @@},"archive": false}+ },+ "unsupported": {+ "windows": {+ "*/mf": "Media Foundation decoding needs a D3D11 device with video support; CI Windows VMs render on the WARP software adapter, which has none, so the product falls back to FFmpeg there. Media Foundation is not exercised by CI."+ }+ },+ "recording": {+ "_doc": "Real capture (verify-recording.yml): distinct case ids each runner must record per tier (crates/verify/tests/recording.rs specs; system-only reports unsupported when the stimulus is silent).",+ "cases": {+ "fast": 8,+ "full": 12+ }}}diff --git a/scripts/verify/summary.py b/scripts/verify/summary.pyindex 66cca3b89..a44bbba54 100755--- a/scripts/verify/summary.py+++ b/scripts/verify/summary.py@@ -22,6 +22,11 @@ def load_records(dirs):records = []for d in dirs:for path in sorted(glob.glob(os.path.join(d, "**", "*.jsonl"), recursive=True)):+ # Retry records repeat cases of the main pass (counted with the+ # retries below), so they are not cases of their own.+ rel = os.path.relpath(path, d).replace(os.sep, "/")+ if rel.startswith("retry/"):+ continuewith open(path, encoding="utf-8") as f:for line in f:line = line.strip()@@ -132,6 +137,21 @@ def render(records, title, junit, flaky, text, retries=None):f"{counts.get('fail', 0)} FAILED, {counts.get('xpass', 0)} unexpectedly passing, "f"{counts.get('unsupported', 0)} unsupported ({seconds:.0f} s of case time)")+ excused = {}+ for r in records:+ for kind in (r.get("excused") or {}).values():+ excused[kind] = excused.get(kind, 0) + 1+ if excused.get("flaky") or excused.get("flaky, starved"):+ head += (+ f"; expected failures excused by flaky entries: {excused.get('flaky', 0)}, "+ f"only under a proven machine stall: {excused.get('flaky, starved', 0)}"+ )+ second = [+ r for r in records+ if any(str(n).startswith("inconclusive, retrying") for n in r.get("notes") or [])+ ]+ if second:+ head += f"; {len(second)} real-capture case(s) settled on a second attempt after a proven input fault"unmeasured = [r for r in records if r.get("unobserved")]if unmeasured:head += f"; {len(unmeasured)} with classes not measured because an earlier check failed"