Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): executed-case floor, FULL tier for engine crates, required support

Richie McIlroy3146aa3dd50b
diff --git a/.github/actions/engine-verify-setup/action.yml b/.github/actions/engine-verify-setup/action.yml
index 24295974c..b14586b33 100644
--- a/.github/actions/engine-verify-setup/action.yml
+++ b/.github/actions/engine-verify-setup/action.yml
@@ -48,7 +48,7 @@ runs:
- name: cargo-nextest
uses: taiki-e/install-action@v2
with:
# One cache entry per OS per day (saved by main, nightly and the CI dev
# branches), restored by PRs. Unlike swatinem/rust-cache it includes the
@@ -129,12 +129,13 @@ runs:
run: echo "CAP_VERIFY_FIXTURE_DIR=${{ runner.temp }}/verify-fixtures" >> "$GITHUB_ENV"
# Fixtures are generated by code (including the real-pipeline Studio and
- # Instant recordings, ~1 min of real-time capture); the key is the whole
- # verify crate source plus the native deps. Exact key only: a fixture made
+ # Instant recordings, ~1 min of real-time capture); the key is every
+ # crate's source and build script, the vendored crates (wgpu-hal, tao),
+ # the workspace manifest, lockfile and toolchain, plus the native deps. Exact key only: a fixture made
# by an older generator is never reused.
- name: Restore fixtures
id: fixtures
uses: actions/cache/restore@v4
with:
path: ${{ runner.temp }}/verify-fixtures
- key: verify-fixtures-v1-${{ runner.os }}-${{ runner.environment }}-${{ inputs.tier }}-${{ hashFiles('crates/**/src/**', 'crates/**/Cargo.toml', 'Cargo.lock', 'scripts/setup.js') }}
+ key: verify-fixtures-v1-${{ runner.os }}-${{ runner.environment }}-${{ inputs.tier }}-${{ hashFiles('crates/**/src/**', 'crates/**/build.rs', 'crates/**/Cargo.toml', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'vendor/**', 'scripts/setup.js') }}
diff --git a/.github/workflows/engine-verify-os.yml b/.github/workflows/engine-verify-os.yml
index 1bb925b9d..95da06162 100644
--- a/.github/workflows/engine-verify-os.yml
+++ b/.github/workflows/engine-verify-os.yml
@@ -57,6 +57,11 @@ env:
CAP_VERIFY_NEXTEST_PROFILE: ci
CAP_VERIFY_LOCKED: "1"
CAP_VERIFY_CARGO_CONFIG: ${{ inputs.cargo-config }}
+ # A case this runner cannot run fails unless scripts/verify/suites.json
+ # allows it as unsupported on this platform (with the reason).
+ CAP_VERIFY_REQUIRE_SUPPORTED: "1"
+ # Machine class for host-scoped expectations (crates/verify-expect hosts).
+ CAP_VERIFY_HOST: ${{ startsWith(inputs.runner, 'blacksmith-') && 'blacksmith' || 'github-hosted' }}
jobs:
build:
@@ -159,6 +164,13 @@ jobs:
target/ci-verify/
key: ${{ steps.setup.outputs.build-cache-key }}
+ # The retry and flaky reruns read test names back from Python (CRLF on
+ # Windows): check that such a name selects exactly its test.
+ - name: Rerun selection self-check
+ if: ${{ !inputs.archive }}
+ shell: bash
+ run: scripts/verify/run.sh "${{ inputs.tier }}" --suites "${{ inputs.suites }}" --check-rerun-selection
+
- name: Download archive
if: inputs.archive
uses: actions/download-artifact@v4
diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.yml
index bbce3c29d..ece6d8f89 100644
--- a/.github/workflows/engine-verify.yml
+++ b/.github/workflows/engine-verify.yml
@@ -4,8 +4,10 @@ name: Engine verify
# and captions engines (crates/verify, crates/verify-expect, the decoder model
# fuzzer in cap-rendering). See crates/verify/README.md.
#
-# Tiers: FAST on every PR push; FULL with the `verify-full` label
-# (engine-verify-label.yml), in the merge queue and nightly. Only suites whose
+# Tiers: FULL for every pull request that changes an engine crate
+# (scripts/verify/plan.py ENGINE_CRATES) or a high-risk region, with the
+# `verify-full` label (engine-verify-label.yml), in the merge queue and
+# nightly; FAST for other pull requests. Only suites whose
# code a change can reach run (scripts/verify/plan.py maps changed paths to
# suites through the workspace dependency graph), and only on the platforms
# where they test platform code. The "Engine verify" job is the one to require.
@@ -71,6 +73,7 @@ jobs:
run: ${{ steps.plan.outputs.run }}
tier: ${{ steps.escalated.outputs.tier }}
escalate: ${{ steps.risk.outputs.escalate }}
+ engine: ${{ steps.risk.outputs.engine }}
matrix: ${{ steps.plan.outputs.matrix }}
save-cache: ${{ steps.tier.outputs.save-cache }}
steps:
@@ -162,13 +165,14 @@ jobs:
env:
TIER: ${{ steps.tier.outputs.tier }}
ESCALATE: ${{ steps.risk.outputs.escalate }}
+ ENGINE: ${{ steps.risk.outputs.engine }}
EVENT: ${{ github.event_name }}
BASE: ${{ inputs.base }}
run: |
tier="$TIER"
- if [[ "$ESCALATE" == true && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; then
+ if [[ ( "$ESCALATE" == true || "$ENGINE" == true ) && ( "$EVENT" == pull_request || -n "$BASE" ) && "$tier" == fast ]]; then
tier=full
- echo "High-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY"
+ echo "Engine or high-risk code changed: running the FULL tier." | tee -a "$GITHUB_STEP_SUMMARY"
fi
echo "tier=$tier" >> "$GITHUB_OUTPUT"
@@ -265,6 +269,7 @@ jobs:
RUN: ${{ needs.plan.outputs.run }}
VERIFY: ${{ needs.verify.result }}
TIER: ${{ needs.plan.outputs.tier }}
+ MATRIX: ${{ needs.plan.outputs.matrix }}
run: |
if [[ "$PLAN" != success ]]; then
echo "planning failed" >&2
@@ -287,4 +292,14 @@ jobs:
echo "engine verification $VERIFY ($TIER tier): see the platform jobs" >&2
exit 1
fi
+ # Executed-case floor: every planned suite ran at least its known
+ # number of cases on every platform, and every named test ran.
+ floor=0
+ python3 scripts/verify/floor.py --matrix "$MATRIX" --tier "$TIER" "$RUNNER_TEMP/reports" > "$RUNNER_TEMP/floor.txt" || floor=$?
+ cat "$RUNNER_TEMP/floor.txt"
+ { echo "### Executed cases"; echo '```text'; cat "$RUNNER_TEMP/floor.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
+ if [[ $floor != 0 ]]; then
+ echo "::error::fewer cases or tests ran than planned (see Executed cases)"
+ exit 1
+ fi
echo "engine verification passed ($TIER tier)"
diff --git a/scripts/verify/floor.py b/scripts/verify/floor.py
new file mode 100644
index 000000000..498fb5d54
--- /dev/null
+++ b/scripts/verify/floor.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Executed-case floor for an engine-verify run.
+
+ floor.py --matrix JSON --tier fast|full REPORTS_DIR
+ floor.py --recording --runners JSON --tier fast|full REPORTS_DIR
+
+REPORTS_DIR holds the downloaded report artifacts
+(verify-report-<platform>-<tier>-<shard>/). For every platform and suite the
+plan ran, the distinct case ids whose id starts with the suite's `cases.prefix`
+(scripts/verify/suites.json) must reach the floor for this tier and platform,
+and a suite with an exact `test` name must appear in the run's junit. A suite
+that silently shrinks (a renamed test, a skip glob or catalog edit that drops
+cases, a shard that never ran) fails the gate instead of passing vacuously.
+With --recording, REPORTS_DIR holds the real-capture artifacts
+(verify-recording-<runner>/verify-report): every runner must have recorded at
+least suites.json's `recording.cases` floor of distinct real-capture cases.
+Prints one line per check; exit status 1 when any check fails.
+"""
+
+import argparse
+import glob
+import json
+import os
+import sys
+import xml.etree.ElementTree as ET
+
+ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".."))
+
+
+def platform_reports(reports, platform, tier):
+ return sorted(glob.glob(os.path.join(reports, f"verify-report-{platform}-{tier}-*")))
+
+
+def case_ids(dirs):
+ ids = set()
+ for d in dirs:
+ for path in glob.glob(os.path.join(d, "**", "*.jsonl"), recursive=True):
+ rel = os.path.relpath(path, d).replace(os.sep, "/")
+ # Retry and flaky-rerun records repeat cases of the main pass.
+ if rel.startswith("retry/") or ".rerun" in rel:
+ continue
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ try:
+ record = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if record.get("case"):
+ ids.add(record["case"])
+ return ids
+
+
+def junit_tests(dirs):
+ names = set()
+ for d in dirs:
+ path = os.path.join(d, "junit.xml")
+ if not os.path.exists(path):
+ continue
+ for case in ET.parse(path).getroot().iter("testcase"):
+ names.add(case.get("name", ""))
+ return names
+
+
+def check(config, matrix, tier, reports):
+ lines, failed = [], 0
+ suites = {s["name"]: s for s in config["suites"]}
+ for entry in matrix.get("include", []):
+ platform = entry["platform"]
+ dirs = platform_reports(reports, platform, tier)
+ expected_shards = int(entry.get("shards", 1))
+ if len(dirs) < expected_shards:
+ failed += 1
+ lines.append(f"FAIL {platform}: {len(dirs)} of {expected_shards} shard reports present")
+ ids = case_ids(dirs)
+ tests = junit_tests(dirs)
+ for name in [n for n in entry["suites"].split(",") if n]:
+ suite = suites[name]
+ floor = suite.get("cases", {}).get(tier, {}).get(platform)
+ if floor is not None:
+ prefix = suite["cases"]["prefix"]
+ ran = sum(1 for i in ids if i.startswith(prefix))
+ ok = ran >= floor
+ failed += not ok
+ lines.append(
+ f"{'ok ' if ok else 'FAIL'} {platform} {name}: {ran} cases ran (floor {floor}, prefix {prefix})"
+ )
+ if suite.get("test"):
+ ok = suite["test"] in tests
+ failed += not ok
+ lines.append(
+ f"{'ok ' if ok else 'FAIL'} {platform} {name}: test {suite['test']} {'ran' if ok else 'is missing from the junit report'}"
+ )
+ return lines, failed
+
+
+def check_recording(config, runners, tier, reports):
+ lines, failed = [], 0
+ floor = config.get("recording", {}).get("cases", {}).get(tier, 0)
+ for runner in runners:
+ d = os.path.join(reports, f"verify-recording-{runner}")
+ ids = {
+ i for i in case_ids([d])
+ if i.startswith("recording/") and "(self-check" not in i
+ }
+ ok = os.path.isdir(d) and len(ids) >= floor
+ failed += not ok
+ lines.append(f"{'ok ' if ok else 'FAIL'} {runner}: {len(ids)} real-capture cases recorded (floor {floor})")
+ return lines, failed
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--matrix")
+ parser.add_argument("--recording", action="store_true")
+ parser.add_argument("--runners")
+ parser.add_argument("--tier", required=True)
+ parser.add_argument("reports")
+ args = parser.parse_args()
+ with open(os.path.join(ROOT, "scripts", "verify", "suites.json")) as f:
+ config = json.load(f)
+ if args.recording:
+ lines, failed = check_recording(config, json.loads(args.runners), args.tier, args.reports)
+ else:
+ lines, failed = check(config, json.loads(args.matrix), args.tier, args.reports)
+ print("\n".join(lines))
+ sys.exit(1 if failed else 0)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/verify/plan.py b/scripts/verify/plan.py
index 104487cd2..91618fa9d 100755
--- a/scripts/verify/plan.py
+++ b/scripts/verify/plan.py
@@ -9,6 +9,9 @@
Maps changed paths to suites through the workspace dependency graph
(`cargo metadata`) and prints GitHub step outputs: run, reason,
suites, matrix.
+ plan.py recording [--files FILE | --all]
+ run=, reason= for the real-capture workflow: whether a change reaches
+ the recorder's or the stimulus's dependency closure or the harness.
plan.py risk --files F [--summary FILE] [--output FILE]
changed files -> high-risk regions and guards (scripts/verify/guards.toml):
escalate=, regions=, guards= lines plus ::warning annotations.
@@ -35,6 +38,33 @@ def load_guards():
return tomllib.load(f)
+# Engine crates: a pull request that changes any of them runs the FULL tier
+# (with Blacksmith runners there is no queue to save).
+ENGINE_CRATES = [
+ "crates/rendering/**",
+ "crates/editor/**",
+ "crates/export/**",
+ "crates/recording/**",
+ "crates/enc-*/**",
+ "crates/audio/**",
+ "crates/video-decode/**",
+ "crates/project/**",
+ "crates/scap-*/**",
+ "crates/camera*/**",
+ "crates/media/**",
+ "crates/media-info/**",
+ "crates/frame-converter/**",
+ "crates/gpu-converters/**",
+ "crates/cursor-capture/**",
+ "crates/timestamp/**",
+ "crates/cap-muxer/**",
+]
+
+
+def engine_files(files):
+ return sorted({p for p in files for pattern in ENGINE_CRATES if match(p, pattern)})
+
+
def risk(files, guards=None):
"""(regions touched, guards whose files are touched) for changed files."""
guards = guards or load_guards()
@@ -81,7 +111,10 @@ def load_config():
def match(path, pattern):
if pattern.endswith("/**"):
- return path.startswith(pattern[:-2])
+ prefix = pattern[:-2]
+ if any(c in prefix for c in "*?["):
+ return fnmatch.fnmatchcase(path, prefix + "*")
+ return path.startswith(prefix)
return fnmatch.fnmatchcase(path, pattern)
@@ -154,11 +187,15 @@ def cmd_select(args):
def cmd_env(args):
"""Shell `export` lines for the env of the selected suites (suites.json
- `env`: {"KEY": "value"} or {"KEY": {"<platform>": "value"}}). Variables
- already set in the environment win."""
+ `env`: {"KEY": "value"} or {"KEY": {"<platform>": "value"}}) and the
+ platform's unsupported-case allowlist. Variables already set in the
+ environment win."""
import shlex
config = load_config()
names = [n for n in (args.suites or "").split(",") if n]
+ allow = ",".join(config.get("unsupported", {}).get(args.platform, {}).keys())
+ if allow and "CAP_VERIFY_UNSUPPORTED_ALLOW" not in os.environ:
+ print(f"export CAP_VERIFY_UNSUPPORTED_ALLOW={shlex.quote(allow)}")
for suite in select(config, args.tier, args.platform, names or None, args.local):
for key, value in suite.get("env", {}).items():
if isinstance(value, dict):
@@ -311,6 +348,56 @@ def cmd_changes(args):
print(f"{key}={value}")
+# Real capture (crates/verify/tests/recording.rs): the recorder and the
+# stimulus, their whole dependency closure, and the harness itself.
+RECORDING = {
+ "roots": ["cap-recording", "cap-verify-stimulus"],
+ "own": [
+ "crates/verify/tests/recording.rs",
+ "crates/verify/src/**",
+ "crates/verify/Cargo.toml",
+ "crates/verify/expectations.toml",
+ "crates/verify-expect/**",
+ "crates/verify-stimulus/**",
+ ".github/workflows/verify-recording.yml",
+ ],
+}
+
+
+def recording_affected(config, files, metadata):
+ """(run, reason) for the real-capture workflow."""
+ for path in files:
+ for pattern in config["always"] + RECORDING["own"]:
+ if match(path, pattern):
+ return True, f"{path} affects real capture"
+ dirs, deps = workspace_graph(metadata)
+ reach = set().union(*(closure(r, deps) for r in RECORDING["roots"]))
+ for path in files:
+ # Other suites' test files share the cap-verify package but not the
+ # recording test binary.
+ if path.startswith("crates/verify/tests/"):
+ continue
+ package = owning_package(path, dirs)
+ if package in reach:
+ return True, f"{path} ({package}) is in the recorder's dependency closure"
+ return False, "no recording code changed"
+
+
+def cmd_recording(args):
+ config = load_config()
+ if args.all:
+ run, reason = True, "full run"
+ else:
+ with open(args.files) as f:
+ files = [line.strip() for line in f if line.strip()]
+ if len(files) >= args.max_files:
+ run, reason = True, f"{len(files)} changed files (list may be truncated)"
+ else:
+ run, reason = recording_affected(config, files, load_metadata(args.metadata))
+ print(f"run={'true' if run else 'false'}")
+ print(f"reason={reason}")
+
+
def cmd_risk(args):
with open(args.files) as f:
files = [line.strip() for line in f if line.strip()]
@@ -321,8 +408,12 @@ def cmd_risk(args):
f.write(md + "\n")
for note in notes:
print(note)
+ engine = engine_files(files)
+ if engine:
+ print(f"::notice title=Engine crates changed::FULL tier for {', '.join(engine[:6])}{' ...' if len(engine) > 6 else ''}")
outputs = [
f"escalate={'true' if regions or touched else 'false'}",
+ f"engine={'true' if engine else 'false'}",
f"regions={','.join(name for name, _ in regions)}",
f"guards={','.join(g['id'] for g in touched)}",
]
@@ -374,6 +465,27 @@ def cmd_test(_args):
ok = got == expected
failed += not ok
print(f"{'ok ' if ok else 'BAD'} risk {files} -> {got}")
+ for files, expected in [
+ (["crates/rendering/src/lib.rs"], True),
+ (["crates/enc-ffmpeg/src/lib.rs"], True),
+ (["crates/camera-ffmpeg/src/lib.rs"], True),
+ (["apps/desktop-gpui/src/main.rs", "crates/verify/src/oracle.rs"], False),
+ ]:
+ ok = bool(engine_files(files)) == expected
+ failed += not ok
+ print(f"{'ok ' if ok else 'BAD'} engine {files} -> {bool(engine_files(files))}")
+ for files, expected in [
+ (["crates/frame-converter/src/lib.rs"], True),
+ (["crates/recording/src/studio_recording.rs"], True),
+ (["crates/verify/tests/recording.rs"], True),
+ (["vendor/wgpu-hal/src/lib.rs"], True),
+ (["Cargo.toml"], True),
+ (["crates/verify/tests/export.rs", "apps/web/app/page.tsx"], False),
+ ]:
+ got = recording_affected(config, files, metadata)[0]
+ ok = got == expected
+ failed += not ok
+ print(f"{'ok ' if ok else 'BAD'} recording {files} -> {got}")
rendering_only = affected(config, ["crates/rendering/src/decoder/model/mod.rs"], metadata, "fast")[0]
if "decoder-model-ffmpeg" not in rendering_only:
failed += 1
@@ -404,6 +516,12 @@ def main():
p.add_argument("--max-files", type=int, default=3000)
p.add_argument("--suites")
p.set_defaults(func=cmd_changes)
+ p = sub.add_parser("recording")
+ p.add_argument("--files")
+ p.add_argument("--all", action="store_true")
+ p.add_argument("--metadata")
+ p.add_argument("--max-files", type=int, default=3000)
+ p.set_defaults(func=cmd_recording)
p = sub.add_parser("risk")
p.add_argument("--files", required=True)
p.add_argument("--summary", help="append the markdown report to this file")
diff --git a/scripts/verify/suites.json b/scripts/verify/suites.json
index bbf24b582..b1e11d671 100644
--- a/scripts/verify/suites.json
+++ b/scripts/verify/suites.json
@@ -1,5 +1,5 @@
{
- "_doc": "Engine verification suites. Read by scripts/verify/plan.py (CI planning, change detection) and scripts/verify/run.sh (local and CI runs). Fields: cargo = the cargo target selection that builds the suite; test = optional exact test name inside that binary; platforms = where each tier runs it (platform-independent logic runs on linux only); roots = workspace packages whose dependency closure decides whether a change can affect the suite; own = files that only affect this suite (a change confined to them selects just this suite). skip = case-id globs left out of a tier on a platform (CAP_VERIFY_SKIP), to keep the FAST tier within its time budget; the FULL tier runs them. Exception: hosted Windows runners have no D3D11 video device, so every Windows case decodes with FFmpeg (the code Linux runs) and renders on the WARP software rasterizer; there the 120 s jitter-camera export and playback and the 3000-seek storms took 6 to more than 15 minutes each (killed at the 15 minute per-test limit in 4 of 5 full runs), so FULL leaves them to Linux and macOS.",
+ "_doc": "Engine verification suites. Read by scripts/verify/plan.py (CI planning, change detection) and scripts/verify/run.sh (local and CI runs). Fields: cargo = the cargo target selection that builds the suite; test = optional exact test name inside that binary; platforms = where each tier runs it (platform-independent logic runs on linux only); roots = workspace packages whose dependency closure decides whether a change can affect the suite; own = files that only affect this suite (a change confined to them selects just this suite). skip = case-id globs left out of a tier on a platform (CAP_VERIFY_SKIP), to keep the FAST tier within its time budget; the FULL tier runs them. Exception: hosted Windows runners have no D3D11 video device, so every Windows case decodes with FFmpeg (the code Linux runs) and renders on the WARP software rasterizer; there the 120 s jitter-camera export and playback and the 3000-seek storms took 6 to more than 15 minutes each (killed at the 15 minute per-test limit in 4 of 5 full runs), so FULL leaves them to Linux and macOS. cases = the executed-case floor per tier and platform: the gate fails when fewer distinct cases whose id starts with prefix ran across all shards (scripts/verify/floor.py). unsupported = per-platform case globs that may report unsupported (with the reason); CI sets CAP_VERIFY_REQUIRE_SUPPORTED=1, so any other unsupported case fails.",
"suites": [
{
"name": "verify-self",
@@ -27,16 +27,24 @@
"skip": {
"fast": {
"macos": ["export/*/ffmpeg", "export/backward-hold/*/mp430/*"],
- "windows": [
- "export/*/mp460/*",
- "export/backward-hold/*",
- "export/jitter-camera-*",
- "export/*/uhd-*"
- ]
+ "windows": ["export/*/mp460/*", "export/backward-hold/*", "export/jitter-camera-*", "export/*/uhd-*"]
},
"full": {
"windows": ["export/jitter-camera-120s/*"]
}
+ },
+ "cases": {
+ "prefix": "export/",
+ "fast": {
+ "linux": 42,
+ "macos": 41,
+ "windows": 43
+ },
+ "full": {
+ "linux": 43,
+ "macos": 69,
+ "windows": 67
+ }
}
},
{
@@ -57,6 +65,18 @@
"full": {
"windows": ["playback/jitter-camera-120s/*"]
}
+ },
+ "cases": {
+ "prefix": "playback/",
+ "fast": {
+ "linux": 19,
+ "macos": 19
+ },
+ "full": {
+ "linux": 19,
+ "macos": 38,
+ "windows": 36
+ }
}
},
{
@@ -77,6 +97,18 @@
"full": {
"windows": ["seek/*/storm*"]
}
+ },
+ "cases": {
+ "prefix": "seek/",
+ "fast": {
+ "linux": 11,
+ "macos": 11
+ },
+ "full": {
+ "linux": 11,
+ "macos": 22,
+ "windows": 16
+ }
}
},
{
@@ -100,6 +132,19 @@
"windows": ["audio/*/mp4-plain30", "audio/*/gif30", "audio/*/mov30"],
"macos": ["audio/*/mp4-plain30", "audio/*/gif30"]
}
+ },
+ "cases": {
+ "prefix": "audio/",
+ "fast": {
+ "linux": 56,
+ "macos": 40,
+ "windows": 39
+ },
+ "full": {
+ "linux": 58,
+ "macos": 58,
+ "windows": 58
+ }
}
},
{
@@ -112,7 +157,16 @@
"full": ["linux"]
},
"roots": ["cap-verify"],
- "own": ["crates/verify/tests/editing.rs", "crates/verify/tests/edits/**"]
+ "own": ["crates/verify/tests/editing.rs", "crates/verify/tests/edits/**"],
+ "cases": {
+ "prefix": "editing/",
+ "fast": {
+ "linux": 22
+ },
+ "full": {
+ "linux": 78
+ }
+ }
},
{
"name": "captions",
@@ -124,7 +178,16 @@
"full": ["linux"]
},
"roots": ["cap-verify"],
- "own": ["crates/verify/tests/captions.rs"]
+ "own": ["crates/verify/tests/captions.rs"],
+ "cases": {
+ "prefix": "captions/",
+ "fast": {
+ "linux": 19
+ },
+ "full": {
+ "linux": 19
+ }
+ }
},
{
"name": "decoder-model-ffmpeg",
@@ -137,7 +200,16 @@
"full": ["linux"]
},
"roots": ["cap-rendering"],
- "own": []
+ "own": [],
+ "cases": {
+ "prefix": "decoder-model/ffmpeg/",
+ "fast": {
+ "linux": 29
+ },
+ "full": {
+ "linux": 29
+ }
+ }
},
{
"name": "decoder-model-avf",
@@ -150,7 +222,16 @@
"full": ["macos"]
},
"roots": ["cap-rendering"],
- "own": []
+ "own": [],
+ "cases": {
+ "prefix": "decoder-model/avf/",
+ "fast": {
+ "macos": 31
+ },
+ "full": {
+ "macos": 31
+ }
+ }
},
{
"name": "native-rendering",
@@ -162,10 +243,16 @@
"full": ["macos"]
},
"roots": ["cap-rendering"],
- "own": [
- "crates/rendering/tests/iosurface-source-lifetime.rs",
- "crates/rendering/tests/expectations.toml"
- ]
+ "own": ["crates/rendering/tests/iosurface-source-lifetime.rs", "crates/rendering/tests/expectations.toml"],
+ "cases": {
+ "prefix": "native/",
+ "fast": {
+ "macos": 1
+ },
+ "full": {
+ "macos": 1
+ }
+ }
}
],
"always": [
@@ -208,5 +295,17 @@
},
"archive": false
}
+ },
+ "unsupported": {
+ "windows": {
+ "*/mf": "Media Foundation decoding needs a D3D11 device with video support; CI Windows VMs render on the WARP software adapter, which has none, so the product falls back to FFmpeg there. Media Foundation is not exercised by CI."
+ }
+ },
+ "recording": {
+ "_doc": "Real capture (verify-recording.yml): distinct case ids each runner must record per tier (crates/verify/tests/recording.rs specs; system-only reports unsupported when the stimulus is silent).",
+ "cases": {
+ "fast": 8,
+ "full": 12
+ }
}
}
diff --git a/scripts/verify/summary.py b/scripts/verify/summary.py
index 66cca3b89..a44bbba54 100755
--- a/scripts/verify/summary.py
+++ b/scripts/verify/summary.py
@@ -22,6 +22,11 @@ def load_records(dirs):
records = []
for d in dirs:
for path in sorted(glob.glob(os.path.join(d, "**", "*.jsonl"), recursive=True)):
+ # Retry records repeat cases of the main pass (counted with the
+ # retries below), so they are not cases of their own.
+ rel = os.path.relpath(path, d).replace(os.sep, "/")
+ if rel.startswith("retry/"):
+ continue
with open(path, encoding="utf-8") as f:
for line in f:
line = line.strip()
@@ -132,6 +137,21 @@ def render(records, title, junit, flaky, text, retries=None):
f"{counts.get('fail', 0)} FAILED, {counts.get('xpass', 0)} unexpectedly passing, "
f"{counts.get('unsupported', 0)} unsupported ({seconds:.0f} s of case time)"
)
+ excused = {}
+ for r in records:
+ for kind in (r.get("excused") or {}).values():
+ excused[kind] = excused.get(kind, 0) + 1
+ if excused.get("flaky") or excused.get("flaky, starved"):
+ head += (
+ f"; expected failures excused by flaky entries: {excused.get('flaky', 0)}, "
+ f"only under a proven machine stall: {excused.get('flaky, starved', 0)}"
+ )
+ second = [
+ r for r in records
+ if any(str(n).startswith("inconclusive, retrying") for n in r.get("notes") or [])
+ ]
+ if second:
+ head += f"; {len(second)} real-capture case(s) settled on a second attempt after a proven input fault"
unmeasured = [r for r in records if r.get("unobserved")]
if unmeasured:
head += f"; {len(unmeasured)} with classes not measured because an earlier check failed"