Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

test(verify): fragment span, orphaned muxer and crash suite wiring

Richie McIlroy14f25d18c522
diff --git a/.github/workflows/verify-crash.yml b/.github/workflows/verify-crash.yml
new file mode 100644
index 000000000..302f109a1
--- /dev/null
+++ b/.github/workflows/verify-crash.yml
@@ -0,0 +1,47 @@
+name: Verify crash safety
+
+# The `crash` engine-verify suite (crates/verify/tests/crash.rs) on all three
+# platforms: in-process vs out-of-process muxer equality and recorder, muxer
+# and finalize kills against what is physically on disk. engine-verify.yml
+# runs it like every other suite; this workflow runs it alone on Blacksmith
+# runners for crash-safety proof branches.
+
+on:
+ push:
+ branches:
+ - ci/fixproof-crash*
+ - scratch/crash-*
+ workflow_dispatch:
+ inputs:
+ tier:
+ description: Tier
+ type: choice
+ options: [fast, full]
+ default: full
+
+permissions:
+ contents: read
+
+jobs:
+ verify:
+ name: ${{ matrix.platform }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - platform: macos
+ runner: blacksmith-6vcpu-macos-15
+ target: aarch64-apple-darwin
+ - platform: linux
+ runner: blacksmith-8vcpu-ubuntu-2404
+ target: x86_64-unknown-linux-gnu
+ - platform: windows
+ runner: blacksmith-8vcpu-windows-2025
+ target: x86_64-pc-windows-msvc
+ uses: ./.github/workflows/engine-verify-os.yml
+ with:
+ platform: ${{ matrix.platform }}
+ runner: ${{ matrix.runner }}
+ target: ${{ matrix.target }}
+ tier: ${{ inputs.tier || 'full' }}
+ suites: crash
diff --git a/crates/verify/tests/crash.rs b/crates/verify/tests/crash.rs
index 9c9bf77df..deb0c5bac 100644
--- a/crates/verify/tests/crash.rs
+++ b/crates/verify/tests/crash.rs
@@ -644,15 +644,86 @@ fn decode_markers(path: &Path) -> anyhow::Result<Vec<(u32, f64)>> {
Ok(out)
}
-/// Every frame inside the physically complete fragments, per writer
-/// directory in order.
+/// The frames of every physically complete fragment, per fragment, writer
+/// directories in order.
+fn fragments_on_disk(display: &Path, scratch: &Path) -> anyhow::Result<Vec<Vec<(u32, f64)>>> {
+ let mut out = Vec::new();
+ for dir in writer_dirs(display) {
+ for fragment in complete_fragments(&dir) {
+ out.push(decode_fragments(
+ &dir,
+ std::slice::from_ref(&fragment),
+ scratch,
+ )?);
+ }
+ }
+ Ok(out)
+}
+
+/// Every frame inside the physically complete fragments, in order.
fn on_disk(display: &Path, scratch: &Path) -> anyhow::Result<Vec<(u32, f64)>> {
- let mut frames = Vec::new();
+ Ok(fragments_on_disk(display, scratch)?.concat())
+}
+
+/// A fragment covers the capture time from its first frame to the first
+/// frame of the next one, and the encoders must start a new one once
+/// `GOP_SECS` have passed: the span may exceed `GOP_SECS` only by the
+/// interval before the frame that closed it.
+fn check_fragment_spans(case: &mut Case, fragments: &[Vec<(u32, f64)>], handed: &[Handed]) {
+ let time: BTreeMap<u32, f64> = handed.iter().map(|h| (h.index, h.secs)).collect();
+ let next_after = |index: u32| handed.iter().find(|h| h.index > index).map(|h| h.secs);
+ let mut worst = 0.0f64;
+ for fragment in fragments {
+ let (Some(first), Some(last)) = (fragment.first(), fragment.last()) else {
+ continue;
+ };
+ let (Some(&start), Some(&end)) = (time.get(&first.0), time.get(&last.0)) else {
+ continue;
+ };
+ let Some(next) = next_after(last.0) else {
+ continue;
+ };
+ let span = next - start;
+ worst = worst.max(span);
+ if span > GOP_SECS + (next - end) + 1e-6 {
+ case.problem(
+ "fragment-span",
+ format!(
+ "fragment of frames {}..={} covers {span:.2}s of capture, more than {GOP_SECS}s plus the {:.2}s before the frame that closed it",
+ first.0,
+ last.0,
+ next - end
+ ),
+ );
+ }
+ }
+ case.note(format!(
+ "longest complete fragment covers {worst:.2}s of capture"
+ ));
+}
+
+/// Out-of-process only: after the recorder died, the orphaned cap-muxer
+/// sees EOF and must finish the fragment it had open (nothing it received
+/// may be lost).
+fn check_open_fragment(case: &mut Case, display: &Path) {
for dir in writer_dirs(display) {
- let fragments = complete_fragments(&dir);
- frames.extend(decode_fragments(&dir, &fragments, scratch)?);
+ for entry in std::fs::read_dir(&dir).into_iter().flatten().flatten() {
+ let path = entry.path();
+ let name = entry.file_name().to_string_lossy().into_owned();
+ if name.starts_with("segment_")
+ && name.ends_with(".m4s.tmp")
+ && !fragment_complete(&path)
+ {
+ let size = entry.metadata().map(|m| m.len()).unwrap_or(0);
+ case.problem(
+ "open-fragment-lost",
+ format!(
+ "the orphaned cap-muxer left {name} incomplete ({size} bytes): the packets it had received are lost"
+ ),
+ );
+ }
+ }
}
- Ok(frames)
}
// ---------------------------------------------------------------------------
@@ -1003,7 +1074,12 @@ fn recorder_kill(
handed.last().map(|h| h.index),
if stopped { " while stopping" } else { "" }
));
- let physical = on_disk(&display, &scratch)?;
+ if arm == Arm::OutOfProcess {
+ check_open_fragment(case, &display);
+ }
+ let fragments = fragments_on_disk(&display, &scratch)?;
+ check_fragment_spans(case, &fragments, &handed);
+ let physical = fragments.concat();
let recovered = match finalize(&display, &scratch.join("recovered.mp4")) {
Ok(frames) => frames,
Err(e) if physical.is_empty() => {
diff --git a/scripts/verify/run.sh b/scripts/verify/run.sh
index 46c0c6757..e5145ffcb 100755
--- a/scripts/verify/run.sh
+++ b/scripts/verify/run.sh
@@ -104,6 +104,12 @@ if [[ "$mode" == list ]]; then
exit 0
fi
+# The crash suite spawns the out-of-process muxer; build it with the same
+# profile so it sits beside the test binaries (target/<profile>/cap-muxer).
+if [[ " ${suite_names//,/ } " == *" crash "* && "$mode" != archive ]]; then
+ cargo build --profile "$profile" -p cap-muxer ${CAP_VERIFY_LOCKED:+--locked}
+fi
+
have_nextest=0
if command -v cargo-nextest >/dev/null 2>&1 || cargo nextest --version >/dev/null 2>&1; then
have_nextest=1
diff --git a/scripts/verify/suites.json b/scripts/verify/suites.json
index 73e1c17c1..197cc3c8c 100644
--- a/scripts/verify/suites.json
+++ b/scripts/verify/suites.json
@@ -111,6 +111,19 @@
}
}
},
+ {
+ "name": "crash",
+ "about": "crash safety of fragmented Studio recording: in-process vs out-of-process muxer equality, recorder and muxer kills, recovery against what is physically on disk (needs the cap-muxer binary, built by run.sh)",
+ "cargo": ["-p", "cap-verify", "--test", "crash"],
+ "binaries": ["cap-verify::crash"],
+ "test": "crash_safety",
+ "platforms": {
+ "fast": ["linux", "macos", "windows"],
+ "full": ["linux", "macos", "windows"]
+ },
+ "roots": ["cap-verify", "cap-muxer"],
+ "own": ["crates/verify/tests/crash.rs"]
+ },
{
"name": "audio",
"about": "export and offline playback audio against sample-accurate markers (mic, system audio, cuts, speeds, mute, volume, A/V offset)",