Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
test(verify): crash-safety suite for fragmented Studio recording
Richie McIlroybfd6879e1621
diff --git a/crates/verify/tests/crash.rs b/crates/verify/tests/crash.rsnew file mode 100644index 000000000..9c9bf77df--- /dev/null+++ b/crates/verify/tests/crash.rs@@ -0,0 +1,1204 @@+//! Crash safety of fragmented Studio recording: the out-of-process muxer+//! against the in-process one, and recovery after the recorder, the muxer or+//! the disk fails.+//!+//! Synthetic marker frames go through the product's Studio display muxers.+//! macOS: the in-process `MacOSFragmentedM4SMuxer` and the out-of-process+//! `OutOfProcessFragmentedM4SMuxer` (a `cap-muxer` child), fed NV12 IOSurface+//! sample buffers like ScreenCaptureKit delivers. Linux and Windows: the+//! shared `SegmentedVideoEncoder` writer through `ffmpeg::SegmentedVideoMuxer`+//! (the Linux Studio display muxer; on Windows the writer behind+//! `WindowsFragmentedM4SMuxer`, whose Direct3D capture frames cannot be+//! synthesized, so Windows has no out-of-process case here).+//!+//! Every frame handed to the pipeline is logged (index, capture time) line by+//! line, so the truth survives a SIGKILL of the recorder. Recovery is judged+//! against what is physically on disk after the failure: every fragment whose+//! box structure is complete must come back frame for frame at its capture+//! time. Frames lost to a failure must be the newest ones and fit in the+//! fragment being written: one GOP (`GOP_SECS` x fps frames, the encoder's+//! keyframe interval, so the longest a segment can be when frames arrive at+//! the nominal rate) plus `IN_FLIGHT` frames still inside the pipeline.++mod common;++use std::{+ collections::BTreeMap,+ io::Write,+ path::{Path, PathBuf},+ process::{Child, Command, Stdio},+ time::{Duration, Instant},+};++use anyhow::{Context, anyhow, bail};+use cap_media_info::{RawVideoFormat, VideoInfo};+use cap_recording::{+ ChannelVideoSource, ChannelVideoSourceConfig, OutputPipeline, recovery::RecoveryManager,+};+use cap_timestamp::{Timestamp, Timestamps};+use cap_verify::{+ fixture::Rng,+ media,+ report::{self, Case, Tier},+ video_code::{self, FrameMarker, StreamKind},+};++const FPS: u32 = 30;+const WIDTH: u32 = 640;+const HEIGHT: u32 = 360;+/// `DEFAULT_KEYFRAME_INTERVAL_SECS` in crates/enc-ffmpeg/src/video/h264.rs:+/// the encoder emits a keyframe every `GOP_SECS * fps` frames and both DASH+/// writers cut a segment only at a keyframe.+const GOP_SECS: f64 = 2.0;+const GOP: u32 = (GOP_SECS as u32) * FPS;+/// Frames that can still be inside the pipeline when the process dies: the+/// one the encoder holds back to stamp real durations (segmented_stream.rs+/// `queue_frame`), the one being encoded, and the one in the muxer channel.+/// The emitter runs at the nominal rate on an idle encoder, so the channel+/// holds at most one frame at a time.+const IN_FLIGHT: u32 = 3;+/// Recorded length of every case.+const SECONDS: f64 = 8.0;+/// Truth timestamps vs presentation times: the encoder time base and the+/// container round to at most 1/90000 s; half a millisecond is far below the+/// 33 ms frame period and above any rounding.+const PTS_TOLERANCE: f64 = 0.0005;+const CHILD_ENV: &str = "CAP_VERIFY_CRASH_CHILD";++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum Arm {+ /// The default in-process writer.+ InProcess,+ /// `out_of_process_muxer` on (macOS only here).+ OutOfProcess,+}++impl Arm {+ fn label(self) -> &'static str {+ match self {+ Arm::InProcess => "inproc",+ Arm::OutOfProcess => "oop",+ }+ }++ fn parse(text: &str) -> Option<Self> {+ match text {+ "inproc" => Some(Arm::InProcess),+ "oop" => Some(Arm::OutOfProcess),+ _ => None,+ }+ }+}++fn arms() -> Vec<Arm> {+ if cfg!(target_os = "macos") {+ vec![Arm::InProcess, Arm::OutOfProcess]+ } else {+ vec![Arm::InProcess]+ }+}++/// One frame the emitter handed to the pipeline.+#[derive(Clone, Copy, Debug)]+struct Handed {+ index: u32,+ /// Seconds after the first handed frame.+ secs: f64,+}++/// What a recording run left behind: the truth and the display directory.+struct Run {+ handed: Vec<Handed>,+ display: PathBuf,+}++// ---------------------------------------------------------------------------+// Recording (shared by the in-test runs and the child recorder)+// ---------------------------------------------------------------------------++#[cfg(target_os = "macos")]+mod source {+ use super::*;+ use cap_recording::{+ MacOSFragmentedM4SMuxer, MacOSFragmentedM4SMuxerConfig, OutOfProcessFragmentedM4SMuxer,+ OutOfProcessFragmentedM4SMuxerConfig, screen_capture,+ };+ use cidre::cm;++ pub type Frame = screen_capture::VideoFrame;++ pub fn info() -> VideoInfo {+ VideoInfo::from_raw(RawVideoFormat::Nv12, WIDTH, HEIGHT, FPS)+ }++ pub type Picture = cidre::arc::R<cm::SampleBuf>;++ /// Renders ahead of time so emission keeps the nominal rate.+ pub fn picture(index: u32) -> anyhow::Result<Picture> {+ let rgba = video_code::marker_rgba(+ FrameMarker::new(StreamKind::Screen, 0, index),+ WIDTH as usize,+ HEIGHT as usize,+ );+ let buf = cap_verify::fixture_avf::pixel_buffer(WIDTH as usize, HEIGHT as usize, &rgba)?;+ let format_desc = cm::VideoFormatDesc::with_image_buf(&buf)+ .map_err(|e| anyhow!("format description: {e:?}"))?;+ let timing = cm::SampleTimingInfo {+ duration: cm::Time::new(1, FPS as i32),+ pts: cm::Time::new(i64::from(index), FPS as i32),+ dts: cm::Time::invalid(),+ };+ cm::SampleBuf::with_image_buf(&buf, true, None, std::ptr::null(), &format_desc, &timing)+ .map_err(|e| anyhow!("sample buffer: {e:?}"))+ }++ pub fn frame(picture: Picture, at: Instant) -> Frame {+ screen_capture::VideoFrame {+ sample_buf: picture,+ timestamp: Timestamp::Instant(at),+ }+ }++ pub async fn pipeline(+ display: &Path,+ arm: Arm,+ rx: flume::Receiver<Frame>,+ timestamps: Timestamps,+ ) -> anyhow::Result<OutputPipeline> {+ let builder = OutputPipeline::builder(display.to_path_buf())+ .with_video::<ChannelVideoSource<Frame>>(ChannelVideoSourceConfig::new(info(), rx))+ .with_timestamps(timestamps);+ match arm {+ Arm::InProcess => {+ builder+ .build::<MacOSFragmentedM4SMuxer>(MacOSFragmentedM4SMuxerConfig::default())+ .await+ }+ Arm::OutOfProcess => {+ builder+ .build::<OutOfProcessFragmentedM4SMuxer>(+ OutOfProcessFragmentedM4SMuxerConfig::default(),+ )+ .await+ }+ }+ }+}++#[cfg(not(target_os = "macos"))]+mod source {+ use super::*;+ use cap_recording::ffmpeg::{FFmpegVideoFrame, SegmentedVideoMuxer, SegmentedVideoMuxerConfig};++ pub type Frame = FFmpegVideoFrame;++ pub fn info() -> VideoInfo {+ VideoInfo::from_raw(RawVideoFormat::Bgra, WIDTH, HEIGHT, FPS)+ }++ pub type Picture = ffmpeg::frame::Video;++ /// Renders ahead of time so emission keeps the nominal rate.+ pub fn picture(index: u32) -> anyhow::Result<Picture> {+ let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, WIDTH, HEIGHT);+ let stride = frame.stride(0);+ video_code::draw_marker(+ FrameMarker::new(StreamKind::Screen, 0, index),+ frame.data_mut(0),+ WIDTH as usize,+ HEIGHT as usize,+ stride,+ video_code::PixelOrder::Bgra,+ );+ Ok(frame)+ }++ pub fn frame(picture: Picture, at: Instant) -> Frame {+ FFmpegVideoFrame {+ inner: picture,+ timestamp: Timestamp::Instant(at),+ }+ }++ pub async fn pipeline(+ display: &Path,+ arm: Arm,+ rx: flume::Receiver<Frame>,+ timestamps: Timestamps,+ ) -> anyhow::Result<OutputPipeline> {+ anyhow::ensure!(arm == Arm::InProcess, "no out-of-process muxer here");+ OutputPipeline::builder(display.to_path_buf())+ .with_video::<ChannelVideoSource<Frame>>(ChannelVideoSourceConfig::new(info(), rx))+ .with_timestamps(timestamps)+ .build::<SegmentedVideoMuxer>(SegmentedVideoMuxerConfig::default())+ .await+ }+}++/// Line-per-event truth log that a SIGKILL cannot lose (no buffering).+struct TruthLog(Option<std::fs::File>);++impl TruthLog {+ fn line(&mut self, text: &str) {+ if let Some(file) = &mut self.0 {+ let _ = file.write_all(format!("{text}\n").as_bytes());+ }+ }+}++/// Hooks a case runs while the recorder is live.+#[derive(Default)]+struct Live {+ /// Called with every handed index (after it was handed).+ on_frame: Option<Box<dyn FnMut(u32) + Send>>,+}++/// Records `seconds` of marker frames into `display` and stops like the+/// recorder (`OutputPipeline::stop`).+/// Every `step`-th slot of the nominal rate carries a frame (1 = steady; more+/// is a mostly static screen, where ScreenCaptureKit delivers only changes).+async fn record(+ display: &Path,+ arm: Arm,+ seconds: f64,+ step: u32,+ truth: Option<&Path>,+ mut live: Live,+) -> anyhow::Result<Run> {+ std::fs::create_dir_all(display)?;+ let mut log = TruthLog(match truth {+ Some(path) => Some(std::fs::File::create(path)?),+ None => None,+ });+ let count = (seconds * f64::from(FPS)) as u32;+ let slots: Vec<u32> = (0..count).step_by(step.max(1) as usize).collect();+ let mut pictures = slots+ .iter()+ .copied()+ .map(source::picture)+ .collect::<anyhow::Result<Vec<_>>>()?+ .into_iter();+ let timestamps = Timestamps::now();+ let (tx, rx) = flume::bounded::<source::Frame>(8);+ let mut pipeline = source::pipeline(display, arm, rx, timestamps).await?;+ let mut health = pipeline.take_health_rx();+ let base = Instant::now() + Duration::from_millis(50);+ let mut handed = Vec::new();+ for index in slots {+ let at = base + Duration::from_secs_f64(f64::from(index) / f64::from(FPS));+ tokio::time::sleep_until(at.into()).await;+ let picture = pictures.next().expect("rendered");+ let frame = source::frame(picture, at);+ tx.send_async(frame)+ .await+ .map_err(|_| anyhow!("pipeline stopped taking frames at {index}"))?;+ let secs = f64::from(index) / f64::from(FPS);+ handed.push(Handed { index, secs });+ log.line(&format!("f {index} {secs:.6}"));+ if let Some(hook) = live.on_frame.as_mut() {+ hook(index);+ }+ }+ tokio::time::sleep(Duration::from_millis(200)).await;+ log.line("stop");+ drop(tx);+ let finished = pipeline.stop().await;+ if let Some(health) = health.as_mut() {+ while let Ok(event) = health.try_recv() {+ log.line(&format!("health {event:?}"));+ }+ }+ finished.map_err(|e| anyhow!("stop: {e:#}"))?;+ log.line("done");+ Ok(Run {+ handed,+ display: display.to_path_buf(),+ })+}++fn read_truth(path: &Path) -> anyhow::Result<(Vec<Handed>, bool, Vec<String>)> {+ let text = std::fs::read_to_string(path).with_context(|| format!("read {path:?}"))?;+ let mut handed = Vec::new();+ let mut stopped = false;+ let mut health = Vec::new();+ for line in text.lines() {+ let mut parts = line.split(' ');+ match parts.next() {+ Some("f") => {+ let (Some(index), Some(secs)) = (parts.next(), parts.next()) else {+ continue;+ };+ // A kill can cut the last line short.+ if let (Ok(index), Ok(secs)) = (index.parse(), secs.parse()) {+ if line.len() > 2 && secs_complete(line) {+ handed.push(Handed { index, secs });+ }+ }+ }+ Some("stop") => stopped = true,+ Some("health") => health.push(line[7..].to_string()),+ _ => {}+ }+ }+ Ok((handed, stopped, health))+}++/// The writer prints 6 decimals; a shorter field was cut by the kill.+fn secs_complete(line: &str) -> bool {+ line.rsplit(' ')+ .next()+ .and_then(|s| s.split('.').nth(1))+ .is_some_and(|d| d.len() == 6)+}++// ---------------------------------------------------------------------------+// The child recorder (a SIGKILL target)+// ---------------------------------------------------------------------------++/// Runs only as a child of `crash_safety` (`CAP_VERIFY_CRASH_CHILD` set):+/// records into the given directory until it is killed or finishes.+#[test]+fn crash_child_recorder() {+ let Ok(spec) = std::env::var(CHILD_ENV) else {+ return;+ };+ common::init();+ let fields: BTreeMap<&str, &str> = spec+ .split(';')+ .filter_map(|kv| kv.split_once('='))+ .collect();+ let arm = Arm::parse(fields["arm"]).expect("arm");+ let display = PathBuf::from(fields["display"]);+ let truth = PathBuf::from(fields["truth"]);+ let seconds: f64 = fields["seconds"].parse().expect("seconds");+ let step: u32 = fields["step"].parse().expect("step");+ let rt = common::runtime();+ rt.block_on(record(+ &display,+ arm,+ seconds,+ step,+ Some(&truth),+ Live::default(),+ ))+ .expect("child recording");+}++struct ChildRecorder {+ child: Child,+ truth: PathBuf,+}++impl ChildRecorder {+ fn spawn(+ display: &Path,+ truth: &Path,+ arm: Arm,+ seconds: f64,+ step: u32,+ ) -> anyhow::Result<Self> {+ let exe = std::env::current_exe()?;+ let child = Command::new(exe)+ .args([+ "crash_child_recorder",+ "--exact",+ "--nocapture",+ "--test-threads=1",+ ])+ .env(+ CHILD_ENV,+ format!(+ "arm={};display={};truth={};seconds={seconds};step={step}",+ arm.label(),+ display.display(),+ truth.display()+ ),+ )+ .stdout(Stdio::null())+ .stderr(Stdio::inherit())+ .spawn()?;+ Ok(Self {+ child,+ truth: truth.to_path_buf(),+ })+ }++ /// Polls the truth log until `ready` holds (or the child exits).+ fn wait_until(+ &mut self,+ timeout: Duration,+ mut ready: impl FnMut(&[Handed], bool) -> bool,+ ) -> anyhow::Result<()> {+ let deadline = Instant::now() + timeout;+ loop {+ if self.truth.exists() {+ let (handed, stopped, _) = read_truth(&self.truth)?;+ if ready(&handed, stopped) {+ return Ok(());+ }+ }+ if let Some(status) = self.child.try_wait()? {+ bail!("child recorder exited early: {status}");+ }+ if Instant::now() > deadline {+ bail!("child recorder did not reach the kill point in {timeout:?}");+ }+ std::thread::sleep(Duration::from_millis(2));+ }+ }++ /// SIGKILL (TerminateProcess on Windows); returns the cap-muxer child+ /// pids it left behind.+ fn kill(mut self) -> anyhow::Result<Vec<u32>> {+ let orphans = muxer_children(self.child.id());+ self.child.kill()?;+ self.child.wait()?;+ Ok(orphans)+ }+}++/// `cap-muxer` processes whose parent is `pid`.+fn muxer_children(pid: u32) -> Vec<u32> {+ if cfg!(windows) {+ return Vec::new();+ }+ let Ok(out) = Command::new("pgrep")+ .args(["-P", &pid.to_string(), "-x", "cap-muxer"])+ .output()+ else {+ return Vec::new();+ };+ String::from_utf8_lossy(&out.stdout)+ .lines()+ .filter_map(|l| l.trim().parse().ok())+ .collect()+}++fn alive(pid: u32) -> bool {+ Command::new("kill")+ .args(["-0", &pid.to_string()])+ .stderr(Stdio::null())+ .status()+ .is_ok_and(|s| s.success())+}++fn signal(pid: u32, sig: &str) -> anyhow::Result<()> {+ let status = Command::new("kill")+ .args([&format!("-{sig}"), &pid.to_string()])+ .status()?;+ anyhow::ensure!(status.success(), "kill -{sig} {pid} failed");+ Ok(())+}++/// After a recorder crash the orphaned cap-muxer sees EOF and finishes its+/// open segment; recovery happens on the next launch, after it exited.+fn wait_for_exit(pids: &[u32], case: &mut Case) {+ let started = Instant::now();+ for &pid in pids {+ while alive(pid) && started.elapsed() < Duration::from_secs(20) {+ std::thread::sleep(Duration::from_millis(20));+ }+ if alive(pid) {+ case.problem(+ "orphan",+ format!("cap-muxer {pid} still running 20 s after the recorder died"),+ );+ }+ }+ if !pids.is_empty() {+ case.note(format!(+ "orphaned cap-muxer exited {:.2}s after the kill",+ started.elapsed().as_secs_f64()+ ));+ }+}++// ---------------------------------------------------------------------------+// What is physically on disk+// ---------------------------------------------------------------------------++/// Top-level ISO BMFF boxes span the file exactly and include moof + mdat:+/// the writer finished this fragment (independent of recovery.rs).+fn fragment_complete(path: &Path) -> bool {+ let Ok(data) = std::fs::read(path) else {+ return false;+ };+ let mut at = 0usize;+ let (mut moof, mut mdat) = (false, false);+ while at + 8 <= data.len() {+ let size32 = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()) as usize;+ let kind = &data[at + 4..at + 8];+ let size = match size32 {+ 0 => return false,+ 1 => {+ if at + 16 > data.len() {+ return false;+ }+ u64::from_be_bytes(data[at + 8..at + 16].try_into().unwrap()) as usize+ }+ n => n,+ };+ if size < 8 {+ return false;+ }+ moof |= kind == b"moof";+ mdat |= kind == b"mdat";+ at += size;+ }+ at == data.len() && moof && mdat+}++fn segment_index(name: &str) -> Option<u32> {+ name.strip_prefix("segment_")?+ .trim_end_matches(".tmp")+ .strip_suffix(".m4s")?+ .parse()+ .ok()+}++/// Physically complete fragments of one writer directory (finished+/// `segment_NNN.m4s`, or a finished `.m4s.tmp` the writer had not renamed+/// yet), in index order.+fn complete_fragments(dir: &Path) -> Vec<PathBuf> {+ let Ok(entries) = std::fs::read_dir(dir) else {+ return Vec::new();+ };+ let mut found: BTreeMap<u32, PathBuf> = BTreeMap::new();+ for entry in entries.flatten() {+ let path = entry.path();+ let Some(name) = path.file_name().and_then(|n| n.to_str()) else {+ continue;+ };+ let Some(index) = segment_index(name) else {+ continue;+ };+ if !fragment_complete(&path) {+ continue;+ }+ // Prefer the renamed file when both exist.+ if name.ends_with(".tmp") && found.contains_key(&index) {+ continue;+ }+ found.insert(index, path);+ }+ found.into_values().collect()+}++/// Writer directories: the display directory and every `respawn-N` beside+/// it (out-of-process muxer restarts), in order.+fn writer_dirs(display: &Path) -> Vec<PathBuf> {+ let mut dirs = vec![display.to_path_buf()];+ let mut respawns: Vec<(u32, PathBuf)> = std::fs::read_dir(display)+ .into_iter()+ .flatten()+ .flatten()+ .filter_map(|e| {+ let name = e.file_name().to_string_lossy().into_owned();+ let n = name.strip_prefix("respawn-")?.parse().ok()?;+ Some((n, e.path()))+ })+ .collect();+ respawns.sort();+ dirs.extend(respawns.into_iter().map(|(_, p)| p));+ dirs+}++/// Decoded (marker index, presentation seconds) of a set of fragments,+/// joined with their init segment into a temporary file.+fn decode_fragments(+ dir: &Path,+ fragments: &[PathBuf],+ scratch: &Path,+) -> anyhow::Result<Vec<(u32, f64)>> {+ if fragments.is_empty() {+ return Ok(Vec::new());+ }+ let mut joined = std::fs::read(dir.join("init.mp4")).context("init.mp4")?;+ for fragment in fragments {+ joined.extend(std::fs::read(fragment)?);+ }+ let path = scratch.join(format!(+ "joined-{}.mp4",+ dir.file_name().unwrap_or_default().to_string_lossy()+ ));+ std::fs::write(&path, joined)?;+ let decoded = decode_markers(&path);+ let _ = std::fs::remove_file(&path);+ decoded+}++fn decode_markers(path: &Path) -> anyhow::Result<Vec<(u32, f64)>> {+ let mut out = Vec::new();+ let mut unreadable = 0usize;+ media::decode_video(path, |picture| {+ let readout = video_code::read_frame(&picture.view());+ match readout.get(StreamKind::Screen).marker() {+ Some(marker) => out.push((marker.index, picture.pts_secs)),+ None => unreadable += 1,+ }+ })?;+ anyhow::ensure!(+ unreadable == 0,+ "{unreadable} pictures without a readable marker"+ );+ Ok(out)+}++/// Every frame inside the physically complete fragments, per writer+/// directory in order.+fn on_disk(display: &Path, scratch: &Path) -> anyhow::Result<Vec<(u32, f64)>> {+ let mut frames = Vec::new();+ for dir in writer_dirs(display) {+ let fragments = complete_fragments(&dir);+ frames.extend(decode_fragments(&dir, &fragments, scratch)?);+ }+ Ok(frames)+}++// ---------------------------------------------------------------------------+// Oracles+// ---------------------------------------------------------------------------++/// The recovered display: the app's display finalization+/// (`RecoveryManager::finalize_to_progressive_mp4`, the same call for a clean+/// stop and for crash recovery).+fn finalize(display: &Path, out: &Path) -> anyhow::Result<Vec<(u32, f64)>> {+ RecoveryManager::finalize_to_progressive_mp4(display, out)+ .map_err(|e| anyhow!("finalize: {e}"))?;+ decode_markers(out)+}++/// Checks `frames` (decoded index, pts) against the truth: no duplicates,+/// increasing indices, each frame at its capture time relative to the first+/// recovered frame. Returns the recovered index set.+fn check_timeline(case: &mut Case, label: &str, frames: &[(u32, f64)], handed: &[Handed]) {+ let truth: BTreeMap<u32, f64> = handed.iter().map(|h| (h.index, h.secs)).collect();+ let Some(&(first_index, first_pts)) = frames.first() else {+ case.problem("missing", format!("{label}: no frames"));+ return;+ };+ let Some(&first_truth) = truth.get(&first_index) else {+ case.problem(+ "unexpected",+ format!("{label}: frame {first_index} was never handed"),+ );+ return;+ };+ let mut previous: Option<u32> = None;+ let mut worst = (0.0f64, 0u32);+ let mut off = Vec::new();+ for &(index, pts) in frames {+ if let Some(p) = previous+ && index <= p+ {+ case.problem(+ "order",+ format!("{label}: frame {index} after {p} (repeat or reorder)"),+ );+ }+ previous = Some(index);+ let Some(&t) = truth.get(&index) else {+ case.problem(+ "unexpected",+ format!("{label}: frame {index} was never handed"),+ );+ continue;+ };+ let error = (pts - first_pts) - (t - first_truth);+ if error.abs() > worst.0.abs() {+ worst = (error, index);+ }+ if error.abs() > PTS_TOLERANCE {+ off.push(index);+ }+ }+ if !off.is_empty() {+ case.problem(+ "timestamps",+ format!(+ "{label}: {} frames off their capture time by more than {:.1} ms, worst {:+.1} ms at frame {} (first {:?})",+ off.len(),+ PTS_TOLERANCE * 1e3,+ worst.0 * 1e3,+ worst.1,+ &off[..off.len().min(5)]+ ),+ );+ }+ case.note(format!(+ "{label}: {} frames {}..={}, worst pts error {:+.2} ms",+ frames.len(),+ frames.first().map(|f| f.0).unwrap_or(0),+ frames.last().map(|f| f.0).unwrap_or(0),+ worst.0 * 1e3+ ));+}++/// Recovery keeps everything physically complete and loses at most the+/// fragment being written plus the frames in flight, all at the end.+fn check_recovery(+ case: &mut Case,+ recovered: &[(u32, f64)],+ physical: &[(u32, f64)],+ handed: &[Handed],+) {+ check_timeline(case, "recovered", recovered, handed);+ let recovered_set: std::collections::BTreeSet<u32> = recovered.iter().map(|f| f.0).collect();+ let physical_set: std::collections::BTreeSet<u32> = physical.iter().map(|f| f.0).collect();+ let dropped: Vec<u32> = physical_set.difference(&recovered_set).copied().collect();+ if !dropped.is_empty() {+ case.problem(+ "dropped-complete",+ format!(+ "recovery dropped {} frames that were inside complete fragments on disk: {}..={}",+ dropped.len(),+ dropped[0],+ dropped[dropped.len() - 1]+ ),+ );+ }+ let extra: Vec<u32> = recovered_set.difference(&physical_set).copied().collect();+ if !extra.is_empty() {+ case.problem(+ "unexpected",+ format!(+ "recovery produced {} frames that no complete fragment holds: {:?}",+ extra.len(),+ &extra[..extra.len().min(5)]+ ),+ );+ }+ let last_physical = physical.iter().map(|f| f.0).max();+ let lost: Vec<&Handed> = handed+ .iter()+ .filter(|h| last_physical.is_none_or(|p| h.index > p))+ .collect();+ let lost_tail = lost.len() as u32;+ // Capture time the crash lost: from the first lost frame to the last+ // handed one, plus that frame's own period.+ let lost_secs = match (lost.first(), lost.last()) {+ (Some(a), Some(b)) => b.secs - a.secs + 1.0 / f64::from(FPS),+ _ => 0.0,+ };+ let interior: Vec<u32> = handed+ .iter()+ .map(|h| h.index)+ .filter(|i| last_physical.is_some_and(|p| *i < p) && !physical_set.contains(i))+ .collect();+ if !interior.is_empty() {+ case.problem(+ "missing",+ format!(+ "{} handed frames missing before the last complete fragment's end: {:?}",+ interior.len(),+ &interior[..interior.len().min(5)]+ ),+ );+ }+ let bound = GOP + IN_FLIGHT;+ // In time: one fragment of capture time, plus the in-flight frames and+ // the frame that closes the fragment at the delivered interval (a static+ // screen delivers far fewer frames than the nominal rate).+ let interval = handed+ .windows(2)+ .map(|w| w[1].secs - w[0].secs)+ .fold(1.0 / f64::from(FPS), f64::max);+ let bound_secs = GOP_SECS + f64::from(IN_FLIGHT + 1) * interval;+ case.note(format!(+ "handed {} frames, complete on disk up to {:?}, lost tail {lost_tail} frames / {lost_secs:.2}s (bound {bound} frames / {bound_secs:.2}s)",+ handed.len(),+ last_physical+ ));+ if lost_tail > bound {+ case.problem(+ "loss",+ format!(+ "the crash lost the last {lost_tail} handed frames, more than one fragment ({GOP}) plus {IN_FLIGHT} in flight"+ ),+ );+ }+ if lost_secs > bound_secs + 1e-6 {+ case.problem(+ "loss-time",+ format!(+ "the crash lost {lost_secs:.2}s of capture, more than one {GOP_SECS}s fragment plus {} frames in flight ({bound_secs:.2}s at the delivered interval)", IN_FLIGHT + 1+ ),+ );+ }+}++// ---------------------------------------------------------------------------+// Cases+// ---------------------------------------------------------------------------++fn scratch_root() -> PathBuf {+ std::env::var_os("CAP_VERIFY_SCRATCH")+ .map(PathBuf::from)+ .unwrap_or_else(std::env::temp_dir)+ .join("cap-verify-crash")+}++fn fresh(dir: &Path) -> anyhow::Result<()> {+ let _ = std::fs::remove_dir_all(dir);+ std::fs::create_dir_all(dir)?;+ Ok(())+}++/// Clean recordings through both writers must give the same frames at the+/// same times, all of them, at their capture times.+fn equality(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> {+ let scratch = scratch_root().join("equality");+ fresh(&scratch)?;+ let mut results = Vec::new();+ for arm in arms() {+ let display = scratch.join(arm.label()).join("display");+ let run = rt.block_on(record(&display, arm, SECONDS, 1, None, Live::default()))?;+ let out = scratch.join(format!("{}.mp4", arm.label()));+ let frames = finalize(&run.display, &out)?;+ check_timeline(case, arm.label(), &frames, &run.handed);+ let handed: std::collections::BTreeSet<u32> = run.handed.iter().map(|h| h.index).collect();+ let got: std::collections::BTreeSet<u32> = frames.iter().map(|f| f.0).collect();+ let missing: Vec<u32> = handed.difference(&got).copied().collect();+ if !missing.is_empty() {+ case.problem(+ "missing",+ format!(+ "{}: {} of {} handed frames missing after a clean stop: {:?}",+ arm.label(),+ missing.len(),+ handed.len(),+ &missing[..missing.len().min(8)]+ ),+ );+ }+ results.push((arm, frames));+ }+ if let [(_, a), (_, b)] = results.as_slice() {+ let differ = a.len() != b.len()+ || a.iter()+ .zip(b)+ .any(|(x, y)| x.0 != y.0 || (x.1 - y.1).abs() > PTS_TOLERANCE);+ if differ {+ case.problem(+ "unequal",+ format!(+ "in-process and out-of-process outputs differ: {} vs {} frames",+ a.len(),+ b.len()+ ),+ );+ } else {+ case.note(format!("in-process == out-of-process: {} frames", a.len()));+ }+ }+ let _ = std::fs::remove_dir_all(&scratch);+ Ok(())+}++/// Static-screen recording: a new picture every `STATIC_STEP` slots (3 per+/// second at 30 fps), killed after `STATIC_KILL_SECS`.+const STATIC_STEP: u32 = 10;+const STATIC_SECONDS: f64 = 14.0;+const STATIC_KILL_SECS: f64 = 11.0;++/// Kills at 2*GOP + k for k = 0..=12 and reports the first k at which the+/// second fragment was complete on disk.+fn writer_latency(arm: Arm, case: &mut Case) -> anyhow::Result<()> {+ let mut first_complete = None;+ for k in 0..=12u32 {+ let name = format!("latency{k}");+ let scratch = scratch_root().join(format!("kill-{}-{name}", arm.label()));+ fresh(&scratch)?;+ let display = scratch.join("display");+ let truth = scratch.join("truth.txt");+ let mut child = ChildRecorder::spawn(&display, &truth, arm, 5.0, 1)?;+ let n = 2 * GOP + k;+ child.wait_until(Duration::from_secs(40), |handed, _| {+ handed.last().is_some_and(|h| h.index >= n)+ })?;+ let orphans = child.kill()?;+ wait_for_exit(&orphans, case);+ let physical = on_disk(&display, &scratch)?;+ let done = physical.iter().any(|f| f.0 == 2 * GOP - 1);+ case.note(format!(+ "kill after frame {n}: fragment 2 complete = {done}"+ ));+ let _ = std::fs::remove_dir_all(&scratch);+ if done && first_complete.is_none() {+ first_complete = Some(k);+ }+ }+ case.note(format!(+ "{}: fragment complete on disk {first_complete:?} frames after its closing keyframe was handed",+ arm.label()+ ));+ Ok(())+}++/// When the recorder process dies. Indices are handed-frame indices.+#[derive(Clone, Copy, Debug)]+enum KillPoint {+ /// After frame N was handed.+ Frame(u32),+ /// Some milliseconds after the recorder started stopping.+ Finalize(u64),+}++fn kill_points(seed: u64) -> Vec<(String, KillPoint)> {+ let mut rng = Rng::new(seed);+ let total = (SECONDS * f64::from(FPS)) as u32;+ let mut points = Vec::new();+ // Mid-fragment, at seeded random frames past the first fragment.+ for k in 0..2 {+ let lo = GOP + 15;+ let index = lo + (rng.next_u64() % u64::from(total - lo - 10)) as u32;+ points.push((format!("random{k}"), KillPoint::Frame(index)));+ }+ // Around the second rollover: the keyframe that closes fragment 2 is+ // frame 2*GOP; the writer renames the segment when the packet after it+ // arrives (the encoder holds one back), and the manifest learns about+ // it on its next 10-frame scan.+ for offset in [0u32, 2, 5] {+ points.push((+ format!("rollover+{offset}"),+ KillPoint::Frame(2 * GOP + offset),+ ));+ }+ for delay in [0u64, 40] {+ points.push((format!("finalize+{delay}ms"), KillPoint::Finalize(delay)));+ }+ points+}++/// Kills a child recorder at `point` and recovers what it left.+/// `seconds`/`step` shape the recording (see `record`).+fn recorder_kill(+ arm: Arm,+ name: &str,+ point: KillPoint,+ seconds: f64,+ step: u32,+ case: &mut Case,+) -> anyhow::Result<()> {+ let scratch = scratch_root().join(format!("kill-{}-{name}", arm.label()));+ fresh(&scratch)?;+ let display = scratch.join("display");+ let truth = scratch.join("truth.txt");+ let mut child = ChildRecorder::spawn(&display, &truth, arm, seconds, step)?;+ let timeout = Duration::from_secs_f64(seconds + 30.0);+ match point {+ KillPoint::Frame(n) => child.wait_until(timeout, |handed, _| {+ handed.last().is_some_and(|h| h.index >= n)+ })?,+ KillPoint::Finalize(ms) => {+ child.wait_until(timeout, |_, stopped| stopped)?;+ std::thread::sleep(Duration::from_millis(ms));+ }+ }+ let orphans = child.kill()?;+ wait_for_exit(&orphans, case);+ let (handed, stopped, _) = read_truth(&truth)?;+ case.note(format!(+ "killed after frame {:?}{}",+ handed.last().map(|h| h.index),+ if stopped { " while stopping" } else { "" }+ ));+ let physical = on_disk(&display, &scratch)?;+ let recovered = match finalize(&display, &scratch.join("recovered.mp4")) {+ Ok(frames) => frames,+ Err(e) if physical.is_empty() => {+ case.note(format!("nothing complete on disk; recovery: {e:#}"));+ Vec::new()+ }+ Err(e) => return Err(e),+ };+ check_recovery(case, &recovered, &physical, &handed);+ if std::env::var_os("CAP_VERIFY_KEEP").is_none() {+ let _ = std::fs::remove_dir_all(&scratch);+ }+ Ok(())+}++/// Out-of-process only: kill cap-muxer mid-recording. The recording must+/// continue in a respawned muxer, and recovery must keep every complete+/// fragment, at its capture time (a muxer crash must not shift later video+/// against audio).+#[cfg(unix)]+fn muxer_kill(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> {+ let scratch = scratch_root().join("muxer-kill");+ fresh(&scratch)?;+ let display = scratch.join("display");+ let kill_at = GOP + GOP / 2;+ let me = std::process::id();+ let killed = std::sync::Arc::new(std::sync::Mutex::new(None::<u32>));+ let killed_hook = killed.clone();+ let live = Live {+ on_frame: Some(Box::new(move |index| {+ if index == kill_at+ && let Some(&pid) = muxer_children(me).first()+ {+ let _ = signal(pid, "KILL");+ *killed_hook.lock().unwrap() = Some(pid);+ }+ })),+ };+ let run = rt.block_on(record(&display, Arm::OutOfProcess, SECONDS, 1, None, live))?;+ let Some(pid) = *killed.lock().unwrap() else {+ bail!("no cap-muxer child to kill at frame {kill_at}");+ };+ case.note(format!("killed cap-muxer {pid} after frame {kill_at}"));+ let respawns = writer_dirs(&display).len() - 1;+ if respawns == 0 {+ case.problem("respawn", "no respawned muxer after the kill");+ }+ let physical = on_disk(&display, &scratch)?;+ let recovered = finalize(&display, &scratch.join("recovered.mp4"))?;+ check_timeline(case, "recovered", &recovered, &run.handed);+ let recovered_set: std::collections::BTreeSet<u32> = recovered.iter().map(|f| f.0).collect();+ let dropped: Vec<u32> = physical+ .iter()+ .map(|f| f.0)+ .filter(|i| !recovered_set.contains(i))+ .collect();+ if !dropped.is_empty() {+ case.problem(+ "dropped-complete",+ format!(+ "finalize dropped {} frames inside complete fragments",+ dropped.len()+ ),+ );+ }+ // Loss: from the last frame the dead muxer finished to the first frame+ // the respawned one kept (the next keyframe): at most the killed+ // fragment plus one GOP until the encoder's next keyframe.+ let indices: Vec<u32> = recovered.iter().map(|f| f.0).collect();+ let gap = indices+ .windows(2)+ .map(|w| w[1] - w[0] - 1)+ .max()+ .unwrap_or(0);+ let last = run.handed.last().map(|h| h.index).unwrap_or(0);+ let tail = last.saturating_sub(*indices.last().unwrap_or(&0));+ case.note(format!(+ "{respawns} respawn(s); recovered {} of {} frames, largest gap {gap} frames, tail {tail}",+ recovered.len(),+ run.handed.len()+ ));+ if gap > 2 * GOP + IN_FLIGHT {+ case.problem(+ "loss",+ format!(+ "a muxer crash lost {gap} consecutive frames ({:.2}s), more than two fragments plus {IN_FLIGHT} in flight",+ f64::from(gap) / f64::from(FPS)+ ),+ );+ }+ if tail > IN_FLIGHT {+ case.problem(+ "loss",+ format!("the respawned muxer lost the last {tail} frames of a clean stop"),+ );+ }+ let _ = std::fs::remove_dir_all(&scratch);+ Ok(())+}++#[test]+fn crash_safety() {+ if std::env::var(CHILD_ENV).is_ok() {+ return;+ }+ common::init();+ if cfg!(target_os = "macos") && std::env::var_os("CAP_MUXER_BIN").is_none() {+ // Built next to the test binary by `cargo build -p cap-muxer`.+ if let Some(bin) = std::env::current_exe()+ .ok()+ .and_then(|exe| Some(exe.parent()?.parent()?.join("cap-muxer")))+ .filter(|p| p.exists())+ {+ let _ = cap_recording::oop_muxer::set_muxer_binary_override(bin);+ }+ }+ let rt = common::runtime();+ let tier = report::tier();+ let mut failures = Vec::new();+ let mut settle = |case: Case| {+ for note in &case.notes {+ eprintln!(" note: {note}");+ }+ if let Err(e) = report::settle(case) {+ failures.push(e);+ }+ };++ let oop_ready = cap_recording::oop_muxer::resolve_muxer_binary();++ if cfg!(target_os = "macos") && report::selected("crash/equality") {+ let mut case = Case::new("crash/equality");+ if let Err(e) = &oop_ready {+ case.problem("harness", format!("cap-muxer not found: {e:#}"));+ } else if let Err(e) = equality(&rt, &mut case) {+ case.problem("error", format!("{e:#}"));+ }+ settle(case);+ }++ #[cfg(unix)]+ if cfg!(target_os = "macos") && report::selected("crash/muxer-kill/oop") {+ let mut case = Case::new("crash/muxer-kill/oop");+ if let Err(e) = muxer_kill(&rt, &mut case) {+ case.problem("error", format!("{e:#}"));+ }+ settle(case);+ }++ let seed: u64 = std::env::var("CAP_VERIFY_CRASH_SEED")+ .ok()+ .and_then(|s| s.parse().ok())+ .unwrap_or(20261016);+ for arm in arms() {+ for (name, point) in kill_points(seed) {+ let id = format!("crash/recorder-kill/{}/{name}", arm.label());+ if !report::selected(&id) || (tier == Tier::Fast && name.starts_with("random1")) {+ continue;+ }+ let mut case = Case::new(&id);+ case.note(format!("seed {seed}, point {point:?}"));+ if let Err(e) = recorder_kill(arm, &name, point, SECONDS, 1, &mut case) {+ case.problem("error", format!("{e:#}"));+ }+ settle(case);+ }+ // A mostly static screen: one frame every STATIC_STEP slots. A+ // fragment must still close every GOP_SECS of capture time.+ let id = format!("crash/recorder-kill/{}/static-screen", arm.label());+ if report::selected(&id) {+ let mut case = Case::new(&id);+ let kill = (STATIC_KILL_SECS * f64::from(FPS)) as u32;+ if let Err(e) = recorder_kill(+ arm,+ "static",+ KillPoint::Frame(kill),+ STATIC_SECONDS,+ STATIC_STEP,+ &mut case,+ ) {+ case.problem("error", format!("{e:#}"));+ }+ settle(case);+ }+ // Writer latency: how many frames after the keyframe that closes a+ // fragment the fragment is complete on disk (measurement only).+ let id = format!("crash/writer-latency/{}", arm.label());+ if tier == Tier::Full && report::selected(&id) {+ let mut case = Case::new(&id);+ if let Err(e) = writer_latency(arm, &mut case) {+ case.problem("error", format!("{e:#}"));+ }+ settle(case);+ }+ }+ drop(settle);+ assert!(failures.is_empty(), "{}", failures.join("\n"));+}