Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
//! Crash safety of fragmented Studio recording: the out-of-process muxer
//! against the in-process one, and recovery after the recorder, the muxer or
//! the disk fails.
//!
//! Synthetic marker frames go through the product's Studio display muxers.
//! macOS: the in-process `MacOSFragmentedM4SMuxer` and the out-of-process
//! `OutOfProcessFragmentedM4SMuxer` (a `cap-muxer` child), fed NV12 IOSurface
//! sample buffers like ScreenCaptureKit delivers. Linux and Windows: the
//! shared `SegmentedVideoEncoder` writer through `ffmpeg::SegmentedVideoMuxer`
//! (the Linux Studio display muxer; on Windows the writer behind
//! `WindowsFragmentedM4SMuxer`, whose Direct3D capture frames cannot be
//! synthesized, so Windows has no out-of-process case here).
//!
//! Every frame handed to the pipeline is logged (index, capture time) line by
//! line, so the truth survives a SIGKILL of the recorder. Recovery is judged
//! against what is physically on disk after the failure: every fragment whose
//! box structure is complete must come back frame for frame at its capture
//! time. Frames lost to a failure must be the newest ones and fit in the
//! fragment being written: one GOP (`GOP_SECS` x fps frames, the encoder's
//! keyframe interval, so the longest a segment can be when frames arrive at
//! the nominal rate) plus the writer's frames still inside the pipeline.
mod common;
use std::{
collections::BTreeMap,
io::Write,
path::{Path, PathBuf},
process::{Child, Command, Stdio},
time::{Duration, Instant},
};
use anyhow::{Context, anyhow, bail};
use cap_media_info::{RawVideoFormat, VideoInfo};
use cap_recording::{
ChannelVideoSource, ChannelVideoSourceConfig, OutputPipeline, recovery::RecoveryManager,
};
use cap_timestamp::{Timestamp, Timestamps};
use cap_verify::{
fixture::Rng,
media,
report::{self, Case, Tier},
video_code::{self, FrameMarker, StreamKind},
};
use tracing_subscriber::fmt::writer::MakeWriterExt;
const FPS: u32 = 30;
const WIDTH: u32 = 640;
const HEIGHT: u32 = 360;
/// `DEFAULT_KEYFRAME_INTERVAL_SECS` in crates/enc-ffmpeg/src/video/h264.rs:
/// the encoder emits a keyframe every `GOP_SECS * fps` frames and both DASH
/// writers cut a segment only at a keyframe.
const GOP_SECS: f64 = 2.0;
const GOP: u32 = (GOP_SECS as u32) * FPS;
/// Frames that can still be inside the pipeline when the process dies: the
/// one the encoder holds back to stamp real durations (segmented_stream.rs
/// `queue_frame`), the one being encoded, and the one in the muxer channel.
/// The emitter runs at the nominal rate on an idle encoder, so the channel
/// holds at most one frame at a time.
const IN_FLIGHT: u32 = 3;
fn recorder_in_flight(arm: Arm, encoder: &str) -> u32 {
// VideoToolbox can retain four frames before emitting their packets,
// in addition to the duration holder, active encode and channel slots.
// The OOP writer drains its received packets after recorder EOF.
IN_FLIGHT + u32::from(encoder == "h264_videotoolbox" && arm == Arm::InProcess) * 4
}
fn selected_encoder(log: &str) -> Option<&str> {
log.lines().find_map(|line| {
if line.contains("Selected hardware H264 encoder with zero-copy VideoToolbox input") {
Some("h264_videotoolbox")
} else {
[
"Selected hardware H264 encoder ",
"Using SOFTWARE H264 encoder (high CPU usage expected) ",
"Using SOFTWARE encoder for high throughput (hardware cannot keep up at this resolution/fps) ",
]
.into_iter()
.find_map(|message| line.split_once(message))
.and_then(|(_, fields)| {
fields
.split_whitespace()
.find_map(|field| field.strip_prefix("encoder="))
})
}
})
}
/// Recorded length of every case.
const SECONDS: f64 = 8.0;
/// Truth timestamps vs presentation times: the encoder time base and the
/// container round to at most 1/90000 s; half a millisecond is far below the
/// 33 ms frame period and above any rounding.
const PTS_TOLERANCE: f64 = 0.0005;
const CHILD_ENV: &str = "CAP_VERIFY_CRASH_CHILD";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Arm {
/// The default in-process writer.
InProcess,
/// `out_of_process_muxer` on (macOS only here).
OutOfProcess,
}
impl Arm {
fn label(self) -> &'static str {
match self {
Arm::InProcess => "inproc",
Arm::OutOfProcess => "oop",
}
}
fn parse(text: &str) -> Option<Self> {
match text {
"inproc" => Some(Arm::InProcess),
"oop" => Some(Arm::OutOfProcess),
_ => None,
}
}
}
fn arms() -> Vec<Arm> {
if cfg!(target_os = "macos") {
vec![Arm::InProcess, Arm::OutOfProcess]
} else {
vec![Arm::InProcess]
}
}
/// One frame the emitter handed to the pipeline.
#[derive(Clone, Copy, Debug)]
struct Handed {
index: u32,
/// Seconds after the first handed frame.
secs: f64,
}
/// What a recording run left behind: the truth and the display directory.
struct Run {
handed: Vec<Handed>,
display: PathBuf,
}
#[cfg(target_os = "macos")]
mod source {
use super::*;
use cap_recording::{
MacOSFragmentedM4SMuxer, MacOSFragmentedM4SMuxerConfig, OutOfProcessFragmentedM4SMuxer,
OutOfProcessFragmentedM4SMuxerConfig, screen_capture,
};
use cidre::cm;
pub type Frame = screen_capture::VideoFrame;
pub fn info() -> VideoInfo {
VideoInfo::from_raw(RawVideoFormat::Nv12, WIDTH, HEIGHT, FPS)
}
pub type Picture = cidre::arc::R<cm::SampleBuf>;
/// Renders ahead of time so emission keeps the nominal rate.
pub fn picture(index: u32) -> anyhow::Result<Picture> {
picture_sized(index, WIDTH, HEIGHT, FPS)
}
pub fn picture_sized(index: u32, width: u32, height: u32, fps: u32) -> anyhow::Result<Picture> {
let rgba = video_code::marker_rgba(
FrameMarker::new(StreamKind::Screen, 0, index),
width as usize,
height as usize,
);
let buf = cap_verify::fixture_avf::pixel_buffer(width as usize, height as usize, &rgba)?;
let format_desc = cm::VideoFormatDesc::with_image_buf(&buf)
.map_err(|e| anyhow!("format description: {e:?}"))?;
let timing = cm::SampleTimingInfo {
duration: cm::Time::new(1, fps as i32),
pts: cm::Time::new(i64::from(index), fps as i32),
dts: cm::Time::invalid(),
};
cm::SampleBuf::with_image_buf(&buf, true, None, std::ptr::null(), &format_desc, &timing)
.map_err(|e| anyhow!("sample buffer: {e:?}"))
}
pub fn frame(picture: Picture, at: Instant) -> Frame {
screen_capture::VideoFrame {
sample_buf: picture,
timestamp: Timestamp::Instant(at),
}
}
pub async fn pipeline(
display: &Path,
arm: Arm,
rx: flume::Receiver<Frame>,
timestamps: Timestamps,
) -> anyhow::Result<OutputPipeline> {
pipeline_with(display, arm, rx, timestamps, info()).await
}
pub async fn pipeline_with(
display: &Path,
arm: Arm,
rx: flume::Receiver<Frame>,
timestamps: Timestamps,
info: VideoInfo,
) -> anyhow::Result<OutputPipeline> {
let builder = OutputPipeline::builder(display.to_path_buf())
.with_video::<ChannelVideoSource<Frame>>(ChannelVideoSourceConfig::new(info, rx))
.with_timestamps(timestamps);
match arm {
Arm::InProcess => {
builder
.build::<ObservedMuxer<MacOSFragmentedM4SMuxer>>(
MacOSFragmentedM4SMuxerConfig {
timed_fragments: true,
..Default::default()
},
)
.await
}
Arm::OutOfProcess => {
builder
.build::<ObservedMuxer<OutOfProcessFragmentedM4SMuxer>>(
OutOfProcessFragmentedM4SMuxerConfig::default(),
)
.await
}
}
}
}
#[cfg(not(target_os = "macos"))]
mod source {
use super::*;
use cap_recording::ffmpeg::{FFmpegVideoFrame, SegmentedVideoMuxer, SegmentedVideoMuxerConfig};
pub type Frame = FFmpegVideoFrame;
pub fn info() -> VideoInfo {
VideoInfo::from_raw(RawVideoFormat::Bgra, WIDTH, HEIGHT, FPS)
}
pub type Picture = ffmpeg::frame::Video;
/// Renders ahead of time so emission keeps the nominal rate.
pub fn picture(index: u32) -> anyhow::Result<Picture> {
let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, WIDTH, HEIGHT);
let stride = frame.stride(0);
video_code::draw_marker(
FrameMarker::new(StreamKind::Screen, 0, index),
frame.data_mut(0),
WIDTH as usize,
HEIGHT as usize,
stride,
video_code::PixelOrder::Bgra,
);
Ok(frame)
}
pub fn frame(picture: Picture, at: Instant) -> Frame {
FFmpegVideoFrame {
inner: picture,
timestamp: Timestamp::Instant(at),
}
}
pub async fn pipeline(
display: &Path,
arm: Arm,
rx: flume::Receiver<Frame>,
timestamps: Timestamps,
) -> anyhow::Result<OutputPipeline> {
anyhow::ensure!(arm == Arm::InProcess, "no out-of-process muxer here");
OutputPipeline::builder(display.to_path_buf())
.with_video::<ChannelVideoSource<Frame>>(ChannelVideoSourceConfig::new(info(), rx))
.with_timestamps(timestamps)
// The Studio display config (capture_pipeline.rs, X11Capture):
// 2 s segments.
.build::<ObservedMuxer<SegmentedVideoMuxer>>(SegmentedVideoMuxerConfig {
timed_fragments: true,
segment_duration: Duration::from_secs(2),
..Default::default()
})
.await
}
}
struct ObservedMuxer<M> {
inner: M,
output_path: PathBuf,
}
impl<M: cap_recording::Muxer> cap_recording::Muxer for ObservedMuxer<M>
where
M::Config: Send,
{
type Config = M::Config;
async fn setup(
config: Self::Config,
output_path: PathBuf,
video_config: Option<VideoInfo>,
audio_config: Option<cap_media_info::AudioInfo>,
pause_flag: std::sync::Arc<std::sync::atomic::AtomicBool>,
tasks: &mut cap_recording::TaskPool,
) -> anyhow::Result<Self> {
let inner = M::setup(
config,
output_path.clone(),
video_config,
audio_config,
pause_flag,
tasks,
)
.await?;
Ok(Self { inner, output_path })
}
fn stop(&mut self) {
self.inner.stop();
std::fs::write(self.output_path.join("mux-video-stopped"), b"stopped")
.expect("write EOF acknowledgement");
}
fn finish(&mut self, timestamp: Duration) -> anyhow::Result<anyhow::Result<()>> {
finalization_barrier(&self.output_path)?;
self.inner.finish(timestamp)
}
fn set_health_sender(&mut self, tx: cap_recording::HealthSender) {
self.inner.set_health_sender(tx);
}
}
impl<M: cap_recording::VideoMuxer<VideoFrame = source::Frame>> cap_recording::VideoMuxer
for ObservedMuxer<M>
where
M::Config: Send,
{
type VideoFrame = source::Frame;
fn send_video_frame(
&mut self,
frame: Self::VideoFrame,
timestamp: Duration,
) -> anyhow::Result<()> {
self.inner.send_video_frame(frame, timestamp)
}
}
impl<M: cap_recording::AudioMuxer> cap_recording::AudioMuxer for ObservedMuxer<M>
where
M::Config: Send,
{
fn send_audio_frame(
&mut self,
frame: cap_recording::AudioFrame,
timestamp: Duration,
) -> anyhow::Result<()> {
self.inner.send_audio_frame(frame, timestamp)
}
}
fn finalization_barrier(display: &Path) -> anyhow::Result<()> {
let hold = display.join("hold-finalization");
if !hold.exists() {
return Ok(());
}
std::fs::write(display.join("finalization-entered"), b"entered")?;
let deadline = Instant::now() + Duration::from_secs(45);
while hold.exists() {
anyhow::ensure!(Instant::now() < deadline, "finalization barrier timed out");
std::thread::sleep(Duration::from_millis(2));
}
Ok(())
}
/// Line-per-event truth log that a SIGKILL cannot lose (no buffering).
struct TruthLog(Option<std::fs::File>);
impl TruthLog {
fn line(&mut self, text: &str) {
if let Some(file) = &mut self.0 {
let _ = file.write_all(format!("{text}\n").as_bytes());
}
}
}
/// Hooks a case runs while the recorder is live.
#[derive(Default)]
struct Live {
/// Called with every handed index (after it was handed).
on_frame: Option<Box<dyn FnMut(u32) + Send>>,
}
/// Records `seconds` of marker frames into `display` and stops like the
/// recorder (`OutputPipeline::stop`).
/// Every `step`-th slot of the nominal rate carries a frame (1 = steady; more
/// is a mostly static screen, where ScreenCaptureKit delivers only changes).
async fn record(
display: &Path,
arm: Arm,
seconds: f64,
step: u32,
truth: Option<&Path>,
mut live: Live,
) -> anyhow::Result<Run> {
std::fs::create_dir_all(display)?;
let mut log = TruthLog(match truth {
Some(path) => Some(std::fs::File::create(path)?),
None => None,
});
let count = (seconds * f64::from(FPS)) as u32;
let slots: Vec<u32> = (0..count).step_by(step.max(1) as usize).collect();
let mut pictures = slots
.iter()
.copied()
.map(source::picture)
.collect::<anyhow::Result<Vec<_>>>()?
.into_iter();
let timestamps = Timestamps::now();
let (tx, rx) = flume::bounded::<source::Frame>(8);
let mut pipeline = source::pipeline(display, arm, rx, timestamps).await?;
let mut health = pipeline.take_health_rx();
let base = Instant::now() + Duration::from_millis(50);
let mut handed = Vec::new();
for index in slots {
let at = base + Duration::from_secs_f64(f64::from(index) / f64::from(FPS));
tokio::time::sleep_until(at.into()).await;
let picture = pictures.next().expect("rendered");
let frame = source::frame(picture, at);
tx.send_async(frame)
.await
.map_err(|_| anyhow!("pipeline stopped taking frames at {index}"))?;
let secs = f64::from(index) / f64::from(FPS);
handed.push(Handed { index, secs });
log.line(&format!("f {index} {secs:.6}"));
if let Some(hook) = live.on_frame.as_mut() {
hook(index);
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
drop(tx);
tokio::time::timeout(Duration::from_secs(5), async {
while !display.join("mux-video-stopped").exists() {
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.context("waiting for mux-video EOF")?;
log.line("stop");
let finished = pipeline.stop().await;
if let Some(health) = health.as_mut() {
while let Ok(event) = health.try_recv() {
log.line(&format!("health {event:?}"));
}
}
finished.map_err(|e| anyhow!("stop: {e:#}"))?;
log.line("done");
Ok(Run {
handed,
display: display.to_path_buf(),
})
}
fn read_truth(path: &Path) -> anyhow::Result<(Vec<Handed>, bool, Vec<String>)> {
let text = std::fs::read_to_string(path).with_context(|| format!("read {path:?}"))?;
let mut handed = Vec::new();
let mut stopped = false;
let mut health = Vec::new();
for line in text.lines() {
let mut parts = line.split(' ');
match parts.next() {
Some("f") => {
let (Some(index), Some(secs)) = (parts.next(), parts.next()) else {
continue;
};
// A kill can cut the last line short.
if let (Ok(index), Ok(secs)) = (index.parse(), secs.parse()) {
if line.len() > 2 && secs_complete(line) {
handed.push(Handed { index, secs });
}
}
}
Some("stop") => stopped = true,
Some("health") => health.push(line[7..].to_string()),
_ => {}
}
}
Ok((handed, stopped, health))
}
/// The writer prints 6 decimals; a shorter field was cut by the kill.
fn secs_complete(line: &str) -> bool {
line.rsplit(' ')
.next()
.and_then(|s| s.split('.').nth(1))
.is_some_and(|d| d.len() == 6)
}
/// Runs only as a child of `crash_safety` (`CAP_VERIFY_CRASH_CHILD` set):
/// records into the given directory until it is killed or finishes.
#[test]
fn crash_child_recorder() {
let Ok(spec) = std::env::var(CHILD_ENV) else {
return;
};
let fields: BTreeMap<&str, &str> = spec
.split(';')
.filter_map(|kv| kv.split_once('='))
.collect();
let arm = Arm::parse(fields["arm"]).expect("arm");
let display = PathBuf::from(fields["display"]);
let truth = PathBuf::from(fields["truth"]);
let seconds: f64 = fields["seconds"].parse().expect("seconds");
let step: u32 = fields["step"].parse().expect("step");
let encoder_log = std::fs::File::create(truth.with_extension("encoder.log"))
.expect("create recorder encoder log");
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::new(std::env::var("CAP_VERIFY_LOG").unwrap_or_default())
.add_directive(
"cap_enc_ffmpeg::video::h264=info"
.parse()
.expect("encoder filter"),
),
)
.without_time()
.with_level(false)
.with_target(false)
.with_ansi(false)
.with_writer(encoder_log.and(std::io::stderr))
.try_init()
.expect("capture recorder encoder selection");
common::init();
if arm == Arm::OutOfProcess {
cap_recording::oop_muxer::resolve_muxer_binary().expect("child recorder: cap-muxer binary");
}
let rt = common::runtime();
rt.block_on(record(
&display,
arm,
seconds,
step,
Some(&truth),
Live::default(),
))
.expect("child recording");
}
struct ChildRecorder {
child: Child,
truth: PathBuf,
}
impl ChildRecorder {
fn spawn(
display: &Path,
truth: &Path,
arm: Arm,
seconds: f64,
step: u32,
) -> anyhow::Result<Self> {
let exe = std::env::current_exe()?;
let mut command = Command::new(exe);
if let Ok(muxer) = cap_recording::oop_muxer::resolve_muxer_binary() {
command.env(cap_recording::oop_muxer::ENV_BIN_PATH, muxer);
}
let child = command
.args([
"crash_child_recorder",
"--exact",
"--nocapture",
"--test-threads=1",
])
.env(
CHILD_ENV,
format!(
"arm={};display={};truth={};seconds={seconds};step={step}",
arm.label(),
display.display(),
truth.display()
),
)
.stdout(Stdio::null())
.stderr(Stdio::inherit())
.spawn()?;
Ok(Self {
child,
truth: truth.to_path_buf(),
})
}
/// Polls the truth log until `ready` holds (or the child exits).
fn wait_until(
&mut self,
timeout: Duration,
mut ready: impl FnMut(&[Handed], bool) -> bool,
) -> anyhow::Result<()> {
let deadline = Instant::now() + timeout;
loop {
if self.truth.exists() {
let (handed, stopped, _) = read_truth(&self.truth)?;
if ready(&handed, stopped) {
return Ok(());
}
}
if let Some(status) = self.child.try_wait()? {
bail!("child recorder exited early: {status}");
}
if Instant::now() > deadline {
bail!("child recorder did not reach the kill point in {timeout:?}");
}
std::thread::sleep(Duration::from_millis(2));
}
}
/// SIGKILL (TerminateProcess on Windows); returns the cap-muxer child
/// pids it left behind.
fn kill(mut self) -> anyhow::Result<Vec<u32>> {
let orphans = muxer_children(self.child.id());
self.child.kill()?;
self.child.wait()?;
Ok(orphans)
}
}
impl Drop for ChildRecorder {
fn drop(&mut self) {
if !matches!(self.child.try_wait(), Ok(Some(_))) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
}
/// `cap-muxer` processes whose parent is `pid`.
fn muxer_children(pid: u32) -> Vec<u32> {
if cfg!(windows) {
return Vec::new();
}
let Ok(out) = Command::new("pgrep")
.args(["-P", &pid.to_string(), "-x", "cap-muxer"])
.output()
else {
return Vec::new();
};
String::from_utf8_lossy(&out.stdout)
.lines()
.filter_map(|l| l.trim().parse().ok())
.collect()
}
fn alive(pid: u32) -> bool {
Command::new("kill")
.args(["-0", &pid.to_string()])
.stderr(Stdio::null())
.status()
.is_ok_and(|s| s.success())
}
fn signal(pid: u32, sig: &str) -> anyhow::Result<()> {
let status = Command::new("kill")
.args([&format!("-{sig}"), &pid.to_string()])
.status()?;
anyhow::ensure!(status.success(), "kill -{sig} {pid} failed");
Ok(())
}
/// After a recorder crash the orphaned cap-muxer sees EOF and finishes its
/// open segment; recovery happens on the next launch, after it exited.
fn wait_for_exit(pids: &[u32], case: &mut Case) {
let started = Instant::now();
for &pid in pids {
while alive(pid) && started.elapsed() < Duration::from_secs(20) {
std::thread::sleep(Duration::from_millis(20));
}
if alive(pid) {
case.problem(
"orphan",
format!("cap-muxer {pid} still running 20 s after the recorder died"),
);
}
}
if !pids.is_empty() {
case.note(format!(
"orphaned cap-muxer exited {:.2}s after the kill",
started.elapsed().as_secs_f64()
));
}
}
/// Top-level ISO BMFF boxes span the file exactly and include moof + mdat:
/// the writer finished this fragment (independent of recovery.rs).
fn fragment_complete(path: &Path) -> bool {
let Ok(data) = std::fs::read(path) else {
return false;
};
let mut at = 0usize;
let (mut moof, mut mdat) = (false, false);
while at + 8 <= data.len() {
let size32 = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()) as usize;
let kind = &data[at + 4..at + 8];
let size = match size32 {
0 => return false,
1 => {
if at + 16 > data.len() {
return false;
}
u64::from_be_bytes(data[at + 8..at + 16].try_into().unwrap()) as usize
}
n => n,
};
if size < 8 {
return false;
}
moof |= kind == b"moof";
mdat |= kind == b"mdat";
at += size;
}
at == data.len() && moof && mdat
}
// A segment can contain several moof/mdat fragments and end mid-write.
// This oracle deliberately does not use the recovery parser being tested.
fn complete_fragment_ends(data: &[u8]) -> Vec<usize> {
let mut ends = Vec::new();
let mut at = 0usize;
let mut pending_moof = false;
while let Some(header) = data.get(at..).and_then(|tail| tail.get(..8)) {
let size32 = u32::from_be_bytes(header[..4].try_into().unwrap());
let (size, header_len) = match size32 {
0 => break,
1 => {
let Some(extended) = data.get(at..).and_then(|tail| tail.get(8..16)) else {
break;
};
let Ok(size) = usize::try_from(u64::from_be_bytes(extended.try_into().unwrap()))
else {
break;
};
(size, 16)
}
size => (size as usize, 8),
};
let Some(end) = at.checked_add(size).filter(|end| *end <= data.len()) else {
break;
};
if size < header_len {
break;
}
match &header[4..8] {
b"moof" if pending_moof => break,
b"moof" => pending_moof = true,
b"mdat" if pending_moof => {
ends.push(end);
pending_moof = false;
}
b"mdat" => break,
_ => {}
}
at = end;
}
ends
}
fn segment_index(name: &str) -> Option<u32> {
name.strip_prefix("segment_")?
.trim_end_matches(".tmp")
.strip_suffix(".m4s")?
.parse()
.ok()
}
/// Segments with at least one complete on-disk fragment, including a
/// complete prefix of an interrupted `.m4s.tmp`, in index order.
fn complete_fragments(dir: &Path) -> Vec<PathBuf> {
let Ok(entries) = std::fs::read_dir(dir) else {
return Vec::new();
};
let mut found: BTreeMap<u32, PathBuf> = BTreeMap::new();
for entry in entries.flatten() {
let path = entry.path();
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
let Some(index) = segment_index(name) else {
continue;
};
if std::fs::read(&path)
.map(|data| complete_fragment_ends(&data).is_empty())
.unwrap_or(true)
{
continue;
}
// Prefer the renamed file when both exist.
if name.ends_with(".tmp") && found.contains_key(&index) {
continue;
}
found.insert(index, path);
}
found.into_values().collect()
}
/// Writer directories: the display directory and every `respawn-N` beside
/// it (out-of-process muxer restarts), in order.
fn writer_dirs(display: &Path) -> Vec<PathBuf> {
let mut dirs = vec![display.to_path_buf()];
let mut respawns: Vec<(u32, PathBuf)> = std::fs::read_dir(display)
.into_iter()
.flatten()
.flatten()
.filter_map(|e| {
let name = e.file_name().to_string_lossy().into_owned();
let n = name.strip_prefix("respawn-")?.parse().ok()?;
Some((n, e.path()))
})
.collect();
respawns.sort();
dirs.extend(respawns.into_iter().map(|(_, p)| p));
dirs
}
/// Decoded (marker index, presentation seconds) of a set of fragments,
/// joined with their init segment into a temporary file.
fn decode_fragments(
dir: &Path,
fragments: &[PathBuf],
scratch: &Path,
) -> anyhow::Result<Vec<(u32, f64)>> {
if fragments.is_empty() {
return Ok(Vec::new());
}
let mut joined = std::fs::read(dir.join("init.mp4")).context("init.mp4")?;
for fragment in fragments {
let data = std::fs::read(fragment)?;
if let Some(&end) = complete_fragment_ends(&data).last() {
joined.extend_from_slice(&data[..end]);
}
}
let path = scratch.join(format!(
"joined-{}.mp4",
dir.file_name().unwrap_or_default().to_string_lossy()
));
std::fs::write(&path, joined)?;
let decoded = decode_markers(&path);
let _ = std::fs::remove_file(&path);
decoded
}
fn decode_markers(path: &Path) -> anyhow::Result<Vec<(u32, f64)>> {
let mut out = Vec::new();
let mut unreadable = 0usize;
media::decode_video(path, |picture| {
let readout = video_code::read_frame(&picture.view());
match readout.get(StreamKind::Screen).marker() {
Some(marker) => out.push((marker.index, picture.pts_secs)),
None => unreadable += 1,
}
})?;
anyhow::ensure!(
unreadable == 0,
"{unreadable} pictures without a readable marker"
);
Ok(out)
}
/// The frames of every physically complete fragment, per fragment, writer
/// directories in order.
fn fragments_on_disk(display: &Path, scratch: &Path) -> anyhow::Result<Vec<Vec<(u32, f64)>>> {
let mut out = Vec::new();
for dir in writer_dirs(display) {
for fragment in complete_fragments(&dir) {
let data = std::fs::read(&fragment)?;
let init = std::fs::read(dir.join("init.mp4"))?;
let path = scratch.join("fragment-prefix.mp4");
let mut previous = Vec::new();
for end in complete_fragment_ends(&data) {
std::fs::write(&path, [&init[..], &data[..end]].concat())?;
let frames = decode_markers(&path)?;
anyhow::ensure!(
frames.starts_with(&previous),
"decoding a longer fragment prefix changed earlier frames"
);
out.push(frames[previous.len()..].to_vec());
previous = frames;
}
let _ = std::fs::remove_file(&path);
}
}
Ok(out)
}
fn pending_fragment_frames(
display: &Path,
scratch: &Path,
) -> anyhow::Result<std::collections::BTreeSet<u32>> {
let mut frames = std::collections::BTreeSet::new();
for dir in writer_dirs(display) {
let Ok(bytes) = std::fs::read(dir.join("manifest.json")) else {
continue;
};
let manifest: serde_json::Value = serde_json::from_slice(&bytes)?;
let entries = manifest
.get("segments")
.or_else(|| manifest.get("fragments"));
let Some(entries) = entries.and_then(serde_json::Value::as_array) else {
continue;
};
for fragment in complete_fragments(&dir) {
let Some(index) = fragment
.file_name()
.and_then(|n| n.to_str())
.and_then(segment_index)
else {
continue;
};
let pending = entries.iter().any(|entry| {
entry
.get("is_complete")
.and_then(serde_json::Value::as_bool)
== Some(false)
&& entry
.get("path")
.and_then(serde_json::Value::as_str)
.and_then(|path| Path::new(path).file_name())
.and_then(|name| name.to_str())
.and_then(segment_index)
== Some(index)
});
if pending {
frames.extend(
decode_fragments(&dir, &[fragment], scratch)?
.into_iter()
.map(|frame| frame.0),
);
}
}
}
Ok(frames)
}
/// Every frame inside the physically complete fragments, in order.
fn on_disk(display: &Path, scratch: &Path) -> anyhow::Result<Vec<(u32, f64)>> {
Ok(fragments_on_disk(display, scratch)?.concat())
}
/// A timed fragment covers capture time from its first frame to the first
/// frame of the next one, and the writer must flush a new one once
/// `GOP_SECS` have passed: the span may exceed `GOP_SECS` only by the
/// interval before the frame that closed it.
fn check_fragment_spans(case: &mut Case, fragments: &[Vec<(u32, f64)>], handed: &[Handed]) {
let time: BTreeMap<u32, f64> = handed.iter().map(|h| (h.index, h.secs)).collect();
let next_after = |index: u32| handed.iter().find(|h| h.index > index).map(|h| h.secs);
let mut worst = 0.0f64;
for fragment in fragments {
let (Some(first), Some(last)) = (fragment.first(), fragment.last()) else {
continue;
};
let (Some(&start), Some(&end)) = (time.get(&first.0), time.get(&last.0)) else {
continue;
};
let Some(next) = next_after(last.0) else {
continue;
};
let span = next - start;
worst = worst.max(span);
if span > GOP_SECS + (next - end) + 1e-6 {
case.problem(
"fragment-span",
format!(
"fragment of frames {}..={} covers {span:.2}s of capture, more than {GOP_SECS}s plus the {:.2}s before the frame that closed it",
first.0,
last.0,
next - end
),
);
}
}
case.note(format!(
"longest complete fragment covers {worst:.2}s of capture"
));
}
/// Out-of-process only: after the recorder died, the orphaned cap-muxer
/// sees EOF and must finish the fragment it had open (nothing it received
/// may be lost).
fn check_open_fragment(case: &mut Case, display: &Path) {
for dir in writer_dirs(display) {
for entry in std::fs::read_dir(&dir).into_iter().flatten().flatten() {
let path = entry.path();
let name = entry.file_name().to_string_lossy().into_owned();
if name.starts_with("segment_")
&& name.ends_with(".m4s.tmp")
&& !fragment_complete(&path)
{
let size = entry.metadata().map(|m| m.len()).unwrap_or(0);
case.problem(
"open-fragment-lost",
format!(
"the orphaned cap-muxer left {name} incomplete ({size} bytes): the packets it had received are lost"
),
);
}
}
}
}
/// The recovered display: the app's display finalization
/// (`RecoveryManager::finalize_to_progressive_mp4`, the same call for a clean
/// stop and for crash recovery).
fn finalize(display: &Path, out: &Path) -> anyhow::Result<Vec<(u32, f64)>> {
RecoveryManager::finalize_to_progressive_mp4(display, out)
.map_err(|e| anyhow!("finalize: {e}"))?;
decode_markers(out)
}
/// Checks `frames` (decoded index, pts) against the truth: no duplicates,
/// increasing indices, each frame at its capture time relative to the first
/// recovered frame. Returns the recovered index set.
fn check_timeline(case: &mut Case, label: &str, frames: &[(u32, f64)], handed: &[Handed]) {
let truth: BTreeMap<u32, f64> = handed.iter().map(|h| (h.index, h.secs)).collect();
let Some(&(first_index, first_pts)) = frames.first() else {
case.problem("missing", format!("{label}: no frames"));
return;
};
let Some(&first_truth) = truth.get(&first_index) else {
case.problem(
"unexpected",
format!("{label}: frame {first_index} was never handed"),
);
return;
};
let mut previous: Option<u32> = None;
let mut worst = (0.0f64, 0u32);
let mut off = Vec::new();
for &(index, pts) in frames {
if let Some(p) = previous
&& index <= p
{
case.problem(
"order",
format!("{label}: frame {index} after {p} (repeat or reorder)"),
);
}
previous = Some(index);
let Some(&t) = truth.get(&index) else {
case.problem(
"unexpected",
format!("{label}: frame {index} was never handed"),
);
continue;
};
let error = (pts - first_pts) - (t - first_truth);
if error.abs() > worst.0.abs() {
worst = (error, index);
}
if error.abs() > PTS_TOLERANCE {
off.push(index);
}
}
if !off.is_empty() {
case.problem(
"timestamps",
format!(
"{label}: {} frames off their capture time by more than {:.1} ms, worst {:+.1} ms at frame {} (first {:?})",
off.len(),
PTS_TOLERANCE * 1e3,
worst.0 * 1e3,
worst.1,
&off[..off.len().min(5)]
),
);
}
case.note(format!(
"{label}: {} frames {}..={}, worst pts error {:+.2} ms",
frames.len(),
frames.first().map(|f| f.0).unwrap_or(0),
frames.last().map(|f| f.0).unwrap_or(0),
worst.0 * 1e3
));
}
/// Recovery keeps everything physically complete and loses at most the
/// fragment being written plus the frames in flight, all at the end.
fn check_recovery(
case: &mut Case,
in_flight: u32,
recovered: &[(u32, f64)],
physical: &[(u32, f64)],
handed: &[Handed],
pending: &std::collections::BTreeSet<u32>,
) {
check_timeline(case, "recovered", recovered, handed);
let recovered_set: std::collections::BTreeSet<u32> = recovered.iter().map(|f| f.0).collect();
let physical_set: std::collections::BTreeSet<u32> = physical.iter().map(|f| f.0).collect();
let dropped: Vec<u32> = physical_set.difference(&recovered_set).copied().collect();
if !dropped.is_empty() {
case.count("dropped-complete", "frames", dropped.len() as f64);
case.count(
"dropped-complete",
"outside-pending",
dropped
.iter()
.filter(|index| !pending.contains(index))
.count() as f64,
);
case.problem(
"dropped-complete",
format!(
"recovery dropped {} frames that were inside complete fragments on disk: {}..={}",
dropped.len(),
dropped[0],
dropped[dropped.len() - 1]
),
);
}
let extra: Vec<u32> = recovered_set.difference(&physical_set).copied().collect();
if !extra.is_empty() {
case.problem(
"unexpected",
format!(
"recovery produced {} frames that no complete fragment holds: {:?}",
extra.len(),
&extra[..extra.len().min(5)]
),
);
}
let last_physical = physical.iter().map(|f| f.0).max();
let lost: Vec<&Handed> = handed
.iter()
.filter(|h| last_physical.is_none_or(|p| h.index > p))
.collect();
let lost_tail = lost.len() as u32;
// Capture time the crash lost: from the first lost frame to the last
// handed one, plus that frame's own period.
let lost_secs = match (lost.first(), lost.last()) {
(Some(a), Some(b)) => b.secs - a.secs + 1.0 / f64::from(FPS),
_ => 0.0,
};
let interior: Vec<u32> = handed
.iter()
.map(|h| h.index)
.filter(|i| last_physical.is_some_and(|p| *i < p) && !physical_set.contains(i))
.collect();
if !interior.is_empty() {
case.count("missing", "frames", interior.len() as f64);
case.problem(
"missing",
format!(
"{} handed frames missing before the last complete fragment's end: {:?}",
interior.len(),
&interior[..interior.len().min(5)]
),
);
}
let bound = GOP + in_flight;
// In time: one fragment of capture time, plus the in-flight frames and
// the frame that closes the fragment at the delivered interval (a static
// screen delivers far fewer frames than the nominal rate).
let interval = handed
.windows(2)
.map(|w| w[1].secs - w[0].secs)
.fold(1.0 / f64::from(FPS), f64::max);
let bound_secs = GOP_SECS + f64::from(in_flight + 1) * interval;
case.note(format!(
"handed {} frames, complete on disk up to {:?}, lost tail {lost_tail} frames / {lost_secs:.2}s (bound {bound} frames / {bound_secs:.2}s)",
handed.len(),
last_physical
));
if lost_tail > bound {
case.measure("loss", "frames", f64::from(lost_tail));
case.problem(
"loss",
format!(
"the crash lost the last {lost_tail} handed frames, more than one fragment ({GOP}) plus {in_flight} in flight"
),
);
}
if lost_secs > bound_secs + 1e-6 {
case.measure("loss-time", "secs", (lost_secs * 100.0).round() / 100.0);
case.problem(
"loss-time",
format!(
"the crash lost {lost_secs:.2}s of capture, more than one {GOP_SECS}s fragment plus {} frames in flight ({bound_secs:.2}s at the delivered interval)", in_flight + 1
),
);
}
}
fn scratch_root() -> PathBuf {
std::env::var_os("CAP_VERIFY_SCRATCH")
.map(PathBuf::from)
.unwrap_or_else(std::env::temp_dir)
.join("cap-verify-crash")
}
fn fresh(dir: &Path) -> anyhow::Result<()> {
let _ = std::fs::remove_dir_all(dir);
std::fs::create_dir_all(dir)?;
Ok(())
}
/// Clean recordings through both writers must give the same frames at the
/// same times, all of them, at their capture times.
fn equality(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> {
let scratch = scratch_root().join("equality");
fresh(&scratch)?;
let mut results = Vec::new();
for arm in arms() {
let display = scratch.join(arm.label()).join("display");
let run = rt.block_on(record(&display, arm, SECONDS, 1, None, Live::default()))?;
let out = scratch.join(format!("{}.mp4", arm.label()));
let frames = finalize(&run.display, &out)?;
check_timeline(case, arm.label(), &frames, &run.handed);
let handed: std::collections::BTreeSet<u32> = run.handed.iter().map(|h| h.index).collect();
let got: std::collections::BTreeSet<u32> = frames.iter().map(|f| f.0).collect();
let missing: Vec<u32> = handed.difference(&got).copied().collect();
if !missing.is_empty() {
case.problem(
"missing",
format!(
"{}: {} of {} handed frames missing after a clean stop: {:?}",
arm.label(),
missing.len(),
handed.len(),
&missing[..missing.len().min(8)]
),
);
}
results.push((arm, frames));
}
if let [(_, a), (_, b)] = results.as_slice() {
let differ = a.len() != b.len()
|| a.iter()
.zip(b)
.any(|(x, y)| x.0 != y.0 || (x.1 - y.1).abs() > PTS_TOLERANCE);
if differ {
case.problem(
"unequal",
format!(
"in-process and out-of-process outputs differ: {} vs {} frames",
a.len(),
b.len()
),
);
} else {
case.note(format!("in-process == out-of-process: {} frames", a.len()));
}
}
let _ = std::fs::remove_dir_all(&scratch);
Ok(())
}
/// Static-screen recording: a new picture every `STATIC_STEP` slots (3 per
/// second at 30 fps), killed after `STATIC_KILL_SECS`.
const STATIC_STEP: u32 = 10;
const STATIC_SECONDS: f64 = 14.0;
const STATIC_KILL_SECS: f64 = 11.0;
/// Kills at 2*GOP + k for k = 0..=12 and reports the first k at which the
/// second fragment was complete on disk.
fn writer_latency(arm: Arm, case: &mut Case) -> anyhow::Result<()> {
let mut first_complete = None;
for k in 0..=12u32 {
let name = format!("latency{k}");
let scratch = scratch_root().join(format!("kill-{}-{name}", arm.label()));
fresh(&scratch)?;
let display = scratch.join("display");
let truth = scratch.join("truth.txt");
let mut child = ChildRecorder::spawn(&display, &truth, arm, 5.0, 1)?;
let n = 2 * GOP + k;
child.wait_until(Duration::from_secs(40), |handed, _| {
handed.last().is_some_and(|h| h.index >= n)
})?;
let orphans = child.kill()?;
wait_for_exit(&orphans, case);
let physical = on_disk(&display, &scratch)?;
let done = physical.iter().any(|f| f.0 == 2 * GOP - 1);
case.note(format!(
"kill after frame {n}: fragment 2 complete = {done}"
));
let _ = std::fs::remove_dir_all(&scratch);
if done && first_complete.is_none() {
first_complete = Some(k);
}
}
case.note(format!(
"{}: fragment complete on disk {first_complete:?} frames after its closing keyframe was handed",
arm.label()
));
Ok(())
}
/// When the recorder process dies. Indices are handed-frame indices.
#[derive(Clone, Copy, Debug)]
enum KillPoint {
/// After frame N was handed.
Frame(u32),
/// Some milliseconds after the recorder started stopping.
Finalize(u64),
}
fn kill_points(seed: u64) -> Vec<(String, KillPoint)> {
let mut rng = Rng::new(seed);
let total = (SECONDS * f64::from(FPS)) as u32;
let mut points = Vec::new();
// Mid-fragment, at seeded random frames past the first fragment.
for k in 0..2 {
let lo = GOP + 15;
let index = lo + (rng.next_u64() % u64::from(total - lo - 10)) as u32;
points.push((format!("random{k}"), KillPoint::Frame(index)));
}
// Around the second rollover: the keyframe that closes fragment 2 is
// frame 2*GOP; the writer renames the segment when the packet after it
// arrives (the encoder holds one back), and the manifest learns about
// it on its next 10-frame scan.
for offset in [0u32, 2, 5] {
points.push((
format!("rollover+{offset}"),
KillPoint::Frame(2 * GOP + offset),
));
}
for delay in [0u64, 40] {
points.push((format!("finalize+{delay}ms"), KillPoint::Finalize(delay)));
}
points
}
/// Kills a child recorder at `point` and recovers what it left.
/// `seconds`/`step` shape the recording (see `record`).
fn recorder_kill(
arm: Arm,
name: &str,
point: KillPoint,
seconds: f64,
step: u32,
case: &mut Case,
) -> anyhow::Result<()> {
let scratch = scratch_root().join(format!("kill-{}-{name}", arm.label()));
fresh(&scratch)?;
let display = scratch.join("display");
let truth = scratch.join("truth.txt");
if matches!(point, KillPoint::Finalize(_)) {
std::fs::create_dir_all(&display)?;
std::fs::write(display.join("hold-finalization"), b"hold")?;
}
let mut child = ChildRecorder::spawn(&display, &truth, arm, seconds, step)?;
let timeout = Duration::from_secs_f64(seconds + 30.0);
match point {
KillPoint::Frame(n) => child.wait_until(timeout, |handed, _| {
handed.last().is_some_and(|h| h.index >= n)
})?,
KillPoint::Finalize(ms) => {
child.wait_until(timeout, |_, stopped| {
stopped && display.join("finalization-entered").exists()
})?;
std::thread::sleep(Duration::from_millis(ms));
}
}
let orphans = child.kill()?;
wait_for_exit(&orphans, case);
let (handed, stopped, _) = read_truth(&truth)?;
if matches!(point, KillPoint::Finalize(_)) {
check_finalize_coverage(case, &display, &truth)?;
}
case.note(format!(
"killed after frame {:?}{}",
handed.last().map(|h| h.index),
if stopped { " while stopping" } else { "" }
));
if arm == Arm::OutOfProcess {
check_open_fragment(case, &display);
}
let fragments = fragments_on_disk(&display, &scratch)?;
// Studio flushes timed fragments; the OOP writer instead drains its
// keyframe-delimited segment after the recorder exits (#2646, #2644).
if arm == Arm::InProcess {
check_fragment_spans(case, &fragments, &handed);
}
let physical = fragments.concat();
let pending = pending_fragment_frames(&display, &scratch)?;
let recovered = match finalize(&display, &scratch.join("recovered.mp4")) {
Ok(frames) => frames,
Err(e) if physical.is_empty() => {
case.note(format!("nothing complete on disk; recovery: {e:#}"));
Vec::new()
}
Err(e) => return Err(e),
};
let in_flight = if arm == Arm::InProcess {
let log = std::fs::read_to_string(truth.with_extension("encoder.log"))?;
let encoder =
selected_encoder(&log).context("recorder did not report its selected encoder")?;
case.note(format!("recorder encoder: {encoder}"));
recorder_in_flight(arm, encoder)
} else {
IN_FLIGHT
};
check_recovery(case, in_flight, &recovered, &physical, &handed, &pending);
if std::env::var_os("CAP_VERIFY_KEEP").is_none() {
let _ = std::fs::remove_dir_all(&scratch);
}
Ok(())
}
/// Out-of-process only: kill cap-muxer mid-recording. The recording must
/// continue in a respawned muxer, and recovery must keep every complete
/// fragment, at its capture time (a muxer crash must not shift later video
/// against audio).
#[cfg(unix)]
fn muxer_kill(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> {
let scratch = scratch_root().join("muxer-kill");
fresh(&scratch)?;
let display = scratch.join("display");
let kill_at = GOP + GOP / 2;
let me = std::process::id();
let killed = std::sync::Arc::new(std::sync::Mutex::new(None::<u32>));
let killed_hook = killed.clone();
let live = Live {
on_frame: Some(Box::new(move |index| {
if index == kill_at
&& let Some(&pid) = muxer_children(me).first()
{
let _ = signal(pid, "KILL");
*killed_hook.lock().unwrap() = Some(pid);
}
})),
};
let run = rt.block_on(record(&display, Arm::OutOfProcess, SECONDS, 1, None, live))?;
let Some(pid) = *killed.lock().unwrap() else {
bail!("no cap-muxer child to kill at frame {kill_at}");
};
case.note(format!("killed cap-muxer {pid} after frame {kill_at}"));
let respawns = writer_dirs(&display).len() - 1;
if respawns == 0 {
case.problem("respawn", "no respawned muxer after the kill");
}
let physical = on_disk(&display, &scratch)?;
let recovered = finalize(&display, &scratch.join("recovered.mp4"))?;
check_timeline(case, "recovered", &recovered, &run.handed);
let recovered_set: std::collections::BTreeSet<u32> = recovered.iter().map(|f| f.0).collect();
let dropped: Vec<u32> = physical
.iter()
.map(|f| f.0)
.filter(|i| !recovered_set.contains(i))
.collect();
if !dropped.is_empty() {
case.count("dropped-complete", "frames", dropped.len() as f64);
case.problem(
"dropped-complete",
format!(
"finalize dropped {} frames inside complete fragments",
dropped.len()
),
);
}
// Loss: from the last frame the dead muxer finished to the first frame
// the respawned one kept (the next keyframe): at most the killed
// fragment plus one GOP until the encoder's next keyframe.
let indices: Vec<u32> = recovered.iter().map(|f| f.0).collect();
let gap = indices
.windows(2)
.map(|w| w[1] - w[0] - 1)
.max()
.unwrap_or(0);
let last = run.handed.last().map(|h| h.index).unwrap_or(0);
let tail = last.saturating_sub(*indices.last().unwrap_or(&0));
case.note(format!(
"{respawns} respawn(s); recovered {} of {} frames, largest gap {gap} frames, tail {tail}",
recovered.len(),
run.handed.len()
));
if gap > 2 * GOP + IN_FLIGHT {
case.problem(
"loss",
format!(
"a muxer crash lost {gap} consecutive frames ({:.2}s), more than two fragments plus {IN_FLIGHT} in flight",
f64::from(gap) / f64::from(FPS)
),
);
}
if tail > IN_FLIGHT {
case.problem(
"loss",
format!("the respawned muxer lost the last {tail} frames of a clean stop"),
);
}
let _ = std::fs::remove_dir_all(&scratch);
Ok(())
}
/// CPU seconds and resident memory of a process (`ps`).
#[cfg(target_os = "macos")]
fn process_usage(pid: u32) -> Option<(f64, u64)> {
let out = Command::new("ps")
.args(["-o", "time=,rss=", "-p", &pid.to_string()])
.output()
.ok()?;
let text = String::from_utf8_lossy(&out.stdout);
let mut fields = text.split_whitespace();
let time = fields.next()?;
let rss: u64 = fields.next()?.parse().ok()?;
let mut secs = 0.0;
for part in time.split(':') {
secs = secs * 60.0 + part.parse::<f64>().ok()?;
}
Some((secs, rss * 1024))
}
/// Total CPU seconds of every VideoToolbox encoder service process (the
/// hardware encoder's host process; shared by every client on the machine).
#[cfg(target_os = "macos")]
fn encoder_service_cpu() -> f64 {
let Ok(out) = Command::new("pgrep")
.args(["-x", "VTEncoderXPCService"])
.output()
else {
return 0.0;
};
String::from_utf8_lossy(&out.stdout)
.lines()
.filter_map(|l| l.trim().parse().ok())
.filter_map(process_usage)
.map(|u| u.0)
.sum()
}
/// Overhead of the out-of-process muxer at a real Studio size: 1920x1080
/// NV12 at 60 fps for `PERF_SECONDS`, interleaved A/B rounds. Reports CPU
/// seconds of the recorder and of cap-muxer per recorded second and their
/// peak resident memory (measurement only; the default decision reads it).
#[cfg(target_os = "macos")]
const PERF_SECONDS: f64 = 10.0;
#[cfg(target_os = "macos")]
fn overhead(rt: &tokio::runtime::Runtime, case: &mut Case) -> anyhow::Result<()> {
let (width, height, fps) = (1920u32, 1080u32, 60u32);
let ring: Vec<source::Picture> = (0..fps)
.map(|i| source::picture_sized(i, width, height, fps))
.collect::<anyhow::Result<_>>()?;
let me = std::process::id();
let mut results: BTreeMap<&str, Vec<(f64, f64, u64, u64)>> = BTreeMap::new();
for round in 0..3 {
let order = if round % 2 == 0 {
[Arm::InProcess, Arm::OutOfProcess]
} else {
[Arm::OutOfProcess, Arm::InProcess]
};
for arm in order {
let scratch = scratch_root().join(format!("perf-{}-{round}", arm.label()));
fresh(&scratch)?;
let display = scratch.join("display");
let ring = ring.clone();
let sample = rt.block_on(async {
let info = VideoInfo::from_raw(RawVideoFormat::Nv12, width, height, fps);
let (tx, rx) = flume::bounded::<source::Frame>(8);
let pipeline =
source::pipeline_with(&display, arm, rx, Timestamps::now(), info).await?;
let before = process_usage(me).map(|u| u.0).unwrap_or(0.0);
let service_before = encoder_service_cpu();
let base = Instant::now() + Duration::from_millis(50);
let count = (PERF_SECONDS * f64::from(fps)) as u32;
let (mut peak_self, mut peak_muxer, mut muxer_cpu) = (0u64, 0u64, 0.0f64);
for index in 0..count {
let at = base + Duration::from_secs_f64(f64::from(index) / f64::from(fps));
tokio::time::sleep_until(at.into()).await;
let picture = ring[(index % fps) as usize].clone();
tx.send_async(source::frame(picture, at))
.await
.map_err(|_| anyhow!("pipeline stopped"))?;
if index % (fps / 2) == 0 {
if let Some((_, rss)) = process_usage(me) {
peak_self = peak_self.max(rss);
}
for pid in muxer_children(me) {
if let Some((cpu, rss)) = process_usage(pid) {
peak_muxer = peak_muxer.max(rss);
muxer_cpu = muxer_cpu.max(cpu);
}
}
}
}
for pid in muxer_children(me) {
if let Some((cpu, _)) = process_usage(pid) {
muxer_cpu = muxer_cpu.max(cpu);
}
}
drop(tx);
pipeline.stop().await.map_err(|e| anyhow!("stop: {e:#}"))?;
let after = process_usage(me).map(|u| u.0).unwrap_or(0.0);
let service = (encoder_service_cpu() - service_before).max(0.0);
anyhow::Ok((
(after - before) / PERF_SECONDS,
muxer_cpu / PERF_SECONDS + service / PERF_SECONDS,
peak_self,
peak_muxer,
))
})?;
case.note(format!(
"round {round} {}: recorder {:.3} CPU s/s, cap-muxer + VideoToolbox service {:.3} CPU s/s, recorder peak {} MB, cap-muxer peak {} MB",
arm.label(),
sample.0,
sample.1,
sample.2 / 1_000_000,
sample.3 / 1_000_000
));
results.entry(arm.label()).or_default().push(sample);
let _ = std::fs::remove_dir_all(&scratch);
}
}
for (arm, samples) in &results {
let mut total: Vec<f64> = samples.iter().map(|s| s.0 + s.1).collect();
total.sort_by(f64::total_cmp);
case.note(format!(
"{arm}: median total CPU {:.3} s per recorded second (min {:.3}, max {:.3})",
total[total.len() / 2],
total[0],
total[total.len() - 1]
));
}
Ok(())
}
#[test]
fn crash_safety() {
if std::env::var(CHILD_ENV).is_ok() {
return;
}
common::init();
if cfg!(target_os = "macos") && std::env::var_os("CAP_MUXER_BIN").is_none() {
// Built next to the test binary by `cargo build -p cap-muxer`.
if let Some(bin) = std::env::current_exe()
.ok()
.and_then(|exe| Some(exe.parent()?.parent()?.join("cap-muxer")))
.filter(|p| p.exists())
{
let _ = cap_recording::oop_muxer::set_muxer_binary_override(bin);
}
}
let rt = common::runtime();
let tier = report::tier();
let mut failures = Vec::new();
let mut settle = |case: Case| {
for note in &case.notes {
eprintln!(" note: {note}");
}
if let Err(e) = report::settle(case) {
failures.push(e);
}
};
let oop_ready = cap_recording::oop_muxer::resolve_muxer_binary();
if cfg!(target_os = "macos") && report::selected("crash/equality") {
let mut case = Case::new("crash/equality");
if let Err(e) = &oop_ready {
case.problem("harness", format!("cap-muxer not found: {e:#}"));
} else if let Err(e) = equality(&rt, &mut case) {
case.problem("error", format!("{e:#}"));
}
settle(case);
}
#[cfg(target_os = "macos")]
if tier == Tier::Full && report::selected("crash/oop-overhead") {
let mut case = Case::new("crash/oop-overhead");
if let Err(e) = overhead(&rt, &mut case) {
case.problem("error", format!("{e:#}"));
}
settle(case);
}
#[cfg(unix)]
if cfg!(target_os = "macos") && report::selected("crash/muxer-kill/oop") {
let mut case = Case::new("crash/muxer-kill/oop");
if let Err(e) = muxer_kill(&rt, &mut case) {
case.problem("error", format!("{e:#}"));
}
settle(case);
}
let seed: u64 = std::env::var("CAP_VERIFY_CRASH_SEED")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(20261016);
for arm in arms() {
for (name, point) in kill_points(seed) {
let id = format!("crash/recorder-kill/{}/{name}", arm.label());
if !report::selected(&id) || (tier == Tier::Fast && name.starts_with("random1")) {
continue;
}
let mut case = Case::new(&id);
case.note(format!("seed {seed}, point {point:?}"));
if let Err(e) = recorder_kill(arm, &name, point, SECONDS, 1, &mut case) {
case.problem("error", format!("{e:#}"));
}
settle(case);
}
// A mostly static screen still has the same crash-loss budget.
let id = format!("crash/recorder-kill/{}/static-screen", arm.label());
if report::selected(&id) {
let mut case = Case::new(&id);
let kill = (STATIC_KILL_SECS * f64::from(FPS)) as u32;
if let Err(e) = recorder_kill(
arm,
"static",
KillPoint::Frame(kill),
STATIC_SECONDS,
STATIC_STEP,
&mut case,
) {
case.problem("error", format!("{e:#}"));
}
settle(case);
}
// Writer latency: how many frames after the keyframe that closes a
// fragment the fragment is complete on disk (measurement only).
let id = format!("crash/writer-latency/{}", arm.label());
if tier == Tier::Full && report::selected(&id) {
let mut case = Case::new(&id);
if let Err(e) = writer_latency(arm, &mut case) {
case.problem("error", format!("{e:#}"));
}
settle(case);
}
}
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
fn check_finalize_coverage(case: &mut Case, display: &Path, truth: &Path) -> anyhow::Result<()> {
let done = std::fs::read_to_string(truth)?
.lines()
.any(|line| line == "done");
let entered = display.join("finalization-entered").is_file();
let held = display.join("hold-finalization").is_file();
case.note(format!(
"finalization coverage: entered={entered}, held={held}, done={done}"
));
if done || !entered || !held {
case.problem("harness", "kill missed the held finalization window");
}
Ok(())
}
#[test]
fn finalize_barrier_is_default_off_and_blocks_until_released() {
let dir = tempfile::tempdir().unwrap();
finalization_barrier(dir.path()).unwrap();
assert!(!dir.path().join("finalization-entered").exists());
std::fs::write(dir.path().join("hold-finalization"), b"hold").unwrap();
let display = dir.path().to_path_buf();
let (tx, rx) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
finalization_barrier(&display).unwrap();
tx.send(()).unwrap();
});
let deadline = Instant::now() + Duration::from_secs(5);
while !dir.path().join("finalization-entered").exists() {
assert!(Instant::now() < deadline);
std::thread::sleep(Duration::from_millis(1));
}
assert!(rx.try_recv().is_err());
std::fs::remove_file(dir.path().join("hold-finalization")).unwrap();
rx.recv_timeout(Duration::from_secs(5)).unwrap();
handle.join().unwrap();
}
#[test]
fn finalize_done_is_a_coverage_miss() {
let dir = tempfile::tempdir().unwrap();
let truth = dir.path().join("truth.txt");
std::fs::write(&truth, b"stop\ndone\n").unwrap();
std::fs::write(dir.path().join("finalization-entered"), b"entered").unwrap();
std::fs::write(dir.path().join("hold-finalization"), b"hold").unwrap();
let mut case = Case::new("finalize-coverage-test");
check_finalize_coverage(&mut case, dir.path(), &truth).unwrap();
assert!(case.classes.contains("harness"));
}
#[test]
fn duplicate_and_extra_frames_still_fail_after_eof() {
let handed = vec![
Handed {
index: 0,
secs: 0.0,
},
Handed {
index: 1,
secs: 1.0 / f64::from(FPS),
},
];
for frames in [
vec![(0, 0.0), (1, 1.0 / 30.0), (1, 2.0 / 30.0)],
vec![(0, 0.0), (1, 1.0 / 30.0), (2, 2.0 / 30.0)],
] {
let mut case = Case::new("eof-mutation-test");
check_timeline(&mut case, "mutated", &frames, &handed);
assert!(case.classes.contains("order") || case.classes.contains("unexpected"));
}
}
#[test]
fn pending_fragment_bound_rejects_other_complete_frame_loss() {
let handed = vec![
Handed {
index: 0,
secs: 0.0,
},
Handed {
index: 1,
secs: 1.0 / f64::from(FPS),
},
];
let physical = vec![(0, 0.0), (1, 1.0 / f64::from(FPS))];
// The bound the pending-fragment (#2640) entry used: frames inside
// complete pending fragments only.
let bound = cap_verify_expect::Bound::from([
("count".into(), cap_verify_expect::Range::at(1.0)),
(
"frames".into(),
cap_verify_expect::Range {
min: 1.0,
max: 60.0,
},
),
("outside-pending".into(), cap_verify_expect::Range::at(0.0)),
]);
for (outside, accepted) in [(0.0, true), (1.0, false)] {
let metrics = BTreeMap::from([
("count".into(), cap_verify_expect::Range::at(1.0)),
("frames".into(), cap_verify_expect::Range::at(1.0)),
(
"outside-pending".into(),
cap_verify_expect::Range::at(outside),
),
]);
assert_eq!(
cap_verify_expect::within(&bound, Some(&metrics)).is_ok(),
accepted
);
}
let mut case = Case::new("pending-fragment-test");
check_recovery(
&mut case,
IN_FLIGHT,
&physical[..1],
&physical,
&handed,
&std::collections::BTreeSet::from([1]),
);
assert!(case.classes.contains("dropped-complete"));
assert!(!case.classes.contains("loss"));
}
#[test]
fn recorder_encoder_selection_ignores_failed_candidates() {
let failed = "Hardware H264 encoder failed pre-flight self-test, trying next candidate encoder=h264_videotoolbox";
assert_eq!(selected_encoder(failed), None);
for (selection, expected) in [
(
"Selected hardware H264 encoder with zero-copy VideoToolbox input width=640",
"h264_videotoolbox",
),
(
"Selected hardware H264 encoder encoder=h264_videotoolbox width=640",
"h264_videotoolbox",
),
(
"Using SOFTWARE H264 encoder (high CPU usage expected) encoder=libx264 width=640",
"libx264",
),
(
"Using SOFTWARE encoder for high throughput (hardware cannot keep up at this resolution/fps) encoder=libx264 width=640",
"libx264",
),
] {
assert_eq!(
selected_encoder(&format!("{failed}\nrecording: {selection}\n")),
Some(expected)
);
}
}
#[test]
fn recorder_loss_budget_accounts_for_codec_buffering_without_hiding_excess_loss() {
let physical: Vec<_> = (0..120).map(|i| (i, f64::from(i) / 30.0)).collect();
for arm in [Arm::InProcess, Arm::OutOfProcess] {
for encoder in ["libx264", "h264_videotoolbox"] {
for lost in [63, 64, 65, 67, 68, 69] {
let handed: Vec<_> = (0..120 + lost)
.map(|index| Handed {
index,
secs: f64::from(index) / 30.0,
})
.collect();
let mut case = Case::new("recorder-loss-budget");
check_recovery(
&mut case,
recorder_in_flight(arm, encoder),
&physical,
&physical,
&handed,
&Default::default(),
);
let videotoolbox = encoder == "h264_videotoolbox" && arm == Arm::InProcess;
assert_eq!(
case.classes.contains("loss"),
if videotoolbox { lost >= 68 } else { lost >= 64 },
"{arm:?}, {encoder}, {lost} lost frames"
);
assert_eq!(
case.classes.contains("loss-time"),
if videotoolbox { lost >= 69 } else { lost >= 65 },
"{arm:?}, {encoder}, {lost} lost frames"
);
assert!(!case.classes.contains("dropped-complete"));
}
}
}
}
#[test]
fn complete_prefix_keeps_whole_fragments_before_every_truncated_tail() {
let atom = |kind: &[u8; 4], payload: &[u8]| {
let mut bytes = ((payload.len() + 8) as u32).to_be_bytes().to_vec();
bytes.extend_from_slice(kind);
bytes.extend_from_slice(payload);
bytes
};
let first = [atom(b"moof", &[1, 2]), atom(b"mdat", &[3, 4, 5])].concat();
let second = [atom(b"moof", &[6]), atom(b"mdat", &[7, 8, 9])].concat();
for cut in 0..first.len() {
assert!(complete_fragment_ends(&first[..cut]).is_empty(), "{cut}");
}
for cut in 0..second.len() {
let bytes = [&first[..], &second[..cut]].concat();
assert_eq!(complete_fragment_ends(&bytes), vec![first.len()], "{cut}");
}
let joined = [first.clone(), second].concat();
assert_eq!(
complete_fragment_ends(&joined),
vec![first.len(), joined.len()]
);
for tail in [
vec![0; 8],
[4u32.to_be_bytes().as_slice(), b"moof"].concat(),
[u32::MAX.to_be_bytes().as_slice(), b"moof"].concat(),
[atom(b"moof", &[]), atom(b"moof", &[]), atom(b"mdat", &[])].concat(),
] {
assert_eq!(
complete_fragment_ends(&[first.clone(), tail].concat()),
vec![first.len()]
);
}
let extended = [
atom(b"moof", &[]),
[
1u32.to_be_bytes().as_slice(),
b"mdat",
19u64.to_be_bytes().as_slice(),
&[1, 2, 3],
]
.concat(),
]
.concat();
assert_eq!(complete_fragment_ends(&extended), vec![extended.len()]);
}
#[test]
fn timed_fragment_span_still_rejects_a_delayed_flush() {
let handed: Vec<_> = (0..=120)
.step_by(10)
.map(|index| Handed {
index,
secs: f64::from(index) / f64::from(FPS),
})
.collect();
for (count, fails) in [(6, false), (12, true)] {
let fragment = handed[..count].iter().map(|h| (h.index, h.secs)).collect();
let mut case = Case::new("timed-fragment-test");
check_fragment_spans(&mut case, &[fragment], &handed);
assert_eq!(case.classes.contains("fragment-span"), fails);
}
}