Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
#![allow(dead_code)]
use std::path::{Path, PathBuf};
use cap_project::{
ClipTransition, ClipTransitionType, ProjectConfiguration, RecordingMeta, TimelineSegment, XY,
};
use cap_verify::{
fixture::{self, FixtureSpec, SourceTruth},
media,
model::ReferenceModel,
oracle::{self, StreamExpectation},
report::{self, Case},
video_code::{self, FrameReadout, StreamKind},
};
pub fn runtime() -> tokio::runtime::Runtime {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.thread_stack_size(16 * 1024 * 1024)
.enable_all()
.build()
.expect("runtime")
}
/// `CAP_VERIFY_LOG=<filter>` (EnvFilter syntax, e.g. `wgpu_hal=warn,cap_rendering=info`)
/// prints product and wgpu logs to stderr, for triage on remote runners.
pub fn init() {
media::init();
static LOG: std::sync::Once = std::sync::Once::new();
LOG.call_once(|| {
if let Ok(filter) = std::env::var("CAP_VERIFY_LOG") {
let _ = tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::new(filter))
.with_writer(std::io::stderr)
.try_init();
}
});
}
/// A ready fixture project plus its decoded source truth.
pub struct Project {
pub name: String,
pub path: PathBuf,
pub meta: RecordingMeta,
pub truth: SourceTruth,
}
pub fn project(spec: &FixtureSpec) -> Project {
init();
let path = fixture::ensure_fixture(spec, fixture::write_fixture)
.unwrap_or_else(|e| panic!("fixture {}: {e:?}", spec.name));
load(&spec.name, path)
}
pub fn load(name: &str, path: PathBuf) -> Project {
let meta = RecordingMeta::load_for_project(&path).expect("fixture meta");
let truth = SourceTruth::read(&path).expect("fixture truth");
Project {
name: name.to_string(),
path,
meta,
truth,
}
}
/// A fixture recorded through the real recording pipeline (real time on
/// first use, cached afterwards).
pub fn recorded(spec: &cap_verify::pipeline::RecordedSpec) -> Project {
init();
let path = fixture::ensure_fixture_keyed(spec.cache_key(), |dir| {
runtime().block_on(cap_verify::pipeline::record_studio(spec, dir))
})
.unwrap_or_else(|e| panic!("recorded fixture {}: {e:?}", spec.name));
load(&format!("rec-{}", spec.name), path)
}
impl Project {
pub fn config(&self) -> ProjectConfiguration {
self.meta.project_config()
}
pub fn model(&self, config: &ProjectConfiguration) -> ReferenceModel {
ReferenceModel::new(&self.meta, config, self.truth.clone()).expect("model")
}
}
/// Replaces the timeline with `(start, end, timescale)` cuts of clip 0.
pub fn set_cuts(config: &mut ProjectConfiguration, cuts: &[(f64, f64, f64)]) {
let timeline = config.timeline.as_mut().expect("timeline");
timeline.segments = cuts
.iter()
.map(|&(start, end, timescale)| TimelineSegment {
recording_clip: 0,
start,
end,
timescale,
name: None,
speed_audio_mode: None,
hide_cursor: None,
volume: None,
})
.collect();
}
/// Replaces the timeline with `(clip, start, end, timescale)` segments.
pub fn set_clip_cuts(config: &mut ProjectConfiguration, cuts: &[(u32, f64, f64, f64)]) {
let timeline = config.timeline.as_mut().expect("timeline");
timeline.segments = cuts
.iter()
.map(|&(clip, start, end, timescale)| TimelineSegment {
recording_clip: clip,
start,
end,
timescale,
name: None,
speed_audio_mode: None,
hide_cursor: None,
volume: None,
})
.collect();
}
pub fn add_transition(config: &mut ProjectConfiguration, segment_index: u32, duration: f64) {
let timeline = config.timeline.as_mut().expect("timeline");
timeline.transitions.push(ClipTransition {
segment_index,
kind: ClipTransitionType::CrossFade,
duration,
});
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Format {
Mp4,
/// MP4 through the turbo export split into `chunks`. With a failing
/// chunk, the export has to fall back to the standard pipeline.
Mp4Turbo {
chunks: usize,
failing_chunk: Option<usize>,
},
Gif,
Mov,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Decoder {
/// The platform default: AVAssetReader on macOS, Media Foundation on
/// Windows, FFmpeg on Linux.
Default,
/// `--force-ffmpeg-decoder`.
Ffmpeg,
}
impl Decoder {
pub fn label(self) -> &'static str {
match self {
Decoder::Default if cfg!(target_os = "macos") => "avf",
Decoder::Default if cfg!(target_os = "windows") => "mf",
Decoder::Default => "ffmpeg",
Decoder::Ffmpeg => "ffmpeg",
}
}
/// Decoders that are actually distinct on this platform.
pub fn all() -> Vec<Decoder> {
if cfg!(any(target_os = "macos", target_os = "windows")) {
vec![Decoder::Default, Decoder::Ffmpeg]
} else {
vec![Decoder::Default]
}
}
/// The `DecoderType` display prefix this choice must produce. A silent
/// fallback (Media Foundation or AVAssetReader failing over to FFmpeg)
/// would otherwise test the wrong decoder under this label.
pub fn expected_type(self) -> &'static str {
match self {
Decoder::Default if cfg!(target_os = "macos") => "AVAssetReader",
Decoder::Default if cfg!(target_os = "windows") => "MediaFoundation",
_ => "FFmpeg",
}
}
}
/// Records which decoder each stream actually used and flags any stream
/// that is not on the requested decoder.
pub fn check_decoders(case: &mut Case, decoder: Decoder, segments: &[cap_editor::SegmentMedia]) {
let mut used = Vec::new();
for (i, segment) in segments.iter().enumerate() {
let streams =
std::iter::once(("screen", Some(segment.decoders.screen_decoder_status()))).chain(
std::iter::once(("camera", segment.decoders.camera_decoder_status())),
);
for (stream, status) in streams {
let Some(status) = status else { continue };
let kind = status.decoder_type.to_string();
used.push(format!("{stream}{i}={kind}"));
if !kind.starts_with(decoder.expected_type()) {
case.problem(
"decoder-fallback",
format!(
"segment {i} {stream} decoded by {kind}, expected {} (fallback reason: {})",
decoder.expected_type(),
status.fallback_reason.as_deref().unwrap_or("none")
),
);
}
}
}
case.note(format!("decoders: {}", used.join(" ")));
}
/// Why `decoder` cannot run on this machine at all, probed once per
/// process. Only Windows has such a case: the Media Foundation decoder
/// needs a D3D11 device with video support, which GPU-less VMs (hosted CI
/// runners, where D3D11 falls back to WARP) cannot create, so the product
/// falls back to FFmpeg there. Any other Media Foundation failure stays a
/// `decoder-fallback` test failure.
pub fn decoder_unsupported(project: &Project, decoder: Decoder) -> Option<String> {
if decoder != Decoder::Default || !cfg!(target_os = "windows") {
return None;
}
static PROBE: std::sync::OnceLock<Option<String>> = std::sync::OnceLock::new();
PROBE
.get_or_init(|| {
let meta = project.meta.studio_meta()?;
let segments = runtime()
.block_on(cap_editor::create_segments(&project.meta, meta, false))
.ok()?;
let reason = segments
.first()?
.decoders
.screen_decoder_status()
.fallback_reason
.clone()?;
reason.contains("D3D11CreateDevice").then(|| {
format!(
"Media Foundation needs a D3D11 device with video support and this machine has none (WARP has no video decode): {reason}"
)
})
})
.clone()
}
/// Starts a case, or settles it as unsupported and returns `None` when this
/// machine cannot run `decoder` (see [`decoder_unsupported`]).
pub fn begin(id: &str, project: &Project, decoder: Decoder) -> Option<Case> {
let mut case = Case::new(id);
if let Some(reason) = decoder_unsupported(project, decoder) {
case.unsupported(reason);
if case.unsupported.is_some() {
let _ = report::settle(case);
return None;
}
}
Some(case)
}
/// Opens the project's decoders the way the exporter does and checks them.
/// The exporter's own decoders are private, so this probes the same
/// `create_segments` decision on the same files.
pub async fn probe_decoders(case: &mut Case, project: &Project, decoder: Decoder) {
let Some(meta) = project.meta.studio_meta() else {
return;
};
match cap_editor::create_segments(&project.meta, meta, decoder == Decoder::Ffmpeg).await {
Ok(segments) => check_decoders(case, decoder, &segments),
Err(e) => case.problem("error", format!("decoder probe failed: {e}")),
}
}
pub struct ExportRun {
pub custom_bpp: Option<f32>,
pub fps: u32,
pub resolution: (u32, u32),
pub format: Format,
pub decoder: Decoder,
}
pub async fn export(
project: &Project,
config: ProjectConfiguration,
run: &ExportRun,
out: &Path,
) -> Result<PathBuf, String> {
let force = run.decoder == Decoder::Ffmpeg;
let resolution_base = XY::new(run.resolution.0, run.resolution.1);
if let Format::Mp4Turbo {
chunks,
failing_chunk,
} = run.format
{
let mut base = cap_export::ExporterBase::builder(project.path.clone())
.with_config(config)
.with_force_ffmpeg_decoder(force)
.with_output_path(out.to_path_buf())
.build_for_mp4(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
false,
)))
.await
.map_err(|e| format!("exporter build: {e}"))?;
if let Some(chunk) = failing_chunk {
base.fail_turbo_chunk(chunk);
}
return cap_export::mp4::Mp4ExportSettings {
fps: run.fps,
resolution_base,
compression: cap_export::mp4::ExportCompression::Social,
custom_bpp: run.custom_bpp,
force_ffmpeg_decoder: force,
optimize_filesize: false,
}
.export_prepared_turbo(
base,
cap_export::mp4::turbo::TurboOptions {
chunks: Some(chunks),
strict: failing_chunk.is_none(),
},
|_| true,
)
.await;
}
let base = cap_export::ExporterBase::builder(project.path.clone())
.with_config(config)
.with_force_ffmpeg_decoder(force)
.with_output_path(out.to_path_buf())
.build()
.await
.map_err(|e| format!("exporter build: {e}"))?;
match run.format {
Format::Mp4 | Format::Mp4Turbo { .. } => {
cap_export::mp4::Mp4ExportSettings {
fps: run.fps,
resolution_base,
compression: cap_export::mp4::ExportCompression::Social,
custom_bpp: run.custom_bpp,
force_ffmpeg_decoder: force,
optimize_filesize: false,
}
.export(base, |_| true)
.await
}
Format::Mov => {
cap_export::mov::MovExportSettings {
fps: run.fps,
resolution_base,
cursor_only: false,
}
.export(base, |_| true)
.await
}
Format::Gif => {
cap_export::gif::GifExportSettings {
fps: run.fps,
resolution_base,
quality: None,
}
.export(base, |_| true)
.await
}
}
}
pub struct OutputFrame {
pub pts_secs: f64,
pub readout: FrameReadout,
/// Hash of the decoded picture, for determinism checks.
pub hash: u64,
/// Picture size.
pub size: (usize, usize),
/// Camera key region of every `LAYOUT_SAMPLE_EVERY`th frame, as fractions
/// of the picture size.
pub camera_region: Option<video_code::Region>,
/// Mean Y, Cb, Cr code values of a flat band of the screen's key colour
/// and the stream's colour tag, sampled like `camera_region` (YUV
/// outputs only).
pub screen_colour: Option<ScreenColour>,
}
#[derive(Clone, Copy, Debug)]
pub struct ScreenColour {
pub ycbcr: [f64; 3],
pub space: ffmpeg::util::color::Space,
pub range: ffmpeg::util::color::Range,
}
/// Measured minus expected (Y, Cb, Cr) code values for `rgb` encoded the way
/// the stream's colour tag says (BT.709 unless tagged BT.601; limited range
/// unless tagged full).
pub fn colour_error(colour: &ScreenColour, rgb: [u8; 3]) -> [f64; 3] {
use ffmpeg::util::color::{Range, Space};
let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0);
let (kr, kb) = if matches!(colour.space, Space::BT470BG | Space::SMPTE170M) {
(0.299, 0.114)
} else {
(0.2126, 0.0722)
};
let y = kr * r + (1.0 - kr - kb) * g + kb * b;
let (cb, cr) = ((b - y) / (2.0 * (1.0 - kb)), (r - y) / (2.0 * (1.0 - kr)));
let expected = if colour.range == Range::JPEG {
[255.0 * y, 128.0 + 255.0 * cb, 128.0 + 255.0 * cr]
} else {
[16.0 + 219.0 * y, 128.0 + 224.0 * cb, 128.0 + 224.0 * cr]
};
[0, 1, 2].map(|k| colour.ycbcr[k] - expected[k])
}
/// Mean code values and tag of a rectangle (fractions of the picture).
pub fn rect_colour(
frame: &ffmpeg::frame::Video,
view: &impl video_code::PixelSource,
rect: [f64; 4],
) -> Option<ScreenColour> {
if !matches!(
frame.format(),
ffmpeg::format::Pixel::YUV420P | ffmpeg::format::Pixel::NV12
) {
return None;
}
let (w, h) = (view.width() as f64, view.height() as f64);
let (x0, x1) = (
(rect[0] * w) as usize,
((rect[2] * w) as usize).max((rect[0] * w) as usize + 1),
);
let (y0, y1) = (
(rect[1] * h) as usize,
((rect[3] * h) as usize).max((rect[1] * h) as usize + 1),
);
let mut sum = [0.0; 3];
let mut n = 0.0;
for y in y0..y1 {
for x in x0..x1 {
let (cb, cr) = view.chroma(x, y);
sum[0] += view.luma(x, y);
sum[1] += cb + 128.0;
sum[2] += cr + 128.0;
n += 1.0;
}
}
Some(ScreenColour {
ycbcr: sum.map(|v| v / n),
space: frame.color_space(),
range: frame.color_range(),
})
}
/// Mean code values of the band 8-16% down the screen's key region, 20-80%
/// across (above the code square and the default camera position).
fn screen_colour(
frame: &ffmpeg::frame::Video,
view: &impl video_code::PixelSource,
) -> Option<ScreenColour> {
if !matches!(
frame.format(),
ffmpeg::format::Pixel::YUV420P | ffmpeg::format::Pixel::NV12
) {
return None;
}
let region = video_code::locate_region(view, StreamKind::Screen)?;
let (w, h) = (region.x1 - region.x0, region.y1 - region.y0);
let (x0, x1) = (
(region.x0 + 0.2 * w) as usize,
(region.x0 + 0.8 * w) as usize,
);
let (y0, y1) = (
(region.y0 + 0.08 * h) as usize,
(region.y0 + 0.16 * h) as usize,
);
let mut sum = [0.0; 3];
let mut n = 0.0;
for y in y0..y1.max(y0 + 1) {
for x in x0..x1.max(x0 + 1) {
let (cb, cr) = view.chroma(x, y);
sum[0] += view.luma(x, y);
sum[1] += cb + 128.0;
sum[2] += cr + 128.0;
n += 1.0;
}
}
Some(ScreenColour {
ycbcr: sum.map(|v| v / n),
space: frame.color_space(),
range: frame.color_range(),
})
}
/// Frames between camera placement samples (placement does not change
/// between frames of the verification projects).
pub const LAYOUT_SAMPLE_EVERY: usize = 10;
fn picture_hash(frame: &ffmpeg::frame::Video) -> u64 {
let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
let planes = if frame.format() == ffmpeg::format::Pixel::NV12 {
2
} else {
3
};
for plane in 0..planes.min(frame.planes()) {
let width = frame.plane_width(plane) as usize
* if frame.format() == ffmpeg::format::Pixel::NV12 && plane == 1 {
2
} else {
1
};
let stride = frame.stride(plane);
let data = frame.data(plane);
for row in 0..frame.plane_height(plane) as usize {
for &byte in &data[row * stride..row * stride + width] {
hash ^= u64::from(byte);
hash = hash.wrapping_mul(0x0100_0000_01b3);
}
}
}
hash
}
/// Decodes an output and reads every frame, using the model's expectation
/// for frame `i` (when given) to verify transition blends.
pub fn read_output(
path: &Path,
expect: Option<&dyn Fn(usize) -> cap_verify::model::FrameExpectation>,
) -> Vec<OutputFrame> {
let mut frames = Vec::new();
media::decode_video(path, |picture| {
let view = picture.view();
let readout = match expect {
Some(expect) => read_expecting(&view, &expect(frames.len())),
None => video_code::read_frame(&view),
};
let size = (
picture.frame.width() as usize,
picture.frame.height() as usize,
);
let camera_region = (frames.len() % LAYOUT_SAMPLE_EVERY == 0
&& matches!(readout.camera, video_code::Readout::Marker { .. }))
.then(|| video_code::locate_region(&view, StreamKind::Camera))
.flatten()
.map(|r| video_code::Region {
x0: r.x0 / size.0 as f64,
x1: r.x1 / size.0 as f64,
y0: r.y0 / size.1 as f64,
y1: r.y1 / size.1 as f64,
});
let screen_colour = (frames.len() % LAYOUT_SAMPLE_EVERY == 0)
.then(|| screen_colour(&picture.frame, &view))
.flatten();
frames.push(OutputFrame {
pts_secs: picture.pts_secs,
readout,
hash: picture_hash(&picture.frame),
size,
camera_region,
screen_colour,
})
})
.unwrap_or_else(|e| panic!("decode {path:?}: {e:?}"));
frames
}
pub fn read_expecting(
img: &impl video_code::PixelSource,
expectation: &cap_verify::model::FrameExpectation,
) -> FrameReadout {
let readout = FrameReadout {
screen: oracle::read_expecting(img, StreamKind::Screen, &expectation.screen),
camera: oracle::read_expecting(img, StreamKind::Camera, &expectation.camera),
};
if std::env::var("CAP_VERIFY_TRACE_BLEND").as_deref() == Ok("1") {
for (kind, expected, observed) in [
(StreamKind::Screen, &expectation.screen, &readout.screen),
(StreamKind::Camera, &expectation.camera, &readout.camera),
] {
if matches!(expected, StreamExpectation::Blend { .. })
&& let Ok(cells) = video_code::read_cells(img, kind)
{
eprintln!(
"[cap-verify-blend] {}",
serde_json::json!({
"stream": kind,
"expected": expected,
"observed": observed,
"contrast": cells.contrast,
"levels": cells.levels,
})
);
}
}
}
readout
}
/// Checks a sequential output (export or sequential playback) frame by frame
/// against the model.
pub fn check_sequence(
case: &mut Case,
model: &ReferenceModel,
frames: &[OutputFrame],
fps: u32,
check_timestamps: bool,
) {
let expected_count = model.output_frames(fps);
if frames.len() as u32 != expected_count {
case.measure(
"frame-count",
"missing",
f64::from(expected_count) - frames.len() as f64,
);
case.problem(
"frame-count",
format!(
"output has {} frames, the model expects {expected_count} ({:.3}s at {fps} fps)",
frames.len(),
model.duration()
),
);
}
if check_timestamps {
let off_grid: Vec<(usize, f64)> = frames
.iter()
.enumerate()
.filter(|(i, f)| (f.pts_secs - *i as f64 / f64::from(fps)).abs() > 0.5 / f64::from(fps))
.map(|(i, f)| (i, f.pts_secs))
.collect();
if !off_grid.is_empty() {
// Signature: how many frames, from which frame, and how far off
// the grid in frame periods (a one-slot shift is exactly 1).
for &(i, pts) in &off_grid {
case.measure("timestamps", "frame", i as f64);
case.measure(
"timestamps",
"off-periods",
((pts * f64::from(fps) - i as f64).abs() * 100.0).round() / 100.0,
);
}
case.count("timestamps", "frames", off_grid.len() as f64);
case.problem(
"timestamps",
format!(
"output timestamps off the {fps} fps grid in {} frames: {:?}",
off_grid.len(),
&off_grid[..off_grid.len().min(5)]
),
);
}
}
let n = frames.len().min(expected_count as usize);
let expectations: Vec<_> = (0..n as u32).map(|i| model.expect(i, fps)).collect();
for kind in StreamKind::ALL {
let exp: Vec<StreamExpectation> =
expectations.iter().map(|e| e.get(kind).clone()).collect();
let obs: Vec<_> = frames[..n]
.iter()
.map(|f| f.readout.get(kind).clone())
.collect();
case.stream(oracle::check_stream(kind, &exp, &obs, fps));
}
}
/// Product contract for exported MP4s: a keyframe at least every
/// `DEFAULT_KEYFRAME_INTERVAL_SECS` (2 s, crates/enc-ffmpeg h264.rs), so
/// shared videos seek quickly.
pub const MAX_KEYFRAME_INTERVAL_SECS: f64 = 2.0;
/// Colour checks on a flat key-colour band (code values).
pub const COLOUR_LEVEL_TOLERANCE: f64 = 4.0;
pub const COLOUR_MATRIX_TOLERANCE: f64 = 2.0;
/// Presentation checks every export gets on top of frame identity: the
/// keyframe interval (MP4), camera mirroring (the code reads mirrored exactly
/// when the project mirrors the camera; the screen never is), and camera
/// placement (the camera's key region lies in the corner/edge the project
/// asks for).
pub fn check_presentation(
case: &mut Case,
config: &ProjectConfiguration,
frames: &[OutputFrame],
path: &Path,
fps: u32,
keyframes: bool,
) {
if keyframes {
match media::keyframe_times(path) {
Err(e) => case.problem("error", format!("keyframes: {e}")),
Ok(times) => {
let end = frames.len() as f64 / f64::from(fps);
let slack = 1.0 / f64::from(fps) + 1e-3;
let mut previous = 0.0;
let mut worst: Option<(f64, f64)> = None;
if times.first().is_none_or(|&t| t > slack) {
case.problem(
"keyframes",
format!("the first frame is not a keyframe (keyframes {times:?})"),
);
}
for &t in times.iter().chain(std::iter::once(&end)) {
let gap = t - previous;
if gap > MAX_KEYFRAME_INTERVAL_SECS + slack
&& worst.is_none_or(|(_, g)| gap > g)
{
worst = Some((previous, gap));
}
previous = t;
}
if let Some((from, gap)) = worst {
case.problem(
"keyframes",
format!(
"{gap:.3} s without a keyframe from {from:.3} s (contract: at most {MAX_KEYFRAME_INTERVAL_SECS} s; {} keyframes in {end:.3} s)",
times.len()
),
);
}
}
}
}
let mut wrong_mirror = Vec::new();
for (i, frame) in frames.iter().enumerate() {
for kind in StreamKind::ALL {
if let video_code::Readout::Marker { mirrored, .. } = frame.readout.get(kind) {
let expected = kind == StreamKind::Camera && config.camera.mirror;
if *mirrored != expected {
wrong_mirror.push((i, kind.name(), *mirrored));
}
}
}
}
if !wrong_mirror.is_empty() {
case.problem(
"mirror",
format!(
"{} frames show a stream mirrored wrongly (camera mirror setting {}): first {:?}",
wrong_mirror.len(),
config.camera.mirror,
&wrong_mirror[..wrong_mirror.len().min(5)]
),
);
}
// Colour level: the screen's key colour must decode, under the stream's
// own colour tag (what players use), to the source luma. (Its chroma
// barely separates BT.601 from BT.709; `check_colour` covers the matrix
// with a saturated colour.)
let level: Vec<(usize, f64)> = frames
.iter()
.enumerate()
.filter_map(|(i, frame)| {
let colour = frame.screen_colour?;
let [dy, _, _] = colour_error(&colour, StreamKind::Screen.key_rgb());
(dy.abs() > COLOUR_LEVEL_TOLERANCE).then_some((i, dy))
})
.collect();
for (_, dy) in &level {
case.measure("colour-level", "luma", *dy);
}
if let Some((i, dy)) = level.first() {
case.problem(
"colour-level",
format!(
"screen key colour luma off by {dy:+.1} code values in {} sampled frames (first frame {i}; tolerance {COLOUR_LEVEL_TOLERANCE})",
level.len()
),
);
}
use cap_project::{CameraXPosition, CameraYPosition};
let mut misplaced = Vec::new();
for (i, frame) in frames.iter().enumerate() {
let Some(r) = frame.camera_region else {
continue;
};
let (cx, cy) = ((r.x0 + r.x1) / 2.0, (r.y0 + r.y1) / 2.0);
let x_ok = match config.camera.position.x {
CameraXPosition::Left => cx < 0.5,
CameraXPosition::Center => (cx - 0.5).abs() < 0.05,
CameraXPosition::Right => cx > 0.5,
};
let y_ok = match config.camera.position.y {
CameraYPosition::Top => cy < 0.5,
CameraYPosition::Bottom => cy > 0.5,
};
if !(x_ok && y_ok) {
misplaced.push((
i,
(cx * 1000.0).round() / 1000.0,
(cy * 1000.0).round() / 1000.0,
));
}
}
if !misplaced.is_empty() {
case.problem(
"layout",
format!(
"camera drawn away from its {:?}/{:?} position in {} sampled frames (frame, centre x, centre y as fractions): {:?}",
config.camera.position.x,
config.camera.position.y,
misplaced.len(),
&misplaced[..misplaced.len().min(5)]
),
);
}
}
pub fn finish(case: Case) {
if let Err(message) = report::settle(case) {
panic!("{message}");
}
}
pub fn scratch_dir(name: &str) -> tempfile::TempDir {
let root = fixture::fixture_root().join("scratch");
std::fs::create_dir_all(&root).expect("scratch root");
tempfile::Builder::new()
.prefix(&format!("{}-", name.replace('/', "_")))
.tempdir_in(root)
.expect("scratch dir")
}
/// Drives the editor's decode + render path offline, one requested frame
/// at a time (no wall clock): `create_segments` decoders, `FrameRenderer`,
/// `RendererLayers`. Seeks use `get_frames_initial` like the preview
/// renderer; frames that follow a seek use `get_frames` like playback.
pub struct Player {
constants: std::sync::Arc<cap_rendering::RenderVideoConstants>,
segments: Vec<cap_editor::SegmentMedia>,
renderer: Option<cap_rendering::FrameRenderer<'static>>,
layers: cap_rendering::RendererLayers,
zooms: Vec<cap_rendering::ZoomTransformTimeline>,
outgoing_zooms: std::collections::HashMap<u32, cap_rendering::ZoomTransformTimeline>,
config: std::sync::Arc<ProjectConfiguration>,
duration: f64,
resolution: XY<u32>,
}
impl Player {
pub async fn open(
project: &Project,
config: ProjectConfiguration,
decoder: Decoder,
resolution: (u32, u32),
) -> Result<Self, String> {
let meta = project.meta.studio_meta().ok_or("not studio")?.clone();
let recordings = cap_rendering::ProjectRecordingsMeta::new(&project.path, &meta)?;
let constants = std::sync::Arc::new(
cap_rendering::RenderVideoConstants::new(
&recordings.segments,
project.meta.clone(),
meta.clone(),
)
.await
.map_err(|e| format!("render constants: {e}"))?,
);
let segments =
cap_editor::create_segments(&project.meta, &meta, decoder == Decoder::Ffmpeg).await?;
let mut layers = cap_rendering::RendererLayers::new_with_options(
&constants.device,
&constants.queue,
constants.is_software_adapter,
);
let (sw, sh) = segments[0].decoders.screen_video_dimensions();
let camera = segments[0].decoders.camera_video_dimensions();
layers.prepare_for_video_dimensions(
&constants.device,
sw,
sh,
camera.map(|c| c.0),
camera.map(|c| c.1),
);
let duration = config.timeline.as_ref().map_or(0.0, |t| t.duration());
let zooms = segments
.iter()
.map(|s| {
cap_rendering::ZoomTransformTimeline::from_project(
&config,
&s.cursor,
duration,
constants.options.screen_size,
)
})
.collect();
// The renderer borrows the constants; both live as long as the
// player, and the Arc keeps the constants' address stable.
let constants_ref: &'static cap_rendering::RenderVideoConstants =
unsafe { &*std::sync::Arc::as_ptr(&constants) };
Ok(Self {
renderer: Some(cap_rendering::FrameRenderer::new(constants_ref)),
constants,
segments,
layers,
zooms,
outgoing_zooms: Default::default(),
config: std::sync::Arc::new(config),
duration,
resolution: XY::new(resolution.0, resolution.1),
})
}
/// Flags streams that are not on `decoder` (checked after the run, so a
/// mid-run fallback is caught too).
pub fn check_decoders(&self, case: &mut Case, decoder: Decoder) {
check_decoders(case, decoder, &self.segments);
case.note(format!(
"render adapter: {}{}",
self.constants.adapter_name(),
if self.constants.is_software_adapter {
" (software)"
} else {
""
}
));
}
pub fn prefer_sequential(&self) {
for segment in &self.segments {
segment.decoders.prefer_sequential_access();
}
}
/// Renders output frame `frame` and reads its markers. `None` when the
/// timeline has no segment there or the decoders returned nothing.
pub async fn frame(
&mut self,
frame: u32,
fps: u32,
seek: bool,
expectation: Option<&cap_verify::model::FrameExpectation>,
) -> Result<FrameReadout, String> {
let t = f64::from(frame) / f64::from(fps);
let mapping = self
.config
.timeline
.as_ref()
.and_then(|timeline| timeline.get_frame_mapping(t))
.ok_or_else(|| format!("no timeline segment at {t:.3}s"))?;
let (incoming, outgoing) = match mapping {
cap_project::TimelineFrameMapping::Single { source, .. }
| cap_project::TimelineFrameMapping::Hold { source, .. } => {
((source.segment.recording_clip, source.source_time), None)
}
cap_project::TimelineFrameMapping::Transition {
outgoing,
incoming,
kind,
progress,
..
} => (
(incoming.segment.recording_clip, incoming.source_time),
Some((
outgoing.segment.recording_clip,
outgoing.source_time,
kind,
progress,
)),
),
};
let incoming_frames = self.fetch(incoming.0, incoming.1, seek).await?;
let rendered = if let Some((clip, time, kind, progress)) = outgoing {
let outgoing_frames = self.fetch(clip, time, seek).await?;
let screen_size = self.constants.options.screen_size;
let outgoing_zoom = self.outgoing_zooms.entry(clip).or_insert_with(|| {
cap_rendering::ZoomTransformTimeline::from_project_for_outgoing_clip(
&self.config,
&self.segments[clip as usize].cursor,
self.duration,
screen_size,
clip,
)
});
outgoing_zoom.ensure_precomputed_until((frame as f32 + 1.0) / fps as f32);
let outgoing_uniforms = cap_rendering::ProjectUniforms::new(
&self.constants,
&self.config,
frame,
fps,
self.resolution,
&self.segments[clip as usize].cursor,
&outgoing_frames,
self.duration,
outgoing_zoom,
);
let zoom = &mut self.zooms[incoming.0 as usize];
zoom.ensure_precomputed_until((frame as f32 + 1.0) / fps as f32);
let incoming_uniforms = cap_rendering::ProjectUniforms::new(
&self.constants,
&self.config,
frame,
fps,
self.resolution,
&self.segments[incoming.0 as usize].cursor,
&incoming_frames,
self.duration,
zoom,
);
self.renderer
.as_mut()
.expect("renderer")
.render_transition_immediate(
cap_rendering::TransitionRenderInput {
segment_frames: outgoing_frames,
uniforms: outgoing_uniforms,
cursor: &self.segments[clip as usize].cursor,
render_display: true,
},
cap_rendering::TransitionRenderInput {
segment_frames: incoming_frames,
uniforms: incoming_uniforms,
cursor: &self.segments[incoming.0 as usize].cursor,
render_display: true,
},
kind,
progress as f32,
&mut self.layers,
)
.await
.map_err(|e| format!("render transition: {e}"))?
} else {
let media = &self.segments[incoming.0 as usize];
let zoom = &mut self.zooms[incoming.0 as usize];
zoom.ensure_precomputed_until((frame as f32 + 1.0) / fps as f32);
let uniforms = cap_rendering::ProjectUniforms::new(
&self.constants,
&self.config,
frame,
fps,
self.resolution,
&media.cursor,
&incoming_frames,
self.duration,
zoom,
);
self.renderer
.as_mut()
.expect("renderer")
.render_immediate(
incoming_frames,
uniforms,
&media.cursor,
true,
&mut self.layers,
)
.await
.map_err(|e| format!("render: {e}"))?
};
let view = video_code::PackedFrame {
data: &rendered.data,
width: rendered.width as usize,
height: rendered.height as usize,
stride: rendered.padded_bytes_per_row as usize,
order: video_code::PixelOrder::Rgba,
};
Ok(match expectation {
Some(expectation) => read_expecting(&view, expectation),
None => video_code::read_frame(&view),
})
}
}
impl Player {
/// Decodes the frames for recording `clip` at recording `time`, as a
/// seek (`get_frames_initial`) or as playback (`get_frames`).
async fn fetch(
&self,
clip: u32,
time: f64,
seek: bool,
) -> Result<cap_rendering::DecodedSegmentFrames, String> {
let media = &self.segments[clip as usize];
let offsets = self
.config
.clips
.iter()
.find(|c| c.index == clip)
.map(|c| c.offsets)
.unwrap_or_default();
let needs_camera = !self.config.camera.hide;
if seek {
media
.decoders
.get_frames_initial(time as f32, needs_camera, true, offsets)
.await
} else {
media
.decoders
.get_frames(time as f32, needs_camera, true, offsets)
.await
}
.ok_or_else(|| format!("decoders returned no frame for clip {clip} at {time:.3}s"))
}
}
impl Drop for Player {
fn drop(&mut self) {
// Drop the renderer before the constants it borrows.
self.renderer.take();
}
}
/// Checks a list of (output frame, readout) pairs against the model.
pub fn check_requests(
case: &mut Case,
model: &ReferenceModel,
frames: &[u32],
readouts: &[FrameReadout],
fps: u32,
) {
let expectations: Vec<_> = frames.iter().map(|&f| model.expect(f, fps)).collect();
for kind in StreamKind::ALL {
let exp: Vec<StreamExpectation> =
expectations.iter().map(|e| e.get(kind).clone()).collect();
let obs: Vec<_> = readouts.iter().map(|r| r.get(kind).clone()).collect();
case.stream(oracle::check_stream_at(kind, &exp, &obs, frames, fps));
}
}
pub fn assert_passed(failures: Vec<String>) {
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
/// What the playback loop reported about one real-time run.
#[derive(Debug, Default)]
pub struct RealtimeTelemetry {
/// From `Playback::start` to the clock starting (warm-up done), if it did.
pub ready_after: Option<std::time::Duration>,
pub warmup_buffered: Option<(usize, usize)>,
/// Frames the loop skipped on purpose, by reason.
pub skipped: std::collections::BTreeMap<String, u64>,
pub renderer_dropped: u64,
}
impl RealtimeTelemetry {
pub fn describe(&self) -> String {
format!(
"playback ready after {}, warm-up buffered {}, skipped {:?}, renderer dropped {}",
self.ready_after.map_or("never".to_string(), |d| format!(
"{:.0} ms",
d.as_secs_f64() * 1000.0
)),
self.warmup_buffered
.map_or("-".to_string(), |(b, t)| format!("{b}/{t}")),
self.skipped,
self.renderer_dropped
)
}
}
/// Runs the editor's real-time playback loop (`cap_editor::Playback`: warm-up,
/// prefetch, frame cache, wall clock) from `start_frame` and returns every
/// frame it delivered, read with `expect(frame number)` (so transition frames
/// are read as blends), plus the loop's telemetry. The `seconds` window starts
/// when the playback clock starts (warm-up finished), not when `start`
/// returns, so a slow warm-up does not eat the measured window.
/// Video only (`create_segments_without_audio`): the audio clock has its own
/// oracle. Dropped frames are normal under load; delivered ones must be right.
#[allow(clippy::too_many_arguments)]
pub async fn realtime_playback(
case: &mut Case,
project: &Project,
config: ProjectConfiguration,
decoder: Decoder,
resolution: (u32, u32),
fps: u32,
start_frame: u32,
seconds: f64,
expect: &dyn Fn(u32) -> cap_verify::model::FrameExpectation,
) -> Result<(Vec<(u32, FrameReadout)>, RealtimeTelemetry), String> {
use std::sync::{Arc, Mutex};
const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
let meta = project.meta.studio_meta().ok_or("not studio")?.clone();
let recordings = cap_rendering::ProjectRecordingsMeta::new(&project.path, &meta)?;
let constants = Arc::new(
cap_rendering::RenderVideoConstants::new(
&recordings.segments,
project.meta.clone(),
meta.clone(),
)
.await
.map_err(|e| format!("render constants: {e}"))?,
);
let segments = Arc::new(
cap_editor::create_segments_without_audio(&project.meta, &meta, decoder == Decoder::Ffmpeg)
.await?,
);
// The case is labelled by decoder: it must be the decoder that runs.
check_decoders(case, decoder, &segments);
let layers_rx = cap_editor::start_renderer_layers_creation(&constants, &config);
let layers_rx = cap_editor::finish_renderer_layers_creation(layers_rx).await;
let delivered: Arc<Mutex<Vec<cap_rendering::RenderedFrame>>> = Default::default();
let sink = delivered.clone();
let frame_cb: cap_editor::EditorFrameCallback = Box::new(move |output, _| {
if let cap_editor::EditorFrameOutput::Rgba(frame) = output {
sink.lock().unwrap().push(frame);
}
});
let renderer = Arc::new(cap_editor::Renderer::spawn(
constants.clone(),
frame_cb,
layers_rx,
)?);
let (_project_tx, project_rx) = tokio::sync::watch::channel(config);
let (audio_output, _audio_sink) = cap_editor::AudioOutput::new_offline();
let (telemetry, mut telemetry_rx) = cap_editor::PlaybackTelemetry::channel();
let playback = cap_editor::Playback {
renderer,
render_constants: constants,
start_frame_number: start_frame,
project: project_rx,
segment_medias: segments,
music: cap_editor::MusicTracks::new(),
audio_output: Arc::new(audio_output),
telemetry: Some(telemetry),
};
let started = std::time::Instant::now();
let handle = playback
.start(fps, XY::new(resolution.0, resolution.1))
.await
.map_err(|e| format!("playback start: {e:?}"))?;
let mut report = RealtimeTelemetry::default();
let absorb = |event: cap_editor::PlaybackTelemetryEvent, report: &mut RealtimeTelemetry| {
use cap_editor::PlaybackTelemetryEvent as E;
match event {
E::WarmupComplete {
buffered_frames,
target_frames,
..
} => report.warmup_buffered = Some((buffered_frames, target_frames)),
E::ClockStarted { .. } => report.ready_after = Some(started.elapsed()),
E::FrameSkipped {
skipped, reason, ..
} => *report.skipped.entry(format!("{reason:?}")).or_default() += u64::from(skipped),
E::RendererDropped { .. } => report.renderer_dropped += 1,
_ => {}
}
};
let ready_deadline = tokio::time::Instant::now() + READY_TIMEOUT;
while report.ready_after.is_none() {
match tokio::time::timeout_at(ready_deadline, telemetry_rx.recv()).await {
Ok(Some(event)) => absorb(event, &mut report),
_ => break,
}
}
if report.ready_after.is_some() {
tokio::time::sleep(std::time::Duration::from_secs_f64(seconds)).await;
}
handle.stop();
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
while let Ok(event) = telemetry_rx.try_recv() {
absorb(event, &mut report);
}
let frames = std::mem::take(&mut *delivered.lock().unwrap());
let frames = frames
.into_iter()
.map(|frame| {
let view = video_code::PackedFrame {
data: &frame.data,
width: frame.width as usize,
height: frame.height as usize,
stride: frame.padded_bytes_per_row as usize,
order: video_code::PixelOrder::Rgba,
};
(
frame.frame_number,
read_expecting(&view, &expect(frame.frame_number)),
)
})
.collect();
Ok((frames, report))
}