Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
#![allow(dead_code)]
#[cfg(any(target_os = "macos", target_os = "windows"))]
use ffmpeg::sys::AVHWDeviceType;
use ffmpeg::{format, frame};
use std::{
cell::RefCell,
collections::BTreeMap,
path::PathBuf,
rc::Rc,
sync::{Arc, OnceLock, mpsc},
};
use tokio::sync::oneshot;
use tracing::info;
use crate::{DecodedFrame, PixelFormat};
#[cfg(target_os = "windows")]
use cap_video_decode::FrameTextures;
use crate::cpu_yuv::YuvMatrix;
use super::{
DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, ManagedVideoControl, ManagedVideoError,
ManagedVideoInput, VideoDecoderMessage,
frame_converter::{FrameConverter, PlanePool},
pts_to_frame, record_pts_hole,
};
#[cfg(test)]
mod tests;
#[derive(Clone)]
struct ProcessedFrame {
number: u32,
data: Arc<Vec<u8>>,
width: u32,
height: u32,
format: PixelFormat,
matrix: YuvMatrix,
y_stride: u32,
uv_stride: u32,
planes: Arc<PlanePool>,
#[cfg(target_os = "linux")]
cuda: Option<Arc<crate::linux_gpu::CudaNv12Frame>>,
}
fn empty_plane_slot() -> Arc<Vec<u8>> {
static EMPTY: OnceLock<Arc<Vec<u8>>> = OnceLock::new();
EMPTY.get_or_init(|| Arc::new(Vec::new())).clone()
}
impl Drop for ProcessedFrame {
fn drop(&mut self) {
// The cache and the renderer share this allocation. Recycle it only
// once both have released the frame; otherwise the next decode would
// write into a buffer the renderer still uploads.
if Arc::strong_count(&self.data) != 1 {
return;
}
let data = std::mem::replace(&mut self.data, empty_plane_slot());
if let Ok(buf) = Arc::try_unwrap(data) {
self.planes.recycle(buf);
}
}
}
impl ProcessedFrame {
fn to_decoded_frame(&self) -> DecodedFrame {
#[cfg(target_os = "linux")]
if let Some(cuda) = &self.cuda {
return DecodedFrame::new_nv12_cuda(Arc::clone(cuda), Arc::clone(&self.data))
.with_matrix(self.matrix);
}
let frame = match self.format {
PixelFormat::Rgba => {
DecodedFrame::new_with_arc(Arc::clone(&self.data), self.width, self.height)
}
PixelFormat::Nv12 => DecodedFrame::new_nv12_with_arc(
Arc::clone(&self.data),
self.width,
self.height,
self.y_stride,
self.uv_stride,
),
PixelFormat::Yuv420p => DecodedFrame::new_yuv420p_with_arc(
Arc::clone(&self.data),
self.width,
self.height,
self.y_stride,
self.uv_stride,
),
};
frame.with_matrix(self.matrix)
}
}
#[derive(Clone)]
struct OutputFrame {
number: u32,
frame: DecodedFrame,
}
struct PendingRequest {
time: f32,
frame: u32,
reply: oneshot::Sender<DecodedFrame>,
}
const MAX_FRAME_CACHE_BYTES: usize = 128 * 1024 * 1024;
/// Cache weight of a frame still on the GPU (it pins an NVDEC surface).
#[cfg(target_os = "linux")]
const CUDA_FRAME_CACHE_WEIGHT: usize = 16 * 1024 * 1024;
fn extract_yuv_planes(
frame: &frame::Video,
planes: &PlanePool,
) -> Option<(Vec<u8>, PixelFormat, u32, u32)> {
let height = frame.height();
match frame.format() {
format::Pixel::YUV420P => {
let y_stride = frame.stride(0) as u32;
let u_stride = frame.stride(1) as u32;
let v_stride = frame.stride(2) as u32;
let uv_height = frame.plane_height(1);
let y_size = (y_stride * height) as usize;
let u_size = (u_stride * uv_height) as usize;
let data = pack_yuv420p(
planes,
&frame.data(0)[..y_size],
&frame.data(1)[..u_size],
frame.data(2),
u_stride,
v_stride,
uv_height,
);
Some((data, PixelFormat::Yuv420p, y_stride, u_stride))
}
format::Pixel::NV12 => {
let y_stride = frame.stride(0) as u32;
let uv_stride = frame.stride(1) as u32;
let y_size = (y_stride * height) as usize;
let uv_size = (uv_stride * frame.plane_height(1)) as usize;
let data = pack_nv12(planes, &frame.data(0)[..y_size], &frame.data(1)[..uv_size]);
Some((data, PixelFormat::Nv12, y_stride, uv_stride))
}
_ => None,
}
}
fn pack_nv12(planes: &PlanePool, y: &[u8], uv: &[u8]) -> Vec<u8> {
let mut data = planes.take(y.len() + uv.len());
data.extend_from_slice(y);
data.extend_from_slice(uv);
data
}
fn pack_yuv420p(
planes: &PlanePool,
y: &[u8],
u: &[u8],
v: &[u8],
u_stride: u32,
v_stride: u32,
uv_height: u32,
) -> Vec<u8> {
let mut data = planes.take(y.len() + u.len() + u.len());
data.extend_from_slice(y);
data.extend_from_slice(u);
// Downstream consumers index both chroma planes with the single
// advertised uv_stride (the U stride). FFmpeg almost always gives
// U and V identical linesizes, but if they ever differ, repack the
// V rows to the U stride — otherwise every V row shifts and the
// whole image takes on a uniform wrong-chroma cast.
if v_stride == u_stride {
data.extend_from_slice(&v[..u.len()]);
} else {
let u_stride = u_stride as usize;
let v_stride = v_stride as usize;
let row_bytes = u_stride.min(v_stride);
let pad = u_stride - row_bytes;
for row in v.chunks(v_stride).take(uv_height as usize) {
data.extend_from_slice(&row[..row_bytes]);
data.resize(data.len() + pad, 128);
}
}
data
}
impl CachedFrame {
fn process_cpu(&mut self, converter: &mut FrameConverter) -> ProcessedFrame {
match self {
Self::Raw { frame, number } => {
#[cfg(target_os = "linux")]
if crate::linux_gpu::enabled() && frame.format() == format::Pixel::CUDA {
// Keep NVDEC's surface: a new reference holds it alive
// while the compositor copies it device-to-device.
let data = unsafe {
let mut reference = frame::Video::empty();
ffmpeg::ffi::av_frame_ref(reference.as_mut_ptr(), frame.as_ptr());
let raw = reference.as_ptr();
let cuda = crate::linux_gpu::CudaNv12Frame::new(
(*raw).data[0] as u64,
(*raw).data[1] as u64,
(*raw).linesize[0] as usize,
(*raw).linesize[1] as usize,
frame.width(),
frame.height(),
Box::new(reference),
);
ProcessedFrame {
data: Arc::new(Vec::new()),
number: *number,
width: frame.width(),
height: frame.height(),
format: PixelFormat::Nv12,
matrix: YuvMatrix::from_color_space(frame.color_space()),
y_stride: cuda.y_pitch as u32,
uv_stride: cuda.uv_pitch as u32,
planes: Arc::clone(converter.plane_pool()),
cuda: Some(Arc::new(cuda)),
}
};
*self = Self::Processed(data.clone());
return data;
}
let planes = Arc::clone(converter.plane_pool());
let data = if let Some((yuv_data, pixel_format, y_stride, uv_stride)) =
extract_yuv_planes(frame, &planes)
{
ProcessedFrame {
data: Arc::new(yuv_data),
number: *number,
width: frame.width(),
height: frame.height(),
format: pixel_format,
matrix: YuvMatrix::from_color_space(frame.color_space()),
y_stride,
uv_stride,
planes,
#[cfg(target_os = "linux")]
cuda: None,
}
} else {
let frame_buffer = converter.convert(frame);
ProcessedFrame {
data: Arc::new(frame_buffer),
number: *number,
width: frame.width(),
height: frame.height(),
format: PixelFormat::Rgba,
matrix: YuvMatrix::from_color_space(frame.color_space()),
y_stride: frame.width() * 4,
uv_stride: 0,
planes,
#[cfg(target_os = "linux")]
cuda: None,
}
};
*self = Self::Processed(data.clone());
data
}
Self::Processed(data) => data.clone(),
#[cfg(target_os = "windows")]
Self::Gpu { .. } => unreachable!(),
}
}
fn produce(&mut self, converter: &mut FrameConverter) -> OutputFrame {
match self {
#[cfg(target_os = "windows")]
Self::Gpu { frame, number, .. } => OutputFrame {
number: *number,
frame: frame.clone(),
},
Self::Raw { .. } => {
let data = self.process_cpu(converter);
OutputFrame {
number: data.number,
frame: data.to_decoded_frame(),
}
}
Self::Processed(data) => OutputFrame {
number: data.number,
frame: data.clone().to_decoded_frame(),
},
}
}
}
#[derive(Clone)]
enum CachedFrame {
Raw {
frame: frame::Video,
number: u32,
},
Processed(ProcessedFrame),
#[cfg(target_os = "windows")]
Gpu {
frame: DecodedFrame,
number: u32,
textures: Arc<FrameTextures>,
},
}
impl CachedFrame {
fn estimated_bytes(&self) -> usize {
// Frames still on the GPU pin an NVDEC surface each; weighting them
// at 16 MB caps the 128 MB cache at 8 surfaces, inside the pool.
#[cfg(target_os = "linux")]
match self {
Self::Raw { frame, .. } if frame.format() == format::Pixel::CUDA => {
return CUDA_FRAME_CACHE_WEIGHT;
}
Self::Processed(frame) if frame.cuda.is_some() => return CUDA_FRAME_CACHE_WEIGHT,
_ => {}
}
match self {
Self::Raw { frame, .. } => {
let height = frame.height() as usize;
match frame.format() {
format::Pixel::YUV420P => {
frame.stride(0).saturating_mul(height).saturating_add(
frame
.stride(1)
.saturating_add(frame.stride(2))
.saturating_mul(frame.plane_height(1) as usize),
)
}
format::Pixel::NV12 => frame.stride(0).saturating_mul(height).saturating_add(
frame
.stride(1)
.saturating_mul(frame.plane_height(1) as usize),
),
_ => (frame.width() as usize)
.saturating_mul(height)
.saturating_mul(4),
}
}
Self::Processed(frame) => frame.data.len(),
#[cfg(target_os = "windows")]
Self::Gpu { frame, .. } => (frame.width() as usize)
.saturating_mul(frame.height() as usize)
.saturating_mul(3),
}
}
}
/// Whether an idle decoder should decode one more frame past `anchor` (the
/// frame the renderer last received): at most `readahead` frames ahead, so the
/// held hardware surfaces stay within the decoder's pool.
fn wants_readahead<T>(cache: &BTreeMap<u32, T>, anchor: Option<u32>, readahead: u32) -> bool {
anchor.is_some_and(|last| {
cache.range(last..).count() <= readahead as usize
&& cache
.keys()
.next_back()
.is_none_or(|&newest| newest < last.saturating_add(readahead))
})
}
fn insert_cached_frame(
cache: &mut BTreeMap<u32, CachedFrame>,
number: u32,
frame: CachedFrame,
requested_frame: u32,
last_active_frame: Option<u32>,
max_bytes: usize,
) {
let frame_bytes = frame.estimated_bytes();
let mut cache_bytes = cache
.values()
.map(CachedFrame::estimated_bytes)
.fold(0usize, usize::saturating_add);
while !cache.is_empty()
&& (cache.len() >= FRAME_CACHE_SIZE || cache_bytes.saturating_add(frame_bytes) > max_bytes)
{
let Some(last_active_frame) = last_active_frame else {
cache.clear();
break;
};
let first = *cache.keys().next().unwrap();
let last = *cache.keys().next_back().unwrap();
let evicted = if requested_frame > last_active_frame {
first
} else if requested_frame < last_active_frame || number <= last {
last
} else {
first
};
if let Some(frame) = cache.remove(&evicted) {
cache_bytes = cache_bytes.saturating_sub(frame.estimated_bytes());
}
}
cache.insert(number, frame);
}
enum DecoderInput {
Ordinary(PathBuf),
Managed(ManagedVideoInput),
}
impl DecoderInput {
fn open(
&self,
hardware: Option<ffmpeg::sys::AVHWDeviceType>,
managed: Option<&Arc<ManagedVideoControl>>,
) -> Result<cap_video_decode::FFmpegDecoder, String> {
match self {
Self::Ordinary(path) => cap_video_decode::FFmpegDecoder::new(path.clone(), hardware),
Self::Managed(input) => {
let control = managed
.cloned()
.ok_or("Managed video input requires a cancellation owner")?;
cap_video_decode::FFmpegDecoder::new_relocatable_interruptible(
&input.source,
input.paths.iter().map(PathBuf::as_path),
hardware,
Arc::new(move || control.is_cancelled()),
)
}
}
}
}
fn managed_error(control: &ManagedVideoControl, error: ManagedVideoError) -> ManagedVideoError {
if control.is_cancelled() {
ManagedVideoError::Cancelled
} else {
error
}
}
fn managed_stopped(control: Option<&Arc<ManagedVideoControl>>) -> bool {
control.is_some_and(|control| control.is_cancelled() || control.terminal().is_some())
}
fn first_frame(
frames: &mut impl Iterator<Item = Result<frame::Video, ffmpeg::Error>>,
managed: Option<&Arc<ManagedVideoControl>>,
) -> Result<Option<frame::Video>, ManagedVideoError> {
let Some(control) = managed else {
return Ok(frames.flatten().next());
};
if let Some(error) = control.terminal() {
return Err(error);
}
if control.is_cancelled() {
return Err(control.fail(ManagedVideoError::Cancelled));
}
let frame = frames.next();
if control.is_cancelled() {
return Err(control.fail(ManagedVideoError::Cancelled));
}
match frame {
Some(Ok(frame)) => Ok(Some(frame)),
Some(Err(error)) => Err(control.fail(ManagedVideoError::Decode(error.to_string()))),
None => Ok(None),
}
}
pub struct FfmpegDecoder;
impl FfmpegDecoder {
pub fn spawn(
name: &'static str,
path: PathBuf,
fps: u32,
rx: mpsc::Receiver<VideoDecoderMessage>,
ready_tx: oneshot::Sender<Result<DecoderInitResult, String>>,
) -> Result<(), String> {
Self::spawn_with_hw_config(name, path, fps, rx, ready_tx, true)
}
pub fn spawn_with_hw_config(
name: &'static str,
path: PathBuf,
fps: u32,
rx: mpsc::Receiver<VideoDecoderMessage>,
ready_tx: oneshot::Sender<Result<DecoderInitResult, String>>,
use_hw_acceleration: bool,
) -> Result<(), String> {
Self::spawn_worker(
name,
DecoderInput::Ordinary(path),
fps,
rx,
ready_tx,
use_hw_acceleration,
None,
)
.map(drop)
}
#[cfg(test)]
pub(super) fn spawn_ordinary_test_worker(
path: PathBuf,
fps: u32,
rx: mpsc::Receiver<VideoDecoderMessage>,
ready_tx: oneshot::Sender<Result<DecoderInitResult, String>>,
use_hw_acceleration: bool,
) -> std::thread::JoinHandle<()> {
Self::spawn_worker(
"ordinary-test",
DecoderInput::Ordinary(path),
fps,
rx,
ready_tx,
use_hw_acceleration,
None,
)
.unwrap()
}
pub(super) fn spawn_managed(
name: &'static str,
input: ManagedVideoInput,
fps: u32,
rx: mpsc::Receiver<VideoDecoderMessage>,
ready_tx: oneshot::Sender<Result<DecoderInitResult, String>>,
use_hw_acceleration: bool,
managed: Arc<ManagedVideoControl>,
) -> Result<std::thread::JoinHandle<()>, String> {
Self::spawn_worker(
name,
DecoderInput::Managed(input),
fps,
rx,
ready_tx,
use_hw_acceleration,
Some(managed),
)
}
fn spawn_worker(
name: &'static str,
input: DecoderInput,
fps: u32,
rx: mpsc::Receiver<VideoDecoderMessage>,
ready_tx: oneshot::Sender<Result<DecoderInitResult, String>>,
use_hw_acceleration: bool,
managed: Option<Arc<ManagedVideoControl>>,
) -> Result<std::thread::JoinHandle<()>, String> {
let is_managed = managed.is_some();
let worker = move || {
if let Some(control) = &managed {
control.run_worker(|| {
Self::run(
name,
input,
fps,
rx,
ready_tx,
use_hw_acceleration,
managed.clone(),
);
});
} else {
Self::run(name, input, fps, rx, ready_tx, use_hw_acceleration, None);
}
};
if is_managed {
std::thread::Builder::new()
.name(format!("managed-video-{name}"))
.spawn(worker)
.map_err(|error| error.to_string())
} else {
Ok(std::thread::spawn(worker))
}
}
fn run(
name: &'static str,
input: DecoderInput,
fps: u32,
rx: mpsc::Receiver<VideoDecoderMessage>,
ready_tx: oneshot::Sender<Result<DecoderInitResult, String>>,
use_hw_acceleration: bool,
managed: Option<Arc<ManagedVideoControl>>,
) {
let hw_device_type = if use_hw_acceleration {
#[cfg(target_os = "windows")]
{
Some(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA)
}
#[cfg(target_os = "macos")]
{
Some(AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX)
}
#[cfg(not(any(target_os = "windows", target_os = "macos")))]
{
// Opt-in NVDEC for Linux GPU hosts (render farm workers).
match std::env::var("CAP_LINUX_HW_DECODE").as_deref() {
Ok("cuda") => Some(ffmpeg::sys::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA),
_ => None,
}
}
} else {
None
};
let mut this = match input.open(hw_device_type, managed.as_ref()) {
Err(e) => {
if let Some(control) = &managed {
control.fail(managed_error(
control,
ManagedVideoError::Initialization(e.clone()),
));
}
let _ = ready_tx.send(Err(e));
return;
}
Ok(v) => {
let is_hw = v.is_hardware_accelerated();
let width = v.decoder().width();
let height = v.decoder().height();
info!(
"FFmpeg decoder created for '{}': {}x{}, hw_accel={}",
name, width, height, is_hw
);
v
}
};
let time_base = this.decoder().time_base();
let video_width = this.decoder().width();
let video_height = this.decoder().height();
let is_hw = this.is_hardware_accelerated();
let mut cache = BTreeMap::<u32, CachedFrame>::new();
let mut last_active_frame = None::<u32>;
// pts holes (start frame -> first frame after the hole) discovered
// from decode-order jumps; facts about the file, so they survive
// resets and cache eviction.
let mut pts_holes = BTreeMap::<u32, u32>::new();
// Content of the most recently served VFR hold, kept across
// requests so a hole keeps rendering its true frame even after
// the pre-hole frame leaves the cache.
let mut gap_hold: Option<OutputFrame> = None;
// Previous vended frame number since the last reset, for pts-hole
// discovery (the reader vends in pts order between resets).
let mut prev_vended: Option<u32> = None;
// Newest frame the reader has vended since its last reset: where the
// reader is. The last sent frame is not, as it can be a cache hit.
let mut reader_at: Option<u32> = None;
let last_sent_frame = Rc::new(RefCell::new(None::<OutputFrame>));
let first_ever_frame = Rc::new(RefCell::new(None::<OutputFrame>));
let mut frames = this.frames();
let mut converter = FrameConverter::new();
let first_frame_result = match first_frame(&mut frames, managed.as_ref()) {
Ok(frame) => frame,
Err(error) => {
let _ = ready_tx.send(Err(error.to_string()));
return;
}
};
if first_frame_result.is_none() && is_hw {
let mut sw_this = match input.open(None, managed.as_ref()) {
Err(e) => {
if let Some(control) = &managed {
control.fail(managed_error(
control,
ManagedVideoError::Initialization(e.clone()),
));
}
let _ = ready_tx.send(Err(format!("Software fallback failed: {e}")));
return;
}
Ok(v) => v,
};
let sw_time_base = sw_this.decoder().time_base();
let mut sw_cache = BTreeMap::<u32, CachedFrame>::new();
let mut sw_last_active_frame = None::<u32>;
// pts holes (start frame -> first frame after the hole) from
// decode-order jumps; facts about the file that survive resets
// and cache eviction.
let mut sw_pts_holes = BTreeMap::<u32, u32>::new();
// Content of the most recently served VFR hold.
let mut sw_gap_hold: Option<OutputFrame> = None;
// Previous vended frame number since the last reset.
let mut sw_prev_vended: Option<u32> = None;
// Newest frame the reader has vended since its last reset: where the
// reader is. The last sent frame is not, as it can be a cache hit.
let mut sw_reader_at: Option<u32> = None;
let sw_last_sent_frame = Rc::new(RefCell::new(None::<OutputFrame>));
let sw_first_ever_frame = Rc::new(RefCell::new(None::<OutputFrame>));
let mut sw_frames = sw_this.frames();
let mut sw_converter = FrameConverter::new();
let first_software_frame = match first_frame(&mut sw_frames, managed.as_ref()) {
Ok(frame) => frame,
Err(error) => {
let _ = ready_tx.send(Err(error.to_string()));
return;
}
};
if first_software_frame.is_none()
&& let Some(control) = &managed
{
let error = control.fail(ManagedVideoError::NoFrames);
let _ = ready_tx.send(Err(error.to_string()));
return;
}
if let Some(frame) = first_software_frame {
let current_frame = pts_to_frame(frame.pts().unwrap_or(0), sw_time_base, fps);
let mut cache_frame = CachedFrame::Raw {
frame,
number: current_frame,
};
let output = cache_frame.produce(&mut sw_converter);
sw_cache.insert(current_frame, cache_frame);
*sw_first_ever_frame.borrow_mut() = Some(output.clone());
*sw_last_sent_frame.borrow_mut() = Some(output);
// The pre-decoded frame is a real vend: without seeding
// this, an opening static hold would never be recorded.
sw_prev_vended = Some(current_frame);
sw_reader_at = Some(current_frame);
}
let sw_decoder_type = DecoderType::FFmpegSoftware;
let sw_init_result = DecoderInitResult {
width: video_width,
height: video_height,
decoder_type: sw_decoder_type,
};
let _ = ready_tx.send(Ok(sw_init_result));
while let Ok(r) = rx.recv() {
if managed_stopped(managed.as_ref()) {
return;
}
let mut pending_requests: Vec<PendingRequest> = Vec::with_capacity(8);
let mut push_request =
|requested_time: f32, reply: oneshot::Sender<DecodedFrame>| {
if reply.is_closed() {
return;
}
let requested_time = requested_time.max(0.0);
let requested_frame = super::request_frame(requested_time, fps);
pending_requests.push(PendingRequest {
time: requested_time,
frame: requested_frame,
reply,
});
};
match r {
VideoDecoderMessage::GetFrame(requested_time, _, reply) => {
push_request(requested_time, reply);
}
}
while let Ok(msg) = rx.try_recv() {
if managed_stopped(managed.as_ref()) {
return;
}
match msg {
VideoDecoderMessage::GetFrame(requested_time, _, reply) => {
push_request(requested_time, reply);
}
}
}
pending_requests.sort_by_key(|r| r.frame);
for PendingRequest {
time: requested_time,
frame: requested_frame,
reply,
} in pending_requests
{
if managed_stopped(managed.as_ref()) {
return;
}
if reply.is_closed() {
continue;
}
let last_sent_number = sw_last_sent_frame.borrow().as_ref().map(|f| f.number);
let is_backward_seek = last_sent_number
.map(|last| requested_frame < last)
.unwrap_or(false);
if let Some(cached) = sw_cache.get_mut(&requested_frame) {
let data = cached.produce(&mut sw_converter);
if reply.send(data.frame.clone()).is_err() {
log::debug!(
"FFmpeg '{name}': Failed to send cached frame {requested_frame}: receiver dropped"
);
}
*sw_last_sent_frame.borrow_mut() = Some(data);
continue;
}
// Nothing precedes the stream's first frame, so it holds
// for any earlier request; a later one needs a seek.
if let Some(first_frame) = sw_first_ever_frame.borrow().clone()
&& requested_frame <= first_frame.number
{
*sw_last_sent_frame.borrow_mut() = Some(first_frame.clone());
let _ = reply.send(first_frame.frame);
continue;
}
// Requests inside a KNOWN pts hole are answered with the
// hole's start frame — the true VFR hold — without
// touching the decoder; see the hardware path below.
if let Some(hole_start) = sw_pts_holes
.range(..=requested_frame)
.next_back()
.filter(|&(_, &end)| requested_frame < end)
.map(|(&start, _)| start)
{
let data = sw_cache
.get_mut(&hole_start)
.map(|c| c.produce(&mut sw_converter))
.or_else(|| sw_gap_hold.clone().filter(|h| h.number == hole_start));
if let Some(data) = data {
sw_gap_hold = Some(data.clone());
*sw_last_sent_frame.borrow_mut() = Some(data.clone());
let _ = reply.send(data.frame);
continue;
}
}
if is_backward_seek {
if let Err(error) = sw_this.reset(requested_time)
&& let Some(control) = &managed
{
control.fail(managed_error(
control,
ManagedVideoError::Seek(error.to_string()),
));
return;
}
sw_frames = sw_this.frames();
*sw_last_sent_frame.borrow_mut() = None;
sw_cache.clear();
sw_prev_vended = None;
sw_reader_at = None;
}
if reply.is_closed() {
continue;
}
let reply_cell = Rc::new(RefCell::new(Some(reply)));
let reply_for_respond = reply_cell.clone();
let mut respond = {
let last_sent_frame = sw_last_sent_frame.clone();
Some(move |data: OutputFrame| {
let frame_number = data.number;
*last_sent_frame.borrow_mut() = Some(data.clone());
if let Some(reply) = reply_for_respond.borrow_mut().take()
&& reply.send(data.frame).is_err()
{
log::debug!(
"Failed to send decoded frame {frame_number}: receiver dropped"
);
}
})
};
let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2);
let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2;
// The reader has already passed this uncached frame: it fell
// outside the cache window, or the last sent frame was a cache
// hit away from the reader. Reading on cannot reach it.
let behind_reader = sw_reader_at.is_some_and(|at| requested_frame <= at);
let is_far_forward = sw_last_sent_frame.borrow().as_ref().map_or_else(
|| {
if sw_first_ever_frame.borrow().is_some() {
let first_frame_num = sw_first_ever_frame
.borrow()
.as_ref()
.map(|f| f.number)
.unwrap_or(0);
requested_frame.saturating_sub(first_frame_num)
> FRAME_CACHE_SIZE as u32
} else {
requested_frame != 0
}
},
|last| {
requested_frame.saturating_sub(last.number) > FRAME_CACHE_SIZE as u32
},
);
if is_far_forward || behind_reader {
if let Err(error) = sw_this.reset(requested_time)
&& let Some(control) = &managed
{
control.fail(managed_error(
control,
ManagedVideoError::Seek(error.to_string()),
));
return;
}
sw_frames = sw_this.frames();
*sw_last_sent_frame.borrow_mut() = None;
sw_cache.clear();
sw_prev_vended = None;
sw_reader_at = None;
}
let mut exit = false;
// Newest vended frame below the cache window: after a reset
// the reader re-vends from the keyframe before the request,
// and that frame is the true VFR hold for a request inside
// a pts hole.
let mut hold_candidate: Option<(u32, CachedFrame)> = None;
for frame in &mut sw_frames {
if let Some(control) = &managed {
if control.is_cancelled() {
control.fail(ManagedVideoError::Cancelled);
return;
}
if let Err(error) = &frame {
control.fail(ManagedVideoError::Decode(error.to_string()));
return;
}
}
if reply_cell.borrow().as_ref().is_none_or(|r| r.is_closed()) {
respond.take();
// The frame just pulled is discarded: it breaks
// the vend continuity that hole discovery
// relies on.
sw_prev_vended = None;
// The reader has still moved past it.
if let Ok(frame) = &frame
&& let Some(pts) = frame.pts()
{
sw_reader_at = Some(pts_to_frame(pts, sw_time_base, fps));
}
break;
}
let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else {
// A skipped frame breaks the vend continuity
// that hole discovery relies on.
sw_prev_vended = None;
continue;
};
let Some(pts) = frame.pts() else {
sw_prev_vended = None;
continue;
};
let current_frame = pts_to_frame(pts, sw_time_base, fps);
if let Some(prev) = sw_prev_vended
&& current_frame > prev + 1
{
record_pts_hole(&mut sw_pts_holes, prev, current_frame);
}
sw_prev_vended = Some(current_frame);
sw_reader_at = Some(current_frame);
let mut cache_frame = CachedFrame::Raw {
frame,
number: current_frame,
};
if sw_first_ever_frame.borrow().is_none() {
let output = cache_frame.produce(&mut sw_converter);
*sw_first_ever_frame.borrow_mut() = Some(output);
}
let exceeds_cache_bounds = current_frame > cache_max;
let too_small_for_cache_bounds = current_frame < cache_min;
if too_small_for_cache_bounds {
// Keep the newest pre-request frame as the VFR hold
// candidate instead of discarding it; a frame below
// the cache window can never exceed the request, so
// nothing else in this iteration applies to it.
if current_frame <= requested_frame {
hold_candidate = Some((current_frame, cache_frame));
}
continue;
}
{
cache_frame.produce(&mut sw_converter);
insert_cached_frame(
&mut sw_cache,
current_frame,
cache_frame,
requested_frame,
sw_last_active_frame,
MAX_FRAME_CACHE_BYTES,
);
// Serve exact matches from the cache so
// sequentially played frames stay available as
// at-or-before holds for later gap requests.
if current_frame == requested_frame
&& let Some(respond) = respond.take()
{
let output = sw_cache
.get_mut(¤t_frame)
.unwrap()
.produce(&mut sw_converter);
(respond)(output);
break;
}
}
if current_frame > requested_frame && respond.is_some() {
// A frame at-or-before the request is the true content for
// that time in a VFR recording: a pts gap means the frame
// stayed on screen, however long the gap is. Prefer the
// newest such frame among the cache, the hold candidate a
// reset re-vended, and the persistent gap hold.
let cached_before = sw_cache
.range(..=requested_frame)
.next_back()
.map(|(&n, _)| n);
let candidate_number = hold_candidate
.as_ref()
.map(|(n, _)| *n)
.filter(|&n| n <= requested_frame);
let hold_before = sw_gap_hold.clone().filter(|h| {
sw_pts_holes.get(&h.number).is_some_and(|&end| {
h.number <= requested_frame && requested_frame < end
})
});
let last_before = sw_last_sent_frame
.borrow()
.as_ref()
.filter(|l| l.number <= requested_frame)
.cloned();
let best_number = [
cached_before,
candidate_number,
hold_before.as_ref().map(|h| h.number),
last_before.as_ref().map(|l| l.number),
]
.into_iter()
.flatten()
.max();
if let Some(best_number) = best_number
&& let Some(respond) = respond.take()
{
let output = if cached_before == Some(best_number) {
sw_cache
.get_mut(&best_number)
.unwrap()
.produce(&mut sw_converter)
} else if candidate_number == Some(best_number) {
let (_, mut candidate) = hold_candidate.take().unwrap();
candidate.produce(&mut sw_converter)
} else if hold_before.as_ref().map(|h| h.number)
== Some(best_number)
{
hold_before.unwrap()
} else {
last_before.unwrap()
};
sw_gap_hold = Some(output.clone());
(respond)(output);
} else if let Some(respond) = respond.take() {
let output = sw_cache
.get_mut(¤t_frame)
.unwrap()
.produce(&mut sw_converter);
(respond)(output);
}
// The request is answered; stop here so the
// next sample stays in the decoder for the
// next request. Pulling it now would discard
// it at the reply guard and poison the vend
// continuity that hole discovery relies on.
break;
}
exit = exit || exceeds_cache_bounds;
if exit {
break;
}
}
sw_last_active_frame = Some(requested_frame);
if let Some(respond) = respond.take() {
// The newest frame at-or-before the request is always a
// valid VFR hold, regardless of how far back it is —
// whether it is cached, was re-vended below the cache
// window by a reset, or is the persistent gap hold.
let cached_before = sw_cache
.range(..=requested_frame)
.next_back()
.map(|(&n, _)| n);
let candidate_number = hold_candidate
.as_ref()
.map(|(n, _)| *n)
.filter(|&n| n <= requested_frame);
let hold_before = sw_gap_hold.clone().filter(|h| {
sw_pts_holes.get(&h.number).is_some_and(|&end| {
h.number <= requested_frame && requested_frame < end
})
});
let last_before = sw_last_sent_frame
.borrow()
.as_ref()
.filter(|l| l.number <= requested_frame)
.cloned();
let best_number = [
cached_before,
candidate_number,
hold_before.as_ref().map(|h| h.number),
last_before.as_ref().map(|l| l.number),
]
.into_iter()
.flatten()
.max();
if let Some(best_number) = best_number {
let output = if cached_before == Some(best_number) {
sw_cache
.get_mut(&best_number)
.unwrap()
.produce(&mut sw_converter)
} else if candidate_number == Some(best_number) {
let (_, mut candidate) = hold_candidate.take().unwrap();
candidate.produce(&mut sw_converter)
} else if hold_before.as_ref().map(|h| h.number) == Some(best_number) {
hold_before.unwrap()
} else {
last_before.unwrap()
};
sw_gap_hold = Some(output.clone());
(respond)(output);
} else {
let last_frame_clone = sw_last_sent_frame.borrow().clone();
let first_frame_clone = sw_first_ever_frame.borrow().clone();
if let Some(last_frame) = last_frame_clone {
(respond)(last_frame);
} else if let Some(first_frame) = first_frame_clone {
(respond)(first_frame);
} else {
let black_frame_data =
vec![0u8; (video_width * video_height * 4) as usize];
let black_frame = OutputFrame {
number: requested_frame,
frame: DecodedFrame::new_with_arc(
Arc::new(black_frame_data),
video_width,
video_height,
),
};
(respond)(black_frame);
}
}
}
}
}
return;
}
if first_frame_result.is_none()
&& let Some(control) = &managed
{
let error = control.fail(ManagedVideoError::NoFrames);
let _ = ready_tx.send(Err(error.to_string()));
return;
}
if let Some(frame) = first_frame_result {
let current_frame = pts_to_frame(frame.pts().unwrap_or(0), time_base, fps);
let mut cache_frame = CachedFrame::Raw {
frame,
number: current_frame,
};
let output = cache_frame.produce(&mut converter);
cache.insert(current_frame, cache_frame);
*first_ever_frame.borrow_mut() = Some(output.clone());
*last_sent_frame.borrow_mut() = Some(output);
// The pre-decoded frame is a real vend: without seeding this,
// an opening static hold would never be recorded.
prev_vended = Some(current_frame);
reader_at = Some(current_frame);
info!(
"FFmpeg decoder '{}': pre-decoded first frame {} ({}x{})",
name, current_frame, video_width, video_height
);
}
// A failed hardware setup falls back to software inside FFmpeg, which
// only shows once the first frame has been decoded.
let decoder_type = if frames.is_hardware_accelerated() {
DecoderType::FFmpegHardware
} else {
DecoderType::FFmpegSoftware
};
let init_result = DecoderInitResult {
width: video_width,
height: video_height,
decoder_type,
};
let _ = ready_tx.send(Ok(init_result));
// Export read-ahead: with no request pending, keep decoding forward
// into the cache so the renderer's next sequential request is served
// at once instead of waiting out NVDEC latency (~4.5 ms per 3K frame,
// which otherwise sits in every export frame). Bounded by `readahead`
// frames past the last request and by the cache's byte budget (hw
// surfaces). Off unless CAP_DECODER_READAHEAD is set: editor
// playback keeps its on-demand behaviour.
// More than the cache holds would evict the frames just decoded ahead.
#[cfg(target_os = "linux")]
let readahead_limit = (MAX_FRAME_CACHE_BYTES / CUDA_FRAME_CACHE_WEIGHT) as u32;
#[cfg(not(target_os = "linux"))]
let readahead_limit = FRAME_CACHE_SIZE as u32;
let readahead = std::env::var("CAP_DECODER_READAHEAD")
.ok()
.and_then(|value| value.parse::<u32>().ok())
.unwrap_or(0)
.min(readahead_limit);
let mut readahead_exhausted = false;
loop {
let r = if readahead > 0 {
match rx.try_recv() {
Ok(message) => message,
Err(mpsc::TryRecvError::Disconnected) => break,
Err(mpsc::TryRecvError::Empty) => {
// Cache hits don't touch `last_active_frame`; the
// last frame actually sent is the renderer's position.
let anchor = last_sent_frame
.borrow()
.as_ref()
.map(|sent| sent.number)
.or(last_active_frame);
if !readahead_exhausted && wants_readahead(&cache, anchor, readahead) {
if managed_stopped(managed.as_ref()) {
return;
}
match frames.next() {
Some(Ok(frame)) => {
if let Some(pts) = frame.pts() {
let current_frame = pts_to_frame(pts, time_base, fps);
if let Some(prev) = prev_vended
&& current_frame > prev + 1
{
record_pts_hole(&mut pts_holes, prev, current_frame);
}
prev_vended = Some(current_frame);
reader_at = Some(current_frame);
let mut cache_frame = CachedFrame::Raw {
frame,
number: current_frame,
};
cache_frame.produce(&mut converter);
insert_cached_frame(
&mut cache,
current_frame,
cache_frame,
current_frame,
last_active_frame,
MAX_FRAME_CACHE_BYTES,
);
} else {
prev_vended = None;
}
}
Some(Err(_)) => prev_vended = None,
None => readahead_exhausted = true,
}
continue;
}
match rx.recv() {
Ok(message) => message,
Err(_) => break,
}
}
}
} else {
match rx.recv() {
Ok(message) => message,
Err(_) => break,
}
};
if managed_stopped(managed.as_ref()) {
return;
}
let mut pending_requests: Vec<PendingRequest> = Vec::with_capacity(8);
let mut push_request = |requested_time: f32, reply: oneshot::Sender<DecodedFrame>| {
if reply.is_closed() {
return;
}
let requested_time = requested_time.max(0.0);
let requested_frame = super::request_frame(requested_time, fps);
pending_requests.push(PendingRequest {
time: requested_time,
frame: requested_frame,
reply,
});
};
match r {
VideoDecoderMessage::GetFrame(requested_time, _, reply) => {
push_request(requested_time, reply);
}
}
while let Ok(msg) = rx.try_recv() {
if managed_stopped(managed.as_ref()) {
return;
}
match msg {
VideoDecoderMessage::GetFrame(requested_time, _, reply) => {
push_request(requested_time, reply);
}
}
}
pending_requests.sort_by_key(|r| r.frame);
for PendingRequest {
time: requested_time,
frame: requested_frame,
reply,
} in pending_requests
{
if managed_stopped(managed.as_ref()) {
return;
}
if reply.is_closed() {
continue;
}
let last_sent_number = last_sent_frame.borrow().as_ref().map(|f| f.number);
let is_backward_seek = last_sent_number
.map(|last| requested_frame < last)
.unwrap_or(false);
if let Some(cached) = cache.get_mut(&requested_frame) {
let data = cached.produce(&mut converter);
if reply.send(data.frame.clone()).is_err() {
log::debug!(
"FFmpeg '{name}': Failed to send cached frame {requested_frame}: receiver dropped"
);
}
*last_sent_frame.borrow_mut() = Some(data);
continue;
}
// Nothing precedes the stream's first frame, so it holds
// for any earlier request; a later one needs a seek.
if let Some(first_frame) = first_ever_frame.borrow().clone()
&& requested_frame <= first_frame.number
{
*last_sent_frame.borrow_mut() = Some(first_frame.clone());
let _ = reply.send(first_frame.frame);
continue;
}
// Requests inside a KNOWN pts hole are answered with the
// hole's start frame — the true VFR hold (the frame simply
// stayed on screen) — without touching the decoder. This
// keeps the post-hole frames cached and the reader parked,
// however long the hole runs. Only recorded holes qualify: a
// bare "some cached frame lies beyond the request" test would
// also match disjoint cache islands left by seeks, and
// serving stale content there would freeze playback.
if let Some(hole_start) = pts_holes
.range(..=requested_frame)
.next_back()
.filter(|&(_, &end)| requested_frame < end)
.map(|(&start, _)| start)
{
let data = cache
.get_mut(&hole_start)
.map(|c| c.produce(&mut converter))
.or_else(|| gap_hold.clone().filter(|h| h.number == hole_start));
if let Some(data) = data {
gap_hold = Some(data.clone());
*last_sent_frame.borrow_mut() = Some(data.clone());
let _ = reply.send(data.frame);
continue;
}
}
if is_backward_seek {
if let Err(error) = this.reset(requested_time)
&& let Some(control) = &managed
{
control.fail(managed_error(
control,
ManagedVideoError::Seek(error.to_string()),
));
return;
}
frames = this.frames();
readahead_exhausted = false;
*last_sent_frame.borrow_mut() = None;
cache.clear();
prev_vended = None;
reader_at = None;
}
if reply.is_closed() {
continue;
}
let reply_cell = Rc::new(RefCell::new(Some(reply)));
let reply_for_respond = reply_cell.clone();
let mut respond = {
let last_sent_frame = last_sent_frame.clone();
Some(move |data: OutputFrame| {
let frame_number = data.number;
*last_sent_frame.borrow_mut() = Some(data.clone());
if let Some(reply) = reply_for_respond.borrow_mut().take()
&& reply.send(data.frame).is_err()
{
log::debug!(
"Failed to send decoded frame {frame_number}: receiver dropped"
);
}
})
};
let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2);
let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2;
// The reader has already passed this uncached frame: it fell
// outside the cache window, or the last sent frame was a cache
// hit away from the reader. Reading on cannot reach it.
let behind_reader = reader_at.is_some_and(|at| requested_frame <= at);
let is_far_forward = last_sent_frame.borrow().as_ref().map_or_else(
|| {
if first_ever_frame.borrow().is_some() {
let first_frame_num = first_ever_frame
.borrow()
.as_ref()
.map(|f| f.number)
.unwrap_or(0);
requested_frame.saturating_sub(first_frame_num)
> FRAME_CACHE_SIZE as u32
} else {
requested_frame != 0
}
},
|last| requested_frame.saturating_sub(last.number) > FRAME_CACHE_SIZE as u32,
);
if is_far_forward || behind_reader {
if let Err(error) = this.reset(requested_time)
&& let Some(control) = &managed
{
control.fail(managed_error(
control,
ManagedVideoError::Seek(error.to_string()),
));
return;
}
frames = this.frames();
readahead_exhausted = false;
*last_sent_frame.borrow_mut() = None;
cache.clear();
prev_vended = None;
reader_at = None;
}
let mut exit = false;
// Newest vended frame below the cache window: after a reset
// the reader re-vends from the keyframe before the request,
// and that frame is the true VFR hold for a request inside
// a pts hole.
let mut hold_candidate: Option<(u32, CachedFrame)> = None;
for frame in &mut frames {
if let Some(control) = &managed {
if control.is_cancelled() {
control.fail(ManagedVideoError::Cancelled);
return;
}
if let Err(error) = &frame {
control.fail(ManagedVideoError::Decode(error.to_string()));
return;
}
}
if reply_cell.borrow().as_ref().is_none_or(|r| r.is_closed()) {
respond.take();
// The frame just pulled is discarded: it breaks the
// vend continuity that hole discovery relies on.
prev_vended = None;
// The reader has still moved past it.
if let Ok(frame) = &frame
&& let Some(pts) = frame.pts()
{
reader_at = Some(pts_to_frame(pts, time_base, fps));
}
break;
}
let Ok(frame) = frame.map_err(|e| format!("read frame / {e}")) else {
// A skipped frame breaks the vend continuity that
// hole discovery relies on.
prev_vended = None;
continue;
};
let Some(pts) = frame.pts() else {
prev_vended = None;
continue;
};
let current_frame = pts_to_frame(pts, time_base, fps);
if let Some(prev) = prev_vended
&& current_frame > prev + 1
{
record_pts_hole(&mut pts_holes, prev, current_frame);
}
prev_vended = Some(current_frame);
reader_at = Some(current_frame);
let mut cache_frame = CachedFrame::Raw {
frame,
number: current_frame,
};
if first_ever_frame.borrow().is_none() {
let output = cache_frame.produce(&mut converter);
*first_ever_frame.borrow_mut() = Some(output);
}
let exceeds_cache_bounds = current_frame > cache_max;
let too_small_for_cache_bounds = current_frame < cache_min;
if too_small_for_cache_bounds {
// Keep the newest pre-request frame as the VFR hold
// candidate instead of discarding it; a frame below
// the cache window can never exceed the request, so
// nothing else in this iteration applies to it.
if current_frame <= requested_frame {
hold_candidate = Some((current_frame, cache_frame));
}
continue;
}
{
cache_frame.produce(&mut converter);
insert_cached_frame(
&mut cache,
current_frame,
cache_frame,
requested_frame,
last_active_frame,
MAX_FRAME_CACHE_BYTES,
);
// Serve exact matches from the cache so
// sequentially played frames stay available as
// at-or-before holds for later gap requests.
if current_frame == requested_frame
&& let Some(respond) = respond.take()
{
let output = cache
.get_mut(¤t_frame)
.unwrap()
.produce(&mut converter);
(respond)(output);
break;
}
}
if current_frame > requested_frame && respond.is_some() {
// A frame at-or-before the request is the true content for
// that time in a VFR recording: a pts gap means the frame
// stayed on screen, however long the gap is. Prefer the
// newest such frame among the cache, the hold candidate a
// reset re-vended, and the persistent gap hold.
let cached_before =
cache.range(..=requested_frame).next_back().map(|(&n, _)| n);
let candidate_number = hold_candidate
.as_ref()
.map(|(n, _)| *n)
.filter(|&n| n <= requested_frame);
let hold_before = gap_hold.clone().filter(|h| {
pts_holes.get(&h.number).is_some_and(|&end| {
h.number <= requested_frame && requested_frame < end
})
});
let last_before = last_sent_frame
.borrow()
.as_ref()
.filter(|l| l.number <= requested_frame)
.cloned();
let best_number = [
cached_before,
candidate_number,
hold_before.as_ref().map(|h| h.number),
last_before.as_ref().map(|l| l.number),
]
.into_iter()
.flatten()
.max();
if let Some(best_number) = best_number
&& let Some(respond) = respond.take()
{
let output = if cached_before == Some(best_number) {
cache.get_mut(&best_number).unwrap().produce(&mut converter)
} else if candidate_number == Some(best_number) {
let (_, mut candidate) = hold_candidate.take().unwrap();
candidate.produce(&mut converter)
} else if hold_before.as_ref().map(|h| h.number) == Some(best_number) {
hold_before.unwrap()
} else {
last_before.unwrap()
};
gap_hold = Some(output.clone());
(respond)(output);
} else if let Some(respond) = respond.take() {
let output = cache
.get_mut(¤t_frame)
.unwrap()
.produce(&mut converter);
(respond)(output);
}
// The request is answered; stop here so the next
// sample stays in the decoder for the next
// request. Pulling it now would discard it at the
// reply guard and poison the vend continuity that
// hole discovery relies on.
break;
}
exit = exit || exceeds_cache_bounds;
if exit {
break;
}
}
last_active_frame = Some(requested_frame);
if let Some(respond) = respond.take() {
// The newest frame at-or-before the request is always a valid
// VFR hold, regardless of how far back it is — whether it is
// cached, was re-vended below the cache window by a reset, or
// is the persistent gap hold.
let cached_before =
cache.range(..=requested_frame).next_back().map(|(&n, _)| n);
let candidate_number = hold_candidate
.as_ref()
.map(|(n, _)| *n)
.filter(|&n| n <= requested_frame);
let hold_before = gap_hold.clone().filter(|h| {
pts_holes.get(&h.number).is_some_and(|&end| {
h.number <= requested_frame && requested_frame < end
})
});
let last_before = last_sent_frame
.borrow()
.as_ref()
.filter(|l| l.number <= requested_frame)
.cloned();
let best_number = [
cached_before,
candidate_number,
hold_before.as_ref().map(|h| h.number),
last_before.as_ref().map(|l| l.number),
]
.into_iter()
.flatten()
.max();
if let Some(best_number) = best_number {
let output = if cached_before == Some(best_number) {
cache.get_mut(&best_number).unwrap().produce(&mut converter)
} else if candidate_number == Some(best_number) {
let (_, mut candidate) = hold_candidate.take().unwrap();
candidate.produce(&mut converter)
} else if hold_before.as_ref().map(|h| h.number) == Some(best_number) {
hold_before.unwrap()
} else {
last_before.unwrap()
};
gap_hold = Some(output.clone());
(respond)(output);
} else {
let last_frame_clone = last_sent_frame.borrow().clone();
let first_frame_clone = first_ever_frame.borrow().clone();
if let Some(last_frame) = last_frame_clone {
(respond)(last_frame);
} else if let Some(first_frame) = first_frame_clone {
(respond)(first_frame);
} else {
let black_frame_data =
vec![0u8; (video_width * video_height * 4) as usize];
let black_frame = OutputFrame {
number: requested_frame,
frame: DecodedFrame::new_with_arc(
Arc::new(black_frame_data),
video_width,
video_height,
),
};
(respond)(black_frame);
}
}
}
}
}
}
}
#[cfg(test)]
mod cache_tests {
use super::*;
#[test]
fn readahead_stops_at_its_window() {
let mut cache = BTreeMap::new();
assert!(!wants_readahead(&cache, None, 4), "no request served yet");
assert!(wants_readahead(&cache, Some(10), 4));
for number in 10..14 {
cache.insert(number, frame(number, 1));
}
assert!(wants_readahead(&cache, Some(10), 4));
cache.insert(14, frame(14, 1));
assert!(!wants_readahead(&cache, Some(10), 4), "window full");
assert!(wants_readahead(&cache, Some(12), 4), "renderer moved on");
assert!(!wants_readahead(&cache, Some(10), 0), "disabled");
}
fn frame(number: u32, bytes: usize) -> CachedFrame {
CachedFrame::Processed(ProcessedFrame {
number,
data: Arc::new(vec![0; bytes]),
width: 2,
height: 2,
format: PixelFormat::Nv12,
matrix: YuvMatrix::Bt601,
y_stride: 2,
uv_stride: 2,
planes: Arc::new(PlanePool::new()),
#[cfg(target_os = "linux")]
cuda: None,
})
}
#[test]
fn forward_playback_keeps_recent_frames_within_byte_budget() {
let mut cache = BTreeMap::new();
for number in 0..5 {
insert_cached_frame(&mut cache, number, frame(number, 6), 5, Some(4), 13);
}
assert_eq!(cache.keys().copied().collect::<Vec<_>>(), vec![3, 4]);
assert_eq!(
cache
.values()
.map(CachedFrame::estimated_bytes)
.sum::<usize>(),
12
);
}
#[test]
fn backward_seek_keeps_earliest_frames_within_byte_budget() {
let mut cache = BTreeMap::new();
for number in [5, 4, 3, 2] {
insert_cached_frame(&mut cache, number, frame(number, 5), 2, Some(6), 11);
}
assert_eq!(cache.keys().copied().collect::<Vec<_>>(), vec![2, 3]);
assert_eq!(
cache
.values()
.map(CachedFrame::estimated_bytes)
.sum::<usize>(),
10
);
}
#[test]
fn oversize_frame_remains_available_without_retaining_older_frames() {
let mut cache = BTreeMap::new();
insert_cached_frame(&mut cache, 1, frame(1, 4), 1, Some(1), 8);
insert_cached_frame(&mut cache, 2, frame(2, 12), 2, Some(1), 8);
assert_eq!(cache.keys().copied().collect::<Vec<_>>(), vec![2]);
assert_eq!(cache[&2].estimated_bytes(), 12);
}
#[test]
fn mismatched_v_stride_is_repacked_to_the_u_stride() {
let pool = PlanePool::new();
let y = vec![4u8; 8];
let u = vec![9u8; 8];
let wide_v = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
let packed = pack_yuv420p(&pool, &y, &u, &wide_v, 4, 6, 2);
assert_eq!(
packed,
[
4, 4, 4, 4, 4, 4, 4, 4, 9, 9, 9, 9, 9, 9, 9, 9, 1, 2, 3, 4, 7, 8, 9, 10,
]
);
let short_v = vec![1, 2, 3, 4];
let packed = pack_yuv420p(&pool, &y, &u, &short_v, 4, 2, 2);
assert_eq!(
&packed[y.len() + u.len()..],
&[1, 2, 128, 128, 3, 4, 128, 128]
);
let v = [7, 8, 9];
let packed = pack_yuv420p(&pool, &[1, 2, 3, 4], &[5, 6], &v, 2, 2, 1);
assert_eq!(packed, [1, 2, 3, 4, 5, 6, 7, 8]);
}
#[test]
fn released_software_frame_reuses_its_plane_buffer() {
for pixel_format in [
format::Pixel::YUV420P,
format::Pixel::NV12,
format::Pixel::RGBA,
] {
let mut converter = FrameConverter::new();
let (cached, output) = decoded_packed(&mut converter, pixel_format, 1);
let ptr = output.frame.data().as_ptr() as usize;
assert_eq!(output.frame.data()[0], 1);
drop(output);
drop(cached);
let (_cached, output) = decoded_packed(&mut converter, pixel_format, 2);
assert_eq!(
output.frame.data().as_ptr() as usize,
ptr,
"{pixel_format:?}"
);
assert_eq!(output.frame.data()[0], 2);
}
}
#[test]
fn renderer_held_frame_is_not_recycled() {
let mut converter = FrameConverter::new();
let (cached, output) = decoded_packed(&mut converter, format::Pixel::YUV420P, 7);
let ptr = output.frame.data().as_ptr() as usize;
let y = output.frame.y_plane().unwrap().to_vec();
drop(cached);
let (_cached, next) = decoded_packed(&mut converter, format::Pixel::YUV420P, 8);
assert_ne!(next.frame.data().as_ptr() as usize, ptr);
assert_eq!(output.frame.y_plane().unwrap(), y.as_slice());
assert_eq!(output.frame.data()[0], 7);
}
#[test]
fn releasing_the_cache_keeps_two_spare_planes() {
let mut converter = FrameConverter::new();
let mut held = Vec::new();
for fill in 1..7u8 {
held.push(decoded_packed(&mut converter, format::Pixel::YUV420P, fill));
}
for (cached, output) in held {
drop(output);
drop(cached);
}
assert_eq!(converter.plane_pool().spare_count(), 2);
}
#[test]
fn pooled_1080p_yuv_copy_reuses_one_allocation() {
let pool = PlanePool::new();
let width = 1920usize;
let height = 1080usize;
let uv_height = height / 2;
let u_stride = width / 2;
let y = vec![16u8; width * height];
let u = vec![128u8; u_stride * uv_height];
let v = vec![90u8; u_stride * uv_height];
let total = y.len() + u.len() + v.len();
let u_stride = u_stride as u32;
let uv_height = uv_height as u32;
let warm = pack_yuv420p(&pool, &y, &u, &v, u_stride, u_stride, uv_height);
assert_eq!(warm.len(), total);
assert_eq!(warm[0], 16);
assert_eq!(warm[y.len()], 128);
assert_eq!(warm[y.len() + u.len()], 90);
let ptr = warm.as_ptr() as usize;
pool.recycle(warm);
let reused = pack_yuv420p(&pool, &y, &u, &v, u_stride, u_stride, uv_height);
assert_eq!(reused.as_ptr() as usize, ptr);
assert_eq!(reused.len(), total);
pool.recycle(reused);
let iters = 40;
let fresh = time_fresh_copies(&y, &u, &v, iters);
let pooled = time_pooled_copies(&pool, &y, &u, &v, u_stride, uv_height, iters);
let fresh_again = time_fresh_copies(&y, &u, &v, iters);
let fresh_alloc = time_fresh_alloc(total, iters);
let pooled_alloc = time_pooled_alloc(&pool, total, iters);
eprintln!(
"yuv420p 1920x1080 bytes={total} iters={iters} fresh_copy_us={} pooled_copy_us={} fresh_copy_again_us={} fresh_alloc_us={} pooled_alloc_us={}",
fresh.as_micros(),
pooled.as_micros(),
fresh_again.as_micros(),
fresh_alloc.as_micros(),
pooled_alloc.as_micros()
);
}
fn decoded_packed(
converter: &mut FrameConverter,
pixel_format: format::Pixel,
fill: u8,
) -> (CachedFrame, OutputFrame) {
let mut source = frame::Video::new(pixel_format, 32, 16);
for plane in 0..source.planes() {
source.data_mut(plane).fill(fill.wrapping_add(plane as u8));
}
let mut cached = CachedFrame::Raw {
frame: source,
number: u32::from(fill),
};
let output = cached.produce(converter);
(cached, output)
}
fn time_fresh_copies(y: &[u8], u: &[u8], v: &[u8], iters: u32) -> std::time::Duration {
let start = std::time::Instant::now();
for _ in 0..iters {
let mut buf = Vec::with_capacity(y.len() + u.len() + v.len());
buf.extend_from_slice(y);
buf.extend_from_slice(u);
buf.extend_from_slice(v);
std::hint::black_box(buf.as_ptr());
}
start.elapsed()
}
fn time_pooled_copies(
pool: &PlanePool,
y: &[u8],
u: &[u8],
v: &[u8],
u_stride: u32,
uv_height: u32,
iters: u32,
) -> std::time::Duration {
let start = std::time::Instant::now();
for _ in 0..iters {
let buf = pack_yuv420p(pool, y, u, v, u_stride, u_stride, uv_height);
std::hint::black_box(buf.as_ptr());
pool.recycle(buf);
}
start.elapsed()
}
fn time_fresh_alloc(total: usize, iters: u32) -> std::time::Duration {
let start = std::time::Instant::now();
for _ in 0..iters {
let buf = Vec::<u8>::with_capacity(total);
std::hint::black_box(buf.capacity());
}
start.elapsed()
}
fn time_pooled_alloc(pool: &PlanePool, total: usize, iters: u32) -> std::time::Duration {
let warm = pool.take(total);
pool.recycle(warm);
let start = std::time::Instant::now();
for _ in 0..iters {
let buf = pool.take(total);
std::hint::black_box(buf.capacity());
pool.recycle(buf);
}
start.elapsed()
}
}
// pub fn find_decoder(
// s: &format::context::Input,
// st: &format::stream::Stream,
// codec_id: codec::Id,
// ) -> Option<Codec> {
// unsafe {
// use ffmpeg::media::Type;
// let codec = match st.parameters().medium() {
// Type::Video => Some((*s.as_ptr()).video_codec),
// Type::Audio => Some((*s.as_ptr()).audio_codec),
// Type::Subtitle => Some((*s.as_ptr()).subtitle_codec),
// _ => None,
// };
// if let Some(codec) = codec {
// if !codec.is_null() {
// return Some(Codec::wrap(codec));
// }
// }
// let found = avcodec_find_decoder(codec_id.into());
// if found.is_null() {
// return None;
// }
// Some(Codec::wrap(found))
// }
// }
// struct PeekableReceiver<T> {
// rx: mpsc::Receiver<T>,
// peeked: Option<T>,
// }
// impl<T> PeekableReceiver<T> {
// fn peek(&mut self) -> Option<&T> {
// if self.peeked.is_some() {
// self.peeked.as_ref()
// } else {
// match self.rx.try_recv() {
// Ok(value) => {
// self.peeked = Some(value);
// self.peeked.as_ref()
// }
// Err(_) => None,
// }
// }
// }
// fn recv(&mut self) -> Result<T, mpsc::RecvError> {
// if let Some(value) = self.peeked.take() {
// Ok(value)
// } else {
// self.rx.recv()
// }
// }
// }
#[cfg(test)]
mod managed_frame_tests {
use super::*;
use std::cell::Cell;
#[test]
fn managed_first_frame_stops_after_one_sticky_error() {
let control = ManagedVideoControl::new();
let pulls = Cell::new(0);
let mut frames = std::iter::from_fn(|| {
pulls.set(pulls.get() + 1);
Some(Err(ffmpeg::Error::InvalidData))
});
assert!(matches!(
first_frame(&mut frames, Some(&control)),
Err(ManagedVideoError::Decode(_))
));
assert_eq!(pulls.get(), 1);
assert!(first_frame(&mut frames, Some(&control)).is_err());
assert_eq!(pulls.get(), 1);
}
#[test]
fn ordinary_first_frame_still_skips_recoverable_errors() {
let mut frames = [Err(ffmpeg::Error::InvalidData), Ok(frame::Video::empty())].into_iter();
assert!(first_frame(&mut frames, None).unwrap().is_some());
assert!(frames.next().is_none());
}
#[test]
fn cancelled_first_frame_never_reads_or_reports_eof() {
let control = ManagedVideoControl::new();
control.cancel();
let mut frames = std::iter::from_fn(|| {
panic!("Cancellation must be observed before requesting another frame")
});
assert!(matches!(
first_frame(&mut frames, Some(&control)),
Err(ManagedVideoError::Cancelled)
));
}
}