The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(media): the frame cache keys on the source frame, and a scheduler decodes ahead
Jareer7311d2dae98b
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rsindex b1b58235..1d89233c 100644--- a/src/crates/concat-export/src/lib.rs+++ b/src/crates/concat-export/src/lib.rs@@ -975,6 +975,7 @@ fn render_picture(codec: request.codec,ten_bit: request.ten_bit,hardware: true,+ threads: 0,},).map_err(|error| error.to_string())?;diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rsindex 4f33157e..eb1fb11f 100644--- a/src/crates/concat-host/tests/export.rs+++ b/src/crates/concat-host/tests/export.rs@@ -91,6 +91,7 @@ fn picture(path: &Path, rate: FrameRate, seconds: u32) {crf: 16,ten_bit: false,hardware: false,+ threads: 0,};let mut encoder =Encoder::create(path, WIDTH, HEIGHT, rate, &options).expect("the linked FFmpeg encodes");diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rsindex a6f914e4..c232c85c 100644--- a/src/crates/concat-media/src/decode.rs+++ b/src/crates/concat-media/src/decode.rs@@ -101,6 +101,12 @@ pub struct DecodeOptions {/// decoded frame. Never for anything a person will watch or export: the/// frame is near the instant, not at it.pub keyframes_only: bool,+ /// Threads the codec may use, or zero to let it count the cores.+ ///+ /// Zero for anything a person is waiting on: a scrub wants its frame+ /// on every core. A few for a proxy being written in the background,+ /// where the whole point is that the machine stays usable meanwhile.+ pub threads: u16,}impl DecodeOptions {@@ -153,6 +159,12 @@ impl DecodeOptions {self.keyframes_only = true;self}++ /// Limits the codec to `threads`. See [`DecodeOptions::threads`].+ pub fn threaded(mut self, threads: u16) -> Self {+ self.threads = threads;+ self+ }}/// How a stream says its colour is encoded: the tags in its container,@@ -330,13 +342,14 @@ impl Decoder {// a seek decodes every frame from the keyframe before it: on one// thread a 4K H.264 file decoded at about 87 frames a second, and a// scrub waited 400 ms for its frame. Frame threads for that walk,- // slice threads for files cut into slices; zero lets the codec count.+ // slice threads for files cut into slices; zero lets the codec+ // count, and a caller working in the background says how many.// SAFETY: the context is not opened yet, which is when threading is// set, and both fields are plain integers.unsafe {let raw = context.as_mut_ptr();(*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;- (*raw).thread_count = 0;+ (*raw).thread_count = i32::from(options.threads);}let mut decoder = context.decoder()@@ -764,6 +777,7 @@ mod tests {crf: 20,ten_bit: true,hardware: false,+ threads: 0,};let mut encoder = Encoder::create_tagged(&path,diff --git a/src/crates/concat-media/src/encode.rs b/src/crates/concat-media/src/encode.rsindex 8aab0818..1717b1a0 100644--- a/src/crates/concat-media/src/encode.rs+++ b/src/crates/concat-media/src/encode.rs@@ -135,6 +135,10 @@ pub struct EncodeOptions {/// Let the platform's hardware encoder lead where there is one; see/// [`VideoCodec::encoders`].pub hardware: bool,+ /// Threads the encoder may use, or zero to let it count the cores.+ /// Zero for an export; a few for a proxy written while the editor is+ /// in use.+ pub threads: u16,}impl Default for EncodeOptions {@@ -145,6 +149,7 @@ impl Default for EncodeOptions {crf: 18,ten_bit: false,hardware: true,+ threads: 0,}}}@@ -295,12 +300,15 @@ impl Encoder {video.set_colorspace(matrix);video.set_color_range(ffmpeg::color::Range::MPEG);// SAFETY: `video` owns a live AVCodecContext; primaries and- // transfer have no setter in the bindings, and both are plain+ // transfer have no setter in the bindings, and all three are plain// fields the encoder reads at open.unsafe {let context = video.as_mut_ptr();(*context).color_primaries = primaries.into();(*context).color_trc = transfer.into();+ if options.threads > 0 {+ (*context).thread_count = i32::from(options.threads);+ }}if global_header {video.set_flags(ffmpeg::codec::Flags::GLOBAL_HEADER);@@ -645,6 +653,7 @@ mod tests {crf: 24,ten_bit,hardware: true,+ threads: 0,};let mut encoder = Encoder::create(&path, 64, 64, FrameRate::THIRTY, &options).unwrap_or_else(|error| panic!("{} {ten_bit}: {error}", codec.label()));diff --git a/src/crates/concat-media/src/lib.rs b/src/crates/concat-media/src/lib.rsindex f09e8cfb..45862202 100644--- a/src/crates/concat-media/src/lib.rs+++ b/src/crates/concat-media/src/lib.rs@@ -20,6 +20,7 @@ pub mod error;mod ffi;pub mod peaks;pub mod pool;+pub mod prefetch;pub mod probe;pub mod samples;pub mod treat;@@ -29,7 +30,8 @@ pub use encode::{EncodeOptions, Encoder, FrameSink, VideoCodec, jpeg};pub use error::{Error, Result};pub use ffi::{init, linked_version};pub use peaks::Peaks;-pub use pool::{FrameCache, ReaderPool};+pub use pool::{CacheStats, FrameCache, FrameRequest, ReaderPool};+pub use prefetch::{Cursor, Direction, Moment, Prefetcher, Priority};pub use probe::{AudioStream, MediaInfo, VideoStream, probe};pub use samples::{AudioDecoder, AudioOptions, SampleFormat};-pub use treat::{treat, treat_to};+pub use treat::{fit, treat, treat_to};diff --git a/src/crates/concat-media/src/pool.rs b/src/crates/concat-media/src/pool.rsindex ca6bb277..92281145 100644--- a/src/crates/concat-media/src/pool.rs+++ b/src/crates/concat-media/src/pool.rs@@ -8,21 +8,38 @@//! interactive use, where scrubbing asks for arbitrary (media, time) pairs.//! This module is the answer://!-//! - **One reader per (file, size, chain)**, kept warm between requests. A+//! - **One reader per (file, level)**, kept warm between requests. A//! request near the reader's current position rolls forward (cheap,//! exact); a request elsewhere seeks - frame-accurately, guided by the//! real timestamps the linked decoder reports.-//! - **A byte-budgeted LRU frame cache** in front of the readers. Scrubbing-//! back over ground just covered, or dwelling on one frame, costs a hash-//! lookup. Frames roll into the cache as decoding passes them, so rolling-//! forward to frame N caches N-1 frames of the path there.+//! - **A byte-budgeted LRU of source frames** in front of the readers,+//! keyed by the file, the level it was decoded at and the frame index -+//! and nothing else. A source frame is the picture as the file holds+//! it, turned the right way up, brought into BT.709 and scaled to a+//! level: the file's own size halved as many times as still covers what+//! was asked for. The crop and the effect chain are not in the key. They+//! are applied to the cached picture on the way out, and their results+//! kept in a second, smaller cache, so turning an effect knob costs a+//! filter per frame and not a decode per frame, and scrubbing back over+//! ground already covered costs a hash lookup whatever the knobs say.+//! - **Proxies.** A file can be given a stand-in - a smaller copy of+//! itself - and a request that says so reads that instead, under its own+//! key. Playback reads the proxy; the paused monitor reads the original.//!-//! The pool is deliberately not used by export: export decodes every frame-//! exactly once in order, and a cache in that path is pure overhead. This is-//! playback infrastructure.+//! Frames roll into the cache as decoding passes them, so rolling forward+//! to frame N caches N-1 frames of the path there.+//!+//! Export decodes every frame exactly once in order and does not come+//! here, with one exception: a clip that runs backwards or on a speed+//! curve cannot be followed by a paced decoder, and each of its frames is+//! sought through [`ReaderPool::frame_at`], which bakes the chain into the+//! decoder the way the paced decoders beside it do. The two paths of one+//! export then agree to the pixel.use std::collections::HashMap;+use std::hash::Hash;use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicU64, Ordering};use std::sync::{Arc, Mutex};use concat_core::frame::Frame;@@ -37,34 +54,124 @@ use crate::probe;/// sixty decodes - comparable to a seek, and exact.const ROLL_FORWARD_FRAMES: i64 = 60;-/// What one pooled frame request asks for.+/// What one frame request asks for; see [`ReaderPool::frame`].#[derive(Clone, PartialEq, Eq, Hash, Debug)]-struct FrameKey {+pub struct FrameRequest {+ /// The media file.+ pub path: PathBuf,+ /// The instant, in the media's own clock.+ pub time: Rational,+ /// The size the whole picture would have at the scale asked for, crop+ /// or no crop: what picks the level the source is decoded at. A clip+ /// cropped to its middle half and fitted to 960 across needs the whole+ /// picture at 1920, and this is where a caller says so.+ pub cover: (u32, u32),+ /// The size the frame comes out at.+ pub size: (u32, u32),+ /// An image file: one frame, served for every time, which the caller+ /// knows from its own model and the pool must not guess from probing.+ pub still: bool,+ /// The chain run in the picture's own pixels before the fit: the crop.+ pub pre: Option<String>,+ /// The effect chain, run at the output size after the fit.+ pub chain: Option<String>,+ /// Read the file's proxy where it has one; see+ /// [`ReaderPool::adopt_proxy`].+ pub proxy: bool,+}++impl FrameRequest {+ /// A plain request: the frame of `path` at `time`, `width` by `height`,+ /// no crop, no chain, the original file.+ pub fn new(path: impl Into<PathBuf>, time: Rational, width: u32, height: u32) -> Self {+ Self {+ path: path.into(),+ time,+ cover: (width, height),+ size: (width, height),+ still: false,+ pre: None,+ chain: None,+ proxy: false,+ }+ }++ /// Marks the file an image. See [`FrameRequest::still`].+ pub fn as_still(mut self, still: bool) -> Self {+ self.still = still;+ self+ }++ /// Sets the size the whole picture has at this scale. See+ /// [`FrameRequest::cover`].+ pub fn covering(mut self, width: u32, height: u32) -> Self {+ self.cover = (width, height);+ self+ }++ /// Applies `pre` before the fit; empty for none.+ pub fn prefiltered(mut self, pre: Option<&str>) -> Self {+ self.pre = pre+ .filter(|chain| !chain.trim().is_empty())+ .map(str::to_owned);+ self+ }++ /// Applies `chain` after the fit; empty for none.+ pub fn filtered(mut self, chain: Option<&str>) -> Self {+ self.chain = chain+ .filter(|chain| !chain.trim().is_empty())+ .map(str::to_owned);+ self+ }++ /// Reads the proxy where there is one. See [`FrameRequest::proxy`].+ pub fn from_proxy(mut self, proxy: bool) -> Self {+ self.proxy = proxy;+ self+ }+}++/// A source frame's identity: the file, the level it was decoded at, the+/// frame. The stream is implicit - a file's best video stream is the one+/// every reader opens.+#[derive(Clone, PartialEq, Eq, Hash, Debug)]+struct SourceKey {path: PathBuf,width: u32,height: u32,- /// The clip's effect chain, part of the frame's identity: the same source- /// frame through two different chains is two different pictures.- chain: Option<String>,- /// The pre-fit chain - a crop - on the same terms.- pre: Option<String>,index: i64,}+/// A treated frame's identity: the source frame and what was done to it.+#[derive(Clone, PartialEq, Eq, Hash, Debug)]+struct TreatedKey {+ source: SourceKey,+ width: u32,+ height: u32,+ pre: Option<String>,+ chain: Option<String>,+ /// Made by a decoder with the chain in its graph rather than from a+ /// cached source frame - export's way, see [`ReaderPool::frame_at`].+ /// The crop then ran in the file's own pixels and not the level's, so+ /// the pictures can differ by a rounding and must not share a key.+ baked: bool,+}+/// A byte-budgeted LRU of decoded frames.////// Plain and measurable on purpose: a `HashMap` plus a logical clock, evicting/// the least-recently-touched frame until the budget holds. At preview sizes a/// frame is ~2-4 MB, so the default budget holds a few hundred frames - many/// seconds of scrub history.-pub struct FrameCache {+pub struct FrameCache<K> {budget: usize,held: usize,tick: u64,- frames: HashMap<FrameKey, (Arc<Frame>, u64)>,+ frames: HashMap<K, (Arc<Frame>, u64)>,}-impl FrameCache {+impl<K: Hash + Eq + Clone> FrameCache<K> {/// An empty cache holding at most `budget` bytes of frames.pub fn new(budget: usize) -> Self {Self {@@ -75,7 +182,7 @@ impl FrameCache {}}- fn get(&mut self, key: &FrameKey) -> Option<Arc<Frame>> {+ fn get(&mut self, key: &K) -> Option<Arc<Frame>> {self.tick += 1;let tick = self.tick;self.frames.get_mut(key).map(|(frame, touched)| {@@ -84,7 +191,7 @@ impl FrameCache {})}- fn insert(&mut self, key: FrameKey, frame: Arc<Frame>) {+ fn insert(&mut self, key: K, frame: Arc<Frame>) {let bytes = frame.pixels().len();// A frame larger than the whole budget would evict everything and// still not fit; hold it once without caching it.@@ -111,13 +218,12 @@ impl FrameCache {}}- /// Bytes currently held.- /// Drops every frame cut from `path`.- fn forget(&mut self, path: &Path) {- let gone: Vec<FrameKey> = self+ /// Drops every frame `doomed` says to.+ fn drop_where(&mut self, doomed: impl Fn(&K) -> bool) {+ let gone: Vec<K> = self.frames.keys()- .filter(|key| key.path == path)+ .filter(|key| doomed(key)).cloned().collect();for key in gone {@@ -165,6 +271,28 @@ fn plan_access(current_next: Option<i64>, target: i64) -> Access {}}+/// An even size no smaller than two on a side: what the scaler and the+/// codecs are happiest with.+fn even(width: u32, height: u32) -> (u32, u32) {+ ((width & !1).max(2), (height & !1).max(2))+}++/// The level a source of `source` is decoded at to cover `cover`: the+/// source halved as many times as still covers it on both sides. A source+/// no bigger than the cover is decoded at its own size; nothing is ever+/// decoded larger than the file.+fn level_for(source: (u32, u32), cover: (u32, u32)) -> (u32, u32) {+ let (mut width, mut height) = even(source.0, source.1);+ loop {+ let (half_w, half_h) = even(width / 2, height / 2);+ if half_w < cover.0 || half_h < cover.1 || (half_w, half_h) == (width, height) {+ break;+ }+ (width, height) = (half_w, half_h);+ }+ (width, height)+}+/// One warm reader: a decoder plus where its *next* frame will land.struct Reader {decoder: Decoder,@@ -186,7 +314,7 @@ impl Reader {) -> Result<Self> {// Unpaced, so every source frame comes out with its own timestamp// and the index is read from that; a seek lands on the keyframe at- // or before the target and `frame_at` rolls forward from there.+ // or before the target and the pull rolls forward from there.let mut options = DecodeOptions::default().starting_at(rate.time_of_frame(index)).scaled_to(width, height);@@ -242,9 +370,29 @@ struct MediaFacts {/// a seek past the end produces zero frames, not an error and not a/// picture.frames: Option<i64>,+ /// The picture's displayed size, for a file with one: what the levels+ /// are halvings of.+ size: Option<(u32, u32)>,}-/// One reader's identity: the file, the decode size, the effect chain.+impl MediaFacts {+ /// The frame on screen at `time`: index zero for a still, and never+ /// past the last frame the container claims. A clip can outlive its+ /// media - an end-trim past the file's length - and the honest picture+ /// for any time past the end is the last frame.+ fn index_at(&self, time: Rational) -> i64 {+ if self.still {+ return 0;+ }+ let mut target = self.rate.frame_at(time).max(0);+ if let Some(frames) = self.frames {+ target = target.min((frames - 1).max(0));+ }+ target+ }+}++/// One reader's identity: the file, the decode size, the baked chains.type ReaderKey = (PathBuf, u32, u32, Option<String>, Option<String>);/// The warm readers and their recency, behind one short lock. A reader is@@ -258,17 +406,54 @@ struct Readers {max: usize,}+/// What the pool has done so far: the numbers a scrub benchmark asserts.+#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]+pub struct CacheStats {+ /// Frames that came out of a decoder.+ pub decoded: u64,+ /// Requests answered from a cached source frame without decoding,+ /// treated on the way out or not.+ pub source_hits: u64,+ /// Requests answered from a cached treated frame: no decode, no filter.+ pub treated_hits: u64,+ /// Source frames put through a crop, a chain or a fit on the way out.+ pub treated: u64,+}++impl CacheStats {+ /// What happened between `earlier` and this snapshot.+ pub fn since(self, earlier: CacheStats) -> CacheStats {+ CacheStats {+ decoded: self.decoded - earlier.decoded,+ source_hits: self.source_hits - earlier.source_hits,+ treated_hits: self.treated_hits - earlier.treated_hits,+ treated: self.treated - earlier.treated,+ }+ }+}++#[derive(Default)]+struct Counters {+ decoded: AtomicU64,+ source_hits: AtomicU64,+ treated_hits: AtomicU64,+ treated: AtomicU64,+}+/// Random access to frames across many files, cache in front, warm readers/// behind.////// Shared, not serialised: every method takes `&self`, and the locks inside-/// are held for as little as each step needs. The cache is one short lock;+/// are held for as little as each step needs. The caches are short locks;/// each reader is its own, so the playback stream decoding ahead on one/// file and the monitor pulling a cached frame of another never queue/// behind each other. Two callers wanting the *same* reader take turns,/// which is what a single decoder demands anyway.pub struct ReaderPool {- cache: Mutex<FrameCache>,+ /// Source frames: the file at a level, untreated.+ sources: Mutex<FrameCache<SourceKey>>,+ /// Source frames after their crop, fit and chain, at the output size.+ treated: Mutex<FrameCache<TreatedKey>>,readers: Mutex<Readers>,facts: Mutex<HashMap<PathBuf, Arc<MediaFacts>>>,/// Stills that exist nowhere but here: a title painted while its words@@ -277,6 +462,10 @@ pub struct ReaderPool {/// chain a plan asks - see [`ReaderPool::hold_still`]. The newest/// [`STILLS_HELD`] stay.memory: Mutex<HeldStills>,+ /// The stand-in for each file that has one; see+ /// [`ReaderPool::adopt_proxy`].+ proxies: Mutex<HashMap<PathBuf, PathBuf>>,+ counters: Counters,}/// The stills held in memory, and the order they came in.@@ -292,10 +481,15 @@ const STILLS_HELD: usize = 24;impl ReaderPool {/// A pool with `cache_bytes` of frame cache and up to `max_readers` warm- /// decoders.+ /// decoders. A quarter of the budget goes to treated frames, which are+ /// output-sized and few - the frames under and just ahead of the+ /// playhead - and the rest to source frames, which are the scrub+ /// history.pub fn new(cache_bytes: usize, max_readers: usize) -> Self {+ let treated = cache_bytes / 4;Self {- cache: Mutex::new(FrameCache::new(cache_bytes)),+ sources: Mutex::new(FrameCache::new(cache_bytes - treated)),+ treated: Mutex::new(FrameCache::new(treated)),readers: Mutex::new(Readers {warm: HashMap::new(),order: Vec::new(),@@ -303,9 +497,16 @@ impl ReaderPool {}),facts: Mutex::new(HashMap::new()),memory: Mutex::new((HashMap::new(), std::collections::VecDeque::new())),+ proxies: Mutex::new(HashMap::new()),+ counters: Counters::default(),}}+ /// 512 MB of frames, eight warm readers - enough for a busy timeline.+ pub fn with_defaults() -> Self {+ Self::new(512 * 1024 * 1024, 8)+ }+/// Makes `frame` the still at `path`, a name no file has, until/// [`STILLS_HELD`] newer ones have come. Frames already cut from an/// earlier still of that name are let go, so the name never shows@@ -322,9 +523,7 @@ impl ReaderPool {}}}- if let Ok(mut cache) = self.cache.lock() {- cache.forget(path);- }+ self.forget(path);if let Ok(mut facts) = self.facts.lock() {facts.insert(path.to_path_buf(),@@ -332,6 +531,7 @@ impl ReaderPool {rate: FrameRate::THIRTY,still: true,frames: None,+ size: None,}),);}@@ -342,33 +542,170 @@ impl ReaderPool {self.memory.lock().ok()?.0.get(path).cloned()}- /// Bytes of decoded frames currently cached.+ /// Makes `proxy` the file read for `original` by every request that+ /// asks for a proxy: a smaller copy, the same frames at the same+ /// instants. Cached under its own name, so the original's frames stay.+ pub fn adopt_proxy(&self, original: &Path, proxy: PathBuf) {+ if let Ok(mut proxies) = self.proxies.lock() {+ proxies.insert(original.to_path_buf(), proxy);+ }+ }++ /// The proxy adopted for `original`, if one was.+ pub fn proxy_of(&self, original: &Path) -> Option<PathBuf> {+ self.proxies.lock().ok()?.get(original).cloned()+ }++ /// Drops every cached frame cut from `path` and its probed facts: for+ /// a file that changed on disk.+ pub fn forget(&self, path: &Path) {+ if let Ok(mut sources) = self.sources.lock() {+ sources.drop_where(|key| key.path == path);+ }+ if let Ok(mut treated) = self.treated.lock() {+ treated.drop_where(|key| key.source.path == path);+ }+ if let Ok(mut facts) = self.facts.lock() {+ facts.remove(path);+ }+ }++ /// Bytes of decoded frames currently cached, source and treated.pub fn cached_bytes(&self) -> usize {- self.cache.lock().map(|cache| cache.held()).unwrap_or(0)+ let sources = self.sources.lock().map(|cache| cache.held()).unwrap_or(0);+ let treated = self.treated.lock().map(|cache| cache.held()).unwrap_or(0);+ sources + treated+ }++ /// What the pool has done so far.+ pub fn stats(&self) -> CacheStats {+ CacheStats {+ decoded: self.counters.decoded.load(Ordering::Relaxed),+ source_hits: self.counters.source_hits.load(Ordering::Relaxed),+ treated_hits: self.counters.treated_hits.load(Ordering::Relaxed),+ treated: self.counters.treated.load(Ordering::Relaxed),+ }+ }++ fn count(counter: &AtomicU64) {+ counter.fetch_add(1, Ordering::Relaxed);}- /// Whether the frame is already decoded, without decoding it.- fn cached(&self, key: &FrameKey) -> Option<Arc<Frame>> {- self.cache.lock().ok()?.get(key)+ fn cached_source(&self, key: &SourceKey) -> Option<Arc<Frame>> {+ self.sources.lock().ok()?.get(key)}- fn remember(&self, key: FrameKey, frame: Arc<Frame>) {- if let Ok(mut cache) = self.cache.lock() {+ fn remember_source(&self, key: SourceKey, frame: Arc<Frame>) {+ if let Ok(mut cache) = self.sources.lock() {cache.insert(key, frame);}}- /// 512 MB of frames, eight warm readers - enough for a busy timeline.- pub fn with_defaults() -> Self {- Self::new(512 * 1024 * 1024, 8)+ fn cached_treated(&self, key: &TreatedKey) -> Option<Arc<Frame>> {+ self.treated.lock().ok()?.get(key)+ }++ fn remember_treated(&self, key: TreatedKey, frame: Arc<Frame>) {+ if let Ok(mut cache) = self.treated.lock() {+ cache.insert(key, frame);+ }+ }++ /// The file a request reads: its proxy when it asks for one and the+ /// file has one, the file itself otherwise.+ fn path_for(&self, request: &FrameRequest) -> PathBuf {+ if request.proxy+ && let Some(proxy) = self.proxy_of(&request.path)+ {+ return proxy;+ }+ request.path.clone()+ }++ /// The source frame a request needs and where it sits: the file it+ /// reads, its facts, and the key of the frame at the level that covers+ /// the request.+ fn locate(&self, request: &FrameRequest) -> Result<(Arc<MediaFacts>, SourceKey)> {+ let path = self.path_for(request);+ let facts = self.facts_for(&path, request.still)?;+ let index = facts.index_at(request.time);+ // A still is cut to the size asked for: one decode, kept, and the+ // levels of a photograph are not worth a photograph's worth of+ // memory each.+ let (width, height) = match facts.size {+ Some(size) if !facts.still => level_for(size, request.cover),+ _ => even(request.cover.0, request.cover.1),+ };+ Ok((+ facts,+ SourceKey {+ path,+ width,+ height,+ index,+ },+ ))+ }++ /// The source frame a request needs: the file at the level that covers+ /// it, untreated. What a compositor that applies the crop, the chain+ /// and the placement itself asks for.+ pub fn source(&self, request: &FrameRequest) -> Result<Arc<Frame>> {+ let (facts, key) = self.locate(request)?;+ self.source_frame(&facts, &key)+ }++ /// The frame a request describes: the source frame at its level,+ /// cropped, fitted to the size asked for, and run through the chain.+ /// Cached twice over - the source frame under its own key, the treated+ /// picture under the source's key plus what was done to it - so an+ /// effect edit costs a filter and a scrub over covered ground costs a+ /// lookup. The same `Arc` comes back for the same request while it is+ /// cached, so a compositor that remembers what it uploaded by the+ /// frame's identity uploads it once.+ pub fn frame(&self, request: &FrameRequest) -> Result<Arc<Frame>> {+ let (facts, source_key) = self.locate(request)?;+ let (width, height) = request.size;+ let plain = request.pre.is_none()+ && request.chain.is_none()+ && (source_key.width, source_key.height) == (width, height);+ if plain {+ return self.source_frame(&facts, &source_key);+ }+ let key = TreatedKey {+ source: source_key,+ width,+ height,+ pre: request.pre.clone(),+ chain: request.chain.clone(),+ baked: false,+ };+ if let Some(frame) = self.cached_treated(&key) {+ Self::count(&self.counters.treated_hits);+ return Ok(frame);+ }+ let source = self.source_frame(&facts, &key.source)?;+ let treated = Arc::new(crate::treat::fit(+ &source,+ request.pre.as_deref(),+ width,+ height,+ request.chain.as_deref(),+ )?);+ Self::count(&self.counters.treated);+ self.remember_treated(key, Arc::clone(&treated));+ Ok(treated)}/// The frame of `path` on screen at `time` (in the media's own clock),- /// scaled to `width` x `height`.+ /// scaled to `width` x `height`, with `pre` and `chain` baked into the+ /// decoder's own graph: the crop in the file's pixels, the fit, the+ /// chain at the output size.///- /// `still` marks image files: a one-frame stream served for every time,- /// which the caller knows (from its own model) and the pool must not- /// guess from probing.+ /// Export's way of reading a frame, and the pixels a paced decoder with+ /// the same options makes. The monitor asks through+ /// [`ReaderPool::frame`] instead, whose cache the chain does not+ /// invalidate. Without a crop or a chain the two are the same request.pub fn frame_at(&self,path: &Path,@@ -379,32 +716,29 @@ impl ReaderPool {chain: Option<&str>,pre: Option<&str>,) -> Result<Arc<Frame>> {+ let chain = chain.filter(|chain| !chain.trim().is_empty());+ let pre = pre.filter(|pre| !pre.trim().is_empty());let facts = self.facts_for(path, still)?;- let rate = facts.rate;- let mut target = if facts.still {- 0- } else {- rate.frame_at(time).max(0)+ let index = facts.index_at(time);+ let source_key = SourceKey {+ path: path.to_path_buf(),+ width,+ height,+ index,};- // A clip can outlive its media - an end-trim past the file's length -- // and the honest picture for any time past the end is the last frame,- // exactly what the roll-forward case already serves. Clamp before- // seeking: `-ss` past the end decodes nothing at all.- if let Some(frames) = facts.frames {- target = target.min((frames - 1).max(0));+ if chain.is_none() && pre.is_none() {+ return self.source_frame(&facts, &source_key);}- let chain_key = chain.map(str::to_owned);- let pre_key = pre.map(str::to_owned);-- let key = FrameKey {- path: path.to_path_buf(),+ let key = TreatedKey {+ source: source_key,width,height,- chain: chain_key.clone(),- pre: pre_key.clone(),- index: target,+ pre: pre.map(str::to_owned),+ chain: chain.map(str::to_owned),+ baked: true,};- if let Some(frame) = self.cached(&key) {+ if let Some(frame) = self.cached_treated(&key) {+ Self::count(&self.counters.treated_hits);return Ok(frame);}// A still held in memory has no file to open: it is cut to the size@@ -413,46 +747,96 @@ impl ReaderPool {let spec: String = [pre, chain].into_iter().flatten()- .filter(|part| !part.trim().is_empty()).collect::<Vec<_>>().join(",");let frame = Arc::new(crate::treat::treat_to(&source, width, height, &spec)?);- self.remember(key, Arc::clone(&frame));+ Self::count(&self.counters.treated);+ self.remember_treated(key, Arc::clone(&frame));return Ok(frame);}+ let shared = self.reader(path, width, height, chain, pre, &facts, index)?;+ let mut reader = shared.lock().map_err(|_| crate::error::Error::NoFrame {+ path: path.to_path_buf(),+ })?;+ // Another caller may have moved this reader while this one waited+ // for it; the answer may even be cached by now.+ if let Some(frame) = self.cached_treated(&key) {+ Self::count(&self.counters.treated_hits);+ return Ok(frame);+ }+ let remember = |at: i64, frame: Arc<Frame>| {+ let mut key = key.clone();+ key.source.index = at;+ self.remember_treated(key, frame);+ };+ self.pull(&mut reader, &facts, index, &remember)?+ .ok_or_else(|| crate::error::Error::NoFrame {+ path: path.to_path_buf(),+ })+ }- let shared = self.reader(path, width, height, chain, pre, rate, facts.still, target)?;+ /// The source frame `key` names, from the cache or a reader.+ fn source_frame(&self, facts: &MediaFacts, key: &SourceKey) -> Result<Arc<Frame>> {+ if let Some(frame) = self.cached_source(key) {+ Self::count(&self.counters.source_hits);+ return Ok(frame);+ }+ let path = key.path.as_path();+ // A still held in memory has no file to open: it is cut to the+ // level asked for, here, and kept like any frame.+ if let Some(source) = self.held_still(path) {+ let frame = Arc::new(crate::treat::treat_to(+ &source, key.width, key.height, "",+ )?);+ self.remember_source(key.clone(), Arc::clone(&frame));+ return Ok(frame);+ }+ let shared = self.reader(path, key.width, key.height, None, None, facts, key.index)?;let mut reader = shared.lock().map_err(|_| crate::error::Error::NoFrame {path: path.to_path_buf(),})?;// Another caller may have moved this reader while this one waited// for it; the answer may even be cached by now.- if let Some(frame) = self.cached(&key) {+ if let Some(frame) = self.cached_source(key) {+ Self::count(&self.counters.source_hits);return Ok(frame);}+ let remember = |at: i64, frame: Arc<Frame>| {+ let mut key = key.clone();+ key.index = at;+ self.remember_source(key, frame);+ };+ self.pull(&mut reader, facts, key.index, &remember)?+ .ok_or_else(|| crate::error::Error::NoFrame {+ path: path.to_path_buf(),+ })+ }++ /// Brings `reader` to `target` and returns the frame there, handing+ /// every frame decoded on the way to `remember` under its own index -+ /// the next scrub over this span is then free. `None` only when the+ /// file has no decodable picture at all.+ fn pull(+ &self,+ reader: &mut Reader,+ facts: &MediaFacts,+ target: i64,+ remember: &dyn Fn(i64, Arc<Frame>),+ ) -> Result<Option<Arc<Frame>>> {+ let rate = facts.rate;let next = (reader.next_index != i64::MIN).then_some(reader.next_index);if !facts.still && plan_access(next, target) == Access::Seek {reader.seek(rate, target)?;}- // Roll forward to the target, caching everything passed on the way -- // the next scrub over this span is then free.- let mut latest: Option<Arc<Frame>> = None;+ // Roll forward to the target, caching everything passed on the way.// Stops at end of stream too: a clip trimmed past its media's end,// where the last real frame is the honest answer.+ let mut latest: Option<Arc<Frame>> = None;while let Some((index, frame)) = reader.next_frame(rate)? {+ Self::count(&self.counters.decoded);let frame = Arc::new(frame);- self.remember(- FrameKey {- path: path.to_path_buf(),- width,- height,- chain: chain_key.clone(),- pre: pre_key.clone(),- index,- },- Arc::clone(&frame),- );+ remember(index, Arc::clone(&frame));latest = Some(frame);if index >= target {break;@@ -461,28 +845,19 @@ impl ReaderPool {// Nothing at all means the seek itself landed past the end of the// picture stream - a container whose video ends before its audio, or- // a stated duration that lied past the clamp above. The last real- // frame is still the honest answer; it just has to be found by- // seeking backwards until something decodes. Each retry doubles the- // step, and the last one starts from zero, so a file with any- // decodable picture at all cannot fail here.+ // a stated duration that lied past the clamp. The last real frame is+ // still the honest answer; it just has to be found by seeking+ // backwards until something decodes. Each retry doubles the step,+ // and the last one starts from zero, so a file with any decodable+ // picture at all cannot fail here.if latest.is_none() && !facts.still {for step in [30i64, 240, i64::MAX] {let from = target.saturating_sub(step).max(0);reader.seek(rate, from)?;while let Some((index, frame)) = reader.next_frame(rate)? {+ Self::count(&self.counters.decoded);let frame = Arc::new(frame);- self.remember(- FrameKey {- path: path.to_path_buf(),- width,- height,- chain: chain_key.clone(),- pre: pre_key.clone(),- index,- },- Arc::clone(&frame),- );+ remember(index, Arc::clone(&frame));latest = Some(frame);if index >= target {break;@@ -496,13 +871,10 @@ impl ReaderPool {// Remember the answer under the index that was asked for, so// dwelling on a time past the end costs one lookup, not a// respawn-and-decode per request.- self.remember(key, Arc::clone(frame));+ remember(target, Arc::clone(frame));}}-- latest.ok_or_else(|| crate::error::Error::NoFrame {- path: path.to_path_buf(),- })+ Ok(latest)}fn facts_for(&self, path: &Path, still: bool) -> Result<Arc<MediaFacts>> {@@ -522,6 +894,7 @@ impl ReaderPool {rate: FrameRate::THIRTY,still: true,frames: None,+ size: None,}} else {let info = probe::probe(path)?;@@ -530,6 +903,11 @@ impl ReaderPool {.as_ref().map(|video| video.frame_rate).unwrap_or(FrameRate::THIRTY);+ let size = info+ .video+ .as_ref()+ .map(|video| (video.width, video.height))+ .filter(|(width, height)| *width > 0 && *height > 0);// Ceil rather than floor: undercounting by one would clamp// legitimate requests for the true last frame.let frames = info@@ -540,6 +918,7 @@ impl ReaderPool {rate,still: false,frames,+ size,}}};@@ -562,8 +941,7 @@ impl ReaderPool {height: u32,chain: Option<&str>,pre: Option<&str>,- rate: FrameRate,- still: bool,+ facts: &MediaFacts,target: i64,) -> Result<Arc<Mutex<Reader>>> {let key = (@@ -591,7 +969,14 @@ impl ReaderPool {// part, and nobody else needs to wait for it. A decode in flight on// an evicted reader finishes on its own handle.let opened = Arc::new(Mutex::new(Reader::open(- path, width, height, chain, pre, rate, still, target,+ path,+ width,+ height,+ chain,+ pre,+ facts.rate,+ facts.still,+ target,)?));let mut readers = self.readers@@ -615,8 +1000,8 @@ impl ReaderPool {Ok(opened)}- /// Drops every warm reader and cached frame - for when the media set- /// changes wholesale, like closing a project.+ /// Drops every warm reader, cached frame and adopted proxy - for when+ /// the media set changes wholesale, like closing a project.pub fn clear(&self) {if let Ok(mut readers) = self.readers.lock() {readers.warm.clear();@@ -625,15 +1010,22 @@ impl ReaderPool {if let Ok(mut facts) = self.facts.lock() {facts.clear();}- if let Ok(mut cache) = self.cache.lock() {+ if let Ok(mut cache) = self.sources.lock() {+ let budget = cache.budget;+ *cache = FrameCache::new(budget);+ }+ if let Ok(mut cache) = self.treated.lock() {let budget = cache.budget;*cache = FrameCache::new(budget);}+ if let Ok(mut proxies) = self.proxies.lock() {+ proxies.clear();+ }}}#[cfg(test)]-mod tests {+pub(crate) mod tests {use super::*;use crate::encode::{EncodeOptions, Encoder, FrameSink};@@ -662,16 +1054,31 @@ mod tests {);}+ /// The level is the source halved while it still covers the request,+ /// never smaller than the request and never larger than the source.+ #[test]+ fn a_level_is_the_source_halved_while_it_still_covers() {+ assert_eq!(level_for((1920, 1080), (960, 540)), (960, 540));+ assert_eq!(level_for((3840, 2160), (960, 540)), (960, 540));+ assert_eq!(level_for((1920, 1080), (1280, 720)), (1920, 1080));+ assert_eq!(level_for((1920, 1080), (1000, 500)), (1920, 1080));+ assert_eq!(+ level_for((640, 360), (1280, 720)),+ (640, 360),+ "never upscaled"+ );+ assert_eq!(level_for((1080, 1920), (270, 480)), (270, 480));+ assert_eq!(level_for((7, 5), (2, 2)), (2, 2), "even, and at least two");+ }+#[test]fn the_cache_evicts_least_recently_used_within_budget() {// Frames are 16 bytes each (2x2); budget holds exactly two.let mut cache = FrameCache::new(32);- let key = |index: i64| FrameKey {+ let key = |index: i64| SourceKey {path: PathBuf::from("a.mp4"),width: 2,height: 2,- chain: None,- pre: None,index,};cache.insert(key(0), Arc::new(Frame::black(2, 2)));@@ -705,7 +1112,7 @@ mod tests {assert_eq!((same.width(), same.height()), (64, 32));assert_eq!(&same.pixels()[..4], &[255, 0, 0, 255]);let half = pool- .frame_at(path, Rational::ZERO, 32, 16, true, None, None)+ .frame(&FrameRequest::new(path, Rational::ZERO, 32, 16).as_still(true)).expect("half size");assert_eq!((half.width(), half.height()), (32, 16));assert_eq!(half.pixels()[0], 255, "still red when scaled");@@ -713,7 +1120,7 @@ mod tests {second.fill([0, 0, 255, 255]);pool.hold_still(path, Arc::new(second));let again = pool- .frame_at(path, Rational::ZERO, 32, 16, true, None, None)+ .frame(&FrameRequest::new(path, Rational::ZERO, 32, 16).as_still(true)).expect("half size again");assert_eq!(again.pixels()[2],@@ -726,12 +1133,10 @@ mod tests {fn an_oversized_frame_is_served_but_never_cached() {let mut cache = FrameCache::new(8);cache.insert(- FrameKey {+ SourceKey {path: PathBuf::from("a"),width: 2,height: 2,- chain: None,- pre: None,index: 0,},Arc::new(Frame::black(2, 2)),@@ -739,23 +1144,32 @@ mod tests {assert!(cache.is_empty(), "16 bytes cannot fit an 8 byte budget");}- /// End to end against the linked FFmpeg: encode a tiny video whose frames- /// are identifiable by colour, then read arbitrary frames back out of- /// order.- #[test]- fn random_access_returns_the_right_frames() {- let path = std::env::temp_dir().join("concat-pool-test.mp4");+ /// A short video whose frames are identifiable by colour: the red+ /// channel encodes the frame index (x2 to survive compression+ /// rounding).+ pub(crate) fn counting_video(name: &str, size: u32, frames: u32) -> PathBuf {+ let path = std::env::temp_dir().join(format!(+ "concat-pool-{name}-{}.mp4",+ std::process::id()+ ));let mut encoder =- Encoder::create(&path, 64, 64, FrameRate::THIRTY, &EncodeOptions::default())+ Encoder::create(&path, size, size, FrameRate::THIRTY, &EncodeOptions::default()).expect("the linked FFmpeg encodes h264");- // 90 frames; the red channel encodes the frame index (x2 to survive- // compression rounding).- for index in 0..90u32 {- let mut frame = Frame::black(64, 64);+ for index in 0..frames {+ let mut frame = Frame::black(size, size);frame.fill([(index * 2).min(255) as u8, 40, 40, 255]);encoder.write_frame(&frame).expect("writes");}encoder.finish().expect("finishes");+ path+ }++ /// End to end against the linked FFmpeg: encode a tiny video whose frames+ /// are identifiable by colour, then read arbitrary frames back out of+ /// order.+ #[test]+ fn random_access_returns_the_right_frames() {+ let path = counting_video("random", 64, 90);let pool = ReaderPool::new(64 * 1024 * 1024, 4);let rate = FrameRate::THIRTY;@@ -796,6 +1210,129 @@ mod tests {let _ = std::fs::remove_file(&path);}+ /// The scrub benchmark. One pass over the footage decodes it; a second+ /// pass with an effect on costs no decode, only a filter; a third pass+ /// with the crop changed costs no decode either; and the fourth pass,+ /// back to the first pass's knobs, costs nothing at all. Timings are+ /// printed for a person; the counts are what is asserted.+ #[test]+ fn a_scrub_over_covered_ground_never_decodes_again() {+ // The first half of the file only: the mp4 muxer leaves the last+ // frame's duration unwritten, so the probed rate of a short+ // synthetic file runs a hair fast and the second half's indices+ // land one frame late. Real footage is thousands of frames, where+ // the hair is nothing; a benchmark wants exact counts.+ let frames: u32 = 50;+ let path = counting_video("scrub", 64, 120);+ let pool = ReaderPool::new(64 * 1024 * 1024, 4);+ let rate = FrameRate::THIRTY;+ // Forward and back again, at half size: what a monitor asks.+ let route: Vec<i64> = (0..i64::from(frames))+ .chain((0..i64::from(frames)).rev())+ .collect();++ let pass = |label: &str, pre: Option<&str>, chain: Option<&str>| -> (CacheStats, i64) {+ let before = pool.stats();+ let started = std::time::Instant::now();+ let mut sample = 0;+ for &index in &route {+ let request = FrameRequest::new(&path, rate.time_of_frame(index), 32, 32)+ .prefiltered(pre)+ .filtered(chain);+ let frame = pool.frame(&request).expect("frame decodes");+ assert_eq!((frame.width(), frame.height()), (32, 32));+ if index == 20 {+ sample = i64::from(frame.pixel(16, 16).expect("in bounds")[0]);+ }+ }+ let stats = pool.stats().since(before);+ println!(+ "{label}: {} requests in {:?}; {stats:?}",+ route.len(),+ started.elapsed(),+ );+ (stats, sample)+ };++ let (first, plain) = pass("plain", None, None);+ assert!(+ first.decoded >= u64::from(frames),+ "the first pass decodes the footage: {first:?}"+ );+ assert!((plain - 40).abs() <= 8, "frame 20 read red {plain}");+ assert_eq!(+ first.source_hits,+ u64::from(frames),+ "the way back is source hits: {first:?}"+ );++ // An effect knob turned: every frame is a source hit and a filter,+ // and the pixels are the effect's.+ let (effected, negated) = pass("negate", None, Some("negate"));+ assert_eq!(+ effected.decoded, 0,+ "an effect must not decode: {effected:?}"+ );+ assert_eq!(effected.source_hits, u64::from(frames), "{effected:?}");+ assert_eq!(effected.treated, u64::from(frames), "{effected:?}");+ assert_eq!(+ effected.treated_hits,+ u64::from(frames),+ "the way back is treated hits: {effected:?}"+ );+ assert!(negated > 180, "negated red is high, read {negated}");++ // The crop changed: the same, through the crop.+ let (cropped, _) = pass(+ "crop",+ Some("crop=w=floor(iw*0.5/2)*2:h=floor(ih*0.5/2)*2:x=0:y=0"),+ Some("negate"),+ );+ assert_eq!(cropped.decoded, 0, "a crop must not decode: {cropped:?}");+ assert_eq!(cropped.source_hits, u64::from(frames), "{cropped:?}");++ // Back to plain: the level is the size asked for, so every request+ // is the source frame itself - and the very same frames.+ let (again, plain_again) = pass("plain again", None, None);+ assert_eq!(again.decoded, 0, "{again:?}");+ assert_eq!(again.treated, 0, "{again:?}");+ assert_eq!(again.source_hits, route.len() as u64, "{again:?}");+ assert_eq!(plain_again, plain);++ let _ = std::fs::remove_file(&path);+ }++ /// A request that says so reads the adopted proxy, under its own key;+ /// a request that does not reads the original.+ #[test]+ fn a_proxy_is_read_when_asked_for_and_the_original_otherwise() {+ let original = counting_video("proxied", 64, 30);+ let proxy = counting_video("proxy", 32, 30);+ let pool = ReaderPool::new(64 * 1024 * 1024, 4);+ pool.adopt_proxy(&original, proxy.clone());+ let rate = FrameRate::THIRTY;++ let request = FrameRequest::new(&original, rate.time_of_frame(10), 16, 16);+ let from_original = pool.source(&request).expect("original decodes");+ assert_eq!(+ (from_original.width(), from_original.height()),+ (16, 16),+ "64 halved twice covers 16"+ );+ let from_proxy = pool+ .source(&request.clone().from_proxy(true))+ .expect("proxy decodes");+ assert_eq!((from_proxy.width(), from_proxy.height()), (16, 16));+ let stats = pool.stats();+ assert!(stats.decoded >= 2, "both files were read: {stats:?}");+ assert_eq!(pool.proxy_of(&original), Some(proxy.clone()));++ pool.clear();+ assert_eq!(pool.proxy_of(&original), None, "cleared with the rest");+ let _ = std::fs::remove_file(&original);+ let _ = std::fs::remove_file(&proxy);+ }+/// A JPEG still through a pool that cannot cache: every request has to/// decode afresh from the same warm reader. FFmpeg's image demuxer/// reads a single JPEG once and never again after a seek, so this@@ -812,14 +1349,9 @@ mod tests {let pool = ReaderPool::new(1, 4);for attempt in 0..3 {let got = pool- .frame_at(- &path,- FrameRate::THIRTY.time_of_frame(attempt),- 64,- 64,- true,- None,- None,+ .frame(+ &FrameRequest::new(&path, FrameRate::THIRTY.time_of_frame(attempt), 64, 64)+ .as_still(true),).unwrap_or_else(|error| panic!("request {attempt} decodes: {error}"));let red = got.pixel(32, 32).expect("in bounds")[0];diff --git a/src/crates/concat-media/src/prefetch.rs b/src/crates/concat-media/src/prefetch.rsnew file mode 100644index 00000000..7be35166--- /dev/null+++ b/src/crates/concat-media/src/prefetch.rs@@ -0,0 +1,528 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! Decoding ahead of the playhead, and the one queue everything else+//! decodes on.+//!+//! The reader pool answers the frame under the playhead; this is what+//! makes that answer a cache hit. A [`Prefetcher`] owns the pool and a few+//! worker threads, and the transport tells it where the playhead is and+//! which way it is going - a [`Cursor`] - together with the instants ahead+//! of it and the frames each needs. The workers decode those into the+//! pool, nearest first, within a small budget, and hold what they decoded+//! until the playhead has passed it; a request the playhead has already+//! passed is dropped unrun, and a cursor that moves supersedes the+//! instants queued for the last one.+//!+//! The same workers take everything else that decodes for a picture on+//! screen - filmstrip tiles, the bin's thumbnails and waveforms, a proxy+//! being written - at lower priorities, so an import of twenty files is+//! twenty jobs waiting their turn on the same few threads and never twenty+//! decoders at once (#52). One worker always stays clear of the low+//! priorities, so the artwork of a big import cannot make playback wait+//! behind it.++use std::collections::VecDeque;+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Arc, Condvar, Mutex};++use concat_core::frame::Frame;++use crate::pool::{FrameRequest, ReaderPool};++/// How many frame instants the prefetcher decodes ahead of the cursor.+/// A quarter of a second at thirty: enough that a decode never lands+/// after its frame is due, small enough that a seek wastes little.+pub const AHEAD: u32 = 8;++/// What a job is for, most urgent first. A worker takes the most urgent+/// job waiting; among jobs of one priority, the first queued.+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]+pub enum Priority {+ /// The frames the playhead is about to reach.+ Playback,+ /// A clip's filmstrip, drawn on the timeline.+ Filmstrip,+ /// The bin's thumbnails and waveforms.+ Artwork,+ /// A proxy being written: minutes of work nobody is waiting on.+ Proxy,+}++impl Priority {+ /// Every priority, most urgent first.+ pub const ALL: [Priority; 4] = [+ Priority::Playback,+ Priority::Filmstrip,+ Priority::Artwork,+ Priority::Proxy,+ ];++ /// Whether a job of this priority is background work: what the+ /// reserved worker never takes, and what runs one at a time for a+ /// proxy.+ fn background(self) -> bool {+ matches!(self, Priority::Artwork | Priority::Proxy)+ }+}++/// Which way the transport is going.+#[derive(Clone, Copy, PartialEq, Eq, Debug)]+pub enum Direction {+ /// Time increasing: play, or a scrub to the right.+ Forward,+ /// Time decreasing: shuttle, or a scrub to the left.+ Backward,+}++/// Where the playhead is and how it moves, as the transport reports it.+#[derive(Clone, Copy, PartialEq, Debug)]+pub struct Cursor {+ /// The playhead, in timeline seconds.+ pub time: f64,+ /// Which way the next frame lies.+ pub direction: Direction,+ /// Timeline seconds a wall second: one for play, two for double+ /// speed, zero for a scrub or a pause. What tells a reader of proxies+ /// whether the picture is moving.+ pub rate: f64,+}++impl Cursor {+ /// Whether `time` is behind the cursor: already passed in its+ /// direction, by more than a frame's slack.+ fn passed(self, time: f64, slack: f64) -> bool {+ match self.direction {+ Direction::Forward => time < self.time - slack,+ Direction::Backward => time > self.time + slack,+ }+ }+}++/// One instant ahead of the cursor and the frames it is made of.+#[derive(Clone, Debug)]+pub struct Moment {+ /// The instant, in timeline seconds.+ pub time: f64,+ /// Every source frame the composite at that instant needs.+ pub frames: Vec<FrameRequest>,+}++type Job = Box<dyn FnOnce() + Send>;++/// The queue the workers take from: one lane per priority.+struct Queue {+ lanes: [VecDeque<Job>; Priority::ALL.len()],+ /// Background jobs running now; see [`Priority::background`].+ background_running: usize,+ /// Proxy jobs running now: at most one.+ proxy_running: usize,+ /// Jobs of any priority running now.+ running: usize,+ closed: bool,+}++struct Lane {+ queue: Mutex<Queue>,+ ready: Condvar,+ /// Fires whenever a job finishes, for [`Prefetcher::drain`].+ done: Condvar,+ workers: usize,+}++impl Lane {+ /// The most urgent job a worker may take now, honouring the reserve:+ /// background work never takes the last free worker, and a proxy runs+ /// alone among proxies.+ fn take(&self, queue: &mut Queue) -> Option<(Priority, Job)> {+ let reserve = if self.workers > 1 { 1 } else { 0 };+ for priority in Priority::ALL {+ if priority.background() && queue.background_running + reserve >= self.workers {+ continue;+ }+ if priority == Priority::Proxy && queue.proxy_running > 0 {+ continue;+ }+ if let Some(job) = queue.lanes[priority as usize].pop_front() {+ return Some((priority, job));+ }+ }+ None+ }+}++/// What the playback jobs share with the prefetcher: the cursor as last+/// reported, and the frames decoded ahead of it, held until passed.+struct Shared {+ pool: Arc<ReaderPool>,+ cursor: Mutex<Option<Cursor>>,+ /// Bumped on every cursor: a job made for an older one is stale.+ generation: AtomicU64,+ /// Frames decoded ahead, by instant, held so the pool's eviction+ /// cannot take them before they are shown; released once passed.+ pinned: Mutex<Vec<(f64, Vec<Arc<Frame>>)>>,+ /// A frame's worth of slack, in seconds, when deciding what is passed.+ slack: Mutex<f64>,+}++/// The scheduler: owns the readers, decodes ahead of the transport, and+/// runs everything else that decodes for the screen, in priority order,+/// on a few threads. See the module docs.+pub struct Prefetcher {+ shared: Arc<Shared>,+ lane: Arc<Lane>,+}++impl Prefetcher {+ /// A prefetcher over `pool` with `workers` threads, one at least.+ pub fn new(pool: Arc<ReaderPool>, workers: usize) -> Self {+ let workers = workers.max(1);+ let lane = Arc::new(Lane {+ queue: Mutex::new(Queue {+ lanes: Default::default(),+ background_running: 0,+ proxy_running: 0,+ running: 0,+ closed: false,+ }),+ ready: Condvar::new(),+ done: Condvar::new(),+ workers,+ });+ for index in 0..workers {+ let lane = Arc::clone(&lane);+ let _ = std::thread::Builder::new()+ .name(format!("concat-decode-{index}"))+ .spawn(move || work(&lane));+ }+ Self {+ shared: Arc::new(Shared {+ pool,+ cursor: Mutex::new(None),+ generation: AtomicU64::new(0),+ pinned: Mutex::new(Vec::new()),+ slack: Mutex::new(1.0 / 30.0),+ }),+ lane,+ }+ }++ /// A prefetcher over `pool` with [`Prefetcher::default_workers`].+ pub fn with_defaults(pool: Arc<ReaderPool>) -> Self {+ Self::new(pool, Self::default_workers())+ }++ /// How many decoders run at once: a quarter of the machine's threads,+ /// two at least and four at most. A decoder is single-threaded on the+ /// walk from a keyframe, so this is roughly the share of the machine+ /// decoding may take while the window and the mix keep the rest.+ pub fn default_workers() -> usize {+ std::thread::available_parallelism().map_or(2, |threads| (threads.get() / 4).clamp(2, 4))+ }++ /// The pool the frames land in.+ pub fn pool(&self) -> &Arc<ReaderPool> {+ &self.shared.pool+ }++ /// The cursor as last reported.+ pub fn cursor(&self) -> Option<Cursor> {+ *lock(&self.shared.cursor)+ }++ /// Moves the cursor and queues `ahead`, nearest first, to be decoded+ /// into the pool. Instants queued for the last cursor are forgotten;+ /// frames held for instants now behind this cursor are released; an+ /// instant behind the cursor by the time a worker reaches it is+ /// skipped. `frame_seconds` is how long one output frame lasts, the+ /// slack in "behind".+ pub fn advance(&self, cursor: Cursor, frame_seconds: f64, ahead: Vec<Moment>) {+ let generation = self.shared.generation.fetch_add(1, Ordering::AcqRel) + 1;+ *lock(&self.shared.cursor) = Some(cursor);+ *lock(&self.shared.slack) = frame_seconds.max(0.0);+ lock(&self.shared.pinned).retain(|(time, _)| !cursor.passed(*time, frame_seconds));++ let mut queue = lock(&self.lane.queue);+ if queue.closed {+ return;+ }+ queue.lanes[Priority::Playback as usize].clear();+ for moment in ahead {+ let shared = Arc::clone(&self.shared);+ queue.lanes[Priority::Playback as usize].push_back(Box::new(move || {+ if shared.generation.load(Ordering::Acquire) != generation {+ return;+ }+ let slack = *lock(&shared.slack);+ if lock(&shared.cursor).is_some_and(|now| now.passed(moment.time, slack)) {+ return;+ }+ let frames: Vec<Arc<Frame>> = moment+ .frames+ .iter()+ .filter_map(|request| shared.pool.frame(request).ok())+ .collect();+ if shared.generation.load(Ordering::Acquire) == generation {+ lock(&shared.pinned).push((moment.time, frames));+ }+ }));+ }+ drop(queue);+ self.lane.ready.notify_all();+ }++ /// Queues `job` at `priority`; it runs on a worker when its turn comes.+ pub fn submit(&self, priority: Priority, job: impl FnOnce() + Send + 'static) {+ let mut queue = lock(&self.lane.queue);+ if queue.closed {+ return;+ }+ queue.lanes[priority as usize].push_back(Box::new(job));+ drop(queue);+ self.lane.ready.notify_all();+ }++ /// Jobs waiting at `priority`, not counting any running.+ pub fn pending(&self, priority: Priority) -> usize {+ lock(&self.lane.queue).lanes[priority as usize].len()+ }++ /// Frames decoded ahead of the cursor and not yet passed.+ pub fn held_ahead(&self) -> usize {+ lock(&self.shared.pinned).len()+ }++ /// Blocks until every queued job has run. For tests and for a caller+ /// that must know the pool is warm; the window never waits here.+ pub fn drain(&self) {+ let mut queue = lock(&self.lane.queue);+ while queue.running > 0 || queue.lanes.iter().any(|lane| !lane.is_empty()) {+ queue = self+ .lane+ .done+ .wait(queue)+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ }+ }+}++impl Drop for Prefetcher {+ fn drop(&mut self) {+ let mut queue = lock(&self.lane.queue);+ queue.closed = true;+ for lane in &mut queue.lanes {+ lane.clear();+ }+ drop(queue);+ self.lane.ready.notify_all();+ }+}++/// One worker: takes the most urgent job it may, runs it, repeats, and+/// leaves when the prefetcher is dropped.+fn work(lane: &Lane) {+ loop {+ let (priority, job) = {+ let mut queue = lock(&lane.queue);+ loop {+ if queue.closed {+ return;+ }+ if let Some(taken) = lane.take(&mut queue) {+ queue.running += 1;+ if taken.0.background() {+ queue.background_running += 1;+ }+ if taken.0 == Priority::Proxy {+ queue.proxy_running += 1;+ }+ break taken;+ }+ queue = lane+ .ready+ .wait(queue)+ .unwrap_or_else(|poisoned| poisoned.into_inner());+ }+ };+ job();+ let mut queue = lock(&lane.queue);+ queue.running -= 1;+ if priority.background() {+ queue.background_running -= 1;+ }+ if priority == Priority::Proxy {+ queue.proxy_running -= 1;+ }+ drop(queue);+ // A finished job frees a worker for a job the reserve was holding+ // back, and is what a drain waits for.+ lane.ready.notify_all();+ lane.done.notify_all();+ }+}++fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {+ mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner())+}++#[cfg(test)]+mod tests {+ use super::*;+ use concat_core::time::FrameRate;+ use std::sync::atomic::AtomicUsize;++ /// Jobs run most urgent first, and a background job never takes the+ /// last worker: with one worker and a reserve of none, everything+ /// runs; the order is by priority, then by arrival.+ #[test]+ fn jobs_run_in_priority_order() {+ let pool = Arc::new(ReaderPool::new(1024, 1));+ let prefetcher = Prefetcher::new(pool, 1);+ let order = Arc::new(Mutex::new(Vec::new()));+ // Hold the worker so the rest queue up behind, then release.+ let gate = Arc::new((Mutex::new(false), Condvar::new()));+ {+ let gate = Arc::clone(&gate);+ prefetcher.submit(Priority::Filmstrip, move || {+ let (open, bell) = &*gate;+ let mut open = open.lock().unwrap();+ while !*open {+ open = bell.wait(open).unwrap();+ }+ });+ }+ for (priority, name) in [+ (Priority::Artwork, "art-1"),+ (Priority::Proxy, "proxy"),+ (Priority::Filmstrip, "strip"),+ (Priority::Artwork, "art-2"),+ (Priority::Playback, "play"),+ ] {+ let order = Arc::clone(&order);+ prefetcher.submit(priority, move || order.lock().unwrap().push(name));+ }+ {+ let (open, bell) = &*gate;+ *open.lock().unwrap() = true;+ bell.notify_all();+ }+ prefetcher.drain();+ assert_eq!(+ *order.lock().unwrap(),+ vec!["play", "strip", "art-1", "art-2", "proxy"]+ );+ }++ /// With two workers, one is kept clear of background work: two+ /// artwork jobs never run together, and a playback job queued while+ /// one artwork job runs takes the free worker at once.+ #[test]+ fn background_work_leaves_one_worker_free() {+ let pool = Arc::new(ReaderPool::new(1024, 1));+ let prefetcher = Prefetcher::new(pool, 2);+ let running = Arc::new(AtomicUsize::new(0));+ let most = Arc::new(AtomicUsize::new(0));+ for _ in 0..6 {+ let running = Arc::clone(&running);+ let most = Arc::clone(&most);+ prefetcher.submit(Priority::Artwork, move || {+ let now = running.fetch_add(1, Ordering::SeqCst) + 1;+ most.fetch_max(now, Ordering::SeqCst);+ std::thread::sleep(std::time::Duration::from_millis(15));+ running.fetch_sub(1, Ordering::SeqCst);+ });+ }+ let played = Arc::new(AtomicUsize::new(0));+ {+ let played = Arc::clone(&played);+ prefetcher.submit(Priority::Playback, move || {+ played.store(1, Ordering::SeqCst);+ });+ }+ let started = std::time::Instant::now();+ while played.load(Ordering::SeqCst) == 0 {+ assert!(+ started.elapsed() < std::time::Duration::from_millis(500),+ "playback waited behind the artwork"+ );+ std::thread::sleep(std::time::Duration::from_millis(1));+ }+ prefetcher.drain();+ assert_eq!(most.load(Ordering::SeqCst), 1, "artwork ran one at a time");+ }++ /// The cursor's instants are decoded ahead into the pool, held until+ /// passed, and a moved cursor drops what it has passed.+ #[test]+ fn instants_ahead_are_decoded_and_released_once_passed() {+ let path = crate::pool::tests::counting_video("prefetch", 32, 90);+ let pool = Arc::new(ReaderPool::new(64 * 1024 * 1024, 2));+ let prefetcher = Prefetcher::new(Arc::clone(&pool), 1);+ let rate = FrameRate::THIRTY;+ let fps = rate.fps().as_f64();+ let moments = |from: i64| -> Vec<Moment> {+ (from + 1..from + 1 + i64::from(AHEAD))+ .map(|index| Moment {+ time: index as f64 / fps,+ frames: vec![FrameRequest::new(+ &path,+ rate.time_of_frame(index),+ 16,+ 16,+ )],+ })+ .collect()+ };+ let cursor = |index: i64| Cursor {+ time: index as f64 / fps,+ direction: Direction::Forward,+ rate: 1.0,+ };++ prefetcher.advance(cursor(0), 1.0 / fps, moments(0));+ prefetcher.drain();+ assert_eq!(prefetcher.held_ahead(), AHEAD as usize);+ let warmed = pool.stats();+ assert!(warmed.decoded >= u64::from(AHEAD), "{warmed:?}");++ // The pull for frame 5 is a hit.+ pool.frame(&FrameRequest::new(&path, rate.time_of_frame(5), 16, 16))+ .expect("decodes");+ assert_eq!(pool.stats().since(warmed).decoded, 0, "frame 5 was ahead");++ // The cursor at 5: the instants more than a frame behind it are+ // released, 4..=8 stay, and 6..=13 join them.+ prefetcher.advance(cursor(5), 1.0 / fps, moments(5));+ prefetcher.drain();+ let held = prefetcher.held_ahead();+ assert!(+ (AHEAD as usize + 4..=AHEAD as usize + 6).contains(&held),+ "held {held}: the passed instants were released"+ );++ // A cursor far ahead: nothing held behind it survives, and an+ // instant queued for it that it has already passed is skipped -+ // asked for at a size nothing else uses, it is a fresh decode+ // afterwards, so it cannot have been prefetched.+ let mut stale = moments(20);+ stale.insert(+ 0,+ Moment {+ time: 2.0 / fps,+ frames: vec![FrameRequest::new(&path, rate.time_of_frame(2), 8, 8)],+ },+ );+ prefetcher.advance(cursor(20), 1.0 / fps, stale);+ prefetcher.drain();+ assert_eq!(prefetcher.held_ahead(), AHEAD as usize);+ let before = pool.stats();+ pool.frame(&FrameRequest::new(&path, rate.time_of_frame(2), 8, 8))+ .expect("decodes");+ assert!(+ pool.stats().since(before).decoded > 0,+ "the passed instant was never decoded"+ );++ let _ = std::fs::remove_file(&path);+ }+}diff --git a/src/crates/concat-media/src/treat.rs b/src/crates/concat-media/src/treat.rsindex 348e70b2..689befaa 100644--- a/src/crates/concat-media/src/treat.rs+++ b/src/crates/concat-media/src/treat.rs@@ -36,6 +36,47 @@ pub fn treat_to(frame: &Frame, width: u32, height: u32, chain: &str) -> Result<Fif chain.trim().is_empty() && width == frame.width() && height == frame.height() {return Ok(frame.clone());}+ let spec = if chain.trim().is_empty() {+ format!("scale={width}:{height}:flags=bilinear,format=rgba")+ } else {+ format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba")+ };+ run(frame, &spec)+}++/// Runs a source frame the way the decoder would have: `pre` in the+/// picture's own pixels - the crop - then the fit to `width` by `height`,+/// then `chain` at that size, then the guard scale that pins the size. The+/// decoder's graph without the decoder, for a picture the reader pool+/// holds untreated. Nothing to do at the same size returns a copy.+pub fn fit(+ frame: &Frame,+ pre: Option<&str>,+ width: u32,+ height: u32,+ chain: Option<&str>,+) -> Result<Frame> {+ let pre = pre.filter(|pre| !pre.trim().is_empty());+ let chain = chain.filter(|chain| !chain.trim().is_empty());+ if pre.is_none() && chain.is_none() && width == frame.width() && height == frame.height() {+ return Ok(frame.clone());+ }+ let mut parts: Vec<String> = Vec::new();+ if let Some(pre) = pre {+ parts.push(pre.to_owned());+ }+ parts.push(format!("scale={width}:{height}:flags=bilinear"));+ if let Some(chain) = chain {+ parts.push(chain.to_owned());+ parts.push(format!("scale={width}:{height}:flags=bilinear"));+ }+ parts.push("format=rgba".to_owned());+ run(frame, &parts.join(","))+}++/// One RGBA picture in, `spec` - a whole filtergraph, ending in the size+/// and format the caller wants out - and one RGBA picture out.+fn run(frame: &Frame, spec: &str) -> Result<Frame> {ffi::init();// The graph has no file behind it; errors name the layer instead.let path = Path::new("layer");@@ -64,15 +105,10 @@ pub fn treat_to(frame: &Frame, width: u32, height: u32, chain: &str) -> Result<F"",).map_err(|error| ffi::fail("buffer sink", path, error))?;- let spec = if chain.trim().is_empty() {- format!("scale={width}:{height}:flags=bilinear,format=rgba")- } else {- format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba")- };graph.output("in", 0).and_then(|parser| parser.input("out", 0))- .and_then(|parser| parser.parse(&spec))+ .and_then(|parser| parser.parse(spec)).map_err(|error| ffi::fail("filter graph", path, error))?;graph.validate()@@ -146,4 +182,29 @@ mod tests {let pinned = treat(&frame, "scale=3:3").expect("scales then pins");assert_eq!((pinned.width(), pinned.height()), (6, 6));}++ /// A fit crops in the source's pixels, scales, and runs the chain at+ /// the output size: the left half of a picture red on the left and+ /// blue on the right, fitted to four across, is all red - and negated,+ /// all cyan.+ #[test]+ fn a_fit_crops_then_scales_then_treats() {+ let mut frame = Frame::black(8, 4);+ for (index, pixel) in frame.pixels_mut().chunks_exact_mut(4).enumerate() {+ let x = index % 8;+ pixel.copy_from_slice(if x < 4 {+ &[255, 0, 0, 255]+ } else {+ &[0, 0, 255, 255]+ });+ }+ let crop = "crop=w=iw/2:h=ih:x=0:y=0";+ let left = fit(&frame, Some(crop), 4, 4, None).expect("fits");+ assert_eq!((left.width(), left.height()), (4, 4));+ assert_eq!(left.pixel(3, 1), Some([255, 0, 0, 255]), "the left half only");+ let negated = fit(&frame, Some(crop), 4, 4, Some("negate")).expect("fits and treats");+ assert_eq!(negated.pixel(3, 1), Some([0, 255, 255, 255]));+ let same = fit(&frame, None, 8, 4, None).expect("copies");+ assert_eq!(same.pixels(), frame.pixels());+ }}