Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(media): the frame cache keys on the source frame, and a scheduler decodes ahead

Jareer7311d2dae98b
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index b1b58235..1d89233c 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -975,6 +975,7 @@ fn render_picture(
codec: request.codec,
ten_bit: request.ten_bit,
hardware: true,
+ threads: 0,
},
)
.map_err(|error| error.to_string())?;
diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rs
index 4f33157e..eb1fb11f 100644
--- a/src/crates/concat-host/tests/export.rs
+++ b/src/crates/concat-host/tests/export.rs
@@ -91,6 +91,7 @@ fn picture(path: &Path, rate: FrameRate, seconds: u32) {
crf: 16,
ten_bit: false,
hardware: false,
+ threads: 0,
};
let mut encoder =
Encoder::create(path, WIDTH, HEIGHT, rate, &options).expect("the linked FFmpeg encodes");
diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rs
index a6f914e4..c232c85c 100644
--- a/src/crates/concat-media/src/decode.rs
+++ b/src/crates/concat-media/src/decode.rs
@@ -101,6 +101,12 @@ pub struct DecodeOptions {
/// decoded frame. Never for anything a person will watch or export: the
/// frame is near the instant, not at it.
pub keyframes_only: bool,
+ /// Threads the codec may use, or zero to let it count the cores.
+ ///
+ /// Zero for anything a person is waiting on: a scrub wants its frame
+ /// on every core. A few for a proxy being written in the background,
+ /// where the whole point is that the machine stays usable meanwhile.
+ pub threads: u16,
}
impl DecodeOptions {
@@ -153,6 +159,12 @@ impl DecodeOptions {
self.keyframes_only = true;
self
}
+
+ /// Limits the codec to `threads`. See [`DecodeOptions::threads`].
+ pub fn threaded(mut self, threads: u16) -> Self {
+ self.threads = threads;
+ self
+ }
}
/// How a stream says its colour is encoded: the tags in its container,
@@ -330,13 +342,14 @@ impl Decoder {
// a seek decodes every frame from the keyframe before it: on one
// thread a 4K H.264 file decoded at about 87 frames a second, and a
// scrub waited 400 ms for its frame. Frame threads for that walk,
- // slice threads for files cut into slices; zero lets the codec count.
+ // slice threads for files cut into slices; zero lets the codec
+ // count, and a caller working in the background says how many.
// SAFETY: the context is not opened yet, which is when threading is
// set, and both fields are plain integers.
unsafe {
let raw = context.as_mut_ptr();
(*raw).thread_type = ffmpeg::sys::FF_THREAD_FRAME | ffmpeg::sys::FF_THREAD_SLICE;
- (*raw).thread_count = 0;
+ (*raw).thread_count = i32::from(options.threads);
}
let mut decoder = context
.decoder()
@@ -764,6 +777,7 @@ mod tests {
crf: 20,
ten_bit: true,
hardware: false,
+ threads: 0,
};
let mut encoder = Encoder::create_tagged(
&path,
diff --git a/src/crates/concat-media/src/encode.rs b/src/crates/concat-media/src/encode.rs
index 8aab0818..1717b1a0 100644
--- a/src/crates/concat-media/src/encode.rs
+++ b/src/crates/concat-media/src/encode.rs
@@ -135,6 +135,10 @@ pub struct EncodeOptions {
/// Let the platform's hardware encoder lead where there is one; see
/// [`VideoCodec::encoders`].
pub hardware: bool,
+ /// Threads the encoder may use, or zero to let it count the cores.
+ /// Zero for an export; a few for a proxy written while the editor is
+ /// in use.
+ pub threads: u16,
}
impl Default for EncodeOptions {
@@ -145,6 +149,7 @@ impl Default for EncodeOptions {
crf: 18,
ten_bit: false,
hardware: true,
+ threads: 0,
}
}
}
@@ -295,12 +300,15 @@ impl Encoder {
video.set_colorspace(matrix);
video.set_color_range(ffmpeg::color::Range::MPEG);
// SAFETY: `video` owns a live AVCodecContext; primaries and
- // transfer have no setter in the bindings, and both are plain
+ // transfer have no setter in the bindings, and all three are plain
// fields the encoder reads at open.
unsafe {
let context = video.as_mut_ptr();
(*context).color_primaries = primaries.into();
(*context).color_trc = transfer.into();
+ if options.threads > 0 {
+ (*context).thread_count = i32::from(options.threads);
+ }
}
if global_header {
video.set_flags(ffmpeg::codec::Flags::GLOBAL_HEADER);
@@ -645,6 +653,7 @@ mod tests {
crf: 24,
ten_bit,
hardware: true,
+ threads: 0,
};
let mut encoder = Encoder::create(&path, 64, 64, FrameRate::THIRTY, &options)
.unwrap_or_else(|error| panic!("{} {ten_bit}: {error}", codec.label()));
diff --git a/src/crates/concat-media/src/lib.rs b/src/crates/concat-media/src/lib.rs
index f09e8cfb..45862202 100644
--- a/src/crates/concat-media/src/lib.rs
+++ b/src/crates/concat-media/src/lib.rs
@@ -20,6 +20,7 @@ pub mod error;
mod ffi;
pub mod peaks;
pub mod pool;
+pub mod prefetch;
pub mod probe;
pub mod samples;
pub mod treat;
@@ -29,7 +30,8 @@ pub use encode::{EncodeOptions, Encoder, FrameSink, VideoCodec, jpeg};
pub use error::{Error, Result};
pub use ffi::{init, linked_version};
pub use peaks::Peaks;
-pub use pool::{FrameCache, ReaderPool};
+pub use pool::{CacheStats, FrameCache, FrameRequest, ReaderPool};
+pub use prefetch::{Cursor, Direction, Moment, Prefetcher, Priority};
pub use probe::{AudioStream, MediaInfo, VideoStream, probe};
pub use samples::{AudioDecoder, AudioOptions, SampleFormat};
-pub use treat::{treat, treat_to};
+pub use treat::{fit, treat, treat_to};
diff --git a/src/crates/concat-media/src/pool.rs b/src/crates/concat-media/src/pool.rs
index ca6bb277..92281145 100644
--- a/src/crates/concat-media/src/pool.rs
+++ b/src/crates/concat-media/src/pool.rs
@@ -8,21 +8,38 @@
//! interactive use, where scrubbing asks for arbitrary (media, time) pairs.
//! This module is the answer:
//!
-//! - **One reader per (file, size, chain)**, kept warm between requests. A
+//! - **One reader per (file, level)**, kept warm between requests. A
//! request near the reader's current position rolls forward (cheap,
//! exact); a request elsewhere seeks - frame-accurately, guided by the
//! real timestamps the linked decoder reports.
-//! - **A byte-budgeted LRU frame cache** in front of the readers. Scrubbing
-//! back over ground just covered, or dwelling on one frame, costs a hash
-//! lookup. Frames roll into the cache as decoding passes them, so rolling
-//! forward to frame N caches N-1 frames of the path there.
+//! - **A byte-budgeted LRU of source frames** in front of the readers,
+//! keyed by the file, the level it was decoded at and the frame index -
+//! and nothing else. A source frame is the picture as the file holds
+//! it, turned the right way up, brought into BT.709 and scaled to a
+//! level: the file's own size halved as many times as still covers what
+//! was asked for. The crop and the effect chain are not in the key. They
+//! are applied to the cached picture on the way out, and their results
+//! kept in a second, smaller cache, so turning an effect knob costs a
+//! filter per frame and not a decode per frame, and scrubbing back over
+//! ground already covered costs a hash lookup whatever the knobs say.
+//! - **Proxies.** A file can be given a stand-in - a smaller copy of
+//! itself - and a request that says so reads that instead, under its own
+//! key. Playback reads the proxy; the paused monitor reads the original.
//!
-//! The pool is deliberately not used by export: export decodes every frame
-//! exactly once in order, and a cache in that path is pure overhead. This is
-//! playback infrastructure.
+//! Frames roll into the cache as decoding passes them, so rolling forward
+//! to frame N caches N-1 frames of the path there.
+//!
+//! Export decodes every frame exactly once in order and does not come
+//! here, with one exception: a clip that runs backwards or on a speed
+//! curve cannot be followed by a paced decoder, and each of its frames is
+//! sought through [`ReaderPool::frame_at`], which bakes the chain into the
+//! decoder the way the paced decoders beside it do. The two paths of one
+//! export then agree to the pixel.
use std::collections::HashMap;
+use std::hash::Hash;
use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use concat_core::frame::Frame;
@@ -37,34 +54,124 @@ use crate::probe;
/// sixty decodes - comparable to a seek, and exact.
const ROLL_FORWARD_FRAMES: i64 = 60;
-/// What one pooled frame request asks for.
+/// What one frame request asks for; see [`ReaderPool::frame`].
#[derive(Clone, PartialEq, Eq, Hash, Debug)]
-struct FrameKey {
+pub struct FrameRequest {
+ /// The media file.
+ pub path: PathBuf,
+ /// The instant, in the media's own clock.
+ pub time: Rational,
+ /// The size the whole picture would have at the scale asked for, crop
+ /// or no crop: what picks the level the source is decoded at. A clip
+ /// cropped to its middle half and fitted to 960 across needs the whole
+ /// picture at 1920, and this is where a caller says so.
+ pub cover: (u32, u32),
+ /// The size the frame comes out at.
+ pub size: (u32, u32),
+ /// An image file: one frame, served for every time, which the caller
+ /// knows from its own model and the pool must not guess from probing.
+ pub still: bool,
+ /// The chain run in the picture's own pixels before the fit: the crop.
+ pub pre: Option<String>,
+ /// The effect chain, run at the output size after the fit.
+ pub chain: Option<String>,
+ /// Read the file's proxy where it has one; see
+ /// [`ReaderPool::adopt_proxy`].
+ pub proxy: bool,
+}
+
+impl FrameRequest {
+ /// A plain request: the frame of `path` at `time`, `width` by `height`,
+ /// no crop, no chain, the original file.
+ pub fn new(path: impl Into<PathBuf>, time: Rational, width: u32, height: u32) -> Self {
+ Self {
+ path: path.into(),
+ time,
+ cover: (width, height),
+ size: (width, height),
+ still: false,
+ pre: None,
+ chain: None,
+ proxy: false,
+ }
+ }
+
+ /// Marks the file an image. See [`FrameRequest::still`].
+ pub fn as_still(mut self, still: bool) -> Self {
+ self.still = still;
+ self
+ }
+
+ /// Sets the size the whole picture has at this scale. See
+ /// [`FrameRequest::cover`].
+ pub fn covering(mut self, width: u32, height: u32) -> Self {
+ self.cover = (width, height);
+ self
+ }
+
+ /// Applies `pre` before the fit; empty for none.
+ pub fn prefiltered(mut self, pre: Option<&str>) -> Self {
+ self.pre = pre
+ .filter(|chain| !chain.trim().is_empty())
+ .map(str::to_owned);
+ self
+ }
+
+ /// Applies `chain` after the fit; empty for none.
+ pub fn filtered(mut self, chain: Option<&str>) -> Self {
+ self.chain = chain
+ .filter(|chain| !chain.trim().is_empty())
+ .map(str::to_owned);
+ self
+ }
+
+ /// Reads the proxy where there is one. See [`FrameRequest::proxy`].
+ pub fn from_proxy(mut self, proxy: bool) -> Self {
+ self.proxy = proxy;
+ self
+ }
+}
+
+/// A source frame's identity: the file, the level it was decoded at, the
+/// frame. The stream is implicit - a file's best video stream is the one
+/// every reader opens.
+#[derive(Clone, PartialEq, Eq, Hash, Debug)]
+struct SourceKey {
path: PathBuf,
width: u32,
height: u32,
- /// The clip's effect chain, part of the frame's identity: the same source
- /// frame through two different chains is two different pictures.
- chain: Option<String>,
- /// The pre-fit chain - a crop - on the same terms.
- pre: Option<String>,
index: i64,
}
+/// A treated frame's identity: the source frame and what was done to it.
+#[derive(Clone, PartialEq, Eq, Hash, Debug)]
+struct TreatedKey {
+ source: SourceKey,
+ width: u32,
+ height: u32,
+ pre: Option<String>,
+ chain: Option<String>,
+ /// Made by a decoder with the chain in its graph rather than from a
+ /// cached source frame - export's way, see [`ReaderPool::frame_at`].
+ /// The crop then ran in the file's own pixels and not the level's, so
+ /// the pictures can differ by a rounding and must not share a key.
+ baked: bool,
+}
+
/// A byte-budgeted LRU of decoded frames.
///
/// Plain and measurable on purpose: a `HashMap` plus a logical clock, evicting
/// the least-recently-touched frame until the budget holds. At preview sizes a
/// frame is ~2-4 MB, so the default budget holds a few hundred frames - many
/// seconds of scrub history.
-pub struct FrameCache {
+pub struct FrameCache<K> {
budget: usize,
held: usize,
tick: u64,
- frames: HashMap<FrameKey, (Arc<Frame>, u64)>,
+ frames: HashMap<K, (Arc<Frame>, u64)>,
}
-impl FrameCache {
+impl<K: Hash + Eq + Clone> FrameCache<K> {
/// An empty cache holding at most `budget` bytes of frames.
pub fn new(budget: usize) -> Self {
Self {
@@ -75,7 +182,7 @@ impl FrameCache {
}
}
- fn get(&mut self, key: &FrameKey) -> Option<Arc<Frame>> {
+ fn get(&mut self, key: &K) -> Option<Arc<Frame>> {
self.tick += 1;
let tick = self.tick;
self.frames.get_mut(key).map(|(frame, touched)| {
@@ -84,7 +191,7 @@ impl FrameCache {
})
}
- fn insert(&mut self, key: FrameKey, frame: Arc<Frame>) {
+ fn insert(&mut self, key: K, frame: Arc<Frame>) {
let bytes = frame.pixels().len();
// A frame larger than the whole budget would evict everything and
// still not fit; hold it once without caching it.
@@ -111,13 +218,12 @@ impl FrameCache {
}
}
- /// Bytes currently held.
- /// Drops every frame cut from `path`.
- fn forget(&mut self, path: &Path) {
- let gone: Vec<FrameKey> = self
+ /// Drops every frame `doomed` says to.
+ fn drop_where(&mut self, doomed: impl Fn(&K) -> bool) {
+ let gone: Vec<K> = self
.frames
.keys()
- .filter(|key| key.path == path)
+ .filter(|key| doomed(key))
.cloned()
.collect();
for key in gone {
@@ -165,6 +271,28 @@ fn plan_access(current_next: Option<i64>, target: i64) -> Access {
}
}
+/// An even size no smaller than two on a side: what the scaler and the
+/// codecs are happiest with.
+fn even(width: u32, height: u32) -> (u32, u32) {
+ ((width & !1).max(2), (height & !1).max(2))
+}
+
+/// The level a source of `source` is decoded at to cover `cover`: the
+/// source halved as many times as still covers it on both sides. A source
+/// no bigger than the cover is decoded at its own size; nothing is ever
+/// decoded larger than the file.
+fn level_for(source: (u32, u32), cover: (u32, u32)) -> (u32, u32) {
+ let (mut width, mut height) = even(source.0, source.1);
+ loop {
+ let (half_w, half_h) = even(width / 2, height / 2);
+ if half_w < cover.0 || half_h < cover.1 || (half_w, half_h) == (width, height) {
+ break;
+ }
+ (width, height) = (half_w, half_h);
+ }
+ (width, height)
+}
+
/// One warm reader: a decoder plus where its *next* frame will land.
struct Reader {
decoder: Decoder,
@@ -186,7 +314,7 @@ impl Reader {
) -> Result<Self> {
// Unpaced, so every source frame comes out with its own timestamp
// and the index is read from that; a seek lands on the keyframe at
- // or before the target and `frame_at` rolls forward from there.
+ // or before the target and the pull rolls forward from there.
let mut options = DecodeOptions::default()
.starting_at(rate.time_of_frame(index))
.scaled_to(width, height);
@@ -242,9 +370,29 @@ struct MediaFacts {
/// a seek past the end produces zero frames, not an error and not a
/// picture.
frames: Option<i64>,
+ /// The picture's displayed size, for a file with one: what the levels
+ /// are halvings of.
+ size: Option<(u32, u32)>,
}
-/// One reader's identity: the file, the decode size, the effect chain.
+impl MediaFacts {
+ /// The frame on screen at `time`: index zero for a still, and never
+ /// past the last frame the container claims. A clip can outlive its
+ /// media - an end-trim past the file's length - and the honest picture
+ /// for any time past the end is the last frame.
+ fn index_at(&self, time: Rational) -> i64 {
+ if self.still {
+ return 0;
+ }
+ let mut target = self.rate.frame_at(time).max(0);
+ if let Some(frames) = self.frames {
+ target = target.min((frames - 1).max(0));
+ }
+ target
+ }
+}
+
+/// One reader's identity: the file, the decode size, the baked chains.
type ReaderKey = (PathBuf, u32, u32, Option<String>, Option<String>);
/// The warm readers and their recency, behind one short lock. A reader is
@@ -258,17 +406,54 @@ struct Readers {
max: usize,
}
+/// What the pool has done so far: the numbers a scrub benchmark asserts.
+#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
+pub struct CacheStats {
+ /// Frames that came out of a decoder.
+ pub decoded: u64,
+ /// Requests answered from a cached source frame without decoding,
+ /// treated on the way out or not.
+ pub source_hits: u64,
+ /// Requests answered from a cached treated frame: no decode, no filter.
+ pub treated_hits: u64,
+ /// Source frames put through a crop, a chain or a fit on the way out.
+ pub treated: u64,
+}
+
+impl CacheStats {
+ /// What happened between `earlier` and this snapshot.
+ pub fn since(self, earlier: CacheStats) -> CacheStats {
+ CacheStats {
+ decoded: self.decoded - earlier.decoded,
+ source_hits: self.source_hits - earlier.source_hits,
+ treated_hits: self.treated_hits - earlier.treated_hits,
+ treated: self.treated - earlier.treated,
+ }
+ }
+}
+
+#[derive(Default)]
+struct Counters {
+ decoded: AtomicU64,
+ source_hits: AtomicU64,
+ treated_hits: AtomicU64,
+ treated: AtomicU64,
+}
+
/// Random access to frames across many files, cache in front, warm readers
/// behind.
///
/// Shared, not serialised: every method takes `&self`, and the locks inside
-/// are held for as little as each step needs. The cache is one short lock;
+/// are held for as little as each step needs. The caches are short locks;
/// each reader is its own, so the playback stream decoding ahead on one
/// file and the monitor pulling a cached frame of another never queue
/// behind each other. Two callers wanting the *same* reader take turns,
/// which is what a single decoder demands anyway.
pub struct ReaderPool {
- cache: Mutex<FrameCache>,
+ /// Source frames: the file at a level, untreated.
+ sources: Mutex<FrameCache<SourceKey>>,
+ /// Source frames after their crop, fit and chain, at the output size.
+ treated: Mutex<FrameCache<TreatedKey>>,
readers: Mutex<Readers>,
facts: Mutex<HashMap<PathBuf, Arc<MediaFacts>>>,
/// Stills that exist nowhere but here: a title painted while its words
@@ -277,6 +462,10 @@ pub struct ReaderPool {
/// chain a plan asks - see [`ReaderPool::hold_still`]. The newest
/// [`STILLS_HELD`] stay.
memory: Mutex<HeldStills>,
+ /// The stand-in for each file that has one; see
+ /// [`ReaderPool::adopt_proxy`].
+ proxies: Mutex<HashMap<PathBuf, PathBuf>>,
+ counters: Counters,
}
/// The stills held in memory, and the order they came in.
@@ -292,10 +481,15 @@ const STILLS_HELD: usize = 24;
impl ReaderPool {
/// A pool with `cache_bytes` of frame cache and up to `max_readers` warm
- /// decoders.
+ /// decoders. A quarter of the budget goes to treated frames, which are
+ /// output-sized and few - the frames under and just ahead of the
+ /// playhead - and the rest to source frames, which are the scrub
+ /// history.
pub fn new(cache_bytes: usize, max_readers: usize) -> Self {
+ let treated = cache_bytes / 4;
Self {
- cache: Mutex::new(FrameCache::new(cache_bytes)),
+ sources: Mutex::new(FrameCache::new(cache_bytes - treated)),
+ treated: Mutex::new(FrameCache::new(treated)),
readers: Mutex::new(Readers {
warm: HashMap::new(),
order: Vec::new(),
@@ -303,9 +497,16 @@ impl ReaderPool {
}),
facts: Mutex::new(HashMap::new()),
memory: Mutex::new((HashMap::new(), std::collections::VecDeque::new())),
+ proxies: Mutex::new(HashMap::new()),
+ counters: Counters::default(),
}
}
+ /// 512 MB of frames, eight warm readers - enough for a busy timeline.
+ pub fn with_defaults() -> Self {
+ Self::new(512 * 1024 * 1024, 8)
+ }
+
/// Makes `frame` the still at `path`, a name no file has, until
/// [`STILLS_HELD`] newer ones have come. Frames already cut from an
/// earlier still of that name are let go, so the name never shows
@@ -322,9 +523,7 @@ impl ReaderPool {
}
}
}
- if let Ok(mut cache) = self.cache.lock() {
- cache.forget(path);
- }
+ self.forget(path);
if let Ok(mut facts) = self.facts.lock() {
facts.insert(
path.to_path_buf(),
@@ -332,6 +531,7 @@ impl ReaderPool {
rate: FrameRate::THIRTY,
still: true,
frames: None,
+ size: None,
}),
);
}
@@ -342,33 +542,170 @@ impl ReaderPool {
self.memory.lock().ok()?.0.get(path).cloned()
}
- /// Bytes of decoded frames currently cached.
+ /// Makes `proxy` the file read for `original` by every request that
+ /// asks for a proxy: a smaller copy, the same frames at the same
+ /// instants. Cached under its own name, so the original's frames stay.
+ pub fn adopt_proxy(&self, original: &Path, proxy: PathBuf) {
+ if let Ok(mut proxies) = self.proxies.lock() {
+ proxies.insert(original.to_path_buf(), proxy);
+ }
+ }
+
+ /// The proxy adopted for `original`, if one was.
+ pub fn proxy_of(&self, original: &Path) -> Option<PathBuf> {
+ self.proxies.lock().ok()?.get(original).cloned()
+ }
+
+ /// Drops every cached frame cut from `path` and its probed facts: for
+ /// a file that changed on disk.
+ pub fn forget(&self, path: &Path) {
+ if let Ok(mut sources) = self.sources.lock() {
+ sources.drop_where(|key| key.path == path);
+ }
+ if let Ok(mut treated) = self.treated.lock() {
+ treated.drop_where(|key| key.source.path == path);
+ }
+ if let Ok(mut facts) = self.facts.lock() {
+ facts.remove(path);
+ }
+ }
+
+ /// Bytes of decoded frames currently cached, source and treated.
pub fn cached_bytes(&self) -> usize {
- self.cache.lock().map(|cache| cache.held()).unwrap_or(0)
+ let sources = self.sources.lock().map(|cache| cache.held()).unwrap_or(0);
+ let treated = self.treated.lock().map(|cache| cache.held()).unwrap_or(0);
+ sources + treated
+ }
+
+ /// What the pool has done so far.
+ pub fn stats(&self) -> CacheStats {
+ CacheStats {
+ decoded: self.counters.decoded.load(Ordering::Relaxed),
+ source_hits: self.counters.source_hits.load(Ordering::Relaxed),
+ treated_hits: self.counters.treated_hits.load(Ordering::Relaxed),
+ treated: self.counters.treated.load(Ordering::Relaxed),
+ }
+ }
+
+ fn count(counter: &AtomicU64) {
+ counter.fetch_add(1, Ordering::Relaxed);
}
- /// Whether the frame is already decoded, without decoding it.
- fn cached(&self, key: &FrameKey) -> Option<Arc<Frame>> {
- self.cache.lock().ok()?.get(key)
+ fn cached_source(&self, key: &SourceKey) -> Option<Arc<Frame>> {
+ self.sources.lock().ok()?.get(key)
}
- fn remember(&self, key: FrameKey, frame: Arc<Frame>) {
- if let Ok(mut cache) = self.cache.lock() {
+ fn remember_source(&self, key: SourceKey, frame: Arc<Frame>) {
+ if let Ok(mut cache) = self.sources.lock() {
cache.insert(key, frame);
}
}
- /// 512 MB of frames, eight warm readers - enough for a busy timeline.
- pub fn with_defaults() -> Self {
- Self::new(512 * 1024 * 1024, 8)
+ fn cached_treated(&self, key: &TreatedKey) -> Option<Arc<Frame>> {
+ self.treated.lock().ok()?.get(key)
+ }
+
+ fn remember_treated(&self, key: TreatedKey, frame: Arc<Frame>) {
+ if let Ok(mut cache) = self.treated.lock() {
+ cache.insert(key, frame);
+ }
+ }
+
+ /// The file a request reads: its proxy when it asks for one and the
+ /// file has one, the file itself otherwise.
+ fn path_for(&self, request: &FrameRequest) -> PathBuf {
+ if request.proxy
+ && let Some(proxy) = self.proxy_of(&request.path)
+ {
+ return proxy;
+ }
+ request.path.clone()
+ }
+
+ /// The source frame a request needs and where it sits: the file it
+ /// reads, its facts, and the key of the frame at the level that covers
+ /// the request.
+ fn locate(&self, request: &FrameRequest) -> Result<(Arc<MediaFacts>, SourceKey)> {
+ let path = self.path_for(request);
+ let facts = self.facts_for(&path, request.still)?;
+ let index = facts.index_at(request.time);
+ // A still is cut to the size asked for: one decode, kept, and the
+ // levels of a photograph are not worth a photograph's worth of
+ // memory each.
+ let (width, height) = match facts.size {
+ Some(size) if !facts.still => level_for(size, request.cover),
+ _ => even(request.cover.0, request.cover.1),
+ };
+ Ok((
+ facts,
+ SourceKey {
+ path,
+ width,
+ height,
+ index,
+ },
+ ))
+ }
+
+ /// The source frame a request needs: the file at the level that covers
+ /// it, untreated. What a compositor that applies the crop, the chain
+ /// and the placement itself asks for.
+ pub fn source(&self, request: &FrameRequest) -> Result<Arc<Frame>> {
+ let (facts, key) = self.locate(request)?;
+ self.source_frame(&facts, &key)
+ }
+
+ /// The frame a request describes: the source frame at its level,
+ /// cropped, fitted to the size asked for, and run through the chain.
+ /// Cached twice over - the source frame under its own key, the treated
+ /// picture under the source's key plus what was done to it - so an
+ /// effect edit costs a filter and a scrub over covered ground costs a
+ /// lookup. The same `Arc` comes back for the same request while it is
+ /// cached, so a compositor that remembers what it uploaded by the
+ /// frame's identity uploads it once.
+ pub fn frame(&self, request: &FrameRequest) -> Result<Arc<Frame>> {
+ let (facts, source_key) = self.locate(request)?;
+ let (width, height) = request.size;
+ let plain = request.pre.is_none()
+ && request.chain.is_none()
+ && (source_key.width, source_key.height) == (width, height);
+ if plain {
+ return self.source_frame(&facts, &source_key);
+ }
+ let key = TreatedKey {
+ source: source_key,
+ width,
+ height,
+ pre: request.pre.clone(),
+ chain: request.chain.clone(),
+ baked: false,
+ };
+ if let Some(frame) = self.cached_treated(&key) {
+ Self::count(&self.counters.treated_hits);
+ return Ok(frame);
+ }
+ let source = self.source_frame(&facts, &key.source)?;
+ let treated = Arc::new(crate::treat::fit(
+ &source,
+ request.pre.as_deref(),
+ width,
+ height,
+ request.chain.as_deref(),
+ )?);
+ Self::count(&self.counters.treated);
+ self.remember_treated(key, Arc::clone(&treated));
+ Ok(treated)
}
/// The frame of `path` on screen at `time` (in the media's own clock),
- /// scaled to `width` x `height`.
+ /// scaled to `width` x `height`, with `pre` and `chain` baked into the
+ /// decoder's own graph: the crop in the file's pixels, the fit, the
+ /// chain at the output size.
///
- /// `still` marks image files: a one-frame stream served for every time,
- /// which the caller knows (from its own model) and the pool must not
- /// guess from probing.
+ /// Export's way of reading a frame, and the pixels a paced decoder with
+ /// the same options makes. The monitor asks through
+ /// [`ReaderPool::frame`] instead, whose cache the chain does not
+ /// invalidate. Without a crop or a chain the two are the same request.
pub fn frame_at(
&self,
path: &Path,
@@ -379,32 +716,29 @@ impl ReaderPool {
chain: Option<&str>,
pre: Option<&str>,
) -> Result<Arc<Frame>> {
+ let chain = chain.filter(|chain| !chain.trim().is_empty());
+ let pre = pre.filter(|pre| !pre.trim().is_empty());
let facts = self.facts_for(path, still)?;
- let rate = facts.rate;
- let mut target = if facts.still {
- 0
- } else {
- rate.frame_at(time).max(0)
+ let index = facts.index_at(time);
+ let source_key = SourceKey {
+ path: path.to_path_buf(),
+ width,
+ height,
+ index,
};
- // A clip can outlive its media - an end-trim past the file's length -
- // and the honest picture for any time past the end is the last frame,
- // exactly what the roll-forward case already serves. Clamp before
- // seeking: `-ss` past the end decodes nothing at all.
- if let Some(frames) = facts.frames {
- target = target.min((frames - 1).max(0));
+ if chain.is_none() && pre.is_none() {
+ return self.source_frame(&facts, &source_key);
}
- let chain_key = chain.map(str::to_owned);
- let pre_key = pre.map(str::to_owned);
-
- let key = FrameKey {
- path: path.to_path_buf(),
+ let key = TreatedKey {
+ source: source_key,
width,
height,
- chain: chain_key.clone(),
- pre: pre_key.clone(),
- index: target,
+ pre: pre.map(str::to_owned),
+ chain: chain.map(str::to_owned),
+ baked: true,
};
- if let Some(frame) = self.cached(&key) {
+ if let Some(frame) = self.cached_treated(&key) {
+ Self::count(&self.counters.treated_hits);
return Ok(frame);
}
// A still held in memory has no file to open: it is cut to the size
@@ -413,46 +747,96 @@ impl ReaderPool {
let spec: String = [pre, chain]
.into_iter()
.flatten()
- .filter(|part| !part.trim().is_empty())
.collect::<Vec<_>>()
.join(",");
let frame = Arc::new(crate::treat::treat_to(&source, width, height, &spec)?);
- self.remember(key, Arc::clone(&frame));
+ Self::count(&self.counters.treated);
+ self.remember_treated(key, Arc::clone(&frame));
return Ok(frame);
}
+ let shared = self.reader(path, width, height, chain, pre, &facts, index)?;
+ let mut reader = shared.lock().map_err(|_| crate::error::Error::NoFrame {
+ path: path.to_path_buf(),
+ })?;
+ // Another caller may have moved this reader while this one waited
+ // for it; the answer may even be cached by now.
+ if let Some(frame) = self.cached_treated(&key) {
+ Self::count(&self.counters.treated_hits);
+ return Ok(frame);
+ }
+ let remember = |at: i64, frame: Arc<Frame>| {
+ let mut key = key.clone();
+ key.source.index = at;
+ self.remember_treated(key, frame);
+ };
+ self.pull(&mut reader, &facts, index, &remember)?
+ .ok_or_else(|| crate::error::Error::NoFrame {
+ path: path.to_path_buf(),
+ })
+ }
- let shared = self.reader(path, width, height, chain, pre, rate, facts.still, target)?;
+ /// The source frame `key` names, from the cache or a reader.
+ fn source_frame(&self, facts: &MediaFacts, key: &SourceKey) -> Result<Arc<Frame>> {
+ if let Some(frame) = self.cached_source(key) {
+ Self::count(&self.counters.source_hits);
+ return Ok(frame);
+ }
+ let path = key.path.as_path();
+ // A still held in memory has no file to open: it is cut to the
+ // level asked for, here, and kept like any frame.
+ if let Some(source) = self.held_still(path) {
+ let frame = Arc::new(crate::treat::treat_to(
+ &source, key.width, key.height, "",
+ )?);
+ self.remember_source(key.clone(), Arc::clone(&frame));
+ return Ok(frame);
+ }
+ let shared = self.reader(path, key.width, key.height, None, None, facts, key.index)?;
let mut reader = shared.lock().map_err(|_| crate::error::Error::NoFrame {
path: path.to_path_buf(),
})?;
// Another caller may have moved this reader while this one waited
// for it; the answer may even be cached by now.
- if let Some(frame) = self.cached(&key) {
+ if let Some(frame) = self.cached_source(key) {
+ Self::count(&self.counters.source_hits);
return Ok(frame);
}
+ let remember = |at: i64, frame: Arc<Frame>| {
+ let mut key = key.clone();
+ key.index = at;
+ self.remember_source(key, frame);
+ };
+ self.pull(&mut reader, facts, key.index, &remember)?
+ .ok_or_else(|| crate::error::Error::NoFrame {
+ path: path.to_path_buf(),
+ })
+ }
+
+ /// Brings `reader` to `target` and returns the frame there, handing
+ /// every frame decoded on the way to `remember` under its own index -
+ /// the next scrub over this span is then free. `None` only when the
+ /// file has no decodable picture at all.
+ fn pull(
+ &self,
+ reader: &mut Reader,
+ facts: &MediaFacts,
+ target: i64,
+ remember: &dyn Fn(i64, Arc<Frame>),
+ ) -> Result<Option<Arc<Frame>>> {
+ let rate = facts.rate;
let next = (reader.next_index != i64::MIN).then_some(reader.next_index);
if !facts.still && plan_access(next, target) == Access::Seek {
reader.seek(rate, target)?;
}
- // Roll forward to the target, caching everything passed on the way -
- // the next scrub over this span is then free.
- let mut latest: Option<Arc<Frame>> = None;
+ // Roll forward to the target, caching everything passed on the way.
// Stops at end of stream too: a clip trimmed past its media's end,
// where the last real frame is the honest answer.
+ let mut latest: Option<Arc<Frame>> = None;
while let Some((index, frame)) = reader.next_frame(rate)? {
+ Self::count(&self.counters.decoded);
let frame = Arc::new(frame);
- self.remember(
- FrameKey {
- path: path.to_path_buf(),
- width,
- height,
- chain: chain_key.clone(),
- pre: pre_key.clone(),
- index,
- },
- Arc::clone(&frame),
- );
+ remember(index, Arc::clone(&frame));
latest = Some(frame);
if index >= target {
break;
@@ -461,28 +845,19 @@ impl ReaderPool {
// Nothing at all means the seek itself landed past the end of the
// picture stream - a container whose video ends before its audio, or
- // a stated duration that lied past the clamp above. The last real
- // frame is still the honest answer; it just has to be found by
- // seeking backwards until something decodes. Each retry doubles the
- // step, and the last one starts from zero, so a file with any
- // decodable picture at all cannot fail here.
+ // a stated duration that lied past the clamp. The last real frame is
+ // still the honest answer; it just has to be found by seeking
+ // backwards until something decodes. Each retry doubles the step,
+ // and the last one starts from zero, so a file with any decodable
+ // picture at all cannot fail here.
if latest.is_none() && !facts.still {
for step in [30i64, 240, i64::MAX] {
let from = target.saturating_sub(step).max(0);
reader.seek(rate, from)?;
while let Some((index, frame)) = reader.next_frame(rate)? {
+ Self::count(&self.counters.decoded);
let frame = Arc::new(frame);
- self.remember(
- FrameKey {
- path: path.to_path_buf(),
- width,
- height,
- chain: chain_key.clone(),
- pre: pre_key.clone(),
- index,
- },
- Arc::clone(&frame),
- );
+ remember(index, Arc::clone(&frame));
latest = Some(frame);
if index >= target {
break;
@@ -496,13 +871,10 @@ impl ReaderPool {
// Remember the answer under the index that was asked for, so
// dwelling on a time past the end costs one lookup, not a
// respawn-and-decode per request.
- self.remember(key, Arc::clone(frame));
+ remember(target, Arc::clone(frame));
}
}
-
- latest.ok_or_else(|| crate::error::Error::NoFrame {
- path: path.to_path_buf(),
- })
+ Ok(latest)
}
fn facts_for(&self, path: &Path, still: bool) -> Result<Arc<MediaFacts>> {
@@ -522,6 +894,7 @@ impl ReaderPool {
rate: FrameRate::THIRTY,
still: true,
frames: None,
+ size: None,
}
} else {
let info = probe::probe(path)?;
@@ -530,6 +903,11 @@ impl ReaderPool {
.as_ref()
.map(|video| video.frame_rate)
.unwrap_or(FrameRate::THIRTY);
+ let size = info
+ .video
+ .as_ref()
+ .map(|video| (video.width, video.height))
+ .filter(|(width, height)| *width > 0 && *height > 0);
// Ceil rather than floor: undercounting by one would clamp
// legitimate requests for the true last frame.
let frames = info
@@ -540,6 +918,7 @@ impl ReaderPool {
rate,
still: false,
frames,
+ size,
}
}
};
@@ -562,8 +941,7 @@ impl ReaderPool {
height: u32,
chain: Option<&str>,
pre: Option<&str>,
- rate: FrameRate,
- still: bool,
+ facts: &MediaFacts,
target: i64,
) -> Result<Arc<Mutex<Reader>>> {
let key = (
@@ -591,7 +969,14 @@ impl ReaderPool {
// part, and nobody else needs to wait for it. A decode in flight on
// an evicted reader finishes on its own handle.
let opened = Arc::new(Mutex::new(Reader::open(
- path, width, height, chain, pre, rate, still, target,
+ path,
+ width,
+ height,
+ chain,
+ pre,
+ facts.rate,
+ facts.still,
+ target,
)?));
let mut readers = self
.readers
@@ -615,8 +1000,8 @@ impl ReaderPool {
Ok(opened)
}
- /// Drops every warm reader and cached frame - for when the media set
- /// changes wholesale, like closing a project.
+ /// Drops every warm reader, cached frame and adopted proxy - for when
+ /// the media set changes wholesale, like closing a project.
pub fn clear(&self) {
if let Ok(mut readers) = self.readers.lock() {
readers.warm.clear();
@@ -625,15 +1010,22 @@ impl ReaderPool {
if let Ok(mut facts) = self.facts.lock() {
facts.clear();
}
- if let Ok(mut cache) = self.cache.lock() {
+ if let Ok(mut cache) = self.sources.lock() {
+ let budget = cache.budget;
+ *cache = FrameCache::new(budget);
+ }
+ if let Ok(mut cache) = self.treated.lock() {
let budget = cache.budget;
*cache = FrameCache::new(budget);
}
+ if let Ok(mut proxies) = self.proxies.lock() {
+ proxies.clear();
+ }
}
}
#[cfg(test)]
-mod tests {
+pub(crate) mod tests {
use super::*;
use crate::encode::{EncodeOptions, Encoder, FrameSink};
@@ -662,16 +1054,31 @@ mod tests {
);
}
+ /// The level is the source halved while it still covers the request,
+ /// never smaller than the request and never larger than the source.
+ #[test]
+ fn a_level_is_the_source_halved_while_it_still_covers() {
+ assert_eq!(level_for((1920, 1080), (960, 540)), (960, 540));
+ assert_eq!(level_for((3840, 2160), (960, 540)), (960, 540));
+ assert_eq!(level_for((1920, 1080), (1280, 720)), (1920, 1080));
+ assert_eq!(level_for((1920, 1080), (1000, 500)), (1920, 1080));
+ assert_eq!(
+ level_for((640, 360), (1280, 720)),
+ (640, 360),
+ "never upscaled"
+ );
+ assert_eq!(level_for((1080, 1920), (270, 480)), (270, 480));
+ assert_eq!(level_for((7, 5), (2, 2)), (2, 2), "even, and at least two");
+ }
+
#[test]
fn the_cache_evicts_least_recently_used_within_budget() {
// Frames are 16 bytes each (2x2); budget holds exactly two.
let mut cache = FrameCache::new(32);
- let key = |index: i64| FrameKey {
+ let key = |index: i64| SourceKey {
path: PathBuf::from("a.mp4"),
width: 2,
height: 2,
- chain: None,
- pre: None,
index,
};
cache.insert(key(0), Arc::new(Frame::black(2, 2)));
@@ -705,7 +1112,7 @@ mod tests {
assert_eq!((same.width(), same.height()), (64, 32));
assert_eq!(&same.pixels()[..4], &[255, 0, 0, 255]);
let half = pool
- .frame_at(path, Rational::ZERO, 32, 16, true, None, None)
+ .frame(&FrameRequest::new(path, Rational::ZERO, 32, 16).as_still(true))
.expect("half size");
assert_eq!((half.width(), half.height()), (32, 16));
assert_eq!(half.pixels()[0], 255, "still red when scaled");
@@ -713,7 +1120,7 @@ mod tests {
second.fill([0, 0, 255, 255]);
pool.hold_still(path, Arc::new(second));
let again = pool
- .frame_at(path, Rational::ZERO, 32, 16, true, None, None)
+ .frame(&FrameRequest::new(path, Rational::ZERO, 32, 16).as_still(true))
.expect("half size again");
assert_eq!(
again.pixels()[2],
@@ -726,12 +1133,10 @@ mod tests {
fn an_oversized_frame_is_served_but_never_cached() {
let mut cache = FrameCache::new(8);
cache.insert(
- FrameKey {
+ SourceKey {
path: PathBuf::from("a"),
width: 2,
height: 2,
- chain: None,
- pre: None,
index: 0,
},
Arc::new(Frame::black(2, 2)),
@@ -739,23 +1144,32 @@ mod tests {
assert!(cache.is_empty(), "16 bytes cannot fit an 8 byte budget");
}
- /// End to end against the linked FFmpeg: encode a tiny video whose frames
- /// are identifiable by colour, then read arbitrary frames back out of
- /// order.
- #[test]
- fn random_access_returns_the_right_frames() {
- let path = std::env::temp_dir().join("concat-pool-test.mp4");
+ /// A short video whose frames are identifiable by colour: the red
+ /// channel encodes the frame index (x2 to survive compression
+ /// rounding).
+ pub(crate) fn counting_video(name: &str, size: u32, frames: u32) -> PathBuf {
+ let path = std::env::temp_dir().join(format!(
+ "concat-pool-{name}-{}.mp4",
+ std::process::id()
+ ));
let mut encoder =
- Encoder::create(&path, 64, 64, FrameRate::THIRTY, &EncodeOptions::default())
+ Encoder::create(&path, size, size, FrameRate::THIRTY, &EncodeOptions::default())
.expect("the linked FFmpeg encodes h264");
- // 90 frames; the red channel encodes the frame index (x2 to survive
- // compression rounding).
- for index in 0..90u32 {
- let mut frame = Frame::black(64, 64);
+ for index in 0..frames {
+ let mut frame = Frame::black(size, size);
frame.fill([(index * 2).min(255) as u8, 40, 40, 255]);
encoder.write_frame(&frame).expect("writes");
}
encoder.finish().expect("finishes");
+ path
+ }
+
+ /// End to end against the linked FFmpeg: encode a tiny video whose frames
+ /// are identifiable by colour, then read arbitrary frames back out of
+ /// order.
+ #[test]
+ fn random_access_returns_the_right_frames() {
+ let path = counting_video("random", 64, 90);
let pool = ReaderPool::new(64 * 1024 * 1024, 4);
let rate = FrameRate::THIRTY;
@@ -796,6 +1210,129 @@ mod tests {
let _ = std::fs::remove_file(&path);
}
+ /// The scrub benchmark. One pass over the footage decodes it; a second
+ /// pass with an effect on costs no decode, only a filter; a third pass
+ /// with the crop changed costs no decode either; and the fourth pass,
+ /// back to the first pass's knobs, costs nothing at all. Timings are
+ /// printed for a person; the counts are what is asserted.
+ #[test]
+ fn a_scrub_over_covered_ground_never_decodes_again() {
+ // The first half of the file only: the mp4 muxer leaves the last
+ // frame's duration unwritten, so the probed rate of a short
+ // synthetic file runs a hair fast and the second half's indices
+ // land one frame late. Real footage is thousands of frames, where
+ // the hair is nothing; a benchmark wants exact counts.
+ let frames: u32 = 50;
+ let path = counting_video("scrub", 64, 120);
+ let pool = ReaderPool::new(64 * 1024 * 1024, 4);
+ let rate = FrameRate::THIRTY;
+ // Forward and back again, at half size: what a monitor asks.
+ let route: Vec<i64> = (0..i64::from(frames))
+ .chain((0..i64::from(frames)).rev())
+ .collect();
+
+ let pass = |label: &str, pre: Option<&str>, chain: Option<&str>| -> (CacheStats, i64) {
+ let before = pool.stats();
+ let started = std::time::Instant::now();
+ let mut sample = 0;
+ for &index in &route {
+ let request = FrameRequest::new(&path, rate.time_of_frame(index), 32, 32)
+ .prefiltered(pre)
+ .filtered(chain);
+ let frame = pool.frame(&request).expect("frame decodes");
+ assert_eq!((frame.width(), frame.height()), (32, 32));
+ if index == 20 {
+ sample = i64::from(frame.pixel(16, 16).expect("in bounds")[0]);
+ }
+ }
+ let stats = pool.stats().since(before);
+ println!(
+ "{label}: {} requests in {:?}; {stats:?}",
+ route.len(),
+ started.elapsed(),
+ );
+ (stats, sample)
+ };
+
+ let (first, plain) = pass("plain", None, None);
+ assert!(
+ first.decoded >= u64::from(frames),
+ "the first pass decodes the footage: {first:?}"
+ );
+ assert!((plain - 40).abs() <= 8, "frame 20 read red {plain}");
+ assert_eq!(
+ first.source_hits,
+ u64::from(frames),
+ "the way back is source hits: {first:?}"
+ );
+
+ // An effect knob turned: every frame is a source hit and a filter,
+ // and the pixels are the effect's.
+ let (effected, negated) = pass("negate", None, Some("negate"));
+ assert_eq!(
+ effected.decoded, 0,
+ "an effect must not decode: {effected:?}"
+ );
+ assert_eq!(effected.source_hits, u64::from(frames), "{effected:?}");
+ assert_eq!(effected.treated, u64::from(frames), "{effected:?}");
+ assert_eq!(
+ effected.treated_hits,
+ u64::from(frames),
+ "the way back is treated hits: {effected:?}"
+ );
+ assert!(negated > 180, "negated red is high, read {negated}");
+
+ // The crop changed: the same, through the crop.
+ let (cropped, _) = pass(
+ "crop",
+ Some("crop=w=floor(iw*0.5/2)*2:h=floor(ih*0.5/2)*2:x=0:y=0"),
+ Some("negate"),
+ );
+ assert_eq!(cropped.decoded, 0, "a crop must not decode: {cropped:?}");
+ assert_eq!(cropped.source_hits, u64::from(frames), "{cropped:?}");
+
+ // Back to plain: the level is the size asked for, so every request
+ // is the source frame itself - and the very same frames.
+ let (again, plain_again) = pass("plain again", None, None);
+ assert_eq!(again.decoded, 0, "{again:?}");
+ assert_eq!(again.treated, 0, "{again:?}");
+ assert_eq!(again.source_hits, route.len() as u64, "{again:?}");
+ assert_eq!(plain_again, plain);
+
+ let _ = std::fs::remove_file(&path);
+ }
+
+ /// A request that says so reads the adopted proxy, under its own key;
+ /// a request that does not reads the original.
+ #[test]
+ fn a_proxy_is_read_when_asked_for_and_the_original_otherwise() {
+ let original = counting_video("proxied", 64, 30);
+ let proxy = counting_video("proxy", 32, 30);
+ let pool = ReaderPool::new(64 * 1024 * 1024, 4);
+ pool.adopt_proxy(&original, proxy.clone());
+ let rate = FrameRate::THIRTY;
+
+ let request = FrameRequest::new(&original, rate.time_of_frame(10), 16, 16);
+ let from_original = pool.source(&request).expect("original decodes");
+ assert_eq!(
+ (from_original.width(), from_original.height()),
+ (16, 16),
+ "64 halved twice covers 16"
+ );
+ let from_proxy = pool
+ .source(&request.clone().from_proxy(true))
+ .expect("proxy decodes");
+ assert_eq!((from_proxy.width(), from_proxy.height()), (16, 16));
+ let stats = pool.stats();
+ assert!(stats.decoded >= 2, "both files were read: {stats:?}");
+ assert_eq!(pool.proxy_of(&original), Some(proxy.clone()));
+
+ pool.clear();
+ assert_eq!(pool.proxy_of(&original), None, "cleared with the rest");
+ let _ = std::fs::remove_file(&original);
+ let _ = std::fs::remove_file(&proxy);
+ }
+
/// A JPEG still through a pool that cannot cache: every request has to
/// decode afresh from the same warm reader. FFmpeg's image demuxer
/// reads a single JPEG once and never again after a seek, so this
@@ -812,14 +1349,9 @@ mod tests {
let pool = ReaderPool::new(1, 4);
for attempt in 0..3 {
let got = pool
- .frame_at(
- &path,
- FrameRate::THIRTY.time_of_frame(attempt),
- 64,
- 64,
- true,
- None,
- None,
+ .frame(
+ &FrameRequest::new(&path, FrameRate::THIRTY.time_of_frame(attempt), 64, 64)
+ .as_still(true),
)
.unwrap_or_else(|error| panic!("request {attempt} decodes: {error}"));
let red = got.pixel(32, 32).expect("in bounds")[0];
diff --git a/src/crates/concat-media/src/prefetch.rs b/src/crates/concat-media/src/prefetch.rs
new file mode 100644
index 00000000..7be35166
--- /dev/null
+++ b/src/crates/concat-media/src/prefetch.rs
@@ -0,0 +1,528 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Decoding ahead of the playhead, and the one queue everything else
+//! decodes on.
+//!
+//! The reader pool answers the frame under the playhead; this is what
+//! makes that answer a cache hit. A [`Prefetcher`] owns the pool and a few
+//! worker threads, and the transport tells it where the playhead is and
+//! which way it is going - a [`Cursor`] - together with the instants ahead
+//! of it and the frames each needs. The workers decode those into the
+//! pool, nearest first, within a small budget, and hold what they decoded
+//! until the playhead has passed it; a request the playhead has already
+//! passed is dropped unrun, and a cursor that moves supersedes the
+//! instants queued for the last one.
+//!
+//! The same workers take everything else that decodes for a picture on
+//! screen - filmstrip tiles, the bin's thumbnails and waveforms, a proxy
+//! being written - at lower priorities, so an import of twenty files is
+//! twenty jobs waiting their turn on the same few threads and never twenty
+//! decoders at once (#52). One worker always stays clear of the low
+//! priorities, so the artwork of a big import cannot make playback wait
+//! behind it.
+
+use std::collections::VecDeque;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Condvar, Mutex};
+
+use concat_core::frame::Frame;
+
+use crate::pool::{FrameRequest, ReaderPool};
+
+/// How many frame instants the prefetcher decodes ahead of the cursor.
+/// A quarter of a second at thirty: enough that a decode never lands
+/// after its frame is due, small enough that a seek wastes little.
+pub const AHEAD: u32 = 8;
+
+/// What a job is for, most urgent first. A worker takes the most urgent
+/// job waiting; among jobs of one priority, the first queued.
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
+pub enum Priority {
+ /// The frames the playhead is about to reach.
+ Playback,
+ /// A clip's filmstrip, drawn on the timeline.
+ Filmstrip,
+ /// The bin's thumbnails and waveforms.
+ Artwork,
+ /// A proxy being written: minutes of work nobody is waiting on.
+ Proxy,
+}
+
+impl Priority {
+ /// Every priority, most urgent first.
+ pub const ALL: [Priority; 4] = [
+ Priority::Playback,
+ Priority::Filmstrip,
+ Priority::Artwork,
+ Priority::Proxy,
+ ];
+
+ /// Whether a job of this priority is background work: what the
+ /// reserved worker never takes, and what runs one at a time for a
+ /// proxy.
+ fn background(self) -> bool {
+ matches!(self, Priority::Artwork | Priority::Proxy)
+ }
+}
+
+/// Which way the transport is going.
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum Direction {
+ /// Time increasing: play, or a scrub to the right.
+ Forward,
+ /// Time decreasing: shuttle, or a scrub to the left.
+ Backward,
+}
+
+/// Where the playhead is and how it moves, as the transport reports it.
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct Cursor {
+ /// The playhead, in timeline seconds.
+ pub time: f64,
+ /// Which way the next frame lies.
+ pub direction: Direction,
+ /// Timeline seconds a wall second: one for play, two for double
+ /// speed, zero for a scrub or a pause. What tells a reader of proxies
+ /// whether the picture is moving.
+ pub rate: f64,
+}
+
+impl Cursor {
+ /// Whether `time` is behind the cursor: already passed in its
+ /// direction, by more than a frame's slack.
+ fn passed(self, time: f64, slack: f64) -> bool {
+ match self.direction {
+ Direction::Forward => time < self.time - slack,
+ Direction::Backward => time > self.time + slack,
+ }
+ }
+}
+
+/// One instant ahead of the cursor and the frames it is made of.
+#[derive(Clone, Debug)]
+pub struct Moment {
+ /// The instant, in timeline seconds.
+ pub time: f64,
+ /// Every source frame the composite at that instant needs.
+ pub frames: Vec<FrameRequest>,
+}
+
+type Job = Box<dyn FnOnce() + Send>;
+
+/// The queue the workers take from: one lane per priority.
+struct Queue {
+ lanes: [VecDeque<Job>; Priority::ALL.len()],
+ /// Background jobs running now; see [`Priority::background`].
+ background_running: usize,
+ /// Proxy jobs running now: at most one.
+ proxy_running: usize,
+ /// Jobs of any priority running now.
+ running: usize,
+ closed: bool,
+}
+
+struct Lane {
+ queue: Mutex<Queue>,
+ ready: Condvar,
+ /// Fires whenever a job finishes, for [`Prefetcher::drain`].
+ done: Condvar,
+ workers: usize,
+}
+
+impl Lane {
+ /// The most urgent job a worker may take now, honouring the reserve:
+ /// background work never takes the last free worker, and a proxy runs
+ /// alone among proxies.
+ fn take(&self, queue: &mut Queue) -> Option<(Priority, Job)> {
+ let reserve = if self.workers > 1 { 1 } else { 0 };
+ for priority in Priority::ALL {
+ if priority.background() && queue.background_running + reserve >= self.workers {
+ continue;
+ }
+ if priority == Priority::Proxy && queue.proxy_running > 0 {
+ continue;
+ }
+ if let Some(job) = queue.lanes[priority as usize].pop_front() {
+ return Some((priority, job));
+ }
+ }
+ None
+ }
+}
+
+/// What the playback jobs share with the prefetcher: the cursor as last
+/// reported, and the frames decoded ahead of it, held until passed.
+struct Shared {
+ pool: Arc<ReaderPool>,
+ cursor: Mutex<Option<Cursor>>,
+ /// Bumped on every cursor: a job made for an older one is stale.
+ generation: AtomicU64,
+ /// Frames decoded ahead, by instant, held so the pool's eviction
+ /// cannot take them before they are shown; released once passed.
+ pinned: Mutex<Vec<(f64, Vec<Arc<Frame>>)>>,
+ /// A frame's worth of slack, in seconds, when deciding what is passed.
+ slack: Mutex<f64>,
+}
+
+/// The scheduler: owns the readers, decodes ahead of the transport, and
+/// runs everything else that decodes for the screen, in priority order,
+/// on a few threads. See the module docs.
+pub struct Prefetcher {
+ shared: Arc<Shared>,
+ lane: Arc<Lane>,
+}
+
+impl Prefetcher {
+ /// A prefetcher over `pool` with `workers` threads, one at least.
+ pub fn new(pool: Arc<ReaderPool>, workers: usize) -> Self {
+ let workers = workers.max(1);
+ let lane = Arc::new(Lane {
+ queue: Mutex::new(Queue {
+ lanes: Default::default(),
+ background_running: 0,
+ proxy_running: 0,
+ running: 0,
+ closed: false,
+ }),
+ ready: Condvar::new(),
+ done: Condvar::new(),
+ workers,
+ });
+ for index in 0..workers {
+ let lane = Arc::clone(&lane);
+ let _ = std::thread::Builder::new()
+ .name(format!("concat-decode-{index}"))
+ .spawn(move || work(&lane));
+ }
+ Self {
+ shared: Arc::new(Shared {
+ pool,
+ cursor: Mutex::new(None),
+ generation: AtomicU64::new(0),
+ pinned: Mutex::new(Vec::new()),
+ slack: Mutex::new(1.0 / 30.0),
+ }),
+ lane,
+ }
+ }
+
+ /// A prefetcher over `pool` with [`Prefetcher::default_workers`].
+ pub fn with_defaults(pool: Arc<ReaderPool>) -> Self {
+ Self::new(pool, Self::default_workers())
+ }
+
+ /// How many decoders run at once: a quarter of the machine's threads,
+ /// two at least and four at most. A decoder is single-threaded on the
+ /// walk from a keyframe, so this is roughly the share of the machine
+ /// decoding may take while the window and the mix keep the rest.
+ pub fn default_workers() -> usize {
+ std::thread::available_parallelism().map_or(2, |threads| (threads.get() / 4).clamp(2, 4))
+ }
+
+ /// The pool the frames land in.
+ pub fn pool(&self) -> &Arc<ReaderPool> {
+ &self.shared.pool
+ }
+
+ /// The cursor as last reported.
+ pub fn cursor(&self) -> Option<Cursor> {
+ *lock(&self.shared.cursor)
+ }
+
+ /// Moves the cursor and queues `ahead`, nearest first, to be decoded
+ /// into the pool. Instants queued for the last cursor are forgotten;
+ /// frames held for instants now behind this cursor are released; an
+ /// instant behind the cursor by the time a worker reaches it is
+ /// skipped. `frame_seconds` is how long one output frame lasts, the
+ /// slack in "behind".
+ pub fn advance(&self, cursor: Cursor, frame_seconds: f64, ahead: Vec<Moment>) {
+ let generation = self.shared.generation.fetch_add(1, Ordering::AcqRel) + 1;
+ *lock(&self.shared.cursor) = Some(cursor);
+ *lock(&self.shared.slack) = frame_seconds.max(0.0);
+ lock(&self.shared.pinned).retain(|(time, _)| !cursor.passed(*time, frame_seconds));
+
+ let mut queue = lock(&self.lane.queue);
+ if queue.closed {
+ return;
+ }
+ queue.lanes[Priority::Playback as usize].clear();
+ for moment in ahead {
+ let shared = Arc::clone(&self.shared);
+ queue.lanes[Priority::Playback as usize].push_back(Box::new(move || {
+ if shared.generation.load(Ordering::Acquire) != generation {
+ return;
+ }
+ let slack = *lock(&shared.slack);
+ if lock(&shared.cursor).is_some_and(|now| now.passed(moment.time, slack)) {
+ return;
+ }
+ let frames: Vec<Arc<Frame>> = moment
+ .frames
+ .iter()
+ .filter_map(|request| shared.pool.frame(request).ok())
+ .collect();
+ if shared.generation.load(Ordering::Acquire) == generation {
+ lock(&shared.pinned).push((moment.time, frames));
+ }
+ }));
+ }
+ drop(queue);
+ self.lane.ready.notify_all();
+ }
+
+ /// Queues `job` at `priority`; it runs on a worker when its turn comes.
+ pub fn submit(&self, priority: Priority, job: impl FnOnce() + Send + 'static) {
+ let mut queue = lock(&self.lane.queue);
+ if queue.closed {
+ return;
+ }
+ queue.lanes[priority as usize].push_back(Box::new(job));
+ drop(queue);
+ self.lane.ready.notify_all();
+ }
+
+ /// Jobs waiting at `priority`, not counting any running.
+ pub fn pending(&self, priority: Priority) -> usize {
+ lock(&self.lane.queue).lanes[priority as usize].len()
+ }
+
+ /// Frames decoded ahead of the cursor and not yet passed.
+ pub fn held_ahead(&self) -> usize {
+ lock(&self.shared.pinned).len()
+ }
+
+ /// Blocks until every queued job has run. For tests and for a caller
+ /// that must know the pool is warm; the window never waits here.
+ pub fn drain(&self) {
+ let mut queue = lock(&self.lane.queue);
+ while queue.running > 0 || queue.lanes.iter().any(|lane| !lane.is_empty()) {
+ queue = self
+ .lane
+ .done
+ .wait(queue)
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ }
+ }
+}
+
+impl Drop for Prefetcher {
+ fn drop(&mut self) {
+ let mut queue = lock(&self.lane.queue);
+ queue.closed = true;
+ for lane in &mut queue.lanes {
+ lane.clear();
+ }
+ drop(queue);
+ self.lane.ready.notify_all();
+ }
+}
+
+/// One worker: takes the most urgent job it may, runs it, repeats, and
+/// leaves when the prefetcher is dropped.
+fn work(lane: &Lane) {
+ loop {
+ let (priority, job) = {
+ let mut queue = lock(&lane.queue);
+ loop {
+ if queue.closed {
+ return;
+ }
+ if let Some(taken) = lane.take(&mut queue) {
+ queue.running += 1;
+ if taken.0.background() {
+ queue.background_running += 1;
+ }
+ if taken.0 == Priority::Proxy {
+ queue.proxy_running += 1;
+ }
+ break taken;
+ }
+ queue = lane
+ .ready
+ .wait(queue)
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ }
+ };
+ job();
+ let mut queue = lock(&lane.queue);
+ queue.running -= 1;
+ if priority.background() {
+ queue.background_running -= 1;
+ }
+ if priority == Priority::Proxy {
+ queue.proxy_running -= 1;
+ }
+ drop(queue);
+ // A finished job frees a worker for a job the reserve was holding
+ // back, and is what a drain waits for.
+ lane.ready.notify_all();
+ lane.done.notify_all();
+ }
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
+ mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use concat_core::time::FrameRate;
+ use std::sync::atomic::AtomicUsize;
+
+ /// Jobs run most urgent first, and a background job never takes the
+ /// last worker: with one worker and a reserve of none, everything
+ /// runs; the order is by priority, then by arrival.
+ #[test]
+ fn jobs_run_in_priority_order() {
+ let pool = Arc::new(ReaderPool::new(1024, 1));
+ let prefetcher = Prefetcher::new(pool, 1);
+ let order = Arc::new(Mutex::new(Vec::new()));
+ // Hold the worker so the rest queue up behind, then release.
+ let gate = Arc::new((Mutex::new(false), Condvar::new()));
+ {
+ let gate = Arc::clone(&gate);
+ prefetcher.submit(Priority::Filmstrip, move || {
+ let (open, bell) = &*gate;
+ let mut open = open.lock().unwrap();
+ while !*open {
+ open = bell.wait(open).unwrap();
+ }
+ });
+ }
+ for (priority, name) in [
+ (Priority::Artwork, "art-1"),
+ (Priority::Proxy, "proxy"),
+ (Priority::Filmstrip, "strip"),
+ (Priority::Artwork, "art-2"),
+ (Priority::Playback, "play"),
+ ] {
+ let order = Arc::clone(&order);
+ prefetcher.submit(priority, move || order.lock().unwrap().push(name));
+ }
+ {
+ let (open, bell) = &*gate;
+ *open.lock().unwrap() = true;
+ bell.notify_all();
+ }
+ prefetcher.drain();
+ assert_eq!(
+ *order.lock().unwrap(),
+ vec!["play", "strip", "art-1", "art-2", "proxy"]
+ );
+ }
+
+ /// With two workers, one is kept clear of background work: two
+ /// artwork jobs never run together, and a playback job queued while
+ /// one artwork job runs takes the free worker at once.
+ #[test]
+ fn background_work_leaves_one_worker_free() {
+ let pool = Arc::new(ReaderPool::new(1024, 1));
+ let prefetcher = Prefetcher::new(pool, 2);
+ let running = Arc::new(AtomicUsize::new(0));
+ let most = Arc::new(AtomicUsize::new(0));
+ for _ in 0..6 {
+ let running = Arc::clone(&running);
+ let most = Arc::clone(&most);
+ prefetcher.submit(Priority::Artwork, move || {
+ let now = running.fetch_add(1, Ordering::SeqCst) + 1;
+ most.fetch_max(now, Ordering::SeqCst);
+ std::thread::sleep(std::time::Duration::from_millis(15));
+ running.fetch_sub(1, Ordering::SeqCst);
+ });
+ }
+ let played = Arc::new(AtomicUsize::new(0));
+ {
+ let played = Arc::clone(&played);
+ prefetcher.submit(Priority::Playback, move || {
+ played.store(1, Ordering::SeqCst);
+ });
+ }
+ let started = std::time::Instant::now();
+ while played.load(Ordering::SeqCst) == 0 {
+ assert!(
+ started.elapsed() < std::time::Duration::from_millis(500),
+ "playback waited behind the artwork"
+ );
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ }
+ prefetcher.drain();
+ assert_eq!(most.load(Ordering::SeqCst), 1, "artwork ran one at a time");
+ }
+
+ /// The cursor's instants are decoded ahead into the pool, held until
+ /// passed, and a moved cursor drops what it has passed.
+ #[test]
+ fn instants_ahead_are_decoded_and_released_once_passed() {
+ let path = crate::pool::tests::counting_video("prefetch", 32, 90);
+ let pool = Arc::new(ReaderPool::new(64 * 1024 * 1024, 2));
+ let prefetcher = Prefetcher::new(Arc::clone(&pool), 1);
+ let rate = FrameRate::THIRTY;
+ let fps = rate.fps().as_f64();
+ let moments = |from: i64| -> Vec<Moment> {
+ (from + 1..from + 1 + i64::from(AHEAD))
+ .map(|index| Moment {
+ time: index as f64 / fps,
+ frames: vec![FrameRequest::new(
+ &path,
+ rate.time_of_frame(index),
+ 16,
+ 16,
+ )],
+ })
+ .collect()
+ };
+ let cursor = |index: i64| Cursor {
+ time: index as f64 / fps,
+ direction: Direction::Forward,
+ rate: 1.0,
+ };
+
+ prefetcher.advance(cursor(0), 1.0 / fps, moments(0));
+ prefetcher.drain();
+ assert_eq!(prefetcher.held_ahead(), AHEAD as usize);
+ let warmed = pool.stats();
+ assert!(warmed.decoded >= u64::from(AHEAD), "{warmed:?}");
+
+ // The pull for frame 5 is a hit.
+ pool.frame(&FrameRequest::new(&path, rate.time_of_frame(5), 16, 16))
+ .expect("decodes");
+ assert_eq!(pool.stats().since(warmed).decoded, 0, "frame 5 was ahead");
+
+ // The cursor at 5: the instants more than a frame behind it are
+ // released, 4..=8 stay, and 6..=13 join them.
+ prefetcher.advance(cursor(5), 1.0 / fps, moments(5));
+ prefetcher.drain();
+ let held = prefetcher.held_ahead();
+ assert!(
+ (AHEAD as usize + 4..=AHEAD as usize + 6).contains(&held),
+ "held {held}: the passed instants were released"
+ );
+
+ // A cursor far ahead: nothing held behind it survives, and an
+ // instant queued for it that it has already passed is skipped -
+ // asked for at a size nothing else uses, it is a fresh decode
+ // afterwards, so it cannot have been prefetched.
+ let mut stale = moments(20);
+ stale.insert(
+ 0,
+ Moment {
+ time: 2.0 / fps,
+ frames: vec![FrameRequest::new(&path, rate.time_of_frame(2), 8, 8)],
+ },
+ );
+ prefetcher.advance(cursor(20), 1.0 / fps, stale);
+ prefetcher.drain();
+ assert_eq!(prefetcher.held_ahead(), AHEAD as usize);
+ let before = pool.stats();
+ pool.frame(&FrameRequest::new(&path, rate.time_of_frame(2), 8, 8))
+ .expect("decodes");
+ assert!(
+ pool.stats().since(before).decoded > 0,
+ "the passed instant was never decoded"
+ );
+
+ let _ = std::fs::remove_file(&path);
+ }
+}
diff --git a/src/crates/concat-media/src/treat.rs b/src/crates/concat-media/src/treat.rs
index 348e70b2..689befaa 100644
--- a/src/crates/concat-media/src/treat.rs
+++ b/src/crates/concat-media/src/treat.rs
@@ -36,6 +36,47 @@ pub fn treat_to(frame: &Frame, width: u32, height: u32, chain: &str) -> Result<F
if chain.trim().is_empty() && width == frame.width() && height == frame.height() {
return Ok(frame.clone());
}
+ let spec = if chain.trim().is_empty() {
+ format!("scale={width}:{height}:flags=bilinear,format=rgba")
+ } else {
+ format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba")
+ };
+ run(frame, &spec)
+}
+
+/// Runs a source frame the way the decoder would have: `pre` in the
+/// picture's own pixels - the crop - then the fit to `width` by `height`,
+/// then `chain` at that size, then the guard scale that pins the size. The
+/// decoder's graph without the decoder, for a picture the reader pool
+/// holds untreated. Nothing to do at the same size returns a copy.
+pub fn fit(
+ frame: &Frame,
+ pre: Option<&str>,
+ width: u32,
+ height: u32,
+ chain: Option<&str>,
+) -> Result<Frame> {
+ let pre = pre.filter(|pre| !pre.trim().is_empty());
+ let chain = chain.filter(|chain| !chain.trim().is_empty());
+ if pre.is_none() && chain.is_none() && width == frame.width() && height == frame.height() {
+ return Ok(frame.clone());
+ }
+ let mut parts: Vec<String> = Vec::new();
+ if let Some(pre) = pre {
+ parts.push(pre.to_owned());
+ }
+ parts.push(format!("scale={width}:{height}:flags=bilinear"));
+ if let Some(chain) = chain {
+ parts.push(chain.to_owned());
+ parts.push(format!("scale={width}:{height}:flags=bilinear"));
+ }
+ parts.push("format=rgba".to_owned());
+ run(frame, &parts.join(","))
+}
+
+/// One RGBA picture in, `spec` - a whole filtergraph, ending in the size
+/// and format the caller wants out - and one RGBA picture out.
+fn run(frame: &Frame, spec: &str) -> Result<Frame> {
ffi::init();
// The graph has no file behind it; errors name the layer instead.
let path = Path::new("layer");
@@ -64,15 +105,10 @@ pub fn treat_to(frame: &Frame, width: u32, height: u32, chain: &str) -> Result<F
"",
)
.map_err(|error| ffi::fail("buffer sink", path, error))?;
- let spec = if chain.trim().is_empty() {
- format!("scale={width}:{height}:flags=bilinear,format=rgba")
- } else {
- format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba")
- };
graph
.output("in", 0)
.and_then(|parser| parser.input("out", 0))
- .and_then(|parser| parser.parse(&spec))
+ .and_then(|parser| parser.parse(spec))
.map_err(|error| ffi::fail("filter graph", path, error))?;
graph
.validate()
@@ -146,4 +182,29 @@ mod tests {
let pinned = treat(&frame, "scale=3:3").expect("scales then pins");
assert_eq!((pinned.width(), pinned.height()), (6, 6));
}
+
+ /// A fit crops in the source's pixels, scales, and runs the chain at
+ /// the output size: the left half of a picture red on the left and
+ /// blue on the right, fitted to four across, is all red - and negated,
+ /// all cyan.
+ #[test]
+ fn a_fit_crops_then_scales_then_treats() {
+ let mut frame = Frame::black(8, 4);
+ for (index, pixel) in frame.pixels_mut().chunks_exact_mut(4).enumerate() {
+ let x = index % 8;
+ pixel.copy_from_slice(if x < 4 {
+ &[255, 0, 0, 255]
+ } else {
+ &[0, 0, 255, 255]
+ });
+ }
+ let crop = "crop=w=iw/2:h=ih:x=0:y=0";
+ let left = fit(&frame, Some(crop), 4, 4, None).expect("fits");
+ assert_eq!((left.width(), left.height()), (4, 4));
+ assert_eq!(left.pixel(3, 1), Some([255, 0, 0, 255]), "the left half only");
+ let negated = fit(&frame, Some(crop), 4, 4, Some("negate")).expect("fits and treats");
+ assert_eq!(negated.pixel(3, 1), Some([0, 255, 255, 255]));
+ let same = fit(&frame, None, 8, 4, None).expect("copies");
+ assert_eq!(same.pixels(), frame.pixels());
+ }
}