Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(host): one scheduler decodes for the screen, and a large file gets a proxy

Jareerab708384c03a
diff --git a/src/crates/concat-api/src/lib.rs b/src/crates/concat-api/src/lib.rs
index e0f24bf9..6de01e46 100644
--- a/src/crates/concat-api/src/lib.rs
+++ b/src/crates/concat-api/src/lib.rs
@@ -502,6 +502,7 @@ impl Api {
time,
width,
height,
+ moving: false,
},
)
.map_err(ApiError::failed)?;
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index 1d89233c..4b04b5c7 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -1433,14 +1433,38 @@ pub fn preview_sources(
request: &PreviewFrameRequest,
gpu: bool,
) -> Result<PreviewSources, String> {
- preview_sources_of(pool, &preview_timeline(request, gpu), request.time)
+ preview_sources_of(pool, &preview_timeline(request, gpu), request.time, false)
}
-/// [`preview_sources`] for one instant of a plan already built.
+/// The pool's request for one planned layer: the source frame at the
+/// level that covers the output, cropped, fitted and run through its
+/// chain on the way out, so the cached picture is the file's and no knob
+/// invalidates it. `proxy` reads the file's stand-in where it has one:
+/// for a picture that is moving, never for the paused monitor.
+fn frame_request(
+ plan: &PreviewPlan,
+ layer: &concat_render::PlannedLayer,
+ (width, height): (u32, u32),
+ chain: Option<&str>,
+ pre: Option<&str>,
+ still: bool,
+ proxy: bool,
+) -> concat_media::FrameRequest {
+ concat_media::FrameRequest::new(&layer.media, layer.source_time, width, height)
+ .covering(plan.width.max(width), plan.height.max(height))
+ .as_still(still)
+ .prefiltered(pre)
+ .filtered(chain)
+ .from_proxy(proxy)
+}
+
+/// [`preview_sources`] for one instant of a plan already built. With
+/// `proxy`, a file with a stand-in is read from it.
pub fn preview_sources_of(
pool: &concat_media::ReaderPool,
plan: &PreviewPlan,
seconds: f64,
+ proxy: bool,
) -> Result<PreviewSources, String> {
let rate = plan.rate;
let BuiltTimeline {
@@ -1472,15 +1496,15 @@ pub fn preview_sources_of(
let pre = pre_chains.get(&layer.clip).map(String::as_str);
// A source that fails to decode contributes nothing rather than
// blanking the monitor - same grace the exporter extends.
- match pool.frame_at(
- std::path::Path::new(&layer.media),
- layer.source_time,
- decode_width,
- decode_height,
- stills.contains(&layer.clip),
+ match pool.frame(&frame_request(
+ plan,
+ layer,
+ (decode_width, decode_height),
chain,
pre,
- ) {
+ stills.contains(&layer.clip),
+ proxy,
+ )) {
Ok(frame) => {
let highlighted = highlight == Some(layer.clip);
let frame = match cutouts.get(&layer.clip).and_then(|job| {
@@ -1611,13 +1635,31 @@ pub fn preview_prefetch(
preview_prefetch_of(pool, &preview_timeline(request, gpu), request.time, frames);
}
-/// [`preview_prefetch`] from a plan already built.
+/// [`preview_prefetch`] from a plan already built: every frame of the
+/// next `frames` instants pulled through the pool, here and now.
pub fn preview_prefetch_of(
pool: &concat_media::ReaderPool,
plan: &PreviewPlan,
seconds: f64,
frames: u32,
) {
+ for moment in preview_moments(plan, seconds, frames, false) {
+ for request in &moment.frames {
+ let _ = pool.frame(request);
+ }
+ }
+}
+
+/// The instants after `seconds` and the frames each is made of, as a
+/// prefetcher takes them: the next `frames` output instants of `plan`,
+/// nearest first, with every visible layer's request at each. Nothing is
+/// decoded here; see `concat_media::Prefetcher::advance`.
+pub fn preview_moments(
+ plan: &PreviewPlan,
+ seconds: f64,
+ frames: u32,
+ proxy: bool,
+) -> Vec<concat_media::Moment> {
let rate = plan.rate;
let BuiltTimeline {
timeline,
@@ -1628,28 +1670,34 @@ pub fn preview_prefetch_of(
..
} = &plan.built;
let fps = rate.fps().as_f64();
-
- for ahead in 0..frames {
- let time = seconds + f64::from(ahead) / fps;
- let plan_at = plan_frame(timeline, quantise(time, rate));
- for layer in &plan_at.layers {
- let (decode_width, decode_height) = decode_sizes
- .get(&layer.clip)
- .copied()
- .unwrap_or((plan.width, plan.height));
- let chain = filter_chains.get(&layer.clip).map(String::as_str);
- let pre = pre_chains.get(&layer.clip).map(String::as_str);
- let _ = pool.frame_at(
- std::path::Path::new(&layer.media),
- layer.source_time,
- decode_width,
- decode_height,
- stills.contains(&layer.clip),
- chain,
- pre,
- );
- }
- }
+ (1..=frames)
+ .map(|ahead| {
+ let time = seconds + f64::from(ahead) / fps;
+ let plan_at = plan_frame(timeline, quantise(time, rate));
+ concat_media::Moment {
+ time,
+ frames: plan_at
+ .layers
+ .iter()
+ .map(|layer| {
+ let size = decode_sizes
+ .get(&layer.clip)
+ .copied()
+ .unwrap_or((plan.width, plan.height));
+ frame_request(
+ plan,
+ layer,
+ size,
+ filter_chains.get(&layer.clip).map(String::as_str),
+ pre_chains.get(&layer.clip).map(String::as_str),
+ stills.contains(&layer.clip),
+ proxy,
+ )
+ })
+ .collect(),
+ }
+ })
+ .collect()
}
#[cfg(test)]
diff --git a/src/crates/concat-host/src/lib.rs b/src/crates/concat-host/src/lib.rs
index 76964d14..665dc993 100644
--- a/src/crates/concat-host/src/lib.rs
+++ b/src/crates/concat-host/src/lib.rs
@@ -29,6 +29,7 @@ pub mod models;
pub mod playback;
pub mod preview;
pub mod projects;
+pub mod proxy;
pub mod session;
pub mod templates;
pub mod titles;
@@ -40,3 +41,18 @@ pub use jobs::{Job, SingleFlight};
pub use projects::ProjectInfo;
pub use session::{EditorView, Session, SettingsView};
pub use titles::{TitleClip, Titles};
+
+/// The one scheduler every decode for the screen goes through: the
+/// monitor's frames and the frames ahead of the playhead first, then
+/// filmstrips, the bin's artwork and proxies in turn, on a few threads
+/// with one always kept clear of the background work. Owns the reader
+/// pool the monitors read; see `concat_media::Prefetcher`.
+pub fn scheduler() -> &'static std::sync::Arc<concat_media::Prefetcher> {
+ static SCHEDULER: std::sync::OnceLock<std::sync::Arc<concat_media::Prefetcher>> =
+ std::sync::OnceLock::new();
+ SCHEDULER.get_or_init(|| {
+ std::sync::Arc::new(concat_media::Prefetcher::with_defaults(
+ std::sync::Arc::new(concat_media::ReaderPool::with_defaults()),
+ ))
+ })
+}
diff --git a/src/crates/concat-host/src/preview.rs b/src/crates/concat-host/src/preview.rs
index 9f62c46a..d7beeb91 100644
--- a/src/crates/concat-host/src/preview.rs
+++ b/src/crates/concat-host/src/preview.rs
@@ -32,6 +32,10 @@ pub struct FrameSpec {
pub width: u32,
/// Preview frame height in pixels.
pub height: u32,
+ /// The picture is moving - playing - rather than paused or scrubbed:
+ /// what reads a file's proxy where it has one, and tells the
+ /// scheduler which way to decode ahead.
+ pub moving: bool,
}
/// The reader pool behind the monitor, shareable across threads.
@@ -70,7 +74,7 @@ impl Monitor {
/// A monitor with the engine's default pool budget.
pub fn new() -> Self {
Self {
- pool: Arc::new(concat_media::ReaderPool::with_defaults()),
+ pool: Arc::clone(crate::scheduler().pool()),
plan: Arc::new(Mutex::new(None)),
#[cfg(feature = "gpu")]
gpu: None,
@@ -83,7 +87,7 @@ impl Monitor {
#[cfg(feature = "gpu")]
pub fn with_gpu(device: wgpu::Device, queue: wgpu::Queue) -> Self {
Self {
- pool: Arc::new(concat_media::ReaderPool::with_defaults()),
+ pool: Arc::clone(crate::scheduler().pool()),
plan: Arc::new(Mutex::new(None)),
gpu: Some(Arc::new(Mutex::new(
concat_render::WgpuCompositor::with_device(device, queue),
@@ -124,7 +128,7 @@ impl Monitor {
spec: FrameSpec,
) -> Result<concat_export::PreviewSources, String> {
let plan = self.plan_for(clips, settings, spec, true);
- concat_export::preview_sources_of(&self.pool, &plan, spec.time)
+ concat_export::preview_sources_of(&self.pool, &plan, spec.time, spec.moving)
}
/// Draws [`Monitor::frame_sources`] into a texture on the device this
@@ -235,16 +239,18 @@ impl Monitor {
spec: FrameSpec,
) -> Result<Vec<u8>, String> {
let plan = self.plan_for(clips, settings, spec, false);
- let sources = concat_export::preview_sources_of(&self.pool, &plan, spec.time)?;
+ let sources = concat_export::preview_sources_of(&self.pool, &plan, spec.time, spec.moving)?;
Ok(sources
.composite(&mut concat_render::CpuCompositor)
.into_pixels())
}
- /// Decode-ahead for the playback stream: warms the pool for the next
- /// `frames` instants after `spec.time`, so the following [`Monitor::frame`]
- /// pulls are cache hits instead of decode waits. Clamped, so a confused
- /// caller cannot park the pool's mutex on a long decode march.
+ /// Decode-ahead for the playback stream: hands the scheduler the next
+ /// `frames` instants after `spec.time`, so the following
+ /// [`Monitor::frame`] pulls are cache hits instead of decode waits.
+ /// Clamped, so a confused caller cannot queue a long decode march. A
+ /// moving picture reads proxies, the way [`Monitor::frame`] does for
+ /// it, so what is decoded ahead is what will be asked for.
pub fn prefetch(
&self,
clips: Arc<Vec<ExportClip>>,
@@ -253,7 +259,17 @@ impl Monitor {
frames: u32,
) {
let plan = self.plan_for(clips, settings, spec, self.has_gpu());
- concat_export::preview_prefetch_of(&self.pool, &plan, spec.time, frames.min(8));
+ let moments = concat_export::preview_moments(&plan, spec.time, frames.min(8), spec.moving);
+ let fps = (settings.rate_num as f64 / settings.rate_den.max(1) as f64).max(1.0);
+ crate::scheduler().advance(
+ concat_media::Cursor {
+ time: spec.time,
+ direction: concat_media::Direction::Forward,
+ rate: if spec.moving { 1.0 } else { 0.0 },
+ },
+ 1.0 / fps,
+ moments,
+ );
}
/// Forgets every cached frame, reader and plan, for when the project
diff --git a/src/crates/concat-host/src/proxy.rs b/src/crates/concat-host/src/proxy.rs
new file mode 100644
index 00000000..31565142
--- /dev/null
+++ b/src/crates/concat-host/src/proxy.rs
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! Proxies: a smaller copy of a large file, written once into the
+//! project's cache, that playback and the filmstrips read instead of the
+//! original.
+//!
+//! A 4K file decodes at a few frames a second on a laptop without
+//! hardware decode; its quarter-size copy decodes at dozens. So a file
+//! larger than HD gets one, written on the scheduler's proxy lane - one
+//! at a time, on two threads, so the machine stays usable meanwhile - and
+//! adopted into the reader pool as the file's stand-in. A request that
+//! asks for the proxy reads it; the paused monitor and the export never
+//! do, and see the original. The copy is named after the file's path,
+//! size and modification time, so a file replaced on disk gets a fresh
+//! one and the old copy is swept with the rest of the cache.
+
+use std::collections::HashSet;
+use std::hash::{Hash, Hasher};
+use std::path::{Path, PathBuf};
+use std::sync::{Mutex, OnceLock};
+
+use concat_media::decode::{DecodeOptions, Decoder, FrameSource};
+use concat_media::{EncodeOptions, Encoder, FrameSink, Priority, VideoCodec};
+
+/// A file with more pixels than this gets a proxy: anything larger than
+/// full HD.
+const PIXELS_WITHOUT_PROXY: u64 = 1920 * 1080;
+/// The proxy's long side is the original's divided by this, and never
+/// shorter than [`SHORTEST_LONG_SIDE`].
+const DIVISOR: u32 = 4;
+const SHORTEST_LONG_SIDE: u32 = 960;
+
+/// Whether a `width` by `height` file is worth a proxy.
+pub fn wanted(width: u32, height: u32) -> bool {
+ u64::from(width) * u64::from(height) > PIXELS_WITHOUT_PROXY
+}
+
+/// The proxy's size for a `width` by `height` original: the long side a
+/// quarter of the original's, not under [`SHORTEST_LONG_SIDE`] and never
+/// over the original, the aspect kept, both sides even.
+pub fn size_for(width: u32, height: u32) -> (u32, u32) {
+ let (width, height) = (width.max(2), height.max(2));
+ let long = width.max(height);
+ let target = (long / DIVISOR).max(SHORTEST_LONG_SIDE).min(long);
+ let scale = f64::from(target) / f64::from(long);
+ let side = |px: u32| (((f64::from(px) * scale).round() as u32) & !1).max(2);
+ (side(width), side(height))
+}
+
+/// Where the proxy of `media` lives under `project`: named by the file's
+/// path, size and modification time. None for a file that cannot be
+/// stat'ed, which cannot be read either.
+pub fn path_for(project: &Path, media: &str) -> Option<PathBuf> {
+ let meta = std::fs::metadata(media).ok()?;
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ media.hash(&mut hasher);
+ meta.len().hash(&mut hasher);
+ if let Ok(modified) = meta.modified()
+ && let Ok(since) = modified.duration_since(std::time::UNIX_EPOCH)
+ {
+ since.as_secs().hash(&mut hasher);
+ }
+ Some(
+ project
+ .join("cache")
+ .join("proxy")
+ .join(format!("{:016x}.mp4", hasher.finish())),
+ )
+}
+
+/// The proxy of `media` under `project`, when it has been written.
+pub fn existing(project: &Path, media: &str) -> Option<PathBuf> {
+ path_for(project, media).filter(|path| path.is_file())
+}
+
+/// Sees to it that `media` - `width` by `height` - has a proxy when it
+/// is worth one: an existing copy is adopted into the reader pool at
+/// once; otherwise one is queued to be written on the proxy lane and
+/// adopted when done. Asking again while it is being written is free.
+/// Returns whether the file has, or will have, a proxy.
+pub fn ensure(project: &Path, media: &str, width: u32, height: u32) -> bool {
+ if !wanted(width, height) {
+ return false;
+ }
+ let Some(target) = path_for(project, media) else {
+ return false;
+ };
+ let pool = crate::scheduler().pool();
+ if target.is_file() {
+ if pool.proxy_of(Path::new(media)).as_deref() != Some(target.as_path()) {
+ pool.adopt_proxy(Path::new(media), target);
+ }
+ return true;
+ }
+ static WRITING: OnceLock<Mutex<HashSet<PathBuf>>> = OnceLock::new();
+ let writing = WRITING.get_or_init(|| Mutex::new(HashSet::new()));
+ {
+ let mut writing = writing
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ if !writing.insert(target.clone()) {
+ return true;
+ }
+ }
+ let source = media.to_owned();
+ let size = size_for(width, height);
+ crate::scheduler().submit(Priority::Proxy, move || {
+ match write(&source, &target, size) {
+ Ok(()) => {
+ crate::scheduler()
+ .pool()
+ .adopt_proxy(Path::new(&source), target.clone());
+ log::info!("{source}: proxy written to {}", target.display());
+ }
+ Err(error) => log::warn!("{source}: no proxy: {error}"),
+ }
+ writing
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .remove(&target);
+ });
+ true
+}
+
+/// Writes the proxy of `source` at `size` to `target`: H.264, a fast
+/// preset, two threads, in software, into a temporary name that is moved
+/// into place only when whole, so a copy that is there is a copy that is
+/// complete.
+pub fn write(source: &str, target: &Path, (width, height): (u32, u32)) -> Result<(), String> {
+ let info = concat_media::probe(source).map_err(|error| error.to_string())?;
+ let rate = info
+ .video
+ .as_ref()
+ .map(|video| video.frame_rate)
+ .ok_or_else(|| "no video stream".to_owned())?;
+ if let Some(folder) = target.parent() {
+ std::fs::create_dir_all(folder)
+ .map_err(|error| format!("{}: {error}", folder.display()))?;
+ }
+ // Written beside the target under a name the muxer still knows the
+ // format of, and moved into place only when whole.
+ let partial = target.with_extension("part.mp4");
+ let options = DecodeOptions::default()
+ .scaled_to(width, height)
+ .in_software()
+ .threaded(2);
+ let mut decoder = Decoder::open(source, &options).map_err(|error| error.to_string())?;
+ let mut encoder = Encoder::create(
+ &partial,
+ width,
+ height,
+ rate,
+ &EncodeOptions {
+ codec: VideoCodec::H264,
+ preset: "veryfast".to_owned(),
+ crf: 23,
+ ten_bit: false,
+ hardware: false,
+ threads: 2,
+ },
+ )
+ .map_err(|error| error.to_string())?;
+ let result = (|| {
+ while let Some(frame) = decoder.next_frame().map_err(|error| error.to_string())? {
+ encoder
+ .write_frame(&frame)
+ .map_err(|error| error.to_string())?;
+ }
+ encoder.finish().map_err(|error| error.to_string())
+ })();
+ match result {
+ Ok(()) => std::fs::rename(&partial, target)
+ .map_err(|error| format!("{}: {error}", target.display())),
+ Err(error) => {
+ let _ = std::fs::remove_file(&partial);
+ Err(error)
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn only_files_larger_than_hd_get_a_proxy() {
+ assert!(!wanted(1920, 1080));
+ assert!(!wanted(1280, 720));
+ assert!(wanted(3840, 2160));
+ assert!(wanted(2560, 1440));
+ assert!(!wanted(0, 0));
+ }
+
+ #[test]
+ fn a_proxy_is_a_quarter_of_the_long_side_but_never_tiny_or_larger() {
+ assert_eq!(size_for(3840, 2160), (960, 540));
+ assert_eq!(size_for(7680, 4320), (1920, 1080));
+ // A tall file keeps its way up.
+ assert_eq!(size_for(2160, 3840), (540, 960));
+ // A file too small for a quarter gets the floor, never upscaled.
+ assert_eq!(size_for(2560, 1440), (960, 540));
+ assert_eq!(size_for(640, 480), (640, 480));
+ // Odd sizes come out even, and nothing is ever under two.
+ let (w, h) = size_for(3841, 2161);
+ assert!(w % 2 == 0 && h % 2 == 0);
+ assert_eq!(size_for(1, 1), (2, 2));
+ }
+
+ /// A proxy written from a file larger than HD is a quarter of it, is
+ /// adopted by the pool, and serves a moving request while the paused
+ /// one still reads the original.
+ #[test]
+ fn a_written_proxy_is_read_for_a_moving_picture_only() {
+ use concat_core::frame::Frame;
+ use concat_core::time::FrameRate;
+ let dir = std::env::temp_dir().join(format!("concat-proxy-write-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).expect("temp dir");
+ let media = dir.join("big.mp4");
+ {
+ let mut encoder = Encoder::create(
+ &media,
+ 2560,
+ 1440,
+ FrameRate::THIRTY,
+ &EncodeOptions {
+ preset: "ultrafast".to_owned(),
+ hardware: false,
+ ..EncodeOptions::default()
+ },
+ )
+ .expect("encodes");
+ for index in 0..6u8 {
+ let mut frame = Frame::black(2560, 1440);
+ frame.fill([index * 40, 20, 200, 255]);
+ encoder.write_frame(&frame).expect("writes");
+ }
+ encoder.finish().expect("finishes");
+ }
+ let project = dir.join("project");
+ let source = media.to_string_lossy().into_owned();
+ assert!(ensure(&project, &source, 2560, 1440), "worth a proxy");
+ crate::scheduler().drain();
+ let proxy = existing(&project, &source).expect("the proxy was written");
+ let info = concat_media::probe(&proxy).expect("probes");
+ let video = info.video.expect("has pictures");
+ assert_eq!((video.width, video.height), (960, 540));
+ let pool = crate::scheduler().pool();
+ assert_eq!(
+ pool.proxy_of(Path::new(&source)).as_deref(),
+ Some(proxy.as_path())
+ );
+ // Second time round: nothing to write, still adopted.
+ assert!(ensure(&project, &source, 2560, 1440));
+ let time = FrameRate::THIRTY.time_of_frame(2);
+ let moving = pool
+ .frame(&concat_media::FrameRequest::new(&source, time, 480, 270).from_proxy(true))
+ .expect("reads the proxy");
+ assert_eq!((moving.width(), moving.height()), (480, 270));
+ let paused = pool
+ .frame(&concat_media::FrameRequest::new(&source, time, 480, 270))
+ .expect("reads the original");
+ assert_eq!((paused.width(), paused.height()), (480, 270));
+ // A file not worth a proxy is left alone.
+ assert!(!ensure(&project, &source, 1280, 720));
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn the_proxy_path_follows_the_file_and_changes_when_the_file_does() {
+ let dir = std::env::temp_dir().join(format!("concat-proxy-path-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).expect("temp dir");
+ let media = dir.join("clip.mp4");
+ std::fs::write(&media, b"one").expect("writes");
+ let project = dir.join("project");
+ let first = path_for(&project, &media.to_string_lossy()).expect("a path");
+ assert!(first.starts_with(project.join("cache").join("proxy")));
+ assert!(existing(&project, &media.to_string_lossy()).is_none());
+ std::fs::write(&media, b"one more byte").expect("writes");
+ let second = path_for(&project, &media.to_string_lossy()).expect("a path");
+ assert_ne!(first, second, "a changed file is another proxy");
+ assert!(path_for(&project, &dir.join("missing.mp4").to_string_lossy()).is_none());
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+}
diff --git a/src/crates/concat-media/src/pool.rs b/src/crates/concat-media/src/pool.rs
index 92281145..9109a96b 100644
--- a/src/crates/concat-media/src/pool.rs
+++ b/src/crates/concat-media/src/pool.rs
@@ -626,8 +626,26 @@ impl ReaderPool {
/// reads, its facts, and the key of the frame at the level that covers
/// the request.
fn locate(&self, request: &FrameRequest) -> Result<(Arc<MediaFacts>, SourceKey)> {
- let path = self.path_for(request);
- let facts = self.facts_for(&path, request.still)?;
+ let mut path = self.path_for(request);
+ let facts = match self.facts_for(&path, request.still) {
+ Ok(facts) => facts,
+ // A proxy that cannot be read - the cache cleared under the
+ // pool, a folder moved - is forgotten, and the original read
+ // instead: a stand-in that has gone is not a missing picture.
+ Err(error) if path != request.path => {
+ log::warn!(
+ "{}: proxy {} unreadable ({error}); reading the original",
+ request.path.display(),
+ path.display()
+ );
+ if let Ok(mut proxies) = self.proxies.lock() {
+ proxies.remove(&request.path);
+ }
+ path = request.path.clone();
+ self.facts_for(&path, request.still)?
+ }
+ Err(error) => return Err(error),
+ };
let index = facts.index_at(request.time);
// A still is cut to the size asked for: one decode, kept, and the
// levels of a photograph are not worth a photograph's worth of
@@ -785,9 +803,7 @@ impl ReaderPool {
// A still held in memory has no file to open: it is cut to the
// level asked for, here, and kept like any frame.
if let Some(source) = self.held_still(path) {
- let frame = Arc::new(crate::treat::treat_to(
- &source, key.width, key.height, "",
- )?);
+ let frame = Arc::new(crate::treat::treat_to(&source, key.width, key.height, "")?);
self.remember_source(key.clone(), Arc::clone(&frame));
return Ok(frame);
}
@@ -1148,13 +1164,16 @@ pub(crate) mod tests {
/// channel encodes the frame index (x2 to survive compression
/// rounding).
pub(crate) fn counting_video(name: &str, size: u32, frames: u32) -> PathBuf {
- let path = std::env::temp_dir().join(format!(
- "concat-pool-{name}-{}.mp4",
- std::process::id()
- ));
- let mut encoder =
- Encoder::create(&path, size, size, FrameRate::THIRTY, &EncodeOptions::default())
- .expect("the linked FFmpeg encodes h264");
+ let path =
+ std::env::temp_dir().join(format!("concat-pool-{name}-{}.mp4", std::process::id()));
+ let mut encoder = Encoder::create(
+ &path,
+ size,
+ size,
+ FrameRate::THIRTY,
+ &EncodeOptions::default(),
+ )
+ .expect("the linked FFmpeg encodes h264");
for index in 0..frames {
let mut frame = Frame::black(size, size);
frame.fill([(index * 2).min(255) as u8, 40, 40, 255]);
@@ -1302,6 +1321,30 @@ pub(crate) mod tests {
let _ = std::fs::remove_file(&path);
}
+ /// A proxy that has gone from disk is forgotten on the first request
+ /// that asks for it, and the original serves the frame.
+ #[test]
+ fn a_proxy_that_vanished_falls_back_to_the_original() {
+ let path = counting_video("proxy-gone", 32, 10);
+ let pool = ReaderPool::new(64 * 1024 * 1024, 2);
+ let gone =
+ std::env::temp_dir().join(format!("concat-no-such-proxy-{}.mp4", std::process::id()));
+ pool.adopt_proxy(&path, gone.clone());
+ assert_eq!(pool.proxy_of(&path), Some(gone));
+ let frame = pool
+ .frame(
+ &FrameRequest::new(&path, FrameRate::THIRTY.time_of_frame(3), 32, 32)
+ .from_proxy(true),
+ )
+ .expect("the original serves the frame");
+ assert_eq!((frame.width(), frame.height()), (32, 32));
+ assert!(
+ pool.proxy_of(&path).is_none(),
+ "the dead proxy is forgotten"
+ );
+ let _ = std::fs::remove_file(&path);
+ }
+
/// A request that says so reads the adopted proxy, under its own key;
/// a request that does not reads the original.
#[test]
diff --git a/src/crates/concat-media/src/prefetch.rs b/src/crates/concat-media/src/prefetch.rs
index 7be35166..3792cc23 100644
--- a/src/crates/concat-media/src/prefetch.rs
+++ b/src/crates/concat-media/src/prefetch.rs
@@ -292,6 +292,14 @@ impl Prefetcher {
lock(&self.shared.pinned).len()
}
+ /// The instants held ahead, in timeline seconds, in no order.
+ pub fn held_instants(&self) -> Vec<f64> {
+ lock(&self.shared.pinned)
+ .iter()
+ .map(|(time, _)| *time)
+ .collect()
+ }
+
/// Blocks until every queued job has run. For tests and for a caller
/// that must know the pool is warm; the window never waits here.
pub fn drain(&self) {
@@ -362,7 +370,9 @@ fn work(lane: &Lane) {
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
- mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
+ mutex
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[cfg(test)]
@@ -451,6 +461,72 @@ mod tests {
assert_eq!(most.load(Ordering::SeqCst), 1, "artwork ran one at a time");
}
+ /// A cursor that turns round keeps what is now ahead of it and lets
+ /// go of what is now behind: shuttling back over a play does not
+ /// throw the frames just played away, and does not hold what lies
+ /// the other way.
+ #[test]
+ fn a_reversed_cursor_keeps_what_is_ahead_of_it_now() {
+ let path = crate::pool::tests::counting_video("prefetch-reverse", 32, 90);
+ let pool = Arc::new(ReaderPool::new(64 * 1024 * 1024, 2));
+ let prefetcher = Prefetcher::new(Arc::clone(&pool), 1);
+ let rate = FrameRate::THIRTY;
+ let fps = rate.fps().as_f64();
+ let moment = |index: i64| Moment {
+ time: index as f64 / fps,
+ frames: vec![FrameRequest::new(&path, rate.time_of_frame(index), 16, 16)],
+ };
+ prefetcher.advance(
+ Cursor {
+ time: 10.0 / fps,
+ direction: Direction::Forward,
+ rate: 1.0,
+ },
+ 1.0 / fps,
+ (11..19).map(moment).collect(),
+ );
+ prefetcher.drain();
+ assert_eq!(prefetcher.held_ahead(), 8);
+ // Turned round at 14: 15.. are behind it now and go; 11..=14 stay
+ // (14 within a frame's slack), and 13 down to 6 are asked for.
+ prefetcher.advance(
+ Cursor {
+ time: 14.0 / fps,
+ direction: Direction::Backward,
+ rate: -1.0,
+ },
+ 1.0 / fps,
+ (6..14).rev().map(moment).collect(),
+ );
+ prefetcher.drain();
+ let held = prefetcher.held_instants();
+ let slack = 1.0 / fps + 1e-6;
+ assert!(
+ held.iter().all(|time| *time <= 14.0 / fps + slack),
+ "held behind a backward cursor: {held:?}"
+ );
+ assert!(held.len() >= 8, "the way back is decoded: {held:?}");
+ // Nothing the cursor has behind it is ever queued again: an
+ // instant past it is skipped, not decoded.
+ let before = pool.stats();
+ prefetcher.advance(
+ Cursor {
+ time: 5.0 / fps,
+ direction: Direction::Backward,
+ rate: -1.0,
+ },
+ 1.0 / fps,
+ vec![moment(30)],
+ );
+ prefetcher.drain();
+ assert_eq!(
+ pool.stats().since(before).decoded,
+ 0,
+ "30 is behind a backward cursor at 5"
+ );
+ let _ = std::fs::remove_file(&path);
+ }
+
/// The cursor's instants are decoded ahead into the pool, held until
/// passed, and a moved cursor drops what it has passed.
#[test]
@@ -464,12 +540,7 @@ mod tests {
(from + 1..from + 1 + i64::from(AHEAD))
.map(|index| Moment {
time: index as f64 / fps,
- frames: vec![FrameRequest::new(
- &path,
- rate.time_of_frame(index),
- 16,
- 16,
- )],
+ frames: vec![FrameRequest::new(&path, rate.time_of_frame(index), 16, 16)],
})
.collect()
};
diff --git a/src/crates/concat-media/src/treat.rs b/src/crates/concat-media/src/treat.rs
index 689befaa..2e1a5f25 100644
--- a/src/crates/concat-media/src/treat.rs
+++ b/src/crates/concat-media/src/treat.rs
@@ -201,7 +201,11 @@ mod tests {
let crop = "crop=w=iw/2:h=ih:x=0:y=0";
let left = fit(&frame, Some(crop), 4, 4, None).expect("fits");
assert_eq!((left.width(), left.height()), (4, 4));
- assert_eq!(left.pixel(3, 1), Some([255, 0, 0, 255]), "the left half only");
+ assert_eq!(
+ left.pixel(3, 1),
+ Some([255, 0, 0, 255]),
+ "the left half only"
+ );
let negated = fit(&frame, Some(crop), 4, 4, Some("negate")).expect("fits and treats");
assert_eq!(negated.pixel(3, 1), Some([0, 255, 255, 255]));
let same = fit(&frame, None, 8, 4, None).expect("copies");
diff --git a/src/crates/concat/src/host.rs b/src/crates/concat/src/host.rs
index e51a65c7..79c83316 100644
--- a/src/crates/concat/src/host.rs
+++ b/src/crates/concat/src/host.rs
@@ -12,9 +12,8 @@
//! which is why it can live in a plain `RefCell` with no lock.
use std::cell::RefCell;
-use std::collections::VecDeque;
use std::rc::Rc;
-use std::sync::{Arc, Condvar, Mutex, OnceLock};
+use std::sync::Arc;
use concat_host::export::Exporter;
pub use concat_host::media::{strip_window, window_span, window_start};
@@ -197,68 +196,33 @@ fn deliver<T: Send + 'static>(
});
}
-/// The bin's artwork lane: its own few threads, taking jobs off one queue.
-///
-/// Not the shared workers. Those are the monitor's: a frame, its prefetch,
-/// and one more, and an import of twenty files asks for twenty thumbnails,
-/// twenty filmstrips and twenty waveforms at once - on the shared workers
-/// the monitor would wait behind all of it, and as one thread per job it
-/// was every core at a hundred percent until the bin had its pictures, the
-/// machine unusable for a minute (#52). The artwork is decoration; it can
-/// arrive a few files at a time. So it queues here, on threads the monitor
-/// never needs, and only [`art_workers`] decoders ever run together,
-/// whatever the size of the import.
-struct ArtLane {
- queue: Mutex<VecDeque<Box<dyn FnOnce() + Send>>>,
- ready: Condvar,
-}
-
-/// How many artwork decoders run at once: a quarter of the machine's
-/// threads, one at least and three at most. A decoder is single-threaded,
-/// so this is roughly the share of the machine the bin's pictures may take
-/// while the window, playback and the monitor keep the rest.
-fn art_workers() -> usize {
- std::thread::available_parallelism().map_or(1, |threads| (threads.get() / 4).clamp(1, 3))
-}
-
-/// Like [`spawn`], on the artwork lane: `work` waits its turn behind the
-/// other artwork rather than starting a thread of its own.
+/// Like [`spawn`], on the engine's scheduler at the artwork priority:
+/// `work` waits its turn behind the monitor's frames, the frames ahead of
+/// the playhead and the filmstrips, and only a few artwork jobs ever run
+/// together, whatever the size of the import (#52). See
+/// `concat_host::scheduler`.
pub fn spawn_art<T: Send + 'static>(
work: impl FnOnce() -> T + Send + 'static,
then: impl FnOnce(&mut Studio, &App, &Models, T) + Send + 'static,
) {
- static LANE: OnceLock<Arc<ArtLane>> = OnceLock::new();
- let lane = LANE.get_or_init(|| {
- let lane = Arc::new(ArtLane {
- queue: Mutex::new(VecDeque::new()),
- ready: Condvar::new(),
- });
- for index in 0..art_workers() {
- let lane = Arc::clone(&lane);
- let _ = std::thread::Builder::new()
- .name(format!("media-art-{index}"))
- .spawn(move || {
- loop {
- let job = {
- let mut queue = lane.queue.lock().unwrap_or_else(|e| e.into_inner());
- loop {
- if let Some(job) = queue.pop_front() {
- break job;
- }
- queue = lane.ready.wait(queue).unwrap_or_else(|e| e.into_inner());
- }
- };
- job();
- }
- });
- }
- lane
- });
- lane.queue
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .push_back(Box::new(move || deliver(work(), then)));
- lane.ready.notify_one();
+ spawn_at(concat_media::Priority::Artwork, work, then);
+}
+
+/// Like [`spawn_art`], at the filmstrip priority: the lanes' pictures
+/// come before the bin's, after the monitor's.
+pub fn spawn_strip<T: Send + 'static>(
+ work: impl FnOnce() -> T + Send + 'static,
+ then: impl FnOnce(&mut Studio, &App, &Models, T) + Send + 'static,
+) {
+ spawn_at(concat_media::Priority::Filmstrip, work, then);
+}
+
+fn spawn_at<T: Send + 'static>(
+ priority: concat_media::Priority,
+ work: impl FnOnce() -> T + Send + 'static,
+ then: impl FnOnce(&mut Studio, &App, &Models, T) + Send + 'static,
+) {
+ concat_host::scheduler().submit(priority, move || deliver(work(), then));
}
/// Runs `body` on the event-loop thread from anywhere, with a full publish
@@ -614,9 +578,14 @@ pub fn media_art(
let peaks = (kind == MediaKind::Audio || has_audio)
.then(|| media::peaks(&path, None, Some(&project)).ok().map(Arc::new))
.flatten();
+ // The filmstrip reads the proxy where the file has one: the tiles are
+ // small, and a 4K file's frames cost more than they show.
+ let strip_source = concat_host::proxy::existing(std::path::Path::new(&project), &path)
+ .map(|proxy| proxy.to_string_lossy().into_owned())
+ .unwrap_or_else(|| path.clone());
let strip = if pictures {
match kind {
- MediaKind::Video => media::filmstrip(&path, STRIP_FRAMES, STRIP_HEIGHT)
+ MediaKind::Video => media::filmstrip(&strip_source, STRIP_FRAMES, STRIP_HEIGHT)
.ok()
.map(|frame| (frame, STRIP_FRAMES)),
MediaKind::Image => thumbnail.clone().map(|frame| (frame, 1)),
diff --git a/src/crates/concat/src/panes/monitor.rs b/src/crates/concat/src/panes/monitor.rs
index 30b267a2..1f24f36f 100644
--- a/src/crates/concat/src/panes/monitor.rs
+++ b/src/crates/concat/src/panes/monitor.rs
@@ -182,6 +182,7 @@ impl MonitorPane {
time: f64::from(studio.playhead),
width,
height,
+ moving: studio.playing,
};
let monitor = studio.host.monitor.clone();
self.busy = true;
diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rs
index d08cc2c6..5d631c40 100644
--- a/src/crates/concat/src/studio.rs
+++ b/src/crates/concat/src/studio.rs
@@ -49,8 +49,8 @@ use crate::dock::{
use crate::format::{colour_of, frames_timecode, hex_of, hex_with_alpha, wave_path, when_phrase};
use crate::host::{
CachedStrip, Host, MediaArt, WindowArt, cached_media_art, cached_window_art, image_at,
- image_of, media_art, on_ui, spawn, spawn_art, strip_window, window_art, window_span,
- window_start,
+ image_of, media_art, on_ui, spawn, spawn_art, spawn_strip, strip_window, window_art,
+ window_span, window_start,
};
use crate::i18n::{self, t, tf};
use crate::panes::settings::installed;
@@ -1791,6 +1791,22 @@ impl Studio {
return;
};
let project_path = session.path().to_owned();
+ // A file larger than HD gets a proxy for playback and the
+ // filmstrips, written once on the scheduler's proxy lane; see
+ // concat_host::proxy.
+ for item in &self.project().media {
+ if item.kind == model::MediaKind::Video
+ && !item.placeholder
+ && let (Some(width), Some(height)) = (item.width, item.height)
+ {
+ concat_host::proxy::ensure(
+ std::path::Path::new(&project_path),
+ &item.path,
+ width,
+ height,
+ );
+ }
+ }
/// One job: the art key it fills, and what to decode.
struct Want {
key: String,
@@ -2000,7 +2016,7 @@ impl Studio {
}
self.window_pending.insert(key);
let project = project_path.clone();
- spawn_art(
+ spawn_strip(
move || window_art(id, path, project, level, cell, duration),
|studio, _, _, art: WindowArt| {
let key = window_key(&art.id, art.level, art.cell);