Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat: clock-paced playback stream with engine decode-ahead

Jareer894f98ee1b2d
diff --git a/desktop/src-tauri/src/export.rs b/desktop/src-tauri/src/export.rs
index 0dab9e19..a5deb6da 100644
--- a/desktop/src-tauri/src/export.rs
+++ b/desktop/src-tauri/src/export.rs
@@ -14,7 +14,7 @@ use tauri::Emitter;
pub use wolfcut_export::{
ClipKind, ExportClip, ExportRequest, PreviewFrameRequest, Reporter, TransitionSpec,
- preview_frame, render,
+ preview_frame, preview_prefetch, render,
};
/// One progress event, as the UI's export dialog consumes it.
diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs
index 65df2185..e0d4f0cf 100644
--- a/desktop/src-tauri/src/lib.rs
+++ b/desktop/src-tauri/src/lib.rs
@@ -705,6 +705,40 @@ async fn preview_frame(
.map(tauri::ipc::Response::new)
}
+/// Decode-ahead for the playback stream.
+///
+/// Warms the pool for the next few frame instants after `request.time`, so
+/// the following `preview_frame` pulls are cache hits instead of decode
+/// waits. Fire-and-forget from the UI: a source that will not decode fails
+/// the pull too, and the pull is the path that reports it.
+#[tauri::command]
+async fn preview_prefetch(
+ state: tauri::State<'_, PoolState>,
+ editor: tauri::State<'_, editor_api::EditorState>,
+ request: PreviewSpec,
+ frames: u32,
+) -> Result<(), String> {
+ let (clips, settings) = editor_api::flattened_clips(&editor)?;
+ let request = export::PreviewFrameRequest {
+ time: request.time,
+ width: request.width,
+ height: request.height,
+ rate_num: settings.rate_num,
+ rate_den: settings.rate_den,
+ clips,
+ };
+ let pool = std::sync::Arc::clone(&state.0);
+ tauri::async_runtime::spawn_blocking(move || {
+ if let Ok(mut pool) = pool.lock() {
+ // Clamped so a confused caller cannot park the pool's mutex on a
+ // long decode march the presenter then queues behind.
+ export::preview_prefetch(&mut pool, &request, frames.min(8));
+ }
+ })
+ .await
+ .map_err(|error| format!("prefetch task failed: {error}"))
+}
+
/// Renders the timeline to a file.
///
/// Runs on a blocking thread and reports progress through the
@@ -860,6 +894,7 @@ pub fn run() {
export_project,
cancel_export,
preview_frame,
+ preview_prefetch,
template_list,
template_save,
template_instantiate,
diff --git a/desktop/src/App.tsx b/desktop/src/App.tsx
index c22d7e90..0f8c8d0b 100644
--- a/desktop/src/App.tsx
+++ b/desktop/src/App.tsx
@@ -961,6 +961,7 @@ function Editor({
project,
renderableClipCount,
frame,
+ fps: session ? session.rateNum / session.rateDen : 30,
latest,
});
diff --git a/desktop/src/hooks/useEngineTruth.ts b/desktop/src/hooks/useEngineTruth.ts
index 86621c1a..60ac08d3 100644
--- a/desktop/src/hooks/useEngineTruth.ts
+++ b/desktop/src/hooks/useEngineTruth.ts
@@ -4,10 +4,13 @@
*
* Paused: when the playhead settles, fetch the exporter's own composite at
* up to 960px and hold it over the approximation until the playhead moves.
- * Playing: while two or more visual layers sit under the playhead, pull
- * frames continuously at up to 640px - request at the transport's position,
- * present, request the next when it lands (desktop decision 0009). Single
- * layers keep the smooth element preview.
+ * Playing: while two or more visual layers sit under the playhead, stream
+ * frames at up to 640px against the transport clock - requests are
+ * quantised to the project's frame grid and issued one measured round-trip
+ * ahead of the interpolated playhead, so a frame lands about when it is
+ * due; each presented frame also warms the engine's cache for the instants
+ * after it (desktop decision 0009). Single layers keep the smooth element
+ * preview.
*
* The request carries an instant and a resolution, nothing else: the engine
* flattens its own session (engine decision 0009). The UI used to serialise
@@ -17,7 +20,7 @@
import { useEffect, useRef, useState } from "react";
import { clipsAt, type EditorProject } from "../lib/editor";
-import { previewFrame } from "../lib/engine";
+import { previewFrame, previewPrefetch } from "../lib/engine";
export interface EngineStill {
bytes: ArrayBuffer;
@@ -41,6 +44,7 @@ export function useEngineTruth({
project,
renderableClipCount,
frame,
+ fps,
latest,
}: {
playing: boolean;
@@ -51,6 +55,8 @@ export function useEngineTruth({
/** Non-text clips on the active timeline - zero means nothing to composite. */
renderableClipCount: number;
frame: { width: number; height: number };
+ /** The project's frame rate; the grid streamed requests snap to. */
+ fps: number;
/** Live values, read mid-flight without restarting the loops. */
latest: { current: { playhead: number; frame: { width: number; height: number }; project: EditorProject } };
}): EngineStill | null {
@@ -81,14 +87,21 @@ export function useEngineTruth({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [playing, loaded, playhead, project, renderableClipCount, frame]);
- // The playback stream. Pull, not push: one request in flight, never a
- // queue of stale frames; the pool's warm readers make consecutive
- // requests rolls, not seeks. Rate beats resolution while moving.
+ // The playback stream, paced by the transport clock. Still pull - one
+ // request in flight, never a queue of stale frames - but the requests are
+ // snapped to the project's frame grid and issued one lead ahead of the
+ // interpolated playhead, where the lead tracks the measured round-trip:
+ // a frame arrives about when it is due instead of a round-trip late, and
+ // the same instant is never composited twice. After each presented frame
+ // a fire-and-forget prefetch warms the engine's cache for the instants
+ // after it, which keeps the pool's readers rolling forward and turns the
+ // next pulls into cache hits. Rate beats resolution while moving.
useEffect(() => {
if (!playing || !loaded) return;
let live = true;
// Invalidate any paused-dwell fetch still in flight from before play.
stillToken.current += 1;
+ const rate = Number.isFinite(fps) && fps > 0 ? fps : 30;
const wait = (ms: number) => new Promise((resolve) => window.setTimeout(resolve, ms));
const visualLayers = (time: number) =>
clipsAt(latest.current.project, time).filter(
@@ -96,21 +109,44 @@ export function useEngineTruth({
).length;
const run = async () => {
+ // The lead starts at one frame and follows the round-trip, clamped to
+ // [1, 3] frames - a pathological decode must not push requests seconds
+ // ahead of what the user is hearing.
+ let lead = 1 / rate;
+ let presented = -1;
while (live) {
- const time = latest.current.playhead;
- if (visualLayers(time) < 2) {
+ const now = latest.current.playhead;
+ if (visualLayers(now) < 2) {
setEngineStill(null);
+ presented = -1;
await wait(120);
continue;
}
+ const target = Math.round((now + lead) * rate);
+ if (target === presented) {
+ // The clock has not reached the next frame yet; a quarter-frame
+ // nap keeps the loop from spinning on an already-shown instant.
+ await wait(250 / rate);
+ continue;
+ }
const { width, height } = capped(latest.current.frame, 640);
+ const started = performance.now();
try {
- const bytes = await previewFrame({ time, width, height });
- if (live) setEngineStill({ bytes, width, height });
+ const bytes = await previewFrame({ time: target / rate, width, height });
+ const trip = (performance.now() - started) / 1000;
+ lead = Math.min(Math.max(0.8 * lead + 0.2 * trip, 1 / rate), 3 / rate);
+ presented = target;
+ if (live) {
+ setEngineStill({ bytes, width, height });
+ void previewPrefetch({ time: (target + 1) / rate, width, height }, 2).catch(
+ () => undefined,
+ );
+ }
} catch {
// A failed frame keeps the approximation on screen; back off so a
// persistently failing source cannot spin the pool.
if (live) setEngineStill(null);
+ presented = -1;
await wait(200);
}
}
@@ -119,7 +155,7 @@ export function useEngineTruth({
return () => {
live = false;
};
- }, [playing, loaded, latest]);
+ }, [playing, loaded, latest, fps]);
return engineStill;
}
diff --git a/desktop/src/lib/engine.ts b/desktop/src/lib/engine.ts
index 1f7832a0..e7eafed1 100644
--- a/desktop/src/lib/engine.ts
+++ b/desktop/src/lib/engine.ts
@@ -395,6 +395,22 @@ export async function previewFrame(request: {
return invoke<ArrayBuffer>("preview_frame", { request });
}
+/**
+ * Warms the engine's frame cache for the instants about to play.
+ *
+ * Fire and forget, from the playback stream only: after presenting a frame,
+ * asking for the next `frames` starting at `request.time` turns the next
+ * `previewFrame` pulls into cache hits instead of decode waits. Failures
+ * are silent here - a source that will not decode fails the pull too, and
+ * the pull is the path that reports it.
+ */
+export async function previewPrefetch(
+ request: { time: number; width: number; height: number },
+ frames: number,
+): Promise<void> {
+ return invoke<void>("preview_prefetch", { request, frames });
+}
+
/**
* Writes one file into the project's cache folder and returns its path.
*
diff --git a/engine/crates/wolfcut-export/src/lib.rs b/engine/crates/wolfcut-export/src/lib.rs
index 8bb7939a..355c29fb 100644
--- a/engine/crates/wolfcut-export/src/lib.rs
+++ b/engine/crates/wolfcut-export/src/lib.rs
@@ -727,28 +727,8 @@ pub fn preview_frame(
request: &PreviewFrameRequest,
) -> Result<Vec<u8>, String> {
let rate = FrameRate::new(Rational::new(request.rate_num, request.rate_den));
-
- let mut resolved = request.clips.clone();
- resolve_transitions(&mut resolved, rate, false);
- let visible: Vec<&ExportClip> = resolved
- .iter()
- .filter(|clip| clip.kind.is_visual() && !clip.hidden)
- .collect();
-
- // build_timeline reads only the output format off the request; the shim
- // keeps one conversion path rather than a preview-flavoured copy of it.
- let shim = ExportRequest {
- output: String::new(),
- width: request.width,
- height: request.height,
- rate_num: request.rate_num,
- rate_den: request.rate_den,
- crf: 18,
- preset: String::new(),
- clips: Vec::new(),
- };
let BuiltTimeline { timeline, stills, decode_sizes, filter_chains } =
- build_timeline(&shim, rate, &visible);
+ preview_timeline(request, rate);
let plan = plan_frame(&timeline, quantise(request.time, rate));
let mut sources: Vec<(std::sync::Arc<Frame>, f32, Transform)> =
@@ -800,6 +780,72 @@ pub fn preview_frame(
Ok(composed.into_pixels())
}
+/// The preview's timeline, built the exporter's way.
+fn preview_timeline(request: &PreviewFrameRequest, rate: FrameRate) -> BuiltTimeline {
+ let mut resolved = request.clips.clone();
+ resolve_transitions(&mut resolved, rate, false);
+ let visible: Vec<&ExportClip> = resolved
+ .iter()
+ .filter(|clip| clip.kind.is_visual() && !clip.hidden)
+ .collect();
+
+ // build_timeline reads only the output format off the request; the shim
+ // keeps one conversion path rather than a preview-flavoured copy of it.
+ let shim = ExportRequest {
+ output: String::new(),
+ width: request.width,
+ height: request.height,
+ rate_num: request.rate_num,
+ rate_den: request.rate_den,
+ crf: 18,
+ preset: String::new(),
+ clips: Vec::new(),
+ };
+ build_timeline(&shim, rate, &visible)
+}
+
+/// Warms the reader pool for the frames about to be presented.
+///
+/// The playback stream's decode-ahead half: while the UI presents the frame
+/// at `request.time`, this decodes the next `frames` frame instants into the
+/// pool's cache so the next pulls are hits, not decode waits. Requests stay
+/// monotonic and one frame apart, which is exactly what keeps the pool's
+/// readers rolling forward instead of respawning FFmpeg to seek.
+///
+/// Compositing is skipped - the pull composites - and so are failures: a
+/// source that will not decode fails the pull too, and that is the path
+/// with an error channel.
+pub fn preview_prefetch(
+ pool: &mut wolfcut_media::ReaderPool,
+ request: &PreviewFrameRequest,
+ frames: u32,
+) {
+ let rate = FrameRate::new(Rational::new(request.rate_num, request.rate_den));
+ let BuiltTimeline { timeline, stills, decode_sizes, filter_chains } =
+ preview_timeline(request, rate);
+ let fps = rate.fps().as_f64();
+
+ for ahead in 0..frames {
+ let time = request.time + f64::from(ahead) / fps;
+ let plan = plan_frame(&timeline, quantise(time, rate));
+ for layer in &plan.layers {
+ let (decode_width, decode_height) = decode_sizes
+ .get(&layer.clip)
+ .copied()
+ .unwrap_or((request.width, request.height));
+ let chain = filter_chains.get(&layer.clip).map(String::as_str);
+ let _ = pool.frame_at(
+ std::path::Path::new(&layer.media),
+ layer.source_time,
+ decode_width,
+ decode_height,
+ stills.contains(&layer.clip),
+ chain,
+ );
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;