The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
fix(media): screen recordings and other variable-rate clips play every frame at its moment in the preview (#457)
Jareerbac31a3f5922
diff --git a/src/crates/concat-host/src/media.rs b/src/crates/concat-host/src/media.rsindex c35fb35d..839e59b3 100644--- a/src/crates/concat-host/src/media.rs+++ b/src/crates/concat-host/src/media.rs@@ -873,6 +873,7 @@ mod tests {width: 1440,height: 1080,frame_rate: concat_core::time::FrameRate::from_int(25),+ cadence: concat_core::time::FrameRate::from_int(25),color_range: None,signal: Signal::Sdr,}),diff --git a/src/crates/concat-media/src/pool.rs b/src/crates/concat-media/src/pool.rsindex d1d05750..3f4d42f7 100644--- a/src/crates/concat-media/src/pool.rs+++ b/src/crates/concat-media/src/pool.rs@@ -370,6 +370,11 @@ struct Reader {/// The frame index `next_frame` will produce, in the media's own rate,/// or `i64::MIN` right after a seek, when only the timestamps know.next_index: i64,+ /// The last frame handed out since the last seek, and its index.+ last: Option<(i64, Arc<Frame>)>,+ /// A frame decoded past a target that had no frame of its own, and its+ /// index: handed out next, before anything more is decoded.+ ahead: Option<(i64, Arc<Frame>)>,}impl Reader {@@ -410,10 +415,18 @@ impl Reader {Ok(Self {decoder,next_index: i64::MIN,+ last: None,+ ahead: None,})}- fn next_frame(&mut self, rate: FrameRate) -> Result<Option<(i64, Frame)>> {+ /// The next frame and its index, and whether it was decoded just now+ /// rather than kept from a pull that went past it.+ fn next_frame(&mut self, rate: FrameRate) -> Result<Option<(i64, Arc<Frame>, bool)>> {+ if let Some((index, frame)) = self.ahead.take() {+ self.next_index = index + 1;+ return Ok(Some((index, frame, false)));+ }let Some(frame) = self.decoder.next_frame()? else {return Ok(None);};@@ -425,14 +438,30 @@ impl Reader {.map(|position| rate.frame_at(position + rate.frame_duration() / Rational::from_int(2))).unwrap_or(self.next_index.max(0));self.next_index = index + 1;- Ok(Some((index, frame)))+ Ok(Some((index, Arc::new(frame), true)))}fn seek(&mut self, rate: FrameRate, index: i64) -> Result<()> {self.decoder.seek(rate.time_of_frame(index))?;self.next_index = i64::MIN;+ self.last = None;+ self.ahead = None;Ok(())}++ /// The frame on screen at `target` when no frame has that index: the+ /// last one handed out, when it is before the target and the frame+ /// kept after it is past the target. A variable-rate stream has no+ /// frame for the length of a still stretch, and every index inside it+ /// is served from here without decoding anything.+ fn held_at(&self, target: i64) -> Option<Arc<Frame>> {+ match (&self.last, &self.ahead) {+ (Some((before, frame)), Some((after, _))) if *before <= target && target < *after => {+ Some(Arc::clone(frame))+ }+ _ => None,+ }+ }}/// The media facts the pool needs per file, probed once.@@ -1042,15 +1071,38 @@ impl ReaderPool {reader.seek(rate, target)?;}+ if let Some(frame) = reader.held_at(target) {+ remember(target, Arc::clone(&frame));+ reader.next_index = target + 1;+ return Ok(Some(frame));+ }+// Roll forward to the target, caching everything passed on the way.// Stops at end of stream too: a clip trimmed past its media's end,- // where the last real frame is the honest answer.+ // where the last real frame is the honest answer. A frame past the+ // target, when the one before it fell short, means no frame has the+ // target's index - a still stretch in a variable-rate recording -+ // and the one before is on screen there: it is the answer, and the+ // one past is kept for the reader to hand out next. Answering with+ // the frame past showed a recording's next movement up to the+ // length of the pause early (#457).let mut latest: Option<Arc<Frame>> = None;let mut reached = i64::MIN;- while let Some((index, frame)) = reader.next_frame(rate)? {- Self::count(&self.counters.decoded);- let frame = Arc::new(frame);- remember(index, Arc::clone(&frame));+ while let Some((index, frame, fresh)) = reader.next_frame(rate)? {+ if fresh {+ Self::count(&self.counters.decoded);+ remember(index, Arc::clone(&frame));+ }+ if index > target+ && let Some((before, held)) = reader.last.clone()+ && before < target+ {+ reader.ahead = Some((index, frame));+ reader.next_index = target + 1;+ remember(target, Arc::clone(&held));+ return Ok(Some(held));+ }+ reader.last = Some((index, Arc::clone(&frame)));latest = Some(frame);reached = index;if index >= target {@@ -1078,10 +1130,10 @@ impl ReaderPool {for step in [30i64, 240, i64::MAX] {let from = target.saturating_sub(step).max(0);reader.seek(rate, from)?;- while let Some((index, frame)) = reader.next_frame(rate)? {+ while let Some((index, frame, _)) = reader.next_frame(rate)? {Self::count(&self.counters.decoded);- let frame = Arc::new(frame);remember(index, Arc::clone(&frame));+ reader.last = Some((index, Arc::clone(&frame)));latest = Some(frame);if index >= target {break;@@ -1122,10 +1174,14 @@ impl ReaderPool {}} else {let info = probe::probe(path)?;+ // Indexed at the cadence, not the average: a variable-rate+ // recording's frames at its average shared slots while it+ // moved - a 60 fps screen capture that averaged 23 played+ // at 23 - and fell between slots elsewhere.let rate = info.video.as_ref()- .map(|video| video.frame_rate)+ .map(|video| video.cadence).unwrap_or(FrameRate::THIRTY);let size = info.video@@ -1289,13 +1345,47 @@ pub(crate) mod tests {#[test]fn a_reader_serves_a_frame_just_behind_it_as_well_as_one_ahead() {assert_eq!(claim_cost(10, 12), Some(2), "just ahead rolls");- assert_eq!(claim_cost(10, 9), Some(1), "one behind: its neighbour won the race");+ assert_eq!(+ claim_cost(10, 9),+ Some(1),+ "one behind: its neighbour won the race"+ );let ahead = i64::from(crate::prefetch::AHEAD);assert_eq!(claim_cost(10 + ahead, 10), Some(ahead));assert_eq!(claim_cost(11 + ahead, 10), None, "further behind seeks");assert_eq!(claim_cost(10, 10 + ROLL_FORWARD_FRAMES + 1), None);}+ /// Playing through a variable-rate recording's still stretch shows the+ /// picture held there at every instant of it, and only then the next+ /// movement - read once, not decoded again for every frame of the+ /// pause. The fixture is white to 0.3 s at 30 fps, with no frame until+ /// black starts at 2 s; its average rate is under 8, and indexed by it+ /// the white frames shared slots and the black came half a pause early+ /// (#457).+ #[test]+ fn playback_through_a_still_screen_holds_the_picture_until_it_moves() {+ let path = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/vfr-still-screen.mp4");+ let pool = ReaderPool::new(64 << 20, 4);+ let mut decoded_in_pause = 0;+ for index in 0..69 {+ let time = Rational::new(index, 30);+ let before = pool.stats();+ let frame = pool+ .frame(&FrameRequest::new(path, time, 32, 32))+ .unwrap_or_else(|error| panic!("{}: {error}", time.as_f64()));+ if (10..59).contains(&index) {+ decoded_in_pause += pool.stats().since(before).decoded;+ }+ let white = frame.pixels()[0] > 200;+ assert_eq!(white, index < 60, "at {:.3} s", time.as_f64());+ }+ assert!(+ decoded_in_pause <= 1,+ "{decoded_in_pause} frames decoded while the screen held still"+ );+ }+/// The level is the source halved while it still covers the request,/// never smaller than the request and never larger than the source.#[test]diff --git a/src/crates/concat-media/src/probe.rs b/src/crates/concat-media/src/probe.rsindex f8568bab..7291d546 100644--- a/src/crates/concat-media/src/probe.rs+++ b/src/crates/concat-media/src/probe.rs@@ -28,6 +28,14 @@ pub struct VideoStream {pub height: u32,/// Average frame rate, exact.pub frame_rate: FrameRate,+ /// The rate one frame can follow another at: the frame rate for a+ /// constant-rate stream. A variable-rate one - a screen recording, a+ /// phone clip - runs at its capture rate while the picture moves and+ /// holds still between, so its average is well under the rate its+ /// frames come at while they come; its cadence is the grid its+ /// timestamps sit on, where that is a rate a camera or a screen records+ /// at. What a reader indexes frames by, so no two land on one slot.+ pub cadence: FrameRate,/// The levels the stream says its numbers span, or `None` where it/// says nothing - which a player then takes for video range. What a/// person compares the picture against when it looks washed out.@@ -184,6 +192,7 @@ fn video_stream(stream: &ffmpeg::format::stream::Stream<'_>, path: &Path) -> Respath: path.to_path_buf(),detail: "video stream has no usable frame rate".to_owned(),})?;+ let cadence = pick_cadence(frame_rate, rational(stream.rate()));// SAFETY: the parameters are live for as long as `stream` is, and the// range, the transfer and the primaries are plain fields libavformat@@ -207,6 +216,7 @@ fn video_stream(stream: &ffmpeg::format::stream::Stream<'_>, path: &Path) -> Reswidth,height,frame_rate: FrameRate::new(frame_rate),+ cadence: FrameRate::new(cadence),color_range: ColorRange::from_ffmpeg(ffmpeg::color::Range::from(range)),signal,})@@ -262,10 +272,47 @@ fn pick_rate(average: Option<Rational>, guess: Option<Rational>) -> Option<Ratio}}+/// The rate a stream's frames come at while they come; see+/// [`VideoStream::cadence`].+///+/// `rate` is what [`pick_rate`] chose. Where the timestamp grid beside it+/// is faster by more than the two percent `pick_rate` forgives - the+/// stream is variable - and is itself a rate a picture is captured at, no+/// faster than 240 fps, the grid is the cadence: a screen recording at 60+/// whose still stretches bring the average to 23. A grid past that is a+/// clock's denominator - 90000 for #394's recording - and says nothing+/// about the frames, so the rate stands.+fn pick_cadence(rate: Rational, grid: Option<Rational>) -> Rational {+ grid.filter(|grid| FrameRate::checked(grid.numerator(), grid.denominator()).is_some())+ .filter(|grid| grid.as_f64() <= 240.0 && grid.as_f64() > rate.as_f64() * 1.02)+ .unwrap_or(rate)+}+#[cfg(test)]mod tests {use super::*;+ /// A variable-rate stream's frames are indexed at the grid they come+ /// on, not its average; a grid that is a clock's denominator, or a+ /// constant stream, keeps the rate.+ #[test]+ fn a_variable_stream_is_paced_by_its_timestamp_grid() {+ let sixty = Rational::from_int(60);+ let average = Rational::new(4620, 179);+ assert_eq!(pick_cadence(average, Some(sixty)), sixty);+ assert_eq!(+ pick_cadence(average, Some(Rational::from_int(90_000))),+ average+ );+ let thirty = Rational::from_int(30);+ assert_eq!(pick_cadence(thirty, Some(thirty)), thirty);+ assert_eq!(pick_cadence(thirty, None), thirty);+ let path = concat!(env!("CARGO_MANIFEST_DIR"), "/fixtures/vfr-still-screen.mp4");+ let video = probe(path).expect("probes").video.expect("a picture");+ assert!(video.frame_rate.fps().as_f64() < 10.0, "the average is low");+ assert_eq!(video.cadence, FrameRate::THIRTY);+ }+#[test]fn falls_back_when_avg_frame_rate_is_zero() {let zero = Some(Rational::ZERO);