Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(video-decode): start AVAssetReader seeks on the exact keyframe tick
Richie McIlroy5ca972e170a7
diff --git a/crates/video-decode/src/avassetreader.rs b/crates/video-decode/src/avassetreader.rsindex 14f1c1205..bc29aba9e 100644--- a/crates/video-decode/src/avassetreader.rs+++ b/crates/video-decode/src/avassetreader.rs@@ -14,6 +14,7 @@ use tokio::runtime::Handle as TokioHandle;#[derive(Clone)]pub struct KeyframeIndex {keyframes: Vec<(u32, f64)>,+ time_base: ffmpeg::Rational,fps: f64,duration_secs: f64,pixel_format: Option<cv::PixelFormat>,@@ -276,6 +277,7 @@ impl KeyframeIndex {Ok(Self {keyframes,+ time_base,fps,duration_secs,pixel_format,@@ -401,20 +403,44 @@ impl KeyframeIndex {pub fn cached_video_info(&self) -> Option<(cv::PixelFormat, u32, u32)> {self.pixel_format.map(|pf| (pf, self.width, self.height))}-}-fn compute_seek_time(keyframe_index: Option<&Arc<KeyframeIndex>>, requested_time: f32) -> f32 {- if let Some(kf_index) = keyframe_index {- let keyframes = kf_index.keyframes();- // Rounded frame numbers can alias keyframes across a VFR gap and seek- // past the image that should still be held at the requested time.- let after_request =- keyframes.partition_point(|(_, time)| *time <= f64::from(requested_time));- if let Some((_, keyframe_time)) = keyframes.get(after_request.saturating_sub(1)) {- return *keyframe_time as f32;+ // The reader decodes from the sync sample at or before its start time, so+ // a start rounded to just below a keyframe decodes the whole previous GOP+ // and vends its last frame retimed to the start. Starting on the exact+ // stream tick keeps the decode on the requested keyframe.+ fn exact_time(&self, secs: f64) -> cm::Time {+ let numerator = i64::from(self.time_base.numerator());+ let denominator = self.time_base.denominator();+ if numerator <= 0 || denominator <= 0 {+ return cm::Time::with_secs(secs, 100);}+ let ticks = (secs * f64::from(denominator) / numerator as f64).round() as i64;+ cm::Time::new(ticks * numerator, denominator)}- requested_time+}++fn seek_keyframe_time(keyframe_index: &KeyframeIndex, requested_time: f32) -> Option<f64> {+ let keyframes = keyframe_index.keyframes();+ // Rounded frame numbers can alias keyframes across a VFR gap and seek+ // past the image that should still be held at the requested time.+ let after_request = keyframes.partition_point(|(_, time)| *time <= f64::from(requested_time));+ keyframes+ .get(after_request.saturating_sub(1))+ .map(|(_, time)| *time)+}++fn compute_seek_time(keyframe_index: Option<&Arc<KeyframeIndex>>, requested_time: f32) -> f32 {+ keyframe_index+ .and_then(|index| seek_keyframe_time(index, requested_time))+ .map_or(requested_time, |time| time as f32)+}++fn reader_start_time(keyframe_index: Option<&Arc<KeyframeIndex>>, requested_time: f32) -> cm::Time {+ keyframe_index+ .and_then(|index| {+ seek_keyframe_time(index, requested_time).map(|time| index.exact_time(time))+ })+ .unwrap_or_else(|| cm::Time::with_secs(requested_time as f64, 100))}pub struct AVAssetReaderDecoder {@@ -490,7 +516,7 @@ impl AVAssetReaderDecoder {let source = Self::get_reader_track_output(&path,- seek_time,+ reader_start_time(keyframe_index.as_ref(), start_time),&tokio_handle,pixel_format,width,@@ -516,7 +542,7 @@ impl AVAssetReaderDecoder {self.source = Self::get_reader_track_output(&self.path,- seek_time,+ reader_start_time(self.keyframe_index.as_ref(), requested_time),&self.tokio_handle,self.pixel_format,self.width,@@ -558,14 +584,14 @@ impl AVAssetReaderDecoder {fn get_reader_track_output(path: &Path,- time: f32,+ start: cm::Time,handle: &TokioHandle,pixel_format: cv::PixelFormat,width: u32,height: u32,) -> Result<ReaderSource, String> {let time_range = cm::TimeRange {- start: cm::Time::with_secs(time as f64, 100),+ start,duration: cm::Time::infinity(),};@@ -723,6 +749,7 @@ mod tests {fn seek_index(keyframes: Keyframes) -> Arc<KeyframeIndex> {Arc::new(KeyframeIndex {keyframes,+ time_base: ffmpeg::Rational::new(1, 600),fps: 2.0,duration_secs: 8.0,pixel_format: None,@@ -755,6 +782,17 @@ mod tests {assert_eq!(compute_seek_time(None, 3.0), 3.0);}+ #[test]+ fn reader_starts_on_the_exact_keyframe_tick() {+ let index = seek_index(vec![(0, 0.0), (1, 577.0 / 600.0), (3, 1147.0 / 600.0)]);+ let start = reader_start_time(Some(&index), 1.5);+ assert_eq!((start.value, start.scale), (577, 600));+ let start = reader_start_time(Some(&index), 1.92);+ assert_eq!((start.value, start.scale), (1147, 600));+ let start = reader_start_time(None, 1.5);+ assert_eq!((start.value, start.scale), (150, 100));+ }+fn both_indexes(path: &Path) -> Option<(Option<Keyframes>, Keyframes)> {ffmpeg::init().unwrap();let mut input = avformat::input(path).ok()?;