Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(recording): reuse one MJPEG decoder across Windows camera frames
Khushal Bhardwajc021d97a16c9
diff --git a/crates/recording/src/output_pipeline/mjpeg.rs b/crates/recording/src/output_pipeline/mjpeg.rsnew file mode 100644index 000000000..db9257a19--- /dev/null+++ b/crates/recording/src/output_pipeline/mjpeg.rs@@ -0,0 +1,154 @@+use anyhow::anyhow;++#[derive(Default)]+pub(crate) struct MjpegDecoder {+ decoder: Option<ffmpeg::codec::decoder::Video>,+}++impl MjpegDecoder {+ pub(crate) fn decode(&mut self, data: &[u8]) -> anyhow::Result<ffmpeg::frame::Video> {+ let mut decoder = match self.decoder.take() {+ Some(decoder) => decoder,+ None => open_mjpeg_decoder()?,+ };++ let packet = ffmpeg::Packet::copy(data);+ decoder+ .send_packet(&packet)+ .map_err(|e| anyhow!("Failed to send MJPEG packet: {e}"))?;++ let mut decoded_frame = ffmpeg::frame::Video::empty();+ decoder+ .receive_frame(&mut decoded_frame)+ .map_err(|e| anyhow!("Failed to decode MJPEG frame: {e}"))?;++ self.decoder = Some(decoder);+ Ok(decoded_frame)+ }+}++fn open_mjpeg_decoder() -> anyhow::Result<ffmpeg::codec::decoder::Video> {+ let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG)+ .ok_or_else(|| anyhow!("MJPEG codec not found"))?;++ ffmpeg::codec::context::Context::new_with_codec(codec)+ .decoder()+ .video()+ .map_err(|e| anyhow!("Failed to create MJPEG decoder: {e}"))+}++#[cfg(test)]+mod tests {+ use super::*;+ use ffmpeg::format::Pixel;++ const WIDTH: u32 = 64;+ const HEIGHT: u32 = 48;++ fn source_luma(base: u8, x: usize) -> u8 {+ base + x as u8+ }++ fn encode_mjpeg(base_luma: u8, chroma: u8) -> Vec<u8> {+ let codec = ffmpeg::codec::encoder::find(ffmpeg::codec::Id::MJPEG).unwrap();+ let mut encoder = ffmpeg::codec::context::Context::new_with_codec(codec)+ .encoder()+ .video()+ .unwrap();+ encoder.set_width(WIDTH);+ encoder.set_height(HEIGHT);+ encoder.set_format(Pixel::YUVJ420P);+ encoder.set_time_base((1, 30));+ let mut encoder = encoder.open().unwrap();++ let mut frame = ffmpeg::frame::Video::new(Pixel::YUVJ420P, WIDTH, HEIGHT);+ let luma_stride = frame.stride(0);+ for row in frame.data_mut(0).chunks_mut(luma_stride) {+ for (x, pixel) in row.iter_mut().take(WIDTH as usize).enumerate() {+ *pixel = source_luma(base_luma, x);+ }+ }+ frame.data_mut(1).fill(chroma);+ frame.data_mut(2).fill(chroma);+ frame.set_pts(Some(0));++ encoder.send_frame(&frame).unwrap();+ encoder.send_eof().unwrap();+ let mut packet = ffmpeg::Packet::empty();+ encoder.receive_packet(&mut packet).unwrap();+ packet.data().unwrap().to_vec()+ }++ fn decode_with_fresh_decoder(data: &[u8]) -> ffmpeg::frame::Video {+ let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG).unwrap();+ let mut decoder = ffmpeg::codec::context::Context::new_with_codec(codec)+ .decoder()+ .video()+ .unwrap();+ decoder.send_packet(&ffmpeg::Packet::copy(data)).unwrap();+ let mut frame = ffmpeg::frame::Video::empty();+ decoder.receive_frame(&mut frame).unwrap();+ frame+ }++ fn luma_rows(frame: &ffmpeg::frame::Video) -> Vec<&[u8]> {+ let width = frame.width() as usize;+ frame+ .data(0)+ .chunks(frame.stride(0))+ .take(frame.height() as usize)+ .map(|row| &row[..width])+ .collect()+ }++ fn assert_matches_fresh_decode(decoded: &ffmpeg::frame::Video, data: &[u8], base_luma: u8) {+ let expected = decode_with_fresh_decoder(data);++ assert_eq!(decoded.width(), WIDTH);+ assert_eq!(decoded.height(), HEIGHT);+ assert_eq!(decoded.format(), expected.format());+ assert_eq!(decoded.width(), expected.width());+ assert_eq!(decoded.height(), expected.height());+ assert_eq!(luma_rows(decoded), luma_rows(&expected));++ let rows = luma_rows(decoded);+ for (x, y) in [(4, 4), (32, 24), (60, 44)] {+ let actual = rows[y][x];+ let source = source_luma(base_luma, x);+ assert!(+ actual.abs_diff(source) <= 6,+ "luma at ({x}, {y}) was {actual}, expected about {source}"+ );+ }+ }++ #[test]+ fn mjpeg_decoder_reused_across_frames_matches_fresh_decoder() {+ let frames: Vec<(u8, Vec<u8>)> =+ [(16u8, 128u8), (60, 90), (100, 160), (140, 40), (30, 200)]+ .into_iter()+ .map(|(base_luma, chroma)| (base_luma, encode_mjpeg(base_luma, chroma)))+ .collect();++ let mut decoder = MjpegDecoder::default();+ for (base_luma, data) in &frames {+ let decoded = decoder.decode(data).unwrap();+ assert_matches_fresh_decode(&decoded, data, *base_luma);+ }+ }++ #[test]+ fn mjpeg_decoder_recovers_after_invalid_input() {+ let valid = encode_mjpeg(80, 128);+ let garbage: Vec<u8> = (0..512u32).map(|i| (i * 37 % 251) as u8).collect();++ let mut decoder = MjpegDecoder::default();+ assert_matches_fresh_decode(&decoder.decode(&valid).unwrap(), &valid, 80);++ assert!(decoder.decode(&garbage).is_err());+ assert_matches_fresh_decode(&decoder.decode(&valid).unwrap(), &valid, 80);++ assert!(decoder.decode(&[]).is_err());+ assert_matches_fresh_decode(&decoder.decode(&valid).unwrap(), &valid, 80);+ }+}diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rsindex d5223a339..25ecb5c52 100644--- a/crates/recording/src/output_pipeline/mod.rs+++ b/crates/recording/src/output_pipeline/mod.rs@@ -7,6 +7,8 @@ mod frame_pacing;mod macos_fragmented_m4s;#[cfg(target_os = "macos")]mod macos_frame_convert;+#[cfg(any(windows, test))]+mod mjpeg;#[cfg(target_os = "macos")]mod oop_fragmented_m4s;pub mod oop_muxer;diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rsindex cf9003cf2..faab33d6e 100644--- a/crates/recording/src/output_pipeline/win.rs+++ b/crates/recording/src/output_pipeline/win.rs@@ -3,6 +3,7 @@ use crate::{output_pipeline::{HealthSender, PipelineHealthEvent, emit_health,frame_pacing::{FramePacer, Paced},+ mjpeg::MjpegDecoder,},screen_capture,};@@ -960,6 +961,7 @@ impl Muxer for WindowsCameraMuxer {let mut last_frame: Option<NativeCameraFrame> = None;let mut first_timestamp: Option<Duration> = None;let mut frame_count = 0u64;+ let mut mjpeg_decoder = MjpegDecoder::default();loop {let ts = match pacer.next(&video_rx, &pause_flag) {@@ -976,7 +978,7 @@ impl Muxer for WindowsCameraMuxer {let normalized_ts = normalize_camera_timestamp(ts, &mut first_timestamp);- let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame) {+ let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame, &mut mjpeg_decoder) {Ok(f) => f,Err(e) => {error!("Failed to convert camera frame: {e}");@@ -1345,11 +1347,14 @@ fn convert_uyvy_to_yuyv(dst}-pub fn camera_frame_to_ffmpeg(frame: &NativeCameraFrame) -> anyhow::Result<ffmpeg::frame::Video> {+pub(crate) fn camera_frame_to_ffmpeg(+ frame: &NativeCameraFrame,+ mjpeg_decoder: &mut MjpegDecoder,+) -> anyhow::Result<ffmpeg::frame::Video> {use cap_mediafoundation_utils::IMFMediaBufferExt;if frame.pixel_format == cap_camera_windows::PixelFormat::MJPEG {- return decode_mjpeg_frame(frame);+ return decode_mjpeg_frame(frame, mjpeg_decoder);}let ffmpeg_format = match frame.pixel_format {@@ -1625,7 +1630,10 @@ fn copy_plane(}}-fn decode_mjpeg_frame(frame: &NativeCameraFrame) -> anyhow::Result<ffmpeg::frame::Video> {+fn decode_mjpeg_frame(+ frame: &NativeCameraFrame,+ mjpeg_decoder: &mut MjpegDecoder,+) -> anyhow::Result<ffmpeg::frame::Video> {use cap_mediafoundation_utils::IMFMediaBufferExt;let buffer_guard = frame@@ -1637,26 +1645,7 @@ fn decode_mjpeg_frame(frame: &NativeCameraFrame) -> anyhow::Result<ffmpeg::frame.map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?;let data = &*lock;- let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG)- .ok_or_else(|| anyhow!("MJPEG codec not found"))?;-- let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);- let mut decoder = decoder_context- .decoder()- .video()- .map_err(|e| anyhow!("Failed to create MJPEG decoder: {e}"))?;-- let packet = ffmpeg::Packet::copy(data);- decoder- .send_packet(&packet)- .map_err(|e| anyhow!("Failed to send MJPEG packet: {e}"))?;-- let mut decoded_frame = ffmpeg::frame::Video::empty();- decoder- .receive_frame(&mut decoded_frame)- .map_err(|e| anyhow!("Failed to decode MJPEG frame: {e}"))?;-- Ok(decoded_frame)+ mjpeg_decoder.decode(data)}fn flip_buffer_size(diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rsindex 67278a913..b517b5759 100644--- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs+++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs@@ -3,6 +3,7 @@ use super::core::{combine_finish_errors, wait_for_blocking_thread_finish,};use super::frame_pacing::{FramePacer, Paced};+use super::mjpeg::MjpegDecoder;use super::win::reference_video_frame;use crate::{AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer,@@ -830,6 +831,7 @@ impl WindowsFragmentedM4SCameraMuxer {let mut duplicated_frames = 0u64;let mut disk_monitor = DiskSpaceMonitor::new();let mut disk_exhausted = false;+ let mut mjpeg_decoder = MjpegDecoder::default();const SLOW_THRESHOLD_MS: u128 = 5;let normalize_timestamp =@@ -848,10 +850,11 @@ impl WindowsFragmentedM4SCameraMuxer {slow_convert_count: &mut u32,slow_encode_count: &mut u32,total_frames: &mut u64,- encoder: &Arc<Mutex<SegmentedVideoEncoder>>|+ encoder: &Arc<Mutex<SegmentedVideoEncoder>>,+ mjpeg_decoder: &mut MjpegDecoder|-> anyhow::Result<()> {let convert_start = std::time::Instant::now();- let ffmpeg_frame_result = camera_frame_to_ffmpeg(camera_frame);+ let ffmpeg_frame_result = camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder);let convert_elapsed_ms = convert_start.elapsed().as_millis();if convert_elapsed_ms > SLOW_THRESHOLD_MS {@@ -949,6 +952,7 @@ impl WindowsFragmentedM4SCameraMuxer {&mut slow_encode_count,&mut total_frames,&encoder_clone,+ &mut mjpeg_decoder,) {warn!("Failed to encode drained camera frame: {e}");break;@@ -983,6 +987,7 @@ impl WindowsFragmentedM4SCameraMuxer {&mut slow_encode_count,&mut total_frames,&encoder_clone,+ &mut mjpeg_decoder,) {error!("Camera encoder mutex poisoned - encoder thread likely panicked, stopping");return Err(e);diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rsindex c99bf9605..c377d3388 100644--- a/crates/recording/src/output_pipeline/win_segmented_camera.rs+++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs@@ -375,7 +375,7 @@ impl WindowsSegmentedCameraMuxer {}fn create_segment(&mut self, first_frame: &NativeCameraFrame) -> anyhow::Result<()> {- use crate::output_pipeline::win::camera_frame_to_ffmpeg;+ use crate::output_pipeline::{mjpeg::MjpegDecoder, win::camera_frame_to_ffmpeg};let segment_path = self.current_segment_path();@@ -595,6 +595,7 @@ impl WindowsSegmentedCameraMuxer {);let mut first_timestamp: Option<Duration> = None;+ let mut mjpeg_decoder = MjpegDecoder::default();while let Ok(Some((frame, timestamp))) = video_rx.recv() {let relative = if let Some(first) = first_timestamp {@@ -604,7 +605,7 @@ impl WindowsSegmentedCameraMuxer {Duration::ZERO};- let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame) {+ let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame, &mut mjpeg_decoder) {Ok(f) => f,Err(e) => {warn!("Failed to convert camera frame: {e}");