Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(recording): reuse one MJPEG decoder across Windows camera frames

Khushal Bhardwajc021d97a16c9
diff --git a/crates/recording/src/output_pipeline/mjpeg.rs b/crates/recording/src/output_pipeline/mjpeg.rs
new file mode 100644
index 000000000..db9257a19
--- /dev/null
+++ b/crates/recording/src/output_pipeline/mjpeg.rs
@@ -0,0 +1,154 @@
+use anyhow::anyhow;
+
+#[derive(Default)]
+pub(crate) struct MjpegDecoder {
+ decoder: Option<ffmpeg::codec::decoder::Video>,
+}
+
+impl MjpegDecoder {
+ pub(crate) fn decode(&mut self, data: &[u8]) -> anyhow::Result<ffmpeg::frame::Video> {
+ let mut decoder = match self.decoder.take() {
+ Some(decoder) => decoder,
+ None => open_mjpeg_decoder()?,
+ };
+
+ let packet = ffmpeg::Packet::copy(data);
+ decoder
+ .send_packet(&packet)
+ .map_err(|e| anyhow!("Failed to send MJPEG packet: {e}"))?;
+
+ let mut decoded_frame = ffmpeg::frame::Video::empty();
+ decoder
+ .receive_frame(&mut decoded_frame)
+ .map_err(|e| anyhow!("Failed to decode MJPEG frame: {e}"))?;
+
+ self.decoder = Some(decoder);
+ Ok(decoded_frame)
+ }
+}
+
+fn open_mjpeg_decoder() -> anyhow::Result<ffmpeg::codec::decoder::Video> {
+ let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG)
+ .ok_or_else(|| anyhow!("MJPEG codec not found"))?;
+
+ ffmpeg::codec::context::Context::new_with_codec(codec)
+ .decoder()
+ .video()
+ .map_err(|e| anyhow!("Failed to create MJPEG decoder: {e}"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use ffmpeg::format::Pixel;
+
+ const WIDTH: u32 = 64;
+ const HEIGHT: u32 = 48;
+
+ fn source_luma(base: u8, x: usize) -> u8 {
+ base + x as u8
+ }
+
+ fn encode_mjpeg(base_luma: u8, chroma: u8) -> Vec<u8> {
+ let codec = ffmpeg::codec::encoder::find(ffmpeg::codec::Id::MJPEG).unwrap();
+ let mut encoder = ffmpeg::codec::context::Context::new_with_codec(codec)
+ .encoder()
+ .video()
+ .unwrap();
+ encoder.set_width(WIDTH);
+ encoder.set_height(HEIGHT);
+ encoder.set_format(Pixel::YUVJ420P);
+ encoder.set_time_base((1, 30));
+ let mut encoder = encoder.open().unwrap();
+
+ let mut frame = ffmpeg::frame::Video::new(Pixel::YUVJ420P, WIDTH, HEIGHT);
+ let luma_stride = frame.stride(0);
+ for row in frame.data_mut(0).chunks_mut(luma_stride) {
+ for (x, pixel) in row.iter_mut().take(WIDTH as usize).enumerate() {
+ *pixel = source_luma(base_luma, x);
+ }
+ }
+ frame.data_mut(1).fill(chroma);
+ frame.data_mut(2).fill(chroma);
+ frame.set_pts(Some(0));
+
+ encoder.send_frame(&frame).unwrap();
+ encoder.send_eof().unwrap();
+ let mut packet = ffmpeg::Packet::empty();
+ encoder.receive_packet(&mut packet).unwrap();
+ packet.data().unwrap().to_vec()
+ }
+
+ fn decode_with_fresh_decoder(data: &[u8]) -> ffmpeg::frame::Video {
+ let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG).unwrap();
+ let mut decoder = ffmpeg::codec::context::Context::new_with_codec(codec)
+ .decoder()
+ .video()
+ .unwrap();
+ decoder.send_packet(&ffmpeg::Packet::copy(data)).unwrap();
+ let mut frame = ffmpeg::frame::Video::empty();
+ decoder.receive_frame(&mut frame).unwrap();
+ frame
+ }
+
+ fn luma_rows(frame: &ffmpeg::frame::Video) -> Vec<&[u8]> {
+ let width = frame.width() as usize;
+ frame
+ .data(0)
+ .chunks(frame.stride(0))
+ .take(frame.height() as usize)
+ .map(|row| &row[..width])
+ .collect()
+ }
+
+ fn assert_matches_fresh_decode(decoded: &ffmpeg::frame::Video, data: &[u8], base_luma: u8) {
+ let expected = decode_with_fresh_decoder(data);
+
+ assert_eq!(decoded.width(), WIDTH);
+ assert_eq!(decoded.height(), HEIGHT);
+ assert_eq!(decoded.format(), expected.format());
+ assert_eq!(decoded.width(), expected.width());
+ assert_eq!(decoded.height(), expected.height());
+ assert_eq!(luma_rows(decoded), luma_rows(&expected));
+
+ let rows = luma_rows(decoded);
+ for (x, y) in [(4, 4), (32, 24), (60, 44)] {
+ let actual = rows[y][x];
+ let source = source_luma(base_luma, x);
+ assert!(
+ actual.abs_diff(source) <= 6,
+ "luma at ({x}, {y}) was {actual}, expected about {source}"
+ );
+ }
+ }
+
+ #[test]
+ fn mjpeg_decoder_reused_across_frames_matches_fresh_decoder() {
+ let frames: Vec<(u8, Vec<u8>)> =
+ [(16u8, 128u8), (60, 90), (100, 160), (140, 40), (30, 200)]
+ .into_iter()
+ .map(|(base_luma, chroma)| (base_luma, encode_mjpeg(base_luma, chroma)))
+ .collect();
+
+ let mut decoder = MjpegDecoder::default();
+ for (base_luma, data) in &frames {
+ let decoded = decoder.decode(data).unwrap();
+ assert_matches_fresh_decode(&decoded, data, *base_luma);
+ }
+ }
+
+ #[test]
+ fn mjpeg_decoder_recovers_after_invalid_input() {
+ let valid = encode_mjpeg(80, 128);
+ let garbage: Vec<u8> = (0..512u32).map(|i| (i * 37 % 251) as u8).collect();
+
+ let mut decoder = MjpegDecoder::default();
+ assert_matches_fresh_decode(&decoder.decode(&valid).unwrap(), &valid, 80);
+
+ assert!(decoder.decode(&garbage).is_err());
+ assert_matches_fresh_decode(&decoder.decode(&valid).unwrap(), &valid, 80);
+
+ assert!(decoder.decode(&[]).is_err());
+ assert_matches_fresh_decode(&decoder.decode(&valid).unwrap(), &valid, 80);
+ }
+}
diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rs
index d5223a339..25ecb5c52 100644
--- a/crates/recording/src/output_pipeline/mod.rs
+++ b/crates/recording/src/output_pipeline/mod.rs
@@ -7,6 +7,8 @@ mod frame_pacing;
mod macos_fragmented_m4s;
#[cfg(target_os = "macos")]
mod macos_frame_convert;
+#[cfg(any(windows, test))]
+mod mjpeg;
#[cfg(target_os = "macos")]
mod oop_fragmented_m4s;
pub mod oop_muxer;
diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs
index cf9003cf2..faab33d6e 100644
--- a/crates/recording/src/output_pipeline/win.rs
+++ b/crates/recording/src/output_pipeline/win.rs
@@ -3,6 +3,7 @@ use crate::{
output_pipeline::{
HealthSender, PipelineHealthEvent, emit_health,
frame_pacing::{FramePacer, Paced},
+ mjpeg::MjpegDecoder,
},
screen_capture,
};
@@ -960,6 +961,7 @@ impl Muxer for WindowsCameraMuxer {
let mut last_frame: Option<NativeCameraFrame> = None;
let mut first_timestamp: Option<Duration> = None;
let mut frame_count = 0u64;
+ let mut mjpeg_decoder = MjpegDecoder::default();
loop {
let ts = match pacer.next(&video_rx, &pause_flag) {
@@ -976,7 +978,7 @@ impl Muxer for WindowsCameraMuxer {
let normalized_ts = normalize_camera_timestamp(ts, &mut first_timestamp);
- let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame) {
+ let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame, &mut mjpeg_decoder) {
Ok(f) => f,
Err(e) => {
error!("Failed to convert camera frame: {e}");
@@ -1345,11 +1347,14 @@ fn convert_uyvy_to_yuyv(
dst
}
-pub fn camera_frame_to_ffmpeg(frame: &NativeCameraFrame) -> anyhow::Result<ffmpeg::frame::Video> {
+pub(crate) fn camera_frame_to_ffmpeg(
+ frame: &NativeCameraFrame,
+ mjpeg_decoder: &mut MjpegDecoder,
+) -> anyhow::Result<ffmpeg::frame::Video> {
use cap_mediafoundation_utils::IMFMediaBufferExt;
if frame.pixel_format == cap_camera_windows::PixelFormat::MJPEG {
- return decode_mjpeg_frame(frame);
+ return decode_mjpeg_frame(frame, mjpeg_decoder);
}
let ffmpeg_format = match frame.pixel_format {
@@ -1625,7 +1630,10 @@ fn copy_plane(
}
}
-fn decode_mjpeg_frame(frame: &NativeCameraFrame) -> anyhow::Result<ffmpeg::frame::Video> {
+fn decode_mjpeg_frame(
+ frame: &NativeCameraFrame,
+ mjpeg_decoder: &mut MjpegDecoder,
+) -> anyhow::Result<ffmpeg::frame::Video> {
use cap_mediafoundation_utils::IMFMediaBufferExt;
let buffer_guard = frame
@@ -1637,26 +1645,7 @@ fn decode_mjpeg_frame(frame: &NativeCameraFrame) -> anyhow::Result<ffmpeg::frame
.map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?;
let data = &*lock;
- let codec = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::MJPEG)
- .ok_or_else(|| anyhow!("MJPEG codec not found"))?;
-
- let decoder_context = ffmpeg::codec::context::Context::new_with_codec(codec);
- let mut decoder = decoder_context
- .decoder()
- .video()
- .map_err(|e| anyhow!("Failed to create MJPEG decoder: {e}"))?;
-
- let packet = ffmpeg::Packet::copy(data);
- decoder
- .send_packet(&packet)
- .map_err(|e| anyhow!("Failed to send MJPEG packet: {e}"))?;
-
- let mut decoded_frame = ffmpeg::frame::Video::empty();
- decoder
- .receive_frame(&mut decoded_frame)
- .map_err(|e| anyhow!("Failed to decode MJPEG frame: {e}"))?;
-
- Ok(decoded_frame)
+ mjpeg_decoder.decode(data)
}
fn flip_buffer_size(
diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs
index 67278a913..b517b5759 100644
--- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs
+++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs
@@ -3,6 +3,7 @@ use super::core::{
combine_finish_errors, wait_for_blocking_thread_finish,
};
use super::frame_pacing::{FramePacer, Paced};
+use super::mjpeg::MjpegDecoder;
use super::win::reference_video_frame;
use crate::{
AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer,
@@ -830,6 +831,7 @@ impl WindowsFragmentedM4SCameraMuxer {
let mut duplicated_frames = 0u64;
let mut disk_monitor = DiskSpaceMonitor::new();
let mut disk_exhausted = false;
+ let mut mjpeg_decoder = MjpegDecoder::default();
const SLOW_THRESHOLD_MS: u128 = 5;
let normalize_timestamp =
@@ -848,10 +850,11 @@ impl WindowsFragmentedM4SCameraMuxer {
slow_convert_count: &mut u32,
slow_encode_count: &mut u32,
total_frames: &mut u64,
- encoder: &Arc<Mutex<SegmentedVideoEncoder>>|
+ encoder: &Arc<Mutex<SegmentedVideoEncoder>>,
+ mjpeg_decoder: &mut MjpegDecoder|
-> anyhow::Result<()> {
let convert_start = std::time::Instant::now();
- let ffmpeg_frame_result = camera_frame_to_ffmpeg(camera_frame);
+ let ffmpeg_frame_result = camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder);
let convert_elapsed_ms = convert_start.elapsed().as_millis();
if convert_elapsed_ms > SLOW_THRESHOLD_MS {
@@ -949,6 +952,7 @@ impl WindowsFragmentedM4SCameraMuxer {
&mut slow_encode_count,
&mut total_frames,
&encoder_clone,
+ &mut mjpeg_decoder,
) {
warn!("Failed to encode drained camera frame: {e}");
break;
@@ -983,6 +987,7 @@ impl WindowsFragmentedM4SCameraMuxer {
&mut slow_encode_count,
&mut total_frames,
&encoder_clone,
+ &mut mjpeg_decoder,
) {
error!("Camera encoder mutex poisoned - encoder thread likely panicked, stopping");
return Err(e);
diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs
index c99bf9605..c377d3388 100644
--- a/crates/recording/src/output_pipeline/win_segmented_camera.rs
+++ b/crates/recording/src/output_pipeline/win_segmented_camera.rs
@@ -375,7 +375,7 @@ impl WindowsSegmentedCameraMuxer {
}
fn create_segment(&mut self, first_frame: &NativeCameraFrame) -> anyhow::Result<()> {
- use crate::output_pipeline::win::camera_frame_to_ffmpeg;
+ use crate::output_pipeline::{mjpeg::MjpegDecoder, win::camera_frame_to_ffmpeg};
let segment_path = self.current_segment_path();
@@ -595,6 +595,7 @@ impl WindowsSegmentedCameraMuxer {
);
let mut first_timestamp: Option<Duration> = None;
+ let mut mjpeg_decoder = MjpegDecoder::default();
while let Ok(Some((frame, timestamp))) = video_rx.recv() {
let relative = if let Some(first) = first_timestamp {
@@ -604,7 +605,7 @@ impl WindowsSegmentedCameraMuxer {
Duration::ZERO
};
- let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame) {
+ let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame, &mut mjpeg_decoder) {
Ok(f) => f,
Err(e) => {
warn!("Failed to convert camera frame: {e}");