Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(desktop-gpui): encode recording thumbnails with FFmpeg's MJPEG encoder
Theophilus200303a00c909d58
diff --git a/apps/desktop-gpui/src/library.rs b/apps/desktop-gpui/src/library.rsindex dc57098a7..29a4757e0 100644--- a/apps/desktop-gpui/src/library.rs+++ b/apps/desktop-gpui/src/library.rs@@ -1143,14 +1143,33 @@ pub fn thumbnail_rgba(path: &Path) -> Option<image::RgbaImage> {/// rather than deferred.////// Faithful in the parts that matter to the output: decode packets until the-/// *first* video frame comes out, scale to RGB24 at the source's own size-/// (`size: None` at both call sites, so a native-resolution JPEG, not a-/// thumbnail), save as JPEG. Blocking; callers hand it to `spawn_blocking`.+/// *first* video frame comes out and save it as a JPEG at the source's own+/// size (`size: None` at every call site, so a native-resolution JPEG, not a+/// thumbnail). Blocking; callers hand it to `spawn_blocking`.pub fn create_screenshot(input: &Path,output: &Path,size: Option<(u32, u32)>,) -> Result<(), String> {+ let frame = first_video_frame(input)?;+ let size = size.unwrap_or((frame.width(), frame.height()));+ let jpeg = match encode_mjpeg(&frame, size) {+ Some(jpeg) => jpeg,+ None => encode_rgb_jpeg(&frame, size)?,+ };+ if let Some(parent) = output.parent() {+ std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;+ }+ let staged = output.with_extension(format!("jpg.tmp-{}", crate::store::new_uuid_v4()));+ std::fs::write(&staged, jpeg)+ .and_then(|()| std::fs::rename(&staged, output))+ .map_err(|e| {+ let _ = std::fs::remove_file(&staged);+ e.to_string()+ })+}++fn first_video_frame(input: &Path) -> Result<ffmpeg::frame::Video, String> {let mut ictx = ffmpeg::format::input(input).map_err(|e| e.to_string())?;let input_stream = ictx.streams()@@ -1164,65 +1183,123 @@ pub fn create_screenshot(.video().map_err(|e| e.to_string())?;- let mut scaler = ffmpeg::software::scaling::context::Context::get(- decoder.format(),- decoder.width(),- decoder.height(),- ffmpeg::format::Pixel::RGB24,- size.map_or(decoder.width(), |s| s.0),- size.map_or(decoder.height(), |s| s.1),- ffmpeg::software::scaling::flag::Flags::BILINEAR,- )- .map_err(|e| e.to_string())?;-let mut frame = ffmpeg::frame::Video::empty();for (stream, packet) in ictx.packets() {if stream.index() != video_stream_index {continue;}decoder.send_packet(&packet).map_err(|e| e.to_string())?;- if decoder.receive_frame(&mut frame).is_err() {- continue;+ if decoder.receive_frame(&mut frame).is_ok() {+ return Ok(frame);}+ }- let mut rgb_frame = ffmpeg::frame::Video::empty();- scaler- .run(&frame, &mut rgb_frame)- .map_err(|e| e.to_string())?;-- // The scaler's rows are padded to its own stride; the image buffer is- // tight, so the copy is row by row.- let width = rgb_frame.width() as usize;- let height = rgb_frame.height() as usize;- let src_stride = rgb_frame.stride(0);- let dst_stride = width * 3;- let buffer = cap_gpui_kernels::frame::copy_rows(- rgb_frame.data(0),- dst_stride,- src_stride,- height,- false,- )- .ok_or("Failed to create image from frame data")?;+ Err("Failed to create screenshot".to_string())+}- let image = image::RgbImage::from_raw(width as u32, height as u32, buffer)- .ok_or("Failed to create image from frame data")?;- if let Some(parent) = output.parent() {- std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;- }- let staged = output.with_extension(format!("jpg.tmp-{}", crate::store::new_uuid_v4()));- cap_gpui_kernels::codec::save_rgb_image(&image, &staged, image::ImageFormat::Jpeg)- .map_err(|e| {- let _ = std::fs::remove_file(&staged);- e.to_string()- })?;- return std::fs::rename(&staged, output).map_err(|e| {- let _ = std::fs::remove_file(&staged);- e.to_string()- });+/// FFmpeg's MJPEG encoder takes the decoded YUV frame directly, so a 4K+/// thumbnail skips the RGB round trip and `image`'s much slower JPEG encoder.+/// 4:4:4 at qscale 4 keeps the unsubsampled layout of the quality-75 JPEG+/// `image` wrote, with at least its quality.+fn encode_mjpeg(frame: &ffmpeg::frame::Video, (width, height): (u32, u32)) -> Option<Vec<u8>> {+ const QSCALE: i32 = 4;+ let format = ffmpeg::format::Pixel::YUV444P;+ let codec = ffmpeg::codec::encoder::find(ffmpeg::codec::Id::MJPEG)?;+ let mut scaler = ffmpeg::software::scaling::context::Context::get(+ frame.format(),+ frame.width(),+ frame.height(),+ format,+ width,+ height,+ ffmpeg::software::scaling::flag::Flags::BILINEAR,+ )+ .ok()?;+ write_full_range(&mut scaler)?;+ let mut yuv = ffmpeg::frame::Video::empty();+ scaler.run(frame, &mut yuv).ok()?;++ let mut encoder = ffmpeg::codec::context::Context::new_with_codec(codec)+ .encoder()+ .video()+ .ok()?;+ encoder.set_width(width);+ encoder.set_height(height);+ encoder.set_format(format);+ encoder.set_color_range(ffmpeg::color::Range::JPEG);+ encoder.set_time_base((1, 1));+ encoder.set_flags(ffmpeg::codec::Flags::QSCALE);+ encoder.set_global_quality(QSCALE * ffmpeg::ffi::FF_QP2LAMBDA);+ encoder.set_qmin(QSCALE);+ encoder.set_qmax(QSCALE);+ let mut encoder = encoder.open().ok()?;++ yuv.set_pts(Some(0));+ encoder.send_frame(&yuv).ok()?;+ encoder.send_eof().ok()?;+ let mut packet = ffmpeg::Packet::empty();+ encoder.receive_packet(&mut packet).ok()?;+ packet.data().map(<[u8]>::to_vec)+}++/// JPEG stores full-range YUV, while swscale writes limited range unless told+/// otherwise; the source keeps whatever range swscale detected for it.+fn write_full_range(scaler: &mut ffmpeg::software::scaling::context::Context) -> Option<()> {+ let mut inv_table = std::ptr::null_mut();+ let mut table = std::ptr::null_mut();+ let (mut src_range, mut dst_range) = (0, 0);+ let (mut brightness, mut contrast, mut saturation) = (0, 0, 0);+ unsafe {+ let context = scaler.as_mut_ptr();+ let read = ffmpeg::ffi::sws_getColorspaceDetails(+ context,+ &mut inv_table,+ &mut src_range,+ &mut table,+ &mut dst_range,+ &mut brightness,+ &mut contrast,+ &mut saturation,+ );+ (read >= 0+ && ffmpeg::ffi::sws_setColorspaceDetails(+ context, inv_table, src_range, table, 1, brightness, contrast, saturation,+ ) >= 0)+ .then_some(())}+}- Err("Failed to create screenshot".to_string())+/// The original path, for an FFmpeg build without the MJPEG encoder.+fn encode_rgb_jpeg(+ frame: &ffmpeg::frame::Video,+ (width, height): (u32, u32),+) -> Result<Vec<u8>, String> {+ let mut rgb_frame = ffmpeg::frame::Video::empty();+ ffmpeg::software::scaling::context::Context::get(+ frame.format(),+ frame.width(),+ frame.height(),+ ffmpeg::format::Pixel::RGB24,+ width,+ height,+ ffmpeg::software::scaling::flag::Flags::BILINEAR,+ )+ .map_err(|e| e.to_string())?+ .run(frame, &mut rgb_frame)+ .map_err(|e| e.to_string())?;++ // The scaler's rows are padded to its own stride; the image buffer is+ // tight, so the copy is row by row.+ let row_bytes = width as usize * 3;+ let buffer = cap_gpui_kernels::frame::copy_rows(+ rgb_frame.data(0),+ row_bytes,+ rgb_frame.stride(0),+ height as usize,+ false,+ )+ .ok_or("Failed to create image from frame data")?;+ cap_gpui_kernels::codec::encode_jpeg_rgb(&buffer, width, height, 75).map_err(|e| e.to_string())}pub fn bundle_thumbnail_path(project_dir: &Path) -> PathBuf {@@ -1293,6 +1370,148 @@ mod tests {bundle}+ fn psnr(jpeg: &[u8], screen: &image::RgbImage) -> f64 {+ let decoded = image::load_from_memory(jpeg).unwrap().into_rgb8();+ assert_eq!(decoded.dimensions(), screen.dimensions());+ let squared: f64 = decoded+ .as_raw()+ .iter()+ .zip(screen.as_raw())+ .map(|(&a, &b)| (f64::from(a) - f64::from(b)).powi(2))+ .sum();+ 10.0 * (255.0_f64.powi(2) * screen.as_raw().len() as f64 / squared).log10()+ }++ /// `screen` as a recording would hand it over: limited-range 4:2:0.+ fn recorded_frame(screen: &image::RgbImage) -> ffmpeg::frame::Video {+ let (width, height) = screen.dimensions();+ let mut rgb = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::RGB24, width, height);+ let stride = rgb.stride(0);+ for (row, pixels) in rgb+ .data_mut(0)+ .chunks_mut(stride)+ .zip(screen.as_raw().chunks(width as usize * 3))+ {+ row[..pixels.len()].copy_from_slice(pixels);+ }+ let mut frame = ffmpeg::frame::Video::empty();+ ffmpeg::software::scaling::context::Context::get(+ ffmpeg::format::Pixel::RGB24,+ width,+ height,+ ffmpeg::format::Pixel::YUV420P,+ width,+ height,+ ffmpeg::software::scaling::flag::Flags::BILINEAR,+ )+ .unwrap()+ .run(&rgb, &mut frame)+ .unwrap();+ frame+ }++ #[test]+ fn screenshot_is_written_with_the_mjpeg_encoder() {+ ffmpeg::init().unwrap();+ let root = temp_dir("mjpeg-screenshot");+ let source = root.join("display.mp4");+ std::fs::write(+ &source,+ include_bytes!("../../media-server/src/__tests__/fixtures/test-with-audio.mp4"),+ )+ .unwrap();+ let output = bundle_thumbnail_path(&root);++ create_screenshot(&source, &output, None).unwrap();++ let frame = first_video_frame(&source).unwrap();+ let written = std::fs::read(&output).unwrap();+ let size = (frame.width(), frame.height());+ assert_eq!(written, encode_mjpeg(&frame, size).unwrap());+ let decoded = image::load_from_memory_with_format(&written, image::ImageFormat::Jpeg);+ assert_eq!(decoded.unwrap().to_rgb8().dimensions(), size);+ std::fs::remove_dir_all(root).unwrap();+ }++ #[test]+ fn mjpeg_screenshot_is_at_least_as_close_to_the_screen() {+ ffmpeg::init().unwrap();+ let screen = image::RgbImage::from_fn(640, 360, |x, y| {+ let text = (x / 2 + y / 3) % 7 == 0 && y % 24 < 14;+ match (x / 160, text) {+ (0, _) => image::Rgb([61, 106, 242]),+ (_, true) => image::Rgb([212, 212, 212]),+ (3, false) => image::Rgb([(x - 480) as u8, 53, (y / 2) as u8 + 60]),+ _ => image::Rgb([30, 31, 34]),+ }+ });+ let frame = recorded_frame(&screen);+ let size = screen.dimensions();++ let mjpeg = psnr(&encode_mjpeg(&frame, size).unwrap(), &screen);+ let rgb = psnr(&encode_rgb_jpeg(&frame, size).unwrap(), &screen);++ assert!(mjpeg >= rgb, "mjpeg {mjpeg:.2} dB, rgb {rgb:.2} dB");+ }++ /// Each source is a video; a `.png` beside it with the same name is the+ /// screen it recorded, which both JPEGs are scored against.+ #[test]+ #[ignore = "needs CAP_SCREENSHOT_BENCH_SOURCES"]+ fn benchmark_create_screenshot() {+ ffmpeg::init().unwrap();+ let sources = std::env::var("CAP_SCREENSHOT_BENCH_SOURCES").unwrap();+ let median = |mut runs: Vec<f64>| {+ runs.sort_by(f64::total_cmp);+ runs[runs.len() / 2]+ };+ println!(+ "source,width,height,decode_ms,rgb_ms,mjpeg_ms,rgb_bytes,mjpeg_bytes,rgb_psnr,mjpeg_psnr"+ );+ for source in sources.split(',').map(Path::new) {+ let started = std::time::Instant::now();+ let frame = first_video_frame(source).unwrap();+ let decode = started.elapsed().as_secs_f64() * 1e3;+ let size = (frame.width(), frame.height());+ let rgb = || encode_rgb_jpeg(&frame, size).unwrap();+ let mjpeg = || encode_mjpeg(&frame, size).unwrap();+ let time = |encode: &dyn Fn() -> Vec<u8>| {+ let started = std::time::Instant::now();+ std::hint::black_box(encode());+ started.elapsed().as_secs_f64() * 1e3+ };+ let (mut rgb_runs, mut mjpeg_runs) = (Vec::new(), Vec::new());+ for run in 0..15 {+ if run % 2 == 0 {+ rgb_runs.push(time(&rgb));+ mjpeg_runs.push(time(&mjpeg));+ } else {+ mjpeg_runs.push(time(&mjpeg));+ rgb_runs.push(time(&rgb));+ }+ }+ let (rgb, mjpeg) = (rgb(), mjpeg());+ let screen = image::open(source.with_extension("png")).map(|png| png.into_rgb8());+ let score = |jpeg: &[u8]| {+ screen+ .as_ref()+ .map_or(String::new(), |screen| format!("{:.2}", psnr(jpeg, screen)))+ };+ println!(+ "{},{},{},{decode:.1},{:.1},{:.1},{},{},{},{}",+ source.display(),+ size.0,+ size.1,+ median(rgb_runs),+ median(mjpeg_runs),+ rgb.len(),+ mjpeg.len(),+ score(&rgb),+ score(&mjpeg),+ );+ }+ }+fn temp_dir(tag: &str) -> PathBuf {let dir = std::env::temp_dir().join(format!("cap-gpui-library-{tag}-{}-{:?}",