Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(desktop-gpui): encode recording thumbnails with FFmpeg's MJPEG encoder

Theophilus200303a00c909d58
diff --git a/apps/desktop-gpui/src/library.rs b/apps/desktop-gpui/src/library.rs
index dc57098a7..29a4757e0 100644
--- a/apps/desktop-gpui/src/library.rs
+++ b/apps/desktop-gpui/src/library.rs
@@ -1143,14 +1143,33 @@ pub fn thumbnail_rgba(path: &Path) -> Option<image::RgbaImage> {
/// rather than deferred.
///
/// Faithful in the parts that matter to the output: decode packets until the
-/// *first* video frame comes out, scale to RGB24 at the source's own size
-/// (`size: None` at both call sites, so a native-resolution JPEG, not a
-/// thumbnail), save as JPEG. Blocking; callers hand it to `spawn_blocking`.
+/// *first* video frame comes out and save it as a JPEG at the source's own
+/// size (`size: None` at every call site, so a native-resolution JPEG, not a
+/// thumbnail). Blocking; callers hand it to `spawn_blocking`.
pub fn create_screenshot(
input: &Path,
output: &Path,
size: Option<(u32, u32)>,
) -> Result<(), String> {
+ let frame = first_video_frame(input)?;
+ let size = size.unwrap_or((frame.width(), frame.height()));
+ let jpeg = match encode_mjpeg(&frame, size) {
+ Some(jpeg) => jpeg,
+ None => encode_rgb_jpeg(&frame, size)?,
+ };
+ if let Some(parent) = output.parent() {
+ std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
+ }
+ let staged = output.with_extension(format!("jpg.tmp-{}", crate::store::new_uuid_v4()));
+ std::fs::write(&staged, jpeg)
+ .and_then(|()| std::fs::rename(&staged, output))
+ .map_err(|e| {
+ let _ = std::fs::remove_file(&staged);
+ e.to_string()
+ })
+}
+
+fn first_video_frame(input: &Path) -> Result<ffmpeg::frame::Video, String> {
let mut ictx = ffmpeg::format::input(input).map_err(|e| e.to_string())?;
let input_stream = ictx
.streams()
@@ -1164,65 +1183,123 @@ pub fn create_screenshot(
.video()
.map_err(|e| e.to_string())?;
- let mut scaler = ffmpeg::software::scaling::context::Context::get(
- decoder.format(),
- decoder.width(),
- decoder.height(),
- ffmpeg::format::Pixel::RGB24,
- size.map_or(decoder.width(), |s| s.0),
- size.map_or(decoder.height(), |s| s.1),
- ffmpeg::software::scaling::flag::Flags::BILINEAR,
- )
- .map_err(|e| e.to_string())?;
-
let mut frame = ffmpeg::frame::Video::empty();
for (stream, packet) in ictx.packets() {
if stream.index() != video_stream_index {
continue;
}
decoder.send_packet(&packet).map_err(|e| e.to_string())?;
- if decoder.receive_frame(&mut frame).is_err() {
- continue;
+ if decoder.receive_frame(&mut frame).is_ok() {
+ return Ok(frame);
}
+ }
- let mut rgb_frame = ffmpeg::frame::Video::empty();
- scaler
- .run(&frame, &mut rgb_frame)
- .map_err(|e| e.to_string())?;
-
- // The scaler's rows are padded to its own stride; the image buffer is
- // tight, so the copy is row by row.
- let width = rgb_frame.width() as usize;
- let height = rgb_frame.height() as usize;
- let src_stride = rgb_frame.stride(0);
- let dst_stride = width * 3;
- let buffer = cap_gpui_kernels::frame::copy_rows(
- rgb_frame.data(0),
- dst_stride,
- src_stride,
- height,
- false,
- )
- .ok_or("Failed to create image from frame data")?;
+ Err("Failed to create screenshot".to_string())
+}
- let image = image::RgbImage::from_raw(width as u32, height as u32, buffer)
- .ok_or("Failed to create image from frame data")?;
- if let Some(parent) = output.parent() {
- std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
- }
- let staged = output.with_extension(format!("jpg.tmp-{}", crate::store::new_uuid_v4()));
- cap_gpui_kernels::codec::save_rgb_image(&image, &staged, image::ImageFormat::Jpeg)
- .map_err(|e| {
- let _ = std::fs::remove_file(&staged);
- e.to_string()
- })?;
- return std::fs::rename(&staged, output).map_err(|e| {
- let _ = std::fs::remove_file(&staged);
- e.to_string()
- });
+/// FFmpeg's MJPEG encoder takes the decoded YUV frame directly, so a 4K
+/// thumbnail skips the RGB round trip and `image`'s much slower JPEG encoder.
+/// 4:4:4 at qscale 4 keeps the unsubsampled layout of the quality-75 JPEG
+/// `image` wrote, with at least its quality.
+fn encode_mjpeg(frame: &ffmpeg::frame::Video, (width, height): (u32, u32)) -> Option<Vec<u8>> {
+ const QSCALE: i32 = 4;
+ let format = ffmpeg::format::Pixel::YUV444P;
+ let codec = ffmpeg::codec::encoder::find(ffmpeg::codec::Id::MJPEG)?;
+ let mut scaler = ffmpeg::software::scaling::context::Context::get(
+ frame.format(),
+ frame.width(),
+ frame.height(),
+ format,
+ width,
+ height,
+ ffmpeg::software::scaling::flag::Flags::BILINEAR,
+ )
+ .ok()?;
+ write_full_range(&mut scaler)?;
+ let mut yuv = ffmpeg::frame::Video::empty();
+ scaler.run(frame, &mut yuv).ok()?;
+
+ let mut encoder = ffmpeg::codec::context::Context::new_with_codec(codec)
+ .encoder()
+ .video()
+ .ok()?;
+ encoder.set_width(width);
+ encoder.set_height(height);
+ encoder.set_format(format);
+ encoder.set_color_range(ffmpeg::color::Range::JPEG);
+ encoder.set_time_base((1, 1));
+ encoder.set_flags(ffmpeg::codec::Flags::QSCALE);
+ encoder.set_global_quality(QSCALE * ffmpeg::ffi::FF_QP2LAMBDA);
+ encoder.set_qmin(QSCALE);
+ encoder.set_qmax(QSCALE);
+ let mut encoder = encoder.open().ok()?;
+
+ yuv.set_pts(Some(0));
+ encoder.send_frame(&yuv).ok()?;
+ encoder.send_eof().ok()?;
+ let mut packet = ffmpeg::Packet::empty();
+ encoder.receive_packet(&mut packet).ok()?;
+ packet.data().map(<[u8]>::to_vec)
+}
+
+/// JPEG stores full-range YUV, while swscale writes limited range unless told
+/// otherwise; the source keeps whatever range swscale detected for it.
+fn write_full_range(scaler: &mut ffmpeg::software::scaling::context::Context) -> Option<()> {
+ let mut inv_table = std::ptr::null_mut();
+ let mut table = std::ptr::null_mut();
+ let (mut src_range, mut dst_range) = (0, 0);
+ let (mut brightness, mut contrast, mut saturation) = (0, 0, 0);
+ unsafe {
+ let context = scaler.as_mut_ptr();
+ let read = ffmpeg::ffi::sws_getColorspaceDetails(
+ context,
+ &mut inv_table,
+ &mut src_range,
+ &mut table,
+ &mut dst_range,
+ &mut brightness,
+ &mut contrast,
+ &mut saturation,
+ );
+ (read >= 0
+ && ffmpeg::ffi::sws_setColorspaceDetails(
+ context, inv_table, src_range, table, 1, brightness, contrast, saturation,
+ ) >= 0)
+ .then_some(())
}
+}
- Err("Failed to create screenshot".to_string())
+/// The original path, for an FFmpeg build without the MJPEG encoder.
+fn encode_rgb_jpeg(
+ frame: &ffmpeg::frame::Video,
+ (width, height): (u32, u32),
+) -> Result<Vec<u8>, String> {
+ let mut rgb_frame = ffmpeg::frame::Video::empty();
+ ffmpeg::software::scaling::context::Context::get(
+ frame.format(),
+ frame.width(),
+ frame.height(),
+ ffmpeg::format::Pixel::RGB24,
+ width,
+ height,
+ ffmpeg::software::scaling::flag::Flags::BILINEAR,
+ )
+ .map_err(|e| e.to_string())?
+ .run(frame, &mut rgb_frame)
+ .map_err(|e| e.to_string())?;
+
+ // The scaler's rows are padded to its own stride; the image buffer is
+ // tight, so the copy is row by row.
+ let row_bytes = width as usize * 3;
+ let buffer = cap_gpui_kernels::frame::copy_rows(
+ rgb_frame.data(0),
+ row_bytes,
+ rgb_frame.stride(0),
+ height as usize,
+ false,
+ )
+ .ok_or("Failed to create image from frame data")?;
+ cap_gpui_kernels::codec::encode_jpeg_rgb(&buffer, width, height, 75).map_err(|e| e.to_string())
}
pub fn bundle_thumbnail_path(project_dir: &Path) -> PathBuf {
@@ -1293,6 +1370,148 @@ mod tests {
bundle
}
+ fn psnr(jpeg: &[u8], screen: &image::RgbImage) -> f64 {
+ let decoded = image::load_from_memory(jpeg).unwrap().into_rgb8();
+ assert_eq!(decoded.dimensions(), screen.dimensions());
+ let squared: f64 = decoded
+ .as_raw()
+ .iter()
+ .zip(screen.as_raw())
+ .map(|(&a, &b)| (f64::from(a) - f64::from(b)).powi(2))
+ .sum();
+ 10.0 * (255.0_f64.powi(2) * screen.as_raw().len() as f64 / squared).log10()
+ }
+
+ /// `screen` as a recording would hand it over: limited-range 4:2:0.
+ fn recorded_frame(screen: &image::RgbImage) -> ffmpeg::frame::Video {
+ let (width, height) = screen.dimensions();
+ let mut rgb = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::RGB24, width, height);
+ let stride = rgb.stride(0);
+ for (row, pixels) in rgb
+ .data_mut(0)
+ .chunks_mut(stride)
+ .zip(screen.as_raw().chunks(width as usize * 3))
+ {
+ row[..pixels.len()].copy_from_slice(pixels);
+ }
+ let mut frame = ffmpeg::frame::Video::empty();
+ ffmpeg::software::scaling::context::Context::get(
+ ffmpeg::format::Pixel::RGB24,
+ width,
+ height,
+ ffmpeg::format::Pixel::YUV420P,
+ width,
+ height,
+ ffmpeg::software::scaling::flag::Flags::BILINEAR,
+ )
+ .unwrap()
+ .run(&rgb, &mut frame)
+ .unwrap();
+ frame
+ }
+
+ #[test]
+ fn screenshot_is_written_with_the_mjpeg_encoder() {
+ ffmpeg::init().unwrap();
+ let root = temp_dir("mjpeg-screenshot");
+ let source = root.join("display.mp4");
+ std::fs::write(
+ &source,
+ include_bytes!("../../media-server/src/__tests__/fixtures/test-with-audio.mp4"),
+ )
+ .unwrap();
+ let output = bundle_thumbnail_path(&root);
+
+ create_screenshot(&source, &output, None).unwrap();
+
+ let frame = first_video_frame(&source).unwrap();
+ let written = std::fs::read(&output).unwrap();
+ let size = (frame.width(), frame.height());
+ assert_eq!(written, encode_mjpeg(&frame, size).unwrap());
+ let decoded = image::load_from_memory_with_format(&written, image::ImageFormat::Jpeg);
+ assert_eq!(decoded.unwrap().to_rgb8().dimensions(), size);
+ std::fs::remove_dir_all(root).unwrap();
+ }
+
+ #[test]
+ fn mjpeg_screenshot_is_at_least_as_close_to_the_screen() {
+ ffmpeg::init().unwrap();
+ let screen = image::RgbImage::from_fn(640, 360, |x, y| {
+ let text = (x / 2 + y / 3) % 7 == 0 && y % 24 < 14;
+ match (x / 160, text) {
+ (0, _) => image::Rgb([61, 106, 242]),
+ (_, true) => image::Rgb([212, 212, 212]),
+ (3, false) => image::Rgb([(x - 480) as u8, 53, (y / 2) as u8 + 60]),
+ _ => image::Rgb([30, 31, 34]),
+ }
+ });
+ let frame = recorded_frame(&screen);
+ let size = screen.dimensions();
+
+ let mjpeg = psnr(&encode_mjpeg(&frame, size).unwrap(), &screen);
+ let rgb = psnr(&encode_rgb_jpeg(&frame, size).unwrap(), &screen);
+
+ assert!(mjpeg >= rgb, "mjpeg {mjpeg:.2} dB, rgb {rgb:.2} dB");
+ }
+
+ /// Each source is a video; a `.png` beside it with the same name is the
+ /// screen it recorded, which both JPEGs are scored against.
+ #[test]
+ #[ignore = "needs CAP_SCREENSHOT_BENCH_SOURCES"]
+ fn benchmark_create_screenshot() {
+ ffmpeg::init().unwrap();
+ let sources = std::env::var("CAP_SCREENSHOT_BENCH_SOURCES").unwrap();
+ let median = |mut runs: Vec<f64>| {
+ runs.sort_by(f64::total_cmp);
+ runs[runs.len() / 2]
+ };
+ println!(
+ "source,width,height,decode_ms,rgb_ms,mjpeg_ms,rgb_bytes,mjpeg_bytes,rgb_psnr,mjpeg_psnr"
+ );
+ for source in sources.split(',').map(Path::new) {
+ let started = std::time::Instant::now();
+ let frame = first_video_frame(source).unwrap();
+ let decode = started.elapsed().as_secs_f64() * 1e3;
+ let size = (frame.width(), frame.height());
+ let rgb = || encode_rgb_jpeg(&frame, size).unwrap();
+ let mjpeg = || encode_mjpeg(&frame, size).unwrap();
+ let time = |encode: &dyn Fn() -> Vec<u8>| {
+ let started = std::time::Instant::now();
+ std::hint::black_box(encode());
+ started.elapsed().as_secs_f64() * 1e3
+ };
+ let (mut rgb_runs, mut mjpeg_runs) = (Vec::new(), Vec::new());
+ for run in 0..15 {
+ if run % 2 == 0 {
+ rgb_runs.push(time(&rgb));
+ mjpeg_runs.push(time(&mjpeg));
+ } else {
+ mjpeg_runs.push(time(&mjpeg));
+ rgb_runs.push(time(&rgb));
+ }
+ }
+ let (rgb, mjpeg) = (rgb(), mjpeg());
+ let screen = image::open(source.with_extension("png")).map(|png| png.into_rgb8());
+ let score = |jpeg: &[u8]| {
+ screen
+ .as_ref()
+ .map_or(String::new(), |screen| format!("{:.2}", psnr(jpeg, screen)))
+ };
+ println!(
+ "{},{},{},{decode:.1},{:.1},{:.1},{},{},{},{}",
+ source.display(),
+ size.0,
+ size.1,
+ median(rgb_runs),
+ median(mjpeg_runs),
+ rgb.len(),
+ mjpeg.len(),
+ score(&rgb),
+ score(&mjpeg),
+ );
+ }
+ }
+
fn temp_dir(tag: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"cap-gpui-library-{tag}-{}-{:?}",