Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

Merge remote-tracking branch 'origin/main' into perf/mf-decoder-state-machine

Marcos Daniel Rivera Gonzalez67cdf7bed8b7
diff --cc crates/rendering/src/decoder/media_foundation.rs
index 09c2f21d6,894d37cbe..562754e44
--- a/crates/rendering/src/decoder/media_foundation.rs
+++ b/crates/rendering/src/decoder/media_foundation.rs
@@@ -229,7 -229,11 +229,8 @@@ impl MFDecoder
let video_width = decoder.width();
let video_height = decoder.height();
+ let matrix = super::probe_yuv_matrix(&path);
- let mut cache = BTreeMap::<u32, CachedFrame>::new();
- let mut cache_bytes = 0usize;
- let mut last_decoded_frame: Option<u32> = None;
let mut health = DecoderHealthMonitor::new();
let init_result = DecoderInitResult {
@@@ -239,61 -243,38 +240,61 @@@
};
let _ = ready_tx.send(Ok(init_result));
- struct PendingRequest {
- frame: u32,
- time: f32,
- sender: oneshot::Sender<DecodedFrame>,
- }
-
- while let Ok(r) = rx.recv() {
- let mut pending_requests: Vec<PendingRequest> = Vec::with_capacity(8);
+ let pixel_count = (video_width as u64) * (video_height as u64);
+ let readahead_frames = if pixel_count > 4_000_000 {
+ 8
+ } else if pixel_count > 2_000_000 {
+ 15
+ } else {
+ 30
+ };
+ let mut requests = FrameRequests::new(readahead_frames);
+ let answer = |sender: oneshot::Sender<DecodedFrame>, frame: Option<CachedFrame>| {
+ let _ = sender.send(frame.map_or_else(
+ || {
+ DecodedFrame::new(
+ vec![0u8; (video_width * video_height * 4) as usize],
+ video_width,
+ video_height,
+ )
+ },
- |frame| frame.to_decoded_frame(),
++ |frame| frame.to_decoded_frame().with_matrix(matrix),
+ ));
+ };
- let mut push_request =
- |requested_time: f32, sender: oneshot::Sender<DecodedFrame>| {
- if sender.is_closed() {
- return;
- }
- let frame = (requested_time * fps as f32).floor() as u32;
- pending_requests.push(PendingRequest {
- frame,
- time: requested_time,
- sender,
- });
+ loop {
+ loop {
+ let message = match requests.next_step() {
+ Step::Wait => rx.recv().ok(),
+ _ => match rx.try_recv() {
+ Err(mpsc::TryRecvError::Empty) => break,
+ message => message.ok(),
+ },
};
-
- match r {
- VideoDecoderMessage::GetFrame(requested_time, _, sender) => {
- push_request(requested_time, sender);
+ // Each request borrows a decoder handle, which owns the sender, so once the
+ // channel is gone nobody is waiting for an answer.
+ let Some(message) = message else {
+ return;
+ };
+ let VideoDecoderMessage::GetFrame(time, _, sender) = message;
+ if sender.is_closed() {
+ continue;
}
- }
- while let Ok(msg) = rx.try_recv() {
- match msg {
- VideoDecoderMessage::GetFrame(requested_time, _, sender) => {
- push_request(requested_time, sender);
+ let frame = (time * fps as f32).floor() as u32;
+ match requests.request(frame, sender) {
+ Some((sender, cached)) => answer(sender, cached),
+ None => {
+ health.record_request();
+ if health.should_warn_unhealthy() {
+ warn!(
+ name = name,
+ consecutive_errors = health.consecutive_errors,
+ texture_failures = health.consecutive_texture_read_failures,
+ total_decoded = health.total_frames_decoded,
+ "MediaFoundation decoder unhealthy, performance may degrade"
+ );
+ }
}
}
}
diff --cc crates/rendering/src/yuv_converter.rs
index 4bd420ddb,edac1b916..d14006320
--- a/crates/rendering/src/yuv_converter.rs
+++ b/crates/rendering/src/yuv_converter.rs
@@@ -1686,3 -1776,241 +1776,241 @@@ impl YuvToRgbaConverter
Some(self.current_output_texture())
}
}
+
+ /// The GPU conversions against the CPU reference in `cpu_yuv`: the export's
+ /// RGBA to NV12 shader must produce BT.709 limited range (the matrix the
+ /// encoders tag), and the decode shaders must apply the matrix they are asked
+ /// for. Skipped where no adapter exists.
+ #[cfg(test)]
+ mod colour_matrix_tests {
+ use super::*;
+ use crate::frame_pipeline::RgbaToNv12Converter;
+
+ // Primaries, secondaries, grey axis and the suite's red background.
+ const CHART: [[u8; 3]; 8] = [
+ [0, 0, 0],
+ [255, 255, 255],
+ [128, 128, 128],
+ [255, 0, 0],
+ [0, 255, 0],
+ [0, 0, 255],
+ [255, 255, 0],
+ [220, 40, 40],
+ ];
+ // Each chart colour fills a 4x2 block, so chroma averaging is exact.
+ const WIDTH: u32 = 4 * CHART.len() as u32;
+ const HEIGHT: u32 = 2;
+
+ fn device() -> Option<(wgpu::Device, wgpu::Queue)> {
+ let instance = crate::create_wgpu_instance_sync();
+ let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
+ power_preference: wgpu::PowerPreference::LowPower,
+ force_fallback_adapter: false,
+ compatible_surface: None,
+ }))
+ .ok()?;
+ pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()
+ }
+
+ fn reference_bt709(rgb: [u8; 3]) -> [f64; 3] {
+ let (kr, kb) = (0.2126, 0.0722);
+ let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0);
+ let y = kr * r + (1.0 - kr - kb) * g + kb * b;
+ [
+ 16.0 + 219.0 * y,
+ 128.0 + 224.0 * (b - y) / (2.0 * (1.0 - kb)),
+ 128.0 + 224.0 * (r - y) / (2.0 * (1.0 - kr)),
+ ]
+ }
+
+ fn chart_texture(device: &wgpu::Device, queue: &wgpu::Queue) -> wgpu::Texture {
+ let texture = device.create_texture(&wgpu::TextureDescriptor {
+ label: Some("colour chart"),
+ size: wgpu::Extent3d {
+ width: WIDTH,
+ height: HEIGHT,
+ depth_or_array_layers: 1,
+ },
+ mip_level_count: 1,
+ sample_count: 1,
+ dimension: wgpu::TextureDimension::D2,
+ format: wgpu::TextureFormat::Rgba8Unorm,
+ usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+ view_formats: &[],
+ });
+ let data: Vec<u8> = (0..HEIGHT)
+ .flat_map(|_| {
+ CHART
+ .iter()
+ .flat_map(|[r, g, b]| [*r, *g, *b, 255].repeat(4))
+ })
+ .collect();
+ queue.write_texture(
+ texture.as_image_copy(),
+ &data,
+ wgpu::TexelCopyBufferLayout {
+ offset: 0,
+ bytes_per_row: Some(WIDTH * 4),
+ rows_per_image: None,
+ },
+ wgpu::Extent3d {
+ width: WIDTH,
+ height: HEIGHT,
+ depth_or_array_layers: 1,
+ },
+ );
+ texture
+ }
+
+ /// The chart as BT.709 NV12 from the CPU encoder (tight strides).
+ fn chart_nv12() -> (Vec<u8>, Vec<u8>) {
+ let mut y = Vec::new();
+ let mut uv = Vec::new();
+ for _ in 0..HEIGHT {
+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {
+ y.extend([cpu_yuv::bt709_luma(r, g, b); 4]);
+ }
+ }
+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {
+ let [cb, cr] = cpu_yuv::bt709_chroma(r, g, b);
+ uv.extend([cb, cr, cb, cr]);
+ }
+ (y, uv)
+ }
+
+ fn read_rgba(device: &wgpu::Device, queue: &wgpu::Queue, texture: &wgpu::Texture) -> Vec<u8> {
+ let padded = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
+ let buffer = device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("chart readback"),
+ size: u64::from(padded * HEIGHT),
+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
+ mapped_at_creation: false,
+ });
+ let mut encoder = device.create_command_encoder(&Default::default());
+ encoder.copy_texture_to_buffer(
+ texture.as_image_copy(),
+ wgpu::TexelCopyBufferInfo {
+ buffer: &buffer,
+ layout: wgpu::TexelCopyBufferLayout {
+ offset: 0,
+ bytes_per_row: Some(padded),
+ rows_per_image: None,
+ },
+ },
+ wgpu::Extent3d {
+ width: WIDTH,
+ height: HEIGHT,
+ depth_or_array_layers: 1,
+ },
+ );
+ queue.submit(std::iter::once(encoder.finish()));
+ buffer.slice(..).map_async(wgpu::MapMode::Read, |_| {});
+ device.poll(wgpu::PollType::Wait).expect("poll");
+ let mapped = buffer.slice(..).get_mapped_range();
+ (0..HEIGHT as usize)
+ .flat_map(|row| {
+ let start = row * padded as usize;
+ mapped[start..start + WIDTH as usize * 4].to_vec()
+ })
+ .collect()
+ }
+
+ #[tokio::test]
+ async fn export_shader_encodes_bt709_limited_range() {
+ let Some((device, queue)) = device() else {
+ eprintln!("no GPU adapter available, skipping");
+ return;
+ };
+ let source = chart_texture(&device, &queue);
+ let mut converter = RgbaToNv12Converter::new(&device);
+ let mut encoder = device.create_command_encoder(&Default::default());
+ assert!(
+ converter
+ .submit_conversion(&device, &queue, &mut encoder, &source, WIDTH, HEIGHT, 0, 30)
+ .await
+ .expect("conversion submission")
+ );
- queue.submit(std::iter::once(encoder.finish()));
- converter.after_submit(&queue);
++ let submission = queue.submit(std::iter::once(encoder.finish()));
++ converter.after_submit(&queue, &submission);
+ let frame = converter
+ .take_pending()
+ .expect("conversion pending")
+ .wait_with_pool(&device, None)
+ .await
+ .expect("conversion completes");
+ let stride = frame.y_stride as usize;
+ let uv_base = stride * HEIGHT as usize;
+ for (i, rgb) in CHART.iter().enumerate() {
+ let [y, cb, cr] = reference_bt709(*rgb);
+ let x = i * 4;
+ let actual = [
+ frame.data[x],
+ frame.data[uv_base + x],
+ frame.data[uv_base + x + 1],
+ ];
+ for (a, e) in actual.iter().zip([y, cb, cr]) {
+ assert!(
+ (f64::from(*a) - e).abs() <= 0.5 + 1e-3,
+ "{rgb:?}: got {actual:?}, BT.709 reference {y:.2} {cb:.2} {cr:.2}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn decode_shaders_apply_the_requested_matrix() {
+ let Some((device, queue)) = device() else {
+ eprintln!("no GPU adapter available, skipping");
+ return;
+ };
+ let (y, uv) = chart_nv12();
+ // The same chroma as separate U and V planes for the planar shader.
+ let u: Vec<u8> = uv.iter().step_by(2).copied().collect();
+ let v: Vec<u8> = uv.iter().skip(1).step_by(2).copied().collect();
+ let mut converter = YuvToRgbaConverter::new(&device);
+ for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {
+ converter.set_matrix(matrix);
+ for planar in [false, true] {
+ if planar {
+ converter
+ .convert_yuv420p(
+ &device,
+ &queue,
+ &y,
+ &u,
+ &v,
+ WIDTH,
+ HEIGHT,
+ WIDTH,
+ WIDTH / 2,
+ )
+ .expect("convert YUV420P");
+ } else {
+ converter
+ .convert_nv12(&device, &queue, &y, &uv, WIDTH, HEIGHT, WIDTH, WIDTH)
+ .expect("convert NV12");
+ }
+ let rgba = read_rgba(&device, &queue, converter.output_texture().unwrap());
+ for (i, rgb) in CHART.iter().enumerate() {
+ let x = i * 4;
+ let expected = matrix.ycbcr_to_rgb(
+ i32::from(y[x]) - 16,
+ i32::from(uv[x]) - 128,
+ i32::from(uv[x + 1]) - 128,
+ );
+ let actual = &rgba[x * 4..x * 4 + 3];
+ assert!(
+ actual.iter().zip(expected).all(|(a, e)| a.abs_diff(e) <= 1),
+ "{matrix:?} planar={planar} {rgb:?}: GPU {actual:?}, CPU {expected:?}"
+ );
+ if matrix == YuvMatrix::Bt709 {
+ assert!(
+ actual.iter().zip(rgb).all(|(a, e)| a.abs_diff(*e) <= 2),
+ "planar={planar}: BT.709 round trip of {rgb:?} gave {actual:?}"
+ );
+ }
+ }
+ }
+ }
+ }
+ }