Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
Merge remote-tracking branch 'origin/main' into perf/mf-decoder-state-machine
Marcos Daniel Rivera Gonzalez67cdf7bed8b7
diff --cc crates/rendering/src/decoder/media_foundation.rsindex 09c2f21d6,894d37cbe..562754e44--- a/crates/rendering/src/decoder/media_foundation.rs+++ b/crates/rendering/src/decoder/media_foundation.rs@@@ -229,7 -229,11 +229,8 @@@ impl MFDecoderlet video_width = decoder.width();let video_height = decoder.height();+ let matrix = super::probe_yuv_matrix(&path);- let mut cache = BTreeMap::<u32, CachedFrame>::new();- let mut cache_bytes = 0usize;- let mut last_decoded_frame: Option<u32> = None;let mut health = DecoderHealthMonitor::new();let init_result = DecoderInitResult {@@@ -239,61 -243,38 +240,61 @@@};let _ = ready_tx.send(Ok(init_result));- struct PendingRequest {- frame: u32,- time: f32,- sender: oneshot::Sender<DecodedFrame>,- }-- while let Ok(r) = rx.recv() {- let mut pending_requests: Vec<PendingRequest> = Vec::with_capacity(8);+ let pixel_count = (video_width as u64) * (video_height as u64);+ let readahead_frames = if pixel_count > 4_000_000 {+ 8+ } else if pixel_count > 2_000_000 {+ 15+ } else {+ 30+ };+ let mut requests = FrameRequests::new(readahead_frames);+ let answer = |sender: oneshot::Sender<DecodedFrame>, frame: Option<CachedFrame>| {+ let _ = sender.send(frame.map_or_else(+ || {+ DecodedFrame::new(+ vec![0u8; (video_width * video_height * 4) as usize],+ video_width,+ video_height,+ )+ },- |frame| frame.to_decoded_frame(),++ |frame| frame.to_decoded_frame().with_matrix(matrix),+ ));+ };- let mut push_request =- |requested_time: f32, sender: oneshot::Sender<DecodedFrame>| {- if sender.is_closed() {- return;- }- let frame = (requested_time * fps as f32).floor() as u32;- pending_requests.push(PendingRequest {- frame,- time: requested_time,- sender,- });+ loop {+ loop {+ let message = match requests.next_step() {+ Step::Wait => rx.recv().ok(),+ _ => match rx.try_recv() {+ Err(mpsc::TryRecvError::Empty) => break,+ message => message.ok(),+ },};-- match r {- VideoDecoderMessage::GetFrame(requested_time, _, sender) => {- push_request(requested_time, sender);+ // Each request borrows a decoder handle, which owns the sender, so once the+ // channel is gone nobody is waiting for an answer.+ let Some(message) = message else {+ return;+ };+ let VideoDecoderMessage::GetFrame(time, _, sender) = message;+ if sender.is_closed() {+ continue;}- }- while let Ok(msg) = rx.try_recv() {- match msg {- VideoDecoderMessage::GetFrame(requested_time, _, sender) => {- push_request(requested_time, sender);+ let frame = (time * fps as f32).floor() as u32;+ match requests.request(frame, sender) {+ Some((sender, cached)) => answer(sender, cached),+ None => {+ health.record_request();+ if health.should_warn_unhealthy() {+ warn!(+ name = name,+ consecutive_errors = health.consecutive_errors,+ texture_failures = health.consecutive_texture_read_failures,+ total_decoded = health.total_frames_decoded,+ "MediaFoundation decoder unhealthy, performance may degrade"+ );+ }}}}diff --cc crates/rendering/src/yuv_converter.rsindex 4bd420ddb,edac1b916..d14006320--- a/crates/rendering/src/yuv_converter.rs+++ b/crates/rendering/src/yuv_converter.rs@@@ -1686,3 -1776,241 +1776,241 @@@ impl YuvToRgbaConverterSome(self.current_output_texture())}}++ /// The GPU conversions against the CPU reference in `cpu_yuv`: the export's+ /// RGBA to NV12 shader must produce BT.709 limited range (the matrix the+ /// encoders tag), and the decode shaders must apply the matrix they are asked+ /// for. Skipped where no adapter exists.+ #[cfg(test)]+ mod colour_matrix_tests {+ use super::*;+ use crate::frame_pipeline::RgbaToNv12Converter;++ // Primaries, secondaries, grey axis and the suite's red background.+ const CHART: [[u8; 3]; 8] = [+ [0, 0, 0],+ [255, 255, 255],+ [128, 128, 128],+ [255, 0, 0],+ [0, 255, 0],+ [0, 0, 255],+ [255, 255, 0],+ [220, 40, 40],+ ];+ // Each chart colour fills a 4x2 block, so chroma averaging is exact.+ const WIDTH: u32 = 4 * CHART.len() as u32;+ const HEIGHT: u32 = 2;++ fn device() -> Option<(wgpu::Device, wgpu::Queue)> {+ let instance = crate::create_wgpu_instance_sync();+ let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {+ power_preference: wgpu::PowerPreference::LowPower,+ force_fallback_adapter: false,+ compatible_surface: None,+ }))+ .ok()?;+ pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()+ }++ fn reference_bt709(rgb: [u8; 3]) -> [f64; 3] {+ let (kr, kb) = (0.2126, 0.0722);+ let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0);+ let y = kr * r + (1.0 - kr - kb) * g + kb * b;+ [+ 16.0 + 219.0 * y,+ 128.0 + 224.0 * (b - y) / (2.0 * (1.0 - kb)),+ 128.0 + 224.0 * (r - y) / (2.0 * (1.0 - kr)),+ ]+ }++ fn chart_texture(device: &wgpu::Device, queue: &wgpu::Queue) -> wgpu::Texture {+ let texture = device.create_texture(&wgpu::TextureDescriptor {+ label: Some("colour chart"),+ size: wgpu::Extent3d {+ width: WIDTH,+ height: HEIGHT,+ depth_or_array_layers: 1,+ },+ mip_level_count: 1,+ sample_count: 1,+ dimension: wgpu::TextureDimension::D2,+ format: wgpu::TextureFormat::Rgba8Unorm,+ usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,+ view_formats: &[],+ });+ let data: Vec<u8> = (0..HEIGHT)+ .flat_map(|_| {+ CHART+ .iter()+ .flat_map(|[r, g, b]| [*r, *g, *b, 255].repeat(4))+ })+ .collect();+ queue.write_texture(+ texture.as_image_copy(),+ &data,+ wgpu::TexelCopyBufferLayout {+ offset: 0,+ bytes_per_row: Some(WIDTH * 4),+ rows_per_image: None,+ },+ wgpu::Extent3d {+ width: WIDTH,+ height: HEIGHT,+ depth_or_array_layers: 1,+ },+ );+ texture+ }++ /// The chart as BT.709 NV12 from the CPU encoder (tight strides).+ fn chart_nv12() -> (Vec<u8>, Vec<u8>) {+ let mut y = Vec::new();+ let mut uv = Vec::new();+ for _ in 0..HEIGHT {+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {+ y.extend([cpu_yuv::bt709_luma(r, g, b); 4]);+ }+ }+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {+ let [cb, cr] = cpu_yuv::bt709_chroma(r, g, b);+ uv.extend([cb, cr, cb, cr]);+ }+ (y, uv)+ }++ fn read_rgba(device: &wgpu::Device, queue: &wgpu::Queue, texture: &wgpu::Texture) -> Vec<u8> {+ let padded = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;+ let buffer = device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("chart readback"),+ size: u64::from(padded * HEIGHT),+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,+ mapped_at_creation: false,+ });+ let mut encoder = device.create_command_encoder(&Default::default());+ encoder.copy_texture_to_buffer(+ texture.as_image_copy(),+ wgpu::TexelCopyBufferInfo {+ buffer: &buffer,+ layout: wgpu::TexelCopyBufferLayout {+ offset: 0,+ bytes_per_row: Some(padded),+ rows_per_image: None,+ },+ },+ wgpu::Extent3d {+ width: WIDTH,+ height: HEIGHT,+ depth_or_array_layers: 1,+ },+ );+ queue.submit(std::iter::once(encoder.finish()));+ buffer.slice(..).map_async(wgpu::MapMode::Read, |_| {});+ device.poll(wgpu::PollType::Wait).expect("poll");+ let mapped = buffer.slice(..).get_mapped_range();+ (0..HEIGHT as usize)+ .flat_map(|row| {+ let start = row * padded as usize;+ mapped[start..start + WIDTH as usize * 4].to_vec()+ })+ .collect()+ }++ #[tokio::test]+ async fn export_shader_encodes_bt709_limited_range() {+ let Some((device, queue)) = device() else {+ eprintln!("no GPU adapter available, skipping");+ return;+ };+ let source = chart_texture(&device, &queue);+ let mut converter = RgbaToNv12Converter::new(&device);+ let mut encoder = device.create_command_encoder(&Default::default());+ assert!(+ converter+ .submit_conversion(&device, &queue, &mut encoder, &source, WIDTH, HEIGHT, 0, 30)+ .await+ .expect("conversion submission")+ );- queue.submit(std::iter::once(encoder.finish()));- converter.after_submit(&queue);++ let submission = queue.submit(std::iter::once(encoder.finish()));++ converter.after_submit(&queue, &submission);+ let frame = converter+ .take_pending()+ .expect("conversion pending")+ .wait_with_pool(&device, None)+ .await+ .expect("conversion completes");+ let stride = frame.y_stride as usize;+ let uv_base = stride * HEIGHT as usize;+ for (i, rgb) in CHART.iter().enumerate() {+ let [y, cb, cr] = reference_bt709(*rgb);+ let x = i * 4;+ let actual = [+ frame.data[x],+ frame.data[uv_base + x],+ frame.data[uv_base + x + 1],+ ];+ for (a, e) in actual.iter().zip([y, cb, cr]) {+ assert!(+ (f64::from(*a) - e).abs() <= 0.5 + 1e-3,+ "{rgb:?}: got {actual:?}, BT.709 reference {y:.2} {cb:.2} {cr:.2}"+ );+ }+ }+ }++ #[test]+ fn decode_shaders_apply_the_requested_matrix() {+ let Some((device, queue)) = device() else {+ eprintln!("no GPU adapter available, skipping");+ return;+ };+ let (y, uv) = chart_nv12();+ // The same chroma as separate U and V planes for the planar shader.+ let u: Vec<u8> = uv.iter().step_by(2).copied().collect();+ let v: Vec<u8> = uv.iter().skip(1).step_by(2).copied().collect();+ let mut converter = YuvToRgbaConverter::new(&device);+ for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {+ converter.set_matrix(matrix);+ for planar in [false, true] {+ if planar {+ converter+ .convert_yuv420p(+ &device,+ &queue,+ &y,+ &u,+ &v,+ WIDTH,+ HEIGHT,+ WIDTH,+ WIDTH / 2,+ )+ .expect("convert YUV420P");+ } else {+ converter+ .convert_nv12(&device, &queue, &y, &uv, WIDTH, HEIGHT, WIDTH, WIDTH)+ .expect("convert NV12");+ }+ let rgba = read_rgba(&device, &queue, converter.output_texture().unwrap());+ for (i, rgb) in CHART.iter().enumerate() {+ let x = i * 4;+ let expected = matrix.ycbcr_to_rgb(+ i32::from(y[x]) - 16,+ i32::from(uv[x]) - 128,+ i32::from(uv[x + 1]) - 128,+ );+ let actual = &rgba[x * 4..x * 4 + 3];+ assert!(+ actual.iter().zip(expected).all(|(a, e)| a.abs_diff(e) <= 1),+ "{matrix:?} planar={planar} {rgb:?}: GPU {actual:?}, CPU {expected:?}"+ );+ if matrix == YuvMatrix::Bt709 {+ assert!(+ actual.iter().zip(rgb).all(|(a, e)| a.abs_diff(*e) <= 2),+ "planar={planar}: BT.709 round trip of {rgb:?} gave {actual:?}"+ );+ }+ }+ }+ }+ }+ }