Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(compositor): convert full-range VideoToolbox frames with full-range BT.709 on macOS

EtienneLescot76894bc66365
diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs
index e190ec385..92dc08a87 100644
--- a/crates/compositor/src/compositor_macos.rs
+++ b/crates/compositor/src/compositor_macos.rs
@@ -106,6 +106,7 @@ extern "C" {
/// excursion ('xf20') : le P010 que VideoToolbox sort pour un flux 10 bits (HEVC Main 10).
const CV_420_10_BIPLANAR_VIDEO: u32 = u32::from_be_bytes(*b"x420");
const CV_420_10_BIPLANAR_FULL: u32 = u32::from_be_bytes(*b"xf20");
+const CV_420_8_BIPLANAR_FULL: u32 = u32::from_be_bytes(*b"420f");
/// `retain` ObjC sur un `id`. `CVMetalTextureGetTexture` rend une référence
/// *empruntée* au `CVMetalTextureRef` qui la porte : relâcher ce dernier sans
@@ -204,6 +205,28 @@ impl CVMetalTextureCache {
}
}
+/// Vue du plan CbCr dont le canal bleu vaut 1 : c'est ainsi que `sample_yuv` (`shaders.metal`)
+/// sait qu'une frame est en pleine plage (`420f`, `xf20`). Un `RG` lu sans swizzle rend 0 en
+/// bleu, donc la plage limitée reste le cas par défaut, sans rien lier de plus à chaque draw.
+fn full_range_view(uv: &metal::TextureRef) -> metal::Texture {
+ use objc::{msg_send, sel, sel_impl};
+ // `MTLTextureSwizzleChannels` : rouge, vert, un, alpha ; absent des bindings `metal` 0.29.
+ #[repr(C)]
+ struct Swizzle([u8; 4]);
+ const RED: u8 = 2;
+ const GREEN: u8 = 3;
+ const ONE: u8 = 1;
+ const ALPHA: u8 = 5;
+ let one = metal::NSRange::new(0, 1);
+ unsafe {
+ msg_send![uv, newTextureViewWithPixelFormat:uv.pixel_format()
+ textureType:uv.texture_type()
+ levels:one
+ slices:one
+ swizzle:Swizzle([RED, GREEN, ONE, ALPHA])]
+ }
+}
+
impl Drop for CVMetalTextureCache {
fn drop(&mut self) {
unsafe {
@@ -1178,7 +1201,8 @@ impl Compositor {
)
})?;
let cache = &self.metal_texture_cache;
- let (y_fmt, uv_fmt) = match CVPixelBufferGetPixelFormatType(pb) {
+ let pixel_format = CVPixelBufferGetPixelFormatType(pb);
+ let (y_fmt, uv_fmt) = match pixel_format {
CV_420_10_BIPLANAR_VIDEO | CV_420_10_BIPLANAR_FULL => {
(metal::MTLPixelFormat::R16Unorm, metal::MTLPixelFormat::RG16Unorm)
}
@@ -1186,6 +1210,9 @@ impl Compositor {
};
let y = cache.make_texture_from_pixel_buffer(pb, 0, y_fmt)?;
let uv = cache.make_texture_from_pixel_buffer(pb, 1, uv_fmt)?;
+ if matches!(pixel_format, CV_420_8_BIPLANAR_FULL | CV_420_10_BIPLANAR_FULL) {
+ return Ok((y, full_range_view(&uv)));
+ }
Ok((y, uv))
}
diff --git a/crates/compositor/src/shaders.metal b/crates/compositor/src/shaders.metal
index 2a469a504..6b68122bf 100644
--- a/crates/compositor/src/shaders.metal
+++ b/crates/compositor/src/shaders.metal
@@ -141,6 +141,18 @@ inline float3 yuv709_limited(float y, float2 cbcr)
return clamp(rgb, 0.0, 1.0);
}
+// BT.709 pleine plage -> RGB : Y sur 0..255, CbCr centrés sur 128 et étalés sur 255.
+inline float3 yuv709_full(float y, float2 cbcr)
+{
+ float Cb = cbcr.x - 128.0 / 255.0;
+ float Cr = cbcr.y - 128.0 / 255.0;
+ float3 rgb;
+ rgb.r = y + 1.5748 * Cr;
+ rgb.g = y - 0.1873 * Cb - 0.4681 * Cr;
+ rgb.b = y + 1.8556 * Cb;
+ return clamp(rgb, 0.0, 1.0);
+}
+
// `texture2d<float>::sample` rend TOUJOURS un `float4` en MSL, là où le HLSL
// `Texture2D<float>` rend un scalaire : d'où les `.r` / `.rg` que le port d'origine
// n'avait pas (et qui ne compilaient pas).
@@ -149,8 +161,9 @@ inline float3 sample_yuv(float2 uv,
texture2d<float, access::sample> texUV)
{
float y = texY.sample(samp, uv).r;
- float2 cbcr = texUV.sample(samp, uv).rg;
- return yuv709_limited(y, cbcr);
+ float4 c = texUV.sample(samp, uv);
+ // Bleu à 1 : vue pleine plage posée par `full_range_view` (420f, xf20).
+ return c.b > 0.5 ? yuv709_full(y, c.rg) : yuv709_limited(y, c.rg);
}
// SDF segment à bouts ronds — la primitive des flèches d'annotation.
diff --git a/crates/compositor/tests/hevc10_decode.rs b/crates/compositor/tests/hevc10_decode.rs
index ae743d76f..33dd9edad 100644
--- a/crates/compositor/tests/hevc10_decode.rs
+++ b/crates/compositor/tests/hevc10_decode.rs
@@ -19,12 +19,13 @@ use openscreen_compositor::config;
use openscreen_compositor::d3d::Gpu;
use openscreen_compositor::pipeline::ClipSource;
use openscreen_compositor::scene::Scene;
-use openscreen_compositor::timeline_walk::render_frames;
+use openscreen_compositor::timeline_walk::{render_frames, RenderedFrame};
fn scene_json(source: &str) -> String {
+ let source = serde_json::to_string(source).expect("chemin JSON");
format!(
r##"{{
- "clips": [{{"screenPath":"{source}","webcamPath":"","sourceStartSec":0,"sourceEndSec":1,"webcamOffsetSec":0,"hasAudio":false}}],
+ "clips": [{{"screenPath":{source},"webcamPath":"","sourceStartSec":0,"sourceEndSec":1,"webcamOffsetSec":0,"hasAudio":false}}],
"layout": {{"preset":"no-webcam","webcamSize":1.0,"webcamShape":"rounded","webcamMirror":false,"webcamPosition":null,"webcamReactiveZoom":false}},
"effects": {{"padding":0,"blur":false,"shadow":0.0,"roundnessFrac":0,"motionBlur":0.0}},
"background": {{"kind":"color","color":"#303030"}},
@@ -36,13 +37,9 @@ fn scene_json(source: &str) -> String {
)
}
-#[test]
-fn a_10_bit_hevc_frame_keeps_its_colours() {
- let Some(source) = std::env::var_os("OPENSCREEN_HEVC10_SOURCE") else {
- eprintln!("OPENSCREEN_HEVC10_SOURCE absent — test sauté");
- return;
- };
- let source = source.to_string_lossy().replace('\\', "/");
+/// La frame 10 de `source`, rendue seule sur un fond uni, sans effet.
+fn render_frame_10(source: &str) -> RenderedFrame {
+ let source = source.replace('\\', "/");
let gpu = Gpu::create_auto(false).expect("device");
let clips = [ClipSource {
screen: source.clone(),
@@ -62,21 +59,50 @@ fn a_10_bit_hevc_frame_keeps_its_colours() {
let comp = Compositor::new_sized(&gpu, 640, 360).expect("compositeur");
comp.set_live_params(live_params_from_scene(&scene));
comp.set_scene(Some(scene));
- let frames =
+ let mut frames =
unsafe { render_frames(&clips, &gpu, &comp, &cfg, 30, &[10]) }.expect("render_frames");
- let frame = &frames[0];
+ frames.swap_remove(0)
+}
+/// Chaque moitié de la mire garde `left` / `right`, à `tol` près par canal.
+fn assert_halves(frame: &RenderedFrame, left: [u8; 3], right: [u8; 3], tol: u8) {
let px = |x: u32, y: u32| {
let i = ((y * frame.width + x) * 4) as usize;
[frame.rgba[i], frame.rgba[i + 1], frame.rgba[i + 2]]
};
- let close =
- |got: [u8; 3], want: [u8; 3]| got.iter().zip(want).all(|(&g, w)| g.abs_diff(w) <= 40);
let (w, h) = (frame.width, frame.height);
- for (x, want) in [(w / 4, [255, 0, 0]), (3 * w / 4, [0, 0, 255])] {
+ for (x, want) in [(w / 4, left), (3 * w / 4, right)] {
for y in [h / 4, h / 2, 3 * h / 4] {
let got = px(x, y);
- assert!(close(got, want), "({x},{y}) : {got:?}, attendu ≈ {want:?}");
+ let close = got.iter().zip(want).all(|(&g, w)| g.abs_diff(w) <= tol);
+ assert!(close, "({x},{y}) : {got:?}, attendu ≈ {want:?}");
}
}
}
+
+#[test]
+fn a_10_bit_hevc_frame_keeps_its_colours() {
+ let Some(source) = std::env::var_os("OPENSCREEN_HEVC10_SOURCE") else {
+ eprintln!("OPENSCREEN_HEVC10_SOURCE absent — test sauté");
+ return;
+ };
+ let frame = render_frame_10(&source.to_string_lossy());
+ assert_halves(&frame, [255, 0, 0], [0, 0, 255], 40);
+}
+
+/// Une frame pleine plage (`420f`, `xf20`) n'est pas relue comme limitée : sans le
+/// correctif, le gris sombre tombe à ~19 et le gris clair monte à ~242.
+///
+/// ```sh
+/// ffmpeg -f lavfi -i "color=c=0x202020:size=640x360:rate=30:duration=1,drawbox=x=320:y=0:w=320:h=360:color=0xe0e0e0:t=fill" /// -vf scale=out_range=full,format=p010le -color_range pc -c:v hevc_videotoolbox -profile:v main10 -tag:v hvc1 grey10full.mov
+/// OPENSCREEN_FULL_RANGE_SOURCE=$PWD/grey10full.mov cargo test -p openscreen-compositor --test hevc10_decode
+/// ```
+#[test]
+fn a_full_range_frame_keeps_its_levels() {
+ let Some(source) = std::env::var_os("OPENSCREEN_FULL_RANGE_SOURCE") else {
+ eprintln!("OPENSCREEN_FULL_RANGE_SOURCE absent — test sauté");
+ return;
+ };
+ let frame = render_frame_10(&source.to_string_lossy());
+ assert_halves(&frame, [0x20; 3], [0xe0; 3], 6);
+}