Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(hdr): an HDR timeline keeps its clips' light above white, and the monitor shows it tone-mapped

Jareer120da88df2d9
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 9bdae3a5..f3a4a7f1 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -111,9 +111,12 @@ flowchart LR
iPhone's HLG, HDR10's PQ - is decoded deep (`DecodeOptions::deep`: sixteen
bits a channel, its own signal, no CPU tone map) wherever its frame goes
straight to the GPU, and converted as it uploads (`DEEP_SHADER`): the
- transfer undone, the primaries brought to Rec. 709's, and, while every
- timeline is SDR, conformed to it by BT.2390's roll-off. A clip with an
- FFmpeg chain or a cutout keeps the eight-bit tone map in the decoder.
+ transfer undone, the primaries brought to Rec. 709's, and, on an SDR
+ timeline, conformed to it by BT.2390's roll-off. On an HDR timeline it
+ keeps its light above white (`FramePlan::output`), and the resolve rolls
+ the whole frame off for an SDR screen instead, with the same maths. A clip
+ with an FFmpeg chain or a cutout keeps the eight-bit tone map in the
+ decoder.
Effect packages written for
gamma-encoded `0..1` see that through the prelude's `legacy_in` and
`legacy_out` (`concat-effects/src/shader.rs`) until the GPU-only effects
diff --git a/src/crates/concat-api/src/lib.rs b/src/crates/concat-api/src/lib.rs
index 65fd7b3c..fefa2509 100644
--- a/src/crates/concat-api/src/lib.rs
+++ b/src/crates/concat-api/src/lib.rs
@@ -688,6 +688,7 @@ impl Api {
width,
height,
moving: false,
+ color_space: session.video().color_space,
},
)
.map_err(ApiError::failed)?;
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index d5cff700..594941f8 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -33,7 +33,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use concat_core::SpeedCurve;
use concat_core::animate::{Animation, Ease as AnimEase, Key as AnimKey, Track as AnimTrack};
-use concat_core::frame::Frame;
+use concat_core::frame::{Frame, Signal};
use concat_core::shader::{RevealMap, ShaderPass};
use concat_core::time::{FrameRate, Rational};
use concat_core::timeline::{Clip, ClipId, MediaRef, Timeline, Track, TrackKind, Transform};
@@ -42,7 +42,7 @@ use concat_media::audio::{self, AudioClip};
use concat_media::{
DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, RateMode, VideoCodec,
};
-use concat_project::model::{AppliedFilter, Cutout};
+use concat_project::model::{AppliedFilter, ColorSpace, Cutout};
use concat_render::{
Compositor, FramePlan, PlannedLayer, PlannedTreatment, Transition, WgpuCompositor, plan_frame,
};
@@ -357,6 +357,11 @@ pub struct ExportRequest {
/// https://github.com/jub0t/Concat/issues/103
#[serde(default, deserialize_with = "range_by_name")]
pub color_range: concat_media::ColorRange,
+ /// What the timeline is output in. SDR when a request does not say.
+ /// Until the HDR export lands an HDR timeline is written tone-mapped
+ /// to SDR, as the monitor shows it.
+ #[serde(default)]
+ pub color_space: ColorSpace,
/// The flattened clip list to render.
pub clips: Vec<ExportClip>,
}
@@ -1279,6 +1284,7 @@ fn render_picture(
height: request.height,
layers,
treatments: Vec::new(),
+ output: output_of(request.color_space),
},
&treatments,
&transitions,
@@ -1382,13 +1388,14 @@ fn composite_treated(
return compositor.render(&plan);
}
- let (width, height) = (plan.width, plan.height);
+ let (width, height, output) = (plan.width, plan.height, plan.output);
let stage = |layers: Vec<PlannedLayer>| FramePlan {
time,
width,
height,
layers,
treatments: Vec::new(),
+ output,
};
let layers = std::mem::take(&mut plan.layers);
let mut ground: Option<Frame> = None;
@@ -1449,6 +1456,7 @@ fn transition_stages(plan: &FramePlan, span: &TransitionSpan) -> (FramePlan, Fra
height: plan.height,
layers,
treatments: Vec::new(),
+ output: plan.output,
};
let from = plan
.layers
@@ -1483,13 +1491,14 @@ fn combine_transition(
) -> Option<Frame> {
let pass =
Catalogue::builtin().transition_pass(&span.id, &span.params, span.progress(plan.time))?;
- let (width, height, time) = (plan.width, plan.height, plan.time);
+ let (width, height, time, output) = (plan.width, plan.height, plan.time, plan.output);
let stage = |layers: Vec<PlannedLayer>| FramePlan {
time,
width,
height,
layers,
treatments: Vec::new(),
+ output,
};
let (from_plan, to_plan) = transition_stages(plan, span);
let from = compositor.render(&from_plan);
@@ -1528,6 +1537,18 @@ pub struct PreviewFrameRequest {
pub rate_den: i64,
/// The flattened clip list, exactly as an export would take it.
pub clips: Vec<ExportClip>,
+ /// What the timeline is output in; SDR when a request does not say.
+ #[serde(default)]
+ pub color_space: ColorSpace,
+}
+
+/// The signal a timeline of `space` is drawn for (`FramePlan::output`).
+pub fn output_of(space: ColorSpace) -> Signal {
+ match space {
+ ColorSpace::Sdr => Signal::Sdr,
+ ColorSpace::Hlg => Signal::Hlg,
+ ColorSpace::Pq => Signal::Pq,
+ }
}
/// Composites the true frame at one instant, for the paused monitor.
@@ -1769,6 +1790,7 @@ pub fn preview_sources_of(
height: plan.height,
layers,
treatments: Vec::new(),
+ output: plan.output,
};
// Every treatment that is shaders alone goes into the plan now, so a
// caller drawing the plan itself has them; see
@@ -1806,6 +1828,8 @@ pub struct PreviewPlan {
rate: FrameRate,
width: u32,
height: u32,
+ /// What the timeline is output in.
+ output: Signal,
}
/// Builds the plan for `clips` at one output size and rate. The costly
@@ -1818,6 +1842,7 @@ pub fn preview_plan(
height: u32,
rate_num: i64,
rate_den: i64,
+ color_space: ColorSpace,
gpu: bool,
) -> PreviewPlan {
let rate = FrameRate::new(Rational::new(rate_num, rate_den));
@@ -1843,6 +1868,7 @@ pub fn preview_plan(
rate_mode: RateMode::Vbr,
bitrate_kbps: 0,
color_range: concat_media::ColorRange::Limited,
+ color_space,
clips: Vec::new(),
};
PreviewPlan {
@@ -1850,6 +1876,7 @@ pub fn preview_plan(
rate,
width,
height,
+ output: output_of(color_space),
}
}
@@ -1861,6 +1888,7 @@ fn preview_timeline(request: &PreviewFrameRequest, gpu: bool) -> PreviewPlan {
request.height,
request.rate_num,
request.rate_den,
+ request.color_space,
gpu,
)
}
@@ -2055,6 +2083,7 @@ mod tests {
height: 8,
layers: vec![ground, top],
treatments: Vec::new(),
+ output: concat_core::frame::Signal::Sdr,
}
};
let negate = Treatment {
@@ -2183,6 +2212,7 @@ mod tests {
rate_num: 30,
rate_den: 1,
clips: vec![first, second],
+ color_space: concat_project::model::ColorSpace::Sdr,
};
let bytes = preview_frame(&pool, &request).expect("previews");
let (px, py) = ((f64::from(width) * x) as u32, height / 2);
@@ -2268,6 +2298,7 @@ mod tests {
rate_num: 30,
rate_den: 1,
clips: vec![request_clip],
+ color_space: concat_project::model::ColorSpace::Sdr,
};
let pool = concat_media::ReaderPool::new(16 * 1024 * 1024, 2);
let bytes = preview_frame(&pool, &request).expect("previews");
@@ -2332,6 +2363,7 @@ mod tests {
rate_num: 30,
rate_den: 1,
clips: vec![request_clip],
+ color_space: concat_project::model::ColorSpace::Sdr,
};
let pool = concat_media::ReaderPool::new(16 * 1024 * 1024, 2);
@@ -2356,6 +2388,7 @@ mod tests {
rate_num: 30,
rate_den: 1,
clips: vec![outliving],
+ color_space: concat_project::model::ColorSpace::Sdr,
};
let bytes = preview_frame(&pool, &late).expect("previews past the media's end");
assert!(
@@ -2379,6 +2412,7 @@ mod tests {
rate_num: 30,
rate_den: 1,
clips: vec![effected],
+ color_space: concat_project::model::ColorSpace::Sdr,
};
let bytes = preview_frame(&pool, &filtered).expect("previews with a chain");
assert!(
diff --git a/src/crates/concat-host/src/export.rs b/src/crates/concat-host/src/export.rs
index 79102cd6..fe3f9406 100644
--- a/src/crates/concat-host/src/export.rs
+++ b/src/crates/concat-host/src/export.rs
@@ -72,6 +72,7 @@ pub fn request(session: &Session, spec: &ExportSpec, titles: Vec<ExportClip>) ->
rate_mode: spec.rate_mode,
bitrate_kbps: spec.bitrate_kbps,
color_range: spec.color_range,
+ color_space: session.video().color_space,
clips,
}
}
diff --git a/src/crates/concat-host/src/preview.rs b/src/crates/concat-host/src/preview.rs
index 8ca23724..f3a1dab7 100644
--- a/src/crates/concat-host/src/preview.rs
+++ b/src/crates/concat-host/src/preview.rs
@@ -21,6 +21,7 @@ use std::sync::{Arc, Mutex};
use concat_export::ExportClip;
use concat_project::DocumentSettings;
+use concat_project::model::ColorSpace;
/// A frame request: the instant and the size, with the clips coming from
/// the session that owns them.
@@ -36,6 +37,9 @@ pub struct FrameSpec {
/// what reads a file's proxy where it has one, and tells the
/// scheduler which way to decode ahead.
pub moving: bool,
+ /// What the timeline is output in: an HDR one keeps its clips' light
+ /// above white and is rolled off for the SDR screen.
+ pub color_space: ColorSpace,
}
/// The reader pool behind the monitor, shareable across threads.
@@ -56,6 +60,7 @@ struct PlanEntry {
width: u32,
height: u32,
rate: (i64, i64),
+ color_space: ColorSpace,
gpu: bool,
plan: Arc<concat_export::PreviewPlan>,
}
@@ -227,6 +232,7 @@ impl Monitor {
&& entry.width == spec.width
&& entry.height == spec.height
&& entry.rate == rate
+ && entry.color_space == spec.color_space
&& entry.gpu == gpu
&& (Arc::ptr_eq(&entry.clips, &clips) || *entry.clips == *clips)
{
@@ -238,6 +244,7 @@ impl Monitor {
spec.height,
rate.0,
rate.1,
+ spec.color_space,
gpu,
));
*slot = Some(PlanEntry {
@@ -245,6 +252,7 @@ impl Monitor {
width: spec.width,
height: spec.height,
rate,
+ color_space: spec.color_space,
gpu,
plan: Arc::clone(&plan),
});
diff --git a/src/crates/concat-render/src/compositor.rs b/src/crates/concat-render/src/compositor.rs
index d6baf43a..2ca9078e 100644
--- a/src/crates/concat-render/src/compositor.rs
+++ b/src/crates/concat-render/src/compositor.rs
@@ -8,7 +8,7 @@
//! one implementation: a machine without a GPU runs it on the platform's
//! software adapter (WARP, lavapipe). The CPU compositor that was the
//! reference lives on in the tests alone ([`crate::reference`]), as the
-//! oracle the GPU is held to while both draw 8-bit, gamma-encoded pictures.
+//! oracle the GPU's plain layers are held to.
use concat_core::frame::Frame;
use concat_core::shader::TransitionPass;
diff --git a/src/crates/concat-render/src/gpu.rs b/src/crates/concat-render/src/gpu.rs
index 841f758b..3a2e63a4 100644
--- a/src/crates/concat-render/src/gpu.rs
+++ b/src/crates/concat-render/src/gpu.rs
@@ -7,16 +7,15 @@
//! uploaded as textures, drawn as transformed quads into an offscreen
//! target, and read back as a [`Frame`] or handed on as a texture.
//!
-//! Choices the tests' CPU oracle (`crate::reference`) mirrors, so the two
-//! can be diffed while both draw 8-bit pictures:
-//!
-//! - Everything is drawn in half floats (`WORK`, `Rgba16Float`): a frame is
-//! converted into it on the GPU as it is uploaded, and the finished frame
-//! out of it into eight bits once, at the end (`resolve`). The values are
-//! still the stored, gamma-encoded ones, so blending matches the oracle's;
-//! when the working space goes linear, the suite moves to expectations of
-//! its own.
-//! - Layer quads are sampled bilinearly with clamp-to-edge.
+//! - Everything is drawn in half floats (`WORK`, `Rgba16Float`), in light:
+//! extended linear Rec. 709, 1.0 the white of an SDR picture (203 nits).
+//! A frame is converted into it on the GPU as it is uploaded, and the
+//! finished frame out of it once, at the end (`resolve`): as it is for an
+//! SDR timeline, rolled off by BT.2390 for an HDR one on an SDR screen.
+//! An HDR clip is conformed to SDR as it uploads for an SDR timeline, and
+//! keeps its light above white for an HDR one (`FramePlan::output`).
+//! - Layer quads are sampled bilinearly with clamp-to-edge, as the tests'
+//! CPU oracle (`crate::reference`) does.
//!
//! Construction is fallible: [`WgpuCompositor::new`] takes the machine's GPU,
//! or its software adapter where it has none (WARP on Windows, lavapipe on
@@ -211,43 +210,22 @@ fn fs_resolve(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
let clipped = clamp(colour.rgb, vec3<f32>(0.0), vec3<f32>(1.0));
return vec4<f32>(pow(clipped, vec3<f32>(1.0 / 2.4)), colour.a);
}
-"#;
-
-/// A deep frame (sixteen bits a channel, Rec. 2020, its source's own signal)
-/// copied into the working space: the upload of an HDR or wide-gamut clip.
-/// One entry a signal (`concat_core::frame::Signal`): the transfer undone
-/// into light, 1.0 the white of an SDR picture (203 nits, ITU-R BT.2408),
-/// and the primaries brought to Rec. 709's, where a colour outside them is
-/// negative rather than clipped.
-///
-/// Every timeline is SDR for now, so an HDR clip is conformed to SDR as it
-/// arrives, as Final Cut does a clip at a time: BT.2390's roll-off (ITU-R
-/// BT.2390, the EETF) takes its highlights from a 1000-nit master down to
-/// SDR white, on the brightest channel so a hue keeps its place. An HDR
-/// timeline will skip it and keep the highlights.
-const DEEP_SHADER: &str = r#"
-@group(0) @binding(0) var deep: texture_2d<u32>;
-
-@vertex
-fn vs_full(@builtin(vertex_index) index: u32) -> @builtin(position) vec4<f32> {
- let corner = vec2<f32>(f32((index << 1u) & 2u), f32(index & 2u));
- return vec4<f32>(corner * 2.0 - 1.0, 0.0, 1.0);
-}
-fn deep_at(at: vec4<f32>) -> vec4<f32> {
- return vec4<f32>(textureLoad(deep, vec2<i32>(at.xy), 0)) / 65535.0;
-}
-
-// Rec. 2020 primaries to Rec. 709's, linear light (ITU-R BT.2087's matrix
-// inverted, to full precision).
-fn from_2020(rgb: vec3<f32>) -> vec3<f32> {
- return vec3<f32>(
- dot(vec3<f32>(1.660491002, -0.587641139, -0.072849863), rgb),
- dot(vec3<f32>(-0.124550475, 1.132899897, -0.008349423), rgb),
- dot(vec3<f32>(-0.018150763, -0.100578898, 1.118729661), rgb),
- );
+// An HDR timeline for an SDR screen: its light rolled off by BT.2390 as an
+// HDR clip's is on an SDR timeline - the same maths, so a lone clip looks
+// the same on either - then encoded as SDR is.
+@fragment
+fn fs_resolve_tone_mapped(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
+ let colour = textureLoad(picture, vec2<i32>(at.xy), 0);
+ let light = to_sdr(colour.rgb * 203.0) / 203.0;
+ let clipped = clamp(light, vec3<f32>(0.0), vec3<f32>(1.0));
+ return vec4<f32>(pow(clipped, vec3<f32>(1.0 / 2.4)), colour.a);
}
+"#;
+/// SMPTE ST 2084 and BT.2390's roll-off, which the deep upload and the
+/// resolve both reach for: appended to each of their modules.
+const TONE_MAP: &str = r#"
// SMPTE ST 2084: a signal to nits, and back.
const PQ_M1: f32 = 0.1593017578125;
const PQ_M2: f32 = 78.84375;
@@ -289,6 +267,43 @@ fn to_sdr(nits: vec3<f32>) -> vec3<f32> {
let mapped = pq_nits(vec3<f32>(e2 * master)).r;
return nits * (mapped / peak);
}
+"#;
+
+/// A deep frame (sixteen bits a channel, Rec. 2020, its source's own signal)
+/// copied into the working space: the upload of an HDR or wide-gamut clip.
+/// One entry a signal (`concat_core::frame::Signal`): the transfer undone
+/// into light, 1.0 the white of an SDR picture (203 nits, ITU-R BT.2408),
+/// and the primaries brought to Rec. 709's, where a colour outside them is
+/// negative rather than clipped.
+///
+/// On an SDR timeline an HDR clip is conformed to SDR as it arrives, as
+/// Final Cut does a clip at a time: BT.2390's roll-off (ITU-R BT.2390, the
+/// EETF) takes its highlights from a 1000-nit master down to SDR white, on
+/// the brightest channel so a hue keeps its place. On an HDR timeline it
+/// keeps them (the `_hdr` entries), and the resolve rolls the whole frame
+/// off instead where the screen is SDR.
+const DEEP_SHADER: &str = r#"
+@group(0) @binding(0) var deep: texture_2d<u32>;
+
+@vertex
+fn vs_full(@builtin(vertex_index) index: u32) -> @builtin(position) vec4<f32> {
+ let corner = vec2<f32>(f32((index << 1u) & 2u), f32(index & 2u));
+ return vec4<f32>(corner * 2.0 - 1.0, 0.0, 1.0);
+}
+
+fn deep_at(at: vec4<f32>) -> vec4<f32> {
+ return vec4<f32>(textureLoad(deep, vec2<i32>(at.xy), 0)) / 65535.0;
+}
+
+// Rec. 2020 primaries to Rec. 709's, linear light (ITU-R BT.2087's matrix
+// inverted, to full precision).
+fn from_2020(rgb: vec3<f32>) -> vec3<f32> {
+ return vec3<f32>(
+ dot(vec3<f32>(1.660491002, -0.587641139, -0.072849863), rgb),
+ dot(vec3<f32>(-0.124550475, 1.132899897, -0.008349423), rgb),
+ dot(vec3<f32>(-0.018150763, -0.100578898, 1.118729661), rgb),
+ );
+}
@fragment
fn fs_upload_sdr_wide(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
@@ -297,29 +312,46 @@ fn fs_upload_sdr_wide(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32
return vec4<f32>(from_2020(lin), colour.a);
}
-@fragment
-fn fs_upload_pq(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
- let colour = deep_at(at);
- let nits = to_sdr(pq_nits(colour.rgb));
- return vec4<f32>(from_2020(nits / 203.0), colour.a);
-}
-
// ARIB STD-B67's inverse OETF to scene light, then BT.2100's OOTF for a
// 1000-nit display (a system gamma of 1.2), which puts HLG's reference
// white - 75 % of the signal - at 203 nits.
-@fragment
-fn fs_upload_hlg(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
- let colour = deep_at(at);
+fn hlg_nits(signal: vec3<f32>) -> vec3<f32> {
let a = 0.17883277;
let b = 0.28466892;
let c = 0.55991073;
- let e = clamp(colour.rgb, vec3<f32>(0.0), vec3<f32>(1.0));
+ let e = clamp(signal, vec3<f32>(0.0), vec3<f32>(1.0));
let low = e * e / 3.0;
let high = (exp((e - c) / a) + b) / 12.0;
let scene = select(high, low, e <= vec3<f32>(0.5));
let ys = dot(scene, vec3<f32>(0.2627, 0.6780, 0.0593));
- let nits = to_sdr(1000.0 * pow(max(ys, 1e-6), 0.2) * scene);
- return vec4<f32>(from_2020(nits / 203.0), colour.a);
+ return 1000.0 * pow(max(ys, 1e-6), 0.2) * scene;
+}
+
+// On an SDR timeline, conformed to SDR as they arrive.
+@fragment
+fn fs_upload_pq(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
+ let colour = deep_at(at);
+ return vec4<f32>(from_2020(to_sdr(pq_nits(colour.rgb)) / 203.0), colour.a);
+}
+
+@fragment
+fn fs_upload_hlg(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
+ let colour = deep_at(at);
+ return vec4<f32>(from_2020(to_sdr(hlg_nits(colour.rgb)) / 203.0), colour.a);
+}
+
+// On an HDR timeline, kept: the highlights stay above white, 1000 nits of
+// the master at 4.93.
+@fragment
+fn fs_upload_pq_hdr(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
+ let colour = deep_at(at);
+ return vec4<f32>(from_2020(pq_nits(colour.rgb) / 203.0), colour.a);
+}
+
+@fragment
+fn fs_upload_hlg_hdr(@builtin(position) at: vec4<f32>) -> @location(0) vec4<f32> {
+ let colour = deep_at(at);
+ return vec4<f32>(from_2020(hlg_nits(colour.rgb) / 203.0), colour.a);
}
"#;
@@ -440,14 +472,21 @@ pub struct WgpuCompositor {
ground_pipelines: [wgpu::RenderPipeline; 2],
/// An uploaded frame into the working format.
upload_pipeline: wgpu::RenderPipeline,
- /// A finished frame out of the working format into eight bits.
+ /// A finished frame out of the working format into eight bits: as it
+ /// is for an SDR timeline, and rolled off for an HDR one on an SDR
+ /// screen.
resolve_pipeline: wgpu::RenderPipeline,
+ tone_mapped_pipeline: wgpu::RenderPipeline,
+ /// What the plan being drawn is output in (`FramePlan::output`): taken
+ /// up by `prepare`, and read by the uploads and the resolve after it.
+ output: concat_core::frame::Signal,
/// The eight-bit textures frames are written into on their way to the
/// pool, one a size, each with its bind group.
staging: HashMap<(u32, u32), (wgpu::Texture, wgpu::BindGroup)>,
/// A deep frame into the working space, by signal: Rec. 2020 gamma,
- /// HLG, PQ (see `DEEP_SHADER`).
- deep_pipelines: [wgpu::RenderPipeline; 3],
+ /// HLG and PQ conformed to SDR, then HLG and PQ kept HDR (see
+ /// `DEEP_SHADER`).
+ deep_pipelines: [wgpu::RenderPipeline; 5],
/// The layout a deep staging texture is bound with: sixteen-bit
/// integers, read texel by texel.
deep_layout: wgpu::BindGroupLayout,
@@ -798,7 +837,7 @@ impl WgpuCompositor {
// The two copies in and out of the working format; see COPY_SHADER.
let copy_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("concat copy"),
- source: wgpu::ShaderSource::Wgsl(COPY_SHADER.into()),
+ source: wgpu::ShaderSource::Wgsl(format!("{COPY_SHADER}{TONE_MAP}").into()),
});
let copy_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("concat copy"),
@@ -834,6 +873,8 @@ impl WgpuCompositor {
};
let upload_pipeline = copy_into(WORK, "fs_upload");
let resolve_pipeline = copy_into(wgpu::TextureFormat::Rgba8Unorm, "fs_resolve");
+ let tone_mapped_pipeline =
+ copy_into(wgpu::TextureFormat::Rgba8Unorm, "fs_resolve_tone_mapped");
let deep_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("concat deep"),
@@ -850,14 +891,21 @@ impl WgpuCompositor {
});
let deep_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("concat deep"),
- source: wgpu::ShaderSource::Wgsl(DEEP_SHADER.into()),
+ source: wgpu::ShaderSource::Wgsl(format!("{DEEP_SHADER}{TONE_MAP}").into()),
});
let deep_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("concat deep"),
bind_group_layouts: &[Some(&deep_layout)],
immediate_size: 0,
});
- let deep_pipelines = ["fs_upload_sdr_wide", "fs_upload_hlg", "fs_upload_pq"].map(|entry| {
+ let deep_pipelines = [
+ "fs_upload_sdr_wide",
+ "fs_upload_hlg",
+ "fs_upload_pq",
+ "fs_upload_hlg_hdr",
+ "fs_upload_pq_hdr",
+ ]
+ .map(|entry| {
device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some(entry),
layout: Some(&deep_pipeline_layout),
@@ -927,6 +975,8 @@ impl WgpuCompositor {
ground_pipelines,
upload_pipeline,
resolve_pipeline,
+ tone_mapped_pipeline,
+ output: concat_core::frame::Signal::Sdr,
staging: HashMap::new(),
deep_pipelines,
deep_layout,
@@ -1033,6 +1083,7 @@ impl WgpuCompositor {
/// rest is drawn on. What comes back are the draws and quads for the
/// final render, into whichever target the caller wants.
fn prepare(&mut self, plan: &FramePlan) -> (Vec<Draw>, Vec<Vertex>) {
+ self.output = plan.output;
self.used.values_mut().for_each(|used| *used = 0);
self.composites += 1;
let (width, height) = (plan.width, plan.height);
@@ -1774,8 +1825,15 @@ impl WgpuCompositor {
/// pixels: the one that already does, moved into this frame's claimed
/// run, or a fresh claim with the pixels uploaded into it.
fn upload(&mut self, frame: &Frame) -> usize {
+ use concat_core::frame::{Depth, Signal};
let key = (frame.width(), frame.height());
- let identity = frame.id();
+ // An HDR frame kept HDR for an HDR timeline is another picture
+ // from the same frame conformed for an SDR one, so it is held
+ // apart: frame ids count up from one, and never reach the top bit.
+ let keep = frame.depth() == Depth::Sixteen
+ && matches!(frame.signal(), Signal::Hlg | Signal::Pq)
+ && self.output != Signal::Sdr;
+ let identity = frame.id() | if keep { 1 << 63 } else { 0 };
let used = self.used.get(&key).copied().unwrap_or(0);
if let Some(pool) = self.pool.get_mut(&key)
&& let Some(found) = (used..pool.len()).find(|&slot| pool[slot].holds == identity)
@@ -1794,7 +1852,7 @@ impl WgpuCompositor {
// a clip's own colour will be brought into the working space. The
// write lands before the submit that copies it, and the next
// upload's write after it, so one staging texture a size serves.
- let deep = frame.depth() == concat_core::frame::Depth::Sixteen;
+ let deep = frame.depth() == Depth::Sixteen;
let (staging, staging_group) = if deep {
self.deep_staging_for(frame.width(), frame.height())
} else {
@@ -1828,11 +1886,12 @@ impl WgpuCompositor {
label: Some("concat upload"),
});
let pipeline = if deep {
- use concat_core::frame::Signal;
- &self.deep_pipelines[match frame.signal() {
- Signal::Hlg => 1,
- Signal::Pq => 2,
- Signal::Sdr | Signal::SdrWide => 0,
+ &self.deep_pipelines[match (frame.signal(), keep) {
+ (Signal::Hlg, false) => 1,
+ (Signal::Pq, false) => 2,
+ (Signal::Hlg, true) => 3,
+ (Signal::Pq, true) => 4,
+ (Signal::Sdr | Signal::SdrWide, _) => 0,
}]
} else {
&self.upload_pipeline
@@ -1854,7 +1913,8 @@ impl WgpuCompositor {
/// Records the finished frame on the pooled `canvas` of `size` copied
/// into `into`, eight bits a channel: the one conversion out of the
- /// working format, where a timeline's output transform will go.
+ /// working format, where the timeline's output transform is - as it is
+ /// for SDR, rolled off for an HDR timeline on an SDR screen.
fn resolve(
&self,
encoder: &mut wgpu::CommandEncoder,
@@ -1862,9 +1922,14 @@ impl WgpuCompositor {
canvas: usize,
into: &wgpu::TextureView,
) {
+ let pipeline = if self.output == concat_core::frame::Signal::Sdr {
+ &self.resolve_pipeline
+ } else {
+ &self.tone_mapped_pipeline
+ };
Self::copy(
encoder,
- &self.resolve_pipeline,
+ pipeline,
&self.pool[&size][canvas].bind_group,
into,
);
diff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rs
index cb2da8b8..d07d2d1f 100644
--- a/src/crates/concat-render/src/gpu/tests.rs
+++ b/src/crates/concat-render/src/gpu/tests.rs
@@ -651,6 +651,7 @@ fn a_transition_kept_on_the_device_matches_the_read_back_one() {
Arc::new(solid(width, height, colour)),
)],
treatments: Vec::new(),
+ output: concat_core::frame::Signal::Sdr,
};
let (from, to) = (plan([220, 30, 30, 255]), plan([30, 30, 220, 255]));
let catalogue = concat_effects::Catalogue::builtin();
@@ -704,6 +705,7 @@ fn a_frame_drawn_before_is_not_uploaded_again() {
height: 36,
layers: vec![PlannedLayer::picture(detached_clip(), Arc::clone(frame))],
treatments: Vec::new(),
+ output: concat_core::frame::Signal::Sdr,
};
gpu.render_texture(&plan).expect("draws");
};
@@ -732,6 +734,7 @@ fn the_pool_stays_within_its_limits() {
height: 4,
layers: vec![PlannedLayer::picture(detached_clip(), frame)],
treatments: Vec::new(),
+ output: concat_core::frame::Signal::Sdr,
};
gpu.render_texture(&plan).expect("draws");
}
@@ -899,6 +902,83 @@ fn an_hdr_frame_is_conformed_to_sdr_on_the_way_in() {
);
}
+/// `plan` drawn into the working space and read back as it stands there,
+/// before the resolve: light, 1.0 SDR white, nothing clipped.
+fn working(gpu: &mut WgpuCompositor, plan: &FramePlan) -> Vec<[f32; 4]> {
+ let (draws, vertices) = gpu.prepare(plan);
+ gpu.write_vertices(&vertices);
+ let (canvas, texture, view) = gpu.canvas(plan.width, plan.height);
+ let encoder = gpu.encode(&view, &texture, &draws, wgpu::Color::BLACK);
+ gpu.queue.submit([encoder.finish()]);
+ let out = read_floats(gpu, (plan.width, plan.height), canvas);
+ gpu.retire();
+ out
+}
+
+/// On an HDR timeline an HDR clip keeps its light above white: HLG's peak
+/// is 1000 nits, 4.93 times SDR white, and its reference white SDR white
+/// itself - where on an SDR timeline the peak is conformed down to white.
+/// The same frame drawn for both is two pictures, kept apart. For the SDR
+/// screen the resolve rolls the HDR timeline off with the maths the SDR
+/// timeline's upload uses, so a lone clip looks the same on either; an SDR
+/// clip's white goes a little under white there, leaving room above it.
+#[test]
+fn an_hdr_timeline_keeps_the_light_above_white() {
+ use concat_core::frame::Signal;
+ let Some(mut gpu) = gpu() else { return };
+ let peak = deep(4, 4, 1.0, Signal::Hlg);
+ let hdr = |layers| FramePlan {
+ output: Signal::Hlg,
+ ..plan(4, 4, layers)
+ };
+ let level = |pixels: Vec<[f32; 4]>| pixels[5];
+ let sdr_light = level(working(&mut gpu, &plan(4, 4, vec![layer(peak.clone())])));
+ assert!(
+ (sdr_light[1] - 1.0).abs() < 0.01,
+ "conformed: {sdr_light:?}"
+ );
+ let hdr_light = level(working(&mut gpu, &hdr(vec![layer(peak.clone())])));
+ assert!(
+ hdr_light[..3]
+ .iter()
+ .all(|channel| (channel - 1000.0 / 203.0).abs() < 0.05),
+ "kept: {hdr_light:?}"
+ );
+ let white = level(working(
+ &mut gpu,
+ &hdr(vec![layer(deep(4, 4, 0.75, Signal::Hlg))]),
+ ));
+ assert!(
+ (white[1] - 1.0).abs() < 0.01,
+ "HLG reference white: {white:?}"
+ );
+
+ let shown = |gpu: &mut WgpuCompositor, plan: &FramePlan| {
+ let out = gpu.render(plan);
+ let p = out.pixel(1, 1).expect("in the frame");
+ [p[0], p[1], p[2]]
+ };
+ for signal_level in [0.75, 1.0, 0.5] {
+ let frame = deep(4, 4, signal_level, Signal::Hlg);
+ let on_sdr = shown(&mut gpu, &plan(4, 4, vec![layer(frame.clone())]));
+ let on_hdr = shown(&mut gpu, &hdr(vec![layer(frame)]));
+ assert!(
+ on_sdr.iter().zip(on_hdr).all(|(a, b)| a.abs_diff(b) <= 1),
+ "HLG {signal_level}: {on_sdr:?} on SDR, {on_hdr:?} on HDR"
+ );
+ }
+ let sdr_white = shown(
+ &mut gpu,
+ &hdr(vec![layer(solid(4, 4, [255, 255, 255, 255]))]),
+ );
+ assert!(
+ sdr_white
+ .iter()
+ .all(|channel| (225..=240).contains(channel)),
+ "SDR white on an HDR timeline: {sdr_white:?}"
+ );
+}
+
/// A pass of a format 2 package - the scene-linear contract - with `body`
/// as its shader and no knobs, at `intensity`.
fn scene_linear(id: &str, body: &str, intensity: f32) -> ShaderPass {
diff --git a/src/crates/concat-render/src/plan.rs b/src/crates/concat-render/src/plan.rs
index b1bec277..0337cfd1 100644
--- a/src/crates/concat-render/src/plan.rs
+++ b/src/crates/concat-render/src/plan.rs
@@ -25,7 +25,7 @@
use std::path::PathBuf;
use std::sync::Arc;
-use concat_core::frame::Frame;
+use concat_core::frame::{Frame, Signal};
use concat_core::shader::ShaderPass;
use concat_core::time::Rational;
use concat_core::timeline::{Blend, ClipId, Timeline, TrackKind, Transform};
@@ -442,6 +442,10 @@ pub struct FramePlan {
pub layers: Vec<PlannedLayer>,
/// The treatments live at this instant, in ascending track order.
pub treatments: Vec<PlannedTreatment>,
+ /// What the timeline is output in: SDR, or HDR in HLG or PQ. On an HDR
+ /// timeline an HDR clip keeps its light above white as it uploads, and
+ /// the resolve rolls the frame off for an SDR screen.
+ pub output: Signal,
}
impl FramePlan {
@@ -453,6 +457,7 @@ impl FramePlan {
height,
layers: Vec::new(),
treatments: Vec::new(),
+ output: Signal::Sdr,
}
}
@@ -535,6 +540,7 @@ pub fn plan_frame(timeline: &Timeline, time: Rational) -> FramePlan {
height: timeline.height,
layers,
treatments: Vec::new(),
+ output: Signal::Sdr,
}
}
diff --git a/src/crates/concat/src/panes/monitor.rs b/src/crates/concat/src/panes/monitor.rs
index 4ddaf9f7..5fbc4c7a 100644
--- a/src/crates/concat/src/panes/monitor.rs
+++ b/src/crates/concat/src/panes/monitor.rs
@@ -184,6 +184,7 @@ impl MonitorPane {
width,
height,
moving: studio.playing,
+ color_space: studio.project().active().video.color_space,
};
let monitor = studio.host.monitor.clone();
self.busy = true;