The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(render): one frame plan describes the frame, and both compositors draw it alike
Jareer4153a50e9828
diff --git a/src/Cargo.lock b/src/Cargo.lockindex 0651a592..78cfc4e9 100644--- a/src/Cargo.lock+++ b/src/Cargo.lock@@ -1337,6 +1337,7 @@ version = "0.2.2"dependencies = ["concat-core","concat-effects",+ "log","pollster","wgpu",]diff --git a/src/crates/concat-cli/src/main.rs b/src/crates/concat-cli/src/main.rsindex 4503b3a0..1949dc5e 100644--- a/src/crates/concat-cli/src/main.rs+++ b/src/crates/concat-cli/src/main.rs@@ -25,7 +25,7 @@ use concat_core::timeline::{Clip, MediaRef, Timeline, Track, TrackKind};use concat_media::{DecodeOptions, Decoder, EncodeOptions, Encoder, FrameSink, FrameSource, ReaderPool,};-use concat_render::{Compositor, CpuCompositor, Layer, plan_frame};+use concat_render::{Compositor, CpuCompositor, plan_frame};#[derive(Parser)]#[command(name = "concat-cli", version, about = "Concat engine command line")]@@ -314,33 +314,23 @@ fn render(input: &PathBuf, output: &PathBuf, frames: u64, fade: u64) -> Result<(println!("rendering {frames} frames at {width}x{height} {rate}");for index in 0..frames {- let plan = plan_frame(&timeline, rate.time_of_frame(index as i64));-- let composed = if plan.is_empty() {- // A gap in the timeline is black, not an error.- compositor.composite(width, height, &[])- } else {- let mut sources = Vec::with_capacity(plan.layers.len());- for layer in &plan.layers {- let frame = pool.frame_at(- &layer.media,- layer.source_time,- width,- height,- false,- None,- None,- )?;- sources.push((frame, layer.opacity));- }- let layers: Vec<Layer<'_>> = sources- .iter()- .map(|(frame, opacity)| {- Layer::new(frame).with_opacity(opacity * fade_in(index, fade))- })- .collect();- compositor.composite(width, height, &layers)- };+ let mut plan = plan_frame(&timeline, rate.time_of_frame(index as i64));+ // The plan says what is on screen; the pool fills in the pictures.+ // A gap in the timeline is a plan with no layers, and black.+ for layer in &mut plan.layers {+ let frame = pool.frame_at(+ &layer.media,+ layer.source_time,+ width,+ height,+ false,+ None,+ None,+ )?;+ layer.source = Some(frame);+ layer.opacity *= fade_in(index, fade);+ }+ let composed = compositor.render(&plan);encoder.write_frame(&composed)?;}diff --git a/src/crates/concat-effects/src/cube.rs b/src/crates/concat-effects/src/cube.rsindex 653c1788..7cefc9c0 100644--- a/src/crates/concat-effects/src/cube.rs+++ b/src/crates/concat-effects/src/cube.rs@@ -11,6 +11,10 @@use concat_core::Lut;+/// Texels a side the largest table may have: 65 is the largest anyone+/// ships, and a 3D texture past it is memory a look does not need.+pub const LARGEST_TABLE: u32 = 65;+/// Parses the text of a `.cube` file into a table.pub fn parse(text: &str) -> Result<Lut, String> {let mut size: Option<u32> = None;@@ -82,6 +86,11 @@ pub fn parse(text: &str) -> Result<Lut, String> {*value = (*value - domain_min[channel]) / span;}}+ if size > LARGEST_TABLE {+ return Err(format!(+ "a table {size} a side is larger than the {LARGEST_TABLE} the renderer uploads"+ ));+ }Lut::from_rgb(size, &rgb).ok_or_else(|| "the table did not fit".to_owned())}@@ -89,6 +98,21 @@ pub fn parse(text: &str) -> Result<Lut, String> {mod tests {use super::*;+ /// A table larger than the renderer uploads is refused at load, with+ /// its size in the message, rather than becoming a texture the+ /// device may not have room for.+ #[test]+ fn a_table_past_the_budget_is_refused() {+ let size = LARGEST_TABLE + 1;+ let mut text = format!("LUT_3D_SIZE {size}\n");+ for _ in 0..size * size * size {+ text.push_str("0 0 0\n");+ }+ let error = parse(&text).expect_err("too large");+ assert!(error.contains(&size.to_string()), "{error}");+ assert!(parse(&identity(2)).is_ok());+ }+fn identity(n: u32) -> String {let mut text = format!("TITLE \"id\"\nLUT_3D_SIZE {n}\n# rows\n");for b in 0..n {diff --git a/src/crates/concat-effects/src/shader.rs b/src/crates/concat-effects/src/shader.rsindex bda8f6ad..cdf4d105 100644--- a/src/crates/concat-effects/src/shader.rs+++ b/src/crates/concat-effects/src/shader.rs@@ -428,6 +428,7 @@ impl Shader {"the shader declares no `fn effect(uv: vec2<f32>) -> vec4<f32>`".to_owned(),);}+ budget(&module)?;let (members, span) = module.types@@ -566,6 +567,82 @@ impl Shader {}}+/// The bindings the prelude declares, and the only ones a package may+/// use: the layer, the frame block and the parameters, the table.+const BINDINGS: [(u32, u32); 6] = [(0, 0), (0, 1), (1, 0), (1, 1), (2, 0), (2, 1)];++/// What a package may not do, however well it parses: bind anything the+/// host did not declare, or loop without an end. A community shader runs+/// on the person's GPU with the host's rights, and a loop with no bound+/// hangs the device for every process on the machine; a binding the host+/// does not know is one it cannot serve. Caught at load, where a broken+/// package is a load error and not a black frame.+fn budget(module: &naga::Module) -> Result<(), String> {+ for (_, global) in module.global_variables.iter() {+ if let Some(binding) = &global.binding+ && !BINDINGS.contains(&(binding.group, binding.binding))+ {+ return Err(format!(+ "the shader binds @group({}) @binding({}), which the host does not provide",+ binding.group, binding.binding+ ));+ }+ }+ for (_, function) in module.functions.iter() {+ bounded(&function.body)?;+ }+ for entry in &module.entry_points {+ bounded(&entry.function.body)?;+ }+ Ok(())+}++/// Every loop in `block` has a way out: a `break` in its body or a+/// `break if` in its continuing block.+fn bounded(block: &naga::Block) -> Result<(), String> {+ for statement in block.iter() {+ match statement {+ naga::Statement::Loop {+ body,+ continuing,+ break_if,+ } => {+ if break_if.is_none() && !breaks(body) {+ return Err(+ "the shader has a loop with no break: it would never end".to_owned()+ );+ }+ bounded(body)?;+ bounded(continuing)?;+ }+ naga::Statement::Block(inner) => bounded(inner)?,+ naga::Statement::If { accept, reject, .. } => {+ bounded(accept)?;+ bounded(reject)?;+ }+ naga::Statement::Switch { cases, .. } => {+ for case in cases {+ bounded(&case.body)?;+ }+ }+ _ => {}+ }+ }+ Ok(())+}++/// Whether `block` breaks out of the loop it is the body of, at any+/// depth short of a nested loop, whose breaks are its own.+fn breaks(block: &naga::Block) -> bool {+ block.iter().any(|statement| match statement {+ naga::Statement::Break => true,+ naga::Statement::Block(inner) => breaks(inner),+ naga::Statement::If { accept, reject, .. } => breaks(accept) || breaks(reject),+ naga::Statement::Switch { cases, .. } => cases.iter().any(|case| breaks(&case.body)),+ _ => false,+ })+}+fn is_f32(scalar: &naga::Scalar) -> bool {scalar.kind == naga::ScalarKind::Float && scalar.width == 4}@@ -641,6 +718,43 @@ fn effect(uv: vec2<f32>) -> vec4<f32> {assert_eq!(shader.params_bytes(&BTreeMap::new(), &[]).len(), 16);}+ /// A community shader may not bind what the host did not declare,+ /// and may not loop without an end: both are refused at load, before+ /// the package can reach a device. A loop with a way out is fine.+ #[test]+ fn a_hostile_shader_is_refused_at_load() {+ let manifest = manifest("");+ let unbounded = Shader::compile(+ &manifest,+ "fn effect(uv: vec2<f32>) -> vec4<f32> { var c = sample(uv); loop { c.r = c.r * 0.5; } return c; }",+ );+ assert!(+ unbounded.unwrap_err().contains("no break"),+ "an endless loop"+ );+ let nested = Shader::compile(+ &manifest,+ "fn effect(uv: vec2<f32>) -> vec4<f32> { var c = sample(uv); for (var i = 0; i < 4; i++) { loop { c.r = c.r * 0.5; } } return c; }",+ );+ assert!(+ nested.unwrap_err().contains("no break"),+ "an endless loop inside a bounded one"+ );+ let extra = Shader::compile(+ &manifest,+ "@group(3) @binding(0) var other: texture_2d<f32>;\nfn effect(uv: vec2<f32>) -> vec4<f32> { return sample(uv) + textureSample(other, source_sampler, uv); }",+ );+ assert!(+ extra.unwrap_err().contains("@group(3)"),+ "a binding the host does not provide"+ );+ let bounded = Shader::compile(+ &manifest,+ "fn effect(uv: vec2<f32>) -> vec4<f32> { var c = sample(uv); var i = 0; loop { i++; if (i > 3) { break; } c.r = c.r * 0.5; } for (var j = 0; j < 2; j++) { c.g = c.g * 0.5; } return c; }",+ );+ assert!(bounded.is_ok(), "{:?}", bounded.err());+ }+#[test]fn a_missing_field_or_a_broken_body_is_refused() {let manifest = manifest(diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rsindex 2587bfa9..f7bfa82e 100644--- a/src/crates/concat-export/src/lib.rs+++ b/src/crates/concat-export/src/lib.rs@@ -43,7 +43,7 @@ use concat_media::{};use concat_project::model::{AppliedFilter, Cutout};use concat_render::{- Compositor, CpuCompositor, Layer, Placement, Treatment as GpuTreatment, plan_frame,+ Compositor, CpuCompositor, FramePlan, PlannedLayer, PlannedTreatment, plan_frame,};use concat_vision::{Mapping, MaskStore};use serde::Deserialize;@@ -885,48 +885,30 @@ pub fn audio_pieces(clip: &ExportClip) -> Vec<AudioClip> {.collect()}-/// One decoded picture on its way to the compositor, with everything the-/// layer it becomes needs. Generic over how the frame is held: owned in the-/// export, shared out of the reader pool in the monitor.-struct Source<F> {- frame: F,- opacity: f32,- transform: Transform,+/// The plan's layer for one clip at one instant, filled in: the picture+/// decoded, its effects resolved, on its track. The crop, the flips and+/// the transition fades come baked from the decoder's chain, and the+/// cutout has already cut the picture, so the layer carries none of+/// them; moving those into the plan is the next step, and the plan has+/// the fields for it.+fn filled(+ layer: &PlannedLayer,+ frame: std::sync::Arc<Frame>,track: usize,- blend: concat_core::timeline::Blend,- passes: Vec<ShaderPass>,+ effects: Vec<ShaderPass>,+) -> PlannedLayer {+ PlannedLayer {+ source: Some(frame),+ track,+ effects,+ ..layer.clone()+ }}-/// The fraction-to-pixel placement of one composited layer: fitted and-/// centred is the base, the clip's transform moves it from there. The one-/// definition the exporter and the preview share - these two paths must-/// never place a picture differently, or the paused truth lies about the-/// file.-fn place_layer<'a>(- frame: &'a Frame,- opacity: f32,- transform: &Transform,- width: u32,- height: u32,-) -> Layer<'a> {- let x = (i64::from(width) - i64::from(frame.width())) / 2;- let y = (i64::from(height) - i64::from(frame.height())) / 2;- let placement = if transform.is_identity() {- Placement::IDENTITY- } else {- Placement {- scale: transform.scale as f32,- rotation: transform.rotation.to_radians() as f32,- translate_x: (transform.offset_x * f64::from(width)) as f32,- translate_y: (transform.offset_y * f64::from(height)) as f32,- stretch_x: transform.stretch_x as f32,- stretch_y: transform.stretch_y as f32,- }- };- Layer::new(frame)- .at(x as i32, y as i32)- .with_opacity(opacity)- .with_placement(placement)+/// A stack already drawn, as a layer over the whole frame on the lowest+/// track: texel for pixel, since it is the frame's own size.+fn ground_layer(ground: Frame) -> PlannedLayer {+ PlannedLayer::picture(concat_render::detached_clip(), std::sync::Arc::new(ground))}/// The best compositor this machine offers: the GPU when the `gpu` feature is@@ -996,7 +978,7 @@ fn render_picture(let time = rate.time_of_frame(index);let plan = plan_frame(&timeline, time);- let mut sources: Vec<Source<Frame>> = Vec::with_capacity(plan.layers.len());+ let mut layers: Vec<PlannedLayer> = Vec::with_capacity(plan.layers.len());for layer in &plan.layers {if !layer.paced {let (decode_width, decode_height) = decode_sizes@@ -1017,15 +999,13 @@ fn render_picture(let frame = cutouts.get(&layer.clip).and_then(|job| job.cut(&frame, layer.source_time))- .unwrap_or_else(|| frame.as_ref().clone());- sources.push(Source {+ .map_or(frame, std::sync::Arc::new);+ layers.push(filled(+ layer,frame,- opacity: layer.opacity,- transform: layer.transform,- track: tracks.get(&layer.clip).copied().unwrap_or(0),- blend: layer.blend,- passes: passes_at(&chains, &timeline, layer.clip, time),- });+ tracks.get(&layer.clip).copied().unwrap_or(0),+ passes_at(&chains, &timeline, layer.clip, time),+ ));}continue;}@@ -1091,43 +1071,24 @@ fn render_picture(Some(cut) => cut,None => frame,};- sources.push(Source {- frame,- opacity: layer.opacity,- transform: layer.transform,- track: tracks.get(&layer.clip).copied().unwrap_or(0),- blend: layer.blend,- passes: passes_at(&chains, &timeline, layer.clip, time),- });+ layers.push(filled(+ layer,+ std::sync::Arc::new(frame),+ tracks.get(&layer.clip).copied().unwrap_or(0),+ passes_at(&chains, &timeline, layer.clip, time),+ ));}}- let seconds = time.as_f64() as f32;- let layers: Vec<(Layer<'_>, usize)> = sources- .iter()- .map(|source| {- (- place_layer(- &source.frame,- source.opacity,- &source.transform,- request.width,- request.height,- )- .with_blend(source.blend)- .with_passes(&source.passes)- .at_time(seconds),- source.track,- )- })- .collect();-let composed = composite_treated(&mut *compositor,- request.width,- request.height,- time,- &layers,+ FramePlan {+ time,+ width: request.width,+ height: request.height,+ layers,+ treatments: Vec::new(),+ },&treatments,);encoder@@ -1176,63 +1137,59 @@ fn mix(a: &Frame, b: &Frame, amount: f32) -> Frame {out}-/// Composites `sources` - each placed layer with the track it came from,-/// bottom-most first - with every treatment live at `time` applied to the-/// stack beneath its track. Without a live treatment this is one composite;-/// with one, the stack is drawn up to the treatment's track, run through its-/// chain, blended back by its strength, and used as the ground for the rest.+/// Draws `plan` with every treatment live at its instant applied to the+/// stack beneath its track. A treatment that is shaders alone goes into+/// the plan, and either compositor applies it where it draws; one that+/// needs FFmpeg for a package with no shader cannot, so the stack is+/// drawn up to its track, run through its chain here, blended back by+/// its strength, and used as the ground for the rest.fn composite_treated(compositor: &mut dyn Compositor,- width: u32,- height: u32,- time: Rational,- sources: &[(Layer<'_>, usize)],+ mut plan: FramePlan,treatments: &[Treatment],) -> Frame {+ let time = plan.time;let mut live: Vec<&Treatment> = treatments.iter().filter(|treatment| treatment.covers(time)).collect();if live.is_empty() {- let layers: Vec<Layer<'_>> = sources.iter().map(|(layer, _)| *layer).collect();- return compositor.composite(width, height, &layers);+ plan.treatments.clear();+ return compositor.render(&plan);}live.sort_by_key(|treatment| treatment.track);-- // A compositor that runs passes applies every treatment without a- // pixel leaving the GPU, so long as none of them needs FFmpeg for- // a package that has no shader.if live.iter().all(|treatment| treatment.chain.is_empty()) {- let resolved: Vec<(Vec<ShaderPass>, f32)> = live- .iter()- .map(|treatment| (treatment.passes_at(time), treatment.strength_at(time)))- .collect();- let gpu: Vec<GpuTreatment<'_>> = live+ plan.treatments = live.iter()- .zip(&resolved)- .map(|(treatment, (passes, strength))| GpuTreatment {+ .map(|treatment| PlannedTreatment {track: treatment.track,- passes,- strength: *strength,+ effects: treatment.passes_at(time),+ strength: treatment.strength_at(time),}).collect();- if let Some(frame) =- compositor.composite_treated(width, height, time.as_f64() as f32, sources, &gpu)- {- return frame;- }+ return compositor.render(&plan);}+ let (width, height) = (plan.width, plan.height);+ let stage = |layers: Vec<PlannedLayer>| FramePlan {+ time,+ width,+ height,+ layers,+ treatments: Vec::new(),+ };+ let layers = std::mem::take(&mut plan.layers);let mut ground: Option<Frame> = None;let mut next = 0;for treatment in live {let below = {- let mut layers: Vec<Layer<'_>> = ground.as_ref().map(Layer::new).into_iter().collect();- while next < sources.len() && sources[next].1 < treatment.track {- layers.push(sources[next].0);+ let mut stack: Vec<PlannedLayer> =+ ground.take().map(ground_layer).into_iter().collect();+ while next < layers.len() && layers[next].track < treatment.track {+ stack.push(layers[next].clone());next += 1;}- compositor.composite(width, height, &layers)+ compositor.render(&stage(stack))};let strength = treatment.strength_at(time);let treated = if strength <= 0.0 {@@ -1244,10 +1201,9 @@ fn composite_treated(let shaded = if passes.is_empty() {below.clone()} else {- let ground = [Layer::new(&below)- .with_passes(&passes)- .at_time(time.as_f64() as f32)];- compositor.composite(width, height, &ground)+ let mut layer = ground_layer(below.clone());+ layer.effects = passes;+ compositor.render(&stage(vec![layer]))};let result = if treatment.chain.is_empty() {Ok(shaded)@@ -1265,9 +1221,9 @@ fn composite_treated(};ground = Some(treated);}- let mut layers: Vec<Layer<'_>> = ground.as_ref().map(Layer::new).into_iter().collect();- layers.extend(sources[next..].iter().map(|(layer, _)| *layer));- compositor.composite(width, height, &layers)+ let mut stack: Vec<PlannedLayer> = ground.map(ground_layer).into_iter().collect();+ stack.extend(layers.into_iter().skip(next));+ compositor.render(&stage(stack))}/// One paused-monitor frame: the same clip list the exporter takes, one@@ -1308,129 +1264,40 @@ pub fn preview_frame(Ok(sources.composite(&mut CpuCompositor).into_pixels())}-/// The decoded pictures under the playhead and where each goes: what a-/// compositor needs to draw the paused monitor's frame, decoded but not yet-/// drawn, so a caller with a GPU can draw them where they are shown.+/// The paused monitor's frame, described but not yet drawn: the plan+/// with every picture decoded and placed, so a caller with a GPU can draw+/// it where it is shown.pub struct PreviewSources {- sources: Vec<Source<std::sync::Arc<Frame>>>,- width: u32,- height: u32,- time: Rational,+ plan: FramePlan,treatments: Vec<Treatment>,- /// Each treatment's passes resolved at `time`, by the same index.- treatment_passes: Vec<Vec<ShaderPass>>,}impl PreviewSources {- /// The layers, placed the exporter's way, bottom-most first. Bare: a- /// caller drawing these itself is skipping the treatments, so check- /// [`PreviewSources::has_treatments`] first.- pub fn layers(&self) -> Vec<Layer<'_>> {- let seconds = self.time.as_f64() as f32;- self.sources- .iter()- .map(|source| {- place_layer(- source.frame.as_ref(),- source.opacity,- &source.transform,- self.width,- self.height,- )- .with_blend(source.blend)- .with_passes(&source.passes)- .at_time(seconds)- })- .collect()+ /// The frame's description, with every treatment that is shaders+ /// alone already in it; a compositor draws this and nothing else. A+ /// treatment that needs FFmpeg is not in it: see+ /// [`PreviewSources::needs_cpu`].+ pub fn plan(&self) -> &FramePlan {+ &self.plan}- /// Whether a layer clip is live at this instant, in which case the frame- /// cannot be drawn from `layers` alone: see [`PreviewSources::composite`].- pub fn has_treatments(&self) -> bool {+ /// Whether a live treatment needs FFmpeg for a package with no+ /// shader, in which case the frame can only be drawn whole through+ /// [`PreviewSources::composite`].+ pub fn needs_cpu(&self) -> bool {self.treatments.iter()- .any(|treatment| treatment.covers(self.time))- }-- /// The layers as [`PreviewSources::layers`] gives them, each with the- /// track it came from, for a compositor applying treatments itself.- pub fn placed(&self) -> Vec<(Layer<'_>, usize)> {- let seconds = self.seconds();- self.sources- .iter()- .map(|source| {- (- place_layer(- source.frame.as_ref(),- source.opacity,- &source.transform,- self.width,- self.height,- )- .with_blend(source.blend)- .with_passes(&source.passes)- .at_time(seconds),- source.track,- )- })- .collect()+ .any(|treatment| treatment.covers(self.plan.time) && !treatment.chain.is_empty())}- /// The treatments live at this instant, in ascending track order, as- /// a GPU compositor takes them - or None when one of them needs FFmpeg- /// for a package with no shader, and only [`PreviewSources::composite`]- /// can draw the frame.- pub fn live_treatments(&self) -> Option<Vec<GpuTreatment<'_>>> {- let mut live: Vec<(usize, &Treatment)> = self- .treatments- .iter()- .enumerate()- .filter(|(_, treatment)| treatment.covers(self.time))- .collect();- if live- .iter()- .any(|(_, treatment)| !treatment.chain.is_empty())- {- return None;- }- live.sort_by_key(|(_, treatment)| treatment.track);- Some(- live.iter()- .map(|(index, treatment)| GpuTreatment {- track: treatment.track,- passes: &self.treatment_passes[*index],- strength: treatment.strength_at(self.time),- })- .collect(),- )- }-- /// The instant, in seconds, for passes that move.+ /// The instant, in seconds.pub fn seconds(&self) -> f32 {- self.time.as_f64() as f32+ self.plan.seconds()}- /// The frame, treatments included, drawn with `compositor`.+ /// The frame, drawn by `compositor`, every treatment applied.pub fn composite(&self, compositor: &mut dyn Compositor) -> Frame {- let placed = self.placed();- composite_treated(- compositor,- self.width,- self.height,- self.time,- &placed,- &self.treatments,- )- }-- /// The output width the layers were placed for.- pub fn width(&self) -> u32 {- self.width- }-- /// The output height the layers were placed for.- pub fn height(&self) -> u32 {- self.height+ composite_treated(compositor, self.plan.clone(), &self.treatments)}}@@ -1492,7 +1359,7 @@ pub fn preview_sources_of(let time = quantise(seconds, rate);let plan_at = plan_frame(timeline, time);- let mut sources: Vec<Source<std::sync::Arc<Frame>>> = Vec::with_capacity(plan_at.layers.len());+ let mut layers: Vec<PlannedLayer> = Vec::with_capacity(plan_at.layers.len());let mut failures: Vec<String> = Vec::new();for layer in &plan_at.layers {let (decode_width, decode_height) = decode_sizes@@ -1524,14 +1391,12 @@ pub fn preview_sources_of(Some(drawn) => std::sync::Arc::new(drawn),None => frame,};- sources.push(Source {+ layers.push(filled(+ layer,frame,- opacity: layer.opacity,- transform: layer.transform,- track: tracks.get(&layer.clip).copied().unwrap_or(0),- blend: layer.blend,- passes: passes_at(chains, timeline, layer.clip, time),- })+ tracks.get(&layer.clip).copied().unwrap_or(0),+ passes_at(chains, timeline, layer.clip, time),+ ))}Err(error) => failures.push(format!("{}: {error}", layer.media.display())),}@@ -1541,22 +1406,42 @@ pub fn preview_sources_of(// frame: compositing zero sources yields opaque black, and the caller// would draw that "truth" over its own perfectly good approximation. An// *empty plan* still composites - a gap in the timeline really is black.- if sources.is_empty() && !plan_at.layers.is_empty() {+ if layers.is_empty() && !plan_at.layers.is_empty() {return Err(format!("no layer decoded for the paused preview: {}",failures.join(" / ")));}- Ok(PreviewSources {- sources,+ let mut frame_plan = FramePlan {+ time,width: plan.width,height: plan.height,- time,- treatment_passes: treatments+ layers,+ treatments: Vec::new(),+ };+ // Every treatment that is shaders alone goes into the plan now, so a+ // caller drawing the plan itself has them; see+ // `PreviewSources::needs_cpu` for the one kind that cannot.+ let live: Vec<&Treatment> = treatments+ .iter()+ .filter(|treatment| treatment.covers(time))+ .collect();+ if live.iter().all(|treatment| treatment.chain.is_empty()) {+ frame_plan.treatments = live.iter()- .map(|treatment| treatment.passes_at(time))- .collect(),+ .map(|treatment| PlannedTreatment {+ track: treatment.track,+ effects: treatment.passes_at(time),+ strength: treatment.strength_at(time),+ })+ .collect();+ frame_plan+ .treatments+ .sort_by_key(|treatment| treatment.track);+ }+ Ok(PreviewSources {+ plan: frame_plan,treatments: treatments.clone(),})}@@ -1728,7 +1613,28 @@ mod tests {// track 2 at the top-left.let red = solid(8, 8, [255, 0, 0, 255]);let blue = solid(2, 2, [0, 0, 255, 255]);- let sources = [(Layer::new(&red), 0usize), (Layer::new(&blue), 2usize)];+ let stack = |time: Rational| {+ let ground = ground_layer(red.clone());+ let mut top = PlannedLayer::picture(+ concat_render::detached_clip(),+ std::sync::Arc::new(blue.clone()),+ );+ top.track = 2;+ // Centred at (4, 4) by the fit; three pixels up and left puts+ // it in the corner.+ top.transform = Transform {+ offset_x: -3.0 / 8.0,+ offset_y: -3.0 / 8.0,+ ..Transform::default()+ };+ FramePlan {+ time,+ width: 8,+ height: 8,+ layers: vec![ground, top],+ treatments: Vec::new(),+ }+ };let negate = Treatment {start: Rational::ZERO,end: Rational::from_int(10),@@ -1742,10 +1648,7 @@ mod tests {let mut compositor = CpuCompositor;let out = composite_treated(&mut compositor,- 8,- 8,- Rational::from_int(1),- &sources,+ stack(Rational::from_int(1)),std::slice::from_ref(&negate),);// Red negated is cyan where nothing sits on top...@@ -1759,27 +1662,13 @@ mod tests {strength: 0.5,..negate.clone()};- let out = composite_treated(- &mut compositor,- 8,- 8,- Rational::from_int(1),- &sources,- &[half],- );+ let out = composite_treated(&mut compositor, stack(Rational::from_int(1)), &[half]);let pixel = &out.pixels()[(7 * 8 + 7) * 4..(7 * 8 + 7) * 4 + 3];assert!(pixel[0] > 120 && pixel[0] < 136, "{pixel:?}");assert!(pixel[1] > 120 && pixel[1] < 136, "{pixel:?}");// Outside its span the treatment does nothing.- let out = composite_treated(- &mut compositor,- 8,- 8,- Rational::from_int(20),- &sources,- &[negate],- );+ let out = composite_treated(&mut compositor, stack(Rational::from_int(20)), &[negate]);assert_eq!(at_of(&out, 7, 7), [255, 0, 0]);fn at_of(frame: &Frame, x: usize, y: usize) -> [u8; 3] {let p = &frame.pixels()[(y * 8 + x) * 4..(y * 8 + x) * 4 + 3];diff --git a/src/crates/concat-host/src/preview.rs b/src/crates/concat-host/src/preview.rsindex d7beeb91..9c57737c 100644--- a/src/crates/concat-host/src/preview.rs+++ b/src/crates/concat-host/src/preview.rs@@ -160,30 +160,21 @@ impl Monitor {.as_ref().ok_or_else(|| "the monitor has no GPU device".to_owned())?;let mut gpu = gpu.lock().map_err(|_| "compositor poisoned".to_owned())?;- if sources.has_treatments() {- // A layer whose look is a shader is applied where the stack is- // drawn, on the GPU; only a layer that needs FFmpeg for a- // package with no shader takes the frame through the CPU.- if let Some(treatments) = sources.live_treatments() {- let layers = sources.placed();- return gpu- .composite_texture_treated(- spec.width,- spec.height,- sources.seconds(),- &layers,- &treatments,- )- .ok_or_else(|| "the GPU device was lost".to_owned());- }+ if sources.needs_cpu() {+ // A layer that needs FFmpeg for a package with no shader takes+ // the frame through the CPU; the picture then goes up as one+ // layer of its own.let frame = sources.composite(&mut *gpu);- let layers = [concat_render::Layer::new(&frame)];+ let mut plan = concat_render::FramePlan::empty(spec.width, spec.height);+ plan.layers.push(concat_render::PlannedLayer::picture(+ concat_render::detached_clip(),+ Arc::new(frame),+ ));return gpu- .composite_texture(spec.width, spec.height, &layers)+ .render_texture(&plan).ok_or_else(|| "the GPU device was lost".to_owned());}- let layers = sources.layers();- gpu.composite_texture(spec.width, spec.height, &layers)+ gpu.render_texture(sources.plan()).ok_or_else(|| "the GPU device was lost".to_owned())}diff --git a/src/crates/concat-render/Cargo.toml b/src/crates/concat-render/Cargo.tomlindex bec4cd01..5e6ebe7f 100644--- a/src/crates/concat-render/Cargo.toml+++ b/src/crates/concat-render/Cargo.toml@@ -19,6 +19,8 @@ wgpu = ["gpu"][dependencies]concat-core.workspace = true+# The CPU compositor says, once, when it draws a package untreated.+log = "0.4"wgpu = { workspace = true, optional = true }pollster = { workspace = true, optional = true }diff --git a/src/crates/concat-render/src/compositor.rs b/src/crates/concat-render/src/compositor.rsindex 82162eb3..a447e043 100644--- a/src/crates/concat-render/src/compositor.rs+++ b/src/crates/concat-render/src/compositor.rs@@ -1,175 +1,39 @@// SPDX-License-Identifier: AGPL-3.0-or-later// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors-//! Blending layers into one frame.+//! Blending a plan's layers into one frame.//!-//! [`CpuCompositor`] is the reference implementation: obvious, dependency-free,-//! and slow enough that it will eventually only be used for tests and for-//! checking the GPU path's output. The `Compositor` trait is the seam that-//! `wgpu` will slot into.+//! [`CpuCompositor`] is the reference implementation: obvious and+//! dependency-free, the one the tests read and the one a machine without+//! a GPU renders with. The [`Compositor`] trait is the seam the wgpu+//! compositor slots into, and a [`FramePlan`] is all either takes.+//!+//! The picture of a layer is made the way [`crate::plan`] describes: the+//! crop, the flips and the fit, then the effects, then the mask, the fades+//! and the wipes at the placement. Effects run through the CPU kernels in+//! [`crate::kernels`], keyed by the package; a package without one is drawn+//! untreated and said so once, which is an honest fallback where a second+//! implementation would be a drifting one.++use std::borrow::Cow;use concat_core::frame::{BYTES_PER_PIXEL, Frame};use concat_core::shader::ShaderPass;use concat_core::timeline::Blend;-/// A layer's placement beyond its base position, in output pixels.-///-/// Applied about the layer's centre: scale first, then rotation, then the-/// translation. Pixel-space on purpose - the resolution-independent form-/// lives in `concat-core::timeline::Transform`, and whoever builds layers-/// converts exactly once.-#[derive(Clone, Copy, PartialEq, Debug)]-pub struct Placement {- /// Multiplier over the layer's own size.- pub scale: f32,- /// Clockwise rotation about the layer's centre, in radians.- pub rotation: f32,- /// Horizontal offset of the layer's centre from its base position, pixels.- pub translate_x: f32,- /// Vertical offset of the layer's centre from its base position, pixels.- pub translate_y: f32,- /// A multiplier on the width beyond `scale`; 1 keeps the aspect.- pub stretch_x: f32,- /// The same for the height.- pub stretch_y: f32,-}--impl Placement {- /// Unscaled, unrotated, unmoved.- pub const IDENTITY: Placement = Placement {- scale: 1.0,- rotation: 0.0,- translate_x: 0.0,- translate_y: 0.0,- stretch_x: 1.0,- stretch_y: 1.0,- };-- /// True when applying this placement would change nothing.- pub fn is_identity(&self) -> bool {- *self == Self::IDENTITY- }-}--impl Default for Placement {- fn default() -> Self {- Self::IDENTITY- }-}--/// One source image, positioned and weighted.-#[derive(Clone, Copy, Debug)]-pub struct Layer<'a> {- /// The pixels to draw.- pub frame: &'a Frame,- /// Blend strength over what is beneath, in `0.0..=1.0`.- pub opacity: f32,- /// Horizontal offset of the layer's top-left corner. May be negative.- pub x: i32,- /// Vertical offset of the layer's top-left corner. May be negative.- pub y: i32,- /// Scale, rotation and translation about the layer's centre.- pub placement: Placement,- /// How the layer's colour meets the ground.- pub blend: Blend,- /// Shader passes to run over the layer's pixels before it is placed, in- /// order. The GPU compositor runs them; the CPU reference cannot, and- /// draws the layer untreated.- pub passes: &'a [ShaderPass],- /// Seconds into the timeline, for passes that move.- pub time: f32,-}--impl<'a> Layer<'a> {- /// A layer drawn at the origin, fully opaque.- pub fn new(frame: &'a Frame) -> Self {- Self {- frame,- opacity: 1.0,- x: 0,- y: 0,- placement: Placement::IDENTITY,- blend: Blend::Normal,- passes: &[],- time: 0.0,- }- }-- /// Sets the shader passes to run over the layer first.- pub fn with_passes(mut self, passes: &'a [ShaderPass]) -> Self {- self.passes = passes;- self- }-- /// Sets the timeline instant the passes see.- pub fn at_time(mut self, time: f32) -> Self {- self.time = time;- self- }-- /// Sets the blend mode.- pub fn with_blend(mut self, blend: Blend) -> Self {- self.blend = blend;- self- }-- /// Sets the blend strength.- pub fn with_opacity(mut self, opacity: f32) -> Self {- self.opacity = opacity;- self- }-- /// Sets the top-left corner.- pub fn at(mut self, x: i32, y: i32) -> Self {- self.x = x;- self.y = y;- self- }-- /// Sets the scale/rotation/translation.- pub fn with_placement(mut self, placement: Placement) -> Self {- self.placement = placement;- self- }-}+use crate::kernels;+use crate::plan::{FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading};-/// A layer clip at one instant, for a compositor that can apply it over-/// the stack beneath its track without leaving the GPU: its passes, and how-/// hard the result is blended back over the untreated stack.-#[derive(Clone, Copy, Debug)]-pub struct Treatment<'a> {- /// The track the layer sits on; every layer on a lower track is under it.- pub track: usize,- /// The passes to run over the stack beneath, in order.- pub passes: &'a [ShaderPass],- /// How much of the treated stack to keep over the untreated, `0..=1`.- pub strength: f32,-}--/// Blends layers into a single output frame.+/// Draws a plan into a frame.pub trait Compositor {- /// Draws `layers` bottom-most first over an opaque black background.+ /// Draws `plan`'s layers bottom-most first over an opaque black+ /// background, every treatment applied over the stack beneath its+ /// track.////// Layers may hang off any edge; anything outside the output is clipped./// The result is always fully opaque - it is what goes to screen or to an/// encoder, and neither has anything to show through.- fn composite(&mut self, width: u32, height: u32, layers: &[Layer<'_>]) -> Frame;-- /// Draws `layers`, each with the track it came from and bottom-most- /// first, with every treatment applied over the stack beneath its- /// track, the treatments in ascending track order. `None` from a- /// compositor that cannot run passes, which the CPU reference cannot:- /// the caller then applies the treatments its own way.- fn composite_treated(- &mut self,- _width: u32,- _height: u32,- _time: f32,- _layers: &[(Layer<'_>, usize)],- _treatments: &[Treatment<'_>],- ) -> Option<Frame> {- None- }+ fn render(&mut self, plan: &FramePlan) -> Frame;}/// A straightforward CPU compositor.@@ -177,41 +41,173 @@ pub trait Compositor {pub struct CpuCompositor;impl Compositor for CpuCompositor {- fn composite(&mut self, width: u32, height: u32, layers: &[Layer<'_>]) -> Frame {- let mut output = Frame::black(width, height);-- for layer in layers {- let opacity = layer.opacity.clamp(0.0, 1.0);- if opacity <= 0.0 {+ fn render(&mut self, plan: &FramePlan) -> Frame {+ let mut ground = Frame::black(plan.width, plan.height);+ let mut treatments: Vec<&PlannedTreatment> = plan.treatments.iter().collect();+ treatments.sort_by_key(|treatment| treatment.track);++ // The stack is drawn up to each treatment's track, treated, and+ // blended back by the strength; the result is the ground the rest+ // is drawn on.+ let mut next = 0;+ for treatment in treatments {+ while next < plan.layers.len() && plan.layers[next].track < treatment.track {+ draw(&mut ground, plan, &plan.layers[next]);+ next += 1;+ }+ let strength = treatment.strength.clamp(0.0, 1.0);+ if strength <= 0.0 || treatment.effects.is_empty() {continue;}+ let treated = run_effects(&ground, &treatment.effects, plan.seconds());+ ground = if strength >= 1.0 {+ treated+ } else {+ mix_frames(&ground, &treated, strength)+ };+ }+ for layer in &plan.layers[next..] {+ draw(&mut ground, plan, layer);+ }+ ground+ }+}- let source = layer.frame;+/// `picture` through `effects` in order: each package's kernel at full+/// strength, mixed back over the untouched picture by the pass's intensity+/// the way the shader's last line does. A package without a kernel leaves+/// the picture as it was.+pub(crate) fn run_effects(picture: &Frame, effects: &[ShaderPass], seconds: f32) -> Frame {+ let mut current = picture.clone();+ for pass in effects {+ let Some(treated) = kernels::run(pass, ¤t, seconds) else {+ kernels::fallback_once(&pass.package);+ continue;+ };+ let intensity = pass.intensity.clamp(0.0, 1.0);+ current = if intensity >= 1.0 {+ treated+ } else {+ mix_frames(¤t, &treated, intensity)+ };+ }+ current+}- if !layer.placement.is_identity() {- blend_transformed(&mut output, layer, opacity);- continue;+/// `a` towards `b` by `amount`, per channel.+pub(crate) fn mix_frames(a: &Frame, b: &Frame, amount: f32) -> Frame {+ let amount = amount.clamp(0.0, 1.0);+ let mut out = a.clone();+ for (pixel, over) in out.pixels_mut().iter_mut().zip(b.pixels().iter()) {+ let base = f32::from(*pixel);+ *pixel = (base + (f32::from(*over) - base) * amount).round() as u8;+ }+ out+}++/// Draws one layer over `output`.+fn draw(output: &mut Frame, plan: &FramePlan, layer: &PlannedLayer) {+ let opacity = layer.weight();+ if opacity <= 0.0 {+ return;+ }+ let Some(source) = &layer.source else {+ return;+ };+ let geometry = layer.geometry(source, plan.width, plan.height);++ // The picture the placement samples: the source as it came, or made+ // first when the effects have to see it cropped, flipped and fitted.+ let (picture, geometry, flip_h, flip_v): (Cow<'_, Frame>, Geometry, bool, bool) =+ if layer.needs_preparing(&geometry) {+ let made = prepare(source, &geometry, layer.flip_h, layer.flip_v);+ let treated = run_effects(&made, &layer.effects, plan.seconds());+ (Cow::Owned(treated), geometry.prepared(), false, false)+ } else if !layer.effects.is_empty() {+ let treated = run_effects(source, &layer.effects, plan.seconds());+ (Cow::Owned(treated), geometry, layer.flip_h, layer.flip_v)+ } else {+ (+ Cow::Borrowed(source.as_ref()),+ geometry,+ layer.flip_h,+ layer.flip_v,+ )+ };++ let weigh = Weighing {+ shading: layer.shading(),+ mask: layer.mask.as_deref(),+ opacity,+ blend: layer.blend,+ };+ if geometry.is_aligned() && geometry.fits_source() && !flip_h && !flip_v {+ blend_aligned(output, &picture, &geometry, &weigh);+ } else {+ blend_transformed(output, &picture, &geometry, flip_h, flip_v, &weigh);+ }+}++/// The picture at its fitted size: the crop rectangle of `source`, flipped+/// as asked, resampled bilinearly. What the effects run over when the+/// source is not already that.+fn prepare(source: &Frame, geometry: &Geometry, flip_h: bool, flip_v: bool) -> Frame {+ let (width, height) = geometry.fitted;+ let mut out = Frame::transparent(width, height);+ let stride = width as usize * BYTES_PER_PIXEL;+ let pixels = out.pixels_mut();+ for y in 0..height {+ for x in 0..width {+ let u = (x as f32 + 0.5) / width as f32;+ let v = (y as f32 + 0.5) / height as f32;+ let (sx, sy) = geometry.source_of(u, v, flip_h, flip_v);+ let sample = sample_bilinear(source, sx, sy);+ let at = y as usize * stride + x as usize * BYTES_PER_PIXEL;+ for (channel, value) in sample.iter().enumerate() {+ pixels[at + channel] = value.round().clamp(0.0, 255.0) as u8;}+ }+ }+ out+}- let Some((dst_x, src_x, columns)) = overlap(layer.x, source.width(), width) else {- continue;- };- let Some((dst_y, src_y, rows)) = overlap(layer.y, source.height(), height) else {- continue;- };+/// What weighs a layer's pixel on its way to the ground: the fades and+/// wipes, the matte, the opacity, and the blend it meets the ground with.+struct Weighing<'a> {+ shading: Shading,+ mask: Option<&'a Frame>,+ opacity: f32,+ blend: Blend,+}- blend_region(- &mut output,- source,- (dst_x, dst_y),- (src_x, src_y),- (columns, rows),- opacity,- layer.blend,+impl Weighing<'_> {+ /// One pixel of the picture, `sample` its straight colour and alpha in+ /// `0..=255` at `u`, `v` across the picture, blended into `under`.+ #[inline]+ fn blend_into(&self, under: &mut [u8], sample: [f32; 4], u: f32, v: f32) {+ if !self.shading.keeps(u) {+ return;+ }+ let mut alpha = (sample[3] / 255.0) * self.opacity;+ if let Some(mask) = self.mask {+ let (mx, my) = (+ u * mask.width() as f32 - 0.5,+ v * mask.height() as f32 - 0.5,);+ alpha *= sample_bilinear(mask, mx, my)[3] / 255.0;}-- output+ if alpha <= 0.0 {+ return;+ }+ for channel in 0..3 {+ let colour = self.shading.colour(channel, sample[channel] / 255.0) * 255.0;+ let over = colour * alpha;+ let ground = f32::from(under[channel]);+ under[channel] = mix(self.blend, over, ground, alpha)+ .round()+ .clamp(0.0, 255.0) as u8;+ }+ under[3] = 255;}}@@ -230,30 +226,59 @@ fn mix(blend: Blend, over: f32, under: f32, alpha: f32) -> f32 {}}+/// `source` at texel coordinate `sx`, `sy` - its centre at `.0` - sampled+/// bilinearly with the edge texels clamped, the way the GPU's sampler+/// reads it. Straight colour and alpha in `0..=255`.+pub(crate) fn sample_bilinear(source: &Frame, sx: f32, sy: f32) -> [f32; 4] {+ let width = source.width() as usize;+ let height = source.height() as usize;+ let stride = width * BYTES_PER_PIXEL;+ let pixels = source.pixels();+ let x0 = (sx.floor().max(0.0) as usize).min(width - 1);+ let y0 = (sy.floor().max(0.0) as usize).min(height - 1);+ let x1 = (x0 + 1).min(width - 1);+ let y1 = (y0 + 1).min(height - 1);+ let fx = (sx - x0 as f32).clamp(0.0, 1.0);+ let fy = (sy - y0 as f32).clamp(0.0, 1.0);+ let mut sample = [0.0f32; 4];+ for (corner_x, corner_y, weight) in [+ (x0, y0, (1.0 - fx) * (1.0 - fy)),+ (x1, y0, fx * (1.0 - fy)),+ (x0, y1, (1.0 - fx) * fy),+ (x1, y1, fx * fy),+ ] {+ let at = corner_y * stride + corner_x * BYTES_PER_PIXEL;+ for channel in 0..4 {+ sample[channel] += f32::from(pixels[at + channel]) * weight;+ }+ }+ sample+}+/// Draws one layer through its placement: inverse-mapped, bilinearly sampled.////// Each covered output pixel is carried backwards through the placement into-/// source coordinates and sampled there. Inverse mapping is what makes the-/// result hole-free at any scale or angle; bilinear is the cheapest filter-/// that does not shimmer on motion. Only the transformed bounding box is+/// the picture, and from there through the flips and the crop into the+/// source, and sampled there. Inverse mapping is what makes the result+/// hole-free at any scale or angle; bilinear is the cheapest filter that+/// does not shimmer on motion. Only the transformed bounding box is/// visited, so a small layer stays cheap on a large frame.-fn blend_transformed(output: &mut Frame, layer: &Layer<'_>, opacity: f32) {- let source = layer.frame;- let placement = layer.placement;- // One factor per axis: the scale, and the stretch that axis carries.- let scale_x = (placement.scale * placement.stretch_x).max(1e-6);- let scale_y = (placement.scale * placement.stretch_y).max(1e-6);- let (sin, cos) = placement.rotation.sin_cos();-- let src_w = source.width() as f32;- let src_h = source.height() as f32;- // The layer's centre in output space, after translation.- let centre_x = layer.x as f32 + src_w / 2.0 + placement.translate_x;- let centre_y = layer.y as f32 + src_h / 2.0 + placement.translate_y;+fn blend_transformed(+ output: &mut Frame,+ picture: &Frame,+ geometry: &Geometry,+ flip_h: bool,+ flip_v: bool,+ weigh: &Weighing<'_>,+) {+ let (scale_x, scale_y) = geometry.scale;+ let (sin, cos) = geometry.rotation.sin_cos();+ let (fitted_w, fitted_h) = (geometry.fitted.0 as f32, geometry.fitted.1 as f32);+ let (centre_x, centre_y) = geometry.centre;// Bounding box of the transformed rectangle, clamped to the output.- let half_w = src_w * scale_x / 2.0;- let half_h = src_h * scale_y / 2.0;+ let half_w = fitted_w * scale_x / 2.0;+ let half_h = fitted_h * scale_y / 2.0;let reach_x = (half_w * cos.abs()) + (half_h * sin.abs());let reach_y = (half_w * sin.abs()) + (half_h * cos.abs());@@ -266,57 +291,24 @@ fn blend_transformed(output: &mut Frame, layer: &Layer<'_>, opacity: f32) {}let dst_stride = output.width() as usize * BYTES_PER_PIXEL;- let src_stride = source.width() as usize * BYTES_PER_PIXEL;- let src_pixels = source.pixels();let dst_pixels = output.pixels_mut();for y in y_from..y_to {for x in x_from..x_to {- // Sample at the pixel centre, mapped back into source space:+ // Sample at the pixel centre, mapped back into the picture:// untranslate, unrotate, unscale, then re-origin at the corner.let dx = (x as f32 + 0.5) - centre_x;let dy = (y as f32 + 0.5) - centre_y;- let sx = (dx * cos + dy * sin) / scale_x + src_w / 2.0 - 0.5;- let sy = (-dx * sin + dy * cos) / scale_y + src_h / 2.0 - 0.5;-- if sx < -0.5 || sy < -0.5 || sx > src_w - 0.5 || sy > src_h - 0.5 {+ let px = (dx * cos + dy * sin) / scale_x + fitted_w / 2.0;+ let py = (-dx * sin + dy * cos) / scale_y + fitted_h / 2.0;+ if px < 0.0 || py < 0.0 || px > fitted_w || py > fitted_h {continue;}-- let x0 = sx.floor().max(0.0) as usize;- let y0 = sy.floor().max(0.0) as usize;- let x1 = (x0 + 1).min(source.width() as usize - 1);- let y1 = (y0 + 1).min(source.height() as usize - 1);- let fx = (sx - x0 as f32).clamp(0.0, 1.0);- let fy = (sy - y0 as f32).clamp(0.0, 1.0);-- let mut sample = [0.0f32; 4];- for (corner_x, corner_y, weight) in [- (x0, y0, (1.0 - fx) * (1.0 - fy)),- (x1, y0, fx * (1.0 - fy)),- (x0, y1, (1.0 - fx) * fy),- (x1, y1, fx * fy),- ] {- let at = corner_y * src_stride + corner_x * BYTES_PER_PIXEL;- for channel in 0..4 {- sample[channel] += f32::from(src_pixels[at + channel]) * weight;- }- }-- let alpha = (sample[3] / 255.0) * opacity;- if alpha <= 0.0 {- continue;- }-+ let (u, v) = (px / fitted_w, py / fitted_h);+ let (sx, sy) = geometry.source_of(u, v, flip_h, flip_v);+ let sample = sample_bilinear(picture, sx, sy);let at = y as usize * dst_stride + x as usize * BYTES_PER_PIXEL;- for channel in 0..3 {- let over = sample[channel] * alpha;- let under = f32::from(dst_pixels[at + channel]);- dst_pixels[at + channel] = mix(layer.blend, over, under, alpha)- .round()- .clamp(0.0, 255.0) as u8;- }- dst_pixels[at + 3] = 255;+ weigh.blend_into(&mut dst_pixels[at..at + BYTES_PER_PIXEL], sample, u, v);}}}@@ -338,75 +330,115 @@ fn overlap(offset: i32, source: u32, destination: u32) -> Option<(u32, u32, u32)(count > 0).then_some((dst_start as u32, src_start as u32, count as u32))}-/// Source-over alpha blend of an aligned rectangle.-fn blend_region(- output: &mut Frame,- source: &Frame,- (dst_x, dst_y): (u32, u32),- (src_x, src_y): (u32, u32),- (columns, rows): (u32, u32),- opacity: f32,- blend: Blend,-) {- let src_stride = source.width() as usize * BYTES_PER_PIXEL;+/// Blends a picture placed by whole pixels, texel for pixel: the aligned+/// case, no resampling.+fn blend_aligned(output: &mut Frame, picture: &Frame, geometry: &Geometry, weigh: &Weighing<'_>) {+ let (x, y) = geometry.corner();+ let Some((dst_x, src_x, columns)) = overlap(x, picture.width(), output.width()) else {+ return;+ };+ let Some((dst_y, src_y, rows)) = overlap(y, picture.height(), output.height()) else {+ return;+ };+ let src_stride = picture.width() as usize * BYTES_PER_PIXEL;let dst_stride = output.width() as usize * BYTES_PER_PIXEL;let span = columns as usize * BYTES_PER_PIXEL;+ let (fitted_w, fitted_h) = (picture.width() as f32, picture.height() as f32);- let src_pixels = source.pixels();+ let src_pixels = picture.pixels();let dst_pixels = output.pixels_mut();for row in 0..rows as usize {let src_offset = (src_y as usize + row) * src_stride + src_x as usize * BYTES_PER_PIXEL;let dst_offset = (dst_y as usize + row) * dst_stride + dst_x as usize * BYTES_PER_PIXEL;+ let v = (src_y as f32 + row as f32 + 0.5) / fitted_h;let src_row = &src_pixels[src_offset..src_offset + span];let dst_row = &mut dst_pixels[dst_offset..dst_offset + span];- for (src_pixel, dst_pixel) in src_row+ for (column, (src_pixel, dst_pixel)) in src_row.chunks_exact(BYTES_PER_PIXEL).zip(dst_row.chunks_exact_mut(BYTES_PER_PIXEL))+ .enumerate(){- let alpha = (f32::from(src_pixel[3]) / 255.0) * opacity;- if alpha <= 0.0 {- continue;- }- for channel in 0..3 {- let over = f32::from(src_pixel[channel]) * alpha;- let under = f32::from(dst_pixel[channel]);- dst_pixel[channel] = mix(blend, over, under, alpha).round().clamp(0.0, 255.0) as u8;- }- dst_pixel[3] = 255;+ let u = (src_x as f32 + column as f32 + 0.5) / fitted_w;+ let sample = [+ f32::from(src_pixel[0]),+ f32::from(src_pixel[1]),+ f32::from(src_pixel[2]),+ f32::from(src_pixel[3]),+ ];+ weigh.blend_into(dst_pixel, sample, u, v);}}}#[cfg(test)]mod tests {+ use std::sync::Arc;++ use concat_core::time::FrameRate;+ use concat_core::timeline::{Clip, ClipId, MediaRef, Timeline, Track, TrackKind, Transform};++ use super::*;+ use crate::plan::{Crop, Transition};++ fn a_clip() -> ClipId {+ let mut timeline = Timeline::new(8, 8, FrameRate::THIRTY);+ let track = timeline.add_track(Track::new("V1", TrackKind::Video));+ timeline+ .add_clip(+ track,+ Clip::new(+ MediaRef::new("a.mp4"),+ concat_core::time::Rational::ZERO,+ concat_core::time::Rational::ONE,+ ),+ )+ .expect("track exists")+ }++ fn layer(frame: Frame) -> PlannedLayer {+ PlannedLayer::picture(a_clip(), Arc::new(frame))+ }++ fn plan(width: u32, height: u32, layers: Vec<PlannedLayer>) -> FramePlan {+ FramePlan {+ layers,+ ..FramePlan::empty(width, height)+ }+ }++ fn render(width: u32, height: u32, layers: Vec<PlannedLayer>) -> Frame {+ CpuCompositor.render(&plan(width, height, layers))+ }++ fn solid(width: u32, height: u32, rgba: [u8; 4]) -> Frame {+ let mut frame = Frame::transparent(width, height);+ frame.fill(rgba);+ frame+ }++ /// A layer moved by whole pixels; the transform is fractions of the+ /// output, so the test speaks in pixels and converts.+ fn moved(mut layer: PlannedLayer, x: f64, y: f64, width: u32, height: u32) -> PlannedLayer {+ layer.transform = Transform {+ offset_x: x / f64::from(width),+ offset_y: y / f64::from(height),+ ..Transform::default()+ };+ layer+ }+/// Multiply by white and screen with black both leave the ground as it/// was; add brightens it, darken cannot, lighten takes the brighter.#[test]fn the_blend_modes_do_what_their_names_say() {let over = |rgba: [u8; 4], blend: Blend| {- let ground_frame = {- let mut frame = Frame::black(2, 2);- for pixel in frame.pixels_mut().chunks_exact_mut(4) {- pixel.copy_from_slice(&[100, 100, 100, 255]);- }- frame- };- let mut layer_frame = Frame::black(2, 2);- for pixel in layer_frame.pixels_mut().chunks_exact_mut(4) {- pixel.copy_from_slice(&rgba);- }- let out = CpuCompositor.composite(- 2,- 2,- &[- Layer::new(&ground_frame),- Layer::new(&layer_frame).with_blend(blend),- ],- );- out.pixels()[0]+ let ground = solid(2, 2, [100, 100, 100, 255]);+ let mut top = layer(solid(2, 2, rgba));+ top.blend = blend;+ render(2, 2, vec![layer(ground), top]).pixels()[0]};assert_eq!(over([255, 255, 255, 255], Blend::Multiply), 100);assert_eq!(over([0, 0, 0, 255], Blend::Screen), 100);@@ -418,62 +450,67 @@ mod tests {assert_eq!(over([200, 200, 200, 255], Blend::Normal), 200);}- use super::*;-- fn solid(width: u32, height: u32, rgba: [u8; 4]) -> Frame {- let mut frame = Frame::transparent(width, height);- frame.fill(rgba);- frame- }-#[test]fn no_layers_gives_opaque_black() {- let frame = CpuCompositor.composite(2, 2, &[]);+ let frame = render(2, 2, vec![]);assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));}#[test]fn an_opaque_layer_replaces_the_background() {- let red = solid(2, 2, [255, 0, 0, 255]);- let frame = CpuCompositor.composite(2, 2, &[Layer::new(&red)]);+ let frame = render(2, 2, vec![layer(solid(2, 2, [255, 0, 0, 255]))]);assert_eq!(frame.pixel(1, 1), Some([255, 0, 0, 255]));}#[test]fn half_opacity_lands_halfway() {- let white = solid(1, 1, [255, 255, 255, 255]);- let frame = CpuCompositor.composite(1, 1, &[Layer::new(&white).with_opacity(0.5)]);+ let mut white = layer(solid(1, 1, [255, 255, 255, 255]));+ white.opacity = 0.5;+ let frame = render(1, 1, vec![white]);assert_eq!(frame.pixel(0, 0), Some([128, 128, 128, 255]));}#[test]fn source_alpha_and_layer_opacity_multiply() {- let half_alpha = solid(1, 1, [255, 255, 255, 128]);- let frame = CpuCompositor.composite(1, 1, &[Layer::new(&half_alpha).with_opacity(0.5)]);+ let mut half = layer(solid(1, 1, [255, 255, 255, 128]));+ half.opacity = 0.5;+ let frame = render(1, 1, vec![half]);// 128/255 * 0.5 ~= 0.251assert_eq!(frame.pixel(0, 0), Some([64, 64, 64, 255]));}#[test]fn later_layers_draw_on_top() {- let red = solid(1, 1, [255, 0, 0, 255]);- let blue = solid(1, 1, [0, 0, 255, 255]);- let frame = CpuCompositor.composite(1, 1, &[Layer::new(&red), Layer::new(&blue)]);+ let frame = render(+ 1,+ 1,+ vec![+ layer(solid(1, 1, [255, 0, 0, 255])),+ layer(solid(1, 1, [0, 0, 255, 255])),+ ],+ );assert_eq!(frame.pixel(0, 0), Some([0, 0, 255, 255]));}#[test]fn a_transparent_layer_changes_nothing() {- let clear = Frame::transparent(2, 2);- let frame = CpuCompositor.composite(2, 2, &[Layer::new(&clear)]);+ let frame = render(2, 2, vec![layer(Frame::transparent(2, 2))]);+ assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));+ }++ #[test]+ fn a_layer_without_a_picture_draws_nothing() {+ let mut missing = layer(solid(2, 2, [255, 0, 0, 255]));+ missing.source = None;+ let frame = render(2, 2, vec![missing]);assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));}#[test]fn offset_layers_are_clipped_not_wrapped() {- let red = solid(2, 2, [255, 0, 0, 255]);// Placed so only its bottom-right pixel lands on the output's top-left.- let frame = CpuCompositor.composite(2, 2, &[Layer::new(&red).at(-1, -1)]);+ let red = moved(layer(solid(2, 2, [255, 0, 0, 255])), -1.0, -1.0, 2, 2);+ let frame = render(2, 2, vec![red]);assert_eq!(frame.pixel(0, 0), Some([255, 0, 0, 255]));assert_eq!(frame.pixel(1, 1),@@ -484,42 +521,45 @@ mod tests {#[test]fn a_layer_entirely_off_screen_is_skipped() {- let red = solid(2, 2, [255, 0, 0, 255]);- let frame = CpuCompositor.composite(2, 2, &[Layer::new(&red).at(50, 50)]);+ let red = moved(layer(solid(2, 2, [255, 0, 0, 255])), 50.0, 50.0, 2, 2);+ let frame = render(2, 2, vec![red]);assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));}+ /// A picture larger than the output is fitted into it, not cropped:+ /// an 8 by 8 red square on a 2 by 2 output is the whole square, small.#[test]- fn a_layer_larger_than_the_output_is_cropped() {- let red = solid(8, 8, [255, 0, 0, 255]);- let frame = CpuCompositor.composite(2, 2, &[Layer::new(&red)]);+ fn a_layer_larger_than_the_output_is_fitted() {+ let frame = render(2, 2, vec![layer(solid(8, 8, [255, 0, 0, 255]))]);assert_eq!(frame.width(), 2);assert_eq!(frame.pixel(1, 1), Some([255, 0, 0, 255]));+ // And a wide picture on a square output keeps its shape: bars+ // above and below.+ let frame = render(4, 4, vec![layer(solid(8, 4, [255, 0, 0, 255]))]);+ assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));+ assert_eq!(frame.pixel(0, 1), Some([255, 0, 0, 255]));+ assert_eq!(frame.pixel(3, 2), Some([255, 0, 0, 255]));+ assert_eq!(frame.pixel(3, 3), Some([0, 0, 0, 255]));}#[test]fn scaling_doubles_coverage() {- let red = solid(2, 2, [255, 0, 0, 255]);- let placement = Placement {+ // A 2x2 layer centred on a 4x4 output scaled x2 about its centre+ // covers the whole output.+ let mut red = layer(solid(2, 2, [255, 0, 0, 255]));+ red.transform = Transform {scale: 2.0,- ..Placement::IDENTITY+ ..Transform::default()};- let frame =- CpuCompositor.composite(4, 4, &[Layer::new(&red).at(1, 1).with_placement(placement)]);- // A 2x2 layer based at (1,1) scaled x2 about its centre covers the- // whole 4x4 output.+ let frame = render(4, 4, vec![red]);assert_eq!(frame.pixel(0, 0), Some([255, 0, 0, 255]));assert_eq!(frame.pixel(3, 3), Some([255, 0, 0, 255]));}#[test]fn translation_moves_the_layer() {- let red = solid(1, 1, [255, 0, 0, 255]);- let placement = Placement {- translate_x: 2.0,- ..Placement::IDENTITY- };- let frame = CpuCompositor.composite(3, 1, &[Layer::new(&red).with_placement(placement)]);+ let red = moved(layer(solid(1, 1, [255, 0, 0, 255])), 1.0, 0.0, 3, 1);+ let frame = render(3, 1, vec![red]);assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));assert_eq!(frame.pixel(2, 0), Some([255, 0, 0, 255]));}@@ -529,27 +569,136 @@ mod tests {let mut strip = Frame::transparent(2, 1);strip.set_pixel(0, 0, [255, 0, 0, 255]);strip.set_pixel(1, 0, [0, 0, 255, 255]);- let placement = Placement {- rotation: std::f32::consts::PI,- ..Placement::IDENTITY+ let mut turned = layer(strip);+ turned.transform = Transform {+ rotation: 180.0,+ ..Transform::default()};- let frame = CpuCompositor.composite(2, 1, &[Layer::new(&strip).with_placement(placement)]);+ let frame = render(2, 1, vec![turned]);assert_eq!(frame.pixel(0, 0), Some([0, 0, 255, 255]));assert_eq!(frame.pixel(1, 0), Some([255, 0, 0, 255]));}#[test]fn a_transformed_layer_is_clipped_at_the_frame_edge() {- let red = solid(2, 2, [255, 0, 0, 255]);- let placement = Placement {+ let mut red = layer(solid(2, 2, [255, 0, 0, 255]));+ red.transform = Transform {scale: 100.0,- ..Placement::IDENTITY+ ..Transform::default()};- let frame = CpuCompositor.composite(2, 2, &[Layer::new(&red).with_placement(placement)]);+ let frame = render(2, 2, vec![red]);assert_eq!(frame.width(), 2);assert_eq!(frame.pixel(1, 1), Some([255, 0, 0, 255]));}+ /// A flip mirrors the picture, with and without a resample.+ #[test]+ fn flips_mirror_the_picture() {+ let mut strip = Frame::transparent(2, 2);+ strip.set_pixel(0, 0, [255, 0, 0, 255]);+ strip.set_pixel(1, 0, [0, 0, 255, 255]);+ strip.set_pixel(0, 1, [0, 255, 0, 255]);+ strip.set_pixel(1, 1, [255, 255, 255, 255]);+ let mut flipped = layer(strip.clone());+ flipped.flip_h = true;+ let frame = render(2, 2, vec![flipped]);+ assert_eq!(frame.pixel(0, 0), Some([0, 0, 255, 255]));+ assert_eq!(frame.pixel(1, 0), Some([255, 0, 0, 255]));+ let mut flipped = layer(strip);+ flipped.flip_v = true;+ let frame = render(2, 2, vec![flipped]);+ assert_eq!(frame.pixel(0, 0), Some([0, 255, 0, 255]));+ assert_eq!(frame.pixel(1, 1), Some([0, 0, 255, 255]));+ }++ /// A crop keeps the part it names and fits it to the output.+ #[test]+ fn a_crop_keeps_what_it_names() {+ let mut strip = Frame::transparent(4, 2);+ for x in 0..4 {+ for y in 0..2 {+ strip.set_pixel(+ x,+ y,+ if x < 2 {+ [255, 0, 0, 255]+ } else {+ [0, 0, 255, 255]+ },+ );+ }+ }+ let mut cropped = layer(strip);+ cropped.crop = Crop::of([0.5, 0.0, 0.0, 0.0]);+ let frame = render(2, 2, vec![cropped]);+ assert_eq!(frame.pixel(0, 0), Some([0, 0, 255, 255]));+ assert_eq!(frame.pixel(1, 1), Some([0, 0, 255, 255]));+ }++ /// The matte's alpha weighs the picture, sampled across it whatever+ /// its size; a fade pulls the colour; a wipe hides the far side.+ #[test]+ fn the_mask_the_fade_and_the_wipe_weigh_the_picture() {+ let red = solid(4, 4, [255, 0, 0, 255]);+ let mut matte = Frame::transparent(2, 1);+ matte.set_pixel(0, 0, [0, 0, 0, 255]);+ let mut masked = layer(red.clone());+ masked.mask = Some(Arc::new(matte));+ let frame = render(4, 4, vec![masked]);+ assert_eq!(frame.pixel(0, 0), Some([255, 0, 0, 255]));+ assert_eq!(frame.pixel(3, 3), Some([0, 0, 0, 255]));++ let mut faded = layer(red.clone());+ faded.transitions = vec![Transition::FadeTo {+ colour: [1.0; 3],+ amount: 0.5,+ }];+ let frame = render(4, 4, vec![faded]);+ assert_eq!(frame.pixel(0, 0), Some([255, 128, 128, 255]));++ let mut wiped = layer(red);+ wiped.transitions = vec![Transition::Wipe {+ uncovered: 0.5,+ from_right: true,+ }];+ let frame = render(4, 4, vec![wiped]);+ assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));+ assert_eq!(frame.pixel(3, 0), Some([255, 0, 0, 255]));+ }++ /// A treatment on track 1 runs over what track 0 drew and not over+ /// what track 2 draws on top of it, and its strength blends the result+ /// back. With no kernel for its package the ground is left as it was.+ #[test]+ fn a_treatment_without_a_kernel_leaves_the_stack_as_it_was() {+ let red = layer(solid(8, 8, [255, 0, 0, 255]));+ let mut blue = layer(solid(2, 2, [0, 0, 255, 255]));+ blue.track = 2;+ blue.transform = Transform {+ offset_x: -3.0 / 8.0,+ offset_y: -3.0 / 8.0,+ ..Transform::default()+ };+ let unknown = ShaderPass {+ package: "test.nothing".to_owned(),+ key: "test.nothing@1".to_owned(),+ source: Arc::from(""),+ params: vec![0; 16],+ values: Default::default(),+ intensity: 1.0,+ lut: None,+ };+ let mut frame_plan = plan(8, 8, vec![red, blue]);+ frame_plan.treatments = vec![PlannedTreatment {+ track: 1,+ effects: vec![unknown],+ strength: 1.0,+ }];+ let out = CpuCompositor.render(&frame_plan);+ assert_eq!(out.pixel(7, 7), Some([255, 0, 0, 255]));+ assert_eq!(out.pixel(0, 0), Some([0, 0, 255, 255]));+ }+#[test]fn overlap_spans_are_correct() {assert_eq!(overlap(0, 4, 4), Some((0, 0, 4)));diff --git a/src/crates/concat-render/src/gpu.rs b/src/crates/concat-render/src/gpu.rsindex 86b30b3c..275c892b 100644--- a/src/crates/concat-render/src/gpu.rs+++ b/src/crates/concat-render/src/gpu.rs@@ -32,19 +32,29 @@ use concat_core::frame::Frame;use concat_core::shader::{Lut, ShaderPass};use concat_core::timeline::Blend;-use crate::compositor::{Compositor, CpuCompositor, Layer, Treatment};+use crate::compositor::{Compositor, CpuCompositor};+use crate::plan::{FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading};/// Bytes per row must be a multiple of this for a texture-to-buffer copy.const ROW_ALIGN: usize = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT as usize;-/// One vertex of a layer quad: clip-space position, texel coordinates, and-/// the layer's opacity riding along so no uniforms are needed.+/// One vertex of a layer quad: clip-space position, the texel it samples,+/// and everything that weighs the pixel riding along - the opacity, the+/// fades folded into a scale and an offset of the colour, the wipes as+/// two edges, and where on the picture the pixel is - so no uniforms are+/// needed and one draw call is one layer.#[repr(C)]#[derive(Clone, Copy)]struct Vertex {position: [f32; 2],uv: [f32; 2],opacity: f32,+ scale: f32,+ offset: [f32; 3],+ edges: [f32; 2],+ /// `0..1` across the picture as it is seen: what the mask is sampled+ /// at and what the wipes measure.+ pic: [f32; 2],}/// Vertex data as raw bytes. `Vertex` is `repr(C)` and all `f32`, so its byte@@ -65,12 +75,20 @@ struct VsIn {@location(0) position: vec2<f32>,@location(1) uv: vec2<f32>,@location(2) opacity: f32,+ @location(3) scale: f32,+ @location(4) offset: vec3<f32>,+ @location(5) edges: vec2<f32>,+ @location(6) pic: vec2<f32>,}struct VsOut {@builtin(position) position: vec4<f32>,@location(0) uv: vec2<f32>,@location(1) opacity: f32,+ @location(2) scale: f32,+ @location(3) offset: vec3<f32>,+ @location(4) edges: vec2<f32>,+ @location(5) pic: vec2<f32>,}@vertex@@ -79,19 +97,31 @@ fn vs_main(in: VsIn) -> VsOut {out.position = vec4<f32>(in.position, 0.0, 1.0);out.uv = in.uv;out.opacity = in.opacity;+ out.scale = in.scale;+ out.offset = in.offset;+ out.edges = in.edges;+ out.pic = in.pic;return out;}@group(0) @binding(0) var layer_texture: texture_2d<f32>;@group(0) @binding(1) var layer_sampler: sampler;+@group(1) @binding(0) var mask_texture: texture_2d<f32>;+@group(1) @binding(1) var mask_sampler: sampler;@fragmentfn fs_main(in: VsOut) -> @location(0) vec4<f32> {let colour = textureSample(layer_texture, layer_sampler, in.uv);- let alpha = colour.a * in.opacity;+ let mask = textureSample(mask_texture, mask_sampler, in.pic);+ // The wipes: a pixel past the moving edge is not drawn at all.+ let kept = select(0.0, 1.0, in.pic.x < in.edges.x && in.pic.x >= in.edges.y);+ let alpha = colour.a * in.opacity * mask.a * kept;+ // The fades: the colour scaled and offset, the same line the CPU+ // reference computes.+ let shaded = colour.rgb * in.scale + in.offset;// Premultiplied output; the pipeline blends ONE / ONE_MINUS_SRC_ALPHA,// which together is the same source-over the CPU path computes.- return vec4<f32>(colour.rgb * alpha, alpha);+ return vec4<f32>(shaded * alpha, alpha);}"#;@@ -104,9 +134,15 @@ struct CompiledShader {bind_group: wgpu::BindGroup,}-/// One draw of a composite: the pooled texture's size and index, and how-/// it meets the ground.-type Draw = (u32, u32, usize, Blend);+/// One draw of a composite: the pooled texture drawn, how it meets the+/// ground, and the pooled texture masking it - the one white pixel for a+/// layer without a mask.+struct Draw {+ size: (u32, u32),+ texture: usize,+ blend: Blend,+ mask: (u32, u32, usize),+}/// A cached layer texture and its bind group, reusable for any layer of the/// same size.@@ -168,6 +204,8 @@ pub struct WgpuCompositor {idle: HashMap<(u32, u32), u32>,target: Option<Target>,presentable: Option<Presentable>,+ /// A one-pixel opaque white picture: the mask of a layer without one.+ white: Frame,/// Set when a readback fails - a lost or reset device. The compositor/// then answers every composite from the CPU reference instead: slower,/// always correct, and never a panic in the middle of an export.@@ -265,14 +303,19 @@ impl WgpuCompositor {});let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {label: Some("concat compositor"),- bind_group_layouts: &[Some(&bind_layout)],+ // Group 0 is the layer, group 1 its mask: the same shape, a+ // texture and a sampler.+ bind_group_layouts: &[Some(&bind_layout), Some(&bind_layout)],immediate_size: 0,});let vertex_layout = wgpu::VertexBufferLayout {array_stride: std::mem::size_of::<Vertex>() as u64,step_mode: wgpu::VertexStepMode::Vertex,- attributes: &wgpu::vertex_attr_array![0 => Float32x2, 1 => Float32x2, 2 => Float32],+ attributes: &wgpu::vertex_attr_array![+ 0 => Float32x2, 1 => Float32x2, 2 => Float32, 3 => Float32,+ 4 => Float32x3, 5 => Float32x2, 6 => Float32x2+ ],};// ONE / ONE_MINUS_SRC_ALPHA over premultiplied shader output is@@ -383,6 +426,11 @@ impl WgpuCompositor {target: None,presentable: None,idle: HashMap::new(),+ white: {+ let mut white = Frame::transparent(1, 1);+ white.fill([255, 255, 255, 255]);+ white+ },dead: false,}}@@ -433,70 +481,220 @@ impl WgpuCompositor {texture}- /// Uploads every visible layer and writes its quad; the draws, in order.- fn prepare(- &mut self,- width: u32,- height: u32,- layers: &[Layer<'_>],- ) -> Vec<(u32, u32, usize, Blend)> {+ /// Every layer of `plan` uploaded, treated and placed, with every+ /// treatment applied over the stack beneath its track without a pixel+ /// leaving the GPU: the stack below a treatment is drawn into a pooled+ /// texture, the passes run over that, and the result, blended back+ /// over the untreated stack by the strength, becomes the ground the+ /// rest is drawn on. What comes back are the draws and quads for the+ /// final render, into whichever target the caller wants.+ fn prepare(&mut self, plan: &FramePlan) -> (Vec<Draw>, Vec<Vertex>) {self.used.values_mut().for_each(|used| *used = 0);- let mut draws: Vec<(u32, u32, usize, Blend)> = Vec::with_capacity(layers.len());- let mut vertices: Vec<Vertex> = Vec::with_capacity(layers.len() * 6);- for layer in layers {- self.push_layer(&mut draws, &mut vertices, width, height, layer);+ let (width, height) = (plan.width, plan.height);+ let seconds = plan.seconds();+ let mut treatments: Vec<&PlannedTreatment> = plan.treatments.iter().collect();+ treatments.sort_by_key(|treatment| treatment.track);+ let mut ground: Option<usize> = None;+ let mut next = 0;+ for treatment in treatments {+ let mut draws = Vec::new();+ let mut vertices = Vec::new();+ if let Some(index) = ground {+ self.push_pooled(&mut draws, &mut vertices, width, height, index, 1.0);+ }+ while next < plan.layers.len() && plan.layers[next].track < treatment.track {+ self.push_layer(&mut draws, &mut vertices, plan, &plan.layers[next]);+ next += 1;+ }+ let below = self.render_pooled(width, height, &draws, &vertices, wgpu::Color::BLACK);+ let strength = treatment.strength.clamp(0.0, 1.0);+ if strength <= 0.0 || treatment.effects.is_empty() {+ ground = Some(below);+ continue;+ }+ let treated = self.run_passes(width, height, below, &treatment.effects, seconds);+ ground = Some(if strength >= 1.0 {+ treated+ } else {+ let mut draws = Vec::new();+ let mut vertices = Vec::new();+ self.push_pooled(&mut draws, &mut vertices, width, height, below, 1.0);+ self.push_pooled(&mut draws, &mut vertices, width, height, treated, strength);+ self.render_pooled(width, height, &draws, &vertices, wgpu::Color::BLACK)+ });}- self.write_vertices(&vertices);- draws+ let mut draws = Vec::new();+ let mut vertices = Vec::new();+ if let Some(index) = ground {+ self.push_pooled(&mut draws, &mut vertices, width, height, index, 1.0);+ }+ for layer in &plan.layers[next..] {+ self.push_layer(&mut draws, &mut vertices, plan, layer);+ }+ (draws, vertices)}- /// One layer's draw: its pixels uploaded, its passes run, its quad- /// placed in an output `width` by `height`.+ /// One layer's draw: its picture uploaded, made and treated, its quad+ /// placed. The picture the quad samples is the source itself, with the+ /// crop and the flips folded into the texel coordinates, unless the+ /// effects have to see it cropped, flipped and fitted first - then it+ /// is made at its fitted size the way the CPU reference makes it, and+ /// the effects run over that.fn push_layer(&mut self,- draws: &mut Vec<(u32, u32, usize, Blend)>,+ draws: &mut Vec<Draw>,vertices: &mut Vec<Vertex>,- width: u32,- height: u32,- layer: &Layer<'_>,+ plan: &FramePlan,+ layer: &PlannedLayer,) {- if layer.opacity <= 0.0 {+ let opacity = layer.weight();+ if opacity <= 0.0 {return;}- let mut index = self.upload(layer.frame);- if !layer.passes.is_empty() {- index = self.run_passes(- layer.frame.width(),- layer.frame.height(),- index,- layer.passes,- layer.time,+ let Some(source) = &layer.source else {+ return;+ };+ let geometry = layer.geometry(source, plan.width, plan.height);+ let seconds = plan.seconds();+ let (size, texture, uvs, flips) = if layer.needs_preparing(&geometry) {+ let uploaded = self.upload(source);+ let made = self.make_picture(+ uploaded,+ (source.width(), source.height()),+ &geometry,+ layer.flip_h,+ layer.flip_v,);- }- draws.push((- layer.frame.width(),- layer.frame.height(),- index,- layer.blend,+ let treated = self.run_passes(+ geometry.fitted.0,+ geometry.fitted.1,+ made,+ &layer.effects,+ seconds,+ );+ (+ geometry.fitted,+ treated,+ geometry.prepared(),+ (false, false),+ )+ } else {+ let mut index = self.upload(source);+ if !layer.effects.is_empty() {+ index = self.run_passes(+ source.width(),+ source.height(),+ index,+ &layer.effects,+ seconds,+ );+ }+ (+ (source.width(), source.height()),+ index,+ geometry,+ (layer.flip_h, layer.flip_v),+ )+ };+ let mask = self.mask_of(layer.mask.as_deref());+ draws.push(Draw {+ size,+ texture,+ blend: layer.blend,+ mask,+ });+ vertices.extend_from_slice(&Self::quad(+ &geometry,+ &uvs,+ flips,+ opacity,+ layer.shading(),+ plan.width,+ plan.height,));- vertices.extend_from_slice(&Self::quad(layer, width, height));+ }++ /// The mask a draw binds: the layer's, uploaded, or the one white+ /// pixel for a layer without one.+ fn mask_of(&mut self, mask: Option<&Frame>) -> (u32, u32, usize) {+ match mask {+ Some(mask) => (mask.width(), mask.height(), self.upload(mask)),+ None => {+ let white = self.white.clone();+ (1, 1, self.upload(&white))+ }+ }+ }++ /// The source through its crop, flips and fit, drawn into a pooled+ /// texture of the fitted size over nothing: the picture as it will be+ /// seen, for the effects to run over. `source` is the pooled index of+ /// the upload.+ fn make_picture(+ &mut self,+ source: usize,+ source_size: (u32, u32),+ geometry: &Geometry,+ flip_h: bool,+ flip_v: bool,+ ) -> usize {+ let (width, height) = geometry.fitted;+ let mask = self.mask_of(None);+ let draws = vec![Draw {+ size: source_size,+ texture: source,+ blend: Blend::Normal,+ mask,+ }];+ let corner = |x: f32, y: f32, u: f32, v: f32| {+ let (su, sv) = geometry.uv_of(u, v, flip_h, flip_v);+ Vertex {+ position: [x, y],+ uv: [su, sv],+ opacity: 1.0,+ scale: 1.0,+ offset: [0.0; 3],+ edges: [2.0, -1.0],+ pic: [u, v],+ }+ };+ let vertices = [+ corner(-1.0, 1.0, 0.0, 0.0),+ corner(1.0, 1.0, 1.0, 0.0),+ corner(-1.0, -1.0, 0.0, 1.0),+ corner(1.0, 1.0, 1.0, 0.0),+ corner(1.0, -1.0, 1.0, 1.0),+ corner(-1.0, -1.0, 0.0, 1.0),+ ];+ self.render_pooled(width, height, &draws, &vertices, wgpu::Color::TRANSPARENT)}/// A draw of a pooled texture the size of the output, over the whole/// of it: how a stack already drawn is used as the ground for more.fn push_pooled(- draws: &mut Vec<(u32, u32, usize, Blend)>,+ &mut self,+ draws: &mut Vec<Draw>,vertices: &mut Vec<Vertex>,width: u32,height: u32,index: usize,opacity: f32,) {- draws.push((width, height, index, Blend::Normal));+ let mask = self.mask_of(None);+ draws.push(Draw {+ size: (width, height),+ texture: index,+ blend: Blend::Normal,+ mask,+ });let corner = |x: f32, y: f32, u: f32, v: f32| Vertex {position: [x, y],uv: [u, v],opacity: opacity.clamp(0.0, 1.0),+ scale: 1.0,+ offset: [0.0; 3],+ edges: [2.0, -1.0],+ pic: [u, v],};vertices.extend_from_slice(&[corner(-1.0, 1.0, 0.0, 0.0),@@ -526,110 +724,101 @@ impl WgpuCompositor {}}- /// Renders `draws` into a fresh pooled texture of the output size and- /// returns its index: a stack drawn so far, kept on the GPU as the- /// ground for a treatment or for the layers above it.+ /// Renders `draws` into a fresh pooled texture of `width` by `height`+ /// over `clear`, and returns its index: a stack drawn so far, kept on+ /// the GPU as the ground for a treatment or for the layers above it,+ /// or a picture made for its effects.fn render_pooled(&mut self,width: u32,height: u32,- draws: &[(u32, u32, usize, Blend)],+ draws: &[Draw],vertices: &[Vertex],+ clear: wgpu::Color,) -> usize {let target = self.claim(width, height);self.write_vertices(vertices);let view = self.pool[&(width, height)][target].texture.create_view(&wgpu::TextureViewDescriptor::default());- let encoder = self.encode(&view, draws);+ let encoder = self.encode(&view, draws, clear);self.queue.submit([encoder.finish()]);target}- /// The draws of a treated frame, every treatment applied over the- /// stack beneath its track without a pixel leaving the GPU: the stack- /// below is drawn into a pooled texture, the passes run over that, and- /// the result, blended back over the untreated stack by the strength,- /// becomes the ground the rest is drawn on. What comes back are the- /// draws for the final render, into whichever target the caller wants.- fn prepare_treated(- &mut self,- width: u32,- height: u32,- time: f32,- layers: &[(Layer<'_>, usize)],- treatments: &[Treatment<'_>],- ) -> (Vec<Draw>, Vec<Vertex>) {- self.used.values_mut().for_each(|used| *used = 0);- let mut ground: Option<usize> = None;- let mut next = 0;- for treatment in treatments {- let mut draws = Vec::new();- let mut vertices = Vec::new();- if let Some(index) = ground {- Self::push_pooled(&mut draws, &mut vertices, width, height, index, 1.0);- }- while next < layers.len() && layers[next].1 < treatment.track {- self.push_layer(&mut draws, &mut vertices, width, height, &layers[next].0);- next += 1;- }- let below = self.render_pooled(width, height, &draws, &vertices);- let strength = treatment.strength.clamp(0.0, 1.0);- if strength <= 0.0 || treatment.passes.is_empty() {- ground = Some(below);- continue;- }- let treated = self.run_passes(width, height, below, treatment.passes, time);- ground = Some(if strength >= 1.0 {- treated- } else {- let mut draws = Vec::new();- let mut vertices = Vec::new();- Self::push_pooled(&mut draws, &mut vertices, width, height, below, 1.0);- Self::push_pooled(&mut draws, &mut vertices, width, height, treated, strength);- self.render_pooled(width, height, &draws, &vertices)- });- }- let mut draws = Vec::new();- let mut vertices = Vec::new();- if let Some(index) = ground {- Self::push_pooled(&mut draws, &mut vertices, width, height, index, 1.0);- }- for (layer, _) in &layers[next..] {- self.push_layer(&mut draws, &mut vertices, width, height, layer);+ /// Draws `plan` into a texture that stays on the GPU, and hands it+ /// back: `Rgba8Unorm`, bindable and renderable, exactly the plan's+ /// size. The texture is one of a small ring, so the caller may keep+ /// showing the previous one while this draws. `None` when the device+ /// is dead; the caller then falls back to [`Compositor::render`] on a+ /// CPU compositor.+ pub fn render_texture(&mut self, plan: &FramePlan) -> Option<wgpu::Texture> {+ if self.dead {+ return None;}- (draws, vertices)+ let (draws, vertices) = self.prepare(plan);+ self.write_vertices(&vertices);+ let texture = self.presentable(plan.width, plan.height);+ let view = texture.create_view(&wgpu::TextureViewDescriptor::default());+ let encoder = self.encode(&view, &draws, wgpu::Color::BLACK);+ self.queue.submit([encoder.finish()]);+ self.retire();+ Some(texture)}- /// [`WgpuCompositor::composite_texture`] with treatments; see- /// [`Treatment`] and `prepare_treated`.- pub fn composite_texture_treated(- &mut self,- width: u32,- height: u32,- time: f32,- layers: &[(Layer<'_>, usize)],- treatments: &[Treatment<'_>],- ) -> Option<wgpu::Texture> {+ /// Runs `pass` once over a sixteen-pixel-square picture and waits at+ /// most `timeout` for the device: what a community package has to+ /// survive before it is enabled. A pass the driver refuses is an+ /// error; a pass that does not finish in time is an error too, and+ /// this compositor is dead from then on, since a device mid-hang+ /// cannot be trusted with the next frame.+ pub fn trial(&mut self, pass: &ShaderPass, timeout: std::time::Duration) -> Result<(), String> {if self.dead {- return None;+ return Err("the GPU device is dead".to_owned());}- let (draws, vertices) = self.prepare_treated(width, height, time, layers, treatments);+ let mut picture = Frame::transparent(16, 16);+ picture.fill([128, 96, 64, 255]);+ let mut layer =+ PlannedLayer::picture(crate::plan::detached_clip(), std::sync::Arc::new(picture));+ layer.effects = vec![pass.clone()];+ let plan = FramePlan {+ layers: vec![layer],+ ..FramePlan::empty(16, 16)+ };+ let scope = self.device.push_error_scope(wgpu::ErrorFilter::Validation);+ let (draws, vertices) = self.prepare(&plan);self.write_vertices(&vertices);- let texture = self.presentable(width, height);+ let texture = self.presentable(16, 16);let view = texture.create_view(&wgpu::TextureViewDescriptor::default());- let encoder = self.encode(&view, &draws);+ let encoder = self.encode(&view, &draws, wgpu::Color::BLACK);self.queue.submit([encoder.finish()]);+ let waited = self.device.poll(wgpu::PollType::Wait {+ submission_index: None,+ timeout: Some(timeout),+ });+ let refused = pollster::block_on(scope.pop());self.retire();- Some(texture)+ if let Some(error) = refused {+ return Err(format!("the driver refused the pass: {error}"));+ }+ match waited {+ Ok(_) => Ok(()),+ Err(error) => {+ self.dead = true;+ Err(format!(+ "the pass did not finish within {timeout:?}: {error}"+ ))+ }+ }}- /// The render pass: every draw over black into `view`. Returns the+ /// The render pass: every draw over `clear` into `view`. Returns the/// encoder so the caller can add a readback before submitting.fn encode(&self,view: &wgpu::TextureView,- draws: &[(u32, u32, usize, Blend)],+ draws: &[Draw],+ clear: wgpu::Color,) -> wgpu::CommandEncoder {let mut encoder = self.device@@ -644,7 +833,7 @@ impl WgpuCompositor {depth_slice: None,resolve_target: None,ops: wgpu::Operations {- load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),+ load: wgpu::LoadOp::Clear(clear),store: wgpu::StoreOp::Store,},})],@@ -654,15 +843,16 @@ impl WgpuCompositor {multiview_mask: None,});pass.set_vertex_buffer(0, self.vertices.slice(..));- for (draw, (layer_width, layer_height, pooled, blend)) in draws.iter().enumerate() {+ for (index, draw) in draws.iter().enumerate() {let which = Blend::ALL.iter()- .position(|mode| mode == blend)+ .position(|mode| *mode == draw.blend).unwrap_or(0);pass.set_pipeline(&self.pipelines[which]);- let bind_group = &self.pool[&(*layer_width, *layer_height)][*pooled].bind_group;- pass.set_bind_group(0, bind_group, &[]);- let first = (draw * 6) as u32;+ pass.set_bind_group(0, &self.pool[&draw.size][draw.texture].bind_group, &[]);+ let (mask_w, mask_h, mask) = draw.mask;+ pass.set_bind_group(1, &self.pool[&(mask_w, mask_h)][mask].bind_group, &[]);+ let first = (index * 6) as u32;pass.draw(first..first + 6, 0..1);}}@@ -690,30 +880,6 @@ impl WgpuCompositor {}}- /// Composites `layers` into a texture that stays on the GPU, and hands- /// it back: `Rgba8Unorm`, bindable and renderable, exactly `width` by- /// `height`. The texture is one of a small ring, so the caller may keep- /// showing the previous one while this draws. `None` when the device is- /// dead; the caller then falls back to [`Compositor::composite`] on a- /// CPU compositor.- pub fn composite_texture(- &mut self,- width: u32,- height: u32,- layers: &[Layer<'_>],- ) -> Option<wgpu::Texture> {- if self.dead {- return None;- }- let draws = self.prepare(width, height, layers);- let texture = self.presentable(width, height);- let view = texture.create_view(&wgpu::TextureViewDescriptor::default());- let encoder = self.encode(&view, &draws);- self.queue.submit([encoder.finish()]);- self.retire();- Some(texture)- }-/// The reusable render target for this output size.fn target(&mut self, width: u32, height: u32) -> &Target {let stale = self@@ -1064,35 +1230,43 @@ impl WgpuCompositor {current}- /// The six vertices of one layer's quad, transformed the same way the CPU- /// path's inverse map is derived: scale, then clockwise rotation, about- /// the layer's centre, then translation.- fn quad(layer: &Layer<'_>, out_width: u32, out_height: u32) -> [Vertex; 6] {- let placement = layer.placement;- let width = layer.frame.width() as f32;- let height = layer.frame.height() as f32;-- let centre_x = layer.x as f32 + width / 2.0 + placement.translate_x;- let centre_y = layer.y as f32 + height / 2.0 + placement.translate_y;- let (sin, cos) = placement.rotation.sin_cos();- let scale_x = (placement.scale * placement.stretch_x).max(1e-6);- let scale_y = (placement.scale * placement.stretch_y).max(1e-6);+ /// The six vertices of one layer's quad: the fitted picture scaled,+ /// turned about its centre and moved as the geometry says, the forward+ /// form of the CPU path's inverse map, sampling `uvs` through the+ /// flips, weighed by the opacity and the shading.+ fn quad(+ geometry: &Geometry,+ uvs: &Geometry,+ (flip_h, flip_v): (bool, bool),+ opacity: f32,+ shading: Shading,+ out_width: u32,+ out_height: u32,+ ) -> [Vertex; 6] {+ let (fitted_w, fitted_h) = (geometry.fitted.0 as f32, geometry.fitted.1 as f32);+ let (centre_x, centre_y) = geometry.centre;+ let (sin, cos) = geometry.rotation.sin_cos();+ let (scale_x, scale_y) = geometry.scale;let corner = |sx: f32, sy: f32, u: f32, v: f32| {- // Source-space offset from centre, scaled per axis, then rotated- // clockwise in y-down coordinates - the forward form of the CPU- // inverse map.- let dx = sx * width / 2.0 * scale_x;- let dy = sy * height / 2.0 * scale_y;+ // Picture-space offset from the centre, scaled per axis, then+ // rotated clockwise in y-down coordinates.+ let dx = sx * fitted_w / 2.0 * scale_x;+ let dy = sy * fitted_h / 2.0 * scale_y;let px = centre_x + dx * cos - dy * sin;let py = centre_y + dx * sin + dy * cos;+ let (tu, tv) = uvs.uv_of(u, v, flip_h, flip_v);Vertex {position: [px / out_width as f32 * 2.0 - 1.0,1.0 - py / out_height as f32 * 2.0,],- uv: [u, v],- opacity: layer.opacity.clamp(0.0, 1.0),+ uv: [tu, tv],+ opacity,+ scale: shading.scale,+ offset: shading.offset,+ edges: [shading.left_edge, shading.right_edge],+ pic: [u, v],}};@@ -1161,59 +1335,34 @@ impl WgpuCompositor {}impl Compositor for WgpuCompositor {- fn composite(&mut self, width: u32, height: u32, layers: &[Layer<'_>]) -> Frame {+ fn render(&mut self, plan: &FramePlan) -> Frame {// A dead device never comes back for this instance; the CPU// reference is the same pixels, slower - never a mid-export panic.if self.dead {- return CpuCompositor.composite(width, height, layers);+ return CpuCompositor.render(plan);}-- let draws = self.prepare(width, height, layers);- match self.render_and_read(width, height, &draws) {+ let (draws, vertices) = self.prepare(plan);+ self.write_vertices(&vertices);+ match self.render_and_read(plan.width, plan.height, &draws) {Some(frame) => frame,None => {self.dead = true;- CpuCompositor.composite(width, height, layers)+ CpuCompositor.render(plan)}}}-- fn composite_treated(- &mut self,- width: u32,- height: u32,- time: f32,- layers: &[(Layer<'_>, usize)],- treatments: &[Treatment<'_>],- ) -> Option<Frame> {- if self.dead {- return None;- }- let (draws, vertices) = self.prepare_treated(width, height, time, layers, treatments);- self.write_vertices(&vertices);- let frame = self.render_and_read(width, height, &draws);- if frame.is_none() {- self.dead = true;- }- frame- }}impl WgpuCompositor {/// Draws into the readback target and copies it out: one submit, one/// wait. `None` when the device did not deliver the pixels.- fn render_and_read(- &mut self,- width: u32,- height: u32,- draws: &[(u32, u32, usize, Blend)],- ) -> Option<Frame> {+ fn render_and_read(&mut self, width: u32, height: u32, draws: &[Draw]) -> Option<Frame> {self.target(width, height);let target = self.target.as_ref().expect("just ensured");let view = target.texture.create_view(&wgpu::TextureViewDescriptor::default());- let mut encoder = self.encode(&view, draws);+ let mut encoder = self.encode(&view, draws, wgpu::Color::BLACK);encoder.copy_texture_to_buffer(wgpu::TexelCopyTextureInfo {texture: &target.texture,@@ -1242,237 +1391,4 @@ impl WgpuCompositor {}#[cfg(test)]-mod tests {- use super::*;- use crate::CpuCompositor;- use crate::compositor::Placement;-- /// A pass runs over the layer before it is placed: an invert shader- /// over a red frame composites cyan, and at half intensity the mix.- #[test]- fn a_pass_treats_the_layer_before_it_is_placed() {- let Some(mut gpu) = gpu() else { return };- let manifest = concat_effects::Manifest::parse(- "[effect]\nid = \"test.invert\"\nname = \"Invert\"\nkind = \"filter\"\n[wgsl]\nentry = \"effect.wgsl\"\n",- )- .expect("a manifest");- let shader = concat_effects::Shader::compile(- &manifest,- "fn effect(uv: vec2<f32>) -> vec4<f32> { let c = sample(uv); return vec4<f32>(vec3<f32>(1.0) - c.rgb, c.a); }",- )- .expect("compiles");- let red = {- let mut frame = Frame::black(4, 4);- for pixel in frame.pixels_mut().chunks_exact_mut(4) {- pixel.copy_from_slice(&[255, 0, 0, 255]);- }- frame- };- let full = [shader.pass(&Default::default(), &[], 1.0, None)];- let out = gpu.composite(4, 4, &[Layer::new(&red).with_passes(&full)]);- assert_eq!(&out.pixels()[..3], &[0, 255, 255]);- let half = [shader.pass(&Default::default(), &[], 0.5, None)];- let out = gpu.composite(4, 4, &[Layer::new(&red).with_passes(&half)]);- let p = &out.pixels()[..3];- assert!(- p[0] > 120 && p[0] < 136 && p[1] > 120 && p[1] < 136,- "{p:?}"- );- }-- /// A pass reads its table through `lut()`: a table that answers green- /// to every colour turns a red frame green, and a pass without one is- /// handed the identity and changes nothing.- #[test]- fn a_pass_samples_its_table_and_the_identity_without_one() {- let Some(mut gpu) = gpu() else { return };- let manifest = concat_effects::Manifest::parse(- "[effect]\nid = \"test.table\"\nname = \"Table\"\nkind = \"effect\"\n[wgsl]\nentry = \"effect.wgsl\"\n",- )- .expect("a manifest");- let shader = concat_effects::Shader::compile(- &manifest,- "fn effect(uv: vec2<f32>) -> vec4<f32> { let c = sample(uv); return vec4<f32>(lut(c.rgb), c.a); }",- )- .expect("compiles");- let red = solid(4, 4, [255, 0, 0, 255]);- let green = Lut::from_rgb(2, &[0.0, 1.0, 0.0].repeat(8)).expect("a table");- let tabled = [shader.pass(- &Default::default(),- &[],- 1.0,- Some(std::sync::Arc::new(green)),- )];- let out = gpu.composite(4, 4, &[Layer::new(&red).with_passes(&tabled)]);- assert_eq!(&out.pixels()[..3], &[0, 255, 0]);- let plain = [shader.pass(&Default::default(), &[], 1.0, None)];- let out = gpu.composite(4, 4, &[Layer::new(&red).with_passes(&plain)]);- assert_eq!(&out.pixels()[..3], &[255, 0, 0]);- }-- /// A treatment runs its passes over the stack beneath its track and- /// nothing above it, blended back by its strength, without the frame- /// leaving the GPU: an invert over a red ground under a blue quarter- /// turns the ground cyan and leaves the blue alone.- #[test]- fn a_treatment_treats_the_stack_beneath_its_track_only() {- let Some(mut gpu) = gpu() else { return };- let manifest = concat_effects::Manifest::parse(- "[effect]\nid = \"test.invert2\"\nname = \"Invert\"\nkind = \"effect\"\n[wgsl]\nentry = \"effect.wgsl\"\n",- )- .expect("a manifest");- let shader = concat_effects::Shader::compile(- &manifest,- "fn effect(uv: vec2<f32>) -> vec4<f32> { let c = sample(uv); return vec4<f32>(vec3<f32>(1.0) - c.rgb, c.a); }",- )- .expect("compiles");- let passes = [shader.pass(&Default::default(), &[], 1.0, None)];- let red = solid(8, 8, [255, 0, 0, 255]);- let blue = solid(4, 4, [0, 0, 255, 255]);- let layers = [(Layer::new(&red), 0), (Layer::new(&blue), 2)];- let full = [Treatment {- track: 1,- passes: &passes,- strength: 1.0,- }];- let out = gpu- .composite_treated(8, 8, 0.0, &layers, &full)- .expect("drawn");- // Top-left is under the blue quarter; bottom-right is treated ground.- assert_eq!(&out.pixels()[..3], &[0, 0, 255]);- let last = out.pixels().len() - 4;- assert_eq!(&out.pixels()[last..last + 3], &[0, 255, 255]);- let half = [Treatment {- track: 1,- passes: &passes,- strength: 0.5,- }];- let out = gpu- .composite_treated(8, 8, 0.0, &layers, &half)- .expect("drawn");- let p = &out.pixels()[last..last + 3];- assert!(- p[0] > 120 && p[0] < 136 && p[1] > 120 && p[1] < 136,- "{p:?}"- );- }-- fn gpu() -> Option<WgpuCompositor> {- let compositor = WgpuCompositor::new();- if compositor.is_none() {- eprintln!("no usable GPU adapter; skipping");- }- compositor- }-- fn solid(width: u32, height: u32, rgba: [u8; 4]) -> Frame {- let mut frame = Frame::transparent(width, height);- frame.fill(rgba);- frame- }-- /// Both backends composite the same layers; every pixel must agree within- /// `tolerance` per channel. The CPU path is the reference.- fn assert_matches_cpu(width: u32, height: u32, layers: &[Layer<'_>], tolerance: i32) {- let Some(mut gpu) = gpu() else { return };- let expected = CpuCompositor.composite(width, height, layers);- let actual = gpu.composite(width, height, layers);-- for y in 0..height {- for x in 0..width {- let want = expected.pixel(x, y).expect("in bounds");- let got = actual.pixel(x, y).expect("in bounds");- for channel in 0..4 {- let difference = (i32::from(want[channel]) - i32::from(got[channel])).abs();- assert!(- difference <= tolerance,- "({x},{y}) channel {channel}: cpu {want:?} vs gpu {got:?}",- );- }- }- }- }-- #[test]- fn empty_output_is_opaque_black() {- let Some(mut gpu) = gpu() else { return };- let frame = gpu.composite(4, 4, &[]);- assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));- assert_eq!(frame.pixel(3, 3), Some([0, 0, 0, 255]));- }-- #[test]- fn plain_layers_match_the_cpu_reference() {- let red = solid(4, 4, [255, 0, 0, 255]);- let blue = solid(2, 2, [0, 0, 255, 255]);- let layers = [Layer::new(&red), Layer::new(&blue).at(1, 1)];- assert_matches_cpu(4, 4, &layers, 1);- }-- #[test]- fn opacity_blending_matches_the_cpu_reference() {- let white = solid(4, 4, [255, 255, 255, 255]);- let half_alpha = solid(4, 4, [40, 200, 90, 128]);- let layers = [- Layer::new(&white).with_opacity(0.5),- Layer::new(&half_alpha).with_opacity(0.7),- ];- assert_matches_cpu(4, 4, &layers, 2);- }-- #[test]- fn offset_layers_clip_like_the_cpu_reference() {- let red = solid(4, 4, [255, 0, 0, 255]);- let layers = [Layer::new(&red).at(-2, 3)];- assert_matches_cpu(6, 6, &layers, 1);- }-- #[test]- fn scaling_matches_the_cpu_reference_away_from_edges() {- let Some(mut gpu) = gpu() else { return };- let red = solid(4, 4, [255, 0, 0, 255]);- let placement = Placement {- scale: 2.0,- ..Placement::IDENTITY- };- let layers = [Layer::new(&red).at(2, 2).with_placement(placement)];-- // Interior pixels are unambiguous; the half-pixel border where the two- // backends rasterise differently is not asserted.- let frame = gpu.composite(8, 8, &layers);- for (x, y) in [(1, 1), (4, 4), (6, 6)] {- assert_eq!(frame.pixel(x, y), Some([255, 0, 0, 255]), "at ({x},{y})");- }- assert_eq!(- CpuCompositor.composite(8, 8, &layers).pixel(4, 4),- Some([255, 0, 0, 255])- );- }-- #[test]- fn a_half_turn_swaps_the_ends() {- let Some(mut gpu) = gpu() else { return };- let mut strip = Frame::transparent(2, 1);- strip.set_pixel(0, 0, [255, 0, 0, 255]);- strip.set_pixel(1, 0, [0, 0, 255, 255]);- let placement = Placement {- rotation: std::f32::consts::PI,- ..Placement::IDENTITY- };- let frame = gpu.composite(2, 1, &[Layer::new(&strip).with_placement(placement)]);- assert_eq!(frame.pixel(0, 0), Some([0, 0, 255, 255]));- assert_eq!(frame.pixel(1, 0), Some([255, 0, 0, 255]));- }-- #[test]- fn output_size_changes_are_handled() {- let Some(mut gpu) = gpu() else { return };- let red = solid(2, 2, [255, 0, 0, 255]);- // 3 wide: an unpadded row is 12 bytes, so this exercises row padding.- let small = gpu.composite(3, 2, &[Layer::new(&red)]);- assert_eq!(small.pixel(1, 1), Some([255, 0, 0, 255]));- let large = gpu.composite(16, 8, &[Layer::new(&red).at(14, 6)]);- assert_eq!(large.pixel(15, 7), Some([255, 0, 0, 255]));- assert_eq!(large.pixel(0, 0), Some([0, 0, 0, 255]));- }-}+mod tests;diff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rsnew file mode 100644index 00000000..3b7280ca--- /dev/null+++ b/src/crates/concat-render/src/gpu/tests.rs@@ -0,0 +1,397 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! The GPU compositor against the CPU reference: the same plans, and the+//! pictures must agree. Every test skips on a machine without an adapter,+//! and says so.++use std::collections::BTreeMap;+use std::sync::Arc;++use concat_core::frame::Frame;+use concat_core::shader::{Lut, ShaderPass};+use concat_core::timeline::{Blend, Transform};++use super::*;+use crate::CpuCompositor;+use crate::metrics::ssim;+use crate::plan::{Crop, Transition, detached_clip};++fn gpu() -> Option<WgpuCompositor> {+ let compositor = WgpuCompositor::new();+ if compositor.is_none() {+ eprintln!("no usable GPU adapter; skipping");+ }+ compositor+}++fn solid(width: u32, height: u32, rgba: [u8; 4]) -> Frame {+ let mut frame = Frame::transparent(width, height);+ frame.fill(rgba);+ frame+}++/// A picture with something in it everywhere: a diagonal gradient and a+/// dark bar, so a misplaced or mirrored picture is caught.+fn gradient(width: u32, height: u32) -> Frame {+ let mut frame = Frame::transparent(width, height);+ for y in 0..height {+ for x in 0..width {+ let r = (x * 255 / width.max(1)) as u8;+ let g = (y * 255 / height.max(1)) as u8;+ let bar = x > width / 3 && x < width / 2 && y > height / 4;+ let b = if bar { 20 } else { 200 };+ frame.set_pixel(x, y, [r, g, b, 255]);+ }+ }+ frame+}++fn layer(frame: Frame) -> PlannedLayer {+ PlannedLayer::picture(detached_clip(), Arc::new(frame))+}++fn plan(width: u32, height: u32, layers: Vec<PlannedLayer>) -> FramePlan {+ FramePlan {+ layers,+ ..FramePlan::empty(width, height)+ }+}++/// A real package's pass: `id` with `body` as its shader and `params` as+/// the manifest's parameter table, resolved to `values`.+fn package(+ id: &str,+ body: &str,+ params: &str,+ values: &[(&str, f64)],+ intensity: f32,+) -> ShaderPass {+ let manifest = concat_effects::Manifest::parse(&format!(+ "[effect]\nid = \"{id}\"\nname = \"Test\"\nkind = \"effect\"\n{params}\n[wgsl]\nentry = \"effect.wgsl\"\n"+ ))+ .expect("a manifest");+ let shader = concat_effects::Shader::compile(&manifest, body).expect("compiles");+ let values: BTreeMap<String, f64> = values+ .iter()+ .map(|(key, value)| ((*key).to_owned(), *value))+ .collect();+ shader.pass(&values, &manifest.params, intensity, None)+}++const INVERT: &str = "fn effect(uv: vec2<f32>) -> vec4<f32> { let c = sample(uv); return vec4<f32>(vec3<f32>(1.0) - c.rgb, c.a); }";++/// Both compositors draw `plan`; the pictures must agree to `least` by+/// SSIM, and every pixel's alpha must be opaque.+fn assert_parity(name: &str, plan: &FramePlan, least: f64) {+ let Some(mut gpu) = gpu() else { return };+ let expected = CpuCompositor.render(plan);+ let actual = gpu.render(plan);+ let score = ssim(&expected, &actual);+ eprintln!("parity {name}: ssim {score:.4}");+ assert!(+ score >= least,+ "{name}: cpu and gpu differ, ssim {score:.4} < {least}"+ );+ assert!(+ actual.pixels().chunks_exact(4).all(|px| px[3] == 255),+ "{name}: opaque"+ );+}++#[test]+fn empty_output_is_opaque_black() {+ let Some(mut gpu) = gpu() else { return };+ let frame = gpu.render(&FramePlan::empty(4, 4));+ assert_eq!(frame.pixel(0, 0), Some([0, 0, 0, 255]));+ assert_eq!(frame.pixel(3, 3), Some([0, 0, 0, 255]));+}++#[test]+fn plain_layers_match_the_cpu_reference_to_the_pixel() {+ let Some(mut gpu) = gpu() else { return };+ let mut top = layer(solid(3, 3, [0, 0, 255, 160]));+ top.transform = Transform {+ offset_x: 2.0 / 8.0,+ offset_y: 1.0 / 8.0,+ ..Transform::default()+ };+ let plan = plan(8, 8, vec![layer(solid(8, 8, [255, 0, 0, 255])), top]);+ let expected = CpuCompositor.render(&plan);+ let actual = gpu.render(&plan);+ for (index, (want, got)) in expected+ .pixels()+ .chunks_exact(4)+ .zip(actual.pixels().chunks_exact(4))+ .enumerate()+ {+ for channel in 0..4 {+ let difference = (i32::from(want[channel]) - i32::from(got[channel])).abs();+ assert!(+ difference <= 1,+ "pixel {index} channel {channel}: cpu {want:?} vs gpu {got:?}"+ );+ }+ }+}++/// A pass runs over the layer before it is placed: an invert shader+/// over a red frame composites cyan, and at half intensity the mix.+#[test]+fn a_pass_treats_the_layer_before_it_is_placed() {+ let Some(mut gpu) = gpu() else { return };+ let red = solid(4, 4, [255, 0, 0, 255]);+ let mut full = layer(red.clone());+ full.effects = vec![package("test.invert", INVERT, "", &[], 1.0)];+ let out = gpu.render(&plan(4, 4, vec![full]));+ assert_eq!(&out.pixels()[..3], &[0, 255, 255]);+ let mut half = layer(red);+ half.effects = vec![package("test.invert", INVERT, "", &[], 0.5)];+ let out = gpu.render(&plan(4, 4, vec![half]));+ let p = &out.pixels()[..3];+ assert!(+ p[0] > 120 && p[0] < 136 && p[1] > 120 && p[1] < 136,+ "{p:?}"+ );+}++/// A pass reads its table through `lut()`: a table that answers green+/// to every colour turns a red frame green, and a pass without one is+/// handed the identity and changes nothing.+#[test]+fn a_pass_samples_its_table_and_the_identity_without_one() {+ let Some(mut gpu) = gpu() else { return };+ let body = "fn effect(uv: vec2<f32>) -> vec4<f32> { let c = sample(uv); return vec4<f32>(lut(c.rgb), c.a); }";+ let green = Lut::from_rgb(2, &[0.0, 1.0, 0.0].repeat(8)).expect("a table");+ let mut tabled = layer(solid(4, 4, [255, 0, 0, 255]));+ let mut pass = package("test.table", body, "", &[], 1.0);+ pass.lut = Some(Arc::new(green));+ tabled.effects = vec![pass];+ let out = gpu.render(&plan(4, 4, vec![tabled]));+ assert_eq!(&out.pixels()[..3], &[0, 255, 0]);+ let mut plain = layer(solid(4, 4, [255, 0, 0, 255]));+ plain.effects = vec![package("test.table", body, "", &[], 1.0)];+ let out = gpu.render(&plan(4, 4, vec![plain]));+ assert_eq!(&out.pixels()[..3], &[255, 0, 0]);+}++/// A treatment runs its passes over the stack beneath its track and+/// nothing above it, blended back by its strength, without the frame+/// leaving the GPU: an invert over a red ground under a blue quarter+/// turns the ground cyan and leaves the blue alone.+#[test]+fn a_treatment_treats_the_stack_beneath_its_track_only() {+ let Some(mut gpu) = gpu() else { return };+ let ground = layer(solid(8, 8, [255, 0, 0, 255]));+ let mut blue = layer(solid(4, 4, [0, 0, 255, 255]));+ blue.track = 2;+ blue.transform = Transform {+ offset_x: -2.0 / 8.0,+ offset_y: -2.0 / 8.0,+ ..Transform::default()+ };+ let treated = |strength: f32| FramePlan {+ treatments: vec![PlannedTreatment {+ track: 1,+ effects: vec![package("concat.invert", INVERT, "", &[], 1.0)],+ strength,+ }],+ ..plan(8, 8, vec![ground.clone(), blue.clone()])+ };+ let out = gpu.render(&treated(1.0));+ // Top-left is under the blue quarter; bottom-right is treated ground.+ assert_eq!(&out.pixels()[..3], &[0, 0, 255]);+ let last = out.pixels().len() - 4;+ assert_eq!(&out.pixels()[last..last + 3], &[0, 255, 255]);+ let out = gpu.render(&treated(0.5));+ let p = &out.pixels()[last..last + 3];+ assert!(+ p[0] > 120 && p[0] < 136 && p[1] > 120 && p[1] < 136,+ "{p:?}"+ );+ // The CPU reference agrees on the whole picture.+ assert_parity("treatment", &treated(0.5), 0.99);+}++/// The parity suite: one plan per thing a frame can ask for, drawn by+/// both compositors, alike by SSIM.+#[test]+fn every_kind_of_layer_matches_the_cpu_reference() {+ let sepia = include_str!("../../../concat-effects/packages/concat.sepia/effect.wgsl");+ let blur = include_str!("../../../concat-effects/packages/concat.box-blur/effect.wgsl");+ let radius =+ "[[param]]\nkey = \"radius\"\nlabel = \"Radius\"\nmin = 0\nmax = 20\ndefault = 4\n";++ assert_parity("plain", &plan(64, 48, vec![layer(gradient(64, 48))]), 0.999);++ // Fitted: a wide picture inside a square, letterboxed.+ assert_parity("fitted", &plan(64, 64, vec![layer(gradient(48, 16))]), 0.99);++ let mut placed = layer(gradient(32, 24));+ placed.transform = Transform {+ scale: 1.4,+ rotation: 30.0,+ offset_x: 0.1,+ offset_y: -0.05,+ stretch_x: 1.2,+ ..Transform::default()+ };+ assert_parity(+ "placed",+ &plan(+ 64,+ 64,+ vec![layer(solid(64, 64, [40, 40, 40, 255])), placed],+ ),+ 0.98,+ );++ let mut cropped = layer(gradient(64, 48));+ cropped.crop = Crop::of([0.25, 0.1, 0.2, 0.3]);+ assert_parity("cropped", &plan(64, 48, vec![cropped]), 0.99);++ let mut flipped = layer(gradient(64, 48));+ flipped.flip_h = true;+ flipped.flip_v = true;+ assert_parity("flipped", &plan(64, 48, vec![flipped]), 0.999);++ let mut blended = layer(gradient(64, 48));+ blended.opacity = 0.5;+ blended.blend = Blend::Multiply;+ assert_parity(+ "blended",+ &plan(64, 48, vec![layer(gradient(64, 48)), blended]),+ 0.99,+ );++ let mut masked = layer(gradient(64, 64));+ let mut mask = Frame::transparent(32, 32);+ for y in 0..32u32 {+ for x in 0..32u32 {+ let d = ((x as f32 - 16.0).powi(2) + (y as f32 - 16.0).powi(2)).sqrt();+ let a = ((16.0 - d) / 8.0).clamp(0.0, 1.0);+ mask.set_pixel(x, y, [255, 255, 255, (a * 255.0) as u8]);+ }+ }+ masked.mask = Some(Arc::new(mask));+ assert_parity(+ "masked",+ &plan(64, 64, vec![layer(solid(64, 64, [0, 60, 0, 255])), masked]),+ 0.99,+ );++ let mut cut = layer(gradient(64, 48));+ cut.transitions = vec![+ Transition::FadeTo {+ colour: [0.0, 0.0, 0.0],+ amount: 0.5,+ },+ Transition::Wipe {+ uncovered: 0.6,+ from_right: false,+ },+ ];+ assert_parity("faded and wiped", &plan(64, 48, vec![cut]), 0.99);++ let mut toned = layer(gradient(64, 48));+ toned.effects = vec![package("concat.sepia", sepia, "", &[], 1.0)];+ assert_parity("sepia kernel", &plan(64, 48, vec![toned]), 0.99);++ let mut keyed = layer(gradient(64, 48));+ keyed.effects = vec![package("concat.sepia", sepia, "", &[], 0.4)];+ assert_parity(+ "sepia at a keyed intensity",+ &plan(64, 48, vec![keyed]),+ 0.99,+ );++ // A blur over a cropped, flipped picture: the effects have to see it+ // made first, on both sides.+ let mut prepared = layer(gradient(64, 48));+ prepared.crop = Crop::of([0.1, 0.0, 0.1, 0.2]);+ prepared.flip_h = true;+ prepared.effects = vec![package(+ "concat.box-blur",+ blur,+ radius,+ &[("radius", 3.0)],+ 1.0,+ )];+ assert_parity(+ "blur over a made picture",+ &plan(80, 60, vec![prepared]),+ 0.98,+ );++ let mut under = layer(gradient(64, 48));+ under.track = 0;+ let mut over = layer(solid(16, 16, [0, 0, 255, 255]));+ over.track = 2;+ let treated = FramePlan {+ treatments: vec![PlannedTreatment {+ track: 1,+ effects: vec![package("concat.sepia", sepia, "", &[], 1.0)],+ strength: 0.6,+ }],+ ..plan(64, 48, vec![under, over])+ };+ assert_parity("treated stack", &treated, 0.99);+}++/// What cannot be drawn is not drawn, the same way on both sides: a+/// layer with a NaN opacity, one far outside the frame, one with no+/// picture, and an opacity past one is one.+#[test]+fn what_cannot_be_drawn_is_skipped_alike() {+ let Some(mut gpu) = gpu() else { return };+ let mut nan = layer(solid(8, 8, [255, 0, 0, 255]));+ nan.opacity = f32::NAN;+ let mut away = layer(solid(8, 8, [255, 0, 0, 255]));+ away.transform = Transform {+ offset_x: 40.0,+ ..Transform::default()+ };+ let mut none = layer(solid(8, 8, [255, 0, 0, 255]));+ none.source = None;+ let plan = plan(8, 8, vec![nan, away, none]);+ assert_eq!(gpu.render(&plan).pixel(4, 4), Some([0, 0, 0, 255]));+ assert_eq!(+ CpuCompositor.render(&plan).pixel(4, 4),+ Some([0, 0, 0, 255])+ );+ let mut over = layer(solid(8, 8, [0, 200, 0, 255]));+ over.opacity = 7.0;+ let plan = FramePlan {+ layers: vec![over],+ ..FramePlan::empty(8, 8)+ };+ assert_eq!(gpu.render(&plan).pixel(1, 1), Some([0, 200, 0, 255]));+}++/// A package that runs within the budget passes its trial; a device that+/// answers in time is kept.+#[test]+fn a_benign_package_survives_its_trial() {+ let Some(mut gpu) = gpu() else { return };+ let pass = package("test.trial", INVERT, "", &[], 1.0);+ gpu.trial(&pass, std::time::Duration::from_secs(5))+ .expect("an invert is quick");+ assert!(!gpu.is_dead());+ // The compositor is still good for a frame afterwards.+ let frame = gpu.render(&plan(4, 4, vec![layer(solid(4, 4, [0, 0, 255, 255]))]));+ assert_eq!(&frame.pixels()[..3], &[0, 0, 255]);+}++#[test]+fn output_size_changes_are_handled() {+ let Some(mut gpu) = gpu() else { return };+ let small = gpu.render(&plan(4, 4, vec![layer(solid(4, 4, [255, 255, 255, 255]))]));+ assert_eq!((small.width(), small.height()), (4, 4));+ let large = gpu.render(&plan(+ 16,+ 8,+ vec![layer(solid(16, 8, [255, 255, 255, 255]))],+ ));+ assert_eq!((large.width(), large.height()), (16, 8));+ assert_eq!(large.pixel(15, 7), Some([255, 255, 255, 255]));+}diff --git a/src/crates/concat-render/src/kernels.rs b/src/crates/concat-render/src/kernels.rsnew file mode 100644index 00000000..67ea0d99--- /dev/null+++ b/src/crates/concat-render/src/kernels.rs@@ -0,0 +1,297 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! The CPU's effects: one kernel per built-in package, keyed by its id.+//!+//! A package is a WGSL shader, and the GPU runs it. The CPU reference+//! cannot run WGSL at any useful speed, so it carries a kernel of its own+//! for each package it knows, written to the same arithmetic as the+//! shader and read from the same resolved values, and checked against the+//! GPU by the parity suite. A package it does not know - a community+//! package, or a built-in nobody has ported - is drawn untreated and said+//! so once per process, which is honest where a filter string that+//! silently differs was not.++use std::collections::HashSet;+use std::sync::{Mutex, OnceLock};++use concat_core::frame::{BYTES_PER_PIXEL, Frame};+use concat_core::shader::ShaderPass;++use crate::compositor::sample_bilinear;++/// Every package the CPU renders.+pub const KERNELS: &[&str] = &[+ "concat.black-white",+ "concat.invert",+ "concat.sepia",+ "concat.hue-shift",+ "concat.vignette",+ "concat.box-blur",+ "concat.gaussian-blur",+];++/// Whether the CPU has a kernel for the package.+pub fn has_kernel(package: &str) -> bool {+ KERNELS.contains(&package)+}++/// Runs the kernel for `pass.package` over `picture` at full strength, the+/// intensity mix being the caller's; `None` when there is no kernel for+/// the package. `seconds` is the timeline instant, for a kernel that+/// moves; none of these do.+pub fn run(pass: &ShaderPass, picture: &Frame, seconds: f32) -> Option<Frame> {+ let _ = seconds;+ let knob = |key: &str| pass.values.get(key).copied().unwrap_or(0.0) as f32;+ Some(match pass.package.as_str() {+ "concat.black-white" => per_pixel(picture, |[r, g, b, a]| {+ let y = luma(r, g, b);+ [y, y, y, a]+ }),+ "concat.invert" => per_pixel(picture, |[r, g, b, a]| [1.0 - r, 1.0 - g, 1.0 - b, a]),+ "concat.sepia" => per_pixel(picture, |[r, g, b, a]| {+ [+ (r * 0.393 + g * 0.769 + b * 0.189).clamp(0.0, 1.0),+ (r * 0.349 + g * 0.686 + b * 0.168).clamp(0.0, 1.0),+ (r * 0.272 + g * 0.534 + b * 0.131).clamp(0.0, 1.0),+ a,+ ]+ }),+ "concat.hue-shift" => {+ let (sn, cs) = knob("angle").to_radians().sin_cos();+ per_pixel(picture, |[r, g, b, a]| {+ // A turn of the hue is a plain rotation in YIQ, where the+ // two chroma axes are at right angles.+ let y = 0.299 * r + 0.587 * g + 0.114 * b;+ let i = 0.596 * r - 0.274 * g - 0.322 * b;+ let q = 0.211 * r - 0.523 * g + 0.312 * b;+ let (i, q) = (i * cs - q * sn, i * sn + q * cs);+ [+ (y + 0.956 * i + 0.621 * q).clamp(0.0, 1.0),+ (y - 0.272 * i - 0.647 * q).clamp(0.0, 1.0),+ (y - 1.106 * i + 1.703 * q).clamp(0.0, 1.0),+ a,+ ]+ })+ }+ "concat.vignette" => {+ let s = knob("strength") / 100.0;+ per_point(picture, |[r, g, b, a], u, v| {+ let d = ((u - 0.5) * 2.0).hypot((v - 0.5) * 2.0);+ let fall = smoothstep(1.4 - s * 0.9, 1.4 + 0.2 - s * 0.3, d);+ let keep = 1.0 - fall * (0.4 + 0.6 * s);+ [r * keep, g * keep, b * keep, a]+ })+ }+ "concat.box-blur" => {+ let step = (knob("radius") / 4.0).max(0.5);+ gathered(picture, step, |_, _| 1.0)+ }+ "concat.gaussian-blur" => {+ let sigma = knob("radius").max(0.5);+ let step = (sigma / 2.5).max(1.0);+ gathered(picture, step, move |dx, dy| {+ (-(dx * dx + dy * dy) / (2.0 * sigma * sigma)).exp()+ })+ }+ _ => return None,+ })+}++/// Says once, in the log, that `package` is drawn untreated on the CPU.+pub fn fallback_once(package: &str) {+ static SEEN: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();+ let seen = SEEN.get_or_init(|| Mutex::new(HashSet::new()));+ let mut seen = seen.lock().unwrap_or_else(|poisoned| poisoned.into_inner());+ if seen.insert(package.to_owned()) {+ log::warn!("{package}: no CPU kernel for this package; the picture is drawn untreated");+ }+}++/// Luminance, Rec. 709.+fn luma(r: f32, g: f32, b: f32) -> f32 {+ 0.2126 * r + 0.7152 * g + 0.0722 * b+}++/// WGSL's `smoothstep`.+fn smoothstep(edge0: f32, edge1: f32, x: f32) -> f32 {+ let t = ((x - edge0) / (edge1 - edge0)).clamp(0.0, 1.0);+ t * t * (3.0 - 2.0 * t)+}++/// Every pixel through `f`, straight colour and alpha in `0..=1`.+fn per_pixel(picture: &Frame, f: impl Fn([f32; 4]) -> [f32; 4]) -> Frame {+ per_point(picture, |rgba, _, _| f(rgba))+}++/// Every pixel through `f` with where it is, `u` and `v` in `0..1` across+/// the picture at the pixel's centre.+fn per_point(picture: &Frame, f: impl Fn([f32; 4], f32, f32) -> [f32; 4]) -> Frame {+ let (width, height) = (picture.width(), picture.height());+ let mut out = picture.clone();+ let stride = width as usize * BYTES_PER_PIXEL;+ let pixels = out.pixels_mut();+ for y in 0..height as usize {+ let v = (y as f32 + 0.5) / height as f32;+ for x in 0..width as usize {+ let u = (x as f32 + 0.5) / width as f32;+ let at = y * stride + x * BYTES_PER_PIXEL;+ let rgba = f(+ [+ f32::from(pixels[at]) / 255.0,+ f32::from(pixels[at + 1]) / 255.0,+ f32::from(pixels[at + 2]) / 255.0,+ f32::from(pixels[at + 3]) / 255.0,+ ],+ u,+ v,+ );+ for (channel, value) in rgba.iter().enumerate() {+ pixels[at + channel] = (value * 255.0).round().clamp(0.0, 255.0) as u8;+ }+ }+ }+ out+}++/// The blurs' 9×9 gather: every pixel becomes the weighted mean of the+/// samples `step` pixels apart around it, `weight` of the offset in+/// pixels, sampled bilinearly with the edge clamped the way the GPU's+/// sampler does.+fn gathered(picture: &Frame, step: f32, weight: impl Fn(f32, f32) -> f32) -> Frame {+ let (width, height) = (picture.width(), picture.height());+ let mut out = picture.clone();+ let stride = width as usize * BYTES_PER_PIXEL;+ let taps: Vec<(f32, f32, f32)> = (-4..=4)+ .flat_map(|y| (-4..=4).map(move |x| (x as f32 * step, y as f32 * step)))+ .map(|(dx, dy)| (dx, dy, weight(dx, dy)))+ .collect();+ let total: f32 = taps.iter().map(|(_, _, w)| *w).sum();+ let pixels = out.pixels_mut();+ for y in 0..height as usize {+ for x in 0..width as usize {+ let mut sum = [0.0f32; 4];+ for (dx, dy, w) in &taps {+ let sample = sample_bilinear(picture, x as f32 + dx, y as f32 + dy);+ for (channel, value) in sample.iter().enumerate() {+ sum[channel] += value * w;+ }+ }+ let at = y * stride + x * BYTES_PER_PIXEL;+ for (channel, value) in sum.iter().enumerate() {+ pixels[at + channel] = (value / total).round().clamp(0.0, 255.0) as u8;+ }+ }+ }+ out+}++#[cfg(test)]+mod tests {+ use std::collections::BTreeMap;+ use std::sync::Arc;++ use super::*;++ fn pass(package: &str, values: &[(&str, f64)]) -> ShaderPass {+ ShaderPass {+ package: package.to_owned(),+ key: package.to_owned(),+ source: Arc::from(""),+ params: vec![0; ShaderPass::MIN_PARAMS],+ values: values+ .iter()+ .map(|(key, value)| ((*key).to_owned(), *value))+ .collect::<BTreeMap<_, _>>(),+ intensity: 1.0,+ lut: None,+ }+ }++ fn solid(rgba: [u8; 4]) -> Frame {+ let mut frame = Frame::transparent(4, 4);+ frame.fill(rgba);+ frame+ }++ #[test]+ fn every_listed_package_runs_and_an_unknown_one_does_not() {+ let picture = solid([200, 100, 50, 255]);+ for package in KERNELS {+ assert!(has_kernel(package));+ let knobs = [("radius", 2.0), ("strength", 50.0), ("angle", 90.0)];+ let out = run(&pass(package, &knobs), &picture, 0.0)+ .unwrap_or_else(|| panic!("{package} has a kernel"));+ assert_eq!((out.width(), out.height()), (4, 4));+ }+ assert!(!has_kernel("someone.else"));+ assert!(run(&pass("someone.else", &[]), &picture, 0.0).is_none());+ fallback_once("someone.else");+ fallback_once("someone.else");+ }++ #[test]+ fn the_colour_kernels_do_their_arithmetic() {+ let red = solid([255, 0, 0, 255]);+ let bw = run(&pass("concat.black-white", &[]), &red, 0.0).expect("kernel");+ let y = (0.2126f32 * 255.0).round() as u8;+ assert_eq!(bw.pixel(0, 0), Some([y, y, y, 255]));+ let inverted = run(&pass("concat.invert", &[]), &red, 0.0).expect("kernel");+ assert_eq!(inverted.pixel(0, 0), Some([0, 255, 255, 255]));+ let sepia = run(&pass("concat.sepia", &[]), &red, 0.0).expect("kernel");+ assert_eq!(sepia.pixel(0, 0), Some([100, 89, 69, 255]));+ // A full turn of the hue is the colour again, near enough.+ let turned =+ run(&pass("concat.hue-shift", &[("angle", 360.0)]), &red, 0.0).expect("kernel");+ let [r, g, b, _] = turned.pixel(0, 0).expect("in");+ assert!(r >= 250 && g <= 5 && b <= 5, "{r} {g} {b}");+ // Alpha rides through every colour kernel untouched.+ let half = solid([255, 0, 0, 128]);+ assert_eq!(+ run(&pass("concat.invert", &[]), &half, 0.0)+ .expect("kernel")+ .pixel(0, 0),+ Some([0, 255, 255, 128])+ );+ }++ #[test]+ fn a_blur_of_a_flat_picture_is_the_picture_and_the_edges_stay_put() {+ let flat = solid([90, 120, 30, 255]);+ for package in ["concat.box-blur", "concat.gaussian-blur"] {+ let out = run(&pass(package, &[("radius", 8.0)]), &flat, 0.0).expect("kernel");+ assert_eq!(out.pixel(0, 0), Some([90, 120, 30, 255]), "{package}");+ assert_eq!(out.pixel(3, 3), Some([90, 120, 30, 255]), "{package}");+ }+ // A negative radius is the floor, not a panic.+ let out = run(&pass("concat.box-blur", &[("radius", -9.0)]), &flat, 0.0).expect("kernel");+ assert_eq!(out.pixel(1, 1), Some([90, 120, 30, 255]));+ let mut edge = Frame::transparent(4, 1);+ edge.set_pixel(0, 0, [255, 255, 255, 255]);+ let out = run(&pass("concat.box-blur", &[("radius", 4.0)]), &edge, 0.0).expect("kernel");+ let [near, ..] = out.pixel(0, 0).expect("in");+ let [far, ..] = out.pixel(3, 0).expect("in");+ assert!(+ near > far,+ "the white spreads but stays brightest where it was: {near} vs {far}"+ );+ }++ #[test]+ fn a_vignette_darkens_the_corners_and_not_the_centre() {+ let mut flat = Frame::transparent(64, 64);+ flat.fill([200, 200, 200, 255]);+ let out =+ run(&pass("concat.vignette", &[("strength", 100.0)]), &flat, 0.0).expect("kernel");+ let [centre, ..] = out.pixel(32, 32).expect("in");+ let [corner, ..] = out.pixel(0, 0).expect("in");+ assert!(centre >= 195, "{centre}");+ assert!(corner < centre, "{corner} < {centre}");+ let out = run(&pass("concat.vignette", &[("strength", 0.0)]), &flat, 0.0).expect("kernel");+ assert_eq!(+ out.pixel(0, 0),+ Some([200, 200, 200, 255]),+ "no strength, no fall"+ );+ }+}diff --git a/src/crates/concat-render/src/lib.rs b/src/crates/concat-render/src/lib.rsindex ba75dc96..f81f2e19 100644--- a/src/crates/concat-render/src/lib.rs+++ b/src/crates/concat-render/src/lib.rs@@ -7,21 +7,31 @@//!//! 1. [`plan`] answers "what is on screen at this instant, from where, and how//! strongly". It touches no pixels and does no IO, so it is fast, exactly-//! testable, and identical for the CPU and GPU backends.-//! 2. [`compositor`] takes that plan plus the decoded pixels and blends them.+//! testable, and identical for the CPU and GPU backends. The executor+//! fills the plan out with the decoded pictures and everything the+//! model has no field for, and a [`FramePlan`] is then the whole+//! description of the frame.+//! 2. [`compositor`] takes that plan and nothing else, and draws it.//!//! Only step 2 is backend-specific. The CPU compositor is the reference-//! implementation; the GPU one exists to be fast and must match it.+//! implementation; the GPU one exists to be fast and must match it, which+//! the parity suite holds it to.pub mod compositor;#[cfg(feature = "gpu")]pub mod gpu;+pub mod kernels;+pub mod metrics;pub mod plan;-pub use compositor::{Compositor, CpuCompositor, Layer, Placement, Treatment};+pub use compositor::{Compositor, CpuCompositor};#[cfg(feature = "gpu")]pub use gpu::WgpuCompositor;-pub use plan::{FramePlan, PlannedLayer, plan_frame};+pub use metrics::ssim;+pub use plan::{+ Crop, FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading, Transition, detached_clip,+ plan_frame,+};/// The wgpu the compositor is built on, for callers that share its device.#[cfg(feature = "gpu")]pub use wgpu;diff --git a/src/crates/concat-render/src/metrics.rs b/src/crates/concat-render/src/metrics.rsnew file mode 100644index 00000000..fd8acf5f--- /dev/null+++ b/src/crates/concat-render/src/metrics.rs@@ -0,0 +1,113 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! How alike two frames are, as a number: what the parity suite holds the+//! GPU to, and what a benchmark reports.+//!+//! [`ssim`] is the structural similarity index over the luminance, on+//! 8×8 windows, the usual constants: one for identical pictures, near+//! zero for unrelated ones. It forgives a rounding and a resampling's+//! blur and does not forgive a missing layer, a wrong placement or a+//! shader that did something else, which is the distinction a parity+//! check needs.++use concat_core::frame::{BYTES_PER_PIXEL, Frame};++/// The window the index is averaged over, a side.+const WINDOW: usize = 8;++/// The structural similarity of `a` and `b`, `0..=1`; zero when their+/// sizes differ, one when they are the same picture.+pub fn ssim(a: &Frame, b: &Frame) -> f64 {+ if a.width() != b.width() || a.height() != b.height() || a.width() == 0 || a.height() == 0 {+ return 0.0;+ }+ let (width, height) = (a.width() as usize, a.height() as usize);+ let luma = |frame: &Frame| -> Vec<f64> {+ frame+ .pixels()+ .chunks_exact(BYTES_PER_PIXEL)+ .map(|px| {+ 0.2126 * f64::from(px[0]) + 0.7152 * f64::from(px[1]) + 0.0722 * f64::from(px[2])+ })+ .collect()+ };+ let (la, lb) = (luma(a), luma(b));+ let (c1, c2) = ((0.01f64 * 255.0).powi(2), (0.03f64 * 255.0).powi(2));+ let mut total = 0.0;+ let mut windows = 0.0;+ let step = WINDOW.min(width).min(height).max(1);+ for y in (0..height).step_by(step) {+ for x in (0..width).step_by(step) {+ let (w, h) = (step.min(width - x), step.min(height - y));+ let n = (w * h) as f64;+ let (mut ma, mut mb) = (0.0, 0.0);+ for row in y..y + h {+ for col in x..x + w {+ ma += la[row * width + col];+ mb += lb[row * width + col];+ }+ }+ ma /= n;+ mb /= n;+ let (mut va, mut vb, mut cov) = (0.0, 0.0, 0.0);+ for row in y..y + h {+ for col in x..x + w {+ let (da, db) = (la[row * width + col] - ma, lb[row * width + col] - mb);+ va += da * da;+ vb += db * db;+ cov += da * db;+ }+ }+ let denominator = if n > 1.0 { n - 1.0 } else { 1.0 };+ va /= denominator;+ vb /= denominator;+ cov /= denominator;+ total += ((2.0 * ma * mb + c1) * (2.0 * cov + c2))+ / ((ma * ma + mb * mb + c1) * (va + vb + c2));+ windows += 1.0;+ }+ }+ (total / windows).clamp(0.0, 1.0)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn gradient(width: u32, height: u32, seed: u8) -> Frame {+ let mut frame = Frame::black(width, height);+ for y in 0..height {+ for x in 0..width {+ let v = (x * 7 + y * 3 + u32::from(seed)) % 256;+ frame.set_pixel(x, y, [v as u8, (v / 2) as u8, 255 - v as u8, 255]);+ }+ }+ frame+ }++ #[test]+ fn the_same_picture_is_one_and_a_different_size_is_zero() {+ let a = gradient(32, 24, 0);+ assert!((ssim(&a, &a) - 1.0).abs() < 1e-9);+ assert_eq!(ssim(&a, &gradient(16, 24, 0)), 0.0);+ assert_eq!(ssim(&Frame::black(0, 0), &Frame::black(0, 0)), 0.0);+ }++ #[test]+ fn a_rounding_is_forgiven_and_a_missing_layer_is_not() {+ let a = gradient(32, 32, 0);+ let mut rounded = a.clone();+ for px in rounded.pixels_mut().chunks_exact_mut(4) {+ px[0] = px[0].saturating_add(1);+ }+ assert!(ssim(&a, &rounded) > 0.99, "{}", ssim(&a, &rounded));+ assert!(ssim(&a, &Frame::black(32, 32)) < 0.5);+ let mut flat = Frame::black(32, 32);+ flat.fill([128, 128, 128, 255]);+ assert!(ssim(&a, &flat) < 0.9);+ // A window smaller than eight a side still counts.+ let tiny = gradient(3, 5, 0);+ assert!((ssim(&tiny, &tiny) - 1.0).abs() < 1e-9);+ }+}diff --git a/src/crates/concat-render/src/plan.rs b/src/crates/concat-render/src/plan.rsindex 597c8b60..a9740cff 100644--- a/src/crates/concat-render/src/plan.rs+++ b/src/crates/concat-render/src/plan.rs@@ -1,26 +1,189 @@// SPDX-License-Identifier: AGPL-3.0-or-later// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors-//! Deciding what is on screen at a given instant.+//! Deciding what is on screen at a given instant, and describing it whole.//!-//! A [`FramePlan`] is the contract between the editor model and whatever draws-//! pixels. It names the media, the exact source timestamp and the blend-//! strength for each visible layer, bottom-most first, and it does so without-//! opening a single file.+//! A [`FramePlan`] is the contract between the editor model and whatever+//! draws pixels: the one description of a frame that both compositors+//! consume, and nothing else reaches them. It is built in two steps.//!-//! Keeping this pure is what lets you unit-test "does a cut land on the right-//! frame" in microseconds instead of by exporting a file and squinting at it.+//! [`plan_frame`] answers "what is on screen, from where, and how strongly":+//! for each visible layer, bottom-most first, the media, the exact source+//! timestamp, the placement, the opacity and the blend. It touches no+//! pixels and does no IO, so it is fast, exactly testable, and identical for+//! the CPU and GPU backends. The executor then fills in what the model has+//! no field for - the decoded picture, the crop, the flips, the effect chain+//! resolved for this frame, the mask, the transitions at the cut, the+//! treatments live over the stack - and hands the plan to a compositor.+//!+//! What a compositor does with one layer, in order: the crop, the flips and+//! the fit make the picture; the effects run over it; the mask, the fades+//! and the wipes weigh it; the placement puts it in the frame; the blend+//! meets what is beneath. The geometry and the weighing are worked out+//! here, once, so the two backends cannot drift on them.use std::path::PathBuf;+use std::sync::Arc;+use concat_core::frame::Frame;+use concat_core::shader::ShaderPass;use concat_core::time::Rational;-use concat_core::timeline::{ClipId, Timeline, TrackKind, Transform};+use concat_core::timeline::{Blend, ClipId, Timeline, TrackKind, Transform};++/// Fractions cut off a source's left, top, right and bottom before it is+/// fitted, as the document stores them.+#[derive(Clone, Copy, PartialEq, Debug, Default)]+pub struct Crop {+ /// Cut off the left edge, `0..1` of the width.+ pub left: f32,+ /// Cut off the top edge, `0..1` of the height.+ pub top: f32,+ /// Cut off the right edge, `0..1` of the width.+ pub right: f32,+ /// Cut off the bottom edge, `0..1` of the height.+ pub bottom: f32,+}++impl Crop {+ /// The whole source.+ pub const NONE: Crop = Crop {+ left: 0.0,+ top: 0.0,+ right: 0.0,+ bottom: 0.0,+ };++ /// The document's four fractions, in its order.+ pub fn of(edges: [f64; 4]) -> Crop {+ let [left, top, right, bottom] = edges.map(|edge| edge.clamp(0.0, 1.0) as f32);+ Crop {+ left,+ top,+ right,+ bottom,+ }+ }++ /// True when nothing is cut.+ pub fn is_none(&self) -> bool {+ self.left <= 0.0 && self.top <= 0.0 && self.right <= 0.0 && self.bottom <= 0.0+ }++ /// What is left of a `width` by `height` source, in its pixels: the+ /// origin and the size. At least a tenth each way, as the document's+ /// crop is bounded, and never past the source's edge.+ pub fn rect(&self, width: u32, height: u32) -> [f32; 4] {+ let (w, h) = (width as f32, height as f32);+ let x = (self.left.clamp(0.0, 1.0) * w).min(w - 1.0).max(0.0);+ let y = (self.top.clamp(0.0, 1.0) * h).min(h - 1.0).max(0.0);+ let cw = ((1.0 - self.left - self.right).max(0.1) * w)+ .max(1.0)+ .min(w - x);+ let ch = ((1.0 - self.top - self.bottom).max(0.1) * h)+ .max(1.0)+ .min(h - y);+ [x, y, cw, ch]+ }+}++/// What a cut does to a layer at one frame: the picture pulled towards a+/// colour, or uncovered behind a moving edge. Resolved by the executor+/// from the transition's shape and the frame's place in it; the compositor+/// only ever sees an amount.+#[derive(Clone, Copy, PartialEq, Debug)]+pub enum Transition {+ /// The picture mixed towards `colour` by `amount`, `0..=1`: a fade to+ /// black or to white, at one frame of it.+ FadeTo {+ /// The colour faded to, `0..=1` a channel.+ colour: [f32; 3],+ /// How far towards it: one is the colour outright.+ amount: f32,+ },+ /// A straight vertical edge with the picture shown on one side and+ /// gone on the other. `uncovered` is the fraction of the picture's+ /// width shown: from the left edge, or from the right when `from_right`.+ Wipe {+ /// How much of the width is shown, `0..=1`.+ uncovered: f32,+ /// Whether the shown part is at the right edge rather than the left.+ from_right: bool,+ },+}++/// A layer's transitions folded into what they do to one pixel: the colour+/// scaled and offset, and the span of the width kept. Every fade composes+/// into one affine map of the colour, and every wipe into one edge a side,+/// so a compositor applies the lot in one line.+#[derive(Clone, Copy, PartialEq, Debug)]+pub struct Shading {+ /// What the colour is multiplied by.+ pub scale: f32,+ /// What is then added to it.+ pub offset: [f32; 3],+ /// A pixel is kept where its `u` is below this.+ pub left_edge: f32,+ /// A pixel is kept where its `u` is at or beyond this.+ pub right_edge: f32,+}++impl Shading {+ /// Nothing done: the colour as it is, the whole width kept.+ pub const NONE: Shading = Shading {+ scale: 1.0,+ offset: [0.0; 3],+ left_edge: 2.0,+ right_edge: -1.0,+ };++ /// The transitions of a layer, folded in order.+ pub fn of(transitions: &[Transition]) -> Shading {+ let mut shading = Shading::NONE;+ for transition in transitions {+ match *transition {+ Transition::FadeTo { colour, amount } => {+ let amount = amount.clamp(0.0, 1.0);+ shading.scale *= 1.0 - amount;+ for (channel, target) in shading.offset.iter_mut().zip(colour) {+ *channel = *channel * (1.0 - amount) + target * amount;+ }+ }+ Transition::Wipe {+ uncovered,+ from_right,+ } => {+ let uncovered = uncovered.clamp(0.0, 1.0);+ if from_right {+ shading.right_edge = shading.right_edge.max(1.0 - uncovered);+ } else {+ shading.left_edge = shading.left_edge.min(uncovered);+ }+ }+ }+ }+ shading+ }++ /// Whether a pixel `u` of the way across the picture is shown at all.+ pub fn keeps(&self, u: f32) -> bool {+ u < self.left_edge && u >= self.right_edge+ }++ /// One channel of the colour, `0..=1`, through the fades.+ pub fn colour(&self, channel: usize, value: f32) -> f32 {+ value * self.scale + self.offset[channel]+ }+}/// One visible layer at one instant.-#[derive(Clone, PartialEq, Debug)]+#[derive(Clone, Debug)]pub struct PlannedLayer {/// Which clip produced this layer.pub clip: ClipId,+ /// The track the layer sits on, bottom-most zero: every layer on a+ /// lower track is under it, and a treatment names the track it sits+ /// above.+ pub track: usize,/// The file to pull pixels from.pub media: PathBuf,/// The timestamp *within that file* to pull.@@ -29,19 +192,238 @@ pub struct PlannedLayer {/// frame per output frame must decode at `output_rate / speed` to stay in/// step with `source_time`.pub speed: Rational,- /// Blend strength over everything beneath, in `0.0..=1.0`.- pub opacity: f32,- /// The clip's placement in the frame, resolution-independent.- pub transform: Transform,/// Whether one decoder paced at `output rate / speed` follows this clip;/// false for a curve or a reverse, where each frame is sought.pub paced: bool,+ /// The decoded picture, once the executor has fetched it: the source at+ /// whatever size the decoder gave it. `None` until then, and a layer+ /// without one draws nothing rather than failing the frame.+ pub source: Option<Arc<Frame>>,+ /// What is cut off the source's edges before it is fitted. `NONE` for+ /// a picture the decoder already cropped.+ pub crop: Crop,+ /// Mirrored left to right, after the crop and before the effects.+ pub flip_h: bool,+ /// Mirrored top to bottom, the same.+ pub flip_v: bool,+ /// The clip's placement in the frame, resolution-independent, over the+ /// fitted picture.+ pub transform: Transform,+ /// The effects to run over the picture, in order, each resolved for+ /// this frame. The GPU runs the shader; the CPU runs its kernel for the+ /// package, or draws the picture untreated when it has none.+ pub effects: Vec<ShaderPass>,+ /// A matte over the picture, its alpha the coverage, sampled across the+ /// picture whatever its own size: what a cutout leaves. `None` keeps+ /// the picture whole.+ pub mask: Option<Arc<Frame>>,+ /// What the cuts at either end do to the picture at this frame.+ pub transitions: Vec<Transition>,/// How the layer's colour meets what is beneath it.- pub blend: concat_core::timeline::Blend,+ pub blend: Blend,+ /// Blend strength over everything beneath, in `0.0..=1.0`.+ pub opacity: f32,+}++impl PlannedLayer {+ /// A layer of `frame` with nothing else set: opaque, unmoved, whole,+ /// untreated, on track zero. Where a test or a tool that already has+ /// its pixels starts from.+ pub fn picture(clip: ClipId, frame: Arc<Frame>) -> PlannedLayer {+ PlannedLayer {+ clip,+ track: 0,+ media: PathBuf::new(),+ source_time: Rational::ZERO,+ speed: Rational::ONE,+ paced: true,+ source: Some(frame),+ crop: Crop::NONE,+ flip_h: false,+ flip_v: false,+ transform: Transform::default(),+ effects: Vec::new(),+ mask: None,+ transitions: Vec::new(),+ blend: Blend::Normal,+ opacity: 1.0,+ }+ }++ /// How hard the layer is drawn: its opacity, held to `0..=1`, and+ /// nothing at all for a number that is not one - a NaN from a bad+ /// key is an invisible layer, not a frame of garbage.+ pub fn weight(&self) -> f32 {+ if self.opacity.is_finite() {+ self.opacity.clamp(0.0, 1.0)+ } else {+ 0.0+ }+ }++ /// The transitions folded; see [`Shading`].+ pub fn shading(&self) -> Shading {+ Shading::of(&self.transitions)+ }++ /// Where the picture's texels come from and where they land, for an+ /// output `width` by `height`; see [`Geometry`].+ pub fn geometry(&self, source: &Frame, width: u32, height: u32) -> Geometry {+ Geometry::of(self, source.width(), source.height(), width, height)+ }++ /// Whether the picture has to be made before the effects can run: the+ /// effects read texels of the picture as it will be seen, so a crop, a+ /// flip or a size the fit changes cannot wait for the placement. With+ /// no effects, all three fold into the placement for nothing.+ pub fn needs_preparing(&self, geometry: &Geometry) -> bool {+ !self.effects.is_empty()+ && (!self.crop.is_none() || self.flip_h || self.flip_v || !geometry.fits_source())+ }+}++/// Where a layer's picture comes from and where it lands: the arithmetic+/// both compositors share, so a picture cannot be placed two ways.+///+/// The picture is the crop rectangle of the source, fitted inside the+/// output - contain, the aspect kept - and centred; that fitted size is+/// what the effects run at and what the transform then scales, turns and+/// moves about the centre. A decoder that already cropped and fitted the+/// source hands back a picture the fit leaves alone.+#[derive(Clone, Copy, PartialEq, Debug)]+pub struct Geometry {+ /// The source pixels the picture is made of: x, y, width, height.+ pub source_rect: [f32; 4],+ /// The size of the source the rectangle is of.+ pub source_size: (u32, u32),+ /// The picture's size once fitted: the size the effects run at.+ pub fitted: (u32, u32),+ /// Where the picture's centre lands in the output, after translation.+ pub centre: (f32, f32),+ /// The transform's scale per axis over the fitted size: the scale+ /// times the stretch.+ pub scale: (f32, f32),+ /// Clockwise rotation about the centre, radians.+ pub rotation: f32,+}++impl Geometry {+ /// The geometry of `layer` over a `source_width` by `source_height`+ /// source in an output `width` by `height`.+ pub fn of(+ layer: &PlannedLayer,+ source_width: u32,+ source_height: u32,+ width: u32,+ height: u32,+ ) -> Geometry {+ let source_rect = layer.crop.rect(source_width, source_height);+ let [_, _, cw, ch] = source_rect;+ let fit = (width as f32 / cw).min(height as f32 / ch);+ let fitted = (+ ((cw * fit).round() as u32).max(1),+ ((ch * fit).round() as u32).max(1),+ );+ // Centred by whole pixels, the way a frame the decoder fitted has+ // always been placed, then moved by the transform's fractions.+ let base_x = (i64::from(width) - i64::from(fitted.0)) / 2;+ let base_y = (i64::from(height) - i64::from(fitted.1)) / 2;+ let transform = &layer.transform;+ let centre = (+ base_x as f32 + fitted.0 as f32 / 2.0 + (transform.offset_x * f64::from(width)) as f32,+ base_y as f32 + fitted.1 as f32 / 2.0 + (transform.offset_y * f64::from(height)) as f32,+ );+ let scale = (+ ((transform.scale * transform.stretch_x) as f32).max(1e-6),+ ((transform.scale * transform.stretch_y) as f32).max(1e-6),+ );+ Geometry {+ source_rect,+ source_size: (source_width, source_height),+ fitted,+ centre,+ scale,+ rotation: transform.rotation.to_radians() as f32,+ }+ }++ /// The geometry once the picture has been made at its fitted size:+ /// the whole of it, unflipped, the same placement.+ pub fn prepared(&self) -> Geometry {+ Geometry {+ source_rect: [0.0, 0.0, self.fitted.0 as f32, self.fitted.1 as f32],+ source_size: self.fitted,+ ..*self+ }+ }++ /// True when the fitted picture is the whole source at its own size.+ pub fn fits_source(&self) -> bool {+ let [x, y, w, h] = self.source_rect;+ x == 0.0+ && y == 0.0+ && w == self.source_size.0 as f32+ && h == self.source_size.1 as f32+ && self.fitted == self.source_size+ }++ /// True when the placement is the fitted picture, unscaled, unturned,+ /// and moved by whole pixels: the case a compositor can copy rather+ /// than resample.+ pub fn is_aligned(&self) -> bool {+ self.scale == (1.0, 1.0)+ && self.rotation == 0.0+ && (self.centre.0 - self.fitted.0 as f32 / 2.0).fract() == 0.0+ && (self.centre.1 - self.fitted.1 as f32 / 2.0).fract() == 0.0+ }++ /// The top-left corner of an aligned placement, in output pixels.+ pub fn corner(&self) -> (i32, i32) {+ (+ (self.centre.0 - self.fitted.0 as f32 / 2.0).round() as i32,+ (self.centre.1 - self.fitted.1 as f32 / 2.0).round() as i32,+ )+ }++ /// The source pixel under a point of the picture, `u` and `v` in+ /// `0..1` across the picture as it is seen, through `flip_h` and+ /// `flip_v`: the texel coordinate a sampler reads at, its centre at+ /// `.0`, like the GPU's.+ pub fn source_of(&self, u: f32, v: f32, flip_h: bool, flip_v: bool) -> (f32, f32) {+ let u = if flip_h { 1.0 - u } else { u };+ let v = if flip_v { 1.0 - v } else { v };+ let [x, y, w, h] = self.source_rect;+ (x + u * w - 0.5, y + v * h - 0.5)+ }++ /// The same, as texture coordinates in `0..1` of the source.+ pub fn uv_of(&self, u: f32, v: f32, flip_h: bool, flip_v: bool) -> (f32, f32) {+ let u = if flip_h { 1.0 - u } else { u };+ let v = if flip_v { 1.0 - v } else { v };+ let [x, y, w, h] = self.source_rect;+ (+ (x + u * w) / self.source_size.0 as f32,+ (y + v * h) / self.source_size.1 as f32,+ )+ }+}++/// A layer clip live at this frame: its effects run over everything+/// composited beneath its track, and the result is blended back over the+/// untreated stack by its strength.+#[derive(Clone, Debug)]+pub struct PlannedTreatment {+ /// The track the layer sits on; every layer on a lower track is under it.+ pub track: usize,+ /// The effects to run over the stack beneath, in order, resolved for+ /// this frame.+ pub effects: Vec<ShaderPass>,+ /// How much of the treated stack to keep over the untreated, `0..=1`.+ pub strength: f32,}/// Everything needed to draw one output frame.-#[derive(Clone, PartialEq, Debug)]+#[derive(Clone, Debug)]pub struct FramePlan {/// The timeline instant this describes.pub time: Rational,@@ -51,25 +433,60 @@ pub struct FramePlan {pub height: u32,/// Visible layers, bottom-most first.pub layers: Vec<PlannedLayer>,+ /// The treatments live at this instant, in ascending track order.+ pub treatments: Vec<PlannedTreatment>,}impl FramePlan {+ /// A frame with nothing on it: black.+ pub fn empty(width: u32, height: u32) -> FramePlan {+ FramePlan {+ time: Rational::ZERO,+ width,+ height,+ layers: Vec::new(),+ treatments: Vec::new(),+ }+ }+/// True if nothing is on screen. The result is a black frame, not an error:/// gaps in a timeline are ordinary.pub fn is_empty(&self) -> bool {self.layers.is_empty()}++ /// The instant in seconds, for effects that move.+ pub fn seconds(&self) -> f32 {+ self.time.as_f64() as f32+ }+}++/// A clip id for a layer that belongs to no timeline: a picture handed+/// in directly, a trial run of a package, a test. Minted from a timeline+/// of its own, since an id is an arena's.+pub fn detached_clip() -> ClipId {+ use concat_core::time::FrameRate;+ use concat_core::timeline::{Clip, MediaRef, Track};+ let mut timeline = Timeline::new(16, 16, FrameRate::THIRTY);+ let track = timeline.add_track(Track::new("detached", TrackKind::Video));+ timeline+ .add_clip(+ track,+ Clip::new(MediaRef::new("detached"), Rational::ZERO, Rational::ONE),+ )+ .expect("the track was just added")}/// Works out what is on screen at `time`.////// Audio tracks and disabled tracks are skipped. A track with no clip under the-/// playhead simply contributes nothing.+/// playhead simply contributes nothing. Every layer comes back with no+/// picture, no effects and no mask: those are the executor's to fill in.pub fn plan_frame(timeline: &Timeline, time: Rational) -> FramePlan {let mut layers = Vec::new();- for (track_id, track) in timeline.tracks() {- if !track.enabled || track.kind != TrackKind::Video {+ for (track, (track_id, lane)) in timeline.tracks().enumerate() {+ if !lane.enabled || lane.kind != TrackKind::Video {continue;}let Some(clip_id) = timeline.clip_on_track_at(track_id, time) else {@@ -84,15 +501,23 @@ pub fn plan_frame(timeline: &Timeline, time: Rational) -> FramePlan {layers.push(PlannedLayer {clip: clip_id,+ track,media: clip.media.path.clone(),source_time,speed: clip.speed,+ paced: clip.is_paced(),+ source: None,+ crop: Crop::NONE,+ flip_h: false,+ flip_v: false,+ transform: clip.transform_at(time),+ effects: Vec::new(),+ mask: None,+ transitions: Vec::new(),+ blend: clip.blend,// The fade ramp multiplies in here, so the compositor only ever// sees a per-frame opacity - it has no idea fades exist.opacity: (clip.opacity_at(time) * clip.video_fade_factor(time)).clamp(0.0, 1.0),- transform: clip.transform_at(time),- paced: clip.is_paced(),- blend: clip.blend,});}@@ -101,6 +526,7 @@ pub fn plan_frame(timeline: &Timeline, time: Rational) -> FramePlan {width: timeline.width,height: timeline.height,layers,+ treatments: Vec::new(),}}@@ -144,6 +570,8 @@ mod tests {vec![PathBuf::from("bottom.mp4"), PathBuf::from("top.mp4")]);assert_eq!((plan.width, plan.height), (640, 360));+ assert_eq!(plan.layers[0].track, 0);+ assert_eq!(plan.layers[1].track, 1);}#[test]@@ -235,4 +663,77 @@ mod tests {assert_eq!(plan_frame(&timeline, seconds(1)).layers[0].opacity, 1.0);}++ fn a_clip() -> ClipId {+ let mut timeline = Timeline::new(8, 8, FrameRate::THIRTY);+ let track = timeline.add_track(Track::new("V1", TrackKind::Video));+ timeline+ .add_clip(+ track,+ Clip::new(MediaRef::new("a.mp4"), seconds(0), seconds(1)),+ )+ .expect("track exists")+ }++ /// A crop's rectangle is the source's pixels, never past its edge; a+ /// fitted picture keeps its aspect and sits in the middle by whole+ /// pixels; a picture the decoder fitted already is left as it is.+ #[test]+ fn the_geometry_fits_the_crop_and_centres_it() {+ let mut layer = PlannedLayer::picture(a_clip(), Arc::new(Frame::black(100, 50)));+ let whole = Geometry::of(&layer, 100, 50, 100, 50);+ assert!(whole.fits_source());+ assert!(whole.is_aligned());+ assert_eq!(whole.corner(), (0, 0));++ let fitted = Geometry::of(&layer, 100, 50, 200, 200);+ assert_eq!(fitted.fitted, (200, 100));+ assert_eq!(fitted.centre, (100.0, 100.0));+ assert_eq!(fitted.corner(), (0, 50));+ assert!(!fitted.fits_source());++ layer.crop = Crop::of([0.25, 0.0, 0.25, 0.0]);+ let cropped = Geometry::of(&layer, 100, 50, 100, 50);+ assert_eq!(cropped.source_rect, [25.0, 0.0, 50.0, 50.0]);+ assert_eq!(cropped.fitted, (50, 50));+ assert_eq!(cropped.corner(), (25, 0));+ // The first texel of the crop is a sampler's half-texel in.+ assert_eq!(cropped.source_of(0.0, 0.0, false, false), (24.5, -0.5));+ assert_eq!(cropped.source_of(1.0, 0.0, true, false), (24.5, -0.5));+ assert_eq!(cropped.uv_of(1.0, 1.0, false, false), (0.75, 1.0));+ }++ /// Fades compose into one map of the colour and wipes into one edge a+ /// side, so two fades to black at half are three quarters black and a+ /// wipe keeps only what its edge has uncovered.+ #[test]+ fn transitions_fold_into_one_shading() {+ let shading = Shading::of(&[+ Transition::FadeTo {+ colour: [0.0; 3],+ amount: 0.5,+ },+ Transition::FadeTo {+ colour: [1.0; 3],+ amount: 0.5,+ },+ ]);+ assert!((shading.colour(0, 1.0) - 0.75).abs() < 1e-6);+ assert!((shading.colour(0, 0.0) - 0.5).abs() < 1e-6);+ assert!(shading.keeps(0.0) && shading.keeps(0.999));++ let wipe = Shading::of(&[Transition::Wipe {+ uncovered: 0.25,+ from_right: true,+ }]);+ assert!(!wipe.keeps(0.5));+ assert!(wipe.keeps(0.8));+ let wipe = Shading::of(&[Transition::Wipe {+ uncovered: 0.25,+ from_right: false,+ }]);+ assert!(wipe.keeps(0.1));+ assert!(!wipe.keeps(0.5));+ assert_eq!(Shading::of(&[]), Shading::NONE);+ }}