Skip to content

EffectCraft: An open-source motion graphics and compositing tool written by scratch in pure Rust

Merge main into fix/2022-render-banding (#1631)

Claude85e061fe0c5d
diff --cc crates/export/src/pipeline.rs
index 128ab267c,ae52d3478..66d603b85
--- a/crates/export/src/pipeline.rs
+++ b/crates/export/src/pipeline.rs
@@@ -297,6 -316,63 +309,65 @@@ impl<'a> Cx<'a>
img
}
+ /// Output frame `i` as the H.264 export's 4:2:0 planes at `w`×`h`, rendered and converted
+ /// by the accelerator ([`effectcraft_render::Accelerator::comp_frame_yuv420`]). Only for
+ /// frames that need nothing done after the render (progressive, at the project's depth, no
+ /// crop or resize, rendered at exactly `w`×`h`). `None`: render it with [`Cx::frame`].
+ pub fn yuv420_frame(&self, comp: &Comp, i: u64, w: u32, h: u32) -> Option<Vec<u8>> {
+ if self.deferred() {
+ return None;
+ }
+ let FrameSample::Progressive(t) = self.job.settings.sample(comp, i) else { return None };
- let depth = self.job.settings.color_depth;
- if depth != ColorDepth::Current && depth.resolve(self.project.settings.bit_depth).levels().is_some() {
++ // Above 8 bpc the colour is dithered on its way to 8 bits (`Cx::pixels`), which the
++ // accelerator's conversion doesn't do: those frames render as usual so H.264 doesn't
++ // band (#2022). The project's depth already includes Render Settings ▸ Color Depth.
++ if self.project.settings.bit_depth != BitDepth::Bpc8 {
+ return None;
+ }
+ let om = self.job.output;
+ let scale = self.job.settings.resolution.clamp(0.01, 4.0);
+ let size = |n: u32| ((n as f64 * scale).round() as u32).max(1);
+ if om.resize.enabled || (size(comp.width), size(comp.height)) != (w, h) || om.crop.edges(w, h, scale) != (0, 0, 0, 0) {
+ return None;
+ }
+ let t0 = Instant::now();
+ let r = self.renderer();
+ let planes = r.active_accel()?.comp_frame_yuv420(&r, self.job.comp, t, comp.background)?;
+ let (cw, ch) = (w.div_ceil(2) as usize, h.div_ceil(2) as usize);
+ if planes.len() != w as usize * h as usize + 2 * cw * ch {
+ return None;
+ }
+ if let Some(ft) = &self.frame_times {
+ ft.lock().unwrap_or_else(|e| e.into_inner()).push((i, t0.elapsed().as_secs_f64()));
+ }
+ Some(planes)
+ }
+
+ /// Output frames `ks` for the H.264 encoder: the accelerator's planes
+ /// ([`Cx::yuv420_frame`]) where it can, otherwise `fallback` of the frame rendered as usual.
+ pub(crate) fn h264_frames<T: Send>(
+ &self,
+ comp: &Comp,
+ ks: Vec<u64>,
+ w: u32,
+ h: u32,
+ planes: impl Fn(Vec<u8>) -> T + Sync + Send,
+ fallback: impl Fn(u64, Image) -> T + Sync + Send,
+ ) -> Vec<T> {
+ // As in `frames`: once the GPU has stopped for good, the rest render in parallel.
+ let gpu_live = || self.job.accel.is_none_or(|a| a.available());
+ map_frames_while(
+ ks,
+ self.gpu_cap(),
+ gpu_live,
+ |k| self.yuv420_frame(comp, k, w, h).ok_or_else(|| self.frame(comp, k)),
+ |k, r| match r {
+ Ok(p) => planes(p),
+ Err(img) => fallback(k, img),
+ },
+ )
+ }
+
/// Output Module ▸ Crop, then Resize.
fn crop_resize(&self, img: Image) -> Image {
let om = self.job.output;
diff --cc crates/export/tests/it/h264_gpu_yuv.rs
index 000000000,59e670779..0f99a7b54
mode 000000,100644..100644
--- a/crates/export/tests/it/h264_gpu_yuv.rs
+++ b/crates/export/tests/it/h264_gpu_yuv.rs
@@@ -1,0 -1,180 +1,194 @@@
+ //! H.264 export with an accelerator that converts frames to 4:2:0 itself
+ //! (`Accelerator::comp_frame_yuv420`): the file is the same as the CPU conversion's, and frames
+ //! the accelerator can't take (odd sizes, crop) fall back to the RGBA path.
+
+ use std::path::{Path, PathBuf};
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ use effectcraft_color::Label;
-use effectcraft_export::render_queue::{OutputFormat, OutputModule, RenderSettings, TimeSpan};
++use effectcraft_export::render_queue::{ColorDepth, OutputFormat, OutputModule, RenderSettings, TimeSpan};
+ use effectcraft_export::{Job, export};
+ use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, Project, Solid, build};
+ use effectcraft_raster::Image;
+ use effectcraft_render::{Accelerator, NoFootage, Renderer};
+ use effectcraft_time::{FrameRate, Tick};
+
+ /// Renders on the CPU; converts to 4:2:0 with the export's own function when `yuv` is set.
+ /// `deferred`: claims deferred readbacks (a browser worker's GPU), whose frames render in passes.
+ /// `stop_after`: the device stops for good after that many planes (`available` turns false).
+ struct Fake {
+ yuv: bool,
+ yuv_frames: AtomicUsize,
+ deferred: bool,
+ stop_after: usize,
+ /// Frames asking for planes right now, and the most at once after the stop.
+ asking: AtomicUsize,
+ most_after_stop: AtomicUsize,
+ }
+
+ impl Fake {
+ fn new(yuv: bool, deferred: bool, stop_after: usize) -> Self {
+ Fake { yuv, yuv_frames: AtomicUsize::new(0), deferred, stop_after, asking: AtomicUsize::new(0), most_after_stop: AtomicUsize::new(0) }
+ }
+ }
+
+ impl Accelerator for Fake {
+ fn name(&self) -> String {
+ "fake".into()
+ }
+ fn comp_frame(&self, r: &Renderer, comp: ItemId, t: Tick) -> Option<Image> {
+ Some(r.comp_frame_cpu(comp, t))
+ }
+ fn comp_frame_yuv420(&self, r: &Renderer, comp: ItemId, t: Tick, bg: [f32; 3]) -> Option<Vec<u8>> {
+ if !self.yuv {
+ return None;
+ }
+ if self.yuv_frames.fetch_add(1, Ordering::SeqCst) >= self.stop_after {
+ self.yuv_frames.fetch_sub(1, Ordering::SeqCst);
+ // Stopped: every frame still asks first, so asks that overlap are frames rendering at once.
+ let now = self.asking.fetch_add(1, Ordering::SeqCst) + 1;
+ self.most_after_stop.fetch_max(now, Ordering::SeqCst);
+ std::thread::sleep(std::time::Duration::from_millis(30));
+ self.asking.fetch_sub(1, Ordering::SeqCst);
+ return None;
+ }
+ Some(effectcraft_raster::yuv::yuv420_over(&r.comp_frame_cpu(comp, t), bg))
+ }
+ fn available(&self) -> bool {
+ self.yuv_frames.load(Ordering::SeqCst) < self.stop_after
+ }
+ fn miss_gate(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicBool>> {
+ self.deferred.then(Default::default)
+ }
+ fn supports_effect(&self, _id: &str) -> bool {
+ false
+ }
+ fn effects(&self, _chain: &[effectcraft_render::FxStep], _buf: &effectcraft_effects::Buf, _levels: Option<f32>) -> Option<effectcraft_effects::Buf> {
+ None
+ }
+ }
+
+ fn out_dir(name: &str) -> PathBuf {
+ let d = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/export").join(name);
+ let _ = std::fs::remove_dir_all(&d);
+ std::fs::create_dir_all(&d).expect("mkdir");
+ d
+ }
+
+ fn project() -> (Project, ItemId) {
+ let mut p = Project::default();
+ p.settings.gpu_acceleration = true;
+ let mut comp = Comp::new(64, 48, FrameRate::new(10, 1), Tick::from_seconds_f64(0.6));
+ comp.background = [0.1, 0.1, 0.4];
+ let sid = p.add_item("Red", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 0.2, 0.0], width: 30, height: 20, pixel_aspect: 1.0 }));
+ let mut l = build::layer(&mut p, &comp, "Red", LayerSource::Solid { item: sid }, (30, 20), None);
+ l.in_point = comp.frame_rate.tick_of(2);
+ comp.layers.push(l);
+ let cid = p.add_item("Comp 1", Label::Sandstone, None, ItemKind::Comp(comp.into()));
+ (p, cid)
+ }
+
+ fn run(p: &Project, cid: ItemId, accel: Option<&dyn Accelerator>, s: &RenderSettings, path: &Path) -> Vec<u8> {
+ let om = OutputModule { bitrate_kbps: 4000, ..OutputModule::for_format(OutputFormat::H264) };
+ let path = path.to_string_lossy().to_string();
+ let job = Job {
+ project: p,
+ footage: &NoFootage,
+ expr: None,
+ accel,
+ comp: cid,
+ settings: s,
+ output: &om,
+ path: &path,
+ sink: None,
+ options: Default::default(),
+ nested_switches: true,
+ };
+ export(&job, &mut |_| true).expect("export");
+ std::fs::read(&path).expect("read")
+ }
+
+ #[test]
+ fn accelerator_planes_encode_like_the_cpu_conversion() {
+ let d = out_dir("h264-gpu-yuv");
+ let (mut p, cid) = project();
+ // Across more than one 16-frame batch so the second batch renders beside the encoder.
+ p.comp_mut(cid).unwrap().duration = Tick::from_seconds_f64(2.0);
+ let s = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() };
+ let yuv = Fake::new(true, false, usize::MAX);
+ let rgba = Fake::new(false, false, usize::MAX);
+ let yuv_mp4 = d.join("yuv.mp4");
+ let a = run(&p, cid, Some(&yuv), &s, &yuv_mp4);
+ let b = run(&p, cid, Some(&rgba), &s, &d.join("rgba.mp4"));
+ let c = run(&p, cid, None, &s, &d.join("cpu.mp4"));
+ assert_eq!(yuv.yuv_frames.load(Ordering::SeqCst), 20, "every frame came from the accelerator");
+ assert!(a == b && b == c, "the three files differ");
+ let f = effectcraft_media::probe(&yuv_mp4).expect("probe yuv.mp4");
+ assert_eq!((f.width, f.height), (64, 48));
+ assert_eq!(f.frame_rate.frame_at(f.duration - Tick(1)) + 1, 20);
+ let pool = effectcraft_media::MediaPool::new();
+ for k in 0..20 {
+ let img = pool.frame_at(&f, f.frame_rate.tick_of(k)).expect("decode frame");
+ assert_eq!((img.width, img.height), (64, 48));
+ }
+ }
+
+ #[test]
+ fn frames_the_accelerator_cannot_take_fall_back_to_rgba() {
+ let d = out_dir("h264-gpu-yuv-fallback");
+ let (p, cid) = project();
+ let base = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() };
+ // Renders at 62×47, encodes at 62×46 (the codec size): planes of the render would not fit.
+ let odd = RenderSettings { resolution: 31.0 / 32.0, ..base.clone() };
+ let yuv = Fake::new(true, false, usize::MAX);
+ let a = run(&p, cid, Some(&yuv), &odd, &d.join("yuv.mp4"));
+ let c = run(&p, cid, None, &odd, &d.join("cpu.mp4"));
+ assert_eq!(yuv.yuv_frames.load(Ordering::SeqCst), 0);
+ assert!(a == c);
+ }
+
++/// Above 8 bpc the colour is dithered on its way to 8 bits, which the accelerator's planes
++/// don't do: those frames take the RGBA path, so an H.264 export doesn't band (#2022).
++#[test]
++fn renders_above_8_bpc_skip_the_planes() {
++ let d = out_dir("h264-gpu-yuv-16bpc");
++ let (p, cid) = project();
++ let s = RenderSettings { time_span: TimeSpan::LengthOfComp, color_depth: ColorDepth::Bpc16, ..Default::default() };
++ let yuv = Fake::new(true, false, usize::MAX);
++ let a = run(&p, cid, Some(&yuv), &s, &d.join("yuv.mp4"));
++ let c = run(&p, cid, None, &s, &d.join("cpu.mp4"));
++ assert_eq!(yuv.yuv_frames.load(Ordering::SeqCst), 0, "no planes asked for above 8 bpc");
++ assert!(a == c);
++}
++
+ /// With deferred readbacks the export keeps rendering frames in passes (`Cx::frames`), never
+ /// through the blocking per-frame path the planes use.
+ #[test]
+ fn deferred_accelerators_keep_the_pass_based_path() {
+ let d = out_dir("h264-gpu-yuv-deferred");
+ let (p, cid) = project();
+ let s = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() };
+ let deferred = Fake::new(true, true, usize::MAX);
+ let a = run(&p, cid, Some(&deferred), &s, &d.join("deferred.mp4"));
+ let c = run(&p, cid, None, &s, &d.join("cpu.mp4"));
+ assert_eq!(deferred.yuv_frames.load(Ordering::SeqCst), 0, "no planes asked for");
+ assert!(a == c);
+ }
+
+ /// An accelerator that stops for good mid-export (`available` false): the frames left render
+ /// on the CPU side by side, not one at a time under the GPU's frames-in-flight cap (as
+ /// `Cx::frames` does), and the file is still the CPU's.
+ #[test]
+ fn frames_after_the_accelerator_stops_render_on_the_cpu() {
+ let d = out_dir("h264-gpu-yuv-stops");
+ let (p, cid) = project();
+ let s = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() };
+ let stops = Fake::new(true, false, 1);
+ // Enough threads for frames to overlap wherever the machine runs it.
+ let pool = rayon::ThreadPoolBuilder::new().num_threads(4).build().unwrap();
+ let a = pool.install(|| run(&p, cid, Some(&stops), &s, &d.join("stops.mp4")));
+ let c = run(&p, cid, None, &s, &d.join("cpu.mp4"));
+ assert_eq!(stops.yuv_frames.load(Ordering::SeqCst), 1, "planes only until the stop");
+ assert!(stops.most_after_stop.load(Ordering::SeqCst) > 1, "frames after the stop rendered one at a time");
+ assert!(a == c);
+ }
diff --cc crates/export/tests/it/main.rs
index 489b07201,9b37d1809..266afc396
--- a/crates/export/tests/it/main.rs
+++ b/crates/export/tests/it/main.rs
@@@ -1,7 -1,7 +1,8 @@@
//! The crate's integration tests, one binary (each module is a former `tests/<name>.rs`).
mod audio_sample_positions;
+mod banding;
+ mod h264_gpu_yuv;
mod hevc_av1_roundtrip;
#[cfg(not(target_arch = "wasm32"))]
mod mov_audio_sample_positions;