Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

Filter layers: a look over everything beneath it, for as long as it runs

Jareercff7c59fc6ef
diff --git a/engine/crates/concat-export/src/flatten.rs b/engine/crates/concat-export/src/flatten.rs
index 3b472e84..23415c9b 100644
--- a/engine/crates/concat-export/src/flatten.rs
+++ b/engine/crates/concat-export/src/flatten.rs
@@ -34,22 +34,55 @@ pub fn flatten_timeline(project: &Project, timeline_id: Option<&str>) -> Vec<Exp
if clip.kind == ModelClipKind::Text {
return None;
}
- let media = project.media.iter().find(|item| item.id == clip.media_id)?;
let index = timeline
.tracks
.iter()
.position(|track| track.id == clip.track_id)?;
let track = &timeline.tracks[index];
+ // A layer: no file, a chain over the stack beneath its track,
+ // its opacity the strength and its fades the ramps.
+ if clip.kind == ModelClipKind::Layer {
+ return Some(ExportClip {
+ path: String::new(),
+ kind: ClipKind::Layer,
+ start: clip.start,
+ duration: clip.duration,
+ source_start: 0.0,
+ track: index,
+ hidden: !track.visible,
+ muted: true,
+ volume: 0.0,
+ fade_in: clip.fade_in,
+ fade_out: clip.fade_out,
+ filter_chain: String::new(),
+ speed: 1.0,
+ preserve_pitch: true,
+ scale: 1.0,
+ offset_x: 0.0,
+ offset_y: 0.0,
+ rotation: 0.0,
+ opacity: clip.opacity,
+ video_filter_chain: video_effect_chain(&clip.video_effects),
+ transition: None,
+ video_fade_in: 0.0,
+ media_width: None,
+ media_height: None,
+ has_audio: Some(false),
+ });
+ }
+
+ let media = project.media.iter().find(|item| item.id == clip.media_id)?;
+
Some(ExportClip {
path: media.path.clone(),
kind: match clip.kind {
ModelClipKind::Video => ClipKind::Video,
ModelClipKind::Audio => ClipKind::Audio,
ModelClipKind::Image => ClipKind::Image,
- // Filtered above; unreachable spelled as a skip so a new
+ // Handled above; unreachable spelled as a skip so a new
// kind fails soft.
- ModelClipKind::Text => return None,
+ ModelClipKind::Text | ModelClipKind::Layer => return None,
},
start: clip.start,
duration: clip.duration,
diff --git a/engine/crates/concat-export/src/lib.rs b/engine/crates/concat-export/src/lib.rs
index 8dabfe18..59554f3e 100644
--- a/engine/crates/concat-export/src/lib.rs
+++ b/engine/crates/concat-export/src/lib.rs
@@ -46,6 +46,10 @@ pub enum ClipKind {
Audio,
/// A still: a one-frame stream, decoded looping.
Image,
+ /// A treatment with no pixels of its own: its chain runs over everything
+ /// composited beneath its track for its span, blended back by its
+ /// opacity, ramped by its fades. See `composite_treated`.
+ Layer,
}
impl ClipKind {
@@ -351,7 +355,7 @@ pub fn render(request: &ExportRequest, mut reporter: Reporter<'_>) -> Result<Str
// decoded, which is handled where the decoder is opened.
let visible: Vec<&ExportClip> = resolved
.iter()
- .filter(|clip| clip.kind.is_visual() && !clip.hidden)
+ .filter(|clip| (clip.kind.is_visual() || clip.kind == ClipKind::Layer) && !clip.hidden)
.collect();
let audible: Vec<&ExportClip> = resolved
.iter()
@@ -503,6 +507,8 @@ fn render_picture(
stills,
decode_sizes,
filter_chains,
+ tracks,
+ treatments,
} = build_timeline(request, rate, visible);
let mut encoder = Encoder::create(
@@ -533,7 +539,8 @@ fn render_picture(
let time = rate.time_of_frame(index);
let plan = plan_frame(&timeline, time);
- let mut sources: Vec<(Frame, f32, Transform)> = Vec::with_capacity(plan.layers.len());
+ let mut sources: Vec<(Frame, f32, Transform, usize)> =
+ Vec::with_capacity(plan.layers.len());
for layer in &plan.layers {
let decoder = match decoders.entry(layer.clip) {
std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
@@ -585,18 +592,29 @@ fn render_picture(
// aborting the export - a clip trimmed past its media's end is a
// mistake in the edit, not a failure of the renderer.
if let Some(frame) = decoder.next_frame().map_err(|error| error.to_string())? {
- sources.push((frame, layer.opacity, layer.transform));
+ let track = tracks.get(&layer.clip).copied().unwrap_or(0);
+ sources.push((frame, layer.opacity, layer.transform, track));
}
}
- let layers: Vec<Layer<'_>> = sources
+ let layers: Vec<(Layer<'_>, usize)> = sources
.iter()
- .map(|(frame, opacity, transform)| {
- place_layer(frame, *opacity, transform, request.width, request.height)
+ .map(|(frame, opacity, transform, track)| {
+ (
+ place_layer(frame, *opacity, transform, request.width, request.height),
+ *track,
+ )
})
.collect();
- let composed = compositor.composite(request.width, request.height, &layers);
+ let composed = composite_treated(
+ &mut *compositor,
+ request.width,
+ request.height,
+ time,
+ &layers,
+ &treatments,
+ );
encoder
.write_frame(&composed)
.map_err(|error| error.to_string())?;
@@ -624,6 +642,109 @@ struct BuiltTimeline {
decode_sizes: HashMap<ClipId, (u32, u32)>,
/// The clip's effect chain, where it has one.
filter_chains: HashMap<ClipId, String>,
+ /// Each picture's track, so a treatment knows what lies beneath it.
+ tracks: HashMap<ClipId, usize>,
+ /// The layers: treatments over the stack, by span.
+ treatments: Vec<Treatment>,
+}
+
+/// A layer clip, as the compositor needs it: when, over which tracks, what
+/// chain, and how hard.
+#[derive(Clone, Debug)]
+struct Treatment {
+ start: Rational,
+ end: Rational,
+ track: usize,
+ chain: String,
+ strength: f32,
+ ramp_in: f64,
+ ramp_out: f64,
+}
+
+impl Treatment {
+ fn covers(&self, time: Rational) -> bool {
+ self.start <= time && time < self.end
+ }
+
+ /// How hard the treatment is applied at `time`: the strength, eased in
+ /// and out over the ramps at either end.
+ fn strength_at(&self, time: Rational) -> f32 {
+ let at = time.as_f64() - self.start.as_f64();
+ let left = self.end.as_f64() - time.as_f64();
+ let mut ramp = 1.0_f64;
+ if self.ramp_in > 0.0 && at < self.ramp_in {
+ ramp = ramp.min(at / self.ramp_in);
+ }
+ if self.ramp_out > 0.0 && left < self.ramp_out {
+ ramp = ramp.min(left / self.ramp_out);
+ }
+ (f64::from(self.strength) * ramp.clamp(0.0, 1.0)) as f32
+ }
+}
+
+/// `a` towards `b` by `amount`, per channel.
+fn mix(a: &Frame, b: &Frame, amount: f32) -> Frame {
+ let amount = amount.clamp(0.0, 1.0);
+ let mut out = a.clone();
+ for (pixel, over) in out.pixels_mut().iter_mut().zip(b.pixels().iter()) {
+ let base = f32::from(*pixel);
+ *pixel = (base + (f32::from(*over) - base) * amount).round() as u8;
+ }
+ out
+}
+
+/// Composites `sources` - each placed layer with the track it came from,
+/// bottom-most first - with every treatment live at `time` applied to the
+/// stack beneath its track. Without a live treatment this is one composite;
+/// with one, the stack is drawn up to the treatment's track, run through its
+/// chain, blended back by its strength, and used as the ground for the rest.
+fn composite_treated(
+ compositor: &mut dyn Compositor,
+ width: u32,
+ height: u32,
+ time: Rational,
+ sources: &[(Layer<'_>, usize)],
+ treatments: &[Treatment],
+) -> Frame {
+ let mut live: Vec<&Treatment> = treatments
+ .iter()
+ .filter(|treatment| treatment.covers(time))
+ .collect();
+ if live.is_empty() {
+ let layers: Vec<Layer<'_>> = sources.iter().map(|(layer, _)| *layer).collect();
+ return compositor.composite(width, height, &layers);
+ }
+ live.sort_by_key(|treatment| treatment.track);
+
+ let mut ground: Option<Frame> = None;
+ let mut next = 0;
+ for treatment in live {
+ let below = {
+ let mut layers: Vec<Layer<'_>> = ground.as_ref().map(Layer::new).into_iter().collect();
+ while next < sources.len() && sources[next].1 < treatment.track {
+ layers.push(sources[next].0);
+ next += 1;
+ }
+ compositor.composite(width, height, &layers)
+ };
+ let strength = treatment.strength_at(time);
+ let treated = if strength <= 0.0 {
+ below
+ } else {
+ match concat_media::treat(&below, &treatment.chain) {
+ Ok(treated) if strength >= 1.0 => treated,
+ Ok(treated) => mix(&below, &treated, strength),
+ // A chain FFmpeg refuses leaves the picture as it was
+ // rather than blanking it; the export says nothing because
+ // the catalogue validated every template at load.
+ Err(_) => below,
+ }
+ };
+ ground = Some(treated);
+ }
+ let mut layers: Vec<Layer<'_>> = ground.as_ref().map(Layer::new).into_iter().collect();
+ layers.extend(sources[next..].iter().map(|(layer, _)| *layer));
+ compositor.composite(width, height, &layers)
}
/// Converts the flattened clip list into an engine timeline.
@@ -636,6 +757,8 @@ fn build_timeline(
let mut stills = std::collections::HashSet::new();
let mut decode_sizes: HashMap<ClipId, (u32, u32)> = HashMap::new();
let mut filter_chains: HashMap<ClipId, String> = HashMap::new();
+ let mut tracks_of: HashMap<ClipId, usize> = HashMap::new();
+ let mut treatments: Vec<Treatment> = Vec::new();
let lanes = visible.iter().map(|clip| clip.track).max().unwrap_or(0) + 1;
let tracks: Vec<_> = (0..lanes)
@@ -652,6 +775,23 @@ fn build_timeline(
continue;
}
+ // A layer has no pixels to decode: it is a treatment over the
+ // stack, kept beside the timeline rather than in it.
+ if clip.kind == ClipKind::Layer {
+ if !clip.video_filter_chain.is_empty() {
+ treatments.push(Treatment {
+ start,
+ end: start + duration,
+ track: clip.track,
+ chain: clip.video_filter_chain.clone(),
+ strength: clip.opacity.clamp(0.0, 1.0) as f32,
+ ramp_in: clip.fade_in.max(0.0),
+ ramp_out: clip.fade_out.max(0.0),
+ });
+ }
+ continue;
+ }
+
let mut engine_clip = Clip::new(MediaRef::new(&clip.path), start, duration);
engine_clip.source_start = quantise(clip.source_start, rate);
// The same clamp the audio path applies, so a 2x clip means the same
@@ -672,6 +812,7 @@ fn build_timeline(
engine_clip.video_fade_in = quantise(clip.video_fade_in, rate);
if let Some(id) = timeline.add_clip(tracks[clip.track], engine_clip) {
+ tracks_of.insert(id, clip.track);
if clip.kind == ClipKind::Image {
stills.insert(id);
}
@@ -689,6 +830,8 @@ fn build_timeline(
stills,
decode_sizes,
filter_chains,
+ tracks: tracks_of,
+ treatments,
}
}
@@ -743,34 +886,66 @@ pub fn preview_frame(
request: &PreviewFrameRequest,
) -> Result<Vec<u8>, String> {
let sources = preview_sources(pool, request)?;
- let layers = sources.layers();
// CPU on purpose: one frame at preview size is milliseconds, and holding
// a GPU context alive for occasional scrubs is not worth its memory. The
// window composites on its own device through `preview_sources`.
- let composed = CpuCompositor.composite(request.width, request.height, &layers);
- Ok(composed.into_pixels())
+ Ok(sources.composite(&mut CpuCompositor).into_pixels())
}
/// The decoded pictures under the playhead and where each goes: what a
/// compositor needs to draw the paused monitor's frame, decoded but not yet
/// drawn, so a caller with a GPU can draw them where they are shown.
pub struct PreviewSources {
- sources: Vec<(std::sync::Arc<Frame>, f32, Transform)>,
+ sources: Vec<(std::sync::Arc<Frame>, f32, Transform, usize)>,
width: u32,
height: u32,
+ time: Rational,
+ treatments: Vec<Treatment>,
}
impl PreviewSources {
- /// The layers, placed the exporter's way, bottom-most first.
+ /// The layers, placed the exporter's way, bottom-most first. Bare: a
+ /// caller drawing these itself is skipping the treatments, so check
+ /// [`PreviewSources::has_treatments`] first.
pub fn layers(&self) -> Vec<Layer<'_>> {
self.sources
.iter()
- .map(|(frame, opacity, transform)| {
+ .map(|(frame, opacity, transform, _)| {
place_layer(frame.as_ref(), *opacity, transform, self.width, self.height)
})
.collect()
}
+ /// Whether a layer clip is live at this instant, in which case the frame
+ /// cannot be drawn from `layers` alone: see [`PreviewSources::composite`].
+ pub fn has_treatments(&self) -> bool {
+ self.treatments
+ .iter()
+ .any(|treatment| treatment.covers(self.time))
+ }
+
+ /// The frame, treatments included, drawn with `compositor`.
+ pub fn composite(&self, compositor: &mut dyn Compositor) -> Frame {
+ let placed: Vec<(Layer<'_>, usize)> = self
+ .sources
+ .iter()
+ .map(|(frame, opacity, transform, track)| {
+ (
+ place_layer(frame.as_ref(), *opacity, transform, self.width, self.height),
+ *track,
+ )
+ })
+ .collect();
+ composite_treated(
+ compositor,
+ self.width,
+ self.height,
+ self.time,
+ &placed,
+ &self.treatments,
+ )
+ }
+
/// The output width the layers were placed for.
pub fn width(&self) -> u32 {
self.width
@@ -795,10 +970,13 @@ pub fn preview_sources(
stills,
decode_sizes,
filter_chains,
+ tracks,
+ treatments,
} = preview_timeline(request, rate);
- let plan = plan_frame(&timeline, quantise(request.time, rate));
+ let time = quantise(request.time, rate);
+ let plan = plan_frame(&timeline, time);
- let mut sources: Vec<(std::sync::Arc<Frame>, f32, Transform)> =
+ let mut sources: Vec<(std::sync::Arc<Frame>, f32, Transform, usize)> =
Vec::with_capacity(plan.layers.len());
let mut failures: Vec<String> = Vec::new();
for layer in &plan.layers {
@@ -817,7 +995,12 @@ pub fn preview_sources(
stills.contains(&layer.clip),
chain,
) {
- Ok(frame) => sources.push((frame, layer.opacity, layer.transform)),
+ Ok(frame) => sources.push((
+ frame,
+ layer.opacity,
+ layer.transform,
+ tracks.get(&layer.clip).copied().unwrap_or(0),
+ )),
Err(error) => failures.push(format!("{}: {error}", layer.media.display())),
}
}
@@ -837,6 +1020,8 @@ pub fn preview_sources(
sources,
width: request.width,
height: request.height,
+ time,
+ treatments,
})
}
@@ -846,7 +1031,7 @@ fn preview_timeline(request: &PreviewFrameRequest, rate: FrameRate) -> BuiltTime
resolve_transitions(&mut resolved, rate, false);
let visible: Vec<&ExportClip> = resolved
.iter()
- .filter(|clip| clip.kind.is_visual() && !clip.hidden)
+ .filter(|clip| (clip.kind.is_visual() || clip.kind == ClipKind::Layer) && !clip.hidden)
.collect();
// build_timeline reads only the output format off the request; the shim
@@ -886,6 +1071,7 @@ pub fn preview_prefetch(
stills,
decode_sizes,
filter_chains,
+ ..
} = preview_timeline(request, rate);
let fps = rate.fps().as_f64();
@@ -912,6 +1098,79 @@ pub fn preview_prefetch(
#[cfg(test)]
mod tests {
+ /// A treatment on track 1 runs over what track 0 drew and not over what
+ /// track 2 draws on top of it, and its strength blends the result back.
+ #[test]
+ fn a_treatment_covers_the_stack_beneath_its_track_only() {
+ fn solid(width: u32, height: u32, rgba: [u8; 4]) -> Frame {
+ let mut frame = Frame::black(width, height);
+ for pixel in frame.pixels_mut().chunks_exact_mut(4) {
+ pixel.copy_from_slice(&rgba);
+ }
+ frame
+ }
+ // Red fills the frame on track 0; a small blue square sits on
+ // track 2 at the top-left.
+ let red = solid(8, 8, [255, 0, 0, 255]);
+ let blue = solid(2, 2, [0, 0, 255, 255]);
+ let sources = [(Layer::new(&red), 0usize), (Layer::new(&blue), 2usize)];
+ let negate = Treatment {
+ start: Rational::ZERO,
+ end: Rational::from_int(10),
+ track: 1,
+ chain: "negate".to_owned(),
+ strength: 1.0,
+ ramp_in: 0.0,
+ ramp_out: 0.0,
+ };
+ let mut compositor = CpuCompositor;
+ let out = composite_treated(
+ &mut compositor,
+ 8,
+ 8,
+ Rational::from_int(1),
+ &sources,
+ &[negate.clone()],
+ );
+ // Red negated is cyan where nothing sits on top...
+ let at = |x: usize, y: usize| &out.pixels()[(y * 8 + x) * 4..(y * 8 + x) * 4 + 3];
+ assert_eq!(at(7, 7), &[0, 255, 255]);
+ // ...and the blue square above the treatment is untouched.
+ assert_eq!(at(0, 0), &[0, 0, 255]);
+
+ // At half strength the ground is halfway between red and cyan.
+ let half = Treatment {
+ strength: 0.5,
+ ..negate.clone()
+ };
+ let out = composite_treated(
+ &mut compositor,
+ 8,
+ 8,
+ Rational::from_int(1),
+ &sources,
+ &[half],
+ );
+ let pixel = &out.pixels()[(7 * 8 + 7) * 4..(7 * 8 + 7) * 4 + 3];
+ assert!(pixel[0] > 120 && pixel[0] < 136, "{pixel:?}");
+ assert!(pixel[1] > 120 && pixel[1] < 136, "{pixel:?}");
+
+ // Outside its span the treatment does nothing.
+ let out = composite_treated(
+ &mut compositor,
+ 8,
+ 8,
+ Rational::from_int(20),
+ &sources,
+ &[negate],
+ );
+ assert_eq!(at_of(&out, 7, 7), [255, 0, 0]);
+ fn at_of(frame: &Frame, x: usize, y: usize) -> [u8; 3] {
+ let p = &frame.pixels()[(y * 8 + x) * 4..(y * 8 + x) * 4 + 3];
+ [p[0], p[1], p[2]]
+ }
+ }
+
use super::*;
fn clip(kind: &str, track: usize, start: f64, duration: f64, source_start: f64) -> ExportClip {
diff --git a/engine/crates/concat-host/src/preview.rs b/engine/crates/concat-host/src/preview.rs
index e0b6a07f..98817579 100644
--- a/engine/crates/concat-host/src/preview.rs
+++ b/engine/crates/concat-host/src/preview.rs
@@ -103,8 +103,17 @@ impl Monitor {
.ok_or_else(|| "the monitor has no GPU device".to_owned())?;
let request = Self::request(clips, settings, spec);
let sources = concat_export::preview_sources(&self.pool, &request)?;
- let layers = sources.layers();
let mut gpu = gpu.lock().map_err(|_| "compositor poisoned".to_owned())?;
+ if sources.has_treatments() {
+ // A layer's chain runs on the CPU, so the frame is drawn there
+ // and uploaded whole; only instants under a layer pay for it.
+ let frame = sources.composite(&mut *gpu);
+ let layers = [concat_render::Layer::new(&frame)];
+ return gpu
+ .composite_texture(spec.width, spec.height, &layers)
+ .ok_or_else(|| "the GPU device was lost".to_owned());
+ }
+ let layers = sources.layers();
gpu.composite_texture(spec.width, spec.height, &layers)
.ok_or_else(|| "the GPU device was lost".to_owned())
}
diff --git a/engine/crates/concat-media/src/lib.rs b/engine/crates/concat-media/src/lib.rs
index ec4ecbc1..0acb73b1 100644
--- a/engine/crates/concat-media/src/lib.rs
+++ b/engine/crates/concat-media/src/lib.rs
@@ -22,6 +22,7 @@ pub mod peaks;
pub mod pool;
pub mod probe;
pub mod samples;
+pub mod treat;
pub use decode::{DecodeOptions, Decoder, FrameSource, SeekableSource};
pub use encode::{EncodeOptions, Encoder, FrameSink, jpeg};
@@ -31,3 +32,4 @@ pub use peaks::Peaks;
pub use pool::{FrameCache, ReaderPool};
pub use probe::{AudioStream, MediaInfo, VideoStream, probe};
pub use samples::{AudioDecoder, AudioOptions, SampleFormat};
+pub use treat::treat;
diff --git a/engine/crates/concat-media/src/treat.rs b/engine/crates/concat-media/src/treat.rs
new file mode 100644
index 00000000..4e31f3fe
--- /dev/null
+++ b/engine/crates/concat-media/src/treat.rs
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors
+
+//! One picture through a filter chain, in memory.
+//!
+//! The decoder runs every clip's own chain as the pixels come out of the
+//! codec. A *layer*, meaning a look or an effect placed over a span of the
+//! timeline, has no pixels of its own: it treats whatever has been
+//! composited beneath it. That picture exists only in memory, so this is
+//! the decoder's filtergraph without the decoder: an RGBA buffer in, the
+//! chain, an RGBA buffer out. A guard scale pins the size, as the decoder's
+//! does, so a chain that resizes cannot change the frame under the
+//! compositor.
+
+use std::path::Path;
+
+use concat_core::frame::Frame;
+use ffmpeg_the_third as ffmpeg;
+use ffmpeg_the_third::filter;
+use ffmpeg_the_third::format::Pixel;
+use ffmpeg_the_third::util::frame::video::Video;
+
+use crate::error::{Error, Result};
+use crate::ffi;
+
+/// Runs `frame` through `chain` and returns the result at the same size.
+/// An empty chain returns a copy.
+pub fn treat(frame: &Frame, chain: &str) -> Result<Frame> {
+ if chain.trim().is_empty() {
+ return Ok(frame.clone());
+ }
+ ffi::init();
+ // The graph has no file behind it; errors name the layer instead.
+ let path = Path::new("layer");
+ let (width, height) = (frame.width(), frame.height());
+
+ let mut graph = filter::Graph::new();
+ let args = format!(
+ "video_size={width}x{height}:pix_fmt={}:time_base=1/1000:pixel_aspect=1/1",
+ Into::<ffmpeg::sys::AVPixelFormat>::into(Pixel::RGBA).0
+ );
+ let missing = |name: &str| Error::Missing {
+ what: "filter",
+ name: name.to_owned(),
+ };
+ graph
+ .add(
+ &filter::find("buffer").ok_or_else(|| missing("buffer"))?,
+ "in",
+ &args,
+ )
+ .map_err(|error| ffi::fail("buffer source", path, error))?;
+ graph
+ .add(
+ &filter::find("buffersink").ok_or_else(|| missing("buffersink"))?,
+ "out",
+ "",
+ )
+ .map_err(|error| ffi::fail("buffer sink", path, error))?;
+ let spec = format!("{chain},scale={width}:{height}:flags=bilinear,format=rgba");
+ graph
+ .output("in", 0)
+ .and_then(|parser| parser.input("out", 0))
+ .and_then(|parser| parser.parse(&spec))
+ .map_err(|error| ffi::fail("filter graph", path, error))?;
+ graph
+ .validate()
+ .map_err(|error| ffi::fail("filter graph", path, error))?;
+
+ // The picture, as a padded FFmpeg frame.
+ let mut source = Video::new(Pixel::RGBA, width, height);
+ {
+ let row = width as usize * 4;
+ let stride = source.stride(0);
+ let data = source.data_mut(0);
+ for (y, line) in frame.pixels().chunks_exact(row).enumerate() {
+ data[y * stride..y * stride + row].copy_from_slice(line);
+ }
+ }
+ source.set_pts(Some(0));
+ {
+ let mut context = graph.get("in").expect("the graph has an input");
+ context
+ .source()
+ .add(&source)
+ .map_err(|error| ffi::fail("filter", path, error))?;
+ }
+ let mut filtered = Video::empty();
+ {
+ let mut context = graph.get("out").expect("the graph has an output");
+ context
+ .sink()
+ .frame(&mut filtered)
+ .map_err(|error| ffi::fail("filter output", path, error))?;
+ }
+
+ // Back to a packed buffer; only the row padding differs.
+ let out_width = filtered.width();
+ let out_height = filtered.height();
+ let row = out_width as usize * 4;
+ let stride = filtered.stride(0);
+ let data = filtered.data(0);
+ let mut pixels = Vec::with_capacity(row * out_height as usize);
+ for y in 0..out_height as usize {
+ pixels.extend_from_slice(&data[y * stride..y * stride + row]);
+ }
+ Frame::from_rgba(out_width, out_height, pixels).ok_or_else(|| Error::Probe {
+ path: path.to_path_buf(),
+ detail: "the filtergraph produced a frame of the wrong size".to_owned(),
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// A solid red picture through `negate` comes back cyan, the same size.
+ #[test]
+ fn a_chain_changes_the_pixels_and_keeps_the_size() {
+ let mut frame = Frame::black(8, 4);
+ for pixel in frame.pixels_mut().chunks_exact_mut(4) {
+ pixel.copy_from_slice(&[255, 0, 0, 255]);
+ }
+ let out = treat(&frame, "negate").expect("negate is a filter FFmpeg has");
+ assert_eq!((out.width(), out.height()), (8, 4));
+ assert_eq!(&out.pixels()[..4], &[0, 255, 255, 255]);
+ }
+
+ /// An empty chain is a copy, and a chain that resizes is pinned back.
+ #[test]
+ fn empty_is_a_copy_and_the_size_is_guarded() {
+ let frame = Frame::black(6, 6);
+ let same = treat(&frame, "").expect("copies");
+ assert_eq!(same.pixels(), frame.pixels());
+ let pinned = treat(&frame, "scale=3:3").expect("scales then pins");
+ assert_eq!((pinned.width(), pinned.height()), (6, 6));
+ }
+}
diff --git a/engine/crates/concat-project/src/commands.rs b/engine/crates/concat-project/src/commands.rs
index 90a389f9..a2d3f8a3 100644
--- a/engine/crates/concat-project/src/commands.rs
+++ b/engine/crates/concat-project/src/commands.rs
@@ -22,6 +22,8 @@ const UNKNOWN_DURATION: f64 = 5.0;
const DEFAULT_IMAGE_DURATION: f64 = 5.0;
/// How long a title lasts when first placed.
const DEFAULT_TEXT_DURATION: f64 = 4.0;
+/// How long a layer covers when first placed. Editorial default, not a fact.
+const DEFAULT_LAYER_DURATION: f64 = 5.0;
const MIN_CLIP_DURATION: f64 = 1.0 / 60.0;
/// The engine's speed range (concat-media `SPEED_RANGE`), verbatim.
const MIN_SPEED: f64 = 0.0625;
@@ -247,6 +249,23 @@ pub enum Command {
#[serde(default)]
offset_y: Option<f64>,
},
+ /// Places a layer: a look or an effect over a span of the timeline that
+ /// treats everything beneath it. No media; the chain starts as the one
+ /// package named, at its defaults, and the clip's opacity is how hard it
+ /// is applied.
+ AddLayerClip {
+ /// The lane to place it on; None picks the first free track.
+ track_id: Option<String>,
+ /// Timeline position in seconds, floored at 0.
+ start: f64,
+ /// Seconds on the timeline; the editorial default when absent.
+ #[serde(default)]
+ duration: Option<f64>,
+ /// The package the layer applies, e.g. "concat.warm".
+ effect_id: String,
+ /// What the lane calls it.
+ name: String,
+ },
/// Repositions any number of clips in one edit - one undo step for a
/// whole multi-selection drag. Unknown clips and tracks are tolerated
/// per [`ClipMove`].
@@ -885,6 +904,65 @@ pub fn apply(
})
}
+ Command::AddLayerClip {
+ track_id,
+ start,
+ duration,
+ effect_id,
+ name,
+ } => {
+ let duration = duration
+ .unwrap_or(DEFAULT_LAYER_DURATION)
+ .max(MIN_CLIP_DURATION);
+ let timeline = project.active_mut();
+ let track_id = match track_id {
+ Some(id) if timeline.track(&id).is_some() => id,
+ Some(_) => return Err(CommandError::TrackGone),
+ None => {
+ first_free_track(timeline, start, duration).ok_or(CommandError::NoTracks)?
+ }
+ };
+ let id = mint.next("c");
+ timeline.clips.push(Clip {
+ id: id.clone(),
+ track_id,
+ media_id: String::new(),
+ name: if name.trim().is_empty() {
+ effect_id.clone()
+ } else {
+ name
+ },
+ kind: ClipKind::Layer,
+ start: start.max(0.0),
+ duration,
+ source_start: 0.0,
+ volume: 1.0,
+ fade_in: 0.0,
+ fade_out: 0.0,
+ scale: 1.0,
+ offset_x: 0.0,
+ offset_y: 0.0,
+ rotation: 0.0,
+ opacity: 1.0,
+ speed: 1.0,
+ preserve_pitch: true,
+ filters: Vec::new(),
+ video_effects: vec![AppliedFilter {
+ id: effect_id,
+ params: Default::default(),
+ enabled: true,
+ }],
+ muted: None,
+ detached_from: None,
+ transition_in: None,
+ text: None,
+ });
+ Ok(Outcome {
+ created_id: Some(id),
+ applied: true,
+ })
+ }
+
Command::MoveClips { moves } => {
let timeline = project.active_mut();
let track_ids: HashSet<String> = timeline
diff --git a/engine/crates/concat-project/src/doc.rs b/engine/crates/concat-project/src/doc.rs
index 6cfde28f..7d92caba 100644
--- a/engine/crates/concat-project/src/doc.rs
+++ b/engine/crates/concat-project/src/doc.rs
@@ -173,8 +173,10 @@ fn read_clips(raw: Option<&Value>, tracks: &[Track], media: &[MediaItem]) -> Vec
return None;
}
- let is_text = entry.get("kind").and_then(Value::as_str) == Some("text");
- let media_id = if is_text {
+ let kind_name = entry.get("kind").and_then(Value::as_str);
+ let is_text = kind_name == Some("text");
+ let is_layer = kind_name == Some("layer");
+ let media_id = if is_text || is_layer {
String::new()
} else {
let media_id = entry.get("mediaId")?.as_str()?.to_owned();
@@ -188,8 +190,10 @@ fn read_clips(raw: Option<&Value>, tracks: &[Track], media: &[MediaItem]) -> Vec
let kind = if is_text {
ClipKind::Text
+ } else if is_layer {
+ ClipKind::Layer
} else {
- match entry.get("kind").and_then(Value::as_str) {
+ match kind_name {
Some("audio") => ClipKind::Audio,
Some("image") => ClipKind::Image,
_ => ClipKind::Video,
diff --git a/engine/crates/concat-project/src/model.rs b/engine/crates/concat-project/src/model.rs
index cfbfee50..159ab841 100644
--- a/engine/crates/concat-project/src/model.rs
+++ b/engine/crates/concat-project/src/model.rs
@@ -41,6 +41,11 @@ pub enum ClipKind {
Image,
/// A title. No media behind it; the content lives in [`Clip::text`].
Text,
+ /// A treatment over everything beneath it for as long as it runs - a
+ /// look or an effect placed as a layer. No media behind it; the chain
+ /// lives in [`Clip::video_effects`], its strength in [`Clip::opacity`]
+ /// and its ramps in the fades.
+ Layer,
}
impl ClipKind {
diff --git a/engine/crates/concat/src/studio.rs b/engine/crates/concat/src/studio.rs
index b6544f48..1a288e49 100644
--- a/engine/crates/concat/src/studio.rs
+++ b/engine/crates/concat/src/studio.rs
@@ -556,6 +556,7 @@ fn kind_of(clip: &Clip) -> ClipKind {
model::ClipKind::Audio => ClipKind::Audio,
model::ClipKind::Image => ClipKind::Image,
model::ClipKind::Text => ClipKind::Text,
+ model::ClipKind::Layer => ClipKind::Filter,
}
}
@@ -916,6 +917,7 @@ impl Studio {
.map(|clip| match clip.kind {
model::ClipKind::Video | model::ClipKind::Image => LANE_LARGE,
model::ClipKind::Audio | model::ClipKind::Text => LANE_MEDIUM,
+ model::ClipKind::Layer => LANE_SMALL,
})
.fold(0.0_f32, f32::max);
if tallest > 0.0 { tallest } else { LANE_MEDIUM }
@@ -1559,6 +1561,16 @@ impl Studio {
duration: LAYER_DURATION,
row: 0,
}),
+ // A look dragged from the Filters page: a layer over a span.
+ // The package id rides in `media`, there being no file.
+ "filter" => Some(DropPlan {
+ kind: ClipKind::Filter,
+ label: label.to_owned(),
+ media: id.to_owned(),
+ start: 0.0,
+ duration: LAYER_DURATION,
+ row: 0,
+ }),
_ => None,
}
}
@@ -1596,6 +1608,14 @@ impl Studio {
duration: Some(f64::from(plan.duration)),
offset_y: None,
})
+ } else if plan.kind == ClipKind::Filter {
+ self.apply(Command::AddLayerClip {
+ track_id: Some(track_id),
+ start: f64::from(plan.start),
+ duration: Some(f64::from(plan.duration)),
+ effect_id: plan.media.clone(),
+ name: plan.label.clone(),
+ })
} else {
self.apply(Command::AddClip {
media_id: plan.media.clone(),
@@ -1623,6 +1643,14 @@ impl Studio {
duration: Some(f64::from(plan.duration)),
offset_y: None,
})
+ } else if plan.kind == ClipKind::Filter {
+ self.apply(Command::AddLayerClip {
+ track_id: None,
+ start,
+ duration: Some(f64::from(plan.duration)),
+ effect_id: plan.media.clone(),
+ name: plan.label.clone(),
+ })
} else {
self.apply(Command::AddClipAtFirstFree {
media_id: plan.media.clone(),
@@ -2346,7 +2374,7 @@ impl Studio {
.clips
.iter()
.filter(|clip| {
- clip.kind != model::ClipKind::Audio
+ (clip.kind.is_visual() || clip.kind == model::ClipKind::Text)
&& showing.contains(clip.track_id.as_str())
&& clip.start <= playhead
&& playhead < clip.start + clip.duration
@@ -3516,7 +3544,7 @@ impl Studio {
.clips
.iter()
.filter(|clip| {
- clip.kind != model::ClipKind::Audio
+ (clip.kind.is_visual() || clip.kind == model::ClipKind::Text)
&& clip.start <= playhead
&& playhead < clip.start + clip.duration
})