Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(effects): a pass carries its package and the values it was written from

Jareer203ef2c920f5
diff --git a/src/crates/concat-core/src/shader.rs b/src/crates/concat-core/src/shader.rs
index c5c67afe..1cd08c11 100644
--- a/src/crates/concat-core/src/shader.rs
+++ b/src/crates/concat-core/src/shader.rs
@@ -12,14 +12,25 @@
//! uniform buffer as it is, because the catalogue already laid the bytes
//! out the way the shader's `Params` struct wants them.
//!
+//! The parameters are uniforms, never strings: a knob with keys is worth
+//! something different each frame, and the catalogue resolves it to the
+//! value for the frame and writes the buffer. The same resolved values
+//! ride along by name in `values`, for a renderer that runs the package
+//! through a kernel of its own rather than the shader - the CPU reference
+//! - so both read one resolution of the clip's settings.
+//!
//! It lives here, in the crate every other one can see, so the catalogue
//! that builds it and the compositor that runs it need not know each other.
+use std::collections::BTreeMap;
use std::sync::Arc;
/// One fragment pass over a layer.
#[derive(Clone, Debug, PartialEq)]
pub struct ShaderPass {
+ /// The package's id, `author.name`: what a renderer without a shader
+ /// stage keys its own kernel for the package by.
+ pub package: String,
/// What to cache the compiled pipeline under: the package's id and
/// version, so a package that changes its shader gets a new pipeline
/// and one that only changes its knobs keeps the old.
@@ -30,6 +41,9 @@ pub struct ShaderPass {
/// The `Params` uniform, laid out to the struct's offsets. Sixteen bytes
/// at least, so an empty struct still has a buffer.
pub params: Vec<u8>,
+ /// The same values by the manifest's keys, every declared parameter
+ /// present, resolved for the frame: what `params` was written from.
+ pub values: BTreeMap<String, f64>,
/// How much of the result to keep over the untouched layer, `0..=1`. A
/// look at half strength is half the look; an effect is always one.
pub intensity: f32,
diff --git a/src/crates/concat-effects/src/shader.rs b/src/crates/concat-effects/src/shader.rs
index 24cfcff5..bda8f6ad 100644
--- a/src/crates/concat-effects/src/shader.rs
+++ b/src/crates/concat-effects/src/shader.rs
@@ -378,6 +378,7 @@ struct Slot {
/// A package's shader, stitched, checked and laid out.
#[derive(Clone, Debug)]
pub struct Shader {
+ package: String,
key: String,
source: Arc<str>,
slots: Vec<Slot>,
@@ -475,6 +476,7 @@ impl Shader {
}
Ok(Shader {
+ package: manifest.effect.id.clone(),
key: format!("{}@{}", manifest.effect.id, manifest.effect.version),
source: Arc::from(source),
slots,
@@ -542,7 +544,9 @@ impl Shader {
bytes
}
- /// A pass over a layer with these values.
+ /// A pass over a layer with these values: the uniform buffer written
+ /// from them, and the values themselves for a renderer that reads by
+ /// name.
pub fn pass(
&self,
values: &BTreeMap<String, f64>,
@@ -551,9 +555,11 @@ impl Shader {
lut: Option<Arc<Lut>>,
) -> ShaderPass {
ShaderPass {
+ package: self.package.clone(),
key: self.key.clone(),
source: Arc::clone(&self.source),
params: self.params_bytes(values, params),
+ values: values.clone(),
intensity,
lut,
}
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index b1b58235..50edf758 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -24,7 +24,7 @@ pub mod chains;
pub mod flatten;
mod resolve;
-use resolve::{BuiltTimeline, RidingChain, Treatment, animation_of, build_timeline, quantise};
+use resolve::{BuiltTimeline, Treatment, animation_of, build_timeline, quantise};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
@@ -958,8 +958,7 @@ fn render_picture(
tracks,
treatments,
pre_chains,
- passes,
- riding,
+ chains,
cutouts,
highlight: _,
} = build_timeline(request, rate, visible, gpu);
@@ -994,7 +993,6 @@ fn render_picture(
reporter.cancelled()?;
let time = rate.time_of_frame(index);
- let frame_seconds = time.as_f64();
let plan = plan_frame(&timeline, time);
let mut sources: Vec<Source<Frame>> = Vec::with_capacity(plan.layers.len());
@@ -1025,7 +1023,7 @@ fn render_picture(
transform: layer.transform,
track: tracks.get(&layer.clip).copied().unwrap_or(0),
blend: layer.blend,
- passes: passes_at(&passes, &riding, layer.clip, frame_seconds),
+ passes: passes_at(&chains, &timeline, layer.clip, time),
});
}
continue;
@@ -1098,7 +1096,7 @@ fn render_picture(
transform: layer.transform,
track: tracks.get(&layer.clip).copied().unwrap_or(0),
blend: layer.blend,
- passes: passes_at(&passes, &riding, layer.clip, frame_seconds),
+ passes: passes_at(&chains, &timeline, layer.clip, time),
});
}
}
@@ -1148,23 +1146,22 @@ fn render_picture(
encoder.finish().map_err(|error| error.to_string())
}
-/// The shader passes for one clip at one frame: built for the instant when
-/// the chain rides, and the ones built once otherwise.
+/// The shader passes for one clip at one instant: every keyed knob at its
+/// value there, the rest at their settings, laid out as the uniforms the
+/// shaders read.
fn passes_at(
- passes: &HashMap<ClipId, Vec<ShaderPass>>,
- riding: &HashMap<ClipId, RidingChain>,
+ chains: &HashMap<ClipId, Vec<AppliedFilter>>,
+ timeline: &Timeline,
clip: ClipId,
- seconds: f64,
+ time: Rational,
) -> Vec<ShaderPass> {
- if let Some(chain) = riding.get(&clip) {
- let at = if chain.duration > 0.0 {
- ((seconds - chain.start) / chain.duration).clamp(0.0, 1.0)
- } else {
- 0.0
- };
- return Catalogue::builtin().shader_passes_at(&chain.effects, at);
- }
- passes.get(&clip).cloned().unwrap_or_default()
+ let Some(effects) = chains.get(&clip) else {
+ return Vec::new();
+ };
+ let at = timeline
+ .clip(clip)
+ .map_or(0.0, |engine_clip| engine_clip.fraction_at(time));
+ Catalogue::builtin().shader_passes_at(effects, at)
}
/// `a` towards `b` by `amount`, per channel.
@@ -1205,12 +1202,17 @@ fn composite_treated(
// pixel leaving the GPU, so long as none of them needs FFmpeg for
// a package that has no shader.
if live.iter().all(|treatment| treatment.chain.is_empty()) {
+ let resolved: Vec<(Vec<ShaderPass>, f32)> = live
+ .iter()
+ .map(|treatment| (treatment.passes_at(time), treatment.strength_at(time)))
+ .collect();
let gpu: Vec<GpuTreatment<'_>> = live
.iter()
- .map(|treatment| GpuTreatment {
+ .zip(&resolved)
+ .map(|(treatment, (passes, strength))| GpuTreatment {
track: treatment.track,
- passes: &treatment.passes,
- strength: treatment.strength_at(time),
+ passes,
+ strength: *strength,
})
.collect();
if let Some(frame) =
@@ -1237,11 +1239,12 @@ fn composite_treated(
} else {
// Shader passes run through the compositor over the ground as a
// layer of its own; whatever is left for FFmpeg runs after.
- let shaded = if treatment.passes.is_empty() {
+ let passes = treatment.passes_at(time);
+ let shaded = if passes.is_empty() {
below.clone()
} else {
let ground = [Layer::new(&below)
- .with_passes(&treatment.passes)
+ .with_passes(&passes)
.at_time(time.as_f64() as f32)];
compositor.composite(width, height, &ground)
};
@@ -1313,6 +1316,8 @@ pub struct PreviewSources {
height: u32,
time: Rational,
treatments: Vec<Treatment>,
+ /// Each treatment's passes resolved at `time`, by the same index.
+ treatment_passes: Vec<Vec<ShaderPass>>,
}
impl PreviewSources {
@@ -1375,20 +1380,24 @@ impl PreviewSources {
/// for a package with no shader, and only [`PreviewSources::composite`]
/// can draw the frame.
pub fn live_treatments(&self) -> Option<Vec<GpuTreatment<'_>>> {
- let mut live: Vec<&Treatment> = self
+ let mut live: Vec<(usize, &Treatment)> = self
.treatments
.iter()
- .filter(|treatment| treatment.covers(self.time))
+ .enumerate()
+ .filter(|(_, treatment)| treatment.covers(self.time))
.collect();
- if live.iter().any(|treatment| !treatment.chain.is_empty()) {
+ if live
+ .iter()
+ .any(|(_, treatment)| !treatment.chain.is_empty())
+ {
return None;
}
- live.sort_by_key(|treatment| treatment.track);
+ live.sort_by_key(|(_, treatment)| treatment.track);
Some(
live.iter()
- .map(|treatment| GpuTreatment {
+ .map(|(index, treatment)| GpuTreatment {
track: treatment.track,
- passes: &treatment.passes,
+ passes: &self.treatment_passes[*index],
strength: treatment.strength_at(self.time),
})
.collect(),
@@ -1450,14 +1459,12 @@ pub fn preview_sources_of(
tracks,
treatments,
pre_chains,
- passes,
- riding,
+ chains,
cutouts,
highlight,
} = &plan.built;
let highlight = *highlight;
let time = quantise(seconds, rate);
- let frame_seconds = time.as_f64();
let plan_at = plan_frame(timeline, time);
let mut sources: Vec<Source<std::sync::Arc<Frame>>> = Vec::with_capacity(plan_at.layers.len());
@@ -1498,7 +1505,7 @@ pub fn preview_sources_of(
transform: layer.transform,
track: tracks.get(&layer.clip).copied().unwrap_or(0),
blend: layer.blend,
- passes: passes_at(passes, riding, layer.clip, frame_seconds),
+ passes: passes_at(chains, timeline, layer.clip, time),
})
}
Err(error) => failures.push(format!("{}: {error}", layer.media.display())),
@@ -1521,6 +1528,10 @@ pub fn preview_sources_of(
width: plan.width,
height: plan.height,
time,
+ treatment_passes: treatments
+ .iter()
+ .map(|treatment| treatment.passes_at(time))
+ .collect(),
treatments: treatments.clone(),
})
}
@@ -1674,7 +1685,7 @@ mod tests {
end: Rational::from_int(10),
track: 1,
chain: "negate".to_owned(),
- passes: Vec::new(),
+ effects: Vec::new(),
strength: 1.0,
ramp_in: 0.0,
ramp_out: 0.0,
diff --git a/src/crates/concat-export/src/resolve.rs b/src/crates/concat-export/src/resolve.rs
index d1d4e7df..ada2c296 100644
--- a/src/crates/concat-export/src/resolve.rs
+++ b/src/crates/concat-export/src/resolve.rs
@@ -92,12 +92,10 @@ pub(crate) struct BuiltTimeline {
pub(crate) tracks: HashMap<ClipId, usize>,
/// The clip's pre-fit chain - its crop - where it has one.
pub(crate) pre_chains: HashMap<ClipId, String>,
- /// The clip's shader passes, on a GPU renderer.
- pub(crate) passes: HashMap<ClipId, Vec<ShaderPass>>,
- /// The clips whose chains ride: a knob with keys is worth something
- /// different each frame, so their passes are built per frame from the
- /// chain and the clip's span rather than read from `passes`.
- pub(crate) riding: HashMap<ClipId, RidingChain>,
+ /// The clip's applied effects, on a GPU renderer: the passes are
+ /// resolved from them at each frame, because a knob with keys is worth
+ /// something different each frame and the resolution is cheap.
+ pub(crate) chains: HashMap<ClipId, Vec<AppliedFilter>>,
/// The clip whose cutout is drawn tinted rather than cut, if one is.
pub(crate) highlight: Option<ClipId>,
/// The layers: treatments over the stack, by span.
@@ -106,14 +104,6 @@ pub(crate) struct BuiltTimeline {
pub(crate) cutouts: HashMap<ClipId, CutoutJob>,
}
-/// A picture chain with keys on it, and where its clip sits, so a frame's
-/// passes can be built at the right point of the ride.
-pub(crate) struct RidingChain {
- pub(crate) effects: Vec<AppliedFilter>,
- pub(crate) start: f64,
- pub(crate) duration: f64,
-}
-
/// A layer clip, as the compositor needs it: when, over which tracks, what
/// chain, and how hard.
#[derive(Clone, Debug)]
@@ -122,9 +112,10 @@ pub(crate) struct Treatment {
pub(crate) end: Rational,
pub(crate) track: usize,
pub(crate) chain: String,
- /// The layer's shader passes, when the renderer runs them; the chain is
- /// then whatever the GPU cannot.
- pub(crate) passes: Vec<ShaderPass>,
+ /// The layer's applied effects, when the renderer runs shaders; the
+ /// chain is then whatever the GPU cannot. Resolved to passes at each
+ /// frame, so a keyed knob rides.
+ pub(crate) effects: Vec<AppliedFilter>,
pub(crate) strength: f32,
pub(crate) ramp_in: f64,
pub(crate) ramp_out: f64,
@@ -135,6 +126,17 @@ impl Treatment {
self.start <= time && time < self.end
}
+ /// The shader passes at `time`, each keyed knob at its value there.
+ pub(crate) fn passes_at(&self, time: Rational) -> Vec<ShaderPass> {
+ let span = (self.end - self.start).as_f64();
+ let at = if span > 0.0 {
+ ((time - self.start).as_f64() / span).clamp(0.0, 1.0)
+ } else {
+ 0.0
+ };
+ Catalogue::builtin().shader_passes_at(&self.effects, at)
+ }
+
/// How hard the treatment is applied at `time`: the strength, eased in
/// and out over the ramps at either end.
pub(crate) fn strength_at(&self, time: Rational) -> f32 {
@@ -165,8 +167,7 @@ pub(crate) fn build_timeline(
let mut tracks_of: HashMap<ClipId, usize> = HashMap::new();
let mut treatments: Vec<Treatment> = Vec::new();
let mut pre_chains: HashMap<ClipId, String> = HashMap::new();
- let mut passes: HashMap<ClipId, Vec<ShaderPass>> = HashMap::new();
- let mut riding: HashMap<ClipId, RidingChain> = HashMap::new();
+ let mut chains: HashMap<ClipId, Vec<AppliedFilter>> = HashMap::new();
let mut cutouts: HashMap<ClipId, CutoutJob> = HashMap::new();
let mut highlight: Option<ClipId> = None;
@@ -189,14 +190,18 @@ pub(crate) fn build_timeline(
// stack, kept beside the timeline rather than in it.
if clip.kind == ClipKind::Layer {
let chain = full_chain(clip, gpu);
- let layer_passes = if gpu { shader_passes(clip) } else { Vec::new() };
- if !chain.is_empty() || !layer_passes.is_empty() {
+ let effects = if gpu {
+ shaded(&clip.effects)
+ } else {
+ Vec::new()
+ };
+ if !chain.is_empty() || !effects.is_empty() {
treatments.push(Treatment {
start,
end: start + duration,
track: clip.track,
chain,
- passes: layer_passes,
+ effects,
strength: clip.opacity.clamp(0.0, 1.0) as f32,
ramp_in: clip.fade_in.max(0.0),
ramp_out: clip.fade_out.max(0.0),
@@ -247,23 +252,9 @@ pub(crate) fn build_timeline(
pre_chains.insert(id, pre);
}
if gpu {
- let clip_passes = shader_passes(clip);
- if !clip_passes.is_empty() {
- passes.insert(id, clip_passes);
- }
- if clip
- .effects
- .iter()
- .any(|link| link.enabled && !link.keys.is_empty())
- {
- riding.insert(
- id,
- RidingChain {
- effects: clip.effects.clone(),
- start: clip.start,
- duration: clip.duration,
- },
- );
+ let effects = shaded(&clip.effects);
+ if !effects.is_empty() {
+ chains.insert(id, effects);
}
}
if let Some(job) = CutoutJob::of(clip) {
@@ -283,8 +274,7 @@ pub(crate) fn build_timeline(
tracks: tracks_of,
treatments,
pre_chains,
- passes,
- riding,
+ chains,
cutouts,
highlight,
}
@@ -336,9 +326,20 @@ pub(crate) fn full_chain(clip: &ExportClip, gpu: bool) -> String {
parts.join(",")
}
-/// The clip's shader passes, for a renderer that runs them.
-pub(crate) fn shader_passes(clip: &ExportClip) -> Vec<ShaderPass> {
- Catalogue::builtin().shader_passes(&clip.effects)
+/// The enabled entries of a chain whose package has a shader: the ones a
+/// renderer that runs shaders resolves to passes each frame.
+pub(crate) fn shaded(effects: &[AppliedFilter]) -> Vec<AppliedFilter> {
+ let catalogue = Catalogue::builtin();
+ effects
+ .iter()
+ .filter(|applied| {
+ applied.enabled
+ && catalogue
+ .get(&applied.id)
+ .is_some_and(|package| package.shader().is_some())
+ })
+ .cloned()
+ .collect()
}
/// The engine's keys for a flattened clip's animation, or None for none.