Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(render): a stabilized clip is drawn through its correction, on the monitor and in the export alike (#253)

Jareer72cbfe50aecf
diff --git a/src/crates/concat-export/src/flatten.rs b/src/crates/concat-export/src/flatten.rs
index 0157f8c2..cdce86d7 100644
--- a/src/crates/concat-export/src/flatten.rs
+++ b/src/crates/concat-export/src/flatten.rs
@@ -31,8 +31,9 @@ pub fn flatten_timeline(project: &Project, timeline_id: Option<&str>) -> Vec<Exp
}
/// [`flatten_timeline`], knowing the project folder: a clip with a cutout
-/// is then told where its media's masks are, and renders cut. Without the
-/// folder the cutout is carried but not rendered.
+/// is then told where its media's masks are, and renders cut, and a
+/// stabilized one where its media's movement is, and renders steadied.
+/// Without the folder both are carried but not rendered.
pub fn flatten_timeline_in(
project: &Project,
timeline_id: Option<&str>,
@@ -149,6 +150,15 @@ pub fn flatten_timeline_in(
}
_ => String::new(),
},
+ stabilize: clip.stabilize.filter(|_| kind == ClipKind::Video),
+ motion_dir: match (&clip.stabilize, project_dir) {
+ (Some(_), Some(dir)) if kind == ClipKind::Video => {
+ concat_vision::stabilize::motion_dir(dir, &media.path)
+ .to_string_lossy()
+ .into_owned()
+ }
+ _ => String::new(),
+ },
..ExportClip::blank(kind, clip.start, clip.duration, index)
})
})
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index cf6d6964..a97b1194 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -220,6 +220,16 @@ pub struct ExportClip {
/// - or empty when the flattener had no project folder to name it by.
#[serde(default)]
pub mask_dir: String,
+ /// The camera's shake taken out, as the document holds it. Rendered
+ /// only with `motion_dir`: a clip whose movement is not measured yet
+ /// shows as shot.
+ #[serde(default)]
+ pub stabilize: Option<concat_project::model::Stabilize>,
+ /// Where this clip's media has its measured movement - see
+ /// `concat_vision::stabilize::motion_dir` - or empty when the
+ /// flattener had no project folder to name it by.
+ #[serde(default)]
+ pub motion_dir: String,
/// A title's per-word reveal order, baked at rasterize time and set by
/// the host - never by the wire format, like `transition_chain`. Every
/// shader pass over this clip carries it, so a reveal effect can read
@@ -322,6 +332,8 @@ impl ExportClip {
has_audio: None,
cutout: None,
mask_dir: String::new(),
+ stabilize: None,
+ motion_dir: String::new(),
highlighted: false,
reveal_map: None,
backdrop: None,
@@ -1313,7 +1325,8 @@ pub fn audio_pieces(clip: &ExportClip) -> Vec<AudioClip> {
/// The plan's layer for one clip at one instant, filled in: the picture
/// decoded, its effects resolved, on its track, and its crop and flips
-/// where the plan draws them (`geometry`; see `resolve::planned_geometry`).
+/// where the plan draws them (`geometry`; see `resolve::planned_geometry`),
+/// and its stabilization's correction at the frame it shows.
/// A clip without planned geometry has them baked in by the decoder's
/// chain, as the transition fades still are, and the cutout has already
/// cut the picture either way.
@@ -1324,6 +1337,7 @@ fn filled(
effects: Vec<ShaderPass>,
geometry: Option<&resolve::PlannedGeometry>,
transitions: Vec<Transition>,
+ stabilized: Option<&resolve::StabilizeJob>,
) -> PlannedLayer {
let mut filled = PlannedLayer {
source: Some(frame),
@@ -1332,6 +1346,9 @@ fn filled(
transitions,
..layer.clone()
};
+ if let Some(job) = stabilized {
+ filled.warp = job.warp_at(layer.source_time);
+ }
if let Some(geometry) = geometry {
filled.crop = geometry.crop;
filled.flip_h = geometry.flip_h;
@@ -1602,6 +1619,7 @@ fn render_picture(
chains,
reveal_maps,
cutouts,
+ stabilized,
highlight: _,
backdrops,
} = build_timeline(request, rate, visible, transitions);
@@ -1697,6 +1715,7 @@ fn render_picture(
passes_at(&chains, &reveal_maps, &timeline, layer.clip, time),
geometry.get(&layer.clip),
shapes_at(&shapes, layer, time, rate),
+ stabilized.get(&layer.clip),
));
continue;
}
@@ -1764,6 +1783,7 @@ fn render_picture(
passes_at(&chains, &reveal_maps, &timeline, layer.clip, time),
geometry.get(&layer.clip),
shapes_at(&shapes, layer, time, rate),
+ stabilized.get(&layer.clip),
));
}
}
@@ -2214,6 +2234,7 @@ pub fn preview_sources_of(
chains,
reveal_maps,
cutouts,
+ stabilized,
highlight,
backdrops,
} = &plan.built;
@@ -2256,6 +2277,7 @@ pub fn preview_sources_of(
passes_at(chains, reveal_maps, timeline, layer.clip, time),
geometry.get(&layer.clip),
shapes_at(shapes, layer, time, rate),
+ stabilized.get(&layer.clip),
))
}
Err(error) => failures.push(format!("{}: {error}", layer.media.display())),
diff --git a/src/crates/concat-export/src/resolve.rs b/src/crates/concat-export/src/resolve.rs
index 31013621..ab676ca3 100644
--- a/src/crates/concat-export/src/resolve.rs
+++ b/src/crates/concat-export/src/resolve.rs
@@ -12,6 +12,7 @@
//! what this builds and never look at an `ExportClip` again.
use super::*;
+use concat_vision::stabilize::{self, Correction, Motion};
/// The tint a highlighted cutout wears: the interface's accent, the same
/// lime the brushes and the selection are drawn in.
@@ -83,6 +84,63 @@ impl CutoutJob {
}
}
+/// A stabilization as the frame loop runs it: the clip's correction for
+/// every frame it shows, worked out once from the media's measured
+/// movement and the clip's settings when the timeline is built. Nothing
+/// is cached past that: the working out is a few milliseconds even for a
+/// long clip, and a changed setting is a rebuilt timeline anyway.
+pub(crate) struct StabilizeJob {
+ correction: Correction,
+}
+
+impl StabilizeJob {
+ /// The job for a clip, or `None` when it is not stabilized or its
+ /// media's movement has not been measured.
+ pub(crate) fn of(clip: &ExportClip) -> Option<StabilizeJob> {
+ let settings = clip.stabilize?;
+ if clip.kind != ClipKind::Video || clip.motion_dir.is_empty() {
+ return None;
+ }
+ let motion = Motion::read(Path::new(&clip.motion_dir))?;
+ // The same clamp the picture's own pacing applies; a curve's mean
+ // is its speed.
+ let speed = audio::clamp_speed(clip.speed);
+ let range = (
+ clip.source_start,
+ clip.source_start + clip.duration.max(0.0) * speed,
+ );
+ let correction = stabilize::path::correct(
+ &motion,
+ range,
+ speed,
+ &stabilize::Settings {
+ smoothness: settings.smoothness,
+ max_zoom: settings.max_zoom,
+ tripod: settings.tripod,
+ keep_rotation: settings.keep_rotation,
+ },
+ );
+ (!correction.is_empty()).then_some(StabilizeJob { correction })
+ }
+
+ /// The correction for the frame on screen at `source_time`, as the
+ /// compositor takes it.
+ pub(crate) fn warp_at(&self, source_time: Rational) -> concat_render::Warp {
+ let map = self.correction.at(source_time.as_f64());
+ concat_render::Warp {
+ a: map.a as f32,
+ b: map.b as f32,
+ tx: map.tx as f32,
+ ty: map.ty as f32,
+ }
+ }
+
+ /// The zoom the clip is drawn at, `>= 1`.
+ pub(crate) fn zoom(&self) -> f64 {
+ self.correction.zoom
+ }
+}
+
/// An engine timeline plus the per-clip facts the decoders need that the
/// engine's model has no field for.
pub(crate) struct BuiltTimeline {
@@ -118,6 +176,8 @@ pub(crate) struct BuiltTimeline {
/// What fills the frame behind each picture that has a background; see
/// [`crate::with_backdrops`].
pub(crate) backdrops: HashMap<ClipId, crate::ExportBackdrop>,
+ /// The clips whose camera shake is taken out.
+ pub(crate) stabilized: HashMap<ClipId, StabilizeJob>,
}
/// A packaged transition over a cut. The incoming clip has been overlapped
@@ -216,6 +276,7 @@ pub(crate) fn build_timeline(
let mut reveal_maps: HashMap<ClipId, Arc<RevealMap>> = HashMap::new();
let mut cutouts: HashMap<ClipId, CutoutJob> = HashMap::new();
let mut backdrops: HashMap<ClipId, crate::ExportBackdrop> = HashMap::new();
+ let mut stabilized: HashMap<ClipId, StabilizeJob> = HashMap::new();
let mut highlight: Option<ClipId> = None;
let lanes = visible.iter().map(|clip| clip.track).max().unwrap_or(0) + 1;
@@ -282,9 +343,30 @@ pub(crate) fn build_timeline(
stills.insert(id);
}
let planned = planned_geometry(clip);
+ let steadied = StabilizeJob::of(clip);
if let Some(size) = fitted_size(request, clip, planned.is_some()) {
+ // A stabilized picture is drawn larger by its zoom: decoded
+ // that much larger too, up to the file's own size, so the
+ // zoom costs no sharpness the file had.
+ let size = match (&steadied, clip.media_width, clip.media_height) {
+ (Some(job), Some(width), Some(height)) if job.zoom() > 1.0 => {
+ let grow = job
+ .zoom()
+ .min(f64::from(width) / f64::from(size.0.max(1)))
+ .min(f64::from(height) / f64::from(size.1.max(1)))
+ .max(1.0);
+ (
+ ((f64::from(size.0) * grow).round() as u32).max(2) & !1,
+ ((f64::from(size.1) * grow).round() as u32).max(2) & !1,
+ )
+ }
+ _ => size,
+ };
decode_sizes.insert(id, size);
}
+ if let Some(job) = steadied {
+ stabilized.insert(id, job);
+ }
if !clip.transition_shapes.is_empty() {
shapes.insert(id, clip.transition_shapes.clone());
}
@@ -326,6 +408,7 @@ pub(crate) fn build_timeline(
chains,
reveal_maps,
cutouts,
+ stabilized,
highlight,
backdrops,
}
@@ -343,12 +426,15 @@ pub(crate) struct PlannedGeometry {
/// it has neither. The plan crops and flips the picture as it draws it,
/// before the clip's effects run over it: a vignette is centred on the
/// picture as cropped, a wipe on a mirrored clip still wipes the way the
-/// frame is seen.
+/// frame is seen. A stabilized clip always has them drawn by the plan,
+/// even when it has neither: its correction is of the whole picture as
+/// shot, before any crop, so the decoder must hand over the whole of it.
pub(crate) fn planned_geometry(clip: &ExportClip) -> Option<PlannedGeometry> {
let crop = clip
.crop
.map_or(concat_render::Crop::NONE, concat_render::Crop::of);
- if crop.is_none() && !clip.flip_h && !clip.flip_v {
+ let stabilized = clip.stabilize.is_some() && !clip.motion_dir.is_empty();
+ if crop.is_none() && !clip.flip_h && !clip.flip_v && !stabilized {
return None;
}
Some(PlannedGeometry {
diff --git a/src/crates/concat-project/src/commands/mod.rs b/src/crates/concat-project/src/commands/mod.rs
index d663d511..90834942 100644
--- a/src/crates/concat-project/src/commands/mod.rs
+++ b/src/crates/concat-project/src/commands/mod.rs
@@ -15,8 +15,8 @@ use serde::{Deserialize, Serialize};
use crate::model::{
AppliedFilter, AudioTrack, Background, Clip, ClipKind, ColorRange, ColorSpace, Crop,
CustomFont, Cutout, CutoutMode, Enhanced, Framing, KeyEase, KeyProperty, MediaItem, MediaKind,
- MediaOrigin, Project, ShapeStyle, SpeedPoint, Stroke, TextStyle, Timeline, Track, Transition,
- VideoSettings,
+ MediaOrigin, Project, ShapeStyle, SpeedPoint, Stabilize, Stroke, TextStyle, Timeline, Track,
+ Transition, VideoSettings,
};
mod audio;
@@ -463,6 +463,16 @@ pub enum Command {
/// The cutout, or None to take it off.
cutout: Option<Cutout>,
},
+ /// Turns a picture's stabilization on, sets its settings, or takes it
+ /// off. Tidied on the way in; see [`Stabilize::tidy`]. The camera's
+ /// movement is measured by the host the first time a clip of the
+ /// media asks for it; until then the clip shows as shot.
+ SetClipStabilize {
+ /// The clip. An unknown id is a no-op.
+ clip_id: String,
+ /// The settings, or None to take stabilization off.
+ stabilize: Option<Stabilize>,
+ },
/// Paints one stroke onto a clip's cutout. A clip with no cutout gets a
/// custom one; an automatic cutout becomes custom, since a stroke is a
/// correction to it. A stroke with no points is a no-op.
@@ -1002,6 +1012,9 @@ impl Command {
bad([cutout.feather]) || cutout.strokes.iter().any(bad_stroke)
}),
Command::AddCutoutStroke { stroke, .. } => bad_stroke(stroke),
+ Command::SetClipStabilize { stabilize, .. } => stabilize
+ .as_ref()
+ .is_some_and(|stabilize| bad([stabilize.smoothness, stabilize.max_zoom])),
Command::MoveClips { moves } => bad(moves.iter().map(|wanted| wanted.start)),
Command::TrimClip { delta, .. } => bad([*delta]),
Command::SplitClips { time, .. } => bad([*time]),
@@ -1139,6 +1152,7 @@ pub fn apply(
| Command::SetClipSpeed { .. }
| Command::SetClipCutout { .. }
| Command::AddCutoutStroke { .. }
+ | Command::SetClipStabilize { .. }
| Command::SetClipKey { .. }
| Command::ClearClipKey { .. }
| Command::ClearClipKeys { .. }
diff --git a/src/crates/concat-project/src/commands/properties.rs b/src/crates/concat-project/src/commands/properties.rs
index 9417fc83..b14819ae 100644
--- a/src/crates/concat-project/src/commands/properties.rs
+++ b/src/crates/concat-project/src/commands/properties.rs
@@ -140,6 +140,26 @@ pub(super) fn apply(
})
}
+ Command::SetClipStabilize {
+ clip_id,
+ stabilize,
+ } => {
+ let timeline = project.active_mut();
+ let Some(clip) = timeline.clip_mut(&clip_id) else {
+ return Ok(Outcome::default());
+ };
+ // Only footage has a camera that moved: a still, a title, a
+ // shape or a layer has nothing to steady.
+ if clip.kind != ClipKind::Video && stabilize.is_some() {
+ return Ok(Outcome::default());
+ }
+ let applied = assign(&mut clip.stabilize, stabilize.map(Stabilize::tidy));
+ Ok(Outcome {
+ created_id: None,
+ applied,
+ })
+ }
+
Command::AddCutoutStroke { clip_id, stroke } => {
let timeline = project.active_mut();
let Some(clip) = timeline.clip_mut(&clip_id) else {
diff --git a/src/crates/concat-project/src/model.rs b/src/crates/concat-project/src/model.rs
index a8cdf590..1ccb6e44 100644
--- a/src/crates/concat-project/src/model.rs
+++ b/src/crates/concat-project/src/model.rs
@@ -566,6 +566,73 @@ impl Stroke {
}
}
+/// A clip's stabilization: the camera's shake taken out of the picture.
+///
+/// The camera's movement is measured once, frame to frame, by the host's
+/// analysis and cached in the project folder beside the masks; these are
+/// the settings the smooth path and the correction are worked out from,
+/// live, whenever they change. Off is no `Stabilize` at all.
+#[derive(Clone, Copy, PartialEq, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", default)]
+pub struct Stabilize {
+ /// How much of the camera's movement is smoothed away, `0..=1`: low
+ /// keeps a pan or a glide and takes out the jitter, high leaves only
+ /// the slowest of moves. Measured on the timeline, so a clip sped up
+ /// is smoothed as calmly as one at its own pace.
+ pub smoothness: f64,
+ /// The most the picture may be zoomed to hide the edges the correction
+ /// moves into view, `1..=MAX_STABILIZE_ZOOM`. The zoom used is the
+ /// least the whole clip needs, up to this; past it the correction is
+ /// eased where it would show an edge.
+ pub max_zoom: f64,
+ /// Lock the frame entirely, as on a tripod, rather than follow the
+ /// camera's intended move.
+ #[serde(skip_serializing_if = "is_false")]
+ pub tripod: bool,
+ /// Leave the camera's roll as shot: only its drift and its zoom are
+ /// corrected.
+ #[serde(skip_serializing_if = "is_false")]
+ pub keep_rotation: bool,
+}
+
+/// The smoothness a stabilization starts with.
+pub const DEFAULT_STABILIZE_SMOOTHNESS: f64 = 0.5;
+/// The zoom a stabilization may use to begin with: a fifth larger.
+pub const DEFAULT_STABILIZE_ZOOM: f64 = 1.2;
+/// The most a stabilization may zoom: half again.
+pub const MAX_STABILIZE_ZOOM: f64 = 1.5;
+
+impl Default for Stabilize {
+ fn default() -> Self {
+ Stabilize {
+ smoothness: DEFAULT_STABILIZE_SMOOTHNESS,
+ max_zoom: DEFAULT_STABILIZE_ZOOM,
+ tripod: false,
+ keep_rotation: false,
+ }
+ }
+}
+
+impl Stabilize {
+ /// Every number held to its range, a number that is not one to its
+ /// default.
+ pub fn tidy(self) -> Stabilize {
+ Stabilize {
+ smoothness: if self.smoothness.is_finite() {
+ self.smoothness.clamp(0.0, 1.0)
+ } else {
+ DEFAULT_STABILIZE_SMOOTHNESS
+ },
+ max_zoom: if self.max_zoom.is_finite() {
+ self.max_zoom.clamp(1.0, MAX_STABILIZE_ZOOM)
+ } else {
+ DEFAULT_STABILIZE_ZOOM
+ },
+ ..self
+ }
+ }
+}
+
/// A crop: what is taken off each edge, as fractions of the source.
#[derive(Clone, Copy, PartialEq, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -1467,6 +1534,10 @@ pub struct Clip {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(deserialize_with = "wire::maybe")]
pub cutout: Option<Cutout>,
+ /// The camera's shake taken out; see [`Stabilize`]. None is off.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[serde(deserialize_with = "wire::maybe")]
+ pub stabilize: Option<Stabilize>,
/// Keep voices at their natural pitch when `speed` is not 1. On by
/// default; off gives the tape-machine chipmunk/slow-motion sound.
pub preserve_pitch: bool,
@@ -1604,6 +1675,7 @@ impl Clip {
blend: String::new(),
crop: None,
cutout: None,
+ stabilize: None,
filters: Vec::new(),
video_effects: Vec::new(),
audio_stream: None,
@@ -1660,6 +1732,7 @@ impl Clip {
.map(Crop::tidy)
.filter(|crop| !crop.is_none());
self.cutout = self.cutout.take().map(Cutout::tidy);
+ self.stabilize = self.stabilize.take().map(Stabilize::tidy);
if let Some(transition) = self.transition_in.as_mut() {
transition.duration = finite(transition.duration, 1.0).max(MIN_TRANSITION);
}
diff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rs
index 16bdb59b..78684f83 100644
--- a/src/crates/concat-render/src/gpu/tests.rs
+++ b/src/crates/concat-render/src/gpu/tests.rs
@@ -15,7 +15,7 @@ use concat_core::timeline::{Blend, Transform};
use super::*;
use crate::CpuCompositor;
use crate::metrics::ssim;
-use crate::plan::{Backdrop, Crop, Transition, detached_clip};
+use crate::plan::{Backdrop, Crop, Transition, Warp, detached_clip};
fn gpu() -> Option<WgpuCompositor> {
let compositor = WgpuCompositor::new();
@@ -158,6 +158,104 @@ fn plain_layers_match_the_cpu_reference_to_the_pixel() {
}
}
+/// A stabilized frame is its source resampled through the correction: a
+/// warp of a whole number of pixels draws each pixel from that far over,
+/// to the level, and the edge it pulls into view holds the source's last
+/// column, as the sampler reads past an edge.
+#[test]
+fn a_stabilized_frame_is_the_source_through_its_warp() {
+ let Some(mut gpu) = gpu() else { return };
+ let (w, h) = (64u32, 48u32);
+ let source = gradient(w, h);
+ let mut shifted = layer(source.clone());
+ // Each point drawn is read 6 px right of and 3 px below itself.
+ shifted.warp = Warp {
+ a: 1.0,
+ b: 0.0,
+ tx: 6.0 / h as f32,
+ ty: 3.0 / h as f32,
+ };
+ let out = gpu.render(&plan(w, h, vec![shifted]));
+ for y in 0..h {
+ for x in 0..w {
+ let want = source
+ .pixel((x + 6).min(w - 1), (y + 3).min(h - 1))
+ .expect("inside");
+ let got = out.pixel(x, y).expect("inside");
+ for channel in 0..3 {
+ let difference = (i32::from(want[channel]) - i32::from(got[channel])).abs();
+ assert!(
+ difference <= 1,
+ "({x}, {y}) channel {channel}: want {want:?}, got {got:?}"
+ );
+ }
+ }
+ }
+
+ // A turn and a zoom about the centre: a point of the picture reads
+ // from where the turn and the zoom send it. Probed on the smooth
+ // gradient, away from the bar, whose edge the two blend differently
+ // (the GPU in light, the reference in its stored levels).
+ let (angle, zoom) = (8f32.to_radians(), 1.0 / 1.15);
+ let mut turned = layer(source.clone());
+ turned.warp = Warp {
+ a: zoom * angle.cos(),
+ b: zoom * angle.sin(),
+ tx: 0.0,
+ ty: 0.0,
+ };
+ let out = gpu.render(&plan(w, h, vec![turned.clone()]));
+ let probe = |x: f32, y: f32| {
+ let (nx, ny) = ((x - w as f32 / 2.0) / h as f32, (y - h as f32 / 2.0) / h as f32);
+ let (mx, my) = (
+ turned.warp.a * nx - turned.warp.b * ny,
+ turned.warp.b * nx + turned.warp.a * ny,
+ );
+ (mx * h as f32 + w as f32 / 2.0, my * h as f32 + h as f32 / 2.0)
+ };
+ for (x, y) in [(52u32, 10u32), (10, 36), (48, 40), (40, 6)] {
+ let (sx, sy) = probe(x as f32 + 0.5, y as f32 + 0.5);
+ let want = crate::reference::sample_bilinear(&source, sx - 0.5, sy - 0.5);
+ let got = out.pixel(x, y).expect("inside");
+ for channel in 0..3 {
+ assert!(
+ (want[channel] - f32::from(got[channel])).abs() <= 3.0,
+ "({x}, {y}) channel {channel}: want {want:?}, got {got:?}"
+ );
+ }
+ }
+
+ // And the CPU reference agrees, alone, cropped, and made first for an
+ // effect to run over.
+ assert_parity("stabilized", &plan(w, h, vec![turned.clone()]), 0.99);
+ let mut cropped = turned.clone();
+ cropped.crop = Crop::of([0.1, 0.0, 0.2, 0.1]);
+ cropped.flip_h = true;
+ assert_parity("stabilized and cropped", &plan(w, h, vec![cropped.clone()]), 0.99);
+ let mut prepared = cropped;
+ prepared.effects = vec![package("test.invert", INVERT, "", &[], 1.0)];
+ let geometry = prepared.geometry(&source, w, h);
+ assert!(prepared.needs_preparing(&geometry));
+ let inverted = gpu.render(&plan(w, h, vec![prepared]));
+ let mut plain = turned;
+ plain.crop = Crop::of([0.1, 0.0, 0.2, 0.1]);
+ plain.flip_h = true;
+ let straight = gpu.render(&plan(w, h, vec![plain]));
+ // The invert ran over the warped picture: inside the fitted picture,
+ // each pixel is the other's inverse.
+ let middle = (w / 2, h / 2);
+ let (a, b) = (
+ inverted.pixel(middle.0, middle.1).expect("inside"),
+ straight.pixel(middle.0, middle.1).expect("inside"),
+ );
+ for channel in 0..3 {
+ assert!(
+ (i32::from(a[channel]) + i32::from(b[channel]) - 255).abs() <= 2,
+ "{a:?} vs {b:?}"
+ );
+ }
+}
+
/// A pass runs over the layer before it is placed: an invert shader
/// over a red frame composites cyan, and at half intensity the mix, in
/// light - half of each channel, which BT.1886's 2.4 gamma stores at 191.
diff --git a/src/crates/concat-render/src/lib.rs b/src/crates/concat-render/src/lib.rs
index 153a2517..2cf68820 100644
--- a/src/crates/concat-render/src/lib.rs
+++ b/src/crates/concat-render/src/lib.rs
@@ -32,6 +32,7 @@ pub use gpu::WgpuCompositor;
pub use metrics::ssim;
pub use plan::{
Backdrop, Crop, FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading, Transition,
+ Warp,
detached_clip, plan_frame,
};
#[cfg(test)]
diff --git a/src/crates/concat-render/src/plan.rs b/src/crates/concat-render/src/plan.rs
index 68c8ac84..1996d738 100644
--- a/src/crates/concat-render/src/plan.rs
+++ b/src/crates/concat-render/src/plan.rs
@@ -16,9 +16,9 @@
//! resolved for this frame, the mask, the transitions at the cut, the
//! treatments live over the stack - and hands the plan to a compositor.
//!
-//! What a compositor does with one layer, in order: the crop, the flips and
-//! the fit make the picture; the effects run over it; the mask, the fades
-//! and the wipes weigh it; the placement puts it in the frame; the blend
+//! What a compositor does with one layer, in order: the warp a stabilized
+//! clip is corrected by, the crop, the flips and the fit make the picture;
+//! the effects run over it; the mask, the fades and the wipes weigh it; the placement puts it in the frame; the blend
//! meets what is beneath. The geometry and the weighing are worked out
//! here, once, so the two backends cannot drift on them.
@@ -86,6 +86,62 @@ impl Crop {
}
}
+/// A turn, a zoom and a shift the source is resampled through before its
+/// crop: what a stabilized clip's correction is at one frame.
+///
+/// It maps a point of the source as drawn to the point of the source as
+/// decoded that is drawn there, in the picture's normalised units - the
+/// source's centre the origin, `y` down, its height one - so it means the
+/// same whatever size the decoder gave the picture. `x' = a·x − b·y + tx`,
+/// `y' = b·x + a·y + ty`.
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct Warp {
+ /// The zoom times the cosine of the turn.
+ pub a: f32,
+ /// The zoom times its sine.
+ pub b: f32,
+ /// The shift across, in source heights.
+ pub tx: f32,
+ /// The shift down.
+ pub ty: f32,
+}
+
+impl Warp {
+ /// The source as decoded.
+ pub const NONE: Warp = Warp {
+ a: 1.0,
+ b: 0.0,
+ tx: 0.0,
+ ty: 0.0,
+ };
+
+ /// True when nothing moves.
+ pub fn is_none(&self) -> bool {
+ *self == Warp::NONE
+ }
+
+ /// Where a point of the drawn source, `(x, y)` in the pixels of a
+ /// `width` by `height` source, is read from in the decoded one.
+ pub fn source_pixel(&self, (x, y): (f32, f32), (width, height): (u32, u32)) -> (f32, f32) {
+ if self.is_none() {
+ return (x, y);
+ }
+ let (w, h) = (width as f32, height as f32);
+ let (nx, ny) = ((x - w / 2.0) / h, (y - h / 2.0) / h);
+ let (mx, my) = (
+ self.a * nx - self.b * ny + self.tx,
+ self.b * nx + self.a * ny + self.ty,
+ );
+ (mx * h + w / 2.0, my * h + h / 2.0)
+ }
+}
+
+impl Default for Warp {
+ fn default() -> Self {
+ Warp::NONE
+ }
+}
+
/// What a cut does to a layer at one frame: the picture pulled towards a
/// colour, or uncovered behind a moving edge. Resolved by the executor
/// from the transition's shape and the frame's place in it; the compositor
@@ -205,6 +261,9 @@ pub struct PlannedLayer {
/// whatever size the decoder gave it. `None` until then, and a layer
/// without one draws nothing rather than failing the frame.
pub source: Option<Arc<Frame>>,
+ /// The source resampled before anything else is done to it: a
+ /// stabilized clip's correction at this frame. `NONE` for every other.
+ pub warp: Warp,
/// What is cut off the source's edges before it is fitted. `NONE` for
/// a picture the decoder already cropped.
pub crop: Crop,
@@ -245,6 +304,7 @@ impl PlannedLayer {
speed: Rational::ONE,
paced: true,
source: Some(frame),
+ warp: Warp::NONE,
crop: Crop::NONE,
flip_h: false,
flip_v: false,
@@ -350,7 +410,9 @@ impl PlannedLayer {
///
/// It covers the frame, centred, whatever the picture's own scale,
/// position and turn: the background stays put while the picture moves
- /// over it. It belongs to the picture all the same: it is on the
+ /// over it. A blur is of the picture as drawn, a stabilized one's
+ /// correction and all, so it is as steady as the picture over it. It
+ /// belongs to the picture all the same: it is on the
/// picture's track, as strong as the picture is at this frame, and goes
/// through the picture's fades and wipes, so a dissolve or a fade to
/// black takes the background with the picture rather than leaving it
@@ -386,12 +448,14 @@ impl PlannedLayer {
Backdrop::Colour([r, g, b]) => {
let mut solid = Frame::black(16, 16);
solid.fill([*r, *g, *b, 255]);
+ layer.warp = Warp::NONE;
layer.crop = Crop::NONE;
layer.flip_h = false;
layer.flip_v = false;
(Arc::new(solid), Vec::new())
}
Backdrop::Image(still) => {
+ layer.warp = Warp::NONE;
layer.crop = Crop::NONE;
layer.flip_h = false;
layer.flip_v = false;
@@ -496,6 +560,9 @@ pub struct Geometry {
pub source_rect: [f32; 4],
/// The size of the source the rectangle is of.
pub source_size: (u32, u32),
+ /// The source resampled before the rectangle is taken of it; see
+ /// [`Warp`].
+ pub warp: Warp,
/// The picture's size once fitted: the size the effects run at.
pub fitted: (u32, u32),
/// Where the picture's centre lands in the output, after translation.
@@ -540,6 +607,7 @@ impl Geometry {
Geometry {
source_rect,
source_size: (source_width, source_height),
+ warp: layer.warp,
fitted,
centre,
scale,
@@ -553,6 +621,7 @@ impl Geometry {
Geometry {
source_rect: [0.0, 0.0, self.fitted.0 as f32, self.fitted.1 as f32],
source_size: self.fitted,
+ warp: Warp::NONE,
..*self
}
}
@@ -560,7 +629,8 @@ impl Geometry {
/// True when the fitted picture is the whole source at its own size.
pub fn fits_source(&self) -> bool {
let [x, y, w, h] = self.source_rect;
- x == 0.0
+ self.warp.is_none()
+ && x == 0.0
&& y == 0.0
&& w == self.source_size.0 as f32
&& h == self.source_size.1 as f32
@@ -593,7 +663,10 @@ impl Geometry {
let u = if flip_h { 1.0 - u } else { u };
let v = if flip_v { 1.0 - v } else { v };
let [x, y, w, h] = self.source_rect;
- (x + u * w - 0.5, y + v * h - 0.5)
+ let (sx, sy) = self
+ .warp
+ .source_pixel((x + u * w, y + v * h), self.source_size);
+ (sx - 0.5, sy - 0.5)
}
/// The same, as texture coordinates in `0..1` of the source.
@@ -601,9 +674,12 @@ impl Geometry {
let u = if flip_h { 1.0 - u } else { u };
let v = if flip_v { 1.0 - v } else { v };
let [x, y, w, h] = self.source_rect;
+ let (sx, sy) = self
+ .warp
+ .source_pixel((x + u * w, y + v * h), self.source_size);
(
- (x + u * w) / self.source_size.0 as f32,
- (y + v * h) / self.source_size.1 as f32,
+ sx / self.source_size.0 as f32,
+ sy / self.source_size.1 as f32,
)
}
}
@@ -713,6 +789,7 @@ pub fn plan_frame(timeline: &Timeline, time: Rational) -> FramePlan {
speed: clip.speed,
paced: clip.is_paced(),
source: None,
+ warp: Warp::NONE,
crop: Crop::NONE,
flip_h: false,
flip_v: false,