Skip to content

Open source CapCut alternative.

TypeScript

fix(site): keep tracking and clip effects in step [rebuild]

David Lec812c87b1ef5
diff --git a/site/packages/effects-kit/src/clipEffects.test.ts b/site/packages/effects-kit/src/clipEffects.test.ts
new file mode 100644
index 00000000..bc60b5a9
--- /dev/null
+++ b/site/packages/effects-kit/src/clipEffects.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, test } from "bun:test";
+import { Canvas } from "skia-canvas";
+import { applyClipEffects, applyEffectToCanvas, type ClipEffect } from "./effects";
+
+type Surface = HTMLCanvasElement;
+const S = 64;
+
+/** An opaque grey square in the middle of a transparent canvas — a logo. */
+function logo(): Surface {
+ const c = new Canvas(S, S) as unknown as Surface;
+ const ctx = c.getContext("2d")!;
+ ctx.fillStyle = "#808080";
+ ctx.fillRect(16, 16, 32, 32);
+ return c;
+}
+const scratch = () => new Canvas(S, S) as unknown as Surface;
+const pixel = (c: Surface, x: number, y: number) => Array.from(c.getContext("2d")!.getImageData(x, y, 1, 1).data);
+const noGrain = () => null;
+
+describe("a clip's effects over a transparent picture", () => {
+ for (const effect of ["vignette", "flash", "lightleak", "grain"] as const) {
+ test(`${effect} paints the picture and leaves the clear corners clear`, () => {
+ const c = logo();
+ applyClipEffects(c, scratch(), [{ effect, amount: 1 }], 0, noGrain);
+ expect(pixel(c, 2, 2)[3]).toBe(0);
+ expect(pixel(c, S / 2, S / 2)[3]).toBe(255);
+ });
+ }
+});
+
+describe("filter effects in a row", () => {
+ test("draw as one joined filter with the result of drawing each in turn", () => {
+ const list: ClipEffect[] = [{ effect: "negative" }, { effect: "huecycle", amount: 0.5 }];
+ const joined = logo();
+ applyClipEffects(joined, scratch(), list, 0.4, noGrain);
+ const each = logo();
+ for (const e of list) applyEffectToCanvas(each, scratch(), e.effect, e.amount, 0.4, noGrain);
+ const a = pixel(joined, S / 2, S / 2);
+ const b = pixel(each, S / 2, S / 2);
+ for (let i = 0; i < 4; i++) expect(Math.abs(a[i] - b[i])).toBeLessThanOrEqual(2);
+ expect(pixel(joined, 2, 2)[3]).toBe(0);
+ });
+});
diff --git a/site/packages/effects-kit/src/effects.ts b/site/packages/effects-kit/src/effects.ts
index 701a5e55..3e8153a6 100644
--- a/site/packages/effects-kit/src/effects.ts
+++ b/site/packages/effects-kit/src/effects.ts
@@ -104,16 +104,13 @@ const lookOf = (effect: string): LookStyle | null =>
export const ALL_EFFECT_IDS: EffectId[] = [...EFFECT_IDS, ...AUDIO_EFFECT_IDS];
-/** The picture treatments a clip can wear itself. */
-export type ClipEffectId = "grain" | "vhs" | "glitch" | "blur" | "vignette" | "lightleak" | "flash" | "negative" | "huecycle";
-
/**
* The effects a clip wears treat that clip's picture alone, inside its mask,
* for the clip's whole length: a masked copy over the shot wears a negative
* and only the window turns. Zoom and shake move the frame, which a clip
* does with its own zoom and pose keys; the looks are the clip's grade.
*/
-export const CLIP_EFFECT_IDS: ClipEffectId[] = [
+export const CLIP_EFFECT_IDS = [
"negative",
"huecycle",
"grain",
@@ -123,7 +120,15 @@ export const CLIP_EFFECT_IDS: ClipEffectId[] = [
"vignette",
"lightleak",
"flash",
-];
+] as const satisfies readonly VisualEffectId[];
+
+/** The picture treatments a clip can wear itself. */
+export type ClipEffectId = (typeof CLIP_EFFECT_IDS)[number];
+
+/** How an effect meets the canvas edge: `opaque` paints the whole canvas (a
+ * frame), `keep` paints only where the picture is (a clip's own picture,
+ * which may be a still with clear corners). */
+export type EffectEdges = "opaque" | "keep";
/** One effect a clip wears; `amount` 0..1, absent = 0.5. */
export interface ClipEffect {
@@ -443,7 +448,8 @@ export function applyEffectToCanvas(
grainTileFor: (tick: number) => CanvasImageSource | null,
focus?: { x: number; y: number },
ramp?: number,
- dur?: number
+ dur?: number,
+ edges: EffectEdges = "opaque"
): void {
const state = effectPreviewState(effect, amount, tLocal, focus, ramp, dur);
const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
@@ -453,10 +459,36 @@ export function applyEffectToCanvas(
const H = canvas.height;
const scale = Math.min(W, H) / 1080;
+ // Paint laid over the picture. A clear picture edge takes it only where
+ // the picture is: a plain paint lands atop, and a blended one is first cut
+ // to the picture's coverage on the scratch.
+ const over = (mode: GlobalCompositeOperation, alpha: number, paint: (c: CanvasRenderingContext2D) => void) => {
+ const direct = edges === "opaque" || mode === "source-over";
+ const target = direct ? ctx : sctx;
+ target.save();
+ target.globalAlpha = direct ? alpha : 1;
+ target.globalCompositeOperation = edges === "keep" && mode === "source-over" ? "source-atop" : direct ? mode : "source-over";
+ if (!direct) target.clearRect(0, 0, W, H);
+ paint(target);
+ target.restore();
+ if (direct) return;
+ sctx.save();
+ sctx.globalCompositeOperation = "destination-in";
+ sctx.drawImage(canvas, 0, 0);
+ sctx.restore();
+ ctx.save();
+ ctx.globalAlpha = alpha;
+ ctx.globalCompositeOperation = mode;
+ ctx.drawImage(scratch, 0, 0);
+ ctx.restore();
+ };
+
const needsRedraw = state.cssFilter || state.dx || state.dy || (state.zoom && state.zoom !== 1);
if (needsRedraw) {
sctx.clearRect(0, 0, W, H);
sctx.drawImage(canvas, 0, 0);
+ // Over a clear edge the old picture would show through the redraw.
+ if (edges === "keep") ctx.clearRect(0, 0, W, H);
ctx.save();
if (state.cssFilter) ctx.filter = state.cssFilter;
const zoom = state.zoom ?? 1;
@@ -506,15 +538,11 @@ export function applyEffectToCanvas(
ctx.restore();
}
- if (state.washes) {
- ctx.save();
- for (const w of state.washes) {
- ctx.globalCompositeOperation = w.mode;
- ctx.globalAlpha = w.alpha;
- ctx.fillStyle = w.color;
- ctx.fillRect(0, 0, W, H);
- }
- ctx.restore();
+ for (const w of state.washes ?? []) {
+ over(w.mode, w.alpha, (c) => {
+ c.fillStyle = w.color;
+ c.fillRect(0, 0, W, H);
+ });
}
if (state.leak) {
@@ -525,17 +553,15 @@ export function applyEffectToCanvas(
g.addColorStop(0.38, "rgba(255,150,60,0.5)");
g.addColorStop(0.72, "rgba(255,120,40,0)");
g.addColorStop(1, "rgba(255,120,40,0)");
- ctx.save();
- ctx.globalAlpha = alpha;
- ctx.globalCompositeOperation = "screen";
- ctx.fillStyle = g;
- ctx.fillRect(0, 0, W, H);
+ const fill = (style: CanvasGradient) => (c: CanvasRenderingContext2D) => {
+ c.fillStyle = style;
+ c.fillRect(0, 0, W, H);
+ };
// The screen pass lights the darks and dies on a bright frame; a plain
// pass at a fraction of the alpha tints the brights, so the bloom reads
// on footage of any brightness.
- ctx.globalAlpha = alpha * LEAK_TINT;
- ctx.globalCompositeOperation = "source-over";
- ctx.fillRect(0, 0, W, H);
+ over("screen", alpha, fill(g));
+ over("source-over", alpha * LEAK_TINT, fill(g));
// The streak bands, each a linear gradient along its own tilt, blended
// the same two ways as the bloom.
for (const s of state.leak.streaks) {
@@ -553,27 +579,19 @@ export function applyEffectToCanvas(
sg.addColorStop(cl(s.p - 2 * s.w), "rgba(255,200,140,0)");
sg.addColorStop(cl(s.p), "rgba(255,200,140,0.9)");
sg.addColorStop(cl(s.p + 2 * s.w), "rgba(255,200,140,0)");
- ctx.fillStyle = sg;
- ctx.globalAlpha = s.alpha;
- ctx.globalCompositeOperation = "screen";
- ctx.fillRect(0, 0, W, H);
- ctx.globalAlpha = s.alpha * LEAK_TINT;
- ctx.globalCompositeOperation = "source-over";
- ctx.fillRect(0, 0, W, H);
+ over("screen", s.alpha, fill(sg));
+ over("source-over", s.alpha * LEAK_TINT, fill(sg));
}
- ctx.restore();
}
if (state.grain) {
const tile = grainTileFor(Math.floor(tLocal * 30));
if (tile) {
- ctx.save();
- ctx.globalAlpha = state.grain;
- ctx.globalCompositeOperation = "overlay";
- for (let y = 0; y < H; y += 256) {
- for (let x = 0; x < W; x += 256) ctx.drawImage(tile, x, y);
- }
- ctx.restore();
+ over("overlay", state.grain, (c) => {
+ for (let y = 0; y < H; y += 256) {
+ for (let x = 0; x < W; x += 256) c.drawImage(tile, x, y);
+ }
+ });
}
}
@@ -588,19 +606,63 @@ export function applyEffectToCanvas(
);
g.addColorStop(0, "rgba(0,0,0,0)");
g.addColorStop(1, `rgba(0,0,0,${Math.min(0.85, state.vignette)})`);
- ctx.save();
- ctx.fillStyle = g;
- ctx.fillRect(0, 0, W, H);
- ctx.restore();
+ over("source-over", 1, (c) => {
+ c.fillStyle = g;
+ c.fillRect(0, 0, W, H);
+ });
}
if (state.flash) {
- ctx.save();
- ctx.globalAlpha = state.flash;
- ctx.fillStyle = "#FFFFFF";
- ctx.fillRect(0, 0, W, H);
- ctx.restore();
+ over("source-over", state.flash, (c) => {
+ c.fillStyle = "#FFFFFF";
+ c.fillRect(0, 0, W, H);
+ });
+ }
+}
+
+/**
+ * A clip's effects over its picture, in order, at `tLocal` on the effects'
+ * clock. A run of effects that are a canvas filter and nothing else — a
+ * negative, a color cycle, a blur — draws once with the filters joined, and
+ * paint lands only where the picture is, so a still's clear corners stay
+ * clear.
+ */
+export function applyClipEffects(
+ canvas: HTMLCanvasElement | OffscreenCanvas,
+ scratch: HTMLCanvasElement | OffscreenCanvas,
+ effects: readonly ClipEffect[],
+ tLocal: number,
+ grainTileFor: (tick: number) => CanvasImageSource | null
+): void {
+ let filters: string[] = [];
+ const flush = () => {
+ if (filters.length > 0) redrawFiltered(canvas, scratch, filters.join(" "));
+ filters = [];
+ };
+ for (const e of effects) {
+ const { cssFilter, ...passes } = effectPreviewState(e.effect, e.amount, tLocal);
+ if (Object.values(passes).every((v) => v === undefined)) {
+ if (cssFilter) filters.push(cssFilter);
+ continue;
+ }
+ flush();
+ applyEffectToCanvas(canvas, scratch, e.effect, e.amount, tLocal, grainTileFor, undefined, undefined, undefined, "keep");
}
+ flush();
+}
+
+/** The canvas redrawn through `filter`, alpha and all. */
+function redrawFiltered(canvas: HTMLCanvasElement | OffscreenCanvas, scratch: HTMLCanvasElement | OffscreenCanvas, filter: string) {
+ const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
+ const sctx = scratch.getContext("2d") as CanvasRenderingContext2D | null;
+ if (!ctx || !sctx) return;
+ sctx.clearRect(0, 0, canvas.width, canvas.height);
+ sctx.drawImage(canvas, 0, 0);
+ ctx.save();
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ ctx.filter = filter;
+ ctx.drawImage(scratch, 0, 0);
+ ctx.restore();
}
/* ---------------------------------------------------------------- ffmpeg */
diff --git a/site/packages/effects-kit/src/keys.ts b/site/packages/effects-kit/src/keys.ts
index 5b79f9c4..fa02b3d4 100644
--- a/site/packages/effects-kit/src/keys.ts
+++ b/site/packages/effects-kit/src/keys.ts
@@ -122,12 +122,10 @@ export function sortedKeys<K extends { t: number }>(keys: K[]): K[] {
return [...keys].sort((a, b) => a.t - b.t);
}
-function inOrder(keys: { t: number }[]): boolean {
- for (let i = 1; i < keys.length; i++) {
- if (keys[i].t < keys[i - 1].t) return false;
- }
- return true;
-}
+/** Each stored track in play order, sorted once: a track is replaced on
+ * every edit and never changed in place, so a tracked mask's key per frame
+ * costs one sort when it lands and nothing on the frames that read it. */
+const playOrder = new WeakMap<object, { t: number }[]>();
/**
* The interpolation core every key track shares: find the surrounding keys,
@@ -139,10 +137,11 @@ export function lerpKeys<K extends { t: number; ease?: EaseId }>(
tLocal: number,
mix: (a: K, b: K, p: number) => K
): K {
- // Stored tracks are already in order, and a tracked mask carries a key per
- // frame: reading them in place and halving the search keeps a playback
- // frame free of a copy and a sort.
- const ks = inOrder(keys) ? keys : sortedKeys(keys);
+ let ks = playOrder.get(keys) as K[] | undefined;
+ if (!ks) {
+ ks = sortedKeys(keys);
+ playOrder.set(keys, ks);
+ }
if (tLocal <= ks[0].t) return ks[0];
const last = ks[ks.length - 1];
if (tLocal >= last.t) return last;
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index b3afbc56..67bf64a3 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -189,12 +189,12 @@ export const INSPECTOR_TOOLS = [
{
name: "track_motion",
description:
- `Track a target through a video clip's footage, frame by frame, with on-device landmark models, and drive an item with the path. Targets: ${TRACK_TARGETS.map((t) => `${t.id} (${t.covers})`).join("; ")}. use "mask" gives the item a pen mask whose outline follows the target on every frame, keyed for you — a filtered copy of the shot that shows only between the hands, a blur only on a face; it replaces the item's mask, and one masked item carries the whole span, so never split a clip into pieces to move a mask. use "follow" keys the item's pose so it rides the target — a sticker on a head, a label beside a hand — keeping where it sits relative to the target at the start (follow_mode says whether it also scales and turns with it); it replaces the item's pose keys. use "read" changes nothing and returns when the target is in view and where it goes. The source is the video clip whose picture is read: a clip is its own source unless source_id names another; an element defaults to the video clip under it. Runs in the editor tab, about a second per second of footage.`,
+ `Track a target through a video clip's footage, frame by frame, with on-device landmark models, and drive an item with the path. Targets: ${TRACK_TARGETS.map((t) => `${t.id} (${t.covers})`).join("; ")}. use "mask" gives the item a pen mask whose outline follows the target on every frame, keyed for you — a filtered copy of the shot that shows only between the hands, a blur only on a face; it replaces the item's mask, and one masked item carries the whole span, so never split a clip into pieces to move a mask. use "follow" keys the item's pose so it rides the target — a sticker on a head, a label beside a hand — keeping where it sits relative to the target at the start (follow_mode says whether it also scales and turns with it); it replaces the item's pose keys. use "read" changes nothing and returns when the target is in view and where it goes. The source is the video clip whose picture is read: a mask or read on a video clip reads its own footage; a follow, an element, or an image clip reads the video clip under it; source_id names another. A clip never follows its own footage. Runs in the editor tab, about a second per second of footage.`,
inputSchema: obj({
id: str("The item to drive: a video clip id or an overlay element id"),
target: { type: "string", enum: TRACK_TARGET_IDS, description: "What to track" },
use: { type: "string", enum: TRACK_USES, description: "mask: trace the target as a keyed pen mask; follow: key the item's pose to ride it; read: report only" },
- source_id: str("Video clip whose footage is tracked; default: the item itself when it is a clip, else the clip under the element"),
+ source_id: str("Video clip whose footage is tracked; default: a video clip's own footage for mask and read, else the video clip under the item"),
from: num("Timeline seconds to start tracking; default where the item and the source first overlap"),
to: num("Timeline seconds to stop tracking; default where they stop overlapping"),
follow_mode: { type: "string", enum: FOLLOW_MODES, description: "follow only: move = position; move_scale = position and size; move_scale_turn = position, size and turn (default move_scale)" },
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index a5935ed4..e1b2108f 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -30,7 +30,6 @@ import {
ALL_EFFECT_IDS,
CLIP_EFFECT_IDS,
type ClipEffect,
- type ClipEffectId,
applyLutToImageData,
autoGradeFromImageData,
buildClipLut,
@@ -2972,9 +2971,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// Each entry names a clip effect once, with an optional amount in range.
const effects: ClipEffect[] = [];
for (const e of input.effects as { effect?: unknown; amount?: unknown }[]) {
- if (!(CLIP_EFFECT_IDS as unknown[]).includes(e?.effect))
- throw new ToolError(`effect must be one of ${CLIP_EFFECT_IDS.join(", ")}.`);
- const effect = e.effect as ClipEffectId;
+ const effect = CLIP_EFFECT_IDS.find((id) => id === e?.effect);
+ if (!effect) throw new ToolError(`effect must be one of ${CLIP_EFFECT_IDS.join(", ")}.`);
if (effects.some((x) => x.effect === effect)) throw new ToolError(`${effect} is listed twice; a clip wears an effect once.`);
effects.push(isNum(e.amount) ? { effect, amount: Math.max(0.05, Math.min(1, e.amount)) } : { effect });
}
diff --git a/site/src/cut/lib/composite.ts b/site/src/cut/lib/composite.ts
index d771e8a1..f6cb6a57 100644
--- a/site/src/cut/lib/composite.ts
+++ b/site/src/cut/lib/composite.ts
@@ -19,7 +19,7 @@
* it was reached by playing there or by rendering the 135th frame.
*/
-import { applyDetail, applyEffectToCanvas, applyLutToImageData, applyMaskToCanvas, detailActive, grainTile, hexToHlgHex, lookCssFilter, lookPost, maskComposite, paintStrokeInk, removalActive, retimeOf, type ClipColorRecipe, type OutputSpace } from "@donkeycut/effects-kit";
+import { applyClipEffects, applyDetail, applyLutToImageData, applyMaskToCanvas, detailActive, grainTile, hexToHlgHex, lookCssFilter, lookPost, maskComposite, paintStrokeInk, removalActive, type ClipColorRecipe, type OutputSpace } from "@donkeycut/effects-kit";
import { cameraCardLayout, traceCardPath, type CardLayout } from "./cameraCard";
import { applyDetailGpu } from "./detailGpu";
import { applyLutGpu } from "./gradeGpu";
@@ -85,6 +85,10 @@ const REC709_SOURCE: RecipeSource = { profile: "rec709" };
/** How many clips a compositor remembers the last drawn LUT for. */
const LAST_LUT_MAX = 64;
+/** Whether the clip wears its effects on its framed picture. A keyed clip
+ * wears them on its source picture instead (see `gradedSource`). */
+const framedEffects = (clip: VideoClip) => !!clip.effects?.length && !removalActive(clip.removal);
+
export class FrameCompositor {
/** Scratch buffers, kept for the compositor's life: the grade pass, the
* look's self-copy passes, and the vignette gradient. Allocating these per
@@ -344,9 +348,10 @@ export class FrameCompositor {
* spatial pass, then the look's color pass as a canvas filter, in the order
* the export's chain runs. Source alpha rides through every pass, so
* transparent stills keep their transparency. The look's post passes
- * (vignette, grain, glow…) draw over the composited layer instead. The
- * clip's own effects run last, in the picture's pixels, so its mask and
- * pose carry them with it.
+ * (vignette, grain, glow…) draw over the composited layer instead. A
+ * keyed clip wears its own effects here, in the picture's pixels, the way
+ * the export's keyed pieces bake them; every other clip wears them framed,
+ * in the mask and pose pass.
*/
private gradedSource(
frame: Extract<Frame, { kind: "ready" }>,
@@ -357,10 +362,10 @@ export class FrameCompositor {
const compiled = this.lutFor(clip, recipe);
const detail = detailActive(clip?.grade) ? clip!.grade! : null;
const lookCss = lookCssFilter(clip?.look, clip?.lookAmount);
- const fx = clip?.effects?.length ? clip : null;
+ const fx = clip?.effects?.length && removalActive(clip.removal) ? clip : null;
if (!compiled && !detail && !lookCss && !fx) return frame.image;
const graded = this.colorPasses(frame, compiled, detail, lookCss);
- return fx ? this.clipEffects(graded, frame.width, frame.height, fx, at) : graded;
+ return fx ? (this.clipEffects(graded, frame.width, frame.height, fx, at) as CanvasImageSource) : graded;
}
/** The LUT, the spatial pass and the look's color pass, in that order. */
@@ -417,24 +422,28 @@ export class FrameCompositor {
return scratch;
}
- /** The clip's effects over its picture, in order, on the clip's own clock:
- * a color cycle turns from the clip's first frame and a flash pops there. */
- private clipEffects(picture: CanvasImageSource, w: number, h: number, clip: VideoClip, at: number): CanvasImageSource {
+ /** The clip's effects over `picture`'s `src` rect, in order, on the
+ * effects' own clock: a color cycle turns from the clip's first frame (or
+ * from where a split left it) and a flash pops there. */
+ private clipEffects(
+ picture: CanvasImageSource,
+ w: number,
+ h: number,
+ clip: VideoClip,
+ at: number,
+ src = { x: 0, y: 0, w, h }
+ ): Surface {
const { surface: canvas } = this.scratch("clipFxCanvas", w, h);
const { surface: scratch } = this.scratch("clipFxScratch", w, h);
const ctx = canvas.getContext("2d") as Ctx | null;
- if (!ctx) return picture;
+ if (!ctx) return canvas;
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.globalAlpha = 1;
ctx.globalCompositeOperation = "source-over";
ctx.clearRect(0, 0, w, h);
- ctx.drawImage(picture, 0, 0, w, h);
- const tLocal = at - clip.start;
- const len = retimeOf(clip).len;
- for (const e of clip.effects!) {
- applyEffectToCanvas(canvas, scratch, e.effect, e.amount, tLocal, grainTile, undefined, undefined, len);
- }
- return canvas as CanvasImageSource;
+ ctx.drawImage(picture, src.x, src.y, src.w, src.h, 0, 0, w, h);
+ applyClipEffects(canvas, scratch, clip.effects ?? [], at - clip.start + (clip.effectsFrom ?? 0), grainTile);
+ return canvas;
}
/**
@@ -709,6 +718,7 @@ export class FrameCompositor {
(clipPosed(clip) ||
// A camera card casts its own shadow; the box style stands aside.
(!!clip.boxStyle?.shadow && !clip.card) ||
+ framedEffects(clip) ||
!!(clip.mask && (clip.mask.kind !== "subject" || this.subjectMatteProvider)))
);
}
@@ -752,12 +762,29 @@ export class FrameCompositor {
kf: undefined,
rotation: undefined,
opacity: undefined,
+ ...(framedEffects(clip) ? { effects: undefined } : {}),
...(clip.boxStyle ? { boxStyle: { ...clip.boxStyle, shadow: undefined } } : {}),
});
} finally {
this.canvas = prev;
}
const rect = rectOf(clip);
+
+ // The clip's effects treat its framed picture, look and all, ahead of
+ // the mask and pose — the export's chain runs them on the framed segment
+ // the same way. Example: a vignette darkens the corners of the clip's box,
+ // wherever the source's own corners were cropped to.
+ if (framedEffects(clip)) {
+ const bx = Math.max(0, Math.floor(rect.x * W));
+ const by = Math.max(0, Math.floor(rect.y * H));
+ const bw = Math.min(W, Math.ceil((rect.x + rect.w) * W)) - bx;
+ const bh = Math.min(H, Math.ceil((rect.y + rect.h) * H)) - by;
+ if (bw > 0 && bh > 0) {
+ const worn = this.clipEffects(layer as CanvasImageSource, bw, bh, clip, at, { x: bx, y: by, w: bw, h: bh });
+ lctx.clearRect(bx, by, bw, bh);
+ lctx.drawImage(worn as CanvasImageSource, bx, by);
+ }
+ }
const { surface: cover } = this.scratch("maskScratch", W, H);
if (mask && mask.kind !== "subject") {
applyMaskToCanvas(
diff --git a/site/src/cut/lib/devHooks.ts b/site/src/cut/lib/devHooks.ts
index 8005527a..2149e2be 100644
--- a/site/src/cut/lib/devHooks.ts
+++ b/site/src/cut/lib/devHooks.ts
@@ -16,8 +16,6 @@ import { awaitingFrame, startTrace, stopTrace, traceReport } from "./perfTrace";
import { markSignedBatch } from "./mediaLinks";
import { prefetchCloudMedia } from "./mediaSync";
import { playheadAt } from "./playhead";
-import { bindCutTracking } from "./chatRuntime";
-import { trackFootage } from "./tracking";
import { projectDuration, useEditor } from "./store";
export function installDevHooks(): void {
@@ -54,10 +52,6 @@ export function installDevHooks(): void {
// The export eval starts a real export through the store, the way the
// dialog does, and watches the dock's feed for the file.
useExports,
- // The tracking check reads a clip's raw landmarks beside the keys the
- // tracker wrote, to see where the path parts from the picture.
- trackFootage,
- bindCutTracking,
};
// The perf eval arms and reads the frame trace through here. Off until
// `start()` is called, so an ordinary dev session records nothing.
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index 0ae6265d..40ce8e55 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -30,7 +30,7 @@ import { drawBlock } from "./blockSource";
import { createRasterCanvas, rasterCanvasToPng } from "./raster";
import { clipLen, clipSpeed, getClipSpans, openedTimeline, overlayLayers, projectDuration, spanSequence, useEditor } from "./store";
import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
-import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk, retimeOf, type SpeedNode } from "@donkeycut/effects-kit";
+import { isMaskAnimated, isOverlayAnimated, matteLumaToAlpha, normalizeGrade, paintMaskLuma, paintStrokeInk, retimeOf, type Mask, type SpeedNode } from "@donkeycut/effects-kit";
import { renderElementFrames, renderElementPng, renderStillPng } from "./textRender";
import { clipCovers, clipKeyed, clipPosed, clipPoseAt, clipZoom, contentRect, frameOf, isStickerOverlay, isTextOverlay, laneOf, overlayAnimStyle, projectBackground, rectOf, regionPx, removalActive, shadowInk, subjectMasked, parkedTimeline } from "./types";
import { liveReader } from "./liveReader";
@@ -484,6 +484,12 @@ interface SpecMask {
* server re-stamps the output fps over it. */
const MASK_SAMPLE_FPS = 15;
+/** The rate a moving mask samples at: the matte cadence, or the export's own
+ * frame rate for a mask keyed more densely than that. A tracked outline
+ * moves on every frame, and a coarser sample would step behind the hand. */
+const maskSampleFps = (mask: Mask | undefined, dur: number, fps: number) =>
+ (mask?.kf?.length ?? 0) / Math.max(dur, 1e-3) > MASK_SAMPLE_FPS ? Math.max(fps, MASK_SAMPLE_FPS) : MASK_SAMPLE_FPS;
+
/** Paint a clip's mask coverage for its export segment: one luma PNG for a
* resting mask, a 15fps sampled sequence for a keyframed one. Keyframed
* opacity folds into the luma — a clip fading under its pose track exports
@@ -496,6 +502,7 @@ async function renderClipMaskPictures(
W: number,
H: number,
dur: number,
+ fps: number,
tag: string,
pngs: ExportPayload["pngs"]
): Promise<SpecMask | undefined> {
@@ -548,8 +555,9 @@ async function renderClipMaskPictures(
pngs.push({ name, blob: await blobAt(0) });
return { file: name };
}
- const step = 1 / MASK_SAMPLE_FPS;
- const n = Math.max(1, Math.round(dur * MASK_SAMPLE_FPS));
+ const rate = maskSampleFps(m, dur, fps);
+ const step = 1 / rate;
+ const n = Math.max(1, Math.round(dur * rate));
const frames: { file: string; duration: number }[] = [];
for (let i = 0; i < n; i++) {
const name = `${tag}_f${i}.png`;
@@ -666,6 +674,7 @@ async function renderClipShadowPictures(
W: number,
H: number,
dur: number,
+ fps: number,
tag: string,
pngs: ExportPayload["pngs"],
assets: MediaAsset[]
@@ -757,8 +766,9 @@ async function renderClipShadowPictures(
pngs.push({ name, blob: await blobAt(0) });
return { file: name };
}
- const step = 1 / MASK_SAMPLE_FPS;
- const n = Math.max(1, Math.round(dur * MASK_SAMPLE_FPS));
+ const rate = maskSampleFps(mask, dur, fps);
+ const step = 1 / rate;
+ const n = Math.max(1, Math.round(dur * rate));
const frames: { file: string; duration: number }[] = [];
for (let i = 0; i < n; i++) {
const name = `${tag}_f${i}.png`;
@@ -951,6 +961,7 @@ export async function buildExportPayload(
look: sp.clip.look,
lookAmount: sp.clip.lookAmount,
effects: sp.clip.effects,
+ effectsFrom: sp.clip.effectsFrom,
hidden: sp.clip.hidden,
// A still: the server loops the image for the clip's length instead of
// trimming a source span.
@@ -989,6 +1000,7 @@ export async function buildExportPayload(
settings.width,
settings.height,
dur,
+ settings.fps,
`mask_c${i}`,
pngs
);
@@ -1018,6 +1030,7 @@ export async function buildExportPayload(
settings.width,
settings.height,
dur,
+ settings.fps,
`shadow_c${i}`,
pngs,
doc.assets
@@ -1050,13 +1063,16 @@ export async function buildExportPayload(
2160,
Math.round(Math.min(settings.width, settings.height) * clipZoom(c))
),
- bakeLook: false,
+ // Effects bake into the keyed layer after the look, as the preview
+ // wears them, so the look bakes in with them.
+ bakeLook: !!c.effects?.length,
});
if (pieces) {
pngs.push({ name: `removal_c${i}_rgb.mp4`, blob: pieces.rgb });
pngs.push({ name: `removal_c${i}_a.mp4`, blob: pieces.alpha });
clipEntries[i].removal = { rgb: `removal_c${i}_rgb.mp4`, alpha: `removal_c${i}_a.mp4` };
clipEntries[i].grade = undefined;
+ if (c.effects?.length) clipEntries[i].look = undefined;
clipEntries[i].effects = undefined;
}
}
@@ -1179,6 +1195,7 @@ export async function buildExportPayload(
look: c.look,
lookAmount: c.lookAmount,
effects: c.effects,
+ effectsFrom: c.effectsFrom,
mask: undefined as SpecMask | undefined,
shadow: undefined as SpecMask | undefined,
kf: posed(c).kf,
@@ -1208,6 +1225,7 @@ export async function buildExportPayload(
settings.width,
settings.height,
olen,
+ settings.fps,
`mask_ov${i}`,
pngs
);
@@ -1233,6 +1251,7 @@ export async function buildExportPayload(
settings.width,
settings.height,
olen,
+ settings.fps,
`shadow_ov${i}`,
pngs,
doc.assets
diff --git a/site/src/cut/lib/itemKinds.test.ts b/site/src/cut/lib/itemKinds.test.ts
index 06815871..9c3b2a4f 100644
--- a/site/src/cut/lib/itemKinds.test.ts
+++ b/site/src/cut/lib/itemKinds.test.ts
@@ -5,7 +5,7 @@ import { ITEM_KINDS, ITEM_KIND_IDS, clipboardItemAcross, clipboardItemAssetIds,
import { payloadAssets } from "./cutClipboard";
import { setPlayhead } from "./playhead";
import { assetIdsInUse, useEditor } from "./store";
-import { emptySubtitles, type MediaAsset, type Selection } from "./types";
+import { clipPoseAt, emptySubtitles, type MediaAsset, type Selection } from "./types";
/**
* Every kind of item the timeline can select is one entry in the item-kind
@@ -167,6 +167,34 @@ describe("splitting keyed items", () => {
expect(right.mask!.kf![0]).toMatchObject({ t: 0, x: 0.1 });
});
+ test("a clip's effects carry on through the cut", () => {
+ const clip = { id: "e1", assetId: "v0", start: 10, in: 0, out: 4, track: 0, effects: [{ effect: "huecycle" }], effectsFrom: 0.5 };
+ const [left, right] = ITEM_KINDS.clip.split!(clip as never, 12);
+ expect(left.effectsFrom).toBe(0.5);
+ expect(right.effectsFrom).toBeCloseTo(2.5, 9);
+ });
+
+ test("an eased move keeps its curve on both sides of the cut", () => {
+ const clip = {
+ id: "e2", assetId: "v0", start: 0, in: 0, out: 4, track: 1,
+ kf: [{ ...poseKey(0, 0), ease: "sine.inOut" as const }, poseKey(4, 1)],
+ };
+ const whole = (t: number) => clipPoseAt(clip as never, t).x;
+ const [left, right] = ITEM_KINDS.clip.split!(clip as never, 1);
+ for (const t of [0.25, 0.5, 0.9]) expect(clipPoseAt(left, t).x).toBeCloseTo(whole(t), 2);
+ for (const t of [0.2, 1, 2.5]) expect(clipPoseAt(right, t).x).toBeCloseTo(whole(1 + t), 2);
+ });
+
+ test("a clip's mask keys stay on its footage through a head trim and a speed change", () => {
+ const key = (t: number) => ({ t, x: 0, y: 0, w: 0.5, h: 0.5, rotation: 0, feather: 0 });
+ const clip = { id: "m1", assetId: "v0", start: 0, in: 0, out: 4, track: 0, muted: false, mask: { kind: "rect" as const, kf: [key(1), key(3)] } };
+ useEditor.setState({ clips: [clip] });
+ useEditor.getState().updateClipTransient("m1", { start: 0.5, in: 0.5 });
+ expect(useEditor.getState().clips[0].mask!.kf!.map((k) => k.t)).toEqual([0.5, 2.5]);
+ useEditor.getState().setClipSpeed("m1", 2);
+ expect(useEditor.getState().clips[0].mask!.kf!.map((k) => k.t)).toEqual([0.25, 1.25]);
+ });
+
test("an element's press stays at its moment on the timeline", () => {
const o = {
id: "h1", kind: "shape", start: 0, end: 4, x: 0.5, y: 0.5,
diff --git a/site/src/cut/lib/itemKinds.ts b/site/src/cut/lib/itemKinds.ts
index 1a9f769e..f675cacf 100644
--- a/site/src/cut/lib/itemKinds.ts
+++ b/site/src/cut/lib/itemKinds.ts
@@ -13,7 +13,7 @@
// compile until it has an entry, and `itemKinds.test.ts` walks the table so
// every entry proves it copies and pastes.
-import { KEY_EPSILON, maskKeyAt, poseAt, retimeOf, shiftCamera, type Mask, type OverlayAnim, type OverlayKey, type OverlayPose } from "@donkeycut/effects-kit";
+import { KEY_EPSILON, maskKeyAt, poseAt, retimeOf, shiftCamera, sortedKeys, type EaseId, type Mask, type OverlayAnim, type OverlayKey, type OverlayPose } from "@donkeycut/effects-kit";
import { splitEmphasis, withEmphasis } from "./captionEmphasis";
import {
clipPoseAt,
@@ -81,16 +81,40 @@ export interface ItemKindDef<K extends ItemKind> {
const deep = <T>(v: T): T => (v === undefined ? v : (JSON.parse(JSON.stringify(v)) as T));
+/** How finely an eased move cut in two is redrawn as straight steps. */
+const SPLIT_EASE_STEP = 1 / 30;
+
/** A key track cut at `cut` seconds into its item. Each half keeps the keys on
* its side, closed by a key holding the value at the cut, and the right
* half's keys count from its own start — so both halves play what the whole
- * did. A tracked mask keys every frame, and a split copy has to hold it. */
-function splitKeys<K extends { t: number }>(keys: K[] | undefined, cut: number, valueAt: (t: number) => K): [K[] | undefined, K[] | undefined] {
+ * did. A tracked mask keys every frame, and a split copy has to hold it. An
+ * eased move the cut lands in has no named curve for either part, so it is
+ * redrawn as straight steps along the original curve. */
+function splitKeys<K extends { t: number; ease?: EaseId }>(
+ keys: K[] | undefined,
+ cut: number,
+ valueAt: (t: number) => K
+): [K[] | undefined, K[] | undefined] {
if (!keys || keys.length === 0) return [keys, keys];
- const edge = valueAt(cut);
+ const sorted = sortedKeys(keys);
+
+ // The eased move under the cut, as plain keys every step along its curve.
+ // Example: a sine.inOut move from 0s to 4s cut at 1s plays the same curve
+ // through 0–1s on the left and 1–4s on the right.
+ const i = sorted.findIndex((k, n) => k.t < cut - KEY_EPSILON && sorted[n + 1]?.t > cut + KEY_EPSILON);
+ const eased = i >= 0 && sorted[i].ease ? sorted[i] : null;
+ const steps = eased ? Math.ceil((sorted[i + 1].t - eased.t) / SPLIT_EASE_STEP) : 0;
+ const along = Array.from({ length: Math.max(0, steps - 1) }, (_, n) => {
+ const t = eased!.t + ((n + 1) * (sorted[i + 1].t - eased!.t)) / steps;
+ return { ...valueAt(t), t, ease: undefined };
+ });
+ const plain = sorted.map((k) => (k === eased ? { ...k, ease: undefined } : k));
+ const all = [...plain, ...along].sort((a, b) => a.t - b.t);
+
+ const edge = { ...valueAt(cut), ease: undefined };
return [
- [...keys.filter((k) => k.t < cut - KEY_EPSILON), { ...edge, t: cut }],
- [{ ...edge, t: 0 }, ...keys.filter((k) => k.t > cut + KEY_EPSILON).map((k) => ({ ...k, t: k.t - cut }))],
+ [...all.filter((k) => k.t < cut - KEY_EPSILON), { ...edge, t: cut }],
+ [{ ...edge, t: 0 }, ...all.filter((k) => k.t > cut + KEY_EPSILON).map((k) => ({ ...k, t: k.t - cut }))],
];
}
@@ -144,9 +168,11 @@ export const ITEM_KINDS: { [K in ItemKind]: ItemKindDef<K> } = {
const [keysL, keysR] = splitTracks(c, at - c.start, (t) => clipPoseAt(c, t));
// A split edit stays on the outer edges: the new cut between the halves
// carries none.
+ // The right half's effects carry on from where the cut leaves them.
+ const effectsR = c.effects?.length ? { effectsFrom: (c.effectsFrom ?? 0) + (at - c.start) } : {};
return [
{ ...left, ...keysL, transition: undefined, transitionStyle: undefined, animOut: undefined, audioTail: undefined },
- { ...right, ...keysR, animIn: undefined, audioLead: undefined },
+ { ...right, ...keysR, ...effectsR, animIn: undefined, audioLead: undefined },
];
},
clone: deep,
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index fcfce8f3..309e9b15 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -145,6 +145,7 @@ const clipTreatment = (c: VideoClip) => ({
...(c.grade ? { grade: c.grade } : {}),
...(c.mask ? { mask: c.mask } : {}),
...(c.effects?.length ? { effects: c.effects } : {}),
+ ...(c.effects?.length && c.effectsFrom ? { effectsFrom: c.effectsFrom } : {}),
...(c.boxStyle ? { boxStyle: c.boxStyle } : {}),
...(c.kf?.length ? { kf: c.kf } : {}),
...(c.hidden ? { hidden: true } : {}),
@@ -185,6 +186,7 @@ const templateTreatment = (l: TemplateLayer) => ({
...(l.grade ? { grade: l.grade } : {}),
...(l.mask ? { mask: l.mask } : {}),
...(l.effects?.length ? { effects: l.effects } : {}),
+ ...(l.effects?.length && l.effectsFrom ? { effectsFrom: l.effectsFrom } : {}),
...(l.boxStyle ? { boxStyle: l.boxStyle } : {}),
...(l.kf?.length ? { kf: l.kf } : {}),
...(l.hidden ? { hidden: true } : {}),
@@ -1303,6 +1305,23 @@ function overlaysClearOf(overlays: Overlay[], ids: Set<string>): Overlay[] {
return settleCameras(overlays, cleared);
}
+/**
+ * A clip with `patch` applied. A mask keyed on the clip's own picture stays
+ * on its footage when the patch retimes the clip: a key at source second 2
+ * still lands on source second 2 after a head trim or a speed change, so a
+ * tracked outline keeps tracing the hand it was drawn around.
+ */
+function patchClip(c: VideoClip, patch: Partial<VideoClip>): VideoClip {
+ const next = { ...c, ...patch };
+ const kf = next.mask?.kf;
+ if (!kf?.length || "mask" in patch) return next;
+ const was = retimeOf(c);
+ const now = retimeOf(next);
+ if (was.key === now.key) return next;
+ const moved = kf.map((k) => ({ ...k, t: now.tAt(was.srcAt(k.t)) })).sort((a, b) => a.t - b.t);
+ return { ...next, mask: { ...next.mask!, kf: moved } };
+}
+
/** Apply a source trim or speed change with the shared same-row collision rule. */
function resizeClipFootprint(clip: VideoClip, patch: Partial<VideoClip>) {
useEditor.getState().pushHistory();
@@ -3695,7 +3714,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
updateClipTransient: (id, patch) =>
set((s) => ({
- clips: s.clips.map((c) => (c.id === id ? { ...c, ...patch } : c)),
+ clips: s.clips.map((c) => (c.id === id ? patchClip(c, patch) : c)),
})),
updateClipsTransient: (patches) =>
@@ -3704,7 +3723,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
return {
clips: s.clips.map((c) => {
const patch = byId.get(c.id);
- return patch ? { ...c, ...patch } : c;
+ return patch ? patchClip(c, patch) : c;
}),
};
}),
@@ -3716,7 +3735,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
const byId = new Map(patches.clips.map((p) => [p.id, p.patch]));
out.clips = s.clips.map((c) => {
const patch = byId.get(c.id);
- return patch ? { ...c, ...patch } : c;
+ return patch ? patchClip(c, patch) : c;
});
}
if (patches.audioClips?.length) {
diff --git a/site/src/cut/lib/trackKeys.test.ts b/site/src/cut/lib/trackKeys.test.ts
index 0efc12ce..c69fd642 100644
--- a/site/src/cut/lib/trackKeys.test.ts
+++ b/site/src/cut/lib/trackKeys.test.ts
@@ -116,7 +116,6 @@ describe("following a target", () => {
axis: [{ x: 0.5, y: 0.5 - 0.2 * (9 / 16) }, { x: 0.5, y: 0.5 + 0.2 * (9 / 16) }],
};
const keys = followKeys(
- { kind: "clip", clip: clip() },
"9:16",
t,
[at(0.5, 0.5), turned],
@@ -131,7 +130,7 @@ describe("following a target", () => {
});
test("moments without the target write no key", () => {
- const keys = followKeys({ kind: "clip", clip: clip() }, "9:16", times(3), [at(0.5, 0.5), null, at(0.6, 0.5)], { x: 0.5, y: 0.5, scale: 1, rotation: 0, opacity: 1 }, "move", 0);
+ const keys = followKeys("9:16", times(3), [at(0.5, 0.5), null, at(0.6, 0.5)], { x: 0.5, y: 0.5, scale: 1, rotation: 0, opacity: 1 }, "move", 0);
expect(keys.map((k) => k.x)).toEqual([0.5, 0.6]);
});
});
diff --git a/site/src/cut/lib/trackKeys.ts b/site/src/cut/lib/trackKeys.ts
index 935bf4a0..48b66ade 100644
--- a/site/src/cut/lib/trackKeys.ts
+++ b/site/src/cut/lib/trackKeys.ts
@@ -14,28 +14,19 @@
* surface agree on what a track means.
*/
-import { evalOverlayFrame, type Mask, type MaskKey, type OverlayKey } from "@donkeycut/effects-kit";
+import { evalOverlayFrame, type Mask, type MaskKey, type MaskPoint, type OverlayKey, type OverlayPose } from "@donkeycut/effects-kit";
+import type { Settings } from "@/lib/config/registry";
import { clipCovers, clipZoom, contentRect, frameOf, rectOf, type Aspect, type Overlay, type VideoClip } from "./types";
-export interface Pt {
- x: number;
- y: number;
-}
-
/** One moment of a target: its outline, its center, and two points along
* its own axis, whose angle is the target's turn and whose length its size. */
export interface TrackSample {
- outline: Pt[];
- center: Pt;
- axis: [Pt, Pt];
+ outline: MaskPoint[];
+ center: MaskPoint;
+ axis: [MaskPoint, MaskPoint];
}
-export interface TrackTuning {
- smoothCutoff: number;
- smoothBeta: number;
- bridgeSeconds: number;
- keyTolerance: number;
-}
+type TrackTuning = Pick<Settings["cutTracking"], "smoothCutoff" | "smoothBeta" | "bridgeSeconds">;
/** The derivative's own smoothing cutoff, Hz — the one-euro default. */
const DERIVATIVE_CUTOFF = 1;
@@ -56,7 +47,7 @@ const flatten = (s: TrackSample): number[] => [
function unflatten(v: number[]): TrackSample {
const n = (v.length - 6) / 2;
- const at = (i: number): Pt => ({ x: v[i * 2], y: v[i * 2 + 1] });
+ const at = (i: number): MaskPoint => ({ x: v[i * 2], y: v[i * 2 + 1] });
return {
outline: Array.from({ length: n }, (_, i) => at(i)),
center: at(n),
@@ -141,7 +132,7 @@ export function pictureToFrame(
clip: VideoClip,
picture: { width: number; height: number },
aspect: Aspect
-): (p: Pt) => Pt {
+): (p: MaskPoint) => MaskPoint {
const fr = frameOf(aspect);
const r = rectOf(clip);
const box = { x: r.x * fr.w, y: r.y * fr.h, w: r.w * fr.w, h: r.h * fr.h };
@@ -160,15 +151,13 @@ export function pictureToFrame(
/** An item a track writes onto: a video clip or an overlay element. */
export type TrackItem = { kind: "clip"; clip: VideoClip } | { kind: "overlay"; overlay: Overlay };
-const itemStart = (item: TrackItem) => (item.kind === "clip" ? item.clip.start : item.overlay.start);
-
/**
* A frame point as an offset from the item's mask anchor, in the item's own
* unposed space at `tLocal`: a clip's masks sit before its pose, so its
* region center is the anchor; an element's mask paints under the element's
* transform, so the point is carried back through it.
*/
-function toMaskSpace(item: TrackItem, aspect: Aspect, tLocal: number): (p: Pt) => Pt {
+function toMaskSpace(item: TrackItem, aspect: Aspect, tLocal: number): (p: MaskPoint) => MaskPoint {
if (item.kind === "clip") {
const r = rectOf(item.clip);
const ax = r.x + r.w / 2;
@@ -194,8 +183,8 @@ function toMaskSpace(item: TrackItem, aspect: Aspect, tLocal: number): (p: Pt) =
* A pen mask that traces the target: one key per kept moment, each carrying
* the outline there. Where the target is gone the outline folds to a point at
* its last place, so the masked picture shows nothing until it returns.
- * `times` are timeline seconds; `frame` holds the settled samples already
- * placed in the project frame.
+ * `times` are seconds into the item; `frame` holds the settled samples
+ * already placed in the project frame.
*/
export function trackedMask(
item: TrackItem,
@@ -212,7 +201,7 @@ export function trackedMask(
// Each moment's outline in the item's mask space.
const outlines = frame.map((s, i) => {
- const local = toMaskSpace(item, aspect, times[i] - itemStart(item));
+ const local = toMaskSpace(item, aspect, times[i]);
if (s) last = s.center;
const pts = s ? s.outline : Array.from({ length: corners }, () => last);
return pts.map(local);
@@ -220,7 +209,7 @@ export function trackedMask(
const kept = keepTurns(times, outlines.map((o) => o.flatMap((p) => [p.x, p.y])), tolerance);
const feather = base.feather ?? 0;
const kf: MaskKey[] = kept.map((i) => ({
- t: round4(times[i] - itemStart(item)),
+ t: round4(times[i]),
x: 0,
y: 0,
w: 1,
@@ -243,34 +232,24 @@ export function trackedMask(
};
}
-/** Where an item rests before a follow moves it: its center in frame
- * fractions, its scale, turn and opacity. */
-export interface RestPose {
- x: number;
- y: number;
- scale: number;
- rotation: number;
- opacity: number;
-}
-
/**
* Pose keys that carry an item with the target. The item keeps where it sits
* relative to the target at the first tracked moment — beside a head, over a
* hand — and from there moves with it; `move_scale` also grows and shrinks
* with the target's size, `move_scale_turn` also turns with it. Moments where
- * the target is gone write no key, so the item holds.
+ * the target is gone write no key, so the item holds. `times` are seconds
+ * into the item.
*/
export function followKeys(
- item: TrackItem,
aspect: Aspect,
times: number[],
frame: (TrackSample | null)[],
- rest: RestPose,
+ rest: OverlayPose,
mode: FollowMode,
tolerance: number
): OverlayKey[] {
const fr = frameOf(aspect);
- const px = (p: Pt) => ({ x: p.x * fr.w, y: p.y * fr.h });
+ const px = (p: MaskPoint) => ({ x: p.x * fr.w, y: p.y * fr.h });
const measure = (s: TrackSample) => {
const a = px(s.axis[0]);
const b = px(s.axis[1]);
@@ -299,7 +278,7 @@ export function followKeys(
poses.push({
i,
key: {
- t: round4(times[i] - itemStart(item)),
+ t: round4(times[i]),
x: round4((m.c.x + ox) / fr.w),
y: round4((m.c.y + oy) / fr.h),
scale: round4(rest.scale * grow),
diff --git a/site/src/cut/lib/trackMotion.test.ts b/site/src/cut/lib/trackMotion.test.ts
new file mode 100644
index 00000000..d0bc6bf8
--- /dev/null
+++ b/site/src/cut/lib/trackMotion.test.ts
@@ -0,0 +1,89 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+import { stubModule } from "@/lib/testing/stubModule";
+import type { TrackSample } from "./trackKeys";
+import type { MediaAsset, VideoClip } from "./types";
+
+// The landmarkers need a browser; the stub reports a still square target and
+// records which footage was read. `during` runs while the "read" is in flight.
+const reads: string[] = [];
+let during: () => void = () => {};
+const still: TrackSample = {
+ outline: [
+ { x: 0.4, y: 0.4 },
+ { x: 0.6, y: 0.4 },
+ { x: 0.6, y: 0.6 },
+ { x: 0.4, y: 0.6 },
+ ],
+ center: { x: 0.5, y: 0.5 },
+ axis: [{ x: 0.4, y: 0.5 }, { x: 0.6, y: 0.5 }],
+};
+await stubModule<typeof import("./tracking")>("./tracking", import.meta.url, {
+ trackFootage: async (src, _size, _target, times) => {
+ reads.push(src);
+ during();
+ return times.map(() => still);
+ },
+});
+const { trackMotion } = await import("./trackMotion");
+const { useEditor } = await import("./store");
+
+const video = (id: string, type: MediaAsset["type"] = "video"): MediaAsset => ({
+ id,
+ fileName: `${id}.mp4`,
+ name: id,
+ type,
+ duration: 10,
+ url: `blob:${id}`,
+ width: 1920,
+ height: 1080,
+});
+const clip = (o: Partial<VideoClip> & { id: string; assetId: string }): VideoClip => ({
+ track: 0,
+ start: 0,
+ in: 0,
+ out: 10,
+ muted: false,
+ ...o,
+});
+
+beforeEach(() => {
+ reads.length = 0;
+ during = () => {};
+ useEditor.setState({
+ assets: [video("shot"), video("pip"), video("logo", "image")],
+ clips: [clip({ id: "base", assetId: "shot" }), clip({ id: "top", assetId: "pip", track: 1, start: 2, out: 4 })],
+ overlays: [],
+ aspect: "16:9",
+ });
+});
+
+describe("the footage a track reads", () => {
+ test("a clip follows the footage under it, never its own", async () => {
+ await trackMotion({ id: "top", target: "head", use: "follow" });
+ expect(reads).toEqual(["blob:shot"]);
+ });
+
+ test("a clip cannot be told to follow its own footage", async () => {
+ await expect(trackMotion({ id: "top", target: "head", use: "follow", sourceId: "top" })).rejects.toThrow(/own footage/);
+ });
+
+ test("a mask on a video clip traces its own footage", async () => {
+ await trackMotion({ id: "top", target: "head", use: "mask" });
+ expect(reads).toEqual(["blob:pip"]);
+ });
+
+ test("a mask on an image clip traces the footage under it", async () => {
+ useEditor.setState({ clips: [clip({ id: "base", assetId: "shot" }), clip({ id: "top", assetId: "logo", track: 1, start: 2, out: 2 })] });
+ await trackMotion({ id: "top", target: "head", use: "mask" });
+ expect(reads).toEqual(["blob:shot"]);
+ });
+});
+
+describe("keys land on the item's clock", () => {
+ test("moving the item during the read leaves the keys where the footage put them", async () => {
+ during = () => useEditor.getState().updateClipTransient("top", { start: 3 });
+ await trackMotion({ id: "top", target: "head", use: "mask", sourceId: "base" });
+ const kf = useEditor.getState().clips.find((c) => c.id === "top")!.mask!.kf!;
+ expect(kf[0].t).toBeCloseTo(0, 3);
+ });
+});
diff --git a/site/src/cut/lib/trackMotion.ts b/site/src/cut/lib/trackMotion.ts
index dfcf5ea1..09d127a8 100644
--- a/site/src/cut/lib/trackMotion.ts
+++ b/site/src/cut/lib/trackMotion.ts
@@ -5,13 +5,13 @@
* one path the chat's track_motion tool and the inspector's Track control
* both run.
*
- * It picks the footage (the item itself, a named clip, or the clip under an
- * element), samples the stretch the two share on the timeline's clock, reads
+ * It picks the footage (a named clip, a video clip's own picture for a mask
+ * or a read, or the clip under the item), samples the stretch the two share on the timeline's clock, reads
* it through the landmarkers (tracking.ts), settles and places the path
* (trackKeys.ts), and lands the result on the item as one undo step.
*/
-import { MASK_FEATHER_MAX, poseAt, retimeOf } from "@donkeycut/effects-kit";
+import { MASK_FEATHER_MAX, poseAt, retimeOf, type MaskPoint } from "@donkeycut/effects-kit";
import { cutTracking } from "./chatRuntime";
import { clipLen, useEditor } from "./store";
import { trackFootage } from "./tracking";
@@ -21,7 +21,6 @@ import {
settleTrack,
trackedMask,
type FollowMode,
- type Pt,
type TrackItem,
type TrackSample,
} from "./trackKeys";
@@ -34,15 +33,12 @@ const MIN_TRACK_SECONDS = 0.05;
const TRACK_SPANS_MAX = 12;
const TRACK_PATH_POINTS = 24;
-/** A track that cannot run, in words for the person or the model. */
-export class TrackError extends Error {}
-
export interface TrackRequest {
/** The item driven: a video clip or an overlay element. */
id: string;
target: TrackTarget;
use: TrackUse;
- /** The video clip whose footage is read; see `trackMotion`. */
+ /** The video clip whose footage is read; see `trackSource`. */
sourceId?: string;
/** Timeline seconds bounding the track. */
from?: number;
@@ -58,19 +54,16 @@ export async function trackMotion(req: TrackRequest) {
const s = useEditor.getState();
const o = s.overlays.find((x) => x.id === req.id);
const own = o ? undefined : s.clips.find((c) => c.id === req.id);
- if (!o && !own) throw new TrackError("No overlay element or video clip with that id.");
- if (o && req.use === "mask" && isEffectOverlay(o)) throw new TrackError("Effect elements have no pixels to mask.");
+ if (!o && !own) throw new Error("No overlay element or video clip with that id.");
+ if (o && req.use === "mask" && isEffectOverlay(o)) throw new Error("Effect elements have no pixels to mask.");
- // The footage read: the named clip, the item itself, or the clip that
- // most covers the element.
+ // The item's clock as the read starts: keys count from here, so a move
+ // during the read cannot slide them off the footage.
const span = o ? { start: o.start, end: o.end } : { start: own!.start, end: own!.start + clipLen(own!) };
- const named = req.sourceId === undefined ? undefined : s.clips.find((c) => c.id === req.sourceId);
- if (req.sourceId !== undefined && !named) throw new TrackError("No video clip with that source_id.");
- const source = named ?? own ?? clipUnder(s.clips, s.assets, span);
- if (!source) throw new TrackError("No video clip plays under that element — pass source_id.");
+ const source = trackSource(s.clips, s.assets, req, own, span);
const asset = s.assets.find((a) => a.id === source.assetId);
const picture = asset?.type === "video" && asset.width && asset.height ? { width: asset.width, height: asset.height } : null;
- if (!asset || !picture) throw new TrackError("The source clip has no video picture to track.");
+ if (!asset || !picture) throw new Error("The source clip has no video picture to track.");
// The stretch tracked: where the item and the source overlap, narrowed by
// from/to, sampled evenly on the timeline's clock.
@@ -78,7 +71,7 @@ export async function trackMotion(req: TrackRequest) {
const a = Math.max(span.start, source.start, req.from ?? -Infinity);
const b = Math.min(span.end, srcEnd, req.to ?? Infinity);
if (b - a < MIN_TRACK_SECONDS)
- throw new TrackError(`The item and the source clip share no stretch to track (${round2(source.start)}–${round2(srcEnd)}s holds the footage).`);
+ throw new Error(`The item and the source clip share no stretch to track (${round2(source.start)}–${round2(srcEnd)}s holds the footage).`);
const tune = cutTracking();
const fps = Math.min(tune.sampleFps, tune.maxFrames / (b - a));
const n = Math.max(2, Math.floor((b - a) * fps) + 1);
@@ -92,7 +85,7 @@ export async function trackMotion(req: TrackRequest) {
const settled = settleTrack(times, raw, tune);
const place = pictureToFrame(source, picture, s.aspect);
const framed = settled.map((x) =>
- x ? { outline: x.outline.map(place), center: place(x.center), axis: [place(x.axis[0]), place(x.axis[1])] as [Pt, Pt] } : null
+ x ? { outline: x.outline.map(place), center: place(x.center), axis: [place(x.axis[0]), place(x.axis[1])] as [MaskPoint, MaskPoint] } : null
);
const found = framed.filter((x) => x !== null).length;
const summary = {
@@ -106,21 +99,22 @@ export async function trackMotion(req: TrackRequest) {
};
if (req.use === "read") return { ...summary, path: trackPath(times, framed) };
const label = trackTargetOf(req.target).label.toLowerCase();
- if (found === 0) throw new TrackError(`No ${label} found in the source between ${round2(a)}s and ${round2(b)}s.`);
+ if (found === 0) throw new Error(`No ${label} found in the source between ${round2(a)}s and ${round2(b)}s.`);
// The item as it stands now: the read took a while, and the person may
// have moved it meanwhile.
const now = useEditor.getState();
const liveO = o && now.overlays.find((x) => x.id === o.id);
const liveClip = own && now.clips.find((c) => c.id === own.id);
- if (!liveO && !liveClip) throw new TrackError("The item was deleted while it was being tracked.");
+ if (!liveO && !liveClip) throw new Error("The item was deleted while it was being tracked.");
const item: TrackItem = liveO ? { kind: "overlay", overlay: liveO } : { kind: "clip", clip: liveClip! };
+ const local = times.map((t) => t - span.start);
// A keyed pen mask that traces the target.
if (req.use === "mask") {
const feather = Math.max(0, Math.min(MASK_FEATHER_MAX, req.feather ?? 0));
const invert = req.invert === true;
- const mask = trackedMask(item, now.aspect, times, framed, { feather, invert }, tune.keyTolerance)!;
+ const mask = trackedMask(item, now.aspect, local, framed, { feather, invert }, tune.keyTolerance)!;
if (liveO) now.updateOverlay(liveO.id, { mask });
else now.updateClip(liveClip!.id, { mask });
return { ...summary, id: req.id, mask: { kind: mask.kind, keys: mask.kf!.length, feather, invert } };
@@ -128,20 +122,52 @@ export async function trackMotion(req: TrackRequest) {
// Pose keys that carry the item with the target.
const follow = req.follow ?? "move_scale";
- const rest = liveO ? poseAt(liveO, a - liveO.start) : clipPoseAt(liveClip!, a - liveClip!.start);
- const kf = followKeys(item, now.aspect, times, framed, rest, follow, tune.keyTolerance);
+ const rest = liveO ? poseAt(liveO, a - span.start) : clipPoseAt(liveClip!, a - span.start);
+ const kf = followKeys(now.aspect, local, framed, rest, follow, tune.keyTolerance);
if (liveO) now.updateOverlay(liveO.id, { kf });
else now.updateClip(liveClip!.id, { kf });
return { ...summary, id: req.id, follow_mode: follow, keys: kf.length };
}
-/** The video clip that covers most of `span` — the footage under an element.
+/**
+ * The footage a track reads: the clip `source_id` names; for a mask or a read
+ * on a video clip, the clip's own picture; else the video clip under the
+ * item. A follow always rides other footage — a clip carried along with its
+ * own subject would move that subject twice.
+ */
+function trackSource(
+ clips: VideoClip[],
+ assets: MediaAsset[],
+ req: TrackRequest,
+ own: VideoClip | undefined,
+ span: { start: number; end: number }
+): VideoClip {
+ const isVideo = (c: VideoClip) => assets.find((x) => x.id === c.assetId)?.type === "video";
+ if (req.sourceId !== undefined) {
+ const named = clips.find((c) => c.id === req.sourceId);
+ if (!named) throw new Error("No video clip with that source_id.");
+ if (req.use === "follow" && named.id === own?.id) {
+ throw new Error("A clip cannot follow its own footage — pass the source_id of the clip it should ride.");
+ }
+ return named;
+ }
+ if (own && req.use !== "follow" && isVideo(own)) return own;
+ const under = clipUnder(clips, span, req.id, isVideo);
+ if (!under) throw new Error(`No other video clip plays under that ${own ? "clip" : "element"} — pass source_id.`);
+ return under;
+}
+
+/** The video clip that covers most of `span` — the footage under an item.
* A tie goes to the clip nearest track 0, the shot itself. */
-function clipUnder(clips: VideoClip[], assets: MediaAsset[], span: { start: number; end: number }): VideoClip | undefined {
+function clipUnder(
+ clips: VideoClip[],
+ span: { start: number; end: number },
+ itemId: string,
+ isVideo: (c: VideoClip) => boolean
+): VideoClip | undefined {
const shared = (c: VideoClip) => Math.min(span.end, c.start + clipLen(c)) - Math.max(span.start, c.start);
- const video = (c: VideoClip) => assets.find((x) => x.id === c.assetId)?.type === "video";
return clips
- .filter((c) => !c.hidden && video(c) && shared(c) > 0)
+ .filter((c) => c.id !== itemId && !c.hidden && isVideo(c) && shared(c) > 0)
.sort((p, q) => shared(q) - shared(p) || Math.abs(p.track ?? 0) - Math.abs(q.track ?? 0))[0];
}
diff --git a/site/src/cut/lib/tracking.ts b/site/src/cut/lib/tracking.ts
index 9a299c8f..e48cdbd8 100644
--- a/site/src/cut/lib/tracking.ts
+++ b/site/src/cut/lib/tracking.ts
@@ -18,7 +18,8 @@ import { cutTracking } from "./chatRuntime";
import { framesAt } from "./mediaRead";
import { visionFileset, withQuietWasmLogs } from "./mediapipe";
import { trackTargetOf, type TrackTarget } from "./trackTargets";
-import type { Pt, TrackSample } from "./trackKeys";
+import type { MaskPoint } from "@donkeycut/effects-kit";
+import type { TrackSample } from "./trackKeys";
const HAND_MODEL = "/mediapipe/hand_landmarker.task";
const FACE_MODEL = "/mediapipe/face_landmarker.task";
@@ -68,7 +69,7 @@ export async function trackFootage(
const order = times.map((_, i) => i).sort((a, b) => times[a] - times[b]);
const out: (TrackSample | null)[] = times.map(() => null);
const detector = await openDetector(target, settings.minConfidence);
- let prev: Pt | null = null;
+ let prev: MaskPoint | null = null;
let seen = 0;
let clock = -1;
try {
@@ -146,10 +147,10 @@ async function openDetector(target: TrackTarget, confidence: number): Promise<De
* there. `aspect` is the frame's width over height: hulls, angles and
* distances are measured in square pixels. `prev` is the target's center a
* frame earlier, which keeps the track on the same hand, face or body. */
-export function sampleOf(target: TrackTarget, found: Landmarks[], aspect: number, prev: Pt | null): TrackSample | null {
+export function sampleOf(target: TrackTarget, found: Landmarks[], aspect: number, prev: MaskPoint | null): TrackSample | null {
const pad = trackTargetOf(target).pad;
- const sq = (p: Pt): Pt => ({ x: p.x * aspect, y: p.y });
- const unsq = (p: Pt): Pt => ({ x: p.x / aspect, y: p.y });
+ const sq = (p: MaskPoint): MaskPoint => ({ x: p.x * aspect, y: p.y });
+ const unsq = (p: MaskPoint): MaskPoint => ({ x: p.x / aspect, y: p.y });
const unsqAll = (s: TrackSample): TrackSample => ({
outline: s.outline.map(unsq),
center: unsq(s.center),
@@ -219,7 +220,7 @@ export function sampleOf(target: TrackTarget, found: Landmarks[], aspect: number
/** The hand on the asked-for side of the picture. With both in view, side
* decides; with one, it stays the tracked hand while it is near where that
* hand was, and otherwise counts when it sits on that half of the picture. */
-function pickHand(found: Landmarks[], side: "left_hand" | "right_hand", prev: Pt | null): Landmarks | null {
+function pickHand(found: Landmarks[], side: "left_hand" | "right_hand", prev: MaskPoint | null): Landmarks | null {
if (found.length === 0) return null;
const left = side === "left_hand";
if (found.length >= 2) {
@@ -234,7 +235,7 @@ function pickHand(found: Landmarks[], side: "left_hand" | "right_hand", prev: Pt
/** The detection nearest where the target was; the largest when the track
* has no history. Null when every detection is too far to be the same one. */
-function pickNearest(found: Landmarks[], prev: Pt | null, centerOf: (l: Landmarks) => Pt): Landmarks | null {
+function pickNearest(found: Landmarks[], prev: MaskPoint | null, centerOf: (l: Landmarks) => MaskPoint): Landmarks | null {
if (found.length === 0) return null;
if (!prev) {
const spread = (l: Landmarks) => {
@@ -259,31 +260,31 @@ function chainLoop(edges: { start: number; end: number }[]): number[] {
return loop;
}
-const mean = (pts: Pt[]): Pt => ({
+const mean = (pts: MaskPoint[]): MaskPoint => ({
x: pts.reduce((s, p) => s + p.x, 0) / pts.length,
y: pts.reduce((s, p) => s + p.y, 0) / pts.length,
});
-const dist = (a: Pt, b: Pt) => Math.hypot(a.x - b.x, a.y - b.y);
+const dist = (a: MaskPoint, b: MaskPoint) => Math.hypot(a.x - b.x, a.y - b.y);
/** Points in order around `c`, starting from the top. */
-function aroundCenter(pts: Pt[], c: Pt): Pt[] {
- const angle = (p: Pt) => (Math.atan2(p.y - c.y, p.x - c.x) + Math.PI * 2.5) % (Math.PI * 2);
+function aroundCenter(pts: MaskPoint[], c: MaskPoint): MaskPoint[] {
+ const angle = (p: MaskPoint) => (Math.atan2(p.y - c.y, p.x - c.x) + Math.PI * 2.5) % (Math.PI * 2);
return [...pts].sort((a, b) => angle(a) - angle(b));
}
/** Points pushed out from `c` by `pad` of their distance. */
-const grow = (pts: Pt[], c: Pt, pad: number): Pt[] =>
+const grow = (pts: MaskPoint[], c: MaskPoint, pad: number): MaskPoint[] =>
pts.map((p) => ({ x: c.x + (p.x - c.x) * (1 + pad), y: c.y + (p.y - c.y) * (1 + pad) }));
/** The convex hull of `pts`, grown by `pad` and resampled to RING_POINTS
* corners at even angles around its middle, so outlines from different
* frames line up corner for corner. Null for fewer than three points. */
-export function hullRing(pts: Pt[], pad: number): Pt[] | null {
+export function hullRing(pts: MaskPoint[], pad: number): MaskPoint[] | null {
const hull = convexHull(pts);
if (hull.length < 3) return null;
const c = mean(hull);
- const ring: Pt[] = [];
+ const ring: MaskPoint[] = [];
for (let k = 0; k < RING_POINTS; k++) {
const theta = (k / RING_POINTS) * Math.PI * 2 - Math.PI / 2;
const d = { x: Math.cos(theta), y: Math.sin(theta) };
@@ -304,19 +305,19 @@ export function hullRing(pts: Pt[], pad: number): Pt[] | null {
return grow(ring, c, pad);
}
-const cross = (u: Pt, v: Pt) => u.x * v.y - u.y * v.x;
+const cross = (u: MaskPoint, v: MaskPoint) => u.x * v.y - u.y * v.x;
/** Andrew's monotone chain, counter-clockwise without repeats. */
-function convexHull(input: Pt[]): Pt[] {
+function convexHull(input: MaskPoint[]): MaskPoint[] {
const pts = [...input].sort((a, b) => a.x - b.x || a.y - b.y);
if (pts.length < 3) return pts;
- const turn = (o: Pt, a: Pt, b: Pt) => (a.x - o.x) * (b.y - o.y) - (a.y - o.y) * (b.x - o.x);
- const lower: Pt[] = [];
+ const turn = (o: MaskPoint, a: MaskPoint, b: MaskPoint) => (a.x - o.x) * (b.y - o.y) - (a.y - o.y) * (b.x - o.x);
+ const lower: MaskPoint[] = [];
for (const p of pts) {
while (lower.length >= 2 && turn(lower[lower.length - 2], lower[lower.length - 1], p) <= 0) lower.pop();
lower.push(p);
}
- const upper: Pt[] = [];
+ const upper: MaskPoint[] = [];
for (let i = pts.length - 1; i >= 0; i--) {
const p = pts[i];
while (upper.length >= 2 && turn(upper[upper.length - 2], upper[upper.length - 1], p) <= 0) upper.pop();
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index a1b394c6..365a7236 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -684,6 +684,9 @@ export interface VideoClip {
/** Effects this clip wears over its own picture, inside its mask, in
* order (absent = none). */
effects?: ClipEffect[];
+ /** Seconds the effects have already run at the clip's first frame, so a
+ * split's right half carries on where the left stopped (absent = 0). */
+ effectsFrom?: number;
/** Hidden clips stay on the timeline (grayed) but render as black — excluded
* from the played/exported picture without disturbing the layout. */
hidden?: boolean;
@@ -1191,6 +1194,7 @@ export interface TemplateLayer {
grade?: ColorGrade;
mask?: Mask;
effects?: ClipEffect[];
+ effectsFrom?: number;
boxStyle?: BoxStyle;
kf?: OverlayKey[];
hidden?: boolean;
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index 7987afa7..c5c715bb 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -611,6 +611,24 @@ describe("clip masks in the filtergraph", () => {
expect(xfadeMismatches(g)).toEqual([]);
});
+ test("a track-0 clip wears its effects over its look, as the preview does", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 3, look: "vintage", effects: [{ effect: "negative" }] }), clip("b.mp4", { out: 2 })],
+ });
+ const lookAt = g.findIndex((c) => c.includes("[lki0]"));
+ const fxAt = g.findIndex((c) => c.includes("lutyuv=y='minval+maxval-val'"));
+ expect(lookAt).toBeGreaterThanOrEqual(0);
+ expect(fxAt).toBeGreaterThan(lookAt);
+ expect(xfadeMismatches(g)).toEqual([]);
+ });
+
+ test("a split's right half carries its effects' clock on", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 3, effects: [{ effect: "huecycle", amount: 0.5 }], effectsFrom: 2 }), clip("b.mp4", { out: 2 })],
+ });
+ expect(g.some((c) => c.includes("hue=H='2*PI*0.8*(t--2)'"))).toBe(true);
+ });
+
test("a mask under head/tail alpha fades keeps both", async () => {
const g = await graphFor({
clips: [clip("a.mp4", { out: 6 })],
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index f5ab209e..4960eaaf 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -186,6 +186,8 @@ export interface ExportSpec {
lookAmount?: number;
/** Effects the clip wears over its own picture, on its own clock. */
effects?: ClipEffect[];
+ /** Seconds that clock has run at the clip's first frame. */
+ effectsFrom?: number;
/** Hidden clips keep their slot but render black + silent. */
hidden?: boolean;
/** A still image: looped for the clip's length instead of trimmed. */
@@ -296,6 +298,8 @@ export interface ExportSpec {
lookAmount?: number;
/** Effects the clip wears over its own picture, under its mask. */
effects?: ClipEffect[];
+ /** Seconds that clock has run at the clip's first frame. */
+ effectsFrom?: number;
/** Client-painted grayscale coverage trimming this overlay's picture,
* box-sized (a letterboxed segment pads out to its box when masked);
* `subject` trims by the shared person matte instead. */
@@ -1347,12 +1351,15 @@ export async function runExport(
const filters: string[] = [];
- // A clip's own effects run on its segment's clock, from 0 to its length,
- // over its framed picture: chain fragment `core` in, the treated fragment
- // out. The fades, pad and mask that follow carry them, as in the preview.
- const wearEffects = (core: string, effects: ClipEffect[] | undefined, len: number, w: number, h: number, pixFmt: string, tag: string) => {
- for (const [n, e] of (effects ?? []).entries()) {
- const lines = effectFilterLines(`${tag}fi${n}`, `${tag}fo${n}`, e.effect, e.amount, 0, len, w, h, `${tag}x${n}`, undefined, undefined, { ...chroma, pixFmt });
+ // A clip's own effects run on its segment's clock over its framed
+ // picture, after its look: chain fragment `core` in, the treated fragment
+ // out. The clock starts `from` seconds in (a split's right half carries
+ // on), so the effects begin at -from on the segment's clock. The fades,
+ // pad and mask that follow carry them, as in the preview.
+ const wearEffects = (core: string, c: { effects?: ClipEffect[]; effectsFrom?: number }, len: number, w: number, h: number, pixFmt: string, tag: string) => {
+ const from = c.effectsFrom ?? 0;
+ for (const [n, e] of (c.effects ?? []).entries()) {
+ const lines = effectFilterLines(`${tag}fi${n}`, `${tag}fo${n}`, e.effect, e.amount, -from, len, w, h, `${tag}x${n}`, undefined, undefined, { ...chroma, pixFmt });
if (!lines) continue;
filters.push(`${core}[${tag}fi${n}]`, ...lines);
core = `[${tag}fo${n}]null`;
@@ -2077,21 +2084,24 @@ export async function runExport(
let core = framedTimebase(timebase, frame, `c${j}`, rmIn ? {} : c, rt, fps, filters);
if (plan) core = plan.run(core, `c${j}`);
core += `,format=${segFmt}`;
- // The clip's effects treat its picture, ahead of the letterbox bars.
- core = wearEffects(core, c.effects, dur, W, H, segFmt, `cw${j}`);
- core += padding;
// The look bakes in after grade + framing, before the edge effects, so
// animations move already-graded pixels (matching the preview order). A
// removal clip's look runs after the flatten below instead — the chain's
- // internal blends would drop the keyed layer's alpha.
- if (c.look && !rmIn) {
+ // internal blends would drop the keyed layer's alpha. A clip wearing
+ // effects takes its look on the picture ahead of the letterbox bars, so
+ // the effects can treat the looked picture there, as the preview does.
+ const lookAt = (at: string) => {
+ if (!c.look || rmIn) return at;
const lines = lookFilterLines(`lki${j}`, `lko${j}`, c.look, c.lookAmount, H, clipFmt, `c${j}`, depth);
- if (lines) {
- filters.push(`${core}[lki${j}]`);
- filters.push(...lines);
- core = `[lko${j}]null`;
- }
- }
+ if (!lines) return at;
+ filters.push(`${at}[lki${j}]`);
+ filters.push(...lines);
+ return `[lko${j}]null`;
+ };
+ const wears = !!c.effects?.length;
+ if (wears) core = wearEffects(lookAt(core), c, dur, W, H, segFmt, `cw${j}`);
+ core += padding;
+ if (!wears) core = lookAt(core);
// The border ring lands after the look (true stroke color) and before
// the fades, so it fades and masks with the clip like the preview.
const brIdx = clipBorderInput.get(j);
@@ -2505,7 +2515,7 @@ export async function runExport(
core = `[olko${k}]null`;
}
}
- core = wearEffects(core, oc.effects, olen, boxW, boxH, lookFmt, `ow${k}`);
+ core = wearEffects(core, oc, olen, boxW, boxH, lookFmt, `ow${k}`);
if (boxPad) core += boxPad;
// The border ring lands after the look and box pad, before the edge
// ramps, so it fades and masks with the clip like the preview.