Skip to content

Open source CapCut alternative.

TypeScript

feat(site): let a clip wear its own effects [rebuild]

David Led6d342fb0e53
diff --git a/evals/cut-perf.latest-report.json b/evals/cut-perf.latest-report.json
index 4446d78c..72563b3d 100644
--- a/evals/cut-perf.latest-report.json
+++ b/evals/cut-perf.latest-report.json
@@ -51,10 +51,10 @@
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.3,
- "p95": 31.6,
- "mean": 17.72,
- "max": 32.9,
+ "p50": 15.8,
+ "p95": 32.6,
+ "mean": 18.51,
+ "max": 33.3,
"n": 41
},
"longTasks": {
@@ -64,7 +64,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 61,
+ "liveSamples": 58,
"machine": "desktop"
},
{
@@ -75,16 +75,16 @@
"late": {
"p50": 0,
"p95": 1,
- "mean": 0.15,
- "max": 2,
+ "mean": 0.12,
+ "max": 1,
"n": 41
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.6,
- "p95": 32.6,
- "mean": 17.7,
- "max": 49.6,
+ "p50": 15.8,
+ "p95": 31.5,
+ "mean": 17.46,
+ "max": 33,
"n": 41
},
"longTasks": {
@@ -94,7 +94,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 67,
+ "liveSamples": 74,
"machine": "desktop"
},
{
@@ -105,16 +105,16 @@
"late": {
"p50": 0,
"p95": 0,
- "mean": 0.07,
- "max": 2,
+ "mean": 0.02,
+ "max": 1,
"n": 41
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.2,
- "p95": 16.5,
- "mean": 16.95,
- "max": 76.7,
+ "p50": 15.6,
+ "p95": 15.9,
+ "mean": 15.07,
+ "max": 32.4,
"n": 41
},
"longTasks": {
@@ -124,7 +124,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 79,
+ "liveSamples": 96,
"machine": "desktop"
},
{
@@ -135,16 +135,16 @@
"late": {
"p50": 0,
"p95": 0,
- "mean": 0.02,
+ "mean": 0.01,
"max": 1,
"n": 81
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.2,
- "p95": 15.7,
- "mean": 14.77,
- "max": 29.7,
+ "p50": 15.8,
+ "p95": 16.7,
+ "mean": 15.6,
+ "max": 32.1,
"n": 81
},
"longTasks": {
@@ -165,16 +165,16 @@
"late": {
"p50": 0,
"p95": 1,
- "mean": 0.17,
- "max": 2,
+ "mean": 0.15,
+ "max": 1,
"n": 41
},
"frameMs": 16.7,
"scrub": {
- "p50": 14.9,
- "p95": 30.2,
- "mean": 17.04,
- "max": 45,
+ "p50": 15.9,
+ "p95": 32.9,
+ "mean": 18.1,
+ "max": 33.3,
"n": 41
},
"longTasks": {
@@ -184,7 +184,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 57,
+ "liveSamples": 64,
"machine": "desktop"
},
{
@@ -196,15 +196,15 @@
"p50": 0,
"p95": 1,
"mean": 0.17,
- "max": 2,
+ "max": 1,
"n": 41
},
"frameMs": 16.7,
"scrub": {
- "p50": 14.9,
- "p95": 31.4,
- "mean": 17.18,
- "max": 51.8,
+ "p50": 15.8,
+ "p95": 32.6,
+ "mean": 18.48,
+ "max": 32.8,
"n": 41
},
"longTasks": {
@@ -214,7 +214,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 60,
+ "liveSamples": 44,
"machine": "desktop"
},
{
@@ -225,16 +225,16 @@
"late": {
"p50": 1,
"p95": 1,
- "mean": 0.9,
+ "mean": 0.88,
"max": 1,
"n": 41
},
"frameMs": 16.7,
"scrub": {
- "p50": 31.2,
- "p95": 32.3,
- "mean": 28.93,
- "max": 32.6,
+ "p50": 32.3,
+ "p95": 33.1,
+ "mean": 30.26,
+ "max": 33.3,
"n": 41
},
"longTasks": {
@@ -253,7 +253,7 @@
"pass": true,
"notes": [],
"drops": 0,
- "presented": 1039,
+ "presented": 1038,
"boundaryDrops": 0,
"longTasks": {
"p50": 0,
@@ -326,13 +326,13 @@
"pass": true,
"notes": [],
"drops": 0,
- "presented": 3243,
+ "presented": 3235,
"lag": {
"p50": 0.02,
"p95": 0.03,
"mean": 0.02,
"max": 0.03,
- "n": 3243
+ "n": 3235
},
"stallShare": 0,
"decay": {
@@ -363,13 +363,13 @@
"pass": true,
"notes": [],
"drops": 0,
- "presented": 223,
+ "presented": 222,
"lag": {
"p50": 0.02,
"p95": 0.03,
- "mean": 0.01,
+ "mean": 0.02,
"max": 0.03,
- "n": 223
+ "n": 222
},
"stallShare": 0,
"decay": {
@@ -400,13 +400,13 @@
"pass": true,
"notes": [],
"drops": 0,
- "presented": 1028,
+ "presented": 1029,
"lag": {
"p50": 0.02,
"p95": 0.03,
"mean": 0.02,
"max": 0.03,
- "n": 1028
+ "n": 1029
},
"stallShare": 0,
"decay": {
@@ -514,7 +514,7 @@
"presented": 3444,
"lag": {
"p50": 0.01,
- "p95": 0.02,
+ "p95": 0.01,
"mean": 0.01,
"max": 0.02,
"n": 3444
@@ -550,7 +550,7 @@
"drops": 0,
"presented": 3444,
"lag": {
- "p50": 0.01,
+ "p50": 0.02,
"p95": 0.03,
"mean": 0.02,
"max": 0.03,
@@ -603,11 +603,11 @@
"reported": 0.04
},
"longTasks": {
- "p50": 60,
- "p95": 60,
- "mean": 57,
- "max": 60,
- "n": 2
+ "p50": 55,
+ "p95": 55,
+ "mean": 55,
+ "max": 55,
+ "n": 1
},
"liveSamples": 10,
"liveSources": 1,
@@ -621,30 +621,30 @@
"bucket": "playback",
"pass": true,
"notes": [],
- "drops": 0,
- "presented": 3444,
+ "drops": 26,
+ "presented": 3422,
"lag": {
"p50": 0.01,
"p95": 0.02,
"mean": 0.01,
- "max": 0.02,
- "n": 3444
+ "max": 0.13,
+ "n": 3422
},
"stallShare": 0,
"decay": {
"first": 0,
- "last": 0
+ "last": 0.019
},
"clock": {
"lead": 0.04,
"reported": 0.04
},
"longTasks": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "max": 0,
- "n": 0
+ "p50": 162,
+ "p95": 162,
+ "mean": 132.5,
+ "max": 162,
+ "n": 2
},
"liveSamples": 10,
"liveSources": 1,
@@ -661,7 +661,7 @@
"drops": 0,
"presented": 3444,
"lag": {
- "p50": 0.01,
+ "p50": 0.02,
"p95": 0.03,
"mean": 0.02,
"max": 0.03,
@@ -696,13 +696,13 @@
"pass": true,
"notes": [],
"drops": 0,
- "presented": 1618,
+ "presented": 1621,
"lag": {
"p50": 0.02,
"p95": 0.03,
"mean": 0.02,
"max": 0.03,
- "n": 1618
+ "n": 1621
},
"stallShare": 0,
"decay": {
@@ -765,18 +765,18 @@
"bucket": "playback",
"pass": true,
"notes": [],
- "drops": 3,
- "presented": 1095,
+ "drops": 2,
+ "presented": 1094,
"lag": {
"p50": 0.02,
"p95": 0.03,
- "mean": 0.01,
+ "mean": 0.02,
"max": 0.03,
"n": 1092
},
"stallShare": 0,
"decay": {
- "first": 0.008,
+ "first": 0.005,
"last": 0
},
"clock": {
@@ -784,10 +784,10 @@
"reported": 0.04
},
"longTasks": {
- "p50": 164,
- "p95": 164,
- "mean": 115.5,
- "max": 164,
+ "p50": 156,
+ "p95": 156,
+ "mean": 108.5,
+ "max": 156,
"n": 2
},
"liveSamples": 10,
diff --git a/site/packages/effects-kit/src/effects.test.ts b/site/packages/effects-kit/src/effects.test.ts
index 14f940fc..d8b2bd8e 100644
--- a/site/packages/effects-kit/src/effects.test.ts
+++ b/site/packages/effects-kit/src/effects.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
-import { codeScale, EFFECT_IDS, effectFilterLines, effectPreviewState } from "./effects";
+import { CLIP_EFFECT_IDS, codeScale, EFFECT_IDS, effectFilterLines, effectPreviewState } from "./effects";
// The bundled engine ffmpeg is LGPL: a recipe reaching for a GPL-only filter
// renders on a dev machine (Homebrew ffmpeg) and fails in the shipped app.
@@ -43,6 +43,29 @@ const filterNamesOf = (lines: string[]): string[] => {
};
describe("effect recipes", () => {
+ test("every clip effect is a picture effect with both recipes", () => {
+ for (const id of CLIP_EFFECT_IDS) {
+ expect((EFFECT_IDS as string[]).includes(id)).toBe(true);
+ expect(effectFilterLines("in", "out", id, 0.5, 0, 2, 540, 960, "t")).not.toBe(null);
+ }
+ });
+
+ test("a negative flips every color whatever the amount", () => {
+ expect(effectPreviewState("negative", 0.1, 3).cssFilter).toBe("invert(1)");
+ expect(effectPreviewState("negative", 1, 0).cssFilter).toBe("invert(1)");
+ });
+
+ test("a color cycle turns the hue at the amount's pace, from its own start", () => {
+ // Amount 0.5 is 0.8 turns a second: a quarter second in is 72 degrees.
+ expect(effectPreviewState("huecycle", 0.5, 0).cssFilter).toBe("hue-rotate(0deg)");
+ expect(effectPreviewState("huecycle", 0.5, 0.25).cssFilter).toBe("hue-rotate(72deg)");
+ // A faster amount turns further over the same time.
+ const deg = (k: number) => Number(effectPreviewState("huecycle", k, 0.1).cssFilter.match(/[\d.]+/)![0]);
+ expect(deg(1)).toBeGreaterThan(deg(0.2));
+ // The export turns on the same clock, measured from the element's start.
+ expect(effectFilterLines("in", "out", "huecycle", 0.5, 2, 5, 1080, 1920, "t")![0]).toContain("hue=H='2*PI*0.8*(t-2)'");
+ });
+
test("every effect exports through the LGPL-safe filter set", () => {
for (const id of EFFECT_IDS) {
const lines = effectFilterLines("in", "out", id, 0.7, 1, 4, 1080, 1920, "t");
diff --git a/site/packages/effects-kit/src/effects.ts b/site/packages/effects-kit/src/effects.ts
index bfc7c9d9..701a5e55 100644
--- a/site/packages/effects-kit/src/effects.ts
+++ b/site/packages/effects-kit/src/effects.ts
@@ -23,6 +23,8 @@ export type VisualEffectId =
| "lightleak"
| "flash"
| "shake"
+ | "negative"
+ | "huecycle"
/** The graded looks, placeable like any other effect. */
| "vintage"
| "horror"
@@ -66,6 +68,8 @@ export const EFFECT_IDS: VisualEffectId[] = [
"lightleak",
"flash",
"shake",
+ "negative",
+ "huecycle",
"vintage",
"horror",
"halation",
@@ -100,6 +104,37 @@ const lookOf = (effect: string): LookStyle | null =>
export const ALL_EFFECT_IDS: EffectId[] = [...EFFECT_IDS, ...AUDIO_EFFECT_IDS];
+/** The picture treatments a clip can wear itself. */
+export type ClipEffectId = "grain" | "vhs" | "glitch" | "blur" | "vignette" | "lightleak" | "flash" | "negative" | "huecycle";
+
+/**
+ * The effects a clip wears treat that clip's picture alone, inside its mask,
+ * for the clip's whole length: a masked copy over the shot wears a negative
+ * and only the window turns. Zoom and shake move the frame, which a clip
+ * does with its own zoom and pose keys; the looks are the clip's grade.
+ */
+export const CLIP_EFFECT_IDS: ClipEffectId[] = [
+ "negative",
+ "huecycle",
+ "grain",
+ "vhs",
+ "glitch",
+ "blur",
+ "vignette",
+ "lightleak",
+ "flash",
+];
+
+/** One effect a clip wears; `amount` 0..1, absent = 0.5. */
+export interface ClipEffect {
+ effect: ClipEffectId;
+ amount?: number;
+}
+
+/** The effects with nothing to dial: a negative flips every color whatever
+ * the amount says. */
+export const AMOUNTLESS_EFFECTS: EffectId[] = ["negative"];
+
export const EFFECT_LABELS: Record<EffectId, string> = {
...AUDIO_EFFECT_LABELS,
zoom: "Zoom",
@@ -111,6 +146,8 @@ export const EFFECT_LABELS: Record<EffectId, string> = {
lightleak: "Light leak",
flash: "Flash",
shake: "Shake",
+ negative: "Negative",
+ huecycle: "Color cycle",
vintage: LOOK_LABELS.vintage,
horror: LOOK_LABELS.horror,
halation: LOOK_LABELS.halation,
@@ -239,6 +276,12 @@ export function defineEffect(id: string, recipe: EffectRecipe): void {
* read it so the export matches the preview. */
const SHAKE_AMP = 22;
+/** How fast a color cycle turns the hue wheel, in turns per second, at the
+ * lowest and highest amount. */
+const HUE_TURNS_MIN = 0.1;
+const HUE_TURNS_MAX = 1.5;
+const hueTurns = (k: number) => HUE_TURNS_MIN + (HUE_TURNS_MAX - HUE_TURNS_MIN) * k;
+
/** How far a zoom pushes in at full depth. The picture renders at this much
* more than frame size and is cropped back around the focus point. */
const ZOOM_RANGE = 0.6;
@@ -373,6 +416,14 @@ export function effectPreviewState(
zoom: 1 + (amp * 2) / 1080,
};
}
+ case "negative":
+ // Every color flipped to its opposite: black to white, skin to cyan.
+ return { cssFilter: "invert(1)" };
+ case "huecycle": {
+ // The hue wheel turns at the amount's pace from the element's start.
+ const deg = (360 * hueTurns(k) * Math.max(0, tLocal)) % 360;
+ return { cssFilter: `hue-rotate(${fmt(deg)}deg)` };
+ }
default:
return { cssFilter: "" };
}
@@ -733,6 +784,15 @@ export function effectFilterLines(
`[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass${chroma.overlay}[${outLabel}]`,
];
}
+ case "negative":
+ // Each plane flips about its own legal range, so black and white swap
+ // exactly in a limited-range chain and the chroma mirrors about grey.
+ return [
+ `[${inLabel}]lutyuv=y='minval+maxval-val':u='minval+maxval-val':v='minval+maxval-val':${en}[${outLabel}]`,
+ ];
+ case "huecycle":
+ // The canvas recipe's turning wheel, in radians on the frame's clock.
+ return [`[${inLabel}]hue=H='2*PI*${fmt(hueTurns(k))}*(t-${fmt(start)})':${en}[${outLabel}]`];
default:
return null;
}
diff --git a/site/src/clients/chatgpt/server/guidance.ts b/site/src/clients/chatgpt/server/guidance.ts
index c130ff8d..30a3f985 100644
--- a/site/src/clients/chatgpt/server/guidance.ts
+++ b/site/src/clients/chatgpt/server/guidance.ts
@@ -54,7 +54,7 @@ For one exact instance among similar subjects, ask the user to use Custom, Brush
"graphics": `# Titles, graphics and motion
Use add_title or add_text_sequence for words, add_shape for geometry, add_sticker for an imported image, and add_effect for a treatment over a time range. Read describe_commands for the current supported styles and parameters. set_background sets the project's background color. Use ids and fonts reported by inspect_project.
update_overlay changes text, position, size, color and other element fields, and accepts ids for changes shared by several elements. Layout coordinates are frame fractions. Keep text inside the project's reported safe zones and check legibility with capture_frame.
-set_overlay_keyframes and set_clip_keyframes animate poses at exact times. set_group_camera moves one camera over a group of elements, and blur and motion blur soften what the camera is not on. set_overlay_animation supplies entrances, exits, loops, word motion, a typing caret and a one-shot press at a chosen moment. set_mask clips a layer to a shape and can animate the reveal. track_motion keys a mask or a pose to a hand, a head or a person in the footage, frame by frame. Read text-videos for timed text and text-creativity for design choices.
+set_overlay_keyframes and set_clip_keyframes animate poses at exact times. set_group_camera moves one camera over a group of elements, and blur and motion blur soften what the camera is not on. set_overlay_animation supplies entrances, exits, loops, word motion, a typing caret and a one-shot press at a chosen moment. set_mask clips a layer to a shape and can animate the reveal. track_motion keys a mask or a pose to a hand, a head or a person in the footage, frame by frame. set_clip_effects treats one clip alone, inside its mask. Read text-videos for timed text and text-creativity for design choices.
Build composite effects as layers: add_overlay_video for copies of footage, masks for visible regions, shapes for bands, and text for labels. Match source trims so copied footage stays synchronized. Work out geometry and timing, build one representative piece, capture_frame to check it, then repeat and check the whole composition. create_sticker generates a new sticker and spends credits when the user requests it.`,
"audio-and-subtitles": `# Audio and subtitles
diff --git a/site/src/cut/components/ClipEffectsSection.tsx b/site/src/cut/components/ClipEffectsSection.tsx
new file mode 100644
index 00000000..29fbbf60
--- /dev/null
+++ b/site/src/cut/components/ClipEffectsSection.tsx
@@ -0,0 +1,102 @@
+"use client";
+
+import { X } from "lucide-react";
+import {
+ AMOUNTLESS_EFFECTS,
+ CLIP_EFFECT_IDS,
+ EFFECT_LABELS,
+ type ClipEffect,
+ type ClipEffectId,
+} from "@donkeycut/effects-kit";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
+import { Row, Section, useSliderCheckpoint } from "@/cut/components/panelBits";
+import { parsePercentInput } from "@/cut/components/ScrubValue";
+import { ValueSlider } from "@/cut/components/ValueSlider";
+import { useEditor } from "@/cut/lib/store";
+import type { VideoClip } from "@/cut/lib/types";
+
+/**
+ * The effects a clip wears: each treats this clip's picture alone, inside its
+ * mask, for the clip's whole length. A masked copy over the shot wearing a
+ * negative turns only the window. The same list the chat's set_clip_effects
+ * writes.
+ */
+export function ClipEffectsSection({ clip }: { clip: VideoClip }) {
+ const ck = useSliderCheckpoint();
+ const list = clip.effects ?? [];
+ const st = () => useEditor.getState();
+ const write = (effects: ClipEffect[]) => st().updateClip(clip.id, { effects: effects.length ? effects : undefined });
+
+ // An effect is worn once; the picker offers the ones the clip lacks.
+ const free = CLIP_EFFECT_IDS.filter((id) => !list.some((e) => e.effect === id));
+
+ // The amount drags as one undo step, the way every inspector slider does.
+ const setAmount = (i: number, amount: number) => {
+ ck.begin();
+ st().updateClipTransient(clip.id, { effects: list.map((e, j) => (j === i ? { ...e, amount } : e)) });
+ };
+
+ return (
+ <Section
+ title="Effects"
+ info="Each effect treats this clip's picture alone, inside its mask, for the clip's whole length. An effect from the Effects tab treats the whole frame under its window."
+ aside={
+ free.length > 0 && (
+ <Select
+ value={null}
+ items={Object.fromEntries(free.map((id) => [id, EFFECT_LABELS[id]]))}
+ onValueChange={(v) => {
+ if (v) write([...list, { effect: v as ClipEffectId }]);
+ }}
+ >
+ <SelectTrigger className="h-7 w-28 text-[12px]">
+ <SelectValue placeholder="Add effect" />
+ </SelectTrigger>
+ <SelectContent>
+ {free.map((id) => (
+ <SelectItem key={id} value={id} className="text-[12px]">
+ {EFFECT_LABELS[id]}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ )
+ }
+ >
+ {list.map((e, i) => (
+ <Row key={e.effect} label={EFFECT_LABELS[e.effect]}>
+ {!AMOUNTLESS_EFFECTS.includes(e.effect) && (
+ <ValueSlider
+ label={`${EFFECT_LABELS[e.effect]} amount`}
+ sliderClassName="data-horizontal:w-24"
+ valueClassName="w-9 text-muted-foreground"
+ value={e.amount ?? 0.5}
+ min={0.05}
+ max={1}
+ step={0.01}
+ snap={[0.5]}
+ format={(v) => String(Math.round(v * 100))}
+ parse={parsePercentInput}
+ onDraft={(v) => setAmount(i, v)}
+ onCommit={(v) => {
+ setAmount(i, v);
+ ck.end();
+ }}
+ />
+ )}
+ <Tooltip>
+ <TooltipTrigger
+ aria-label={`Remove ${EFFECT_LABELS[e.effect]}`}
+ className="grid size-6 place-items-center rounded text-muted-foreground transition-colors hover:text-foreground"
+ onClick={() => write(list.filter((_, j) => j !== i))}
+ >
+ <X className="size-3.5" />
+ </TooltipTrigger>
+ <TooltipContent>Remove</TooltipContent>
+ </Tooltip>
+ </Row>
+ ))}
+ </Section>
+ );
+}
diff --git a/site/src/cut/components/EffectsPanel.tsx b/site/src/cut/components/EffectsPanel.tsx
index 2a6f8afd..861afd24 100644
--- a/site/src/cut/components/EffectsPanel.tsx
+++ b/site/src/cut/components/EffectsPanel.tsx
@@ -80,7 +80,7 @@ const GROUPS = [
{ id: "audio", label: "Sound" },
] as const;
-const MOVING: EffectId[] = ["zoom", "grain", "vhs", "glitch", "lightleak", "flash", "shake"];
+const MOVING: EffectId[] = ["zoom", "grain", "vhs", "glitch", "lightleak", "flash", "shake", "huecycle"];
const groupOf = (id: EffectId): EffectGroup =>
isAudioEffect(id) ? "audio" : MOVING.includes(id) ? "moving" : "filters";
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index e72690c5..99eb8379 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -31,6 +31,7 @@ import {
MASK_KINDS,
MASK_RADIUS_MAX,
PEN_MIN_POINTS,
+ CLIP_EFFECT_IDS,
} from "@donkeycut/effects-kit";
import { bool, ids, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
import { builtinLutCatalogText } from "@/cut/lib/builtinLuts";
@@ -359,6 +360,22 @@ export const INSPECTOR_TOOLS = [
clear: bool("Remove the box styling entirely"),
}, ["clipId"]),
},
+ {
+ name: "set_clip_effects",
+ description:
+ `Set the effects a video clip wears over its own picture (${CLIP_EFFECT_IDS.join(", ")}). They treat that clip alone — inside its mask, under its pose — for its whole length, on the clip's own clock: a masked copy over the shot wearing negative and huecycle shows a color-cycling negative window while the shot around it plays untouched. add_effect places an element that treats the whole frame under its window. Pass the full list in order; [] removes them all. amount 0.05..1 (default 0.5) is the strength; for huecycle it is how fast the hue turns (0.5 is a turn every 1.25s); negative has none.`,
+ inputSchema: obj({
+ clipId: str("Video clip id"),
+ effects: {
+ type: "array",
+ description: "The clip's effects, applied in this order",
+ items: obj({
+ effect: { type: "string", enum: [...CLIP_EFFECT_IDS], description: "Effect id" },
+ amount: num("Strength 0.05..1 (default 0.5)"),
+ }, ["effect"]),
+ },
+ }, ["clipId", "effects"]),
+ },
{
name: "set_speed",
description:
diff --git a/site/src/cut/components/Inspector.tsx b/site/src/cut/components/Inspector.tsx
index e3d9540b..64d2ccbc 100644
--- a/site/src/cut/components/Inspector.tsx
+++ b/site/src/cut/components/Inspector.tsx
@@ -7,6 +7,7 @@ import { Button } from "@/components/ui/button";
import { EmojiPicker } from "@/cut/components/EmojiPicker";
import { FontPicker } from "@/cut/components/FontPicker";
import { ClipAnimationTiles, previewBar } from "@/cut/components/TransitionsPanel";
+import { ClipEffectsSection } from "@/cut/components/ClipEffectsSection";
import { TrackRow } from "@/cut/components/TrackRow";
import {
Select,
@@ -86,6 +87,7 @@ import {
maskSizeAxes,
penClosed,
restingMaskFrame,
+ AMOUNTLESS_EFFECTS,
} from "@donkeycut/effects-kit";
import { BLOCK_COLOR } from "@/cut/lib/blockSource";
import { bindRecording, unbindRecording } from "@/cut/lib/soundBind";
@@ -732,6 +734,7 @@ function ClipFramePanel({ clip }: { clip: VideoClip }) {
{!clip.card && <ClipBorderSection clip={clip} />}
{!clip.card && <ClipShadowSection clip={clip} />}
<ClipMaskSection clip={clip} />
+ <ClipEffectsSection clip={clip} />
</div>
</>
);
@@ -4500,7 +4503,7 @@ function EffectPanel({ overlay: o }: { overlay: EffectOverlay }) {
<div className="flex flex-col gap-1 px-3.5 pb-4">
{o.effect === "zoom" ? (
<ZoomTarget overlay={o} />
- ) : (
+ ) : AMOUNTLESS_EFFECTS.includes(o.effect) ? null : (
<Row label="Amount">
<ValueSlider
label="Effect amount"
diff --git a/site/src/cut/components/inspectorToolFields.test.ts b/site/src/cut/components/inspectorToolFields.test.ts
index 3a4ff9dc..77c83247 100644
--- a/site/src/cut/components/inspectorToolFields.test.ts
+++ b/site/src/cut/components/inspectorToolFields.test.ts
@@ -32,6 +32,7 @@ const ITEM_TOOLS = new Set([
"set_split_edit",
"set_framing",
"set_clip_style",
+ "set_clip_effects",
"set_camera_card",
"set_speed",
"set_speed_curve",
@@ -65,6 +66,7 @@ const PANEL_SOURCES = [
"SpeedCurveStrip.tsx",
"RemovalPanel.tsx",
"CameraCardSection.tsx",
+ "ClipEffectsSection.tsx",
"AnimationTiles.tsx",
"TransitionsPanel.tsx",
"GroupPanel.tsx",
diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.ts
index 5e5f9576..2573499f 100644
--- a/site/src/cut/lib/aiContext.ts
+++ b/site/src/cut/lib/aiContext.ts
@@ -88,6 +88,7 @@ function clipEffects(clip: VideoClip, live: boolean, frame?: CardFrame) {
? { animOut: { style: clip.animOut.style, seconds: r(clip.animOut.seconds) } }
: {}),
...color,
+ ...(clip.effects?.length ? { effects: clip.effects } : {}),
...(clip.boxStyle ? { boxStyle: clip.boxStyle } : {}),
...describeRemoval(clip, live),
...(clip.card ? { card: describeCameraCard(clip, live, frame) } : {}),
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index f5f4ee9f..93281076 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -28,6 +28,9 @@ import { textSpots } from "./textPlace";
import {
OUTPUT_SPACES,
ALL_EFFECT_IDS,
+ CLIP_EFFECT_IDS,
+ type ClipEffect,
+ type ClipEffectId,
applyLutToImageData,
autoGradeFromImageData,
buildClipLut,
@@ -2972,6 +2975,23 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
};
},
+ set_clip_effects: (s, input) => {
+ const clip = requireItem(s.clips, input.clipId, "video clip");
+ if (!Array.isArray(input.effects)) throw new ToolError("effects must be a list; [] removes them all.");
+
+ // Each entry names a clip effect once, with an optional amount in range.
+ const effects: ClipEffect[] = [];
+ for (const e of input.effects as { effect?: unknown; amount?: unknown }[]) {
+ if (!(CLIP_EFFECT_IDS as unknown[]).includes(e?.effect))
+ throw new ToolError(`effect must be one of ${CLIP_EFFECT_IDS.join(", ")}.`);
+ const effect = e.effect as ClipEffectId;
+ if (effects.some((x) => x.effect === effect)) throw new ToolError(`${effect} is listed twice; a clip wears an effect once.`);
+ effects.push(isNum(e.amount) ? { effect, amount: Math.max(0.05, Math.min(1, e.amount)) } : { effect });
+ }
+ s.updateClip(clip.id, { effects: effects.length ? effects : undefined });
+ return { id: clip.id, effects };
+ },
+
set_clip_style: (s, input) => {
const clip = requireItem(s.clips, input.clipId, "video clip");
if (input.clear === true) {
@@ -6097,6 +6117,7 @@ function sweepCandidates(s: Editor, kind: ItemKind): SweepCandidate[] {
* call out, and every handler stays a single-item write. */
const SWEEP_TARGETS: Readonly<Record<string, string>> = {
set_clip_muted: "clipId",
+ set_clip_effects: "clipId",
set_clip_hidden: "clipId",
set_clip_volume: "clipId",
set_split_edit: "clipId",
diff --git a/site/src/cut/lib/composite.ts b/site/src/cut/lib/composite.ts
index 1cac4c67..77451fb2 100644
--- a/site/src/cut/lib/composite.ts
+++ b/site/src/cut/lib/composite.ts
@@ -19,7 +19,7 @@
* it was reached by playing there or by rendering the 135th frame.
*/
-import { applyDetail, applyLutToImageData, applyMaskToCanvas, detailActive, grainTile, hexToHlgHex, lookCssFilter, lookPost, maskComposite, paintStrokeInk, removalActive, type ClipColorRecipe, type OutputSpace } from "@donkeycut/effects-kit";
+import { applyDetail, applyEffectToCanvas, applyLutToImageData, applyMaskToCanvas, detailActive, grainTile, hexToHlgHex, lookCssFilter, lookPost, maskComposite, paintStrokeInk, removalActive, retimeOf, type ClipColorRecipe, type OutputSpace } from "@donkeycut/effects-kit";
import { cameraCardLayout, cardActive, traceCardPath, type CardLayout } from "./cameraCard";
import { applyDetailGpu } from "./detailGpu";
import { applyLutGpu } from "./gradeGpu";
@@ -94,6 +94,10 @@ export class FrameCompositor {
private gradeLutScratch: Surface | null = null;
private lookScratch: Surface | null = null;
private vignetteCanvas: Surface | null = null;
+ /** The clip's own effects: the picture copies into `clipFxCanvas` and
+ * each effect runs over it, `clipFxScratch` holding its self-copies. */
+ private clipFxCanvas: Surface | null = null;
+ private clipFxScratch: Surface | null = null;
/** The masked/keyframed-layer pass: the layer draws into `layerScratch`,
* its mask's coverage paints into `maskScratch`, a keyframed pose blits
* through `poseScratch`, and the result composites back onto the frame. */
@@ -219,6 +223,8 @@ export class FrameCompositor {
field:
| "gradeCanvas"
| "gradeLutScratch"
+ | "clipFxCanvas"
+ | "clipFxScratch"
| "lookScratch"
| "vignetteCanvas"
| "layerScratch"
@@ -338,17 +344,32 @@ export class FrameCompositor {
* spatial pass, then the look's color pass as a canvas filter, in the order
* the export's chain runs. Source alpha rides through every pass, so
* transparent stills keep their transparency. The look's post passes
- * (vignette, grain, glow…) draw over the composited layer instead.
+ * (vignette, grain, glow…) draw over the composited layer instead. The
+ * clip's own effects run last, in the picture's pixels, so its mask and
+ * pose carry them with it.
*/
private gradedSource(
frame: Extract<Frame, { kind: "ready" }>,
- clip: VideoClip | undefined
+ clip: VideoClip | undefined,
+ at: number
): CanvasImageSource {
const recipe = this.recipeFor(clip, frame.source);
const compiled = this.lutFor(clip, recipe);
const detail = detailActive(clip?.grade) ? clip!.grade! : null;
const lookCss = lookCssFilter(clip?.look, clip?.lookAmount);
- if (!compiled && !detail && !lookCss) return frame.image;
+ const fx = clip?.effects?.length ? clip : null;
+ if (!compiled && !detail && !lookCss && !fx) return frame.image;
+ const graded = this.colorPasses(frame, compiled, detail, lookCss);
+ return fx ? this.clipEffects(graded, frame.width, frame.height, fx, at) : graded;
+ }
+
+ /** The LUT, the spatial pass and the look's color pass, in that order. */
+ private colorPasses(
+ frame: Extract<Frame, { kind: "ready" }>,
+ compiled: ClipLut | null,
+ detail: VideoClip["grade"] | null,
+ lookCss: string
+ ): CanvasImageSource {
const w = frame.width;
const h = frame.height;
let picture: CanvasImageSource = frame.image;
@@ -396,6 +417,26 @@ export class FrameCompositor {
return scratch;
}
+ /** The clip's effects over its picture, in order, on the clip's own clock:
+ * a color cycle turns from the clip's first frame and a flash pops there. */
+ private clipEffects(picture: CanvasImageSource, w: number, h: number, clip: VideoClip, at: number): CanvasImageSource {
+ const { surface: canvas } = this.scratch("clipFxCanvas", w, h);
+ const { surface: scratch } = this.scratch("clipFxScratch", w, h);
+ const ctx = canvas.getContext("2d") as Ctx | null;
+ if (!ctx) return picture;
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.globalAlpha = 1;
+ ctx.globalCompositeOperation = "source-over";
+ ctx.clearRect(0, 0, w, h);
+ ctx.drawImage(picture, 0, 0, w, h);
+ const tLocal = at - clip.start;
+ const len = retimeOf(clip).len;
+ for (const e of clip.effects!) {
+ applyEffectToCanvas(canvas, scratch, e.effect, e.amount, tLocal, grainTile, undefined, undefined, len);
+ }
+ return canvas as CanvasImageSource;
+ }
+
/**
* The clip's graded picture with its background removal applied: the baked
* AI matte the host provides multiplies an alpha into the frame, the
@@ -445,7 +486,7 @@ export class FrameCompositor {
clip: VideoClip | undefined,
at: number
): CanvasImageSource {
- const base = this.gradedSource(frame, clip);
+ const base = this.gradedSource(frame, clip, at);
const r = clip?.removal;
if (!r || !removalActive(r) || (clip && this.removalBypass === clip.id)) return base;
const w = frame.width;
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index 51110f47..39a7d803 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -840,6 +840,7 @@ const cardEntry = (tag: string) => ({
removal: { rgb: `${tag}_rgb.mp4`, alpha: `${tag}_a.mp4` },
grade: undefined,
look: undefined,
+ effects: undefined,
fit: "fill" as const,
zoom: 1,
panX: 0,
@@ -961,6 +962,7 @@ export async function buildExportPayload(
animOut: sp.clip.animOut,
look: sp.clip.look,
lookAmount: sp.clip.lookAmount,
+ effects: sp.clip.effects,
hidden: sp.clip.hidden,
// A still: the server loops the image for the clip's length instead of
// trimming a source span.
@@ -1067,6 +1069,7 @@ export async function buildExportPayload(
pngs.push({ name: `removal_c${i}_a.mp4`, blob: pieces.alpha });
clipEntries[i].removal = { rgb: `removal_c${i}_rgb.mp4`, alpha: `removal_c${i}_a.mp4` };
clipEntries[i].grade = undefined;
+ clipEntries[i].effects = undefined;
}
}
}
@@ -1187,6 +1190,7 @@ export async function buildExportPayload(
color: specColor(assetById.get(c.assetId)),
look: c.look,
lookAmount: c.lookAmount,
+ effects: c.effects,
mask: undefined as SpecMask | undefined,
shadow: undefined as SpecMask | undefined,
kf: posed(c).kf,
@@ -1276,6 +1280,7 @@ export async function buildExportPayload(
entry.removal = { rgb: `removal_ov${i}_rgb.mp4`, alpha: `removal_ov${i}_a.mp4` };
entry.grade = undefined;
entry.look = undefined;
+ entry.effects = undefined;
}
}
}
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 1c050ead..e6a45c53 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -144,6 +144,7 @@ const clipTreatment = (c: VideoClip) => ({
...(c.sound ? { sound: c.sound } : {}),
...(c.grade ? { grade: c.grade } : {}),
...(c.mask ? { mask: c.mask } : {}),
+ ...(c.effects?.length ? { effects: c.effects } : {}),
...(c.boxStyle ? { boxStyle: c.boxStyle } : {}),
...(c.kf?.length ? { kf: c.kf } : {}),
...(c.hidden ? { hidden: true } : {}),
@@ -183,6 +184,7 @@ const templateTreatment = (l: TemplateLayer) => ({
...(l.sound ? { sound: l.sound } : {}),
...(l.grade ? { grade: l.grade } : {}),
...(l.mask ? { mask: l.mask } : {}),
+ ...(l.effects?.length ? { effects: l.effects } : {}),
...(l.boxStyle ? { boxStyle: l.boxStyle } : {}),
...(l.kf?.length ? { kf: l.kf } : {}),
...(l.hidden ? { hidden: true } : {}),
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index 7f15ce8d..05a255d9 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -25,7 +25,7 @@ import {
type TextOverlay as KitTextOverlay,
type WordEffectId,
} from "@donkeycut/effects-kit";
-import type { CodeFormat, SourceProfile } from "@donkeycut/effects-kit";
+import type { ClipEffect, CodeFormat, SourceProfile } from "@donkeycut/effects-kit";
import type { CameraCard } from "./cameraCard";
import { getBackend, type CutBackend } from "./backend";
import type { VideoProject } from "./genvideo/types";
@@ -677,6 +677,9 @@ export interface VideoClip {
look?: LookStyle;
/** Look strength 0..1; absent = 1 (full). */
lookAmount?: number;
+ /** Effects this clip wears over its own picture, inside its mask, in
+ * order (absent = none). */
+ effects?: ClipEffect[];
/** Hidden clips stay on the timeline (grayed) but render as black — excluded
* from the played/exported picture without disturbing the layout. */
hidden?: boolean;
@@ -1183,6 +1186,7 @@ export interface TemplateLayer {
* removal stays behind, its matte baked from the footage it was cut from. */
grade?: ColorGrade;
mask?: Mask;
+ effects?: ClipEffect[];
boxStyle?: BoxStyle;
kf?: OverlayKey[];
hidden?: boolean;
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 16a2ddee..37b5ba00 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -387,12 +387,12 @@ Titles (add_title / update_overlay): text, size (frame px; the design short side
Shapes (add_shape): rect, ellipse, line, arrow. w/h are frame fractions; rect/ellipse take fill + fill_opacity, corner radius (rect), and an outline (stroke_color/stroke_width); line/arrow draw in \`fill\` with h as their thickness, and rotation gives them their direction (0° points right, 90° points down).
Stickers (add_sticker): a project image asset (generated cutouts in the Elements panel carry origin "sticker"); w is a frame-width fraction and height follows the source's aspect unless h sets it. Lottie JSON uploads become animated stickers that loop for the element's duration.
Custom stickers (create_sticker): one call generates the image, removes the background, adds the die-cut outline, and places it — use it when the user asks for a sticker of something they don't have (it spends credits like image generation).
-Effects (add_effect): time-ranged treatments over the finished cut. Three families, one list: the picture treatments (${EFFECT_TREATMENTS.join(", ")}), the graded looks (${LOOK_EFFECTS.join(", ")}), and the audio treatments (${AUDIO_EFFECT_IDS.join(", ")}). A picture effect treats the footage and everything laid over it; an audio effect treats everything audible under its window at once — clip sound, upper-track sound and the soundtrack — so "make this bit sound like a phone call", "put an echo on that line", or "muffle the music here" is an effect element over that stretch, and a per-clip volume, fade or duck is the audio tools instead. A handover on the sound at a cut belongs to the transition on that cut — the audiocross style — not here. Placed like any element and tuned with \`amount\` (update_overlay), and trims like anything else: a picture effect dropped over a clip opens covering that clip, while an audio effect opens at the standard length wherever it was placed — a treatment on the sound is aimed at a moment, so give it the window you mean. One effect at a time reads best, amounts under ~0.5. Effects are stylized post treatments on a stretch of timeline; pure color — exposure, warmth, a cinematic or film palette on one clip — is the per-clip grading tools instead (color-grading skill), whose preset catalog is separate from the look effects.
+Effects (add_effect): time-ranged treatments over the finished cut. Three families, one list: the picture treatments (${EFFECT_TREATMENTS.join(", ")}), the graded looks (${LOOK_EFFECTS.join(", ")}), and the audio treatments (${AUDIO_EFFECT_IDS.join(", ")}). A picture effect treats the footage and everything laid over it, and set_clip_effects puts one on a single clip, inside its mask; an audio effect treats everything audible under its window at once — clip sound, upper-track sound and the soundtrack — so "make this bit sound like a phone call", "put an echo on that line", or "muffle the music here" is an effect element over that stretch, and a per-clip volume, fade or duck is the audio tools instead. A handover on the sound at a cut belongs to the transition on that cut — the audiocross style — not here. Placed like any element and tuned with \`amount\` (update_overlay), and trims like anything else: a picture effect dropped over a clip opens covering that clip, while an audio effect opens at the standard length wherever it was placed — a treatment on the sound is aimed at a moment, so give it the window you mean. One effect at a time reads best, amounts under ~0.5. Effects are stylized post treatments on a stretch of timeline; pure color — exposure, warmth, a cinematic or film palette on one clip — is the per-clip grading tools instead (color-grading skill), whose preset catalog is separate from the look effects.
Advanced text (update_overlay): italic, align (multi-line), letter_spacing (em), line_height, wrap_width (frame-width fraction; side grips wrap text, height follows content; corners scale type proportionally), stroke_color/stroke_width (em outline behind the fill), and richer shadows. Bundled Google families join the system set — ${GOOGLE_FONT_IDS.join(", ")} — and so do the user's own: a .ttf/.otf/.woff/.woff2 file dropped on the Library is offered in every project under a font id like "font:<key>". Editor_state lists the ones this account has under \`fonts\` — use an id from there instead of guessing at a family name, and when someone asks for a typeface that is not in the list, tell them the Library takes font files.
Animation (set_overlay_animation): every element takes preset In/Out ramps (${OVERLAY_ANIM_STYLE_IDS.join(", ")} — ${TEXT_ONLY_ANIM_STYLE_IDS.join(", ")} for titles only; the per-letter styles sweep a title character by character; ${DIVE_ANIM_STYLE_IDS.join(", ")} flies into the last letter until its ink fills the frame, the built-in zoom-through transition), a Loop (${OVERLAY_LOOP_STYLE_IDS.join(", ")}) that runs its whole duration, and a Move that says what it does WHILE it holds. A title takes one more: Words (${WORD_EFFECT_IDS.join(", ")}), which plays the line word by word — an emphasis travelling along a line that is fully up, or a build assembling the line as it is spoken. A typewriter entrance takes a caret (caret, caret_blink, caret_blinks, caret_hide), the blinking bar of a typed comment or search box. Every element takes a Hit (${OVERLAY_HIT_STYLE_IDS.join(", ")}) at hit_at seconds inside it: press is a finger tapping a button — the Post or Like button of a call-to-action card — and a click from stock_search kind "sound" placed at the same moment makes it heard. Entrances 0.3–0.6s read well; one loop per scene, sparingly; a move on the lines that matter. All of them are slots on the element, applied internally, and they compose over its keyframe track. A slot is the reach when the motion is per-letter or per-word — typewriter, the glyph sweeps, the word effects — which one pose track cannot express; what the whole element does is keyed.
Keyframes (set_overlay_keyframes): for motion no preset covers — a path across the frame, a slow push in, a drift that holds first. A key is a whole pose (position, scale, rotation, opacity, blur) at a time in seconds from the element's start; the pose moves between keys along each key's ease (sine.inOut, power2/power3 in/out, default linear) and holds outside them, and presets still compose on top. This is the default way to animate an element: a key names the exact pose at the exact second, where a named ramp only approximates it. Two or three keys carry almost everything. In the UI the diamond in the inspector adds a key at the playhead, and dragging the element then records into it. set_clip_keyframes is the same track on a video clip (any track): pans, push-ins, picture-in-picture flights, spins, and fades on the footage itself.
Camera over a board of cards (set_group_camera): lay the cards out across the frame, group them, and key one camera that steps from card to card, pushes onto the one being talked about, pulls back, or whips — the cards keep their own keys inside it. Depth of field is the neighbours' own keys: blur and dim them at the moments the camera pushes in. motion_blur on the camera (or on an element, through update_overlay) streaks fast moves and leaves still frames sharp.
-Masks (set_mask): trim any element or video clip to a shape — ${MASK_SHAPE_KINDS.join(", ")} — with feather, invert, rounded corners on a box, and its own keyframe track for reveals and wipes. The pen kind takes an outline as points, for any region the fixed shapes miss: a doorway, a screen — read the region's corners off a captured frame in frame fractions, and stack a masked copy of the shot (a filter, a title) so it shows through only there. A region that moves with a hand, the window between the hands, a head or a person is track_motion's: use "mask" keys a pen outline onto it on every frame, one item for the whole span; use "follow" carries an element along with it. The subject kind trims by the person in the shot (on-device segmentation; preview and export match): inverted sits the item behind the speaker, plain keeps it only on the speaker. Behind-speaker reads well when the speaker is clearly separated from the background; with no detectable person it degrades to the plain picture. Group and Ungroup are in the timeline toolbar (⌘/Ctrl+G and ⇧⌘/Ctrl+G).
+Masks (set_mask): trim any element or video clip to a shape — ${MASK_SHAPE_KINDS.join(", ")} — with feather, invert, rounded corners on a box, and its own keyframe track for reveals and wipes. The pen kind takes an outline as points, for any region the fixed shapes miss: a doorway, a screen — read the region's corners off a captured frame in frame fractions, and stack a masked copy of the shot, treated by set_clip_effects or a grade, so the treatment shows only there. A region that moves with a hand, the window between the hands, a head or a person is track_motion's: use "mask" keys a pen outline onto it on every frame, one item for the whole span; use "follow" carries an element along with it. The subject kind trims by the person in the shot (on-device segmentation; preview and export match): inverted sits the item behind the speaker, plain keeps it only on the speaker. Behind-speaker reads well when the speaker is clearly separated from the background; with no detectable person it degrades to the plain picture. Group and Ungroup are in the timeline toolbar (⌘/Ctrl+G and ⇧⌘/Ctrl+G).
Composite builds: these primitives compose. Stacked overlay video copies, masks and their keys, pose keyframes, staggered starts, and shapes construct effects beyond any preset list. Given a reference (a screenshot or clip of an effect the user wants), reverse-engineer it:
1. Read the reference as layers. For each visible piece, decide what it is: live footage, a cropped or repeated slice of the footage, a solid graphic, text, or empty background.
2. Map each layer to a primitive. A slice of the picture = a copy of the same asset on its own overlay track (add_overlay_video, layout full, matching trim so every copy's picture lines up with the base) trimmed by a mask whose x/y offsets place it — a full-frame copy under a mask shows the true part of the picture at that spot, so masked copies tile back into the whole image. Gap lines and borders come from sizing masks under their spacing (track 0 left empty plays black through the gaps) or from stroked shapes. Solid panels are shapes; words are titles.
diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.ts
index b57aa8c5..d8b53185 100644
--- a/site/src/cut/server/exportPipeline.test.ts
+++ b/site/src/cut/server/exportPipeline.test.ts
@@ -564,6 +564,49 @@ describe("clip masks in the filtergraph", () => {
expect(xfadeMismatches(g)).toEqual([]);
});
+ test("a masked overlay's own effects run on its clock, before the pad and the mask", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 6 })],
+ overlayVideos: [
+ {
+ file: "ov.mp4",
+ in: 0,
+ out: 2,
+ start: 1,
+ track: 1,
+ muted: true,
+ effects: [{ effect: "negative" }, { effect: "huecycle", amount: 0.5 }],
+ frame: { x: 0.5, y: 0.5, w: 0.5, h: 0.5 },
+ mask: { file: "mask_ov0.png" },
+ },
+ ],
+ });
+ const joined = g.join(";");
+ // Both effects gate on the segment's own clock, 0 to its 2s length, in
+ // the order the clip lists them.
+ const negAt = g.findIndex((c) => c.includes("lutyuv=y='minval+maxval-val'") && c.includes("gte(t,0)*lt(t,2)"));
+ const hueAt = g.findIndex((c) => c.includes("hue=H='2*PI*0.8*(t-0)'"));
+ const padAt = g.findIndex((c) => c.includes("pad=540:960") && c.includes("[email protected]"));
+ expect(negAt).toBeGreaterThanOrEqual(0);
+ expect(hueAt).toBeGreaterThan(negAt);
+ expect(padAt).toBeGreaterThan(hueAt);
+ expect(joined).toContain("alphamerge");
+ expect(xfadeMismatches(g)).toEqual([]);
+ });
+
+ test("a track-0 clip's effects treat its picture ahead of the letterbox bars", async () => {
+ const g = await graphFor({
+ clips: [clip("a.mp4", { out: 3, effects: [{ effect: "negative" }] }), clip("b.mp4", { out: 2 })],
+ });
+ const fxAt = g.findIndex((c) => c.includes("lutyuv=y='minval+maxval-val'"));
+ expect(fxAt).toBeGreaterThanOrEqual(0);
+ // The chain the effect hands on to pads after it, and only the first clip
+ // wears it.
+ expect(g.slice(fxAt).some((c) => c.includes("[cw0fo0]null") && c.includes("pad=1080:1920"))).toBe(true);
+ expect(g.filter((c) => c.includes("lutyuv=y='minval+maxval-val'"))).toHaveLength(1);
+ expect(xfadeMismatches(g)).toEqual([]);
+ });
+
test("a mask under head/tail alpha fades keeps both", async () => {
const g = await graphFor({
clips: [clip("a.mp4", { out: 6 })],
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index 7528ef01..b4d45fd7 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -16,7 +16,7 @@ import { shiftSpan, type SpecSound } from "../lib/soundSource";
import { masterRawMix, packStems } from "./exportAudio";
import type { StemDef } from "../lib/stems";
import { CLIP_MAX_ZOOM, regionPx, TRANSITION_XFADE, TRANSITION_ZOOM, type ColorGrade, type TransitionStyle } from "../lib/types";
-import { audioFxFilters, buildClipLut, buildTransferLut, CLARITY_EPS, compositeSpaceFor, detailActive, detailGain, detailRadius, effectFilterLines, graphicsToHlg, hexToHlgHex, hlgToPq, isAudioEffect, lookFilterLines, lutToCube, mirrorRetimable, parseLutFile, recipeKey, retimeOf, shortestTurn, slowRuns, srcSpan, sortedKeys, soundFilters, type ChainChroma, type ClipColorRecipe, type ClipSound, type CodeFormat, type OutputSpace, type OverlayKey, type ParsedLut, type Retime, type SpeedNode } from "@donkeycut/effects-kit";
+import { audioFxFilters, buildClipLut, buildTransferLut, CLARITY_EPS, compositeSpaceFor, detailActive, detailGain, detailRadius, effectFilterLines, graphicsToHlg, hexToHlgHex, hlgToPq, isAudioEffect, lookFilterLines, lutToCube, mirrorRetimable, parseLutFile, recipeKey, retimeOf, shortestTurn, slowRuns, srcSpan, sortedKeys, soundFilters, type ChainChroma, type ClipColorRecipe, type ClipEffect, type ClipSound, type CodeFormat, type OutputSpace, type OverlayKey, type ParsedLut, type Retime, type SpeedNode } from "@donkeycut/effects-kit";
// The render pipeline itself: spec in, finished mp4 out. Shared by the local
// engine's job registry (jobs.ts) and the cloud render worker, which stage
@@ -184,6 +184,8 @@ export interface ExportSpec {
* spec carries only the id — the chain is built server-side). */
look?: string;
lookAmount?: number;
+ /** Effects the clip wears over its own picture, on its own clock. */
+ effects?: ClipEffect[];
/** Hidden clips keep their slot but render black + silent. */
hidden?: boolean;
/** A still image: looped for the clip's length instead of trimmed. */
@@ -292,6 +294,8 @@ export interface ExportSpec {
* overlays may carry alpha the look chain would flatten). */
look?: string;
lookAmount?: number;
+ /** Effects the clip wears over its own picture, under its mask. */
+ effects?: ClipEffect[];
/** Client-painted grayscale coverage trimming this overlay's picture,
* box-sized (a letterboxed segment pads out to its box when masked);
* `subject` trims by the shared person matte instead. */
@@ -1340,6 +1344,19 @@ export async function runExport(
const filters: string[] = [];
+ // A clip's own effects run on its segment's clock, from 0 to its length,
+ // over its framed picture: chain fragment `core` in, the treated fragment
+ // out. The fades, pad and mask that follow carry them, as in the preview.
+ const wearEffects = (core: string, effects: ClipEffect[] | undefined, len: number, w: number, h: number, pixFmt: string, tag: string) => {
+ for (const [n, e] of (effects ?? []).entries()) {
+ const lines = effectFilterLines(`${tag}fi${n}`, `${tag}fo${n}`, e.effect, e.amount, 0, len, w, h, `${tag}x${n}`, undefined, undefined, { ...chroma, pixFmt });
+ if (!lines) continue;
+ filters.push(`${core}[${tag}fi${n}]`, ...lines);
+ core = `[${tag}fo${n}]null`;
+ }
+ return core;
+ };
+
// One color mapping per clip: the source conversion, the library LUT and
// the grade baked into one 3D LUT (effects-kit colorPipeline.ts), written
// to the job dir as .cube once per distinct recipe and applied with lut3d.
@@ -2056,7 +2073,10 @@ export async function runExport(
// picture before the letterbox pad, so the bars stay the frame color.
let core = framedTimebase(timebase, frame, `c${j}`, rmIn ? {} : c, rt, fps, filters);
if (plan) core = plan.run(core, `c${j}`);
- core += `,format=${segFmt}${padding}`;
+ core += `,format=${segFmt}`;
+ // The clip's effects treat its picture, ahead of the letterbox bars.
+ core = wearEffects(core, c.effects, dur, W, H, segFmt, `cw${j}`);
+ core += padding;
// The look bakes in after grade + framing, before the edge effects, so
// animations move already-graded pixels (matching the preview order). A
// removal clip's look runs after the flatten below instead — the chain's
@@ -2482,6 +2502,7 @@ export async function runExport(
core = `[olko${k}]null`;
}
}
+ core = wearEffects(core, oc.effects, olen, boxW, boxH, lookFmt, `ow${k}`);
if (boxPad) core += boxPad;
// The border ring lands after the look and box pad, before the edge
// ramps, so it fades and masks with the clip like the preview.