Open source CapCut alternative.
TypeScript
feat(site): flash a cutout in, a frame at a time [rebuild]
David Lee83ed195c26a
diff --git a/evals/cut-perf.latest-report.json b/evals/cut-perf.latest-report.jsonindex f2a1f059..f4316029 100644--- a/evals/cut-perf.latest-report.json+++ b/evals/cut-perf.latest-report.json@@ -51,10 +51,10 @@},"frameMs": 16.7,"scrub": {- "p50": 15.7,- "p95": 32.6,- "mean": 18.12,- "max": 33,+ "p50": 15.6,+ "p95": 32.3,+ "mean": 18.16,+ "max": 32.4,"n": 41},"longTasks": {@@ -64,7 +64,7 @@"max": 0,"n": 0},- "liveSamples": 56,+ "liveSamples": 64,"machine": "desktop"},{@@ -81,10 +81,10 @@},"frameMs": 16.7,"scrub": {- "p50": 15.7,- "p95": 32.5,- "mean": 17.98,- "max": 48.2,+ "p50": 15.8,+ "p95": 32.1,+ "mean": 17.64,+ "max": 32.5,"n": 41},"longTasks": {@@ -94,7 +94,7 @@"max": 0,"n": 0},- "liveSamples": 63,+ "liveSamples": 57,"machine": "desktop"},{@@ -111,10 +111,10 @@},"frameMs": 16.7,"scrub": {- "p50": 15.6,- "p95": 16.4,- "mean": 15.91,- "max": 33.3,+ "p50": 15.7,+ "p95": 16.5,+ "mean": 16.02,+ "max": 33.1,"n": 41},"longTasks": {@@ -124,7 +124,7 @@"max": 0,"n": 0},- "liveSamples": 68,+ "liveSamples": 78,"machine": "desktop"},{@@ -135,16 +135,16 @@"late": {"p50": 0,"p95": 0,- "mean": 0.02,+ "mean": 0.01,"max": 1,"n": 81},"frameMs": 16.7,"scrub": {- "p50": 15.7,- "p95": 16.3,- "mean": 15.45,- "max": 32.6,+ "p50": 15.9,+ "p95": 16.1,+ "mean": 15.61,+ "max": 32.4,"n": 81},"longTasks": {@@ -172,9 +172,9 @@"frameMs": 16.7,"scrub": {"p50": 15.8,- "p95": 32.6,- "mean": 18.38,- "max": 33.1,+ "p95": 32,+ "mean": 18.5,+ "max": 32.9,"n": 41},"longTasks": {@@ -184,7 +184,7 @@"max": 0,"n": 0},- "liveSamples": 45,+ "liveSamples": 47,"machine": "desktop"},{@@ -203,8 +203,8 @@"scrub": {"p50": 15.8,"p95": 32.4,- "mean": 18.53,- "max": 32.7,+ "mean": 18.5,+ "max": 32.6,"n": 41},"longTasks": {@@ -214,7 +214,7 @@"max": 0,"n": 0},- "liveSamples": 46,+ "liveSamples": 47,"machine": "desktop"},{@@ -231,10 +231,10 @@},"frameMs": 16.7,"scrub": {- "p50": 31.9,- "p95": 33.3,- "mean": 30.34,- "max": 33.4,+ "p50": 32.4,+ "p95": 32.7,+ "mean": 30.71,+ "max": 33.1,"n": 41},"longTasks": {@@ -289,13 +289,13 @@"pass": true,"notes": [],"drops": 0,- "presented": 810,+ "presented": 809,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 810+ "n": 809},"stallShare": 0,"decay": {@@ -326,13 +326,13 @@"pass": true,"notes": [],"drops": 0,- "presented": 3233,+ "presented": 3234,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 3233+ "n": 3234},"stallShare": 0,"decay": {@@ -344,17 +344,17 @@"reported": 0.04},"longTasks": {- "p50": 0,- "p95": 0,- "mean": 0,- "max": 0,- "n": 0+ "p50": 84,+ "p95": 84,+ "mean": 84,+ "max": 84,+ "n": 1},"liveSamples": 120,"liveSources": 12,"keptSources": 20,"warmMb": 354,- "memMb": 1110,+ "memMb": 1111,"machine": "desktop"},{@@ -437,13 +437,13 @@"pass": true,"notes": [],"drops": 0,- "presented": 3442,+ "presented": 3444,"lag": {"p50": 0.01,"p95": 0.01,"mean": 0.01,"max": 0.02,- "n": 3442+ "n": 3444},"stallShare": 0,"decay": {@@ -455,11 +455,11 @@"reported": 0.04},"longTasks": {- "p50": 50,- "p95": 50,- "mean": 50,- "max": 50,- "n": 1+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0},"liveSamples": 10,"liveSources": 1,@@ -492,11 +492,11 @@"reported": 0.04},"longTasks": {- "p50": 63,- "p95": 63,- "mean": 63,- "max": 63,- "n": 1+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0},"liveSamples": 10,"liveSources": 1,@@ -513,9 +513,9 @@"drops": 0,"presented": 3444,"lag": {- "p50": 0.01,+ "p50": 0.02,"p95": 0.03,- "mean": 0.01,+ "mean": 0.02,"max": 0.03,"n": 3444},@@ -529,11 +529,11 @@"reported": 0.04},"longTasks": {- "p50": 51,- "p95": 51,- "mean": 51,- "max": 51,- "n": 1+ "p50": 56,+ "p95": 56,+ "mean": 54.5,+ "max": 56,+ "n": 2},"liveSamples": 10,"liveSources": 1,@@ -585,13 +585,13 @@"pass": true,"notes": [],"drops": 0,- "presented": 3444,+ "presented": 3443,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 3444+ "n": 3443},"stallShare": 0,"decay": {@@ -691,8 +691,8 @@"bucket": "playback","pass": true,"notes": [],- "drops": 2,- "presented": 1094,+ "drops": 3,+ "presented": 1095,"lag": {"p50": 0.02,"p95": 0.03,@@ -702,7 +702,7 @@},"stallShare": 0,"decay": {- "first": 0.005,+ "first": 0.008,"last": 0},"clock": {@@ -710,10 +710,10 @@"reported": 0.04},"longTasks": {- "p50": 153,- "p95": 153,- "mean": 109,- "max": 153,+ "p50": 150,+ "p95": 150,+ "mean": 111,+ "max": 150,"n": 2},"liveSamples": 10,diff --git a/site/packages/effects-kit/src/clipEffects.test.ts b/site/packages/effects-kit/src/clipEffects.test.tsindex c154fd60..db7e251b 100644--- a/site/packages/effects-kit/src/clipEffects.test.ts+++ b/site/packages/effects-kit/src/clipEffects.test.ts@@ -28,6 +28,21 @@ describe("a clip's effects over a transparent picture", () => {}});+describe("a flash a cutout wears", () => {+ test("fading from its tone lands the cutout as a solid silhouette and clears across the clip", () => {+ // The next shot's subject cut out and flashed in: on the clip's first+ // frame it is the tone alone, inside its own edges; by its end, itself.+ const fade: ClipEffect[] = [{ effect: "flash", amount: 1, fade: "from" }];+ const first = logo();+ applyClipEffects(first, scratch(), fade, 0, noGrain, 0.2);+ expect(pixel(first, S / 2, S / 2)).toEqual([255, 255, 255, 255]);+ expect(pixel(first, 2, 2)[3]).toBe(0);+ const last = logo();+ applyClipEffects(last, scratch(), fade, 0.2, noGrain, 0.2);+ expect(pixel(last, S / 2, S / 2)).toEqual([128, 128, 128, 255]);+ });+});+describe("filter effects in a row", () => {test("draw as one joined filter with the result of drawing each in turn", () => {const list: ClipEffect[] = [{ effect: "negative" }, { effect: "huecycle", amount: 0.5 }];diff --git a/site/packages/effects-kit/src/effects.ts b/site/packages/effects-kit/src/effects.tsindex b16e2288..c3cc5ac8 100644--- a/site/packages/effects-kit/src/effects.ts+++ b/site/packages/effects-kit/src/effects.ts@@ -1523,7 +1523,9 @@ export function applyClipEffects(scratch: HTMLCanvasElement | OffscreenCanvas,effects: readonly ClipEffect[],tLocal: number,- grainTileFor: (tick: number) => CanvasImageSource | null+ grainTileFor: (tick: number) => CanvasImageSource | null,+ /** The clip's length on the same clock, which a flash's fade runs across. */+ dur?: number): void {let filters: string[] = [];const flush = () => {@@ -1531,13 +1533,13 @@ export function applyClipEffects(filters = [];};for (const e of effects) {- const { cssFilter, ...passes } = effectPreviewState(e.effect, e.amount, tLocal, undefined, undefined, undefined, e);+ const { cssFilter, ...passes } = effectPreviewState(e.effect, e.amount, tLocal, undefined, undefined, dur, e);if (Object.values(passes).every((v) => v === undefined)) {if (cssFilter) filters.push(cssFilter);continue;}flush();- applyEffectToCanvas(canvas, scratch, e.effect, e.amount, tLocal, grainTileFor, undefined, undefined, undefined, "keep", e);+ applyEffectToCanvas(canvas, scratch, e.effect, e.amount, tLocal, grainTileFor, undefined, undefined, dur, "keep", e);}flush();}diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.tsindex d90f7925..3645d884 100644--- a/site/src/cut/components/Inspector.tools.ts+++ b/site/src/cut/components/Inspector.tools.ts@@ -35,6 +35,7 @@ import {PEN_MIN_POINTS,CLIP_EFFECT_IDS,FLASH_RHYTHMS,+ FLASH_FADES,FLASH_RATE_MAX,FLASH_RATE_MIN,FLASH_TONES,@@ -412,7 +413,7 @@ export const INSPECTOR_TOOLS = [{name: "set_clip_effects",description:- `Set the effects a video clip wears over its own picture (${CLIP_EFFECT_IDS.join(", ")}). They treat that clip alone — inside its mask, under its pose — for its whole length, on the clip's own clock: a masked copy over the shot wearing negative and huecycle shows a color-cycling negative window while the shot around it plays untouched. add_effect places an element that treats the whole frame under its window. Pass the full list in order; [] removes them all. amount 0.05..1 (default 0.5) is the strength; for huecycle it is how fast the hue turns (0.5 is a turn every 1.25s); negative has none, but takes a strobe rate: a cutout copy wearing negative at rate ${FLASH_RATE_MAX} flashes between negative and normal every frame. A flash takes add_effect's tone, rate and rhythm on the clip's own clock: a white flicker at rate 12–15, amount 0.15–0.25, pops a masked shot's exposure unevenly while the black around the mask stays black.`,+ `Set the effects a video clip wears over its own picture (${CLIP_EFFECT_IDS.join(", ")}). They treat that clip alone — inside its mask, under its pose — for its whole length, on the clip's own clock: a masked copy over the shot wearing negative and huecycle shows a color-cycling negative window while the shot around it plays untouched. add_effect places an element that treats the whole frame under its window. Pass the full list in order; [] removes them all. amount 0.05..1 (default 0.5) is the strength; for huecycle it is how fast the hue turns (0.5 is a turn every 1.25s); negative has none, but takes a strobe rate: a cutout copy wearing negative at rate ${FLASH_RATE_MAX} flashes between negative and normal every frame. A flash takes add_effect's tone, rate, rhythm and fade on the clip's own clock: a white flicker at rate 12–15, amount 0.15–0.25, pops a masked shot's exposure unevenly while the black around the mask stays black. A subject flashed in over a shot — the next shot's subject cut out (set_removal) on a track above, landing as a solid white silhouette, then itself, then blown out, then itself — is that cutout split into a clip per frame of the flicker: fade "from" at amount 1 for the silhouette, a pop at 0.6-0.9 for each blown-out frame, nothing for the frames that show it plain.`,inputSchema: obj({clipId: str("Video clip id"),effects: {@@ -424,6 +425,7 @@ export const INSPECTOR_TOOLS = [tone: { type: "string", enum: [...FLASH_TONES], description: 'flash only: "white" (default) or "black"' },rate: num(`flash and negative: strobe pulses a second, ${FLASH_RATE_MIN}..${FLASH_RATE_MAX}, each held for its first half (${FLASH_RATE_MAX} at 30 fps flips every other frame); omit for one pop at the clip's start, or a negative flipped throughout`),rhythm: { type: "string", enum: [...FLASH_RHYTHMS], description: 'flash with a rate only: "strobe" (default) or "flicker", each pulse its own strength, some skipped' },+ fade: { type: "string", enum: [...FLASH_FADES], description: 'flash only: ramp evenly across the clip — "from" starts as the tone alone and clears, "to" ends on it. At amount 1 a one-frame clip fading from white is a solid silhouette of what it shows' },}, ["effect"]),},}, ["clipId", "effects"]),diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 5bc1107d..d7e41cb3 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -112,6 +112,7 @@ import {FLASH_RATE_MAX,FLASH_RATE_MIN,FLASH_FADE_CHOICES,+ FLASH_FADES,FLASH_FADE_NONE,FLASH_RHYTHMS,loopPeriod,@@ -3950,7 +3951,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// a flash also takes its tone, strobe rate and rhythm, a negative its// strobe rate.const effects: ClipEffect[] = [];- for (const e of input.effects as { effect?: unknown; amount?: unknown; tone?: unknown; rate?: unknown; rhythm?: unknown }[]) {+ for (const e of input.effects as { effect?: unknown; amount?: unknown; tone?: unknown; rate?: unknown; rhythm?: unknown; fade?: unknown }[]) {const effect = CLIP_EFFECT_IDS.find((id) => id === e?.effect);if (!effect) throw new ToolError(`effect must be one of ${CLIP_EFFECT_IDS.join(", ")}.`);if (effects.some((x) => x.effect === effect)) throw new ToolError(`${effect} is listed twice; a clip wears an effect once.`);@@ -3970,6 +3971,11 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {if (rhythm) {entry.rhythm = rhythm;}+ // A fade runs across the clip: "from" lands it as the tone alone.+ if (e.fade !== undefined && e.fade !== FLASH_FADE_NONE) {+ if (!(FLASH_FADES as readonly unknown[]).includes(e.fade)) throw new ToolError(`fade is one of ${FLASH_FADE_CHOICES.join(", ")}.`);+ entry.fade = e.fade as FlashFade;+ }}effects.push(entry);}diff --git a/site/src/cut/lib/composite.ts b/site/src/cut/lib/composite.tsindex c680503e..cd851e00 100644--- a/site/src/cut/lib/composite.ts+++ b/site/src/cut/lib/composite.ts@@ -25,6 +25,7 @@ import { applyDetailGpu } from "./detailGpu";import { applyLutGpu } from "./gradeGpu";import { clipRecipe, peekClipLut, requestClipLut, type ClipLut, type RecipeSource } from "./lutBuild";import { createRasterCanvas } from "./raster";+import { clipLen } from "./timelineEnd";import {bandEdge,CLOCK_SWEEP,@@ -471,7 +472,10 @@ export class FrameCompositor {ctx.globalCompositeOperation = "source-over";ctx.clearRect(0, 0, w, h);ctx.drawImage(picture, src.x, src.y, src.w, src.h, 0, 0, w, h);- applyClipEffects(canvas, scratch, clip.effects ?? [], at - clip.start + (clip.effectsFrom ?? 0), grainTile);+ // The effects' clock starts where the unsplit clip did, so a fade runs+ // across all of it, as the export's does.+ const from = clip.effectsFrom ?? 0;+ applyClipEffects(canvas, scratch, clip.effects ?? [], at - clip.start + from, grainTile, from + clipLen(clip));return canvas;}diff --git a/site/src/cut/lib/effectUpdateTool.test.ts b/site/src/cut/lib/effectUpdateTool.test.tsindex 77db956a..3e4434b3 100644--- a/site/src/cut/lib/effectUpdateTool.test.ts+++ b/site/src/cut/lib/effectUpdateTool.test.ts@@ -60,6 +60,13 @@ describe("update_overlay on an effect", () => {expect(useEditor.getState().clips[0].effects).toEqual([{ effect: "negative", rate: 15 }]);});+ test("a cutout clip's flash fades from its tone, the silhouette it lands as", async () => {+ useEditor.setState({ clips: [{ id: "c1", assetId: "a", start: 0, in: 0, out: 2, speed: 1 } as never] });+ await runAiTool("set_clip_effects", { clipId: "c1", effects: [{ effect: "flash", amount: 1, fade: "from" }] });+ expect(useEditor.getState().clips[0].effects).toEqual([{ effect: "flash", amount: 1, fade: "from" }]);+ expect(await errorOf(() => runAiTool("set_clip_effects", { clipId: "c1", effects: [{ effect: "flash", fade: "out" }] }))).toContain("fade is one of");+ });+test("a flash's fade comes off again with none, back to the pop", async () => {const { id } = (await runAiTool("add_effect", { effect: "flash", start: 2, end: 3, fade: "to" })) as { id: string };expect(overlay(id)).toMatchObject({ fade: "to" });diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 7325e79a..ed75ce1d 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -358,7 +358,7 @@ Someone points at a video — a file they imported, a link you brought in, a cli**Lay down the sound, then block out the cuts.** Cuts timed to a reference are timed to its sound: detach_audio with the reference's \`assetId\` puts its whole track on the soundtrack and none of its picture. add_clip with \`blocks\` lays the skeleton from \`shots\` in one call — one block per shot on track 0, each its shot's \`from\` and \`to\` as read so every cut lands on its frame, flash cuts included, \`color\` what sample_color reads behind it and \`label\` naming what the person drops there. The label rides the timeline chip; the frame shows the colour alone. Footage dropped on a block, or replace_item, takes its place and its length, so everything built on top keeps its timing. When the ask is the reference's own picture playing, add_clip \`spans\` lays its shots down instead, one span per shot at those same times.-**Fill the shots from Pexels, when asked.** The reference's own frames never go in the cut; Pexels' free footage does. Search it with WebFetch — the one web tool you have, and it reaches only Pexels: https://www.pexels.com/search/videos/<words>/?orientation=landscape (portrait for a 9:16 cut), asking for every result's link, id and title. Search per shot by what it shows — the subject and what they wear, the action, the angle, the light, the time of day — and look again with other words when a page has nothing close; one creator's series often covers a whole template, so a good hit's neighbours are worth a search of their own. import_url \`urls\` takes the video page links (pexels.com/video/<slug>-<id>/) and brings each down as its 1080p file, the Pexels id in its name. Watch the candidates beside the reference shot and keep the one that matches subject, angle and motion — a candidate is footage to judge, not a source to replicate: one scan pass over the stretch you would use settles it, and the whole-watch and word-for-word reading above are the reference's alone; replace_item puts it on that shot's block, trim_clip sets its in so the motion lands where the reference's does, set_framing zooms and pans to the reference's framing, and match_color_grade with ref_asset_id the reference and ref_at a moment inside the shot grades it to the reference's frame. Pexels files carry their own sound: when the reference's music is the soundtrack, set_clip_muted with \`ids\` mutes every footage clip in one call, so only the music plays. A shot whose camera moves — a push, a punch-in on the beat, a shake, a tilt — is measured with track_camera over that shot of the reference and keyed on its clip with set_clip_keyframes from the \`path\` and \`moves\` it returns. When it says \`still\`, the reference froze a frame and moved it: freeze_frame at a timeline second inside the shot's clip, for the shot's length, at that clip's index, delete_item the moving clip, and key the moves on the still. An overlay that shows the next shot's subject over the current one is that shot's clip again, on a track above. Name every Pexels id you used when you close.+**Fill the shots from Pexels, when asked.** The reference's own frames never go in the cut; Pexels' free footage does. Search it with WebFetch — the one web tool you have, and it reaches only Pexels: https://www.pexels.com/search/videos/<words>/?orientation=landscape (portrait for a 9:16 cut), asking for every result's link, id and title. Search per shot by what it shows — the subject and what they wear, the action, the angle, the light, the time of day — and look again with other words when a page has nothing close; one creator's series often covers a whole template, so a good hit's neighbours are worth a search of their own. import_url \`urls\` takes the video page links (pexels.com/video/<slug>-<id>/) and brings each down as its 1080p file, the Pexels id in its name. Watch the candidates beside the reference shot and keep the one that matches subject, angle and motion — a candidate is footage to judge, not a source to replicate: one scan pass over the stretch you would use settles it, and the whole-watch and word-for-word reading above are the reference's alone; replace_item puts it on that shot's block, trim_clip sets its in so the motion lands where the reference's does, set_framing zooms and pans to the reference's framing, and match_color_grade with ref_asset_id the reference and ref_at a moment inside the shot grades it to the reference's frame. Pexels files carry their own sound: when the reference's music is the soundtrack, set_clip_muted with \`ids\` mutes every footage clip in one call, so only the music plays. A shot whose camera moves — a push, a punch-in on the beat, a shake, a tilt — is measured with track_camera over that shot of the reference and keyed on its clip with set_clip_keyframes from the \`path\` and \`moves\` it returns. When it says \`still\`, the reference froze a frame and moved it: freeze_frame at a timeline second inside the shot's clip, for the shot's length, at that clip's index, delete_item the moving clip, and key the moves on the still. An overlay that shows the next shot's subject over the current one is that shot's clip again on a track above, its subject extracted with set_removal. Such a cutout is usually flashed in on its own picture, a frame at a time: a solid white silhouette, then the subject, then blown out, then the subject. Read those frames one by one, split the cutout a clip per frame, and give each its set_clip_effects flash: fade \"from\" at amount 1 for the silhouette, a pop at 0.6-0.9 for a blown-out frame, none for a plain one. A cutout that flickers between the subject and its negative every frame is one clip with a negative effect strobing at half the timeline's frame rate, which flips it each frame. Name every Pexels id you used when you close.**Rebuild the text as elements.** One title per card, a stretch of cards per add_title call — the words exactly as they read, size in px at 1080 wide, and x/y where suggest_font sets it on that card — a title is set from the middle of its line, so the centre of its ink box (track_element, match ink, the text's colour) sits off it by the face's own offset. A run of lines timed one after another is add_text_sequence in one call. Match the case the reference uses, and its colour from sample_color. The face comes off the frame: suggest_font with the card's moment, a region close around its words and the words themselves ranks every face Donkey has and hands back the font, weight, size, tracking and x/y to set. Call it on each card: cards in one face share its weight, size and tracking, but each sits its own way in its space, so its x/y comes from its own read, never from a neighbour's place. A title that cycles through typefaces is one call per face it shows. A no_match verdict means the reference uses a face Donkey does not bundle: say so, and set the closest one it named. Without a suggestion, size comes off the ink: capitals stand about 0.7 of the size tall, so size is the ink height in px at 1080 wide ÷ 0.7 (h × 1920 ÷ 0.7 on 9:16). A cycle whose faces change on the music's beats times itself with set_overlay_animation font_cycle_beats on the soundtrack clip after detect_beats; one whose changes follow the reference's own rhythm takes font_cycle_switches, the timeline seconds detect_cuts reads each change at.