Skip to content

Open source CapCut alternative.

TypeScript

feat(site): fill a replicated template from Pexels and measure its camera [rebuild]

David Lecfdb2fa84ccf
diff --git a/evals/cut-perf.latest-report.json b/evals/cut-perf.latest-report.json
index a8f1de8b..f2a1f059 100644
--- a/evals/cut-perf.latest-report.json
+++ b/evals/cut-perf.latest-report.json
@@ -51,10 +51,10 @@
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.3,
- "p95": 31.8,
- "mean": 17.99,
- "max": 33.7,
+ "p50": 15.7,
+ "p95": 32.6,
+ "mean": 18.12,
+ "max": 33,
"n": 41
},
"longTasks": {
@@ -64,7 +64,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 55,
+ "liveSamples": 56,
"machine": "desktop"
},
{
@@ -81,10 +81,10 @@
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.9,
+ "p50": 15.7,
"p95": 32.5,
- "mean": 17.68,
- "max": 33,
+ "mean": 17.98,
+ "max": 48.2,
"n": 41
},
"longTasks": {
@@ -94,7 +94,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 53,
+ "liveSamples": 63,
"machine": "desktop"
},
{
@@ -112,9 +112,9 @@
"frameMs": 16.7,
"scrub": {
"p50": 15.6,
- "p95": 17.3,
- "mean": 15.83,
- "max": 30.9,
+ "p95": 16.4,
+ "mean": 15.91,
+ "max": 33.3,
"n": 41
},
"longTasks": {
@@ -124,7 +124,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 56,
+ "liveSamples": 68,
"machine": "desktop"
},
{
@@ -141,10 +141,10 @@
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.4,
- "p95": 17.3,
- "mean": 15.73,
- "max": 32.9,
+ "p50": 15.7,
+ "p95": 16.3,
+ "mean": 15.45,
+ "max": 32.6,
"n": 81
},
"longTasks": {
@@ -165,16 +165,16 @@
"late": {
"p50": 0,
"p95": 1,
- "mean": 0.2,
- "max": 2,
+ "mean": 0.17,
+ "max": 1,
"n": 41
},
"frameMs": 16.7,
"scrub": {
- "p50": 16,
- "p95": 32.7,
- "mean": 18.98,
- "max": 52,
+ "p50": 15.8,
+ "p95": 32.6,
+ "mean": 18.38,
+ "max": 33.1,
"n": 41
},
"longTasks": {
@@ -184,7 +184,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 46,
+ "liveSamples": 45,
"machine": "desktop"
},
{
@@ -201,10 +201,10 @@
},
"frameMs": 16.7,
"scrub": {
- "p50": 15.7,
- "p95": 32.9,
- "mean": 18.45,
- "max": 33.9,
+ "p50": 15.8,
+ "p95": 32.4,
+ "mean": 18.53,
+ "max": 32.7,
"n": 41
},
"longTasks": {
@@ -214,7 +214,7 @@
"max": 0,
"n": 0
},
- "liveSamples": 44,
+ "liveSamples": 46,
"machine": "desktop"
},
{
@@ -232,9 +232,9 @@
"frameMs": 16.7,
"scrub": {
"p50": 31.9,
- "p95": 33.7,
- "mean": 30.29,
- "max": 33.9,
+ "p95": 33.3,
+ "mean": 30.34,
+ "max": 33.4,
"n": 41
},
"longTasks": {
@@ -326,13 +326,13 @@
"pass": true,
"notes": [],
"drops": 0,
- "presented": 3239,
+ "presented": 3233,
"lag": {
"p50": 0.02,
"p95": 0.03,
"mean": 0.02,
"max": 0.03,
- "n": 3239
+ "n": 3233
},
"stallShare": 0,
"decay": {
@@ -437,13 +437,13 @@
"pass": true,
"notes": [],
"drops": 0,
- "presented": 3444,
+ "presented": 3442,
"lag": {
"p50": 0.01,
- "p95": 0.02,
+ "p95": 0.01,
"mean": 0.01,
"max": 0.02,
- "n": 3444
+ "n": 3442
},
"stallShare": 0,
"decay": {
@@ -455,11 +455,11 @@
"reported": 0.04
},
"longTasks": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "max": 0,
- "n": 0
+ "p50": 50,
+ "p95": 50,
+ "mean": 50,
+ "max": 50,
+ "n": 1
},
"liveSamples": 10,
"liveSources": 1,
@@ -492,11 +492,11 @@
"reported": 0.04
},
"longTasks": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "max": 0,
- "n": 0
+ "p50": 63,
+ "p95": 63,
+ "mean": 63,
+ "max": 63,
+ "n": 1
},
"liveSamples": 10,
"liveSources": 1,
@@ -515,7 +515,7 @@
"lag": {
"p50": 0.01,
"p95": 0.03,
- "mean": 0.02,
+ "mean": 0.01,
"max": 0.03,
"n": 3444
},
@@ -529,11 +529,11 @@
"reported": 0.04
},
"longTasks": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "max": 0,
- "n": 0
+ "p50": 51,
+ "p95": 51,
+ "mean": 51,
+ "max": 51,
+ "n": 1
},
"liveSamples": 10,
"liveSources": 1,
@@ -691,18 +691,18 @@
"bucket": "playback",
"pass": true,
"notes": [],
- "drops": 3,
- "presented": 1095,
+ "drops": 2,
+ "presented": 1094,
"lag": {
"p50": 0.02,
"p95": 0.03,
- "mean": 0.01,
+ "mean": 0.02,
"max": 0.03,
"n": 1092
},
"stallShare": 0,
"decay": {
- "first": 0.008,
+ "first": 0.005,
"last": 0
},
"clock": {
@@ -712,7 +712,7 @@
"longTasks": {
"p50": 153,
"p95": 153,
- "mean": 113.5,
+ "mean": 109,
"max": 153,
"n": 2
},
diff --git a/site/packages/effects-kit/src/effects.test.ts b/site/packages/effects-kit/src/effects.test.ts
index 8d1c5d29..ae0c64c7 100644
--- a/site/packages/effects-kit/src/effects.test.ts
+++ b/site/packages/effects-kit/src/effects.test.ts
@@ -55,6 +55,15 @@ describe("effect recipes", () => {
expect(effectPreviewState("negative", 1, 0).cssFilter).toBe("invert(1)");
});
+ test("a negative with a rate strobes: flipped for the first half of each pulse", () => {
+ // Rate 15 at 30 fps: every other frame flipped, the picture between.
+ const at = (frame: number) => effectPreviewState("negative", 0.5, frame / 30, undefined, undefined, 2, { rate: 15 }).cssFilter;
+ expect([0, 1, 2, 3, 4].map(at)).toEqual(["invert(1)", "", "invert(1)", "", "invert(1)"]);
+ // The export gates its flip by the same pulse.
+ const lines = effectFilterLines("in", "out", "negative", 0.5, 1, 2, 540, 960, "t", undefined, undefined, undefined, { rate: 15 })!;
+ expect(lines.join(";")).toMatch(/enable='[^']*lt\(mod\(\(t-1\)\*15\+/);
+ });
+
test("a color cycle turns the hue at the amount's pace, from its own start", () => {
// Amount 0.5 is 0.8 turns a second: a quarter second in is 72 degrees.
expect(effectPreviewState("huecycle", 0.5, 0).cssFilter).toBe("hue-rotate(0deg)");
diff --git a/site/packages/effects-kit/src/effects.ts b/site/packages/effects-kit/src/effects.ts
index 65915b7d..b16e2288 100644
--- a/site/packages/effects-kit/src/effects.ts
+++ b/site/packages/effects-kit/src/effects.ts
@@ -1156,9 +1156,12 @@ export function effectPreviewState(
zoom: 1 + (amp * 2) / 1080,
};
}
- case "negative":
+ case "negative": {
// Every color flipped to its opposite: black to white, skin to cyan.
- return { cssFilter: "invert(1)" };
+ // With a rate it strobes, flipped for the first half of each pulse.
+ const rate = flashRate(pulse);
+ return { cssFilter: !rate || flashOn(tLocal, rate) ? "invert(1)" : "" };
+ }
case "huecycle": {
// The hue wheel turns at the amount's pace from the element's start.
const deg = (360 * hueTurns(k) * Math.max(0, tLocal)) % 360;
@@ -1900,12 +1903,18 @@ export function effectFilterLines(
`[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass${chroma.overlay}[${outLabel}]`,
];
}
- case "negative":
+ case "negative": {
// Each plane flips about its own legal range, so black and white swap
// exactly in a limited-range chain and the chroma mirrors about grey.
+ // A strobe's flip is gated to the first half of each pulse.
+ const rate = flashRate(pulse);
+ const on = rate
+ ? `enable='gte(t,${edge(start)})*lt(t,${edge(end)})*lt(mod((t-${fmt(start)})*${fmt(rate)}+${PULSE_NUDGE},1),0.5)'`
+ : en;
return [
- `[${inLabel}]lutyuv=y='minval+maxval-val':u='minval+maxval-val':v='minval+maxval-val':${en}[${outLabel}]`,
+ `[${inLabel}]lutyuv=y='minval+maxval-val':u='minval+maxval-val':v='minval+maxval-val':${on}[${outLabel}]`,
];
+ }
case "huecycle":
// The canvas recipe's turning wheel, in radians on the frame's clock.
return [`[${inLabel}]hue=H='2*PI*${fmt(hueTurns(k))}*(t-${fmt(start)})':${en}[${outLabel}]`];
diff --git a/site/src/clients/chatgpt/server/catalog.ts b/site/src/clients/chatgpt/server/catalog.ts
index 0f7ca893..81b882ca 100644
--- a/site/src/clients/chatgpt/server/catalog.ts
+++ b/site/src/clients/chatgpt/server/catalog.ts
@@ -60,6 +60,7 @@ export const READ_COMMANDS = new Set([
"compare_to_source",
"sample_color",
"track_element",
+ "track_camera",
"suggest_font",
"listen_audio",
"detect_silence",
diff --git a/site/src/clients/chatgpt/server/guidance.ts b/site/src/clients/chatgpt/server/guidance.ts
index 97f72066..33487def 100644
--- a/site/src/clients/chatgpt/server/guidance.ts
+++ b/site/src/clients/chatgpt/server/guidance.ts
@@ -1,9 +1,14 @@
import { AI_SKILLS } from "@/cut/server/ai/catalog";
import { DOC_EXPORT_PRESETS } from "@/cut/lib/exportPresets";
-/** Translate the shared catalog's state-read reference for this client. */
+/** Translate the shared catalog's references for this client: its state
+ * read, its import, and the editor chat's Pexels-only web fetch, which
+ * ChatGPT's own browsing stands in for. */
export function chatgptDescription(text: string): string {
- return text.replace(/\bget_state\b/g, "inspect_project");
+ return text
+ .replace(/\bget_state\b/g, "inspect_project")
+ .replace(/\bimport_url\b/g, "import_media")
+ .replace(/Search it with WebFetch — the one web tool you have, and it reaches only Pexels:/g, "Search it on the web:");
}
// These guides describe editing operations shared by every client. Additions
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.ts
index e558c2d6..4197a485 100644
--- a/site/src/cut/components/AiPanel.tools.ts
+++ b/site/src/cut/components/AiPanel.tools.ts
@@ -21,6 +21,7 @@ import {
SAMPLE_COLOR_MAX,
SAMPLE_COLORS_K_MAX,
TRACK_FRAMES_MAX,
+ CAMERA_FRAMES_MAX,
TRACK_MATCHES,
TRACK_TOLERANCE_DEFAULT,
} from "@/cut/lib/measure/limits";
@@ -301,6 +302,20 @@ export const AI_PANEL_TOOLS = [
["asset_id", "from", "seed"]
),
},
+ {
+ name: "track_camera",
+ description:
+ `Measure how the camera moves over a stretch of a video, inside its picture: each frame's zoom, shift and turn against the frame on \`from\`, so a punch-in, a push, a beat zoom pulse or a shake is keyed with set_clip_keyframes to the reference's own numbers. Give one shot's stretch (SOURCE seconds; a cut inside it breaks the read). \`path\` lists every frame: t, scale (1.08 is 8% closer), dx/dy (how far the picture's centre moved, fractions of the frame, + is right and down), rotation (degrees clockwise) and residual (what the move leaves unexplained, 0..255 a pixel; null on a frame too blurred to read). A frame whose picture moves in two layers — a cutout held still over a background that shakes, or pulsing over a still one — also has \`second\`, the other layer's pose and the \`share\` of the picture it holds: build it as two tracks, the footage keyed with one pose and its cutout (set_removal) above it keyed with the other. The keys copy it straight across: a clip key at the same moment with scale, rotation, and x = 0.5 + dx, y = 0.5 + dy — relative to the first frame, so start the clip's own pose there. \`moves\` lists each move of each number with from/to, start/end, the ease that fits it (${EASE_IDS.join(", ")}) and \`error\`; a pulse is a move out and a move back. \`still\` true means one picture holds under the whole move — residual under 6 throughout: the reference froze that frame and moved it, so freeze_frame the shot's clip at \`from\` and key the moves on the freeze. A live shot with a moving camera leaves a higher residual: its numbers are still the move. Up to ${CAMERA_FRAMES_MAX} frames a call; when \`truncated\`, continue from \`coveredTo\`.`,
+ inputSchema: obj(
+ {
+ asset_id: str("Project asset id (video)"),
+ from: num("Source second the stretch starts: the frame every other is measured against"),
+ to: num("Source second it ends, inside the same shot"),
+ picture: pictureArg,
+ },
+ ["asset_id", "from", "to"]
+ ),
+ },
{
name: "suggest_font",
description:
@@ -359,7 +374,7 @@ export const AI_PANEL_TOOLS = [
{
name: "import_url",
description:
- "Read a web or media URL. For Donkey folder links use read_folder; for Donkey project links use read_project. Supports TikTok, YouTube, Instagram Reels, an X/Twitter post or Article, an ordinary web page, or a direct video/audio/image link — with the bundled downloader and import what it holds into the project. Free and local. A web page comes back as its article text plus the pictures on it; a post as its video or photos; and the source's own words (returned as sourceText) are quoted for the user beside the media automatically — don't retype them in your reply. A source that is only words returns sourceText with no assets, which is a success: read it and answer from it. This is how you look something up: point it at the page and read what comes back. When the user wants only the sound — a song, a soundtrack, a podcast — pass audio_only and the source's audio track lands as an audio asset. Media lands on a card in this chat and the user drags it to the timeline, Media, or the Library; place it yourself (add_clip) only when they asked for it in the cut. A short clip downloads in seconds; a long video can take a couple of minutes.",
+ "Read a web or media URL. For Donkey folder links use read_folder; for Donkey project links use read_project. Supports TikTok, YouTube, Instagram Reels, an X/Twitter post or Article, an ordinary web page, a Pexels video page (pexels.com/video/<slug>-<id>/ — it comes down as its 1080p file, named with its Pexels id) or one of its files (videos.pexels.com/video-files/<id>/…), or a direct video/audio/image link — with the bundled downloader and import what it holds into the project. Free and local. A web page comes back as its article text plus the pictures on it; a post as its video or photos; and the source's own words (returned as sourceText) are quoted for the user beside the media automatically — don't retype them in your reply. A source that is only words returns sourceText with no assets, which is a success: read it and answer from it. This is how you look something up: point it at the page and read what comes back. When the user wants only the sound — a song, a soundtrack, a podcast — pass audio_only and the source's audio track lands as an audio asset. Media lands on a card in this chat and the user drags it to the timeline, Media, or the Library; place it yourself (add_clip) only when they asked for it in the cut. A short clip downloads in seconds; a long video can take a couple of minutes.",
inputSchema: obj(
{
url: str("The page or media URL to download"),
diff --git a/site/src/cut/components/EffectsPanel.tools.ts b/site/src/cut/components/EffectsPanel.tools.ts
index 9a3dc61b..f41d277f 100644
--- a/site/src/cut/components/EffectsPanel.tools.ts
+++ b/site/src/cut/components/EffectsPanel.tools.ts
@@ -34,7 +34,7 @@ export const EFFECT_SETTING_ARGS = {
enum: [...FLASH_TONES],
description: 'flash only: "white" pops the picture bright (default), "black" dips it dark',
},
- rate: num(`flash only: strobe pulses a second, ${FLASH_RATE_MIN}..${FLASH_RATE_MAX}; each pulse holds the tone for its first half. Omit for one pop at the start`),
+ rate: num(`flash and negative: strobe pulses a second, ${FLASH_RATE_MIN}..${FLASH_RATE_MAX}; each pulse holds the tone (or the flip) for its first half — ${FLASH_RATE_MAX} at 30 fps is every other frame. Omit for one pop at the start (a negative: flipped throughout)`),
rhythm: {
type: "string",
enum: [...FLASH_RHYTHMS],
@@ -51,7 +51,7 @@ export const EFFECTS_TOOLS = [
{
name: "add_effect",
description:
- `Add a time-ranged effect element over the cut: a zoom into part of the frame, a picture treatment (${EFFECT_TREATMENTS.join(", ")}), a graded look (${LOOK_EFFECTS.join(", ")}), or an audio treatment (${AUDIO_EFFECT_IDS.join(", ")}) over the sound. A picture effect filters everything visible under its window (start..end); an audio effect treats everything audible under it — clip sound, upper tracks and soundtrack together. They stack. glitch is clean footage broken by runs of corrupted frames — dark pixel-stretch smears, the picture doubled sideways over an orange burn, teal close-ups, blown-out flashes — the glitch ending of a teaser, where amount is how often it hits. A glitch pinned to one kind (glitch "smear", "split", "tint", "blowout" or "rgb") hits with it on every frame; "rgb" pulls the color channels apart, red down and blue up — a 2–3 frame rgb glitch is the RGB-split hit of a collapsing title or HUD. lightleak drifts a warm bloom and light streaks over the picture for as long as it is up; with leak "burn" it is a film burn played once across the element's own length: it climbs from the bottom edge behind a hot orange front, flares near white, settles orange and hands the picture back in its last tenth — the burn-out of a teaser, a 0.4–0.6s element over the end of the last shot, amount 1 for a fully opaque burn. With leak "scorch" it flashes film burn through the picture: every frame maps the picture onto a saturated hot tone by brightness — yellow, orange, or red over pink — across the whole frame or a soft-edged patch of it, a new tone and patch each frame; short 2–12 frame scorches over the last shots, amount 1, are the burn-through flicker of a teaser's ending. With leak "film" it is the film-strip flash over a cut, played once across the element: the picture washes warm orange, a brighter band glows across the lower middle and the strip's sprocket holes blaze cream along the top edge, full through the middle of the element and gone by its ends — a 3–5 frame element centered on a cut, amount 1, is the film-strip light-leak transition of a recap montage (one per section change). With leak "char" it is the burn's climb in dark stock: a hot orange front rises from the bottom edge, the frame cools through ember to black behind it, a thin bright light line crosses, and the picture comes back in the last tenth — never a pale flare; an 8–10 frame element, amount 1, over a black card's end and the next shot's first frames is the dark film-burn handover of a recap opener. lightbars is the streaming-service sting: the camera flies down a ribbon of glowing vertical lines (reds, oranges and pinks on the left, blues on the right) across the element's own length — it opens as one narrow warm column just left of the frame's middle (x about 0.46, a twelfth of the frame wide), bursts out to fill the frame about a quarter of the way in, keeps flowing outward as deeper lines spread after the near ones, flares red and pink about three quarters in, then leaves one red line just right of the middle that widens and sweeps off the right edge as the element ends, on black. Bars add light onto the picture, so over black (no clip under it) they are the whole picture; a 1.6–1.7s element, amount 1, right after a logo reveal whose last red vertical stroke stands where the column opens. flash pops the picture white at the element's start and decays in about 0.4s; tone "black" dips it dark instead, and a strobe rate holds the tone on and off that many times a second for the whole element — a black strobe at 5–6 with amount 0.9–1 is the dark beat-flicker over a teaser's last shot. rhythm "flicker" deals each pulse its own strength and leaves some dark: an uneven exposure flicker, a white flicker at rate 12–15 and amount 0.15–0.25. To flicker one shot inside its mask, give the clip the flash through set_clip_effects. Tasteful default: one effect at a time, amount under ~0.5.`,
+ `Add a time-ranged effect element over the cut: a zoom into part of the frame, a picture treatment (${EFFECT_TREATMENTS.join(", ")}), a graded look (${LOOK_EFFECTS.join(", ")}), or an audio treatment (${AUDIO_EFFECT_IDS.join(", ")}) over the sound. A picture effect filters everything visible under its window (start..end); an audio effect treats everything audible under it — clip sound, upper tracks and soundtrack together. They stack. glitch is clean footage broken by runs of corrupted frames — dark pixel-stretch smears, the picture doubled sideways over an orange burn, teal close-ups, blown-out flashes — the glitch ending of a teaser, where amount is how often it hits. A glitch pinned to one kind (glitch "smear", "split", "tint", "blowout" or "rgb") hits with it on every frame; "rgb" pulls the color channels apart, red down and blue up — a 2–3 frame rgb glitch is the RGB-split hit of a collapsing title or HUD. lightleak drifts a warm bloom and light streaks over the picture for as long as it is up; with leak "burn" it is a film burn played once across the element's own length: it climbs from the bottom edge behind a hot orange front, flares near white, settles orange and hands the picture back in its last tenth — the burn-out of a teaser, a 0.4–0.6s element over the end of the last shot, amount 1 for a fully opaque burn. With leak "scorch" it flashes film burn through the picture: every frame maps the picture onto a saturated hot tone by brightness — yellow, orange, or red over pink — across the whole frame or a soft-edged patch of it, a new tone and patch each frame; short 2–12 frame scorches over the last shots, amount 1, are the burn-through flicker of a teaser's ending. With leak "film" it is the film-strip flash over a cut, played once across the element: the picture washes warm orange, a brighter band glows across the lower middle and the strip's sprocket holes blaze cream along the top edge, full through the middle of the element and gone by its ends — a 3–5 frame element centered on a cut, amount 1, is the film-strip light-leak transition of a recap montage (one per section change). With leak "char" it is the burn's climb in dark stock: a hot orange front rises from the bottom edge, the frame cools through ember to black behind it, a thin bright light line crosses, and the picture comes back in the last tenth — never a pale flare; an 8–10 frame element, amount 1, over a black card's end and the next shot's first frames is the dark film-burn handover of a recap opener. lightbars is the streaming-service sting: the camera flies down a ribbon of glowing vertical lines (reds, oranges and pinks on the left, blues on the right) across the element's own length — it opens as one narrow warm column just left of the frame's middle (x about 0.46, a twelfth of the frame wide), bursts out to fill the frame about a quarter of the way in, keeps flowing outward as deeper lines spread after the near ones, flares red and pink about three quarters in, then leaves one red line just right of the middle that widens and sweeps off the right edge as the element ends, on black. Bars add light onto the picture, so over black (no clip under it) they are the whole picture; a 1.6–1.7s element, amount 1, right after a logo reveal whose last red vertical stroke stands where the column opens. flash pops the picture white at the element's start and decays in about 0.4s; tone "black" dips it dark instead, and a strobe rate holds the tone on and off that many times a second for the whole element — a black strobe at 5–6 with amount 0.9–1 is the dark beat-flicker over a teaser's last shot. rhythm "flicker" deals each pulse its own strength and leaves some dark: an uneven exposure flicker, a white flicker at rate 12–15 and amount 0.15–0.25. To flicker one shot inside its mask, give the clip the flash through set_clip_effects. negative flips every colour for the whole element; with a rate it strobes between the negative and the picture, and at ${FLASH_RATE_MAX} on a 30 fps cut it alternates every frame. Tasteful default: one effect at a time, amount under ~0.5.`,
inputSchema: obj({
effect: { type: "string", enum: [...ALL_EFFECT_IDS], description: "Effect id" },
start: num("Start time s (default: playhead)"),
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index ff263756..d90f7925 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -412,7 +412,7 @@ export const INSPECTOR_TOOLS = [
{
name: "set_clip_effects",
description:
- `Set the effects a video clip wears over its own picture (${CLIP_EFFECT_IDS.join(", ")}). They treat that clip alone — inside its mask, under its pose — for its whole length, on the clip's own clock: a masked copy over the shot wearing negative and huecycle shows a color-cycling negative window while the shot around it plays untouched. add_effect places an element that treats the whole frame under its window. Pass the full list in order; [] removes them all. amount 0.05..1 (default 0.5) is the strength; for huecycle it is how fast the hue turns (0.5 is a turn every 1.25s); negative has none. A flash takes add_effect's tone, rate and rhythm on the clip's own clock: a white flicker at rate 12–15, amount 0.15–0.25, pops a masked shot's exposure unevenly while the black around the mask stays black.`,
+ `Set the effects a video clip wears over its own picture (${CLIP_EFFECT_IDS.join(", ")}). They treat that clip alone — inside its mask, under its pose — for its whole length, on the clip's own clock: a masked copy over the shot wearing negative and huecycle shows a color-cycling negative window while the shot around it plays untouched. add_effect places an element that treats the whole frame under its window. Pass the full list in order; [] removes them all. amount 0.05..1 (default 0.5) is the strength; for huecycle it is how fast the hue turns (0.5 is a turn every 1.25s); negative has none, but takes a strobe rate: a cutout copy wearing negative at rate ${FLASH_RATE_MAX} flashes between negative and normal every frame. A flash takes add_effect's tone, rate and rhythm on the clip's own clock: a white flicker at rate 12–15, amount 0.15–0.25, pops a masked shot's exposure unevenly while the black around the mask stays black.`,
inputSchema: obj({
clipId: str("Video clip id"),
effects: {
@@ -422,7 +422,7 @@ export const INSPECTOR_TOOLS = [
effect: { type: "string", enum: [...CLIP_EFFECT_IDS], description: "Effect id" },
amount: num("Strength 0.05..1 (default 0.5)"),
tone: { type: "string", enum: [...FLASH_TONES], description: 'flash only: "white" (default) or "black"' },
- rate: num(`flash only: strobe pulses a second, ${FLASH_RATE_MIN}..${FLASH_RATE_MAX}; omit for one pop at the clip's start`),
+ rate: num(`flash and negative: strobe pulses a second, ${FLASH_RATE_MIN}..${FLASH_RATE_MAX}, each held for its first half (${FLASH_RATE_MAX} at 30 fps flips every other frame); omit for one pop at the clip's start, or a negative flipped throughout`),
rhythm: { type: "string", enum: [...FLASH_RHYTHMS], description: 'flash with a rate only: "strobe" (default) or "flicker", each pulse its own strength, some skipped' },
}, ["effect"]),
},
@@ -600,10 +600,11 @@ export const INSPECTOR_TOOLS = [
{
name: "match_color_grade",
description:
- "Match a video clip's color to a reference, computed from the pixels: the grade sliders for tone, the ends of the range, white balance and saturation are fitted so the clip's frame, rendered through its real color pipeline, lands on the reference's lightness distribution and per-band color in CIELAB. The fitted sliders carry the whole look, so the match replaces the clip's entire grade, preset and LUT included. The reference is ref_asset_id (an attached or imported image) or ref_clip_id (another clip's frame). Use when the user shares a look to copy or asks to match two shots; refine afterward with the other grading tools, checking read_color_stats.",
+ "Match a video clip's color to a reference, computed from the pixels: the grade sliders for tone, the ends of the range, white balance and saturation are fitted so the clip's frame, rendered through its real color pipeline, lands on the reference's lightness distribution and per-band color in CIELAB. The fitted sliders carry the whole look, so the match replaces the clip's entire grade, preset and LUT included. The reference is ref_asset_id (an attached or imported image, or a video with ref_at, the second of it to match) or ref_clip_id (another clip's frame). A video reference reads inside the picture its measuring tools found, so a replication grades each shot's footage to the reference's own frame of that shot — ref_at a moment inside the shot, away from flashes and overlays. Use when the user shares a look to copy or asks to match two shots; refine afterward with the other grading tools, checking read_color_stats.",
inputSchema: obj({
clipId: str("Video clip to grade"),
- ref_asset_id: str("Reference image asset id"),
+ ref_asset_id: str("Reference asset id: an image, or a video read at ref_at"),
+ ref_at: num("Source second of a video ref_asset_id to match — a moment inside the shot the clip stands in for"),
strength: num(
"0..1, default 1: how far to take the match. Lower it when the reference shows other content than the clip (a bright window for a dark room, a black foreground the clip lacks) and the full match lifts or crushes tones the reference never had"
),
diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.ts
index 48dd8c85..97a9f25d 100644
--- a/site/src/cut/lib/aiContext.ts
+++ b/site/src/cut/lib/aiContext.ts
@@ -960,6 +960,7 @@ function describeOverlay(o: Overlay) {
...(o.effect === "glitch" && o.glitch ? { glitch: o.glitch } : {}),
...(o.effect === "flash" && o.tone ? { tone: o.tone } : {}),
...(o.effect === "flash" && o.rate ? { rate: r(o.rate), rhythm: o.rhythm ?? "strobe" } : {}),
+ ...(o.effect === "negative" && o.rate ? { rate: r(o.rate) } : {}),
...(o.effect === "flash" && o.fade ? { fade: o.fade } : {}),
};
}
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 2a5ddac9..5bc1107d 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -243,9 +243,9 @@ import { fontKind, fontLook } from "./fontCatalog";
import { CUT_FONT_MATCH } from "./fontMatchSettings";
import { COMPARE_MODES, MAX_COMPARE, SWEEP_TARGET } from "./frameDiff";
import { compareMoments, sweepSource } from "./sourceCompare";
-import { findPicture, sampleColors, scanCuts, trackElement, type FrameBox } from "./measure/sourceMeasure";
+import { findPicture, sampleColors, scanCuts, sourcePictureAt, trackCamera, trackElement, type FrameBox } from "./measure/sourceMeasure";
import { FULL_FRAME, isFullFrame } from "./measure/picture";
-import { DETECT_CHANGES_MAX, DETECT_CUTS_MAX, DETECT_CUTS_SPAN_MAX, SAMPLE_COLOR_MAX, SAMPLE_COLORS_K_MAX, TRACK_FRAMES_MAX, TRACK_MATCHES, TRACK_PATH_MAX, TRACK_TOLERANCE_DEFAULT } from "./measure/limits";
+import { DETECT_CHANGES_MAX, DETECT_CUTS_MAX, DETECT_CUTS_SPAN_MAX, SAMPLE_COLOR_MAX, SAMPLE_COLORS_K_MAX, TRACK_FRAMES_MAX, CAMERA_FRAMES_MAX, TRACK_MATCHES, TRACK_PATH_MAX, TRACK_TOLERANCE_DEFAULT } from "./measure/limits";
import { renderStageFrame, storeStageStill } from "./stageFrame";
import { createRasterCanvas, decodeRasterImageUrl, rasterCanvasToDataUrl } from "./raster";
import { buildAiContext, describeCameraCard, describeDoc, describeParkedTimeline, hiddenFromChat, OTHER_CHATS_MEDIA_NOTE, roundT } from "./aiContext";
@@ -573,7 +573,7 @@ async function readGradeOf(clip: VideoClip): Promise<ReadGrade | undefined> {
async function colorStatsForRef(
s: ReturnType<typeof useEditor.getState>,
- ref: { clipId?: string; assetId?: string; graded?: boolean }
+ ref: { clipId?: string; assetId?: string; at?: number; graded?: boolean }
): Promise<ColorStats> {
if (ref.clipId) {
const clip = requireItem(s.clips, ref.clipId, "video clip");
@@ -586,10 +586,21 @@ async function colorStatsForRef(
}
if (ref.assetId) {
const asset = requireItem(s.assets, ref.assetId, "project asset");
+ if (asset.type === "video") {
+ // A reference video reads at a second, inside the picture its measuring
+ // tools found, so a screen recording's grade is the shot's, not the app's.
+ if (!isNum(ref.at))
+ throw new ToolError("A video reference needs ref_at, the second of it to match (a moment inside the shot).");
+ const at = clamp(ref.at, 0, Math.max(0, asset.duration - 0.001));
+ const px = await withSweepPaused(() => sourcePictureAt(asset.url, at, asset.picture ?? FULL_FRAME)).catch((e) => {
+ throw new ToolError(e instanceof Error ? e.message : `Could not read "${asset.name}".`, { cause: e });
+ });
+ const stats = colorStatsFromImageData(px.data);
+ if (!stats) throw new ToolError(`Could not read pixels off "${asset.name}" at ${roundT(at)}s.`);
+ return stats;
+ }
if (asset.type !== "image")
- throw new ToolError(
- "Color stats read images and clip frames — for a video asset, place it on the timeline and pass its clip id."
- );
+ throw new ToolError("Color stats read images, video frames and clip frames — that asset has no picture.");
const img = await decodeRasterImageUrl(asset.url);
if (!img || !img.width) throw new ToolError("Could not decode that image.");
const w = 192;
@@ -1845,6 +1856,57 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
};
},
+ track_camera: async (s, input) => {
+ const asset = measuredSource(s, input, "video");
+ const end = Math.max(0, asset.duration - 0.001);
+ const from = clamp(isNum(input.from) ? input.from : 0, 0, end);
+ if (!isNum(input.to) || input.to <= from) throw new ToolError("Pass to, the source second the stretch ends, after from.");
+ const to = clamp(input.to, 0, asset.duration);
+ const area = pictureArea(asset, input);
+ const run = await withSweepPaused(() => trackCamera(asset.url, { from, to, area, maxFrames: CAMERA_FRAMES_MAX })).catch((e) => {
+ throw new ToolError(e instanceof Error ? e.message : `Could not read "${asset.name}".`, { cause: e });
+ });
+ if (run.path.length < 2) throw new ToolError(`Only ${run.path.length} frame between ${from}s and ${to}s — give a longer stretch.`);
+ const r3 = (v: number) => Math.round(v * 1000) / 1000;
+ return {
+ ...(run.fps > 0 ? { fps: Math.round(run.fps * 100) / 100 } : {}),
+ framesRead: run.path.length,
+ still: run.still,
+ path: run.path.map((p) => ({
+ t: roundT(p.t),
+ scale: r3(p.scale),
+ dx: r3(p.dx),
+ dy: r3(p.dy),
+ rotation: Math.round(p.rotation * 10) / 10,
+ residual: Number.isFinite(p.residual) ? Math.round(p.residual * 10) / 10 : null,
+ ...(p.second
+ ? {
+ second: {
+ scale: r3(p.second.scale),
+ dx: r3(p.second.dx),
+ dy: r3(p.second.dy),
+ rotation: Math.round(p.second.rotation * 10) / 10,
+ share: Math.round(p.second.share * 100) / 100,
+ },
+ }
+ : {}),
+ })),
+ moves: run.moves.map((m) => ({
+ prop: m.prop,
+ from: roundT(m.from),
+ to: roundT(m.to),
+ start: m.prop === "rotation" ? Math.round(m.start * 10) / 10 : r3(m.start),
+ end: m.prop === "rotation" ? Math.round(m.end * 10) / 10 : r3(m.end),
+ ease: m.ease,
+ error: Math.round(m.error * 1000) / 1000,
+ })),
+ coveredTo: roundT(run.coveredTo),
+ truncated: run.truncated,
+ ...pictureReport(area),
+ source: sourceRef(asset),
+ };
+ },
+
watch_video: async (s, input) => {
// A look at a reference project's footage decides a mapping and writes
// nothing: no note owed, no frame map kept, no sweep started.
@@ -1895,6 +1957,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
detail,
at: aimed,
shots: shotMoments.get(asset.id) ?? [],
+ // A screen recording is watched inside its picture, as it is measured.
+ ...(asset.picture ? { picture: asset.picture } : {}),
budgetMs: 90_000,
})
).catch((e) => {
@@ -3883,13 +3947,15 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
if (!Array.isArray(input.effects)) throw new ToolError("effects must be a list; [] removes them all.");
// Each entry names a clip effect once, with an optional amount in range;
- // a flash also takes its tone, strobe rate and rhythm.
+ // a flash also takes its tone, strobe rate and rhythm, a negative its
+ // strobe rate.
const effects: ClipEffect[] = [];
for (const e of input.effects as { effect?: unknown; amount?: unknown; tone?: unknown; rate?: unknown; rhythm?: unknown }[]) {
const effect = CLIP_EFFECT_IDS.find((id) => id === e?.effect);
if (!effect) throw new ToolError(`effect must be one of ${CLIP_EFFECT_IDS.join(", ")}.`);
if (effects.some((x) => x.effect === effect)) throw new ToolError(`${effect} is listed twice; a clip wears an effect once.`);
const entry: ClipEffect = isNum(e.amount) ? { effect, amount: Math.max(0.05, Math.min(1, e.amount)) } : { effect };
+ if (effect === "negative" && isNum(e.rate) && e.rate > 0) entry.rate = clamp(e.rate, FLASH_RATE_MIN, FLASH_RATE_MAX);
if (effect === "flash") {
if (e.tone !== undefined && !(FLASH_TONES as readonly unknown[]).includes(e.tone)) {
throw new ToolError(`tone is one of ${FLASH_TONES.join(", ")}.`);
@@ -6309,6 +6375,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const reference = await colorStatsForRef(s, {
clipId: typeof input.ref_clip_id === "string" ? input.ref_clip_id : undefined,
assetId: typeof input.ref_asset_id === "string" ? input.ref_asset_id : undefined,
+ at: isNum(input.ref_at) ? input.ref_at : undefined,
graded: true,
});
const raw = await clipRawFrame(s, clip, 192, 108);
@@ -6478,6 +6545,7 @@ export const MEDIA_RUNTIME_TOOLS: ReadonlySet<string> = new Set([
"detect_cuts",
"sample_color",
"track_element",
+ "track_camera",
"suggest_font",
"render_preview",
"freeze_frame",
@@ -8125,6 +8193,8 @@ function effectSettings(effect: EffectId, input: Record<string, unknown>): Parti
}
patch.glitch = input.glitch as GlitchKind;
}
+ // A negative strobes at a rate; none flips the whole element.
+ if (effect === "negative" && isNum(input.rate)) patch.rate = input.rate > 0 ? clamp(input.rate, FLASH_RATE_MIN, FLASH_RATE_MAX) : undefined;
if (effect !== "flash") return patch;
// A flash's tone and strobe: white and a single pop store as absence.
if (input.tone !== undefined) {
diff --git a/site/src/cut/lib/colorOffscreen.test.ts b/site/src/cut/lib/colorOffscreen.test.ts
index 38d717e2..5694db31 100644
--- a/site/src/cut/lib/colorOffscreen.test.ts
+++ b/site/src/cut/lib/colorOffscreen.test.ts
@@ -23,11 +23,23 @@ await stubModule<typeof import("./baseFrame")>("./baseFrame", import.meta.url, {
},
});
+// A reference video answers its picture at a second: mid grey (120 and 160).
+const pictured: { url: string; t: number; area: unknown }[] = [];
+await stubModule<typeof import("./measure/sourceMeasure")>("./measure/sourceMeasure", import.meta.url, {
+ sourcePictureAt: async (url, t, area) => {
+ pictured.push({ url, t, area });
+ const data = new Uint8ClampedArray(16 * 9 * 4);
+ for (let j = 0; j < 16 * 9; j++) data.fill(120 + (j % 2) * 40, j * 4, j * 4 + 3), (data[j * 4 + 3] = 255);
+ return { data, width: 16, height: 9 };
+ },
+});
+
const { runAiTool } = await import("./aiTools");
const { useEditor } = await import("./store");
beforeEach(async () => {
decoded.length = 0;
+ pictured.length = 0;
useEditor.setState({
projectId: "p1",
clips: [],
@@ -83,3 +95,26 @@ test("strength 0 says it took none of the match", async () => {
const out = (await runAiTool("match_color_grade", { clipId: clip.id, ref_clip_id: clip.id, strength: 0 })) as { note?: string };
expect(out.note).toContain("strength 0");
});
+
+test("match_color_grade reads a reference video at a second, inside its picture", async () => {
+ const picture = { x: 0.5, y: 0.31, w: 1, h: 0.32 };
+ useEditor.setState({
+ assets: [
+ ...useEditor.getState().assets,
+ { id: "ref", fileName: "ref.mp4", name: "Ref", type: "video", duration: 13, url: "blob:ref", picture },
+ ],
+ });
+ const [clip] = useEditor.getState().clips;
+ await runAiTool("match_color_grade", { clipId: clip.id, ref_asset_id: "ref", ref_at: 4.2 });
+ expect(pictured).toEqual([{ url: "blob:ref", t: 4.2, area: picture }]);
+ // The dark clip lifts toward the grey reference.
+ expect(useEditor.getState().clips[0].grade?.exposure).toBeGreaterThan(10);
+});
+
+test("a reference video needs the second to read", async () => {
+ useEditor.setState({
+ assets: [...useEditor.getState().assets, { id: "ref", fileName: "ref.mp4", name: "Ref", type: "video", duration: 13, url: "blob:ref" }],
+ });
+ const [clip] = useEditor.getState().clips;
+ await expect(runAiTool("match_color_grade", { clipId: clip.id, ref_asset_id: "ref" })).rejects.toThrow(/ref_at/);
+});
diff --git a/site/src/cut/lib/effectUpdateTool.test.ts b/site/src/cut/lib/effectUpdateTool.test.ts
index 3c8002a5..77db956a 100644
--- a/site/src/cut/lib/effectUpdateTool.test.ts
+++ b/site/src/cut/lib/effectUpdateTool.test.ts
@@ -50,6 +50,16 @@ describe("update_overlay on an effect", () => {
expect(overlay(glitch)).toMatchObject({ glitch: "rgb" });
});
+ test("a negative strobes at a rate, on the timeline and on a clip", async () => {
+ const { id } = (await runAiTool("add_effect", { effect: "negative", start: 4.9, end: 5.43, rate: 15 })) as { id: string };
+ expect(overlay(id)).toMatchObject({ effect: "negative", rate: 15 });
+ await runAiTool("update_overlay", { id, rate: 0 });
+ expect((overlay(id) as { rate?: number }).rate).toBeUndefined();
+ useEditor.setState({ clips: [{ id: "c1", assetId: "a", start: 0, in: 0, out: 2, speed: 1 } as never] });
+ await runAiTool("set_clip_effects", { clipId: "c1", effects: [{ effect: "negative", rate: 15 }] });
+ expect(useEditor.getState().clips[0].effects).toEqual([{ effect: "negative", rate: 15 }]);
+ });
+
test("a flash's fade comes off again with none, back to the pop", async () => {
const { id } = (await runAiTool("add_effect", { effect: "flash", start: 2, end: 3, fade: "to" })) as { id: string };
expect(overlay(id)).toMatchObject({ fade: "to" });
diff --git a/site/src/cut/lib/measure/camera.test.ts b/site/src/cut/lib/measure/camera.test.ts
new file mode 100644
index 00000000..176ca3b9
--- /dev/null
+++ b/site/src/cut/lib/measure/camera.test.ts
@@ -0,0 +1,101 @@
+import { describe, expect, test } from "bun:test";
+import { cameraPath, holdsStill, type Gray } from "./camera";
+
+// A picture that goes on past the frame: a sum of waves, so any zoom, shift or
+// turn of it can be drawn exactly.
+const WAVES = Array.from({ length: 14 }, (_, i) => ({
+ fx: Math.cos(i * 2.4) * (0.2 + 0.08 * (i % 5)),
+ fy: Math.sin(i * 2.4) * (0.2 + 0.08 * (i % 5)),
+ ph: i * 1.7,
+}));
+/** `live` walks a bright subject across the picture, as a skier moves through
+ * a live shot under the camera. */
+const picture = (x: number, y: number, live = 0) =>
+ 128 +
+ WAVES.reduce((v, w) => v + 9 * Math.sin(w.fx * x + w.fy * y + w.ph), 0) +
+ (live ? 90 * Math.exp(-((x - 30 + live * 6) ** 2 + (y - 5) ** 2) / 120) : 0);
+
+/** The picture as the camera sees it at a pose: what sits at (x, y) on the
+ * frame is the picture's point there with the move undone about the centre. */
+function shot(pose: { scale: number; dx: number; dy: number; rotation: number }, live = 0, w = 192, h = 108): Gray {
+ const data = new Float32Array(w * h);
+ const cx = (w - 1) / 2;
+ const cy = (h - 1) / 2;
+ const th = (pose.rotation * Math.PI) / 180;
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const px = x - cx - pose.dx * w;
+ const py = y - cy - pose.dy * h;
+ const rx = (Math.cos(th) * px + Math.sin(th) * py) / pose.scale;
+ const ry = (-Math.sin(th) * px + Math.cos(th) * py) / pose.scale;
+ data[y * w + x] = picture(rx, ry, live);
+ }
+ }
+ return { width: w, height: h, data };
+}
+
+describe("cameraPath", () => {
+ test("finds a beat zoom with a nudge and a tilt", () => {
+ const poses = [
+ { scale: 1, dx: 0, dy: 0, rotation: 0 },
+ { scale: 1.04, dx: 0.01, dy: -0.005, rotation: 0.6 },
+ { scale: 1.085, dx: 0.02, dy: 0.01, rotation: -1.2 },
+ { scale: 1.02, dx: -0.015, dy: 0.02, rotation: 0.3 },
+ ];
+ const path = cameraPath(poses.map((p) => shot(p)));
+ path.forEach((got, i) => {
+ expect(Math.abs(got.scale - poses[i].scale)).toBeLessThan(0.006);
+ expect(Math.abs(got.dx - poses[i].dx)).toBeLessThan(0.004);
+ expect(Math.abs(got.dy - poses[i].dy)).toBeLessThan(0.006);
+ expect(Math.abs(got.rotation - poses[i].rotation)).toBeLessThan(0.3);
+ expect(got.residual).toBeLessThan(3);
+ });
+ });
+
+ test("follows a shake that walks far from where it began", () => {
+ const poses = Array.from({ length: 8 }, (_, i) => ({ scale: 1.05, dx: 0, dy: 0.012 * i * (i % 2 ? 1 : -1), rotation: 0 }));
+ poses[0] = { scale: 1, dx: 0, dy: 0, rotation: 0 };
+ const path = cameraPath(poses.map((p) => shot(p)));
+ path.forEach((got, i) => expect(Math.abs(got.dy - poses[i].dy)).toBeLessThan(0.006));
+ });
+});
+
+test("a cutout held still over a background that shakes is two layers", () => {
+ // The subject: the picture's left 45% stays put while the rest slides.
+ const layered = (dx: number) => {
+ const still = shot({ scale: 1, dx: 0, dy: 0, rotation: 0 });
+ const moved = shot({ scale: 1, dx, dy: 0, rotation: 0 });
+ const data = moved.data.slice();
+ for (let y = 0; y < still.height; y++)
+ for (let x = 0; x < Math.round(still.width * 0.45); x++) data[y * still.width + x] = still.data[y * still.width + x];
+ return { ...moved, data };
+ };
+ const path = cameraPath([layered(0), layered(0.03), layered(0.06), layered(0.03)]);
+ const moving = path.slice(1).map((p) => {
+ // Whichever layer is the camera, the other carries the shake.
+ const shift = Math.abs(p.dx) > 0.01 ? p.dx : p.second?.dx;
+ return Math.round((shift ?? 0) * 100) / 100;
+ });
+ expect(moving).toEqual([0.03, 0.06, 0.03]);
+ expect(path.slice(1).every((p) => p.second)).toBe(true);
+});
+
+describe("holdsStill", () => {
+ // A beat zoom: out to 1.08 and back, twice.
+ const pulse = [1, 1.04, 1.08, 1.04, 1, 1.04, 1.08, 1.04, 1].map((scale) => ({ scale, dx: 0, dy: 0, rotation: 0 }));
+
+ test("a frozen frame pulsed on the beat holds still", () => {
+ const frames = pulse.map((p) => shot(p));
+ expect(holdsStill(frames, cameraPath(frames))).toBe(true);
+ });
+
+ test("a live picture under the same pulse has moved on when the camera comes back", () => {
+ const frames = pulse.map((p, i) => shot(p, i + 1));
+ expect(holdsStill(frames, cameraPath(frames))).toBe(false);
+ });
+
+ test("a steady push never comes back, so it cannot tell", () => {
+ const frames = [1, 1.02, 1.04, 1.06].map((scale) => shot({ scale, dx: 0, dy: 0, rotation: 0 }));
+ expect(holdsStill(frames, cameraPath(frames))).toBeNull();
+ });
+});
diff --git a/site/src/cut/lib/measure/camera.ts b/site/src/cut/lib/measure/camera.ts
new file mode 100644
index 00000000..f8e046fb
--- /dev/null
+++ b/site/src/cut/lib/measure/camera.ts
@@ -0,0 +1,466 @@
+/** The camera's move over a stretch of a source: each frame's zoom, shift and
+ * turn against the stretch's first frame, found by warping that frame until
+ * it lands on this one. A punch-in, a beat zoom, a shake or a slow push in a
+ * reference keys as a clip's pose track (set_clip_keyframes) from these
+ * numbers, and holdsStill tells a picture that holds under the move — a
+ * freeze the template zooms and shakes — from a live one.
+ * Pure data in, pure data out; sourceMeasure.ts feeds it decoded frames. */
+
+import type { Pixels } from "./color";
+
+/** A frame as luma, one float a pixel. */
+export interface Gray {
+ width: number;
+ height: number;
+ data: Float32Array;
+}
+
+/** How a frame sits against the reference frame: `scale` > 1 is zoomed in,
+ * `dx`/`dy` move the picture's centre (frame fractions, + is right and
+ * down), `rotation` turns it clockwise in degrees, and `residual` is the mean
+ * per-pixel difference left after the warp, 0..255. */
+export interface CameraPose {
+ scale: number;
+ dx: number;
+ dy: number;
+ rotation: number;
+ residual: number;
+ /** A second layer moving its own way: a cutout held over a background
+ * that shakes, or one pulsing over a still. Its pose is read at half size,
+ * and `share` is the part of the matched patches it holds. */
+ second?: { scale: number; dx: number; dy: number; rotation: number; share: number };
+}
+
+/** The long side frames are read at for the camera: a pixel is half a
+ * percent of it, and a patch match lands to a tenth of one. */
+export const CAMERA_LONG_SIDE = 192;
+
+export function grayOf(px: Pixels): Gray {
+ const data = new Float32Array(px.width * px.height);
+ for (let i = 0; i < data.length; i++) {
+ data[i] = 0.2126 * px.data[i * 4] + 0.7152 * px.data[i * 4 + 1] + 0.0722 * px.data[i * 4 + 2];
+ }
+ return { width: px.width, height: px.height, data };
+}
+
+/** Patches across the frame, cols × rows; the camera is fitted to the ones
+ * that agree, so a subject crossing some of them is outvoted. */
+const GRID = { cols: 10, rows: 6 };
+/** A patch's half side, and how far it is searched each way past where a
+ * pose puts it: wide on the half-size pass that catches a shake's jump, close
+ * on the full-size one that places it to a tenth of a pixel. */
+const PATCH_R = 6;
+const COARSE_R = 8;
+const FINE_R = 3;
+/** A patch flatter than this (high-passed std) has nothing to match: sky,
+ * snow, a blown-out flash. */
+const MIN_TEXTURE = 3;
+/** Frames the fit needs this many agreeing patches to trust. */
+const MIN_PATCHES = 6;
+
+/** The frame minus its own blur: edges and texture without the light over
+ * them, so a vignette fixed to the screen, a sky's gradient and a beat's
+ * exposure pulse do not pull the fit. */
+function highPass(g: Gray, r = 4): Gray {
+ const { width: w, height: h } = g;
+ const integral = new Float64Array((w + 1) * (h + 1));
+ for (let y = 0; y < h; y++) {
+ let row = 0;
+ for (let x = 0; x < w; x++) {
+ row += g.data[y * w + x];
+ integral[(y + 1) * (w + 1) + x + 1] = integral[y * (w + 1) + x + 1] + row;
+ }
+ }
+ const data = new Float32Array(w * h);
+ for (let y = 0; y < h; y++) {
+ const y0 = Math.max(0, y - r);
+ const y1 = Math.min(h, y + r + 1);
+ for (let x = 0; x < w; x++) {
+ const x0 = Math.max(0, x - r);
+ const x1 = Math.min(w, x + r + 1);
+ const sum = integral[y1 * (w + 1) + x1] - integral[y0 * (w + 1) + x1] - integral[y1 * (w + 1) + x0] + integral[y0 * (w + 1) + x0];
+ data[y * w + x] = g.data[y * w + x] - sum / ((x1 - x0) * (y1 - y0));
+ }
+ }
+ return { width: w, height: h, data };
+}
+
+/** Half the size, each pixel the mean of four. */
+function halve(g: Gray): Gray {
+ const w = Math.max(1, Math.floor(g.width / 2));
+ const h = Math.max(1, Math.floor(g.height / 2));
+ const data = new Float32Array(w * h);
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const i = 2 * y * g.width + 2 * x;
+ data[y * w + x] = (g.data[i] + g.data[i + 1] + g.data[i + g.width] + g.data[i + g.width + 1]) / 4;
+ }
+ }
+ return { width: w, height: h, data };
+}
+
+function sample(g: Gray, x: number, y: number): number {
+ if (x < 0 || y < 0 || x > g.width - 1.001 || y > g.height - 1.001) return NaN;
+ const x0 = Math.floor(x);
+ const y0 = Math.floor(y);
+ const fx = x - x0;
+ const fy = y - y0;
+ const i = y0 * g.width + x0;
+ return (
+ (g.data[i] * (1 - fx) + g.data[i + 1] * fx) * (1 - fy) + (g.data[i + g.width] * (1 - fx) + g.data[i + g.width + 1] * fx) * fy
+ );
+}
+
+/** Mean absolute difference between `cur` and `ref` warped by the pose (shift
+ * in pixels of these frames), over the pixels whose source lies inside `ref`.
+ * Infinity when too little overlaps. */
+function cost(ref: Gray, cur: Gray, s: number, dxPx: number, dyPx: number, rotDeg: number, step: number): number {
+ const { width: w, height: h } = cur;
+ const cx = (w - 1) / 2;
+ const cy = (h - 1) / 2;
+ const th = (rotDeg * Math.PI) / 180;
+ const cos = Math.cos(th) / s;
+ const sin = Math.sin(th) / s;
+ let sum = 0;
+ let n = 0;
+ for (let y = 0; y < h; y += step) {
+ for (let x = 0; x < w; x += step) {
+ // The reference point that lands on (x, y): undo the shift, the turn
+ // and the zoom about the centre.
+ const px = x - cx - dxPx;
+ const py = y - cy - dyPx;
+ const v = sample(ref, cx + cos * px + sin * py, cy - sin * px + cos * py);
+ if (Number.isNaN(v)) continue;
+ sum += Math.abs(v - cur.data[y * w + x]);
+ n++;
+ }
+ }
+ // A warp that leaves less than a third of the frame overlapping proves nothing.
+ return n * step * step < (w * h) / 3 ? Infinity : sum / n;
+}
+
+interface Pose {
+ s: number;
+ rot: number;
+ tx: number;
+ ty: number;
+}
+
+/** Where a point of the reference (from the centre) lands under a pose. */
+function place(p: Pose, x: number, y: number): [number, number] {
+ const th = (p.rot * Math.PI) / 180;
+ const a = p.s * Math.cos(th);
+ const b = p.s * Math.sin(th);
+ return [a * x - b * y + p.tx, b * x + a * y + p.ty];
+}
+
+/** A match within this many pixels of where a pose puts it agrees with it. */
+const AGREE_PX = 1;
+
+/** The least-squares zoom, turn and shift over the pairs `idx`:
+ * q = [a -b; b a] p + t, solved with the means taken out. */
+function solvePose(from: [number, number][], to: [number, number][], idx: number[]): Pose | null {
+ let mpx = 0, mpy = 0, mqx = 0, mqy = 0;
+ for (const i of idx) {
+ mpx += from[i][0]; mpy += from[i][1]; mqx += to[i][0]; mqy += to[i][1];
+ }
+ mpx /= idx.length; mpy /= idx.length; mqx /= idx.length; mqy /= idx.length;
+ let saa = 0, sab = 0, spp = 0;
+ for (const i of idx) {
+ const px = from[i][0] - mpx, py = from[i][1] - mpy;
+ const qx = to[i][0] - mqx, qy = to[i][1] - mqy;
+ saa += px * qx + py * qy;
+ sab += px * qy - py * qx;
+ spp += px * px + py * py;
+ }
+ if (spp <= 0) return null;
+ const a = saa / spp;
+ const b = sab / spp;
+ return { s: Math.hypot(a, b), rot: (Math.atan2(b, a) * 180) / Math.PI, tx: mqx - (a * mpx - b * mpy), ty: mqy - (b * mpx + a * mpy) };
+}
+
+/** The pairs a pose agrees with. */
+function agreeing(pose: Pose, from: [number, number][], to: [number, number][], px: number): number[] {
+ return from.flatMap((p, i) => {
+ const [x, y] = place(pose, p[0], p[1]);
+ return Math.hypot(x - to[i][0], y - to[i][1]) <= px ? [i] : [];
+ });
+}
+
+/** The zoom, turn and shift that carry `from` onto `to` (points from the
+ * centre) by consensus: every two pairs propose a pose, the one most pairs
+ * agree with wins, and it is refitted over them. Snow, sky and a subject
+ * moving through mismatch at random and agree with nothing. Null with fewer
+ * than MIN_PATCHES agreeing. */
+function fitPose(from: [number, number][], to: [number, number][], px = AGREE_PX): { pose: Pose; kept: number; agree: number[] } | null {
+ if (from.length < MIN_PATCHES) return null;
+ let best: number[] = [];
+ for (let i = 0; i < from.length; i++) {
+ for (let j = i + 1; j < from.length; j++) {
+ // Two points too close together cannot tell a zoom or a turn.
+ if (Math.hypot(from[i][0] - from[j][0], from[i][1] - from[j][1]) < 8 * px) continue;
+ const pose = solvePose(from, to, [i, j]);
+ if (!pose || pose.s < 0.5 || pose.s > 2) continue;
+ const agree = agreeing(pose, from, to, px);
+ if (agree.length > best.length) best = agree;
+ }
+ }
+ if (best.length < MIN_PATCHES) return null;
+ let pose = solvePose(from, to, best)!;
+ let agree = agreeing(pose, from, to, px);
+ if (agree.length >= MIN_PATCHES) pose = solvePose(from, to, agree)!;
+ else agree = best;
+ return { pose, kept: agree.length, agree };
+}
+
+interface Patch {
+ /** Its centre from the frame's centre. */
+ x: number;
+ y: number;
+ pixels: Float32Array;
+}
+
+/** The textured patches of the reference frame, on the grid. */
+function patchesOf(g: Gray): Patch[] {
+ const margin = PATCH_R + 2;
+ const cx = (g.width - 1) / 2;
+ const cy = (g.height - 1) / 2;
+ const out: Patch[] = [];
+ for (let r = 0; r < GRID.rows; r++) {
+ for (let c = 0; c < GRID.cols; c++) {
+ const x = Math.round(margin + ((g.width - 2 * margin) * (c + 0.5)) / GRID.cols);
+ const y = Math.round(margin + ((g.height - 2 * margin) * (r + 0.5)) / GRID.rows);
+ const pixels = new Float32Array((2 * PATCH_R + 1) ** 2);
+ let k = 0, sum = 0, sq = 0;
+ for (let v = -PATCH_R; v <= PATCH_R; v++) {
+ for (let u = -PATCH_R; u <= PATCH_R; u++) {
+ const p = g.data[(y + v) * g.width + x + u];
+ pixels[k++] = p;
+ sum += p;
+ sq += p * p;
+ }
+ }
+ const std = Math.sqrt(Math.max(0, sq / k - (sum / k) ** 2));
+ if (std >= MIN_TEXTURE) out.push({ x: x - cx, y: y - cy, pixels });
+ }
+ }
+ return out;
+}
+
+/** Where a reference patch sits in `cur`, searched around where `pose` puts
+ * it, with the patch zoomed and turned as the pose has it; null when it runs
+ * off the frame or its best match is no better than its neighbours'. */
+function matchPatch(cur: Gray, patch: Patch, pose: Pose, SEARCH_R: number): [number, number] | null {
+ const cx = (cur.width - 1) / 2;
+ const cy = (cur.height - 1) / 2;
+ const [px, py] = place(pose, patch.x, patch.y);
+ const th = (pose.rot * Math.PI) / 180;
+ const a = pose.s * Math.cos(th);
+ const b = pose.s * Math.sin(th);
+ const n = 2 * SEARCH_R + 1;
+ const costs = new Float32Array(n * n).fill(Infinity);
+ let best = Infinity;
+ let bi = -1;
+ for (let oy = -SEARCH_R; oy <= SEARCH_R; oy++) {
+ for (let ox = -SEARCH_R; ox <= SEARCH_R; ox++) {
+ let sum = 0;
+ let k = 0;
+ let off = false;
+ for (let v = -PATCH_R; v <= PATCH_R && !off; v++) {
+ for (let u = -PATCH_R; u <= PATCH_R; u++) {
+ const val = sample(cur, cx + px + ox + a * u - b * v, cy + py + oy + b * u + a * v);
+ if (Number.isNaN(val)) {
+ off = true;
+ break;
+ }
+ sum += Math.abs(val - patch.pixels[k++]);
+ }
+ }
+ if (off) continue;
+ const i = (oy + SEARCH_R) * n + ox + SEARCH_R;
+ costs[i] = sum;
+ if (sum < best) {
+ best = sum;
+ bi = i;
+ }
+ }
+ }
+ if (bi < 0) return null;
+ const bx = (bi % n) - SEARCH_R;
+ const by = Math.floor(bi / n) - SEARCH_R;
+ // A best on the search's edge may lie past it.
+ if (Math.abs(bx) === SEARCH_R || Math.abs(by) === SEARCH_R) return null;
+ // The parabola through the best and its neighbours puts it between pixels.
+ const sub = (l: number, c: number, r: number) => {
+ const d = l - 2 * c + r;
+ return d > 0 && Number.isFinite(d) ? Math.max(-0.5, Math.min(0.5, (l - r) / (2 * d))) : 0;
+ };
+ const at = (x: number, y: number) => costs[(y + SEARCH_R) * n + x + SEARCH_R];
+ return [
+ px + bx + sub(at(bx - 1, by), best, at(bx + 1, by)),
+ py + by + sub(at(bx, by - 1), best, at(bx, by + 1)),
+ ];
+}
+
+/** The pose that carries `patches` of the reference onto `cur`, searched
+ * `reach` pixels around where `start` puts each one. */
+function matchAll(cur: Gray, patches: Patch[], start: Pose, reach: number) {
+ const from: [number, number][] = [];
+ const to: [number, number][] = [];
+ for (const p of patches) {
+ const hit = matchPatch(cur, p, start, reach);
+ if (hit) {
+ from.push([p.x, p.y]);
+ to.push(hit);
+ }
+ }
+ return { from, to };
+}
+
+function fitFrom(cur: Gray, patches: Patch[], start: Pose, reach: number, px = AGREE_PX) {
+ const { from, to } = matchAll(cur, patches, start, reach);
+ return fitPose(from, to, px);
+}
+
+/** The layers the matches split into: the pose most of them agree with, and
+ * the pose most of the rest agree with, when enough do. */
+function layersOf(cur: Gray, patches: Patch[], start: Pose, reach: number, px: number) {
+ const { from, to } = matchAll(cur, patches, start, reach);
+ const first = fitPose(from, to, px);
+ if (!first) return [];
+ const taken = new Set(first.agree);
+ const rest = from.flatMap((_, i) => (taken.has(i) ? [] : [i]));
+ const second = fitPose(rest.map((i) => from[i]), rest.map((i) => to[i]), px);
+ const share = (n: number) => n / from.length;
+ return second && apart(first.pose, second.pose, cur, 2 * px)
+ ? [{ pose: first.pose, share: share(first.kept) }, { pose: second.pose, share: share(second.kept) }]
+ : [{ pose: first.pose, share: share(first.kept) }];
+}
+
+/** Whether two poses put some corner of the frame more than `px` apart. */
+function apart(a: Pose, b: Pose, g: Gray, px: number): boolean {
+ const hw = g.width / 2;
+ const hh = g.height / 2;
+ return [[-hw, -hh], [hw, -hh], [-hw, hh], [hw, hh], [0, 0]].some(([x, y]) => {
+ const [ax, ay] = place(a, x, y);
+ const [bx, by] = place(b, x, y);
+ return Math.hypot(ax - bx, ay - by) > px;
+ });
+}
+
+/** How far apart two poses put the frame's corners, at most. */
+function gap(a: Pose, b: Pose, g: Gray): number {
+ const hw = g.width / 2;
+ const hh = g.height / 2;
+ return Math.max(
+ ...[[-hw, -hh], [hw, -hh], [-hw, hh], [hw, hh]].map(([x, y]) => {
+ const [ax, ay] = place(a, x, y);
+ const [bx, by] = place(b, x, y);
+ return Math.hypot(ax - bx, ay - by);
+ })
+ );
+}
+
+/** Layers holding shares this close are near even. */
+const EVEN_SHARE = 0.1;
+/** A second layer holds at least this share of the matched patches in a
+ * frame, and shows in at least SECOND_FRAMES of the stretch's frames: less
+ * is a few patches mismatching together, not a layer. */
+const SECOND_SHARE = 0.25;
+const SECOND_FRAMES = 0.25;
+
+const half = (p: Pose): Pose => ({ ...p, tx: p.tx / 2, ty: p.ty / 2 });
+const REST: Pose = { s: 1, rot: 0, tx: 0, ty: 0 };
+
+/** Each frame's pose against `frames[0]`, read at CAMERA_LONG_SIDE: the
+ * reference frame's textured patches are found again in each frame and one
+ * zoom, turn and shift is fitted to the patches that agree. A half-size pass
+ * from the frame before's pose, and from rest, catches a jump; the better
+ * fit is placed at full size. A frame too blurred or too changed to fit
+ * keeps the pose before it, with residual Infinity. */
+export function cameraPath(frames: Gray[]): CameraPose[] {
+ if (frames.length === 0) return [];
+ const ref = highPass(frames[0]);
+ const refHalf = highPass(halve(frames[0]), 2);
+ const patches = patchesOf(ref);
+ const patchesHalf = patchesOf(refHalf);
+ const out: CameraPose[] = [{ scale: 1, dx: 0, dy: 0, rotation: 0, residual: 0 }];
+ let prev: Pose = REST;
+ for (let i = 1; i < frames.length; i++) {
+ const cur = highPass(frames[i]);
+ const curHalf = highPass(halve(frames[i]), 2);
+ // The layers seen from the frame before and from rest. The camera is the
+ // one holding most of the picture — of two near even, the one carrying
+ // on from the frame before — and another holding a fair share of its
+ // own is the second layer.
+ const layers = [...layersOf(curHalf, patchesHalf, half(prev), COARSE_R, AGREE_PX * 0.75), ...layersOf(curHalf, patchesHalf, REST, COARSE_R, AGREE_PX * 0.75)]
+ .sort((a, b) => b.share - a.share);
+ const main = layers
+ .filter((l) => l.share >= layers[0].share - EVEN_SHARE)
+ .sort((a, b) => gap(a.pose, half(prev), curHalf) - gap(b.pose, half(prev), curHalf))[0];
+ const other = main && layers.find((l) => l.share >= SECOND_SHARE && apart(l.pose, main.pose, curHalf, 1.5));
+ const start = main ? { ...main.pose, tx: main.pose.tx * 2, ty: main.pose.ty * 2 } : prev;
+ const fit = fitFrom(cur, patches, start, FINE_R);
+ if (!fit) {
+ out.push({ ...out[out.length - 1], residual: Infinity });
+ continue;
+ }
+ prev = fit.pose;
+ out.push({
+ scale: fit.pose.s,
+ dx: fit.pose.tx / cur.width,
+ dy: fit.pose.ty / cur.height,
+ rotation: fit.pose.rot,
+ residual: cost(ref, cur, fit.pose.s, fit.pose.tx, fit.pose.ty, fit.pose.rot, 2),
+ ...(other
+ ? {
+ second: {
+ scale: other.pose.s,
+ dx: other.pose.tx / curHalf.width,
+ dy: other.pose.ty / curHalf.height,
+ rotation: other.pose.rot,
+ share: other.share,
+ },
+ }
+ : {}),
+ });
+ }
+ const layered = out.filter((p) => p.second).length;
+ if (layered < SECOND_FRAMES * (out.length - 1)) for (const p of out) delete p.second;
+ return out;
+}
+
+/** Two frames the camera holds the same way differ by less than this a pixel
+ * when the picture under it is one frame: a returning live shot has moved on. */
+export const STILL_RESIDUAL = 2;
+
+/** Poses this close show the same part of the picture the same way. */
+const SAME_POSE = { scale: 0.004, d: 0.003, rotation: 0.2 };
+
+/** Whether one picture holds under the whole move — a freeze the template
+ * zooms or shakes — by the frames where the camera comes back to a pose it
+ * held at least two frames before: a frozen picture is the same frame again,
+ * a live one has moved on. The warp's own residual cannot tell, since a
+ * screen-fixed vignette or grain over a zoomed freeze leaves one too. Null
+ * when the camera never returns, as in a steady push. */
+export function holdsStill(frames: Gray[], path: CameraPose[]): boolean | null {
+ const diffs: number[] = [];
+ for (let i = 2; i < frames.length; i++) {
+ for (let j = 0; j <= i - 2; j++) {
+ const a = path[i];
+ const b = path[j];
+ if (
+ Math.abs(a.scale - b.scale) < SAME_POSE.scale &&
+ Math.abs(a.dx - b.dx) < SAME_POSE.d &&
+ Math.abs(a.dy - b.dy) < SAME_POSE.d &&
+ Math.abs(a.rotation - b.rotation) < SAME_POSE.rotation
+ ) {
+ diffs.push(cost(frames[j], frames[i], 1, 0, 0, 0, 1));
+ break;
+ }
+ }
+ }
+ if (diffs.length < 2) return null;
+ diffs.sort((x, y) => x - y);
+ return diffs[Math.floor(diffs.length / 2)] < STILL_RESIDUAL;
+}
diff --git a/site/src/cut/lib/measure/cuts.test.ts b/site/src/cut/lib/measure/cuts.test.ts
index aeb93a0a..dae01bc1 100644
--- a/site/src/cut/lib/measure/cuts.test.ts
+++ b/site/src/cut/lib/measure/cuts.test.ts
@@ -136,3 +136,17 @@ test("a slow pan over a gradient reports no changes", () => {
expect(cuts).toEqual([]);
expect(changes).toEqual([]);
});
+
+test("a flash frame inside fast streaking footage keeps both of its cuts", () => {
+ // Dark footage whose streaks change most of the picture every frame, then
+ // one white flash frame and a bright shot: the flash's change rides no
+ // higher than its busy neighbours, but the whole picture leaps to white.
+ const streak = (k: number) => card(64, 64, [20, 30, 45], [60, 200, 210], [(k * 13) % 48, ((k * 13) % 48) + 16, 0, 64]);
+ const frames = [
+ ...Array.from({ length: 6 }, (_, k) => streak(k)),
+ solid(64, 64, [240, 240, 230]),
+ ...Array(4).fill(solid(64, 64, [90, 160, 200])),
+ ];
+ const { cuts } = scoreFrames(frames);
+ expect([6, 7].filter((i) => cuts.includes(i))).toEqual([6, 7]);
+});
diff --git a/site/src/cut/lib/measure/cuts.ts b/site/src/cut/lib/measure/cuts.ts
index efafc9a7..1d386a7f 100644
--- a/site/src/cut/lib/measure/cuts.ts
+++ b/site/src/cut/lib/measure/cuts.ts
@@ -17,6 +17,10 @@ export const CUT_TUNING = {
SPIKE: 3,
/** Frames either side a cut is judged against. */
NEIGHBOURS: 3,
+ /** Mean-luma step (0..255) of a frame that flashes or blacks out the whole
+ * picture at once: a cut however busy the frames around it are, since
+ * motion moves things around the frame and leaves its brightness be. */
+ FLASH_LUMA: 60,
/** Per-channel step a flat fill takes when it cuts to a near colour (a dark
* green to a slightly lighter one): under CELL_TOL, over grain. */
FAINT_TOL: 6,
@@ -105,6 +109,8 @@ export interface FrameStep {
faint: number;
fine: number;
box: ChangeBox | null;
+ /** How far the whole picture's mean luma moved, 0..255. */
+ luma?: number;
}
/** A change inside a shot: from frame `frame` for `frames` frames, the most
@@ -150,6 +156,13 @@ function fineChange(a: Float32Array, b: Float32Array): { pct: number; box: Chang
return { pct: (100 * n) / (grid * grid), box };
}
+/** A grid's mean luma (Rec. 709 weights), 0..255. */
+function meanLuma(g: Float32Array): number {
+ let sum = 0;
+ for (let i = 0; i < g.length; i += 3) sum += 0.2126 * g[i] + 0.7152 * g[i + 1] + 0.0722 * g[i + 2];
+ return sum / Math.max(1, g.length / 3);
+}
+
/** How `next` differs from `prev` on both grids. */
export function frameStep(prev: FrameGrids, next: FrameGrids): FrameStep {
const fine = fineChange(prev.fine, next.fine);
@@ -158,6 +171,7 @@ export function frameStep(prev: FrameGrids, next: FrameGrids): FrameStep {
faint: diffCellPct(prev.coarse, next.coarse, CUT_TUNING.FAINT_TOL),
fine: fine.pct,
box: fine.box,
+ luma: Math.abs(meanLuma(next.coarse) - meanLuma(prev.coarse)),
};
}
@@ -185,12 +199,15 @@ function neighbourMedian(scores: number[], i: number, skip?: Set<number>): numbe
* Changes on consecutive frames are one change that runs those frames: a
* fade-in over three frames reads as one change of three. */
export function pickShotEdges(steps: FrameStep[], fines?: ArrayLike<number>[]): { cuts: number[]; changes: ShotChange[] } {
- const { FAINT_ALL, SPIKE, CHANGE_MIN } = CUT_TUNING;
+ const { FAINT_ALL, SPIKE, CHANGE_MIN, MIN_CHANGE, FLASH_LUMA } = CUT_TUNING;
const hard = pickCuts(steps.map((s) => s.pct));
const faintScores = steps.map((s) => s.faint);
- // The faint cuts, beside the hard ones.
+ // The faint cuts and the flashes, beside the hard ones.
const cutSet = new Set(hard);
+ steps.forEach((s, i) => {
+ if (s.pct >= MIN_CHANGE && (s.luma ?? 0) >= FLASH_LUMA) cutSet.add(i);
+ });
for (let i = 0; i < steps.length; i++) {
if (cutSet.has(i) || !(faintScores[i] >= FAINT_ALL)) {
continue;
diff --git a/site/src/cut/lib/measure/limits.ts b/site/src/cut/lib/measure/limits.ts
index 9d7420a2..c027e2b1 100644
--- a/site/src/cut/lib/measure/limits.ts
+++ b/site/src/cut/lib/measure/limits.ts
@@ -21,3 +21,6 @@ export const TRACK_TOLERANCE_DEFAULT = 40;
export const TRACK_MATCHES = ["shape", "ink"] as const;
/** track_element: path entries listed per call; a longer path is thinned. */
export const TRACK_PATH_MAX = 60;
+/** track_camera: frames read per call — four seconds at 30 fps, every one
+ * listed, so a beat zoom's peaks are all there. */
+export const CAMERA_FRAMES_MAX = 120;
diff --git a/site/src/cut/lib/measure/sourceMeasure.ts b/site/src/cut/lib/measure/sourceMeasure.ts
index bedd23c8..6b94f1c6 100644
--- a/site/src/cut/lib/measure/sourceMeasure.ts
+++ b/site/src/cut/lib/measure/sourceMeasure.ts
@@ -8,6 +8,7 @@ import { baseFrameSink, colorReadAt } from "../baseFrame";
import { openMedia, UnreadableMediaError, videoTrackOf, type FrameSize } from "../mediaRead";
import { createRasterCanvas, decodeRasterImageUrl } from "../raster";
import type { WrappedCanvas } from "mediabunny";
+import { CAMERA_LONG_SIDE, cameraPath, grayOf, holdsStill, type CameraPose, type Gray } from "./camera";
import { hexOf, regionColor, parseHex, type PixelRect, type Pixels, type Rgb } from "./color";
import { frameStep, gridsOf, pickShotEdges, type FrameGrids, type FrameStep, type ShotChange } from "./cuts";
import { fitMoves, type Move } from "./easeFit";
@@ -228,6 +229,19 @@ export async function scanCuts(
});
}
+/** A source's picture at second `t`: the frame inside `area`, read at about
+ * `longSide` on that area's long side, as the preview shows it — what a grade
+ * is matched to. */
+export async function sourcePictureAt(url: string, t: number, area: FrameBox = FULL_FRAME, longSide = 192): Promise<Pixels> {
+ return withSourceVideo(url, async (v) => {
+ const size = sizeAt(v.width, v.height, longSide, area);
+ for await (const f of v.framesAt([t], size)) {
+ if (f) return cropPixels(f, toPixels(f, area));
+ }
+ throw new Error(`Could not read the frame at ${t}s.`);
+ });
+}
+
export interface ColorSample {
t: number;
box: FrameBox;
@@ -371,6 +385,68 @@ export async function trackElement(
};
}
+export interface CameraRun {
+ fps: number;
+ /** Per frame read: its time and how the picture sits against the first. */
+ path: ({ t: number } & CameraPose)[];
+ moves: (Move & { prop: "scale" | "dx" | "dy" | "rotation" })[];
+ /** Whether one picture holds under the whole move — a freeze frame — or
+ * null when the camera never comes back to a pose to tell by. */
+ still: boolean | null;
+ coveredTo: number;
+ truncated: boolean;
+}
+
+/** How the camera moves over [from, to) of a video, inside its picture box:
+ * each frame's zoom, shift and turn against the frame on `from`, and the
+ * moves of each fitted with an ease. */
+export async function trackCamera(
+ url: string,
+ opts: { from: number; to: number; maxFrames: number; area?: FrameBox }
+): Promise<CameraRun> {
+ const area = opts.area ?? FULL_FRAME;
+ let truncated = false;
+ let coveredTo = opts.to;
+ const times: number[] = [];
+ const durations: number[] = [];
+ const frames: Gray[] = [];
+ await withSourceVideo(url, async (v) => {
+ const at = sizeAt(v.width, v.height, CAMERA_LONG_SIDE, area);
+ for await (const f of v.frames(Math.max(0, opts.from - LEAD_S), opts.to, at)) {
+ if (frames.length >= opts.maxFrames) {
+ truncated = true;
+ coveredTo = times[times.length - 1];
+ break;
+ }
+ times.push(f.t);
+ durations.push(f.duration);
+ frames.push(grayOf(cropPixels(f, toPixels(f, area))));
+ }
+ });
+ const poses = cameraPath(frames);
+ const path = poses.map((p, i) => ({ t: times[i], ...p }));
+ // A measurement step per number: a fifth of a read pixel, or the zoom or
+ // turn that moves the frame's edge that far.
+ const moves: CameraRun["moves"] = [];
+ if (path.length > 2) {
+ const w = frames[0].width;
+ const h = frames[0].height;
+ const steps = { scale: 0.4 / Math.max(w, h), dx: 0.2 / w, dy: 0.2 / h, rotation: 0.2 };
+ for (const prop of ["scale", "dx", "dy", "rotation"] as const) {
+ for (const m of fitMoves(times, path.map((p) => p[prop]), steps[prop])) moves.push({ ...m, prop });
+ }
+ moves.sort((a, b) => a.from - b.from);
+ }
+ return {
+ fps: rateOf(times, durations),
+ path,
+ moves,
+ still: holdsStill(frames, poses),
+ coveredTo,
+ truncated,
+ };
+}
+
/** The box a video's picture fills inside its frame, found from frames
* sampled evenly over [from, to): the part that changes, apart from bars and
* an app's chrome around it. Null when too little moves to tell. */
diff --git a/site/src/cut/lib/measureTools.test.ts b/site/src/cut/lib/measureTools.test.ts
index bdff42db..8c40623d 100644
--- a/site/src/cut/lib/measureTools.test.ts
+++ b/site/src/cut/lib/measureTools.test.ts
@@ -176,6 +176,22 @@ test("track_element follows a growing bar and names its ease", async () => {
expect(out.moves.find((m) => m.prop === "x")).toBeUndefined();
});
+test("track_camera reads a held shot as one picture that never moves", async () => {
+ const out = (await runAiTool("track_camera", { asset_id: "ref", from: 1.1, to: 1.4 })) as {
+ still: boolean;
+ path: { t: number; scale: number; dx: number; dy: number; rotation: number }[];
+ moves: unknown[];
+ };
+ expect(out.still).toBe(true);
+ expect(out.path.length).toBeGreaterThan(10);
+ for (const p of out.path) expect([p.scale, p.dx, p.dy, p.rotation]).toEqual([1, 0, 0, 0]);
+ expect(out.moves).toEqual([]);
+});
+
+test("track_camera needs the stretch's end", async () => {
+ expect(await message(runAiTool("track_camera", { asset_id: "ref", from: 1.1 }))).toContain("to");
+});
+
test("track_element's ink box measures a title at one moment", async () => {
const out = (await runAiTool("track_element", {
asset_id: "ref",
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 560da59a..f326bfd5 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -1632,6 +1632,11 @@ export async function sampleWatchFrames(
* detect_cuts found and the frame each change inside a shot settles on.
* Each gets a decode of its own, however close to the next. */
shots?: number[];
+ /** The part of the frame that is the source's picture (a screen
+ * recording's player, a letterboxed film), centre and size as frame
+ * fractions: frames are judged and returned inside it, so a shot changing
+ * in a phone's player is not outvoted by the app around it. */
+ picture?: { x: number; y: number; w: number; h: number };
}
): Promise<WatchFrames> {
const input = openMedia(sourceUrl);
@@ -1649,14 +1654,26 @@ export async function sampleWatchFrames(
const times = candidateTimes(from, to, interval, opts.at ?? [], opts.shots ?? []);
const geom = watchGeometry(opts.detail);
- const srcW = await track.getDisplayWidth();
- const srcH = await track.getDisplayHeight();
+ const fullW = await track.getDisplayWidth();
+ const fullH = await track.getDisplayHeight();
+ const area = opts.picture && (opts.picture.w < 0.999 || opts.picture.h < 0.999) ? opts.picture : null;
+ // Cells are sized to the picture; frames decode at whatever size gives
+ // the picture those pixels, and `blit` cuts the picture out of each.
+ const srcW = area ? fullW * area.w : fullW;
+ const srcH = area ? fullH * area.h : fullH;
+ const decodeSize = (w: number, h: number) =>
+ area ? { width: Math.round(w / area.w), height: Math.round(h / area.h), fit: "fill" as const } : { width: w, height: h, fit: "fill" as const };
+ const blit = (ctx: CanvasRenderingContext2D, src: CanvasImageSource & { width: number; height: number }, w: number, h: number) => {
+ if (!area) return ctx.drawImage(src, 0, 0, w, h);
+ ctx.drawImage(src, (area.x - area.w / 2) * src.width, (area.y - area.h / 2) * src.height, area.w * src.width, area.h * src.height, 0, 0, w, h);
+ };
const twoPass = !opts.metadataOnly && geom.rewalk;
// Selection reads a 192² signature whatever the detail, so a pass that
// returns no pictures — or that walks the kept times again for big ones —
// decodes candidates at the cheap size.
const selectOnly = opts.metadataOnly || twoPass;
const [cw, ch] = cellDims(srcW, srcH, selectOnly ? SELECT_CELL : geom.cell);
+ const cellSize = decodeSize(cw, ch);
const maxFrames = geom.maxFrames;
// Cell canvases are held only until the selector rules on them (decisions
// run one frame behind), so memory stays near the kept set, never the
@@ -1680,20 +1697,20 @@ export async function sampleWatchFrames(
if (!sigCtx) throw new Error("Could not sample the video.");
const deadline = Date.now() + (opts.budgetMs ?? Infinity);
const outOfTime = () => Date.now() > deadline;
- const sigG16Of = (canvas: CanvasImageSource): Float32Array => {
+ const sigG16Of = (canvas: CanvasImageSource & { width: number; height: number }): Float32Array => {
sigCtx.imageSmoothingQuality = "high";
- sigCtx.drawImage(canvas, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ blit(sigCtx, canvas, SIGNATURE_SIZE, SIGNATURE_SIZE);
const data = sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data;
return frameSig({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 4, data }).g16;
};
// The timestamp of the first frame in (lo, hi] that differs from `pre`,
// decoded in order; null when the walk finds none.
const firstNewFrame = async (lo: number, hi: number, pre: Float32Array): Promise<number | null> => {
- const walk = baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).canvases(lo, hi + CUT_WALK_EPS);
+ const walk = baseFrameSink(track, cellSize, reading).canvases(lo, hi + CUT_WALK_EPS);
try {
for await (const frame of walk) {
if (frame.timestamp <= lo - CUT_WALK_EPS) continue;
- const g = sigG16Of(frame.canvas as CanvasImageSource);
+ const g = sigG16Of(frame.canvas);
if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) return frame.timestamp;
}
} finally {
@@ -1702,12 +1719,12 @@ export async function sampleWatchFrames(
return null;
};
const sigG16At = async (t: number): Promise<Float32Array | null> => {
- const one = await baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).getCanvas(t);
- return one ? sigG16Of(one.canvas as CanvasImageSource) : null;
+ const one = await baseFrameSink(track, cellSize, reading).getCanvas(t);
+ return one ? sigG16Of(one.canvas) : null;
};
// The times are ascending, so the whole span decodes once, in order.
- const cells = baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).canvasesAtTimestamps(times);
+ const cells = baseFrameSink(track, cellSize, reading).canvasesAtTimestamps(times);
let paused = false;
let capped = false;
try {
@@ -1728,12 +1745,12 @@ export async function sampleWatchFrames(
const copy = createRasterCanvas(cw, ch);
const cctx = copy.getContext("2d") as CanvasRenderingContext2D | null;
if (!cctx) throw new Error("Could not sample the video.");
- cctx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, cw, ch);
+ blit(cctx, next.value.canvas, cw, ch);
held.set(candTimes.length, copy);
}
candTimes.push(times[i]);
sigCtx.imageSmoothingQuality = "high";
- sigCtx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ blit(sigCtx, next.value.canvas, SIGNATURE_SIZE, SIGNATURE_SIZE);
const rgb = {
width: SIGNATURE_SIZE,
height: SIGNATURE_SIZE,
@@ -1767,7 +1784,7 @@ export async function sampleWatchFrames(
const shot = createRasterCanvas(nw, nh);
const shotCtx = shot.getContext("2d") as CanvasRenderingContext2D | null;
if (!shotCtx) throw new Error("Could not sample the video.");
- const walk = baseFrameSink(track, { width: nw, height: nh, fit: "fill" }, reading).canvasesAtTimestamps(
+ const walk = baseFrameSink(track, decodeSize(nw, nh), reading).canvasesAtTimestamps(
keptAll.map((i) => candTimes[i])
);
try {
@@ -1776,7 +1793,7 @@ export async function sampleWatchFrames(
const frame = await walk.next();
if (frame.done) break;
if (!frame.value) continue;
- shotCtx.drawImage(frame.value.canvas as CanvasImageSource, 0, 0, nw, nh);
+ blit(shotCtx, frame.value.canvas, nw, nh);
encoded.set(idx, await rasterCanvasToDataUrl(shot, "image/jpeg", geom.quality));
}
} finally {
diff --git a/site/src/cut/lib/pi/mutationLedger.ts b/site/src/cut/lib/pi/mutationLedger.ts
index 5a2f164a..dcef1370 100644
--- a/site/src/cut/lib/pi/mutationLedger.ts
+++ b/site/src/cut/lib/pi/mutationLedger.ts
@@ -15,6 +15,7 @@ const READ_ONLY = new Set([
"compare_to_source",
"sample_color",
"track_element",
+ "track_camera",
"suggest_font",
"watch_video",
"note_source",
diff --git a/site/src/cut/lib/watch/sweep.ts b/site/src/cut/lib/watch/sweep.ts
index 6066fd14..29454432 100644
--- a/site/src/cut/lib/watch/sweep.ts
+++ b/site/src/cut/lib/watch/sweep.ts
@@ -218,6 +218,7 @@ async function run(): Promise<void> {
// The transcript is already in by now (speech runs first), so the
// map is drawn on the source's own lines rather than a blind grid.
at: speechOnsets(asset.speech?.segments ?? [], span.from, span.to),
+ ...(asset.picture ? { picture: asset.picture } : {}),
});
if (r.coveredTo > span.from) {
const cur = useEditor.getState().assets.find((a) => a.id === id);
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 2d3aad12..d67d3a57 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -330,7 +330,7 @@ Your eyes and ears:
- detect_silence(clip_id | asset_id, threshold_db?, min_silence?): silent stretches in source seconds; with clip_id each also carries its timeline times.
- detect_beats(clip_id | asset_id, regenerate?): the music's beat grid — bpm and beat times in source seconds, plus timelineBeats ready for split_at with clip_id. The grid persists on the asset, draws as yellow dots on its clips, and every drag or trim snaps to it. "Cut to the beat" / "sync to the music" means detect, then split or place at the timeline beat times; the stored grid carries the user's hand edits, so re-detect only when they ask.
- capture_frame(t?): one composited frame, at the given time in seconds or the playhead — for checking the final look. Surveying footage is watch_video's job.
-- detect_cuts(asset_id, from?, to?), sample_color(asset_id, samples), track_element(asset_id, from, to?, seed): a source in numbers — every hard cut on its frame, the hex of a region at a second, an element's box per frame with the ease of each move. find_picture(asset_id) finds the box a recorded or letterboxed source's picture fills, and they read inside it. Replicating a video runs on them.
+- detect_cuts(asset_id, from?, to?), sample_color(asset_id, samples), track_element(asset_id, from, to?, seed), track_camera(asset_id, from, to): a source in numbers — every hard cut on its frame, the hex of a region at a second, an element's box per frame with the ease of each move, the camera's zoom, shift and turn per frame and whether the picture under it is a frozen still. find_picture(asset_id) finds the box a recorded or letterboxed source's picture fills, and they read inside it. Replicating a video runs on them.
The flow for "edit this for me":
1. get_state — every clip, its trim (in/out), speed, gaps, and the other tracks.
@@ -349,22 +349,24 @@ Time math (get this right):
Cutting speech well: pace is part of the message, and it is set in one place — refine_speech_cuts' \`pace\`. "Fast paced", "punchy", "tight" means pace: fast there; a calm piece means relaxed. Placing your own edges closer to the words is not a faster cut, it is a clipped one: a word's tail runs past its cue, the ear hears the truncation, and no amount of tightening reads as energy. So cut loose — up to a second of spare air on each side — and let the refine step take it back to the beat the pace asks for. Shorten a long pause to that beat rather than deleting it; speech butted directly together reads as rushed however fast the piece is meant to be. Prefer split_at + delete_item, then place_clip to close the gap (a beat of black may be wanted — ask the cut, not the tool); trim_clip only tightens a clip's edges.`,
"replicating-a-video": `# Replicating a video you can watch
-Someone points at a video — a file they imported, a link you brought in, a clip on their timeline — and asks for one like it. Everything on the screen gets rebuilt out of this editor's own parts: every card of type, every shape, every colour, every move, at the time it happens. The footage inside the shots is the one thing you cannot rebuild, because it is the person's to bring — so each shot stands as a block in the colour the reference has behind it, with nothing written on it.
+Someone points at a video — a file they imported, a link you brought in, a clip on their timeline — and asks for one like it. Everything on the screen gets rebuilt out of this editor's own parts: every card of type, every shape, every colour, every move, at the time it happens. The footage inside the shots is the one thing you cannot rebuild, because it is the person's to bring — so each shot stands as a block in the colour the reference has behind it, with nothing written on it. When the ask is for the shots filled — with stock footage, Pexels clips, real footage — you find it yourself on Pexels (below) and the blocks become those clips.
**Clipping a long source.** A talk, an interview or a stream is mostly material that only makes sense in place. find_highlights ranks its stretches from the transcript, so an hour is read for its handful of moments without watching all of it; watch_video the spans it returns before committing, since the ranking never saw the picture. add_clip \`spans\` lays a pick down — several stretches of one source as one run of clips, the material between them left out — and a two-span result is one clip whose halves were never next to each other.
**Watch the whole thing first.** Every decision below comes off the record, so the record has to be complete: watch from 0 and keep going until \`unwatchedSeconds\` is 0, writing each pass into note_source before the next one. Anything over a couple of minutes gets a coarse pass end to end first (\`interval_seconds\` 15-30) to learn its shape, then a dense pass (0.5) over the stretches that carry text or cut fast. Read every card word for word — its text, its face, where it sits in the frame, when it comes and goes — because a turn later the sheets are gone and the notes are all you have. Reading type is what \`detail\` is for: the dense pass over a stretch carrying words runs at \`detail\` read, and a face or treatment you are about to name runs at original, where the frame arrives at the source's own resolution. When the cards ARE the narration — a caption track of what the speaker is saying — the transcript already holds every word, and the close passes are for how they are set: face, weight, size, colour, placement, how many words ride a card, how it enters and leaves. Read a few of those at original and take the rest from the transcript.
-**Measure it.** A contact sheet tells you what happens; the numbers come from the source's pixels. A reference recorded off a screen — a phone filming an editing app, a letterboxed film — has its picture inside a bigger frame: find_picture finds that box and keeps it on the source (detect_cuts does on its first read), every measuring tool and compare_to_source then reads inside it, and their x, y, w, h are fractions of it, so they land on the cut's frame as they are; set_aspect to its \`aspect\`. detect_cuts reads every frame: each cut on its frame, the \`shots\` between them, and the \`changes\` inside a shot — a word landing, a word swapping fill for outline, a shape popping in. Run it over the whole source before the read passes, which then land a frame in every shot and change, and time each title from them. Its times are to the millisecond, and every tool lands such a time on the frame it names: write them back as read. sample_color reads the hex of any region at any second. track_element gives an element's box frame by frame and the ease of each move; with match ink it gives a title's ink box.
+**Measure it.** A contact sheet tells you what happens; the numbers come from the source's pixels. A reference recorded off a screen — a phone filming an editing app, a letterboxed film — has its picture inside a bigger frame: find_picture finds that box and keeps it on the source (detect_cuts does on its first read), every measuring tool and compare_to_source then reads inside it, and their x, y, w, h are fractions of it, so they land on the cut's frame as they are; set_aspect to its \`aspect\`. detect_cuts reads every frame: each cut on its frame, the \`shots\` between them, and the \`changes\` inside a shot — a word landing, a word swapping fill for outline, a shape popping in. Run it over the whole source before the read passes, which then land a frame in every shot and change, and time each title from them. Its times are to the millisecond, and every tool lands such a time on the frame it names: write them back as read. sample_color reads the hex of any region at any second. track_element gives an element's box frame by frame and the ease of each move; with match ink it gives a title's ink box. track_camera gives a shot's camera move frame by frame — zoom, shift, turn — and says when the picture under it is one frozen frame.
**Lay down the sound, then block out the cuts.** Cuts timed to a reference are timed to its sound: detach_audio with the reference's \`assetId\` puts its whole track on the soundtrack and none of its picture. add_clip with \`blocks\` lays the skeleton from \`shots\` in one call — one block per shot on track 0, each its shot's \`from\` and \`to\` as read so every cut lands on its frame, flash cuts included, \`color\` what sample_color reads behind it and \`label\` naming what the person drops there. The label rides the timeline chip; the frame shows the colour alone. Footage dropped on a block, or replace_item, takes its place and its length, so everything built on top keeps its timing. When the ask is the reference's own picture playing, add_clip \`spans\` lays its shots down instead, one span per shot at those same times.
+**Fill the shots from Pexels, when asked.** The reference's own frames never go in the cut; Pexels' free footage does. Search it with WebFetch — the one web tool you have, and it reaches only Pexels: https://www.pexels.com/search/videos/<words>/?orientation=landscape (portrait for a 9:16 cut), asking for every result's link, id and title. Search per shot by what it shows — the subject and what they wear, the action, the angle, the light, the time of day — and look again with other words when a page has nothing close; one creator's series often covers a whole template, so a good hit's neighbours are worth a search of their own. import_url \`urls\` takes the video page links (pexels.com/video/<slug>-<id>/) and brings each down as its 1080p file, the Pexels id in its name. Watch the candidates beside the reference shot and keep the one that matches subject, angle and motion — a candidate is footage to judge, not a source to replicate: one scan pass over the stretch you would use settles it, and the whole-watch and word-for-word reading above are the reference's alone; replace_item puts it on that shot's block, trim_clip sets its in so the motion lands where the reference's does, set_framing zooms and pans to the reference's framing, and match_color_grade with ref_asset_id the reference and ref_at a moment inside the shot grades it to the reference's frame. Pexels files carry their own sound: when the reference's music is the soundtrack, set_clip_muted with \`ids\` mutes every footage clip in one call, so only the music plays. A shot whose camera moves — a push, a punch-in on the beat, a shake, a tilt — is measured with track_camera over that shot of the reference and keyed on its clip with set_clip_keyframes from the \`path\` and \`moves\` it returns. When it says \`still\`, the reference froze a frame and moved it: freeze_frame at a timeline second inside the shot's clip, for the shot's length, at that clip's index, delete_item the moving clip, and key the moves on the still. An overlay that shows the next shot's subject over the current one is that shot's clip again, on a track above. Name every Pexels id you used when you close.
+
**Rebuild the text as elements.** One title per card, a stretch of cards per add_title call — the words exactly as they read, size in px at 1080 wide, and x/y where suggest_font sets it on that card — a title is set from the middle of its line, so the centre of its ink box (track_element, match ink, the text's colour) sits off it by the face's own offset. A run of lines timed one after another is add_text_sequence in one call. Match the case the reference uses, and its colour from sample_color. The face comes off the frame: suggest_font with the card's moment, a region close around its words and the words themselves ranks every face Donkey has and hands back the font, weight, size, tracking and x/y to set. Call it on each card: cards in one face share its weight, size and tracking, but each sits its own way in its space, so its x/y comes from its own read, never from a neighbour's place. A title that cycles through typefaces is one call per face it shows. A no_match verdict means the reference uses a face Donkey does not bundle: say so, and set the closest one it named. Without a suggestion, size comes off the ink: capitals stand about 0.7 of the size tall, so size is the ink height in px at 1080 wide ÷ 0.7 (h × 1920 ÷ 0.7 on 9:16). A cycle whose faces change on the music's beats times itself with set_overlay_animation font_cycle_beats on the soundtrack clip after detect_beats; one whose changes follow the reference's own rhythm takes font_cycle_switches, the timeline seconds detect_cuts reads each change at.
**Then the motion.** set_overlay_keyframes gives each card its own pose track — where it sits, at what scale, at what second. track_element over a move hands you the keys: each move's from/to, start/end and ease, with scale as w over the resting w. set_overlay_animation composes on top for the entrance, the exit, a loop, and what a line does while it holds; 0.3-0.6s reads as an entrance, and a per-letter style is what makes a card read as typed. add_shape, a stretch per call, is the rest of the graphics: bars, cards, rules, a block of color behind a line.
**Look at what you made.** A build that reads right from your own notes is still a guess until you measure it. compare_to_source does: mode sweep scores every frame of the cut against the source — 0 is the same picture, under ${SWEEP_TARGET} reads as one — and names the worst stretches with a grid of where each differs; mode moments shows those peaks beside the source's frames. Sweep as you build, since an error in the first card is in every card after it: look at the worst stretches, fix what differs, sweep again, and stop when the median is under ${SWEEP_TARGET} and no stretch stands out. A \`shift\` means the right thing lands early or late by that much. Read each picture for colour, the words and their face, where a thing sits, its size, and how far along a move is; a title the same colour as the block behind it reads as correct in the document and shows up here. A block keeps its footage's difference, so judge a shot by the cells over what you built.
-**Say what plays.** Close by naming what fills the shots, and what you checked. A shell gets what to drop in: how many shots, how long each one runs, and that the titles and timing are built around them.`,
+**Say what plays.** Close by naming what fills the shots — each Pexels id by shot — and what you checked. A shell gets what to drop in: how many shots, how long each one runs, and that the titles and timing are built around them.`,
"transitions-and-fades": `# Transitions, animations, looks & fades
Route the ask to the right feature:
@@ -660,13 +662,13 @@ Rules:
- Styling is part of the ask, never a bonus. Words the user asked to put on screen — a title, the lines from the audio, a caption run — go on in one plain readable face at one place, timed to what they are for. Per-line typefaces, tilts, color cards, moves, word effects and a repainted frame are a design, and a design is something the user asks for: a lyric video, kinetic type, a named style or vibe, a reference to match, "make it fun". Call add_text_sequence with no \`look\` for the plain job, and leave the caption styling and the background as you found them.
- A skill attached to the message (<skill_relevance>) is the reference for its area: read it before touching that area's settings. It never changes which tool the ask maps to — the tool descriptions decide that ("a better version" is a render, whatever skill rode along). list_skills / read_skill cover an area the attachment left out.
- A reference image or clip the user attaches ("do this", "put our video in this style") carries one of two asks — decide which before acting. When its footage's look is the point (color, lighting, film grade), route to color grading — match_color_grade computes the grade from the reference's own pixels, then the grading tools refine (color-grading skill) — or to look effects or restyle when the reference is a stylized treatment. When it demonstrates an editing device — a tile grid, a mosaic reveal, a split layout, an animated title treatment — the ask is to construct that device on the user's own footage: read the graphics skill's composite-builds section and build it with overlay tracks, masks, and keyframes. Your tools compose into effects far beyond the preset lists, so reach for construction before declaring something out of reach. Construction is free and undoable; a whole-look re-render (restyle_scene) spends credits — when the reading is ambiguous, say what each path would do and let the user pick.
-- A video held up as a reference — a file they imported, a link you brought in, a clip on the timeline — is watched all the way through before anything is copied from it, then rebuilt out of this editor's parts: everything on the screen, every card of type, every shape, every colour and every move, at the time it happens. The footage inside the shots is the one thing you cannot rebuild — add_clip \`blocks\` holds each shot's place in the colour behind it, for the person to fill — and when they want the reference's own picture playing, add_clip \`spans\` lays its shots down instead. What you built is a guess until you look at it beside the thing it copies: compare_to_source scores the cut against the source frame by frame and shows the worst moments side by side, and what differs gets fixed and checked again. The replicating-a-video skill has the flow.
+- A video held up as a reference — a file they imported, a link you brought in, a clip on the timeline — is watched all the way through before anything is copied from it, then rebuilt out of this editor's parts: everything on the screen, every card of type, every shape, every colour and every move, at the time it happens. The footage inside the shots is the one thing you cannot rebuild — add_clip \`blocks\` holds each shot's place in the colour behind it, for the person to fill, or for Pexels footage you find when they ask for the shots filled — and when they want the reference's own picture playing, add_clip \`spans\` lays its shots down instead. What you built is a guess until you look at it beside the thing it copies: compare_to_source scores the cut against the source frame by frame and shows the worst moments side by side, and what differs gets fixed and checked again. The replicating-a-video skill has the flow.
- A Donkey Cut project link or share link the user gives you is a document: read it with read_project (the document is the edit — never watch a reference to learn what its document already states), decide which of this project's sources plays each of its parts from the reference's observed notes and transcripts (a reference look with watch_video's project_link where those leave it dark), then call replicate_project once — the whole edit, or only the items they asked for: a title, one clip's treatment, the music, the caption look — and tune from there. copy_project_media brings files over on their own. The replicating-a-project skill has the flow.
- Captions that sit off the speech are re-timed, never re-transcribed: align_to_audio measures the cut's own waveform and moves each edge onto the voice, on a caption track or on the elements of a text run. Re-transcribing to fix timing throws away the user's wording, and hand-retiming with update_cue guesses at what the audio already says.
- Transcription tools write tracks the user sees — run subtitles_generate / captions_generate only when captions were asked for, never just to read the words: existing cues are in editor_state, sources you have watched or listened to carry their own transcript (media entries marked speech: "transcribed"; get_state includes the timed segments), and audio plays to you via listen_audio. Don't transcribe a video with no speech (subtitles_from_visuals narrates silent footage), and never invent a spoken transcript.
- Voiceovers duck other audio by default so they stay audible. Steer a voiceover's delivery with \`direction\` and inline tags like [whispers] rather than rewriting the script.
- generate_image / generate_video / generate_character_video / voiceover_generate / read_subtitles_aloud / generate_music make media through hosted models (spends the user's Donkey credits, needs sign-in); call them when the user asked for the media itself — a request for the prompt or script gets text in chat. generate_music makes a music track — a vocal-free background bed by default ("add music to this video"), or a full sung song (instrumental:false); it's sung music, so a spoken narration is voiceover_generate. Bundled stock media (stock_search / stock_add) is free — use it when it fits. Media the user attached is in \`media\`; pass those asset ids as generation references when they say "use this", and place project assets in the cut with add_clip when they ask for them there ("make a movie from these photos"). Generated media previews on a chat card the user can expand, drag in, or file away; add it to the timeline (add_to_timeline:true or an index) only when they asked for it in the cut. When it lands on a card, close with a statement of where it is ("The voiceover is on the card — drag it in or tell me to place it."). Write a rich, specific prompt from their shorthand. Video renders take a minute or two. "Generate a video of/about X" means one generate_video clip; only when they name a narrated multi-shot production (a story, episode, narrated short) does generate_scene plan it and stop at the storyboard — the user reviews the frames (regenerate_shot redraws any before approval, no credits), then call approve_scene, since it renders many paid shots (regenerate_shot / recut_scene / restyle_scene revise it after). Close a plan message with the approval path as a statement: "Approve it — say so or hit Approve on the card — and I'll render the shots." Tweaks before approval are theirs to name.
-- You can see and hear: audio the user attaches to their message plays right in it — answer "what does this say" from what you hear, no tool call. For project footage, watch_video samples a source's frames into timestamped contact sheets (scene changes included) — watch before cutting footage you haven't seen, and write each look down with note_source — sheets are conversation-local, notes stay on the source; listen_audio plays a source's audio — an audio asset or a video/clip's own track, one stretch per call, with \`coveredTo\` and \`unheardSeconds\` saying how far it reached and what nobody has played (an ask about how a whole source sounds listens on from \`unheardFrom\` until that is 0); detect_silence finds dead air; find_highlights ranks the stretches of a long source worth clipping, from what is said over them; detect_beats reads the music's beat grid (persisted on the asset, drawn as dots on its clips, snapped to by every drag) for cutting to the music; capture_frame shows one rendered frame, at the time you pass or the playhead; compare_to_source scores the cut against the source it copies and shows the two side by side, which is how copied work is checked; detect_cuts, sample_color and track_element measure a source's cuts, colours and moving elements as numbers. The watching-and-cutting skill has the flow.
+- You can see and hear: audio the user attaches to their message plays right in it — answer "what does this say" from what you hear, no tool call. For project footage, watch_video samples a source's frames into timestamped contact sheets (scene changes included) — watch before cutting footage you haven't seen, and write each look down with note_source — sheets are conversation-local, notes stay on the source; listen_audio plays a source's audio — an audio asset or a video/clip's own track, one stretch per call, with \`coveredTo\` and \`unheardSeconds\` saying how far it reached and what nobody has played (an ask about how a whole source sounds listens on from \`unheardFrom\` until that is 0); detect_silence finds dead air; find_highlights ranks the stretches of a long source worth clipping, from what is said over them; detect_beats reads the music's beat grid (persisted on the asset, drawn as dots on its clips, snapped to by every drag) for cutting to the music; capture_frame shows one rendered frame, at the time you pass or the playhead; compare_to_source scores the cut against the source it copies and shows the two side by side, which is how copied work is checked; detect_cuts, sample_color, track_element and track_camera measure a source's cuts, colours, moving elements and camera moves as numbers. The watching-and-cutting skill has the flow.
- A figure the video states — a number, date, price, name or release in the narration, a title or a caption — goes through check_facts before it goes on screen. Report every claim it does not support, with what the sources say and where, and leave the user's words as they wrote them until they ask.
- End on a statement. The final sentence of every reply states what changed, what failed, or the one suggestion you're making; "let me know if", "would you like", "do you want me to", and every other trailing offer or question are banned closers. The only question that may end a reply is the single clarifying one a genuinely ambiguous or blocked ask earns.`;
}
diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.ts
index 05792b8e..2a7728fe 100644
--- a/site/src/cut/server/ai/headlessTools.ts
+++ b/site/src/cut/server/ai/headlessTools.ts
@@ -43,6 +43,7 @@ const PAGE_MEDIA_TOOLS: ReadonlySet<string> = new Set([
"detect_cuts",
"sample_color",
"track_element",
+ "track_camera",
"suggest_font",
"render_preview",
"create_sticker",
diff --git a/site/src/cut/server/http/ai.ts b/site/src/cut/server/http/ai.ts
index 10b34685..db3355bc 100644
--- a/site/src/cut/server/http/ai.ts
+++ b/site/src/cut/server/http/ai.ts
@@ -44,6 +44,10 @@ import { codexCommand } from "../tool-path";
import { codexAccountModels, createCodexCatalog } from "../ai/codexModels";
import { errorMessage } from "../util";
+/** The one place a Claude chat's WebFetch may go: Pexels' pages, where stock
+ * footage is searched by its own search URLs. Every other host is refused. */
+const PEXELS_FETCH = ["WebFetch(domain:www.pexels.com)", "WebFetch(domain:pexels.com)"];
+
interface ChatBody {
threadId: string;
messages: UIMessage[];
@@ -152,7 +156,11 @@ async function runClaude(
? { pathToClaudeCodeExecutable: process.env.DONKEY_CUT_CLAUDE }
: {}),
systemPrompt: systemPrompt(),
- tools: [], // no built-in tools — the editor MCP server is the whole surface
+ // The editor MCP server is the whole surface, but for one built-in:
+ // WebFetch, held to Pexels, where a replication's stock footage is
+ // searched. Pexels answers a server's own fetch with a challenge, and
+ // the fetch tool reads its search pages; import_url brings a video in.
+ tools: ["WebFetch"],
mcpServers: {
cut: {
type: "stdio",
@@ -160,7 +168,7 @@ async function runClaude(
alwaysLoad: true,
},
},
- allowedTools: ["mcp__cut"],
+ allowedTools: ["mcp__cut", ...PEXELS_FETCH],
permissionMode: "dontAsk",
settingSources: [], // don't drag the user's CLAUDE.md/settings into app chats
// The editor MCP is the whole tool surface. settingSources:[] only drops
diff --git a/site/src/cut/server/pexels.test.ts b/site/src/cut/server/pexels.test.ts
new file mode 100644
index 00000000..e08b42f3
--- /dev/null
+++ b/site/src/cut/server/pexels.test.ts
@@ -0,0 +1,54 @@
+import { describe, expect, test } from "bun:test";
+import { pexelsFileUrl, pexelsVideo } from "./pexels";
+
+describe("pexelsVideo", () => {
+ test("reads the id and the words of a video page link", () => {
+ expect(pexelsVideo("https://www.pexels.com/video/a-man-snowboarding-6947537/")).toEqual({
+ id: "6947537",
+ title: "A man snowboarding",
+ });
+ expect(pexelsVideo("https://pexels.com/es-es/video/ski-resort-853808")).toEqual({ id: "853808", title: "Ski resort" });
+ });
+
+ test("reads a file link on the CDN, which is its own file", () => {
+ expect(pexelsVideo("https://videos.pexels.com/video-files/31934986/13602850_1920_1080_30fps.mp4")).toEqual({
+ id: "31934986",
+ title: "Pexels 31934986",
+ file: "https://videos.pexels.com/video-files/31934986/13602850_1920_1080_30fps.mp4",
+ });
+ });
+
+ test("leaves every other link alone", () => {
+ expect(pexelsVideo("https://www.pexels.com/search/videos/snow/")).toBeNull();
+ expect(pexelsVideo("https://www.tiktok.com/@a/video/7626749283018263815")).toBeNull();
+ expect(pexelsVideo("https://www.pexels.com/photo/snow-123/")).toBeNull();
+ });
+});
+
+describe("pexelsFileUrl", () => {
+ // The CDN's files for one video, as a HEAD answers them.
+ const cdn = (files: string[]) => async (url: string) => files.some((f) => url.endsWith(`/${f}`));
+
+ test("takes the 1080p file at whatever rate it was shot", async () => {
+ const has = cdn(["6286880-uhd_3840_2160_30fps.mp4", "6286880-hd_1920_1080_30fps.mp4", "6286880-hd_1280_720_60fps.mp4"]);
+ expect(await pexelsFileUrl("6286880", has)).toBe(
+ "https://videos.pexels.com/video-files/6286880/6286880-hd_1920_1080_30fps.mp4"
+ );
+ });
+
+ test("a portrait video's 1080p is 1080 wide", async () => {
+ expect(await pexelsFileUrl("9", cdn(["9-hd_1080_1920_25fps.mp4", "9-sd_540_960_25fps.mp4"]))).toBe(
+ "https://videos.pexels.com/video-files/9/9-hd_1080_1920_25fps.mp4"
+ );
+ });
+
+ test("an old video with no 1080p falls back to the largest it has", async () => {
+ expect(await pexelsFileUrl("1581362", cdn(["1581362-sd_960_540_30fps.mp4", "1581362-sd_640_360_30fps.mp4"]))).toBe(
+ "https://videos.pexels.com/video-files/1581362/1581362-sd_960_540_30fps.mp4"
+ );
+ });
+
+ test("nothing on the CDN is null", async () => {
+ expect(await pexelsFileUrl("1", cdn([]))).toBeNull();
+ });
+});
diff --git a/site/src/cut/server/pexels.ts b/site/src/cut/server/pexels.ts
new file mode 100644
index 00000000..d22c6be9
--- /dev/null
+++ b/site/src/cut/server/pexels.ts
@@ -0,0 +1,67 @@
+// A Pexels video page names its video but the page itself answers a server's
+// fetch with a challenge, so its file is found on the CDN instead: an older
+// video keeps its renditions at a predictable path, named by size and frame
+// rate. A newer one names its files by a second id only the page shows, so
+// its CDN link comes off the page and imports as it is.
+
+const PAGE_RE = /^https?:\/\/(?:www\.)?pexels\.com\/(?:[a-z]{2}-[a-z]{2}\/)?video\/([a-z0-9-]*?)-?(\d+)\/?(?:[?#].*)?$/i;
+
+const CDN = "https://videos.pexels.com/video-files";
+
+const FILE_RE = /^https?:\/\/videos\.pexels\.com\/video-files\/(\d+)\/[^/?#]+\.(?:mp4|mov|webm)(?:[?#].*)?$/i;
+
+/** The renditions worth importing, best first: 1080p either way up, then the
+ * larger and smaller sizes a video shot before 1080p or above it may hold
+ * instead. */
+const SIZES = [
+ "hd_1920_1080",
+ "hd_1080_1920",
+ "uhd_2560_1440",
+ "uhd_1440_2560",
+ "hd_1280_720",
+ "hd_720_1280",
+ "uhd_3840_2160",
+ "uhd_2160_3840",
+ "sd_960_540",
+ "sd_540_960",
+ "sd_640_360",
+ "sd_360_640",
+];
+
+/** The rates a rendition's name carries; one video's sizes can differ. */
+const RATES = [24, 25, 30, 50, 60];
+
+/** The video a Pexels link points at: its id and a title — the words of a
+ * page's slug — and, for a link to one of its files on the CDN, that file.
+ * Null for any other link. */
+export function pexelsVideo(url: string): { id: string; title: string; file?: string } | null {
+ const f = FILE_RE.exec(url.trim());
+ if (f) return { id: f[1], title: `Pexels ${f[1]}`, file: url.trim() };
+ const m = PAGE_RE.exec(url.trim());
+ if (!m) return null;
+ const words = m[1].replace(/-/g, " ").trim();
+ return { id: m[2], title: words ? words[0].toUpperCase() + words.slice(1) : `Pexels ${m[2]}` };
+}
+
+/** Whether a file is there, by a HEAD request. */
+async function headOk(url: string): Promise<boolean> {
+ try {
+ const res = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(10_000) });
+ return res.ok;
+ } catch {
+ return false;
+ }
+}
+
+/** The best rendition of a Pexels video on the CDN, or null when it holds
+ * none. Each size's rates are asked at once and the first size with a file
+ * wins. */
+export async function pexelsFileUrl(id: string, exists: (url: string) => Promise<boolean> = headOk): Promise<string | null> {
+ for (const size of SIZES) {
+ const urls = RATES.map((r) => `${CDN}/${id}/${id}-${size}_${r}fps.mp4`);
+ const found = await Promise.all(urls.map(exists));
+ const hit = urls.find((_, i) => found[i]);
+ if (hit) return hit;
+ }
+ return null;
+}
diff --git a/site/src/cut/server/urlDownload.ts b/site/src/cut/server/urlDownload.ts
index 39b52d04..baa764ea 100644
--- a/site/src/cut/server/urlDownload.ts
+++ b/site/src/cut/server/urlDownload.ts
@@ -7,12 +7,13 @@ import { Defuddle } from "defuddle/node";
import { parseHTML } from "linkedom";
import { convertToMp4, fitsAlready, streamCodecs } from "./convert";
import type { LibrarySource } from "./library";
+import { pexelsFileUrl, pexelsVideo } from "./pexels";
// Bring back whatever a URL points at. yt-dlp, resolved from PATH, pulls
// anything with a video or audio stream (TikTok, YouTube, Instagram, …); X's
// auth-free syndication endpoint covers the posts it can't touch — photos, an
-// Article, a post that is only words; and any other page falls back to its
-// readable text. Shared by the local engine's import routes (urlImport.ts) and
+// Article, a post that is only words; a Pexels video page brings its file
+// from the CDN; and any other page falls back to its readable text. Shared by the local engine's import routes (urlImport.ts) and
// the cloud render worker's import_url jobs.
const MEDIA_EXT = /\.(mp4|mov|m4v|webm|mkv|mp3|m4a|aac|wav|ogg|flac)$/i;
@@ -85,6 +86,8 @@ export async function download(
opts: DownloadOptions = {}
): Promise<Downloaded> {
if (IMAGE_URL_RE.test(url)) return downloadDirectImage(url, tmp);
+ const pexels = pexelsVideo(url);
+ if (pexels) return downloadPexels(pexels, url, tmp, opts);
const rungs: (() => Promise<Downloaded | null>)[] = [];
const postId = X_STATUS_RE.exec(url)?.[1];
// An X post's lower rungs are the post itself — its stills, its words — so a
@@ -120,6 +123,31 @@ export async function download(
throw first instanceof Error ? first : new Error("Nothing could be imported from that URL.");
}
+/** A Pexels video by its page link: the page refuses a server, so its best
+ * rendition comes straight off the CDN, titled from the page's slug and kept
+ * against the page, so the asset says which Pexels video it is. */
+async function downloadPexels(
+ video: { id: string; title: string; file?: string },
+ pageUrl: string,
+ tmp: string,
+ opts: DownloadOptions
+): Promise<Downloaded> {
+ const file = video.file ?? (await pexelsFileUrl(video.id));
+ // Newer videos name their files by a second id only the page shows.
+ if (!file)
+ throw new Error(
+ `Pexels video ${video.id} keeps its files under names only its page shows: read the page for its videos.pexels.com/video-files/${video.id}/… link (the 1920 or 1080 wide one) and import that link.`
+ );
+ const got = await downloadMedia(file, tmp, opts);
+ const title = (video.file ? video.title : `${video.title} (Pexels ${video.id})`).slice(0, 120);
+ return {
+ ...got,
+ files: got.files.slice(0, 1).map((f) => ({ ...f, title })),
+ source: { url: video.file ? `https://www.pexels.com/video/${video.id}/` : pageUrl, title, uploader: "Pexels" },
+ text: undefined,
+ };
+}
+
/** yt-dlp knows this site as a video host and still could not bring the video
* back. The import fails on it rather than falling through to the page. */
class MediaHostError extends Error {}