Open source CapCut alternative.
TypeScript
Merge branch 'gap/measure' into gap/all
David Led8cdb57ff2ae
diff --cc site/src/cut/components/AiPanel.tools.tsindex 0ff12de0,906d0a62..361063f6--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@@ -12,7 -12,30 +12,31 @@@ import { WATCH_DETAIL_NOTES, WATCH_DETAimport { FACT_CLAIM_MAX_CHARS, FACT_CLAIMS_MAX } from "@/lib/inference/factCheck";import { URL_BATCH_MAX } from "@/cut/lib/urlBatch";import { FIND_MEDIA_DEFAULT, FIND_MEDIA_MAX, FIND_MEDIA_SNAPSHOT_CAP } from "@/cut/lib/findMedia";+import { COMPARE_MODES, MAX_COMPARE, SWEEP_TARGET } from "@/cut/lib/frameDiff";+ import { EASE_IDS } from "@donkeycut/effects-kit";+ import {+ DETECT_CUTS_MAX,+ DETECT_CUTS_SPAN_MAX,+ SAMPLE_COLOR_MAX,+ SAMPLE_COLORS_K_MAX,+ TRACK_FRAMES_MAX,+ TRACK_MATCHES,+ TRACK_TOLERANCE_DEFAULT,+ } from "@/cut/lib/measure/limits";++ /** A box on a source's frame as the editor places one: centre and size. */+ const frameBox = (what: string) => ({+ ...obj(+ {+ x: num("Centre x, fraction of frame width"),+ y: num("Centre y, fraction of frame height"),+ w: num("Width, fraction of frame width"),+ h: num("Height, fraction of frame height"),+ },+ ["x", "y", "w", "h"]+ ),+ description: what,+ });export const AI_PANEL_TOOLS = [{@@@ -165,14 -179,71 +189,75 @@@],},},+ offset: num("Source second minus timeline second, for every moment without its own source and for the sweep (default 0)"),+ from: num("sweep: timeline second to start at (default 0)"),+ to: num("sweep: timeline second to stop at (default the cut's end)"),+ step: num("sweep: seconds between samples (default one frame, widened so a long span fits the call)"),},- ["asset_id", "times"]+ ["asset_id"]),},+ {+ name: "detect_cuts",+ description:+ `Find every hard cut in a stretch of a video source, to the frame. Reads every frame between from and to at a few dozen pixels, scores how much of the picture changed since the frame before, and reports each frame that opens a new shot: \`t\` in SOURCE seconds to the millisecond (the new shot's first frame), its \`frame\` index at the source's \`fps\`, and \`change\`, the percent of the picture that changed — near 100 is a new shot, 10-30 a card or a block of type swapping over the same background. \`shots\` lists the stretches between the cuts with their length in frames. watch_video's sceneChanges come from samples a step apart and lose a shot shorter than the step; this reads every frame, so a five-frame shot in a fast montage is found. Run it before blocking out a reference: add_clip \`blocks\` takes one block per shot at that shot's length. Hard cuts only — a dissolve or a fade changes a little each frame and reads as none. Up to ${DETECT_CUTS_SPAN_MAX}s and ${DETECT_CUTS_MAX} cuts a call; when \`truncated\`, call again from \`coveredTo\`.`,+ inputSchema: obj(+ {+ asset_id: str("Project asset id of the video"),+ from: num("Source start s (default 0)"),+ to: num(`Source end s (default the source's end, at most ${DETECT_CUTS_SPAN_MAX}s past from)`),+ },+ ["asset_id"]+ ),+ },+ {+ name: "sample_color",+ description:+ `Read the colour of a source's pixels: the hex of a region or a point of a frame at a source second, so a block, a shape, a title or a background takes the reference's colour as a number. Contact-sheet cells are JPEGs and shift colour; this reads the decoded frame as the preview shows it. Each sample is a moment \`t\` (SOURCE seconds) and a box — x/y its centre and w/h its size as frame fractions, the box add_shape takes; leave out w/h to read a point. Returns per sample \`hex\`, the region's median colour, and \`spread\`, how far its pixels stray from it on average (0-255: under 10 is a flat fill, over 40 a mix — read \`dominant\` there). \`colors\` 2-${SAMPLE_COLORS_K_MAX} adds \`dominant\`, the colours the region is made of with their shares, largest first: a card and the type on it, a bar and the shot behind it. Read a point inside a flat fill for its colour, a box over a title for its ink and its ground. Up to ${SAMPLE_COLOR_MAX} samples a call, at one moment or each at its own. Works on a still image too (t is ignored).`,+ inputSchema: obj(+ {+ asset_id: str("Project asset id (video or image)"),+ samples: {+ type: "array",+ description: `Up to ${SAMPLE_COLOR_MAX} regions to read`,+ items: obj(+ {+ t: num("Source second"),+ x: num("Centre x, fraction of frame width"),+ y: num("Centre y, fraction of frame height"),+ w: num("Width, fraction of frame width (omit with h for a point)"),+ h: num("Height, fraction of frame height"),+ },+ ["x", "y"]+ ),+ },+ colors: num(`Also list this many dominant colours per region, 2..${SAMPLE_COLORS_K_MAX} (default: none)`),+ },+ ["asset_id", "samples"]+ ),+ },+ {+ name: "track_element",+ description:+ `Measure where an element of a source's picture sits, frame by frame, and how it moves, so a grow, a slide or a pop is keyed with set_overlay_keyframes to the reference's own numbers. Give a stretch of a video (SOURCE seconds) and a \`seed\` box on the element at \`from\`; boxes in and out are x/y centre and w/h size as frame fractions — what add_shape, add_title and the keys take. The element is found by colour: \`color\` (sample_color reads it), or else the seed's median colour on the first frame, so draw the seed inside the element's edges at a moment it is on screen. \`match\` shape follows the one connected piece of that colour the seed sits on — a card, a bar, a sticker — as it moves and changes size; ink takes every pixel of that colour inside the seed (or \`within\`), the ink of a line of type whose letters are separate pieces: its centre is where the title sits and its height is about 0.7 of the font size when the line is capitals. Leave out \`to\` to measure one frame. \`moves\` lists each move of x, y, w and h with from/to (the source seconds it starts and lands), start/end values, the ease that fits it (${EASE_IDS.join(", ")}) and \`error\`, the RMS miss as a share of the move: under 0.03 is that curve, over 0.1 none fits and the move keys from \`path\` point by point. A size move keys as scale: w over the element's resting w. \`path\` lists the box wherever it changes (a hold is one entry); a null box is a frame where the element was not found. Up to ${TRACK_FRAMES_MAX} frames a call; when \`truncated\`, continue from \`coveredTo\`. Works on a still image too (one frame).`,+ inputSchema: obj(+ {+ asset_id: str("Project asset id (video or image)"),+ from: num("Source second the seed shows the element at"),+ to: num("Source end s; omit to measure the one frame at from"),+ seed: frameBox("A box on the element at from — inside its edges for shape, around the line for ink"),+ within: frameBox("Only look inside this box (default: the whole frame for shape, the seed for ink)"),+ color: str("The element's colour as #RRGGBB (default: the seed's median colour)"),+ tolerance: num(`Per-channel distance from color that still counts as it, 0..255 (default ${TRACK_TOLERANCE_DEFAULT}); raise it for a gradient or a soft edge`),+ match: {+ type: "string",+ enum: [...TRACK_MATCHES],+ description: "shape: the one connected piece the seed sits on (default). ink: every pixel of the colour inside the seed or within — type",+ },+ },+ ["asset_id", "from", "seed"]+ ),+ },{name: "find_highlights",description:diff --cc site/src/cut/lib/aiTools.tsindex 8a07b4b5,c30154c7..81794399--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@@ -232,12 -233,12 +232,14 @@@ import { loadLibraryLut, lutIdOf, lutLaimport { type RawFrame, type ReadGrade, rawFrameAt, sampleClipRawFrame, sourceProfileOf, toBaseRendering, toGradedRendering } from "./baseFrame";import { applyOverlayPatchSettled, assetClipUses, assetIdsOnAnyTimeline, clipLen, footprints, nextFreeStart, track0Clips, laneGapAt, getClipSpans, overlayLaneOrder, overlayLayers, parkedTransitions, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";import { playheadAt } from "./playhead";-import { mixSpecFor, renderProjectFrame, renderProjectFrames } from "./exportRender";+import { mixSpecFor, renderProjectFrame } from "./exportRender";import { renderMix } from "./audioMix";import { measureLoudnessSliced, roundedLoudness, type LoudnessMeasure } from "./loudness";- import { readMediaFileSize } from "./mediaRead";+ import { framesAt, readMediaFileSize } from "./mediaRead";+import { COMPARE_MODES, MAX_COMPARE, SWEEP_TARGET } from "./frameDiff";+import { compareMoments, sweepSource } from "./sourceCompare";+ import { sampleColors, scanCuts, trackElement, type FrameBox } from "./measure/sourceMeasure";+ import { DETECT_CUTS_MAX, DETECT_CUTS_SPAN_MAX, SAMPLE_COLOR_MAX, SAMPLE_COLORS_K_MAX, TRACK_FRAMES_MAX, TRACK_MATCHES, TRACK_PATH_MAX, TRACK_TOLERANCE_DEFAULT } from "./measure/limits";import { renderStageFrame, storeStageStill } from "./stageFrame";import { createRasterCanvas, decodeRasterImageUrl, rasterCanvasToDataUrl } from "./raster";import { buildAiContext, describeCameraCard, describeDoc, hiddenFromChat, OTHER_CHATS_MEDIA_NOTE, roundT } from "./aiContext";diff --cc site/src/cut/server/ai/catalog.tsindex 65c6c92b,7ad19be3..12f21073--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@@ -353,13 -353,15 +354,15 @@@ Someone points at a video — a file th**Clipping a long source.** A talk, an interview or a stream is mostly material that only makes sense in place. find_highlights ranks its stretches from the transcript, so an hour is read for its handful of moments without watching all of it; watch_video the spans it returns before committing, since the ranking never saw the picture. add_clip \`spans\` lays a pick down — several stretches of one source as one run of clips, the material between them left out — and a two-span result is one clip whose halves were never next to each other.**Watch the whole thing first.** Every decision below comes off the record, so the record has to be complete: watch from 0 and keep going until \`unwatchedSeconds\` is 0, writing each pass into note_source before the next one. Anything over a couple of minutes gets a coarse pass end to end first (\`interval_seconds\` 15-30) to learn its shape, then a dense pass (0.5) over the stretches that carry text or cut fast. Read every card word for word — its text, its face, where it sits in the frame, when it comes and goes — because a turn later the sheets are gone and the notes are all you have. Reading type is what \`detail\` is for: the dense pass over a stretch carrying words runs at \`detail\` read, and a face or treatment you are about to name runs at original, where the frame arrives at the source's own resolution. A scan cell is a quarter of the frame's width; a small caption in one is a smudge, and a smudge read as a word is how a card ends up saying something the video never said. When the cards ARE the narration — a caption track of what the speaker is saying — the transcript already holds every word, and the close passes are for how they are set: face, weight, size, colour, placement, how many words ride a card, how it enters and leaves. Read a few of those at original and take the rest from the transcript.- **Block out the cuts.** \`sceneChanges\` are the cut times: shot 1 runs from 0 to the first, shot n from one to the next, the last to the end. add_clip with \`blocks\` lays the whole skeleton down in one call — one block per shot on track 0, each the length that shot runs, \`color\` set to what the reference has behind that shot and \`label\` naming what the person drops there. The label rides the timeline chip and the editor state; the frame itself shows the colour alone, so a shot never plays a sentence describing itself. A block is filled by dropping footage on it — it takes the block's place and its length — or with replace_item, so everything you build on top keeps its timing. Cuts timed to a reference are timed to what it sounds like, so that sound belongs on the soundtrack under them: detach_audio with the reference's \`assetId\` puts its whole track there and places none of its picture. When the ask is the reference's own picture playing, add_clip \`spans\` lays its shots down instead, one span per shot at those same times.+ **Measure it.** A contact sheet tells you what happens; the numbers come from the source's pixels. detect_cuts reads every frame and returns each cut on its frame and the \`shots\` between them with their lengths — run it over the whole source, continuing from \`coveredTo\`. sample_color reads the hex of any region at any second. track_element gives an element's box frame by frame and the ease of each move; with match ink it gives a title's ink box.- **Rebuild the text as elements.** One add_title per card — the words exactly as they read, the face from the graphics skill's font ids, size in px at 1080 wide, and x/y as the fraction of the frame it sits at (a lower-third card is around y 0.75, a centered statement 0.5). A run of lines timed one after another is add_text_sequence in one call. Match the case the reference uses; match its weight and its color against the frame behind it.+ **Block out the cuts.** add_clip with \`blocks\` lays the skeleton from detect_cuts' \`shots\` in one call — one block per shot on track 0, each that shot's length, \`color\` what sample_color reads behind it and \`label\` naming what the person drops there. The label rides the timeline chip and the editor state; the frame itself shows the colour alone, so a shot never plays a sentence describing itself. A block is filled by dropping footage on it — it takes the block's place and its length — or with replace_item, so everything you build on top keeps its timing. Cuts timed to a reference are timed to what it sounds like, so that sound belongs on the soundtrack under them: detach_audio with the reference's \`assetId\` puts its whole track there and places none of its picture. When the ask is the reference's own picture playing, add_clip \`spans\` lays its shots down instead, one span per shot at those same times.- **Then the motion.** set_overlay_keyframes gives each card its own pose track — where it sits, at what scale, at what second — which is how the timing lands on the reference's own frames. set_overlay_animation composes on top for the entrance, the exit, a loop, and what a line does while it holds; 0.3-0.6s reads as an entrance, and a per-letter style is what makes a card read as typed. add_shape is the rest of the graphics: bars, cards, rules, a block of color behind a line.+ **Rebuild the text as elements.** One add_title per card — the words exactly as they read, the face from the graphics skill's font ids, size in px at 1080 wide, and x/y the centre of its ink box (track_element, match ink, the text's colour); capitals stand about 0.7 of the size tall, so size is the ink height in px at 1080 wide ÷ 0.7 (h × 1920 ÷ 0.7 on 9:16). A run of lines timed one after another is add_text_sequence in one call. Match the case and weight the reference uses, and its colour from sample_color.++ **Then the motion.** set_overlay_keyframes gives each card its own pose track — where it sits, at what scale, at what second. track_element over a move hands you the keys: each move's from/to, start/end and ease, with scale as w over the resting w. set_overlay_animation composes on top for the entrance, the exit, a loop, and what a line does while it holds; 0.3-0.6s reads as an entrance, and a per-letter style is what makes a card read as typed. add_shape is the rest of the graphics: bars, cards, rules, a block of color behind a line.-**Look at what you made.** Watching told you what the reference does; it does not tell you whether what you made does it too, and a build that reads right from your own notes is still a guess until you see it. compare_to_source puts the source's frame and yours side by side at the same moment, a few at a time. Run it as you build, not once at the end: the first few shots, then each stretch as you finish it — an error in the first card is in every card after it. Spread the moments over what you laid down, including the middle of a move, and read each pair for colour, the words and their face, where a thing sits, its size, and how far along a move is. A title the same colour as the block behind it, or a shape covering the thing it was meant to sit beside, reads as correct in the document and is only visible here. Fix what differs, check the moments you changed, and keep going until a pass turns up nothing.+**Look at what you made.** Watching told you what the reference does; it does not tell you whether what you made does it too, and a build that reads right from your own notes is still a guess until you measure it. compare_to_source does: mode sweep scores every frame of the cut against the source — 0 is the same picture, under ${SWEEP_TARGET} reads as one — and names the worst stretches with a grid of where each differs; mode moments shows those peaks beside the source's frames. Sweep as you build, since an error in the first card is in every card after it: look at the worst stretches, fix what differs, sweep again, and stop when the median is under ${SWEEP_TARGET} and no stretch stands out. A \`shift\` means the right thing lands early or late by that much. Read each picture for colour, the words and their face, where a thing sits, its size, and how far along a move is; a title the same colour as the block behind it reads as correct in the document and shows up here. A block keeps its footage's difference, so judge a shot by the cells over what you built.**Say what plays.** Close by naming what fills the shots, and what you checked. A shell gets what to drop in: how many shots, how long each one runs, and that the titles and timing are built around them.`,@@@ -663,7 -665,7 +666,7 @@@ Rules- Transcription tools write tracks the user sees — run subtitles_generate / captions_generate only when captions were asked for, never just to read the words: existing cues are in editor_state, sources you have watched or listened to carry their own transcript (media entries marked speech: "transcribed"; get_state includes the timed segments), and audio plays to you via listen_audio. Don't transcribe a video with no speech (subtitles_from_visuals narrates silent footage), and never invent a spoken transcript.- Voiceovers duck other audio by default so they stay audible. Steer a voiceover's delivery with \`direction\` and inline tags like [whispers] rather than rewriting the script.- generate_image / generate_video / generate_character_video / voiceover_generate / read_subtitles_aloud / generate_music make media through hosted models (spends the user's Donkey credits, needs sign-in); call them when the user asked for the media itself — a request for the prompt or script gets text in chat. generate_music makes a music track — a vocal-free background bed by default ("add music to this video"), or a full sung song (instrumental:false); it's sung music, so a spoken narration is voiceover_generate. Bundled stock media (stock_search / stock_add) is free — use it when it fits. Media the user attached is in \`media\`; pass those asset ids as generation references when they say "use this", and place project assets in the cut with add_clip when they ask for them there ("make a movie from these photos"). Generated media previews on a chat card the user can expand, drag in, or file away; add it to the timeline (add_to_timeline:true or an index) only when they asked for it in the cut. When it lands on a card, close with a statement of where it is ("The voiceover is on the card — drag it in or tell me to place it."). Write a rich, specific prompt from their shorthand. Video renders take a minute or two. "Generate a video of/about X" means one generate_video clip; only when they name a narrated multi-shot production (a story, episode, narrated short) does generate_scene plan it and stop at the storyboard — the user reviews the frames (regenerate_shot redraws any before approval, no credits), then call approve_scene, since it renders many paid shots (regenerate_shot / recut_scene / restyle_scene revise it after). Close a plan message with the approval path as a statement: "Approve it — say so or hit Approve on the card — and I'll render the shots." Tweaks before approval are theirs to name.- - You can see and hear: audio the user attaches to their message plays right in it — answer "what does this say" from what you hear, no tool call. For project footage, watch_video samples a source's frames into timestamped contact sheets (scene changes included) — watch before cutting footage you haven't seen, and write each look down with note_source — sheets are conversation-local, notes stay on the source; listen_audio plays a source's audio — an audio asset or a video/clip's own track, one stretch per call, with \`coveredTo\` and \`unheardSeconds\` saying how far it reached and what nobody has played (an ask about how a whole source sounds listens on from \`unheardFrom\` until that is 0); detect_silence finds dead air; find_highlights ranks the stretches of a long source worth clipping, from what is said over them; detect_beats reads the music's beat grid (persisted on the asset, drawn as dots on its clips, snapped to by every drag) for cutting to the music; capture_frame shows one rendered frame, at the time you pass or the playhead; compare_to_source scores the cut against the source it copies and shows the two side by side, which is how copied work is checked. The watching-and-cutting skill has the flow.-- You can see and hear: audio the user attaches to their message plays right in it — answer "what does this say" from what you hear, no tool call. For project footage, watch_video samples a source's frames into timestamped contact sheets (scene changes included) — watch before cutting footage you haven't seen, and write each look down with note_source — sheets are conversation-local, notes stay on the source; listen_audio plays a source's audio — an audio asset or a video/clip's own track, one stretch per call, with \`coveredTo\` and \`unheardSeconds\` saying how far it reached and what nobody has played (an ask about how a whole source sounds listens on from \`unheardFrom\` until that is 0); detect_silence finds dead air; find_highlights ranks the stretches of a long source worth clipping, from what is said over them; detect_beats reads the music's beat grid (persisted on the asset, drawn as dots on its clips, snapped to by every drag) for cutting to the music; capture_frame shows one rendered frame, at the time you pass or the playhead; compare_to_source puts that frame beside the source's own at the same moment, which is how work copied from something is checked against it; detect_cuts, sample_color and track_element measure a source's cuts, colours and moving elements as numbers. The watching-and-cutting skill has the flow.++- You can see and hear: audio the user attaches to their message plays right in it — answer "what does this say" from what you hear, no tool call. For project footage, watch_video samples a source's frames into timestamped contact sheets (scene changes included) — watch before cutting footage you haven't seen, and write each look down with note_source — sheets are conversation-local, notes stay on the source; listen_audio plays a source's audio — an audio asset or a video/clip's own track, one stretch per call, with \`coveredTo\` and \`unheardSeconds\` saying how far it reached and what nobody has played (an ask about how a whole source sounds listens on from \`unheardFrom\` until that is 0); detect_silence finds dead air; find_highlights ranks the stretches of a long source worth clipping, from what is said over them; detect_beats reads the music's beat grid (persisted on the asset, drawn as dots on its clips, snapped to by every drag) for cutting to the music; capture_frame shows one rendered frame, at the time you pass or the playhead; compare_to_source scores the cut against the source it copies and shows the two side by side, which is how copied work is checked; detect_cuts, sample_color and track_element measure a source's cuts, colours and moving elements as numbers. The watching-and-cutting skill has the flow.- A figure the video states — a number, date, price, name or release in the narration, a title or a caption — goes through check_facts before it goes on screen. Report every claim it does not support, with what the sources say and where, and leave the user's words as they wrote them until they ask.- End on a statement. The final sentence of every reply states what changed, what failed, or the one suggestion you're making; "let me know if", "would you like", "do you want me to", and every other trailing offer or question are banned closers. The only question that may end a reply is the single clarifying one a genuinely ambiguous or blocked ask earns.`;}