Open source CapCut alternative.
TypeScript
fix(site): split a named item, and read a faint cut as a cutout [rebuild]
David Le9cf74cbd4096
diff --git a/site/src/cut/components/Timeline.tools.ts b/site/src/cut/components/Timeline.tools.tsindex dbb75b07..ff7b9f22 100644--- a/site/src/cut/components/Timeline.tools.ts+++ b/site/src/cut/components/Timeline.tools.ts@@ -100,8 +100,11 @@ export const TIMELINE_TOOLS = [{name: "split_at",description:- `Split selected items and group members at a time, like pressing S. Supported kinds: ${ITEM_KIND_IDS.filter(canSplitItem).join(", ")}. Transitions remain whole. With no selection, split the video on track 0. Omit t for the playhead. Times stay fixed; the result includes the new selected ids.`,- inputSchema: obj({ t: num("Timeline seconds to cut at (optional)") }),+ `Split items and their group members at a time, like pressing S: the items \`ids\` names, else the selection, else the video on track 0. Supported kinds: ${ITEM_KIND_IDS.filter(canSplitItem).join(", ")}. Transitions remain whole. Omit t for the playhead. Times stay fixed; the result includes the new selected ids.`,+ inputSchema: obj({+ t: num("Timeline seconds to cut at (optional)"),+ ids: { type: "array", items: { type: "string" }, description: "Ids of the items to split, any track or kind (optional; default the selection)" },+ }),},{name: "move_clip",diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex d7e41cb3..617186cb 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -2771,11 +2771,25 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {},split_at: (s, input) => {+ const ids = Array.isArray(input.ids) ? input.ids.filter((id): id is string => typeof id === "string") : [];+ const items = ids.map((id) => {+ const kind = ITEM_KIND_IDS.find((k) => ITEM_KINDS[k].list(s).some((item) => item.id === id));+ if (!kind) throw new ToolError(`No timeline item ${id}.`);+ return { kind, id };+ });const before = ITEM_KIND_IDS.reduce((n, kind) => n + ITEM_KINDS[kind].list(s).length, 0);- s.splitAtPlayhead(isNum(input.t) ? input.t : undefined);+ s.splitAtPlayhead(isNum(input.t) ? input.t : undefined, items);const after = useEditor.getState();const made = ITEM_KIND_IDS.reduce((n, kind) => n + ITEM_KINDS[kind].list(after).length, 0) - before;- if (made === 0) throw new ToolError("Nothing to split at that time.");+ if (made === 0) {+ // An item splits only strictly inside its span: on its edge it is cut already.+ if (items.length) throw new ToolError("Nothing to split: none of those items runs across that time.");+ if (s.multiSelection.length || s.selection)+ throw new ToolError("Nothing to split: the selection doesn't run across that time. Name what to split with ids.");+ throw new ToolError(+ "Nothing to split: with nothing selected only the track-0 video splits, and none of it runs across that time (a clip edge is a cut already). Name an overlay clip, title or other item with ids."+ );+ }return { split: made, selection: after.multiSelection, ...tracksAfter() };},diff --git a/site/src/cut/lib/splitEdit.test.ts b/site/src/cut/lib/splitEdit.test.tsindex 23a0f101..30f57572 100644--- a/site/src/cut/lib/splitEdit.test.ts+++ b/site/src/cut/lib/splitEdit.test.ts@@ -164,3 +164,20 @@ describe("a recording bound to the video", () => {await expect(runAiTool("sync_audio", { clipId: "a", audio_asset_id: "cam" })).rejects.toThrow();});});++describe("split_at", () => {+ test("ids names what to split, an overlay clip included, with nothing selected", async () => {+ useEditor.setState({ clips: [...useEditor.getState().clips, clip("top", 2, 30, 34, { track: 1 })] });+ const result = await runAiTool("split_at", { t: 3, ids: ["top"] });+ const tops = useEditor.getState().clips.filter((c) => c.track === 1).sort((x, y) => x.start - y.start);+ expect(result).toMatchObject({ split: 1 });+ expect(tops.map((c) => [c.start, c.in, c.out])).toEqual([[2, 30, 31], [3, 31, 34]]);+ expect(useEditor.getState().clips.filter((c) => c.track === 0)).toHaveLength(2);+ });++ test("a time on a clip edge says how to reach another item", async () => {+ useEditor.setState({ clips: [...useEditor.getState().clips, clip("top", 2, 30, 34, { track: 1 })] });+ await expect(runAiTool("split_at", { t: 4 })).rejects.toThrow("ids");+ await expect(runAiTool("split_at", { t: 7, ids: ["top"] })).rejects.toThrow("runs across");+ });+});diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex 67718be6..77425d99 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -861,8 +861,9 @@ export interface EditorState {/** "Detach Audio": lift the selected clip's sound onto the* soundtrack track (and mute the clip) so it can be cut independently. */detachAudio: () => void;- /** Split at the given time, or the playhead when omitted. */- splitAtPlayhead: (at?: number) => void;+ /** Split at the given time, or the playhead when omitted: the given items,+ * else the selection, else the track-0 video there. */+ splitAtPlayhead: (at?: number, items?: NonNullable<Selection>[]) => void;setPublish: (patch: Partial<{ caption: string; tags: string; soundTitle: string; handle: string }>) => void;setNotes: (patch: Partial<{ text: string; publishedAt: string; links: string[] }>) => void;/** Kick off (and poll) an on-device transcription of the current cut. */@@ -4052,11 +4053,11 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {}));},- splitAtPlayhead: (at) => {+ splitAtPlayhead: (at, items) => {const s = get();const t = at ?? playheadAt();if (s.readOnly || !Number.isFinite(t)) return;- const selected = s.multiSelection.length ? s.multiSelection : s.selection ? [s.selection] : s.clips+ const selected = items?.length ? items : s.multiSelection.length ? s.multiSelection : s.selection ? [s.selection] : s.clips.filter((c) => c.track === 0 && t > c.start && t < c.start + clipLen(c)).map((c) => ({ kind: "clip" as const, id: c.id }));const result = splitTimelineItems(s, expandTimelineGroups(s, selected), t, uid);diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex ed75ce1d..cfb3e0ee 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -354,7 +354,7 @@ Someone points at a video — a file they imported, a link you brought in, a cli**Clipping a long source.** A talk, an interview or a stream is mostly material that only makes sense in place. find_highlights ranks its stretches from the transcript, so an hour is read for its handful of moments without watching all of it; watch_video the spans it returns before committing, since the ranking never saw the picture. add_clip \`spans\` lays a pick down — several stretches of one source as one run of clips, the material between them left out — and a two-span result is one clip whose halves were never next to each other.**Watch the whole thing first.** Every decision below comes off the record, so the record has to be complete: watch from 0 and keep going until \`unwatchedSeconds\` is 0, writing each pass into note_source before the next one. Anything over a couple of minutes gets a coarse pass end to end first (\`interval_seconds\` 15-30) to learn its shape, then a dense pass (0.5) over the stretches that carry text or cut fast. Read every card word for word — its text, its face, where it sits in the frame, when it comes and goes — because a turn later the sheets are gone and the notes are all you have. Reading type is what \`detail\` is for: the dense pass over a stretch carrying words runs at \`detail\` read, and a face or treatment you are about to name runs at original, where the frame arrives at the source's own resolution. When the cards ARE the narration — a caption track of what the speaker is saying — the transcript already holds every word, and the close passes are for how they are set: face, weight, size, colour, placement, how many words ride a card, how it enters and leaves. Read a few of those at original and take the rest from the transcript.-**Measure it.** A contact sheet tells you what happens; the numbers come from the source's pixels. A reference recorded off a screen — a phone filming an editing app, a letterboxed film — has its picture inside a bigger frame: find_picture finds that box and keeps it on the source (detect_cuts does on its first read), every measuring tool and compare_to_source then reads inside it, and their x, y, w, h are fractions of it, so they land on the cut's frame as they are; set_aspect to its \`aspect\`. detect_cuts reads every frame: each cut on its frame, the \`shots\` between them, and the \`changes\` inside a shot — a word landing, a word swapping fill for outline, a shape popping in. Run it over the whole source before the read passes, which then land a frame in every shot and change, and time each title from them. Its times are to the millisecond, and every tool lands such a time on the frame it names: write them back as read. sample_color reads the hex of any region at any second. track_element gives an element's box frame by frame and the ease of each move; with match ink it gives a title's ink box. track_camera gives a shot's camera move frame by frame — zoom, shift, turn — and says when the picture under it is one frozen frame. Run it over every shot before you describe how it moves: a beat zoom on a frozen frame, a shake or a slow push reads on a contact sheet as a held shot with exposure flicker, and detect_cuts lists its pulses as whole-frame \`changes\` — a run of those inside one shot is a camera move until track_camera says otherwise.+**Measure it.** A contact sheet tells you what happens; the numbers come from the source's pixels. A reference recorded off a screen — a phone filming an editing app, a letterboxed film — has its picture inside a bigger frame: find_picture finds that box and keeps it on the source (detect_cuts does on its first read), every measuring tool and compare_to_source then reads inside it, and their x, y, w, h are fractions of it, so they land on the cut's frame as they are; set_aspect to its \`aspect\`. detect_cuts reads every frame: each cut on its frame, the \`shots\` between them, and the \`changes\` inside a shot — a word landing, a word swapping fill for outline, a shape popping in. Run it over the whole source before the read passes, which then land a frame in every shot and change, and time each title from them. Its times are to the millisecond, and every tool lands such a time on the frame it names: write them back as read. sample_color reads the hex of any region at any second. track_element gives an element's box frame by frame and the ease of each move; with match ink it gives a title's ink box. track_camera gives a shot's camera move frame by frame — zoom, shift, turn — and says when the picture under it is one frozen frame. Run it over every shot before you describe how it moves: a beat zoom on a frozen frame, a shake or a slow push reads on a contact sheet as a held shot with exposure flicker, and detect_cuts lists its pulses as whole-frame \`changes\` — a run of those inside one shot is a camera move until track_camera says otherwise. A cut whose \`change\` sits far below a new shot's, or a few frames that open just before a real cut, is rarely a shot of its own: it is usually the next shot's subject laid over this one, a cutout flashed in. Look at those frames one at a time, and build every one you find; it is part of the edit's look, not a glitch to skip.**Lay down the sound, then block out the cuts.** Cuts timed to a reference are timed to its sound: detach_audio with the reference's \`assetId\` puts its whole track on the soundtrack and none of its picture. add_clip with \`blocks\` lays the skeleton from \`shots\` in one call — one block per shot on track 0, each its shot's \`from\` and \`to\` as read so every cut lands on its frame, flash cuts included, \`color\` what sample_color reads behind it and \`label\` naming what the person drops there. The label rides the timeline chip; the frame shows the colour alone. Footage dropped on a block, or replace_item, takes its place and its length, so everything built on top keeps its timing. When the ask is the reference's own picture playing, add_clip \`spans\` lays its shots down instead, one span per shot at those same times.diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex f081a387..77338022 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -464,6 +464,11 @@ function ffmpegEncoders(): Promise<Set<string>> {}));}+/** Drops the encoder list, for a test that points PATH at another ffmpeg. */+export function forgetFfmpegEncodersForTest(): void {+ encodersCache = null;+}+/** Whether this ffmpeg writes VP9: the worker's build does, the Mac's LGPL* build does not. */export async function hasVp9Encoder(): Promise<boolean> {diff --git a/site/src/cut/server/frames.test.ts b/site/src/cut/server/frames.test.tsindex c06fd584..551d4a2a 100644--- a/site/src/cut/server/frames.test.ts+++ b/site/src/cut/server/frames.test.ts@@ -4,6 +4,7 @@ import { mkdir, mkdtemp, rm, stat } from "node:fs/promises";import os from "node:os";import path from "node:path";import { fileURLToPath } from "node:url";+import { forgetFfmpegEncodersForTest } from "./exportPipeline";import { makeFreezeFrame } from "./frames";import { mediaDir } from "./projects";@@ -22,6 +23,7 @@ beforeAll(async () => {pathBefore = process.env.PATH;process.env.DONKEY_CUT_DATA_DIR = data;process.env.PATH = `${VENDOR}:${pathBefore}`;+ forgetFfmpegEncodersForTest();await mkdir(mediaDir(PROJECT), { recursive: true });const made = spawnSync("ffmpeg", ["-v", "error", "-f", "lavfi", "-i", "color=c=0x336699:s=64x36", "-frames:v", "1", path.join(mediaDir(PROJECT), "src.png")]);if (made.status !== 0) throw new Error(String(made.stderr));@@ -29,6 +31,7 @@ beforeAll(async () => {afterAll(async () => {process.env.PATH = pathBefore;+ forgetFfmpegEncodersForTest();if (dataBefore === undefined) delete process.env.DONKEY_CUT_DATA_DIR;else process.env.DONKEY_CUT_DATA_DIR = dataBefore;await rm(data, { recursive: true, force: true });