Skip to content

Open source CapCut alternative.

TypeScript

fix(site): split a named item, and read a faint cut as a cutout [rebuild]

David Le9cf74cbd4096
diff --git a/site/src/cut/components/Timeline.tools.ts b/site/src/cut/components/Timeline.tools.ts
index dbb75b07..ff7b9f22 100644
--- a/site/src/cut/components/Timeline.tools.ts
+++ b/site/src/cut/components/Timeline.tools.ts
@@ -100,8 +100,11 @@ export const TIMELINE_TOOLS = [
{
name: "split_at",
description:
- `Split selected items and group members at a time, like pressing S. Supported kinds: ${ITEM_KIND_IDS.filter(canSplitItem).join(", ")}. Transitions remain whole. With no selection, split the video on track 0. Omit t for the playhead. Times stay fixed; the result includes the new selected ids.`,
- inputSchema: obj({ t: num("Timeline seconds to cut at (optional)") }),
+ `Split items and their group members at a time, like pressing S: the items \`ids\` names, else the selection, else the video on track 0. Supported kinds: ${ITEM_KIND_IDS.filter(canSplitItem).join(", ")}. Transitions remain whole. Omit t for the playhead. Times stay fixed; the result includes the new selected ids.`,
+ inputSchema: obj({
+ t: num("Timeline seconds to cut at (optional)"),
+ ids: { type: "array", items: { type: "string" }, description: "Ids of the items to split, any track or kind (optional; default the selection)" },
+ }),
},
{
name: "move_clip",
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index d7e41cb3..617186cb 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -2771,11 +2771,25 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
},
split_at: (s, input) => {
+ const ids = Array.isArray(input.ids) ? input.ids.filter((id): id is string => typeof id === "string") : [];
+ const items = ids.map((id) => {
+ const kind = ITEM_KIND_IDS.find((k) => ITEM_KINDS[k].list(s).some((item) => item.id === id));
+ if (!kind) throw new ToolError(`No timeline item ${id}.`);
+ return { kind, id };
+ });
const before = ITEM_KIND_IDS.reduce((n, kind) => n + ITEM_KINDS[kind].list(s).length, 0);
- s.splitAtPlayhead(isNum(input.t) ? input.t : undefined);
+ s.splitAtPlayhead(isNum(input.t) ? input.t : undefined, items);
const after = useEditor.getState();
const made = ITEM_KIND_IDS.reduce((n, kind) => n + ITEM_KINDS[kind].list(after).length, 0) - before;
- if (made === 0) throw new ToolError("Nothing to split at that time.");
+ if (made === 0) {
+ // An item splits only strictly inside its span: on its edge it is cut already.
+ if (items.length) throw new ToolError("Nothing to split: none of those items runs across that time.");
+ if (s.multiSelection.length || s.selection)
+ throw new ToolError("Nothing to split: the selection doesn't run across that time. Name what to split with ids.");
+ throw new ToolError(
+ "Nothing to split: with nothing selected only the track-0 video splits, and none of it runs across that time (a clip edge is a cut already). Name an overlay clip, title or other item with ids."
+ );
+ }
return { split: made, selection: after.multiSelection, ...tracksAfter() };
},
diff --git a/site/src/cut/lib/splitEdit.test.ts b/site/src/cut/lib/splitEdit.test.ts
index 23a0f101..30f57572 100644
--- a/site/src/cut/lib/splitEdit.test.ts
+++ b/site/src/cut/lib/splitEdit.test.ts
@@ -164,3 +164,20 @@ describe("a recording bound to the video", () => {
await expect(runAiTool("sync_audio", { clipId: "a", audio_asset_id: "cam" })).rejects.toThrow();
});
});
+
+describe("split_at", () => {
+ test("ids names what to split, an overlay clip included, with nothing selected", async () => {
+ useEditor.setState({ clips: [...useEditor.getState().clips, clip("top", 2, 30, 34, { track: 1 })] });
+ const result = await runAiTool("split_at", { t: 3, ids: ["top"] });
+ const tops = useEditor.getState().clips.filter((c) => c.track === 1).sort((x, y) => x.start - y.start);
+ expect(result).toMatchObject({ split: 1 });
+ expect(tops.map((c) => [c.start, c.in, c.out])).toEqual([[2, 30, 31], [3, 31, 34]]);
+ expect(useEditor.getState().clips.filter((c) => c.track === 0)).toHaveLength(2);
+ });
+
+ test("a time on a clip edge says how to reach another item", async () => {
+ useEditor.setState({ clips: [...useEditor.getState().clips, clip("top", 2, 30, 34, { track: 1 })] });
+ await expect(runAiTool("split_at", { t: 4 })).rejects.toThrow("ids");
+ await expect(runAiTool("split_at", { t: 7, ids: ["top"] })).rejects.toThrow("runs across");
+ });
+});
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 67718be6..77425d99 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -861,8 +861,9 @@ export interface EditorState {
/** "Detach Audio": lift the selected clip's sound onto the
* soundtrack track (and mute the clip) so it can be cut independently. */
detachAudio: () => void;
- /** Split at the given time, or the playhead when omitted. */
- splitAtPlayhead: (at?: number) => void;
+ /** Split at the given time, or the playhead when omitted: the given items,
+ * else the selection, else the track-0 video there. */
+ splitAtPlayhead: (at?: number, items?: NonNullable<Selection>[]) => void;
setPublish: (patch: Partial<{ caption: string; tags: string; soundTitle: string; handle: string }>) => void;
setNotes: (patch: Partial<{ text: string; publishedAt: string; links: string[] }>) => void;
/** Kick off (and poll) an on-device transcription of the current cut. */
@@ -4052,11 +4053,11 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
}));
},
- splitAtPlayhead: (at) => {
+ splitAtPlayhead: (at, items) => {
const s = get();
const t = at ?? playheadAt();
if (s.readOnly || !Number.isFinite(t)) return;
- const selected = s.multiSelection.length ? s.multiSelection : s.selection ? [s.selection] : s.clips
+ const selected = items?.length ? items : s.multiSelection.length ? s.multiSelection : s.selection ? [s.selection] : s.clips
.filter((c) => c.track === 0 && t > c.start && t < c.start + clipLen(c))
.map((c) => ({ kind: "clip" as const, id: c.id }));
const result = splitTimelineItems(s, expandTimelineGroups(s, selected), t, uid);
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index ed75ce1d..cfb3e0ee 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -354,7 +354,7 @@ Someone points at a video — a file they imported, a link you brought in, a cli
**Clipping a long source.** A talk, an interview or a stream is mostly material that only makes sense in place. find_highlights ranks its stretches from the transcript, so an hour is read for its handful of moments without watching all of it; watch_video the spans it returns before committing, since the ranking never saw the picture. add_clip \`spans\` lays a pick down — several stretches of one source as one run of clips, the material between them left out — and a two-span result is one clip whose halves were never next to each other.
**Watch the whole thing first.** Every decision below comes off the record, so the record has to be complete: watch from 0 and keep going until \`unwatchedSeconds\` is 0, writing each pass into note_source before the next one. Anything over a couple of minutes gets a coarse pass end to end first (\`interval_seconds\` 15-30) to learn its shape, then a dense pass (0.5) over the stretches that carry text or cut fast. Read every card word for word — its text, its face, where it sits in the frame, when it comes and goes — because a turn later the sheets are gone and the notes are all you have. Reading type is what \`detail\` is for: the dense pass over a stretch carrying words runs at \`detail\` read, and a face or treatment you are about to name runs at original, where the frame arrives at the source's own resolution. When the cards ARE the narration — a caption track of what the speaker is saying — the transcript already holds every word, and the close passes are for how they are set: face, weight, size, colour, placement, how many words ride a card, how it enters and leaves. Read a few of those at original and take the rest from the transcript.
-**Measure it.** A contact sheet tells you what happens; the numbers come from the source's pixels. A reference recorded off a screen — a phone filming an editing app, a letterboxed film — has its picture inside a bigger frame: find_picture finds that box and keeps it on the source (detect_cuts does on its first read), every measuring tool and compare_to_source then reads inside it, and their x, y, w, h are fractions of it, so they land on the cut's frame as they are; set_aspect to its \`aspect\`. detect_cuts reads every frame: each cut on its frame, the \`shots\` between them, and the \`changes\` inside a shot — a word landing, a word swapping fill for outline, a shape popping in. Run it over the whole source before the read passes, which then land a frame in every shot and change, and time each title from them. Its times are to the millisecond, and every tool lands such a time on the frame it names: write them back as read. sample_color reads the hex of any region at any second. track_element gives an element's box frame by frame and the ease of each move; with match ink it gives a title's ink box. track_camera gives a shot's camera move frame by frame — zoom, shift, turn — and says when the picture under it is one frozen frame. Run it over every shot before you describe how it moves: a beat zoom on a frozen frame, a shake or a slow push reads on a contact sheet as a held shot with exposure flicker, and detect_cuts lists its pulses as whole-frame \`changes\` — a run of those inside one shot is a camera move until track_camera says otherwise.
+**Measure it.** A contact sheet tells you what happens; the numbers come from the source's pixels. A reference recorded off a screen — a phone filming an editing app, a letterboxed film — has its picture inside a bigger frame: find_picture finds that box and keeps it on the source (detect_cuts does on its first read), every measuring tool and compare_to_source then reads inside it, and their x, y, w, h are fractions of it, so they land on the cut's frame as they are; set_aspect to its \`aspect\`. detect_cuts reads every frame: each cut on its frame, the \`shots\` between them, and the \`changes\` inside a shot — a word landing, a word swapping fill for outline, a shape popping in. Run it over the whole source before the read passes, which then land a frame in every shot and change, and time each title from them. Its times are to the millisecond, and every tool lands such a time on the frame it names: write them back as read. sample_color reads the hex of any region at any second. track_element gives an element's box frame by frame and the ease of each move; with match ink it gives a title's ink box. track_camera gives a shot's camera move frame by frame — zoom, shift, turn — and says when the picture under it is one frozen frame. Run it over every shot before you describe how it moves: a beat zoom on a frozen frame, a shake or a slow push reads on a contact sheet as a held shot with exposure flicker, and detect_cuts lists its pulses as whole-frame \`changes\` — a run of those inside one shot is a camera move until track_camera says otherwise. A cut whose \`change\` sits far below a new shot's, or a few frames that open just before a real cut, is rarely a shot of its own: it is usually the next shot's subject laid over this one, a cutout flashed in. Look at those frames one at a time, and build every one you find; it is part of the edit's look, not a glitch to skip.
**Lay down the sound, then block out the cuts.** Cuts timed to a reference are timed to its sound: detach_audio with the reference's \`assetId\` puts its whole track on the soundtrack and none of its picture. add_clip with \`blocks\` lays the skeleton from \`shots\` in one call — one block per shot on track 0, each its shot's \`from\` and \`to\` as read so every cut lands on its frame, flash cuts included, \`color\` what sample_color reads behind it and \`label\` naming what the person drops there. The label rides the timeline chip; the frame shows the colour alone. Footage dropped on a block, or replace_item, takes its place and its length, so everything built on top keeps its timing. When the ask is the reference's own picture playing, add_clip \`spans\` lays its shots down instead, one span per shot at those same times.
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index f081a387..77338022 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -464,6 +464,11 @@ function ffmpegEncoders(): Promise<Set<string>> {
}));
}
+/** Drops the encoder list, for a test that points PATH at another ffmpeg. */
+export function forgetFfmpegEncodersForTest(): void {
+ encodersCache = null;
+}
+
/** Whether this ffmpeg writes VP9: the worker's build does, the Mac's LGPL
* build does not. */
export async function hasVp9Encoder(): Promise<boolean> {
diff --git a/site/src/cut/server/frames.test.ts b/site/src/cut/server/frames.test.ts
index c06fd584..551d4a2a 100644
--- a/site/src/cut/server/frames.test.ts
+++ b/site/src/cut/server/frames.test.ts
@@ -4,6 +4,7 @@ import { mkdir, mkdtemp, rm, stat } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
+import { forgetFfmpegEncodersForTest } from "./exportPipeline";
import { makeFreezeFrame } from "./frames";
import { mediaDir } from "./projects";
@@ -22,6 +23,7 @@ beforeAll(async () => {
pathBefore = process.env.PATH;
process.env.DONKEY_CUT_DATA_DIR = data;
process.env.PATH = `${VENDOR}:${pathBefore}`;
+ forgetFfmpegEncodersForTest();
await mkdir(mediaDir(PROJECT), { recursive: true });
const made = spawnSync("ffmpeg", ["-v", "error", "-f", "lavfi", "-i", "color=c=0x336699:s=64x36", "-frames:v", "1", path.join(mediaDir(PROJECT), "src.png")]);
if (made.status !== 0) throw new Error(String(made.stderr));
@@ -29,6 +31,7 @@ beforeAll(async () => {
afterAll(async () => {
process.env.PATH = pathBefore;
+ forgetFfmpegEncodersForTest();
if (dataBefore === undefined) delete process.env.DONKEY_CUT_DATA_DIR;
else process.env.DONKEY_CUT_DATA_DIR = dataBefore;
await rm(data, { recursive: true, force: true });