Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): count picture-box saves as edits

David Le32695d26fac9
diff --git a/site/src/clients/chatgpt/server/catalog.test.ts b/site/src/clients/chatgpt/server/catalog.test.ts
index ba4a3ee5..ff5c06da 100644
--- a/site/src/clients/chatgpt/server/catalog.test.ts
+++ b/site/src/clients/chatgpt/server/catalog.test.ts
@@ -9,6 +9,10 @@ test("first-class history tools can queue their editor commands", () => {
expect(READ_COMMANDS.has("redo")).toBe(false);
});
+test("a command that keeps a picture box on the source saves the document", () => {
+ expect(READ_COMMANDS.has("detect_cuts")).toBe(false);
+});
+
test("ChatGPT exposes file metadata as a read command", () => {
expect(READ_COMMANDS.has("get_asset_info")).toBe(true);
expect(COMMANDS.find((t) => t.name === "get_asset_info")?.inputSchema)
diff --git a/site/src/clients/chatgpt/server/catalog.ts b/site/src/clients/chatgpt/server/catalog.ts
index 9f2f3c3f..0f7ca893 100644
--- a/site/src/clients/chatgpt/server/catalog.ts
+++ b/site/src/clients/chatgpt/server/catalog.ts
@@ -58,7 +58,6 @@ export const READ_COMMANDS = new Set([
"watch_video",
"capture_frame",
"compare_to_source",
- "detect_cuts",
"sample_color",
"track_element",
"suggest_font",
diff --git a/site/src/clients/chatgpt/server/guidance.ts b/site/src/clients/chatgpt/server/guidance.ts
index 5f0c5c66..97f72066 100644
--- a/site/src/clients/chatgpt/server/guidance.ts
+++ b/site/src/clients/chatgpt/server/guidance.ts
@@ -41,7 +41,7 @@ Inspect the project for asset ids, transcripts, trims, notes and existing covera
watch_video returns timestamped contact sheets, sceneChanges and coverage. Stamps are source seconds. Continue from coveredTo when truncated. For a request about the whole source, cover it until unwatchedSeconds reaches zero. For long footage, sweep coarsely to learn its structure, then inspect relevant spans closely. A source map or a transcript alone does not establish visual coverage. Use detail read or original to read text and match typography; scan is for structure. Check unreadSeconds and unreadFrom when legible coverage matters.
After every watch, record what you observed through edit_project with note_source: source span, actions, cuts, and exact on-screen words. A further watch can be refused until those observations are recorded. Notes persist on the asset; returned contact sheets belong to this conversation. Read observed, recorded and unnoted to resume from existing evidence.
listen_audio returns playable source audio. Use the transcript for words and listen for delivery, music and timing. Respect coveredTo; continue from unheardFrom until unheardSeconds reaches zero when judging a whole source. Record observations with note_source. A clip_id scopes listening to its source trim; from and to are source seconds.
-Use signal measurements for editing decisions: detect_silence locates quiet spans; detect_beats returns a beat grid and timelineBeats for a clip; measure_level measures loudness, including integratedLufs and truePeakDbtp per clip and for the whole mix, and returns volumeToMatch against a target. read_color_stats measures exposure, tonal range and color before grading. compare_to_source checks the edit against a reference; capture_frame shows the composited picture. detect_cuts, sample_color and track_element measure a source's cuts, colours and moving elements as numbers, inside the picture box find_picture keeps for a recorded or letterboxed source. These read commands belong in inspect_project's commands array.
+Use signal measurements for editing decisions: detect_silence locates quiet spans; detect_beats returns a beat grid and timelineBeats for a clip; measure_level measures loudness, including integratedLufs and truePeakDbtp per clip and for the whole mix, and returns volumeToMatch against a target. read_color_stats measures exposure, tonal range and color before grading. compare_to_source checks the edit against a reference; capture_frame shows the composited picture. detect_cuts, sample_color and track_element measure a source's cuts, colours and moving elements as numbers, inside the picture box find_picture keeps for a recorded or letterboxed source. sample_color and track_element belong in inspect_project's commands array; detect_cuts keeps the box it finds on a source's first read, so it goes through edit_project.
find_filler identifies transcript fillers; find_highlights ranks transcript spans. Their results need the relevant visual or audio check before cutting. Use edit_project for these commands unless list_commands marks them reads. Caption creation is a separate edit requested by the user.
Plan cuts around complete thoughts and measured pauses. Split and delete in batches, read the resulting ids, then place clips to close intended gaps. Leave room around speech boundaries and run refine_speech_cuts with the requested pace. Listen around adjusted edges, widen a clipped word's trim, and check the changed cut visually. Source seconds drive watches, listens and trims; timeline seconds drive placement and splits. Use the clip timing data to map between them.
Report only the coverage you inspected and the changes tool results confirm. render_preview produces a playable saved revision for the user to review.`,
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 8bf6c956..7b3e1343 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -1587,7 +1587,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// A source read for the first time has its picture found and kept, so
// a change that fills the picture counts in full when the picture is a
// small part of a recorded frame.
- if (input.picture === undefined && !asset.picture) {
+ const keepsPicture = input.picture === undefined && !asset.picture;
+ if (keepsPicture) {
await storePicture(asset, 0, asset.duration);
}
const area = pictureArea(useEditor.getState().assets.find((a) => a.id === asset.id) ?? asset, input);
@@ -1623,6 +1624,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
cuts: scan.cuts.map((c) => ({ t: roundT(c.t), frame: c.frame, change: c.change })),
shots,
...pictureReport(area),
+ ...(keepsPicture ? { pictureKept: true as const } : {}),
source: sourceRef(asset),
note:
(scan.cuts.length === 0 ? "No hard cut in this span. " : "") +
diff --git a/site/src/cut/lib/measureTools.test.ts b/site/src/cut/lib/measureTools.test.ts
index 580fffa1..bdff42db 100644
--- a/site/src/cut/lib/measureTools.test.ts
+++ b/site/src/cut/lib/measureTools.test.ts
@@ -115,6 +115,13 @@ test("detect_cuts finds every hard cut to the frame, a five-frame shot included"
expect(decoded).toHaveLength(FRAMES);
});
+test("detect_cuts says when its first read of a source kept the picture box", async () => {
+ const first = (await runAiTool("detect_cuts", { asset_id: "ref", from: 1.05, to: 2 })) as { pictureKept?: true };
+ expect(first.pictureKept).toBe(true);
+ const again = (await runAiTool("detect_cuts", { asset_id: "ref", from: 1.05, to: 2 })) as { pictureKept?: true };
+ expect(again.pictureKept).toBeUndefined();
+});
+
test("detect_cuts scans only the span asked for, and a sound has no cuts", async () => {
const out = (await runAiTool("detect_cuts", { asset_id: "ref", from: 1.05, to: 2 })) as { cuts: { t: number }[] };
expect(out.cuts.map((c) => c.t)).toEqual([1.083]);
diff --git a/site/src/cut/lib/pi/mutationLedger.test.ts b/site/src/cut/lib/pi/mutationLedger.test.ts
index fb3dd966..864d88e6 100644
--- a/site/src/cut/lib/pi/mutationLedger.test.ts
+++ b/site/src/cut/lib/pi/mutationLedger.test.ts
@@ -2,6 +2,17 @@ import { describe, expect, test } from "bun:test";
import { harvestIds, ledgerText, recordCall, type LedgerRecord } from "./mutationLedger";
describe("recordCall", () => {
+ // Both keep a picture box on the source: find_picture always, detect_cuts
+ // on its first read of a source, which its result says.
+ test("a reading that kept a picture box is an edit", () => {
+ const records: LedgerRecord[] = [];
+ recordCall(records, "find_picture", { picture: { x: 0.5, y: 0.5, w: 1, h: 0.5 } }, undefined);
+ recordCall(records, "detect_cuts", { cuts: [], pictureKept: true }, undefined);
+ recordCall(records, "detect_cuts", { cuts: [] }, undefined);
+ recordCall(records, "detect_cuts", undefined, "Could not read the source.");
+ expect(records.map((r) => r.name)).toEqual(["find_picture", "detect_cuts"]);
+ });
+
test("a read leaves no record and a mutation keeps its ids", () => {
const records: LedgerRecord[] = [];
recordCall(records, "get_state", { videoTrack: [] }, undefined);
diff --git a/site/src/cut/lib/pi/mutationLedger.ts b/site/src/cut/lib/pi/mutationLedger.ts
index 64ea841f..5a2f164a 100644
--- a/site/src/cut/lib/pi/mutationLedger.ts
+++ b/site/src/cut/lib/pi/mutationLedger.ts
@@ -13,8 +13,6 @@ const READ_ONLY = new Set([
"request_tools",
"capture_frame",
"compare_to_source",
- "find_picture",
- "detect_cuts",
"sample_color",
"track_element",
"suggest_font",
@@ -44,6 +42,12 @@ const READ_ONLY = new Set([
export const isMutatingTool = (name: string): boolean => !READ_ONLY.has(name);
+// Readings that write only now and then, with how their result says it did:
+// detect_cuts keeps the picture box it finds on its first read of a source.
+const WRITES_WHEN: Record<string, (response: unknown) => boolean> = {
+ detect_cuts: (response) => (response as { pictureKept?: unknown } | undefined)?.pictureKept === true,
+};
+
export interface LedgerRecord {
name: string;
/** Ids the call's result carried, by field name. */
@@ -111,6 +115,8 @@ export function recordCall(
error: string | undefined
): void {
if (!isMutatingTool(name)) return;
+ const wrote = WRITES_WHEN[name];
+ if (wrote && (error || !wrote(response))) return;
if (error) {
records.push({ name, ids: [], error });
return;