Open source CapCut alternative.
TypeScript
feat(cut): verify a grade through read_color_stats graded and capture_frame [rebuild]
David Lea9b41831a787
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.tsindex 6340aba7..fe8c321b 100644--- a/site/src/cut/components/Inspector.tools.ts+++ b/site/src/cut/components/Inspector.tools.ts@@ -477,10 +477,11 @@ export const INSPECTOR_TOOLS = [{name: "read_color_stats",description:- "Read color statistics off a frame — per-channel and luma quantiles (0..255), channel means, mean saturation, and warmth (red/blue midtone ratio; >1 warm, <1 cool). Pass clipId for a video clip's current frame as its base rendering (through the source conversion, before any LUT or grade, so the numbers describe the footage itself; seek into the clip first if it errors) or assetId for an image (a chat attachment or Media import). Read stats before grading — never grade blind — and read them again after to verify the move.",+ "Read color statistics off a frame — per-channel and luma quantiles (0..255), channel means, mean saturation, and warmth (red/blue midtone ratio; >1 warm, <1 cool). Pass clipId for a video clip's current frame (seek into the clip first if it errors) or assetId for an image (a chat attachment or Media import). A clip reads as its base rendering — through the source conversion, before any LUT or grade, so the numbers describe the footage itself; graded:true reads it through the clip's LUT, preset and adjustments, which is how a move is verified. Read the base before grading — never grade blind — and the graded frame after.",inputSchema: obj({- clipId: str("Video clip id (its current frame's base rendering)"),+ clipId: str("Video clip id (its current frame)"),assetId: str("Image asset id (chat attachment or import)"),+ graded: bool("Read the clip as graded (LUT, preset and adjustments applied) to verify a move; default reads the ungraded base"),}),},{diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 8b30cc47..9c65c16b 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -28,7 +28,9 @@ import { textSpots } from "./textPlace";import {OUTPUT_SPACES,ALL_EFFECT_IDS,+ applyLutToImageData,autoGradeFromImageData,+ buildClipLut,colorStatsFromImageData,GRADE_BASIC_FIELDS,GRADE_DETAIL_FIELDS,@@ -177,7 +179,7 @@ import { clampLayersToAssets, mediaTypeFits, templateFromDoc } from "./projectTeimport { isSoundPresetTemplate, listSoundPresets, saveSoundPreset } from "./soundPresets";import { isGradePresetTemplate, listSavedGrades, saveGradePreset } from "./gradePresets";import { isStylePresetTemplate } from "./stylePresets";-import { lutLabel } from "./linkedLibrary";+import { loadLibraryLut, lutIdOf, lutLabel } from "./linkedLibrary";import { sampleClipBaseFrameData, sourceProfileOf, toBaseRendering } from "./baseFrame";import { applyOverlayPatchSettled, clipLen, track0Clips, laneGapAt, getClipSpans, overlayLaneOrder, overlayLayers, parkedTransitions, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";import { playheadAt } from "./playhead";@@ -372,10 +374,11 @@ const GRADE_KEYS = [/** Color stats for a grading reference: a clip's current frame as the base* rendering (through the source conversion, before any grade — what Auto- * reads) or an image asset's pixels. */+ * reads), or `graded` through the clip's LUT, preset and adjustments too, or+ * an image asset's pixels. */async function colorStatsForRef(s: ReturnType<typeof useEditor.getState>,- ref: { clipId?: string; assetId?: string }+ ref: { clipId?: string; assetId?: string; graded?: boolean }): Promise<ColorStats> {if (ref.clipId) {const clip = requireItem(s.clips, ref.clipId, "video clip");@@ -384,6 +387,14 @@ async function colorStatsForRef(throw new ToolError("No decoded frame for that clip yet — seek into it so it is on screen, then retry.");+ if (ref.graded && clip.grade) {+ // The same recipe the Color panel's tiles draw with: the base rendering+ // through the clip's LUT, preset and adjustments.+ const lutId = lutIdOf(clip.grade);+ const userLut = lutId ? await loadLibraryLut(lutId) : undefined;+ const lut = buildClipLut({ profile: "rec709", grade: clip.grade, output: "sdr", size: 33 }, userLut);+ if (lut) applyLutToImageData(data, lut);+ }const stats = colorStatsFromImageData(data);if (!stats) throw new ToolError("Could not read pixels off that clip's frame.");return stats;@@ -4719,6 +4730,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {const stats = await colorStatsForRef(s, {clipId: typeof input.clipId === "string" ? input.clipId : undefined,assetId: typeof input.assetId === "string" ? input.assetId : undefined,+ graded: input.graded === true,});return statsReport(stats);},@@ -4838,7 +4850,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {return {id: next.id,grade: next.grade ?? "already matched (no change needed)",- note: "Verify with read_color_stats on the clip and refine with the grading tools.",+ note: "Verify with read_color_stats graded:true on the clip and capture_frame, then refine with the grading tools.",};},diff --git a/site/src/cut/lib/colorTools.test.ts b/site/src/cut/lib/colorTools.test.tsindex 391e4703..4c1cdae1 100644--- a/site/src/cut/lib/colorTools.test.ts+++ b/site/src/cut/lib/colorTools.test.ts@@ -44,6 +44,12 @@ await stubModule<typeof import("./gradePresets")>("./gradePresets", import.meta.},});+// A flat mid-grey frame stands in for a decoded clip frame, so the stats+// tool reads known pixels.+await stubModule<typeof import("./baseFrame")>("./baseFrame", import.meta.url, {+ sampleClipBaseFrameData: (_clipId, w = 96, h = 54) => new Uint8ClampedArray(w * h * 4).fill(128),+});+const { runAiTool } = await import("./aiTools");const { useEditor } = await import("./store");const { syncLinkedLibrary } = await import("./linkedLibrary");@@ -174,3 +180,13 @@ test("save_color_grade keeps the grade under a name and apply_saved_grade puts iexpect(gradeOf(c)).toEqual({ temperature: 15 });await expect(runAiTool("apply_saved_grade", { clipId: b, preset_id: "cold" })).rejects.toThrow(/No saved grade/);});++test("read_color_stats reads the base frame, and graded:true reads it through the clip's grade", async () => {+ const [, id] = clipIds();+ await runAiTool("set_color_grade", { clipId: id, temperature: 40 });+ type Stats = { warmth: number };+ const base = (await runAiTool("read_color_stats", { clipId: id })) as Stats;+ const graded = (await runAiTool("read_color_stats", { clipId: id, graded: true })) as Stats;+ expect(base.warmth).toBe(1);+ expect(graded.warmth).toBeGreaterThan(1.05);+});diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex b8447ff6..6c23b8cb 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -265,7 +265,7 @@ Times are in seconds on the shared timeline. The playhead is currentTime; a skim"color-grading": `# Color gradingGrade like a colorist: read the image, correct it, then style it — in that order.-**Read before you write.** Call read_color_stats on the clip before grading (never grade blind) and again after a move to verify it landed. The stats describe the base rendering — the footage through its source conversion, before any LUT or grade. The quantiles tell you where the tones sit: p2 well above 0 = lifted/milky blacks, p98 well below 255 = no real whites (flat), a p50 far from ~118 = exposure off. warmth > 1 reads warm, < 1 cool; meanSat under ~0.15 reads muted, over ~0.35 vivid. The clip's current color state is in editor_state (colorPreset, grade, lut, curves/wheels/hslBands); each asset's sourceColor says what its code values mean.+**Read before you write, look after.** Call read_color_stats on the clip before grading (never grade blind): by default it reads the base rendering — the footage through its source conversion, before any LUT or grade. After a move, read it again with graded:true to confirm the numbers went where you meant, and capture_frame to look at the result — faces, whites, and the frame as a whole — before calling the grade done. The quantiles tell you where the tones sit: p2 well above 0 = lifted/milky blacks, p98 well below 255 = no real whites (flat), a p50 far from ~118 = exposure off. warmth > 1 reads warm, < 1 cool; meanSat under ~0.15 reads muted, over ~0.35 vivid. The clip's current color state is in editor_state (colorPreset, grade, lut, curves/wheels/hslBands); each asset's sourceColor says what its code values mean.**Source color first.** Log and HDR footage (${SOURCE_PROFILES.filter((p) => p.id !== "rec709" && p.id !== "srgb").map((p) => `${p.id} = ${p.label}`).join(", ")}) is detected from the file and converted to Rec.709 display values before anything else, so it grades like ordinary footage. When a clip reads flat and grey and its sourceColor is rec709, the header did not say — set_source_color with the right profile fixes the whole clip in one move. A camera maker's Log-to-Rec.709 LUT is the other route: set_source_color rec709 on that asset, then set_color_lut, so the picture is converted once.