Skip to content

Open source CapCut alternative.

TypeScript

feat(site): color grading chat tools

David Le5ca73225f954
diff --git a/site/scripts/lib/cut-eval/cases.ts b/site/scripts/lib/cut-eval/cases.ts
index d034359a..87c413de 100644
--- a/site/scripts/lib/cut-eval/cases.ts
+++ b/site/scripts/lib/cut-eval/cases.ts
@@ -9,6 +9,7 @@
* multi-tool — composed edits over several calls
*/
+import { gradePresetsInCategory } from "@donkeycut/effects-kit";
import {
AUDIO_STATE,
CLIP_REFS,
@@ -760,5 +761,117 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
state: STYLED_STATE,
maxToolCalls: 2,
},
+ {
+ // Grading: a mood ask lands as one set_color_preset with a pick from
+ // the matching catalog category — a hand-rolled slider stack fails.
+ name: "cinematic-ask-picks-preset",
+ bucket: "single-tool",
+ input: () => [userTurn("give my video a cinematic color grade")],
+ reply: /cinematic|grade|preset|teal|look/i,
+ requiredTools: ["set_color_preset"],
+ maxToolCalls: 4,
+ simulate: () => (name, args) => {
+ if (name !== "set_color_preset") return undefined;
+ const pick = String(args.preset);
+ const ids = gradePresetsInCategory("cinematic").map((p) => p.id);
+ if (!ids.includes(pick))
+ throw new Error(`preset ${pick} — expected a cinematic pick (${ids.join(", ")})`);
+ return { id: args.clipId ?? "c1", preset: pick, amount: Number(args.amount ?? 1) };
+ },
+ },
+ {
+ // Semantic grading: "warmer" is temperature above zero and "crush the
+ // blacks" pushes shadows down (or deepens the low end of the master
+ // curve) — the interceptor fails any move in the wrong direction.
+ name: "warmer-crushed-blacks-in-range",
+ bucket: "single-tool",
+ input: () => [userTurn("make the beach clip warmer and crush the blacks a little")],
+ reply: /warm|black|shadow|grade/i,
+ requiredTools: ["set_color_grade"],
+ maxToolCalls: 5,
+ simulate: () => (name, args) => {
+ if (name === "set_color_grade") {
+ if (typeof args.temperature === "number" && args.temperature <= 0)
+ throw new Error(`temperature ${args.temperature} — warmer means above 0`);
+ if (typeof args.shadows === "number" && args.shadows >= 0)
+ throw new Error(`shadows ${args.shadows} — crushing blacks means below 0`);
+ return { id: args.clipId ?? "c1", grade: { temperature: args.temperature ?? 0, shadows: args.shadows ?? 0 } };
+ }
+ if (name === "set_color_curves") {
+ if (typeof args.fade === "number" && args.fade > 0)
+ throw new Error(`fade ${args.fade} — fade lifts the blacks, the ask crushes them`);
+ const master = Array.isArray(args.master) ? (args.master as number[][]) : [];
+ for (const [x, y] of master)
+ if (x <= 80 && y > x)
+ throw new Error(`master point [${x}, ${y}] lifts the low end — crushing pulls it down`);
+ return { id: args.clipId ?? "c1", curves: { master } };
+ }
+ return undefined;
+ },
+ },
+ {
+ // Per-hue isolation: "just the sky" is one hue band, so the move is
+ // set_color_hsl on aqua or blue — a whole-frame saturation push leaks
+ // onto everything else and fails.
+ name: "sky-bluer-single-band",
+ bucket: "single-tool",
+ input: () => [userTurn("make the sky bluer without changing anything else")],
+ reply: /sky|blue/i,
+ requiredTools: ["set_color_hsl"],
+ maxToolCalls: 4,
+ simulate: () => (name, args) => {
+ if (name === "set_color_grade" && (typeof args.saturation === "number" || typeof args.hue === "number"))
+ throw new Error("whole-frame color move — the ask isolates one hue, use set_color_hsl");
+ if (name !== "set_color_hsl") return undefined;
+ const band = String(args.band);
+ if (band !== "aqua" && band !== "blue")
+ throw new Error(`band ${band} — the sky lives in aqua/blue`);
+ const sat = Number(args.sat ?? 0);
+ const hue = Number(args.hue ?? 0);
+ if (sat === 0 && hue === 0)
+ throw new Error("all-zero band clears it — bluer needs a sat or hue move");
+ if (sat < 0) throw new Error(`sat ${sat} — bluer means saturation up`);
+ return { id: args.clipId ?? "c1", band, hue, sat, luma: Number(args.luma ?? 0) };
+ },
+ },
+ {
+ // Reference matching: a shared look-to-copy routes to the computed
+ // match on the attached photo — an eyeballed slider stack fails. The
+ // stats stub keeps a read-before/verify-after pass cheap and grounded.
+ name: "reference-photo-matches-grade",
+ bucket: "single-tool",
+ input: () => [
+ userTurn("match my video's colors to this photo", { attachRefs: [PHOTO_REFS[0]] }),
+ ],
+ reply: /match|color|grade|photo/i,
+ requiredTools: ["match_color_grade"],
+ maxToolCalls: 6,
+ simulate: () => (name, args) => {
+ if (name !== "match_color_grade") return undefined;
+ if (args.ref_asset_id !== PHOTO_REFS[0].id)
+ throw new Error(`ref ${args.ref_asset_id ?? args.ref_clip_id} — the attached photo is asset ${PHOTO_REFS[0].id}`);
+ if (args.clipId !== "c1")
+ throw new Error(`clip ${args.clipId} — the video on the timeline is c1`);
+ return {
+ id: "c1",
+ grade: { curves: { r: [[0, 0], [128, 138], [255, 255]], b: [[0, 0], [128, 120], [255, 255]] }, saturation: 6 },
+ note: "Verify with read_color_stats on the clip and refine with the grading tools.",
+ };
+ },
+ stubs: {
+ read_color_stats: {
+ quantiles: "values 0..255 at p2, p10, p25, p50, p75, p90, p98",
+ r: [8, 24, 62, 118, 172, 214, 244],
+ g: [7, 22, 58, 112, 166, 208, 240],
+ b: [10, 28, 70, 130, 182, 220, 248],
+ luma: [8, 23, 61, 116, 170, 211, 242],
+ meanR: 121,
+ meanG: 114,
+ meanB: 133,
+ meanSat: 0.24,
+ warmth: 0.91,
+ },
+ },
+ },
];
}
diff --git a/site/scripts/lib/cut-eval/fixtures.ts b/site/scripts/lib/cut-eval/fixtures.ts
index 1113c46a..a713f99a 100644
--- a/site/scripts/lib/cut-eval/fixtures.ts
+++ b/site/scripts/lib/cut-eval/fixtures.ts
@@ -23,6 +23,7 @@ export const SAFE_TOOLS = new Set([
"watch_video",
"detect_silence",
"listen_audio",
+ "read_color_stats",
"library_list",
"stock_search",
"list_voices",
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index 751d0d91..65a10c10 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -6,8 +6,52 @@
* model's toolset and `aiTools.ts` keys its handlers on `InspectorToolName`.
*/
+import {
+ GRADE_BASIC_FIELDS,
+ GRADE_HUE_MAX,
+ GRADE_MAX,
+ GRADE_PRESET_IDS,
+ gradePresetCatalogText,
+ HSL_BANDS,
+} from "@donkeycut/effects-kit";
import { bool, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
+/** Per-field slider hints; ranges interpolate the exported constants so the
+ * schema can never drift from the model the renderer clamps to. */
+const GRADE_FIELD_HINTS: Record<string, string> = {
+ exposure: "(±1 stop)",
+ contrast: "",
+ highlights: "trim (-) or boost (+) the brights",
+ shadows: "lift (+) or crush (-) the darks",
+ temperature: "positive = warmer",
+ tint: "negative = green, positive = magenta",
+ saturation: `(-${GRADE_MAX} = grayscale)`,
+ vibrance: "saturation weighted toward the muted colors",
+};
+
+const gradeFieldProps = Object.fromEntries(
+ GRADE_BASIC_FIELDS.map((f) => [
+ f.key,
+ num(
+ `-${GRADE_MAX}..${GRADE_MAX}, 0 neutral${GRADE_FIELD_HINTS[f.key] ? ` — ${GRADE_FIELD_HINTS[f.key]}` : ""}`
+ ),
+ ])
+);
+
+/** [input, output] control points, 0..255, for one curve channel. */
+const curvePoints = (channel: string) => ({
+ type: "array" as const,
+ description: `${channel} curve control points as [input, output] pairs (0..255, sorted by input); replaces that channel; an empty list clears it`,
+ items: { type: "array" as const, items: { type: "number" as const } },
+});
+
+const wheelArg = (zone: string) =>
+ obj({
+ dx: num(`Chroma x -${GRADE_MAX}..${GRADE_MAX} (the puck's position; angle = hue, distance = strength)`),
+ dy: num(`Chroma y -${GRADE_MAX}..${GRADE_MAX}`),
+ luma: num(`${zone} brightness trim -${GRADE_MAX}..${GRADE_MAX}`),
+ });
+
export const INSPECTOR_TOOLS = [
{
name: "update_overlay",
@@ -173,17 +217,92 @@ export const INSPECTOR_TOOLS = [
{
name: "set_color_grade",
description:
- "Color-grade a video clip (any track; stills too). Fields patch the clip's current grade: only the ones you pass change, and every value 0 is neutral. reset:true clears the whole grade first; auto:true fits a starting grade from the clip's decoded frame (auto-tone: exposure to middle gray, contrast stretch, gray-world white balance — needs the clip's frame decoded, so seek into it first if this errors), and explicit fields then override it. Preview, timeline thumbnails, and export all render the same result.",
+ "Color-grade a video clip's basic sliders (any track; stills too). Fields patch the clip's current grade: only the ones you pass change, and every value 0 is neutral. Manual adjustments layer OVER the clip's color preset (set_color_preset), never replacing it. reset:true clears the manual grade first (the preset stays); auto:true fits a starting grade from the clip's decoded frame (auto-tone: exposure to middle gray, contrast stretch, gray-world white balance — needs the clip's frame decoded, so seek into it first if this errors), and explicit fields then override it. Preview, timeline thumbnails, and export all render the same result. Read read_color_stats before grading so the numbers ground the move.",
inputSchema: obj({
clipId: str("Video clip id"),
- brightness: num("-50..50, 0 neutral"),
- contrast: num("-50..50, 0 neutral"),
- saturation: num("-50..50 (-50 = grayscale), 0 neutral"),
- exposure: num("-50..50 (±1 stop), 0 neutral"),
- temperature: num("-50..50, positive = warmer, 0 neutral"),
- hue: num("Hue rotation in degrees, -180..180"),
+ ...gradeFieldProps,
+ brightness: num(`-${GRADE_MAX}..${GRADE_MAX}, 0 neutral (plain gain; prefer exposure)`),
+ hue: num(`Whole-frame hue rotation in degrees, -${GRADE_HUE_MAX}..${GRADE_HUE_MAX} (for one hue only, use set_color_hsl)`),
auto: bool("Fit a starting grade from the clip's current frame"),
- reset: bool("Clear the existing grade before applying fields"),
+ reset: bool("Clear the existing manual grade before applying fields (keeps the preset)"),
+ }, ["clipId"]),
+ },
+ {
+ name: "set_color_preset",
+ description:
+ `Apply a named color preset to a video clip, layered under its manual grade. Presets by category — ${gradePresetCatalogText()}. amount 0..1 scales the preset toward neutral (default 1); protect_skin keeps its color shifts off skin tones. Pass preset "none" to clear. Preview, tiles, and export all render the same result.`,
+ inputSchema: obj({
+ clipId: str("Video clip id"),
+ preset: {
+ type: "string",
+ enum: [...GRADE_PRESET_IDS, "none"],
+ description: 'Preset id, or "none" to clear',
+ },
+ amount: num("Intensity 0..1 (default 1)"),
+ protect_skin: bool("Keep the preset's color shifts off skin tones"),
+ }, ["clipId", "preset"]),
+ },
+ {
+ name: "set_color_curves",
+ description:
+ `Set a video clip's tone curves: per-channel [input, output] control points (0..255) through a monotone spline — master shapes all three channels, red/green/blue shape one each. A channel you pass replaces that channel; an empty list clears it; others keep their curve. The semantic knobs write the master curve for you: curve_contrast (-${GRADE_MAX}..${GRADE_MAX}) is an s-curve around mid-gray, fade (0..${GRADE_MAX}) lifts the blacks for a faded-film look; passing either replaces the master curve. reset:true clears every curve first.`,
+ inputSchema: obj({
+ clipId: str("Video clip id"),
+ master: curvePoints("Master"),
+ red: curvePoints("Red"),
+ green: curvePoints("Green"),
+ blue: curvePoints("Blue"),
+ curve_contrast: num(`S-curve contrast -${GRADE_MAX}..${GRADE_MAX} (compiled into master points)`),
+ fade: num(`Lifted blacks 0..${GRADE_MAX} (compiled into master points)`),
+ reset: bool("Clear all curves before applying"),
+ }, ["clipId"]),
+ },
+ {
+ name: "set_color_wheels",
+ description:
+ `Set a video clip's color wheels — shadows, midtones, highlights — for split-toning: each wheel pushes its tonal range toward a hue (dx/dy is the wheel puck: angle = hue, 0° = red, 120° = green, 240° = blue; distance = strength) and trims its brightness (luma). A wheel you pass is replaced whole; {dx:0,dy:0,luma:0} clears it. reset:true clears all three.`,
+ inputSchema: obj({
+ clipId: str("Video clip id"),
+ shadows: wheelArg("Shadow"),
+ midtones: wheelArg("Midtone"),
+ highlights: wheelArg("Highlight"),
+ reset: bool("Clear all wheels before applying"),
+ }, ["clipId"]),
+ },
+ {
+ name: "set_color_hsl",
+ description:
+ `Adjust one hue band of a video clip — shift its hue (±${GRADE_MAX} ≈ ±30°), scale its saturation, and lift or darken its luminance — leaving every other color alone ("just the sky bluer", "mute the greens"). Bands: ${HSL_BANDS.map((b) => b.id).join(", ")}. The band's three values are replaced whole (omitted values 0); all-zero clears the band. reset_all clears every band first.`,
+ inputSchema: obj({
+ clipId: str("Video clip id"),
+ band: {
+ type: "string",
+ enum: HSL_BANDS.map((b) => b.id),
+ description: "Hue band to adjust",
+ },
+ hue: num(`Hue shift -${GRADE_MAX}..${GRADE_MAX} (≈ ±30°)`),
+ sat: num(`Saturation -${GRADE_MAX}..${GRADE_MAX}`),
+ luma: num(`Luminance -${GRADE_MAX}..${GRADE_MAX}`),
+ reset_all: bool("Clear every hue band first"),
+ }, ["clipId"]),
+ },
+ {
+ name: "read_color_stats",
+ description:
+ "Read color statistics off a frame — per-channel and luma quantiles (0..255), channel means, mean saturation, and warmth (red/blue midtone ratio; >1 warm, <1 cool). Pass clipId for a video clip's current frame (pre-grade, so the numbers describe the footage itself; seek into the clip first if it errors) or assetId for an image (a chat attachment or Media import). Read stats before grading — never grade blind — and read them again after to verify the move.",
+ inputSchema: obj({
+ clipId: str("Video clip id (its current decoded frame)"),
+ assetId: str("Image asset id (chat attachment or import)"),
+ }),
+ },
+ {
+ name: "match_color_grade",
+ description:
+ "Match a video clip's color to a reference, computed from the pixels: per-channel quantile mapping compiled into tone curves plus a saturation delta. The computed curves carry the whole look, so the match replaces the clip's entire grade, preset included. The reference is ref_asset_id (an attached or imported image) or ref_clip_id (another clip's current frame). Use when the user shares a look to copy or asks to match two shots; refine afterward with the other grading tools, checking read_color_stats.",
+ inputSchema: obj({
+ clipId: str("Video clip to grade"),
+ ref_asset_id: str("Reference image asset id"),
+ ref_clip_id: str("Reference video clip id (its current decoded frame)"),
}, ["clipId"]),
},
] as const satisfies readonly AiToolDef[];
diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.ts
index a2654c2b..b4f117fa 100644
--- a/site/src/cut/lib/aiContext.ts
+++ b/site/src/cut/lib/aiContext.ts
@@ -31,15 +31,39 @@ function transitionToNext(sp: ClipSpan, index: number, spans: ClipSpan[]) {
return { style, seconds: r(sp.transitionOut) };
}
-/** The clip's own effects — entrance/exit animations and filter look — in
- * the shape set_animation/set_look take. */
+/** The clip's own effects — entrance/exit animations, box styling, and its
+ * color state in the shape the grading tools take: the preset ref plus a
+ * compact summary of the manual grade (nonzero sliders verbatim; curves and
+ * wheels as presence flags; the touched hue bands by name). */
function clipEffects(clip: VideoClip) {
+ const grade = clip.grade;
+ const sliders: Record<string, number> = {};
+ for (const [k, v] of Object.entries(grade ?? {})) {
+ if (typeof v === "number" && v !== 0) sliders[k] = v;
+ }
+ const color = grade
+ ? {
+ ...(grade.preset
+ ? {
+ colorPreset: {
+ id: grade.preset.id,
+ amount: r(grade.preset.amount ?? 1),
+ ...(grade.preset.skin ? { protectSkin: true } : {}),
+ },
+ }
+ : {}),
+ ...(Object.keys(sliders).length ? { grade: sliders } : {}),
+ ...(grade.curves ? { curves: Object.keys(grade.curves) } : {}),
+ ...(grade.wheels ? { wheels: Object.keys(grade.wheels) } : {}),
+ ...(grade.hsl ? { hslBands: Object.keys(grade.hsl) } : {}),
+ }
+ : {};
return {
...(clip.animIn ? { animIn: { style: clip.animIn.style, seconds: r(clip.animIn.seconds) } } : {}),
...(clip.animOut
? { animOut: { style: clip.animOut.style, seconds: r(clip.animOut.seconds) } }
: {}),
- ...(clip.look ? { look: clip.look, lookAmount: r(clip.lookAmount ?? 1) } : {}),
+ ...color,
...(clip.boxStyle ? { boxStyle: clip.boxStyle } : {}),
};
}
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 4f63cf73..4cfe89f2 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -2,9 +2,20 @@
import {
autoGradeFromImageData,
+ colorStatsFromImageData,
EFFECT_IDS,
+ GRADE_BASIC_FIELDS,
+ GRADE_PRESETS,
+ HSL_BANDS,
+ matchGrade,
normalizeGrade,
+ semanticMasterCurve,
+ type ColorStats,
+ type CurvePoint,
type EffectId,
+ type GradeCurves,
+ type HslBand,
+ type WheelTuple,
type Mask,
OVERLAY_ANIM_DEFAULT_SECONDS,
OVERLAY_ANIM_MAX_SECONDS,
@@ -75,7 +86,7 @@ import { STOCK_VIDEOS } from "./stockVideoManifest";
import { applyOverlayPatchSettled, track0Clips, laneGapAt, getClipSpans, nextFreeStart, overlayLaneOrder, overlayLayers, parkedTransitions, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";
import { playheadAt } from "./playhead";
import { renderProjectFrame } from "./exportRender";
-import { rasterCanvasToDataUrl } from "./raster";
+import { createRasterCanvas, decodeRasterImageUrl, rasterCanvasToDataUrl } from "./raster";
import { buildAiContext } from "./aiContext";
import { sampleClipFrameData } from "./previewCanvas";
import { CAPTION_STYLES, laneCues, subtitleLaneCount } from "./subtitles";
@@ -227,7 +238,64 @@ const ANIM_STYLE_SYNONYMS: Record<string, AnimStyle> = {
wipedown: "slidedown",
};
-const GRADE_KEYS = ["brightness", "contrast", "saturation", "exposure", "temperature", "hue"] as const;
+/** The basic sliders the grade tool patches — the panel's field list plus the
+ * legacy pair, so the schema and handler can never disagree. */
+const GRADE_KEYS = [...GRADE_BASIC_FIELDS.map((f) => f.key), "brightness", "hue"] as const;
+
+/** Color stats for a grading reference: a clip's current decoded frame
+ * (pre-grade, like Auto reads) or an image asset's pixels. */
+async function colorStatsForRef(
+ s: ReturnType<typeof useEditor.getState>,
+ ref: { clipId?: string; assetId?: string }
+): Promise<ColorStats> {
+ if (ref.clipId) {
+ const clip = requireItem(s.clips, ref.clipId, "video clip");
+ const data = sampleClipFrameData(clip.id, 192, 108);
+ if (!data)
+ throw new ToolError(
+ "No decoded frame for that clip yet — seek into it so it is on screen, then retry."
+ );
+ const stats = colorStatsFromImageData(data);
+ if (!stats) throw new ToolError("Could not read pixels off that clip's frame.");
+ return stats;
+ }
+ if (ref.assetId) {
+ const asset = requireItem(s.assets, ref.assetId, "project asset");
+ if (asset.type !== "image")
+ throw new ToolError(
+ "Color stats read images and clip frames — for a video asset, place it on the timeline and pass its clip id."
+ );
+ const img = await decodeRasterImageUrl(asset.url);
+ if (!img || !img.width) throw new ToolError("Could not decode that image.");
+ const w = 192;
+ const h = Math.max(1, Math.round((img.height / img.width) * w));
+ const c = createRasterCanvas(w, h);
+ const ctx = c.getContext("2d") as CanvasRenderingContext2D | null;
+ if (!ctx) throw new ToolError("Could not read that image's pixels.");
+ ctx.drawImage(img.source, 0, 0, w, h);
+ const stats = colorStatsFromImageData(ctx.getImageData(0, 0, w, h).data);
+ if (!stats) throw new ToolError("Could not read pixels off that image.");
+ return stats;
+ }
+ throw new ToolError("Pass clipId or assetId.");
+}
+
+/** Stats shaped for the model: rounded, with the quantile probes labeled. */
+function statsReport(stats: ColorStats) {
+ const r1 = (v: number) => Math.round(v * 100) / 100;
+ return {
+ quantiles: "values 0..255 at p2, p10, p25, p50, p75, p90, p98",
+ r: stats.r,
+ g: stats.g,
+ b: stats.b,
+ luma: stats.luma,
+ meanR: Math.round(stats.meanR),
+ meanG: Math.round(stats.meanG),
+ meanB: Math.round(stats.meanB),
+ meanSat: r1(stats.meanSat),
+ warmth: r1(stats.warmth),
+ };
+}
/** Long side of a captured frame: enough for the model to read a title, small
* enough to ride a turn. */
@@ -2794,14 +2862,16 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
set_color_grade: (s, input) => {
const clip = requireItem(s.clips, input.clipId, "video clip");
- let grade: ColorGrade = input.reset === true ? {} : { ...clip.grade };
+ // reset and auto rebuild the manual layer; the preset layer stays.
+ let grade: ColorGrade =
+ input.reset === true ? { preset: clip.grade?.preset } : { ...clip.grade };
if (input.auto === true) {
const data = sampleClipFrameData(clip.id);
if (!data)
throw new ToolError(
"No decoded frame for that clip yet — seek into it so it is on screen, then retry."
);
- grade = { ...autoGradeFromImageData(data) };
+ grade = { ...autoGradeFromImageData(data), preset: clip.grade?.preset };
}
for (const k of GRADE_KEYS) if (isNum(input[k])) grade[k] = input[k];
s.updateClip(clip.id, { grade: normalizeGrade(grade) });
@@ -2809,6 +2879,131 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return { id: next.id, grade: next.grade ?? "neutral" };
},
+ set_color_preset: (s, input) => {
+ const clip = requireItem(s.clips, input.clipId, "video clip");
+ const id = typeof input.preset === "string" ? input.preset : "";
+ if (id !== "none" && !GRADE_PRESETS[id])
+ throw new ToolError(`No color preset "${id}" — pass one of the catalog ids or "none".`);
+ const preset =
+ id === "none"
+ ? undefined
+ : {
+ id,
+ amount: isNum(input.amount) ? Math.max(0, Math.min(1, input.amount)) : undefined,
+ skin: input.protect_skin === true || undefined,
+ };
+ s.updateClip(clip.id, { grade: normalizeGrade({ ...clip.grade, preset }) });
+ const next = useEditor.getState().clips.find((c) => c.id === clip.id)!;
+ return {
+ id: next.id,
+ preset: next.grade?.preset ?? "none",
+ ...(preset ? { label: GRADE_PRESETS[id].label } : {}),
+ ...(preset && next.grade?.preset?.amount === 0
+ ? { note: "amount 0 holds the preset at neutral — raise it for the look to show." }
+ : {}),
+ };
+ },
+
+ set_color_curves: (s, input) => {
+ const clip = requireItem(s.clips, input.clipId, "video clip");
+ const curves: GradeCurves = input.reset === true ? {} : { ...clip.grade?.curves };
+ const channels = [
+ ["m", input.master],
+ ["r", input.red],
+ ["g", input.green],
+ ["b", input.blue],
+ ] as const;
+ for (const [ch, pts] of channels) {
+ if (!Array.isArray(pts)) continue;
+ if (pts.length === 0) delete curves[ch];
+ else curves[ch] = pts as CurvePoint[];
+ }
+ if (isNum(input.curve_contrast) || isNum(input.fade)) {
+ const master = semanticMasterCurve(
+ isNum(input.curve_contrast) ? input.curve_contrast : 0,
+ isNum(input.fade) ? input.fade : 0
+ );
+ if (master) curves.m = master;
+ else delete curves.m;
+ }
+ s.updateClip(clip.id, { grade: normalizeGrade({ ...clip.grade, curves }) });
+ const next = useEditor.getState().clips.find((c) => c.id === clip.id)!;
+ return { id: next.id, curves: next.grade?.curves ?? "neutral" };
+ },
+
+ set_color_wheels: (s, input) => {
+ const clip = requireItem(s.clips, input.clipId, "video clip");
+ const wheels = input.reset === true ? {} : { ...clip.grade?.wheels };
+ const zones = [
+ ["s", input.shadows],
+ ["m", input.midtones],
+ ["h", input.highlights],
+ ] as const;
+ for (const [zone, w] of zones) {
+ if (!w || typeof w !== "object") continue;
+ const t = w as { dx?: number; dy?: number; luma?: number };
+ wheels[zone] = [t.dx ?? 0, t.dy ?? 0, t.luma ?? 0] as WheelTuple;
+ }
+ s.updateClip(clip.id, { grade: normalizeGrade({ ...clip.grade, wheels }) });
+ const next = useEditor.getState().clips.find((c) => c.id === clip.id)!;
+ return { id: next.id, wheels: next.grade?.wheels ?? "neutral" };
+ },
+
+ set_color_hsl: (s, input) => {
+ const clip = requireItem(s.clips, input.clipId, "video clip");
+ const hsl = input.reset_all === true ? {} : { ...clip.grade?.hsl };
+ const band = typeof input.band === "string" ? (input.band as HslBand) : undefined;
+ if (band) {
+ if (!HSL_BANDS.some((b) => b.id === band))
+ throw new ToolError(`band must be one of ${HSL_BANDS.map((b) => b.id).join(", ")}.`);
+ hsl[band] = [
+ isNum(input.hue) ? input.hue : 0,
+ isNum(input.sat) ? input.sat : 0,
+ isNum(input.luma) ? input.luma : 0,
+ ];
+ } else if (input.reset_all !== true) {
+ throw new ToolError("Pass a band to adjust, or reset_all to clear every band.");
+ }
+ s.updateClip(clip.id, { grade: normalizeGrade({ ...clip.grade, hsl }) });
+ const next = useEditor.getState().clips.find((c) => c.id === clip.id)!;
+ return { id: next.id, hsl: next.grade?.hsl ?? "neutral" };
+ },
+
+ read_color_stats: async (s, input) => {
+ const stats = await colorStatsForRef(s, {
+ clipId: typeof input.clipId === "string" ? input.clipId : undefined,
+ assetId: typeof input.assetId === "string" ? input.assetId : undefined,
+ });
+ return statsReport(stats);
+ },
+
+ match_color_grade: async (s, input) => {
+ const clip = requireItem(s.clips, input.clipId, "video clip");
+ const source = await colorStatsForRef(s, { clipId: clip.id });
+ const reference = await colorStatsForRef(s, {
+ clipId: typeof input.ref_clip_id === "string" ? input.ref_clip_id : undefined,
+ assetId: typeof input.ref_asset_id === "string" ? input.ref_asset_id : undefined,
+ });
+ const grade = matchGrade(source, reference);
+ // The frames already agree: the clip's own grade stays as it is rather
+ // than being wiped by a match that has nothing to say.
+ if (!grade)
+ return {
+ id: clip.id,
+ grade: clip.grade ?? "neutral",
+ note: "The footage already matches the reference — nothing changed.",
+ };
+ // The computed curves carry the whole look, so the match replaces the
+ // clip's grade — a preset left underneath would double the move.
+ s.updateClip(clip.id, { grade });
+ const next = useEditor.getState().clips.find((c) => c.id === clip.id)!;
+ return {
+ id: next.id,
+ grade: next.grade ?? "already matched (no change needed)",
+ note: "Verify with read_color_stats on the clip and refine with the grading tools.",
+ };
+ },
+
merge_cue: (s, input) => {
const cue = requireItem(s.subtitles.cues, input.id, "subtitle cue");
const mates = laneCues(s.subtitles, cue.lane ?? 0);
@@ -3004,7 +3199,7 @@ function textSequenceLines(
const lines = raw
.filter((l): l is Record<string, unknown> => typeof l === "object" && l !== null)
.map((l) => ({
- text: typeof l.text === "string" ? l.text.trim() : "",
+ text: typeof l.text === "string" ? l.text.replace(/\\n/g, "\n").trim() : "",
start: isNum(l.start) ? Math.max(0, l.start) : NaN,
end: isNum(l.end) ? l.end : NaN,
...(typeof l.color === "string" ? { color: l.color } : {}),
@@ -3393,7 +3588,10 @@ function overlayPatch(input: Record<string, unknown>, kind: "text" | "shape" | "
if (typeof input.hidden === "boolean") patch.hidden = input.hidden || undefined;
if (kind === "text") {
- if (typeof input.text === "string" && input.text.trim()) patch.text = input.text;
+ // A literal backslash-n in the call is always a meant newline — models
+ // double-escape multi-line titles.
+ if (typeof input.text === "string" && input.text.trim())
+ patch.text = input.text.replace(/\\n/g, "\n");
if (isNum(input.size)) patch.size = clamp(Math.round(input.size), 16, 320);
if (typeof input.color === "string") patch.color = input.color;
// Only ids the registry knows: an unknown one resolves to the default
diff --git a/site/src/cut/lib/pi/mutationLedger.ts b/site/src/cut/lib/pi/mutationLedger.ts
index 3d1229f9..7e505a5d 100644
--- a/site/src/cut/lib/pi/mutationLedger.ts
+++ b/site/src/cut/lib/pi/mutationLedger.ts
@@ -15,6 +15,7 @@ const READ_ONLY = new Set([
"note_source",
"detect_silence",
"listen_audio",
+ "read_color_stats",
"library_list",
"library_list_folder",
"stock_search",
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 2f5db3fe..f512926e 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -30,7 +30,13 @@ import { SUBTITLES_TOOLS } from "@/cut/components/SubtitlesPanel.tools";
import { TIMELINE_TOOLS } from "@/cut/components/Timeline.tools";
import { TOP_BAR_TOOLS } from "@/cut/components/TopBar.tools";
import { TRANSITIONS_TOOLS } from "@/cut/components/TransitionsPanel.tools";
-import { LOOK_EFFECTS, OVERLAY_ANIM_STYLE_IDS, OVERLAY_LOOP_STYLE_IDS } from "@donkeycut/effects-kit";
+import {
+ gradePresetCatalogText,
+ HSL_BANDS,
+ LOOK_EFFECTS,
+ OVERLAY_ANIM_STYLE_IDS,
+ OVERLAY_LOOP_STYLE_IDS,
+} from "@donkeycut/effects-kit";
import type { AiToolDef } from "../../lib/aiToolDef";
import { GOOGLE_FONT_IDS } from "../../lib/fontCatalog";
import { FONTS, TRANSITION_STYLE_IDS } from "../../lib/types";
@@ -77,7 +83,7 @@ Times are in seconds on the shared timeline. The playhead is currentTime; a skim
- A clip's timeline length is (out-in)/speed; total duration runs to the last clip's end, gaps included. Transitions blend at cuts and never change layout or length.
- trim_clip changes in/out inside the source media. in >= 0, out <= source duration, out-in >= 0.1.
- set_speed sets a clip's playback rate; it changes the clip's timeline length, and later titles/captions ripple to stay in sync.
-- set_transition joins a clip into the next one (0–2s, ${TRANSITION_STYLE_IDS.length} styles); set_animation animates one clip's own entrance/exit — read the transitions-and-fades skill before styling cuts. Splitting or deleting clears the affected transition. A transition is a bar on its own row at an absolute time, so an edit can leave one lining up with nothing: every timeline mutation reports those as \`parkedTransitions\`, and clearing them (remove_transition) is part of finishing the edit. Grading a clip's picture is add_effect: an effect dropped over a clip opens covering it.
+- set_transition joins a clip into the next one (0–2s, ${TRANSITION_STYLE_IDS.length} styles); set_animation animates one clip's own entrance/exit — read the transitions-and-fades skill before styling cuts. Splitting or deleting clears the affected transition. A transition is a bar on its own row at an absolute time, so an edit can leave one lining up with nothing: every timeline mutation reports those as \`parkedTransitions\`, and clearing them (remove_transition) is part of finishing the edit. Grading a clip's picture is the color tools (set_color_preset, set_color_grade, set_color_curves, set_color_wheels, set_color_hsl — read the color-grading skill); a stylized post treatment (grain, VHS, halation…) is add_effect, dropped over the stretch of timeline it covers.
- split_at cuts the track-0 clip under that time into two clips at the exact frame. With a soundtrack or overlay clip selected it splits that instead.
- The user can multi-select (⌘/⇧-click) and delete several items at once; a hover chip on each video clip toggles its own audio.
- Anything can be hidden instead of deleted: it stays on the timeline (grayed) but drops out of playback and export — a hidden video clip plays black and silent. Per item: set_clip_hidden, or the \`hidden\` field on update_overlay_video/update_overlay/update_audio. Whole rows: set_track_hidden (video/soundtrack/text/subtitles) and set_track_muted (video) — the eye/speaker toggles on the track headers.
@@ -89,6 +95,25 @@ Times are in seconds on the shared timeline. The playhead is currentTime; a skim
- The user can copy/paste any selected segment (video, overlay video, audio, title) with ⌘C/⌘V; pastes aim for the playhead and slide right to free space.
- Zoom: set_view pxPerSec (12..800) or fit. The timeline panel height: set_view timelineH (170..600).`,
+ "color-grading": `# Color grading
+Grade like a colorist: read the image, correct it, then style it — in that order.
+
+**Read before you write.** Call read_color_stats on the clip before grading (never grade blind) and again after a move to verify it landed. The quantiles tell you where the tones sit: p2 well above 0 = lifted/milky blacks, p98 well below 255 = no real whites (flat), a p50 far from ~118 = exposure off. warmth > 1 reads warm, < 1 cool; meanSat under ~0.15 reads muted, over ~0.35 vivid. The clip's current color state is in editor_state (colorPreset, grade, curves/wheels/hslBands).
+
+**Correct first.** Balance and tone before any look: auto (set_color_grade auto:true) is a computed starting point — exposure to middle gray, contrast stretch, gray-world white balance — then nudge. Neutralize a cast with temperature/tint before styling warmth on top, or the style fights the cast. Every value 0 is neutral and fields patch, so small moves are cheap; slider range is ±50.
+
+**One model, layered.** A clip's color is one grade: an optional preset underneath, manual adjustments on top. set_color_preset never touches the manual layer and vice versa; intensity (amount) scales only the preset. Preview, thumbnails, and export render the same numbers.
+
+**Which tool for which job.**
+- set_color_preset — a fast, coherent base. Catalog: ${gradePresetCatalogText()}. Start amount around 0.6–0.8 and tune to taste; protect_skin when a strong palette would discolor faces.
+- set_color_grade — the basic sliders: exposure/contrast for tone, highlights/shadows for the ends, temperature/tint for balance, saturation/vibrance for color amount (vibrance favors the muted colors and spares what is already rich).
+- set_color_curves — tonal shape: curve_contrast for an s-curve, fade for lifted film blacks, explicit points when you need a precise remap; red/green/blue channels split-tone tonally (a touch of blue lifted in shadows = cool shadows).
+- set_color_wheels — split-toning by range: push shadows toward one hue and highlights toward another (teal shadows dx<0, warm highlights dx>0 is the classic pairing); luma trims each range's brightness.
+- set_color_hsl — one hue only (${HSL_BANDS.map((b) => b.id).join(", ")}): "sky bluer" = aqua/blue sat up, "mute the greens" = green sat down, skin smoothing = orange band, gently.
+- match_color_grade — a reference exists (attached image or another clip): the grade is computed from the pixels (quantile-matched curves + saturation), then refine with the tools above. Matching adjacent clips for consistency is this too — match the odd clip to its neighbor.
+
+**Craft.** Skin lives around orange at modest saturation — check faces after strong saturation, hue, or wheel moves, and reach for protect_skin or a gentle orange-band correction rather than abandoning the look. Keep adjacent clips consistent: one preset + per-clip correction reads as one film; different presets per clip read as a slideshow. Strong looks sit better with contrast established first. Grades are per-clip — "grade the whole video" means every clip, same recipe.`,
+
"watching-and-cutting": `# Watching footage & cutting by content
When a request depends on what the footage actually contains — "cut the dead air", "clip the best moment", "remove the boring part", "split where the scene changes" — watch it first. Never guess at content you haven't seen.
@@ -146,7 +171,7 @@ Titles (add_title / update_overlay): text, size (frame px; the design short side
Shapes (add_shape): rect, ellipse, line, arrow. w/h are frame fractions; rect/ellipse take fill + fill_opacity, corner radius (rect), and an outline (stroke_color/stroke_width); line/arrow draw in \`fill\` with h as their thickness, and rotation gives them their direction (0° points right, 90° points down).
Stickers (add_sticker): a project image asset (generated cutouts in the Elements panel carry origin "sticker"); w is a frame-width fraction and height follows the source's aspect. Lottie JSON uploads become animated stickers that loop for the element's duration.
Custom stickers (create_sticker): one call generates the image, removes the background, adds the die-cut outline, and places it — use it when the user asks for a sticker of something they don't have (it spends credits like image generation).
-Effects (add_effect): time-ranged treatments over the finished picture — the footage and everything laid over it. Two families, one list: the treatments (${EFFECT_TREATMENTS.join(", ")}) and the graded looks (${LOOK_EFFECTS.join(", ")}). Placed like any element and tuned with \`amount\` (update_overlay); an effect added over a clip opens covering that clip, and trims like anything else. One effect at a time reads best, amounts under ~0.5.
+Effects (add_effect): time-ranged treatments over the finished picture — the footage and everything laid over it. Two families, one list: the treatments (${EFFECT_TREATMENTS.join(", ")}) and the graded looks (${LOOK_EFFECTS.join(", ")}). Placed like any element and tuned with \`amount\` (update_overlay); an effect added over a clip opens covering that clip, and trims like anything else. One effect at a time reads best, amounts under ~0.5. Effects are stylized post treatments on a stretch of timeline; pure color — exposure, warmth, a cinematic or film palette on one clip — is the per-clip grading tools instead (color-grading skill), whose preset catalog is separate from the look effects.
Advanced text (update_overlay): italic, align (multi-line), letter_spacing (em), line_height, stroke_color/stroke_width (em outline behind the fill), and richer shadows. Bundled Google families join the system set — ${GOOGLE_FONT_IDS.join(", ")} — and so do the user's own: a .ttf/.otf/.woff/.woff2 file dropped on the Library is offered in every project under a font id like "font:<id>". Editor_state lists the ones this account has under \`fonts\` — use an id from there instead of guessing at a family name, and when someone asks for a typeface that is not in the list, tell them the Library takes font files.
Animation (set_overlay_animation): every element takes preset In/Out ramps (${OVERLAY_ANIM_STYLE_IDS.join(", ")} — typewriter for titles only; the per-letter styles sweep a title character by character), a Loop (${OVERLAY_LOOP_STYLE_IDS.join(", ")}) that runs its whole duration, and a Move that says what it does WHILE it holds. Entrances 0.3–0.6s read well; one loop per scene, sparingly; a move on the lines that matter. Every one of these is keyframe data — the same poses set_overlay_keyframes writes — so a named move lands as ordinary keys the user can drag, and anything the names do not cover is keyed directly.
Keyframes (set_overlay_keyframes): for motion no preset covers — a path across the frame, a slow push in, a drift that holds first. A key is a whole pose (position, scale, rotation, opacity) at a time in seconds from the element's start; the pose moves linearly between keys and holds outside them, and presets still compose on top. Two or three keys carry almost everything; reach for a preset first and keyframes when the user describes a specific path or timing. In the UI the diamond in the inspector adds a key at the playhead, and dragging the element then records into it. set_clip_keyframes is the same track on a video clip (any track): pans, push-ins, picture-in-picture flights, spins, and fades on the footage itself.
@@ -341,7 +366,7 @@ Rules:
- Times are seconds. The frame follows project.aspect in editor_state — any "W:H" ratio, short side 1080px; project.frame has the exact pixels. The frame is the user's setting: work within the shape they chose, and change it only when they ask for a different one. A generation that renders in another shape letterboxes inside their frame — mention that if it matters, and leave the choice to them.
- Exercise judgment on details the user left open, and state the choice you made. Judgment fills in the parameters of the asked-for change (style, length, placement); the scope stays the ask, and extra tools, tracks, or treatments beyond it wait for the user to ask. Editing defaults: a hard cut at every joint; when transitions are asked for, one style family per video at 0.4–0.6s. The editing-taste skill carries the full doctrine.
- Read list_skills / read_skill before working in an area you're unsure about — they document every setting. A tool whose description already answers the ask goes straight through.
-- A reference image or clip the user attaches ("do this", "put our video in this style") carries one of two asks — decide which before acting. When its footage's look is the point (color, lighting, film grade), route to looks, grades, or restyle. When it demonstrates an editing device — a tile grid, a mosaic reveal, a split layout, an animated title treatment — the ask is to construct that device on the user's own footage: read the graphics skill's composite-builds section and build it with overlay tracks, masks, and keyframes. Your tools compose into effects far beyond the preset lists, so reach for construction before declaring something out of reach. Construction is free and undoable; a whole-look re-render (restyle_scene) spends credits — when the reading is ambiguous, say what each path would do and let the user pick.
+- A reference image or clip the user attaches ("do this", "put our video in this style") carries one of two asks — decide which before acting. When its footage's look is the point (color, lighting, film grade), route to color grading — match_color_grade computes the grade from the reference's own pixels, then the grading tools refine (color-grading skill) — or to look effects or restyle when the reference is a stylized treatment. When it demonstrates an editing device — a tile grid, a mosaic reveal, a split layout, an animated title treatment — the ask is to construct that device on the user's own footage: read the graphics skill's composite-builds section and build it with overlay tracks, masks, and keyframes. Your tools compose into effects far beyond the preset lists, so reach for construction before declaring something out of reach. Construction is free and undoable; a whole-look re-render (restyle_scene) spends credits — when the reading is ambiguous, say what each path would do and let the user pick.
- Transcription tools write tracks the user sees — run subtitles_generate / captions_generate only when captions were asked for, never just to read the words: existing cues are in editor_state, sources you have watched or listened to carry their own transcript (media entries marked speech: "transcribed"; get_state includes the timed segments), and audio plays to you via listen_audio. Don't transcribe a video with no speech (subtitles_from_visuals narrates silent footage), and never invent a spoken transcript.
- Voiceovers duck other audio by default so they stay audible. Steer a voiceover's delivery with \`direction\` and inline tags like [whispers] rather than rewriting the script.
- generate_image / generate_video / generate_character_video / voiceover_generate / read_subtitles_aloud / generate_music make media through hosted models (spends the user's Donkey credits, needs sign-in); call them when the user asked for the media itself — a request for the prompt or script gets text in chat. generate_music makes a music track — a vocal-free background bed by default ("add music to this video"), or a full sung song (instrumental:false); it's sung music, so a spoken narration is voiceover_generate. Bundled stock media (stock_search / stock_add) is free — use it when it fits. Media the user attached is in \`media\`; pass those asset ids as generation references when they say "use this", and place project assets in the cut with add_clip when they ask for them there ("make a movie from these photos"). Generated media previews on a chat card the user can expand, drag in, or file away; add it to the timeline (add_to_timeline:true or an index) only when they asked for it in the cut. When it lands on a card, close with a statement of where it is ("The voiceover is on the card — drag it in or tell me to place it."). Write a rich, specific prompt from their shorthand. Video renders take a minute or two. "Generate a video of/about X" means one generate_video clip; only when they name a narrated multi-shot production (a story, episode, narrated short) does generate_scene plan it and stop at the storyboard — the user reviews the frames (regenerate_shot redraws any before approval, no credits), then call approve_scene, since it renders many paid shots (regenerate_shot / recut_scene / restyle_scene revise it after). Close a plan message with the approval path as a statement: "Approve it — say so or hit Approve on the card — and I'll render the shots." Tweaks before approval are theirs to name.