Skip to content

Open source CapCut alternative.

TypeScript

feat(site): fit color matches through the pipeline [rebuild]

David Lec7347bebef49
diff --git a/site/packages/effects-kit/src/gradeMatch.test.ts b/site/packages/effects-kit/src/gradeMatch.test.ts
index 8056e326..e9446aa9 100644
--- a/site/packages/effects-kit/src/gradeMatch.test.ts
+++ b/site/packages/effects-kit/src/gradeMatch.test.ts
@@ -1,6 +1,10 @@
import { describe, expect, test } from "bun:test";
+import type { ColorGrade } from "./colorGrade";
import { createGradeTransform } from "./gradeMath";
-import { autoGradeFromImageData, colorStatsFromImageData, matchGrade, type ColorStats } from "./gradeMatch";
+import { type Lab, deltaE2000, displayToLab } from "./colorDifference";
+import { type ClipColorRecipe, createClipTransform } from "./colorPipeline";
+import { type SourceProfile, appleLogEncode, bt1886Decode } from "./colorSpace";
+import { autoGrade, colorStatsFromImageData, matchGrade, recipeStatsRender } from "./gradeMatch";
/** A synthetic frame with tonal and color variety. */
function testFrame(): Uint8ClampedArray {
@@ -19,116 +23,72 @@ function testFrame(): Uint8ClampedArray {
return data;
}
-function graded(data: Uint8ClampedArray, f: (v: number, c: number) => number): Uint8ClampedArray {
- const out = new Uint8ClampedArray(data.length);
- for (let i = 0; i < data.length; i += 4) {
- for (let c = 0; c < 3; c++) out[i + c] = Math.max(0, Math.min(255, Math.round(f(data[i + c], c))));
- out[i + 3] = data[i + 3];
+/** A clip's code values rendered through its recipe with `grade`, as pixels. */
+function rendered(px: Uint8ClampedArray, recipe: ClipColorRecipe, grade?: ColorGrade): Lab[] {
+ const fn = createClipTransform({ ...recipe, grade });
+ const out: Lab[] = [];
+ for (let i = 0; i < px.length; i += 4) {
+ const v = fn ? fn(px[i] / 255, px[i + 1] / 255, px[i + 2] / 255) : [px[i] / 255, px[i + 1] / 255, px[i + 2] / 255];
+ out.push(displayToLab(Math.round(v[0] * 255), Math.round(v[1] * 255), Math.round(v[2] * 255)));
}
return out;
}
+const meanDeltaE = (a: Lab[], b: Lab[]) => a.reduce((s, p, i) => s + deltaE2000(p, b[i]), 0) / a.length;
+
+const recipeFor = (profile: SourceProfile): ClipColorRecipe => ({ profile, output: "sdr", size: 0 });
+
describe("matchGrade", () => {
test("a frame matched to itself is neutral", () => {
- const stats = colorStatsFromImageData(testFrame())!;
- expect(matchGrade(stats, stats)).toBeUndefined();
+ const render = recipeStatsRender(testFrame(), recipeFor("rec709"));
+ expect(matchGrade(render, render(undefined)!)).toBeUndefined();
});
- test("recovers a known channel shift within tolerance", () => {
- const src = testFrame();
- // Warm the reference: red up, blue down — a temperature-like cast.
- const ref = graded(src, (v, c) => (c === 0 ? v * 1.18 : c === 2 ? v * 0.85 : v));
- const srcStats = colorStatsFromImageData(src)!;
- const refStats = colorStatsFromImageData(ref)!;
- const grade = matchGrade(srcStats, refStats);
- expect(grade !== undefined).toBe(true);
- const transform = createGradeTransform(grade)!;
- // Apply the match and compare channel means against the reference.
- let mr = 0;
- let mg = 0;
- let mb = 0;
- let n = 0;
- for (let i = 0; i < src.length; i += 4) {
- const [r, g, b] = transform(src[i] / 255, src[i + 1] / 255, src[i + 2] / 255);
- mr += r * 255;
- mg += g * 255;
- mb += b * 255;
- n++;
+ // The eval: a reference made by grading the clip's own frame with a known
+ // grade, so a perfect match exists. The fitted grade has to land the frame
+ // on that reference pixel for pixel — mean CIEDE2000 under 1, below what
+ // an observer sees across a cut — on Rec.709 and on Apple Log graded in
+ // scene light.
+ const TARGETS: ColorGrade[] = [
+ { exposure: 12, temperature: 20 },
+ { contrast: 25, temperature: -18, saturation: 10 },
+ { exposure: -10, tint: 12, saturation: -25 },
+ { highlights: -20, shadows: 15, temperature: 10 },
+ ];
+ for (const profile of ["rec709", "apple-log-2"] as SourceProfile[]) {
+ for (const target of TARGETS) {
+ test(`${profile}: recovers ${JSON.stringify(target)}`, () => {
+ const px = testFrame();
+ const recipe = recipeFor(profile);
+ const render = recipeStatsRender(px, recipe);
+ const grade = matchGrade(render, render(target)!);
+ expect(grade).toBeDefined();
+ expect(meanDeltaE(rendered(px, recipe, grade), rendered(px, recipe, target))).toBeLessThan(1);
+ });
}
- mr /= n;
- mg /= n;
- mb /= n;
- expect(Math.abs(mr - refStats.meanR)).toBeLessThan(10);
- expect(Math.abs(mg - refStats.meanG)).toBeLessThan(10);
- expect(Math.abs(mb - refStats.meanB)).toBeLessThan(10);
- });
+ }
- test("matched curves reach both ends of the range", () => {
- const src = testFrame();
- const ref = graded(src, (v, c) => (c === 0 ? v * 1.18 : c === 2 ? v * 0.85 : v));
- const grade = matchGrade(colorStatsFromImageData(src)!, colorStatsFromImageData(ref)!)!;
- // The spline holds flat past its end points, so a curve stopping at the
- // p2/p98 probes would flatten the darkest and brightest pixels.
- for (const ch of ["r", "g", "b"] as const) {
- const pts = grade.curves?.[ch];
- if (!pts) continue;
- expect(pts[0][0]).toBe(0);
- expect(pts[pts.length - 1][0]).toBe(255);
- for (let i = 1; i < pts.length; i++) {
- expect(pts[i][0]).toBeGreaterThan(pts[i - 1][0]);
- expect(pts[i][1]).toBeGreaterThanOrEqual(pts[i - 1][1]);
- }
+ test("strength scales every fitted slider", () => {
+ const render = recipeStatsRender(testFrame(), recipeFor("rec709"));
+ const reference = render({ exposure: 16, temperature: 24 })!;
+ const full = matchGrade(render, reference)!;
+ const half = matchGrade(render, reference, 0.5)!;
+ for (const [k, v] of Object.entries(full)) {
+ expect(Math.abs((half as Record<string, number>)[k] ?? 0) - Math.abs((v as number) / 2)).toBeLessThanOrEqual(0.5);
}
+ expect(matchGrade(render, reference, 0)).toBeUndefined();
});
- test("a reference of other content never posterizes: every curve segment keeps a bounded slope", () => {
- // A shadow on a blue-and-orange wall matched to a shadow on beige
- // pavement: the green quantiles jump 54 -> 164 across eight code values,
- // which as a curve tears the wall into flat bands of false colour.
- const stats = (r: number[], g: number[], b: number[]): ColorStats => ({
- r,
- g,
- b,
- luma: g,
- meanR: r[3],
- meanG: g[3],
- meanB: b[3],
- meanSat: 0.3,
- warmth: 1,
- });
- const source = stats([50, 59, 126, 153, 227, 230, 232], [42, 52, 113, 147, 155, 160, 171], [23, 48, 60, 74, 132, 149, 161]);
- const reference = stats([33, 37, 40, 48, 203, 209, 213], [39, 45, 47, 54, 164, 173, 178], [40, 45, 48, 55, 135, 144, 150]);
- const grade = matchGrade(source, reference)!;
- for (const ch of ["r", "g", "b"] as const) {
- const pts = grade.curves?.[ch];
- expect(pts).toBeDefined();
- for (let i = 1; i < pts!.length; i++) {
- const slope = (pts![i][1] - pts![i - 1][1]) / (pts![i][0] - pts![i - 1][0]);
- // Rounding to whole code values moves a short segment's slope a little.
- const slack = 1.5 / (pts![i][0] - pts![i - 1][0]);
- expect(slope).toBeLessThanOrEqual(3 + slack);
- if (pts![i][1] < 255 && pts![i - 1][1] > 0) expect(slope).toBeGreaterThanOrEqual(1 / 3 - slack);
- }
- }
- // The middle of the range still lands where the reference puts it.
- const mid = grade.curves!.g!.find(([x]) => x === 147)!;
- expect(mid[1]).toBe(54);
- });
-
- test("strength scales the match: half strength moves every curve point half as far", () => {
- const src = testFrame();
- const ref = graded(src, (v, c) => (c === 0 ? v * 1.3 : c === 2 ? v * 0.7 : v));
- const srcStats = colorStatsFromImageData(src)!;
- const refStats = colorStatsFromImageData(ref)!;
- const full = matchGrade(srcStats, refStats)!;
- const half = matchGrade(srcStats, refStats, 0.5)!;
- for (const ch of ["r", "b"] as const) {
- const f = full.curves![ch]!;
- const h = half.curves![ch]!;
- expect(h.map(([x]) => x)).toEqual(f.map(([x]) => x));
- for (let i = 0; i < f.length; i++) expect(Math.abs(h[i][1] - (f[i][0] + f[i][1]) / 2)).toBeLessThanOrEqual(0.5);
- }
- expect(matchGrade(srcStats, refStats, 0)).toBeUndefined();
+ test("stats read lightness and each band's cast in CIELAB", () => {
+ const grey = colorStatsFromImageData(frame([119, 119, 119]))!;
+ // BT.1886 code 119 is 16% luminance: L* 47.
+ expect(Math.abs(grey.lightness[3] - 47)).toBeLessThan(1);
+ const mid = grey.tones[1];
+ expect(mid.share).toBe(1);
+ expect(Math.abs(mid.a) + Math.abs(mid.b)).toBeLessThan(0.5);
+ const warm = colorStatsFromImageData(frame([160, 120, 90], [40, 60, 90]))!;
+ expect(warm.tones[1].b).toBeGreaterThan(10);
+ expect(warm.tones[0].b).toBeLessThan(-10);
});
test("stats expose warmth and saturation for the assistant", () => {
@@ -155,29 +115,31 @@ function frame(...pixels: [number, number, number][]): Uint8ClampedArray {
return data;
}
-describe("autoGradeFromImageData", () => {
+const autoGrade709 = (px: Uint8ClampedArray) => autoGrade(px, recipeFor("rec709"));
+
+describe("autoGrade", () => {
test("a mid-gray frame needs nearly nothing", () => {
- const g = autoGradeFromImageData(frame([125, 125, 125]));
+ const g = autoGrade709(frame([125, 125, 125]));
expect(Math.abs(g?.exposure ?? 0) <= 1).toBe(true);
expect(g?.contrast).toBeUndefined();
expect(g?.temperature).toBeUndefined();
});
test("a dark frame gets pushed up toward middle gray, in stops", () => {
- const g = autoGradeFromImageData(frame([30, 30, 30], [50, 50, 50]));
+ const g = autoGrade709(frame([30, 30, 30], [50, 50, 50]));
expect((g?.exposure ?? 0) > 10).toBe(true);
expect(g?.exposure ?? 0).toBeLessThanOrEqual(50);
});
test("a compressed tonal range gets a contrast stretch", () => {
- const g = autoGradeFromImageData(frame([100, 100, 100], [150, 150, 150]));
+ const g = autoGrade709(frame([100, 100, 100], [150, 150, 150]));
expect((g?.contrast ?? 0) > 0).toBe(true);
});
test("gray-world counters color casts both ways and leaves hue and saturation alone", () => {
- const cool = autoGradeFromImageData(frame([90, 120, 170]));
+ const cool = autoGrade709(frame([90, 120, 170]));
expect((cool?.temperature ?? 0) > 0).toBe(true);
- const warm = autoGradeFromImageData(frame([170, 120, 90]));
+ const warm = autoGrade709(frame([170, 120, 90]));
expect((warm?.temperature ?? 0) < 0).toBe(true);
expect(cool?.saturation).toBeUndefined();
expect(cool?.hue).toBeUndefined();
@@ -186,7 +148,7 @@ describe("autoGradeFromImageData", () => {
test("the fitted white balance neutralizes the cast it measured", () => {
const cast: [number, number, number] = [150, 120, 95];
- const g = autoGradeFromImageData(frame(cast))!;
+ const g = autoGrade709(frame(cast))!;
const t = createGradeTransform({ temperature: g.temperature })!;
const out = t(cast[0] / 255, cast[1] / 255, cast[2] / 255);
// Damped correction: the red/blue gap closes by at least half.
@@ -194,6 +156,32 @@ describe("autoGradeFromImageData", () => {
});
test("an empty sample yields no grade", () => {
- expect(autoGradeFromImageData(new Uint8ClampedArray(0))).toBeUndefined();
+ expect(autoGrade709(new Uint8ClampedArray(0))).toBeUndefined();
+ });
+
+ test("on Apple Log the exposure lands the rendered frame on its target in true stops", () => {
+ // A flat log frame two stops under: grey card codes from a scene at 4.5%.
+ const recipe = recipeFor("apple-log-2");
+ const code = Math.round(appleLogEncode(0.045) * 255);
+ const px = frame([code, code, code], [code + 30, code + 30, code + 30]);
+ const g = autoGrade(px, recipe)!;
+ const before = Math.log2(renderedLogAverage(px, recipe));
+ const after = Math.log2(renderedLogAverage(px, recipe, { exposure: g.exposure }));
+ const want = before + 0.8 * (Math.log2(0.18) - before);
+ expect(Math.abs(after - want)).toBeLessThan(0.1);
});
});
+
+/** Geometric mean display luminance of a frame rendered through a recipe. */
+function renderedLogAverage(px: Uint8ClampedArray, recipe: ClipColorRecipe, grade?: ColorGrade): number {
+ const fn = createClipTransform({ ...recipe, grade });
+ let sum = 0;
+ let n = 0;
+ for (let i = 0; i < px.length; i += 4) {
+ const v = fn ? fn(px[i] / 255, px[i + 1] / 255, px[i + 2] / 255) : [px[i] / 255, px[i + 1] / 255, px[i + 2] / 255];
+ const luma = 0.2126 * Math.round(v[0] * 255) + 0.7152 * Math.round(v[1] * 255) + 0.0722 * Math.round(v[2] * 255);
+ sum += Math.log(Math.max(1e-4, bt1886Decode(luma / 255)));
+ n++;
+ }
+ return Math.exp(sum / n);
+}
diff --git a/site/packages/effects-kit/src/gradeMatch.ts b/site/packages/effects-kit/src/gradeMatch.ts
index 6435584f..6a17417b 100644
--- a/site/packages/effects-kit/src/gradeMatch.ts
+++ b/site/packages/effects-kit/src/gradeMatch.ts
@@ -2,15 +2,18 @@
* Measuring pictures and fitting grades to them: color statistics off a
* frame, the auto grade, and reference matching. Every fit is expressed in
* the grade's own parameters (exposure in stops, white balance as an
- * illuminant, contrast about grey, curves), so the result is an ordinary
- * ColorGrade the user or the assistant refines afterward. Callers hand these
- * the pixels of the base rendering — the picture after the source
- * conversion — so a log clip is measured as the display sees it.
+ * illuminant, contrast about grey), so the result is an ordinary ColorGrade
+ * the user or the assistant refines afterward. Stats and the auto grade read
+ * the base rendering — the picture after the source conversion — so a log
+ * clip is measured as the display sees it; a match renders the clip's code
+ * values through its real recipe for every grade it tries.
*/
-import { type ColorGrade, type CurvePoint, GRADE_MAX, normalizeGrade } from "./colorGrade";
-import { REC709_PRIMARIES, bt1886Decode, mat3Apply } from "./colorSpace";
-import { CONTRAST_POWER_AT_MAX, EXPOSURE_STOPS_AT_MAX, whiteBalanceFor } from "./gradeMath";
+import { type ColorGrade, GRADE_MAX, normalizeGrade } from "./colorGrade";
+import { displayToLab } from "./colorDifference";
+import { type ClipColorRecipe, createClipTransform } from "./colorPipeline";
+import { bt1886Decode } from "./colorSpace";
+import { CONTRAST_POWER_AT_MAX } from "./gradeMath";
/** Quantile probes used by stats and matching. */
export const STAT_QUANTILES = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98] as const;
@@ -29,10 +32,33 @@ export interface ColorStats {
meanSat: number;
/** Red/blue mean ratio in the midtones; >1 reads warm, <1 reads cool. */
warmth: number;
+ /** CIELAB lightness L* (0..100) at each STAT_QUANTILES probe. */
+ lightness: number[];
+ /** The color of the shadows, midtones and highlights (TONE_BANDS): the
+ * share of pixels in the band, the mean a* and b* — its cast, green–magenta
+ * and blue–yellow — and their spread, which saturation scales. */
+ tones: ToneColor[];
}
+export interface ToneColor {
+ share: number;
+ a: number;
+ b: number;
+ spreadA: number;
+ spreadB: number;
+}
+
+/** Upper L* edge of each tonal band, darkest first. */
+const TONE_BANDS = [35, 70, 100];
+
+/** Lightness bins per L* unit: a tenth of the smallest step anyone sees. */
+const L_BINS_PER_UNIT = 10;
+
export function colorStatsFromImageData(data: Uint8ClampedArray): ColorStats | undefined {
const hist = [new Float64Array(256), new Float64Array(256), new Float64Array(256), new Float64Array(256)];
+ const lightHist = new Float64Array(100 * L_BINS_PER_UNIT + 1);
+ // Per band: count, Σa, Σb, Σa², Σb².
+ const bands = TONE_BANDS.map(() => [0, 0, 0, 0, 0]);
let count = 0;
let sumR = 0;
let sumG = 0;
@@ -62,17 +88,25 @@ export function colorStatsFromImageData(data: Uint8ClampedArray): ColorStats | u
midB += b;
mids++;
}
+ const [L, a, lb] = displayToLab(r, g, b);
+ lightHist[Math.max(0, Math.min(lightHist.length - 1, Math.round(L * L_BINS_PER_UNIT)))]++;
+ const band = bands[Math.max(0, TONE_BANDS.findIndex((top) => L <= top))];
+ band[0]++;
+ band[1] += a;
+ band[2] += lb;
+ band[3] += a * a;
+ band[4] += lb * lb;
count++;
}
if (!count) return undefined;
- const quantiles = (h: Float64Array) =>
+ const quantiles = (h: Float64Array, scale = 1) =>
STAT_QUANTILES.map((p) => {
let acc = 0;
- for (let v = 0; v < 256; v++) {
+ for (let v = 0; v < h.length; v++) {
acc += h[v];
- if (acc >= count * p) return v;
+ if (acc >= count * p) return v / scale;
}
- return 255;
+ return (h.length - 1) / scale;
});
return {
r: quantiles(hist[0]),
@@ -84,6 +118,19 @@ export function colorStatsFromImageData(data: Uint8ClampedArray): ColorStats | u
meanB: sumB / count,
meanSat: sumSat / count,
warmth: mids ? midR / Math.max(1, midB) : 1,
+ lightness: quantiles(lightHist, L_BINS_PER_UNIT),
+ tones: bands.map(([n, sa, sb, saa, sbb]) => {
+ if (!n) return { share: 0, a: 0, b: 0, spreadA: 0, spreadB: 0 };
+ const a = sa / n;
+ const b = sb / n;
+ return {
+ share: n / count,
+ a,
+ b,
+ spreadA: Math.sqrt(Math.max(0, saa / n - a * a)),
+ spreadB: Math.sqrt(Math.max(0, sbb / n - b * b)),
+ };
+ }),
};
}
@@ -91,47 +138,124 @@ export function colorStatsFromImageData(data: Uint8ClampedArray): ColorStats | u
/* Auto grade */
/* ------------------------------------------------------------------ */
+/** Pixels of a frame rendered through its recipe with a candidate grade. */
+type PixelRender = (grade: ColorGrade | undefined) => Uint8ClampedArray;
+
+/** The most pixels a fit renders per try: enough for quantiles a tenth of
+ * an L* apart, few enough that a few hundred tries take well under a second. */
+const FIT_SAMPLE = 4096;
+
+/** A PixelRender over a frame's code values (RGBA) and the recipe that draws
+ * them, with the candidate grade in the recipe's place: every try runs the
+ * real pipeline, so a log clip is fitted in scene light exactly as the
+ * preview renders it. At most FIT_SAMPLE pixels, evenly spaced. */
+function recipeRender(px: Uint8ClampedArray, recipe: ClipColorRecipe): PixelRender {
+ const total = px.length >> 2;
+ const stride = Math.max(1, Math.ceil(total / FIT_SAMPLE));
+ const sample = new Float32Array(Math.ceil(total / stride) * 3);
+ let n = 0;
+ for (let i = 0; i < total; i += stride) {
+ sample[n * 3] = px[i * 4] / 255;
+ sample[n * 3 + 1] = px[i * 4 + 1] / 255;
+ sample[n * 3 + 2] = px[i * 4 + 2] / 255;
+ n++;
+ }
+ return (grade) => {
+ const fn = createClipTransform({ ...recipe, grade });
+ const out = new Uint8ClampedArray(n * 4);
+ for (let i = 0; i < n; i++) {
+ const r = sample[i * 3];
+ const g = sample[i * 3 + 1];
+ const b = sample[i * 3 + 2];
+ const v = fn ? fn(r, g, b) : [r, g, b];
+ out[i * 4] = Math.round(v[0] * 255);
+ out[i * 4 + 1] = Math.round(v[1] * 255);
+ out[i * 4 + 2] = Math.round(v[2] * 255);
+ out[i * 4 + 3] = 255;
+ }
+ return out;
+ };
+}
+
+const lumaOf = (data: Uint8ClampedArray, i: number) => 0.2126 * data[i] + 0.7152 * data[i + 1] + 0.0722 * data[i + 2];
+
+/** Geometric mean of display luminance: where the frame's exposure sits. */
+function logAverage(data: Uint8ClampedArray): number {
+ let sum = 0;
+ for (let i = 0; i < data.length; i += 4) sum += Math.log(Math.max(1e-4, bt1886Decode(lumaOf(data, i) / 255)));
+ return Math.exp(sum / Math.max(1, data.length >> 2));
+}
+
+/** Mean red minus mean blue of the midtones in display light — the cast
+ * grey-world reads; darker and clipped pixels carry no reliable illuminant. */
+function midtoneCast(data: Uint8ClampedArray): number {
+ let gap = 0;
+ let mids = 0;
+ for (let i = 0; i < data.length; i += 4) {
+ const luma = lumaOf(data, i);
+ if (luma < 16 || luma > 240) continue;
+ gap += bt1886Decode(data[i] / 255) - bt1886Decode(data[i + 2] / 255);
+ mids++;
+ }
+ return mids ? gap / mids : 0;
+}
+
+/** The slider value in [-GRADE_MAX, GRADE_MAX] where a monotone `f` crosses
+ * zero by bisection, or the end nearer zero when it never does. */
+function sliderRoot(f: (v: number) => number): number {
+ let lo = -GRADE_MAX;
+ let hi = GRADE_MAX;
+ let flo = f(lo);
+ const fhi = f(hi);
+ if (Math.sign(flo) === Math.sign(fhi)) return Math.abs(flo) < Math.abs(fhi) ? lo : hi;
+ for (let i = 0; i < 14; i++) {
+ const mid = (lo + hi) / 2;
+ const fm = f(mid);
+ if (Math.sign(fm) === Math.sign(flo)) {
+ lo = mid;
+ flo = fm;
+ } else hi = mid;
+ }
+ return (lo + hi) / 2;
+}
+
+/** How much of each measured correction Auto applies: a starting point the
+ * user refines, short of a full normalization. */
+const AUTO_EXPOSURE_SHARE = 0.8;
+const AUTO_CONTRAST_SHARE = 0.6;
+const AUTO_WB_SHARE = 0.6;
+const AUTO_CONTRAST_RANGE = [-10, 40] as const;
+const AUTO_WB_MAX = 40;
+
/**
- * Auto grade from a frame's RGBA pixels, the classic auto-tone pipeline:
+ * Auto grade from a frame's code values (RGBA) and the recipe that draws
+ * them, the classic auto-tone pipeline measured through the real render:
*
- * 1. Exposure — map the frame's log-average (geometric mean) luminance onto
- * 18% middle grey, in stops.
- * 2. Contrast — auto-levels: clip 0.5% off each end of the luma histogram
- * and stretch what remains toward full range, expressed as the power
- * about grey the contrast slider applies.
- * 3. White balance — grey-world: the illuminant whose CAT02 correction
- * equalizes the red and blue means of the midtones, found on the
- * temperature slider.
+ * 1. Exposure — the slider that moves the rendered frame's log-average
+ * luminance AUTO_EXPOSURE_SHARE of the way to 18% middle grey.
+ * 2. Contrast — auto-levels: clip 0.5% off each end of the rendered luma
+ * histogram and stretch what remains toward full range, expressed as the
+ * power about grey the contrast slider applies.
+ * 3. White balance — grey-world: the temperature whose render brings the
+ * midtones' red and blue means together, applied in part.
*
- * Corrections are damped and capped inside the slider range so the result
- * is a starting point the user refines. Saturation and hue are left alone.
+ * Saturation and hue are left alone.
*/
-export function autoGradeFromImageData(data: Uint8ClampedArray): ColorGrade | undefined {
+export function autoGrade(px: Uint8ClampedArray, recipe: ClipColorRecipe): ColorGrade | undefined {
+ if (px.length < 4) return undefined;
+ const render = recipeRender(px, recipe);
+ const grade: ColorGrade = {};
+
+ const start = Math.log(logAverage(render(undefined)));
+ const target = start + AUTO_EXPOSURE_SHARE * (Math.log(0.18) - start);
+ grade.exposure = Math.round(sliderRoot((e) => Math.log(logAverage(render({ exposure: e }))) - target));
+
+ // Auto-levels on the exposed render. A near-flat histogram (solid color,
+ // title card) carries no tonal-range signal: contrast stays.
+ const exposed = render(normalizeGrade(grade));
const hist = new Float64Array(256);
- let count = 0;
- let logSum = 0;
- let sumR = 0;
- let sumG = 0;
- let sumB = 0;
- let mids = 0;
- for (let i = 0; i < data.length; i += 4) {
- const r = data[i];
- const g = data[i + 1];
- const b = data[i + 2];
- const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b;
- hist[Math.min(255, Math.round(luma))]++;
- logSum += Math.log(Math.max(1e-4, bt1886Decode(luma / 255)));
- count++;
- // Cast statistics from the midtones only — near-black and clipped pixels
- // carry no reliable illuminant signal.
- if (luma >= 16 && luma <= 240) {
- sumR += bt1886Decode(r / 255);
- sumG += bt1886Decode(g / 255);
- sumB += bt1886Decode(b / 255);
- mids++;
- }
- }
- if (!count) return undefined;
+ for (let i = 0; i < exposed.length; i += 4) hist[Math.min(255, Math.round(lumaOf(exposed, i)))]++;
+ const count = exposed.length >> 2;
const percentile = (p: number) => {
let acc = 0;
for (let v = 0; v < 256; v++) {
@@ -140,43 +264,17 @@ export function autoGradeFromImageData(data: Uint8ClampedArray): ColorGrade | un
}
return 255;
};
-
- const grade: ColorGrade = {};
- const ev = Math.log2(0.18 / Math.exp(logSum / count));
- grade.exposure = Math.round((GRADE_MAX * ev * 0.8) / EXPOSURE_STOPS_AT_MAX);
- // Auto-levels spread after the exposure shift moves it, on encoded values.
- // A near-flat histogram (solid color, title card) carries no tonal-range
- // signal — leave contrast alone rather than stretch noise.
- const gain = Math.pow(2, ((grade.exposure / GRADE_MAX) * EXPOSURE_STOPS_AT_MAX) / 2.4);
- const lo = Math.min(255, percentile(0.005) * gain);
- const hi = Math.min(255, percentile(0.995) * gain);
+ const lo = percentile(0.005);
+ const hi = percentile(0.995);
if (hi - lo >= 16) {
const power = 255 / (hi - lo);
- grade.contrast = Math.max(
- -10,
- Math.min(40, Math.round(((power - 1) * 0.6 * GRADE_MAX) / CONTRAST_POWER_AT_MAX))
- );
- }
- if (mids) {
- const mean: [number, number, number] = [sumR / mids, sumG / mids, sumB / mids];
- // Grey-world on the temperature axis: the slider value whose white
- // balance brings red and blue to the same level.
- const imbalance = (t: number) => {
- const wb = whiteBalanceFor(t, 0, REC709_PRIMARIES);
- const c = wb ? mat3Apply(wb, mean[0], mean[1], mean[2]) : mean;
- return c[0] - c[2];
- };
- let best = 0;
- let bestErr = Math.abs(imbalance(0));
- for (let t = -GRADE_MAX; t <= GRADE_MAX; t++) {
- const err = Math.abs(imbalance(t));
- if (err < bestErr) {
- bestErr = err;
- best = t;
- }
- }
- grade.temperature = Math.max(-40, Math.min(40, Math.round(best * 0.6)));
+ const [min, max] = AUTO_CONTRAST_RANGE;
+ grade.contrast = Math.max(min, Math.min(max, Math.round(((power - 1) * AUTO_CONTRAST_SHARE * GRADE_MAX) / CONTRAST_POWER_AT_MAX)));
}
+
+ const toned = normalizeGrade(grade);
+ const balanced = sliderRoot((t) => midtoneCast(render({ ...toned, temperature: t })));
+ grade.temperature = Math.max(-AUTO_WB_MAX, Math.min(AUTO_WB_MAX, Math.round(balanced * AUTO_WB_SHARE)));
return normalizeGrade(grade);
}
@@ -184,101 +282,134 @@ export function autoGradeFromImageData(data: Uint8ClampedArray): ColorGrade | un
/* Reference matching */
/* ------------------------------------------------------------------ */
-/** The steepest a matched curve may climb, and (inverted) the flattest, per
- * segment. Quantiles of a reference with other content — a beige pavement
- * for a painted wall — can jump a hundred code values across a few, and a
- * curve that steep tears smooth gradients into flat bands of false colour;
- * one that flat crushes a channel while its neighbours keep climbing, so the
- * channels cross and highlights change hue. */
-export const MATCH_SLOPE_MAX = 2.5;
-
-function quantileCurve(src: number[], ref: number[]): CurvePoint[] | undefined {
- const pts: CurvePoint[] = [];
- const median = STAT_QUANTILES.indexOf(0.5);
- let anchor = 0;
- let lastX = -1;
- for (let i = 0; i < src.length; i++) {
- const x = Math.round(src[i]);
- const y = Math.round(ref[i]);
- if (x <= lastX) continue;
- if (i <= median) anchor = pts.length;
- pts.push([x, y]);
- lastX = x;
- }
- // Monotone outputs keep the compiled spline monotone.
- for (let i = 1; i < pts.length; i++) {
- if (pts[i][1] < pts[i - 1][1]) pts[i][1] = pts[i - 1][1];
- }
- // The median lands where the reference puts it; outward from it each
- // segment's slope stays inside [1/MATCH_SLOPE_MAX, MATCH_SLOPE_MAX].
- const clampY = (y: number) => Math.max(0, Math.min(255, Math.round(y)));
- for (let i = anchor + 1; i < pts.length; i++) {
- const dx = pts[i][0] - pts[i - 1][0];
- const y = pts[i - 1][1];
- pts[i][1] = clampY(Math.min(y + dx * MATCH_SLOPE_MAX, Math.max(y + dx / MATCH_SLOPE_MAX, pts[i][1])));
- }
- for (let i = anchor - 1; i >= 0; i--) {
- const dx = pts[i + 1][0] - pts[i][0];
- const y = pts[i + 1][1];
- pts[i][1] = clampY(Math.max(y - dx * MATCH_SLOPE_MAX, Math.min(y - dx / MATCH_SLOPE_MAX, pts[i][1])));
+/** The fields a match moves: exposure and contrast about grey, the two ends
+ * of the range, white balance and saturation — the corrections that bring
+ * one shot to another. Each is a slider the user reads afterward. */
+const MATCH_FIELDS = ["exposure", "contrast", "highlights", "shadows", "temperature", "tint", "saturation"] as const;
+
+/** Cost of a slider's distance from zero, in squared ΔL* units at full
+ * throw: when two settings fit equally, the smaller one wins, so a field the
+ * reference does not need stays at zero. */
+const MATCH_PULL = 4;
+
+/** Slider step of the finite differences the fit takes its slopes from. */
+const MATCH_PROBE = 2;
+
+/** The fit stops once no slider would move by more than this much. */
+const MATCH_SETTLED = 0.25;
+const MATCH_ITERATIONS = 40;
+
+/** The stats of a frame rendered with a candidate grade. */
+export type StatsRender = (grade: ColorGrade | undefined) => ColorStats | undefined;
+
+/** How far apart two frames read, as residuals in CIELAB units: the
+ * lightness distribution probe by probe, and each tonal band's cast and
+ * spread weighted by the share of the reference it holds — the moments a Lab
+ * color transfer matches, read per band so a split tone counts too. */
+function statsResiduals(a: ColorStats, b: ColorStats): number[] {
+ const out = a.lightness.map((l, i) => l - b.lightness[i]);
+ for (let i = 0; i < b.tones.length; i++) {
+ const p = a.tones[i];
+ const q = b.tones[i];
+ const w = Math.sqrt(TONE_BANDS.length * q.share);
+ out.push(w * (p.a - q.a), w * (p.b - q.b), w * (p.spreadA - q.spreadA), w * (p.spreadB - q.spreadB));
}
- if (pts.length < 2 || pts.every(([x, y]) => Math.abs(x - y) <= 2)) return undefined;
- // The spline holds flat outside its end points, so a curve that stops at the
- // p2/p98 probes would crush everything darker and brighter into those two
- // values. Carry the end segments' slopes out to 0 and 255 instead, keeping
- // the shadow and specular separation the reference has.
- const [x0, y0] = pts[0];
- const [x1, y1] = pts[1];
- const [xa, ya] = pts[pts.length - 2];
- const [xb, yb] = pts[pts.length - 1];
- if (xb < 255) {
- const slope = (yb - ya) / Math.max(1, xb - xa);
- pts.push([255, Math.round(Math.min(255, Math.max(yb, yb + slope * (255 - xb))))]);
+ return out;
+}
+
+/** Solve A x = y for a small dense system by Gaussian elimination with
+ * partial pivoting; null when A is singular. */
+function solve(A: number[][], y: number[]): number[] | null {
+ const n = y.length;
+ const m = A.map((row, i) => [...row, y[i]]);
+ for (let c = 0; c < n; c++) {
+ let pivot = c;
+ for (let r = c + 1; r < n; r++) if (Math.abs(m[r][c]) > Math.abs(m[pivot][c])) pivot = r;
+ if (Math.abs(m[pivot][c]) < 1e-12) return null;
+ [m[c], m[pivot]] = [m[pivot], m[c]];
+ for (let r = c + 1; r < n; r++) {
+ const f = m[r][c] / m[c][c];
+ for (let k = c; k <= n; k++) m[r][k] -= f * m[c][k];
+ }
}
- if (x0 > 0) {
- const slope = (y1 - y0) / Math.max(1, x1 - x0);
- pts.unshift([0, Math.round(Math.max(0, Math.min(y0, y0 - slope * x0)))]);
+ const x = new Array<number>(n).fill(0);
+ for (let r = n - 1; r >= 0; r--) {
+ let v = m[r][n];
+ for (let k = r + 1; k < n; k++) v -= m[r][k] * x[k];
+ x[r] = v / m[r][r];
}
- return pts;
+ return x;
}
-/** Move each curve point `strength` of the way from identity to where the
- * match put it. */
-function scaleCurve(pts: CurvePoint[] | undefined, strength: number): CurvePoint[] | undefined {
- if (!pts || strength >= 1) return pts;
- const out = pts.map(([x, y]) => [x, Math.round(x + (y - x) * strength)] as CurvePoint);
- return out.every(([x, y]) => Math.abs(x - y) <= 2) ? undefined : out;
+/** A StatsRender over a frame's code values (RGBA) and the recipe that
+ * draws them: recipeRender, measured. */
+export function recipeStatsRender(px: Uint8ClampedArray, recipe: ClipColorRecipe): StatsRender {
+ const render = recipeRender(px, recipe);
+ return (grade) => colorStatsFromImageData(render(grade));
}
/**
- * Compute the grade that moves `source` toward `reference`: per-channel
- * quantile curves, or a saturation delta when the tones already agree.
- * `strength` (0..1) takes the match part of the way — for a reference whose
- * content differs from the clip's (a bright window for a dark room), where
- * the full match lifts or crushes what the reference never had. Returns
- * undefined when the frames already agree.
+ * The grade that brings a frame to `reference`, fitted in the grade's own
+ * sliders: Levenberg–Marquardt least squares over MATCH_FIELDS, every try
+ * rendered through `render` and scored by statsResiduals in CIELAB, with a
+ * light pull toward zero on each slider. `strength` (0..1) scales the fitted
+ * sliders — for a reference whose content differs from the clip's (a bright
+ * window for a dark room), where the full match lifts or crushes what the
+ * reference never had. Returns undefined when the frames already agree.
*/
-export function matchGrade(source: ColorStats, reference: ColorStats, strength = 1): ColorGrade | undefined {
+export function matchGrade(render: StatsRender, reference: ColorStats, strength = 1): ColorGrade | undefined {
const k = Math.max(0, Math.min(1, strength));
if (k === 0) return undefined;
- const grade: ColorGrade = {};
- const r = scaleCurve(quantileCurve(source.r, reference.r), k);
- const g = scaleCurve(quantileCurve(source.g, reference.g), k);
- const b = scaleCurve(quantileCurve(source.b, reference.b), k);
- if (r || g || b) {
- grade.curves = {};
- if (r) grade.curves.r = r;
- if (g) grade.curves.g = g;
- if (b) grade.curves.b = b;
- }
- // Per-channel curves already stretch or squeeze chroma along with the
- // channels, so the saturation term is for frames that agree in tone and
- // differ in chroma alone.
- if (!grade.curves && source.meanSat > 0.01) {
- const delta = Math.round((reference.meanSat / source.meanSat - 1) * GRADE_MAX * k);
- if (Math.abs(delta) >= 2) {
- grade.saturation = Math.max(-GRADE_MAX, Math.min(GRADE_MAX, delta));
+ const gradeOf = (p: number[]): ColorGrade => Object.fromEntries(MATCH_FIELDS.map((f, i) => [f, p[i]]));
+ const pull = Math.sqrt(MATCH_PULL) / GRADE_MAX;
+ const residuals = (p: number[]): number[] | null => {
+ const stats = render(normalizeGrade(gradeOf(p)));
+ return stats ? [...statsResiduals(stats, reference), ...p.map((v) => pull * v)] : null;
+ };
+ const sumSq = (r: number[]) => r.reduce((a, v) => a + v * v, 0);
+ const clampSlider = (v: number) => Math.max(-GRADE_MAX, Math.min(GRADE_MAX, v));
+
+ // Levenberg–Marquardt: slopes by finite differences, a damped
+ // Gauss–Newton step, damping eased after a step that helps and raised
+ // after one that does not.
+ let p = MATCH_FIELDS.map(() => 0);
+ let r = residuals(p);
+ if (!r) return undefined;
+ let damping = 1e-2;
+ for (let it = 0; it < MATCH_ITERATIONS; it++) {
+ const J = p.map((v, j) => {
+ const q = [...p];
+ q[j] = v + MATCH_PROBE <= GRADE_MAX ? v + MATCH_PROBE : v - MATCH_PROBE;
+ const rq = residuals(q) ?? r!;
+ return rq.map((x, i) => (x - r![i]) / (q[j] - v));
+ });
+ const A = J.map((ci) => J.map((cj) => ci.reduce((a, x, i) => a + x * cj[i], 0)));
+ const g = J.map((ci) => ci.reduce((a, x, i) => a + x * r![i], 0));
+ let stepped = false;
+ let largest = 0;
+ while (damping < 1e6) {
+ const damped = A.map((row, i) => row.map((v, j) => (i === j ? v * (1 + damping) + 1e-9 : v)));
+ const delta = solve(damped, g.map((v) => -v));
+ if (!delta) break;
+ const next = p.map((v, i) => clampSlider(v + delta[i]));
+ largest = Math.max(...next.map((v, i) => Math.abs(v - p[i])));
+ const rn = residuals(next);
+ if (rn && sumSq(rn) < sumSq(r)) {
+ p = next;
+ r = rn;
+ damping = Math.max(1e-6, damping / 3);
+ stepped = true;
+ break;
+ }
+ damping *= 4;
}
+ if (!stepped || largest < MATCH_SETTLED) break;
}
- return normalizeGrade(grade);
+
+ const scaled: ColorGrade = {};
+ MATCH_FIELDS.forEach((f, i) => {
+ const v = Math.round(p[i] * k);
+ if (v) scaled[f] = v;
+ });
+ return normalizeGrade(scaled);
}
diff --git a/site/src/cut/components/ColorPanel.tsx b/site/src/cut/components/ColorPanel.tsx
index 16e820e5..549aa485 100644
--- a/site/src/cut/components/ColorPanel.tsx
+++ b/site/src/cut/components/ColorPanel.tsx
@@ -5,7 +5,7 @@ import { useQueryClient } from "@tanstack/react-query";
import { ChevronLeft, Loader2, RotateCcw, SlidersHorizontal, Trash2, Wand2, X } from "lucide-react";
import {
applyLutToImageData,
- autoGradeFromImageData,
+ autoGrade as fitAutoGrade,
buildClipLut,
GRADE_BASIC_FIELDS,
GRADE_DETAIL_FIELDS,
@@ -36,7 +36,7 @@ import {
} from "@donkeycut/effects-kit";
import { getPreviewCanvas } from "@/cut/lib/previewCanvas";
import { beginLutDraft, endLutDraft } from "@/cut/lib/lutBuild";
-import { sampleClipBaseFrameData, sourceProfileOf, toBaseRendering } from "@/cut/lib/baseFrame";
+import { sampleClipRawFrame, sourceProfileOf, toBaseRendering } from "@/cut/lib/baseFrame";
import { useEditor } from "@/cut/lib/store";
import { usePanelState, useRememberedScroll } from "@/cut/lib/panelState";
import type { MediaAsset, VideoClip } from "@/cut/lib/types";
@@ -381,7 +381,7 @@ function useLutTiles(): LutTile[] {
* already wears. */
function withLutToggled(grade: ColorGrade | undefined, lutId: string): ColorGrade {
const next: ColorGrade = { ...grade };
- if (grade?.lut?.id === lutId) delete next.lut;
+ if (lutIdOf(grade) === lutId) delete next.lut;
else next.lut = { id: lutId, amount: grade?.lut?.amount };
return next;
}
@@ -514,7 +514,7 @@ function PresetView({ clip, peers, onAdjust }: { clip: VideoClip; peers?: readon
frame={frame}
profile={profile}
selected={
- "lutId" in p ? lut?.id === p.lutId : "saved" in p ? gradeKey(p.grade) === currentKey : active?.id === p.id
+ "lutId" in p ? lutIdOf(clip.grade) === p.lutId : "saved" in p ? gradeKey(p.grade) === currentKey : active?.id === p.id
}
onPick={() => pick(p)}
onRemove={"saved" in p ? () => remove(p) : undefined}
@@ -575,11 +575,18 @@ function PresetView({ clip, peers, onAdjust }: { clip: VideoClip; peers?: readon
const presetLutCache = new Map<string, GradeLut | null>();
const presetThumbCache = new Map<string, string>();
-function presetLut(p: GradePreset, userLut: ParsedLut | undefined): GradeLut | null {
- const key = `${p.id}|${userLut ? "lut" : ""}`;
+/** Thumbs are 128px wide: a 17³ lattice draws them without visible error and
+ * builds in a few milliseconds even through a log clip's scene grade. */
+const THUMB_LUT_SIZE = 17;
+
+/** The preset through the clip's own recipe, from raw code values to the
+ * display: the source conversion, the grade (in scene light for log) and its
+ * LUT in one lattice. */
+function presetLut(p: GradePreset, profile: SourceProfile, userLut: ParsedLut | undefined): GradeLut | null {
+ const key = `${p.id}|${profile}|${userLut ? "lut" : ""}`;
let lut = presetLutCache.get(key);
if (lut === undefined) {
- lut = buildClipLut({ profile: "rec709", grade: p.grade, output: "sdr", size: 33 }, userLut);
+ lut = buildClipLut({ profile, grade: p.grade, output: "sdr", size: THUMB_LUT_SIZE }, userLut);
presetLutCache.set(key, lut);
}
return lut;
@@ -632,7 +639,7 @@ function PresetTile({
const img = new Image();
img.onload = () => {
if (gone) return;
- const lut = presetLut(preset, userLut);
+ const lut = presetLut(preset, profile, userLut);
const w = 128;
const h = Math.max(1, Math.round((img.naturalHeight / (img.naturalWidth || 1)) * w));
const c = document.createElement("canvas");
@@ -643,8 +650,8 @@ function PresetTile({
ctx.drawImage(img, 0, 0, w, h);
if (lut || converted) {
const px = ctx.getImageData(0, 0, w, h);
- if (converted) toBaseRendering(px.data, profile);
if (lut) applyLutToImageData(px.data, lut);
+ else toBaseRendering(px.data, profile);
ctx.putImageData(px, 0, 0);
}
const url = c.toDataURL("image/jpeg", 0.72);
@@ -747,13 +754,13 @@ function AdjustView({ clip, peers, onBack }: { clip: VideoClip; peers?: readonly
const grade = clip.grade;
// Manual adjustments only — reset-all keeps the preset layer.
const manualDirty = TOOLS.some((t) => toolDirty(grade, t.id));
- // Fit a starting grade from the clip's base rendering (the decoder frame
- // through the source conversion alone, never the graded preview, which
- // would fold the current grade back into the fit); the sliders show the
- // result and stay fully adjustable after.
+ // Fit a starting grade from the clip's code values rendered through its
+ // own recipe (never the graded preview, which would fold the current grade
+ // back into the fit); the sliders show the result and stay fully
+ // adjustable after.
const autoGrade = () => {
- const data = sampleClipBaseFrameData(clip.id);
- if (data) commit({ ...autoGradeFromImageData(data), preset: grade?.preset, lut: grade?.lut });
+ const raw = sampleClipRawFrame(clip.id);
+ if (raw) commit({ ...fitAutoGrade(raw.px, raw.recipe), preset: grade?.preset, lut: grade?.lut });
};
const resetAll = () => commit(grade?.preset ? { preset: grade.preset } : undefined);
return (
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index da8eadb6..c3d957db 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -479,7 +479,7 @@ export const INSPECTOR_TOOLS = [
{
name: "set_color_preset",
description:
- `Apply a named color preset to a video clip, layered under its manual grade. Presets by category — ${gradePresetCatalogText()}. amount 0..1 scales the preset toward neutral (default 1); protect_skin keeps its color shifts off skin tones. Pass preset "none" to clear. Preview, tiles, and export all render the same result.`,
+ `Apply a named color preset to a video clip, layered over its manual grade. Presets by category — ${gradePresetCatalogText()}. amount 0..1 scales the preset toward neutral (default 1); protect_skin keeps its color shifts off skin tones. Pass preset "none" to clear. Preview, tiles, and export all render the same result.`,
inputSchema: obj({
clipId: str("Video clip id"),
...manyClips,
@@ -600,14 +600,14 @@ export const INSPECTOR_TOOLS = [
{
name: "match_color_grade",
description:
- "Match a video clip's color to a reference, computed from the pixels: per-channel quantile mapping compiled into tone curves (each segment's slope bounded, so a reference of other content shifts the look without tearing gradients into bands) plus a saturation delta. The computed curves carry the whole look, so the match replaces the clip's entire grade, preset and LUT included. The reference is ref_asset_id (an attached or imported image) or ref_clip_id (another clip's frame). Use when the user shares a look to copy or asks to match two shots; refine afterward with the other grading tools, checking read_color_stats.",
+ "Match a video clip's color to a reference, computed from the pixels: the grade sliders for tone, the ends of the range, white balance and saturation are fitted so the clip's frame, rendered through its real color pipeline, lands on the reference's lightness distribution and per-band color in CIELAB. The fitted sliders carry the whole look, so the match replaces the clip's entire grade, preset and LUT included. The reference is ref_asset_id (an attached or imported image) or ref_clip_id (another clip's frame). Use when the user shares a look to copy or asks to match two shots; refine afterward with the other grading tools, checking read_color_stats.",
inputSchema: obj({
clipId: str("Video clip to grade"),
ref_asset_id: str("Reference image asset id"),
strength: num(
"0..1, default 1: how far to take the match. Lower it when the reference shows other content than the clip (a bright window for a dark room, a black foreground the clip lacks) and the full match lifts or crushes tones the reference never had"
),
- ref_clip_id: str("Reference video clip id (its base rendering: the frame on screen, or one decoded mid-trim when it is off screen)"),
+ ref_clip_id: str("Reference video clip id, read as it shows through its own grade: the frame on screen, or one decoded mid-trim when it is off screen"),
}, ["clipId"]),
},
] as const satisfies readonly AiToolDef[];
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 8e6e344b..f248ec57 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -31,15 +31,14 @@ import {
ALL_EFFECT_IDS,
CLIP_EFFECT_IDS,
type ClipEffect,
- applyLutToImageData,
- autoGradeFromImageData,
- buildClipLut,
+ autoGrade,
colorStatsFromImageData,
GRADE_BASIC_FIELDS,
GRADE_DETAIL_FIELDS,
GRADE_PRESETS,
HSL_BANDS,
matchGrade,
+ recipeStatsRender,
normalizeGrade,
SOURCE_PROFILES,
WHEEL_LABELS,
@@ -230,7 +229,7 @@ import { isGradePresetTemplate, listSavedGrades, saveGradePreset } from "./grade
import { isStylePresetTemplate } from "./stylePresets";
import { findLibraryAsset, listLibrary } from "./libraryList";
import { loadLibraryLut, lutIdOf, lutLabel } from "./linkedLibrary";
-import { baseFrameAt, sampleClipBaseFrameData, sourceProfileOf, toBaseRendering } from "./baseFrame";
+import { type RawFrame, type ReadGrade, rawFrameAt, sampleClipRawFrame, sourceProfileOf, toBaseRendering, toGradedRendering } from "./baseFrame";
import { applyOverlayPatchSettled, assetClipUses, assetIdsOnAnyTimeline, clipLen, footprints, nextFreeStart, track0Clips, laneGapAt, getClipSpans, overlayLaneOrder, overlayLayers, parkedTransitions, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";
import { playheadAt } from "./playhead";
import { mixSpecFor, renderProjectFrame, renderProjectFrames } from "./exportRender";
@@ -456,30 +455,52 @@ const GRADE_KEYS = [
* rendering (through the source conversion, before any grade — what Auto
* reads), or `graded` through the clip's LUT, preset and adjustments too, or
* an image asset's pixels. */
-/** A clip's frame decoded from its file at the middle of its trim, for a clip
- * the preview has no frame of — off screen, or not decoded yet. The decode
- * reads the base rendering, as the on-screen sample does. */
-async function offscreenClipFrameData(
+/** A clip's code values and recipe: the preview's decoded frame when it has
+ * one, else a decode of the file at the middle of the trim — off screen, or
+ * not decoded yet. */
+async function clipRawFrame(
s: ReturnType<typeof useEditor.getState>,
clip: VideoClip,
w: number,
h: number
-): Promise<Uint8ClampedArray | null> {
+): Promise<RawFrame> {
+ const shown = sampleClipRawFrame(clip.id, w, h);
+ if (shown) return shown;
const asset = s.assets.find((a) => a.id === clip.assetId);
- if (!asset?.url) return null;
+ if (!asset?.url) throw new ToolError("Could not decode a frame of that clip's file.");
// A still on the track has no frames to seek: draw the image itself.
if (asset.type === "image") {
const img = await decodeRasterImageUrl(asset.url);
- if (!img || !img.width) return null;
+ if (!img || !img.width) throw new ToolError("Could not decode a frame of that clip's file.");
const c = createRasterCanvas(w, h);
const ctx = c.getContext("2d") as CanvasRenderingContext2D | null;
- if (!ctx) return null;
+ if (!ctx) throw new ToolError("Could not decode a frame of that clip's file.");
ctx.drawImage(img.source, 0, 0, w, h);
- return toBaseRendering(ctx.getImageData(0, 0, w, h).data, sourceProfileOf(asset));
+ return { px: ctx.getImageData(0, 0, w, h).data, recipe: { profile: sourceProfileOf(asset), output: "sdr", size: 0 } };
}
- const frame = await baseFrameAt(asset.url, (clip.in + clip.out) / 2, { width: w, height: h, fit: "fill" });
- const ctx = frame?.canvas.getContext("2d") as CanvasRenderingContext2D | null | undefined;
- return ctx ? ctx.getImageData(0, 0, w, h).data : null;
+ const raw = await rawFrameAt(asset.url, (clip.in + clip.out) / 2, { width: w, height: h, fit: "fill" });
+ if (!raw) throw new ToolError("Could not decode a frame of that clip's file.");
+ return raw;
+}
+
+/** A clip's frame as the base rendering, or through its whole grade. */
+async function clipFrameData(
+ s: ReturnType<typeof useEditor.getState>,
+ clip: VideoClip,
+ w: number,
+ h: number,
+ graded?: ReadGrade
+): Promise<Uint8ClampedArray> {
+ const { px, recipe } = await clipRawFrame(s, clip, w, h);
+ return graded ? toGradedRendering(px, recipe.profile, recipe, graded) : toBaseRendering(px, recipe.profile, recipe);
+}
+
+/** The grade a clip's graded reads render through: its own, with the library
+ * LUT it names loaded. */
+async function readGradeOf(clip: VideoClip): Promise<ReadGrade | undefined> {
+ if (!clip.grade) return undefined;
+ const lutId = lutIdOf(clip.grade);
+ return { grade: clip.grade, userLut: lutId ? await loadLibraryLut(lutId) : undefined };
}
async function colorStatsForRef(
@@ -488,16 +509,9 @@ async function colorStatsForRef(
): Promise<ColorStats> {
if (ref.clipId) {
const clip = requireItem(s.clips, ref.clipId, "video clip");
- const data = sampleClipBaseFrameData(clip.id, 192, 108) ?? (await offscreenClipFrameData(s, clip, 192, 108));
- if (!data) throw new ToolError("Could not decode a frame of that clip's file.");
- if (ref.graded && clip.grade) {
- // The same recipe the Color panel's tiles draw with: the base rendering
- // through the clip's LUT, preset and adjustments.
- const lutId = lutIdOf(clip.grade);
- const userLut = lutId ? await loadLibraryLut(lutId) : undefined;
- const lut = buildClipLut({ profile: "rec709", grade: clip.grade, output: "sdr", size: 33 }, userLut);
- if (lut) applyLutToImageData(data, lut);
- }
+ // Graded reads run the clip's whole recipe on its code values, the way
+ // the preview draws it: a log clip's grade works in scene light.
+ const data = await clipFrameData(s, clip, 192, 108, ref.graded ? await readGradeOf(clip) : undefined);
const stats = colorStatsFromImageData(data);
if (!stats) throw new ToolError("Could not read pixels off that clip's frame.");
return stats;
@@ -5310,9 +5324,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
let grade: ColorGrade =
input.reset === true ? { preset: clip.grade?.preset, lut: clip.grade?.lut } : { ...clip.grade };
if (input.auto === true) {
- const data = sampleClipBaseFrameData(clip.id) ?? (await offscreenClipFrameData(s, clip, 96, 54));
- if (!data) throw new ToolError("Could not decode a frame of that clip's file.");
- grade = { ...autoGradeFromImageData(data), preset: clip.grade?.preset, lut: clip.grade?.lut };
+ const raw = await clipRawFrame(s, clip, 96, 54);
+ grade = { ...autoGrade(raw.px, raw.recipe), preset: clip.grade?.preset, lut: clip.grade?.lut };
}
for (const k of GRADE_KEYS) if (isNum(input[k])) grade[k] = input[k];
s.updateClip(clip.id, { grade: normalizeGrade(grade) });
@@ -5510,13 +5523,17 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
match_color_grade: async (s, input) => {
const clip = requireItem(s.clips, input.clipId, "video clip");
- const source = await colorStatsForRef(s, { clipId: clip.id });
+ // The reference reads as it shows — a clip through its own grade — and
+ // the fit renders this clip's code values through its real recipe for
+ // every grade it tries.
const reference = await colorStatsForRef(s, {
clipId: typeof input.ref_clip_id === "string" ? input.ref_clip_id : undefined,
assetId: typeof input.ref_asset_id === "string" ? input.ref_asset_id : undefined,
+ graded: true,
});
+ const raw = await clipRawFrame(s, clip, 192, 108);
const strength = isNum(input.strength) ? input.strength : 1;
- const grade = matchGrade(source, reference, strength);
+ const grade = matchGrade(recipeStatsRender(raw.px, raw.recipe), reference, strength);
// The frames already agree, or strength 0 asked for none of the match:
// the clip's own grade stays as it is.
if (!grade)
diff --git a/site/src/cut/lib/baseFrame.ts b/site/src/cut/lib/baseFrame.ts
index 937ea618..e66fabbe 100644
--- a/site/src/cut/lib/baseFrame.ts
+++ b/site/src/cut/lib/baseFrame.ts
@@ -3,10 +3,11 @@
/**
* A clip's frame as the base rendering sees it: the decoder's code values put
* through the source conversion alone — Apple Log, HLG or PQ into Rec.709
- * display values — and nothing of the grade. Analysis (Auto, the chat's
- * colour stats, a match), thumbnails, watch frames and freeze frames read
- * this, so a log clip's numbers and pictures describe what the grade works
- * on, never the flat log encoding. The frames are read the way the preview
+ * display values — and nothing of the grade. The chat's colour stats,
+ * thumbnails, watch frames and freeze frames read this, so a log clip's
+ * numbers and pictures describe what the display shows, never the flat log
+ * encoding; Auto and a match read the raw frame and render each grade they
+ * try through the clip's recipe. The frames are read the way the preview
* reads them (sourceColor.ts): tags overridden so the decoder draws code
* values, and the matrix it drew them through undone first.
*/
@@ -14,9 +15,12 @@
import {
applyLutToImageData,
buildClipLut,
+ type ClipColorRecipe,
+ type ColorGrade,
drawnColorKey,
type DrawnColor,
type GradeLut,
+ type ParsedLut,
type SourceProfile,
} from "@donkeycut/effects-kit";
import { frameSink, openMedia, videoTrackOf, type FrameCanvasSink, type FrameSize, type FrameSinkOptions } from "./mediaRead";
@@ -53,19 +57,48 @@ export function toBaseRendering(
return px;
}
-/** A small RGBA readout of a clip's base rendering; null when the clip has no
- * decoded frame ready. */
-export function sampleClipBaseFrameData(clipId: string, w = 96, h = 54): Uint8ClampedArray | null {
+/** A grade to read a frame through: the clip's grade and the library LUT it
+ * names, loaded. */
+export type ReadGrade = { grade: ColorGrade; userLut?: ParsedLut };
+
+/** Thumb-size readouts sample a 33³ lattice: quantiles and means sit within
+ * a code of the full-size preview. */
+const GRADED_READ_LUT_SIZE = 33;
+
+/** Convert raw code values in place into the picture the grade renders on an
+ * SDR project: the clip's whole recipe in one lattice, so a log clip's grade
+ * runs in scene light exactly as the preview runs it. */
+export function toGradedRendering(
+ px: Uint8ClampedArray,
+ profile: SourceProfile,
+ drawn: Partial<DrawnColor> | undefined,
+ read: ReadGrade
+): Uint8ClampedArray {
+ const lut = buildClipLut({ ...drawn, profile, output: "sdr", size: GRADED_READ_LUT_SIZE, grade: read.grade }, read.userLut);
+ if (!lut) return toBaseRendering(px, profile, drawn);
+ applyLutToImageData(px, lut);
+ return px;
+}
+
+/** A frame's code values (RGBA) and the ungraded recipe that draws them on
+ * an SDR project: what a match renders each grade it tries through. */
+export interface RawFrame {
+ px: Uint8ClampedArray;
+ recipe: ClipColorRecipe;
+}
+
+/** A small readout of a clip's code values as the preview decoded them; null
+ * when the clip has no decoded frame ready. */
+export function sampleClipRawFrame(clipId: string, w = 96, h = 54): RawFrame | null {
const px = sampleClipFrameData(clipId, w, h);
if (!px) return null;
const s = useEditor.getState();
const clip = s.clips.find((c) => c.id === clipId);
const asset = clip ? s.assets.find((a) => a.id === clip.assetId) : undefined;
- if (!asset) return toBaseRendering(px, "rec709");
- // The frame came through the engine's read of the clip; convert it the way
- // the compositor does.
- const input = (sampleClipColorRead(clipId) ?? colorRead(asset, previewFile(asset))).recipe();
- return toBaseRendering(px, input.profile, input);
+ // The frame came through the engine's read of the clip; its recipe is the
+ // one the compositor draws it with.
+ const input = asset ? (sampleClipColorRead(clipId) ?? colorRead(asset, previewFile(asset))).recipe() : undefined;
+ return { px, recipe: { ...input, profile: input?.profile ?? "rec709", output: "sdr", size: 0 } };
}
/** How the frames at `url` are to be read: the project asset the URL belongs
@@ -125,6 +158,28 @@ export async function baseFrameAt(url: string, time: number, size?: FrameSize):
}
}
+/** The code values of one frame of the file at `url`, read the way the
+ * preview reads them, with the recipe that draws them. */
+export async function rawFrameAt(url: string, time: number, size: FrameSize): Promise<RawFrame | null> {
+ const input = openMedia(url);
+ try {
+ const track = await videoTrackOf(input);
+ if (!track) throw new Error("This file has no readable video.");
+ const read = colorReadAt(url);
+ const colorSpace = read.colorSpace;
+ const frame = await frameSink(track, size, colorSpace ? { colorSpace } : {}).getCanvas(Math.max(0, time));
+ const ctx = frame?.canvas.getContext("2d") as CanvasRenderingContext2D | null | undefined;
+ if (!frame || !ctx) return null;
+ const drawn = await read.settled();
+ return {
+ px: ctx.getImageData(0, 0, frame.canvas.width, frame.canvas.height).data,
+ recipe: { ...drawn, output: "sdr", size: 0 },
+ };
+ } finally {
+ input.dispose();
+ }
+}
+
/** Base-rendering frames at ascending `times`, in one decode pass; null for a
* time the track has no frame for. */
export async function* baseFramesAt(url: string, times: number[], size?: FrameSize): AsyncGenerator<WrappedCanvas | null> {
diff --git a/site/src/cut/lib/colorOffscreen.test.ts b/site/src/cut/lib/colorOffscreen.test.ts
index 512d5657..8697cfe0 100644
--- a/site/src/cut/lib/colorOffscreen.test.ts
+++ b/site/src/cut/lib/colorOffscreen.test.ts
@@ -9,22 +9,17 @@ const decoded: { url: string; time: number; width?: number; height?: number }[]
// Nothing is on screen; the file decode answers a flat dark frame.
await stubModule<typeof import("./baseFrame")>("./baseFrame", import.meta.url, {
- sampleClipBaseFrameData: () => null,
- baseFrameAt: async (url, time, size) => {
- decoded.push({ url, time, width: size?.width, height: size?.height });
- const w = size?.width ?? 16;
- const h = size?.height ?? 9;
+ sampleClipRawFrame: () => null,
+ rawFrameAt: async (url, time, size) => {
+ decoded.push({ url, time, width: size.width, height: size.height });
+ const w = size.width ?? 16;
+ const h = size.height ?? 9;
// Each file is two greys 40 apart, pixel by pixel: "blob:one" dark
// (20 and 60), any other mid (100 and 140).
const base = url === "blob:one" ? 20 : 100;
const data = new Uint8ClampedArray(w * h * 4);
for (let j = 0; j < w * h; j++) data.fill(base + (j % 2) * 40, j * 4, j * 4 + 3), (data[j * 4 + 3] = 255);
- const canvas = {
- width: w,
- height: h,
- getContext: () => ({ getImageData: () => ({ data, width: w, height: h }) }),
- };
- return { canvas, timestamp: time, duration: 1 / 30 } as never;
+ return { px: data, recipe: { profile: "rec709", output: "sdr", size: 0 } };
},
});
@@ -75,14 +70,12 @@ test("match_color_grade strength takes the match part of the way", async () => {
});
await runAiTool("add_clip", { asset_id: "a2" });
const [dark, mid] = useEditor.getState().clips;
- const curveAt20 = () => {
- const g = useEditor.getState().clips.find((c) => c.id === dark.id)!.grade;
- return g?.curves?.g?.find(([x]) => x === 20)?.[1];
- };
+ const exposure = () => useEditor.getState().clips.find((c) => c.id === dark.id)!.grade?.exposure ?? 0;
await runAiTool("match_color_grade", { clipId: dark.id, ref_clip_id: mid.id });
- expect(curveAt20()).toBe(100);
+ const full = exposure();
+ expect(full).toBeGreaterThan(10);
await runAiTool("match_color_grade", { clipId: dark.id, ref_clip_id: mid.id, strength: 0.5 });
- expect(curveAt20()).toBe(60);
+ expect(Math.abs(exposure() - full / 2)).toBeLessThanOrEqual(1);
});
test("strength 0 says it took none of the match, not that the footage already matches", async () => {
diff --git a/site/src/cut/lib/colorTools.test.ts b/site/src/cut/lib/colorTools.test.ts
index 4c1cdae1..74752e24 100644
--- a/site/src/cut/lib/colorTools.test.ts
+++ b/site/src/cut/lib/colorTools.test.ts
@@ -5,6 +5,7 @@ import type { LibraryAsset, LibraryData } from "./library";
import type { SavedGrade } from "./gradePresets";
import type { MediaAsset } from "./types";
import { BUILTIN_LUTS } from "./builtinLuts";
+import { sourceProfileOf } from "./sourceColor";
// The colour tools: every slider the panel renders, four wheels, the LUT
// off the shelf, one grade copied across the timeline as one undo step, the
@@ -47,7 +48,12 @@ await stubModule<typeof import("./gradePresets")>("./gradePresets", import.meta.
// A flat mid-grey frame stands in for a decoded clip frame, so the stats
// tool reads known pixels.
await stubModule<typeof import("./baseFrame")>("./baseFrame", import.meta.url, {
- sampleClipBaseFrameData: (_clipId, w = 96, h = 54) => new Uint8ClampedArray(w * h * 4).fill(128),
+ sampleClipRawFrame: (clipId, w = 96, h = 54) => {
+ const s = useEditor.getState();
+ const asset = s.assets.find((a) => a.id === s.clips.find((c) => c.id === clipId)?.assetId);
+ const px = new Uint8ClampedArray(w * h * 4).fill(128);
+ return { px, recipe: { profile: asset ? sourceProfileOf(asset) : "rec709", output: "sdr", size: 0 } };
+ },
});
const { runAiTool } = await import("./aiTools");
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 7f53a265..c31569e8 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -279,13 +279,13 @@ Grade like a colorist: read the image, correct it, then style it — in that ord
**Read before you write, look after.** Call read_color_stats on the clip before grading (never grade blind): by default it reads the base rendering — the footage through its source conversion, before any LUT or grade. After a move, read it again with graded:true to confirm the numbers went where you meant, and capture_frame to look at the result — faces, whites, and the frame as a whole — before calling the grade done. The quantiles tell you where the tones sit: p2 well above 0 = lifted/milky blacks, p98 well below 255 = no real whites (flat), a p50 far from ~118 = exposure off. warmth > 1 reads warm, < 1 cool; meanSat under ~0.15 reads muted, over ~0.35 vivid. The clip's current color state is in editor_state (colorPreset, grade, lut, curves/wheels/hslBands); each asset's sourceColor says what its code values mean.
-**Source color first.** Log and HDR footage (${SOURCE_PROFILES.filter((p) => p.id !== "rec709" && p.id !== "srgb").map((p) => `${p.id} = ${p.label}`).join(", ")}) is detected from the file and converted to Rec.709 display values before anything else, so it grades like ordinary footage. When a clip reads flat and grey and its sourceColor is rec709, the header did not say — set_source_color with the right profile fixes the whole clip in one move. A camera maker's Log-to-Rec.709 LUT is the other route: set_source_color rec709 on that asset, then set_color_lut, so the picture is converted once.
+**Source color first.** Log and HDR footage (${SOURCE_PROFILES.filter((p) => p.id !== "rec709" && p.id !== "srgb").map((p) => `${p.id} = ${p.label}`).join(", ")}) is detected from the file. Log footage is graded in scene light and then rendered for the display, so exposure moves real stops and bright areas roll off smoothly; HDR footage is converted to display values first. When a clip reads flat and grey and its sourceColor is rec709, the header did not say — set_source_color with the right profile fixes the whole clip in one move. A camera maker's Log-to-Rec.709 LUT is the other route: set_source_color rec709 on that asset, then set_color_lut, so the picture is converted once.
**HDR delivery.** The project's color space (editor_state project.colorSpace; set_project_color: ${OUTPUT_SPACES.map((o) => o.id).join(", ")}) decides what the clips convert to. In an HDR project log and HDR footage keeps its range, SDR footage and every title or sticker sits at reference white (BT.2408, 203 nits), and the grade works in the HDR container; the export is 10-bit HEVC Main 10 or ProRes tagged Rec.2020 HLG or PQ. "Export in HDR" or "make this HLG/PQ" is set_project_color; the export then carries it.
**Correct next.** Balance and tone before any look: auto (set_color_grade auto:true) is a computed starting point — exposure to middle gray, contrast stretch, gray-world white balance — then nudge. Neutralize a cast with temperature/tint before styling warmth on top, or the style fights the cast. Every value 0 is neutral and fields patch, so small moves are cheap; centred sliders run ±${GRADE_MAX}, fade/sharpen/clarity 0..${GRADE_MAX}.
-**One model, layered.** A clip's color is one pipeline: the source conversion, then its LUT (scaled by amount), then the preset, then the manual adjustments on top. set_color_preset never touches the manual layer and vice versa; intensity (amount) scales only the preset; the LUT sits under both. Preview, thumbnails, and export render the same numbers.
+**One model, layered.** A clip's color is one pipeline: the source conversion, then its LUT (scaled by amount), then the manual adjustments that balance the shot, then the preset's look on top. On log footage the adjustments and preset run in scene light and the LUT on the rendered picture. set_color_preset never touches the manual layer and vice versa; intensity (amount) scales only the preset. Preview, thumbnails, and export render the same numbers.
**Which tool for which job.**
- set_color_lut — a film look from a LUT: the built-in ones (${builtinLutCatalogText()}) and the user's .cube/.3dl files, all in editor_state.luts; amount 0..1 blends it. "none" removes it. Pick by what the look does to the picture, and start around 0.7 on footage that is already graded or strongly colored.
@@ -294,7 +294,7 @@ Grade like a colorist: read the image, correct it, then style it — in that ord
- set_color_curves — tonal shape: curve_contrast for an s-curve, explicit points when you need a precise remap; red/green/blue channels split-tone tonally (a touch of blue lifted in shadows = cool shadows).
- set_color_wheels — the four wheels ${WHEEL_ZONES.map((z) => WHEEL_LABELS[z].toLowerCase()).join("/")}: lift for shadows, gamma for midtones, gain for highlights, offset for the whole picture; split-toning is a hue on lift and another on gain (teal shadows dx<0, warm highlights dx>0 is the classic pairing); luma trims each range's brightness.
- set_color_hsl — one hue only (${HSL_BANDS.map((b) => b.id).join(", ")}): "sky bluer" = aqua/blue sat up, "mute the greens" = green sat down, skin smoothing = orange band, gently.
-- match_color_grade — a reference exists (attached image or another clip): the grade is computed from the pixels (quantile-matched curves + saturation), then refine with the tools above. Matching adjacent clips for consistency is this too — match the odd clip to its neighbor.
+- match_color_grade — a reference exists (attached image or another clip): the grade sliders are fitted from the pixels through the clip's real pipeline, then refine with the tools above. Matching adjacent clips for consistency is this too — match the odd clip to its neighbor.
- copy_color_grade — "apply this look to everything": one clip's whole grade (LUT included) onto ids or all_clips, as one undo step. Every set_color_* tool also takes ids or all_clips for the same patch on several clips.
- save_color_grade / apply_saved_grade — keep a finished grade under a name in the Library (editor_state.savedGrades) and put it on clips in this or any project.