Skip to content

Open source CapCut alternative.

TypeScript

fix(site): fit color through the clip's own look [rebuild]

David Le4cf2af11e49f
diff --git a/site/packages/effects-kit/src/colorGrade.ts b/site/packages/effects-kit/src/colorGrade.ts
index b40d37f4..4201005d 100644
--- a/site/packages/effects-kit/src/colorGrade.ts
+++ b/site/packages/effects-kit/src/colorGrade.ts
@@ -37,7 +37,7 @@ export type HslBand =
* hueShift spans about ±30 degrees at full range. */
export type HslTuple = [number, number, number];
-/** A preset layered under the manual adjustments. `amount` 0..1 scales the
+/** A preset layered over the manual adjustments. `amount` 0..1 scales the
* preset toward neutral (default 1); `skin` keeps the preset's color shifts
* off skin tones. Unknown ids render as neutral, so docs can carry presets
* from newer catalogs without breaking. */
diff --git a/site/packages/effects-kit/src/gradeMatch.test.ts b/site/packages/effects-kit/src/gradeMatch.test.ts
index e9446aa9..b4aedb06 100644
--- a/site/packages/effects-kit/src/gradeMatch.test.ts
+++ b/site/packages/effects-kit/src/gradeMatch.test.ts
@@ -4,7 +4,8 @@ import { createGradeTransform } from "./gradeMath";
import { type Lab, deltaE2000, displayToLab } from "./colorDifference";
import { type ClipColorRecipe, createClipTransform } from "./colorPipeline";
import { type SourceProfile, appleLogEncode, bt1886Decode } from "./colorSpace";
-import { autoGrade, colorStatsFromImageData, matchGrade, recipeStatsRender } from "./gradeMatch";
+import { type StatsRender, autoGrade, colorStatsFromImageData, matchGrade, recipeStatsRender } from "./gradeMatch";
+import type { ParsedLut } from "./lutFile";
/** A synthetic frame with tonal and color variety. */
function testFrame(): Uint8ClampedArray {
@@ -39,9 +40,22 @@ const meanDeltaE = (a: Lab[], b: Lab[]) => a.reduce((s, p, i) => s + deltaE2000(
const recipeFor = (profile: SourceProfile): ClipColorRecipe => ({ profile, output: "sdr", size: 0 });
describe("matchGrade", () => {
- test("a frame matched to itself is neutral", () => {
+ test("a frame matched to itself is neutral", async () => {
const render = recipeStatsRender(testFrame(), recipeFor("rec709"));
- expect(matchGrade(render, render(undefined)!)).toBeUndefined();
+ expect(await matchGrade(render, render(undefined)!)).toBeUndefined();
+ });
+
+ test("a long fit hands the thread back while it runs", async () => {
+ const base = recipeStatsRender(testFrame(), recipeFor("rec709"));
+ // Every try costs a few milliseconds, as a log clip's does in the tab.
+ const slow: StatsRender = (g) => {
+ for (let i = 0; i < 8; i++) base(g);
+ return base(g);
+ };
+ const order: string[] = [];
+ setTimeout(() => order.push("tick"), 0);
+ await matchGrade(slow, base({ exposure: 30 })!).then(() => order.push("fit"));
+ expect(order).toEqual(["tick", "fit"]);
});
// The eval: a reference made by grading the clip's own frame with a known
@@ -57,26 +71,26 @@ describe("matchGrade", () => {
];
for (const profile of ["rec709", "apple-log-2"] as SourceProfile[]) {
for (const target of TARGETS) {
- test(`${profile}: recovers ${JSON.stringify(target)}`, () => {
+ test(`${profile}: recovers ${JSON.stringify(target)}`, async () => {
const px = testFrame();
const recipe = recipeFor(profile);
const render = recipeStatsRender(px, recipe);
- const grade = matchGrade(render, render(target)!);
+ const grade = await matchGrade(render, render(target)!);
expect(grade).toBeDefined();
expect(meanDeltaE(rendered(px, recipe, grade), rendered(px, recipe, target))).toBeLessThan(1);
});
}
}
- test("strength scales every fitted slider", () => {
+ test("strength scales every fitted slider", async () => {
const render = recipeStatsRender(testFrame(), recipeFor("rec709"));
const reference = render({ exposure: 16, temperature: 24 })!;
- const full = matchGrade(render, reference)!;
- const half = matchGrade(render, reference, 0.5)!;
+ const full = (await matchGrade(render, reference))!;
+ const half = (await matchGrade(render, reference, 0.5))!;
for (const [k, v] of Object.entries(full)) {
expect(Math.abs((half as Record<string, number>)[k] ?? 0) - Math.abs((v as number) / 2)).toBeLessThanOrEqual(0.5);
}
- expect(matchGrade(render, reference, 0)).toBeUndefined();
+ expect(await matchGrade(render, reference, 0)).toBeUndefined();
});
test("stats read lightness and each band's cast in CIELAB", () => {
@@ -117,7 +131,26 @@ function frame(...pixels: [number, number, number][]): Uint8ClampedArray {
const autoGrade709 = (px: Uint8ClampedArray) => autoGrade(px, recipeFor("rec709"));
+/** A two-node cube that halves every channel: a LUT that darkens. */
+function halvingLut(): ParsedLut {
+ const data = new Float32Array(8 * 3);
+ for (let i = 0; i < 8; i++) {
+ data[i * 3] = (i & 1) * 0.5;
+ data[i * 3 + 1] = ((i >> 1) & 1) * 0.5;
+ data[i * 3 + 2] = ((i >> 2) & 1) * 0.5;
+ }
+ return { cube: { size: 2, data, min: [0, 0, 0], max: [1, 1, 1] } };
+}
+
describe("autoGrade", () => {
+ test("the fit sees the picture through the clip's LUT and preset", () => {
+ const px = frame([125, 125, 125]);
+ const recipe: ClipColorRecipe = { ...recipeFor("rec709"), grade: { lut: { id: "dark" } } };
+ const bare = autoGrade709(px)?.exposure ?? 0;
+ const through = autoGrade(px, recipe, halvingLut())?.exposure ?? 0;
+ expect(through).toBeGreaterThan(bare + 10);
+ });
+
test("a mid-gray frame needs nearly nothing", () => {
const g = autoGrade709(frame([125, 125, 125]));
expect(Math.abs(g?.exposure ?? 0) <= 1).toBe(true);
diff --git a/site/packages/effects-kit/src/gradeMatch.ts b/site/packages/effects-kit/src/gradeMatch.ts
index 6a17417b..85877e56 100644
--- a/site/packages/effects-kit/src/gradeMatch.ts
+++ b/site/packages/effects-kit/src/gradeMatch.ts
@@ -14,6 +14,7 @@ import { displayToLab } from "./colorDifference";
import { type ClipColorRecipe, createClipTransform } from "./colorPipeline";
import { bt1886Decode } from "./colorSpace";
import { CONTRAST_POWER_AT_MAX } from "./gradeMath";
+import type { ParsedLut } from "./lutFile";
/** Quantile probes used by stats and matching. */
export const STAT_QUANTILES = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98] as const;
@@ -146,10 +147,13 @@ type PixelRender = (grade: ColorGrade | undefined) => Uint8ClampedArray;
const FIT_SAMPLE = 4096;
/** A PixelRender over a frame's code values (RGBA) and the recipe that draws
- * them, with the candidate grade in the recipe's place: every try runs the
+ * them, with the candidate grade as the manual layer: every try runs the
* real pipeline, so a log clip is fitted in scene light exactly as the
- * preview renders it. At most FIT_SAMPLE pixels, evenly spaced. */
-function recipeRender(px: Uint8ClampedArray, recipe: ClipColorRecipe): PixelRender {
+ * preview renders it. The recipe grade's preset and LUT (`userLut` its
+ * table) stay under every try, as they stay on the clip. At most FIT_SAMPLE
+ * pixels, evenly spaced. */
+function recipeRender(px: Uint8ClampedArray, recipe: ClipColorRecipe, userLut?: ParsedLut): PixelRender {
+ const { preset, lut } = recipe.grade ?? {};
const total = px.length >> 2;
const stride = Math.max(1, Math.ceil(total / FIT_SAMPLE));
const sample = new Float32Array(Math.ceil(total / stride) * 3);
@@ -161,7 +165,7 @@ function recipeRender(px: Uint8ClampedArray, recipe: ClipColorRecipe): PixelRend
n++;
}
return (grade) => {
- const fn = createClipTransform({ ...recipe, grade });
+ const fn = createClipTransform({ ...recipe, grade: { ...grade, ...(preset ? { preset } : {}), ...(lut ? { lut } : {}) } }, userLut);
const out = new Uint8ClampedArray(n * 4);
for (let i = 0; i < n; i++) {
const r = sample[i * 3];
@@ -239,11 +243,12 @@ const AUTO_WB_MAX = 40;
* 3. White balance — grey-world: the temperature whose render brings the
* midtones' red and blue means together, applied in part.
*
- * Saturation and hue are left alone.
+ * Saturation and hue are left alone. The recipe's grade carries the preset
+ * and LUT the clip keeps, so the fit lands on the picture the clip shows.
*/
-export function autoGrade(px: Uint8ClampedArray, recipe: ClipColorRecipe): ColorGrade | undefined {
+export function autoGrade(px: Uint8ClampedArray, recipe: ClipColorRecipe, userLut?: ParsedLut): ColorGrade | undefined {
if (px.length < 4) return undefined;
- const render = recipeRender(px, recipe);
+ const render = recipeRender(px, recipe, userLut);
const grade: ColorGrade = {};
const start = Math.log(logAverage(render(undefined)));
@@ -299,6 +304,28 @@ const MATCH_PROBE = 2;
const MATCH_SETTLED = 0.25;
const MATCH_ITERATIONS = 40;
+/** Longest stretch the fit works before it hands the thread back, so a match
+ * in the tab never holds a preview frame. */
+const FIT_SLICE_MS = 8;
+
+/** Let the thread run other tasks. Headless runtimes have setImmediate,
+ * which lets timers and I/O run first; a page posts a message, which runs
+ * like a zero timer without the clamping a background tab applies to timers. */
+function nextTask(): Promise<void> {
+ return new Promise((resolve) => {
+ if (typeof setImmediate === "function") {
+ setImmediate(resolve);
+ return;
+ }
+ const ch = new MessageChannel();
+ ch.port1.onmessage = () => {
+ ch.port1.close();
+ resolve();
+ };
+ ch.port2.postMessage(null);
+ });
+}
+
/** The stats of a frame rendered with a candidate grade. */
export type StatsRender = (grade: ColorGrade | undefined) => ColorStats | undefined;
@@ -356,13 +383,19 @@ export function recipeStatsRender(px: Uint8ClampedArray, recipe: ClipColorRecipe
* sliders — for a reference whose content differs from the clip's (a bright
* window for a dark room), where the full match lifts or crushes what the
* reference never had. Returns undefined when the frames already agree.
+ * The fit runs in slices of FIT_SLICE_MS between other tasks.
*/
-export function matchGrade(render: StatsRender, reference: ColorStats, strength = 1): ColorGrade | undefined {
+export async function matchGrade(render: StatsRender, reference: ColorStats, strength = 1): Promise<ColorGrade | undefined> {
const k = Math.max(0, Math.min(1, strength));
if (k === 0) return undefined;
const gradeOf = (p: number[]): ColorGrade => Object.fromEntries(MATCH_FIELDS.map((f, i) => [f, p[i]]));
const pull = Math.sqrt(MATCH_PULL) / GRADE_MAX;
- const residuals = (p: number[]): number[] | null => {
+ let sliceStart = performance.now();
+ const residuals = async (p: number[]): Promise<number[] | null> => {
+ if (performance.now() - sliceStart > FIT_SLICE_MS) {
+ await nextTask();
+ sliceStart = performance.now();
+ }
const stats = render(normalizeGrade(gradeOf(p)));
return stats ? [...statsResiduals(stats, reference), ...p.map((v) => pull * v)] : null;
};
@@ -373,16 +406,17 @@ export function matchGrade(render: StatsRender, reference: ColorStats, strength
// Gauss–Newton step, damping eased after a step that helps and raised
// after one that does not.
let p = MATCH_FIELDS.map(() => 0);
- let r = residuals(p);
+ let r = await residuals(p);
if (!r) return undefined;
let damping = 1e-2;
for (let it = 0; it < MATCH_ITERATIONS; it++) {
- const J = p.map((v, j) => {
+ const J: number[][] = [];
+ for (let j = 0; j < p.length; j++) {
const q = [...p];
- q[j] = v + MATCH_PROBE <= GRADE_MAX ? v + MATCH_PROBE : v - MATCH_PROBE;
- const rq = residuals(q) ?? r!;
- return rq.map((x, i) => (x - r![i]) / (q[j] - v));
- });
+ q[j] = p[j] + MATCH_PROBE <= GRADE_MAX ? p[j] + MATCH_PROBE : p[j] - MATCH_PROBE;
+ const rq = (await residuals(q)) ?? r;
+ J.push(rq.map((x, i) => (x - r![i]) / (q[j] - p[j])));
+ }
const A = J.map((ci) => J.map((cj) => ci.reduce((a, x, i) => a + x * cj[i], 0)));
const g = J.map((ci) => ci.reduce((a, x, i) => a + x * r![i], 0));
let stepped = false;
@@ -393,7 +427,7 @@ export function matchGrade(render: StatsRender, reference: ColorStats, strength
if (!delta) break;
const next = p.map((v, i) => clampSlider(v + delta[i]));
largest = Math.max(...next.map((v, i) => Math.abs(v - p[i])));
- const rn = residuals(next);
+ const rn = await residuals(next);
if (rn && sumSq(rn) < sumSq(r)) {
p = next;
r = rn;
diff --git a/site/src/cut/components/ColorPanel.tsx b/site/src/cut/components/ColorPanel.tsx
index 549aa485..adf1c45c 100644
--- a/site/src/cut/components/ColorPanel.tsx
+++ b/site/src/cut/components/ColorPanel.tsx
@@ -758,9 +758,15 @@ function AdjustView({ clip, peers, onBack }: { clip: VideoClip; peers?: readonly
// own recipe (never the graded preview, which would fold the current grade
// back into the fit); the sliders show the result and stay fully
// adjustable after.
- const autoGrade = () => {
+ const autoGrade = async () => {
const raw = sampleClipRawFrame(clip.id);
- if (raw) commit({ ...fitAutoGrade(raw.px, raw.recipe), preset: grade?.preset, lut: grade?.lut });
+ if (!raw) {
+ return;
+ }
+ const kept: ColorGrade = { preset: grade?.preset, lut: grade?.lut };
+ const lutId = lutIdOf(kept);
+ const userLut = lutId ? await loadLibraryLut(lutId) : undefined;
+ commit({ ...fitAutoGrade(raw.px, { ...raw.recipe, grade: kept }, userLut), ...kept });
};
const resetAll = () => commit(grade?.preset ? { preset: grade.preset } : undefined);
return (
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index c3d957db..ff263756 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -465,7 +465,7 @@ export const INSPECTOR_TOOLS = [
{
name: "set_color_grade",
description:
- `Color-grade a video clip's basic sliders (any track; stills too): the Light group (${GRADE_BASIC_FIELDS.filter((f) => f.group === "light").map((f) => f.key).join(", ")}), the Color group (${GRADE_BASIC_FIELDS.filter((f) => f.group === "color").map((f) => f.key).join(", ")}) and the Detail pair (${GRADE_DETAIL_FIELDS.map((f) => f.key).join(", ")}). Fields patch the clip's current grade: only the ones you pass change, and every value 0 is neutral. The light sliders work in linear light on the picture after its source conversion and LUT; manual adjustments layer OVER the clip's color preset (set_color_preset), never replacing it. reset:true clears the manual grade first (the preset and LUT stay); auto:true fits a starting grade from the clip's base rendering (auto-tone: exposure to middle gray, contrast stretch, gray-world white balance), and explicit fields then override it. Preview, timeline thumbnails, and export all render the same result. Read read_color_stats before grading so the numbers ground the move. Pass ids or all_clips to land the same patch on several clips as one undo step.`,
+ `Color-grade a video clip's basic sliders (any track; stills too): the Light group (${GRADE_BASIC_FIELDS.filter((f) => f.group === "light").map((f) => f.key).join(", ")}), the Color group (${GRADE_BASIC_FIELDS.filter((f) => f.group === "color").map((f) => f.key).join(", ")}) and the Detail pair (${GRADE_DETAIL_FIELDS.map((f) => f.key).join(", ")}). Fields patch the clip's current grade: only the ones you pass change, and every value 0 is neutral. The light sliders work in linear light: on the picture after its source conversion and LUT, and on log footage on the camera's scene light, ahead of the ACES output transform and the LUT. The manual grade balances the shot first and the clip's color preset (set_color_preset) styles the corrected picture on top; both stay. reset:true clears the manual grade first (the preset and LUT stay); auto:true fits a starting grade from the clip's base rendering (auto-tone: exposure to middle gray, contrast stretch, gray-world white balance), and explicit fields then override it. Preview, timeline thumbnails, and export all render the same result. Read read_color_stats before grading so the numbers ground the move. Pass ids or all_clips to land the same patch on several clips as one undo step.`,
inputSchema: obj({
clipId: str("Video clip id"),
...manyClips,
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 783c1dbd..dea5bb23 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -299,6 +299,7 @@ import {
DEFAULT_BACKGROUND,
clampKeyPos,
clampOverlayPos,
+ previewUrl,
clipZoom,
frameOf,
IMAGE_CLIP_SECONDS,
@@ -479,7 +480,9 @@ async function clipRawFrame(
ctx.drawImage(img.source, 0, 0, w, h);
return { px: ctx.getImageData(0, 0, w, h).data, recipe: { profile: sourceProfileOf(asset), output: "sdr", size: 0 } };
}
- const raw = await rawFrameAt(asset.url, (clip.in + clip.out) / 2, { width: w, height: h, fit: "fill" });
+ // The file the cut plays: a light proxy reads faster than its master, and
+ // carries its own color recipe.
+ const raw = await rawFrameAt(previewUrl(asset), (clip.in + clip.out) / 2, { width: w, height: h, fit: "fill" });
if (!raw) throw new ToolError("Could not decode a frame of that clip's file.");
return raw;
}
@@ -5326,7 +5329,10 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
input.reset === true ? { preset: clip.grade?.preset, lut: clip.grade?.lut } : { ...clip.grade };
if (input.auto === true) {
const raw = await clipRawFrame(s, clip, 96, 54);
- grade = { ...autoGrade(raw.px, raw.recipe), preset: clip.grade?.preset, lut: clip.grade?.lut };
+ const kept: ColorGrade = { preset: clip.grade?.preset, lut: clip.grade?.lut };
+ const lutId = lutIdOf(kept);
+ const userLut = lutId ? await loadLibraryLut(lutId) : undefined;
+ grade = { ...autoGrade(raw.px, { ...raw.recipe, grade: kept }, userLut), ...kept };
}
for (const k of GRADE_KEYS) if (isNum(input[k])) grade[k] = input[k];
s.updateClip(clip.id, { grade: normalizeGrade(grade) });
@@ -5534,7 +5540,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
});
const raw = await clipRawFrame(s, clip, 192, 108);
const strength = isNum(input.strength) ? input.strength : 1;
- const grade = matchGrade(recipeStatsRender(raw.px, raw.recipe), reference, strength);
+ const grade = await matchGrade(recipeStatsRender(raw.px, raw.recipe), reference, strength);
// The frames already agree, or strength 0 asked for none of the match:
// the clip's own grade stays as it is.
if (!grade)