Open source CapCut alternative.
TypeScript
feat(site): read a striped fill's lines with sample_color
David Le921c91aabf24
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex ad3f8b50..2a1ea061 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -274,7 +274,7 @@ export const AI_PANEL_TOOLS = [{name: "sample_color",description:- `Read the colour of a source's pixels: the hex of a region or a point of a frame at a source second, so a block, a shape, a title or a background takes the reference's colour as a number. Contact-sheet cells are JPEGs and shift colour; this reads the decoded frame as the preview shows it. Each sample is a moment \`t\` (SOURCE seconds) and a box — x/y its centre and w/h its size as frame fractions, the box add_shape takes; leave out w/h to read a point. Returns per sample \`hex\`, the region's median colour, and \`spread\`, how far its pixels stray from it on average (0-255: under 10 is a flat fill, over 40 a mix — read \`dominant\` there). \`colors\` 2-${SAMPLE_COLORS_K_MAX} adds \`dominant\`, the colours the region is made of with their shares, largest first: a card and the type on it, a bar and the shot behind it. Read a point inside a flat fill for its colour, a box over a title for its ink and its ground; the ink's share tells a filled word (a quarter of its box or more) from an outlined one, so a word that swaps between the two is read shot by shot. Up to ${SAMPLE_COLOR_MAX} samples a call, at one moment or each at its own. Works on a still image too (t is ignored).`,+ `Read the colour of a source's pixels: the hex of a region or a point of a frame at a source second, so a block, a shape, a title or a background takes the reference's colour as a number. Contact-sheet cells are JPEGs and shift colour; this reads the decoded frame as the preview shows it. Each sample is a moment \`t\` (SOURCE seconds) and a box — x/y its centre and w/h its size as frame fractions, the box add_shape takes; leave out w/h to read a point. Returns per sample \`hex\`, the region's median colour, and \`spread\`, how far its pixels stray from it on average (0-255: under 10 is a flat fill, over 40 a mix — read \`dominant\` there). \`colors\` 2-${SAMPLE_COLORS_K_MAX} adds \`dominant\`, the colours the region is made of with their shares, largest first: a card and the type on it, a bar and the shot behind it. Read a point inside a flat fill for its colour, a box over a title for its ink and its ground; the ink's share tells a filled word (a quarter of its box or more) from an outlined one, so a word that swaps between the two is read shot by shot. A box over a fill laid in fine parallel lines — a striped card, a hatched bar — adds \`stripes\`: the lines' \`angle\` (clockwise from vertical), \`width\` and \`gap\` in px at 1080, and the \`line\` and \`ground\` colours, exactly what add_shape's stripe pattern takes; read a box well inside the fill, a few dozen lines across. Up to ${SAMPLE_COLOR_MAX} samples a call, at one moment or each at its own. Works on a still image too (t is ignored).`,inputSchema: obj({asset_id: str("Project asset id (video or image)"),diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex aba91f48..9eeb7c10 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -2036,6 +2036,12 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {...(smp.point ? {} : { w: rF(smp.box.w), h: rF(smp.box.h) }),...read[i],})),+ ...(read.some((r) => r.stripes)+ ? {+ stripesNote:+ "A sample with `stripes` is laid in parallel lines: lay a block in its ground colour, then the same box over it as add_shape with fill the line colour, pattern stripes and pattern_width, pattern_gap and pattern_angle as they are.",+ }+ : {}),...pictureReport(area),source: sourceRef(asset),};diff --git a/site/src/cut/lib/measure/color.test.ts b/site/src/cut/lib/measure/color.test.tsindex 176c5a82..0fdb20f4 100644--- a/site/src/cut/lib/measure/color.test.ts+++ b/site/src/cut/lib/measure/color.test.ts@@ -1,5 +1,5 @@import { expect, test } from "bun:test";-import { hexOf, parseHex, regionColor } from "./color";+import { hexOf, parseHex, regionColor, regionStripes } from "./color";/** A w×h RGBA frame painted by `paint(x, y)`. */const frame = (w: number, h: number, paint: (x: number, y: number) => [number, number, number]) => {@@ -41,3 +41,42 @@ test("a region running off the frame is clipped to it", () => {const out = regionColor(px, { x: 8, y: 8, w: 10, h: 10 });expect(out.hex).toBe("#000000");});++/** Lines `width` px thick every `period` px, their normal `deg` clockwise from+ * the x axis, so the lines themselves run `deg` clockwise from vertical. */+const striped = (deg: number, period: number, width: number, line: [number, number, number], ground: [number, number, number]) => {+ const c = Math.cos((deg * Math.PI) / 180);+ const s = Math.sin((deg * Math.PI) / 180);+ return frame(200, 200, (x, y) => ((((x * c + y * s) % period) + period) % period < width ? line : ground));+};++test("a striped region reads its lines: angle, thickness, gap and both colours", () => {+ const px = striped(-45, 10, 4, [70, 190, 215], [50, 175, 200]);+ const out = regionStripes(px, { x: 20, y: 20, w: 160, h: 160 }, 2);+ expect(out).not.toBeNull();+ expect(out!.angle).toBe(-45);+ // Thickness and gap in px at 1080: two design px per pixel here.+ expect(out!.width).toBeCloseTo(8, 0);+ expect(out!.gap).toBeCloseTo(12, 0);+ expect(out!.line).toBe(hexOf(70, 190, 215));+ expect(out!.ground).toBe(hexOf(50, 175, 200));+});++test("upright and tilted stripes read their direction the way a shape's pattern takes it", () => {+ expect(regionStripes(striped(0, 12, 4, [255, 255, 255], [0, 0, 0]), { x: 0, y: 0, w: 200, h: 200 }, 1)!.angle).toBe(0);+ expect(regionStripes(striped(30, 12, 4, [255, 255, 255], [0, 0, 0]), { x: 0, y: 0, w: 200, h: 200 }, 1)!.angle).toBe(30);+ expect(regionStripes(striped(90, 12, 4, [255, 255, 255], [0, 0, 0]), { x: 0, y: 0, w: 200, h: 200 }, 1)!.angle).toBe(90);+});++test("a flat fill, a split card and a little noise read no stripes", () => {+ expect(regionStripes(frame(80, 80, () => [56, 180, 202]), { x: 0, y: 0, w: 80, h: 80 }, 1)).toBeNull();+ const split = frame(80, 80, (x) => (x < 50 ? [56, 180, 202] : [20, 70, 80]));+ expect(regionStripes(split, { x: 0, y: 0, w: 80, h: 80 }, 1)).toBeNull();+ let seed = 7;+ const grain = frame(80, 80, () => {+ seed = (seed * 16807) % 2147483647;+ const d = (seed % 9) - 4;+ return [56 + d, 180 + d, 202 + d];+ });+ expect(regionStripes(grain, { x: 0, y: 0, w: 80, h: 80 }, 1)).toBeNull();+});diff --git a/site/src/cut/lib/measure/color.ts b/site/src/cut/lib/measure/color.tsindex 777c07ee..127b229b 100644--- a/site/src/cut/lib/measure/color.ts+++ b/site/src/cut/lib/measure/color.ts@@ -1,5 +1,5 @@/** Colour reads of a region of a frame: its median colour, how far the region- * strays from it, and the few colours it is made of. Pure pixel math on RGBA+ * strays from it, the few colours it is made of, and the stripes it is laid in. Pure pixel math on RGBA* data; sourceMeasure.ts hands it decoded frames. *//** An RGBA frame, as ImageData carries it. */@@ -26,6 +26,27 @@ const CLUSTER_ROUNDS = 10;/** Dominant colours below this share of the region are noise: edges, grain. */const MIN_SHARE = 0.01;+/** Stripes are read off the middle of a region at most this many pixels a+ * side: a few dozen lines, and the angle and period searches stay cheap. */+const STRIPE_SIDE = 160;+/** Bin width along the stripes' normal for the angle search, pixels. */+const STRIPE_BIN = 0.5;+/** The fewest whole periods the read must hold, so the longest period read is+ * a third of the side. */+const STRIPE_PERIODS = 3;+/** Period steps: fine where a step of 0.02 px still moves the fold, then a+ * fixed share of the period. */+const STRIPE_STEP = 0.02;+const STRIPE_STEP_SHARE = 0.005;+/** Phase bins a period folds into. */+const STRIPE_PHASES = 16;+/** Share of the region's luminance variance the folded period must explain:+ * stripes are most of what varies; type, an edge or grain isn't. */+const STRIPE_SHARE = 0.35;+/** Least luminance swing between line and gap, 0..255 — under it the lines+ * are too faint to lay. */+const STRIPE_CONTRAST = 4;+const hex2 = (v: number) => Math.round(Math.max(0, Math.min(255, v))).toString(16).padStart(2, "0").toUpperCase();export const hexOf = (r: number, g: number, b: number): string => `#${hex2(r)}${hex2(g)}${hex2(b)}`;@@ -184,3 +205,183 @@ function dominantColors(px: Pixels, r: PixelRect, k: number): { hex: string; sha.filter((c) => c.share >= MIN_SHARE).sort((a, b) => b.share - a.share);}++/** Parallel lines a region is laid in, as add_shape's stripe pattern takes+ * them: `angle` the lines' direction in degrees clockwise from vertical,+ * -90..90; `width` and `gap` in px at 1080; `line` and `ground` their colours. */+export interface Stripes {+ angle: number;+ width: number;+ gap: number;+ line: string;+ ground: string;+}++/** The stripes a region is laid in, or null when it isn't. `unit` is design+ * px at 1080 per frame pixel. The angle is the one whose profile across the+ * lines swings widest; the period the shortest that folds the pixels onto+ * one line about as well as the best does (twice the period folds just as+ * well); the line is whichever of the brighter and darker phases covers+ * less of the period. */+export function regionStripes(px: Pixels, rect: PixelRect, unit: number): Stripes | null {+ const r = clipRect(px, rect);+ const w = Math.min(STRIPE_SIDE, r.w);+ const h = Math.min(STRIPE_SIDE, r.h);+ if (Math.min(w, h) < 4 * STRIPE_PERIODS) return null;+ const x0 = r.x + Math.floor((r.w - w) / 2);+ const y0 = r.y + Math.floor((r.h - h) / 2);+ const n = w * h;+ const lum = new Float32Array(n);+ let mean = 0;+ for (let y = 0; y < h; y++) {+ for (let x = 0; x < w; x++) {+ const i = ((y0 + y) * px.width + x0 + x) * 4;+ const v = 0.2126 * px.data[i] + 0.7152 * px.data[i + 1] + 0.0722 * px.data[i + 2];+ lum[y * w + x] = v;+ mean += v;+ }+ }+ mean /= n;+ let pixVar = 0;+ for (let k = 0; k < n; k++) {+ lum[k] -= mean;+ pixVar += lum[k] * lum[k];+ }+ pixVar /= n;+ // A square wave of the least contrast varies by a quarter of its swing squared.+ if (pixVar < (STRIPE_CONTRAST * STRIPE_CONTRAST) / 4) return null;++ // The angle: the normal whose binned profile varies most.+ const reach = Math.ceil(Math.hypot(w, h) / STRIPE_BIN) + 2;+ const sum = new Float64Array(reach * 2);+ const cnt = new Uint32Array(reach * 2);+ const profileVar = (deg: number) => {+ const c = Math.cos((deg * Math.PI) / 180) / STRIPE_BIN;+ const s = Math.sin((deg * Math.PI) / 180) / STRIPE_BIN;+ sum.fill(0);+ cnt.fill(0);+ for (let y = 0; y < h; y++) {+ for (let x = 0; x < w; x++) {+ const b = Math.floor(x * c + y * s) + reach;+ sum[b] += lum[y * w + x];+ cnt[b]++;+ }+ }+ let v = 0;+ let bins = 0;+ for (let b = 0; b < reach * 2; b++) {+ if (cnt[b] < Math.min(w, h) / 2) continue;+ v += (sum[b] / cnt[b]) ** 2;+ bins++;+ }+ return bins ? v / bins : 0;+ };+ let theta = 0;+ let best = -1;+ for (let d = -90; d < 90; d++) {+ const v = profileVar(d);+ if (v > best) [theta, best] = [d, v];+ }+ for (let d = theta - 1; d <= theta + 1; d += 0.25) {+ const v = profileVar(d);+ if (v > best) [theta, best] = [d, v];+ }++ // The period: fold each pixel by its place along the normal.+ const c0 = Math.cos((theta * Math.PI) / 180);+ const s0 = Math.sin((theta * Math.PI) / 180);+ const along = new Float32Array(n);+ for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) along[y * w + x] = x * c0 + y * s0;+ const phSum = new Float64Array(STRIPE_PHASES);+ const phCnt = new Uint32Array(STRIPE_PHASES);+ const fold = (period: number) => {+ phSum.fill(0);+ phCnt.fill(0);+ for (let k = 0; k < n; k++) {+ const q = along[k] / period;+ const b = Math.min(STRIPE_PHASES - 1, Math.floor((q - Math.floor(q)) * STRIPE_PHASES));+ phSum[b] += lum[k];+ phCnt[b]++;+ }+ };+ // A fold's swing, kept only as far as neighbouring phases agree: on a pixel+ // grid a period's fraction (10 px folded at 10/4.5) sorts the same pixels+ // into the same bins out of order and swings just as wide, but only the+ // true period lays one line in one run of phases.+ const foldVar = (period: number) => {+ fold(period);+ let v = 0;+ let step = 0;+ for (let b = 0; b < STRIPE_PHASES; b++) {+ const m = phSum[b] / Math.max(1, phCnt[b]);+ const next = phSum[(b + 1) % STRIPE_PHASES] / Math.max(1, phCnt[(b + 1) % STRIPE_PHASES]);+ v += m * m;+ step += (m - next) ** 2;+ }+ if (v === 0) return 0;+ return (v / STRIPE_PHASES) * Math.max(0, 1 - step / (2 * v));+ };+ const tries: { period: number; v: number }[] = [];+ for (let p = 2; p <= Math.min(w, h) / STRIPE_PERIODS; p += Math.max(STRIPE_STEP, p * STRIPE_STEP_SHARE)) {+ tries.push({ period: p, v: foldVar(p) });+ }+ if (tries.length === 0) return null;+ const top = Math.max(...tries.map((t) => t.v));+ if (top < STRIPE_SHARE * pixVar) return null;+ let at = tries.findIndex((t) => t.v >= 0.8 * top);+ while (at + 1 < tries.length && tries[at + 1].v >= tries[at].v) at++;+ const period = tries[at].period;++ // Line and ground: split the period's phases at the middle of their swing.+ fold(period);+ const phMean = Array.from(phSum, (v, b) => v / Math.max(1, phCnt[b]));+ const hi = Math.max(...phMean);+ const lo = Math.min(...phMean);+ if (hi - lo < STRIPE_CONTRAST) return null;+ const bright = phMean.map((v) => v > (hi + lo) / 2);+ const lineBright = bright.filter(Boolean).length <= STRIPE_PHASES / 2;+ // Colours from the phases clear of an edge, where no pixel mixes the two.+ const core = (b: number) =>+ bright[(b + STRIPE_PHASES - 1) % STRIPE_PHASES] === bright[b] && bright[(b + 1) % STRIPE_PHASES] === bright[b];+ const ink = [0, 0, 0, 0];+ const ground = [0, 0, 0, 0];+ const all = [0, 0, 0];+ for (let y = 0; y < h; y++) {+ for (let x = 0; x < w; x++) {+ const q = along[y * w + x] / period;+ const b = Math.min(STRIPE_PHASES - 1, Math.floor((q - Math.floor(q)) * STRIPE_PHASES));+ const i = ((y0 + y) * px.width + x0 + x) * 4;+ all[0] += px.data[i];+ all[1] += px.data[i + 1];+ all[2] += px.data[i + 2];+ if (!core(b)) continue;+ const t = bright[b] === lineBright ? ink : ground;+ t[0] += px.data[i];+ t[1] += px.data[i + 1];+ t[2] += px.data[i + 2];+ t[3]++;+ }+ }+ if (ink[3] === 0 || ground[3] === 0) return null;+ const inkRgb = ink.slice(0, 3).map((v) => v / ink[3]);+ const groundRgb = ground.slice(0, 3).map((v) => v / ground[3]);+ // The line's share of the period: how far the region's mean sits from the+ // ground towards the line, so lines thinner than a phase still measure.+ let num = 0;+ let den = 0;+ for (let ch = 0; ch < 3; ch++) {+ const d = inkRgb[ch] - groundRgb[ch];+ num += (all[ch] / n - groundRgb[ch]) * d;+ den += d * d;+ }+ const lineShare = Math.max(0, Math.min(1, num / den));+ const angle = theta <= -90 ? theta + 180 : theta;+ const r1 = (v: number) => Math.round(v * 10) / 10;+ return {+ angle: Math.round(angle),+ width: r1(period * lineShare * unit),+ gap: r1(period * (1 - lineShare) * unit),+ line: hexOf(inkRgb[0], inkRgb[1], inkRgb[2]),+ ground: hexOf(groundRgb[0], groundRgb[1], groundRgb[2]),+ };+}diff --git a/site/src/cut/lib/measure/sourceMeasure.ts b/site/src/cut/lib/measure/sourceMeasure.tsindex 68860545..f0501fc8 100644--- a/site/src/cut/lib/measure/sourceMeasure.ts+++ b/site/src/cut/lib/measure/sourceMeasure.ts@@ -9,7 +9,7 @@ import { openMedia, UnreadableMediaError, videoTrackOf, type FrameSize } from ".import { createRasterCanvas, decodeRasterImageUrl } from "../raster";import type { WrappedCanvas } from "mediabunny";import { CAMERA_LONG_SIDE, cameraPath, grayOf, holdsStill, type CameraPose, type Gray } from "./camera";-import { hexOf, regionColor, parseHex, type PixelRect, type Pixels, type Rgb } from "./color";+import { hexOf, regionColor, regionStripes, parseHex, type PixelRect, type Pixels, type Rgb, type Stripes } from "./color";import { frameStep, gridsOf, pickShotEdges, type FrameGrids, type FrameStep, type PictureFade, type ShotChange } from "./cuts";import { fitMoves, type Move } from "./easeFit";import { FULL_FRAME, areaOf, findPictureBox, intoArea, type PictureFind } from "./picture";@@ -263,31 +263,40 @@ export interface ColorSample {box: FrameBox;}+export type ColorRead = ReturnType<typeof regionColor> & { stripes?: Stripes };+/** The colour of each sample's region at its moment, read in one ordered- * pass over the source. `k` ≥ 2 adds the k colours each region is made of. */+ * pass over the source. `k` ≥ 2 adds the k colours each region is made of;+ * a region laid in parallel lines adds its `stripes`, in px at 1080 of the+ * area's short side. */export async function sampleColors(url: string,kind: "video" | "image",samples: ColorSample[],k: number,area: FrameBox = FULL_FRAME-): Promise<ReturnType<typeof regionColor>[]> {+): Promise<ColorRead[]> {// Each sample's box is in the area's fractions; the reads are the frame's.samples = samples.map((s) => ({ ...s, box: intoArea(area, s.box) }));+ const read = (px: Pixels, box: FrameBox): ColorRead => {+ const rect = toPixels(px, box);+ const stripes = regionStripes(px, rect, 1080 / Math.min(area.w * px.width, area.h * px.height));+ return { ...regionColor(px, rect, k), ...(stripes ? { stripes } : {}) };+ };if (kind === "image") {const px = await imagePixels(url, COLOR_LONG_SIDE);- return samples.map((s) => regionColor(px, toPixels(px, s.box), k));+ return samples.map((s) => read(px, s.box));}const times = [...new Set(samples.map((s) => s.t))].sort((a, b) => a - b);return withSourceVideo(url, async (v) => {const size = sizeAt(v.width, v.height, COLOR_LONG_SIDE, area);- const out = new Array<ReturnType<typeof regionColor>>(samples.length);+ const out = new Array<ColorRead>(samples.length);let at = 0;for await (const f of v.framesAt(times, size)) {const t = times[at++];if (!f) throw new Error(`Could not read the frame at ${t}s.`);samples.forEach((s, i) => {- if (s.t === t) out[i] = regionColor(f, toPixels(f, s.box), k);+ if (s.t === t) out[i] = read(f, s.box);});}return out;