Skip to content

Open source CapCut alternative.

TypeScript

feat(site): read a striped fill's lines with sample_color

David Le921c91aabf24
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.ts
index ad3f8b50..2a1ea061 100644
--- a/site/src/cut/components/AiPanel.tools.ts
+++ b/site/src/cut/components/AiPanel.tools.ts
@@ -274,7 +274,7 @@ export const AI_PANEL_TOOLS = [
{
name: "sample_color",
description:
- `Read the colour of a source's pixels: the hex of a region or a point of a frame at a source second, so a block, a shape, a title or a background takes the reference's colour as a number. Contact-sheet cells are JPEGs and shift colour; this reads the decoded frame as the preview shows it. Each sample is a moment \`t\` (SOURCE seconds) and a box — x/y its centre and w/h its size as frame fractions, the box add_shape takes; leave out w/h to read a point. Returns per sample \`hex\`, the region's median colour, and \`spread\`, how far its pixels stray from it on average (0-255: under 10 is a flat fill, over 40 a mix — read \`dominant\` there). \`colors\` 2-${SAMPLE_COLORS_K_MAX} adds \`dominant\`, the colours the region is made of with their shares, largest first: a card and the type on it, a bar and the shot behind it. Read a point inside a flat fill for its colour, a box over a title for its ink and its ground; the ink's share tells a filled word (a quarter of its box or more) from an outlined one, so a word that swaps between the two is read shot by shot. Up to ${SAMPLE_COLOR_MAX} samples a call, at one moment or each at its own. Works on a still image too (t is ignored).`,
+ `Read the colour of a source's pixels: the hex of a region or a point of a frame at a source second, so a block, a shape, a title or a background takes the reference's colour as a number. Contact-sheet cells are JPEGs and shift colour; this reads the decoded frame as the preview shows it. Each sample is a moment \`t\` (SOURCE seconds) and a box — x/y its centre and w/h its size as frame fractions, the box add_shape takes; leave out w/h to read a point. Returns per sample \`hex\`, the region's median colour, and \`spread\`, how far its pixels stray from it on average (0-255: under 10 is a flat fill, over 40 a mix — read \`dominant\` there). \`colors\` 2-${SAMPLE_COLORS_K_MAX} adds \`dominant\`, the colours the region is made of with their shares, largest first: a card and the type on it, a bar and the shot behind it. Read a point inside a flat fill for its colour, a box over a title for its ink and its ground; the ink's share tells a filled word (a quarter of its box or more) from an outlined one, so a word that swaps between the two is read shot by shot. A box over a fill laid in fine parallel lines — a striped card, a hatched bar — adds \`stripes\`: the lines' \`angle\` (clockwise from vertical), \`width\` and \`gap\` in px at 1080, and the \`line\` and \`ground\` colours, exactly what add_shape's stripe pattern takes; read a box well inside the fill, a few dozen lines across. Up to ${SAMPLE_COLOR_MAX} samples a call, at one moment or each at its own. Works on a still image too (t is ignored).`,
inputSchema: obj(
{
asset_id: str("Project asset id (video or image)"),
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index aba91f48..9eeb7c10 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -2036,6 +2036,12 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
...(smp.point ? {} : { w: rF(smp.box.w), h: rF(smp.box.h) }),
...read[i],
})),
+ ...(read.some((r) => r.stripes)
+ ? {
+ stripesNote:
+ "A sample with `stripes` is laid in parallel lines: lay a block in its ground colour, then the same box over it as add_shape with fill the line colour, pattern stripes and pattern_width, pattern_gap and pattern_angle as they are.",
+ }
+ : {}),
...pictureReport(area),
source: sourceRef(asset),
};
diff --git a/site/src/cut/lib/measure/color.test.ts b/site/src/cut/lib/measure/color.test.ts
index 176c5a82..0fdb20f4 100644
--- a/site/src/cut/lib/measure/color.test.ts
+++ b/site/src/cut/lib/measure/color.test.ts
@@ -1,5 +1,5 @@
import { expect, test } from "bun:test";
-import { hexOf, parseHex, regionColor } from "./color";
+import { hexOf, parseHex, regionColor, regionStripes } from "./color";
/** A w×h RGBA frame painted by `paint(x, y)`. */
const frame = (w: number, h: number, paint: (x: number, y: number) => [number, number, number]) => {
@@ -41,3 +41,42 @@ test("a region running off the frame is clipped to it", () => {
const out = regionColor(px, { x: 8, y: 8, w: 10, h: 10 });
expect(out.hex).toBe("#000000");
});
+
+/** Lines `width` px thick every `period` px, their normal `deg` clockwise from
+ * the x axis, so the lines themselves run `deg` clockwise from vertical. */
+const striped = (deg: number, period: number, width: number, line: [number, number, number], ground: [number, number, number]) => {
+ const c = Math.cos((deg * Math.PI) / 180);
+ const s = Math.sin((deg * Math.PI) / 180);
+ return frame(200, 200, (x, y) => ((((x * c + y * s) % period) + period) % period < width ? line : ground));
+};
+
+test("a striped region reads its lines: angle, thickness, gap and both colours", () => {
+ const px = striped(-45, 10, 4, [70, 190, 215], [50, 175, 200]);
+ const out = regionStripes(px, { x: 20, y: 20, w: 160, h: 160 }, 2);
+ expect(out).not.toBeNull();
+ expect(out!.angle).toBe(-45);
+ // Thickness and gap in px at 1080: two design px per pixel here.
+ expect(out!.width).toBeCloseTo(8, 0);
+ expect(out!.gap).toBeCloseTo(12, 0);
+ expect(out!.line).toBe(hexOf(70, 190, 215));
+ expect(out!.ground).toBe(hexOf(50, 175, 200));
+});
+
+test("upright and tilted stripes read their direction the way a shape's pattern takes it", () => {
+ expect(regionStripes(striped(0, 12, 4, [255, 255, 255], [0, 0, 0]), { x: 0, y: 0, w: 200, h: 200 }, 1)!.angle).toBe(0);
+ expect(regionStripes(striped(30, 12, 4, [255, 255, 255], [0, 0, 0]), { x: 0, y: 0, w: 200, h: 200 }, 1)!.angle).toBe(30);
+ expect(regionStripes(striped(90, 12, 4, [255, 255, 255], [0, 0, 0]), { x: 0, y: 0, w: 200, h: 200 }, 1)!.angle).toBe(90);
+});
+
+test("a flat fill, a split card and a little noise read no stripes", () => {
+ expect(regionStripes(frame(80, 80, () => [56, 180, 202]), { x: 0, y: 0, w: 80, h: 80 }, 1)).toBeNull();
+ const split = frame(80, 80, (x) => (x < 50 ? [56, 180, 202] : [20, 70, 80]));
+ expect(regionStripes(split, { x: 0, y: 0, w: 80, h: 80 }, 1)).toBeNull();
+ let seed = 7;
+ const grain = frame(80, 80, () => {
+ seed = (seed * 16807) % 2147483647;
+ const d = (seed % 9) - 4;
+ return [56 + d, 180 + d, 202 + d];
+ });
+ expect(regionStripes(grain, { x: 0, y: 0, w: 80, h: 80 }, 1)).toBeNull();
+});
diff --git a/site/src/cut/lib/measure/color.ts b/site/src/cut/lib/measure/color.ts
index 777c07ee..127b229b 100644
--- a/site/src/cut/lib/measure/color.ts
+++ b/site/src/cut/lib/measure/color.ts
@@ -1,5 +1,5 @@
/** Colour reads of a region of a frame: its median colour, how far the region
- * strays from it, and the few colours it is made of. Pure pixel math on RGBA
+ * strays from it, the few colours it is made of, and the stripes it is laid in. Pure pixel math on RGBA
* data; sourceMeasure.ts hands it decoded frames. */
/** An RGBA frame, as ImageData carries it. */
@@ -26,6 +26,27 @@ const CLUSTER_ROUNDS = 10;
/** Dominant colours below this share of the region are noise: edges, grain. */
const MIN_SHARE = 0.01;
+/** Stripes are read off the middle of a region at most this many pixels a
+ * side: a few dozen lines, and the angle and period searches stay cheap. */
+const STRIPE_SIDE = 160;
+/** Bin width along the stripes' normal for the angle search, pixels. */
+const STRIPE_BIN = 0.5;
+/** The fewest whole periods the read must hold, so the longest period read is
+ * a third of the side. */
+const STRIPE_PERIODS = 3;
+/** Period steps: fine where a step of 0.02 px still moves the fold, then a
+ * fixed share of the period. */
+const STRIPE_STEP = 0.02;
+const STRIPE_STEP_SHARE = 0.005;
+/** Phase bins a period folds into. */
+const STRIPE_PHASES = 16;
+/** Share of the region's luminance variance the folded period must explain:
+ * stripes are most of what varies; type, an edge or grain isn't. */
+const STRIPE_SHARE = 0.35;
+/** Least luminance swing between line and gap, 0..255 — under it the lines
+ * are too faint to lay. */
+const STRIPE_CONTRAST = 4;
+
const hex2 = (v: number) => Math.round(Math.max(0, Math.min(255, v))).toString(16).padStart(2, "0").toUpperCase();
export const hexOf = (r: number, g: number, b: number): string => `#${hex2(r)}${hex2(g)}${hex2(b)}`;
@@ -184,3 +205,183 @@ function dominantColors(px: Pixels, r: PixelRect, k: number): { hex: string; sha
.filter((c) => c.share >= MIN_SHARE)
.sort((a, b) => b.share - a.share);
}
+
+/** Parallel lines a region is laid in, as add_shape's stripe pattern takes
+ * them: `angle` the lines' direction in degrees clockwise from vertical,
+ * -90..90; `width` and `gap` in px at 1080; `line` and `ground` their colours. */
+export interface Stripes {
+ angle: number;
+ width: number;
+ gap: number;
+ line: string;
+ ground: string;
+}
+
+/** The stripes a region is laid in, or null when it isn't. `unit` is design
+ * px at 1080 per frame pixel. The angle is the one whose profile across the
+ * lines swings widest; the period the shortest that folds the pixels onto
+ * one line about as well as the best does (twice the period folds just as
+ * well); the line is whichever of the brighter and darker phases covers
+ * less of the period. */
+export function regionStripes(px: Pixels, rect: PixelRect, unit: number): Stripes | null {
+ const r = clipRect(px, rect);
+ const w = Math.min(STRIPE_SIDE, r.w);
+ const h = Math.min(STRIPE_SIDE, r.h);
+ if (Math.min(w, h) < 4 * STRIPE_PERIODS) return null;
+ const x0 = r.x + Math.floor((r.w - w) / 2);
+ const y0 = r.y + Math.floor((r.h - h) / 2);
+ const n = w * h;
+ const lum = new Float32Array(n);
+ let mean = 0;
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const i = ((y0 + y) * px.width + x0 + x) * 4;
+ const v = 0.2126 * px.data[i] + 0.7152 * px.data[i + 1] + 0.0722 * px.data[i + 2];
+ lum[y * w + x] = v;
+ mean += v;
+ }
+ }
+ mean /= n;
+ let pixVar = 0;
+ for (let k = 0; k < n; k++) {
+ lum[k] -= mean;
+ pixVar += lum[k] * lum[k];
+ }
+ pixVar /= n;
+ // A square wave of the least contrast varies by a quarter of its swing squared.
+ if (pixVar < (STRIPE_CONTRAST * STRIPE_CONTRAST) / 4) return null;
+
+ // The angle: the normal whose binned profile varies most.
+ const reach = Math.ceil(Math.hypot(w, h) / STRIPE_BIN) + 2;
+ const sum = new Float64Array(reach * 2);
+ const cnt = new Uint32Array(reach * 2);
+ const profileVar = (deg: number) => {
+ const c = Math.cos((deg * Math.PI) / 180) / STRIPE_BIN;
+ const s = Math.sin((deg * Math.PI) / 180) / STRIPE_BIN;
+ sum.fill(0);
+ cnt.fill(0);
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const b = Math.floor(x * c + y * s) + reach;
+ sum[b] += lum[y * w + x];
+ cnt[b]++;
+ }
+ }
+ let v = 0;
+ let bins = 0;
+ for (let b = 0; b < reach * 2; b++) {
+ if (cnt[b] < Math.min(w, h) / 2) continue;
+ v += (sum[b] / cnt[b]) ** 2;
+ bins++;
+ }
+ return bins ? v / bins : 0;
+ };
+ let theta = 0;
+ let best = -1;
+ for (let d = -90; d < 90; d++) {
+ const v = profileVar(d);
+ if (v > best) [theta, best] = [d, v];
+ }
+ for (let d = theta - 1; d <= theta + 1; d += 0.25) {
+ const v = profileVar(d);
+ if (v > best) [theta, best] = [d, v];
+ }
+
+ // The period: fold each pixel by its place along the normal.
+ const c0 = Math.cos((theta * Math.PI) / 180);
+ const s0 = Math.sin((theta * Math.PI) / 180);
+ const along = new Float32Array(n);
+ for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) along[y * w + x] = x * c0 + y * s0;
+ const phSum = new Float64Array(STRIPE_PHASES);
+ const phCnt = new Uint32Array(STRIPE_PHASES);
+ const fold = (period: number) => {
+ phSum.fill(0);
+ phCnt.fill(0);
+ for (let k = 0; k < n; k++) {
+ const q = along[k] / period;
+ const b = Math.min(STRIPE_PHASES - 1, Math.floor((q - Math.floor(q)) * STRIPE_PHASES));
+ phSum[b] += lum[k];
+ phCnt[b]++;
+ }
+ };
+ // A fold's swing, kept only as far as neighbouring phases agree: on a pixel
+ // grid a period's fraction (10 px folded at 10/4.5) sorts the same pixels
+ // into the same bins out of order and swings just as wide, but only the
+ // true period lays one line in one run of phases.
+ const foldVar = (period: number) => {
+ fold(period);
+ let v = 0;
+ let step = 0;
+ for (let b = 0; b < STRIPE_PHASES; b++) {
+ const m = phSum[b] / Math.max(1, phCnt[b]);
+ const next = phSum[(b + 1) % STRIPE_PHASES] / Math.max(1, phCnt[(b + 1) % STRIPE_PHASES]);
+ v += m * m;
+ step += (m - next) ** 2;
+ }
+ if (v === 0) return 0;
+ return (v / STRIPE_PHASES) * Math.max(0, 1 - step / (2 * v));
+ };
+ const tries: { period: number; v: number }[] = [];
+ for (let p = 2; p <= Math.min(w, h) / STRIPE_PERIODS; p += Math.max(STRIPE_STEP, p * STRIPE_STEP_SHARE)) {
+ tries.push({ period: p, v: foldVar(p) });
+ }
+ if (tries.length === 0) return null;
+ const top = Math.max(...tries.map((t) => t.v));
+ if (top < STRIPE_SHARE * pixVar) return null;
+ let at = tries.findIndex((t) => t.v >= 0.8 * top);
+ while (at + 1 < tries.length && tries[at + 1].v >= tries[at].v) at++;
+ const period = tries[at].period;
+
+ // Line and ground: split the period's phases at the middle of their swing.
+ fold(period);
+ const phMean = Array.from(phSum, (v, b) => v / Math.max(1, phCnt[b]));
+ const hi = Math.max(...phMean);
+ const lo = Math.min(...phMean);
+ if (hi - lo < STRIPE_CONTRAST) return null;
+ const bright = phMean.map((v) => v > (hi + lo) / 2);
+ const lineBright = bright.filter(Boolean).length <= STRIPE_PHASES / 2;
+ // Colours from the phases clear of an edge, where no pixel mixes the two.
+ const core = (b: number) =>
+ bright[(b + STRIPE_PHASES - 1) % STRIPE_PHASES] === bright[b] && bright[(b + 1) % STRIPE_PHASES] === bright[b];
+ const ink = [0, 0, 0, 0];
+ const ground = [0, 0, 0, 0];
+ const all = [0, 0, 0];
+ for (let y = 0; y < h; y++) {
+ for (let x = 0; x < w; x++) {
+ const q = along[y * w + x] / period;
+ const b = Math.min(STRIPE_PHASES - 1, Math.floor((q - Math.floor(q)) * STRIPE_PHASES));
+ const i = ((y0 + y) * px.width + x0 + x) * 4;
+ all[0] += px.data[i];
+ all[1] += px.data[i + 1];
+ all[2] += px.data[i + 2];
+ if (!core(b)) continue;
+ const t = bright[b] === lineBright ? ink : ground;
+ t[0] += px.data[i];
+ t[1] += px.data[i + 1];
+ t[2] += px.data[i + 2];
+ t[3]++;
+ }
+ }
+ if (ink[3] === 0 || ground[3] === 0) return null;
+ const inkRgb = ink.slice(0, 3).map((v) => v / ink[3]);
+ const groundRgb = ground.slice(0, 3).map((v) => v / ground[3]);
+ // The line's share of the period: how far the region's mean sits from the
+ // ground towards the line, so lines thinner than a phase still measure.
+ let num = 0;
+ let den = 0;
+ for (let ch = 0; ch < 3; ch++) {
+ const d = inkRgb[ch] - groundRgb[ch];
+ num += (all[ch] / n - groundRgb[ch]) * d;
+ den += d * d;
+ }
+ const lineShare = Math.max(0, Math.min(1, num / den));
+ const angle = theta <= -90 ? theta + 180 : theta;
+ const r1 = (v: number) => Math.round(v * 10) / 10;
+ return {
+ angle: Math.round(angle),
+ width: r1(period * lineShare * unit),
+ gap: r1(period * (1 - lineShare) * unit),
+ line: hexOf(inkRgb[0], inkRgb[1], inkRgb[2]),
+ ground: hexOf(groundRgb[0], groundRgb[1], groundRgb[2]),
+ };
+}
diff --git a/site/src/cut/lib/measure/sourceMeasure.ts b/site/src/cut/lib/measure/sourceMeasure.ts
index 68860545..f0501fc8 100644
--- a/site/src/cut/lib/measure/sourceMeasure.ts
+++ b/site/src/cut/lib/measure/sourceMeasure.ts
@@ -9,7 +9,7 @@ import { openMedia, UnreadableMediaError, videoTrackOf, type FrameSize } from ".
import { createRasterCanvas, decodeRasterImageUrl } from "../raster";
import type { WrappedCanvas } from "mediabunny";
import { CAMERA_LONG_SIDE, cameraPath, grayOf, holdsStill, type CameraPose, type Gray } from "./camera";
-import { hexOf, regionColor, parseHex, type PixelRect, type Pixels, type Rgb } from "./color";
+import { hexOf, regionColor, regionStripes, parseHex, type PixelRect, type Pixels, type Rgb, type Stripes } from "./color";
import { frameStep, gridsOf, pickShotEdges, type FrameGrids, type FrameStep, type PictureFade, type ShotChange } from "./cuts";
import { fitMoves, type Move } from "./easeFit";
import { FULL_FRAME, areaOf, findPictureBox, intoArea, type PictureFind } from "./picture";
@@ -263,31 +263,40 @@ export interface ColorSample {
box: FrameBox;
}
+export type ColorRead = ReturnType<typeof regionColor> & { stripes?: Stripes };
+
/** The colour of each sample's region at its moment, read in one ordered
- * pass over the source. `k` ≥ 2 adds the k colours each region is made of. */
+ * pass over the source. `k` ≥ 2 adds the k colours each region is made of;
+ * a region laid in parallel lines adds its `stripes`, in px at 1080 of the
+ * area's short side. */
export async function sampleColors(
url: string,
kind: "video" | "image",
samples: ColorSample[],
k: number,
area: FrameBox = FULL_FRAME
-): Promise<ReturnType<typeof regionColor>[]> {
+): Promise<ColorRead[]> {
// Each sample's box is in the area's fractions; the reads are the frame's.
samples = samples.map((s) => ({ ...s, box: intoArea(area, s.box) }));
+ const read = (px: Pixels, box: FrameBox): ColorRead => {
+ const rect = toPixels(px, box);
+ const stripes = regionStripes(px, rect, 1080 / Math.min(area.w * px.width, area.h * px.height));
+ return { ...regionColor(px, rect, k), ...(stripes ? { stripes } : {}) };
+ };
if (kind === "image") {
const px = await imagePixels(url, COLOR_LONG_SIDE);
- return samples.map((s) => regionColor(px, toPixels(px, s.box), k));
+ return samples.map((s) => read(px, s.box));
}
const times = [...new Set(samples.map((s) => s.t))].sort((a, b) => a - b);
return withSourceVideo(url, async (v) => {
const size = sizeAt(v.width, v.height, COLOR_LONG_SIDE, area);
- const out = new Array<ReturnType<typeof regionColor>>(samples.length);
+ const out = new Array<ColorRead>(samples.length);
let at = 0;
for await (const f of v.framesAt(times, size)) {
const t = times[at++];
if (!f) throw new Error(`Could not read the frame at ${t}s.`);
samples.forEach((s, i) => {
- if (s.t === t) out[i] = regionColor(f, toPixels(f, s.box), k);
+ if (s.t === t) out[i] = read(f, s.box);
});
}
return out;