Skip to content

Open source CapCut alternative.

TypeScript

feat(site): browser watch pipeline with asset metadata and background sweep [rebuild]

David Le582b05eaed6a
diff --git a/site/package.json b/site/package.json
index f2dbfdfe..7539c823 100644
--- a/site/package.json
+++ b/site/package.json
@@ -20,6 +20,7 @@
"engine:dev": "bun run src/cut/engine/main.ts",
"engine:build": "bash scripts/build-cut-engine.sh",
"worker:build": "bun build --target=node --minify --external=pg-native src/cut/worker/main.ts --outfile dist/cut-worker/main.js",
+ "bench:watch": "bun run scripts/bench-watch.ts",
"eval:cut-chat": "bun run scripts/eval-cut-chat.ts",
"email:dev": "email dev --dir src/emails --port 3001",
"test": "bun test src packages/effects-kit/src",
diff --git a/site/scripts/bench-watch.ts b/site/scripts/bench-watch.ts
new file mode 100644
index 00000000..dbfa1499
--- /dev/null
+++ b/site/scripts/bench-watch.ts
@@ -0,0 +1,467 @@
+/**
+ * Watch-selection benchmark: how well does each frame-selection method turn a
+ * video into ≤36 contact-sheet cells?
+ *
+ * Methods, both fed the same candidate pass (the production sampler's steady
+ * floor at max(1s, duration/150), software decode for determinism):
+ * A fixed interval — the old sampler: steady clamp(range/32, 2, 30)s
+ * C dense+dedup — the shipped selector (lib/watch/dedupSelector)
+ *
+ * Fixtures build deterministically into dist/watch-bench/ (gitignored):
+ * montage — 20 stock clips re-encoded and concatenated; ground truth =
+ * the known cut times (does each segment get a frame?)
+ * slideshow — synthesized text cards on a static background; ground truth =
+ * the card-change times (the settled-channel case: thin strokes
+ * that measure ~0% on a coarse grid)
+ * talking heads — stock character clips as-is (dedup should collapse these)
+ * jfk-rice — public-domain speech footage via yt-dlp when available
+ *
+ * Outputs: dist/watch-bench/stats.csv, per-video keep/drop report.html for
+ * method C, candidate JPEGs for eyeballing. Run: npx bun scripts/bench-watch.ts
+ */
+import { spawn } from "node:child_process";
+import { mkdir, readdir, rm, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { createDedupSelector } from "../src/cut/lib/watch/dedupSelector";
+import { SIGNATURE_SIZE, type FrameDecision } from "../src/cut/lib/watch/types";
+
+const SITE = path.resolve(import.meta.dir, "..");
+const OUT = path.resolve(SITE, "..", "dist", "watch-bench");
+const STOCK = path.join(SITE, "public", "cut-stock-video");
+const RAW_BYTES = SIGNATURE_SIZE * SIGNATURE_SIZE * 3;
+const MAX_KEPT = 36;
+
+const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));
+
+interface RunOut {
+ code: number | null;
+ stdout: Buffer;
+ stderr: string;
+}
+
+function run(cmd: string, args: string[], onStdout?: (d: Buffer) => void): Promise<RunOut> {
+ return new Promise((resolve, reject) => {
+ const p = spawn(cmd, args);
+ const chunks: Buffer[] = [];
+ let stderr = "";
+ p.stdout.on("data", (d: Buffer) => (onStdout ? onStdout(d) : chunks.push(d)));
+ p.stderr.on("data", (d) => {
+ if (stderr.length < 4_000_000) stderr += d.toString();
+ });
+ p.on("error", reject);
+ p.on("close", (code) => resolve({ code, stdout: Buffer.concat(chunks), stderr }));
+ });
+}
+
+async function ffmpeg(args: string[], onStdout?: (d: Buffer) => void): Promise<RunOut> {
+ const r = await run("ffmpeg", ["-hide_banner", "-nostats", "-y", ...args], onStdout);
+ if (r.code !== 0 && !onStdout)
+ throw new Error(`ffmpeg failed: ${r.stderr.split("\n").slice(-4).join("\n")}`);
+ return r;
+}
+
+// ---------------------------------------------------------------- fixtures
+
+/** Deterministic PRNG so the slideshow renders identically every run. */
+function lcg(seed: number) {
+ let s = seed >>> 0;
+ return () => ((s = (s * 1664525 + 1013904223) >>> 0), s / 2 ** 32);
+}
+
+const CARD_W = 640;
+const CARD_H = 360;
+const CARD_SECONDS = 6;
+const CARD_COUNT = 8;
+
+/** One text card as a PPM (P6) buffer: a block digit header plus rows of
+ * 2px "text" strokes whose lengths change per card — thin ink on a static
+ * background, the change a coarse grid cannot see. */
+function renderCard(card: number): Buffer {
+ const px = new Uint8Array(CARD_W * CARD_H * 3);
+ for (let i = 0; i < px.length; i += 3) {
+ px[i] = 16;
+ px[i + 1] = 16;
+ px[i + 2] = 20;
+ }
+ const fill = (x: number, y: number, w: number, h: number) => {
+ for (let yy = y; yy < y + h && yy < CARD_H; yy++)
+ for (let xx = x; xx < x + w && xx < CARD_W; xx++) {
+ const i = (yy * CARD_W + xx) * 3;
+ px[i] = 235;
+ px[i + 1] = 235;
+ px[i + 2] = 240;
+ }
+ };
+ // Seven-segment "chapter" digit, so sheets are eyeball-checkable. It only
+ // advances every second card — the in-between transitions change nothing
+ // but the 2px stroke rows, the change only the settled channel can see.
+ const SEGS: Record<string, [number, number, number, number]> = {
+ top: [0, 0, 60, 8],
+ mid: [0, 46, 60, 8],
+ bot: [0, 92, 60, 8],
+ tl: [0, 0, 8, 50],
+ tr: [52, 0, 8, 50],
+ bl: [0, 50, 8, 50],
+ br: [52, 50, 8, 50],
+ };
+ const DIGIT: Record<number, string[]> = {
+ 0: ["top", "bot", "tl", "tr", "bl", "br"],
+ 1: ["tr", "br"],
+ 2: ["top", "mid", "bot", "tr", "bl"],
+ 3: ["top", "mid", "bot", "tr", "br"],
+ 4: ["mid", "tl", "tr", "br"],
+ 5: ["top", "mid", "bot", "tl", "br"],
+ 6: ["top", "mid", "bot", "tl", "bl", "br"],
+ 7: ["top", "tr", "br"],
+ 8: ["top", "mid", "bot", "tl", "tr", "bl", "br"],
+ 9: ["top", "mid", "bot", "tl", "tr", "br"],
+ };
+ for (const seg of DIGIT[(Math.floor(card / 2) + 1) % 10]) {
+ const [x, y, w, h] = SEGS[seg];
+ fill(40 + x, 30 + y, w, h);
+ }
+ // "Body text": a seeded paragraph of thin word-run strokes. Line pitch,
+ // word lengths, and gaps are the card's identity — different text lays its
+ // ink at different spots, the way real caption swaps do.
+ const rand = lcg(1000 + card * 77);
+ const lineH = 14 + Math.floor(rand() * 9);
+ const lines = 7 + Math.floor(rand() * 4);
+ for (let row = 0; row < lines; row++) {
+ const y = 160 + row * lineH + Math.floor(rand() * 4);
+ let x = 40 + Math.floor(rand() * 30);
+ while (x < 560) {
+ const w = 20 + Math.floor(rand() * 60);
+ if (rand() > 0.25) fill(x, y, Math.min(w, 580 - x), 3);
+ x += w + 8 + Math.floor(rand() * 18);
+ }
+ }
+ return Buffer.concat([Buffer.from(`P6\n${CARD_W} ${CARD_H}\n255\n`), Buffer.from(px)]);
+}
+
+const WRITE_SECONDS = 40;
+const WRITE_STEP = 2; // a new word-run lands every 2s
+// One page for the whole clip: a cleared page would leave the old full page
+// in the selector's kept window, sitting exactly where the new ink lands —
+// the window's changed-vs-everything rule then reads the new ink as already
+// seen. Page swaps are a hard-cut case; the montage covers those.
+const WRITE_PAGE = WRITE_SECONDS;
+
+/** One second of the "handwriting" fixture: dark word-runs accumulating on a
+ * bright static page, one more every WRITE_STEP seconds — a slow thin-stroke
+ * morph, the settled channel's hardest case. */
+function renderWritingFrame(sec: number): Buffer {
+ const px = new Uint8Array(CARD_W * CARD_H * 3);
+ for (let i = 0; i < px.length; i += 3) {
+ px[i] = 235;
+ px[i + 1] = 233;
+ px[i + 2] = 225;
+ }
+ const ink = (x: number, y: number, w: number, h: number) => {
+ for (let yy = y; yy < y + h && yy < CARD_H; yy++)
+ for (let xx = x; xx < x + w && xx < CARD_W; xx++) {
+ const i = (yy * CARD_W + xx) * 3;
+ px[i] = 30;
+ px[i + 1] = 28;
+ px[i + 2] = 40;
+ }
+ };
+ const page = Math.floor(sec / WRITE_PAGE);
+ const steps = Math.floor((sec % WRITE_PAGE) / WRITE_STEP) + 1;
+ const rand = lcg(4000 + page * 131);
+ let x = 60;
+ let y = 60;
+ for (let s = 0; s < steps; s++) {
+ const w = 40 + Math.floor(rand() * 90);
+ if (x + w > 580) {
+ x = 60;
+ y += 24 + Math.floor(rand() * 6);
+ }
+ ink(x, y, w, 3);
+ x += w + 14 + Math.floor(rand() * 12);
+ }
+ return Buffer.concat([Buffer.from(`P6\n${CARD_W} ${CARD_H}\n255\n`), Buffer.from(px)]);
+}
+
+interface Fixture {
+ slug: string;
+ file: string;
+ /** Ground-truth state boundaries (ascending): a good selection has ≥1 kept
+ * frame inside every [boundary[i], boundary[i+1]) span. */
+ states?: number[];
+}
+
+async function buildFixtures(): Promise<Fixture[]> {
+ const fx = path.join(OUT, "fixtures");
+ await mkdir(fx, { recursive: true });
+ const fixtures: Fixture[] = [];
+
+ // montage: 20 stock clips, 7s each, one size, concatenated. Real encoded
+ // part durations drift from the asked-for 7s, so the ground-truth cut
+ // times are measured, then remembered beside the file.
+ const montage = path.join(fx, "montage.mp4");
+ const montageStates = path.join(fx, "montage.states.json");
+ const stock = (await readdir(STOCK)).filter((f) => f.endsWith(".mp4")).sort().slice(0, 20);
+ if (!existsSync(montage) || !existsSync(montageStates)) {
+ const parts: string[] = [];
+ const cuts: number[] = [];
+ let at = 0;
+ for (const [i, f] of stock.entries()) {
+ const part = path.join(fx, `part-${String(i).padStart(2, "0")}.ts`);
+ await ffmpeg([
+ "-ss", "0", "-t", "7", "-i", path.join(STOCK, f),
+ "-vf", "scale=640:360:force_original_aspect_ratio=increase,crop=640:360,fps=30",
+ "-an", "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", part,
+ ]);
+ parts.push(part);
+ cuts.push(at);
+ const probe = await run("ffprobe", [
+ "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", part,
+ ]);
+ at += parseFloat(probe.stdout.toString()) || 7;
+ }
+ const list = path.join(fx, "concat.txt");
+ await writeFile(list, parts.map((p) => `file '${p}'`).join("\n"));
+ await ffmpeg(["-f", "concat", "-safe", "0", "-i", list, "-c", "copy", montage]);
+ await writeFile(montageStates, JSON.stringify(cuts));
+ await Promise.all(parts.map((p) => rm(p, { force: true })));
+ await rm(list, { force: true });
+ }
+ fixtures.push({
+ slug: "montage",
+ file: montage,
+ states: JSON.parse(await Bun.file(montageStates).text()) as number[],
+ });
+
+ // slideshow: synthesized text cards, CARD_SECONDS each.
+ const slideshow = path.join(fx, "slideshow.mp4");
+ if (!existsSync(slideshow)) {
+ for (let c = 0; c < CARD_COUNT; c++)
+ await writeFile(path.join(fx, `card-${String(c).padStart(3, "0")}.ppm`), renderCard(c));
+ await ffmpeg([
+ "-framerate", `1/${CARD_SECONDS}`, "-i", path.join(fx, "card-%03d.ppm"),
+ "-r", "30", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
+ slideshow,
+ ]);
+ for (let c = 0; c < CARD_COUNT; c++)
+ await rm(path.join(fx, `card-${String(c).padStart(3, "0")}.ppm`), { force: true });
+ }
+ fixtures.push({
+ slug: "slideshow",
+ file: slideshow,
+ states: Array.from({ length: CARD_COUNT }, (_, i) => i * CARD_SECONDS),
+ });
+
+ // handwriting: ink accumulating on a static page.
+ const writing = path.join(fx, "writing.mp4");
+ if (!existsSync(writing)) {
+ for (let s = 0; s < WRITE_SECONDS; s++)
+ await writeFile(path.join(fx, `write-${String(s).padStart(3, "0")}.ppm`), renderWritingFrame(s));
+ await ffmpeg([
+ "-framerate", "1", "-i", path.join(fx, "write-%03d.ppm"),
+ "-r", "30", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
+ writing,
+ ]);
+ for (let s = 0; s < WRITE_SECONDS; s++)
+ await rm(path.join(fx, `write-${String(s).padStart(3, "0")}.ppm`), { force: true });
+ }
+ fixtures.push({
+ slug: "writing",
+ file: writing,
+ states: Array.from({ length: WRITE_SECONDS / WRITE_STEP }, (_, i) => i * WRITE_STEP),
+ });
+
+ // talking heads: character clips as-is (no ground truth — the measure is
+ // how few frames a static speaker costs).
+ for (const slug of ["character-cafe-analyst", "character-startup-founder"]) {
+ const file = path.join(STOCK, `${slug}.mp4`);
+ if (existsSync(file)) fixtures.push({ slug, file });
+ }
+
+ // jfk-rice: public-domain speech footage (1962), 60–240s section.
+ const jfk = path.join(fx, "jfk-rice.mp4");
+ if (!existsSync(jfk)) {
+ const probe = await run("yt-dlp", ["--version"]).catch(() => null);
+ if (probe && probe.code === 0) {
+ console.log("fetching jfk-rice.mp4 (yt-dlp)...");
+ const dl = await run("yt-dlp", [
+ "--download-sections", "*60-240",
+ "-f", "bv*[height<=480]+ba/b[height<=480]",
+ "--merge-output-format", "mp4",
+ "-o", jfk,
+ "https://www.youtube.com/watch?v=WZyRbnpGyzQ",
+ ]);
+ if (dl.code !== 0) console.log(" jfk-rice download failed — skipping that fixture");
+ } else {
+ console.log("yt-dlp not found — skipping the jfk-rice fixture");
+ }
+ }
+ if (existsSync(jfk)) fixtures.push({ slug: "jfk-rice", file: jfk });
+
+ return fixtures;
+}
+
+// ---------------------------------------------------- candidate extraction
+
+interface Candidate {
+ t: number;
+ jpg: string; // path on disk
+ raw: Uint8Array;
+}
+
+/** The production candidate pass (software decode): a steady floor at
+ * max(1s, duration/150), split into 480px JPEGs and raw selector frames. */
+async function extractCandidates(file: string, dir: string): Promise<{ cands: Candidate[]; ms: number }> {
+ await rm(dir, { recursive: true, force: true });
+ await mkdir(dir, { recursive: true });
+ const probe = await run("ffprobe", [
+ "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file,
+ ]);
+ const dur = parseFloat(probe.stdout.toString()) || 0;
+ const interval = Math.max(1, dur / 150);
+ const graph = [
+ // showinfo sits after select, so its log lines are exactly the emitted
+ // candidates in order — the only place the source timestamps survive.
+ `[0:v]select='isnan(prev_selected_t)+gte(t-prev_selected_t,${interval})',` +
+ "showinfo,split=2[jl][rl]",
+ "[jl]scale=480:-2[jpg]",
+ `[rl]scale=${SIGNATURE_SIZE}:${SIGNATURE_SIZE}:flags=area,format=rgb24[raw]`,
+ ].join(";");
+ const raws: Uint8Array[] = [];
+ let pendingBuf: Buffer = Buffer.alloc(0);
+ const t0 = performance.now();
+ const r = await ffmpeg(
+ [
+ "-loglevel", "info",
+ "-i", file,
+ "-filter_complex", graph,
+ "-map", "[jpg]", "-fps_mode", "vfr", "-frames:v", "700", "-q:v", "5",
+ path.join(dir, "cand-%04d.jpg"),
+ "-map", "[raw]", "-fps_mode", "vfr", "-frames:v", "700", "-f", "rawvideo", "pipe:1",
+ ],
+ (d) => {
+ pendingBuf = pendingBuf.length === 0 ? d : Buffer.concat([pendingBuf, d]);
+ while (pendingBuf.length >= RAW_BYTES) {
+ raws.push(new Uint8Array(pendingBuf.subarray(0, RAW_BYTES)));
+ pendingBuf = Buffer.from(pendingBuf.subarray(RAW_BYTES));
+ }
+ }
+ );
+ const ms = performance.now() - t0;
+ if (r.code !== 0) throw new Error(`candidate pass failed: ${r.stderr.split("\n").slice(-4).join("\n")}`);
+ const meta: { t: number }[] = [];
+ for (const line of r.stderr.split("\n")) {
+ const t = /Parsed_showinfo.*pts_time:\s*(-?[\d.]+)/.exec(line);
+ if (t) meta.push({ t: Math.max(0, parseFloat(t[1])) });
+ }
+ const files = (await readdir(dir)).filter((f) => f.endsWith(".jpg")).sort();
+ const n = Math.min(files.length, meta.length, raws.length);
+ const cands: Candidate[] = [];
+ for (let i = 0; i < n; i++) cands.push({ t: meta[i].t, jpg: files[i], raw: raws[i] });
+ return { cands, ms };
+}
+
+// ----------------------------------------------------------------- methods
+
+/** A: the old cloud sampler — one frame per steady interval, capped at 36. */
+function methodA(cands: Candidate[], from: number, to: number): number[] {
+ const interval = clamp((to - from) / 32, 2, 30);
+ const kept: number[] = [];
+ for (let t = from; t < to && kept.length < MAX_KEPT; t += interval) {
+ let best = -1;
+ for (let i = 0; i < cands.length; i++) {
+ if (best === -1 || Math.abs(cands[i].t - t) < Math.abs(cands[best].t - t)) best = i;
+ }
+ if (best >= 0 && kept[kept.length - 1] !== best) kept.push(best);
+ }
+ return kept;
+}
+
+/** C: the shipped selector. */
+function methodC(cands: Candidate[]): { kept: number[]; decisions: FrameDecision[]; ms: number } {
+ const t0 = performance.now();
+ const selector = createDedupSelector({ maxFrames: MAX_KEPT });
+ for (const c of cands)
+ selector.push({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 3, data: c.raw });
+ const sel = selector.finish();
+ return { kept: sel.kept, decisions: sel.decisions, ms: performance.now() - t0 };
+}
+
+// ------------------------------------------------------------------ scoring
+
+/** Fraction of ground-truth states holding at least one kept frame. */
+function coverage(cands: Candidate[], kept: number[], states: number[], end: number): number {
+ const times = kept.map((i) => cands[i].t);
+ let hit = 0;
+ for (let s = 0; s < states.length; s++) {
+ const lo = states[s];
+ const hi = s + 1 < states.length ? states[s + 1] : end;
+ if (times.some((t) => t >= lo && t < hi)) hit++;
+ }
+ return hit / states.length;
+}
+
+function reportHtml(cands: Candidate[], decisions: FrameDecision[], slug: string): string {
+ const rows = decisions
+ .map((d) => {
+ const c = cands[d.index];
+ if (!c) return "";
+ const cls = d.verdict === "drop" ? "drop" : d.verdict === "thinned" ? "thin" : "keep";
+ const extra =
+ d.settledScore !== undefined
+ ? ` · settled ${d.settledScore.toFixed(1)}%/${d.settledGate!.toFixed(1)}%`
+ : "";
+ return (
+ `<figure class="${cls}"><img src="cands/${c.jpg}" loading="lazy">` +
+ `<figcaption>#${d.index} @${c.t.toFixed(1)}s · ${d.verdict} · dist ${d.globalDist.toFixed(1)}%${extra}</figcaption></figure>`
+ );
+ })
+ .join("");
+ return `<!doctype html><meta charset="utf-8"><title>${slug} — watch selection</title>
+<style>body{font:14px system-ui;margin:20px;background:#111;color:#ddd}
+.grid{display:flex;flex-wrap:wrap;gap:10px}figure{margin:0;width:200px}
+img{width:100%;border-radius:4px}figcaption{font-size:11px;color:#999;padding:2px 0}
+.drop img{opacity:.35;outline:2px solid #a33}.thin img{opacity:.35;outline:2px solid #a80}
+.keep img{outline:2px solid #3a6}</style>
+<h2>${slug}</h2><p>green kept · red duplicate · orange removed by the 36-frame cap</p>
+<div class="grid">${rows}</div>`;
+}
+
+// --------------------------------------------------------------------- main
+
+const fixtures = await buildFixtures();
+const csv: string[] = ["video,method,candidates,kept,sheets,coverage,selectMs,extractMs"];
+for (const f of fixtures) {
+ const dir = path.join(OUT, f.slug);
+ const { cands, ms: extractMs } = await extractCandidates(f.file, path.join(dir, "cands"));
+ if (cands.length === 0) {
+ console.log(`${f.slug}: no candidates — skipped`);
+ continue;
+ }
+ const from = 0;
+ const to = cands[cands.length - 1].t + 1;
+ const a = methodA(cands, from, to);
+ const c = methodC(cands);
+ await writeFile(path.join(dir, "report.html"), reportHtml(cands, c.decisions, f.slug));
+
+ console.log(`\n== ${f.slug} (${cands.length} candidates, ${to.toFixed(0)}s, extract ${(extractMs / 1000).toFixed(1)}s)`);
+ for (const [name, kept, ms] of [
+ ["A fixed", a, 0],
+ ["C dedup", c.kept, c.ms],
+ ] as const) {
+ const cov = f.states ? coverage(cands, [...kept], f.states, to) : null;
+ const sheets = Math.ceil(kept.length / 9);
+ console.log(
+ ` ${name}: kept ${String(kept.length).padStart(3)} sheets ${sheets}` +
+ (cov !== null ? ` state-coverage ${(cov * 100).toFixed(0)}%` : "") +
+ (ms ? ` select ${ms.toFixed(0)}ms` : "")
+ );
+ csv.push(
+ `${f.slug},${name.split(" ")[0]},${cands.length},${kept.length},${sheets},` +
+ `${cov !== null ? cov.toFixed(3) : ""},${ms ? ms.toFixed(0) : ""},${extractMs.toFixed(0)}`
+ );
+ }
+}
+await writeFile(path.join(OUT, "stats.csv"), csv.join("\n") + "\n");
+console.log(`\nstats: ${path.join(OUT, "stats.csv")}`);
+console.log(`reports: ${OUT}/<slug>/report.html`);
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.ts
index eeee5ebb..43274c61 100644
--- a/site/src/cut/components/AiPanel.tools.ts
+++ b/site/src/cut/components/AiPanel.tools.ts
@@ -19,13 +19,13 @@ export const AI_PANEL_TOOLS = [
{
name: "watch_video",
description:
- "Watch a video source with your own eyes: samples its frames at scene changes plus a steady floor into timestamped contact-sheet images, and returns the detected scene-change times (natural cut candidates). Pass clip_id to watch a timeline clip's source (the result includes that clip's source↔timeline time math) or asset_id for any project video or image. The stamp burned into each cell is SOURCE seconds — what trim_clip's in/out use — not timeline seconds. Coverage is capped per call: survey the whole range first, then call again with a narrow from/to and a small interval_seconds where the cut needs care; the result says where coverage stopped. Read the watching-and-cutting skill before editing footage by content.",
+ "Watch a video source with your own eyes: samples candidate frames on a dense steady floor, keeps only the ones that actually differ, and tiles them into timestamped contact-sheet images — so every cell is a distinct moment and a gap between stamps means nothing changed there. Also returns sceneChanges, hard-cut times refined to about a third of a second of the true boundary (the time is where the new shot first appears) — though a shot shorter than the sampling step can fall between candidates, so where short cuts matter, re-watch narrow with a small interval_seconds. When the clip has captions or the source's own transcript exists, a fused timeline places each kept frame inside the speech it belongs to. Pass clip_id to watch a timeline clip's source (the result includes that clip's source↔timeline time math) or asset_id for any project video or image. The stamp burned into each cell is SOURCE seconds — what trim_clip's in/out use — not timeline seconds. Coverage ends where the result says it does: when truncated is true (span bound, frame cap, or time budget), continue from coveredTo — everything before it was seen, nothing after it was. Watching builds the source's frame map (moment times + cut candidates), persisted on the asset; media entries list those spans as `mapped`, and your first watch starts a quiet background sweep that maps the rest (`watching: true` while it runs). The map aims your watches — a mapped span is NOT footage you have seen; only sheets returned in this conversation are, so watch any span you need to actually look at. Read the watching-and-cutting skill before editing footage by content.",
inputSchema: obj({
clip_id: str("Video clip id, track 0 or overlay (defaults from/to to its trimmed in/out)"),
asset_id: str("Project asset id (video or image) — watch the source itself"),
from: num("Source start s (default: the clip's in, else 0)"),
to: num("Source end s (default: the clip's out, else the source's end; spans at most 600s per call)"),
- interval_seconds: num("Target seconds between sampled frames, 0.5–30 (default spreads ~32 frames across the range)"),
+ interval_seconds: num("Seconds between candidate frames, 0.5–30 (default 1; near-duplicates are dropped, so a dense default costs nothing)"),
}),
},
{
@@ -44,7 +44,7 @@ export const AI_PANEL_TOOLS = [
{
name: "listen_audio",
description:
- "Listen with your own ears to a project asset's sound — an audio asset, or the audio track of a video (its speech, music, burned-in narration) — so you can answer what it says or how it sounds. The sound rides back inline (≈12MB cap). Pass clip_id for a timeline clip's source (scopes to its trim) or asset_id for the whole source; add from/to (source seconds) to hear one stretch of a long file. Audio the user attached to their message already plays in it. To WRITE a caption track, use subtitles_generate.",
+ "Listen with your own ears to a project asset's sound — an audio asset, or the audio track of a video (its speech, music, burned-in narration) — so you can answer what it says or how it sounds. For the WORDS alone, check the source's transcript first: media entries marked speech: 'transcribed' carry timed segments (get_state includes them) at no audio cost; listen when you need the sound itself — tone, music, delivery, timing by ear. The sound rides back inline (≈12MB cap). Pass clip_id for a timeline clip's source (scopes to its trim) or asset_id for the whole source; add from/to (source seconds) to hear one stretch of a long file. Audio the user attached to their message already plays in it. To WRITE a caption track, use subtitles_generate.",
inputSchema: obj({
clip_id: str("Clip id — video or soundtrack (defaults from/to to its trimmed in/out)"),
asset_id: str("Project asset id (audio or video) — listen to the whole source"),
diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.ts
index dd52ef7f..1d98dbc7 100644
--- a/site/src/cut/lib/aiContext.ts
+++ b/site/src/cut/lib/aiContext.ts
@@ -5,6 +5,7 @@ import { chatOwner } from "./chatAssets";
import { useGenerate } from "./generate";
import { getClipSpans, overlayLayers, totalDuration, useEditor } from "./store";
import { laneCues, subtitleLaneCount } from "./subtitles";
+import { watchSweepActive } from "./watch/sweep";
import { frameOf, rectOf, regionLabel, type ClipSpan, type Overlay, type VideoClip } from "./types";
const r = (n: number) => Math.round(n * 100) / 100;
@@ -135,6 +136,32 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nu
type: a.type,
duration: r(a.duration),
...(a.origin ? { origin: a.origin } : {}),
+ // Source spans whose frame map exists (persisted with the project):
+ // distinct-moment times and cut candidates recorded by an earlier watch
+ // or by the background sweep. The map aims a watch — it is not seen
+ // footage; only sheets returned in this chat are. `watching` marks the
+ // sweep still filling the rest; the spans grow as segments land.
+ ...(a.watch && a.watch.ranges.length > 0
+ ? { mapped: a.watch.ranges.map((rg) => ({ from: r(rg.from), to: r(rg.to) })) }
+ : {}),
+ ...(watchSweepActive(a.id) ? { watching: true } : {}),
+ // The source's own transcript (built quietly by the sweep; no subtitle
+ // track involved). The snapshot carries the verdict; get_state carries
+ // the segments — read those for the words instead of inlining audio.
+ ...(a.speech
+ ? {
+ speech: a.speech.noSpeech ? "none" : "transcribed",
+ ...(opts?.fullCues && a.speech.segments.length > 0
+ ? {
+ transcript: a.speech.segments.map((sg) => ({
+ start: r(sg.start),
+ end: r(sg.end),
+ text: sg.text,
+ })),
+ }
+ : {}),
+ }
+ : {}),
})),
mediaTruncated: visibleAssets.length > cueCap,
// AI video renders for this project, live from the job store — what
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 5e44ce73..fdb380ce 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -59,9 +59,10 @@ import {
importImage,
importStockVideo,
importUrlMedia,
- makeContactSheetsClientSide,
- makeStillSheetClientSide,
+ composeSheets,
+ makeStillFrame,
renderAudioSpanWav,
+ sampleWatchFrames,
} from "./media";
import { isLottieAsset } from "./lottieAssets";
import { BREATH, REACH, refineEdge, type SilenceSpan } from "./cutRefine";
@@ -74,6 +75,9 @@ import { applyOverlayPatchSettled, track0Clips, laneGapAt, getClipSpans, nextFre
import { buildAiContext } from "./aiContext";
import { sampleClipFrameData } from "./previewCanvas";
import { laneCues, subtitleLaneCount } from "./subtitles";
+import { fuseTimeline, renderFusedTimeline } from "./watch/fuse";
+import { mergeWatch } from "./watch/merge";
+import { queueWatchSweep, withSweepPaused } from "./watch/sweep";
import { synthesizeMusic } from "./audioGen";
import { composeMusicPrompt } from "./composeGen";
import { stockAssetInDoc } from "./genvideo/docWriter";
@@ -510,78 +514,93 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
},
watch_video: async (s, input) => {
- const { projectId, asset, clip, speed, from, to } = resolveWatchRange(s, input);
+ const { asset, clip, speed, from, to } = resolveWatchRange(s, input);
if (asset.type === "audio")
throw new ToolError(`"${asset.name}" is audio — listen_audio hears it, detect_silence finds its dead air.`);
- interface WatchBody {
- sheets: { image: string; frames: { t: number; scene?: number }[] }[];
- layout: { grid: number; margin: number; padding: number };
- sceneChanges: number[];
- coveredTo: number;
- truncated: boolean;
- error?: string;
- }
- // A still is one sheet with no time axis.
+ // One path for both backends: the browser decodes the source (the same
+ // URL the preview plays) and the shared selector keeps distinct frames.
if (asset.type === "image") {
- let body: WatchBody;
- if (getBackend().kind === "cloud") {
- body = await makeStillSheetClientSide(asset.url).catch((e) => {
- throw new ToolError(e instanceof Error ? e.message : "Could not read the image.");
- });
- } else {
- const res = await apiFetch(`/api/cut/projects/${projectId}/watch`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ file: asset.fileName, still: true }),
- });
- body = await apiJson<WatchBody>(res);
- if (!res.ok) throw new ToolError(body.error ?? "Could not read the image.");
- }
+ const body = await makeStillFrame(asset.url).catch((e) => {
+ throw new ToolError(e instanceof Error ? e.message : "Could not read the image.");
+ });
return {
- images: body.sheets.map((sh) => sh.image),
+ images: [body.frames[0].image],
source: { assetId: asset.id, name: asset.name },
note: "A still image — one frame, no time axis.",
};
}
- let body: WatchBody;
- if (getBackend().kind === "cloud") {
- // Same defaults, clamps, and caps as the engine's watch handler.
- body = await makeContactSheetsClientSide(asset.url, {
+ // The sweep yields its decoders to this call for its whole duration.
+ // The budget keeps the call inside the tool bridge's 120s deadline: a
+ // slow decode salvages what it covered (truncated + coveredTo) rather
+ // than timing out with nothing.
+ const body = await withSweepPaused(() =>
+ sampleWatchFrames(asset.url, {
from,
...(to !== undefined ? { to } : {}),
...(isNum(input.interval_seconds) ? { interval: input.interval_seconds } : {}),
- }).catch((e) => {
- throw new ToolError(e instanceof Error ? e.message : "Could not watch the video.");
- });
- } else {
- const res = await apiFetch(`/api/cut/projects/${projectId}/watch`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({
- file: asset.fileName,
- from,
- ...(to !== undefined ? { to } : {}),
- ...(isNum(input.interval_seconds) ? { interval: input.interval_seconds } : {}),
- }),
- });
- body = await apiJson<WatchBody>(res);
- if (!res.ok) throw new ToolError(body.error ?? "Could not watch the video.");
+ budgetMs: 90_000,
+ })
+ ).catch((e) => {
+ throw new ToolError(e instanceof Error ? e.message : "Could not watch the video.");
+ });
+ // The metadata persists on the asset (it saves with the project and
+ // dies with the asset), so what has been seen survives this thread.
+ // Merge against the asset's CURRENT record — a sweep segment may have
+ // landed while this call was decoding, and a merge computed from the
+ // pre-decode snapshot would overwrite it.
+ const st = useEditor.getState();
+ const liveAsset = st.assets.find((x) => x.id === asset.id);
+ if (liveAsset) {
+ // A pass that covered nothing records nothing.
+ if (body.coveredTo > from) {
+ st.updateAsset(asset.id, {
+ watch: mergeWatch(liveAsset.watch, {
+ from,
+ to: body.coveredTo,
+ frames: body.frames.map((f) => ({ t: f.t, via: f.via })),
+ sceneChanges: body.sceneChanges,
+ }),
+ });
+ }
+ // A first look queues the quiet background sweep of the rest of the
+ // source — metadata only, merged in as segments land.
+ queueWatchSweep(asset.id);
}
- // The engine's ffmpeg has no text renderer; the cells get their source-
- // time stamps here on a canvas. A sheet that fails to stamp rides plain —
- // sheetFrames stays the authority either way.
- let stamped = true;
- const images = await Promise.all(
- body.sheets.map((sh) =>
- stampSheet(sh.image, sh.frames.map((f) => f.t), body.layout).catch(() => {
- stamped = false;
- return sh.image;
- })
- )
- );
+ // Tile the kept frames 3×3 and stamp each cell's source time.
+ const sheets = await composeSheets(body.frames).catch(() => null);
+ const keptTimes = body.frames.map((f) => f.t);
+ // Kept frames woven into the speech on one clock, so the model reads
+ // precomputed frame↔speech alignment. Project captions win (cue times
+ // are timeline seconds, mapped to source through the clip); with none,
+ // the asset's own transcript serves — it is already source time, so it
+ // fuses for asset-only watches too.
+ let speech: { start: number; end: number; text: string }[] = [];
+ if (clip) {
+ speech = laneCues(s.subtitles, s.subtitleLane)
+ .map((c) => ({
+ start: clip.in + (c.start - clip.start) * speed,
+ end: clip.in + (c.end - clip.start) * speed,
+ text: c.text,
+ }))
+ .filter((c) => c.end > from && c.start < body.coveredTo);
+ }
+ if (speech.length === 0) speech = (liveAsset ?? asset).speech?.segments ?? [];
+ let timelineText: string | undefined;
+ if (speech.length > 0) {
+ timelineText = renderFusedTimeline(
+ fuseTimeline(keptTimes, speech, { from, to: body.coveredTo })
+ ).slice(0, 4000);
+ }
+ // Each cell's time plus why the selector kept it — "global" a hard cut,
+ // "action" local motion, "settled" new settled detail (text, ink, UI).
+ const cellInfo = body.frames.map((f) => ({ t: round2(f.t), via: f.via }));
+ const perSheet = sheets ? sheets.map((sh) => sh.frames.length) : [cellInfo.length];
+ let cellAt = 0;
return {
- images,
- sheetFrames: body.sheets.map((sh) => sh.frames.map((f) => round2(f.t))),
+ images: sheets ? sheets.map((sh) => sh.image) : body.frames.map((f) => f.image),
+ sheetFrames: perSheet.map((n) => cellInfo.slice(cellAt, (cellAt += n))),
+ distinctFrames: body.frames.length,
+ candidates: body.candidates,
sceneChanges: body.sceneChanges.map(round2),
coveredTo: round2(body.coveredTo),
truncated: body.truncated,
@@ -598,12 +617,15 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
},
}
: {}),
+ ...(timelineText ? { timeline: timelineText } : {}),
note:
- "Cells read left→right then top→bottom; each stamp is SOURCE seconds." +
+ "Cells read left→right then top→bottom; each stamp is SOURCE seconds. " +
+ "Cells are distinct moments (near-duplicates removed), so gaps between stamps mean nothing changed there. " +
+ "sheetFrames says why each cell was kept: global = hard cut, action = local motion, settled = new settled detail (text/UI)." +
(body.truncated
? ` Coverage stopped at ${round2(body.coveredTo)}s — call again with from=${round2(body.coveredTo)} to continue.`
: "") +
- (stamped ? "" : " (Stamps unavailable — sheetFrames lists each cell's time.)"),
+ (sheets ? "" : " (Sheets unavailable — each image is one frame; sheetFrames lists the times.)"),
};
},
@@ -658,6 +680,9 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const inline = wholeAudio
? await refToInlineAudio(refFromAsset(asset))
: await listenToSource(projectId, asset, from, to);
+ // Listening shows interest in the source's sound — queue the background
+ // sweep so its transcript (and, for video, its visual map) fills in.
+ queueWatchSweep(asset.id);
if (!inline)
throw new ToolError(
"That stretch is too long to listen to inline (≈12MB cap) — pass a narrower from/to."
@@ -2349,45 +2374,6 @@ async function synthesizeVoiceover(
};
}
-/** Draw each cell's source-time stamp onto a contact sheet. The bundled
- * ffmpeg ships without a text renderer (LGPL build, no freetype), so the
- * stamps land here, where a canvas always can. */
-async function stampSheet(
- image: string,
- times: number[],
- layout: { grid: number; margin: number; padding: number }
-): Promise<string> {
- const img = new Image();
- await new Promise<void>((resolve, reject) => {
- img.onload = () => resolve();
- img.onerror = () => reject(new Error("Bad sheet image."));
- img.src = image;
- });
- const canvas = document.createElement("canvas");
- canvas.width = img.naturalWidth;
- canvas.height = img.naturalHeight;
- const ctx = canvas.getContext("2d");
- if (!ctx) throw new Error("No 2d context.");
- ctx.drawImage(img, 0, 0);
- const { grid, margin, padding } = layout;
- const cellW = (canvas.width - 2 * margin - (grid - 1) * padding) / grid;
- const cellH = (canvas.height - 2 * margin - (grid - 1) * padding) / grid;
- const size = Math.max(13, Math.round(cellH / 12));
- ctx.font = `bold ${size}px ui-monospace, monospace`;
- ctx.textBaseline = "bottom";
- times.forEach((t, i) => {
- const x = margin + (i % grid) * (cellW + padding);
- const y = margin + Math.floor(i / grid) * (cellH + padding);
- const label = `${round2(t)}s`;
- const w = ctx.measureText(label).width;
- ctx.fillStyle = "rgba(0,0,0,0.6)";
- ctx.fillRect(x + 4, y + cellH - size - 10, w + 10, size + 8);
- ctx.fillStyle = "#fff";
- ctx.fillText(label, x + 9, y + cellH - 6);
- });
- return canvas.toDataURL("image/jpeg", 0.8);
-}
-
/** Resolve a watch/listen target: a timeline clip (its source plus trim and
* placement) or a bare project asset. */
function resolveWatchTarget(
diff --git a/site/src/cut/lib/localStt.ts b/site/src/cut/lib/localStt.ts
index 851bfb43..81003652 100644
--- a/site/src/cut/lib/localStt.ts
+++ b/site/src/cut/lib/localStt.ts
@@ -12,10 +12,33 @@ import type { SubtitleCue } from "./types";
const POLL_MS = 700;
+/** The engine transcribes one job at a time, so every caller in this client
+ * takes its turn here instead of racing for the slot and failing with
+ * "already running" — a user's caption run simply waits out the background
+ * sweep's current chunk, and the sweep waits out the user. */
+let sttTurn: Promise<void> = Promise.resolve();
+export function withEngineStt<T>(work: () => Promise<T>): Promise<T> {
+ const run = sttTurn.then(work, work);
+ sttTurn = run.then(
+ () => undefined,
+ () => undefined
+ );
+ return run;
+}
+
/** Transcribe a rendered mix on this Mac. Returns null when `isStale` trips
* (the caller moved on); throws when the engine can't do it, which is the
* caller's cue to fall back to the hosted route. */
-export async function engineTranscribeSamples(
+export function engineTranscribeSamples(
+ samples: Float32Array,
+ duration: number,
+ locale: string | undefined,
+ isStale?: () => boolean
+): Promise<SubtitleCue[] | null> {
+ return withEngineStt(() => engineTranscribe(samples, duration, locale, isStale));
+}
+
+async function engineTranscribe(
samples: Float32Array,
duration: number,
locale: string | undefined,
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index a0b34d19..1c603bf0 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -19,8 +19,11 @@ import {
videoTrackOf,
} from "./mediaRead";
import { useEditor } from "./store";
-import type { AssetType, AudioClip, MediaAsset, ProjectSummary, StoredAsset, VideoClip } from "./types";
+import type { AssetType, AudioClip, MediaAsset, ProjectSummary, StoredAsset, VideoClip, WatchKeepReason } from "./types";
import { IMAGE_CLIP_SECONDS, mediaUrl } from "./types";
+import { createDedupSelector, DEDUP_TUNING } from "./watch/dedupSelector";
+import { diffCellPct, frameSig } from "./watch/signatures";
+import { SIGNATURE_SIZE } from "./watch/types";
const uid = () => crypto.randomUUID().slice(0, 8);
@@ -825,18 +828,24 @@ export async function renderAudioSpanWav(
return encodeWav((await ctx.startRendering()).getChannelData(0));
}
-// The engine's contact-sheet geometry (server/frames.ts), mirrored exactly so
-// the tool's stampSheet lands each cell's time stamp in the same place in
-// both modes.
+// Watch geometry: cells at a fixed long side, tiled and stamped by
+// composeSheets. One sampler serves the local and cloud backends alike — the
+// browser decodes the source (local media rides the engine's media URL, cloud
+// media the CDN) and the shared selector keeps the frames that differ.
const SHEET_GRID = 3; // cells per row and column
const SHEET_CELL = 480; // cell long side, px
const SHEET_GAP = 4; // tile margin and padding, px
const SHEET_MAX = 4; // sheets per call
const SHEET_QUALITY = 0.8; // jpeg encode
-
-export interface WatchSheets {
- sheets: { image: string; frames: { t: number }[] }[];
- layout: { grid: number; margin: number; padding: number };
+const CANDIDATE_CAP = 150; // decodes per call — the densest sweep
+const REFINE_TARGET_S = 0.35; // localize each hard cut to about this window
+const REFINE_MAX_PROBES = 48; // extra seeks the refinement pass may spend per call
+
+/** The watch sample: kept frames, ascending source time, each carrying why
+ * the selector kept it (WatchKeepReason in types.ts). */
+export interface WatchFrames {
+ frames: { t: number; image: string; via: WatchKeepReason }[];
+ candidates: number;
sceneChanges: number[];
coveredTo: number;
truncated: boolean;
@@ -844,16 +853,14 @@ export interface WatchSheets {
const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));
-// The engine scales with ffmpeg's scale=480:-2, which rounds the short side
-// to even; mirror it so cell geometry matches to the pixel.
+// Round the cell's short side to even, so encoded dimensions stay codec-safe.
const cellDims = (w: number, h: number): [number, number] =>
w >= h
? [SHEET_CELL, Math.max(2, 2 * Math.round((SHEET_CELL * h) / w / 2))]
: [Math.max(2, 2 * Math.round((SHEET_CELL * w) / h / 2)), SHEET_CELL];
-/** Cloud twin of the engine's watch route for a still image: one downscaled
- * cell, no time axis. */
-export async function makeStillSheetClientSide(sourceUrl: string): Promise<WatchSheets> {
+/** Watch a still image: one downscaled frame, no time axis. */
+export async function makeStillFrame(sourceUrl: string): Promise<WatchFrames> {
const img = new Image();
img.crossOrigin = "anonymous";
await new Promise<void>((resolve, reject) => {
@@ -872,23 +879,43 @@ export async function makeStillSheetClientSide(sourceUrl: string): Promise<Watch
ctx.imageSmoothingQuality = "high";
ctx.drawImage(img, 0, 0, w, h);
return {
- sheets: [{ image: canvas.toDataURL("image/jpeg", SHEET_QUALITY), frames: [{ t: 0 }] }],
- layout: { grid: 1, margin: 0, padding: 0 },
+ frames: [{ t: 0, image: canvas.toDataURL("image/jpeg", SHEET_QUALITY), via: "first" }],
+ candidates: 1,
sceneChanges: [],
coveredTo: 0,
truncated: false,
};
}
-/** Cloud twin of the engine's watch route: seek the source in the browser and
- * tile downscaled frames into timestamped contact sheets, with the route's
- * defaults, clamps, and per-call caps. Frames land on the steady interval
- * only — scene detection needs a decoder — so sceneChanges stays empty and
- * scene fields are omitted. */
-export async function makeContactSheetsClientSide(
+/** Watch a video source: decode candidates on a dense steady floor, then let
+ * the shared selector keep the frames that actually differ. sceneChanges are
+ * hard-cut times: each "global" keep says a cut lies inside the grid step
+ * before it, and a short bisection pass narrows that window to about
+ * REFINE_TARGET_S — the reported time is where the new shot is first seen.
+ * A shot shorter than the candidate step can still fall between samples; a
+ * narrow re-watch with a small interval is the recall pass.
+ *
+ * Coverage is honest by construction: the keep-cap, a pause, and the time
+ * budget all END the pass (coveredTo says where) instead of thinning or
+ * guessing — everything inside [from, coveredTo] was decoded and judged, and
+ * every kept frame in it is returned. A source that decodes no frames at all
+ * throws; it is never reported as watched.
+ *
+ * metadataOnly skips the cell copies and JPEG encodes — frames come back with
+ * an empty image — for the background sweep, which only wants the numbers.
+ * shouldPause is polled per candidate; budgetMs bounds the whole call the
+ * same way, for callers racing a tool deadline. */
+export async function sampleWatchFrames(
sourceUrl: string,
- opts: { from: number; to?: number; interval?: number }
-): Promise<WatchSheets> {
+ opts: {
+ from: number;
+ to?: number;
+ interval?: number;
+ metadataOnly?: boolean;
+ shouldPause?: () => boolean;
+ budgetMs?: number;
+ }
+): Promise<WatchFrames> {
const input = openMedia(sourceUrl);
try {
const track = await videoTrackOf(input);
@@ -899,79 +926,214 @@ export async function makeContactSheetsClientSide(
throw new Error("Could not read the media duration — pass to (seconds).");
if (!(wanted > from)) throw new Error("from/to describe an empty range.");
const to = Math.min(wanted, from + 600); // bound the work per call; callers resume from coveredTo
- const interval =
- opts.interval !== undefined
- ? clamp(opts.interval, 0.5, 30)
- : clamp((to - from) / 32, 2, 30);
-
- const perSheet = SHEET_GRID * SHEET_GRID;
+ const interval = clamp(opts.interval ?? 1, 0.5, 30);
+ // The candidate floor: as asked, widened only when the range would blow
+ // the per-call decode budget.
+ const step = Math.max(interval, (to - from) / CANDIDATE_CAP);
const times: number[] = [];
- for (let t = from; t < to && times.length < SHEET_MAX * perSheet; t += interval)
+ for (let t = from; t < to && times.length < CANDIDATE_CAP; t += step)
times.push(round2(t));
const [cw, ch] = cellDims(await track.getDisplayWidth(), await track.getDisplayHeight());
- const sheetW = 2 * SHEET_GAP + SHEET_GRID * cw + (SHEET_GRID - 1) * SHEET_GAP;
- const sheetH = 2 * SHEET_GAP + SHEET_GRID * ch + (SHEET_GRID - 1) * SHEET_GAP;
- const sheets: WatchSheets["sheets"] = [];
+ const maxFrames = SHEET_MAX * SHEET_GRID * SHEET_GRID;
+ // Cell canvases are held only until the selector rules on them (decisions
+ // run one frame behind), so memory stays near the kept set, never the
+ // candidate sweep.
+ const held = new Map<number, HTMLCanvasElement>();
+ const candTimes: number[] = [];
+ const candG16: Float32Array[] = []; // per-candidate coarse grid, for cut refinement
+ let keptCount = 0;
+ const selector = createDedupSelector({
+ maxFrames,
+ onDecision: (d) => {
+ if (d.verdict === "drop") held.delete(d.index);
+ else keptCount++;
+ },
+ });
+ const sig = document.createElement("canvas");
+ sig.width = SIGNATURE_SIZE;
+ sig.height = SIGNATURE_SIZE;
+ const sigCtx = sig.getContext("2d", { willReadFrequently: true });
+ if (!sigCtx) throw new Error("Could not sample the video.");
+ const deadline = Date.now() + (opts.budgetMs ?? Infinity);
+ const outOfTime = () => Date.now() > deadline;
+ const sigG16At = async (t: number): Promise<Float32Array | null> => {
+ const one = await frameSink(track, { width: cw, height: ch, fit: "fill" })
+ .canvasesAtTimestamps([t])
+ .next();
+ if (one.done || !one.value) return null;
+ sigCtx.imageSmoothingQuality = "high";
+ sigCtx.drawImage(one.value.canvas, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ const data = sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data;
+ return frameSig({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 4, data }).g16;
+ };
- // The times are ascending, so the whole span decodes once, in order, and
- // each cell is drawn as its frame comes past.
+ // The times are ascending, so the whole span decodes once, in order.
const cells = frameSink(track, { width: cw, height: ch, fit: "fill" }).canvasesAtTimestamps(
times
);
- let ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null = null;
- let canvas: HTMLCanvasElement | null = null;
- // Only the cells that actually got a frame are reported. A cell the decoder
- // had nothing for stays black, and listing its timestamp anyway would tell
- // the model there is a picture of that moment on the sheet — it would then
- // describe a black tile as the content at that time, or read every later
- // cell against the wrong one.
- let drawn: { t: number }[] = [];
+ let paused = false;
+ let capped = false;
for (let i = 0; i < times.length; i++) {
- const j = i % perSheet;
- if (j === 0) {
- canvas = document.createElement("canvas");
- canvas.width = sheetW;
- canvas.height = sheetH;
- ctx = canvas.getContext("2d");
- if (!ctx) throw new Error("Could not sample the video.");
- ctx.fillStyle = "#000";
- ctx.fillRect(0, 0, sheetW, sheetH);
- ctx.imageSmoothingQuality = "high";
- drawn = [];
+ if (opts.shouldPause?.() || outOfTime()) {
+ paused = true;
+ break;
+ }
+ // The keep-cap ends the pass — everything reported stays everything
+ // seen, and the caller resumes from coveredTo.
+ if (keptCount >= maxFrames) {
+ capped = true;
+ break;
}
const next = await cells.next();
- if (!next.done && next.value) {
- const x = SHEET_GAP + (j % SHEET_GRID) * (cw + SHEET_GAP);
- const y = SHEET_GAP + Math.floor(j / SHEET_GRID) * (ch + SHEET_GAP);
- ctx!.drawImage(next.value.canvas, x, y, cw, ch);
- drawn.push({ t: times[i] });
+ if (next.done || !next.value) continue; // a moment the decoder had no frame for
+ if (!opts.metadataOnly) {
+ const copy = document.createElement("canvas");
+ copy.width = cw;
+ copy.height = ch;
+ const cctx = copy.getContext("2d");
+ if (!cctx) throw new Error("Could not sample the video.");
+ cctx.drawImage(next.value.canvas, 0, 0);
+ held.set(candTimes.length, copy);
}
- const lastCell = j === perSheet - 1 || i === times.length - 1;
- if (lastCell && drawn.length > 0) {
- sheets.push({
- image: await canvasDataUrl(canvas!, "image/jpeg", SHEET_QUALITY),
- frames: drawn,
- });
+ candTimes.push(times[i]);
+ sigCtx.imageSmoothingQuality = "high";
+ sigCtx.drawImage(next.value.canvas, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ const rgb = {
+ width: SIGNATURE_SIZE,
+ height: SIGNATURE_SIZE,
+ channels: 4 as const,
+ data: sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,
+ };
+ candG16.push(frameSig(rgb).g16);
+ selector.push(rgb);
+ }
+ const selection = selector.finish();
+ // Nothing decoded and nothing interrupted the pass: the source is
+ // unreadable. Say so — never record an unseen span as watched.
+ if (candTimes.length === 0 && !paused) throw new Error("Could not sample the video.");
+ // finish() judges the one pending frame, which can nudge past the cap.
+ const keptIdx = selection.kept.slice(0, maxFrames);
+ const frames: WatchFrames["frames"] = [];
+ for (const idx of keptIdx) {
+ const canvas = held.get(idx);
+ if (!opts.metadataOnly && !canvas) continue;
+ const verdict = selection.decisions[idx].verdict;
+ frames.push({
+ t: candTimes[idx],
+ image: canvas ? await canvasDataUrl(canvas, "image/jpeg", SHEET_QUALITY) : "",
+ via: verdict === "keep-global" ? "global"
+ : verdict === "keep-action" ? "action"
+ : verdict === "keep-settled" ? "settled"
+ : "first",
+ });
+ }
+ held.clear();
+
+ // A "global" keep only says the cut lies in the grid step before it.
+ // Bisect that step with a few extra decodes so each reported cut time
+ // sits within REFINE_TARGET_S of the first new-shot frame; the kept
+ // image stays the grid sample.
+ let probes = 0;
+ const sceneChanges: number[] = [];
+ for (const idx of keptIdx) {
+ if (selection.decisions[idx].verdict !== "keep-global") continue;
+ let lo = idx > 0 ? candTimes[idx - 1] : from;
+ let hi = candTimes[idx];
+ const pre = idx > 0 ? candG16[idx - 1] : null;
+ while (
+ pre !== null &&
+ hi - lo > REFINE_TARGET_S &&
+ probes < REFINE_MAX_PROBES &&
+ !outOfTime() &&
+ !opts.shouldPause?.()
+ ) {
+ probes++;
+ const mid = (lo + hi) / 2;
+ const g = await sigG16At(mid);
+ if (!g) break;
+ if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) hi = mid;
+ else lo = mid;
}
+ sceneChanges.push(round2(hi));
}
- const lastT = times.length > 0 ? times[times.length - 1] : from;
- const capped = times.length >= SHEET_MAX * perSheet && lastT < to - interval;
- // The per-call span bound is itself truncation — the caller asked for more.
- const truncated = capped || to < wanted;
+ // An interrupted or capped scan only covered what it decoded and kept;
+ // the caller merges that much and comes back for the rest.
+ const lastKeptT = keptIdx.length > 0 ? candTimes[keptIdx[keptIdx.length - 1]] : from;
+ const coveredTo = paused
+ ? (candTimes[candTimes.length - 1] ?? from)
+ : capped
+ ? lastKeptT
+ : to;
return {
- sheets,
- layout: { grid: SHEET_GRID, margin: SHEET_GAP, padding: SHEET_GAP },
- sceneChanges: [],
- coveredTo: capped ? lastT : to,
- truncated,
+ frames,
+ candidates: selection.candidateCount,
+ sceneChanges,
+ coveredTo,
+ // The per-call span bound is itself truncation — the caller asked for more.
+ truncated: paused || capped || to < wanted,
};
} finally {
input.dispose();
}
}
+/** Tile kept frames into 3×3 contact sheets and stamp each cell's source
+ * time. The last sheet may run short; its frames list only the real cells. */
+export async function composeSheets(
+ cells: { t: number; image: string }[]
+): Promise<{ image: string; frames: { t: number }[] }[]> {
+ if (cells.length === 0) return [];
+ const images = await Promise.all(
+ cells.map(
+ (c) =>
+ new Promise<HTMLImageElement>((resolve, reject) => {
+ const img = new Image();
+ img.onload = () => resolve(img);
+ img.onerror = () => reject(new Error("Bad frame image."));
+ img.src = c.image;
+ })
+ )
+ );
+ const cw = images[0].naturalWidth;
+ const ch = images[0].naturalHeight;
+ const perSheet = SHEET_GRID * SHEET_GRID;
+ const sheetW = 2 * SHEET_GAP + SHEET_GRID * cw + (SHEET_GRID - 1) * SHEET_GAP;
+ const sheetH = 2 * SHEET_GAP + SHEET_GRID * ch + (SHEET_GRID - 1) * SHEET_GAP;
+ const size = Math.max(13, Math.round(ch / 12));
+ const sheets: { image: string; frames: { t: number }[] }[] = [];
+ for (let s = 0; s < cells.length; s += perSheet) {
+ const batch = cells.slice(s, s + perSheet);
+ const canvas = document.createElement("canvas");
+ canvas.width = sheetW;
+ canvas.height = sheetH;
+ const ctx = canvas.getContext("2d");
+ if (!ctx) throw new Error("No 2d context.");
+ ctx.fillStyle = "#000";
+ ctx.fillRect(0, 0, sheetW, sheetH);
+ ctx.imageSmoothingQuality = "high";
+ ctx.font = `bold ${size}px ui-monospace, monospace`;
+ ctx.textBaseline = "bottom";
+ batch.forEach((c, j) => {
+ const x = SHEET_GAP + (j % SHEET_GRID) * (cw + SHEET_GAP);
+ const y = SHEET_GAP + Math.floor(j / SHEET_GRID) * (ch + SHEET_GAP);
+ ctx.drawImage(images[s + j], x, y, cw, ch);
+ const label = `${round2(c.t)}s`;
+ const w = ctx.measureText(label).width;
+ ctx.fillStyle = "rgba(0,0,0,0.6)";
+ ctx.fillRect(x + 4, y + ch - size - 10, w + 10, size + 8);
+ ctx.fillStyle = "#fff";
+ ctx.fillText(label, x + 9, y + ch - 6);
+ });
+ sheets.push({
+ image: await canvasDataUrl(canvas, "image/jpeg", SHEET_QUALITY),
+ frames: batch.map((c) => ({ t: c.t })),
+ });
+ }
+ return sheets;
+}
+
/** Generate filmstrip thumbnails / waveform peaks and merge them into the
* store. Safe to call repeatedly; skips assets that are already enriched.
* `src` overrides where the frames are read from — an import still uploading
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index 214cb090..f1a1cfbf 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -55,7 +55,7 @@ import {
import { renderMix, transcribeSamples, type CloudTranscribeSpec } from "./cloudTranscribe";
import { alignCues } from "./cueAlign";
import { useGenNotify } from "./genNotify";
-import { engineTranscribeSamples } from "./localStt";
+import { engineTranscribeSamples, withEngineStt } from "./localStt";
import { trackLocale } from "./subtitles";
import { ANIM_STYLE_IDS, animStyleOfTransition, clipPoseAt, emptySubtitles, frameOf, IMAGE_CLIP_SECONDS, isEffectOverlay, isStickerOverlay, MAX_SUBTITLE_LANES, mediaUrl, migrateBehindSubject, migrateLegacyTransitions, normalizeAspect, overlayAnimStyle, SPEED_FLOOR, SPEED_MIN, stampOverlayKinds, stripDefaultOverlayKinds, TRANSITION_MAX, TRANSITION_STYLE_IDS, transitionStyleOfAnim } from "./types";
import { readTextStyle } from "./textStyle";
@@ -910,7 +910,8 @@ export async function runTranscription(projectId: string, spec: object): Promise
// A cloud project's media isn't on this Mac, so the engine can't render the
// mix — the browser does, and then hands it to whoever can transcribe it:
// this Mac when the app is running (on-device, free, real word timings), the
- // metered hosted route otherwise. A local project skips all of it; the
+ // hosted route otherwise (included up to the account's allowance, metered
+ // past it). A local project skips all of it; the
// engine already holds the media and runs the whole job.
if (getBackend().kind === "cloud") {
const s = spec as CloudTranscribeSpec;
@@ -935,24 +936,29 @@ export async function runTranscription(projectId: string, spec: object): Promise
const cues = await transcribeSamples(samples, s.locale, stale);
return cues && alignCues(cues, samples, mix.sampleRate, { snap: 0.6 });
}
- const res = await apiFetch(`/api/cut/projects/${projectId}/transcribe`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify(spec),
- });
- const body = await apiJson<{ id?: string }>(res);
- if (!res.ok || !body.id) throw new Error(body.error ?? "Transcription failed to start.");
- for (;;) {
- await new Promise((r) => setTimeout(r, 600));
- if (useEditor.getState().projectId !== projectId) return null;
- const st = await apiFetch(`/api/cut/projects/${projectId}/transcribe?job=${body.id}`);
- if (!st.ok) throw new Error("The transcription job was lost — try again.");
- const status = (await st.json()) as { status: string; error?: string; cues?: SubtitleCue[] };
- if (status.status === "error") throw new Error(status.error ?? "Transcription failed.");
- if (status.status === "done") {
- return useEditor.getState().projectId === projectId ? (status.cues ?? []) : null;
+ // The engine runs one transcription at a time; the client-side turn queue
+ // keeps this job from colliding with a background sweep chunk (or vice
+ // versa) and failing on the busy slot.
+ return withEngineStt(async () => {
+ const res = await apiFetch(`/api/cut/projects/${projectId}/transcribe`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(spec),
+ });
+ const body = await apiJson<{ id?: string }>(res);
+ if (!res.ok || !body.id) throw new Error(body.error ?? "Transcription failed to start.");
+ for (;;) {
+ await new Promise((r) => setTimeout(r, 600));
+ if (useEditor.getState().projectId !== projectId) return null;
+ const st = await apiFetch(`/api/cut/projects/${projectId}/transcribe?job=${body.id}`);
+ if (!st.ok) throw new Error("The transcription job was lost — try again.");
+ const status = (await st.json()) as { status: string; error?: string; cues?: SubtitleCue[] };
+ if (status.status === "error") throw new Error(status.error ?? "Transcription failed.");
+ if (status.status === "done") {
+ return useEditor.getState().projectId === projectId ? (status.cues ?? []) : null;
+ }
}
- }
+ });
}
// Doc-mutating state: in a read-only shared view the set wrapper drops these
@@ -4071,7 +4077,7 @@ export const docOverlays = (() => {
export function storedAssets(assets: MediaAsset[]): StoredAsset[] {
return assets
.filter((a) => !a.upload)
- .map(({ id, fileName, name, type, duration, width, height, origin, chatId, language }) => ({
+ .map(({ id, fileName, name, type, duration, width, height, origin, chatId, language, watch, speech }) => ({
id,
fileName,
name,
@@ -4082,6 +4088,8 @@ export function storedAssets(assets: MediaAsset[]): StoredAsset[] {
...(origin !== undefined ? { origin } : {}),
...(chatId !== undefined ? { chatId } : {}),
...(language !== undefined ? { language } : {}),
+ ...(watch !== undefined ? { watch } : {}),
+ ...(speech !== undefined ? { speech } : {}),
}));
}
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index 9baf509f..fd50db1e 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -118,6 +118,39 @@ export function nearestAspect<T extends string>(aspect: Aspect, supported: reado
}
/** Asset fields persisted in project.json. */
+/** Why watch_video kept a frame: "first" opens the range, "global" is a
+ * whole-frame change (a hard cut), "action" is hard local motion (a small
+ * subject), "settled" is new settled detail (text, ink, UI). */
+export type WatchKeepReason = "first" | "global" | "action" | "settled";
+
+/** What the assistant has seen of a source: watch_video's kept frames and
+ * detected cuts, merged across the watched spans. Times are source seconds.
+ * Media files are immutable per fileName, so this never goes stale; it lives
+ * on the asset, so it saves with the project and dies with the asset. */
+export interface AssetWatch {
+ /** Watched source spans, merged and ascending. */
+ ranges: { from: number; to: number }[];
+ /** Kept distinct frames, ascending, each with why it was kept. */
+ frames: { t: number; via: WatchKeepReason }[];
+ /** Hard-cut moments among the kept frames. */
+ sceneChanges: number[];
+}
+
+/** A source's own transcript — working data for the assistant, kept apart
+ * from subtitle tracks (those are user-visible and only written on request).
+ * Times are source seconds. Same lifecycle as AssetWatch: saves with the
+ * project, dies with the asset, never stale (media files are immutable). */
+export interface AssetSpeech {
+ /** Transcribed source spans, merged and ascending. */
+ ranges: { from: number; to: number }[];
+ /** Timed speech segments, ascending. */
+ segments: { start: number; end: number; text: string }[];
+ /** Set once the whole source is covered and no speech was heard — a
+ * result, so silent sources are never re-transcribed. */
+ noSpeech?: true;
+ locale?: string;
+}
+
export interface StoredAsset {
id: string;
fileName: string; // file inside the project's media/ folder
@@ -126,6 +159,10 @@ export interface StoredAsset {
duration: number; // seconds
width?: number;
height?: number;
+ /** Watch metadata for video sources — what the assistant has seen. */
+ watch?: AssetWatch;
+ /** The source's own transcript — what the assistant has heard. */
+ speech?: AssetSpeech;
/** How this asset entered the project. Absent = the user imported it (drag,
* drop, or upload), so it belongs in the Media panel. Any value marks media
* Cut created or fetched — it lives where it was made (the timeline, a
diff --git a/site/src/cut/lib/watch/dedupSelector.test.ts b/site/src/cut/lib/watch/dedupSelector.test.ts
new file mode 100644
index 00000000..b1257bac
--- /dev/null
+++ b/site/src/cut/lib/watch/dedupSelector.test.ts
@@ -0,0 +1,189 @@
+import { describe, expect, test } from "bun:test";
+
+import { createDedupSelector } from "./dedupSelector";
+import { frameSig, settledCellScores } from "./signatures";
+import { type RgbFrame, SIGNATURE_SIZE } from "./types";
+
+const S = SIGNATURE_SIZE;
+
+/** A solid frame, optionally with painted rectangles. */
+function paint(
+ base: [number, number, number],
+ rects: { x: number; y: number; w: number; h: number; color: [number, number, number] }[] = []
+): RgbFrame {
+ const data = new Uint8Array(S * S * 3);
+ for (let i = 0; i < data.length; i += 3) {
+ data[i] = base[0];
+ data[i + 1] = base[1];
+ data[i + 2] = base[2];
+ }
+ for (const r of rects) {
+ for (let y = r.y; y < r.y + r.h; y++) {
+ for (let x = r.x; x < r.x + r.w; x++) {
+ const i = (y * S + x) * 3;
+ data[i] = r.color[0];
+ data[i + 1] = r.color[1];
+ data[i + 2] = r.color[2];
+ }
+ }
+ }
+ return { width: S, height: S, channels: 3, data };
+}
+
+function select(frames: RgbFrame[], maxFrames = 36) {
+ const sel = createDedupSelector({ maxFrames });
+ for (const f of frames) sel.push(f);
+ return sel.finish();
+}
+
+const GRAY: [number, number, number] = [120, 120, 120];
+const WHITE: [number, number, number] = [255, 255, 255];
+const BLACK: [number, number, number] = [10, 10, 10];
+
+describe("global channel", () => {
+ test("an identical stream keeps only the first frame", () => {
+ const r = select(Array.from({ length: 10 }, () => paint(GRAY)));
+ expect(r.kept).toEqual([0]);
+ expect(r.decisions[0].verdict).toBe("first");
+ expect(r.decisions.slice(1, -1).every((d) => d.verdict === "drop")).toBe(true);
+ expect(r.candidateCount).toBe(10);
+ });
+
+ test("a hard cut keeps the new shot", () => {
+ const r = select([paint(BLACK), paint(BLACK), paint(WHITE), paint(WHITE)]);
+ expect(r.kept).toEqual([0, 2]);
+ expect(r.decisions[2].verdict).toBe("keep-global");
+ });
+
+ test("the window blocks A-B-A alternation", () => {
+ const a = () => paint(BLACK);
+ const b = () => paint(WHITE);
+ const r = select([a(), b(), a(), b(), a()]);
+ // Both shots enter once; their reappearances match the kept window.
+ expect(r.kept).toEqual([0, 1]);
+ });
+});
+
+describe("action channel", () => {
+ test("a small hard-moving subject keeps its frames", () => {
+ // A 12px dot crossing a static scene: ~0.4% of pixels, far under the
+ // global threshold, but hard change in a few 32-grid cells.
+ const dot = (x: number) =>
+ paint(GRAY, [{ x, y: 90, w: 12, h: 12, color: BLACK }]);
+ const r = select([dot(20), dot(80), dot(140)]);
+ expect(r.kept).toEqual([0, 1, 2]);
+ expect(r.decisions[1].verdict).toBe("keep-action");
+ });
+});
+
+describe("settled channel", () => {
+ // A "caption" lands on a static scene and stays: 1px rows of ink — thin
+ // enough that every cell mean stays inside the global and action
+ // tolerances, so only the settled channel can see it.
+ const caption = (color: [number, number, number]) =>
+ paint(GRAY, [
+ { x: 24, y: 150, w: 144, h: 1, color },
+ { x: 24, y: 156, w: 144, h: 1, color },
+ { x: 24, y: 162, w: 144, h: 1, color },
+ ]);
+
+ test("a caption appearing on a static scene is kept", () => {
+ const r = select([paint(GRAY), paint(GRAY), caption(BLACK), caption(BLACK), caption(BLACK)]);
+ expect(r.kept.includes(2)).toBe(true);
+ const d = r.decisions[2];
+ expect(d.verdict).toBe("keep-settled");
+ expect(d.globalDist).toBeLessThanOrEqual(8);
+ expect(d.settledScore).toBeGreaterThan(d.settledGate!);
+ });
+
+ test("change still in flight waits for the settled state", () => {
+ // The stroke keeps moving every frame; only the final state (nothing
+ // after it to prove motion) can take a settled keep.
+ const moving = (y: number) => paint(GRAY, [{ x: 24, y, w: 144, h: 1, color: BLACK }]);
+ const r = select([paint(GRAY), moving(60), moving(90), moving(120)]);
+ expect(r.decisions[1].verdict).toBe("drop");
+ expect(r.decisions[2].verdict).toBe("drop");
+ expect(r.decisions[3].verdict).toBe("keep-settled");
+ });
+
+ test("soft-contrast drift dies at the strict tolerance", () => {
+ // A faint wash (Δ90: over the soft tolerance, under the strict 105) must
+ // not take a frame — smoke and fades live there.
+ const wash = paint(GRAY, [
+ { x: 24, y: 150, w: 144, h: 1, color: [210, 210, 210] },
+ { x: 24, y: 156, w: 144, h: 1, color: [210, 210, 210] },
+ { x: 24, y: 162, w: 144, h: 1, color: [210, 210, 210] },
+ ]);
+ const r = select([paint(GRAY), paint(GRAY), wash, wash, wash]);
+ expect(r.decisions[2].verdict).toBe("drop");
+ });
+
+ test("the cooldown delays a settled keep right after another", () => {
+ const frames = [paint(GRAY), paint(GRAY), caption(BLACK), caption(BLACK)];
+ // A second small settled change (a new white stroke) lands right after
+ // the caption's keep.
+ const second = paint(GRAY, [
+ { x: 24, y: 150, w: 144, h: 1, color: BLACK },
+ { x: 24, y: 156, w: 144, h: 1, color: BLACK },
+ { x: 24, y: 162, w: 144, h: 1, color: BLACK },
+ { x: 24, y: 30, w: 144, h: 1, color: WHITE },
+ ]);
+ const r = select([...frames, second, second, second]);
+ // Judged against the raised gate, the immediate follow-up waits...
+ expect(r.decisions[4].verdict).toBe("drop");
+ expect(r.decisions[4].settledGate!).toBeGreaterThan(6.8);
+ // ...and the state is kept once the gate decays back down.
+ expect(r.decisions[5].verdict).toBe("keep-settled");
+ });
+});
+
+describe("cap thinning", () => {
+ test("survivors thin evenly and are marked", () => {
+ // 12 hard cuts alternating far-apart colors in distinct positions.
+ const frames = Array.from({ length: 12 }, (_, i) =>
+ paint(i % 2 === 0 ? BLACK : WHITE, [
+ { x: (i * 13) % 150, y: 8, w: 30, h: 30, color: [200, 40, 40] },
+ ])
+ );
+ const r = select(frames, 4);
+ expect(r.kept.length).toBe(4);
+ expect(r.kept[0]).toBe(0);
+ const thinned = r.decisions.filter((d) => d.verdict === "thinned");
+ expect(thinned.length).toBeGreaterThan(0);
+ });
+});
+
+describe("streaming contract", () => {
+ test("decisions stream one frame behind push and flush on finish", () => {
+ const seen: number[] = [];
+ const sel = createDedupSelector({ maxFrames: 36, onDecision: (d) => seen.push(d.index) });
+ sel.push(paint(BLACK));
+ expect(seen).toEqual([]);
+ sel.push(paint(WHITE));
+ expect(seen).toEqual([0]);
+ sel.finish();
+ expect(seen).toEqual([0, 1]);
+ });
+
+ test("RGB and RGBA inputs agree", () => {
+ const rgb = paint(GRAY, [{ x: 10, y: 10, w: 40, h: 40, color: BLACK }]);
+ const rgba = new Uint8ClampedArray(S * S * 4);
+ for (let p = 0, i = 0, o = 0; p < S * S; p++, i += 3, o += 4) {
+ rgba[o] = rgb.data[i];
+ rgba[o + 1] = rgb.data[i + 1];
+ rgba[o + 2] = rgb.data[i + 2];
+ rgba[o + 3] = 255;
+ }
+ const a = frameSig(rgb);
+ const b = frameSig({ width: S, height: S, channels: 4, data: rgba });
+ expect(Array.from(a.fine)).toEqual(Array.from(b.fine));
+ expect(Array.from(a.g16)).toEqual(Array.from(b.g16));
+ });
+
+ test("the shift tolerance absorbs a one-pixel jitter", () => {
+ const a = paint(GRAY, [{ x: 40, y: 40, w: 50, h: 50, color: BLACK }]);
+ const b = paint(GRAY, [{ x: 41, y: 40, w: 50, h: 50, color: BLACK }]);
+ const scores = settledCellScores(frameSig(b).fine, [frameSig(a).fine], null, 80);
+ expect(Math.max(...scores)).toBe(0);
+ });
+});
diff --git a/site/src/cut/lib/watch/dedupSelector.ts b/site/src/cut/lib/watch/dedupSelector.ts
new file mode 100644
index 00000000..8f85e6c5
--- /dev/null
+++ b/site/src/cut/lib/watch/dedupSelector.ts
@@ -0,0 +1,174 @@
+import {
+ diffCellPct,
+ frameSig,
+ type FrameSig,
+ hardCellCount,
+ settledCellScores,
+} from "./signatures";
+import type {
+ FrameDecision,
+ FrameSelector,
+ RgbFrame,
+ SelectionResult,
+ SelectorOptions,
+ Verdict,
+} from "./types";
+
+/** The default watch selector: three change detectors vote on each candidate
+ * against a sliding window of the last kept frames. The window (rather than
+ * only the previous frame) keeps A-B-A alternation out — a shot the model has
+ * already seen stays out even when a different frame sat in between.
+ *
+ * - global: % of 16-grid cells changed. Sees cuts, pans, motion; blind to
+ * small local changes (each cell averages a whole region away).
+ * - action: a few 32-grid cells changing hard mark a frame as new no matter
+ * the percentage — a small subject can never move 8% of the pixels, so the
+ * one second that matters would otherwise dedup away.
+ * - settled: on a static scene, pixels that differ strongly from every kept
+ * frame in the window (shift-tolerant, so grain and jitter re-match) and
+ * have stopped changing toward the next frame. Sees caption swaps, thin
+ * ink, and local UI updates that measure 0.0% on the global channel. A
+ * cooldown raises its bar after each keep so sustained "settling" motion
+ * can't take a frame every time; a stricter second tolerance must fire in
+ * the same cell so soft-contrast drift (smoke, fades) stays out.
+ *
+ * Memory holds the window plus a one-frame lookahead, never the whole stream. */
+
+export const DEDUP_TUNING = {
+ /** Kept frames each candidate is compared against. */
+ WINDOW: 4,
+ /** Per-channel value change for a 16-grid cell to count as changed. */
+ GLOBAL_TOL: 25,
+ /** % of 16-grid cells that must change for a frame to count as new. */
+ THRESHOLD: 8,
+ /** Action channel: per-channel change for a 32-grid cell to count as hard. */
+ ACTION_HARD_TOL: 45,
+ /** Action channel: hard cells needed to keep. */
+ ACTION_HARD_CELLS: 3,
+ /** Settled channel runs only when the global diff vs the previous candidate
+ * sits under this (the scene is otherwise static), or on the final frame —
+ * a state that appears at the end has nothing after it to prove it settled. */
+ SETTLED_STATIC_MAX: 3,
+ /** Settled mask tolerance; the strict pass that must fire in the same cell. */
+ SETTLED_TOL: 80,
+ SETTLED_STRICT_TOL: 105,
+ /** Base gate = GATE_FACTOR · THRESHOLD (max-cell %); strict gate = STRICT_FACTOR · base. */
+ SETTLED_GATE_FACTOR: 0.85,
+ SETTLED_STRICT_FACTOR: 0.4,
+ /** Cooldown: each settled keep raises the gate additively; it decays per frame. */
+ SETTLED_COOLDOWN_ADD: 2,
+ SETTLED_COOLDOWN_DECAY: 0.7,
+} as const;
+
+export function createDedupSelector(
+ opts: SelectorOptions,
+ tuning: Partial<typeof DEDUP_TUNING> = {}
+): FrameSelector {
+ const T = { ...DEDUP_TUNING, ...tuning };
+ const gateBase = T.SETTLED_GATE_FACTOR * T.THRESHOLD;
+ const strictGate = T.SETTLED_STRICT_FACTOR * gateBase;
+
+ const window: FrameSig[] = [];
+ const decisions: FrameDecision[] = [];
+ const keptIdx: number[] = [];
+ let pending: FrameSig | null = null;
+ let prevG16: Float32Array | null = null; // previous candidate, kept or dropped
+ let cooldown = 1;
+ let count = 0;
+ let finished = false;
+
+ function judge(sig: FrameSig, next: FrameSig | null): void {
+ const index = decisions.length;
+ let verdict: Verdict = "drop";
+ let globalDist = 100;
+ let settledScore: number | undefined;
+ let settledGate: number | undefined;
+
+ if (window.length === 0) {
+ verdict = "first";
+ } else {
+ globalDist = Math.min(
+ ...window.map((k) => diffCellPct(sig.g16, k.g16, T.GLOBAL_TOL))
+ );
+ if (globalDist > T.THRESHOLD) {
+ verdict = "keep-global";
+ } else if (
+ window.every(
+ (k) =>
+ diffCellPct(sig.g16, k.g16, T.GLOBAL_TOL) > T.THRESHOLD ||
+ hardCellCount(sig.g32, k.g32, T.ACTION_HARD_TOL) >= T.ACTION_HARD_CELLS
+ )
+ ) {
+ verdict = "keep-action";
+ } else {
+ const motion = prevG16 === null ? 100 : diffCellPct(sig.g16, prevG16, T.GLOBAL_TOL);
+ if (motion < T.SETTLED_STATIC_MAX || next === null) {
+ const soft = settledCellScores(
+ sig.fine,
+ window.map((k) => k.fine),
+ next?.fine ?? null,
+ T.SETTLED_TOL
+ );
+ settledScore = Math.max(...soft);
+ settledGate = gateBase * cooldown;
+ if (settledScore > settledGate) {
+ // The strict tolerance must fire in the SAME cell — unrelated hard
+ // noise elsewhere must not validate a soft drift.
+ const hard = settledCellScores(
+ sig.fine,
+ window.map((k) => k.fine),
+ next?.fine ?? null,
+ T.SETTLED_STRICT_TOL
+ );
+ if (soft.some((s, i) => s > settledGate! && hard[i] > strictGate)) {
+ verdict = "keep-settled";
+ cooldown += T.SETTLED_COOLDOWN_ADD;
+ }
+ }
+ }
+ }
+ }
+
+ prevG16 = sig.g16;
+ cooldown = Math.max(1, cooldown * T.SETTLED_COOLDOWN_DECAY);
+ if (verdict !== "drop") {
+ keptIdx.push(index);
+ window.push(sig);
+ if (window.length > T.WINDOW) window.shift();
+ }
+ const d: FrameDecision = { index, verdict, globalDist, settledScore, settledGate };
+ decisions.push(d);
+ opts.onDecision?.(d);
+ }
+
+ return {
+ push(frame: RgbFrame) {
+ if (finished) throw new Error("Selector already finished.");
+ const sig = frameSig(frame);
+ count++;
+ if (pending) judge(pending, sig);
+ pending = sig;
+ },
+ finish(): SelectionResult {
+ if (!finished) {
+ finished = true;
+ if (pending) judge(pending, null);
+ pending = null;
+ }
+ let kept = keptIdx;
+ if (kept.length > opts.maxFrames) {
+ // Thin evenly after the channels have voted, so the survivors stay
+ // spread across the range.
+ const step = kept.length / opts.maxFrames;
+ const keep = new Set<number>();
+ for (let i = 0; i < opts.maxFrames; i++) keep.add(Math.floor(i * step));
+ kept = kept.filter((idx, pos) => {
+ if (keep.has(pos)) return true;
+ decisions[idx].verdict = "thinned";
+ return false;
+ });
+ }
+ return { kept, decisions, candidateCount: count };
+ },
+ };
+}
diff --git a/site/src/cut/lib/watch/fuse.test.ts b/site/src/cut/lib/watch/fuse.test.ts
new file mode 100644
index 00000000..8b32b7b8
--- /dev/null
+++ b/site/src/cut/lib/watch/fuse.test.ts
@@ -0,0 +1,64 @@
+import { describe, expect, test } from "bun:test";
+
+import { fuseTimeline, renderFusedTimeline } from "./fuse";
+
+const cue = (start: number, end: number, text: string) => ({ start, end, text });
+
+describe("fuseTimeline", () => {
+ test("frames land inside their speech span", () => {
+ const spans = fuseTimeline(
+ [1, 3.5, 9],
+ [cue(0.5, 4, "hello there"), cue(8, 10, "and goodbye")],
+ { from: 0, to: 12 }
+ );
+ const speech = spans.filter((s) => s.text);
+ expect(speech[0].frames).toEqual([1, 3.5]);
+ expect(speech[1].frames).toEqual([9]);
+ });
+
+ test("a long silence becomes its own span; a short gap does not", () => {
+ const spans = fuseTimeline([5], [cue(0, 4, "a"), cue(4.5, 8, "b"), cue(11, 12, "c")], {
+ from: 0,
+ to: 12,
+ });
+ // 4→4.5 is under the minimum; 8→11 earns a silence span holding no text.
+ expect(spans.map((s) => s.text ?? null)).toEqual(["a", "b", null, "c"]);
+ expect(spans[2].start).toBe(8);
+ expect(spans[2].end).toBe(11);
+ });
+
+ test("cues clip to the watched range and empty cues drop", () => {
+ const spans = fuseTimeline([], [cue(0, 20, "long"), cue(2, 3, " ")], { from: 5, to: 10 });
+ expect(spans.length).toBe(1);
+ expect(spans[0].start).toBe(5);
+ expect(spans[0].end).toBe(10);
+ expect(spans[0].text).toBe("long");
+ });
+
+ test("no cues yields one frame-only span over the range", () => {
+ const spans = fuseTimeline([2, 4], [], { from: 0, to: 6 });
+ expect(spans.length).toBe(1);
+ expect(spans[0].frames).toEqual([2, 4]);
+ });
+
+ test("a frame in a crack attaches to the closest span", () => {
+ // Overlapping cues leave 4→5 unclaimed by the sweep; 4.9 sits nearest the
+ // second cue's span.
+ const spans = fuseTimeline([4.9], [cue(0, 4, "a"), cue(5, 8, "b")], { from: 0, to: 8 });
+ const b = spans.find((s) => s.text === "b")!;
+ expect(b.frames).toEqual([4.9]);
+ });
+});
+
+describe("renderFusedTimeline", () => {
+ test("spans render on one clock with their frames", () => {
+ const out = renderFusedTimeline(
+ fuseTimeline([1, 6], [cue(0.5, 4, "hello")], { from: 0, to: 8 })
+ );
+ expect(out).toBe('[0.5s–4s] "hello" frames: 1s\n[4s–8s] (no speech) frames: 6s');
+ });
+
+ test("spans with nothing to say render nothing", () => {
+ expect(renderFusedTimeline(fuseTimeline([], [], { from: 0, to: 8 }))).toBe("");
+ });
+});
diff --git a/site/src/cut/lib/watch/fuse.ts b/site/src/cut/lib/watch/fuse.ts
new file mode 100644
index 00000000..309b5955
--- /dev/null
+++ b/site/src/cut/lib/watch/fuse.ts
@@ -0,0 +1,71 @@
+/** Fused frame/transcript timeline: kept frame times woven into the caption
+ * cues on one clock, so the model reads precomputed alignment — each speech
+ * span lists the frames that fall inside it, and longer silences become
+ * frame-only spans. Matching frames to lines by eye misaligns on long
+ * ranges; this join is exact. All times are source seconds, the same numbers
+ * stamped on the sheet cells. */
+
+export interface FusedSpan {
+ start: number;
+ end: number;
+ text?: string;
+ frames: number[];
+}
+
+/** A silence between cues must reach this length to earn its own span. */
+const MIN_SILENCE_SPAN = 1.5;
+
+export function fuseTimeline(
+ frameTimes: number[],
+ cues: { start: number; end: number; text: string }[],
+ range: { from: number; to: number }
+): FusedSpan[] {
+ const spans: FusedSpan[] = [];
+ const inRange = cues
+ .map((c) => ({
+ start: Math.max(c.start, range.from),
+ end: Math.min(c.end, range.to),
+ text: c.text.trim(),
+ }))
+ .filter((c) => c.end > c.start && c.text.length > 0)
+ .sort((a, b) => a.start - b.start || a.end - b.end);
+
+ let cursor = range.from;
+ for (const c of inRange) {
+ if (c.start - cursor >= MIN_SILENCE_SPAN) spans.push({ start: cursor, end: c.start, frames: [] });
+ spans.push({ start: c.start, end: Math.max(c.end, c.start), text: c.text, frames: [] });
+ cursor = Math.max(cursor, c.end);
+ }
+ if (range.to - cursor >= MIN_SILENCE_SPAN || spans.length === 0)
+ spans.push({ start: cursor, end: range.to, frames: [] });
+
+ for (const t of [...frameTimes].sort((a, b) => a - b)) {
+ const hit =
+ spans.find((sp) => sp.start <= t && t < sp.end) ??
+ // A frame in a crack between spans attaches to the closest one.
+ spans.reduce((best, sp) =>
+ Math.min(Math.abs(t - sp.start), Math.abs(t - sp.end)) <
+ Math.min(Math.abs(t - best.start), Math.abs(t - best.end))
+ ? sp
+ : best
+ );
+ hit.frames.push(t);
+ }
+ return spans;
+}
+
+const sec = (t: number) => `${Math.round(t * 100) / 100}s`;
+
+/** Render spans as the tool-result text block. Only spans that carry speech
+ * or frames earn a line. */
+export function renderFusedTimeline(spans: FusedSpan[]): string {
+ const lines: string[] = [];
+ for (const sp of spans) {
+ if (!sp.text && sp.frames.length === 0) continue;
+ const head = `[${sec(sp.start)}–${sec(sp.end)}]`;
+ const body = sp.text ? ` "${sp.text}"` : " (no speech)";
+ const frames = sp.frames.length > 0 ? ` frames: ${sp.frames.map(sec).join(", ")}` : "";
+ lines.push(head + body + frames);
+ }
+ return lines.join("\n");
+}
diff --git a/site/src/cut/lib/watch/merge.test.ts b/site/src/cut/lib/watch/merge.test.ts
new file mode 100644
index 00000000..6eaa9a8a
--- /dev/null
+++ b/site/src/cut/lib/watch/merge.test.ts
@@ -0,0 +1,108 @@
+import { describe, expect, test } from "bun:test";
+
+import { mergeSpeech, mergeWatch, nextUncoveredSpan } from "./merge";
+import type { AssetWatch } from "../types";
+
+const frame = (t: number, via: AssetWatch["frames"][number]["via"] = "global") => ({ t, via });
+
+describe("mergeWatch", () => {
+ test("a first watch records its span, frames, and cuts", () => {
+ const w = mergeWatch(undefined, {
+ from: 0,
+ to: 10,
+ frames: [frame(0, "first"), frame(4.128)],
+ sceneChanges: [4.128],
+ });
+ expect(w.ranges).toEqual([{ from: 0, to: 10 }]);
+ expect(w.frames).toEqual([frame(0, "first"), frame(4.13)]);
+ expect(w.sceneChanges).toEqual([4.13]);
+ });
+
+ test("re-watching a span replaces what it covered", () => {
+ const first = mergeWatch(undefined, {
+ from: 0,
+ to: 10,
+ frames: [frame(0, "first"), frame(4), frame(9)],
+ sceneChanges: [4],
+ });
+ const again = mergeWatch(first, { from: 3, to: 10, frames: [frame(5)], sceneChanges: [] });
+ expect(again.ranges).toEqual([{ from: 0, to: 10 }]);
+ expect(again.frames).toEqual([frame(0, "first"), frame(5)]);
+ expect(again.sceneChanges).toEqual([]);
+ });
+
+ test("a new span extends coverage and keeps old frames sorted in", () => {
+ const first = mergeWatch(undefined, { from: 20, to: 30, frames: [frame(25)], sceneChanges: [] });
+ const both = mergeWatch(first, { from: 0, to: 10, frames: [frame(5)], sceneChanges: [] });
+ expect(both.ranges).toEqual([
+ { from: 0, to: 10 },
+ { from: 20, to: 30 },
+ ]);
+ expect(both.frames).toEqual([frame(5), frame(25)]);
+ });
+
+ test("touching spans fuse into one", () => {
+ const first = mergeWatch(undefined, { from: 0, to: 10, frames: [], sceneChanges: [] });
+ const fused = mergeWatch(first, { from: 10, to: 20, frames: [], sceneChanges: [] });
+ expect(fused.ranges).toEqual([{ from: 0, to: 20 }]);
+ });
+
+ test("merging leaves the previous record untouched", () => {
+ const first = mergeWatch(undefined, { from: 0, to: 10, frames: [frame(4)], sceneChanges: [4] });
+ mergeWatch(first, { from: 5, to: 20, frames: [], sceneChanges: [] });
+ expect(first.ranges).toEqual([{ from: 0, to: 10 }]);
+ expect(first.frames).toEqual([frame(4)]);
+ });
+});
+
+describe("mergeSpeech", () => {
+ const seg = (start: number, end: number, text = "words") => ({ start, end, text });
+
+ test("chunks accumulate in order and empty text drops", () => {
+ const a = mergeSpeech(undefined, { from: 0, to: 120, segments: [seg(3, 5), seg(8, 9, " ")] }, 300);
+ const b = mergeSpeech(a, { from: 120, to: 240, segments: [seg(130, 133)] }, 300);
+ expect(b.ranges).toEqual([{ from: 0, to: 240 }]);
+ expect(b.segments).toEqual([seg(3, 5), seg(130, 133)]);
+ expect(b.noSpeech).toBeUndefined();
+ });
+
+ test("re-transcribing a span replaces its segments", () => {
+ const a = mergeSpeech(undefined, { from: 0, to: 120, segments: [seg(3, 5, "old")] }, 300);
+ const b = mergeSpeech(a, { from: 0, to: 120, segments: [seg(3, 5, "new")] }, 300);
+ expect(b.segments).toEqual([seg(3, 5, "new")]);
+ });
+
+ test("silence settles as a verdict only once the whole source is covered", () => {
+ const half = mergeSpeech(undefined, { from: 0, to: 150, segments: [] }, 300);
+ expect(half.noSpeech).toBeUndefined();
+ const all = mergeSpeech(half, { from: 150, to: 300, segments: [] }, 300);
+ expect(all.noSpeech).toBe(true);
+ });
+
+ test("the locale rides along", () => {
+ const a = mergeSpeech(undefined, { from: 0, to: 100, segments: [], locale: "de-DE" }, 300);
+ expect(mergeSpeech(a, { from: 100, to: 200, segments: [] }, 300).locale).toBe("de-DE");
+ });
+});
+
+describe("nextUncoveredSpan", () => {
+ test("an unwatched source starts at zero, capped at the segment size", () => {
+ expect(nextUncoveredSpan(undefined, 2000, 600)).toEqual({ from: 0, to: 600 });
+ });
+
+ test("coverage from zero continues where it stopped", () => {
+ const w = mergeWatch(undefined, { from: 0, to: 600, frames: [], sceneChanges: [] });
+ expect(nextUncoveredSpan(w, 900, 600)).toEqual({ from: 600, to: 900 });
+ });
+
+ test("a gap before the coverage is filled first, ending at the covered part", () => {
+ const w = mergeWatch(undefined, { from: 100, to: 300, frames: [], sceneChanges: [] });
+ expect(nextUncoveredSpan(w, 400, 600)).toEqual({ from: 0, to: 100 });
+ });
+
+ test("full coverage and tail slivers report done", () => {
+ const w = mergeWatch(undefined, { from: 0, to: 599, frames: [], sceneChanges: [] });
+ expect(nextUncoveredSpan(w, 600, 600)).toBe(null);
+ expect(nextUncoveredSpan(mergeWatch(undefined, { from: 0, to: 600, frames: [], sceneChanges: [] }), 600, 600)).toBe(null);
+ });
+});
diff --git a/site/src/cut/lib/watch/merge.ts b/site/src/cut/lib/watch/merge.ts
new file mode 100644
index 00000000..9b116613
--- /dev/null
+++ b/site/src/cut/lib/watch/merge.ts
@@ -0,0 +1,106 @@
+import type { AssetSpeech, AssetWatch, WatchKeepReason } from "../types";
+
+/** Spans closer than this merge into one (matches the 2-dp times stored). */
+const EPS = 0.011;
+
+/** Uncovered slivers shorter than this are left alone — a frame there would
+ * duplicate its neighbors. */
+const MIN_GAP_S = 2;
+
+/** The first stretch of [0, duration] a coverage record leaves uncovered,
+ * capped at `cap` seconds. Null once coverage is effectively complete.
+ * Works for any record carrying merged ranges (watch and speech alike). */
+export function nextUncoveredSpan(
+ watch: { ranges: { from: number; to: number }[] } | undefined,
+ duration: number,
+ cap = 600
+): { from: number; to: number } | null {
+ let cursor = 0;
+ for (const rg of watch?.ranges ?? []) {
+ if (rg.from - cursor >= MIN_GAP_S)
+ return { from: cursor, to: Math.min(rg.from, cursor + cap) };
+ cursor = Math.max(cursor, rg.to);
+ }
+ if (duration - cursor < MIN_GAP_S) return null;
+ return { from: cursor, to: Math.min(duration, cursor + cap) };
+}
+
+const round2 = (n: number) => Math.round(n * 100) / 100;
+
+/** Fold a new watch result into an asset's stored watch metadata. The new
+ * span is authoritative for what it covered: stored frames and cuts inside
+ * [from, to] are replaced by the new ones, spans union, and everything stays
+ * sorted. Watching again over the same stretch refreshes it; watching a new
+ * stretch extends it. */
+/** Fold a transcribed chunk into an asset's stored transcript, the same way
+ * mergeWatch folds frames: the chunk is authoritative for its span, spans
+ * union, segments stay sorted by start. `noSpeech` settles only when the
+ * whole source is covered and nothing was heard. */
+export function mergeSpeech(
+ prev: AssetSpeech | undefined,
+ add: {
+ from: number;
+ to: number;
+ segments: { start: number; end: number; text: string }[];
+ locale?: string;
+ },
+ duration: number
+): AssetSpeech {
+ const from = round2(Math.min(add.from, add.to));
+ const to = round2(Math.max(add.from, add.to));
+ const inside = (t: number) => t >= from - EPS && t <= to + EPS;
+
+ const segments = (prev?.segments ?? [])
+ .filter((sg) => !inside(sg.start))
+ .concat(
+ add.segments
+ .filter((sg) => sg.text.trim().length > 0)
+ .map((sg) => ({ start: round2(sg.start), end: round2(sg.end), text: sg.text.trim() }))
+ )
+ .sort((a, b) => a.start - b.start);
+
+ const ranges: AssetSpeech["ranges"] = [];
+ for (const rg of [...(prev?.ranges ?? []), { from, to }].sort((a, b) => a.from - b.from)) {
+ const last = ranges[ranges.length - 1];
+ if (last && rg.from <= last.to + EPS) last.to = Math.max(last.to, rg.to);
+ else ranges.push({ from: rg.from, to: rg.to });
+ }
+ const covered = nextUncoveredSpan({ ranges }, duration) === null;
+ return {
+ ranges,
+ segments,
+ ...(covered && segments.length === 0 ? { noSpeech: true as const } : {}),
+ ...(add.locale ?? prev?.locale ? { locale: add.locale ?? prev?.locale } : {}),
+ };
+}
+
+export function mergeWatch(
+ prev: AssetWatch | undefined,
+ add: {
+ from: number;
+ to: number;
+ frames: { t: number; via: WatchKeepReason }[];
+ sceneChanges: number[];
+ }
+): AssetWatch {
+ const from = round2(Math.min(add.from, add.to));
+ const to = round2(Math.max(add.from, add.to));
+ const inside = (t: number) => t >= from - EPS && t <= to + EPS;
+
+ const frames = (prev?.frames ?? [])
+ .filter((f) => !inside(f.t))
+ .concat(add.frames.map((f) => ({ t: round2(f.t), via: f.via })))
+ .sort((a, b) => a.t - b.t);
+ const sceneChanges = (prev?.sceneChanges ?? [])
+ .filter((t) => !inside(t))
+ .concat(add.sceneChanges.map(round2))
+ .sort((a, b) => a - b);
+
+ const ranges: AssetWatch["ranges"] = [];
+ for (const rg of [...(prev?.ranges ?? []), { from, to }].sort((a, b) => a.from - b.from)) {
+ const last = ranges[ranges.length - 1];
+ if (last && rg.from <= last.to + EPS) last.to = Math.max(last.to, rg.to);
+ else ranges.push({ from: rg.from, to: rg.to });
+ }
+ return { ranges, frames, sceneChanges };
+}
diff --git a/site/src/cut/lib/watch/signatures.ts b/site/src/cut/lib/watch/signatures.ts
new file mode 100644
index 00000000..0443f125
--- /dev/null
+++ b/site/src/cut/lib/watch/signatures.ts
@@ -0,0 +1,147 @@
+import { type RgbFrame, SIGNATURE_SIZE } from "./types";
+
+/** The pixel comparisons behind frame selection. All math runs on three
+ * fixed views of a SIGNATURE_SIZE² frame:
+ * - fine: the pixels themselves (packed RGB) — local detail
+ * - g16 / g32: per-cell channel means — whole-frame structure
+ * Comparisons use the max channel difference, so hues with equal luma still
+ * read as different. */
+
+const S = SIGNATURE_SIZE;
+
+export interface FrameSig {
+ fine: Uint8Array; // S*S*3 packed RGB
+ g16: Float32Array; // 16*16*3 cell means
+ g32: Float32Array; // 32*32*3 cell means
+}
+
+function meanGrid(fine: Uint8Array, grid: number): Float32Array {
+ const cell = S / grid;
+ const out = new Float32Array(grid * grid * 3);
+ const norm = 1 / (cell * cell);
+ for (let gy = 0; gy < grid; gy++) {
+ for (let gx = 0; gx < grid; gx++) {
+ let r = 0;
+ let g = 0;
+ let b = 0;
+ for (let y = gy * cell; y < (gy + 1) * cell; y++) {
+ let i = (y * S + gx * cell) * 3;
+ for (let x = 0; x < cell; x++) {
+ r += fine[i++];
+ g += fine[i++];
+ b += fine[i++];
+ }
+ }
+ const o = (gy * grid + gx) * 3;
+ out[o] = r * norm;
+ out[o + 1] = g * norm;
+ out[o + 2] = b * norm;
+ }
+ }
+ return out;
+}
+
+export function frameSig(f: RgbFrame): FrameSig {
+ if (f.width !== S || f.height !== S)
+ throw new Error(`Selector frames must be ${S}×${S} (got ${f.width}×${f.height}).`);
+ let fine: Uint8Array;
+ if (f.channels === 3) {
+ fine = Uint8Array.from(f.data); // copy — the caller may reuse its buffer
+ } else {
+ fine = new Uint8Array(S * S * 3);
+ for (let p = 0, i = 0, o = 0; p < S * S; p++, i += 4, o += 3) {
+ fine[o] = f.data[i];
+ fine[o + 1] = f.data[i + 1];
+ fine[o + 2] = f.data[i + 2];
+ }
+ }
+ return { fine, g16: meanGrid(fine, 16), g32: meanGrid(fine, 32) };
+}
+
+/** % of grid cells whose max channel differs by more than tol. */
+export function diffCellPct(a: Float32Array, b: Float32Array, tol: number): number {
+ const cells = a.length / 3;
+ let changed = 0;
+ for (let i = 0; i < a.length; i += 3) {
+ const d = Math.max(
+ Math.abs(a[i] - b[i]),
+ Math.abs(a[i + 1] - b[i + 1]),
+ Math.abs(a[i + 2] - b[i + 2])
+ );
+ if (d > tol) changed++;
+ }
+ return (100 * changed) / cells;
+}
+
+/** Count of grid cells whose max channel differs by more than tol. */
+export function hardCellCount(a: Float32Array, b: Float32Array, tol: number): number {
+ let n = 0;
+ for (let i = 0; i < a.length; i += 3) {
+ const d = Math.max(
+ Math.abs(a[i] - b[i]),
+ Math.abs(a[i + 1] - b[i + 1]),
+ Math.abs(a[i + 2] - b[i + 2])
+ );
+ if (d > tol) n++;
+ }
+ return n;
+}
+
+/** True when pixel (x,y) of cur differs by more than tol from every ref pixel
+ * within ±1 — the shift tolerance lets jitter, weave, and grain re-match, so
+ * only genuinely new pixels count. */
+function shiftChanged(ref: Uint8Array, cur: Uint8Array, x: number, y: number, tol: number): boolean {
+ const ci = (y * S + x) * 3;
+ const r = cur[ci];
+ const g = cur[ci + 1];
+ const b = cur[ci + 2];
+ for (let dy = -1; dy <= 1; dy++) {
+ const sy = Math.min(S - 1, Math.max(0, y + dy));
+ for (let dx = -1; dx <= 1; dx++) {
+ const sx = Math.min(S - 1, Math.max(0, x + dx));
+ const ri = (sy * S + sx) * 3;
+ const d = Math.max(
+ Math.abs(ref[ri] - r),
+ Math.abs(ref[ri + 1] - g),
+ Math.abs(ref[ri + 2] - b)
+ );
+ if (d <= tol) return false;
+ }
+ }
+ return true;
+}
+
+export const SETTLED_GRID = 16;
+
+/** Per-cell % of pixels that hold a settled new state: changed (beyond tol,
+ * shift-tolerant) vs EVERY kept frame in the window, and — when a next frame
+ * exists — no longer changing toward it. A caption that just landed scores
+ * high in its cells; motion mid-flight scores zero because the next frame
+ * moves it again. Returns SETTLED_GRID² cell percentages. */
+export function settledCellScores(
+ cur: Uint8Array,
+ keptWindow: Uint8Array[],
+ next: Uint8Array | null,
+ tol: number
+): Float32Array {
+ const grid = SETTLED_GRID;
+ const cell = S / grid;
+ const counts = new Float32Array(grid * grid);
+ for (let y = 0; y < S; y++) {
+ const gy = Math.floor(y / cell);
+ for (let x = 0; x < S; x++) {
+ let changed = true;
+ for (const kf of keptWindow) {
+ if (!shiftChanged(kf, cur, x, y, tol)) {
+ changed = false;
+ break;
+ }
+ }
+ if (changed && next !== null && shiftChanged(next, cur, x, y, tol)) changed = false;
+ if (changed) counts[gy * grid + Math.floor(x / cell)]++;
+ }
+ }
+ const norm = 100 / (cell * cell);
+ for (let i = 0; i < counts.length; i++) counts[i] *= norm;
+ return counts;
+}
diff --git a/site/src/cut/lib/watch/sweep.ts b/site/src/cut/lib/watch/sweep.ts
new file mode 100644
index 00000000..7be014c7
--- /dev/null
+++ b/site/src/cut/lib/watch/sweep.ts
@@ -0,0 +1,247 @@
+"use client";
+
+import { getBackend, hasLocalCompute } from "../backend";
+import {
+ isFreeTranscriptionExhausted,
+ renderMix,
+ transcribeSamples,
+ type CloudTranscribeSpec,
+} from "../cloudTranscribe";
+import { alignCues } from "../cueAlign";
+import { engineTranscribeSamples } from "../localStt";
+import { sampleWatchFrames } from "../media";
+import { runTranscription, useEditor } from "../store";
+import { trackLocale } from "../subtitles";
+import { mergeSpeech, mergeWatch, nextUncoveredSpan } from "./merge";
+
+/** Background watch sweep: after the assistant's first look at (or listen to)
+ * a source, quietly build the rest of its working metadata — the transcript
+ * first, then the visual map, segment by segment, merged into the asset's
+ * stored records as each piece lands. No images are made, no subtitle track
+ * is written, and the user is never charged: transcription runs on the free
+ * paths (the local engine, a Mac's engine behind a cloud project, or the
+ * hosted route's included allowance) and stops when none remains. The
+ * sweep yields the decoders to playback and to foreground watch calls,
+ * yields the engine's transcriber to user caption generation, survives its
+ * own interruption because coverage persists per segment, and ends when
+ * nothing is left uncovered. */
+
+const SEGMENT_S = 600; // visual sweep segment
+const SPEECH_CHUNK_S = 120; // transcription chunk — short, so a user's own
+// caption generation never waits long behind the engine's one-at-a-time STT
+const SPEECH_PAD_S = 2; // re-listen into the previous chunk so no word is cut
+const STT_MAX_FAILS = 4;
+const WATCH_MAX_FAILS = 4; // media reads ride the network — retry transient failures
+const RETRY_MS = 5_000; // wait while playback or a foreground watch holds the decoders
+const BREATHER_MS = 1_000; // idle gap between segments
+
+const queued = new Set<string>();
+const sttFails = new Map<string, number>();
+const watchFails = new Map<string, number>();
+let running = false;
+let syncWatches = 0;
+// The UTC month in which the hosted route said this account's included chunks
+// were used up. The sweep stops asking until the month changes (or a reload);
+// the route re-decides from the account's actual allowance window — a Pro
+// billing period, or the calendar month — on the next ask.
+let hostedFreeExhaustedMonth: string | null = null;
+const utcMonth = () => new Date().toISOString().slice(0, 7);
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+/** The sweep steps aside whenever someone is actively using the decoders:
+ * the preview playing, or the assistant inside a foreground watch call. */
+const decodersBusy = () => syncWatches > 0 || useEditor.getState().playing;
+
+/** A free transcriber exists: the local engine, a Mac's engine behind a cloud
+ * project, or the hosted route's monthly included allowance while it lasts. */
+const sttFree = () =>
+ getBackend().kind !== "cloud" || hasLocalCompute() || hostedFreeExhaustedMonth !== utcMonth();
+
+/** Run a foreground watch with the sweep paused for its duration, so the
+ * call the assistant is waiting on gets the decoders to itself. */
+export async function withSweepPaused<T>(work: () => Promise<T>): Promise<T> {
+ syncWatches++;
+ try {
+ return await work();
+ } finally {
+ syncWatches--;
+ }
+}
+
+/** True while a background sweep is still filling this asset's metadata. */
+export function watchSweepActive(assetId: string): boolean {
+ return queued.has(assetId);
+}
+
+/** Queue an asset for background sweeping. Safe to call repeatedly; a fully
+ * covered asset dequeues on its first turn. */
+export function queueWatchSweep(assetId: string): void {
+ if (queued.has(assetId)) return;
+ queued.add(assetId);
+ if (!running) void run();
+}
+
+/** The free transcribers only. A background job must be unable to spend the
+ * user's credits by construction, so the cloud case here talks to its
+ * transcriber directly: the Mac's engine when one is reachable, else the
+ * hosted route pinned to `freeOnly` — the route refuses past the account's
+ * included allowance, and that refusal ends hosted sweeping for the session. */
+async function transcribeFree(
+ projectId: string,
+ spec: CloudTranscribeSpec
+): Promise<{ start: number; end: number; text: string }[] | null> {
+ if (getBackend().kind !== "cloud") return runTranscription(projectId, spec);
+ const stale = () => useEditor.getState().projectId !== projectId;
+ const mix = await renderMix(projectId, spec);
+ if (stale()) return null;
+ if (!mix) return []; // nothing audible — no speech, like the engine's short-circuit
+ const samples = mix.getChannelData(0);
+ if (hasLocalCompute()) {
+ try {
+ return await engineTranscribeSamples(samples, spec.duration, spec.locale, stale);
+ } catch {
+ // The app went away mid-call, or on-device speech is unavailable on
+ // this Mac — the hosted included path below still serves this chunk.
+ }
+ }
+ try {
+ // The hosted model reads cue times off the audio by ear; snap them to the
+ // mix the same way the user's caption path does.
+ const cues = await transcribeSamples(samples, spec.locale, stale, { freeOnly: true });
+ return cues && alignCues(cues, samples, mix.sampleRate, { snap: 0.6 });
+ } catch (error) {
+ if (isFreeTranscriptionExhausted(error)) hostedFreeExhaustedMonth = utcMonth();
+ throw error;
+ }
+}
+
+/** Transcribe one chunk of the source into its asset-level transcript. The
+ * chunk re-listens SPEECH_PAD_S into covered ground so a word straddling the
+ * boundary is heard whole; segments the previous chunk already owns drop. */
+async function transcribeChunk(
+ assetId: string,
+ span: { from: number; to: number }
+): Promise<void> {
+ const s = useEditor.getState();
+ const asset = s.assets.find((a) => a.id === assetId);
+ if (!s.projectId || !asset) return;
+ const locale = asset.language ?? trackLocale(s.subtitles, 0);
+ const specFrom = Math.max(0, span.from - (span.from > 0 ? SPEECH_PAD_S : 0));
+ const cues = await transcribeFree(s.projectId, {
+ duration: span.to - specFrom,
+ locale,
+ clips: [],
+ audio: [{ file: asset.fileName, in: specFrom, out: span.to, start: 0, volume: 1 }],
+ });
+ if (cues === null) return; // another project took over mid-run
+ const segments = cues
+ .map((c) => ({ start: c.start + specFrom, end: c.end + specFrom, text: c.text }))
+ .filter((sg) => sg.start >= span.from - 0.05);
+ const cur = useEditor.getState().assets.find((a) => a.id === assetId);
+ if (!cur) return;
+ useEditor.getState().updateAsset(assetId, {
+ speech: mergeSpeech(cur.speech, { from: span.from, to: span.to, segments, locale }, cur.duration),
+ });
+}
+
+async function run(): Promise<void> {
+ running = true;
+ try {
+ while (queued.size > 0) {
+ const id: string = queued.values().next().value!;
+ const s = useEditor.getState();
+ const asset = s.assets.find((a) => a.id === id);
+ // The asset may be gone (deleted, or another project opened) — the
+ // sweep dies with it, like its metadata does.
+ if (
+ !asset ||
+ asset.upload ||
+ !(asset.duration > 0) ||
+ (asset.type !== "video" && asset.type !== "audio")
+ ) {
+ queued.delete(id);
+ sttFails.delete(id);
+ watchFails.delete(id);
+ continue;
+ }
+
+ // Transcript first: it is small, unlocks the fused timeline for every
+ // later watch, and answers "what is said" without inlining audio.
+ // It waits out the user's own caption generation — the engine
+ // transcribes one job at a time and the user goes first.
+ const speechSpan =
+ sttFree() && (sttFails.get(id) ?? 0) < STT_MAX_FAILS && s.subtitleStatus !== "running"
+ ? nextUncoveredSpan(asset.speech, asset.duration, SPEECH_CHUNK_S)
+ : null;
+ if (speechSpan) {
+ try {
+ await transcribeChunk(id, speechSpan);
+ sttFails.delete(id);
+ } catch {
+ // Likely the engine's single transcription slot — back off; the
+ // fail cap keeps an unreadable source from looping forever.
+ sttFails.set(id, (sttFails.get(id) ?? 0) + 1);
+ await sleep(RETRY_MS);
+ }
+ await sleep(BREATHER_MS);
+ continue;
+ }
+
+ // Visual map second; audio-only sources are done once heard.
+ const span =
+ asset.type === "video" ? nextUncoveredSpan(asset.watch, asset.duration, SEGMENT_S) : null;
+ if (!span) {
+ queued.delete(id);
+ sttFails.delete(id);
+ watchFails.delete(id);
+ continue;
+ }
+ if (decodersBusy()) {
+ await sleep(RETRY_MS);
+ continue;
+ }
+ try {
+ const r = await sampleWatchFrames(asset.url, {
+ from: span.from,
+ to: span.to,
+ metadataOnly: true,
+ shouldPause: decodersBusy,
+ });
+ if (r.coveredTo > span.from) {
+ const cur = useEditor.getState().assets.find((a) => a.id === id);
+ if (!cur) {
+ queued.delete(id);
+ continue;
+ }
+ watchFails.delete(id);
+ useEditor.getState().updateAsset(id, {
+ watch: mergeWatch(cur.watch, {
+ from: span.from,
+ to: r.coveredTo,
+ frames: r.frames.map((f) => ({ t: f.t, via: f.via })),
+ sceneChanges: r.sceneChanges,
+ }),
+ });
+ } else {
+ await sleep(RETRY_MS); // paused before any progress
+ }
+ } catch {
+ // Media reads ride the network, so a failure is retried like the
+ // speech pass's; the cap keeps a truly unreadable source from
+ // looping forever.
+ const fails = (watchFails.get(id) ?? 0) + 1;
+ watchFails.set(id, fails);
+ if (fails >= WATCH_MAX_FAILS) {
+ queued.delete(id);
+ watchFails.delete(id);
+ } else {
+ await sleep(RETRY_MS);
+ }
+ }
+ await sleep(BREATHER_MS);
+ }
+ } finally {
+ running = false;
+ }
+}
diff --git a/site/src/cut/lib/watch/types.ts b/site/src/cut/lib/watch/types.ts
new file mode 100644
index 00000000..f587fda7
--- /dev/null
+++ b/site/src/cut/lib/watch/types.ts
@@ -0,0 +1,63 @@
+/** Frame selection for watch_video: both backends hand candidate frames to a
+ * selector that keeps the distinct moments. The engine feeds it raw RGB frames
+ * piped from ffmpeg; the browser feeds it ImageData. Pure data in, pure data
+ * out — no DOM, no Node. */
+
+/** Every candidate reaches the selector at this size; signatures divide it
+ * evenly (192 = 12·16 = 6·32). */
+export const SIGNATURE_SIZE = 192;
+
+/** One candidate frame's pixels at SIGNATURE_SIZE × SIGNATURE_SIZE.
+ * channels 3 = packed RGB (ffmpeg rawvideo), 4 = RGBA (canvas ImageData —
+ * alpha is ignored). The selector copies what it needs; callers may reuse
+ * the buffer. */
+export interface RgbFrame {
+ width: number;
+ height: number;
+ channels: 3 | 4;
+ data: Uint8Array | Uint8ClampedArray;
+}
+
+export type Verdict =
+ | "first" // the opening frame — always kept
+ | "keep-global" // whole-frame change vs the kept window
+ | "keep-action" // hard local change on a small subject
+ | "keep-settled" // a settled local change on a static scene (text, ink, UI)
+ | "drop" // a near-duplicate of the kept window
+ | "thinned"; // kept by the channels, removed by the max-frames cap
+
+export interface FrameDecision {
+ index: number;
+ verdict: Verdict;
+ /** Min % of 16-grid cells changed vs the kept window (the global channel). */
+ globalDist: number;
+ /** Max settled-mask cell %, when the settled channel ran. */
+ settledScore?: number;
+ /** The settled gate (cooldown included) that score was judged against. */
+ settledGate?: number;
+}
+
+export interface SelectionResult {
+ /** Kept candidate indices, ascending, after the max-frames cap. */
+ kept: number[];
+ decisions: FrameDecision[];
+ candidateCount: number;
+}
+
+export interface SelectorOptions {
+ /** Cap on kept frames; survivors are thinned evenly when the channels keep more. */
+ maxFrames: number;
+ /** Fires as each candidate's channel verdict lands (one frame behind push —
+ * the settled channel looks ahead one frame). "thinned" is not known until
+ * finish(); treat a keep here as provisional. */
+ onDecision?: (d: FrameDecision) => void;
+}
+
+/** Push candidates in time order, then finish. Decisions stream one frame
+ * behind push; finish flushes the last frame and applies the cap. */
+export interface FrameSelector {
+ push(frame: RgbFrame): void;
+ finish(): SelectionResult;
+}
+
+export type SelectorFactory = (opts: SelectorOptions) => FrameSelector;
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index ac1343f4..0cbe512f 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -87,7 +87,7 @@ Times are in seconds on the shared timeline. The playhead is currentTime; a skim
When a request depends on what the footage actually contains — "cut the dead air", "clip the best moment", "remove the boring part", "split where the scene changes" — watch it first. Never guess at content you haven't seen.
Your eyes and ears:
-- watch_video(clip_id | asset_id, from?, to?, interval_seconds?): samples the SOURCE at scene changes plus a steady floor and returns timestamped contact sheets — cells read left→right, top→bottom, and each burned stamp is source seconds — plus sceneChanges, the natural cut candidates. Coverage is capped per call: survey the whole range at the default interval first, then zoom into the moments that matter with a narrow from/to and interval_seconds 0.5–1. When truncated, continue from coveredTo.
+- watch_video(clip_id | asset_id, from?, to?, interval_seconds?): samples the SOURCE on a dense steady floor, keeps only the frames that actually differ, and returns them as timestamped contact sheets — cells read left→right, top→bottom, each burned stamp is source seconds, and a gap between stamps means nothing changed there — plus sceneChanges, hard-cut times refined to about a third of a second of the true boundary, and — when the clip has captions or the source's transcript exists — a fused timeline placing each frame inside the speech it belongs to. Coverage ends where the result says: when truncated (span bound, frame cap, or time budget), continue from coveredTo. A shot shorter than the sampling step can fall between candidates — where short cuts matter, re-watch narrow with interval_seconds 0.5–1. Watching builds the source's frame map, persisted on the asset; each media entry's "mapped" spans list where that map exists, and the first watch or listen of a source starts a background sweep that maps the rest, transcript included ("watching": true marks it running; the spans grow as segments land). Use the map to aim your watches — a mapped span is NOT footage you have seen; only the sheets returned to you in this conversation are, so watch any span before you cut inside it.
- detect_silence(clip_id | asset_id, threshold_db?, min_silence?): silent stretches in source seconds; with clip_id each also carries its timeline times.
- capture_frame: one composited frame at the playhead — for checking the final look, not for surveying footage.
@@ -206,7 +206,7 @@ Rules:
- Times are seconds. The frame follows project.aspect in editor_state — any "W:H" ratio, short side 1080px; project.frame has the exact pixels. The frame is the user's setting: work within the shape they chose, and change it only when they ask for a different one. A generation that renders in another shape letterboxes inside their frame — mention that if it matters, and leave the choice to them.
- Read list_skills / read_skill before working in an area you're unsure about — they document every setting. A tool whose description already answers the ask goes straight through.
- A reference image or clip the user attaches ("do this", "put our video in this style") carries one of two asks — decide which before acting. When its footage's look is the point (color, lighting, film grade), route to looks, grades, or restyle. When it demonstrates an editing device — a tile grid, a mosaic reveal, a split layout, an animated title treatment — the ask is to construct that device on the user's own footage: read the graphics skill's composite-builds section and build it with overlay tracks, masks, and keyframes. Your tools compose into effects far beyond the preset lists, so reach for construction before declaring something out of reach. Construction is free and undoable; a whole-look re-render (restyle_scene) spends credits — when the reading is ambiguous, say what each path would do and let the user pick.
-- Transcription tools write tracks the user sees — run subtitles_generate / captions_generate only when captions were asked for, never just to read the words: existing cues are in editor_state, and audio plays to you via listen_audio. Don't transcribe a video with no speech (subtitles_from_visuals narrates silent footage), and never invent a spoken transcript.
+- Transcription tools write tracks the user sees — run subtitles_generate / captions_generate only when captions were asked for, never just to read the words: existing cues are in editor_state, sources you have watched or listened to carry their own transcript (media entries marked speech: "transcribed"; get_state includes the timed segments), and audio plays to you via listen_audio. Don't transcribe a video with no speech (subtitles_from_visuals narrates silent footage), and never invent a spoken transcript.
- Voiceovers duck other audio by default so they stay audible. Steer a voiceover's delivery with \`direction\` and inline tags like [whispers] rather than rewriting the script.
- generate_image / generate_video / generate_character_video / voiceover_generate / read_subtitles_aloud / generate_music make media through hosted models (spends the user's Donkey credits, needs sign-in); call them when the user asked for the media itself — a request for the prompt or script gets text in chat. generate_music makes a music track — a vocal-free background bed by default ("add music to this video"), or a full sung song (instrumental:false); it's sung music, so a spoken narration is voiceover_generate. Bundled stock media (stock_search / stock_add) is free — use it when it fits. Media the user attached is in \`media\`; pass those asset ids as generation references when they say "use this", and place project assets in the cut with add_clip when they ask for them there ("make a movie from these photos"). Generated media previews on a chat card the user can expand, drag in, or file away; add it to the timeline (add_to_timeline:true or an index) only when they asked for it in the cut. Write a rich, specific prompt from their shorthand. Video renders take a minute or two. "Generate a video of/about X" means one generate_video clip; only when they name a narrated multi-shot production (a story, episode, narrated short) does generate_scene plan it and stop at the storyboard — the user reviews the frames (regenerate_shot redraws any before approval, no credits), then call approve_scene, since it renders many paid shots (regenerate_shot / recut_scene / restyle_scene revise it after).
- You can see and hear: audio the user attaches to their message plays right in it — answer "what does this say" from what you hear, no tool call. For project footage, watch_video samples a source's frames into timestamped contact sheets (scene changes included) — watch before cutting footage you haven't seen; listen_audio plays a source's audio — an audio asset or a video/clip's own track; detect_silence finds dead air; capture_frame shows the one rendered frame at the playhead. The watching-and-cutting skill has the flow.`;
diff --git a/site/src/cut/server/frames.ts b/site/src/cut/server/frames.ts
index e884acaa..292c31c1 100644
--- a/site/src/cut/server/frames.ts
+++ b/site/src/cut/server/frames.ts
@@ -1,5 +1,5 @@
import { spawn } from "node:child_process";
-import { mkdtemp, readdir, readFile, rm, unlink } from "node:fs/promises";
+import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { mediaPath } from "./projects";
@@ -191,147 +191,6 @@ function runCapture(args: string[], timeoutMs: number): Promise<CaptureRun> {
const errTail = (stderr: string) => stderr.trim().split("\n").slice(-3).join("\n");
-const sheetDataUrl = async (file: string) =>
- `data:image/jpeg;base64,${(await readFile(file)).toString("base64")}`;
-
-const SHEET_GRID = 3; // cells per row and column
-const SHEET_CELL = 480; // cell long side, px
-const SHEET_GAP = 4; // tile margin and padding, px — the client stamps cells by this geometry
-const SCENE_THRESHOLD = 0.3; // ffmpeg scene-score cut point
-const WATCH_BUDGET_MS = 100_000; // all attempts together; leaves salvage room under the 120s tool cap
-
-export interface SheetFrame {
- t: number;
- scene?: number;
-}
-
-export interface WatchResult {
- sheets: { image: string; frames: SheetFrame[] }[];
- /** Cell geometry, so the client can stamp each cell with its time. */
- layout: { grid: number; margin: number; padding: number };
- sceneChanges: number[];
- coveredTo: number;
- truncated: boolean;
-}
-
-/**
- * Sample a media file into timestamped contact sheets: frames picked at scene
- * changes plus a steady density floor, each cell stamped with its source time,
- * tiled 3×3. This is how the assistant watches footage — a bounded survey of
- * what the file actually shows (maxSheets per call; callers narrow the range
- * to see more).
- */
-export async function makeContactSheets(
- projectId: string,
- sourceFile: string,
- opts: { from: number; to: number; interval: number; maxSheets: number; still?: boolean }
-): Promise<WatchResult> {
- const src = mediaPath(projectId, sourceFile);
- const tmp = await mkdtemp(path.join(os.tmpdir(), "veditor-watch-"));
- try {
- const dims = await probeDims(src);
- const scale = dims.width >= dims.height ? `scale=${SHEET_CELL}:-2` : `scale=-2:${SHEET_CELL}`;
-
- // A still is its own one-cell sheet — no sampling, no time axis.
- if (opts.still) {
- const out = path.join(tmp, "sheet-01.jpg");
- const r = await runCapture(
- ["-hide_banner", "-nostats", "-y", "-i", src, "-vf", scale, "-frames:v", "1", "-q:v", "5", out],
- 30_000
- );
- if (!r.ok) throw new Error(errTail(r.stderr) || "Could not read the image.");
- return {
- sheets: [{ image: await sheetDataUrl(out), frames: [{ t: 0 }] }],
- layout: { grid: 1, margin: 0, padding: 0 },
- sceneChanges: [],
- coveredTo: 0,
- truncated: false,
- };
- }
-
- const { from, to, interval, maxSheets } = opts;
- const minGap = Math.max(0.4, interval / 3);
- const graph = [
- // First frame always; a scene cut once minGap has passed; the floor.
- `select='isnan(prev_selected_t)+gt(scene,${SCENE_THRESHOLD})*gte(t-prev_selected_t,${num(minGap)})+gte(t-prev_selected_t,${num(interval)})'`,
- "metadata=print",
- scale,
- `tile=${SHEET_GRID}x${SHEET_GRID}:padding=${SHEET_GAP}:margin=${SHEET_GAP}`,
- ].join(",");
- const args = (hw: boolean) => [
- "-hide_banner", "-nostats", "-loglevel", "info", "-y",
- ...(hw ? ["-hwaccel", "videotoolbox"] : []),
- "-ss", num(from),
- "-t", num(to - from),
- "-i", src,
- "-map", "0:v:0", "-an", "-sn", "-dn",
- "-vf", graph,
- "-fps_mode", "vfr",
- "-frames:v", String(maxSheets),
- "-q:v", "5",
- path.join(tmp, "sheet-%02d.jpg"),
- ];
-
- // Hardware decode first; a source it refuses gets one software retry.
- const deadline = Date.now() + WATCH_BUDGET_MS;
- let run = await runCapture(args(true), WATCH_BUDGET_MS);
- if (!run.ok && !run.timedOut && deadline - Date.now() > 5_000) {
- // Await every stale-sheet delete before the retry writes new sheets to the
- // same paths, or a slow unlink can land after ffmpeg and wipe a fresh sheet.
- await Promise.all(
- (await readdir(tmp)).map((f) => unlink(path.join(tmp, f)).catch(() => {})),
- );
- run = await runCapture(args(false), deadline - Date.now());
- }
-
- const files = (await readdir(tmp)).filter((f) => f.endsWith(".jpg")).sort();
- // A timeout or a mid-run decode error may leave the last sheet half-written.
- if (!run.ok && files.length > 0) files.pop();
- if (files.length === 0) {
- throw new Error(
- run.timedOut
- ? "Sampling timed out before the first sheet was ready — try a shorter range or a larger interval."
- : errTail(run.stderr) || "Could not sample the video."
- );
- }
-
- // metadata=print logs each selected frame ("frame:N … pts_time:T" then
- // "lavfi.scene_score=S"); times are relative to the input seek.
- const frames: SheetFrame[] = [];
- for (const line of run.stderr.split("\n")) {
- const t = /frame:\d+.*pts_time:([\d.]+)/.exec(line);
- if (t) {
- frames.push({ t: round(from + parseFloat(t[1])) });
- continue;
- }
- const s = /lavfi\.scene_score=([\d.]+)/.exec(line);
- if (s && frames.length > 0) frames[frames.length - 1].scene = parseFloat(s[1]);
- }
- const perSheet = SHEET_GRID * SHEET_GRID;
- const kept = frames.slice(0, files.length * perSheet);
- const sheets = await Promise.all(
- files.map(async (f, i) => ({
- image: await sheetDataUrl(path.join(tmp, f)),
- frames: kept.slice(i * perSheet, (i + 1) * perSheet),
- }))
- );
- const lastT = kept.length > 0 ? kept[kept.length - 1].t : from;
- // An aborted run (timeout or error) that salvaged sheets covered only what
- // it decoded — never claim the full range for it.
- const truncated =
- (!run.ok || frames.length >= maxSheets * perSheet) && lastT < to - interval;
- return {
- sheets,
- layout: { grid: SHEET_GRID, margin: SHEET_GAP, padding: SHEET_GAP },
- sceneChanges: kept.filter((f) => (f.scene ?? 0) >= SCENE_THRESHOLD).map((f) => f.t),
- coveredTo: truncated ? lastT : to,
- truncated,
- };
- } finally {
- void rm(tmp, { recursive: true, force: true });
- }
-}
-
export interface SilenceRange {
start: number;
end: number;
diff --git a/site/src/cut/server/http/projects.ts b/site/src/cut/server/http/projects.ts
index de4b8fde..2467a182 100644
--- a/site/src/cut/server/http/projects.ts
+++ b/site/src/cut/server/http/projects.ts
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import { normalizeAspect, type ProjectDoc } from "@/cut/lib/types";
-import { detectSilence, extractAudio, makeContactSheets, makeFreezeFrame, probeDims, probeDuration } from "../frames";
+import { detectSilence, extractAudio, makeFreezeFrame, probeDims, probeDuration } from "../frames";
import {
createProject,
createProjectFolder,
@@ -396,47 +396,6 @@ export const projectsApi = {
}
},
- /** Sample a media file into timestamped contact sheets (the assistant's eyes). */
- async watch(req: Request, { id }: { id: string }) {
- try {
- if (!(await readProject(id))) return err("Project not found.", 404);
- const body = (await req.json()) as {
- file?: string;
- from?: number;
- to?: number;
- interval?: number;
- maxSheets?: number;
- still?: boolean;
- };
- if (!body.file) return err("file is required.", 400);
- if (body.still) {
- return Response.json(
- await makeContactSheets(id, body.file, { from: 0, to: 0, interval: 1, maxSheets: 1, still: true })
- );
- }
- const from = Math.max(0, typeof body.from === "number" ? body.from : 0);
- const wanted = typeof body.to === "number" ? body.to : await probeDuration(mediaPath(id, body.file));
- if (typeof body.to !== "number" && wanted <= 0)
- return err("Could not read the media duration — pass to (seconds).", 400);
- if (!(wanted > from)) return err("from/to describe an empty range.", 400);
- const to = Math.min(wanted, from + 600); // bound the decode per call; callers resume from coveredTo
- const interval =
- typeof body.interval === "number"
- ? clamp(body.interval, 0.5, 30)
- : clamp((to - from) / 32, 2, 30);
- const maxSheets = clamp(Math.round(typeof body.maxSheets === "number" ? body.maxSheets : 4), 1, 4);
- const out = await makeContactSheets(id, body.file, { from, to, interval, maxSheets });
- // The per-call decode cap is itself truncation — the caller asked for more.
- if (to < wanted && !out.truncated) {
- out.truncated = true;
- out.coveredTo = to;
- }
- return Response.json(out);
- } catch (e) {
- return caught(e, "Could not sample the video.");
- }
- },
-
/** Report silent stretches in a media file's audio. */
async silence(req: Request, { id }: { id: string }) {
try {
diff --git a/site/src/cut/server/http/routes.ts b/site/src/cut/server/http/routes.ts
index accdb817..3b6aa2ef 100644
--- a/site/src/cut/server/http/routes.ts
+++ b/site/src/cut/server/http/routes.ts
@@ -46,7 +46,6 @@ export const CUT_ROUTES: CutRoute[] = [
{ method: "GET", path: "/api/cut/projects/:id/transcribe", handler: (req, p) => projectsApi.transcribePoll(req, { id: p.id }) },
{ method: "POST", path: "/api/cut/projects/:id/image", handler: (req, p) => projectsApi.importImage(req, { id: p.id }) },
{ method: "POST", path: "/api/cut/projects/:id/freeze", handler: (req, p) => projectsApi.freeze(req, { id: p.id }) },
- { method: "POST", path: "/api/cut/projects/:id/watch", handler: (req, p) => projectsApi.watch(req, { id: p.id }) },
{ method: "POST", path: "/api/cut/projects/:id/silence", handler: (req, p) => projectsApi.silence(req, { id: p.id }) },
{ method: "POST", path: "/api/cut/projects/:id/audio", handler: (req, p) => projectsApi.audio(req, { id: p.id }) },
{ method: "POST", path: "/api/cut/projects/:id/duplicate", handler: (req, p) => projectsApi.duplicate(req, { id: p.id }) },