Open source CapCut alternative.
TypeScript
feat(site): filmstrip tiles show true frames and split at scene cuts [rebuild]
David Le8ea06e1137d5
diff --git a/site/src/cut/components/Timeline.tsx b/site/src/cut/components/Timeline.tsxindex 9461ab56..eb1a22a8 100644--- a/site/src/cut/components/Timeline.tsx+++ b/site/src/cut/components/Timeline.tsx@@ -43,6 +43,7 @@ import { useExports } from "@/cut/lib/exportStore";import { isDragActive, startDrag, subscribeDragActive } from "@/cut/lib/drag";import { CLIP_GAP, startLaneMove, startLaneTrim, type LaneDrag, type LaneKind } from "@/cut/lib/laneTracks";import { downloadMedia, ensurePeaks, importImage, importStockMusic, importStockVideo, peekEdgeFrame, requestEdgeFrame, revealMedia, stripFailedFor, subscribeStripStatus } from "@/cut/lib/media";+import { planFilmstrip, type FilmTile } from "@/cut/lib/filmstrip";import { track0Clips, laneGapAt, sameLane, type LaneRef, clipLen, clipSpeed, getClipSpans, overlayLaneOrder, overlayLayers, projectDuration, resolveTransitions, rippleInsert, useEditor } from "@/cut/lib/store";import type { VideoTrackPlacement } from "@/cut/lib/store";import { playheadAt, setSkim, skimAt, subscribePlayhead, usePlayhead, useSkim } from "@/cut/lib/playhead";@@ -3512,14 +3513,19 @@ function ClipView({() => stripFailedFor(asset.id),() => false);+ const boxRef = useRef<HTMLDivElement>(null);+ // Bumped as tile captures land, so the memo re-plans and picks them up.+ const [filmGen, onTileFrame] = useReducer((x: number) => x + 1, 0);const filmstrip = useMemo(() =>filmstripFrames(asset, filmIn, w, pps, speed, VIDEO_H - 4, 26, {start: startFrame,end: endFrame,}),- [asset, filmIn, w, pps, speed, startFrame, endFrame]+ // eslint-disable-next-line react-hooks/exhaustive-deps+ [asset, filmIn, w, pps, speed, startFrame, endFrame, filmGen]);+ useTileFrames(boxRef, clip.id, asset, filmstrip, filmIn, w, pps, speed, onTileFrame);// The move gesture is the shared lane behavior (parting, snapping); its// verticality is the video placement system — the other tracks and the new@@ -3561,6 +3567,7 @@ function ClipView({// Inline so it beats SELECTED_SHADOW's z-10 class on the same element.zIndex: drag ? 20 : undefined,}}+ ref={boxRef}data-tl-sel={`clip:${clip.id}`}onPointerDown={(e) => startLaneMove(e, lane, clip.id, ui)}onContextMenu={(e) => {@@ -3923,7 +3930,9 @@ function DropGhostFilm({* tiles at each end). Tiles keep the asset's aspect until the box would blow* the tile cap; past that they widen by whole grid cells, still on the grid.* The first and last tiles pin to the segment's exact boundary frames once- * captured; middle tiles keep the nearest pre-sampled thumb. */+ * captured; middle tiles show the nearest pre-sampled thumb until the zoom+ * outruns the strip's sampling, when each carries a true frame at its own+ * moment as its capture lands (see `planFilmstrip`). */function filmstripFrames(asset: MediaAsset | undefined,filmIn: number,@@ -3933,24 +3942,21 @@ function filmstripFrames(tileH: number,minTileW: number,edges?: { start: string | null; end: string | null }-) {+): FilmTile[] {if (!asset?.thumbs?.length || !asset.thumbStep) return [];- const aspect = (asset.width ?? 16) / Math.max(1, asset.height ?? 9);- const natural = Math.max(minTileW, Math.round(tileH * aspect));- const cells = Math.max(1, Math.ceil(w / natural));- const imgW = natural * Math.ceil(cells / 120);- const stepT = (imgW / pps) * speed;- const filmOut = filmIn + (w / pps) * speed;- const first = Math.max(0, Math.floor(filmIn / stepT));- const last = Math.max(first, Math.ceil(filmOut / stepT) - 1);- const frames = Array.from({ length: last - first + 1 }, (_, i) => {- const k = first + i;- const timeAt = (k + 0.5) * stepT;- const idx = Math.min(- asset.thumbs!.length - 1,- Math.max(0, Math.floor(timeAt / asset.thumbStep!))- );- return { src: asset.thumbs![idx], left: ((k * stepT - filmIn) / speed) * pps, width: imgW };+ const frames = planFilmstrip({+ thumbs: asset.thumbs,+ thumbStep: asset.thumbStep,+ duration: asset.duration,+ aspect: (asset.width ?? 16) / Math.max(1, asset.height ?? 9),+ filmIn,+ w,+ pps,+ speed,+ tileH,+ minTileW,+ cuts: asset.sceneCuts,+ exactFrame: asset.type === "video" ? (t) => peekEdgeFrame(asset.url, t) : undefined,});if (edges?.start) frames[0] = { ...frames[0], src: edges.start };if (edges?.end && frames.length > 1) {@@ -3959,6 +3965,93 @@ function filmstripFrames(return frames;}+/** How far past the scroller's view a tile still gets its true frame, so a+ * small scroll shows captured tiles at once. */+const TILE_CAPTURE_MARGIN = 240;++/** Capture the true frames a zoomed strip's tiles ask for. The plan names a+ * capture time on each tile the pre-sampled strip is too coarse for; this+ * effect requests the ones inside the scroller's view (plus a margin) through+ * the edge reader's background lane and re-renders as they land. Scroll fills+ * in newly visible tiles; a zoom or trim re-aims the same per-tile slots, so+ * superseded captures fall away on their own. */+function useTileFrames(+ boxRef: React.RefObject<HTMLDivElement | null>,+ clipId: string,+ asset: MediaAsset | undefined,+ tiles: FilmTile[],+ filmIn: number,+ w: number,+ pps: number,+ speed: number,+ onFrame: () => void+) {+ // The landed-frame re-render hands this hook a fresh `tiles` array with the+ // same geometry; the ref keeps that from re-running the capture effect,+ // while scroll and the effect body always read the newest plan.+ const tilesRef = useRef(tiles);+ useEffect(() => {+ tilesRef.current = tiles;+ });+ const wanted = tiles.some((t) => t.wantT !== null);+ useEffect(() => {+ const box = boxRef.current;+ const url = asset?.type === "video" ? asset.url : null;+ if (!box || !url || !wanted) return;+ const scroller = box.closest("[data-tl-scroll]") as HTMLElement | null;+ let live = true;+ let scheduled = false;+ let bumping = false;+ // One request per capture time per plan: a failed read stays quiet until+ // a zoom or trim re-plans, and a scroll walk re-asks for nothing.+ const asked = new Set<string>();+ const landed = () => {+ if (bumping) return;+ bumping = true;+ requestAnimationFrame(() => {+ bumping = false;+ if (live) onFrame();+ });+ };+ const request = () => {+ scheduled = false;+ if (!live) return;+ let lo = 0;+ let hi = w;+ if (scroller) {+ const boxR = box.getBoundingClientRect();+ const view = scroller.getBoundingClientRect();+ lo = view.left - boxR.left - TILE_CAPTURE_MARGIN;+ hi = view.right - boxR.left + TILE_CAPTURE_MARGIN;+ }+ tilesRef.current.forEach((t, i) => {+ if (t.wantT === null || t.exact) return;+ if (t.left + t.width < lo || t.left > hi) return;+ const key = `${i}:${t.wantT.toFixed(2)}`;+ if (asked.has(key)) return;+ asked.add(key);+ void requestEdgeFrame(`${clipId}:tile:${i}`, url, t.wantT, undefined, {+ background: true,+ }).then((src) => {+ if (live && src) landed();+ });+ });+ };+ const schedule = () => {+ if (scheduled) return;+ scheduled = true;+ requestAnimationFrame(request);+ };+ schedule();+ scroller?.addEventListener("scroll", schedule, { passive: true });+ return () => {+ live = false;+ scroller?.removeEventListener("scroll", schedule);+ };+ // Geometry names the plan; `tiles` itself rides the ref above.+ }, [boxRef, clipId, asset, wanted, filmIn, w, pps, speed, onFrame]);+}+/** The exact source frame at a clip edge. Returns null (the nearest sampled* thumb shows instead) until the capture lands; a changed edge time falls back* immediately so a trim drag never shows a stale exact frame. */diff --git a/site/src/cut/lib/devHooks.ts b/site/src/cut/lib/devHooks.tsindex 80ea064e..dcb6afe6 100644--- a/site/src/cut/lib/devHooks.ts+++ b/site/src/cut/lib/devHooks.ts@@ -10,7 +10,7 @@import { renderProjectToMp4 } from "./exportRender";import { useGenerate } from "./generate";import { useGenScene } from "./genScene";-import { enrichAsset, importFileToProject } from "./media";+import { edgeFramesPending, enrichAsset, importFileToProject } from "./media";import { awaitingFrame, startTrace, stopTrace } from "./perfTrace";import { playheadAt } from "./playhead";import { projectDuration, useEditor } from "./store";@@ -23,6 +23,9 @@ export function installDevHooks(): void {useGenScene,importFileToProject,enrichAsset,+ // The filmstrip eval waits on zero pending captures before it reads the+ // strip's tiles back out of the DOM.+ edgeFramesPending,playheadAt,// The export eval renders a doc through the tab's own pipeline and reads// the bytes back out.diff --git a/site/src/cut/lib/filmstrip.test.ts b/site/src/cut/lib/filmstrip.test.tsnew file mode 100644index 00000000..1baf4e30--- /dev/null+++ b/site/src/cut/lib/filmstrip.test.ts@@ -0,0 +1,231 @@+import { describe, expect, test } from "bun:test";+import {+ FILM_TILE_CAP,+ MIN_SUB_TILE_PX,+ pickThumbTimes,+ planFilmstrip,+ WANT_GRID_S,+} from "./filmstrip";++/** The user's case: a ~24s clip with fast scene cuts carries 12 pre-sampled+ * thumbs, one every ~2s. */+const DURATION = 23.66;+const THUMB_STEP = DURATION / 12;+const thumbs = Array.from({ length: 12 }, (_, i) => `thumb-${i}`);++/** A whole-clip box drawn at `pps`, its width following the zoom the way a+ * real clip's does. */+const plan = (over: Partial<Parameters<typeof planFilmstrip>[0]> = {}) =>+ planFilmstrip({+ thumbs,+ thumbStep: THUMB_STEP,+ duration: DURATION,+ aspect: 16 / 9,+ filmIn: 0,+ w: (over.duration ?? DURATION) * (over.pps ?? 60),+ pps: 60,+ speed: 1,+ tileH: 60,+ minTileW: 26,+ ...over,+ });++describe("planFilmstrip", () => {+ test("tiles cover the drawn width on a source-time grid", () => {+ const tiles = plan();+ expect(tiles.length).toBeGreaterThan(0);+ expect(tiles[0].left).toBeLessThanOrEqual(0);+ const end = tiles[tiles.length - 1];+ expect(end.left + end.width).toBeGreaterThanOrEqual(DURATION * 60);+ for (let i = 1; i < tiles.length; i++) {+ expect(tiles[i].left).toBeCloseTo(tiles[i - 1].left + tiles[i - 1].width, 6);+ }+ });++ test("each tile shows the pre-sampled thumb nearest its own moment", () => {+ for (const t of plan()) {+ const idx = Number(t.src.slice("thumb-".length));+ const sampledAt = (idx + 0.5) * THUMB_STEP;+ // The end tiles overhang the source and clamp to the last thumb.+ const inSource = Math.min(t.srcT, DURATION);+ expect(Math.abs(sampledAt - inSource)).toBeLessThanOrEqual(THUMB_STEP / 2 + 1e-9);+ }+ });++ test("zoomed out past the strip's sampling, no tile asks for a capture", () => {+ const tiles = plan({ pps: 30 });+ expect(tiles.every((t) => t.wantT === null && !t.exact)).toBe(true);+ });++ test("zoomed in, every tile asks for a capture at its own midpoint", () => {+ // Half the capture grid is the whole allowance: a capture pulled further+ // toward a tile's edge lands on the neighboring scene whenever a cut sits+ // inside the tile, and the strip flips a whole tile early.+ for (const pps of [120, 200, 300, 480, 800]) {+ const tiles = plan({ pps });+ for (const t of tiles) {+ expect(t.wantT === null).toBe(false);+ if (t.wantT! <= 0 || t.wantT! >= DURATION - 0.05) continue;+ expect(Math.abs(t.wantT! - t.srcT)).toBeLessThanOrEqual(WANT_GRID_S / 2 + 1e-9);+ expect(t.exact).toBe(false);+ }+ }+ });++ test("capture times land on fine-grid centers, clear of frame boundaries", () => {+ // Grid times keep cache keys stable while the strip scrolls, trims, and+ // re-plans; centers keep captures off the whole- and half-second moments+ // where scene cuts live, so a straddling tile stays on its midpoint's+ // side of the cut.+ for (const t of plan({ pps: 300 })) {+ if (t.wantT === null || t.wantT <= 0 || t.wantT >= DURATION - 0.05) continue;+ const cells = t.wantT / WANT_GRID_S - 0.5;+ expect(Math.abs(cells - Math.round(cells))).toBeLessThan(1e-6);+ }+ });++ test("a tile whose midpoint sits a hair before a cut stays before it", () => {+ // The regression the timeline showed: a midpoint at ~2.97 must capture+ // below 3.0, never at it — the frame at 3.0 is the next scene's first.+ const pps = 270; // stepT ≈ 0.396 puts a tile midpoint at ~2.972+ const nearCut = plan({ pps }).filter((t) => t.srcT > 2.95 && t.srcT < 3);+ expect(nearCut.length).toBeGreaterThan(0);+ for (const t of nearCut) expect(t.wantT!).toBeLessThan(3);+ });++ test("capture times stay inside the source", () => {+ for (const t of plan({ pps: 800 })) {+ expect(t.wantT!).toBeGreaterThanOrEqual(0);+ expect(t.wantT!).toBeLessThanOrEqual(DURATION - 0.05 + 1e-9);+ }+ });++ test("a captured frame replaces the thumb and marks the tile exact", () => {+ const tiles = plan({ pps: 300, exactFrame: (t) => (t < 12 ? `frame-${t.toFixed(3)}` : null) });+ for (const t of tiles) {+ if (t.wantT! < 12) {+ expect(t.exact).toBe(true);+ expect(t.src).toBe(`frame-${t.wantT!.toFixed(3)}`);+ } else {+ expect(t.exact).toBe(false);+ expect(t.src.startsWith("thumb-")).toBe(true);+ }+ }+ });++ test("a trimmed left edge keeps tiles on the same source grid", () => {+ const pps = 300;+ const whole = plan({ pps });+ const trimmed = plan({ pps, filmIn: 3.2, w: (DURATION - 3.2) * pps });+ const wholeTimes = new Set(whole.map((t) => t.srcT.toFixed(4)));+ for (const t of trimmed) expect(wholeTimes.has(t.srcT.toFixed(4))).toBe(true);+ });++ test("past the tile cap, tiles widen by whole cells and stay capped", () => {+ const tiles = plan({ pps: 800, duration: 600, w: 600 * 800, thumbStep: 600 / 24 });+ expect(tiles.length).toBeLessThanOrEqual(FILM_TILE_CAP + 1);+ });++ test("an empty strip plans nothing", () => {+ expect(plan({ thumbs: [] })).toEqual([]);+ });++ test("a scene cut inside a refining tile splits it at the cut", () => {+ const pps = 300;+ const cut = 3.0;+ const tiles = plan({ pps, cuts: [cut] });+ // Some tile boundary sits exactly on the cut...+ const edges = tiles.map((t) => t.left);+ expect(edges.some((x) => Math.abs(x - cut * pps) < 1e-6)).toBe(true);+ // ...and each side captures strictly on its own side of it.+ for (const t of tiles) {+ const a = t.left / pps;+ const b = (t.left + t.width) / pps;+ if (b <= cut + 1e-9) expect(t.wantT!).toBeLessThan(cut);+ if (a >= cut - 1e-9) expect(t.wantT!).toBeGreaterThan(cut);+ }+ });++ test("split tiles still tile the strip without gaps", () => {+ const tiles = plan({ pps: 300, cuts: [1.7, 3.0, 9.13] });+ for (let i = 1; i < tiles.length; i++) {+ expect(tiles[i].left).toBeCloseTo(tiles[i - 1].left + tiles[i - 1].width, 6);+ }+ });++ test("a cut a sliver from a tile boundary does not split off a sub-pixel tile", () => {+ const pps = 300;+ const stepT = plan({ pps })[1].width / pps;+ const sliver = stepT + MIN_SUB_TILE_PX / pps / 2; // just inside tile 1+ const tiles = plan({ pps, cuts: [sliver] });+ for (const t of tiles) expect(t.width).toBeGreaterThanOrEqual(MIN_SUB_TILE_PX - 1e-6);+ });++ test("zoomed out, cuts split nothing", () => {+ const zoomedOut = plan({ pps: 30, cuts: [3.0, 6.0, 9.0] });+ expect(zoomedOut).toEqual(plan({ pps: 30 }));+ });++ test("no cuts leaves the plan unchanged", () => {+ expect(plan({ pps: 300, cuts: [] })).toEqual(plan({ pps: 300 }));+ });+});++describe("pickThumbTimes", () => {+ /** Probes every `at` seconds with scene cuts at the given times. */+ const probesWithCuts = (duration: number, at: number, cuts: number[]) => {+ const out = [];+ for (let t = at / 2; t < duration; t += at) {+ const cut = cuts.some((c) => t - at < c && c <= t);+ out.push({ t, cut });+ }+ return out;+ };++ test("no probes: every bucket grabs its midpoint", () => {+ const times = pickThumbTimes(12, 2, 24, []);+ times.forEach((t, k) => expect(t).toBeCloseTo(Math.min(24 - 0.05, (k + 0.5) * 2), 6));+ });++ test("no cuts: every bucket grabs near its midpoint", () => {+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, []));+ times.forEach((t, k) => {+ expect(Math.abs(t - (k + 0.5) * 2)).toBeLessThanOrEqual(0.25);+ });+ });++ test("a cut late in a bucket: the grab stays with the dominant scene", () => {+ // The regression the timeline showed: a scene cut at 3.0s inside the+ // bucket [2, 4) — most of the bucket is the first scene, so the thumb+ // must come from before the cut... with the cut at the bucket's own+ // 3/4 point, the run before it is the longer one.+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, [3.5]));+ const bucket1 = times[1]; // bucket [2, 4)+ expect(bucket1).toBeLessThan(3.5);+ expect(bucket1).toBeGreaterThanOrEqual(2);+ });++ test("a cut early in a bucket: the grab jumps past it", () => {+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, [2.5]));+ const bucket1 = times[1]; // bucket [2, 4): the scene after 2.5 owns it+ expect(bucket1).toBeGreaterThan(2.5);+ expect(bucket1).toBeLessThan(4);+ });++ test("grabs never land right on a cut", () => {+ const cuts = [3, 6, 9, 12, 15, 18, 21];+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, cuts));+ for (const t of times) {+ for (const c of cuts) expect(Math.abs(t - c)).toBeGreaterThan(0.1);+ }+ });++ test("grabs stay inside their own bucket, ascending", () => {+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, [1, 4.2, 17.9]));+ times.forEach((t, k) => {+ expect(t).toBeGreaterThanOrEqual(k * 2);+ expect(t).toBeLessThan((k + 1) * 2);+ });+ for (let i = 1; i < times.length; i++) expect(times[i]).toBeGreaterThan(times[i - 1]);+ });+});diff --git a/site/src/cut/lib/filmstrip.ts b/site/src/cut/lib/filmstrip.tsnew file mode 100644index 00000000..56fd158f--- /dev/null+++ b/site/src/cut/lib/filmstrip.ts@@ -0,0 +1,173 @@+/**+ * Filmstrip tile planning: which frame each tile of a clip's strip shows.+ *+ * Tiles sit on a source-time grid, so the strip holds still while a trimmed+ * edge sweeps across it. Each tile spans `stepT` seconds of source; the asset+ * carries a pre-sampled strip with one thumb every `thumbStep` seconds+ * (roughly two). While a tile spans at least that much, the nearest+ * pre-sampled thumb is a faithful picture of the tile's whole span.+ *+ * Zoomed past that, a tile spans a fraction of the sampling interval, and on+ * footage that cuts between scenes the nearest thumb can show a moment up to+ * `thumbStep / 2` away — a visibly wrong scene under the playhead. Those+ * tiles each name a capture time (`wantT`) for a true frame at their own+ * moment: the tile's midpoint, on a fine fixed grid (tenths of a second) so+ * cache keys hold still through scrolls and trims. The midpoint is the one+ * time that keeps a tile on its own side of a scene cut — a capture pulled+ * toward the tile's edge lands on the neighboring scene whenever a cut sits+ * inside the tile, and the strip then flips a whole tile early. `exactFrame`+ * answers with captures already in hand; a tile without one keeps the nearest+ * pre-sampled thumb until its capture lands.+ *+ * Detected scene cuts (`cuts`, measured at import) split any refining tile+ * they pass through: each side captures its own scene, so the strip changes+ * picture at the cut's own pixel and the boundary area under the pointer+ * shows what actually plays there.+ */++export type FilmTile = {+ src: string;+ /** Pixel offset within the clip box. */+ left: number;+ width: number;+ /** The tile's midpoint in source seconds. */+ srcT: number;+ /** Capture time for a true frame; null while the pre-sampled strip is+ * already finer than the tile grid. */+ wantT: number | null;+ /** `src` is a true frame at `wantT`. */+ exact: boolean;+};++/** Widest a strip gets before tiles widen by whole grid cells. */+export const FILM_TILE_CAP = 120;++/** Capture times land on this grid, tenths of a second: fine enough that a+ * capture sits imperceptibly close to its tile's midpoint, coarse enough that+ * a strip re-planned by a scroll or a trim asks for the same times again. */+export const WANT_GRID_S = 0.1;++/** Narrowest sub-tile a scene cut may split off. */+export const MIN_SUB_TILE_PX = 5;++/** A probe of the source: a moment, and whether the picture changed scene+ * between the previous probe and this one. */+export type ThumbProbe = { t: number; cut: boolean };++/** Where the pre-sampled strip grabs each bucket's thumb.+ *+ * The strip is read back by position (`thumbs[floor(t / thumbStep)]`), so+ * each bucket owns one thumb; this picks the moment inside the bucket that+ * thumb is grabbed at. Probes carry scene-change signals measured off small+ * decoded frames (the watch pipeline's grid signatures): each bucket grabs+ * the middle of its longest stretch of stable content, so a bucket crossed+ * by a scene cut shows the scene that owns most of it — a blind midpoint+ * grab lands right at a cut often enough to paint whole buckets with their+ * neighbor's scene. A bucket with no probes grabs its midpoint. */+export function pickThumbTimes(+ count: number,+ thumbStep: number,+ duration: number,+ probes: ThumbProbe[]+): number[] {+ const clampT = (t: number) => Math.max(0, Math.min(duration - 0.05, t));+ return Array.from({ length: count }, (_, k) => {+ const lo = k * thumbStep;+ const hi = (k + 1) * thumbStep;+ const inBucket = probes.filter((p) => p.t >= lo && p.t < hi);+ const mid = (k + 0.5) * thumbStep;+ if (inBucket.length === 0) return clampT(mid);+ const runs: ThumbProbe[][] = [[inBucket[0]]];+ for (let i = 1; i < inBucket.length; i++) {+ if (inBucket[i].cut) runs.push([]);+ runs[runs.length - 1].push(inBucket[i]);+ }+ const center = (r: ThumbProbe[]) => r[Math.floor((r.length - 1) / 2)].t;+ const best = runs.reduce((a, b) =>+ b.length > a.length ||+ (b.length === a.length && Math.abs(center(b) - mid) < Math.abs(center(a) - mid))+ ? b+ : a+ );+ return clampT(center(best));+ });+}++export function planFilmstrip(p: {+ thumbs: string[];+ thumbStep: number;+ /** Source length in seconds; capture times clamp inside it. */+ duration: number;+ /** Source width/height ratio. */+ aspect: number;+ /** Source time at the clip's left edge. */+ filmIn: number;+ /** Drawn width of the clip box, px. */+ w: number;+ pps: number;+ speed: number;+ tileH: number;+ minTileW: number;+ /** Scene changes in the source, seconds. A refining tile a cut passes+ * through splits at it, so the strip changes picture at the cut's own+ * pixel; the boundary area then shows what actually plays there. */+ cuts?: number[];+ /** Synchronous lookup of a captured true frame at a `wantT`. */+ exactFrame?: (wantT: number) => string | null;+}): FilmTile[] {+ if (!p.thumbs.length || !p.thumbStep) return [];+ const natural = Math.max(p.minTileW, Math.round(p.tileH * p.aspect));+ const cells = Math.max(1, Math.ceil(p.w / natural));+ const imgW = natural * Math.ceil(cells / FILM_TILE_CAP);+ const stepT = (imgW / p.pps) * p.speed;+ const filmOut = p.filmIn + (p.w / p.pps) * p.speed;+ const first = Math.max(0, Math.floor(p.filmIn / stepT));+ const last = Math.max(first, Math.ceil(filmOut / stepT) - 1);+ const refine = stepT < p.thumbStep;+ // A sub-tile narrower than a few pixels reads as a rendering artifact; a+ // cut that close to a tile boundary already flips within that distance.+ const minSubT = (MIN_SUB_TILE_PX / p.pps) * p.speed;+ const tiles: FilmTile[] = [];+ for (let k = first; k <= last; k++) {+ const t0 = k * stepT;+ const t1 = (k + 1) * stepT;+ const inner = refine+ ? (p.cuts ?? []).filter((c) => c > t0 + minSubT && c < t1 - minSubT).slice(0, 2)+ : [];+ const bounds = [t0, ...inner, t1];+ for (let s = 0; s + 1 < bounds.length; s++) {+ const a = bounds[s];+ const b = bounds[s + 1];+ const mid = (a + b) / 2;+ const idx = Math.min(p.thumbs.length - 1, Math.max(0, Math.floor(mid / p.thumbStep)));+ let src = p.thumbs[idx];+ let wantT: number | null = null;+ let exact = false;+ if (refine) {+ // Grid centers, never grid boundaries: cuts sit on frame boundaries+ // (whole and half seconds above all), and a capture that lands+ // exactly on a cut decodes the next scene's first frame — a tile+ // whose midpoint sits a hair before a cut would flip to the wrong+ // side of it. A sub-tile too narrow for a grid center inside it+ // captures at its own midpoint.+ const snapped = (Math.floor(mid / WANT_GRID_S) + 0.5) * WANT_GRID_S;+ wantT = snapped > a && snapped < b ? snapped : mid;+ wantT = Math.max(0, Math.min(p.duration - 0.05, wantT));+ const hit = p.exactFrame?.(wantT) ?? null;+ if (hit) {+ src = hit;+ exact = true;+ }+ }+ tiles.push({+ src,+ left: ((a - p.filmIn) / p.speed) * p.pps,+ width: ((b - a) / p.speed) * p.pps,+ srcT: mid,+ wantT,+ exact,+ });+ }+ }+ return tiles;+}diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 6e39e407..69876ae3 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -41,6 +41,7 @@ import {rasterCanvasToDataUrl,type RasterSurface,} from "./raster";+import { pickThumbTimes, type ThumbProbe } from "./filmstrip";import { useEditor } from "./store";import type { AssetType, AudioClip, MediaAsset, ProjectSummary, StoredAsset, VideoClip, WatchKeepReason } from "./types";import { contentRect, IMAGE_CLIP_SECONDS, mediaUrl } from "./types";@@ -1471,14 +1472,18 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {const key = stripCacheKey(useEditor.getState().projectId, asset.fileName);const cached = await readCachedStrip(key, asset.duration);if (cached) {- useEditor.getState().updateAsset(asset.id, { thumbs: cached.thumbs, thumbStep: cached.thumbStep });+ useEditor.getState().updateAsset(asset.id, {+ thumbs: cached.thumbs,+ thumbStep: cached.thumbStep,+ sceneCuts: cached.cuts ?? [],+ });} else {- const { thumbs, thumbStep } = await makeThumbs(src, asset.duration);+ const { thumbs, thumbStep, cuts } = await makeThumbs(src, asset.duration);// An empty strip is a failure, not a result: persisting it would// leave the asset looking permanently mid-load.if (!thumbs.length) throw new UnreadableMediaError("No frames could be read for the filmstrip.");- useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep });- writeCachedStrip(key, { thumbs, thumbStep, duration: asset.duration, at: Date.now() });+ useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep, sceneCuts: cuts });+ writeCachedStrip(key, { thumbs, thumbStep, cuts, duration: asset.duration, at: Date.now() });}enrichRetries.delete(asset.id);} else if (asset.type === "audio" && !asset.peaks?.length) {@@ -1528,10 +1533,17 @@ const STRIP_DB = "cut-filmstrips";const STRIP_STORE = "strips";const STRIP_CAP = 500; // prune oldest beyond this many cached strips-type CachedStrip = { thumbs: string[]; thumbStep: number; duration: number; at: number };+type CachedStrip = {+ thumbs: string[];+ thumbStep: number;+ cuts?: number[];+ duration: number;+ at: number;+};function stripCacheKey(projectId: string | null, fileName: string) {- return `${projectId ?? ""}/${fileName}@${THUMB_H}`;+ // "s3" marks strips that carry scene cuts; strips cached before them regenerate.+ return `${projectId ?? ""}/${fileName}@${THUMB_H}s3`;}function openStripDb(): Promise<IDBDatabase> {@@ -1594,13 +1606,123 @@ function writeCachedStrip(key: string, strip: CachedStrip) {})();}+/** Probes per strip bucket for scene-change detection, and the cap on the+ * whole probe sweep so long files stay a bounded pass. */+const THUMB_PROBE_CAP = 120;+/** % of 16-grid cells that must change between adjacent probes to call a+ * scene cut. Far above the watch selector's "meaningfully new" bar (8%), so+ * motion and pans pass under it and only a real cut splits a bucket. */+const THUMB_PROBE_CUT_PCT = 45;++/** g16 grid signatures for ascending `times`, decoded small in one sweep;+ * null where the decoder had no frame. */+async function g16Sweep(url: string, times: number[]): Promise<(Float32Array | null)[]> {+ const sig = createRasterCanvas(SIGNATURE_SIZE, SIGNATURE_SIZE);+ const ctx = sig.getContext("2d", {+ willReadFrequently: true,+ }) as CanvasRenderingContext2D | null;+ if (!ctx) return times.map(() => null);+ const out: (Float32Array | null)[] = [];+ for await (const frame of framesAt(url, times, { height: SIGNATURE_SIZE })) {+ if (!frame) {+ out.push(null);+ continue;+ }+ ctx.drawImage(frame.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);+ out.push(+ frameSig({+ width: SIGNATURE_SIZE,+ height: SIGNATURE_SIZE,+ channels: 4,+ data: ctx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,+ }).g16+ );+ }+ return out;+}++/** Scene-change signals across the clip: one small decoded frame every+ * fraction of a bucket, compared through the watch pipeline's grid signature.+ * A probe is flagged `cut` when the picture changed scene since the probe+ * before it. */+async function thumbProbes(url: string, duration: number, count: number): Promise<ThumbProbe[]> {+ const perBucket = Math.max(3, Math.floor(THUMB_PROBE_CAP / count));+ const n = count * perBucket;+ const times = Array.from({ length: n }, (_, i) =>+ Math.max(0, Math.min(duration - 0.05, ((i + 0.5) * duration) / n))+ );+ const sigs = await g16Sweep(url, times);+ return times.map((t, i) => {+ const p = i > 0 ? sigs[i - 1] : null;+ const q = sigs[i];+ return {+ t,+ cut: !!p && !!q && diffCellPct(q, p, DEDUP_TUNING.GLOBAL_TOL) > THUMB_PROBE_CUT_PCT,+ };+ });+}++/** How many cut brackets get the dense second sweep, samples per bracket, and+ * the most cuts an asset stores at all. */+const CUT_REFINE_MAX = 32;+const CUT_REFINE_STEPS = 8;+const SCENE_CUTS_MAX = 256;++/** The clip's scene-change moments. The probe pass brackets each cut between+ * two probes; a denser sweep inside each bracket places the cut within an+ * eighth of the probe spacing (tens of milliseconds on a typical clip).+ * Brackets past the refinement cap keep their bracket midpoint. */+async function sceneCutTimes(url: string, probes: ThumbProbe[]): Promise<number[]> {+ const brackets: [number, number][] = [];+ for (let i = 1; i < probes.length; i++) {+ if (probes[i].cut) brackets.push([probes[i - 1].t, probes[i].t]);+ }+ const refine = brackets.slice(0, CUT_REFINE_MAX);+ const cuts = brackets.slice(CUT_REFINE_MAX).map(([a, b]) => (a + b) / 2);+ if (refine.length) {+ const S = CUT_REFINE_STEPS;+ const times = refine.flatMap(([a, b]) =>+ Array.from({ length: S + 1 }, (_, j) => a + (j * (b - a)) / S)+ );+ const sigs = await g16Sweep(url, times);+ refine.forEach(([a, b], r) => {+ let at = (a + b) / 2;+ let strongest = -1;+ for (let j = 1; j <= S; j++) {+ const p = sigs[r * (S + 1) + j - 1];+ const q = sigs[r * (S + 1) + j];+ if (!p || !q) continue;+ const d = diffCellPct(q, p, DEDUP_TUNING.GLOBAL_TOL);+ if (d > strongest) {+ strongest = d;+ at = a + ((j - 0.5) * (b - a)) / S;+ }+ }+ cuts.push(at);+ });+ }+ return cuts+ .map((c) => Math.round(c * 100) / 100)+ .sort((x, y) => x - y)+ .slice(0, SCENE_CUTS_MAX);+}+async function makeThumbs(url: string, duration: number) {// One frame every ~2s (min 10, max 24) so long clips don't repeat frames.const count = Math.min(24, Math.max(10, Math.round(duration / 2)));const thumbStep = duration / count;- const times = Array.from({ length: count }, (_, i) =>- Math.max(0, Math.min(duration - 0.05, (i + 0.5) * thumbStep))- );+ // Scene-aware grab points: probe the clip for scene changes, then grab each+ // bucket's thumb from its dominant scene (bucket midpoints when the probe+ // pass yields nothing).+ let probes: ThumbProbe[] = [];+ let cuts: number[] = [];+ try {+ probes = await thumbProbes(url, duration, count);+ cuts = await sceneCutTimes(url, probes);+ } catch {+ // Probing is advisory; the strip still builds from midpoints.+ }+ const times = pickThumbTimes(count, thumbStep, duration, probes);// Ascending times over one decode pass — the strip is a single sweep of the// file rather than `count` seeks into it.//@@ -1623,7 +1745,7 @@ async function makeThumbs(url: string, duration: number) {thumbs.push(fill);}}- return { thumbs, thumbStep };+ return { thumbs, thumbStep, cuts };}// Exact edge frames: a clip's first and last filmstrip tiles show the true@@ -1658,6 +1780,9 @@ type EdgeReader = {const edgeCache = new Map<string, string>();const edgeQueue = new Map<string, EdgeRequest>();+/** Filmstrip tile captures wait here, behind every queued clip-edge capture,+ * so a trim drag's frames land first while a zoomed strip fills in behind. */+const edgeBackQueue = new Map<string, EdgeRequest>();const edgePool = new Map<string, Promise<EdgeReader>>();let edgePumping = false;@@ -1670,30 +1795,39 @@ export function peekEdgeFrame(url: string, time: number, height = THUMB_H): strireturn edgeCache.get(edgeKey(url, time, height)) ?? null;}-/** Capture the frame at `time`, latest-wins per `slot` (a clip edge). Resolves- * with the frame, or null when superseded by a newer request or on a failed- * read. */+/** Capture the frame at `time`, latest-wins per `slot` (a clip edge or a+ * strip tile). Resolves with the frame, or null when superseded by a newer+ * request or on a failed read. `background` requests wait behind every queued+ * foreground one. */export function requestEdgeFrame(slot: string,url: string,time: number,- height = THUMB_H+ height = THUMB_H,+ opts?: { background?: boolean }): Promise<string | null> {const key = edgeKey(url, time, height);const hit = edgeCache.get(key);if (hit) return Promise.resolve(hit);+ const queue = opts?.background ? edgeBackQueue : edgeQueue;return new Promise((resolve) => {- const prev = edgeQueue.get(slot);+ const prev = queue.get(slot);if (prev?.key === key) {prev.resolvers.push(resolve);} else {prev?.resolvers.forEach((r) => r(null));- edgeQueue.set(slot, { url, time, height, key, resolvers: [resolve] });+ queue.set(slot, { url, time, height, key, resolvers: [resolve] });}void pumpEdgeFrames();});}+/** Captures still queued or mid-read. The filmstrip eval waits on zero to+ * know a zoomed strip has finished filling in. */+export function edgeFramesPending(): number {+ return edgeQueue.size + edgeBackQueue.size + (edgePumping ? 1 : 0);+}+function edgeReader(url: string): Promise<EdgeReader> {const hit = edgePool.get(url);if (hit) {@@ -1736,10 +1870,11 @@ async function pumpEdgeFrames() {edgePumping = true;try {for (;;) {- const next = edgeQueue.entries().next();+ const queue = edgeQueue.size ? edgeQueue : edgeBackQueue;+ const next = queue.entries().next();if (next.done) break;const [slot, req] = next.value;- edgeQueue.delete(slot);+ queue.delete(slot);let src = edgeCache.get(req.key) ?? null;if (!src) {try {diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.tsindex ff15e109..2736c1e6 100644--- a/site/src/cut/lib/types.ts+++ b/site/src/cut/lib/types.ts@@ -183,6 +183,10 @@ export interface StoredAsset {/** BCP-47 of the audio's spoken language, when known (stamped on voiceovers* at synthesis) — what transcription should run its recognizer in. */language?: string;+ /** Scene changes detected in the source at import (seconds, near frame+ * accurate). The timeline strip splits its tiles on these so the strip+ * changes picture where the source does. */+ sceneCuts?: number[];/** For origin "chat": the chat thread that made it. Deleting that thread* deletes the assets it still owns (see chatAssets.ts). */chatId?: string;