Open source CapCut alternative.
TypeScript
fix(cut): land watch scene changes on their frame
David Leb571e9b3b9ed
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 31c4bf7c..520814bb 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1302,6 +1302,7 @@ export async function captureFreezeFrame(}const round2 = (n: number) => Math.round(n * 100) / 100;+const round3 = (n: number) => Math.round(n * 1000) / 1000;/** Decoded audio for a span of a source, chunk by chunk, so scanning a long* file costs one decoded chunk at a time rather than a decoded copy of the@@ -1442,6 +1443,7 @@ const CANDIDATE_CAP = 150; // decodes per call — the densest sweepconst SELECT_CELL = 480; // what the selector decodes at when the frames are biggerconst REFINE_TARGET_S = 0.35; // localize each hard cut to about this windowconst REFINE_MAX_PROBES = 48; // extra seeks the refinement pass may spend per call+const CUT_WALK_EPS = 0.0005; // half a millisecond: lands a walk's ends on the frames they nameconst AIMED_SHARE = 0.5; // most of the cap aimed moments may take from the floorconst MERGE_EPS = 0.15; // two candidate times this close are one decode@@ -1570,8 +1572,9 @@ export async function makeStillFrame(* moments `at` aims at, then let the shared selector keep the frames that* actually differ. sceneChanges are* hard-cut times: each "global" keep says a cut lies inside the grid step- * before it, and a short bisection pass narrows that window to about- * REFINE_TARGET_S — the reported time is where the new shot is first seen.+ * before it, a short bisection pass narrows that window to about+ * REFINE_TARGET_S, and an in-order walk of the window's frames reports the+ * first frame of the new shot, to the millisecond.* A shot shorter than the candidate step can still fall between samples; a* narrow re-watch with a small interval is the recall pass.*@@ -1648,14 +1651,31 @@ export async function sampleWatchFrames(if (!sigCtx) throw new Error("Could not sample the video.");const deadline = Date.now() + (opts.budgetMs ?? Infinity);const outOfTime = () => Date.now() > deadline;- const sigG16At = async (t: number): Promise<Float32Array | null> => {- const one = await baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).getCanvas(t);- if (!one) return null;+ const sigG16Of = (canvas: CanvasImageSource): Float32Array => {sigCtx.imageSmoothingQuality = "high";- sigCtx.drawImage(one.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);+ sigCtx.drawImage(canvas, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);const data = sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data;return frameSig({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 4, data }).g16;};+ // The timestamp of the first frame in (lo, hi] that differs from `pre`,+ // decoded in order; null when the walk finds none.+ const firstNewFrame = async (lo: number, hi: number, pre: Float32Array): Promise<number | null> => {+ const walk = baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).canvases(lo, hi + CUT_WALK_EPS);+ try {+ for await (const frame of walk) {+ if (frame.timestamp <= lo - CUT_WALK_EPS) continue;+ const g = sigG16Of(frame.canvas as CanvasImageSource);+ if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) return frame.timestamp;+ }+ } finally {+ await walk.return(undefined);+ }+ return null;+ };+ const sigG16At = async (t: number): Promise<Float32Array | null> => {+ const one = await baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).getCanvas(t);+ return one ? sigG16Of(one.canvas as CanvasImageSource) : null;+ };// The times are ascending, so the whole span decodes once, in order.const cells = baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).canvasesAtTimestamps(times);@@ -1761,9 +1781,8 @@ export async function sampleWatchFrames(encoded.clear();// A "global" keep only says the cut lies in the grid step before it.- // Bisect that step with a few extra decodes so each reported cut time- // sits within REFINE_TARGET_S of the first new-shot frame; the kept- // image stays the grid sample.+ // Bisect that step with a few extra decodes down to REFINE_TARGET_S, then+ // walk what is left frame by frame; the kept image stays the grid sample.let probes = 0;const sceneChanges: number[] = [];for (const idx of keptIdx) {@@ -1785,7 +1804,14 @@ export async function sampleWatchFrames(if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) hi = mid;else lo = mid;}- sceneChanges.push(round2(hi));+ // The window is a few frames wide now: walk it in order and report the+ // first frame of the new shot itself, so a cut lands on its frame.+ const first =+ pre !== null && probes < REFINE_MAX_PROBES && !outOfTime() && !opts.shouldPause?.()+ ? await firstNewFrame(lo, hi, pre)+ : null;+ if (first !== null) probes++;+ sceneChanges.push(round3(first ?? hi));}// An interrupted or capped scan only covered what it decoded and kept;diff --git a/site/src/cut/lib/sampleWatchFrames.test.ts b/site/src/cut/lib/sampleWatchFrames.test.tsindex e1297ed2..a5af9871 100644--- a/site/src/cut/lib/sampleWatchFrames.test.ts+++ b/site/src/cut/lib/sampleWatchFrames.test.ts@@ -116,3 +116,49 @@ test("the top of the ladder is bounded, and only it walks twice", () => {expect(watchGeometry("scan").rewalk).toBe(false);expect(watchGeometry("read").rewalk).toBe(false);});++test("a hard cut is reported on the first frame of the new shot, to the millisecond", async () => {+ // A 60 fps source whose picture turns from black to white at frame 61+ // (1.0167s). The sweep only says the cut sits between two samples a second+ // apart; the reported time has to be the frame itself.+ const FPS = 60;+ const CUT_FRAME = 61;+ const shade = (t: number) => (Math.floor(t * FPS + 1e-6) >= CUT_FRAME ? 255 : 0);+ type Painted = { getContext: () => unknown; shade: number };+ const surface = (): Painted => {+ const own: Painted = {+ shade: 0,+ getContext: () => ({+ drawImage(src: Painted) { own.shade = src.shade; },+ getImageData(_x: number, _y: number, width: number, height: number) {+ return { data: new Uint8ClampedArray(width * height * 4).fill(own.shade) };+ },+ }),+ };+ return own;+ };+ const shot = (t: number) => {+ const canvas = surface();+ canvas.shade = shade(t);+ return { canvas, timestamp: Math.floor(t * FPS + 1e-6) / FPS, duration: 1 / FPS } as unknown as WrappedCanvas;+ };+ const input = { dispose() {} } as unknown as Input;+ const track = { getDisplayWidth: async () => 320, getDisplayHeight: async () => 180 } as unknown as InputVideoTrack;+ const spies = [+ spyOn(mediaRead, "openMedia").mockReturnValue(input),+ spyOn(mediaRead, "videoTrackOf").mockResolvedValue(track),+ spyOn(raster, "createRasterCanvas").mockImplementation(() => surface() as unknown as raster.RasterSurface),+ spyOn(mediaRead, "frameSink").mockReturnValue({+ getCanvas: async (t) => shot(t),+ async *canvases(start = 0, end = Infinity) {+ for (let f = Math.floor(start * FPS + 1e-6); f / FPS < end; f++) yield shot(f / FPS);+ },+ async *canvasesAtTimestamps(times) {+ for await (const t of times) yield shot(t);+ },+ }),+ ];+ restores.push(...spies.map((spy) => () => spy.mockRestore()));+ const result = await sampleWatchFrames("fixture.mp4", { from: 0, to: 3, interval: 1, metadataOnly: true });+ expect(result.sceneChanges).toEqual([1.017]);+});