Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): land watch scene changes on their frame

David Leb571e9b3b9ed
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 31c4bf7c..520814bb 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -1302,6 +1302,7 @@ export async function captureFreezeFrame(
}
const round2 = (n: number) => Math.round(n * 100) / 100;
+const round3 = (n: number) => Math.round(n * 1000) / 1000;
/** Decoded audio for a span of a source, chunk by chunk, so scanning a long
* file costs one decoded chunk at a time rather than a decoded copy of the
@@ -1442,6 +1443,7 @@ const CANDIDATE_CAP = 150; // decodes per call — the densest sweep
const SELECT_CELL = 480; // what the selector decodes at when the frames are bigger
const REFINE_TARGET_S = 0.35; // localize each hard cut to about this window
const REFINE_MAX_PROBES = 48; // extra seeks the refinement pass may spend per call
+const CUT_WALK_EPS = 0.0005; // half a millisecond: lands a walk's ends on the frames they name
const AIMED_SHARE = 0.5; // most of the cap aimed moments may take from the floor
const MERGE_EPS = 0.15; // two candidate times this close are one decode
@@ -1570,8 +1572,9 @@ export async function makeStillFrame(
* moments `at` aims at, then let the shared selector keep the frames that
* actually differ. sceneChanges are
* hard-cut times: each "global" keep says a cut lies inside the grid step
- * before it, and a short bisection pass narrows that window to about
- * REFINE_TARGET_S — the reported time is where the new shot is first seen.
+ * before it, a short bisection pass narrows that window to about
+ * REFINE_TARGET_S, and an in-order walk of the window's frames reports the
+ * first frame of the new shot, to the millisecond.
* A shot shorter than the candidate step can still fall between samples; a
* narrow re-watch with a small interval is the recall pass.
*
@@ -1648,14 +1651,31 @@ export async function sampleWatchFrames(
if (!sigCtx) throw new Error("Could not sample the video.");
const deadline = Date.now() + (opts.budgetMs ?? Infinity);
const outOfTime = () => Date.now() > deadline;
- const sigG16At = async (t: number): Promise<Float32Array | null> => {
- const one = await baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).getCanvas(t);
- if (!one) return null;
+ const sigG16Of = (canvas: CanvasImageSource): Float32Array => {
sigCtx.imageSmoothingQuality = "high";
- sigCtx.drawImage(one.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ sigCtx.drawImage(canvas, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
const data = sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data;
return frameSig({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 4, data }).g16;
};
+ // The timestamp of the first frame in (lo, hi] that differs from `pre`,
+ // decoded in order; null when the walk finds none.
+ const firstNewFrame = async (lo: number, hi: number, pre: Float32Array): Promise<number | null> => {
+ const walk = baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).canvases(lo, hi + CUT_WALK_EPS);
+ try {
+ for await (const frame of walk) {
+ if (frame.timestamp <= lo - CUT_WALK_EPS) continue;
+ const g = sigG16Of(frame.canvas as CanvasImageSource);
+ if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) return frame.timestamp;
+ }
+ } finally {
+ await walk.return(undefined);
+ }
+ return null;
+ };
+ const sigG16At = async (t: number): Promise<Float32Array | null> => {
+ const one = await baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).getCanvas(t);
+ return one ? sigG16Of(one.canvas as CanvasImageSource) : null;
+ };
// The times are ascending, so the whole span decodes once, in order.
const cells = baseFrameSink(track, { width: cw, height: ch, fit: "fill" }, reading).canvasesAtTimestamps(times);
@@ -1761,9 +1781,8 @@ export async function sampleWatchFrames(
encoded.clear();
// A "global" keep only says the cut lies in the grid step before it.
- // Bisect that step with a few extra decodes so each reported cut time
- // sits within REFINE_TARGET_S of the first new-shot frame; the kept
- // image stays the grid sample.
+ // Bisect that step with a few extra decodes down to REFINE_TARGET_S, then
+ // walk what is left frame by frame; the kept image stays the grid sample.
let probes = 0;
const sceneChanges: number[] = [];
for (const idx of keptIdx) {
@@ -1785,7 +1804,14 @@ export async function sampleWatchFrames(
if (diffCellPct(g, pre, DEDUP_TUNING.GLOBAL_TOL) > DEDUP_TUNING.THRESHOLD) hi = mid;
else lo = mid;
}
- sceneChanges.push(round2(hi));
+ // The window is a few frames wide now: walk it in order and report the
+ // first frame of the new shot itself, so a cut lands on its frame.
+ const first =
+ pre !== null && probes < REFINE_MAX_PROBES && !outOfTime() && !opts.shouldPause?.()
+ ? await firstNewFrame(lo, hi, pre)
+ : null;
+ if (first !== null) probes++;
+ sceneChanges.push(round3(first ?? hi));
}
// An interrupted or capped scan only covered what it decoded and kept;
diff --git a/site/src/cut/lib/sampleWatchFrames.test.ts b/site/src/cut/lib/sampleWatchFrames.test.ts
index e1297ed2..a5af9871 100644
--- a/site/src/cut/lib/sampleWatchFrames.test.ts
+++ b/site/src/cut/lib/sampleWatchFrames.test.ts
@@ -116,3 +116,49 @@ test("the top of the ladder is bounded, and only it walks twice", () => {
expect(watchGeometry("scan").rewalk).toBe(false);
expect(watchGeometry("read").rewalk).toBe(false);
});
+
+test("a hard cut is reported on the first frame of the new shot, to the millisecond", async () => {
+ // A 60 fps source whose picture turns from black to white at frame 61
+ // (1.0167s). The sweep only says the cut sits between two samples a second
+ // apart; the reported time has to be the frame itself.
+ const FPS = 60;
+ const CUT_FRAME = 61;
+ const shade = (t: number) => (Math.floor(t * FPS + 1e-6) >= CUT_FRAME ? 255 : 0);
+ type Painted = { getContext: () => unknown; shade: number };
+ const surface = (): Painted => {
+ const own: Painted = {
+ shade: 0,
+ getContext: () => ({
+ drawImage(src: Painted) { own.shade = src.shade; },
+ getImageData(_x: number, _y: number, width: number, height: number) {
+ return { data: new Uint8ClampedArray(width * height * 4).fill(own.shade) };
+ },
+ }),
+ };
+ return own;
+ };
+ const shot = (t: number) => {
+ const canvas = surface();
+ canvas.shade = shade(t);
+ return { canvas, timestamp: Math.floor(t * FPS + 1e-6) / FPS, duration: 1 / FPS } as unknown as WrappedCanvas;
+ };
+ const input = { dispose() {} } as unknown as Input;
+ const track = { getDisplayWidth: async () => 320, getDisplayHeight: async () => 180 } as unknown as InputVideoTrack;
+ const spies = [
+ spyOn(mediaRead, "openMedia").mockReturnValue(input),
+ spyOn(mediaRead, "videoTrackOf").mockResolvedValue(track),
+ spyOn(raster, "createRasterCanvas").mockImplementation(() => surface() as unknown as raster.RasterSurface),
+ spyOn(mediaRead, "frameSink").mockReturnValue({
+ getCanvas: async (t) => shot(t),
+ async *canvases(start = 0, end = Infinity) {
+ for (let f = Math.floor(start * FPS + 1e-6); f / FPS < end; f++) yield shot(f / FPS);
+ },
+ async *canvasesAtTimestamps(times) {
+ for await (const t of times) yield shot(t);
+ },
+ }),
+ ];
+ restores.push(...spies.map((spy) => () => spy.mockRestore()));
+ const result = await sampleWatchFrames("fixture.mp4", { from: 0, to: 3, interval: 1, metadataOnly: true });
+ expect(result.sceneChanges).toEqual([1.017]);
+});