Open source CapCut alternative.
TypeScript
fix(cut): close decoded frames when media scans stop [rebuild]
David Lefb169aa44417
diff --git a/site/src/cut/lib/headless/nodeMedia.test.ts b/site/src/cut/lib/headless/nodeMedia.test.tsnew file mode 100644index 00000000..be679fbf--- /dev/null+++ b/site/src/cut/lib/headless/nodeMedia.test.ts@@ -0,0 +1,17 @@+import { expect, spyOn, test } from "bun:test";+import { VideoSampleSink, type VideoSample } from "mediabunny";+import { NodeFrameSink } from "./nodeMedia";++test("a headless transform failure closes its source sample", async () => {+ const failure = new Error("Transform failed");+ let closed = 0;+ const sample = { transform: async () => { throw failure; }, close: () => { closed++; } } as unknown as VideoSample;+ const getSample = spyOn(VideoSampleSink.prototype, "getSample").mockResolvedValue(sample);+ // Only the sink's sample source is replaced; getCanvas runs the real ownership path.+ const sink = Object.create(NodeFrameSink.prototype) as NodeFrameSink;+ Object.assign(sink, { samples: Object.create(VideoSampleSink.prototype) });+ try {+ await expect(sink.getCanvas(0)).rejects.toBe(failure);+ expect(closed).toBe(1);+ } finally { getSample.mockRestore(); }+});diff --git a/site/src/cut/lib/headless/nodeMedia.ts b/site/src/cut/lib/headless/nodeMedia.tsindex 0f14282c..4da20da8 100644--- a/site/src/cut/lib/headless/nodeMedia.ts+++ b/site/src/cut/lib/headless/nodeMedia.ts@@ -24,7 +24,7 @@ import { createRasterCanvas } from "../raster";*//** A sink over one video track that hands back server canvases. */-class NodeFrameSink implements FrameCanvasSink {+export class NodeFrameSink implements FrameCanvasSink {private readonly samples: VideoSampleSink;constructor(@@ -39,12 +39,16 @@ class NodeFrameSink implements FrameCanvasSink {const { width, height, fit } = this.size ?? {};// Always a transform, even at native size: it is what bakes the// container's rotation into the pixels.- const out = await sample.transform({- ...(width !== undefined ? { width: Math.round(width) } : {}),- ...(height !== undefined ? { height: Math.round(height) } : {}),- ...(width !== undefined && height !== undefined ? { fit: fit ?? "fill" } : fit ? { fit } : {}),- });- sample.close();+ let out: VideoSample;+ try {+ out = await sample.transform({+ ...(width !== undefined ? { width: Math.round(width) } : {}),+ ...(height !== undefined ? { height: Math.round(height) } : {}),+ ...(width !== undefined && height !== undefined ? { fit: fit ?? "fill" } : fit ? { fit } : {}),+ });+ } finally {+ sample.close();+ }try {const pixels = new Uint8Array(out.allocationSize({ format: "RGBA" }));await out.copyTo(pixels, { format: "RGBA" });diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 417b01fc..7edcbd67 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1430,12 +1430,10 @@ export async function sampleWatchFrames(const deadline = Date.now() + (opts.budgetMs ?? Infinity);const outOfTime = () => Date.now() > deadline;const sigG16At = async (t: number): Promise<Float32Array | null> => {- const one = await frameSink(track, { width: cw, height: ch, fit: "fill" })- .canvasesAtTimestamps([t])- .next();- if (one.done || !one.value) return null;+ const one = await frameSink(track, { width: cw, height: ch, fit: "fill" }).getCanvas(t);+ if (!one) return null;sigCtx.imageSmoothingQuality = "high";- sigCtx.drawImage(one.value.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);+ sigCtx.drawImage(one.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);const data = sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data;return frameSig({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 4, data }).g16;};@@ -1446,37 +1444,41 @@ export async function sampleWatchFrames();let paused = false;let capped = false;- for (let i = 0; i < times.length; i++) {- if (opts.shouldPause?.() || outOfTime()) {- paused = true;- break;- }- // The keep-cap ends the pass — everything reported stays everything- // seen, and the caller resumes from coveredTo.- if (keptCount >= maxFrames) {- capped = true;- break;- }- const next = await cells.next();- if (next.done || !next.value) continue; // a moment the decoder had no frame for- if (!opts.metadataOnly) {- const copy = createRasterCanvas(cw, ch);- const cctx = copy.getContext("2d") as CanvasRenderingContext2D | null;- if (!cctx) throw new Error("Could not sample the video.");- cctx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, cw, ch);- held.set(candTimes.length, copy);+ try {+ for (let i = 0; i < times.length; i++) {+ if (opts.shouldPause?.() || outOfTime()) {+ paused = true;+ break;+ }+ // The keep-cap ends the pass — everything reported stays everything+ // seen, and the caller resumes from coveredTo.+ if (keptCount >= maxFrames) {+ capped = true;+ break;+ }+ const next = await cells.next();+ if (next.done || !next.value) continue; // a moment the decoder had no frame for+ if (!opts.metadataOnly) {+ const copy = createRasterCanvas(cw, ch);+ const cctx = copy.getContext("2d") as CanvasRenderingContext2D | null;+ if (!cctx) throw new Error("Could not sample the video.");+ cctx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, cw, ch);+ held.set(candTimes.length, copy);+ }+ candTimes.push(times[i]);+ sigCtx.imageSmoothingQuality = "high";+ sigCtx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);+ const rgb = {+ width: SIGNATURE_SIZE,+ height: SIGNATURE_SIZE,+ channels: 4 as const,+ data: sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,+ };+ candG16.push(frameSig(rgb).g16);+ selector.push(rgb);}- candTimes.push(times[i]);- sigCtx.imageSmoothingQuality = "high";- sigCtx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);- const rgb = {- width: SIGNATURE_SIZE,- height: SIGNATURE_SIZE,- channels: 4 as const,- data: sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,- };- candG16.push(frameSig(rgb).g16);- selector.push(rgb);+ } finally {+ await cells.return();}const selection = selector.finish();// Nothing decoded and nothing interrupted the pass: the source isdiff --git a/site/src/cut/lib/sampleWatchFrames.test.ts b/site/src/cut/lib/sampleWatchFrames.test.tsnew file mode 100644index 00000000..59f0fb54--- /dev/null+++ b/site/src/cut/lib/sampleWatchFrames.test.ts@@ -0,0 +1,58 @@+import { afterEach, expect, spyOn, test } from "bun:test";+import type { Input, InputVideoTrack, WrappedCanvas } from "mediabunny";+import { sampleWatchFrames } from "./media";+import * as mediaRead from "./mediaRead";+import * as raster from "./raster";++const restores: (() => void)[] = [];+afterEach(() => { for (const restore of restores.splice(0)) restore(); });++function fixture(failPixels = false) {+ let closed = 0;+ let disposed = false;+ const context = {+ drawImage() {},+ getImageData(_x: number, _y: number, width: number, height: number) {+ if (failPixels) throw new Error("Frame read failed");+ return { data: new Uint8ClampedArray(width * height * 4) };+ },+ };+ const canvas = { getContext: () => context } as unknown as raster.RasterSurface;+ const frame = { canvas, timestamp: 0, duration: 0.5 } as WrappedCanvas;+ const input = { dispose() { disposed = true; } } as unknown as Input;+ const track = { getDisplayWidth: async () => 320, getDisplayHeight: async () => 180 } as unknown as InputVideoTrack;+ const spies = [+ spyOn(mediaRead, "openMedia").mockReturnValue(input),+ spyOn(mediaRead, "videoTrackOf").mockResolvedValue(track),+ spyOn(raster, "createRasterCanvas").mockReturnValue(canvas),+ spyOn(mediaRead, "frameSink").mockReturnValue({+ getCanvas: async () => frame,+ async *canvases() { yield frame; },+ async *canvasesAtTimestamps(times) {+ try { for await (const time of times) yield { ...frame, timestamp: time }; }+ finally { closed++; }+ },+ }),+ ];+ restores.push(...spies.map((spy) => () => spy.mockRestore()));+ return { closed: () => closed, disposed: () => disposed };+}++test("pausing a scene scan closes its decoded-frame iterator", async () => {+ const run = fixture();+ let checks = 0;+ const result = await sampleWatchFrames("fixture.mp4", {+ from: 0, to: 5, interval: 0.5, metadataOnly: true, shouldPause: () => ++checks > 1,+ });+ expect(result.truncated).toBe(true);+ expect(result.candidates).toBe(1);+ expect(run.closed()).toBe(1);+ expect(run.disposed()).toBe(true);+});++test("a failed frame read closes the iterator and its input", async () => {+ const run = fixture(true);+ await expect(sampleWatchFrames("fixture.mp4", { from: 0, to: 5, metadataOnly: true })).rejects.toThrow("Frame read failed");+ expect(run.closed()).toBe(1);+ expect(run.disposed()).toBe(true);+});