Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): close decoded frames when media scans stop [rebuild]

David Lefb169aa44417
diff --git a/site/src/cut/lib/headless/nodeMedia.test.ts b/site/src/cut/lib/headless/nodeMedia.test.ts
new file mode 100644
index 00000000..be679fbf
--- /dev/null
+++ b/site/src/cut/lib/headless/nodeMedia.test.ts
@@ -0,0 +1,17 @@
+import { expect, spyOn, test } from "bun:test";
+import { VideoSampleSink, type VideoSample } from "mediabunny";
+import { NodeFrameSink } from "./nodeMedia";
+
+test("a headless transform failure closes its source sample", async () => {
+ const failure = new Error("Transform failed");
+ let closed = 0;
+ const sample = { transform: async () => { throw failure; }, close: () => { closed++; } } as unknown as VideoSample;
+ const getSample = spyOn(VideoSampleSink.prototype, "getSample").mockResolvedValue(sample);
+ // Only the sink's sample source is replaced; getCanvas runs the real ownership path.
+ const sink = Object.create(NodeFrameSink.prototype) as NodeFrameSink;
+ Object.assign(sink, { samples: Object.create(VideoSampleSink.prototype) });
+ try {
+ await expect(sink.getCanvas(0)).rejects.toBe(failure);
+ expect(closed).toBe(1);
+ } finally { getSample.mockRestore(); }
+});
diff --git a/site/src/cut/lib/headless/nodeMedia.ts b/site/src/cut/lib/headless/nodeMedia.ts
index 0f14282c..4da20da8 100644
--- a/site/src/cut/lib/headless/nodeMedia.ts
+++ b/site/src/cut/lib/headless/nodeMedia.ts
@@ -24,7 +24,7 @@ import { createRasterCanvas } from "../raster";
*/
/** A sink over one video track that hands back server canvases. */
-class NodeFrameSink implements FrameCanvasSink {
+export class NodeFrameSink implements FrameCanvasSink {
private readonly samples: VideoSampleSink;
constructor(
@@ -39,12 +39,16 @@ class NodeFrameSink implements FrameCanvasSink {
const { width, height, fit } = this.size ?? {};
// Always a transform, even at native size: it is what bakes the
// container's rotation into the pixels.
- const out = await sample.transform({
- ...(width !== undefined ? { width: Math.round(width) } : {}),
- ...(height !== undefined ? { height: Math.round(height) } : {}),
- ...(width !== undefined && height !== undefined ? { fit: fit ?? "fill" } : fit ? { fit } : {}),
- });
- sample.close();
+ let out: VideoSample;
+ try {
+ out = await sample.transform({
+ ...(width !== undefined ? { width: Math.round(width) } : {}),
+ ...(height !== undefined ? { height: Math.round(height) } : {}),
+ ...(width !== undefined && height !== undefined ? { fit: fit ?? "fill" } : fit ? { fit } : {}),
+ });
+ } finally {
+ sample.close();
+ }
try {
const pixels = new Uint8Array(out.allocationSize({ format: "RGBA" }));
await out.copyTo(pixels, { format: "RGBA" });
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 417b01fc..7edcbd67 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -1430,12 +1430,10 @@ export async function sampleWatchFrames(
const deadline = Date.now() + (opts.budgetMs ?? Infinity);
const outOfTime = () => Date.now() > deadline;
const sigG16At = async (t: number): Promise<Float32Array | null> => {
- const one = await frameSink(track, { width: cw, height: ch, fit: "fill" })
- .canvasesAtTimestamps([t])
- .next();
- if (one.done || !one.value) return null;
+ const one = await frameSink(track, { width: cw, height: ch, fit: "fill" }).getCanvas(t);
+ if (!one) return null;
sigCtx.imageSmoothingQuality = "high";
- sigCtx.drawImage(one.value.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ sigCtx.drawImage(one.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
const data = sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data;
return frameSig({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 4, data }).g16;
};
@@ -1446,37 +1444,41 @@ export async function sampleWatchFrames(
);
let paused = false;
let capped = false;
- for (let i = 0; i < times.length; i++) {
- if (opts.shouldPause?.() || outOfTime()) {
- paused = true;
- break;
- }
- // The keep-cap ends the pass — everything reported stays everything
- // seen, and the caller resumes from coveredTo.
- if (keptCount >= maxFrames) {
- capped = true;
- break;
- }
- const next = await cells.next();
- if (next.done || !next.value) continue; // a moment the decoder had no frame for
- if (!opts.metadataOnly) {
- const copy = createRasterCanvas(cw, ch);
- const cctx = copy.getContext("2d") as CanvasRenderingContext2D | null;
- if (!cctx) throw new Error("Could not sample the video.");
- cctx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, cw, ch);
- held.set(candTimes.length, copy);
+ try {
+ for (let i = 0; i < times.length; i++) {
+ if (opts.shouldPause?.() || outOfTime()) {
+ paused = true;
+ break;
+ }
+ // The keep-cap ends the pass — everything reported stays everything
+ // seen, and the caller resumes from coveredTo.
+ if (keptCount >= maxFrames) {
+ capped = true;
+ break;
+ }
+ const next = await cells.next();
+ if (next.done || !next.value) continue; // a moment the decoder had no frame for
+ if (!opts.metadataOnly) {
+ const copy = createRasterCanvas(cw, ch);
+ const cctx = copy.getContext("2d") as CanvasRenderingContext2D | null;
+ if (!cctx) throw new Error("Could not sample the video.");
+ cctx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, cw, ch);
+ held.set(candTimes.length, copy);
+ }
+ candTimes.push(times[i]);
+ sigCtx.imageSmoothingQuality = "high";
+ sigCtx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ const rgb = {
+ width: SIGNATURE_SIZE,
+ height: SIGNATURE_SIZE,
+ channels: 4 as const,
+ data: sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,
+ };
+ candG16.push(frameSig(rgb).g16);
+ selector.push(rgb);
}
- candTimes.push(times[i]);
- sigCtx.imageSmoothingQuality = "high";
- sigCtx.drawImage(next.value.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
- const rgb = {
- width: SIGNATURE_SIZE,
- height: SIGNATURE_SIZE,
- channels: 4 as const,
- data: sigCtx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,
- };
- candG16.push(frameSig(rgb).g16);
- selector.push(rgb);
+ } finally {
+ await cells.return();
}
const selection = selector.finish();
// Nothing decoded and nothing interrupted the pass: the source is
diff --git a/site/src/cut/lib/sampleWatchFrames.test.ts b/site/src/cut/lib/sampleWatchFrames.test.ts
new file mode 100644
index 00000000..59f0fb54
--- /dev/null
+++ b/site/src/cut/lib/sampleWatchFrames.test.ts
@@ -0,0 +1,58 @@
+import { afterEach, expect, spyOn, test } from "bun:test";
+import type { Input, InputVideoTrack, WrappedCanvas } from "mediabunny";
+import { sampleWatchFrames } from "./media";
+import * as mediaRead from "./mediaRead";
+import * as raster from "./raster";
+
+const restores: (() => void)[] = [];
+afterEach(() => { for (const restore of restores.splice(0)) restore(); });
+
+function fixture(failPixels = false) {
+ let closed = 0;
+ let disposed = false;
+ const context = {
+ drawImage() {},
+ getImageData(_x: number, _y: number, width: number, height: number) {
+ if (failPixels) throw new Error("Frame read failed");
+ return { data: new Uint8ClampedArray(width * height * 4) };
+ },
+ };
+ const canvas = { getContext: () => context } as unknown as raster.RasterSurface;
+ const frame = { canvas, timestamp: 0, duration: 0.5 } as WrappedCanvas;
+ const input = { dispose() { disposed = true; } } as unknown as Input;
+ const track = { getDisplayWidth: async () => 320, getDisplayHeight: async () => 180 } as unknown as InputVideoTrack;
+ const spies = [
+ spyOn(mediaRead, "openMedia").mockReturnValue(input),
+ spyOn(mediaRead, "videoTrackOf").mockResolvedValue(track),
+ spyOn(raster, "createRasterCanvas").mockReturnValue(canvas),
+ spyOn(mediaRead, "frameSink").mockReturnValue({
+ getCanvas: async () => frame,
+ async *canvases() { yield frame; },
+ async *canvasesAtTimestamps(times) {
+ try { for await (const time of times) yield { ...frame, timestamp: time }; }
+ finally { closed++; }
+ },
+ }),
+ ];
+ restores.push(...spies.map((spy) => () => spy.mockRestore()));
+ return { closed: () => closed, disposed: () => disposed };
+}
+
+test("pausing a scene scan closes its decoded-frame iterator", async () => {
+ const run = fixture();
+ let checks = 0;
+ const result = await sampleWatchFrames("fixture.mp4", {
+ from: 0, to: 5, interval: 0.5, metadataOnly: true, shouldPause: () => ++checks > 1,
+ });
+ expect(result.truncated).toBe(true);
+ expect(result.candidates).toBe(1);
+ expect(run.closed()).toBe(1);
+ expect(run.disposed()).toBe(true);
+});
+
+test("a failed frame read closes the iterator and its input", async () => {
+ const run = fixture(true);
+ await expect(sampleWatchFrames("fixture.mp4", { from: 0, to: 5, metadataOnly: true })).rejects.toThrow("Frame read failed");
+ expect(run.closed()).toBe(1);
+ expect(run.disposed()).toBe(true);
+});