Skip to content

Open source CapCut alternative.

TypeScript

fix(site): read a matte frame from its middle

David Lec06c3cecf12e
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index cef1ee6f..be4f0159 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -40,7 +40,7 @@ import {
import { registerSourceColor, registerSourceSampler } from "@/cut/lib/previewCanvas";
import { colorRead, previewFile, type ColorRead } from "@/cut/lib/sourceColor";
import { backdropStill } from "@/cut/lib/backdropStills";
-import { MatteAlpha } from "@/cut/lib/matteAlpha";
+import { MatteAlpha, matteReadTime } from "@/cut/lib/matteAlpha";
import { createRasterCanvas, type RasterSurface } from "@/cut/lib/raster";
import { disposeDetailGpu } from "@/cut/lib/detailGpu";
@@ -526,7 +526,7 @@ class Engine {
const asset = s.assets.find((a) => a.id === m.assetId);
if (!asset) return null;
const dur = Math.max(0.001, asset.duration);
- const mt = Math.max(0, Math.min(sourceTimeOf(clip, at) - m.in, dur - 0.001));
+ const mt = matteReadTime(Math.max(0, sourceTimeOf(clip, at) - m.in), dur);
// Coverage tolerates far less resolution than the picture, and the luma →
// alpha read below is a per-frame pixel pass; capping the matte's decode
// keeps that pass off the frame budget.
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index f3e5078c..f6114679 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -53,7 +53,7 @@ import { applyEffectToCanvas, effectRecipe, diveView, divesAt, evalOverlayFrame,
import { backdropStill, loadBackdropStill } from "./backdropStills";
import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";
import { overlayLiveAt } from "./overlayLive";
-import { MatteAlpha, matteStamp } from "./matteAlpha";
+import { MatteAlpha, matteReadTime, matteStamp } from "./matteAlpha";
import { createRasterCanvas, snapshotRaster, type RasterSurface } from "./raster";
import { exportFrameSynth, SYNTH_EDGE, synthWeight, type FrameSynth } from "./frameSynth";
import { cutRenderEnv, renderElementCanvas, renderStillPng, requireStickers } from "./textRender";
@@ -1691,7 +1691,7 @@ export class FramePainter {
// converted frame is reused until the read crosses into the next one.
const stamp = matteStamp(m.assetId, mt);
const held = alpha.held(stamp);
- const frame = held ? null : await this.readerFor(asset).frameAt(mt);
+ const frame = held ? null : await this.readerFor(asset).frameAt(matteReadTime(mt, dur));
const img = held ?? (frame?.kind === "ready" ? alpha.of(frame, stamp) : null);
if (img) this.matteFrames.set(key, img);
}
diff --git a/site/src/cut/lib/matteAlpha.test.ts b/site/src/cut/lib/matteAlpha.test.ts
new file mode 100644
index 00000000..c9cfab3d
--- /dev/null
+++ b/site/src/cut/lib/matteAlpha.test.ts
@@ -0,0 +1,24 @@
+import { describe, expect, test } from "bun:test";
+import { MATTE_FPS } from "@donkeycut/effects-kit";
+import { matteReadTime, matteStamp } from "./matteAlpha";
+
+describe("matte frame lookup", () => {
+ // A 30 fps cut puts every other frame on a 15 fps matte frame's edge, and
+ // the preview and the export reach that time through different arithmetic.
+ const edge = 4 / MATTE_FPS;
+
+ test("a time on a matte frame's edge reads that frame from either side of float error", () => {
+ expect(matteStamp("m", edge - 1e-9)).toBe(matteStamp("m", edge + 1e-9));
+ expect(matteReadTime(edge - 1e-9, 10)).toBeCloseTo(4.5 / MATTE_FPS, 9);
+ expect(matteReadTime(edge + 1e-9, 10)).toBeCloseTo(4.5 / MATTE_FPS, 9);
+ });
+
+ test("a time inside a matte frame reads that frame's middle", () => {
+ expect(matteReadTime(4.4 / MATTE_FPS, 10)).toBeCloseTo(4.5 / MATTE_FPS, 9);
+ expect(matteReadTime(0, 10)).toBeCloseTo(0.5 / MATTE_FPS, 9);
+ });
+
+ test("the last frame's read stays inside the matte", () => {
+ expect(matteReadTime(0.299, 0.3)).toBeLessThan(0.3);
+ });
+});
diff --git a/site/src/cut/lib/matteAlpha.ts b/site/src/cut/lib/matteAlpha.ts
index 3b8d4cd5..3af75fbe 100644
--- a/site/src/cut/lib/matteAlpha.ts
+++ b/site/src/cut/lib/matteAlpha.ts
@@ -21,10 +21,23 @@ interface MatteFrame {
height: number;
}
+/** How far past a read's time its matte frame is found, in matte frames: a
+ * 30 fps cut puts every other frame on a matte frame's edge, and the nudge
+ * keeps float error in that time from tipping it into the frame before. */
+const MATTE_NUDGE = 1e-3;
+
+/** The baked matte frame source time `mt` falls in. */
+const matteFrameOf = (mt: number) => Math.floor(mt * MATTE_FPS + MATTE_NUDGE);
+
/** Which baked matte frame source time `mt` falls in — equal stamps are the
* same pixels. */
-export const matteStamp = (assetId: string, mt: number): string =>
- `${assetId}:${Math.floor(mt * MATTE_FPS)}`;
+export const matteStamp = (assetId: string, mt: number): string => `${assetId}:${matteFrameOf(mt)}`;
+
+/** Where to read the matte frame `mt` falls in: the middle of that frame, so
+ * every reader lands on the same one whether it picks the frame at or before
+ * the time or the nearest. Kept inside a matte `dur` seconds long. */
+export const matteReadTime = (mt: number, dur: number): number =>
+ Math.min((matteFrameOf(mt) + 0.5) / MATTE_FPS, Math.max(0, dur - 0.001));
/** One matte's alpha picture, held on a single surface and converted only
* when the frame's stamp moves. */
@@ -73,7 +86,7 @@ export function matteStage(m: Matte | undefined, assets: MediaAsset[]) {
const stamp = matteStamp(m.assetId, mt);
ready = alpha.held(stamp);
if (ready) return;
- const f = await reader.frameAt(mt);
+ const f = await reader.frameAt(matteReadTime(mt, dur));
ready = f.kind === "ready" ? alpha.of(f, stamp) : null;
},
get: () => ready,