Open source CapCut alternative.
TypeScript
fix(site): read a matte frame from its middle
David Lec06c3cecf12e
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex cef1ee6f..be4f0159 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -40,7 +40,7 @@ import {import { registerSourceColor, registerSourceSampler } from "@/cut/lib/previewCanvas";import { colorRead, previewFile, type ColorRead } from "@/cut/lib/sourceColor";import { backdropStill } from "@/cut/lib/backdropStills";-import { MatteAlpha } from "@/cut/lib/matteAlpha";+import { MatteAlpha, matteReadTime } from "@/cut/lib/matteAlpha";import { createRasterCanvas, type RasterSurface } from "@/cut/lib/raster";import { disposeDetailGpu } from "@/cut/lib/detailGpu";@@ -526,7 +526,7 @@ class Engine {const asset = s.assets.find((a) => a.id === m.assetId);if (!asset) return null;const dur = Math.max(0.001, asset.duration);- const mt = Math.max(0, Math.min(sourceTimeOf(clip, at) - m.in, dur - 0.001));+ const mt = matteReadTime(Math.max(0, sourceTimeOf(clip, at) - m.in), dur);// Coverage tolerates far less resolution than the picture, and the luma →// alpha read below is a per-frame pixel pass; capping the matte's decode// keeps that pass off the frame budget.diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex f3e5078c..f6114679 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -53,7 +53,7 @@ import { applyEffectToCanvas, effectRecipe, diveView, divesAt, evalOverlayFrame,import { backdropStill, loadBackdropStill } from "./backdropStills";import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";import { overlayLiveAt } from "./overlayLive";-import { MatteAlpha, matteStamp } from "./matteAlpha";+import { MatteAlpha, matteReadTime, matteStamp } from "./matteAlpha";import { createRasterCanvas, snapshotRaster, type RasterSurface } from "./raster";import { exportFrameSynth, SYNTH_EDGE, synthWeight, type FrameSynth } from "./frameSynth";import { cutRenderEnv, renderElementCanvas, renderStillPng, requireStickers } from "./textRender";@@ -1691,7 +1691,7 @@ export class FramePainter {// converted frame is reused until the read crosses into the next one.const stamp = matteStamp(m.assetId, mt);const held = alpha.held(stamp);- const frame = held ? null : await this.readerFor(asset).frameAt(mt);+ const frame = held ? null : await this.readerFor(asset).frameAt(matteReadTime(mt, dur));const img = held ?? (frame?.kind === "ready" ? alpha.of(frame, stamp) : null);if (img) this.matteFrames.set(key, img);}diff --git a/site/src/cut/lib/matteAlpha.test.ts b/site/src/cut/lib/matteAlpha.test.tsnew file mode 100644index 00000000..c9cfab3d--- /dev/null+++ b/site/src/cut/lib/matteAlpha.test.ts@@ -0,0 +1,24 @@+import { describe, expect, test } from "bun:test";+import { MATTE_FPS } from "@donkeycut/effects-kit";+import { matteReadTime, matteStamp } from "./matteAlpha";++describe("matte frame lookup", () => {+ // A 30 fps cut puts every other frame on a 15 fps matte frame's edge, and+ // the preview and the export reach that time through different arithmetic.+ const edge = 4 / MATTE_FPS;++ test("a time on a matte frame's edge reads that frame from either side of float error", () => {+ expect(matteStamp("m", edge - 1e-9)).toBe(matteStamp("m", edge + 1e-9));+ expect(matteReadTime(edge - 1e-9, 10)).toBeCloseTo(4.5 / MATTE_FPS, 9);+ expect(matteReadTime(edge + 1e-9, 10)).toBeCloseTo(4.5 / MATTE_FPS, 9);+ });++ test("a time inside a matte frame reads that frame's middle", () => {+ expect(matteReadTime(4.4 / MATTE_FPS, 10)).toBeCloseTo(4.5 / MATTE_FPS, 9);+ expect(matteReadTime(0, 10)).toBeCloseTo(0.5 / MATTE_FPS, 9);+ });++ test("the last frame's read stays inside the matte", () => {+ expect(matteReadTime(0.299, 0.3)).toBeLessThan(0.3);+ });+});diff --git a/site/src/cut/lib/matteAlpha.ts b/site/src/cut/lib/matteAlpha.tsindex 3b8d4cd5..3af75fbe 100644--- a/site/src/cut/lib/matteAlpha.ts+++ b/site/src/cut/lib/matteAlpha.ts@@ -21,10 +21,23 @@ interface MatteFrame {height: number;}+/** How far past a read's time its matte frame is found, in matte frames: a+ * 30 fps cut puts every other frame on a matte frame's edge, and the nudge+ * keeps float error in that time from tipping it into the frame before. */+const MATTE_NUDGE = 1e-3;++/** The baked matte frame source time `mt` falls in. */+const matteFrameOf = (mt: number) => Math.floor(mt * MATTE_FPS + MATTE_NUDGE);+/** Which baked matte frame source time `mt` falls in — equal stamps are the* same pixels. */-export const matteStamp = (assetId: string, mt: number): string =>- `${assetId}:${Math.floor(mt * MATTE_FPS)}`;+export const matteStamp = (assetId: string, mt: number): string => `${assetId}:${matteFrameOf(mt)}`;++/** Where to read the matte frame `mt` falls in: the middle of that frame, so+ * every reader lands on the same one whether it picks the frame at or before+ * the time or the nearest. Kept inside a matte `dur` seconds long. */+export const matteReadTime = (mt: number, dur: number): number =>+ Math.min((matteFrameOf(mt) + 0.5) / MATTE_FPS, Math.max(0, dur - 0.001));/** One matte's alpha picture, held on a single surface and converted only* when the frame's stamp moves. */@@ -73,7 +86,7 @@ export function matteStage(m: Matte | undefined, assets: MediaAsset[]) {const stamp = matteStamp(m.assetId, mt);ready = alpha.held(stamp);if (ready) return;- const f = await reader.frameAt(mt);+ const f = await reader.frameAt(matteReadTime(mt, dur));ready = f.kind === "ready" ? alpha.of(f, stamp) : null;},get: () => ready,