Open source CapCut alternative.
TypeScript
fix(site): end each element on its last frame [rebuild]
David Lef385702b5fac
diff --git a/site/src/cut/components/OverlayLayer.tsx b/site/src/cut/components/OverlayLayer.tsxindex 3f33dddb..e8b47c70 100644--- a/site/src/cut/components/OverlayLayer.tsx+++ b/site/src/cut/components/OverlayLayer.tsx@@ -8,7 +8,8 @@ import { startDrag } from "@/cut/lib/drag";import { startSelectionDrag, togglePreviewSelection } from "@/cut/components/previewSelectionDrag";import { pictureGroupSelected } from "@/cut/lib/previewSelection";import { useSkim, usePreviewTime } from "@/cut/lib/playhead";-import { useEditor } from "@/cut/lib/store";+import { projectDuration, useEditor } from "@/cut/lib/store";+import { overlayLiveAt, stageTimeAt } from "@/cut/lib/overlayLive";import {captionStyle,cueAt,@@ -212,6 +213,10 @@ export function OverlayLayer({// Titles preview under the skimmer too (paused only), matching the canvas.const t = usePreviewTime();const skimTime = useSkim();+ const total = useEditor(projectDuration);+ // The moment the elements are on screen for: the canvas's own, which holds+ // the last frame with the playhead on the cut's end.+ const at = stageTimeAt(t, total);const live = useLiveEffects();// The grade and frame motion of every effect above this slice.const graded = useMemo(@@ -350,7 +355,7 @@ export function OverlayLayer({// handles or drag. The timeline bar is the thing you grab.if (o.kind === "effect") return null;const selected = selectedIds.has(o.id) || sel?.id === o.id;- const inRange = t >= o.start && t <= o.end;+ const inRange = overlayLiveAt(o, at);// While hover-scrubbing (paused, skimmer active) the preview must show the// exact frame under the skimmer — a selected but out-of-frame title can't// leak into a frame it isn't part of. Off the skimmer, a selected title@@ -411,7 +416,7 @@ export function OverlayLayer({style={{ filter, opacity: ghostAlpha }}>{overlays.map((o) => {- if (o.hidden || o.kind === "effect" || t < o.start || t > o.end) return null;+ if (o.hidden || o.kind === "effect" || !overlayLiveAt(o, at)) return null;return (<OverlayItemkey={o.id}@@ -582,9 +587,12 @@ function SubtitleCaption({const multiSelection = useEditor((s) => s.multiSelection);const stagePress = useContext(StagePress);const t = usePreviewTime();+ const total = useEditor(projectDuration);+ // The cue on screen, with the last one held when the playhead sits on the+ // cut's end, the way the canvas holds the last frame.const cues = laneCues(subtitles, lane);- const cue = cueAt(cues, t);+ const cue = cueAt(cues, stageTimeAt(t, total));if (!cue || !cue.text.trim()) return null;// The caption moves its whole track, so it takes the same rule the elements// take: the cue on screen has to be the selection before a drag moves it.diff --git a/site/src/cut/components/StageEffects.tsx b/site/src/cut/components/StageEffects.tsxindex e22f1266..1e4c042b 100644--- a/site/src/cut/components/StageEffects.tsx+++ b/site/src/cut/components/StageEffects.tsx@@ -12,7 +12,8 @@ import {washBackground,} from "@donkeycut/effects-kit";import { previewAt, subscribePlayhead, usePreviewTime } from "@/cut/lib/playhead";-import { useEditor } from "@/cut/lib/store";+import { projectDuration, useEditor } from "@/cut/lib/store";+import { stageTimeAt } from "@/cut/lib/overlayLive";import { isEffectOverlay, laneOf } from "@/cut/lib/types";import {hasEffects,@@ -52,7 +53,11 @@ export { stageEffectTransform, stageSliceStructure, type LiveEffect, type StageSexport function useLiveEffects(): LiveEffect[] {const overlays = useEditor((s) => s.overlays);const t = usePreviewTime();- return useMemo(() => liveEffectsAt(overlays, t), [overlays, t]);+ const total = useEditor(projectDuration);++ // Graded at the canvas's moment, which holds the last frame on the cut's end.+ const at = stageTimeAt(t, total);+ return useMemo(() => liveEffectsAt(overlays, at), [overlays, at]);}/**@@ -105,7 +110,9 @@ export function StagePictureFx({ children }: { children: React.ReactNode }) {return;}const apply = () => {- const states = liveEffectsAt(overlays, previewAt()).map((e) => e.state);+ // The canvas's moment: the last frame's, with the playhead on the end.+ const at = stageTimeAt(previewAt(), projectDuration(useEditor.getState()));+ const states = liveEffectsAt(overlays, at).map((e) => e.state);const box = ref.current;const split = box ? rgbSplitFilter(states, box.clientWidth, box.clientHeight) : "";write([stageEffectFilter(states), split].filter(Boolean).join(" "), stageEffectTransform(states) ?? "");diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 7a4d49d1..858e2648 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -16,6 +16,7 @@ import { assetIsSilent, CLIP_MIN_SECONDS, clipCovers, rectOf } from "@/cut/lib/timport type { AudioClip, ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";import { OWN_SOUND, soundSourceOf, soundWindow } from "@/cut/lib/soundSource";import { SubjectMaskCompositor } from "@/cut/lib/behindPass";+import { stageTimeAt } from "@/cut/lib/overlayLive";import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";import { hdrCanvasSupport } from "@/cut/lib/hdrCanvas";import { displayIsHdr, HdrPresenter, onDisplayRangeChange, setHdrPreviewState } from "@/cut/lib/hdrPresent";@@ -953,7 +954,6 @@ class Engine {// background. Playing, that was the last frame before the stop, a single// dark frame the paused redraw painted over; paused, it is a skim off the// right edge of the cut. Both hold the last frame.- const end = Math.max(0, total - 0.001);let t: number;if (playing) {const head = playheadAt();@@ -961,9 +961,9 @@ class Engine {this.mixer.start(Math.min(head, total));this.written = head;}- t = Math.max(0, Math.min(this.mixer.now(), end));+ t = stageTimeAt(this.mixer.now(), total);} else {- t = Math.max(0, Math.min(previewAt(), end));+ t = stageTimeAt(previewAt(), total);if (this.mixer.running) {this.mixer.stop();flushMeter();diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.tsindex ce801111..a6c61691 100644--- a/site/src/cut/lib/behindPass.ts+++ b/site/src/cut/lib/behindPass.ts@@ -37,6 +37,7 @@ import {import { ElementFx, elementLook } from "@donkeycut/effects-kit";import { personSegmenter, segmentSubjectAlpha } from "./cutout";import { allowance, holdMemory } from "./memoryBudget";+import { overlayLiveAt } from "./overlayLive";import { createRasterCanvas, snapshotRaster } from "./raster";import { cutRenderEnv } from "./textRender";import {@@ -160,7 +161,7 @@ function drawable(o: Overlay): boolean {/** The behind-the-speaker elements live at `t` (any drawable kind). */export function behindOverlaysAt(overlays: Overlay[], t: number): Overlay[] {return overlays.filter(- (o) => behindSubjectOverlay(o) && drawable(o) && t >= o.start && t <= o.end+ (o) => behindSubjectOverlay(o) && drawable(o) && overlayLiveAt(o, t));}@@ -486,7 +487,7 @@ export class SubjectMaskCompositor {* before the frame is, so no frame of a render shows a stale one. */async ready(overlays: Overlay[], w: number, h: number, assets: MediaAsset[], t: number): Promise<void> {for (const o of overlays) {- if (!behindSubjectOverlay(o) || !drawable(o) || t < o.start || t > o.end) continue;+ if (!behindSubjectOverlay(o) || !drawable(o) || !overlayLiveAt(o, t)) continue;const tLocal = Math.max(0, t - o.start);const look = lookAt(o, tLocal, evalOverlayFrame(o, tLocal, w / h));if (this.entryFor(o).byWord.has(look.key)) continue;@@ -699,7 +700,7 @@ export class SubjectMaskCompositor {this.sweepRasters();const active = behindOverlaysAt(overlays, t);const wantsMatte = overlays.some(- (o) => frontSubjectOverlay(o) && drawable(o) && t >= o.start && t <= o.end+ (o) => frontSubjectOverlay(o) && drawable(o) && overlayLiveAt(o, t));if (active.length === 0 && !wantsMatte) return;const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;diff --git a/site/src/cut/lib/effectStack.ts b/site/src/cut/lib/effectStack.tsindex 331af727..518519bf 100644--- a/site/src/cut/lib/effectStack.ts+++ b/site/src/cut/lib/effectStack.ts@@ -18,6 +18,7 @@ import {washBackground,type EffectPreviewState,} from "@donkeycut/effects-kit";+import { overlayLiveAt } from "./overlayLive";import { isEffectOverlay, laneOf, type Overlay } from "./types";/** One effect live at a moment, with the lane that places it in the stack. */@@ -31,7 +32,7 @@ export function liveEffectsAt(overlays: Overlay[], t: number): LiveEffect[] {const live: LiveEffect[] = [];for (const o of overlays) {if (!isEffectOverlay(o) || isAudioEffect(o.effect)) continue;- if (o.hidden || t < o.start || t > o.end) continue;+ if (o.hidden || !overlayLiveAt(o, t)) continue;live.push({lane: laneOf(o),state: effectPreviewState(effectRecipe(o), o.amount, t - o.start, o.focus, o.ramp, o.end - o.start, o),diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex b03d88c9..f3e5078c 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -52,6 +52,7 @@ import { ElementFx, elementLook, paintElementCrop } from "@donkeycut/effects-kitimport { applyEffectToCanvas, effectRecipe, diveView, divesAt, evalOverlayFrame, measureDiveFocus, retimeOf, grainTile, smoothsAt, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, planAnimatedLayers, type DiveFocus, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";import { backdropStill, loadBackdropStill } from "./backdropStills";import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";+import { overlayLiveAt } from "./overlayLive";import { MatteAlpha, matteStamp } from "./matteAlpha";import { createRasterCanvas, snapshotRaster, type RasterSurface } from "./raster";import { exportFrameSynth, SYNTH_EDGE, synthWeight, type FrameSynth } from "./frameSynth";@@ -603,7 +604,7 @@ function liveEffectsAt(overlays: Overlay[], t: number): EffectOverlay[] {return overlays.filter(isEffectOverlay).filter((o) => !isAudioEffect(o.effect))- .filter((o) => !o.hidden && t >= o.start && t < o.end)+ .filter((o) => !o.hidden && overlayLiveAt(o, t)).sort((a, b) => laneOf(b) - laneOf(a));}@@ -1324,7 +1325,7 @@ async function drawStamps(| null;if (!ctx) return;for (const layer of layers) {- if (t < layer.start || t >= layer.end) continue;+ if (!overlayLiveAt(layer, t)) continue;if (layer.lottie) {// Nothing to cache: the animation's own canvas is sought per frame and// drawn at the sticker's rect under the same delta transform.diff --git a/site/src/cut/lib/overlayLive.test.ts b/site/src/cut/lib/overlayLive.test.tsnew file mode 100644index 00000000..189697cb--- /dev/null+++ b/site/src/cut/lib/overlayLive.test.ts@@ -0,0 +1,114 @@+import { beforeAll, describe, expect, test } from "bun:test";+import type { ShapeOverlay } from "@donkeycut/effects-kit";+import { behindOverlaysAt, SubjectMaskCompositor } from "./behindPass";+import { liveEffectsAt } from "./effectStack";+import { renderProjectFrame } from "./exportRender";+import { installSkiaRaster } from "./headless/skiaRaster";+import { overlayLiveAt, stageTimeAt } from "./overlayLive";+import type { ExportDoc } from "./renderSnapshot";+import type { Overlay, TextOverlay } from "./types";++// A shot's elements end on the frame the next shot starts. That frame belongs+// to the incoming shot: frame 508 at 30 fps (t = 16.9333) shows the next+// shot's elements alone, in the preview and in the export.++const FPS = 30;+const CUT = 508 / FPS;+const size = { width: 320, height: 180 };++beforeAll(async () => {+ expect(await installSkiaRaster()).toBe(true);+});++const rect = (id: string, start: number, end: number, fill: string): ShapeOverlay => ({+ id, kind: "shape", shape: "rect", start, end,+ x: 0.5, y: 0.5, w: 0.5, h: 0.5, fill,+});++const flash = (id: string, start: number, end: number) =>+ ({ id, kind: "effect", effect: "lightleak", leak: "film", amount: 1, start, end, x: 0.5, y: 0.5, lane: 0 }) as Overlay;++const behind = (id: string, start: number, end: number): TextOverlay => ({+ id, text: "SOON", start, end,+ x: 0.5, y: 0.5, size: 160, font: "sf", weight: 900, color: "#ffffff",+ plate: false, shadow: false,+ mask: { kind: "subject", invert: true } as TextOverlay["mask"],+});++const docOf = (overlays: Overlay[]): ExportDoc => ({+ aspect: "16:9", assets: [], clips: [], audioClips: [], overlays,+ subtitles: { cues: [], showOnVideo: false, showOnTimeline: false },+});++/** Pixels where one channel is lit and the other two are dark. */+function inkOf(data: Uint8ClampedArray, channel: number): number {+ let n = 0;+ for (let i = 0; i < data.length; i += 4) {+ const lit = data[i + channel] > 128;+ const others = [0, 1, 2].filter((c) => c !== channel).every((c) => data[i + c] < 64);+ if (lit && others) {+ n++;+ }+ }+ return n;+}++describe("the frame on a cut", () => {+ const outgoing = rect("out", 15, CUT, "#ff0000");+ const incoming = rect("in", CUT, 18.5, "#0000ff");++ test("the preview's elements show the incoming shot's alone", () => {+ const live = [outgoing, incoming].filter((o) => overlayLiveAt(o, stageTimeAt(CUT, 20)));+ expect(live.map((o) => o.id)).toEqual(["in"]);+ });++ test("the preview's effects grade with the incoming shot's alone", () => {+ const live = liveEffectsAt([flash("a", 15, CUT), flash("b", CUT, 18.5)], CUT);+ expect(live.length).toBe(1);+ });++ test("the behind-the-speaker pass draws the incoming shot's alone", async () => {+ const overlays = [behind("out", 15, CUT), behind("in", CUT, 18.5)];+ expect(behindOverlaysAt(overlays, CUT).map((o) => o.id)).toEqual(["in"]);++ // The export prepares a picture for each element it is about to draw.+ const pass = new SubjectMaskCompositor();+ await pass.ready(overlays, 540, 960, [], CUT);+ const rasters = (pass as unknown as { rasters: Map<string, unknown> }).rasters;+ expect([...rasters.keys()]).toEqual(["in"]);+ pass.dispose();+ });++ test("the export draws the incoming shot's alone", async () => {+ const frame = await renderProjectFrame(docOf([outgoing, incoming]), CUT, size, () => "");+ const data = (frame.getContext("2d") as CanvasRenderingContext2D).getImageData(0, 0, size.width, size.height).data;+ expect(inkOf(data, 0)).toBe(0);+ expect(inkOf(data, 2)).toBeGreaterThan(1000);+ });++ test("a cut every frame shows one shot's element per frame", () => {+ const shots = Array.from({ length: 10 }, (_, i) => rect(`s${i}`, (500 + i) / FPS, (501 + i) / FPS, "#ffffff"));+ for (let i = 0; i < 10; i++) {+ const live = shots.filter((o) => overlayLiveAt(o, stageTimeAt((500 + i) / FPS, 20)));+ expect(live.map((o) => o.id)).toEqual([`s${i}`]);+ }+ });+});++// Playback stops with the playhead on the cut's end. The paused frame there+// is the cut's last frame, so what runs to the end still shows.+describe("paused at the cut's end", () => {+ const total = CUT;++ test("an element running to the end shows", () => {+ expect(overlayLiveAt(rect("last", 15, total, "#ff0000"), stageTimeAt(total, total))).toBe(true);+ });++ test("an effect running to the end grades", () => {+ expect(liveEffectsAt([flash("last", 15, total)], stageTimeAt(total, total)).length).toBe(1);+ });++ test("a moment inside the cut is its own", () => {+ expect(stageTimeAt(3, total)).toBe(3);+ });+});diff --git a/site/src/cut/lib/overlayLive.ts b/site/src/cut/lib/overlayLive.tsnew file mode 100644index 00000000..fd3268cf--- /dev/null+++ b/site/src/cut/lib/overlayLive.ts@@ -0,0 +1,27 @@+/**+ * When an element is on screen.+ *+ * An element draws from its start up to its end, end excluded: the frame on+ * a cut belongs to the shot that starts there. The preview, the export and+ * the behind-the-speaker pass all ask here, so they draw the same elements+ * on every frame.+ */++/** How far inside the cut's end the stage draws a moment on the end. */+const END_HOLD = 0.001;++/** Whether an element draws at `t`. One ending at 2 and one starting at 2+ * meet on that frame, and the frame shows the second alone. */+export function overlayLiveAt(o: { start: number; end: number }, t: number): boolean {+ return t >= o.start && t < o.end;+}++/**+ * The moment the stage draws for a playhead at `t`, in a cut `total` long.+ * Playback stops with the playhead on the end, a moment past the final shot,+ * so the stage draws the moment just inside it: paused there, the picture is+ * the cut's last frame, with every element that runs to the end.+ */+export function stageTimeAt(t: number, total: number): number {+ return Math.max(0, Math.min(t, total - END_HOLD));+}