Skip to content

Open source CapCut alternative.

TypeScript

fix(site): end each element on its last frame [rebuild]

David Lef385702b5fac
diff --git a/site/src/cut/components/OverlayLayer.tsx b/site/src/cut/components/OverlayLayer.tsx
index 3f33dddb..e8b47c70 100644
--- a/site/src/cut/components/OverlayLayer.tsx
+++ b/site/src/cut/components/OverlayLayer.tsx
@@ -8,7 +8,8 @@ import { startDrag } from "@/cut/lib/drag";
import { startSelectionDrag, togglePreviewSelection } from "@/cut/components/previewSelectionDrag";
import { pictureGroupSelected } from "@/cut/lib/previewSelection";
import { useSkim, usePreviewTime } from "@/cut/lib/playhead";
-import { useEditor } from "@/cut/lib/store";
+import { projectDuration, useEditor } from "@/cut/lib/store";
+import { overlayLiveAt, stageTimeAt } from "@/cut/lib/overlayLive";
import {
captionStyle,
cueAt,
@@ -212,6 +213,10 @@ export function OverlayLayer({
// Titles preview under the skimmer too (paused only), matching the canvas.
const t = usePreviewTime();
const skimTime = useSkim();
+ const total = useEditor(projectDuration);
+ // The moment the elements are on screen for: the canvas's own, which holds
+ // the last frame with the playhead on the cut's end.
+ const at = stageTimeAt(t, total);
const live = useLiveEffects();
// The grade and frame motion of every effect above this slice.
const graded = useMemo(
@@ -350,7 +355,7 @@ export function OverlayLayer({
// handles or drag. The timeline bar is the thing you grab.
if (o.kind === "effect") return null;
const selected = selectedIds.has(o.id) || sel?.id === o.id;
- const inRange = t >= o.start && t <= o.end;
+ const inRange = overlayLiveAt(o, at);
// While hover-scrubbing (paused, skimmer active) the preview must show the
// exact frame under the skimmer — a selected but out-of-frame title can't
// leak into a frame it isn't part of. Off the skimmer, a selected title
@@ -411,7 +416,7 @@ export function OverlayLayer({
style={{ filter, opacity: ghostAlpha }}
>
{overlays.map((o) => {
- if (o.hidden || o.kind === "effect" || t < o.start || t > o.end) return null;
+ if (o.hidden || o.kind === "effect" || !overlayLiveAt(o, at)) return null;
return (
<OverlayItem
key={o.id}
@@ -582,9 +587,12 @@ function SubtitleCaption({
const multiSelection = useEditor((s) => s.multiSelection);
const stagePress = useContext(StagePress);
const t = usePreviewTime();
+ const total = useEditor(projectDuration);
+ // The cue on screen, with the last one held when the playhead sits on the
+ // cut's end, the way the canvas holds the last frame.
const cues = laneCues(subtitles, lane);
- const cue = cueAt(cues, t);
+ const cue = cueAt(cues, stageTimeAt(t, total));
if (!cue || !cue.text.trim()) return null;
// The caption moves its whole track, so it takes the same rule the elements
// take: the cue on screen has to be the selection before a drag moves it.
diff --git a/site/src/cut/components/StageEffects.tsx b/site/src/cut/components/StageEffects.tsx
index e22f1266..1e4c042b 100644
--- a/site/src/cut/components/StageEffects.tsx
+++ b/site/src/cut/components/StageEffects.tsx
@@ -12,7 +12,8 @@ import {
washBackground,
} from "@donkeycut/effects-kit";
import { previewAt, subscribePlayhead, usePreviewTime } from "@/cut/lib/playhead";
-import { useEditor } from "@/cut/lib/store";
+import { projectDuration, useEditor } from "@/cut/lib/store";
+import { stageTimeAt } from "@/cut/lib/overlayLive";
import { isEffectOverlay, laneOf } from "@/cut/lib/types";
import {
hasEffects,
@@ -52,7 +53,11 @@ export { stageEffectTransform, stageSliceStructure, type LiveEffect, type StageS
export function useLiveEffects(): LiveEffect[] {
const overlays = useEditor((s) => s.overlays);
const t = usePreviewTime();
- return useMemo(() => liveEffectsAt(overlays, t), [overlays, t]);
+ const total = useEditor(projectDuration);
+
+ // Graded at the canvas's moment, which holds the last frame on the cut's end.
+ const at = stageTimeAt(t, total);
+ return useMemo(() => liveEffectsAt(overlays, at), [overlays, at]);
}
/**
@@ -105,7 +110,9 @@ export function StagePictureFx({ children }: { children: React.ReactNode }) {
return;
}
const apply = () => {
- const states = liveEffectsAt(overlays, previewAt()).map((e) => e.state);
+ // The canvas's moment: the last frame's, with the playhead on the end.
+ const at = stageTimeAt(previewAt(), projectDuration(useEditor.getState()));
+ const states = liveEffectsAt(overlays, at).map((e) => e.state);
const box = ref.current;
const split = box ? rgbSplitFilter(states, box.clientWidth, box.clientHeight) : "";
write([stageEffectFilter(states), split].filter(Boolean).join(" "), stageEffectTransform(states) ?? "");
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index 7a4d49d1..858e2648 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -16,6 +16,7 @@ import { assetIsSilent, CLIP_MIN_SECONDS, clipCovers, rectOf } from "@/cut/lib/t
import type { AudioClip, ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";
import { OWN_SOUND, soundSourceOf, soundWindow } from "@/cut/lib/soundSource";
import { SubjectMaskCompositor } from "@/cut/lib/behindPass";
+import { stageTimeAt } from "@/cut/lib/overlayLive";
import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";
import { hdrCanvasSupport } from "@/cut/lib/hdrCanvas";
import { displayIsHdr, HdrPresenter, onDisplayRangeChange, setHdrPreviewState } from "@/cut/lib/hdrPresent";
@@ -953,7 +954,6 @@ class Engine {
// background. Playing, that was the last frame before the stop, a single
// dark frame the paused redraw painted over; paused, it is a skim off the
// right edge of the cut. Both hold the last frame.
- const end = Math.max(0, total - 0.001);
let t: number;
if (playing) {
const head = playheadAt();
@@ -961,9 +961,9 @@ class Engine {
this.mixer.start(Math.min(head, total));
this.written = head;
}
- t = Math.max(0, Math.min(this.mixer.now(), end));
+ t = stageTimeAt(this.mixer.now(), total);
} else {
- t = Math.max(0, Math.min(previewAt(), end));
+ t = stageTimeAt(previewAt(), total);
if (this.mixer.running) {
this.mixer.stop();
flushMeter();
diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.ts
index ce801111..a6c61691 100644
--- a/site/src/cut/lib/behindPass.ts
+++ b/site/src/cut/lib/behindPass.ts
@@ -37,6 +37,7 @@ import {
import { ElementFx, elementLook } from "@donkeycut/effects-kit";
import { personSegmenter, segmentSubjectAlpha } from "./cutout";
import { allowance, holdMemory } from "./memoryBudget";
+import { overlayLiveAt } from "./overlayLive";
import { createRasterCanvas, snapshotRaster } from "./raster";
import { cutRenderEnv } from "./textRender";
import {
@@ -160,7 +161,7 @@ function drawable(o: Overlay): boolean {
/** The behind-the-speaker elements live at `t` (any drawable kind). */
export function behindOverlaysAt(overlays: Overlay[], t: number): Overlay[] {
return overlays.filter(
- (o) => behindSubjectOverlay(o) && drawable(o) && t >= o.start && t <= o.end
+ (o) => behindSubjectOverlay(o) && drawable(o) && overlayLiveAt(o, t)
);
}
@@ -486,7 +487,7 @@ export class SubjectMaskCompositor {
* before the frame is, so no frame of a render shows a stale one. */
async ready(overlays: Overlay[], w: number, h: number, assets: MediaAsset[], t: number): Promise<void> {
for (const o of overlays) {
- if (!behindSubjectOverlay(o) || !drawable(o) || t < o.start || t > o.end) continue;
+ if (!behindSubjectOverlay(o) || !drawable(o) || !overlayLiveAt(o, t)) continue;
const tLocal = Math.max(0, t - o.start);
const look = lookAt(o, tLocal, evalOverlayFrame(o, tLocal, w / h));
if (this.entryFor(o).byWord.has(look.key)) continue;
@@ -699,7 +700,7 @@ export class SubjectMaskCompositor {
this.sweepRasters();
const active = behindOverlaysAt(overlays, t);
const wantsMatte = overlays.some(
- (o) => frontSubjectOverlay(o) && drawable(o) && t >= o.start && t <= o.end
+ (o) => frontSubjectOverlay(o) && drawable(o) && overlayLiveAt(o, t)
);
if (active.length === 0 && !wantsMatte) return;
const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
diff --git a/site/src/cut/lib/effectStack.ts b/site/src/cut/lib/effectStack.ts
index 331af727..518519bf 100644
--- a/site/src/cut/lib/effectStack.ts
+++ b/site/src/cut/lib/effectStack.ts
@@ -18,6 +18,7 @@ import {
washBackground,
type EffectPreviewState,
} from "@donkeycut/effects-kit";
+import { overlayLiveAt } from "./overlayLive";
import { isEffectOverlay, laneOf, type Overlay } from "./types";
/** One effect live at a moment, with the lane that places it in the stack. */
@@ -31,7 +32,7 @@ export function liveEffectsAt(overlays: Overlay[], t: number): LiveEffect[] {
const live: LiveEffect[] = [];
for (const o of overlays) {
if (!isEffectOverlay(o) || isAudioEffect(o.effect)) continue;
- if (o.hidden || t < o.start || t > o.end) continue;
+ if (o.hidden || !overlayLiveAt(o, t)) continue;
live.push({
lane: laneOf(o),
state: effectPreviewState(effectRecipe(o), o.amount, t - o.start, o.focus, o.ramp, o.end - o.start, o),
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index b03d88c9..f3e5078c 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -52,6 +52,7 @@ import { ElementFx, elementLook, paintElementCrop } from "@donkeycut/effects-kit
import { applyEffectToCanvas, effectRecipe, diveView, divesAt, evalOverlayFrame, measureDiveFocus, retimeOf, grainTile, smoothsAt, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, planAnimatedLayers, type DiveFocus, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";
import { backdropStill, loadBackdropStill } from "./backdropStills";
import { hasSubjectOverlays, SubjectMaskCompositor } from "./behindPass";
+import { overlayLiveAt } from "./overlayLive";
import { MatteAlpha, matteStamp } from "./matteAlpha";
import { createRasterCanvas, snapshotRaster, type RasterSurface } from "./raster";
import { exportFrameSynth, SYNTH_EDGE, synthWeight, type FrameSynth } from "./frameSynth";
@@ -603,7 +604,7 @@ function liveEffectsAt(overlays: Overlay[], t: number): EffectOverlay[] {
return overlays
.filter(isEffectOverlay)
.filter((o) => !isAudioEffect(o.effect))
- .filter((o) => !o.hidden && t >= o.start && t < o.end)
+ .filter((o) => !o.hidden && overlayLiveAt(o, t))
.sort((a, b) => laneOf(b) - laneOf(a));
}
@@ -1324,7 +1325,7 @@ async function drawStamps(
| null;
if (!ctx) return;
for (const layer of layers) {
- if (t < layer.start || t >= layer.end) continue;
+ if (!overlayLiveAt(layer, t)) continue;
if (layer.lottie) {
// Nothing to cache: the animation's own canvas is sought per frame and
// drawn at the sticker's rect under the same delta transform.
diff --git a/site/src/cut/lib/overlayLive.test.ts b/site/src/cut/lib/overlayLive.test.ts
new file mode 100644
index 00000000..189697cb
--- /dev/null
+++ b/site/src/cut/lib/overlayLive.test.ts
@@ -0,0 +1,114 @@
+import { beforeAll, describe, expect, test } from "bun:test";
+import type { ShapeOverlay } from "@donkeycut/effects-kit";
+import { behindOverlaysAt, SubjectMaskCompositor } from "./behindPass";
+import { liveEffectsAt } from "./effectStack";
+import { renderProjectFrame } from "./exportRender";
+import { installSkiaRaster } from "./headless/skiaRaster";
+import { overlayLiveAt, stageTimeAt } from "./overlayLive";
+import type { ExportDoc } from "./renderSnapshot";
+import type { Overlay, TextOverlay } from "./types";
+
+// A shot's elements end on the frame the next shot starts. That frame belongs
+// to the incoming shot: frame 508 at 30 fps (t = 16.9333) shows the next
+// shot's elements alone, in the preview and in the export.
+
+const FPS = 30;
+const CUT = 508 / FPS;
+const size = { width: 320, height: 180 };
+
+beforeAll(async () => {
+ expect(await installSkiaRaster()).toBe(true);
+});
+
+const rect = (id: string, start: number, end: number, fill: string): ShapeOverlay => ({
+ id, kind: "shape", shape: "rect", start, end,
+ x: 0.5, y: 0.5, w: 0.5, h: 0.5, fill,
+});
+
+const flash = (id: string, start: number, end: number) =>
+ ({ id, kind: "effect", effect: "lightleak", leak: "film", amount: 1, start, end, x: 0.5, y: 0.5, lane: 0 }) as Overlay;
+
+const behind = (id: string, start: number, end: number): TextOverlay => ({
+ id, text: "SOON", start, end,
+ x: 0.5, y: 0.5, size: 160, font: "sf", weight: 900, color: "#ffffff",
+ plate: false, shadow: false,
+ mask: { kind: "subject", invert: true } as TextOverlay["mask"],
+});
+
+const docOf = (overlays: Overlay[]): ExportDoc => ({
+ aspect: "16:9", assets: [], clips: [], audioClips: [], overlays,
+ subtitles: { cues: [], showOnVideo: false, showOnTimeline: false },
+});
+
+/** Pixels where one channel is lit and the other two are dark. */
+function inkOf(data: Uint8ClampedArray, channel: number): number {
+ let n = 0;
+ for (let i = 0; i < data.length; i += 4) {
+ const lit = data[i + channel] > 128;
+ const others = [0, 1, 2].filter((c) => c !== channel).every((c) => data[i + c] < 64);
+ if (lit && others) {
+ n++;
+ }
+ }
+ return n;
+}
+
+describe("the frame on a cut", () => {
+ const outgoing = rect("out", 15, CUT, "#ff0000");
+ const incoming = rect("in", CUT, 18.5, "#0000ff");
+
+ test("the preview's elements show the incoming shot's alone", () => {
+ const live = [outgoing, incoming].filter((o) => overlayLiveAt(o, stageTimeAt(CUT, 20)));
+ expect(live.map((o) => o.id)).toEqual(["in"]);
+ });
+
+ test("the preview's effects grade with the incoming shot's alone", () => {
+ const live = liveEffectsAt([flash("a", 15, CUT), flash("b", CUT, 18.5)], CUT);
+ expect(live.length).toBe(1);
+ });
+
+ test("the behind-the-speaker pass draws the incoming shot's alone", async () => {
+ const overlays = [behind("out", 15, CUT), behind("in", CUT, 18.5)];
+ expect(behindOverlaysAt(overlays, CUT).map((o) => o.id)).toEqual(["in"]);
+
+ // The export prepares a picture for each element it is about to draw.
+ const pass = new SubjectMaskCompositor();
+ await pass.ready(overlays, 540, 960, [], CUT);
+ const rasters = (pass as unknown as { rasters: Map<string, unknown> }).rasters;
+ expect([...rasters.keys()]).toEqual(["in"]);
+ pass.dispose();
+ });
+
+ test("the export draws the incoming shot's alone", async () => {
+ const frame = await renderProjectFrame(docOf([outgoing, incoming]), CUT, size, () => "");
+ const data = (frame.getContext("2d") as CanvasRenderingContext2D).getImageData(0, 0, size.width, size.height).data;
+ expect(inkOf(data, 0)).toBe(0);
+ expect(inkOf(data, 2)).toBeGreaterThan(1000);
+ });
+
+ test("a cut every frame shows one shot's element per frame", () => {
+ const shots = Array.from({ length: 10 }, (_, i) => rect(`s${i}`, (500 + i) / FPS, (501 + i) / FPS, "#ffffff"));
+ for (let i = 0; i < 10; i++) {
+ const live = shots.filter((o) => overlayLiveAt(o, stageTimeAt((500 + i) / FPS, 20)));
+ expect(live.map((o) => o.id)).toEqual([`s${i}`]);
+ }
+ });
+});
+
+// Playback stops with the playhead on the cut's end. The paused frame there
+// is the cut's last frame, so what runs to the end still shows.
+describe("paused at the cut's end", () => {
+ const total = CUT;
+
+ test("an element running to the end shows", () => {
+ expect(overlayLiveAt(rect("last", 15, total, "#ff0000"), stageTimeAt(total, total))).toBe(true);
+ });
+
+ test("an effect running to the end grades", () => {
+ expect(liveEffectsAt([flash("last", 15, total)], stageTimeAt(total, total)).length).toBe(1);
+ });
+
+ test("a moment inside the cut is its own", () => {
+ expect(stageTimeAt(3, total)).toBe(3);
+ });
+});
diff --git a/site/src/cut/lib/overlayLive.ts b/site/src/cut/lib/overlayLive.ts
new file mode 100644
index 00000000..fd3268cf
--- /dev/null
+++ b/site/src/cut/lib/overlayLive.ts
@@ -0,0 +1,27 @@
+/**
+ * When an element is on screen.
+ *
+ * An element draws from its start up to its end, end excluded: the frame on
+ * a cut belongs to the shot that starts there. The preview, the export and
+ * the behind-the-speaker pass all ask here, so they draw the same elements
+ * on every frame.
+ */
+
+/** How far inside the cut's end the stage draws a moment on the end. */
+const END_HOLD = 0.001;
+
+/** Whether an element draws at `t`. One ending at 2 and one starting at 2
+ * meet on that frame, and the frame shows the second alone. */
+export function overlayLiveAt(o: { start: number; end: number }, t: number): boolean {
+ return t >= o.start && t < o.end;
+}
+
+/**
+ * The moment the stage draws for a playhead at `t`, in a cut `total` long.
+ * Playback stops with the playhead on the end, a moment past the final shot,
+ * so the stage draws the moment just inside it: paused there, the picture is
+ * the cut's last frame, with every element that runs to the end.
+ */
+export function stageTimeAt(t: number, total: number): number {
+ return Math.max(0, Math.min(t, total - END_HOLD));
+}