Skip to content

Open source CapCut alternative.

TypeScript

feat(site): rebuild the Cut preview on WebCodecs

David Lee8a0a9fe727f
diff --git a/site/src/cut/components/Preview.tsx b/site/src/cut/components/Preview.tsx
index 40bb358e..fd5d785c 100644
--- a/site/src/cut/components/Preview.tsx
+++ b/site/src/cut/components/Preview.tsx
@@ -1,9 +1,16 @@
"use client";
-import { useEffect, useRef, useState, type RefObject } from "react";
+import { useEffect, useMemo, useRef, useState, type RefObject } from "react";
import { usePlayback } from "@/cut/hooks/usePlayback";
import { clearAssetDrag, setAssetDragData } from "@/cut/lib/assetDrag";
import { startDrag } from "@/cut/lib/drag";
+import {
+ playheadAt,
+ previewAt,
+ subscribePlayhead,
+ usePreviewSelector,
+ useSkim,
+} from "@/cut/lib/playhead";
import { getClipSpans, projectDuration, useEditor } from "@/cut/lib/store";
import {
capturePoster,
@@ -16,19 +23,20 @@ import { clipKeyed, clipPoseAt, frameOf, isFullRect, rectOf, REGION_MAX_SCALE, t
import { hasMaskKeys, type MaskKey } from "@donkeycut/effects-kit";
import { cn } from "@/lib/utils";
import { MaskGizmoCore, OverlayLayer } from "./OverlayLayer";
-import { StageEffectPaint, stageSlices, useEffectLanes, useStageEffects } from "./StageEffects";
+import {
+ StageEffectPaint,
+ StagePictureFx,
+ stageSliceStructure,
+ useEffectLanes,
+} from "./StageEffects";
/** The clip under the playhead, when it overflows the frame in fill mode. */
-function pannableSpan(s: {
- clips: VideoClip[];
- assets: MediaAsset[];
- currentTime: number;
- aspect: Aspect;
-}): ClipSpan | null {
+function pannableSpan(
+ s: { clips: VideoClip[]; assets: MediaAsset[]; aspect: Aspect },
+ t: number
+): ClipSpan | null {
const spans = getClipSpans(s.clips, s.assets);
- const span =
- spans.find((sp) => s.currentTime >= sp.start && sp.start + sp.len > s.currentTime) ??
- spans[spans.length - 1];
+ const span = spans.find((sp) => t >= sp.start && sp.start + sp.len > t) ?? spans[spans.length - 1];
// Pan only makes sense for a full-frame fill clip; a regioned clip is moved
// with its own preview handle instead.
if (!span || span.clip.fit !== "fill" || !isFullRect(rectOf(span.clip))) return null;
@@ -41,6 +49,34 @@ function pannableSpan(s: {
return ox > 1 || oy > 1 ? span : null;
}
+/**
+ * Show the grab cursor while the clip under the playhead can be panned.
+ *
+ * Which clip that is changes sixty times a second, and the answer is one class
+ * on one element. Rendering the whole preview to find out would put every
+ * frame of playback through React; the class goes on from a subscription
+ * instead, and only when the answer actually changes.
+ */
+function usePannableCursor(target: RefObject<HTMLElement | null>) {
+ const clips = useEditor((s) => s.clips);
+ const assets = useEditor((s) => s.assets);
+ const aspect = useEditor((s) => s.aspect);
+ useEffect(() => {
+ const el = target.current;
+ if (!el) return;
+ let on: boolean | null = null;
+ const apply = () => {
+ const next = pannableSpan({ clips, assets, aspect }, previewAt()) !== null;
+ if (next === on) return;
+ on = next;
+ el.classList.toggle("cursor-grab", next);
+ el.classList.toggle("active:cursor-grabbing", next);
+ };
+ apply();
+ return subscribePlayhead(apply);
+ }, [target, clips, assets, aspect]);
+}
+
/** Paint the picture this project's preview last showed, then hand the canvas
* back to the engine. Cloud media is a network away, so without this the
* preview holds black for about a second on every open while the first decoder
@@ -73,18 +109,20 @@ function useCachedFirstFrame(canvasRef: RefObject<HTMLCanvasElement | null>) {
export function Preview() {
const canvasRef = useRef<HTMLCanvasElement>(null);
const wrapRef = useRef<HTMLDivElement>(null);
+ const stageRef = useRef<HTMLDivElement>(null);
const [stage, setStage] = useState({ w: 270, h: 480 });
- const pannable = useEditor((s) => pannableSpan(s) !== null);
const aspect = useEditor((s) => s.aspect);
const frame = frameOf(aspect);
usePlayback(canvasRef);
- // An effect grades what plays under it, so the stage is built in slices:
- // the picture, then the elements of each lane band with the look of the
- // effects above them, and each effect's paints sitting where the effect does.
- const stageFx = useStageEffects();
+ // An effect grades what plays under it, so the stage is built in slices: the
+ // picture, then the elements of each lane band with the look of the effects
+ // above them, and each effect's paints sitting where the effect does. Only
+ // which lanes hold effects decides the shape, so this component never renders
+ // for the clock; each slice reads the clock for itself.
const effectLanes = useEffectLanes();
- const { picture, slices } = stageSlices(stageFx, effectLanes);
+ const slices = useMemo(() => stageSliceStructure(effectLanes), [effectLanes]);
+ usePannableCursor(stageRef);
useEffect(() => {
setPreviewCanvas(canvasRef.current);
@@ -93,6 +131,17 @@ export function Preview() {
useCachedFirstFrame(canvasRef);
+ // The canvas backing store matches the pixels the screen will actually show,
+ // capped at the project's own frame. Painting a 4K backing store into a box
+ // a few hundred pixels wide put every grade, look and mask pass through
+ // millions of pixels nobody could see; the export renders at full size on its
+ // own surface, so nothing about the file changes.
+ const surface = useMemo(() => {
+ const dpr = typeof window === "undefined" ? 1 : Math.min(2, window.devicePixelRatio || 1);
+ const w = Math.max(2, Math.min(frame.w, Math.round(stage.w * dpr)));
+ return { w, h: Math.max(2, Math.round((w * frame.h) / frame.w)) };
+ }, [stage.w, frame.w, frame.h]);
+
useEffect(() => {
const wrap = wrapRef.current;
if (!wrap) return;
@@ -115,14 +164,14 @@ export function Preview() {
const s = useEditor.getState();
const total = projectDuration(s);
if (!total) return;
- if (!s.playing && s.currentTime >= total - 0.01) s.seek(0);
+ if (!s.playing && playheadAt() >= total - 0.01) s.seek(0);
s.setPlaying(!s.playing);
};
// Drag a fill-mode clip inside the frame to choose the visible crop.
const panDrag = (e: React.PointerEvent) => {
const s = useEditor.getState();
- const span = pannableSpan(s);
+ const span = pannableSpan(s, previewAt());
if (!span) return false;
const fr = frameOf(s.aspect);
const { width = 1, height = 1 } = span.asset;
@@ -168,7 +217,7 @@ export function Preview() {
const px = (e.clientX - rct.left) / rct.width;
const py = (e.clientY - rct.top) / rct.height;
const s = useEditor.getState();
- const t = s.currentTime;
+ const t = previewAt();
let best: { id: string; track: number } | null = null;
for (const c of s.clips) {
if (c.hidden) continue;
@@ -198,9 +247,9 @@ export function Preview() {
so a box dragged past the frame edge stays visible and grabbable. */}
<div className="relative" style={{ width: stage.w, height: stage.h }}>
<div
+ ref={stageRef}
className={cn(
- "stage absolute inset-0 overflow-hidden rounded-xl bg-black shadow-[0_0_0_1px_rgba(0,0,0,0.08),0_12px_36px_rgba(0,0,0,0.18)]",
- pannable && "cursor-grab active:cursor-grabbing"
+ "stage absolute inset-0 overflow-hidden rounded-xl bg-black shadow-[0_0_0_1px_rgba(0,0,0,0.08),0_12px_36px_rgba(0,0,0,0.18)]"
)}
onPointerDown={(e) => {
if (
@@ -229,12 +278,12 @@ export function Preview() {
}
}}
>
+ <StagePictureFx>
<canvas
ref={canvasRef}
- width={frame.w}
- height={frame.h}
+ width={surface.w}
+ height={surface.h}
className="block size-full"
- style={{ transform: picture.transform, filter: picture.filter }}
// Drag the viewport to reference what's on screen: the clip under
// the playhead travels as a media drag (timeline placement, chat
// attachment, generation reference). Pan on a fill clip wins —
@@ -243,7 +292,7 @@ export function Preview() {
onDragStart={(e) => {
const s = useEditor.getState();
const spans = getClipSpans(s.clips, s.assets);
- const t = s.currentTime;
+ const t = previewAt();
const span =
spans.find((sp) => t >= sp.start && sp.start + sp.len > t) ??
spans[spans.length - 1];
@@ -252,20 +301,20 @@ export function Preview() {
}}
onDragEnd={clearAssetDrag}
/>
+ </StagePictureFx>
<ClipMaskGizmo stage={stage} />
{slices.map((slice) =>
slice.kind === "elements" ? (
<OverlayLayer
key={slice.key}
stageWidth={stage.w}
- transform={slice.transform}
- filter={slice.filter}
+ gradeAbove={slice.gradeAbove}
from={slice.from}
to={slice.to}
captions={slice.captions}
/>
) : (
- <StageEffectPaint key={slice.key} states={slice.states} />
+ <StageEffectPaint key={slice.key} lane={slice.lane} />
)
)}
</div>
@@ -319,16 +368,24 @@ function OverlayPipHandle({ stage }: { stage: { w: number; h: number } }) {
const clips = useEditor((s) => s.clips);
const assets = useEditor((s) => s.assets);
const aspect = useEditor((s) => s.aspect);
- const currentTime = useEditor((s) => s.currentTime);
- const skimTime = useEditor((s) => s.skimTime);
- const playing = useEditor((s) => s.playing);
+ const skimTime = useSkim();
+ // The handle only cares whether its clip is on screen, so it subscribes to
+ // that answer instead of to the clock: one render when the clip comes and
+ // goes, rather than one per frame while it stays.
+ const selectedClip = selection?.kind === "clip" ? clips.find((c) => c.id === selection.id) : null;
+ const live = usePreviewSelector((t) => {
+ if (!selectedClip) return false;
+ const speed = selectedClip.speed && selectedClip.speed > 0 ? selectedClip.speed : 1;
+ const len = Math.max(0.1, (selectedClip.out - selectedClip.in) / speed);
+ return t >= selectedClip.start && t < selectedClip.start + len;
+ });
const [guides, setGuides] = useState<{ x: number | null; y: number | null }>({
x: null,
y: null,
});
// While hover-scrubbing the preview shows the skimmer's frame, where the
// selected clip may not even be on screen — selection stays, the handle hides.
- if (!playing && skimTime !== null) return null;
+ if (skimTime !== null) return null;
// Resolve the selected, live, regioned clip (any track) plus how to patch its
// rect. A clip's own footprint equals its span length, so one path serves
@@ -339,12 +396,10 @@ function OverlayPipHandle({ stage }: { stage: { w: number; h: number } }) {
// far the scaled picture overhangs the box (frame px), and the pan to start
// the gesture from.
let pan: { id: string; ox: number; oy: number; panX0: number; panY0: number } | null = null;
- if (selection?.kind === "clip") {
- const clip = clips.find((c) => c.id === selection.id);
+ {
+ const clip = selectedClip;
if (clip && !clip.hidden) {
- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
- const len = Math.max(0.1, (clip.out - clip.in) / speed);
- if (currentTime >= clip.start && currentTime < clip.start + len) {
+ if (live) {
rect = rectOf(clip);
apply = (frame) => useEditor.getState().updateClipTransient(clip.id, { frame });
const asset = assets.find((a) => a.id === clip.assetId);
@@ -481,17 +536,19 @@ function OverlayPipHandle({ stage }: { stage: { w: number; h: number } }) {
function ClipMaskGizmo({ stage }: { stage: { w: number; h: number } }) {
const selection = useEditor((s) => s.selection);
const clips = useEditor((s) => s.clips);
- const currentTime = useEditor((s) => s.currentTime);
- const skimTime = useEditor((s) => s.skimTime);
- const playing = useEditor((s) => s.playing);
- if (!playing && skimTime !== null) return null;
- if (selection?.kind !== "clip") return null;
- const clip = clips.find((c) => c.id === selection.id);
- if (!clip?.mask || clip.mask.kind === "subject" || clip.hidden) return null;
+ const skimTime = useSkim();
+ // A keyframed mask travels with the clock, so this one does follow every
+ // frame — but only while a masked clip is the selection.
+ const masked =
+ selection?.kind === "clip" ? clips.find((c) => c.id === selection.id) ?? null : null;
+ const armed = !!masked?.mask && masked.mask.kind !== "subject" && !masked.hidden;
+ const tLocal = usePreviewSelector((t) => (armed && masked ? t - masked.start : -1));
+ if (skimTime !== null) return null;
+ const clip = masked;
+ if (!armed || !clip?.mask) return null;
const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
const len = Math.max(0.1, (clip.out - clip.in) / speed);
- if (currentTime < clip.start || currentTime >= clip.start + len) return null;
- const tLocal = currentTime - clip.start;
+ if (tLocal < 0 || tLocal >= len) return null;
const rect = rectOf(clip);
const pose = clipKeyed(clip) ? clipPoseAt(clip, tLocal) : null;
const ax = pose ? pose.x : rect.x + rect.w / 2;
diff --git a/site/src/cut/components/StageEffects.tsx b/site/src/cut/components/StageEffects.tsx
index fc675484..3fb1ea71 100644
--- a/site/src/cut/components/StageEffects.tsx
+++ b/site/src/cut/components/StageEffects.tsx
@@ -1,17 +1,22 @@
"use client";
-import { useMemo } from "react";
+import { useEffect, useMemo, useRef } from "react";
import {
- effectPreviewState,
grainTileUrl,
LEAK_TINT,
leakGradient,
streakGradient,
- type EffectPreviewState,
} from "@donkeycut/effects-kit";
+import { previewAt, subscribePlayhead, usePreviewTime } from "@/cut/lib/playhead";
import { useEditor } from "@/cut/lib/store";
import { isEffectOverlay, laneOf } from "@/cut/lib/types";
-import type { LiveEffect } from "@/cut/lib/effectStack";
+import {
+ hasEffects,
+ liveEffectsAt,
+ stageEffectFilter,
+ stageEffectTransform,
+ type LiveEffect,
+} from "@/cut/lib/effectStack";
import "./grain.css";
/**
@@ -26,27 +31,59 @@ import "./grain.css";
* Both exports walk the same stack, so what plays here is what renders.
*/
-export { stageEffectTransform, stageSlices, type LiveEffect, type StageSlice } from "@/cut/lib/effectStack";
+export { stageEffectTransform, stageSliceStructure, type LiveEffect, type StageSlice } from "@/cut/lib/effectStack";
-/** Every effect live at `t`, deepest lane last. Skimming previews them too,
- * the same as the elements beside them. */
-export function useStageEffects(): LiveEffect[] {
+/**
+ * Every effect live right now, deepest lane last. Skimming previews them too,
+ * the same as the elements beside them.
+ *
+ * This subscribes at frame rate, so only the leaves of the stage call it — the
+ * paints of one lane, the elements of one band. The component that lays the
+ * stage out stays off the clock.
+ */
+export function useLiveEffects(): LiveEffect[] {
const overlays = useEditor((s) => s.overlays);
- const currentTime = useEditor((s) => s.currentTime);
- const skimTime = useEditor((s) => s.skimTime);
- const playing = useEditor((s) => s.playing);
- const t = !playing && skimTime !== null ? skimTime : currentTime;
- return useMemo(
- () =>
- overlays
- .filter(isEffectOverlay)
- .filter((o) => !o.hidden && t >= o.start && t <= o.end)
- .map((o) => ({
- lane: laneOf(o),
- state: effectPreviewState(o.effect, o.amount, t - o.start, o.focus, o.ramp, o.end - o.start),
- }))
- .sort((a, b) => a.lane - b.lane),
- [overlays, t]
+ const t = usePreviewTime();
+ return useMemo(() => liveEffectsAt(overlays, t), [overlays, t]);
+}
+
+/**
+ * Wear the whole stack's grade and frame motion over the picture.
+ *
+ * The picture is a canvas the engine paints imperatively, and its grade is two
+ * style properties on the box around it. Writing them from a subscription keeps
+ * a project with no effects at zero React work per frame, and one with effects
+ * at two string comparisons.
+ */
+export function StagePictureFx({ children }: { children: React.ReactNode }) {
+ const overlays = useEditor((s) => s.overlays);
+ const ref = useRef<HTMLDivElement>(null);
+ useEffect(() => {
+ const write = (filter: string, transform: string) => {
+ const style = ref.current?.style;
+ if (!style) return;
+ if (style.filter !== filter) style.filter = filter;
+ if (style.transform !== transform) style.transform = transform;
+ };
+ if (!hasEffects(overlays)) {
+ write("", "");
+ return;
+ }
+ const apply = () => {
+ const states = liveEffectsAt(overlays, previewAt()).map((e) => e.state);
+ write(stageEffectFilter(states) ?? "", stageEffectTransform(states) ?? "");
+ };
+ apply();
+ const stop = subscribePlayhead(apply);
+ return () => {
+ stop();
+ write("", "");
+ };
+ }, [overlays]);
+ return (
+ <div ref={ref} className="absolute inset-0">
+ {children}
+ </div>
);
}
@@ -63,7 +100,12 @@ export function useEffectLanes(): number[] {
/** The frame-wide paints of the effects on one lane, over everything under
* them and under everything above. */
-export function StageEffectPaint({ states }: { states: EffectPreviewState[] }) {
+export function StageEffectPaint({ lane }: { lane: number }) {
+ const live = useLiveEffects();
+ const states = useMemo(
+ () => live.filter((e) => e.lane === lane).map((e) => e.state),
+ [live, lane]
+ );
if (states.length === 0) return null;
const grainUrl = grainTileUrl();
const grain = Math.max(0, ...states.map((s) => s.grain ?? 0));
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index 552f5594..0e64b7a9 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -1,588 +1,331 @@
"use client";
import { useEffect, type RefObject } from "react";
-import { clipSpeed, getClipSpans, overlayLayers, projectDuration, useEditor } from "@/cut/lib/store";
+import {
+ clipSpeed,
+ getClipSpans,
+ overlayLayers,
+ projectDuration,
+ useEditor,
+} from "@/cut/lib/store";
+import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";
import { isFullRect, projectFadeSeconds, rectOf } from "@/cut/lib/types";
import type { ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";
import { SubjectMaskCompositor } from "@/cut/lib/behindPass";
import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";
-import { duckGainAt, overlayPlan, prerollLead, trackZeroPlan } from "@/cut/lib/framePlan";
-import { reportMediaElementError } from "@/cut/lib/mediaLinks";
+import { duckGainAt, overlayPlan, trackZeroPlan } from "@/cut/lib/framePlan";
+import { type ClipFrameSource, FrameSourcePool, mappingKey } from "@/cut/lib/frameSource";
+import { PreviewMixer, type Voice } from "@/cut/lib/previewMixer";
+import { markLiveSamples, markPresent, markTick, tracing } from "@/cut/lib/perfTrace";
import { registerSourceSampler } from "@/cut/lib/previewCanvas";
-// A video clip on any track is backed by a <video> for footage or an <img>
-// for a still image. These helpers read either kind uniformly so the
-// compositor stays one code path — an image never seeks, plays, or carries
-// audio.
-type MediaEl = HTMLVideoElement | HTMLImageElement;
-const isImageEl = (el: MediaEl): el is HTMLImageElement =>
- typeof HTMLImageElement !== "undefined" && el instanceof HTMLImageElement;
-const elReady = (el: MediaEl) =>
- isImageEl(el) ? el.complete && el.naturalWidth > 0 : el.readyState >= 2 && el.videoWidth > 0;
-const elW = (el: MediaEl) => (isImageEl(el) ? el.naturalWidth : el.videoWidth);
-const elH = (el: MediaEl) => (isImageEl(el) ? el.naturalHeight : el.videoHeight);
-// A source that will never become ready: a video decode error, or an image
-// that finished loading (`complete`) with no pixels (a broken/unreachable URL).
-// The compositor paints through these instead of wedging on them.
-const elErrored = (el: MediaEl) =>
- isImageEl(el) ? el.complete && el.naturalWidth === 0 : !!el.error;
-/** Browsers reject playbackRate outside roughly 0.0625–16, so the element
- * rate is clamped; beyond it the periodic seek correction carries the true
- * speed, at the cost of a choppier preview. Export renders the real rate. */
-const safeRate = (speed: number) => Math.min(16, Math.max(0.0625, speed));
-
-// Decode-ahead window. Every tick, clips whose entrance is within this many
-// seconds of the playhead get their decoder built now and seeked to their first
-// frame, so the file is already buffering (preload="auto") and frame 0 is
-// decoded before the playhead reaches them — a cut lands with no cold-start
-// hitch. Capped so a montage of tiny clips can't start a fetch storm that
-// starves the clip actually on screen.
-const WARM_HORIZON_S = 8;
-const WARM_MAX = 4;
+/**
+ * The preview engine.
+ *
+ * Three things meet here and nothing else does. The frame plan says what the
+ * cut looks like at an instant; the frame sources hold decoded pictures ready
+ * to answer for it; the compositor paints. The engine's whole job is to pick
+ * the instant, ask, and draw.
+ *
+ * The instant comes from the mixer's clock while playing and from the playhead
+ * while paused, so sound and picture are two readings of one number rather than
+ * two clocks chasing each other. A cut is not an event: at the moment the plan
+ * names a different clip, that clip's frames are already decoded, because the
+ * scheduler has been reading ahead of the playhead the whole time.
+ *
+ * The loop runs while something is moving. A paused editor with nothing dirty
+ * schedules no frames at all.
+ */
-// Live decoder budget. A decoder is not free: it holds an open fetch against
-// the media file, a demuxer, a decoded-frame buffer, and one of the small
-// number of hardware decode slots the browser hands out per tab. A clip's
-// element used to live from the first time the playhead came near it until the
-// clip was deleted, so a long cut split many times accumulated decoders for
-// every clip ever visited — all of them buffering the same file at once. Past
-// the hardware limit the extras fall back to software decode, and since sound
-// stays real-time while frames arrive late, the picture drifts behind the
-// audio. Only reloading the page cleared it.
-//
-// So the pool is capped. Clips this frame is built from are never candidates;
-// beyond them the least recently touched elements are torn down, leaving room
-// for a few just-played clips so stepping back a cut stays hot. Rebuilding is
-// cheap — the bytes are in the HTTP cache, and the same rebuild path already
-// runs for stalled decoders.
+/** How far ahead of the playhead a clip's decoder is opened and started. */
+const WARM_HORIZON_S = 2.5;
+/** Inside this, an upcoming clip is walked rather than merely opened. */
+const WARM_STREAM_S = 0.75;
+/** Decoders alive at once. Past the tab's hardware decode slots the rest fall
+ * back to software, and frames start arriving late. */
const DECODER_BUDGET = 12;
-// Decoder health. A decoder can stop making progress two ways, and both used
-// to be permanent for the life of the tab. It can error — and an errored
-// element stays errored, because re-minting the media links does not repoint
-// it: a signed URL is stable for its whole signing window, so the mint hands
-// back the very string that just failed and nothing swaps. Or it can wedge
-// before its first frame with nothing arriving, which nothing was watching for
-// at all. Either way that clip played black for the rest of the session while
-// its neighbours played fine. Decoders are cheap and replaceable, so a clip
-// whose element stops advancing gets a new one, bounded so a source that
-// really is unplayable settles into the paint-through path instead of
-// refetching forever.
-const DECODER_STALL_MS = 10_000;
-const DECODER_RETRY_MS = 1_000;
-const DECODER_REBUILDS = 3;
-
-// Ahead of a clip's entrance its element is played muted and undrawn, so the
-// decoder is already running across the in-point when the cut lands and the
-// handoff play() resumes hot instead of spinning a cold decoder up. How long
-// that roll gets is `prerollLead`, and when a clip is close enough to be worth
-// keeping alive is the frame plan's `upcoming` — both live there, so the
-// picture and the decoders agree on when a clip is about to be needed.
+/** Source time of a clip at timeline time `t`. */
+const sourceTimeOf = (clip: VideoClip, t: number) =>
+ clip.in + Math.max(0, t - clip.start) * clipSpeed(clip);
-const pauseEl = (el: MediaEl) => {
- if (!isImageEl(el) && !el.paused) el.pause();
-};
+/** The decode identity of a clip — see `mappingKey`. */
+const keyOf = (clip: VideoClip, asset: MediaAsset) =>
+ mappingKey(asset.id, clipSpeed(clip), clip.in, clip.start);
-/** Decoder pool key. Clips whose frames are identical at every timeline
- * instant — same source, same speed, same source-time offset — share one
- * element: srcT(t) = in + (t − start)·speed, so equal (asset, speed,
- * in − start·speed) means equal pictures at all times. A tiled reveal that
- * plays one source across many tracks costs one decoder instead of one per
- * tile, and a split of one source hands the playing element straight across
- * the cut — no resume at all. Different trims of a source keep separate
- * elements, so a same-source dissolve still blends two distinct frames. */
-const mappingKey = (clip: VideoClip, asset: MediaAsset) => {
- const speed = clipSpeed(clip);
- return `${asset.id}|${speed}|${(clip.in - clip.start * speed).toFixed(3)}`;
-};
-/** Build the decoder element for a clip's asset: an <img> for a still, a
- * hidden <video> for footage. */
-function makeMediaEl(asset: MediaAsset): MediaEl {
- if (asset.type === "image") {
- const img = document.createElement("img");
- img.crossOrigin = "anonymous";
- // Detached elements never reach the global error listener — report
- // failures explicitly so an expired signed URL re-mints the batch.
- img.addEventListener("error", () => reportMediaElementError(img));
- img.src = asset.url;
- return img;
- }
- const v = document.createElement("video");
- v.playsInline = true;
- v.preload = "auto";
- v.crossOrigin = "anonymous";
- v.addEventListener("error", () => reportMediaElementError(v));
- v.src = asset.url;
- return v;
-}
-/** Release an element's source. Images just drop the src; videos stop and
- * unload the decoder. */
-function teardown(el: MediaEl) {
- if (isImageEl(el)) {
- el.removeAttribute("src");
- return;
- }
- el.pause();
- el.removeAttribute("src");
- el.load();
-}
-
-// The canvas backing store is the full frame resolution (1080×1920 or
-// 1920×1080, set by Preview from the project aspect) so the preview stays
-// sharp on Retina displays. The engine reads the size off the canvas each
-// frame, so an aspect switch takes effect seamlessly.
-
-/**
- * Preview engine. One hidden <video> per track-0 clip and one <audio> per
- * soundtrack clip; the active clip's video element is the master clock while
- * playing and every frame is composited onto the preview canvas (contain-fit,
- * matching the export's letterboxing).
- */
class Engine {
- // Keyed by `mappingKey`, not asset id: two trims of the same source get
- // their own decoders, so a cross-dissolve can show both at once and the
- // incoming clip warms during the overlap instead of fighting the outgoing
- // one over a single element's seek head (the black flash between
- // same-source segments) — while clips that show identical frames (a tiled
- // reveal, a plain split) share one.
- private videoEls = new Map<string, MediaEl>();
- // The overlay tracks' pool, keyed the same way, kept apart from track 0's
- // so the two never trade elements mid-frame.
- private overlayEls = new Map<string, MediaEl>();
- private audioEls = new Map<string, HTMLAudioElement>();
- // Per-decoder health, keyed like the element maps: the last progress
- // reading and when it was taken, plus how many times this decoder has
- // been rebuilt on the current source.
- private health = new Map<string, { mark: number; at: number; rebuilds: number }>();
- // Decoder recency, keyed like the element maps: the tick that last asked
- // for this element. Anything below the current tick is idle and can be
- // evicted; the current tick's entries are what the frame on screen is made
- // of.
- private used = new Map<string, number>();
- // Captured entrance frames for live blend windows, keyed like the element
- // maps: the bitmap the blend draws while the incoming element pre-rolls.
- private heldEntrance = new Map<string, { src: string; canvas: HTMLCanvasElement }>();
- private frame = 0;
- // The clip array the decoder maps were last reconciled against.
- private clipsSeen: VideoClip[] | null = null;
+ private comp: FrameCompositor;
+ private pool = new FrameSourcePool(DECODER_BUDGET, () => this.wake());
+ private mixer = new PreviewMixer();
+ private behind = new SubjectMaskCompositor(true);
+
private raf = 0;
private disposed = false;
- // Wall-clock stamp for advancing time where track 0 has nothing playing —
- // in a gap or past its end there is no track-0 video element to act as the
- // master clock.
- private lastPlayNow = 0;
- // Everything this engine puts on the canvas goes through here. The engine
- // owns the decoders and the clock; the compositor owns the picture, and an
- // export renders through the same one.
- private comp: FrameCompositor;
+ /** Set when something changed that the canvas has yet to show. */
+ private dirty = true;
+ /** Sources this frame is being drawn from. The warm pass leaves these alone:
+ * a warm ask lands on the same source as a live clip whenever they share a
+ * mapping — a plain split reads straight across its own cut — and steering a
+ * walk that is already carrying the picture would tear it down every tick. */
+ private used = new Set<ClipFrameSource>();
+ /** The playhead value the engine itself wrote last, so its own echo is
+ * tellable from an outside move — a seek while playing. */
+ private written: number | null = null;
+
+ private unsubscribe: () => void;
+ private unwatch: () => void;
+ private sizeWatch: MutationObserver;
constructor(private canvas: HTMLCanvasElement) {
this.comp = new FrameCompositor(canvas);
this.tick = this.tick.bind(this);
- this.raf = requestAnimationFrame(this.tick);
+ // A moved playhead and an edited document both change the picture. Either
+ // one wakes the loop; nothing else does, so an idle editor costs nothing.
+ this.unsubscribe = subscribePlayhead(() => this.wake());
+ this.unwatch = useEditor.subscribe(() => this.wake());
+ // The backing store follows the stage box, and assigning a canvas's width
+ // or height erases it. That write comes from layout — no playhead move, no
+ // document edit — so the attributes themselves are the wake signal.
+ this.sizeWatch = new MutationObserver(() => this.wake());
+ this.sizeWatch.observe(canvas, { attributes: true, attributeFilter: ["width", "height"] });
+ this.wake();
}
dispose() {
this.disposed = true;
- cancelAnimationFrame(this.raf);
- for (const el of this.videoEls.values()) teardown(el);
- for (const el of this.overlayEls.values()) teardown(el);
- for (const el of this.audioEls.values()) el.pause();
- this.videoEls.clear();
- this.overlayEls.clear();
- this.audioEls.clear();
- this.health.clear();
- this.used.clear();
- this.heldEntrance.clear();
- }
-
- /** Release one decoder. The element maps are disjoint by key, so this
- * serves track 0 and the overlay tracks alike. */
- private dropDecoder(key: string) {
- const el = this.videoEls.get(key) ?? this.overlayEls.get(key);
- if (el) teardown(el);
- this.videoEls.delete(key);
- this.overlayEls.delete(key);
- this.health.delete(key);
- this.used.delete(key);
- this.heldEntrance.delete(key);
- }
-
- /** Hold the live decoder pool to `DECODER_BUDGET`, oldest first. Elements
- * touched during the tick that just ran are the frame on screen — the
- * master, its transition partner, the pre-rolling next clip, the live
- * overlays, the warm set — and are never candidates, so a busy frame simply
- * evicts less rather than tearing down something it is about to draw. */
- private evictIdleDecoders() {
- const total = this.videoEls.size + this.overlayEls.size;
- if (total <= DECODER_BUDGET) return;
- const idle = [...this.videoEls.keys(), ...this.overlayEls.keys()]
- .filter((id) => (this.used.get(id) ?? 0) < this.frame)
- .sort((a, b) => (this.used.get(a) ?? 0) - (this.used.get(b) ?? 0));
- for (const id of idle.slice(0, total - DECODER_BUDGET)) this.dropDecoder(id);
- }
-
- /** The cached element for a key, rebuilt when its source no longer matches —
- * a swap keeps the clip but repoints the asset (a shot re-render), and the
- * old decoder would otherwise keep playing (or erroring on) the old file. */
- private elFor(map: Map<string, MediaEl>, key: string, asset: MediaAsset): MediaEl {
- // Asking for an element is what marks it in use — every path that
- // draws, seeks, warms or pre-rolls a clip comes through here, so recency
- // needs no bookkeeping at the call sites.
- this.used.set(key, this.frame);
- let el = map.get(key);
- if (el && el.getAttribute("src") !== asset.url) {
- teardown(el);
- // A repoint is a new source, so it starts with a full rebuild budget.
- this.health.delete(key);
- el = undefined;
- }
- if (el && this.spent(key, el)) {
- teardown(el);
- el = undefined;
- }
- if (!el) {
- el = makeMediaEl(asset);
- map.set(key, el);
- }
- return el;
- }
-
- /** Whether this clip's decoder has stopped making progress and should be
- * replaced. A ready element is healthy and clears its streak; one whose
- * readiness and buffer are both still advancing is on its way there. What is
- * left is an element that errored, or that has sat at the same reading past
- * `DECODER_STALL_MS` — dead either way, and worth one more decoder. Says yes
- * at most `DECODER_REBUILDS` times per source, so an unplayable file stops
- * costing fetches and falls through to the paint-through path. */
- private spent(key: string, el: MediaEl): boolean {
- const now = performance.now();
- const h = this.health.get(key) ?? { mark: -1, at: now, rebuilds: 0 };
- if (elReady(el)) {
- this.health.set(key, { mark: -1, at: now, rebuilds: 0 });
- return false;
- }
- // Progress reading, not a clock: readiness plus how far the buffer reaches,
- // so a big file arriving slowly over a thin link is never mistaken for a
- // wedge and torn down mid-fetch.
- const mark = isImageEl(el)
- ? 0
- : el.readyState + (el.buffered.length ? el.buffered.end(el.buffered.length - 1) : 0);
- if (mark > h.mark) {
- this.health.set(key, { ...h, mark, at: now });
- return false;
- }
- const since = now - h.at;
- // A hidden document defers media loads, so no progress there is no verdict.
- const visible = typeof document === "undefined" || document.visibilityState === "visible";
- const dead = visible && (elErrored(el) ? since > DECODER_RETRY_MS : since > DECODER_STALL_MS);
- if (!dead || h.rebuilds >= DECODER_REBUILDS) {
- this.health.set(key, { ...h, mark });
- return false;
- }
- this.health.set(key, { mark: -1, at: now, rebuilds: h.rebuilds + 1 });
- console.debug(
- `[cut-preview] decoder rebuild: ${key} ` +
- `(${elErrored(el) ? "errored" : "stalled"}, attempt ${h.rebuilds + 1}/${DECODER_REBUILDS})`
- );
- return true;
- }
-
- private videoFor(clip: VideoClip, asset: MediaAsset): MediaEl {
- return this.elFor(this.videoEls, mappingKey(clip, asset), asset);
- }
-
- /** Decode-ahead for track 0: build each soon-to-enter clip's element
- * and seek it to its entrance frame now, so its file is fetching and frame 0
- * is decoded before the playhead arrives. Only clips strictly ahead of `t`
- * are touched — the live master (and any dissolve partner) steers its own
- * clock through `composite`. Bounded by `WARM_HORIZON_S`/`WARM_MAX`; spans are
- * start-ordered, so we can stop once one is past the horizon. */
- private warmAhead(spans: ClipSpan[], t: number, activeKey: string | null) {
- let warmed = 0;
- // One parking seek per element: spans sharing a decoder would otherwise
- // pull its seek head between their targets every tick. Start-ordered, so
- // the nearest entrance wins; the active element is the live picture and
- // is never parked out from under it.
- const parked = new Set<string>();
- for (let i = 0; i < spans.length; i++) {
- const span = spans[i];
- if (span.start <= t) continue; // current or past — not ours to warm
- if (span.start > t + WARM_HORIZON_S) break;
- const key = mappingKey(span.clip, span.asset);
- if (key === activeKey || parked.has(key)) continue;
- parked.add(key);
- const speed = clipSpeed(span.clip);
- // A clip a transition blends in shows its entrance frame for the whole
- // blend window, so it parks exactly there, as early as possible — a
- // seek at window-open would leave the frame undecodable mid-blend, and
- // the push/wipe geometry would carve a black region where the picture
- // belongs. It never pre-rolls: the blend is its warm-up.
- const blended = (spans[i - 1]?.transitionOut ?? 0) > 0;
- const lead = blended ? 0 : prerollLead(span.clip.in, speed);
- // The imminent clip inside its own pre-roll window is `warmNext`'s to
- // play hot; a parking seek here would fight it, so leave it alone. A clip
- // with no roll to give (an untrimmed one) has no such window, and stays
- // parked here right up to the cut.
- if (lead > 0 && span.start - t <= lead) continue;
- const el = this.videoFor(span.clip, span.asset); // creating it starts the fetch
- // Park a not-yet-imminent clip exactly where the pre-roll will play from
- // (`warmNext`'s `from`), not on its entrance frame: parking at `in` and
- // then seeking back for the roll-in discards the warmed buffer, and over
- // the network (cloud media) that late re-seek is what used to stall the
- // handoff. Parked here, the pre-roll's seek is a no-op on already-buffered
- // bytes and the cut lands hot.
- if (!isImageEl(el) && !el.seeking && el.paused) {
- const target = span.clip.in - lead * speed;
- if (Math.abs(el.currentTime - target) > 0.1) el.currentTime = target;
- }
- if (++warmed >= WARM_MAX) break;
- }
+ if (this.raf) cancelAnimationFrame(this.raf);
+ this.unsubscribe();
+ this.unwatch();
+ this.sizeWatch.disconnect();
+ this.pool.closeAll();
+ this.mixer.dispose();
+ }
+
+ /**
+ * The height sources decode at: the canvas, rounded up to a coarse step.
+ *
+ * The pool keys sources on this number. Keyed on the raw canvas height, a
+ * divider drag — which resizes the backing store pixel by pixel — would mint
+ * a fresh decoder for every intermediate size and blow through the tab's
+ * decode slots in a second. Rounding up keeps the identity still while the
+ * box moves and never decodes below display size, and the step is small
+ * enough that the extra rows decoded above it stay cheap.
+ */
+ private decodeHeight(): number {
+ return Math.ceil(this.canvas.height / 180) * 180 || 180;
+ }
+
+ /** The engine's own playhead writes go through here, so `written` always
+ * says what the engine last wrote. */
+ private writeHead(t: number) {
+ this.written = t;
+ setPlayhead(t);
+ }
+
+ /** Something changed: draw at the next opportunity. */
+ private wake() {
+ this.dirty = true;
+ this.schedule();
+ }
+
+ /**
+ * Ask for one frame, and only one.
+ *
+ * Drawing writes the playhead, and writing the playhead wakes the engine, so
+ * a tick that scheduled its own successor unconditionally would leave two
+ * callbacks pending where it meant to leave one — and each of those would
+ * leave two more. Every path books a frame through here, and here refuses to
+ * book a second.
+ */
+ private schedule() {
+ if (this.raf || this.disposed) return;
+ this.raf = requestAnimationFrame(this.tick);
}
- /** Decode-ahead counterpart for overlay tracks: warm each overlay clip whose
- * entrance is within the horizon (overlay clips aren't start-ordered, so this
- * scans rather than breaking early). A warmed element sits paused on its first
- * frame until the tick's overlay path takes it live. */
- private warmOverlaysAhead(t: number) {
+ /** The clip's raw, ungraded decoder frame for analysis (the color panel's
+ * Auto), or null when nothing has been decoded for it. */
+ sourceFor(clipId: string): CanvasImageSource | null {
const s = useEditor.getState();
- // Elements live on screen at `t` — with shared decoders an upcoming clip
- // can ride the same element a playing clip is steering, and a parking
- // seek would yank the live picture.
- const live = new Set<string>();
- for (const c of overlayLayers(s.clips)) {
- if (c.hidden || c.start > t) continue;
- if (t >= c.start + (c.out - c.in) / clipSpeed(c)) continue;
- const asset = s.assets.find((a) => a.id === c.assetId);
- if (asset) live.add(mappingKey(c, asset));
- }
- let warmed = 0;
- const parked = new Set<string>();
- for (const c of overlayLayers(s.clips)) {
- if (c.hidden || c.start <= t || c.start > t + WARM_HORIZON_S) continue;
- const asset = s.assets.find((a) => a.id === c.assetId);
- if (!asset) continue;
- const key = mappingKey(c, asset);
- if (live.has(key) || parked.has(key)) continue;
- parked.add(key);
- const el = this.overlayVideoFor(c, asset);
- if (!isImageEl(el) && el.paused && !el.seeking && Math.abs(el.currentTime - c.in) > 0.1) {
- el.currentTime = c.in;
+ const clip = s.clips.find((c) => c.id === clipId);
+ const asset = clip && s.assets.find((a) => a.id === clip.assetId);
+ if (!clip || !asset) return null;
+ const src = this.pool.get(keyOf(clip, asset), asset, this.decodeHeight());
+ return src.frameAt(sourceTimeOf(clip, previewAt()))?.image ?? null;
+ }
+
+ /**
+ * The picture a clip shows at timeline time `t`.
+ *
+ * Asking is never a wait. A source with the exact frame gives it; one running
+ * behind gives the nearest it has, which is a held frame rather than a hole;
+ * one that has decoded nothing yet is `pending`, which tells the compositor to
+ * leave what is already on screen alone. Painting black for a decoder that is
+ * merely a moment behind is what strobing looks like.
+ */
+ private frameFor(span: ClipSpan, t: number, playing: boolean): Frame {
+ const src = this.pool.get(keyOf(span.clip, span.asset), span.asset, this.decodeHeight());
+ this.used.add(src);
+ const st = sourceTimeOf(span.clip, t);
+ src.want(st, playing);
+ if (src.failed) return MISSING_FRAME;
+ const frame = src.frameAt(st);
+ if (!frame) return PENDING_FRAME;
+ return { kind: "ready", image: frame.image, width: frame.width, height: frame.height };
+ }
+
+ /** Open and start the decoders for clips about to arrive — on track 0 and
+ * the overlay tracks alike — so a cut or an overlay's entrance lands on a
+ * ring that is already full. */
+ private warm(t: number, playing: boolean) {
+ const s = useEditor.getState();
+ const lists = [getClipSpans(s.clips, s.assets)];
+ for (const track of new Set(overlayLayers(s.clips).map((c) => c.track))) {
+ lists.push(getClipSpans(s.clips, s.assets, track));
+ }
+ for (const spans of lists) {
+ for (const sp of spans) {
+ if (sp.start <= t || sp.start > t + WARM_HORIZON_S) continue;
+ const src = this.pool.get(keyOf(sp.clip, sp.asset), sp.asset, this.decodeHeight());
+ // A source already carrying this frame's picture — or already warmed
+ // for a nearer span — needs nothing. Its walk reads across the join on
+ // its own, and a second ask would restart it.
+ if (this.used.has(src)) continue;
+ this.used.add(src);
+ // Start where the clip opens. A walk is the useful thing to have ready —
+ // the first step past a join then costs one frame rather than a seek —
+ // but a walk started for every clip on the horizon spends decode a drag
+ // elsewhere on the timeline is waiting for. So the clip about to be
+ // reached gets a walk, and the ones behind it get the single frame that
+ // lets a cut land on something.
+ const imminent = sp.start - t <= WARM_STREAM_S;
+ src.want(sp.clip.in, playing || imminent);
}
- if (++warmed >= WARM_MAX) break;
}
}
- /** Pre-roll the imminent next clip so its handoff is hot. A decode-ahead
- * element sits paused on its entrance frame with a cold decode pipeline, so
- * the handoff `play()` has to spin the decoder up — and past a trimmed
- * in-point, decode forward from the prior keyframe — before the clock
- * advances: the residual hitch at a cut, and the freeze that parking the
- * element back on its frame never cured. Instead, in the moments before the
- * cut, play the element muted and undrawn from `lead` of source before its
- * entrance, so it is already running across the in-point — arriving there as
- * the playhead reaches the cut — and the real `play()` resumes hot with
- * nothing skipped. Rolling only makes sense while the playhead is moving:
- * paused inside the window, the element parks on the seat, and a roll left
- * over from playback is stopped — an unwatched roll runs past the cut to the
- * end of the file and restarts forever. */
- private warmNext(span: ClipSpan, t: number, play: boolean) {
- const el = this.videoFor(span.clip, span.asset);
- if (isImageEl(el)) return; // a still needs no pipeline
- const speed = clipSpeed(span.clip);
- // Only as long a roll as there is source ahead of the in-point to cover it
- // — nothing at all for an untrimmed clip, which stays parked on its already
- // hot keyframe 0 instead. Farther out than that, it is warmAhead's to park
- // and buffer.
- const lead = prerollLead(span.clip.in, speed);
- if (t < span.start - lead) return;
- const rate = safeRate(speed);
- if (el.playbackRate !== rate) el.playbackRate = rate;
- el.muted = true; // silent until it becomes master and unmutes
- // Seat it `lead` of source before the entrance; playing, it then rolls
- // forward so it reaches `in` right as the playhead reaches the cut.
- const from = span.clip.in - lead * speed;
- if (!play) {
- if (!el.paused) el.pause();
- else if (!el.seeking && Math.abs(el.currentTime - from) > 0.1) el.currentTime = from;
- return;
- }
- // Already rolling: let it run — it crosses `in` on its own as the cut lands.
- if (!el.paused) return;
- if (!el.seeking && Math.abs(el.currentTime - from) > 0.1) el.currentTime = from;
- if (el.readyState >= 2 && !el.seeking) void el.play().catch(() => {});
+ /** Overlay clips live at `t`, with their assets and ramps, in draw order. */
+ private liveOverlays(t: number) {
+ const s = useEditor.getState();
+ return overlayPlan(
+ [...new Set(overlayLayers(s.clips).map((c) => c.track))],
+ (track) => getClipSpans(s.clips, s.assets, track),
+ t
+ );
}
- /** Park a clip's element on its entrance frame, paused and silent — the
- * picture a transition blends in before the clip starts playing. */
- private holdAtEntrance(span: ClipSpan): MediaEl {
- const el = this.videoFor(span.clip, span.asset);
- if (isImageEl(el)) return el;
- el.muted = true;
- if (!el.paused) el.pause();
- else if (!el.seeking && Math.abs(el.currentTime - span.clip.in) > 0.05) {
- el.currentTime = span.clip.in;
- }
- return el;
+ /** Whole-video fade gain at `t`: ramps 0→1 over the project fade-in and 1→0
+ * over the fade-out at the end of the cut. */
+ private projectFadeGain(t: number, total: number) {
+ const s = useEditor.getState();
+ const fadeIn = projectFadeSeconds(s.fadeIn, total);
+ const fadeOut = projectFadeSeconds(s.fadeOut, total);
+ let g = 1;
+ if (fadeIn > 0 && t < fadeIn) g = Math.min(g, Math.max(0, t / fadeIn));
+ if (fadeOut > 0 && t > total - fadeOut) g = Math.min(g, Math.max(0, (total - t) / fadeOut));
+ return Math.min(1, g);
}
- /** The incoming side of a live blend: the entrance frame the window draws,
- * and a decoder made hot for the cut about to land. The element holds
- * parked on its entrance frame only until that frame has been captured
- * once; from there the bitmap carries the blend's picture and the element
- * itself pre-rolls muted toward the in-point (`warmNext`), so the handoff
- * `play()` resumes a decoder that is already running. Holding the element
- * itself for the whole window meant resuming it cold at the cut — and past
- * a trimmed in-point, decoding forward from the prior keyframe — which
- * froze the picture and the playhead right as the transition landed. */
- private blendIncoming(span: ClipSpan, t: number, play: boolean): Frame {
- const el = this.videoFor(span.clip, span.asset);
- if (isImageEl(el)) return this.frameOf(el); // a still is its own held frame
- const key = mappingKey(span.clip, span.asset);
- const src = `${span.asset.url}#${span.clip.in.toFixed(3)}`;
- let held = this.heldEntrance.get(key);
- if (held && held.src !== src) held = undefined;
- if (!held) {
- const parkedOnEntrance =
- elReady(el) && el.paused && !el.seeking && Math.abs(el.currentTime - span.clip.in) <= 0.05;
- if (!parkedOnEntrance) {
- // Nothing to capture yet: hold the element on the frame, as before.
- this.holdAtEntrance(span);
- return this.frameOf(el);
- }
- const canvas = document.createElement("canvas");
- canvas.width = elW(el);
- canvas.height = elH(el);
- canvas.getContext("2d")?.drawImage(el, 0, 0);
- held = { src, canvas };
- this.heldEntrance.set(key, held);
+ /**
+ * Everything audible at `t`, with the gain the frame plan gives it.
+ *
+ * The same ramps that dim the picture dim the sound: a clip fading out of a
+ * dissolve takes its audio with it, an upper-track clip's transition carries
+ * its own, and a live voiceover ducks the rest.
+ */
+ private voicesAt(t: number, spans: ClipSpan[], master: ClipSpan | undefined): Voice[] {
+ const s = useEditor.getState();
+ const out: Voice[] = [];
+ const duck = duckGainAt(s.audioClips, t);
+ if (master && !master.clip.muted && !master.clip.hidden) {
+ const plan = trackZeroPlan(master, spans, t);
+ out.push({
+ id: master.clip.id,
+ url: master.asset.url,
+ start: master.start,
+ in: master.clip.in,
+ out: master.clip.out,
+ speed: clipSpeed(master.clip),
+ gain: plan.gain * duck * (master.clip.volume ?? 1),
+ });
}
- this.warmNext(span, t, play);
- return { kind: "ready", image: held.canvas, width: held.canvas.width, height: held.canvas.height };
- }
-
- /** Seek/rate/play one clip's element toward its frame at timeline time `t`,
- * without touching any other element (the caller pauses stale ones). */
- private prepare(span: ClipSpan, t: number, play: boolean, muted: boolean): MediaEl {
- const el = this.videoFor(span.clip, span.asset);
- // A still never seeks, plays, or carries audio — it's ready as soon as the
- // <img> decodes. Skip every video-clock operation.
- if (isImageEl(el)) return el;
- const speed = clipSpeed(span.clip);
- const rate = safeRate(speed);
- if (el.playbackRate !== rate) el.playbackRate = rate;
- const target = span.clip.in + Math.max(0, t - span.start) * speed;
- // While playing, the element is its own clock and advances on its own, so
- // only re-seek on a real jump (a clip switch or a scrub) — never for the
- // sub-second lag between this frame's `target` (built from last frame's
- // clock read) and the freely-running element. At high speed that lag is
- // `speed × frameInterval` every frame, which a tight threshold would seek
- // backward each tick, stalling playback. When paused (scrubbing) keep it
- // tight so the frame under the mouse tracks precisely.
- // Let an in-flight seek finish before issuing the next one — restarting
- // the decoder every mousemove makes scrubbing stutter.
- const tol = play ? 0.34 : 0.05;
- if (Math.abs(el.currentTime - target) > tol && !el.seeking) el.currentTime = target;
- el.muted = muted;
- if (play) {
- if (el.paused && el.readyState >= 2) void el.play().catch(() => {});
- } else if (!el.paused) {
- el.pause();
+ for (const { clip, asset, gain } of this.liveOverlays(t)) {
+ if (clip.muted || clip.hidden || asset.type === "image") continue;
+ out.push({
+ id: clip.id,
+ url: asset.url,
+ start: clip.start,
+ in: clip.in,
+ out: clip.out,
+ speed: clipSpeed(clip),
+ gain: gain * (clip.volume ?? 1) * duck,
+ });
}
- return el;
- }
-
- private pauseExcept(keep: Set<string>) {
- for (const [clipId, el] of this.videoEls) {
- if (!keep.has(clipId)) pauseEl(el);
+ for (const a of s.audioClips) {
+ const asset = s.assets.find((x) => x.id === a.assetId);
+ if (!asset || a.hidden) continue;
+ const speed = a.speed && a.speed > 0 ? a.speed : 1;
+ const len = Math.max(0.1, (a.out - a.in) / speed);
+ if (t < a.start || t >= a.start + len) continue;
+ // Fade envelope: linear ramps at either end of the clip.
+ const rel = t - a.start;
+ const fi = a.fadeIn ?? 0;
+ const fo = a.fadeOut ?? 0;
+ let g = 1;
+ if (fi > 0 && rel < fi) g *= rel / fi;
+ if (fo > 0 && rel > len - fo) g *= Math.max(0, (len - rel) / fo);
+ // A ducking voiceover never ducks itself, or the others that duck.
+ const dg = a.duck !== undefined && a.duck < 1 ? 1 : duck;
+ out.push({
+ id: a.id,
+ url: asset.url,
+ start: a.start,
+ in: a.in,
+ out: a.out,
+ speed,
+ gain: a.volume * g * dg,
+ });
}
+ return out;
}
- /** Draw a video element into a sub-region of the frame. "fill" covers the
- * region and crops the overflow (clipped to the rect); "fit" contains the
- * whole picture inside it, centered. `zoom` scales the picture around the
- * region's center (zoom transitions), clipping the overflow to the rect. */
- /** The clip's raw, ungraded decoder frame for analysis (the color panel's
- * Auto), or null when no ready decoder exists for the clip. */
- sourceFor(clipId: string): CanvasImageSource | null {
- const s = useEditor.getState();
- const clip = s.clips.find((c) => c.id === clipId);
- const asset = clip && s.assets.find((a) => a.id === clip.assetId);
- if (!clip || !asset) return null;
- const key = mappingKey(clip, asset);
- const el = this.videoEls.get(key) ?? this.overlayEls.get(key);
- return el && elReady(el) ? el : null;
- }
-
- /** The picture a media element is currently showing, in the form the
- * compositor takes. An element that exists but has no decodable frame is
- * `pending`, which is what keeps a skim from strobing black. */
- private frameOf(el: MediaEl | null | undefined): Frame {
- if (!el) return MISSING_FRAME;
- if (!elReady(el)) return PENDING_FRAME;
- return { kind: "ready", image: el, width: elW(el), height: elH(el) };
- }
-
- /** Draw `masterSpan` full-frame, plus any transition live at time `t` in
- * its style's geometry (blend, dip, push, wipe, shape reveal), plus the
- * clips' own entrance/exit animations. Returns the master element (the
- * playback clock). */
- private composite(masterSpan: ClipSpan, spans: ClipSpan[], t: number, play: boolean) {
- // A hidden clip is silent as well as black.
- // What the frame should be at `t` — the same answer an export gets.
- const plan = trackZeroPlan(masterSpan, spans, t);
- // A hidden clip is silent as well as black.
- const masterEl = this.prepare(masterSpan, t, play, masterSpan.clip.muted || !!masterSpan.clip.hidden);
- const masterKey = mappingKey(masterSpan.clip, masterSpan.asset);
- const keep = new Set([masterKey]);
- // Prime the next clip's decoder+audio pipeline shortly before its entrance
- // (the dissolve start, or the hard cut) so the handoff `play()` resumes hot
- // — no cold-start spin-up freezing the picture and playhead at the cut.
- // A clip sharing the master's element (a split of the same source) needs
- // neither: the element crosses the cut already playing.
- if (plan.upcoming && plan.upcoming !== plan.incoming) {
- const upKey = mappingKey(plan.upcoming.clip, plan.upcoming.asset);
- if (upKey !== masterKey) {
- if (masterSpan.transitionOut > 0) this.holdAtEntrance(plan.upcoming);
- else this.warmNext(plan.upcoming, t, play);
- keep.add(upKey);
- }
- }
- // The incoming side of a live blend. Each trim owns its element, so the
- // two sides decode side by side — a true blend even between trims of the
- // same source. A split sharing the master's mapping draws the master's own
- // frames (the pictures are identical by construction); everything else
- // shows its captured entrance frame while its element pre-rolls for the
- // handoff (`blendIncoming`).
- let incFrame: Frame = MISSING_FRAME;
- if (plan.incoming) {
- const incKey = mappingKey(plan.incoming.clip, plan.incoming.asset);
- if (incKey === masterKey) {
- incFrame = this.frameOf(masterEl);
- } else {
- incFrame = this.blendIncoming(plan.incoming, t, play);
- keep.add(incKey);
- }
- }
- // A live voiceover ducks the master clip's sound under it. The clip's own
- // volume rides on top (the element clamps at 1; export honors up to 1.5).
- const duck = duckGainAt(useEditor.getState().audioClips, t);
- if (!isImageEl(masterEl)) {
- masterEl.volume = Math.max(
- 0,
- Math.min(1, plan.gain * duck * (masterSpan.clip.volume ?? 1))
+ /** Draw track 0 at `t`: the master clip, whatever is blending into it, the
+ * neighbour frame behind an edge animation, and the clip's own veil. */
+ private drawTrackZero(
+ master: ClipSpan,
+ spans: ClipSpan[],
+ t: number,
+ playing: boolean,
+ masterFrame: Frame
+ ) {
+ const plan = trackZeroPlan(master, spans, t);
+ // The incoming side of a live blend decodes alongside the outgoing one, so
+ // a dissolve blends two real pictures — including between two trims of the
+ // same file, which keep their own sources by construction.
+ const incFrame = plan.incoming
+ ? this.frameFor(plan.incoming, Math.max(plan.incoming.start, t), playing)
+ : MISSING_FRAME;
+ if (plan.backdrop) {
+ // A neighbour's held frame behind a live edge animation, drawn at the
+ // exact source moment the plan asks for.
+ const b = plan.backdrop;
+ const src = this.pool.get(keyOf(b.span.clip, b.span.asset), b.span.asset, this.decodeHeight());
+ this.used.add(src);
+ src.want(b.at, false);
+ const f = src.frameAt(b.at);
+ this.comp.drawLayer(
+ f ? { kind: "ready", image: f.image, width: f.width, height: f.height } : PENDING_FRAME,
+ b.span.clip,
+ false,
+ 1,
+ t
);
}
- this.pauseExcept(keep);
- // No clear here — the tick clears once before compositing, so a regioned
- // clip draws into its rect over the already-black frame.
- if (plan.backdrop) this.drawBackdropFrame(plan.backdrop.span, plan.backdrop.at, t);
this.comp.drawCrossJoin(
plan.style,
plan.p,
{
- masterFrame: this.frameOf(masterEl),
- masterClip: masterSpan.clip,
+ masterFrame,
+ masterClip: master.clip,
masterAlpha: plan.masterAlpha,
masterZoom: plan.masterZoom,
masterFx: {
@@ -597,506 +340,164 @@ class Engine {
t
);
// Veil only the master clip's own footprint, like the export's per-clip
- // fade filter: a regioned clip darkens inside its rect while a track
- // behind shows through the margins; tracks drawn after (above) stay lit.
- if (plan.veil > 0) this.comp.fillBlackVeil(plan.veil, rectOf(masterSpan.clip));
- return masterEl;
- }
-
- private overlayVideoFor(clip: VideoClip, asset: MediaAsset): MediaEl {
- return this.elFor(this.overlayEls, mappingKey(clip, asset), asset);
- }
-
- // The subject-mask pass: behind-tagged elements leave the DOM overlay path
- // and composite here — video, then the element raster, then the segmented
- // person back on top — and the pass publishes the matte the DOM's front
- // subject-masked elements read (see behindPass.ts). Cheap no-op when
- // nothing reads the matte. Subject-masked video clips pull their matte
- // mid-stack through the compositor's provider, reading the canvas as it
- // stands beneath them.
- private behind = new SubjectMaskCompositor(true);
-
- private drawBehind(t: number) {
- const s = useEditor.getState();
- this.comp.subjectMatteProvider = (at) => this.behind.clipMatteOf(this.canvas, at);
- this.behind.draw(this.canvas, s.overlays, s.assets, t);
- }
-
- // Effect elements are not a canvas pass: they filter the finished picture,
- // elements included, so the stage applies them over everything (see
- // StageEffects) exactly as the export does.
-
- /** Draw a neighbor clip's held frame full-frame beneath an edge animation.
- * A paused element parks on the wanted frame (the previous clip already
- * rests at its out point after the handoff; warm-ahead parks the next one
- * on its entrance frame); a pre-rolling element draws where it is — frames
- * moments ahead of its entrance, close enough at the cut. Until the
- * element has a decodable frame this draws nothing and the tick's black
- * clear shows, same as before. */
- private drawBackdropFrame(span: ClipSpan, at: number, t: number) {
- const el = this.videoFor(span.clip, span.asset);
- if (!isImageEl(el) && el.paused && !el.seeking && Math.abs(el.currentTime - at) > 0.15) {
- el.currentTime = Math.max(0, at);
- }
- this.comp.drawLayer(this.frameOf(el), span.clip, false, 1, t);
- }
-
- /** Whole-video fade gain at time `t`: ramps 0→1 over the project fade-in and
- * 1→0 over the fade-out at the end of the cut. 1 when neither applies. */
- private projectFadeGain(t: number, total: number) {
- const s = useEditor.getState();
- const fadeIn = projectFadeSeconds(s.fadeIn, total);
- const fadeOut = projectFadeSeconds(s.fadeOut, total);
- let g = 1;
- if (fadeIn > 0 && t < fadeIn) g = Math.min(g, Math.max(0, t / fadeIn));
- if (fadeOut > 0 && t > total - fadeOut) g = Math.min(g, Math.max(0, (total - t) / fadeOut));
- return Math.min(1, g);
- }
-
- /** Overlay clips live at time `t`, with their assets and transition ramps,
- * in draw order. */
- private liveOverlays(t: number) {
- const s = useEditor.getState();
- return overlayPlan(
- [...new Set(overlayLayers(s.clips).map((c) => c.track))],
- (track) => getClipSpans(s.clips, s.assets, track),
- t
- );
- }
-
- /** Seek/rate/play one overlay clip's element toward its frame at timeline
- * time `t` (the overlay counterpart of `prepare`). `gain` carries the
- * clip's transition ramp, so a fading picture takes its sound with it. */
- private prepareOverlay(clip: VideoClip, asset: MediaAsset, t: number, play: boolean, gain = 1): MediaEl {
- const el = this.overlayVideoFor(clip, asset);
- if (isImageEl(el)) return el;
- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;
- const rate = safeRate(speed);
- if (el.playbackRate !== rate) el.playbackRate = rate;
- const target = clip.in + Math.max(0, t - clip.start) * speed;
- const tol = play ? 0.34 : 0.05;
- if (Math.abs(el.currentTime - target) > tol && !el.seeking) el.currentTime = target;
- // Overlay audio previews like the export mixes it: the clip's own volume,
- // ducked under a live voiceover, silent when muted. (The tick dims it
- // further with the project fade.)
- el.muted = !!clip.muted;
- el.volume = Math.max(
- 0,
- Math.min(1, gain * (clip.volume ?? 1) * duckGainAt(useEditor.getState().audioClips, t))
- );
- if (play) {
- if (el.paused && el.readyState >= 2) void el.play().catch(() => {});
- } else if (!el.paused) {
- el.pause();
- }
- return el;
- }
-
- /** The live overlays, each already primed toward `t`. Computed once so the
- * skim path's readiness check and the draw step share the same
- * filter/sort/seek instead of repeating it per frame. */
- private prepareOverlays(t: number, play: boolean) {
- return this.liveOverlays(t).map(({ clip, asset, alpha, zoom, gain }) => ({
- clip,
- key: mappingKey(clip, asset),
- alpha,
- zoom,
- el: this.prepareOverlay(clip, asset, t, play, gain),
- }));
- }
-
- /** Draw the overlay tracks over track 0 in z-order (further-back first). A
- * full-frame clip covers what's under it; a regioned one shares the frame,
- * letting lower tracks show in its margins. Collects the clips it touched
- * into `active`. Pass `prepared` (from `prepareOverlays`) to reuse an
- * already-primed set. */
- private drawOverlays(
- t: number,
- play: boolean,
- active: Set<string>,
- prepared?: { clip: VideoClip; key: string; el: MediaEl; alpha: number; zoom: number }[]
- ) {
- for (const { clip, key, el, alpha, zoom } of prepared ?? this.prepareOverlays(t, play)) {
- active.add(key);
- if (!elReady(el)) continue;
+ // fade filter: a regioned clip darkens inside its rect while a track behind
+ // shows through the margins.
+ if (plan.veil > 0) this.comp.fillBlackVeil(plan.veil, rectOf(master.clip));
+ }
+
+ /** Draw the overlay tracks over track 0, further-back first. */
+ private drawOverlays(t: number, playing: boolean) {
+ for (const { clip, asset, alpha, zoom } of this.liveOverlays(t)) {
+ const span: ClipSpan = {
+ clip,
+ asset,
+ start: clip.start,
+ len: Math.max(0.1, (clip.out - clip.in) / clipSpeed(clip)),
+ transitionOut: 0,
+ };
+ const frame = this.frameFor(span, t, playing);
+ if (frame.kind !== "ready") continue;
const rect = rectOf(clip);
const cover = clip.fit === "fill" || (clip.fit == null && isFullRect(rect));
- this.comp.drawIntoRect(this.frameOf(el), rect, cover, alpha, t, zoom, clip);
- }
- }
-
- /** Pause overlay elements not drawn this frame; drop those no clip needs. */
- private cleanupOverlays(active: Set<string>) {
- const s = useEditor.getState();
- const live = new Set<string>();
- for (const c of overlayLayers(s.clips)) {
- const asset = s.assets.find((a) => a.id === c.assetId);
- if (asset) live.add(mappingKey(c, asset));
- }
- for (const [key, el] of this.overlayEls) {
- if (active.has(key)) continue;
- pauseEl(el);
- if (!live.has(key)) this.dropDecoder(key);
+ this.comp.drawIntoRect(frame, rect, cover, alpha, t, zoom, clip);
}
}
- private syncSoundtrack(t: number, playing: boolean, fadeGain = 1) {
- const s = useEditor.getState();
- const live = new Set<string>();
- for (const a of s.audioClips) {
- live.add(a.id);
- const asset = s.assets.find((x) => x.id === a.assetId);
- if (!asset) continue;
- let el = this.audioEls.get(a.id);
- // Same-id source swaps (a regenerated voiceover) rebuild the element.
- if (el && el.getAttribute("src") !== asset.url) {
- el.pause();
- el = undefined;
- }
- if (!el) {
- const audio = new Audio();
- audio.preload = "auto";
- audio.addEventListener("error", () => reportMediaElementError(audio));
- audio.src = asset.url;
- el = audio;
- this.audioEls.set(a.id, el);
- }
- // Detached audio can carry its video clip's rate; footprint is (out-in)/speed.
- const speed = a.speed && a.speed > 0 ? a.speed : 1;
- const len = Math.max(0.1, (a.out - a.in) / speed);
- // A hidden clip is muted from the mix — keep its element but never play it.
- const active = playing && !a.hidden && t >= a.start && t < a.start + len;
- if (active) {
- // Fade envelope: linear ramps at either end of the clip.
- const rel = t - a.start;
- const fi = a.fadeIn ?? 0;
- const fo = a.fadeOut ?? 0;
- let gain = 1;
- if (fi > 0 && rel < fi) gain *= rel / fi;
- if (fo > 0 && rel > len - fo) gain *= Math.max(0, (len - rel) / fo);
- // A live voiceover ducks the other soundtrack clips (music) too;
- // ducking clips never duck each other.
- const dg = a.duck !== undefined && a.duck < 1 ? 1 : duckGainAt(s.audioClips, t);
- el.volume = Math.max(0, Math.min(1, a.volume * gain * dg * fadeGain));
- const rate = safeRate(speed);
- if (el.playbackRate !== rate) el.playbackRate = rate;
- const expected = a.in + rel * speed;
- if (Math.abs(el.currentTime - expected) > 0.25) el.currentTime = expected;
- if (el.paused) void el.play().catch(() => {});
- } else if (!el.paused) {
- el.pause();
- }
- }
- for (const [id, el] of this.audioEls) {
- if (!live.has(id)) {
- el.pause();
- this.audioEls.delete(id);
- }
- }
- }
-
- // The last tick's wall-clock stamp, for detecting gaps in the tick stream.
- private lastTickAt = 0;
-
private tick() {
+ this.raf = 0;
if (this.disposed) return;
- this.raf = requestAnimationFrame(this.tick);
- const now = performance.now();
- // The stall clock must count only time the engine was alive. rAF freezes
- // with the tab hidden while `health.at` stamps keep aging — and Chrome
- // defers a hidden tab's media loads, so a decoder created near the switch
- // is still cold on return. The first tick back would then read every such
- // decoder as minutes past the stall limit and tear it down, killing the
- // very load Chrome had just released. A gap in the tick stream restarts
- // each decoder's stall window instead.
- if (this.lastTickAt && now - this.lastTickAt > 1_000) {
- for (const h of this.health.values()) h.at = now;
- }
- this.lastTickAt = now;
- this.frame++;
- this.render();
- // After the frame, never during it: what `render` touched is exactly what
- // must survive, and it takes every early return there is to say so.
- this.evictIdleDecoders();
- this.diagPulse();
- }
-
- // Diagnostic heartbeat. The decoder pool is invisible from DevTools — the
- // elements are detached — so every ten seconds the engine prints each live
- // decoder's clock, readiness and buffer. A decoder found playing while the
- // project is paused is flagged loudly: every paused-path element is supposed
- // to be parked, so one still running marks a leak. Filter the console on
- // "cut-preview".
- private lastDiagAt = 0;
-
- private diagPulse() {
- const now = performance.now();
- if (now - this.lastDiagAt < 10_000) return;
- this.lastDiagAt = now;
+ markTick();
const playing = useEditor.getState().playing;
- const rows: string[] = [];
- const scan = (map: Map<string, MediaEl>, kind: string) => {
- for (const [id, el] of map) {
- if (isImageEl(el)) {
- rows.push(`${kind} ${id} img`);
- continue;
- }
- const buf = el.buffered.length ? el.buffered.end(el.buffered.length - 1).toFixed(1) : "0";
- rows.push(
- `${kind} ${id} ct=${el.currentTime.toFixed(2)} rs=${el.readyState} buf=${buf}` +
- `${el.paused ? "" : " PLAYING"}${el.error ? ` err=${el.error.code}` : ""}`
- );
- if (!playing && !el.paused) {
- console.warn(`[cut-preview] decoder playing while paused: clip ${id} at ${el.currentTime.toFixed(2)}s`);
- }
- }
- };
- scan(this.videoEls, "track0");
- scan(this.overlayEls, "overlay");
- console.debug(
- `[cut-preview] playing=${playing} decoders=${this.videoEls.size + this.overlayEls.size} ` +
- `audio=${this.audioEls.size}\n` + rows.join("\n")
- );
+ // Playing redraws every frame; paused, only what changed since the last
+ // paint. Either way the loop stops as soon as there is nothing to do.
+ if (playing || this.dirty) {
+ this.dirty = false;
+ this.render(playing);
+ }
+ if (playing || this.dirty) this.schedule();
}
- private render() {
+ private render(playing: boolean) {
const s = useEditor.getState();
const spans = getClipSpans(s.clips, s.assets);
- // Drop decoders no clip needs anymore (deleted or replaced). Every edit
- // rewrites the clip array, so its identity changing is the signal to
- // re-check — a moving playhead leaves it alone, and the scan never runs
- // per frame.
- if (this.clipsSeen !== s.clips) {
- this.clipsSeen = s.clips;
- const live = new Set<string>();
- for (const c of s.clips) {
- const asset = s.assets.find((a) => a.id === c.assetId);
- if (asset) live.add(mappingKey(c, asset));
- }
- for (const key of [...this.videoEls.keys(), ...this.overlayEls.keys()]) {
- if (!live.has(key)) this.dropDecoder(key);
+ const total = projectDuration(s);
+ this.pool.beginFrame();
+ this.used.clear();
+
+ // The clock, before anything that can return early.
+ //
+ // Whether the mixer is running *is* whether the cut is playing — there is no
+ // second flag to fall out of step with it. Kept as one, a frame that
+ // returned early (a clip still opening, an empty timeline) could leave the
+ // engine believing playback had already started while the mixer had stopped,
+ // and the next play would read a clock that was never anchored: the picture
+ // would sit at zero for the length of the cut.
+ //
+ // The playhead can also move while the clock runs — an arrow-key skip, the
+ // assistant's seek, a preview range starting mid-play. The engine writes
+ // the playhead itself every playing frame, so a value it did not write is
+ // someone seeking, and the clock re-anchors there rather than snapping the
+ // playhead back.
+ let t: number;
+ if (playing) {
+ const head = playheadAt();
+ if (!this.mixer.running || (this.written !== null && head !== this.written)) {
+ this.mixer.start(Math.min(head, total));
+ this.written = head;
}
+ t = Math.max(0, Math.min(this.mixer.now(), total));
+ } else {
+ // Pinned just inside the end: a preview time at exactly `total` lies past
+ // the final span, and a skim off the right edge of the cut should hold
+ // the last frame rather than clear to black.
+ const end = Math.max(0, total - 0.001);
+ t = Math.max(0, Math.min(previewAt(), end));
+ if (this.mixer.running) this.mixer.stop();
}
- // Whole-project length so time past track 0's end (a longer video track or
- // soundtrack) is still reachable while scrubbing and playing.
- const total = projectDuration(s);
- // Nothing anywhere — no track-0 clip, no overlay layer, no soundtrack, no
- // title or sticker — resets to a black frame at 0. An empty track 0 with
- // any one of those still plays: the tick body draws those layers and
- // advances the wall clock, so the guard must not bail on
- // `spans.length === 0` alone.
+ // Nothing anywhere resets to a black frame at 0.
if (
spans.length === 0 &&
overlayLayers(s.clips).length === 0 &&
s.audioClips.length === 0 &&
s.overlays.length === 0
) {
- this.pauseExcept(new Set());
+ this.mixer.stop();
this.comp.drawLayer(MISSING_FRAME, undefined, true, 1, 0);
- this.syncSoundtrack(0, false);
- if (s.playing) useEditor.setState({ playing: false, currentTime: 0 });
- return;
- }
-
- let t = Math.min(s.currentTime, total);
-
- // Skimming: while paused with the mouse over the timeline, the
- // frame on screen lives at the skim point, not the playhead. The playhead
- // (currentTime) is never touched.
- const pt =
- !s.playing && s.skimTime !== null
- ? Math.max(0, Math.min(s.skimTime, total - 0.001))
- : t;
-
- // Keep the next few clips decoded and buffering ahead of the frame being
- // shown (the skim point while skimming, else the playhead — paused too, so
- // pressing play resumes clean). Runs before either branch since both
- // benefit; warms only clips ahead of the anchor, so it never touches the
- // element the branches are about to drive — anchored at the playhead while
- // skimming, its entrance-frame parking seeks would fight the skimmed
- // clip's own scrub seeks every tick and freeze the preview on any clip
- // ahead of the playhead. The span live at the anchor names the element
- // that must not be parked out from under the picture (a shared decoder).
- const anchorSpan = spans.find((sp) => pt >= sp.start && pt < sp.start + sp.len);
- this.warmAhead(spans, pt, anchorSpan ? mappingKey(anchorSpan.clip, anchorSpan.asset) : null);
- this.warmOverlaysAhead(pt);
-
- if (!s.playing) {
- // Not advancing: drop the wall-clock stamp so the first playing tick
- // starts a fresh delta instead of leaping over the paused stretch.
- this.lastPlayNow = 0;
- const span = anchorSpan;
- // Prime every layer live at `pt` — the track-0 element and each overlay
- // track — before repainting (create them, issue any seeks). A cold
- // element or an unbuffered seek has no decodable frame yet, and painting
- // around it tears the composite: black before the track-0 seek resolves,
- // or track 0 flashing through where an overlay covers it. Hold the last
- // painted frame until every live layer has a frame, so each scrubbed
- // frame is the same composite playback and export show.
- let ready = true;
- if (span) {
- const el = this.prepare(span, Math.min(pt, span.start + span.len), false, true);
- // A broken source never becomes ready; paint without it rather than
- // wedging the preview on it.
- if (!elErrored(el) && !elReady(el)) ready = false;
- }
- // Prime the overlays once; the readiness scan and the draw step below
- // reuse these instead of re-filtering/seeking them a second time.
- const overlaysLive = this.prepareOverlays(pt, false);
- for (const { el } of overlaysLive) {
- if (!elErrored(el) && !elReady(el)) ready = false;
- }
- if (!ready) {
- this.pauseExcept(new Set(span ? [mappingKey(span.clip, span.asset)] : []));
- this.syncSoundtrack(t, false);
- return;
+ if (s.playing) {
+ useEditor.setState({ playing: false });
+ this.writeHead(0);
}
- const active = new Set<string>();
- this.comp.clear();
- // Where track 0 has nothing live there is no master frame — just black
- // and the other tracks still running at `pt`.
- if (span) this.composite(span, spans, Math.min(pt, span.start + span.len), false);
- else this.pauseExcept(new Set());
- this.drawOverlays(pt, false, active, overlaysLive);
- this.drawBehind(pt);
- this.comp.drawProjectFade(this.projectFadeGain(pt, total));
- this.cleanupOverlays(active);
- this.syncSoundtrack(t, false);
return;
}
- let span = anchorSpan;
+ const master = spans.find((sp) => t >= sp.start && t < sp.start + sp.len);
- // A just-started or just-scrubbed clip may still be decoding. Hold the last
- // painted frame rather than clearing to black — same as the skim path. (At a
- // dissolve boundary the incoming clip is already warm from the overlap, so
- // this only bites a genuinely cold first frame.)
- if (span) {
- const el = this.prepare(span, t, true, span.clip.muted || !!span.clip.hidden);
- // A broken source (unreachable still, decode error) never becomes ready;
- // fall through so the wall clock advances past it instead of freezing.
- if (!elReady(el) && !elErrored(el)) {
- this.pauseExcept(new Set([mappingKey(span.clip, span.asset)]));
- this.syncSoundtrack(t, true);
- return;
- }
+ // Ask for the master's picture before clearing. A clip nothing has decoded
+ // yet answers `pending`, and clearing on the strength of that would black
+ // the frame out for as long as the file takes to open — the strobe this
+ // whole design exists to remove. Leave what is on screen, stay dirty, and
+ // paint when there is something to paint.
+ const masterFrame = master ? this.frameFor(master, t, playing) : MISSING_FRAME;
+ if (masterFrame.kind === "pending") {
+ this.warm(t, playing);
+ this.dirty = true;
+ return;
}
- // Clear to black, then prime the master element (and any dissolve
- // partner) over it and read the clock.
- const active = new Set<string>();
this.comp.clear();
+ if (master) this.drawTrackZero(master, spans, t, playing, masterFrame);
+ this.drawOverlays(t, playing);
+ // The subject-mask pass reads the canvas as it stands beneath it, then
+ // publishes the matte the DOM's front subject-masked elements read.
+ this.comp.subjectMatteProvider = (at) => this.behind.clipMatteOf(this.canvas, at);
+ this.behind.draw(this.canvas, s.overlays, s.assets, t);
+ const fadeGain = this.projectFadeGain(t, total);
+ this.comp.drawProjectFade(fadeGain);
- if (span) {
- let el = this.composite(span, spans, t, true);
- // The element clock is the truth but it's coarse: currentTime advances in
- // steps bigger than a frame, and copying it straight to the playhead
- // makes the indicator stutter and step backward. Advance by wall clock
- // instead — smooth by construction and never backward — and let the
- // element clock steer it: the playhead may run at most 60ms ahead of the
- // clock (a stalled element halts it with the picture) and snaps forward
- // only when the clock genuinely leads.
- const now = performance.now();
- const dt = this.lastPlayNow ? Math.min(0.25, (now - this.lastPlayNow) / 1000) : 0;
- let atEnd: boolean;
- if (isImageEl(el) || elErrored(el)) {
- // A still has no element clock, and a broken source's clock never
- // advances (steering by it would pin the playhead at the clip start) —
- // move purely by wall clock and end at the clip's timeline footprint.
- t = Math.max(span.start, Math.min(t + dt, span.start + span.len));
- atEnd = t >= span.start + span.len - 0.0001;
+ // Decoders for what is about to arrive, and the ones nothing needs closed.
+ this.warm(t, playing);
+ this.pool.evict();
+
+ if (playing) {
+ this.mixer.setMasterGain(fadeGain);
+ this.mixer.update(t, this.voicesAt(t, spans, master));
+ // A scoped effect preview auto-pauses at its stop mark; the end of the
+ // cut stops playback outright.
+ const stop = s.previewStopAt;
+ if (stop != null && t >= stop) {
+ useEditor.setState({ playing: false, previewStopAt: null });
+ this.writeHead(Math.min(t, stop));
+ } else if (t >= total - 0.001) {
+ useEditor.setState({ playing: false, previewStopAt: null });
+ this.writeHead(total);
} else {
- const speed = clipSpeed(span.clip);
- const derived = span.start + (el.currentTime - span.clip.in) / speed;
- const cand = Math.min(t + dt, derived + 0.06);
- t = derived - cand > 0.25 ? derived : Math.max(t, cand);
- t = Math.max(span.start, Math.min(t, span.start + span.len));
- atEnd = el.currentTime >= span.clip.out - 0.02 || el.ended;
- }
- // Clip boundary: hand off to the next clip when it abuts (or dissolves),
- // fall into the gap when it doesn't — track 0 plays black there and the
- // wall clock advances — or (if track 0 is done but another track runs
- // on) fall through to the wall-clock tail.
- if (atEnd) {
- const idx = spans.indexOf(span);
- const next = spans[idx + 1];
- if (next && next.start <= span.start + span.len + 0.001) {
- // Jump past the finished clip's whole footprint (including any
- // cross-dissolve overlap), not back to next.start — which still sits
- // inside the outgoing clip's footprint, so find() would re-pick the
- // clip we just finished and playback would ping-pong across the
- // dissolve forever.
- t = Math.max(next.start + 0.0001, span.start + span.len);
- span = next;
- el = this.composite(next, spans, t, true);
- } else if (next) {
- // A gap before the next clip: step just past this clip's footprint
- // so the next tick's find() sees no active span and the wall-clock
- // path carries time (and the soundtrack) across the black stretch.
- t = Math.max(t, span.start + span.len + 0.0001);
- pauseEl(el);
- } else if (t >= total - 0.001) {
- // Track 0 and every other track finished.
- useEditor.setState({ playing: false, currentTime: total, previewStopAt: null });
- pauseEl(el);
- this.drawOverlays(t, true, active);
- this.drawBehind(t);
- this.comp.drawProjectFade(this.projectFadeGain(total, total));
- this.cleanupOverlays(active);
- this.syncSoundtrack(total, false);
- return;
- }
- }
- this.lastPlayNow = now;
- } else {
- // Nothing live on track 0 but another track is still playing: no master
- // element, so advance time by the wall clock and let the overlays follow.
- this.pauseExcept(new Set());
- const now = performance.now();
- const dt = this.lastPlayNow ? Math.min(0.25, (now - this.lastPlayNow) / 1000) : 0;
- this.lastPlayNow = now;
- t = t + dt;
- if (t >= total - 0.001) {
- useEditor.setState({ playing: false, currentTime: total, previewStopAt: null });
- this.drawOverlays(t, true, active);
- this.drawBehind(t);
- this.comp.drawProjectFade(this.projectFadeGain(total, total));
- this.cleanupOverlays(active);
- this.syncSoundtrack(total, false);
- return;
+ this.writeHead(t);
}
}
- this.drawOverlays(t, true, active);
- this.drawBehind(t);
- // The whole-video fade veils the finished frame and dims the sound —
- // the master's element volume (set by composite) and the soundtrack.
- const fadeGain = this.projectFadeGain(t, total);
- if (fadeGain < 1) {
- if (span) {
- const mel = this.videoEls.get(mappingKey(span.clip, span.asset));
- if (mel && !isImageEl(mel)) mel.volume = Math.min(mel.volume, fadeGain);
- }
- for (const el of this.overlayEls.values()) {
- if (!isImageEl(el)) el.volume = Math.min(el.volume, fadeGain);
+ if (tracing()) {
+ let srcTs: number | null = null;
+ let exact = true;
+ if (master) {
+ const src = this.pool.get(
+ keyOf(master.clip, master.asset),
+ master.asset,
+ this.decodeHeight()
+ );
+ const st = sourceTimeOf(master.clip, t);
+ srcTs = src.frameAt(st)?.timestamp ?? null;
+ exact = src.hasExact(st);
}
- }
- this.comp.drawProjectFade(fadeGain);
- this.cleanupOverlays(active);
- // A scoped effect preview auto-pauses at its stop mark (the frame this
- // tick just painted is the stop frame — close enough at tick rate).
- if (s.previewStopAt != null && t >= s.previewStopAt) {
- useEditor.setState({
- playing: false,
- currentTime: Math.min(t, s.previewStopAt),
- previewStopAt: null,
+ markPresent({
+ t,
+ srcTs,
+ clipId: master?.clip.id ?? null,
+ exact,
+ stale: masterFrame.kind !== "ready",
});
- this.syncSoundtrack(Math.min(t, s.previewStopAt), false);
- return;
+ markLiveSamples(this.pool.held);
}
- useEditor.setState({ currentTime: t });
- this.syncSoundtrack(t, true, fadeGain);
}
}
@@ -1105,8 +506,8 @@ export function usePlayback(canvasRef: RefObject<HTMLCanvasElement | null>) {
const canvas = canvasRef.current;
if (!canvas) return;
const engine = new Engine(canvas);
- // Dev-only automation hook, like installDevHooks: lets a headless run (or
- // a debugging session) read the live decoder pool the way diagPulse does.
+ // Dev-only automation hook, like installDevHooks: lets a headless run (or a
+ // debugging session) reach the live engine.
if (process.env.NODE_ENV !== "production") {
(window as unknown as Record<string, unknown>).__cutDevEngine = engine;
}
diff --git a/site/src/cut/lib/effectStack.ts b/site/src/cut/lib/effectStack.ts
index bab4acf8..b2a9f3f0 100644
--- a/site/src/cut/lib/effectStack.ts
+++ b/site/src/cut/lib/effectStack.ts
@@ -11,7 +11,8 @@
* The math is here, apart from React, because both exports walk the same stack.
*/
-import type { EffectPreviewState } from "@donkeycut/effects-kit";
+import { effectPreviewState, type EffectPreviewState } from "@donkeycut/effects-kit";
+import { isEffectOverlay, laneOf, type Overlay } from "./types";
/** One effect live at a moment, with the lane that places it in the stack. */
export interface LiveEffect {
@@ -19,6 +20,23 @@ export interface LiveEffect {
state: EffectPreviewState;
}
+/** The effects playing at `t`, shallowest lane first. */
+export function liveEffectsAt(overlays: Overlay[], t: number): LiveEffect[] {
+ const live: LiveEffect[] = [];
+ for (const o of overlays) {
+ if (!isEffectOverlay(o) || o.hidden || t < o.start || t > o.end) continue;
+ live.push({
+ lane: laneOf(o),
+ state: effectPreviewState(o.effect, o.amount, t - o.start, o.focus, o.ramp, o.end - o.start),
+ });
+ }
+ return live.sort((a, b) => a.lane - b.lane);
+}
+
+/** Whether any effect could be live at all. The stage asks before doing the
+ * per-frame work of finding out which ones are. */
+export const hasEffects = (overlays: Overlay[]) => overlays.some(isEffectOverlay);
+
/** How the content under an effect moves: a zoom's push in, a shake, and the
* horizontal jump that reads as tearing. Empty for the effects that only
* filter. */
@@ -54,7 +72,7 @@ export function stageEffectTransform(states: EffectPreviewState[]): string | und
}
/** The grade of a run of effects, as one CSS filter. */
-const stageEffectFilter = (states: EffectPreviewState[]) =>
+export const stageEffectFilter = (states: EffectPreviewState[]) =>
states
.map((s) => s.cssFilter)
.filter(Boolean)
@@ -68,23 +86,24 @@ export type StageSlice =
/** Elements on lanes in [from, to) — `from` counts down from the top. */
from: number;
to: number;
- filter?: string;
- transform?: string;
+ /** Effects shallower than this lane grade the slice. Null on the top
+ * slice, which nothing grades. */
+ gradeAbove: number | null;
/** The topmost slice carries the captions, which nothing grades. */
captions: boolean;
}
- | { kind: "paint"; key: string; states: EffectPreviewState[] };
+ | { kind: "paint"; key: string; lane: number };
/**
- * The stage bottom to top: what the picture itself wears, then alternating
- * element slices and the paints of the effect that sits over each.
+ * The stage bottom to top: alternating element slices and the paints of the
+ * effect that sits over each.
+ *
+ * The shape depends only on which lanes hold effects, so it holds still while
+ * the playhead moves and changes only when a row does. What each slice *wears*
+ * at this instant is worked out by the slice itself, which is what keeps the
+ * clock out of the component that lays the stage out.
*/
-export function stageSlices(
- live: LiveEffect[],
- lanes: number[]
-): { picture: { filter?: string; transform?: string }; slices: StageSlice[] } {
- const above = (lane: number) => live.filter((e) => e.lane < lane).map((e) => e.state);
- const all = live.map((e) => e.state);
+export function stageSliceStructure(lanes: number[]): StageSlice[] {
const slices: StageSlice[] = [];
// Deepest first: elements under that effect, then its paints over them.
for (let i = lanes.length - 1; i >= 0; i--) {
@@ -94,25 +113,18 @@ export function stageSlices(
key: `band-${lane}`,
from: lane + 1,
to: i + 1 < lanes.length ? lanes[i + 1] : Infinity,
- filter: stageEffectFilter(above(lane + 1)),
- transform: stageEffectTransform(above(lane + 1)),
+ gradeAbove: lane + 1,
captions: false,
});
- slices.push({
- kind: "paint",
- key: `paint-${lane}`,
- states: live.filter((e) => e.lane === lane).map((e) => e.state),
- });
+ slices.push({ kind: "paint", key: `paint-${lane}`, lane });
}
slices.push({
kind: "elements",
key: "band-top",
from: 0,
to: lanes.length ? lanes[0] : Infinity,
+ gradeAbove: null,
captions: true,
});
- return {
- picture: { filter: stageEffectFilter(all), transform: stageEffectTransform(all) },
- slices,
- };
+ return slices;
}
diff --git a/site/src/cut/lib/frameSource.test.ts b/site/src/cut/lib/frameSource.test.ts
new file mode 100644
index 00000000..b8074fd9
--- /dev/null
+++ b/site/src/cut/lib/frameSource.test.ts
@@ -0,0 +1,171 @@
+import { describe, expect, test } from "bun:test";
+import { FrameRing, FrameSourcePool, mappingKey, type Timed } from "./frameSource";
+import type { MediaAsset } from "./types";
+
+/** A decoded frame at 30fps, as the sink would hand it over. */
+const f = (timestamp: number, duration = 1 / 30): Timed => ({ timestamp, duration });
+
+const asset = (id: string): MediaAsset => ({
+ id,
+ name: `${id}.mp4`,
+ fileName: `${id}.mp4`,
+ type: "video",
+ url: `https://example.test/${id}.mp4`,
+ duration: 10,
+});
+
+describe("FrameRing", () => {
+ test("hands back the frame covering the asked-for time", () => {
+ const ring = new FrameRing<Timed>(8);
+ for (let i = 0; i < 5; i++) ring.push(f(i / 30));
+ // Between two frames, the one that started is the one on screen.
+ expect(ring.at(2.5 / 30)?.timestamp).toBeCloseTo(2 / 30);
+ expect(ring.at(2 / 30)?.timestamp).toBeCloseTo(2 / 30);
+ });
+
+ test("answers past the end with the newest frame it holds", () => {
+ // A decoder running behind the playhead should show the last real frame
+ // rather than nothing — this is the "never withhold" rule.
+ const ring = new FrameRing<Timed>(8);
+ ring.push(f(0));
+ ring.push(f(1 / 30));
+ expect(ring.at(5)?.timestamp).toBeCloseTo(1 / 30);
+ });
+
+ test("answers before the first frame with the earliest it holds", () => {
+ // At a clip's head the frame the cut opens on is the right thing to show.
+ const ring = new FrameRing<Timed>(8);
+ ring.push(f(2));
+ ring.push(f(2 + 1 / 30));
+ expect(ring.at(1)?.timestamp).toBeCloseTo(2);
+ });
+
+ test("empty ring has no answer", () => {
+ expect(new FrameRing<Timed>(4).at(0)).toBe(null);
+ });
+
+ test("covers() separates a real frame from a held one", () => {
+ const ring = new FrameRing<Timed>(8);
+ ring.push(f(1));
+ expect(ring.covers(1)).toBe(true);
+ expect(ring.covers(1 + 0.5 / 30)).toBe(true);
+ // Past this frame's own duration nothing is covered, even though `at`
+ // still hands the frame over.
+ expect(ring.covers(1 + 2 / 30)).toBe(false);
+ expect(ring.at(1 + 2 / 30)?.timestamp).toBeCloseTo(1);
+ expect(ring.covers(0.5)).toBe(false);
+ });
+
+ test("drops the oldest past capacity, so the canvas pool never wraps under it", () => {
+ const ring = new FrameRing<Timed>(3);
+ for (let i = 0; i < 6; i++) ring.push(f(i));
+ expect(ring.size).toBe(3);
+ expect(ring.oldest).toBeCloseTo(3);
+ expect(ring.newest).toBeCloseTo(5);
+ // The evicted frames are gone, not merely unreachable.
+ expect(ring.at(0)?.timestamp).toBeCloseTo(3);
+ });
+
+ test("keep() drops what a reader moved away from", () => {
+ const ring = new FrameRing<Timed>(16);
+ for (let i = 0; i < 10; i++) ring.push(f(i));
+ ring.keep(4, 6);
+ expect(ring.size).toBe(3);
+ expect(ring.oldest).toBeCloseTo(4);
+ expect(ring.newest).toBeCloseTo(6);
+ });
+});
+
+describe("mappingKey", () => {
+ test("clips showing identical pictures share a decoder", () => {
+ // A plain split: same file, same speed, and the same source time at
+ // timeline zero. One decoder reads straight across the cut.
+ const left = mappingKey("a", 1, 0, 0);
+ const right = mappingKey("a", 1, 4, 4);
+ expect(left).toBe(right);
+ });
+
+ test("different trims of one file keep their own decoders", () => {
+ // Both live at timeline 4, showing different moments — a same-source
+ // dissolve has to blend two distinct frames.
+ expect(mappingKey("a", 1, 0, 4)).not.toBe(mappingKey("a", 1, 8, 4));
+ });
+
+ test("a speed change is a different mapping", () => {
+ expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("a", 2, 0, 0));
+ });
+
+ test("different files never share", () => {
+ expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("b", 1, 0, 0));
+ });
+});
+
+describe("FrameSourcePool", () => {
+ test("one source per mapping and size, reused across frames", () => {
+ const pool = new FrameSourcePool(4);
+ pool.beginFrame();
+ const a = pool.get("k", asset("a"), 480);
+ const b = pool.get("k", asset("a"), 480);
+ expect(a).toBe(b);
+ // A different preview size is a different decode.
+ expect(pool.get("k", asset("a"), 240)).not.toBe(a);
+ expect(pool.size).toBe(2);
+ });
+
+ test("a source follows its asset's URL", () => {
+ // A signed URL re-mints, or a shot re-renders onto its asset id. The
+ // mapping still names the same pictures, but they now live somewhere else,
+ // so the old source is closed and a fresh one reads the new URL.
+ const pool = new FrameSourcePool(4);
+ pool.beginFrame();
+ const before = pool.get("k", asset("a"), 480);
+ expect(pool.get("k", asset("a"), 480)).toBe(before);
+ const moved = { ...asset("a"), url: "https://example.test/a.mp4?e=2&s=next" };
+ const after = pool.get("k", moved, 480);
+ expect(after).not.toBe(before);
+ expect(after.url).toBe(moved.url);
+ expect(pool.size).toBe(1);
+ });
+
+ test("evicts the least recently asked-for, never this frame's", () => {
+ const pool = new FrameSourcePool(2);
+ pool.beginFrame();
+ const old = pool.get("old", asset("a"), 480);
+ pool.beginFrame();
+ const mid = pool.get("mid", asset("b"), 480);
+ // Let the grace expire on both, then draw a third from a frame of its own.
+ for (let i = 0; i < 200; i++) pool.beginFrame();
+ const live = pool.get("live", asset("c"), 480);
+ expect(pool.size).toBe(3);
+ pool.evict();
+ // Down to the budget, and the one this frame is drawn from survives.
+ expect(pool.size).toBe(2);
+ expect(pool.get("live", asset("c"), 480)).toBe(live);
+ expect(pool.get("mid", asset("b"), 480)).toBe(mid);
+ expect(pool.get("old", asset("a"), 480)).not.toBe(old);
+ });
+
+ test("holds a source through the grace, so a busy cut never thrashes", () => {
+ // A cut with more clips than the budget would otherwise close and reopen
+ // decoders every frame, and reopening costs far more than the memory it
+ // saves. Nothing is closed until it has gone unwanted for a while.
+ const pool = new FrameSourcePool(1);
+ pool.beginFrame();
+ const a = pool.get("a", asset("a"), 480);
+ pool.beginFrame();
+ const b = pool.get("b", asset("b"), 480);
+ pool.evict();
+ expect(pool.size).toBe(2);
+ expect(pool.get("a", asset("a"), 480)).toBe(a);
+ expect(pool.get("b", asset("b"), 480)).toBe(b);
+ });
+
+ test("stays put while under budget", () => {
+ const pool = new FrameSourcePool(4);
+ pool.beginFrame();
+ const a = pool.get("a", asset("a"), 480);
+ pool.beginFrame();
+ pool.evict();
+ expect(pool.get("a", asset("a"), 480)).toBe(a);
+ });
+});
diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.ts
new file mode 100644
index 00000000..6f6da9ce
--- /dev/null
+++ b/site/src/cut/lib/frameSource.ts
@@ -0,0 +1,621 @@
+"use client";
+
+/**
+ * Frames for the preview, decoded straight off the file.
+ *
+ * The preview used to ask an `HTMLVideoElement` for a picture by writing
+ * `currentTime` and waiting: a seek costs a decode from the nearest keyframe,
+ * the element decides when it lands, and a cut meant handing the clock from one
+ * element to another mid-flight. Everything that felt slow about scrubbing and
+ * everything that stuttered at a join came from that one arrangement.
+ *
+ * Here a clip owns a decoder and a small ring of already-decoded frames.
+ * Playing walks the file forward and fills the ring ahead of the playhead;
+ * scrubbing asks for a single time and the newest request wins. Either way the
+ * question the compositor asks — what does this clip look like now — is
+ * answered from memory, without awaiting anything. If the exact frame has not
+ * arrived, the nearest one already decoded is handed over and the real one
+ * replaces it a moment later. A held frame is worth more than a stall.
+ *
+ * The ring is bounded by the sink's own canvas pool: `mediabunny` cycles a
+ * fixed set of canvases, so keeping a reference to more of them than the pool
+ * holds would hand out a canvas that has since been drawn over. The ring's
+ * capacity and the pool size move together, and that is the whole memory story.
+ */
+
+import type { CanvasSink, Input, WrappedCanvas } from "mediabunny";
+import { frameSink, openMedia, videoTrackOf } from "./mediaRead";
+import type { MediaAsset } from "./types";
+
+/**
+ * Decoded frames a source keeps around where it is being read.
+ *
+ * Small on purpose. Each one is a full-size canvas, and ten sources holding a
+ * dozen apiece is hundreds of megabytes — enough to take the tab down. The sink
+ * pre-decodes ahead internally, so the ring only has to bridge the gap between
+ * two frames of the display: the frame being shown, a couple ahead of it, and
+ * one spare for the blend at a dissolve.
+ */
+const RING = 10;
+/** Canvases the sink cycles. Two more than the ring, so the frame being decoded
+ * never lands on one the compositor is drawing this instant. */
+const POOL = RING + 2;
+/**
+ * How far ahead of where it is read a playing source decodes.
+ *
+ * This has to stay inside what the ring can hold. Reading further ahead than
+ * `RING` frames would push the frame actually on screen out of the ring to make
+ * room for one nobody has asked for yet — the picture would run ahead of the
+ * playhead and the decoder would burn through the file to put it there.
+ */
+export const DECODE_AHEAD_S = 0.3;
+/** Two source times closer than this are the same frame. */
+const SAME = 1e-4;
+/** Frames a source must go unwanted before the pool will close it. About a
+ * second of playback — long enough that crossing a cut and coming back finds
+ * the decoder still open. */
+const EVICT_GRACE = 90;
+/** A failed open tries again this much later, growing per attempt, and gives
+ * up after this many. A network blip heals on its own; a file that is truly
+ * unreadable stops costing anything after a few seconds. */
+const RETRY_MS = 1000;
+const RETRIES = 3;
+
+/** A clip's picture at an instant, with where it came from. */
+export interface SourceFrame {
+ image: CanvasImageSource;
+ width: number;
+ height: number;
+ /** The frame's own timestamp in the source, for telling a held frame from a
+ * fresh one. */
+ timestamp: number;
+}
+
+const frameOfCanvas = (c: WrappedCanvas): SourceFrame => ({
+ image: c.canvas,
+ width: c.canvas.width,
+ height: c.canvas.height,
+ timestamp: c.timestamp,
+});
+
+/** What the ring stores: a frame and the stretch of source it stands for. */
+export interface Timed {
+ timestamp: number;
+ duration: number;
+}
+
+/**
+ * Decoded frames in the order they arrived, oldest dropped first.
+ *
+ * The order matters more than it looks. The sink hands out canvases from a
+ * fixed pool and reuses the oldest once it wraps, so holding a reference past
+ * the pool's length means holding a canvas something else has drawn on. Bounded
+ * by arrival, the ring can only ever name canvases the pool still considers
+ * ours.
+ */
+export class FrameRing<T extends Timed> {
+ private items: T[] = [];
+
+ constructor(private readonly cap: number) {}
+
+ get size(): number {
+ return this.items.length;
+ }
+
+ get newest(): number {
+ return this.items.length ? this.items[this.items.length - 1].timestamp : -Infinity;
+ }
+
+ /** The earliest timestamp held, which is where a walk's buffer begins. */
+ get oldest(): number {
+ let min = Infinity;
+ for (const i of this.items) min = Math.min(min, i.timestamp);
+ return min;
+ }
+
+ push(item: T): void {
+ this.items.push(item);
+ while (this.items.length > this.cap) this.items.shift();
+ }
+
+ clear(): void {
+ this.items = [];
+ }
+
+ /**
+ * The frame covering `t`: the last one starting at or before it. Before the
+ * first frame held, the earliest — at a clip's head that is the frame the cut
+ * opens on, and showing it beats showing nothing.
+ */
+ at(t: number): T | null {
+ let best: T | null = null;
+ let earliest: T | null = null;
+ for (const i of this.items) {
+ if (!earliest || i.timestamp < earliest.timestamp) earliest = i;
+ if (i.timestamp <= t + SAME && (!best || i.timestamp > best.timestamp)) best = i;
+ }
+ return best ?? earliest;
+ }
+
+ /**
+ * Whether `t` is genuinely covered, rather than answered by a held frame.
+ *
+ * A frame stands for its own timestamp up to the next one, which is what its
+ * duration says. Asking the frame rather than the ring's extent gets the same
+ * answer for a walk that has run past `t` and for a single frame fetched at
+ * `t`, so a scrub and a playback agree on what "we have this one" means.
+ */
+ covers(t: number): boolean {
+ return this.items.some(
+ (i) => t >= i.timestamp - SAME && t < i.timestamp + Math.max(i.duration, SAME) + SAME
+ );
+ }
+
+ /** How many frames held start at or after `t`. */
+ aheadOf(t: number): number {
+ let n = 0;
+ for (const i of this.items) if (i.timestamp >= t - SAME) n++;
+ return n;
+ }
+
+ /** How many frames held lie inside [from, to]. */
+ between(from: number, to: number): number {
+ let n = 0;
+ for (const i of this.items) if (i.timestamp >= from - SAME && i.timestamp <= to + SAME) n++;
+ return n;
+ }
+
+ /** Drop frames outside a window a reader has moved away from. */
+ keep(from: number, to: number): void {
+ this.items = this.items.filter((i) => i.timestamp >= from - SAME && i.timestamp <= to + SAME);
+ }
+}
+
+/**
+ * One decoder and its ring.
+ *
+ * A source is opened for a clip's *mapping* rather than for its asset: same
+ * file, same speed, same source-time offset means the same picture at every
+ * instant, so a tiled reveal playing one file across many tracks decodes once,
+ * and a plain split reads straight across its own cut. Two different trims of
+ * one file keep their own sources, which is what lets a same-source dissolve
+ * show two distinct frames.
+ */
+export class ClipFrameSource {
+ private input: Input | null = null;
+ private sink: CanvasSink | null = null;
+ private ring = new FrameRing<WrappedCanvas>(RING);
+ /** A still's single frame; stills never stream. */
+ private still: SourceFrame | null = null;
+ private opening: Promise<void> | null = null;
+ private closed = false;
+ /** Set when the file turns out to hold no picture this browser can read. */
+ private unreadable = false;
+ /** Failed opens so far, and the timer that will clear `unreadable` for the
+ * next try. */
+ private attempts = 0;
+ private retryTimer = 0;
+
+ /** The forward walk, when one is running, where it started, and the last
+ * frame it landed. Steering reads only these two numbers: the ring is a
+ * cache that may also hold frames left by other gestures — a jump's single
+ * frame, a spent walk — and a min/max over that sparse set has holes in it
+ * that read as coverage. */
+ private stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = null;
+ private streamFrom = 0;
+ private streamTail = 0;
+ /** The walk ran off the end of the file. Its span is still the truth — there
+ * are no frames past its tail to decode — so a reader inside it must not
+ * keep restarting a walk that can only end again. */
+ private streamDone = false;
+ private streaming = false;
+ /** The time a paused reader last asked for, and whether a read is in flight.
+ * Latest wins: a fast drag decodes where it stopped, never every point it
+ * crossed. */
+ private wanted: number | null = null;
+ private reading = false;
+
+ /** The engine's tick number when this was last asked for anything, for the
+ * pool's eviction order. */
+ touched = 0;
+
+ /** The URL this source is reading. The pool compares it against the store's
+ * current one, so a re-minted signed URL replaces the source under it. */
+ get url(): string {
+ return this.asset.url;
+ }
+
+ constructor(
+ private readonly asset: MediaAsset,
+ private readonly height: number,
+ /** Called whenever a frame lands. A paused editor draws the nearest frame
+ * it has and stops; without a nudge, the exact frame would decode into a
+ * ring nothing looks at again. */
+ private readonly onFrame: () => void = () => {}
+ ) {}
+
+ /** Decoded frames held right now, for the pool's budget and the perf trace. */
+ get held(): number {
+ return this.ring.size + (this.still ? 1 : 0);
+ }
+
+ get ready(): boolean {
+ return this.still !== null || this.ring.size > 0;
+ }
+
+ get failed(): boolean {
+ return this.unreadable;
+ }
+
+ /**
+ * The best picture this source has for source time `t`, without waiting.
+ *
+ * Exact when the ring holds the frame covering `t`; otherwise the nearest
+ * frame before it, which is what a decoder running slightly behind should
+ * show. Null only when nothing has been decoded yet.
+ */
+ frameAt(t: number): SourceFrame | null {
+ if (this.still) return this.still;
+ const c = this.ring.at(t);
+ return c ? frameOfCanvas(c) : null;
+ }
+
+ /** Whether the ring holds the exact frame covering `t`. */
+ hasExact(t: number): boolean {
+ return this.still !== null || this.ring.covers(t);
+ }
+
+ /**
+ * Say where this clip is being read, and let the source decide how to keep up.
+ *
+ * `playing` walks forward from `t` and stays `DECODE_AHEAD_S` ahead of it;
+ * paused, a single frame is fetched for `t` and the newest ask wins.
+ */
+ want(t: number, playing: boolean): void {
+ if (this.closed || this.unreadable) return;
+ if (this.asset.type === "image") {
+ void this.openStill();
+ return;
+ }
+ void this.open();
+ if (playing) {
+ this.wanted = null;
+ this.pumpStream(t);
+ } else {
+ if (this.hasExact(t)) return;
+ // A drag creeping along is a forward walk, not a series of jumps. Asking
+ // the file for each position separately means decoding from the nearest
+ // keyframe every time — the same cost the `<video>` element charged. If
+ // the wanted frame is inside the walk or just ahead of the last frame
+ // that arrived, keep walking and each step costs one frame. The last
+ // arrival is the only ring fact consulted: anything older may be a
+ // leftover from some other gesture entirely.
+ const walkEdge = Math.max(this.streamFrom, this.streamTail);
+ const nearWalk = this.stream !== null && t >= walkEdge - SAME && t < walkEdge + 0.5;
+ const nearLast =
+ this.ring.size > 0 && t >= this.ring.newest - SAME && t < this.ring.newest + 0.5;
+ if (nearWalk || nearLast) {
+ this.pumpStream(t);
+ return;
+ }
+ // A real jump: drop the walk and fetch the one frame. Newest ask wins, so
+ // a fast drag decodes where it stopped rather than everywhere it crossed.
+ this.stopStream();
+ this.wanted = t;
+ void this.pumpSeek();
+ }
+ }
+
+ /** Frames outside [from, to] are of no further use — a clip whose window
+ * moved on, or a scrub that jumped. */
+ trim(from: number, to: number): void {
+ this.ring.keep(from, to);
+ }
+
+ close(): void {
+ this.closed = true;
+ this.stopStream();
+ this.ring.clear();
+ if (this.retryTimer) clearTimeout(this.retryTimer);
+ this.retryTimer = 0;
+ // A still's bitmap holds real memory the GC can't see the size of.
+ if (this.still && this.still.image instanceof ImageBitmap) this.still.image.close();
+ this.still = null;
+ this.input?.dispose();
+ this.input = null;
+ this.sink = null;
+ }
+
+ /**
+ * An open failed. That is usually a moment — a network blip, a signed URL a
+ * few seconds past its window — so the source tells the link keeper (which
+ * re-mints an expired URL; the pool then swaps this source out under the new
+ * one) and books itself another try. Only a file that keeps failing stays
+ * marked unreadable, and only after it has had its chances.
+ */
+ private fail(): void {
+ this.unreadable = true;
+ this.opening = null;
+ void import("./mediaLinks").then((m) => m.reportMediaUrlError(this.asset.url));
+ if (this.closed || this.attempts >= RETRIES || typeof window === "undefined") return;
+ this.attempts++;
+ this.retryTimer = window.setTimeout(() => {
+ this.retryTimer = 0;
+ if (this.closed) return;
+ this.unreadable = false;
+ // Nothing asks a failed source for frames, so nothing would notice it is
+ // willing again without a nudge.
+ this.onFrame();
+ }, RETRY_MS * this.attempts);
+ }
+
+ private open(): Promise<void> {
+ if (this.opening) return this.opening;
+ this.opening = (async () => {
+ const input = openMedia(this.asset.url);
+ try {
+ const track = await videoTrackOf(input);
+ if (!track) {
+ input.dispose();
+ this.unreadable = true;
+ return;
+ }
+ if (this.closed) {
+ input.dispose();
+ return;
+ }
+ this.input = input;
+ // Height alone: the sink keeps the source's aspect and applies the
+ // file's rotation, so a phone clip arrives upright at preview size and
+ // no caller has to know it was ever sideways.
+ this.sink = frameSink(track, { height: this.height }, POOL);
+ } catch {
+ input.dispose();
+ this.fail();
+ }
+ })();
+ return this.opening;
+ }
+
+ private async openStill(): Promise<void> {
+ if (this.opening) return this.opening;
+ this.opening = (async () => {
+ try {
+ const res = await fetch(this.asset.url, { mode: "cors" });
+ const bitmap = await createImageBitmap(await res.blob());
+ if (this.closed) return bitmap.close();
+ this.still = {
+ image: bitmap,
+ width: bitmap.width,
+ height: bitmap.height,
+ timestamp: 0,
+ };
+ this.onFrame();
+ } catch {
+ this.fail();
+ }
+ })();
+ return this.opening;
+ }
+
+ private stopStream(): void {
+ const s = this.stream;
+ this.stream = null;
+ this.streaming = false;
+ void s?.return(undefined).catch(() => {});
+ }
+
+ /**
+ * Keep the forward walk running and roughly `DECODE_AHEAD_S` ahead of `t`.
+ *
+ * A walk already covering `t` is left alone — restarting it would throw away
+ * the buffer it has built and re-decode from the nearest keyframe, which is
+ * the hitch this design exists to remove. It restarts only for a real jump:
+ * a scrub, or a clip re-entered from somewhere else.
+ */
+ private pumpStream(t: number): void {
+ // Whether the walk covers `t` is a question about the walk — where it
+ // started and the last frame it has landed. A walk whose first frame is
+ // still decoding is already the right walk: that first step is a keyframe
+ // seek, longer than a display frame, and judging it by frames landed would
+ // tear it down on every tick and it would never land anything. It gets a
+ // second of slack for the reader to advance while that first frame comes.
+ if (this.stream) {
+ // The walk serves `t` if the frame is already held, or if `t` sits just
+ // past the tail — where the next pulls will land. Its span behind the
+ // tail proves nothing: frames there have been aging out of the ring the
+ // whole time, so a reader that went back to the walk's beginning would
+ // find a record of coverage and no frames.
+ const landed = this.streamTail > this.streamFrom + SAME;
+ const ahead = landed
+ ? t >= this.streamTail - SAME && t <= this.streamTail + DECODE_AHEAD_S
+ : t >= this.streamFrom - SAME && t <= this.streamFrom + DECODE_AHEAD_S + 1;
+ if (ahead || this.ring.covers(t)) {
+ void this.drain(t);
+ return;
+ }
+ this.stopStream();
+ } else if (this.streamDone && (this.ring.covers(t) || t >= this.streamTail - SAME)) {
+ // The walk ended at the file's last frame. Inside its span the frame is
+ // already held, and past its tail there is nothing left to decode — the
+ // newest frame held is the answer. Restarting here would re-decode the
+ // whole tail from its keyframe once per tick, forever.
+ return;
+ }
+ this.streamFrom = t;
+ this.streamTail = t;
+ this.streamDone = false;
+ void this.startStream(t);
+ }
+
+ private async startStream(from: number): Promise<void> {
+ await this.open();
+ if (this.closed || !this.sink || this.stream) return;
+ // The read may have been overtaken while the file was opening.
+ if (Math.abs(this.streamFrom - from) > SAME) return;
+ this.stream = this.sink.canvases(Math.max(0, from));
+ void this.drain(from);
+ }
+
+ /** Pull frames until the ring reaches `DECODE_AHEAD_S` past `t`. */
+ private async drain(t: number): Promise<void> {
+ if (this.streaming || !this.stream) return;
+ this.streaming = true;
+ try {
+ for (;;) {
+ const stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = this.stream;
+ if (!stream || this.closed) break;
+ // Stop at the lookahead, and stop early once the walk's own frames at
+ // or past `t` fill the ring — pushing another would drop one that is
+ // still wanted. Only the walk's frames count: leftovers from other
+ // gestures merely age out of the cache as the walk pushes.
+ if (this.streamTail >= t + DECODE_AHEAD_S) break;
+ if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;
+ const { value, done } = await stream.next();
+ // A restart or a close while awaiting: this walk is no longer the one.
+ if (this.stream !== stream) break;
+ if (done || !value) {
+ this.stream = null;
+ this.streamDone = true;
+ break;
+ }
+ this.streamTail = value.timestamp;
+ this.ring.push(value);
+ this.onFrame();
+ }
+ } catch {
+ this.stopStream();
+ } finally {
+ this.streaming = false;
+ }
+ }
+
+ /** Fetch the single frame a paused reader is waiting on. */
+ private async pumpSeek(): Promise<void> {
+ if (this.reading) return;
+ this.reading = true;
+ try {
+ await this.open();
+ for (;;) {
+ const t = this.wanted;
+ if (t === null || this.closed || !this.sink) break;
+ this.wanted = null;
+ const c = await this.sink.getCanvas(Math.max(0, t));
+ if (this.closed) break;
+ if (c) {
+ this.ring.push(c);
+ this.onFrame();
+ }
+ // Another position was asked for while this one decoded; that one is
+ // where the pointer actually is now.
+ if (this.wanted === null) break;
+ }
+ } catch {
+ this.wanted = null;
+ } finally {
+ this.reading = false;
+ }
+ }
+}
+
+/**
+ * The live decoders.
+ *
+ * A decoder is not free: an open read against the file, a demuxer, and one of
+ * the small number of hardware decode slots a tab is given. Past that limit
+ * decoding falls back to software, and since sound stays real-time while frames
+ * arrive late, the picture drifts behind the audio. So the pool is capped, and
+ * the sources this frame is built from are never the ones evicted.
+ */
+export class FrameSourcePool {
+ private sources = new Map<string, ClipFrameSource>();
+ private tick = 0;
+
+ constructor(
+ private budget = 10,
+ private readonly onFrame: () => void = () => {}
+ ) {}
+
+ /** Advance the clock the eviction order is measured on. */
+ beginFrame(): void {
+ this.tick++;
+ }
+
+ /**
+ * The source for one clip mapping, opening it if this is the first ask.
+ * `key` must identify a mapping — same file, speed and source-time offset —
+ * so clips showing identical pictures share one decoder.
+ */
+ get(key: string, asset: MediaAsset, height: number): ClipFrameSource {
+ const id = `${key}|${height}`;
+ let src = this.sources.get(id);
+ // The mapping names which pictures; the URL is where they are read from,
+ // and it moves — a signed link re-mints, a shot re-renders onto its asset.
+ // A source is only current while it reads the store's current URL.
+ if (src && src.url !== asset.url) {
+ src.close();
+ this.sources.delete(id);
+ src = undefined;
+ }
+ if (!src) {
+ src = new ClipFrameSource(asset, height, this.onFrame);
+ this.sources.set(id, src);
+ }
+ src.touched = this.tick;
+ return src;
+ }
+
+ /** Decoded frames held across every source, for the budget and the trace. */
+ get held(): number {
+ let n = 0;
+ for (const s of this.sources.values()) n += s.held;
+ return n;
+ }
+
+ get size(): number {
+ return this.sources.size;
+ }
+
+ /**
+ * Close the decoders nothing has asked for lately.
+ *
+ * "Lately" is doing real work here. Evicting everything untouched by the
+ * current frame reads as tidy and behaves terribly: a cut with more clips
+ * than the budget closes and reopens decoders every single frame, and the
+ * reopening — a fresh read of the file, a fresh decoder — is far more
+ * expensive than the memory it was saving. A source is only a candidate once
+ * nothing has wanted it for a while, which is long enough that a clip being
+ * crossed back and forth over is never the one closed.
+ */
+ evict(): void {
+ if (this.sources.size <= this.budget) return;
+ const idle = [...this.sources.entries()]
+ .filter(([, s]) => this.tick - s.touched >= EVICT_GRACE)
+ .sort((a, b) => a[1].touched - b[1].touched);
+ let over = this.sources.size - this.budget;
+ for (const [id, src] of idle) {
+ if (over-- <= 0) break;
+ src.close();
+ this.sources.delete(id);
+ }
+ }
+
+ closeAll(): void {
+ for (const s of this.sources.values()) s.close();
+ this.sources.clear();
+ }
+}
+
+/**
+ * The decode identity of a clip: the frames it shows are a function of its
+ * file, its speed, and where its source time stands at timeline zero. Two clips
+ * agreeing on all three show the same picture at every instant.
+ */
+export function mappingKey(
+ assetId: string,
+ speed: number,
+ inPoint: number,
+ start: number
+): string {
+ return `${assetId}|${speed}|${(inPoint - start * speed).toFixed(3)}`;
+}
diff --git a/site/src/cut/lib/mediaLinks.ts b/site/src/cut/lib/mediaLinks.ts
index 28dd573b..8ba21a77 100644
--- a/site/src/cut/lib/mediaLinks.ts
+++ b/site/src/cut/lib/mediaLinks.ts
@@ -18,11 +18,11 @@
// hands back the same string, so the reload is what clears an element that
// failed for any other reason.
//
-// A re-mint swaps the store's asset URLs in place. Components and the preview
-// engine both key their elements off asset.url, so the swap heals them on
-// render. The store marks the batch at load; a global capture-phase error
-// listener watches DOM elements, and the preview engine reports its detached
-// decoders through reportMediaElementError.
+// A re-mint swaps the store's asset URLs in place. Media elements key off
+// asset.url and the preview's decoder pool compares each source against it, so
+// the swap heals both on the next render. The store marks the batch at load; a
+// global capture-phase error listener watches DOM elements, and the preview's
+// frame sources report their failed reads through reportMediaUrlError.
import { fetchSignedMediaUrls } from "./backend/cloud";
import { mediaUrl } from "./types";
@@ -143,6 +143,18 @@ function retryElement(el: HTMLImageElement | HTMLMediaElement, src: string, atte
}, ELEMENT_RETRY_BASE_MS * attempt);
}
+/** A decoder failed to read a media URL. There is no element to reload — the
+ * frame source retries on its own — but an expiring URL may genuinely have
+ * expired, and only a re-mint heals that. The store swap repoints the decoder
+ * pool, which opens a fresh source under the new URL. */
+export function reportMediaUrlError(src: string) {
+ if (!src || src.startsWith("blob:") || src.startsWith("data:")) return;
+ if (!isExpiringUrl(src)) return;
+ if (Date.now() - lastForcedAt < FORCED_GAP_MS) return;
+ lastForcedAt = Date.now();
+ void refreshSignedUrls(true);
+}
+
/** A media element failed to load. An expiring URL may genuinely have expired,
* so re-mint the whole batch — the store swap repoints every consumer. Then
* reload the element either way: re-minting is not a cure on its own, because
diff --git a/site/src/cut/lib/perfTrace.ts b/site/src/cut/lib/perfTrace.ts
new file mode 100644
index 00000000..e62dc3f7
--- /dev/null
+++ b/site/src/cut/lib/perfTrace.ts
@@ -0,0 +1,152 @@
+"use client";
+
+/**
+ * What the preview actually did, frame by frame.
+ *
+ * Scrub latency and a stutter at a cut are both claims about time, and neither
+ * can be settled by watching. This records the two facts that settle them: when
+ * a time was asked for, and when a frame for that time reached the screen. From
+ * those, "the picture is one frame behind the pointer" and "the join dropped a
+ * frame" become numbers a script can hold to a budget.
+ *
+ * The trace is off until something arms it, and while it is off every call here
+ * is a null check. The eval arms it through `window.__cutPerf`; nothing in the
+ * product turns it on.
+ */
+
+/** One composited frame that reached the canvas. */
+export interface PresentRecord {
+ /** Timeline time the frame was composited for. */
+ t: number;
+ /** When it was painted, on the `performance.now()` clock. */
+ at: number;
+ /** Source timestamp of the master layer's picture, for spotting a frame that
+ * was shown twice or skipped. */
+ srcTs: number | null;
+ /** The clip the master picture came from, so a boundary is findable. */
+ clipId: string | null;
+ /**
+ * Whether the frame drawn is the one that belongs at `t`.
+ *
+ * This is the measure that matters, and it is not the same as "a new
+ * picture". Thirty-frame footage on a sixty-hertz display repeats every
+ * source frame once and is perfectly smooth; a decoder that fell behind
+ * repeats one too, and is not. Asking whether the source actually held the
+ * wanted frame tells those apart.
+ */
+ exact: boolean;
+ /** True when no decoded frame existed at all and the paint fell back to
+ * holding or to black. A stale present never resolves a scrub. */
+ stale: boolean;
+}
+
+/** A time the editor was asked to show, and when it was asked for. */
+export interface SeekRecord {
+ t: number;
+ at: number;
+ /** How long until a frame for this time was painted. Null while unresolved —
+ * either still pending, or superseded by a later seek. */
+ latencyMs: number | null;
+}
+
+export interface LongTaskRecord {
+ at: number;
+ ms: number;
+}
+
+export interface Trace {
+ presents: PresentRecord[];
+ seeks: SeekRecord[];
+ longTasks: LongTaskRecord[];
+ /** rAF callbacks the engine ran, for checking that an idle editor is idle. */
+ ticks: number;
+ /** Decoded frames the sources are holding open, sampled per present. */
+ liveSamples: number[];
+ startedAt: number;
+}
+
+/** Times equal within this are the same instant. Half a frame at 120fps —
+ * tight enough that a real lag never passes, loose enough that float drift on
+ * the timeline never fails. */
+const SAME_TIME = 0.004;
+
+let trace: Trace | null = null;
+let observer: PerformanceObserver | null = null;
+/** The seek still waiting for its picture. A later seek replaces it, which is
+ * what makes a fast drag measure the position it settled on. */
+let pendingSeek: SeekRecord | null = null;
+
+/** Whether anything is listening. The engine's hot path checks this first. */
+export const tracing = () => trace !== null;
+
+export function startTrace(): void {
+ trace = {
+ presents: [],
+ seeks: [],
+ longTasks: [],
+ ticks: 0,
+ liveSamples: [],
+ startedAt: performance.now(),
+ };
+ pendingSeek = null;
+ if (typeof PerformanceObserver === "undefined") return;
+ try {
+ observer = new PerformanceObserver((list) => {
+ for (const entry of list.getEntries()) {
+ trace?.longTasks.push({ at: entry.startTime, ms: entry.duration });
+ }
+ });
+ observer.observe({ entryTypes: ["longtask"] });
+ } catch {
+ // Long-task timing is Chromium-only. Its absence costs one metric, not the
+ // run.
+ observer = null;
+ }
+}
+
+export function stopTrace(): Trace | null {
+ const out = trace;
+ observer?.disconnect();
+ observer = null;
+ trace = null;
+ pendingSeek = null;
+ return out;
+}
+
+/** A time was asked for — a scrub, a skim, a click on the ruler. */
+export function markSeek(t: number): void {
+ if (!trace) return;
+ const rec: SeekRecord = { t, at: performance.now(), latencyMs: null };
+ trace.seeks.push(rec);
+ pendingSeek = rec;
+}
+
+/** A frame reached the canvas. */
+export function markPresent(rec: Omit<PresentRecord, "at">): void {
+ if (!trace) return;
+ const at = performance.now();
+ trace.presents.push({ ...rec, at });
+ // A held or black frame is not an answer to the question the scrub asked —
+ // neither is the neighbouring frame drawn while the real one decodes. The
+ // clock keeps running until the frame that belongs at that time is on screen.
+ if (pendingSeek && rec.exact && !rec.stale && Math.abs(rec.t - pendingSeek.t) <= SAME_TIME) {
+ pendingSeek.latencyMs = at - pendingSeek.at;
+ pendingSeek = null;
+ }
+}
+
+/** Whether a time has been asked for that no frame has answered yet. Lets a
+ * driver wait for the picture to catch up instead of guessing at a delay. */
+export const awaitingFrame = () => pendingSeek !== null;
+
+/** One turn of the engine's loop. */
+export function markTick(): void {
+ if (!trace) return;
+ trace.ticks++;
+}
+
+/** How many decoded frames are being held open right now. */
+export function markLiveSamples(n: number): void {
+ if (!trace) return;
+ trace.liveSamples.push(n);
+}
diff --git a/site/src/cut/lib/playhead.ts b/site/src/cut/lib/playhead.ts
new file mode 100644
index 00000000..32d99870
--- /dev/null
+++ b/site/src/cut/lib/playhead.ts
@@ -0,0 +1,130 @@
+"use client";
+
+/**
+ * Where the preview is, and where the mouse is hovering over the timeline.
+ *
+ * These two numbers change more often than anything else in Cut — sixty times a
+ * second while playing, and once per pointer event while dragging. They used to
+ * live in the editor store beside the clips, which meant every one of those
+ * changes ran the document's write normalizer and woke every mounted selector
+ * in the editor, a few hundred of them, to ask whether a title bar had moved.
+ *
+ * So they live here instead, in the smallest store that can hold them: a
+ * number, a set of listeners, and nothing else. Surfaces that draw the playhead
+ * subscribe; surfaces that only need to know where it is when the user does
+ * something read it directly. The document store is left to describe the
+ * document.
+ *
+ * The clamp to the project's length lives with the callers that know the
+ * length. This module holds a time and tells people it changed.
+ */
+
+import { useSyncExternalStore } from "react";
+import { markSeek } from "./perfTrace";
+
+let at = 0;
+let skim: number | null = null;
+const listeners = new Set<() => void>();
+
+function notify() {
+ for (const fn of listeners) fn();
+}
+
+function subscribe(fn: () => void) {
+ listeners.add(fn);
+ return () => {
+ listeners.delete(fn);
+ };
+}
+
+/**
+ * Run `fn` on every move, outside React.
+ *
+ * For surfaces whose whole response to the clock is one DOM write — the
+ * playhead's transform, the grade on the picture. Re-rendering a component to
+ * set one style property costs a reconcile per frame and buys nothing.
+ */
+export const subscribePlayhead = subscribe;
+
+/** The playhead, in timeline seconds. */
+export const playheadAt = () => at;
+
+/** The skimmer, or null when the pointer is off the timeline. */
+export const skimAt = () => skim;
+
+/**
+ * The time the picture should show: the skimmer while one is live, the playhead
+ * otherwise. One place decides this, so the preview and everything drawing
+ * against the preview agree on which moment is on screen.
+ */
+export const previewAt = () => skim ?? at;
+
+/** Move the playhead. Callers clamp to the project length first. */
+export function setPlayhead(t: number): void {
+ if (t === at) return;
+ at = t;
+ markSeek(t);
+ notify();
+}
+
+export function setSkim(t: number | null): void {
+ if (t === skim) return;
+ skim = t;
+ if (t !== null) markSeek(t);
+ notify();
+}
+
+/**
+ * Hold the playhead inside a timeline that just got shorter. Deleting the last
+ * of a long row leaves the playhead standing past the new end, which puts the
+ * readout ahead of the total and — since the playhead is placed with a
+ * transform, and transforms count toward scrollable overflow — stretches the
+ * scroll area into empty space.
+ */
+export function clampPlayhead(total: number): void {
+ if (at > total) setPlayhead(total);
+}
+
+/** The playhead, as a subscription. Re-renders the caller on every change. */
+export function usePlayhead(): number {
+ return useSyncExternalStore(subscribe, playheadAt, () => 0);
+}
+
+export function useSkim(): number | null {
+ return useSyncExternalStore(subscribe, skimAt, () => null);
+}
+
+export function usePreviewTime(): number {
+ return useSyncExternalStore(subscribe, previewAt, () => 0);
+}
+
+/**
+ * Something the preview time decides, without re-rendering when only the time
+ * moved — whether a cue is the one being spoken, whether a clip is on screen.
+ * `select` must return a primitive: React compares snapshots by identity, and a
+ * fresh object every read would loop.
+ */
+export function usePreviewSelector<T extends string | number | boolean | null>(
+ select: (t: number) => T
+): T {
+ return useSyncExternalStore(
+ subscribe,
+ () => select(previewAt()),
+ () => select(0)
+ );
+}
+
+/**
+ * The preview time rounded to `steps` per second, for surfaces that want the
+ * clock without wanting every frame of it — a picker's swatches, a panel's
+ * live thumbnail. Rounding inside the snapshot means React compares the coarse
+ * value and skips the render, so a 2 Hz caller re-renders twice a second while
+ * the preview runs at sixty.
+ */
+export function usePreviewTimeEvery(steps: number): number {
+ return useSyncExternalStore(
+ subscribe,
+ () => Math.round(previewAt() * steps),
+ () => 0
+ );
+}
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
new file mode 100644
index 00000000..de798032
--- /dev/null
+++ b/site/src/cut/lib/previewMixer.ts
@@ -0,0 +1,343 @@
+"use client";
+
+/**
+ * The cut's sound while you watch it.
+ *
+ * Sound used to ride the same `<video>` and `<audio>` elements the picture came
+ * from: the clip under the playhead was both the speaker and the clock, and
+ * everything else chased it. That made the element's decode latency into the
+ * project's timing, and it left no way to be sure the preview mixed a cut the
+ * way the export would.
+ *
+ * Here every audible thing is a buffer scheduled on one `AudioContext`. The
+ * context's own clock is what the preview runs on — it advances in real time,
+ * never backward, and is sample-accurate — so the picture follows the sound
+ * rather than the other way round. Clips are scheduled a window at a time so
+ * starting playback costs one short decode instead of the whole timeline, and a
+ * clip playing at a speed other than 1 is time-stretched rather than resampled,
+ * which is what keeps its pitch and matches what the export writes.
+ *
+ * Gains are the only thing touched per frame, and only their value: the clip's
+ * own volume, its fades, the duck under a live voiceover, and the whole-project
+ * fade. The ramps themselves are the frame plan's, so the mix follows the same
+ * description the picture does.
+ */
+
+import { decodeAudioSpan } from "./mediaRead";
+import { timeStretch } from "./timeStretch";
+
+/** How much of a voice is decoded and scheduled at once, in timeline seconds. */
+const WINDOW_S = 20;
+/** A stretched window is resynthesised on the main thread, so it is kept short
+ * enough that the work lands inside a frame or two. */
+const WINDOW_STRETCHED_S = 4;
+/** Schedule the next window once the playhead is this close to running out. */
+const AHEAD_S = 2;
+
+/** One audible thing on the timeline: a clip's own sound, or a soundtrack item. */
+export interface Voice {
+ /** Stable across frames — the clip's id. A voice whose id survives keeps
+ * playing; one that vanishes is stopped. */
+ id: string;
+ url: string;
+ /** Where it starts on the timeline, and the source span it plays. */
+ start: number;
+ in: number;
+ out: number;
+ speed: number;
+ /** Everything the frame plan says about how loud it is right now. */
+ gain: number;
+}
+
+interface Scheduled {
+ node: AudioBufferSourceNode;
+ /** Timeline time this window ends at, so the next one starts there. */
+ until: number;
+}
+
+interface LiveVoice {
+ url: string;
+ speed: number;
+ in: number;
+ out: number;
+ start: number;
+ gain: GainNode;
+ /** Windows already handed to the context, oldest first. */
+ windows: Scheduled[];
+ /** Timeline time everything scheduled so far runs out at. */
+ scheduled: number;
+ /** A decode in flight, so a slow read doesn't start a second one. */
+ filling: boolean;
+ /** Set when the source can't be read; the voice stays silent rather than
+ * retrying every frame. */
+ dead: boolean;
+}
+
+/**
+ * The preview's clock and mixer.
+ *
+ * One of these lives for as long as the editor is open. It holds the context,
+ * the anchor that maps timeline seconds onto it, and one gain-fed voice per
+ * audible clip.
+ */
+export class PreviewMixer {
+ private ctx: AudioContext | null = null;
+ private master: GainNode | null = null;
+ private voices = new Map<string, LiveVoice>();
+ /** Timeline time, context time and wall time of the same instant. Everything
+ * scheduled is placed against this. */
+ private anchor: { timeline: number; ctx: number; wall: number } | null = null;
+ private decoded = new Map<string, AudioBuffer>();
+
+ /** Whether the clock is running. */
+ get running(): boolean {
+ return this.anchor !== null;
+ }
+
+ private context(): AudioContext {
+ if (!this.ctx) {
+ this.ctx = new AudioContext();
+ this.master = this.ctx.createGain();
+ this.master.connect(this.ctx.destination);
+ }
+ return this.ctx;
+ }
+
+ /**
+ * Where the playhead is, from the sound's own clock.
+ *
+ * This is the whole point of the arrangement: the number the picture is drawn
+ * against comes from the same clock the samples leave on, so they cannot
+ * drift apart no matter what the main thread is doing.
+ */
+ now(): number {
+ if (!this.anchor) return 0;
+ const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;
+ // A browser holds an `AudioContext` suspended until the page has been
+ // interacted with, and a suspended context's clock does not move. Reading
+ // it blindly would freeze the picture on a project played before the user
+ // has touched anything, and picking it up once it finally starts would jump
+ // the playhead by however long it stayed asleep. So the wall carries the cut
+ // until the audio clock agrees with it, and a clock that disagrees is
+ // re-anchored rather than believed.
+ if (!this.ctx || this.ctx.state !== "running") return wall;
+ const audio = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);
+ if (Math.abs(audio - wall) > 0.25) {
+ this.anchor.ctx = this.ctx.currentTime - (wall - this.anchor.timeline);
+ return wall;
+ }
+ return audio;
+ }
+
+ /**
+ * Begin playing at timeline time `t`.
+ *
+ * The anchor is this very instant: the next `now()` reads `t` and counts
+ * forward. Anchoring even slightly in the future reads *behind* `t` first,
+ * which walks the playhead backwards on every play and, at the head of the
+ * timeline, puts the clock before the first clip. The first audio window
+ * pays for its own decode instead — `fill` starts a late window at the
+ * current time with the missed sliver skipped, so the sound joins in sync a
+ * few milliseconds in.
+ */
+ start(t: number): void {
+ const ctx = this.context();
+ void ctx.resume();
+ this.stopVoices();
+ this.anchor = { timeline: t, ctx: ctx.currentTime, wall: performance.now() };
+ }
+
+ /** Stop the clock and silence everything. */
+ stop(): void {
+ this.anchor = null;
+ this.stopVoices();
+ }
+
+ /** The whole-project fade, over the finished mix. */
+ setMasterGain(g: number): void {
+ if (this.master) this.master.gain.value = Math.max(0, Math.min(1, g));
+ }
+
+ /**
+ * Bring the mix in line with what is audible at timeline time `t`.
+ *
+ * Called every frame while playing. Voices that just became live are opened
+ * and scheduled; voices that ran out are dropped; the rest have their gain
+ * set to whatever the frame plan worked out. Nothing here awaits, so a slow
+ * decode delays that one voice and nothing else.
+ */
+ update(t: number, voices: Voice[]): void {
+ if (!this.anchor || !this.ctx) return;
+ const seen = new Set<string>();
+ for (const v of voices) {
+ seen.add(v.id);
+ // An edit to a playing clip arrives as the same id with new geometry — a
+ // trim, a move, a speed change, a re-minted URL. Gain is the only part of
+ // a voice that tunes live; everything else is baked into what was already
+ // scheduled, so the voice reopens and the sound follows the edit the way
+ // the picture does.
+ const prev = this.voices.get(v.id);
+ if (
+ prev &&
+ (prev.url !== v.url ||
+ prev.start !== v.start ||
+ prev.in !== v.in ||
+ prev.out !== v.out ||
+ prev.speed !== v.speed)
+ ) {
+ this.release(prev);
+ this.voices.delete(v.id);
+ }
+ const live = this.voices.get(v.id) ?? this.open(v, t);
+ if (!live) continue;
+ live.gain.gain.value = Math.max(0, Math.min(1.5, v.gain));
+ if (!live.dead && !live.filling && live.scheduled < t + AHEAD_S) void this.fill(v.id, live);
+ // Windows that have finished playing hold their buffers alive.
+ live.windows = live.windows.filter((w) => w.until > t - 1);
+ }
+ for (const [id, live] of this.voices) {
+ if (seen.has(id)) continue;
+ this.release(live);
+ this.voices.delete(id);
+ }
+ }
+
+ /** Drop the decode cache for a source that has been replaced. */
+ forget(url: string): void {
+ for (const key of [...this.decoded.keys()]) {
+ if (key.startsWith(`${url}|`)) this.decoded.delete(key);
+ }
+ }
+
+ dispose(): void {
+ this.stop();
+ this.decoded.clear();
+ void this.ctx?.close();
+ this.ctx = null;
+ this.master = null;
+ }
+
+ private open(v: Voice, t: number): LiveVoice | null {
+ const ctx = this.ctx;
+ if (!ctx || !this.master) return null;
+ const gain = ctx.createGain();
+ gain.gain.value = 0;
+ gain.connect(this.master);
+ const live: LiveVoice = {
+ url: v.url,
+ speed: v.speed,
+ in: v.in,
+ out: v.out,
+ start: v.start,
+ gain,
+ windows: [],
+ // Scheduling begins where the playhead is: a voice opened mid-clip —
+ // playback started inside it, or the clip edited while audible — decodes
+ // from the moment it is heard, not from a head nobody will hear.
+ scheduled: Math.max(v.start, t),
+ filling: false,
+ dead: false,
+ };
+ this.voices.set(v.id, live);
+ return live;
+ }
+
+ private release(live: LiveVoice): void {
+ for (const w of live.windows) {
+ try {
+ w.node.stop();
+ } catch {
+ // Already finished; stopping a spent node throws and means nothing.
+ }
+ w.node.disconnect();
+ }
+ live.windows = [];
+ live.gain.disconnect();
+ }
+
+ private stopVoices(): void {
+ for (const live of this.voices.values()) this.release(live);
+ this.voices.clear();
+ }
+
+ /**
+ * Decode and schedule the next window of one voice.
+ *
+ * The window is measured in timeline seconds and the source span behind it
+ * scales with the clip's speed, so a clip at 2× reads twice as much file to
+ * fill the same stretch of timeline.
+ */
+ private async fill(id: string, live: LiveVoice): Promise<void> {
+ if (!this.ctx || !this.anchor) return;
+ live.filling = true;
+ try {
+ const stretched = Math.abs(live.speed - 1) > 1e-3;
+ const span = stretched ? WINDOW_STRETCHED_S : WINDOW_S;
+ const from = live.scheduled;
+ const sourceFrom = live.in + (from - live.start) * live.speed;
+ if (sourceFrom >= live.out - 1e-3) {
+ live.dead = true;
+ return;
+ }
+ const sourceTo = Math.min(live.out, sourceFrom + span * live.speed);
+ const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;
+ let buffer = this.decoded.get(key) ?? null;
+ if (!buffer) {
+ const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);
+ if (!raw) {
+ live.dead = true;
+ return;
+ }
+ buffer = stretched ? this.stretch(raw, 1 / live.speed) : raw;
+ // A handful of windows is all that's worth keeping — stepping back over
+ // a cut replays them, and everything older is a full decode away anyway.
+ if (this.decoded.size > 24) this.decoded.clear();
+ this.decoded.set(key, buffer);
+ }
+ // The mixer may have been stopped, or the voice dropped, while decoding.
+ const still = this.voices.get(id);
+ if (!still || still !== live || !this.anchor || !this.ctx) return;
+ const node = this.ctx.createBufferSource();
+ node.buffer = buffer;
+ node.connect(live.gain);
+ const at = this.anchor.ctx + (from - this.anchor.timeline);
+ const now = this.ctx.currentTime;
+ const until = from + buffer.duration;
+ if (until <= this.now()) {
+ // The decode outlived the window it was for — a long read, or a tab
+ // that was in the background. Skip it and let the next fill catch up.
+ live.scheduled = Math.max(live.scheduled, this.now());
+ return;
+ }
+ if (at >= now) {
+ node.start(at);
+ } else {
+ // Late: start where the clock actually is, so the sound stays lined up
+ // with the picture instead of replaying what has already gone by.
+ node.start(now, now - at);
+ }
+ live.windows.push({ node, until });
+ live.scheduled = until;
+ } catch {
+ live.dead = true;
+ } finally {
+ live.filling = false;
+ }
+ }
+
+ /** Re-lay a buffer at a different length, keeping its pitch. */
+ private stretch(buffer: AudioBuffer, factor: number): AudioBuffer {
+ const channels: Float32Array[] = [];
+ for (let c = 0; c < buffer.numberOfChannels; c++) channels.push(buffer.getChannelData(c));
+ const out = timeStretch(channels, buffer.sampleRate, factor);
+ const result = new AudioBuffer({
+ length: Math.max(1, out[0]?.length ?? 1),
+ numberOfChannels: Math.max(1, out.length),
+ sampleRate: buffer.sampleRate,
+ });
+ // A fresh view per channel: `copyToChannel` wants a plain Float32Array,
+ // and the stretch hands back views whose buffer type it declines.
+ out.forEach((data, i) => result.copyToChannel(new Float32Array(data), i));
+ return result;
+ }
+}