Open source CapCut alternative.
TypeScript
feat(site): rebuild the Cut preview on WebCodecs
David Lee8a0a9fe727f
diff --git a/site/src/cut/components/Preview.tsx b/site/src/cut/components/Preview.tsxindex 40bb358e..fd5d785c 100644--- a/site/src/cut/components/Preview.tsx+++ b/site/src/cut/components/Preview.tsx@@ -1,9 +1,16 @@"use client";-import { useEffect, useRef, useState, type RefObject } from "react";+import { useEffect, useMemo, useRef, useState, type RefObject } from "react";import { usePlayback } from "@/cut/hooks/usePlayback";import { clearAssetDrag, setAssetDragData } from "@/cut/lib/assetDrag";import { startDrag } from "@/cut/lib/drag";+import {+ playheadAt,+ previewAt,+ subscribePlayhead,+ usePreviewSelector,+ useSkim,+} from "@/cut/lib/playhead";import { getClipSpans, projectDuration, useEditor } from "@/cut/lib/store";import {capturePoster,@@ -16,19 +23,20 @@ import { clipKeyed, clipPoseAt, frameOf, isFullRect, rectOf, REGION_MAX_SCALE, timport { hasMaskKeys, type MaskKey } from "@donkeycut/effects-kit";import { cn } from "@/lib/utils";import { MaskGizmoCore, OverlayLayer } from "./OverlayLayer";-import { StageEffectPaint, stageSlices, useEffectLanes, useStageEffects } from "./StageEffects";+import {+ StageEffectPaint,+ StagePictureFx,+ stageSliceStructure,+ useEffectLanes,+} from "./StageEffects";/** The clip under the playhead, when it overflows the frame in fill mode. */-function pannableSpan(s: {- clips: VideoClip[];- assets: MediaAsset[];- currentTime: number;- aspect: Aspect;-}): ClipSpan | null {+function pannableSpan(+ s: { clips: VideoClip[]; assets: MediaAsset[]; aspect: Aspect },+ t: number+): ClipSpan | null {const spans = getClipSpans(s.clips, s.assets);- const span =- spans.find((sp) => s.currentTime >= sp.start && sp.start + sp.len > s.currentTime) ??- spans[spans.length - 1];+ const span = spans.find((sp) => t >= sp.start && sp.start + sp.len > t) ?? spans[spans.length - 1];// Pan only makes sense for a full-frame fill clip; a regioned clip is moved// with its own preview handle instead.if (!span || span.clip.fit !== "fill" || !isFullRect(rectOf(span.clip))) return null;@@ -41,6 +49,34 @@ function pannableSpan(s: {return ox > 1 || oy > 1 ? span : null;}+/**+ * Show the grab cursor while the clip under the playhead can be panned.+ *+ * Which clip that is changes sixty times a second, and the answer is one class+ * on one element. Rendering the whole preview to find out would put every+ * frame of playback through React; the class goes on from a subscription+ * instead, and only when the answer actually changes.+ */+function usePannableCursor(target: RefObject<HTMLElement | null>) {+ const clips = useEditor((s) => s.clips);+ const assets = useEditor((s) => s.assets);+ const aspect = useEditor((s) => s.aspect);+ useEffect(() => {+ const el = target.current;+ if (!el) return;+ let on: boolean | null = null;+ const apply = () => {+ const next = pannableSpan({ clips, assets, aspect }, previewAt()) !== null;+ if (next === on) return;+ on = next;+ el.classList.toggle("cursor-grab", next);+ el.classList.toggle("active:cursor-grabbing", next);+ };+ apply();+ return subscribePlayhead(apply);+ }, [target, clips, assets, aspect]);+}+/** Paint the picture this project's preview last showed, then hand the canvas* back to the engine. Cloud media is a network away, so without this the* preview holds black for about a second on every open while the first decoder@@ -73,18 +109,20 @@ function useCachedFirstFrame(canvasRef: RefObject<HTMLCanvasElement | null>) {export function Preview() {const canvasRef = useRef<HTMLCanvasElement>(null);const wrapRef = useRef<HTMLDivElement>(null);+ const stageRef = useRef<HTMLDivElement>(null);const [stage, setStage] = useState({ w: 270, h: 480 });- const pannable = useEditor((s) => pannableSpan(s) !== null);const aspect = useEditor((s) => s.aspect);const frame = frameOf(aspect);usePlayback(canvasRef);- // An effect grades what plays under it, so the stage is built in slices:- // the picture, then the elements of each lane band with the look of the- // effects above them, and each effect's paints sitting where the effect does.- const stageFx = useStageEffects();+ // An effect grades what plays under it, so the stage is built in slices: the+ // picture, then the elements of each lane band with the look of the effects+ // above them, and each effect's paints sitting where the effect does. Only+ // which lanes hold effects decides the shape, so this component never renders+ // for the clock; each slice reads the clock for itself.const effectLanes = useEffectLanes();- const { picture, slices } = stageSlices(stageFx, effectLanes);+ const slices = useMemo(() => stageSliceStructure(effectLanes), [effectLanes]);+ usePannableCursor(stageRef);useEffect(() => {setPreviewCanvas(canvasRef.current);@@ -93,6 +131,17 @@ export function Preview() {useCachedFirstFrame(canvasRef);+ // The canvas backing store matches the pixels the screen will actually show,+ // capped at the project's own frame. Painting a 4K backing store into a box+ // a few hundred pixels wide put every grade, look and mask pass through+ // millions of pixels nobody could see; the export renders at full size on its+ // own surface, so nothing about the file changes.+ const surface = useMemo(() => {+ const dpr = typeof window === "undefined" ? 1 : Math.min(2, window.devicePixelRatio || 1);+ const w = Math.max(2, Math.min(frame.w, Math.round(stage.w * dpr)));+ return { w, h: Math.max(2, Math.round((w * frame.h) / frame.w)) };+ }, [stage.w, frame.w, frame.h]);+useEffect(() => {const wrap = wrapRef.current;if (!wrap) return;@@ -115,14 +164,14 @@ export function Preview() {const s = useEditor.getState();const total = projectDuration(s);if (!total) return;- if (!s.playing && s.currentTime >= total - 0.01) s.seek(0);+ if (!s.playing && playheadAt() >= total - 0.01) s.seek(0);s.setPlaying(!s.playing);};// Drag a fill-mode clip inside the frame to choose the visible crop.const panDrag = (e: React.PointerEvent) => {const s = useEditor.getState();- const span = pannableSpan(s);+ const span = pannableSpan(s, previewAt());if (!span) return false;const fr = frameOf(s.aspect);const { width = 1, height = 1 } = span.asset;@@ -168,7 +217,7 @@ export function Preview() {const px = (e.clientX - rct.left) / rct.width;const py = (e.clientY - rct.top) / rct.height;const s = useEditor.getState();- const t = s.currentTime;+ const t = previewAt();let best: { id: string; track: number } | null = null;for (const c of s.clips) {if (c.hidden) continue;@@ -198,9 +247,9 @@ export function Preview() {so a box dragged past the frame edge stays visible and grabbable. */}<div className="relative" style={{ width: stage.w, height: stage.h }}><div+ ref={stageRef}className={cn(- "stage absolute inset-0 overflow-hidden rounded-xl bg-black shadow-[0_0_0_1px_rgba(0,0,0,0.08),0_12px_36px_rgba(0,0,0,0.18)]",- pannable && "cursor-grab active:cursor-grabbing"+ "stage absolute inset-0 overflow-hidden rounded-xl bg-black shadow-[0_0_0_1px_rgba(0,0,0,0.08),0_12px_36px_rgba(0,0,0,0.18)]")}onPointerDown={(e) => {if (@@ -229,12 +278,12 @@ export function Preview() {}}}>+ <StagePictureFx><canvasref={canvasRef}- width={frame.w}- height={frame.h}+ width={surface.w}+ height={surface.h}className="block size-full"- style={{ transform: picture.transform, filter: picture.filter }}// Drag the viewport to reference what's on screen: the clip under// the playhead travels as a media drag (timeline placement, chat// attachment, generation reference). Pan on a fill clip wins —@@ -243,7 +292,7 @@ export function Preview() {onDragStart={(e) => {const s = useEditor.getState();const spans = getClipSpans(s.clips, s.assets);- const t = s.currentTime;+ const t = previewAt();const span =spans.find((sp) => t >= sp.start && sp.start + sp.len > t) ??spans[spans.length - 1];@@ -252,20 +301,20 @@ export function Preview() {}}onDragEnd={clearAssetDrag}/>+ </StagePictureFx><ClipMaskGizmo stage={stage} />{slices.map((slice) =>slice.kind === "elements" ? (<OverlayLayerkey={slice.key}stageWidth={stage.w}- transform={slice.transform}- filter={slice.filter}+ gradeAbove={slice.gradeAbove}from={slice.from}to={slice.to}captions={slice.captions}/>) : (- <StageEffectPaint key={slice.key} states={slice.states} />+ <StageEffectPaint key={slice.key} lane={slice.lane} />))}</div>@@ -319,16 +368,24 @@ function OverlayPipHandle({ stage }: { stage: { w: number; h: number } }) {const clips = useEditor((s) => s.clips);const assets = useEditor((s) => s.assets);const aspect = useEditor((s) => s.aspect);- const currentTime = useEditor((s) => s.currentTime);- const skimTime = useEditor((s) => s.skimTime);- const playing = useEditor((s) => s.playing);+ const skimTime = useSkim();+ // The handle only cares whether its clip is on screen, so it subscribes to+ // that answer instead of to the clock: one render when the clip comes and+ // goes, rather than one per frame while it stays.+ const selectedClip = selection?.kind === "clip" ? clips.find((c) => c.id === selection.id) : null;+ const live = usePreviewSelector((t) => {+ if (!selectedClip) return false;+ const speed = selectedClip.speed && selectedClip.speed > 0 ? selectedClip.speed : 1;+ const len = Math.max(0.1, (selectedClip.out - selectedClip.in) / speed);+ return t >= selectedClip.start && t < selectedClip.start + len;+ });const [guides, setGuides] = useState<{ x: number | null; y: number | null }>({x: null,y: null,});// While hover-scrubbing the preview shows the skimmer's frame, where the// selected clip may not even be on screen — selection stays, the handle hides.- if (!playing && skimTime !== null) return null;+ if (skimTime !== null) return null;// Resolve the selected, live, regioned clip (any track) plus how to patch its// rect. A clip's own footprint equals its span length, so one path serves@@ -339,12 +396,10 @@ function OverlayPipHandle({ stage }: { stage: { w: number; h: number } }) {// far the scaled picture overhangs the box (frame px), and the pan to start// the gesture from.let pan: { id: string; ox: number; oy: number; panX0: number; panY0: number } | null = null;- if (selection?.kind === "clip") {- const clip = clips.find((c) => c.id === selection.id);+ {+ const clip = selectedClip;if (clip && !clip.hidden) {- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;- const len = Math.max(0.1, (clip.out - clip.in) / speed);- if (currentTime >= clip.start && currentTime < clip.start + len) {+ if (live) {rect = rectOf(clip);apply = (frame) => useEditor.getState().updateClipTransient(clip.id, { frame });const asset = assets.find((a) => a.id === clip.assetId);@@ -481,17 +536,19 @@ function OverlayPipHandle({ stage }: { stage: { w: number; h: number } }) {function ClipMaskGizmo({ stage }: { stage: { w: number; h: number } }) {const selection = useEditor((s) => s.selection);const clips = useEditor((s) => s.clips);- const currentTime = useEditor((s) => s.currentTime);- const skimTime = useEditor((s) => s.skimTime);- const playing = useEditor((s) => s.playing);- if (!playing && skimTime !== null) return null;- if (selection?.kind !== "clip") return null;- const clip = clips.find((c) => c.id === selection.id);- if (!clip?.mask || clip.mask.kind === "subject" || clip.hidden) return null;+ const skimTime = useSkim();+ // A keyframed mask travels with the clock, so this one does follow every+ // frame — but only while a masked clip is the selection.+ const masked =+ selection?.kind === "clip" ? clips.find((c) => c.id === selection.id) ?? null : null;+ const armed = !!masked?.mask && masked.mask.kind !== "subject" && !masked.hidden;+ const tLocal = usePreviewSelector((t) => (armed && masked ? t - masked.start : -1));+ if (skimTime !== null) return null;+ const clip = masked;+ if (!armed || !clip?.mask) return null;const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;const len = Math.max(0.1, (clip.out - clip.in) / speed);- if (currentTime < clip.start || currentTime >= clip.start + len) return null;- const tLocal = currentTime - clip.start;+ if (tLocal < 0 || tLocal >= len) return null;const rect = rectOf(clip);const pose = clipKeyed(clip) ? clipPoseAt(clip, tLocal) : null;const ax = pose ? pose.x : rect.x + rect.w / 2;diff --git a/site/src/cut/components/StageEffects.tsx b/site/src/cut/components/StageEffects.tsxindex fc675484..3fb1ea71 100644--- a/site/src/cut/components/StageEffects.tsx+++ b/site/src/cut/components/StageEffects.tsx@@ -1,17 +1,22 @@"use client";-import { useMemo } from "react";+import { useEffect, useMemo, useRef } from "react";import {- effectPreviewState,grainTileUrl,LEAK_TINT,leakGradient,streakGradient,- type EffectPreviewState,} from "@donkeycut/effects-kit";+import { previewAt, subscribePlayhead, usePreviewTime } from "@/cut/lib/playhead";import { useEditor } from "@/cut/lib/store";import { isEffectOverlay, laneOf } from "@/cut/lib/types";-import type { LiveEffect } from "@/cut/lib/effectStack";+import {+ hasEffects,+ liveEffectsAt,+ stageEffectFilter,+ stageEffectTransform,+ type LiveEffect,+} from "@/cut/lib/effectStack";import "./grain.css";/**@@ -26,27 +31,59 @@ import "./grain.css";* Both exports walk the same stack, so what plays here is what renders.*/-export { stageEffectTransform, stageSlices, type LiveEffect, type StageSlice } from "@/cut/lib/effectStack";+export { stageEffectTransform, stageSliceStructure, type LiveEffect, type StageSlice } from "@/cut/lib/effectStack";-/** Every effect live at `t`, deepest lane last. Skimming previews them too,- * the same as the elements beside them. */-export function useStageEffects(): LiveEffect[] {+/**+ * Every effect live right now, deepest lane last. Skimming previews them too,+ * the same as the elements beside them.+ *+ * This subscribes at frame rate, so only the leaves of the stage call it — the+ * paints of one lane, the elements of one band. The component that lays the+ * stage out stays off the clock.+ */+export function useLiveEffects(): LiveEffect[] {const overlays = useEditor((s) => s.overlays);- const currentTime = useEditor((s) => s.currentTime);- const skimTime = useEditor((s) => s.skimTime);- const playing = useEditor((s) => s.playing);- const t = !playing && skimTime !== null ? skimTime : currentTime;- return useMemo(- () =>- overlays- .filter(isEffectOverlay)- .filter((o) => !o.hidden && t >= o.start && t <= o.end)- .map((o) => ({- lane: laneOf(o),- state: effectPreviewState(o.effect, o.amount, t - o.start, o.focus, o.ramp, o.end - o.start),- }))- .sort((a, b) => a.lane - b.lane),- [overlays, t]+ const t = usePreviewTime();+ return useMemo(() => liveEffectsAt(overlays, t), [overlays, t]);+}++/**+ * Wear the whole stack's grade and frame motion over the picture.+ *+ * The picture is a canvas the engine paints imperatively, and its grade is two+ * style properties on the box around it. Writing them from a subscription keeps+ * a project with no effects at zero React work per frame, and one with effects+ * at two string comparisons.+ */+export function StagePictureFx({ children }: { children: React.ReactNode }) {+ const overlays = useEditor((s) => s.overlays);+ const ref = useRef<HTMLDivElement>(null);+ useEffect(() => {+ const write = (filter: string, transform: string) => {+ const style = ref.current?.style;+ if (!style) return;+ if (style.filter !== filter) style.filter = filter;+ if (style.transform !== transform) style.transform = transform;+ };+ if (!hasEffects(overlays)) {+ write("", "");+ return;+ }+ const apply = () => {+ const states = liveEffectsAt(overlays, previewAt()).map((e) => e.state);+ write(stageEffectFilter(states) ?? "", stageEffectTransform(states) ?? "");+ };+ apply();+ const stop = subscribePlayhead(apply);+ return () => {+ stop();+ write("", "");+ };+ }, [overlays]);+ return (+ <div ref={ref} className="absolute inset-0">+ {children}+ </div>);}@@ -63,7 +100,12 @@ export function useEffectLanes(): number[] {/** The frame-wide paints of the effects on one lane, over everything under* them and under everything above. */-export function StageEffectPaint({ states }: { states: EffectPreviewState[] }) {+export function StageEffectPaint({ lane }: { lane: number }) {+ const live = useLiveEffects();+ const states = useMemo(+ () => live.filter((e) => e.lane === lane).map((e) => e.state),+ [live, lane]+ );if (states.length === 0) return null;const grainUrl = grainTileUrl();const grain = Math.max(0, ...states.map((s) => s.grain ?? 0));diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 552f5594..0e64b7a9 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -1,588 +1,331 @@"use client";import { useEffect, type RefObject } from "react";-import { clipSpeed, getClipSpans, overlayLayers, projectDuration, useEditor } from "@/cut/lib/store";+import {+ clipSpeed,+ getClipSpans,+ overlayLayers,+ projectDuration,+ useEditor,+} from "@/cut/lib/store";+import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";import { isFullRect, projectFadeSeconds, rectOf } from "@/cut/lib/types";import type { ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";import { SubjectMaskCompositor } from "@/cut/lib/behindPass";import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";-import { duckGainAt, overlayPlan, prerollLead, trackZeroPlan } from "@/cut/lib/framePlan";-import { reportMediaElementError } from "@/cut/lib/mediaLinks";+import { duckGainAt, overlayPlan, trackZeroPlan } from "@/cut/lib/framePlan";+import { type ClipFrameSource, FrameSourcePool, mappingKey } from "@/cut/lib/frameSource";+import { PreviewMixer, type Voice } from "@/cut/lib/previewMixer";+import { markLiveSamples, markPresent, markTick, tracing } from "@/cut/lib/perfTrace";import { registerSourceSampler } from "@/cut/lib/previewCanvas";-// A video clip on any track is backed by a <video> for footage or an <img>-// for a still image. These helpers read either kind uniformly so the-// compositor stays one code path — an image never seeks, plays, or carries-// audio.-type MediaEl = HTMLVideoElement | HTMLImageElement;-const isImageEl = (el: MediaEl): el is HTMLImageElement =>- typeof HTMLImageElement !== "undefined" && el instanceof HTMLImageElement;-const elReady = (el: MediaEl) =>- isImageEl(el) ? el.complete && el.naturalWidth > 0 : el.readyState >= 2 && el.videoWidth > 0;-const elW = (el: MediaEl) => (isImageEl(el) ? el.naturalWidth : el.videoWidth);-const elH = (el: MediaEl) => (isImageEl(el) ? el.naturalHeight : el.videoHeight);-// A source that will never become ready: a video decode error, or an image-// that finished loading (`complete`) with no pixels (a broken/unreachable URL).-// The compositor paints through these instead of wedging on them.-const elErrored = (el: MediaEl) =>- isImageEl(el) ? el.complete && el.naturalWidth === 0 : !!el.error;-/** Browsers reject playbackRate outside roughly 0.0625–16, so the element- * rate is clamped; beyond it the periodic seek correction carries the true- * speed, at the cost of a choppier preview. Export renders the real rate. */-const safeRate = (speed: number) => Math.min(16, Math.max(0.0625, speed));--// Decode-ahead window. Every tick, clips whose entrance is within this many-// seconds of the playhead get their decoder built now and seeked to their first-// frame, so the file is already buffering (preload="auto") and frame 0 is-// decoded before the playhead reaches them — a cut lands with no cold-start-// hitch. Capped so a montage of tiny clips can't start a fetch storm that-// starves the clip actually on screen.-const WARM_HORIZON_S = 8;-const WARM_MAX = 4;+/**+ * The preview engine.+ *+ * Three things meet here and nothing else does. The frame plan says what the+ * cut looks like at an instant; the frame sources hold decoded pictures ready+ * to answer for it; the compositor paints. The engine's whole job is to pick+ * the instant, ask, and draw.+ *+ * The instant comes from the mixer's clock while playing and from the playhead+ * while paused, so sound and picture are two readings of one number rather than+ * two clocks chasing each other. A cut is not an event: at the moment the plan+ * names a different clip, that clip's frames are already decoded, because the+ * scheduler has been reading ahead of the playhead the whole time.+ *+ * The loop runs while something is moving. A paused editor with nothing dirty+ * schedules no frames at all.+ */-// Live decoder budget. A decoder is not free: it holds an open fetch against-// the media file, a demuxer, a decoded-frame buffer, and one of the small-// number of hardware decode slots the browser hands out per tab. A clip's-// element used to live from the first time the playhead came near it until the-// clip was deleted, so a long cut split many times accumulated decoders for-// every clip ever visited — all of them buffering the same file at once. Past-// the hardware limit the extras fall back to software decode, and since sound-// stays real-time while frames arrive late, the picture drifts behind the-// audio. Only reloading the page cleared it.-//-// So the pool is capped. Clips this frame is built from are never candidates;-// beyond them the least recently touched elements are torn down, leaving room-// for a few just-played clips so stepping back a cut stays hot. Rebuilding is-// cheap — the bytes are in the HTTP cache, and the same rebuild path already-// runs for stalled decoders.+/** How far ahead of the playhead a clip's decoder is opened and started. */+const WARM_HORIZON_S = 2.5;+/** Inside this, an upcoming clip is walked rather than merely opened. */+const WARM_STREAM_S = 0.75;+/** Decoders alive at once. Past the tab's hardware decode slots the rest fall+ * back to software, and frames start arriving late. */const DECODER_BUDGET = 12;-// Decoder health. A decoder can stop making progress two ways, and both used-// to be permanent for the life of the tab. It can error — and an errored-// element stays errored, because re-minting the media links does not repoint-// it: a signed URL is stable for its whole signing window, so the mint hands-// back the very string that just failed and nothing swaps. Or it can wedge-// before its first frame with nothing arriving, which nothing was watching for-// at all. Either way that clip played black for the rest of the session while-// its neighbours played fine. Decoders are cheap and replaceable, so a clip-// whose element stops advancing gets a new one, bounded so a source that-// really is unplayable settles into the paint-through path instead of-// refetching forever.-const DECODER_STALL_MS = 10_000;-const DECODER_RETRY_MS = 1_000;-const DECODER_REBUILDS = 3;--// Ahead of a clip's entrance its element is played muted and undrawn, so the-// decoder is already running across the in-point when the cut lands and the-// handoff play() resumes hot instead of spinning a cold decoder up. How long-// that roll gets is `prerollLead`, and when a clip is close enough to be worth-// keeping alive is the frame plan's `upcoming` — both live there, so the-// picture and the decoders agree on when a clip is about to be needed.+/** Source time of a clip at timeline time `t`. */+const sourceTimeOf = (clip: VideoClip, t: number) =>+ clip.in + Math.max(0, t - clip.start) * clipSpeed(clip);-const pauseEl = (el: MediaEl) => {- if (!isImageEl(el) && !el.paused) el.pause();-};+/** The decode identity of a clip — see `mappingKey`. */+const keyOf = (clip: VideoClip, asset: MediaAsset) =>+ mappingKey(asset.id, clipSpeed(clip), clip.in, clip.start);-/** Decoder pool key. Clips whose frames are identical at every timeline- * instant — same source, same speed, same source-time offset — share one- * element: srcT(t) = in + (t − start)·speed, so equal (asset, speed,- * in − start·speed) means equal pictures at all times. A tiled reveal that- * plays one source across many tracks costs one decoder instead of one per- * tile, and a split of one source hands the playing element straight across- * the cut — no resume at all. Different trims of a source keep separate- * elements, so a same-source dissolve still blends two distinct frames. */-const mappingKey = (clip: VideoClip, asset: MediaAsset) => {- const speed = clipSpeed(clip);- return `${asset.id}|${speed}|${(clip.in - clip.start * speed).toFixed(3)}`;-};-/** Build the decoder element for a clip's asset: an <img> for a still, a- * hidden <video> for footage. */-function makeMediaEl(asset: MediaAsset): MediaEl {- if (asset.type === "image") {- const img = document.createElement("img");- img.crossOrigin = "anonymous";- // Detached elements never reach the global error listener — report- // failures explicitly so an expired signed URL re-mints the batch.- img.addEventListener("error", () => reportMediaElementError(img));- img.src = asset.url;- return img;- }- const v = document.createElement("video");- v.playsInline = true;- v.preload = "auto";- v.crossOrigin = "anonymous";- v.addEventListener("error", () => reportMediaElementError(v));- v.src = asset.url;- return v;-}-/** Release an element's source. Images just drop the src; videos stop and- * unload the decoder. */-function teardown(el: MediaEl) {- if (isImageEl(el)) {- el.removeAttribute("src");- return;- }- el.pause();- el.removeAttribute("src");- el.load();-}--// The canvas backing store is the full frame resolution (1080×1920 or-// 1920×1080, set by Preview from the project aspect) so the preview stays-// sharp on Retina displays. The engine reads the size off the canvas each-// frame, so an aspect switch takes effect seamlessly.--/**- * Preview engine. One hidden <video> per track-0 clip and one <audio> per- * soundtrack clip; the active clip's video element is the master clock while- * playing and every frame is composited onto the preview canvas (contain-fit,- * matching the export's letterboxing).- */class Engine {- // Keyed by `mappingKey`, not asset id: two trims of the same source get- // their own decoders, so a cross-dissolve can show both at once and the- // incoming clip warms during the overlap instead of fighting the outgoing- // one over a single element's seek head (the black flash between- // same-source segments) — while clips that show identical frames (a tiled- // reveal, a plain split) share one.- private videoEls = new Map<string, MediaEl>();- // The overlay tracks' pool, keyed the same way, kept apart from track 0's- // so the two never trade elements mid-frame.- private overlayEls = new Map<string, MediaEl>();- private audioEls = new Map<string, HTMLAudioElement>();- // Per-decoder health, keyed like the element maps: the last progress- // reading and when it was taken, plus how many times this decoder has- // been rebuilt on the current source.- private health = new Map<string, { mark: number; at: number; rebuilds: number }>();- // Decoder recency, keyed like the element maps: the tick that last asked- // for this element. Anything below the current tick is idle and can be- // evicted; the current tick's entries are what the frame on screen is made- // of.- private used = new Map<string, number>();- // Captured entrance frames for live blend windows, keyed like the element- // maps: the bitmap the blend draws while the incoming element pre-rolls.- private heldEntrance = new Map<string, { src: string; canvas: HTMLCanvasElement }>();- private frame = 0;- // The clip array the decoder maps were last reconciled against.- private clipsSeen: VideoClip[] | null = null;+ private comp: FrameCompositor;+ private pool = new FrameSourcePool(DECODER_BUDGET, () => this.wake());+ private mixer = new PreviewMixer();+ private behind = new SubjectMaskCompositor(true);+private raf = 0;private disposed = false;- // Wall-clock stamp for advancing time where track 0 has nothing playing —- // in a gap or past its end there is no track-0 video element to act as the- // master clock.- private lastPlayNow = 0;- // Everything this engine puts on the canvas goes through here. The engine- // owns the decoders and the clock; the compositor owns the picture, and an- // export renders through the same one.- private comp: FrameCompositor;+ /** Set when something changed that the canvas has yet to show. */+ private dirty = true;+ /** Sources this frame is being drawn from. The warm pass leaves these alone:+ * a warm ask lands on the same source as a live clip whenever they share a+ * mapping — a plain split reads straight across its own cut — and steering a+ * walk that is already carrying the picture would tear it down every tick. */+ private used = new Set<ClipFrameSource>();+ /** The playhead value the engine itself wrote last, so its own echo is+ * tellable from an outside move — a seek while playing. */+ private written: number | null = null;++ private unsubscribe: () => void;+ private unwatch: () => void;+ private sizeWatch: MutationObserver;constructor(private canvas: HTMLCanvasElement) {this.comp = new FrameCompositor(canvas);this.tick = this.tick.bind(this);- this.raf = requestAnimationFrame(this.tick);+ // A moved playhead and an edited document both change the picture. Either+ // one wakes the loop; nothing else does, so an idle editor costs nothing.+ this.unsubscribe = subscribePlayhead(() => this.wake());+ this.unwatch = useEditor.subscribe(() => this.wake());+ // The backing store follows the stage box, and assigning a canvas's width+ // or height erases it. That write comes from layout — no playhead move, no+ // document edit — so the attributes themselves are the wake signal.+ this.sizeWatch = new MutationObserver(() => this.wake());+ this.sizeWatch.observe(canvas, { attributes: true, attributeFilter: ["width", "height"] });+ this.wake();}dispose() {this.disposed = true;- cancelAnimationFrame(this.raf);- for (const el of this.videoEls.values()) teardown(el);- for (const el of this.overlayEls.values()) teardown(el);- for (const el of this.audioEls.values()) el.pause();- this.videoEls.clear();- this.overlayEls.clear();- this.audioEls.clear();- this.health.clear();- this.used.clear();- this.heldEntrance.clear();- }-- /** Release one decoder. The element maps are disjoint by key, so this- * serves track 0 and the overlay tracks alike. */- private dropDecoder(key: string) {- const el = this.videoEls.get(key) ?? this.overlayEls.get(key);- if (el) teardown(el);- this.videoEls.delete(key);- this.overlayEls.delete(key);- this.health.delete(key);- this.used.delete(key);- this.heldEntrance.delete(key);- }-- /** Hold the live decoder pool to `DECODER_BUDGET`, oldest first. Elements- * touched during the tick that just ran are the frame on screen — the- * master, its transition partner, the pre-rolling next clip, the live- * overlays, the warm set — and are never candidates, so a busy frame simply- * evicts less rather than tearing down something it is about to draw. */- private evictIdleDecoders() {- const total = this.videoEls.size + this.overlayEls.size;- if (total <= DECODER_BUDGET) return;- const idle = [...this.videoEls.keys(), ...this.overlayEls.keys()]- .filter((id) => (this.used.get(id) ?? 0) < this.frame)- .sort((a, b) => (this.used.get(a) ?? 0) - (this.used.get(b) ?? 0));- for (const id of idle.slice(0, total - DECODER_BUDGET)) this.dropDecoder(id);- }-- /** The cached element for a key, rebuilt when its source no longer matches —- * a swap keeps the clip but repoints the asset (a shot re-render), and the- * old decoder would otherwise keep playing (or erroring on) the old file. */- private elFor(map: Map<string, MediaEl>, key: string, asset: MediaAsset): MediaEl {- // Asking for an element is what marks it in use — every path that- // draws, seeks, warms or pre-rolls a clip comes through here, so recency- // needs no bookkeeping at the call sites.- this.used.set(key, this.frame);- let el = map.get(key);- if (el && el.getAttribute("src") !== asset.url) {- teardown(el);- // A repoint is a new source, so it starts with a full rebuild budget.- this.health.delete(key);- el = undefined;- }- if (el && this.spent(key, el)) {- teardown(el);- el = undefined;- }- if (!el) {- el = makeMediaEl(asset);- map.set(key, el);- }- return el;- }-- /** Whether this clip's decoder has stopped making progress and should be- * replaced. A ready element is healthy and clears its streak; one whose- * readiness and buffer are both still advancing is on its way there. What is- * left is an element that errored, or that has sat at the same reading past- * `DECODER_STALL_MS` — dead either way, and worth one more decoder. Says yes- * at most `DECODER_REBUILDS` times per source, so an unplayable file stops- * costing fetches and falls through to the paint-through path. */- private spent(key: string, el: MediaEl): boolean {- const now = performance.now();- const h = this.health.get(key) ?? { mark: -1, at: now, rebuilds: 0 };- if (elReady(el)) {- this.health.set(key, { mark: -1, at: now, rebuilds: 0 });- return false;- }- // Progress reading, not a clock: readiness plus how far the buffer reaches,- // so a big file arriving slowly over a thin link is never mistaken for a- // wedge and torn down mid-fetch.- const mark = isImageEl(el)- ? 0- : el.readyState + (el.buffered.length ? el.buffered.end(el.buffered.length - 1) : 0);- if (mark > h.mark) {- this.health.set(key, { ...h, mark, at: now });- return false;- }- const since = now - h.at;- // A hidden document defers media loads, so no progress there is no verdict.- const visible = typeof document === "undefined" || document.visibilityState === "visible";- const dead = visible && (elErrored(el) ? since > DECODER_RETRY_MS : since > DECODER_STALL_MS);- if (!dead || h.rebuilds >= DECODER_REBUILDS) {- this.health.set(key, { ...h, mark });- return false;- }- this.health.set(key, { mark: -1, at: now, rebuilds: h.rebuilds + 1 });- console.debug(- `[cut-preview] decoder rebuild: ${key} ` +- `(${elErrored(el) ? "errored" : "stalled"}, attempt ${h.rebuilds + 1}/${DECODER_REBUILDS})`- );- return true;- }-- private videoFor(clip: VideoClip, asset: MediaAsset): MediaEl {- return this.elFor(this.videoEls, mappingKey(clip, asset), asset);- }-- /** Decode-ahead for track 0: build each soon-to-enter clip's element- * and seek it to its entrance frame now, so its file is fetching and frame 0- * is decoded before the playhead arrives. Only clips strictly ahead of `t`- * are touched — the live master (and any dissolve partner) steers its own- * clock through `composite`. Bounded by `WARM_HORIZON_S`/`WARM_MAX`; spans are- * start-ordered, so we can stop once one is past the horizon. */- private warmAhead(spans: ClipSpan[], t: number, activeKey: string | null) {- let warmed = 0;- // One parking seek per element: spans sharing a decoder would otherwise- // pull its seek head between their targets every tick. Start-ordered, so- // the nearest entrance wins; the active element is the live picture and- // is never parked out from under it.- const parked = new Set<string>();- for (let i = 0; i < spans.length; i++) {- const span = spans[i];- if (span.start <= t) continue; // current or past — not ours to warm- if (span.start > t + WARM_HORIZON_S) break;- const key = mappingKey(span.clip, span.asset);- if (key === activeKey || parked.has(key)) continue;- parked.add(key);- const speed = clipSpeed(span.clip);- // A clip a transition blends in shows its entrance frame for the whole- // blend window, so it parks exactly there, as early as possible — a- // seek at window-open would leave the frame undecodable mid-blend, and- // the push/wipe geometry would carve a black region where the picture- // belongs. It never pre-rolls: the blend is its warm-up.- const blended = (spans[i - 1]?.transitionOut ?? 0) > 0;- const lead = blended ? 0 : prerollLead(span.clip.in, speed);- // The imminent clip inside its own pre-roll window is `warmNext`'s to- // play hot; a parking seek here would fight it, so leave it alone. A clip- // with no roll to give (an untrimmed one) has no such window, and stays- // parked here right up to the cut.- if (lead > 0 && span.start - t <= lead) continue;- const el = this.videoFor(span.clip, span.asset); // creating it starts the fetch- // Park a not-yet-imminent clip exactly where the pre-roll will play from- // (`warmNext`'s `from`), not on its entrance frame: parking at `in` and- // then seeking back for the roll-in discards the warmed buffer, and over- // the network (cloud media) that late re-seek is what used to stall the- // handoff. Parked here, the pre-roll's seek is a no-op on already-buffered- // bytes and the cut lands hot.- if (!isImageEl(el) && !el.seeking && el.paused) {- const target = span.clip.in - lead * speed;- if (Math.abs(el.currentTime - target) > 0.1) el.currentTime = target;- }- if (++warmed >= WARM_MAX) break;- }+ if (this.raf) cancelAnimationFrame(this.raf);+ this.unsubscribe();+ this.unwatch();+ this.sizeWatch.disconnect();+ this.pool.closeAll();+ this.mixer.dispose();+ }++ /**+ * The height sources decode at: the canvas, rounded up to a coarse step.+ *+ * The pool keys sources on this number. Keyed on the raw canvas height, a+ * divider drag — which resizes the backing store pixel by pixel — would mint+ * a fresh decoder for every intermediate size and blow through the tab's+ * decode slots in a second. Rounding up keeps the identity still while the+ * box moves and never decodes below display size, and the step is small+ * enough that the extra rows decoded above it stay cheap.+ */+ private decodeHeight(): number {+ return Math.ceil(this.canvas.height / 180) * 180 || 180;+ }++ /** The engine's own playhead writes go through here, so `written` always+ * says what the engine last wrote. */+ private writeHead(t: number) {+ this.written = t;+ setPlayhead(t);+ }++ /** Something changed: draw at the next opportunity. */+ private wake() {+ this.dirty = true;+ this.schedule();+ }++ /**+ * Ask for one frame, and only one.+ *+ * Drawing writes the playhead, and writing the playhead wakes the engine, so+ * a tick that scheduled its own successor unconditionally would leave two+ * callbacks pending where it meant to leave one — and each of those would+ * leave two more. Every path books a frame through here, and here refuses to+ * book a second.+ */+ private schedule() {+ if (this.raf || this.disposed) return;+ this.raf = requestAnimationFrame(this.tick);}- /** Decode-ahead counterpart for overlay tracks: warm each overlay clip whose- * entrance is within the horizon (overlay clips aren't start-ordered, so this- * scans rather than breaking early). A warmed element sits paused on its first- * frame until the tick's overlay path takes it live. */- private warmOverlaysAhead(t: number) {+ /** The clip's raw, ungraded decoder frame for analysis (the color panel's+ * Auto), or null when nothing has been decoded for it. */+ sourceFor(clipId: string): CanvasImageSource | null {const s = useEditor.getState();- // Elements live on screen at `t` — with shared decoders an upcoming clip- // can ride the same element a playing clip is steering, and a parking- // seek would yank the live picture.- const live = new Set<string>();- for (const c of overlayLayers(s.clips)) {- if (c.hidden || c.start > t) continue;- if (t >= c.start + (c.out - c.in) / clipSpeed(c)) continue;- const asset = s.assets.find((a) => a.id === c.assetId);- if (asset) live.add(mappingKey(c, asset));- }- let warmed = 0;- const parked = new Set<string>();- for (const c of overlayLayers(s.clips)) {- if (c.hidden || c.start <= t || c.start > t + WARM_HORIZON_S) continue;- const asset = s.assets.find((a) => a.id === c.assetId);- if (!asset) continue;- const key = mappingKey(c, asset);- if (live.has(key) || parked.has(key)) continue;- parked.add(key);- const el = this.overlayVideoFor(c, asset);- if (!isImageEl(el) && el.paused && !el.seeking && Math.abs(el.currentTime - c.in) > 0.1) {- el.currentTime = c.in;+ const clip = s.clips.find((c) => c.id === clipId);+ const asset = clip && s.assets.find((a) => a.id === clip.assetId);+ if (!clip || !asset) return null;+ const src = this.pool.get(keyOf(clip, asset), asset, this.decodeHeight());+ return src.frameAt(sourceTimeOf(clip, previewAt()))?.image ?? null;+ }++ /**+ * The picture a clip shows at timeline time `t`.+ *+ * Asking is never a wait. A source with the exact frame gives it; one running+ * behind gives the nearest it has, which is a held frame rather than a hole;+ * one that has decoded nothing yet is `pending`, which tells the compositor to+ * leave what is already on screen alone. Painting black for a decoder that is+ * merely a moment behind is what strobing looks like.+ */+ private frameFor(span: ClipSpan, t: number, playing: boolean): Frame {+ const src = this.pool.get(keyOf(span.clip, span.asset), span.asset, this.decodeHeight());+ this.used.add(src);+ const st = sourceTimeOf(span.clip, t);+ src.want(st, playing);+ if (src.failed) return MISSING_FRAME;+ const frame = src.frameAt(st);+ if (!frame) return PENDING_FRAME;+ return { kind: "ready", image: frame.image, width: frame.width, height: frame.height };+ }++ /** Open and start the decoders for clips about to arrive — on track 0 and+ * the overlay tracks alike — so a cut or an overlay's entrance lands on a+ * ring that is already full. */+ private warm(t: number, playing: boolean) {+ const s = useEditor.getState();+ const lists = [getClipSpans(s.clips, s.assets)];+ for (const track of new Set(overlayLayers(s.clips).map((c) => c.track))) {+ lists.push(getClipSpans(s.clips, s.assets, track));+ }+ for (const spans of lists) {+ for (const sp of spans) {+ if (sp.start <= t || sp.start > t + WARM_HORIZON_S) continue;+ const src = this.pool.get(keyOf(sp.clip, sp.asset), sp.asset, this.decodeHeight());+ // A source already carrying this frame's picture — or already warmed+ // for a nearer span — needs nothing. Its walk reads across the join on+ // its own, and a second ask would restart it.+ if (this.used.has(src)) continue;+ this.used.add(src);+ // Start where the clip opens. A walk is the useful thing to have ready —+ // the first step past a join then costs one frame rather than a seek —+ // but a walk started for every clip on the horizon spends decode a drag+ // elsewhere on the timeline is waiting for. So the clip about to be+ // reached gets a walk, and the ones behind it get the single frame that+ // lets a cut land on something.+ const imminent = sp.start - t <= WARM_STREAM_S;+ src.want(sp.clip.in, playing || imminent);}- if (++warmed >= WARM_MAX) break;}}- /** Pre-roll the imminent next clip so its handoff is hot. A decode-ahead- * element sits paused on its entrance frame with a cold decode pipeline, so- * the handoff `play()` has to spin the decoder up — and past a trimmed- * in-point, decode forward from the prior keyframe — before the clock- * advances: the residual hitch at a cut, and the freeze that parking the- * element back on its frame never cured. Instead, in the moments before the- * cut, play the element muted and undrawn from `lead` of source before its- * entrance, so it is already running across the in-point — arriving there as- * the playhead reaches the cut — and the real `play()` resumes hot with- * nothing skipped. Rolling only makes sense while the playhead is moving:- * paused inside the window, the element parks on the seat, and a roll left- * over from playback is stopped — an unwatched roll runs past the cut to the- * end of the file and restarts forever. */- private warmNext(span: ClipSpan, t: number, play: boolean) {- const el = this.videoFor(span.clip, span.asset);- if (isImageEl(el)) return; // a still needs no pipeline- const speed = clipSpeed(span.clip);- // Only as long a roll as there is source ahead of the in-point to cover it- // — nothing at all for an untrimmed clip, which stays parked on its already- // hot keyframe 0 instead. Farther out than that, it is warmAhead's to park- // and buffer.- const lead = prerollLead(span.clip.in, speed);- if (t < span.start - lead) return;- const rate = safeRate(speed);- if (el.playbackRate !== rate) el.playbackRate = rate;- el.muted = true; // silent until it becomes master and unmutes- // Seat it `lead` of source before the entrance; playing, it then rolls- // forward so it reaches `in` right as the playhead reaches the cut.- const from = span.clip.in - lead * speed;- if (!play) {- if (!el.paused) el.pause();- else if (!el.seeking && Math.abs(el.currentTime - from) > 0.1) el.currentTime = from;- return;- }- // Already rolling: let it run — it crosses `in` on its own as the cut lands.- if (!el.paused) return;- if (!el.seeking && Math.abs(el.currentTime - from) > 0.1) el.currentTime = from;- if (el.readyState >= 2 && !el.seeking) void el.play().catch(() => {});+ /** Overlay clips live at `t`, with their assets and ramps, in draw order. */+ private liveOverlays(t: number) {+ const s = useEditor.getState();+ return overlayPlan(+ [...new Set(overlayLayers(s.clips).map((c) => c.track))],+ (track) => getClipSpans(s.clips, s.assets, track),+ t+ );}- /** Park a clip's element on its entrance frame, paused and silent — the- * picture a transition blends in before the clip starts playing. */- private holdAtEntrance(span: ClipSpan): MediaEl {- const el = this.videoFor(span.clip, span.asset);- if (isImageEl(el)) return el;- el.muted = true;- if (!el.paused) el.pause();- else if (!el.seeking && Math.abs(el.currentTime - span.clip.in) > 0.05) {- el.currentTime = span.clip.in;- }- return el;+ /** Whole-video fade gain at `t`: ramps 0→1 over the project fade-in and 1→0+ * over the fade-out at the end of the cut. */+ private projectFadeGain(t: number, total: number) {+ const s = useEditor.getState();+ const fadeIn = projectFadeSeconds(s.fadeIn, total);+ const fadeOut = projectFadeSeconds(s.fadeOut, total);+ let g = 1;+ if (fadeIn > 0 && t < fadeIn) g = Math.min(g, Math.max(0, t / fadeIn));+ if (fadeOut > 0 && t > total - fadeOut) g = Math.min(g, Math.max(0, (total - t) / fadeOut));+ return Math.min(1, g);}- /** The incoming side of a live blend: the entrance frame the window draws,- * and a decoder made hot for the cut about to land. The element holds- * parked on its entrance frame only until that frame has been captured- * once; from there the bitmap carries the blend's picture and the element- * itself pre-rolls muted toward the in-point (`warmNext`), so the handoff- * `play()` resumes a decoder that is already running. Holding the element- * itself for the whole window meant resuming it cold at the cut — and past- * a trimmed in-point, decoding forward from the prior keyframe — which- * froze the picture and the playhead right as the transition landed. */- private blendIncoming(span: ClipSpan, t: number, play: boolean): Frame {- const el = this.videoFor(span.clip, span.asset);- if (isImageEl(el)) return this.frameOf(el); // a still is its own held frame- const key = mappingKey(span.clip, span.asset);- const src = `${span.asset.url}#${span.clip.in.toFixed(3)}`;- let held = this.heldEntrance.get(key);- if (held && held.src !== src) held = undefined;- if (!held) {- const parkedOnEntrance =- elReady(el) && el.paused && !el.seeking && Math.abs(el.currentTime - span.clip.in) <= 0.05;- if (!parkedOnEntrance) {- // Nothing to capture yet: hold the element on the frame, as before.- this.holdAtEntrance(span);- return this.frameOf(el);- }- const canvas = document.createElement("canvas");- canvas.width = elW(el);- canvas.height = elH(el);- canvas.getContext("2d")?.drawImage(el, 0, 0);- held = { src, canvas };- this.heldEntrance.set(key, held);+ /**+ * Everything audible at `t`, with the gain the frame plan gives it.+ *+ * The same ramps that dim the picture dim the sound: a clip fading out of a+ * dissolve takes its audio with it, an upper-track clip's transition carries+ * its own, and a live voiceover ducks the rest.+ */+ private voicesAt(t: number, spans: ClipSpan[], master: ClipSpan | undefined): Voice[] {+ const s = useEditor.getState();+ const out: Voice[] = [];+ const duck = duckGainAt(s.audioClips, t);+ if (master && !master.clip.muted && !master.clip.hidden) {+ const plan = trackZeroPlan(master, spans, t);+ out.push({+ id: master.clip.id,+ url: master.asset.url,+ start: master.start,+ in: master.clip.in,+ out: master.clip.out,+ speed: clipSpeed(master.clip),+ gain: plan.gain * duck * (master.clip.volume ?? 1),+ });}- this.warmNext(span, t, play);- return { kind: "ready", image: held.canvas, width: held.canvas.width, height: held.canvas.height };- }-- /** Seek/rate/play one clip's element toward its frame at timeline time `t`,- * without touching any other element (the caller pauses stale ones). */- private prepare(span: ClipSpan, t: number, play: boolean, muted: boolean): MediaEl {- const el = this.videoFor(span.clip, span.asset);- // A still never seeks, plays, or carries audio — it's ready as soon as the- // <img> decodes. Skip every video-clock operation.- if (isImageEl(el)) return el;- const speed = clipSpeed(span.clip);- const rate = safeRate(speed);- if (el.playbackRate !== rate) el.playbackRate = rate;- const target = span.clip.in + Math.max(0, t - span.start) * speed;- // While playing, the element is its own clock and advances on its own, so- // only re-seek on a real jump (a clip switch or a scrub) — never for the- // sub-second lag between this frame's `target` (built from last frame's- // clock read) and the freely-running element. At high speed that lag is- // `speed × frameInterval` every frame, which a tight threshold would seek- // backward each tick, stalling playback. When paused (scrubbing) keep it- // tight so the frame under the mouse tracks precisely.- // Let an in-flight seek finish before issuing the next one — restarting- // the decoder every mousemove makes scrubbing stutter.- const tol = play ? 0.34 : 0.05;- if (Math.abs(el.currentTime - target) > tol && !el.seeking) el.currentTime = target;- el.muted = muted;- if (play) {- if (el.paused && el.readyState >= 2) void el.play().catch(() => {});- } else if (!el.paused) {- el.pause();+ for (const { clip, asset, gain } of this.liveOverlays(t)) {+ if (clip.muted || clip.hidden || asset.type === "image") continue;+ out.push({+ id: clip.id,+ url: asset.url,+ start: clip.start,+ in: clip.in,+ out: clip.out,+ speed: clipSpeed(clip),+ gain: gain * (clip.volume ?? 1) * duck,+ });}- return el;- }-- private pauseExcept(keep: Set<string>) {- for (const [clipId, el] of this.videoEls) {- if (!keep.has(clipId)) pauseEl(el);+ for (const a of s.audioClips) {+ const asset = s.assets.find((x) => x.id === a.assetId);+ if (!asset || a.hidden) continue;+ const speed = a.speed && a.speed > 0 ? a.speed : 1;+ const len = Math.max(0.1, (a.out - a.in) / speed);+ if (t < a.start || t >= a.start + len) continue;+ // Fade envelope: linear ramps at either end of the clip.+ const rel = t - a.start;+ const fi = a.fadeIn ?? 0;+ const fo = a.fadeOut ?? 0;+ let g = 1;+ if (fi > 0 && rel < fi) g *= rel / fi;+ if (fo > 0 && rel > len - fo) g *= Math.max(0, (len - rel) / fo);+ // A ducking voiceover never ducks itself, or the others that duck.+ const dg = a.duck !== undefined && a.duck < 1 ? 1 : duck;+ out.push({+ id: a.id,+ url: asset.url,+ start: a.start,+ in: a.in,+ out: a.out,+ speed,+ gain: a.volume * g * dg,+ });}+ return out;}- /** Draw a video element into a sub-region of the frame. "fill" covers the- * region and crops the overflow (clipped to the rect); "fit" contains the- * whole picture inside it, centered. `zoom` scales the picture around the- * region's center (zoom transitions), clipping the overflow to the rect. */- /** The clip's raw, ungraded decoder frame for analysis (the color panel's- * Auto), or null when no ready decoder exists for the clip. */- sourceFor(clipId: string): CanvasImageSource | null {- const s = useEditor.getState();- const clip = s.clips.find((c) => c.id === clipId);- const asset = clip && s.assets.find((a) => a.id === clip.assetId);- if (!clip || !asset) return null;- const key = mappingKey(clip, asset);- const el = this.videoEls.get(key) ?? this.overlayEls.get(key);- return el && elReady(el) ? el : null;- }-- /** The picture a media element is currently showing, in the form the- * compositor takes. An element that exists but has no decodable frame is- * `pending`, which is what keeps a skim from strobing black. */- private frameOf(el: MediaEl | null | undefined): Frame {- if (!el) return MISSING_FRAME;- if (!elReady(el)) return PENDING_FRAME;- return { kind: "ready", image: el, width: elW(el), height: elH(el) };- }-- /** Draw `masterSpan` full-frame, plus any transition live at time `t` in- * its style's geometry (blend, dip, push, wipe, shape reveal), plus the- * clips' own entrance/exit animations. Returns the master element (the- * playback clock). */- private composite(masterSpan: ClipSpan, spans: ClipSpan[], t: number, play: boolean) {- // A hidden clip is silent as well as black.- // What the frame should be at `t` — the same answer an export gets.- const plan = trackZeroPlan(masterSpan, spans, t);- // A hidden clip is silent as well as black.- const masterEl = this.prepare(masterSpan, t, play, masterSpan.clip.muted || !!masterSpan.clip.hidden);- const masterKey = mappingKey(masterSpan.clip, masterSpan.asset);- const keep = new Set([masterKey]);- // Prime the next clip's decoder+audio pipeline shortly before its entrance- // (the dissolve start, or the hard cut) so the handoff `play()` resumes hot- // — no cold-start spin-up freezing the picture and playhead at the cut.- // A clip sharing the master's element (a split of the same source) needs- // neither: the element crosses the cut already playing.- if (plan.upcoming && plan.upcoming !== plan.incoming) {- const upKey = mappingKey(plan.upcoming.clip, plan.upcoming.asset);- if (upKey !== masterKey) {- if (masterSpan.transitionOut > 0) this.holdAtEntrance(plan.upcoming);- else this.warmNext(plan.upcoming, t, play);- keep.add(upKey);- }- }- // The incoming side of a live blend. Each trim owns its element, so the- // two sides decode side by side — a true blend even between trims of the- // same source. A split sharing the master's mapping draws the master's own- // frames (the pictures are identical by construction); everything else- // shows its captured entrance frame while its element pre-rolls for the- // handoff (`blendIncoming`).- let incFrame: Frame = MISSING_FRAME;- if (plan.incoming) {- const incKey = mappingKey(plan.incoming.clip, plan.incoming.asset);- if (incKey === masterKey) {- incFrame = this.frameOf(masterEl);- } else {- incFrame = this.blendIncoming(plan.incoming, t, play);- keep.add(incKey);- }- }- // A live voiceover ducks the master clip's sound under it. The clip's own- // volume rides on top (the element clamps at 1; export honors up to 1.5).- const duck = duckGainAt(useEditor.getState().audioClips, t);- if (!isImageEl(masterEl)) {- masterEl.volume = Math.max(- 0,- Math.min(1, plan.gain * duck * (masterSpan.clip.volume ?? 1))+ /** Draw track 0 at `t`: the master clip, whatever is blending into it, the+ * neighbour frame behind an edge animation, and the clip's own veil. */+ private drawTrackZero(+ master: ClipSpan,+ spans: ClipSpan[],+ t: number,+ playing: boolean,+ masterFrame: Frame+ ) {+ const plan = trackZeroPlan(master, spans, t);+ // The incoming side of a live blend decodes alongside the outgoing one, so+ // a dissolve blends two real pictures — including between two trims of the+ // same file, which keep their own sources by construction.+ const incFrame = plan.incoming+ ? this.frameFor(plan.incoming, Math.max(plan.incoming.start, t), playing)+ : MISSING_FRAME;+ if (plan.backdrop) {+ // A neighbour's held frame behind a live edge animation, drawn at the+ // exact source moment the plan asks for.+ const b = plan.backdrop;+ const src = this.pool.get(keyOf(b.span.clip, b.span.asset), b.span.asset, this.decodeHeight());+ this.used.add(src);+ src.want(b.at, false);+ const f = src.frameAt(b.at);+ this.comp.drawLayer(+ f ? { kind: "ready", image: f.image, width: f.width, height: f.height } : PENDING_FRAME,+ b.span.clip,+ false,+ 1,+ t);}- this.pauseExcept(keep);- // No clear here — the tick clears once before compositing, so a regioned- // clip draws into its rect over the already-black frame.- if (plan.backdrop) this.drawBackdropFrame(plan.backdrop.span, plan.backdrop.at, t);this.comp.drawCrossJoin(plan.style,plan.p,{- masterFrame: this.frameOf(masterEl),- masterClip: masterSpan.clip,+ masterFrame,+ masterClip: master.clip,masterAlpha: plan.masterAlpha,masterZoom: plan.masterZoom,masterFx: {@@ -597,506 +340,164 @@ class Engine {t);// Veil only the master clip's own footprint, like the export's per-clip- // fade filter: a regioned clip darkens inside its rect while a track- // behind shows through the margins; tracks drawn after (above) stay lit.- if (plan.veil > 0) this.comp.fillBlackVeil(plan.veil, rectOf(masterSpan.clip));- return masterEl;- }-- private overlayVideoFor(clip: VideoClip, asset: MediaAsset): MediaEl {- return this.elFor(this.overlayEls, mappingKey(clip, asset), asset);- }-- // The subject-mask pass: behind-tagged elements leave the DOM overlay path- // and composite here — video, then the element raster, then the segmented- // person back on top — and the pass publishes the matte the DOM's front- // subject-masked elements read (see behindPass.ts). Cheap no-op when- // nothing reads the matte. Subject-masked video clips pull their matte- // mid-stack through the compositor's provider, reading the canvas as it- // stands beneath them.- private behind = new SubjectMaskCompositor(true);-- private drawBehind(t: number) {- const s = useEditor.getState();- this.comp.subjectMatteProvider = (at) => this.behind.clipMatteOf(this.canvas, at);- this.behind.draw(this.canvas, s.overlays, s.assets, t);- }-- // Effect elements are not a canvas pass: they filter the finished picture,- // elements included, so the stage applies them over everything (see- // StageEffects) exactly as the export does.-- /** Draw a neighbor clip's held frame full-frame beneath an edge animation.- * A paused element parks on the wanted frame (the previous clip already- * rests at its out point after the handoff; warm-ahead parks the next one- * on its entrance frame); a pre-rolling element draws where it is — frames- * moments ahead of its entrance, close enough at the cut. Until the- * element has a decodable frame this draws nothing and the tick's black- * clear shows, same as before. */- private drawBackdropFrame(span: ClipSpan, at: number, t: number) {- const el = this.videoFor(span.clip, span.asset);- if (!isImageEl(el) && el.paused && !el.seeking && Math.abs(el.currentTime - at) > 0.15) {- el.currentTime = Math.max(0, at);- }- this.comp.drawLayer(this.frameOf(el), span.clip, false, 1, t);- }-- /** Whole-video fade gain at time `t`: ramps 0→1 over the project fade-in and- * 1→0 over the fade-out at the end of the cut. 1 when neither applies. */- private projectFadeGain(t: number, total: number) {- const s = useEditor.getState();- const fadeIn = projectFadeSeconds(s.fadeIn, total);- const fadeOut = projectFadeSeconds(s.fadeOut, total);- let g = 1;- if (fadeIn > 0 && t < fadeIn) g = Math.min(g, Math.max(0, t / fadeIn));- if (fadeOut > 0 && t > total - fadeOut) g = Math.min(g, Math.max(0, (total - t) / fadeOut));- return Math.min(1, g);- }-- /** Overlay clips live at time `t`, with their assets and transition ramps,- * in draw order. */- private liveOverlays(t: number) {- const s = useEditor.getState();- return overlayPlan(- [...new Set(overlayLayers(s.clips).map((c) => c.track))],- (track) => getClipSpans(s.clips, s.assets, track),- t- );- }-- /** Seek/rate/play one overlay clip's element toward its frame at timeline- * time `t` (the overlay counterpart of `prepare`). `gain` carries the- * clip's transition ramp, so a fading picture takes its sound with it. */- private prepareOverlay(clip: VideoClip, asset: MediaAsset, t: number, play: boolean, gain = 1): MediaEl {- const el = this.overlayVideoFor(clip, asset);- if (isImageEl(el)) return el;- const speed = clip.speed && clip.speed > 0 ? clip.speed : 1;- const rate = safeRate(speed);- if (el.playbackRate !== rate) el.playbackRate = rate;- const target = clip.in + Math.max(0, t - clip.start) * speed;- const tol = play ? 0.34 : 0.05;- if (Math.abs(el.currentTime - target) > tol && !el.seeking) el.currentTime = target;- // Overlay audio previews like the export mixes it: the clip's own volume,- // ducked under a live voiceover, silent when muted. (The tick dims it- // further with the project fade.)- el.muted = !!clip.muted;- el.volume = Math.max(- 0,- Math.min(1, gain * (clip.volume ?? 1) * duckGainAt(useEditor.getState().audioClips, t))- );- if (play) {- if (el.paused && el.readyState >= 2) void el.play().catch(() => {});- } else if (!el.paused) {- el.pause();- }- return el;- }-- /** The live overlays, each already primed toward `t`. Computed once so the- * skim path's readiness check and the draw step share the same- * filter/sort/seek instead of repeating it per frame. */- private prepareOverlays(t: number, play: boolean) {- return this.liveOverlays(t).map(({ clip, asset, alpha, zoom, gain }) => ({- clip,- key: mappingKey(clip, asset),- alpha,- zoom,- el: this.prepareOverlay(clip, asset, t, play, gain),- }));- }-- /** Draw the overlay tracks over track 0 in z-order (further-back first). A- * full-frame clip covers what's under it; a regioned one shares the frame,- * letting lower tracks show in its margins. Collects the clips it touched- * into `active`. Pass `prepared` (from `prepareOverlays`) to reuse an- * already-primed set. */- private drawOverlays(- t: number,- play: boolean,- active: Set<string>,- prepared?: { clip: VideoClip; key: string; el: MediaEl; alpha: number; zoom: number }[]- ) {- for (const { clip, key, el, alpha, zoom } of prepared ?? this.prepareOverlays(t, play)) {- active.add(key);- if (!elReady(el)) continue;+ // fade filter: a regioned clip darkens inside its rect while a track behind+ // shows through the margins.+ if (plan.veil > 0) this.comp.fillBlackVeil(plan.veil, rectOf(master.clip));+ }++ /** Draw the overlay tracks over track 0, further-back first. */+ private drawOverlays(t: number, playing: boolean) {+ for (const { clip, asset, alpha, zoom } of this.liveOverlays(t)) {+ const span: ClipSpan = {+ clip,+ asset,+ start: clip.start,+ len: Math.max(0.1, (clip.out - clip.in) / clipSpeed(clip)),+ transitionOut: 0,+ };+ const frame = this.frameFor(span, t, playing);+ if (frame.kind !== "ready") continue;const rect = rectOf(clip);const cover = clip.fit === "fill" || (clip.fit == null && isFullRect(rect));- this.comp.drawIntoRect(this.frameOf(el), rect, cover, alpha, t, zoom, clip);- }- }-- /** Pause overlay elements not drawn this frame; drop those no clip needs. */- private cleanupOverlays(active: Set<string>) {- const s = useEditor.getState();- const live = new Set<string>();- for (const c of overlayLayers(s.clips)) {- const asset = s.assets.find((a) => a.id === c.assetId);- if (asset) live.add(mappingKey(c, asset));- }- for (const [key, el] of this.overlayEls) {- if (active.has(key)) continue;- pauseEl(el);- if (!live.has(key)) this.dropDecoder(key);+ this.comp.drawIntoRect(frame, rect, cover, alpha, t, zoom, clip);}}- private syncSoundtrack(t: number, playing: boolean, fadeGain = 1) {- const s = useEditor.getState();- const live = new Set<string>();- for (const a of s.audioClips) {- live.add(a.id);- const asset = s.assets.find((x) => x.id === a.assetId);- if (!asset) continue;- let el = this.audioEls.get(a.id);- // Same-id source swaps (a regenerated voiceover) rebuild the element.- if (el && el.getAttribute("src") !== asset.url) {- el.pause();- el = undefined;- }- if (!el) {- const audio = new Audio();- audio.preload = "auto";- audio.addEventListener("error", () => reportMediaElementError(audio));- audio.src = asset.url;- el = audio;- this.audioEls.set(a.id, el);- }- // Detached audio can carry its video clip's rate; footprint is (out-in)/speed.- const speed = a.speed && a.speed > 0 ? a.speed : 1;- const len = Math.max(0.1, (a.out - a.in) / speed);- // A hidden clip is muted from the mix — keep its element but never play it.- const active = playing && !a.hidden && t >= a.start && t < a.start + len;- if (active) {- // Fade envelope: linear ramps at either end of the clip.- const rel = t - a.start;- const fi = a.fadeIn ?? 0;- const fo = a.fadeOut ?? 0;- let gain = 1;- if (fi > 0 && rel < fi) gain *= rel / fi;- if (fo > 0 && rel > len - fo) gain *= Math.max(0, (len - rel) / fo);- // A live voiceover ducks the other soundtrack clips (music) too;- // ducking clips never duck each other.- const dg = a.duck !== undefined && a.duck < 1 ? 1 : duckGainAt(s.audioClips, t);- el.volume = Math.max(0, Math.min(1, a.volume * gain * dg * fadeGain));- const rate = safeRate(speed);- if (el.playbackRate !== rate) el.playbackRate = rate;- const expected = a.in + rel * speed;- if (Math.abs(el.currentTime - expected) > 0.25) el.currentTime = expected;- if (el.paused) void el.play().catch(() => {});- } else if (!el.paused) {- el.pause();- }- }- for (const [id, el] of this.audioEls) {- if (!live.has(id)) {- el.pause();- this.audioEls.delete(id);- }- }- }-- // The last tick's wall-clock stamp, for detecting gaps in the tick stream.- private lastTickAt = 0;-private tick() {+ this.raf = 0;if (this.disposed) return;- this.raf = requestAnimationFrame(this.tick);- const now = performance.now();- // The stall clock must count only time the engine was alive. rAF freezes- // with the tab hidden while `health.at` stamps keep aging — and Chrome- // defers a hidden tab's media loads, so a decoder created near the switch- // is still cold on return. The first tick back would then read every such- // decoder as minutes past the stall limit and tear it down, killing the- // very load Chrome had just released. A gap in the tick stream restarts- // each decoder's stall window instead.- if (this.lastTickAt && now - this.lastTickAt > 1_000) {- for (const h of this.health.values()) h.at = now;- }- this.lastTickAt = now;- this.frame++;- this.render();- // After the frame, never during it: what `render` touched is exactly what- // must survive, and it takes every early return there is to say so.- this.evictIdleDecoders();- this.diagPulse();- }-- // Diagnostic heartbeat. The decoder pool is invisible from DevTools — the- // elements are detached — so every ten seconds the engine prints each live- // decoder's clock, readiness and buffer. A decoder found playing while the- // project is paused is flagged loudly: every paused-path element is supposed- // to be parked, so one still running marks a leak. Filter the console on- // "cut-preview".- private lastDiagAt = 0;-- private diagPulse() {- const now = performance.now();- if (now - this.lastDiagAt < 10_000) return;- this.lastDiagAt = now;+ markTick();const playing = useEditor.getState().playing;- const rows: string[] = [];- const scan = (map: Map<string, MediaEl>, kind: string) => {- for (const [id, el] of map) {- if (isImageEl(el)) {- rows.push(`${kind} ${id} img`);- continue;- }- const buf = el.buffered.length ? el.buffered.end(el.buffered.length - 1).toFixed(1) : "0";- rows.push(- `${kind} ${id} ct=${el.currentTime.toFixed(2)} rs=${el.readyState} buf=${buf}` +- `${el.paused ? "" : " PLAYING"}${el.error ? ` err=${el.error.code}` : ""}`- );- if (!playing && !el.paused) {- console.warn(`[cut-preview] decoder playing while paused: clip ${id} at ${el.currentTime.toFixed(2)}s`);- }- }- };- scan(this.videoEls, "track0");- scan(this.overlayEls, "overlay");- console.debug(- `[cut-preview] playing=${playing} decoders=${this.videoEls.size + this.overlayEls.size} ` +- `audio=${this.audioEls.size}\n` + rows.join("\n")- );+ // Playing redraws every frame; paused, only what changed since the last+ // paint. Either way the loop stops as soon as there is nothing to do.+ if (playing || this.dirty) {+ this.dirty = false;+ this.render(playing);+ }+ if (playing || this.dirty) this.schedule();}- private render() {+ private render(playing: boolean) {const s = useEditor.getState();const spans = getClipSpans(s.clips, s.assets);- // Drop decoders no clip needs anymore (deleted or replaced). Every edit- // rewrites the clip array, so its identity changing is the signal to- // re-check — a moving playhead leaves it alone, and the scan never runs- // per frame.- if (this.clipsSeen !== s.clips) {- this.clipsSeen = s.clips;- const live = new Set<string>();- for (const c of s.clips) {- const asset = s.assets.find((a) => a.id === c.assetId);- if (asset) live.add(mappingKey(c, asset));- }- for (const key of [...this.videoEls.keys(), ...this.overlayEls.keys()]) {- if (!live.has(key)) this.dropDecoder(key);+ const total = projectDuration(s);+ this.pool.beginFrame();+ this.used.clear();++ // The clock, before anything that can return early.+ //+ // Whether the mixer is running *is* whether the cut is playing — there is no+ // second flag to fall out of step with it. Kept as one, a frame that+ // returned early (a clip still opening, an empty timeline) could leave the+ // engine believing playback had already started while the mixer had stopped,+ // and the next play would read a clock that was never anchored: the picture+ // would sit at zero for the length of the cut.+ //+ // The playhead can also move while the clock runs — an arrow-key skip, the+ // assistant's seek, a preview range starting mid-play. The engine writes+ // the playhead itself every playing frame, so a value it did not write is+ // someone seeking, and the clock re-anchors there rather than snapping the+ // playhead back.+ let t: number;+ if (playing) {+ const head = playheadAt();+ if (!this.mixer.running || (this.written !== null && head !== this.written)) {+ this.mixer.start(Math.min(head, total));+ this.written = head;}+ t = Math.max(0, Math.min(this.mixer.now(), total));+ } else {+ // Pinned just inside the end: a preview time at exactly `total` lies past+ // the final span, and a skim off the right edge of the cut should hold+ // the last frame rather than clear to black.+ const end = Math.max(0, total - 0.001);+ t = Math.max(0, Math.min(previewAt(), end));+ if (this.mixer.running) this.mixer.stop();}- // Whole-project length so time past track 0's end (a longer video track or- // soundtrack) is still reachable while scrubbing and playing.- const total = projectDuration(s);- // Nothing anywhere — no track-0 clip, no overlay layer, no soundtrack, no- // title or sticker — resets to a black frame at 0. An empty track 0 with- // any one of those still plays: the tick body draws those layers and- // advances the wall clock, so the guard must not bail on- // `spans.length === 0` alone.+ // Nothing anywhere resets to a black frame at 0.if (spans.length === 0 &&overlayLayers(s.clips).length === 0 &&s.audioClips.length === 0 &&s.overlays.length === 0) {- this.pauseExcept(new Set());+ this.mixer.stop();this.comp.drawLayer(MISSING_FRAME, undefined, true, 1, 0);- this.syncSoundtrack(0, false);- if (s.playing) useEditor.setState({ playing: false, currentTime: 0 });- return;- }-- let t = Math.min(s.currentTime, total);-- // Skimming: while paused with the mouse over the timeline, the- // frame on screen lives at the skim point, not the playhead. The playhead- // (currentTime) is never touched.- const pt =- !s.playing && s.skimTime !== null- ? Math.max(0, Math.min(s.skimTime, total - 0.001))- : t;-- // Keep the next few clips decoded and buffering ahead of the frame being- // shown (the skim point while skimming, else the playhead — paused too, so- // pressing play resumes clean). Runs before either branch since both- // benefit; warms only clips ahead of the anchor, so it never touches the- // element the branches are about to drive — anchored at the playhead while- // skimming, its entrance-frame parking seeks would fight the skimmed- // clip's own scrub seeks every tick and freeze the preview on any clip- // ahead of the playhead. The span live at the anchor names the element- // that must not be parked out from under the picture (a shared decoder).- const anchorSpan = spans.find((sp) => pt >= sp.start && pt < sp.start + sp.len);- this.warmAhead(spans, pt, anchorSpan ? mappingKey(anchorSpan.clip, anchorSpan.asset) : null);- this.warmOverlaysAhead(pt);-- if (!s.playing) {- // Not advancing: drop the wall-clock stamp so the first playing tick- // starts a fresh delta instead of leaping over the paused stretch.- this.lastPlayNow = 0;- const span = anchorSpan;- // Prime every layer live at `pt` — the track-0 element and each overlay- // track — before repainting (create them, issue any seeks). A cold- // element or an unbuffered seek has no decodable frame yet, and painting- // around it tears the composite: black before the track-0 seek resolves,- // or track 0 flashing through where an overlay covers it. Hold the last- // painted frame until every live layer has a frame, so each scrubbed- // frame is the same composite playback and export show.- let ready = true;- if (span) {- const el = this.prepare(span, Math.min(pt, span.start + span.len), false, true);- // A broken source never becomes ready; paint without it rather than- // wedging the preview on it.- if (!elErrored(el) && !elReady(el)) ready = false;- }- // Prime the overlays once; the readiness scan and the draw step below- // reuse these instead of re-filtering/seeking them a second time.- const overlaysLive = this.prepareOverlays(pt, false);- for (const { el } of overlaysLive) {- if (!elErrored(el) && !elReady(el)) ready = false;- }- if (!ready) {- this.pauseExcept(new Set(span ? [mappingKey(span.clip, span.asset)] : []));- this.syncSoundtrack(t, false);- return;+ if (s.playing) {+ useEditor.setState({ playing: false });+ this.writeHead(0);}- const active = new Set<string>();- this.comp.clear();- // Where track 0 has nothing live there is no master frame — just black- // and the other tracks still running at `pt`.- if (span) this.composite(span, spans, Math.min(pt, span.start + span.len), false);- else this.pauseExcept(new Set());- this.drawOverlays(pt, false, active, overlaysLive);- this.drawBehind(pt);- this.comp.drawProjectFade(this.projectFadeGain(pt, total));- this.cleanupOverlays(active);- this.syncSoundtrack(t, false);return;}- let span = anchorSpan;+ const master = spans.find((sp) => t >= sp.start && t < sp.start + sp.len);- // A just-started or just-scrubbed clip may still be decoding. Hold the last- // painted frame rather than clearing to black — same as the skim path. (At a- // dissolve boundary the incoming clip is already warm from the overlap, so- // this only bites a genuinely cold first frame.)- if (span) {- const el = this.prepare(span, t, true, span.clip.muted || !!span.clip.hidden);- // A broken source (unreachable still, decode error) never becomes ready;- // fall through so the wall clock advances past it instead of freezing.- if (!elReady(el) && !elErrored(el)) {- this.pauseExcept(new Set([mappingKey(span.clip, span.asset)]));- this.syncSoundtrack(t, true);- return;- }+ // Ask for the master's picture before clearing. A clip nothing has decoded+ // yet answers `pending`, and clearing on the strength of that would black+ // the frame out for as long as the file takes to open — the strobe this+ // whole design exists to remove. Leave what is on screen, stay dirty, and+ // paint when there is something to paint.+ const masterFrame = master ? this.frameFor(master, t, playing) : MISSING_FRAME;+ if (masterFrame.kind === "pending") {+ this.warm(t, playing);+ this.dirty = true;+ return;}- // Clear to black, then prime the master element (and any dissolve- // partner) over it and read the clock.- const active = new Set<string>();this.comp.clear();+ if (master) this.drawTrackZero(master, spans, t, playing, masterFrame);+ this.drawOverlays(t, playing);+ // The subject-mask pass reads the canvas as it stands beneath it, then+ // publishes the matte the DOM's front subject-masked elements read.+ this.comp.subjectMatteProvider = (at) => this.behind.clipMatteOf(this.canvas, at);+ this.behind.draw(this.canvas, s.overlays, s.assets, t);+ const fadeGain = this.projectFadeGain(t, total);+ this.comp.drawProjectFade(fadeGain);- if (span) {- let el = this.composite(span, spans, t, true);- // The element clock is the truth but it's coarse: currentTime advances in- // steps bigger than a frame, and copying it straight to the playhead- // makes the indicator stutter and step backward. Advance by wall clock- // instead — smooth by construction and never backward — and let the- // element clock steer it: the playhead may run at most 60ms ahead of the- // clock (a stalled element halts it with the picture) and snaps forward- // only when the clock genuinely leads.- const now = performance.now();- const dt = this.lastPlayNow ? Math.min(0.25, (now - this.lastPlayNow) / 1000) : 0;- let atEnd: boolean;- if (isImageEl(el) || elErrored(el)) {- // A still has no element clock, and a broken source's clock never- // advances (steering by it would pin the playhead at the clip start) —- // move purely by wall clock and end at the clip's timeline footprint.- t = Math.max(span.start, Math.min(t + dt, span.start + span.len));- atEnd = t >= span.start + span.len - 0.0001;+ // Decoders for what is about to arrive, and the ones nothing needs closed.+ this.warm(t, playing);+ this.pool.evict();++ if (playing) {+ this.mixer.setMasterGain(fadeGain);+ this.mixer.update(t, this.voicesAt(t, spans, master));+ // A scoped effect preview auto-pauses at its stop mark; the end of the+ // cut stops playback outright.+ const stop = s.previewStopAt;+ if (stop != null && t >= stop) {+ useEditor.setState({ playing: false, previewStopAt: null });+ this.writeHead(Math.min(t, stop));+ } else if (t >= total - 0.001) {+ useEditor.setState({ playing: false, previewStopAt: null });+ this.writeHead(total);} else {- const speed = clipSpeed(span.clip);- const derived = span.start + (el.currentTime - span.clip.in) / speed;- const cand = Math.min(t + dt, derived + 0.06);- t = derived - cand > 0.25 ? derived : Math.max(t, cand);- t = Math.max(span.start, Math.min(t, span.start + span.len));- atEnd = el.currentTime >= span.clip.out - 0.02 || el.ended;- }- // Clip boundary: hand off to the next clip when it abuts (or dissolves),- // fall into the gap when it doesn't — track 0 plays black there and the- // wall clock advances — or (if track 0 is done but another track runs- // on) fall through to the wall-clock tail.- if (atEnd) {- const idx = spans.indexOf(span);- const next = spans[idx + 1];- if (next && next.start <= span.start + span.len + 0.001) {- // Jump past the finished clip's whole footprint (including any- // cross-dissolve overlap), not back to next.start — which still sits- // inside the outgoing clip's footprint, so find() would re-pick the- // clip we just finished and playback would ping-pong across the- // dissolve forever.- t = Math.max(next.start + 0.0001, span.start + span.len);- span = next;- el = this.composite(next, spans, t, true);- } else if (next) {- // A gap before the next clip: step just past this clip's footprint- // so the next tick's find() sees no active span and the wall-clock- // path carries time (and the soundtrack) across the black stretch.- t = Math.max(t, span.start + span.len + 0.0001);- pauseEl(el);- } else if (t >= total - 0.001) {- // Track 0 and every other track finished.- useEditor.setState({ playing: false, currentTime: total, previewStopAt: null });- pauseEl(el);- this.drawOverlays(t, true, active);- this.drawBehind(t);- this.comp.drawProjectFade(this.projectFadeGain(total, total));- this.cleanupOverlays(active);- this.syncSoundtrack(total, false);- return;- }- }- this.lastPlayNow = now;- } else {- // Nothing live on track 0 but another track is still playing: no master- // element, so advance time by the wall clock and let the overlays follow.- this.pauseExcept(new Set());- const now = performance.now();- const dt = this.lastPlayNow ? Math.min(0.25, (now - this.lastPlayNow) / 1000) : 0;- this.lastPlayNow = now;- t = t + dt;- if (t >= total - 0.001) {- useEditor.setState({ playing: false, currentTime: total, previewStopAt: null });- this.drawOverlays(t, true, active);- this.drawBehind(t);- this.comp.drawProjectFade(this.projectFadeGain(total, total));- this.cleanupOverlays(active);- this.syncSoundtrack(total, false);- return;+ this.writeHead(t);}}- this.drawOverlays(t, true, active);- this.drawBehind(t);- // The whole-video fade veils the finished frame and dims the sound —- // the master's element volume (set by composite) and the soundtrack.- const fadeGain = this.projectFadeGain(t, total);- if (fadeGain < 1) {- if (span) {- const mel = this.videoEls.get(mappingKey(span.clip, span.asset));- if (mel && !isImageEl(mel)) mel.volume = Math.min(mel.volume, fadeGain);- }- for (const el of this.overlayEls.values()) {- if (!isImageEl(el)) el.volume = Math.min(el.volume, fadeGain);+ if (tracing()) {+ let srcTs: number | null = null;+ let exact = true;+ if (master) {+ const src = this.pool.get(+ keyOf(master.clip, master.asset),+ master.asset,+ this.decodeHeight()+ );+ const st = sourceTimeOf(master.clip, t);+ srcTs = src.frameAt(st)?.timestamp ?? null;+ exact = src.hasExact(st);}- }- this.comp.drawProjectFade(fadeGain);- this.cleanupOverlays(active);- // A scoped effect preview auto-pauses at its stop mark (the frame this- // tick just painted is the stop frame — close enough at tick rate).- if (s.previewStopAt != null && t >= s.previewStopAt) {- useEditor.setState({- playing: false,- currentTime: Math.min(t, s.previewStopAt),- previewStopAt: null,+ markPresent({+ t,+ srcTs,+ clipId: master?.clip.id ?? null,+ exact,+ stale: masterFrame.kind !== "ready",});- this.syncSoundtrack(Math.min(t, s.previewStopAt), false);- return;+ markLiveSamples(this.pool.held);}- useEditor.setState({ currentTime: t });- this.syncSoundtrack(t, true, fadeGain);}}@@ -1105,8 +506,8 @@ export function usePlayback(canvasRef: RefObject<HTMLCanvasElement | null>) {const canvas = canvasRef.current;if (!canvas) return;const engine = new Engine(canvas);- // Dev-only automation hook, like installDevHooks: lets a headless run (or- // a debugging session) read the live decoder pool the way diagPulse does.+ // Dev-only automation hook, like installDevHooks: lets a headless run (or a+ // debugging session) reach the live engine.if (process.env.NODE_ENV !== "production") {(window as unknown as Record<string, unknown>).__cutDevEngine = engine;}diff --git a/site/src/cut/lib/effectStack.ts b/site/src/cut/lib/effectStack.tsindex bab4acf8..b2a9f3f0 100644--- a/site/src/cut/lib/effectStack.ts+++ b/site/src/cut/lib/effectStack.ts@@ -11,7 +11,8 @@* The math is here, apart from React, because both exports walk the same stack.*/-import type { EffectPreviewState } from "@donkeycut/effects-kit";+import { effectPreviewState, type EffectPreviewState } from "@donkeycut/effects-kit";+import { isEffectOverlay, laneOf, type Overlay } from "./types";/** One effect live at a moment, with the lane that places it in the stack. */export interface LiveEffect {@@ -19,6 +20,23 @@ export interface LiveEffect {state: EffectPreviewState;}+/** The effects playing at `t`, shallowest lane first. */+export function liveEffectsAt(overlays: Overlay[], t: number): LiveEffect[] {+ const live: LiveEffect[] = [];+ for (const o of overlays) {+ if (!isEffectOverlay(o) || o.hidden || t < o.start || t > o.end) continue;+ live.push({+ lane: laneOf(o),+ state: effectPreviewState(o.effect, o.amount, t - o.start, o.focus, o.ramp, o.end - o.start),+ });+ }+ return live.sort((a, b) => a.lane - b.lane);+}++/** Whether any effect could be live at all. The stage asks before doing the+ * per-frame work of finding out which ones are. */+export const hasEffects = (overlays: Overlay[]) => overlays.some(isEffectOverlay);+/** How the content under an effect moves: a zoom's push in, a shake, and the* horizontal jump that reads as tearing. Empty for the effects that only* filter. */@@ -54,7 +72,7 @@ export function stageEffectTransform(states: EffectPreviewState[]): string | und}/** The grade of a run of effects, as one CSS filter. */-const stageEffectFilter = (states: EffectPreviewState[]) =>+export const stageEffectFilter = (states: EffectPreviewState[]) =>states.map((s) => s.cssFilter).filter(Boolean)@@ -68,23 +86,24 @@ export type StageSlice =/** Elements on lanes in [from, to) — `from` counts down from the top. */from: number;to: number;- filter?: string;- transform?: string;+ /** Effects shallower than this lane grade the slice. Null on the top+ * slice, which nothing grades. */+ gradeAbove: number | null;/** The topmost slice carries the captions, which nothing grades. */captions: boolean;}- | { kind: "paint"; key: string; states: EffectPreviewState[] };+ | { kind: "paint"; key: string; lane: number };/**- * The stage bottom to top: what the picture itself wears, then alternating- * element slices and the paints of the effect that sits over each.+ * The stage bottom to top: alternating element slices and the paints of the+ * effect that sits over each.+ *+ * The shape depends only on which lanes hold effects, so it holds still while+ * the playhead moves and changes only when a row does. What each slice *wears*+ * at this instant is worked out by the slice itself, which is what keeps the+ * clock out of the component that lays the stage out.*/-export function stageSlices(- live: LiveEffect[],- lanes: number[]-): { picture: { filter?: string; transform?: string }; slices: StageSlice[] } {- const above = (lane: number) => live.filter((e) => e.lane < lane).map((e) => e.state);- const all = live.map((e) => e.state);+export function stageSliceStructure(lanes: number[]): StageSlice[] {const slices: StageSlice[] = [];// Deepest first: elements under that effect, then its paints over them.for (let i = lanes.length - 1; i >= 0; i--) {@@ -94,25 +113,18 @@ export function stageSlices(key: `band-${lane}`,from: lane + 1,to: i + 1 < lanes.length ? lanes[i + 1] : Infinity,- filter: stageEffectFilter(above(lane + 1)),- transform: stageEffectTransform(above(lane + 1)),+ gradeAbove: lane + 1,captions: false,});- slices.push({- kind: "paint",- key: `paint-${lane}`,- states: live.filter((e) => e.lane === lane).map((e) => e.state),- });+ slices.push({ kind: "paint", key: `paint-${lane}`, lane });}slices.push({kind: "elements",key: "band-top",from: 0,to: lanes.length ? lanes[0] : Infinity,+ gradeAbove: null,captions: true,});- return {- picture: { filter: stageEffectFilter(all), transform: stageEffectTransform(all) },- slices,- };+ return slices;}diff --git a/site/src/cut/lib/frameSource.test.ts b/site/src/cut/lib/frameSource.test.tsnew file mode 100644index 00000000..b8074fd9--- /dev/null+++ b/site/src/cut/lib/frameSource.test.ts@@ -0,0 +1,171 @@+import { describe, expect, test } from "bun:test";+import { FrameRing, FrameSourcePool, mappingKey, type Timed } from "./frameSource";+import type { MediaAsset } from "./types";++/** A decoded frame at 30fps, as the sink would hand it over. */+const f = (timestamp: number, duration = 1 / 30): Timed => ({ timestamp, duration });++const asset = (id: string): MediaAsset => ({+ id,+ name: `${id}.mp4`,+ fileName: `${id}.mp4`,+ type: "video",+ url: `https://example.test/${id}.mp4`,+ duration: 10,+});++describe("FrameRing", () => {+ test("hands back the frame covering the asked-for time", () => {+ const ring = new FrameRing<Timed>(8);+ for (let i = 0; i < 5; i++) ring.push(f(i / 30));+ // Between two frames, the one that started is the one on screen.+ expect(ring.at(2.5 / 30)?.timestamp).toBeCloseTo(2 / 30);+ expect(ring.at(2 / 30)?.timestamp).toBeCloseTo(2 / 30);+ });++ test("answers past the end with the newest frame it holds", () => {+ // A decoder running behind the playhead should show the last real frame+ // rather than nothing — this is the "never withhold" rule.+ const ring = new FrameRing<Timed>(8);+ ring.push(f(0));+ ring.push(f(1 / 30));+ expect(ring.at(5)?.timestamp).toBeCloseTo(1 / 30);+ });++ test("answers before the first frame with the earliest it holds", () => {+ // At a clip's head the frame the cut opens on is the right thing to show.+ const ring = new FrameRing<Timed>(8);+ ring.push(f(2));+ ring.push(f(2 + 1 / 30));+ expect(ring.at(1)?.timestamp).toBeCloseTo(2);+ });++ test("empty ring has no answer", () => {+ expect(new FrameRing<Timed>(4).at(0)).toBe(null);+ });++ test("covers() separates a real frame from a held one", () => {+ const ring = new FrameRing<Timed>(8);+ ring.push(f(1));+ expect(ring.covers(1)).toBe(true);+ expect(ring.covers(1 + 0.5 / 30)).toBe(true);+ // Past this frame's own duration nothing is covered, even though `at`+ // still hands the frame over.+ expect(ring.covers(1 + 2 / 30)).toBe(false);+ expect(ring.at(1 + 2 / 30)?.timestamp).toBeCloseTo(1);+ expect(ring.covers(0.5)).toBe(false);+ });++ test("drops the oldest past capacity, so the canvas pool never wraps under it", () => {+ const ring = new FrameRing<Timed>(3);+ for (let i = 0; i < 6; i++) ring.push(f(i));+ expect(ring.size).toBe(3);+ expect(ring.oldest).toBeCloseTo(3);+ expect(ring.newest).toBeCloseTo(5);+ // The evicted frames are gone, not merely unreachable.+ expect(ring.at(0)?.timestamp).toBeCloseTo(3);+ });++ test("keep() drops what a reader moved away from", () => {+ const ring = new FrameRing<Timed>(16);+ for (let i = 0; i < 10; i++) ring.push(f(i));+ ring.keep(4, 6);+ expect(ring.size).toBe(3);+ expect(ring.oldest).toBeCloseTo(4);+ expect(ring.newest).toBeCloseTo(6);+ });+});++describe("mappingKey", () => {+ test("clips showing identical pictures share a decoder", () => {+ // A plain split: same file, same speed, and the same source time at+ // timeline zero. One decoder reads straight across the cut.+ const left = mappingKey("a", 1, 0, 0);+ const right = mappingKey("a", 1, 4, 4);+ expect(left).toBe(right);+ });++ test("different trims of one file keep their own decoders", () => {+ // Both live at timeline 4, showing different moments — a same-source+ // dissolve has to blend two distinct frames.+ expect(mappingKey("a", 1, 0, 4)).not.toBe(mappingKey("a", 1, 8, 4));+ });++ test("a speed change is a different mapping", () => {+ expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("a", 2, 0, 0));+ });++ test("different files never share", () => {+ expect(mappingKey("a", 1, 0, 0)).not.toBe(mappingKey("b", 1, 0, 0));+ });+});++describe("FrameSourcePool", () => {+ test("one source per mapping and size, reused across frames", () => {+ const pool = new FrameSourcePool(4);+ pool.beginFrame();+ const a = pool.get("k", asset("a"), 480);+ const b = pool.get("k", asset("a"), 480);+ expect(a).toBe(b);+ // A different preview size is a different decode.+ expect(pool.get("k", asset("a"), 240)).not.toBe(a);+ expect(pool.size).toBe(2);+ });++ test("a source follows its asset's URL", () => {+ // A signed URL re-mints, or a shot re-renders onto its asset id. The+ // mapping still names the same pictures, but they now live somewhere else,+ // so the old source is closed and a fresh one reads the new URL.+ const pool = new FrameSourcePool(4);+ pool.beginFrame();+ const before = pool.get("k", asset("a"), 480);+ expect(pool.get("k", asset("a"), 480)).toBe(before);+ const moved = { ...asset("a"), url: "https://example.test/a.mp4?e=2&s=next" };+ const after = pool.get("k", moved, 480);+ expect(after).not.toBe(before);+ expect(after.url).toBe(moved.url);+ expect(pool.size).toBe(1);+ });++ test("evicts the least recently asked-for, never this frame's", () => {+ const pool = new FrameSourcePool(2);+ pool.beginFrame();+ const old = pool.get("old", asset("a"), 480);+ pool.beginFrame();+ const mid = pool.get("mid", asset("b"), 480);+ // Let the grace expire on both, then draw a third from a frame of its own.+ for (let i = 0; i < 200; i++) pool.beginFrame();+ const live = pool.get("live", asset("c"), 480);+ expect(pool.size).toBe(3);+ pool.evict();+ // Down to the budget, and the one this frame is drawn from survives.+ expect(pool.size).toBe(2);+ expect(pool.get("live", asset("c"), 480)).toBe(live);+ expect(pool.get("mid", asset("b"), 480)).toBe(mid);+ expect(pool.get("old", asset("a"), 480)).not.toBe(old);+ });++ test("holds a source through the grace, so a busy cut never thrashes", () => {+ // A cut with more clips than the budget would otherwise close and reopen+ // decoders every frame, and reopening costs far more than the memory it+ // saves. Nothing is closed until it has gone unwanted for a while.+ const pool = new FrameSourcePool(1);+ pool.beginFrame();+ const a = pool.get("a", asset("a"), 480);+ pool.beginFrame();+ const b = pool.get("b", asset("b"), 480);+ pool.evict();+ expect(pool.size).toBe(2);+ expect(pool.get("a", asset("a"), 480)).toBe(a);+ expect(pool.get("b", asset("b"), 480)).toBe(b);+ });++ test("stays put while under budget", () => {+ const pool = new FrameSourcePool(4);+ pool.beginFrame();+ const a = pool.get("a", asset("a"), 480);+ pool.beginFrame();+ pool.evict();+ expect(pool.get("a", asset("a"), 480)).toBe(a);+ });+});diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.tsnew file mode 100644index 00000000..6f6da9ce--- /dev/null+++ b/site/src/cut/lib/frameSource.ts@@ -0,0 +1,621 @@+"use client";++/**+ * Frames for the preview, decoded straight off the file.+ *+ * The preview used to ask an `HTMLVideoElement` for a picture by writing+ * `currentTime` and waiting: a seek costs a decode from the nearest keyframe,+ * the element decides when it lands, and a cut meant handing the clock from one+ * element to another mid-flight. Everything that felt slow about scrubbing and+ * everything that stuttered at a join came from that one arrangement.+ *+ * Here a clip owns a decoder and a small ring of already-decoded frames.+ * Playing walks the file forward and fills the ring ahead of the playhead;+ * scrubbing asks for a single time and the newest request wins. Either way the+ * question the compositor asks — what does this clip look like now — is+ * answered from memory, without awaiting anything. If the exact frame has not+ * arrived, the nearest one already decoded is handed over and the real one+ * replaces it a moment later. A held frame is worth more than a stall.+ *+ * The ring is bounded by the sink's own canvas pool: `mediabunny` cycles a+ * fixed set of canvases, so keeping a reference to more of them than the pool+ * holds would hand out a canvas that has since been drawn over. The ring's+ * capacity and the pool size move together, and that is the whole memory story.+ */++import type { CanvasSink, Input, WrappedCanvas } from "mediabunny";+import { frameSink, openMedia, videoTrackOf } from "./mediaRead";+import type { MediaAsset } from "./types";++/**+ * Decoded frames a source keeps around where it is being read.+ *+ * Small on purpose. Each one is a full-size canvas, and ten sources holding a+ * dozen apiece is hundreds of megabytes — enough to take the tab down. The sink+ * pre-decodes ahead internally, so the ring only has to bridge the gap between+ * two frames of the display: the frame being shown, a couple ahead of it, and+ * one spare for the blend at a dissolve.+ */+const RING = 10;+/** Canvases the sink cycles. Two more than the ring, so the frame being decoded+ * never lands on one the compositor is drawing this instant. */+const POOL = RING + 2;+/**+ * How far ahead of where it is read a playing source decodes.+ *+ * This has to stay inside what the ring can hold. Reading further ahead than+ * `RING` frames would push the frame actually on screen out of the ring to make+ * room for one nobody has asked for yet — the picture would run ahead of the+ * playhead and the decoder would burn through the file to put it there.+ */+export const DECODE_AHEAD_S = 0.3;+/** Two source times closer than this are the same frame. */+const SAME = 1e-4;+/** Frames a source must go unwanted before the pool will close it. About a+ * second of playback — long enough that crossing a cut and coming back finds+ * the decoder still open. */+const EVICT_GRACE = 90;+/** A failed open tries again this much later, growing per attempt, and gives+ * up after this many. A network blip heals on its own; a file that is truly+ * unreadable stops costing anything after a few seconds. */+const RETRY_MS = 1000;+const RETRIES = 3;++/** A clip's picture at an instant, with where it came from. */+export interface SourceFrame {+ image: CanvasImageSource;+ width: number;+ height: number;+ /** The frame's own timestamp in the source, for telling a held frame from a+ * fresh one. */+ timestamp: number;+}++const frameOfCanvas = (c: WrappedCanvas): SourceFrame => ({+ image: c.canvas,+ width: c.canvas.width,+ height: c.canvas.height,+ timestamp: c.timestamp,+});++/** What the ring stores: a frame and the stretch of source it stands for. */+export interface Timed {+ timestamp: number;+ duration: number;+}++/**+ * Decoded frames in the order they arrived, oldest dropped first.+ *+ * The order matters more than it looks. The sink hands out canvases from a+ * fixed pool and reuses the oldest once it wraps, so holding a reference past+ * the pool's length means holding a canvas something else has drawn on. Bounded+ * by arrival, the ring can only ever name canvases the pool still considers+ * ours.+ */+export class FrameRing<T extends Timed> {+ private items: T[] = [];++ constructor(private readonly cap: number) {}++ get size(): number {+ return this.items.length;+ }++ get newest(): number {+ return this.items.length ? this.items[this.items.length - 1].timestamp : -Infinity;+ }++ /** The earliest timestamp held, which is where a walk's buffer begins. */+ get oldest(): number {+ let min = Infinity;+ for (const i of this.items) min = Math.min(min, i.timestamp);+ return min;+ }++ push(item: T): void {+ this.items.push(item);+ while (this.items.length > this.cap) this.items.shift();+ }++ clear(): void {+ this.items = [];+ }++ /**+ * The frame covering `t`: the last one starting at or before it. Before the+ * first frame held, the earliest — at a clip's head that is the frame the cut+ * opens on, and showing it beats showing nothing.+ */+ at(t: number): T | null {+ let best: T | null = null;+ let earliest: T | null = null;+ for (const i of this.items) {+ if (!earliest || i.timestamp < earliest.timestamp) earliest = i;+ if (i.timestamp <= t + SAME && (!best || i.timestamp > best.timestamp)) best = i;+ }+ return best ?? earliest;+ }++ /**+ * Whether `t` is genuinely covered, rather than answered by a held frame.+ *+ * A frame stands for its own timestamp up to the next one, which is what its+ * duration says. Asking the frame rather than the ring's extent gets the same+ * answer for a walk that has run past `t` and for a single frame fetched at+ * `t`, so a scrub and a playback agree on what "we have this one" means.+ */+ covers(t: number): boolean {+ return this.items.some(+ (i) => t >= i.timestamp - SAME && t < i.timestamp + Math.max(i.duration, SAME) + SAME+ );+ }++ /** How many frames held start at or after `t`. */+ aheadOf(t: number): number {+ let n = 0;+ for (const i of this.items) if (i.timestamp >= t - SAME) n++;+ return n;+ }++ /** How many frames held lie inside [from, to]. */+ between(from: number, to: number): number {+ let n = 0;+ for (const i of this.items) if (i.timestamp >= from - SAME && i.timestamp <= to + SAME) n++;+ return n;+ }++ /** Drop frames outside a window a reader has moved away from. */+ keep(from: number, to: number): void {+ this.items = this.items.filter((i) => i.timestamp >= from - SAME && i.timestamp <= to + SAME);+ }+}++/**+ * One decoder and its ring.+ *+ * A source is opened for a clip's *mapping* rather than for its asset: same+ * file, same speed, same source-time offset means the same picture at every+ * instant, so a tiled reveal playing one file across many tracks decodes once,+ * and a plain split reads straight across its own cut. Two different trims of+ * one file keep their own sources, which is what lets a same-source dissolve+ * show two distinct frames.+ */+export class ClipFrameSource {+ private input: Input | null = null;+ private sink: CanvasSink | null = null;+ private ring = new FrameRing<WrappedCanvas>(RING);+ /** A still's single frame; stills never stream. */+ private still: SourceFrame | null = null;+ private opening: Promise<void> | null = null;+ private closed = false;+ /** Set when the file turns out to hold no picture this browser can read. */+ private unreadable = false;+ /** Failed opens so far, and the timer that will clear `unreadable` for the+ * next try. */+ private attempts = 0;+ private retryTimer = 0;++ /** The forward walk, when one is running, where it started, and the last+ * frame it landed. Steering reads only these two numbers: the ring is a+ * cache that may also hold frames left by other gestures — a jump's single+ * frame, a spent walk — and a min/max over that sparse set has holes in it+ * that read as coverage. */+ private stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = null;+ private streamFrom = 0;+ private streamTail = 0;+ /** The walk ran off the end of the file. Its span is still the truth — there+ * are no frames past its tail to decode — so a reader inside it must not+ * keep restarting a walk that can only end again. */+ private streamDone = false;+ private streaming = false;+ /** The time a paused reader last asked for, and whether a read is in flight.+ * Latest wins: a fast drag decodes where it stopped, never every point it+ * crossed. */+ private wanted: number | null = null;+ private reading = false;++ /** The engine's tick number when this was last asked for anything, for the+ * pool's eviction order. */+ touched = 0;++ /** The URL this source is reading. The pool compares it against the store's+ * current one, so a re-minted signed URL replaces the source under it. */+ get url(): string {+ return this.asset.url;+ }++ constructor(+ private readonly asset: MediaAsset,+ private readonly height: number,+ /** Called whenever a frame lands. A paused editor draws the nearest frame+ * it has and stops; without a nudge, the exact frame would decode into a+ * ring nothing looks at again. */+ private readonly onFrame: () => void = () => {}+ ) {}++ /** Decoded frames held right now, for the pool's budget and the perf trace. */+ get held(): number {+ return this.ring.size + (this.still ? 1 : 0);+ }++ get ready(): boolean {+ return this.still !== null || this.ring.size > 0;+ }++ get failed(): boolean {+ return this.unreadable;+ }++ /**+ * The best picture this source has for source time `t`, without waiting.+ *+ * Exact when the ring holds the frame covering `t`; otherwise the nearest+ * frame before it, which is what a decoder running slightly behind should+ * show. Null only when nothing has been decoded yet.+ */+ frameAt(t: number): SourceFrame | null {+ if (this.still) return this.still;+ const c = this.ring.at(t);+ return c ? frameOfCanvas(c) : null;+ }++ /** Whether the ring holds the exact frame covering `t`. */+ hasExact(t: number): boolean {+ return this.still !== null || this.ring.covers(t);+ }++ /**+ * Say where this clip is being read, and let the source decide how to keep up.+ *+ * `playing` walks forward from `t` and stays `DECODE_AHEAD_S` ahead of it;+ * paused, a single frame is fetched for `t` and the newest ask wins.+ */+ want(t: number, playing: boolean): void {+ if (this.closed || this.unreadable) return;+ if (this.asset.type === "image") {+ void this.openStill();+ return;+ }+ void this.open();+ if (playing) {+ this.wanted = null;+ this.pumpStream(t);+ } else {+ if (this.hasExact(t)) return;+ // A drag creeping along is a forward walk, not a series of jumps. Asking+ // the file for each position separately means decoding from the nearest+ // keyframe every time — the same cost the `<video>` element charged. If+ // the wanted frame is inside the walk or just ahead of the last frame+ // that arrived, keep walking and each step costs one frame. The last+ // arrival is the only ring fact consulted: anything older may be a+ // leftover from some other gesture entirely.+ const walkEdge = Math.max(this.streamFrom, this.streamTail);+ const nearWalk = this.stream !== null && t >= walkEdge - SAME && t < walkEdge + 0.5;+ const nearLast =+ this.ring.size > 0 && t >= this.ring.newest - SAME && t < this.ring.newest + 0.5;+ if (nearWalk || nearLast) {+ this.pumpStream(t);+ return;+ }+ // A real jump: drop the walk and fetch the one frame. Newest ask wins, so+ // a fast drag decodes where it stopped rather than everywhere it crossed.+ this.stopStream();+ this.wanted = t;+ void this.pumpSeek();+ }+ }++ /** Frames outside [from, to] are of no further use — a clip whose window+ * moved on, or a scrub that jumped. */+ trim(from: number, to: number): void {+ this.ring.keep(from, to);+ }++ close(): void {+ this.closed = true;+ this.stopStream();+ this.ring.clear();+ if (this.retryTimer) clearTimeout(this.retryTimer);+ this.retryTimer = 0;+ // A still's bitmap holds real memory the GC can't see the size of.+ if (this.still && this.still.image instanceof ImageBitmap) this.still.image.close();+ this.still = null;+ this.input?.dispose();+ this.input = null;+ this.sink = null;+ }++ /**+ * An open failed. That is usually a moment — a network blip, a signed URL a+ * few seconds past its window — so the source tells the link keeper (which+ * re-mints an expired URL; the pool then swaps this source out under the new+ * one) and books itself another try. Only a file that keeps failing stays+ * marked unreadable, and only after it has had its chances.+ */+ private fail(): void {+ this.unreadable = true;+ this.opening = null;+ void import("./mediaLinks").then((m) => m.reportMediaUrlError(this.asset.url));+ if (this.closed || this.attempts >= RETRIES || typeof window === "undefined") return;+ this.attempts++;+ this.retryTimer = window.setTimeout(() => {+ this.retryTimer = 0;+ if (this.closed) return;+ this.unreadable = false;+ // Nothing asks a failed source for frames, so nothing would notice it is+ // willing again without a nudge.+ this.onFrame();+ }, RETRY_MS * this.attempts);+ }++ private open(): Promise<void> {+ if (this.opening) return this.opening;+ this.opening = (async () => {+ const input = openMedia(this.asset.url);+ try {+ const track = await videoTrackOf(input);+ if (!track) {+ input.dispose();+ this.unreadable = true;+ return;+ }+ if (this.closed) {+ input.dispose();+ return;+ }+ this.input = input;+ // Height alone: the sink keeps the source's aspect and applies the+ // file's rotation, so a phone clip arrives upright at preview size and+ // no caller has to know it was ever sideways.+ this.sink = frameSink(track, { height: this.height }, POOL);+ } catch {+ input.dispose();+ this.fail();+ }+ })();+ return this.opening;+ }++ private async openStill(): Promise<void> {+ if (this.opening) return this.opening;+ this.opening = (async () => {+ try {+ const res = await fetch(this.asset.url, { mode: "cors" });+ const bitmap = await createImageBitmap(await res.blob());+ if (this.closed) return bitmap.close();+ this.still = {+ image: bitmap,+ width: bitmap.width,+ height: bitmap.height,+ timestamp: 0,+ };+ this.onFrame();+ } catch {+ this.fail();+ }+ })();+ return this.opening;+ }++ private stopStream(): void {+ const s = this.stream;+ this.stream = null;+ this.streaming = false;+ void s?.return(undefined).catch(() => {});+ }++ /**+ * Keep the forward walk running and roughly `DECODE_AHEAD_S` ahead of `t`.+ *+ * A walk already covering `t` is left alone — restarting it would throw away+ * the buffer it has built and re-decode from the nearest keyframe, which is+ * the hitch this design exists to remove. It restarts only for a real jump:+ * a scrub, or a clip re-entered from somewhere else.+ */+ private pumpStream(t: number): void {+ // Whether the walk covers `t` is a question about the walk — where it+ // started and the last frame it has landed. A walk whose first frame is+ // still decoding is already the right walk: that first step is a keyframe+ // seek, longer than a display frame, and judging it by frames landed would+ // tear it down on every tick and it would never land anything. It gets a+ // second of slack for the reader to advance while that first frame comes.+ if (this.stream) {+ // The walk serves `t` if the frame is already held, or if `t` sits just+ // past the tail — where the next pulls will land. Its span behind the+ // tail proves nothing: frames there have been aging out of the ring the+ // whole time, so a reader that went back to the walk's beginning would+ // find a record of coverage and no frames.+ const landed = this.streamTail > this.streamFrom + SAME;+ const ahead = landed+ ? t >= this.streamTail - SAME && t <= this.streamTail + DECODE_AHEAD_S+ : t >= this.streamFrom - SAME && t <= this.streamFrom + DECODE_AHEAD_S + 1;+ if (ahead || this.ring.covers(t)) {+ void this.drain(t);+ return;+ }+ this.stopStream();+ } else if (this.streamDone && (this.ring.covers(t) || t >= this.streamTail - SAME)) {+ // The walk ended at the file's last frame. Inside its span the frame is+ // already held, and past its tail there is nothing left to decode — the+ // newest frame held is the answer. Restarting here would re-decode the+ // whole tail from its keyframe once per tick, forever.+ return;+ }+ this.streamFrom = t;+ this.streamTail = t;+ this.streamDone = false;+ void this.startStream(t);+ }++ private async startStream(from: number): Promise<void> {+ await this.open();+ if (this.closed || !this.sink || this.stream) return;+ // The read may have been overtaken while the file was opening.+ if (Math.abs(this.streamFrom - from) > SAME) return;+ this.stream = this.sink.canvases(Math.max(0, from));+ void this.drain(from);+ }++ /** Pull frames until the ring reaches `DECODE_AHEAD_S` past `t`. */+ private async drain(t: number): Promise<void> {+ if (this.streaming || !this.stream) return;+ this.streaming = true;+ try {+ for (;;) {+ const stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = this.stream;+ if (!stream || this.closed) break;+ // Stop at the lookahead, and stop early once the walk's own frames at+ // or past `t` fill the ring — pushing another would drop one that is+ // still wanted. Only the walk's frames count: leftovers from other+ // gestures merely age out of the cache as the walk pushes.+ if (this.streamTail >= t + DECODE_AHEAD_S) break;+ if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;+ const { value, done } = await stream.next();+ // A restart or a close while awaiting: this walk is no longer the one.+ if (this.stream !== stream) break;+ if (done || !value) {+ this.stream = null;+ this.streamDone = true;+ break;+ }+ this.streamTail = value.timestamp;+ this.ring.push(value);+ this.onFrame();+ }+ } catch {+ this.stopStream();+ } finally {+ this.streaming = false;+ }+ }++ /** Fetch the single frame a paused reader is waiting on. */+ private async pumpSeek(): Promise<void> {+ if (this.reading) return;+ this.reading = true;+ try {+ await this.open();+ for (;;) {+ const t = this.wanted;+ if (t === null || this.closed || !this.sink) break;+ this.wanted = null;+ const c = await this.sink.getCanvas(Math.max(0, t));+ if (this.closed) break;+ if (c) {+ this.ring.push(c);+ this.onFrame();+ }+ // Another position was asked for while this one decoded; that one is+ // where the pointer actually is now.+ if (this.wanted === null) break;+ }+ } catch {+ this.wanted = null;+ } finally {+ this.reading = false;+ }+ }+}++/**+ * The live decoders.+ *+ * A decoder is not free: an open read against the file, a demuxer, and one of+ * the small number of hardware decode slots a tab is given. Past that limit+ * decoding falls back to software, and since sound stays real-time while frames+ * arrive late, the picture drifts behind the audio. So the pool is capped, and+ * the sources this frame is built from are never the ones evicted.+ */+export class FrameSourcePool {+ private sources = new Map<string, ClipFrameSource>();+ private tick = 0;++ constructor(+ private budget = 10,+ private readonly onFrame: () => void = () => {}+ ) {}++ /** Advance the clock the eviction order is measured on. */+ beginFrame(): void {+ this.tick++;+ }++ /**+ * The source for one clip mapping, opening it if this is the first ask.+ * `key` must identify a mapping — same file, speed and source-time offset —+ * so clips showing identical pictures share one decoder.+ */+ get(key: string, asset: MediaAsset, height: number): ClipFrameSource {+ const id = `${key}|${height}`;+ let src = this.sources.get(id);+ // The mapping names which pictures; the URL is where they are read from,+ // and it moves — a signed link re-mints, a shot re-renders onto its asset.+ // A source is only current while it reads the store's current URL.+ if (src && src.url !== asset.url) {+ src.close();+ this.sources.delete(id);+ src = undefined;+ }+ if (!src) {+ src = new ClipFrameSource(asset, height, this.onFrame);+ this.sources.set(id, src);+ }+ src.touched = this.tick;+ return src;+ }++ /** Decoded frames held across every source, for the budget and the trace. */+ get held(): number {+ let n = 0;+ for (const s of this.sources.values()) n += s.held;+ return n;+ }++ get size(): number {+ return this.sources.size;+ }++ /**+ * Close the decoders nothing has asked for lately.+ *+ * "Lately" is doing real work here. Evicting everything untouched by the+ * current frame reads as tidy and behaves terribly: a cut with more clips+ * than the budget closes and reopens decoders every single frame, and the+ * reopening — a fresh read of the file, a fresh decoder — is far more+ * expensive than the memory it was saving. A source is only a candidate once+ * nothing has wanted it for a while, which is long enough that a clip being+ * crossed back and forth over is never the one closed.+ */+ evict(): void {+ if (this.sources.size <= this.budget) return;+ const idle = [...this.sources.entries()]+ .filter(([, s]) => this.tick - s.touched >= EVICT_GRACE)+ .sort((a, b) => a[1].touched - b[1].touched);+ let over = this.sources.size - this.budget;+ for (const [id, src] of idle) {+ if (over-- <= 0) break;+ src.close();+ this.sources.delete(id);+ }+ }++ closeAll(): void {+ for (const s of this.sources.values()) s.close();+ this.sources.clear();+ }+}++/**+ * The decode identity of a clip: the frames it shows are a function of its+ * file, its speed, and where its source time stands at timeline zero. Two clips+ * agreeing on all three show the same picture at every instant.+ */+export function mappingKey(+ assetId: string,+ speed: number,+ inPoint: number,+ start: number+): string {+ return `${assetId}|${speed}|${(inPoint - start * speed).toFixed(3)}`;+}diff --git a/site/src/cut/lib/mediaLinks.ts b/site/src/cut/lib/mediaLinks.tsindex 28dd573b..8ba21a77 100644--- a/site/src/cut/lib/mediaLinks.ts+++ b/site/src/cut/lib/mediaLinks.ts@@ -18,11 +18,11 @@// hands back the same string, so the reload is what clears an element that// failed for any other reason.//-// A re-mint swaps the store's asset URLs in place. Components and the preview-// engine both key their elements off asset.url, so the swap heals them on-// render. The store marks the batch at load; a global capture-phase error-// listener watches DOM elements, and the preview engine reports its detached-// decoders through reportMediaElementError.+// A re-mint swaps the store's asset URLs in place. Media elements key off+// asset.url and the preview's decoder pool compares each source against it, so+// the swap heals both on the next render. The store marks the batch at load; a+// global capture-phase error listener watches DOM elements, and the preview's+// frame sources report their failed reads through reportMediaUrlError.import { fetchSignedMediaUrls } from "./backend/cloud";import { mediaUrl } from "./types";@@ -143,6 +143,18 @@ function retryElement(el: HTMLImageElement | HTMLMediaElement, src: string, atte}, ELEMENT_RETRY_BASE_MS * attempt);}+/** A decoder failed to read a media URL. There is no element to reload — the+ * frame source retries on its own — but an expiring URL may genuinely have+ * expired, and only a re-mint heals that. The store swap repoints the decoder+ * pool, which opens a fresh source under the new URL. */+export function reportMediaUrlError(src: string) {+ if (!src || src.startsWith("blob:") || src.startsWith("data:")) return;+ if (!isExpiringUrl(src)) return;+ if (Date.now() - lastForcedAt < FORCED_GAP_MS) return;+ lastForcedAt = Date.now();+ void refreshSignedUrls(true);+}+/** A media element failed to load. An expiring URL may genuinely have expired,* so re-mint the whole batch — the store swap repoints every consumer. Then* reload the element either way: re-minting is not a cure on its own, becausediff --git a/site/src/cut/lib/perfTrace.ts b/site/src/cut/lib/perfTrace.tsnew file mode 100644index 00000000..e62dc3f7--- /dev/null+++ b/site/src/cut/lib/perfTrace.ts@@ -0,0 +1,152 @@+"use client";++/**+ * What the preview actually did, frame by frame.+ *+ * Scrub latency and a stutter at a cut are both claims about time, and neither+ * can be settled by watching. This records the two facts that settle them: when+ * a time was asked for, and when a frame for that time reached the screen. From+ * those, "the picture is one frame behind the pointer" and "the join dropped a+ * frame" become numbers a script can hold to a budget.+ *+ * The trace is off until something arms it, and while it is off every call here+ * is a null check. The eval arms it through `window.__cutPerf`; nothing in the+ * product turns it on.+ */++/** One composited frame that reached the canvas. */+export interface PresentRecord {+ /** Timeline time the frame was composited for. */+ t: number;+ /** When it was painted, on the `performance.now()` clock. */+ at: number;+ /** Source timestamp of the master layer's picture, for spotting a frame that+ * was shown twice or skipped. */+ srcTs: number | null;+ /** The clip the master picture came from, so a boundary is findable. */+ clipId: string | null;+ /**+ * Whether the frame drawn is the one that belongs at `t`.+ *+ * This is the measure that matters, and it is not the same as "a new+ * picture". Thirty-frame footage on a sixty-hertz display repeats every+ * source frame once and is perfectly smooth; a decoder that fell behind+ * repeats one too, and is not. Asking whether the source actually held the+ * wanted frame tells those apart.+ */+ exact: boolean;+ /** True when no decoded frame existed at all and the paint fell back to+ * holding or to black. A stale present never resolves a scrub. */+ stale: boolean;+}++/** A time the editor was asked to show, and when it was asked for. */+export interface SeekRecord {+ t: number;+ at: number;+ /** How long until a frame for this time was painted. Null while unresolved —+ * either still pending, or superseded by a later seek. */+ latencyMs: number | null;+}++export interface LongTaskRecord {+ at: number;+ ms: number;+}++export interface Trace {+ presents: PresentRecord[];+ seeks: SeekRecord[];+ longTasks: LongTaskRecord[];+ /** rAF callbacks the engine ran, for checking that an idle editor is idle. */+ ticks: number;+ /** Decoded frames the sources are holding open, sampled per present. */+ liveSamples: number[];+ startedAt: number;+}++/** Times equal within this are the same instant. Half a frame at 120fps —+ * tight enough that a real lag never passes, loose enough that float drift on+ * the timeline never fails. */+const SAME_TIME = 0.004;++let trace: Trace | null = null;+let observer: PerformanceObserver | null = null;+/** The seek still waiting for its picture. A later seek replaces it, which is+ * what makes a fast drag measure the position it settled on. */+let pendingSeek: SeekRecord | null = null;++/** Whether anything is listening. The engine's hot path checks this first. */+export const tracing = () => trace !== null;++export function startTrace(): void {+ trace = {+ presents: [],+ seeks: [],+ longTasks: [],+ ticks: 0,+ liveSamples: [],+ startedAt: performance.now(),+ };+ pendingSeek = null;+ if (typeof PerformanceObserver === "undefined") return;+ try {+ observer = new PerformanceObserver((list) => {+ for (const entry of list.getEntries()) {+ trace?.longTasks.push({ at: entry.startTime, ms: entry.duration });+ }+ });+ observer.observe({ entryTypes: ["longtask"] });+ } catch {+ // Long-task timing is Chromium-only. Its absence costs one metric, not the+ // run.+ observer = null;+ }+}++export function stopTrace(): Trace | null {+ const out = trace;+ observer?.disconnect();+ observer = null;+ trace = null;+ pendingSeek = null;+ return out;+}++/** A time was asked for — a scrub, a skim, a click on the ruler. */+export function markSeek(t: number): void {+ if (!trace) return;+ const rec: SeekRecord = { t, at: performance.now(), latencyMs: null };+ trace.seeks.push(rec);+ pendingSeek = rec;+}++/** A frame reached the canvas. */+export function markPresent(rec: Omit<PresentRecord, "at">): void {+ if (!trace) return;+ const at = performance.now();+ trace.presents.push({ ...rec, at });+ // A held or black frame is not an answer to the question the scrub asked —+ // neither is the neighbouring frame drawn while the real one decodes. The+ // clock keeps running until the frame that belongs at that time is on screen.+ if (pendingSeek && rec.exact && !rec.stale && Math.abs(rec.t - pendingSeek.t) <= SAME_TIME) {+ pendingSeek.latencyMs = at - pendingSeek.at;+ pendingSeek = null;+ }+}++/** Whether a time has been asked for that no frame has answered yet. Lets a+ * driver wait for the picture to catch up instead of guessing at a delay. */+export const awaitingFrame = () => pendingSeek !== null;++/** One turn of the engine's loop. */+export function markTick(): void {+ if (!trace) return;+ trace.ticks++;+}++/** How many decoded frames are being held open right now. */+export function markLiveSamples(n: number): void {+ if (!trace) return;+ trace.liveSamples.push(n);+}diff --git a/site/src/cut/lib/playhead.ts b/site/src/cut/lib/playhead.tsnew file mode 100644index 00000000..32d99870--- /dev/null+++ b/site/src/cut/lib/playhead.ts@@ -0,0 +1,130 @@+"use client";++/**+ * Where the preview is, and where the mouse is hovering over the timeline.+ *+ * These two numbers change more often than anything else in Cut — sixty times a+ * second while playing, and once per pointer event while dragging. They used to+ * live in the editor store beside the clips, which meant every one of those+ * changes ran the document's write normalizer and woke every mounted selector+ * in the editor, a few hundred of them, to ask whether a title bar had moved.+ *+ * So they live here instead, in the smallest store that can hold them: a+ * number, a set of listeners, and nothing else. Surfaces that draw the playhead+ * subscribe; surfaces that only need to know where it is when the user does+ * something read it directly. The document store is left to describe the+ * document.+ *+ * The clamp to the project's length lives with the callers that know the+ * length. This module holds a time and tells people it changed.+ */++import { useSyncExternalStore } from "react";+import { markSeek } from "./perfTrace";++let at = 0;+let skim: number | null = null;+const listeners = new Set<() => void>();++function notify() {+ for (const fn of listeners) fn();+}++function subscribe(fn: () => void) {+ listeners.add(fn);+ return () => {+ listeners.delete(fn);+ };+}++/**+ * Run `fn` on every move, outside React.+ *+ * For surfaces whose whole response to the clock is one DOM write — the+ * playhead's transform, the grade on the picture. Re-rendering a component to+ * set one style property costs a reconcile per frame and buys nothing.+ */+export const subscribePlayhead = subscribe;++/** The playhead, in timeline seconds. */+export const playheadAt = () => at;++/** The skimmer, or null when the pointer is off the timeline. */+export const skimAt = () => skim;++/**+ * The time the picture should show: the skimmer while one is live, the playhead+ * otherwise. One place decides this, so the preview and everything drawing+ * against the preview agree on which moment is on screen.+ */+export const previewAt = () => skim ?? at;++/** Move the playhead. Callers clamp to the project length first. */+export function setPlayhead(t: number): void {+ if (t === at) return;+ at = t;+ markSeek(t);+ notify();+}++export function setSkim(t: number | null): void {+ if (t === skim) return;+ skim = t;+ if (t !== null) markSeek(t);+ notify();+}++/**+ * Hold the playhead inside a timeline that just got shorter. Deleting the last+ * of a long row leaves the playhead standing past the new end, which puts the+ * readout ahead of the total and — since the playhead is placed with a+ * transform, and transforms count toward scrollable overflow — stretches the+ * scroll area into empty space.+ */+export function clampPlayhead(total: number): void {+ if (at > total) setPlayhead(total);+}++/** The playhead, as a subscription. Re-renders the caller on every change. */+export function usePlayhead(): number {+ return useSyncExternalStore(subscribe, playheadAt, () => 0);+}++export function useSkim(): number | null {+ return useSyncExternalStore(subscribe, skimAt, () => null);+}++export function usePreviewTime(): number {+ return useSyncExternalStore(subscribe, previewAt, () => 0);+}++/**+ * Something the preview time decides, without re-rendering when only the time+ * moved — whether a cue is the one being spoken, whether a clip is on screen.+ * `select` must return a primitive: React compares snapshots by identity, and a+ * fresh object every read would loop.+ */+export function usePreviewSelector<T extends string | number | boolean | null>(+ select: (t: number) => T+): T {+ return useSyncExternalStore(+ subscribe,+ () => select(previewAt()),+ () => select(0)+ );+}++/**+ * The preview time rounded to `steps` per second, for surfaces that want the+ * clock without wanting every frame of it — a picker's swatches, a panel's+ * live thumbnail. Rounding inside the snapshot means React compares the coarse+ * value and skips the render, so a 2 Hz caller re-renders twice a second while+ * the preview runs at sixty.+ */+export function usePreviewTimeEvery(steps: number): number {+ return useSyncExternalStore(+ subscribe,+ () => Math.round(previewAt() * steps),+ () => 0+ );+}diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsnew file mode 100644index 00000000..de798032--- /dev/null+++ b/site/src/cut/lib/previewMixer.ts@@ -0,0 +1,343 @@+"use client";++/**+ * The cut's sound while you watch it.+ *+ * Sound used to ride the same `<video>` and `<audio>` elements the picture came+ * from: the clip under the playhead was both the speaker and the clock, and+ * everything else chased it. That made the element's decode latency into the+ * project's timing, and it left no way to be sure the preview mixed a cut the+ * way the export would.+ *+ * Here every audible thing is a buffer scheduled on one `AudioContext`. The+ * context's own clock is what the preview runs on — it advances in real time,+ * never backward, and is sample-accurate — so the picture follows the sound+ * rather than the other way round. Clips are scheduled a window at a time so+ * starting playback costs one short decode instead of the whole timeline, and a+ * clip playing at a speed other than 1 is time-stretched rather than resampled,+ * which is what keeps its pitch and matches what the export writes.+ *+ * Gains are the only thing touched per frame, and only their value: the clip's+ * own volume, its fades, the duck under a live voiceover, and the whole-project+ * fade. The ramps themselves are the frame plan's, so the mix follows the same+ * description the picture does.+ */++import { decodeAudioSpan } from "./mediaRead";+import { timeStretch } from "./timeStretch";++/** How much of a voice is decoded and scheduled at once, in timeline seconds. */+const WINDOW_S = 20;+/** A stretched window is resynthesised on the main thread, so it is kept short+ * enough that the work lands inside a frame or two. */+const WINDOW_STRETCHED_S = 4;+/** Schedule the next window once the playhead is this close to running out. */+const AHEAD_S = 2;++/** One audible thing on the timeline: a clip's own sound, or a soundtrack item. */+export interface Voice {+ /** Stable across frames — the clip's id. A voice whose id survives keeps+ * playing; one that vanishes is stopped. */+ id: string;+ url: string;+ /** Where it starts on the timeline, and the source span it plays. */+ start: number;+ in: number;+ out: number;+ speed: number;+ /** Everything the frame plan says about how loud it is right now. */+ gain: number;+}++interface Scheduled {+ node: AudioBufferSourceNode;+ /** Timeline time this window ends at, so the next one starts there. */+ until: number;+}++interface LiveVoice {+ url: string;+ speed: number;+ in: number;+ out: number;+ start: number;+ gain: GainNode;+ /** Windows already handed to the context, oldest first. */+ windows: Scheduled[];+ /** Timeline time everything scheduled so far runs out at. */+ scheduled: number;+ /** A decode in flight, so a slow read doesn't start a second one. */+ filling: boolean;+ /** Set when the source can't be read; the voice stays silent rather than+ * retrying every frame. */+ dead: boolean;+}++/**+ * The preview's clock and mixer.+ *+ * One of these lives for as long as the editor is open. It holds the context,+ * the anchor that maps timeline seconds onto it, and one gain-fed voice per+ * audible clip.+ */+export class PreviewMixer {+ private ctx: AudioContext | null = null;+ private master: GainNode | null = null;+ private voices = new Map<string, LiveVoice>();+ /** Timeline time, context time and wall time of the same instant. Everything+ * scheduled is placed against this. */+ private anchor: { timeline: number; ctx: number; wall: number } | null = null;+ private decoded = new Map<string, AudioBuffer>();++ /** Whether the clock is running. */+ get running(): boolean {+ return this.anchor !== null;+ }++ private context(): AudioContext {+ if (!this.ctx) {+ this.ctx = new AudioContext();+ this.master = this.ctx.createGain();+ this.master.connect(this.ctx.destination);+ }+ return this.ctx;+ }++ /**+ * Where the playhead is, from the sound's own clock.+ *+ * This is the whole point of the arrangement: the number the picture is drawn+ * against comes from the same clock the samples leave on, so they cannot+ * drift apart no matter what the main thread is doing.+ */+ now(): number {+ if (!this.anchor) return 0;+ const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;+ // A browser holds an `AudioContext` suspended until the page has been+ // interacted with, and a suspended context's clock does not move. Reading+ // it blindly would freeze the picture on a project played before the user+ // has touched anything, and picking it up once it finally starts would jump+ // the playhead by however long it stayed asleep. So the wall carries the cut+ // until the audio clock agrees with it, and a clock that disagrees is+ // re-anchored rather than believed.+ if (!this.ctx || this.ctx.state !== "running") return wall;+ const audio = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);+ if (Math.abs(audio - wall) > 0.25) {+ this.anchor.ctx = this.ctx.currentTime - (wall - this.anchor.timeline);+ return wall;+ }+ return audio;+ }++ /**+ * Begin playing at timeline time `t`.+ *+ * The anchor is this very instant: the next `now()` reads `t` and counts+ * forward. Anchoring even slightly in the future reads *behind* `t` first,+ * which walks the playhead backwards on every play and, at the head of the+ * timeline, puts the clock before the first clip. The first audio window+ * pays for its own decode instead — `fill` starts a late window at the+ * current time with the missed sliver skipped, so the sound joins in sync a+ * few milliseconds in.+ */+ start(t: number): void {+ const ctx = this.context();+ void ctx.resume();+ this.stopVoices();+ this.anchor = { timeline: t, ctx: ctx.currentTime, wall: performance.now() };+ }++ /** Stop the clock and silence everything. */+ stop(): void {+ this.anchor = null;+ this.stopVoices();+ }++ /** The whole-project fade, over the finished mix. */+ setMasterGain(g: number): void {+ if (this.master) this.master.gain.value = Math.max(0, Math.min(1, g));+ }++ /**+ * Bring the mix in line with what is audible at timeline time `t`.+ *+ * Called every frame while playing. Voices that just became live are opened+ * and scheduled; voices that ran out are dropped; the rest have their gain+ * set to whatever the frame plan worked out. Nothing here awaits, so a slow+ * decode delays that one voice and nothing else.+ */+ update(t: number, voices: Voice[]): void {+ if (!this.anchor || !this.ctx) return;+ const seen = new Set<string>();+ for (const v of voices) {+ seen.add(v.id);+ // An edit to a playing clip arrives as the same id with new geometry — a+ // trim, a move, a speed change, a re-minted URL. Gain is the only part of+ // a voice that tunes live; everything else is baked into what was already+ // scheduled, so the voice reopens and the sound follows the edit the way+ // the picture does.+ const prev = this.voices.get(v.id);+ if (+ prev &&+ (prev.url !== v.url ||+ prev.start !== v.start ||+ prev.in !== v.in ||+ prev.out !== v.out ||+ prev.speed !== v.speed)+ ) {+ this.release(prev);+ this.voices.delete(v.id);+ }+ const live = this.voices.get(v.id) ?? this.open(v, t);+ if (!live) continue;+ live.gain.gain.value = Math.max(0, Math.min(1.5, v.gain));+ if (!live.dead && !live.filling && live.scheduled < t + AHEAD_S) void this.fill(v.id, live);+ // Windows that have finished playing hold their buffers alive.+ live.windows = live.windows.filter((w) => w.until > t - 1);+ }+ for (const [id, live] of this.voices) {+ if (seen.has(id)) continue;+ this.release(live);+ this.voices.delete(id);+ }+ }++ /** Drop the decode cache for a source that has been replaced. */+ forget(url: string): void {+ for (const key of [...this.decoded.keys()]) {+ if (key.startsWith(`${url}|`)) this.decoded.delete(key);+ }+ }++ dispose(): void {+ this.stop();+ this.decoded.clear();+ void this.ctx?.close();+ this.ctx = null;+ this.master = null;+ }++ private open(v: Voice, t: number): LiveVoice | null {+ const ctx = this.ctx;+ if (!ctx || !this.master) return null;+ const gain = ctx.createGain();+ gain.gain.value = 0;+ gain.connect(this.master);+ const live: LiveVoice = {+ url: v.url,+ speed: v.speed,+ in: v.in,+ out: v.out,+ start: v.start,+ gain,+ windows: [],+ // Scheduling begins where the playhead is: a voice opened mid-clip —+ // playback started inside it, or the clip edited while audible — decodes+ // from the moment it is heard, not from a head nobody will hear.+ scheduled: Math.max(v.start, t),+ filling: false,+ dead: false,+ };+ this.voices.set(v.id, live);+ return live;+ }++ private release(live: LiveVoice): void {+ for (const w of live.windows) {+ try {+ w.node.stop();+ } catch {+ // Already finished; stopping a spent node throws and means nothing.+ }+ w.node.disconnect();+ }+ live.windows = [];+ live.gain.disconnect();+ }++ private stopVoices(): void {+ for (const live of this.voices.values()) this.release(live);+ this.voices.clear();+ }++ /**+ * Decode and schedule the next window of one voice.+ *+ * The window is measured in timeline seconds and the source span behind it+ * scales with the clip's speed, so a clip at 2× reads twice as much file to+ * fill the same stretch of timeline.+ */+ private async fill(id: string, live: LiveVoice): Promise<void> {+ if (!this.ctx || !this.anchor) return;+ live.filling = true;+ try {+ const stretched = Math.abs(live.speed - 1) > 1e-3;+ const span = stretched ? WINDOW_STRETCHED_S : WINDOW_S;+ const from = live.scheduled;+ const sourceFrom = live.in + (from - live.start) * live.speed;+ if (sourceFrom >= live.out - 1e-3) {+ live.dead = true;+ return;+ }+ const sourceTo = Math.min(live.out, sourceFrom + span * live.speed);+ const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;+ let buffer = this.decoded.get(key) ?? null;+ if (!buffer) {+ const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);+ if (!raw) {+ live.dead = true;+ return;+ }+ buffer = stretched ? this.stretch(raw, 1 / live.speed) : raw;+ // A handful of windows is all that's worth keeping — stepping back over+ // a cut replays them, and everything older is a full decode away anyway.+ if (this.decoded.size > 24) this.decoded.clear();+ this.decoded.set(key, buffer);+ }+ // The mixer may have been stopped, or the voice dropped, while decoding.+ const still = this.voices.get(id);+ if (!still || still !== live || !this.anchor || !this.ctx) return;+ const node = this.ctx.createBufferSource();+ node.buffer = buffer;+ node.connect(live.gain);+ const at = this.anchor.ctx + (from - this.anchor.timeline);+ const now = this.ctx.currentTime;+ const until = from + buffer.duration;+ if (until <= this.now()) {+ // The decode outlived the window it was for — a long read, or a tab+ // that was in the background. Skip it and let the next fill catch up.+ live.scheduled = Math.max(live.scheduled, this.now());+ return;+ }+ if (at >= now) {+ node.start(at);+ } else {+ // Late: start where the clock actually is, so the sound stays lined up+ // with the picture instead of replaying what has already gone by.+ node.start(now, now - at);+ }+ live.windows.push({ node, until });+ live.scheduled = until;+ } catch {+ live.dead = true;+ } finally {+ live.filling = false;+ }+ }++ /** Re-lay a buffer at a different length, keeping its pitch. */+ private stretch(buffer: AudioBuffer, factor: number): AudioBuffer {+ const channels: Float32Array[] = [];+ for (let c = 0; c < buffer.numberOfChannels; c++) channels.push(buffer.getChannelData(c));+ const out = timeStretch(channels, buffer.sampleRate, factor);+ const result = new AudioBuffer({+ length: Math.max(1, out[0]?.length ?? 1),+ numberOfChannels: Math.max(1, out.length),+ sampleRate: buffer.sampleRate,+ });+ // A fresh view per channel: `copyToChannel` wants a plain Float32Array,+ // and the stretch hands back views whose buffer type it declines.+ out.forEach((data, i) => result.copyToChannel(new Float32Array(data), i));+ return result;+ }+}