Skip to content

Open source CapCut alternative.

TypeScript
49.8 KB
Raw Blame
"use client";


import { useEffect, type RefObject } from "react";
import { noteStageDraw, noteStagePicture } from "@/cut/lib/posterCache";
import {
  getClipSpans,
  overlayLayers,
  projectDuration,
  useEditor,
} from "@/cut/lib/store";
import { headSrc, retimeOf, smoothsAt } from "@donkeycut/effects-kit";
import { blendInto, SYNTH_EDGE, synthWeight } from "@/cut/lib/frameSynth";
import { playheadAt, previewAt, setPlayhead, subscribePlayhead } from "@/cut/lib/playhead";
import { assetIsSilent, CLIP_MIN_SECONDS, clipCovers, rectOf } from "@/cut/lib/types";
import type { AudioClip, ClipSpan, MediaAsset, VideoClip } from "@/cut/lib/types";
import { OWN_SOUND, soundSourceOf, soundWindow } from "@/cut/lib/soundSource";
import { SubjectMaskCompositor } from "@/cut/lib/behindPass";
import { stageTimeAt } from "@/cut/lib/overlayLive";
import { FrameCompositor, MISSING_FRAME, PENDING_FRAME, type Frame } from "@/cut/lib/composite";
import { hdrCanvasSupport } from "@/cut/lib/hdrCanvas";
import { displayIsHdr, HdrPresenter, onDisplayRangeChange, setHdrPreviewState } from "@/cut/lib/hdrPresent";
import { crossHandles, duckGainAt, overlayPlan, soundCrossGain, trackZeroAt, trackZeroPlan } from "@/cut/lib/framePlan";
import { type ClipFrameSource, FrameSourcePool, mappingKey, walkCostMs } from "@/cut/lib/frameSource";
import { liveAudioFxAt } from "@/cut/lib/audioEffects";
import { PreviewMixer, type Voice } from "@/cut/lib/previewMixer";
import {
  flushMeter,
  markAudioClock,
  markLiveSamples,
  markLiveSources,
  markPresent,
  meterAudioClock,
  meterFrame,
  meterState,
  metering,
  markTick,
  tracing,
} from "@/cut/lib/perfTrace";
import { registerSourceColor, registerSourceSampler } from "@/cut/lib/previewCanvas";
import { colorRead, previewFile, type ColorRead } from "@/cut/lib/sourceColor";
import { backdropStill } from "@/cut/lib/backdropStills";
import { MatteAlpha, matteReadTime } from "@/cut/lib/matteAlpha";
import { createRasterCanvas, type RasterSurface } from "@/cut/lib/raster";
import { disposeDetailGpu } from "@/cut/lib/detailGpu";

/**
 * The preview engine.
 *
 * Three things meet here and nothing else does. The frame plan says what the
 * cut looks like at an instant; the frame sources hold decoded pictures ready
 * to answer for it; the compositor paints. The engine's whole job is to pick
 * the instant, ask, and draw.
 *
 * The instant comes from the mixer's clock while playing and from the playhead
 * while paused, so sound and picture are two readings of one number rather than
 * two clocks chasing each other. A cut is not an event: at the moment the plan
 * names a different clip, that clip's frames are already decoded, because the
 * scheduler has been reading ahead of the playhead the whole time.
 *
 * The loop runs while something is moving. A paused editor with nothing dirty
 * schedules no frames at all.
 */

/** How far ahead of the playhead a clip's decoder is opened and started. */
const WARM_HORIZON_S = 2.5;
/** Inside this, an upcoming clip is walked as well as opened. */
const WARM_STREAM_S = 0.75;
/** How long the master's picture has to be missing before a play is held at a
 * standstill. Short enough that a real stall is caught before the playhead has
 * gone anywhere, long enough that a decoder a frame behind at a join is not a
 * stutter and a cut in the sound. */
const STALL_HOLD_MS = 200;
/**
 * Decoders alive at once.
 *
 * Past the tab's hardware decode slots the rest fall back to software, where
 * they are the same processor split further, so it reads as a number that
 * should shrink on a machine with few slots. It does not. Shrinking it makes
 * a fast cut suspend and reopen sources it is about to need again, and a
 * reopen on such a machine costs a fresh decoder and a full walk from the
 * keyframe — the expensive thing, paid repeatedly. Measured on the eval's
 * few-slot profiles, cutting this to six moved late frames from 42% to 52%
 * and turned a flat play into one that decayed across its length.
 */
const DECODER_BUDGET = 12;

/** Source time of a clip at timeline time `t`. */
const sourceTimeOf = (clip: VideoClip, t: number) =>
  retimeOf(clip).srcAt(Math.max(0, t - clip.start));

/**
 * When the preview gives up resolution to keep the picture with the sound.
 *
 * A machine whose decoder cannot deliver a frame per frame period falls
 * further behind for as long as the play lasts: the sound goes on, the picture
 * does not, and nothing in the reader can invent the frames. Decoding smaller
 * is the one lever that reduces the work itself. The
 * trade is deliberate — a softer picture that stays with its sound beats a
 * sharp one seconds behind it.
 */
const TRAIL_LAG_S = 0.5;
/** How long the picture has to stay behind before the trade is taken. A cold
 * open hops once and catches up; this is about the state it cannot leave. */
const TRAIL_MS = 3_000;
/** Below this the preview would be mush, so it stops trading. */
const DECODE_FLOOR = 360;
/**
 * How long a play runs before the trade is on the table at all.
 *
 * Starting a play is the one moment every machine trails: nothing is decoded
 * yet, each source is finding its keyframe, and the first read of a file comes
 * off the link. That settles on its own within a few seconds and says nothing
 * about whether the machine can hold the cut. Trading resolution away for it
 * would leave every preview softer for the rest of the session on the strength
 * of its first breath.
 */
const PLAY_SETTLE_MS = 6_000;

/** The decode identity of a clip — see `mappingKey`. */
const keyOf = (clip: VideoClip, asset: MediaAsset) =>
  mappingKey(asset.id, retimeOf(clip), clip.start);

/** Assets by id, built once per assets array. */
const assetIndexes = new WeakMap<readonly MediaAsset[], Map<string, MediaAsset>>();
function assetIndex(assets: readonly MediaAsset[]): Map<string, MediaAsset> {
  let index = assetIndexes.get(assets);
  if (!index) assetIndexes.set(assets, (index = new Map(assets.map((a) => [a.id, a]))));
  return index;
}

/**
 * Which way a clip's source is read at this moment: playing, a reversed clip
 * reads down its source; paused, the reader's own direction along the
 * timeline turns around on a reversed clip.
 */
const readsDown = (clip: VideoClip, playing: boolean, backward: boolean) =>
  playing ? !!clip.reverse : backward !== !!clip.reverse;

let engineSerial = 0;

/** Dev-only: engine lifecycle events, for the perf eval's timeline. Bounded so
 * an ordinary dev session never accumulates it. */
export function engineLog(msg: string): void {
  if (process.env.NODE_ENV === "production" || typeof window === "undefined") return;
  const w = window as unknown as { __cutEngineLog?: { at: number; msg: string }[] };
  const log = (w.__cutEngineLog ??= []);
  log.push({ at: Math.round(performance.now()), msg });
  if (log.length > 500) log.splice(0, log.length - 500);
}

/**
 * The source span a clip's voice plays, and where it lands.
 *
 * A clip normally plays exactly what it is trimmed to. A cross dissolve widens
 * that: the clip reaches into the handle on either side — source it was
 * trimmed away from — so it is still sounding after the picture has cut, or
 * already sounding before it. The widened span rides the voice for the clip's
 * whole life rather than only inside the handover, so the mixer decodes and
 * schedules one stable voice and the frame-by-frame gain is all that moves.
 */
/**
 * How far ahead of the playhead a clip's sound is opened, in timeline seconds.
 *
 * A voice opened the frame its clip begins pays the open — a container parsed,
 * a decoder configured, a first read — while the clip is already playing, and
 * a clip shorter than that open is gone before its sound arrives. The clips
 * about to begin ride along with the audible ones, so the play reaches each
 * one with its file open and its first window scheduled.
 */
const SOUND_LOOKAHEAD_S = 3;

const voiceSpan = (sp: ClipSpan) => {
  // The handles a crossing or a split edit reaches into play at the pace of
  // the footage beside them, so the voice's span widens through the clip's
  // own map and its retime is rebuilt over the wider span. A recording bound
  // to the video is read through the same map, on its own clock, and starts
  // late where it rolled after the camera did.
  const w = soundWindow(sp.clip, sp.sound ?? OWN_SOUND, -sp.soundBack, sp.len + sp.soundAhead);
  if (!w) return null;
  return {
    url: (sp.sound?.asset ?? sp.asset).url,
    start: sp.start - sp.soundBack + w.at,
    in: w.lo,
    out: w.hi,
    retime: retimeOf(w.span),
    sound: sp.clip.sound,
  };
};

/** A span with nothing to hear: its video has no sound track and no
 * recording bound to it. */
const spanSilent = (sp: ClipSpan) => !sp.sound && assetIsSilent(sp.asset);

/** An audio clip's sound: its own file, or the recording bound to the video
 * it was detached from. */
const audioVoice = (a: AudioClip, asset: MediaAsset, assets: MediaAsset[]) => {
  const src = soundSourceOf(asset, assets);
  if (!src.bound) return { url: asset.url, start: a.start, in: a.in, out: a.out, retime: retimeOf(a) };
  const w = soundWindow(a, src, 0, retimeOf(a).len);
  if (!w) return null;
  return { url: src.asset.url, start: a.start + w.at, in: w.lo, out: w.hi, retime: retimeOf(w.span) };
};

class Engine {
  /** Which engine this is, for the perf eval's pool dump. */
  readonly serial = ++engineSerial;

  private comp: FrameCompositor;
  private pool = new FrameSourcePool(DECODER_BUDGET, () => this.wake());
  private mixer = new PreviewMixer();
  private behind = new SubjectMaskCompositor(true);

  private raf = 0;
  private disposed = false;
  /** Set when something changed that the canvas has yet to show. */
  private dirty = true;
  /** Sources this frame is being drawn from. The warm pass leaves these alone:
   * a warm ask lands on the same source as a live clip whenever they share a
   * mapping — a plain split reads straight across its own cut — and steering a
   * walk that is already carrying the picture would tear it down every tick. */
  private used = new Set<ClipFrameSource>();
  /** The playhead value the engine itself wrote last, so its own echo is
   * tellable from an outside move — a seek while playing. */
  private written: number | null = null;
  /** Wall time the master's picture went missing, 0 while it is there. */
  private stalledAt = 0;
  /** How many 180-steps the preview has traded away, for the engine's life. */
  private dropped = 0;
  private steppedThisPlay = false;
  private trailingSince = 0;
  private playSince = 0;
  private wasPlaying = false;
  /** The last paused moment drawn, and whether the reader is heading down the
   * timeline. A paused reader's direction is a fact about the gesture, known
   * here and passed to every source read, since a source only ever sees its
   * own asks. */
  private lastRead: number | null = null;
  private readBackward = false;
  private lastWalkCost = 0;
  /** Dev only: where a slow tick spent its time, phase by phase. */
  private phases: string[] = [];
  private phaseAt = 0;
  private phase(label: string) {
    if (process.env.NODE_ENV === "production") return;
    const now = performance.now();
    this.phases.push(`${label} ${Math.round(now - this.phaseAt)}`);
    this.phaseAt = now;
  }
  /** Whether the tick being drawn is a playing one — read by the matte
   * provider, which the compositor calls mid-frame. */
  private renderPlaying = false;
  /** Per matte read: the last decoded matte luma frame converted to alpha,
   * so the conversion runs once per new frame rather than once per draw. Keyed
   * by the matte's mapping — a card that takes over its cutout's matte shares
   * the cutout's conversion, while two clips reading one matte at different
   * times each keep their own — and bounded LRU, so a long session's re-bakes
   * and deleted clips never pile up canvases. */
  private matteAlpha = new Map<string, MatteAlpha>();
  private static readonly MATTE_ALPHA_MAX = 8;
  /** Per clip, the canvas its smoothed slow motion is blended on. Bounded
   * like the mattes: the clips drawn longest ago give theirs up. */
  private smoothed = new Map<string, RasterSurface>();
  private static readonly SMOOTHED_MAX = 4;

  private unsubscribe: () => void;
  private unwatch: () => void;
  private sizeWatch: MutationObserver;
  private onHidden: () => void;
  private unwatchDisplay: () => void;

  /** The HDR present pass over the composite, once the browser has handed
   * over an extended-range canvas; null until then and where it cannot. */
  private presenter: HdrPresenter | null = null;
  /** Whether the presenter is still being made — a project that turns HDR
   * later asks once. */
  private presenting = false;
  /** Whether the browser hands over an extended-range canvas: unknown until
   * the first HDR frame asks, so an SDR project never probes the GPU. */
  private canPresent: boolean | null = null;

  constructor(
    private canvas: HTMLCanvasElement,
    /** The extended-range canvas over the stage the HDR picture lands on. */
    private presentCanvas: HTMLCanvasElement | null
  ) {
    this.comp = new FrameCompositor(canvas);
    this.comp.removalMatteProvider = (clip, at) => this.matteFor(clip.removal?.matte, clip, at);
    this.comp.cardMatteProvider = (clip, at) => this.matteFor(clip.card?.matte, clip, at);
    // A clip's color recipe describes the frames its source hands over — the
    // file that source reads, read the way it reads it; a LUT built off the
    // thread repaints when it lands.
    this.comp.sourceProvider = (clip) => this.colorOf(clip)?.recipe() ?? { profile: "rec709" };
    this.comp.wake = () => this.wake();
    // A backdrop still decodes out of band; the frame that finds it missing
    // draws without it and the landing repaints.
    this.comp.backdropImageProvider = (assetId) =>
      backdropStill(useEditor.getState().assets, assetId, () => this.wake());
    this.tick = this.tick.bind(this);
    // A moved playhead and an edited document both change the picture. Either
    // one wakes the loop; nothing else does, so an idle editor costs nothing.
    this.unsubscribe = subscribePlayhead(() => this.wake());
    this.unwatch = useEditor.subscribe(() => this.wake());
    // The backing store follows the stage box, and assigning a canvas's width
    // or height erases it. That write comes from layout — no playhead move, no
    // document edit — so the attributes themselves are the wake signal.
    this.sizeWatch = new MutationObserver(() => this.wake());
    this.sizeWatch.observe(canvas, { attributes: true, attributeFilter: ["width", "height"] });
    // An editor left in a background tab keeps everything it was holding: the
    // frame loop is where sources are stood down, and a hidden tab is given
    // almost no frames to do it in. So the memory an editing session builds up
    // — decoders, the canvases behind them, the bytes they were decoded from —
    // stays out of the machine's reach for as long as the tab sits there,
    // which is exactly when the person has gone to do something else with it.
    // Letting go costs one keyframe seek per visible clip on return.
    this.onHidden = () => {
      if (document.visibilityState !== "hidden" || useEditor.getState().playing) return;
      engineLog(`engine ${this.serial} backgrounded, releasing`);
      this.pool.closeAll();
      this.behind.clear();
      this.matteAlpha.clear();
      this.smoothed.clear();
      this.mixer.releaseCaches();
      // The canvas keeps showing the frame it last painted, so nothing is
      // blank while the tab is away. The wake is for the return: it books the
      // repaint that reopens the sources, and a hidden tab runs it when it is
      // hidden no longer.
      this.wake();
    };
    document.addEventListener("visibilitychange", this.onHidden);
    // A window dragged onto or off an HDR display changes what the stage can
    // show; the answer is read per frame and the change wakes a repaint.
    this.unwatchDisplay = onDisplayRangeChange(() => this.wake());
    engineLog(`engine ${this.serial} constructed`);
    this.wake();
  }

  /**
   * Whether this frame draws the HDR picture: the project delivers HDR, the
   * display carries it, and the extended-range canvas is in hand. The
   * presenter is made on the first HDR frame asked for, off the loop; until
   * it lands the stage shows the grade rendered as SDR.
   */
  private hdrFrame(colorSpace: string): boolean {
    if (colorSpace === "sdr" || !this.presentCanvas || !displayIsHdr()) return false;
    if (this.canPresent === null) {
      if (!this.presenting) {
        this.presenting = true;
        void hdrCanvasSupport().then((support) => {
          this.presenting = false;
          if (this.disposed) return;
          this.canPresent = support.webgpu;
          this.wake();
        });
      }
      return false;
    }
    if (!this.canPresent) return false;
    if (this.presenter?.gone) {
      this.presenter.dispose();
      this.presenter = null;
    }
    if (this.presenter) return true;
    if (!this.presenting) {
      this.presenting = true;
      void HdrPresenter.create(this.presentCanvas).then((p) => {
        this.presenting = false;
        if (this.disposed) {
          p?.dispose();
          return;
        }
        if (p) this.presenter = p;
        else this.canPresent = false;
        this.wake();
      });
    }
    return false;
  }

  dispose() {
    engineLog(`engine ${this.serial} disposed`);
    flushMeter();
    this.disposed = true;
    if (this.raf) cancelAnimationFrame(this.raf);
    this.unsubscribe();
    this.unwatch();
    this.sizeWatch.disconnect();
    document.removeEventListener("visibilitychange", this.onHidden);
    this.unwatchDisplay();
    this.presenter?.dispose();
    this.presenter = null;
    setHdrPreviewState("off");
    this.pool.dispose();
    this.behind.dispose();
    this.mixer.dispose();
    disposeDetailGpu();
  }

  /**
   * The height sources decode at: the canvas, rounded up to a coarse step.
   *
   * The pool keys sources on this number. Keyed on the raw canvas height, a
   * divider drag — which resizes the backing store pixel by pixel — would mint
   * a fresh decoder for every intermediate size and blow through the tab's
   * decode slots in a second. Rounding up keeps the identity still while the
   * box moves and never decodes below display size, and the step is small
   * enough that the extra rows decoded above it stay cheap.
   *
   * Nothing the window does moves it, for a second reason: the pool keys on
   * it, so a height that follows the canvas would abandon every open decoder
   * each time the box moved and pay a fresh keyframe walk for each.
   *
   * `noteLag` moves it exactly once per play, and pays that price knowingly.
   * The abandoned sources keep their decoders until the pool stands them down,
   * so for a second or two the machine holds about twice the decoders it needs
   * — on a machine already behind, which is the whole reason the step is being
   * taken. It is worth it because the alternative is a play that never catches
   * up at all, and because it happens once: the step is one-way, so the cost is
   * paid a single time and a wobbling picture never pays it again.
   */
  private decodeHeight(): number {
    const asked = Math.ceil(this.canvas.height / 180) * 180 || 180;
    return Math.max(DECODE_FLOOR, asked - this.dropped * 180);
  }

  /**
   * The picture has been behind the sound for a while, so decode smaller.
   *
   * One step per play and never back up. Stepping re-keys the pool, so every
   * open decoder is abandoned and each source pays a fresh keyframe walk;
   * paying that twice to undo it would cost more than the pixels are worth,
   * and paying it repeatedly inside one play is the storm the fixed height
   * exists to avoid.
   */
  private noteLag(lag: number, now: number, byteBound: boolean): void {
    if (lag <= TRAIL_LAG_S) {
      this.trailingSince = 0;
      return;
    }
    // Behind because the bytes are not here yet, which decoding smaller does
    // not help. The clock does not start while that is true, so the stretch
    // that earns the trade is one the decoder was responsible for.
    if (byteBound) return;
    if (!this.trailingSince) {
      this.trailingSince = now;
      return;
    }
    if (now - this.trailingSince < TRAIL_MS) return;
    if (now - this.playSince < PLAY_SETTLE_MS) return;
    if (this.steppedThisPlay || this.decodeHeight() <= DECODE_FLOOR) return;
    this.steppedThisPlay = true;
    this.trailingSince = 0;
    this.dropped++;
    engineLog(`decode step down to ${this.decodeHeight()}`);
    this.wake();
  }

  /** The engine's own playhead writes go through here, so `written` always
   * says what the engine last wrote. */
  private writeHead(t: number) {
    this.written = t;
    setPlayhead(t);
  }

  /** Something changed: draw at the next opportunity. */
  private wake() {
    this.dirty = true;
    this.schedule();
  }

  /**
   * Ask for one frame, and only one.
   *
   * Drawing writes the playhead, and writing the playhead wakes the engine, so
   * a tick that scheduled its own successor unconditionally would leave two
   * callbacks pending where it meant to leave one — and each of those would
   * leave two more. Every path books a frame through here, and here refuses to
   * book a second.
   */
  private schedule() {
    if (this.raf || this.disposed) return;
    this.raf = requestAnimationFrame(this.tick);
  }

  /** Every open decoder and what it is doing, for the perf eval's pool dump. */
  poolState(): Record<string, unknown>[] {
    return [
      { serial: this.serial, disposed: this.disposed, raf: this.raf, dirty: this.dirty },
      ...this.pool.debugState(),
    ];
  }

  /** How the clip's frames are read: the frame its open source last handed
   * out, else the file the preview would open for it. */
  colorOf(clip: VideoClip): ColorRead | null {
    const asset = assetIndex(useEditor.getState().assets).get(clip.assetId);
    if (!asset) return null;
    const src = this.pool.peek(keyOf(clip, asset), this.decodeHeight());
    return src ? src.shownColorRead : colorRead(asset, previewFile(asset));
  }

  /** The clip's raw, ungraded decoder frame for analysis (the color panel's
   * Auto), or null when nothing has been decoded for it. */
  sourceFor(clipId: string): CanvasImageSource | null {
    const s = useEditor.getState();
    const clip = s.clips.find((c) => c.id === clipId);
    const asset = clip && s.assets.find((a) => a.id === clip.assetId);
    if (!clip || !asset) return null;
    const src = this.pool.get(keyOf(clip, asset), asset, this.decodeHeight());
    return src.frameAt(sourceTimeOf(clip, previewAt()), clip.in, clip.out)?.image ?? null;
  }

  /**
   * The alpha matte `m` — a removal clip's, or a camera card's — at timeline
   * time `t`: the baked matte video read like any other source, its luma
   * turned into alpha and held under the read's mapping. Null while nothing
   * has decoded yet — the picture draws plain until the matte lands.
   */
  private matteFor(
    m: { assetId: string; in: number } | undefined,
    clip: VideoClip,
    at: number
  ): CanvasImageSource | null {
    if (!m) return null;
    const s = useEditor.getState();
    const asset = s.assets.find((a) => a.id === m.assetId);
    if (!asset) return null;
    const dur = Math.max(0.001, asset.duration);
    const mt = matteReadTime(Math.max(0, sourceTimeOf(clip, at) - m.in), dur);
    // Coverage tolerates far less resolution than the picture, and the luma →
    // alpha read below is a per-frame pixel pass; capping the matte's decode
    // keeps that pass off the frame budget.
    const key = mappingKey(asset.id, retimeOf(clip), clip.start, m.in);
    const src = this.pool.get(
      key,
      asset,
      Math.min(this.decodeHeight(), 480)
    );
    this.used.add(src);
    src.want(
      mt,
      this.renderPlaying,
      readsDown(clip, this.renderPlaying, this.readBackward),
      retimeOf(clip).rateAt(at - clip.start)
    );
    const frame = src.frameAt(mt, 0, dur);
    if (!frame) return null;
    const alpha = this.matteAlpha.get(key) ?? new MatteAlpha();
    // Re-insert on every touch so the cap below drops the matte drawn longest
    // ago.
    this.matteAlpha.delete(key);
    this.matteAlpha.set(key, alpha);
    if (this.matteAlpha.size > Engine.MATTE_ALPHA_MAX) {
      const oldest = this.matteAlpha.keys().next().value;
      if (oldest !== undefined) this.matteAlpha.delete(oldest);
    }
    return alpha.of(frame, String(frame.timestamp));
  }

  /**
   * The picture a clip shows at timeline time `t`.
   *
   * Asking is never a wait. A source with the exact frame gives it; one running
   * behind gives the nearest it has, which is a held frame rather than a hole;
   * one that has decoded nothing yet is `pending`, which tells the compositor to
   * leave what is already on screen alone. Painting black for a decoder that is
   * merely a moment behind is what strobing looks like.
   */
  private frameFor(span: ClipSpan, t: number, playing: boolean, master = false): Frame {
    const src = this.pool.get(keyOf(span.clip, span.asset), span.asset, this.decodeHeight());
    this.used.add(src);
    const st = sourceTimeOf(span.clip, t);
    src.want(
      st,
      playing,
      readsDown(span.clip, playing, this.readBackward),
      retimeOf(span.clip).rateAt(t - span.start)
    );
    // A failed source that already decoded frames keeps showing the nearest
    // one it holds — a transient blip (a network drop, a signed URL mid
    // re-mint) reads as a held frame, and only a source with nothing at all
    // to show goes missing. The clip's own span bounds the answer: two clips
    // split from one file share a source, and a held frame from across the
    // split would show the other scene at a paused playhead.
    const frame = this.smoothFrame(span, t, st, src);
    // The picture on track 0 is the one being watched, so it is the one the
    // machine is judged on — see the meter in `perfTrace`.
    if (master && playing) {
      const lag = frame ? Math.max(0, st - frame.timestamp) : 0;
      // The trade does not wait for anyone to turn diagnostics on.
      this.noteLag(lag, performance.now(), src.byteBound);
      if (metering()) meterFrame(lag, !frame);
    }
    if (frame)
      return { kind: "ready", image: frame.image, width: frame.width, height: frame.height, source: frame.read?.recipe() };
    return src.failed ? MISSING_FRAME : PENDING_FRAME;
  }

  /**
   * The clip's picture at source time `st`, blended between the two source
   * frames around it where the clip smooths its slow motion and the rate is
   * under 1×. The blend is the preview's stand-in for the export's motion
   * estimate: two draws, inside the frame budget. Everywhere else, and at
   * either edge of the pair, the source frame shows as is.
   */
  private smoothFrame(span: ClipSpan, t: number, st: number, src: ClipFrameSource) {
    const { clip } = span;
    if (!smoothsAt(clip, retimeOf(clip), t - span.start)) return src.frameAt(st, clip.in, clip.out);
    const pair = src.pairAt(st, clip.in, clip.out);
    if (!pair) return null;
    if (!pair.b) return pair.a;
    const w = synthWeight(st, pair.a.timestamp, pair.b.timestamp);
    if (w <= SYNTH_EDGE) return pair.a;
    if (w >= 1 - SYNTH_EDGE) return pair.b;
    const { width, height } = pair.a;
    let canvas = this.smoothed.get(clip.id);
    // Re-insert on every touch so the cap below drops the clip drawn longest ago.
    this.smoothed.delete(clip.id);
    if (!canvas) canvas = createRasterCanvas(width, height);
    if (canvas.width !== width) canvas.width = width;
    if (canvas.height !== height) canvas.height = height;
    this.smoothed.set(clip.id, canvas);
    if (this.smoothed.size > Engine.SMOOTHED_MAX) {
      const oldest = this.smoothed.keys().next().value;
      if (oldest !== undefined) this.smoothed.delete(oldest);
    }
    blendInto(canvas, pair.a.image, pair.b.image, w);
    return { image: canvas as CanvasImageSource, width, height, timestamp: pair.a.timestamp, read: pair.a.read };
  }

  /** Open and start the decoders for clips about to arrive — on track 0 and
   * the overlay tracks alike — so a cut or an overlay's entrance lands on a
   * ring that is already full. */
  private warm(t: number, playing: boolean) {
    if (process.env.NODE_ENV !== "production") {
      const cost = walkCostMs();
      if (cost && cost !== this.lastWalkCost) {
        this.lastWalkCost = cost;
        engineLog(`walk cost ${cost}ms`);
      }
    }
    const s = useEditor.getState();
    const lists = [getClipSpans(s.clips, s.assets)];
    for (const track of new Set(overlayLayers(s.clips).map((c) => c.track))) {
      lists.push(getClipSpans(s.clips, s.assets, track));
    }
    // A pointer backing down the timeline arrives at the previous clip's last
    // frame, so that clip's tail is landed backward ahead of it: the cut is
    // crossed onto windows already there.
    if (this.readBackward) {
      for (const spans of lists) {
        for (const sp of spans) {
          const end = sp.start + sp.len;
          if (end > t || t - end > WARM_STREAM_S) continue;
          const src = this.pool.get(keyOf(sp.clip, sp.asset), sp.asset, this.decodeHeight());
          if (this.used.has(src)) continue;
          this.used.add(src);
          // A reversed clip's tail is its source's `in`, and backing into it
          // reads up the source from there.
          const tail = sp.clip.reverse
            ? sp.clip.in
            : Math.max(sp.clip.in, sourceTimeOf(sp.clip, end) - 0.001);
          src.want(tail, false, !sp.clip.reverse);
        }
      }
    }
    // The clips ahead are for a reader moving that way. One backing down the
    // timeline is leaving them, and a walk started for the nearest — sent as
    // if playing — would take the decoder the backward walk is using and let
    // go of the frames it landed.
    if (this.readBackward) return;
    for (const spans of lists) {
      for (const sp of spans) {
        if (sp.start <= t || sp.start > t + WARM_HORIZON_S) continue;
        const src = this.pool.get(keyOf(sp.clip, sp.asset), sp.asset, this.decodeHeight());
        // A source already carrying this frame's picture — or already warmed
        // for a nearer span — needs nothing. Its walk reads across the join on
        // its own, and a second ask would restart it.
        if (this.used.has(src)) continue;
        this.used.add(src);
        // Start where the clip opens. A walk is the useful thing to have ready —
        // the first step past a join then costs one frame rather than a seek —
        // but a walk started for every clip on the horizon spends decode a drag
        // elsewhere on the timeline is waiting for. So the clip about to be
        // reached gets a walk, and the ones behind it get the single frame that
        // lets a cut land on something.
        const imminent = sp.start - t <= WARM_STREAM_S;
        src.want(headSrc(sp.clip), playing || imminent, !!sp.clip.reverse);
      }
    }
  }

  /** Overlay clips live at `t`, with their assets and ramps, in draw order. */
  private liveOverlays(t: number) {
    const s = useEditor.getState();
    return overlayPlan(
      [...new Set(overlayLayers(s.clips).map((c) => c.track))],
      (track) => getClipSpans(s.clips, s.assets, track),
      t
    );
  }

  /** Each upper track's spans, for the sound side — a cross dissolve up there
   * crosses the same way track 0's does. */
  private overlaySpans(): ClipSpan[][] {
    const s = useEditor.getState();
    return [...new Set(overlayLayers(s.clips).map((c) => c.track))].map((track) =>
      getClipSpans(s.clips, s.assets, track)
    );
  }

  /**
   * Everything audible at `t`, with the gain the frame plan gives it.
   *
   * The same ramps that dim the picture dim the sound: a clip fading out of a
   * dissolve takes its audio with it, an upper-track clip's transition carries
   * its own, and a live voiceover ducks the rest. A cross dissolve is the
   * ramp on its own — the picture cuts and only these gains move. It and a
   * split edit are what make a clip audible outside its own footprint, so
   * both sides of a crossing really play at once.
   */
  private voicesAt(t: number, spans: ClipSpan[], master: ClipSpan | undefined): Voice[] {
    const s = useEditor.getState();
    const out: Voice[] = [];
    const duck = duckGainAt(s.audioClips, t);
    if (master && !master.clip.muted && !master.clip.hidden && !spanSilent(master)) {
      const plan = trackZeroPlan(master, spans, t);
      const v = voiceSpan(master);
      if (v)
        out.push({
          id: master.clip.id,
          ...v,
          gain: plan.gain * soundCrossGain(spans, master, t) * duck * (master.clip.volume ?? 1),
        });
    }
    // The other half of a crossing, and a split edit's lead or tail: a clip
    // playing into its handle, past its own footprint, carried by the cross
    // and split ramps alone.
    for (const spans0 of [spans, ...this.overlaySpans()]) {
      for (const { span, gain } of crossHandles(spans0, t)) {
        if (spanSilent(span)) continue;
        // The clip's own id, not a second voice: the handle is the same voice
        // carrying on past the cut, and a new id would stop and re-open it
        // there — a decode gap in the middle of the crossing.
        const v = voiceSpan(span);
        if (v) out.push({ id: span.clip.id, ...v, gain: gain * (span.clip.volume ?? 1) * duck });
      }
    }
    for (const { span, clip, gain } of this.liveOverlays(t)) {
      if (clip.muted || clip.hidden || spanSilent(span)) continue;
      const v = voiceSpan(span);
      if (v) out.push({ id: clip.id, ...v, gain: gain * (clip.volume ?? 1) * duck });
    }
    for (const a of s.audioClips) {
      const asset = s.assets.find((x) => x.id === a.assetId);
      if (!asset || a.hidden) continue;
      const rt = retimeOf(a);
      const len = Math.max(CLIP_MIN_SECONDS, rt.len);
      if (t < a.start || t >= a.start + len) continue;
      const v = audioVoice(a, asset, s.assets);
      if (!v || (v.url === asset.url && assetIsSilent(asset))) continue;
      // Fade envelope: linear ramps at either end of the clip.
      const rel = t - a.start;
      const fi = a.fadeIn ?? 0;
      const fo = a.fadeOut ?? 0;
      let g = 1;
      if (fi > 0 && rel < fi) g *= rel / fi;
      if (fo > 0 && rel > len - fo) g *= Math.max(0, (len - rel) / fo);
      // A ducking voiceover never ducks itself, or the others that duck.
      const dg = a.duck !== undefined && a.duck < 1 ? 1 : duck;
      out.push({ id: a.id, ...v, gain: a.volume * g * dg, sound: a.sound });
    }
    this.upcomingVoices(t, spans, duck, out);
    return out;
  }

  /**
   * The clips that begin within `SOUND_LOOKAHEAD_S` of `t`, added to `out` so
   * the mixer opens them before the playhead arrives. Each carries the gain it
   * will have on its first frame; the frames inside the clip take over from
   * there. A clip's sound opens from where it first plays, its handle or split
   * edit lead included. A clip already voiced — the far side of a crossing —
   * is left as it is.
   */
  private upcomingVoices(t: number, spans: ClipSpan[], duck: number, out: Voice[]) {
    const s = useEditor.getState();
    const have = new Set(out.map((v) => v.id));
    const soon = (start: number) => start > t && start <= t + SOUND_LOOKAHEAD_S;
    for (const spans0 of [spans, ...this.overlaySpans()]) {
      for (const span of spans0) {
        if (!soon(span.start - span.soundBack) || have.has(span.clip.id)) continue;
        if (span.clip.muted || span.clip.hidden || spanSilent(span)) continue;
        const v = voiceSpan(span);
        if (!v) continue;
        have.add(span.clip.id);
        out.push({ id: span.clip.id, ...v, gain: span.soundLead ? 0 : (span.clip.volume ?? 1) * duck });
      }
    }
    for (const a of s.audioClips) {
      if (!soon(a.start) || have.has(a.id) || a.hidden) continue;
      const asset = s.assets.find((x) => x.id === a.assetId);
      if (!asset) continue;
      const v = audioVoice(a, asset, s.assets);
      if (!v || (v.url === asset.url && assetIsSilent(asset))) continue;
      const dg = a.duck !== undefined && a.duck < 1 ? 1 : duck;
      out.push({ id: a.id, ...v, gain: (a.fadeIn ?? 0) > 0 ? 0 : a.volume * dg, sound: a.sound });
    }
  }

  /** Draw track 0 at `t`: the master clip, whatever is blending into it, the
   * neighbour frame behind an edge animation, and the clip's own veil. */
  private drawTrackZero(
    master: ClipSpan,
    spans: ClipSpan[],
    t: number,
    playing: boolean,
    masterFrame: Frame
  ) {
    const plan = trackZeroPlan(master, spans, t);
    // The incoming side of a live blend decodes alongside the outgoing one, so
    // a dissolve blends two real pictures — including between two trims of the
    // same file, which keep their own sources by construction.
    const incFrame = plan.incoming
      ? this.frameFor(plan.incoming, Math.max(plan.incoming.start, t), playing)
      : MISSING_FRAME;
    if (plan.backdrop) {
      // A neighbour's held frame behind a live edge animation, drawn at the
      // exact source moment the plan asks for.
      const b = plan.backdrop;
      const src = this.pool.get(keyOf(b.span.clip, b.span.asset), b.span.asset, this.decodeHeight());
      this.used.add(src);
      src.want(b.at, false);
      const f = src.frameAt(b.at, b.span.clip.in, b.span.clip.out);
      this.comp.drawLayer(
        f ? { kind: "ready", image: f.image, width: f.width, height: f.height, source: f.read?.recipe() } : PENDING_FRAME,
        b.span.clip,
        false,
        1,
        t
      );
    }
    this.comp.drawCrossJoin(
      plan.style,
      plan.p,
      {
        masterFrame,
        masterClip: master.clip,
        masterAlpha: plan.masterAlpha,
        masterZoom: plan.masterZoom,
        masterFx: {
          dx: plan.masterFxFrac.dx * this.canvas.width,
          dy: plan.masterFxFrac.dy * this.canvas.height,
          blur: plan.masterFxFrac.blur * Math.min(this.canvas.width, this.canvas.height),
        },
        incFrame,
        incClip: plan.incoming?.clip,
        incAlpha: plan.incAlpha,
        incZoom: plan.incZoom,
      },
      t
    );
    // Veil only the master clip's own footprint, like the export's per-clip
    // fade filter: a regioned clip darkens inside its rect while a track behind
    // shows through the margins.
    if (plan.veil > 0) this.comp.fillBlackVeil(plan.veil, rectOf(master.clip));
  }

  /** Draw the overlay tracks over track 0, further-back first. */
  private drawOverlays(t: number, playing: boolean) {
    for (const { span, clip, alpha, zoom } of this.liveOverlays(t)) {
      const frame = this.frameFor(span, t, playing);
      if (frame.kind !== "ready") continue;
      this.comp.drawIntoRect(frame, rectOf(clip), clipCovers(clip), alpha, t, zoom, clip);
    }
  }

  /** When the last tick ran, for spotting a starved loop in the dev log. */
  private lastTickAt = 0;

  private tick() {
    this.raf = 0;
    if (this.disposed) return;
    markTick();
    if (process.env.NODE_ENV !== "production") {
      const now = performance.now();
      if (this.lastTickAt && now - this.lastTickAt > 500) {
        engineLog(`engine ${this.serial} tick gap ${Math.round(now - this.lastTickAt)}ms`);
      }
      this.lastTickAt = now;
    }
    const playing = useEditor.getState().playing;
    if (playing !== this.wasPlaying) {
      this.wasPlaying = playing;
      this.trailingSince = 0;
      if (playing) {
        this.steppedThisPlay = false;
        this.playSince = performance.now();
      }
    }
    // Playing redraws every frame; paused, only what changed since the last
    // paint. Either way the loop stops as soon as there is nothing to do.
    if (playing || this.dirty) {
      this.dirty = false;
      const at = performance.now();
      this.phases.length = 0;
      this.phaseAt = at;
      this.render(playing);
      // A tick that held the thread past a few frames, with the phase that
      // held it. The playhead write is where the timeline and the inspector
      // re-render; the rest is the engine's own.
      const ms = performance.now() - at;
      if (process.env.NODE_ENV !== "production" && ms > 50)
        engineLog(`engine ${this.serial} tick ${Math.round(ms)}ms: ${this.phases.join(", ")}`);
    }
    if (playing || this.dirty) this.schedule();
  }

  private render(playing: boolean) {
    const s = useEditor.getState();
    const spans = getClipSpans(s.clips, s.assets);
    const total = projectDuration(s);
    this.phase("spans");
    // The frame's color is the project's, read fresh each frame: every clear
    // and every letterbox below paints it, so a cut with no footage at all
    // still plays a picture.
    this.comp.background = s.background;
    // An HDR project composites in HLG, the space its export composites in,
    // and the present pass decodes that for the display; without the
    // display or the canvas for it, the same grade renders as SDR.
    const hdr = this.hdrFrame(s.colorSpace);
    this.comp.output = hdr ? "hlg" : "sdr";
    setHdrPreviewState(s.colorSpace === "sdr" ? "off" : hdr ? "hdr" : "sdr");
    this.comp.removalBypass = s.removalPeek;
    this.renderPlaying = playing;
    this.pool.beginFrame();
    this.used.clear();

    // The clock, before anything that can return early.
    //
    // Whether the mixer is running *is* whether the cut is playing — there is no
    // second flag to fall out of step with it. Kept as one, a frame that
    // returned early (a clip still opening, an empty timeline) could leave the
    // engine believing playback had already started while the mixer had stopped,
    // and the next play would read a clock that was never anchored: the picture
    // would sit at zero for the length of the cut.
    //
    // The playhead can also move while the clock runs — an arrow-key skip, the
    // assistant's seek, a preview range starting mid-play. The engine writes
    // the playhead itself every playing frame, so a value it did not write is
    // someone seeking, and the clock re-anchors there rather than snapping the
    // playhead back.
    // Pinned just inside the end: a time at exactly `total` lies past the final
    // span, so a frame drawn there has no master clip and clears to the
    // background. Playing, that was the last frame before the stop, a single
    // dark frame the paused redraw painted over; paused, it is a skim off the
    // right edge of the cut. Both hold the last frame.
    let t: number;
    if (playing) {
      const head = playheadAt();
      if (!this.mixer.running || (this.written !== null && head !== this.written)) {
        this.mixer.start(Math.min(head, total));
        this.written = head;
      }
      t = stageTimeAt(this.mixer.now(), total);
    } else {
      t = stageTimeAt(previewAt(), total);
      if (this.mixer.running) {
        this.mixer.stop();
        flushMeter();
      }
    }

    // Nothing anywhere resets to a black frame at 0.
    if (
      spans.length === 0 &&
      overlayLayers(s.clips).length === 0 &&
      s.audioClips.length === 0 &&
      s.overlays.length === 0
    ) {
      this.mixer.stop();
      this.stalledAt = 0;
      if (s.buffering) useEditor.setState({ buffering: false });
      this.comp.drawLayer(MISSING_FRAME, undefined, true, 1, 0);
      noteStageDraw();
      if (s.playing) {
        useEditor.setState({ playing: false });
        this.writeHead(0);
      }
      return;
    }

    // Which way a paused reader is going, before anything reads a source on
    // its behalf. The direction is the last move's: a pointer at rest is
    // still heading the way it was, and the frames landed under it stay.
    if (playing) {
      this.readBackward = false;
      this.lastRead = null;
    } else {
      if (this.lastRead !== null && Math.abs(t - this.lastRead) > 1e-4)
        this.readBackward = t < this.lastRead;
      this.lastRead = t;
    }

    // The export's rule, so a cut a float sum lands a hair late flips on its frame here too.
    const master = trackZeroAt(spans, [], t)?.span;
    this.phase("clock");

    // Ask for the master's picture before clearing. A clip nothing has decoded
    // yet answers `pending`, and clearing on the strength of that would black
    // the frame out for as long as the file takes to open — the strobe this
    // whole design exists to remove. Leave what is on screen and stay dirty;
    // the clock, the audio, and the stop checks below still run, so a slow
    // open can't freeze the playhead or carry playback past a stop mark.
    const masterFrame = master ? this.frameFor(master, t, playing, true) : MISSING_FRAME;
    const pendingMaster = masterFrame.kind === "pending";
    this.phase("master");
    // A play whose master clip has decoded nothing has nothing to play: the
    // clock would carry the playhead across a stretch of the cut nobody saw or
    // heard, and land somewhere further on when the file finally opened. So it
    // stands still and says so, and the play carries on from where it stopped.
    // A moment of it is a decoder a frame or two behind rather than a stall,
    // and stopping the clock for that would stutter the picture and cut the
    // sound at every join, so a stall has to hold before it counts. A gap in
    // the timeline is empty on purpose and plays through.
    if (playing && master && pendingMaster) {
      this.stalledAt ||= performance.now();
      if (performance.now() - this.stalledAt >= STALL_HOLD_MS) {
        this.mixer.hold(t);
        this.writeHead(t);
        if (!s.buffering) useEditor.setState({ buffering: true });
        this.dirty = true;
        return;
      }
    } else {
      this.stalledAt = 0;
      if (s.buffering) useEditor.setState({ buffering: false });
    }
    if (pendingMaster) {
      this.dirty = true;
    } else {
      this.comp.clear();
      if (master) this.drawTrackZero(master, spans, t, playing, masterFrame);
      this.drawOverlays(t, playing);
      // The subject-mask pass reads the canvas as it stands beneath it, then
      // publishes the matte the DOM's front subject-masked elements read.
      this.comp.subjectMatteProvider = (at) => this.behind.clipMatteOf(this.canvas, at);
      this.behind.draw(this.canvas, s.overlays, s.assets, t);
      // The HDR picture: the HLG composite decoded onto the extended-range
      // canvas over the stage. One pass, on the frame just drawn.
      if (hdr) this.presenter!.present(this.canvas);
      // A paused draw is the picture standing still, which is when a
      // readback of it costs nothing anyone is watching; a play's first
      // frame is read back once the play stops and the frame is redrawn.
      if (!playing && masterFrame.kind === "ready") noteStagePicture();
      noteStageDraw();
    }
    this.phase("draw");

    // Decoders for what is about to arrive, and the ones nothing needs closed.
    this.warm(t, playing);
    this.pool.evict();
    this.phase("warm");

    if (playing) {
      // The audio effect elements over this moment, each at the level its
      // window puts it at; the mixer carries them over the whole mix.
      this.mixer.setEffects(
        liveAudioFxAt(s.overlays, t).map(({ span, wet }) => ({
          id: span.id,
          effect: span.effect,
          amount: span.amount,
          wet,
        }))
      );
      this.mixer.update(t, this.voicesAt(t, spans, master));
      this.phase("audio");
      // A scoped effect preview auto-pauses at its stop mark; the end of the
      // cut stops playback outright.
      const stop = s.previewStopAt;
      if (stop != null && t >= stop) {
        useEditor.setState({ playing: false, previewStopAt: null });
        this.writeHead(Math.min(t, stop));
      } else if (t >= total - 0.001) {
        useEditor.setState({ playing: false, previewStopAt: null });
        this.writeHead(total);
      } else {
        this.writeHead(t);
      }
      this.phase("head");
    } else {
      // The sound's readers on the same footing as the picture's: the clip
      // under a parked playhead has its file open before the play begins,
      // rather than opening it once the clock is already running.
      this.mixer.warm(t, this.voicesAt(t, spans, master));
      this.phase("audio");
    }

    if (tracing()) {
      let srcTs: number | null = null;
      let wantSrc: number | null = null;
      let exact = true;
      if (master) {
        const src = this.pool.get(
          keyOf(master.clip, master.asset),
          master.asset,
          this.decodeHeight()
        );
        const st = sourceTimeOf(master.clip, t);
        wantSrc = st;
        srcTs = src.frameAt(st, master.clip.in, master.clip.out)?.timestamp ?? null;
        exact = src.hasExact(st);
      }
      markPresent({
        t,
        srcTs,
        wantSrc,
        clipId: master?.clip.id ?? null,
        exact,
        stale: masterFrame.kind !== "ready",
      });
      markLiveSamples(this.pool.held);
      const clock = this.mixer.clockLead();
      markAudioClock(clock.lead, clock.reported);
      markLiveSources(this.pool.active, this.pool.size, this.pool.warmPixels);
    }

    if (playing && metering()) {
      const clock = this.mixer.clockLead();
      meterAudioClock(clock.lead, clock.reported);
      meterState({
        sources: this.pool.active,
        held: this.pool.held,
        warmMb: Math.round(this.pool.warmPixels * 4e-6),
        clips: s.clips.length,
        decodeHeight: this.decodeHeight(),
        canvasHeight: this.canvas.height,
      });
    }
  }
}

export function usePlayback(
  canvasRef: RefObject<HTMLCanvasElement | null>,
  presentRef?: RefObject<HTMLCanvasElement | null>
) {
  useEffect(() => {
    const canvas = canvasRef.current;
    if (!canvas) return;
    const engine = new Engine(canvas, presentRef?.current ?? null);
    // Dev-only automation hook, like installDevHooks: lets a headless run (or a
    // debugging session) reach the live engine.
    if (process.env.NODE_ENV !== "production") {
      (window as unknown as Record<string, unknown>).__cutDevEngine = engine;
    }
    registerSourceSampler((clipId) => engine.sourceFor(clipId));
    registerSourceColor((clipId) => {
      const clip = useEditor.getState().clips.find((c) => c.id === clipId);
      return clip ? engine.colorOf(clip) : null;
    });
    return () => {
      registerSourceSampler(null);
      registerSourceColor(null);
      engine.dispose();
    };
  }, [canvasRef, presentRef]);
}