Skip to content

Open source CapCut alternative.

TypeScript
49.7 KB
Raw Blame
"use client";

/**
 * The cut's sound while you watch it.
 *
 * Sound used to ride the same `<video>` and `<audio>` elements the picture came
 * from: the clip under the playhead was both the speaker and the clock, and
 * everything else chased it. That made the element's decode latency into the
 * project's timing, and it left no way to be sure the preview mixed a cut the
 * way the export would.
 *
 * Here every audible thing is a buffer scheduled on one `AudioContext`. The
 * context's own clock is what the preview runs on — it advances in real time,
 * never backward, and is sample-accurate — so the picture follows the sound
 * rather than the other way round.
 *
 * Every voice holds a reader walking its own file forward and hands the context
 * about a second of sound at a time. Reading in order is the part that matters:
 * a voice asking for a long span at once is asking for the whole interleaved
 * stretch of file that span is scattered through, which over a network is
 * minutes of bytes for seconds of sound and a voice permanently behind its own
 * playhead. Walking keeps the sound exactly as far into the file as the picture
 * is. A clip playing at a speed other than 1 is time-stretched rather than
 * resampled, which is what keeps its pitch and matches what the export writes;
 * a stretch needs its span in hand, so those voices read a short window instead
 * of walking.
 *
 * Gains are the only thing touched per frame, and only their value: the clip's
 * own volume, its fades, the duck under a live voiceover, the whole-project
 * fade, and how much of each audio effect is heard. The ramps themselves are
 * the frame plan's, so the mix follows the same description the picture does.
 *
 * The audio effect elements sit between the voices and that final fade, each
 * treating what the one before it handed on — the order the export's filter
 * chain runs them in.
 */

import {
  audioFxRecipe,
  buildAudioFx,
  soundRecipe,
  type AudioFxNodes,
  type ClipSound,
  type Retime,
} from "@donkeycut/effects-kit";
import type { WrappedAudioBuffer } from "mediabunny";
import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";
import { allowance, holdMemory } from "./memoryBudget";
import { meterAudioLate, meterClockJump } from "./perfTrace";
import { fitSpan, retimeFits } from "./retimeFit";

/** How much of a voice is pulled off its walk and scheduled at once, in source
 * seconds. Long enough that a stretch of sound is one scheduling decision;
 * short enough that the bytes it waits on are the ones the picture is reading
 * anyway. */
const GROUP_S = 1;
/** The first window scheduled from a standing start — a play begun, a seek
 * landed, a hold released. The picture opens on a frame already decoded, and a
 * sound that waits for a whole group joins it a group-read late; a short first
 * window puts the sound beside the first frame, and the fills behind it extend
 * the lead by whole groups. A link that cannot feed the second window in time
 * starts it where the clock stands, the way any late window starts — a beat of
 * sound and a catch-up beats a silence the length of the group. */
const FIRST_GROUP_S = 0.3;
/** Most reads one group is assembled from. A packet of compressed audio holds
 * a fraction of a second, so a group is tens of reads, and this is the ceiling
 * over them. */
const GROUP_PULLS = 128;
/** A stretched window is resynthesised on the main thread, so it is kept short
 * enough that the work lands inside a frame or two. */
const WINDOW_STRETCHED_S = 4;
/**
 * Decoded windows kept in hand: stretched windows, and short clips read whole.
 *
 * Two dozen four-second windows of stereo at 48kHz — enough that stepping back
 * over a cut replays what it just played, and bounded because everything older
 * than that is a full decode away regardless.
 */
const WINDOW_CACHE_BYTES = 24 * WINDOW_STRETCHED_S * 48_000 * 2 * 4;
/** A clip whose whole source span is at most this long is one read: decoded
 * in full the first time it is heard and held, so the sound effects a cut is
 * dotted with — a click, a whoosh, a hit, each shorter than the open a walk
 * over its file costs — play whole on every pass, and a second pass over the
 * same one costs no read at all. */
const SHORT_CLIP_S = 3;

/** What a set of decoded windows costs: float samples, one per channel. */
/** A window laid through a clip's map, with the source second its first
 * sample really sits at — see `FittedSpan.head`. */
interface FittedWindow {
  buffer: AudioBuffer;
  /** Seconds past where the window was asked for that its sound begins —
   * see `FittedSpan.shift`. */
  shift: number;
  /** Seconds into the buffer the window begins: a whole short clip held from
   * its head, joined part-way through. */
  skip?: number;
}

/** A voice read whole — see `SHORT_CLIP_S`. */
const isShort = (live: { in: number; out: number }): boolean => live.out - live.in <= SHORT_CLIP_S;

const decodedBytes = (held: Map<string, FittedWindow>): number => {
  let n = 0;
  for (const { buffer: b } of held.values()) n += b.length * b.numberOfChannels * 4;
  return n;
};
/** Schedule more of a voice once the playhead is this close to running out.
 * The scheduling runs on the main thread, and what is scheduled is all the
 * sound there is while that thread is blocked — a delete that takes seconds
 * to land, an import decoding — so the lead is longer than the freezes a slow
 * machine has been measured to take. */
const AHEAD_S = 5;
/** Most the picture will be held back for the output device, whatever the
 * device claims. Well past a Bluetooth headset, and short of anything that
 * would read as the playhead refusing to move. */
const MAX_LATENCY_S = 0.4;
/** How long a play follows the sound's clock alone before the wall may carry
 * it. A new context's device opens after the context does: Chrome on a Mac
 * reports `running` at once, holds `currentTime` at zero for a quarter second
 * and reports no output latency until the device is open. The wall running on
 * through that read as a clock jump, put the playhead ahead of a sound that
 * had not started, and stepped it back once the latency arrived. */
const START_WAIT_S = 1;
/** Sound late by more than this, mid-play, is a hole the listener hears: the
 * scheduling runs seconds ahead, so a window that misses its moment means the
 * thread was gone for that long. A voice's first window is a different thing
 * — it is read while the play is already running, by design — so only a long
 * one counts there. */
const LATE_GAP_S = 0.05;
const LATE_FIRST_S = 0.5;

/** One audible thing on the timeline: a clip's own sound, or a soundtrack item. */
export interface Voice {
  /** Stable across frames — the clip's id. A voice whose id survives keeps
   * playing; one that vanishes is stopped. */
  id: string;
  url: string;
  /** Where it starts on the timeline, and the source span it plays. */
  start: number;
  in: number;
  out: number;
  /** The map from this voice's timeline span onto its source span. */
  retime: Retime;
  /** Everything the frame plan says about how loud it is right now. */
  gain: number;
  /** The clip's own treatment, run on this voice alone before it joins the
   * bus. */
  sound?: ClipSound;
}

interface Scheduled {
  node: AudioBufferSourceNode;
  /** Timeline time this window ends at, so the next one starts there. */
  until: number;
}

interface LiveVoice {
  url: string;
  retime: Retime;
  in: number;
  out: number;
  start: number;
  /** Where this voice's scheduled windows land. The clip's own treatment
   * runs between here and the gain, so it hears the clip at its recorded
   * level — the same place the fold and the ffmpeg stanza run it, ahead of
   * the level, fades and duck the gain carries. */
  head: GainNode;
  gain: GainNode;
  /** The treatment's chain, and what it was built from, so a frame that
   * changes nothing about it touches no nodes. Rebuilt in place when the
   * treatment changes: the windows already scheduled play on through the
   * new chain, since they are connected to the head rather than to it. */
  fx: AudioFxNodes | null;
  soundKey: string;
  /** Windows already handed to the context, oldest first. */
  windows: Scheduled[];
  /** Timeline time everything scheduled so far runs out at. */
  scheduled: number;
  /** The reader walking this voice's source, open from where it last read to.
   * A voice that moves inside the same file re-aims it; a read that failed, or
   * a source that moved to another address, drops it. */
  walk: AudioWalk | null;
  /** A read in flight, so a slow one doesn't start a second one, and when it
   * started, so one that has stopped answering can be disowned. */
  filling: boolean;
  fillAt: number;
  /** Bumped by everything that moves the voice. A read carries the number it
   * started under and acts on what it got only while that still stands. */
  gen: number;
  /** Set when the voice has read to the end of its source. A position rather
   * than a verdict: anything that moves the voice clears it, so a seek back
   * into the clip plays again. */
  ended: boolean;
  /** The furthest point in the source a walk has ended at. A second walk that
   * gets no further has found where the track really stops. */
  reached: number;
  /** Consecutive failed reads, and the wall time (performance.now) the next
   * one may be tried at. */
  attempts: number;
  retryAt: number;
}

/** A read that has not answered in this long has stopped being a read — a
 * request left hanging by a link that went away. The voice disowns it and asks
 * again; waiting it out would cost the rest of the play. */
const FILL_STALL_MS = 15_000;
/** A failed read is asked again quickly a few times, then on a slow cadence for
 * as long as the failure holds. Nothing here is ever terminal, and that is the
 * point: a source that cannot be read this second — bytes still landing, a
 * token a moment past its window, a link that dropped — is nearly always
 * readable shortly, the picture's own readers wait it out exactly this way
 * (frameSource.ts), and a voice that stopped asking was silent for the life of
 * the page while the picture beside it healed. */
const READ_RETRY_MS = 1_000;
const READ_RETRIES = 3;
const READ_RECHECK_MS = 10_000;
/** How close to the clip's end a walk may stop and be believed. A track
 * routinely runs a fraction short of the picture it came with. */
const END_SLACK_S = 0.25;

/** Whether `source` sits at or past the end of the span the voice plays —
 * `out` on a voice reading forward, `in` on one reading backward — within
 * `slack` seconds. */
const atSpanEnd = (live: { retime: Retime; in: number; out: number }, source: number, slack = 1e-3) =>
  live.retime.reverse ? source <= live.in + slack : source >= live.out - slack;

/** A source second as progress through the voice: it rises as the voice
 * plays, whichever way the voice reads. */
const progress = (live: { retime: Retime }, source: number) =>
  live.retime.reverse ? -source : source;

/** One audio effect element the graph is carrying: its recipe's chain, and
 * the pair of gains that cross the mix between treated and untreated. */
interface LiveFx {
  nodes: AudioFxNodes;
  wet: GainNode;
  dry: GainNode;
  /** Where the treated and untreated sides meet — the next stage's input. */
  out: GainNode;
}

/** What the mixer is told about one effect element on the timeline: which
 * effect at what strength, and how much of it is heard right now. */
export interface FxVoice {
  id: string;
  effect: string;
  amount?: number;
  /** 0 = the untreated mix, 1 = the treated one. */
  wet: number;
}

/**
 * The preview's clock and mixer.
 *
 * One of these lives for as long as the editor is open. It holds the context,
 * the anchor that maps timeline seconds onto it, and one gain-fed voice per
 * audible clip.
 */
export class PreviewMixer {
  private ctx: AudioContext | null = null;
  private master: GainNode | null = null;
  /** Where every voice lands, and what the effect chains treat. */
  private bus: GainNode | null = null;
  private fx = new Map<string, LiveFx>();
  /** What the built chains stand for, so a frame that changes nothing about
   * them touches gains alone. */
  private fxKey = "";
  private voices = new Map<string, LiveVoice>();
  /** Whether the clock is being kept still by `hold`, so entering a hold
   * silences the mix once rather than on every frame of it. */
  private holding = false;
  /** Where the clock is being kept while `holding`. */
  private heldAt = 0;
  /** Timeline time, context time and wall time of the same instant. Everything
   * scheduled is placed against this. */
  private anchor: { timeline: number; ctx: number; wall: number } | null = null;
  /** Wall time (performance.now) the play began, for `START_WAIT_S`. */
  private startedAt = 0;
  /** The latest time `now()` has answered this play, so it never answers
   * less: a device reporting more latency mid-play holds the picture until
   * the sound reaches it. */
  private shown = 0;
  private decoded = new Map<string, FittedWindow>();
  private readonly releaseMemory = holdMemory("mixerAudio", () => decodedBytes(this.decoded));

  /** Whether the clock is running. */
  get running(): boolean {
    return this.anchor !== null;
  }

  private context(): AudioContext {
    if (!this.ctx) {
      this.ctx = new AudioContext();
      this.master = this.ctx.createGain();
      this.master.connect(this.ctx.destination);
      // Voices meet on the bus and the effect chains treat what is on it —
      // the order the export's ffmpeg graph has.
      this.bus = this.ctx.createGain();
      this.bus.connect(this.master);
    }
    return this.ctx;
  }

  /**
   * How far behind the graph's clock the sound is actually heard.
   *
   * `ctx.currentTime` is the moment the graph has *rendered* to, and a rendered
   * sample has not left the machine yet: it is sitting in the output buffer the
   * device will read next. `baseLatency` is what the graph itself holds and
   * `outputLatency` is what the device does, and their sum is the gap between
   * a sample being written and a person hearing it.
   *
   * The size of that gap is the platform's to decide, and platforms disagree by
   * an order of magnitude. CoreAudio hands back something in the tens of
   * milliseconds. WASAPI in shared mode — every Windows browser, by default —
   * routinely holds several times that, and a Bluetooth output holds more
   * again. A picture drawn against the unadjusted clock therefore runs ahead of
   * its own sound by an amount that is invisible where this is developed and
   * plainly wrong on the machines most people edit on.
   *
   * Capped, because the number comes from a driver: a device reporting
   * something absurd would otherwise hold the picture at the play position for
   * as long as it claimed.
   */
  private latency(): number {
    const ctx = this.ctx;
    if (!ctx) return 0;
    const reported = (ctx.baseLatency || 0) + (ctx.outputLatency || 0);
    return Math.min(MAX_LATENCY_S, Math.max(0, reported));
  }

  /** The moment leaving the speakers, given the moment the graph has rendered
   * to. Held at the play position over the first buffer: the sound starts at
   * once and the picture waits out the device rather than stepping backwards
   * into what has already been heard. */
  private heard(graph: number): number {
    return Math.max(this.anchor?.timeline ?? 0, graph - this.latency());
  }

  /**
   * Where the playhead is, from the sound's own clock.
   *
   * This is the whole point of the arrangement: the number the picture is drawn
   * against comes from the same clock the samples leave on, so they cannot
   * drift apart no matter what the main thread is doing. Leaving is the word —
   * what the graph has rendered is a buffer ahead of what is audible, and the
   * picture is drawn against the audible one.
   */
  now(): number {
    if (!this.anchor) return 0;
    // A hold is the clock standing still on purpose; nothing about it is drift.
    if (this.holding) return this.heldAt;
    if (this.starting()) return this.show(this.heard(this.followSound()));
    const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;
    // A browser holds an `AudioContext` suspended until the page has been
    // interacted with, and a suspended context's clock does not move. Reading
    // it blindly would freeze the picture on a project played before the user
    // has touched anything, and picking it up once it finally starts would jump
    // the playhead by however long it stayed asleep. So the wall carries the cut
    // until the audio clock agrees with it, and a clock that disagrees is
    // re-anchored rather than believed.
    if (!this.ctx || this.ctx.state !== "running") return this.show(wall);
    const graph = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);
    // Measured on the graph's clock, which is the one the anchor was taken on.
    // Comparing the heard clock against the wall would read the device's own
    // buffer as drift and tear down every scheduled window to correct for it,
    // which on the machines with the largest buffers is the loudest thing this
    // class could do.
    if (Math.abs(graph - wall) > 0.25) {
      this.anchor.ctx = this.ctx.currentTime - (wall - this.anchor.timeline);
      // Everything already scheduled was placed against the old anchor and is
      // now wrong by the size of the jump — a context that sat suspended until
      // the user's next gesture, an output device switch. The clock alone
      // moving would leave those windows playing the wrong moment for up to
      // their full length, so they stop here and the next update refills from
      // where the clock now stands.
      meterClockJump();
      this.restart(wall);
      return this.show(this.heard(wall));
    }
    return this.show(this.heard(graph));
  }

  /** Whether the play is inside its `START_WAIT_S`. */
  private starting(): boolean {
    return performance.now() - this.startedAt < START_WAIT_S * 1000;
  }

  /**
   * The graph's clock, with the wall kept level with it.
   *
   * While the device opens the graph's clock stands still and so does the
   * playhead, at the play position. The wall is pulled along so that once the
   * wait is over it carries on from the same moment: a context that never
   * starts — a page nobody has touched — plays on in silence from there.
   */
  private followSound(): number {
    const anchor = this.anchor!;
    const graph = anchor.timeline + (this.ctx!.currentTime - anchor.ctx);
    anchor.wall = performance.now() - (graph - anchor.timeline) * 1000;
    return graph;
  }

  /** `t`, or the latest time already answered if that is later. */
  private show(t: number): number {
    this.shown = Math.max(this.shown, t);
    return this.shown;
  }

  /**
   * Where the playhead is, with the clock allowed to re-aim nothing.
   *
   * A read asks with this. Asking with `now()` lets a read that took a moment
   * trip the re-anchor, and re-anchoring disowns every voice — including the
   * one this read was for, whose sound is then thrown away by the act of
   * finding out where to put it.
   */
  private at(): number {
    if (!this.anchor) return 0;
    if (this.starting()) return Math.max(this.shown, this.heard(this.followSound()));
    const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;
    if (!this.ctx || this.ctx.state !== "running") return Math.max(this.shown, wall);
    const graph = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);
    return Math.max(this.shown, this.heard(Math.abs(graph - wall) > 0.25 ? wall : graph));
  }

  /**
   * How far the picture is being held behind the graph's clock right now, and
   * what the output device says it holds. Zero when nothing is playing.
   *
   * These are two independent readings of the same thing — one taken from what
   * the clock actually returns, one from the device — and the perf eval holds
   * them against each other. A preview that went back to drawing against the
   * rendered clock reports a lead of zero on a device claiming a fifth of a
   * second, which is the picture running that far ahead of its own sound.
   */
  clockLead(): { lead: number; reported: number } {
    const ctx = this.ctx;
    const reported = ctx ? (ctx.baseLatency || 0) + (ctx.outputLatency || 0) : 0;
    if (!this.anchor || !ctx || ctx.state !== "running") return { lead: 0, reported };
    const graph = this.anchor.timeline + (ctx.currentTime - this.anchor.ctx);
    return { lead: Math.max(0, graph - this.heard(graph)), reported };
  }

  /** Stop every scheduled window and re-aim each voice at `timeline`, keeping
   * the file each one is reading: a clock that jumped moved the voices inside
   * the same sources. */
  private restart(timeline: number): void {
    for (const live of this.voices.values()) {
      this.stopWindows(live);
      this.reaim(live, Math.max(live.start, timeline));
      // The voice is somewhere else in its file now, so where a walk stopped
      // at the old position says nothing about this one.
      live.reached = -Infinity;
    }
  }

  /** Stop and drop every window a voice has handed to the context. */
  private stopWindows(live: LiveVoice): void {
    for (const w of live.windows) {
      try {
        w.node.stop();
      } catch {
        // Already finished; stopping a spent node throws and means nothing.
      }
      w.node.disconnect();
    }
    live.windows = [];
  }

  /**
   * The voice is no longer reading where it was.
   *
   * This is the one that gives the file up, for the cases where the file is
   * the problem or is no longer the right one: a read that failed or stopped
   * answering, a voice released, a source that moved to another address. A
   * voice that has merely fallen behind keeps its reader and is re-aimed. The
   * read still in flight is disowned in the same breath: it carries the old
   * number and everything it comes back with is ignored.
   */
  private moved(live: LiveVoice): void {
    live.gen++;
    live.filling = false;
    live.ended = false;
    live.walk?.close();
    live.walk = null;
  }

  /**
   * Begin playing at timeline time `t`.
   *
   * The anchor is this very instant: the next `now()` reads `t` and counts
   * forward. Anchoring even slightly in the future reads *behind* `t` first,
   * which walks the playhead backwards on every play and, at the head of the
   * timeline, puts the clock before the first clip. The first audio window
   * pays for its own decode instead — `fill` starts a late window at the
   * current time with the missed sliver skipped, so the sound joins in sync a
   * few milliseconds in.
   */
  start(t: number): void {
    const ctx = this.context();
    void ctx.resume();
    this.restart(t);
    this.anchor = { timeline: t, ctx: ctx.currentTime, wall: performance.now() };
    this.startedAt = this.anchor.wall;
    this.shown = t;
  }

  /**
   * Keep the clock at `t` while the cut has nothing to show.
   *
   * The context's clock runs whatever happens to it, so the anchor moves under
   * it: the timeline stands still while real time goes on, and the play picks
   * up where it stopped. Sound stops with it — a mix playing on over a picture
   * that is not moving is the sound of a cut coming apart, and what is already
   * scheduled would land at the wrong moment when the picture came back.
   */
  hold(t: number): void {
    if (!this.ctx || !this.anchor) return;
    if (!this.holding) {
      this.holding = true;
      this.heldAt = t;
      for (const live of this.voices.values()) {
        this.stopWindows(live);
        const from = Math.max(live.start, t);
        this.reaim(live, from);
        // The held moment's sound comes down while the picture opens: one
        // read aims the walk and pulls the bytes it sits in — the primer a
        // parked playhead uses — so the release finds an open file already
        // read where the first window starts.
        const source = live.retime.srcAt(from - live.start);
        if (!live.filling && !atSpanEnd(live, source)) void this.prime(live, source);
      }
    }
    // Both clocks are pulled back to where the hold began, every frame of it:
    // the context's runs on, and the wall's is what carries a cut whose context
    // is not running.
    this.anchor.ctx = this.ctx.currentTime - (this.heldAt - this.anchor.timeline);
    this.anchor.wall = performance.now() - (this.heldAt - this.anchor.timeline) * 1000;
  }

  /** Stop the clock and silence everything, keeping each voice's reader: the
   * playhead is parked over the same clips it was playing, and the next play
   * is usually from where this one stopped. */
  stop(): void {
    this.anchor = null;
    this.holding = false;
    for (const live of this.voices.values()) this.stopWindows(live);
  }

  /**
   * Carry these audio effects over the bus, in the order given.
   *
   * The chains run in series — the second effect treats what the first one
   * handed on, the way the export's filter chain does — and each one is
   * crossed into the mix by its own `wet`. The graph is rebuilt only when the
   * list itself changes; a caller that passes the effects near the playhead
   * therefore rebuilds while the entering or leaving chain is silent, and
   * every frame in between only moves gains.
   */
  setEffects(fx: FxVoice[]): void {
    const ctx = this.ctx;
    const bus = this.bus;
    const master = this.master;
    if (!ctx || !bus || !master) return;
    const key = fx.map((f) => `${f.id}:${f.effect}:${f.amount ?? ""}`).join("|");
    if (key !== this.fxKey) {
      this.fxKey = key;
      this.teardownFx();
      // The bus's old outlet goes first, so the stages built below are the
      // only path from it to the fade.
      bus.disconnect();
      let node: AudioNode = bus;
      for (const f of fx) {
        const recipe = audioFxRecipe(f.effect, f.amount);
        if (!recipe) continue;
        const nodes = buildAudioFx(ctx, recipe, ctx.currentTime);
        // One stage: what came in, and the treated copy of it, meeting at a
        // sum that the next stage reads.
        const wet = ctx.createGain();
        const dry = ctx.createGain();
        const out = ctx.createGain();
        wet.gain.value = 0;
        dry.gain.value = 1;
        node.connect(nodes.input);
        nodes.output.connect(wet);
        wet.connect(out);
        node.connect(dry);
        dry.connect(out);
        node = out;
        this.fx.set(f.id, { nodes, wet, dry, out });
      }
      node.connect(master);
    }
    for (const f of fx) {
      const live = this.fx.get(f.id);
      if (!live) continue;
      const w = Math.max(0, Math.min(1, f.wet));
      live.wet.gain.value = w;
      live.dry.gain.value = 1 - w;
    }
  }

  /** Drop every effect chain, stopping the modulators they started. */
  private teardownFx(): void {
    for (const live of this.fx.values()) {
      for (const src of live.nodes.sources) {
        try {
          src.stop();
        } catch {
          // Already stopped; a spent node throws and means nothing.
        }
      }
      live.nodes.input.disconnect();
      live.nodes.output.disconnect();
      live.wet.disconnect();
      live.dry.disconnect();
      live.out.disconnect();
    }
    this.fx.clear();
  }

  /**
   * Bring the mix in line with what is audible at timeline time `t`.
   *
   * Called every frame while playing. Voices that just became live are opened
   * and scheduled; voices that ran out are dropped; the rest have their gain
   * set to whatever the frame plan worked out. Nothing here awaits, so a slow
   * decode delays that one voice and nothing else.
   */
  update(t: number, voices: Voice[]): void {
    if (!this.anchor || !this.ctx) return;
    this.holding = false;
    // A context that stops running stops the sound and leaves the picture
    // playing on wall time — an output device taken away, a page the browser
    // put to sleep. Nothing else asks it back, since `start` is the only other
    // resume, so a play that lost its context stayed silent to the end of the
    // cut.
    if (this.ctx.state !== "running") void this.ctx.resume().catch(() => {});
    const seen = new Set<string>();
    for (const v of voices) {
      seen.add(v.id);
      // An edit to a playing clip arrives as the same id with new geometry — a
      // trim, a move, a speed change. Gain is the only part of a voice that
      // tunes live; everything else is baked into what was already scheduled,
      // so the voice reopens and the sound follows the edit the way the
      // picture does.
      const prev = this.voices.get(v.id);
      if (
        prev &&
        (prev.start !== v.start ||
          prev.in !== v.in ||
          prev.out !== v.out ||
          prev.retime.key !== v.retime.key)
      ) {
        this.release(prev);
        this.voices.delete(v.id);
      } else if (prev && prev.url !== v.url) {
        // The same sound at a new address: a signed link re-minted, an import
        // that landed in project storage and moved off the URL it arrived on.
        // The bytes already scheduled are the same sound, so they play on and
        // only the next read moves — dropping them would take the sound out
        // for as long as a cold open of the new address takes, which on a
        // long file is most of a minute and sounds like the audio quitting.
        this.forget(prev.url);
        prev.url = v.url;
        this.moved(prev);
        // A new address answers for itself, so nothing the old one did holds
        // this voice back from asking straight away.
        prev.attempts = 0;
        prev.retryAt = 0;
        prev.reached = -Infinity;
      }
      const live = this.voices.get(v.id) ?? this.open(v, t);
      if (!live) continue;
      live.gain.gain.value = Math.max(0, Math.min(3, v.gain));
      this.treat(live, v.sound);
      // A read that hangs holds the voice as surely as one that fails, and it
      // holds it quietly: nothing else would ever notice, and the voice would
      // wait on it for the rest of the play.
      if (live.filling && performance.now() - live.fillAt > FILL_STALL_MS) this.miss(live);
      if (
        !live.ended &&
        !live.filling &&
        live.scheduled < t + AHEAD_S &&
        performance.now() >= live.retryAt
      )
        void this.fill(v.id, live);
      // Windows that have finished playing hold their buffers alive.
      live.windows = live.windows.filter((w) => w.until > t - 1);
    }
    for (const [id, live] of this.voices) {
      if (seen.has(id)) continue;
      this.release(live);
      this.voices.delete(id);
    }
  }

  /** A window that missed its moment. Reported to the diagnostics meter once
   * it is past what the first read of a voice costs by design. */
  private noteLate(live: LiveVoice, lateS: number): void {
    if (lateS > (live.windows.length ? LATE_GAP_S : LATE_FIRST_S)) meterAudioLate(lateS);
  }

  /**
   * Get the sound ready for a play that has not started.
   *
   * Opening a reader is most of a second on a long file and several on a slow
   * link — a container parsed, a decoder configured, a seek into the middle of
   * it — and the sound used to do all of that after the play began. The picture
   * never did: the clip under a parked playhead has had a decoder open on it
   * all along, which is why the picture starts on the frame and the sound came
   * in seconds later, worst right after a seek.
   *
   * The playhead standing over a clip is all the notice the sound needs. Each
   * audible voice opens its file and takes one read at the moment it is parked
   * on, so the play finds an open file aimed where it is about to read. Nothing
   * is scheduled and nothing is heard; the context is left alone until a play
   * asks for it, which is also why this waits for one — a page that has not
   * played yet has no context to warm into.
   */
  warm(t: number, voices: Voice[]): void {
    if (this.running || !this.ctx) return;
    const seen = new Set<string>();
    for (const v of voices) {
      seen.add(v.id);
      const prev = this.voices.get(v.id);
      if (
        prev &&
        (prev.url !== v.url ||
          prev.start !== v.start ||
          prev.in !== v.in ||
          prev.out !== v.out ||
          prev.retime.key !== v.retime.key)
      ) {
        this.release(prev);
        this.voices.delete(v.id);
      }
      const live = this.voices.get(v.id) ?? this.open(v, t);
      if (!live) continue;
      live.gain.gain.value = 0;
      this.treat(live, v.sound);
      const from = Math.max(v.start, t);
      if (live.scheduled !== from) this.reaim(live, from);
      const source = live.retime.srcAt(from - live.start);
      // Already reading, at the end of its source, or standing close enough
      // that the play's first read is the one it was going to make anyway.
      if (live.filling || atSpanEnd(live, source)) continue;
      if (live.walk && live.walk.position >= source && live.walk.position - source <= 1) continue;
      void this.prime(live, source);
    }
    for (const [id, live] of this.voices) {
      if (seen.has(id)) continue;
      this.release(live);
      this.voices.delete(id);
    }
  }

  /** One read taken to open the file and put the decoder on the moment the
   * playhead is parked at. What it reads is thrown away — the walk standing
   * there is the whole point — and a failure is left for the play to meet, so
   * a parked playhead never reports anything about a link. */
  private async prime(live: LiveVoice, source: number): Promise<void> {
    const gen = live.gen;
    live.filling = true;
    live.fillAt = performance.now();
    try {
      // A voice laid through its map has no walk: its first window is the
      // primer, decoded now and held for the play that follows.
      if (retimeFits(live.retime)) {
        await this.stretchedWindow(live, source);
      } else if (isShort(live)) {
        await this.shortWindow(live, source);
      } else {
        const walk = (live.walk ??= openAudioWalk(live.url, source, live.out));
        if (Math.abs(walk.position - source) > 0.02) walk.seek(source);
        await walk.next();
      }
      // The read answering is the source answering: a voice backed off by an
      // outage is willing again the moment its file reads, so the play that
      // follows is never left waiting out a cadence the outage booked.
      if (live.gen === gen) {
        live.attempts = 0;
        live.retryAt = 0;
      }
    } catch {
      if (live.gen === gen) this.moved(live);
    } finally {
      if (live.gen === gen) live.filling = false;
    }
  }

  /** Let go of every decoded window. The next play decodes them again. */
  releaseCaches(): void {
    this.decoded.clear();
  }

  /** Drop the decode cache for a source that has been replaced. */
  private forget(url: string): void {
    for (const key of [...this.decoded.keys()]) {
      if (key.startsWith(`${url}|`)) this.decoded.delete(key);
    }
  }

  dispose(): void {
    this.stop();
    this.stopVoices();
    this.teardownFx();
    this.fxKey = "";
    this.decoded.clear();
    this.releaseMemory();
    void this.ctx?.close();
    this.ctx = null;
    this.master = null;
    this.bus = null;
  }

  private open(v: Voice, t: number): LiveVoice | null {
    const ctx = this.ctx;
    if (!ctx || !this.bus) return null;
    const gain = ctx.createGain();
    gain.gain.value = 0;
    gain.connect(this.bus);
    const head = ctx.createGain();
    head.connect(gain);
    const live: LiveVoice = {
      url: v.url,
      retime: v.retime,
      in: v.in,
      out: v.out,
      start: v.start,
      head,
      gain,
      fx: null,
      soundKey: "",
      windows: [],
      // Scheduling begins where the playhead is: a voice opened mid-clip —
      // playback started inside it, or the clip edited while audible — decodes
      // from the moment it is heard, not from a head nobody will hear.
      scheduled: Math.max(v.start, t),
      walk: null,
      filling: false,
      fillAt: 0,
      gen: 0,
      ended: false,
      reached: -Infinity,
      attempts: 0,
      retryAt: 0,
    };
    this.voices.set(v.id, live);
    return live;
  }

  private release(live: LiveVoice): void {
    this.stopWindows(live);
    this.moved(live);
    live.head.disconnect();
    live.gain.disconnect();
    if (live.fx) {
      live.fx.input.disconnect();
      live.fx.output.disconnect();
      live.fx = null;
    }
  }

  /**
   * Put the clip's sound treatment between the voice's gain and the bus, or
   * take it out when the clip has none. Keyed on the treatment itself, so
   * every frame that leaves it alone costs a string compare and nothing
   * else, and a slider drag rebuilds a chain a few nodes long.
   */
  private treat(live: LiveVoice, sound: ClipSound | undefined): void {
    const ctx = this.ctx;
    const bus = this.bus;
    if (!ctx || !bus) return;
    const key = sound ? JSON.stringify(sound) : "";
    if (key === live.soundKey) return;
    live.soundKey = key;
    live.head.disconnect();
    if (live.fx) {
      live.fx.input.disconnect();
      live.fx.output.disconnect();
      live.fx = null;
    }
    const recipe = soundRecipe(sound);
    if (!recipe) {
      live.head.connect(live.gain);
      return;
    }
    const fx = buildAudioFx(ctx, recipe, ctx.currentTime);
    live.head.connect(fx.input);
    fx.output.connect(live.gain);
    live.fx = fx;
  }

  private stopVoices(): void {
    for (const live of this.voices.values()) this.release(live);
    this.voices.clear();
  }

  /**
   * Carry one voice a little further ahead of the playhead.
   *
   * The voice's walk hands back the sound in the order it plays, and a group of
   * it is scheduled as one, placed by the source timestamps the buffers carry.
   * A stretch of file with no audio in it therefore moves the sound to where it
   * belongs and everything after it keeps its own time.
   *
   * Nothing here is terminal. A read that fails or stops answering leaves the
   * voice waiting a moment and asking again, for as long as the play lasts,
   * because every reason a source stops answering mid-play is a reason it
   * starts again.
   */
  private async fill(id: string, live: LiveVoice): Promise<void> {
    if (!this.ctx || !this.anchor) return;
    const gen = ++live.gen;
    live.filling = true;
    live.fillAt = performance.now();
    // The voice can be moved out from under this read while it runs — a seek,
    // a clock jump, this very read disowned as stalled — and only the read that
    // still stands may act on what it got.
    const current = () => this.voices.get(id) === live && live.gen === gen;
    try {
      const from = live.scheduled;
      const sourceFrom = live.retime.srcAt(from - live.start);
      if (atSpanEnd(live, sourceFrom)) {
        live.ended = true;
        return;
      }
      let buffer: AudioBuffer | null;
      // How far past `from` this window's first sample belongs. A window laid
      // through the map can start short of the second it was asked for, when
      // the read ran into the end of the source; played at `from` it would
      // run early for its whole length.
      let shift = 0;
      let skip = 0;
      try {
        // Nothing scheduled means the sound is standing still — the play just
        // began, a seek landed, a hold let go — and the picture is not
        // waiting. The first window is short so the sound joins it now.
        const group = live.windows.length ? GROUP_S : FIRST_GROUP_S;
        if (retimeFits(live.retime)) {
          const fit = await this.stretchedWindow(live, sourceFrom);
          buffer = fit?.buffer ?? null;
          shift = Math.max(0, fit?.shift ?? 0);
        } else if (isShort(live)) {
          const fit = await this.shortWindow(live, sourceFrom);
          buffer = fit?.buffer ?? null;
          skip = fit?.skip ?? 0;
        } else {
          buffer = await this.walkGroup(live, sourceFrom, current, group);
        }
      } catch {
        // The read is the only part of this that says anything about the
        // source, so it is the only part a failure is spent on.
        if (current()) this.miss(live);
        return;
      }
      if (!current()) return;
      if (!buffer) {
        // A walk reports the same ending whether it read the track's last
        // sample or gave up on it: bytes that never came, a decoder closed
        // under it, a link that dropped mid-file. Believed, an ending a few
        // seconds into a long clip plays the rest of the cut in silence while
        // the picture reads happily past it — so an ending short of the clip
        // is a failed read, and another walk goes after the sound it did not
        // reach. Two things stop that running away: a walk that gets no
        // further than the one before it has found where the track really
        // stops, and a track that runs a fraction short of its picture is
        // inside the slack.
        if (atSpanEnd(live, sourceFrom, END_SLACK_S) || progress(live, sourceFrom) <= live.reached + 1e-3) {
          live.ended = true;
        } else {
          live.reached = progress(live, sourceFrom);
          // A source that stopped answering mid-file is the one case a
          // re-mint cannot speak to, so this failure keeps to itself.
          this.miss(live, false);
        }
        return;
      }
      const tNow = this.at();
      if (!current() || live.scheduled !== from) return;
      const at2 = from + shift;
      const until = at2 + buffer.duration - skip;
      if (until <= tNow) {
        this.noteLate(live, tNow - at2);
        // The read outlived the moment it was for — a long wait on bytes, a
        // tab in the background. Where it left the walk is behind the clock
        // now, so the voice re-aims at the clock and gives up the stretch in
        // between; that stretch was going by either way.
        this.reaim(live, Math.max(live.scheduled, tNow));
        return;
      }
      const node = this.ctx.createBufferSource();
      node.buffer = buffer;
      node.connect(live.head);
      const at = this.anchor.ctx + (at2 - this.anchor.timeline);
      const now = this.ctx.currentTime;
      if (at >= now) {
        node.start(at, skip);
      } else {
        // Late: start where the clock actually is, so the sound stays lined up
        // with the picture and skips what has already gone by.
        this.noteLate(live, now - at);
        node.start(now, skip + (now - at));
      }
      live.windows.push({ node, until });
      live.scheduled = until;
      live.attempts = 0;
    } catch {
      // Past the read, everything is scheduling: a context closed or swapped
      // under the voice while this group decoded. The source is fine, so the
      // voice spaces out its next attempt and keeps every one of its tries.
      this.backOff(live, 1);
    } finally {
      if (live.gen === gen) live.filling = false;
    }
  }

  /**
   * The next `GROUP_S` of a voice's sound, pulled off its walk.
   *
   * The walk is opened where the voice stands when it hasn't got one — the
   * first ask, or the first after something moved it — and kept open after
   * that, so a clip is one open file, one container parse and one decoder for
   * as long as it plays.
   */
  private async walkGroup(
    live: LiveVoice,
    sourceFrom: number,
    current: () => boolean,
    group: number
  ): Promise<AudioBuffer | null> {
    const walk = (live.walk ??= openAudioWalk(live.url, sourceFrom, live.out));
    // A walk outlives the read that opened it, so where it stands is its own
    // to say. Anything that moved it — a re-aim, a priming read taken before
    // the play began — is answered here rather than trusted.
    if (Math.abs(walk.position - sourceFrom) > 0.02) walk.seek(sourceFrom);
    const parts: WrappedAudioBuffer[] = [];
    let end = sourceFrom;
    // Bounded by the sound it holds and by the reads that go into it: a track
    // answering with buffers that carry no time would otherwise be pulled for
    // as long as the frame lasted.
    for (let pulls = 0; end - sourceFrom < group && pulls < GROUP_PULLS; pulls++) {
      const part = await walk.next();
      if (!current()) return null;
      if (!part) break;
      // The stall guard is about a source that stopped answering, and a walk
      // that is handing sound back has not: every part it delivers is the read
      // saying it is still there. Without this a cold open of a long file over
      // a slow link is disowned on the clock, re-opened from nothing, and
      // disowned again, and the voice never gets a group out.
      live.fillAt = performance.now();
      parts.push(part);
      end = part.timestamp + part.duration;
    }
    return assembleAudio(parts, sourceFrom, live.out);
  }

  /**
   * A window of a voice playing at a speed other than 1, re-laid at the length
   * the timeline gives it.
   *
   * Held briefly, because stepping back over a cut replays the same window and
   * everything older is a full decode away anyway.
   */
  private async stretchedWindow(live: LiveVoice, sourceFrom: number): Promise<FittedWindow | null> {
    const rt = live.retime;
    const tFrom = rt.tAt(sourceFrom);
    // The window runs on from `sourceFrom` the way the voice reads: up the
    // source, or down it on a reversed clip, and never past the span.
    const far = rt.srcAt(tFrom + WINDOW_STRETCHED_S);
    const lo = rt.reverse ? Math.max(live.in, far) : sourceFrom;
    const hi = rt.reverse ? sourceFrom : Math.min(live.out, far);
    const key = `${live.url}|${lo.toFixed(3)}|${hi.toFixed(3)}|${rt.key}`;
    const held = this.decoded.get(key);
    if (held) return held;
    const raw = await decodeAudioSpan(live.url, lo, hi);
    if (!raw) return null;
    // The window is laid by the clip's own map from where it starts, so its
    // length is exactly the timeline the picture spends on it.
    return this.keep(key, this.fit(raw, rt, lo, hi));
  }

  /**
   * A short clip's whole sound, decoded once and held.
   *
   * A walk over the file costs an open before its first read answers, and a
   * clip shorter than that open is over before its sound arrives; the clip is
   * one read instead, and the read is kept. Joined part-way through — a play
   * begun inside it — the window starts that far into the buffer.
   */
  private async shortWindow(live: LiveVoice, sourceFrom: number): Promise<FittedWindow | null> {
    const key = `${live.url}|${live.in.toFixed(3)}|${live.out.toFixed(3)}|${live.retime.key}`;
    const skip = Math.max(0, sourceFrom - live.in);
    const held = this.decoded.get(key);
    if (held) return { ...held, skip };
    const raw = await decodeAudioSpan(live.url, live.in, live.out);
    if (!raw) return null;
    return { ...this.keep(key, { buffer: raw, shift: 0 }), skip };
  }

  /** Keep a decoded window, letting the oldest go until the cache is back
   * inside its share. Emptying the whole cache would throw away the window the
   * playhead is standing on along with the ones it has finished, and the next
   * frame would pay a full decode for sound it was already holding. */
  private keep(key: string, window: FittedWindow): FittedWindow {
    this.decoded.set(key, window);
    const cap = allowance("mixerAudio", WINDOW_CACHE_BYTES);
    let cached = decodedBytes(this.decoded);
    for (const [oldest, held] of this.decoded) {
      if (cached <= cap || this.decoded.size <= 1) break;
      cached -= held.buffer.length * held.buffer.numberOfChannels * 4;
      this.decoded.delete(oldest);
    }
    return window;
  }

  /**
   * Move a voice up to the clock, keeping the file it is reading.
   *
   * A group that lands after the moment it was for is the ordinary shape of a
   * slow open: the file takes a few seconds to parse, and by the time the
   * first sound comes back the playhead has gone past it. Answering that by
   * giving up the walk starts the whole open again a second behind where it
   * just was, and again after that — the voice pays the open cost for the rest
   * of the play and never once catches the clock, which is a cut that plays
   * out in silence while the picture rolls on. The reader moves instead: same
   * file, same decoder, aimed where the sound is wanted.
   */
  private reaim(live: LiveVoice, timeline: number): void {
    live.gen++;
    live.filling = false;
    live.ended = false;
    live.scheduled = timeline;
    live.walk?.seek(live.retime.srcAt(timeline - live.start));
  }

  /** A read that failed, came back empty, or stopped answering. The voice
   * gives up where it was, waits, and asks again — and asks for as long as the
   * play lasts. */
  private miss(live: LiveVoice, report = true): void {
    this.moved(live);
    live.attempts += 1;
    this.backOff(live, live.attempts);
    if (!report) return;
    // There is no element to reload and nothing else asks on the sound's
    // behalf, so a signed URL past its window would stay past it. The picture's
    // readers report their failed reads the same way; the re-mint that follows
    // swaps the store's URLs and every voice reopens under the new one.
    void import("./mediaLinks")
      .then((m) => m.reportMediaUrlError(live.url))
      .catch(() => {});
  }

  /** Hold the voice off until its next try, `step` failures in. */
  private backOff(live: LiveVoice, step: number): void {
    live.retryAt =
      performance.now() + (step <= READ_RETRIES ? READ_RETRY_MS * step : READ_RECHECK_MS);
  }

  /** Re-lay a buffer at a different length, keeping its pitch. */
  private fit(
    buffer: AudioBuffer,
    rt: Retime,
    lo: number,
    hi: number
  ): { buffer: AudioBuffer; shift: number } {
    const channels: Float32Array[] = [];
    for (let c = 0; c < buffer.numberOfChannels; c++) channels.push(buffer.getChannelData(c));
    const out = fitSpan(channels, buffer.sampleRate, rt, lo, hi);
    const result = new AudioBuffer({
      length: Math.max(1, out.channels[0]?.length ?? 1),
      numberOfChannels: Math.max(1, out.channels.length),
      sampleRate: buffer.sampleRate,
    });
    // A fresh view per channel: `copyToChannel` wants a plain Float32Array,
    // and the stretch hands back views whose buffer type it declines.
    out.channels.forEach((data, i) => result.copyToChannel(new Float32Array(data), i));
    return { buffer: result, shift: out.shift };
  }
}