Open source CapCut alternative.
TypeScript
fix(cut): a voice reads its file where the picture reads it [rebuild]
David Le53d2f9bc4a3b
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.tsindex 8439adf6..5b9e4b49 100644--- a/site/src/cut/lib/mediaRead.ts+++ b/site/src/cut/lib/mediaRead.ts@@ -439,15 +439,28 @@ export async function decodeAudioSpan(): Promise<AudioBuffer | null> {const chunks: WrappedAudioBuffer[] = [];for await (const chunk of audioChunks(src, from, to)) chunks.push(chunk);+ return assembleAudio(chunks, from, to);+}++/**+ * Lay decoded chunks into one buffer covering `[from, to)`, or null when there+ * are none.+ *+ * The buffer begins exactly at `from`, which is what callers assume when they+ * play it from 0. Decoding lands on packet boundaries, so the first chunk+ * usually starts a little earlier; that head is trimmed rather than being+ * allowed to shift the whole span early.+ */+export function assembleAudio(+ chunks: WrappedAudioBuffer[],+ from: number,+ to?: number+): AudioBuffer | null {if (chunks.length === 0) return null;const sampleRate = chunks[0].buffer.sampleRate;const numberOfChannels = Math.max(...chunks.map((c) => c.buffer.numberOfChannels));const last = chunks[chunks.length - 1];- // The buffer begins exactly at `from`, which is what callers assume when they- // play it from 0. Decoding lands on packet boundaries, so the first chunk- // usually starts a little earlier; that head is trimmed below rather than- // being allowed to shift the whole span early.const start = from;const end = Math.min(to ?? Infinity, last.timestamp + last.duration);const length = Math.max(1, Math.round((end - start) * sampleRate));@@ -469,6 +482,61 @@ export async function decodeAudioSpan(return out;}+/** A reader part-way through one file's audio: `next` hands back the buffers+ * in the order they play and answers null at the end of the span, and `close`+ * gives up the file however far it got. */+export interface AudioWalk {+ next(): Promise<WrappedAudioBuffer | null>;+ close(): void;+}++/**+ * Walk a file's audio forward from `from`, a buffer at a time.+ *+ * Asking for a span at a time is what a caller reaches for first, and over a+ * network it is the wrong shape: audio packets are interleaved with the+ * picture through the whole file, so twenty seconds of sound is scattered+ * across twenty seconds of *everything* — a hundred megabytes of a phone+ * recording to fill a few hundred kilobytes of samples. A reader asking that+ * far ahead is asking for bytes the link has not reached yet, and it goes on+ * asking for them for as long as it plays, which is a sound that arrives after+ * the moment it was for.+ *+ * A walk reads the file once, in order, at the pace the caller pulls, so the+ * sound is exactly as far into the file as the picture is. It is the same+ * shape the frame readers hold, and it is what lets one open file, one+ * container parse and one decoder serve a whole clip.+ */+export function openAudioWalk(src: string | Blob, from = 0, to?: number): AudioWalk {+ const input = openMedia(src);+ let closed = false;+ let opening: Promise<AsyncGenerator<WrappedAudioBuffer, void, unknown> | null> | null = null;+ const open = () =>+ (opening ??= (async () => {+ const track = await audioTrackOf(input);+ return track ? new AudioBufferSink(track).buffers(from, to) : null;+ })());+ return {+ async next() {+ if (closed) return null;+ const gen = await open();+ if (!gen || closed) return null;+ const step = await gen.next();+ return step.done ? null : step.value;+ },+ close() {+ if (closed) return;+ closed = true;+ // The generator is told it is over, which is where the decoder it holds+ // is released, and the file goes at once behind it: disposing is what+ // cancels a read still in the air, and a walk being closed because it+ // stopped answering is waiting on exactly one of those.+ void opening?.then((gen) => gen?.return()).catch(() => {});+ input.dispose();+ },+ };+}+/*** Normalized 0..1 waveform peaks across the whole file, `buckets` wide.*diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.tsnew file mode 100644index 00000000..73588b89--- /dev/null+++ b/site/src/cut/lib/previewMixer.test.ts@@ -0,0 +1,285 @@+import { describe, expect, mock, test } from "bun:test";+import { AudioBuffer } from "node-web-audio-api";++/**+ * The preview's sound, against a link that only just keeps up.+ *+ * This is how a cloud project actually plays: the file arrives over+ * the network at about the rate it is watched, a little of it already here.+ * Every case below plays a single long clip through and measures the one thing+ * a listener measures — how much of the cut made a sound — because every way+ * this has failed looked identical from outside. The picture went on playing+ * and the sound stopped, once, for good.+ */++const RATE = 48_000;+/** How much source audio one read off the walk hands back. */+const PART_S = 0.2;+const CLIP_S = 70;+/** The link: bytes arrive at the rate the clip plays, with a little of the+ * file already resident when playback starts. A voice that has to read far+ * ahead of the playhead can never be satisfied here; one that reads where the+ * picture reads is comfortable. */+const LINK_RATE = 1;+const HEAD_START_S = 2.5;++// ── the clock everything shares ─────────────────────────────────────────────+let wall = 0;+let ctxTime = 0;+let resumes = 0;+const globals = globalThis as Record<string, unknown>;+globals.AudioBuffer ??= AudioBuffer;+globals.performance = { now: () => wall * 1000 } as Performance;++// ── the sound leaving the graph ─────────────────────────────────────────────+interface Played {+ from: number;+ to: number;+}+let played: Played[] = [];++class FakeGain {+ gain = { value: 0 };+ connect() {}+ disconnect() {}+}++class FakeSource {+ buffer: { duration: number } | null = null;+ connect() {}+ disconnect() {}+ start(when: number, offset = 0) {+ const length = (this.buffer?.duration ?? 0) - offset;+ if (length > 0) played.push({ from: when, to: when + length });+ }+ stop() {}+}++class FakeContext {+ state = "running";+ baseLatency = 0;+ outputLatency = 0;+ destination = {};+ get currentTime() {+ return ctxTime;+ }+ createGain() {+ return new FakeGain();+ }+ createBufferSource() {+ return new FakeSource();+ }+ resume() {+ resumes++;+ this.state = "running";+ return Promise.resolve();+ }+ close() {+ return Promise.resolve();+ }+}+globals.AudioContext = FakeContext;++// ── the file, arriving over the link ────────────────────────────────────────+/** Reads that throw, and reads that never answer, over a stretch of wall time. */+let failUntil = 0;+let hangUntil = 0;++const waiting: { at: number; wake: () => void }[] = [];++/** Resolves once the link has delivered the file as far as `sourceTime`. */+function delivered(sourceTime: number): Promise<void> {+ const at = Math.max(0, (sourceTime - HEAD_START_S) / LINK_RATE);+ if (wall >= at) return Promise.resolve();+ return new Promise((wake) => waiting.push({ at, wake }));+}++const walks = { opened: 0 };++const openAudioWalk = (_url: string, from: number, to?: number) => {+ walks.opened++;+ const end = to ?? CLIP_S;+ let pos = from;+ let closed = false;+ return {+ async next() {+ if (closed) return null;+ if (wall < hangUntil) return new Promise<null>(() => {});+ await delivered(pos + PART_S);+ if (closed) return null;+ if (wall < failUntil) throw new Error("the link dropped");+ if (pos >= end - 1e-6) return null;+ const duration = Math.min(PART_S, end - pos);+ const part = {+ buffer: new AudioBuffer({+ length: Math.round(duration * RATE),+ numberOfChannels: 1,+ sampleRate: RATE,+ }),+ timestamp: pos,+ duration,+ };+ pos += duration;+ return part;+ },+ close() {+ closed = true;+ },+ };+};++const assembleAudio = (+ parts: { timestamp: number; duration: number }[],+ from: number,+ to?: number+) => {+ if (parts.length === 0) return null;+ const last = parts[parts.length - 1];+ const end = Math.min(to ?? Infinity, last.timestamp + last.duration);+ return new AudioBuffer({+ length: Math.max(1, Math.round((end - from) * RATE)),+ numberOfChannels: 1,+ sampleRate: RATE,+ });+};++const reMints = { asked: 0 };++/** A span asked for in one go, which the link cannot answer until it has+ * delivered the file as far as the span's end. */+const decodeAudioSpan = async (_url: string, from: number, to: number) => {+ await delivered(to);+ if (wall < failUntil) throw new Error("the link dropped");+ if (from >= CLIP_S - 1e-6) return null;+ const end = Math.min(to, CLIP_S);+ return new AudioBuffer({+ length: Math.max(1, Math.round((end - from) * RATE)),+ numberOfChannels: 1,+ sampleRate: RATE,+ });+};++// Only the reads are stood in for. A module mock replaces the whole module for+// every test file in the run, so the rest of each one is carried through as it+// really is.+const media = await import("./mediaRead");+const links = await import("./mediaLinks");+mock.module("./mediaRead", () => ({ ...media, openAudioWalk, assembleAudio, decodeAudioSpan }));+mock.module("./mediaLinks", () => ({+ ...links,+ reportMediaUrlError: () => {+ reMints.asked++;+ },+}));++const { PreviewMixer } = await import("./previewMixer");++// ── driving a play ──────────────────────────────────────────────────────────+const FRAME_S = 0.05;++/** Let everything the frame set going settle before the clock moves again. */+async function settle(): Promise<void> {+ for (let i = 0; i < 32; i++) await Promise.resolve();+}++function step(): void {+ wall += FRAME_S;+ ctxTime += FRAME_S;+ for (const w of waiting.splice(0)) {+ if (w.at <= wall) w.wake();+ else waiting.push(w);+ }+}++const voice = {+ id: "clip",+ url: "clip.mp4",+ start: 0,+ in: 0,+ out: CLIP_S,+ speed: 1,+ gain: 1,+};++/** Play the clip through, running `during` on each frame at the timeline time+ * it is drawing. Answers what was heard. */+async function play(during?: (t: number, ctx: FakeContext) => void): Promise<Played[]> {+ wall = 0;+ ctxTime = 0;+ resumes = 0;+ played = [];+ waiting.length = 0;+ walks.opened = 0;+ reMints.asked = 0;+ failUntil = 0;+ hangUntil = 0;+ const mixer = new PreviewMixer();+ mixer.start(0);+ try {+ for (let t = 0; t < CLIP_S; ) {+ step();+ t = Math.min(mixer.now(), CLIP_S);+ during?.(t, (mixer as unknown as { ctx: FakeContext }).ctx);+ mixer.update(t, [voice]);+ await settle();+ }+ } finally {+ mixer.dispose();+ }+ return played;+}++/** How much of `[from, to)` made a sound. */+function heard(sound: Played[], from: number, to: number): number {+ const spans = sound+ .map((p) => ({ from: Math.max(from, p.from), to: Math.min(to, p.to) }))+ .filter((p) => p.to > p.from)+ .sort((a, b) => a.from - b.from);+ let covered = 0;+ let at = from;+ for (const s of spans) {+ if (s.to <= at) continue;+ covered += s.to - Math.max(at, s.from);+ at = s.to;+ }+ return covered / (to - from);+}++describe("the preview's sound over a link that only just keeps up", () => {+ test("plays the whole clip through", async () => {+ const sound = await play();+ // The failure this is here for is a cut that sounds for its first stretch+ // and then plays out in silence, so the end of it is what matters.+ expect(heard(sound, 0, CLIP_S)).toBeGreaterThan(0.95);+ expect(heard(sound, CLIP_S - 20, CLIP_S)).toBeGreaterThan(0.95);+ // One file, one reader: a walk is opened once and kept.+ expect(walks.opened).toBe(1);+ });++ test("comes back from a long outage", async () => {+ const sound = await play((t) => {+ if (t >= 20 && t < 45) failUntil = 45;+ });+ expect(heard(sound, 0, 18)).toBeGreaterThan(0.9);+ expect(heard(sound, 50, CLIP_S)).toBeGreaterThan(0.9);+ // A read that fails has no element to reload; only a re-mint heals a URL+ // past its signing window, so the voice asks for one.+ expect(reMints.asked).toBeGreaterThan(0);+ });++ test("disowns a read that stops answering", async () => {+ const sound = await play((t) => {+ if (t >= 20 && t < 30) hangUntil = 30;+ });+ expect(heard(sound, 0, 18)).toBeGreaterThan(0.9);+ expect(heard(sound, 50, CLIP_S)).toBeGreaterThan(0.9);+ });++ test("picks its context back up when the browser puts it down", async () => {+ const sound = await play((t, ctx) => {+ if (t >= 20 && t < 20.1) ctx.state = "suspended";+ });+ expect(resumes).toBeGreaterThan(1);+ expect(heard(sound, 25, CLIP_S)).toBeGreaterThan(0.9);+ });+});diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsindex 69cd4423..6c4b6dd6 100644--- a/site/src/cut/lib/previewMixer.ts+++ b/site/src/cut/lib/previewMixer.ts@@ -12,10 +12,18 @@* Here every audible thing is a buffer scheduled on one `AudioContext`. The* context's own clock is what the preview runs on — it advances in real time,* never backward, and is sample-accurate — so the picture follows the sound- * rather than the other way round. Clips are scheduled a window at a time so- * starting playback costs one short decode instead of the whole timeline, and a- * clip playing at a speed other than 1 is time-stretched rather than resampled,- * which is what keeps its pitch and matches what the export writes.+ * rather than the other way round.+ *+ * Every voice holds a reader walking its own file forward and hands the context+ * about a second of sound at a time. Reading in order is the part that matters:+ * a voice asking for a long span at once is asking for the whole interleaved+ * stretch of file that span is scattered through, which over a network is+ * minutes of bytes for seconds of sound and a voice permanently behind its own+ * playhead. Walking keeps the sound exactly as far into the file as the picture+ * is. A clip playing at a speed other than 1 is time-stretched rather than+ * resampled, which is what keeps its pitch and matches what the export writes;+ * a stretch needs its span in hand, so those voices read a short window instead+ * of walking.** Gains are the only thing touched per frame, and only their value: the clip's* own volume, its fades, the duck under a live voiceover, the whole-project@@ -28,15 +36,23 @@*/import { audioFxRecipe, buildAudioFx, type AudioFxNodes } from "@donkeycut/effects-kit";-import { decodeAudioSpan } from "./mediaRead";+import type { WrappedAudioBuffer } from "mediabunny";+import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";import { timeStretch } from "./timeStretch";-/** How much of a voice is decoded and scheduled at once, in timeline seconds. */-const WINDOW_S = 20;+/** How much of a voice is pulled off its walk and scheduled at once, in source+ * seconds. Long enough that a stretch of sound is one scheduling decision;+ * short enough that the bytes it waits on are the ones the picture is reading+ * anyway. */+const GROUP_S = 1;+/** Most reads one group is assembled from. A packet of compressed audio holds+ * a fraction of a second, so a group is tens of reads, and this is the ceiling+ * over them. */+const GROUP_PULLS = 128;/** A stretched window is resynthesised on the main thread, so it is kept short* enough that the work lands inside a frame or two. */const WINDOW_STRETCHED_S = 4;-/** Schedule the next window once the playhead is this close to running out. */+/** Schedule more of a voice once the playhead is this close to running out. */const AHEAD_S = 2;/** Most the picture will be held back for the output device, whatever the* device claims. Well past a Bluetooth headset, and short of anything that@@ -75,26 +91,41 @@ interface LiveVoice {windows: Scheduled[];/** Timeline time everything scheduled so far runs out at. */scheduled: number;- /** A decode in flight, so a slow read doesn't start a second one. */+ /** The reader walking this voice's source, open from where it last read to.+ * A walk is a position and nothing more, so anything that moves the voice —+ * a seek, a clock jump, a read that failed — drops it. */+ walk: AudioWalk | null;+ /** A read in flight, so a slow one doesn't start a second one, and when it+ * started, so one that has stopped answering can be disowned. */filling: boolean;- /** Set when the voice has run off the end of its source, or its source has- * refused enough times to stop asking. */- dead: boolean;- /** Consecutive failed reads. A file that has just been imported is often- * unreadable for a moment — the bytes are still going to storage, the- * signed URL has not been minted — and a voice that gave up on the first- * miss stayed silent for the life of the page. */- misses: number;- /** Wall time (performance.now) the next read may be attempted at. */+ fillAt: number;+ /** Bumped by everything that moves the voice. A read carries the number it+ * started under and acts on what it got only while that still stands. */+ gen: number;+ /** Set when the voice has read to the end of its source. A position rather+ * than a verdict: anything that moves the voice clears it, so a seek back+ * into the clip plays again. */+ ended: boolean;+ /** Consecutive failed reads, and the wall time (performance.now) the next+ * one may be tried at. */+ attempts: number;retryAt: number;}-/** How many times a voice re-reads a source that will not answer before it- * gives up, and the backoff between tries. Six tries spans about half a- * minute, which covers an import landing behind the playhead. */-const READ_RETRIES = 6;-const RETRY_BASE_MS = 400;-const RETRY_MAX_MS = 8_000;+/** A read that has not answered in this long has stopped being a read — a+ * request left hanging by a link that went away. The voice disowns it and asks+ * again; waiting it out would cost the rest of the play. */+const FILL_STALL_MS = 15_000;+/** A failed read is asked again quickly a few times, then on a slow cadence for+ * as long as the failure holds. Nothing here is ever terminal, and that is the+ * point: a source that cannot be read this second — bytes still landing, a+ * token a moment past its window, a link that dropped — is nearly always+ * readable shortly, the picture's own readers wait it out exactly this way+ * (frameSource.ts), and a voice that stopped asking was silent for the life of+ * the page while the picture beside it healed. */+const READ_RETRY_MS = 1_000;+const READ_RETRIES = 3;+const READ_RECHECK_MS = 10_000;/** One audio effect element the graph is carrying: its recipe's chain, and* the pair of gains that cross the mix between treated and untreated. */@@ -250,22 +281,45 @@ export class PreviewMixer {return { lead: Math.max(0, graph - this.heard(graph)), reported };}- /** Stop every scheduled window and mark each voice to refill at `timeline`. */+ /** Stop every scheduled window and re-aim each voice at `timeline`. */private restart(timeline: number): void {for (const live of this.voices.values()) {- for (const w of live.windows) {- try {- w.node.stop();- } catch {- // Already finished; stopping a spent node throws and means nothing.- }- w.node.disconnect();- }- live.windows = [];+ this.stopWindows(live);+ this.moved(live);live.scheduled = Math.max(live.start, timeline);}}+ /** Stop and drop every window a voice has handed to the context. */+ private stopWindows(live: LiveVoice): void {+ for (const w of live.windows) {+ try {+ w.node.stop();+ } catch {+ // Already finished; stopping a spent node throws and means nothing.+ }+ w.node.disconnect();+ }+ live.windows = [];+ }++ /**+ * The voice is no longer reading where it was.+ *+ * A seek, a clock jump, a read that failed or stopped answering — each of+ * them leaves the walk pointing at a moment nobody is waiting for, and a walk+ * is only a position, so it goes. The read still in flight against it is+ * disowned in the same breath: it carries the old number and everything it+ * comes back with is ignored.+ */+ private moved(live: LiveVoice): void {+ live.gen++;+ live.filling = false;+ live.ended = false;+ live.walk?.close();+ live.walk = null;+ }+/*** Begin playing at timeline time `t`.*@@ -377,6 +431,12 @@ export class PreviewMixer {*/update(t: number, voices: Voice[]): void {if (!this.anchor || !this.ctx) return;+ // A context that stops running stops the sound and leaves the picture+ // playing on wall time — an output device taken away, a page the browser+ // put to sleep. Nothing else asks it back, since `start` is the only other+ // resume, so a play that lost its context stayed silent to the end of the+ // cut.+ if (this.ctx.state !== "running") void this.ctx.resume().catch(() => {});const seen = new Set<string>();for (const v of voices) {seen.add(v.id);@@ -400,8 +460,12 @@ export class PreviewMixer {const live = this.voices.get(v.id) ?? this.open(v, t);if (!live) continue;live.gain.gain.value = Math.max(0, Math.min(1.5, v.gain));+ // A read that hangs holds the voice as surely as one that fails, and it+ // holds it quietly: nothing else would ever notice, and the voice would+ // wait on it for the rest of the play.+ if (live.filling && performance.now() - live.fillAt > FILL_STALL_MS) this.miss(live);if (- !live.dead &&+ !live.ended &&!live.filling &&live.scheduled < t + AHEAD_S &&performance.now() >= live.retryAt@@ -453,9 +517,12 @@ export class PreviewMixer {// playback started inside it, or the clip edited while audible — decodes// from the moment it is heard, not from a head nobody will hear.scheduled: Math.max(v.start, t),+ walk: null,filling: false,- dead: false,- misses: 0,+ fillAt: 0,+ gen: 0,+ ended: false,+ attempts: 0,retryAt: 0,};this.voices.set(v.id, live);@@ -463,15 +530,8 @@ export class PreviewMixer {}private release(live: LiveVoice): void {- for (const w of live.windows) {- try {- w.node.stop();- } catch {- // Already finished; stopping a spent node throws and means nothing.- }- w.node.disconnect();- }- live.windows = [];+ this.stopWindows(live);+ this.moved(live);live.gain.disconnect();}@@ -481,52 +541,65 @@ export class PreviewMixer {}/**- * Decode and schedule the next window of one voice.+ * Carry one voice a little further ahead of the playhead.*- * The window is measured in timeline seconds and the source span behind it- * scales with the clip's speed, so a clip at 2× reads twice as much file to- * fill the same stretch of timeline.+ * The voice's walk hands back the sound in the order it plays, and a group of+ * it is scheduled as one, placed by the source timestamps the buffers carry.+ * A stretch of file with no audio in it therefore moves the sound to where it+ * belongs and everything after it keeps its own time.+ *+ * Nothing here is terminal. A read that fails or stops answering leaves the+ * voice waiting a moment and asking again, for as long as the play lasts,+ * because every reason a source stops answering mid-play is a reason it+ * starts again.*/private async fill(id: string, live: LiveVoice): Promise<void> {if (!this.ctx || !this.anchor) return;+ const gen = ++live.gen;live.filling = true;+ live.fillAt = performance.now();+ // The voice can be moved out from under this read while it runs — a seek,+ // a clock jump, this very read disowned as stalled — and only the read that+ // still stands may act on what it got.+ const current = () => this.voices.get(id) === live && live.gen === gen;try {- const stretched = Math.abs(live.speed - 1) > 1e-3;- const span = stretched ? WINDOW_STRETCHED_S : WINDOW_S;const from = live.scheduled;const sourceFrom = live.in + (from - live.start) * live.speed;if (sourceFrom >= live.out - 1e-3) {- live.dead = true;+ live.ended = true;return;}- const sourceTo = Math.min(live.out, sourceFrom + span * live.speed);- const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;- let buffer = this.decoded.get(key) ?? null;- if (!buffer) {+ let buffer: AudioBuffer | null;+ try {+ buffer =+ Math.abs(live.speed - 1) > 1e-3+ ? await this.stretchedWindow(live, sourceFrom)+ : await this.walkGroup(live, sourceFrom, current);+ } catch {// The read is the only part of this that says anything about the- // source, so it is the only part a miss is spent on.- const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo).catch(() => null);- if (!raw) {- this.miss(live);- return;- }- buffer = stretched ? this.stretch(raw, 1 / live.speed) : raw;- // A handful of windows is all that's worth keeping — stepping back over- // a cut replays them, and everything older is a full decode away anyway.- if (this.decoded.size > 24) this.decoded.clear();- this.decoded.set(key, buffer);+ // source, so it is the only part a failure is spent on.+ if (current()) this.miss(live);+ return;+ }+ if (!current()) return;+ if (!buffer) {+ // The reader answered with nothing left to play. A track can stop+ // before the picture does, and a clip whose file carries no sound at+ // all answers this way from the first ask.+ live.ended = true;+ return;}- // The mixer may have been stopped, or the voice dropped, while decoding.- if (this.voices.get(id) !== live || !this.anchor || !this.ctx) return;- // Reading the clock can restart the voices (`restart`), so it happens- // before the placement below is computed — and a restarted voice, moved- // to a different position while this window decoded, abandons it.+ // Reading the clock can re-aim the voices (`restart`), so it happens+ // before the placement below is computed.const tNow = this.now();- if (live.scheduled !== from) return;+ if (!current() || live.scheduled !== from) return;const until = from + buffer.duration;if (until <= tNow) {- // The decode outlived the window it was for — a long read, or a tab- // that was in the background. Skip it and let the next fill catch up.+ // The read outlived the moment it was for — a long wait on bytes, a+ // tab in the background. Where it left the walk is behind the clock+ // now, so the voice re-aims at the clock and gives up the stretch in+ // between; that stretch was going by either way.+ this.moved(live);live.scheduled = Math.max(live.scheduled, tNow);return;}@@ -544,33 +617,86 @@ export class PreviewMixer {}live.windows.push({ node, until });live.scheduled = until;- live.misses = 0;+ live.attempts = 0;} catch {// Past the read, everything is scheduling: a context closed or swapped- // under the voice while this window decoded. The source is fine, so the- // voice spaces out its next attempt and keeps every one of its reads.- this.backOff(live, live.misses + 1);+ // under the voice while this group decoded. The source is fine, so the+ // voice spaces out its next attempt and keeps every one of its tries.+ this.backOff(live, 1);} finally {- live.filling = false;+ if (live.gen === gen) live.filling = false;}}- /** A read that came back empty or threw. The source may simply not be there- * yet — a just-imported file still on its way to storage — so the voice- * waits and asks again, and stays silent only once its tries are spent. */- private miss(live: LiveVoice): void {- live.misses += 1;- if (live.misses >= READ_RETRIES) {- live.dead = true;- return;+ /**+ * The next `GROUP_S` of a voice's sound, pulled off its walk.+ *+ * The walk is opened where the voice stands when it hasn't got one — the+ * first ask, or the first after something moved it — and kept open after+ * that, so a clip is one open file, one container parse and one decoder for+ * as long as it plays.+ */+ private async walkGroup(+ live: LiveVoice,+ sourceFrom: number,+ current: () => boolean+ ): Promise<AudioBuffer | null> {+ const walk = (live.walk ??= openAudioWalk(live.url, sourceFrom, live.out));+ const parts: WrappedAudioBuffer[] = [];+ let end = sourceFrom;+ // Bounded by the sound it holds and by the reads that go into it: a track+ // answering with buffers that carry no time would otherwise be pulled for+ // as long as the frame lasted.+ for (let pulls = 0; end - sourceFrom < GROUP_S && pulls < GROUP_PULLS; pulls++) {+ const part = await walk.next();+ if (!current()) return null;+ if (!part) break;+ parts.push(part);+ end = part.timestamp + part.duration;}- this.backOff(live, live.misses);+ return assembleAudio(parts, sourceFrom, live.out);+ }++ /**+ * A window of a voice playing at a speed other than 1, re-laid at the length+ * the timeline gives it.+ *+ * Held briefly, because stepping back over a cut replays the same window and+ * everything older is a full decode away anyway.+ */+ private async stretchedWindow(live: LiveVoice, sourceFrom: number): Promise<AudioBuffer | null> {+ const sourceTo = Math.min(live.out, sourceFrom + WINDOW_STRETCHED_S * live.speed);+ const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;+ const held = this.decoded.get(key);+ if (held) return held;+ const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);+ if (!raw) return null;+ const buffer = this.stretch(raw, 1 / live.speed);+ if (this.decoded.size > 24) this.decoded.clear();+ this.decoded.set(key, buffer);+ return buffer;+ }++ /** A read that failed, came back empty, or stopped answering. The voice+ * gives up where it was, waits, and asks again — and asks for as long as the+ * play lasts. */+ private miss(live: LiveVoice): void {+ this.moved(live);+ live.attempts += 1;+ this.backOff(live, live.attempts);+ // There is no element to reload and nothing else asks on the sound's+ // behalf, so a signed URL past its window would stay past it. The picture's+ // readers report their failed reads the same way; the re-mint that follows+ // swaps the store's URLs and every voice reopens under the new one.+ void import("./mediaLinks")+ .then((m) => m.reportMediaUrlError(live.url))+ .catch(() => {});}- /** Hold the voice off until the next attempt, `step` failures in. */+ /** Hold the voice off until its next try, `step` failures in. */private backOff(live: LiveVoice, step: number): void {live.retryAt =- performance.now() + Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** Math.max(0, step - 1));+ performance.now() + (step <= READ_RETRIES ? READ_RETRY_MS * step : READ_RECHECK_MS);}/** Re-lay a buffer at a different length, keeping its pitch. */