Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): a voice reads its file where the picture reads it [rebuild]

David Le53d2f9bc4a3b
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.ts
index 8439adf6..5b9e4b49 100644
--- a/site/src/cut/lib/mediaRead.ts
+++ b/site/src/cut/lib/mediaRead.ts
@@ -439,15 +439,28 @@ export async function decodeAudioSpan(
): Promise<AudioBuffer | null> {
const chunks: WrappedAudioBuffer[] = [];
for await (const chunk of audioChunks(src, from, to)) chunks.push(chunk);
+ return assembleAudio(chunks, from, to);
+}
+
+/**
+ * Lay decoded chunks into one buffer covering `[from, to)`, or null when there
+ * are none.
+ *
+ * The buffer begins exactly at `from`, which is what callers assume when they
+ * play it from 0. Decoding lands on packet boundaries, so the first chunk
+ * usually starts a little earlier; that head is trimmed rather than being
+ * allowed to shift the whole span early.
+ */
+export function assembleAudio(
+ chunks: WrappedAudioBuffer[],
+ from: number,
+ to?: number
+): AudioBuffer | null {
if (chunks.length === 0) return null;
const sampleRate = chunks[0].buffer.sampleRate;
const numberOfChannels = Math.max(...chunks.map((c) => c.buffer.numberOfChannels));
const last = chunks[chunks.length - 1];
- // The buffer begins exactly at `from`, which is what callers assume when they
- // play it from 0. Decoding lands on packet boundaries, so the first chunk
- // usually starts a little earlier; that head is trimmed below rather than
- // being allowed to shift the whole span early.
const start = from;
const end = Math.min(to ?? Infinity, last.timestamp + last.duration);
const length = Math.max(1, Math.round((end - start) * sampleRate));
@@ -469,6 +482,61 @@ export async function decodeAudioSpan(
return out;
}
+/** A reader part-way through one file's audio: `next` hands back the buffers
+ * in the order they play and answers null at the end of the span, and `close`
+ * gives up the file however far it got. */
+export interface AudioWalk {
+ next(): Promise<WrappedAudioBuffer | null>;
+ close(): void;
+}
+
+/**
+ * Walk a file's audio forward from `from`, a buffer at a time.
+ *
+ * Asking for a span at a time is what a caller reaches for first, and over a
+ * network it is the wrong shape: audio packets are interleaved with the
+ * picture through the whole file, so twenty seconds of sound is scattered
+ * across twenty seconds of *everything* — a hundred megabytes of a phone
+ * recording to fill a few hundred kilobytes of samples. A reader asking that
+ * far ahead is asking for bytes the link has not reached yet, and it goes on
+ * asking for them for as long as it plays, which is a sound that arrives after
+ * the moment it was for.
+ *
+ * A walk reads the file once, in order, at the pace the caller pulls, so the
+ * sound is exactly as far into the file as the picture is. It is the same
+ * shape the frame readers hold, and it is what lets one open file, one
+ * container parse and one decoder serve a whole clip.
+ */
+export function openAudioWalk(src: string | Blob, from = 0, to?: number): AudioWalk {
+ const input = openMedia(src);
+ let closed = false;
+ let opening: Promise<AsyncGenerator<WrappedAudioBuffer, void, unknown> | null> | null = null;
+ const open = () =>
+ (opening ??= (async () => {
+ const track = await audioTrackOf(input);
+ return track ? new AudioBufferSink(track).buffers(from, to) : null;
+ })());
+ return {
+ async next() {
+ if (closed) return null;
+ const gen = await open();
+ if (!gen || closed) return null;
+ const step = await gen.next();
+ return step.done ? null : step.value;
+ },
+ close() {
+ if (closed) return;
+ closed = true;
+ // The generator is told it is over, which is where the decoder it holds
+ // is released, and the file goes at once behind it: disposing is what
+ // cancels a read still in the air, and a walk being closed because it
+ // stopped answering is waiting on exactly one of those.
+ void opening?.then((gen) => gen?.return()).catch(() => {});
+ input.dispose();
+ },
+ };
+}
+
/**
* Normalized 0..1 waveform peaks across the whole file, `buckets` wide.
*
diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.ts
new file mode 100644
index 00000000..73588b89
--- /dev/null
+++ b/site/src/cut/lib/previewMixer.test.ts
@@ -0,0 +1,285 @@
+import { describe, expect, mock, test } from "bun:test";
+import { AudioBuffer } from "node-web-audio-api";
+
+/**
+ * The preview's sound, against a link that only just keeps up.
+ *
+ * This is how a cloud project actually plays: the file arrives over
+ * the network at about the rate it is watched, a little of it already here.
+ * Every case below plays a single long clip through and measures the one thing
+ * a listener measures — how much of the cut made a sound — because every way
+ * this has failed looked identical from outside. The picture went on playing
+ * and the sound stopped, once, for good.
+ */
+
+const RATE = 48_000;
+/** How much source audio one read off the walk hands back. */
+const PART_S = 0.2;
+const CLIP_S = 70;
+/** The link: bytes arrive at the rate the clip plays, with a little of the
+ * file already resident when playback starts. A voice that has to read far
+ * ahead of the playhead can never be satisfied here; one that reads where the
+ * picture reads is comfortable. */
+const LINK_RATE = 1;
+const HEAD_START_S = 2.5;
+
+// ── the clock everything shares ─────────────────────────────────────────────
+let wall = 0;
+let ctxTime = 0;
+let resumes = 0;
+const globals = globalThis as Record<string, unknown>;
+globals.AudioBuffer ??= AudioBuffer;
+globals.performance = { now: () => wall * 1000 } as Performance;
+
+// ── the sound leaving the graph ─────────────────────────────────────────────
+interface Played {
+ from: number;
+ to: number;
+}
+let played: Played[] = [];
+
+class FakeGain {
+ gain = { value: 0 };
+ connect() {}
+ disconnect() {}
+}
+
+class FakeSource {
+ buffer: { duration: number } | null = null;
+ connect() {}
+ disconnect() {}
+ start(when: number, offset = 0) {
+ const length = (this.buffer?.duration ?? 0) - offset;
+ if (length > 0) played.push({ from: when, to: when + length });
+ }
+ stop() {}
+}
+
+class FakeContext {
+ state = "running";
+ baseLatency = 0;
+ outputLatency = 0;
+ destination = {};
+ get currentTime() {
+ return ctxTime;
+ }
+ createGain() {
+ return new FakeGain();
+ }
+ createBufferSource() {
+ return new FakeSource();
+ }
+ resume() {
+ resumes++;
+ this.state = "running";
+ return Promise.resolve();
+ }
+ close() {
+ return Promise.resolve();
+ }
+}
+globals.AudioContext = FakeContext;
+
+// ── the file, arriving over the link ────────────────────────────────────────
+/** Reads that throw, and reads that never answer, over a stretch of wall time. */
+let failUntil = 0;
+let hangUntil = 0;
+
+const waiting: { at: number; wake: () => void }[] = [];
+
+/** Resolves once the link has delivered the file as far as `sourceTime`. */
+function delivered(sourceTime: number): Promise<void> {
+ const at = Math.max(0, (sourceTime - HEAD_START_S) / LINK_RATE);
+ if (wall >= at) return Promise.resolve();
+ return new Promise((wake) => waiting.push({ at, wake }));
+}
+
+const walks = { opened: 0 };
+
+const openAudioWalk = (_url: string, from: number, to?: number) => {
+ walks.opened++;
+ const end = to ?? CLIP_S;
+ let pos = from;
+ let closed = false;
+ return {
+ async next() {
+ if (closed) return null;
+ if (wall < hangUntil) return new Promise<null>(() => {});
+ await delivered(pos + PART_S);
+ if (closed) return null;
+ if (wall < failUntil) throw new Error("the link dropped");
+ if (pos >= end - 1e-6) return null;
+ const duration = Math.min(PART_S, end - pos);
+ const part = {
+ buffer: new AudioBuffer({
+ length: Math.round(duration * RATE),
+ numberOfChannels: 1,
+ sampleRate: RATE,
+ }),
+ timestamp: pos,
+ duration,
+ };
+ pos += duration;
+ return part;
+ },
+ close() {
+ closed = true;
+ },
+ };
+};
+
+const assembleAudio = (
+ parts: { timestamp: number; duration: number }[],
+ from: number,
+ to?: number
+) => {
+ if (parts.length === 0) return null;
+ const last = parts[parts.length - 1];
+ const end = Math.min(to ?? Infinity, last.timestamp + last.duration);
+ return new AudioBuffer({
+ length: Math.max(1, Math.round((end - from) * RATE)),
+ numberOfChannels: 1,
+ sampleRate: RATE,
+ });
+};
+
+const reMints = { asked: 0 };
+
+/** A span asked for in one go, which the link cannot answer until it has
+ * delivered the file as far as the span's end. */
+const decodeAudioSpan = async (_url: string, from: number, to: number) => {
+ await delivered(to);
+ if (wall < failUntil) throw new Error("the link dropped");
+ if (from >= CLIP_S - 1e-6) return null;
+ const end = Math.min(to, CLIP_S);
+ return new AudioBuffer({
+ length: Math.max(1, Math.round((end - from) * RATE)),
+ numberOfChannels: 1,
+ sampleRate: RATE,
+ });
+};
+
+// Only the reads are stood in for. A module mock replaces the whole module for
+// every test file in the run, so the rest of each one is carried through as it
+// really is.
+const media = await import("./mediaRead");
+const links = await import("./mediaLinks");
+mock.module("./mediaRead", () => ({ ...media, openAudioWalk, assembleAudio, decodeAudioSpan }));
+mock.module("./mediaLinks", () => ({
+ ...links,
+ reportMediaUrlError: () => {
+ reMints.asked++;
+ },
+}));
+
+const { PreviewMixer } = await import("./previewMixer");
+
+// ── driving a play ──────────────────────────────────────────────────────────
+const FRAME_S = 0.05;
+
+/** Let everything the frame set going settle before the clock moves again. */
+async function settle(): Promise<void> {
+ for (let i = 0; i < 32; i++) await Promise.resolve();
+}
+
+function step(): void {
+ wall += FRAME_S;
+ ctxTime += FRAME_S;
+ for (const w of waiting.splice(0)) {
+ if (w.at <= wall) w.wake();
+ else waiting.push(w);
+ }
+}
+
+const voice = {
+ id: "clip",
+ url: "clip.mp4",
+ start: 0,
+ in: 0,
+ out: CLIP_S,
+ speed: 1,
+ gain: 1,
+};
+
+/** Play the clip through, running `during` on each frame at the timeline time
+ * it is drawing. Answers what was heard. */
+async function play(during?: (t: number, ctx: FakeContext) => void): Promise<Played[]> {
+ wall = 0;
+ ctxTime = 0;
+ resumes = 0;
+ played = [];
+ waiting.length = 0;
+ walks.opened = 0;
+ reMints.asked = 0;
+ failUntil = 0;
+ hangUntil = 0;
+ const mixer = new PreviewMixer();
+ mixer.start(0);
+ try {
+ for (let t = 0; t < CLIP_S; ) {
+ step();
+ t = Math.min(mixer.now(), CLIP_S);
+ during?.(t, (mixer as unknown as { ctx: FakeContext }).ctx);
+ mixer.update(t, [voice]);
+ await settle();
+ }
+ } finally {
+ mixer.dispose();
+ }
+ return played;
+}
+
+/** How much of `[from, to)` made a sound. */
+function heard(sound: Played[], from: number, to: number): number {
+ const spans = sound
+ .map((p) => ({ from: Math.max(from, p.from), to: Math.min(to, p.to) }))
+ .filter((p) => p.to > p.from)
+ .sort((a, b) => a.from - b.from);
+ let covered = 0;
+ let at = from;
+ for (const s of spans) {
+ if (s.to <= at) continue;
+ covered += s.to - Math.max(at, s.from);
+ at = s.to;
+ }
+ return covered / (to - from);
+}
+
+describe("the preview's sound over a link that only just keeps up", () => {
+ test("plays the whole clip through", async () => {
+ const sound = await play();
+ // The failure this is here for is a cut that sounds for its first stretch
+ // and then plays out in silence, so the end of it is what matters.
+ expect(heard(sound, 0, CLIP_S)).toBeGreaterThan(0.95);
+ expect(heard(sound, CLIP_S - 20, CLIP_S)).toBeGreaterThan(0.95);
+ // One file, one reader: a walk is opened once and kept.
+ expect(walks.opened).toBe(1);
+ });
+
+ test("comes back from a long outage", async () => {
+ const sound = await play((t) => {
+ if (t >= 20 && t < 45) failUntil = 45;
+ });
+ expect(heard(sound, 0, 18)).toBeGreaterThan(0.9);
+ expect(heard(sound, 50, CLIP_S)).toBeGreaterThan(0.9);
+ // A read that fails has no element to reload; only a re-mint heals a URL
+ // past its signing window, so the voice asks for one.
+ expect(reMints.asked).toBeGreaterThan(0);
+ });
+
+ test("disowns a read that stops answering", async () => {
+ const sound = await play((t) => {
+ if (t >= 20 && t < 30) hangUntil = 30;
+ });
+ expect(heard(sound, 0, 18)).toBeGreaterThan(0.9);
+ expect(heard(sound, 50, CLIP_S)).toBeGreaterThan(0.9);
+ });
+
+ test("picks its context back up when the browser puts it down", async () => {
+ const sound = await play((t, ctx) => {
+ if (t >= 20 && t < 20.1) ctx.state = "suspended";
+ });
+ expect(resumes).toBeGreaterThan(1);
+ expect(heard(sound, 25, CLIP_S)).toBeGreaterThan(0.9);
+ });
+});
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
index 69cd4423..6c4b6dd6 100644
--- a/site/src/cut/lib/previewMixer.ts
+++ b/site/src/cut/lib/previewMixer.ts
@@ -12,10 +12,18 @@
* Here every audible thing is a buffer scheduled on one `AudioContext`. The
* context's own clock is what the preview runs on — it advances in real time,
* never backward, and is sample-accurate — so the picture follows the sound
- * rather than the other way round. Clips are scheduled a window at a time so
- * starting playback costs one short decode instead of the whole timeline, and a
- * clip playing at a speed other than 1 is time-stretched rather than resampled,
- * which is what keeps its pitch and matches what the export writes.
+ * rather than the other way round.
+ *
+ * Every voice holds a reader walking its own file forward and hands the context
+ * about a second of sound at a time. Reading in order is the part that matters:
+ * a voice asking for a long span at once is asking for the whole interleaved
+ * stretch of file that span is scattered through, which over a network is
+ * minutes of bytes for seconds of sound and a voice permanently behind its own
+ * playhead. Walking keeps the sound exactly as far into the file as the picture
+ * is. A clip playing at a speed other than 1 is time-stretched rather than
+ * resampled, which is what keeps its pitch and matches what the export writes;
+ * a stretch needs its span in hand, so those voices read a short window instead
+ * of walking.
*
* Gains are the only thing touched per frame, and only their value: the clip's
* own volume, its fades, the duck under a live voiceover, the whole-project
@@ -28,15 +36,23 @@
*/
import { audioFxRecipe, buildAudioFx, type AudioFxNodes } from "@donkeycut/effects-kit";
-import { decodeAudioSpan } from "./mediaRead";
+import type { WrappedAudioBuffer } from "mediabunny";
+import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";
import { timeStretch } from "./timeStretch";
-/** How much of a voice is decoded and scheduled at once, in timeline seconds. */
-const WINDOW_S = 20;
+/** How much of a voice is pulled off its walk and scheduled at once, in source
+ * seconds. Long enough that a stretch of sound is one scheduling decision;
+ * short enough that the bytes it waits on are the ones the picture is reading
+ * anyway. */
+const GROUP_S = 1;
+/** Most reads one group is assembled from. A packet of compressed audio holds
+ * a fraction of a second, so a group is tens of reads, and this is the ceiling
+ * over them. */
+const GROUP_PULLS = 128;
/** A stretched window is resynthesised on the main thread, so it is kept short
* enough that the work lands inside a frame or two. */
const WINDOW_STRETCHED_S = 4;
-/** Schedule the next window once the playhead is this close to running out. */
+/** Schedule more of a voice once the playhead is this close to running out. */
const AHEAD_S = 2;
/** Most the picture will be held back for the output device, whatever the
* device claims. Well past a Bluetooth headset, and short of anything that
@@ -75,26 +91,41 @@ interface LiveVoice {
windows: Scheduled[];
/** Timeline time everything scheduled so far runs out at. */
scheduled: number;
- /** A decode in flight, so a slow read doesn't start a second one. */
+ /** The reader walking this voice's source, open from where it last read to.
+ * A walk is a position and nothing more, so anything that moves the voice —
+ * a seek, a clock jump, a read that failed — drops it. */
+ walk: AudioWalk | null;
+ /** A read in flight, so a slow one doesn't start a second one, and when it
+ * started, so one that has stopped answering can be disowned. */
filling: boolean;
- /** Set when the voice has run off the end of its source, or its source has
- * refused enough times to stop asking. */
- dead: boolean;
- /** Consecutive failed reads. A file that has just been imported is often
- * unreadable for a moment — the bytes are still going to storage, the
- * signed URL has not been minted — and a voice that gave up on the first
- * miss stayed silent for the life of the page. */
- misses: number;
- /** Wall time (performance.now) the next read may be attempted at. */
+ fillAt: number;
+ /** Bumped by everything that moves the voice. A read carries the number it
+ * started under and acts on what it got only while that still stands. */
+ gen: number;
+ /** Set when the voice has read to the end of its source. A position rather
+ * than a verdict: anything that moves the voice clears it, so a seek back
+ * into the clip plays again. */
+ ended: boolean;
+ /** Consecutive failed reads, and the wall time (performance.now) the next
+ * one may be tried at. */
+ attempts: number;
retryAt: number;
}
-/** How many times a voice re-reads a source that will not answer before it
- * gives up, and the backoff between tries. Six tries spans about half a
- * minute, which covers an import landing behind the playhead. */
-const READ_RETRIES = 6;
-const RETRY_BASE_MS = 400;
-const RETRY_MAX_MS = 8_000;
+/** A read that has not answered in this long has stopped being a read — a
+ * request left hanging by a link that went away. The voice disowns it and asks
+ * again; waiting it out would cost the rest of the play. */
+const FILL_STALL_MS = 15_000;
+/** A failed read is asked again quickly a few times, then on a slow cadence for
+ * as long as the failure holds. Nothing here is ever terminal, and that is the
+ * point: a source that cannot be read this second — bytes still landing, a
+ * token a moment past its window, a link that dropped — is nearly always
+ * readable shortly, the picture's own readers wait it out exactly this way
+ * (frameSource.ts), and a voice that stopped asking was silent for the life of
+ * the page while the picture beside it healed. */
+const READ_RETRY_MS = 1_000;
+const READ_RETRIES = 3;
+const READ_RECHECK_MS = 10_000;
/** One audio effect element the graph is carrying: its recipe's chain, and
* the pair of gains that cross the mix between treated and untreated. */
@@ -250,22 +281,45 @@ export class PreviewMixer {
return { lead: Math.max(0, graph - this.heard(graph)), reported };
}
- /** Stop every scheduled window and mark each voice to refill at `timeline`. */
+ /** Stop every scheduled window and re-aim each voice at `timeline`. */
private restart(timeline: number): void {
for (const live of this.voices.values()) {
- for (const w of live.windows) {
- try {
- w.node.stop();
- } catch {
- // Already finished; stopping a spent node throws and means nothing.
- }
- w.node.disconnect();
- }
- live.windows = [];
+ this.stopWindows(live);
+ this.moved(live);
live.scheduled = Math.max(live.start, timeline);
}
}
+ /** Stop and drop every window a voice has handed to the context. */
+ private stopWindows(live: LiveVoice): void {
+ for (const w of live.windows) {
+ try {
+ w.node.stop();
+ } catch {
+ // Already finished; stopping a spent node throws and means nothing.
+ }
+ w.node.disconnect();
+ }
+ live.windows = [];
+ }
+
+ /**
+ * The voice is no longer reading where it was.
+ *
+ * A seek, a clock jump, a read that failed or stopped answering — each of
+ * them leaves the walk pointing at a moment nobody is waiting for, and a walk
+ * is only a position, so it goes. The read still in flight against it is
+ * disowned in the same breath: it carries the old number and everything it
+ * comes back with is ignored.
+ */
+ private moved(live: LiveVoice): void {
+ live.gen++;
+ live.filling = false;
+ live.ended = false;
+ live.walk?.close();
+ live.walk = null;
+ }
+
/**
* Begin playing at timeline time `t`.
*
@@ -377,6 +431,12 @@ export class PreviewMixer {
*/
update(t: number, voices: Voice[]): void {
if (!this.anchor || !this.ctx) return;
+ // A context that stops running stops the sound and leaves the picture
+ // playing on wall time — an output device taken away, a page the browser
+ // put to sleep. Nothing else asks it back, since `start` is the only other
+ // resume, so a play that lost its context stayed silent to the end of the
+ // cut.
+ if (this.ctx.state !== "running") void this.ctx.resume().catch(() => {});
const seen = new Set<string>();
for (const v of voices) {
seen.add(v.id);
@@ -400,8 +460,12 @@ export class PreviewMixer {
const live = this.voices.get(v.id) ?? this.open(v, t);
if (!live) continue;
live.gain.gain.value = Math.max(0, Math.min(1.5, v.gain));
+ // A read that hangs holds the voice as surely as one that fails, and it
+ // holds it quietly: nothing else would ever notice, and the voice would
+ // wait on it for the rest of the play.
+ if (live.filling && performance.now() - live.fillAt > FILL_STALL_MS) this.miss(live);
if (
- !live.dead &&
+ !live.ended &&
!live.filling &&
live.scheduled < t + AHEAD_S &&
performance.now() >= live.retryAt
@@ -453,9 +517,12 @@ export class PreviewMixer {
// playback started inside it, or the clip edited while audible — decodes
// from the moment it is heard, not from a head nobody will hear.
scheduled: Math.max(v.start, t),
+ walk: null,
filling: false,
- dead: false,
- misses: 0,
+ fillAt: 0,
+ gen: 0,
+ ended: false,
+ attempts: 0,
retryAt: 0,
};
this.voices.set(v.id, live);
@@ -463,15 +530,8 @@ export class PreviewMixer {
}
private release(live: LiveVoice): void {
- for (const w of live.windows) {
- try {
- w.node.stop();
- } catch {
- // Already finished; stopping a spent node throws and means nothing.
- }
- w.node.disconnect();
- }
- live.windows = [];
+ this.stopWindows(live);
+ this.moved(live);
live.gain.disconnect();
}
@@ -481,52 +541,65 @@ export class PreviewMixer {
}
/**
- * Decode and schedule the next window of one voice.
+ * Carry one voice a little further ahead of the playhead.
*
- * The window is measured in timeline seconds and the source span behind it
- * scales with the clip's speed, so a clip at 2× reads twice as much file to
- * fill the same stretch of timeline.
+ * The voice's walk hands back the sound in the order it plays, and a group of
+ * it is scheduled as one, placed by the source timestamps the buffers carry.
+ * A stretch of file with no audio in it therefore moves the sound to where it
+ * belongs and everything after it keeps its own time.
+ *
+ * Nothing here is terminal. A read that fails or stops answering leaves the
+ * voice waiting a moment and asking again, for as long as the play lasts,
+ * because every reason a source stops answering mid-play is a reason it
+ * starts again.
*/
private async fill(id: string, live: LiveVoice): Promise<void> {
if (!this.ctx || !this.anchor) return;
+ const gen = ++live.gen;
live.filling = true;
+ live.fillAt = performance.now();
+ // The voice can be moved out from under this read while it runs — a seek,
+ // a clock jump, this very read disowned as stalled — and only the read that
+ // still stands may act on what it got.
+ const current = () => this.voices.get(id) === live && live.gen === gen;
try {
- const stretched = Math.abs(live.speed - 1) > 1e-3;
- const span = stretched ? WINDOW_STRETCHED_S : WINDOW_S;
const from = live.scheduled;
const sourceFrom = live.in + (from - live.start) * live.speed;
if (sourceFrom >= live.out - 1e-3) {
- live.dead = true;
+ live.ended = true;
return;
}
- const sourceTo = Math.min(live.out, sourceFrom + span * live.speed);
- const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;
- let buffer = this.decoded.get(key) ?? null;
- if (!buffer) {
+ let buffer: AudioBuffer | null;
+ try {
+ buffer =
+ Math.abs(live.speed - 1) > 1e-3
+ ? await this.stretchedWindow(live, sourceFrom)
+ : await this.walkGroup(live, sourceFrom, current);
+ } catch {
// The read is the only part of this that says anything about the
- // source, so it is the only part a miss is spent on.
- const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo).catch(() => null);
- if (!raw) {
- this.miss(live);
- return;
- }
- buffer = stretched ? this.stretch(raw, 1 / live.speed) : raw;
- // A handful of windows is all that's worth keeping — stepping back over
- // a cut replays them, and everything older is a full decode away anyway.
- if (this.decoded.size > 24) this.decoded.clear();
- this.decoded.set(key, buffer);
+ // source, so it is the only part a failure is spent on.
+ if (current()) this.miss(live);
+ return;
+ }
+ if (!current()) return;
+ if (!buffer) {
+ // The reader answered with nothing left to play. A track can stop
+ // before the picture does, and a clip whose file carries no sound at
+ // all answers this way from the first ask.
+ live.ended = true;
+ return;
}
- // The mixer may have been stopped, or the voice dropped, while decoding.
- if (this.voices.get(id) !== live || !this.anchor || !this.ctx) return;
- // Reading the clock can restart the voices (`restart`), so it happens
- // before the placement below is computed — and a restarted voice, moved
- // to a different position while this window decoded, abandons it.
+ // Reading the clock can re-aim the voices (`restart`), so it happens
+ // before the placement below is computed.
const tNow = this.now();
- if (live.scheduled !== from) return;
+ if (!current() || live.scheduled !== from) return;
const until = from + buffer.duration;
if (until <= tNow) {
- // The decode outlived the window it was for — a long read, or a tab
- // that was in the background. Skip it and let the next fill catch up.
+ // The read outlived the moment it was for — a long wait on bytes, a
+ // tab in the background. Where it left the walk is behind the clock
+ // now, so the voice re-aims at the clock and gives up the stretch in
+ // between; that stretch was going by either way.
+ this.moved(live);
live.scheduled = Math.max(live.scheduled, tNow);
return;
}
@@ -544,33 +617,86 @@ export class PreviewMixer {
}
live.windows.push({ node, until });
live.scheduled = until;
- live.misses = 0;
+ live.attempts = 0;
} catch {
// Past the read, everything is scheduling: a context closed or swapped
- // under the voice while this window decoded. The source is fine, so the
- // voice spaces out its next attempt and keeps every one of its reads.
- this.backOff(live, live.misses + 1);
+ // under the voice while this group decoded. The source is fine, so the
+ // voice spaces out its next attempt and keeps every one of its tries.
+ this.backOff(live, 1);
} finally {
- live.filling = false;
+ if (live.gen === gen) live.filling = false;
}
}
- /** A read that came back empty or threw. The source may simply not be there
- * yet — a just-imported file still on its way to storage — so the voice
- * waits and asks again, and stays silent only once its tries are spent. */
- private miss(live: LiveVoice): void {
- live.misses += 1;
- if (live.misses >= READ_RETRIES) {
- live.dead = true;
- return;
+ /**
+ * The next `GROUP_S` of a voice's sound, pulled off its walk.
+ *
+ * The walk is opened where the voice stands when it hasn't got one — the
+ * first ask, or the first after something moved it — and kept open after
+ * that, so a clip is one open file, one container parse and one decoder for
+ * as long as it plays.
+ */
+ private async walkGroup(
+ live: LiveVoice,
+ sourceFrom: number,
+ current: () => boolean
+ ): Promise<AudioBuffer | null> {
+ const walk = (live.walk ??= openAudioWalk(live.url, sourceFrom, live.out));
+ const parts: WrappedAudioBuffer[] = [];
+ let end = sourceFrom;
+ // Bounded by the sound it holds and by the reads that go into it: a track
+ // answering with buffers that carry no time would otherwise be pulled for
+ // as long as the frame lasted.
+ for (let pulls = 0; end - sourceFrom < GROUP_S && pulls < GROUP_PULLS; pulls++) {
+ const part = await walk.next();
+ if (!current()) return null;
+ if (!part) break;
+ parts.push(part);
+ end = part.timestamp + part.duration;
}
- this.backOff(live, live.misses);
+ return assembleAudio(parts, sourceFrom, live.out);
+ }
+
+ /**
+ * A window of a voice playing at a speed other than 1, re-laid at the length
+ * the timeline gives it.
+ *
+ * Held briefly, because stepping back over a cut replays the same window and
+ * everything older is a full decode away anyway.
+ */
+ private async stretchedWindow(live: LiveVoice, sourceFrom: number): Promise<AudioBuffer | null> {
+ const sourceTo = Math.min(live.out, sourceFrom + WINDOW_STRETCHED_S * live.speed);
+ const key = `${live.url}|${sourceFrom.toFixed(3)}|${sourceTo.toFixed(3)}|${live.speed}`;
+ const held = this.decoded.get(key);
+ if (held) return held;
+ const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);
+ if (!raw) return null;
+ const buffer = this.stretch(raw, 1 / live.speed);
+ if (this.decoded.size > 24) this.decoded.clear();
+ this.decoded.set(key, buffer);
+ return buffer;
+ }
+
+ /** A read that failed, came back empty, or stopped answering. The voice
+ * gives up where it was, waits, and asks again — and asks for as long as the
+ * play lasts. */
+ private miss(live: LiveVoice): void {
+ this.moved(live);
+ live.attempts += 1;
+ this.backOff(live, live.attempts);
+ // There is no element to reload and nothing else asks on the sound's
+ // behalf, so a signed URL past its window would stay past it. The picture's
+ // readers report their failed reads the same way; the re-mint that follows
+ // swaps the store's URLs and every voice reopens under the new one.
+ void import("./mediaLinks")
+ .then((m) => m.reportMediaUrlError(live.url))
+ .catch(() => {});
}
- /** Hold the voice off until the next attempt, `step` failures in. */
+ /** Hold the voice off until its next try, `step` failures in. */
private backOff(live: LiveVoice, step: number): void {
live.retryAt =
- performance.now() + Math.min(RETRY_MAX_MS, RETRY_BASE_MS * 2 ** Math.max(0, step - 1));
+ performance.now() + (step <= READ_RETRIES ? READ_RETRY_MS * step : READ_RECHECK_MS);
}
/** Re-lay a buffer at a different length, keeping its pitch. */