Skip to content

Open source CapCut alternative.

TypeScript

fix(site): a short sound effect plays in the preview

David Le81d4bb7cd321
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index 956b191e..a20e43bf 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -144,6 +144,17 @@ export function engineLog(msg: string): void {
* whole life rather than only inside the handover, so the mixer decodes and
* schedules one stable voice and the frame-by-frame gain is all that moves.
*/
+/**
+ * How far ahead of the playhead a clip's sound is opened, in timeline seconds.
+ *
+ * A voice opened the frame its clip begins pays the open — a container parsed,
+ * a decoder configured, a first read — while the clip is already playing, and
+ * a clip shorter than that open is gone before its sound arrives. The clips
+ * about to begin ride along with the audible ones, so the play reaches each
+ * one with its file open and its first window scheduled.
+ */
+const SOUND_LOOKAHEAD_S = 3;
+
const voiceSpan = (sp: ClipSpan) => {
// The handles a crossing reaches into play at the pace of the footage
// beside them, so the voice's span widens through the clip's own map and
@@ -684,9 +695,50 @@ class Engine {
sound: a.sound,
});
}
+ this.upcomingVoices(t, spans, duck, out);
return out;
}
+ /**
+ * The clips that begin within `SOUND_LOOKAHEAD_S` of `t`, added to `out` so
+ * the mixer opens them before the playhead arrives. Each carries the gain it
+ * will have on its first frame; the frames inside the clip take over from
+ * there. A clip already voiced — the far side of a crossing — is left as it is.
+ */
+ private upcomingVoices(t: number, spans: ClipSpan[], duck: number, out: Voice[]) {
+ const s = useEditor.getState();
+ const have = new Set(out.map((v) => v.id));
+ const soon = (start: number) => start > t && start <= t + SOUND_LOOKAHEAD_S;
+ for (const spans0 of [spans, ...this.overlaySpans()]) {
+ for (const span of spans0) {
+ if (!soon(span.start) || have.has(span.clip.id)) continue;
+ if (span.clip.muted || span.clip.hidden || assetIsSilent(span.asset)) continue;
+ have.add(span.clip.id);
+ out.push({
+ id: span.clip.id,
+ ...voiceSpan(span),
+ gain: (span.clip.volume ?? 1) * duck,
+ });
+ }
+ }
+ for (const a of s.audioClips) {
+ if (!soon(a.start) || have.has(a.id) || a.hidden) continue;
+ const asset = s.assets.find((x) => x.id === a.assetId);
+ if (!asset || assetIsSilent(asset)) continue;
+ const dg = a.duck !== undefined && a.duck < 1 ? 1 : duck;
+ out.push({
+ id: a.id,
+ url: asset.url,
+ start: a.start,
+ in: a.in,
+ out: a.out,
+ retime: retimeOf(a),
+ gain: (a.fadeIn ?? 0) > 0 ? 0 : a.volume * dg,
+ sound: a.sound,
+ });
+ }
+ }
+
/** Draw track 0 at `t`: the master clip, whatever is blending into it, the
* neighbour frame behind an edge animation, and the clip's own veil. */
private drawTrackZero(
diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.ts
index 31f4599b..f0afea8d 100644
--- a/site/src/cut/lib/previewMixer.test.ts
+++ b/site/src/cut/lib/previewMixer.test.ts
@@ -200,10 +200,12 @@ const assembleAudio = (
};
const reMints = { asked: 0 };
+const decodes = { asked: 0 };
/** A span asked for in one go, which the link cannot answer until it has
* delivered the file as far as the span's end. */
const decodeAudioSpan = async (_url: string, from: number, to: number) => {
+ decodes.asked++;
await delivered(to);
if (wall < failUntil) throw new Error("the link dropped");
if (from >= CLIP_S - 1e-6) return null;
@@ -268,6 +270,7 @@ function reset(file: { endEarlyAt?: number; trackEnd?: number } = {}): void {
waiting.length = 0;
walks.opened = 0;
reMints.asked = 0;
+ decodes.asked = 0;
failUntil = 0;
hangUntil = 0;
openCostS = 0;
@@ -461,6 +464,53 @@ describe("the preview's sound over a link that only just keeps up", () => {
expect(walks.opened).toBe(1);
});
+ test("a short sound ahead of the playhead plays whole, and again for free", async () => {
+ // The failure this is here for: a click a third of a second long was
+ // opened the frame the playhead reached it, and the walk's open outlived
+ // the clip, so the voice was released before a window was scheduled and
+ // the effect was never heard. The frame plan now hands the mixer the clips
+ // about to begin, and a short clip is one decode, kept for the next pass.
+ reset();
+ headStart = CLIP_S;
+ openCostS = 1;
+ const click = {
+ id: "click",
+ url: "click.mp3",
+ start: 5,
+ in: 0,
+ out: 0.3,
+ retime: retimeOf({ in: 0, out: 0.3, speed: 1 }),
+ gain: 1,
+ };
+ const passOver = async (mixer: InstanceType<typeof PreviewMixer>) => {
+ for (let t = 0; t < 8; ) {
+ step();
+ t = mixer.now();
+ const soon = t >= click.start - 3 && t < click.start + click.out;
+ mixer.update(t, soon ? [voice, click] : [voice]);
+ await settle();
+ }
+ };
+ const mixer = new PreviewMixer();
+ mixer.start(0);
+ liveCtx = (mixer as unknown as { ctx: FakeContext }).ctx;
+ await passOver(mixer);
+ const first = played.filter((p) => p.from >= 4.9 && p.from <= 5.1);
+ expect(first.length).toBe(1);
+ expect(first[0].to - first[0].from).toBeCloseTo(0.3, 2);
+ expect(decodes.asked).toBe(1);
+ // Back to the top and over it again: the click plays from what is held.
+ mixer.stop();
+ played = [];
+ ctxTime = 0;
+ mixer.start(0);
+ liveCtx = (mixer as unknown as { ctx: FakeContext }).ctx;
+ await passOver(mixer);
+ mixer.dispose();
+ expect(played.filter((p) => p.from >= 4.9 && p.from <= 5.1).length).toBe(1);
+ expect(decodes.asked).toBe(1);
+ });
+
test("a seek keeps its reader and lands with its sound", async () => {
// The failure this is here for: picking a position mid-play re-anchors the
// clock, and a voice that gave up its walk there paid a fresh open — a
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
index 853eb081..f6497e88 100644
--- a/site/src/cut/lib/previewMixer.ts
+++ b/site/src/cut/lib/previewMixer.ts
@@ -70,13 +70,19 @@ const GROUP_PULLS = 128;
* enough that the work lands inside a frame or two. */
const WINDOW_STRETCHED_S = 4;
/**
- * Stretched windows kept in hand.
+ * Decoded windows kept in hand: stretched windows, and short clips read whole.
*
* Two dozen four-second windows of stereo at 48kHz — enough that stepping back
* over a cut replays what it just played, and bounded because everything older
* than that is a full decode away regardless.
*/
-const STRETCH_CACHE_BYTES = 24 * WINDOW_STRETCHED_S * 48_000 * 2 * 4;
+const WINDOW_CACHE_BYTES = 24 * WINDOW_STRETCHED_S * 48_000 * 2 * 4;
+/** A clip whose whole source span is at most this long is one read: decoded
+ * in full the first time it is heard and held, so the sound effects a cut is
+ * dotted with — a click, a whoosh, a hit, each shorter than the open a walk
+ * over its file costs — play whole on every pass, and a second pass over the
+ * same one costs no read at all. */
+const SHORT_CLIP_S = 3;
/** What a set of decoded windows costs: float samples, one per channel. */
/** A window laid through a clip's map, with the source second its first
@@ -86,8 +92,14 @@ interface FittedWindow {
/** Seconds past where the window was asked for that its sound begins —
* see `FittedSpan.shift`. */
shift: number;
+ /** Seconds into the buffer the window begins: a whole short clip held from
+ * its head, joined part-way through. */
+ skip?: number;
}
+/** A voice read whole — see `SHORT_CLIP_S`. */
+const isShort = (live: { in: number; out: number }): boolean => live.out - live.in <= SHORT_CLIP_S;
+
const decodedBytes = (held: Map<string, FittedWindow>): number => {
let n = 0;
for (const { buffer: b } of held.values()) n += b.length * b.numberOfChannels * 4;
@@ -720,6 +732,8 @@ export class PreviewMixer {
// primer, decoded now and held for the play that follows.
if (retimeFits(live.retime)) {
await this.stretchedWindow(live, source);
+ } else if (isShort(live)) {
+ await this.shortWindow(live, source);
} else {
const walk = (live.walk ??= openAudioWalk(live.url, source, live.out));
if (Math.abs(walk.position - source) > 0.02) walk.seek(source);
@@ -882,6 +896,7 @@ export class PreviewMixer {
// the read ran into the end of the source; played at `from` it would
// run early for its whole length.
let shift = 0;
+ let skip = 0;
try {
// Nothing scheduled means the sound is standing still — the play just
// began, a seek landed, a hold let go — and the picture is not
@@ -891,6 +906,10 @@ export class PreviewMixer {
const fit = await this.stretchedWindow(live, sourceFrom);
buffer = fit?.buffer ?? null;
shift = Math.max(0, fit?.shift ?? 0);
+ } else if (isShort(live)) {
+ const fit = await this.shortWindow(live, sourceFrom);
+ buffer = fit?.buffer ?? null;
+ skip = fit?.skip ?? 0;
} else {
buffer = await this.walkGroup(live, sourceFrom, current, group);
}
@@ -925,7 +944,7 @@ export class PreviewMixer {
const tNow = this.at();
if (!current() || live.scheduled !== from) return;
const at2 = from + shift;
- const until = at2 + buffer.duration;
+ const until = at2 + buffer.duration - skip;
if (until <= tNow) {
this.noteLate(live, tNow - at2);
// The read outlived the moment it was for — a long wait on bytes, a
@@ -941,12 +960,12 @@ export class PreviewMixer {
const at = this.anchor.ctx + (at2 - this.anchor.timeline);
const now = this.ctx.currentTime;
if (at >= now) {
- node.start(at);
+ node.start(at, skip);
} else {
// Late: start where the clock actually is, so the sound stays lined up
// with the picture and skips what has already gone by.
this.noteLate(live, now - at);
- node.start(now, now - at);
+ node.start(now, skip + (now - at));
}
live.windows.push({ node, until });
live.scheduled = until;
@@ -1023,20 +1042,41 @@ export class PreviewMixer {
if (!raw) return null;
// The window is laid by the clip's own map from where it starts, so its
// length is exactly the timeline the picture spends on it.
- const buffer = this.fit(raw, rt, lo, hi);
- // Oldest first until the cache is back inside its share. Emptying the
- // whole cache would throw away the window the playhead is standing on along
- // with the ones it has finished, and the next frame would pay a full decode
- // for sound it was already holding.
- this.decoded.set(key, buffer);
- const cap = allowance("mixerAudio", STRETCH_CACHE_BYTES);
+ return this.keep(key, this.fit(raw, rt, lo, hi));
+ }
+
+ /**
+ * A short clip's whole sound, decoded once and held.
+ *
+ * A walk over the file costs an open before its first read answers, and a
+ * clip shorter than that open is over before its sound arrives; the clip is
+ * one read instead, and the read is kept. Joined part-way through — a play
+ * begun inside it — the window starts that far into the buffer.
+ */
+ private async shortWindow(live: LiveVoice, sourceFrom: number): Promise<FittedWindow | null> {
+ const key = `${live.url}|${live.in.toFixed(3)}|${live.out.toFixed(3)}|${live.retime.key}`;
+ const skip = Math.max(0, sourceFrom - live.in);
+ const held = this.decoded.get(key);
+ if (held) return { ...held, skip };
+ const raw = await decodeAudioSpan(live.url, live.in, live.out);
+ if (!raw) return null;
+ return { ...this.keep(key, { buffer: raw, shift: 0 }), skip };
+ }
+
+ /** Keep a decoded window, letting the oldest go until the cache is back
+ * inside its share. Emptying the whole cache would throw away the window the
+ * playhead is standing on along with the ones it has finished, and the next
+ * frame would pay a full decode for sound it was already holding. */
+ private keep(key: string, window: FittedWindow): FittedWindow {
+ this.decoded.set(key, window);
+ const cap = allowance("mixerAudio", WINDOW_CACHE_BYTES);
let cached = decodedBytes(this.decoded);
for (const [oldest, held] of this.decoded) {
if (cached <= cap || this.decoded.size <= 1) break;
cached -= held.buffer.length * held.buffer.numberOfChannels * 4;
this.decoded.delete(oldest);
}
- return buffer;
+ return window;
}
/**