Open source CapCut alternative.
TypeScript
fix(site): a short sound effect plays in the preview
David Le81d4bb7cd321
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 956b191e..a20e43bf 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -144,6 +144,17 @@ export function engineLog(msg: string): void {* whole life rather than only inside the handover, so the mixer decodes and* schedules one stable voice and the frame-by-frame gain is all that moves.*/+/**+ * How far ahead of the playhead a clip's sound is opened, in timeline seconds.+ *+ * A voice opened the frame its clip begins pays the open — a container parsed,+ * a decoder configured, a first read — while the clip is already playing, and+ * a clip shorter than that open is gone before its sound arrives. The clips+ * about to begin ride along with the audible ones, so the play reaches each+ * one with its file open and its first window scheduled.+ */+const SOUND_LOOKAHEAD_S = 3;+const voiceSpan = (sp: ClipSpan) => {// The handles a crossing reaches into play at the pace of the footage// beside them, so the voice's span widens through the clip's own map and@@ -684,9 +695,50 @@ class Engine {sound: a.sound,});}+ this.upcomingVoices(t, spans, duck, out);return out;}+ /**+ * The clips that begin within `SOUND_LOOKAHEAD_S` of `t`, added to `out` so+ * the mixer opens them before the playhead arrives. Each carries the gain it+ * will have on its first frame; the frames inside the clip take over from+ * there. A clip already voiced — the far side of a crossing — is left as it is.+ */+ private upcomingVoices(t: number, spans: ClipSpan[], duck: number, out: Voice[]) {+ const s = useEditor.getState();+ const have = new Set(out.map((v) => v.id));+ const soon = (start: number) => start > t && start <= t + SOUND_LOOKAHEAD_S;+ for (const spans0 of [spans, ...this.overlaySpans()]) {+ for (const span of spans0) {+ if (!soon(span.start) || have.has(span.clip.id)) continue;+ if (span.clip.muted || span.clip.hidden || assetIsSilent(span.asset)) continue;+ have.add(span.clip.id);+ out.push({+ id: span.clip.id,+ ...voiceSpan(span),+ gain: (span.clip.volume ?? 1) * duck,+ });+ }+ }+ for (const a of s.audioClips) {+ if (!soon(a.start) || have.has(a.id) || a.hidden) continue;+ const asset = s.assets.find((x) => x.id === a.assetId);+ if (!asset || assetIsSilent(asset)) continue;+ const dg = a.duck !== undefined && a.duck < 1 ? 1 : duck;+ out.push({+ id: a.id,+ url: asset.url,+ start: a.start,+ in: a.in,+ out: a.out,+ retime: retimeOf(a),+ gain: (a.fadeIn ?? 0) > 0 ? 0 : a.volume * dg,+ sound: a.sound,+ });+ }+ }+/** Draw track 0 at `t`: the master clip, whatever is blending into it, the* neighbour frame behind an edge animation, and the clip's own veil. */private drawTrackZero(diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.tsindex 31f4599b..f0afea8d 100644--- a/site/src/cut/lib/previewMixer.test.ts+++ b/site/src/cut/lib/previewMixer.test.ts@@ -200,10 +200,12 @@ const assembleAudio = (};const reMints = { asked: 0 };+const decodes = { asked: 0 };/** A span asked for in one go, which the link cannot answer until it has* delivered the file as far as the span's end. */const decodeAudioSpan = async (_url: string, from: number, to: number) => {+ decodes.asked++;await delivered(to);if (wall < failUntil) throw new Error("the link dropped");if (from >= CLIP_S - 1e-6) return null;@@ -268,6 +270,7 @@ function reset(file: { endEarlyAt?: number; trackEnd?: number } = {}): void {waiting.length = 0;walks.opened = 0;reMints.asked = 0;+ decodes.asked = 0;failUntil = 0;hangUntil = 0;openCostS = 0;@@ -461,6 +464,53 @@ describe("the preview's sound over a link that only just keeps up", () => {expect(walks.opened).toBe(1);});+ test("a short sound ahead of the playhead plays whole, and again for free", async () => {+ // The failure this is here for: a click a third of a second long was+ // opened the frame the playhead reached it, and the walk's open outlived+ // the clip, so the voice was released before a window was scheduled and+ // the effect was never heard. The frame plan now hands the mixer the clips+ // about to begin, and a short clip is one decode, kept for the next pass.+ reset();+ headStart = CLIP_S;+ openCostS = 1;+ const click = {+ id: "click",+ url: "click.mp3",+ start: 5,+ in: 0,+ out: 0.3,+ retime: retimeOf({ in: 0, out: 0.3, speed: 1 }),+ gain: 1,+ };+ const passOver = async (mixer: InstanceType<typeof PreviewMixer>) => {+ for (let t = 0; t < 8; ) {+ step();+ t = mixer.now();+ const soon = t >= click.start - 3 && t < click.start + click.out;+ mixer.update(t, soon ? [voice, click] : [voice]);+ await settle();+ }+ };+ const mixer = new PreviewMixer();+ mixer.start(0);+ liveCtx = (mixer as unknown as { ctx: FakeContext }).ctx;+ await passOver(mixer);+ const first = played.filter((p) => p.from >= 4.9 && p.from <= 5.1);+ expect(first.length).toBe(1);+ expect(first[0].to - first[0].from).toBeCloseTo(0.3, 2);+ expect(decodes.asked).toBe(1);+ // Back to the top and over it again: the click plays from what is held.+ mixer.stop();+ played = [];+ ctxTime = 0;+ mixer.start(0);+ liveCtx = (mixer as unknown as { ctx: FakeContext }).ctx;+ await passOver(mixer);+ mixer.dispose();+ expect(played.filter((p) => p.from >= 4.9 && p.from <= 5.1).length).toBe(1);+ expect(decodes.asked).toBe(1);+ });+test("a seek keeps its reader and lands with its sound", async () => {// The failure this is here for: picking a position mid-play re-anchors the// clock, and a voice that gave up its walk there paid a fresh open — adiff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsindex 853eb081..f6497e88 100644--- a/site/src/cut/lib/previewMixer.ts+++ b/site/src/cut/lib/previewMixer.ts@@ -70,13 +70,19 @@ const GROUP_PULLS = 128;* enough that the work lands inside a frame or two. */const WINDOW_STRETCHED_S = 4;/**- * Stretched windows kept in hand.+ * Decoded windows kept in hand: stretched windows, and short clips read whole.** Two dozen four-second windows of stereo at 48kHz — enough that stepping back* over a cut replays what it just played, and bounded because everything older* than that is a full decode away regardless.*/-const STRETCH_CACHE_BYTES = 24 * WINDOW_STRETCHED_S * 48_000 * 2 * 4;+const WINDOW_CACHE_BYTES = 24 * WINDOW_STRETCHED_S * 48_000 * 2 * 4;+/** A clip whose whole source span is at most this long is one read: decoded+ * in full the first time it is heard and held, so the sound effects a cut is+ * dotted with — a click, a whoosh, a hit, each shorter than the open a walk+ * over its file costs — play whole on every pass, and a second pass over the+ * same one costs no read at all. */+const SHORT_CLIP_S = 3;/** What a set of decoded windows costs: float samples, one per channel. *//** A window laid through a clip's map, with the source second its first@@ -86,8 +92,14 @@ interface FittedWindow {/** Seconds past where the window was asked for that its sound begins —* see `FittedSpan.shift`. */shift: number;+ /** Seconds into the buffer the window begins: a whole short clip held from+ * its head, joined part-way through. */+ skip?: number;}+/** A voice read whole — see `SHORT_CLIP_S`. */+const isShort = (live: { in: number; out: number }): boolean => live.out - live.in <= SHORT_CLIP_S;+const decodedBytes = (held: Map<string, FittedWindow>): number => {let n = 0;for (const { buffer: b } of held.values()) n += b.length * b.numberOfChannels * 4;@@ -720,6 +732,8 @@ export class PreviewMixer {// primer, decoded now and held for the play that follows.if (retimeFits(live.retime)) {await this.stretchedWindow(live, source);+ } else if (isShort(live)) {+ await this.shortWindow(live, source);} else {const walk = (live.walk ??= openAudioWalk(live.url, source, live.out));if (Math.abs(walk.position - source) > 0.02) walk.seek(source);@@ -882,6 +896,7 @@ export class PreviewMixer {// the read ran into the end of the source; played at `from` it would// run early for its whole length.let shift = 0;+ let skip = 0;try {// Nothing scheduled means the sound is standing still — the play just// began, a seek landed, a hold let go — and the picture is not@@ -891,6 +906,10 @@ export class PreviewMixer {const fit = await this.stretchedWindow(live, sourceFrom);buffer = fit?.buffer ?? null;shift = Math.max(0, fit?.shift ?? 0);+ } else if (isShort(live)) {+ const fit = await this.shortWindow(live, sourceFrom);+ buffer = fit?.buffer ?? null;+ skip = fit?.skip ?? 0;} else {buffer = await this.walkGroup(live, sourceFrom, current, group);}@@ -925,7 +944,7 @@ export class PreviewMixer {const tNow = this.at();if (!current() || live.scheduled !== from) return;const at2 = from + shift;- const until = at2 + buffer.duration;+ const until = at2 + buffer.duration - skip;if (until <= tNow) {this.noteLate(live, tNow - at2);// The read outlived the moment it was for — a long wait on bytes, a@@ -941,12 +960,12 @@ export class PreviewMixer {const at = this.anchor.ctx + (at2 - this.anchor.timeline);const now = this.ctx.currentTime;if (at >= now) {- node.start(at);+ node.start(at, skip);} else {// Late: start where the clock actually is, so the sound stays lined up// with the picture and skips what has already gone by.this.noteLate(live, now - at);- node.start(now, now - at);+ node.start(now, skip + (now - at));}live.windows.push({ node, until });live.scheduled = until;@@ -1023,20 +1042,41 @@ export class PreviewMixer {if (!raw) return null;// The window is laid by the clip's own map from where it starts, so its// length is exactly the timeline the picture spends on it.- const buffer = this.fit(raw, rt, lo, hi);- // Oldest first until the cache is back inside its share. Emptying the- // whole cache would throw away the window the playhead is standing on along- // with the ones it has finished, and the next frame would pay a full decode- // for sound it was already holding.- this.decoded.set(key, buffer);- const cap = allowance("mixerAudio", STRETCH_CACHE_BYTES);+ return this.keep(key, this.fit(raw, rt, lo, hi));+ }++ /**+ * A short clip's whole sound, decoded once and held.+ *+ * A walk over the file costs an open before its first read answers, and a+ * clip shorter than that open is over before its sound arrives; the clip is+ * one read instead, and the read is kept. Joined part-way through — a play+ * begun inside it — the window starts that far into the buffer.+ */+ private async shortWindow(live: LiveVoice, sourceFrom: number): Promise<FittedWindow | null> {+ const key = `${live.url}|${live.in.toFixed(3)}|${live.out.toFixed(3)}|${live.retime.key}`;+ const skip = Math.max(0, sourceFrom - live.in);+ const held = this.decoded.get(key);+ if (held) return { ...held, skip };+ const raw = await decodeAudioSpan(live.url, live.in, live.out);+ if (!raw) return null;+ return { ...this.keep(key, { buffer: raw, shift: 0 }), skip };+ }++ /** Keep a decoded window, letting the oldest go until the cache is back+ * inside its share. Emptying the whole cache would throw away the window the+ * playhead is standing on along with the ones it has finished, and the next+ * frame would pay a full decode for sound it was already holding. */+ private keep(key: string, window: FittedWindow): FittedWindow {+ this.decoded.set(key, window);+ const cap = allowance("mixerAudio", WINDOW_CACHE_BYTES);let cached = decodedBytes(this.decoded);for (const [oldest, held] of this.decoded) {if (cached <= cap || this.decoded.size <= 1) break;cached -= held.buffer.length * held.buffer.numberOfChannels * 4;this.decoded.delete(oldest);}- return buffer;+ return window;}/**