Open source CapCut alternative.
TypeScript
fix(cut): start a play on the sound's clock
David Le264cb83ed990
diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.tsindex 293ce036..7e942232 100644--- a/site/src/cut/lib/previewMixer.test.ts+++ b/site/src/cut/lib/previewMixer.test.ts@@ -80,11 +80,18 @@ class FakeSource {* an output device being switched, a tab the browser has put down. */let resumeBlockedUntil = 0;let liveCtx: FakeContext | null = null;+/** The output device behind a new context: until the wall reaches this the+ * context says it is running while its clock stands at zero and it reports no+ * output latency — Chrome on a Mac, a quarter second after the press. */+let deviceOpensAt = 0;+let deviceLatency = 0;class FakeContext {state = "running";baseLatency = 0;- outputLatency = 0;+ get outputLatency() {+ return wall >= deviceOpensAt ? deviceLatency : 0;+ }destination = {};get currentTime() {return ctxTime;@@ -253,7 +260,7 @@ function step(): void {wall += FRAME_S;// The graph's clock is the context's, and a context that is not running does// not keep one.- if (liveCtx?.state === "running") ctxTime += FRAME_S;+ if (liveCtx?.state === "running" && wall >= deviceOpensAt) ctxTime += FRAME_S;for (const w of waiting.splice(0)) {if (w.at <= wall) w.wake();else waiting.push(w);@@ -284,6 +291,8 @@ function reset(file: { endEarlyAt?: number; trackEnd?: number } = {}): void {hangUntil = 0;openCostS = 0;resumeBlockedUntil = 0;+ deviceOpensAt = 0;+ deviceLatency = 0;headStart = HEAD_START_S;liveCtx = null;endEarlyAt = file.endEarlyAt ?? Infinity;@@ -473,6 +482,34 @@ describe("the preview's sound over a link that only just keeps up", () => {expect(walks.opened).toBe(1);});+ test("a play's clock waits for its device and never steps back", async () => {+ // The failure this is here for: a first play created the context, whose+ // clock stood at zero while the output device opened. The wall ran on, the+ // standstill read as a clock jump, and the playhead was re-anchored a+ // quarter second ahead of a sound that had not started. Once the device+ // reported its latency the picture stepped back by that much.+ reset();+ deviceOpensAt = 0.4;+ deviceLatency = 0.2;+ const mixer = new PreviewMixer();+ mixer.start(0);+ liveCtx = (mixer as unknown as { ctx: FakeContext }).ctx;+ const times: number[] = [];+ for (let i = 0; i < 40; i++) {+ step();+ const t = mixer.now();+ times.push(t);+ // The sound starts on the context's clock, and leaves the speakers the+ // device's latency after that.+ expect(t).toBeLessThanOrEqual(Math.max(0, ctxTime - liveCtx.outputLatency) + 1e-9);+ mixer.update(t, [voice]);+ await settle();+ }+ mixer.dispose();+ for (let i = 1; i < times.length; i++) expect(times[i]).toBeGreaterThanOrEqual(times[i - 1]);+ expect(times[times.length - 1]).toBeGreaterThan(1);+ });+test("a short sound ahead of the playhead plays whole, and again for free", async () => {// The failure this is here for: a click a third of a second long was// opened the frame the playhead reached it, and the walk's open outliveddiff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsindex bd0730db..b6f8d1a7 100644--- a/site/src/cut/lib/previewMixer.ts+++ b/site/src/cut/lib/previewMixer.ts@@ -115,6 +115,13 @@ const AHEAD_S = 5;* device claims. Well past a Bluetooth headset, and short of anything that* would read as the playhead refusing to move. */const MAX_LATENCY_S = 0.4;+/** How long a play follows the sound's clock alone before the wall may carry+ * it. A new context's device opens after the context does: Chrome on a Mac+ * reports `running` at once, holds `currentTime` at zero for a quarter second+ * and reports no output latency until the device is open. The wall running on+ * through that read as a clock jump, put the playhead ahead of a sound that+ * had not started, and stepped it back once the latency arrived. */+const START_WAIT_S = 1;/** Sound late by more than this, mid-play, is a hole the listener hears: the* scheduling runs seconds ahead, so a window that misses its moment means the* thread was gone for that long. A voice's first window is a different thing@@ -268,6 +275,12 @@ export class PreviewMixer {/** Timeline time, context time and wall time of the same instant. Everything* scheduled is placed against this. */private anchor: { timeline: number; ctx: number; wall: number } | null = null;+ /** Wall time (performance.now) the play began, for `START_WAIT_S`. */+ private startedAt = 0;+ /** The latest time `now()` has answered this play, so it never answers+ * less: a device reporting more latency mid-play holds the picture until+ * the sound reaches it. */+ private shown = 0;private decoded = new Map<string, FittedWindow>();private readonly releaseMemory = holdMemory("mixerAudio", () => decodedBytes(this.decoded));@@ -338,6 +351,7 @@ export class PreviewMixer {if (!this.anchor) return 0;// A hold is the clock standing still on purpose; nothing about it is drift.if (this.holding) return this.heldAt;+ if (this.starting()) return this.show(this.heard(this.followSound()));const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;// A browser holds an `AudioContext` suspended until the page has been// interacted with, and a suspended context's clock does not move. Reading@@ -346,7 +360,7 @@ export class PreviewMixer {// the playhead by however long it stayed asleep. So the wall carries the cut// until the audio clock agrees with it, and a clock that disagrees is// re-anchored rather than believed.- if (!this.ctx || this.ctx.state !== "running") return wall;+ if (!this.ctx || this.ctx.state !== "running") return this.show(wall);const graph = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);// Measured on the graph's clock, which is the one the anchor was taken on.// Comparing the heard clock against the wall would read the device's own@@ -363,9 +377,35 @@ export class PreviewMixer {// where the clock now stands.meterClockJump();this.restart(wall);- return this.heard(wall);+ return this.show(this.heard(wall));}- return this.heard(graph);+ return this.show(this.heard(graph));+ }++ /** Whether the play is inside its `START_WAIT_S`. */+ private starting(): boolean {+ return performance.now() - this.startedAt < START_WAIT_S * 1000;+ }++ /**+ * The graph's clock, with the wall kept level with it.+ *+ * While the device opens the graph's clock stands still and so does the+ * playhead, at the play position. The wall is pulled along so that once the+ * wait is over it carries on from the same moment: a context that never+ * starts — a page nobody has touched — plays on in silence from there.+ */+ private followSound(): number {+ const anchor = this.anchor!;+ const graph = anchor.timeline + (this.ctx!.currentTime - anchor.ctx);+ anchor.wall = performance.now() - (graph - anchor.timeline) * 1000;+ return graph;+ }++ /** `t`, or the latest time already answered if that is later. */+ private show(t: number): number {+ this.shown = Math.max(this.shown, t);+ return this.shown;}/**@@ -378,10 +418,11 @@ export class PreviewMixer {*/private at(): number {if (!this.anchor) return 0;+ if (this.starting()) return Math.max(this.shown, this.heard(this.followSound()));const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;- if (!this.ctx || this.ctx.state !== "running") return wall;+ if (!this.ctx || this.ctx.state !== "running") return Math.max(this.shown, wall);const graph = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);- return this.heard(Math.abs(graph - wall) > 0.25 ? wall : graph);+ return Math.max(this.shown, this.heard(Math.abs(graph - wall) > 0.25 ? wall : graph));}/**@@ -462,6 +503,8 @@ export class PreviewMixer {void ctx.resume();this.restart(t);this.anchor = { timeline: t, ctx: ctx.currentTime, wall: performance.now() };+ this.startedAt = this.anchor.wall;+ this.shown = t;}/**