Open source CapCut alternative.
TypeScript
fix(cut): a voice goes after the sound its walk stopped short of
David Ledae8fc0941f7
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.tsindex 5fa71852..b02f0842 100644--- a/site/src/cut/hooks/usePlayback.ts+++ b/site/src/cut/hooks/usePlayback.ts@@ -140,6 +140,12 @@ const voiceSpan = (sp: ClipSpan) => {class Engine {/** Which engine this is, for the perf eval's pool dump. */readonly serial = ++engineSerial;++ /** The mixer's own account of itself; see `PreviewMixer.soundState`. */+ soundState() {+ return this.mixer.soundState();+ }+private comp: FrameCompositor;private pool = new FrameSourcePool(DECODER_BUDGET, () => this.wake());private mixer = new PreviewMixer();@@ -745,9 +751,16 @@ export function usePlayback(canvasRef: RefObject<HTMLCanvasElement | null>) {if (process.env.NODE_ENV !== "production") {(window as unknown as Record<string, unknown>).__cutDevEngine = engine;}+ // A preview that has gone quiet looks identical from outside whatever the+ // reason, and it goes quiet on machines and links that are not here. This+ // reads back what the sound is doing — the clock, and where each voice has+ // read to — in any build. Numbers only, and only when asked for.+ const win = window as unknown as Record<string, unknown>;+ win.__cutSound = () => engine.soundState();registerSourceSampler((clipId) => engine.sourceFor(clipId));return () => {registerSourceSampler(null);+ delete win.__cutSound;engine.dispose();};}, [canvasRef]);diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.tsindex 73588b89..7911fc4f 100644--- a/site/src/cut/lib/previewMixer.test.ts+++ b/site/src/cut/lib/previewMixer.test.ts@@ -84,6 +84,11 @@ globals.AudioContext = FakeContext;/** Reads that throw, and reads that never answer, over a stretch of wall time. */let failUntil = 0;let hangUntil = 0;+/** A walk that reports itself over before the track really is — bytes that+ * never came, a decoder closed under it. Spent the first time it fires. */+let endEarlyAt = Infinity;+/** Where the track truly stops, which can be short of the clip. */+let trackEnd = CLIP_S;const waiting: { at: number; wake: () => void }[] = [];@@ -99,16 +104,25 @@ const walks = { opened: 0 };const openAudioWalk = (_url: string, from: number, to?: number) => {walks.opened++;const end = to ?? CLIP_S;+ // This walk's own false ending, spent on opening: a walk that gives up stays+ // given up, and the walk sent after it reads the file properly.+ const givesUpAt = endEarlyAt;+ endEarlyAt = Infinity;let pos = from;let closed = false;+ let done = false;return {async next() {- if (closed) return null;+ if (closed || done) return null;if (wall < hangUntil) return new Promise<null>(() => {});await delivered(pos + PART_S);if (closed) return null;if (wall < failUntil) throw new Error("the link dropped");- if (pos >= end - 1e-6) return null;+ if (pos >= givesUpAt) {+ done = true;+ return null;+ }+ if (pos >= Math.min(end, trackEnd) - 1e-6) return null;const duration = Math.min(PART_S, end - pos);const part = {buffer: new AudioBuffer({@@ -203,7 +217,10 @@ const voice = {/** Play the clip through, running `during` on each frame at the timeline time* it is drawing. Answers what was heard. */-async function play(during?: (t: number, ctx: FakeContext) => void): Promise<Played[]> {+async function play(+ during?: (t: number, ctx: FakeContext) => void,+ file: { endEarlyAt?: number; trackEnd?: number } = {}+): Promise<Played[]> {wall = 0;ctxTime = 0;resumes = 0;@@ -213,6 +230,8 @@ async function play(during?: (t: number, ctx: FakeContext) => void): Promise<PlareMints.asked = 0;failUntil = 0;hangUntil = 0;+ endEarlyAt = file.endEarlyAt ?? Infinity;+ trackEnd = file.trackEnd ?? CLIP_S;const mixer = new PreviewMixer();mixer.start(0);try {@@ -275,6 +294,25 @@ describe("the preview's sound over a link that only just keeps up", () => {expect(heard(sound, 50, CLIP_S)).toBeGreaterThan(0.9);});+ test("goes after the sound a walk stopped short of", async () => {+ // The failure this is here for: a walk reports the same ending whether it+ // read the track's last sample or gave up on it, and a voice that believes+ // an ending five seconds into a long clip plays the rest of it in silence.+ const sound = await play(undefined, { endEarlyAt: 5 });+ expect(heard(sound, 0, 5)).toBeGreaterThan(0.9);+ expect(heard(sound, 10, CLIP_S)).toBeGreaterThan(0.9);+ expect(walks.opened).toBeGreaterThan(1); // it went back for the rest+ });++ test("believes an ending two walks agree on", async () => {+ // A track really can stop before the picture does. One walk goes back for+ // what it missed; a second that gets no further has found the end.+ const sound = await play(undefined, { trackEnd: 30 });+ expect(heard(sound, 0, 29)).toBeGreaterThan(0.9);+ expect(heard(sound, 32, CLIP_S)).toBe(0);+ expect(walks.opened).toBeLessThan(4);+ });+test("picks its context back up when the browser puts it down", async () => {const sound = await play((t, ctx) => {if (t >= 20 && t < 20.1) ctx.state = "suspended";diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsindex 6c4b6dd6..d19c2d2b 100644--- a/site/src/cut/lib/previewMixer.ts+++ b/site/src/cut/lib/previewMixer.ts@@ -106,6 +106,9 @@ interface LiveVoice {* than a verdict: anything that moves the voice clears it, so a seek back* into the clip plays again. */ended: boolean;+ /** The furthest point in the source a walk has ended at. A second walk that+ * gets no further has found where the track really stops. */+ reached: number;/** Consecutive failed reads, and the wall time (performance.now) the next* one may be tried at. */attempts: number;@@ -126,6 +129,9 @@ const FILL_STALL_MS = 15_000;const READ_RETRY_MS = 1_000;const READ_RETRIES = 3;const READ_RECHECK_MS = 10_000;+/** How close to the clip's end a walk may stop and be believed. A track+ * routinely runs a fraction short of the picture it came with. */+const END_SLACK_S = 0.25;/** One audio effect element the graph is carrying: its recipe's chain, and* the pair of gains that cross the mix between treated and untreated. */@@ -263,6 +269,22 @@ export class PreviewMixer {return this.heard(graph);}+ /**+ * Where the playhead is, with the clock allowed to re-aim nothing.+ *+ * A read asks with this. Asking with `now()` lets a read that took a moment+ * trip the re-anchor, and re-anchoring disowns every voice — including the+ * one this read was for, whose sound is then thrown away by the act of+ * finding out where to put it.+ */+ private at(): number {+ if (!this.anchor) return 0;+ const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;+ if (!this.ctx || this.ctx.state !== "running") return wall;+ const graph = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);+ return this.heard(Math.abs(graph - wall) > 0.25 ? wall : graph);+ }+/*** How far the picture is being held behind the graph's clock right now, and* what the output device says it holds. Zero when nothing is playing.@@ -481,6 +503,41 @@ export class PreviewMixer {}}+ /**+ * What the sound is doing right now: the clock it runs on and where each+ * voice has read to. Numbers only — no media, and nothing naming a project.+ *+ * A preview that has gone quiet looks the same from outside whatever the+ * reason, and the reasons live in here. Reachable in any build as+ * `__cutSound()`, because the machine a cut goes quiet on is rarely the one+ * a developer is sitting at.+ */+ soundState() {+ const ctx = this.ctx;+ const wall = performance.now();+ return {+ running: this.running,+ ctx: ctx+ ? { state: ctx.state, time: +ctx.currentTime.toFixed(2), ...this.clockLead() }+ : null,+ at: +this.at().toFixed(2),+ voices: [...this.voices].map(([id, v]) => ({+ id,+ url: v.url.slice(0, 60),+ scheduled: +v.scheduled.toFixed(2),+ out: v.out,+ windows: v.windows.length,+ reading: v.filling,+ readingMs: v.filling ? Math.round(wall - v.fillAt) : 0,+ walk: !!v.walk,+ ended: v.ended,+ reached: Number.isFinite(v.reached) ? +v.reached.toFixed(2) : null,+ attempts: v.attempts,+ retryInMs: Math.max(0, Math.round(v.retryAt - wall)),+ })),+ };+ }+/** Drop the decode cache for a source that has been replaced. */forget(url: string): void {for (const key of [...this.decoded.keys()]) {@@ -522,6 +579,7 @@ export class PreviewMixer {fillAt: 0,gen: 0,ended: false,+ reached: -Infinity,attempts: 0,retryAt: 0,};@@ -583,15 +641,27 @@ export class PreviewMixer {}if (!current()) return;if (!buffer) {- // The reader answered with nothing left to play. A track can stop- // before the picture does, and a clip whose file carries no sound at- // all answers this way from the first ask.- live.ended = true;+ // A walk reports the same ending whether it read the track's last+ // sample or gave up on it: bytes that never came, a decoder closed+ // under it, a link that dropped mid-file. Believed, an ending a few+ // seconds into a long clip plays the rest of the cut in silence while+ // the picture reads happily past it — so an ending short of the clip+ // is a failed read, and another walk goes after the sound it did not+ // reach. Two things stop that running away: a walk that gets no+ // further than the one before it has found where the track really+ // stops, and a track that runs a fraction short of its picture is+ // inside the slack.+ if (sourceFrom >= live.out - END_SLACK_S || sourceFrom <= live.reached + 1e-3) {+ live.ended = true;+ } else {+ live.reached = sourceFrom;+ // A source that stopped answering mid-file is the one case a+ // re-mint cannot speak to, so this failure keeps to itself.+ this.miss(live, false);+ }return;}- // Reading the clock can re-aim the voices (`restart`), so it happens- // before the placement below is computed.- const tNow = this.now();+ const tNow = this.at();if (!current() || live.scheduled !== from) return;const until = from + buffer.duration;if (until <= tNow) {@@ -680,10 +750,11 @@ export class PreviewMixer {/** A read that failed, came back empty, or stopped answering. The voice* gives up where it was, waits, and asks again — and asks for as long as the* play lasts. */- private miss(live: LiveVoice): void {+ private miss(live: LiveVoice, report = true): void {this.moved(live);live.attempts += 1;this.backOff(live, live.attempts);+ if (!report) return;// There is no element to reload and nothing else asks on the sound's// behalf, so a signed URL past its window would stay past it. The picture's// readers report their failed reads the same way; the re-mint that follows