Skip to content

Open source CapCut alternative.

TypeScript

refactor(site): the sound's readers survive a clock jump

David Le7dba03639b35
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index b02f0842..17e4400d 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -141,11 +141,6 @@ class Engine {
/** Which engine this is, for the perf eval's pool dump. */
readonly serial = ++engineSerial;
- /** The mixer's own account of itself; see `PreviewMixer.soundState`. */
- soundState() {
- return this.mixer.soundState();
- }
-
private comp: FrameCompositor;
private pool = new FrameSourcePool(DECODER_BUDGET, () => this.wake());
private mixer = new PreviewMixer();
@@ -751,16 +746,9 @@ export function usePlayback(canvasRef: RefObject<HTMLCanvasElement | null>) {
if (process.env.NODE_ENV !== "production") {
(window as unknown as Record<string, unknown>).__cutDevEngine = engine;
}
- // A preview that has gone quiet looks identical from outside whatever the
- // reason, and it goes quiet on machines and links that are not here. This
- // reads back what the sound is doing — the clock, and where each voice has
- // read to — in any build. Numbers only, and only when asked for.
- const win = window as unknown as Record<string, unknown>;
- win.__cutSound = () => engine.soundState();
registerSourceSampler((clipId) => engine.sourceFor(clipId));
return () => {
registerSourceSampler(null);
- delete win.__cutSound;
engine.dispose();
};
}, [canvasRef]);
diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.ts
index bbcf7621..e9edb81d 100644
--- a/site/src/cut/lib/previewMixer.test.ts
+++ b/site/src/cut/lib/previewMixer.test.ts
@@ -64,6 +64,11 @@ class FakeSource {
}
}
+/** While the wall is short of this, a suspended context refuses to come back —
+ * an output device being switched, a tab the browser has put down. */
+let resumeBlockedUntil = 0;
+let liveCtx: FakeContext | null = null;
+
class FakeContext {
state = "running";
baseLatency = 0;
@@ -80,7 +85,7 @@ class FakeContext {
}
resume() {
resumes++;
- this.state = "running";
+ if (wall >= resumeBlockedUntil) this.state = "running";
return Promise.resolve();
}
close() {
@@ -229,7 +234,9 @@ async function settle(): Promise<void> {
function step(): void {
wall += FRAME_S;
- ctxTime += FRAME_S;
+ // The graph's clock is the context's, and a context that is not running does
+ // not keep one.
+ if (liveCtx?.state === "running") ctxTime += FRAME_S;
for (const w of waiting.splice(0)) {
if (w.at <= wall) w.wake();
else waiting.push(w);
@@ -262,11 +269,14 @@ async function play(
failUntil = 0;
hangUntil = 0;
openCostS = 0;
+ resumeBlockedUntil = 0;
+ liveCtx = null;
endEarlyAt = file.endEarlyAt ?? Infinity;
trackEnd = file.trackEnd ?? CLIP_S;
voice.url = "clip.mp4";
const mixer = new PreviewMixer();
mixer.start(0);
+ liveCtx = (mixer as unknown as { ctx: FakeContext }).ctx;
try {
for (let t = 0; t < CLIP_S; ) {
step();
@@ -385,6 +395,25 @@ describe("the preview's sound over a link that only just keeps up", () => {
expect(walks.opened).toBe(2);
});
+ test("keeps its readers when the clock jumps", async () => {
+ // A context that sat suspended — an output device switched, a tab the
+ // browser put down — comes back with its clock seconds behind the wall,
+ // and every voice is re-aimed at where the playhead now is. They are in
+ // the same files they were reading a moment ago, so they keep them.
+ const sound = await play((t, ctx) => {
+ if (t < 20 || t > 20.1) return;
+ ctx.state = "suspended";
+ resumeBlockedUntil = wall + 5;
+ });
+ expect(heard(sound, 0, 18)).toBeGreaterThan(0.9);
+ // The context's clock stood still while it was down, so what is heard from
+ // here on runs that much behind the timeline; it is unbroken either way.
+ expect(heard(sound, 28, 60)).toBeGreaterThan(0.95);
+ // One file, one container parse: the jump moved the reader, it did not
+ // cost a new one.
+ expect(walks.opened).toBe(1);
+ });
+
test("picks its context back up when the browser puts it down", async () => {
const sound = await play((t, ctx) => {
if (t >= 20 && t < 20.1) ctx.state = "suspended";
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
index d1f35289..a4d78397 100644
--- a/site/src/cut/lib/previewMixer.ts
+++ b/site/src/cut/lib/previewMixer.ts
@@ -92,8 +92,8 @@ interface LiveVoice {
/** Timeline time everything scheduled so far runs out at. */
scheduled: number;
/** The reader walking this voice's source, open from where it last read to.
- * A walk is a position and nothing more, so anything that moves the voice —
- * a seek, a clock jump, a read that failed — drops it. */
+ * A voice that moves inside the same file re-aims it; a read that failed, or
+ * a source that moved to another address, drops it. */
walk: AudioWalk | null;
/** A read in flight, so a slow one doesn't start a second one, and when it
* started, so one that has stopped answering can be disowned. */
@@ -303,12 +303,16 @@ export class PreviewMixer {
return { lead: Math.max(0, graph - this.heard(graph)), reported };
}
- /** Stop every scheduled window and re-aim each voice at `timeline`. */
+ /** Stop every scheduled window and re-aim each voice at `timeline`, keeping
+ * the file each one is reading: a clock that jumped moved the voices inside
+ * the same sources. */
private restart(timeline: number): void {
for (const live of this.voices.values()) {
this.stopWindows(live);
- this.moved(live);
- live.scheduled = Math.max(live.start, timeline);
+ this.reaim(live, Math.max(live.start, timeline));
+ // The voice is somewhere else in its file now, so where a walk stopped
+ // at the old position says nothing about this one.
+ live.reached = -Infinity;
}
}
@@ -328,11 +332,12 @@ export class PreviewMixer {
/**
* The voice is no longer reading where it was.
*
- * A seek, a clock jump, a read that failed or stopped answering — each of
- * them leaves the walk pointing at a moment nobody is waiting for, and a walk
- * is only a position, so it goes. The read still in flight against it is
- * disowned in the same breath: it carries the old number and everything it
- * comes back with is ignored.
+ * This is the one that gives the file up, for the cases where the file is
+ * the problem or is no longer the right one: a read that failed or stopped
+ * answering, a voice released, a source that moved to another address. A
+ * voice that has merely fallen behind keeps its reader and is re-aimed. The
+ * read still in flight is disowned in the same breath: it carries the old
+ * number and everything it comes back with is ignored.
*/
private moved(live: LiveVoice): void {
live.gen++;
@@ -517,43 +522,8 @@ export class PreviewMixer {
}
}
- /**
- * What the sound is doing right now: the clock it runs on and where each
- * voice has read to. Numbers only — no media, and nothing naming a project.
- *
- * A preview that has gone quiet looks the same from outside whatever the
- * reason, and the reasons live in here. Reachable in any build as
- * `__cutSound()`, because the machine a cut goes quiet on is rarely the one
- * a developer is sitting at.
- */
- soundState() {
- const ctx = this.ctx;
- const wall = performance.now();
- return {
- running: this.running,
- ctx: ctx
- ? { state: ctx.state, time: +ctx.currentTime.toFixed(2), ...this.clockLead() }
- : null,
- at: +this.at().toFixed(2),
- voices: [...this.voices].map(([id, v]) => ({
- id,
- url: v.url.slice(0, 60),
- scheduled: +v.scheduled.toFixed(2),
- out: v.out,
- windows: v.windows.length,
- reading: v.filling,
- readingMs: v.filling ? Math.round(wall - v.fillAt) : 0,
- walk: !!v.walk,
- ended: v.ended,
- reached: Number.isFinite(v.reached) ? +v.reached.toFixed(2) : null,
- attempts: v.attempts,
- retryInMs: Math.max(0, Math.round(v.retryAt - wall)),
- })),
- };
- }
-
/** Drop the decode cache for a source that has been replaced. */
- forget(url: string): void {
+ private forget(url: string): void {
for (const key of [...this.decoded.keys()]) {
if (key.startsWith(`${url}|`)) this.decoded.delete(key);
}
@@ -683,7 +653,7 @@ export class PreviewMixer {
// tab in the background. Where it left the walk is behind the clock
// now, so the voice re-aims at the clock and gives up the stretch in
// between; that stretch was going by either way.
- this.reaim(live, tNow);
+ this.reaim(live, Math.max(live.scheduled, tNow));
return;
}
const node = this.ctx.createBufferSource();
@@ -778,12 +748,12 @@ export class PreviewMixer {
* out in silence while the picture rolls on. The reader moves instead: same
* file, same decoder, aimed where the sound is wanted.
*/
- private reaim(live: LiveVoice, t: number): void {
+ private reaim(live: LiveVoice, timeline: number): void {
live.gen++;
live.filling = false;
live.ended = false;
- live.scheduled = Math.max(live.scheduled, t);
- live.walk?.seek(live.in + (live.scheduled - live.start) * live.speed);
+ live.scheduled = timeline;
+ live.walk?.seek(live.in + (timeline - live.start) * live.speed);
}
/** A read that failed, came back empty, or stopped answering. The voice