Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): a voice goes after the sound its walk stopped short of

David Ledae8fc0941f7
diff --git a/site/src/cut/hooks/usePlayback.ts b/site/src/cut/hooks/usePlayback.ts
index 5fa71852..b02f0842 100644
--- a/site/src/cut/hooks/usePlayback.ts
+++ b/site/src/cut/hooks/usePlayback.ts
@@ -140,6 +140,12 @@ const voiceSpan = (sp: ClipSpan) => {
class Engine {
/** Which engine this is, for the perf eval's pool dump. */
readonly serial = ++engineSerial;
+
+ /** The mixer's own account of itself; see `PreviewMixer.soundState`. */
+ soundState() {
+ return this.mixer.soundState();
+ }
+
private comp: FrameCompositor;
private pool = new FrameSourcePool(DECODER_BUDGET, () => this.wake());
private mixer = new PreviewMixer();
@@ -745,9 +751,16 @@ export function usePlayback(canvasRef: RefObject<HTMLCanvasElement | null>) {
if (process.env.NODE_ENV !== "production") {
(window as unknown as Record<string, unknown>).__cutDevEngine = engine;
}
+ // A preview that has gone quiet looks identical from outside whatever the
+ // reason, and it goes quiet on machines and links that are not here. This
+ // reads back what the sound is doing — the clock, and where each voice has
+ // read to — in any build. Numbers only, and only when asked for.
+ const win = window as unknown as Record<string, unknown>;
+ win.__cutSound = () => engine.soundState();
registerSourceSampler((clipId) => engine.sourceFor(clipId));
return () => {
registerSourceSampler(null);
+ delete win.__cutSound;
engine.dispose();
};
}, [canvasRef]);
diff --git a/site/src/cut/lib/previewMixer.test.ts b/site/src/cut/lib/previewMixer.test.ts
index 73588b89..7911fc4f 100644
--- a/site/src/cut/lib/previewMixer.test.ts
+++ b/site/src/cut/lib/previewMixer.test.ts
@@ -84,6 +84,11 @@ globals.AudioContext = FakeContext;
/** Reads that throw, and reads that never answer, over a stretch of wall time. */
let failUntil = 0;
let hangUntil = 0;
+/** A walk that reports itself over before the track really is — bytes that
+ * never came, a decoder closed under it. Spent the first time it fires. */
+let endEarlyAt = Infinity;
+/** Where the track truly stops, which can be short of the clip. */
+let trackEnd = CLIP_S;
const waiting: { at: number; wake: () => void }[] = [];
@@ -99,16 +104,25 @@ const walks = { opened: 0 };
const openAudioWalk = (_url: string, from: number, to?: number) => {
walks.opened++;
const end = to ?? CLIP_S;
+ // This walk's own false ending, spent on opening: a walk that gives up stays
+ // given up, and the walk sent after it reads the file properly.
+ const givesUpAt = endEarlyAt;
+ endEarlyAt = Infinity;
let pos = from;
let closed = false;
+ let done = false;
return {
async next() {
- if (closed) return null;
+ if (closed || done) return null;
if (wall < hangUntil) return new Promise<null>(() => {});
await delivered(pos + PART_S);
if (closed) return null;
if (wall < failUntil) throw new Error("the link dropped");
- if (pos >= end - 1e-6) return null;
+ if (pos >= givesUpAt) {
+ done = true;
+ return null;
+ }
+ if (pos >= Math.min(end, trackEnd) - 1e-6) return null;
const duration = Math.min(PART_S, end - pos);
const part = {
buffer: new AudioBuffer({
@@ -203,7 +217,10 @@ const voice = {
/** Play the clip through, running `during` on each frame at the timeline time
* it is drawing. Answers what was heard. */
-async function play(during?: (t: number, ctx: FakeContext) => void): Promise<Played[]> {
+async function play(
+ during?: (t: number, ctx: FakeContext) => void,
+ file: { endEarlyAt?: number; trackEnd?: number } = {}
+): Promise<Played[]> {
wall = 0;
ctxTime = 0;
resumes = 0;
@@ -213,6 +230,8 @@ async function play(during?: (t: number, ctx: FakeContext) => void): Promise<Pla
reMints.asked = 0;
failUntil = 0;
hangUntil = 0;
+ endEarlyAt = file.endEarlyAt ?? Infinity;
+ trackEnd = file.trackEnd ?? CLIP_S;
const mixer = new PreviewMixer();
mixer.start(0);
try {
@@ -275,6 +294,25 @@ describe("the preview's sound over a link that only just keeps up", () => {
expect(heard(sound, 50, CLIP_S)).toBeGreaterThan(0.9);
});
+ test("goes after the sound a walk stopped short of", async () => {
+ // The failure this is here for: a walk reports the same ending whether it
+ // read the track's last sample or gave up on it, and a voice that believes
+ // an ending five seconds into a long clip plays the rest of it in silence.
+ const sound = await play(undefined, { endEarlyAt: 5 });
+ expect(heard(sound, 0, 5)).toBeGreaterThan(0.9);
+ expect(heard(sound, 10, CLIP_S)).toBeGreaterThan(0.9);
+ expect(walks.opened).toBeGreaterThan(1); // it went back for the rest
+ });
+
+ test("believes an ending two walks agree on", async () => {
+ // A track really can stop before the picture does. One walk goes back for
+ // what it missed; a second that gets no further has found the end.
+ const sound = await play(undefined, { trackEnd: 30 });
+ expect(heard(sound, 0, 29)).toBeGreaterThan(0.9);
+ expect(heard(sound, 32, CLIP_S)).toBe(0);
+ expect(walks.opened).toBeLessThan(4);
+ });
+
test("picks its context back up when the browser puts it down", async () => {
const sound = await play((t, ctx) => {
if (t >= 20 && t < 20.1) ctx.state = "suspended";
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
index 6c4b6dd6..d19c2d2b 100644
--- a/site/src/cut/lib/previewMixer.ts
+++ b/site/src/cut/lib/previewMixer.ts
@@ -106,6 +106,9 @@ interface LiveVoice {
* than a verdict: anything that moves the voice clears it, so a seek back
* into the clip plays again. */
ended: boolean;
+ /** The furthest point in the source a walk has ended at. A second walk that
+ * gets no further has found where the track really stops. */
+ reached: number;
/** Consecutive failed reads, and the wall time (performance.now) the next
* one may be tried at. */
attempts: number;
@@ -126,6 +129,9 @@ const FILL_STALL_MS = 15_000;
const READ_RETRY_MS = 1_000;
const READ_RETRIES = 3;
const READ_RECHECK_MS = 10_000;
+/** How close to the clip's end a walk may stop and be believed. A track
+ * routinely runs a fraction short of the picture it came with. */
+const END_SLACK_S = 0.25;
/** One audio effect element the graph is carrying: its recipe's chain, and
* the pair of gains that cross the mix between treated and untreated. */
@@ -263,6 +269,22 @@ export class PreviewMixer {
return this.heard(graph);
}
+ /**
+ * Where the playhead is, with the clock allowed to re-aim nothing.
+ *
+ * A read asks with this. Asking with `now()` lets a read that took a moment
+ * trip the re-anchor, and re-anchoring disowns every voice — including the
+ * one this read was for, whose sound is then thrown away by the act of
+ * finding out where to put it.
+ */
+ private at(): number {
+ if (!this.anchor) return 0;
+ const wall = this.anchor.timeline + (performance.now() - this.anchor.wall) / 1000;
+ if (!this.ctx || this.ctx.state !== "running") return wall;
+ const graph = this.anchor.timeline + (this.ctx.currentTime - this.anchor.ctx);
+ return this.heard(Math.abs(graph - wall) > 0.25 ? wall : graph);
+ }
+
/**
* How far the picture is being held behind the graph's clock right now, and
* what the output device says it holds. Zero when nothing is playing.
@@ -481,6 +503,41 @@ export class PreviewMixer {
}
}
+ /**
+ * What the sound is doing right now: the clock it runs on and where each
+ * voice has read to. Numbers only — no media, and nothing naming a project.
+ *
+ * A preview that has gone quiet looks the same from outside whatever the
+ * reason, and the reasons live in here. Reachable in any build as
+ * `__cutSound()`, because the machine a cut goes quiet on is rarely the one
+ * a developer is sitting at.
+ */
+ soundState() {
+ const ctx = this.ctx;
+ const wall = performance.now();
+ return {
+ running: this.running,
+ ctx: ctx
+ ? { state: ctx.state, time: +ctx.currentTime.toFixed(2), ...this.clockLead() }
+ : null,
+ at: +this.at().toFixed(2),
+ voices: [...this.voices].map(([id, v]) => ({
+ id,
+ url: v.url.slice(0, 60),
+ scheduled: +v.scheduled.toFixed(2),
+ out: v.out,
+ windows: v.windows.length,
+ reading: v.filling,
+ readingMs: v.filling ? Math.round(wall - v.fillAt) : 0,
+ walk: !!v.walk,
+ ended: v.ended,
+ reached: Number.isFinite(v.reached) ? +v.reached.toFixed(2) : null,
+ attempts: v.attempts,
+ retryInMs: Math.max(0, Math.round(v.retryAt - wall)),
+ })),
+ };
+ }
+
/** Drop the decode cache for a source that has been replaced. */
forget(url: string): void {
for (const key of [...this.decoded.keys()]) {
@@ -522,6 +579,7 @@ export class PreviewMixer {
fillAt: 0,
gen: 0,
ended: false,
+ reached: -Infinity,
attempts: 0,
retryAt: 0,
};
@@ -583,15 +641,27 @@ export class PreviewMixer {
}
if (!current()) return;
if (!buffer) {
- // The reader answered with nothing left to play. A track can stop
- // before the picture does, and a clip whose file carries no sound at
- // all answers this way from the first ask.
- live.ended = true;
+ // A walk reports the same ending whether it read the track's last
+ // sample or gave up on it: bytes that never came, a decoder closed
+ // under it, a link that dropped mid-file. Believed, an ending a few
+ // seconds into a long clip plays the rest of the cut in silence while
+ // the picture reads happily past it — so an ending short of the clip
+ // is a failed read, and another walk goes after the sound it did not
+ // reach. Two things stop that running away: a walk that gets no
+ // further than the one before it has found where the track really
+ // stops, and a track that runs a fraction short of its picture is
+ // inside the slack.
+ if (sourceFrom >= live.out - END_SLACK_S || sourceFrom <= live.reached + 1e-3) {
+ live.ended = true;
+ } else {
+ live.reached = sourceFrom;
+ // A source that stopped answering mid-file is the one case a
+ // re-mint cannot speak to, so this failure keeps to itself.
+ this.miss(live, false);
+ }
return;
}
- // Reading the clock can re-aim the voices (`restart`), so it happens
- // before the placement below is computed.
- const tNow = this.now();
+ const tNow = this.at();
if (!current() || live.scheduled !== from) return;
const until = from + buffer.duration;
if (until <= tNow) {
@@ -680,10 +750,11 @@ export class PreviewMixer {
/** A read that failed, came back empty, or stopped answering. The voice
* gives up where it was, waits, and asks again — and asks for as long as the
* play lasts. */
- private miss(live: LiveVoice): void {
+ private miss(live: LiveVoice, report = true): void {
this.moved(live);
live.attempts += 1;
this.backOff(live, live.attempts);
+ if (!report) return;
// There is no element to reload and nothing else asks on the sound's
// behalf, so a signed URL past its window would stay past it. The picture's
// readers report their failed reads the same way; the re-mint that follows