Skip to content

Open source CapCut alternative.

TypeScript

fix(site): preview decodes low-latency and a paused seek rides the walk

David Le2fbe8e2105b7
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index 695f431f..d3a63cb1 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -177,7 +177,7 @@ export class ClipReader {
this.input = input;
// A small pool keeps the render's canvas allocation flat over thousands
// of frames.
- this.sink = frameSink(track, undefined, 4);
+ this.sink = frameSink(track, undefined, { poolSize: 4 });
return this.sink;
})());
}
diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.ts
index 867d3444..4a175e0e 100644
--- a/site/src/cut/lib/frameSource.ts
+++ b/site/src/cut/lib/frameSource.ts
@@ -226,7 +226,9 @@ export class ClipFrameSource {
* are no frames past its tail to decode — so a reader inside it must not
* keep restarting a walk that can only end again. */
private streamDone = false;
- private streaming = false;
+ /** The drain in flight, when one is. A seek that replaces the stream awaits
+ * this so the old drain has truly let go before the new walk is pulled. */
+ private drainRun: Promise<void> | null = null;
/** The time a paused reader last asked for, and whether a read is in flight.
* Latest wins: a fast drag decodes where it stopped, never every point it
* crossed. */
@@ -394,7 +396,7 @@ export class ClipFrameSource {
// Height alone: the sink keeps the source's aspect and applies the
// file's rotation, so a phone clip arrives upright at preview size and
// no caller has to know it was ever sideways.
- this.sink = frameSink(track, { height: this.height }, POOL);
+ this.sink = frameSink(track, { height: this.height }, { poolSize: POOL, lowLatency: true });
// A clean open ends any failure streak: the next outage starts from
// the quick retries again.
this.attempts = 0;
@@ -431,7 +433,6 @@ export class ClipFrameSource {
private stopStream(): void {
const s = this.stream;
this.stream = null;
- this.streaming = false;
void s?.return(undefined).catch(() => {});
}
@@ -487,6 +488,9 @@ export class ClipFrameSource {
private async startStream(from: number): Promise<void> {
await this.open();
+ // A drain still letting go of a replaced walk holds the pull loop; the new
+ // walk starts once it has.
+ await this.drainRun;
if (this.closed || !this.sink || this.stream) return;
// The read may have been overtaken while the file was opening.
if (Math.abs(this.streamFrom - from) > SAME) return;
@@ -494,40 +498,53 @@ export class ClipFrameSource {
void this.drain(from);
}
- /** Pull frames until the ring reaches `DECODE_AHEAD_S` past `t`. */
- private async drain(t: number): Promise<void> {
- if (this.streaming || !this.stream) return;
- this.streaming = true;
- try {
- for (;;) {
- const stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = this.stream;
- if (!stream || this.closed) break;
- // Stop at the lookahead, and stop early once the walk's own frames at
- // or past `t` fill the ring — pushing another would drop one that is
- // still wanted. Only the walk's frames count: leftovers from other
- // gestures merely age out of the cache as the walk pushes.
- if (this.streamTail >= t + DECODE_AHEAD_S) break;
- if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;
- const { value, done } = await stream.next();
- // A restart or a close while awaiting: this walk is no longer the one.
- if (this.stream !== stream) break;
- if (done || !value) {
- this.stream = null;
- this.streamDone = true;
- break;
+ /** Pull frames until the ring reaches `DECODE_AHEAD_S` past `t`. Returns the
+ * run in flight when one is already pulling. */
+ private drain(t: number): Promise<void> {
+ if (this.drainRun) return this.drainRun;
+ if (!this.stream) return Promise.resolve();
+ this.drainRun = (async () => {
+ try {
+ for (;;) {
+ const stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = this.stream;
+ if (!stream || this.closed) break;
+ // Stop at the lookahead, and stop early once the walk's own frames at
+ // or past `t` fill the ring — pushing another would drop one that is
+ // still wanted. Only the walk's frames count: leftovers from other
+ // gestures merely age out of the cache as the walk pushes.
+ if (this.streamTail >= t + DECODE_AHEAD_S) break;
+ if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;
+ const { value, done } = await stream.next();
+ // A restart or a close while awaiting: this walk is no longer the one.
+ if (this.stream !== stream) break;
+ if (done || !value) {
+ this.stream = null;
+ this.streamDone = true;
+ break;
+ }
+ this.streamTail = value.timestamp;
+ this.ring.push(value);
+ this.onFrame();
}
- this.streamTail = value.timestamp;
- this.ring.push(value);
- this.onFrame();
+ } catch {
+ this.stopStream();
+ } finally {
+ this.drainRun = null;
}
- } catch {
- this.stopStream();
- } finally {
- this.streaming = false;
- }
+ })();
+ return this.drainRun;
}
- /** Fetch the single frame a paused reader is waiting on. */
+ /**
+ * Serve the position a paused reader is waiting on, newest ask first.
+ *
+ * The seek is a walk anchored at the ask. A single-frame fetch would build a
+ * decoder, run it from the keyframe to the frame, flush it, and throw it
+ * away — and the flush and the teardown are the slow part on some machines.
+ * The walk decodes the same span once, lands the covering frame as it passes
+ * it, and is still warm exactly where the user just landed, so the creep or
+ * the play that usually follows starts on frames already in hand.
+ */
private async pumpSeek(): Promise<void> {
if (this.reading) return;
this.reading = true;
@@ -537,12 +554,18 @@ export class ClipFrameSource {
const t = this.wanted;
if (t === null || this.closed || !this.sink) break;
this.wanted = null;
- const c = await this.sink.getCanvas(Math.max(0, t));
+ this.stopStream();
+ // Let the replaced walk's drain finish letting go, so the new walk is
+ // never pulled by a loop still holding the old one.
+ await this.drainRun;
if (this.closed) break;
- if (c) {
- this.ring.push(c);
- this.onFrame();
- }
+ // Playback took over while this ask waited; its walk owns the stream.
+ if (this.stream) break;
+ this.streamDone = false;
+ this.streamFrom = t;
+ this.streamTail = t;
+ this.stream = this.sink.canvases(Math.max(0, t));
+ await this.drain(t);
// Another position was asked for while this one decoded; that one is
// where the pointer actually is now.
if (this.wanted === null) break;
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index d3fe0a7a..6e39e407 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -1715,7 +1715,7 @@ function edgeReader(url: string): Promise<EdgeReader> {
const sinkFor = (height: number) => {
let sink = sinks.get(height);
if (!sink) {
- sink = frameSink(track, { height }, 4);
+ sink = frameSink(track, { height }, { poolSize: 4, lowLatency: true });
sinks.set(height, sink);
}
return sink;
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.ts
index 7bac2b29..6e6ac047 100644
--- a/site/src/cut/lib/mediaRead.ts
+++ b/site/src/cut/lib/mediaRead.ts
@@ -170,14 +170,28 @@ export interface FrameCanvasSink {
): AsyncGenerator<WrappedCanvas | null, void, unknown>;
}
+export interface FrameSinkOptions {
+ poolSize?: number;
+ /** Ask the decoder to emit each frame as soon as it is decoded. Interactive
+ * readers — the preview, a trim drag — set this: Windows hardware decoders
+ * otherwise hold output until their input queue runs deep, and frames arrive
+ * in bursts with stalls between them. Batch readers leave it off and keep
+ * the deeper pipeline's throughput. */
+ lowLatency?: boolean;
+}
+
export type FrameSinkFactory = (
track: InputVideoTrack,
size?: FrameSize,
- poolSize?: number
+ opts?: FrameSinkOptions
) => FrameCanvasSink;
-const canvasSinkFactory: FrameSinkFactory = (track, size, poolSize) =>
- new CanvasSink(track, { ...size, ...(poolSize ? { poolSize } : {}) });
+const canvasSinkFactory: FrameSinkFactory = (track, size, opts) =>
+ new CanvasSink(track, {
+ ...size,
+ ...(opts?.poolSize ? { poolSize: opts.poolSize } : {}),
+ ...(opts?.lowLatency ? { decoderOptions: { optimizeForLatency: true } } : {}),
+ });
let sinkFactory: FrameSinkFactory = canvasSinkFactory;
@@ -195,9 +209,9 @@ export function setFrameSinkFactory(f: FrameSinkFactory): void {
export function frameSink(
track: InputVideoTrack,
size?: FrameSize,
- poolSize?: number
+ opts?: FrameSinkOptions
): FrameCanvasSink {
- return sinkFactory(track, capToTrack(track, size), poolSize);
+ return sinkFactory(track, capToTrack(track, size), opts);
}
function capToTrack(track: InputVideoTrack, size?: FrameSize): FrameSize | undefined {