Open source CapCut alternative.
TypeScript
fix(site): preview decodes low-latency and a paused seek rides the walk
David Le2fbe8e2105b7
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex 695f431f..d3a63cb1 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -177,7 +177,7 @@ export class ClipReader {this.input = input;// A small pool keeps the render's canvas allocation flat over thousands// of frames.- this.sink = frameSink(track, undefined, 4);+ this.sink = frameSink(track, undefined, { poolSize: 4 });return this.sink;})());}diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.tsindex 867d3444..4a175e0e 100644--- a/site/src/cut/lib/frameSource.ts+++ b/site/src/cut/lib/frameSource.ts@@ -226,7 +226,9 @@ export class ClipFrameSource {* are no frames past its tail to decode — so a reader inside it must not* keep restarting a walk that can only end again. */private streamDone = false;- private streaming = false;+ /** The drain in flight, when one is. A seek that replaces the stream awaits+ * this so the old drain has truly let go before the new walk is pulled. */+ private drainRun: Promise<void> | null = null;/** The time a paused reader last asked for, and whether a read is in flight.* Latest wins: a fast drag decodes where it stopped, never every point it* crossed. */@@ -394,7 +396,7 @@ export class ClipFrameSource {// Height alone: the sink keeps the source's aspect and applies the// file's rotation, so a phone clip arrives upright at preview size and// no caller has to know it was ever sideways.- this.sink = frameSink(track, { height: this.height }, POOL);+ this.sink = frameSink(track, { height: this.height }, { poolSize: POOL, lowLatency: true });// A clean open ends any failure streak: the next outage starts from// the quick retries again.this.attempts = 0;@@ -431,7 +433,6 @@ export class ClipFrameSource {private stopStream(): void {const s = this.stream;this.stream = null;- this.streaming = false;void s?.return(undefined).catch(() => {});}@@ -487,6 +488,9 @@ export class ClipFrameSource {private async startStream(from: number): Promise<void> {await this.open();+ // A drain still letting go of a replaced walk holds the pull loop; the new+ // walk starts once it has.+ await this.drainRun;if (this.closed || !this.sink || this.stream) return;// The read may have been overtaken while the file was opening.if (Math.abs(this.streamFrom - from) > SAME) return;@@ -494,40 +498,53 @@ export class ClipFrameSource {void this.drain(from);}- /** Pull frames until the ring reaches `DECODE_AHEAD_S` past `t`. */- private async drain(t: number): Promise<void> {- if (this.streaming || !this.stream) return;- this.streaming = true;- try {- for (;;) {- const stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = this.stream;- if (!stream || this.closed) break;- // Stop at the lookahead, and stop early once the walk's own frames at- // or past `t` fill the ring — pushing another would drop one that is- // still wanted. Only the walk's frames count: leftovers from other- // gestures merely age out of the cache as the walk pushes.- if (this.streamTail >= t + DECODE_AHEAD_S) break;- if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;- const { value, done } = await stream.next();- // A restart or a close while awaiting: this walk is no longer the one.- if (this.stream !== stream) break;- if (done || !value) {- this.stream = null;- this.streamDone = true;- break;+ /** Pull frames until the ring reaches `DECODE_AHEAD_S` past `t`. Returns the+ * run in flight when one is already pulling. */+ private drain(t: number): Promise<void> {+ if (this.drainRun) return this.drainRun;+ if (!this.stream) return Promise.resolve();+ this.drainRun = (async () => {+ try {+ for (;;) {+ const stream: AsyncGenerator<WrappedCanvas, void, unknown> | null = this.stream;+ if (!stream || this.closed) break;+ // Stop at the lookahead, and stop early once the walk's own frames at+ // or past `t` fill the ring — pushing another would drop one that is+ // still wanted. Only the walk's frames count: leftovers from other+ // gestures merely age out of the cache as the walk pushes.+ if (this.streamTail >= t + DECODE_AHEAD_S) break;+ if (this.ring.between(Math.max(t, this.streamFrom), this.streamTail) >= RING - 1) break;+ const { value, done } = await stream.next();+ // A restart or a close while awaiting: this walk is no longer the one.+ if (this.stream !== stream) break;+ if (done || !value) {+ this.stream = null;+ this.streamDone = true;+ break;+ }+ this.streamTail = value.timestamp;+ this.ring.push(value);+ this.onFrame();}- this.streamTail = value.timestamp;- this.ring.push(value);- this.onFrame();+ } catch {+ this.stopStream();+ } finally {+ this.drainRun = null;}- } catch {- this.stopStream();- } finally {- this.streaming = false;- }+ })();+ return this.drainRun;}- /** Fetch the single frame a paused reader is waiting on. */+ /**+ * Serve the position a paused reader is waiting on, newest ask first.+ *+ * The seek is a walk anchored at the ask. A single-frame fetch would build a+ * decoder, run it from the keyframe to the frame, flush it, and throw it+ * away — and the flush and the teardown are the slow part on some machines.+ * The walk decodes the same span once, lands the covering frame as it passes+ * it, and is still warm exactly where the user just landed, so the creep or+ * the play that usually follows starts on frames already in hand.+ */private async pumpSeek(): Promise<void> {if (this.reading) return;this.reading = true;@@ -537,12 +554,18 @@ export class ClipFrameSource {const t = this.wanted;if (t === null || this.closed || !this.sink) break;this.wanted = null;- const c = await this.sink.getCanvas(Math.max(0, t));+ this.stopStream();+ // Let the replaced walk's drain finish letting go, so the new walk is+ // never pulled by a loop still holding the old one.+ await this.drainRun;if (this.closed) break;- if (c) {- this.ring.push(c);- this.onFrame();- }+ // Playback took over while this ask waited; its walk owns the stream.+ if (this.stream) break;+ this.streamDone = false;+ this.streamFrom = t;+ this.streamTail = t;+ this.stream = this.sink.canvases(Math.max(0, t));+ await this.drain(t);// Another position was asked for while this one decoded; that one is// where the pointer actually is now.if (this.wanted === null) break;diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex d3fe0a7a..6e39e407 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1715,7 +1715,7 @@ function edgeReader(url: string): Promise<EdgeReader> {const sinkFor = (height: number) => {let sink = sinks.get(height);if (!sink) {- sink = frameSink(track, { height }, 4);+ sink = frameSink(track, { height }, { poolSize: 4, lowLatency: true });sinks.set(height, sink);}return sink;diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.tsindex 7bac2b29..6e6ac047 100644--- a/site/src/cut/lib/mediaRead.ts+++ b/site/src/cut/lib/mediaRead.ts@@ -170,14 +170,28 @@ export interface FrameCanvasSink {): AsyncGenerator<WrappedCanvas | null, void, unknown>;}+export interface FrameSinkOptions {+ poolSize?: number;+ /** Ask the decoder to emit each frame as soon as it is decoded. Interactive+ * readers — the preview, a trim drag — set this: Windows hardware decoders+ * otherwise hold output until their input queue runs deep, and frames arrive+ * in bursts with stalls between them. Batch readers leave it off and keep+ * the deeper pipeline's throughput. */+ lowLatency?: boolean;+}+export type FrameSinkFactory = (track: InputVideoTrack,size?: FrameSize,- poolSize?: number+ opts?: FrameSinkOptions) => FrameCanvasSink;-const canvasSinkFactory: FrameSinkFactory = (track, size, poolSize) =>- new CanvasSink(track, { ...size, ...(poolSize ? { poolSize } : {}) });+const canvasSinkFactory: FrameSinkFactory = (track, size, opts) =>+ new CanvasSink(track, {+ ...size,+ ...(opts?.poolSize ? { poolSize: opts.poolSize } : {}),+ ...(opts?.lowLatency ? { decoderOptions: { optimizeForLatency: true } } : {}),+ });let sinkFactory: FrameSinkFactory = canvasSinkFactory;@@ -195,9 +209,9 @@ export function setFrameSinkFactory(f: FrameSinkFactory): void {export function frameSink(track: InputVideoTrack,size?: FrameSize,- poolSize?: number+ opts?: FrameSinkOptions): FrameCanvasSink {- return sinkFactory(track, capToTrack(track, size), poolSize);+ return sinkFactory(track, capToTrack(track, size), opts);}function capToTrack(track: InputVideoTrack, size?: FrameSize): FrameSize | undefined {