Open source CapCut alternative.
TypeScript
fix(site): backward scrubbing reads from a cache of the span being backed through
David Le1ea6a6f71543
diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.tsindex 094277d4..db50435e 100644--- a/site/src/cut/lib/frameSource.ts+++ b/site/src/cut/lib/frameSource.ts@@ -25,6 +25,7 @@import type { Input, InputVideoTrack, WrappedCanvas } from "mediabunny";import { frameSink, keyframeTimeAt, openMedia, videoTrackOf, type FrameCanvasSink } from "./mediaRead";+import { createRasterCanvas } from "./raster";import type { MediaAsset } from "./types";/**@@ -62,6 +63,16 @@ const LAG_HOP_S = 2;const FIRST_FRAME_S = 4;/** Two source times closer than this are the same frame. */const SAME = 1e-4;+/** Frames the backward-skim cache keeps of one keyframe span. A drag moving+ * backward reads against the decode direction: every position it crosses would+ * cost a fresh decoder session and a re-decode from the same keyframe. Instead+ * the span is decoded once and this many frames of it are copied out, evenly+ * spread, so backward positions answer from memory at this granularity. */+const BACK_FRAMES = 16;+/** The most source time one backward-skim decode covers. Spans are usually one+ * keyframe interval — a second or two — and this keeps a sparsely-keyframed+ * file from turning one cache fill into a decode of half the file. */+const BACK_SPAN_S = 2.5;/** Frames a source must go unwanted before the pool will close it. About a* second of playback — long enough that crossing a cut and coming back finds* the decoder still open. */@@ -102,6 +113,28 @@ export interface Timed {duration: number;}+/** A frame the backward-skim cache holds. Copied off the sink's pool onto its+ * own canvas, so it stays valid however many frames decode after it. */+interface CopiedFrame extends Timed {+ frame: SourceFrame;+}++const copyOf = (c: WrappedCanvas): CopiedFrame => {+ const canvas = createRasterCanvas(c.canvas.width, c.canvas.height);+ const ctx = (canvas as HTMLCanvasElement).getContext("2d");+ ctx?.drawImage(c.canvas, 0, 0);+ return {+ timestamp: c.timestamp,+ duration: c.duration,+ frame: {+ image: canvas,+ width: c.canvas.width,+ height: c.canvas.height,+ timestamp: c.timestamp,+ },+ };+};+/*** Decoded frames in the order they arrived, oldest dropped first.*@@ -242,6 +275,19 @@ export class ClipFrameSource {private wanted: number | null = null;private reading = false;+ /** The backward-skim cache: strided copies of the keyframe span a drag is+ * moving backward through, decoded once and answered from memory. The main+ * ring cannot serve this — its canvases belong to the sink's pool and its+ * lookahead points the wrong way — so backward motion gets its own frames,+ * read through its own sink so the pull never recycles a canvas the ring+ * still shows. */+ private back: { from: number; to: number; ring: FrameRing<CopiedFrame> } | null = null;+ private backRun: Promise<void> | null = null;+ private backSink: FrameCanvasSink | null = null;+ /** The last paused position asked for, which is how a backward drag is+ * recognized: the next ask lands earlier than this one. */+ private lastAsk: number | null = null;+/** The engine's tick number when this was last asked for anything, for the* pool's eviction order. */touched = 0;@@ -285,12 +331,22 @@ export class ClipFrameSource {frameAt(t: number, from = -Infinity, to = Infinity): SourceFrame | null {if (this.still) return this.still;const c = this.ring.at(t, from, to);- return c ? frameOfCanvas(c) : null;- }-- /** Whether the ring holds the exact frame covering `t`. */+ const b = this.back?.ring.at(t, from, to) ?? null;+ const cf = c ? frameOfCanvas(c) : null;+ const bf = b ? b.frame : null;+ if (!cf || !bf) return cf ?? bf;+ // Of the two answers, the one standing closest at or before `t`; a frame+ // past `t` only stands in when neither holds one before it.+ const cBefore = cf.timestamp <= t + SAME;+ const bBefore = bf.timestamp <= t + SAME;+ if (cBefore !== bBefore) return cBefore ? cf : bf;+ if (cBefore) return cf.timestamp >= bf.timestamp ? cf : bf;+ return cf.timestamp <= bf.timestamp ? cf : bf;+ }++ /** Whether a frame covering `t` exactly is already held. */hasExact(t: number): boolean {- return this.still !== null || this.ring.covers(t);+ return this.still !== null || this.ring.covers(t) || (this.back?.ring.covers(t) ?? false);}/**@@ -308,8 +364,17 @@ export class ClipFrameSource {void this.open();if (playing) {this.wanted = null;+ this.lastAsk = null;+ // Playback reads forward; the backward cache's copies are dead weight.+ if (this.back) this.back = null;this.pumpStream(t);} else {+ const prev = this.lastAsk;+ this.lastAsk = t;+ // Moving backward reads against the decode direction, where the walk+ // and its lookahead cannot help. Fill the backward cache for the span+ // being backed through, so the positions still to come answer from it.+ if (prev !== null && t < prev - SAME && !this.backCovers(t)) void this.backfill(t, prev);if (this.hasExact(t)) return;// A drag creeping along is a forward walk, not a series of jumps. Asking// the file for each position separately means decoding from the nearest@@ -342,6 +407,8 @@ export class ClipFrameSource {this.closed = true;this.stopStream();this.ring.clear();+ this.back = null;+ this.backSink = null;if (this.retryTimer) clearTimeout(this.retryTimer);this.retryTimer = 0;// A still's bitmap holds real memory the GC can't see the size of.@@ -515,12 +582,75 @@ export class ClipFrameSource {const kt = await keyframeTimeAt(this.track, Math.max(0, t)).catch(() => null);if (kt === null || this.closed) return;if (this.ring.between(kt, t) > 0) return;+ if (this.back && this.back.ring.between(kt, t) > 0) return;const c = await this.sink.getCanvas(kt).catch(() => null);if (!c || this.closed) return;this.ring.push(c);this.onFrame();}+ /** Whether the backward cache spans `t` — filled or still filling. */+ private backCovers(t: number): boolean {+ return this.back !== null && t >= this.back.from - SAME && t <= this.back.to + SAME;+ }++ /**+ * Decode the span a drag is backing through, once, into the backward cache.+ *+ * The span runs from the keyframe at or before `t` up to where the drag+ * came from. It decodes forward — the only direction a decoder goes — and+ * every `BACK_FRAMES`th of it is copied out as it passes, so by the time+ * the pointer reaches any position in the span, a frame within one stride+ * of it answers from memory. The cache is swapped in before the pull so a+ * fast drag reads the early copies while the later ones still decode.+ *+ * The pull runs on its own sink. The main sink cycles a fixed canvas pool+ * that the ring holds references into; sixty frames pulled through it would+ * redraw every canvas the compositor is showing.+ */+ private backfill(t: number, upTo: number): void {+ if (this.backRun || this.closed) return;+ this.backRun = (async () => {+ try {+ await this.open();+ if (this.closed || !this.track) return;+ const kt = await keyframeTimeAt(this.track, Math.max(0, t)).catch(() => null);+ if (kt === null || this.closed) return;+ if (this.back && Math.abs(this.back.from - kt) < SAME) return;+ if (!this.backSink)+ this.backSink = frameSink(+ this.track,+ { height: this.height },+ { poolSize: 6, lowLatency: true }+ );+ const to = Math.min(Math.max(upTo, kt + SAME), kt + BACK_SPAN_S);+ const step = (to - kt) / BACK_FRAMES;+ const ring = new FrameRing<CopiedFrame>(BACK_FRAMES + 1);+ this.back = { from: kt, to, ring };+ const stream = this.backSink.canvases(Math.max(0, kt));+ try {+ let next = kt;+ for (;;) {+ const { value, done } = await stream.next();+ if (done || !value || this.closed) break;+ if (value.timestamp > to + SAME) break;+ if (value.timestamp + SAME >= next) {+ ring.push(copyOf(value));+ next = value.timestamp + step;+ this.onFrame();+ }+ }+ } finally {+ void stream.return(undefined).catch(() => {});+ }+ } catch {+ this.back = null;+ } finally {+ this.backRun = null;+ }+ })();+ }+/** Pull frames until the ring reaches `DECODE_AHEAD_S` past `t`. Returns the* run in flight when one is already pulling. */private drain(t: number): Promise<void> {@@ -594,6 +724,16 @@ export class ClipFrameSource {// The pointer has already moved on; chase it rather than finishing// an exact decode nobody is looking at.if (this.wanted !== null) continue;+ // The backward cache is already showing a frame for this span, so the+ // exact decode can wait a beat. A drag still moving supersedes the ask+ // during the wait, and the whole backward pass costs one decoder+ // session — the cache fill — while the exact frame still lands a beat+ // after the pointer rests.+ if (this.backCovers(t)) {+ await new Promise((r) => setTimeout(r, 40));+ if (this.closed) break;+ if (this.wanted !== null) continue;+ }this.stopStream();// Let the replaced walk's drain finish letting go, so the new walk is// never pulled by a loop still holding the old one.