Skip to content

Open source CapCut alternative.

TypeScript

fix(site): backward-skim cache windows at the ask and releases its copies

David Le92ec7829af1e
diff --git a/site/src/cut/lib/frameSource.test.ts b/site/src/cut/lib/frameSource.test.ts
index b3de9301..94f2d170 100644
--- a/site/src/cut/lib/frameSource.test.ts
+++ b/site/src/cut/lib/frameSource.test.ts
@@ -66,6 +66,24 @@ describe("FrameRing", () => {
expect(ring.at(0)?.timestamp).toBeCloseTo(3);
});
+ test("nearer merges two rings' answers under at()'s own policy", () => {
+ // The backward-skim cache and the main ring each answer separately; the
+ // frame served is the one a single ring holding both would have picked.
+ const a = f(1);
+ const b = f(2);
+ const late = f(5);
+ // A frame at or before the ask beats one after it.
+ expect(FrameRing.nearer(3, a, late)).toBe(a);
+ expect(FrameRing.nearer(3, late, a)).toBe(a);
+ // Of two before, the later; of two after, the earlier.
+ expect(FrameRing.nearer(3, a, b)).toBe(b);
+ expect(FrameRing.nearer(0.5, a, b)).toBe(a);
+ // A lone answer stands.
+ expect(FrameRing.nearer(3, null, b)).toBe(b);
+ expect(FrameRing.nearer(3, a, null)).toBe(a);
+ expect(FrameRing.nearer(3, null, null)).toBe(null);
+ });
+
test("a window keeps the answer on the asking clip's side of a split", () => {
// Two clips split from one file share a ring. The clip past the split asks
// inside its own span; a garden frame from before the split is nearer but
diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.ts
index db50435e..95c9f166 100644
--- a/site/src/cut/lib/frameSource.ts
+++ b/site/src/cut/lib/frameSource.ts
@@ -63,15 +63,15 @@ const LAG_HOP_S = 2;
const FIRST_FRAME_S = 4;
/** Two source times closer than this are the same frame. */
const SAME = 1e-4;
-/** Frames the backward-skim cache keeps of one keyframe span. A drag moving
- * backward reads against the decode direction: every position it crosses would
- * cost a fresh decoder session and a re-decode from the same keyframe. Instead
- * the span is decoded once and this many frames of it are copied out, evenly
- * spread, so backward positions answer from memory at this granularity. */
+/** Frames the backward-skim cache keeps of one span: the span is decoded once
+ * and this many frames of it are copied out, evenly spread, so the positions a
+ * backward drag crosses answer from memory at this granularity. */
const BACK_FRAMES = 16;
-/** The most source time one backward-skim decode covers. Spans are usually one
- * keyframe interval — a second or two — and this keeps a sparsely-keyframed
- * file from turning one cache fill into a decode of half the file. */
+/** The most source time one backward-skim fill covers. The window ends at the
+ * ask and reaches this far below it, clamped to the keyframe the span decodes
+ * from, so a sparsely-keyframed file — a screen recording with keyframes many
+ * seconds apart — fills the stretch the drag is entering and pays for that
+ * stretch alone. */
const BACK_SPAN_S = 2.5;
/** Frames a source must go unwanted before the pool will close it. About a
* second of playback — long enough that crossing a cut and coming back finds
@@ -113,26 +113,16 @@ export interface Timed {
duration: number;
}
-/** A frame the backward-skim cache holds. Copied off the sink's pool onto its
- * own canvas, so it stays valid however many frames decode after it. */
-interface CopiedFrame extends Timed {
- frame: SourceFrame;
-}
-
-const copyOf = (c: WrappedCanvas): CopiedFrame => {
+/** Copy a decoded frame off the sink's pool onto its own canvas, so it stays
+ * valid however many frames decode after it. Null when no drawing context is
+ * available — a frame that could not be drawn must never enter a cache that
+ * answers "we have this one". */
+const copyOf = (c: WrappedCanvas): WrappedCanvas | null => {
const canvas = createRasterCanvas(c.canvas.width, c.canvas.height);
const ctx = (canvas as HTMLCanvasElement).getContext("2d");
- ctx?.drawImage(c.canvas, 0, 0);
- return {
- timestamp: c.timestamp,
- duration: c.duration,
- frame: {
- image: canvas,
- width: c.canvas.width,
- height: c.canvas.height,
- timestamp: c.timestamp,
- },
- };
+ if (!ctx) return null;
+ ctx.drawImage(c.canvas, 0, 0);
+ return { canvas, timestamp: c.timestamp, duration: c.duration };
};
/**
@@ -187,15 +177,26 @@ export class FrameRing<T extends Timed> {
* its last painted picture until the right frame lands.
*/
at(t: number, from = -Infinity, to = Infinity): T | null {
- let best: T | null = null;
- let earliest: T | null = null;
+ let pick: T | null = null;
for (const i of this.items) {
if (i.timestamp > to + SAME) continue;
if (i.timestamp + Math.max(i.duration, SAME) < from - SAME) continue;
- if (!earliest || i.timestamp < earliest.timestamp) earliest = i;
- if (i.timestamp <= t + SAME && (!best || i.timestamp > best.timestamp)) best = i;
+ pick = FrameRing.nearer(t, pick, i);
}
- return best ?? earliest;
+ return pick;
+ }
+
+ /** Of two frames, the one `at` would pick for `t`: a frame at or before `t`
+ * beats one after it; of two before, the later; of two after, the earlier.
+ * The one selection policy, shared with callers merging answers from two
+ * rings. */
+ static nearer<T extends Timed>(t: number, a: T | null, b: T | null): T | null {
+ if (!a || !b) return a ?? b;
+ const aBefore = a.timestamp <= t + SAME;
+ const bBefore = b.timestamp <= t + SAME;
+ if (aBefore !== bBefore) return aBefore ? a : b;
+ if (aBefore) return a.timestamp >= b.timestamp ? a : b;
+ return a.timestamp <= b.timestamp ? a : b;
}
/**
@@ -281,7 +282,7 @@ export class ClipFrameSource {
* lookahead points the wrong way — so backward motion gets its own frames,
* read through its own sink so the pull never recycles a canvas the ring
* still shows. */
- private back: { from: number; to: number; ring: FrameRing<CopiedFrame> } | null = null;
+ private back: { from: number; to: number; ring: FrameRing<WrappedCanvas> } | null = null;
private backRun: Promise<void> | null = null;
private backSink: FrameCanvasSink | null = null;
/** The last paused position asked for, which is how a backward drag is
@@ -309,7 +310,12 @@ export class ClipFrameSource {
/** Decoded frames held right now, for the pool's budget and the perf trace. */
get held(): number {
- return this.ring.size + (this.still ? 1 : 0);
+ return this.ring.size + (this.back?.ring.size ?? 0) + (this.still ? 1 : 0);
+ }
+
+ /** Let go of the backward-skim cache and its copies. */
+ dropBack(): void {
+ this.back = null;
}
get ready(): boolean {
@@ -330,18 +336,8 @@ export class ClipFrameSource {
*/
frameAt(t: number, from = -Infinity, to = Infinity): SourceFrame | null {
if (this.still) return this.still;
- const c = this.ring.at(t, from, to);
- const b = this.back?.ring.at(t, from, to) ?? null;
- const cf = c ? frameOfCanvas(c) : null;
- const bf = b ? b.frame : null;
- if (!cf || !bf) return cf ?? bf;
- // Of the two answers, the one standing closest at or before `t`; a frame
- // past `t` only stands in when neither holds one before it.
- const cBefore = cf.timestamp <= t + SAME;
- const bBefore = bf.timestamp <= t + SAME;
- if (cBefore !== bBefore) return cBefore ? cf : bf;
- if (cBefore) return cf.timestamp >= bf.timestamp ? cf : bf;
- return cf.timestamp <= bf.timestamp ? cf : bf;
+ const c = FrameRing.nearer(t, this.ring.at(t, from, to), this.back?.ring.at(t, from, to) ?? null);
+ return c ? frameOfCanvas(c) : null;
}
/** Whether a frame covering `t` exactly is already held. */
@@ -366,16 +362,21 @@ export class ClipFrameSource {
this.wanted = null;
this.lastAsk = null;
// Playback reads forward; the backward cache's copies are dead weight.
- if (this.back) this.back = null;
+ this.back = null;
this.pumpStream(t);
} else {
const prev = this.lastAsk;
this.lastAsk = t;
+ // Reading far from the cache's span means the gesture it served has
+ // moved on; the copies are dead weight until a drag backs through
+ // somewhere again, and that drag fills its own span.
+ if (this.back && (t > this.back.to + BACK_SPAN_S || t < this.back.from - BACK_SPAN_S))
+ this.back = null;
+ if (this.hasExact(t)) return;
// Moving backward reads against the decode direction, where the walk
// and its lookahead cannot help. Fill the backward cache for the span
// being backed through, so the positions still to come answer from it.
- if (prev !== null && t < prev - SAME && !this.backCovers(t)) void this.backfill(t, prev);
- if (this.hasExact(t)) return;
+ if (prev !== null && t < prev - SAME && !this.backCovers(t)) this.backfill(t, prev);
// A drag creeping along is a forward walk, not a series of jumps. Asking
// the file for each position separately means decoding from the nearest
// keyframe every time — the same cost the `<video>` element charged. If
@@ -597,16 +598,20 @@ export class ClipFrameSource {
/**
* Decode the span a drag is backing through, once, into the backward cache.
*
- * The span runs from the keyframe at or before `t` up to where the drag
- * came from. It decodes forward — the only direction a decoder goes — and
- * every `BACK_FRAMES`th of it is copied out as it passes, so by the time
- * the pointer reaches any position in the span, a frame within one stride
- * of it answers from memory. The cache is swapped in before the pull so a
- * fast drag reads the early copies while the later ones still decode.
+ * The window ends at the position the drag came from, reaches at most
+ * `BACK_SPAN_S` below the ask, and is clamped to the keyframe the decode
+ * starts from, so it always contains the ask: on a densely-keyframed file
+ * that is the whole keyframe interval, and on a sparse one — a screen
+ * recording with keyframes many seconds apart — it is the stretch the drag
+ * is entering, with the next window down filled when the drag reaches it.
+ * `BACK_FRAMES` evenly-spread timestamps are fetched in one decode pass and
+ * copied out. The cache is swapped in before the pull so a fast drag reads
+ * the early copies while the later ones still decode, and the pull stands
+ * down the moment playback takes the source over.
*
* The pull runs on its own sink. The main sink cycles a fixed canvas pool
- * that the ring holds references into; sixty frames pulled through it would
- * redraw every canvas the compositor is showing.
+ * that the ring holds references into; a span of frames pulled through it
+ * would redraw every canvas the compositor is showing.
*/
private backfill(t: number, upTo: number): void {
if (this.backRun || this.closed) return;
@@ -615,28 +620,33 @@ export class ClipFrameSource {
await this.open();
if (this.closed || !this.track) return;
const kt = await keyframeTimeAt(this.track, Math.max(0, t)).catch(() => null);
- if (kt === null || this.closed) return;
- if (this.back && Math.abs(this.back.from - kt) < SAME) return;
+ if (kt === null || this.closed || this.backCovers(t)) return;
if (!this.backSink)
this.backSink = frameSink(
this.track,
{ height: this.height },
{ poolSize: 6, lowLatency: true }
);
- const to = Math.min(Math.max(upTo, kt + SAME), kt + BACK_SPAN_S);
- const step = (to - kt) / BACK_FRAMES;
- const ring = new FrameRing<CopiedFrame>(BACK_FRAMES + 1);
- this.back = { from: kt, to, ring };
- const stream = this.backSink.canvases(Math.max(0, kt));
+ const from = Math.max(kt, t - BACK_SPAN_S, 0);
+ const to = Math.min(Math.max(upTo, t + SAME), from + BACK_SPAN_S);
+ const step = (to - from) / BACK_FRAMES;
+ const asks: number[] = [];
+ for (let ts = from; ts <= to + SAME; ts += step) asks.push(ts);
+ const ring = new FrameRing<WrappedCanvas>(asks.length);
+ this.back = { from, to, ring };
+ const stream = this.backSink.canvasesAtTimestamps(asks);
try {
- let next = kt;
+ let last = -1;
for (;;) {
const { value, done } = await stream.next();
- if (done || !value || this.closed) break;
- if (value.timestamp > to + SAME) break;
- if (value.timestamp + SAME >= next) {
- ring.push(copyOf(value));
- next = value.timestamp + step;
+ // `lastAsk` clearing means playback owns the source now; the
+ // cache was dropped and this pull is decoding for nobody.
+ if (done || this.closed || this.lastAsk === null) break;
+ if (!value || value.timestamp === last) continue;
+ last = value.timestamp;
+ const copy = copyOf(value);
+ if (copy) {
+ ring.push(copy);
this.onFrame();
}
}
@@ -733,6 +743,9 @@ export class ClipFrameSource {
await new Promise((r) => setTimeout(r, 40));
if (this.closed) break;
if (this.wanted !== null) continue;
+ // Playback may have started its walk during the wait; that walk
+ // owns the stream and must not be torn down under it.
+ if (this.stream) break;
}
this.stopStream();
// Let the replaced walk's drain finish letting go, so the new walk is
@@ -828,6 +841,11 @@ export class FrameSourcePool {
* crossed back and forth over is never the one closed.
*/
evict(): void {
+ // A backward-skim cache on a source nothing reads any more is copies held
+ // for a gesture that ended; the source itself may be worth keeping warm,
+ // its cache is not.
+ for (const s of this.sources.values())
+ if (this.tick - s.touched >= EVICT_GRACE) s.dropBack();
if (this.sources.size <= this.budget) return;
const idle = [...this.sources.entries()]
.filter(([, s]) => this.tick - s.touched >= EVICT_GRACE)