Skip to content

Open source CapCut alternative.

TypeScript

perf(site): one memory ceiling for the tab, shared by every cache

David Lebf5cc6ea2652
diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.ts
index 61fb7bf4..d5867d1d 100644
--- a/site/src/cut/lib/behindPass.ts
+++ b/site/src/cut/lib/behindPass.ts
@@ -19,6 +19,7 @@ import {
wordSampleWindows,
} from "@donkeycut/effects-kit";
import { personSegmenter, segmentSubjectAlpha } from "./cutout";
+import { holdMemory } from "./memoryBudget";
import { createRasterCanvas } from "./raster";
import { renderElementPng } from "./textRender";
import {
@@ -121,11 +122,32 @@ export class SubjectMaskCompositor {
private segmenter: Segmenter | null = null;
private segKicked = false;
- // One picture per element, and one per emphasized word when the element
- // lights its words as they are said — keyed by the element object, so an
- // edit hands back a fresh picture rather than a stale one.
- private rasters = new WeakMap<Overlay, Map<number, ImageBitmap>>();
- private pending = new WeakMap<Overlay, Set<number>>();
+ /**
+ * One picture per element, and one per emphasized word when the element
+ * lights its words as they are said.
+ *
+ * Filed under the element's id and stamped with the element object the
+ * picture was drawn from. The store hands back a new object for every edit,
+ * so a stamp that no longer matches is a picture of an element that has since
+ * changed: it closes and redraws. Keying on the object itself would say the
+ * same thing, and would leave every superseded picture holding pixels outside
+ * the heap that only `close()` gives back, with a keystroke's worth minted
+ * per keystroke.
+ *
+ * `pending` rides in the entry so it is dropped whenever the pictures are.
+ * It marks a draw already asked for, and a marker outliving the picture it
+ * was asked for would leave the element blank for good.
+ */
+ private rasters = new Map<
+ string,
+ { of: Overlay; byWord: Map<number, ImageBitmap>; pending: Set<number> }
+ >();
+ private readonly releaseMemory = holdMemory("pictures", () => {
+ let n = 0;
+ for (const e of this.rasters.values())
+ for (const b of e.byWord.values()) n += b.width * b.height * 4;
+ return n;
+ });
private person: HTMLCanvasElement | null = null;
private small: HTMLCanvasElement | null = null;
private mask: { at: number; alpha: HTMLCanvasElement | null } = { at: -1e9, alpha: null };
@@ -143,6 +165,23 @@ export class SubjectMaskCompositor {
});
}
+ /** This element's entry, emptied first when the pictures in it were drawn
+ * from an older version of the element. */
+ private entryFor(o: Overlay) {
+ let entry = this.rasters.get(o.id);
+ if (entry && entry.of !== o) {
+ for (const b of entry.byWord.values()) b.close();
+ entry.byWord.clear();
+ entry.pending.clear();
+ entry.of = o;
+ }
+ if (!entry) {
+ entry = { of: o, byWord: new Map(), pending: new Set() };
+ this.rasters.set(o.id, entry);
+ }
+ return entry;
+ }
+
private rasterFor(
o: Overlay,
w: number,
@@ -150,12 +189,11 @@ export class SubjectMaskCompositor {
assets: MediaAsset[],
word: number
): ImageBitmap | null {
- const hit = this.rasters.get(o)?.get(word);
+ const entry = this.entryFor(o);
+ const hit = entry.byWord.get(word);
if (hit) return hit;
- const inFlight = this.pending.get(o) ?? new Set<number>();
- if (!inFlight.has(word)) {
- inFlight.add(word);
- this.pending.set(o, inFlight);
+ if (!entry.pending.has(word)) {
+ entry.pending.add(word);
// Neutral picture: position aside, the per-frame pose owns rotation and
// opacity, so baking them here would apply each of them twice.
void this.drawRaster(o, w, h, assets, word).catch(() => {});
@@ -182,9 +220,29 @@ export class SubjectMaskCompositor {
assets
);
const bmp = await createImageBitmap(png);
- const byWord = this.rasters.get(o) ?? new Map<number, ImageBitmap>();
- byWord.set(word, bmp);
- this.rasters.set(o, byWord);
+ // The element may have been edited, or the pictures released, while this
+ // one was being drawn; either way its entry is gone or stamped with a
+ // different object, and this picture is of the wrong thing.
+ const entry = this.rasters.get(o.id);
+ if (!entry || entry.of !== o) {
+ bmp.close();
+ return;
+ }
+ entry.byWord.get(word)?.close();
+ entry.byWord.set(word, bmp);
+ }
+
+ /** Give back every picture this pass is holding. The pass stays usable and
+ * draws them again when it is next asked. */
+ clear(): void {
+ for (const e of this.rasters.values()) for (const b of e.byWord.values()) b.close();
+ this.rasters.clear();
+ }
+
+ /** Give the pictures back and stop reporting what this pass holds. */
+ dispose(): void {
+ this.clear();
+ this.releaseMemory();
}
/** Export path: everything resident before the first frame draws. */
@@ -199,7 +257,7 @@ export class SubjectMaskCompositor {
// An element that lights its words needs one picture per word, all of
// them resident before the first frame draws.
rasterSlices(o).map(async (word) => {
- if (this.rasters.get(o)?.has(word)) return;
+ if (this.entryFor(o).byWord.has(word)) return;
try {
await this.drawRaster(o, w, h, assets, word);
} catch {
@@ -342,6 +400,18 @@ export class SubjectMaskCompositor {
t: number,
opts: { minMaskInterval?: number } = {}
): void {
+ // A deleted element's picture has nothing left to draw it. Filing by id
+ // means the entry outlives the element, so the document's own list is what
+ // says which entries are still owed — checked only when there are more of
+ // them than elements, which is the only way one can be stale.
+ if (this.rasters.size > overlays.length) {
+ const ids = new Set(overlays.map((o) => o.id));
+ for (const [id, e] of this.rasters) {
+ if (ids.has(id)) continue;
+ for (const b of e.byWord.values()) b.close();
+ this.rasters.delete(id);
+ }
+ }
const active = behindOverlaysAt(overlays, t);
const wantsMatte = overlays.some(
(o) => frontSubjectOverlay(o) && drawable(o) && t >= o.start && t <= o.end
diff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.ts
index a0ee5038..40751f2d 100644
--- a/site/src/cut/lib/exportRender.ts
+++ b/site/src/cut/lib/exportRender.ts
@@ -32,6 +32,7 @@ import { renderMix, type MixClip, type MixItem, type MixSpec } from "./audioMix"
import { FrameCompositor, MISSING_FRAME, type Frame } from "./composite";
import { overlayPlan, trackZeroPlan } from "./framePlan";
import { frameSink, openMedia, videoTrackOf } from "./mediaRead";
+import { allowance, canvasBytes } from "./memoryBudget";
import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";
import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";
import { applyEffectToCanvas, evalOverlayFrame, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, MATTE_FPS, matteLumaToAlpha, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";
@@ -126,6 +127,18 @@ export function bitrateFor(settings: ExportSettings): number {
/** The frame reader for one open video source. */
type ClipSink = ReturnType<typeof frameSink>;
+/** Canvases one clip reader cycles. Small: a render draws one frame at a time
+ * and never looks back, so the pool only has to keep the frame being drawn
+ * clear of the frame being decoded. */
+const READER_POOL = 4;
+/** Readers a render keeps open when memory is not the constraint. A cut drawn
+ * from more files than this reopens the ones it comes back to. */
+const READERS_TUNED = 12;
+/** Frames a reader may go unasked-for before it is a candidate to close. One
+ * frame's own readers are never candidates, so what a join or a stack of
+ * layers needs together is never taken away from the frame needing it. */
+const READER_GRACE = 2;
+
/** A clip's source time at timeline time `t`. */
function sourceTimeAt(span: ClipSpan, t: number): number {
const speed = span.clip.speed && span.clip.speed > 0 ? span.clip.speed : 1;
@@ -153,6 +166,20 @@ export class ClipReader {
* the preview holds the rest — the rebuilt reader decodes in software,
* which always opens. */
private software = false;
+ /** The frame number this reader was last asked for, for the painter's
+ * between-frames eviction. */
+ usedAt = 0;
+
+ /**
+ * Pixels in a frame of the source.
+ *
+ * The sink is opened with no size, so its canvases are the file's own — a
+ * 4K clip in a 1080p render still cycles 4K canvases. Sizing the budget off
+ * the render's canvas would under-count that by the ratio between them.
+ */
+ get sourcePixels(): number {
+ return (this.asset.width ?? 1920) * (this.asset.height ?? 1080);
+ }
constructor(
private asset: MediaAsset,
@@ -188,7 +215,7 @@ export class ClipReader {
// A small pool keeps the render's canvas allocation flat over thousands
// of frames.
this.sink = frameSink(track, undefined, {
- poolSize: 4,
+ poolSize: READER_POOL,
...(this.software ? { software: true } : {}),
});
return this.sink;
@@ -826,6 +853,8 @@ export class FramePainter {
private comp: FrameCompositor;
private stamps: StampCache;
private readers = new Map<string, ClipReader>();
+ /** Frames drawn, which is the clock the readers' eviction order runs on. */
+ private frameNo = 0;
private behind: SubjectMaskCompositor | null = null;
private fxScratch: RasterSurface | null = null;
/** This frame's removal mattes, one alpha image per keyed clip — fetched
@@ -907,12 +936,58 @@ export class FramePainter {
for (const sp of list) this.spanOfClip.set(sp.clip.id, sp);
}
+ /**
+ * The reader for one asset, opening it if this frame is the first to ask.
+ *
+ * Readers are kept between frames because a render walks time forward and
+ * asks the same clip for thousands of consecutive frames; reopening one per
+ * frame would pay a keyframe seek for every one of them. They are not kept
+ * forever: each holds a decoder and a pool of canvases at the render size, so
+ * a cut drawing from a hundred files would otherwise finish the render
+ * holding a hundred decoders. Past the budget the least recently drawn are
+ * let go, which costs one reopen if the cut returns to them.
+ */
private readerFor(asset: MediaAsset): ClipReader {
let r = this.readers.get(asset.id);
if (!r) this.readers.set(asset.id, (r = new ClipReader(asset, () => this.resolve(asset))));
+ r.usedAt = this.frameNo;
return r;
}
+ /**
+ * Close the readers this cut has moved on from.
+ *
+ * Readers are kept between frames because a render walks time forward and
+ * asks the same clip for thousands of consecutive frames; reopening one per
+ * frame would pay a keyframe seek for every one. They are not kept forever:
+ * each holds a decoder and a pool of canvases at the *source's* own size, so
+ * a cut drawing from a hundred files would otherwise end the render holding a
+ * hundred decoders.
+ *
+ * This runs between frames, before anything has been asked for, and spares
+ * everything the last couple of frames used. A frame reaches for several
+ * readers at once — a join's two sides, a backdrop, a matte, every overlay
+ * layer — and closing one the frame is midway through would leave the
+ * compositor drawing from a disposed decoder. Closing them here, past the
+ * grace, cannot.
+ */
+ private evictReaders(): void {
+ const cost = (r: ClipReader) => canvasBytes(READER_POOL * r.sourcePixels);
+ const cap = allowance("canvases", READERS_TUNED * canvasBytes(READER_POOL * 1920 * 1080));
+ let held = 0;
+ for (const r of this.readers.values()) held += cost(r);
+ if (held <= cap) return;
+ const old = [...this.readers.entries()]
+ .filter(([, r]) => this.frameNo - r.usedAt > READER_GRACE)
+ .sort((a, b) => a[1].usedAt - b[1].usedAt);
+ for (const [id, r] of old) {
+ if (held <= cap) break;
+ held -= cost(r);
+ this.readers.delete(id);
+ r.dispose();
+ }
+ }
+
/** Pull a removal clip's baked matte frame for timeline time `t` and stage
* it, luma turned to alpha, where the compositor's provider reads. A clip
* whose bake is still owed stages nothing. */
@@ -952,6 +1027,8 @@ export class FramePainter {
/** Draw the whole cut at timeline time `t` onto the canvas. */
async drawAt(t: number): Promise<void> {
+ this.frameNo++;
+ this.evictReaders();
const { canvas, comp, doc, stamps } = this;
const W = canvas.width;
const H = canvas.height;
@@ -1047,6 +1124,7 @@ export class FramePainter {
dispose(): void {
this.stamps.dispose();
+ this.behind?.dispose();
for (const r of this.readers.values()) r.dispose();
this.readers.clear();
}
diff --git a/site/src/cut/lib/frameSource.test.ts b/site/src/cut/lib/frameSource.test.ts
index 6699554b..207f947a 100644
--- a/site/src/cut/lib/frameSource.test.ts
+++ b/site/src/cut/lib/frameSource.test.ts
@@ -389,6 +389,34 @@ describe("FrameSourcePool", () => {
expect(big.map((s) => s.suspended)).toEqual([true, true, true, true, false, false]);
});
+ test("says what it is holding, live sources included", () => {
+ const pool = new FrameSourcePool(4);
+ expect(pool.decoderBytes).toBe(0);
+ expect(pool.canvasBytes).toBe(0);
+ pool.beginFrame();
+ pool.get("a", asset("a", 3840, 2160), 2160);
+ // A live 4K decoder holds sixteen planar frames: about two hundred
+ // megabytes, none of which the warm shelf's own number would ever show.
+ expect(Math.round(pool.decoderBytes / 2 ** 20)).toBe(190);
+ expect(pool.warmPixels).toBe(0);
+ expect(pool.canvasBytes).toBeGreaterThan(0);
+ // One 4K source is well inside the budget, so nothing stands down.
+ for (let i = 0; i < 200; i++) pool.beginFrame();
+ pool.evict();
+ expect(pool.decoderBytes).toBeGreaterThan(0);
+
+ // Standing down gives the decoder back and keeps the canvases, which is
+ // the whole point of the warm shelf.
+ const tight = new FrameSourcePool(0);
+ tight.beginFrame();
+ tight.get("a", asset("a", 1920, 1080), 1080);
+ const canvases = tight.canvasBytes;
+ for (let i = 0; i < 200; i++) tight.beginFrame();
+ tight.evict();
+ expect(tight.decoderBytes).toBe(0);
+ expect(tight.canvasBytes).toBe(canvases);
+ });
+
test("standing down costs a decoder, not a source", () => {
const pool = new FrameSourcePool(1);
pool.beginFrame();
diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.ts
index 2d789829..ebd7db89 100644
--- a/site/src/cut/lib/frameSource.ts
+++ b/site/src/cut/lib/frameSource.ts
@@ -25,6 +25,7 @@
import type { Input, InputVideoTrack, WrappedCanvas } from "mediabunny";
import { frameSink, keyframeTimeAt, openMedia, videoTrackOf, type FrameCanvasSink } from "./mediaRead";
+import { allowance, canvasBytes, decodedFrameBytes, holdMemory } from "./memoryBudget";
import { meterPull, meterSource, meterWalk } from "./perfTrace";
import type { MediaAsset } from "./types";
@@ -305,6 +306,16 @@ const WARM_KEEP_MAX = 32;
* at 1080p and holds four at 4K.
*/
const LIVE_PIXELS = 34e6;
+/**
+ * Decoded frames a live decoder is holding at once.
+ *
+ * The patched decode pump gives a stream a head start of this many packets and
+ * counts what has gone in against what has come back out, so a stream that is
+ * not being read stops here, short of the platform decoder. It is
+ * the multiplier that turns a source's frame size into what its decoder costs
+ * — see `site/patches/mediabunny+1.55.5.patch`.
+ */
+const DECODER_FRAMES = 16;
/** A failed open tries again this much later, growing per attempt over the
* first `RETRIES` tries, so a network blip heals within seconds. */
const RETRY_MS = 1000;
@@ -651,6 +662,25 @@ export class ClipFrameSource {
return POOL * width * this.height;
}
+ /**
+ * Every canvas this source is holding.
+ *
+ * `keptPixels` answers what a stood-down source would still cost, which is
+ * what the warm shelf is filled by. A source being read holds that same pool
+ * and, while a backward gesture is on it, two more: the fine windows at the
+ * decode size and the coarse span at half of it. Those are the biggest thing
+ * in the tab during a scrub, so a report of what the editor holds has to
+ * count them.
+ */
+ get canvasPixels(): number {
+ const one = this.keptPixels / POOL;
+ return (
+ this.keptPixels +
+ (this.backSink ? one * BACK_POOL : 0) +
+ (this.backCoarseSink ? (one / 4) * BACK_COARSE_POOL : 0)
+ );
+ }
+
/** Pixels in a frame of the file, which is what its decoder's held frames
* cost whatever size the picture is drawn at. */
get decodePixels(): number {
@@ -658,6 +688,12 @@ export class ClipFrameSource {
return (this.asset.width ?? 1920) * (this.asset.height ?? 1080);
}
+ /** What this source's decoder is holding in the platform's decoder, which is
+ * nothing at all once it has stood down. */
+ get decoderBytes(): number {
+ return this.asleep ? 0 : decodedFrameBytes(this.decodePixels) * DECODER_FRAMES;
+ }
+
/** The URL this source is reading. The pool compares it against the store's
* current one, so a re-minted signed URL replaces the source under it. */
get url(): string {
@@ -1645,10 +1681,19 @@ export class FrameSourcePool {
private sources = new Map<string, ClipFrameSource>();
private tick = 0;
+ private readonly release: () => void;
+
constructor(
private budget = 10,
private readonly onFrame: () => void = () => {}
- ) {}
+ ) {
+ const stopDecoders = holdMemory("decoders", () => this.decoderBytes);
+ const stopCanvases = holdMemory("canvases", () => this.canvasBytes);
+ this.release = () => {
+ stopDecoders();
+ stopCanvases();
+ };
+ }
/** Advance the clock the eviction order is measured on. */
beginFrame(): void {
@@ -1715,6 +1760,28 @@ export class FrameSourcePool {
return n;
}
+ /** What the live decoders are holding in the platform's decoder. */
+ get decoderBytes(): number {
+ let n = 0;
+ for (const s of this.sources.values()) n += s.decoderBytes;
+ return n;
+ }
+
+ /**
+ * Canvas backing behind every source, live and stood down.
+ *
+ * The warm shelf's own budget counts only the stood-down ones, because those
+ * are the only ones it may close. This counts all of them, because all of
+ * them are memory the machine has to find — a scrub holding two backward
+ * windows open is the largest the number ever gets, and a report that left
+ * it out would say the editor was idle at its busiest moment.
+ */
+ get canvasBytes(): number {
+ let n = 0;
+ for (const s of this.sources.values()) n += s.canvasPixels;
+ return canvasBytes(n);
+ }
+
/** Every open source and what it is doing, for the perf eval's pool dump. */
debugState(): Record<string, unknown>[] {
return [...this.sources.entries()].map(([id, s]) => ({ id, tick: this.tick, ...s.debugState() }));
@@ -1757,14 +1824,17 @@ export class FrameSourcePool {
live--;
}
- // Then by pixels: the live decoders are kept most recently read first
- // until the source budget runs out, and the idle ones past it stand down.
- let livePixels = 0;
+ // Then by what those decoders hold: the live ones are kept most recently
+ // read first until the budget runs out, and the idle ones past it stand
+ // down. The budget is the smaller of what the preview was tuned to want
+ // and this machine's share, so a laptop sheds decoders a desktop keeps.
+ const decoderCap = allowance("decoders", decodedFrameBytes(LIVE_PIXELS) * DECODER_FRAMES);
+ let liveBytes = 0;
for (const [id, src] of [...this.sources.entries()].sort((a, b) => b[1].touched - a[1].touched)) {
if (src.suspended) continue;
- livePixels += src.decodePixels;
- if (livePixels <= LIVE_PIXELS || this.tick - src.touched < EVICT_GRACE) continue;
- poolLog(`pool suspend ${id} (pixels)`);
+ liveBytes += src.decoderBytes;
+ if (liveBytes <= decoderCap || this.tick - src.touched < EVICT_GRACE) continue;
+ poolLog(`pool suspend ${id} (memory)`);
src.suspend();
}
@@ -1772,13 +1842,14 @@ export class FrameSourcePool {
// the canvas budget runs out, and what does not fit closes for real. Only
// stood-down sources are on the shelf, so nothing holding a decoder — and
// nothing the picture is being drawn from — is ever closed here.
- let pixels = 0;
+ const warmCap = allowance("canvases", canvasBytes(WARM_PIXELS));
+ let held = 0;
let warm = 0;
for (const [id, src] of [...this.sources.entries()].sort((a, b) => b[1].touched - a[1].touched)) {
if (!src.suspended) continue;
- pixels += src.keptPixels;
+ held += canvasBytes(src.keptPixels);
warm++;
- if (pixels <= WARM_PIXELS && warm <= WARM_KEEP_MAX) continue;
+ if (held <= warmCap && warm <= WARM_KEEP_MAX) continue;
poolLog(`pool close ${id}`);
src.close();
this.sources.delete(id);
@@ -1790,6 +1861,12 @@ export class FrameSourcePool {
for (const s of this.sources.values()) s.close();
this.sources.clear();
}
+
+ /** Close every source and stop reporting what this pool holds. */
+ dispose(): void {
+ this.closeAll();
+ this.release();
+ }
}
/**
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.ts
index 47a6661c..98d14ab7 100644
--- a/site/src/cut/lib/mediaRead.ts
+++ b/site/src/cut/lib/mediaRead.ts
@@ -44,6 +44,7 @@ import {
import { confirmEngine, engineConnected, isEngineUrl } from "./api";
import { resolveRegisteredBlob } from "./backend/browser/registry";
import { chunkSourceOptions } from "./chunkCache";
+import { allowance, holdMemory } from "./memoryBudget";
/** What a file turns out to be, read from its container. */
export interface MediaProbe {
@@ -176,22 +177,72 @@ function trackedFetch(url: string): typeof fetch {
* ranged fetches of blob URLs are unreliable across browsers. Signed cloud
* media reads through the chunk cache (chunkCache.ts), so bytes touched once
* are on disk for the next read of the same object. */
+/**
+ * Readers open at once, at the busiest the editor gets.
+ *
+ * The preview's decoder pool, the timeline's edge readers, and a render's
+ * per-clip readers all hold one each. The number is what a per-reader cache has
+ * to be divided by for the total to stay inside the tab's share: sizing each
+ * cache as if it were the only one is how a cache with a sensible-looking limit
+ * becomes a gigabyte.
+ */
+const READERS = 16;
+/** Bytes of the file one reader keeps in memory. Enough that a walk reading
+ * forward never re-fetches what it just read, and small enough that the
+ * readers together stay inside the tab's ceiling. */
+const READ_CACHE = 32 * 2 ** 20;
+
+/** Chunked readers open right now, for the memory report. */
+let openReaders = 0;
+
+/** Decided on the first open and held: the ceiling does not move, so asking
+ * again would return the same number and re-record the same pressure. Reading
+ * what the readers hold must not itself count as a request for room. */
+let readCache = 0;
+const readCacheSize = (): number =>
+ (readCache ||= Math.round(allowance("reads", READ_CACHE * READERS) / READERS));
+
+holdMemory("reads", () => openReaders * readCacheSize());
+
export function openMedia(src: string | Blob): Input {
const blob = typeof src === "string" ? resolveRegisteredBlob(src) ?? src : src;
let source;
+ let chunkedRead = false;
if (typeof blob === "string") {
const chunked = chunkSourceOptions(blob);
source = chunked
- ? new CustomSource({ ...chunked, prefetchProfile: "network", maxCacheSize: 64 * 2 ** 20 })
+ ? new CustomSource({
+ ...chunked,
+ prefetchProfile: "network",
+ maxCacheSize: readCacheSize(),
+ })
: new UrlSource(blob, {
parallelism: urlParallelism(blob),
getRetryDelay: urlRetryDelay,
fetchFn: trackedFetch(blob),
});
+ chunkedRead = !!chunked;
} else {
source = new BlobSource(blob);
}
- return new Input({ formats: ALL_FORMATS, source });
+ const input = new Input({ formats: ALL_FORMATS, source });
+ // The cache lives inside the source, where nothing can ask it how big it has
+ // grown. Counting the readers that own one is what makes the total
+ // reportable, and the count is only right if it is taken where the reader is
+ // let go — so the dispose every caller already makes is where it is taken.
+ if (chunkedRead) {
+ openReaders++;
+ const dispose = input.dispose.bind(input);
+ let closed = false;
+ input.dispose = () => {
+ if (!closed) {
+ closed = true;
+ openReaders--;
+ }
+ return dispose();
+ };
+ }
+ return input;
}
/** Run `fn` against an open input and dispose it however that ends. */
diff --git a/site/src/cut/lib/memoryBudget.test.ts b/site/src/cut/lib/memoryBudget.test.ts
new file mode 100644
index 00000000..dac69b8a
--- /dev/null
+++ b/site/src/cut/lib/memoryBudget.test.ts
@@ -0,0 +1,105 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import {
+ allowance,
+ canvasBytes,
+ decodedFrameBytes,
+ heapBytes,
+ holdMemory,
+ mb,
+ memoryCeiling,
+ memoryUsage,
+ takeMemoryPressure,
+} from "./memoryBudget";
+
+const MB = 2 ** 20;
+
+const drops: (() => void)[] = [];
+const hold = (bucket: Parameters<typeof holdMemory>[0], bytes: number) => {
+ drops.push(holdMemory(bucket, () => bytes));
+};
+
+afterEach(() => {
+ for (const drop of drops.splice(0)) drop();
+ takeMemoryPressure();
+});
+
+describe("the ceiling", () => {
+ test("is a share of the machine, decided once", () => {
+ // The environment the tests run in reports no `deviceMemory`, which is the
+ // same silence Safari and Firefox give a real page: the assumed size
+ // stands in, and a quarter of it is the tab's.
+ expect(memoryCeiling()).toBe(2 * 2 ** 30);
+ expect(memoryCeiling()).toBe(memoryCeiling());
+ });
+
+ test("hands a bucket the smaller of its tuning and its share", () => {
+ // A size the machine can afford is left alone, so a preview tuned on a
+ // machine with room behaves there exactly as it was tuned to.
+ const small = 4 * MB;
+ expect(allowance("decoders", small)).toBe(small);
+ expect(takeMemoryPressure()).toEqual([]);
+
+ // A size it cannot is cut to the share, and the cut is recorded: the
+ // machine is deciding, not the tuning.
+ const huge = 99 * 2 ** 30;
+ expect(allowance("decoders", huge)).toBeLessThan(huge);
+ expect(allowance("decoders", huge)).toBe(Math.round(memoryCeiling() * 0.4));
+ expect(takeMemoryPressure()).toEqual(["decoders"]);
+ // Reading the pressure clears it, so a later window reports its own.
+ expect(takeMemoryPressure()).toEqual([]);
+ });
+
+ test("splits the whole ceiling between the buckets and no more", () => {
+ const huge = 99 * 2 ** 30;
+ const shares =
+ allowance("decoders", huge) +
+ allowance("canvases", huge) +
+ allowance("reads", huge) +
+ allowance("audio", huge) +
+ allowance("pictures", huge);
+ expect(shares).toBeLessThanOrEqual(memoryCeiling());
+ expect(shares).toBeGreaterThan(memoryCeiling() * 0.99);
+ });
+});
+
+describe("what is being held", () => {
+ test("adds up what every cache reports, by bucket", () => {
+ hold("decoders", 10 * MB);
+ hold("decoders", 5 * MB);
+ hold("reads", 3 * MB);
+ const usage = memoryUsage();
+ expect(mb(usage.decoders)).toBe(15);
+ expect(mb(usage.reads)).toBe(3);
+ expect(mb(usage.canvases)).toBe(0);
+ expect(mb(usage.total)).toBe(18);
+ });
+
+ test("forgets a cache that has been let go", () => {
+ hold("audio", 8 * MB);
+ expect(mb(memoryUsage().audio)).toBe(8);
+ drops.pop()!();
+ expect(memoryUsage().audio).toBe(0);
+ });
+
+ test("reports nothing rather than guessing when the browser is silent", () => {
+ // No `performance.memory` here, as in every browser but Chrome. A zero is
+ // the honest answer; a made-up number would read as a measurement.
+ expect(heapBytes()).toBe(0);
+ });
+});
+
+describe("what a picture costs", () => {
+ test("counts canvas backing at four bytes a pixel", () => {
+ expect(canvasBytes(1920 * 1080)).toBe(1920 * 1080 * 4);
+ });
+
+ test("counts a decoded frame at the planar size the decoder hands back", () => {
+ // Three halves of the luma plane: full-resolution luma, two quarter-
+ // resolution chroma planes.
+ expect(decodedFrameBytes(1920 * 1080)).toBe(Math.round(1920 * 1080 * 1.5));
+ });
+
+ test("a 4K decoder costs more than a 1080p one by the area", () => {
+ expect(decodedFrameBytes(3840 * 2160)).toBe(decodedFrameBytes(1920 * 1080) * 4);
+ });
+});
diff --git a/site/src/cut/lib/memoryBudget.ts b/site/src/cut/lib/memoryBudget.ts
new file mode 100644
index 00000000..5fa4696e
--- /dev/null
+++ b/site/src/cut/lib/memoryBudget.ts
@@ -0,0 +1,226 @@
+/**
+ * What the editor is allowed to hold in memory, and what it is holding.
+ *
+ * A cut is built out of expensive pictures. Decoded frames sit in the
+ * platform decoder, canvases sit behind every source the pool keeps warm, and
+ * the bytes of the files themselves sit in each reader's cache. Every one of
+ * those is capped somewhere, but the caps were written one subsystem at a
+ * time, in whatever currency that subsystem thinks in — a count of decoders, a
+ * number of source pixels, a number of canvas pixels, a number of open files.
+ * Nothing added them up, so the tab's real ceiling was whatever those numbers
+ * happened to sum to on the machine the change was tuned on.
+ *
+ * This is the one place the total is decided. Holders register what they are
+ * standing on, in bytes, and ask what they may stand on: the smaller of the
+ * size they were tuned to want and their share of the tab's ceiling. On a
+ * machine with memory to spare the tuned size is the smaller one and nothing
+ * about the preview changes. On a machine without, the share binds first and
+ * every holder sheds together, so no one subsystem eats the room the others
+ * needed.
+ *
+ * The ceiling is a model. The memory that actually runs a machine out is
+ * memory a page cannot see: decoded frames live in the platform's decoder
+ * process, canvas backing lives in the GPU process, and
+ * `measureUserAgentSpecificMemory` — which needs a cross-origin isolation this
+ * page does not have — reports the size of neither. So the accounting is done
+ * at the allocation sites, where the sizes are known exactly.
+ */
+
+/** Where the tab's memory goes. Each is a family of allocations with one
+ * eviction story, so a bucket is what a share can sensibly be given to. */
+export type MemoryBucket =
+ /** Frames held inside the platform decoder, on behalf of live decoders. */
+ | "decoders"
+ /** Canvas backing: the rings frames land on, and the warm shelf. */
+ | "canvases"
+ /** File bytes held by open readers. */
+ | "reads"
+ /** Decoded sound waiting to be scheduled. */
+ | "audio"
+ /** Rasterized pictures: element bitmaps, thumbnails, mattes. */
+ | "pictures";
+
+const BUCKETS: MemoryBucket[] = ["decoders", "canvases", "reads", "audio", "pictures"];
+
+/**
+ * Each bucket's share of the ceiling.
+ *
+ * The split follows what the buckets actually cost at the sizes people edit
+ * at: a live 4K decoder holds hundreds of megabytes of frames where the
+ * canvases drawn from it hold tens, and the readers under both hold the file
+ * bytes those frames were decoded from. The shares sum to one, and a bucket
+ * holding less than its share does not lend the remainder — a holder that
+ * grows into borrowed room has to give it back mid-gesture, which is the churn
+ * the caps exist to prevent.
+ */
+const SHARES: Record<MemoryBucket, number> = {
+ decoders: 0.4,
+ canvases: 0.25,
+ reads: 0.25,
+ audio: 0.05,
+ pictures: 0.05,
+};
+
+/**
+ * The tab's share of the machine.
+ *
+ * A quarter of what the browser reports leaves the operating system, the
+ * browser's own processes and the person's other tabs the rest. Past that the
+ * machine starts swapping, and a swapping machine previews worse than one
+ * decoding at half the size — the picture stops for a page fault, which no
+ * amount of decode headroom recovers.
+ */
+const CEILING_SHARE = 0.25;
+/**
+ * What to assume when the browser does not say.
+ *
+ * `deviceMemory` is Chrome and Edge only; Safari and Firefox report nothing.
+ * Eight is also the largest number Chrome will report, so it already means
+ * "eight or more" — assuming it for the browsers that stay silent puts them
+ * on the same footing as the machines we can see, which are overwhelmingly
+ * the same machines.
+ */
+const ASSUMED_GB = 8;
+/** The smallest ceiling worth honouring: below this the editor cannot hold a
+ * single 1080p source open, so a machine reporting less memory still gets this
+ * much. */
+const CEILING_MIN = 384 * 2 ** 20;
+
+const GB = 2 ** 30;
+
+let ceiling = 0;
+
+/**
+ * Bytes the editor may hold, decided once.
+ *
+ * `deviceMemory` is coarse by design — it rounds to a power of two and caps at
+ * eight — which is enough to tell the machines that need protecting from the
+ * machines that do not, and it is the only figure a page is given.
+ */
+export function memoryCeiling(): number {
+ if (ceiling) return ceiling;
+ const reported =
+ typeof navigator === "undefined"
+ ? 0
+ : ((navigator as unknown as { deviceMemory?: number }).deviceMemory ?? 0);
+ const gb = reported > 0 ? reported : ASSUMED_GB;
+ ceiling = Math.max(CEILING_MIN, Math.round(gb * GB * CEILING_SHARE));
+ return ceiling;
+}
+
+/** Bytes a holder in `bucket` may stand on: the smaller of the size it was
+ * tuned to want and its share of the ceiling. */
+export function allowance(bucket: MemoryBucket, tuned: number): number {
+ const share = Math.round(memoryCeiling() * SHARES[bucket]);
+ if (share < tuned) bound[bucket] = true;
+ return Math.min(tuned, share);
+}
+
+/** Buckets whose share came in under what they were tuned to want: the
+ * ceiling is deciding their size. Read and cleared by whoever is reporting;
+ * it says the machine is the constraint. */
+const bound: Record<MemoryBucket, boolean> = {
+ decoders: false,
+ canvases: false,
+ reads: false,
+ audio: false,
+ pictures: false,
+};
+
+/** Which buckets the ceiling is currently deciding the size of, clearing the
+ * record. */
+export function takeMemoryPressure(): MemoryBucket[] {
+ const hit = BUCKETS.filter((b) => bound[b]);
+ for (const b of hit) bound[b] = false;
+ return hit;
+}
+
+type Reporter = () => number;
+
+const holders = new Map<MemoryBucket, Set<Reporter>>();
+
+/**
+ * Say what a cache is standing on, so the total can be reported.
+ *
+ * Reporters are pulled: a cache that has to remember to announce every
+ * allocation eventually forgets one, and the number it forgot is the number a
+ * report needs. Returns the way to stop reporting.
+ */
+export function holdMemory(bucket: MemoryBucket, bytes: Reporter): () => void {
+ let set = holders.get(bucket);
+ if (!set) holders.set(bucket, (set = new Set()));
+ set.add(bytes);
+ return () => {
+ set.delete(bytes);
+ };
+}
+
+export type MemoryUsage = Record<MemoryBucket, number> & { total: number };
+
+/** What every registered cache is holding right now, by bucket. */
+export function memoryUsage(): MemoryUsage {
+ const out = { decoders: 0, canvases: 0, reads: 0, audio: 0, pictures: 0, total: 0 };
+ for (const bucket of BUCKETS) {
+ let n = 0;
+ for (const bytes of holders.get(bucket) ?? []) n += bytes();
+ out[bucket] = n;
+ out.total += n;
+ }
+ return out;
+}
+
+/** Least time between two walks of every reporter, for callers that want a
+ * number rather than a peak. */
+const TOTAL_TTL_MS = 1_000;
+let totalAt = 0;
+let total = 0;
+
+/**
+ * What everything is holding, from a walk no more than a second old.
+ *
+ * Reading the total walks every cache that reports one. A caller on a path
+ * that is already in trouble — the record of a frame that blocked — wants the
+ * number without paying that walk again for a figure that cannot have moved
+ * far since the last one.
+ */
+export function memoryTotalCached(): number {
+ const now = Date.now();
+ if (now - totalAt >= TOTAL_TTL_MS) {
+ totalAt = now;
+ total = memoryUsage().total;
+ }
+ return total;
+}
+
+/**
+ * The JavaScript heap, where a browser reports one.
+ *
+ * It is a small part of what an editor holds — the pictures are all outside it
+ * — and Chrome quantizes what it returns. It is worth reporting anyway: a heap
+ * climbing across a session is the signature of a cache that never evicts,
+ * which the modeled numbers beside it cannot show.
+ */
+export function heapBytes(): number {
+ const perf = typeof performance === "undefined" ? null : performance;
+ const mem = (perf as unknown as { memory?: { usedJSHeapSize?: number } } | null)?.memory;
+ return mem?.usedJSHeapSize ?? 0;
+}
+
+const MB = 2 ** 20;
+
+/** Bytes as whole megabytes, which is the resolution every report of this
+ * wants. */
+export const mb = (bytes: number): number => Math.round(bytes / MB);
+
+/** Four bytes a pixel: the canvas backing behind a picture of that size. */
+export const canvasBytes = (pixels: number): number => pixels * 4;
+
+/**
+ * A decoded frame of a picture that size.
+ *
+ * Hardware decoders hand back planar 4:2:0 — a full-resolution luma plane and
+ * two quarter-resolution chroma planes — which is where the three halves come
+ * from. A frame drawn at a smaller size still costs this: the decode happens
+ * at the file's own resolution and the scaling happens after.
+ */
+export const decodedFrameBytes = (pixels: number): number => Math.round(pixels * 1.5);
diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.ts
index 2769604c..b7014a39 100644
--- a/site/src/cut/lib/previewMixer.ts
+++ b/site/src/cut/lib/previewMixer.ts
@@ -44,6 +44,7 @@ import {
} from "@donkeycut/effects-kit";
import type { WrappedAudioBuffer } from "mediabunny";
import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";
+import { allowance, holdMemory } from "./memoryBudget";
import { meterAudioLate, meterClockJump } from "./perfTrace";
import { timeStretch } from "./timeStretch";
@@ -67,6 +68,21 @@ const GROUP_PULLS = 128;
/** A stretched window is resynthesised on the main thread, so it is kept short
* enough that the work lands inside a frame or two. */
const WINDOW_STRETCHED_S = 4;
+/**
+ * Stretched windows kept in hand.
+ *
+ * Two dozen four-second windows of stereo at 48kHz — enough that stepping back
+ * over a cut replays what it just played, and bounded because everything older
+ * than that is a full decode away regardless.
+ */
+const STRETCH_CACHE_BYTES = 24 * WINDOW_STRETCHED_S * 48_000 * 2 * 4;
+
+/** What a set of decoded windows costs: float samples, one per channel. */
+const decodedBytes = (held: Map<string, AudioBuffer>): number => {
+ let n = 0;
+ for (const b of held.values()) n += b.length * b.numberOfChannels * 4;
+ return n;
+};
/** Schedule more of a voice once the playhead is this close to running out.
* The scheduling runs on the main thread, and what is scheduled is all the
* sound there is while that thread is blocked — a delete that takes seconds
@@ -219,6 +235,7 @@ export class PreviewMixer {
* scheduled is placed against this. */
private anchor: { timeline: number; ctx: number; wall: number } | null = null;
private decoded = new Map<string, AudioBuffer>();
+ private readonly releaseMemory = holdMemory("audio", () => decodedBytes(this.decoded));
/** Whether the clock is running. */
get running(): boolean {
@@ -694,6 +711,11 @@ export class PreviewMixer {
}
}
+ /** Let go of every decoded window. The next play decodes them again. */
+ releaseCaches(): void {
+ this.decoded.clear();
+ }
+
/** Drop the decode cache for a source that has been replaced. */
private forget(url: string): void {
for (const key of [...this.decoded.keys()]) {
@@ -707,6 +729,7 @@ export class PreviewMixer {
this.teardownFx();
this.fxKey = "";
this.decoded.clear();
+ this.releaseMemory();
void this.ctx?.close();
this.ctx = null;
this.master = null;
@@ -956,8 +979,18 @@ export class PreviewMixer {
const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);
if (!raw) return null;
const buffer = this.stretch(raw, 1 / live.speed);
- if (this.decoded.size > 24) this.decoded.clear();
+ // Oldest first until the cache is back inside its share. Emptying the
+ // whole cache would throw away the window the playhead is standing on along
+ // with the ones it has finished, and the next frame would pay a full decode
+ // for sound it was already holding.
this.decoded.set(key, buffer);
+ const cap = allowance("audio", STRETCH_CACHE_BYTES);
+ let cached = decodedBytes(this.decoded);
+ for (const [oldest, buf] of this.decoded) {
+ if (cached <= cap || this.decoded.size <= 1) break;
+ cached -= buf.length * buf.numberOfChannels * 4;
+ this.decoded.delete(oldest);
+ }
return buffer;
}
diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.ts
index f64909da..b80e0c82 100644
--- a/site/src/cut/lib/textRender.ts
+++ b/site/src/cut/lib/textRender.ts
@@ -13,6 +13,7 @@ import {
type WordDraw,
} from "@donkeycut/effects-kit";
import type { CSSProperties } from "react";
+import { holdMemory } from "./memoryBudget";
import { createRasterCanvas, decodeRasterImage, rasterCanvasToPng } from "./raster";
import { fontStack, type MediaAsset, type Overlay } from "./types";
@@ -78,6 +79,16 @@ export function wordDrawCss(d: WordDraw, lineHeight = LINE_HEIGHT): CSSPropertie
/** Decoded sticker images by asset id, shared across one page's renders. An
* <img> decode handles SVG too (createImageBitmap on an SVG blob does not). */
const stickerCache = new Map<string, Promise<StickerImage | null>>();
+/** What each decoded sticker costs, filled in as they land. The cache above
+ * holds promises, which cannot be asked their size; this is what makes the
+ * pictures the page is holding countable. */
+const stickerBytes = new Map<string, number>();
+
+holdMemory("pictures", () => {
+ let n = 0;
+ for (const bytes of stickerBytes.values()) n += bytes;
+ return n;
+});
function decodeSticker(url: string): Promise<StickerImage | null> {
return fetch(url)
@@ -105,6 +116,7 @@ export function cutRenderEnv(assets: MediaAsset[]): RenderEnv {
hit = decodeSticker(asset.url).then((img) => {
// A failed decode is not worth pinning; the next render retries.
if (!img) stickerCache.delete(asset.id);
+ else stickerBytes.set(asset.id, img.width * img.height * 4);
return img;
});
stickerCache.set(asset.id, hit);