Open source CapCut alternative.
TypeScript
perf(site): one memory ceiling for the tab, shared by every cache
David Lebf5cc6ea2652
diff --git a/site/src/cut/lib/behindPass.ts b/site/src/cut/lib/behindPass.tsindex 61fb7bf4..d5867d1d 100644--- a/site/src/cut/lib/behindPass.ts+++ b/site/src/cut/lib/behindPass.ts@@ -19,6 +19,7 @@ import {wordSampleWindows,} from "@donkeycut/effects-kit";import { personSegmenter, segmentSubjectAlpha } from "./cutout";+import { holdMemory } from "./memoryBudget";import { createRasterCanvas } from "./raster";import { renderElementPng } from "./textRender";import {@@ -121,11 +122,32 @@ export class SubjectMaskCompositor {private segmenter: Segmenter | null = null;private segKicked = false;- // One picture per element, and one per emphasized word when the element- // lights its words as they are said — keyed by the element object, so an- // edit hands back a fresh picture rather than a stale one.- private rasters = new WeakMap<Overlay, Map<number, ImageBitmap>>();- private pending = new WeakMap<Overlay, Set<number>>();+ /**+ * One picture per element, and one per emphasized word when the element+ * lights its words as they are said.+ *+ * Filed under the element's id and stamped with the element object the+ * picture was drawn from. The store hands back a new object for every edit,+ * so a stamp that no longer matches is a picture of an element that has since+ * changed: it closes and redraws. Keying on the object itself would say the+ * same thing, and would leave every superseded picture holding pixels outside+ * the heap that only `close()` gives back, with a keystroke's worth minted+ * per keystroke.+ *+ * `pending` rides in the entry so it is dropped whenever the pictures are.+ * It marks a draw already asked for, and a marker outliving the picture it+ * was asked for would leave the element blank for good.+ */+ private rasters = new Map<+ string,+ { of: Overlay; byWord: Map<number, ImageBitmap>; pending: Set<number> }+ >();+ private readonly releaseMemory = holdMemory("pictures", () => {+ let n = 0;+ for (const e of this.rasters.values())+ for (const b of e.byWord.values()) n += b.width * b.height * 4;+ return n;+ });private person: HTMLCanvasElement | null = null;private small: HTMLCanvasElement | null = null;private mask: { at: number; alpha: HTMLCanvasElement | null } = { at: -1e9, alpha: null };@@ -143,6 +165,23 @@ export class SubjectMaskCompositor {});}+ /** This element's entry, emptied first when the pictures in it were drawn+ * from an older version of the element. */+ private entryFor(o: Overlay) {+ let entry = this.rasters.get(o.id);+ if (entry && entry.of !== o) {+ for (const b of entry.byWord.values()) b.close();+ entry.byWord.clear();+ entry.pending.clear();+ entry.of = o;+ }+ if (!entry) {+ entry = { of: o, byWord: new Map(), pending: new Set() };+ this.rasters.set(o.id, entry);+ }+ return entry;+ }+private rasterFor(o: Overlay,w: number,@@ -150,12 +189,11 @@ export class SubjectMaskCompositor {assets: MediaAsset[],word: number): ImageBitmap | null {- const hit = this.rasters.get(o)?.get(word);+ const entry = this.entryFor(o);+ const hit = entry.byWord.get(word);if (hit) return hit;- const inFlight = this.pending.get(o) ?? new Set<number>();- if (!inFlight.has(word)) {- inFlight.add(word);- this.pending.set(o, inFlight);+ if (!entry.pending.has(word)) {+ entry.pending.add(word);// Neutral picture: position aside, the per-frame pose owns rotation and// opacity, so baking them here would apply each of them twice.void this.drawRaster(o, w, h, assets, word).catch(() => {});@@ -182,9 +220,29 @@ export class SubjectMaskCompositor {assets);const bmp = await createImageBitmap(png);- const byWord = this.rasters.get(o) ?? new Map<number, ImageBitmap>();- byWord.set(word, bmp);- this.rasters.set(o, byWord);+ // The element may have been edited, or the pictures released, while this+ // one was being drawn; either way its entry is gone or stamped with a+ // different object, and this picture is of the wrong thing.+ const entry = this.rasters.get(o.id);+ if (!entry || entry.of !== o) {+ bmp.close();+ return;+ }+ entry.byWord.get(word)?.close();+ entry.byWord.set(word, bmp);+ }++ /** Give back every picture this pass is holding. The pass stays usable and+ * draws them again when it is next asked. */+ clear(): void {+ for (const e of this.rasters.values()) for (const b of e.byWord.values()) b.close();+ this.rasters.clear();+ }++ /** Give the pictures back and stop reporting what this pass holds. */+ dispose(): void {+ this.clear();+ this.releaseMemory();}/** Export path: everything resident before the first frame draws. */@@ -199,7 +257,7 @@ export class SubjectMaskCompositor {// An element that lights its words needs one picture per word, all of// them resident before the first frame draws.rasterSlices(o).map(async (word) => {- if (this.rasters.get(o)?.has(word)) return;+ if (this.entryFor(o).byWord.has(word)) return;try {await this.drawRaster(o, w, h, assets, word);} catch {@@ -342,6 +400,18 @@ export class SubjectMaskCompositor {t: number,opts: { minMaskInterval?: number } = {}): void {+ // A deleted element's picture has nothing left to draw it. Filing by id+ // means the entry outlives the element, so the document's own list is what+ // says which entries are still owed — checked only when there are more of+ // them than elements, which is the only way one can be stale.+ if (this.rasters.size > overlays.length) {+ const ids = new Set(overlays.map((o) => o.id));+ for (const [id, e] of this.rasters) {+ if (ids.has(id)) continue;+ for (const b of e.byWord.values()) b.close();+ this.rasters.delete(id);+ }+ }const active = behindOverlaysAt(overlays, t);const wantsMatte = overlays.some((o) => frontSubjectOverlay(o) && drawable(o) && t >= o.start && t <= o.enddiff --git a/site/src/cut/lib/exportRender.ts b/site/src/cut/lib/exportRender.tsindex a0ee5038..40751f2d 100644--- a/site/src/cut/lib/exportRender.ts+++ b/site/src/cut/lib/exportRender.ts@@ -32,6 +32,7 @@ import { renderMix, type MixClip, type MixItem, type MixSpec } from "./audioMix"import { FrameCompositor, MISSING_FRAME, type Frame } from "./composite";import { overlayPlan, trackZeroPlan } from "./framePlan";import { frameSink, openMedia, videoTrackOf } from "./mediaRead";+import { allowance, canvasBytes } from "./memoryBudget";import { getClipSpans, overlayLayers, projectDuration, spanSequence } from "./store";import { captionStyle, cueOverlay, cueWordFrames, laneCues, laneHidden, subtitleLaneCount, trackPos } from "./subtitles";import { applyEffectToCanvas, evalOverlayFrame, grainTile, isAudioEffect, isMaskAnimated, isOverlayAnimated, maskFrameAt, MATTE_FPS, matteLumaToAlpha, planAnimatedLayers, type LottieHandle, type OverlayAnim, type PaintPhase } from "@donkeycut/effects-kit";@@ -126,6 +127,18 @@ export function bitrateFor(settings: ExportSettings): number {/** The frame reader for one open video source. */type ClipSink = ReturnType<typeof frameSink>;+/** Canvases one clip reader cycles. Small: a render draws one frame at a time+ * and never looks back, so the pool only has to keep the frame being drawn+ * clear of the frame being decoded. */+const READER_POOL = 4;+/** Readers a render keeps open when memory is not the constraint. A cut drawn+ * from more files than this reopens the ones it comes back to. */+const READERS_TUNED = 12;+/** Frames a reader may go unasked-for before it is a candidate to close. One+ * frame's own readers are never candidates, so what a join or a stack of+ * layers needs together is never taken away from the frame needing it. */+const READER_GRACE = 2;+/** A clip's source time at timeline time `t`. */function sourceTimeAt(span: ClipSpan, t: number): number {const speed = span.clip.speed && span.clip.speed > 0 ? span.clip.speed : 1;@@ -153,6 +166,20 @@ export class ClipReader {* the preview holds the rest — the rebuilt reader decodes in software,* which always opens. */private software = false;+ /** The frame number this reader was last asked for, for the painter's+ * between-frames eviction. */+ usedAt = 0;++ /**+ * Pixels in a frame of the source.+ *+ * The sink is opened with no size, so its canvases are the file's own — a+ * 4K clip in a 1080p render still cycles 4K canvases. Sizing the budget off+ * the render's canvas would under-count that by the ratio between them.+ */+ get sourcePixels(): number {+ return (this.asset.width ?? 1920) * (this.asset.height ?? 1080);+ }constructor(private asset: MediaAsset,@@ -188,7 +215,7 @@ export class ClipReader {// A small pool keeps the render's canvas allocation flat over thousands// of frames.this.sink = frameSink(track, undefined, {- poolSize: 4,+ poolSize: READER_POOL,...(this.software ? { software: true } : {}),});return this.sink;@@ -826,6 +853,8 @@ export class FramePainter {private comp: FrameCompositor;private stamps: StampCache;private readers = new Map<string, ClipReader>();+ /** Frames drawn, which is the clock the readers' eviction order runs on. */+ private frameNo = 0;private behind: SubjectMaskCompositor | null = null;private fxScratch: RasterSurface | null = null;/** This frame's removal mattes, one alpha image per keyed clip — fetched@@ -907,12 +936,58 @@ export class FramePainter {for (const sp of list) this.spanOfClip.set(sp.clip.id, sp);}+ /**+ * The reader for one asset, opening it if this frame is the first to ask.+ *+ * Readers are kept between frames because a render walks time forward and+ * asks the same clip for thousands of consecutive frames; reopening one per+ * frame would pay a keyframe seek for every one of them. They are not kept+ * forever: each holds a decoder and a pool of canvases at the render size, so+ * a cut drawing from a hundred files would otherwise finish the render+ * holding a hundred decoders. Past the budget the least recently drawn are+ * let go, which costs one reopen if the cut returns to them.+ */private readerFor(asset: MediaAsset): ClipReader {let r = this.readers.get(asset.id);if (!r) this.readers.set(asset.id, (r = new ClipReader(asset, () => this.resolve(asset))));+ r.usedAt = this.frameNo;return r;}+ /**+ * Close the readers this cut has moved on from.+ *+ * Readers are kept between frames because a render walks time forward and+ * asks the same clip for thousands of consecutive frames; reopening one per+ * frame would pay a keyframe seek for every one. They are not kept forever:+ * each holds a decoder and a pool of canvases at the *source's* own size, so+ * a cut drawing from a hundred files would otherwise end the render holding a+ * hundred decoders.+ *+ * This runs between frames, before anything has been asked for, and spares+ * everything the last couple of frames used. A frame reaches for several+ * readers at once — a join's two sides, a backdrop, a matte, every overlay+ * layer — and closing one the frame is midway through would leave the+ * compositor drawing from a disposed decoder. Closing them here, past the+ * grace, cannot.+ */+ private evictReaders(): void {+ const cost = (r: ClipReader) => canvasBytes(READER_POOL * r.sourcePixels);+ const cap = allowance("canvases", READERS_TUNED * canvasBytes(READER_POOL * 1920 * 1080));+ let held = 0;+ for (const r of this.readers.values()) held += cost(r);+ if (held <= cap) return;+ const old = [...this.readers.entries()]+ .filter(([, r]) => this.frameNo - r.usedAt > READER_GRACE)+ .sort((a, b) => a[1].usedAt - b[1].usedAt);+ for (const [id, r] of old) {+ if (held <= cap) break;+ held -= cost(r);+ this.readers.delete(id);+ r.dispose();+ }+ }+/** Pull a removal clip's baked matte frame for timeline time `t` and stage* it, luma turned to alpha, where the compositor's provider reads. A clip* whose bake is still owed stages nothing. */@@ -952,6 +1027,8 @@ export class FramePainter {/** Draw the whole cut at timeline time `t` onto the canvas. */async drawAt(t: number): Promise<void> {+ this.frameNo++;+ this.evictReaders();const { canvas, comp, doc, stamps } = this;const W = canvas.width;const H = canvas.height;@@ -1047,6 +1124,7 @@ export class FramePainter {dispose(): void {this.stamps.dispose();+ this.behind?.dispose();for (const r of this.readers.values()) r.dispose();this.readers.clear();}diff --git a/site/src/cut/lib/frameSource.test.ts b/site/src/cut/lib/frameSource.test.tsindex 6699554b..207f947a 100644--- a/site/src/cut/lib/frameSource.test.ts+++ b/site/src/cut/lib/frameSource.test.ts@@ -389,6 +389,34 @@ describe("FrameSourcePool", () => {expect(big.map((s) => s.suspended)).toEqual([true, true, true, true, false, false]);});+ test("says what it is holding, live sources included", () => {+ const pool = new FrameSourcePool(4);+ expect(pool.decoderBytes).toBe(0);+ expect(pool.canvasBytes).toBe(0);+ pool.beginFrame();+ pool.get("a", asset("a", 3840, 2160), 2160);+ // A live 4K decoder holds sixteen planar frames: about two hundred+ // megabytes, none of which the warm shelf's own number would ever show.+ expect(Math.round(pool.decoderBytes / 2 ** 20)).toBe(190);+ expect(pool.warmPixels).toBe(0);+ expect(pool.canvasBytes).toBeGreaterThan(0);+ // One 4K source is well inside the budget, so nothing stands down.+ for (let i = 0; i < 200; i++) pool.beginFrame();+ pool.evict();+ expect(pool.decoderBytes).toBeGreaterThan(0);++ // Standing down gives the decoder back and keeps the canvases, which is+ // the whole point of the warm shelf.+ const tight = new FrameSourcePool(0);+ tight.beginFrame();+ tight.get("a", asset("a", 1920, 1080), 1080);+ const canvases = tight.canvasBytes;+ for (let i = 0; i < 200; i++) tight.beginFrame();+ tight.evict();+ expect(tight.decoderBytes).toBe(0);+ expect(tight.canvasBytes).toBe(canvases);+ });+test("standing down costs a decoder, not a source", () => {const pool = new FrameSourcePool(1);pool.beginFrame();diff --git a/site/src/cut/lib/frameSource.ts b/site/src/cut/lib/frameSource.tsindex 2d789829..ebd7db89 100644--- a/site/src/cut/lib/frameSource.ts+++ b/site/src/cut/lib/frameSource.ts@@ -25,6 +25,7 @@import type { Input, InputVideoTrack, WrappedCanvas } from "mediabunny";import { frameSink, keyframeTimeAt, openMedia, videoTrackOf, type FrameCanvasSink } from "./mediaRead";+import { allowance, canvasBytes, decodedFrameBytes, holdMemory } from "./memoryBudget";import { meterPull, meterSource, meterWalk } from "./perfTrace";import type { MediaAsset } from "./types";@@ -305,6 +306,16 @@ const WARM_KEEP_MAX = 32;* at 1080p and holds four at 4K.*/const LIVE_PIXELS = 34e6;+/**+ * Decoded frames a live decoder is holding at once.+ *+ * The patched decode pump gives a stream a head start of this many packets and+ * counts what has gone in against what has come back out, so a stream that is+ * not being read stops here, short of the platform decoder. It is+ * the multiplier that turns a source's frame size into what its decoder costs+ * — see `site/patches/mediabunny+1.55.5.patch`.+ */+const DECODER_FRAMES = 16;/** A failed open tries again this much later, growing per attempt over the* first `RETRIES` tries, so a network blip heals within seconds. */const RETRY_MS = 1000;@@ -651,6 +662,25 @@ export class ClipFrameSource {return POOL * width * this.height;}+ /**+ * Every canvas this source is holding.+ *+ * `keptPixels` answers what a stood-down source would still cost, which is+ * what the warm shelf is filled by. A source being read holds that same pool+ * and, while a backward gesture is on it, two more: the fine windows at the+ * decode size and the coarse span at half of it. Those are the biggest thing+ * in the tab during a scrub, so a report of what the editor holds has to+ * count them.+ */+ get canvasPixels(): number {+ const one = this.keptPixels / POOL;+ return (+ this.keptPixels ++ (this.backSink ? one * BACK_POOL : 0) ++ (this.backCoarseSink ? (one / 4) * BACK_COARSE_POOL : 0)+ );+ }+/** Pixels in a frame of the file, which is what its decoder's held frames* cost whatever size the picture is drawn at. */get decodePixels(): number {@@ -658,6 +688,12 @@ export class ClipFrameSource {return (this.asset.width ?? 1920) * (this.asset.height ?? 1080);}+ /** What this source's decoder is holding in the platform's decoder, which is+ * nothing at all once it has stood down. */+ get decoderBytes(): number {+ return this.asleep ? 0 : decodedFrameBytes(this.decodePixels) * DECODER_FRAMES;+ }+/** The URL this source is reading. The pool compares it against the store's* current one, so a re-minted signed URL replaces the source under it. */get url(): string {@@ -1645,10 +1681,19 @@ export class FrameSourcePool {private sources = new Map<string, ClipFrameSource>();private tick = 0;+ private readonly release: () => void;+constructor(private budget = 10,private readonly onFrame: () => void = () => {}- ) {}+ ) {+ const stopDecoders = holdMemory("decoders", () => this.decoderBytes);+ const stopCanvases = holdMemory("canvases", () => this.canvasBytes);+ this.release = () => {+ stopDecoders();+ stopCanvases();+ };+ }/** Advance the clock the eviction order is measured on. */beginFrame(): void {@@ -1715,6 +1760,28 @@ export class FrameSourcePool {return n;}+ /** What the live decoders are holding in the platform's decoder. */+ get decoderBytes(): number {+ let n = 0;+ for (const s of this.sources.values()) n += s.decoderBytes;+ return n;+ }++ /**+ * Canvas backing behind every source, live and stood down.+ *+ * The warm shelf's own budget counts only the stood-down ones, because those+ * are the only ones it may close. This counts all of them, because all of+ * them are memory the machine has to find — a scrub holding two backward+ * windows open is the largest the number ever gets, and a report that left+ * it out would say the editor was idle at its busiest moment.+ */+ get canvasBytes(): number {+ let n = 0;+ for (const s of this.sources.values()) n += s.canvasPixels;+ return canvasBytes(n);+ }+/** Every open source and what it is doing, for the perf eval's pool dump. */debugState(): Record<string, unknown>[] {return [...this.sources.entries()].map(([id, s]) => ({ id, tick: this.tick, ...s.debugState() }));@@ -1757,14 +1824,17 @@ export class FrameSourcePool {live--;}- // Then by pixels: the live decoders are kept most recently read first- // until the source budget runs out, and the idle ones past it stand down.- let livePixels = 0;+ // Then by what those decoders hold: the live ones are kept most recently+ // read first until the budget runs out, and the idle ones past it stand+ // down. The budget is the smaller of what the preview was tuned to want+ // and this machine's share, so a laptop sheds decoders a desktop keeps.+ const decoderCap = allowance("decoders", decodedFrameBytes(LIVE_PIXELS) * DECODER_FRAMES);+ let liveBytes = 0;for (const [id, src] of [...this.sources.entries()].sort((a, b) => b[1].touched - a[1].touched)) {if (src.suspended) continue;- livePixels += src.decodePixels;- if (livePixels <= LIVE_PIXELS || this.tick - src.touched < EVICT_GRACE) continue;- poolLog(`pool suspend ${id} (pixels)`);+ liveBytes += src.decoderBytes;+ if (liveBytes <= decoderCap || this.tick - src.touched < EVICT_GRACE) continue;+ poolLog(`pool suspend ${id} (memory)`);src.suspend();}@@ -1772,13 +1842,14 @@ export class FrameSourcePool {// the canvas budget runs out, and what does not fit closes for real. Only// stood-down sources are on the shelf, so nothing holding a decoder — and// nothing the picture is being drawn from — is ever closed here.- let pixels = 0;+ const warmCap = allowance("canvases", canvasBytes(WARM_PIXELS));+ let held = 0;let warm = 0;for (const [id, src] of [...this.sources.entries()].sort((a, b) => b[1].touched - a[1].touched)) {if (!src.suspended) continue;- pixels += src.keptPixels;+ held += canvasBytes(src.keptPixels);warm++;- if (pixels <= WARM_PIXELS && warm <= WARM_KEEP_MAX) continue;+ if (held <= warmCap && warm <= WARM_KEEP_MAX) continue;poolLog(`pool close ${id}`);src.close();this.sources.delete(id);@@ -1790,6 +1861,12 @@ export class FrameSourcePool {for (const s of this.sources.values()) s.close();this.sources.clear();}++ /** Close every source and stop reporting what this pool holds. */+ dispose(): void {+ this.closeAll();+ this.release();+ }}/**diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.tsindex 47a6661c..98d14ab7 100644--- a/site/src/cut/lib/mediaRead.ts+++ b/site/src/cut/lib/mediaRead.ts@@ -44,6 +44,7 @@ import {import { confirmEngine, engineConnected, isEngineUrl } from "./api";import { resolveRegisteredBlob } from "./backend/browser/registry";import { chunkSourceOptions } from "./chunkCache";+import { allowance, holdMemory } from "./memoryBudget";/** What a file turns out to be, read from its container. */export interface MediaProbe {@@ -176,22 +177,72 @@ function trackedFetch(url: string): typeof fetch {* ranged fetches of blob URLs are unreliable across browsers. Signed cloud* media reads through the chunk cache (chunkCache.ts), so bytes touched once* are on disk for the next read of the same object. */+/**+ * Readers open at once, at the busiest the editor gets.+ *+ * The preview's decoder pool, the timeline's edge readers, and a render's+ * per-clip readers all hold one each. The number is what a per-reader cache has+ * to be divided by for the total to stay inside the tab's share: sizing each+ * cache as if it were the only one is how a cache with a sensible-looking limit+ * becomes a gigabyte.+ */+const READERS = 16;+/** Bytes of the file one reader keeps in memory. Enough that a walk reading+ * forward never re-fetches what it just read, and small enough that the+ * readers together stay inside the tab's ceiling. */+const READ_CACHE = 32 * 2 ** 20;++/** Chunked readers open right now, for the memory report. */+let openReaders = 0;++/** Decided on the first open and held: the ceiling does not move, so asking+ * again would return the same number and re-record the same pressure. Reading+ * what the readers hold must not itself count as a request for room. */+let readCache = 0;+const readCacheSize = (): number =>+ (readCache ||= Math.round(allowance("reads", READ_CACHE * READERS) / READERS));++holdMemory("reads", () => openReaders * readCacheSize());+export function openMedia(src: string | Blob): Input {const blob = typeof src === "string" ? resolveRegisteredBlob(src) ?? src : src;let source;+ let chunkedRead = false;if (typeof blob === "string") {const chunked = chunkSourceOptions(blob);source = chunked- ? new CustomSource({ ...chunked, prefetchProfile: "network", maxCacheSize: 64 * 2 ** 20 })+ ? new CustomSource({+ ...chunked,+ prefetchProfile: "network",+ maxCacheSize: readCacheSize(),+ }): new UrlSource(blob, {parallelism: urlParallelism(blob),getRetryDelay: urlRetryDelay,fetchFn: trackedFetch(blob),});+ chunkedRead = !!chunked;} else {source = new BlobSource(blob);}- return new Input({ formats: ALL_FORMATS, source });+ const input = new Input({ formats: ALL_FORMATS, source });+ // The cache lives inside the source, where nothing can ask it how big it has+ // grown. Counting the readers that own one is what makes the total+ // reportable, and the count is only right if it is taken where the reader is+ // let go — so the dispose every caller already makes is where it is taken.+ if (chunkedRead) {+ openReaders++;+ const dispose = input.dispose.bind(input);+ let closed = false;+ input.dispose = () => {+ if (!closed) {+ closed = true;+ openReaders--;+ }+ return dispose();+ };+ }+ return input;}/** Run `fn` against an open input and dispose it however that ends. */diff --git a/site/src/cut/lib/memoryBudget.test.ts b/site/src/cut/lib/memoryBudget.test.tsnew file mode 100644index 00000000..dac69b8a--- /dev/null+++ b/site/src/cut/lib/memoryBudget.test.ts@@ -0,0 +1,105 @@+import { afterEach, describe, expect, test } from "bun:test";+import {+ allowance,+ canvasBytes,+ decodedFrameBytes,+ heapBytes,+ holdMemory,+ mb,+ memoryCeiling,+ memoryUsage,+ takeMemoryPressure,+} from "./memoryBudget";++const MB = 2 ** 20;++const drops: (() => void)[] = [];+const hold = (bucket: Parameters<typeof holdMemory>[0], bytes: number) => {+ drops.push(holdMemory(bucket, () => bytes));+};++afterEach(() => {+ for (const drop of drops.splice(0)) drop();+ takeMemoryPressure();+});++describe("the ceiling", () => {+ test("is a share of the machine, decided once", () => {+ // The environment the tests run in reports no `deviceMemory`, which is the+ // same silence Safari and Firefox give a real page: the assumed size+ // stands in, and a quarter of it is the tab's.+ expect(memoryCeiling()).toBe(2 * 2 ** 30);+ expect(memoryCeiling()).toBe(memoryCeiling());+ });++ test("hands a bucket the smaller of its tuning and its share", () => {+ // A size the machine can afford is left alone, so a preview tuned on a+ // machine with room behaves there exactly as it was tuned to.+ const small = 4 * MB;+ expect(allowance("decoders", small)).toBe(small);+ expect(takeMemoryPressure()).toEqual([]);++ // A size it cannot is cut to the share, and the cut is recorded: the+ // machine is deciding, not the tuning.+ const huge = 99 * 2 ** 30;+ expect(allowance("decoders", huge)).toBeLessThan(huge);+ expect(allowance("decoders", huge)).toBe(Math.round(memoryCeiling() * 0.4));+ expect(takeMemoryPressure()).toEqual(["decoders"]);+ // Reading the pressure clears it, so a later window reports its own.+ expect(takeMemoryPressure()).toEqual([]);+ });++ test("splits the whole ceiling between the buckets and no more", () => {+ const huge = 99 * 2 ** 30;+ const shares =+ allowance("decoders", huge) ++ allowance("canvases", huge) ++ allowance("reads", huge) ++ allowance("audio", huge) ++ allowance("pictures", huge);+ expect(shares).toBeLessThanOrEqual(memoryCeiling());+ expect(shares).toBeGreaterThan(memoryCeiling() * 0.99);+ });+});++describe("what is being held", () => {+ test("adds up what every cache reports, by bucket", () => {+ hold("decoders", 10 * MB);+ hold("decoders", 5 * MB);+ hold("reads", 3 * MB);+ const usage = memoryUsage();+ expect(mb(usage.decoders)).toBe(15);+ expect(mb(usage.reads)).toBe(3);+ expect(mb(usage.canvases)).toBe(0);+ expect(mb(usage.total)).toBe(18);+ });++ test("forgets a cache that has been let go", () => {+ hold("audio", 8 * MB);+ expect(mb(memoryUsage().audio)).toBe(8);+ drops.pop()!();+ expect(memoryUsage().audio).toBe(0);+ });++ test("reports nothing rather than guessing when the browser is silent", () => {+ // No `performance.memory` here, as in every browser but Chrome. A zero is+ // the honest answer; a made-up number would read as a measurement.+ expect(heapBytes()).toBe(0);+ });+});++describe("what a picture costs", () => {+ test("counts canvas backing at four bytes a pixel", () => {+ expect(canvasBytes(1920 * 1080)).toBe(1920 * 1080 * 4);+ });++ test("counts a decoded frame at the planar size the decoder hands back", () => {+ // Three halves of the luma plane: full-resolution luma, two quarter-+ // resolution chroma planes.+ expect(decodedFrameBytes(1920 * 1080)).toBe(Math.round(1920 * 1080 * 1.5));+ });++ test("a 4K decoder costs more than a 1080p one by the area", () => {+ expect(decodedFrameBytes(3840 * 2160)).toBe(decodedFrameBytes(1920 * 1080) * 4);+ });+});diff --git a/site/src/cut/lib/memoryBudget.ts b/site/src/cut/lib/memoryBudget.tsnew file mode 100644index 00000000..5fa4696e--- /dev/null+++ b/site/src/cut/lib/memoryBudget.ts@@ -0,0 +1,226 @@+/**+ * What the editor is allowed to hold in memory, and what it is holding.+ *+ * A cut is built out of expensive pictures. Decoded frames sit in the+ * platform decoder, canvases sit behind every source the pool keeps warm, and+ * the bytes of the files themselves sit in each reader's cache. Every one of+ * those is capped somewhere, but the caps were written one subsystem at a+ * time, in whatever currency that subsystem thinks in — a count of decoders, a+ * number of source pixels, a number of canvas pixels, a number of open files.+ * Nothing added them up, so the tab's real ceiling was whatever those numbers+ * happened to sum to on the machine the change was tuned on.+ *+ * This is the one place the total is decided. Holders register what they are+ * standing on, in bytes, and ask what they may stand on: the smaller of the+ * size they were tuned to want and their share of the tab's ceiling. On a+ * machine with memory to spare the tuned size is the smaller one and nothing+ * about the preview changes. On a machine without, the share binds first and+ * every holder sheds together, so no one subsystem eats the room the others+ * needed.+ *+ * The ceiling is a model. The memory that actually runs a machine out is+ * memory a page cannot see: decoded frames live in the platform's decoder+ * process, canvas backing lives in the GPU process, and+ * `measureUserAgentSpecificMemory` — which needs a cross-origin isolation this+ * page does not have — reports the size of neither. So the accounting is done+ * at the allocation sites, where the sizes are known exactly.+ */++/** Where the tab's memory goes. Each is a family of allocations with one+ * eviction story, so a bucket is what a share can sensibly be given to. */+export type MemoryBucket =+ /** Frames held inside the platform decoder, on behalf of live decoders. */+ | "decoders"+ /** Canvas backing: the rings frames land on, and the warm shelf. */+ | "canvases"+ /** File bytes held by open readers. */+ | "reads"+ /** Decoded sound waiting to be scheduled. */+ | "audio"+ /** Rasterized pictures: element bitmaps, thumbnails, mattes. */+ | "pictures";++const BUCKETS: MemoryBucket[] = ["decoders", "canvases", "reads", "audio", "pictures"];++/**+ * Each bucket's share of the ceiling.+ *+ * The split follows what the buckets actually cost at the sizes people edit+ * at: a live 4K decoder holds hundreds of megabytes of frames where the+ * canvases drawn from it hold tens, and the readers under both hold the file+ * bytes those frames were decoded from. The shares sum to one, and a bucket+ * holding less than its share does not lend the remainder — a holder that+ * grows into borrowed room has to give it back mid-gesture, which is the churn+ * the caps exist to prevent.+ */+const SHARES: Record<MemoryBucket, number> = {+ decoders: 0.4,+ canvases: 0.25,+ reads: 0.25,+ audio: 0.05,+ pictures: 0.05,+};++/**+ * The tab's share of the machine.+ *+ * A quarter of what the browser reports leaves the operating system, the+ * browser's own processes and the person's other tabs the rest. Past that the+ * machine starts swapping, and a swapping machine previews worse than one+ * decoding at half the size — the picture stops for a page fault, which no+ * amount of decode headroom recovers.+ */+const CEILING_SHARE = 0.25;+/**+ * What to assume when the browser does not say.+ *+ * `deviceMemory` is Chrome and Edge only; Safari and Firefox report nothing.+ * Eight is also the largest number Chrome will report, so it already means+ * "eight or more" — assuming it for the browsers that stay silent puts them+ * on the same footing as the machines we can see, which are overwhelmingly+ * the same machines.+ */+const ASSUMED_GB = 8;+/** The smallest ceiling worth honouring: below this the editor cannot hold a+ * single 1080p source open, so a machine reporting less memory still gets this+ * much. */+const CEILING_MIN = 384 * 2 ** 20;++const GB = 2 ** 30;++let ceiling = 0;++/**+ * Bytes the editor may hold, decided once.+ *+ * `deviceMemory` is coarse by design — it rounds to a power of two and caps at+ * eight — which is enough to tell the machines that need protecting from the+ * machines that do not, and it is the only figure a page is given.+ */+export function memoryCeiling(): number {+ if (ceiling) return ceiling;+ const reported =+ typeof navigator === "undefined"+ ? 0+ : ((navigator as unknown as { deviceMemory?: number }).deviceMemory ?? 0);+ const gb = reported > 0 ? reported : ASSUMED_GB;+ ceiling = Math.max(CEILING_MIN, Math.round(gb * GB * CEILING_SHARE));+ return ceiling;+}++/** Bytes a holder in `bucket` may stand on: the smaller of the size it was+ * tuned to want and its share of the ceiling. */+export function allowance(bucket: MemoryBucket, tuned: number): number {+ const share = Math.round(memoryCeiling() * SHARES[bucket]);+ if (share < tuned) bound[bucket] = true;+ return Math.min(tuned, share);+}++/** Buckets whose share came in under what they were tuned to want: the+ * ceiling is deciding their size. Read and cleared by whoever is reporting;+ * it says the machine is the constraint. */+const bound: Record<MemoryBucket, boolean> = {+ decoders: false,+ canvases: false,+ reads: false,+ audio: false,+ pictures: false,+};++/** Which buckets the ceiling is currently deciding the size of, clearing the+ * record. */+export function takeMemoryPressure(): MemoryBucket[] {+ const hit = BUCKETS.filter((b) => bound[b]);+ for (const b of hit) bound[b] = false;+ return hit;+}++type Reporter = () => number;++const holders = new Map<MemoryBucket, Set<Reporter>>();++/**+ * Say what a cache is standing on, so the total can be reported.+ *+ * Reporters are pulled: a cache that has to remember to announce every+ * allocation eventually forgets one, and the number it forgot is the number a+ * report needs. Returns the way to stop reporting.+ */+export function holdMemory(bucket: MemoryBucket, bytes: Reporter): () => void {+ let set = holders.get(bucket);+ if (!set) holders.set(bucket, (set = new Set()));+ set.add(bytes);+ return () => {+ set.delete(bytes);+ };+}++export type MemoryUsage = Record<MemoryBucket, number> & { total: number };++/** What every registered cache is holding right now, by bucket. */+export function memoryUsage(): MemoryUsage {+ const out = { decoders: 0, canvases: 0, reads: 0, audio: 0, pictures: 0, total: 0 };+ for (const bucket of BUCKETS) {+ let n = 0;+ for (const bytes of holders.get(bucket) ?? []) n += bytes();+ out[bucket] = n;+ out.total += n;+ }+ return out;+}++/** Least time between two walks of every reporter, for callers that want a+ * number rather than a peak. */+const TOTAL_TTL_MS = 1_000;+let totalAt = 0;+let total = 0;++/**+ * What everything is holding, from a walk no more than a second old.+ *+ * Reading the total walks every cache that reports one. A caller on a path+ * that is already in trouble — the record of a frame that blocked — wants the+ * number without paying that walk again for a figure that cannot have moved+ * far since the last one.+ */+export function memoryTotalCached(): number {+ const now = Date.now();+ if (now - totalAt >= TOTAL_TTL_MS) {+ totalAt = now;+ total = memoryUsage().total;+ }+ return total;+}++/**+ * The JavaScript heap, where a browser reports one.+ *+ * It is a small part of what an editor holds — the pictures are all outside it+ * — and Chrome quantizes what it returns. It is worth reporting anyway: a heap+ * climbing across a session is the signature of a cache that never evicts,+ * which the modeled numbers beside it cannot show.+ */+export function heapBytes(): number {+ const perf = typeof performance === "undefined" ? null : performance;+ const mem = (perf as unknown as { memory?: { usedJSHeapSize?: number } } | null)?.memory;+ return mem?.usedJSHeapSize ?? 0;+}++const MB = 2 ** 20;++/** Bytes as whole megabytes, which is the resolution every report of this+ * wants. */+export const mb = (bytes: number): number => Math.round(bytes / MB);++/** Four bytes a pixel: the canvas backing behind a picture of that size. */+export const canvasBytes = (pixels: number): number => pixels * 4;++/**+ * A decoded frame of a picture that size.+ *+ * Hardware decoders hand back planar 4:2:0 — a full-resolution luma plane and+ * two quarter-resolution chroma planes — which is where the three halves come+ * from. A frame drawn at a smaller size still costs this: the decode happens+ * at the file's own resolution and the scaling happens after.+ */+export const decodedFrameBytes = (pixels: number): number => Math.round(pixels * 1.5);diff --git a/site/src/cut/lib/previewMixer.ts b/site/src/cut/lib/previewMixer.tsindex 2769604c..b7014a39 100644--- a/site/src/cut/lib/previewMixer.ts+++ b/site/src/cut/lib/previewMixer.ts@@ -44,6 +44,7 @@ import {} from "@donkeycut/effects-kit";import type { WrappedAudioBuffer } from "mediabunny";import { assembleAudio, decodeAudioSpan, openAudioWalk, type AudioWalk } from "./mediaRead";+import { allowance, holdMemory } from "./memoryBudget";import { meterAudioLate, meterClockJump } from "./perfTrace";import { timeStretch } from "./timeStretch";@@ -67,6 +68,21 @@ const GROUP_PULLS = 128;/** A stretched window is resynthesised on the main thread, so it is kept short* enough that the work lands inside a frame or two. */const WINDOW_STRETCHED_S = 4;+/**+ * Stretched windows kept in hand.+ *+ * Two dozen four-second windows of stereo at 48kHz — enough that stepping back+ * over a cut replays what it just played, and bounded because everything older+ * than that is a full decode away regardless.+ */+const STRETCH_CACHE_BYTES = 24 * WINDOW_STRETCHED_S * 48_000 * 2 * 4;++/** What a set of decoded windows costs: float samples, one per channel. */+const decodedBytes = (held: Map<string, AudioBuffer>): number => {+ let n = 0;+ for (const b of held.values()) n += b.length * b.numberOfChannels * 4;+ return n;+};/** Schedule more of a voice once the playhead is this close to running out.* The scheduling runs on the main thread, and what is scheduled is all the* sound there is while that thread is blocked — a delete that takes seconds@@ -219,6 +235,7 @@ export class PreviewMixer {* scheduled is placed against this. */private anchor: { timeline: number; ctx: number; wall: number } | null = null;private decoded = new Map<string, AudioBuffer>();+ private readonly releaseMemory = holdMemory("audio", () => decodedBytes(this.decoded));/** Whether the clock is running. */get running(): boolean {@@ -694,6 +711,11 @@ export class PreviewMixer {}}+ /** Let go of every decoded window. The next play decodes them again. */+ releaseCaches(): void {+ this.decoded.clear();+ }+/** Drop the decode cache for a source that has been replaced. */private forget(url: string): void {for (const key of [...this.decoded.keys()]) {@@ -707,6 +729,7 @@ export class PreviewMixer {this.teardownFx();this.fxKey = "";this.decoded.clear();+ this.releaseMemory();void this.ctx?.close();this.ctx = null;this.master = null;@@ -956,8 +979,18 @@ export class PreviewMixer {const raw = await decodeAudioSpan(live.url, sourceFrom, sourceTo);if (!raw) return null;const buffer = this.stretch(raw, 1 / live.speed);- if (this.decoded.size > 24) this.decoded.clear();+ // Oldest first until the cache is back inside its share. Emptying the+ // whole cache would throw away the window the playhead is standing on along+ // with the ones it has finished, and the next frame would pay a full decode+ // for sound it was already holding.this.decoded.set(key, buffer);+ const cap = allowance("audio", STRETCH_CACHE_BYTES);+ let cached = decodedBytes(this.decoded);+ for (const [oldest, buf] of this.decoded) {+ if (cached <= cap || this.decoded.size <= 1) break;+ cached -= buf.length * buf.numberOfChannels * 4;+ this.decoded.delete(oldest);+ }return buffer;}diff --git a/site/src/cut/lib/textRender.ts b/site/src/cut/lib/textRender.tsindex f64909da..b80e0c82 100644--- a/site/src/cut/lib/textRender.ts+++ b/site/src/cut/lib/textRender.ts@@ -13,6 +13,7 @@ import {type WordDraw,} from "@donkeycut/effects-kit";import type { CSSProperties } from "react";+import { holdMemory } from "./memoryBudget";import { createRasterCanvas, decodeRasterImage, rasterCanvasToPng } from "./raster";import { fontStack, type MediaAsset, type Overlay } from "./types";@@ -78,6 +79,16 @@ export function wordDrawCss(d: WordDraw, lineHeight = LINE_HEIGHT): CSSPropertie/** Decoded sticker images by asset id, shared across one page's renders. An* <img> decode handles SVG too (createImageBitmap on an SVG blob does not). */const stickerCache = new Map<string, Promise<StickerImage | null>>();+/** What each decoded sticker costs, filled in as they land. The cache above+ * holds promises, which cannot be asked their size; this is what makes the+ * pictures the page is holding countable. */+const stickerBytes = new Map<string, number>();++holdMemory("pictures", () => {+ let n = 0;+ for (const bytes of stickerBytes.values()) n += bytes;+ return n;+});function decodeSticker(url: string): Promise<StickerImage | null> {return fetch(url)@@ -105,6 +116,7 @@ export function cutRenderEnv(assets: MediaAsset[]): RenderEnv {hit = decodeSticker(asset.url).then((img) => {// A failed decode is not worth pinning; the next render retries.if (!img) stickerCache.delete(asset.id);+ else stickerBytes.set(asset.id, img.width * img.height * 4);return img;});stickerCache.set(asset.id, hit);