Skip to content

Open source CapCut alternative.

TypeScript

perf(site): the timeline's strips and the assets' pictures come off disk

David Led27e005d119c
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 4ed2b59b..4d099e1f 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -6,6 +6,7 @@ import { cloudBackend } from "./backend/cloud";
import { pollCloudJob } from "./cloudJob";
import { quotaErrorMessage } from "./backend/cloud";
import { encodeWav } from "./cloudTranscribe";
+import { allowance, canvasBytes, holdMemory } from "./memoryBudget";
import { normalizeLink } from "./link";
import { downloadFromUrl } from "./download";
import { startUpload, uploadInFlight } from "./importQueue";
@@ -44,7 +45,7 @@ import {
} from "./raster";
import { pickThumbTimes, type ThumbProbe } from "./filmstrip";
import { reportSwallowed } from "./report";
-import { useEditor } from "./store";
+import { assetIdsInUse, useEditor } from "./store";
import type { AssetBeats, AssetType, AudioClip, MediaAsset, ProjectSummary, StoredAsset, VideoClip, WatchKeepReason } from "./types";
import { contentRect, IMAGE_CLIP_SECONDS, mediaUrl } from "./types";
import { createDedupSelector, DEDUP_TUNING } from "./watch/dedupSelector";
@@ -2080,6 +2081,106 @@ async function refineSceneCuts(assetId: string, url: string, duration: number) {
// handle already is come back without re-reading the file — which is the whole
// difference between this and re-opening the source per request.
const EDGE_CACHE_CAP = 300;
+/** Canvases one strip sink cycles, and strip heights a reader keeps sinks
+ * for — a zoom in and the height it came from, so a wobble between two levels
+ * rebuilds neither. */
+const EDGE_SINK_POOL = 4;
+const EDGE_SINK_HEIGHTS = 2;
+
+/**
+ * What the timeline's own pictures cost.
+ *
+ * The captured frames are JPEG data URLs — text, two bytes a character in the
+ * heap — and each open reader cycles a pool of canvases per strip height it
+ * has been asked for. Both are held for as long as the timeline is on screen,
+ * so a report of what the tab holds counts them. The readers are counted from
+ * the sinks they actually opened: a reader asked for two heights holds two
+ * pools, and at 16:9 a pool's canvas is about twice its height across.
+ */
+const stripBytes = (): number => {
+ let n = 0;
+ for (const url of edgeCache.values()) n += url.length * 2;
+ for (const open of openEdgeReaders)
+ for (const h of open.sinks.keys()) n += canvasBytes(EDGE_SINK_POOL * h * h * 2);
+ return n;
+};
+
+/** Readers whose sinks have opened, for the count above. A reader still
+ * opening holds nothing yet. */
+const openEdgeReaders = new Set<{ sinks: Map<number, unknown> }>();
+
+/**
+ * What the derived pictures on the assets themselves cost.
+ *
+ * A strip's thumbnails and a waveform's peaks are read once and then live on
+ * the asset for the session: a couple of dozen JPEG data URLs and up to sixty
+ * thousand samples apiece, which on a project of a hundred files is more than
+ * the decoders are allowed between them. Both have a disk cache behind them,
+ * so this is memory standing in front of storage.
+ */
+const derivedBytes = (): number => {
+ let n = 0;
+ for (const a of useEditor.getState().assets) {
+ for (const t of a.thumbs ?? []) n += t.length * 2;
+ n += (a.peaks?.length ?? 0) * 8;
+ }
+ return n;
+};
+
+/**
+ * Give back the strips and waveforms of the files nothing in the cut is using.
+ *
+ * A library entry that was imported and never placed still carries everything
+ * the timeline drew for it once. Every one of those reads back from its own
+ * IndexedDB cache the moment a tile asks again, so holding them is a choice
+ * about latency; under memory pressure it is the wrong one. Assets a clip
+ * points at keep what they have, so nothing on the timeline ever redraws.
+ */
+/** What the strips and waveforms of unplaced library entries cost — the part
+ * of the total that a release could actually give back. */
+function unusedDerivedBytes(): number {
+ const s = useEditor.getState();
+ const used = assetIdsInUse(s);
+ let n = 0;
+ for (const a of s.assets) {
+ if (used.has(a.id)) continue;
+ for (const t of a.thumbs ?? []) n += t.length * 2;
+ n += (a.peaks?.length ?? 0) * 8;
+ }
+ return n;
+}
+
+function releaseUnusedDerived(): void {
+ const s = useEditor.getState();
+ const used = assetIdsInUse(s);
+ for (const a of s.assets) {
+ if (used.has(a.id) || (!a.thumbs?.length && !a.peaks?.length)) continue;
+ s.updateAsset(a.id, { thumbs: undefined, peaks: undefined });
+ }
+}
+
+/** Derived pictures the assets may carry between them. */
+const DERIVED_MAX = 128 * 2 ** 20;
+/** Least time between two sweeps. A capture lands many times a second while a
+ * strip fills, and the answer cannot have changed between two of them. */
+const DERIVED_SWEEP_MS = 5_000;
+let derivedSweptAt = 0;
+
+/**
+ * Give the unplaced library entries' pictures back, at most now and then.
+ *
+ * The test is what a release would recover, so a project whose placed clips
+ * are themselves over the budget asks once and finds nothing, in place of
+ * sweeping every asset on every captured frame and stripping entries a tile
+ * has just read back.
+ */
+function sweepDerived(): void {
+ const now = Date.now();
+ if (now - derivedSweptAt < DERIVED_SWEEP_MS) return;
+ derivedSweptAt = now;
+ const cap = allowance("pictures", DERIVED_MAX);
+ if (derivedBytes() > cap && unusedDerivedBytes() > 0) releaseUnusedDerived();
+}
const EDGE_POOL_CAP = 4;
type EdgeRequest = {
@@ -2096,6 +2197,7 @@ type EdgeRequest = {
type EdgeReader = {
input: ReturnType<typeof openMedia>;
sinkFor: (height: number) => ReturnType<typeof frameSink>;
+ sinks: Map<number, ReturnType<typeof frameSink>>;
};
const edgeCache = new Map<string, string>();
@@ -2104,6 +2206,9 @@ const edgeQueue = new Map<string, EdgeRequest>();
* so a trim drag's frames land first while a zoomed strip fills in behind. */
const edgeBackQueue = new Map<string, EdgeRequest>();
const edgePool = new Map<string, Promise<EdgeReader>>();
+
+holdMemory("pictures", stripBytes);
+holdMemory("pictures", derivedBytes);
let edgePumping = false;
function edgeKey(url: string, time: number, height: number) {
@@ -2165,22 +2270,37 @@ function edgeReader(url: string): Promise<EdgeReader> {
}
// A pool of canvases each sink cycles through, so a drag that reads
// hundreds of frames keeps its allocation flat.
+ // One sink per strip height, because a zoomed timeline asks for a
+ // different one. Only the heights still on screen are worth keeping: each
+ // holds its own pool of canvases, and a drag through a dozen zoom levels
+ // would otherwise leave a dozen pools behind it.
const sinks = new Map<number, ReturnType<typeof frameSink>>();
const sinkFor = (height: number) => {
let sink = sinks.get(height);
- if (!sink) {
- sink = frameSink(track, { height }, { poolSize: 4, lowLatency: true });
+ if (sink) {
+ sinks.delete(height);
sinks.set(height, sink);
+ return sink;
}
+ sink = frameSink(track, { height }, { poolSize: EDGE_SINK_POOL, lowLatency: true });
+ sinks.set(height, sink);
+ while (sinks.size > EDGE_SINK_HEIGHTS) sinks.delete(sinks.keys().next().value!);
return sink;
};
- return { input, sinkFor };
+ const reader = { input, sinkFor, sinks };
+ openEdgeReaders.add(reader);
+ return reader;
})();
edgePool.set(url, opening);
while (edgePool.size > EDGE_POOL_CAP) {
const [oldUrl, old] = edgePool.entries().next().value!;
edgePool.delete(oldUrl);
- old.then((r) => r.input.dispose()).catch(() => {});
+ old
+ .then((r) => {
+ openEdgeReaders.delete(r);
+ r.input.dispose();
+ })
+ .catch(() => {});
}
return opening;
}
@@ -2203,9 +2323,20 @@ async function pumpEdgeFrames() {
if (!frame) throw new Error("No frame at that time.");
src = await rasterCanvasToDataUrl(frame.canvas, "image/jpeg", 0.92);
edgeCache.set(req.key, src);
- while (edgeCache.size > EDGE_CACHE_CAP) {
- edgeCache.delete(edgeCache.keys().next().value!);
+ // Oldest first, by count and by what they cost. A count on its own
+ // says nothing about the size of what is counted: three hundred
+ // frames of a phone video are a different number of megabytes from
+ // three hundred of a screen recording.
+ const cap = allowance("pictures", canvasBytes(EDGE_CACHE_CAP * THUMB_H * THUMB_H * 2));
+ let bytes = stripBytes();
+ while (edgeCache.size > EDGE_CACHE_CAP || (edgeCache.size > 1 && bytes > cap)) {
+ const oldest = edgeCache.keys().next().value!;
+ bytes -= (edgeCache.get(oldest)?.length ?? 0) * 2;
+ edgeCache.delete(oldest);
}
+ // The strips and waveforms sitting on unused library entries are the
+ // other half of this bucket, and they come back off disk.
+ sweepDerived();
} catch {
src = null;
}