Skip to content

Open source CapCut alternative.

TypeScript

perf(site): a landing carries the session's resident bytes

David Le2946ea893352
diff --git a/site/src/cut/lib/chunkAdopt.test.ts b/site/src/cut/lib/chunkAdopt.test.ts
new file mode 100644
index 00000000..f01c175f
--- /dev/null
+++ b/site/src/cut/lib/chunkAdopt.test.ts
@@ -0,0 +1,187 @@
+import { describe, expect, mock, test } from "bun:test";
+
+/**
+ * The byte cache carried across an import's landing.
+ *
+ * A library clip dragged into a project plays from the library's own URL
+ * while the copy runs behind the editor, and everything the session reads —
+ * the play, the waveform and filmstrip decodes — lands in the chunk store
+ * under the library key. The stored copy is the same bytes under a project
+ * key, so the landing adopts what is resident; the failure this guards
+ * against is a first session that reads the whole file twice and stutters
+ * for it, on a clip that plays smoothly ever after a reload.
+ */
+
+// ── an in-memory store standing in for OPFS ─────────────────────────────────
+
+class FakeDir {
+ kind = "directory" as const;
+ dirs = new Map<string, FakeDir>();
+ files = new Map<string, Blob>();
+
+ async getDirectoryHandle(name: string, opts?: { create?: boolean }): Promise<FakeDir> {
+ let dir = this.dirs.get(name);
+ if (!dir) {
+ if (!opts?.create) throw new DOMException("Not found", "NotFoundError");
+ dir = new FakeDir();
+ this.dirs.set(name, dir);
+ }
+ return dir;
+ }
+
+ async getFileHandle(name: string, opts?: { create?: boolean }) {
+ if (!this.files.has(name)) {
+ if (!opts?.create) throw new DOMException("Not found", "NotFoundError");
+ this.files.set(name, new Blob([]));
+ }
+ const files = this.files;
+ return {
+ kind: "file" as const,
+ async getFile() {
+ return files.get(name)!;
+ },
+ async createWritable() {
+ return {
+ async write(data: Blob | ArrayBuffer | string) {
+ files.set(name, data instanceof Blob ? data : new Blob([data]));
+ },
+ async close() {},
+ async abort() {},
+ };
+ },
+ };
+ }
+
+ async removeEntry(name: string): Promise<void> {
+ this.dirs.delete(name);
+ this.files.delete(name);
+ }
+
+ async *[Symbol.asyncIterator](): AsyncGenerator<[string, { kind: string }]> {
+ for (const [name, dir] of this.dirs) yield [name, dir];
+ for (const [name] of this.files) yield [name, { kind: "file" }];
+ }
+}
+
+let root = new FakeDir();
+
+const opfs = await import("./backend/browser/opfs");
+mock.module("./backend/browser/opfs", () => ({
+ ...opfs,
+ supportsBrowserStore: () => true,
+ chunksDir: async () => root as never,
+ // The real readers over the fake handles.
+ readFileAt: async (dir: FakeDir | null, name: string) => {
+ if (!dir) return null;
+ try {
+ return await (await dir.getFileHandle(name)).getFile();
+ } catch {
+ return null;
+ }
+ },
+ writeFileAt: async (dir: FakeDir, name: string, data: Blob | ArrayBuffer | string) => {
+ const w = await (await dir.getFileHandle(name, { create: true })).createWritable();
+ await w.write(data);
+ await w.close();
+ },
+ readJson: async (dir: FakeDir | null, name: string) => {
+ if (!dir) return null;
+ try {
+ const file = await (await dir.getFileHandle(name)).getFile();
+ return JSON.parse(await file.text());
+ } catch {
+ return null;
+ }
+ },
+}));
+
+const { CHUNK_SIZE, adoptChunks, chunkIdentity, encodeResident, decodeResident } = await import(
+ "./chunkCache"
+);
+
+const signed = (path: string) => `https://media.donkeycut.com${path}?e=123&s=abc`;
+const LIB = signed("/cut/u1/library/phone.mp4");
+const PROJ = signed("/cut/u1/projects/p1/media/phone.mp4");
+
+/** Seed a source object: `size` bytes with the given chunks resident. */
+async function seed(url: string, size: number, resident: number[], opts?: { shortChunk?: number }) {
+ const id = chunkIdentity(url)!;
+ const dir = await (
+ await root.getDirectoryHandle(id.keyHash, { create: true })
+ ).getDirectoryHandle(id.versionTag, { create: true });
+ const total = Math.ceil(size / CHUNK_SIZE);
+ for (const idx of resident) {
+ const len =
+ opts?.shortChunk === idx ? 7 : Math.min(CHUNK_SIZE, size - idx * CHUNK_SIZE);
+ dir.files.set(String(idx), new Blob([new Uint8Array(len).fill(idx + 1)]));
+ }
+ dir.files.set(
+ "meta.json",
+ new Blob([
+ JSON.stringify({
+ key: id.key,
+ version: id.version,
+ size,
+ chunk: CHUNK_SIZE,
+ at: 1,
+ resident: encodeResident(new Set(resident), total),
+ }),
+ ])
+ );
+ return dir;
+}
+
+const destDir = async (url: string) => {
+ const id = chunkIdentity(url)!;
+ return (await root.getDirectoryHandle(id.keyHash)).getDirectoryHandle(id.versionTag);
+};
+
+describe("adoptChunks", () => {
+ test("the landing carries the resident bytes to the stored key", async () => {
+ root = new FakeDir();
+ const size = CHUNK_SIZE * 2 + 512;
+ await seed(LIB, size, [0, 2]);
+ await adoptChunks(LIB, PROJ);
+ const dest = await destDir(PROJ);
+ expect(dest.files.get("0")!.size).toBe(CHUNK_SIZE);
+ expect(dest.files.get("2")!.size).toBe(512);
+ expect(dest.files.has("1")).toBe(false);
+ const meta = JSON.parse(await dest.files.get("meta.json")!.text());
+ expect(meta.key).toBe(chunkIdentity(PROJ)!.key);
+ expect(meta.size).toBe(size);
+ expect([...decodeResident(meta.resident, 3)].sort()).toEqual([0, 2]);
+ // The library keeps its own copy for its own shelf.
+ const src = await destDir(LIB);
+ expect(src.files.has("0")).toBe(true);
+ });
+
+ test("a chunk the disk cannot back stays behind", async () => {
+ root = new FakeDir();
+ const size = CHUNK_SIZE * 2;
+ await seed(LIB, size, [0, 1], { shortChunk: 0 });
+ await adoptChunks(LIB, PROJ);
+ const dest = await destDir(PROJ);
+ expect(dest.files.has("0")).toBe(false);
+ expect(dest.files.get("1")!.size).toBe(CHUNK_SIZE);
+ const meta = JSON.parse(await dest.files.get("meta.json")!.text());
+ expect([...decodeResident(meta.resident, 2)]).toEqual([1]);
+ });
+
+ test("a source with nothing resident, or the same key, carries nothing", async () => {
+ root = new FakeDir();
+ await seed(LIB, CHUNK_SIZE, []);
+ await adoptChunks(LIB, PROJ);
+ const projId = chunkIdentity(PROJ)!;
+ expect(root.dirs.has(projId.keyHash)).toBe(false);
+ // The same key at a rotated token is the same object already.
+ await adoptChunks(LIB, `${LIB.split("?")[0]}?e=999&s=zzz`);
+ expect(root.dirs.size).toBe(1);
+ });
+
+ test("an address off the media origin carries nothing", async () => {
+ root = new FakeDir();
+ await adoptChunks("blob:abc", PROJ);
+ await adoptChunks("http://127.0.0.1:41417/f.mp4", PROJ);
+ expect(root.dirs.size).toBe(0);
+ });
+});
diff --git a/site/src/cut/lib/chunkCache.ts b/site/src/cut/lib/chunkCache.ts
index 827a7bc9..17328993 100644
--- a/site/src/cut/lib/chunkCache.ts
+++ b/site/src/cut/lib/chunkCache.ts
@@ -805,6 +805,81 @@ export function chunkSourceOptions(url: string): {
};
}
+/**
+ * Carry one object's resident bytes to another key holding the same bytes.
+ *
+ * An import that lands moves an asset off the URL it arrived on — a library
+ * file copied into the project — and the copy is byte-identical, so what the
+ * session already read under the old key answers for the new one: the play
+ * that ran, the waveform and filmstrip decodes, a prefetch. Left behind,
+ * the whole session reads cold off the new key while the same bytes sit
+ * resident a directory away — the roughness a clip shows on its first
+ * session and never after a reload, because a reload holds one key all
+ * session. Chunk files are copied and the source keeps its own; a carry
+ * that fails costs only the refetch it always cost.
+ */
+export async function adoptChunks(fromUrl: string, toUrl: string): Promise<void> {
+ if (!supportsBrowserStore()) return;
+ const from = chunkIdentity(fromUrl);
+ const to = chunkIdentity(toUrl);
+ if (!from || !to || stateKey(from) === stateKey(to)) return;
+ try {
+ const root = await chunksDir(false);
+ if (!root) return;
+ const srcDir = await (await root.getDirectoryHandle(from.keyHash)).getDirectoryHandle(
+ from.versionTag
+ );
+ const meta = await readJson<ChunkMeta>(srcDir, META);
+ if (
+ !meta ||
+ meta.key !== from.key ||
+ meta.chunk !== CHUNK_SIZE ||
+ !Number.isInteger(meta.size) ||
+ meta.size <= 0
+ )
+ return;
+ const total = chunkCount(meta.size);
+ const resident = meta.resident
+ ? decodeResident(meta.resident, total)
+ : await scanResident(srcDir, total);
+ if (resident.size === 0) return;
+ const dest = await versionDir(to);
+ if (!dest) return;
+ const copied = new Set<number>();
+ for (const idx of [...resident].sort((a, b) => a - b)) {
+ const bytes = await readFileAt(srcDir, String(idx));
+ // A file the map names but the disk cannot back, or one of the wrong
+ // length, answers for bytes it does not hold; it stays behind.
+ if (!bytes || bytes.size !== chunkLen(idx, meta.size)) continue;
+ await writeFileAt(dest, String(idx), bytes);
+ copied.add(idx);
+ }
+ if (copied.size === 0) return;
+ const out: ChunkMeta = {
+ key: to.key,
+ version: to.version,
+ size: meta.size,
+ chunk: CHUNK_SIZE,
+ at: Date.now(),
+ resident: encodeResident(copied, total),
+ };
+ await writeFileAt(dest, META, JSON.stringify(out));
+ // A reader already holding the destination open learns what landed.
+ const open = states.get(stateKey(to));
+ if (open)
+ void open
+ .then((s) => {
+ if (s.dead) return;
+ if (!s.size) s.size = meta.size;
+ for (const i of copied) s.resident.add(i);
+ })
+ .catch(() => {});
+ } catch {
+ // A cache that could not be carried is a cache that never was: the next
+ // open probes and reads over the network, exactly as before.
+ }
+}
+
/** Walk a file's chunks into the store front to back, skipping what is
* already resident or in the air. Stops on abort, on a store that has run out
* of room, and on a network failure — every chunk that landed stays. */
diff --git a/site/src/cut/lib/importQueue.ts b/site/src/cut/lib/importQueue.ts
index e8375cc7..ea966ca9 100644
--- a/site/src/cut/lib/importQueue.ts
+++ b/site/src/cut/lib/importQueue.ts
@@ -198,6 +198,14 @@ async function run(job: Job) {
.then((m) => m.mintLandedUrl(job.projectId, fileName))
.catch(() => null)) ??
mediaUrl(job.projectId, fileName);
+ // The stored copy is the same bytes at a new key: what this session
+ // already read under the source's key — the play so far, the waveform and
+ // filmstrip decodes — carries over before anything opens the new address,
+ // so the swap lands on a warm cache.
+ const playingFrom = useEditor.getState().assets.find((a) => a.id === asset.id)?.url ?? localUrl;
+ await import("./chunkCache")
+ .then((m) => m.adoptChunks(playingFrom, url))
+ .catch(() => {});
useEditor.getState().updateAsset(asset.id, {
fileName,
url,