Skip to content

Open source CapCut alternative.

TypeScript

fix(site): read the clips a cut plays ahead of the playhead

David Le83c4a2742542
diff --git a/site/src/cut/lib/mediaSync.ts b/site/src/cut/lib/mediaSync.ts
index f66c5f10..5e3a9a40 100644
--- a/site/src/cut/lib/mediaSync.ts
+++ b/site/src/cut/lib/mediaSync.ts
@@ -247,14 +247,16 @@ let prefetch: PrefetchRun | null = null;
* and swap onto blob URLs, so an <img> paints local bytes offline. Every file
* reports back through `onDone` exactly once — its blob URL when whole bytes
* landed, null otherwise — so the caller can swap playing assets mid-session
- * and retire per-file loading state either way. Starting a prefetch cancels
+ * and retire per-file loading state either way. Videos wait for `videosAfter`
+ * (the clip spans the cut plays, read first) while the other files go ahead. Starting a prefetch cancels
* any earlier one — the queue follows the open project — and records the open
* for eviction recency. Fire-and-forget; a failed or refused write costs
* nothing but the cache. */
export function prefetchCloudMedia(
projectId: string,
files: { fileName: string; url: string; type?: AssetType }[],
- onDone?: (fileName: string, blobUrl: string | null) => void
+ onDone?: (fileName: string, blobUrl: string | null) => void,
+ videosAfter?: () => Promise<void>
): void {
if (!supportsBrowserStore()) {
for (const f of files) onDone?.(f.fileName, null);
@@ -269,9 +271,18 @@ export function prefetchCloudMedia(
}).catch(() => {});
// Opening a project brings a cache that outgrew its ceiling back under it.
await makeRoom(projectId, 0).catch(() => {});
+ // The small files a still or a sticker paints go first; the videos follow
+ // the spans the cut plays.
+ files = [...files.filter((f) => f.type !== "video"), ...files.filter((f) => f.type === "video")];
+ let waited = false;
for (let i = 0; i < files.length; i++) {
if (run.ctrl.signal.aborted) return;
const f = files[i];
+ if (f.type === "video" && !waited) {
+ waited = true;
+ await videosAfter?.().catch(() => {});
+ if (run.ctrl.signal.aborted) return;
+ }
const r =
f.type === "video"
? await cacheVideoChunks(f.url, run.ctrl.signal)
diff --git a/site/src/cut/lib/spanWarm.test.ts b/site/src/cut/lib/spanWarm.test.ts
new file mode 100644
index 00000000..92ad373f
--- /dev/null
+++ b/site/src/cut/lib/spanWarm.test.ts
@@ -0,0 +1,33 @@
+import { expect, test } from "bun:test";
+import { warmOrder } from "./spanWarm";
+
+// A cut that changes file every half second stalls at every cold join, so the
+// clips still to play are read first, each from the file the preview decodes.
+const assets = [
+ { id: "a", type: "video" as const, url: "https://m/a.mp4" },
+ { id: "b", type: "video" as const, url: "https://m/b.mp4", proxyUrl: "https://m/b.mp4.light.mp4" },
+ { id: "s", type: "image" as const, url: "https://m/s.png" },
+];
+const clips = [
+ { assetId: "a", start: 1, in: 4, out: 4.5 },
+ { assetId: "b", start: 0.5, in: 10, out: 10.5 },
+ { assetId: "a", start: 0, in: 0, out: 0.5 },
+ { assetId: "s", start: 1.5, in: 0, out: 1 },
+];
+
+test("clips read in timeline order, from the proxy where the preview plays one", () => {
+ expect(warmOrder(clips, assets, 0)).toEqual([
+ { url: "https://m/a.mp4", from: 0, to: 0.5 },
+ { url: "https://m/b.mp4.light.mp4", from: 10, to: 10.5 },
+ { url: "https://m/a.mp4", from: 4, to: 4.5 },
+ ]);
+});
+
+test("what plays from the playhead goes first, the clip under it included, then the top", () => {
+ expect(warmOrder(clips, assets, 0.7).map((s) => s.from)).toEqual([10, 4, 0]);
+});
+
+test("a reversed or sped-up clip reads its source span", () => {
+ const [span] = warmOrder([{ assetId: "a", start: 0, in: 6, out: 2, speed: 4 }], assets, 0);
+ expect(span).toEqual({ url: "https://m/a.mp4", from: 2, to: 6 });
+});
diff --git a/site/src/cut/lib/spanWarm.ts b/site/src/cut/lib/spanWarm.ts
new file mode 100644
index 00000000..db511b29
--- /dev/null
+++ b/site/src/cut/lib/spanWarm.ts
@@ -0,0 +1,116 @@
+"use client";
+
+/**
+ * The footage a cloud project's cut plays, read ahead of the playhead.
+ *
+ * A cut that changes file every half second opens a file every half second,
+ * and an open that has to reach the network takes longer than the half second
+ * the preview gives it (`PREROLL_LEAD_S`): every cold join stalls. The
+ * whole-file prefetch (mediaSync.ts) gets there in the end, but it walks
+ * masters front to back one file at a time, and a clip playing through its
+ * proxy never reads its master at all. This reads exactly what each clip
+ * plays — the container's index and the packets from the keyframe before its
+ * in-point to its out-point — from the file the preview decodes
+ * (`previewUrl`), in timeline order from the playhead, through the same chunk
+ * cache the decoder reads. Nothing is decoded; the bytes land resident, so the
+ * join opens off disk.
+ */
+
+import { EncodedPacketSink } from "mediabunny";
+import { openMediaShared, videoTrackOf } from "./mediaRead";
+import { previewUrl, type MediaAsset, type VideoClip } from "./types";
+
+/** One clip's read: a span of source seconds in the file the preview plays. */
+export interface WarmSpan {
+ url: string;
+ from: number;
+ to: number;
+}
+
+/** Files read at once. One is too few to stay ahead of a fast cut over a slow
+ * link; more starves the reads the playing decoder makes itself. */
+const WARM_PARALLEL = 3;
+
+/**
+ * What to read, in order: the clips still to play from `at`, then the ones
+ * before it, which a replay from the top plays next. A clip already under the
+ * playhead goes first. A clip read twice — a file the cut returns to — is
+ * listed once per use; the second read is a disk read.
+ */
+export function warmOrder(
+ clips: Pick<VideoClip, "assetId" | "start" | "in" | "out" | "speed">[],
+ assets: Pick<MediaAsset, "id" | "type" | "url" | "proxyUrl">[],
+ at: number
+): WarmSpan[] {
+ const byId = new Map(assets.map((a) => [a.id, a]));
+ const end = (c: (typeof clips)[number]) => c.start + Math.abs(c.out - c.in) / (c.speed || 1);
+ const sorted = [...clips].sort((a, b) => a.start - b.start);
+ const ahead = sorted.filter((c) => end(c) > at);
+ const behind = sorted.filter((c) => end(c) <= at);
+ return [...ahead, ...behind].flatMap((c) => {
+ const asset = byId.get(c.assetId);
+ if (!asset || asset.type !== "video") return [];
+ const url = previewUrl(asset);
+ return url ? [{ url, from: Math.min(c.in, c.out), to: Math.max(c.in, c.out) }] : [];
+ });
+}
+
+/** Read one span's packets through the shared reader, so the index the
+ * playing decoder parses next is the one this parsed. */
+async function warmSpan(span: WarmSpan, signal: AbortSignal): Promise<void> {
+ const handle = openMediaShared(span.url);
+ try {
+ const track = await videoTrackOf(handle.input);
+ if (!track || signal.aborted) return;
+ const sink = new EncodedPacketSink(track);
+ let packet =
+ (await sink.getKeyPacket(span.from, { verifyKeyPackets: true })) ?? (await sink.getFirstPacket());
+ while (packet && packet.timestamp <= span.to && !signal.aborted) {
+ packet = await sink.getNextPacket(packet);
+ }
+ } finally {
+ handle.release();
+ }
+}
+
+let running: AbortController | null = null;
+let walk: Promise<void> | null = null;
+
+/**
+ * Read a project's clips ahead of `at`, cancelling any read already running —
+ * a new play or a new project restarts the walk from where it now stands.
+ * Resolves when every span has been read or the walk was cancelled. A span
+ * that fails is left for the decoder's own read: this is only a head start.
+ */
+export function warmClipSpans(spans: WarmSpan[]): Promise<void> {
+ running?.abort();
+ const ctrl = new AbortController();
+ running = ctrl;
+ let next = 0;
+ const lane = async () => {
+ while (!ctrl.signal.aborted && next < spans.length) {
+ const span = spans[next++];
+ await warmSpan(span, ctrl.signal).catch(() => {});
+ }
+ };
+ const done = Promise.all(Array.from({ length: WARM_PARALLEL }, lane)).then(() => {
+ if (running === ctrl) running = null;
+ if (walk === done) walk = null;
+ });
+ walk = done;
+ return done;
+}
+
+/** Resolves once no walk is reading: the one running now, and any a play
+ * started in its place. What waits on this — the whole-file prefetch — never
+ * shares the link with a walk the playhead depends on. */
+export async function clipWarmSettled(): Promise<void> {
+ while (walk) await walk;
+}
+
+/** Stop the walk, when the project closes. */
+export function stopClipWarm(): void {
+ running?.abort();
+ running = null;
+ walk = null;
+}
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index bcaba706..18c43639 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -73,6 +73,7 @@ import { fetchSignedMediaUrls, pinDocBase } from "./backend/cloud";
import { cancelRevoke, resolveRegisteredBlob, revokeRegistered } from "./backend/browser/registry";
import { markSignedBatch } from "./mediaLinks";
import { dropLocalMedia, localMediaUrl, prefetchCloudMedia } from "./mediaSync";
+import { clipWarmSettled, stopClipWarm, warmClipSpans, warmOrder } from "./spanWarm";
import {
dropCachedDoc,
readCachedDoc,
@@ -2225,6 +2226,11 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
(x, y) =>
(firstUse.get(x.fileName) ?? Infinity) - (firstUse.get(y.fileName) ?? Infinity)
);
+ // What the cut plays is read first — each clip's span, from the
+ // file the preview decodes, in timeline order (spanWarm.ts) — and
+ // the whole videos wait for it, so their walk never competes with
+ // the first play.
+ void warmClipSpans(warmOrder(doc.clips, assets, playheadAt()));
// Video reads through the chunk cache and plays from its first
// chunk, so only whole-file fetches show as loading.
set({
@@ -2244,7 +2250,7 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
next.delete(fileName);
return { loadingMedia: next };
});
- });
+ }, clipWarmSettled);
}
} else {
markSignedBatch(id, null);
@@ -2360,7 +2366,10 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
closeProject: (id) => {
if (loadTarget === id) { loadGeneration++; loadTarget = null; }
- if (get().projectId === id) set({ projectId: null, loaded: false, playing: false });
+ if (get().projectId === id) {
+ stopClipWarm();
+ set({ projectId: null, loaded: false, playing: false });
+ }
},
openProjectDoc: async (id, doc, assets) => {
@@ -5363,6 +5372,14 @@ useEditor.subscribe((s, prev) => {
docSeq++;
});
+// A play from a new spot reads ahead from there: the walk the open started
+// may still be reading the far end of the cut, or have finished long before
+// an edit put new clips under the playhead.
+useEditor.subscribe((s, prev) => {
+ if (s.playing && !prev.playing && s.loaded && getBackend().kind === "cloud")
+ void warmClipSpans(warmOrder(s.clips, s.assets, playheadAt()));
+});
+
// Every subtitles pass — transcription, translation, AI captions — rides
// subtitleStatus, so this one watch badges the Subtitles rail tile whenever a
// pass settles while the tab is closed (landed() no-ops while it's watched).