Skip to content

Open source CapCut alternative.

TypeScript

refactor(site): move in-page proxy encoding out

David Ledc7ee33525d9
diff --git a/site/src/cut/lib/mediaProxy.test.ts b/site/src/cut/lib/mediaProxy.test.ts
index 7c67f9dd..6ecbf1b9 100644
--- a/site/src/cut/lib/mediaProxy.test.ts
+++ b/site/src/cut/lib/mediaProxy.test.ts
@@ -175,7 +175,7 @@ describe("the in-page proxy's muxer", () => {
// reserved up front makes the finished file read from its start.
test("streams the file out a bounded chunk at a time and reads back", async () => {
const { EncodedPacket, EncodedVideoPacketSource, Input, BufferSource, MP4 } = await import("mediabunny");
- const { PROXY_WRITE_CHUNK, proxyOutput, proxyPacketBudget } = await import("./mediaProxy");
+ const { PROXY_WRITE_CHUNK, proxyOutput, proxyPacketBudget } = await import("./proxyEncode");
const file = new Uint8Array(64 * 1024 * 1024);
let size = 0;
let written = 0;
diff --git a/site/src/cut/lib/mediaProxy.ts b/site/src/cut/lib/mediaProxy.ts
index 08836e45..201cb3b8 100644
--- a/site/src/cut/lib/mediaProxy.ts
+++ b/site/src/cut/lib/mediaProxy.ts
@@ -18,40 +18,20 @@
* the asset through the store.
*/
-import {
- AudioSampleSink,
- AudioSampleSource,
- canEncodeAudio,
- canEncodeVideo,
- Mp4OutputFormat,
- Output,
- QUALITY_HIGH,
- StreamTarget,
- VideoSample,
- VideoSampleSink,
- VideoSampleSource,
- type VideoCodec,
-} from "mediabunny";
import { create } from "zustand";
-import { apiFetch as engineFetch, engineFeatures } from "./api";
-import { getBackend, hasLocalCompute, type CutBackend } from "./backend";
-import { freeName, mediaDir } from "./backend/browser/opfs";
+import { engineFeatures } from "./api";
+import { getBackend, type CutBackend } from "./backend";
+import { mediaDir } from "./backend/browser/opfs";
import { resolveRegisteredBlob } from "./backend/browser/registry";
import { pollCloudJob } from "./cloudJob";
import { cutColor } from "./colorSettings";
import { uploadProjectMediaTo } from "./media";
-import { audioTrackOf, frameSinkIsCustom, openMedia, probeMediaFile, videoTrackOf } from "./mediaRead";
-import { encoderLead } from "./encoderLead";
+import { frameSinkIsCustom, probeMediaFile } from "./mediaRead";
import { forgetLocalMediaUrl, storedMediaUrl } from "./mediaSync";
-import { ensureProresDecoder } from "./proresDecoder";
-import { convertPlanes, isPlanarFormat, planesPassThrough, proxySize } from "./proxyPlanes";
+import { encodeProxyInPage, proxyNameFor, proxyOnTheMac, type Landed } from "./proxyEncode";
import { useEditor } from "./store";
import type { MediaAsset, StoredAsset } from "./types";
-/** The proxy's file name beside its master (server/proxy.ts `proxyNameFor`). */
-export const proxyNameFor = (fileName: string) =>
- `${fileName.replace(/\.[^./\\]+$/, "") || "media"}.proxy.mp4`;
-
/** A video whose master needs a proxy and has none: ProRes (a four-character
* code starting `ap`), read off the header at import. */
export function needsProxy(asset: Pick<StoredAsset, "type" | "color" | "proxy" | "block">): boolean {
@@ -84,23 +64,6 @@ export const useProxyJobs = create<ProxyJobs>((set) => ({
const inFlight = new Set<string>();
-/** The 10-bit codecs the page may encode a proxy with, best first. Chrome
- * encodes 10-bit only as VP9 profile 2. WebKit answers yes to all three and
- * then has no 10-bit `VideoFrame` to feed them — the WASM decoder falls
- * back to 8-bit frames there — so a WebKit page makes no proxy and the Mac
- * beside it does. */
-export const PROXY_CODECS: { codec: VideoCodec; fullCodecString: string }[] = [
- { codec: "hevc", fullCodecString: "hev1.2.4.L153.B0" },
- { codec: "av1", fullCodecString: "av01.0.13M.10" },
- { codec: "vp9", fullCodecString: "vp09.02.51.10" },
-];
-
-/** A result from any residency: the file the project now holds. */
-interface Landed {
- fileName: string;
- sizeBytes: number;
-}
-
/**
* Make sure the asset has a proxy: build one through the residency that can,
* and write it onto the asset. Runs once per asset at a time; a second ask
@@ -223,200 +186,22 @@ async function buildProxy(
// storage, then the Mac beside it.
const inPage = await proxyInPage(projectId, asset, proxyMaxHeight, onProgress);
if (inPage) return inPage;
- const blob = await proxyOnTheMac(asset, proxyMaxHeight, proxyCrf);
+ const master = async () => resolveRegisteredBlob(asset.url) ?? (await (await fetch(asset.url)).blob());
+ const blob = await proxyOnTheMac(master, asset.fileName, proxyMaxHeight, proxyCrf);
if (!blob) return null;
const fileName = await uploadProjectMediaTo(backend, projectId, blob, proxyNameFor(asset.fileName));
return { fileName, sizeBytes: blob.size };
}
-/** The first 10-bit codec this browser encodes at the master's size. */
-async function encodableProxyCodec(width: number, height: number) {
- for (const c of PROXY_CODECS) {
- if (await canEncodeVideo(c.codec, { width, height, fullCodecString: c.fullCodecString })) return c;
- }
- return null;
-}
-
-/** How much encoded output the page holds before it writes to disk. */
-export const PROXY_WRITE_CHUNK = 4 * 1024 * 1024;
-
-/** The packet ceilings the reserved header is sized for: the master's own
- * frame count and the AAC frames its duration makes, each with the headroom
- * the muxer asks for. */
-export function proxyPacketBudget(videoPackets: number, duration: number, sampleRate: number) {
- return {
- video: Math.ceil(videoPackets * 1.33) + 16,
- audio: Math.ceil(((duration * sampleRate) / 1024) * 1.33) + 16,
- };
-}
-
-/** The proxy's muxer: MP4 with its header in space reserved up front, written
- * through `writable` a bounded chunk at a time, so what the page holds does not
- * grow with the master's length. */
-export function proxyOutput(writable: WritableStream) {
- return new Output({
- format: new Mp4OutputFormat({ fastStart: "reserve" }),
- target: new StreamTarget(writable, { chunked: true, chunkSize: PROXY_WRITE_CHUNK }),
- });
-}
-
-/**
- * Build the proxy in this page and write it into the project's own storage:
- * decode the master through the WASM decoder, carry every frame's planes into
- * Rec.709 limited 4:2:0 at 10 bits, and encode with the browser's first
- * 10-bit codec, streaming the file to disk as it goes; a master taller than
- * the proxy may be is brought down in the same pass. Null when the page
- * cannot: no 10-bit encoder, no AAC encoder for the sound, or a decoder
- * handing out 8-bit frames (WebKit, whose `VideoFrame` takes no 10-bit
- * format).
- */
-export async function proxyInPage(
+/** Build the project's proxy in this page, straight into its own storage. */
+async function proxyInPage(
projectId: string,
asset: MediaAsset,
maxHeight: number,
onProgress: (p: number) => void
): Promise<Landed | null> {
- if (typeof VideoFrame === "undefined") return null;
- await ensureProresDecoder();
- const input = openMedia(asset.url);
- try {
- const video = await videoTrackOf(input);
- if (!video) return null;
- const [coded, duration, rotated] = await Promise.all([
- Promise.all([video.getCodedWidth(), video.getCodedHeight()]),
- input.computeDuration(),
- video.getRotation(),
- ]);
- // The planes are converted at the coded size; the display size is the
- // coded one turned by the container's rotation, which the proxy keeps —
- // so a sideways master's cap applies to its coded width.
- const cap = rotated % 180 === 0 ? maxHeight : Math.round((maxHeight * coded[1]) / Math.max(1, coded[0]));
- const target = proxySize(coded[0], coded[1], cap);
- const codec = await encodableProxyCodec(target.width, target.height);
- if (!codec) return null;
- const audio = await audioTrackOf(input);
- if (audio && !(await canEncodeAudio("aac"))) return null;
- const from = { matrix: asset.color?.matrix ?? "bt709", fullRange: asset.color?.fullRange ?? false } as const;
- const budget = proxyPacketBudget(
- (await video.computePacketStats()).packetCount,
- duration,
- audio ? await audio.getSampleRate() : 0
- );
-
- const dir = await mediaDir(projectId, true);
- if (!dir) return null;
- const fileName = await freeName(dir, proxyNameFor(asset.fileName));
- const handle = await dir.getFileHandle(fileName, { create: true });
- const writable = await handle.createWritable();
- const output = proxyOutput(writable);
- let landed = false;
- try {
- const videoSource = new VideoSampleSource({
- codec: codec.codec,
- fullCodecString: codec.fullCodecString,
- quality: QUALITY_HIGH,
- });
- output.addVideoTrack(videoSource, { rotation: rotated, maximumPacketCount: budget.video });
- const audioSource = audio ? new AudioSampleSource({ codec: "aac", quality: QUALITY_HIGH }) : null;
- if (audioSource) output.addAudioTrack(audioSource, { maximumPacketCount: budget.audio });
- await output.start();
-
- const frames = new VideoSampleSink(video);
- for await (const sample of frames.samples()) {
- try {
- const converted = await convertSample(sample, from, target);
- if (!converted) return null;
- try {
- await videoSource.add(converted);
- } finally {
- if (converted !== sample) converted.close();
- }
- if (duration > 0) onProgress(Math.min(0.95, sample.timestamp / duration));
- } finally {
- sample.close();
- }
- }
- if (audio && audioSource) {
- // Each sample goes in early by the encoder's priming, so the edit list
- // trims it and the proxy's sound stays on its picture.
- const lead = await encoderLead("aac", await audio.getSampleRate(), await audio.getNumberOfChannels());
- const sounds = new AudioSampleSink(audio);
- for await (const sample of sounds.samples()) {
- try {
- sample.setTimestamp(sample.timestamp - lead);
- await audioSource.add(sample);
- } finally {
- sample.close();
- }
- }
- }
- await output.finalize();
- // The file the project keeps has to read back as what was asked for.
- const file = await handle.getFile();
- const probe = await probeMediaFile(file);
- const want = rotated % 180 === 0 ? target.width : target.height;
- if (!probe.hasVideo || probe.width !== want) throw new Error("The preview proxy did not read back.");
- landed = true;
- onProgress(1);
- return { fileName, sizeBytes: file.size };
- } finally {
- if (!landed) {
- if (output.state !== "finalized" && output.state !== "canceled") await output.cancel().catch(() => {});
- // A muxer that never started never took the stream.
- if (!writable.locked) await writable.abort().catch(() => {});
- await dir.removeEntry(fileName).catch(() => {});
- }
- }
- } finally {
- input.dispose();
- }
-}
-
-/** The sample as Rec.709 limited 4:2:0 10-bit planes, tagged so; the sample
- * itself when it already is that. Null for a frame the page cannot carry at
- * 10 bits. */
-async function convertSample(
- sample: VideoSample,
- from: { matrix: "bt709" | "bt601" | "bt2020nc"; fullRange: boolean },
- target: { width: number; height: number }
-): Promise<VideoSample | null> {
- const format = sample.format;
- if (!isPlanarFormat(format)) return null;
- const colorSpace = { primaries: "bt709", transfer: "iec61966-2-1", matrix: "bt709", fullRange: false } as const;
- const init = { timestamp: sample.timestamp, duration: sample.duration, colorSpace };
- const data = new ArrayBuffer(sample.allocationSize());
- const layout = await sample.copyTo(data);
- const size = { width: sample.codedWidth, height: sample.codedHeight, maxHeight: target.height };
- if (planesPassThrough(format, from, size) && sample.codedWidth === target.width) {
- return new VideoSample(data, { ...init, format: "I420P10", codedWidth: sample.codedWidth, codedHeight: sample.codedHeight, layout });
- }
- const converted = convertPlanes(
- { format, width: sample.codedWidth, height: sample.codedHeight, data, layout },
- from,
- { maxHeight: target.height }
- );
- return new VideoSample(converted.data, {
- ...init,
- format: converted.format,
- codedWidth: converted.width,
- codedHeight: converted.height,
- layout: converted.layout,
- });
-}
-
-/** Hand the master's bytes to the Mac and take the proxy back; the engine
- * keeps nothing. Null when the app is not there or is older than the route. */
-async function proxyOnTheMac(asset: MediaAsset, maxHeight: number, crf: number): Promise<Blob | null> {
- if (!hasLocalCompute() || !(await engineFeatures()).has("media.proxy")) return null;
- const bytes = resolveRegisteredBlob(asset.url) ?? (await (await fetch(asset.url)).blob());
- const form = new FormData();
- form.append("file", bytes, asset.fileName);
- form.append("maxHeight", String(maxHeight));
- form.append("crf", String(crf));
- const res = await engineFetch("/api/cut/proxy", { method: "POST", body: form });
- if (!res.ok) {
- const body = (await res.json().catch(() => ({}))) as { error?: string };
- throw new Error(body.error ?? "Could not make the preview proxy.");
- }
- return res.blob();
+ const dir = await mediaDir(projectId, true);
+ if (!dir) return null;
+ const from = { matrix: asset.color?.matrix ?? "bt709", fullRange: asset.color?.fullRange ?? false } as const;
+ return encodeProxyInPage(asset.url, from, dir, asset.fileName, maxHeight, onProgress);
}
diff --git a/site/src/cut/lib/proxyEncode.ts b/site/src/cut/lib/proxyEncode.ts
new file mode 100644
index 00000000..e3749129
--- /dev/null
+++ b/site/src/cut/lib/proxyEncode.ts
@@ -0,0 +1,254 @@
+"use client";
+
+/**
+ * The page's half of making a preview proxy: encode one in this tab from a
+ * master the WASM decoder reads, or hand the master to the Mac's engine and
+ * take the proxy back. A project's proxy (mediaProxy.ts) and a library
+ * video's playable copy (the browser shelf's library route) both build here;
+ * each caller says where the file lands.
+ */
+
+import {
+ AudioSampleSink,
+ AudioSampleSource,
+ canEncodeAudio,
+ canEncodeVideo,
+ Mp4OutputFormat,
+ Output,
+ QUALITY_HIGH,
+ StreamTarget,
+ VideoSample,
+ VideoSampleSink,
+ VideoSampleSource,
+ type VideoCodec,
+} from "mediabunny";
+import { apiFetch as engineFetch, engineFeatures } from "./api";
+import { hasLocalCompute } from "./backend";
+import { freeName } from "./backend/browser/opfs";
+import { audioTrackOf, openMedia, probeMediaFile, videoTrackOf } from "./mediaRead";
+import { encoderLead } from "./encoderLead";
+import { ensureProresDecoder } from "./proresDecoder";
+import { convertPlanes, isPlanarFormat, planesPassThrough, proxySize } from "./proxyPlanes";
+
+/** The proxy's file name beside its master (server/proxy.ts `proxyNameFor`). */
+export const proxyNameFor = (fileName: string) =>
+ `${fileName.replace(/\.[^./\\]+$/, "") || "media"}.proxy.mp4`;
+
+/** What the master's code values mean: the matrix and range the proxy is
+ * carried out of. */
+export type ProxyColor = { matrix: "bt709" | "bt601" | "bt2020nc"; fullRange: boolean };
+
+/** The 10-bit codecs the page may encode a proxy with, best first. Chrome
+ * encodes 10-bit only as VP9 profile 2. WebKit answers yes to all three and
+ * then has no 10-bit `VideoFrame` to feed them — the WASM decoder falls
+ * back to 8-bit frames there — so a WebKit page makes no proxy and the Mac
+ * beside it does. */
+export const PROXY_CODECS: { codec: VideoCodec; fullCodecString: string }[] = [
+ { codec: "hevc", fullCodecString: "hev1.2.4.L153.B0" },
+ { codec: "av1", fullCodecString: "av01.0.13M.10" },
+ { codec: "vp9", fullCodecString: "vp09.02.51.10" },
+];
+
+
+/** A result from any residency: the file the shelf or project now holds. */
+export interface Landed {
+ fileName: string;
+ sizeBytes: number;
+}
+
+
+/** The first 10-bit codec this browser encodes at the master's size. */
+async function encodableProxyCodec(width: number, height: number) {
+ for (const c of PROXY_CODECS) {
+ if (await canEncodeVideo(c.codec, { width, height, fullCodecString: c.fullCodecString })) return c;
+ }
+ return null;
+}
+
+/** How much encoded output the page holds before it writes to disk. */
+export const PROXY_WRITE_CHUNK = 4 * 1024 * 1024;
+
+/** The packet ceilings the reserved header is sized for: the master's own
+ * frame count and the AAC frames its duration makes, each with the headroom
+ * the muxer asks for. */
+export function proxyPacketBudget(videoPackets: number, duration: number, sampleRate: number) {
+ return {
+ video: Math.ceil(videoPackets * 1.33) + 16,
+ audio: Math.ceil(((duration * sampleRate) / 1024) * 1.33) + 16,
+ };
+}
+
+/** The proxy's muxer: MP4 with its header in space reserved up front, written
+ * through `writable` a bounded chunk at a time, so what the page holds does not
+ * grow with the master's length. */
+export function proxyOutput(writable: WritableStream) {
+ return new Output({
+ format: new Mp4OutputFormat({ fastStart: "reserve" }),
+ target: new StreamTarget(writable, { chunked: true, chunkSize: PROXY_WRITE_CHUNK }),
+ });
+}
+
+/**
+ * Build the proxy in this page and write it into the project's own storage:
+ * decode the master through the WASM decoder, carry every frame's planes into
+ * Rec.709 limited 4:2:0 at 10 bits, and encode with the browser's first
+ * 10-bit codec, streaming the file to disk as it goes; a master taller than
+ * the proxy may be is brought down in the same pass. Null when the page
+ * cannot: no 10-bit encoder, no AAC encoder for the sound, or a decoder
+ * handing out 8-bit frames (WebKit, whose `VideoFrame` takes no 10-bit
+ * format).
+ */
+export async function encodeProxyInPage(
+ src: string | Blob,
+ from: ProxyColor,
+ dir: FileSystemDirectoryHandle,
+ masterName: string,
+ maxHeight: number,
+ onProgress: (p: number) => void
+): Promise<Landed | null> {
+ if (typeof VideoFrame === "undefined") return null;
+ await ensureProresDecoder();
+ const input = openMedia(src);
+ try {
+ const video = await videoTrackOf(input);
+ if (!video) return null;
+ const [coded, duration, rotated] = await Promise.all([
+ Promise.all([video.getCodedWidth(), video.getCodedHeight()]),
+ input.computeDuration(),
+ video.getRotation(),
+ ]);
+ // The planes are converted at the coded size; the display size is the
+ // coded one turned by the container's rotation, which the proxy keeps —
+ // so a sideways master's cap applies to its coded width.
+ const cap = rotated % 180 === 0 ? maxHeight : Math.round((maxHeight * coded[1]) / Math.max(1, coded[0]));
+ const target = proxySize(coded[0], coded[1], cap);
+ const codec = await encodableProxyCodec(target.width, target.height);
+ if (!codec) return null;
+ const audio = await audioTrackOf(input);
+ if (audio && !(await canEncodeAudio("aac"))) return null;
+ const budget = proxyPacketBudget(
+ (await video.computePacketStats()).packetCount,
+ duration,
+ audio ? await audio.getSampleRate() : 0
+ );
+
+ const fileName = await freeName(dir, proxyNameFor(masterName));
+ const handle = await dir.getFileHandle(fileName, { create: true });
+ const writable = await handle.createWritable();
+ const output = proxyOutput(writable);
+ let landed = false;
+ try {
+ const videoSource = new VideoSampleSource({
+ codec: codec.codec,
+ fullCodecString: codec.fullCodecString,
+ quality: QUALITY_HIGH,
+ });
+ output.addVideoTrack(videoSource, { rotation: rotated, maximumPacketCount: budget.video });
+ const audioSource = audio ? new AudioSampleSource({ codec: "aac", quality: QUALITY_HIGH }) : null;
+ if (audioSource) output.addAudioTrack(audioSource, { maximumPacketCount: budget.audio });
+ await output.start();
+
+ const frames = new VideoSampleSink(video);
+ for await (const sample of frames.samples()) {
+ try {
+ const converted = await convertSample(sample, from, target);
+ if (!converted) return null;
+ try {
+ await videoSource.add(converted);
+ } finally {
+ if (converted !== sample) converted.close();
+ }
+ if (duration > 0) onProgress(Math.min(0.95, sample.timestamp / duration));
+ } finally {
+ sample.close();
+ }
+ }
+ if (audio && audioSource) {
+ // Each sample goes in early by the encoder's priming, so the edit list
+ // trims it and the proxy's sound stays on its picture.
+ const lead = await encoderLead("aac", await audio.getSampleRate(), await audio.getNumberOfChannels());
+ const sounds = new AudioSampleSink(audio);
+ for await (const sample of sounds.samples()) {
+ try {
+ sample.setTimestamp(sample.timestamp - lead);
+ await audioSource.add(sample);
+ } finally {
+ sample.close();
+ }
+ }
+ }
+ await output.finalize();
+ // The file the project keeps has to read back as what was asked for.
+ const file = await handle.getFile();
+ const probe = await probeMediaFile(file);
+ const want = rotated % 180 === 0 ? target.width : target.height;
+ if (!probe.hasVideo || probe.width !== want) throw new Error("The preview proxy did not read back.");
+ landed = true;
+ onProgress(1);
+ return { fileName, sizeBytes: file.size };
+ } finally {
+ if (!landed) {
+ if (output.state !== "finalized" && output.state !== "canceled") await output.cancel().catch(() => {});
+ // A muxer that never started never took the stream.
+ if (!writable.locked) await writable.abort().catch(() => {});
+ await dir.removeEntry(fileName).catch(() => {});
+ }
+ }
+ } finally {
+ input.dispose();
+ }
+}
+
+/** The sample as Rec.709 limited 4:2:0 10-bit planes, tagged so; the sample
+ * itself when it already is that. Null for a frame the page cannot carry at
+ * 10 bits. */
+async function convertSample(
+ sample: VideoSample,
+ from: ProxyColor,
+ target: { width: number; height: number }
+): Promise<VideoSample | null> {
+ const format = sample.format;
+ if (!isPlanarFormat(format)) return null;
+ const colorSpace = { primaries: "bt709", transfer: "iec61966-2-1", matrix: "bt709", fullRange: false } as const;
+ const init = { timestamp: sample.timestamp, duration: sample.duration, colorSpace };
+ const data = new ArrayBuffer(sample.allocationSize());
+ const layout = await sample.copyTo(data);
+ const size = { width: sample.codedWidth, height: sample.codedHeight, maxHeight: target.height };
+ if (planesPassThrough(format, from, size) && sample.codedWidth === target.width) {
+ return new VideoSample(data, { ...init, format: "I420P10", codedWidth: sample.codedWidth, codedHeight: sample.codedHeight, layout });
+ }
+ const converted = convertPlanes(
+ { format, width: sample.codedWidth, height: sample.codedHeight, data, layout },
+ from,
+ { maxHeight: target.height }
+ );
+ return new VideoSample(converted.data, {
+ ...init,
+ format: converted.format,
+ codedWidth: converted.width,
+ codedHeight: converted.height,
+ layout: converted.layout,
+ });
+}
+
+/** Hand the master's bytes to the Mac and take the proxy back; the engine
+ * keeps nothing. Null when the app is not there or is older than the route. */
+export async function proxyOnTheMac(
+ master: () => Promise<Blob>,
+ masterName: string,
+ maxHeight: number,
+ crf: number
+): Promise<Blob | null> {
+ if (!hasLocalCompute() || !(await engineFeatures()).has("media.proxy")) return null;
+ const bytes = await master();
+ const form = new FormData();
+ form.append("file", bytes, masterName);
+ form.append("maxHeight", String(maxHeight));
+ form.append("crf", String(crf));
+ const res = await engineFetch("/api/cut/proxy", { method: "POST", body: form });
+ if (!res.ok) {
+ const body = (await res.json().catch(() => ({}))) as { error?: string };
+ throw new Error(body.error ?? "Could not make the preview proxy.");
+ }
+ return res.blob();
+}