Open source CapCut alternative.
TypeScript
"use client";
/** Conversion uses the project's backend and keeps each asset's identity.
* Browser projects borrow preparation when a decoder or encoder is unavailable;
* the prepared media returns to the project's own storage. */
import {
BufferTarget,
Conversion,
Mp4OutputFormat,
Output,
type ConversionAudioOptions,
type ConversionVideoOptions,
} from "mediabunny";
import { withCloudPreparedMedia } from "@/cut/lib/cloudLibraryUpload";
import { apiFetch as engineFetch } from "./api";
import { getBackend, hasLocalCompute, type CutBackend } from "./backend";
import { resolveRegisteredBlob } from "./backend/browser/registry";
import { pollCloudJob } from "./cloudJob";
import { enrichAsset, uploadProjectMediaTo } from "./media";
import { storedMediaUrl } from "./mediaSync";
import {
audioTrackOf,
openMedia,
probeMediaFile,
readMediaFileSize,
videoTrackOf,
UnreadableMediaError,
} from "./mediaRead";
import type { ConvertMode } from "./convertMode";
import { errorFromBody, UserError } from "./userError";
import { useEditor } from "./store";
import type { MediaAsset } from "./types";
/** One file in a project, as the converter addresses it. */
export interface ConvertSource {
fileName: string;
/** Where the bytes can be read when the conversion runs in this process. */
url: string;
/** Display name, for the message an error carries back to the user. */
name: string;
}
export interface ConvertOptions {
maxHeight?: number;
/** "reencode" rewrites a video stream that already fits (convertMode.ts). */
mode?: ConvertMode;
/** Pinned by work that outlives navigation; defaults to the active one. */
backend?: CutBackend;
}
/** A finished conversion: the new file, and what that file turned out to be,
* so nothing downstream has to read it again. */
export interface FileConversion {
fileName: string;
/** False when the streams were copied into the new container untouched. */
reencoded: boolean;
/** The file was already an MP4 of the right streams, so it was left alone
* and `fileName` is the one the project already had. */
unchanged?: boolean;
duration: number;
width?: number;
height?: number;
}
/** What a conversion did to an asset, reported back to whoever asked. */
export interface ConvertOutcome extends FileConversion {
assetId: string;
/** The asset's display name, rewritten to the new extension. */
name: string;
}
/** What a headless host does for itself, in place of the client's backend
* seam: it holds the project's files, so it converts and deletes them
* directly rather than addressing itself over HTTP. */
export interface HostMediaStore {
convert(projectId: string, source: ConvertSource, opts: ConvertOptions): Promise<FileConversion>;
drop(projectId: string, fileName: string): Promise<void>;
}
/** In-process conversion holds the whole output in memory. A source past this
* belongs on a machine with a disk, and saying so beats an out-of-memory tab
* several minutes in. */
const IN_PROCESS_MAX_BYTES = 2 * 1024 * 1024 * 1024;
const inPage = () => typeof document !== "undefined";
/** Containers that hold the target streams under the name people ask for. */
const MP4_EXT = /\.(mp4|m4v)$/i;
let host: HostMediaStore | null = null;
/** Install the store a headless host does its own media work with. */
export function setHostMediaStore(store: HostMediaStore | null) {
host = store;
}
/** The MP4 name a source file converts to: same stem, `.mp4` extension. */
export const mp4NameFor = (fileName: string) =>
`${fileName.replace(/\.[^./\\]+$/, "") || "media"}.mp4`;
/** A display name after conversion. A name carrying the camera's extension
* gets the new one — the card reading IMG_4450.MOV under a file that is now
* an MP4 is the card lying. A name with no extension is the user's words and
* stays as written. */
export const mp4DisplayName = (name: string) =>
/\.[a-z0-9]{2,4}$/i.test(name) ? mp4NameFor(name) : name;
/**
* Convert one of a project's media files to MP4, leaving the original in
* place. The result lands beside it under a deduped name; what points at
* which file is the caller's to decide.
*/
export async function convertProjectFile(
projectId: string,
source: ConvertSource,
opts: ConvertOptions = {}
): Promise<FileConversion> {
if (host) return host.convert(projectId, source, opts);
const backend = opts.backend ?? getBackend();
// The machine that stores the bytes converts them: the Mac for its own
// projects, the worker beside R2 for cloud ones. A headless process working
// a cloud project is that machine, so it converts in-process.
return backend.kind === "local" || (backend.kind === "cloud" && inPage())
? convertOnBackend(projectId, source, backend, opts)
: convertNearThePage(projectId, source, { ...opts, backend });
}
/**
* Convert one project asset to MP4 and point the asset at the result.
*
* `maxHeight` caps the picture's height, which forces a re-encode when the
* source is taller. Everything else is decided from the file: streams that
* already fit the container are copied, the rest are re-encoded.
*/
export async function convertAssetToMp4(
projectId: string,
asset: MediaAsset,
opts: ConvertOptions = {}
): Promise<ConvertOutcome> {
if (asset.type === "image") throw new UserError("convert_image_refused", "Images have nothing to convert.");
const backend = opts.backend ?? getBackend();
const made = await convertProjectFile(
projectId,
{ fileName: asset.fileName, url: asset.url, name: asset.name },
{ ...opts, backend }
);
if (made.unchanged) return { assetId: asset.id, name: asset.name, ...made };
const url = await storedMediaUrl(projectId, made.fileName, backend);
const patch: Partial<MediaAsset> = {
fileName: made.fileName,
name: mp4DisplayName(asset.name),
url,
sizeBytes: await readMediaFileSize(url),
duration: made.duration,
...(made.width !== undefined ? { width: made.width } : {}),
...(made.height !== undefined ? { height: made.height } : {}),
// The filmstrip and waveform were read off bytes that are about to be
// deleted; enrichAsset fills them again from the new file.
thumbs: undefined,
thumbStep: undefined,
peaks: undefined,
};
useEditor.getState().updateAsset(asset.id, patch);
void enrichAsset({ ...asset, ...patch }).catch(() => {});
// Best effort: a file left behind costs storage while everything else
// already points at the new one.
void dropProjectFile(projectId, asset.fileName, backend).catch(() => {});
return { assetId: asset.id, name: patch.name!, ...made };
}
/** Delete a project file through whichever side owns the storage. */
export function dropProjectFile(
projectId: string,
fileName: string,
backend: CutBackend = getBackend()
): Promise<unknown> {
if (host) return host.drop(projectId, fileName);
return backend.fetch(`/api/cut/projects/${projectId}/media/${encodeURIComponent(fileName)}`, {
method: "DELETE",
});
}
/** Server answer shape, identical from the engine's project route and the
* worker's job result. */
interface ServerConvert {
fileName?: string;
duration?: number;
width?: number;
height?: number;
transcodedVideo?: boolean;
transcodedAudio?: boolean;
unchanged?: boolean;
error?: string;
}
// A conversion that failed with nothing more to say.
const convertFailed = () => new UserError("convert_failed", "Could not convert that file.");
const fromServer = (body: ServerConvert): FileConversion => ({
fileName: body.fileName!,
reencoded: !!body.transcodedVideo || !!body.transcodedAudio,
...(body.unchanged ? { unchanged: true } : {}),
duration: body.duration ?? 0,
...(body.width !== undefined ? { width: body.width, height: body.height } : {}),
});
/** The engine converts the project's file synchronously; the cloud queues a
* worker job and this polls it. Both answer with the same fields. */
async function convertOnBackend(
projectId: string,
source: ConvertSource,
backend: CutBackend,
{ maxHeight, mode }: ConvertOptions
): Promise<FileConversion> {
const res = await backend.fetch(`/api/cut/projects/${projectId}/convert`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ file: source.fileName, ...(maxHeight ? { maxHeight } : {}), ...(mode === "reencode" ? { mode } : {}) }),
});
let body: ServerConvert;
if (backend.kind === "cloud") {
const started = (await res.json().catch(() => ({}))) as { jobId?: string; error?: string };
if (!res.ok || !started.jobId) throw errorFromBody(started, convertFailed());
body = await pollCloudJob(
started.jobId,
backend,
convertFailed(),
{ timedOut: new UserError("convert_timed_out", "The conversion took too long.") }
);
} else {
body = (await res.json().catch(() => ({}))) as ServerConvert;
if (!res.ok) throw errorFromBody(body, convertFailed());
}
if (!body.fileName) throw errorFromBody(body, convertFailed());
return fromServer(body);
}
/** Convert beside the project, borrowing hosted preparation for missing codecs. */
async function convertNearThePage(
projectId: string,
source: ConvertSource,
opts: ConvertOptions
): Promise<FileConversion> {
// A forced re-encode is for a stream the page reads no frames from, so the
// page has nothing to encode it from; a headless runtime decodes it itself.
const decodable = (opts.mode !== "reencode" || !inPage()) && await probeMediaFile(source.url).then(() => true, (error: unknown) => {
if (error instanceof UnreadableMediaError && error.undecodable) return false;
throw error;
});
if (decodable) {
try { return await convertInProcess(projectId, source, opts); }
catch (error) { if (!(error instanceof ConversionUnavailableError)) throw error; }
}
if (hasLocalCompute()) {
try { return await convertOnTheMac(projectId, source, opts); }
catch { /* The browser still needs a decoder; hosted preparation supplies it. */ }
}
const response = resolveRegisteredBlob(source.url) ?? await fetch(source.url).then((res) => {
if (!res.ok) throw new UserError("file_read_failed", `Could not read ${source.name}.`, { name: source.name });
return res.blob();
});
if (!response) throw new UserError("file_read_failed", `Could not read ${source.name}.`, { name: source.name });
return withCloudPreparedMedia(
new File([response], source.fileName, { type: response.type }),
(prepared) => land(projectId, source, prepared, opts),
{ name: source.name, maxHeight: opts.maxHeight, mode: opts.mode },
);
}
class ConversionUnavailableError extends Error {}
/**
* Convert in this process with mediabunny: WebCodecs in a tab, the headless
* runtime's decoders in a worker. Source packets that already fit the
* container are copied, which makes the common .mov of H.264 a fast remux
* with no decoding at all.
*/
async function convertInProcess(
projectId: string,
source: ConvertSource,
opts: ConvertOptions
): Promise<FileConversion> {
const bytes = resolveRegisteredBlob(source.url);
if (bytes && bytes.size > IN_PROCESS_MAX_BYTES) {
throw new ConversionUnavailableError(`${source.name} needs hosted conversion.`);
}
const input = openMedia(source.url);
try {
const [video, audio] = await Promise.all([videoTrackOf(input), audioTrackOf(input)]);
const height = video ? await video.getDisplayHeight() : 0;
const cap = opts.maxHeight && opts.maxHeight > 0 ? Math.round(opts.maxHeight) : 0;
const shrink = !!(cap && height > cap);
const forced = opts.mode === "reencode" && !!video;
// Already the file a conversion would produce — a second pass would copy
// it to a new name and change nothing.
if (
!forced &&
MP4_EXT.test(source.fileName) &&
!shrink &&
(!video || video.codec === "avc") &&
(!audio || audio.codec === "aac")
) {
return {
fileName: source.fileName,
reencoded: false,
unchanged: true,
duration: await input.computeDuration(),
...(video ? { width: await video.getDisplayWidth(), height } : {}),
};
}
// A stream already in the target codec is copied packet for packet; what
// is left over is what gets decoded and re-encoded.
const reencoded =
(!!video && (video.codec !== "avc" || shrink || forced)) || (!!audio && audio.codec !== "aac");
const videoOptions: ConversionVideoOptions = {
codec: "avc",
...(shrink ? { height: cap } : {}),
...(forced ? { forceTranscode: true } : {}),
};
const audioOptions: ConversionAudioOptions = { codec: "aac" };
const output = new Output({
format: new Mp4OutputFormat({ fastStart: "in-memory" }),
target: new BufferTarget(),
});
const conversion = await Conversion.init({
input,
output,
video: (track) => track === video ? videoOptions : { discard: true },
audio: (track) => track === audio ? audioOptions : { discard: true },
});
if (conversion.discardedTracks.some(({ track, reason }) =>
(track === video || track === audio) &&
(reason === "undecodable_source_codec" || reason === "no_encodable_target_codec")
)) throw new ConversionUnavailableError("This media needs hosted conversion.");
if (!conversion.isValid) throw new UserError("file_convert_failed", `Could not convert ${source.name}.`, { name: source.name });
await conversion.execute();
const buffer = output.target.buffer;
if (!buffer) throw new UserError("file_convert_failed", `Could not convert ${source.name}.`, { name: source.name });
const made = await land(projectId, source, new Blob([buffer], { type: "video/mp4" }), opts);
return { ...made, reencoded };
} finally {
input.dispose();
}
}
/** Hand the bytes to the Mac and take the MP4 back. The engine keeps nothing:
* it converts what it is given and answers with the file. */
async function convertOnTheMac(
projectId: string,
source: ConvertSource,
opts: ConvertOptions
): Promise<FileConversion> {
const bytes = resolveRegisteredBlob(source.url) ?? (await (await fetch(source.url)).blob());
const form = new FormData();
form.append("file", bytes, source.fileName);
if (opts.maxHeight) form.append("maxHeight", String(Math.round(opts.maxHeight)));
if (opts.mode === "reencode") form.append("mode", opts.mode);
const res = await engineFetch("/api/cut/convert", { method: "POST", body: form });
if (!res.ok) {
const body: unknown = await res.json().catch(() => null);
throw errorFromBody(body, new UserError("file_convert_failed", `Could not convert ${source.name}.`, { name: source.name }));
}
// The handoff only happens for footage this process cannot decode, which is
// footage the Mac had to re-encode.
return land(projectId, source, await res.blob(), opts);
}
/** Store a converted blob in the project and read back what it is. */
async function land(
projectId: string,
source: ConvertSource,
blob: Blob,
opts: ConvertOptions
): Promise<FileConversion> {
const fileName = await uploadProjectMediaTo(
opts.backend ?? getBackend(),
projectId,
blob,
mp4NameFor(source.fileName)
);
const meta = await probeMediaFile(blob);
return {
fileName,
reencoded: true,
duration: meta.duration,
...(meta.width !== undefined ? { width: meta.width, height: meta.height } : {}),
};
}