Skip to content

Open source CapCut alternative.

TypeScript
15.1 KB
Raw Blame
"use client";

/** Conversion uses the project's backend and keeps each asset's identity.
 * Browser projects borrow preparation when a decoder or encoder is unavailable;
 * the prepared media returns to the project's own storage. */

import {
  BufferTarget,
  Conversion,
  Mp4OutputFormat,
  Output,
  type ConversionAudioOptions,
  type ConversionVideoOptions,
} from "mediabunny";
import { withCloudPreparedMedia } from "@/cut/lib/cloudLibraryUpload";
import { apiFetch as engineFetch } from "./api";
import { getBackend, hasLocalCompute, type CutBackend } from "./backend";
import { resolveRegisteredBlob } from "./backend/browser/registry";
import { pollCloudJob } from "./cloudJob";
import { enrichAsset, uploadProjectMediaTo } from "./media";
import { storedMediaUrl } from "./mediaSync";
import {
  audioTrackOf,
  openMedia,
  probeMediaFile,
  readMediaFileSize,
  videoTrackOf,
  UnreadableMediaError,
} from "./mediaRead";
import type { ConvertMode } from "./convertMode";
import { errorFromBody, UserError } from "./userError";
import { useEditor } from "./store";
import type { MediaAsset } from "./types";

/** One file in a project, as the converter addresses it. */
export interface ConvertSource {
  fileName: string;
  /** Where the bytes can be read when the conversion runs in this process. */
  url: string;
  /** Display name, for the message an error carries back to the user. */
  name: string;
}

export interface ConvertOptions {
  maxHeight?: number;
  /** "reencode" rewrites a video stream that already fits (convertMode.ts). */
  mode?: ConvertMode;
  /** Pinned by work that outlives navigation; defaults to the active one. */
  backend?: CutBackend;
}

/** A finished conversion: the new file, and what that file turned out to be,
 * so nothing downstream has to read it again. */
export interface FileConversion {
  fileName: string;
  /** False when the streams were copied into the new container untouched. */
  reencoded: boolean;
  /** The file was already an MP4 of the right streams, so it was left alone
   * and `fileName` is the one the project already had. */
  unchanged?: boolean;
  duration: number;
  width?: number;
  height?: number;
}

/** What a conversion did to an asset, reported back to whoever asked. */
export interface ConvertOutcome extends FileConversion {
  assetId: string;
  /** The asset's display name, rewritten to the new extension. */
  name: string;
}

/** What a headless host does for itself, in place of the client's backend
 * seam: it holds the project's files, so it converts and deletes them
 * directly rather than addressing itself over HTTP. */
export interface HostMediaStore {
  convert(projectId: string, source: ConvertSource, opts: ConvertOptions): Promise<FileConversion>;
  drop(projectId: string, fileName: string): Promise<void>;
}

/** In-process conversion holds the whole output in memory. A source past this
 * belongs on a machine with a disk, and saying so beats an out-of-memory tab
 * several minutes in. */
const IN_PROCESS_MAX_BYTES = 2 * 1024 * 1024 * 1024;

const inPage = () => typeof document !== "undefined";

/** Containers that hold the target streams under the name people ask for. */
const MP4_EXT = /\.(mp4|m4v)$/i;

let host: HostMediaStore | null = null;

/** Install the store a headless host does its own media work with. */
export function setHostMediaStore(store: HostMediaStore | null) {
  host = store;
}

/** The MP4 name a source file converts to: same stem, `.mp4` extension. */
export const mp4NameFor = (fileName: string) =>
  `${fileName.replace(/\.[^./\\]+$/, "") || "media"}.mp4`;

/** A display name after conversion. A name carrying the camera's extension
 * gets the new one — the card reading IMG_4450.MOV under a file that is now
 * an MP4 is the card lying. A name with no extension is the user's words and
 * stays as written. */
export const mp4DisplayName = (name: string) =>
  /\.[a-z0-9]{2,4}$/i.test(name) ? mp4NameFor(name) : name;

/**
 * Convert one of a project's media files to MP4, leaving the original in
 * place. The result lands beside it under a deduped name; what points at
 * which file is the caller's to decide.
 */
export async function convertProjectFile(
  projectId: string,
  source: ConvertSource,
  opts: ConvertOptions = {}
): Promise<FileConversion> {
  if (host) return host.convert(projectId, source, opts);
  const backend = opts.backend ?? getBackend();
  // The machine that stores the bytes converts them: the Mac for its own
  // projects, the worker beside R2 for cloud ones. A headless process working
  // a cloud project is that machine, so it converts in-process.
  return backend.kind === "local" || (backend.kind === "cloud" && inPage())
    ? convertOnBackend(projectId, source, backend, opts)
    : convertNearThePage(projectId, source, { ...opts, backend });
}

/**
 * Convert one project asset to MP4 and point the asset at the result.
 *
 * `maxHeight` caps the picture's height, which forces a re-encode when the
 * source is taller. Everything else is decided from the file: streams that
 * already fit the container are copied, the rest are re-encoded.
 */
export async function convertAssetToMp4(
  projectId: string,
  asset: MediaAsset,
  opts: ConvertOptions = {}
): Promise<ConvertOutcome> {
  if (asset.type === "image") throw new UserError("convert_image_refused", "Images have nothing to convert.");
  const backend = opts.backend ?? getBackend();
  const made = await convertProjectFile(
    projectId,
    { fileName: asset.fileName, url: asset.url, name: asset.name },
    { ...opts, backend }
  );

  if (made.unchanged) return { assetId: asset.id, name: asset.name, ...made };

  const url = await storedMediaUrl(projectId, made.fileName, backend);
  const patch: Partial<MediaAsset> = {
    fileName: made.fileName,
    name: mp4DisplayName(asset.name),
    url,
    sizeBytes: await readMediaFileSize(url),
    duration: made.duration,
    ...(made.width !== undefined ? { width: made.width } : {}),
    ...(made.height !== undefined ? { height: made.height } : {}),
    // The filmstrip and waveform were read off bytes that are about to be
    // deleted; enrichAsset fills them again from the new file.
    thumbs: undefined,
    thumbStep: undefined,
    peaks: undefined,
  };
  useEditor.getState().updateAsset(asset.id, patch);
  void enrichAsset({ ...asset, ...patch }).catch(() => {});
  // Best effort: a file left behind costs storage while everything else
  // already points at the new one.
  void dropProjectFile(projectId, asset.fileName, backend).catch(() => {});

  return { assetId: asset.id, name: patch.name!, ...made };
}

/** Delete a project file through whichever side owns the storage. */
export function dropProjectFile(
  projectId: string,
  fileName: string,
  backend: CutBackend = getBackend()
): Promise<unknown> {
  if (host) return host.drop(projectId, fileName);
  return backend.fetch(`/api/cut/projects/${projectId}/media/${encodeURIComponent(fileName)}`, {
    method: "DELETE",
  });
}

/** Server answer shape, identical from the engine's project route and the
 * worker's job result. */
interface ServerConvert {
  fileName?: string;
  duration?: number;
  width?: number;
  height?: number;
  transcodedVideo?: boolean;
  transcodedAudio?: boolean;
  unchanged?: boolean;
  error?: string;
}

// A conversion that failed with nothing more to say.
const convertFailed = () => new UserError("convert_failed", "Could not convert that file.");

const fromServer = (body: ServerConvert): FileConversion => ({
  fileName: body.fileName!,
  reencoded: !!body.transcodedVideo || !!body.transcodedAudio,
  ...(body.unchanged ? { unchanged: true } : {}),
  duration: body.duration ?? 0,
  ...(body.width !== undefined ? { width: body.width, height: body.height } : {}),
});

/** The engine converts the project's file synchronously; the cloud queues a
 * worker job and this polls it. Both answer with the same fields. */
async function convertOnBackend(
  projectId: string,
  source: ConvertSource,
  backend: CutBackend,
  { maxHeight, mode }: ConvertOptions
): Promise<FileConversion> {
  const res = await backend.fetch(`/api/cut/projects/${projectId}/convert`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ file: source.fileName, ...(maxHeight ? { maxHeight } : {}), ...(mode === "reencode" ? { mode } : {}) }),
  });
  let body: ServerConvert;
  if (backend.kind === "cloud") {
    const started = (await res.json().catch(() => ({}))) as { jobId?: string; error?: string };
    if (!res.ok || !started.jobId) throw errorFromBody(started, convertFailed());
    body = await pollCloudJob(
      started.jobId,
      backend,
      convertFailed(),
      { timedOut: new UserError("convert_timed_out", "The conversion took too long.") }
    );
  } else {
    body = (await res.json().catch(() => ({}))) as ServerConvert;
    if (!res.ok) throw errorFromBody(body, convertFailed());
  }
  if (!body.fileName) throw errorFromBody(body, convertFailed());
  return fromServer(body);
}

/** Convert beside the project, borrowing hosted preparation for missing codecs. */
async function convertNearThePage(
  projectId: string,
  source: ConvertSource,
  opts: ConvertOptions
): Promise<FileConversion> {
  // A forced re-encode is for a stream the page reads no frames from, so the
  // page has nothing to encode it from; a headless runtime decodes it itself.
  const decodable = (opts.mode !== "reencode" || !inPage()) && await probeMediaFile(source.url).then(() => true, (error: unknown) => {
    if (error instanceof UnreadableMediaError && error.undecodable) return false;
    throw error;
  });
  if (decodable) {
    try { return await convertInProcess(projectId, source, opts); }
    catch (error) { if (!(error instanceof ConversionUnavailableError)) throw error; }
  }
  if (hasLocalCompute()) {
    try { return await convertOnTheMac(projectId, source, opts); }
    catch { /* The browser still needs a decoder; hosted preparation supplies it. */ }
  }
  const response = resolveRegisteredBlob(source.url) ?? await fetch(source.url).then((res) => {
    if (!res.ok) throw new UserError("file_read_failed", `Could not read ${source.name}.`, { name: source.name });
    return res.blob();
  });
  if (!response) throw new UserError("file_read_failed", `Could not read ${source.name}.`, { name: source.name });
  return withCloudPreparedMedia(
    new File([response], source.fileName, { type: response.type }),
    (prepared) => land(projectId, source, prepared, opts),
    { name: source.name, maxHeight: opts.maxHeight, mode: opts.mode },
  );
}

class ConversionUnavailableError extends Error {}

/**
 * Convert in this process with mediabunny: WebCodecs in a tab, the headless
 * runtime's decoders in a worker. Source packets that already fit the
 * container are copied, which makes the common .mov of H.264 a fast remux
 * with no decoding at all.
 */
async function convertInProcess(
  projectId: string,
  source: ConvertSource,
  opts: ConvertOptions
): Promise<FileConversion> {
  const bytes = resolveRegisteredBlob(source.url);
  if (bytes && bytes.size > IN_PROCESS_MAX_BYTES) {
    throw new ConversionUnavailableError(`${source.name} needs hosted conversion.`);
  }
  const input = openMedia(source.url);
  try {
    const [video, audio] = await Promise.all([videoTrackOf(input), audioTrackOf(input)]);
    const height = video ? await video.getDisplayHeight() : 0;
    const cap = opts.maxHeight && opts.maxHeight > 0 ? Math.round(opts.maxHeight) : 0;
    const shrink = !!(cap && height > cap);
    const forced = opts.mode === "reencode" && !!video;
    // Already the file a conversion would produce — a second pass would copy
    // it to a new name and change nothing.
    if (
      !forced &&
      MP4_EXT.test(source.fileName) &&
      !shrink &&
      (!video || video.codec === "avc") &&
      (!audio || audio.codec === "aac")
    ) {
      return {
        fileName: source.fileName,
        reencoded: false,
        unchanged: true,
        duration: await input.computeDuration(),
        ...(video ? { width: await video.getDisplayWidth(), height } : {}),
      };
    }
    // A stream already in the target codec is copied packet for packet; what
    // is left over is what gets decoded and re-encoded.
    const reencoded =
      (!!video && (video.codec !== "avc" || shrink || forced)) || (!!audio && audio.codec !== "aac");
    const videoOptions: ConversionVideoOptions = {
      codec: "avc",
      ...(shrink ? { height: cap } : {}),
      ...(forced ? { forceTranscode: true } : {}),
    };
    const audioOptions: ConversionAudioOptions = { codec: "aac" };

    const output = new Output({
      format: new Mp4OutputFormat({ fastStart: "in-memory" }),
      target: new BufferTarget(),
    });
    const conversion = await Conversion.init({
      input,
      output,
      video: (track) => track === video ? videoOptions : { discard: true },
      audio: (track) => track === audio ? audioOptions : { discard: true },
    });
    if (conversion.discardedTracks.some(({ track, reason }) =>
      (track === video || track === audio) &&
      (reason === "undecodable_source_codec" || reason === "no_encodable_target_codec")
    )) throw new ConversionUnavailableError("This media needs hosted conversion.");
    if (!conversion.isValid) throw new UserError("file_convert_failed", `Could not convert ${source.name}.`, { name: source.name });
    await conversion.execute();
    const buffer = output.target.buffer;
    if (!buffer) throw new UserError("file_convert_failed", `Could not convert ${source.name}.`, { name: source.name });
    const made = await land(projectId, source, new Blob([buffer], { type: "video/mp4" }), opts);
    return { ...made, reencoded };
  } finally {
    input.dispose();
  }
}

/** Hand the bytes to the Mac and take the MP4 back. The engine keeps nothing:
 * it converts what it is given and answers with the file. */
async function convertOnTheMac(
  projectId: string,
  source: ConvertSource,
  opts: ConvertOptions
): Promise<FileConversion> {
  const bytes = resolveRegisteredBlob(source.url) ?? (await (await fetch(source.url)).blob());
  const form = new FormData();
  form.append("file", bytes, source.fileName);
  if (opts.maxHeight) form.append("maxHeight", String(Math.round(opts.maxHeight)));
  if (opts.mode === "reencode") form.append("mode", opts.mode);
  const res = await engineFetch("/api/cut/convert", { method: "POST", body: form });
  if (!res.ok) {
    const body: unknown = await res.json().catch(() => null);
    throw errorFromBody(body, new UserError("file_convert_failed", `Could not convert ${source.name}.`, { name: source.name }));
  }
  // The handoff only happens for footage this process cannot decode, which is
  // footage the Mac had to re-encode.
  return land(projectId, source, await res.blob(), opts);
}

/** Store a converted blob in the project and read back what it is. */
async function land(
  projectId: string,
  source: ConvertSource,
  blob: Blob,
  opts: ConvertOptions
): Promise<FileConversion> {
  const fileName = await uploadProjectMediaTo(
    opts.backend ?? getBackend(),
    projectId,
    blob,
    mp4NameFor(source.fileName)
  );
  const meta = await probeMediaFile(blob);
  return {
    fileName,
    reencoded: true,
    duration: meta.duration,
    ...(meta.width !== undefined ? { width: meta.width, height: meta.height } : {}),
  };
}