Skip to content

Open source CapCut alternative.

TypeScript
19.7 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { spawnSync } from "node:child_process";
import { cp, mkdir, mkdtemp, rm, stat } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { passWindows, runExport, type ExportSpec, type RenderHandle } from "./exportPipeline";

// The graph tests assert shapes; this one hands the real graph to a real
// ffmpeg. It synthesizes every asset kind from lavfi sources, builds one
// project that uses every ExportSpec feature — framing crops, regions,
// transitions, edge animations, looks, grades, masks (painted and subject),
// keyframed poses, borders, shadows, overlay videos, elements (static and
// slideshow), effects, captions, an audio bed with ducking, whole-video
// fades, a background color, a gap, a clip played backward — and renders it
// end to end. A filter ffmpeg rejects, a broken option value, or an
// unconnected pad fails here before it can fail an export.
//
// It runs once per delivery, because the graph is built in the chroma the
// delivery carries: a ProRes 4444 master composites at 4:4:4, where every
// filter and every `overlay` negotiates a different pixel family than the
// 4:2:0 the H.264 file is built in.
const DELIVERIES = [
  { name: "an H.264 MP4", ext: "mp4", codec: "h264", pixFmt: "yuv420p", trc: "bt709", over: {} },
  {
    // ProRes 4444's bitstream carries twelve bits, so the file reads back as
    // yuv444p12le whatever the encoder was handed.
    name: "a ProRes 4444 master",
    ext: "mov",
    codec: "prores",
    pixFmt: "yuv444p12le",
    trc: "bt709",
    over: { codec: "prores4444", container: "mov", audioCodec: "pcm" },
  },
  {
    // An HDR delivery composites at ten bits; every alpha segment (the
    // masked keyed clip, the subject matte, the masked overlay) must reach
    // the Main 10 file without an 8-bit hop.
    name: "an HLG HEVC file",
    ext: "mp4",
    codec: "hevc",
    pixFmt: "yuv420p10le",
    trc: "arib-std-b67",
    over: { codec: "hevc", colorSpace: "hlg" },
  },
] as const satisfies readonly {
  name: string;
  ext: string;
  codec: string;
  /** What the finished file decodes as. */
  pixFmt: string;
  /** The transfer the file's header signals. */
  trc: string;
  over: Partial<ExportSpec>;
}[];

const available = (cmd: string) =>
  spawnSync(cmd, ["-version"], { stdio: "ignore" }).status === 0;
const tools = available("ffmpeg") && available("ffprobe");

const ff = (args: string[]) => {
  const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", ...args], {
    encoding: "utf8",
  });
  if (r.status !== 0) throw new Error(`ffmpeg ${args.join(" ")} failed:\n${r.stderr}`);
};

const probe = (file: string) => {
  const r = spawnSync(
    "ffprobe",
    [
      "-v", "error",
      "-show_entries", "format=duration",
      "-show_entries", "stream=codec_type,codec_name,pix_fmt,width,height,color_transfer",
      "-of", "json",
      file,
    ],
    { encoding: "utf8" }
  );
  if (r.status !== 0) throw new Error(`ffprobe failed:\n${r.stderr}`);
  return JSON.parse(r.stdout) as {
    format: { duration: string };
    streams: {
      codec_type: string;
      codec_name: string;
      pix_fmt?: string;
      width?: number;
      height?: number;
      color_transfer?: string;
    }[];
  };
};

const W = 320;
const H = 568;

/** Synthesizes every asset kind from lavfi sources into `mediaDir` (project
 * media) and `tmpDir` (client-painted stills), and returns one project that
 * uses every ExportSpec feature over them. */
async function everyFeature(mediaDir: string, tmpDir: string): Promise<ExportSpec> {
  // ---- Footage, stills, and the audio bed (project media) ----------
  const vid = (name: string, src: string, hz: number, secs: number) =>
    ff([
      "-f", "lavfi", "-i", `${src}=size=320x240:rate=24:duration=${secs}`,
      "-f", "lavfi", "-i", `sine=frequency=${hz}:sample_rate=44100:duration=${secs}`,
      "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac",
      path.join(mediaDir, name),
    ]);
  vid("a.mp4", "testsrc2", 440, 3);
  vid("b.mp4", "smptebars", 660, 3);
  vid("o.mp4", "testsrc", 880, 3);
  ff([
    "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=1:duration=1",
    "-frames:v", "1", path.join(mediaDir, "img.png"),
  ]);
  ff([
    "-f", "lavfi", "-i", "sine=frequency=220:sample_rate=44100:duration=6",
    "-c:a", "aac", path.join(mediaDir, "music.m4a"),
  ]);

  // ---- Client-painted stills and the person matte (job.tmpDir) -----
  const png = (name: string, chain: string) =>
    ff(["-f", "lavfi", "-i", chain, "-frames:v", "1", path.join(tmpDir, name)]);
  png("mask.png", `color=c=white:s=${W}x${H},format=gray`);
  png("omask.png", "color=c=0xC0C0C0:s=160x160,format=gray");
  png("el.png", `[email protected]:s=${W}x${H},format=rgba`);
  png("f0.png", "color=c=blue:s=120x80,format=rgba");
  png("f1.png", "color=c=green:s=120x80,format=rgba");
  png("blank.png", "[email protected]:s=120x80,format=rgba");
  png("border.png", `[email protected]:s=${W}x${H},format=rgba`);
  png("oborder.png", "[email protected]:s=128x256,format=rgba");
  png("shadow.png", `[email protected]:s=${W}x${H},format=rgba`);
  png("cap.png", `[email protected]:s=${W}x${H},format=rgba`);
  png("sub_blank.png", `[email protected]:s=${W}x${H},format=rgba`);
  ff([
    "-f", "lavfi", "-i", `color=c=gray:s=${W}x${H}:d=6:r=24`,
    "-pix_fmt", "yuv420p", path.join(tmpDir, "matte.mp4"),
  ]);

  // Timeline: 1.6s (sped-up cover crop) + 2s (graded, masked, keyed)
  // + 1s gap + 1.5s regioned still = 6.1s.
  return {
    projectId: "e2e",
    width: W,
    height: H,
    fps: 24,
    crf: 30,
    preset: "ultrafast",
    duration: 6.1,
    background: "#102030",
    clips: [
      {
        file: "a.mp4", in: 0, out: 2, muted: false, volume: 0.8,
        fit: "fill", zoom: 1.4, panX: 0.3, panY: -0.2,
        speed: 1.25,
        look: "vhs", lookAmount: 0.7,
        animIn: { style: "pop", seconds: 0.4 },
        transition: 0.5, transitionStyle: "crosszoom",
      },
      {
        file: "b.mp4", in: 0, out: 2, muted: false,
        fit: "fit", reverse: true,
        grade: { brightness: 5, contrast: 8, saturation: -10, temperature: 12, hue: 10 },
        mask: { file: "mask.png" },
        kf: [
          { t: 0, x: 0.5, y: 0.5, scale: 1, rotation: 0, opacity: 1 },
          { t: 1.5, x: 0.6, y: 0.4, scale: 0.8, rotation: 15, opacity: 1 },
        ],
        animOut: { style: "fade", seconds: 0.4 },
        transition: 0.4, transitionStyle: "wipeleft",
      },
      { file: "", in: 0, out: 1, muted: true, hidden: true },
      {
        file: "img.png", in: 0, out: 1.5, muted: true, image: true,
        frame: { x: 0.5, y: 0.45, w: 0.45, h: 0.5 }, fit: "fill",
        mask: { subject: { feather: 1 } },
        border: "border.png",
        shadow: { file: "shadow.png" },
      },
    ],
    overlayVideos: [
      {
        file: "o.mp4", in: 0.2, out: 1.8, start: 0.8, track: 1,
        frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 },
        fit: "fill", zoom: 1.3, panX: -0.4,
        muted: false, volume: 0.5, speed: 2,
        headFade: 0.3,
        mask: { file: "omask.png" },
        border: "oborder.png",
        kf: [
          { t: 0, x: 0.75, y: 0.7, scale: 1, rotation: 0, opacity: 1 },
          { t: 0.6, x: 0.65, y: 0.75, scale: 1.1, rotation: -10, opacity: 1 },
        ],
      },
    ],
    audio: [
      { file: "music.m4a", in: 0, out: 4, start: 0, volume: 0.6, fadeIn: 0.2, fadeOut: 0.4 },
      { file: "a.mp4", in: 0, out: 1, start: 2, volume: 1, duck: 0.3, speed: 1.1 },
    ],
    overlays: [
      { file: "el.png", start: 0.5, end: 2.5, lane: 1 },
      {
        frames: [
          { file: "f0.png", duration: 0.5 },
          { file: "f1.png", duration: 0.5 },
        ],
        blank: "blank.png", x: 40, y: 60, start: 1, end: 2,
        subject: { invert: true, feather: 2 }, lane: 0,
      },
    ],
    behindMask: { file: "matte.mp4", from: 0.5 },
    effects: [
      { effect: "vignette", amount: 0.6, start: 0.5, end: 2, lane: 0 },
      { effect: "zoom", amount: 0.4, focus: { x: 0.5, y: 0.4 }, ramp: 0.4, start: 2, end: 3, lane: 1 },
    ],
    captions: [
      { file: "cap.png", start: 0.3, end: 1.2 },
      { file: "cap.png", start: 1.4, end: 2.0 },
    ],
  };
}

for (const delivery of DELIVERIES) describe("export pipeline end to end", () => {
  test.skipIf(!tools)(
    `a project using every pipeline feature renders through real ffmpeg as ${delivery.name}`,
    async () => {
      const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));
      const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));
      try {
        const spec: ExportSpec = {
          ...(await everyFeature(mediaDir, tmpDir)),
          ...delivery.over,
        };

        const job: RenderHandle = {
          tmpDir,
          outPath: path.join(tmpDir, `out.${delivery.ext}`),
          progress: 0,
          log: [],
        };
        await runExport(job, spec, (file) => path.join(mediaDir, file));

        const info = await stat(job.outPath);
        expect(info.size).toBeGreaterThan(0);
        const meta = probe(job.outPath);
        expect(Number(meta.format.duration)).toBeGreaterThan(spec.duration - 0.3);
        expect(Number(meta.format.duration)).toBeLessThan(spec.duration + 0.3);
        const video = meta.streams.find((s) => s.codec_type === "video");
        expect(video?.width).toBe(W);
        expect(video?.height).toBe(H);
        expect(video?.codec_name).toBe(delivery.codec);
        // The chroma the composite was built in is the chroma the file holds:
        // 4:4:4 all the way through for the master, 4:2:0 for the MP4.
        expect(video?.pix_fmt).toBe(delivery.pixFmt);
        // The header signals the delivery's transfer whatever the container:
        // a MOV takes it from the frames, an MP4 from the bitstream.
        expect(video?.color_transfer).toBe(delivery.trc);
        expect(meta.streams.some((s) => s.codec_type === "audio")).toBe(true);
      } finally {
        await rm(mediaDir, { recursive: true, force: true });
        await rm(tmpDir, { recursive: true, force: true });
      }
    },
    120_000
  );
});

/** Every frame and sample a file decodes to, one hash per frame. */
// A render's content: each picture's hash, and every decoded sample as
// 16-bit — a single pass hands the muxer sound in whatever packets the graph
// made, a join in even ones, so the samples compare whole.
const ffmpegOut = (args: string[]) => {
  const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", ...args], { maxBuffer: 256 << 20 });
  if (r.status !== 0) {
    throw new Error(`ffmpeg ${args.join(" ")} failed:\n${r.stderr}`);
  }
  return r.stdout;
};
const framesOf = (file: string) => ({
  pictures: ffmpegOut(["-i", file, "-map", "0:v?", "-f", "framemd5", "-"]).toString().split("\n").filter((l) => !l.startsWith("#")),
  samples: new Int16Array(new Uint8Array(ffmpegOut(["-i", file, "-map", "0:a?", "-f", "s16le", "-c:a", "pcm_s16le", "-"])).buffer),
});

// The largest gap between two runs of samples, which must be as long.
const sampleGap = (a: Int16Array, b: Int16Array) => {
  expect(a.length).toBe(b.length);
  return a.reduce((gap, v, i) => Math.max(gap, Math.abs(v - b[i])), 0);
};

/** The spec rendered in passes of `pieces`, at the default memory budget. */
const inPieces = (spec: ExportSpec, pieces: number): ExportSpec => ({ ...spec, passes: { holdMB: 2048, pieces } });
/** More pieces than any fixture has: the whole delivery in one pass. */
const ONE_PASS = 10_000;

describe("a delivery rendered in passes", () => {
  // The same every-feature project rendered in one pass and in passes of two
  // pieces. ProRes is intra-only and the sound is PCM, so the same frames and
  // samples encode to the same bits whichever pass made them: the joined
  // file, its mastered mix and its stems decode exactly as the one pass's do.
  // Both sounds are resampled (the mix to 48kHz, the stems to theirs), the
  // case where a pass cut off the sample grid would show.
  test.skipIf(!tools)(
    "joins into the frames, mix and stems one pass renders",
    async () => {
      const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));
      const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));
      const jobsDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-jobs-"));
      try {
        const spec: ExportSpec = {
          ...(await everyFeature(mediaDir, tmpDir)),
          codec: "prores4444",
          container: "mov",
          audioCodec: "pcm",
          audioSampleRate: 48000,
          loudness: -16,
          truePeakCeiling: -1,
          stemPlan: [
            { name: "Dialogue", lane: null, file: "1 Dialogue.wav" },
            { name: "Music", lane: 0, file: "2 Music.wav" },
          ],
        };
        expect(passWindows(inPieces(spec, 2), () => ({ pixels: 0, fps: 30 })).length).toBeGreaterThan(2);

        // Each render runs in its own copy of the job dir, so the stems it
        // packs stay on disk beside its file.
        const render = async (name: string, passPieces: number) => {
          const dir = path.join(jobsDir, name);
          await cp(tmpDir, dir, { recursive: true });
          const job: RenderHandle = {
            tmpDir: dir,
            outPath: path.join(dir, "out.mov"),
            stemsPath: path.join(dir, "stems.zip"),
            progress: 0,
            log: [],
          };
          await runExport(job, inPieces(spec, passPieces), (file) => path.join(mediaDir, file));
          return { file: job.outPath, stems: spec.stemPlan!.map((_, i) => path.join(dir, `stem_${i}.wav`)) };
        };
        const whole = await render("whole", ONE_PASS);
        const passes = await render("passes", 2);

        expect(framesOf(passes.file)).toEqual(framesOf(whole.file));
        for (const [i, stem] of passes.stems.entries()) {
          expect(framesOf(stem)).toEqual(framesOf(whole.stems[i]));
        }
      } finally {
        await rm(mediaDir, { recursive: true, force: true });
        await rm(tmpDir, { recursive: true, force: true });
        await rm(jobsDir, { recursive: true, force: true });
      }
    },
    240_000
  );
});

describe("a long recording cut out of order", () => {
  // Cuts from all over one 40s recording, played out of source order, with
  // sound handles, a split edit, a transition, a cross dissolve, a layered
  // clip and a sound entry reading the same file late. Rendered in passes of
  // three, each pass opens the file at its own earliest read; the file
  // decodes exactly as the one pass's does.
  test.skipIf(!tools)(
    "renders in passes as in one",
    async () => {
      const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));
      const jobsDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-jobs-"));
      try {
        ff([
          "-f", "lavfi", "-i", "testsrc2=size=320x240:rate=24:duration=40",
          "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=40",
          "-shortest", "-pix_fmt", "yuv420p", "-c:a", "aac", "-g", "48",
          path.join(mediaDir, "talk.mp4"),
        ]);
        const at = [31, 4, 22, 12, 36, 8, 27, 17, 2, 33, 14, 25];
        const clips: ExportSpec["clips"] = at.map((from, k) => ({
          file: "talk.mp4",
          in: from,
          // 37 frames: window edges fall between milliseconds.
          out: from + 37 / 24,
          muted: false,
          soundBack: 0.4,
          soundAhead: 0.4,
          ...(k === 3 ? { speed: 1.5 } : {}),
          ...(k === 5 ? { transition: 0.4 } : {}),
          ...(k === 7 ? { soundCross: 0.2 } : {}),
          ...(k === 9 ? { soundLead: 0.3, splitFade: 0.05 } : {}),
        }));
        const spec: ExportSpec = {
          projectId: "e2e",
          width: W,
          height: H,
          fps: 24,
          crf: 30,
          preset: "ultrafast",
          duration: clips.reduce((sum, c) => sum + (c.out - c.in) / (c.speed ?? 1), 0),
          clips,
          overlayVideos: [
            {
              file: "talk.mp4", in: 29, out: 31, start: 12, track: 1,
              frame: { x: 0.55, y: 0.5, w: 0.4, h: 0.45 }, muted: false, volume: 0.5,
            },
          ],
          audio: [{ file: "talk.mp4", in: 35, out: 38, start: 9, volume: 0.4 }],
          overlays: [],
          codec: "prores4444",
          container: "mov",
          audioCodec: "pcm",
        };
        expect(passWindows(inPieces(spec, 3), () => ({ pixels: 0, fps: 30 })).length).toBeGreaterThan(3);

        const render = async (name: string, passPieces: number) => {
          const dir = path.join(jobsDir, name);
          await mkdir(dir);
          const job: RenderHandle = { tmpDir: dir, outPath: path.join(dir, "out.mov"), progress: 0, log: [] };
          await runExport(job, inPieces(spec, passPieces), (file) => path.join(mediaDir, file));
          return job.outPath;
        };
        const whole = await render("whole", ONE_PASS);
        const passes = await render("passes", 3);
        // An AAC decoder fills noise-coded bands from a generator a seek
        // restarts, so a pass that reads from its window's first source
        // second carries the same sound to within one 16-bit step.
        const [got, want] = [framesOf(passes), framesOf(whole)];
        expect(got.pictures).toEqual(want.pictures);
        expect(sampleGap(got.samples, want.samples)).toBeLessThanOrEqual(1);
      } finally {
        await rm(mediaDir, { recursive: true, force: true });
        await rm(jobsDir, { recursive: true, force: true });
      }
    },
    240_000
  );
});

describe("a flash shorter than an output frame", () => {
  // A 1/60 s red shot on track 1 at 0.01s sits between frames 0 and 1 of a
  // 30 fps file; a blue shot follows it. The red holds frame 1 and the blue
  // opens on frame 2, over black on track 0.
  test.skipIf(!tools)(
    "holds the frame after its start, and the shot after it opens one later",
    async () => {
      const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));
      const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));
      try {
        const solid = (name: string, c: string) =>
          ff(["-f", "lavfi", "-i", `color=c=${c}:s=64x64:r=60:d=1`, "-pix_fmt", "yuv420p", path.join(mediaDir, name)]);
        solid("black.mp4", "black");
        solid("red.mp4", "red");
        solid("blue.mp4", "blue");
        const flash = 1 / 60;
        const spec: ExportSpec = {
          projectId: "p",
          width: 64,
          height: 64,
          fps: 30,
          crf: 18,
          preset: "veryfast",
          duration: 0.5,
          clips: [{ file: "black.mp4", in: 0, out: 0.5, muted: true }],
          overlayVideos: [
            { file: "red.mp4", in: 0, out: flash, start: 0.01, track: 1, muted: true },
            { file: "blue.mp4", in: 0, out: 0.4, start: 0.01 + flash, track: 1, muted: true },
          ],
          audio: [],
          overlays: [],
        };
        const job: RenderHandle = { tmpDir, outPath: path.join(tmpDir, "out.mp4"), stemsPath: path.join(tmpDir, "stems.zip"), progress: 0, log: [] };
        await runExport(job, spec, (file) => path.join(mediaDir, file));
        // A middle pixel of each of the first four frames, as RGB (a 2x2 crop:
        // 4:2:0 frames crop on even edges).
        const px = new Uint8Array(
          ffmpegOut(["-i", job.outPath, "-frames:v", "4", "-vf", "crop=2:2:32:32", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"])
        );
        const hue = (k: number) => {
          const [r, g, b] = px.slice(k * 12, k * 12 + 3);
          return r > 150 && b < 100 ? "red" : b > 150 && r < 100 ? "blue" : r < 40 && g < 40 && b < 40 ? "black" : `${r},${g},${b}`;
        };
        expect([0, 1, 2, 3].map(hue)).toEqual(["black", "red", "blue", "blue"]);
      } finally {
        await rm(mediaDir, { recursive: true, force: true });
        await rm(tmpDir, { recursive: true, force: true });
      }
    },
    60_000
  );
});