Skip to content

Open source CapCut alternative.

TypeScript
9.0 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { stubModule } from "@/lib/testing/stubModule";
import { AudioBuffer, OfflineAudioContext } from "node-web-audio-api";

// Where the offline fold lands a clip's sound when it leaves its picture: a
// split edit reaching past either edge, and a recording bound to the video.
// Each file reads back silence with one click at a source second of its own,
// so where the click lands in the mix says exactly which source second played
// at which timeline second.
const RATE = 44100;

const globals = globalThis as Record<string, unknown>;
globals.AudioBuffer ??= AudioBuffer;
globals.OfflineAudioContext ??= OfflineAudioContext;

/** Each file's click, in its own source seconds, and its length. */
const FILES: Record<string, { click?: number; duration: number }> = {
  "quiet.mp4": { duration: 60 },
  "lead.mp4": { click: 4.8, duration: 60 },
  "tail.mp4": { click: 4.3, duration: 60 },
  "cam.mp4": { click: 12, duration: 60 },
  "rec.wav": { click: 8.5, duration: 60 },
  "late.wav": { click: 1, duration: 60 },
  "layer.mp4": { click: 4.7, duration: 60 },
};

await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, {
  decodeAudioSpan: (async (file: string, from: number, to: number) => {
    const f = FILES[file];
    const lo = Math.max(0, from);
    const hi = Math.min(f.duration, to);
    const buf = new AudioBuffer({ length: Math.max(1, Math.round((hi - lo) * RATE)), numberOfChannels: 1, sampleRate: RATE });
    if (f.click !== undefined && f.click >= lo && f.click < hi) {
      const at = Math.round((f.click - lo) * RATE);
      const ch = buf.getChannelData(0);
      for (let i = 0; i < 32 && at + i < ch.length; i++) ch[at + i] = 1;
    }
    return buf;
  }) as never,
});

const { renderMix } = await import("./audioMix");
type Spec = Parameters<typeof renderMix>[0];

const render = async (spec: Spec) => {
  const buffer = await renderMix(spec, { sampleRate: RATE, channels: 1, resolve: (f) => f });
  expect(buffer).not.toBeNull();
  return buffer!.getChannelData(0);
};

/** The timeline second the click starts at, or null when none sounds. */
const clickAt = (data: Float32Array) => {
  for (let i = 0; i < data.length; i++) if (Math.abs(data[i]) > 0.5) return i / RATE;
  return null;
};

const near = (got: number | null, want: number) => {
  expect(got).not.toBeNull();
  expect(Math.abs(got! - want)).toBeLessThan(2 / RATE);
};

describe("split edits in the offline fold", () => {
  test("a J-cut plays the incoming clip's sound ahead of its picture", async () => {
    // b's picture starts at 4 on source 5; its sound leads by half a second,
    // so source 4.8 plays at 3.8 — over a's picture.
    const mix = await render({
      duration: 8,
      clips: [
        { file: "quiet.mp4", in: 0, out: 4, muted: false },
        { file: "lead.mp4", in: 5, out: 9, muted: false, soundLead: 0.5, soundBack: 0.5, splitFade: 0.03 },
      ],
      items: [],
    });
    near(clickAt(mix), 3.8);
  });

  test("the lead's ramp starts it from silence", async () => {
    // A click inside the ramp: 0.01 s into a 0.03 s fade plays at a third.
    FILES["ramp.mp4"] = { click: 4.51, duration: 60 };
    const mix = await render({
      duration: 8,
      clips: [
        { file: "quiet.mp4", in: 0, out: 4, muted: false },
        { file: "ramp.mp4", in: 5, out: 9, muted: false, soundLead: 0.5, soundBack: 0.5, splitFade: 0.03 },
      ],
      items: [],
    });
    const at = Math.round(3.51 * RATE);
    expect(mix[at]).toBeGreaterThan(0.25);
    expect(mix[at]).toBeLessThan(0.42);
  });

  test("an L-cut carries the outgoing clip's sound past its picture", async () => {
    // a's picture ends at 4 on source 4; its sound runs on, so source 4.3
    // plays at 4.3 — under b's picture.
    const mix = await render({
      duration: 8,
      clips: [
        { file: "tail.mp4", in: 0, out: 4, muted: false, soundTail: 0.5, soundAhead: 0.5, splitFade: 0.03 },
        { file: "quiet.mp4", in: 0, out: 4, muted: false },
      ],
      items: [],
    });
    near(clickAt(mix), 4.3);
  });

  test("without a split edit the trimmed-away sound stays silent", async () => {
    const mix = await render({
      duration: 8,
      clips: [
        { file: "tail.mp4", in: 0, out: 4, muted: false },
        { file: "lead.mp4", in: 5, out: 9, muted: false },
      ],
      items: [],
    });
    expect(clickAt(mix)).toBeNull();
  });

  test("a layer clip's lead lands ahead of its start", async () => {
    const mix = await render({
      duration: 8,
      clips: [{ file: "", in: 0, out: 8, muted: true }],
      items: [{ file: "layer.mp4", in: 5, out: 7, start: 2, volume: 1, soundLead: 0.4, soundBack: 0.4, splitFade: 0.03 }],
    });
    near(clickAt(mix), 1.7);
  });
});

describe("a recording bound to the video", () => {
  test("plays the recording through the offset, and the camera's track stays out", async () => {
    // Recording second = video second − 3, so recording 8.5 is video 11.5,
    // which this clip (in 10, at 0) plays at 1.5. The camera's own click at
    // video 12 must not sound.
    const mix = await render({
      duration: 4,
      clips: [{ file: "cam.mp4", in: 10, out: 14, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],
      items: [],
    });
    near(clickAt(mix), 1.5);
    expect(Math.abs(mix[Math.round(2 * RATE) + 4])).toBeLessThan(0.01);
  });

  test("a recording that rolled after the camera leaves silence before it, in time", async () => {
    // The recording starts at video second 2; the clip starts on video
    // second 1. Recording 1 is video 3, played at timeline 2.
    const mix = await render({
      duration: 4,
      clips: [{ file: "cam.mp4", in: 1, out: 5, muted: false, soundFrom: { file: "late.wav", offset: -2, duration: 60 } }],
      items: [],
    });
    near(clickAt(mix), 2);
  });

  test("a speed change runs the recording through the clip's own map", async () => {
    // At 2× the clip covers video 10..14 in two seconds: video 11.5 plays at 0.75.
    const mix = await render({
      duration: 2,
      clips: [{ file: "cam.mp4", in: 10, out: 14, speed: 2, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],
      items: [],
    });
    const got = clickAt(mix);
    expect(got).not.toBeNull();
    // The stretch smears a click over a few milliseconds.
    expect(Math.abs(got! - 0.75)).toBeLessThan(0.03);
  });

  test("a reversed clip plays the recording backward from its out point", async () => {
    // Reversed, timeline 0 shows video 14 and timeline 4 video 10: video
    // 11.5 lands at 2.5.
    const mix = await render({
      duration: 4,
      clips: [{ file: "cam.mp4", in: 10, out: 14, reverse: true, muted: false, soundFrom: { file: "rec.wav", offset: -3, duration: 60 } }],
      items: [],
    });
    let peak = 0;
    let at = 0;
    for (let i = 0; i < mix.length; i++) if (Math.abs(mix[i]) > peak) { peak = Math.abs(mix[i]); at = i / RATE; }
    expect(Math.abs(at - 2.5)).toBeLessThan(0.002);
  });
});

describe("stems from the document", () => {
  test("the dialogue stem plays the bound recording and the split edit; a lane keeps its own sound", async () => {
    const { mixSpecFor, stemMixSpec } = await import("./exportRender");
    const media = (id: string, type: "video" | "audio", over: Record<string, unknown> = {}) => ({
      id,
      fileName: id,
      name: id,
      type,
      duration: 60,
      url: "",
      ...over,
    });
    // Video second t plays recording t − 2: the recording's click at 8.5 is
    // video 10.5, which clip a (video 10..12) plays at 0.5. Clip b starts at
    // 2 on source 5 and leads by half a second: source 4.8 plays at 1.8. The
    // soundtrack's click at its own 1.0 sits on lane 0.
    const doc = {
      aspect: "16:9",
      assets: [
        media("cam.mp4", "video", { soundFrom: { assetId: "rec.wav", offset: -2 } }),
        media("rec.wav", "audio"),
        media("lead.mp4", "video"),
        media("late.wav", "audio"),
      ],
      clips: [
        { id: "a", assetId: "cam.mp4", start: 0, in: 10, out: 12, track: 0, muted: false },
        { id: "b", assetId: "lead.mp4", start: 2, in: 5, out: 9, track: 0, muted: false, audioLead: 0.5 },
      ],
      audioClips: [{ id: "s", assetId: "late.wav", start: 0, in: 0, out: 4, volume: 1, lane: 0 }],
      overlays: [],
      subtitles: { cues: [] },
    } as unknown as Parameters<typeof mixSpecFor>[0];
    const spec = mixSpecFor(doc, (a) => a.fileName);
    const clicks = (data: Float32Array) => {
      const at: number[] = [];
      let quietSince = -RATE;
      for (let i = 0; i < data.length; i++) {
        if (Math.abs(data[i]) <= 0.5) continue;
        if (i - quietSince > RATE * 0.1) at.push(i / RATE);
        quietSince = i;
      }
      return at;
    };
    const dialogue = clicks(await render(stemMixSpec(spec, { name: "Dialogue", file: "1 Dialogue.wav", lane: null })));
    expect(dialogue).toHaveLength(2);
    near(dialogue[0], 0.5);
    near(dialogue[1], 1.8);
    const lane = clicks(await render(stemMixSpec(spec, { name: "late", file: "2 late.wav", lane: 0 })));
    expect(lane).toHaveLength(1);
    near(lane[0], 1);
  });
});