Skip to content

Write HTML. Render video. Built for agents.

13.3 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { buildStitchedHeader, readMp4Layout, readTracks, writeStitchedMp4 } from "./mp4Stitch.js";

function box(type: string, ...body: Buffer[]): Buffer {
  const header = Buffer.alloc(8);
  header.writeUInt32BE(8 + body.reduce((n, part) => n + part.length, 0));
  header.write(type, 4, "latin1");
  return Buffer.concat([header, ...body]);
}

function u32(...values: number[]): Buffer {
  const buf = Buffer.alloc(values.length * 4);
  values.forEach((value, i) => buf.writeUInt32BE(value, i * 4));
  return buf;
}
const full = (type: string, ...body: Buffer[]) => box(type, u32(0), ...body);
// Version 0 mvhd/tkhd/mdhd with only the fields the stitcher reads filled in.
const header = (type: string, timescaleOrTrack: number, duration: number) =>
  type === "tkhd"
    ? full(type, u32(0, 0, timescaleOrTrack, 0, duration), Buffer.alloc(60))
    : full(type, u32(0, 0, timescaleOrTrack, duration), Buffer.alloc(type === "mvhd" ? 80 : 4));

interface FakeTrack {
  handler: "vide" | "soun";
  timescale: number;
  mediaTime: number;
  durations: number[];
  sizes: number[];
  sync?: number[];
  codec?: string;
}

/** A chunk file shaped like ffmpeg's: ftyp, mdat with samples interleaved one per mp4 chunk, moov
 * (or the moov first, as a faststart file has it). */
function fakeChunk(
  tag: number,
  tracks: FakeTrack[],
  moovFirst = false,
): { file: Buffer; samples: Buffer[][] } {
  const samples = tracks.map((track, t) =>
    track.sizes.map((size, k) => Buffer.alloc(size, (tag * 61 + t * 17 + k * 5 + 1) & 255)),
  );
  const ftyp = box("ftyp", Buffer.from("isom\0\0\x02\0isom", "latin1"));
  const moovAt = (payloadAt: number) => {
    const offsets = tracks.map(() => [] as number[]);
    const payload: Buffer[] = [];
    let at = payloadAt;
    for (let k = 0; k < Math.max(...tracks.map((track) => track.sizes.length)); k++) {
      samples.forEach((track, t) => {
        if (!track[k]) return;
        offsets[t]!.push(at);
        payload.push(track[k]!);
        at += track[k]!.length;
      });
    }
    const traks = tracks.map((track, t) => {
      const duration = track.durations.reduce((n, d) => n + d, 0);
      const entry = track.codec ?? (track.handler === "vide" ? "avc1" : "mp4a");
      const stbl = box(
        "stbl",
        full("stsd", u32(1), box(entry, Buffer.alloc(8, t))),
        full("stts", u32(track.durations.length, ...track.durations.flatMap((d) => [1, d]))),
        ...(track.sync ? [full("stss", u32(track.sync.length, ...track.sync))] : []),
        full("stsc", u32(1, 1, 1, 1)),
        full("stsz", u32(0, track.sizes.length, ...track.sizes)),
        full("stco", u32(offsets[t]!.length, ...offsets[t]!)),
      );
      return box(
        "trak",
        header("tkhd", t + 1, 0),
        box("edts", full("elst", u32(1, 0, track.mediaTime, 0x10000))),
        box(
          "mdia",
          header("mdhd", track.timescale, duration),
          full("hdlr", u32(0), Buffer.from(track.handler), Buffer.alloc(13)),
          box("minf", stbl),
        ),
      );
    });
    return { moov: box("moov", header("mvhd", 1000, 0), ...traks), mdat: box("mdat", ...payload) };
  };
  if (!moovFirst) {
    const { moov, mdat } = moovAt(ftyp.length + 8);
    return { file: Buffer.concat([ftyp, mdat, moov]), samples };
  }
  // The moov's size does not depend on the offsets it holds.
  const { moov, mdat } = moovAt(ftyp.length + moovAt(0).moov.length + 8);
  return { file: Buffer.concat([ftyp, moov, mdat]), samples };
}

const memoryRange = (file: Buffer) => async (start: number, end: number) =>
  file.subarray(start, end);

/** Every sample of `track`, read from `file` where its tables place it. */
function samplesOf(file: Buffer, track: ReturnType<typeof readTracks>[number]): Buffer[] {
  const read: Buffer[] = [];
  let k = 0;
  track.chunkOffsets.forEach((offset, c) => {
    for (let s = 0, at = offset; s < track.samplesPerChunk[c]!; s++) {
      read.push(file.subarray(at, at + track.sizes[k]!));
      at += track.sizes[k++]!;
    }
  });
  return read;
}

/** The stitched file: the header, then `audio`'s and each chunk's mdat payload in order. */
async function stitchInMemory(files: Buffer[], audio?: Buffer): Promise<Buffer> {
  const sources = audio ? [audio, ...files] : files;
  const layouts = await Promise.all(sources.map((file) => readMp4Layout(memoryRange(file))));
  return Buffer.concat([
    audio ? buildStitchedHeader(layouts.slice(1), layouts[0]) : buildStitchedHeader(layouts),
    ...layouts.map((l, i) => sources[i]!.subarray(l.payloadStart, l.payloadEnd)),
  ]);
}

/** A track's tkhd track_ID (version 0, as fakeChunk writes it). */
const trackIdOf = (moov: Buffer, track: ReturnType<typeof readTracks>[number]) =>
  moov.readUInt32BE(track.trak.bodyStart + 8 + 12);

describe("buildStitchedHeader", () => {
  const video = (sizes: number[], mediaTime = 0): FakeTrack => ({
    handler: "vide",
    timescale: 15360,
    mediaTime,
    durations: sizes.map(() => 512),
    sizes,
    sync: [1],
  });
  const audio = (durations: number[], mediaTime: number): FakeTrack => ({
    handler: "soun",
    timescale: 48000,
    mediaTime,
    durations,
    sizes: durations.map((_, k) => 40 + k),
  });

  it("indexes every chunk's samples at their place after the header", async () => {
    const chunks = [
      fakeChunk(0, [video([500, 120, 130]), audio([1024, 1024, 1024, 1024], 1024)]),
      fakeChunk(1, [video([480, 110]), audio([1024, 1024, 640], 0)]),
    ];
    const layouts = await Promise.all(chunks.map((c) => readMp4Layout(memoryRange(c.file))));
    const header = buildStitchedHeader(layouts);
    const file = Buffer.concat([
      header,
      ...layouts.map((l, i) => chunks[i]!.file.subarray(l.payloadStart, l.payloadEnd)),
    ]);

    const stitched = await readMp4Layout(memoryRange(file));
    expect(stitched.payloadStart).toBe(header.length);
    expect(stitched.payloadEnd).toBe(file.length);
    const tracks = readTracks(stitched.moov);
    tracks.forEach((track, t) => {
      expect(samplesOf(file, track)).toEqual(chunks.flatMap((chunk) => chunk.samples[t]!));
    });
    expect(tracks.map((track) => [track.handler, track.mediaTime])).toEqual([
      ["vide", 0],
      ["soun", 1024],
    ]);
    expect(tracks[0]!.sync).toEqual([true, false, false, true, false]);
    expect(tracks[1]!.durations).toEqual([1024, 1024, 1024, 1024, 1024, 1024, 640]);

    const moov = stitched.moov;
    const after = (type: string, from = 0) => moov.indexOf(type, from, "latin1") + 4;
    // Only audio needs an edit list (version 1): it shows its media minus the 1024-sample
    // priming, (6784 - 1024) / 48000 s = 120 ms in the 1000-tick movie timescale.
    const audioElst = after("elst");
    expect(moov.indexOf("elst", audioElst, "latin1")).toBe(-1);
    expect(audioElst).toBeGreaterThan(after("vide"));
    expect(Number(moov.readBigUInt64BE(audioElst + 8))).toBe(120);
    expect(Number(moov.readBigInt64BE(audioElst + 16))).toBe(1024);
    expect(moov.readUInt32BE(after("mdhd", audioElst) + 16)).toBe(6784);
    expect(moov.readUInt32BE(after("mvhd") + 16)).toBe(167);
  });

  it("refuses a chunk whose video starts late, as B-frames make it", async () => {
    const chunks = [fakeChunk(0, [video([500, 120])]), fakeChunk(1, [video([480, 110], 512)])];
    const layouts = await Promise.all(chunks.map((c) => readMp4Layout(memoryRange(c.file))));
    expect(() => buildStitchedHeader(layouts)).toThrow("chunk 1 video starts at media time 512");
  });

  // A tail shorter than one AAC packet owns no packet (the one before it spans the boundary), so
  // its chunk is muxed video-only; the plan's audio track still holds every packet once.
  it("keeps the plan's audio when a chunk owns no audio packets", async () => {
    const chunks = [
      fakeChunk(0, [video([500, 120, 130]), audio([1024, 1024, 1024, 640], 1024)]),
      fakeChunk(1, [video([90])]),
    ];
    const file = await stitchInMemory(chunks.map((chunk) => chunk.file));
    const tracks = readTracks((await readMp4Layout(memoryRange(file))).moov);
    expect(tracks.map((track) => [track.handler, track.mediaTime])).toEqual([
      ["vide", 0],
      ["soun", 1024],
    ]);
    expect(samplesOf(file, tracks[0]!)).toEqual([
      ...chunks[0]!.samples[0]!,
      ...chunks[1]!.samples[0]!,
    ]);
    expect(samplesOf(file, tracks[1]!)).toEqual(chunks[0]!.samples[1]!);
    expect(tracks[1]!.durations).toEqual([1024, 1024, 1024, 640]);
  });

  it("puts a separate audio file's track and payload before every chunk's", async () => {
    const chunks = [fakeChunk(0, [video([500, 120])]), fakeChunk(1, [video([90])])];
    const mix = fakeChunk(2, [audio([1024, 1024, 640], 1024)]);
    const file = await stitchInMemory(
      chunks.map((chunk) => chunk.file),
      mix.file,
    );
    const { moov } = await readMp4Layout(memoryRange(file));
    const tracks = readTracks(moov);
    expect(tracks.map((track) => [track.handler, track.mediaTime])).toEqual([
      ["vide", 0],
      ["soun", 1024],
    ]);
    expect(tracks.map((track) => trackIdOf(moov, track))).toEqual([1, 2]);
    expect(samplesOf(file, tracks[0]!)).toEqual([
      ...chunks[0]!.samples[0]!,
      ...chunks[1]!.samples[0]!,
    ]);
    expect(samplesOf(file, tracks[1]!)).toEqual(mix.samples[0]!);
    expect(tracks[1]!.durations).toEqual([1024, 1024, 640]);
    expect(Math.max(...tracks[1]!.chunkOffsets)).toBeLessThan(Math.min(...tracks[0]!.chunkOffsets));
  });

  it.each([0, 1])(
    "refuses chunk %i carrying audio when the audio is a separate file",
    async (carrier) => {
      const chunk = (i: number) =>
        fakeChunk(i, i === carrier ? [video([500]), audio([1024], 0)] : [video([500])]);
      const layouts = await Promise.all(
        [chunk(0), chunk(1)].map((c) => readMp4Layout(memoryRange(c.file))),
      );
      const mix = await readMp4Layout(memoryRange(fakeChunk(2, [audio([1024], 1024)]).file));
      expect(() => buildStitchedHeader(layouts, mix)).toThrow(
        `chunk ${carrier} carries its own audio`,
      );
    },
  );

  it.each([
    [
      "an audio codec chunk 0 does not use",
      [video([500]), audio([1024], 1024)],
      [video([90]), { ...audio([1024], 0), codec: "Opus" }],
    ],
    ["no video", [video([500]), audio([1024], 1024)], [audio([1024], 0)]],
    ["audio chunk 0 does not carry", [video([500])], [video([90]), audio([1024], 0)]],
  ] as const)("refuses a chunk with %s", async (_, first, second) => {
    const layouts = await Promise.all(
      [fakeChunk(0, [...first]), fakeChunk(1, [...second])].map((c) =>
        readMp4Layout(memoryRange(c.file)),
      ),
    );
    expect(() => buildStitchedHeader(layouts)).toThrow("chunk 1 tracks differ from chunk 0");
  });
});

describe("writeStitchedMp4", () => {
  const video = (sizes: number[], mediaTime = 0): FakeTrack => ({
    handler: "vide",
    timescale: 15360,
    mediaTime,
    durations: sizes.map(() => 512),
    sizes,
    sync: [1],
  });
  let dir: string;
  beforeEach(() => {
    dir = mkdtempSync(join(tmpdir(), "hf-mp4-stitch-"));
  });
  afterEach(() => {
    rmSync(dir, { recursive: true, force: true });
  });
  const write = (name: string, file: Buffer) => {
    writeFileSync(join(dir, name), file);
    return join(dir, name);
  };

  it("writes the header and every payload, replacing an earlier output", async () => {
    const files = [fakeChunk(0, [video([500, 120])]).file, fakeChunk(1, [video([480])]).file];
    const output = write("out.mp4", Buffer.from("stale"));
    await writeStitchedMp4([write("a.mp4", files[0]!), write("b.mp4", files[1]!)], output);
    expect(readFileSync(output).equals(await stitchInMemory(files))).toBe(true);
    expect(readdirSync(dir).sort()).toEqual(["a.mp4", "b.mp4", "out.mp4"]);
  });

  it("writes a separate audio file's payload right after the header", async () => {
    const files = [fakeChunk(0, [video([500, 120])]).file, fakeChunk(1, [video([480])]).file];
    const mix = fakeChunk(2, [
      {
        handler: "soun",
        timescale: 48000,
        mediaTime: 1024,
        durations: [1024, 640],
        sizes: [40, 41],
      },
    ]).file;
    const output = join(dir, "out.mp4");
    await writeStitchedMp4(
      [write("a.mp4", files[0]!), write("b.mp4", files[1]!)],
      output,
      write("audio.m4a", mix),
    );
    expect(readFileSync(output).equals(await stitchInMemory(files, mix))).toBe(true);
  });

  it("opens no output for chunks it refuses", async () => {
    const paths = [
      write("a.mp4", fakeChunk(0, [video([500])]).file),
      write("b.mp4", fakeChunk(1, [video([480], 512)]).file),
    ];
    await expect(writeStitchedMp4(paths, join(dir, "out.mp4"))).rejects.toThrow(
      "chunk 1 video starts at media time 512",
    );
    expect(readdirSync(dir).sort()).toEqual(["a.mp4", "b.mp4"]);
  });

  it("fails a chunk whose mdat is cut short mid-copy and leaves no output", async () => {
    const whole = fakeChunk(1, [video([480, 300])], true).file;
    const paths = [
      write("a.mp4", fakeChunk(0, [video([500])]).file),
      write("b.mp4", whole.subarray(0, whole.length - 200)),
    ];
    await expect(writeStitchedMp4(paths, join(dir, "out.mp4"))).rejects.toThrow(
      "chunk 1 mdat holds 580 of 780 payload bytes",
    );
    expect(readdirSync(dir).sort()).toEqual(["a.mp4", "b.mp4"]);
  });
});