Skip to content

Open source CapCut alternative.

TypeScript
18.3 KB
Raw Blame
/**
 * What color a video file's header says its code values mean.
 *
 * A source's color is settled from its container alone: the sample entry's
 * `colr` atom names the primaries, transfer and matrix and whether the
 * range is full, the codec configuration names the bit depth, and Apple Log
 * footage carries a log identifier string in the movie's metadata. The probe
 * walks the atom tree by their headers and reads the payloads of only the
 * atoms that can carry those: the video track's sample description, and the
 * metadata and user-data atoms of the movie and that track. The sample
 * tables — the bulk of a long movie's header — are stepped over unread. The
 * bytes come in blocks, so a typical file costs two ranged reads and no
 * decode, and the probe runs the same over a dropped File, a URL and a file
 * handle.
 *
 * Anything that is not a MOV/MP4 — a WebM recording, a Matroska file — reads
 * as Rec.709 8-bit limited, which is what such files carry.
 *
 * The matrix decides whether a file is HDR. Social apps transcode phone
 * footage to 8-bit H.264 through the Rec.709 matrix and keep the source's
 * BT.2020 / HLG tags; those files are SDR, so an HLG or PQ transfer reads as
 * HDR only beside a BT.2020 matrix (or an unspecified one on BT.2020
 * primaries).
 */

import type { CodeFormat, SourceProfile } from "@donkeycut/effects-kit";

/** Read `length` bytes at `offset`; fewer come back at the end of the file. */
export type ReadRange = (offset: number, length: number) => Promise<Uint8Array>;

export interface ColorProbe {
  code: CodeFormat;
  /** The profile the header settles on. */
  detected: SourceProfile;
  /** The video sample entry's four-character code (avc1, hvc1, apch, …);
   * "" when the header has no video. */
  codec: string;
}

export const DEFAULT_COLOR: ColorProbe = {
  code: { matrix: "bt709", fullRange: false, bitDepth: 8 },
  detected: "rec709",
  codec: "",
};

/** Bytes fetched per block. The top-level walk usually finds `ftyp`, the
 * data box's header and, in a fast-start file, the whole movie header in the
 * first one. */
const BLOCK = 64 * 1024;
/** A movie header up to this size is fetched in one read as soon as its
 * header is seen; a longer one — hours of footage, megabytes of sample
 * tables — is read block by block, and only where the walk lands. */
const MOOV_PREFETCH = 512 * 1024;
/** The most of any one metadata or sample-description atom scanned for the
 * log identifiers. Those atoms are a few kilobytes; past this the rest is
 * something else (cover art, a thumbnail track). */
const SCAN_MAX = 64 * 1024;
/** Atoms visited before the walk gives up on a malformed file. */
const MAX_ATOMS = 4096;

/** Identifier strings Apple writes into the movie metadata of log footage. */
const APPLE_LOG_2 = "com.apple.apple-wide-gamut.apple-log";
const APPLE_LOG = "com.apple.rec2020.apple-log";
const APPLE_CUSTOM_GAMMA = "com.apple.proapps.customgamma";

const ascii = (b: Uint8Array, at: number) => String.fromCharCode(b[at], b[at + 1], b[at + 2], b[at + 3]);
const u16 = (b: Uint8Array, at: number) => (b[at] << 8) | b[at + 1];
const u32 = (b: Uint8Array, at: number) => ((b[at] << 24) | (b[at + 1] << 16) | (b[at + 2] << 8) | b[at + 3]) >>> 0;
const u64 = (b: Uint8Array, at: number) => u32(b, at) * 2 ** 32 + u32(b, at + 4);

/** Ranged reads through a block cache: every byte is fetched once, in
 * BLOCK-aligned pieces, and a span read back comes out of the cache. */
class Blocks {
  private cache = new Map<number, Promise<Uint8Array>>();
  constructor(
    private read: ReadRange,
    /** The file's length, Infinity when unknown: the end is then wherever a
     * read comes back short. */
    readonly size: number
  ) {}

  /** Fetch `[offset, offset + length)` ahead in one read, into the cache. */
  prefetch(offset: number, length: number): void {
    let first = Math.floor(offset / BLOCK);
    const last = Math.floor((offset + Math.max(1, length) - 1) / BLOCK);
    while (first <= last && this.cache.has(first)) first++;
    if (first > last) return;
    const from = first * BLOCK;
    const whole = this.read(from, (last - first + 1) * BLOCK);
    for (let i = first; i <= last; i++) {
      if (this.cache.has(i)) continue;
      const at = (i - first) * BLOCK;
      const piece = whole.then((b) => b.subarray(Math.min(at, b.length), Math.min(at + BLOCK, b.length)));
      // A prefetched block the walk never reaches must not surface a failed
      // read as unhandled; one it does reach rejects there.
      piece.catch(() => {});
      this.cache.set(i, piece);
    }
  }

  private block(i: number): Promise<Uint8Array> {
    let p = this.cache.get(i);
    if (!p) {
      p = this.read(i * BLOCK, BLOCK);
      this.cache.set(i, p);
    }
    return p;
  }

  /** The bytes of `[offset, offset + length)` that the file holds; fewer at
   * its end. */
  async bytes(offset: number, length: number): Promise<Uint8Array> {
    const end = Math.min(offset + length, this.size);
    if (end <= offset) return new Uint8Array(0);
    const first = Math.floor(offset / BLOCK);
    const last = Math.floor((end - 1) / BLOCK);
    if (first === last) {
      const b = await this.block(first);
      const from = offset - first * BLOCK;
      return b.subarray(Math.min(from, b.length), Math.min(end - first * BLOCK, b.length));
    }
    const out = new Uint8Array(end - offset);
    let filled = 0;
    for (let i = first; i <= last; i++) {
      const b = await this.block(i);
      const from = i === first ? offset - first * BLOCK : 0;
      const to = Math.min(b.length, end - i * BLOCK);
      if (to <= from) break;
      out.set(b.subarray(from, to), filled);
      filled += to - from;
      if (b.length < BLOCK) break;
    }
    return out.subarray(0, filled);
  }
}

interface Atom {
  type: string;
  /** File offset of the payload. */
  start: number;
  /** File offset one past the payload. */
  end: number;
}

/** The atoms of `[from, to)`, read header by header. An atom whose size lies
 * (too small, or running past the span) ends the walk: what follows is not
 * atoms. */
async function* atoms(blocks: Blocks, from: number, to: number, budget: { left: number }): AsyncGenerator<Atom> {
  let at = from;
  while (at + 8 <= to && budget.left-- > 0) {
    const head = await blocks.bytes(at, 16);
    if (head.length < 8) return;
    let size = u32(head, 0);
    let headLen = 8;
    if (size === 1) {
      if (head.length < 16) return;
      size = u64(head, 8);
      headLen = 16;
    } else if (size === 0) {
      size = to - at;
    }
    if (size < headLen || at + size > to) return;
    yield { type: ascii(head, 4), start: at + headLen, end: at + size };
    at += size;
  }
}

async function childAtom(blocks: Blocks, parent: Atom, type: string, budget: { left: number }): Promise<Atom | null> {
  for await (const c of atoms(blocks, parent.start, parent.end, budget)) if (c.type === type) return c;
  return null;
}

/** Whether the byte string `needle` occurs in `hay`. The search is over a
 * metadata atom's bytes — a technical container field. */
function contains(hay: Uint8Array, needle: string): boolean {
  const n = needle.length;
  const first = needle.charCodeAt(0);
  outer: for (let i = 0; i + n <= hay.length; i++) {
    if (hay[i] !== first) continue;
    for (let j = 1; j < n; j++) if (hay[i + j] !== needle.charCodeAt(j)) continue outer;
    return true;
  }
  return false;
}

const MATRIX: Record<number, CodeFormat["matrix"]> = {
  1: "bt709",
  5: "bt601",
  6: "bt601",
  9: "bt2020nc",
};
/** colr's code for BT.2020 primaries, and for "unspecified". */
const BT2020 = 9;
const UNSPECIFIED = 2;

/** ProRes and other codecs whose depth is the four-character code. */
const CODEC_DEPTH: Record<string, number> = {
  apco: 10,
  apcs: 10,
  apcn: 10,
  apch: 10,
  ap4h: 12,
  ap4x: 12,
};

interface Box {
  type: string;
  /** Offset of the payload inside the buffer. */
  start: number;
  /** Offset one past the payload. */
  end: number;
}

/** Child boxes of the span `[from, to)` of an in-memory buffer. */
function* boxes(b: Uint8Array, from: number, to: number): Generator<Box> {
  let at = from;
  while (at + 8 <= to) {
    let size = u32(b, at);
    let head = 8;
    if (size === 1) {
      if (at + 16 > to) return;
      size = u64(b, at + 8);
      head = 16;
    } else if (size === 0) {
      size = to - at;
    }
    if (size < head || at + size > to) return;
    yield { type: ascii(b, at + 4), start: at + head, end: at + size };
    at += size;
  }
}

function child(b: Uint8Array, box: Box, type: string): Box | null {
  for (const c of boxes(b, box.start, box.end)) if (c.type === type) return c;
  return null;
}

/** Bit depth from the codec configuration record inside a sample entry. */
function bitDepthOf(b: Uint8Array, entry: Box, format: string): number {
  const fixed = CODEC_DEPTH[format];
  if (fixed) return fixed;
  const hvcC = child(b, entry, "hvcC");
  // HEVCDecoderConfigurationRecord: bitDepthLumaMinus8 sits at byte 17.
  if (hvcC && hvcC.end - hvcC.start >= 18) return 8 + (b[hvcC.start + 17] & 0x07);
  const avcC = child(b, entry, "avcC");
  if (avcC) {
    // The depth fields follow the parameter sets, and only for the high
    // profiles (100, 110, 122, 144); the other profiles are 8-bit.
    const profile = b[avcC.start + 1];
    if (![100, 110, 122, 144].includes(profile)) return 8;
    let at = avcC.start + 5;
    const spsCount = b[at++] & 0x1f;
    for (let i = 0; i < spsCount && at + 2 <= avcC.end; i++) at += 2 + u16(b, at);
    if (at >= avcC.end) return 8;
    const ppsCount = b[at++];
    for (let i = 0; i < ppsCount && at + 2 <= avcC.end; i++) at += 2 + u16(b, at);
    if (at + 3 > avcC.end) return 8;
    return 8 + (b[at + 1] & 0x07);
  }
  const av1C = child(b, entry, "av1C");
  if (av1C && av1C.end - av1C.start >= 3) {
    const highBitdepth = (b[av1C.start + 2] >> 6) & 1;
    const twelveBit = (b[av1C.start + 2] >> 5) & 1;
    return highBitdepth ? (twelveBit ? 12 : 10) : 8;
  }
  const vpcC = child(b, entry, "vpcC");
  if (vpcC && vpcC.end - vpcC.start >= 6) return b[vpcC.start + 6] >> 4 || 8;
  return 8;
}

/** What the probe reads out of a movie header: the first video sample entry
 * (its payload, the fixed fields dropped) and the metadata bytes that can
 * name a log curve. */
interface MovieFacts {
  entry: { format: string; bytes: Uint8Array } | null;
  metadata: Uint8Array[];
}

/** Atoms whose payload can hold the log identifiers. */
const METADATA = new Set(["meta", "udta"]);

async function scan(blocks: Blocks, atom: Atom): Promise<Uint8Array> {
  return blocks.bytes(atom.start, Math.min(atom.end - atom.start, SCAN_MAX));
}

/** Walk `moov` for the facts, reading only the atoms that carry them. */
async function readMovie(blocks: Blocks, moov: Atom): Promise<MovieFacts> {
  const facts: MovieFacts = { entry: null, metadata: [] };
  const budget = { left: MAX_ATOMS };
  for await (const top of atoms(blocks, moov.start, moov.end, budget)) {
    if (METADATA.has(top.type)) {
      facts.metadata.push(await scan(blocks, top));
      continue;
    }
    if (top.type !== "trak" || facts.entry) continue;
    let isVideo = false;
    let stsd: Atom | null = null;
    const trakMeta: Atom[] = [];
    for await (const t of atoms(blocks, top.start, top.end, budget)) {
      if (METADATA.has(t.type)) trakMeta.push(t);
      if (t.type !== "mdia") continue;
      for await (const m of atoms(blocks, t.start, t.end, budget)) {
        if (m.type === "hdlr") {
          // Full box: version/flags (4), pre_defined (4), handler_type (4).
          const h = await blocks.bytes(m.start, 12);
          isVideo = h.length >= 12 && ascii(h, 8) === "vide";
          if (!isVideo) break;
        } else if (m.type === "minf" && isVideo) {
          const stbl = await childAtom(blocks, m, "stbl", budget);
          stsd = stbl ? await childAtom(blocks, stbl, "stsd", budget) : null;
        }
      }
    }
    if (!isVideo || !stsd) continue;
    // Full box header (4) + entry count (4), then the sample entries.
    const first = (await atoms(blocks, stsd.start + 8, stsd.end, budget).next()).value as Atom | undefined;
    // VisualSampleEntry: 78 bytes of fixed fields before the child boxes.
    if (first && first.end - first.start >= 78) {
      const bytes = await blocks.bytes(first.start + 78, Math.min(first.end - first.start - 78, SCAN_MAX));
      facts.entry = { format: first.type, bytes };
    }
    for (const a of trakMeta) facts.metadata.push(await scan(blocks, a));
  }
  return facts;
}

/** Settle a file's color from its header. `size` is the file's byte length,
 * when known; without it the walk ends where a read comes back short. */
export async function probeColor(readRange: ReadRange, size = Infinity): Promise<ColorProbe> {
  const blocks = new Blocks(readRange, size);
  const moov = await findMoov(blocks);
  if (!moov) return DEFAULT_COLOR;
  return colorOf(await readMovie(blocks, moov));
}

/** Walk the top-level atoms for `moov`. Null when the file is not an ISO
 * base media file or has no movie header in reach. Once the walk is within
 * reach of the end of a file of known length, the rest comes in one read:
 * that is where a camera writes the movie header. */
async function findMoov(blocks: Blocks): Promise<Atom | null> {
  let at = 0;
  for (let seen = 0; seen < 64 && at + 8 <= blocks.size; seen++) {
    if (blocks.size - at <= MOOV_PREFETCH) blocks.prefetch(at, blocks.size - at);
    const head = await blocks.bytes(at, 16);
    if (head.length < 8) return null;
    const type = ascii(head, 4);
    let size = u32(head, 0);
    let headLen = 8;
    if (size === 1) {
      if (head.length < 16) return null;
      size = u64(head, 8);
      headLen = 16;
    } else if (size === 0) {
      size = blocks.size - at;
    }
    if (at === 0 && !["ftyp", "moov", "wide", "free", "mdat", "skip"].includes(type)) return null;
    if (size < headLen) return null;
    if (type === "moov") {
      // A header cut short by the file's end ends there: the atoms that run
      // past it are not read.
      const atom = { type, start: at + headLen, end: Math.min(at + size, blocks.size) };
      if (atom.end - atom.start <= MOOV_PREFETCH) blocks.prefetch(atom.start, atom.end - atom.start);
      return atom;
    }
    at += size;
  }
  return null;
}

/** The profile a sample entry's `colr` and the movie's metadata settle on. */
function colorOf(facts: MovieFacts): ColorProbe {
  if (!facts.entry) return DEFAULT_COLOR;
  const { format, bytes } = facts.entry;
  const entry: Box = { type: format, start: 0, end: bytes.length };
  let matrix: CodeFormat["matrix"] = "bt709";
  let fullRange = false;
  let transfer = 1;
  let hdrMatrix = false;
  for (const c of boxes(bytes, entry.start, entry.end)) {
    if (c.type !== "colr" || c.end - c.start < 10) continue;
    const kind = ascii(bytes, c.start);
    if (kind !== "nclx" && kind !== "nclc") continue;
    const primaries = u16(bytes, c.start + 4);
    transfer = u16(bytes, c.start + 6);
    const code = u16(bytes, c.start + 8);
    // An unspecified matrix on BT.2020 primaries is BT.2020's own.
    hdrMatrix = code === BT2020 || (code === UNSPECIFIED && primaries === BT2020);
    matrix = hdrMatrix ? "bt2020nc" : (MATRIX[code] ?? "bt709");
    fullRange = kind === "nclx" && c.end - c.start >= 11 ? (bytes[c.start + 10] & 0x80) !== 0 : false;
    break;
  }
  const bitDepth = bitDepthOf(bytes, entry, format);
  const named = (id: string) => contains(bytes, id) || facts.metadata.some((m) => contains(m, id));
  let detected: SourceProfile;
  if (named(APPLE_LOG_2)) detected = "apple-log-2";
  else if (named(APPLE_LOG) || named(APPLE_CUSTOM_GAMMA)) detected = "apple-log";
  else if (transfer === 18 && hdrMatrix) detected = "hlg";
  else if (transfer === 16 && hdrMatrix) detected = "pq";
  else if (transfer === 13) detected = "srgb";
  else detected = "rec709";
  return { code: { matrix, fullRange, bitDepth }, detected, codec: format };
}

/** An asset's color record from a probe: the code format and the profile
 * the header settles on. */
export type ProbedColor = CodeFormat & { detected: SourceProfile; codec: string };

export const colorRecord = (probe: ColorProbe): ProbedColor => ({
  ...probe.code,
  detected: probe.detected,
  codec: probe.codec,
});

/** The video assets that carry no color record yet. */
export const lacksColor = <A extends { type: string; color?: unknown; block?: unknown }>(a: A): boolean =>
  a.type === "video" && !a.color && !a.block;

/**
 * Color records for every video asset the timeline draws that has none, one
 * header walk per file, keyed by file name. `used` names the assets the
 * frame reads (`pictureAssetIds`): a file sitting unused in the Media panel is
 * never walked, and a used file whose header cannot be read fails the call,
 * naming it. Each process hands in its own way to reach the bytes — a URL in
 * the tab and the runner, the file on disk in the engine — and every caller
 * fills its assets from the one answer.
 */
export async function probeMissingColors<A extends { id: string; type: string; fileName: string; color?: unknown; block?: unknown }>(
  assets: readonly A[],
  used: ReadonlySet<string>,
  probe: (asset: A) => Promise<ColorProbe>
): Promise<Map<string, ProbedColor>> {
  const found = new Map<string, ProbedColor>();
  const walks = new Map<string, A>();
  for (const a of assets) if (used.has(a.id) && lacksColor(a) && !walks.has(a.fileName)) walks.set(a.fileName, a);
  await Promise.all(
    [...walks].map(async ([fileName, asset]) => {
      try {
        found.set(fileName, colorRecord(await probe(asset)));
      } catch (e) {
        throw new Error(`The color of ${fileName} could not be read: ${e instanceof Error ? e.message : String(e)}`);
      }
    })
  );
  return found;
}

/** `assets` with the colors `probeMissingColors` found filled in. */
export const withFoundColors = <A extends { type: string; fileName: string; color?: unknown; block?: unknown }>(
  assets: A[],
  found: ReadonlyMap<string, ProbedColor>
): A[] =>
  found.size === 0
    ? assets
    : assets.map((a) => (lacksColor(a) && found.has(a.fileName) ? { ...a, color: found.get(a.fileName) } : a));

/** `readRange` over bytes already in hand. */
export const rangeOfBytes =
  (bytes: Uint8Array): ReadRange =>
  async (offset, length) =>
    bytes.subarray(offset, Math.min(bytes.length, offset + length));

/** `readRange` over a Blob or File. */
export const rangeOfBlob =
  (blob: Blob): ReadRange =>
  async (offset, length) =>
    new Uint8Array(await blob.slice(offset, Math.min(blob.size, offset + length)).arrayBuffer());