Skip to content

Open source CapCut alternative.

TypeScript
22.2 KB
Raw Blame
"use client";

/**
 * The cut's audible mix, rendered in the browser.
 *
 * One `OfflineAudioContext` fold that mirrors the engine's ffmpeg audio graph:
 * track-0 clip audio folds sequentially with cross-dissolves overlapping,
 * upper-track clips and soundtrack items land at their absolute starts with
 * their own volumes and fades, a ducking voiceover pulls everything else down
 * while it speaks, and the whole-project fade rides over the sum.
 *
 * Two callers want this and want it at different shapes. Transcription needs a
 * 16 kHz mono buffer, because that is what a speech model reads; an export
 * needs full-rate stereo, because that is what goes in the file. Same fold,
 * different rate and channel count — which is why those are parameters and the
 * fold is written once. A mix that drifted between them would put captions on
 * words that the exported audio says at another time.
 */

import {
  AUDIO_FX_RAMP,
  audioFxRecipe,
  buildAudioFx,
  retimeOf,
  soundRecipe,
  type ClipSound,
  type SpeedNode,
} from "@donkeycut/effects-kit";
import { decodeAudioSpan } from "./mediaRead";
import { CLIP_MIN_SECONDS } from "./types";
import { fitSpan, retimeFits } from "./retimeFit";
import { OWN_SOUND, shiftSpan, soundWindow, type SoundWindow, type SpecSound } from "./soundSource";

/** One track-0 clip's audio in the sequential fold. A spacer (no file) only
 * shapes time. */
export interface MixClip {
  file: string;
  in: number;
  out: number;
  muted: boolean;
  speed?: number;
  speedCurve?: SpeedNode[];
  reverse?: boolean;
  /** Gain on the clip's own audio; absent = 1. */
  volume?: number;
  /** The clip's own treatment, run on its sound before the fades. */
  sound?: ClipSound;
  /** Head and tail ramps of the clip's own fade animation, timeline seconds:
   * the sound goes down to silence with the picture, the gain the preview
   * plays. An edge a transition owns carries none. */
  fadeIn?: number;
  fadeOut?: number;
  /** Cross-dissolve overlap into the next clip, timeline seconds. */
  transition?: number;
  /** Half the cross dissolve into the next clip, timeline seconds: the two
   * clips ramp equal-power past each other over that long either side of the
   * cut. Set instead of `transition` when the handover is on the sound. */
  soundCross?: number;
  /** Source seconds past `out` this clip keeps sounding, into the crossing at
   * its tail, and source seconds before `in` it starts sounding, for the
   * crossing at its head. Both are handle — media the trim left behind — and
   * are what make the two clips audible at once over a hard picture cut. */
  soundAhead?: number;
  soundBack?: number;
  /** A split edit: seconds of `soundBack` the clip's sound plays ahead of its
   * picture and of `soundAhead` past its end, each ramping from silence over
   * `splitFade` at its outer end. */
  soundLead?: number;
  soundTail?: number;
  splitFade?: number;
  /** The video's sound recorded apart from it: the sound reads this file,
   * `offset` seconds later in the source (see soundSource.ts). */
  soundFrom?: SpecSound;
}

/** A clip placed at an absolute time: an upper-track video's audio, a
 * soundtrack bed, or a voiceover. */
export interface MixItem {
  file: string;
  in: number;
  out: number;
  start: number;
  volume: number;
  speed?: number;
  speedCurve?: SpeedNode[];
  reverse?: boolean;
  /** Half the cross dissolve at this item's head and tail, with the handle
   * each side reaches into — an upper-track cut crosses the same way track
   * 0's does. */
  crossIn?: number;
  crossOut?: number;
  soundAhead?: number;
  soundBack?: number;
  muted?: boolean;
  fadeIn?: number;
  fadeOut?: number;
  sound?: ClipSound;
  /** While this item is audible, everything else drops to this gain. */
  duck?: number;
  soundLead?: number;
  soundTail?: number;
  splitFade?: number;
  soundFrom?: SpecSound;
  /** The soundtrack lane this item sits on; absent for a video clip's own
   * sound. What a stem render picks items by. */
  lane?: number;
}

/** One audio effect element: the treatment it names, and the stretch of
 * timeline it treats. */
export interface MixEffect {
  effect: string;
  amount?: number;
  start: number;
  end: number;
}

export interface MixSpec {
  duration: number;
  clips: MixClip[];
  /** Soundtrack, voiceovers, and upper-track clip audio. */
  items: MixItem[];
  /** Audio effect elements over the finished mix, in timeline order. */
  effects?: MixEffect[];
  /** The duck envelope, when it comes from items outside this spec: a stem
   * holds some of the mix's items and still ducks under all of its
   * voiceovers. Absent = read from `items`. */
  ducks?: { start: number; end: number; gain: number }[];
}

export interface MixOptions {
  sampleRate: number;
  channels: number;
  /** Resolves a spec file name to something the reader can open. */
  resolve: (file: string) => string;
}

/** Timeline length of one clip: its source span through its rate. A gap
 * spacer keeps its exact length — flooring it would land everything after it
 * late. */
const clipDur = (c: MixClip) =>
  c.file ? Math.max(CLIP_MIN_SECONDS, retimeOf(c).len) : Math.max(0, c.out - c.in);

/** Timeline footprint of a placed item. */
const itemDur = (a: MixItem) => Math.max(0, retimeOf(a).len);

/**
 * Where the sequential fold puts each track-0 clip, and how long its
 * transition edges are. Clips abut — a transition claims no layout — and its
 * sound is the outgoing clip fading across the blend window while the next
 * enters clean at the cut, exactly like the engine's tail-fade + hard join.
 *
 * A cross dissolve is the other shape: the two clips ramp past each other
 * across the cut, each reaching into its handle so both are really sounding
 * while the picture hard-joins. Those ramps are equal-power and are kept apart
 * from the fades, which are linear and end at silence.
 */
export function foldClips(clips: MixClip[]) {
  const geo = clips.map((clip) => ({
    clip,
    at: 0,
    dur: clipDur(clip),
    fadeIn: 0,
    fadeOut: 0,
    crossIn: 0,
    crossOut: 0,
  }));
  let acc = 0;
  geo.forEach((g, j) => {
    g.at = acc;
    acc += g.dur;
    // The clip's own fade animation ramps its sound; the tail's room is what
    // the head left, the way the engine's stanza splits them.
    const own = Math.min(Math.max(0, g.clip.fadeIn ?? 0), g.dur);
    if (own > 0.01) g.fadeIn = own;
    const ownOut = Math.min(Math.max(0, g.clip.fadeOut ?? 0), Math.max(0, g.dur - g.fadeIn));
    if (ownOut > 0.01) g.fadeOut = ownOut;
    const next = geo[j + 1];
    if (!next) return;
    const fade = Math.min(g.clip.transition ?? 0, g.dur * 0.9);
    if (fade > 0.01) g.fadeOut = Math.max(g.fadeOut, fade);
    const cross = Math.min(g.clip.soundCross ?? 0, g.dur * 0.9, next.dur * 0.9);
    if (cross > 0.01) {
      g.crossOut = cross;
      next.crossIn = cross;
    }
  });
  return geo;
}

/** How far each side of a clip reaches into its handle for a crossing, capped
 * by the ramp that crossing actually got, or for a split edit, which an edge
 * with a crossing never carries. */
const handles = (g: { clip: MixClip; crossIn: number; crossOut: number }) => {
  const rt = retimeOf(g.clip);
  return {
    // Never past the head of the source: a handle is what the trim left
    // behind. A reversed clip's head reaches toward the source's end, and
    // `soundBack` already stops at it; a bound recording's reach was measured
    // on the recording.
    back: Math.min(
      g.clip.soundBack ?? 0,
      Math.max(g.crossIn, g.clip.soundLead ?? 0),
      rt.reverse || g.clip.soundFrom ? Infinity : Math.max(0, -rt.tAt(0))
    ),
    ahead: Math.min(g.clip.soundAhead ?? 0, Math.max(g.crossOut, g.clip.soundTail ?? 0)),
  };
};

type Sounded = {
  file: string;
  in: number;
  out: number;
  speed?: number;
  speedCurve?: SpeedNode[];
  reverse?: boolean;
  soundFrom?: SpecSound;
};

/** What an entry decodes: its own trim widened by the reach at its edges,
 * on the file its sound reads — a bound recording through its offset, the
 * window narrowed to what that recording holds. */
const soundRead = (c: Sounded, back = 0, ahead = 0): (SoundWindow & { file: string }) | null => {
  const len = retimeOf(c).len;
  const from = c.soundFrom;
  const w = soundWindow(c, from ? { offset: from.offset, limit: from.duration } : OWN_SOUND, -back, len + ahead);
  return w && { ...w, file: from ? from.file : c.file };
};

/** The ramps a split edit puts on an entry's gain, in timeline seconds: up
 * from silence at the head of the sound that leads the picture, down to
 * silence at the end of the sound that trails it. Inside the picture the
 * clip's own fades and crossings shape the level, and they never reach out
 * here — so the trailing sound comes back up to `level` at the picture's end.
 * Scheduled after those fades: two events at the picture's end keep the order
 * they were set in, and the last one is the level the tail plays at. */
function scheduleSplit(
  param: AudioParam,
  level: number,
  head: number,
  end: number,
  lead: number,
  tail: number,
  fade: number
) {
  if (lead > 0) {
    const f = Math.min(fade, lead);
    const from = Math.max(0, head - lead);
    param.setValueAtTime(f > 0 ? 0 : level, from);
    if (f > 0) param.linearRampToValueAtTime(level, from + f);
  }
  if (tail > 0) {
    const f = Math.min(fade, tail);
    param.setValueAtTime(level, end);
    if (f > 0) {
      param.setValueAtTime(level, end + tail - f);
      param.linearRampToValueAtTime(0, end + tail);
    }
  }
}

/**
 * The windows during which a ducking item is audible, merged into a single
 * envelope: each entry is a stretch of timeline where everything that is not
 * itself ducking drops to `gain`.
 *
 * Overlapping voiceovers take the lowest gain of the ones speaking, which is
 * what the preview does frame by frame — computed here once for the whole
 * timeline instead, because a scheduled ramp cannot ask "what is live now?".
 */
export function duckWindows(items: MixItem[]): { start: number; end: number; gain: number }[] {
  const edges = items
    .filter((a) => a.duck !== undefined && a.duck < 1)
    .map((a) => ({ start: a.start, end: a.start + itemDur(a), gain: Math.max(0, a.duck!) }))
    .filter((w) => w.end > w.start)
    .sort((a, b) => a.start - b.start);
  const merged: { start: number; end: number; gain: number }[] = [];
  for (const w of edges) {
    const last = merged[merged.length - 1];
    if (last && w.start <= last.end) {
      last.end = Math.max(last.end, w.end);
      last.gain = Math.min(last.gain, w.gain);
    } else {
      merged.push({ ...w });
    }
  }
  return merged;
}

/**
 * Schedule one side of a cross dissolve: an equal-power ramp between silence
 * and `level` across `[from, to]`, rising for the incoming clip and falling
 * for the outgoing one. The two meet at 0.707 in the middle of the window —
 * the cut — which is what keeps the mix at one loudness through the join.
 *
 * Written as short linear steps rather than `setValueCurveAtTime`, which is
 * not carried by every context this fold runs in (the headless renderer's
 * among them).
 */
const CROSS_STEPS = 24;

export function scheduleCross(
  param: AudioParam,
  level: number,
  from: number,
  to: number,
  rising: boolean
) {
  for (let i = 0; i <= CROSS_STEPS; i++) {
    const p = i / CROSS_STEPS;
    const v = level * (rising ? Math.sin((p * Math.PI) / 2) : Math.cos((p * Math.PI) / 2));
    const at = Math.max(0, from + (to - from) * p);
    if (i === 0) param.setValueAtTime(v, at);
    else param.linearRampToValueAtTime(v, at);
  }
}

/** Schedule a duck envelope onto a gain node: 1 everywhere except inside the
 * windows, where it steps down and back. The steps are ramps a few
 * milliseconds long so the drop doesn't click. */
const DUCK_RAMP = 0.08;

function scheduleDuck(gain: GainNode, windows: { start: number; end: number; gain: number }[]) {
  gain.gain.value = 1;
  for (const w of windows) {
    gain.gain.setValueAtTime(1, Math.max(0, w.start - DUCK_RAMP));
    gain.gain.linearRampToValueAtTime(w.gain, w.start);
    gain.gain.setValueAtTime(w.gain, Math.max(w.start, w.end - DUCK_RAMP));
    gain.gain.linearRampToValueAtTime(1, w.end);
  }
}

/**
 * Render `spec` to a buffer, or null when nothing in it makes a sound.
 *
 * A clip that plays at a speed other than 1 is time-stretched to fit, so it
 * keeps its pitch — the same thing ffmpeg's atempo does for the engine and the
 * browser's `preservesPitch` does for the preview.
 */
export async function renderMix(spec: MixSpec, opts: MixOptions): Promise<AudioBuffer | null> {
  const { sampleRate, channels, resolve } = opts;
  const length = Math.max(1, Math.ceil(spec.duration * sampleRate));
  const ctx = new OfflineAudioContext(channels, length, sampleRate);

  // Only the spans that are actually heard are decoded, and only for entries
  // that can be heard at all. Decoding whole sources instead would mean a
  // ten-second trim of a ninety-minute recording pulling the entire recording
  // into memory — around 1.4 GB an hour at 48 kHz stereo — and several of them
  // at once.
  // The track-0 fold, with the source span each clip really decodes: a
  // crossing at either edge widens it into the handle it reaches for.
  const folded = foldClips(spec.clips).map((g) => {
    const { back, ahead } = handles(g);
    const read = g.clip.file && !g.clip.muted ? soundRead(g.clip, back, ahead) : null;
    return { ...g, back, ahead, read };
  });
  // Each placed item's reach: a crossing's handle, or the split edit an edge
  // without one carries.
  const placed = spec.items.map((a) => {
    const back = Math.min(a.soundBack ?? 0, Math.max(a.crossIn ?? 0, a.soundLead ?? 0));
    const ahead = Math.min(a.soundAhead ?? 0, Math.max(a.crossOut ?? 0, a.soundTail ?? 0));
    const read = a.file && !a.muted ? soundRead(a, back, ahead) : null;
    return { a, back, ahead, read };
  });

  const wanted = new Map<string, { file: string; from: number; to: number }>();
  const spanKey = (c: { file: string; lo: number; hi: number }) =>
    `${c.file}|${c.lo.toFixed(3)}|${c.hi.toFixed(3)}`;
  for (const { read } of [...folded, ...placed]) {
    if (read) wanted.set(spanKey(read), { file: read.file, from: read.lo, to: read.hi });
  }

  const buffers = new Map<string, AudioBuffer>();
  await Promise.all(
    [...wanted].map(async ([key, span]) => {
      // A source with no audio track at all — a silent recording, a still —
      // reads back null and mixes as silence, like the engine's hasStream
      // check. A source that could not be *read* is a different thing and is
      // allowed to fail the render: silently dropping it would hand the user a
      // finished file with a clip, or a whole voiceover, missing its sound.
      const buf = await decodeAudioSpan(resolve(span.file), span.from, span.to);
      if (buf) buffers.set(key, buf);
    })
  );

  if (buffers.size === 0) return null;

  // Everything that can be ducked passes through one node; the voiceovers doing
  // the ducking connect past it, so they never duck themselves.
  const ducked = ctx.createGain();
  scheduleDuck(ducked, spec.ducks ?? duckWindows(spec.items));

  // Every sound meets on the bus, which is what the effect elements treat —
  // the ffmpeg graph applies them to the mixed stream in the same place.
  const bus = ctx.createGain();
  ducked.connect(bus);

  // The effects run in series off the bus, each crossed in over its own
  // window: inside it the treated signal is heard, outside it the untouched
  // one, with the edges ramping so the switch cannot click.
  let treated: AudioNode = bus;
  for (const fx of spec.effects ?? []) {
    const recipe = audioFxRecipe(fx.effect, fx.amount);
    const from = Math.max(0, fx.start);
    const to = Math.min(fx.end, spec.duration);
    if (!recipe || !(to > from)) continue;
    const nodes = buildAudioFx(ctx, recipe, from);
    const wet = ctx.createGain();
    const dry = ctx.createGain();
    const out = ctx.createGain();
    const ramp = Math.min(AUDIO_FX_RAMP, Math.max(0.001, (to - from) / 2));
    const cross = (gain: GainNode, inside: number) => {
      const outside = 1 - inside;
      gain.gain.setValueAtTime(outside, 0);
      gain.gain.setValueAtTime(outside, Math.max(0, from - ramp));
      gain.gain.linearRampToValueAtTime(inside, from + ramp);
      gain.gain.setValueAtTime(inside, Math.max(from + ramp, to - ramp));
      gain.gain.linearRampToValueAtTime(outside, to + ramp);
    };
    cross(wet, 1);
    cross(dry, 0);
    treated.connect(nodes.input);
    nodes.output.connect(wet);
    wet.connect(out);
    treated.connect(dry);
    dry.connect(out);
    treated = out;
  }
  treated.connect(ctx.destination);

  /**
   * Fit a span to the time its clip occupies, keeping its pitch.
   *
   * `playbackRate` would do the fitting for free, and resample while it did it —
   * a clip at 1.5× would come back seven semitones high, which neither the
   * preview nor the engine's `atempo` does. So the buffer is stretched to the
   * length it needs first, and then played at its own rate.
   */
  const stretched = new Map<string, { buffer: AudioBuffer; shift: number }>();
  const fitted = (
    key: string,
    buf: AudioBuffer,
    entry: { in: number; out: number; speed?: number; speedCurve?: SpeedNode[]; reverse?: boolean },
    from: number,
    to: number
  ): { buffer: AudioBuffer; shift: number } => {
    const rt = retimeOf(entry);
    if (!retimeFits(rt)) return { buffer: buf, shift: 0 };
    // A span used by more than one clip at the same rate is stretched once.
    const memo = `${key}@${rt.key}`;
    const hit = stretched.get(memo);
    if (hit) return hit;
    const channels = Array.from({ length: buf.numberOfChannels }, (_, c) =>
      buf.getChannelData(c)
    );
    // `from` is the source second the buffer starts at; the fit follows the
    // clip's own map from there and lands exactly on its length.
    const fit = fitSpan(channels, buf.sampleRate, rt, from, to);
    const out = new AudioBuffer({
      length: Math.max(1, fit.channels[0].length),
      numberOfChannels: buf.numberOfChannels,
      sampleRate: buf.sampleRate,
    });
    fit.channels.forEach((data, c) => out.getChannelData(c).set(data));
    const fitted = { buffer: out, shift: fit.shift };
    stretched.set(memo, fitted);
    return fitted;
  };

  // Each buffer holds exactly the span its entry plays, already fitted to the
  // time that entry occupies, so it starts at its own zero and runs at 1.
  // The entry's own treatment runs on its sound alone, between the source and
  // the gain that carries its level, fades and crossings — the order the
  // preview's voice and the ffmpeg stanza both keep.
  const play = (
    buf: AudioBuffer,
    at: number,
    duration: number,
    into: AudioNode,
    sound: ClipSound | undefined,
    shape: (gain: GainNode) => void
  ) => {
    const src = ctx.createBufferSource();
    src.buffer = buf;
    const gain = ctx.createGain();
    shape(gain);
    const recipe = soundRecipe(sound);
    if (recipe) {
      const fx = buildAudioFx(ctx, recipe, Math.max(0, at));
      src.connect(fx.input);
      fx.output.connect(gain);
    } else {
      src.connect(gain);
    }
    gain.connect(into);
    // A sound reaching before the timeline's start joins part-way in.
    const skip = Math.max(0, -at);
    src.start(Math.max(0, at), skip, Math.max(0, duration - skip));
  };

  for (const g of folded) {
    const c = g.clip;
    const read = g.read;
    const key = read ? spanKey(read) : "";
    const buf = read ? buffers.get(key) : undefined;
    if (!read || !buf) continue; // muted, silent, or a gap spacer: only shapes time
    // Laid through the clip's own map, moved onto the file the sound reads.
    const fit = fitted(key, buf, shiftSpan(c, c.soundFrom?.offset ?? 0), read.lo, read.hi);
    const from = g.at - g.back + read.at + fit.shift;
    const end = g.at + g.dur;
    play(fit.buffer, from, read.len, ducked, c.sound, (gain) => {
      const level = Math.max(0, c.volume ?? 1);
      gain.gain.value = level;
      // A crossing and a fade never share an edge — the cut carries one
      // handover — so each side takes whichever it has.
      if (g.crossIn > 0) {
        scheduleCross(gain.gain, level, g.at - g.crossIn, g.at + g.crossIn, true);
      } else if (g.fadeIn > 0) {
        gain.gain.setValueAtTime(0, g.at);
        gain.gain.linearRampToValueAtTime(level, g.at + g.fadeIn);
      }
      if (g.crossOut > 0) {
        scheduleCross(gain.gain, level, end - g.crossOut, end + g.crossOut, false);
      } else if (g.fadeOut > 0) {
        gain.gain.setValueAtTime(level, end - g.fadeOut);
        gain.gain.linearRampToValueAtTime(0, end);
      }
      scheduleSplit(
        gain.gain,
        level,
        g.at,
        end,
        g.crossIn > 0 ? 0 : Math.min(c.soundLead ?? 0, g.back),
        g.crossOut > 0 ? 0 : Math.min(c.soundTail ?? 0, g.ahead),
        c.splitFade ?? 0
      );
    });
  }

  for (const { a, back, ahead, read } of placed) {
    const key = read ? spanKey(read) : "";
    const buf = read ? buffers.get(key) : undefined;
    if (!read || !buf) continue;
    const dur = itemDur(a);
    const end = a.start + dur;
    // A voiceover that ducks everything else is not itself ducked; it still
    // lands on the bus, so the effect elements treat it with the rest.
    const into = a.duck !== undefined && a.duck < 1 ? bus : ducked;
    const fit = fitted(key, buf, shiftSpan(a, a.soundFrom?.offset ?? 0), read.lo, read.hi);
    play(fit.buffer, a.start - back + read.at + fit.shift, read.len, into, a.sound, (gain) => {
      const level = Math.max(0, a.volume);
      gain.gain.value = level;
      const fIn = Math.min(a.fadeIn ?? 0, dur);
      const fOut = Math.min(a.fadeOut ?? 0, dur);
      if (a.crossIn) {
        scheduleCross(gain.gain, level, a.start - a.crossIn, a.start + a.crossIn, true);
      } else if (fIn > 0) {
        gain.gain.setValueAtTime(0, a.start);
        gain.gain.linearRampToValueAtTime(level, a.start + fIn);
      }
      if (a.crossOut) {
        scheduleCross(gain.gain, level, end - a.crossOut, end + a.crossOut, false);
      } else if (fOut > 0) {
        gain.gain.setValueAtTime(level, end - fOut);
        gain.gain.linearRampToValueAtTime(0, end);
      }
      scheduleSplit(
        gain.gain,
        level,
        a.start,
        end,
        a.crossIn ? 0 : Math.min(a.soundLead ?? 0, back),
        a.crossOut ? 0 : Math.min(a.soundTail ?? 0, ahead),
        a.splitFade ?? 0
      );
    });
  }

  return ctx.startRendering();
}