Skip to content

Open source CapCut alternative.

TypeScript
6.7 KB
Raw Blame
/**
 * Where a clip's sound comes from.
 *
 * A camera records the picture and a scratch track beside it; a recorder on
 * the speaker records the sound worth keeping. Binding the recording to the
 * video asset (`soundFrom`) puts every clip of that video on the recording:
 * the clip keeps its trim, rate and direction, and each source second of the
 * video is read off the recording `offset` seconds later. The binding sits on
 * the asset, so a split, a trim, a retime, a reverse
 * or a copy cannot leave the sound behind. Every reader of a clip's sound —
 * the preview, the fold the tab exports and transcribes through, the spec the
 * engine and the worker render, the tools that listen — resolves it here.
 */

import { retimeOf, speedCurveOf, srcSpan, type Retimable, type SpeedNode } from "@donkeycut/effects-kit";
import { assetIsSilent, type ClipSpan, type MediaAsset, type StoredAsset } from "./types";

type Sourced = Pick<StoredAsset, "id" | "type" | "duration" | "soundFrom">;

/** The longest a split edit carries a clip's sound past its picture, either
 * side, in seconds. */
export const SPLIT_EDIT_MAX_S = 5;

/** A file's own track: read from its own clock, as far as it goes. */
export const OWN_SOUND = { offset: 0, limit: Infinity } as const;

/** The sound a video plays: its own file's track, or the recording bound to
 * it, with the seconds added to its source time to land on that file. */
export interface SoundSource<A> {
  asset: A;
  offset: number;
  /** The last second the sound file holds, for a bound recording; the
   * asset's own track is read as far as it goes. */
  limit: number;
  bound: boolean;
}

/** `asset`'s sound. A binding to a recording that is gone, silent, or the
 * video itself plays the camera's own track. One hop: a recording's own
 * binding is never followed. */
export function soundSourceOf<A extends Sourced>(
  asset: A,
  assets: readonly A[] | ReadonlyMap<string, A>
): SoundSource<A> {
  const from = asset.soundFrom;
  if (from && from.assetId !== asset.id && Number.isFinite(from.offset)) {
    const rec = isMap(assets) ? assets.get(from.assetId) : assets.find((a) => a.id === from.assetId);
    if (rec && !assetIsSilent(rec))
      return { asset: rec, offset: from.offset, limit: rec.duration > 0 ? rec.duration : Infinity, bound: true };
  }
  return { asset, ...OWN_SOUND, bound: false };
}

const isMap = <A>(v: readonly A[] | ReadonlyMap<string, A>): v is ReadonlyMap<string, A> => v instanceof Map;

/** Source seconds of the sound file on either side of a clip's trim: what a
 * handle, a split edit or a crossing can reach into before its head and past
 * its tail. A reversed clip's head faces the source's end. */
export function soundRoom(
  c: { in: number; out: number; reverse?: boolean },
  src: { asset: { duration: number }; offset: number }
): { head: number; tail: number } {
  const lo = c.in + src.offset;
  const hi = c.out + src.offset;
  const end = src.asset.duration;
  return c.reverse ? { head: end - hi, tail: lo } : { head: lo, tail: end - hi };
}

/** A span moved onto its sound's clock: the same map from the timeline,
 * `offset` seconds later in the source. Speed curve nodes sit in source
 * seconds, so they move with it. */
export function shiftSpan<T extends Retimable>(c: T, offset: number): T {
  if (!offset) return c;
  const nodes = speedCurveOf(c);
  return {
    ...c,
    in: c.in + offset,
    out: c.out + offset,
    ...(nodes ? { speedCurve: nodes.map(([at, rate]): SpeedNode => [at + offset, rate]) } : {}),
  };
}

/** What one stretch of a span's sound reads from its file. */
export interface SoundWindow {
  /** Seconds on the sound file, low end first. */
  lo: number;
  hi: number;
  /** Timeline seconds into the window the sound begins: a recording that
   * starts after the window does leaves silence ahead of it. */
  at: number;
  /** Timeline seconds the sound runs. */
  len: number;
  /** The stretch as a span of its own on the sound file — the map a renderer
   * lays it through. */
  span: Retimable;
}

/**
 * The sound of span `c` over the window `[fromT, toT]`, timeline seconds from
 * its head (negative reaches before it, past its length beyond it), read from
 * the source `src` describes. The window narrows to what the file holds: a
 * recording rolled after the camera did, or stopped before it, leaves the
 * rest of the window silent. Null when nothing of the window is on the file.
 */
export function soundWindow(
  c: Retimable,
  src: { offset: number; limit: number },
  fromT: number,
  toT: number
): SoundWindow | null {
  const moved = shiftSpan(c, src.offset);
  const rt = retimeOf(moved);
  const want = srcSpan(rt, fromT, toT);
  const lo = Math.max(0, want.lo);
  const hi = Math.min(src.limit, want.hi);
  if (!(hi - lo > 1e-4)) return null;
  const t0 = rt.tAt(rt.reverse ? hi : lo);
  const t1 = rt.tAt(rt.reverse ? lo : hi);
  return {
    lo,
    hi,
    at: Math.max(0, t0 - fromT),
    len: Math.max(0, t1 - t0),
    span: { in: lo, out: hi, speed: moved.speed, speedCurve: moved.speedCurve, reverse: moved.reverse },
  };
}

/** A bound recording as a render spec carries it: the file the sound reads,
 * its clock's offset from the video's, and its length. */
export interface SpecSound {
  file: string;
  offset: number;
  duration: number;
}

/** The spec entry for `asset`'s bound recording, named the way the spec
 * names files; absent when the video plays its own track. */
export function specSound<A extends Sourced>(
  asset: A,
  assets: readonly A[] | ReadonlyMap<string, A>,
  fileOf: (a: A) => string
): SpecSound | undefined {
  const src = soundSourceOf(asset, assets);
  if (!src.bound) return undefined;
  return { file: fileOf(src.asset), offset: src.offset, duration: src.asset.duration };
}

/** A clip's sound as a spec entry carries it: the recording bound to its
 * video, and a split edit's reach past its picture with the ramp at the far
 * end. `fileOf` names files the way the spec does. */
export function spanSoundSpec(sp: ClipSpan, fileOf: (a: MediaAsset) => string) {
  return {
    ...(sp.sound
      ? { soundFrom: { file: fileOf(sp.sound.asset), offset: sp.sound.offset, duration: sp.sound.asset.duration } }
      : {}),
    ...(sp.soundLead || sp.soundTail
      ? { soundLead: sp.soundLead ?? 0, soundTail: sp.soundTail ?? 0, splitFade: sp.splitFade ?? 0 }
      : {}),
  };
}

/** Whether a span plays sound off a bound recording or past its picture —
 * what a renderer without either would get wrong. A clip that plays nothing
 * (hidden, muted, a still, a silent file with no recording) never does. */
export function spanUsesSoundFeature(sp: ClipSpan): boolean {
  const sounds = !sp.clip.hidden && !sp.clip.muted && (!!sp.sound || !assetIsSilent(sp.asset));
  return sounds && (!!sp.sound || !!sp.soundLead || !!sp.soundTail);
}