Skip to content

Open source CapCut alternative.

TypeScript
10.6 KB
Raw Blame
"use client";

// Browser-side audio work for a project whose media the engine doesn't hold,
// and the hosted transcriber that goes with it. Rendering the mix is shared:
// an OfflineAudioContext applies the same trims, speeds, volumes, and
// crossfades the engine's ffmpeg graph does (see server/transcribe.ts), so the
// result is the cut's audible mix in timeline time — whether it then goes to
// this Mac (localStt.ts) or to the hosted route, which includes each
// account's first chunks and meters the rest. The hosted path
// chunks it as 16 kHz mono WAV, POSTs each chunk, and stitches the cues — each
// word carrying its own timing — back into timeline time. Mic dictation reuses
// the same chunk/post/stitch core on a MediaRecorder capture.

import { renderMix as mixAudio } from "./audioMix";
import { apiFetch } from "./backend";
import { cloudBackend } from "./backend/cloud";
import { mediaUrl, type MediaAsset, type SubtitleCue } from "./types";
import { replyError, UserError } from "./userError";
import { wavHeader } from "./wav";

/** Mirror of the engine's TranscribeSpec (server/transcribe.ts) minus projectId. */
export interface CloudTranscribeSpec {
  duration: number;
  locale?: string;
  clips: {
    file: string;
    in: number;
    out: number;
    muted: boolean;
    speed?: number;
    speedCurve?: [number, number][];
    reverse?: boolean;
    /** Cross-dissolve overlap into the next clip, timeline seconds. */
    transition?: number;
  }[];
  audio: { file: string; in: number; out: number; start: number; volume: number; speed?: number; speedCurve?: [number, number][]; reverse?: boolean }[];
}

const RATE = 16000; // the wire format the hosted route expects
// The hosted route's included allowance counts chunks and holds each to this
// length (its MAX_CHUNK_SECONDS leaves a little room), so the chunk is the
// unit of accounting: short enough that a few round trips overlap, long
// enough that a sentence keeps its context.
const CHUNK_SECONDS = 20;
const OVERLAP_SECONDS = 3; // audio shared across chunk seams for stitching
const POSTS_IN_FLIGHT = 4; // short chunks mean more round-trips; overlap them
// A chunk quieter than this end to end carries no speech; skip the round-trip
// (and its credit charge) — the engine path short-circuits silence the same way.
const SILENCE_PEAK = 1e-3;

const uid = () => crypto.randomUUID().slice(0, 8);
const round = (n: number) => Math.round(n * 1000) / 1000;

/** Render the cut's audible mix to a 16 kHz mono buffer — the wire format the
 * speech models read. The fold itself is the shared one (audioMix.ts), so the
 * audio a caption is timed against is the same audio an export writes, at a
 * different rate. Returns null when nothing audible exists. */
export function renderMix(
  projectId: string,
  spec: CloudTranscribeSpec,
  assets: readonly MediaAsset[]
): Promise<AudioBuffer | null> {
  // Each file is read where its asset plays from: a dropped file still
  // uploading has no stored copy yet, and plays from the bytes in hand.
  const urls = new Map(assets.map((a) => [a.fileName, a.url]));
  return mixAudio(
    {
      duration: spec.duration,
      clips: spec.clips,
      // Transcription hears the soundtrack the way the viewer will, minus the
      // shaping it does not change the timing of.
      items: spec.audio,
    },
    { sampleRate: RATE, channels: 1, resolve: (file) => urls.get(file) || mediaUrl(projectId, file) }
  );
}

/** 16-bit PCM WAV (RIFF) from mono 16 kHz float samples. */
export function encodeWav(samples: Float32Array): Blob {
  const data = new DataView(new ArrayBuffer(44 + samples.length * 2));
  new Uint8Array(data.buffer).set(wavHeader(samples.length, 1, RATE, 16));
  for (let i = 0; i < samples.length; i++) {
    const v = Math.max(-1, Math.min(1, samples[i]));
    data.setInt16(44 + i * 2, v < 0 ? v * 0x8000 : v * 0x7fff, true);
  }
  return new Blob([data.buffer], { type: "audio/wav" });
}

function isSilent(samples: Float32Array): boolean {
  for (let i = 0; i < samples.length; i++) {
    if (Math.abs(samples[i]) > SILENCE_PEAK) return false;
  }
  return true;
}

interface WireCue {
  start: number;
  end: number;
  text: string;
  /** Per-word timings, seconds from the start of the chunk. */
  words?: { t0: number; t1: number; w: string }[];
}

// The route's error code for an account whose included chunks are used up,
// which is also the failure's catalog code. A `freeOnly` caller reads it off
// the thrown error and stops transcribing.
const FREE_EXHAUSTED_CODE = "free_transcription_exhausted";

export const isFreeTranscriptionExhausted = (e: unknown): boolean =>
  e instanceof UserError && e.code === FREE_EXHAUSTED_CODE;

/** The transport a run posts its chunks on. Defaults to the active backend;
 * mic dictation names the hosted one, because the take it is falling back with
 * is the one the engine could not take. */
type ChunkTransport = (path: string, init?: RequestInit) => Promise<Response>;

async function postChunk(
  samples: Float32Array,
  offset: number,
  locale: string | undefined,
  freeOnly: boolean,
  request: ChunkTransport
): Promise<WireCue[]> {
  const form = new FormData();
  form.append("audio", new File([encodeWav(samples)], "chunk.wav", { type: "audio/wav" }));
  form.append("offset", String(round(offset)));
  if (locale) form.append("locale", locale);
  if (freeOnly) form.append("freeOnly", "1");
  const res = await request("/api/cut/transcribe", { method: "POST", body: form });
  const body = (await res.json().catch(() => null)) as
    | { cues?: WireCue[]; error?: string; message?: string }
    | null;
  if (!res.ok) {
    // The used-up allowance answers 402 with its own code under `error`, so it
    // is read before the credits wall would claim the status.
    if (body?.error === FREE_EXHAUSTED_CODE) {
      throw new UserError(FREE_EXHAUSTED_CODE, body.message ?? "The free transcription allowance is used up.");
    }
    throw replyError(res.status, body, new UserError("transcription_failed", "Transcription failed."));
  }
  return (body?.cues ?? []).filter(
    (c) =>
      typeof c?.start === "number" &&
      typeof c?.end === "number" &&
      typeof c?.text === "string" &&
      Number.isFinite(c.start) &&
      Number.isFinite(c.end)
  );
}

/** Chunk mono 16 kHz samples, transcribe each chunk, and stitch: cue times
 * re-base by chunk offset, and in each 3s overlap the midpoint of the seam
 * decides ownership — cues whose center falls in the earlier chunk's half are
 * dropped from the later one (and vice versa), so seams never duplicate.
 * Returns null when `isStale` trips mid-run. */
export async function transcribeSamples(
  samples: Float32Array,
  locale: string | undefined,
  isStale?: () => boolean,
  opts?: { freeOnly?: boolean; request?: ChunkTransport }
): Promise<SubtitleCue[] | null> {
  const duration = samples.length / RATE;
  const step = (CHUNK_SECONDS - OVERLAP_SECONDS) * RATE;
  const chunks: { slice: Float32Array; offset: number; last: boolean }[] = [];
  for (let s = 0; s < samples.length; s += step) {
    const slice = samples.subarray(s, Math.min(samples.length, s + CHUNK_SECONDS * RATE));
    const last = s + CHUNK_SECONDS * RATE >= samples.length;
    if (!isSilent(slice)) chunks.push({ slice, offset: s / RATE, last });
    if (last) break;
  }

  // A small pool posts the chunks; a failure parks its error and drains the
  // pool so no further metered calls go out.
  const results: WireCue[][] = chunks.map(() => []);
  let next = 0;
  let failed: unknown;
  const worker = async () => {
    for (;;) {
      const i = next++;
      if (i >= chunks.length || failed !== undefined || isStale?.()) return;
      try {
        results[i] = await postChunk(
          chunks[i].slice,
          chunks[i].offset,
          locale,
          opts?.freeOnly ?? false,
          opts?.request ?? apiFetch
        );
      } catch (error) {
        failed = error;
        return;
      }
    }
  };
  await Promise.all(Array.from({ length: Math.min(POSTS_IN_FLIGHT, chunks.length) }, worker));
  if (isStale?.()) return null;
  if (failed !== undefined) throw failed;

  const cues: SubtitleCue[] = [];
  for (const [i, { slice, offset, last }] of chunks.entries()) {
    const from = i === 0 ? -Infinity : offset + OVERLAP_SECONDS / 2;
    const to = last ? Infinity : offset + slice.length / RATE - OVERLAP_SECONDS / 2;
    for (const c of results[i]) {
      const start = Math.max(0, Math.min(c.start + offset, duration));
      const end = Math.max(0, Math.min(c.end + offset, duration));
      const text = c.text.trim();
      const mid = (start + end) / 2;
      if (!text || end <= start || mid < from || mid >= to) continue;
      // Word timings ride along when the route sends them; a cue that arrives
      // without them still shows, it just doesn't highlight word by word.
      const words = (Array.isArray(c.words) ? c.words : [])
        .filter((w) => typeof w?.t0 === "number" && typeof w?.t1 === "number" && typeof w?.w === "string")
        .map((w) => ({
          t0: round(Math.max(start, Math.min(w.t0 + offset, end))),
          t1: round(Math.max(start, Math.min(w.t1 + offset, end))),
          w: w.w,
        }));
      cues.push({
        id: uid(),
        start: round(start),
        end: round(end),
        text,
        ...(words.length > 0 ? { words } : {}),
      });
    }
  }
  return cues.sort((a, b) => a.start - b.start);
}

/** Transcribe a finished mic recording: decode it, downmix/resample to the
 * wire format, run the chunk pipeline, and join the cue texts. It posts to the
 * hosted route whatever backend the project lives on — the dictation reaches
 * here when this Mac's engine is not the one transcribing. */
export async function cloudTranscribeRecording(blob: Blob, locale?: string): Promise<string> {
  const bytes = await blob.arrayBuffer();
  // Decode at the device rate, then resample/downmix through an offline
  // render — decodeAudioData resamples to its context's rate, but the mono
  // 16 kHz target length isn't known until after the decode.
  const probe = new AudioContext();
  let decoded: AudioBuffer;
  try {
    decoded = await probe.decodeAudioData(bytes);
  } catch {
    throw new UserError("recording_audio_unreadable", "Could not read the recording's audio.");
  } finally {
    void probe.close().catch(() => {});
  }
  const ctx = new OfflineAudioContext(1, Math.max(1, Math.ceil(decoded.duration * RATE)), RATE);
  const src = ctx.createBufferSource();
  src.buffer = decoded;
  src.connect(ctx.destination);
  src.start();
  const mono = (await ctx.startRendering()).getChannelData(0);
  const cues = await transcribeSamples(mono, locale, undefined, { request: cloudBackend.fetch });
  return (cues ?? [])
    .map((c) => c.text)
    .join(" ")
    .trim();
}