Skip to content

Open source CapCut alternative.

TypeScript
22.2 KB
Raw Blame
"use client";

import { geminiModelRoleNames } from "@/lib/inference/gemini-models";
import { bytesFromBase64 } from "./bytes";
import { wavHeader } from "./wav";
import { NoCreditsError, readHostedFailure } from "./hostedError";
import { UserError } from "./userError";
import { hostedPost } from "./hosted";
import { CUT_JUDGE } from "./chatRuntime";
import { askJudge, choice } from "./judge";
import { importFileToProject } from "./media";
import { mediaSlug, type MediaAsset } from "./types";
import {
  DEFAULT_VOICE,
  resolveVoice,
  SPEECH_VOICES,
  VOICE_SAMPLE_TEXT,
  type SpeechVoice,
} from "./voices";

// The voice catalog lives in the dependency-light `voices` module (shared with
// the build script); re-exported here so existing importers keep working.
export { DEFAULT_VOICE, resolveVoice, SPEECH_VOICES, type SpeechVoice };

const VOICE_QUESTION = choice(
  "`wanted` describes the voice the user asked for in a video editor's voiceover. Which of these voices matches the description best?",
  Object.fromEntries(
    SPEECH_VOICES.map((v) => [v.id, `${v.name}: ${v.style}, ${v.gender === "f" ? "a woman" : "a man"}`]),
  ) as Record<string, string>,
);

/** A voice ask resolved to a model id: an id or display name matches as
 * `resolveVoice` does; a description ("a warm older man") is judged against
 * the catalog, and one the judge cannot commit to takes the default. */
export async function resolveVoiceAsk(wanted?: string): Promise<string> {
  const w = wanted?.trim();
  if (!w) return DEFAULT_VOICE;
  const named = resolveVoice(w);
  if (named !== DEFAULT_VOICE || w.toLowerCase() === DEFAULT_VOICE.toLowerCase()) return named;
  if (SPEECH_VOICES.some((v) => v.name.toLowerCase() === w.toLowerCase())) return named;
  try {
    const { answers } = await askJudge(
      (payload, signal) => hostedPost("/api/inference/judge", payload, signal),
      { wanted: w.slice(0, 500) },
      { voice: VOICE_QUESTION },
    );
    const pick = answers.voice;
    // The winner's own probability, not the spread of the distribution: over
    // twenty voices every plausible match takes a share, so a correct pick
    // reads as low confidence.
    const p = pick.probabilities[pick.choice] ?? 0;
    return p >= CUT_JUDGE.voicePick && SPEECH_VOICES.some((v) => v.id === pick.choice)
      ? pick.choice
      : DEFAULT_VOICE;
  } catch {
    return DEFAULT_VOICE;
  }
}

// Client side of AI voiceovers: Gemini speech generation on Donkey's hosted
// inference routes, with the user's Donkey sign-in and credits (same-origin on
// the cut hosts, like image/video generation). Each segment comes back as raw
// PCM; the browser lays the segments out at their timeline offsets, encodes
// one WAV, and saves it into the project through the local engine like any
// other media file.

export interface SpeechSegment {
  text: string;
  /** Timeline offset this line should land at, seconds. */
  at: number;
}

export interface SpeechLanguage {
  id: string;
  label: string;
  /** Voice-picker sample line, written natively in the language. */
  sample: string;
}

// The neutral sample line, shared with the persona catalog; also the default
// text for every language whose own sample is not written out below.
const SAMPLE_TEXT = VOICE_SAMPLE_TEXT;

/** Languages the speech model speaks (BCP-47, fixed by the model). "auto" lets
 * the model match the text; a concrete pick pins pronunciation and picks the
 * sample line's language. */
export const SPEECH_LANGUAGES: SpeechLanguage[] = [
  { id: "auto", label: "Auto — match the text", sample: SAMPLE_TEXT },
  { id: "ar-EG", label: "Arabic", sample: "هذا هو صوتي. لنصنع شيئًا يستحق التذكر." },
  { id: "bn-BD", label: "Bengali", sample: "আমার কণ্ঠ এমনই শোনায়। চলুন মনে রাখার মতো কিছু তৈরি করি।" },
  { id: "nl-NL", label: "Dutch", sample: "Zo klink ik. Laten we iets maken dat het onthouden waard is." },
  { id: "en-US", label: "English", sample: SAMPLE_TEXT },
  { id: "en-IN", label: "English (India)", sample: SAMPLE_TEXT },
  { id: "fr-FR", label: "French", sample: "Voici comment je sonne. Créons quelque chose d'inoubliable." },
  { id: "de-DE", label: "German", sample: "So klinge ich. Lass uns etwas Unvergessliches schaffen." },
  { id: "hi-IN", label: "Hindi", sample: "मेरी आवाज़ ऐसी है। चलिए कुछ यादगार बनाते हैं।" },
  { id: "id-ID", label: "Indonesian", sample: "Seperti inilah suara saya. Mari membuat sesuatu yang layak dikenang." },
  { id: "it-IT", label: "Italian", sample: "Questa è la mia voce. Creiamo qualcosa che valga la pena ricordare." },
  { id: "ja-JP", label: "Japanese", sample: "これが私の声です。記憶に残るものを作りましょう。" },
  { id: "ko-KR", label: "Korean", sample: "제 목소리는 이렇습니다. 기억에 남을 만한 것을 만들어 봅시다." },
  { id: "mr-IN", label: "Marathi", sample: "माझा आवाज असा आहे. चला काहीतरी संस्मरणीय बनवूया." },
  { id: "pl-PL", label: "Polish", sample: "Tak brzmi mój głos. Stwórzmy coś wartego zapamiętania." },
  { id: "pt-BR", label: "Portuguese", sample: "Esta é a minha voz. Vamos criar algo que valha a pena lembrar." },
  { id: "ro-RO", label: "Romanian", sample: "Așa sună vocea mea. Să creăm ceva demn de amintit." },
  { id: "ru-RU", label: "Russian", sample: "Вот как звучит мой голос. Давайте создадим что-то запоминающееся." },
  { id: "es-US", label: "Spanish", sample: "Así sueno. Hagamos algo digno de recordar." },
  { id: "ta-IN", label: "Tamil", sample: "என் குரல் இப்படித்தான் ஒலிக்கிறது. நினைவில் நிற்கும் ஒன்றை உருவாக்குவோம்." },
  { id: "te-IN", label: "Telugu", sample: "నా గొంతు ఇలా వినిపిస్తుంది. గుర్తుండిపోయేది ఒకటి చేద్దాం." },
  { id: "th-TH", label: "Thai", sample: "เสียงของฉันเป็นแบบนี้ มาสร้างสิ่งที่น่าจดจำกันเถอะ" },
  { id: "tr-TR", label: "Turkish", sample: "Sesim böyle. Hatırlanmaya değer bir şey yapalım." },
  { id: "uk-UA", label: "Ukrainian", sample: "Ось як звучить мій голос. Створімо щось варте пам'яті." },
  { id: "vi-VN", label: "Vietnamese", sample: "Giọng của tôi nghe như thế này. Hãy cùng tạo nên điều gì đó đáng nhớ." },
];

export const DEFAULT_LANGUAGE = "auto";

/** Resolve a requested speech language against the catalog: an exact
 * (case-insensitive) BCP-47 match, else a bare language ("es") to its catalog
 * variant, else auto-detect. Shared by every generation surface and the AI
 * copilot so a loose ask still lands on a supported code. */
export function resolveLanguage(wanted?: string): string {
  const w = wanted?.trim().toLowerCase();
  if (!w || w === "auto") return DEFAULT_LANGUAGE;
  const exact = SPEECH_LANGUAGES.find((l) => l.id.toLowerCase() === w);
  if (exact) return exact.id;
  const bare = SPEECH_LANGUAGES.find((l) => l.id.toLowerCase().startsWith(`${w}-`));
  return bare?.id ?? DEFAULT_LANGUAGE;
}

/** What a free-text voice direction resolves to once a model has read it. */
export interface VoiceoverPlan {
  /** Lines to speak, in order — translated into `language` when the direction
   * asked to speak in another tongue, otherwise the originals unchanged. */
  texts: string[];
  /** Delivery instruction with any "say it in X" language ask removed. */
  direction?: string;
  /** BCP-47 code to pin pronunciation: the language the direction named, or the
   * caller's picked language when it named none. */
  language?: string;
}

const PLAN_INSTRUCTIONS = `You prepare a voiceover before a text-to-speech voice reads it.

You get a delivery DIRECTION (free text about how to say it) and the LINES to speak. Decide whether the direction asks for the lines to be spoken in a specific language that differs from the language they are already written in.

Respond with ONLY a JSON object — no prose, no code fences:
{
  "language": string | null,  // BCP-47 tag (e.g. "ko-KR", "es-US", "ja-JP") when the direction asks to speak in a language; otherwise null
  "delivery": string | null,  // the direction with any language request removed (e.g. "warmly"); null if nothing about delivery is left
  "lines": string[] | null    // the LINES translated into "language", same count and order; null when no translation is needed
}

Rules:
- Set "language" only when the direction actually asks to speak in a language ("say it in Korean", "in Spanish", "read this in Japanese").
- When you set "language", translate every line into it, preserving meaning, tone, punctuation, and inline tags like [whispers]. Keep the same number of lines in the same order.
- When the direction is only about tone, pace, or energy ("say warmly"), return "language": null and "lines": null and put the whole direction in "delivery".
- When the direction is empty, return {"language": null, "delivery": null, "lines": null}.`;

/** Read a free-text voice direction for a target language and, when it names
 * one, translate the lines into it. The direction is natural language ("say it
 * in Korean, warmly"), so a model — not string matching — decides whether a
 * language was asked for; the returned plan carries the (possibly translated)
 * lines, the resolved BCP-47 code, and the direction with the language ask
 * stripped out. With no direction there is nothing to read, so the inputs pass
 * through unchanged and no round-trip happens. Any failure falls back to the
 * inputs so a generation never breaks on the prep step. */
export async function planVoiceover(
  texts: string[],
  opts: { direction?: string; language?: string }
): Promise<VoiceoverPlan> {
  const direction = opts.direction?.trim();
  const passthrough: VoiceoverPlan = { texts, direction, language: opts.language };
  if (!direction) return passthrough;
  try {
    const res = await hostedPost("/api/inference/responses", {
      donkeyProvider: "gemini",
      model: geminiModelRoleNames.chat,
      instructions: PLAN_INSTRUCTIONS,
      input: [{ role: "user", content: [{ text: JSON.stringify({ direction, lines: texts }) }] }],
    });
    if (!res.ok) return passthrough;
    const body = (await res.json()) as { output_text?: string };
    const plan = parsePlan(body.output_text);
    if (!plan) return passthrough;

    const language =
      typeof plan.language === "string" && plan.language.trim()
        ? resolveLanguage(plan.language)
        : opts.language;
    const translated =
      Array.isArray(plan.lines) && plan.lines.length === texts.length
        ? plan.lines.filter((l): l is string => typeof l === "string" && l.trim().length > 0)
        : [];
    const lines = translated.length === texts.length ? translated : texts;
    const delivery = typeof plan.delivery === "string" ? plan.delivery.trim() : "";
    return { texts: lines, direction: delivery || undefined, language };
  } catch {
    return passthrough;
  }
}

/** Pull the plan JSON out of the model's reply, tolerating a code fence. */
function parsePlan(
  text: string | undefined
): { language?: unknown; delivery?: unknown; lines?: unknown } | null {
  if (!text) return null;
  const body = text.trim().replace(/^```(?:json)?\s*|\s*```$/g, "");
  try {
    const value = JSON.parse(body);
    return value && typeof value === "object" ? value : null;
  } catch {
    return null;
  }
}

const MAX_SEGMENTS = 200;
const MAX_SEGMENT_CHARS = 2000;
const MAX_TOTAL_CHARS = 20000;
/** Hosted synthesis calls in flight at once for multi-line readouts. */
const CONCURRENCY = 3;

interface PcmClip {
  samples: Int16Array;
  rate: number;
}

const readFailure = (res: Response, fallback: UserError) =>
  readHostedFailure(res, new UserError("sign_in_to_generate_voiceovers", "Sign in to Donkey to generate voiceovers."), fallback);

/** One hosted Gemini speech call: text in, decoded PCM out. A direction is a
 * natural-language delivery instruction ("Say warmly, like an old friend");
 * the model also honors inline tags like [whispers] in the text itself. */
async function synthesizeSegment(
  text: string,
  voice: string,
  direction?: string,
  language?: string
): Promise<PcmClip> {
  const style = direction?.trim();
  const languageCode = resolveLanguage(language);
  const res = await hostedPost("/api/inference/assets", {
    kind: "speech",
    prompt: style ? `${style}: ${text}` : text,
    inputs: { voice },
    ...(languageCode !== DEFAULT_LANGUAGE
      ? { parameters: { languageCode } }
      : {}),
  });
  if (!res.ok) {
    throw await readFailure(res, new UserError("voice_generation_failed", "Voice generation failed."));
  }
  const gen = (await res.json()) as {
    outputs?: { dataBase64?: string; contentType?: string }[];
  };
  const out = gen.outputs?.find((o) => o.dataBase64);
  if (!out?.dataBase64) {
    throw new UserError("audio_output_missing", "The provider returned no audio.");
  }

  const bytes = bytesFromBase64(out.dataBase64);
  const rate = Number(out.contentType?.match(/rate=(\d+)/)?.[1]) || 24000;
  // Gemini TTS returns raw little-endian 16-bit mono PCM.
  return { samples: new Int16Array(bytes.buffer, 0, bytes.byteLength >> 1), rate };
}

/** Nearest-neighbor resample, only for the off chance segments disagree. */
function resample(clip: PcmClip, rate: number): PcmClip {
  if (clip.rate === rate) return clip;
  const out = new Int16Array(Math.round((clip.samples.length * rate) / clip.rate));
  for (let i = 0; i < out.length; i++) {
    out[i] = clip.samples[Math.min(clip.samples.length - 1, Math.round((i * clip.rate) / rate))];
  }
  return { samples: out, rate };
}

/** Lay the clips out at their offsets (summing any overlap, clamped) and wrap
 * the result in a WAV container. */
function assembleWav(clips: PcmClip[], offsets: number[]): Blob {
  const rate = clips[0].rate;
  const aligned = clips.map((c) => resample(c, rate));
  const starts = offsets.map((o) => Math.round(o * rate));
  const total = Math.max(...aligned.map((c, i) => starts[i] + c.samples.length));

  const mix = new Int32Array(total);
  aligned.forEach((c, i) => {
    const from = starts[i];
    for (let j = 0; j < c.samples.length; j++) mix[from + j] += c.samples[j];
  });

  const data = new Int16Array(total);
  for (let i = 0; i < total; i++) {
    data[i] = Math.max(-32768, Math.min(32767, mix[i]));
  }

  return new Blob([wavHeader(total, 1, rate, 16), data.buffer], { type: "audio/wav" });
}

export interface SpeechLayout {
  /** The spoken line's text (translated when the direction asked), so callers
   * can match a line back to its cue. */
  text: string;
  /** Timeline second this line's audio starts at. */
  at: number;
  /** How long the generated audio for this line runs, seconds. */
  duration: number;
}

/**
 * Synthesize segments into one WAV clip. The direction is read for a target
 * language and the lines translated when asked (planVoiceover), each line is
 * spoken, and the clips are laid out into a single WAV whose t=0 is the
 * earliest segment. `offset` is that earliest timeline time; `layout` reports
 * where each line landed so callers can re-time their cues. This is the shared
 * core behind every voiceover surface — the timeline asset, the subtitles
 * readout, and the in-panel preview — so language handling is identical
 * everywhere.
 */
/** The locale a text will be SPOKEN in, read from its writing system. For a
 * non-Latin script the letters are definitive — Hangul speaks Korean whatever
 * any language picker says. Latin scripts can't be told apart this way, so
 * they return undefined and the planned/picked language decides. */
export function scriptSpokenLocale(text: string): string | undefined {
  const scripts: [RegExp, string][] = [
    [/[぀-ヿ]/gu, "ja-JP"], // kana first: Japanese also uses Han
    [/[가-힯ᄀ-ᇿ㄰-㆏]/gu, "ko-KR"],
    [/[一-鿿]/gu, "zh-CN"],
    [/[Ѐ-ӿ]/gu, "ru-RU"],
    [/[؀-ۿ]/gu, "ar-SA"],
    [/[֐-׿]/gu, "he-IL"],
    [/[฀-๿]/gu, "th-TH"],
    [/[ऀ-ॿ]/gu, "hi-IN"],
    [/[Ͱ-Ͽ]/gu, "el-GR"],
  ];
  const letters = (text.match(/\p{L}/gu) ?? []).length;
  if (letters === 0) return undefined;
  for (const [re, locale] of scripts) {
    const count = (text.match(re) ?? []).length;
    if (count >= 4 && count / letters >= 0.2) return locale;
  }
  return undefined;
}

export async function renderSpeechClip(
  segments: SpeechSegment[],
  opts: { voice: string; direction?: string; language?: string }
): Promise<{ blob: Blob; offset: number; layout: SpeechLayout[]; language?: string }> {
  const raw = segments
    .map((s) => ({ text: s.text.trim(), at: Math.max(0, s.at) }))
    .filter((s) => s.text);
  if (raw.length === 0) {
    throw new UserError("speech_text_empty", "Nothing to say.");
  }
  if (raw.length > MAX_SEGMENTS) {
    throw new UserError("speech_lines_too_many", `At most ${MAX_SEGMENTS} lines at once.`, { max: MAX_SEGMENTS });
  }
  if (raw.some((s) => s.text.length > MAX_SEGMENT_CHARS)) {
    throw new UserError("speech_line_too_long", `Each line must stay under ${MAX_SEGMENT_CHARS} characters.`, {
      max: MAX_SEGMENT_CHARS,
    });
  }
  if (raw.reduce((n, s) => n + s.text.length, 0) > MAX_TOTAL_CHARS) {
    throw new UserError("speech_script_too_long", `The script must stay under ${MAX_TOTAL_CHARS} characters.`, {
      max: MAX_TOTAL_CHARS,
    });
  }

  // The one place a free-text direction becomes speech settings: a model reads
  // it for a "say it in X" ask and translates the lines when it names a
  // language, so the voice speaks the translation rather than English with an
  // accent. No direction → passthrough, no round-trip.
  const plan = await planVoiceover(
    raw.map((l) => l.text),
    { direction: opts.direction, language: opts.language }
  );
  const lines = raw.map((l, i) => ({ text: plan.texts[i] ?? l.text, at: l.at }));

  const offset = Math.min(...lines.map((s) => s.at));
  const voice = await resolveVoiceAsk(opts.voice);

  // A small pool: readouts can be dozens of lines, one hosted call each.
  const clips: PcmClip[] = new Array(lines.length);
  let next = 0;
  const worker = async () => {
    while (next < lines.length) {
      const i = next++;
      try {
        try {
          clips[i] = await synthesizeSegment(lines[i].text, voice, plan.direction, plan.language);
        } catch (e) {
          // An empty balance repeats forever — resending only burns the queue.
          if (e instanceof NoCreditsError) throw e;
          // The TTS backend rejects the odd call spuriously; one resend of just
          // this line usually lands and saves the rest of the batch.
          clips[i] = await synthesizeSegment(lines[i].text, voice, plan.direction, plan.language);
        }
      } catch (e) {
        // Name the line that failed so a one-bad-cue batch is actionable.
        if (lines.length > 1 && e instanceof Error && !(e instanceof NoCreditsError)) {
          const snippet = lines[i].text.length > 40 ? `${lines[i].text.slice(0, 40)}…` : lines[i].text;
          throw new UserError("speech_line_failed", `Line ${i + 1} ("${snippet}"): ${e.message}`, {
            line: i + 1,
            snippet,
            reason: e.message,
          });
        }
        throw e;
      }
    }
  };
  await Promise.all(
    Array.from({ length: Math.min(CONCURRENCY, lines.length) }, worker)
  );

  // Each line lands at its cue offset — unless the previous line's speech runs
  // past it. The generated pace differs from the original recording, so such
  // overlaps are common; mixed as-is they'd play two voice lines at once. The
  // pushed-back starts are reported in `layout` so callers re-time their cues.
  const placed: number[] = [];
  let cursor = 0;
  lines.forEach((l, i) => {
    const at = Math.max(l.at - offset, cursor);
    placed.push(at);
    cursor = at + clips[i].samples.length / clips[i].rate;
  });

  const blob = assembleWav(clips, placed);
  const layout: SpeechLayout[] = lines.map((l, i) => ({
    text: l.text,
    at: offset + placed[i],
    duration: clips[i].samples.length / clips[i].rate,
  }));
  // What the clip actually speaks — its writing system first (definitive for
  // non-Latin scripts), then the direction's ask, then the picked language.
  // Stamped on the asset so transcription later runs the right recognizer.
  const language =
    scriptSpokenLocale(lines.map((l) => l.text).join(" ")) ?? plan.language ?? opts.language;
  return { blob, offset, layout, ...(language ? { language } : {}) };
}

/** Save a rendered speech clip into the project's media folder as a voiceover
 * asset. Split from rendering so a preview can play the same clip without
 * committing it. */
export async function speechClipToAsset(
  projectId: string,
  blob: Blob,
  name?: string,
  language?: string
): Promise<MediaAsset> {
  const label = name?.trim() || "AI voice";
  const file = new File([blob], `${mediaSlug(label, "ai-voice")}.wav`, { type: "audio/wav" });
  const asset = await importFileToProject(projectId, file);
  if (!asset) {
    throw new UserError("voiceover_save_failed", "Could not save the voiceover into the project.");
  }
  asset.name = label;
  asset.origin = "voiceover";
  if (language) asset.language = language;
  // No Audio-tab badge here: this is the shared commit path (the panel, the
  // subtitles readout, the chat tools, the scene narration). Only the Audio
  // panel's own generator badges its tab — a chat- or run-made voiceover is
  // announced in the chat, never on the tab. The panel calls landed() itself.
  return asset;
}

/** Render the segments and save the result as a project voiceover asset —
 * render-then-commit in one call, for the surfaces that go straight to the
 * timeline. */
export async function synthesizeSpeech(
  projectId: string,
  segments: SpeechSegment[],
  opts: { voice: string; direction?: string; language?: string; name?: string }
): Promise<{ asset: MediaAsset; offset: number; layout: SpeechLayout[] }> {
  const { blob, offset, layout, language } = await renderSpeechClip(segments, opts);
  const asset = await speechClipToAsset(projectId, blob, opts.name, language);
  return { asset, offset, layout };
}