Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
// fallow-ignore-file complexity
import { execFileSync } from "node:child_process";
import { existsSync, mkdirSync, readdirSync, unlinkSync } from "node:fs";
import { join, dirname, basename } from "node:path";
import { homedir } from "node:os";
import {
ensureModel,
ensureVoices,
DEFAULT_VOICE,
inferLangFromVoiceId,
type SupportedLang,
} from "./manager.js";
import { findPython, hasPythonPackage } from "./python.js";
import { writeNewFileSync } from "../utils/writeNewFile.js";
// ---------------------------------------------------------------------------
// Inline Python script for Kokoro synthesis
// ---------------------------------------------------------------------------
// Kokoro-onnx added the `lang=` kwarg to `Kokoro.create()` in a later release.
// We pass it conditionally so older installs that only accept `voice=`/`speed=`
// continue to work (falling back to Kokoro's default phonemization).
const SYNTH_SCRIPT = `
import sys, json, inspect, re
model_path = sys.argv[1]
voices_path = sys.argv[2]
text = sys.argv[3]
voice = sys.argv[4]
speed = float(sys.argv[5])
output_path = sys.argv[6]
lang = sys.argv[7] if len(sys.argv) > 7 else ""
import kokoro_onnx
import soundfile as sf
model = kokoro_onnx.Kokoro(model_path, voices_path)
kwargs = {"voice": voice, "speed": speed}
supports_lang = "lang" in inspect.signature(model.create).parameters
if lang and supports_lang:
kwargs["lang"] = lang
def is_voice_token_overflow(error):
return bool(re.search(
r"index \\d+ is out of bounds for axis 0 with size 510",
str(error),
re.IGNORECASE,
))
def split_for_retry(value):
midpoint = len(value) // 2
boundaries = [
index + 1
for index, char in enumerate(value)
if (char in "。!?!?\\n" or char.isspace()) and 0 < index < len(value) - 1
]
split_at = min(boundaries, key=lambda index: abs(index - midpoint)) if boundaries else midpoint
return value[:split_at], value[split_at:]
def synthesize_text(value):
try:
return model.create(value, **kwargs)
except IndexError as error:
if not is_voice_token_overflow(error) or len(value) < 2:
raise
left, right = split_for_retry(value)
left_samples, left_rate = synthesize_text(left)
right_samples, right_rate = synthesize_text(right)
if left_rate != right_rate:
raise RuntimeError("Kokoro returned inconsistent sample rates across text chunks")
samples = list(left_samples)
samples.extend(right_samples)
return samples, left_rate
samples, sample_rate = synthesize_text(text)
sf.write(output_path, samples, sample_rate)
duration = len(samples) / sample_rate
print(json.dumps({
"outputPath": output_path,
"sampleRate": sample_rate,
"durationSeconds": round(duration, 3),
"langApplied": bool(lang and supports_lang),
}))
`;
// espeak-ng maps Mandarin Chinese to ISO 639-3 "cmn", not Kokoro's own "zh"
// voice-prefix convention. Translate at the Python/espeak boundary only —
// keep "zh" as the public --lang value since it matches Kokoro's docs.
const ESPEAK_LANG_OVERRIDES: Partial<Record<SupportedLang, string>> = {
zh: "cmn",
};
// Cache the script to avoid rewriting it on every invocation.
// The filename carries a version suffix so older installs automatically
// upgrade when the script body changes (e.g., adding the `lang` kwarg).
const SCRIPT_DIR = join(homedir(), ".cache", "hyperframes", "tts");
const SCRIPT_PATH = join(SCRIPT_DIR, "synth-v3.py");
function ensureSynthScript(): string {
if (!existsSync(SCRIPT_PATH)) {
mkdirSync(SCRIPT_DIR, { recursive: true });
writeNewFileSync(SCRIPT_PATH, SYNTH_SCRIPT);
// Best-effort: delete older versioned scripts left behind by previous
// CLI releases so users don't accumulate stale files in ~/.cache.
const currentName = basename(SCRIPT_PATH);
try {
for (const entry of readdirSync(SCRIPT_DIR)) {
if (entry !== currentName && /^synth(-v\d+)?\.py$/.test(entry)) {
try {
unlinkSync(join(SCRIPT_DIR, entry));
} catch {
// Ignore — orphan cleanup is best-effort.
}
}
}
} catch {
// Ignore — directory read is best-effort.
}
}
return SCRIPT_PATH;
}
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
export interface SynthesizeOptions {
model?: string;
voice?: string;
speed?: number;
/**
* Phonemizer locale. When omitted, inferred from the voice ID prefix
* (e.g., `ef_dora` → `es`). Pass explicitly to override — for example,
* reading English text with a French voice as a stylization.
*/
lang?: SupportedLang;
onProgress?: (message: string) => void;
}
export interface SynthesizeResult {
outputPath: string;
sampleRate: number;
durationSeconds: number;
/** False when the installed kokoro-onnx version does not support the `lang` kwarg. */
langApplied: boolean;
}
/**
* Synthesize text to speech using Kokoro-82M via kokoro-onnx.
*/
// fallow-ignore-next-line complexity
export async function synthesize(
text: string,
outputPath: string,
options?: SynthesizeOptions,
): Promise<SynthesizeResult> {
const voice = options?.voice ?? DEFAULT_VOICE;
const speed = options?.speed ?? 1.0;
const lang: SupportedLang = options?.lang ?? inferLangFromVoiceId(voice);
// 1. Ensure Python 3 is available with kokoro-onnx
options?.onProgress?.("Checking Python runtime...");
const python = findPython();
if (!python) {
throw new Error(
"Python 3 is required for text-to-speech. Install Python 3.10+ and run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)",
);
}
if (!hasPythonPackage(python, "kokoro_onnx")) {
throw new Error(
"The kokoro-onnx package is not installed. Run: pip install kokoro-onnx soundfile (or point HYPERFRAMES_PYTHON at a venv python that has them)",
);
}
if (!hasPythonPackage(python, "soundfile")) {
throw new Error("The soundfile package is not installed. Run: pip install soundfile");
}
// 2. Ensure model and voices are downloaded (parallel on first run)
const [modelPath, voicesPath] = await Promise.all([
ensureModel(options?.model, { onProgress: options?.onProgress }),
ensureVoices({ onProgress: options?.onProgress }),
]);
// 3. Ensure synthesis script is cached
const scriptPath = ensureSynthScript();
// 4. Ensure output directory exists
mkdirSync(dirname(outputPath), { recursive: true });
// 5. Run synthesis
options?.onProgress?.(`Generating speech with voice ${voice} (${lang})...`);
try {
const espeakLang = ESPEAK_LANG_OVERRIDES[lang] ?? lang;
const stdout = execFileSync(
python,
[scriptPath, modelPath, voicesPath, text, voice, String(speed), outputPath, espeakLang],
{
encoding: "utf-8",
timeout: 300_000,
stdio: ["pipe", "pipe", "pipe"],
env: { ...process.env, ORT_DISABLE_TELEMETRY: "1" },
},
);
if (!existsSync(outputPath)) {
throw new Error("Synthesis completed but no output file was created");
}
// Parse the last line of stdout as JSON (in case Python printed warnings before it)
const lines = stdout.trim().split("\n");
const jsonLine = lines[lines.length - 1] ?? "";
const result: {
outputPath: string;
sampleRate: number;
durationSeconds: number;
langApplied: boolean;
} = JSON.parse(jsonLine);
return {
outputPath: result.outputPath,
sampleRate: result.sampleRate,
durationSeconds: result.durationSeconds,
langApplied: result.langApplied,
};
} catch (err: unknown) {
// If the error is our own JSON parse failure but the file was created,
// re-throw with a clearer message rather than returning fabricated data
if (err instanceof SyntaxError && existsSync(outputPath)) {
throw new Error(
"Speech was generated but metadata could not be read. Check the output file manually.",
);
}
let detail = "";
if (err && typeof err === "object" && "stderr" in err) {
const stderr = String(err.stderr).trim();
if (stderr) detail = `\n${stderr.slice(-500)}`;
}
throw new Error(`Speech synthesis failed${detail}`);
}
}