Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.
JavaScript ai-agentsanimationcanvasclaude-codecodexexplainer-videoffmpegjavascriptkokoroproduct-demotext-to-speechvideo-generationwhisper
// Aligns the words we asked the voice model to say with the words Whisper
// heard, so every script word gets a start and end time.
import { createRequire } from 'node:module';
import { normWord } from './script.js';
import { japaneseReadings } from './phonemes.js';
const require = createRequire(import.meta.url);
// Whisper often writes Mandarin in Traditional characters, even when the
// script is Simplified. Both sides are compared in Simplified.
let toSimplified;
function simplified(text) {
toSimplified ??= require('opencc-js/t2cn').Converter({ from: 'tw', to: 'cn' });
return toSimplified(text);
}
// Whisper also often writes another character with the same sound, like 直
// for 值. Mandarin characters match when they share a reading without tones.
const readings = new Map();
function soundsAlike(a, b) {
const as = Array.from(a);
const bs = Array.from(b);
if (as.length !== bs.length) return false;
const { pinyin } = require('pinyin-pro');
const of = c => {
if (!readings.has(c)) readings.set(c, /\p{Script=Han}/u.test(c) ? pinyin(c, { multiple: true, toneType: 'none', type: 'array' }) : [c]);
return readings.get(c);
};
return as.every((c, i) => c === bs[i] || of(c).some(r => of(bs[i]).includes(r)));
}
const katakana = text => text.replace(/[\u3041-\u3096]/g, c => String.fromCharCode(c.charCodeAt(0) + 0x60));
// Gives each piece of text the reading of its characters. A reading that
// covers several characters is shared out between them.
export function spreadReadings(pieces, tokens) {
const kana = Array.from(pieces.join(''), () => '');
for (const [start, end, reading] of tokens) {
const r = Array.from(katakana(reading));
const n = end - start;
for (let k = 0; k < n && start + k < kana.length; k++) kana[start + k] = r.slice(Math.round(k * r.length / n), Math.round((k + 1) * r.length / n)).join('');
}
let at = 0;
return pieces.map(p => {
const length = Array.from(p).length;
at += length;
return kana.slice(at - length, at).join('');
});
}
// Japanese has many words that sound the same, and Whisper often writes a
// word in kana that the script writes in kanji. When Python with misaki is
// there, both sides are compared by their reading.
export async function readAlike(words, heard, language) {
if (language !== 'ja' || !words.length || !heard.length) return { words, heard };
try {
const spoken = words.map(w => String(w.spoken));
const pieces = heard.map(h => String(h.text).trim());
const shown = words.map((w, i) => w.display !== undefined && String(w.display) !== spoken[i] ? String(w.display) : null);
const extra = shown.filter(s => s !== null);
const [a, b, ...c] = await japaneseReadings([spoken.join(''), pieces.join(''), ...extra]);
const spokenKana = spreadReadings(spoken, a);
const heardKana = spreadReadings(pieces, b);
let k = 0;
const shownKana = shown.map(s => s === null ? null : spreadReadings([s], c[k++])[0]);
return {
words: words.map((w, i) => ({ ...w, spoken: spokenKana[i] || w.spoken, display: shown[i] === null ? undefined : shownKana[i] || w.display })),
heard: heard.map((h, i) => ({ ...h, text: heardKana[i] || h.text })),
};
} catch {
return { words, heard };
}
}
const NUMBER_WORDS = {
zero: '0', one: '1', two: '2', three: '3', four: '4', five: '5', six: '6', seven: '7', eight: '8', nine: '9',
ten: '10', eleven: '11', twelve: '12', thirteen: '13', fourteen: '14', fifteen: '15', sixteen: '16',
seventeen: '17', eighteen: '18', nineteen: '19', twenty: '20', thirty: '30', forty: '40', fifty: '50',
sixty: '60', seventy: '70', eighty: '80', ninety: '90', hundred: '100', thousand: '1000',
};
function canon(w) {
const n = normWord(w).replace(/,/g, '');
return NUMBER_WORDS[n] ?? n;
}
const ONES = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];
const TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];
const SCALES = [[1e9, 'billion'], [1e6, 'million'], [1e3, 'thousand']];
const CURRENCY = { $: 'dollars', '€': 'euros', '£': 'pounds' };
// 1102 -> one thousand one hundred two (without "and"; "and" is optional below).
export function intWords(n) {
if (!Number.isFinite(n) || n < 0 || n >= 1e12) return [String(n)];
if (n < 20) return [ONES[n]];
if (n < 100) return [TENS[Math.floor(n / 10)], ...(n % 10 ? [ONES[n % 10]] : [])];
if (n < 1000) return [ONES[Math.floor(n / 100)], 'hundred', ...(n % 100 ? intWords(n % 100) : [])];
for (const [size, name] of SCALES) {
if (n >= size) return [...intWords(Math.floor(n / size)), name, ...(n % size ? intWords(n % size) : [])];
}
return [String(n)];
}
// Kokoro reads a lone four-digit number like a year: 1582 "fifteen eighty
// two", 1900 "nineteen hundred", 1905 "nineteen oh five". Numbers below 1100
// or with fewer than ten in the last three digits (2000, 2005) stay cardinal.
export function yearWords(n) {
if (n < 1100 || n > 9999 || n % 1000 < 10) return null;
const hi = Math.floor(n / 100);
const lo = n % 100;
if (n % 1000 >= 100 && lo === 0) return [...intWords(hi), 'hundred'];
if (n % 1000 >= 100 && lo < 10) return [...intWords(hi), 'oh', ...intWords(lo)];
return [...intWords(hi), ...intWords(lo)];
}
const ORDINALS = { one: 'first', two: 'second', three: 'third', five: 'fifth', eight: 'eighth', nine: 'ninth', twelve: 'twelfth' };
// 29th -> twenty ninth, 101st -> one hundred first.
function ordinalWords(n) {
const words = intWords(n);
const last = words.pop();
const ord = ORDINALS[last] ?? (last.endsWith('y') ? last.slice(0, -1) + 'ieth' : last + 'th');
return [...words.map(canon), ord];
}
// Turns one written word into the tokens a voice would say, in canonical form.
// Whisper writes "$1,050" and "5%" where the voice said "one thousand and
// fifty dollars" and "five percent", and "example.com" or "1.1.1.1" where it
// said words joined by "dot". Both sides are expanded the same way.
export function expand(word, language = 'en') {
if (language !== 'en') {
const n = normWord(word).replace(/[०-९]/g, d => String(d.charCodeAt(0) - 0x0966));
const digits = /^[\p{N}%$€£+.]+$/u.test(n) ? n.replace(/[^\p{N}]/gu, '') : '';
return n ? [digits || n] : [];
}
const raw = String(word).trim().toLowerCase();
// Whisper splits "1.1.1.1" into "1", ".1", ".1", ".1": a leading dot is a spoken "dot".
const lead = /^\.[a-z0-9]/i.test(raw) ? ['dot'] : [];
const n = canon(word);
const ordinal = /^(\d+)(st|nd|rd|th)$/.exec(n);
if (ordinal && !/\d,\d/.test(raw)) return [...lead, ...ordinalWords(Number(ordinal[1]))];
const money = /^([$€£])?(\d+)(?:\.(\d+))?(%)?$/.exec(n);
if (money && /\d/.test(raw) && !/^\d+\.\d+\.\d+/.test(n)) {
const [, cur, int, dec, pct] = money;
const year = !cur && dec === undefined && !pct && /^\d{4}$/.test(int) && !/\d,\d/.test(raw) ? yearWords(Number(int)) : null;
if (year) return [...lead, ...year].map(canon);
const out = [...lead, ...intWords(Number(int))];
if (dec !== undefined) {
if (cur) out.push(CURRENCY[cur], ...intWords(Number(dec)), 'cents');
else out.push('point', ...dec.split('').map((d) => ONES[Number(d)]));
} else if (cur) out.push(CURRENCY[cur]);
if (pct) out.push('percent');
return out.map(canon);
}
if (!/[a-z0-9]\.[a-z0-9]/.test(n)) return n ? [...lead, n] : lead;
const parts = n.split('.').filter(Boolean).map((p) => NUMBER_WORDS[p] ?? p);
const out = [...lead];
parts.forEach((p, i) => {
if (i) out.push('dot');
out.push(p);
});
return out;
}
const isNumberToken = (t) => /^\d+$/.test(t) || ['hundred', 'thousand', 'million', 'billion', '100', '1000'].includes(t);
function editDistance(a, b) {
if (a === b) return 0;
const m = a.length;
const n = b.length;
if (!m) return n;
if (!n) return m;
let prev = Array.from({ length: n + 1 }, (_, j) => j);
for (let i = 1; i <= m; i++) {
const cur = [i];
for (let j = 1; j <= n; j++) {
cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
}
prev = cur;
}
return prev[n];
}
export function similar(a, b) {
if (!a || !b) return false;
if (a === b) return true;
const len = Math.max(a.length, b.length);
if (len <= 3) return false;
return editDistance(a, b) / len <= 0.34;
}
// words: [{ spoken, display? }] in order. heard: [{ text, start, end }] from Whisper.
// Returns per-word { start, end, matched } and the share of spoken tokens matched.
// A word also counts as heard when Whisper writes its shown form, so
// {SQL|sequel} or {async|a sink} is confirmed by "SQL" or "async".
export function alignWords(words, heard, duration, language = 'en') {
const cjk = ['ja', 'zh'].includes(language);
const chars = word => normWord(word).normalize('NFC');
const partsOf = word => cjk ? Array.from(language === 'zh' ? simplified(chars(word)) : chars(word)) : expand(word, language);
const same = (a, b) => a === b || (language === 'zh' && soundsAlike(a, b));
const matches = (a, b) => cjk ? same(a, b) : language !== 'en' && (/\p{N}/u.test(a) || /\p{N}/u.test(b)) ? a === b : similar(a, b);
const tokens = [];
words.forEach((w, wi) => {
const parts = String(w.spoken).split(/\s+/).flatMap(partsOf);
(parts.length ? parts : ['']).forEach((p, k) => {
// "and" inside a spoken number ("one thousand and fifty") is optional.
const prev = tokens[tokens.length - 1];
const optional = language === 'en' && p === 'and' && prev && isNumberToken(prev.norm);
tokens.push({ wi, norm: p, first: k === 0, last: k === parts.length - 1 || !parts.length, optional });
});
const shown = w.display === undefined ? '' : String(w.display).split(/\s+/).flatMap(partsOf).join('');
const first = tokens.findIndex((t) => t.wi === wi);
if (shown && shown !== parts.join('')) tokens[first].shown = shown;
});
// Whisper splits numbers into pieces ("$1", ",000" and "5", "%" and "1",
// ".1"). Join them back before comparing.
const joined = [];
for (const h of heard) {
const prev = joined[joined.length - 1];
const t = String(h.text).trim();
const glue = prev && ((/^[,.]\d/.test(t) && /\d$/.test(prev.text)) || (/^%/.test(t) && /\d$/.test(prev.text)) || (/^\d/.test(t) && /^[$€£]$/.test(prev.text)));
if (glue) {
prev.text += t;
prev.end = h.end;
} else {
joined.push({ ...h, text: t });
}
}
const H = [];
for (const h of joined) {
const parts = partsOf(h.text);
const step = (h.end - h.start) / Math.max(1, parts.length);
parts.forEach((p, k) => H.push({ text: p, norm: p, start: h.start + k * step, end: h.start + (k + 1) * step }));
}
const T = tokens.length;
const N = H.length;
// Dynamic programming over (token i, heard j). Moves:
// match 1:1, skip a token, skip a heard word, a whole multi-token word to one
// heard word ("D N S" vs "DNS"), and one token to two heard words.
const INF = 1e9;
const cost = Array.from({ length: T + 1 }, () => new Float64Array(N + 1).fill(INF));
const move = Array.from({ length: T + 1 }, () => new Int8Array(N + 1));
const shownLength = cjk ? Array.from({ length: T + 1 }, () => new Int32Array(N + 1)) : null;
cost[0][0] = 0;
const groupEnd = new Int32Array(T);
for (let i = 0; i < T; i++) {
let e = i;
while (e + 1 < T && tokens[e + 1].wi === tokens[i].wi) e++;
groupEnd[i] = e;
}
for (let i = 0; i <= T; i++) {
for (let j = 0; j <= N; j++) {
const c = cost[i][j];
if (c >= INF) continue;
if (i < T && c + (tokens[i].optional ? 0 : 1) < cost[i + 1][j]) { cost[i + 1][j] = c + (tokens[i].optional ? 0 : 1); move[i + 1][j] = 2; }
if (j < N && c + 1 < cost[i][j + 1]) { cost[i][j + 1] = c + 1; move[i][j + 1] = 3; }
if (i < T && j < N) {
const ok = matches(tokens[i].norm, H[j].norm);
const v = c + (ok ? 0 : 1.6);
if (v < cost[i + 1][j + 1]) { cost[i + 1][j + 1] = v; move[i + 1][j + 1] = ok ? 1 : 4; }
}
if (i < T && j < N && tokens[i].first && (groupEnd[i] > i || tokens[i].shown)) {
const e = groupEnd[i];
const joined = tokens.slice(i, e + 1).map((t) => t.norm).join('');
const ok = (groupEnd[i] > i && matches(joined, H[j].norm)) || (tokens[i].shown && matches(tokens[i].shown, H[j].norm));
if (ok && c < cost[e + 1][j + 1]) { cost[e + 1][j + 1] = c; move[e + 1][j + 1] = 5; }
}
if (cjk && i < T && tokens[i].first && tokens[i].shown) {
const length = Array.from(tokens[i].shown).length;
const e = groupEnd[i];
if (j + length <= N && same(tokens[i].shown, H.slice(j, j + length).map(h => h.norm).join('')) && c < cost[e + 1][j + length]) {
cost[e + 1][j + length] = c;
move[e + 1][j + length] = 7;
shownLength[e + 1][j + length] = length;
}
}
if (i < T && j + 1 < N && matches(tokens[i].norm, H[j].norm + H[j + 1].norm) && c < cost[i + 1][j + 2]) {
cost[i + 1][j + 2] = c;
move[i + 1][j + 2] = 6;
}
}
}
const spans = words.map(() => null);
const hits = words.map(() => 0);
const need = words.map(() => 0);
for (const t of tokens) if (!t.optional) need[t.wi]++;
const add = (wi, s, e, n = 1) => {
hits[wi] += n;
const cur = spans[wi];
spans[wi] = cur ? { start: Math.min(cur.start, s), end: Math.max(cur.end, e) } : { start: s, end: e };
};
let matched = 0;
let i = T;
let j = N;
while (i > 0 || j > 0) {
const m = move[i][j];
if (m === 1) { add(tokens[i - 1].wi, H[j - 1].start, H[j - 1].end); matched++; i--; j--; }
else if (m === 4) { i--; j--; }
else if (m === 2) { i--; }
else if (m === 3) { j--; }
else if (m === 5) {
let s = i - 1;
while (s > 0 && tokens[s - 1].wi === tokens[i - 1].wi) s--;
add(tokens[i - 1].wi, H[j - 1].start, H[j - 1].end, i - s);
matched += i - s;
i = s;
j--;
} else if (m === 7) {
const length = shownLength[i][j];
let s = i - 1;
while (s > 0 && tokens[s - 1].wi === tokens[i - 1].wi) s--;
add(tokens[i - 1].wi, H[j - length].start, H[j - 1].end, i - s);
matched += i - s;
i = s;
j -= length;
} else if (m === 6) { add(tokens[i - 1].wi, H[j - 2].start, H[j - 1].end); matched++; i--; j -= 2; }
else break;
}
// Interpolate words Whisper did not confirm, by character length, between
// the nearest confirmed neighbours.
// A word counts as confirmed only when every part of it was heard, so
// "example.com" read as "example comm" is reported.
const out = words.map((w, wi) => ({ start: 0, end: 0, matched: !!spans[wi] && hits[wi] >= need[wi] }));
const weight = words.map((w) => Math.max(1, String(w.spoken).length));
let k = 0;
while (k < words.length) {
if (spans[k]) {
out[k].start = spans[k].start;
out[k].end = Math.max(spans[k].end, spans[k].start + 0.05);
k++;
continue;
}
let e = k;
while (e < words.length && !spans[e]) e++;
const from = k > 0 ? out[k - 1].end : 0;
const to = e < words.length ? spans[e].start : duration;
const total = weight.slice(k, e).reduce((a, b) => a + b, 0);
let t = from;
for (let q = k; q < e; q++) {
const d = ((to - from) * weight[q]) / total;
out[q].start = t;
out[q].end = t + d;
t += d;
}
k = e;
}
// Keep times monotonic.
for (let q = 1; q < out.length; q++) {
if (out[q].start < out[q - 1].start) out[q].start = out[q - 1].start;
if (out[q].end < out[q].start) out[q].end = out[q].start + 0.05;
}
const counted = tokens.filter((t) => !t.optional).length;
return { words: out, matchRate: counted ? Math.min(1, matched / counted) : 1 };
}