Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.
JavaScript ai-agentsanimationcanvasclaude-codecodexexplainer-videoffmpegjavascriptkokoroproduct-demotext-to-speechvideo-generationwhisper
Confirm a word when the speech check hears its shown form (#16)
Vincent Schmalbach55e9639ed65e
diff --git a/src/align.js b/src/align.jsindex e27b6c2..70c589a 100644--- a/src/align.js+++ b/src/align.js@@ -93,8 +93,10 @@ export function similar(a, b) {return editDistance(a, b) / len <= 0.34;}-// words: [{ spoken }] in order. heard: [{ text, start, end }] from Whisper.+// words: [{ spoken, display? }] in order. heard: [{ text, start, end }] from Whisper.// Returns per-word { start, end, matched } and the share of spoken tokens matched.+// A word also counts as heard when Whisper writes its shown form, so+// {SQL|sequel} or {async|a sink} is confirmed by "SQL" or "async".export function alignWords(words, heard, duration, language = 'en') {const partsOf = word => expand(word, language);const matches = (a, b) => language !== 'en' && (/\p{N}/u.test(a) || /\p{N}/u.test(b)) ? a === b : similar(a, b);@@ -107,6 +109,9 @@ export function alignWords(words, heard, duration, language = 'en') {const optional = language === 'en' && p === 'and' && prev && isNumberToken(prev.norm);tokens.push({ wi, norm: p, first: k === 0, last: k === parts.length - 1 || !parts.length, optional });});+ const shown = w.display === undefined ? '' : String(w.display).split(/\s+/).flatMap(partsOf).join('');+ const first = tokens.findIndex((t) => t.wi === wi);+ if (shown && shown !== parts.join('')) tokens[first].shown = shown;});// Whisper splits numbers into pieces ("$1", ",000" and "5", "%" and "1",// ".1"). Join them back before comparing.@@ -155,10 +160,11 @@ export function alignWords(words, heard, duration, language = 'en') {const v = c + (ok ? 0 : 1.6);if (v < cost[i + 1][j + 1]) { cost[i + 1][j + 1] = v; move[i + 1][j + 1] = ok ? 1 : 4; }}- if (i < T && j < N && tokens[i].first && groupEnd[i] > i) {+ if (i < T && j < N && tokens[i].first && (groupEnd[i] > i || tokens[i].shown)) {const e = groupEnd[i];const joined = tokens.slice(i, e + 1).map((t) => t.norm).join('');- if (matches(joined, H[j].norm) && c < cost[e + 1][j + 1]) { cost[e + 1][j + 1] = c; move[e + 1][j + 1] = 5; }+ const ok = (groupEnd[i] > i && matches(joined, H[j].norm)) || (tokens[i].shown && matches(tokens[i].shown, H[j].norm));+ if (ok && c < cost[e + 1][j + 1]) { cost[e + 1][j + 1] = c; move[e + 1][j + 1] = 5; }}if (i < T && j + 1 < N && matches(tokens[i].norm, H[j].norm + H[j + 1].norm) && c < cost[i + 1][j + 2]) {cost[i + 1][j + 2] = c;diff --git a/src/qa.js b/src/qa.jsindex b847008..557955e 100644--- a/src/qa.js+++ b/src/qa.js@@ -136,7 +136,7 @@ export async function verify(project, { file = null, log = () => {} } = {}) {log('checking that the narration is understandable in the final mix');const tlFile = path.join(project.paths.build, 'timeline.json');const timeline = fs.existsSync(tlFile) ? JSON.parse(fs.readFileSync(tlFile, 'utf8')) : null;- const spokenWords = (sc) => sc.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken }));+ const spokenWords = (sc) => sc.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken, display: u.display }));const byId = new Map(project.script.scenes.map((sc) => [sc.id, sc]));const sameScenes = timeline && timeline.scenes.length === project.script.scenes.length && timeline.scenes.every((t) => byId.has(t.id)) && Math.abs(timeline.duration - duration) < 1;if (sameScenes) {diff --git a/test/align.test.js b/test/align.test.jsindex 26fe81c..cbe3a1b 100644--- a/test/align.test.js+++ b/test/align.test.js@@ -63,3 +63,18 @@ test('number pieces from Whisper are joined', () => {const r = alignWords([{ spoken: 'one thousand dollars' }, { spoken: 'five percent' }], heard([['$1', 0, 0.4], [',000', 0.4, 0.8], ['5', 0.8, 1], ['%', 1, 1.2]]), 1.2);assert.equal(r.matchRate, 1);});++test('a word counts as heard when Whisper writes its shown form', () => {+ const one = alignWords([{ spoken: 'The' }, { spoken: 'sequel', display: 'SQL' }, { spoken: 'query.' }], heard([['The', 0, 0.2], ['SQL', 0.2, 0.7], ['query.', 0.7, 1.1]]), 1.2);+ assert.equal(one.matchRate, 1);+ assert.ok(one.words[1].matched);+ const two = alignWords([{ spoken: 'The' }, { spoken: 'a sink', display: 'async' }, { spoken: 'keyword.' }], heard([['The', 0, 0.2], ['async', 0.2, 0.7], ['keyword.', 0.7, 1.1]]), 1.2);+ assert.equal(two.matchRate, 1);+ assert.equal(two.words[1].start, 0.2);+ // The spoken form still matches as before.+ const spoken = alignWords([{ spoken: 'a sink', display: 'async' }], heard([['a', 0, 0.2], ['sink', 0.2, 0.5]]), 0.6);+ assert.equal(spoken.matchRate, 1);+ // A different word is still not confirmed.+ const wrong = alignWords([{ spoken: 'sequel', display: 'SQL' }], heard([['table', 0, 0.5]]), 0.6);+ assert.ok(!wrong.words[0].matched);+});