Skip to content

Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.

Confirm a word when the speech check hears its shown form (#16)

Vincent Schmalbach55e9639ed65e
diff --git a/src/align.js b/src/align.js
index e27b6c2..70c589a 100644
--- a/src/align.js
+++ b/src/align.js
@@ -93,8 +93,10 @@ export function similar(a, b) {
return editDistance(a, b) / len <= 0.34;
}
-// words: [{ spoken }] in order. heard: [{ text, start, end }] from Whisper.
+// words: [{ spoken, display? }] in order. heard: [{ text, start, end }] from Whisper.
// Returns per-word { start, end, matched } and the share of spoken tokens matched.
+// A word also counts as heard when Whisper writes its shown form, so
+// {SQL|sequel} or {async|a sink} is confirmed by "SQL" or "async".
export function alignWords(words, heard, duration, language = 'en') {
const partsOf = word => expand(word, language);
const matches = (a, b) => language !== 'en' && (/\p{N}/u.test(a) || /\p{N}/u.test(b)) ? a === b : similar(a, b);
@@ -107,6 +109,9 @@ export function alignWords(words, heard, duration, language = 'en') {
const optional = language === 'en' && p === 'and' && prev && isNumberToken(prev.norm);
tokens.push({ wi, norm: p, first: k === 0, last: k === parts.length - 1 || !parts.length, optional });
});
+ const shown = w.display === undefined ? '' : String(w.display).split(/\s+/).flatMap(partsOf).join('');
+ const first = tokens.findIndex((t) => t.wi === wi);
+ if (shown && shown !== parts.join('')) tokens[first].shown = shown;
});
// Whisper splits numbers into pieces ("$1", ",000" and "5", "%" and "1",
// ".1"). Join them back before comparing.
@@ -155,10 +160,11 @@ export function alignWords(words, heard, duration, language = 'en') {
const v = c + (ok ? 0 : 1.6);
if (v < cost[i + 1][j + 1]) { cost[i + 1][j + 1] = v; move[i + 1][j + 1] = ok ? 1 : 4; }
}
- if (i < T && j < N && tokens[i].first && groupEnd[i] > i) {
+ if (i < T && j < N && tokens[i].first && (groupEnd[i] > i || tokens[i].shown)) {
const e = groupEnd[i];
const joined = tokens.slice(i, e + 1).map((t) => t.norm).join('');
- if (matches(joined, H[j].norm) && c < cost[e + 1][j + 1]) { cost[e + 1][j + 1] = c; move[e + 1][j + 1] = 5; }
+ const ok = (groupEnd[i] > i && matches(joined, H[j].norm)) || (tokens[i].shown && matches(tokens[i].shown, H[j].norm));
+ if (ok && c < cost[e + 1][j + 1]) { cost[e + 1][j + 1] = c; move[e + 1][j + 1] = 5; }
}
if (i < T && j + 1 < N && matches(tokens[i].norm, H[j].norm + H[j + 1].norm) && c < cost[i + 1][j + 2]) {
cost[i + 1][j + 2] = c;
diff --git a/src/qa.js b/src/qa.js
index b847008..557955e 100644
--- a/src/qa.js
+++ b/src/qa.js
@@ -136,7 +136,7 @@ export async function verify(project, { file = null, log = () => {} } = {}) {
log('checking that the narration is understandable in the final mix');
const tlFile = path.join(project.paths.build, 'timeline.json');
const timeline = fs.existsSync(tlFile) ? JSON.parse(fs.readFileSync(tlFile, 'utf8')) : null;
- const spokenWords = (sc) => sc.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken }));
+ const spokenWords = (sc) => sc.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken, display: u.display }));
const byId = new Map(project.script.scenes.map((sc) => [sc.id, sc]));
const sameScenes = timeline && timeline.scenes.length === project.script.scenes.length && timeline.scenes.every((t) => byId.has(t.id)) && Math.abs(timeline.duration - duration) < 1;
if (sameScenes) {
diff --git a/test/align.test.js b/test/align.test.js
index 26fe81c..cbe3a1b 100644
--- a/test/align.test.js
+++ b/test/align.test.js
@@ -63,3 +63,18 @@ test('number pieces from Whisper are joined', () => {
const r = alignWords([{ spoken: 'one thousand dollars' }, { spoken: 'five percent' }], heard([['$1', 0, 0.4], [',000', 0.4, 0.8], ['5', 0.8, 1], ['%', 1, 1.2]]), 1.2);
assert.equal(r.matchRate, 1);
});
+
+test('a word counts as heard when Whisper writes its shown form', () => {
+ const one = alignWords([{ spoken: 'The' }, { spoken: 'sequel', display: 'SQL' }, { spoken: 'query.' }], heard([['The', 0, 0.2], ['SQL', 0.2, 0.7], ['query.', 0.7, 1.1]]), 1.2);
+ assert.equal(one.matchRate, 1);
+ assert.ok(one.words[1].matched);
+ const two = alignWords([{ spoken: 'The' }, { spoken: 'a sink', display: 'async' }, { spoken: 'keyword.' }], heard([['The', 0, 0.2], ['async', 0.2, 0.7], ['keyword.', 0.7, 1.1]]), 1.2);
+ assert.equal(two.matchRate, 1);
+ assert.equal(two.words[1].start, 0.2);
+ // The spoken form still matches as before.
+ const spoken = alignWords([{ spoken: 'a sink', display: 'async' }], heard([['a', 0, 0.2], ['sink', 0.2, 0.5]]), 0.6);
+ assert.equal(spoken.matchRate, 1);
+ // A different word is still not confirmed.
+ const wrong = alignWords([{ spoken: 'sequel', display: 'SQL' }], heard([['table', 0, 0.5]]), 0.6);
+ assert.ok(!wrong.words[0].matched);
+});