Skip to content

Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.

verify: transcribe the final mix scene by scene; s.ui typing volume; demo sound advice

Vincent Schmalbachdd361d39f888
diff --git a/AGENTS.md b/AGENTS.md
index 622cff6..92b4ee5 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -346,7 +346,7 @@ take seconds, spoken words or `"#marks"`, like everywhere else.
| `u.pop(at, cx, cy, fn, { sfx })` | pops anything in around a point |
| `u.wipe(at, { color })` | the accent color spreads out until it fills the frame |
| `u.cursor(keys, { out })` | the mouse; each key `{ at, x, y }` moves it, `{ at, click: true }` clicks with a sound |
-| `u.press(at)`, `u.typed(text, at, cps)`, `u.count(to, at, dur)` | a button press (0 to 1 and back), text typed so far with typing sounds, a number counting up |
+| `u.press(at)`, `u.typed(text, at, cps, { gain, sfx })`, `u.count(to, at, dur)` | a button press (0 to 1 and back), text typed so far with typing sounds, a number counting up |
| `u.colors` | `accent`, `accentDark`, `accentTint`, `background`, `ink`, `text`, `muted`, `faint`, `line`, `surface` |
`check` sees the text of `s.ui` too. It reports text that runs off the frame
@@ -355,6 +355,10 @@ overlaps. A dropdown or dialog may cover what is under it. Look at stills at
the moments the camera is zoomed in, and check the cursor does not hide the
word it points at.
+Many sounds under the voice make it hard to follow, and `verify` then
+understands less of it. Keep long typing quiet (`u.typed(text, at, cps, {
+gain: 0.25 })`) and give each moment one sound, not three.
+
A blur filter on every frame makes rendering slow. The kit blurs things once
and reuses them. Do the same if you add your own blur.
diff --git a/engine/ui.js b/engine/ui.js
index 28e0a86..bf1d6ea 100644
--- a/engine/ui.js
+++ b/engine/ui.js
@@ -911,11 +911,12 @@ export class UI {
return bell((this.s.t - this.at(at)) / (0.26 / this.s.pace));
}
- // The part of `str` typed by now, starting at `at`, with typing sounds.
- typed(str, at, cps = 17) {
+ // The part of `str` typed by now, starting at `at`, with typing sounds
+ // (o.gain, or o.sfx: false for none; keep long typing quiet under the voice).
+ typed(str, at, cps = 17, o = {}) {
const a = this.at(at);
const dur = str.length / cps / this.s.pace;
- this.s.sfx('type', a, { dur, gain: 0.55 });
+ if (o.sfx !== false) this.s.sfx('type', a, { dur, gain: o.gain ?? 0.55 });
return str.slice(0, Math.floor(clamp((this.s.t - a) / dur) * str.length));
}
diff --git a/src/qa.js b/src/qa.js
index 8cb9bb2..5a10067 100644
--- a/src/qa.js
+++ b/src/qa.js
@@ -114,18 +114,47 @@ export async function verify(project, { file = null, log = () => {} } = {}) {
for (const [s, e] of silences) issues.push({ level: 'hint', message: `near silence from ${s.toFixed(1)}s to ${e.toFixed(1)}s` });
}
- // Can the narration be understood over the music and effects?
+ // Can the narration be understood over the music and effects? Each scene
+ // is transcribed on its own: Whisper sometimes drops a whole 30 second
+ // chunk of a long file ("(bell dings)"), and per scene the report can say
+ // where the voice is hard to follow.
let speech = null;
if (a) {
const wav = path.join(project.paths.build, 'verify-16k.wav');
await ffmpeg(['-loglevel', 'error', '-i', video, '-vn', '-ac', '1', '-ar', '16000', '-c:a', 'pcm_s16le', wav]);
const { samples } = readWav(wav);
log('checking that the narration is understandable in the final mix');
- const heard = await transcribeWords(samples, { log });
- const script = project.script.scenes.flatMap((s) => s.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken })));
- const aligned = alignWords(script, heard.words, duration);
- speech = { matchRate: Math.round(aligned.matchRate * 1000) / 1000, transcript: heard.text };
- if (aligned.matchRate < 0.85) issues.push({ level: 'warn', message: `only ${Math.round(aligned.matchRate * 100)}% of the script was understood in the final mix; lower music.volume or check pronunciation` });
+ const tlFile = path.join(project.paths.build, 'timeline.json');
+ const timeline = fs.existsSync(tlFile) ? JSON.parse(fs.readFileSync(tlFile, 'utf8')) : null;
+ const spokenWords = (sc) => sc.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken }));
+ const byId = new Map(project.script.scenes.map((sc) => [sc.id, sc]));
+ const sameScenes = timeline && timeline.scenes.length === project.script.scenes.length && timeline.scenes.every((t) => byId.has(t.id)) && Math.abs(timeline.duration - duration) < 1;
+ if (sameScenes) {
+ const texts = [];
+ const scenes = [];
+ let total = 0;
+ let matched = 0;
+ for (const t of timeline.scenes) {
+ const words = spokenWords(byId.get(t.id));
+ if (!words.length) continue;
+ const from = Math.max(0, Math.floor(t.start * 16000));
+ const to = Math.min(samples.length, Math.ceil((t.start + t.dur) * 16000));
+ const heard = await transcribeWords(samples.subarray(from, to), { log });
+ texts.push(heard.text);
+ const rate = alignWords(words, heard.words, t.dur).matchRate;
+ scenes.push({ id: t.id, matchRate: Math.round(rate * 1000) / 1000 });
+ total += words.length;
+ matched += rate * words.length;
+ if (rate < 0.8) issues.push({ level: 'warn', message: `scene "${t.id}": only ${Math.round(rate * 100)}% of the narration was understood in the final mix; it heard "${heard.text.slice(0, 120)}"` });
+ }
+ const rate = total ? matched / total : 1;
+ speech = { matchRate: Math.round(rate * 1000) / 1000, transcript: texts.join(' '), scenes };
+ } else {
+ const heard = await transcribeWords(samples, { log });
+ const script = project.script.scenes.flatMap(spokenWords);
+ speech = { matchRate: Math.round(alignWords(script, heard.words, duration).matchRate * 1000) / 1000, transcript: heard.text };
+ }
+ if (speech.matchRate < 0.85) issues.push({ level: 'warn', message: `only ${Math.round(speech.matchRate * 100)}% of the script was understood in the final mix; lower music.volume or check pronunciation` });
fs.rmSync(wav, { force: true });
}
const report = {