Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.
JavaScript ai-agentsanimationcanvasclaude-codecodexexplainer-videoffmpegjavascriptkokoroproduct-demotext-to-speechvideo-generationwhisper
verify: transcribe the final mix scene by scene; s.ui typing volume; demo sound advice
Vincent Schmalbachdd361d39f888
diff --git a/AGENTS.md b/AGENTS.mdindex 622cff6..92b4ee5 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -346,7 +346,7 @@ take seconds, spoken words or `"#marks"`, like everywhere else.| `u.pop(at, cx, cy, fn, { sfx })` | pops anything in around a point || `u.wipe(at, { color })` | the accent color spreads out until it fills the frame || `u.cursor(keys, { out })` | the mouse; each key `{ at, x, y }` moves it, `{ at, click: true }` clicks with a sound |-| `u.press(at)`, `u.typed(text, at, cps)`, `u.count(to, at, dur)` | a button press (0 to 1 and back), text typed so far with typing sounds, a number counting up |+| `u.press(at)`, `u.typed(text, at, cps, { gain, sfx })`, `u.count(to, at, dur)` | a button press (0 to 1 and back), text typed so far with typing sounds, a number counting up || `u.colors` | `accent`, `accentDark`, `accentTint`, `background`, `ink`, `text`, `muted`, `faint`, `line`, `surface` |`check` sees the text of `s.ui` too. It reports text that runs off the frame@@ -355,6 +355,10 @@ overlaps. A dropdown or dialog may cover what is under it. Look at stills atthe moments the camera is zoomed in, and check the cursor does not hide theword it points at.+Many sounds under the voice make it hard to follow, and `verify` then+understands less of it. Keep long typing quiet (`u.typed(text, at, cps, {+gain: 0.25 })`) and give each moment one sound, not three.+A blur filter on every frame makes rendering slow. The kit blurs things onceand reuses them. Do the same if you add your own blur.diff --git a/engine/ui.js b/engine/ui.jsindex 28e0a86..bf1d6ea 100644--- a/engine/ui.js+++ b/engine/ui.js@@ -911,11 +911,12 @@ export class UI {return bell((this.s.t - this.at(at)) / (0.26 / this.s.pace));}- // The part of `str` typed by now, starting at `at`, with typing sounds.- typed(str, at, cps = 17) {+ // The part of `str` typed by now, starting at `at`, with typing sounds+ // (o.gain, or o.sfx: false for none; keep long typing quiet under the voice).+ typed(str, at, cps = 17, o = {}) {const a = this.at(at);const dur = str.length / cps / this.s.pace;- this.s.sfx('type', a, { dur, gain: 0.55 });+ if (o.sfx !== false) this.s.sfx('type', a, { dur, gain: o.gain ?? 0.55 });return str.slice(0, Math.floor(clamp((this.s.t - a) / dur) * str.length));}diff --git a/src/qa.js b/src/qa.jsindex 8cb9bb2..5a10067 100644--- a/src/qa.js+++ b/src/qa.js@@ -114,18 +114,47 @@ export async function verify(project, { file = null, log = () => {} } = {}) {for (const [s, e] of silences) issues.push({ level: 'hint', message: `near silence from ${s.toFixed(1)}s to ${e.toFixed(1)}s` });}- // Can the narration be understood over the music and effects?+ // Can the narration be understood over the music and effects? Each scene+ // is transcribed on its own: Whisper sometimes drops a whole 30 second+ // chunk of a long file ("(bell dings)"), and per scene the report can say+ // where the voice is hard to follow.let speech = null;if (a) {const wav = path.join(project.paths.build, 'verify-16k.wav');await ffmpeg(['-loglevel', 'error', '-i', video, '-vn', '-ac', '1', '-ar', '16000', '-c:a', 'pcm_s16le', wav]);const { samples } = readWav(wav);log('checking that the narration is understandable in the final mix');- const heard = await transcribeWords(samples, { log });- const script = project.script.scenes.flatMap((s) => s.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken })));- const aligned = alignWords(script, heard.words, duration);- speech = { matchRate: Math.round(aligned.matchRate * 1000) / 1000, transcript: heard.text };- if (aligned.matchRate < 0.85) issues.push({ level: 'warn', message: `only ${Math.round(aligned.matchRate * 100)}% of the script was understood in the final mix; lower music.volume or check pronunciation` });+ const tlFile = path.join(project.paths.build, 'timeline.json');+ const timeline = fs.existsSync(tlFile) ? JSON.parse(fs.readFileSync(tlFile, 'utf8')) : null;+ const spokenWords = (sc) => sc.units.filter((u) => u.type === 'word').map((u) => ({ spoken: u.spoken }));+ const byId = new Map(project.script.scenes.map((sc) => [sc.id, sc]));+ const sameScenes = timeline && timeline.scenes.length === project.script.scenes.length && timeline.scenes.every((t) => byId.has(t.id)) && Math.abs(timeline.duration - duration) < 1;+ if (sameScenes) {+ const texts = [];+ const scenes = [];+ let total = 0;+ let matched = 0;+ for (const t of timeline.scenes) {+ const words = spokenWords(byId.get(t.id));+ if (!words.length) continue;+ const from = Math.max(0, Math.floor(t.start * 16000));+ const to = Math.min(samples.length, Math.ceil((t.start + t.dur) * 16000));+ const heard = await transcribeWords(samples.subarray(from, to), { log });+ texts.push(heard.text);+ const rate = alignWords(words, heard.words, t.dur).matchRate;+ scenes.push({ id: t.id, matchRate: Math.round(rate * 1000) / 1000 });+ total += words.length;+ matched += rate * words.length;+ if (rate < 0.8) issues.push({ level: 'warn', message: `scene "${t.id}": only ${Math.round(rate * 100)}% of the narration was understood in the final mix; it heard "${heard.text.slice(0, 120)}"` });+ }+ const rate = total ? matched / total : 1;+ speech = { matchRate: Math.round(rate * 1000) / 1000, transcript: texts.join(' '), scenes };+ } else {+ const heard = await transcribeWords(samples, { log });+ const script = project.script.scenes.flatMap(spokenWords);+ speech = { matchRate: Math.round(alignWords(script, heard.words, duration).matchRate * 1000) / 1000, transcript: heard.text };+ }+ if (speech.matchRate < 0.85) issues.push({ level: 'warn', message: `only ${Math.round(speech.matchRate * 100)}% of the script was understood in the final mix; lower music.volume or check pronunciation` });fs.rmSync(wav, { force: true });}const report = {