Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.
JavaScript ai-agentsanimationcanvasclaude-codecodexexplainer-videoffmpegjavascriptkokoroproduct-demotext-to-speechvideo-generationwhisper
Match spoken numbers, amounts and percentages in the speech check, time chart bars on words, accept image references from the current folder
Vincent Schmalbach6409e359845d
diff --git a/AGENTS.md b/AGENTS.mdindex bc271d5..1c54b17 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -463,9 +463,10 @@ s.pie([{ label: 'Images', value: 55 }, { label: 'Other', value: 45 }], { donut:- `s.number(value, options)` counts up with ticks: `from`, `dur` (1.4),`prefix`, `suffix`, `decimals`, `separator`, `group: false`, plus textoptions.-- `s.bars(data, options)`: `x`, `y`, `w`, `h`, `max`, `stagger`, `growDur`,- `values: false`, `prefix`, `suffix`, `format(v)`, `labelSize`,- `valueSize`, `fillStyle`.+- `s.bars(data, options)`: items are `{ label, value, color, at }`, and a bar+ with its own `at` grows when that word is spoken. Options: `x`, `y`, `w`,+ `h`, `max`, `stagger`, `growDur`, `values: false`, `prefix`, `suffix`,+ `format(v)`, `labelSize`, `valueSize`, `fillStyle`.- `s.lineChart(values, options)`: `x`, `y`, `w`, `h`, `min`, `max`,`labels`, `dots: false`, `area: false`, `color`, `dur`.- `s.pie(data, options)`: `x`, `y`, `r`, `donut` (0 to 0.8), `labels: false`,diff --git a/engine/stage.js b/engine/stage.jsindex f97d00d..8c3ff76 100644--- a/engine/stage.js+++ b/engine/stage.js@@ -1061,7 +1061,9 @@ export class Stage {const at = this.time(o.at ?? 0);const stagger = o.stagger ?? 0.22;this._register(o.id, x, y, w, h);- items.forEach((d, i) => this._record('chart', at + 0.3 + i * stagger, 0.8, { sfx: o.sfx }, { label: `bar${i}` }));+ // Each bar can have its own `at` (seconds, a spoken word or "#marker").+ const barAt = items.map((d, i) => (d.at !== undefined ? this.time(d.at) : at + 0.3 + i * stagger));+ items.forEach((d, i) => this._record('chart', barAt[i], 0.8, { sfx: o.sfx }, { label: `bar${i}` }));const axisLife = this._life({ at, out: o.out, sfx: null }, 'chart', { enter: 'draw', dur: 0.5 });if (!axisLife) return geom(x, y, w, h);const alpha = axisLife.q > 0 ? 1 - ease.in(axisLife.q) : 1;@@ -1069,7 +1071,7 @@ export class Stage {ctx.globalAlpha *= alpha;this.pen.draw(ctx, { kind: 'poly', points: [[x - w / 2 - 20, base], [x + w / 2 + 20, base]] }, { stroke: colorOf(th, 'ink'), width: th.stroke.width, seed: this._seed('axis', o), single: true }, { stroke: ease.inOut(axisLife.p), fill: 1 });items.forEach((d, i) => {- const t0 = at + 0.3 + i * stagger;+ const t0 = barAt[i];const g = ease.out(clamp((this.t - t0) / (o.growDur ?? 0.9)));const cx = x - w / 2 + slot * i + slot / 2;const bh = (d.value / max) * h;diff --git a/src/align.js b/src/align.jsindex 46f70b1..e6cadd2 100644--- a/src/align.js+++ b/src/align.js@@ -14,12 +14,43 @@ function canon(w) {return NUMBER_WORDS[n] ?? n;}-// "example.com" and "1.1.1.1" are spoken as words joined by "dot"; Whisper-// writes them with dots. Expand both sides the same way.-function expand(word) {- const n = canon(word);+const ONES = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];+const TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];+const SCALES = [[1e9, 'billion'], [1e6, 'million'], [1e3, 'thousand']];+const CURRENCY = { $: 'dollars', '€': 'euros', '£': 'pounds' };++// 1102 -> one thousand one hundred two (without "and"; "and" is optional below).+export function intWords(n) {+ if (!Number.isFinite(n) || n < 0 || n >= 1e12) return [String(n)];+ if (n < 20) return [ONES[n]];+ if (n < 100) return [TENS[Math.floor(n / 10)], ...(n % 10 ? [ONES[n % 10]] : [])];+ if (n < 1000) return [ONES[Math.floor(n / 100)], 'hundred', ...(n % 100 ? intWords(n % 100) : [])];+ for (const [size, name] of SCALES) {+ if (n >= size) return [...intWords(Math.floor(n / size)), name, ...(n % size ? intWords(n % size) : [])];+ }+ return [String(n)];+}++// Turns one written word into the tokens a voice would say, in canonical form.+// Whisper writes "$1,050" and "5%" where the voice said "one thousand and+// fifty dollars" and "five percent", and "example.com" or "1.1.1.1" where it+// said words joined by "dot". Both sides are expanded the same way.+export function expand(word) {+ const raw = String(word).trim().toLowerCase();// Whisper splits "1.1.1.1" into "1", ".1", ".1", ".1": a leading dot is a spoken "dot".- const lead = /^\s*\.[a-z0-9]/i.test(String(word)) ? ['dot'] : [];+ const lead = /^\.[a-z0-9]/i.test(raw) ? ['dot'] : [];+ const n = canon(word);+ const money = /^([$€£])?(\d+)(?:\.(\d+))?(%)?$/.exec(n);+ if (money && /\d/.test(raw) && !/^\d+\.\d+\.\d+/.test(n)) {+ const [, cur, int, dec, pct] = money;+ const out = [...lead, ...intWords(Number(int))];+ if (dec !== undefined) {+ if (cur) out.push(CURRENCY[cur], ...intWords(Number(dec)), 'cents');+ else out.push('point', ...dec.split('').map((d) => ONES[Number(d)]));+ } else if (cur) out.push(CURRENCY[cur]);+ if (pct) out.push('percent');+ return out.map(canon);+ }if (!/[a-z0-9]\.[a-z0-9]/.test(n)) return n ? [...lead, n] : lead;const parts = n.split('.').filter(Boolean).map((p) => NUMBER_WORDS[p] ?? p);const out = [...lead];@@ -30,6 +61,8 @@ function expand(word) {return out;}+const isNumberToken = (t) => /^\d+$/.test(t) || ['hundred', 'thousand', 'million', 'billion', '100', '1000'].includes(t);+function editDistance(a, b) {if (a === b) return 0;const m = a.length;@@ -61,10 +94,29 @@ export function alignWords(words, heard, duration) {const tokens = [];words.forEach((w, wi) => {const parts = String(w.spoken).split(/\s+/).flatMap(expand);- (parts.length ? parts : ['']).forEach((p, k) => tokens.push({ wi, norm: p, first: k === 0, last: k === parts.length - 1 || !parts.length }));+ (parts.length ? parts : ['']).forEach((p, k) => {+ // "and" inside a spoken number ("one thousand and fifty") is optional.+ const prev = tokens[tokens.length - 1];+ const optional = p === 'and' && prev && isNumberToken(prev.norm);+ tokens.push({ wi, norm: p, first: k === 0, last: k === parts.length - 1 || !parts.length, optional });+ });});- const H = [];+ // Whisper splits numbers into pieces ("$1", ",000" and "5", "%" and "1",+ // ".1"). Join them back before comparing.+ const joined = [];for (const h of heard) {+ const prev = joined[joined.length - 1];+ const t = String(h.text).trim();+ const glue = prev && ((/^[,.]\d/.test(t) && /\d$/.test(prev.text)) || (/^%/.test(t) && /\d$/.test(prev.text)) || (/^\d/.test(t) && /^[$€£]$/.test(prev.text)));+ if (glue) {+ prev.text += t;+ prev.end = h.end;+ } else {+ joined.push({ ...h, text: t });+ }+ }+ const H = [];+ for (const h of joined) {const parts = expand(h.text);const step = (h.end - h.start) / Math.max(1, parts.length);parts.forEach((p, k) => H.push({ text: p, norm: p, start: h.start + k * step, end: h.start + (k + 1) * step }));@@ -89,7 +141,7 @@ export function alignWords(words, heard, duration) {for (let j = 0; j <= N; j++) {const c = cost[i][j];if (c >= INF) continue;- if (i < T && c + 1 < cost[i + 1][j]) { cost[i + 1][j] = c + 1; move[i + 1][j] = 2; }+ if (i < T && c + (tokens[i].optional ? 0 : 1) < cost[i + 1][j]) { cost[i + 1][j] = c + (tokens[i].optional ? 0 : 1); move[i + 1][j] = 2; }if (j < N && c + 1 < cost[i][j + 1]) { cost[i][j + 1] = c + 1; move[i][j + 1] = 3; }if (i < T && j < N) {const ok = similar(tokens[i].norm, H[j].norm);@@ -111,7 +163,7 @@ export function alignWords(words, heard, duration) {const spans = words.map(() => null);const hits = words.map(() => 0);const need = words.map(() => 0);- for (const t of tokens) need[t.wi]++;+ for (const t of tokens) if (!t.optional) need[t.wi]++;const add = (wi, s, e, n = 1) => {hits[wi] += n;const cur = spans[wi];@@ -170,5 +222,6 @@ export function alignWords(words, heard, duration) {if (out[q].start < out[q - 1].start) out[q].start = out[q - 1].start;if (out[q].end < out[q].start) out[q].end = out[q].start + 0.05;}- return { words: out, matchRate: T ? matched / T : 1 };+ const counted = tokens.filter((t) => !t.optional).length;+ return { words: out, matchRate: counted ? Math.min(1, matched / counted) : 1 };}diff --git a/src/images.js b/src/images.jsindex 3c508cb..0885578 100644--- a/src/images.js+++ b/src/images.js@@ -60,8 +60,9 @@ export async function generateImage(project, { name, prompt, model, aspect, refsconst text = `${prompt.trim()}\n\nStyle: ${styleText}\nAspect ratio: ${ratio}.`.trim();const content = [{ type: 'text', text }];for (const r of refs) {- const file = path.resolve(project.dir, r);- if (!fs.existsSync(file)) throw new ProjectError(`reference image ${r} does not exist`);+ // Accept paths relative to the project (assets/a.png) or to the current folder.+ const file = [path.resolve(project.dir, r), path.resolve(r)].find((f) => fs.existsSync(f));+ if (!file) throw new ProjectError(`reference image ${r} does not exist (use a path like assets/${path.basename(r)})`);content.push({ type: 'image_url', image_url: { url: toDataUrl(file) } });}const body = {diff --git a/src/voice.js b/src/voice.jsindex a079768..4f5bdc7 100644--- a/src/voice.js+++ b/src/voice.js@@ -9,7 +9,7 @@ import { alignWords } from './align.js';import { loadTTS, transcribeWords, TTS_MODEL, ASR_MODEL, TTS_SAMPLE_RATE, ttsDtype } from './models.js';import { writeWav, readWav, resample, trimSilence } from './wav.js';-const PIPELINE = 5;+const PIPELINE = 7;function sceneHash(scene, chunks, config) {return cryptodiff --git a/test/align.test.js b/test/align.test.jsindex b7c43f2..26fe81c 100644--- a/test/align.test.js+++ b/test/align.test.js@@ -43,3 +43,23 @@ test('split number tokens with leading dots still match', () => {const r = alignWords([{ spoken: 'one dot one dot one dot one' }], heard([['1', 0, 0.3], ['.1', 0.3, 0.6], ['.1', 0.6, 0.9], ['.1', 0.9, 1.2]]), 1.3);assert.equal(r.matchRate, 1);});++test('spoken amounts match digits, dollars and percent', () => {+ const r = alignWords(+ [{ spoken: 'one thousand and fifty dollars.' }, { spoken: 'at' }, { spoken: 'five percent' }, { spoken: 'one thousand one hundred and two dollars fifty' }],+ heard([['$1,050.', 0, 1], ['at', 1, 1.2], ['5%', 1.2, 1.6], ['$1,102.50', 1.6, 3]]),+ 3,+ );+ assert.equal(r.matchRate, 1);+ assert.ok(r.words.every((w) => w.matched));+});++test('digits in the script match digits Whisper wrote', () => {+ const r = alignWords([{ spoken: 'After' }, { spoken: '10' }, { spoken: 'years' }], heard([['After', 0, 0.3], ['10', 0.3, 0.6], ['years', 0.6, 1]]), 1);+ assert.equal(r.matchRate, 1);+});++test('number pieces from Whisper are joined', () => {+ const r = alignWords([{ spoken: 'one thousand dollars' }, { spoken: 'five percent' }], heard([['$1', 0, 0.4], [',000', 0.4, 0.8], ['5', 0.8, 1], ['%', 1, 1.2]]), 1.2);+ assert.equal(r.matchRate, 1);+});