Skip to content

Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.

Match spoken numbers, amounts and percentages in the speech check, time chart bars on words, accept image references from the current folder

Vincent Schmalbach6409e359845d
diff --git a/AGENTS.md b/AGENTS.md
index bc271d5..1c54b17 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -463,9 +463,10 @@ s.pie([{ label: 'Images', value: 55 }, { label: 'Other', value: 45 }], { donut:
- `s.number(value, options)` counts up with ticks: `from`, `dur` (1.4),
`prefix`, `suffix`, `decimals`, `separator`, `group: false`, plus text
options.
-- `s.bars(data, options)`: `x`, `y`, `w`, `h`, `max`, `stagger`, `growDur`,
- `values: false`, `prefix`, `suffix`, `format(v)`, `labelSize`,
- `valueSize`, `fillStyle`.
+- `s.bars(data, options)`: items are `{ label, value, color, at }`, and a bar
+ with its own `at` grows when that word is spoken. Options: `x`, `y`, `w`,
+ `h`, `max`, `stagger`, `growDur`, `values: false`, `prefix`, `suffix`,
+ `format(v)`, `labelSize`, `valueSize`, `fillStyle`.
- `s.lineChart(values, options)`: `x`, `y`, `w`, `h`, `min`, `max`,
`labels`, `dots: false`, `area: false`, `color`, `dur`.
- `s.pie(data, options)`: `x`, `y`, `r`, `donut` (0 to 0.8), `labels: false`,
diff --git a/engine/stage.js b/engine/stage.js
index f97d00d..8c3ff76 100644
--- a/engine/stage.js
+++ b/engine/stage.js
@@ -1061,7 +1061,9 @@ export class Stage {
const at = this.time(o.at ?? 0);
const stagger = o.stagger ?? 0.22;
this._register(o.id, x, y, w, h);
- items.forEach((d, i) => this._record('chart', at + 0.3 + i * stagger, 0.8, { sfx: o.sfx }, { label: `bar${i}` }));
+ // Each bar can have its own `at` (seconds, a spoken word or "#marker").
+ const barAt = items.map((d, i) => (d.at !== undefined ? this.time(d.at) : at + 0.3 + i * stagger));
+ items.forEach((d, i) => this._record('chart', barAt[i], 0.8, { sfx: o.sfx }, { label: `bar${i}` }));
const axisLife = this._life({ at, out: o.out, sfx: null }, 'chart', { enter: 'draw', dur: 0.5 });
if (!axisLife) return geom(x, y, w, h);
const alpha = axisLife.q > 0 ? 1 - ease.in(axisLife.q) : 1;
@@ -1069,7 +1071,7 @@ export class Stage {
ctx.globalAlpha *= alpha;
this.pen.draw(ctx, { kind: 'poly', points: [[x - w / 2 - 20, base], [x + w / 2 + 20, base]] }, { stroke: colorOf(th, 'ink'), width: th.stroke.width, seed: this._seed('axis', o), single: true }, { stroke: ease.inOut(axisLife.p), fill: 1 });
items.forEach((d, i) => {
- const t0 = at + 0.3 + i * stagger;
+ const t0 = barAt[i];
const g = ease.out(clamp((this.t - t0) / (o.growDur ?? 0.9)));
const cx = x - w / 2 + slot * i + slot / 2;
const bh = (d.value / max) * h;
diff --git a/src/align.js b/src/align.js
index 46f70b1..e6cadd2 100644
--- a/src/align.js
+++ b/src/align.js
@@ -14,12 +14,43 @@ function canon(w) {
return NUMBER_WORDS[n] ?? n;
}
-// "example.com" and "1.1.1.1" are spoken as words joined by "dot"; Whisper
-// writes them with dots. Expand both sides the same way.
-function expand(word) {
- const n = canon(word);
+const ONES = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];
+const TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];
+const SCALES = [[1e9, 'billion'], [1e6, 'million'], [1e3, 'thousand']];
+const CURRENCY = { $: 'dollars', '€': 'euros', '£': 'pounds' };
+
+// 1102 -> one thousand one hundred two (without "and"; "and" is optional below).
+export function intWords(n) {
+ if (!Number.isFinite(n) || n < 0 || n >= 1e12) return [String(n)];
+ if (n < 20) return [ONES[n]];
+ if (n < 100) return [TENS[Math.floor(n / 10)], ...(n % 10 ? [ONES[n % 10]] : [])];
+ if (n < 1000) return [ONES[Math.floor(n / 100)], 'hundred', ...(n % 100 ? intWords(n % 100) : [])];
+ for (const [size, name] of SCALES) {
+ if (n >= size) return [...intWords(Math.floor(n / size)), name, ...(n % size ? intWords(n % size) : [])];
+ }
+ return [String(n)];
+}
+
+// Turns one written word into the tokens a voice would say, in canonical form.
+// Whisper writes "$1,050" and "5%" where the voice said "one thousand and
+// fifty dollars" and "five percent", and "example.com" or "1.1.1.1" where it
+// said words joined by "dot". Both sides are expanded the same way.
+export function expand(word) {
+ const raw = String(word).trim().toLowerCase();
// Whisper splits "1.1.1.1" into "1", ".1", ".1", ".1": a leading dot is a spoken "dot".
- const lead = /^\s*\.[a-z0-9]/i.test(String(word)) ? ['dot'] : [];
+ const lead = /^\.[a-z0-9]/i.test(raw) ? ['dot'] : [];
+ const n = canon(word);
+ const money = /^([$€£])?(\d+)(?:\.(\d+))?(%)?$/.exec(n);
+ if (money && /\d/.test(raw) && !/^\d+\.\d+\.\d+/.test(n)) {
+ const [, cur, int, dec, pct] = money;
+ const out = [...lead, ...intWords(Number(int))];
+ if (dec !== undefined) {
+ if (cur) out.push(CURRENCY[cur], ...intWords(Number(dec)), 'cents');
+ else out.push('point', ...dec.split('').map((d) => ONES[Number(d)]));
+ } else if (cur) out.push(CURRENCY[cur]);
+ if (pct) out.push('percent');
+ return out.map(canon);
+ }
if (!/[a-z0-9]\.[a-z0-9]/.test(n)) return n ? [...lead, n] : lead;
const parts = n.split('.').filter(Boolean).map((p) => NUMBER_WORDS[p] ?? p);
const out = [...lead];
@@ -30,6 +61,8 @@ function expand(word) {
return out;
}
+const isNumberToken = (t) => /^\d+$/.test(t) || ['hundred', 'thousand', 'million', 'billion', '100', '1000'].includes(t);
+
function editDistance(a, b) {
if (a === b) return 0;
const m = a.length;
@@ -61,10 +94,29 @@ export function alignWords(words, heard, duration) {
const tokens = [];
words.forEach((w, wi) => {
const parts = String(w.spoken).split(/\s+/).flatMap(expand);
- (parts.length ? parts : ['']).forEach((p, k) => tokens.push({ wi, norm: p, first: k === 0, last: k === parts.length - 1 || !parts.length }));
+ (parts.length ? parts : ['']).forEach((p, k) => {
+ // "and" inside a spoken number ("one thousand and fifty") is optional.
+ const prev = tokens[tokens.length - 1];
+ const optional = p === 'and' && prev && isNumberToken(prev.norm);
+ tokens.push({ wi, norm: p, first: k === 0, last: k === parts.length - 1 || !parts.length, optional });
+ });
});
- const H = [];
+ // Whisper splits numbers into pieces ("$1", ",000" and "5", "%" and "1",
+ // ".1"). Join them back before comparing.
+ const joined = [];
for (const h of heard) {
+ const prev = joined[joined.length - 1];
+ const t = String(h.text).trim();
+ const glue = prev && ((/^[,.]\d/.test(t) && /\d$/.test(prev.text)) || (/^%/.test(t) && /\d$/.test(prev.text)) || (/^\d/.test(t) && /^[$€£]$/.test(prev.text)));
+ if (glue) {
+ prev.text += t;
+ prev.end = h.end;
+ } else {
+ joined.push({ ...h, text: t });
+ }
+ }
+ const H = [];
+ for (const h of joined) {
const parts = expand(h.text);
const step = (h.end - h.start) / Math.max(1, parts.length);
parts.forEach((p, k) => H.push({ text: p, norm: p, start: h.start + k * step, end: h.start + (k + 1) * step }));
@@ -89,7 +141,7 @@ export function alignWords(words, heard, duration) {
for (let j = 0; j <= N; j++) {
const c = cost[i][j];
if (c >= INF) continue;
- if (i < T && c + 1 < cost[i + 1][j]) { cost[i + 1][j] = c + 1; move[i + 1][j] = 2; }
+ if (i < T && c + (tokens[i].optional ? 0 : 1) < cost[i + 1][j]) { cost[i + 1][j] = c + (tokens[i].optional ? 0 : 1); move[i + 1][j] = 2; }
if (j < N && c + 1 < cost[i][j + 1]) { cost[i][j + 1] = c + 1; move[i][j + 1] = 3; }
if (i < T && j < N) {
const ok = similar(tokens[i].norm, H[j].norm);
@@ -111,7 +163,7 @@ export function alignWords(words, heard, duration) {
const spans = words.map(() => null);
const hits = words.map(() => 0);
const need = words.map(() => 0);
- for (const t of tokens) need[t.wi]++;
+ for (const t of tokens) if (!t.optional) need[t.wi]++;
const add = (wi, s, e, n = 1) => {
hits[wi] += n;
const cur = spans[wi];
@@ -170,5 +222,6 @@ export function alignWords(words, heard, duration) {
if (out[q].start < out[q - 1].start) out[q].start = out[q - 1].start;
if (out[q].end < out[q].start) out[q].end = out[q].start + 0.05;
}
- return { words: out, matchRate: T ? matched / T : 1 };
+ const counted = tokens.filter((t) => !t.optional).length;
+ return { words: out, matchRate: counted ? Math.min(1, matched / counted) : 1 };
}
diff --git a/src/images.js b/src/images.js
index 3c508cb..0885578 100644
--- a/src/images.js
+++ b/src/images.js
@@ -60,8 +60,9 @@ export async function generateImage(project, { name, prompt, model, aspect, refs
const text = `${prompt.trim()}\n\nStyle: ${styleText}\nAspect ratio: ${ratio}.`.trim();
const content = [{ type: 'text', text }];
for (const r of refs) {
- const file = path.resolve(project.dir, r);
- if (!fs.existsSync(file)) throw new ProjectError(`reference image ${r} does not exist`);
+ // Accept paths relative to the project (assets/a.png) or to the current folder.
+ const file = [path.resolve(project.dir, r), path.resolve(r)].find((f) => fs.existsSync(f));
+ if (!file) throw new ProjectError(`reference image ${r} does not exist (use a path like assets/${path.basename(r)})`);
content.push({ type: 'image_url', image_url: { url: toDataUrl(file) } });
}
const body = {
diff --git a/src/voice.js b/src/voice.js
index a079768..4f5bdc7 100644
--- a/src/voice.js
+++ b/src/voice.js
@@ -9,7 +9,7 @@ import { alignWords } from './align.js';
import { loadTTS, transcribeWords, TTS_MODEL, ASR_MODEL, TTS_SAMPLE_RATE, ttsDtype } from './models.js';
import { writeWav, readWav, resample, trimSilence } from './wav.js';
-const PIPELINE = 5;
+const PIPELINE = 7;
function sceneHash(scene, chunks, config) {
return crypto
diff --git a/test/align.test.js b/test/align.test.js
index b7c43f2..26fe81c 100644
--- a/test/align.test.js
+++ b/test/align.test.js
@@ -43,3 +43,23 @@ test('split number tokens with leading dots still match', () => {
const r = alignWords([{ spoken: 'one dot one dot one dot one' }], heard([['1', 0, 0.3], ['.1', 0.3, 0.6], ['.1', 0.6, 0.9], ['.1', 0.9, 1.2]]), 1.3);
assert.equal(r.matchRate, 1);
});
+
+test('spoken amounts match digits, dollars and percent', () => {
+ const r = alignWords(
+ [{ spoken: 'one thousand and fifty dollars.' }, { spoken: 'at' }, { spoken: 'five percent' }, { spoken: 'one thousand one hundred and two dollars fifty' }],
+ heard([['$1,050.', 0, 1], ['at', 1, 1.2], ['5%', 1.2, 1.6], ['$1,102.50', 1.6, 3]]),
+ 3,
+ );
+ assert.equal(r.matchRate, 1);
+ assert.ok(r.words.every((w) => w.matched));
+});
+
+test('digits in the script match digits Whisper wrote', () => {
+ const r = alignWords([{ spoken: 'After' }, { spoken: '10' }, { spoken: 'years' }], heard([['After', 0, 0.3], ['10', 0.3, 0.6], ['years', 0.6, 1]]), 1);
+ assert.equal(r.matchRate, 1);
+});
+
+test('number pieces from Whisper are joined', () => {
+ const r = alignWords([{ spoken: 'one thousand dollars' }, { spoken: 'five percent' }], heard([['$1', 0, 0.4], [',000', 0.4, 0.8], ['5', 0.8, 1], ['%', 1, 1.2]]), 1.2);
+ assert.equal(r.matchRate, 1);
+});