Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.
JavaScript ai-agentsanimationcanvasclaude-codecodexexplainer-videoffmpegjavascriptkokoroproduct-demotext-to-speechvideo-generationwhisper
Japanese: decimals, line breaks and numbers in the speech check (#28)
Vincent Schmalbach618c385da43d
diff --git a/AGENTS.md b/AGENTS.mdindex c23fe3b..5aa7e70 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -88,7 +88,9 @@ When you ask, put everything you need in one message. Typical reasons to ask:check confirmed. If a word is not confirmed, the voice probably said itwrong. Fix it with `{shown|spoken}` in the script and run the commandagain. For a recording, make script.md say what was actually said.- Only the scenes you changed are made again.+ Only the scenes you changed are made again. `check`, `still`, `sheet`,+ `render` and `preview` also make missing narration, so start them after+ `voice` has finished, not at the same time.5. **Plan the pictures.** For each scene, decide what is on screen when eachmarker or important word is spoken. Decide where each thing sits and what@@ -175,7 +177,8 @@ something to a friend at a table.Length and pace:- Aim for 45 to 120 seconds unless the user says otherwise. A finished video- has about 150 words a minute, pauses included.+ has about 150 words a minute, pauses included. In Japanese it is about 270+ characters a minute, and in Chinese about 200.- Start with the question or problem. Explain it in small steps. Give oneconcrete example, then sum up.- One idea per scene, usually 1 to 3 sentences (4 to 15 seconds).@@ -362,7 +365,12 @@ match whole words. Put a `[#marker]` where you need a moment inside a word.Sentences end with `。`, `!` or `?`.Latin words such as API or JavaScript are read in English. For a Japanese-reading, write it out: `{API|エーピーアイ}`.+reading, write it out: `{API|エーピーアイ}`. Numbers can stay in digits,+including decimals like 365.2422. The Japanese reader sometimes picks the+wrong reading for a word, such as ついたち for 一日 (write `1日` for+いちにち) or こよみ for 暦 in グレゴリオ暦 (write `{グレゴリオ暦|グレゴリオれき}`).+When the speech check flags a word, compare what it heard with the script+before you change anything.Whisper often writes a word with other characters that sound the same, sothe speech check compares sounds. Japanese is compared by its kana reading,@@ -683,8 +691,9 @@ same. `render --out` still sets the file name you choose.## Pace-`pace` makes the whole video quicker or slower. At 1.3 the voice speaks 30%-faster. The pauses, the time before and after each scene, the transitions and+`pace` makes the whole video quicker or slower. At 1.3 the voice speaks about+30% faster. The voice model does not speed up exactly, so check the length+`voice` reports. The pauses, the time before and after each scene, the transitions andevery animation also get 30% shorter. Word and marker cues follow the voice.So if you tie pictures to words, the rhythm stays right at any pace. Times inseconds are real seconds of the scene, like `s.t` and `s.cue()`. Sodiff --git a/engine/captions.js b/engine/captions.jsindex b635580..0a8d046 100644--- a/engine/captions.js+++ b/engine/captions.js@@ -3,6 +3,26 @@import { fontString } from './themes.js';import { clamp, ease } from './util.js';+// Some words belong to the word before them, so a phrase or line should not+// start with one: Japanese particles and verb endings (short hiragana pieces+// such as は, ごと, せん, or ones starting like ません), a one-kanji suffix+// such as the 暦 of グレゴリオ暦, Chinese particles such as 的, and counters+// after a number (4年, 365天).+const JAPANESE_ENDINGS = /^(ませ|まし|でし|です|ます|しょう|られ|させ|され|なかっ|ながら|ような|ように|という|として|について|によって|くらい|ぐらい)/;+export function clingsToPrevious(language) {+ const counter = (text, before) => /\d$/.test(before) && /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(text);+ if (language === 'ja') {+ return (text, before = '') => {+ const word = text.replace(/[、。!?「」『』()]+$/, '');+ const ending = /^[\u3041-\u3096]+$/.test(word) && (word.length <= 2 || JAPANESE_ENDINGS.test(word));+ const suffix = /^\p{Script=Han}$/u.test(word) && /[\p{Script=Han}\p{Script=Katakana}]$/u.test(before);+ return ending || suffix || counter(text, before);+ };+ }+ if (language === 'zh') return (text, before = '') => /^[的了吗呢吧啊着过得地们][,。!?、]*$/.test(text) || counter(text, before);+ return () => false;+}+export function buildPhrases(timeline, maxChars) {const cjk = ['ja', 'zh'].includes(timeline.language);const sentenceEnd = cjk ? /[.!?;:।。!?]["'’)」』)]*$/ : /[.!?;:।]["'’)]*$/;@@ -13,7 +33,7 @@ export function buildPhrases(timeline, maxChars) {// phrases as fit, balanced in length, so no phrase is one lonely word.let run = [];const flush = () => {- for (const words of balancePhrases(run, maxChars, cjk ? '' : ' ')) phrases.push({ words, ...(cjk ? { separator: '' } : {}), start: words[0].start, end: words[words.length - 1].end });+ for (const words of balancePhrases(run, maxChars, cjk ? '' : ' ', clingsToPrevious(timeline.language))) phrases.push({ words, ...(cjk ? { separator: '' } : {}), start: words[0].start, end: words[words.length - 1].end });run = [];};for (const w of sc.words) {@@ -32,8 +52,9 @@ export function buildPhrases(timeline, maxChars) {}// Splits words into the fewest phrases of at most maxChars characters, with-// lengths as even as possible and breaks after commas where they fit.-export function balancePhrases(words, maxChars, separator = ' ') {+// lengths as even as possible and breaks after commas where they fit, never+// before a word that clings to the one before it if that can be helped.+export function balancePhrases(words, maxChars, separator = ' ', clings = () => false) {if (!words.length) return [];const len = (a, b) => words.slice(a, b).reduce((n, w) => n + w.text.length + separator.length, -separator.length);if (len(0, words.length) <= maxChars) return [words];@@ -47,7 +68,8 @@ export function balancePhrases(words, maxChars, separator = ' ') {if (l > maxChars && b - a > 1) break;if (!best[a]) continue;const comma = b < n && punctuation.test(words[b - 1].text) ? 150 : 0;- const c = { parts: best[a].parts + 1, cost: best[a].cost + (maxChars - l) ** 2 - comma, from: a };+ const glued = b < n && !comma && clings(words[b].text, words[b - 1].text) ? 250 : 0;+ const c = { parts: best[a].parts + 1, cost: best[a].cost + (maxChars - l) ** 2 - comma + glued, from: a };if (!best[b] || c.parts < best[b].parts || (c.parts === best[b].parts && c.cost < best[b].cost)) best[b] = c;}}diff --git a/src/g2p.py b/src/g2p.pyindex dbf21b8..477b728 100644--- a/src/g2p.py+++ b/src/g2p.py@@ -9,16 +9,46 @@ sys.stderr.reconfigure(encoding='utf-8')mode = sys.argv[1]+DIGITS = ['ゼロ', 'イチ', 'ニ', 'サン', 'ヨン', 'ゴ', 'ロク', 'ナナ', 'ハチ', 'キュー']+++def below_10000(n):+ th, h, t, o = n // 1000, n // 100 % 10, n // 10 % 10, n % 10+ out = {1: 'セン', 3: 'サンゼン', 8: 'ハッセン'}.get(th, DIGITS[th] + 'セン') if th else ''+ out += {1: 'ヒャク', 3: 'サンビャク', 6: 'ロッピャク', 8: 'ハッピャク'}.get(h, DIGITS[h] + 'ヒャク') if h else ''+ out += ('' if t == 1 else DIGITS[t]) + 'ジュー' if t else ''+ return out + (DIGITS[o] if o else '')+++def number(digits):+ # UniDic leaves digits unread. Whisper may write a number in digits or+ # kana, so both sides get the same reading.+ n = int(digits)+ if n == 0 or n >= 10 ** 8 or digits[0] == '0':+ return ''.join(DIGITS[int(d)] for d in digits)+ man, rest = divmod(n, 10000)+ return (below_10000(man) + 'マン' if man else '') + below_10000(rest)++def readings(tagger, text):# [start, end, katakana reading] for each word, in characters.- out, pos = [], 0+ out, pos, decimal = [], 0, Falsefor word in tagger(text):start = text.find(word.surface, pos)if start < 0:continuepos = start + len(word.surface)pron = getattr(word.feature, 'pron', None)- out.append([start, pos, pron if pron and pron != '*' else word.surface])+ surface = word.surface+ if surface.isascii() and surface.isdigit():+ # After a decimal point, digits are read one by one.+ reading = ''.join(DIGITS[int(d)] for d in surface) if decimal else number(surface)+ elif surface == '.' and out and text[start - 1:start].isdigit() and text[pos:pos + 1].isdigit():+ reading = 'テン'+ else:+ reading = pron if pron and pron != '*' else surface+ decimal = reading == 'テン' or (decimal and surface.isdigit())+ out.append([start, pos, reading])return outdiff --git a/src/phonemes.js b/src/phonemes.jsindex d4e0a3b..564355f 100644--- a/src/phonemes.js+++ b/src/phonemes.js@@ -96,7 +96,14 @@ async function englishPhonemes(text) {return (await espeakWasm(text, 'en-us')).join(' ').replace(/ʲ/g, 'j').replace(/r/g, 'ɹ').replace(/x/g, 'k').replace(/ɬ/g, 'l').trim();}+// misaki reads "365.2422" as 365, a pause and 2422. In Japanese the digits+// after the point are read one by one: 365点二四二二.+function japaneseDecimals(text) {+ return text.replace(/(\d)\.(\d+)/g, (m, whole, digits) => whole + '点' + Array.from(digits, d => '〇一二三四五六七八九'[d]).join(''));+}+async function misakiPhonemes(texts, language) {+ if (language === 'ja') texts = texts.map(japaneseDecimals);const parts = texts.map(text => {const list = [];let last = 0;diff --git a/src/subtitles.js b/src/subtitles.jsindex e63fd57..d441503 100644--- a/src/subtitles.js+++ b/src/subtitles.js@@ -1,9 +1,12 @@// Subtitle cues use the shown words and their times from the voice timeline.+import { clingsToPrevious } from '../engine/captions.js';+const sentenceEnd = /[.!?…।]["'’”)\]]*$/;const punctuation = /[,;:.!?…।]["'’”)\]]*$/;function lines(words, language = 'en') {const cjk = ['ja', 'zh'].includes(language);+ const clings = clingsToPrevious(language);const limit = cjk ? 18 : 42;const separator = cjk ? '' : ' ';const text = words.map((w) => w.text).join(separator);@@ -15,7 +18,8 @@ function lines(words, language = 'en') {if (a.length > limit || b.length > limit) continue;// Break after punctuation, unless that leaves a word or two alone on a line.const atPunctuation = (cjk ? /[,;:.!?…।。!?、,;:]["'’”)\]」』)]*$/ : punctuation).test(words[i - 1].text) && Math.min(a.length, b.length) >= (cjk ? 5 : 12);- const score = (atPunctuation ? 30 : 0) - Math.abs(a.length - b.length);+ const glued = !atPunctuation && clings(words[i].text, words[i - 1].text) ? 40 : 0;+ const score = (atPunctuation ? 30 : 0) - Math.abs(a.length - b.length) - glued;if (!best || score > best.score) best = { score, text: `${a}\n${b}` };}return best?.text ?? null;@@ -27,6 +31,7 @@ export function subtitleCues(timeline, { from = 0, to = Infinity } = {}) {const limit = cjk ? 18 : 42;const endSentence = cjk ? /[.!?…।。!?]["'’”)\]」』)]*$/ : sentenceEnd;const atPunctuation = cjk ? /[,;:.!?…।。!?、,;:]["'’”)\]」』)]*$/ : punctuation;+ const clings = clingsToPrevious(language);const cues = [];for (const scene of timeline.scenes) {const words = scene.words || [];@@ -37,9 +42,11 @@ export function subtitleCues(timeline, { from = 0, to = Infinity } = {}) {while (j < words.length) {if (endSentence.test(words[j - 1].text) || words[j].start - words[j - 1].end > 0.6 + 1e-9) break;if (words[j].end + 0.25 - words[i].start > 6 || lines(words.slice(i, j + 1), language) === null) {- for (let k = j - 1; k > i; k--) {- if (atPunctuation.test(words[k - 1].text)) { j = k; break; }- }+ let k = j - 1;+ while (k > i && !atPunctuation.test(words[k - 1].text)) k--;+ if (k > i) j = k;+ // Without punctuation, start the next cue at a word of its own.+ else while (j - 1 > i && clings(words[j].text, words[j - 1].text)) j--;break;}j++;diff --git a/test/cjk.test.js b/test/cjk.test.jsindex 33ed7ca..d1bfa1c 100644--- a/test/cjk.test.js+++ b/test/cjk.test.js@@ -6,7 +6,7 @@ import path from 'node:path';import { tokenizeNarration, parseScript, speechChunks, normWord, resolveMarks } from '../src/script.js';import { alignWords, readAlike, spreadReadings } from '../src/align.js';import { subtitleCues, toSrt, toVtt } from '../src/subtitles.js';-import { buildPhrases, drawCaptions } from '../engine/captions.js';+import { buildPhrases, drawCaptions, clingsToPrevious } from '../engine/captions.js';import { getTheme } from '../engine/themes.js';import { phonemize, misakiAvailable } from '../src/phonemes.js';import { languageFonts, ensureFonts } from '../src/fonts.js';@@ -87,6 +87,27 @@ for (const [language, text] of Object.entries(native)) {});}+test('subtitle lines and caption phrases do not start with a Japanese or Chinese particle', () => {+ const timed = list => list.map((text, i) => ({ text, start: i * 0.3, end: (i + 1) * 0.3 }));+ const ja = timed(['地球', 'の', '動き', 'と', 'カレンダー', 'に', 'ズレ', 'が', 'たまり', 'ます。']);+ const [cue] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: ja }] });+ assert.equal(cue.text, '地球の動きとカレンダーに\nズレがたまります。');+ const phrases = buildPhrases({ language: 'ja', scenes: [{ start: 0, words: ja }] }, 46);+ assert.equal(phrases.length, 2);+ assert.ok(phrases.every(p => !/^[\u3041-\u3096]/.test(p.words[0].text)), phrases.map(p => p.words.map(w => w.text).join('')).join(' / '));+ const [counter] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: timed(['そこで、', '基本', 'は', '4', '年', 'ごと', 'に', '1', '日', 'を', '足し', 'ます。']) }] });+ assert.equal(counter.text, 'そこで、基本は4年ごとに\n1日を足します。');+ const clings = clingsToPrevious('ja');+ for (const [text, before] of [['は', '基本'], ['せん', 'ありま'], ['ませんでした。', 'あり'], ['暦', 'グレゴリオ'], ['年', '4'], ['しょう', 'ので']]) assert.equal(clings(text, before), true, text);+ for (const [text, before] of [['うるう年', 'は'], ['ありま', 'では'], ['足', 'を'], ['地球', 'の'], ['カレンダー', 'の'], ['1', 'に']]) assert.equal(clings(text, before), false, text);+ // Breaking before or after に balances the lines equally; に stays with the word before.+ const [even] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: timed(['アイウエオカキクケ', 'に', 'サシスセソタチツテ。']) }] });+ assert.equal(even.text, 'アイウエオカキクケに\nサシスセソタチツテ。');+ const zh = timed(['这', '是', '我们', '最', '常用', '的', '密码', '保存', '方法', '之一', ',', '也', '很', '安全', '的', '做法', '吗', '?']);+ for (const p of buildPhrases({ language: 'zh', scenes: [{ start: 0, words: zh }] }, 46)) assert.ok(!['的', '吗'].includes(p.words[0].text), p.words.map(w => w.text).join(''));+ for (const c of subtitleCues({ language: 'zh', scenes: [{ start: 0, words: zh }] })) for (const line of c.text.split('\n')) assert.ok(!/^[的吗]/.test(line), line);+});+test('Mandarin alignment reads Traditional characters as Simplified, Japanese keeps them', () => {const rate = language => alignWords([{ spoken: '浏览器' }, { spoken: '会询问网址。' }], [{ text: '瀏覽器會詢問網址', start: 0, end: 1.6 }], 1.6, language).matchRate;assert.equal(rate('zh'), 1);@@ -168,6 +189,7 @@ test('Python batches JSON lines without shell expansion and accepts misaki phoneconst texts = ['解析。\n次の行', '$( ) `*` "中文" $@'];assert.deepEqual(await phonemize(texts, language), texts.map(t => t + '↗'));assert.equal(misakiAvailable(language), true);+ assert.deepEqual(await phonemize(['およそ365.2422日。'], language), [language === 'ja' ? 'およそ365点二四二二日。↗' : 'およそ365.2422日。↗']);// Latin words are read as English between the misaki pieces.assert.deepEqual(await phonemize(['我用JavaScript写代码。', 'API。'], language), ['我用↗ dʒˈɑːvə skɹˈɪpt 写代码。↗', 'ˌeɪpˌiːˈaɪ 。↗']);}