Skip to content

Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.

16.0 KB
Raw Blame
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { tokenizeNarration, parseScript, speechChunks, normWord, resolveMarks } from '../src/script.js';
import { alignWords, readAlike, spreadReadings } from '../src/align.js';
import { subtitleCues, toSrt, toVtt } from '../src/subtitles.js';
import { buildPhrases, drawCaptions, clingsToPrevious } from '../engine/captions.js';
import { getTheme } from '../engine/themes.js';
import { phonemize, misakiAvailable } from '../src/phonemes.js';
import { languageFonts, ensureFonts } from '../src/fonts.js';
import { cacheRoot } from '../src/models.js';
import { normalizeConfig } from '../src/project.js';
import { buildTimeline } from '../src/timeline.js';

function env(t, key, value) {
  const previous = process.env[key];
  process.env[key] = value;
  t.after(() => { if (previous === undefined) delete process.env[key]; else process.env[key] = previous; });
}

const native = { ja: '住所を解析します。次の画面を表示します!終わりです?', zh: '浏览器解析地址。下一步显示网页!完成了吗?' };

for (const [language, text] of Object.entries(native)) {
  test(`${language} segments words, keeps punctuation and preserves markers and overrides`, () => {
    const units = tokenizeNarration('[#first]' + text + '[pause 0.4][#last]{住所|じゅうしょ}です。', 0, language);
    const words = units.filter(u => u.type === 'word');
    assert.equal(words.map(w => w.display).join(''), text + '住所です。');
    assert.ok(words.some(w => w.display === '解析'));
    assert.deepEqual(words.find(w => w.display === '住所' && w.spoken !== w.display), { type: 'word', display: '住所', spoken: 'じゅうしょ', space: true });
    assert.equal(normWord('解析。'), '解析');
    assert.deepEqual(tokenizeNarration('次に「解析」を行う。', 0, language).filter(u => u.type === 'word').map(u => u.display).join(''), '次に「解析」を行う。');
    assert.ok(tokenizeNarration('次に「解析」を行う。', 0, language).filter(u => u.type === 'word').every(u => normWord(u.display)));
    assert.deepEqual(tokenizeNarration('{住所|じゅうしょ}。次。', 0, language).filter(u => u.type === 'word').map(u => u.display), ['住所。', '次。']);
    assert.deepEqual(speechChunks(tokenizeNarration('{住所|じゅうしょ}。「解析」。', 0, language), { language }).map(c => c.text), ['じゅうしょ。', '「解析」。']);
    assert.equal(normWord('が'), normWord('が'.normalize('NFD')));
    const times = units.map((u, i) => u.type === 'word' ? { start: i, end: i + 0.5 } : null);
    assert.equal(resolveMarks(units, times).first, 1);
    assert.equal(resolveMarks(units, times).last, units.findIndex(u => u.spoken === 'じゅうしょ'));
    const chunks = speechChunks(units, { language });
    assert.deepEqual(chunks.map(c => c.text), [...text.match(/[^。!?]+[。!?]/g), 'じゅうしょです。']);
    assert.equal(chunks.at(-1).gapBefore, 0.7);
    assert.equal(parseScript('## intro\n' + text, { language }).scenes[0].text, text);
    for (const ending of ['。', '!', '?', '.', '!', '?']) {
      assert.equal(speechChunks(tokenizeNarration(`解析${ending}解析${ending}`, 0, language), { language }).length, 2);
    }
  });

  test(`${language} alignment ignores Whisper word boundaries and times each word's characters`, () => {
    const words = [{ spoken: '解析' }, { spoken: '地址。' }];
    const whole = alignWords(words, [{ text: '解析地址', start: 0.2, end: 1 }], 1.2, language);
    const single = alignWords(words, [...'解析地址'].map((text, i) => ({ text, start: 0.2 + i * 0.2, end: 0.4 + i * 0.2 })), 1.2, language);
    assert.equal(whole.matchRate, 1);
    assert.deepEqual(whole.words, single.words);
    assert.ok(Math.abs(whole.words[0].start - 0.2) < 1e-9);
    assert.ok(Math.abs(whole.words[0].end - 0.6) < 1e-9);
    const shown = alignWords([{ spoken: 'じゅうしょ', display: '住所' }, { spoken: 'です。' }], [{ text: '住所です', start: 0, end: 1 }], 1, language);
    assert.equal(shown.matchRate, 1);
    assert.ok(shown.words.every(w => w.matched));
    const missing = alignWords([{ spoken: '解析' }, { spoken: '地' }, { spoken: '址' }], [{ text: '解析址', start: 0, end: 0.9 }], 1, language);
    assert.equal(missing.matchRate, 0.75);
    assert.equal(missing.words[1].matched, false);
    assert.ok(missing.words.every(w => w.start <= w.end));
    assert.equal(alignWords([{ spoken: 'かな' }], [{ text: 'かな'.normalize('NFD'), start: 0, end: 1 }], 1, language).matchRate, 1);
  });

  test(`${language} captions and subtitles join without spaces and limit lines at word boundaries`, () => {
    const text = native[language].repeat(3);
    const words = tokenizeNarration(text, 0, language).filter(u => u.type === 'word').map((u, i) => ({ text: u.display, start: i * 0.2, end: (i + 1) * 0.2 }));
    const timeline = buildTimeline({ config: normalizeConfig({ language, captions: true }), script: parseScript('## intro\n' + text, { language }) }, { intro: { duration: words.length * 0.2, words, marks: {} } });
    const cues = subtitleCues(timeline);
    assert.equal(cues.map(c => c.text.replaceAll('\n', '')).join(''), text);
    assert.ok(cues.every(c => c.text.split('\n').every(line => line.length <= 18)));
    assert.ok(toSrt(cues).includes('解析'));
    assert.ok(toVtt(cues).includes('解析'));
    const phrases = buildPhrases(timeline, 46);
    assert.equal(phrases.map(p => p.words.map(w => w.text).join(p.separator)).join(''), text);
    assert.ok(phrases.every(p => p.words.map(w => w.text).join('').length <= 18));
    const drawn = [];
    const ctx = new Proxy({ measureText: text => ({ width: text.length * 44 }), fillText: (text, x) => drawn.push({ text, x }) }, { get: (o, key) => o[key] ?? (() => {}) });
    drawCaptions(ctx, phrases, phrases[0].start + 0.1, getTheme('clean'), 1920, 1080);
    assert.equal(drawn[1].x - drawn[0].x, drawn[0].text.length * 44);
    const long = { language, scenes: [{ start: 0, words: Array.from({ length: 30 }, (_, i) => ({ text: '解析', start: i * 0.1, end: (i + 1) * 0.1 })) }] };
    assert.ok(subtitleCues(long).every(c => c.text.split('\n').every(line => line.length <= 18 && line.length % 2 === 0)));
    assert.throws(() => subtitleCues({ language, scenes: [{ words: [{ text: '長'.repeat(19) }] }] }), /longer than 18/);
  });
}

test('subtitle lines and caption phrases do not start with a Japanese or Chinese particle', () => {
  const timed = list => list.map((text, i) => ({ text, start: i * 0.3, end: (i + 1) * 0.3 }));
  const ja = timed(['地球', 'の', '動き', 'と', 'カレンダー', 'に', 'ズレ', 'が', 'たまり', 'ます。']);
  const [cue] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: ja }] });
  assert.equal(cue.text, '地球の動きとカレンダーに\nズレがたまります。');
  const phrases = buildPhrases({ language: 'ja', scenes: [{ start: 0, words: ja }] }, 46);
  assert.equal(phrases.length, 2);
  assert.ok(phrases.every(p => !/^[\u3041-\u3096]/.test(p.words[0].text)), phrases.map(p => p.words.map(w => w.text).join('')).join(' / '));
  const [counter] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: timed(['そこで、', '基本', 'は', '4', '年', 'ごと', 'に', '1', '日', 'を', '足し', 'ます。']) }] });
  assert.equal(counter.text, 'そこで、基本は4年ごとに\n1日を足します。');
  const clings = clingsToPrevious('ja');
  for (const [text, before] of [['は', '基本'], ['せん', 'ありま'], ['ませんでした。', 'あり'], ['暦', 'グレゴリオ'], ['年', '4'], ['しょう', 'ので']]) assert.equal(clings(text, before), true, text);
  for (const [text, before] of [['うるう年', 'は'], ['ありま', 'では'], ['足', 'を'], ['地球', 'の'], ['カレンダー', 'の'], ['1', 'に']]) assert.equal(clings(text, before), false, text);
  // Breaking before or after に balances the lines equally; に stays with the word before.
  const [even] = subtitleCues({ language: 'ja', scenes: [{ start: 0, words: timed(['アイウエオカキクケ', 'に', 'サシスセソタチツテ。']) }] });
  assert.equal(even.text, 'アイウエオカキクケに\nサシスセソタチツテ。');
  const zh = timed(['这', '是', '我们', '最', '常用', '的', '密码', '保存', '方法', '之一', ',', '也', '很', '安全', '的', '做法', '吗', '?']);
  for (const p of buildPhrases({ language: 'zh', scenes: [{ start: 0, words: zh }] }, 46)) assert.ok(!['的', '吗'].includes(p.words[0].text), p.words.map(w => w.text).join(''));
  for (const c of subtitleCues({ language: 'zh', scenes: [{ start: 0, words: zh }] })) for (const line of c.text.split('\n')) assert.ok(!/^[的吗]/.test(line), line);
});

test('Mandarin alignment reads Traditional characters as Simplified, Japanese keeps them', () => {
  const rate = language => alignWords([{ spoken: '浏览器' }, { spoken: '会询问网址。' }], [{ text: '瀏覽器會詢問網址', start: 0, end: 1.6 }], 1.6, language).matchRate;
  assert.equal(rate('zh'), 1);
  assert.equal(rate('ja'), 0.25);
});

test('Mandarin alignment accepts a character with the same sound, not a different one', () => {
  const words = [{ spoken: '哈希值' }, { spoken: '和' }, { spoken: '盐。' }];
  const heard = text => [{ text, start: 0, end: 1 }];
  // Whisper writes 直 for 值 and 研 for 盐; all are read zhi and yan.
  assert.equal(alignWords(words, heard('哈希直和研'), 1, 'zh').matchRate, 1);
  assert.equal(alignWords(words, heard('哈希職和研'), 1, 'zh').matchRate, 1);
  const wrong = alignWords(words, heard('哈希值和鸡'), 1, 'zh');
  assert.equal(wrong.matchRate, 0.8);
  assert.equal(wrong.words[2].matched, false);
  assert.equal(alignWords(words, heard('哈希直和研'), 1, 'ja').matchRate, 0.6);
  const shown = alignWords([{ spoken: '哈希', display: '哈希值' }], heard('哈希直'), 1, 'zh');
  assert.equal(shown.matchRate, 1);
});

test('Japanese readings are shared out over the characters they cover', () => {
  assert.deepEqual(spreadReadings(['うるう', '年', 'には'], [[0, 4, 'ウルードシ'], [4, 5, 'ニ'], [5, 6, 'ワ']]), ['ウルード', 'シ', 'ニワ']);
  assert.deepEqual(spreadReadings(['ウルードシ', '、'], [[0, 5, 'うるーどし'], [5, 6, '、']]), ['ウルードシ', '、']);
  assert.deepEqual(spreadReadings(['今日', 'は'], [[0, 2, 'キョー']]), ['キョー', '']);
});

test('the Japanese speech check compares readings, and falls back to characters without Python', { skip: process.platform === 'win32' }, async t => {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-readings-'));
  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
  const file = path.join(dir, 'python');
  env(t, 'EXPLAINROO_PYTHON', file);
  // A tiny dictionary: 世紀 and 正規 are both read セーキ.
  fs.writeFileSync(file, `#!/usr/bin/env node
    const assert = require('node:assert/strict');
    assert.equal(process.argv[3], 'ja-readings');
    const words = { 世紀: 'セーキ', 正規: 'セーキ', の: 'ノ', 年: 'トシ', とし: 'トシ' };
    let input = '';
    process.stdin.on('data', d => input += d);
    process.stdin.on('end', () => {
      for (const text of input.trimEnd().split('\\n').map(JSON.parse)) {
        const out = [];
        for (let i = 0; i < text.length;) {
          const w = Object.keys(words).find(w => text.startsWith(w, i));
          if (w) { out.push([i, i + w.length, words[w]]); i += w.length; } else i++;
        }
        console.log(JSON.stringify(out));
      }
    });
  `, { mode: 0o755 });
  const words = [{ spoken: '世紀', display: '世紀' }, { spoken: 'の', display: 'の' }, { spoken: '年。', display: '年。' }];
  const heard = [{ text: '正規の', start: 0, end: 0.6 }, { text: 'とし', start: 0.6, end: 1 }];
  const alike = await readAlike(words, heard, 'ja');
  assert.deepEqual(alike.words.map(w => w.spoken), ['セーキ', 'ノ', 'トシ']);
  assert.deepEqual(alike.heard.map(h => h.text), ['セーキノ', 'トシ']);
  assert.equal(alignWords(alike.words, alike.heard, 1, 'ja').matchRate, 1);
  assert.deepEqual(await readAlike(words, heard, 'zh'), { words, heard });
  fs.rmSync(file);
  assert.deepEqual(await readAlike(words, heard, 'ja'), { words, heard });
  assert.ok(alignWords(words, heard, 1, 'ja').matchRate < 0.7);
});

test('Python batches JSON lines without shell expansion and accepts misaki phonemes unchanged', { skip: process.platform === 'win32' }, async t => {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-python-'));
  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
  const file = path.join(dir, 'fake python');
  env(t, 'EXPLAINROO_PYTHON', file);
  fs.writeFileSync(file, `#!/usr/bin/env node
    const assert = require('node:assert/strict');
    assert.equal(process.argv[2].endsWith('/g2p.py'), true);
    assert.ok(['ja', 'zh'].includes(process.argv[3]));
    let input = '';
    process.stdin.on('data', d => input += d);
    process.stdin.on('end', () => {
      const texts = input.trimEnd().split('\\n').map(JSON.parse);
      process.stdout.write(texts.map(t => JSON.stringify(t + '↗')).join('\\n') + '\\n');
    });
  `, { mode: 0o755 });
  for (const language of ['ja', 'zh']) {
    const texts = ['解析。\n次の行', '$( ) `*` "中文" $@'];
    assert.deepEqual(await phonemize(texts, language), texts.map(t => t + '↗'));
    assert.equal(misakiAvailable(language), true);
    assert.deepEqual(await phonemize(['およそ365.2422日。'], language), [language === 'ja' ? 'およそ365点二四二二日。↗' : 'およそ365.2422日。↗']);
    // Latin words are read as English between the misaki pieces.
    assert.deepEqual(await phonemize(['我用JavaScript写代码。', 'API。'], language), ['我用↗ dʒˈɑːvə skɹˈɪpt 写代码。↗', 'ˌeɪpˌiːˈaɪ 。↗']);
  }
  fs.writeFileSync(file, '#!/usr/bin/env node\nconsole.log("not JSON");');
  await assert.rejects(phonemize(['解析'], 'ja'), /incomplete phonemes/);
  assert.equal(misakiAvailable('ja'), false);
  fs.writeFileSync(file, '#!/usr/bin/env node\nconsole.log(JSON.stringify("one"));');
  await assert.rejects(phonemize(['解析', '次'], 'ja'), /incomplete phonemes/);
  fs.writeFileSync(file, '#!/usr/bin/env node\nconsole.error("ModuleNotFoundError: No module named \'misaki\'"); process.exit(1);');
  await assert.rejects(phonemize(['解析'], 'zh'), /pip install "misaki\[zh\]".*Python said: ModuleNotFoundError: No module named 'misaki'/);
  fs.chmodSync(file, 0o644);
  await assert.rejects(phonemize(['解析'], 'ja'), /EXPLAINROO_PYTHON/);
  fs.rmSync(file);
  assert.equal(misakiAvailable('ja'), false);
  await assert.rejects(phonemize(['解析'], 'ja'), /-m unidic download/);
  assert.deepEqual(await phonemize([], 'ja'), []);
});

test('font paths follow EXPLAINROO_CACHE, offline mode refuses missing or damaged fonts', async t => {
  assert.equal(languageFonts('ja')[0].path, path.join(cacheRoot(), 'fonts', 'ja-400.otf'));
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'explainroo-font-cache-'));
  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
  env(t, 'EXPLAINROO_CACHE', dir);
  env(t, 'EXPLAINROO_OFFLINE', '1');
  assert.deepEqual(languageFonts('en'), []);
  for (const language of ['ja', 'zh']) {
    const fonts = languageFonts(language);
    assert.deepEqual(fonts.map(f => f.weight), [400, 700]);
    assert.equal(fonts[0].path, path.join(dir, 'fonts', `${language}-400.otf`));
    await assert.rejects(ensureFonts(language), /without EXPLAINROO_OFFLINE/);
    fs.mkdirSync(path.join(dir, 'fonts'), { recursive: true });
    fs.writeFileSync(fonts[0].path, 'OTTOcorrupt');
    await assert.rejects(ensureFonts(language), /missing or damaged/);
  }
  await ensureFonts('en');
  process.env.EXPLAINROO_OFFLINE = '0';
  t.mock.method(globalThis, 'fetch', async () => ({ ok: true, arrayBuffer: async () => Buffer.from('OTTOcorrupt') }));
  await assert.rejects(ensureFonts('ja'), /could not be downloaded or checked/);
  assert.equal(fs.readFileSync(languageFonts('ja')[0].path, 'utf8'), 'OTTOcorrupt');
});