Skip to content

Explainer videos and product demos made by your AI agent. Free and open source: a local voice (Kokoro), word timing (Whisper) and a canvas renderer turn a script into a narrated MP4.

6.3 KB
Raw Blame
// Burned-in captions built from the narration's word timings. The current
// phrase is shown; words light up as they are spoken.
import { fontString } from './themes.js';
import { clamp, ease } from './util.js';

// Some words belong to the word before them, so a phrase or line should not
// start with one: Japanese particles and verb endings (short hiragana pieces
// such as は, ごと, せん, or ones starting like ません), a one-kanji suffix
// such as the 暦 of グレゴリオ暦, Chinese particles such as 的, and counters
// after a number (4年, 365天).
const JAPANESE_ENDINGS = /^(ませ|まし|でし|です|ます|しょう|られ|させ|され|なかっ|ながら|ような|ように|という|として|について|によって|くらい|ぐらい)/;
export function clingsToPrevious(language) {
  const counter = (text, before) => /\d$/.test(before) && /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(text);
  if (language === 'ja') {
    return (text, before = '') => {
      const word = text.replace(/[、。!?「」『』()]+$/, '');
      const ending = /^[\u3041-\u3096]+$/.test(word) && (word.length <= 2 || JAPANESE_ENDINGS.test(word));
      const suffix = /^\p{Script=Han}$/u.test(word) && /[\p{Script=Han}\p{Script=Katakana}]$/u.test(before);
      return ending || suffix || counter(text, before);
    };
  }
  if (language === 'zh') return (text, before = '') => /^[的了吗呢吧啊着过得地们][,。!?、]*$/.test(text) || counter(text, before);
  return () => false;
}

export function buildPhrases(timeline, maxChars) {
  const cjk = ['ja', 'zh'].includes(timeline.language);
  const sentenceEnd = cjk ? /[.!?;:।。!?]["'’)」』)]*$/ : /[.!?;:।]["'’)]*$/;
  if (cjk) maxChars = Math.min(maxChars, 18);
  const phrases = [];
  for (const sc of timeline.scenes) {
    // A sentence (or a run of words between long pauses) is split into as few
    // phrases as fit, balanced in length, so no phrase is one lonely word.
    let run = [];
    const flush = () => {
      for (const words of balancePhrases(run, maxChars, cjk ? '' : ' ', clingsToPrevious(timeline.language))) phrases.push({ words, ...(cjk ? { separator: '' } : {}), start: words[0].start, end: words[words.length - 1].end });
      run = [];
    };
    for (const w of sc.words) {
      const g = { text: w.text, start: sc.start + w.start, end: sc.start + w.end };
      if (run.length && g.start - run[run.length - 1].end > 0.55) flush();
      run.push(g);
      if (sentenceEnd.test(g.text)) flush();
    }
    if (run.length) flush();
  }
  phrases.forEach((p, i) => {
    const next = phrases[i + 1];
    p.until = next ? Math.min(next.start, p.end + 0.9) : p.end + 0.9;
  });
  return phrases;
}

// Splits words into the fewest phrases of at most maxChars characters, with
// lengths as even as possible and breaks after commas where they fit, never
// before a word that clings to the one before it if that can be helped.
export function balancePhrases(words, maxChars, separator = ' ', clings = () => false) {
  if (!words.length) return [];
  const len = (a, b) => words.slice(a, b).reduce((n, w) => n + w.text.length + separator.length, -separator.length);
  if (len(0, words.length) <= maxChars) return [words];
  const punctuation = separator === '' ? /[,;:–—、,;:]["'’)」』)]*$/ : /[,;:–—]["'’)]*$/;
  const n = words.length;
  const best = [{ parts: 0, cost: 0, from: -1 }];
  for (let b = 1; b <= n; b++) {
    best[b] = null;
    for (let a = b - 1; a >= 0; a--) {
      const l = len(a, b);
      if (l > maxChars && b - a > 1) break;
      if (!best[a]) continue;
      const comma = b < n && punctuation.test(words[b - 1].text) ? 150 : 0;
      const glued = b < n && !comma && clings(words[b].text, words[b - 1].text) ? 250 : 0;
      const c = { parts: best[a].parts + 1, cost: best[a].cost + (maxChars - l) ** 2 - comma + glued, from: a };
      if (!best[b] || c.parts < best[b].parts || (c.parts === best[b].parts && c.cost < best[b].cost)) best[b] = c;
    }
  }
  const out = [];
  for (let b = n; b > 0; b = best[b].from) out.unshift(words.slice(best[b].from, b));
  return out;
}

// `area` ({ cx, bottom, maxWidth }) places the captions inside a platform's
// safe area, right above the band the app covers with its own text.
export function drawCaptions(ctx, phrases, T, theme, W, H, area = null) {
  const ph = phrases.find((p) => T >= p.start - 0.08 && T < p.until);
  if (!ph) return;
  const vertical = H > W;
  const size = vertical ? 58 : 44;
  const font = fontString(theme, 'body', size, theme.fonts.body.family === 'Inter' ? 600 : undefined);
  ctx.save();
  ctx.font = font;
  ctx.textBaseline = 'middle';
  ctx.textAlign = 'left';
  const space = ctx.measureText(ph.separator ?? ' ').width;
  const widths = ph.words.map((w) => ctx.measureText(w.text).width);
  const maxLine = area ? area.maxWidth - size * 1.1 : W * (vertical ? 0.8 : 0.7);
  const lines = [[]];
  let lw = 0;
  ph.words.forEach((w, i) => {
    const add = (lines[lines.length - 1].length ? space : 0) + widths[i];
    if (lw + add > maxLine && lines[lines.length - 1].length) {
      lines.push([]);
      lw = 0;
    }
    lines[lines.length - 1].push(i);
    lw += (lines[lines.length - 1].length > 1 ? space : 0) + widths[i];
  });
  const lh = size * 1.3;
  const boxW = Math.max(...lines.map((l) => l.reduce((a, i, k) => a + widths[i] + (k ? space : 0), 0))) + size * 1.1;
  const boxH = lines.length * lh + size * 0.55;
  const cx = area ? area.cx : W / 2;
  const cy = area ? area.bottom - boxH / 2 : vertical ? H * 0.74 : H - Math.max(96, H * 0.1) - boxH / 2 + lh / 2;
  const fadeIn = ease.out(clamp((T - (ph.start - 0.08)) / 0.15));
  const fadeOut = 1 - clamp((T - (ph.until - 0.15)) / 0.15);
  ctx.globalAlpha = Math.min(fadeIn, fadeOut);
  ctx.fillStyle = theme.caption.bg;
  ctx.beginPath();
  ctx.roundRect(cx - boxW / 2, cy - boxH / 2, boxW, boxH, Math.min(22, boxH / 2));
  ctx.fill();
  lines.forEach((l, li) => {
    const total = l.reduce((a, i, k) => a + widths[i] + (k ? space : 0), 0);
    let x = cx - total / 2;
    const y = cy - boxH / 2 + size * 0.275 + lh * li + lh / 2;
    for (const i of l) {
      const w = ph.words[i];
      ctx.fillStyle = T >= w.start ? theme.caption.active : theme.caption.fg;
      ctx.fillText(w.text, x, y);
      x += widths[i] + space;
    }
  });
  ctx.restore();
}