Skip to content

An AI skill that turns books into animated, narrated, interactive web experiences

fix(skill): picture-book captions, subtitles, effects and music

yzrf7f219f52e8a
diff --git a/.claude/skills/papermorph/assets/templates/picture-book/chapter.html b/.claude/skills/papermorph/assets/templates/picture-book/chapter.html
index 18afe1c..949863c 100644
--- a/.claude/skills/papermorph/assets/templates/picture-book/chapter.html
+++ b/.claude/skills/papermorph/assets/templates/picture-book/chapter.html
@@ -41,12 +41,14 @@ const BEATS = [
const BGM = [/* ['theme', ['scene1', 'scene2'], .3] */]
.map(([n, beats, gain]) => ({ n, beats, gain, a: Object.assign(new Audio(`audio/bgm/${n}.mp3`), { preload: 'auto', loop: true, volume: 0 }), v: 0 }));
const beatLen = id => { const b = BEATS.find(x => x.id === id); return (b.pre || 0) + TIMINGS[id].dur + .6; };
-let lastTick = performance.now();
+let lastTick = performance.now(), duck = .3;
function bgmTick(now) {
const dt = Math.min(.1, (now - lastTick) / 1000); lastTick = now;
const id = BEATS[P.i]?.id, on = P.started && P.playing && !P.waiting && !P.done;
- const speaking = P.audio && !P.audio.paused && !P.audio.ended;
- const vol = +(document.getElementById('volume')?.value ?? 1) * (speaking ? .3 : .7); // the music ducks well under the voices
+ // the music ducks well under the voices for the whole of a scene that has them (its silent opening and tail too, so
+ // it does not swell between lines), easing over about half a second
+ duck += Math.max(-dt, Math.min(dt, (P.audio ? .3 : .7) - duck));
+ const vol = +(document.getElementById('volume')?.value ?? 1) * duck;
for (const m of BGM) {
const k = m.beats.indexOf(id), current = on && k >= 0 && !(window.MUSIC_OFF?.[id]?.(P.t));
m.v = Math.max(0, Math.min(1, m.v + (current ? 1 : -1) * dt / .8));
diff --git a/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.js b/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.js
index 62f1845..6717439 100644
--- a/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.js
+++ b/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.js
@@ -20,20 +20,24 @@ const artSVG = (name, x, y, w, h) => `<image href="../img/props/${name}.webp" x=
/* ---------- sound effects ----------
sfx(beat, at, name, vol): play audio/sfx/<name>.mp3 (SKILL/scripts/sfx.py) when the clock of beat `beat` passes `at`
- in normal playback (seeking past it does not fire it). Pausing stops them. They play at half volume under the voices. */
+ in normal playback (seeking past it does not fire it). Pausing holds them; a seek stops them. They play at half volume under the voices. */
const SFX = [], sfxOn = [];
const sfx = (beat, at, name, vol = 1) => SFX.push({ beat, at, name, vol });
let sfxLast = { i: -1, t: 0 };
function sfxTick() {
const id = BEATS[P.i]?.id, on = P.started && P.playing && !P.waiting && !P.done;
- if (on && sfxLast.i === P.i && P.t > sfxLast.t && P.t - sfxLast.t < .5)
+ // the clock ran on (or stood still, paused) in the same beat; anything else is a seek, a replay or a new scene,
+ // and what was playing belongs to another moment
+ const same = sfxLast.i === P.i && P.t >= sfxLast.t && P.t - sfxLast.t < .5;
+ if (!same) while (sfxOn.length) sfxOn.pop().pause();
+ if (on && same)
for (const e of SFX) if (e.beat === id && e.at > sfxLast.t && e.at <= P.t) {
const a = new Audio(`audio/sfx/${e.name}.mp3`);
a.volume = .5 * e.vol * +(document.getElementById('volume')?.value ?? 1); // effects sit under the voices
a.playbackRate = P.rate || 1;
a.play().catch(() => {}); sfxOn.push(a); a.onended = () => sfxOn.splice(sfxOn.indexOf(a), 1);
}
- if (!on) while (sfxOn.length) sfxOn.pop().pause();
+ for (const a of sfxOn) if (on === a.paused) on ? a.play().catch(() => {}) : a.pause(); // a pause holds them, play resumes
sfxLast = { i: P.i, t: P.t };
requestAnimationFrame(sfxTick);
}
diff --git a/.claude/skills/papermorph/references/sound.md b/.claude/skills/papermorph/references/sound.md
index ffa81c1..a932d3b 100644
--- a/.claude/skills/papermorph/references/sound.md
+++ b/.claude/skills/papermorph/references/sound.md
@@ -5,7 +5,7 @@ Voices carry the story; everything else sits under them. In early cuts the music
## Voices
- **One voice per character.** List them under `voices` in `narration.<lang>.json` and write each line as `{"speaker": name, "text": …}`; `tts.py` synthesizes every line (Edge voices, with rate and pitch: children were adult voices sped up and pitched up). The narrator is a voice too.
-- **Lines in other languages** are spoken in their own language by a native voice (Edge has Japanese, Chinese, Korean and more) and subtitled in the book's language. The caption shows both: the original and the translation in brackets.
+- **Lines in other languages** are spoken in their own language by a native voice (Edge has Japanese, Chinese, Korean and more) and subtitled in the book's language. The caption shows both: the original and the translation in brackets. Write it as the beat's `"sub"`; `tts.py` marks such a line (its voice's locale differs from the narration file's language) so the page shows its subtitle even with captions off, the way films subtitle foreign dialogue.
- **Mouths** follow the words: `mouths.py` turns each line's word timings into mouth shapes, and `lips(q, scene, pre, line…)` picks the shape for the frame.
- **One clip per scene.** `join.py` joins a scene's lines into one audio clip with short pauses, and inserts silence where the action needs room (`line@T+S`: S seconds of silence T seconds into the line). Gaps for a held stare, a fly landing, a class turning round belong here, not in rushed animation. Changing a line means `tts.py`, `mouths.py` and `join.py` again.
diff --git a/.claude/skills/papermorph/scripts/join.py b/.claude/skills/papermorph/scripts/join.py
index 383f020..0cac55f 100644
--- a/.claude/skills/papermorph/scripts/join.py
+++ b/.claude/skills/papermorph/scripts/join.py
@@ -9,11 +9,18 @@ AUDIO_DIR/shots.js: TIMINGS.SCENE = {dur, marks, cues, parts} with each clip's s
"clip.mark") and the caption cues moved along; the mouth cues of clips with holes are moved too (MOUTHS.clip). Load
shots.js after timings.js and mouths.js. Run after tts.py and mouths.py; needs ffmpeg.
"""
-import json, subprocess, sys, pathlib, tempfile
+import json, re, subprocess, sys, pathlib, tempfile
if len(sys.argv) < 3 or sys.argv[1] in ('-h', '--help'):
print(__doc__); sys.exit(0 if sys.argv[1:2] in (['-h'], ['--help']) else 2)
GAP = .35
d = pathlib.Path(sys.argv[1]); T = json.loads((d / 'timings.js').read_text().split('=', 1)[1].rstrip().rstrip(';'))
+def speech_end(f): # where a clip's sound stops (the start of its trailing silence), in seconds
+ err = subprocess.run(['ffmpeg', '-v', 'info', '-i', str(f), '-af', 'silencedetect=n=-40dB:d=0.15', '-f', 'null', '-'], capture_output=True, text=True).stderr
+ starts = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err)]
+ ends = [float(x) for x in re.findall(r'silence_end: ([\d.]+)', err)]
+ dur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(f)], capture_output=True, text=True).stdout)
+ # the last silence runs to the end of the file (ffmpeg may or may not print its end there)
+ return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .05) and starts[-1] > 0 else dur
out, H = {}, {} # H: every shot's holes, for the mouths
for arg in sys.argv[2:]:
shot, beats = arg.split('='); beats = beats.split(',')
@@ -42,7 +49,14 @@ for arg in sys.argv[2:]:
parts, cues, marks, at = {}, [], {}, 0
for b in beats:
parts[b] = round(at, 3)
- cues += [[round(shift(b, t) + at, 3), c] for t, c in T[b]['cues']]
+ # a caption at 0 (a voice with no word timings) still belongs after a hole that opens the clip; each one ends
+ # .3 s after its words do (the next caption, a hole between it and the next, or the clip's last sound), so it does
+ # not hang over the action; a hole inside its own sentence keeps it up
+ cs, last = T[b]['cues'], speech_end(d / f'{b}.mp3')
+ for k, (t, c) in enumerate(cs):
+ t = max(t, .1); nxt = cs[k + 1][0] if k + 1 < len(cs) else None
+ stop = min([last] + ([nxt] + [h for h, _ in holes[b] if t < h < nxt] if nxt is not None else []))
+ cues.append([round(shift(b, t) + at, 3), c, round(shift(b, stop - 1e-3) + at + .3, 3)] + [1] * bool(T[b].get('forced')))
marks.update({f'{b}.{k}': round(shift(b, v) + at, 3) for k, v in T[b]['marks'].items()})
at += T[b]['dur'] + sum(sec for _, sec in holes[b]) + GAP
dur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(d / f'{shot}.mp3')], capture_output=True, text=True).stdout)
diff --git a/.claude/skills/papermorph/scripts/tts.py b/.claude/skills/papermorph/scripts/tts.py
index b11eba7..6013a44 100644
--- a/.claude/skills/papermorph/scripts/tts.py
+++ b/.claude/skills/papermorph/scripts/tts.py
@@ -202,12 +202,13 @@ async def main(src, out_dir):
if engine not in ("edge", "kokoro"):
raise SystemExit(f"unknown engine {engine!r}: use edge or kokoro")
result = {}
- public = lambda v: {k: v[k] for k in ("dur", "marks", "cues")}
+ public = lambda v: {k: v[k] for k in ("dur", "marks", "cues", "forced") if k in v}
live = {b: public(v) for b, v in cache.items()
if {"dur", "marks", "cues"} <= v.keys() and (out_dir / f"{b}.mp3").exists()}
timings_path = out_dir / "timings.js"
for beat, raw in spec["beats"].items():
who = voices[raw["speaker"]] if isinstance(raw, dict) else spec
+ sub = raw.get("sub") if isinstance(raw, dict) else None # the line's subtitle, one cue for the whole clip
raw = raw["text"] if isinstance(raw, dict) else raw
voice, rate, pitch = who["voice"], who.get("rate", "+0%"), who.get("pitch", "+0Hz")
if pitch != "+0Hz" and engine == "kokoro":
@@ -234,7 +235,11 @@ async def main(src, out_dir):
words = (kokoro_beat(clean, voice, rate, clip) if engine == "kokoro"
else await edge_beat(beat, clean, voice, rate, clip, out_dir, pitch))
entry = {"key": key, "dur": duration(clip), "words": words,
- "marks": mark_times(clean, marks, words), "cues": caption_cues(clean, words)}
+ "marks": mark_times(clean, marks, words), "cues": [[0, sub]] if sub else caption_cues(clean, words)}
+ # a line in another language than the book's (its voice's locale differs): the page shows its subtitle even
+ # with captions off, as films do with foreign dialogue
+ if sub and "-" in voice and voice.split("-")[0].lower() != Path(src).stem.split(".")[-1].lower():
+ entry["forced"] = True
if tmp:
tmp.replace(mp3)
print(f"{beat}: synthesized")