An AI skill that turns books into animated, narrated, interactive web experiences
JavaScript ai-agentsai-educationanimationclaudeclaude-codeclaude-skillsedtecheducationinteractive-textbookscormtext-to-video
fix(skill): picture-book captions, subtitles, effects and music
yzrf7f219f52e8a
diff --git a/.claude/skills/papermorph/assets/templates/picture-book/chapter.html b/.claude/skills/papermorph/assets/templates/picture-book/chapter.htmlindex 18afe1c..949863c 100644--- a/.claude/skills/papermorph/assets/templates/picture-book/chapter.html+++ b/.claude/skills/papermorph/assets/templates/picture-book/chapter.html@@ -41,12 +41,14 @@ const BEATS = [const BGM = [/* ['theme', ['scene1', 'scene2'], .3] */].map(([n, beats, gain]) => ({ n, beats, gain, a: Object.assign(new Audio(`audio/bgm/${n}.mp3`), { preload: 'auto', loop: true, volume: 0 }), v: 0 }));const beatLen = id => { const b = BEATS.find(x => x.id === id); return (b.pre || 0) + TIMINGS[id].dur + .6; };-let lastTick = performance.now();+let lastTick = performance.now(), duck = .3;function bgmTick(now) {const dt = Math.min(.1, (now - lastTick) / 1000); lastTick = now;const id = BEATS[P.i]?.id, on = P.started && P.playing && !P.waiting && !P.done;- const speaking = P.audio && !P.audio.paused && !P.audio.ended;- const vol = +(document.getElementById('volume')?.value ?? 1) * (speaking ? .3 : .7); // the music ducks well under the voices+ // the music ducks well under the voices for the whole of a scene that has them (its silent opening and tail too, so+ // it does not swell between lines), easing over about half a second+ duck += Math.max(-dt, Math.min(dt, (P.audio ? .3 : .7) - duck));+ const vol = +(document.getElementById('volume')?.value ?? 1) * duck;for (const m of BGM) {const k = m.beats.indexOf(id), current = on && k >= 0 && !(window.MUSIC_OFF?.[id]?.(P.t));m.v = Math.max(0, Math.min(1, m.v + (current ? 1 : -1) * dt / .8));diff --git a/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.js b/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.jsindex 62f1845..6717439 100644--- a/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.js+++ b/.claude/skills/papermorph/assets/templates/picture-book/lib/shared.js@@ -20,20 +20,24 @@ const artSVG = (name, x, y, w, h) => `<image href="../img/props/${name}.webp" x=/* ---------- sound effects ----------sfx(beat, at, name, vol): play audio/sfx/<name>.mp3 (SKILL/scripts/sfx.py) when the clock of beat `beat` passes `at`- in normal playback (seeking past it does not fire it). Pausing stops them. They play at half volume under the voices. */+ in normal playback (seeking past it does not fire it). Pausing holds them; a seek stops them. They play at half volume under the voices. */const SFX = [], sfxOn = [];const sfx = (beat, at, name, vol = 1) => SFX.push({ beat, at, name, vol });let sfxLast = { i: -1, t: 0 };function sfxTick() {const id = BEATS[P.i]?.id, on = P.started && P.playing && !P.waiting && !P.done;- if (on && sfxLast.i === P.i && P.t > sfxLast.t && P.t - sfxLast.t < .5)+ // the clock ran on (or stood still, paused) in the same beat; anything else is a seek, a replay or a new scene,+ // and what was playing belongs to another moment+ const same = sfxLast.i === P.i && P.t >= sfxLast.t && P.t - sfxLast.t < .5;+ if (!same) while (sfxOn.length) sfxOn.pop().pause();+ if (on && same)for (const e of SFX) if (e.beat === id && e.at > sfxLast.t && e.at <= P.t) {const a = new Audio(`audio/sfx/${e.name}.mp3`);a.volume = .5 * e.vol * +(document.getElementById('volume')?.value ?? 1); // effects sit under the voicesa.playbackRate = P.rate || 1;a.play().catch(() => {}); sfxOn.push(a); a.onended = () => sfxOn.splice(sfxOn.indexOf(a), 1);}- if (!on) while (sfxOn.length) sfxOn.pop().pause();+ for (const a of sfxOn) if (on === a.paused) on ? a.play().catch(() => {}) : a.pause(); // a pause holds them, play resumessfxLast = { i: P.i, t: P.t };requestAnimationFrame(sfxTick);}diff --git a/.claude/skills/papermorph/references/sound.md b/.claude/skills/papermorph/references/sound.mdindex ffa81c1..a932d3b 100644--- a/.claude/skills/papermorph/references/sound.md+++ b/.claude/skills/papermorph/references/sound.md@@ -5,7 +5,7 @@ Voices carry the story; everything else sits under them. In early cuts the music## Voices- **One voice per character.** List them under `voices` in `narration.<lang>.json` and write each line as `{"speaker": name, "text": …}`; `tts.py` synthesizes every line (Edge voices, with rate and pitch: children were adult voices sped up and pitched up). The narrator is a voice too.-- **Lines in other languages** are spoken in their own language by a native voice (Edge has Japanese, Chinese, Korean and more) and subtitled in the book's language. The caption shows both: the original and the translation in brackets.+- **Lines in other languages** are spoken in their own language by a native voice (Edge has Japanese, Chinese, Korean and more) and subtitled in the book's language. The caption shows both: the original and the translation in brackets. Write it as the beat's `"sub"`; `tts.py` marks such a line (its voice's locale differs from the narration file's language) so the page shows its subtitle even with captions off, the way films subtitle foreign dialogue.- **Mouths** follow the words: `mouths.py` turns each line's word timings into mouth shapes, and `lips(q, scene, pre, line…)` picks the shape for the frame.- **One clip per scene.** `join.py` joins a scene's lines into one audio clip with short pauses, and inserts silence where the action needs room (`line@T+S`: S seconds of silence T seconds into the line). Gaps for a held stare, a fly landing, a class turning round belong here, not in rushed animation. Changing a line means `tts.py`, `mouths.py` and `join.py` again.diff --git a/.claude/skills/papermorph/scripts/join.py b/.claude/skills/papermorph/scripts/join.pyindex 383f020..0cac55f 100644--- a/.claude/skills/papermorph/scripts/join.py+++ b/.claude/skills/papermorph/scripts/join.py@@ -9,11 +9,18 @@ AUDIO_DIR/shots.js: TIMINGS.SCENE = {dur, marks, cues, parts} with each clip's s"clip.mark") and the caption cues moved along; the mouth cues of clips with holes are moved too (MOUTHS.clip). Loadshots.js after timings.js and mouths.js. Run after tts.py and mouths.py; needs ffmpeg."""-import json, subprocess, sys, pathlib, tempfile+import json, re, subprocess, sys, pathlib, tempfileif len(sys.argv) < 3 or sys.argv[1] in ('-h', '--help'):print(__doc__); sys.exit(0 if sys.argv[1:2] in (['-h'], ['--help']) else 2)GAP = .35d = pathlib.Path(sys.argv[1]); T = json.loads((d / 'timings.js').read_text().split('=', 1)[1].rstrip().rstrip(';'))+def speech_end(f): # where a clip's sound stops (the start of its trailing silence), in seconds+ err = subprocess.run(['ffmpeg', '-v', 'info', '-i', str(f), '-af', 'silencedetect=n=-40dB:d=0.15', '-f', 'null', '-'], capture_output=True, text=True).stderr+ starts = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err)]+ ends = [float(x) for x in re.findall(r'silence_end: ([\d.]+)', err)]+ dur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(f)], capture_output=True, text=True).stdout)+ # the last silence runs to the end of the file (ffmpeg may or may not print its end there)+ return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .05) and starts[-1] > 0 else durout, H = {}, {} # H: every shot's holes, for the mouthsfor arg in sys.argv[2:]:shot, beats = arg.split('='); beats = beats.split(',')@@ -42,7 +49,14 @@ for arg in sys.argv[2:]:parts, cues, marks, at = {}, [], {}, 0for b in beats:parts[b] = round(at, 3)- cues += [[round(shift(b, t) + at, 3), c] for t, c in T[b]['cues']]+ # a caption at 0 (a voice with no word timings) still belongs after a hole that opens the clip; each one ends+ # .3 s after its words do (the next caption, a hole between it and the next, or the clip's last sound), so it does+ # not hang over the action; a hole inside its own sentence keeps it up+ cs, last = T[b]['cues'], speech_end(d / f'{b}.mp3')+ for k, (t, c) in enumerate(cs):+ t = max(t, .1); nxt = cs[k + 1][0] if k + 1 < len(cs) else None+ stop = min([last] + ([nxt] + [h for h, _ in holes[b] if t < h < nxt] if nxt is not None else []))+ cues.append([round(shift(b, t) + at, 3), c, round(shift(b, stop - 1e-3) + at + .3, 3)] + [1] * bool(T[b].get('forced')))marks.update({f'{b}.{k}': round(shift(b, v) + at, 3) for k, v in T[b]['marks'].items()})at += T[b]['dur'] + sum(sec for _, sec in holes[b]) + GAPdur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(d / f'{shot}.mp3')], capture_output=True, text=True).stdout)diff --git a/.claude/skills/papermorph/scripts/tts.py b/.claude/skills/papermorph/scripts/tts.pyindex b11eba7..6013a44 100644--- a/.claude/skills/papermorph/scripts/tts.py+++ b/.claude/skills/papermorph/scripts/tts.py@@ -202,12 +202,13 @@ async def main(src, out_dir):if engine not in ("edge", "kokoro"):raise SystemExit(f"unknown engine {engine!r}: use edge or kokoro")result = {}- public = lambda v: {k: v[k] for k in ("dur", "marks", "cues")}+ public = lambda v: {k: v[k] for k in ("dur", "marks", "cues", "forced") if k in v}live = {b: public(v) for b, v in cache.items()if {"dur", "marks", "cues"} <= v.keys() and (out_dir / f"{b}.mp3").exists()}timings_path = out_dir / "timings.js"for beat, raw in spec["beats"].items():who = voices[raw["speaker"]] if isinstance(raw, dict) else spec+ sub = raw.get("sub") if isinstance(raw, dict) else None # the line's subtitle, one cue for the whole clipraw = raw["text"] if isinstance(raw, dict) else rawvoice, rate, pitch = who["voice"], who.get("rate", "+0%"), who.get("pitch", "+0Hz")if pitch != "+0Hz" and engine == "kokoro":@@ -234,7 +235,11 @@ async def main(src, out_dir):words = (kokoro_beat(clean, voice, rate, clip) if engine == "kokoro"else await edge_beat(beat, clean, voice, rate, clip, out_dir, pitch))entry = {"key": key, "dur": duration(clip), "words": words,- "marks": mark_times(clean, marks, words), "cues": caption_cues(clean, words)}+ "marks": mark_times(clean, marks, words), "cues": [[0, sub]] if sub else caption_cues(clean, words)}+ # a line in another language than the book's (its voice's locale differs): the page shows its subtitle even+ # with captions off, as films do with foreign dialogue+ if sub and "-" in voice and voice.split("-")[0].lower() != Path(src).stem.split(".")[-1].lower():+ entry["forced"] = Trueif tmp:tmp.replace(mp3)print(f"{beat}: synthesized")