An AI skill that turns books into animated, narrated, interactive web experiences
JavaScript ai-agentsai-educationanimationclaudeclaude-codeclaude-skillsedtecheducationinteractive-textbookscormtext-to-video
fix(skill): captions and mouths survive silences inside their own speech (#17)
Yuya Shimboridb6f7799b4fc
diff --git a/.claude/skills/papermorph/scripts/join.py b/.claude/skills/papermorph/scripts/join.pyindex c65ecdc..0681c42 100644--- a/.claude/skills/papermorph/scripts/join.py+++ b/.claude/skills/papermorph/scripts/join.py@@ -11,8 +11,8 @@ hole belongs after a word's sound has ended, not where the next word starts. A sAUDIO_DIR/shots.js: TIMINGS.SCENE = {dur, marks, cues, parts} with each clip's start in `parts` (marks become"clip.mark") and the caption cues moved along; the mouth cues of clips with holes are moved too (MOUTHS.clip). Loadshots.js after timings.js and mouths.js. Captions start 0.12 s after Edge's word times (the sound starts that much-later; mouths.py moves its cues the same) and end at the first pause of 0.45 s or more after their words.-With --words, writes NARRATION.words.txt next to the narration (outside the site): each clip's start, its holes and+later; mouths.py moves its cues the same) and end at the first pause of 0.45 s or more after their last word — a+pause inside the sentence, or the clip's silence without word times, never ends one early. With --words, writes NARRATION.words.txt next to the narration (outside the site): each clip's start, its holes andevery word's heard time in the joined scene clip, which a scene's author times actions against (the beat clock is thescene's `pre` plus these). Keep the join command in a script beside the book's notes and rerun it after any change.Run after tts.py and mouths.py; needs ffmpeg.@@ -28,14 +28,16 @@ def speech_end(f): # where a clip's sound stops (the start of its trailing silstarts = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err)]ends = [float(x) for x in re.findall(r'silence_end: ([\d.]+)', err)]dur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(f)], capture_output=True, text=True).stdout)- # the last silence runs to the end of the file (ffmpeg may or may not print its end there)- return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .05) and starts[-1] > 0 else dur+ # the last silence runs to the end of the file (ffmpeg may print its end there, just inside+ # the mp3's priming padding): an end well before the file's duration is a pause inside the clip+ return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .2) and starts[-1] > 0 else durdef pauses(f): # start of every pause of 0.45 s or more inside a clip: a caption ends at the pause after its wordserr = subprocess.run(['ffmpeg', '-v', 'info', '-i', str(f), '-af', 'silencedetect=n=-40dB:d=0.45', '-f', 'null', '-'], capture_output=True, text=True).stderrreturn [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err)]args, words = sys.argv[2:], Noneif '--words' in args:k = args.index('--words'); words = pathlib.Path(args[k + 1]); del args[k:k + 2]+WB = json.loads(words.with_suffix('.timings.json').read_text()) if words else {} # per-beat word times, to place caption endsdef silence(path, sec):subprocess.run(['ffmpeg', '-v', 'error', '-y', '-f', 'lavfi', '-i', 'anullsrc=r=24000:cl=mono', '-t', str(sec), '-b:a', '64k', str(path)], check=True)out, H = {}, {} # H: every shot's holes, for the mouths@@ -72,13 +74,18 @@ for arg in args:for b in beats:parts[b] = round(at, 3)# a caption at 0 (a voice with no word timings) still belongs after a hole that opens the clip; each one ends- # .3 s after its words do (the next caption, a hole between it and the next, or the clip's last sound), so it does- # not hang over the action; a hole inside its own sentence keeps it up+ # .3 s after its words do (the next caption, a hole between it and the next, or the clip's last sound), so it+ # does not hang over the action; a hole or a pause inside its own sentence keeps it up — a pause may close it+ # only once its final word has been heard (with word times; without them only the trusted ends close it)if T[b].get('silent'): at += T[b]['dur'] + GAP; continuecs, last, ps = T[b]['cues'], speech_end(d / f'{b}.mp3'), pauses(d / f'{b}.mp3')+ said = [t for t, _ in (WB.get(b) or {}).get('words', [])]for k, (t, c) in enumerate(cs):t = max(t, .1); nxt = cs[k + 1][0] if k + 1 < len(cs) else None- stop = min([last] + ([nxt] + [h for h, _ in holes[b] if t < h < nxt] if nxt is not None else []) + [x for x in ps if t + .15 < x < (nxt or 1e9)])+ lim = nxt or 1e9+ mine = [w for w in said if t <= w < lim]+ stop = min([last] + ([nxt] + [h for h, _ in holes[b] if t < h < nxt] if nxt is not None else [])+ + [x for x in ps if t + .15 < x < lim and mine and x > max(mine) + .2])cues.append([round(shift(b, t) + at + LAG, 3), c, round(shift(b, stop - 1e-3) + at + .3, 3)] + [1] * bool(T[b].get('forced')))marks.update({f'{b}.{k}': round(shift(b, v) + at, 3) for k, v in T[b]['marks'].items()})at += T[b]['dur'] + sum(sec for _, sec in holes[b]) + GAPdiff --git a/.claude/skills/papermorph/scripts/mouths.py b/.claude/skills/papermorph/scripts/mouths.pyindex fc79dd9..e6540a1 100644--- a/.claude/skills/papermorph/scripts/mouths.py+++ b/.claude/skills/papermorph/scripts/mouths.py@@ -32,8 +32,12 @@ def shapes(word):LAG = .12 # Edge's word times run ahead of the sound (join.py has the same)def speech_end(f): # where the clip's sound stops: the mouth shuts there, not .55 s after the last word startserr = subprocess.run(['ffmpeg', '-v', 'info', '-i', str(f), '-af', 'silencedetect=n=-40dB:d=0.15', '-f', 'null', '-'], capture_output=True, text=True).stderr- st = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err) if float(x) > .2]- return st[-1] if st else 1e9+ starts = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err) if float(x) > .2]+ ends = [float(x) for x in re.findall(r'silence_end: ([\d.]+)', err)]+ dur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(f)], capture_output=True, text=True).stdout or 0)+ # only a silence that never ends (or runs to the file's end, inside mp3 priming padding) is+ # trailing: an internal silence means speech resumes after it, and the mouth cues past it must stay+ return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .2) and starts[-1] > 0 else durres = {}for beat, e in cache.items():words, dur, cues = e['words'], e['dur'], []