Skip to content

An AI skill that turns books into animated, narrated, interactive web experiences

fix(skill): captions and mouths survive silences inside their own speech (#17)

Yuya Shimboridb6f7799b4fc
diff --git a/.claude/skills/papermorph/scripts/join.py b/.claude/skills/papermorph/scripts/join.py
index c65ecdc..0681c42 100644
--- a/.claude/skills/papermorph/scripts/join.py
+++ b/.claude/skills/papermorph/scripts/join.py
@@ -11,8 +11,8 @@ hole belongs after a word's sound has ended, not where the next word starts. A s
AUDIO_DIR/shots.js: TIMINGS.SCENE = {dur, marks, cues, parts} with each clip's start in `parts` (marks become
"clip.mark") and the caption cues moved along; the mouth cues of clips with holes are moved too (MOUTHS.clip). Load
shots.js after timings.js and mouths.js. Captions start 0.12 s after Edge's word times (the sound starts that much
-later; mouths.py moves its cues the same) and end at the first pause of 0.45 s or more after their words.
-With --words, writes NARRATION.words.txt next to the narration (outside the site): each clip's start, its holes and
+later; mouths.py moves its cues the same) and end at the first pause of 0.45 s or more after their last word — a
+pause inside the sentence, or the clip's silence without word times, never ends one early. With --words, writes NARRATION.words.txt next to the narration (outside the site): each clip's start, its holes and
every word's heard time in the joined scene clip, which a scene's author times actions against (the beat clock is the
scene's `pre` plus these). Keep the join command in a script beside the book's notes and rerun it after any change.
Run after tts.py and mouths.py; needs ffmpeg.
@@ -28,14 +28,16 @@ def speech_end(f): # where a clip's sound stops (the start of its trailing sil
starts = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err)]
ends = [float(x) for x in re.findall(r'silence_end: ([\d.]+)', err)]
dur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(f)], capture_output=True, text=True).stdout)
- # the last silence runs to the end of the file (ffmpeg may or may not print its end there)
- return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .05) and starts[-1] > 0 else dur
+ # the last silence runs to the end of the file (ffmpeg may print its end there, just inside
+ # the mp3's priming padding): an end well before the file's duration is a pause inside the clip
+ return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .2) and starts[-1] > 0 else dur
def pauses(f): # start of every pause of 0.45 s or more inside a clip: a caption ends at the pause after its words
err = subprocess.run(['ffmpeg', '-v', 'info', '-i', str(f), '-af', 'silencedetect=n=-40dB:d=0.45', '-f', 'null', '-'], capture_output=True, text=True).stderr
return [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err)]
args, words = sys.argv[2:], None
if '--words' in args:
k = args.index('--words'); words = pathlib.Path(args[k + 1]); del args[k:k + 2]
+WB = json.loads(words.with_suffix('.timings.json').read_text()) if words else {} # per-beat word times, to place caption ends
def silence(path, sec):
subprocess.run(['ffmpeg', '-v', 'error', '-y', '-f', 'lavfi', '-i', 'anullsrc=r=24000:cl=mono', '-t', str(sec), '-b:a', '64k', str(path)], check=True)
out, H = {}, {} # H: every shot's holes, for the mouths
@@ -72,13 +74,18 @@ for arg in args:
for b in beats:
parts[b] = round(at, 3)
# a caption at 0 (a voice with no word timings) still belongs after a hole that opens the clip; each one ends
- # .3 s after its words do (the next caption, a hole between it and the next, or the clip's last sound), so it does
- # not hang over the action; a hole inside its own sentence keeps it up
+ # .3 s after its words do (the next caption, a hole between it and the next, or the clip's last sound), so it
+ # does not hang over the action; a hole or a pause inside its own sentence keeps it up — a pause may close it
+ # only once its final word has been heard (with word times; without them only the trusted ends close it)
if T[b].get('silent'): at += T[b]['dur'] + GAP; continue
cs, last, ps = T[b]['cues'], speech_end(d / f'{b}.mp3'), pauses(d / f'{b}.mp3')
+ said = [t for t, _ in (WB.get(b) or {}).get('words', [])]
for k, (t, c) in enumerate(cs):
t = max(t, .1); nxt = cs[k + 1][0] if k + 1 < len(cs) else None
- stop = min([last] + ([nxt] + [h for h, _ in holes[b] if t < h < nxt] if nxt is not None else []) + [x for x in ps if t + .15 < x < (nxt or 1e9)])
+ lim = nxt or 1e9
+ mine = [w for w in said if t <= w < lim]
+ stop = min([last] + ([nxt] + [h for h, _ in holes[b] if t < h < nxt] if nxt is not None else [])
+ + [x for x in ps if t + .15 < x < lim and mine and x > max(mine) + .2])
cues.append([round(shift(b, t) + at + LAG, 3), c, round(shift(b, stop - 1e-3) + at + .3, 3)] + [1] * bool(T[b].get('forced')))
marks.update({f'{b}.{k}': round(shift(b, v) + at, 3) for k, v in T[b]['marks'].items()})
at += T[b]['dur'] + sum(sec for _, sec in holes[b]) + GAP
diff --git a/.claude/skills/papermorph/scripts/mouths.py b/.claude/skills/papermorph/scripts/mouths.py
index fc79dd9..e6540a1 100644
--- a/.claude/skills/papermorph/scripts/mouths.py
+++ b/.claude/skills/papermorph/scripts/mouths.py
@@ -32,8 +32,12 @@ def shapes(word):
LAG = .12 # Edge's word times run ahead of the sound (join.py has the same)
def speech_end(f): # where the clip's sound stops: the mouth shuts there, not .55 s after the last word starts
err = subprocess.run(['ffmpeg', '-v', 'info', '-i', str(f), '-af', 'silencedetect=n=-40dB:d=0.15', '-f', 'null', '-'], capture_output=True, text=True).stderr
- st = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err) if float(x) > .2]
- return st[-1] if st else 1e9
+ starts = [float(x) for x in re.findall(r'silence_start: ([\d.]+)', err) if float(x) > .2]
+ ends = [float(x) for x in re.findall(r'silence_end: ([\d.]+)', err)]
+ dur = float(subprocess.run(['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', str(f)], capture_output=True, text=True).stdout or 0)
+ # only a silence that never ends (or runs to the file's end, inside mp3 priming padding) is
+ # trailing: an internal silence means speech resumes after it, and the mouth cues past it must stay
+ return starts[-1] if starts and (len(ends) < len(starts) or ends[-1] > dur - .2) and starts[-1] > 0 else dur
res = {}
for beat, e in cache.items():
words, dur, cues = e['words'], e['dur'], []