Local video editor for coding agents: 42 FFmpeg tools with a machine-readable contract and an MCP server, for Claude Code, Cursor and Codex. Cut, captions, reframe, loudness, platform exports. No cloud, no API keys.
Python agent-skillsai-agentsaudio-processingcaptionsclaude-codecodexcoding-agentcursorffmpegloudness-normalizationmcpmcp-servermodel-context-protocolpythonsubtitlestranscodingvideovideo-editingvideo-processingwhisper
caption.py: --karaoke-style word for per-word Dialogue events (scale + 3-colour highlight) (#278)
Shogo Kajiyamae63f2e8d9abd
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex dba1eb5..3506a01 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,7 +4,7 @@## Unreleased-(nothing yet)+- feat(caption): `--karaoke-style word` -- one ASS Dialogue event per word, active word scaled/emboldened, past/upcoming colours (#276)## 1.24.0diff --git a/scripts/caption.py b/scripts/caption.pyindex 3f5e3f7..546bed2 100755--- a/scripts/caption.py+++ b/scripts/caption.py@@ -25,6 +25,7 @@ Text-to-SRT input format (one cue per line, blank lines ignored):Examples:python3 caption.py input.mp4 --srt subs.srtpython3 caption.py input.mp4 --text cues.txt --animate pop --karaoke # word-by-word highlight, TikTok style+ python3 caption.py input.mp4 --text cues.txt --karaoke --karaoke-style word # active word scaled + emboldened, past/upcoming colourspython3 caption.py input.mp4 --srt subs.srt --font "Noto Sans CJK JP" --size 28 --position toppython3 caption.py --text cues.txt --write-srt cues.srt # only produce the SRTpython3 caption.py input.mp4 --text cues.txt # generate + burn in one go@@ -464,9 +465,18 @@ def max_line_em(args, play_w: Optional[int], play_h: Optional[int]) -> Optional[--size is in ASS points against a 288-line script (what libass's force_style uses), so therendered pixel size is size * play_h / 288.++ --karaoke-style word scales its active word up by --karaoke-scale and freezes the line break+ (WrapStyle: 2 in write_ass) so libass can never re-wrap around that scale-up. The budget here+ is narrowed by the same factor so the longest wrapped line -- computed at the UNscaled width --+ still fits once its active word is drawn --karaoke-scale%% larger; without this, a line that+ exactly filled the safe width would overflow it the moment its emphasised word appears."""- return line_em_for_size(args.size, play_w, play_h,- safe_fraction=safe_width_fraction(args, play_w))+ em = line_em_for_size(args.size, play_w, play_h,+ safe_fraction=safe_width_fraction(args, play_w))+ if em is not None and getattr(args, "karaoke", False) and getattr(args, "karaoke_style", "sweep") == "word":+ em = em / max(1.0, getattr(args, "karaoke_scale", 112) / 100.0)+ return emdef parse_ass_dialogue(path: str) -> str:@@ -526,6 +536,23 @@ def word_durations_from_timings(words: List[Tuple[float, float, str]], start: flreturn out+def _karaoke_word_durations(args, video: Optional[str], start: float, end: float, n_words: int) -> List[int]:+ """Centiseconds per word for one cue, shared by --karaoke-style sweep and word.++ Real word timings from the transcript beat both the energy estimate and the even split --+ they are what the speaker actually did, not a proxy for it.+ """+ dur_cs = max(1, int(round((end - start) * 100)))+ durs = word_durations_from_timings(getattr(args, "_word_timings", None) or [], start, end, n_words)+ if durs is None:+ if getattr(args, "karaoke_timing", "even") == "energy" and video:+ durs = word_durations_from_audio(video, start, end, n_words, getattr(args, "audio_stream", 0))+ else:+ per = max(1, dur_cs // max(1, n_words))+ durs = [per] * n_words+ return durs++def write_ass(cues: List[Tuple[float, float, str]], path: str, args, play_w: int, play_h: int, video: str = None) -> None:"""Write a styled ASS file with optional animation and word-by-word highlight."""def t(sec: float) -> str:@@ -544,8 +571,20 @@ def write_ass(cues: List[Tuple[float, float, str]], path: str, args, play_w: intsecondary = ass_color(args.color)outline = ass_color(args.outline_color)back = ass_color(args.outline_color, 0x80)+ word_style = args.karaoke and getattr(args, "karaoke_style", "sweep") == "word"+ word_past = ass_color(args.color)+ word_active = ass_color(args.highlight_color)+ word_upcoming = ass_color(args.upcoming_color)+ word_bord = f"{args.outline * scale * (max(1.0, args.karaoke_scale) / 100.0):.2f}"+ word_scale = int(round(args.karaoke_scale))+ # --karaoke-style word emits one Dialogue per word carrying the WHOLE cue with explicit \N+ # breaks (the layout's own fixed line list -- see max_line_em's docstring). WrapStyle: 2 tells+ # libass to honour only those explicit breaks and never re-wrap: with WrapStyle: 0 (the sweep+ # default, unchanged), scaling the active word up would re-flow the line and it would visibly+ # jump every time the highlight advances (the trap the issue this implements is filed against).header = [- "[Script Info]", "ScriptType: v4.00+", f"PlayResX: {play_w}", f"PlayResY: {play_h}", "WrapStyle: 0", "ScaledBorderAndShadow: yes", "",+ "[Script Info]", "ScriptType: v4.00+", f"PlayResX: {play_w}", f"PlayResY: {play_h}",+ f"WrapStyle: {2 if word_style else 0}", "ScaledBorderAndShadow: yes", "","[V4+ Styles]","Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",f"Style: Default,{ass_font_name(args.font)},{size},{primary},{secondary},{outline},{back},{-1 if args.bold else 0},0,0,0,100,100,0,0,{3 if args.box else 1},{args.outline * scale:.1f},{args.shadow * scale:.1f},{ALIGN[args.position]},{margin_l},{margin_r},{margin},1",@@ -579,20 +618,66 @@ def write_ass(cues: List[Tuple[float, float, str]], path: str, args, play_w: intown = own_sizes[cue_index] if cue_index < len(own_sizes) else Noneif own and own != args.size:fx += "{\\fs%d}" % int(round(own * scale))+ if args.karaoke and word_style:+ # One Dialogue event per WORD, each carrying the whole cue -- \k/\kf can only+ # interpolate colour, so scaling/emboldening the active word (and a third "upcoming"+ # colour) needs a real per-word override block instead. The line list is `body`'s own+ # \N split, already frozen by layout_cues (this function never re-wraps it), so every+ # event uses the identical break -- WrapStyle: 2 above stops libass re-wrapping around+ # the active word's larger \fscx/\fscy, which is the jump the issue is filed against.+ line_words = [[w for w in seg.split(" ") if w] for seg in body.split("\\N")]+ flat = [w for lw in line_words for w in lw]+ real_idx = [i for i, w in enumerate(flat) if w.strip(EMOJI_SENTINEL)]+ if not real_idx:+ # nothing to highlight (an all-emoji or empty cue) -- one plain event, same as+ # today's karaoke-off rendering, rather than zero Dialogue lines for this cue+ out_body = body+ if EMOJI_SENTINEL in out_body:+ out_body = out_body.replace(EMOJI_SENTINEL, emoji_placeholder(getattr(args, "_emoji_box_px", size)))+ lines.append(f"Dialogue: 0,{t(start)},{t(end)},Default,,0,0,0,,{fx}{out_body}")+ continue+ n_words = len(real_idx)+ durs = _karaoke_word_durations(args, video, start, end, n_words)+ starts_cs, acc = [], 0+ for d in durs:+ starts_cs.append(acc)+ acc += d+ for j in range(n_words):+ w_start = start + starts_cs[j] / 100.0+ w_end = start + (starts_cs[j] + durs[j]) / 100.0+ # colour/scale each real word by its rank against the active one (j); an+ # emoji-placeholder token is never active and just keeps its plain text+ styled, flat_pos = [], 0+ for lw in line_words:+ styled_line = []+ for w in lw:+ if not w.strip(EMOJI_SENTINEL):+ styled_line.append(w)+ else:+ rank = real_idx.index(flat_pos)+ if rank < j:+ styled_line.append(f"{{\\c{word_past}}}{w}")+ elif rank == j:+ styled_line.append(+ f"{{\\c{word_active}\\fscx{word_scale}\\fscy{word_scale}\\bord{word_bord}}}{w}{{\\r}}")+ else:+ styled_line.append(f"{{\\c{word_upcoming}}}{w}")+ flat_pos += 1+ styled.append(" ".join(styled_line))+ out_body = "\\N".join(styled)+ if EMOJI_SENTINEL in out_body:+ out_body = out_body.replace(EMOJI_SENTINEL, emoji_placeholder(getattr(args, "_emoji_box_px", size)))+ # the size/fit override (\fsN) belongs on every event; the entrance animation+ # (\fad/\move/pop) only on the cue's FIRST word event -- replaying it on every+ # later word would re-fade/re-pop the whole cue each time the highlight advances+ event_fx = fx if j == 0 else (fx[fx.index("{\\fs"):] if "{\\fs" in fx else "")+ lines.append(f"Dialogue: 0,{t(w_start)},{t(w_end)},Default,,0,0,0,,{event_fx}{out_body}")+ continueif args.karaoke:# split each line into words and give every word an equal share of the cue (\k is in centiseconds)- dur_cs = max(1, int(round((end - start) * 100)))segments = body.split("\\N")words = [w for seg in segments for w in seg.split(" ") if w and w.strip(EMOJI_SENTINEL)]- # real word timings from the transcript beat both the energy estimate and the even- # split -- they are what the speaker actually did, not a proxy for it- durs = word_durations_from_timings(getattr(args, "_word_timings", None) or [], start, end, len(words))- if durs is None:- if getattr(args, "karaoke_timing", "even") == "energy" and video:- durs = word_durations_from_audio(video, start, end, len(words), getattr(args, "audio_stream", 0))- else:- per = max(1, dur_cs // max(1, len(words)))- durs = [per] * len(words)+ durs = _karaoke_word_durations(args, video, start, end, len(words))it = iter(durs)out_segments = []for seg in segments:@@ -840,7 +925,16 @@ def main() -> int:anim = ap.add_argument_group("animation (generates ASS; needs --text or --srt input)")anim.add_argument("--animate", choices=["none", "fade", "pop", "slide"], default=None, help="per-cue entrance animation (default none, or brand caption.animate)")anim.add_argument("--karaoke", action="store_true", help="word-by-word highlight (fills from --color to --highlight-color across each cue)")- anim.add_argument("--highlight-color", default=None, help="karaoke fill colour RRGGBB (default FFD200 or brand primary)")+ anim.add_argument("--karaoke-style", choices=["sweep", "word"], default="sweep",+ help="'sweep' (default): today's \\kf colour fill, one Dialogue event per cue -- unchanged. "+ "'word': one Dialogue event per word, the active word scaled by --karaoke-scale and "+ "emboldened, past words in --color, upcoming words in --upcoming-color -- \\k/\\kf can only "+ "interpolate colour, so this is the only way to also scale/embolden the active word")+ anim.add_argument("--highlight-color", default=None, help="karaoke fill/active-word colour RRGGBB (default FFD200 or brand primary)")+ anim.add_argument("--upcoming-color", default=None,+ help="--karaoke-style word: colour RRGGBB for words not yet spoken (default B4B4B4)")+ anim.add_argument("--karaoke-scale", type=float, default=112.0,+ help="--karaoke-style word: the active word's size as a percentage of the caption size (default 112)")anim.add_argument("--karaoke-timing", choices=["even", "energy"], default="energy",help="how words are timed inside a cue: 'energy' follows the speech loudness in the audio (default), 'even' splits time equally")anim.add_argument("--write-ass", help="where to save the generated ASS (default: next to the output)")@@ -886,6 +980,9 @@ def main() -> int:# to be applied anyway and then refused as "animation is burn only" -- ignore it thereargs.animate = args.animate or (bcap.get("animate", "none") if args.brand and args.mode != "mux" else "none")args.highlight_color = color_hex(args.highlight_color or bc.get("primary", "FFD200"))+ args.upcoming_color = color_hex(args.upcoming_color or bc.get("upcoming", "B4B4B4"))+ if args.karaoke_scale <= 0:+ die("--karaoke-scale must be positive")if args.brand and bcap.get("bold") and not args.bold:args.bold = Trueif args.brand and bcap.get("karaoke") and not args.karaoke:diff --git a/tests/fixtures/mcp_tools.json b/tests/fixtures/mcp_tools.jsonindex ec3b5ec..fb141f9 100644--- a/tests/fixtures/mcp_tools.json+++ b/tests/fixtures/mcp_tools.json@@ -166,6 +166,8 @@"json","json_brief","karaoke",+ "karaoke_scale",+ "karaoke_style","karaoke_timing","language","margin",@@ -192,6 +194,7 @@"timeout","track_title","transcribe",+ "upcoming_color","wrap","write_ass","write_srt"diff --git a/tests/test_picture.py b/tests/test_picture.pyindex e7148c0..83d99d6 100644--- a/tests/test_picture.py+++ b/tests/test_picture.py@@ -1198,6 +1198,100 @@ class PictureTests(MediaFixtures):line2 = [l for l in ass2.read_text(encoding="utf-8-sig").splitlines() if l.startswith("Dialogue")][0]self.assertEqual([int(x) for x in re.findall(r"\\kf(\d+)", line2)], [100, 100, 100, 100])+ # ---------------------------------------------------------------- --karaoke-style word (#276)+ def test_karaoke_style_default_sweep_is_byte_identical_to_omitting_the_flag(self):+ """The mode selector must not change a single byte of today's \\kf output for a caller who+ never asks for --karaoke-style word: the default has to stay exactly 'sweep'."""+ ass_default = OUT / "kstyle_default.ass"+ ass_sweep = OUT / "kstyle_sweep.ass"+ script("caption.py", self.src, "--text", self.cues, "--karaoke", "--write-ass", ass_default,+ "--karaoke-timing", "even", "--preset", "veryfast", "-o", OUT / "kstyle_default.mp4")+ script("caption.py", self.src, "--text", self.cues, "--karaoke", "--karaoke-style", "sweep",+ "--karaoke-timing", "even", "--write-ass", ass_sweep, "--preset", "veryfast", "-o", OUT / "kstyle_sweep.mp4")+ self.assertEqual(ass_default.read_bytes(), ass_sweep.read_bytes(),+ "--karaoke-style sweep (explicit or default) must be byte-identical")+ self.assertIn("WrapStyle: 0", ass_default.read_text(encoding="utf-8-sig"),+ "sweep mode's header is unchanged from before this flag existed")++ def test_karaoke_style_word_emits_one_dialogue_event_per_word_covering_its_span(self):+ gappy = OUT / "gappy_kw.mp4"+ sh("ffmpeg", "-y", "-hide_banner", "-loglevel", "error",+ "-f", "lavfi", "-i", "aevalsrc='0.5*sin(2*PI*440*t)':s=48000",+ "-f", "lavfi", "-i", "testsrc2=size=640x360:rate=30", "-t", "4", "-c:v", "libx264",+ "-preset", "veryfast", "-c:a", "aac", gappy)+ cues = OUT / "kwcues.txt"+ cues.write_text("0:00-0:04 one two three four\n", encoding="utf-8")+ ass = OUT / "kword.ass"+ script("caption.py", gappy, "--text", cues, "--karaoke", "--karaoke-style", "word",+ "--karaoke-timing", "even", "--write-ass", ass, "--fast", "-o", OUT / "kword.mp4")+ text = ass.read_text(encoding="utf-8-sig")+ self.assertIn("WrapStyle: 2", text, "word mode freezes the line break: libass must not re-wrap")+ dialogues = [l for l in text.splitlines() if l.startswith("Dialogue:")]+ self.assertEqual(len(dialogues), 4, "one Dialogue event per word, not per cue")+ # even timing over 4 s / 4 words -> 1 s each+ expected_bounds = [("0:00:00.00", "0:00:01.00"), ("0:00:01.00", "0:00:02.00"),+ ("0:00:02.00", "0:00:03.00"), ("0:00:03.00", "0:00:04.00")]+ for line, (want_start, want_end) in zip(dialogues, expected_bounds):+ fields = line.split(",", 3)+ self.assertEqual(fields[1], want_start)+ self.assertEqual(fields[2], want_end)+ # every event still carries the WHOLE cue text, not just the active word+ for line in dialogues:+ for word in ("one", "two", "three", "four"):+ self.assertIn(word, line)++ def test_karaoke_style_word_active_word_is_scaled_and_a_distinct_colour(self):+ cues = OUT / "kwcues2.txt"+ cues.write_text("0:00-0:02 alpha beta\n", encoding="utf-8")+ ass = OUT / "kword2.ass"+ script("caption.py", self.src, "--text", cues, "--karaoke", "--karaoke-style", "word",+ "--karaoke-timing", "even", "--karaoke-scale", "130", "--highlight-color", "1EC3FC",+ "--color", "FFFFFF", "--upcoming-color", "808080", "--write-ass", ass,+ "--preset", "veryfast", "-o", OUT / "kword2.mp4")+ dialogues = [l for l in ass.read_text(encoding="utf-8-sig").splitlines() if l.startswith("Dialogue:")]+ self.assertEqual(len(dialogues), 2)+ first, second = dialogues+ self.assertIn(r"\fscx130", first)+ self.assertIn(r"\fscy130", first)+ self.assertIn("&H00FCC31E", first, "active word carries --highlight-color")+ self.assertIn("&H00808080", first, "the not-yet-spoken word carries --upcoming-color")+ self.assertIn("&H00FCC31E", second, "the second event's active word (beta) also gets --highlight-color")+ self.assertIn("&H00FFFFFF", second, "the already-spoken word (alpha) falls back to --color")++ def test_karaoke_style_word_preserves_explicit_line_breaks(self):+ """A cue with a manual two-line break (the `|` syntax) must keep exactly that \\N split in+ every per-word event -- the frozen line list from layout_cues, not a re-wrap."""+ cues = OUT / "kwcues3.txt"+ cues.write_text("0:00-0:02 top line | bottom line\n", encoding="utf-8")+ ass = OUT / "kword3.ass"+ script("caption.py", self.src, "--text", cues, "--karaoke", "--karaoke-style", "word",+ "--karaoke-timing", "even", "--write-ass", ass, "--preset", "veryfast", "-o", OUT / "kword3.mp4")+ dialogues = [l for l in ass.read_text(encoding="utf-8-sig").splitlines() if l.startswith("Dialogue:")]+ self.assertEqual(len(dialogues), 4, "top line + bottom line = 4 words -> 4 events")+ for line in dialogues:+ self.assertEqual(line.count("\\N"), 1, "each event keeps the one frozen line break")+ before, _, after = line.partition("\\N")+ self.assertTrue(any(w in before for w in ("top", "line")))+ self.assertTrue(any(w in after for w in ("bottom", "line")))++ def test_karaoke_style_word_handles_emoji_placeholders_and_brace_escaping(self):+ assets = _emoji_assets()+ cues = OUT / "kwcues4.txt"+ cues.write_text("0:00-0:02 say {hi} \U0001F389 now\n", encoding="utf-8")+ ass = OUT / "kword4.ass"+ script("caption.py", self.src, "--text", cues, "--karaoke", "--karaoke-style", "word",+ "--karaoke-timing", "even", "--emoji-assets", assets, "--write-ass", ass,+ "--preset", "veryfast", "-o", OUT / "kword4.mp4")+ text = ass.read_text(encoding="utf-8-sig")+ dialogues = [l for l in text.splitlines() if l.startswith("Dialogue:")]+ # "say", "{hi}" (escaped) and "now" get their own event; the emoji token is a placeholder+ # with no real duration of its own (same as --karaoke-style sweep's \kf0 emoji segment)+ self.assertEqual(len(dialogues), 3)+ self.assertNotIn("{\\pos", text, "cue text must not be able to open a real override block")+ self.assertIn(r"\{hi\}", text, "the braces the user typed are escaped, not dropped")+ for line in dialogues:+ self.assertNotIn("\U0001F389", line, "the emoji glyph itself is replaced by the drawtext/libass placeholder")+# ---------------------------------------------------------------- colour-flag filter-graph injection (adversarial)def test_color_like_flags_refuse_filter_graph_injection(self):"""Every flag that string-formats a colour straight into a filter graph (color=c=...,