Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.
Python ai-filmmakingai-videoai-video-generatorapple-siliconhailuoimage-to-videolip-synclocal-ailora-trainingltx-2ltx-2-5ltx-videomacosminimax-h3mlxmusic-generationoffline-aipinokiotext-to-videovideo-generation
"""Subtitles - captions for a clip or a cut, timed to the audio.
Phosphene usually KNOWS the words: an H3 prompt carries its dialogue in
`<d>[English] ...</d>`, an LTX prompt in `says, "..."`, a music job carries its
lyrics. So the words come from the script and only the timing comes from the
audio. Whisper (large-v3-turbo on MLX) is the ear: it hears the clip with word
times, the script's words are matched to what it heard, and each script line is
then snapped to the audio by forced alignment (Whisper's own cross-attention,
fed the known words, the same model, no second download). A line Whisper did
not hear at all (a sung line under loud music) is still placed: it is aligned
inside the gap between the lines around it. A clip with no script falls back to
Whisper's own words.
Pure logic + a CLI; the panel calls it in a subprocess of an env that has
mlx_whisper (`python subtitles.py make CLIP --prompt-file .. --out ..`).
make clip -> cues.json + .srt + .vtt (+ --burn out.mp4)
burn clip + cues.json -> out.mp4 (libass when ffmpeg has it, else cards)
"""
from __future__ import annotations
import argparse
import difflib
import json
import os
import re
import shutil
import subprocess
import sys
import time
import unicodedata
from dataclasses import asdict, dataclass, field
from pathlib import Path
MODEL = os.environ.get("PHOSPHENE_SUBS_MODEL", "mlx-community/whisper-large-v3-turbo")
SR = 16000
MAX_CHARS = 42 # per caption row (Netflix-style limit)
MIN_CUE = 1.0 # s; a shorter cue is held longer when there is room
HOLD = 0.35 # s after the last word, so the eye can finish reading
LEAD = 0.08 # s before the first word
# ---------------------------------------------------------------- the script
_D_TAG = re.compile(r"<d>\s*(?:\[([^\]]{1,24})\])?\s*(.*?)\s*</d>", re.S | re.I)
_SAYS = re.compile(
r"\b(says|said|asks|asked|replies|whispers|shouts|calls|announces|murmurs|mutters|sings|sang|chants)\b"
r"[^\"“]{0,48}[\"“]([^\"”]{1,400})[\"”]", re.I)
# the same, single-quoted: "sings the words 'He was soft, so we could be hard'"
_SAYS1 = re.compile(
r"\b(says|said|asks|asked|replies|whispers|shouts|calls|announces|murmurs|mutters|sings|sang|chants)\b"
r"[^\"'\u2018\u201c]{0,48}(?<![A-Za-z])['\u2018]([^'\u2019]{1,400})['\u2019](?=[\s,.;:!?)]|$)", re.I)
_SECTION = re.compile(r"^\s*[\[(][^\])]*[\])]\s*$")
@dataclass
class Line:
text: str
kind: str = "say" # say | song
lang: str | None = None # from <d>[Latin]
translation: str | None = None
def script_from_prompt(prompt: str) -> list[Line]:
"""Dialogue lines in prompt order. H3 `<d>[Lang] text</d>` first; else LTX
`he says, "text"`. A sung verb marks the line as a song."""
out: list[Line] = []
for lang, text in _D_TAG.findall(prompt or ""):
text = " ".join(text.split())
if text:
out.append(Line(text, "say", (lang or "").strip() or None))
if out:
return out
found = sorted([(m.start(), m.group(1), m.group(2)) for rx in (_SAYS, _SAYS1) for m in rx.finditer(prompt or "")])
for _, verb, text in found:
text = " ".join(text.split())
if text:
out.append(Line(text, "song" if verb.lower() in ("sings", "sang", "chants") else "say"))
return out
def script_from_lyrics(lyrics: str, translations: dict | None = None) -> list[Line]:
"""Lyrics -> one song line per sung line; [Verse] tags and blank lines go."""
out = []
for raw in (lyrics or "").splitlines():
t = raw.strip()
if not t or _SECTION.match(t):
continue
out.append(Line(t, "song", translation=(translations or {}).get(t)))
return out
# ---------------------------------------------------------------- matching
def norm(w: str) -> str:
w = unicodedata.normalize("NFKD", w)
return re.sub(r"[^a-z0-9']", "", "".join(c for c in w if not unicodedata.combining(c)).lower())
def _sim(a: str, b: str) -> float:
if a == b:
return 1.0
if not a or not b:
return 0.0
return difflib.SequenceMatcher(None, a, b).ratio()
def match_words(script: list[str], heard: list[str]) -> list[int | None]:
"""For each script word, the index of the heard word it is, or None.
LOCAL alignment (Smith-Waterman): free skips at both ends of BOTH sides,
because a clip may hold more audio than its script and a cut may use only
part of a song's lyrics (the verse, not the chorus). Gaps inside cost a
little; fuzzy word similarity lets 'proxies'/'proxy', 'wi-fi'/'wifi' match.
Below that, one stretch of the script is matched; `match_all` repeats it
on what is left so two separate heard passages both land."""
n, m = len(script), len(heard)
out: list[int | None] = [None] * n
if not n or not m:
return out
GAP = -0.35
S = [[0.0] * (m + 1) for _ in range(n + 1)]
B = [[0] * (m + 1) for _ in range(n + 1)] # 0 stop, 1 diag, 2 up, 3 left
best, bi, bj = 0.0, 0, 0
for i in range(1, n + 1):
a = script[i - 1]
for j in range(1, m + 1):
sm = _sim(a, heard[j - 1])
d = S[i - 1][j - 1] + (2.0 * sm - 0.8 if sm >= 0.55 else -1.0)
u, l = S[i - 1][j] + GAP, S[i][j - 1] + GAP
v, k = max((0.0, 0), (d, 1), (u, 2), (l, 3))
S[i][j], B[i][j] = v, k
if v > best:
best, bi, bj = v, i, j
i, j = bi, bj
while i > 0 and j > 0 and B[i][j]:
k = B[i][j]
if k == 1:
if _sim(script[i - 1], heard[j - 1]) >= 0.55:
out[i - 1] = j - 1
i, j = i - 1, j - 1
elif k == 2:
i -= 1
else:
j -= 1
return out
def match_all(script: list[str], heard: list[str]) -> list[int | None]:
"""match_words, then again on the script words before and after the
matched stretch against the heard words before and after it."""
out = match_words(script, heard)
hit = [k for k, v in enumerate(out) if v is not None]
if not hit:
return out
k0, k1 = hit[0], hit[-1]
h0, h1 = out[k0], out[k1]
if k0 > 0 and h0 > 0:
left = match_all(script[:k0], heard[:h0])
out[:k0] = left
if k1 + 1 < len(script) and h1 + 1 < len(heard):
right = match_all(script[k1 + 1:], heard[h1 + 1:])
out[k1 + 1:] = [None if v is None else v + h1 + 1 for v in right]
return out
# ---------------------------------------------------------------- the ear
class Ear:
"""Whisper on MLX: transcribe with word times, and force-align known text."""
def __init__(self, model: str = MODEL):
import mlx.core as mx
import mlx_whisper
from mlx_whisper import audio as wa
from mlx_whisper.load_models import load_model
from mlx_whisper.tokenizer import get_tokenizer
self.mx, self.mw, self.wa = mx, mlx_whisper, wa
self.repo = model
t = time.time()
self.model = load_model(model, dtype=mx.float16)
self.load_s = time.time() - t
self._tok = {}
self._get_tokenizer = get_tokenizer
def tokenizer(self, lang: str | None):
key = lang or "en"
if key not in self._tok:
self._tok[key] = self._get_tokenizer(self.model.is_multilingual, num_languages=self.model.num_languages,
language=key, task="transcribe")
return self._tok[key]
def hear(self, audio, language: str | None = None) -> tuple[list[dict], str]:
"""Words with times. No initial_prompt: feeding the script as a prompt
made it loop ("66.32 66.32 ...") on SUNDAY MASS; the silence threshold
stops it inventing words in the gaps."""
from mlx_whisper.transcribe import ModelHolder
ModelHolder.model, ModelHolder.model_path = self.model, self.repo # reuse, no reload
r = self.mw.transcribe(audio, path_or_hf_repo=self.repo, word_timestamps=True, language=language,
condition_on_previous_text=False, hallucination_silence_threshold=2.0,
temperature=(0.0, 0.2, 0.4), fp16=True)
words = [{"w": w["word"].strip(), "s": float(w["start"]), "e": float(w["end"]), "p": float(w.get("probability", 0))}
for seg in r.get("segments", []) for w in seg.get("words", []) if w["word"].strip()]
return words, r.get("language") or language or "en"
def force(self, audio, t0: float, t1: float, text: str, lang: str | None) -> list[dict]:
"""Word times of KNOWN text inside [t0, t1] (seconds). Whisper's
alignment heads over the decoder fed the given tokens, then DTW: the
same method its own word timestamps use, but on our words."""
import numpy as np
from mlx_whisper.timing import find_alignment
wa, mx = self.wa, self.mx
a0, a1 = max(0, int(t0 * SR)), min(len(audio), int(t1 * SR))
seg = audio[a0:a1]
if len(seg) < SR * 0.2:
return []
n_frames = len(seg) // wa.HOP_LENGTH
mel = wa.log_mel_spectrogram(np.asarray(seg, dtype=np.float32), n_mels=self.model.dims.n_mels,
padding=wa.N_SAMPLES)
mel = wa.pad_or_trim(mel, wa.N_FRAMES, axis=-2).astype(mx.float16)
tok = self.tokenizer(lang)
toks = tok.encode(" " + text.strip())
al = find_alignment(self.model, tok, toks, mel, min(n_frames, wa.N_FRAMES))
return [{"w": a.word.strip(), "s": t0 + float(a.start), "e": t0 + float(a.end), "p": float(a.probability)}
for a in al if a.word.strip()]
def load_audio(path: str):
import numpy as np
raw = subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-i", str(path), "-vn", "-ac", "1", "-ar", str(SR),
"-f", "s16le", "-"], capture_output=True, check=True).stdout
return np.frombuffer(raw, np.int16).astype(np.float32) / 32768.0
# ---------------------------------------------------------------- cues
@dataclass
class Cue:
start: float
end: float
text: str
kind: str = "say"
translation: str | None = None
source: str = "script" # script | heard
sure: bool = True # False: placed in a gap, not confirmed by the ear
words: list = field(default_factory=list)
def _spans(lines, sw, owner, idx, heard):
out = []
for li in range(len(lines)):
hits = [heard[idx[k]] for k in range(len(sw)) if owner[k] == li and idx[k] is not None]
nwords = sum(1 for o in owner if o == li)
out.append([hits[0]["s"], hits[-1]["e"], len(hits) / max(1, nwords), hits]
if hits and len(hits) >= max(1, round(0.34 * nwords)) else None)
return out
def _free_gap(heard, a, b):
"""The longest stretch of [a, b] no heard word sits in."""
pts = sorted((max(a, h["s"]), min(b, h["e"])) for h in heard if h["e"] > a and h["s"] < b)
best, cur = (a, a), a
for s0, e0 in pts:
if s0 - cur > best[1] - best[0]:
best = (cur, s0)
cur = max(cur, e0)
if b - cur > best[1] - best[0]:
best = (cur, b)
return best
def align_script(ear: Ear, audio, lines: list[Line], heard: list[dict], lang: str | None,
refine: bool = True, lo: float = 0.0, hi: float | None = None) -> list[Cue]:
"""Known lines -> cues. `lo`/`hi` bound the search to where this source
plays (a clip's slot in a cut, the span of the music bed).
1. Match the script's words to the words Whisper heard (match_all): a line
with enough matched words is timed by them. Measured on SUNDAY MASS this
is the most accurate timing we have (median 0.06 s from hand-placed).
2. A line it did not hear, BETWEEN heard lines or after them, gets a second
listen: Whisper again on just the free gap (sung lines under music are
often heard on a short window when the long pass missed them), and the
match is redone with those words.
3. Still nothing: forced alignment of the known words in the gap. Whisper's
DTW stretches the first word over any music before it, so the time is a
guess: the cue is kept, marked unsure, shown for review, never burned.
`refine=False` stops after step 1 (the bench's "script-match")."""
dur = len(audio) / SR if hi is None else min(hi, len(audio) / SR)
heard = [h for h in heard if h["e"] > lo and h["s"] < dur]
sw, owner = [], []
for li, ln in enumerate(lines):
for w in ln.text.split():
if norm(w):
sw.append(norm(w))
owner.append(li)
idx = match_all(sw, [norm(h["w"]) for h in heard])
spans = _spans(lines, sw, owner, idx, heard)
if refine and any(spans) and not all(spans):
# step 2: listen again in each free gap that holds an unheard line after the first heard one
first = next(li for li, sp in enumerate(spans) if sp)
extra = []
for li in range(first + 1, len(lines)):
if spans[li]:
continue
p = next((spans[k][1] for k in range(li - 1, -1, -1) if spans[k]), lo)
n_ = next((spans[k][0] for k in range(li + 1, len(lines)) if spans[k]), dur)
g0, g1 = _free_gap(heard + extra, p, n_)
if g1 - g0 >= 0.8:
a0, a1 = int(max(0, g0 - 0.15) * SR), int(min(dur, g1 + 0.15) * SR)
words, _ = ear.hear(audio[a0:a1], lang)
extra += [dict(w, s=w["s"] + a0 / SR, e=w["e"] + a0 / SR) for w in words]
if extra:
heard = sorted(heard + extra, key=lambda h: h["s"])
idx = match_all(sw, [norm(h["w"]) for h in heard])
spans = _spans(lines, sw, owner, idx, heard)
cues: list[Cue] = []
first = next((li for li, sp in enumerate(spans) if sp), None)
for li, ln in enumerate(lines):
sp = spans[li]
if sp:
cues.append(Cue(sp[0], sp[1], ln.text, ln.kind, ln.translation, "script", True,
[{"w": h["w"], "s": round(h["s"], 3), "e": round(h["e"], 3)} for h in sp[3]]))
continue
if not refine or first is None or li < first:
continue # before anything was heard: not in this cut (the verse before the chorus)
p = next((spans[k][1] for k in range(li - 1, -1, -1) if spans[k]), lo)
n_ = next((spans[k][0] for k in range(li + 1, len(lines)) if spans[k]), dur)
g0, g1 = _free_gap(heard, p, n_)
nw = len(ln.text.split())
if g1 - g0 < 0.3 * nw + 0.3:
continue # no room for it: it is not in the audio
words = ear.force(audio, g0, g1, ln.text, ln.lang and _lang_code(ln.lang) or lang)
if not words or sum(w["p"] for w in words) / len(words) < 0.2:
continue
cues.append(Cue(words[0]["s"], words[-1]["e"], ln.text, ln.kind, ln.translation, "script", False,
[dict(w, s=round(w["s"], 3), e=round(w["e"], 3)) for w in words]))
return cues
def drop_unsure(cues: list[Cue]) -> list[Cue]:
"""What is burned in: lines the ear confirmed. A line only placed in a gap
stays in the cue file (the editor shows it for a look) but is not burned."""
return [c for c in cues if c.sure]
def cues_from_heard(heard: list[dict], kind: str = "say") -> list[Cue]:
"""No script: group Whisper's words into readable cues (sentence end,
a pause over 0.6 s, or ~2 rows of text)."""
cues, cur = [], []
for i, w in enumerate(heard):
cur.append(w)
text = " ".join(x["w"] for x in cur)
nxt = heard[i + 1] if i + 1 < len(heard) else None
brk = (nxt is None or nxt["s"] - w["e"] > 0.6 or re.search(r"[.!?…]$", w["w"]) or len(text) > 2 * MAX_CHARS - 10
or (w["e"] - cur[0]["s"]) > 5.5)
if brk:
cues.append(Cue(cur[0]["s"], cur[-1]["e"], text, kind, None, "heard", True,
[dict(x) for x in cur]))
cur = []
return cues
def _lang_code(name: str) -> str | None:
from mlx_whisper.tokenizer import TO_LANGUAGE_CODE, LANGUAGES
n = name.strip().lower()
return n if n in LANGUAGES else TO_LANGUAGE_CODE.get(n)
def _settle(cues: list[Cue]) -> list[Cue]:
"""Two sources speaking at once (a line over the choir): the spoken line
wins the screen, the sung one is cut back around it."""
cues = sorted(cues, key=lambda c: c.start)
out: list[Cue] = []
for c in cues:
if out and c.start < out[-1].end:
p = out[-1]
if norm(p.text.replace(" ", "")) == norm(c.text.replace(" ", "")):
# the same line from two sources (the song's lyrics and the shot's prompt)
if (c.sure, c.end - c.start) > (p.sure, p.end - p.start):
out[-1] = c
continue
if p.kind == "song" and c.kind == "say":
p.end = c.start
if p.end - p.start < 0.4:
out.pop()
elif c.kind == "song" and p.kind == "say":
c.start = p.end
if c.end - c.start < 0.4:
continue
out.append(c)
return out
def finish(cues: list[Cue], dur: float) -> list[Cue]:
"""Lead-in, read-out hold, minimum length, no overlaps."""
cues = sorted(cues, key=lambda c: c.start)
for i, c in enumerate(cues):
nxt = cues[i + 1].start if i + 1 < len(cues) else dur
prv = cues[i - 1].end if i else 0.0
c.start = max(prv, c.start - LEAD, 0.0)
c.end = min(max(c.end + HOLD, c.start + MIN_CUE), max(c.end, nxt - 0.04), dur)
c.start, c.end = round(c.start, 3), round(c.end, 3)
return cues
def wrap(text: str, width: int = MAX_CHARS) -> list[str]:
"""One row if it fits, else two balanced rows (bottom-heavy)."""
if len(text) <= width:
return [text]
words = text.split()
best, cut = None, 1
for k in range(1, len(words)):
a, b = " ".join(words[:k]), " ".join(words[k:])
score = abs(len(a) - len(b)) + (0 if len(a) <= len(b) else 3) + (0 if re.search(r"[,.;:!?]$", words[k - 1]) else 4)
if best is None or score < best:
best, cut = score, k
return [" ".join(words[:cut]), " ".join(words[cut:])]
# ---------------------------------------------------------------- files
def _ts(t: float, sep: str) -> str:
ms = int(round(t * 1000))
return "%02d:%02d:%02d%s%03d" % (ms // 3600000, ms // 60000 % 60, ms // 1000 % 60, sep, ms % 1000)
def to_srt(cues: list[Cue], notes: bool = True) -> str:
out = []
for i, c in enumerate(cues, 1):
body = "\n".join(wrap(c.text))
if c.kind == "song" and notes:
body = "♪ " + body + " ♪"
if c.translation:
body += "\n(" + c.translation + ")"
out.append("%d\n%s --> %s\n%s\n" % (i, _ts(c.start, ","), _ts(c.end, ","), body))
return "\n".join(out)
def to_vtt(cues: list[Cue]) -> str:
out = ["WEBVTT", ""]
for c in cues:
body = "\n".join(wrap(c.text))
if c.kind == "song":
body = "<i>♪ " + body + " ♪</i>"
if c.translation:
body += "\n<i>" + c.translation + "</i>"
out += ["%s --> %s line:88%%" % (_ts(c.start, "."), _ts(c.end, ".")), body, ""]
return "\n".join(out)
# The look that worked on SUNDAY MASS: white dialogue, italic cream sung lines
# between notes, a smaller translation underneath, soft shadow, bottom centre.
STYLE = {"font": "Helvetica Neue", "size": 0.047, "margin": 0.075, "say": "&H00FFFFFF", "song": "&H00C8E8F6",
"tr": "&H10D8E4EA", "outline": 0.0022, "shadow": 0.0028}
def to_ass(cues: list[Cue], w: int, h: int, style: dict | None = None) -> str:
st = dict(STYLE, **(style or {}))
fs, mv = round(h * st["size"]), round(h * st["margin"])
ol, sh = max(1, round(h * st["outline"] * 1.6)), max(1, round(h * st["shadow"] * 1.6))
head = ("[Script Info]\nScriptType: v4.00+\nPlayResX: %d\nPlayResY: %d\nWrapStyle: 2\nScaledBorderAndShadow: yes\n\n"
"[V4+ Styles]\nFormat: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, "
"Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, "
"MarginL, MarginR, MarginV, Encoding\n") % (w, h)
row = "Style: %s,%s,%d,%s,&H000000FF,&HA0000000,&H96000000,0,%d,0,0,100,100,0,0,1,%d,%d,2,%d,%d,%d,1\n"
head += row % ("Say", st["font"], fs, st["say"], 0, ol, sh, mv, mv, mv)
head += row % ("Song", st["font"], fs, st["song"], 1, ol, sh, mv, mv, mv)
head += row % ("Tr", st["font"], round(fs * 0.74), st["tr"], 1, ol, sh, mv, mv, mv)
head += "\n[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n"
def t(x):
cs = int(round(x * 100))
return "%d:%02d:%02d.%02d" % (cs // 360000, cs // 6000 % 60, cs // 100 % 60, cs % 100)
ev = []
for c in cues:
body = r"\N".join(wrap(c.text))
if c.kind == "song":
body = "♪ " + body + " ♪"
lift = mv
if c.translation:
ev.append("Dialogue: 0,%s,%s,Tr,,0,0,%d,,%s" % (t(c.start), t(c.end), mv, c.translation))
lift = mv + round(fs * 0.74 * 1.25)
ev.append("Dialogue: 0,%s,%s,%s,,0,0,%d,,%s" % (t(c.start), t(c.end), "Song" if c.kind == "song" else "Say",
lift, body))
return head + "\n".join(ev) + "\n"
def ffmpeg_has_libass(ff: str) -> bool:
try:
out = subprocess.run([ff, "-hide_banner", "-filters"], capture_output=True, text=True, timeout=10).stdout
except Exception: # noqa: BLE001
return False
return bool(re.search(r"^\s*\S+\s+subtitles\s", out, re.M))
def burn(src: str, cues: list[Cue], dst: str, ff: str | None = None, style: dict | None = None) -> str:
"""Burn the cues in. libass when this ffmpeg has it (the Mac app's and
Pinokio's both do); otherwise a still card per cue laid over the picture."""
ff = ff or os.environ.get("LTX_FFMPEG") or shutil.which("ffmpeg")
w, h = map(int, subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
"stream=width,height", "-of", "csv=p=0", src], capture_output=True, text=True,
check=True).stdout.strip().split(",")[:2])
work = Path(dst).with_suffix(".subs")
work.mkdir(exist_ok=True)
enc = ["-c:v", "libx264", "-crf", "17", "-preset", "medium", "-pix_fmt", "yuv420p", "-c:a", "copy",
"-movflags", "+faststart"]
if ffmpeg_has_libass(ff):
ass = work / "subs.ass"
ass.write_text(to_ass(cues, w, h, style), encoding="utf-8")
esc = str(ass).replace("\\", "\\\\").replace(":", "\\:").replace("'", "\\'")
vf = "subtitles='%s':fontsdir=/System/Library/Fonts" % esc
subprocess.run([ff, "-nostdin", "-v", "error", "-y", "-i", src, "-vf", vf, *enc, dst], check=True)
return "libass"
# fallback: PNG cards
from PIL import Image, ImageDraw, ImageFilter, ImageFont
ins, chain, last = [], [], "0:v"
for i, c in enumerate(cues):
p = work / ("cue_%03d.png" % i)
_card(c, w, h, Image, ImageDraw, ImageFilter, ImageFont).save(p)
ins += ["-i", str(p)]
chain.append("[%s][%d:v]overlay=0:0:enable='between(t,%.3f,%.3f)'[v%d]" % (last, i + 1, c.start, c.end, i))
last = "v%d" % i
if not cues:
shutil.copyfile(src, dst)
return "none"
subprocess.run([ff, "-nostdin", "-v", "error", "-y", "-i", src, *ins, "-filter_complex", ";".join(chain),
"-map", "[%s]" % last, "-map", "0:a?", *enc, dst], check=True)
return "cards"
def _card(c, w, h, Image, ImageDraw, ImageFilter, ImageFont):
F = "/System/Library/Fonts/"
s = h / 720
main = ImageFont.truetype(F + ("SFNSItalic.ttf" if c.kind == "song" else "SFNS.ttf"), round(34 * s))
small = ImageFont.truetype(F + "SFNSItalic.ttf", round(22 * s))
col = (246, 232, 200, 255) if c.kind == "song" else (255, 255, 255, 255)
rows = wrap(c.text)
if c.kind == "song":
rows[0], rows[-1] = "♪ " + rows[0], rows[-1] + " ♪"
img = Image.new("RGBA", (w, h))
d = ImageDraw.Draw(img)
y = h - round((96 if c.translation else 70) * s) - (len(rows) - 1) * round(40 * s)
for r in rows:
d.text((w / 2, y), r, font=main, fill=col, anchor="ms")
y += round(40 * s)
if c.translation:
d.text((w / 2, y - round(8 * s)), c.translation, font=small, fill=(220, 214, 200, 235), anchor="ms")
a = img.getchannel("A")
shadow = Image.new("RGBA", (w, h), (0, 0, 0, 0))
shadow.putalpha(a.filter(ImageFilter.MaxFilter(5)).filter(ImageFilter.GaussianBlur(3 * s)).point(lambda v: int(v * .85)))
return Image.alpha_composite(shadow, img)
# ---------------------------------------------------------------- CLI
def make(clip: str, lines: list[Line], out: str, language: str | None = None, ear: Ear | None = None,
refine: bool = True, sources: list | None = None) -> dict:
"""`lines` for one clip; `sources` = [(t0, t1, lines), ...] for a cut, one
per clip with a script and one per music bed with lyrics."""
t0 = time.time()
ear = ear or Ear()
audio = load_audio(clip)
dur = len(audio) / SR
t1 = time.time()
heard, lang = ear.hear(audio, language)
t2 = time.time()
if sources:
cues = []
for a, b, ls in sources:
cues += align_script(ear, audio, ls, heard, language or lang, refine=refine, lo=a, hi=b)
cues = _settle(cues)
mode = "script"
elif lines:
cues = align_script(ear, audio, lines, heard, language or lang, refine=refine)
mode = "script" if refine else "script-match"
else:
cues = cues_from_heard(heard)
mode = "heard"
cues = finish(cues, dur)
t3 = time.time()
base = Path(out)
base.parent.mkdir(parents=True, exist_ok=True)
rep = {"clip": str(clip), "mode": mode, "language": lang, "duration": round(dur, 2), "model": ear.repo,
"timing": {"load_model": round(ear.load_s, 2), "audio": round(t1 - t0, 2), "hear": round(t2 - t1, 2),
"align": round(t3 - t2, 2)},
"heard": heard, "cues": [asdict(c) for c in cues]}
base.with_suffix(".json").write_text(json.dumps(rep, indent=1, ensure_ascii=False))
base.with_suffix(".srt").write_text(to_srt(cues), encoding="utf-8")
base.with_suffix(".vtt").write_text(to_vtt(cues), encoding="utf-8")
return rep
def _cues(rep: dict) -> list[Cue]:
return [Cue(**c) for c in rep["cues"]]
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
sub = ap.add_subparsers(dest="cmd", required=True)
m = sub.add_parser("make")
m.add_argument("clip")
m.add_argument("--out", required=True, help="base path; writes .json .srt .vtt")
m.add_argument("--prompt", default="")
m.add_argument("--prompt-file")
m.add_argument("--lyrics-file")
m.add_argument("--translations", help="json {line: translation}")
m.add_argument("--language")
m.add_argument("--no-script", action="store_true", help="ignore the script: Whisper's words only")
m.add_argument("--no-refine", action="store_true", help="script words timed by matching only")
m.add_argument("--burn", help="also write a captioned copy here")
m.add_argument("--source", action="append", default=[],
help="T0:T1:prompt=FILE or T0:T1:lyrics=FILE[:tr=FILE] (a cut: repeat per clip / music bed)")
b = sub.add_parser("burn")
b.add_argument("clip")
b.add_argument("cues")
b.add_argument("dst")
b.add_argument("--cards", action="store_true", help="force the PNG-card route")
a = ap.parse_args(argv)
if a.cmd == "make":
prompt = Path(a.prompt_file).read_text() if a.prompt_file else a.prompt
tr = json.loads(Path(a.translations).read_text()) if a.translations else None
lines = [] if a.no_script else script_from_prompt(prompt)
if a.lyrics_file and not a.no_script:
lines += script_from_lyrics(Path(a.lyrics_file).read_text(), tr)
sources = []
for spec in a.source:
t0, t1, rest = spec.split(":", 2)
parts = dict(p.split("=", 1) for p in rest.split(":"))
if "prompt" in parts:
ls = script_from_prompt(Path(parts["prompt"]).read_text())
else:
trs = json.loads(Path(parts["tr"]).read_text()) if "tr" in parts else None
ls = script_from_lyrics(Path(parts["lyrics"]).read_text(), trs)
if ls:
sources.append((float(t0), float(t1), ls))
rep = make(a.clip, lines, a.out, a.language, refine=not a.no_refine, sources=sources or None)
print(json.dumps({k: rep[k] for k in ("mode", "language", "duration", "timing")}))
for c in rep["cues"]:
print("%7.2f %7.2f %-5s %s %s" % (c["start"], c["end"], c["kind"], "" if c["sure"] else "?", c["text"]))
if a.burn:
print("burned with", burn(a.clip, drop_unsure(_cues(rep)), a.burn))
else:
rep = json.loads(Path(a.cues).read_text())
print("burned with", burn(a.clip, _cues(rep), a.dst, ff="/opt/homebrew/bin/ffmpeg" if a.cards else None))
if __name__ == "__main__":
sys.exit(main())