AI-powered video editor that turns raw footage and a creative brief into a polished ad using an ensemble of AI agents (Google Gemini + FFmpeg)
Python
"""ASS caption generation from per-word timestamps."""
from __future__ import annotations
from pathlib import Path
from pysubs2 import Alignment, Color, SSAEvent, SSAFile, SSAStyle
from src.models.schemas import FootageIndex, Shot, WordTimestamp
_SHOT_MATCH_EPSILON = 1e-6
_MAX_WORDS_PER_PHRASE = 4
_MIN_WORDS_PER_PHRASE = 2
_MAX_CHARS_PER_PHRASE = 28
_MAX_PHRASE_DURATION_SECONDS = 1.8
_MAX_INTER_WORD_GAP_SECONDS = 0.45
_TERMINAL_PUNCTUATION = (".", "!", "?", ";")
_HIGHLIGHT_TAG = r"{\1c&H4ACFFF&}"
_RESET_TAG = r"{\r}"
def _require_file(path: str, kind: str) -> None:
if not Path(path).exists():
raise FileNotFoundError(f"{kind} not found: {path}")
def _format_tokens(tokens: list[str]) -> str:
"""Join caption tokens into readable text."""
if not tokens:
return ""
no_leading_space = {".", ",", "!", "?", ";", ":", "%", ")", "]", "}"}
no_trailing_space = {"(", "[", "{", "$", "#"}
parts: list[str] = []
for token in tokens:
if not token:
continue
if parts and token not in no_leading_space:
prev = parts[-1]
if prev not in no_trailing_space:
parts.append(" ")
parts.append(token)
return "".join(parts).strip()
def _resolve_shot(footage_index: FootageIndex, shot_id: str) -> Shot:
sep = shot_id.rfind("#")
if sep == -1:
raise ValueError(
f"shot_id {shot_id!r} is missing a '#' separator; expected "
"format '<source_file>#<start_time>'"
)
source_file = shot_id[:sep]
start_time = float(shot_id[sep + 1 :])
for shot in footage_index.shots:
if (
shot.source_file == source_file
and abs(shot.start_time - start_time) < _SHOT_MATCH_EPSILON
):
return shot
raise ValueError(f"shot_id {shot_id!r} not found in FootageIndex")
def _words_for_window(
shot: Shot,
clip_start: float,
clip_end: float,
) -> list[WordTimestamp]:
"""Select shot words whose midpoint lies within the trimmed clip."""
selected: list[WordTimestamp] = []
for word in shot.words:
midpoint = (word.start + word.end) / 2.0
if not (clip_start <= midpoint < clip_end):
continue
start = max(clip_start, word.start) - clip_start
end = min(clip_end, word.end) - clip_start
if end <= start:
continue
selected.append(
WordTimestamp(word=word.word, start=float(start), end=float(end))
)
return selected
def has_words_in_window(shot: Shot, clip_start: float, clip_end: float) -> bool:
"""Return True when a trimmed clip contains spoken words to caption."""
return bool(_words_for_window(shot, clip_start, clip_end))
def _should_break_phrase(
current: list[WordTimestamp],
next_word: WordTimestamp | None,
) -> bool:
if next_word is None:
return True
if len(current) < _MIN_WORDS_PER_PHRASE:
return False
current_text = _format_tokens([word.word for word in current])
current_duration = current[-1].end - current[0].start
gap = next_word.start - current[-1].end
last_token = current[-1].word
return (
len(current) >= _MAX_WORDS_PER_PHRASE
or len(current_text) >= _MAX_CHARS_PER_PHRASE
or current_duration >= _MAX_PHRASE_DURATION_SECONDS
or gap >= _MAX_INTER_WORD_GAP_SECONDS
or last_token.endswith(_TERMINAL_PUNCTUATION)
)
def _group_words_into_phrases(words: list[WordTimestamp]) -> list[list[WordTimestamp]]:
"""Group relative word timestamps into short readable caption phrases."""
if not words:
return []
phrases: list[list[WordTimestamp]] = []
current: list[WordTimestamp] = []
for index, word in enumerate(words):
current.append(word)
next_word = words[index + 1] if index + 1 < len(words) else None
if _should_break_phrase(current, next_word):
phrases.append(current)
current = []
if current:
phrases.append(current)
if len(phrases) >= 2 and len(phrases[-1]) == 1:
phrases[-2].extend(phrases.pop())
return phrases
def _highlighted_phrase_text(
phrase: list[WordTimestamp],
highlight_index: int,
) -> str:
tokens: list[str] = []
for index, word in enumerate(phrase):
token = word.word
if index == highlight_index:
token = f"{_HIGHLIGHT_TAG}{token}{_RESET_TAG}"
tokens.append(token)
return _format_tokens(tokens)
def _build_subtitle_file() -> SSAFile:
subs = SSAFile()
subs.info["PlayResX"] = "1080"
subs.info["PlayResY"] = "1920"
subs.styles["Default"] = SSAStyle(
fontname="Helvetica",
fontsize=44,
primarycolor=Color(255, 255, 255, 0),
secondarycolor=Color(74, 207, 255, 0),
outlinecolor=Color(0, 0, 0, 0),
backcolor=Color(0, 0, 0, 0),
bold=True,
outline=3,
shadow=0,
alignment=Alignment.BOTTOM_CENTER,
marginl=80,
marginr=80,
marginv=220,
)
return subs
def _to_ms(seconds: float) -> int:
return max(0, int(round(seconds * 1000)))
def generate_ass_captions(
footage_index_path: str,
shot_id: str,
clip_start: float,
clip_end: float,
output: str,
) -> str:
"""Write an ASS subtitle file for a trimmed clip using shot word timings."""
_require_file(footage_index_path, "Footage index")
if clip_end <= clip_start:
raise ValueError(
f"clip_end ({clip_end}) must be greater than clip_start ({clip_start})"
)
index = FootageIndex.model_validate_json(
Path(footage_index_path).read_text(encoding="utf-8")
)
shot = _resolve_shot(index, shot_id)
words = _words_for_window(shot, clip_start, clip_end)
if not words:
raise ValueError(
f"shot_id {shot_id!r} has no captionable words in "
f"[{clip_start}, {clip_end})"
)
subs = _build_subtitle_file()
for phrase in _group_words_into_phrases(words):
phrase_end = phrase[-1].end
for index, word in enumerate(phrase):
next_start = phrase[index + 1].start if index + 1 < len(phrase) else phrase_end
end_time = max(word.end, next_start)
subs.events.append(
SSAEvent(
start=_to_ms(word.start),
end=_to_ms(end_time),
text=_highlighted_phrase_text(phrase, index),
)
)
output_path = Path(output)
output_path.parent.mkdir(parents=True, exist_ok=True)
subs.save(str(output_path))
return str(output_path)
def burn_ass_subtitles(video: str, subtitles: str, output: str) -> str:
"""Burn an ASS subtitle file into a video via FFmpeg's ass filter."""
import subprocess
_require_file(video, "Video")
_require_file(subtitles, "Subtitle file")
output_path = Path(output)
output_path.parent.mkdir(parents=True, exist_ok=True)
subtitle_path = str(Path(subtitles).resolve()).replace("\\", r"\\").replace(":", r"\:")
cmd = [
"ffmpeg",
"-y",
"-i",
video,
"-vf",
f"ass={subtitle_path}",
"-c:a",
"copy",
str(output_path),
]
try:
subprocess.run(cmd, check=True, capture_output=True, text=True)
except subprocess.CalledProcessError as exc:
raise RuntimeError(f"FFmpeg failed: {exc.stderr}") from exc
return str(output_path)