Skip to content

AI-powered video editor that turns raw footage and a creative brief into a polished ad using an ensemble of AI agents (Google Gemini + FFmpeg)

Python
9.0 KB
Raw Blame
"""Pre-processing pipeline: scene detection + transcription.

Walks an input directory of video files, splits each clip into shots using
PySceneDetect, transcribes audio with Faster-Whisper, aligns word-level
timestamps to shot boundaries, and emits a serialized FootageIndex JSON.
"""

from __future__ import annotations

import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import TYPE_CHECKING

from src.models.schemas import FootageIndex, Shot, WordTimestamp

if TYPE_CHECKING:
    from faster_whisper import WhisperModel

VIDEO_SUFFIXES = {".mov", ".mp4", ".m4v", ".mkv"}

#: Directory name patterns used to auto-detect roll type.
_ROLL_TYPE_PATTERNS: dict[str, str] = {
    "a-roll": "a-roll",
    "a_roll": "a-roll",
    "aroll": "a-roll",
    "b-roll": "b-roll",
    "b_roll": "b-roll",
    "broll": "b-roll",
}


def _detect_roll_type(video_path: Path) -> str:
    """Infer 'a-roll', 'b-roll', or 'unknown' from parent directory name."""
    for parent in video_path.parents:
        name_lower = parent.name.lower().replace(" ", "")
        if name_lower in _ROLL_TYPE_PATTERNS:
            return _ROLL_TYPE_PATTERNS[name_lower]
    return "unknown"


def _log(msg: str) -> None:
    """Emit progress to stderr."""
    print(msg, file=sys.stderr, flush=True)


def _detect_shots(
    video_path: Path, scene_threshold: float = 27.0
) -> list[tuple[float, float]]:
    """Detect shots in a video using PySceneDetect ContentDetector.

    Args:
        video_path: Path to the video file.
        scene_threshold: ContentDetector threshold (default 27.0).

    Returns:
        List of (start_seconds, end_seconds) tuples. If no cuts are detected,
        returns a single shot spanning the entire video duration.
    """
    from scenedetect import ContentDetector, SceneManager, open_video

    video = open_video(str(video_path))
    scene_manager = SceneManager()
    scene_manager.add_detector(ContentDetector(threshold=scene_threshold))
    scene_manager.detect_scenes(video, show_progress=False)
    scene_list = scene_manager.get_scene_list()

    if not scene_list:
        try:
            duration = video.duration.get_seconds()
        except Exception:
            duration = 0.0
        return [(0.0, float(duration))]

    return [
        (float(start.get_seconds()), float(end.get_seconds()))
        for start, end in scene_list
    ]


def _transcribe_words(
    model: WhisperModel, video_path: Path
) -> list[tuple[float, float, str]]:
    """Transcribe a video and return word-level timestamps.

    Args:
        model: A pre-loaded Faster-Whisper model.
        video_path: Path to the video file.

    Returns:
        List of (start_seconds, end_seconds, word_text) tuples across the
        entire file. Returns empty list if no speech is detected.
    """
    segments, _info = model.transcribe(
        str(video_path), word_timestamps=True, temperature=0
    )
    words: list[tuple[float, float, str]] = []
    for segment in segments:
        seg_words = getattr(segment, "words", None) or []
        for w in seg_words:
            if w.start is None or w.end is None:
                continue
            words.append((float(w.start), float(w.end), str(w.word)))
    return words


def _normalize_word_text(text: str) -> str:
    """Collapse whitespace in a transcribed word while keeping punctuation."""
    return " ".join(text.split())


def _words_to_text(words: list[WordTimestamp]) -> str:
    """Render a token stream into readable caption/transcript text."""
    if not words:
        return ""

    no_leading_space = {".", ",", "!", "?", ";", ":", "%", ")", "]", "}"}
    no_trailing_space = {"(", "[", "{", "$", "#"}

    parts: list[str] = []
    for word in words:
        token = word.word
        if not token:
            continue
        if parts and token not in no_leading_space:
            prev = parts[-1]
            if prev not in no_trailing_space:
                parts.append(" ")
        parts.append(token)
    return "".join(parts).strip()


def _words_for_shot(
    words: list[tuple[float, float, str]],
    shot_start: float,
    shot_end: float,
) -> list[WordTimestamp]:
    """Return word timestamps whose midpoint falls within the shot window."""
    selected: list[WordTimestamp] = []
    for w_start, w_end, text in words:
        midpoint = (w_start + w_end) / 2.0
        if shot_start <= midpoint < shot_end:
            normalized = _normalize_word_text(text)
            if not normalized:
                continue
            start = max(shot_start, w_start)
            end = min(shot_end, w_end)
            if end <= start:
                continue
            selected.append(
                WordTimestamp(word=normalized, start=float(start), end=float(end))
            )
    return selected


def _process_video(
    model: WhisperModel,
    video_path: Path,
    scene_threshold: float = 27.0,
) -> list[Shot]:
    """Detect shots in a video and assemble Shot objects with aligned transcripts."""
    shot_ranges = _detect_shots(video_path, scene_threshold=scene_threshold)
    try:
        words = _transcribe_words(model, video_path)
    except Exception as exc:
        _log(f"[preprocess]   no transcript for {video_path.name}: {exc}")
        words = []

    roll_type = _detect_roll_type(video_path)
    shots: list[Shot] = []
    for shot_start, shot_end in shot_ranges:
        shot_words = _words_for_shot(words, shot_start, shot_end)
        transcript = _words_to_text(shot_words)
        shots.append(
            Shot(
                source_file=str(video_path),
                start_time=shot_start,
                end_time=shot_end,
                description="",
                energy_level=0,
                relevance_score=0.0,
                transcript=transcript,
                words=shot_words,
                roll_type=roll_type,
            )
        )
    return shots


def preprocess_footage(
    input_dir: str,
    output_path: str,
    *,
    scene_threshold: float = 27.0,
    whisper_model_size: str = "base",
) -> FootageIndex:
    """Pre-process a directory of footage into a FootageIndex.

    Walks ``input_dir`` recursively for video files, runs PySceneDetect to
    find shot boundaries, transcribes each file with Faster-Whisper, aligns
    word-level timestamps to shot ranges, and serializes the resulting
    FootageIndex to ``output_path`` as JSON.

    Args:
        input_dir: Directory containing source video files.
        output_path: File path to write the serialized FootageIndex JSON.
        scene_threshold: PySceneDetect ContentDetector threshold.
        whisper_model_size: Faster-Whisper model size (default "base").

    Returns:
        The constructed FootageIndex (also written to ``output_path``).

    Raises:
        FileNotFoundError: If ``input_dir`` does not exist.
        NotADirectoryError: If ``input_dir`` is not a directory.
    """
    from faster_whisper import WhisperModel

    input_root = Path(input_dir)
    if not input_root.exists():
        raise FileNotFoundError(f"input_dir does not exist: {input_dir}")
    if not input_root.is_dir():
        raise NotADirectoryError(f"input_dir is not a directory: {input_dir}")

    out_path = Path(output_path)
    out_path.parent.mkdir(parents=True, exist_ok=True)

    video_files = sorted(
        p
        for p in input_root.rglob("*")
        if p.is_file() and p.suffix.lower() in VIDEO_SUFFIXES
    )

    _log(
        f"[preprocess] scanning {input_root} — found {len(video_files)} video file(s)"
    )

    if not video_files:
        _log("[preprocess] WARNING: no video files found")

    _log(f"[preprocess] loading Faster-Whisper model '{whisper_model_size}'...")
    model = WhisperModel(whisper_model_size, device="cpu", compute_type="int8")

    all_shots: list[Shot] = []
    total = len(video_files)
    for idx, video_path in enumerate(video_files, start=1):
        _log(f"[{idx}/{total}] processing {video_path.name}...")
        try:
            shots = _process_video(
                model, video_path, scene_threshold=scene_threshold
            )
            _log(f"[{idx}/{total}]   -> {len(shots)} shot(s)")
            all_shots.extend(shots)
        except Exception as exc:
            _log(f"[{idx}/{total}]   ERROR processing {video_path.name}: {exc}")
            continue

    total_duration = sum(
        max(0.0, shot.end_time - shot.start_time) for shot in all_shots
    )

    footage_index = FootageIndex(
        source_dir=str(input_root),
        shots=all_shots,
        total_duration=total_duration,
        created_at=datetime.now(timezone.utc),
    )

    out_path.write_text(
        footage_index.model_dump_json(indent=2),
        encoding="utf-8",
    )
    _log(
        f"[preprocess] wrote FootageIndex with {len(all_shots)} shot(s), "
        f"total_duration={total_duration:.2f}s -> {out_path}"
    )

    return footage_index


if __name__ == "__main__":
    if len(sys.argv) != 3:
        print(
            "Usage: python -m src.pipeline.preprocess <input_dir> <output_path>",
            file=sys.stderr,
        )
        sys.exit(1)
    preprocess_footage(sys.argv[1], sys.argv[2])