Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

20.2 KB
Raw Blame
"""Local full-episode silence analysis and rendering.

Silero VAD (MIT, https://github.com/snakers4/silero-vad) finds speech without
uploading media. Transcript word ranges are
unioned with VAD output before cuts are planned, so known words are never cut.
The derived video and remapped transcript keep every downstream Podcli feature
on one compact timeline while the original source remains untouched.
"""

from __future__ import annotations

import hashlib
import json
import os
import shutil
import tempfile
import urllib.request
import uuid
import wave
from pathlib import Path
from typing import Callable, Iterable, Optional

from config.paths import paths
from services.audio_extract import extract_wav_16k_mono
from services.media_probe import get_media_duration_seconds, has_audio_stream
from utils.proc import run as proc_run

try:
    import numpy as np
    import onnxruntime as ort
    _VAD_RUNTIME_AVAILABLE = True
except ImportError:
    _VAD_RUNTIME_AVAILABLE = False


ProgressCallback = Optional[Callable[[int, str], None]]

SILERO_MODEL_URL = (
    "https://raw.githubusercontent.com/snakers4/silero-vad/"
    "76e3dc408eb2a5c655c34e230d2d5459b4439daa/"
    "src/silero_vad/data/silero_vad_16k_op15.onnx"
)
SILERO_MODEL_SHA256 = "7ed98ddbad84ccac4cd0aeb3099049280713df825c610a8ed34543318f1b2c49"
SILERO_MODEL_FILENAME = "silero_vad_16k_op15.onnx"
SAMPLE_RATE = 16_000
WINDOW_SAMPLES = 512
CONTEXT_SAMPLES = 64


def _emit(callback: ProgressCallback, percent: int, message: str) -> None:
    if callback:
        callback(max(0, min(100, int(percent))), message)


def _model_path() -> Path:
    return Path(paths["cache"]) / "models" / SILERO_MODEL_FILENAME


def _sha256(file_path: Path) -> str:
    digest = hashlib.sha256()
    with file_path.open("rb") as source:
        for block in iter(lambda: source.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def ensure_silero_model(progress_callback: ProgressCallback = None) -> Path:
    """Return verified model path, downloading the 1.3 MB model once if needed."""
    model_path = _model_path()
    if model_path.exists() and _sha256(model_path) == SILERO_MODEL_SHA256:
        return model_path

    model_path.parent.mkdir(parents=True, exist_ok=True)
    temp_path = model_path.with_name(f".{model_path.name}.{uuid.uuid4().hex}.download")
    _emit(progress_callback, 2, "Downloading local speech detector (first use only)")
    try:
        request = urllib.request.Request(
            SILERO_MODEL_URL,
            headers={"User-Agent": "podcli-silence-removal"},
        )
        with urllib.request.urlopen(request, timeout=60) as response, temp_path.open("wb") as target:
            shutil.copyfileobj(response, target)
        if _sha256(temp_path) != SILERO_MODEL_SHA256:
            raise RuntimeError("Silero VAD model checksum mismatch")
        os.replace(temp_path, model_path)
    finally:
        try:
            temp_path.unlink()
        except OSError:
            pass
    return model_path


def _merge_segments(
    segments: Iterable[dict],
    duration: float,
    merge_gap: float = 0.0,
) -> list[dict]:
    cleaned: list[dict] = []
    for raw in segments:
        try:
            start = max(0.0, min(duration, float(raw["start"])))
            end = max(0.0, min(duration, float(raw["end"])))
        except (KeyError, TypeError, ValueError):
            continue
        if end - start < 0.02:
            continue
        cleaned.append({"start": start, "end": end})
    cleaned.sort(key=lambda item: (item["start"], item["end"]))

    merged: list[dict] = []
    for segment in cleaned:
        if merged and segment["start"] <= merged[-1]["end"] + merge_gap:
            merged[-1]["end"] = max(merged[-1]["end"], segment["end"])
        else:
            merged.append(dict(segment))
    return merged


def probabilities_to_speech_segments(
    probabilities: list[float],
    audio_samples: int,
    *,
    threshold: float = 0.5,
    min_speech_ms: int = 250,
    min_silence_ms: int = 180,
) -> list[dict]:
    """Convert Silero probabilities into unpadded speech ranges."""
    negative_threshold = max(0.01, threshold - 0.15)
    min_speech_samples = SAMPLE_RATE * min_speech_ms / 1000
    min_silence_samples = SAMPLE_RATE * min_silence_ms / 1000
    triggered = False
    temporary_end = 0
    current_start = 0
    speech: list[dict] = []

    for index, probability in enumerate(probabilities):
        current_sample = index * WINDOW_SAMPLES
        if probability >= threshold:
            if not triggered:
                triggered = True
                current_start = current_sample
            temporary_end = 0
            continue

        if triggered and probability < negative_threshold:
            if not temporary_end:
                temporary_end = current_sample
            if current_sample - temporary_end >= min_silence_samples:
                if temporary_end - current_start >= min_speech_samples:
                    speech.append({
                        "start": current_start / SAMPLE_RATE,
                        "end": temporary_end / SAMPLE_RATE,
                    })
                triggered = False
                temporary_end = 0

    if triggered and audio_samples - current_start >= min_speech_samples:
        speech.append({
            "start": current_start / SAMPLE_RATE,
            "end": audio_samples / SAMPLE_RATE,
        })
    return speech


def detect_speech(
    video_path: str,
    *,
    threshold: float = 0.5,
    progress_callback: ProgressCallback = None,
) -> list[dict]:
    if not _VAD_RUNTIME_AVAILABLE:
        raise RuntimeError("Local silence detection requires numpy and onnxruntime")

    model_path = ensure_silero_model(progress_callback)
    _emit(progress_callback, 5, "Extracting episode audio")
    wav_path = extract_wav_16k_mono(video_path)
    try:
        session = ort.InferenceSession(str(model_path), providers=["CPUExecutionProvider"])
        state = np.zeros((2, 1, 128), dtype=np.float32)
        context = np.zeros((1, CONTEXT_SAMPLES), dtype=np.float32)
        probabilities: list[float] = []

        with wave.open(wav_path, "rb") as audio:
            if audio.getframerate() != SAMPLE_RATE or audio.getnchannels() != 1 or audio.getsampwidth() != 2:
                raise RuntimeError("Extracted audio is not 16 kHz mono PCM")
            total_samples = audio.getnframes()
            processed = 0
            last_percent = -1
            while True:
                frames = audio.readframes(WINDOW_SAMPLES)
                if not frames:
                    break
                chunk = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
                actual_samples = chunk.size
                if actual_samples < WINDOW_SAMPLES:
                    chunk = np.pad(chunk, (0, WINDOW_SAMPLES - actual_samples))
                model_input = np.concatenate((context, chunk.reshape(1, -1)), axis=1)
                output, state = session.run(None, {
                    "input": model_input,
                    "state": state,
                    "sr": np.array(SAMPLE_RATE, dtype=np.int64),
                })
                probabilities.append(float(output[0][0]))
                context = model_input[:, -CONTEXT_SAMPLES:]
                processed += actual_samples
                percent = 10 + int((processed / max(1, total_samples)) * 55)
                if percent >= last_percent + 3:
                    last_percent = percent
                    _emit(progress_callback, percent, "Finding spoken sections")

        return probabilities_to_speech_segments(probabilities, total_samples, threshold=threshold)
    finally:
        try:
            os.unlink(wav_path)
        except OSError:
            pass


def plan_silence_removal(
    duration: float,
    vad_segments: list[dict],
    transcript_words: list[dict],
    *,
    min_silence_seconds: float = 0.65,
    padding_seconds: float = 0.12,
) -> dict:
    """Build conservative keep ranges from VAD plus transcript word timings."""
    duration = max(0.0, float(duration))
    min_silence_seconds = max(0.2, min(5.0, float(min_silence_seconds)))
    padding_seconds = max(0.0, min(0.5, float(padding_seconds)))
    if duration <= 0:
        raise ValueError("Video duration must be positive")

    protected: list[dict] = []
    for segment in vad_segments:
        protected.append({
            "start": float(segment.get("start", 0)) - padding_seconds,
            "end": float(segment.get("end", 0)) + padding_seconds,
        })
    for word in transcript_words:
        try:
            protected.append({
                "start": float(word["start"]) - padding_seconds,
                "end": float(word["end"]) + padding_seconds,
            })
        except (KeyError, TypeError, ValueError):
            continue

    keep_segments = _merge_segments(protected, duration, merge_gap=min_silence_seconds)
    if not keep_segments:
        keep_segments = [{"start": 0.0, "end": duration}]

    removed_ranges: list[dict] = []
    cursor = 0.0
    for segment in keep_segments:
        if segment["start"] - cursor >= min_silence_seconds:
            removed_ranges.append({"start": cursor, "end": segment["start"]})
        elif cursor < segment["start"]:
            segment["start"] = cursor
        cursor = segment["end"]
    if duration - cursor >= min_silence_seconds:
        removed_ranges.append({"start": cursor, "end": duration})
    elif cursor < duration:
        keep_segments[-1]["end"] = duration

    # Rebuild exact keep ranges as the complement of accepted removals. This
    # prevents short leading/interstitial gaps from disappearing accidentally.
    keep_segments = []
    cursor = 0.0
    for removed in removed_ranges:
        if removed["start"] > cursor:
            keep_segments.append({"start": cursor, "end": removed["start"]})
        cursor = removed["end"]
    if cursor < duration:
        keep_segments.append({"start": cursor, "end": duration})
    if not keep_segments:
        keep_segments = [{"start": 0.0, "end": duration}]

    keep_segments = [
        {"start": round(item["start"], 3), "end": round(item["end"], 3)}
        for item in keep_segments if item["end"] - item["start"] >= 0.04
    ]
    removed_ranges = [
        {"start": round(item["start"], 3), "end": round(item["end"], 3)}
        for item in removed_ranges
    ]
    output_duration = sum(item["end"] - item["start"] for item in keep_segments)
    removed_duration = max(0.0, duration - output_duration)
    return {
        "keep_segments": keep_segments,
        "removed_ranges": removed_ranges,
        "source_duration": round(duration, 3),
        "output_duration": round(output_duration, 3),
        "removed_duration": round(removed_duration, 3),
        "removed_percent": round((removed_duration / duration) * 100, 1),
        "cut_count": len(removed_ranges),
        "min_silence_seconds": min_silence_seconds,
        "padding_seconds": padding_seconds,
        "method": "silero-vad+word-boundaries",
    }


def analyze_silence(
    video_path: str,
    transcript_words: list[dict],
    *,
    threshold: float = 0.5,
    min_silence_seconds: float = 0.65,
    padding_seconds: float = 0.12,
    progress_callback: ProgressCallback = None,
) -> dict:
    duration = get_media_duration_seconds(video_path)
    if duration <= 0:
        raise RuntimeError("Could not determine episode duration")
    speech = detect_speech(video_path, threshold=threshold, progress_callback=progress_callback)
    _emit(progress_callback, 75, "Protecting word boundaries")
    plan = plan_silence_removal(
        duration,
        speech,
        transcript_words,
        min_silence_seconds=min_silence_seconds,
        padding_seconds=padding_seconds,
    )
    plan["vad_threshold"] = threshold
    _emit(progress_callback, 100, "Silence analysis ready")
    return plan


def _map_point(t: float, keep_segments: list[dict]) -> Optional[tuple[float, float]]:
    """Map a single instant through keep_segments. None if t fell in a
    removed range; otherwise its output position as a zero-length range."""
    cursor = 0.0
    for segment in keep_segments:
        if segment["start"] <= t <= segment["end"]:
            mapped = cursor + (t - segment["start"])
            return mapped, mapped
        cursor += segment["end"] - segment["start"]
    return None


def _map_range(
    start: float,
    end: float,
    keep_segments: list[dict],
    *,
    assign_by_midpoint: bool = False,
) -> Optional[tuple[float, float]]:
    # A zero (or inverted) duration item has no overlap for the interval test
    # below to find. overlap_end > overlap_start never holds when they're
    # equal, so it always mapped to None and got dropped. Map it as a point
    # instead: keep it if it falls inside a kept range, drop it if not.
    if end <= start:
        return _map_point(start, keep_segments)
    if assign_by_midpoint:
        # A word that straddles a cut (part of it sits in the removed gap
        # between two kept ranges) belongs wholly to whichever side holds its
        # midpoint, not to a stitched span across the cut. Stitching would
        # silently absorb the cut into the word's own duration.
        midpoint = (start + end) / 2.0
        cursor = 0.0
        for segment in keep_segments:
            seg_start, seg_end = segment["start"], segment["end"]
            if seg_start <= midpoint <= seg_end:
                overlap_start = max(start, seg_start)
                overlap_end = min(end, seg_end)
                return cursor + overlap_start - seg_start, cursor + overlap_end - seg_start
            cursor += seg_end - seg_start
        return None
    output_cursor = 0.0
    mapped_parts: list[tuple[float, float]] = []
    for segment in keep_segments:
        overlap_start = max(start, segment["start"])
        overlap_end = min(end, segment["end"])
        if overlap_end > overlap_start:
            mapped_parts.append((
                output_cursor + overlap_start - segment["start"],
                output_cursor + overlap_end - segment["start"],
            ))
        output_cursor += segment["end"] - segment["start"]
    if not mapped_parts:
        return None
    return mapped_parts[0][0], mapped_parts[-1][1]


def remap_timed_items(
    items: list[dict],
    keep_segments: list[dict],
    *,
    assign_by_midpoint: bool = False,
) -> list[dict]:
    remapped: list[dict] = []
    for item in items:
        try:
            start = float(item["start"])
            end = float(item["end"])
        except (KeyError, TypeError, ValueError):
            continue
        mapped = _map_range(start, end, keep_segments, assign_by_midpoint=assign_by_midpoint)
        if mapped is None:
            continue
        mapped_start, mapped_end = mapped
        # The 10ms floor only makes sense for an interval a cut clipped down to
        # near-nothing. Whisper emits many 10ms words (and float subtraction
        # puts some at 9.99ms), so an item that kept its whole length stays,
        # as does a zero-duration point from _map_point.
        mapped_length = mapped_end - mapped_start
        clipped = (end - start) - mapped_length > 1e-6
        if clipped and mapped_length < 0.01:
            continue
        remapped.append({**item, "start": round(mapped_start, 3), "end": round(mapped_end, 3)})
    return remapped


def remap_transcript(transcript: dict, keep_segments: list[dict]) -> dict:
    remapped = dict(transcript or {})
    remapped["words"] = remap_timed_items(
        list(remapped.get("words") or []), keep_segments, assign_by_midpoint=True
    )
    remapped["segments"] = remap_timed_items(list(remapped.get("segments") or []), keep_segments)
    remapped["duration"] = round(sum(s["end"] - s["start"] for s in keep_segments), 3)
    remapped["silence_removed"] = True
    return remapped


def _reserve_output_path(video_path: str, output_dir: str) -> Path:
    stem = Path(video_path).stem
    for suffix in range(1, 10_000):
        name = f"{stem}_silence_removed_podcli.mp4" if suffix == 1 else f"{stem}_silence_removed_podcli-{suffix}.mp4"
        candidate = Path(output_dir) / name
        if not candidate.exists():
            return candidate
    raise RuntimeError("Could not reserve silence-removed output filename")


def _render_batch(
    video_path: str,
    output_path: str,
    segments: list[dict],
    audio: bool,
) -> None:
    filters: list[str] = []
    concat_inputs: list[str] = []
    for index, segment in enumerate(segments):
        start = segment["start"]
        end = segment["end"]
        filters.append(f"[0:v:0]trim=start={start:.3f}:end={end:.3f},setpts=PTS-STARTPTS[v{index}]")
        concat_inputs.append(f"[v{index}]")
        if audio:
            filters.append(f"[0:a:0]atrim=start={start:.3f}:end={end:.3f},asetpts=PTS-STARTPTS[a{index}]")
            concat_inputs.append(f"[a{index}]")
    filters.append(
        "".join(concat_inputs)
        + f"concat=n={len(segments)}:v=1:a={1 if audio else 0}[vout]"
        + ("[aout]" if audio else "")
    )
    command = [
        "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", video_path,
        "-filter_complex", ";".join(filters),
        "-map", "[vout]",
    ]
    if audio:
        command += ["-map", "[aout]"]
    command += [
        "-c:v", "libx264", "-crf", "18", "-preset", "fast", "-profile:v", "high",
        "-pix_fmt", "yuv420p",
    ]
    if audio:
        command += ["-c:a", "aac", "-b:a", "192k"]
    command += ["-movflags", "+faststart", output_path]
    proc_run(command, timeout=3600, check=True)


def render_silence_removed(
    video_path: str,
    keep_segments: list[dict],
    transcript: dict,
    output_dir: str,
    *,
    progress_callback: ProgressCallback = None,
) -> dict:
    if not os.path.exists(video_path):
        raise FileNotFoundError(f"Video not found: {video_path}")
    duration = get_media_duration_seconds(video_path)
    normalized = _merge_segments(keep_segments, duration)
    if not normalized:
        raise ValueError("No valid speech segments to render")

    Path(output_dir).mkdir(parents=True, exist_ok=True)
    output_path = _reserve_output_path(video_path, output_dir)
    work_dir = Path(tempfile.mkdtemp(prefix="podcli_silence_", dir=paths["working"] if os.path.isdir(paths["working"]) else None))
    batch_size = 80
    chunks: list[Path] = []
    audio = has_audio_stream(video_path)
    try:
        batches = [normalized[index:index + batch_size] for index in range(0, len(normalized), batch_size)]
        for index, batch in enumerate(batches):
            _emit(progress_callback, 5 + int((index / len(batches)) * 85), f"Building compact episode {index + 1}/{len(batches)}")
            chunk_path = work_dir / f"chunk-{index:04d}.mp4"
            _render_batch(video_path, str(chunk_path), batch, audio)
            chunks.append(chunk_path)

        partial = work_dir / "finished.mp4"
        if len(chunks) == 1:
            shutil.copy2(chunks[0], partial)
        else:
            concat_path = work_dir / "chunks.txt"
            concat_path.write_text("".join(f"file '{chunk.as_posix()}'\n" for chunk in chunks), encoding="utf-8")
            proc_run([
                "ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
                "-f", "concat", "-safe", "0", "-i", str(concat_path),
                "-c", "copy", "-movflags", "+faststart", str(partial),
            ], timeout=1800, check=True)
        # Falls back to copy+delete: the work dir and a user-configured output
        # dir can sit on different volumes, where os.replace raises EXDEV.
        shutil.move(str(partial), str(output_path))
        _emit(progress_callback, 96, "Remapping captions and clips")
        remapped = remap_transcript(transcript, normalized)
        manifest = output_path.with_suffix(".silence.json")
        manifest.write_text(json.dumps({
            "source_video": os.path.abspath(video_path),
            "output_video": str(output_path),
            "keep_segments": normalized,
        }, ensure_ascii=False, indent=2), encoding="utf-8")
        _emit(progress_callback, 100, "Compact episode ready")
        stat = output_path.stat()
        return {
            "output_path": str(output_path),
            "filename": output_path.name,
            "file_size_mb": round(stat.st_size / (1024 * 1024), 2),
            "duration": remapped["duration"],
            "transcript": remapped,
            "manifest_path": str(manifest),
        }
    finally:
        shutil.rmtree(work_dir, ignore_errors=True)