Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

11.7 KB
Raw Blame
"""
Speaker diarization service.

Detects who is speaking when in a podcast using pyannote.audio.
Merges speaker labels with Whisper transcription segments for
per-word and per-segment speaker attribution.

Supports 2-person and 3+ person podcasts automatically.
"""

import os
import subprocess
import tempfile
from typing import Optional, Callable

from utils.proc import run as proc_run, ProcError

# Fix torchaudio compatibility: speechbrain calls torchaudio.list_audio_backends()
# which was removed in torchaudio >= 2.10. Monkey-patch before any import.
try:
    import torchaudio
    if not hasattr(torchaudio, "list_audio_backends"):
        torchaudio.list_audio_backends = lambda: ["ffmpeg"]
except ImportError:
    pass

# Suppress noisy PyTorch warnings (std() degrees of freedom, etc.)
import warnings
warnings.filterwarnings("ignore", message="std\\(\\): degrees of freedom")
warnings.filterwarnings("ignore", category=UserWarning, module="pyannote")


def extract_audio_wav(video_path: str, output_path: str) -> str:
    """Extract audio as 16kHz mono WAV (required by pyannote)."""
    cmd = [
        "ffmpeg", "-y",
        "-i", video_path,
        "-vn",                  # No video
        "-acodec", "pcm_s16le", # 16-bit PCM
        "-ar", "16000",         # 16kHz
        "-ac", "1",             # Mono
        output_path,
    ]
    result = proc_run(cmd, timeout=600, check=False)
    if result.returncode != 0:
        raise RuntimeError(f"Audio extraction failed: {result.stderr[-300:]}")
    return output_path


def _load_wav_waveform(audio_path: str):
    """Read a PCM WAV into the {waveform, sample_rate} dict pyannote accepts.

    pyannote 4 loads a file path through torchcodec, which needs FFmpeg's
    shared libraries on the system and raises "Could not load libtorchcodec"
    on machines that only have the ffmpeg binary. extract_audio_wav always
    writes 16 kHz mono PCM, so read it with the stdlib instead and hand the
    pipeline a tensor. Returns None when the file is not plain PCM, in which
    case the caller falls back to passing the path.
    """
    import wave

    try:
        import numpy as np
        import torch

        with wave.open(audio_path, "rb") as wf:
            sample_rate = wf.getframerate()
            channels = wf.getnchannels()
            width = wf.getsampwidth()
            raw = wf.readframes(wf.getnframes())
    except Exception:
        return None

    dtype = {1: "uint8", 2: "int16", 4: "int32"}.get(width)
    if dtype is None or not raw:
        return None

    data = np.frombuffer(raw, dtype=dtype).astype("float32")
    if width == 1:
        data = (data - 128.0) / 128.0
    else:
        data /= float(1 << (8 * width - 1))

    data = data.reshape(-1, channels).T if channels > 1 else data.reshape(1, -1)
    return {"waveform": torch.from_numpy(data.copy()), "sample_rate": sample_rate}


def run_diarization(
    audio_path: str,
    num_speakers: Optional[int] = None,
    min_speakers: int = 2,
    max_speakers: int = 5,
    hf_token: Optional[str] = None,
    progress_callback: Optional[Callable[[int, str], None]] = None,
) -> list[dict]:
    """
    Run speaker diarization on an audio file.

    Returns list of speaker segments:
        [{"speaker": "SPEAKER_00", "start": 0.5, "end": 3.2}, ...]

    Args:
        audio_path: Path to WAV audio file
        num_speakers: Exact number of speakers (if known)
        min_speakers: Minimum speakers to detect
        max_speakers: Maximum speakers to detect
        hf_token: HuggingFace token for pyannote model access
    """
    token = hf_token or os.environ.get("HF_TOKEN", "")

    try:
        from pyannote.audio import Pipeline
    except ImportError:
        msg = "Speaker detection not installed. Run: podcli setup --speakers"
        if progress_callback:
            progress_callback(0, msg)
        raise ImportError(msg)

    if not token:
        msg = (
            "HF_TOKEN not set: speaker detection requires a HuggingFace token.\n"
            "\n"
            "  Setup (one-time):\n"
            "  1. Create a token at https://huggingface.co/settings/tokens\n"
            "  2. Accept model terms at ALL THREE:\n"
            "     → https://huggingface.co/pyannote/speaker-diarization-3.1\n"
            "     → https://huggingface.co/pyannote/segmentation-3.0\n"
            "     → https://huggingface.co/pyannote/speaker-diarization-community-1\n"
            "  3. Set it: podcli env set HF_TOKEN hf_your_token_here\n"
            "     (or in the Web UI: Config → Secrets)"
        )
        if progress_callback:
            progress_callback(0, msg)
        raise PermissionError(msg)

    if progress_callback:
        progress_callback(10, "Loading speaker diarization model...")

    try:
        # pyannote >= 3.1 uses 'token', older versions use 'use_auth_token'
        try:
            pipeline = Pipeline.from_pretrained(
                "pyannote/speaker-diarization-3.1",
                token=token,
            )
        except TypeError:
            pipeline = Pipeline.from_pretrained(
                "pyannote/speaker-diarization-3.1",
                use_auth_token=token,
            )
    except Exception as e:
        err_str = str(e).lower()
        if "token" in err_str or "auth" in err_str or "403" in err_str or "401" in err_str:
            msg = (
                f"HuggingFace auth failed: {e}\n"
                "\n"
                "  Fix: Accept model terms at ALL THREE (while logged into HuggingFace):\n"
                "  → https://huggingface.co/pyannote/speaker-diarization-3.1\n"
                "  → https://huggingface.co/pyannote/segmentation-3.0\n"
                "  → https://huggingface.co/pyannote/speaker-diarization-community-1\n"
                "  Then verify your HF_TOKEN is valid."
            )
            if progress_callback:
                progress_callback(0, msg)
            raise PermissionError(msg) from e
        raise

    # Move pipeline to GPU if available (Apple Silicon MPS or CUDA)
    import torch
    device = "cpu"
    if torch.cuda.is_available():
        device = "cuda"
        pipeline.to(torch.device("cuda"))
    elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        try:
            pipeline.to(torch.device("mps"))
            device = "mps"
        except Exception:
            pass  # Some pyannote components don't support MPS, fall back to CPU

    # Estimate duration for progress message
    import wave
    try:
        with wave.open(audio_path, "rb") as wf:
            audio_dur = wf.getnframes() / wf.getframerate()
        if device != "cpu":
            est_min = max(1, int(audio_dur / 600))  # GPU: ~2x faster
            time_msg = f"Running speaker diarization on {device.upper()} (~{est_min}-{est_min * 2} min)..."
        else:
            est_min = max(1, int(audio_dur / 300))
            time_msg = f"Running speaker diarization on CPU (~{est_min}-{est_min * 2} min for {int(audio_dur / 60)} min audio)..."
    except Exception:
        time_msg = f"Running speaker diarization on {device.upper()}..."

    if progress_callback:
        progress_callback(30, time_msg)

    # Run diarization: this is a blocking call, no progress updates from pyannote
    diarization_params = {}
    if num_speakers:
        diarization_params["num_speakers"] = num_speakers
    else:
        diarization_params["min_speakers"] = min_speakers
        diarization_params["max_speakers"] = max_speakers

    diarization = pipeline(_load_wav_waveform(audio_path) or audio_path, **diarization_params)

    if progress_callback:
        progress_callback(90, "Processing speaker segments...")

    # pyannote >= 4.0 returns DiarizeOutput; extract the Annotation from it
    if hasattr(diarization, "speaker_diarization"):
        diarization = diarization.speaker_diarization

    # Convert to simple list format
    speaker_segments = []
    for turn, _, speaker in diarization.itertracks(yield_label=True):
        speaker_segments.append({
            "speaker": speaker,
            "start": round(turn.start, 3),
            "end": round(turn.end, 3),
        })

    if progress_callback:
        progress_callback(100, f"Detected {len(set(s['speaker'] for s in speaker_segments))} speakers")

    return speaker_segments


def assign_speakers_to_segments(
    segments: list[dict],
    speaker_segments: list[dict],
) -> list[dict]:
    """
    Assign speaker labels to transcription segments.

    For each transcription segment, find the speaker who talks the most
    during that segment's time range (majority vote).
    """
    if not speaker_segments:
        # No diarization data: return segments unchanged
        for seg in segments:
            seg["speaker"] = None
        return segments

    for seg in segments:
        seg_start = seg["start"]
        seg_end = seg["end"]

        # Find overlapping speaker segments
        speaker_overlap = {}
        for sp in speaker_segments:
            overlap_start = max(seg_start, sp["start"])
            overlap_end = min(seg_end, sp["end"])
            overlap = max(0, overlap_end - overlap_start)

            if overlap > 0:
                speaker_overlap[sp["speaker"]] = speaker_overlap.get(sp["speaker"], 0) + overlap

        if speaker_overlap:
            # Assign speaker with most overlap
            seg["speaker"] = max(speaker_overlap, key=speaker_overlap.get)
        else:
            seg["speaker"] = None

    return segments


def assign_speakers_to_words(
    words: list[dict],
    speaker_segments: list[dict],
) -> list[dict]:
    """
    Assign speaker labels to individual words.

    Picks the speaker whose diarization segment overlaps the word the most,
    at turn boundaries several segments can touch a word, and the first
    match is often the previous speaker trailing off.
    """
    if not speaker_segments:
        for w in words:
            w["speaker"] = None
        return words

    for w in words:
        speaker_overlap = {}
        for sp in speaker_segments:
            overlap = min(w["end"], sp["end"]) - max(w["start"], sp["start"])
            if overlap > 0:
                speaker_overlap[sp["speaker"]] = speaker_overlap.get(sp["speaker"], 0) + overlap

        if speaker_overlap:
            w["speaker"] = max(speaker_overlap, key=speaker_overlap.get)
        else:
            # Zero-length or gap words: fall back to midpoint containment.
            midpoint = (w["start"] + w["end"]) / 2
            w["speaker"] = next(
                (sp["speaker"] for sp in speaker_segments if sp["start"] <= midpoint <= sp["end"]),
                None,
            )

    return words


def create_speaker_summary(speaker_segments: list[dict]) -> dict:
    """
    Create a summary of speaker activity.

    Returns:
        {
            "num_speakers": int,
            "speakers": {
                "SPEAKER_00": {"total_time": float, "segments": int, "label": "Speaker 1"},
                ...
            }
        }
    """
    if not speaker_segments:
        return {"num_speakers": 0, "speakers": {}}

    speaker_stats = {}
    for sp in speaker_segments:
        name = sp["speaker"]
        if name not in speaker_stats:
            speaker_stats[name] = {"total_time": 0, "segments": 0}
        speaker_stats[name]["total_time"] += sp["end"] - sp["start"]
        speaker_stats[name]["segments"] += 1

    # Sort by total speaking time (most talkative first)
    sorted_speakers = sorted(speaker_stats.items(), key=lambda x: -x[1]["total_time"])

    # Assign friendly labels
    result = {}
    for i, (name, stats) in enumerate(sorted_speakers):
        result[name] = {
            "total_time": round(stats["total_time"], 1),
            "segments": stats["segments"],
            "label": f"Speaker {i + 1}",
        }

    return {
        "num_speakers": len(result),
        "speakers": result,
    }