Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
"""
Audio-event detection (laughter, cheering, applause, screaming) for reaction-based
clip scoring.
Runs YAMNet (AudioSet, 521 classes) as a self-contained ONNX graph via onnxruntime,
which imports neither torch nor tensorflow, so this belongs on the native /
whisper.cpp runtime path. Audio is extracted at 16 kHz mono (the same format
whisper.cpp uses) and fed to the graph as a raw waveform; the log-mel front-end is
baked into the model, so there is no librosa dependency.
A laugh is the strongest language-independent "something funny happened" signal, and
it anti-correlates with speech at the frame level, which makes it a reliable anchor
for extending a clip backwards to the moment that caused the reaction.
"""
from __future__ import annotations
import csv
import os
import subprocess
import tempfile
import wave
from pathlib import Path
from typing import Optional, Callable
from utils.proc import run as proc_run
# Guarded together: on a partial runtime, reaction detection turns itself off rather
# than taking down every other backend task with an import error.
try:
import numpy as np
import onnxruntime as ort
_RUNTIME_AVAILABLE = True
except ImportError:
_RUNTIME_AVAILABLE = False
_MODELS_DIR = Path(__file__).resolve().parents[1] / "models"
_MODEL_PATH = _MODELS_DIR / "yamnet.onnx"
_CLASS_MAP_PATH = _MODELS_DIR / "yamnet_class_map.csv"
# AudioSet display names, grouped into the reaction channels we score on. Laughter is
# spread across fine-grained children in the ontology, so we collapse the whole family.
_LAUGHTER_KEYS = ("laugh", "giggle", "chuckle", "snicker", "chortle")
_CHEER_NAMES = ("Cheering", "Applause", "Whoop")
_SCREAM_NAMES = ("Screaming",)
_SPEECH_NAMES = ("Speech",)
_REACTION_CHANNELS = ("laughter", "cheering", "screaming")
REACTION_THRESHOLD = 0.15
_session = None
_class_index: dict[str, list[int]] = {}
def _load_class_index() -> dict[str, list[int]]:
names: dict[int, str] = {}
with open(_CLASS_MAP_PATH, newline="", encoding="utf-8") as f:
reader = csv.reader(f)
next(reader) # header: index,mid,display_name
for row in reader:
names[int(row[0])] = row[2]
def by_substr(keys) -> list[int]:
return [i for i, n in names.items() if any(k in n.lower() for k in keys)]
def by_name(wanted) -> list[int]:
return [i for i, n in names.items() if n in wanted]
return {
"laughter": by_substr(_LAUGHTER_KEYS),
"cheering": by_name(_CHEER_NAMES),
"screaming": by_name(_SCREAM_NAMES),
"speech": by_name(_SPEECH_NAMES),
}
def _get_session():
global _session, _class_index
if not _RUNTIME_AVAILABLE or not _MODEL_PATH.exists():
return None
if _session is None:
_session = ort.InferenceSession(
str(_MODEL_PATH), providers=["CPUExecutionProvider"]
)
_class_index = _load_class_index()
return _session
def is_available() -> bool:
"""True if onnxruntime, numpy and the YAMNet model are all present."""
return _RUNTIME_AVAILABLE and _MODEL_PATH.exists()
def _read_waveform_16k_mono(video_path: str, wav_path: Optional[str] = None) -> Optional[np.ndarray]:
"""Extract audio as a float32 [-1, 1] mono waveform at 16 kHz via ffmpeg.
wav_path: optional pre-extracted 16 kHz mono WAV: read directly, no decode.
"""
if wav_path and os.path.exists(wav_path):
try:
with wave.open(wav_path) as w:
frames = w.readframes(w.getnframes())
except (wave.Error, OSError):
return None
if not frames:
return None
return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
# delete=False + manual unlink: on Windows the open NamedTemporaryFile
# handle blocks ffmpeg from writing to the same path.
tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
tmp.close()
try:
cmd = [
"ffmpeg", "-y", "-i", video_path,
"-vn", "-ar", "16000", "-ac", "1",
"-f", "wav", tmp.name, "-loglevel", "error",
]
result = proc_run(cmd, timeout=600, check=False)
if result.returncode != 0:
return None
try:
with wave.open(tmp.name) as w:
frames = w.readframes(w.getnframes())
except (wave.Error, OSError):
return None
finally:
try:
os.unlink(tmp.name)
except OSError:
pass
if not frames:
return None
return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
def extract_audio_events(video_path: str, wav_path: Optional[str] = None) -> list[dict]:
"""
Run YAMNet over a video's audio and return per-frame reaction probabilities.
Returns a list of {time, laughter, cheering, screaming, speech} dicts, one per
YAMNet frame (~0.48 s hop). Empty list if the model/runtime is unavailable or the
audio can't be read, so callers degrade gracefully.
"""
session = _get_session()
if session is None:
return []
waveform = _read_waveform_16k_mono(video_path, wav_path=wav_path)
if waveform is None or waveform.size == 0:
return []
input_name = session.get_inputs()[0].name
# Windowed so one multi-hour session.run() can't block the worker unboundedly.
window = 16000 * 300
parts: list[np.ndarray] = []
times: list[float] = []
for offset in range(0, waveform.size, window):
chunk = waveform[offset:offset + window]
if chunk.size < 16000:
continue
chunk_scores = session.run(None, {input_name: chunk})[0] # [frames, 521]
if chunk_scores.shape[0] == 0:
continue
hop = (chunk.size / 16000.0) / chunk_scores.shape[0]
base = offset / 16000.0
times.extend(base + f * hop for f in range(chunk_scores.shape[0]))
parts.append(chunk_scores)
if not parts:
return []
scores = np.concatenate(parts, axis=0)
n_frames = scores.shape[0]
def channel_max(keys: list[int]) -> np.ndarray:
return scores[:, keys].max(axis=1) if keys else np.zeros(n_frames)
laughter = channel_max(_class_index["laughter"])
cheering = channel_max(_class_index["cheering"])
screaming = channel_max(_class_index["screaming"])
speech = channel_max(_class_index["speech"])
return [
{
"time": round(times[i], 2),
"laughter": round(float(laughter[i]), 3),
"cheering": round(float(cheering[i]), 3),
"screaming": round(float(screaming[i]), 3),
"speech": round(float(speech[i]), 3),
}
for i in range(n_frames)
]
def _reaction_level(event: dict) -> float:
"""Combined reaction strength for one frame: the loudest reaction channel."""
return max(event[channel] for channel in _REACTION_CHANNELS)
def reaction_times(
events_data: list[dict],
threshold: float = REACTION_THRESHOLD,
) -> list[float]:
"""Timestamps of frames whose reaction level clears `threshold`."""
return [e["time"] for e in events_data if _reaction_level(e) >= threshold]
def dominant_reaction(events_data: list[dict], start: float, end: float) -> Optional[str]:
"""Name of the strongest reaction channel in [start, end]: laughter, cheering
or screaming. None if nothing reacted in that range."""
frames = [e for e in events_data if start <= e["time"] <= end]
if not frames:
return None
peak = max(frames, key=_reaction_level)
if _reaction_level(peak) <= 0:
return None
return max(_REACTION_CHANNELS, key=lambda channel: peak[channel])
def compute_event_scores(
events_data: list[dict],
segments: list[dict],
) -> list[float]:
"""
Per-segment reaction score (0-10) from the peak reaction within each segment.
Unlike audio energy (RMS in dB, only meaningful relative to a video's own
baseline), YAMNet probabilities are absolute-calibrated in [0, 1], so the score is
a direct scaling of the peak rather than a z-score. A brief chuckle registers
~0.25, a hearty laugh higher; peaks stand out sharply against a ~0 baseline.
"""
if not events_data:
return [0.0] * len(segments)
scores = []
for seg in segments:
seg_start = seg.get("start", 0)
seg_end = seg.get("end", 0)
levels = [
_reaction_level(e)
for e in events_data
if seg_start <= e["time"] <= seg_end
]
peak = max(levels) if levels else 0.0
scores.append(round(min(10.0, peak * 12.0), 2))
return scores
def get_event_profile(
video_path: str,
segments: list[dict],
progress_callback: Optional[Callable] = None,
reaction_threshold: float = REACTION_THRESHOLD,
wav_path: Optional[str] = None,
events_data: Optional[list[dict]] = None,
) -> dict:
"""
Full pipeline: detect audio events and score all segments.
Returns {events_data, segment_scores, reaction_times} where reaction_times are the
timestamps of frames whose reaction level clears reaction_threshold, the anchors a
reaction-based detector expands backwards from.
"""
if not is_available():
return {"events_data": [], "segment_scores": [0.0] * len(segments), "reaction_times": []}
if progress_callback:
progress_callback(0, "Detecting laughter and reactions...")
# Already-extracted events skip the model pass; scoring them is pure.
if events_data is None:
events_data = extract_audio_events(video_path, wav_path=wav_path)
if progress_callback:
progress_callback(70, "Scoring segments by reaction...")
segment_scores = compute_event_scores(events_data, segments)
if progress_callback:
progress_callback(100, "Reaction analysis complete")
return {
"events_data": events_data,
"segment_scores": segment_scores,
"reaction_times": reaction_times(events_data, reaction_threshold),
}