Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
"""Multi-signal saliency engine: fuse channels, peak-pick, expand reactions, snap.
For profiles whose candidate source is "saliency" (party, action), moments are
generated from a fused interestingness curve instead of from an LLM reading the
transcript. This is what lets podcli auto-cut highlights from footage with no useful
transcript (party videos, action).
Each channel is normalized against THIS video's own distribution (never a global
scale) so different recordings, rooms and mic levels compare fairly, then combined by
the active profile's weights. Peaks on the fused curve become candidate clips; a peak
driven by a laugh or cheer is expanded backwards to capture the moment that caused the
reaction, since the funny thing happens just before people react to it.
"""
from typing import Optional, Callable
import numpy as np
from services.profiles import get_profile, ContentProfile
from services.audio_analyzer import extract_audio_energy
from services.audio_events import extract_audio_events, is_available as audio_events_available
GRID_HZ = 1.0 # common time grid; energy is per-second, so 1 Hz is the natural rate
def _robust_z(x: np.ndarray) -> np.ndarray:
"""Median/MAD normalization: resistant to the heavy tails of RMS and reactions."""
if x.size == 0:
return x
med = np.median(x)
mad = np.median(np.abs(x - med))
scale = 1.4826 * mad if mad > 1e-9 else (np.std(x) or 1.0)
return (x - med) / scale
def _energy_curve(energy_data: list[dict], n_bins: int) -> np.ndarray:
"""Per-second loudness onto the grid; silence (<= -60 dB) floored so it doesn't win."""
curve = np.full(n_bins, -60.0)
for e in energy_data:
b = int(e["time"] * GRID_HZ)
if 0 <= b < n_bins:
curve[b] = max(curve[b], e.get("rms_db", -60.0))
return curve
def _reaction_curve(events_data: list[dict], n_bins: int) -> np.ndarray:
"""Peak reaction (laugh/cheer/scream) per grid bin."""
curve = np.zeros(n_bins)
for e in events_data:
b = int(e["time"] * GRID_HZ)
if 0 <= b < n_bins:
level = max(e.get("laughter", 0), e.get("cheering", 0), e.get("screaming", 0))
curve[b] = max(curve[b], level)
return curve
def _dilate(curve: np.ndarray, radius_bins: int) -> np.ndarray:
"""Spread each spike to its neighbors (grayscale dilation) so a brief, narrow
reaction still aligns with, and can win, the fused peak near it."""
if radius_bins <= 0 or curve.size == 0:
return curve
out = curve.copy()
for r in range(1, radius_bins + 1):
out[r:] = np.maximum(out[r:], curve[:-r])
out[:-r] = np.maximum(out[:-r], curve[r:])
return out
def fuse_channels(channels: dict[str, np.ndarray], profile: ContentProfile) -> np.ndarray:
"""Weighted sum of per-video-normalized channels, weights renormalized over what exists."""
present = {k: v for k, v in channels.items() if profile.channel_weights.get(k, 0) > 0 and v.size}
if not present:
return np.zeros(next(iter(channels.values())).size if channels else 0)
total_w = sum(profile.channel_weights[k] for k in present)
fused = None
for k, curve in present.items():
w = profile.channel_weights[k] / total_w
contrib = w * _robust_z(curve)
fused = contrib if fused is None else fused + contrib
return fused
def pick_peaks(curve: np.ndarray, height: float, min_gap_bins: int) -> list[int]:
"""Local maxima above `height`, then greedy non-maximum suppression by min gap.
Peaks are taken in descending value order and a lower peak is dropped if it falls
within min_gap_bins of an already-chosen higher one (1-D NMS).
"""
if curve.size == 0:
return []
candidates = [
i for i in range(1, len(curve) - 1)
if curve[i] >= curve[i - 1] and curve[i] >= curve[i + 1] and curve[i] >= height
]
if curve[0] >= height and (len(curve) == 1 or curve[0] > curve[1]):
candidates.append(0)
if len(curve) > 1 and curve[-1] >= height and curve[-1] > curve[-2]:
candidates.append(len(curve) - 1)
candidates.sort(key=lambda i: curve[i], reverse=True)
chosen: list[int] = []
for i in candidates:
if all(abs(i - j) >= min_gap_bins for j in chosen):
chosen.append(i)
return sorted(chosen)
def _snap_to_quiet(target_sec: float, energy_curve: np.ndarray, window_sec: float = 1.5) -> float:
"""Nudge a boundary to the quietest second nearby, so cuts land in a lull not mid-action."""
if energy_curve.size == 0:
return target_sec
center = int(round(target_sec * GRID_HZ))
lo = max(0, center - int(window_sec * GRID_HZ))
hi = min(len(energy_curve), center + int(window_sec * GRID_HZ) + 1)
if lo >= hi:
return target_sec
local = energy_curve[lo:hi]
return (lo + int(np.argmin(local))) / GRID_HZ
def sentences_from_words(words: list[dict]) -> list[dict]:
"""Group word-level timestamps into sentences on terminal punctuation.
Whisper *segments* often straddle a sentence boundary ("...was. But then..."), so
snapping to them still starts a clip mid-thought. Splitting on .?! gives real
sentence edges to snap to.
"""
sents: list[dict] = []
start = None
end = None
for w in words:
text = str(w.get("word", "")).strip()
if not text:
continue
if start is None:
start = w.get("start", 0.0)
end = w.get("end", start)
if text.endswith((".", "?", "!")):
sents.append({"start": start, "end": end})
start = None
if start is not None:
sents.append({"start": start, "end": end})
return sents
def _seg_index_at(t: float, segments: list[dict]) -> int:
"""Index of the segment covering t, else the nearest segment start."""
for i, s in enumerate(segments):
if s.get("start", 0) <= t <= s.get("end", 0):
return i
return min(range(len(segments)), key=lambda i: abs(segments[i].get("start", 0) - t))
def _sentence_window(
peak_sec: float,
back_sec: float,
fwd_sec: float,
segments: list[dict],
min_dur: float,
max_dur: float,
) -> tuple[float, float]:
"""Build a clip out of whole segments (sentences) so it never cuts mid-thought.
Starts at a sentence boundary, extends back ~back_sec and forward ~fwd_sec (and
enough to clear min_dur), always snapping to segment edges and never exceeding
max_dur.
"""
idx = _seg_index_at(peak_sec, segments)
start = segments[idx].get("start", peak_sec)
end = segments[idx].get("end", peak_sec)
i = idx
while i > 0 and (peak_sec - segments[i - 1]["start"]) <= back_sec and (end - segments[i - 1]["start"]) <= max_dur:
i -= 1
start = segments[i]["start"]
j = idx
while (
j < len(segments) - 1
and (segments[j + 1]["end"] - start) <= max_dur
and ((end - start) < min_dur or (segments[j]["end"] - peak_sec) < fwd_sec)
):
j += 1
end = segments[j]["end"]
return start, end
def _window_for_peak(
peak_sec: float,
reaction_level: float,
profile: ContentProfile,
duration: float,
energy_curve: np.ndarray,
min_dur: float,
max_dur: float,
segments: Optional[list[dict]] = None,
reaction_threshold: float = 0.06,
) -> tuple[float, float, bool]:
"""Clip window for a peak. A reaction peak expands backwards from the reaction onset.
With a transcript, boundaries snap to whole sentences so each clip is a complete
thought; without one (party footage) they snap to audio lulls instead.
"""
is_reaction = reaction_level >= reaction_threshold
back = profile.reaction_lookback_sec if is_reaction else min_dur / 2.0
fwd = profile.reaction_payoff_sec if is_reaction else min_dur / 2.0
if segments:
start, end = _sentence_window(peak_sec, back, fwd, segments, min_dur, max_dur)
else:
start = _snap_to_quiet(max(0.0, peak_sec - back), energy_curve)
end = _snap_to_quiet(min(duration, peak_sec + fwd), energy_curve)
if end - start < min_dur:
if is_reaction:
start = max(0.0, end - min_dur)
else:
end = min(duration, start + min_dur)
elif end - start > max_dur:
start = end - max_dur if is_reaction else start
end = start + max_dur
return round(max(0.0, start), 1), round(min(duration, end), 1), is_reaction
def detect_highlights(
video_path: str,
profile_name: str = "party",
top_n: int = 8,
min_dur: float = 8.0,
max_dur: float = 60.0,
height_z: float = 1.0,
segments: Optional[list[dict]] = None,
words: Optional[list[dict]] = None,
reaction_threshold: float = 0.06,
progress_callback: Optional[Callable] = None,
energy_data: Optional[list[dict]] = None,
events_data: Optional[list[dict]] = None,
wav_path: Optional[str] = None,
) -> list[dict]:
"""
Generate highlight clips from a video's fused signal curve.
If a transcript is provided, clip boundaries snap to whole sentences so each clip
is a complete thought; `words` (with punctuation) gives true sentence edges and is
preferred over `segments`. Without any transcript, boundaries snap to audio lulls.
`reaction_threshold` is the laughter/cheer level that counts as a reaction: low
(~0.06) for conversational chuckles, higher for loud belly-laughs/crowds.
`energy_data`/`events_data` accept the profiles the orchestration layer already
computed so the source audio isn't decoded again; `wav_path` is a pre-extracted
16 kHz mono WAV used when extraction is still needed.
Returns clip dicts compatible with the render pipeline:
{title, start_second, end_second, duration, score, reasons, preview}.
"""
profile = get_profile(profile_name)
snap_units = sentences_from_words(words) if words else segments
if energy_data is None:
if progress_callback:
progress_callback(10, "Analyzing audio energy...")
energy_data = extract_audio_energy(video_path, wav_path=wav_path)
if events_data is None:
events_data = []
if audio_events_available():
if progress_callback:
progress_callback(40, "Detecting laughter and reactions...")
events_data = extract_audio_events(video_path, wav_path=wav_path)
last_times = [e["time"] for e in energy_data] + [e["time"] for e in events_data]
if not last_times:
return []
duration = max(last_times) + 1.0
n_bins = int(duration * GRID_HZ) + 1
energy_curve = _energy_curve(energy_data, n_bins)
# Dilate reactions by ~2s so a single-frame laugh isn't suppressed by a louder
# energy neighbor and so the fused peak lands on the reaction, not next to it.
reaction_curve = _dilate(_reaction_curve(events_data, n_bins), int(2 * GRID_HZ))
fused = fuse_channels(
{"energy": energy_curve, "audio_event": reaction_curve}, profile
)
if fused.size == 0:
return []
if progress_callback:
progress_callback(70, "Selecting highlight moments...")
min_gap_bins = max(1, int(profile.peak_min_gap_sec * GRID_HZ))
# Reaction moments are primary candidates: a detected laugh/cheer is almost always
# worth a clip regardless of loudness, so they aren't made to out-compete energy in
# the blended curve. Energy peaks then fill the rest, minus any that collide with a
# reaction. Reaction score is offset above energy so reactions rank first.
reaction_peaks = pick_peaks(reaction_curve, reaction_threshold, min_gap_bins)
energy_peaks = pick_peaks(fused, height_z, min_gap_bins)
candidates = [(i, float(reaction_curve[i]), True) for i in reaction_peaks]
reaction_bins = {i for i in reaction_peaks}
for i in energy_peaks:
if all(abs(i - j) >= min_gap_bins for j in reaction_bins):
candidates.append((i, float(fused[i]), False))
def rank_key(c):
i, val, is_reaction = c
return (1 if is_reaction else 0, val)
candidates.sort(key=rank_key, reverse=True)
candidates = candidates[:top_n]
clips = []
for i, val, want_reaction in candidates:
peak_sec = i / GRID_HZ
reaction_level = float(reaction_curve[i]) if want_reaction else 0.0
start, end, is_reaction = _window_for_peak(
peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur,
snap_units, reaction_threshold,
)
if end - start < min_dur * 0.75:
continue
kind = "laugh/cheer" if is_reaction else "high energy"
score = round(10.0 + reaction_level * 10.0, 2) if is_reaction else round(float(val), 2)
clips.append({
"title": f"Highlight ({kind}) at {int(peak_sec // 60):d}:{int(peak_sec % 60):02d}",
"start_second": start,
"end_second": end,
"duration": round(end - start),
"score": score,
"reasons": ["reaction"] if is_reaction else ["energy_peak"],
"preview": "",
"content_type": "highlight",
})
clips.sort(key=lambda c: c["start_second"])
if progress_callback:
progress_callback(100, f"Found {len(clips)} highlights")
return clips
def detect_highlights_pooled(
video_paths: list[str],
profile_name: str = "party",
top_n: int = 15,
min_dur: float = 8.0,
max_dur: float = 60.0,
progress_callback: Optional[Callable] = None,
) -> list[dict]:
"""
Detect highlights across many videos and rank them globally: "the best N bits
from tonight" across a folder of party clips.
Each returned clip carries a `source_file`. Ranking is reaction-first, then by
score, so a genuine laugh in any file outranks a merely loud moment in another.
"""
pooled: list[dict] = []
n = len(video_paths) or 1
for idx, path in enumerate(video_paths):
clips = detect_highlights(
path, profile_name=profile_name, top_n=top_n, min_dur=min_dur, max_dur=max_dur
)
for c in clips:
c["source_file"] = path
pooled.extend(clips)
if progress_callback:
progress_callback(
int((idx + 1) / n * 100), f"{path}: {len(clips)} highlights"
)
pooled.sort(
key=lambda c: (1 if "reaction" in c.get("reasons", []) else 0, c.get("score", 0)),
reverse=True,
)
return pooled[:top_n]