Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
import os
import sys
import numpy as np
import pytest
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
BACKEND_ROOT = os.path.join(ROOT, "backend")
if BACKEND_ROOT not in sys.path:
sys.path.insert(0, BACKEND_ROOT)
from services.multicam_signal import ( # noqa: E402
BOTH,
LEAD_IN,
SILENT,
Camera,
coarse_lag,
fit_clock,
gcc_phat,
level_db,
onset_envelope,
plan_cuts,
segments_to_frames,
speaker_frames,
speech_bounds,
)
RATE = 8000
def speechlike(seconds: float, seed: int) -> np.ndarray:
"""Noise gated into irregular syllable-length bursts, like talking."""
rng = np.random.default_rng(seed)
n = int(seconds * RATE)
gate = np.zeros(n, dtype=np.float32)
t = 0
while t < n:
burst = int(rng.uniform(0.08, 0.35) * RATE)
gap = int(rng.uniform(0.05, 0.6) * RATE)
gate[t:t + burst] = rng.uniform(0.3, 1.0)
t += burst + gap
return (rng.standard_normal(n).astype(np.float32) * gate * 8000).astype(np.float32)
def other_device(x: np.ndarray, seed: int) -> np.ndarray:
"""Same sound through another mic: lowpassed, quieter, with its own hiss."""
rng = np.random.default_rng(seed)
kernel = np.ones(6, dtype=np.float32) / 6
return np.convolve(x, kernel, mode="same") * 0.4 + rng.standard_normal(len(x)).astype(np.float32) * 150
def test_coarse_lag_finds_a_late_starting_source():
ref = speechlike(180, seed=1)
src = other_device(ref[40 * RATE:150 * RATE], seed=2)
match = coarse_lag(onset_envelope(ref, RATE), onset_envelope(src, RATE))
assert match is not None and match.ok
assert abs(match.lag_seconds - 40.0) <= 0.02
def test_coarse_lag_finds_a_source_that_started_before_the_reference():
full = speechlike(200, seed=3)
ref = full[30 * RATE:]
src = other_device(full[:120 * RATE], seed=4)
match = coarse_lag(onset_envelope(ref, RATE), onset_envelope(src, RATE))
assert match is not None and match.ok
assert abs(match.lag_seconds + 30.0) <= 0.02
def test_unrelated_audio_is_not_reported_as_synced():
match = coarse_lag(onset_envelope(speechlike(120, 5), RATE), onset_envelope(speechlike(90, 6), RATE))
assert match is not None and not match.ok
def test_gcc_phat_refines_to_sub_millisecond():
ref = speechlike(30, seed=7)
shift = 0.1234
start = int((5 + shift) * RATE)
src = other_device(ref[start:start + 10 * RATE], seed=8)
pad = int(0.25 * RATE)
window = ref[5 * RATE - pad:5 * RATE + 10 * RATE + pad]
found, prominence = gcc_phat(window, src, RATE, 0.25)
assert abs(found - shift) < 0.001
assert prominence > 10
def test_fit_clock_recovers_drift_and_drops_an_outlier():
speed = 1.0 + 80e-6
pts = [(s, 12.5 + s * speed, 1.0) for s in (60, 1200, 2400, 3600, 4800)]
pts.append((3000, 12.5 + 3000 * speed + 0.4, 1.0))
fit = fit_clock(pts)
assert fit is not None
assert abs(fit.offset - 12.5) < 1e-4
assert abs(fit.speed - speed) < 1e-7
assert fit.checkpoints == 5
def test_fit_clock_rejects_implausible_speed():
fit = fit_clock([(0, 5.0, 1.0), (100, 106.0, 1.0)])
assert fit is not None and fit.speed == 1.0
# Falling back to speed 1.0 does not make the fit trustworthy: the 1 s
# drift between these two points still shows up as a large residual, and
# callers need that signal to flag the sync for review.
assert fit.speed_fallback is True
assert fit.residual_ms == pytest.approx(500.0, abs=1.0)
assert fit.residual_all_ms == pytest.approx(500.0, abs=1.0)
def test_fit_clock_reports_total_checkpoints_and_outlier_residual():
speed = 1.0 + 80e-6
pts = [(s, 12.5 + s * speed, 1.0) for s in (60, 1200, 2400, 3600, 4800)]
pts.append((3000, 12.5 + 3000 * speed + 0.4, 1.0))
fit = fit_clock(pts)
assert fit is not None
assert fit.total_checkpoints == 6
assert fit.checkpoints == 5
# The inlier residual is tiny, but the dropped outlier was 0.4 s off: the
# all-checkpoints residual must still surface that.
assert fit.residual_ms < 0.01
assert fit.residual_all_ms == pytest.approx(400.0, abs=1.0)
assert fit.speed_fallback is False
def _two_person_levels():
host = speechlike(20, seed=10)
guest = speechlike(20, seed=11)
host[10 * RATE:] = 0
guest[:10 * RATE] = 0
# Each mic hears the other person 20 dB down, and the guest mic runs hot.
host_mic = host + guest * 0.1
guest_mic = (guest + host * 0.1) * 3
return level_db(host_mic.astype(np.float32), RATE), level_db(guest_mic.astype(np.float32), RATE)
def test_speaker_frames_ignores_bleed_and_mic_gain():
labels = speaker_frames(list(_two_person_levels()))
first, second = labels[100:900], labels[1100:1900]
assert (first == 0).mean() > 0.6 and (first == 1).mean() < 0.05
assert (second == 1).mean() > 0.6 and (second == 0).mean() < 0.05
def test_speaker_frames_marks_crosstalk_as_both():
a = speechlike(10, seed=12)
b = speechlike(10, seed=13)
labels = speaker_frames([level_db(a, RATE), level_db(b, RATE)])
assert (labels == BOTH).mean() > 0.1
def test_segments_to_frames_marks_overlap():
labels = segments_to_frames(
[{"start": 0, "end": 2, "person": "host"}, {"start": 1, "end": 3, "person": "guest"}],
["host", "guest"],
400,
)
assert labels[50] == 0 and labels[150] == BOTH and labels[250] == 1 and labels[350] == SILENT
def _labels(pattern: list[tuple[int, float]]) -> np.ndarray:
out = []
for value, seconds in pattern:
out += [value] * int(round(seconds / 0.01))
return np.array(out, dtype=np.int32)
CAMS = [
Camera("cam_host", "host", 0, 100),
Camera("cam_guest", "guest", 0, 100),
Camera("cam_wide", "wide", 0, 100),
]
def test_plan_cuts_follows_the_speaker_and_folds_short_blips():
labels = _labels([(0, 10), (1, 0.5), (0, 5), (SILENT, 2), (1, 10)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=27.5, min_shot=2.0)
assert [c["source_id"] for c in cuts] == ["cam_host", "cam_guest"]
assert cuts[0]["start"] == 0 and cuts[-1]["end"] == 27.5
# The guest starts at 17.5 after a pause, so the cut leads in to them.
assert abs(cuts[1]["start"] - (17.5 - LEAD_IN)) < 0.02
def test_plan_cuts_goes_wide_on_crosstalk():
labels = _labels([(0, 6), (BOTH, 3), (1, 6)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=15)
assert [c["source_id"] for c in cuts] == ["cam_host", "cam_wide", "cam_guest"]
def test_plan_cuts_breaks_long_monologues_with_a_wide_shot():
labels = _labels([(0, 70)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=70, max_shot=30, wide_insert=4)
ids = [c["source_id"] for c in cuts]
assert ids.count("cam_wide") == 2
assert all(c["end"] - c["start"] <= 30 for c in cuts)
def test_plan_cuts_switches_to_the_next_camera_part():
cams = [
Camera("guest_part1", "guest", 0, 20),
Camera("guest_part2", "guest", 20, 60),
Camera("cam_host", "host", 0, 60),
]
labels = _labels([(1, 40)])
cuts = plan_cuts(labels, ["host", "guest"], cams, range_start=0, range_end=40, max_shot=0)
assert [c["source_id"] for c in cuts] == ["guest_part1", "guest_part2"]
assert cuts[0]["end"] == 20 and cuts[1]["start"] == 20
def test_plan_cuts_respects_a_range_that_starts_mid_timeline():
labels = _labels([(0, 10), (1, 10)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=5, range_end=15)
assert cuts[0] == {"start": 5.0, "end": 10.0, "source_id": "cam_host"}
assert cuts[1]["source_id"] == "cam_guest" and cuts[1]["end"] == 15.0
def test_speech_bounds():
assert speech_bounds(_labels([(SILENT, 3), (0, 2), (SILENT, 1)])) == (3.0, 5.0)
assert speech_bounds(_labels([(SILENT, 1)])) is None
def test_plan_cuts_never_overlaps_wide_inserts():
labels = _labels([(0, 20)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=20, max_shot=3, wide_insert=4)
assert all(a["end"] <= b["start"] + 1e-6 for a, b in zip(cuts, cuts[1:]))
assert abs(sum(c["end"] - c["start"] for c in cuts) - 20) < 1e-6
def test_sustained_crosstalk_with_blips_becomes_one_wide_shot():
blips = [(BOTH, 0.7), (0, 0.05), (BOTH, 0.9), (1, 0.08), (BOTH, 1.2), (0, 0.05), (BOTH, 1.0)]
labels = _labels([(0, 15), *blips, (1, 10)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=29, max_shot=0)
assert [c["source_id"] for c in cuts] == ["cam_host", "cam_wide", "cam_guest"]
assert cuts[1]["end"] - cuts[1]["start"] > 3.5
def test_backchannels_inside_an_answer_never_cut_away():
# Host says "mm-hm" and laughs with the guest briefly while the guest answers.
labels = _labels([(0, 5), (1, 12), (0, 0.6), (1, 9), (BOTH, 0.8), (1, 14), (0, 6)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=48.4, max_shot=0)
assert [c["source_id"] for c in cuts] == ["cam_host", "cam_guest", "cam_host"]
def test_guest_answers_hold_but_host_monologues_get_a_cutaway():
labels = _labels([(1, 70), (0, 70)])
cams = [Camera(c.source_id, c.person, 0, 200) for c in CAMS]
cuts = plan_cuts(labels, ["host", "guest"], cams, range_start=0, range_end=140, max_shot=30,
guests=frozenset({"guest"}))
ids = [c["source_id"] for c in cuts]
assert ids[0] == "cam_guest" and cuts[0]["end"] == 70
assert "cam_wide" in ids[1:]
held_off = plan_cuts(labels, ["host", "guest"], cams, range_start=0, range_end=140, max_shot=30,
guests=frozenset({"guest"}), hold_guest=False)
assert held_off[1]["source_id"] == "cam_wide" and held_off[1]["start"] < 70
def test_plan_cuts_scales_to_a_three_hour_episode():
import time
rng = np.random.default_rng(0)
runs = rng.integers(5, 150, size=40000)
labels = np.repeat(rng.choice([0, 1, BOTH], size=len(runs)).astype(np.int32), runs)
cams = [Camera("a", "host", 0, 11000), Camera("b", "guest", 0, 11000), Camera("w", "wide", 0, 11000)]
t0 = time.perf_counter()
cuts = plan_cuts(labels, ["host", "guest"], cams, range_start=0, range_end=len(labels) / 100)
assert time.perf_counter() - t0 < 5
assert all(c["end"] - c["start"] >= 2.0 - 1e-6 for c in cuts[1:-1])
def test_remote_style_keeps_questions_on_the_split_and_opens_answers_wide():
labels = _labels([(0, 6), (1, 5), (0, 4), (1, 30)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=45, min_shot=4, max_shot=0,
guests=frozenset({"guest"}), host_solo=False, guest_min=8, guest_delay=4)
# Question, short answer and follow-up all sit on the split; the long answer goes to the guest after 4 s.
assert [c["source_id"] for c in cuts] == ["cam_wide", "cam_guest"]
assert abs(cuts[1]["start"] - 19.0) < 0.05 and cuts[1]["end"] == 45
def test_remote_style_measures_a_guest_turn_by_its_words_not_the_silence_after():
labels = _labels([(0, 6), (1, 3), (SILENT, 6), (0, 6)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=21, min_shot=1, max_shot=0,
guests=frozenset({"guest"}), host_solo=False, guest_min=8, guest_delay=4)
assert [c["source_id"] for c in cuts] == ["cam_wide"]
def test_remote_style_returns_to_the_split_soon_after_the_answer_ends():
labels = _labels([(0, 4), (1, 20), (SILENT, 5), (0, 4)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=33, min_shot=1, max_shot=0,
guests=frozenset({"guest"}), host_solo=False, guest_min=8, guest_delay=4)
solo = next(c for c in cuts if c["source_id"] == "cam_guest")
assert abs(solo["start"] - 8) < 0.05 and abs(solo["end"] - 25) < 0.05
def test_a_cut_on_a_speech_onset_leads_in_through_silence_only():
# Host, a pause, then the guest: the cut moves LEAD_IN into the pause.
cuts = plan_cuts(_labels([(0, 6), (SILENT, 1), (1, 6)]), ["host", "guest"], CAMS,
range_start=0, range_end=13, min_shot=2.0)
assert [c["source_id"] for c in cuts] == ["cam_host", "cam_guest"]
assert cuts[0]["end"] == cuts[1]["start"] == pytest.approx(7.0 - LEAD_IN)
# A pause shorter than the lead-in: the cut stops at the host's last word.
cuts = plan_cuts(_labels([(0, 6), (SILENT, 0.05), (1, 6)]), ["host", "guest"], CAMS,
range_start=0, range_end=12.05, min_shot=2.0)
assert cuts[1]["start"] == pytest.approx(6.0)
# No pause at all: the guest's first word lands on the cut, as before.
cuts = plan_cuts(_labels([(0, 6), (1, 6)]), ["host", "guest"], CAMS, range_start=0, range_end=12, min_shot=2.0)
assert cuts[1]["start"] == pytest.approx(6.0)
# The host's shot can't drop under min_shot to make room.
cuts = plan_cuts(_labels([(SILENT, 1.0), (0, 1.0), (SILENT, 1.05), (1, 6)]), ["host", "guest"], CAMS,
range_start=1.0, range_end=9.05, min_shot=2.0)
assert [c["source_id"] for c in cuts] == ["cam_host", "cam_guest"]
assert cuts[1]["start"] == pytest.approx(3.0)
# Nor reach back before the guest's camera started rolling.
late = [Camera("cam_host", "host", 0, 100), Camera("cam_guest", "guest", 6.95, 100)]
cuts = plan_cuts(_labels([(0, 6), (SILENT, 1), (1, 6)]), ["host", "guest"], late,
range_start=0, range_end=13, min_shot=2.0)
assert cuts[1]["start"] == pytest.approx(6.95)
def test_cuts_away_from_a_continuing_speaker_do_not_lead_in():
# The remote style holds the split for guest_delay into an answer; the cut
# to the guest falls mid-sentence, not on an onset, so it stays put.
labels = _labels([(0, 4), (SILENT, 1), (1, 20)])
cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=25, min_shot=2.0,
guests=frozenset({"guest"}), host_solo=False, guest_min=8.0, guest_delay=4.0)
solo = next(c for c in cuts if c["source_id"] == "cam_guest")
assert solo["start"] == pytest.approx(9.0)