Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

13.8 KB
Raw Blame
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from unittest import mock

import numpy as np

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
BACKEND_ROOT = os.path.join(ROOT, "backend")
if BACKEND_ROOT not in sys.path:
    sys.path.insert(0, BACKEND_ROOT)

from services import clip_generator as cg
from services.opening_hook import (
    keyframes_to_playback,
    order_with_hook,
    snap_hook_to_words,
    validate_hook,
)


class ValidateHookTests(unittest.TestCase):
    def test_none_means_no_hook(self):
        self.assertIsNone(validate_hook(None, 10, 40))

    def test_valid_hook_is_normalized(self):
        hook = validate_hook({"start": 12, "end": 15, "mode": "move"}, 10, 40)
        self.assertEqual(hook, {"start": 12.0, "end": 15.0, "mode": "move"})

    def test_hook_on_the_clip_edges_is_inside(self):
        self.assertIsNotNone(validate_hook({"start": 10, "end": 12, "mode": "repeat"}, 10, 40))
        self.assertIsNotNone(validate_hook({"start": 38, "end": 40, "mode": "repeat"}, 10, 40))

    def test_end_must_follow_start(self):
        with self.assertRaisesRegex(ValueError, "greater than"):
            validate_hook({"start": 15, "end": 15, "mode": "repeat"}, 10, 40)

    def test_shorter_than_one_second_is_rejected(self):
        with self.assertRaisesRegex(ValueError, "between 1 and 15"):
            validate_hook({"start": 12, "end": 12.5, "mode": "repeat"}, 10, 40)

    def test_longer_than_fifteen_seconds_is_rejected(self):
        with self.assertRaisesRegex(ValueError, "between 1 and 15"):
            validate_hook({"start": 12, "end": 28, "mode": "repeat"}, 10, 40)

    def test_outside_the_clip_range_is_rejected(self):
        with self.assertRaisesRegex(ValueError, "not inside the clip body"):
            validate_hook({"start": 8, "end": 11, "mode": "repeat"}, 10, 40)
        with self.assertRaisesRegex(ValueError, "not inside the clip body"):
            validate_hook({"start": 39, "end": 41, "mode": "repeat"}, 10, 40)

    def test_hook_in_a_gap_between_segments_is_rejected(self):
        segments = [{"start": 10, "end": 15}, {"start": 20, "end": 30}]
        with self.assertRaisesRegex(ValueError, "not inside the clip body"):
            validate_hook({"start": 14, "end": 17, "mode": "repeat"}, 10, 30, segments)

    def test_hook_across_touching_segments_is_inside(self):
        segments = [{"start": 10, "end": 15}, {"start": 15, "end": 30}]
        self.assertIsNotNone(
            validate_hook({"start": 14, "end": 17, "mode": "repeat"}, 10, 30, segments)
        )

    def test_unknown_mode_is_rejected(self):
        with self.assertRaisesRegex(ValueError, "repeat"):
            validate_hook({"start": 12, "end": 14, "mode": "loop"}, 10, 40)
        with self.assertRaisesRegex(ValueError, "repeat"):
            validate_hook({"start": 12, "end": 14}, 10, 40)

    def test_non_numeric_bounds_are_rejected(self):
        with self.assertRaisesRegex(ValueError, "numbers"):
            validate_hook({"start": "12", "end": 14, "mode": "repeat"}, 10, 40)
        with self.assertRaisesRegex(ValueError, "numbers"):
            validate_hook({"start": True, "end": 14, "mode": "repeat"}, 10, 40)

    def test_non_object_is_rejected(self):
        with self.assertRaisesRegex(ValueError, "object"):
            validate_hook([12, 14], 10, 40)

    def test_generate_clip_rejects_a_bad_hook_before_rendering(self):
        with tempfile.TemporaryDirectory() as td:
            video = os.path.join(td, "v.mp4")
            open(video, "wb").close()
            with mock.patch.object(cg, "cut_segment") as cut, \
                 mock.patch.object(cg, "cut_multi_segment") as cut_multi:
                with self.assertRaisesRegex(ValueError, "not inside the clip body"):
                    cg.generate_clip(
                        video_path=video, start_second=10, end_second=20,
                        hook={"start": 25, "end": 27, "mode": "repeat"},
                    )
            cut.assert_not_called()
            cut_multi.assert_not_called()


class OrderWithHookTests(unittest.TestCase):
    body = [{"start": 10, "end": 20}, {"start": 22, "end": 30}]

    def test_repeat_plays_the_hook_then_the_whole_body(self):
        ranges = order_with_hook(self.body, {"start": 24, "end": 27, "mode": "repeat"})
        self.assertEqual(ranges, [
            {"start": 24, "end": 27},
            {"start": 10, "end": 20},
            {"start": 22, "end": 30},
        ])

    def test_move_removes_the_hook_from_the_body(self):
        ranges = order_with_hook(self.body, {"start": 24, "end": 27, "mode": "move"})
        self.assertEqual(ranges, [
            {"start": 24, "end": 27},
            {"start": 10, "end": 20},
            {"start": 22, "end": 24},
            {"start": 27, "end": 30},
        ])

    def test_move_drops_a_sliver_left_at_a_segment_edge(self):
        ranges = order_with_hook(self.body, {"start": 22.05, "end": 25, "mode": "move"})
        self.assertEqual(ranges, [
            {"start": 22.05, "end": 25},
            {"start": 10, "end": 20},
            {"start": 25, "end": 30},
        ])

    def test_move_cannot_take_the_whole_body(self):
        with self.assertRaisesRegex(ValueError, "whole clip"):
            order_with_hook([{"start": 10, "end": 14}], {"start": 10, "end": 14, "mode": "move"})


class SnapHookTests(unittest.TestCase):
    words = [
        {"word": "one", "start": 1.0, "end": 1.6},
        {"word": "two", "start": 1.8, "end": 2.4},
        {"word": "three", "start": 2.6, "end": 3.3},
    ]

    def test_edges_inside_words_widen_to_the_whole_word(self):
        hook = snap_hook_to_words({"start": 1.2, "end": 3.0, "mode": "repeat"}, self.words)
        self.assertEqual((hook["start"], hook["end"]), (1.0, 3.3))
        self.assertEqual(hook["mode"], "repeat")

    def test_edges_in_gaps_are_left_alone(self):
        hook = snap_hook_to_words({"start": 1.7, "end": 2.5, "mode": "move"}, self.words)
        self.assertEqual((hook["start"], hook["end"]), (1.7, 2.5))


class KeyframesToPlaybackTests(unittest.TestCase):
    def test_identity_for_one_range_at_the_origin(self):
        kf = [{"t": 0.0, "x_pct": 30}, {"t": 4.0, "x_pct": 70}]
        out = keyframes_to_playback(kf, 10.0, [{"start": 10.0, "end": 20.0}])
        self.assertEqual(out, kf)

    def test_each_range_opens_on_the_keyframe_in_force(self):
        # Source clock: 30% from 10s, 70% from 14s. The hook 15-17 plays first
        # at 70%; the body then replays 10-20 from 30% and switches at 4s.
        kf = [{"t": 0.0, "x_pct": 30}, {"t": 4.0, "x_pct": 70}]
        ranges = [{"start": 15.0, "end": 17.0}, {"start": 10.0, "end": 20.0}]
        out = keyframes_to_playback(kf, 10.0, ranges, part_durations=[2.0, 10.0])
        self.assertEqual(out, [
            {"t": 0.0, "x_pct": 70},
            {"t": 2.0, "x_pct": 30},
            {"t": 6.0, "x_pct": 70},
        ])


@unittest.skipUnless(
    shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg/ffprobe not installed"
)
class OpeningHookRenderTests(unittest.TestCase):
    """Real render of a source whose every second has its own colour and its
    own tone, so the order of the output can be read back frame by frame."""

    FPS = 25
    COLORS = ["red", "green", "blue", "yellow", "cyan", "magenta", "white", "gray", "orange", "purple"]
    RGB = {
        "red": (255, 0, 0), "green": (0, 128, 0), "blue": (0, 0, 255),
        "yellow": (255, 255, 0), "cyan": (0, 255, 255), "magenta": (255, 0, 255),
        "white": (255, 255, 255), "gray": (128, 128, 128), "orange": (255, 165, 0),
        "purple": (128, 0, 128),
    }

    @staticmethod
    def freq(second):
        return 300 + 100 * second

    @classmethod
    def setUpClass(cls):
        cls.tmpdir = tempfile.mkdtemp(prefix="podcli-hook-test-")
        cls.src = os.path.join(cls.tmpdir, "src.mp4")
        n = len(cls.COLORS)
        cmd = ["ffmpeg", "-y", "-loglevel", "error"]
        for color in cls.COLORS:
            cmd += ["-f", "lavfi", "-i", f"color=c={color}:s=320x240:r={cls.FPS}:d=1"]
        for i in range(n):
            cmd += ["-f", "lavfi", "-i", f"sine=frequency={cls.freq(i)}:sample_rate=44100:duration=1"]
        pairs = "".join(f"[{i}:v][{n + i}:a]" for i in range(n))
        cmd += [
            "-filter_complex", f"{pairs}concat=n={n}:v=1:a=1[v][a]",
            "-map", "[v]", "-map", "[a]",
            "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac",
            cls.src,
        ]
        subprocess.run(cmd, check=True, capture_output=True)
        # One word per second, well inside it.
        cls.words = [
            {"word": f"w{i}", "start": i + 0.2, "end": i + 0.7, "speaker": None}
            for i in range(n)
        ]

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.tmpdir, ignore_errors=True)

    def _render(self, mode):
        captured = {}
        real_write = cg.write_sidecars

        def spy(words, base, *a, **kw):
            captured["words"] = words
            return real_write(words, base, *a, **kw)

        with mock.patch.object(cg, "write_sidecars", side_effect=spy):
            result = cg.generate_clip(
                video_path=self.src,
                start_second=2.0,
                end_second=8.0,
                caption_style="subtle",
                crop_strategy="center",
                transcript_words=self.words,
                title=f"hook_{mode}",
                output_dir=os.path.join(self.tmpdir, mode),
                captions=False,
                # Sidecars otherwise follow `captions`; this test inspects
                # the sidecar words directly, so request them explicitly.
                write_subtitles=True,
                clean_fillers=False,
                trim_opening=False,
                hook={"start": 5.0, "end": 7.0, "mode": mode},
            )
        return result, captured["words"]

    def _frame_rgb(self, path, t):
        raw = subprocess.run(
            ["ffmpeg", "-loglevel", "error", "-ss", f"{t:.3f}", "-i", path,
             "-frames:v", "1", "-vf", "scale=32:32", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
            check=True, capture_output=True,
        ).stdout
        px = np.frombuffer(raw, dtype=np.uint8).reshape(-1, 3)
        return tuple(int(c) for c in px.mean(axis=0).round())

    def _dominant_freq(self, path, t, window=0.4):
        raw = subprocess.run(
            ["ffmpeg", "-loglevel", "error", "-ss", f"{t:.3f}", "-i", path, "-t", f"{window}",
             "-vn", "-ac", "1", "-ar", "16000", "-f", "s16le", "-"],
            check=True, capture_output=True,
        ).stdout
        samples = np.frombuffer(raw, dtype=np.int16).astype(np.float64)
        spectrum = np.abs(np.fft.rfft(samples * np.hanning(len(samples))))
        freqs = np.fft.rfftfreq(len(samples), 1 / 16000)
        return float(freqs[int(spectrum.argmax())])

    def _video_duration(self, path):
        out = subprocess.run(
            ["ffprobe", "-v", "error", "-select_streams", "v:0", "-count_frames",
             "-show_entries", "stream=nb_read_frames", "-of", "csv=p=0", path],
            check=True, capture_output=True, text=True,
        ).stdout.strip()
        return int(out) / self.FPS

    def _assert_plays(self, path, playback_t, source_second):
        got = self._frame_rgb(path, playback_t)
        want = self.RGB[self.COLORS[source_second]]
        for g, w in zip(got, want):
            self.assertLessEqual(
                abs(g - w), 40,
                f"at {playback_t}s the frame is {got}, expected source second "
                f"{source_second} ({self.COLORS[source_second]} {want})",
            )
        self.assertAlmostEqual(
            self._dominant_freq(path, playback_t - 0.2), self.freq(source_second), delta=15,
            msg=f"at {playback_t}s the tone is not source second {source_second}",
        )

    def _assert_words(self, words, expected):
        got = [(w["word"], w["start"]) for w in words]
        self.assertEqual([g[0] for g in got], [e[0] for e in expected])
        for (word, start), (_, want) in zip(got, expected):
            self.assertAlmostEqual(start, want, delta=0.05, msg=f"{word} at {start}, expected {want}")

    def test_repeat_plays_the_hook_first_then_the_whole_body(self):
        result, words = self._render("repeat")
        path = result["output_path"]
        # Hook 5-7, then body 2-8: 2 + 6 seconds.
        plan = [(0.5, 5), (1.5, 6), (2.5, 2), (3.5, 3), (4.5, 4), (5.5, 5), (6.5, 6), (7.5, 7)]
        for playback_t, source_second in plan:
            self._assert_plays(path, playback_t, source_second)
        self.assertAlmostEqual(self._video_duration(path), 8.0, delta=1 / self.FPS)
        self.assertAlmostEqual(result["duration"], 8.0, delta=0.1)
        self.assertEqual(result["hook"], {"start": 5.0, "end": 7.0, "mode": "repeat"})
        self._assert_words(words, [
            ("w5", 0.2), ("w6", 1.2),
            ("w2", 2.2), ("w3", 3.2), ("w4", 4.2), ("w5", 5.2), ("w6", 6.2), ("w7", 7.2),
        ])
        with open(result["srt_path"], encoding="utf-8") as f:
            self.assertIn("w5 w6 w2 w3 w4 w5 w6 w7", f.read())

    def test_move_plays_the_hook_first_and_drops_it_from_the_body(self):
        result, words = self._render("move")
        path = result["output_path"]
        # Hook 5-7, then body 2-5 and 7-8: 2 + 3 + 1 seconds.
        plan = [(0.5, 5), (1.5, 6), (2.5, 2), (3.5, 3), (4.5, 4), (5.5, 7)]
        for playback_t, source_second in plan:
            self._assert_plays(path, playback_t, source_second)
        # Each of the three parts can round up to a whole frame on its own,
        # depending on the ffmpeg build; captions follow the probed lengths.
        self.assertAlmostEqual(self._video_duration(path), 6.0, delta=3 / self.FPS)
        self.assertAlmostEqual(result["duration"], 6.0, delta=0.1)
        self._assert_words(words, [
            ("w5", 0.2), ("w6", 1.2), ("w2", 2.2), ("w3", 3.2), ("w4", 4.2), ("w7", 5.2),
        ])


if __name__ == "__main__":
    unittest.main()