Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Rank the suggested moments against each other before choosing (#159)
Nika Siradze8c917c107ccf
diff --git a/backend/cli.py b/backend/cli.pyindex 61f6713..68ce24f 100644--- a/backend/cli.py+++ b/backend/cli.py@@ -664,6 +664,12 @@ def cmd_process(args):config["no_speakers"] = Trueif getattr(args, "no_cache", False):config["no_cache"] = True+ # Told rather than detected. Detection reads the opening seconds, so an+ # episode that starts with music, an English intro clip, or a guest saying+ # hello in a second language gets transcribed in the wrong one for its whole+ # hour, and every caption is then a translation nobody asked for.+ if getattr(args, "language", None):+ config["language"] = args.languageif args.quality:config["quality"] = args.qualityif getattr(args, "allow_ass_fallback", False):@@ -838,6 +844,7 @@ def cmd_process(args):file_path=video_path,model_size=config.get("whisper_model", "base"),engine=os.environ.get("PODCLI_ENGINE") or None,+ language=config.get("language") or None,enable_diarization=not config.get("no_speakers", False),progress_callback=_transcribe_progress,wav_path=shared_wav.get() if config.get("energy_boost", True) else None,@@ -953,7 +960,7 @@ def cmd_process(args):# Try an AI CLI first (uses PodStack knowledge base for intelligent selection)from services import ai_providerfrom services.ai_cli import _engine_label- from services.claude_suggest import blend_signal_scores, suggest_initial_with_claude+ from services.claude_suggest import select_clips_with_signal_scores, suggest_initial_with_claudeproviders = ai_provider.status()["providers"]if clips:@@ -961,14 +968,21 @@ def cmd_process(args):elif providers and config.get("ai_select", True):ai_label = providers[0]["label"]print(f" [3/4] Selecting moments with {ai_label} (PodStack)...")+ candidate_top_n = top_n * 2clips = suggest_initial_with_claude(segments=segments,- top_n=top_n,+ top_n=candidate_top_n,progress_callback=lambda pct, msg: print(f" {msg}") if msg else None,reaction_times=reaction_times,)if clips:- blend_signal_scores(clips, energy_data=energy_data, events_data=events_data)+ clips = select_clips_with_signal_scores(+ clips,+ top_n=top_n,+ energy_data=energy_data,+ events_data=events_data,+ progress_callback=lambda pct, msg: print(f" {msg}") if msg else None,+ )engine_id = next((c.get("_ai_engine") for c in clips if c.get("_ai_engine")), "")actual_engine = _engine_label(engine_id) if engine_id in ("claude", "codex") else ai_labelprint(f" ✓ {actual_engine} selected {len(clips)} clips")@@ -4131,6 +4145,7 @@ def main():proc.add_argument("-o", "--output", help="Output directory (default: ./clips)")proc.add_argument("-p", "--preset", help="Load a saved preset")proc.add_argument("--engine", choices=["whisper-py", "whispercpp", "assemblyai"], help="Transcription engine (default: whisper-py; whispercpp is local; assemblyai uses ASSEMBLYAI_API_KEY)")+ proc.add_argument("--language", help="Language of the recording (e.g. es, pt-BR, ka). Auto-detect if omitted.")proc.add_argument("--assemblyai-api-key", help="AssemblyAI API key for --engine assemblyai. Prefer ASSEMBLYAI_API_KEY; command-line secrets can appear in process listings.")proc.add_argument("--fast", action="store_true", help="Draft mode: tiny Whisper, heuristic selection, center crop, low quality")proc.add_argument("--thumbnails", dest="thumbnails", action="store_true", default=None, help="Force thumbnail generation on")diff --git a/backend/main.py b/backend/main.pyindex 265ab66..d4f0a14 100644--- a/backend/main.py+++ b/backend/main.py@@ -717,7 +717,7 @@ def handle_suggest_clips(task_id: str, params: dict):errors: list[str] = []energy_data, events_data, reaction_times = _signal_profiles_for_suggest(task_id, params)- candidate_top_n = top_n * 2 if energy_data or events_data else top_n+ candidate_top_n = top_n * 2clips = suggest_initial_with_claude(segments=segments,top_n=candidate_top_n,@@ -737,6 +737,7 @@ def handle_suggest_clips(task_id: str, params: dict):top_n=top_n,energy_data=energy_data,events_data=events_data,+ progress_callback=lambda pct, msg: emit_progress(task_id, "suggesting", pct, msg),)emit_result(task_id, "success", data={"clips": clips})diff --git a/backend/services/claude_suggest.py b/backend/services/claude_suggest.pyindex 57f4641..499489e 100644--- a/backend/services/claude_suggest.py+++ b/backend/services/claude_suggest.py@@ -319,12 +319,16 @@ def _drop_clips_overlapping(clips: list[dict], exclude_clips: list[dict]) -> lisdef _select_top_by_score(clips: list[dict], top_n: int) -> list[dict]:- """Keep the highest-scored `top_n` clips, then order them by start time.- Ranking by score must come before truncation — otherwise the earliest clips- ship, not the best ones."""+ """Keep the best `top_n` clips, then order them by start time.++ An explicit `rank` wins over the score, but only when every clip has one.+ """if len(clips) <= top_n:return sorted(clips, key=lambda c: c.get("start_second", 0))- ranked = sorted(clips, key=lambda c: c.get("score", 0), reverse=True)[:top_n]+ if clips and all(isinstance(c.get("rank"), int) for c in clips):+ ranked = sorted(clips, key=lambda c: c["rank"])[:top_n]+ else:+ ranked = sorted(clips, key=lambda c: c.get("score", 0), reverse=True)[:top_n]return sorted(ranked, key=lambda c: c.get("start_second", 0))@@ -535,6 +539,8 @@ def suggest_with_claude(learned = podcli_cloud.prompt_block()if learned:instruction = f"{learned}\n\n{instruction}"+ # local_prompt replaces the prompt outright, so local backends need it too.+ prompt = f"{learned}\n\n{prompt}"attempt = ai_provider.generate(instruction,@@ -820,14 +826,119 @@ def blend_signal_scores(return clips+MIN_RANKABLE_SURPLUS = 2+++def rank_clips_with_ai(+ clips: list[dict],+ top_n: int,+ progress_callback: Optional[Callable[[int, str], None]] = None,+ timeout: int = 45,+) -> list[dict]:+ """Compare a whole candidate pool in one transcript-free pass.++ Mutates and returns `clips`, writing a 1-based `rank` and leaving `score`+ untouched. Falls back to score ordering on any failure.+ """+ if len(clips) < top_n + MIN_RANKABLE_SURPLUS:+ return clips++ lines = []+ for idx, clip in enumerate(clips):+ boost = clip.get("signal_boost")+ lines.append(+ f'{idx}. "{clip.get("title", "Untitled")}" '+ f'[{clip.get("duration", 0)}s, {clip.get("content_type", "unknown")}, '+ f'score {clip.get("score", 0)}'+ + (f', audience reaction +{boost}' if boost else "")+ + "]\n"+ f' quote: {str(clip.get("quote", ""))[:200]}\n'+ f' why: {str(clip.get("why", ""))[:200]}'+ )++ learned = podcli_cloud.prompt_block()+ prompt = f"""You are choosing which {top_n} of these {len(clips)} podcast clips to publish.++IMPORTANT: Return ONLY valid JSON. No markdown, no explanation, no code fences.++Each candidate was found by a separate pass over one stretch of the episode, so+their scores are not comparable to each other. Rank them against each other now.++Rank on:+- Would a stranger stop scrolling for this? That is the whole job.+- Does it stand alone, with no missing setup?+- Does it end on a complete thought rather than trailing off?+- Variety across the set: {top_n} clips that make the same point is one clip.++{f"{learned}{chr(10)}{chr(10)}" if learned else ""}CANDIDATES:+{chr(10).join(lines)}++Return this exact JSON structure, best first, every candidate listed exactly once:+{{+ "ranked": [+ {{"id": 3, "why": "One sentence on why this outranks the rest"}},+ {{"id": 0, "why": "..."}}+ ]+}}"""++ label = {"value": "AI"}++ def announce(name: str) -> None:+ label["value"] = name+ if progress_callback:+ progress_callback(0, f"Ranking {len(clips)} candidates with {name}...")++ project_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")+ payload, result = ai_provider.generate_json(+ prompt,+ timeout=timeout,+ project_dir=project_dir,+ on_attempt=announce,+ purpose="rank_moments",+ )++ def give_up(reason: str) -> list[dict]:+ if progress_callback:+ progress_callback(100, f"Ranking unavailable, using scores ({reason})")+ print(f"Clip ranking skipped: {reason}", file=sys.stderr, flush=True)+ return clips++ if not result.ok:+ return give_up(result.error or "ranking pass produced no response")+ if not isinstance(payload, dict) or not isinstance(payload.get("ranked"), list):+ return give_up(f"{label['value']} returned a ranking in an unreadable shape")++ order: list[int] = []+ for entry in payload["ranked"]:+ if not isinstance(entry, dict):+ continue+ idx = entry.get("id")+ if isinstance(idx, int) and 0 <= idx < len(clips) and idx not in order:+ order.append(idx)++ if len(order) < len(clips):+ return give_up(+ f"{label['value']} ranked {len(order)} of {len(clips)} candidates"+ )++ for position, idx in enumerate(order, start=1):+ clips[idx]["rank"] = position+ if progress_callback:+ progress_callback(100, f"{label['value']} ranked {len(clips)} candidates")+ return clips++def select_clips_with_signal_scores(clips: list[dict],top_n: int,energy_data: list[dict] | None = None,events_data: list[dict] | None = None,+ progress_callback: Optional[Callable[[int, str], None]] = None,) -> list[dict]:+ """Blend the audio signals in, rank against each other, cut to `top_n`."""blended = blend_signal_scores(clips, energy_data=energy_data, events_data=events_data)- return _select_top_by_score(blended, top_n)+ ranked = rank_clips_with_ai(blended, top_n, progress_callback=progress_callback)+ return _select_top_by_score(ranked, top_n)def _bucket_coverage_seconds(existing_clips: list[dict], start: float, end: float) -> float:diff --git a/backend/services/podcli_cloud.py b/backend/services/podcli_cloud.pyindex 1624788..8b0da9c 100644--- a/backend/services/podcli_cloud.py+++ b/backend/services/podcli_cloud.py@@ -292,6 +292,9 @@ def backfill_clips(limit: int = 200) -> tuple[int, int]:return synced, failed+_PROMPT_BLOCK_CACHE: dict[str, str] = {}++def prompt_block() -> str:"""What this workspace has learned, phrased for the selection prompt.@@ -300,15 +303,24 @@ def prompt_block() -> str:something that waits for every user to upgrade their CLI.Short timeout and silent on failure: better clips are the point, but not at- the cost of blocking a suggestion run behind a slow network.+ the cost of blocking a suggestion run behind a slow network. Held for the+ life of the process, keyed on the token so a workspace switch cannot serve+ one workspace's learnings into another's prompt."""- if not signed_in():+ token = read_token()+ if not token:return ""+ if token in _PROMPT_BLOCK_CACHE:+ return _PROMPT_BLOCK_CACHE[token]try:payload = request("GET", "/v1/insights/prompt-block", timeout=10)- except CloudError:- return ""- return (payload or {}).get("block") or ""+ block = (payload or {}).get("block") or ""+ except Exception:+ # Never raises: a stall gives TimeoutError and a non-JSON 200 gives+ # JSONDecodeError, neither of them a CloudError.+ block = ""+ _PROMPT_BLOCK_CACHE[token] = block+ return blockdef templates() -> list[dict]:diff --git a/src/models/index.ts b/src/models/index.tsindex a991f73..8c9aa6f 100644--- a/src/models/index.ts+++ b/src/models/index.ts@@ -106,6 +106,7 @@ export interface SuggestedClip {timestamp_display?: string;content_type?: string;score?: number;+ rank?: number;}export interface UIState {diff --git a/src/ui/web-server.ts b/src/ui/web-server.tsindex d021d7e..14234ce 100644--- a/src/ui/web-server.ts+++ b/src/ui/web-server.ts@@ -3502,6 +3502,7 @@ app.post("/api/claude-suggest", async (req, res) => {preview_text: c.preview_text ?? "",content_type: c.content_type,score: c.score,+ rank: c.rank,suggested_caption_style: c.suggested_caption_style || "hormozi",}));uiState.deselectedIndices = [];@@ -3575,6 +3576,7 @@ app.post("/api/find-moment", async (req, res) => {preview_text: c.preview_text ?? "",content_type: c.content_type,score: c.score,+ rank: c.rank,suggested_caption_style: c.suggested_caption_style || "hormozi",});}diff --git a/tests/test_ai_fallback.py b/tests/test_ai_fallback.pyindex 23fe0a4..f9f2ff9 100644--- a/tests/test_ai_fallback.py+++ b/tests/test_ai_fallback.py@@ -462,3 +462,145 @@ class AICliDiscoveryTests(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class RankClipsTests(unittest.TestCase):+ def _pool(self):+ return [+ {"title": "a", "start_second": 30, "end_second": 60, "duration": 30, "score": 18,+ "quote": "q", "why": "w", "content_type": "hot_take"},+ {"title": "b", "start_second": 90, "end_second": 120, "duration": 30, "score": 17,+ "quote": "q", "why": "w", "content_type": "guest_story"},+ {"title": "c", "start_second": 150, "end_second": 180, "duration": 30, "score": 17,+ "quote": "q", "why": "w", "content_type": "hot_take"},+ {"title": "d", "start_second": 210, "end_second": 240, "duration": 30, "score": 16,+ "quote": "q", "why": "w", "content_type": "hot_take"},+ ]++ def _run(self, stdout, clips=None, top_n=2):+ pool = clips if clips is not None else self._pool()+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=""), \+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \+ mock.patch.object(+ ai,+ "_run_ai_command",+ return_value=subprocess.CompletedProcess(+ args=["claude"], returncode=0, stdout=stdout, stderr=""+ ),+ ):+ return cs.rank_clips_with_ai(pool, top_n), pool++ def test_ranking_decides_the_cut_that_tied_scores_cannot(self):+ ranked, _ = self._run(json.dumps({+ "ranked": [{"id": 2}, {"id": 0}, {"id": 3}, {"id": 1}],+ }))+ selected = cs._select_top_by_score(ranked, 2)+ self.assertEqual([c["title"] for c in selected], ["a", "c"])++ def test_selection_is_returned_in_timeline_order(self):+ ranked, _ = self._run(json.dumps({"ranked": [{"id": 3}, {"id": 1}, {"id": 0}, {"id": 2}]}))+ selected = cs._select_top_by_score(ranked, 2)+ self.assertEqual([c["start_second"] for c in selected], [90, 210])++ def test_a_partial_ranking_is_discarded_whole(self):+ narrated = []+ pool = self._pool()+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=""), \+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \+ mock.patch.object(+ ai,+ "_run_ai_command",+ return_value=subprocess.CompletedProcess(+ args=["claude"], returncode=0,+ stdout=json.dumps({"ranked": [{"id": 2}, {"id": 0}]}), stderr="",+ ),+ ):+ cs.rank_clips_with_ai(pool, 2, progress_callback=lambda _p, m: narrated.append(m))+ self.assertTrue(all("rank" not in c for c in pool))+ self.assertEqual([c["title"] for c in cs._select_top_by_score(pool, 2)], ["a", "b"])+ self.assertTrue(any("Ranking unavailable" in m for m in narrated))++ def test_an_unreadable_answer_leaves_score_ordering_alone(self):+ pool = self._pool()+ self._run("not json at all", clips=pool)+ self.assertTrue(all("rank" not in c for c in pool))++ def test_no_ai_available_is_not_an_error(self):+ pool = self._pool()+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=""), \+ mock.patch.object(ap, "_chain", return_value=[]):+ out = cs.rank_clips_with_ai(pool, 2)+ self.assertIs(out, pool)+ self.assertTrue(all("rank" not in c for c in pool))++ def test_a_pool_with_nothing_to_drop_makes_no_call(self):+ pool = self._pool()[:3]+ with mock.patch.object(ap, "_chain") as chain:+ cs.rank_clips_with_ai(pool, 2)+ chain.assert_not_called()++ def test_the_pass_is_metered_under_its_own_purpose(self):+ seen = {}+ original = ap.generate_json++ def capture(prompt, **kwargs):+ seen.update(kwargs)+ seen["prompt"] = prompt+ return original(prompt, **kwargs)++ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value="RETAINS BEST: hot takes"), \+ mock.patch.object(ap, "generate_json", capture), \+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \+ mock.patch.object(+ ai,+ "_run_ai_command",+ return_value=subprocess.CompletedProcess(+ args=["claude"], returncode=0,+ stdout=json.dumps({"ranked": [{"id": i} for i in range(4)]}), stderr="",+ ),+ ):+ cs.rank_clips_with_ai(self._pool(), 2)++ self.assertEqual(seen["purpose"], "rank_moments")+ self.assertNotIn("stable_prefix", seen)+ self.assertIn("RETAINS BEST: hot takes", seen["prompt"])+++class WorkspaceLearningsTests(unittest.TestCase):+ def _prompt_sent_to_cli(self, learned):+ sent = {}++ def capture(**kwargs):+ with open(kwargs["prompt_file"], encoding="utf-8") as fh:+ sent["prompt"] = fh.read()+ return subprocess.CompletedProcess(+ args=["claude"], returncode=0,+ stdout=json.dumps({"clips": [{+ "title": "A moment",+ "start_second": 10.0,+ "end_second": 40.0,+ "segments": [{"start": 10.0, "end": 40.0}],+ "scores": {"standalone": 5, "hook": 5, "relevance": 4, "quotability": 4},+ "quote": "q", "why": "w",+ }]}),+ stderr="",+ )++ segments = [+ {"start": float(i * 5), "end": float(i * 5 + 5), "text": f"line {i}", "speaker": "A"}+ for i in range(12)+ ]+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=learned), \+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \+ mock.patch.object(ai, "_run_ai_command", side_effect=capture):+ cs.suggest_with_claude(segments=segments, top_n=1)+ return sent.get("prompt", "")++ def test_a_local_cli_receives_the_workspace_learnings(self):+ prompt = self._prompt_sent_to_cli("WHAT WORKS ON THIS CHANNEL: founder stories.")+ self.assertIn("WHAT WORKS ON THIS CHANNEL: founder stories.", prompt)++ def test_the_free_path_prompt_is_untouched(self):+ prompt = self._prompt_sent_to_cli("")+ self.assertFalse(prompt.startswith("\n"))+ self.assertTrue(prompt.startswith("You are a viral clip editor"))diff --git a/tests/test_suggest_handler.py b/tests/test_suggest_handler.pyindex a28d21c..d257491 100644--- a/tests/test_suggest_handler.py+++ b/tests/test_suggest_handler.py@@ -112,6 +112,10 @@ class SuggestReactionTimesTests(unittest.TestCase):captured, _ = self._run({"segments": SEGMENTS})self.assertIsNone(captured["reaction_times"])+ def test_a_pool_is_requested_even_with_no_audio_to_analyse(self):+ captured, _ = self._run({"segments": SEGMENTS, "top_n": 3})+ self.assertEqual(captured["top_n"], 6)+if __name__ == "__main__":unittest.main()