Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Rank the suggested moments against each other before choosing (#159)

Nika Siradze8c917c107ccf
diff --git a/backend/cli.py b/backend/cli.py
index 61f6713..68ce24f 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -664,6 +664,12 @@ def cmd_process(args):
config["no_speakers"] = True
if getattr(args, "no_cache", False):
config["no_cache"] = True
+ # Told rather than detected. Detection reads the opening seconds, so an
+ # episode that starts with music, an English intro clip, or a guest saying
+ # hello in a second language gets transcribed in the wrong one for its whole
+ # hour, and every caption is then a translation nobody asked for.
+ if getattr(args, "language", None):
+ config["language"] = args.language
if args.quality:
config["quality"] = args.quality
if getattr(args, "allow_ass_fallback", False):
@@ -838,6 +844,7 @@ def cmd_process(args):
file_path=video_path,
model_size=config.get("whisper_model", "base"),
engine=os.environ.get("PODCLI_ENGINE") or None,
+ language=config.get("language") or None,
enable_diarization=not config.get("no_speakers", False),
progress_callback=_transcribe_progress,
wav_path=shared_wav.get() if config.get("energy_boost", True) else None,
@@ -953,7 +960,7 @@ def cmd_process(args):
# Try an AI CLI first (uses PodStack knowledge base for intelligent selection)
from services import ai_provider
from services.ai_cli import _engine_label
- from services.claude_suggest import blend_signal_scores, suggest_initial_with_claude
+ from services.claude_suggest import select_clips_with_signal_scores, suggest_initial_with_claude
providers = ai_provider.status()["providers"]
if clips:
@@ -961,14 +968,21 @@ def cmd_process(args):
elif providers and config.get("ai_select", True):
ai_label = providers[0]["label"]
print(f" [3/4] Selecting moments with {ai_label} (PodStack)...")
+ candidate_top_n = top_n * 2
clips = suggest_initial_with_claude(
segments=segments,
- top_n=top_n,
+ top_n=candidate_top_n,
progress_callback=lambda pct, msg: print(f" {msg}") if msg else None,
reaction_times=reaction_times,
)
if clips:
- blend_signal_scores(clips, energy_data=energy_data, events_data=events_data)
+ clips = select_clips_with_signal_scores(
+ clips,
+ top_n=top_n,
+ energy_data=energy_data,
+ events_data=events_data,
+ progress_callback=lambda pct, msg: print(f" {msg}") if msg else None,
+ )
engine_id = next((c.get("_ai_engine") for c in clips if c.get("_ai_engine")), "")
actual_engine = _engine_label(engine_id) if engine_id in ("claude", "codex") else ai_label
print(f" ✓ {actual_engine} selected {len(clips)} clips")
@@ -4131,6 +4145,7 @@ def main():
proc.add_argument("-o", "--output", help="Output directory (default: ./clips)")
proc.add_argument("-p", "--preset", help="Load a saved preset")
proc.add_argument("--engine", choices=["whisper-py", "whispercpp", "assemblyai"], help="Transcription engine (default: whisper-py; whispercpp is local; assemblyai uses ASSEMBLYAI_API_KEY)")
+ proc.add_argument("--language", help="Language of the recording (e.g. es, pt-BR, ka). Auto-detect if omitted.")
proc.add_argument("--assemblyai-api-key", help="AssemblyAI API key for --engine assemblyai. Prefer ASSEMBLYAI_API_KEY; command-line secrets can appear in process listings.")
proc.add_argument("--fast", action="store_true", help="Draft mode: tiny Whisper, heuristic selection, center crop, low quality")
proc.add_argument("--thumbnails", dest="thumbnails", action="store_true", default=None, help="Force thumbnail generation on")
diff --git a/backend/main.py b/backend/main.py
index 265ab66..d4f0a14 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -717,7 +717,7 @@ def handle_suggest_clips(task_id: str, params: dict):
errors: list[str] = []
energy_data, events_data, reaction_times = _signal_profiles_for_suggest(task_id, params)
- candidate_top_n = top_n * 2 if energy_data or events_data else top_n
+ candidate_top_n = top_n * 2
clips = suggest_initial_with_claude(
segments=segments,
top_n=candidate_top_n,
@@ -737,6 +737,7 @@ def handle_suggest_clips(task_id: str, params: dict):
top_n=top_n,
energy_data=energy_data,
events_data=events_data,
+ progress_callback=lambda pct, msg: emit_progress(task_id, "suggesting", pct, msg),
)
emit_result(task_id, "success", data={"clips": clips})
diff --git a/backend/services/claude_suggest.py b/backend/services/claude_suggest.py
index 57f4641..499489e 100644
--- a/backend/services/claude_suggest.py
+++ b/backend/services/claude_suggest.py
@@ -319,12 +319,16 @@ def _drop_clips_overlapping(clips: list[dict], exclude_clips: list[dict]) -> lis
def _select_top_by_score(clips: list[dict], top_n: int) -> list[dict]:
- """Keep the highest-scored `top_n` clips, then order them by start time.
- Ranking by score must come before truncation — otherwise the earliest clips
- ship, not the best ones."""
+ """Keep the best `top_n` clips, then order them by start time.
+
+ An explicit `rank` wins over the score, but only when every clip has one.
+ """
if len(clips) <= top_n:
return sorted(clips, key=lambda c: c.get("start_second", 0))
- ranked = sorted(clips, key=lambda c: c.get("score", 0), reverse=True)[:top_n]
+ if clips and all(isinstance(c.get("rank"), int) for c in clips):
+ ranked = sorted(clips, key=lambda c: c["rank"])[:top_n]
+ else:
+ ranked = sorted(clips, key=lambda c: c.get("score", 0), reverse=True)[:top_n]
return sorted(ranked, key=lambda c: c.get("start_second", 0))
@@ -535,6 +539,8 @@ def suggest_with_claude(
learned = podcli_cloud.prompt_block()
if learned:
instruction = f"{learned}\n\n{instruction}"
+ # local_prompt replaces the prompt outright, so local backends need it too.
+ prompt = f"{learned}\n\n{prompt}"
attempt = ai_provider.generate(
instruction,
@@ -820,14 +826,119 @@ def blend_signal_scores(
return clips
+MIN_RANKABLE_SURPLUS = 2
+
+
+def rank_clips_with_ai(
+ clips: list[dict],
+ top_n: int,
+ progress_callback: Optional[Callable[[int, str], None]] = None,
+ timeout: int = 45,
+) -> list[dict]:
+ """Compare a whole candidate pool in one transcript-free pass.
+
+ Mutates and returns `clips`, writing a 1-based `rank` and leaving `score`
+ untouched. Falls back to score ordering on any failure.
+ """
+ if len(clips) < top_n + MIN_RANKABLE_SURPLUS:
+ return clips
+
+ lines = []
+ for idx, clip in enumerate(clips):
+ boost = clip.get("signal_boost")
+ lines.append(
+ f'{idx}. "{clip.get("title", "Untitled")}" '
+ f'[{clip.get("duration", 0)}s, {clip.get("content_type", "unknown")}, '
+ f'score {clip.get("score", 0)}'
+ + (f', audience reaction +{boost}' if boost else "")
+ + "]\n"
+ f' quote: {str(clip.get("quote", ""))[:200]}\n'
+ f' why: {str(clip.get("why", ""))[:200]}'
+ )
+
+ learned = podcli_cloud.prompt_block()
+ prompt = f"""You are choosing which {top_n} of these {len(clips)} podcast clips to publish.
+
+IMPORTANT: Return ONLY valid JSON. No markdown, no explanation, no code fences.
+
+Each candidate was found by a separate pass over one stretch of the episode, so
+their scores are not comparable to each other. Rank them against each other now.
+
+Rank on:
+- Would a stranger stop scrolling for this? That is the whole job.
+- Does it stand alone, with no missing setup?
+- Does it end on a complete thought rather than trailing off?
+- Variety across the set: {top_n} clips that make the same point is one clip.
+
+{f"{learned}{chr(10)}{chr(10)}" if learned else ""}CANDIDATES:
+{chr(10).join(lines)}
+
+Return this exact JSON structure, best first, every candidate listed exactly once:
+{{
+ "ranked": [
+ {{"id": 3, "why": "One sentence on why this outranks the rest"}},
+ {{"id": 0, "why": "..."}}
+ ]
+}}"""
+
+ label = {"value": "AI"}
+
+ def announce(name: str) -> None:
+ label["value"] = name
+ if progress_callback:
+ progress_callback(0, f"Ranking {len(clips)} candidates with {name}...")
+
+ project_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")
+ payload, result = ai_provider.generate_json(
+ prompt,
+ timeout=timeout,
+ project_dir=project_dir,
+ on_attempt=announce,
+ purpose="rank_moments",
+ )
+
+ def give_up(reason: str) -> list[dict]:
+ if progress_callback:
+ progress_callback(100, f"Ranking unavailable, using scores ({reason})")
+ print(f"Clip ranking skipped: {reason}", file=sys.stderr, flush=True)
+ return clips
+
+ if not result.ok:
+ return give_up(result.error or "ranking pass produced no response")
+ if not isinstance(payload, dict) or not isinstance(payload.get("ranked"), list):
+ return give_up(f"{label['value']} returned a ranking in an unreadable shape")
+
+ order: list[int] = []
+ for entry in payload["ranked"]:
+ if not isinstance(entry, dict):
+ continue
+ idx = entry.get("id")
+ if isinstance(idx, int) and 0 <= idx < len(clips) and idx not in order:
+ order.append(idx)
+
+ if len(order) < len(clips):
+ return give_up(
+ f"{label['value']} ranked {len(order)} of {len(clips)} candidates"
+ )
+
+ for position, idx in enumerate(order, start=1):
+ clips[idx]["rank"] = position
+ if progress_callback:
+ progress_callback(100, f"{label['value']} ranked {len(clips)} candidates")
+ return clips
+
+
def select_clips_with_signal_scores(
clips: list[dict],
top_n: int,
energy_data: list[dict] | None = None,
events_data: list[dict] | None = None,
+ progress_callback: Optional[Callable[[int, str], None]] = None,
) -> list[dict]:
+ """Blend the audio signals in, rank against each other, cut to `top_n`."""
blended = blend_signal_scores(clips, energy_data=energy_data, events_data=events_data)
- return _select_top_by_score(blended, top_n)
+ ranked = rank_clips_with_ai(blended, top_n, progress_callback=progress_callback)
+ return _select_top_by_score(ranked, top_n)
def _bucket_coverage_seconds(existing_clips: list[dict], start: float, end: float) -> float:
diff --git a/backend/services/podcli_cloud.py b/backend/services/podcli_cloud.py
index 1624788..8b0da9c 100644
--- a/backend/services/podcli_cloud.py
+++ b/backend/services/podcli_cloud.py
@@ -292,6 +292,9 @@ def backfill_clips(limit: int = 200) -> tuple[int, int]:
return synced, failed
+_PROMPT_BLOCK_CACHE: dict[str, str] = {}
+
+
def prompt_block() -> str:
"""What this workspace has learned, phrased for the selection prompt.
@@ -300,15 +303,24 @@ def prompt_block() -> str:
something that waits for every user to upgrade their CLI.
Short timeout and silent on failure: better clips are the point, but not at
- the cost of blocking a suggestion run behind a slow network.
+ the cost of blocking a suggestion run behind a slow network. Held for the
+ life of the process, keyed on the token so a workspace switch cannot serve
+ one workspace's learnings into another's prompt.
"""
- if not signed_in():
+ token = read_token()
+ if not token:
return ""
+ if token in _PROMPT_BLOCK_CACHE:
+ return _PROMPT_BLOCK_CACHE[token]
try:
payload = request("GET", "/v1/insights/prompt-block", timeout=10)
- except CloudError:
- return ""
- return (payload or {}).get("block") or ""
+ block = (payload or {}).get("block") or ""
+ except Exception:
+ # Never raises: a stall gives TimeoutError and a non-JSON 200 gives
+ # JSONDecodeError, neither of them a CloudError.
+ block = ""
+ _PROMPT_BLOCK_CACHE[token] = block
+ return block
def templates() -> list[dict]:
diff --git a/src/models/index.ts b/src/models/index.ts
index a991f73..8c9aa6f 100644
--- a/src/models/index.ts
+++ b/src/models/index.ts
@@ -106,6 +106,7 @@ export interface SuggestedClip {
timestamp_display?: string;
content_type?: string;
score?: number;
+ rank?: number;
}
export interface UIState {
diff --git a/src/ui/web-server.ts b/src/ui/web-server.ts
index d021d7e..14234ce 100644
--- a/src/ui/web-server.ts
+++ b/src/ui/web-server.ts
@@ -3502,6 +3502,7 @@ app.post("/api/claude-suggest", async (req, res) => {
preview_text: c.preview_text ?? "",
content_type: c.content_type,
score: c.score,
+ rank: c.rank,
suggested_caption_style: c.suggested_caption_style || "hormozi",
}));
uiState.deselectedIndices = [];
@@ -3575,6 +3576,7 @@ app.post("/api/find-moment", async (req, res) => {
preview_text: c.preview_text ?? "",
content_type: c.content_type,
score: c.score,
+ rank: c.rank,
suggested_caption_style: c.suggested_caption_style || "hormozi",
});
}
diff --git a/tests/test_ai_fallback.py b/tests/test_ai_fallback.py
index 23fe0a4..f9f2ff9 100644
--- a/tests/test_ai_fallback.py
+++ b/tests/test_ai_fallback.py
@@ -462,3 +462,145 @@ class AICliDiscoveryTests(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class RankClipsTests(unittest.TestCase):
+ def _pool(self):
+ return [
+ {"title": "a", "start_second": 30, "end_second": 60, "duration": 30, "score": 18,
+ "quote": "q", "why": "w", "content_type": "hot_take"},
+ {"title": "b", "start_second": 90, "end_second": 120, "duration": 30, "score": 17,
+ "quote": "q", "why": "w", "content_type": "guest_story"},
+ {"title": "c", "start_second": 150, "end_second": 180, "duration": 30, "score": 17,
+ "quote": "q", "why": "w", "content_type": "hot_take"},
+ {"title": "d", "start_second": 210, "end_second": 240, "duration": 30, "score": 16,
+ "quote": "q", "why": "w", "content_type": "hot_take"},
+ ]
+
+ def _run(self, stdout, clips=None, top_n=2):
+ pool = clips if clips is not None else self._pool()
+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=""), \
+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \
+ mock.patch.object(
+ ai,
+ "_run_ai_command",
+ return_value=subprocess.CompletedProcess(
+ args=["claude"], returncode=0, stdout=stdout, stderr=""
+ ),
+ ):
+ return cs.rank_clips_with_ai(pool, top_n), pool
+
+ def test_ranking_decides_the_cut_that_tied_scores_cannot(self):
+ ranked, _ = self._run(json.dumps({
+ "ranked": [{"id": 2}, {"id": 0}, {"id": 3}, {"id": 1}],
+ }))
+ selected = cs._select_top_by_score(ranked, 2)
+ self.assertEqual([c["title"] for c in selected], ["a", "c"])
+
+ def test_selection_is_returned_in_timeline_order(self):
+ ranked, _ = self._run(json.dumps({"ranked": [{"id": 3}, {"id": 1}, {"id": 0}, {"id": 2}]}))
+ selected = cs._select_top_by_score(ranked, 2)
+ self.assertEqual([c["start_second"] for c in selected], [90, 210])
+
+ def test_a_partial_ranking_is_discarded_whole(self):
+ narrated = []
+ pool = self._pool()
+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=""), \
+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \
+ mock.patch.object(
+ ai,
+ "_run_ai_command",
+ return_value=subprocess.CompletedProcess(
+ args=["claude"], returncode=0,
+ stdout=json.dumps({"ranked": [{"id": 2}, {"id": 0}]}), stderr="",
+ ),
+ ):
+ cs.rank_clips_with_ai(pool, 2, progress_callback=lambda _p, m: narrated.append(m))
+ self.assertTrue(all("rank" not in c for c in pool))
+ self.assertEqual([c["title"] for c in cs._select_top_by_score(pool, 2)], ["a", "b"])
+ self.assertTrue(any("Ranking unavailable" in m for m in narrated))
+
+ def test_an_unreadable_answer_leaves_score_ordering_alone(self):
+ pool = self._pool()
+ self._run("not json at all", clips=pool)
+ self.assertTrue(all("rank" not in c for c in pool))
+
+ def test_no_ai_available_is_not_an_error(self):
+ pool = self._pool()
+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=""), \
+ mock.patch.object(ap, "_chain", return_value=[]):
+ out = cs.rank_clips_with_ai(pool, 2)
+ self.assertIs(out, pool)
+ self.assertTrue(all("rank" not in c for c in pool))
+
+ def test_a_pool_with_nothing_to_drop_makes_no_call(self):
+ pool = self._pool()[:3]
+ with mock.patch.object(ap, "_chain") as chain:
+ cs.rank_clips_with_ai(pool, 2)
+ chain.assert_not_called()
+
+ def test_the_pass_is_metered_under_its_own_purpose(self):
+ seen = {}
+ original = ap.generate_json
+
+ def capture(prompt, **kwargs):
+ seen.update(kwargs)
+ seen["prompt"] = prompt
+ return original(prompt, **kwargs)
+
+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value="RETAINS BEST: hot takes"), \
+ mock.patch.object(ap, "generate_json", capture), \
+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \
+ mock.patch.object(
+ ai,
+ "_run_ai_command",
+ return_value=subprocess.CompletedProcess(
+ args=["claude"], returncode=0,
+ stdout=json.dumps({"ranked": [{"id": i} for i in range(4)]}), stderr="",
+ ),
+ ):
+ cs.rank_clips_with_ai(self._pool(), 2)
+
+ self.assertEqual(seen["purpose"], "rank_moments")
+ self.assertNotIn("stable_prefix", seen)
+ self.assertIn("RETAINS BEST: hot takes", seen["prompt"])
+
+
+class WorkspaceLearningsTests(unittest.TestCase):
+ def _prompt_sent_to_cli(self, learned):
+ sent = {}
+
+ def capture(**kwargs):
+ with open(kwargs["prompt_file"], encoding="utf-8") as fh:
+ sent["prompt"] = fh.read()
+ return subprocess.CompletedProcess(
+ args=["claude"], returncode=0,
+ stdout=json.dumps({"clips": [{
+ "title": "A moment",
+ "start_second": 10.0,
+ "end_second": 40.0,
+ "segments": [{"start": 10.0, "end": 40.0}],
+ "scores": {"standalone": 5, "hook": 5, "relevance": 4, "quotability": 4},
+ "quote": "q", "why": "w",
+ }]}),
+ stderr="",
+ )
+
+ segments = [
+ {"start": float(i * 5), "end": float(i * 5 + 5), "text": f"line {i}", "speaker": "A"}
+ for i in range(12)
+ ]
+ with mock.patch.object(cs.podcli_cloud, "prompt_block", return_value=learned), \
+ mock.patch.object(ap, "_chain", return_value=[("cli", "/tmp/claude", "claude")]), \
+ mock.patch.object(ai, "_run_ai_command", side_effect=capture):
+ cs.suggest_with_claude(segments=segments, top_n=1)
+ return sent.get("prompt", "")
+
+ def test_a_local_cli_receives_the_workspace_learnings(self):
+ prompt = self._prompt_sent_to_cli("WHAT WORKS ON THIS CHANNEL: founder stories.")
+ self.assertIn("WHAT WORKS ON THIS CHANNEL: founder stories.", prompt)
+
+ def test_the_free_path_prompt_is_untouched(self):
+ prompt = self._prompt_sent_to_cli("")
+ self.assertFalse(prompt.startswith("\n"))
+ self.assertTrue(prompt.startswith("You are a viral clip editor"))
diff --git a/tests/test_suggest_handler.py b/tests/test_suggest_handler.py
index a28d21c..d257491 100644
--- a/tests/test_suggest_handler.py
+++ b/tests/test_suggest_handler.py
@@ -112,6 +112,10 @@ class SuggestReactionTimesTests(unittest.TestCase):
captured, _ = self._run({"segments": SEGMENTS})
self.assertIsNone(captured["reaction_times"])
+ def test_a_pool_is_requested_even_with_no_audio_to_analyse(self):
+ captured, _ = self._run({"segments": SEGMENTS, "top_n": 3})
+ self.assertEqual(captured["top_n"], 6)
+
if __name__ == "__main__":
unittest.main()