Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
fix: clip pipeline quality, face detection, thumbnails, and MCP routing
Nika Siradze1f8542406b3d
diff --git a/backend/config/caption_styles.py b/backend/config/caption_styles.pyindex 1d04348..6b7ae8a 100644--- a/backend/config/caption_styles.py+++ b/backend/config/caption_styles.py@@ -124,8 +124,8 @@ STYLES = {"outline_color": "&H00000000", # Black"back_color": "&H00000000", # Transparent"bold": True,- "outline_width": 0, # No outline — clean text- "shadow_depth": 0, # No shadow+ "outline_width": 0, # No box border (BackColour used for pill)+ "shadow_depth": 1, # Subtle shadow for text edge smoothing"alignment": 2, # Bottom-center"margin_v": 500, # Position: ~72% from top (natural lower-third)"words_per_chunk": 6, # Show 5-7 words across 2 linesdiff --git a/backend/main.py b/backend/main.pyindex 4af5c9c..34311a2 100644--- a/backend/main.py+++ b/backend/main.py@@ -109,7 +109,13 @@ def handle_batch_clips(task_id: str, params: dict):task_id, "batch", int((_i / total) * 100 + pct / total), msg),)- results.append({"clip_index": i, "status": "success", **result})+ results.append({+ "clip_index": i,+ "status": "success",+ "start_second": clip["start_second"],+ "end_second": clip["end_second"],+ **result,+ })except Exception as e:results.append({"clip_index": i, "status": "error", "error": str(e)})diff --git a/backend/services/caption_renderer.py b/backend/services/caption_renderer.pyindex 349f81e..3d194bb 100644--- a/backend/services/caption_renderer.py+++ b/backend/services/caption_renderer.py@@ -51,7 +51,7 @@ ScaledBorderAndShadow: yes[V4+ Styles]Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding-Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H00000000,&H00000000,-1,0,0,0,100,100,2,0,3,0,0,{style["alignment"]},60,60,{style["margin_v"]},1+Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,-1,0,0,0,100,100,2,0,3,0,{style["shadow_depth"]},{style["alignment"]},60,60,{style["margin_v"]},1[Events]Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Textdiff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.pyindex ffdb72f..40e413a 100644--- a/backend/services/clip_generator.py+++ b/backend/services/clip_generator.py@@ -225,6 +225,10 @@ def generate_clip("duration": round(duration, 2),"file_size_mb": file_size_mb,"title": title,+ "start_second": start_second,+ "end_second": end_second,+ "caption_style": caption_style,+ "crop_strategy": crop_strategy,}finally:diff --git a/backend/services/encoder.py b/backend/services/encoder.pyindex 5dddc5f..570b158 100644--- a/backend/services/encoder.py+++ b/backend/services/encoder.py@@ -105,7 +105,7 @@ def _get_encoder_flags(encoder: str) -> list[str]:flags = {"h264_videotoolbox": ["-c:v", "h264_videotoolbox",- "-b:v", "20M", # High bitrate for crisp 1080x1920+ "-b:v", "6M", # 6 Mbps — plenty for 1080x1920 vertical"-profile:v", "high","-allow_sw", "1", # Allow software fallback],diff --git a/backend/services/thumbnail_generator.py b/backend/services/thumbnail_generator.pyindex f99c115..24330e5 100644--- a/backend/services/thumbnail_generator.py+++ b/backend/services/thumbnail_generator.py@@ -161,19 +161,36 @@ def _get_font(size: int, bold: bool = True, italic: bool = False) -> ImageFont.Freturn ImageFont.load_default()-def _wrap_text(text: str, max_chars: int) -> list[str]:- """Wrap text into lines of max_chars, breaking at word boundaries."""+def _wrap_text(text: str, max_chars: int, font=None, max_width: int = 0) -> list[str]:+ """Wrap text into lines, using pixel width if font provided, otherwise char count."""words = text.split()lines = []current = ""- for word in words:- test = f"{current} {word}".strip()- if len(test) <= max_chars:- current = test- else:- if current:- lines.append(current)- current = word++ if font and max_width > 0:+ # Pixel-accurate wrapping using actual font metrics+ from PIL import ImageDraw, Image+ _measure = ImageDraw.Draw(Image.new("RGBA", (1, 1)))+ for word in words:+ test = f"{current} {word}".strip()+ bbox = _measure.textbbox((0, 0), test, font=font)+ if (bbox[2] - bbox[0]) <= max_width:+ current = test+ else:+ if current:+ lines.append(current)+ current = word+ else:+ # Fallback: char-count wrapping+ for word in words:+ test = f"{current} {word}".strip()+ if len(test) <= max_chars:+ current = test+ else:+ if current:+ lines.append(current)+ current = word+if current:lines.append(current)return lines@@ -451,13 +468,39 @@ def generate_thumbnail(draw = ImageDraw.Draw(img)# ── Layer 4: Text box + text ──+ # Safe text area: 80% of image width with margins+ safe_margin = int(w * 0.08)+ max_text_w = w - safe_margin * 2+font1 = _get_font(cfg["font_size_line1"], bold=cfg.get("line1_bold", True))font2 = _get_font(cfg["font_size_line2"], bold=cfg.get("line2_bold", True), italic=cfg.get("line2_italic", True))l1_text = title_line1.upper() if cfg.get("line1_uppercase", True) else title_line1l2_text = title_line2.upper() if cfg.get("line2_uppercase", True) else title_line2- l1_lines = _wrap_text(l1_text, cfg["max_chars_per_line"])- l2_lines = _wrap_text(l2_text, cfg["max_chars_per_line"])++ # Pixel-aware wrapping with auto-shrink if text still overflows+ def _wrap_and_fit(text, font, font_size, bold=True, italic=False):+ """Wrap text using pixel width, shrink font if any single word is too wide."""+ current_font = font+ current_size = font_size+ for _ in range(5): # max 5 shrink attempts+ lines = _wrap_text(text, cfg["max_chars_per_line"], font=current_font, max_width=max_text_w)+ # Check if any line overflows+ overflow = False+ for line in lines:+ bbox = draw.textbbox((0, 0), line, font=current_font)+ if (bbox[2] - bbox[0]) > max_text_w:+ overflow = True+ break+ if not overflow:+ return lines, current_font+ # Shrink font by 15%+ current_size = max(28, int(current_size * 0.85))+ current_font = _get_font(current_size, bold=bold, italic=italic)+ return lines, current_font++ l1_lines, font1 = _wrap_and_fit(l1_text, font1, cfg["font_size_line1"], bold=cfg.get("line1_bold", True))+ l2_lines, font2 = _wrap_and_fit(l2_text, font2, cfg["font_size_line2"], bold=cfg.get("line2_bold", True), italic=cfg.get("line2_italic", True))line2_style = cfg.get("line2_style", "highlight")hl_pad_x, hl_pad_y = 16, 8@@ -481,9 +524,9 @@ def generate_thumbnail(l2_block_h = sum(s[1] + hl_pad_y * 2 for s in l2_sizes) if line2_style == "highlight" else sum(s[1] for s in l2_sizes)total_content_h = l1_block_h + block_gap + l2_block_h- # Max width for box sizing+ # Max width for box sizing — clamp to image boundsall_widths = [s[0] for s in l1_sizes] + [s[0] + hl_pad_x * 2 for s in l2_sizes]- max_content_w = max(all_widths) if all_widths else 400+ max_content_w = min(max(all_widths) if all_widths else 400, max_text_w)# Vertical position + variation shifttext_y_ratio = cfg.get("text_y", 0.72)@@ -492,16 +535,18 @@ def generate_thumbnail(elif variation == 2:text_y_ratio += 0.03content_top = int(h * text_y_ratio) - total_content_h // 2+ # Clamp: keep text within image bounds+ content_top = max(10, min(content_top, h - total_content_h - 10))# Draw text boxbox_enabled = cfg.get("box_enabled", True)if box_enabled:pad_x = cfg.get("box_padding_x", 50)pad_y = cfg.get("box_padding_y", 35)- box_l = (w - max_content_w) // 2 - pad_x- box_t = content_top - pad_y- box_r = (w + max_content_w) // 2 + pad_x- box_b = content_top + total_content_h + pad_y+ box_l = max(0, (w - max_content_w) // 2 - pad_x)+ box_t = max(0, content_top - pad_y)+ box_r = min(w, (w + max_content_w) // 2 + pad_x)+ box_b = min(h, content_top + total_content_h + pad_y)fill_rgba = _hex_to_rgba(cfg.get("box_fill_color", "#0D0D0DE0"))border_w = cfg.get("box_border_width", 4)diff --git a/backend/services/thumbnail_html.py b/backend/services/thumbnail_html.pyindex 991ac7b..b7e6665 100644--- a/backend/services/thumbnail_html.py+++ b/backend/services/thumbnail_html.py@@ -230,13 +230,17 @@ def _build_html(def _fit_size(text, base_size, base_spacing):"""Shrink font size so text fits on one line."""- # Inter uppercase: average char width ~0.62 × font_size + letter-spacing- char_w = 0.62 * base_size + base_spacing+ # Character width varies by font. Use conservative estimate:+ # uppercase chars ~0.68× font_size, lowercase ~0.52×, average ~0.60×+ # Add safety margin of 5% to prevent edge-case overflow+ is_upper = text == text.upper()+ char_ratio = 0.68 if is_upper else 0.58+ char_w = char_ratio * base_size + base_spacingest_width = len(text) * char_w- if est_width <= usable_w:+ if est_width <= usable_w * 0.95: # 5% safety marginreturn base_size, base_spacing- scale = usable_w / est_width- new_size = max(36, int(base_size * scale))+ scale = (usable_w * 0.95) / est_width+ new_size = max(32, int(base_size * scale))new_spacing = max(0, int(base_spacing * scale))return new_size, new_spacing@@ -340,6 +344,9 @@ body {{line-height: {l1_lh};margin-bottom: {l1_mb};white-space: {l1_nowrap};+ overflow: hidden;+ text-overflow: ellipsis;+ max-width: 100%;}}.line2 {{diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 3f801f0..1eed0ea 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -101,7 +101,7 @@ def cut_segment("-ss", str(start_second),"-i", input_path,"-t", str(duration),- "-c:v", "libx264", "-crf", "12", "-preset", "fast",+ "-c:v", "libx264", "-crf", "18", "-preset", "fast", "-profile:v", "high","-c:a", "aac", "-b:a", "192k","-avoid_negative_ts", "make_zero",output_path,@@ -281,7 +281,7 @@ def _build_speaker_aware_crop(# Cluster faces by position (left vs right)positions = np.array([obs[1] for obs in face_observations])- cluster_radius = width * 0.15+ cluster_radius = width * 0.20 # 20% of frame width — groups face positions with natural head movementclusters = []used = np.zeros(len(positions), dtype=bool)@@ -322,14 +322,19 @@ def _build_speaker_aware_crop(speaker_to_cluster = {}if is_split_screen and len(clusters) >= 2 and len(speakers) == 2:- # Split-screen: both faces always visible.- # Standard podcast layout: host on left, guest on right (or vice versa).- # Map by speaker order — typically speaker labels from diarization- # are consistent: SPEAKER_00 talks first (host intro), SPEAKER_01 is guest.- # Host is usually on the LEFT in Riverside/Zencastr/etc.- # So: first speaker (host) → left cluster, second speaker (guest) → right cluster.- speaker_to_cluster[speakers[0]] = clusters[0] # left- speaker_to_cluster[speakers[1]] = clusters[1] # right+ # Split-screen: both faces are always visible, so we can't use+ # audio-visual correlation (both clusters get equal votes).+ # Use positional heuristic: whoever speaks first in this clip+ # is mapped to the LEFT face (standard podcast layout: host left,+ # host does intro). Sort speakers by first word time, not label.+ speaker_first_word = {}+ for w in transcript_words:+ sp = w.get("speaker")+ if sp and sp not in speaker_first_word:+ speaker_first_word[sp] = w["start"]+ speakers_by_first_word = sorted(speakers, key=lambda s: speaker_first_word.get(s, float("inf")))+ speaker_to_cluster[speakers_by_first_word[0]] = clusters[0] # left+ speaker_to_cluster[speakers_by_first_word[1]] = clusters[1] # rightelse:# Non-split-screen: only one face visible at a time.# Map by which face is visible when each speaker talks.@@ -371,12 +376,15 @@ def _build_speaker_aware_crop(clusters.sort(key=lambda c: c["count"], reverse=True)return str(clusters[0]["crop_x"])- # Merge very short segments (<0.5s) into neighbors to avoid jitter+ # Merge very short segments (<1.0s) into neighbors to avoid jitter.+ # 0.5s was too aggressive — fast back-and-forth crosstalk caused rapid panning.merged = []for seg in segments:if merged and seg[2] == merged[-1][2]:+ # Same speaker: extend previous segmentmerged[-1] = (merged[-1][0], seg[1], seg[2])- elif merged and (seg[1] - seg[0]) < 0.5:+ elif merged and (seg[1] - seg[0]) < 1.0:+ # Too short to pan — absorb into previous segmentmerged[-1] = (merged[-1][0], seg[1], merged[-1][2])else:merged.append(list(seg))@@ -390,40 +398,63 @@ def _build_speaker_aware_crop(default_cluster = speaker_to_cluster.get(dominant_speaker) or clusters[0]default_x = default_cluster["crop_x"]- # Build FFmpeg expression with smooth panning.- # Use linear interpolation over 0.3s transition windows.- pan_duration = 0.3- expr_parts = []+ # Build a timeline: list of (time, target_x) keyframes with smooth transitions.+ # Instead of nested if/between (breaks with many segments), use a linear+ # interpolation chain: lerp between keyframes.+ pan_duration = 0.4 # 400ms smooth pan (was 300ms, felt jerky)++ # Build keyframe list: (time, crop_x)+ keyframes = []+ prev_x = default_xfor start_t, end_t, speaker in segments:cl = speaker_to_cluster.get(speaker)- if not cl or cl["crop_x"] == default_x:- continue- target_x = cl["crop_x"]- # Smooth transition: ramp from default to target over pan_duration- ramp_end = start_t + pan_duration- # if(between(t, start, end), lerp_or_static, ...)- # For smooth pan: use linear interp at segment boundaries+ target_x = cl["crop_x"] if cl else default_x++ if target_x != prev_x:+ # Ease into the new position over pan_duration+ keyframes.append((start_t, prev_x))+ keyframes.append((start_t + pan_duration, target_x))+ prev_x = target_x++ if not keyframes:+ return str(default_x)++ # Build FFmpeg expression using chained lerp:+ # For each keyframe pair, if t is in [t0, t1], lerp from x0 to x1+ # Otherwise fall through to next pair or default+ expr_parts = []+ for i in range(0, len(keyframes) - 1, 2):+ t0, x0 = keyframes[i]+ t1, x1 = keyframes[i + 1]+ # During transition: linear interpolation+ expr_parts.append(+ f"if(between(t\\,{t0:.2f}\\,{t1:.2f})\\,"+ f"{x0}+(({x1}-{x0})*(t-{t0:.2f})/{max(0.01, t1 - t0):.2f})\\,"+ )+ # After transition until next one: hold at target+ if i + 2 < len(keyframes):+ next_t = keyframes[i + 2][0]+ else:+ next_t = durationexpr_parts.append(- f"if(between(t\\,{start_t:.2f}\\,{end_t:.2f})\\,"- f"if(lt(t\\,{ramp_end:.2f})\\,"- f"{default_x}+(({target_x}-{default_x})*(t-{start_t:.2f})/{pan_duration:.2f})\\,"- f"{target_x})\\,"+ f"if(between(t\\,{t1:.2f}\\,{next_t:.2f})\\,"+ f"{x1}\\,")- if not expr_parts:+ # Cap nesting depth to avoid FFmpeg expression parser limits+ if len(expr_parts) > 30:+ print(f"Warning: {len(expr_parts)} pan segments, simplifying to avoid FFmpeg limits", file=sys.stderr)+ # Fallback: just use the dominant speaker positionreturn str(default_x)- expr = ""- for part in expr_parts:- expr += part- expr += str(default_x)- expr += ")" * (len(expr_parts) * 2) # Each part has 2 nested ifs+ expr = "".join(expr_parts) + str(default_x) + ")" * len(expr_parts)return exprexcept ImportError:return None- except Exception:+ except Exception as e:+ print(f"Warning: speaker-aware crop failed: {e}", file=sys.stderr)return None@@ -724,7 +755,7 @@ def normalize_audio("-af", af_filter,"-c:v", "copy","-c:a", "aac",- "-b:a", "128k",+ "-b:a", "192k","-ar", "44100","-movflags", "+faststart",output_path,@@ -858,7 +889,7 @@ def _detect_face_offset(# When two speakers are visible, positions will form two groups.# Pick the largest cluster (most frequently detected face).crop_w = int(height * target_ratio)- cluster_radius = crop_w * 0.25 # positions within 25% of crop width = same face+ cluster_radius = crop_w * 0.35 # positions within 35% of crop width = same face (allows for head movement)positions = np.array(face_positions)@@ -901,5 +932,6 @@ def _detect_face_offset(except ImportError:# OpenCV not installed, fall back to centerreturn None- except Exception:+ except Exception as e:+ print(f"Warning: face detection failed: {e}", file=sys.stderr)return Nonediff --git a/src/server.ts b/src/server.tsindex 3322bba..bc949ba 100644--- a/src/server.ts+++ b/src/server.ts@@ -298,6 +298,34 @@ export function createServer(): McpServer {if (resolved) params.logo_path = resolved;}+ // Resolve clip_number from UI state BEFORE routing+ if (params.clip_number != null && (params.start_second == null || params.end_second == null)) {+ const uiState = await readUIState();+ const suggestions = (uiState?.suggestions as Array<Record<string, unknown>>) || [];+ const settings = (uiState?.settings as Record<string, string>) || {};+ const idx = (params.clip_number as number) - 1;+ if (idx < 0 || idx >= suggestions.length) {+ return {+ content: [{+ type: "text" as const,+ text: `Clip #${params.clip_number} not found. Available: 1-${suggestions.length}`,+ }],+ };+ }+ const suggestion = suggestions[idx];+ if (params.start_second == null) params.start_second = suggestion.start_second as number;+ if (params.end_second == null) params.end_second = suggestion.end_second as number;+ if (!params.video_path) params.video_path = (uiState?.videoPath || uiState?.filePath || "") as string;+ if (!params.title || params.title === "clip") params.title = (suggestion.title as string) || "clip";+ if (!params.caption_style || params.caption_style === "hormozi") {+ params.caption_style = ((suggestion.suggested_caption_style as string) || settings.captionStyle || "hormozi") as typeof params.caption_style;+ }+ if (!params.transcript_words) {+ const transcript = uiState?.transcript as Record<string, unknown> | null;+ if (transcript?.words) params.transcript_words = transcript.words as any;+ }+ }+// Check for duplicatesconst dup = await history.findDuplicate(params.video_path as string,@@ -458,6 +486,47 @@ export function createServer(): McpServer {},async (params) => {try {+ // Resolve clip_numbers/export_selected from UI state BEFORE routing+ let resolvedClips = params.clips;+ let resolvedVideoPath = params.video_path;+ let resolvedTranscriptWords = params.transcript_words;+ if (!resolvedClips && (params.export_selected || params.clip_numbers)) {+ const uiState = await readUIState();+ const suggestions = (uiState?.suggestions as Array<Record<string, unknown>>) || [];+ const settings = (uiState?.settings as Record<string, string>) || {};+ const deselected = (uiState?.deselectedIndices as number[]) || [];+ if (!resolvedVideoPath) resolvedVideoPath = (uiState?.videoPath || uiState?.filePath || "") as string;+ if (!resolvedTranscriptWords) {+ const transcript = uiState?.transcript as Record<string, unknown> | null;+ if (transcript?.words) resolvedTranscriptWords = transcript.words as any;+ }++ if (params.export_selected) {+ resolvedClips = suggestions+ .filter((_: unknown, i: number) => !deselected.includes(i))+ .map((s: Record<string, unknown>, i: number) => ({+ start_second: s.start_second,+ end_second: s.end_second,+ title: s.title || `clip_${i + 1}`,+ caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",+ crop_strategy: settings.cropStrategy || "face",+ })) as any;+ } else if (params.clip_numbers) {+ resolvedClips = (params.clip_numbers as number[])+ .filter((n) => n >= 1 && n <= suggestions.length)+ .map((n) => {+ const s = suggestions[n - 1];+ return {+ start_second: s.start_second,+ end_second: s.end_second,+ title: s.title || `clip_${n}`,+ caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",+ crop_strategy: settings.cropStrategy || "face",+ };+ }) as any;+ }+ }+// Try routing through web server for real-time UI progress// The web server handles SSE broadcasts (export-started, job-update, job-complete)let usedWebServer = false;@@ -467,9 +536,9 @@ export function createServer(): McpServer {method: "POST",headers: { "Content-Type": "application/json" },body: JSON.stringify({- video_path: params.video_path,- clips: params.clips,- transcript_words: params.transcript_words,+ video_path: resolvedVideoPath,+ clips: resolvedClips,+ transcript_words: resolvedTranscriptWords,}),});if (webRes.ok) {diff --git a/src/ui/web-server.ts b/src/ui/web-server.tsindex 50e54d7..5efe116 100644--- a/src/ui/web-server.ts+++ b/src/ui/web-server.ts@@ -568,6 +568,11 @@ app.post("/api/batch-clips", async (req, res) => {};jobs.set(jobId, job);+ broadcastSSE("export-started", { jobId, clipCount: clips.length });+ uiState.phase = "exporting";+ uiState.lastUpdated = Date.now();+ persistState();+res.json({ job_id: jobId, status: "running" });executor@@ -577,6 +582,7 @@ app.post("/api/batch-clips", async (req, res) => {(event) => {job.progress = event.percent;job.message = event.message;+ broadcastSSE("job-update", { jobId, progress: event.percent, message: event.message });}).then(async (result) => {