Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

fix: clip pipeline quality, face detection, thumbnails, and MCP routing

Nika Siradze1f8542406b3d
diff --git a/backend/config/caption_styles.py b/backend/config/caption_styles.py
index 1d04348..6b7ae8a 100644
--- a/backend/config/caption_styles.py
+++ b/backend/config/caption_styles.py
@@ -124,8 +124,8 @@ STYLES = {
"outline_color": "&H00000000", # Black
"back_color": "&H00000000", # Transparent
"bold": True,
- "outline_width": 0, # No outline — clean text
- "shadow_depth": 0, # No shadow
+ "outline_width": 0, # No box border (BackColour used for pill)
+ "shadow_depth": 1, # Subtle shadow for text edge smoothing
"alignment": 2, # Bottom-center
"margin_v": 500, # Position: ~72% from top (natural lower-third)
"words_per_chunk": 6, # Show 5-7 words across 2 lines
diff --git a/backend/main.py b/backend/main.py
index 4af5c9c..34311a2 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -109,7 +109,13 @@ def handle_batch_clips(task_id: str, params: dict):
task_id, "batch", int((_i / total) * 100 + pct / total), msg
),
)
- results.append({"clip_index": i, "status": "success", **result})
+ results.append({
+ "clip_index": i,
+ "status": "success",
+ "start_second": clip["start_second"],
+ "end_second": clip["end_second"],
+ **result,
+ })
except Exception as e:
results.append({"clip_index": i, "status": "error", "error": str(e)})
diff --git a/backend/services/caption_renderer.py b/backend/services/caption_renderer.py
index 349f81e..3d194bb 100644
--- a/backend/services/caption_renderer.py
+++ b/backend/services/caption_renderer.py
@@ -51,7 +51,7 @@ ScaledBorderAndShadow: yes
[V4+ Styles]
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
-Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H00000000,&H00000000,-1,0,0,0,100,100,2,0,3,0,0,{style["alignment"]},60,60,{style["margin_v"]},1
+Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,-1,0,0,0,100,100,2,0,3,0,{style["shadow_depth"]},{style["alignment"]},60,60,{style["margin_v"]},1
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index ffdb72f..40e413a 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -225,6 +225,10 @@ def generate_clip(
"duration": round(duration, 2),
"file_size_mb": file_size_mb,
"title": title,
+ "start_second": start_second,
+ "end_second": end_second,
+ "caption_style": caption_style,
+ "crop_strategy": crop_strategy,
}
finally:
diff --git a/backend/services/encoder.py b/backend/services/encoder.py
index 5dddc5f..570b158 100644
--- a/backend/services/encoder.py
+++ b/backend/services/encoder.py
@@ -105,7 +105,7 @@ def _get_encoder_flags(encoder: str) -> list[str]:
flags = {
"h264_videotoolbox": [
"-c:v", "h264_videotoolbox",
- "-b:v", "20M", # High bitrate for crisp 1080x1920
+ "-b:v", "6M", # 6 Mbps — plenty for 1080x1920 vertical
"-profile:v", "high",
"-allow_sw", "1", # Allow software fallback
],
diff --git a/backend/services/thumbnail_generator.py b/backend/services/thumbnail_generator.py
index f99c115..24330e5 100644
--- a/backend/services/thumbnail_generator.py
+++ b/backend/services/thumbnail_generator.py
@@ -161,19 +161,36 @@ def _get_font(size: int, bold: bool = True, italic: bool = False) -> ImageFont.F
return ImageFont.load_default()
-def _wrap_text(text: str, max_chars: int) -> list[str]:
- """Wrap text into lines of max_chars, breaking at word boundaries."""
+def _wrap_text(text: str, max_chars: int, font=None, max_width: int = 0) -> list[str]:
+ """Wrap text into lines, using pixel width if font provided, otherwise char count."""
words = text.split()
lines = []
current = ""
- for word in words:
- test = f"{current} {word}".strip()
- if len(test) <= max_chars:
- current = test
- else:
- if current:
- lines.append(current)
- current = word
+
+ if font and max_width > 0:
+ # Pixel-accurate wrapping using actual font metrics
+ from PIL import ImageDraw, Image
+ _measure = ImageDraw.Draw(Image.new("RGBA", (1, 1)))
+ for word in words:
+ test = f"{current} {word}".strip()
+ bbox = _measure.textbbox((0, 0), test, font=font)
+ if (bbox[2] - bbox[0]) <= max_width:
+ current = test
+ else:
+ if current:
+ lines.append(current)
+ current = word
+ else:
+ # Fallback: char-count wrapping
+ for word in words:
+ test = f"{current} {word}".strip()
+ if len(test) <= max_chars:
+ current = test
+ else:
+ if current:
+ lines.append(current)
+ current = word
+
if current:
lines.append(current)
return lines
@@ -451,13 +468,39 @@ def generate_thumbnail(
draw = ImageDraw.Draw(img)
# ── Layer 4: Text box + text ──
+ # Safe text area: 80% of image width with margins
+ safe_margin = int(w * 0.08)
+ max_text_w = w - safe_margin * 2
+
font1 = _get_font(cfg["font_size_line1"], bold=cfg.get("line1_bold", True))
font2 = _get_font(cfg["font_size_line2"], bold=cfg.get("line2_bold", True), italic=cfg.get("line2_italic", True))
l1_text = title_line1.upper() if cfg.get("line1_uppercase", True) else title_line1
l2_text = title_line2.upper() if cfg.get("line2_uppercase", True) else title_line2
- l1_lines = _wrap_text(l1_text, cfg["max_chars_per_line"])
- l2_lines = _wrap_text(l2_text, cfg["max_chars_per_line"])
+
+ # Pixel-aware wrapping with auto-shrink if text still overflows
+ def _wrap_and_fit(text, font, font_size, bold=True, italic=False):
+ """Wrap text using pixel width, shrink font if any single word is too wide."""
+ current_font = font
+ current_size = font_size
+ for _ in range(5): # max 5 shrink attempts
+ lines = _wrap_text(text, cfg["max_chars_per_line"], font=current_font, max_width=max_text_w)
+ # Check if any line overflows
+ overflow = False
+ for line in lines:
+ bbox = draw.textbbox((0, 0), line, font=current_font)
+ if (bbox[2] - bbox[0]) > max_text_w:
+ overflow = True
+ break
+ if not overflow:
+ return lines, current_font
+ # Shrink font by 15%
+ current_size = max(28, int(current_size * 0.85))
+ current_font = _get_font(current_size, bold=bold, italic=italic)
+ return lines, current_font
+
+ l1_lines, font1 = _wrap_and_fit(l1_text, font1, cfg["font_size_line1"], bold=cfg.get("line1_bold", True))
+ l2_lines, font2 = _wrap_and_fit(l2_text, font2, cfg["font_size_line2"], bold=cfg.get("line2_bold", True), italic=cfg.get("line2_italic", True))
line2_style = cfg.get("line2_style", "highlight")
hl_pad_x, hl_pad_y = 16, 8
@@ -481,9 +524,9 @@ def generate_thumbnail(
l2_block_h = sum(s[1] + hl_pad_y * 2 for s in l2_sizes) if line2_style == "highlight" else sum(s[1] for s in l2_sizes)
total_content_h = l1_block_h + block_gap + l2_block_h
- # Max width for box sizing
+ # Max width for box sizing — clamp to image bounds
all_widths = [s[0] for s in l1_sizes] + [s[0] + hl_pad_x * 2 for s in l2_sizes]
- max_content_w = max(all_widths) if all_widths else 400
+ max_content_w = min(max(all_widths) if all_widths else 400, max_text_w)
# Vertical position + variation shift
text_y_ratio = cfg.get("text_y", 0.72)
@@ -492,16 +535,18 @@ def generate_thumbnail(
elif variation == 2:
text_y_ratio += 0.03
content_top = int(h * text_y_ratio) - total_content_h // 2
+ # Clamp: keep text within image bounds
+ content_top = max(10, min(content_top, h - total_content_h - 10))
# Draw text box
box_enabled = cfg.get("box_enabled", True)
if box_enabled:
pad_x = cfg.get("box_padding_x", 50)
pad_y = cfg.get("box_padding_y", 35)
- box_l = (w - max_content_w) // 2 - pad_x
- box_t = content_top - pad_y
- box_r = (w + max_content_w) // 2 + pad_x
- box_b = content_top + total_content_h + pad_y
+ box_l = max(0, (w - max_content_w) // 2 - pad_x)
+ box_t = max(0, content_top - pad_y)
+ box_r = min(w, (w + max_content_w) // 2 + pad_x)
+ box_b = min(h, content_top + total_content_h + pad_y)
fill_rgba = _hex_to_rgba(cfg.get("box_fill_color", "#0D0D0DE0"))
border_w = cfg.get("box_border_width", 4)
diff --git a/backend/services/thumbnail_html.py b/backend/services/thumbnail_html.py
index 991ac7b..b7e6665 100644
--- a/backend/services/thumbnail_html.py
+++ b/backend/services/thumbnail_html.py
@@ -230,13 +230,17 @@ def _build_html(
def _fit_size(text, base_size, base_spacing):
"""Shrink font size so text fits on one line."""
- # Inter uppercase: average char width ~0.62 × font_size + letter-spacing
- char_w = 0.62 * base_size + base_spacing
+ # Character width varies by font. Use conservative estimate:
+ # uppercase chars ~0.68× font_size, lowercase ~0.52×, average ~0.60×
+ # Add safety margin of 5% to prevent edge-case overflow
+ is_upper = text == text.upper()
+ char_ratio = 0.68 if is_upper else 0.58
+ char_w = char_ratio * base_size + base_spacing
est_width = len(text) * char_w
- if est_width <= usable_w:
+ if est_width <= usable_w * 0.95: # 5% safety margin
return base_size, base_spacing
- scale = usable_w / est_width
- new_size = max(36, int(base_size * scale))
+ scale = (usable_w * 0.95) / est_width
+ new_size = max(32, int(base_size * scale))
new_spacing = max(0, int(base_spacing * scale))
return new_size, new_spacing
@@ -340,6 +344,9 @@ body {{
line-height: {l1_lh};
margin-bottom: {l1_mb};
white-space: {l1_nowrap};
+ overflow: hidden;
+ text-overflow: ellipsis;
+ max-width: 100%;
}}
.line2 {{
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 3f801f0..1eed0ea 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -101,7 +101,7 @@ def cut_segment(
"-ss", str(start_second),
"-i", input_path,
"-t", str(duration),
- "-c:v", "libx264", "-crf", "12", "-preset", "fast",
+ "-c:v", "libx264", "-crf", "18", "-preset", "fast", "-profile:v", "high",
"-c:a", "aac", "-b:a", "192k",
"-avoid_negative_ts", "make_zero",
output_path,
@@ -281,7 +281,7 @@ def _build_speaker_aware_crop(
# Cluster faces by position (left vs right)
positions = np.array([obs[1] for obs in face_observations])
- cluster_radius = width * 0.15
+ cluster_radius = width * 0.20 # 20% of frame width — groups face positions with natural head movement
clusters = []
used = np.zeros(len(positions), dtype=bool)
@@ -322,14 +322,19 @@ def _build_speaker_aware_crop(
speaker_to_cluster = {}
if is_split_screen and len(clusters) >= 2 and len(speakers) == 2:
- # Split-screen: both faces always visible.
- # Standard podcast layout: host on left, guest on right (or vice versa).
- # Map by speaker order — typically speaker labels from diarization
- # are consistent: SPEAKER_00 talks first (host intro), SPEAKER_01 is guest.
- # Host is usually on the LEFT in Riverside/Zencastr/etc.
- # So: first speaker (host) → left cluster, second speaker (guest) → right cluster.
- speaker_to_cluster[speakers[0]] = clusters[0] # left
- speaker_to_cluster[speakers[1]] = clusters[1] # right
+ # Split-screen: both faces are always visible, so we can't use
+ # audio-visual correlation (both clusters get equal votes).
+ # Use positional heuristic: whoever speaks first in this clip
+ # is mapped to the LEFT face (standard podcast layout: host left,
+ # host does intro). Sort speakers by first word time, not label.
+ speaker_first_word = {}
+ for w in transcript_words:
+ sp = w.get("speaker")
+ if sp and sp not in speaker_first_word:
+ speaker_first_word[sp] = w["start"]
+ speakers_by_first_word = sorted(speakers, key=lambda s: speaker_first_word.get(s, float("inf")))
+ speaker_to_cluster[speakers_by_first_word[0]] = clusters[0] # left
+ speaker_to_cluster[speakers_by_first_word[1]] = clusters[1] # right
else:
# Non-split-screen: only one face visible at a time.
# Map by which face is visible when each speaker talks.
@@ -371,12 +376,15 @@ def _build_speaker_aware_crop(
clusters.sort(key=lambda c: c["count"], reverse=True)
return str(clusters[0]["crop_x"])
- # Merge very short segments (<0.5s) into neighbors to avoid jitter
+ # Merge very short segments (<1.0s) into neighbors to avoid jitter.
+ # 0.5s was too aggressive — fast back-and-forth crosstalk caused rapid panning.
merged = []
for seg in segments:
if merged and seg[2] == merged[-1][2]:
+ # Same speaker: extend previous segment
merged[-1] = (merged[-1][0], seg[1], seg[2])
- elif merged and (seg[1] - seg[0]) < 0.5:
+ elif merged and (seg[1] - seg[0]) < 1.0:
+ # Too short to pan — absorb into previous segment
merged[-1] = (merged[-1][0], seg[1], merged[-1][2])
else:
merged.append(list(seg))
@@ -390,40 +398,63 @@ def _build_speaker_aware_crop(
default_cluster = speaker_to_cluster.get(dominant_speaker) or clusters[0]
default_x = default_cluster["crop_x"]
- # Build FFmpeg expression with smooth panning.
- # Use linear interpolation over 0.3s transition windows.
- pan_duration = 0.3
- expr_parts = []
+ # Build a timeline: list of (time, target_x) keyframes with smooth transitions.
+ # Instead of nested if/between (breaks with many segments), use a linear
+ # interpolation chain: lerp between keyframes.
+ pan_duration = 0.4 # 400ms smooth pan (was 300ms, felt jerky)
+
+ # Build keyframe list: (time, crop_x)
+ keyframes = []
+ prev_x = default_x
for start_t, end_t, speaker in segments:
cl = speaker_to_cluster.get(speaker)
- if not cl or cl["crop_x"] == default_x:
- continue
- target_x = cl["crop_x"]
- # Smooth transition: ramp from default to target over pan_duration
- ramp_end = start_t + pan_duration
- # if(between(t, start, end), lerp_or_static, ...)
- # For smooth pan: use linear interp at segment boundaries
+ target_x = cl["crop_x"] if cl else default_x
+
+ if target_x != prev_x:
+ # Ease into the new position over pan_duration
+ keyframes.append((start_t, prev_x))
+ keyframes.append((start_t + pan_duration, target_x))
+ prev_x = target_x
+
+ if not keyframes:
+ return str(default_x)
+
+ # Build FFmpeg expression using chained lerp:
+ # For each keyframe pair, if t is in [t0, t1], lerp from x0 to x1
+ # Otherwise fall through to next pair or default
+ expr_parts = []
+ for i in range(0, len(keyframes) - 1, 2):
+ t0, x0 = keyframes[i]
+ t1, x1 = keyframes[i + 1]
+ # During transition: linear interpolation
+ expr_parts.append(
+ f"if(between(t\\,{t0:.2f}\\,{t1:.2f})\\,"
+ f"{x0}+(({x1}-{x0})*(t-{t0:.2f})/{max(0.01, t1 - t0):.2f})\\,"
+ )
+ # After transition until next one: hold at target
+ if i + 2 < len(keyframes):
+ next_t = keyframes[i + 2][0]
+ else:
+ next_t = duration
expr_parts.append(
- f"if(between(t\\,{start_t:.2f}\\,{end_t:.2f})\\,"
- f"if(lt(t\\,{ramp_end:.2f})\\,"
- f"{default_x}+(({target_x}-{default_x})*(t-{start_t:.2f})/{pan_duration:.2f})\\,"
- f"{target_x})\\,"
+ f"if(between(t\\,{t1:.2f}\\,{next_t:.2f})\\,"
+ f"{x1}\\,"
)
- if not expr_parts:
+ # Cap nesting depth to avoid FFmpeg expression parser limits
+ if len(expr_parts) > 30:
+ print(f"Warning: {len(expr_parts)} pan segments, simplifying to avoid FFmpeg limits", file=sys.stderr)
+ # Fallback: just use the dominant speaker position
return str(default_x)
- expr = ""
- for part in expr_parts:
- expr += part
- expr += str(default_x)
- expr += ")" * (len(expr_parts) * 2) # Each part has 2 nested ifs
+ expr = "".join(expr_parts) + str(default_x) + ")" * len(expr_parts)
return expr
except ImportError:
return None
- except Exception:
+ except Exception as e:
+ print(f"Warning: speaker-aware crop failed: {e}", file=sys.stderr)
return None
@@ -724,7 +755,7 @@ def normalize_audio(
"-af", af_filter,
"-c:v", "copy",
"-c:a", "aac",
- "-b:a", "128k",
+ "-b:a", "192k",
"-ar", "44100",
"-movflags", "+faststart",
output_path,
@@ -858,7 +889,7 @@ def _detect_face_offset(
# When two speakers are visible, positions will form two groups.
# Pick the largest cluster (most frequently detected face).
crop_w = int(height * target_ratio)
- cluster_radius = crop_w * 0.25 # positions within 25% of crop width = same face
+ cluster_radius = crop_w * 0.35 # positions within 35% of crop width = same face (allows for head movement)
positions = np.array(face_positions)
@@ -901,5 +932,6 @@ def _detect_face_offset(
except ImportError:
# OpenCV not installed, fall back to center
return None
- except Exception:
+ except Exception as e:
+ print(f"Warning: face detection failed: {e}", file=sys.stderr)
return None
diff --git a/src/server.ts b/src/server.ts
index 3322bba..bc949ba 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -298,6 +298,34 @@ export function createServer(): McpServer {
if (resolved) params.logo_path = resolved;
}
+ // Resolve clip_number from UI state BEFORE routing
+ if (params.clip_number != null && (params.start_second == null || params.end_second == null)) {
+ const uiState = await readUIState();
+ const suggestions = (uiState?.suggestions as Array<Record<string, unknown>>) || [];
+ const settings = (uiState?.settings as Record<string, string>) || {};
+ const idx = (params.clip_number as number) - 1;
+ if (idx < 0 || idx >= suggestions.length) {
+ return {
+ content: [{
+ type: "text" as const,
+ text: `Clip #${params.clip_number} not found. Available: 1-${suggestions.length}`,
+ }],
+ };
+ }
+ const suggestion = suggestions[idx];
+ if (params.start_second == null) params.start_second = suggestion.start_second as number;
+ if (params.end_second == null) params.end_second = suggestion.end_second as number;
+ if (!params.video_path) params.video_path = (uiState?.videoPath || uiState?.filePath || "") as string;
+ if (!params.title || params.title === "clip") params.title = (suggestion.title as string) || "clip";
+ if (!params.caption_style || params.caption_style === "hormozi") {
+ params.caption_style = ((suggestion.suggested_caption_style as string) || settings.captionStyle || "hormozi") as typeof params.caption_style;
+ }
+ if (!params.transcript_words) {
+ const transcript = uiState?.transcript as Record<string, unknown> | null;
+ if (transcript?.words) params.transcript_words = transcript.words as any;
+ }
+ }
+
// Check for duplicates
const dup = await history.findDuplicate(
params.video_path as string,
@@ -458,6 +486,47 @@ export function createServer(): McpServer {
},
async (params) => {
try {
+ // Resolve clip_numbers/export_selected from UI state BEFORE routing
+ let resolvedClips = params.clips;
+ let resolvedVideoPath = params.video_path;
+ let resolvedTranscriptWords = params.transcript_words;
+ if (!resolvedClips && (params.export_selected || params.clip_numbers)) {
+ const uiState = await readUIState();
+ const suggestions = (uiState?.suggestions as Array<Record<string, unknown>>) || [];
+ const settings = (uiState?.settings as Record<string, string>) || {};
+ const deselected = (uiState?.deselectedIndices as number[]) || [];
+ if (!resolvedVideoPath) resolvedVideoPath = (uiState?.videoPath || uiState?.filePath || "") as string;
+ if (!resolvedTranscriptWords) {
+ const transcript = uiState?.transcript as Record<string, unknown> | null;
+ if (transcript?.words) resolvedTranscriptWords = transcript.words as any;
+ }
+
+ if (params.export_selected) {
+ resolvedClips = suggestions
+ .filter((_: unknown, i: number) => !deselected.includes(i))
+ .map((s: Record<string, unknown>, i: number) => ({
+ start_second: s.start_second,
+ end_second: s.end_second,
+ title: s.title || `clip_${i + 1}`,
+ caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",
+ crop_strategy: settings.cropStrategy || "face",
+ })) as any;
+ } else if (params.clip_numbers) {
+ resolvedClips = (params.clip_numbers as number[])
+ .filter((n) => n >= 1 && n <= suggestions.length)
+ .map((n) => {
+ const s = suggestions[n - 1];
+ return {
+ start_second: s.start_second,
+ end_second: s.end_second,
+ title: s.title || `clip_${n}`,
+ caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",
+ crop_strategy: settings.cropStrategy || "face",
+ };
+ }) as any;
+ }
+ }
+
// Try routing through web server for real-time UI progress
// The web server handles SSE broadcasts (export-started, job-update, job-complete)
let usedWebServer = false;
@@ -467,9 +536,9 @@ export function createServer(): McpServer {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
- video_path: params.video_path,
- clips: params.clips,
- transcript_words: params.transcript_words,
+ video_path: resolvedVideoPath,
+ clips: resolvedClips,
+ transcript_words: resolvedTranscriptWords,
}),
});
if (webRes.ok) {
diff --git a/src/ui/web-server.ts b/src/ui/web-server.ts
index 50e54d7..5efe116 100644
--- a/src/ui/web-server.ts
+++ b/src/ui/web-server.ts
@@ -568,6 +568,11 @@ app.post("/api/batch-clips", async (req, res) => {
};
jobs.set(jobId, job);
+ broadcastSSE("export-started", { jobId, clipCount: clips.length });
+ uiState.phase = "exporting";
+ uiState.lastUpdated = Date.now();
+ persistState();
+
res.json({ job_id: jobId, status: "running" });
executor
@@ -577,6 +582,7 @@ app.post("/api/batch-clips", async (req, res) => {
(event) => {
job.progress = event.percent;
job.message = event.message;
+ broadcastSSE("job-update", { jobId, progress: event.percent, message: event.message });
}
)
.then(async (result) => {