Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
fix: clip quality, face tracking, split-screen, and thumbnail selection
Nika Siradze5906740f910b
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.pyindex 40e413a..26e2811 100644--- a/backend/services/clip_generator.py+++ b/backend/services/clip_generator.py@@ -30,6 +30,46 @@ _FILLER_WORDS = frozenset([])+def _snap_to_sentence_end(+ transcript_words: list[dict], start_second: float, end_second: float,+ max_extension: float = 3.0,+) -> float:+ """+ Snap clip end_second to the nearest sentence boundary (. ! ?).+ Searches forward up to max_extension seconds. If no boundary found,+ searches backward within the clip. Returns adjusted end_second.+ """+ SENTENCE_ENDINGS = ".!?"++ # Get the last word in the clip range+ clip_words = [w for w in transcript_words if w["end"] > start_second and w["start"] < end_second]+ if not clip_words:+ return end_second++ last_word_text = clip_words[-1].get("word", "").strip()++ # Already ends on a sentence boundary+ if last_word_text and last_word_text[-1] in SENTENCE_ENDINGS:+ return end_second++ # Search forward: find next sentence-ending word within max_extension+ for w in transcript_words:+ if w["start"] >= end_second and w["end"] <= end_second + max_extension:+ text = w.get("word", "").strip()+ if text and text[-1] in SENTENCE_ENDINGS:+ return w["end"]++ # Search backward: find last sentence-ending word within the clip+ for w in reversed(clip_words):+ text = w.get("word", "").strip()+ if text and text[-1] in SENTENCE_ENDINGS:+ # Only snap backward if we don't lose more than 30% of the clip+ if w["end"] > start_second + (end_second - start_second) * 0.7:+ return w["end"]++ return end_second++def _clean_transcript_words(words: list[dict]) -> list[dict]:"""Remove filler words from transcript captions.@@ -92,6 +132,11 @@ def generate_clip(if end_second <= start_second:raise ValueError("end_second must be greater than start_second")+ # Snap clip boundaries to sentence endings if transcript is available.+ # Extends up to 3s forward to find a sentence boundary (. ! ?).+ if transcript_words:+ end_second = _snap_to_sentence_end(transcript_words, start_second, end_second)+duration = end_second - start_secondif duration > 180:raise ValueError(f"Clip too long ({duration:.0f}s). Max 180 seconds for shorts.")diff --git a/backend/services/thumbnail_ai.py b/backend/services/thumbnail_ai.pyindex 87fd5af..7f1780f 100644--- a/backend/services/thumbnail_ai.py+++ b/backend/services/thumbnail_ai.py@@ -53,6 +53,65 @@ def _frame_sharpness(frame, face_roi=None) -> float:return cv2.Laplacian(gray, cv2.CV_64F).var()+def _face_expression_quality(frame, x1: int, y1: int, x2: int, y2: int) -> float:+ """+ Estimate facial expression quality (0 to 1).+ Penalizes: closed/squinting eyes, wide open mouth, motion blur.+ Higher = better thumbnail candidate.+ """+ import cv2+ import numpy as np++ region = frame[y1:y2, x1:x2]+ h, w = region.shape[:2]+ if h < 20 or w < 20:+ return 0.5 # Too small to analyze++ gray = cv2.cvtColor(region, cv2.COLOR_BGR2GRAY)++ # 1) Eye region check (top 25-45% of face)+ eye_y1 = int(h * 0.25)+ eye_y2 = int(h * 0.45)+ eye_region = gray[eye_y1:eye_y2, :]+ if eye_region.size > 0:+ eye_sharpness = cv2.Laplacian(eye_region, cv2.CV_64F).var()+ # Sharp eyes = open eyes (edges around iris/pupil)+ # Blurry eyes = blinking or closed+ if eye_sharpness < 10:+ return 0.3 # Eyes likely closed or blinking+ else:+ eye_sharpness = 50 # Can't measure, assume OK++ # 2) Mouth region check (bottom 30% of face)+ mouth_region = gray[int(h * 0.65):, :]+ if mouth_region.size > 0:+ mouth_var = cv2.Laplacian(mouth_region, cv2.CV_64F).var()+ # Very high variance in mouth = wide open (teeth, tongue visible)+ # Moderate variance = natural expression+ mouth_penalty = 0.0+ if mouth_var > 200:+ mouth_penalty = 0.3 # Wide open mouth+ elif mouth_var > 120:+ mouth_penalty = 0.15 # Slightly open+ else:+ mouth_penalty = 0.0++ # 3) Overall face sharpness — catch motion blur+ face_sharpness = cv2.Laplacian(gray, cv2.CV_64F).var()+ if face_sharpness < 20:+ return 0.35 # Motion blur++ # 4) Face aspect ratio — detect extreme angles or distortion+ aspect = h / max(1, w)+ if aspect < 0.6 or aspect > 1.5:+ return 0.5 # Odd angle++ # Combine scores+ eye_score = min(1.0, eye_sharpness / 80.0)+ quality = eye_score * (1.0 - mouth_penalty)+ return max(0.2, min(1.0, quality))++def extract_candidate_frames(video_path: str,output_dir: str,@@ -135,10 +194,19 @@ def extract_candidate_frames(# Sharpness of face region (proxy for eyes open, good expression)sharpness = _frame_sharpness(frame, (x1, y1, x2, y2))+ # Skip very blurry faces (motion blur, out of focus)+ if sharpness < 15:+ continue++ # Face expression quality — filter blinking, open mouths+ expression_quality = _face_expression_quality(frame, x1, y1, x2, y2)+ if expression_quality < 0.4:+ continue # Bad expression (blinking, mouth wide open, etc.)+# Face size as fraction of frameface_area_pct = (fw * fh) / (w * h) * 100- # Score: confidence × sharpness × size_preference+ # Score: confidence × sharpness × size_preference × expression# Prefer faces that are 5-25% of frame area (natural portrait)# Penalize extreme close-ups (>35%) and tiny faces (<3%)if face_area_pct > 35:@@ -150,7 +218,7 @@ def extract_candidate_frames(else:size_factor = 1.0 # Sweet spot- score = (conf ** 2) * (sharpness ** 0.5) * size_factor+ score = (conf ** 2) * (sharpness ** 0.5) * size_factor * expression_qualitycandidates.append({"frame": frame.copy(),@@ -474,7 +542,7 @@ def generate_variations(def thumbnail_to_video_frame(thumbnail_path: str,output_path: str,- duration: float = 2.0,+ duration: float = 0.8,width: int = 1080,height: int = 1920,) -> str:diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 913d5e2..ecae0f8 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -404,12 +404,24 @@ def _build_speaker_aware_crop(pan_duration = 0.4 # 400ms smooth pan (was 300ms, felt jerky)# Build keyframe list: (time, crop_x)+ # In split-screen, verify target face is visible before panning.+ # If face isn't detected at that time, stay at current position.keyframes = []prev_x = default_xfor start_t, end_t, speaker in segments:cl = speaker_to_cluster.get(speaker)target_x = cl["crop_x"] if cl else default_x+ # Verify target face is visible at this time (avoid panning to empty space)+ if is_split_screen and cl:+ # Check if any face detection near the target cluster exists at segment start+ face_near_target = any(+ abs(obs_t - start_t) < 1.5 and abs(obs_cx - cl["center_x"]) < cluster_radius+ for obs_t, obs_cx, obs_fw in face_observations+ )+ if not face_near_target:+ target_x = prev_x # Stay at current position+if target_x != prev_x:# Ease into the new position over pan_durationkeyframes.append((start_t, prev_x))@@ -921,12 +933,12 @@ def _detect_face_offset(# 2) Fill gaps: if face wasn't detected for some frames, the tracked# array has gaps. That's fine — we interpolate between known points.- # 3) Heavy smoothing to prevent jitter. Use a rolling average with a- # window of ~1.5 seconds. This means small head bobs don't move the- # crop, but sustained movement (leaning, shifting) does.+ # 3) Heavy smoothing to prevent jitter. Use a 2.5-second rolling average+ # so only sustained movement (leaning, shifting) affects the crop.+ # Small head bobs and natural sway are averaged out.if len(tracked_crop_x) > 5:sample_interval = tracked_times[-1] / len(tracked_times) if len(tracked_times) > 1 else 0.25- window = max(3, int(1.5 / max(0.05, sample_interval)))+ window = max(5, int(2.5 / max(0.05, sample_interval)))# Pad edges so smoothing doesn't pull toward zeropadded = np.pad(tracked_crop_x, (window // 2, window // 2), mode="edge")kernel = np.ones(window) / window@@ -938,17 +950,17 @@ def _detect_face_offset(smoothed = np.clip(np.round(smoothed), 0, width - crop_w).astype(int)# --- Check if tracking is even needed ---- # If the face barely moves (range < 3% of frame width), just use static+ # If the face barely moves (range < 5% of frame width), just use staticmovement_range = int(smoothed.max() - smoothed.min())- if movement_range < width * 0.03:+ if movement_range < width * 0.05:return str(int(np.median(smoothed)))# --- Build keyframes: only emit when position changes significantly ---- # Downsample smoothed track to keyframes at ~1s intervals,- # or whenever the crop shifts by more than 20px.+ # Use conservative thresholds: only pan when the speaker truly shifts+ # position, not for natural head movement.keyframes = [(tracked_times[0], int(smoothed[0]))]- min_time_gap = 0.8 # Don't place keyframes closer than 0.8s- min_px_change = 20 # Ignore movement smaller than 20px+ min_time_gap = 2.0 # Don't place keyframes closer than 2s+ min_px_change = 50 # Ignore movement smaller than 50px (~5% of frame)for i in range(1, len(smoothed)):t = tracked_times[i]