Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

fix: clip quality, face tracking, split-screen, and thumbnail selection

Nika Siradze5906740f910b
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index 40e413a..26e2811 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -30,6 +30,46 @@ _FILLER_WORDS = frozenset([
])
+def _snap_to_sentence_end(
+ transcript_words: list[dict], start_second: float, end_second: float,
+ max_extension: float = 3.0,
+) -> float:
+ """
+ Snap clip end_second to the nearest sentence boundary (. ! ?).
+ Searches forward up to max_extension seconds. If no boundary found,
+ searches backward within the clip. Returns adjusted end_second.
+ """
+ SENTENCE_ENDINGS = ".!?"
+
+ # Get the last word in the clip range
+ clip_words = [w for w in transcript_words if w["end"] > start_second and w["start"] < end_second]
+ if not clip_words:
+ return end_second
+
+ last_word_text = clip_words[-1].get("word", "").strip()
+
+ # Already ends on a sentence boundary
+ if last_word_text and last_word_text[-1] in SENTENCE_ENDINGS:
+ return end_second
+
+ # Search forward: find next sentence-ending word within max_extension
+ for w in transcript_words:
+ if w["start"] >= end_second and w["end"] <= end_second + max_extension:
+ text = w.get("word", "").strip()
+ if text and text[-1] in SENTENCE_ENDINGS:
+ return w["end"]
+
+ # Search backward: find last sentence-ending word within the clip
+ for w in reversed(clip_words):
+ text = w.get("word", "").strip()
+ if text and text[-1] in SENTENCE_ENDINGS:
+ # Only snap backward if we don't lose more than 30% of the clip
+ if w["end"] > start_second + (end_second - start_second) * 0.7:
+ return w["end"]
+
+ return end_second
+
+
def _clean_transcript_words(words: list[dict]) -> list[dict]:
"""
Remove filler words from transcript captions.
@@ -92,6 +132,11 @@ def generate_clip(
if end_second <= start_second:
raise ValueError("end_second must be greater than start_second")
+ # Snap clip boundaries to sentence endings if transcript is available.
+ # Extends up to 3s forward to find a sentence boundary (. ! ?).
+ if transcript_words:
+ end_second = _snap_to_sentence_end(transcript_words, start_second, end_second)
+
duration = end_second - start_second
if duration > 180:
raise ValueError(f"Clip too long ({duration:.0f}s). Max 180 seconds for shorts.")
diff --git a/backend/services/thumbnail_ai.py b/backend/services/thumbnail_ai.py
index 87fd5af..7f1780f 100644
--- a/backend/services/thumbnail_ai.py
+++ b/backend/services/thumbnail_ai.py
@@ -53,6 +53,65 @@ def _frame_sharpness(frame, face_roi=None) -> float:
return cv2.Laplacian(gray, cv2.CV_64F).var()
+def _face_expression_quality(frame, x1: int, y1: int, x2: int, y2: int) -> float:
+ """
+ Estimate facial expression quality (0 to 1).
+ Penalizes: closed/squinting eyes, wide open mouth, motion blur.
+ Higher = better thumbnail candidate.
+ """
+ import cv2
+ import numpy as np
+
+ region = frame[y1:y2, x1:x2]
+ h, w = region.shape[:2]
+ if h < 20 or w < 20:
+ return 0.5 # Too small to analyze
+
+ gray = cv2.cvtColor(region, cv2.COLOR_BGR2GRAY)
+
+ # 1) Eye region check (top 25-45% of face)
+ eye_y1 = int(h * 0.25)
+ eye_y2 = int(h * 0.45)
+ eye_region = gray[eye_y1:eye_y2, :]
+ if eye_region.size > 0:
+ eye_sharpness = cv2.Laplacian(eye_region, cv2.CV_64F).var()
+ # Sharp eyes = open eyes (edges around iris/pupil)
+ # Blurry eyes = blinking or closed
+ if eye_sharpness < 10:
+ return 0.3 # Eyes likely closed or blinking
+ else:
+ eye_sharpness = 50 # Can't measure, assume OK
+
+ # 2) Mouth region check (bottom 30% of face)
+ mouth_region = gray[int(h * 0.65):, :]
+ if mouth_region.size > 0:
+ mouth_var = cv2.Laplacian(mouth_region, cv2.CV_64F).var()
+ # Very high variance in mouth = wide open (teeth, tongue visible)
+ # Moderate variance = natural expression
+ mouth_penalty = 0.0
+ if mouth_var > 200:
+ mouth_penalty = 0.3 # Wide open mouth
+ elif mouth_var > 120:
+ mouth_penalty = 0.15 # Slightly open
+ else:
+ mouth_penalty = 0.0
+
+ # 3) Overall face sharpness — catch motion blur
+ face_sharpness = cv2.Laplacian(gray, cv2.CV_64F).var()
+ if face_sharpness < 20:
+ return 0.35 # Motion blur
+
+ # 4) Face aspect ratio — detect extreme angles or distortion
+ aspect = h / max(1, w)
+ if aspect < 0.6 or aspect > 1.5:
+ return 0.5 # Odd angle
+
+ # Combine scores
+ eye_score = min(1.0, eye_sharpness / 80.0)
+ quality = eye_score * (1.0 - mouth_penalty)
+ return max(0.2, min(1.0, quality))
+
+
def extract_candidate_frames(
video_path: str,
output_dir: str,
@@ -135,10 +194,19 @@ def extract_candidate_frames(
# Sharpness of face region (proxy for eyes open, good expression)
sharpness = _frame_sharpness(frame, (x1, y1, x2, y2))
+ # Skip very blurry faces (motion blur, out of focus)
+ if sharpness < 15:
+ continue
+
+ # Face expression quality — filter blinking, open mouths
+ expression_quality = _face_expression_quality(frame, x1, y1, x2, y2)
+ if expression_quality < 0.4:
+ continue # Bad expression (blinking, mouth wide open, etc.)
+
# Face size as fraction of frame
face_area_pct = (fw * fh) / (w * h) * 100
- # Score: confidence × sharpness × size_preference
+ # Score: confidence × sharpness × size_preference × expression
# Prefer faces that are 5-25% of frame area (natural portrait)
# Penalize extreme close-ups (>35%) and tiny faces (<3%)
if face_area_pct > 35:
@@ -150,7 +218,7 @@ def extract_candidate_frames(
else:
size_factor = 1.0 # Sweet spot
- score = (conf ** 2) * (sharpness ** 0.5) * size_factor
+ score = (conf ** 2) * (sharpness ** 0.5) * size_factor * expression_quality
candidates.append({
"frame": frame.copy(),
@@ -474,7 +542,7 @@ def generate_variations(
def thumbnail_to_video_frame(
thumbnail_path: str,
output_path: str,
- duration: float = 2.0,
+ duration: float = 0.8,
width: int = 1080,
height: int = 1920,
) -> str:
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 913d5e2..ecae0f8 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -404,12 +404,24 @@ def _build_speaker_aware_crop(
pan_duration = 0.4 # 400ms smooth pan (was 300ms, felt jerky)
# Build keyframe list: (time, crop_x)
+ # In split-screen, verify target face is visible before panning.
+ # If face isn't detected at that time, stay at current position.
keyframes = []
prev_x = default_x
for start_t, end_t, speaker in segments:
cl = speaker_to_cluster.get(speaker)
target_x = cl["crop_x"] if cl else default_x
+ # Verify target face is visible at this time (avoid panning to empty space)
+ if is_split_screen and cl:
+ # Check if any face detection near the target cluster exists at segment start
+ face_near_target = any(
+ abs(obs_t - start_t) < 1.5 and abs(obs_cx - cl["center_x"]) < cluster_radius
+ for obs_t, obs_cx, obs_fw in face_observations
+ )
+ if not face_near_target:
+ target_x = prev_x # Stay at current position
+
if target_x != prev_x:
# Ease into the new position over pan_duration
keyframes.append((start_t, prev_x))
@@ -921,12 +933,12 @@ def _detect_face_offset(
# 2) Fill gaps: if face wasn't detected for some frames, the tracked
# array has gaps. That's fine — we interpolate between known points.
- # 3) Heavy smoothing to prevent jitter. Use a rolling average with a
- # window of ~1.5 seconds. This means small head bobs don't move the
- # crop, but sustained movement (leaning, shifting) does.
+ # 3) Heavy smoothing to prevent jitter. Use a 2.5-second rolling average
+ # so only sustained movement (leaning, shifting) affects the crop.
+ # Small head bobs and natural sway are averaged out.
if len(tracked_crop_x) > 5:
sample_interval = tracked_times[-1] / len(tracked_times) if len(tracked_times) > 1 else 0.25
- window = max(3, int(1.5 / max(0.05, sample_interval)))
+ window = max(5, int(2.5 / max(0.05, sample_interval)))
# Pad edges so smoothing doesn't pull toward zero
padded = np.pad(tracked_crop_x, (window // 2, window // 2), mode="edge")
kernel = np.ones(window) / window
@@ -938,17 +950,17 @@ def _detect_face_offset(
smoothed = np.clip(np.round(smoothed), 0, width - crop_w).astype(int)
# --- Check if tracking is even needed ---
- # If the face barely moves (range < 3% of frame width), just use static
+ # If the face barely moves (range < 5% of frame width), just use static
movement_range = int(smoothed.max() - smoothed.min())
- if movement_range < width * 0.03:
+ if movement_range < width * 0.05:
return str(int(np.median(smoothed)))
# --- Build keyframes: only emit when position changes significantly ---
- # Downsample smoothed track to keyframes at ~1s intervals,
- # or whenever the crop shifts by more than 20px.
+ # Use conservative thresholds: only pan when the speaker truly shifts
+ # position, not for natural head movement.
keyframes = [(tracked_times[0], int(smoothed[0]))]
- min_time_gap = 0.8 # Don't place keyframes closer than 0.8s
- min_px_change = 20 # Ignore movement smaller than 20px
+ min_time_gap = 2.0 # Don't place keyframes closer than 2s
+ min_px_change = 50 # Ignore movement smaller than 50px (~5% of frame)
for i in range(1, len(smoothed)):
t = tracked_times[i]