Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

feat: per-clip review loop, smooth 2s camera pan, custom feedback, dynamic captions

Nika Siradze6afe35c12a05
diff --git a/backend/cli.py b/backend/cli.py
index 29a63e1..e6510a9 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -448,6 +448,7 @@ def cmd_process(args):
print(f"\n [4/4] Exporting {len(clips)} clips{_ai_label} to {output_dir}/")
results = []
t0 = time.time()
+ _skip_review = False
for i, clip in enumerate(clips):
ok = False
@@ -540,6 +541,150 @@ def cmd_process(args):
except Exception as e:
print(f" ⚠ content: {e}")
+ # ── Per-clip review: open video, ask for feedback ──
+ if ok and not _skip_review and result.get("output_path") and os.path.exists(result["output_path"]):
+ import subprocess as _review_sp
+ _review_sp.Popen(["open", result["output_path"]] if sys.platform == "darwin" else ["xdg-open", result["output_path"]])
+
+ while True:
+ import questionary as _rq
+ from questionary import Style as _RS
+ _rstyle = _RS([
+ ("qmark", "fg:#d4874a bold"), ("question", "bold"),
+ ("answer", "fg:#4ade80"), ("pointer", "fg:#d4874a bold"),
+ ("highlighted", "fg:#d4874a bold"), ("selected", "fg:#4ade80"),
+ ])
+ _raction = _rq.select(
+ f"Clip {i+1}/{len(clips)}: {clip['title'][:40]}",
+ choices=[
+ _rq.Choice("Looks good — next clip", value="next"),
+ _rq.Choice("Change caption style", value="style"),
+ _rq.Choice("Make shorter (trim 5s from end)", value="shorter"),
+ _rq.Choice("Make longer (extend 5s)", value="longer"),
+ _rq.Choice("Start earlier (5s)", value="earlier"),
+ _rq.Choice("Start later (3s)", value="later"),
+ _rq.Choice("Tell me what to change", value="custom"),
+ _rq.Choice("Render the rest without asking", value="skip_review"),
+ ],
+ style=_rstyle,
+ instruction="",
+ ).ask()
+
+ if _raction is None or _raction == "next":
+ break
+
+ if _raction == "skip_review":
+ _skip_review = True
+ break
+
+ if _raction == "custom":
+ _feedback = _rq.text(
+ "What should I change?",
+ style=_rstyle,
+ ).ask()
+ if _feedback and _feedback.strip():
+ fb = _feedback.strip().lower()
+ # Parse common requests
+ if any(w in fb for w in ["shorter", "trim", "cut"]):
+ secs = 5
+ for word in fb.split():
+ try:
+ secs = int(word)
+ break
+ except ValueError:
+ pass
+ clip["end_second"] -= secs
+ clip["duration"] = max(10, clip["duration"] - secs)
+ print(f" Trimming {secs}s from end")
+ elif any(w in fb for w in ["longer", "extend", "more"]):
+ secs = 5
+ for word in fb.split():
+ try:
+ secs = int(word)
+ break
+ except ValueError:
+ pass
+ clip["end_second"] += secs
+ clip["duration"] += secs
+ print(f" Extending {secs}s")
+ elif any(w in fb for w in ["earlier", "before", "back"]):
+ secs = 5
+ for word in fb.split():
+ try:
+ secs = int(word)
+ break
+ except ValueError:
+ pass
+ clip["start_second"] = max(0, clip["start_second"] - secs)
+ print(f" Starting {secs}s earlier")
+ elif any(w in fb for w in ["later", "forward", "skip"]):
+ secs = 3
+ for word in fb.split():
+ try:
+ secs = int(word)
+ break
+ except ValueError:
+ pass
+ clip["start_second"] += secs
+ print(f" Starting {secs}s later")
+ elif any(w in fb for w in ["hormozi", "karaoke", "subtle", "branded"]):
+ for s in ["hormozi", "karaoke", "subtle", "branded"]:
+ if s in fb:
+ config["caption_style"] = s
+ print(f" Changing to {s} style")
+ break
+ else:
+ print(f" Couldn't parse that. Try: 'shorter', 'longer 10', 'start 3s earlier', 'hormozi style'")
+ continue
+ else:
+ continue
+
+ # Apply change
+ if _raction == "style":
+ _new_style = _rq.select("Style:", choices=[
+ _rq.Choice("branded", value="branded"),
+ _rq.Choice("hormozi", value="hormozi"),
+ _rq.Choice("karaoke", value="karaoke"),
+ _rq.Choice("subtle", value="subtle"),
+ ], style=_rstyle).ask()
+ if _new_style:
+ config["caption_style"] = _new_style
+ elif _raction == "shorter":
+ clip["end_second"] -= 5
+ clip["duration"] = max(10, clip["duration"] - 5)
+ elif _raction == "longer":
+ clip["end_second"] += 5
+ clip["duration"] += 5
+ elif _raction == "earlier":
+ clip["start_second"] = max(0, clip["start_second"] - 5)
+ elif _raction == "later":
+ clip["start_second"] += 3
+
+ # Re-render
+ with _Spinner(f"Re-rendering clip {i+1}..."):
+ try:
+ result = generate_clip(
+ video_path=video_path,
+ start_second=clip["start_second"],
+ end_second=clip["end_second"],
+ caption_style=config.get("caption_style", "branded"),
+ crop_strategy=config.get("crop_strategy", "face"),
+ transcript_words=words,
+ title=clip.get("title", f"clip_{i+1}"),
+ output_dir=output_dir,
+ logo_path=config.get("logo_path") or None,
+ outro_path=config.get("outro_path") or None,
+ keep_segments=clip.get("segments"),
+ face_map=face_map,
+ )
+ results[-1] = result
+ print(f" ✓ Re-rendered: {result['file_size_mb']}MB")
+ # Open new version
+ _review_sp.Popen(["open", result["output_path"]] if sys.platform == "darwin" else ["xdg-open", result["output_path"]])
+ except Exception as _re:
+ print(f" ✗ {_re}")
+ break
+
elapsed = time.time() - t0
success = sum(1 for r in results if "output_path" in r)
print(f"\n {success}/{len(clips)} clips exported in {elapsed:.1f}s")
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index cdaeb02..a242faf 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -220,10 +220,53 @@ def _render_with_remotion(
"end": round(w["end"] - time_offset, 3),
})
+ # Extract face Y positions so captions can avoid overlapping faces
+ # Probe video dimensions
+ face_y_norm = None # normalized 0-1 (0=top, 1=bottom)
+ try:
+ probe_cmd = ["ffprobe", "-v", "error", "-select_streams", "v:0",
+ "-show_entries", "stream=height", "-of", "csv=p=0",
+ os.path.abspath(video_path)]
+ vid_h = int(subprocess.run(probe_cmd, capture_output=True, text=True, timeout=5).stdout.strip())
+
+ # Get face center Y from the cropped video's face detection
+ # Use a quick sample at the middle of the clip
+ try:
+ import cv2
+ sys.path.insert(0, os.path.join(project_root, "backend"))
+ from services.face_detector import create_detector, detect_faces
+ cap = cv2.VideoCapture(os.path.abspath(video_path))
+ if cap.isOpened():
+ vid_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ vid_h_actual = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ dur = cap.get(cv2.CAP_PROP_FRAME_COUNT) / (cap.get(cv2.CAP_PROP_FPS) or 30)
+ detector = create_detector(vid_w, vid_h_actual)
+ if detector:
+ # Sample 5 frames
+ ys = []
+ for t in [dur * f for f in [0.1, 0.3, 0.5, 0.7, 0.9]]:
+ cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)
+ ret, frame = cap.read()
+ if ret:
+ faces = detect_faces(detector, frame, vid_w, vid_h_actual)
+ for f in faces:
+ ys.append(f["cy"] / vid_h_actual)
+ if ys:
+ face_y_norm = sum(ys) / len(ys)
+ cap.release()
+ except Exception:
+ pass
+ except Exception:
+ pass
+
words_file = output_path + ".words.json"
try:
+ payload = {
+ "words": adjusted_words,
+ "faceY": face_y_norm, # normalized face center Y (0-1), null if unknown
+ }
with open(words_file, "w") as f:
- json.dump(adjusted_words, f)
+ json.dump(payload, f)
cmd = [
node_path, render_script,
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index bf144f6..45813cd 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -591,8 +591,10 @@ def _track_and_crop(
)
if force_snap or abs(diff) > snap_distance:
- # Teleport — speaker switch or very large jump
- cam_x = target_x
+ # Smooth 2-second pan to the other speaker instead of instant snap
+ # Rate 1.5 with dt~0.1 gives alpha~0.14 per step → reaches 95% in ~2s
+ pan_alpha = 1.0 - math.exp(-1.5 * dt)
+ cam_x += diff * pan_alpha
if force_snap:
last_snap_t = t
elif abs(diff) > dead_zone:
diff --git a/remotion/render.mjs b/remotion/render.mjs
index 148bce1..a51cd18 100644
--- a/remotion/render.mjs
+++ b/remotion/render.mjs
@@ -116,7 +116,10 @@ async function main() {
process.exit(1);
}
- const words = JSON.parse(fs.readFileSync(opts.words, "utf-8"));
+ const wordsData = JSON.parse(fs.readFileSync(opts.words, "utf-8"));
+ // Support both old format (array) and new format ({words, faceY})
+ const words = Array.isArray(wordsData) ? wordsData : wordsData.words || [];
+ const faceY = Array.isArray(wordsData) ? null : wordsData.faceY ?? null;
const styleName = opts.style || "branded";
const fps = parseInt(opts.fps || "30", 10);
@@ -185,6 +188,7 @@ async function main() {
words,
styleName,
logoSrc,
+ faceY,
durationInFrames,
fps,
};
diff --git a/remotion/src/CaptionedClip.tsx b/remotion/src/CaptionedClip.tsx
index 04eefbb..72050b4 100644
--- a/remotion/src/CaptionedClip.tsx
+++ b/remotion/src/CaptionedClip.tsx
@@ -1,5 +1,5 @@
import React from "react";
-import { AbsoluteFill, OffthreadVideo, staticFile } from "remotion";
+import { AbsoluteFill } from "remotion";
import { HormoziCaptions } from "./components/HormoziCaptions";
import { KaraokeCaptions } from "./components/KaraokeCaptions";
import { SubtleCaptions } from "./components/SubtleCaptions";
@@ -11,13 +11,14 @@ export interface CaptionedClipProps {
words: Word[];
style: CaptionStyle;
logoSrc?: string;
+ faceY?: number | null;
}
export const CaptionedClip: React.FC<CaptionedClipProps> = ({
- videoSrc,
words,
style,
logoSrc,
+ faceY,
}) => {
const CaptionComponent = {
hormozi: HormoziCaptions,
@@ -26,12 +27,10 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({
branded: BrandedCaptions,
}[style.name];
- // Render captions only (transparent bg) — FFmpeg composites onto video later
- // This is 10x faster than decoding video through Chrome
return (
<AbsoluteFill style={{ backgroundColor: "transparent" }}>
{style.name === "branded" ? (
- <BrandedCaptions words={words} style={style} logoSrc={logoSrc} />
+ <BrandedCaptions words={words} style={style} logoSrc={logoSrc} faceY={faceY} />
) : (
<CaptionComponent words={words} style={style} />
)}
diff --git a/remotion/src/Root.tsx b/remotion/src/Root.tsx
index 7101208..5ed5c53 100644
--- a/remotion/src/Root.tsx
+++ b/remotion/src/Root.tsx
@@ -11,6 +11,7 @@ const inputProps = getInputProps() as {
words?: Word[];
styleName?: string;
logoSrc?: string;
+ faceY?: number | null;
durationInFrames?: number;
fps?: number;
};
@@ -33,6 +34,7 @@ export const RemotionRoot: React.FC = () => {
words: inputProps.words || [],
style: STYLES[inputProps.styleName || "branded"],
logoSrc: inputProps.logoSrc,
+ faceY: inputProps.faceY ?? null,
}}
/>
</>
diff --git a/remotion/src/components/BrandedCaptions.tsx b/remotion/src/components/BrandedCaptions.tsx
index ab3af42..f217814 100644
--- a/remotion/src/components/BrandedCaptions.tsx
+++ b/remotion/src/components/BrandedCaptions.tsx
@@ -12,6 +12,7 @@ interface Props {
words: Word[];
style: CaptionStyle;
logoSrc?: string;
+ faceY?: number | null; // normalized 0-1 (0=top, 1=bottom)
}
interface Chunk {
@@ -139,9 +140,10 @@ export const BrandedCaptions: React.FC<Props> = ({
words,
style,
logoSrc,
+ faceY,
}) => {
const frame = useCurrentFrame();
- const { fps } = useVideoConfig();
+ const { fps, height } = useVideoConfig();
const currentTime = frame / fps;
const chunks = buildChunks(words, style.wordsPerChunk);
@@ -149,6 +151,18 @@ export const BrandedCaptions: React.FC<Props> = ({
(c) => currentTime >= c.start && currentTime < c.end
);
+ // Dynamic margin: if face is in the lower portion, push captions further down
+ // faceY is normalized 0-1 (0=top, 1=bottom)
+ // Default margin is style.marginBottom. If face center is below 0.55, reduce margin.
+ let dynamicMargin = style.marginBottom;
+ if (faceY != null && faceY > 0.55) {
+ // Face is low — push captions to the very bottom
+ dynamicMargin = Math.max(80, style.marginBottom - Math.round((faceY - 0.55) * height * 0.6));
+ } else if (faceY != null && faceY < 0.35) {
+ // Face is high — can bring captions up a bit
+ dynamicMargin = style.marginBottom + 60;
+ }
+
return (
<>
{logoSrc && (
@@ -172,7 +186,7 @@ export const BrandedCaptions: React.FC<Props> = ({
<div
style={{
position: "absolute",
- bottom: style.marginBottom,
+ bottom: dynamicMargin,
left: 60,
right: 60,
display: "flex",
diff --git a/remotion/src/types.ts b/remotion/src/types.ts
index cc09e5d..4f092c9 100644
--- a/remotion/src/types.ts
+++ b/remotion/src/types.ts
@@ -25,6 +25,7 @@ export interface CaptionProps {
durationInFrames: number;
videoSrc: string;
logoSrc?: string;
+ faceY?: number | null; // normalized 0-1 (0=top, 1=bottom)
}
const FONT = "'DM Sans', sans-serif";