Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Let a caller hand in the face map, and stop a lost crop looking deliberate (#220)
Nika Siradzefe0bf9986b67
diff --git a/backend/cli.py b/backend/cli.pyindex 0764f47..98fec7c 100644--- a/backend/cli.py+++ b/backend/cli.py@@ -84,6 +84,27 @@ def _parse_json_transcript(raw_text):return data.get("words", []), data.get("segments", []), data+def _json_file_arg(raw: str | None, name: str):+ """JSON given inline or named as a file, or nothing.++ Both, because the two callers differ: keyframes are a handful of numbers+ and have always been passed inline, while a face map is a per-second record+ of every face in the video, which an argument list will not carry. Missing+ or malformed loses the framing hint, never the run.+ """+ if not raw:+ return None+ text = raw.strip()+ try:+ if text.startswith("{") or text.startswith("["):+ return json.loads(text)+ with open(text, "r", encoding="utf-8") as handle:+ return json.load(handle)+ except (OSError, ValueError) as exc:+ print(f" Warning: {name} could not be read ({exc}); ignoring it", file=sys.stderr)+ return None++def _cached_face_map(video_path: str):"""Face maps are keyed by video content, not by transcript, so an importedtranscript can still borrow the map from an earlier run on the same file."""@@ -470,6 +491,22 @@ def cmd_studio(args):"--caption-scale", str(getattr(args, "caption_scale", 1.0)),"--crop", args.crop,"--format", getattr(args, "format", None) or "vertical",+ ]+ # Where the faces are, and where somebody put the frame by hand. Both are+ # what the crop needs and neither could be handed to it before, so a caller+ # that already knew — the cloud worker scans the window before it renders —+ # had no way to say so and watched `speaker` fall through to a letterbox.+ # A path is made absolute because the script runs from elsewhere; JSON+ # given inline is handed straight through.+ def _pass_through(value):+ text = str(value).strip()+ return text if text.startswith(("{", "[")) else os.path.abspath(text)++ if getattr(args, "face_map", None):+ cmd += ["--face-map", _pass_through(args.face_map)]+ if getattr(args, "crop_keyframes", None):+ cmd += ["--crop-keyframes", _pass_through(args.crop_keyframes)]+ cmd += ["--logo-position", getattr(args, "logo_position", "top-left"),"--logo-scale", str(getattr(args, "logo_scale", 1.0)),"--intro-seconds", str(args.intro_seconds),@@ -979,6 +1016,45 @@ def cmd_process(args):# Extract face_map before result gets overwritten in clip loopface_map = result.get("face_map")+ # A map handed in on the command line wins over one the transcriber found,+ # because the caller who bothered to pass it scanned this video rather than+ # inferring the layout from who was speaking. It is also the only map there+ # is when the transcript came from an engine that does not diarize, which is+ # the case where `speaker` framing used to fall through to a letterbox.+ given = _json_file_arg(getattr(args, "face_map", None), "--face-map")+ if given:+ face_map = given+ print(" Using the face map passed in (speaker framing preserved)")++ # Still nothing, and the crop about to run needs one. Scanning costs a+ # minute on a long episode and buys back every clip in it: without a map,+ # `speaker` and `face` skip every rung that could place a frame and land on+ # the whole wide source letterboxed into the cut.+ if not face_map and config.get("crop_strategy") in ("face", "speaker", "speaker-hardcut"):+ try:+ from services.face_analysis import analyze_faces+ print(" No face map yet; scanning the episode for faces")+ # The last word's end, because the scan only uses this to decide+ # how many frames to sample; it reads the real frame count off the+ # file itself and spreads the samples across all of it.+ spoken = max((float(w.get("end") or 0) for w in words), default=0.0)+ found = analyze_faces(+ video_path,+ [{"speaker": w["speaker"], "start": w["start"], "end": w["end"]}+ for w in words if w.get("speaker")],+ spoken,+ progress_callback=lambda p, m: None,+ )+ except Exception as exc:+ print(f" Face scan failed ({type(exc).__name__}: {exc}); "+ "the crop will fall back", file=sys.stderr)+ found = None+ if found and found.get("clusters"):+ face_map = found+ print(f" Found {len(found['clusters'])} face position(s)")+ else:+ print(" No faces found; the crop will fall back")+# Check speaker data availability (needed for smart cropping)speakers_in_words = set(w.get("speaker") for w in words if w.get("speaker"))diarization_warning = result.get("diarization_warning")@@ -4399,7 +4475,12 @@ def main():help="Caption placement (default: follows the chosen style)")proc.add_argument("--caption-scale", type=float, choices=[0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5],help="Caption size multiplier (default: 1)")- proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"])+ proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut", "manual"])+ proc.add_argument("--face-map", dest="face_map", default=None,+ help="Where the faces sit in this video, as JSON. Skips detection and lets "+ "speaker framing work on a transcript that carries no speaker labels.")+ proc.add_argument("--crop-keyframes", dest="crop_keyframes", default=None,+ help="Hand-placed crop positions, as JSON. Used by --crop manual.")proc.add_argument("--format", choices=["vertical", "horizontal", "square"], help="Output aspect ratio (default: vertical)")proc.add_argument("--profile", choices=["podcast", "party", "action"], help="Detection profile: podcast (transcript-first, default), party/action (laughter/energy highlights)")proc.add_argument("--logo", help="Logo image (asset name or path)")@@ -4485,7 +4566,12 @@ def main():studio.add_argument("--caption-style", choices=["hormozi", "karaoke", "subtle", "branded"], default="hormozi")studio.add_argument("--caption-position", choices=["auto", "upper", "center", "lower"], default="auto")studio.add_argument("--caption-scale", type=float, default=1.0)- studio.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"], default="face")+ studio.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut", "manual"], default="face")+ studio.add_argument("--face-map", dest="face_map", default=None,+ help="Where the faces sit in this video, as JSON. Skips detection and lets "+ "speaker framing work on a transcript that carries no speaker labels.")+ studio.add_argument("--crop-keyframes", dest="crop_keyframes", default=None,+ help="Hand-placed crop positions, as JSON. Used by --crop manual.")studio.add_argument("--format", choices=["vertical", "horizontal", "square"], default="vertical",help="Output aspect ratio (default: vertical)")studio.add_argument("--template", help="Cut in a saved look (podcli Pro). Name or id.")diff --git a/backend/clip_studio.py b/backend/clip_studio.pyindex 8e79196..a6fe152 100644--- a/backend/clip_studio.py+++ b/backend/clip_studio.py@@ -161,11 +161,82 @@ def _json_arg(raw, name):return None+def _json_file(raw, name):+ """JSON given inline or named as a file, or nothing.++ Both, because the two callers differ: keyframes are a handful of numbers+ and have always been passed inline, while a face map is a per-second record+ of every face in the video and an argument list will not carry one. Telling+ them apart by looking is cheaper than a second flag.+ """+ if not raw:+ return None+ text = raw.strip()+ if text.startswith("{") or text.startswith("["):+ return _json_arg(text, name)+ try:+ with open(text, "r", encoding="utf-8") as handle:+ return json.load(handle)+ except (OSError, ValueError) as exc:+ print(f" Warning: {name} could not be read ({exc}); ignoring it",+ file=sys.stderr, flush=True)+ return None+++# The crops that cannot place a frame without knowing where the faces are.+_WANTS_FACES = ("face", "speaker", "speaker-hardcut")+++def _face_map_for(video, crop, start, end):+ """Where the faces are, scanned now, when the crop needs it and nobody said.++ A face map used to arrive only from the diarizing transcriber, so a cut+ made against a whisper transcript or an imported one had none, and every+ rung of the crop ladder that needs one was skipped. What the caller got+ instead was the whole wide frame letterboxed into the cut, reported as a+ centre crop and looking deliberate.++ Scanning here costs a few seconds on the window actually being cut, which+ is the trade every one of those clips would have taken.+ """+ if crop not in _WANTS_FACES:+ return None+ try:+ from services.face_analysis import analyze_faces+ except ImportError:+ return None++ # The whole file's length rather than the fragment's: the scan spreads its+ # samples across the file it is given, and sizing the count to a 40-second+ # window would take twenty samples across an hour-long episode.+ duration = _probe_duration(video) or max(0.0, (end or 0) - (start or 0))+ if duration <= 0:+ return None++ print(" [fragment] no face map for this cut; scanning it", flush=True)+ try:+ # No speaker segments: those come from diarization, and the whole point+ # of scanning here is that there was none. Clusters and the split+ # screen are still found, which is what a crop needs to place a frame.+ found = analyze_faces(video, [], duration)+ except Exception as exc:+ print(f" Warning: face scan failed ({type(exc).__name__}: {exc}); "+ "the crop will fall back", file=sys.stderr, flush=True)+ return None++ if not found or not found.get("clusters"):+ print(" [fragment] no faces found in this cut", flush=True)+ return None+ print(f" [fragment] found {len(found['clusters'])} face position(s)"+ f"{' , split screen' if found.get('is_split_screen') else ''}", flush=True)+ return found++def _render_fragment(video, start, end, words, style, crop, title, out_dir, fmt="vertical",logo=None, name_card=None, motion=None, caption_position="auto",caption_scale=1.0, logo_position="top-left", logo_scale=1.0,topic=None, progress=None, cards=None, brand=None, font_family=None,- captions=True):+ captions=True, face_map=None, crop_keyframes=None):"""Render the fragment with face-crop + captions via the existing engine."""from services.clip_generator import generate_clipprint(f" [fragment] rendering {start:.1f}s–{end:.1f}s ({style}, crop={crop}, {fmt})", flush=True)@@ -175,6 +246,7 @@ def _render_fragment(video, start, end, words, style, crop, title, out_dir, fmt=caption_font_scale=round(caption_scale * 100),logo_position=logo_position, logo_scale=logo_scale,crop_strategy=crop, format=fmt,+ face_map=face_map, crop_keyframes=crop_keyframes,transcript_words=words, title=title, output_dir=out_dir,logo_path=logo, name_card=name_card, motion=motion,topic=topic, progress=progress, cards=cards, brand=brand, font_family=font_family,@@ -296,6 +368,11 @@ def main():ap.add_argument("--progress", action="store_true",help="Draw how much of the clip is left along the bottom edge")ap.add_argument("--progress-color", default=None)+ ap.add_argument("--face-map", dest="face_map", default=None,+ help="Path to a JSON face map for this video. Lets speaker framing work "+ "on a transcript that carries no speaker labels.")+ ap.add_argument("--crop-keyframes", dest="crop_keyframes", default=None,+ help="Path to hand-placed crop positions as JSON. Used by --crop manual.")ap.add_argument("--cards", default=None,help="On-screen cards as JSON, each with kind/start/end")ap.add_argument("--brand", default=None,@@ -391,6 +468,10 @@ def main():print(" Warning: --motion is not valid JSON; using each style's own motion",flush=True)+ face_map = _json_file(args.face_map, "--face-map")+ if face_map is None:+ face_map = _face_map_for(video, args.crop, start, end)+fragment = _render_fragment(video, start, end, words, args.caption_style, args.crop, "fragment", out_dir,fmt=args.format,@@ -410,6 +491,8 @@ def main():brand=_json_arg(args.brand, "--brand"),font_family=args.font_family,captions=not args.no_captions,+ face_map=face_map,+ crop_keyframes=_json_file(args.crop_keyframes, "--crop-keyframes"),)platforms = [p.strip() for p in platforms_str.split(",") if p.strip()]diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 8f10461..eace9c7 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -247,12 +247,26 @@ def crop_to_vertical(clip_start: The start time of this clip in the original video (for timestamp alignment)."""width, height = get_dimensions(input_path)+ # What the caller asked for, kept because `strategy` is reassigned on the+ # way down the ladder. A clip that asked to follow the speaker and ended up+ # letterboxed should say so in those words, not report a centre crop as+ # though centre was the plan.+ wanted = strategyface_map = _sane_speaker_mappings(face_map)target_w, target_h = target_dimstarget_ratio = target_w / target_h # 0.5625 for the vertical defaultsource_ratio = width / height+ # Manual with nothing to place the frame from used to fall past every+ # branch below and reach the return with no filter built at all, which is+ # an UnboundLocalError rather than a clip. It became reachable the moment+ # a caller could pass --crop manual, so it answers for itself: no+ # keyframes is no hand placement, and the crop it wanted is a face.+ if strategy == "manual" and not crop_keyframes:+ log_event("crop", "fallback", reason="manual_without_keyframes", to="face")+ strategy = "face"+if strategy == "manual" and crop_keyframes:log_event("crop", "chose=manual", keyframes=len(crop_keyframes), source=f"{width}x{height}")crop_h = height@@ -417,7 +431,16 @@ def crop_to_vertical(if strategy == "center":if source_ratio > target_ratio:- log_event("crop", "chose=center-blur-bg", source=f"{width}x{height}")+ log_event("crop", "chose=center-blur-bg", source=f"{width}x{height}", asked=wanted)+ # The whole wide frame, shrunk into a band with a blur behind it.+ # It is the honest answer for a source with nothing to crop to and+ # the wrong one for a clip that asked to follow a face, so it is+ # reported at warn rather than left to look deliberate.+ if wanted != "center":+ log_event(+ "crop", "uncropped", level="warn", asked=wanted,+ reason="no face map, no speaker labels and no face found in this window",+ )# Wide source with no face detected: blurred background + sharp center.# Scales source to fill 9:16 height → blur → overlay sharp fit-to-width.vf_complex = (