Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Let a caller hand in the face map, and stop a lost crop looking deliberate (#220)

Nika Siradzefe0bf9986b67
diff --git a/backend/cli.py b/backend/cli.py
index 0764f47..98fec7c 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -84,6 +84,27 @@ def _parse_json_transcript(raw_text):
return data.get("words", []), data.get("segments", []), data
+def _json_file_arg(raw: str | None, name: str):
+ """JSON given inline or named as a file, or nothing.
+
+ Both, because the two callers differ: keyframes are a handful of numbers
+ and have always been passed inline, while a face map is a per-second record
+ of every face in the video, which an argument list will not carry. Missing
+ or malformed loses the framing hint, never the run.
+ """
+ if not raw:
+ return None
+ text = raw.strip()
+ try:
+ if text.startswith("{") or text.startswith("["):
+ return json.loads(text)
+ with open(text, "r", encoding="utf-8") as handle:
+ return json.load(handle)
+ except (OSError, ValueError) as exc:
+ print(f" Warning: {name} could not be read ({exc}); ignoring it", file=sys.stderr)
+ return None
+
+
def _cached_face_map(video_path: str):
"""Face maps are keyed by video content, not by transcript, so an imported
transcript can still borrow the map from an earlier run on the same file."""
@@ -470,6 +491,22 @@ def cmd_studio(args):
"--caption-scale", str(getattr(args, "caption_scale", 1.0)),
"--crop", args.crop,
"--format", getattr(args, "format", None) or "vertical",
+ ]
+ # Where the faces are, and where somebody put the frame by hand. Both are
+ # what the crop needs and neither could be handed to it before, so a caller
+ # that already knew — the cloud worker scans the window before it renders —
+ # had no way to say so and watched `speaker` fall through to a letterbox.
+ # A path is made absolute because the script runs from elsewhere; JSON
+ # given inline is handed straight through.
+ def _pass_through(value):
+ text = str(value).strip()
+ return text if text.startswith(("{", "[")) else os.path.abspath(text)
+
+ if getattr(args, "face_map", None):
+ cmd += ["--face-map", _pass_through(args.face_map)]
+ if getattr(args, "crop_keyframes", None):
+ cmd += ["--crop-keyframes", _pass_through(args.crop_keyframes)]
+ cmd += [
"--logo-position", getattr(args, "logo_position", "top-left"),
"--logo-scale", str(getattr(args, "logo_scale", 1.0)),
"--intro-seconds", str(args.intro_seconds),
@@ -979,6 +1016,45 @@ def cmd_process(args):
# Extract face_map before result gets overwritten in clip loop
face_map = result.get("face_map")
+ # A map handed in on the command line wins over one the transcriber found,
+ # because the caller who bothered to pass it scanned this video rather than
+ # inferring the layout from who was speaking. It is also the only map there
+ # is when the transcript came from an engine that does not diarize, which is
+ # the case where `speaker` framing used to fall through to a letterbox.
+ given = _json_file_arg(getattr(args, "face_map", None), "--face-map")
+ if given:
+ face_map = given
+ print(" Using the face map passed in (speaker framing preserved)")
+
+ # Still nothing, and the crop about to run needs one. Scanning costs a
+ # minute on a long episode and buys back every clip in it: without a map,
+ # `speaker` and `face` skip every rung that could place a frame and land on
+ # the whole wide source letterboxed into the cut.
+ if not face_map and config.get("crop_strategy") in ("face", "speaker", "speaker-hardcut"):
+ try:
+ from services.face_analysis import analyze_faces
+ print(" No face map yet; scanning the episode for faces")
+ # The last word's end, because the scan only uses this to decide
+ # how many frames to sample; it reads the real frame count off the
+ # file itself and spreads the samples across all of it.
+ spoken = max((float(w.get("end") or 0) for w in words), default=0.0)
+ found = analyze_faces(
+ video_path,
+ [{"speaker": w["speaker"], "start": w["start"], "end": w["end"]}
+ for w in words if w.get("speaker")],
+ spoken,
+ progress_callback=lambda p, m: None,
+ )
+ except Exception as exc:
+ print(f" Face scan failed ({type(exc).__name__}: {exc}); "
+ "the crop will fall back", file=sys.stderr)
+ found = None
+ if found and found.get("clusters"):
+ face_map = found
+ print(f" Found {len(found['clusters'])} face position(s)")
+ else:
+ print(" No faces found; the crop will fall back")
+
# Check speaker data availability (needed for smart cropping)
speakers_in_words = set(w.get("speaker") for w in words if w.get("speaker"))
diarization_warning = result.get("diarization_warning")
@@ -4399,7 +4475,12 @@ def main():
help="Caption placement (default: follows the chosen style)")
proc.add_argument("--caption-scale", type=float, choices=[0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5],
help="Caption size multiplier (default: 1)")
- proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"])
+ proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut", "manual"])
+ proc.add_argument("--face-map", dest="face_map", default=None,
+ help="Where the faces sit in this video, as JSON. Skips detection and lets "
+ "speaker framing work on a transcript that carries no speaker labels.")
+ proc.add_argument("--crop-keyframes", dest="crop_keyframes", default=None,
+ help="Hand-placed crop positions, as JSON. Used by --crop manual.")
proc.add_argument("--format", choices=["vertical", "horizontal", "square"], help="Output aspect ratio (default: vertical)")
proc.add_argument("--profile", choices=["podcast", "party", "action"], help="Detection profile: podcast (transcript-first, default), party/action (laughter/energy highlights)")
proc.add_argument("--logo", help="Logo image (asset name or path)")
@@ -4485,7 +4566,12 @@ def main():
studio.add_argument("--caption-style", choices=["hormozi", "karaoke", "subtle", "branded"], default="hormozi")
studio.add_argument("--caption-position", choices=["auto", "upper", "center", "lower"], default="auto")
studio.add_argument("--caption-scale", type=float, default=1.0)
- studio.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"], default="face")
+ studio.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut", "manual"], default="face")
+ studio.add_argument("--face-map", dest="face_map", default=None,
+ help="Where the faces sit in this video, as JSON. Skips detection and lets "
+ "speaker framing work on a transcript that carries no speaker labels.")
+ studio.add_argument("--crop-keyframes", dest="crop_keyframes", default=None,
+ help="Hand-placed crop positions, as JSON. Used by --crop manual.")
studio.add_argument("--format", choices=["vertical", "horizontal", "square"], default="vertical",
help="Output aspect ratio (default: vertical)")
studio.add_argument("--template", help="Cut in a saved look (podcli Pro). Name or id.")
diff --git a/backend/clip_studio.py b/backend/clip_studio.py
index 8e79196..a6fe152 100644
--- a/backend/clip_studio.py
+++ b/backend/clip_studio.py
@@ -161,11 +161,82 @@ def _json_arg(raw, name):
return None
+def _json_file(raw, name):
+ """JSON given inline or named as a file, or nothing.
+
+ Both, because the two callers differ: keyframes are a handful of numbers
+ and have always been passed inline, while a face map is a per-second record
+ of every face in the video and an argument list will not carry one. Telling
+ them apart by looking is cheaper than a second flag.
+ """
+ if not raw:
+ return None
+ text = raw.strip()
+ if text.startswith("{") or text.startswith("["):
+ return _json_arg(text, name)
+ try:
+ with open(text, "r", encoding="utf-8") as handle:
+ return json.load(handle)
+ except (OSError, ValueError) as exc:
+ print(f" Warning: {name} could not be read ({exc}); ignoring it",
+ file=sys.stderr, flush=True)
+ return None
+
+
+# The crops that cannot place a frame without knowing where the faces are.
+_WANTS_FACES = ("face", "speaker", "speaker-hardcut")
+
+
+def _face_map_for(video, crop, start, end):
+ """Where the faces are, scanned now, when the crop needs it and nobody said.
+
+ A face map used to arrive only from the diarizing transcriber, so a cut
+ made against a whisper transcript or an imported one had none, and every
+ rung of the crop ladder that needs one was skipped. What the caller got
+ instead was the whole wide frame letterboxed into the cut, reported as a
+ centre crop and looking deliberate.
+
+ Scanning here costs a few seconds on the window actually being cut, which
+ is the trade every one of those clips would have taken.
+ """
+ if crop not in _WANTS_FACES:
+ return None
+ try:
+ from services.face_analysis import analyze_faces
+ except ImportError:
+ return None
+
+ # The whole file's length rather than the fragment's: the scan spreads its
+ # samples across the file it is given, and sizing the count to a 40-second
+ # window would take twenty samples across an hour-long episode.
+ duration = _probe_duration(video) or max(0.0, (end or 0) - (start or 0))
+ if duration <= 0:
+ return None
+
+ print(" [fragment] no face map for this cut; scanning it", flush=True)
+ try:
+ # No speaker segments: those come from diarization, and the whole point
+ # of scanning here is that there was none. Clusters and the split
+ # screen are still found, which is what a crop needs to place a frame.
+ found = analyze_faces(video, [], duration)
+ except Exception as exc:
+ print(f" Warning: face scan failed ({type(exc).__name__}: {exc}); "
+ "the crop will fall back", file=sys.stderr, flush=True)
+ return None
+
+ if not found or not found.get("clusters"):
+ print(" [fragment] no faces found in this cut", flush=True)
+ return None
+ print(f" [fragment] found {len(found['clusters'])} face position(s)"
+ f"{' , split screen' if found.get('is_split_screen') else ''}", flush=True)
+ return found
+
+
def _render_fragment(video, start, end, words, style, crop, title, out_dir, fmt="vertical",
logo=None, name_card=None, motion=None, caption_position="auto",
caption_scale=1.0, logo_position="top-left", logo_scale=1.0,
topic=None, progress=None, cards=None, brand=None, font_family=None,
- captions=True):
+ captions=True, face_map=None, crop_keyframes=None):
"""Render the fragment with face-crop + captions via the existing engine."""
from services.clip_generator import generate_clip
print(f" [fragment] rendering {start:.1f}s–{end:.1f}s ({style}, crop={crop}, {fmt})", flush=True)
@@ -175,6 +246,7 @@ def _render_fragment(video, start, end, words, style, crop, title, out_dir, fmt=
caption_font_scale=round(caption_scale * 100),
logo_position=logo_position, logo_scale=logo_scale,
crop_strategy=crop, format=fmt,
+ face_map=face_map, crop_keyframes=crop_keyframes,
transcript_words=words, title=title, output_dir=out_dir,
logo_path=logo, name_card=name_card, motion=motion,
topic=topic, progress=progress, cards=cards, brand=brand, font_family=font_family,
@@ -296,6 +368,11 @@ def main():
ap.add_argument("--progress", action="store_true",
help="Draw how much of the clip is left along the bottom edge")
ap.add_argument("--progress-color", default=None)
+ ap.add_argument("--face-map", dest="face_map", default=None,
+ help="Path to a JSON face map for this video. Lets speaker framing work "
+ "on a transcript that carries no speaker labels.")
+ ap.add_argument("--crop-keyframes", dest="crop_keyframes", default=None,
+ help="Path to hand-placed crop positions as JSON. Used by --crop manual.")
ap.add_argument("--cards", default=None,
help="On-screen cards as JSON, each with kind/start/end")
ap.add_argument("--brand", default=None,
@@ -391,6 +468,10 @@ def main():
print(" Warning: --motion is not valid JSON; using each style's own motion",
flush=True)
+ face_map = _json_file(args.face_map, "--face-map")
+ if face_map is None:
+ face_map = _face_map_for(video, args.crop, start, end)
+
fragment = _render_fragment(
video, start, end, words, args.caption_style, args.crop, "fragment", out_dir,
fmt=args.format,
@@ -410,6 +491,8 @@ def main():
brand=_json_arg(args.brand, "--brand"),
font_family=args.font_family,
captions=not args.no_captions,
+ face_map=face_map,
+ crop_keyframes=_json_file(args.crop_keyframes, "--crop-keyframes"),
)
platforms = [p.strip() for p in platforms_str.split(",") if p.strip()]
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 8f10461..eace9c7 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -247,12 +247,26 @@ def crop_to_vertical(
clip_start: The start time of this clip in the original video (for timestamp alignment).
"""
width, height = get_dimensions(input_path)
+ # What the caller asked for, kept because `strategy` is reassigned on the
+ # way down the ladder. A clip that asked to follow the speaker and ended up
+ # letterboxed should say so in those words, not report a centre crop as
+ # though centre was the plan.
+ wanted = strategy
face_map = _sane_speaker_mappings(face_map)
target_w, target_h = target_dims
target_ratio = target_w / target_h # 0.5625 for the vertical default
source_ratio = width / height
+ # Manual with nothing to place the frame from used to fall past every
+ # branch below and reach the return with no filter built at all, which is
+ # an UnboundLocalError rather than a clip. It became reachable the moment
+ # a caller could pass --crop manual, so it answers for itself: no
+ # keyframes is no hand placement, and the crop it wanted is a face.
+ if strategy == "manual" and not crop_keyframes:
+ log_event("crop", "fallback", reason="manual_without_keyframes", to="face")
+ strategy = "face"
+
if strategy == "manual" and crop_keyframes:
log_event("crop", "chose=manual", keyframes=len(crop_keyframes), source=f"{width}x{height}")
crop_h = height
@@ -417,7 +431,16 @@ def crop_to_vertical(
if strategy == "center":
if source_ratio > target_ratio:
- log_event("crop", "chose=center-blur-bg", source=f"{width}x{height}")
+ log_event("crop", "chose=center-blur-bg", source=f"{width}x{height}", asked=wanted)
+ # The whole wide frame, shrunk into a band with a blur behind it.
+ # It is the honest answer for a source with nothing to crop to and
+ # the wrong one for a clip that asked to follow a face, so it is
+ # reported at warn rather than left to look deliberate.
+ if wanted != "center":
+ log_event(
+ "crop", "uncropped", level="warn", asked=wanted,
+ reason="no face map, no speaker labels and no face found in this window",
+ )
# Wide source with no face detected: blurred background + sharp center.
# Scales source to fill 9:16 height → blur → overlay sharp fit-to-width.
vf_complex = (