Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Snap between framings on the cut the source already made (#178)
Nika Siradze1d246371d9ea
diff --git a/.gitignore b/.gitignoreindex c87914f..926b1a9 100644--- a/.gitignore+++ b/.gitignore@@ -67,3 +67,4 @@ reel_*/# generated by scripts/gen-docs-manifest.mjsdocs-manifest.json+.venv/diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex da9a5f3..98d2dc1 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -6,6 +6,7 @@ audio normalization, and final encoding."""import os+import reimport subprocessimport jsonimport math@@ -109,6 +110,68 @@ def _sane_speaker_mappings(face_map: Optional[dict]) -> Optional[dict]:return out+def _source_cut_times(input_path: str, min_score: float = 0.25) -> list[float]:+ """Times where the source itself cuts, by ffmpeg's own scene score.++ Never raises: a crop that cannot read the cuts still crops, just without+ snapping to them.+ """+ cmd = [+ "ffmpeg", "-v", "error", "-i", input_path,+ "-filter:v", f"select='gt(scene,{min_score})',metadata=print:file=-",+ "-f", "null", "-",+ ]+ try:+ r = proc_run(cmd, timeout=_FFMPEG_TIMEOUT, check=False)+ except Exception:+ return []+ if r.returncode != 0:+ return []+ blob = (r.stdout or "") + (r.stderr or "")+ return [float(m) for m in re.findall(r"pts_time:([0-9.]+)", blob)]+++def _snap_runs_to_source_cuts(runs: list, cuts: list, tolerance: float = 0.2) -> list:+ """Move each run boundary onto the source cut it is trying to describe.++ Runs close at the first sample that noticed the layout change, and the+ sampler runs at about 12Hz, so a boundary sits up to 83ms past the cut it+ belongs to. Two frames of the old crop then land on the new layout, which+ is both the wall left in frame and a second scene change 83ms after the+ source's own. That pair reads as clustered cuts and pulls the transition+ blur onto a cut that did not need it.+ """+ if not cuts or len(runs) < 2:+ return runs+ snapped = list(runs)+ for i in range(len(snapped) - 1):+ boundary = snapped[i][1]+ near = min(cuts, key=lambda c: abs(c - boundary))+ if abs(near - boundary) > tolerance:+ continue+ if near <= snapped[i][0] or near >= snapped[i + 1][1]:+ continue+ snapped[i] = (snapped[i][0], near, snapped[i][2])+ snapped[i + 1] = (near, snapped[i + 1][1], snapped[i + 1][2])+ return snapped+++def _run_step_crop_x_expr(runs: list, crop_x_for) -> str:+ """Hold each layout run's crop until the next run starts, then snap.++ One expression evaluated over one pass, rather than a part per run joined+ end to end. Cutting and concatenating re-encodes every part and every part+ overshoots the length it was asked for, because -t keeps the frame that+ starts before the cut and each part's audio carries its own encoder+ padding. Measured across five runs that was 403ms of drift, and it left the+ video ending 177ms before its own audio.+ """+ expr = str(crop_x_for(runs[0][2]))+ for r_start, _r_end, r_cx in runs[1:]:+ expr = f"if(gte(t\\,{r_start:.3f})\\,{crop_x_for(r_cx)}\\,{expr})"+ return expr++def crop_to_vertical(input_path: str,output_path: str,@@ -1531,10 +1594,73 @@ def _track_and_crop_inner(output_path=output_path, label="crop_mixed_static",)+ # A dissolve needs a window where both framings hold the+ # speaker, and a layout change leaves none: the pad below runs this+ # run's crop over the next run's content, so the subject has+ # already moved out of it. Measured on a 3840-wide two-up that was+ # 209ms of bare wall, 52 luma brighter than either speaker, fading+ # into the right framing. The subject survives the pad only while+ # the next centre is still inside this crop, half a window either+ # side. Past that, snap between the framings in a single pass: the+ # source cut in the same place, so the cut is invisible, and one+ # pass cannot drift against its own audio the way cutting into+ # parts and joining them does.+ reach = crop_w / 2+ if any(abs(runs[i + 1][2] - runs[i][2]) > reach+ for i in range(len(runs) - 1)):+ widest = max(abs(runs[i + 1][2] - runs[i][2])+ for i in range(len(runs) - 1))+ cuts = _source_cut_times(input_path)+ snapped = _snap_runs_to_source_cuts(runs, cuts)+ moved = sum(1 for a, b in zip(runs, snapped) if abs(a[1] - b[1]) > 0.001)+ x_expr = _run_step_crop_x_expr(snapped, _crop_for)+ vf = (f"crop={crop_w}:{crop_h}:x='{x_expr}':y={crop_y},"+ f"scale={target_w}:{target_h}")+ log_event("crop", "chose=mixed-step-cuts", runs=len(runs),+ widest=int(widest), reach=int(reach),+ cuts=len(cuts), snapped=moved)+ stepped = _run_ffmpeg_with_fallback(+ cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],+ cmd_parts_after_enc=[+ "-c:a", "aac", "-b:a", "192k", "-ar", "44100",+ "-movflags", "+faststart",+ # The same explicit length the dissolve pins itself to.+ # Without it the segment's audio runs past its own video+ # and the encoder's padding widens the gap further: 163ms+ # of audio with no picture under it.+ "-t", f"{duration:.3f}",+ ],+ output_path=output_path, label="crop_mixed_steps",+ )+ if stepped:+ return stepped+ log_event("crop", "fallback", reason="mixed_steps_failed",+ to="dissolve")+# Multiple layouts — crop each segment separately, join# with cross-dissolve so transitions feel editorial.work_dir = os.path.dirname(output_path) or "."xfade_dur = 0.18 # Short dissolve++ # A dissolve needs 180ms where both framings hold the speaker, and+ # a layout change leaves none: the padding below runs this run's+ # crop over the next run's content, so the subject has already+ # moved out of it. Measured on a 3840-wide two-up, that was 209ms+ # of bare wall 52 luma brighter than either speaker, dissolving+ # into the right framing. The subject survives the pad only while+ # the next centre is still inside this crop, which is half a window+ # either side; past that there is nothing to dissolve and the cut+ # below is both correct and invisible, because the source cut too.+ reach = crop_w / 2+ dissolve_holds_subject = all(+ abs(runs[i + 1][2] - runs[i][2]) <= reach for i in range(len(runs) - 1)+ )+ if not dissolve_holds_subject:+ widest = max(+ abs(runs[i + 1][2] - runs[i][2]) for i in range(len(runs) - 1)+ )+ log_event("crop", "mixed_cuts", reason="layout_change",+ runs=len(runs), widest=int(widest), reach=int(reach))part_paths = []cut_paths = []# (path, measured duration, padding it carries). Offsets are built@@ -1545,7 +1671,7 @@ def _track_and_crop_inner(part_specs = []try:- for ri, (r_start, r_end, r_cx) in enumerate(runs):+ for ri, (r_start, r_end, r_cx) in enumerate(runs if dissolve_holds_subject else []):r_crop_x = _crop_for(r_cx)# Pad segment slightly for xfade overlappad = xfade_dur if ri < len(runs) - 1 else 0@@ -1581,8 +1707,11 @@ def _track_and_crop_inner(part_specs[-1] = (last_path, last_len, 0.0)if len(part_paths) < 2:- log_event("crop", "fallback", reason="mixed_too_few_parts",- parts=len(part_paths), to="hard-cuts")+ # Only a surprise when a dissolve was actually attempted;+ # skipping it above is a decision, not a fallback.+ if dissolve_holds_subject:+ log_event("crop", "fallback", reason="mixed_too_few_parts",+ parts=len(part_paths), to="hard-cuts")else:# Where each part's own content begins in the output: the# running total of the parts before it, each counted withoutdiff --git a/tests/test_crop_path_golden.py b/tests/test_crop_path_golden.pyindex c20d3b0..7875670 100644--- a/tests/test_crop_path_golden.py+++ b/tests/test_crop_path_golden.py@@ -10,6 +10,7 @@ No cv2, ffmpeg, or video files — the fixtures stand in for decoded frames."""import json+import reimport osimport sysimport tempfile@@ -437,3 +438,113 @@ class CropFailuresNeverFailTheClipTests(unittest.TestCase):if __name__ == "__main__":unittest.main()+++class DissolveNeedsBothFramingsTests(unittest.TestCase):+ """A dissolve across a layout change has nothing to dissolve.++ The stitch pads each run by the dissolve length, which runs that run's crop+ over the next run's content. The subject survives that pad only while the+ next centre is still inside this crop, which is half a window either side.+ On the episode this came from the widest boundary was 1167px against a+ 607px reach, and the dissolve showed 209ms of bare wall.+ """++ @staticmethod+ def holds(runs, crop_w):+ reach = crop_w / 2+ return all(abs(runs[i + 1][2] - runs[i][2]) <= reach for i in range(len(runs) - 1))++ def test_a_layout_change_is_too_wide_to_dissolve(self):+ runs = [(0.0, 2.5, 1006), (2.5, 6.0, 2173)] # 1167px apart+ self.assertFalse(self.holds(runs, 1214))++ def test_a_speaker_shifting_in_their_seat_still_dissolves(self):+ runs = [(0.0, 2.5, 1006), (2.5, 6.0, 1300)] # 294px apart+ self.assertTrue(self.holds(runs, 1214))++ def test_one_wide_boundary_settles_it_for_the_whole_clip(self):+ runs = [(0.0, 2.0, 1006), (2.0, 4.0, 1200), (4.0, 6.0, 2400)]+ self.assertFalse(self.holds(runs, 1214))+++class SteppedRunCropTests(unittest.TestCase):+ """One expression over one pass, so a layout change cannot drift.++ Cutting a run per layout and joining them re-encodes each part, and each+ part overshoots the length it was asked for: 403ms across five runs, with+ the video ending 177ms before its own audio.+ """++ RUNS = [(0.0, 2.629, 1021), (2.629, 6.386, 1940), (6.386, 10.393, 921)]+ CROP = staticmethod(lambda cx: max(0, min(int(cx - 607), 3840 - 1214)))++ def _at(self, expr, t):+ """Evaluate the nested if(gte(t,T),a,b) the way FFmpeg would."""+ expr = expr.replace("\\", "")+ while expr.startswith("if("):+ inner, depth, parts, cur = expr[3:-1], 0, [], ""+ for ch in inner:+ if ch == "(":+ depth += 1+ if ch == ")":+ depth -= 1+ if ch == "," and depth == 0:+ parts.append(cur)+ cur = ""+ else:+ cur += ch+ parts.append(cur)+ cond, yes, no = parts+ thr = float(re.search(r"gte\(t,([\d.]+)\)", cond).group(1))+ expr = yes if t >= thr else no+ return int(expr)++ def test_each_run_holds_its_own_crop(self):+ e = vp._run_step_crop_x_expr(self.RUNS, self.CROP)+ for start, end, cx in self.RUNS:+ mid = (start + end) / 2+ self.assertEqual(self._at(e, mid), self.CROP(cx))++ def test_it_snaps_at_the_boundary_rather_than_gliding(self):+ e = vp._run_step_crop_x_expr(self.RUNS, self.CROP)+ b = self.RUNS[1][0]+ self.assertEqual(self._at(e, b - 0.001), self.CROP(self.RUNS[0][2]))+ self.assertEqual(self._at(e, b), self.CROP(self.RUNS[1][2]))++ def test_a_single_run_is_a_constant(self):+ e = vp._run_step_crop_x_expr([self.RUNS[0]], self.CROP)+ self.assertEqual(e, str(self.CROP(self.RUNS[0][2])))+++class SnapRunsToSourceCutsTests(unittest.TestCase):+ """A boundary belongs on the cut it is describing, not near it.++ The sampler runs at about 12Hz, so a run closes up to 83ms after the cut it+ noticed. Those two frames of old crop on new layout are the wall left in+ frame, and they register as a second scene change 83ms after the source's+ own, which reads as clustered cuts and drags the transition blur onto a cut+ that never needed it.+ """++ CUTS = [0.875, 7.792, 16.958, 23.458]++ def test_a_late_boundary_moves_onto_the_cut(self):+ runs = [(0.0, 7.87, 1000), (7.87, 17.04, 2000), (17.04, 33.4, 1000)]+ out = vp._snap_runs_to_source_cuts(runs, self.CUTS)+ self.assertAlmostEqual(out[0][1], 7.792, places=3)+ self.assertAlmostEqual(out[1][0], 7.792, places=3)+ self.assertAlmostEqual(out[1][1], 16.958, places=3)++ def test_a_boundary_with_no_cut_near_it_is_left_alone(self):+ runs = [(0.0, 12.0, 1000), (12.0, 33.4, 2000)]+ self.assertEqual(vp._snap_runs_to_source_cuts(runs, self.CUTS), runs)++ def test_it_never_collapses_a_run(self):+ runs = [(0.0, 0.9, 1000), (0.9, 1.0, 2000), (1.0, 5.0, 1000)]+ for a, b, _ in vp._snap_runs_to_source_cuts(runs, self.CUTS):+ self.assertLess(a, b)++ def test_no_cuts_detected_changes_nothing(self):+ runs = [(0.0, 7.87, 1000), (7.87, 33.4, 2000)]+ self.assertEqual(vp._snap_runs_to_source_cuts(runs, []), runs)