Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Fix framing, captions, scramble and steps found in the Advaith renders (#249)
Nika Siradze36cd57cc91ad
diff --git a/backend/cli.py b/backend/cli.pyindex 0a2a121..886362a 100644--- a/backend/cli.py+++ b/backend/cli.py@@ -501,6 +501,8 @@ def cmd_studio(args):cmd += ["--language", args.language]if getattr(args, "engine", None):cmd += ["--engine", args.engine]+ if getattr(args, "transcript", None):+ cmd += ["--transcript", args.transcript]env = os.environ.copy()if getattr(args, "assemblyai_api_key", None):env["ASSEMBLYAI_API_KEY"] = args.assemblyai_api_key@@ -5012,6 +5014,7 @@ def main():studio.add_argument("--paragraph", help="Find the fragment by matching this text in the transcript")studio.add_argument("--language", help="Transcription language (e.g. es). Auto-detect if omitted.")studio.add_argument("--engine", choices=["whisper-py", "whispercpp", "assemblyai"], help="Transcription engine")+ studio.add_argument("--transcript", help="Word timings JSON for this video ({words:[...]} or a list); skips transcription")studio.add_argument("--assemblyai-api-key", help="AssemblyAI API key for --engine assemblyai. Prefer ASSEMBLYAI_API_KEY; command-line secrets can appear in process listings.")studio.add_argument("--caption-style", choices=["hormozi", "karaoke", "subtle", "branded", "outline"], default="hormozi")studio.add_argument("--caption-position", choices=["auto", "upper", "center", "lower"], default="auto")diff --git a/backend/services/face_track_helpers.py b/backend/services/face_track_helpers.pyindex 4121f39..769cd7e 100644--- a/backend/services/face_track_helpers.py+++ b/backend/services/face_track_helpers.py@@ -270,6 +270,38 @@ def longest_layout_run(detections: list, split: bool) -> float:return best+def follow_within_runs(+ runs: list,+ face_points: list,+ crop_w: int,+ video_width: int,+ fast: float = 480.0,+ slow: float = 140.0,+) -> list:+ half = crop_w / 2.0+ lowest, highest = half, max(half, video_width - half)+ keys = []+ for start, end, center in runs:+ cam = min(max(center, lowest), highest)+ if keys:+ keys.append((max(keys[-1][0], start - 0.001), keys[-1][1]))+ keys.append((start, cam))+ last = start+ for t, cx, fw in face_points:+ if not start <= t < end:+ continue+ room = max(24.0, (crop_w - fw) / 2.0 - crop_w * 0.04)+ calm = min(crop_w * 0.22, room)+ diff = cx - cam+ if abs(diff) > calm:+ speed = fast if abs(diff) > room else slow+ cam += min(abs(diff), speed * max(t - last, 0.01)) * (1 if diff > 0 else -1)+ cam = min(max(cam, lowest), highest)+ last = t+ keys.append((t, cam))+ return keys++def clip_layout_is_mixed(detections: list,face_map: dict | None = None,diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 4c9075f..5f40794 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -44,6 +44,7 @@ from services.face_track_helpers import (clamp_away_from_dead_zone as _clamp_away_from_dead_zone,clip_layout_is_mixed as _clip_layout_is_mixed,crop_center_keeping_faces_visible as _crop_center_keeping_faces_visible,+ follow_within_runs as _follow_within_runs,followed_face_cx_at as _followed_face_cx_at,safe_default_center as _safe_default_center,update_tripod_camera as _update_tripod_camera,@@ -1590,6 +1591,7 @@ def _track_and_crop_inner(# ── Collect per-frame face positions (speaker-aware) ──────from statistics import medianface_points = [] # [(t, center_x), ...]+ face_widths = []for t, faces in detections:if not faces:continue@@ -1603,6 +1605,7 @@ def _track_and_crop_inner(else:best = max(faces, key=lambda f: f["fw"])face_points.append((t, float(best["cx"])))+ face_widths.append(float(best.get("fw", 0)))# ── Split into stable position runs ─────────────────────# Each run = one locked camera position. Runs split on@@ -1687,7 +1690,14 @@ def _track_and_crop_inner(cuts = _source_cut_times(input_path)snapped = _snap_runs_to_source_cuts(runs, cuts)moved = sum(1 for a, b in zip(runs, snapped) if abs(a[1] - b[1]) > 0.001)- x_expr = _run_step_crop_x_expr(snapped, _crop_for)+ followed = _follow_within_runs(+ snapped,+ [(t, cx, fw) for (t, cx), fw in zip(face_points, face_widths)],+ crop_w, width,+ )+ keys = _simplify_keyframes([(t, _crop_for(cx)) for t, cx in followed], tolerance=8)+ x_expr = (_build_cam_expr(keys, duration, False)+ or _run_step_crop_x_expr(snapped, _crop_for))vf = (f"crop={crop_w}:{crop_h}:x='{x_expr}':y={crop_y},"f"scale={target_w}:{target_h}")log_event("crop", "chose=mixed-step-cuts", runs=len(runs),diff --git a/remotion/src/CaptionedClip.tsx b/remotion/src/CaptionedClip.tsxindex cba700e..d60a0b2 100644--- a/remotion/src/CaptionedClip.tsx+++ b/remotion/src/CaptionedClip.tsx@@ -56,6 +56,8 @@ export interface CaptionedClipProps {theme?: ThemeInput | null;}+const ON_PAGE = 0.8;+export const CaptionedClip: React.FC<CaptionedClipProps> = ({videoSrc,words,@@ -92,7 +94,6 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({const nameCardSeconds = nameCard?.title ? (nameCard.seconds ?? 3) : 0;const pastNameCard = frame / fps >= nameCardSeconds;- const cardPlanned = Boolean(cards?.length);const upNow = pastNameCard ? cardAt(cards ?? [], frame / fps) : null;/** How far in the card is, rather than whether it is.@@ -109,8 +110,7 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({motion: upNow.motion ?? MOTION.card,}).opacity: 0;- const restingShrink = cardPlanned ? 0.75 : 1;- const captionShrink = restingShrink + (0.6 - restingShrink) * cardIn;+ const captionShrink = 1 + (ON_PAGE - 1) * cardIn;/** Captions used to drop toward the bottom edge while a card held the frame,* on the reasoning that there is no chin down there to clear. There is no@@ -149,7 +149,7 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({*/const settledStyle: CaptionStyle = {...captionStyle,- fontSize: style.fontSize * captionSize * 0.6,+ fontSize: voice.fontSize * captionSize * ON_PAGE,};/*@@ -222,7 +222,7 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({title={nameCard.title}subtitle={nameCard.subtitle}seconds={nameCardSeconds}- bottom={(style.marginBottom + 150) * (height / 1920)}+ bottom={captionZone(captionStyle) + 24}/>)}{!theme && nameCard?.title && (diff --git a/remotion/src/components/Scene.tsx b/remotion/src/components/Scene.tsxindex 2f6f1bb..8446ac6 100644--- a/remotion/src/components/Scene.tsx+++ b/remotion/src/components/Scene.tsx@@ -6,7 +6,7 @@ import { FONT } from "../types";import { context, muted, track } from "./brand";import type { Brand } from "./brand";import {- emphasisRuns, GAP_SIZE, MEDIA_HEIGHT, MIN_FIT, SCENE_WIDTH, TYPE_SIZE, sceneHeight,+ emphasisRuns, GAP_SIZE, MEDIA_HEIGHT, MIN_FIT, SCENE_WIDTH, TYPE_SIZE, sceneHeight, stepType,sizeOf,} from "../scene";import type { Align, Block, Gap, Layout, Size, Tone } from "../scene";@@ -278,6 +278,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {if (block.type === "steps") {const last = block.points.length - 1;+ const type = stepType(block.points, paint.width / unit);return (<div style={{ display: "flex", alignItems: "center" }}>{block.points.map((point, i) => (@@ -287,9 +288,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {style={{flex: 1,height: MARK.rule * unit,- background: i === last- ? `linear-gradient(90deg, ${context(brand.ink)}, ${accent})`- : context(brand.ink),+ background: i === last ? accent : context(brand.ink),margin: `0 ${18 * unit}px`,transform: `translateY(${-14 * unit}px)`,}}@@ -308,7 +307,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {<divstyle={{fontFamily: paint.font,- fontSize: TYPE_SIZE.md * unit,+ fontSize: type.value * unit,fontWeight: 700,color: brand.ink,marginTop: 12 * unit,@@ -321,7 +320,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {<divstyle={{fontFamily: paint.font,- fontSize: TYPE_SIZE.xs * unit,+ fontSize: type.note * unit,color: muted(brand.ink),marginTop: 6 * unit,}}diff --git a/remotion/src/scene.ts b/remotion/src/scene.tsindex 4afaa11..28898a3 100644--- a/remotion/src/scene.ts+++ b/remotion/src/scene.ts@@ -168,7 +168,22 @@ const lines = (text: string, size: number, width: number, advance = ADVANCE) =>const BAR_ROW = 26 + 8 + 34 * 1.3;-const STEP_ROW = 26 + 58 + 34 * 1.3 + 24;+const CHAR_WIDTH = 0.58;++export const stepType = (points: StepPoint[], width = SCENE_WIDTH) => {+ const slot = (width - Math.max(0, points.length - 1) * 120) / Math.max(1, points.length);+ const fits = (text: string | undefined, size: number) => !text || text.length * size * CHAR_WIDTH <= slot;+ const value = [TYPE_SIZE.xl, TYPE_SIZE.lg, TYPE_SIZE.md]+ .find((size) => points.every((point) => fits(point.value, size))) ?? TYPE_SIZE.md;+ const note = [TYPE_SIZE.sm, TYPE_SIZE.xs]+ .find((size) => points.every((point) => fits(point.note, size))) ?? TYPE_SIZE.xs;+ return { value, note };+};++const stepRow = (points: StepPoint[], width: number) => {+ const type = stepType(points, width);+ return 26 + type.value + type.note * 2.6 + 24;+};const CHIP_ROW = 132;@@ -215,7 +230,7 @@ export function blockHeight(block: Block, width: number): number {case "meter":return TYPE_SIZE.xxl * 0.72 + GAP_SIZE.tight + METER_ROW;case "steps":- return STEP_ROW;+ return stepRow(block.points, width);case "chip":return Math.max(CHIP_ROW,diff --git a/remotion/src/style/pages.tsx b/remotion/src/style/pages.tsxindex d5a6705..cf2f387 100644--- a/remotion/src/style/pages.tsx+++ b/remotion/src/style/pages.tsx@@ -120,7 +120,7 @@ const Strip: React.FC<{ theme: Theme; seed: number; s: number; color?: string; stheme, seed, s, color, size = 48, children,}) => (<Paper theme={theme} seed={seed} color={color ?? theme.color.stripAlt} scale={s}>- <div style={{ ...fontOf(theme.type.label, size * s), color: theme.color.ink, padding: `${16 * s}px ${34 * s}px` }}>+ <div style={{ ...fontOf(theme.type.label, size * s * (theme.surface === "none" ? 1.35 : 1)), color: theme.color.ink, padding: `${16 * s}px ${34 * s}px` }}>{children}</div></Paper>diff --git a/remotion/src/style/reveal.tsx b/remotion/src/style/reveal.tsxindex cb6f12a..38869ac 100644--- a/remotion/src/style/reveal.tsx+++ b/remotion/src/style/reveal.tsx@@ -56,7 +56,14 @@ const mask = (entrance: Entrance, p: number) => {export const roleFrames = (theme: Theme, role: Role, text?: string) =>entranceFrames(theme.motion, theme.motion.roles[role], text);-const GLYPHS = "ABCDEFGHJKLMNPQRSTUVWXYZ0123456789#%&";+const UPPER = "ABCDEFGHJKLMNPRSTUVXYZ";+const LOWER = "abcdeghknoprsuvxz";+const DIGITS = "0123456789";++const scrambled = (char: string, n: number) => {+ const pool = /[A-Z]/.test(char) ? UPPER : /[a-z]/.test(char) ? LOWER : /[0-9]/.test(char) ? DIGITS : "";+ return pool ? pool[n % pool.length] : char;+};export const Reveal: React.FC<{theme: Theme;@@ -150,26 +157,39 @@ export const RevealText: React.FC<{let seen = 0;let cursor = 0;- const spans: { text: string; from: number; visible: boolean; fresh: boolean }[] = pieces.map((piece) => {+ const spans: { text: string; from: number; visible: boolean; fresh: boolean; cover?: string }[] = pieces.map((piece) => {const word = piece.trim().length > 0;const index = word ? seen++ : seen - 1;const from = cursor;cursor += piece.length;const settled = index < count;const shown = entrance === "scramble" ? f >= at : settled || (!word && index < count);- const body = entrance === "scramble" && !settled && word- ? Array.from(piece).map((_, i) => GLYPHS[((from + i) * 7 + seed * 13) % GLYPHS.length]).join("")- : piece;- return { text: body, from, visible: shown, fresh: entrance === "type" && word && index === count - 1 && count < total };+ const cover = entrance === "scramble" && !settled && word+ ? Array.from(piece).map((char, i) => scrambled(char, (from + i) * 7 + seed * 13)).join("")+ : undefined;+ return { text: piece, from, visible: shown, cover, fresh: entrance === "type" && word && index === count - 1 && count < total };});const render = (lo: number, hi: number) => spans.filter((span) => span.from >= lo && span.from < hi).map((span) => (- <span key={span.from} style={{- opacity: span.visible ? 1 : 0,- filter: span.fresh ? `blur(${3 * s}px)` : undefined,- }}>{span.text}</span>+ span.cover ? (+ <span key={span.from} style={{ opacity: span.visible ? 1 : 0 }}>+ {Array.from(span.text).map((char, i) => (+ <span key={i} style={{ position: "relative" }}>+ <span style={{ visibility: "hidden" }}>{char}</span>+ <span style={{ position: "absolute", left: "50%", top: 0, transform: "translateX(-50%)" }}>+ {Array.from(span.cover ?? "")[i]}+ </span>+ </span>+ ))}+ </span>+ ) : (+ <span key={span.from} style={{+ opacity: span.visible ? 1 : 0,+ filter: span.fresh ? `blur(${3 * s}px)` : undefined,+ }}>{span.text}</span>+ )));if (markStart < 0 || motion.textUnit === "word" && !spans.some((span) => span.from === markStart)) {diff --git a/tests/test_face_track_helpers.py b/tests/test_face_track_helpers.pyindex 4dda790..6513890 100644--- a/tests/test_face_track_helpers.py+++ b/tests/test_face_track_helpers.py@@ -367,6 +367,41 @@ class SeatsFromFramesTests(unittest.TestCase):self.assertIsNone(fth.seats_from_frames([[960]] * 50, self.WIDTH))+class FollowWithinRunsTests(unittest.TestCase):+ CROP = 1215+ WIDTH = 3840++ def _follow(self, runs, points):+ return fth.follow_within_runs(runs, points, self.CROP, self.WIDTH)++ def _at(self, keys, t):+ return [x for kt, x in keys if kt <= t][-1]++ def test_a_still_face_keeps_a_still_camera(self):+ points = [(i * 0.1, 1900.0 + (i % 3) * 20, 640.0) for i in range(100)]+ keys = self._follow([(0.0, 10.0, 1900.0)], points)+ self.assertEqual({x for _, x in keys}, {1900.0})++ def test_a_close_face_that_leans_is_followed_before_it_leaves_frame(self):+ points = [(i * 0.1, 1930.0, 650.0) for i in range(140)]+ points += [(14.0 + i * 0.1, 2300.0, 850.0) for i in range(50)]+ keys = self._follow([(0.0, 19.0, 1930.0)], points)+ half = self.CROP / 2+ self.assertLessEqual(abs(self._at(keys, 15.0) - 2300.0) + 425, half)++ def test_a_layout_change_is_a_cut_not_a_pan(self):+ points = [(i * 0.1, 1950.0, 640.0) for i in range(330)]+ points += [(33.0 + i * 0.1, 1000.0, 680.0) for i in range(15)]+ keys = self._follow([(0.0, 33.0, 1950.0), (33.0, 34.5, 1000.0)], points)+ self.assertEqual(self._at(keys, 32.999), 1950.0)+ self.assertEqual(self._at(keys, 33.0), 1000.0)++ def test_the_camera_never_leaves_the_source(self):+ points = [(i * 0.1, 50.0, 400.0) for i in range(30)]+ keys = self._follow([(0.0, 3.0, 50.0)], points)+ self.assertTrue(all(x >= self.CROP / 2 for _, x in keys))++class ReExportTests(unittest.TestCase):def test_video_processor_reexports_all_helpers(self):from services import video_processor as vpdiff --git a/tests/test_studio_format.py b/tests/test_studio_format.pyindex 03cc699..43b4ce3 100644--- a/tests/test_studio_format.py+++ b/tests/test_studio_format.py@@ -85,6 +85,21 @@ class StudioFormatTests(unittest.TestCase):self.assertEqual(cmd[cmd.index("--logo-scale") + 1], "1.0")+class StudioTranscriptTests(unittest.TestCase):+ def test_supplied_words_reach_the_render_script(self):+ cmd = _run_studio(_studio_args(transcript="/tmp/window-words.json"))+ self.assertEqual(cmd[cmd.index("--transcript") + 1], "/tmp/window-words.json")++ def test_no_transcript_sends_no_transcript_flag(self):+ self.assertNotIn("--transcript", _run_studio(_studio_args()))++ def test_the_flag_is_in_the_help_the_cloud_worker_reads(self):+ import subprocess+ script = os.path.join(os.path.dirname(cli_mod.__file__), "cli.py")+ shown = subprocess.run([sys.executable, script, "studio", "--help"], capture_output=True, text=True)+ self.assertIn("--transcript", shown.stdout)++class StudioStyleTests(unittest.TestCase):"""`--style` travels to clip_studio.py the same way `--brand` does."""