Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Fix framing, captions, scramble and steps found in the Advaith renders (#249)

Nika Siradze36cd57cc91ad
diff --git a/backend/cli.py b/backend/cli.py
index 0a2a121..886362a 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -501,6 +501,8 @@ def cmd_studio(args):
cmd += ["--language", args.language]
if getattr(args, "engine", None):
cmd += ["--engine", args.engine]
+ if getattr(args, "transcript", None):
+ cmd += ["--transcript", args.transcript]
env = os.environ.copy()
if getattr(args, "assemblyai_api_key", None):
env["ASSEMBLYAI_API_KEY"] = args.assemblyai_api_key
@@ -5012,6 +5014,7 @@ def main():
studio.add_argument("--paragraph", help="Find the fragment by matching this text in the transcript")
studio.add_argument("--language", help="Transcription language (e.g. es). Auto-detect if omitted.")
studio.add_argument("--engine", choices=["whisper-py", "whispercpp", "assemblyai"], help="Transcription engine")
+ studio.add_argument("--transcript", help="Word timings JSON for this video ({words:[...]} or a list); skips transcription")
studio.add_argument("--assemblyai-api-key", help="AssemblyAI API key for --engine assemblyai. Prefer ASSEMBLYAI_API_KEY; command-line secrets can appear in process listings.")
studio.add_argument("--caption-style", choices=["hormozi", "karaoke", "subtle", "branded", "outline"], default="hormozi")
studio.add_argument("--caption-position", choices=["auto", "upper", "center", "lower"], default="auto")
diff --git a/backend/services/face_track_helpers.py b/backend/services/face_track_helpers.py
index 4121f39..769cd7e 100644
--- a/backend/services/face_track_helpers.py
+++ b/backend/services/face_track_helpers.py
@@ -270,6 +270,38 @@ def longest_layout_run(detections: list, split: bool) -> float:
return best
+def follow_within_runs(
+ runs: list,
+ face_points: list,
+ crop_w: int,
+ video_width: int,
+ fast: float = 480.0,
+ slow: float = 140.0,
+) -> list:
+ half = crop_w / 2.0
+ lowest, highest = half, max(half, video_width - half)
+ keys = []
+ for start, end, center in runs:
+ cam = min(max(center, lowest), highest)
+ if keys:
+ keys.append((max(keys[-1][0], start - 0.001), keys[-1][1]))
+ keys.append((start, cam))
+ last = start
+ for t, cx, fw in face_points:
+ if not start <= t < end:
+ continue
+ room = max(24.0, (crop_w - fw) / 2.0 - crop_w * 0.04)
+ calm = min(crop_w * 0.22, room)
+ diff = cx - cam
+ if abs(diff) > calm:
+ speed = fast if abs(diff) > room else slow
+ cam += min(abs(diff), speed * max(t - last, 0.01)) * (1 if diff > 0 else -1)
+ cam = min(max(cam, lowest), highest)
+ last = t
+ keys.append((t, cam))
+ return keys
+
+
def clip_layout_is_mixed(
detections: list,
face_map: dict | None = None,
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 4c9075f..5f40794 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -44,6 +44,7 @@ from services.face_track_helpers import (
clamp_away_from_dead_zone as _clamp_away_from_dead_zone,
clip_layout_is_mixed as _clip_layout_is_mixed,
crop_center_keeping_faces_visible as _crop_center_keeping_faces_visible,
+ follow_within_runs as _follow_within_runs,
followed_face_cx_at as _followed_face_cx_at,
safe_default_center as _safe_default_center,
update_tripod_camera as _update_tripod_camera,
@@ -1590,6 +1591,7 @@ def _track_and_crop_inner(
# ── Collect per-frame face positions (speaker-aware) ──────
from statistics import median
face_points = [] # [(t, center_x), ...]
+ face_widths = []
for t, faces in detections:
if not faces:
continue
@@ -1603,6 +1605,7 @@ def _track_and_crop_inner(
else:
best = max(faces, key=lambda f: f["fw"])
face_points.append((t, float(best["cx"])))
+ face_widths.append(float(best.get("fw", 0)))
# ── Split into stable position runs ─────────────────────
# Each run = one locked camera position. Runs split on
@@ -1687,7 +1690,14 @@ def _track_and_crop_inner(
cuts = _source_cut_times(input_path)
snapped = _snap_runs_to_source_cuts(runs, cuts)
moved = sum(1 for a, b in zip(runs, snapped) if abs(a[1] - b[1]) > 0.001)
- x_expr = _run_step_crop_x_expr(snapped, _crop_for)
+ followed = _follow_within_runs(
+ snapped,
+ [(t, cx, fw) for (t, cx), fw in zip(face_points, face_widths)],
+ crop_w, width,
+ )
+ keys = _simplify_keyframes([(t, _crop_for(cx)) for t, cx in followed], tolerance=8)
+ x_expr = (_build_cam_expr(keys, duration, False)
+ or _run_step_crop_x_expr(snapped, _crop_for))
vf = (f"crop={crop_w}:{crop_h}:x='{x_expr}':y={crop_y},"
f"scale={target_w}:{target_h}")
log_event("crop", "chose=mixed-step-cuts", runs=len(runs),
diff --git a/remotion/src/CaptionedClip.tsx b/remotion/src/CaptionedClip.tsx
index cba700e..d60a0b2 100644
--- a/remotion/src/CaptionedClip.tsx
+++ b/remotion/src/CaptionedClip.tsx
@@ -56,6 +56,8 @@ export interface CaptionedClipProps {
theme?: ThemeInput | null;
}
+const ON_PAGE = 0.8;
+
export const CaptionedClip: React.FC<CaptionedClipProps> = ({
videoSrc,
words,
@@ -92,7 +94,6 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({
const nameCardSeconds = nameCard?.title ? (nameCard.seconds ?? 3) : 0;
const pastNameCard = frame / fps >= nameCardSeconds;
- const cardPlanned = Boolean(cards?.length);
const upNow = pastNameCard ? cardAt(cards ?? [], frame / fps) : null;
/*
* How far in the card is, rather than whether it is.
@@ -109,8 +110,7 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({
motion: upNow.motion ?? MOTION.card,
}).opacity
: 0;
- const restingShrink = cardPlanned ? 0.75 : 1;
- const captionShrink = restingShrink + (0.6 - restingShrink) * cardIn;
+ const captionShrink = 1 + (ON_PAGE - 1) * cardIn;
/*
* Captions used to drop toward the bottom edge while a card held the frame,
* on the reasoning that there is no chin down there to clear. There is no
@@ -149,7 +149,7 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({
*/
const settledStyle: CaptionStyle = {
...captionStyle,
- fontSize: style.fontSize * captionSize * 0.6,
+ fontSize: voice.fontSize * captionSize * ON_PAGE,
};
/*
@@ -222,7 +222,7 @@ export const CaptionedClip: React.FC<CaptionedClipProps> = ({
title={nameCard.title}
subtitle={nameCard.subtitle}
seconds={nameCardSeconds}
- bottom={(style.marginBottom + 150) * (height / 1920)}
+ bottom={captionZone(captionStyle) + 24}
/>
)}
{!theme && nameCard?.title && (
diff --git a/remotion/src/components/Scene.tsx b/remotion/src/components/Scene.tsx
index 2f6f1bb..8446ac6 100644
--- a/remotion/src/components/Scene.tsx
+++ b/remotion/src/components/Scene.tsx
@@ -6,7 +6,7 @@ import { FONT } from "../types";
import { context, muted, track } from "./brand";
import type { Brand } from "./brand";
import {
- emphasisRuns, GAP_SIZE, MEDIA_HEIGHT, MIN_FIT, SCENE_WIDTH, TYPE_SIZE, sceneHeight,
+ emphasisRuns, GAP_SIZE, MEDIA_HEIGHT, MIN_FIT, SCENE_WIDTH, TYPE_SIZE, sceneHeight, stepType,
sizeOf,
} from "../scene";
import type { Align, Block, Gap, Layout, Size, Tone } from "../scene";
@@ -278,6 +278,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {
if (block.type === "steps") {
const last = block.points.length - 1;
+ const type = stepType(block.points, paint.width / unit);
return (
<div style={{ display: "flex", alignItems: "center" }}>
{block.points.map((point, i) => (
@@ -287,9 +288,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {
style={{
flex: 1,
height: MARK.rule * unit,
- background: i === last
- ? `linear-gradient(90deg, ${context(brand.ink)}, ${accent})`
- : context(brand.ink),
+ background: i === last ? accent : context(brand.ink),
margin: `0 ${18 * unit}px`,
transform: `translateY(${-14 * unit}px)`,
}}
@@ -308,7 +307,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {
<div
style={{
fontFamily: paint.font,
- fontSize: TYPE_SIZE.md * unit,
+ fontSize: type.value * unit,
fontWeight: 700,
color: brand.ink,
marginTop: 12 * unit,
@@ -321,7 +320,7 @@ const Piece: React.FC<{ block: Block; paint: Paint }> = ({ block, paint }) => {
<div
style={{
fontFamily: paint.font,
- fontSize: TYPE_SIZE.xs * unit,
+ fontSize: type.note * unit,
color: muted(brand.ink),
marginTop: 6 * unit,
}}
diff --git a/remotion/src/scene.ts b/remotion/src/scene.ts
index 4afaa11..28898a3 100644
--- a/remotion/src/scene.ts
+++ b/remotion/src/scene.ts
@@ -168,7 +168,22 @@ const lines = (text: string, size: number, width: number, advance = ADVANCE) =>
const BAR_ROW = 26 + 8 + 34 * 1.3;
-const STEP_ROW = 26 + 58 + 34 * 1.3 + 24;
+const CHAR_WIDTH = 0.58;
+
+export const stepType = (points: StepPoint[], width = SCENE_WIDTH) => {
+ const slot = (width - Math.max(0, points.length - 1) * 120) / Math.max(1, points.length);
+ const fits = (text: string | undefined, size: number) => !text || text.length * size * CHAR_WIDTH <= slot;
+ const value = [TYPE_SIZE.xl, TYPE_SIZE.lg, TYPE_SIZE.md]
+ .find((size) => points.every((point) => fits(point.value, size))) ?? TYPE_SIZE.md;
+ const note = [TYPE_SIZE.sm, TYPE_SIZE.xs]
+ .find((size) => points.every((point) => fits(point.note, size))) ?? TYPE_SIZE.xs;
+ return { value, note };
+};
+
+const stepRow = (points: StepPoint[], width: number) => {
+ const type = stepType(points, width);
+ return 26 + type.value + type.note * 2.6 + 24;
+};
const CHIP_ROW = 132;
@@ -215,7 +230,7 @@ export function blockHeight(block: Block, width: number): number {
case "meter":
return TYPE_SIZE.xxl * 0.72 + GAP_SIZE.tight + METER_ROW;
case "steps":
- return STEP_ROW;
+ return stepRow(block.points, width);
case "chip":
return Math.max(
CHIP_ROW,
diff --git a/remotion/src/style/pages.tsx b/remotion/src/style/pages.tsx
index d5a6705..cf2f387 100644
--- a/remotion/src/style/pages.tsx
+++ b/remotion/src/style/pages.tsx
@@ -120,7 +120,7 @@ const Strip: React.FC<{ theme: Theme; seed: number; s: number; color?: string; s
theme, seed, s, color, size = 48, children,
}) => (
<Paper theme={theme} seed={seed} color={color ?? theme.color.stripAlt} scale={s}>
- <div style={{ ...fontOf(theme.type.label, size * s), color: theme.color.ink, padding: `${16 * s}px ${34 * s}px` }}>
+ <div style={{ ...fontOf(theme.type.label, size * s * (theme.surface === "none" ? 1.35 : 1)), color: theme.color.ink, padding: `${16 * s}px ${34 * s}px` }}>
{children}
</div>
</Paper>
diff --git a/remotion/src/style/reveal.tsx b/remotion/src/style/reveal.tsx
index cb6f12a..38869ac 100644
--- a/remotion/src/style/reveal.tsx
+++ b/remotion/src/style/reveal.tsx
@@ -56,7 +56,14 @@ const mask = (entrance: Entrance, p: number) => {
export const roleFrames = (theme: Theme, role: Role, text?: string) =>
entranceFrames(theme.motion, theme.motion.roles[role], text);
-const GLYPHS = "ABCDEFGHJKLMNPQRSTUVWXYZ0123456789#%&";
+const UPPER = "ABCDEFGHJKLMNPRSTUVXYZ";
+const LOWER = "abcdeghknoprsuvxz";
+const DIGITS = "0123456789";
+
+const scrambled = (char: string, n: number) => {
+ const pool = /[A-Z]/.test(char) ? UPPER : /[a-z]/.test(char) ? LOWER : /[0-9]/.test(char) ? DIGITS : "";
+ return pool ? pool[n % pool.length] : char;
+};
export const Reveal: React.FC<{
theme: Theme;
@@ -150,26 +157,39 @@ export const RevealText: React.FC<{
let seen = 0;
let cursor = 0;
- const spans: { text: string; from: number; visible: boolean; fresh: boolean }[] = pieces.map((piece) => {
+ const spans: { text: string; from: number; visible: boolean; fresh: boolean; cover?: string }[] = pieces.map((piece) => {
const word = piece.trim().length > 0;
const index = word ? seen++ : seen - 1;
const from = cursor;
cursor += piece.length;
const settled = index < count;
const shown = entrance === "scramble" ? f >= at : settled || (!word && index < count);
- const body = entrance === "scramble" && !settled && word
- ? Array.from(piece).map((_, i) => GLYPHS[((from + i) * 7 + seed * 13) % GLYPHS.length]).join("")
- : piece;
- return { text: body, from, visible: shown, fresh: entrance === "type" && word && index === count - 1 && count < total };
+ const cover = entrance === "scramble" && !settled && word
+ ? Array.from(piece).map((char, i) => scrambled(char, (from + i) * 7 + seed * 13)).join("")
+ : undefined;
+ return { text: piece, from, visible: shown, cover, fresh: entrance === "type" && word && index === count - 1 && count < total };
});
const render = (lo: number, hi: number) => spans
.filter((span) => span.from >= lo && span.from < hi)
.map((span) => (
- <span key={span.from} style={{
- opacity: span.visible ? 1 : 0,
- filter: span.fresh ? `blur(${3 * s}px)` : undefined,
- }}>{span.text}</span>
+ span.cover ? (
+ <span key={span.from} style={{ opacity: span.visible ? 1 : 0 }}>
+ {Array.from(span.text).map((char, i) => (
+ <span key={i} style={{ position: "relative" }}>
+ <span style={{ visibility: "hidden" }}>{char}</span>
+ <span style={{ position: "absolute", left: "50%", top: 0, transform: "translateX(-50%)" }}>
+ {Array.from(span.cover ?? "")[i]}
+ </span>
+ </span>
+ ))}
+ </span>
+ ) : (
+ <span key={span.from} style={{
+ opacity: span.visible ? 1 : 0,
+ filter: span.fresh ? `blur(${3 * s}px)` : undefined,
+ }}>{span.text}</span>
+ )
));
if (markStart < 0 || motion.textUnit === "word" && !spans.some((span) => span.from === markStart)) {
diff --git a/tests/test_face_track_helpers.py b/tests/test_face_track_helpers.py
index 4dda790..6513890 100644
--- a/tests/test_face_track_helpers.py
+++ b/tests/test_face_track_helpers.py
@@ -367,6 +367,41 @@ class SeatsFromFramesTests(unittest.TestCase):
self.assertIsNone(fth.seats_from_frames([[960]] * 50, self.WIDTH))
+class FollowWithinRunsTests(unittest.TestCase):
+ CROP = 1215
+ WIDTH = 3840
+
+ def _follow(self, runs, points):
+ return fth.follow_within_runs(runs, points, self.CROP, self.WIDTH)
+
+ def _at(self, keys, t):
+ return [x for kt, x in keys if kt <= t][-1]
+
+ def test_a_still_face_keeps_a_still_camera(self):
+ points = [(i * 0.1, 1900.0 + (i % 3) * 20, 640.0) for i in range(100)]
+ keys = self._follow([(0.0, 10.0, 1900.0)], points)
+ self.assertEqual({x for _, x in keys}, {1900.0})
+
+ def test_a_close_face_that_leans_is_followed_before_it_leaves_frame(self):
+ points = [(i * 0.1, 1930.0, 650.0) for i in range(140)]
+ points += [(14.0 + i * 0.1, 2300.0, 850.0) for i in range(50)]
+ keys = self._follow([(0.0, 19.0, 1930.0)], points)
+ half = self.CROP / 2
+ self.assertLessEqual(abs(self._at(keys, 15.0) - 2300.0) + 425, half)
+
+ def test_a_layout_change_is_a_cut_not_a_pan(self):
+ points = [(i * 0.1, 1950.0, 640.0) for i in range(330)]
+ points += [(33.0 + i * 0.1, 1000.0, 680.0) for i in range(15)]
+ keys = self._follow([(0.0, 33.0, 1950.0), (33.0, 34.5, 1000.0)], points)
+ self.assertEqual(self._at(keys, 32.999), 1950.0)
+ self.assertEqual(self._at(keys, 33.0), 1000.0)
+
+ def test_the_camera_never_leaves_the_source(self):
+ points = [(i * 0.1, 50.0, 400.0) for i in range(30)]
+ keys = self._follow([(0.0, 3.0, 50.0)], points)
+ self.assertTrue(all(x >= self.CROP / 2 for _, x in keys))
+
+
class ReExportTests(unittest.TestCase):
def test_video_processor_reexports_all_helpers(self):
from services import video_processor as vp
diff --git a/tests/test_studio_format.py b/tests/test_studio_format.py
index 03cc699..43b4ce3 100644
--- a/tests/test_studio_format.py
+++ b/tests/test_studio_format.py
@@ -85,6 +85,21 @@ class StudioFormatTests(unittest.TestCase):
self.assertEqual(cmd[cmd.index("--logo-scale") + 1], "1.0")
+class StudioTranscriptTests(unittest.TestCase):
+ def test_supplied_words_reach_the_render_script(self):
+ cmd = _run_studio(_studio_args(transcript="/tmp/window-words.json"))
+ self.assertEqual(cmd[cmd.index("--transcript") + 1], "/tmp/window-words.json")
+
+ def test_no_transcript_sends_no_transcript_flag(self):
+ self.assertNotIn("--transcript", _run_studio(_studio_args()))
+
+ def test_the_flag_is_in_the_help_the_cloud_worker_reads(self):
+ import subprocess
+ script = os.path.join(os.path.dirname(cli_mod.__file__), "cli.py")
+ shown = subprocess.run([sys.executable, script, "studio", "--help"], capture_output=True, text=True)
+ self.assertIn("--transcript", shown.stdout)
+
+
class StudioStyleTests(unittest.TestCase):
"""`--style` travels to clip_studio.py the same way `--brand` does."""