Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Pool many videos in Find Highlights and rank the best across all of them (#57)
Nika Siradzef52467aed673
diff --git a/backend/main.py b/backend/main.pyindex 0e78ec7..f84dc04 100644--- a/backend/main.py+++ b/backend/main.py@@ -365,9 +365,11 @@ def handle_manage_reel(task_id: str, params: dict):Actions: new (detect + build), list, show, edit (adjust one moment + rebuild),build, delete."""+ import hashlibfrom dataclasses import asdictfrom services.reel import (- ReelSession, seed_session, edit_moment, build_reel, list_sessions, delete_session,+ ReelSession, seed_session, seed_session_pooled, edit_moment, build_reel,+ list_sessions, delete_session,)action = params.get("action", "show")@@ -380,9 +382,11 @@ def handle_manage_reel(task_id: str, params: dict):d["clip_exists"] = os.path.exists(d["clip_path"])moments.append(d)reel_file = reel or os.path.join(session.out_dir, "highlights_reel.mp4")+ sources = sorted({m.source for m in session.moments if m.source}) or [session.source]return {"session_id": session.session_id,"source": session.source,+ "sources": sources,"format": session.format,"out_dir": session.out_dir,"reel_path": reel_file if os.path.exists(reel_file) else None,@@ -392,20 +396,26 @@ def handle_manage_reel(task_id: str, params: dict):try:if action == "new":from services.transcript_packer import compute_cache_hash, load_cached_transcript_for_video- video = params["video_path"]- sid = compute_cache_hash(video)- out_dir = params.get("out_dir") or os.path.join(os.getcwd(), f"reel_{sid[:8]}")- cached = load_cached_transcript_for_video(video)- words = cached.get("words") if cached else None- session = seed_session(- sid, video, out_dir, profile=params.get("profile", "auto"),+ video_paths = params.get("video_paths") or []+ common = dict(+ profile=params.get("profile", "auto"),format=params.get("format", "horizontal"),top_n=int(params.get("top_n", 10)),min_dur=float(params.get("min_dur", 15.0)),max_dur=float(params.get("max_dur", 60.0)),- words=words,progress_callback=lambda p, m: emit_progress(task_id, "detecting", p, m),)+ if video_paths:+ sid = hashlib.sha256("\n".join(sorted(video_paths)).encode()).hexdigest()[:16]+ out_dir = params.get("out_dir") or os.path.join(os.getcwd(), f"reel_{sid[:8]}")+ session = seed_session_pooled(sid, video_paths, out_dir, **common)+ else:+ video = params["video_path"]+ sid = compute_cache_hash(video)+ out_dir = params.get("out_dir") or os.path.join(os.getcwd(), f"reel_{sid[:8]}")+ cached = load_cached_transcript_for_video(video)+ words = cached.get("words") if cached else None+ session = seed_session(sid, video, out_dir, words=words, **common)reel = build_reel(session, progress_callback=lambda p, m: emit_progress(task_id, "building", p, m))emit_result(task_id, "success", data=payload(session, reel))elif action == "list":diff --git a/backend/services/reel.py b/backend/services/reel.pyindex 9914f3f..5cfd978 100644--- a/backend/services/reel.py+++ b/backend/services/reel.py@@ -35,6 +35,7 @@ class Moment:end: floatwhy: str = "energy_peak"text: str = ""+ source: str = "" # empty falls back to the session source (single-video reels)enabled: bool = Truedirty: bool = True # needs a re-cut before the next build@@ -110,6 +111,40 @@ def seed_session(return session+def seed_session_pooled(+ session_id: str,+ sources: list[str],+ out_dir: str,+ profile: str = "auto",+ format: str = "horizontal",+ top_n: int = 10,+ min_dur: float = 15.0,+ max_dur: float = 60.0,+ progress_callback: Optional[Callable] = None,+) -> ReelSession:+ """Detect across many videos, rank globally, and persist one editable session."""+ from services.saliency import detect_highlights_pooled++ clips = detect_highlights_pooled(+ sources, profile_name=profile, top_n=top_n, min_dur=min_dur, max_dur=max_dur,+ progress_callback=progress_callback,+ )+ moments = [+ Moment(+ start=round(c["start_second"], 1),+ end=round(c["end_second"], 1),+ why=c.get("reasons", ["energy_peak"])[0],+ source=c.get("source_file", ""),+ )+ for c in clips+ ]+ session = ReelSession(+ session_id, sources[0], profile, out_dir, format=get_format(format).name, moments=moments+ )+ session.save()+ return session++_EDITS = {"longer": lambda m, s: setattr(m, "end", round(m.end + s, 1)),"shorter": lambda m, s: setattr(m, "end", round(max(m.start + 1, m.end - s), 1)),@@ -190,7 +225,7 @@ def build_reel(session: ReelSession, progress_callback: Optional[Callable] = Nonif m.dirty or not os.path.exists(clip):if progress_callback:progress_callback(int(n / len(active) * 90), f"cutting moment {i}")- clip = _cut(session.source, session.out_dir, i, m, session.format)+ clip = _cut(m.source or session.source, session.out_dir, i, m, session.format)m.dirty = Falsefiles.append(clip)session.save()@@ -232,6 +267,7 @@ def list_sessions() -> list[dict]:"format": data.get("format", "horizontal"),"moment_count": len(moments),"enabled_count": sum(1 for m in moments if m.get("enabled", True)),+ "source_count": len({m.get("source") for m in moments if m.get("source")}) or 1,"reel_path": reel if os.path.exists(reel) else None,"mtime": os.path.getmtime(path),})diff --git a/src/server.ts b/src/server.tsindex c78c35c..0f5feec 100644--- a/src/server.ts+++ b/src/server.ts@@ -2077,7 +2077,7 @@ export function createServer(): McpServer {// =============================================server.tool("manage_reel",- "Create and iterate on a highlights reel. Detection runs once with action 'new'; after that, edit individual moments fast (longer/shorter/earlier/later/shift/drop/toggle) and rebuild without re-detecting. Actions: 'new' (video_path, profile, format, top_n, min_dur, max_dur), 'list', 'show' (session_id), 'edit' (session_id, index, op, seconds), 'build' (session_id), 'delete' (session_id).",+ "Create and iterate on a highlights reel. Detection runs once with action 'new'; after that, edit individual moments fast (longer/shorter/earlier/later/shift/drop/toggle) and rebuild without re-detecting. Pass video_paths (a list) to pool many videos and rank the best moments across all of them. Actions: 'new' (video_path or video_paths, profile, format, top_n, min_dur, max_dur), 'list', 'show' (session_id), 'edit' (session_id, index, op, seconds), 'build' (session_id), 'delete' (session_id).",{action: z.enum(["new", "list", "show", "edit", "build", "delete"])@@ -2086,6 +2086,10 @@ export function createServer(): McpServer {.string().optional().describe("For 'new': path to the source video"),+ video_paths: z+ .array(z.string())+ .optional()+ .describe("For 'new': many source videos to pool and rank the best moments across all of them"),session_id: z.string().optional()diff --git a/src/ui/client/HighlightsPage.tsx b/src/ui/client/HighlightsPage.tsxindex 206da83..a537991 100644--- a/src/ui/client/HighlightsPage.tsx+++ b/src/ui/client/HighlightsPage.tsx@@ -7,6 +7,7 @@ interface Moment {end: number;why: string;text: string;+ source?: string;enabled: boolean;dirty: boolean;clip_path?: string;@@ -16,6 +17,7 @@ interface Moment {interface HighlightsResp {session_id: string;source: string;+ sources?: string[];format: string;out_dir: string;reel_path: string | null;@@ -29,6 +31,7 @@ interface SessionSummary {format: string;moment_count: number;enabled_count: number;+ source_count?: number;reel_path: string | null;}@@ -189,7 +192,8 @@ const round1 = (n: number) => Math.round(n * 10) / 10;export default function HighlightsPage() {const [sessions, setSessions] = useState<SessionSummary[]>([]);- const [videoPath, setVideoPath] = useState("");+ const [videoPaths, setVideoPaths] = useState<string[]>([]);+ const [pathDraft, setPathDraft] = useState("");const [browsing, setBrowsing] = useState(false);const [dragOver, setDragOver] = useState(false);const [format, setFormat] = useState<Format>("horizontal");@@ -234,11 +238,16 @@ export default function HighlightsPage() {}}+ const addPath = (p: string) =>+ setVideoPaths((prev) => (p && !prev.includes(p) ? [...prev, p] : prev));+ const removePath = (i: number) => setVideoPaths((prev) => prev.filter((_, n) => n !== i));+async function browse() {setBrowsing(true);try {- const d = await api<{ file_path?: string }>("/browse-file");- if (d.file_path) setVideoPath(d.file_path);+ const d = await api<{ file_path?: string; file_paths?: string[] }>("/browse-file?multiple=1");+ const picked = d.file_paths?.length ? d.file_paths : d.file_path ? [d.file_path] : [];+ setVideoPaths((prev) => [...prev, ...picked.filter((p) => !prev.includes(p))]);} catch {/* dialog cancelled */} finally {@@ -246,16 +255,18 @@ export default function HighlightsPage() {}}- async function uploadDropped(file: File) {+ async function uploadDropped(files: File[]) {setBrowsing(true);- setMsg(`Uploading ${file.name}…`);try {- const fd = new FormData();- fd.append("file", file);- const res = await fetch("/api/upload", { method: "POST", body: fd });- const d = (await res.json()) as { file_path?: string; error?: string };- if (d.file_path) setVideoPath(d.file_path);- else setMsg(d.error || "Upload failed");+ for (const file of files) {+ setMsg(`Uploading ${file.name}…`);+ const fd = new FormData();+ fd.append("file", file);+ const res = await fetch("/api/upload", { method: "POST", body: fd });+ const d = (await res.json()) as { file_path?: string; error?: string };+ if (d.file_path) addPath(d.file_path);+ else setMsg(d.error || "Upload failed");+ }} catch {setMsg("Upload failed");} finally {@@ -267,15 +278,26 @@ export default function HighlightsPage() {function onDrop(e: React.DragEvent) {e.preventDefault();setDragOver(false);- const file = e.dataTransfer.files?.[0];- if (file) uploadDropped(file);+ const files = Array.from(e.dataTransfer.files || []);+ if (files.length) uploadDropped(files);+ }++ function commitDraft() {+ const p = pathDraft.trim();+ if (p) addPath(p);+ setPathDraft("");}- const detect = () =>+ const detect = () => {+ const seed =+ videoPaths.length === 1 ? { video_path: videoPaths[0] } : { video_paths: videoPaths };call(- { action: "new", video_path: videoPath, profile: "auto", format, top_n: topN, min_dur: minDur, max_dur: maxDur },- "Finding the best moments (one-time, ~2 min)…",+ { action: "new", ...seed, profile: "auto", format, top_n: topN, min_dur: minDur, max_dur: maxDur },+ videoPaths.length > 1+ ? `Finding the best moments across ${videoPaths.length} videos (one-time)…`+ : "Finding the best moments (one-time, ~2 min)…",);+ };const open = (id: string) => call({ action: "show", session_id: id }, "Loading…");@@ -312,8 +334,9 @@ export default function HighlightsPage() {}const enabled = session?.moments.filter((m) => m.enabled).length ?? 0;- const streamSrc = session ? `/api/stream-source?path=${encodeURIComponent(session.source)}` : "";const active = session?.moments[selected];+ const activeSource = active?.source || session?.source || "";+ const streamSrc = activeSource ? `/api/stream-source?path=${encodeURIComponent(activeSource)}` : "";return (<div className="app">@@ -324,37 +347,43 @@ export default function HighlightsPage() {<div className="section card"><div className="card-title" style={{ marginBottom: 14 }}>Find highlights</div>- {!videoPath ? (- <div- className={`drop-zone ${dragOver ? "drag-over" : ""}`}- style={{ cursor: browsing ? "default" : "pointer" }}- onClick={browsing ? undefined : browse}- onDragOver={(e) => { e.preventDefault(); setDragOver(true); }}- onDragLeave={() => setDragOver(false)}- onDrop={onDrop}- >- {browsing ? (- <div style={{ display: "flex", alignItems: "center", justifyContent: "center", gap: 10 }}>- <div className="spinner sm" /> <span style={{ fontSize: 13, fontWeight: 600 }}>Working…</span>+ <div+ className={`drop-zone ${dragOver ? "drag-over" : ""}`}+ style={{ cursor: browsing ? "default" : "pointer" }}+ onClick={browsing ? undefined : browse}+ onDragOver={(e) => { e.preventDefault(); setDragOver(true); }}+ onDragLeave={() => setDragOver(false)}+ onDrop={onDrop}+ >+ {browsing ? (+ <div style={{ display: "flex", alignItems: "center", justifyContent: "center", gap: 10 }}>+ <div className="spinner sm" /> <span style={{ fontSize: 13, fontWeight: 600 }}>Working…</span>+ </div>+ ) : (+ <div className="label"><strong>Browse</strong> or drop video files here</div>+ )}+ </div>++ {videoPaths.length > 0 && (+ <div style={{ display: "flex", flexDirection: "column", gap: 6, marginTop: 8 }}>+ {videoPaths.map((p, i) => (+ <div key={p} className="file-badge fade-in">+ <div className="dot" />+ <div className="name">{basename(p)}</div>+ <button className="btn btn-ghost btn-sm" onClick={() => removePath(i)} style={{ padding: "4px 10px", fontSize: 11 }}>+ Remove+ </button></div>- ) : (- <div className="label"><strong>Browse</strong> or drop a video file here</div>- )}- </div>- ) : (- <div className="file-badge fade-in">- <div className="dot" />- <div className="name">{basename(videoPath)}</div>- <button className="btn btn-ghost btn-sm" onClick={() => setVideoPath("")} style={{ padding: "4px 10px", fontSize: 11 }}>- Clear- </button>+ ))}</div>)}<inputtype="text"- placeholder="…or paste a local video path"- value={videoPath}- onChange={(e) => setVideoPath(e.target.value)}+ placeholder="…or paste a local video path, press Enter to add"+ value={pathDraft}+ onChange={(e) => setPathDraft(e.target.value)}+ onKeyDown={(e) => { if (e.key === "Enter") { e.preventDefault(); commitDraft(); } }}+ onBlur={commitDraft}style={{ width: "100%", marginTop: 8, fontFamily: "var(--font-mono)", fontSize: 12 }}/>@@ -378,8 +407,12 @@ export default function HighlightsPage() {</div><div style={{ display: "flex", justifyContent: "flex-end", marginTop: 16 }}>- <button className="btn btn-primary" disabled={busy || !videoPath.trim()} onClick={detect}>- {busy && msg?.startsWith("Finding") ? "Finding…" : "Find highlights"}+ <button className="btn btn-primary" disabled={busy || videoPaths.length === 0} onClick={detect}>+ {busy && msg?.startsWith("Finding")+ ? "Finding…"+ : videoPaths.length > 1+ ? `Find best across ${videoPaths.length} videos`+ : "Find highlights"}</button></div></div>@@ -411,6 +444,9 @@ export default function HighlightsPage() {<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 12 }}><span className="section-label" style={{ margin: 0 }}>Moment {selected + 1}</span><span className="pill pill-blue">{active.why}</span>+ {(session.sources?.length ?? 1) > 1 && active.source && (+ <span className="hint" title={active.source}>{basename(active.source)}</span>+ )}<span className="spacer" style={{ flex: 1 }} />{active.clip_exists && active.clip_path && (<a className="btn btn-ghost btn-sm" href={download(active.clip_path)} style={{ textDecoration: "none", display: "inline-flex", alignItems: "center", gap: 6 }}>@@ -451,6 +487,11 @@ export default function HighlightsPage() {<span className="hint" style={{ width: 20, textAlign: "right" }}>{idx + 1}</span><strong style={{ fontVariantNumeric: "tabular-nums", minWidth: 96 }}>{mmss(m.start)}–{mmss(m.end)}</strong><span className="pill pill-blue">{m.why}</span>+ {(session.sources?.length ?? 1) > 1 && m.source && (+ <span className="hint" title={m.source} style={{ minWidth: 0, overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>+ {basename(m.source)}+ </span>+ )}<span className="hint" style={{ marginLeft: "auto", fontVariantNumeric: "tabular-nums" }}>{Math.round(m.end - m.start)}s</span>{m.clip_exists && m.clip_path && (<a className="btn btn-ghost btn-sm" href={download(m.clip_path)} title="Download clip"@@ -463,7 +504,7 @@ export default function HighlightsPage() {</div></div>) : sessions.length === 0 ? (- <div className="empty-state">No highlights yet. Drop in a video above to find its best moments.</div>+ <div className="empty-state">No highlights yet. Drop in one or more videos above to find their best moments.</div>) : (<><div className="section-label" style={{ marginBottom: 12 }}>Saved</div>@@ -476,7 +517,11 @@ export default function HighlightsPage() {onClick={() => !busy && open(s.session_id)}><div style={{ minWidth: 0, flex: 1 }}>- <div className="clip-card-title" style={{ marginBottom: 4 }}>{basename(s.source) || s.session_id}</div>+ <div className="clip-card-title" style={{ marginBottom: 4 }}>+ {(s.source_count ?? 1) > 1+ ? `${s.source_count} videos`+ : basename(s.source) || s.session_id}+ </div><div className="meta" style={{ gap: 8 }}><span className="pill pill-blue">{s.profile}</span><span className="hint">{FORMAT_LABEL[s.format as Format] || s.format}</span>diff --git a/src/ui/web-server.ts b/src/ui/web-server.tsindex a0a9e65..e016083 100644--- a/src/ui/web-server.ts+++ b/src/ui/web-server.ts@@ -23,7 +23,7 @@ import {import { mkdir, readdir, unlink } from "fs/promises";import path from "path";import { join, dirname, basename, extname, resolve } from "path";-import { execSync, spawn } from "child_process";+import { execSync, execFileSync, spawn } from "child_process";import { lookup } from "dns/promises";import { isIP } from "net";import { tmpdir } from "os";@@ -679,46 +679,56 @@ app.post("/api/select-file", (req, res) => {/*** GET /api/browse-file — Open native OS file dialog and return the selected path*/-app.get("/api/browse-file", (_req, res) => {+app.get("/api/browse-file", (req, res) => {+ const multiple = req.query.multiple === "1" || req.query.multiple === "true";try {- let filePath: string;+ let raw: string;if (process.platform === "darwin") {- const script = `osascript -e 'POSIX path of (choose file of type {"mp4","mov","mkv","webm","mp3","wav","m4a"})'`;- filePath = execSync(script, {- encoding: "utf-8",- timeout: 120_000,- }).trim();+ const mult = multiple ? " with multiple selections allowed" : "";+ raw = execFileSync(+ "osascript",+ [+ "-e", `set sel to choose file of type {"mp4","mov","mkv","webm","mp3","wav","m4a"}${mult}`,+ "-e", 'if class of sel is not list then set sel to {sel}',+ "-e", "set out to \"\"",+ "-e", "repeat with f in sel",+ "-e", "set out to out & POSIX path of f & linefeed",+ "-e", "end repeat",+ "-e", "return out",+ ],+ { encoding: "utf-8", timeout: 120_000 },+ );} else if (process.platform === "win32") {// EncodedCommand (UTF-16LE base64) sidesteps cmd→PowerShell quoting; -STA is required by WinForms dialogs.const ps = ["Add-Type -AssemblyName System.Windows.Forms;","$f = New-Object System.Windows.Forms.OpenFileDialog;","$f.Filter = 'Media files|*.mp4;*.mov;*.mkv;*.webm;*.mp3;*.wav;*.m4a';",- "if ($f.ShowDialog() -eq [System.Windows.Forms.DialogResult]::OK) { Write-Output $f.FileName }",+ multiple ? "$f.Multiselect = $true;" : "",+ "if ($f.ShowDialog() -eq [System.Windows.Forms.DialogResult]::OK) { $f.FileNames -join [Environment]::NewLine }",].join(" ");const encoded = Buffer.from(ps, "utf16le").toString("base64");- filePath = execSync(`powershell -NoProfile -STA -EncodedCommand ${encoded}`, {+ raw = execSync(`powershell -NoProfile -STA -EncodedCommand ${encoded}`, {encoding: "utf-8",timeout: 120_000,- }).trim();+ });} else {- // Linux fallback- filePath = execSync(- `zenity --file-selection --file-filter="Media files|*.mp4 *.mov *.mkv *.webm *.mp3 *.wav *.m4a"`,- { encoding: "utf-8", timeout: 120_000 },- ).trim();+ const args = ["--file-selection", "--file-filter=Media files|*.mp4 *.mov *.mkv *.webm *.mp3 *.wav *.m4a"];+ if (multiple) args.push("--multiple", "--separator=\n");+ raw = execFileSync("zenity", args, { encoding: "utf-8", timeout: 120_000 });}- if (!filePath || !existsSync(filePath)) {+ const paths = raw.split("\n").map((p) => p.trim()).filter((p) => p && existsSync(p));+ if (paths.length === 0) {res.json({ error: "cancelled" });return;}-- const stat = statSync(filePath);- registerSourcePath(filePath);+ for (const p of paths) registerSourcePath(p);+ const stat = statSync(paths[0]);res.json({- file_path: filePath,- filename: basename(filePath),+ file_path: paths[0],+ file_paths: paths,+ filename: basename(paths[0]),size_mb: Math.round((stat.size / (1024 * 1024)) * 100) / 100,});} catch {@@ -1570,6 +1580,9 @@ app.post("/api/reel", async (req, res) => {const result = await executor.execute("manage_reel", req.body || {});const data = (result.data || {}) as any;if (typeof data.source === "string") registerSourcePath(data.source);+ if (Array.isArray(data.sources)) {+ for (const s of data.sources) if (typeof s === "string") registerSourcePath(s);+ }if (typeof data.reel_path === "string") registerSourcePath(data.reel_path);if (Array.isArray(data.moments)) {for (const m of data.moments) {