Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Hold the guest through an answer, cut remote shows, and give the cloud editor its engine (#230)

Nika Siradze6fda6f8cd95b
diff --git a/CLAUDE.md b/CLAUDE.md
index 0ffafcd..0ed9397 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -61,7 +61,7 @@ All 27 tools registered by the MCP server.
| Tool | What it does |
|------|-------------|
-| `manage_multicam` | Map camera and mic files to people, sync them by audio, auto-cut to the speaker, render or export to Premiere / FCPXML |
+| `manage_multicam` | Map camera, mic and call-recording files to people, sync them by audio, auto-cut to the speaker, render, or export a studio shoot to Premiere / FCPXML |
**Content and configuration**
diff --git a/README.md b/README.md
index 79b22af..bb2a7b8 100644
--- a/README.md
+++ b/README.md
@@ -74,8 +74,9 @@ Clips land in `podcli-clips/` in the directory you ran it from, so each show kee
**Editing the full episode**
- Multicam edit: point at one folder of camera and mic files, confirm who each file belongs to, and podcli syncs them by audio and corrects clock drift between devices
-- Automatic camera cuts that follow whoever is talking, with wide shots for crosstalk and long monologues
-- Render the episode as an MP4 with one WAV per person, or export a Premiere XML or FCPXML timeline that points at your original files
+- Automatic camera cuts that follow whoever is talking, with wide shots for crosstalk and a guest's answer held on the guest
+- Remote shows too: one file per person becomes a split screen, and a single gallery recording is split into a camera per tile. Questions stay on the split and the guest goes full frame on long answers
+- Render the episode as an MP4 with one WAV per person. Studio shoots can also go to Premiere or Final Cut as a timeline that points at your original files
- Same edit from the studio (Multicam edit), the CLI (`podcli multicam <folder>`), or an agent (`manage_multicam`). Re-running reuses the sync and the cut, so only the step you changed runs again
**Finding the moments**
diff --git a/backend/cli.py b/backend/cli.py
index 0482658..cbbaedb 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -657,9 +657,12 @@ def _fmt_time(seconds: float) -> str:
def _multicam_table(session) -> None:
+ from services import multicam as mc
+
names = {p.id: p.name for p in session.people}
names.update({"wide": "Everyone (wide)", "": "Shared room mic"})
- print(f"\n {session.name} · people: {', '.join(p.name for p in session.people)} · session {session.session_id}")
+ people = ", ".join(f"{p.name} ({p.role})" for p in session.people)
+ print(f"\n {session.name} · people: {people} · session {session.session_id}")
print(f" {'#':>2} {'File':<28} {'Role':<7} {'Who':<22} {'Length':>8} {'Format':<10} Sync")
for i, s in enumerate(session.sources, 1):
if s.role == "ignore":
@@ -668,14 +671,14 @@ def _multicam_table(session) -> None:
who = " / ".join(names.get(p, p) for p in s.channel_people)
else:
who = names.get(s.person, s.person)
- size = f"{s.width}x{s.height}" if s.kind == "video" else f"{s.audio_channels}ch"
+ size = "layout" if s.virtual else f"{s.width}x{s.height}" if s.kind == "video" else f"{s.audio_channels}ch"
sync = {"reference": "reference", "ok": "synced", "rough": "rough sync", "manual": "set by hand",
- "failed": "not synced"}.get(s.sync.get("status", ""), "")
+ "assumed": "starts with the others", "failed": "not synced"}.get(s.sync.get("status", ""), "")
if s.synced:
drift = s.sync.get("drift_ppm") or 0
sync = f"{s.offset:+.3f}s {sync}" + (f", drift {drift:+.0f} ppm" if abs(drift) >= 5 else "")
flag = " (guessed)" if s.guessed and s.role != "ignore" else ""
- name = os.path.basename(s.path)
+ name = mc.source_label(session, s)
print(f" {i:>2} {name[:28]:<28} {s.role:<7} {who[:22]:<22} {_fmt_time(s.duration):>8} {size:<10} {sync}{flag}")
@@ -717,7 +720,8 @@ def _apply_multicam_fixes(session, fixes: list[str]):
if target.isdigit() and 1 <= int(target) <= len(session.sources):
source = session.sources[int(target) - 1]
else:
- matches = [s for s in session.sources if os.path.basename(s.path).lower().startswith(target.lower())]
+ matches = [s for s in session.sources if not s.virtual
+ and os.path.basename(s.path).lower().startswith(target.lower())]
if len(matches) != 1:
raise ValueError(f"--set {fix!r} matches {len(matches)} files. Use the file's number from the table.")
source = matches[0]
@@ -738,6 +742,7 @@ def _apply_multicam_fixes(session, fixes: list[str]):
def _review_multicam_interactively(session):
"""Let the user fix guessed roles before the slow steps. Returns None to quit."""
import questionary
+ from services import multicam as mc
from services.multicam import rename_people, update_mapping
while True:
@@ -763,7 +768,7 @@ def _review_multicam_interactively(session):
continue
source = questionary.select(
"Which file?",
- choices=[questionary.Choice(os.path.basename(s.path), value=s) for s in session.sources],
+ choices=[questionary.Choice(mc.source_label(session, s), value=s) for s in session.sources],
).ask()
if source is None:
continue
@@ -786,22 +791,43 @@ def _review_multicam_interactively(session):
def _open_multicam_session(target: str, people):
from services import multicam as mc
- if not os.path.isdir(target):
+ if os.path.isdir(target):
+ report, done = _multicam_progress("Reading files")
+ session = mc.new_session(folder=target, people=people, progress_callback=report)
+ done()
+ else:
session = mc.MulticamSession.load(target)
- return mc.rename_people(session, people) if people else session
- report, done = _multicam_progress("Reading files")
- session = mc.new_session(folder=target, people=people, progress_callback=report)
- done()
+ # Reopening keeps the saved names; --people on a re-run renames them.
+ if people and people != [p.name for p in session.people]:
+ session = mc.rename_people(session, people)
return session
def cmd_multicam(args):
- """Edit a full multicam episode: map every file, sync, auto-cut, then render or export."""
+ """Edit a full multicam episode: map every file, sync, auto-cut, then render or export.
+
+ With --json, everything human goes to stderr and stdout carries exactly one
+ JSON object: the edit (plus activity and transcript when asked), a deletion
+ receipt, or {"error": ...}. Scripts and the cloud worker rely on that.
+ """
+ import contextlib
from services import multicam as mc
+ out = sys.stdout
+
+ def fail(message: str, code: int):
+ sys.stdout.flush()
+ if args.json:
+ print(json.dumps({"error": message}), file=out)
+ print(f" ✗ {message}", file=sys.stderr)
+ sys.exit(code)
+
target = _clean_path(args.target) if args.target else ""
if target == "list":
sessions = mc.list_sessions()
+ if args.json:
+ print(json.dumps({"sessions": sessions}))
+ return
if not sessions:
print(" No multicam edits yet. Start one with: podcli multicam <folder>")
for s in sessions:
@@ -809,79 +835,145 @@ def cmd_multicam(args):
print(f" {s['session_id']} {s['name'][:32]:<32} {s['sources']} files {state}")
return
if not target:
+ if args.json:
+ fail("Give an episode folder, a session id, or list.", 2)
print(" Usage: podcli multicam <folder | session id | list> [options]\n"
" Example: podcli multicam ~/Podcasts/ep12 --people \"Nika, Ana\"")
sys.exit(2)
try:
- if args.delete:
- session = (mc.find_session(mc.collect_files(target)) if os.path.isdir(target)
- else mc.MulticamSession.load(target))
- if not session:
- raise ValueError(f"No multicam edit for {target}")
- mc.delete_session(session.session_id)
- print(f" ✓ Deleted multicam edit {session.session_id}. Your recordings are untouched.")
- return
-
- people = [n.strip() for n in (args.people or "").split(",") if n.strip()] or None
- session = _apply_multicam_fixes(_open_multicam_session(target, people), args.set or [])
-
- guessed = any(s.guessed and s.role != "ignore" for s in session.sources)
- if guessed and sys.stdin.isatty() and sys.stdout.isatty() and not args.yes:
- session = _review_multicam_interactively(session)
- if session is None:
- return
- else:
- _multicam_table(session)
- if guessed:
- print(" Using the guessed roles above. Fix any with --set FILE=ROLE:NAME.")
-
- if args.resync or mc.needs_sync(session):
- report, done = _multicam_progress("Syncing")
- session = mc.sync_session(session, force=args.resync, progress_callback=report)
- done()
- _multicam_table(session)
- failed = [s for s in session.sources if s.role != "ignore" and not s.synced]
- if failed:
- names = ", ".join(os.path.basename(s.path) for s in failed)
- print(f"\n ✗ Couldn't sync {names}. Ignore it with --set {os.path.basename(failed[0].path)}=ignore, "
- "or set its offset in the studio (podcli ui).")
- sys.exit(1)
+ with contextlib.redirect_stdout(sys.stderr) if args.json else contextlib.nullcontext():
+ result = _run_multicam(args, mc, target)
+ data = result
+ if isinstance(result, mc.MulticamSession):
+ data = mc.payload(result)
+ if args.json and args.activity:
+ data["activity"] = mc.activity(result)
+ if args.json and args.transcript:
+ data["transcript"] = mc.transcript(result, model_size=args.model, engine=args.engine)
+ if args.json and data is not None:
+ print(json.dumps(data, allow_nan=False), file=out)
+ except Exception as e:
+ fail(str(e) or type(e).__name__, 1)
+
+
+def _load_json(path: str, what: str):
+ try:
+ with open(_clean_path(path), encoding="utf-8") as f:
+ return json.load(f)
+ except (OSError, json.JSONDecodeError) as e:
+ raise ValueError(f"Can't read the {what} from {path}: {e}")
+
+
+def _load_cuts(path: str) -> list:
+ data = _load_json(path, "cut")
+ return data.get("cuts", data) if isinstance(data, dict) else data
+
+
+def _run_multicam(args, mc, target: str):
+ if args.delete:
+ session = (mc.find_session(mc.collect_files(target)) if os.path.isdir(target)
+ else mc.MulticamSession.load(target))
+ if not session:
+ raise ValueError(f"No multicam edit for {target}")
+ mc.delete_session(session.session_id)
+ print(f" ✓ Deleted multicam edit {session.session_id}. Your recordings are untouched.")
+ return {"deleted": True, "session_id": session.session_id}
+
+ people = [n.strip() for n in (args.people or "").split(",") if n.strip()] or None
+ session = _open_multicam_session(target, people)
+ if args.state:
+ session = mc.apply_state(session, _load_json(args.state, "saved edit"))
+ session = _apply_multicam_fixes(session, args.set or [])
+ if args.guests is not None:
+ guests = {n.strip().lower() for n in args.guests.split(",") if n.strip()}
+ unknown = guests - {p.name.lower() for p in session.people}
+ if unknown:
+ raise ValueError(f"No person named {sorted(unknown)[0]!r}. People: {', '.join(p.name for p in session.people)}")
+ session = mc.update_mapping(session, {"people": [
+ {"id": p.id, "name": p.name, "role": "guest" if p.name.lower() in guests else "host"} for p in session.people
+ ]})
+
+ guessed = any(s.guessed and s.role != "ignore" for s in session.sources)
+ if guessed and sys.stdin.isatty() and sys.stdout.isatty() and not (args.yes or args.json):
+ session = _review_multicam_interactively(session)
+ if session is None:
+ return None
+ else:
+ _multicam_table(session)
+ if guessed:
+ print(" Using the guessed roles above. Fix any with --set FILE=ROLE:NAME.")
- cut_edits = {
- **({"range_start": args.start} if args.start is not None else {}),
- **({"range_end": args.end} if args.end is not None else {}),
- **({"cut_settings": {k: v for k, v in (("min_shot", args.min_shot), ("max_shot", args.max_shot)) if v is not None}}
- if args.min_shot is not None or args.max_shot is not None else {}),
- **({"look": args.look} if args.look else {}),
- }
- if cut_edits:
- session = mc.update_mapping(session, cut_edits)
- if not session.cuts:
- report, done = _multicam_progress("Cutting")
- session = mc.plan_session(session, progress_callback=report)
- done()
- stats = mc.cut_stats(session)
- names = {s.id: os.path.basename(s.path) for s in session.sources}
- share = ", ".join(f"{names[k]} {v:.0%}" for k, v in sorted(stats["share"].items(), key=lambda kv: -kv[1]))
- print(f"\n {stats['shots']} shots over {_fmt_time(stats['duration'])}, average {stats['average_shot']:.1f}s ({share})")
-
- for fmt in (["premiere", "fcpxml"] if args.export == "all" else [args.export] if args.export else []):
+ if args.resync or mc.needs_sync(session):
+ report, done = _multicam_progress("Syncing")
+ session = mc.sync_session(session, force=args.resync, progress_callback=report)
+ done()
+ _multicam_table(session)
+ if mc.needs_sync(session):
+ failed = [s for s in session.sources if not s.virtual and s.role != "ignore" and not s.synced]
+ numbers = {s.id: i for i, s in enumerate(session.sources, 1)}
+ names = ", ".join(f"{numbers[s.id]} ({mc.source_label(session, s)})" for s in failed)
+ raise ValueError(f"Couldn't sync {names}. Ignore it with --set {numbers[failed[0].id]}=ignore, "
+ "or set its offset in the studio (podcli ui).")
+
+ cut_edits = {
+ **({"range_start": args.start} if args.start is not None else {}),
+ **({"range_end": args.end} if args.end is not None else {}),
+ **({"cut_settings": {k: v for k, v in (("style", args.style), ("min_shot", args.min_shot),
+ ("max_shot", args.max_shot)) if v is not None}}
+ if any(v is not None for v in (args.style, args.min_shot, args.max_shot)) else {}),
+ **({"look": args.look} if args.look else {}),
+ }
+ if cut_edits:
+ session = mc.update_mapping(session, cut_edits)
+ if args.cuts:
+ session = mc.set_cuts(session, _load_cuts(args.cuts))
+ elif not session.cuts:
+ report, done = _multicam_progress("Cutting")
+ session = mc.plan_session(session, progress_callback=report)
+ done()
+ if args.removals:
+ data = _load_json(args.removals, "removals")
+ session = mc.set_removals(session, data.get("removals", []) if isinstance(data, dict) else data)
+ stats = mc.cut_stats(session)
+ names = {s.id: mc.source_label(session, s) for s in session.sources}
+ share = ", ".join(f"{names[k]} {v:.0%}" for k, v in sorted(stats["share"].items(), key=lambda kv: -kv[1]))
+ print(f"\n {stats['shots']} shots over {_fmt_time(stats['duration'])}, average {stats['average_shot']:.1f}s, "
+ f"{mc.resolved_style(session)} style ({share})")
+ if session.removals:
+ removed = sum(r["end"] - r["start"] for r in session.removals)
+ print(f" {len(session.removals)} stretches removed, {_fmt_time(removed)} in all")
+ if args.activity and not args.json:
+ spoken = mc.activity(session)["people"]
+ print(" Talk time: " + ", ".join(
+ f"{p.name} {_fmt_time(sum(b - a for a, b in spoken.get(p.id, [])))}" for p in session.people))
+
+ for fmt in (["premiere", "fcpxml"] if args.export == "all" else [args.export] if args.export else []):
+ try:
print(f" ✓ {mc.export_xml(session, fmt)}")
- if not args.no_render:
- report, done = _multicam_progress("Rendering")
- outputs = mc.render_session(session, stems=not args.no_stems, progress_callback=report)
- done()
- for path in [outputs["video"], *(outputs.get("stems") or [])]:
- print(f" ✓ {path}")
- print(f"\n Change anything and re-run: podcli multicam {session.session_id} [options]")
- video = session.outputs.get("video")
- if video and os.path.exists(video):
- print(f" Make clips from it: podcli process \"{video}\"")
- except (ValueError, OSError, RuntimeError, ImportError, TypeError) as e:
- sys.stdout.flush()
- print(f" ✗ {e}", file=sys.stderr)
- sys.exit(1)
+ except ValueError as e:
+ print(f" ! Skipped the {fmt} export: {e}", file=sys.stderr)
+ if args.transcript:
+ report, done = _multicam_progress("Transcribing")
+ words = mc.transcript(session, model_size=args.model, engine=args.engine, progress_callback=report)["words"]
+ done()
+ print(f" ✓ Transcribed {len(words)} words")
+ if args.preview:
+ report, done = _multicam_progress("Preview")
+ session = mc.build_preview(session, progress_callback=report)
+ done()
+ print(f" ✓ Preview proxies and mix in {os.path.dirname(session.preview['audio'])}")
+ if not args.no_render:
+ report, done = _multicam_progress("Rendering")
+ outputs = mc.render_session(session, stems=not args.no_stems, progress_callback=report)
+ done()
+ for path in [outputs["video"], *(outputs.get("stems") or [])]:
+ print(f" ✓ {path}")
+ print(f"\n Change anything and re-run: podcli multicam {session.session_id} [options]")
+ video = session.outputs.get("video")
+ if video and os.path.exists(video):
+ print(f" Make clips from it: podcli process \"{video}\"")
+ return session
def cmd_process(args):
@@ -4834,14 +4926,20 @@ def main():
)
mc_p.add_argument("target", nargs="?", help="Episode folder, a session id to resume, or 'list'")
mc_p.add_argument("--people", help="Comma-separated speaker names (default: Host, Guest)")
+ mc_p.add_argument("--guests", help="Comma-separated names of the guests, whose answers are held on their camera "
+ "(default: the last person)")
mc_p.add_argument("--set", action="append", metavar="FILE=ROLE[:NAME]",
help="Fix a file's role: camera:NAME, camera:wide, mic:NAME, mic:NAME,NAME (stereo L/R), mic:room, or ignore. "
- "FILE is its number in the table or the start of its name. Repeatable.")
+ "FILE is its number in the table or the start of its name; tiles and split screens go by "
+ "number. Repeatable.")
mc_p.add_argument("--start", type=float, help="Episode start on the synced timeline, in seconds")
mc_p.add_argument("--end", type=float, help="Episode end on the synced timeline, in seconds")
- mc_p.add_argument("--min-shot", type=float, dest="min_shot", help="Shortest shot in seconds (default 2)")
+ mc_p.add_argument("--style", choices=["auto", "studio", "remote"],
+ help="How to cut: studio (everyone's camera) or remote (split screen, guest full frame on long "
+ "answers). Default auto: remote for call recordings")
+ mc_p.add_argument("--min-shot", type=float, dest="min_shot", help="Shortest shot in seconds (studio 2, remote 4)")
mc_p.add_argument("--max-shot", type=float, dest="max_shot",
- help="Cut to the wide shot after this many seconds on one speaker; 0 never (default 30)")
+ help="Cut to the wide shot after this many seconds on one host; 0 never (studio 30, remote 0)")
mc_p.add_argument("--look", choices=["none", "natural", "warm", "contrast"], help="Color look for the render")
mc_p.add_argument("--export", choices=["premiere", "fcpxml", "all"],
help="Also write an editor timeline that points at the original files")
@@ -4851,6 +4949,22 @@ def main():
help="Sync every file again, including offsets you set by hand")
mc_p.add_argument("-y", "--yes", action="store_true", help="Don't stop to review guessed roles")
mc_p.add_argument("--delete", action="store_true", help="Delete this multicam edit (source files are untouched)")
+ mc_p.add_argument("--state", metavar="FILE",
+ help="Restore a prepared edit (a saved --json result) so these files render without syncing again")
+ mc_p.add_argument("--cuts", metavar="FILE",
+ help="Use this cut instead of the automatic one: a JSON list of {start, end, source_id}, back to back")
+ mc_p.add_argument("--removals", metavar="FILE",
+ help="Cut these stretches out of the episode: a JSON list of {start, end} on the timeline; [] restores everything")
+ mc_p.add_argument("--preview", action="store_true",
+ help="Also build small playback proxies of each camera and a mic mix on the timeline")
+ mc_p.add_argument("--transcript", action="store_true",
+ help="Transcribe the episode with each word credited to whoever's mic was speaking")
+ mc_p.add_argument("--model", default="base", help="Whisper model for --transcript (default base)")
+ mc_p.add_argument("--engine", choices=["whisper-py", "whispercpp", "assemblyai"],
+ help="Transcription engine for --transcript (default: the one podcli is set up with)")
+ mc_p.add_argument("--activity", action="store_true", help="Report who speaks when (talk time, or spans with --json)")
+ mc_p.add_argument("--json", action="store_true",
+ help="Print the edit as one JSON object on stdout; progress and messages go to stderr")
# ── studio ──
studio = sub.add_parser("studio", help="Cut a fragment + add Remotion intro/outro (follow-us) bookends")
@@ -5166,7 +5280,13 @@ def main():
args = parser.parse_args()
- _auto_migrate_cli(args)
+ if getattr(args, "json", False):
+ # --json promises stdout holds only the JSON result.
+ import contextlib
+ with contextlib.redirect_stdout(sys.stderr):
+ _auto_migrate_cli(args)
+ else:
+ _auto_migrate_cli(args)
if getattr(args, "show_help", False) and args.command is None:
print_help()
diff --git a/backend/main.py b/backend/main.py
index 49d96a7..ee36b5a 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -963,14 +963,16 @@ def handle_render_silence_removed(task_id: str, params: dict):
emit_result(task_id, "error", error=str(e))
-MULTICAM_MAP_KEYS = ("people", "sources", "range_start", "range_end", "cut_settings", "speaker_map", "look")
+MULTICAM_MAP_KEYS = ("people", "sources", "range_start", "range_end", "cut_settings", "speaker_map", "look", "removals")
def handle_manage_multicam(task_id: str, params: dict):
"""Multicam podcast editing: map sources, sync, plan cuts, render, export.
Actions: new (folder or files, people), list, show, map, sync, plan, cut
- (swap one shot's camera), previews, render, export (premiere|fcpxml), delete.
+ (swap one shot's camera), set_cuts (replace the whole cut), activity (who
+ speaks when), previews (stills), preview (playback proxies), render,
+ export (premiere|fcpxml), delete.
sync, plan and render apply any mapping fields sent with them first.
"""
from services import multicam as mc
@@ -1004,6 +1006,9 @@ def handle_manage_multicam(task_id: str, params: dict):
return
session = mc.MulticamSession.load(session_id)
+ if action == "render" and any(k in params for k in MULTICAM_MAP_KEYS if k != "look"):
+ # A mapping change can drop the cut, and a render needs one: map, then plan, then render.
+ raise ValueError("render takes only look and stems. Change the mapping with 'map', then 'plan' again.")
if action in ("map", "sync", "plan", "render") and any(k in params for k in MULTICAM_MAP_KEYS):
session = mc.update_mapping(session, params)
data: dict = {}
@@ -1013,6 +1018,12 @@ def handle_manage_multicam(task_id: str, params: dict):
session = mc.sync_session(session, force=bool(params.get("force")), progress_callback=progress("syncing"))
elif action == "plan":
session = mc.plan_session(session, progress_callback=progress("planning"))
+ elif action == "preview":
+ session = mc.build_preview(session, progress_callback=progress("preview"))
+ elif action == "activity":
+ data["activity"] = mc.activity(session)
+ elif action == "set_cuts":
+ session = mc.set_cuts(session, params.get("cuts"))
elif action == "cut":
index, source_id = params.get("index"), params.get("source_id")
if not isinstance(index, int) or not isinstance(source_id, str):
@@ -1021,14 +1032,21 @@ def handle_manage_multicam(task_id: str, params: dict):
elif action == "previews":
data["previews"] = mc.previews(session, looks=bool(params.get("looks")), at=params.get("at"))
elif action == "render":
- mc.render_session(session, stems=params.get("stems", True), progress_callback=progress("rendering"))
+ stems = params.get("stems", True)
+ if not isinstance(stems, bool):
+ raise ValueError("stems is true or false")
+ mc.render_session(session, stems=stems, progress_callback=progress("rendering"))
elif action == "export":
data["export_path"] = mc.export_xml(session, params.get("format", "premiere"))
else:
raise ValueError(f"Unknown multicam action {action!r}")
emit_result(task_id, "success", data={**mc.payload(session), **data})
- except (IndexError, ValueError, OSError, RuntimeError, ImportError, TypeError, AttributeError) as e:
- emit_result(task_id, "error", error=str(e))
+ except KeyError as e:
+ emit_result(task_id, "error", error=f"Missing field {e.args[0]!r}")
+ except Exception as e:
+ # Callers get one sentence; the full trace goes to the log, never to a client.
+ print(traceback.format_exc(), file=sys.stderr, flush=True)
+ emit_result(task_id, "error", error=str(e) or type(e).__name__)
def handle_run_integration_tool(task_id: str, params: dict):
diff --git a/backend/services/integrations/_shared/multicam_xml.py b/backend/services/integrations/_shared/multicam_xml.py
index 229df2a..1d0689d 100644
--- a/backend/services/integrations/_shared/multicam_xml.py
+++ b/backend/services/integrations/_shared/multicam_xml.py
@@ -6,9 +6,10 @@ Resolve. Both reference the original camera and mic files, so nothing is
re-encoded: V1 carries the camera cuts, and each person's mic sits on its own
audio track.
-Clock drift can't be expressed as a clip property in either format. When a mic
-drifts, its track is split at every camera cut and each piece gets its own
-in-point, which keeps it within a millisecond or two of picture.
+Both follow the same render plan as the MP4, so removed stretches drop out of
+picture and mics at the same frames. Clock drift can't be expressed as a clip
+property in either format, so a drifting mic is split at every shot and each
+piece gets its own in-point, which keeps it within a millisecond of picture.
"""
from __future__ import annotations
@@ -36,22 +37,33 @@ def _text(parent: ET.Element, tag: str, value) -> ET.Element:
return el
-def _audio_pieces(session, source, start: float, end: float, fps: float) -> list[tuple[int, int, float]]:
- """(timeline_frame_in, timeline_frame_out, source_seconds_at_in) for one mic across the episode."""
- lo = max(start, source.timeline_start())
- hi = min(end, source.timeline_end())
- if hi <= lo:
- return []
- bounds = [lo, hi]
- if abs(source.speed - 1.0) > 1e-6:
- bounds = [lo] + [c["start"] for c in session.cuts if lo < c["start"] < hi] + [hi]
- pieces = []
- for a, b in zip(bounds, bounds[1:]):
- f0 = round((a - start) * fps)
- f1 = round((b - start) * fps)
- if f1 > f0:
- pieces.append((f0, f1, source.source_time(start + f0 / fps)))
- return pieces
+def _audio_pieces(source, plan, fps: float) -> list[tuple[int, int, float]]:
+ """(output_frame_in, output_frame_out, source_seconds_at_in) for one mic across the episode.
+
+ Follows the render plan, so removed stretches drop out of the mics exactly
+ where they drop out of the picture. Pieces that continue each other on the
+ timeline are joined; a drifting mic is left split at every shot, so each
+ piece gets its own in-point and stays within a millisecond of picture.
+ """
+ pieces: list[list] = []
+ joinable = abs(source.speed - 1.0) <= 1e-6
+ last_end = None
+ for p in plan:
+ a = p.tl_start
+ b = a + p.frames / fps
+ lo, hi = max(a, source.timeline_start()), min(b, source.timeline_end())
+ if hi <= lo:
+ continue
+ f0 = p.out_frame + round((lo - a) * fps)
+ f1 = p.out_frame + round((hi - a) * fps)
+ if f1 <= f0:
+ continue
+ if joinable and pieces and pieces[-1][1] == f0 and last_end is not None and abs(last_end - lo) < 1e-3:
+ pieces[-1][1] = f1
+ else:
+ pieces.append([f0, f1, source.source_time(p.tl_start + (f0 - p.out_frame) / fps)])
+ last_end = hi
+ return [tuple(x) for x in pieces]
def _label(session, source, channel: int, person: str) -> str:
@@ -80,13 +92,10 @@ def _mic_tracks(session, *, split_stereo: bool = False) -> list[tuple[object, in
def write_xmeml(session, out_path: Path, *, width: int, height: int, fps: float) -> None:
- from services.multicam import shot_frames
+ from services.multicam import render_plan
- start = session.cuts[0]["start"]
- end = session.cuts[-1]["end"]
- frames = shot_frames(session, fps)
- shots = [s for s in frames if s[0]]
- total = max(f0 + n for _, f0, n in frames)
+ plan = render_plan(session, fps)
+ total = sum(p.frames for p in plan)
root = ET.Element("xmeml", {"version": "4"})
seq = ET.SubElement(root, "sequence", {"id": "sequence-1"})
@@ -147,16 +156,18 @@ def write_xmeml(session, out_path: Path, *, width: int, height: int, fps: float)
_text(st, "mediatype", "audio")
_text(st, "trackindex", max(1, audio_channel + 1))
- for source_id, f0, n in shots:
- cam = session.source(source_id)
- clipitem(vtrack, cam, f0, f0 + n, cam.source_time(start + f0 / fps), name=os.path.basename(cam.path))
+ for p in plan:
+ if not p.source_id:
+ continue
+ cam = session.source(p.source_id)
+ clipitem(vtrack, cam, p.out_frame, p.out_frame + p.frames, cam.source_time(p.tl_start), name=os.path.basename(cam.path))
audio = ET.SubElement(media, "audio")
_text(audio, "numOutputChannels", 2)
for source, channel, person in _mic_tracks(session, split_stereo=True):
track = ET.SubElement(audio, "track")
label = _label(session, source, channel, person)
- for f0, f1, src_seconds in _audio_pieces(session, source, start, end, fps):
+ for f0, f1, src_seconds in _audio_pieces(source, plan, fps):
clipitem(track, source, f0, f1, src_seconds, name=label, audio_channel=channel)
_text(track, "enabled", "TRUE")
@@ -170,12 +181,10 @@ def write_xmeml(session, out_path: Path, *, width: int, height: int, fps: float)
def write_fcpxml(session, out_path: Path, *, width: int, height: int, fps: float) -> None:
- from services.multicam import shot_frames
+ from services.multicam import render_plan
- start = session.cuts[0]["start"]
- end = session.cuts[-1]["end"]
- frames = shot_frames(session, fps)
- total = max(f0 + n for _, f0, n in frames)
+ plan = render_plan(session, fps)
+ total = sum(p.frames for p in plan)
def t(frame_count: int) -> str:
return fx.rational_time(frame_count, fps)
@@ -193,7 +202,7 @@ def write_fcpxml(session, out_path: Path, *, width: int, height: int, fps: float
resources = [fx.make_format(fmt_id, fps, width, height)]
asset_ids: dict[str, str] = {}
formats: dict[str, str] = {}
- used = {s for s, _, _ in frames if s} | {s.id for s, _, _ in _mic_tracks(session)}
+ used = {p.source_id for p in plan if p.source_id} | {s.id for s, _, _ in _mic_tracks(session)}
for i, source in enumerate(s for s in session.sources if s.id in used):
aid = f"a{i + 1}"
asset_ids[source.id] = aid
@@ -240,22 +249,22 @@ def write_fcpxml(session, out_path: Path, *, width: int, height: int, fps: float
# lanes, so every clip is positioned against the same zero.
gap = ET.SubElement(spine, "gap", {"name": "Episode", "offset": "0s", "start": "0s", "duration": t(total)})
- for source_id, f0, n in frames:
- if not source_id:
+ for p in plan:
+ if not p.source_id:
continue
- cam = session.source(source_id)
+ cam = session.source(p.source_id)
ET.SubElement(gap, "asset-clip", {
- "ref": asset_ids[source_id],
+ "ref": asset_ids[p.source_id],
"lane": "1",
- "offset": t(f0),
- "start": src_t(cam, cam.source_time(start + f0 / fps)),
- "duration": t(n),
+ "offset": t(p.out_frame),
+ "start": src_t(cam, cam.source_time(p.tl_start)),
+ "duration": t(p.frames),
"name": os.path.basename(cam.path),
"srcEnable": "video",
})
for lane, (source, channel, person) in enumerate(_mic_tracks(session), start=1):
- for f0, f1, src_seconds in _audio_pieces(session, source, start, end, fps):
+ for f0, f1, src_seconds in _audio_pieces(source, plan, fps):
clip = ET.SubElement(gap, "asset-clip", {
"ref": asset_ids[source.id],
"lane": str(-lane),
diff --git a/backend/services/multicam.py b/backend/services/multicam.py
index 52256d9..655dcc9 100644
--- a/backend/services/multicam.py
+++ b/backend/services/multicam.py
@@ -14,6 +14,7 @@ from __future__ import annotations
import hashlib
import json
+import math
import os
import re
import shutil
@@ -46,7 +47,33 @@ LOOKS = {
"contrast": "curves=preset=medium_contrast,eq=saturation=1.1",
}
-DEFAULT_CUT = {"min_shot": 2.0, "max_shot": 30.0, "wide_insert": 4.0}
+# How a show is cut. Studio: everyone has a camera, cut to whoever talks.
+# Remote (measured on Deeptech Decoded's call recordings): questions and short
+# answers sit on the split screen, a guest goes full frame 4 s into any answer
+# of 8 s or more and stays there, and hosts never get a solo shot.
+STYLES = {
+ "studio": {"min_shot": 2.0, "max_shot": 30.0, "wide_insert": 4.0, "backchannel": 1.2, "hold_guest": True,
+ "host_solo": True, "guest_min": 0.0, "guest_delay": 0.0},
+ "remote": {"min_shot": 4.0, "max_shot": 0.0, "wide_insert": 4.0, "backchannel": 1.5, "hold_guest": True,
+ "host_solo": False, "guest_min": 8.0, "guest_delay": 4.0},
+}
+CUT_LIMITS = {
+ "min_shot": (0.5, 10.0), "max_shot": (0.0, 600.0), "wide_insert": (1.0, 15.0), "backchannel": (0.0, 5.0),
+ "guest_min": (0.0, 60.0), "guest_delay": (0.0, 30.0),
+}
+
+
+def resolved_style(session: "MulticamSession") -> str:
+ """The cut style in force: a call layout (panes or a split screen) cuts like a remote show."""
+ style = session.cut_settings.get("style", "auto")
+ if style in STYLES:
+ return style
+ return "remote" if any(c.virtual for c in session.cameras()) else "studio"
+
+
+def effective_cut(session: "MulticamSession") -> dict:
+ tweaks = {k: v for k, v in session.cut_settings.items() if k in STYLES["studio"]}
+ return {**STYLES[resolved_style(session)], **tweaks}
def _emit(callback: ProgressCallback, percent: float, message: str) -> None:
@@ -70,6 +97,7 @@ def _work_dir(session_id: str) -> Path:
class Person:
id: str
name: str
+ role: str = "host" # "host" | "guest": a guest's answers are held on their camera
@dataclass
@@ -93,11 +121,20 @@ class Source:
offset: Optional[float] = None # timeline seconds where source time 0 sits
speed: float = 1.0 # timeline seconds per source second (clock drift)
sync: dict = field(default_factory=dict)
+ # Virtual cameras need no file of their own: a pane cropped out of a call
+ # recording (parent + crop), or a split screen of several people (members).
+ parent: str = ""
+ crop: list[float] = field(default_factory=list) # [x, y, w, h] as fractions of the parent's frame
+ members: list[str] = field(default_factory=list) # camera ids laid side by side, left to right
@property
def synced(self) -> bool:
return self.offset is not None
+ @property
+ def virtual(self) -> bool:
+ return bool(self.parent or self.members)
+
def timeline_start(self) -> float:
return float(self.offset or 0.0)
@@ -119,19 +156,21 @@ class MulticamSession:
range_start: Optional[float] = None
range_end: Optional[float] = None
look: str = "none"
- cut_settings: dict = field(default_factory=lambda: dict(DEFAULT_CUT))
+ cut_settings: dict = field(default_factory=lambda: {"style": "auto"}) # the style, plus any tweaks to it
cuts: list[dict] = field(default_factory=list)
speaker_map: dict = field(default_factory=dict)
activity_key: str = ""
outputs: dict = field(default_factory=dict)
skipped: list[str] = field(default_factory=list)
+ preview: dict = field(default_factory=dict)
+ removals: list[dict] = field(default_factory=list)
def path(self) -> Path:
return _sessions_dir() / f"{self.session_id}.json"
def save(self) -> None:
tmp = self.path().with_suffix(".json.tmp")
- tmp.write_text(json.dumps(asdict(self), indent=2), encoding="utf-8")
+ tmp.write_text(json.dumps(asdict(self), indent=2, allow_nan=False), encoding="utf-8")
os.replace(tmp, self.path())
@classmethod
@@ -159,9 +198,21 @@ class MulticamSession:
return [s for s in self.sources if s.role == "camera" and s.kind == "video"]
def person_mics(self) -> list[tuple[Source, int, str]]:
- """(source, channel or -1 for a mono mixdown, person) for each personal mic feed."""
+ """(source, channel or -1 for a mono mixdown, person) for each personal mic feed.
+
+ In a remote recording each person's own file carries their own mic, so
+ a person camera with sound counts as that person's mic when no
+ dedicated mic is mapped to them.
+ """
feeds = []
people = set(self.person_ids())
+ if any(s.members and s.role != "ignore" for s in self.sources):
+ mic_people = {s.person for s in self.sources if s.role == "mic"} | {
+ p for s in self.sources if s.role == "mic" for p in s.channel_people}
+ feeds += [
+ (s, -1, s.person) for s in self.sources
+ if s.role == "camera" and not s.virtual and s.has_audio and s.person in people - mic_people
+ ]
for s in self.sources:
if s.role != "mic":
continue
@@ -184,6 +235,20 @@ class MulticamSession:
MAX_SOURCES = 40
MAX_SCANNED_ENTRIES = 20_000
+MAX_SECONDS = 24 * 3600
+MAX_REMOVALS = 20_000
+MAX_SHOTS = 50_000
+
+
+def _number(value, what: str, lo: float, hi: float) -> float:
+ """A finite number in [lo, hi], or a ValueError naming the field."""
+ try:
+ x = float(value)
+ except (TypeError, ValueError):
+ raise ValueError(f"{what} must be a number") from None
+ if not math.isfinite(x) or not lo <= x <= hi:
+ raise ValueError(f"{what} must be between {lo:g} and {hi:g}")
+ return x
def scan_folder(folder: str) -> list[str]:
@@ -251,7 +316,9 @@ def probe_source(path: str) -> Source:
fps = 30.0
sample_rate = int(audio.get("sample_rate") or 0) if audio else 0
return Source(
- id=hashlib.sha1(os.path.abspath(path).encode()).hexdigest()[:10],
+ # Built from name and size so the same recording gets the same id
+ # on another machine and a saved edit or cut list still points at it.
+ id=hashlib.sha1(f"{os.path.basename(path)}:{os.path.getsize(path)}".encode()).hexdigest()[:10],
path=os.path.abspath(path),
kind="video" if video else "audio",
duration=round(duration, 3),
@@ -341,6 +408,184 @@ def guess_roles(session: MulticamSession) -> None:
s.person = free.pop(0) if free else ""
+# ---------------------------------------------------------------------------
+# Call layouts: panes inside one recording, or a split screen of several
+# ---------------------------------------------------------------------------
+
+def detect_panes(source: Source, samples: int = 8) -> list[list[float]]:
+ """Tiles of a video-call gallery recording, as [x, y, w, h] fractions, left to right.
+
+ Call apps separate tiles with a gutter (a strip of flat colour) or at least
+ a hard straight edge. Both show up as columns or rows that look the same
+ in every sampled frame; a single camera's picture has no such line. Empty
+ when the recording isn't a gallery.
+ """
+ width = 320
+ frames = []
+ with tempfile.TemporaryDirectory(prefix="podcli_panes_") as tmp:
+ for i in range(samples):
+ out = Path(tmp) / f"{i}.gray"
+ proc_run([
+ "ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
+ "-ss", f"{source.duration * (i + 1) / (samples + 1):.2f}", "-i", source.path,
+ "-frames:v", "1", "-vf", f"scale={width}:-2,format=gray", "-f", "rawvideo", str(out),
+ ], timeout=60, check=False)
+ data = np.fromfile(out, dtype=np.uint8) if out.exists() else np.zeros(0, dtype=np.uint8)
+ if len(data) and len(data) % width == 0:
+ frames.append(data.reshape(-1, width).astype(np.float32))
+ if len(frames) < 3:
+ return []
+ stack = np.stack(frames)
+
+ def borders(block: np.ndarray, axis: int) -> list[tuple[int, int]]:
+ """Runs of columns (axis 2) or rows (axis 1) that are app chrome between or around tiles."""
+ span = block.shape[axis]
+ other = 1 if axis == 2 else 2
+ # App chrome (gutters, margins) is one solid colour; even a plain wall in a
+ # downscaled, compressed camera picture varies more than this.
+ flat = block.std(axis=other).max(axis=0) < 2
+ diff = np.abs(np.diff(block, axis=axis)).mean(axis=other).min(axis=0) # a hard edge in every frame
+ edge = np.concatenate([diff > 40, [False]])
+ runs, k = [], 0
+ while k < span:
+ if flat[k] or edge[k]:
+ j = k
+ while j < span and (flat[j] or edge[j]):
+ j += 1
+ runs.append((k, j))
+ k = j
+ else:
+ k += 1
+ margins = [r for r in runs if r[0] == 0 or r[1] == span]
+ # A gutter between tiles is thin; a wide plain gap is backdrop between two people in one shot.
+ inner = [r for r in runs if r not in margins and r[1] - r[0] <= span * 0.05]
+ # Tiles are equal sizes, so the inner borders must sit at every split of
+ # one tiling (1/2; or 1/3 and 2/3; ...). A static edge in a picture, a
+ # door frame or a dark wall, rarely lands on all of them at once.
+ for n in (4, 3, 2):
+ chosen = []
+ for k in range(1, n):
+ at = span * k / n
+ near = [r for r in inner if r[0] - span * 0.03 <= at <= r[1] + span * 0.03]
+ if not near:
+ break
+ chosen.append(min(near, key=lambda r: abs((r[0] + r[1]) / 2 - at)))
+ else:
+ return sorted(margins + chosen)
+ return margins
+
+ def segments(lines: list[tuple[int, int]], span: int) -> list[tuple[float, float]]:
+ bounds, cursor = [], 0
+ for a, b in lines:
+ if a > cursor:
+ bounds.append((cursor, a))
+ cursor = max(cursor, b)
+ if cursor < span:
+ bounds.append((cursor, span))
+ return [(a / span, (b - a) / span) for a, b in bounds if (b - a) / span >= 0.15]
+
+ height = stack.shape[1]
+ panes = []
+ # Rows first, then columns inside each row band: a 2+1 gallery has two
+ # tiles on top and one centred below, so columns differ from band to band.
+ for y, h in segments(borders(stack, 1), height):
+ band = stack[:, int(y * height):int((y + h) * height), :]
+ panes += [[round(x, 4), round(y, 4), round(w, 4), round(h, 4)] for x, w in segments(borders(band, 2), stack.shape[2])]
+ return panes if len(panes) >= 2 else []
+
+
+def add_call_panes(session: MulticamSession) -> None:
+ """One call recording with several tiles: each tile becomes a person's camera.
+
+ The whole frame stays as the wide shot. Pane-to-person follows the people
+ order left to right, flagged as a guess the user can fix.
+ """
+ people = session.person_ids()
+ real = [s for s in session.cameras() if not s.virtual]
+ if len(real) != 1 or len(people) < 2 or any(s.parent for s in session.sources):
+ return
+ whole = real[0]
+ panes = detect_panes(whole)
+ if len(panes) < 2:
+ return
+ whole.person = "wide"
+ for i, crop in enumerate(panes):
+ session.sources.append(Source(
+ id=hashlib.sha1(f"{whole.id}:{crop}".encode()).hexdigest()[:10], path=whole.path, kind="video",
+ duration=whole.duration, has_audio=False, width=int(whole.width * crop[2]),
+ height=int(whole.height * crop[3]), fps=whole.fps,
+ role="camera" if i < len(people) else "ignore", person=people[i] if i < len(people) else "",
+ parent=whole.id, crop=crop,
+ ))
+
+
+def _sounds_remote(session: MulticamSession, cams: list[Source]) -> bool:
+ """True when each person's file hears mostly themselves, as call recordings do.
+
+ Studio cameras all hear the same room, so their sound levels rise and fall
+ together. Compared only while both files were recording; a camera without
+ sound can't tell, so that counts as a studio.
+ """
+ if not all(c.has_audio for c in cams):
+ return False
+ length = int(np.ceil(session.timeline_duration() / sig.FRAME_SECONDS))
+ levels = {c.id: _timeline_levels(session, c, -1, length) for c in cams}
+ for i, a in enumerate(cams):
+ for b in cams[i + 1:]:
+ lo = max(0, int(max(a.timeline_start(), b.timeline_start()) / sig.FRAME_SECONDS))
+ hi = max(0, int(min(a.timeline_end(), b.timeline_end()) / sig.FRAME_SECONDS))
+ la, lb = levels[a.id][lo:hi], levels[b.id][lo:hi]
+ heard = (la > -90) | (lb > -90)
+ if heard.sum() < int(10 / sig.FRAME_SECONDS):
+ return False
+ alike = np.corrcoef(la[heard], lb[heard])[0, 1]
+ if not np.isfinite(alike) or alike > 0.6:
+ return False
+ return True
+
+
+def add_split_if_remote(session: MulticamSession) -> None:
+ """One file per person and no wide camera: lay the people side by side as the wide shot.
+
+ Done for a remote recording, told apart by sound, or whenever the remote
+ style is chosen. The split holds people rather than files, so a person whose
+ recording came in parts keeps one column, drawn from whichever part is
+ rolling at each moment.
+ """
+ people = session.person_ids()
+ cams = [s for s in session.cameras() if not s.virtual and s.synced and s.person in people]
+ seated = [p for p in people if any(c.person == p for c in cams)]
+ if len(seated) < 2 or any(s.person == "wide" for s in session.cameras()) or any(s.members for s in session.sources):
+ return
+ chosen = session.cut_settings.get("style") == "remote"
+ one_each = [next(c for c in cams if c.person == p) for p in seated]
+ if not chosen and not _sounds_remote(session, one_each):
+ return
+ first = one_each[0]
+ session.sources.append(Source(
+ id=hashlib.sha1(f"split:{seated}".encode()).hexdigest()[:10], path="", kind="video",
+ duration=first.duration, has_audio=False, width=first.width, height=first.height, fps=first.fps,
+ role="camera", person="wide", members=seated, guessed=False,
+ ))
+
+
+def place_views(session: MulticamSession) -> None:
+ """Put virtual cameras on the timeline from the files they come from."""
+ for v in session.sources:
+ if v.parent:
+ parent = session.source(v.parent)
+ v.offset, v.speed, v.duration, v.sync = parent.offset, parent.speed, parent.duration, dict(parent.sync)
+ elif v.members:
+ files = [c for c in session.cameras() if not c.virtual and c.synced and c.person in v.members]
+ if files:
+ start = min(c.timeline_start() for c in files)
+ end = max(c.timeline_end() for c in files)
+ v.offset, v.speed, v.duration = round(start, 4), 1.0, round(end - start, 3)
+ v.sync = {"status": "ok"}
+ else:
+ v.offset, v.sync = None, {"status": "failed", "message": "Nobody in the split has a synced camera."}
+
+
def collect_files(folder: str = "", files: Optional[list] = None) -> list[str]:
if files is not None and not (isinstance(files, list) and all(isinstance(f, str) for f in files)):
raise ValueError("files must be a list of file paths")
@@ -363,7 +608,7 @@ def find_session(found: list[str]) -> Optional[MulticamSession]:
session = MulticamSession.load(p.stem)
except (ValueError, OSError, TypeError, json.JSONDecodeError):
continue
- if {s.path for s in session.sources} | set(session.skipped) == wanted:
+ if {s.path for s in session.sources if not s.members} | set(session.skipped) == wanted:
return session
return None
@@ -386,15 +631,15 @@ def new_session(
) -> MulticamSession:
"""Scan and probe one episode's files, or reopen the saved edit for the same files."""
found = collect_files(folder, files)
- raw_people = [p.get("name", "") if isinstance(p, dict) else str(p or "") for p in (people or [])]
- person_names = [n.strip() for n in raw_people if n.strip()]
-
+ # Reopening returns the saved edit untouched; renaming people is an edit the
+ # caller makes after, under the same lock as any other.
existing = find_session(found)
if existing:
- if person_names and person_names != [p.name for p in existing.people]:
- existing = rename_people(existing, person_names)
_emit(progress_callback, 100, f"Reopened the edit for these {len(found)} files")
return existing
+ named = [(str(p.get("name", "")).strip(), p.get("role")) if isinstance(p, dict) else (str(p or "").strip(), None)
+ for p in (people or [])]
+ named = [(n, r) for n, r in named if n]
sources, skipped = [], []
for i, p in enumerate(found):
@@ -406,24 +651,117 @@ def new_session(
_emit(progress_callback, 5 + 90 * i / len(found), f"Skipped {os.path.basename(p)}: {e}")
if not sources:
raise ValueError("None of the files could be read as audio or video.")
-
- person_names = person_names or ["Host", "Guest"]
+ seen: dict[str, int] = {}
+ for src in sources:
+ # Two cameras can write the same file name at the same size (C0001.MP4).
+ n = seen.get(src.id, 0)
+ seen[src.id] = n + 1
+ if n:
+ src.id = hashlib.sha1(f"{src.id}:{n}".encode()).hexdigest()[:10]
+
+ named = named or [("Host", None), ("Guest", None)]
session = MulticamSession(
session_id=uuid.uuid4().hex[:12],
name=name or (Path(folder).name if folder else Path(found[0]).stem),
folder=os.path.abspath(folder) if folder else "",
- people=[Person(id=_person_id(n, i), name=n) for i, n in enumerate(person_names)],
+ # Most shows list the hosts first, so unless told, the last person is the guest.
+ people=[
+ Person(id=_person_id(n, i), name=n,
+ role=r if r in ("host", "guest") else "guest" if i == len(named) - 1 and i > 0 else "host")
+ for i, (n, r) in enumerate(named)
+ ],
sources=sources,
skipped=skipped,
)
guess_roles(session)
+ add_call_panes(session)
session.save()
_emit(progress_callback, 100, f"Found {len(sources)} sources")
return session
+def apply_state(session: MulticamSession, state: dict) -> MulticamSession:
+ """Restore a prepared edit (a saved --json payload) onto freshly opened files.
+
+ A cloud worker is stateless: it downloads the recordings, opens them, and
+ applies the state another worker prepared, so it can render without
+ syncing again. Files are matched by id, which is stable across machines.
+ Every field goes through the same checks as an edit made by hand.
+ """
+ if not isinstance(state, dict) or not isinstance(state.get("sources"), list) or not isinstance(state.get("people"), list):
+ raise ValueError("A saved edit needs its people and sources")
+ saved_sources = [x for x in state["sources"] if isinstance(x, dict) and isinstance(x.get("id"), str)]
+ real = [s for s in session.sources if not s.virtual]
+ real_ids = {s.id for s in real}
+ # Tiles and split screens come from the saved edit as they were; detecting them again could differ.
+ views = []
+ for saved in saved_sources:
+ parent, members = saved.get("parent") or "", saved.get("members") or []
+ if not parent and not members:
+ continue
+ if parent and parent not in real_ids:
+ raise ValueError(f"The saved edit has a tile cut from a file that isn't here ({parent})")
+ crop = [_number(x, "crop", 0, 1) for x in (saved.get("crop") or [])]
+ if parent and len(crop) != 4:
+ raise ValueError("A saved tile needs a crop of x, y, width and height")
+ base = next((s for s in real if s.id == parent), None)
+ views.append(Source(
+ id=saved["id"], path=base.path if base else "", kind="video", duration=base.duration if base else 0.0,
+ has_audio=False, width=int(saved.get("width") or 0), height=int(saved.get("height") or 0),
+ fps=float(saved.get("fps") or 0), parent=parent, crop=crop,
+ members=[str(m) for m in members if isinstance(m, str)],
+ ))
+ session.sources = real + views
+ ids = {s.id for s in session.sources}
+ update_mapping(session, {
+ "people": state["people"],
+ "sources": [
+ {k: x[k] for k in ("id", "role", "person", "channel_people") if k in x}
+ for x in saved_sources if x["id"] in ids
+ ],
+ })
+ for saved in saved_sources:
+ if saved["id"] not in ids or saved.get("parent") or saved.get("members"):
+ continue
+ src = session.source(saved["id"])
+ offset = saved.get("offset")
+ src.offset = None if offset is None else _number(offset, "offset", -MAX_SECONDS, MAX_SECONDS)
+ src.speed = _number(saved.get("speed", 1.0), "speed", 0.99, 1.01)
+ src.sync = dict(saved.get("sync") or {})
+ place_views(session)
+ session.name = str(state.get("name") or session.name)
+ if state.get("reference_id") in ids:
+ session.reference_id = state["reference_id"]
+ overrides = state.get("cut_overrides")
+ if not isinstance(overrides, dict):
+ overrides = state.get("cut_settings") if isinstance(state.get("cut_settings"), dict) else {}
+ update_mapping(session, {
+ "range_start": state.get("range_start"), "range_end": state.get("range_end"),
+ "look": state.get("look") or "none", "cut_settings": overrides,
+ "speaker_map": state.get("speaker_map") or {},
+ })
+ cuts = state.get("cuts") or []
+ if cuts and all(isinstance(c, dict) and c.get("source_id") in ids for c in cuts):
+ set_cuts(session, cuts)
+ set_removals(session, state.get("removals") or [])
+ session.activity_key = ""
+ session.save()
+ return session
+
+
def needs_sync(session: MulticamSession) -> bool:
- return any(s.role != "ignore" and not s.synced for s in session.sources)
+ return any(_in_use(session, s) and not s.virtual and not s.synced for s in session.sources)
+
+
+def _in_use(session: MulticamSession, s: Source) -> bool:
+ """Mapped to something, or the recording a tile in use is cut from (even when the whole frame is ignored)."""
+ return s.role != "ignore" or any(v.parent == s.id and v.role != "ignore" for v in session.sources)
+
+
+def _owners(session: MulticamSession, s: Source) -> set[str]:
+ """Whose voice or face a file carries, when it belongs to particular people."""
+ people = set(session.person_ids())
+ return ({s.person} & people) | (set(s.channel_people) & people)
def _person_id(name: str, index: int) -> str:
@@ -440,6 +778,7 @@ def update_mapping(session: MulticamSession, params: dict) -> MulticamSession:
before = _cut_inputs(session)
if "people" in params:
people = []
+ roles = {p.id: p.role for p in session.people}
for i, entry in enumerate(params["people"] or []):
name = str(entry.get("name") if isinstance(entry, dict) else entry).strip()
if not name:
@@ -447,12 +786,17 @@ def update_mapping(session: MulticamSession, params: dict) -> MulticamSession:
pid = (entry.get("id") if isinstance(entry, dict) else None) or _person_id(name, i)
if pid == "wide" or pid in {p.id for p in people}:
pid = f"{pid}-{i + 1}"
- people.append(Person(id=pid, name=name))
+ role = (entry.get("role") if isinstance(entry, dict) else None) or roles.get(pid, "host")
+ if role not in {"host", "guest"}:
+ raise ValueError(f"A person's role is host or guest; got {role!r}")
+ people.append(Person(id=pid, name=name, role=role))
if not people:
raise ValueError("Add at least one person")
session.people = people
valid = set(session.person_ids())
for s in session.sources:
+ if s.members:
+ s.members = [m for m in s.members if m in valid]
if s.role == "camera" and s.person not in valid | {"wide"}:
s.person = "wide"
if s.role == "mic" and s.person and s.person not in valid:
@@ -483,33 +827,54 @@ def update_mapping(session: MulticamSession, params: dict) -> MulticamSession:
if "channel_people" in edit:
chans = [c if c in valid else "" for c in (edit["channel_people"] or [])]
s.channel_people = chans[: max(0, s.audio_channels)] if any(chans) else []
+ if (("offset" in edit and edit["offset"] is not None) or "nudge" in edit) and s.virtual:
+ raise ValueError("A tile or split screen moves with the files it comes from. Move those instead.")
if "offset" in edit and edit["offset"] is not None:
- s.offset = float(edit["offset"])
+ s.offset = _number(edit["offset"], "offset", -MAX_SECONDS, MAX_SECONDS)
s.sync = {**s.sync, "status": "manual"}
if "nudge" in edit:
if s.offset is None:
raise ValueError(f"{os.path.basename(s.path)} isn't synced yet. Set its offset first.")
- s.offset = float(s.offset) + float(edit["nudge"])
+ s.offset = _number(float(s.offset) + _number(edit["nudge"], "nudge", -3600, 3600), "offset",
+ -MAX_SECONDS, MAX_SECONDS)
s.sync = {**s.sync, "status": "manual"}
s.guessed = False
if "range_start" in params:
- session.range_start = None if params["range_start"] is None else max(0.0, float(params["range_start"]))
+ session.range_start = None if params["range_start"] is None else _number(params["range_start"], "start", 0, MAX_SECONDS)
if "range_end" in params:
- session.range_end = None if params["range_end"] is None else float(params["range_end"])
+ session.range_end = None if params["range_end"] is None else _number(params["range_end"], "end", 0, MAX_SECONDS)
if "look" in params:
if params["look"] not in LOOKS:
raise ValueError(f"Unknown look {params['look']!r}. Use: {', '.join(LOOKS)}")
session.look = params["look"]
if "cut_settings" in params:
- merged = {**session.cut_settings, **(params["cut_settings"] or {})}
- session.cut_settings = {
- "min_shot": max(0.5, min(10.0, float(merged["min_shot"]))),
- "max_shot": max(0.0, min(600.0, float(merged["max_shot"]))),
- "wide_insert": max(1.0, min(15.0, float(merged["wide_insert"]))),
- }
+ incoming = params["cut_settings"] or {}
+ style = incoming.get("style", session.cut_settings.get("style", "auto"))
+ if style != "auto" and style not in STYLES:
+ raise ValueError(f"Cut style is one of auto, {', '.join(STYLES)}; got {style!r}")
+ # A new style starts from its own defaults rather than the old style's tweaks.
+ same = style == session.cut_settings.get("style", "auto")
+ overrides = {k: v for k, v in session.cut_settings.items() if k != "style"} if same else {}
+ for key, value in incoming.items():
+ if key in CUT_LIMITS:
+ lo, hi = CUT_LIMITS[key]
+ overrides[key] = max(lo, min(hi, _number(value, key, -MAX_SECONDS, MAX_SECONDS)))
+ elif key in ("hold_guest", "host_solo"):
+ if not isinstance(value, bool):
+ raise ValueError(f"{key} is true or false")
+ overrides[key] = value
+ elif key != "style":
+ raise ValueError(f"Unknown cut setting {key!r}")
+ session.cut_settings = {"style": style}
+ # Only what differs from the style is a tweak; keeping the rest would pin today's defaults.
+ defaults = STYLES[resolved_style(session)]
+ session.cut_settings.update({k: v for k, v in overrides.items() if v != defaults[k]})
+ if "removals" in params:
+ set_removals(session, params["removals"] or [])
if "speaker_map" in params:
session.speaker_map = {str(k): v for k, v in (params["speaker_map"] or {}).items() if v in valid}
+ place_views(session)
if _cut_inputs(session) != before:
session.cuts = []
session.save()
@@ -519,7 +884,8 @@ def update_mapping(session: MulticamSession, params: dict) -> MulticamSession:
def _cut_inputs(session: MulticamSession) -> str:
return json.dumps([
[(s.id, s.role, s.person, s.channel_people, s.offset) for s in session.sources],
- session.person_ids(), session.range_start, session.range_end, session.cut_settings, session.speaker_map,
+ [(p.id, p.role) for p in session.people], session.range_start, session.range_end,
+ session.cut_settings, session.speaker_map,
], default=str)
@@ -584,7 +950,7 @@ def _fine_points(ref: np.ndarray, src: np.ndarray, lag: float) -> list[tuple[flo
if prominence < 8:
continue
# A drifting clock smears the match across the window, so the measured
- # shift belongs to the window's middle, not its start.
+ # shift belongs to the window's middle.
mid = a + FINE_WINDOW / 2
pts.append((mid, mid + lag + shift, prominence))
return pts
@@ -600,6 +966,15 @@ def _match(ref: np.ndarray, ref_env: np.ndarray, src: np.ndarray) -> tuple[Optio
"message": "No shared sound found. Check the file belongs to this episode, or set the offset by hand.",
}
fit = sig.fit_clock(_fine_points(ref, src, match.lag_seconds))
+ # Two different voices can line up by chance in the coarse pass; a real
+ # match nearly always gets confirmed sample by sample, so an unconfirmed one
+ # has to be overwhelming to count.
+ if fit is None and (match.score < 20 or match.peak_ratio > 0.6):
+ return None, {
+ "status": "failed",
+ "score": round(match.score, 1),
+ "message": "No shared sound found. Check the file belongs to this episode, or set the offset by hand.",
+ }
if fit is None:
return sig.ClockFit(match.lag_seconds, 1.0, 10.0, 0), {
"status": "rough", "score": round(match.score, 1),
@@ -626,7 +1001,7 @@ def sync_session(
shares no sound with the reference gets a second try against every file
that did sync, which covers cameras split into parts with no continuous mic.
"""
- candidates = [s for s in session.sources if s.role != "ignore" and s.has_audio]
+ candidates = [s for s in session.sources if _in_use(session, s) and s.has_audio and not s.virtual]
if not candidates:
raise ValueError("Map at least one camera or mic that has audio before syncing")
# The longest recording is the one most likely to overlap every other file;
@@ -675,15 +1050,42 @@ def sync_session(
anchors.append(s)
break
+ # A remote recording made with headphones: each person's files share no sound
+ # with anyone else's. Riverside and Zoom start every track together, so each
+ # person's group is taken to start with the rest and synced within itself.
+ remaining = [s for s in failed if not s.synced]
+ heard = {p for s in [reference, *others] if s.synced for p in _owners(session, s)}
+ if remaining and all(_owners(session, s) for s in remaining) and not (
+ {p for s in remaining for p in _owners(session, s)} & heard):
+ while remaining:
+ lead = max(remaining, key=lambda s: s.duration)
+ lead.offset, lead.speed = 0.0, 1.0
+ lead.sync = {"status": "assumed", "message": "No sound in common with the others, so it is taken to "
+ "start with them, as call recorders do. Nudge it if it's off."}
+ lead_audio = wav(lead)
+ lead_env = sig.onset_envelope(lead_audio, SYNC_RATE)
+ rest = []
+ for s in remaining:
+ if s is lead:
+ continue
+ fit, report = _match(lead_audio, lead_env, wav(s))
+ if fit:
+ s.offset, s.speed, s.sync = fit.offset, fit.speed, {**report, "via": lead.id}
+ else:
+ rest.append(s)
+ remaining = rest
+
for s in session.sources:
- if s.role != "ignore" and not s.has_audio and s not in manual:
+ if _in_use(session, s) and not s.has_audio and not s.virtual and s not in manual:
s.offset = None
s.sync = {"status": "failed", "message": "This camera recorded no audio. Set its offset by hand."}
- synced = [s for s in session.sources if s.synced and s.role != "ignore"]
+ synced = [s for s in session.sources if s.synced and _in_use(session, s) and not s.virtual]
shift = min(s.timeline_start() for s in synced)
for s in synced:
s.offset = round(float(s.offset) - shift, 4)
+ add_split_if_remote(session)
+ place_views(session)
session.range_start = session.range_end = None
session.cuts = []
session.activity_key = ""
@@ -740,6 +1142,17 @@ def _diarized_labels(session: MulticamSession, length: int, progress_callback: P
return sig.segments_to_frames(segments, people, length)
+def _person_levels(session: MulticamSession, length: int, progress_callback: ProgressCallback = None) -> dict[str, np.ndarray]:
+ """One timeline-aligned level track (dB per 10 ms) per person with a mic of their own."""
+ feeds = [(s, ch, pid) for s, ch, pid in session.person_mics() if s.synced]
+ per_person: dict[str, np.ndarray] = {}
+ for i, (s, ch, pid) in enumerate(feeds):
+ _emit(progress_callback, 10 + 70 * i / len(feeds), f"Listening to {os.path.basename(s.path)}")
+ level = _timeline_levels(session, s, ch, length)
+ per_person[pid] = np.maximum(per_person[pid], level) if pid in per_person else level
+ return per_person
+
+
def speaker_labels(session: MulticamSession, progress_callback: ProgressCallback = None) -> np.ndarray:
"""Per-10ms timeline labels (person index, SILENT, BOTH), cached per mapping."""
length = int(np.ceil(session.timeline_duration() / sig.FRAME_SECONDS))
@@ -750,14 +1163,9 @@ def speaker_labels(session: MulticamSession, progress_callback: ProgressCallback
if len(labels) == length:
return labels
- feeds = [(s, ch, pid) for s, ch, pid in session.person_mics() if s.synced]
- if feeds:
+ per_person = _person_levels(session, length, progress_callback)
+ if per_person:
people = session.person_ids()
- per_person: dict[str, np.ndarray] = {}
- for i, (s, ch, pid) in enumerate(feeds):
- _emit(progress_callback, 10 + 70 * i / len(feeds), f"Listening to {os.path.basename(s.path)}")
- level = _timeline_levels(session, s, ch, length)
- per_person[pid] = np.maximum(per_person[pid], level) if pid in per_person else level
speaking = [p for p in people if p in per_person]
raw = sig.speaker_frames([per_person[p] for p in speaking])
index = np.array([people.index(p) for p in speaking], dtype=np.int32)
@@ -804,7 +1212,9 @@ def plan_session(session: MulticamSession, progress_callback: ProgressCallback =
session.cuts = sig.plan_cuts(
labels, session.person_ids(), cams,
- range_start=start, range_end=end, **session.cut_settings,
+ range_start=start, range_end=end,
+ guests=frozenset(p.id for p in session.people if p.role == "guest"),
+ **effective_cut(session),
)
if not session.cuts:
raise ValueError("No camera covers the episode range. Check the camera mapping and the start and end.")
@@ -828,78 +1238,308 @@ def cut_stats(session: MulticamSession) -> dict:
}
-def set_cut(session: MulticamSession, index: int, source_id: str) -> MulticamSession:
- """Swap one shot to another camera; neighbours on the same camera merge."""
- if not 0 <= index < len(session.cuts):
- raise IndexError(f"No shot {index + 1}")
- cam = session.source(source_id)
- shot = session.cuts[index]
- if cam.role != "camera" or not cam.synced:
- raise ValueError("Pick a synced camera")
- if cam.timeline_start() > shot["start"] + 1e-3 or cam.timeline_end() < shot["end"] - 1e-3:
- raise ValueError(f"{os.path.basename(cam.path)} wasn't recording for this whole shot")
- shot["source_id"] = source_id
- merged: list[dict] = []
- for c in session.cuts:
- if merged and merged[-1]["source_id"] == c["source_id"]:
- merged[-1]["end"] = c["end"]
+def set_cuts(session: MulticamSession, cuts: list) -> MulticamSession:
+ """Replace the cut with a hand-edited one; neighbouring shots on the same camera merge.
+
+ Shots must run back to back, and every camera must have been recording for
+ its whole shot, so a render never reaches past the end of a file.
+ """
+ if not isinstance(cuts, list) or not cuts or len(cuts) > MAX_SHOTS:
+ raise ValueError(f"cuts must be a list of 1 to {MAX_SHOTS} {{start, end, source_id}} shots")
+ clean: list[dict] = []
+ for c in cuts:
+ if not isinstance(c, dict) or not {"start", "end", "source_id"} <= c.keys():
+ raise ValueError("Each shot needs start, end and source_id")
+ start = _number(c["start"], "start", 0, MAX_SECONDS)
+ end = _number(c["end"], "end", 0, MAX_SECONDS)
+ cam = session.source(str(c["source_id"]))
+ if end - start < 0.1:
+ raise ValueError("A shot must be at least 0.1 s long")
+ if clean and abs(clean[-1]["end"] - start) > 1e-3:
+ raise ValueError("Shots must run back to back with no gaps or overlaps")
+ if cam.role != "camera" or not cam.synced:
+ raise ValueError(f"{os.path.basename(cam.path)} isn't a synced camera")
+ if cam.timeline_start() > start + 1e-3 or cam.timeline_end() < end - 1e-3:
+ raise ValueError(f"{os.path.basename(cam.path)} wasn't recording for the whole shot at {start:.1f}s")
+ if clean and clean[-1]["source_id"] == cam.id:
+ clean[-1]["end"] = round(end, 3)
else:
- merged.append(dict(c))
- session.cuts = merged
+ clean.append({"start": round(start, 3), "end": round(end, 3), "source_id": cam.id})
+ previous, session.cuts = session.cuts, clean
+ if not kept_segments(session):
+ session.cuts = previous
+ raise ValueError("The removals cover this whole cut. Restore some of the episode first.")
session.save()
return session
+def set_cut(session: MulticamSession, index: int, source_id: str) -> MulticamSession:
+ """Swap one shot to another camera."""
+ if not 0 <= index < len(session.cuts):
+ raise IndexError(f"No shot {index + 1}")
+ cuts = [dict(c) for c in session.cuts]
+ cuts[index]["source_id"] = source_id
+ return set_cuts(session, cuts)
+
+
+def activity(session: MulticamSession) -> dict:
+ """Who speaks when, as spans per person, for drawing the timeline's speaking lanes."""
+ labels = speaker_labels(session)
+ people = session.person_ids()
+
+ def spans(mask: np.ndarray) -> list[list[float]]:
+ edges = np.flatnonzero(np.diff(np.concatenate([[0], mask.astype(np.int8), [0]])))
+ out: list[list[float]] = []
+ for a, b in zip(edges[::2], edges[1::2]):
+ start, end = a * sig.FRAME_SECONDS, b * sig.FRAME_SECONDS
+ # Breaths and short pauses would draw as noise; join them into one phrase.
+ if out and start - out[-1][1] < 0.25:
+ out[-1][1] = round(end, 2)
+ elif end - start >= 0.15:
+ out.append([round(start, 2), round(end, 2)])
+ return out
+
+ return {
+ "people": {pid: spans((labels == i) | (labels == sig.BOTH)) for i, pid in enumerate(people)},
+ "both": spans(labels == sig.BOTH),
+ }
+
+
# ---------------------------------------------------------------------------
# Previews
# ---------------------------------------------------------------------------
-def _frame(source: Source, at: float, out: Path, look: str = "none", width: int = 480) -> Path:
- vf = ",".join(x for x in [LOOKS.get(look, ""), f"scale={width}:-2"] if x)
+def _picture(session: MulticamSession, cam: Source, tl: float, width: int, height: int) -> tuple[list[str], str]:
+ """ffmpeg inputs and a filter graph drawing `cam` at timeline second tl into [pic], width x height.
+
+ A real camera is fitted to the frame; a pane is cropped out of its call
+ recording first; a split screen fills one equal column per person.
+ """
+ fit = f"scale={width}:{height}:force_original_aspect_ratio=decrease,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2,setsar=1"
+ if cam.members:
+ n = len(cam.members)
+ pane = (width // n) & ~1
+ args, chains = [], []
+ for i, person in enumerate(cam.members):
+ # Each column is that person's file rolling at this moment, or black between parts.
+ rolling = next((c for c in session.cameras() if not c.virtual and c.synced and c.person == person
+ and c.timeline_start() <= tl < c.timeline_end()), None)
+ if rolling:
+ args += ["-ss", f"{max(0.0, rolling.source_time(tl)):.4f}", "-i", rolling.path]
+ else:
+ args += ["-f", "lavfi", "-i", f"color=c=black:s={pane}x{height}"]
+ chains.append(f"[{i}:v:0]scale={pane}:{height}:force_original_aspect_ratio=increase,crop={pane}:{height},setsar=1[p{i}]")
+ joined = "".join(f"[p{i}]" for i in range(n))
+ return args, ";".join(chains) + f";{joined}hstack=inputs={n},pad={width}:{height}:(ow-iw)/2:(oh-ih)/2[pic]"
+ src = session.source(cam.parent) if cam.parent else cam
+ crop = ""
+ if cam.crop:
+ x, y, w, h = cam.crop
+ crop = f"crop=iw*{w:.4f}:ih*{h:.4f}:iw*{x:.4f}:ih*{y:.4f},"
+ return ["-ss", f"{max(0.0, src.source_time(tl)):.4f}", "-i", src.path], f"[0:v:0]{crop}{fit}[pic]"
+
+
+def _still(session: MulticamSession, cam: Source, tl: float, out: Path, look: str = "none", width: int = 480) -> Path:
+ args, graph = _picture(session, cam, tl, width, (width * 9 // 16) & ~1)
+ tail = f";[pic]{LOOKS[look]}[still]" if LOOKS.get(look) else ";[pic]null[still]"
proc_run([
- "ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
- "-ss", f"{max(0.0, at):.3f}", "-i", source.path, "-frames:v", "1",
- "-vf", vf, "-q:v", "4", str(out),
+ "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", *args,
+ "-filter_complex", graph + tail, "-map", "[still]", "-frames:v", "1", "-q:v", "4", str(out),
], timeout=60, check=True)
return out
def previews(session: MulticamSession, *, looks: bool = False, at: Optional[float] = None) -> dict:
- """One still per camera, and optionally one still per look from the first camera."""
+ """One still per camera, and optionally one still per look from the wide camera."""
work = _work_dir(session.session_id)
frames: dict = {"cameras": {}, "looks": {}}
+
+ def moment(s: Source) -> float:
+ if at is not None and s.synced and s.timeline_start() <= at < s.timeline_end():
+ return at
+ return s.timeline_start() + s.duration * s.speed * 0.3
+
for s in session.sources:
if s.kind != "video":
continue
- t = s.duration * 0.3 if at is None or not s.synced else s.source_time(at)
- t = min(max(0.0, t), max(0.0, s.duration - 0.5))
+ t = moment(s)
out = work / f"frame-{s.id}-{int(t * 10)}.jpg"
if not out.exists():
- _frame(s, t, out)
+ _still(session, s, t, out)
frames["cameras"][s.id] = str(out)
if looks:
cams = session.cameras() or [s for s in session.sources if s.kind == "video"]
if cams:
cam = next((c for c in cams if c.person == "wide"), cams[0])
- t = cam.duration * 0.3
for name in LOOKS:
out = work / f"look-{cam.id}-{name}.jpg"
if not out.exists():
- _frame(cam, t, out, look=name, width=640)
+ _still(session, cam, moment(cam), out, look=name, width=640)
frames["looks"][name] = str(out)
return frames
+def build_preview(session: MulticamSession, progress_callback: ProgressCallback = None) -> MulticamSession:
+ """Small proxies of every recorded camera, one mic mix on the timeline, and a still per camera.
+
+ This is what a browser editor (the podcli cloud editor) plays from. Browsers
+ can't play 4K, MXF, or MTS smoothly or at all; 540p H.264 with a keyframe
+ every second seeks instantly. Tiles and split screens get no proxy of their
+ own: an editor draws them from their parent's proxy and crop, or from each
+ member's. Cached until a source or the sync changes.
+ """
+ work = _work_dir(session.session_id)
+ # Panes and split screens are drawn from these by their crop and members.
+ cams = [s for s in session.cameras() if s.synced and not s.virtual]
+ proxies: dict[str, str] = {}
+ for i, cam in enumerate(cams):
+ out = work / f"{cam.id}.proxy.mp4"
+ if not (out.exists() and out.stat().st_mtime >= os.path.getmtime(cam.path)):
+ _emit(progress_callback, 5 + 80 * i / max(1, len(cams)), f"Preparing preview of {os.path.basename(cam.path)}")
+ tmp = out.with_suffix(".tmp.mp4")
+ gop = str(max(1, round(cam.fps or 25)))
+ proc_run([
+ "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", cam.path, "-map", "0:v:0", "-an",
+ "-vf", "scale=-2:540", "-c:v", "libx264", "-preset", "veryfast", "-crf", "28",
+ "-g", gop, "-pix_fmt", "yuv420p", "-movflags", "+faststart", str(tmp),
+ ], timeout=7200, check=True)
+ os.replace(tmp, out)
+ proxies[cam.id] = str(out)
+
+ audio = work / f"preview-{_mix_key(session)}.m4a"
+ if not audio.exists():
+ _emit(progress_callback, 88, "Mixing preview audio")
+ tmp = audio.with_suffix(".tmp.m4a")
+ _write_mix(session, tmp, 0.0, session.timeline_duration(),
+ encode=("-ac", "1", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart"))
+ os.replace(tmp, audio)
+ for old in work.glob("preview-*.m4a"):
+ if old != audio:
+ old.unlink(missing_ok=True)
+ # Edits keep landing while proxies encode, so write onto the latest saved copy.
+ latest = MulticamSession.load(session.session_id)
+ latest.preview = {"proxies": proxies, "audio": str(audio), "stills": previews(latest)["cameras"]}
+ latest.save()
+ _emit(progress_callback, 100, "Preview ready")
+ return latest
+
+
+def transcript(session: MulticamSession, *, model_size: str = "base", engine: Optional[str] = None,
+ language: Optional[str] = None, progress_callback: ProgressCallback = None) -> dict:
+ """Words on the timeline, each credited to the person who said it.
+
+ The mics are mixed and transcribed once: transcribing each mic alone with
+ the other voice silenced throws word timestamps seconds off. Each word goes
+ to the mic that was loudest during it, relative to that mic's own speech
+ level, then sentence edges settle what loose timestamps leave unclear at a
+ turn change. Cached until the mix changes.
+ """
+ from services.transcription import transcribe_file
+
+ work = _work_dir(session.session_id)
+ # Who each word is credited to depends on the mic mapping as well as the mix.
+ credit = [(s.id, ch, pid) for s, ch, pid in session.person_mics()]
+ key = hashlib.sha1(json.dumps(
+ [_mix_key(session), credit, session.speaker_map, model_size, engine, language], default=str,
+ ).encode()).hexdigest()[:12]
+ out = work / f"transcript-{key}.json"
+ if out.exists():
+ return json.loads(out.read_text(encoding="utf-8"))
+ wav = work / "transcript-mix.wav"
+ _emit(progress_callback, 2, "Mixing the mics for transcription")
+ _write_mix(session, wav, 0.0, session.timeline_duration(), encode=("-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le"))
+ try:
+ result = transcribe_file(
+ str(wav), model_size=model_size, engine=engine, language=language, enable_diarization=False,
+ wav_path=str(wav), progress_callback=lambda p, m: _emit(progress_callback, 5 + p * 0.85, m),
+ )
+ finally:
+ wav.unlink(missing_ok=True)
+ labels = speaker_labels(session)
+ people = session.person_ids()
+ levels = {
+ pid: level - np.percentile(level[level > -90], 95) if (level > -90).any() else level
+ for pid, level in _person_levels(session, len(labels)).items()
+ }
+
+ def loudest(start: float, end: float) -> tuple[str, float]:
+ """The credited person, and by how many dB their mic beat the next one."""
+ a, b = max(0, int(start / sig.FRAME_SECONDS)), int(np.ceil(end / sig.FRAME_SECONDS))
+ means = sorted(((float(np.mean(lv[a:b])), pid) for pid, lv in levels.items() if len(lv[a:b])), reverse=True)
+ if means and means[0][0] > -30:
+ return means[0][1], means[0][0] - (means[1][0] if len(means) > 1 else -100.0)
+ return _speaker_at(labels, people, start, end), 0.0
+
+ words, margins, last = [], [], ""
+ for w in result.get("words") or []:
+ start, end = float(w["start"]), float(w["end"])
+ who, margin = loudest(start, end)
+ last = who or last
+ words.append({"start": round(start, 3), "end": round(end, 3), "text": str(w.get("word", "")).strip(), "person": last})
+ margins.append(margin)
+ _settle_turn_edges(words, margins)
+ data = {"words": words, "language": result.get("language")}
+ out.write_text(json.dumps(data), encoding="utf-8")
+ _emit(progress_callback, 100, f"Transcribed {len(words)} words")
+ return data
+
+
+_SENTENCE_END = (".", "?", "!")
+
+
+def _settle_turn_edges(words: list[dict], margins: list[float], clear: float = 6.0) -> None:
+ """Fix credits that loose word timestamps get wrong around a turn change.
+
+ Only words the mics couldn't tell apart (margin under `clear` dB) move. A
+ lone word credited to someone else mid-sentence goes back to the person
+ saying that sentence. The one word that opens a new sentence after the
+ previous speaker's full stop goes to whoever speaks next: "...get home.
+ Right," is the next speaker's "Right,".
+ """
+ def rejoin_strays() -> None:
+ for i in range(1, len(words) - 1):
+ prev, word, nxt = words[i - 1], words[i], words[i + 1]
+ if (margins[i] < clear and word["person"] != prev["person"] and prev["person"] == nxt["person"]
+ and not prev["text"].endswith(_SENTENCE_END) and not word["text"].endswith(_SENTENCE_END)):
+ word["person"] = prev["person"]
+
+ rejoin_strays()
+ for i in range(1, len(words) - 1):
+ prev, word, nxt = words[i - 1], words[i], words[i + 1]
+ if (margins[i] < clear and word["person"] == prev["person"] != nxt["person"]
+ and prev["text"].endswith(_SENTENCE_END) and word["text"][:1].isupper()):
+ word["person"] = nxt["person"]
+ # Moving a sentence opener can leave the word after it stranded; rejoin it too.
+ rejoin_strays()
+
+
+def _speaker_at(labels: np.ndarray, people: list[str], start: float, end: float) -> str:
+ """Whoever speaks most during a word, else around it within a second, else nobody."""
+ for pad in (0.0, 1.0):
+ a = max(0, int((start - pad) / sig.FRAME_SECONDS))
+ b = min(len(labels), int(np.ceil((end + pad) / sig.FRAME_SECONDS)))
+ span = labels[a:b]
+ span = span[span >= 0]
+ if len(span):
+ return people[int(np.bincount(span).argmax())]
+ return ""
+
+
# ---------------------------------------------------------------------------
# Render
# ---------------------------------------------------------------------------
def _output_format(session: MulticamSession) -> tuple[int, int, float]:
- cams = [s for s in session.cameras() if s.synced]
- width = max(s.width for s in cams)
- height = max(s.height for s in cams)
- rates = [_standard_fps(s.fps) for s in cams]
- return width - width % 2, height - height % 2, max(set(rates), key=rates.count)
+ """The episode's frame: the wide shot's, else the biggest landscape camera's.
+
+ One camera's size is used whole, so a guest on a portrait phone never
+ stretches a landscape show into a square.
+ """
+ cams = [s for s in session.cameras() if s.synced and s.width and s.height]
+ wide = [c for c in cams if c.person == "wide"]
+ pick = wide[0] if wide else max(cams, key=lambda c: (c.width >= c.height, c.width * c.height))
+ rates = [_standard_fps(s.fps) for s in cams if not s.virtual] or [_standard_fps(pick.fps)]
+ return pick.width - pick.width % 2, pick.height - pick.height % 2, max(set(rates), key=rates.count)
STANDARD_FPS = (24000 / 1001, 24.0, 25.0, 30000 / 1001, 30.0, 50.0, 60000 / 1001, 60.0)
@@ -918,7 +1558,7 @@ def _output_dir(session: MulticamSession) -> Path:
return d
-def _render_shot(cam: Optional[Source], out: Path, tl_start: float, frames: int,
+def _render_shot(session: MulticamSession, cam: Optional[Source], out: Path, tl_start: float, frames: int,
width: int, height: int, fps: float, look: str) -> None:
"""Encode exactly `frames` frames starting at timeline second tl_start.
@@ -936,21 +1576,13 @@ def _render_shot(cam: Optional[Source], out: Path, tl_start: float, frames: int,
"-f", "lavfi", "-i", f"color=c=black:s={width}x{height}:r={fps:.6f}", *common,
], timeout=600, check=True)
return
- vf = [
- f"scale={width}:{height}:force_original_aspect_ratio=decrease",
- f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2",
- "setsar=1",
- f"fps={fps:.6f}",
- # A camera that stops a few frames early would shorten the shot and slip
- # every later shot against the audio; holding its last frame prevents that.
- "tpad=stop_mode=clone:stop=-1",
- ]
- if LOOKS.get(look):
- vf.append(LOOKS[look])
+ args, graph = _picture(session, cam, tl_start, width, height)
+ # A camera that stops a few frames early would shorten the shot and slip
+ # every later shot against the audio; holding its last frame prevents that.
+ tail = [f"fps={fps:.6f}", "tpad=stop_mode=clone:stop=-1", *([LOOKS[look]] if LOOKS.get(look) else [])]
proc_run([
- "ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
- "-ss", f"{max(0.0, cam.source_time(tl_start)):.4f}", "-i", cam.path,
- "-map", "0:v:0", "-vf", ",".join(vf), *common,
+ "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", *args,
+ "-filter_complex", f"{graph};[pic]{','.join(tail)}[v]", "-map", "[v]", *common,
], timeout=3600, check=True)
@@ -982,24 +1614,43 @@ def _aligned_input(s: Source, channel: int, start: float, duration: float) -> tu
return ["-ss", f"{max(0.0, src_start):.4f}", "-i", s.path], steps
-def _render_audio(session: MulticamSession, out: Path, start: float, duration: float) -> None:
+def _write_mix(session: MulticamSession, out: Path, start: float, duration: float, *,
+ per_feed: tuple[str, ...] = (), after: str = "anull", encode: tuple[str, ...],
+ feeds: Optional[list[tuple[Source, int]]] = None) -> None:
+ """Mix every mic (or the room, or just `feeds`) onto [start, start + duration] of the timeline."""
args: list[str] = []
chains: list[str] = []
- for i, (s, ch) in enumerate(_audio_inputs(session)):
+ for i, (s, ch) in enumerate(feeds if feeds is not None else _audio_inputs(session)):
inp, steps = _aligned_input(s, ch, start, duration)
args += inp
- chains.append(f"[{i}:a:0]{','.join(steps + ['dynaudnorm=f=250:g=15:p=0.9'])}[a{i}]")
+ chains.append(f"[{i}:a:0]{','.join([*steps, *per_feed])}[a{i}]")
n = len(chains)
mix = "".join(f"[a{i}]" for i in range(n)) + f"amix=inputs={n}:normalize=0," if n > 1 else "[a0]"
- graph = ";".join(chains) + f";{mix}loudnorm=I=-16:TP=-1.5:LRA=11,aformat=channel_layouts=stereo[out]"
proc_run([
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", *args,
- "-filter_complex", graph, "-map", "[out]", "-ar", "48000",
- "-c:a", "aac", "-b:a", "192k", str(out),
+ "-filter_complex", ";".join(chains) + f";{mix}{after}[out]", "-map", "[out]", *encode, str(out),
], timeout=7200, check=True)
-def _render_stems(session: MulticamSession, out_dir: Path, start: float, duration: float) -> list[str]:
+def _mix_key(session: MulticamSession) -> str:
+ """Changes whenever which mics are mixed, or where they sit, changes."""
+ feeds = [(s.id, ch, s.offset, s.speed, os.path.getmtime(s.path)) for s, ch in _audio_inputs(session)]
+ return hashlib.sha1(json.dumps(feeds, default=str).encode()).hexdigest()[:12]
+
+
+def _render_audio(session: MulticamSession, out: Path, start: float, duration: float) -> None:
+ _write_mix(
+ session, out, start, duration,
+ per_feed=("dynaudnorm=f=250:g=15:p=0.9",),
+ after="loudnorm=I=-16:TP=-1.5:LRA=11,aformat=channel_layouts=stereo",
+ encode=("-ar", "48000", "-c:a", "pcm_s16le"),
+ )
+
+
+def _render_stems(
+ session: MulticamSession, out_dir: Path, start: float, duration: float,
+ splice: Optional[list[tuple[float, float]]] = None,
+) -> list[str]:
"""One WAV per person; a person recorded across several files (recorder splits) gets them mixed in."""
stems = []
for pid in session.person_ids():
@@ -1015,32 +1666,142 @@ def _render_stems(session: MulticamSession, out_dir: Path, start: float, duratio
graph = ";".join(chains) + (f";{mix}amix=inputs={len(feeds)}:normalize=0[out]" if len(feeds) > 1 else ";[a0]anull[out]")
# Person ids are unique slugs, so two people never share a file name.
out = out_dir / f"{pid}.wav"
+ full = out.with_name(f"{pid}.full.wav") if splice else out
proc_run([
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error", *args,
- "-filter_complex", graph, "-map", "[out]", "-ar", "48000", "-c:a", "pcm_s24le", str(out),
+ "-filter_complex", graph, "-map", "[out]", "-ar", "48000", "-c:a", "pcm_s24le", str(full),
], timeout=3600, check=True)
+ if splice:
+ _splice_audio(full, out, splice, ["-c:a", "pcm_s24le"])
+ full.unlink(missing_ok=True)
stems.append(str(out))
return stems
-def shot_frames(session: MulticamSession, fps: float) -> list[tuple[Optional[str], int, int]]:
- """(source_id or None for a gap, first frame, frame count) on one grid from the first cut.
+@dataclass
+class Piece:
+ source_id: Optional[str] # None renders black: no camera covered this stretch
+ out_frame: int # first frame in the finished episode
+ frames: int
+ tl_start: float # timeline second shown on that first frame
+
+
+def kept_segments(session: MulticamSession) -> list[tuple[float, float]]:
+ """The episode range minus the removed stretches, in timeline seconds."""
+ start, end = session.cuts[0]["start"], session.cuts[-1]["end"]
+ kept, cursor = [], start
+ for r in session.removals:
+ a, b = max(start, r["start"]), min(end, r["end"])
+ if b <= cursor or a >= end:
+ continue
+ if a > cursor:
+ kept.append((cursor, a))
+ cursor = max(cursor, b)
+ if cursor < end:
+ kept.append((cursor, end))
+ return kept
+
+
+def render_plan(session: MulticamSession, fps: float) -> list[Piece]:
+ """Every shot, split at removals, laid end to end on one output frame grid.
- One grid over the whole episode keeps shot lengths from accumulating
- rounding drift against the audio.
+ One grid over the whole episode keeps piece lengths from accumulating
+ rounding drift against the audio; the MP4, the stems, and both editor
+ timelines are all built from this list so they agree frame for frame.
"""
- start = session.cuts[0]["start"]
- out: list[tuple[Optional[str], int, int]] = []
- cursor = 0
- for shot in session.cuts:
- f0 = round((shot["start"] - start) * fps)
- f1 = round((shot["end"] - start) * fps)
- if f0 > cursor:
- out.append((None, cursor, f0 - cursor))
- if f1 > max(f0, cursor):
- out.append((shot["source_id"], max(f0, cursor), f1 - max(f0, cursor)))
- cursor = f1
- return out
+ pieces: list[Piece] = []
+ out_t, out_f = 0.0, 0
+
+ def emit(source_id: Optional[str], a: float, b: float) -> None:
+ nonlocal out_t, out_f
+ out_end = out_t + (b - a)
+ f_end = round(out_end * fps)
+ if f_end > out_f:
+ pieces.append(Piece(source_id, out_f, f_end - out_f, a + (out_f / fps - out_t)))
+ out_f = f_end
+ out_t = out_end
+
+ for a, b in kept_segments(session):
+ t = a
+ for c in session.cuts:
+ if c["end"] <= t:
+ continue
+ if c["start"] >= b:
+ break
+ if c["start"] > t:
+ emit(None, t, c["start"])
+ t = c["start"]
+ e = min(c["end"], b)
+ emit(c["source_id"], t, e)
+ t = e
+ if t < b:
+ emit(None, t, b)
+ return pieces
+
+
+SPLICE_BATCH = 60
+FADE = 0.008
+
+
+def _splice_audio(src: Path, dst: Path, segments: list[tuple[float, float]], codec: list[str]) -> None:
+ """Keep only `segments` (seconds into src), with 8 ms fades so a cut never clicks."""
+ work = dst.parent
+ chunks = []
+ for k in range(0, len(segments), SPLICE_BATCH):
+ batch = segments[k:k + SPLICE_BATCH]
+ parts = []
+ for i, (a, b) in enumerate(batch):
+ fade = min(FADE, (b - a) / 2)
+ parts.append(
+ f"[0:a]atrim=start={a:.5f}:end={b:.5f},asetpts=PTS-STARTPTS,"
+ f"afade=t=in:d={fade:.4f},afade=t=out:st={b - a - fade:.5f}:d={fade:.4f}[s{i}]"
+ )
+ joined = "".join(f"[s{i}]" for i in range(len(batch)))
+ chunk = work / f"{dst.stem}.part{k // SPLICE_BATCH:04d}.wav"
+ proc_run([
+ "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", str(src),
+ "-filter_complex", ";".join(parts) + f";{joined}concat=n={len(batch)}:v=0:a=1[out]",
+ "-map", "[out]", *codec, str(chunk),
+ ], timeout=7200, check=True)
+ chunks.append(chunk)
+ listing = work / f"{dst.stem}.parts.txt"
+ listing.write_text("".join(f"file '{c.as_posix()}'\n" for c in chunks), encoding="utf-8")
+ proc_run([
+ "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-f", "concat", "-safe", "0",
+ "-i", str(listing), "-c", "copy", str(dst),
+ ], timeout=3600, check=True)
+ for c in chunks:
+ c.unlink(missing_ok=True)
+ listing.unlink(missing_ok=True)
+
+
+def set_removals(session: MulticamSession, removals: list) -> MulticamSession:
+ """Stretches cut out of the episode on every camera and mic, as [{start, end, reason?}]."""
+ if not isinstance(removals, list) or len(removals) > MAX_REMOVALS:
+ raise ValueError(f"removals must be a list of at most {MAX_REMOVALS} {{start, end}} stretches")
+ if not all(isinstance(r, dict) and {"start", "end"} <= r.keys() for r in removals):
+ raise ValueError("Each removal needs start and end")
+ limit = session.timeline_duration() or MAX_SECONDS
+ spans = sorted(
+ ((_number(r["start"], "start", -MAX_SECONDS, MAX_SECONDS), _number(r["end"], "end", -MAX_SECONDS, MAX_SECONDS), r)
+ for r in removals),
+ key=lambda x: (x[0], x[1]),
+ )
+ clean: list[dict] = []
+ for start, end, r in spans:
+ a, b = max(0.0, start), min(limit, end)
+ if b - a < 0.02:
+ continue
+ if clean and a <= clean[-1]["end"]:
+ clean[-1]["end"] = round(max(clean[-1]["end"], b), 3)
+ else:
+ clean.append({"start": round(a, 3), "end": round(b, 3), **({"reason": str(r["reason"])} if r.get("reason") else {})})
+ previous, session.removals = session.removals, clean
+ if session.cuts and not kept_segments(session):
+ session.removals = previous
+ raise ValueError("That would remove the whole episode")
+ session.save()
+ return session
def _render_key(session: MulticamSession, stems: bool) -> str:
@@ -1048,7 +1809,7 @@ def _render_key(session: MulticamSession, stems: bool) -> str:
used = {c["source_id"] for c in session.cuts} | {s.id for s, _ in _audio_inputs(session)}
files = [(s.id, s.offset, s.speed, s.channel_people, s.person, os.path.getmtime(s.path))
for s in session.sources if s.id in used and os.path.exists(s.path)]
- blob = json.dumps([session.cuts, session.look, stems, files], sort_keys=True, default=str)
+ blob = json.dumps([session.cuts, session.removals, session.look, stems, files], sort_keys=True, default=str)
return hashlib.sha1(blob.encode()).hexdigest()[:16]
@@ -1070,19 +1831,22 @@ def render_session(
out_dir = _output_dir(session)
work = Path(tempfile.mkdtemp(prefix="podcli_multicam_", dir=paths["working"] if os.path.isdir(paths["working"]) else None))
try:
- jobs = shot_frames(session, fps)
- total_frames = sum(n for _, _, n in jobs)
+ jobs = render_plan(session, fps)
+ total_frames = sum(p.frames for p in jobs)
duration = total_frames / fps
+ end = session.cuts[-1]["end"]
+ # Audio renders over the whole range, then the removed stretches come out.
+ splice = [(a - start, b - start) for a, b in kept_segments(session)] if session.removals else None
done = [0, 0]
lock = threading.Lock()
def run(item):
- i, (source_id, f0, n) = item
+ i, piece = item
chunk = work / f"shot-{i:05d}.mp4"
- cam = session.source(source_id) if source_id else None
- _render_shot(cam, chunk, start + f0 / fps, n, width, height, fps, session.look)
+ cam = session.source(piece.source_id) if piece.source_id else None
+ _render_shot(session, cam, chunk, piece.tl_start, piece.frames, width, height, fps, session.look)
with lock:
- done[0] += n
+ done[0] += piece.frames
done[1] += 1
_emit(progress_callback, 3 + 80 * done[0] / total_frames, f"Cut {done[1]} of {len(jobs)} shots")
return chunk
@@ -1092,8 +1856,11 @@ def render_session(
chunks = list(pool.map(run, enumerate(jobs)))
_emit(progress_callback, 84, "Mixing microphones")
- audio = work / "audio.m4a"
- _render_audio(session, audio, start, duration)
+ audio = work / "audio.wav"
+ _render_audio(session, audio, start, end - start)
+ if splice:
+ _splice_audio(audio, work / "kept.wav", splice, ["-c:a", "pcm_s16le"])
+ audio = work / "kept.wav"
_emit(progress_callback, 92, "Joining shots")
listing = work / "shots.txt"
@@ -1102,7 +1869,7 @@ def render_session(
proc_run([
"ffmpeg", "-y", "-hide_banner", "-loglevel", "error",
"-f", "concat", "-safe", "0", "-i", str(listing), "-i", str(audio),
- "-map", "0:v:0", "-map", "1:a:0", "-c", "copy", "-shortest",
+ "-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy", "-c:a", "aac", "-b:a", "192k", "-shortest",
"-movflags", "+faststart", str(partial),
], timeout=3600, check=True)
video = out_dir / "episode.mp4"
@@ -1112,7 +1879,7 @@ def render_session(
stem_paths = []
if stems:
_emit(progress_callback, 96, "Writing separate mic tracks")
- stem_paths = _render_stems(session, out_dir, start, duration)
+ stem_paths = _render_stems(session, out_dir, start, end - start, splice)
session.outputs = {
**session.outputs,
@@ -1135,6 +1902,9 @@ def export_xml(session: MulticamSession, fmt: str) -> str:
raise ValueError("Plan the cuts before exporting")
if fmt not in {"premiere", "fcpxml"}:
raise ValueError("Export format must be 'premiere' or 'fcpxml'")
+ if any(session.source(c["source_id"]).virtual for c in session.cuts):
+ raise ValueError("Editor timelines can't carry call layouts (tiles cut from one recording, or a split "
+ "screen) yet. Render the MP4 instead.")
width, height, fps = _output_format(session)
out_dir = _output_dir(session)
if fmt == "premiere":
@@ -1162,8 +1932,8 @@ def list_sessions() -> list[dict]:
items.append({
"session_id": s.session_id,
"name": s.name,
- "sources": len(s.sources),
- "cameras": len(s.cameras()),
+ "sources": len([x for x in s.sources if not x.virtual]),
+ "cameras": len([x for x in s.cameras() if not x.virtual]),
"synced": all(x.synced for x in s.sources if x.role != "ignore"),
"shots": len(s.cuts),
"video": s.outputs.get("video") if s.outputs.get("video") and os.path.exists(s.outputs["video"]) else None,
@@ -1178,18 +1948,43 @@ def delete_session(session_id: str) -> bool:
return True
+def source_label(session: MulticamSession, s: Source) -> str:
+ """A file's name, or what a virtual camera shows."""
+ if s.parent:
+ tiles = [v.id for v in session.sources if v.parent == s.parent]
+ return f"Tile {tiles.index(s.id) + 1} of {os.path.basename(session.source(s.parent).path)}"
+ if s.members:
+ names = {p.id: p.name for p in session.people}
+ return "Split: " + " + ".join(names.get(m, m) for m in s.members)
+ return os.path.basename(s.path)
+
+
def payload(session: MulticamSession) -> dict:
data = asdict(session)
data["timeline_duration"] = round(session.timeline_duration(), 3)
data["stats"] = cut_stats(session)
+ data["cut_settings"] = {**effective_cut(session), "style": session.cut_settings.get("style", "auto")}
+ data["cut_overrides"] = dict(session.cut_settings)
+ data["resolved_style"] = resolved_style(session)
+ data["auto_style"] = "remote" if any(c.virtual for c in session.cameras()) else "studio"
data["looks"] = list(LOOKS)
data["has_person_mics"] = bool(session.person_mics())
data["outputs"] = {
k: v for k, v in session.outputs.items()
if not isinstance(v, str) or not os.path.isabs(v) or os.path.exists(v)
}
+ proxies = {k: v for k, v in (session.preview.get("proxies") or {}).items() if os.path.exists(v)}
+ audio = session.preview.get("audio")
+ try:
+ current = os.path.basename(audio or "") == f"preview-{_mix_key(session)}.m4a"
+ except OSError:
+ current = False
+ # The mix file is named by the mic alignment, so a re-sync or remap leaves it stale.
+ ready = current and os.path.exists(audio) and all(c.id in proxies for c in session.cameras() if c.synced and not c.virtual)
+ stills = {k: v for k, v in (session.preview.get("stills") or {}).items() if os.path.exists(v)}
+ data["preview"] = {"proxies": proxies, "audio": audio, "stills": stills} if ready else None
for s in data["sources"]:
- s["name"] = os.path.basename(s["path"])
+ s["name"] = source_label(session, session.source(s["id"]))
s["timeline_start"] = s["offset"]
s["timeline_end"] = None if s["offset"] is None else round(s["offset"] + s["duration"] * s["speed"], 3)
return data
diff --git a/backend/services/multicam_signal.py b/backend/services/multicam_signal.py
index ca82400..22a47f9 100644
--- a/backend/services/multicam_signal.py
+++ b/backend/services/multicam_signal.py
@@ -276,14 +276,27 @@ def plan_cuts(
min_shot: float = 2.0,
max_shot: float = 30.0,
wide_insert: float = 4.0,
+ backchannel: float = 1.2,
+ guests: frozenset[str] = frozenset(),
+ hold_guest: bool = True,
+ host_solo: bool = True,
+ guest_min: float = 0.0,
+ guest_delay: float = 0.0,
) -> list[dict]:
"""Turn per-frame speaker labels into [{start, end, source_id}] on the timeline.
- Silence holds the previous shot; crosstalk goes wide. Shots shorter than
- min_shot fold into a neighbour whose camera also covers them, and a single
- shot longer than max_shot gets a wide_insert-second wide shot in the middle
- when a wide camera exists. Split camera files (part 1, part 2) are handled
- because every decision checks which camera actually covers the moment.
+ Silence holds the previous shot; crosstalk goes wide. A backchannel (an
+ "mm-hm", a laugh, a one-word interjection under backchannel seconds) inside
+ someone else's turn never cuts away from them. Shots shorter than min_shot
+ fold into a neighbour whose camera also covers them. A single shot longer
+ than max_shot gets a wide_insert-second wide shot in the middle when a wide
+ camera exists, except on a guest when hold_guest is set: a guest's answer
+ stays on the guest from start to finish. With host_solo off a host speaks
+ on the wide shot. A guest turn shorter than guest_min stays wide, and a
+ longer one opens wide for guest_delay seconds before going to the guest,
+ as a call recording cuts between the split screen and the guest. Split
+ camera files (part 1, part 2) are handled because every decision checks
+ which camera actually covers the moment.
"""
if not cameras or range_end <= range_start:
return []
@@ -309,16 +322,22 @@ def plan_cuts(
avail = labels[max(0, first):min(len(labels), last)]
lead = max(0, -first)
window[lead:lead + len(avail)] = avail
+ window = suppress_backchannels(window, backchannel)
targets: list[str] = []
+ speech: list[str] = []
previous = "wide" if wide else (people[0] if people else "wide")
for value in window:
if value == SILENT:
targets.append(previous)
+ speech.append("")
continue
current = "wide" if value == BOTH else people[value]
targets.append(current)
+ speech.append(current)
previous = current
+ if wide and (not host_solo or guest_min > 0 or guest_delay > 0):
+ targets = _call_layout(targets, speech, guests, host_solo, guest_min, guest_delay)
codes = {name: i for i, name in enumerate(dict.fromkeys(targets))}
names = list(codes)
shot_runs = _runs(np.array([codes[t] for t in targets], dtype=np.int32))
@@ -346,8 +365,9 @@ def plan_cuts(
broken: list[dict] = []
for shot in shots:
length = shot["end"] - shot["start"]
- is_wide = by_id[shot["source_id"]].person == "wide"
- if is_wide or length <= max_shot:
+ person = by_id[shot["source_id"]].person
+ held = hold_guest and person in guests
+ if person == "wide" or held or length <= max_shot:
broken.append(shot)
continue
# Spacing inserts at least a shortest shot apart keeps them from
@@ -376,6 +396,65 @@ def plan_cuts(
]
+# A guest's full-frame shot runs this far past their last word, then the
+# split returns before the next speaker, as the measured remote edits do.
+SOLO_TAIL = 1.0
+
+
+def _call_layout(targets: list[str], speech: list[str], guests: frozenset[str], host_solo: bool,
+ guest_min: float, guest_delay: float) -> list[str]:
+ """Send host turns and short guest turns to the wide shot; open long guest turns on it.
+
+ `targets` has pauses filled with the last speaker; `speech` keeps them
+ empty, so a guest turn is measured from their first word to their last,
+ and silence after an answer never counts toward it.
+ """
+ out = list(targets)
+ delay = int(guest_delay / FRAME_SECONDS)
+ tail = int(SOLO_TAIL / FRAME_SECONDS)
+ start = 0
+ for i in range(1, len(targets) + 1):
+ if i < len(targets) and targets[i] == targets[start]:
+ continue
+ who = targets[start]
+ if who in guests:
+ spoken = [k for k in range(start, i) if speech[k] == who]
+ first, last = (spoken[0], spoken[-1] + 1) if spoken else (start, start)
+ if (last - first) * FRAME_SECONDS < guest_min:
+ out[start:i] = ["wide"] * (i - start)
+ else:
+ solo_from, solo_to = min(i, first + delay), min(i, last + tail)
+ out[start:i] = ["wide"] * (i - start)
+ out[solo_from:solo_to] = [who] * max(0, solo_to - solo_from)
+ elif who != "wide" and not host_solo:
+ out[start:i] = ["wide"] * (i - start)
+ start = i
+ return out
+
+
+def suppress_backchannels(labels: np.ndarray, backchannel: float) -> np.ndarray:
+ """Relabel short interjections inside one person's turn as that person.
+
+ A run shorter than `backchannel` seconds that sits between two stretches of
+ the same speaker (silence in between doesn't count) belongs to that
+ speaker's turn: a listener's "yeah" or a shared laugh shouldn't pull the
+ camera off someone mid-answer. Both stretches must be at least that long
+ themselves, so a blip inside real crosstalk doesn't swallow it.
+ """
+ limit = int(backchannel / FRAME_SECONDS)
+ if limit <= 0 or len(labels) == 0:
+ return labels
+ out = labels.copy()
+ runs = [r for r in _runs(labels) if r[0] != SILENT]
+ for k in range(1, len(runs) - 1):
+ value, a, b = runs[k]
+ (before, a0, b0), (after, a1, b1) = runs[k - 1], runs[k + 1]
+ if (b - a < limit and before == after and before >= 0 and value != before
+ and b0 - a0 >= limit and b1 - a1 >= limit):
+ out[a:b] = before
+ return out
+
+
def _merge_neighbours(shots: list[dict]) -> list[dict]:
merged: list[dict] = []
for s in shots:
diff --git a/src/server.ts b/src/server.ts
index 634ebb9..2999239 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -2230,20 +2230,23 @@ export function createServer(): McpServer {
"Flow: 'new' (folder or files, people) → check the guessed mapping with 'show' and fix it with 'map' → 'sync' → 'plan' → 'render' or 'export'. " +
"'new' returns the session with guessed roles; calling it again on the same files reopens that edit. " +
"'sync', 'plan' and 'render' start a background job and return job_id: poll job_status, then call 'show'. " +
- "Mapping fields (people, sources, range_start, range_end, cut_settings, speaker_map, look) apply on 'map', 'sync', 'plan' and 'render'. " +
+ "Mapping fields (people, sources, range_start, range_end, cut_settings, speaker_map, look, removals) apply on 'map', 'sync' and 'plan'; 'render' takes only look and stems. " +
"Changing who is in a file or where it sits clears the cut, so run 'plan' again. 'render' skips work when nothing changed. " +
- "Other actions: 'list', 'cut' (index, source_id: swap one shot's camera), 'previews' (still frames per camera, looks: true adds color-look stills), 'delete'.",
+ "Other actions: 'list', 'cut' (index, source_id: swap one shot's camera), 'set_cuts' (cuts: replace the whole cut with back-to-back shots), " +
+ "'activity' (who speaks when, as spans per person), 'previews' (still frames per camera, looks: true adds color-look stills), " +
+ "'preview' (background job: playback proxies, a mic mix and stills, for a browser editor such as podcli cloud), 'delete'. " +
+ "Call recordings work too: one file per person becomes a split screen, one gallery recording is split into a camera per tile; Premiere and FCPXML export refuse those layouts for now.",
{
action: z
- .enum(["new", "list", "show", "map", "sync", "plan", "cut", "previews", "render", "export", "delete"])
+ .enum(["new", "list", "show", "map", "sync", "plan", "cut", "set_cuts", "activity", "previews", "preview", "render", "export", "delete"])
.describe("What to do"),
session_id: z.string().optional().describe("Session id returned by 'new' (every action except new/list)"),
folder: z.string().optional().describe("For 'new': folder holding one episode's recordings, scanned recursively"),
files: z.array(z.string()).optional().describe("For 'new': explicit media file paths, alone or with folder"),
people: z
- .array(z.union([z.string(), z.object({ id: z.string().optional(), name: z.string() })]))
+ .array(z.union([z.string(), z.object({ id: z.string().optional(), name: z.string(), role: z.enum(["host", "guest"]).optional() })]))
.optional()
- .describe("For 'new': speaker names (default Host, Guest). For 'map': the full people list, with ids to keep"),
+ .describe("For 'new': speaker names, or {name, role} (default Host, Guest; without roles the last one is the guest). For 'map': the full people list, with ids to keep and role host or guest; a guest's long answers are held on their camera"),
name: z.string().optional().describe("For 'new': episode name"),
sources: z
.array(
@@ -2262,9 +2265,15 @@ export function createServer(): McpServer {
range_end: z.number().nullable().optional().describe("For 'map'/'plan': episode end on the timeline, null for automatic"),
cut_settings: z
.object({
- min_shot: z.number().optional().describe("Shortest shot in seconds (default 2)"),
- max_shot: z.number().optional().describe("Break a longer single-speaker shot with a wide shot; 0 disables (default 30)"),
+ min_shot: z.number().optional().describe("Shortest shot in seconds (studio 2, remote 4)"),
+ max_shot: z.number().optional().describe("Break a longer single-host shot with a wide shot; 0 disables (studio 30, remote 0)"),
wide_insert: z.number().optional().describe("Length of that wide shot in seconds (default 4)"),
+ backchannel: z.number().optional().describe("Interjections shorter than this many seconds inside someone's turn never cut away (default 1.2)"),
+ hold_guest: z.boolean().optional().describe("Never cut away from a guest mid-answer (default true)"),
+ style: z.enum(["auto", "studio", "remote"]).optional().describe("studio: everyone's camera. remote: split screen, guest full frame on long answers. auto picks remote for call recordings; a new style resets the other settings to its defaults"),
+ host_solo: z.boolean().optional().describe("Give hosts their own camera when they talk (studio true, remote false)"),
+ guest_min: z.number().optional().describe("Guest turns shorter than this stay on the wide shot (remote 8 s)"),
+ guest_delay: z.number().optional().describe("A long guest answer opens on the wide shot for this long first (remote 4 s)"),
})
.optional(),
speaker_map: z.record(z.string(), z.string()).optional().describe("For shared-audio shows: diarization label → person id"),
@@ -2274,7 +2283,16 @@ export function createServer(): McpServer {
format: z.enum(["premiere", "fcpxml"]).optional().describe("For 'export': premiere (FCP7 XML, also opens in Resolve) or fcpxml (Final Cut Pro, Resolve)"),
index: z.number().int().min(0).optional().describe("For 'cut': 0-based shot index"),
source_id: z.string().optional().describe("For 'cut': camera source id to use for that shot"),
+ removals: z
+ .array(z.object({ start: z.number(), end: z.number(), reason: z.string().optional() }))
+ .optional()
+ .describe("Stretches cut out of the episode on every camera and mic, in timeline seconds; [] restores everything"),
+ cuts: z
+ .array(z.object({ start: z.number(), end: z.number(), source_id: z.string() }))
+ .optional()
+ .describe("For 'set_cuts': the full cut, shots back to back on the timeline"),
looks: z.boolean().optional().describe("For 'previews': include one still per color look"),
+ at: z.number().optional().describe("For 'previews': timeline second to take the stills at"),
},
async (params) => {
try {
diff --git a/src/ui/client/multicam-auto-cut.tsx b/src/ui/client/multicam-auto-cut.tsx
new file mode 100644
index 0000000..47fa876
--- /dev/null
+++ b/src/ui/client/multicam-auto-cut.tsx
@@ -0,0 +1,94 @@
+import React, { useState } from "react";
+import { fmt } from "./lib";
+import type { McCutSettings, McSession, McStyle } from "./multicam-types";
+
+// Mirrors the clamps in multicam.update_mapping so the inputs never show a value the server won't use.
+const LIMITS = {
+ min_shot: { min: 0.5, max: 10, step: 0.5 },
+ max_shot: { min: 0, max: 600, step: 1 },
+ wide_insert: { min: 1, max: 15, step: 0.5 },
+ backchannel: { min: 0, max: 5, step: 0.1 },
+ guest_min: { min: 0, max: 60, step: 1 },
+ guest_delay: { min: 0, max: 30, step: 0.5 },
+};
+
+export default function MulticamAutoCut({
+ session,
+ locked,
+ onRecut,
+}: {
+ session: McSession;
+ locked: boolean;
+ onRecut: (extra: Record<string, unknown>) => void;
+}) {
+ const [open, setOpen] = useState(false);
+ const [range, setRange] = useState({ range_start: session.range_start ?? 0, range_end: session.range_end ?? session.timeline_duration });
+ const [settings, setSettings] = useState(session.cut_settings);
+
+ const field = (label: string, value: number, onChange: (v: number) => void, limits = { min: 0, max: session.timeline_duration, step: 0.1 }) => (
+ <label style={{ display: "block" }}>
+ <span className="field-label">{label}</span>
+ <input type="number" {...limits} value={value} onChange={(e) => onChange(Number(e.target.value))} style={{ width: "100%" }} />
+ </label>
+ );
+
+ return (
+ <div style={{ marginTop: 12 }}>
+ <button className="btn btn-ghost btn-sm" onClick={() => setOpen((v) => !v)}>
+ {open ? "Hide auto-cut settings" : "Auto-cut settings"}
+ </button>
+ {open && (
+ <>
+ <div className="row" style={{ margin: "12px 0" }}>
+ {field(`Episode start (${fmt(range.range_start)})`, range.range_start, (v) => setRange({ ...range, range_start: v }))}
+ {field(`Episode end (${fmt(range.range_end)})`, range.range_end, (v) => setRange({ ...range, range_end: v }))}
+ {field("Shortest shot (s)", settings.min_shot, (v) => setSettings({ ...settings, min_shot: v }), LIMITS.min_shot)}
+ {field("Wide cutaway after (s, 0 = never)", settings.max_shot, (v) => setSettings({ ...settings, max_shot: v }), LIMITS.max_shot)}
+ {field("Wide cutaway length (s)", settings.wide_insert, (v) => setSettings({ ...settings, wide_insert: v }), LIMITS.wide_insert)}
+ {field("Ignore interjections under (s)", settings.backchannel, (v) => setSettings({ ...settings, backchannel: v }), LIMITS.backchannel)}
+ {field("Guest answers stay wide under (s)", settings.guest_min, (v) => setSettings({ ...settings, guest_min: v }), LIMITS.guest_min)}
+ {field("Open long answers wide for (s)", settings.guest_delay, (v) => setSettings({ ...settings, guest_delay: v }), LIMITS.guest_delay)}
+ </div>
+ <div style={{ display: "flex", gap: 18, flexWrap: "wrap", alignItems: "center", marginBottom: 12 }}>
+ <label className="hint" style={{ display: "inline-flex", alignItems: "center", gap: 8 }}>
+ Style
+ <select
+ value={settings.style}
+ onChange={(e) => setSettings({ ...settings, style: e.target.value as McStyle })}
+ style={{ width: "auto", padding: "4px 28px 4px 10px", fontSize: 12 }}
+ >
+ <option value="auto">Automatic ({session.auto_style === "remote" ? "remote call" : "studio"})</option>
+ <option value="studio">Studio cameras</option>
+ <option value="remote">Remote call</option>
+ </select>
+ </label>
+ <label className="hint" style={{ display: "inline-flex", alignItems: "center", gap: 8 }}>
+ <input type="checkbox" checked={settings.hold_guest} onChange={(e) => setSettings({ ...settings, hold_guest: e.target.checked })} />
+ Stay on the guest for their whole answer
+ </label>
+ <label className="hint" style={{ display: "inline-flex", alignItems: "center", gap: 8 }}>
+ <input type="checkbox" checked={settings.host_solo} onChange={(e) => setSettings({ ...settings, host_solo: e.target.checked })} />
+ Give hosts their own shot
+ </label>
+ </div>
+ <div style={{ display: "flex", justifyContent: "flex-end" }}>
+ <button
+ className="btn btn-primary btn-sm"
+ disabled={locked}
+ onClick={() => {
+ // Only what was changed is sent: anything else follows the style's own defaults.
+ const changed = Object.fromEntries(
+ Object.entries(settings).filter(([k, v]) => v !== session.cut_settings[k as keyof McCutSettings]),
+ );
+ const rangeChanged = range.range_start !== (session.range_start ?? 0) || range.range_end !== (session.range_end ?? session.timeline_duration);
+ onRecut({ ...(rangeChanged ? range : {}), cut_settings: changed });
+ }}
+ >
+ Cut again
+ </button>
+ </div>
+ </>
+ )}
+ </div>
+ );
+}
diff --git a/src/ui/client/multicam-cut.tsx b/src/ui/client/multicam-cut.tsx
deleted file mode 100644
index 5f8ad1a..0000000
--- a/src/ui/client/multicam-cut.tsx
+++ /dev/null
@@ -1,139 +0,0 @@
-import React, { useEffect, useRef, useState } from "react";
-import { fmt } from "./lib";
-import type { McEdit, McSession } from "./multicam-types";
-import { cameraColorMap, mcStreamUrl, whoLabel } from "./multicam-types";
-
-// Mirrors the clamps in multicam.update_mapping so the inputs never show a value the server won't use.
-const LIMITS = {
- min_shot: { min: 0.5, max: 10, step: 0.5 },
- max_shot: { min: 0, max: 600, step: 1 },
- wide_insert: { min: 1, max: 15, step: 0.5 },
-};
-
-export default function MulticamCut({
- session,
- locked,
- onEdit,
- onRecut,
-}: {
- session: McSession;
- locked: boolean;
- onEdit: McEdit;
- onRecut: (extra: Record<string, unknown>) => void;
-}) {
- const [selected, setSelected] = useState(0);
- const videoRef = useRef<HTMLVideoElement>(null);
-
- const cameras = session.sources.filter((s) => s.role === "camera" && s.offset !== null);
- const colors = cameraColorMap(cameras.map((c) => c.id));
- const index = Math.min(selected, session.cuts.length - 1);
- const shot = session.cuts[index];
- const shotCamera = cameras.find((c) => c.id === shot?.source_id);
-
- useEffect(() => {
- if (!shot || !shotCamera || !videoRef.current) return;
- videoRef.current.currentTime = Math.max(0, (shot.start - (shotCamera.timeline_start ?? 0)) / (shotCamera.speed || 1));
- }, [index, shot?.start, shot?.source_id]);
-
- return (
- <div className="section card">
- <div className="card-title" style={{ marginBottom: 4 }}>Cut</div>
- <p className="card-desc">Cameras follow whoever is speaking. Click a shot to swap its camera.</p>
-
- {!session.has_person_mics && (
- <div className="hint" style={{ marginBottom: 12 }}>
- No personal mics mapped, so cuts follow speaker detection on the shared audio. Map one mic per person for better cuts.
- </div>
- )}
-
- <div style={{ display: "flex", height: 36, borderRadius: "var(--radius-sm)", overflow: "hidden", border: "1px solid var(--border)" }}>
- {session.cuts.map((c, i) => (
- <button
- key={`${c.start}-${c.source_id}`}
- aria-label={`Shot ${i + 1}, ${fmt(c.start)}-${fmt(c.end)}`}
- onClick={() => setSelected(i)}
- style={{
- flex: `${Math.max(0.2, c.end - c.start)} 0 0`, minWidth: 0, padding: 0, border: "none",
- background: colors[c.source_id] || "var(--text3)", opacity: i === index ? 1 : 0.55, cursor: "pointer",
- borderRight: "1px solid var(--bg)",
- }}
- />
- ))}
- </div>
-
- <div style={{ display: "flex", flexWrap: "wrap", gap: 12, margin: "10px 0 16px" }}>
- {cameras.map((cam) => (
- <span key={cam.id} className="hint" style={{ display: "inline-flex", alignItems: "center", gap: 6 }}>
- <span style={{ width: 10, height: 10, borderRadius: 3, background: colors[cam.id] }} />
- {whoLabel(session.people, cam)} {Math.round((session.stats.share?.[cam.id] || 0) * 100)}%
- </span>
- ))}
- <span className="hint" style={{ marginLeft: "auto" }}>
- {session.stats.shots} shots · average {(session.stats.average_shot || 0).toFixed(1)}s · {fmt(session.stats.duration || 0)}
- </span>
- </div>
-
- {shot && (
- <div className="card" style={{ padding: 14, marginBottom: 16 }}>
- <div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 10, flexWrap: "wrap" }}>
- <strong style={{ fontVariantNumeric: "tabular-nums" }}>Shot {index + 1} · {fmt(shot.start)}-{fmt(shot.end)}</strong>
- <div style={{ display: "flex", gap: 6, marginLeft: "auto", flexWrap: "wrap" }}>
- {cameras.map((c) => (
- <button
- key={c.id}
- className={`btn btn-sm ${c.id === shot.source_id ? "btn-primary" : "btn-ghost"}`}
- disabled={locked || c.id === shot.source_id}
- onClick={() => onEdit({ action: "cut", index, source_id: c.id })}
- >
- {whoLabel(session.people, c)}
- </button>
- ))}
- </div>
- </div>
- {shotCamera && (
- <video ref={videoRef} src={mcStreamUrl(shotCamera.path)} controls style={{ width: "100%", borderRadius: "var(--radius-sm)", background: "#000" }} />
- )}
- </div>
- )}
-
- <CutSettings key={JSON.stringify([session.range_start, session.range_end, session.cut_settings])} session={session} locked={locked} onRecut={onRecut} />
- </div>
- );
-}
-
-function CutSettings({ session, locked, onRecut }: { session: McSession; locked: boolean; onRecut: (extra: Record<string, unknown>) => void }) {
- const [open, setOpen] = useState(false);
- const [range, setRange] = useState({ range_start: session.range_start ?? 0, range_end: session.range_end ?? session.timeline_duration });
- const [settings, setSettings] = useState(session.cut_settings);
-
- const field = (label: string, value: number, onChange: (v: number) => void, limits = { min: 0, max: session.timeline_duration, step: 0.1 }) => (
- <div>
- <label className="field-label">{label}</label>
- <input type="number" {...limits} value={value} onChange={(e) => onChange(Number(e.target.value))} style={{ width: "100%" }} />
- </div>
- );
-
- return (
- <>
- <button className="btn btn-ghost btn-sm" onClick={() => setOpen((v) => !v)}>
- {open ? "Hide cut settings" : "Cut settings"}
- </button>
- {open && (
- <>
- <div className="row" style={{ margin: "12px 0" }}>
- {field(`Episode start (${fmt(range.range_start)})`, range.range_start, (v) => setRange({ ...range, range_start: v }))}
- {field(`Episode end (${fmt(range.range_end)})`, range.range_end, (v) => setRange({ ...range, range_end: v }))}
- {field("Shortest shot (s)", settings.min_shot, (v) => setSettings({ ...settings, min_shot: v }), LIMITS.min_shot)}
- {field("Wide cutaway after (s, 0 = never)", settings.max_shot, (v) => setSettings({ ...settings, max_shot: v }), LIMITS.max_shot)}
- {field("Wide cutaway length (s)", settings.wide_insert, (v) => setSettings({ ...settings, wide_insert: v }), LIMITS.wide_insert)}
- </div>
- <div style={{ display: "flex", justifyContent: "flex-end" }}>
- <button className="btn btn-primary btn-sm" disabled={locked} onClick={() => onRecut({ ...range, cut_settings: settings })}>
- Recut
- </button>
- </div>
- </>
- )}
- </>
- );
-}
diff --git a/src/ui/client/multicam-deliver.tsx b/src/ui/client/multicam-deliver.tsx
index e4cf0a1..e8d7c2c 100644
--- a/src/ui/client/multicam-deliver.tsx
+++ b/src/ui/client/multicam-deliver.tsx
@@ -6,17 +6,34 @@ import { mcFileUrl, mcStreamUrl } from "./multicam-types";
const linkStyle = { textDecoration: "none", display: "inline-flex", alignItems: "center", gap: 6 } as const;
-export default function MulticamDeliver({
- session,
+export function DeliverActions({
locked,
+ exportable,
onRender,
onExport,
- onMakeClips,
}: {
- session: McSession;
locked: boolean;
+ exportable: boolean;
onRender: () => void;
onExport: (format: "premiere" | "fcpxml") => void;
+}) {
+ const why = exportable ? undefined : "Editor timelines can't carry call layouts yet. Render the MP4 instead.";
+ return (
+ <>
+ <button className="btn btn-ghost btn-sm" disabled={locked || !exportable} title={why} onClick={() => onExport("premiere")}>Export for Premiere</button>
+ <button className="btn btn-ghost btn-sm" disabled={locked || !exportable} title={why} onClick={() => onExport("fcpxml")}>Export for Final Cut</button>
+ <button className="btn btn-primary btn-sm" disabled={locked} onClick={onRender}>Render episode</button>
+ </>
+ );
+}
+
+export default function MulticamResults({
+ session,
+ locked,
+ onMakeClips,
+}: {
+ session: McSession;
+ locked: boolean;
onMakeClips: () => void;
}) {
const { video, stems = [], premiere, fcpxml, duration } = session.outputs;
@@ -26,18 +43,11 @@ export default function MulticamDeliver({
...(premiere ? [{ path: premiere, label: "Premiere XML" }] : []),
...(fcpxml ? [{ path: fcpxml, label: "Final Cut XML" }] : []),
];
+ if (!downloads.length) return null;
return (
<div className="section card">
- <div className="card-title" style={{ marginBottom: 4 }}>Deliver</div>
- <p className="card-desc">Render the finished MP4, or hand the cut to Premiere or Final Cut as a timeline of your original files.</p>
-
- <div style={{ display: "flex", gap: 10, flexWrap: "wrap" }}>
- <button className="btn btn-primary" disabled={locked} onClick={onRender}>Render episode</button>
- <button className="btn btn-ghost" disabled={locked} onClick={() => onExport("premiere")}>Export for Premiere</button>
- <button className="btn btn-ghost" disabled={locked} onClick={() => onExport("fcpxml")}>Export for Final Cut</button>
- </div>
-
+ <div className="card-title" style={{ marginBottom: 12 }}>Finished files</div>
{video && (
// The file name never changes between renders; the duration in the URL
// makes the browser fetch the new render instead of its cached copy.
@@ -45,21 +55,19 @@ export default function MulticamDeliver({
key={`${video}-${duration}`}
controls
src={`${mcStreamUrl(video)}&v=${duration}`}
- style={{ width: "100%", borderRadius: "var(--radius-sm)", background: "#000", margin: "20px 0 12px" }}
+ style={{ width: "100%", borderRadius: "var(--radius-sm)", background: "#000", marginBottom: 12 }}
/>
)}
- {downloads.length > 0 && (
- <div style={{ display: "flex", flexWrap: "wrap", gap: 8, alignItems: "center", marginTop: video ? 0 : 16 }}>
- {downloads.map((d) => (
- <a key={d.path} className="btn btn-ghost btn-sm" href={mcFileUrl(d.path)} style={linkStyle}>
- <DownloadIcon size={13} /> {d.label}
- </a>
- ))}
- {video && (
- <button className="btn btn-primary btn-sm" disabled={locked} onClick={onMakeClips}>Make clips from this episode</button>
- )}
- </div>
- )}
+ <div style={{ display: "flex", flexWrap: "wrap", gap: 8, alignItems: "center" }}>
+ {downloads.map((d) => (
+ <a key={d.path} className="btn btn-ghost btn-sm" href={mcFileUrl(d.path)} style={linkStyle}>
+ <DownloadIcon size={13} /> {d.label}
+ </a>
+ ))}
+ {video && (
+ <button className="btn btn-primary btn-sm" disabled={locked} onClick={onMakeClips}>Make clips from this episode</button>
+ )}
+ </div>
</div>
);
}
diff --git a/src/ui/client/multicam-look.tsx b/src/ui/client/multicam-look.tsx
deleted file mode 100644
index a8de9a7..0000000
--- a/src/ui/client/multicam-look.tsx
+++ /dev/null
@@ -1,43 +0,0 @@
-import React from "react";
-import type { McEdit, McSession } from "./multicam-types";
-import { mcImageUrl } from "./multicam-types";
-
-export default function MulticamLook({
- session,
- lookPreviews,
- locked,
- onEdit,
-}: {
- session: McSession;
- lookPreviews: Record<string, string>;
- locked: boolean;
- onEdit: McEdit;
-}) {
- return (
- <div className="section card">
- <div className="card-title" style={{ marginBottom: 4 }}>Look</div>
- <p className="card-desc">Pick a color grade for the whole episode.</p>
-
- <div style={{ display: "flex", gap: 12, flexWrap: "wrap" }}>
- {session.looks.map((look) => (
- <button
- key={look}
- className="card"
- disabled={locked}
- aria-pressed={session.look === look}
- onClick={() => onEdit({ action: "map", look })}
- style={{
- margin: 0, padding: 8, width: 160, cursor: locked ? "default" : "pointer", color: "var(--text)",
- borderColor: session.look === look ? "var(--accent)" : "var(--border)",
- }}
- >
- <div style={{ width: "100%", aspectRatio: "16/9", borderRadius: "var(--radius-sm)", background: "var(--surface2)", overflow: "hidden", marginBottom: 8 }}>
- {lookPreviews[look] && <img src={mcImageUrl(lookPreviews[look])} alt="" style={{ width: "100%", height: "100%", objectFit: "cover" }} />}
- </div>
- <div style={{ fontSize: 13, fontWeight: 600, textAlign: "center" }}>{look[0].toUpperCase() + look.slice(1)}</div>
- </button>
- ))}
- </div>
- </div>
- );
-}
diff --git a/src/ui/client/multicam-page.tsx b/src/ui/client/multicam-page.tsx
index e2ab859..57aacc0 100644
--- a/src/ui/client/multicam-page.tsx
+++ b/src/ui/client/multicam-page.tsx
@@ -7,20 +7,19 @@ import { BackIcon } from "./icons";
import MulticamStart from "./multicam-start";
import MulticamSources from "./multicam-sources";
import MulticamSync from "./multicam-sync";
-import MulticamCut from "./multicam-cut";
-import MulticamLook from "./multicam-look";
-import MulticamDeliver from "./multicam-deliver";
-import type { McJobKind, McPreviews, McPreviewsResp, McSession, McSessionSummary } from "./multicam-types";
-
-const emptyPreviews: McPreviews = { cameras: {}, looks: {} };
+import MulticamSummary from "./multicam-summary";
+import MulticamResults, { DeliverActions } from "./multicam-deliver";
+import type { McJobKind, McPreviewsResp, McSession, McSessionSummary } from "./multicam-types";
const JOB_LABEL: Record<McJobKind, string> = {
sync: "Syncing",
plan: "Cutting",
render: "Rendering",
+ preview: "Preparing previews",
};
const errorText = (e: unknown) => (e instanceof Error ? e.message : String(e));
+const post = <T,>(body: Record<string, unknown>) => api<T>("/multicam", { method: "POST", body: JSON.stringify(body) });
export default function MulticamPage() {
const navigate = useNavigate();
@@ -29,22 +28,22 @@ export default function MulticamPage() {
const [sessions, setSessions] = useState<McSessionSummary[]>([]);
const [session, setSession] = useState<McSession | null>(null);
- const [previews, setPreviews] = useState<McPreviews>(emptyPreviews);
+ const [stills, setStills] = useState<Record<string, string>>({});
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const [job, setJob] = useState<{ id: string; kind: McJobKind } | null>(null);
const jobState = useJob(job?.id ?? null);
const [peopleDraft, setPeopleDraft] = useState<string[]>(["Host", "Guest"]);
+ const [sourcesOpen, setSourcesOpen] = useState(false);
- // One lock for the whole page: a running job owns the session until it
- // finishes, so edits can't race it and get overwritten.
+ // A running job owns the session until it finishes, so edits can't race it and get overwritten.
const locked = busy || job !== null;
async function call<T = McSession>(body: Record<string, unknown>): Promise<T | null> {
setBusy(true);
setError(null);
try {
- return await api<T>("/multicam", { method: "POST", body: JSON.stringify(body) });
+ return await post<T>(body);
} catch (e) {
setError(errorText(e));
return null;
@@ -63,40 +62,37 @@ export default function MulticamPage() {
session && call({ ...body, session_id: session.session_id }).then(adopt);
function refreshList() {
- api<{ sessions: McSessionSummary[] }>("/multicam", { method: "POST", body: JSON.stringify({ action: "list" }) })
- .then((r) => setSessions(r.sessions))
- .catch(() => {});
+ post<{ sessions: McSessionSummary[] }>({ action: "list" }).then((r) => setSessions(r.sessions)).catch(() => {});
}
- useEffect(() => {
- refreshList();
- }, []);
+ function startJob(kind: McJobKind, extra: Record<string, unknown> = {}, sessionId = session?.session_id) {
+ if (!sessionId) return;
+ call<{ job_id: string }>({ action: kind, session_id: sessionId, ...extra }).then((r) => {
+ if (r) setJob({ id: r.job_id, kind });
+ });
+ }
+
+ useEffect(refreshList, []);
useEffect(() => {
+ setStills({});
+ setSourcesOpen(false);
if (!openId) {
setSession(null);
setJob(null);
- setPreviews(emptyPreviews);
return;
}
- if (session?.session_id !== openId) {
- setPreviews(emptyPreviews);
- call({ action: "show", session_id: openId }).then(adopt);
- }
+ if (session?.session_id !== openId) call({ action: "show", session_id: openId }).then(adopt);
}, [openId]);
const cutReady = !!session && session.cuts.length > 0;
+
useEffect(() => {
if (!session) return;
- const wantLooks = cutReady && Object.keys(previews.looks).length === 0;
- if (Object.keys(previews.cameras).length > 0 && !wantLooks) return;
- api<McPreviewsResp>("/multicam", {
- method: "POST",
- body: JSON.stringify({ action: "previews", session_id: session.session_id, looks: cutReady }),
- })
- .then((r) => setPreviews(r.previews))
+ post<McPreviewsResp>({ action: "previews", session_id: session.session_id })
+ .then((r) => setStills(r.previews.cameras))
.catch(() => {});
- }, [session?.session_id, cutReady]);
+ }, [session?.session_id, session?.sources.map((s) => s.id).join()]);
useEffect(() => {
if (!job || !jobState) return;
@@ -115,31 +111,19 @@ export default function MulticamPage() {
if (job.kind === "sync" && allSynced) startJob("plan", {}, result.session_id);
}, [jobState?.status]);
- function startJob(kind: McJobKind, extra: Record<string, unknown> = {}, sessionId = session?.session_id) {
- if (!sessionId) return;
- call<{ job_id: string }>({ action: kind, session_id: sessionId, ...extra }).then((r) => {
- if (r) setJob({ id: r.job_id, kind });
- });
- }
-
async function startSession(seed: Record<string, unknown>): Promise<boolean> {
const r = await call({ action: "new", ...seed, people: peopleDraft.filter((n) => n.trim()) });
if (!r) return false;
- setSession(r);
+ adopt(r);
setSearchParams({ session: r.session_id });
refreshList();
return true;
}
- function closeSession() {
- setError(null);
- setSearchParams({});
- }
-
async function deleteSession(id: string) {
if (!window.confirm("Delete this multicam edit? Your recordings stay where they are.")) return;
if (await call({ action: "delete", session_id: id })) {
- if (session?.session_id === id) closeSession();
+ if (session?.session_id === id) setSearchParams({});
refreshList();
}
}
@@ -181,13 +165,38 @@ export default function MulticamPage() {
jobState?.progress ? ` (${Math.round(jobState.progress)}%)` : ""
}`
: null;
- const syncStarted = !!session?.sources.some((s) => s.sync?.status);
+ const synced = !!session?.sources.some((s) => s.sync?.status);
+ const unsynced = session?.sources.filter((s) => s.role !== "ignore" && s.offset === null) || [];
+ const showSources = !session || !cutReady || sourcesOpen || unsynced.length > 0;
return (
<div className="app">
- <PageHeader title="Multicam edit" />
+ <PageHeader
+ title={session ? session.name : "Multicam edit"}
+ back={session && (
+ <button className="btn btn-ghost btn-sm" disabled={busy} onClick={() => setSearchParams({})} style={{ display: "inline-flex", alignItems: "center", gap: 6 }}>
+ <BackIcon /> All multicam edits
+ </button>
+ )}
+ actions={session && cutReady && (
+ <DeliverActions
+ locked={locked}
+ exportable={!session.cuts.some((c) => {
+ const cam = session.sources.find((s) => s.id === c.source_id);
+ return !!cam && (!!cam.parent || cam.members.length > 0);
+ })}
+ onRender={() => startJob("render", { stems: true })}
+ onExport={(format) => mutate({ action: "export", format })}
+ />
+ )}
+ />
{error && <div className="set-note err" style={{ marginBottom: 16 }}>{error}</div>}
+ {progress && (
+ <div className="set-note" style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 16 }}>
+ <div className="spinner sm" /> {progress}
+ </div>
+ )}
{!session ? (
<MulticamStart
@@ -201,42 +210,30 @@ export default function MulticamPage() {
/>
) : (
<div className="stream-in">
- <button className="btn btn-ghost btn-sm" onClick={closeSession} style={{ display: "inline-flex", alignItems: "center", gap: 6, marginBottom: 16 }}>
- <BackIcon /> All multicam edits
- </button>
-
- {progress && (
- <div className="set-note" style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 16 }}>
- <div className="spinner sm" /> {progress}
+ {!showSources && (
+ <div className="section card" style={{ display: "flex", alignItems: "center", gap: 12, padding: "12px 16px" }}>
+ <span className="pill pill-green">Synced</span>
+ <span className="hint">
+ {session.sources.filter((s) => s.role !== "ignore" && !s.parent && !s.members.length).length} files · {session.people.map((p) => p.name).join(", ")}
+ </span>
+ <button className="btn btn-ghost btn-sm" style={{ marginLeft: "auto" }} onClick={() => setSourcesOpen(true)}>
+ Edit sources
+ </button>
</div>
)}
-
- <MulticamSources
- session={session}
- previewCameras={previews.cameras}
- locked={locked}
- onEdit={mutate}
- onSync={onSync}
- />
- {syncStarted && (
- <MulticamSync session={session} locked={locked} onEdit={mutate} onCut={() => startJob("plan")} />
+ {showSources && (
+ <>
+ <MulticamSources session={session} previewCameras={stills} locked={locked} onEdit={mutate} onSync={onSync} />
+ {synced && <MulticamSync session={session} locked={locked} onEdit={mutate} onCut={() => startJob("plan")} />}
+ {cutReady && sourcesOpen && (
+ <button className="btn btn-ghost btn-sm" style={{ marginBottom: 16 }} onClick={() => setSourcesOpen(false)}>Hide sources</button>
+ )}
+ </>
)}
{cutReady && (
<>
- <MulticamCut
- session={session}
- locked={locked}
- onEdit={mutate}
- onRecut={(extra) => startJob("plan", extra)}
- />
- <MulticamLook session={session} lookPreviews={previews.looks} locked={locked} onEdit={mutate} />
- <MulticamDeliver
- session={session}
- locked={locked}
- onRender={() => startJob("render", { stems: true })}
- onExport={(format) => mutate({ action: "export", format })}
- onMakeClips={onMakeClips}
- />
+ <MulticamSummary session={session} locked={locked} onEdit={mutate} onRecut={(extra) => startJob("plan", extra)} />
+ <MulticamResults session={session} locked={locked} onMakeClips={onMakeClips} />
</>
)}
</div>
diff --git a/src/ui/client/multicam-person-chip.tsx b/src/ui/client/multicam-person-chip.tsx
index a2494d0..1d76030 100644
--- a/src/ui/client/multicam-person-chip.tsx
+++ b/src/ui/client/multicam-person-chip.tsx
@@ -6,16 +6,23 @@ export default function PersonChip({
onChange,
onRemove,
onBlur,
+ role,
+ onRole,
+ disabled = false,
}: {
value: string;
onChange: (name: string) => void;
onRemove: () => void;
onBlur?: () => void;
+ role?: "host" | "guest";
+ onRole?: (role: "host" | "guest") => void;
+ disabled?: boolean;
}) {
return (
<div className="file-badge" style={{ padding: "6px 10px", background: "var(--surface2)", border: "1px solid var(--border)" }}>
<input
value={value}
+ disabled={disabled}
aria-label="Person name"
onChange={(e) => onChange(e.target.value)}
onBlur={onBlur}
@@ -23,7 +30,18 @@ export default function PersonChip({
size={Math.max(4, value.length + 1)}
style={{ border: "none", background: "none", width: "auto", color: "var(--text)" }}
/>
- <button className="btn btn-ghost btn-sm" aria-label={`Remove ${value || "person"}`} onClick={onRemove} style={{ padding: "2px 6px" }}>
+ {role && onRole && (
+ <button
+ className={`pill ${role === "guest" ? "pill-sky" : "pill-blue"}`}
+ title={role === "guest" ? "Guest: answers stay on their camera. Click to make a host." : "Host. Click to make a guest."}
+ disabled={disabled}
+ onClick={() => onRole(role === "guest" ? "host" : "guest")}
+ style={{ border: "none", cursor: "pointer" }}
+ >
+ {role === "guest" ? "Guest" : "Host"}
+ </button>
+ )}
+ <button className="btn btn-ghost btn-sm" aria-label={`Remove ${value || "person"}`} disabled={disabled} onClick={onRemove} style={{ padding: "2px 6px" }}>
<TrashIcon size={12} />
</button>
</div>
diff --git a/src/ui/client/multicam-sources.tsx b/src/ui/client/multicam-sources.tsx
index 484478f..3463f04 100644
--- a/src/ui/client/multicam-sources.tsx
+++ b/src/ui/client/multicam-sources.tsx
@@ -45,6 +45,9 @@ export default function MulticamSources({
onChange={(v) => setNameDraft((d) => ({ ...d, [person.id]: v }))}
onBlur={() => commitName(person)}
onRemove={() => savePeople(session.people.filter((p) => p.id !== person.id))}
+ disabled={locked}
+ role={person.role || "host"}
+ onRole={(role) => savePeople(session.people.map((p) => (p.id === person.id ? { ...p, role } : p)))}
/>
))}
<button
@@ -114,9 +117,11 @@ function SourceRow({
<div style={{ minWidth: 0, width: 200 }}>
<div style={{ fontSize: 13, fontWeight: 600, overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }} title={source.path}>
- {basename(source.path)}
+ {source.name || basename(source.path)}
+ </div>
+ <div className="hint">
+ {source.parent ? "Cut from the call recording" : source.members.length ? "Side by side from everyone's own files" : details.join(" · ")}
</div>
- <div className="hint">{details.join(" · ")}</div>
</div>
{source.guessed && source.role !== "ignore" && <span className="pill pill-amber">Guessed</span>}
@@ -128,7 +133,7 @@ function SourceRow({
<option value="ignore">Ignore</option>
</select>
- {source.role === "camera" && (
+ {source.role === "camera" && !source.members.length && (
<select value={source.person || "wide"} disabled={disabled} style={selectStyle} onChange={(e) => onEdit({ person: e.target.value })}>
{personOptions}
<option value="wide">Everyone (wide)</option>
diff --git a/src/ui/client/multicam-summary.tsx b/src/ui/client/multicam-summary.tsx
new file mode 100644
index 0000000..cf1715f
--- /dev/null
+++ b/src/ui/client/multicam-summary.tsx
@@ -0,0 +1,69 @@
+import React from "react";
+import { fmt } from "./lib";
+import MulticamAutoCut from "./multicam-auto-cut";
+import type { McEdit, McSession } from "./multicam-types";
+import { cameraColorMap, whoLabel } from "./multicam-types";
+
+export default function MulticamSummary({
+ session,
+ locked,
+ onEdit,
+ onRecut,
+}: {
+ session: McSession;
+ locked: boolean;
+ onEdit: McEdit;
+ onRecut: (extra: Record<string, unknown>) => void;
+}) {
+ const cameras = session.sources.filter((s) => s.role === "camera" && s.offset !== null);
+ const colors = cameraColorMap(cameras.map((c) => c.id));
+ const { shots = 0, average_shot = 0, duration = 0, share = {} } = session.stats;
+
+ return (
+ <div className="section card">
+ <div className="card-title" style={{ marginBottom: 4 }}>Cut</div>
+ <p className="card-desc">
+ {session.resolved_style === "remote"
+ ? "Questions play on the split screen; the guest goes full frame on long answers."
+ : "Cameras follow whoever is speaking, picked from each person's mic."}
+ </p>
+
+ <div aria-hidden style={{ display: "flex", height: 28, borderRadius: "var(--radius-sm)", overflow: "hidden", border: "1px solid var(--border)" }}>
+ {session.cuts.map((c) => (
+ <div
+ key={`${c.start}-${c.source_id}`}
+ title={`${fmt(c.start)}-${fmt(c.end)}`}
+ style={{ flex: `${Math.max(0.2, c.end - c.start)} 0 0`, background: colors[c.source_id], opacity: 0.85, borderRight: "1px solid var(--bg)" }}
+ />
+ ))}
+ </div>
+
+ <div style={{ display: "flex", flexWrap: "wrap", gap: 12, alignItems: "center", marginTop: 10 }}>
+ {cameras.map((cam) => (
+ <span key={cam.id} className="hint" style={{ display: "inline-flex", alignItems: "center", gap: 6 }}>
+ <span style={{ width: 10, height: 10, borderRadius: 3, background: colors[cam.id] }} />
+ {whoLabel(session.people, cam)} {Math.round((share[cam.id] || 0) * 100)}%
+ </span>
+ ))}
+ <span className="hint">
+ {shots} shots · average {average_shot.toFixed(1)}s · {fmt(duration)}
+ {session.removals.length > 0 &&
+ ` · ${session.removals.length} cut out (${fmt(session.removals.reduce((t, r) => t + r.end - r.start, 0))})`}
+ </span>
+ <label className="hint" style={{ display: "inline-flex", alignItems: "center", gap: 6, marginLeft: "auto" }}>
+ Look
+ <select value={session.look} disabled={locked} onChange={(e) => onEdit({ action: "map", look: e.target.value })} style={{ width: "auto", padding: "4px 28px 4px 10px", fontSize: 12 }}>
+ {session.looks.map((l) => <option key={l} value={l}>{l[0].toUpperCase() + l.slice(1)}</option>)}
+ </select>
+ </label>
+ </div>
+
+ <MulticamAutoCut
+ key={JSON.stringify([session.range_start, session.range_end, session.cut_settings])}
+ session={session}
+ locked={locked}
+ onRecut={onRecut}
+ />
+ </div>
+ );
+}
diff --git a/src/ui/client/multicam-sync.tsx b/src/ui/client/multicam-sync.tsx
index 4e3f3d4..957f151 100644
--- a/src/ui/client/multicam-sync.tsx
+++ b/src/ui/client/multicam-sync.tsx
@@ -8,6 +8,7 @@ const STATUS: Record<string, { label: string; color: string; pill: string }> = {
ok: { label: "Synced", color: "var(--green)", pill: "pill-green" },
rough: { label: "Rough sync", color: "var(--amber)", pill: "pill-amber" },
manual: { label: "Set by hand", color: "var(--blue)", pill: "pill-sky" },
+ assumed: { label: "Starts with the others", color: "var(--amber)", pill: "pill-amber" },
};
const NOT_SYNCED = { label: "Not synced", color: "var(--red)", pill: "pill-red" };
@@ -23,7 +24,7 @@ export default function MulticamSync({
onCut: () => void;
}) {
const duration = session.timeline_duration || 1;
- const active = session.sources.filter((s) => s.role !== "ignore");
+ const active = session.sources.filter((s) => s.role !== "ignore" && !s.parent && !s.members.length);
const ready = active.every((s) => s.offset !== null);
return (
@@ -98,6 +99,7 @@ function SyncRow({ source, people, onEdit, disabled }: { source: McSource; peopl
{source.offset === null && (
<span className="hint" style={{ color: "var(--red)" }}>{source.sync.message || "Sync this file, or type where it starts."}</span>
)}
+ {source.sync.status === "assumed" && <span className="hint">{source.sync.message}</span>}
<div style={{ display: "inline-flex", gap: 6, marginLeft: "auto", alignItems: "center" }}>
{source.offset === null ? (
diff --git a/src/ui/client/multicam-types.ts b/src/ui/client/multicam-types.ts
index c68a51c..94b6f64 100644
--- a/src/ui/client/multicam-types.ts
+++ b/src/ui/client/multicam-types.ts
@@ -5,10 +5,11 @@
export interface McPerson {
id: string;
name: string;
+ role?: "host" | "guest";
}
export type McRole = "camera" | "mic" | "ignore";
-export type McSyncStatus = "reference" | "ok" | "rough" | "failed" | "manual";
+export type McSyncStatus = "reference" | "ok" | "rough" | "failed" | "manual" | "assumed";
export interface McSourceSync {
status?: McSyncStatus;
@@ -39,6 +40,9 @@ export interface McSource {
timeline_start: number | null;
timeline_end: number | null;
sync: McSourceSync;
+ parent: string;
+ crop: number[];
+ members: string[];
}
export interface McCut {
@@ -47,10 +51,18 @@ export interface McCut {
source_id: string;
}
+export type McStyle = "auto" | "studio" | "remote";
+
export interface McCutSettings {
+ style: McStyle;
min_shot: number;
max_shot: number;
wide_insert: number;
+ backchannel: number;
+ hold_guest: boolean;
+ host_solo: boolean;
+ guest_min: number;
+ guest_delay: number;
}
export interface McStats {
@@ -80,6 +92,9 @@ export interface McSession {
look: string;
looks: string[];
cut_settings: McCutSettings;
+ resolved_style: "studio" | "remote";
+ auto_style: "studio" | "remote";
+ removals: { start: number; end: number; reason?: string }[];
cuts: McCut[];
stats: McStats;
speaker_map: Record<string, string>;
@@ -108,7 +123,7 @@ export interface McPreviewsResp extends McSession {
previews: McPreviews;
}
-export type McJobKind = "sync" | "plan" | "render";
+export type McJobKind = "sync" | "plan" | "render" | "preview";
export const mcImageUrl = (path: string) => `/api/multicam/image?path=${encodeURIComponent(path)}`;
export const mcFileUrl = (path: string) => `/api/multicam/file?path=${encodeURIComponent(path)}`;
@@ -126,8 +141,8 @@ export function whoLabel(people: McPerson[], source: McSource): string {
export type McEdit = (body: Record<string, unknown>) => void;
-// Stable color per camera, assigned in first-appearance order across the cuts
-// so the shot strip and legend always agree on which color means which camera.
+// One color per camera in the order the cameras are listed, so the shot strip
+// and the legend agree on which color means which camera.
const CAMERA_COLORS = ["var(--accent)", "var(--green)", "var(--blue)", "var(--amber)", "var(--red)", "var(--text2)"];
export function cameraColorMap(cameraIds: string[]): Record<string, string> {
diff --git a/src/ui/web-server.ts b/src/ui/web-server.ts
index f369b4f..335a173 100644
--- a/src/ui/web-server.ts
+++ b/src/ui/web-server.ts
@@ -2243,8 +2243,9 @@ app.get("/api/reel-download", (req, res) => {
});
// --- Multicam: map sources, sync, cut, render one recording ---
-const MULTICAM_JOB_ACTIONS = new Set(["sync", "plan", "render"]);
-const MULTICAM_WRITE_ACTIONS = new Set(["map", "cut", "export", "delete", ...MULTICAM_JOB_ACTIONS]);
+const MULTICAM_JOB_ACTIONS = new Set(["sync", "plan", "render", "preview"]);
+// activity saves the session when it refreshes its cache, so it waits for jobs like any write.
+const MULTICAM_WRITE_ACTIONS = new Set(["map", "cut", "set_cuts", "export", "delete", "activity", ...MULTICAM_JOB_ACTIONS]);
const multicamPreviewDir = join(paths.working, "multicam");
// A multi-hour, multi-camera render outlasts the default one-hour task limit.
const multicamExecutor = new PythonExecutor(8 * 3600_000);
@@ -2254,19 +2255,13 @@ const multicamRunning = new Map<string, { id: string; action: string }>();
type MulticamPayload = {
session_id?: string;
- sources?: Array<{ path?: string }>;
outputs?: { video?: string };
active_job?: { id: string; action: string };
};
-// Camera and mic files become streamable for the review player, but only the
-// finished episode joins the recent-sources list the other pages offer.
+// Only the finished episode becomes streamable: paths in a request body are the
+// caller's say-so, and registering them would let a request read any media file.
function allowMulticamPaths(data: MulticamPayload | undefined): void {
- for (const s of data?.sources || []) {
- try {
- if (s.path) allowedSourcePaths.add(realpathSync(path.resolve(s.path)));
- } catch {}
- }
if (data?.outputs?.video) registerSourcePath(data.outputs.video);
}
@@ -2327,10 +2322,11 @@ app.post("/api/multicam", async (req, res) => {
});
});
+// Camera stills podcli made for an edit; nothing outside its working folder.
app.get("/api/multicam/image", (req, res) => {
- // Python reports resolved paths (/private/tmp on macOS), so compare real paths on both sides.
let resolved: string | null = null;
try {
+ // Python reports resolved paths (/private/tmp on macOS), so compare real paths on both sides.
const root = realpathSync(multicamPreviewDir);
resolved = safePath(root, path.relative(root, realpathSync(String(req.query.path || ""))));
} catch {}
diff --git a/tests/test_multicam.py b/tests/test_multicam.py
index e218099..a9412e0 100644
--- a/tests/test_multicam.py
+++ b/tests/test_multicam.py
@@ -1,3 +1,4 @@
+import json
import os
import shutil
import subprocess
@@ -91,14 +92,19 @@ def test_update_mapping_validates_roles_and_people(sandbox):
assert session.sources[0].person == ""
-def test_shot_frames_fills_gaps_on_one_grid():
+def test_render_plan_fills_gaps_and_skips_removals_on_one_grid():
session = mc.MulticamSession(session_id="abc123abc126", name="ep", cuts=[
{"start": 10.0, "end": 12.51, "source_id": "a"},
{"start": 12.51, "end": 14.0, "source_id": "b"},
{"start": 15.0, "end": 16.0, "source_id": "a"},
])
- frames = mc.shot_frames(session, 30.0)
- assert frames == [("a", 0, 75), ("b", 75, 45), (None, 120, 30), ("a", 150, 30)]
+ plan = [(p.source_id, p.out_frame, p.frames) for p in mc.render_plan(session, 30.0)]
+ assert plan == [("a", 0, 75), ("b", 75, 45), (None, 120, 30), ("a", 150, 30)]
+
+ session.removals = [{"start": 11.0, "end": 13.0}]
+ plan = mc.render_plan(session, 30.0)
+ assert [(p.source_id, p.out_frame, p.frames) for p in plan] == [("a", 0, 30), ("b", 30, 30), (None, 60, 30), ("a", 90, 30)]
+ assert plan[1].tl_start == pytest.approx(13.0)
# --- End to end with generated media -----------------------------------------
@@ -211,7 +217,7 @@ def test_sync_plan_render_and_export_a_three_camera_episode(sandbox):
tracks = premiere.findall("./sequence/media/audio/track")
assert len(tracks) == 2
vclips = premiere.findall("./sequence/media/video/track/clipitem")
- assert len(vclips) == len([f for f in mc.shot_frames(session, 30.0) if f[0]])
+ assert len(vclips) == len([p for p in mc.render_plan(session, 30.0) if p.source_id])
fcp = ET.parse(mc.export_xml(session, "fcpxml")).getroot()
lanes = {c.get("lane") for c in fcp.iter("asset-clip")}
@@ -222,6 +228,23 @@ def test_sync_plan_render_and_export_a_three_camera_episode(sandbox):
assert mc.render_session(session)["video"] == video
assert os.path.getmtime(video) == before
+ # Removing 2.5 s takes it out of picture, mix, stems and both timelines alike.
+ full = float(probe.stdout)
+ mc.set_removals(session, [{"start": 20.0, "end": 22.5, "reason": "retake"}])
+ trimmed = mc.render_session(session)
+
+ def length(path, stream):
+ out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", stream, "-show_entries",
+ "stream=duration", "-of", "csv=p=0", path], check=True, capture_output=True, text=True)
+ return float(out.stdout.strip().splitlines()[0])
+
+ assert abs(length(trimmed["video"], "v:0") - (full - 2.5)) < 0.1
+ assert abs(length(trimmed["video"], "a:0") - (full - 2.5)) < 0.1
+ assert all(abs(length(stem, "a:0") - (full - 2.5)) < 0.1 for stem in trimmed["stems"])
+ xml = ET.parse(mc.export_xml(session, "premiere")).getroot()
+ assert abs(int(xml.findtext("./sequence/duration")) / 30 - (full - 2.5)) < 0.1
+ mc.set_removals(session, [])
+
# A hand-set offset survives a normal re-sync; force measures it again.
wide = next(s for s in session.sources if s.path.endswith("cam_wide.mp4"))
mc.update_mapping(session, {"sources": [{"id": wide.id, "offset": 2.5}]})
@@ -375,6 +398,8 @@ def run_cli(monkeypatch, *argv):
@pytest.fixture
def episode(sandbox):
+ if not (shutil.which("ffmpeg") and shutil.which("ffprobe")):
+ pytest.skip("ffmpeg not installed")
folder = sandbox / "ep"
folder.mkdir()
a = _speech(40, 5)
@@ -447,3 +472,341 @@ def test_bad_fix_exits_with_a_readable_error(episode, monkeypatch, capsys):
def test_no_target_prints_usage(monkeypatch, capsys):
assert run_cli(monkeypatch) == 2
assert "podcli multicam <folder" in capsys.readouterr().out
+
+
+def test_set_cuts_checks_coverage_and_merges(sandbox):
+ session = mc.MulticamSession(
+ session_id="abc123abc150", name="ep",
+ people=[mc.Person("host", "Host"), mc.Person("guest", "Guest")],
+ sources=[
+ _source("/x/a.mp4", role="camera", person="host", offset=0.0),
+ _source("/x/w.mp4", role="camera", person="wide", offset=10.0, duration=20.0),
+ ],
+ )
+ mc.set_cuts(session, [
+ {"start": 0, "end": 5, "source_id": "a"},
+ {"start": 5, "end": 12, "source_id": "a"},
+ {"start": 12, "end": 20, "source_id": "w"},
+ ])
+ assert session.cuts == [{"start": 0, "end": 12, "source_id": "a"}, {"start": 12, "end": 20, "source_id": "w"}]
+ with pytest.raises(ValueError, match="wasn't recording"):
+ mc.set_cuts(session, [{"start": 0, "end": 12, "source_id": "w"}])
+ with pytest.raises(ValueError, match="back to back"):
+ mc.set_cuts(session, [{"start": 0, "end": 4, "source_id": "a"}, {"start": 5, "end": 9, "source_id": "a"}])
+
+
+def test_cli_json_mode_applies_a_cut_and_builds_the_preview(episode, monkeypatch, capsys, tmp_path):
+ code = run_cli(monkeypatch, str(episode), "--people", "Nika, Ana", "--set", "cam_one=camera:nika",
+ "--set", "cam_two=camera:ana", "--no-render", "--json", "--activity")
+ out = capsys.readouterr().out
+ assert code == 0, out
+ data = json.loads(out)
+ assert data["cuts"] and set(data["activity"]["people"]) == {"nika", "ana"}
+ assert data["activity"]["people"]["nika"][0][0] < 20 <= data["activity"]["people"]["ana"][0][0] + 1
+
+ one = next(s["id"] for s in data["sources"] if s["name"] == "cam_one.mp4")
+ start, end = data["cuts"][0]["start"], data["cuts"][-1]["end"]
+ cuts = tmp_path / "cuts.json"
+ cuts.write_text(json.dumps([{"start": start, "end": end, "source_id": one}]))
+ code = run_cli(monkeypatch, data["session_id"], "--cuts", str(cuts), "--preview", "--no-render", "--json")
+ data = json.loads(capsys.readouterr().out)
+ assert code == 0
+ assert data["cuts"] == [{"start": start, "end": end, "source_id": one}]
+ assert set(data["preview"]["proxies"]) == {s["id"] for s in data["sources"] if s["role"] == "camera"}
+ assert os.path.exists(data["preview"]["audio"])
+
+ # Moving a mic changes the mix, so the old preview audio no longer matches.
+ session = mc.MulticamSession.load(data["session_id"])
+ mic = next(s for s in session.sources if s.role in ("mic", "camera") and s.offset)
+ mc.update_mapping(session, {"sources": [{"id": mic.id, "nudge": 0.04}]})
+ assert mc.payload(mc.MulticamSession.load(data["session_id"]))["preview"] is None
+
+ cuts.write_text("not json")
+ assert run_cli(monkeypatch, data["session_id"], "--cuts", str(cuts), "--json") == 1
+ assert "error" in json.loads(capsys.readouterr().out)
+
+
+def test_last_person_is_the_guest_and_roles_survive_renames(episode):
+ session = mc.new_session(folder=str(episode), people=["Nihal", "Cameron", "Ana"])
+ assert [(p.name, p.role) for p in session.people] == [("Nihal", "host"), ("Cameron", "host"), ("Ana", "guest")]
+ session = mc.rename_people(session, ["Nihal", "Cam", "Ana Smith"])
+ assert [p.role for p in session.people] == ["host", "host", "guest"]
+
+
+def test_cli_guests_flag_sets_roles(episode, monkeypatch, capsys):
+ code = run_cli(monkeypatch, str(episode), "--people", "Nika, Ana", "--guests", "Nika",
+ "--set", "cam_one=camera:nika", "--set", "cam_two=camera:ana", "--no-render", "--json")
+ data = json.loads(capsys.readouterr().out)
+ assert code == 0
+ assert {p["name"]: p["role"] for p in data["people"]} == {"Nika": "guest", "Ana": "host"}
+ assert run_cli(monkeypatch, data["session_id"], "--guests", "Bob", "--no-render") == 1
+ assert "No person named 'bob'" in capsys.readouterr().err
+
+
+def test_a_saved_edit_renders_on_another_machine_without_syncing(episode, monkeypatch, capsys, tmp_path):
+ assert run_cli(monkeypatch, str(episode), "--people", "Nika, Ana", "--set", "cam_one=camera:nika",
+ "--set", "cam_two=camera:ana", "--no-render", "--json") == 0
+ prepared = json.loads(capsys.readouterr().out)
+ state = tmp_path / "state.json"
+ state.write_text(json.dumps(prepared))
+
+ # A fresh worker: the same recordings in another folder, and nothing saved.
+ elsewhere = tmp_path / "worker" / "ep"
+ shutil.copytree(episode, elsewhere)
+ monkeypatch.setitem(paths, "packed", str(tmp_path / "worker" / "home" / "packed"))
+ monkeypatch.setattr(mc, "sync_session", lambda *a, **k: pytest.fail("synced again"))
+ monkeypatch.setattr(mc, "plan_session", lambda *a, **k: pytest.fail("cut again"))
+ assert run_cli(monkeypatch, str(elsewhere), "--state", str(state), "--no-render", "--json") == 0
+ restored = json.loads(capsys.readouterr().out)
+
+ assert restored["session_id"] != prepared["session_id"]
+ assert {s["id"]: s["offset"] for s in restored["sources"]} == {s["id"]: s["offset"] for s in prepared["sources"]}
+ assert restored["cuts"] == prepared["cuts"]
+
+
+def test_transcript_credits_each_word_to_the_mic_that_spoke(episode, monkeypatch):
+ from services import transcription
+
+ calls = []
+
+ def fake(path, **kw):
+ calls.append(path)
+ return {"words": [{"word": "hello", "start": 2.0, "end": 2.4}, {"word": "there", "start": 25.0, "end": 25.5}],
+ "language": "en"}
+
+ monkeypatch.setattr(transcription, "transcribe_file", fake)
+ session = mc.new_session(folder=str(episode), people=["Nika", "Ana"])
+ mc.update_mapping(session, {"sources": [
+ {"id": s.id, "role": "camera", "person": "nika" if "one" in os.path.basename(s.path) else "ana"}
+ for s in session.sources if s.kind == "video"
+ ]})
+ session = mc.sync_session(session)
+ data = mc.transcript(session)
+ assert [(w["text"], w["person"]) for w in data["words"]] == [("hello", "nika"), ("there", "ana")]
+ assert mc.transcript(session) == data and len(calls) == 1
+
+
+def test_turn_edges_follow_sentences_not_loose_timestamps():
+ def words(spec):
+ return [{"text": t, "person": p, "start": i, "end": i + 0.5} for i, (t, p) in enumerate(spec)]
+
+ # "Right," opens the host's sentence but was timed before the guest stopped.
+ w = words([("get", "g"), ("home.", "g"), ("Right,", "g"), ("because", "h"), ("every", "h")])
+ mc._settle_turn_edges(w, [0.0] * len(w))
+ assert [x["person"] for x in w] == ["g", "g", "h", "h", "h"]
+
+ # A lone word mid-sentence credited to the other mic goes back to the speaker.
+ w = words([("the", "g"), ("pain", "h"), ("starts", "g"), ("here.", "g")])
+ mc._settle_turn_edges(w, [0.0] * len(w))
+ assert [x["person"] for x in w] == ["g", "g", "g", "g"]
+
+ # A word the mics credit clearly never moves, even at a turn edge.
+ w = words([("home.", "a"), ("Then", "a"), ("Exactly.", "b")])
+ mc._settle_turn_edges(w, [0.0, 15.0, 15.0])
+ assert [x["person"] for x in w] == ["a", "a", "b"]
+
+
+# --- Remote recordings ------------------------------------------------------------
+
+def _voices(seconds=40):
+ host, guest = _speech(seconds, 21), _speech(seconds, 22)
+ t = np.arange(len(host)) / RATE
+ host[t >= 8] = 0 # a short question...
+ guest[t < 8] = 0 # ...then one long answer
+ return host, guest
+
+
[email protected](not shutil.which("ffmpeg"), reason="ffmpeg not installed")
+def test_one_file_per_person_cuts_like_a_call(sandbox):
+ folder = sandbox / "remote"
+ folder.mkdir()
+ host, guest = _voices()
+ # Each person's local recording hears only them (headphones), so the files share no sound.
+ for name, voice, color in (("nika", host, "red"), ("ana", guest, "blue")):
+ _write_wav(sandbox / f"{name}.wav", voice)
+ _ffmpeg("-f", "lavfi", "-i", f"color=c={color}:s=320x180:r=30:d=40", "-i", str(sandbox / f"{name}.wav"),
+ "-shortest", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", str(folder / f"{name}.mp4"))
+ session = mc.sync_session(mc.new_session(folder=str(folder), people=["Nika", "Ana"]))
+ assert {s.sync["status"] for s in session.sources if not s.virtual} == {"reference", "assumed"}
+ split = next(s for s in session.sources if s.members)
+ assert split.person == "wide" and mc.resolved_style(session) == "remote"
+ assert sorted(p for _, _, p in session.person_mics()) == ["ana", "nika"]
+
+ session = mc.plan_session(session)
+ ids = {s.id: s for s in session.sources}
+ # The question plays on the split; the answer goes to Ana once she's 4 s in.
+ assert ids[session.cuts[0]["source_id"]].members
+ guest_shot = next(c for c in session.cuts if ids[c["source_id"]].person == "ana")
+ assert 11.0 < guest_shot["start"] < 13.5
+
+ video = mc.render_session(session, stems=False)["video"]
+ left, right = _mean_rgb_at(video, 3, crop="crop=iw/2:ih:0:0"), _mean_rgb_at(video, 3, crop="crop=iw/2:ih:iw/2:0")
+ assert left[0] > 150 and right[2] > 150 # split screen: Nika red on the left, Ana blue on the right
+ full = _mean_rgb_at(video, session.cuts[-1]["end"] - session.cuts[0]["start"] - 2)
+ assert full[2] > 150 and full[0] < 80 # Ana alone, full frame
+
+
[email protected](not shutil.which("ffmpeg"), reason="ffmpeg not installed")
+def test_one_gallery_recording_becomes_a_camera_per_tile(sandbox, monkeypatch):
+ from services import speaker_detection
+
+ folder = sandbox / "gallery"
+ folder.mkdir()
+ host, guest = _voices()
+ _write_wav(sandbox / "call.wav", host + guest)
+ # A two-tile call recording: moving pictures in each tile, a flat gutter between them.
+ _ffmpeg("-f", "lavfi", "-i", "testsrc2=s=310x360:r=30:d=40", "-f", "lavfi", "-i", "mandelbrot=s=310x360:r=30",
+ "-i", str(sandbox / "call.wav"),
+ "-filter_complex", "[0:v]pad=320:360:0:0:color=0x202020[l];[1:v]pad=320:360:10:0:color=0x202020[r];[l][r]hstack[v]",
+ "-map", "[v]", "-map", "2:a", "-t", "40", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac",
+ str(folder / "zoom_call.mp4"))
+ monkeypatch.setattr(speaker_detection, "run_diarization", lambda *a, **k: [
+ {"speaker": "SPEAKER_00", "start": 0.5, "end": 7.8}, {"speaker": "SPEAKER_01", "start": 8.2, "end": 39.5}])
+
+ session = mc.new_session(folder=str(folder), people=["Nika", "Ana"])
+ panes = [s for s in session.sources if s.parent]
+ assert [p.person for p in panes] == ["nika", "ana"]
+ assert panes[0].crop[0] < 0.05 and 0.45 < panes[0].crop[2] < 0.52 and panes[1].crop[0] > 0.48
+ session = mc.plan_session(mc.sync_session(session))
+ assert mc.resolved_style(session) == "remote"
+ ids = {s.id: s for s in session.sources}
+ assert ids[session.cuts[0]["source_id"]].person == "wide"
+ assert ids[session.cuts[-1]["source_id"]].person == "ana"
+ mc.render_session(session, stems=False)
+
+
+def _mean_rgb_at(video, at, crop="null"):
+ raw = subprocess.run(
+ ["ffmpeg", "-hide_banner", "-loglevel", "error", "-ss", f"{at}", "-i", str(video),
+ "-frames:v", "1", "-vf", f"{crop},scale=8:8", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
+ check=True, capture_output=True,
+ ).stdout
+ return np.frombuffer(raw, dtype=np.uint8).reshape(-1, 3).mean(axis=0)
+
+
+# --- Audit regressions ------------------------------------------------------------
+
+def _bare(sandbox, n=2):
+ return mc.MulticamSession(
+ session_id="abc123abc199", name="ep",
+ people=[mc.Person("host", "Host"), mc.Person("guest", "Guest", role="guest")],
+ sources=[_source(f"/x/{c}.mp4", role="camera", person=p, offset=0.0, id=c)
+ for c, p in (("a", "host"), ("b", "guest"), ("w", "wide"))[:n + 1]],
+ )
+
+
[email protected]("bad", [float("nan"), float("inf"), "x"])
+def test_non_finite_numbers_are_refused_everywhere(sandbox, bad):
+ session = _bare(sandbox)
+ session.cuts = [{"start": 0, "end": 30, "source_id": "a"}]
+ with pytest.raises(ValueError):
+ mc.set_removals(session, [{"start": 1, "end": bad}])
+ with pytest.raises(ValueError):
+ mc.set_cuts(session, [{"start": 0, "end": bad, "source_id": "a"}])
+ with pytest.raises(ValueError):
+ mc.update_mapping(session, {"sources": [{"id": "a", "offset": bad}]})
+ with pytest.raises(ValueError):
+ mc.update_mapping(session, {"range_end": bad})
+
+
+def test_choosing_a_style_keeps_only_real_tweaks(sandbox):
+ session = _bare(sandbox)
+ mc.update_mapping(session, {"cut_settings": {**mc.STYLES["studio"], "style": "remote"}})
+ # Everything sent equals studio defaults, which differ from remote's: those are tweaks.
+ assert session.cut_settings["style"] == "remote" and session.cut_settings["host_solo"] is True
+ mc.update_mapping(session, {"cut_settings": {"style": "studio"}})
+ assert session.cut_settings == {"style": "studio"}
+ mc.update_mapping(session, {"cut_settings": {"min_shot": 2.0, "max_shot": 12}})
+ assert session.cut_settings == {"style": "studio", "max_shot": 12.0}
+
+
+def test_removals_and_cuts_are_checked_against_each_other(sandbox):
+ session = _bare(sandbox)
+ session.cuts = [{"start": 0, "end": 30, "source_id": "a"}]
+ mc.set_removals(session, [{"start": 9, "end": 21}])
+ with pytest.raises(ValueError, match="cover this whole cut"):
+ mc.set_cuts(session, [{"start": 10, "end": 20, "source_id": "a"}])
+ assert session.cuts == [{"start": 0, "end": 30, "source_id": "a"}]
+
+
+def test_frame_size_comes_from_one_landscape_camera(sandbox):
+ session = _bare(sandbox, n=1)
+ session.sources[1].width, session.sources[1].height = 1080, 1920
+ assert mc._output_format(session)[:2] == (1920, 1080)
+
+
[email protected](not shutil.which("ffmpeg"), reason="ffmpeg not installed")
+def test_studio_cameras_that_start_at_different_times_stay_studio(sandbox):
+ folder = sandbox / "studio"
+ folder.mkdir()
+ host, guest = _voices()
+ room = (host + guest) * 0.6
+ _write_wav(folder / "rec_Tr1.wav", host + 0.08 * guest)
+ _write_wav(folder / "rec_Tr2.wav", guest + 0.08 * host)
+ for name, start in (("nika", 0), ("ana", 12)):
+ _write_wav(sandbox / f"{name}.wav", room[int(start * RATE):])
+ _ffmpeg("-f", "lavfi", "-i", f"color=c=gray:s=160x90:r=30:d={40 - start}", "-i", str(sandbox / f"{name}.wav"),
+ "-shortest", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", str(folder / f"cam_{name}.mp4"))
+ session = mc.sync_session(mc.new_session(folder=str(folder), people=["Nika", "Ana"]))
+ assert not any(s.members for s in session.sources) and mc.resolved_style(session) == "studio"
+
+
[email protected](not shutil.which("ffmpeg"), reason="ffmpeg not installed")
+def test_each_persons_video_and_audio_sync_together_and_start_with_the_rest(sandbox):
+ folder = sandbox / "tracks"
+ folder.mkdir()
+ host, guest = _voices()
+ for name, voice, color in (("nika", host, "red"), ("ana", guest, "blue")):
+ _write_wav(folder / f"{name}_audio.wav", voice)
+ _write_wav(sandbox / f"{name}.wav", voice * 0.5)
+ _ffmpeg("-f", "lavfi", "-i", f"color=c={color}:s=160x90:r=30:d=40", "-i", str(sandbox / f"{name}.wav"),
+ "-shortest", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", str(folder / f"{name}_video.mp4"))
+ session = mc.new_session(folder=str(folder), people=["Nika", "Ana"])
+ mc.update_mapping(session, {"sources": [
+ {"id": s.id, "role": "camera" if s.kind == "video" else "mic", "person": "nika" if "nika" in os.path.basename(s.path) else "ana"}
+ for s in session.sources
+ ]})
+ session = mc.sync_session(session)
+ assert not mc.needs_sync(session), [(os.path.basename(s.path), s.role, s.offset, s.sync) for s in session.sources]
+ assert "assumed" in {s.sync.get("status") for s in session.sources}
+ assert any(s.members for s in session.sources)
+ # Reopening the folder finds this edit, split screen and all.
+ assert mc.new_session(folder=str(folder)).session_id == session.session_id
+
+
[email protected](not shutil.which("ffmpeg"), reason="ffmpeg not installed")
+def test_gallery_with_two_tiles_on_top_and_one_below(sandbox, tmp_path):
+ video = tmp_path / "three.mp4"
+ _ffmpeg("-f", "lavfi", "-i", "testsrc2=s=300x168:r=30:d=6", "-f", "lavfi", "-i", "mandelbrot=s=300x168:r=30",
+ "-f", "lavfi", "-i", "life=s=300x168:r=30:mold=10:ratio=0.2",
+ "-filter_complex",
+ "color=c=0x101010:s=640x360:d=6[bg];[bg][0:v]overlay=10:8[a];[a][1:v]overlay=330:8[b];[b][2:v]overlay=170:184[v]",
+ "-map", "[v]", "-t", "6", "-c:v", "libx264", "-pix_fmt", "yuv420p", str(video))
+ panes = mc.detect_panes(mc.probe_source(str(video)))
+ assert len(panes) == 3
+ top, bottom = sorted(panes, key=lambda p: (p[1], p[0]))[:2], sorted(panes, key=lambda p: (p[1], p[0]))[2]
+ assert all(p[1] < 0.1 for p in top) and bottom[1] > 0.45 and 0.2 < bottom[0] < 0.3
+
+
+def test_render_refuses_mapping_changes_that_would_drop_the_cut(sandbox, monkeypatch):
+ import main as backend
+
+ session = _bare(sandbox)
+ session.cuts = [{"start": 0, "end": 30, "source_id": "a"}]
+ session.save()
+ sent = []
+ monkeypatch.setattr(backend, "emit_result", lambda task, status, data=None, error=None: sent.append((status, error)))
+ backend.handle_manage_multicam("t", {"action": "render", "session_id": session.session_id, "range_start": 5})
+ assert sent[0][0] == "error" and "only look" in sent[0][1]
+ assert mc.MulticamSession.load(session.session_id).cuts
+
+
+def test_cli_json_always_prints_one_object(episode, monkeypatch, capsys, tmp_path):
+ assert run_cli(monkeypatch, "--json") == 2
+ assert "error" in json.loads(capsys.readouterr().out)
+ cuts = tmp_path / "cuts.json"
+ cuts.write_text(json.dumps([{"end": 3, "source_id": "x"}]))
+ assert run_cli(monkeypatch, str(episode), "--cuts", str(cuts), "--no-render", "--json", "-y") == 1
+ assert "error" in json.loads(capsys.readouterr().out)
+ assert run_cli(monkeypatch, str(episode), "--delete", "--json") == 0
+ assert json.loads(capsys.readouterr().out)["deleted"] is True
diff --git a/tests/test_multicam_signal.py b/tests/test_multicam_signal.py
index 859a693..300a5bd 100644
--- a/tests/test_multicam_signal.py
+++ b/tests/test_multicam_signal.py
@@ -198,13 +198,32 @@ def test_plan_cuts_never_overlaps_wide_inserts():
assert abs(sum(c["end"] - c["start"] for c in cuts) - 20) < 1e-6
-def test_flickering_crosstalk_becomes_one_wide_shot():
- rng = np.random.default_rng(1)
- flicker = rng.choice([BOTH, BOTH, 0, 1], size=500).astype(np.int32)
- labels = np.concatenate([_labels([(0, 15)]), flicker, _labels([(1, 10)])])
- cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=30, max_shot=0)
+def test_sustained_crosstalk_with_blips_becomes_one_wide_shot():
+ blips = [(BOTH, 0.7), (0, 0.05), (BOTH, 0.9), (1, 0.08), (BOTH, 1.2), (0, 0.05), (BOTH, 1.0)]
+ labels = _labels([(0, 15), *blips, (1, 10)])
+ cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=29, max_shot=0)
assert [c["source_id"] for c in cuts] == ["cam_host", "cam_wide", "cam_guest"]
- assert cuts[1]["end"] - cuts[1]["start"] > 3.0
+ assert cuts[1]["end"] - cuts[1]["start"] > 3.5
+
+
+def test_backchannels_inside_an_answer_never_cut_away():
+ # Host says "mm-hm" and laughs with the guest briefly while the guest answers.
+ labels = _labels([(0, 5), (1, 12), (0, 0.6), (1, 9), (BOTH, 0.8), (1, 14), (0, 6)])
+ cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=48.4, max_shot=0)
+ assert [c["source_id"] for c in cuts] == ["cam_host", "cam_guest", "cam_host"]
+
+
+def test_guest_answers_hold_but_host_monologues_get_a_cutaway():
+ labels = _labels([(1, 70), (0, 70)])
+ cams = [Camera(c.source_id, c.person, 0, 200) for c in CAMS]
+ cuts = plan_cuts(labels, ["host", "guest"], cams, range_start=0, range_end=140, max_shot=30,
+ guests=frozenset({"guest"}))
+ ids = [c["source_id"] for c in cuts]
+ assert ids[0] == "cam_guest" and cuts[0]["end"] == 70
+ assert "cam_wide" in ids[1:]
+ held_off = plan_cuts(labels, ["host", "guest"], cams, range_start=0, range_end=140, max_shot=30,
+ guests=frozenset({"guest"}), hold_guest=False)
+ assert held_off[1]["source_id"] == "cam_wide" and held_off[1]["start"] < 70
def test_plan_cuts_scales_to_a_three_hour_episode():
@@ -218,3 +237,27 @@ def test_plan_cuts_scales_to_a_three_hour_episode():
cuts = plan_cuts(labels, ["host", "guest"], cams, range_start=0, range_end=len(labels) / 100)
assert time.perf_counter() - t0 < 5
assert all(c["end"] - c["start"] >= 2.0 - 1e-6 for c in cuts[1:-1])
+
+
+def test_remote_style_keeps_questions_on_the_split_and_opens_answers_wide():
+ labels = _labels([(0, 6), (1, 5), (0, 4), (1, 30)])
+ cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=45, min_shot=4, max_shot=0,
+ guests=frozenset({"guest"}), host_solo=False, guest_min=8, guest_delay=4)
+ # Question, short answer and follow-up all sit on the split; the long answer goes to the guest after 4 s.
+ assert [c["source_id"] for c in cuts] == ["cam_wide", "cam_guest"]
+ assert abs(cuts[1]["start"] - 19.0) < 0.05 and cuts[1]["end"] == 45
+
+
+def test_remote_style_measures_a_guest_turn_by_its_words_not_the_silence_after():
+ labels = _labels([(0, 6), (1, 3), (SILENT, 6), (0, 6)])
+ cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=21, min_shot=1, max_shot=0,
+ guests=frozenset({"guest"}), host_solo=False, guest_min=8, guest_delay=4)
+ assert [c["source_id"] for c in cuts] == ["cam_wide"]
+
+
+def test_remote_style_returns_to_the_split_soon_after_the_answer_ends():
+ labels = _labels([(0, 4), (1, 20), (SILENT, 5), (0, 4)])
+ cuts = plan_cuts(labels, ["host", "guest"], CAMS, range_start=0, range_end=33, min_shot=1, max_shot=0,
+ guests=frozenset({"guest"}), host_solo=False, guest_min=8, guest_delay=4)
+ solo = next(c for c in cuts if c["source_id"] == "cam_guest")
+ assert abs(solo["start"] - 8) < 0.05 and abs(solo["end"] - 25) < 0.05