Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Music Video shot modes and provider-aware timing (#8977).
*
* A scene is either a CUTAWAY (the default — any image-to-video lane animates
* the reference frame; the master song is laid under it at render time) or a
* PERFORMANCE shot: a visible singer whose mouth must follow the actual master
* recording. Prompting cannot deliver that; only a provider that accepts the
* exact song audio as conditioning can. This module is the one definition of
* which lanes can, and of the timing math both runtimes share:
*
* - `SOURCE_AUDIO_LIPSYNC` — the capability snapshot per backend. Only fal.ai's
* documented MiniMax H3 lip-sync route (image_url + audio_url, at least 5s of
* audio, output length follows the audio, priced per output second by
* resolution — lib/falVideoModels.js) is listed. Grok's CLI image_to_video lane takes no audio at all, so
* it is cutaway-only here until an actual adapter proves exact-source-audio
* conditioning — REST documentation for a different Grok surface is not that
* proof. Local runtimes are absent for the same reason.
* - `SOURCE_AUDIO_MUSIC` — the music-driven performance lanes: fal.ai
* audio-to-video models (kind `a2v`) and reference models that hear the
* song slice as an audio reference (MiniMax H3 Max, Seedance 2.5), where
* the song drives the whole picture's motion rather than only a singer's mouth. A scene opts in with
* `performanceStyle: 'music'`; it shares the window planning below.
* - `planPerformanceWindow` — the audio window a performance shot submits:
* inside the provider's limits (never relying on its silent truncation), a
* short shot padded to a supported contextual window with an explicit edit
* in-point, a long shot refused so the director splits it on a lyric/phrase
* boundary. Nothing is looped or time-stretched to fill a slot.
* - `grokCoverage` — the 6/10-second Grok request that COVERS a cutaway shot
* (reusing `nearestGrokDuration`), flagging spans a single clip cannot cover.
* - `shotSplitLimit` / `planShotSplit` — how such an over-long shot is split:
* contiguous pieces within the lane's per-take limit, cut at a pause between
* sung lines, a lyric/phrase boundary or a beat (in that order).
*
* - `falSceneTake` — the fal.ai request (model, length, resolution) a scene
* take makes on the fal lane and its estimated cost, shared by the board,
* the submit path and the production autopilot's spend cap.
*
* - `runwaySceneTake` — the same answer for a cutaway on the Runway lane;
* `meteredSceneTake` picks the paid lane's take for a backend.
*
* Dependency-free (only grokVideoClip.js, falVideoModels.js and
* runwayVideoModels.js, themselves dependency-free) so the client
* re-exports it from `client/src/lib/musicVideoShotTiming.js` and the two sides
* cannot drift.
*/
import { GROK_VIDEO_DURATIONS, nearestGrokDuration } from './grokVideoClip.js';
import {
FAL_AUDIO_VIDEO_MODELS,
FAL_DEFAULT_IMAGE_VIDEO_MODEL,
FAL_LIPSYNC_VIDEO_MODEL,
FAL_MUSIC_VIDEO_DEFAULT_MODEL,
coerceFalVideoSeconds,
describeFalVideoRate,
estimateFalVideoCostUsd,
falVideoResolutions,
getFalVideoModel,
resolveFalVideoResolution,
} from './falVideoModels.js';
import {
RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL,
coerceRunwayVideoSeconds,
estimateRunwayVideoCostUsd,
getRunwayVideoModel,
resolveRunwayVideoResolution,
} from './runwayVideoModels.js';
export const MUSIC_VIDEO_SHOT_MODES = Object.freeze(['cutaway', 'performance']);
/**
* A scene with no `shotMode` (every pre-#8977 record) is a cutaway. A
* performance is sung footage, so it only applies on the footage layer
* (#8985): a still or title-card scene is never lip-synced.
*/
export const isPerformanceScene = (scene) => scene?.shotMode === 'performance'
&& (scene.visualLayer == null || scene.visualLayer === 'footage');
/**
* How a performance shot follows the song: `lipsync` (absent on every older
* record) syncs a singer's mouth to the vocal; `music` lets the song slice
* drive the whole picture's motion — a drummer hitting on the beat, a dancer,
* a crowd — on an audio-to-video model (lib/falVideoModels.js kind `a2v`).
*/
export const MUSIC_VIDEO_PERFORMANCE_STYLES = Object.freeze(['lipsync', 'music']);
/** A performance shot a singer lip-syncs (the only kind whose mouth sync is measured). */
export const isLipSyncScene = (scene) => isPerformanceScene(scene) && scene.performanceStyle !== 'music';
/**
* The shot instruction of a performance scene's SELECTED video take, or null —
* for a cutaway scene (even one whose selected take was once generated as a
* performance), or a take with no usable edit points.
*/
export function selectedPerformanceInstruction(scene) {
if (!isPerformanceScene(scene) || !scene.videoHistoryId) return null;
const take = (Array.isArray(scene.takes) ? scene.takes : [])
.find((t) => t?.kind === 'video' && t.assetId === scene.videoHistoryId);
const instruction = take?.shotInstruction;
const edit = instruction?.edit;
if (instruction?.shotMode !== 'performance' || !Number.isFinite(edit?.inSec) || !Number.isFinite(edit?.outSec)) return null;
return edit.inSec >= 0 && edit.outSec > edit.inSec ? instruction : null;
}
/**
* How far into a take's clip its audio window really starts (#11210): the
* provider's leading padding the take's edit points were moved by when it
* landed, or 0. Clip time t plays song time `audioWindow.startSec + t - offset`.
*/
export const clipAudioOffsetSec = (instruction) => (Number.isFinite(instruction?.alignment?.offsetSec) ? instruction.alignment.offsetSec : 0);
// Safety margin kept inside the provider's documented bounds so ffmpeg's
// sample rounding can never land a window a hair under the minimum (rejected)
// or over the maximum (silently clipped by the provider).
export const PERFORMANCE_WINDOW_MARGIN_SEC = 0.05;
/**
* Default output resolution of a Music Video performance take. fal's own
* default is 768P; a sung close-up is the shot a viewer scrutinizes most, so
* the board asks for 1080P unless the project pins another
* (`videoSettings.falLipSyncResolution`).
*/
export const MUSIC_VIDEO_LIPSYNC_DEFAULT_RESOLUTION = '1080P';
/**
* Verified source-audio lip-sync lanes, keyed by Music Video backend.
*
* `maxAudioSec` is PortOS's per-take ceiling, not the provider's: fal accepts
* up to 15 minutes of audio. A performance take is one shot of the edit, and a
* flawed take can only be re-rolled whole, so a take is kept to about one
* sung phrase — 30s — and a longer shot is split on a lyric boundary. 30s also
* matches Seedance 2.5's audio-reference ceiling (30.2s), so a shot split for
* one lip-sync lane fits the other. Takes past 15s are billed at 1.2×; the
* cost label says so, and a director who wants to avoid it splits shorter.
*/
function localAudioLane(style) {
return Object.freeze({
provider: 'local',
style,
label: style === 'music' ? 'Local LTX-2.5 audio-to-video' : 'Local LTX-2.5 lip-sync',
modelId: null,
minAudioSec: 1,
maxAudioSec: 20,
transcription: false,
resolutions: Object.freeze([]),
defaultResolution: null,
costLabel: 'Free, renders on this machine',
});
}
export const SOURCE_AUDIO_LIPSYNC = Object.freeze({
fal: Object.freeze({
provider: 'fal',
style: 'lipsync',
label: 'fal.ai MiniMax H3 lip-sync',
modelId: FAL_LIPSYNC_VIDEO_MODEL,
minAudioSec: 5,
maxAudioSec: 30,
transcription: true,
resolutions: Object.freeze(falVideoResolutions(getFalVideoModel(FAL_LIPSYNC_VIDEO_MODEL))),
defaultResolution: MUSIC_VIDEO_LIPSYNC_DEFAULT_RESOLUTION,
// The published rate at the default resolution; the board shows each
// take's own estimate beside it (falSceneTake).
costLabel: `${describeFalVideoRate(FAL_LIPSYNC_VIDEO_MODEL, MUSIC_VIDEO_LIPSYNC_DEFAULT_RESOLUTION)} — billed to your fal.ai account`,
}),
// LTX audio-to-video on this machine: the song slice is the audio latent the
// picture is denoised against, held fixed through both passes
// (scripts/generate_ltx2.py `_install_a2v_stage2_audio_freeze`). The model is
// the project's local LTX pick, checked at submission; 20s keeps one take
// inside what LTX-2.5 renders well and within a Mac's memory.
local: localAudioLane('lipsync'),
});
// A music-driven take renders at its model's own default resolution.
// LTX audio-to-video is deliberately not a fal lane: PortOS runs LTX-2.3 /
// LTX-2.5 locally, where Make › Supplied song audio conditions a cutaway on
// its own song slice (submitJob.js `_suppliedAudio`).
const musicCapability = (model) => {
const audio = model.audioInput || model.referenceAudio;
// A reference model bills and renders whole seconds from its shortest clip
// up: the song window starts at that length (less the planning margin, so a
// padded window lands exactly on it) and never outruns the longest clip.
const clip = model.duration.kind === 'range' ? model.duration : null;
const defaultResolution = model.resolution.default;
return Object.freeze({
provider: 'fal',
style: 'music',
label: `fal.ai ${model.label}`,
modelId: model.id,
minAudioSec: clip ? Math.max(audio.minSec, clip.min - PERFORMANCE_WINDOW_MARGIN_SEC) : audio.minSec,
// The same one-phrase per-take ceiling as lip-sync (see above).
maxAudioSec: Math.min(audio.maxSec, clip ? clip.max : Infinity, 30),
transcription: false,
resolutions: Object.freeze(falVideoResolutions(model)),
defaultResolution,
costLabel: `${describeFalVideoRate(model.id, defaultResolution)} — billed to your fal.ai account`,
});
};
/**
* Verified music-driven lanes, keyed by backend then model id. The project's
* `videoSettings.falMusicModelId` picks one; absent (or a model this install
* does not curate) = `FAL_MUSIC_VIDEO_DEFAULT_MODEL` (MiniMax H3 Max reference).
*/
const LOCAL_MUSIC_LANE = localAudioLane('music');
export const SOURCE_AUDIO_MUSIC = Object.freeze({
fal: Object.freeze(Object.fromEntries(FAL_AUDIO_VIDEO_MODELS.map((model) => [model.id, musicCapability(model)]))),
});
/**
* The source-audio capability a backend offers a performance shot, or null.
* Without a scene (or for a lip-sync scene) it is the lip-sync lane; a
* music-driven scene gets the audio-to-video model `videoSettings` pins.
*/
export function performanceCapability(backend, { scene = null, videoSettings = null } = {}) {
if (!Object.prototype.hasOwnProperty.call(SOURCE_AUDIO_LIPSYNC, backend || '')) return null;
if (scene?.performanceStyle !== 'music') return SOURCE_AUDIO_LIPSYNC[backend];
if (backend === 'local') return LOCAL_MUSIC_LANE;
const lanes = SOURCE_AUDIO_MUSIC[backend];
const pinned = videoSettings?.falMusicModelId;
return (pinned && Object.prototype.hasOwnProperty.call(lanes, pinned) ? lanes[pinned] : null) || lanes[FAL_MUSIC_VIDEO_DEFAULT_MODEL];
}
/**
* Why a performance shot cannot render on `backend`, or null when it can.
* `backend` is the project's pinned Music Video backend ('' / null = the
* install default, which is never assumed to be a lip-sync provider).
*/
export function performanceBlockedReason(backend) {
if (performanceCapability(backend)) return null;
if (backend === 'grok' || backend === 'runway') {
return `${backend === 'grok' ? 'Grok' : 'Runway'} video is cutaway-only: it cannot synchronize a singer to the song. Pin fal.ai for performance shots, or switch this scene to Cutaway.`;
}
return 'Performance shots need a source-audio lip-sync provider — pin fal.ai as this project\'s video backend.';
}
const round6 = (n) => Math.round(n * 1e6) / 1e6;
/**
* The audio window a performance shot submits, for a scene spanning
* `[startSec, endSec]` of a song `songDurationSec` long.
*
* Returns `{ ok: true, spanSec, windowStartSec, windowEndSec, windowSec,
* editInSec, editOutSec }` — the window is absolute song time; the edit in/out
* points are relative to the generated clip (whose length follows the window).
* Returns `{ ok: false, code, message }` when the shot cannot be planned.
*
* `lineCues` (the project's lyric cues) asks for the whole sung line: the
* window grows to cover every timed cue the shot overlaps, so the take holds
* the full line and can be played whole or cut down. The edit points still
* frame the shot. A line too long for one take keeps the shot's own window and
* says so (`lineTooLong`); `lineStartSec` / `lineEndSec` are the span covered.
*/
export function planPerformanceWindow({ startSec, endSec, songDurationSec, capability, lineCues = null }) {
if (!capability) {
return { ok: false, code: 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED', message: performanceBlockedReason(null) };
}
if (typeof startSec !== 'number' || typeof endSec !== 'number' || !(endSec > startSec) || startSec < 0) {
return { ok: false, code: 'MUSIC_VIDEO_PERFORMANCE_UNTIMED', message: 'A performance shot needs a timed start and end on the song.' };
}
if (!(songDurationSec > 0) || endSec > songDurationSec + 1e-6) {
return { ok: false, code: 'MUSIC_VIDEO_PERFORMANCE_OUT_OF_RANGE', message: 'The performance shot runs past the end of the song.' };
}
const minSec = capability.minAudioSec + PERFORMANCE_WINDOW_MARGIN_SEC;
const maxSec = capability.maxAudioSec - PERFORMANCE_WINDOW_MARGIN_SEC;
const spanSec = endSec - startSec;
if (spanSec > maxSec) {
return {
ok: false,
code: 'MUSIC_VIDEO_PERFORMANCE_TOO_LONG',
message: `This performance shot runs ${spanSec.toFixed(2)}s, but ${capability.label} synchronizes at most ${maxSec.toFixed(2)}s of audio per take. Split the scene on a lyric or phrase boundary.`,
};
}
if (songDurationSec < minSec) {
return {
ok: false,
code: 'MUSIC_VIDEO_PERFORMANCE_SONG_TOO_SHORT',
message: `${capability.label} needs at least ${minSec.toFixed(2)}s of audio; the song is only ${songDurationSec.toFixed(2)}s.`,
};
}
let windowStartSec = startSec;
let windowEndSec = endSec;
let lineTooLong = false;
if (lineCues) {
const line = sungLineSpan(lineCues, startSec, endSec);
const lineStart = Math.max(0, line.startSec);
const lineEnd = Math.min(songDurationSec, line.endSec);
if (lineEnd - lineStart > maxSec) lineTooLong = true;
else {
windowStartSec = lineStart;
windowEndSec = lineEnd;
}
}
const lineStartSec = windowStartSec;
const lineEndSec = windowEndSec;
const coveredSec = windowEndSec - windowStartSec;
if (coveredSec < minSec) {
// Pad a short shot (or line) to a supported contextual window, centered on
// it where the song allows, then shifted to stay inside it. The edit
// in-point records exactly where the shot starts inside the generated clip.
const pad = minSec - coveredSec;
windowStartSec = Math.max(0, windowStartSec - pad / 2);
windowEndSec = windowStartSec + minSec;
if (windowEndSec > songDurationSec) {
windowEndSec = songDurationSec;
windowStartSec = songDurationSec - minSec;
}
}
return {
ok: true,
spanSec: round6(spanSec),
windowStartSec: round6(windowStartSec),
windowEndSec: round6(windowEndSec),
windowSec: round6(windowEndSec - windowStartSec),
editInSec: round6(startSec - windowStartSec),
editOutSec: round6(endSec - windowStartSec),
...(lineCues ? { lineStartSec: round6(lineStartSec), lineEndSec: round6(lineEndSec), lineTooLong } : {}),
};
}
/**
* `[startSec, endSec]` widened to the whole of every timed lyric cue it
* overlaps (a cue that only touches an edge does not count). A shot over no
* timed cue keeps its own span.
*/
function sungLineSpan(cues, startSec, endSec) {
let lineStart = startSec;
let lineEnd = endSec;
for (const cue of Array.isArray(cues) ? cues : []) {
if (!timedNumber(cue?.startSec)) continue;
const cueEnd = timedNumber(cue.endSec) ? cue.endSec : cue.startSec;
if (cueEnd <= startSec + 1e-6 || cue.startSec >= endSec - 1e-6) continue;
lineStart = Math.min(lineStart, cue.startSec);
lineEnd = Math.max(lineEnd, cueEnd);
}
return { startSec: lineStart, endSec: lineEnd };
}
/**
* Lyric cues overlapping `[windowStartSec, windowEndSec]`, re-timed relative to
* the window start and clamped to it. Untimed cues are dropped.
*/
export function clipRelativeCues(cues, windowStartSec, windowEndSec) {
const out = [];
for (const cue of Array.isArray(cues) ? cues : []) {
if (typeof cue?.startSec !== 'number') continue;
const end = typeof cue.endSec === 'number' ? cue.endSec : cue.startSec;
if (end < windowStartSec || cue.startSec > windowEndSec) continue;
out.push({
text: typeof cue.text === 'string' ? cue.text : '',
startSec: round6(Math.max(0, cue.startSec - windowStartSec)),
endSec: round6(Math.min(windowEndSec, end) - windowStartSec),
...(Array.isArray(cue.words) ? { words: cue.words
.filter((word) => word && Number.isFinite(word.startSec) && Number.isFinite(word.endSec) && word.endSec > word.startSec && word.endSec > windowStartSec && word.startSec < windowEndSec)
// Aligned lyric words keep their text as `w` (lyricAlignCore), transcribed ones as `text`.
.map((word) => ({ text: typeof word.text === 'string' ? word.text : typeof word.w === 'string' ? word.w : '', startSec: round6(Math.max(windowStartSec, word.startSec) - windowStartSec), endSec: round6(Math.min(windowEndSec, word.endSec) - windowStartSec) })) } : {}),
});
}
return out;
}
/**
* The Grok CLI request that covers a cutaway shot of `spanSec`: the shortest
* deliverable clip at least that long (6, 10 or 15 s), or the longest when none is.
* `uncoveredSec > 0` means one clip cannot cover the shot — split the scene
* rather than loop or stretch the clip.
*/
export function grokCoverage(spanSec) {
const requestSec = nearestGrokDuration(spanSec);
const longest = GROK_VIDEO_DURATIONS[GROK_VIDEO_DURATIONS.length - 1];
const span = Number(spanSec);
const uncoveredSec = Number.isFinite(span) && span > longest ? round6(span - longest) : 0;
return { requestSec, uncoveredSec, needsSplit: uncoveredSec > 0 };
}
/**
* Phrase intents overlapping a cutaway shot, as approximate clip-relative motion
* prose for providers that take no timed controls (Grok's CLI lane). The timing
* is explicitly approximate — the provider is not bound to it.
*/
export function approximateMotionCues(phrases, startSec, endSec) {
if (typeof startSec !== 'number' || typeof endSec !== 'number' || !(endSec > startSec)) return '';
const beats = [];
for (const phrase of Array.isArray(phrases) ? phrases : []) {
const intent = typeof phrase?.intent === 'string' ? phrase.intent.trim() : '';
if (!intent || typeof phrase.startSec !== 'number') continue;
const phraseEnd = typeof phrase.endSec === 'number' ? phrase.endSec : phrase.startSec;
if (phraseEnd <= startSec || phrase.startSec >= endSec) continue;
beats.push(`around ${Math.max(0, phrase.startSec - startSec).toFixed(1)}s: ${intent}`);
}
return beats.length ? `Approximate motion timing within the clip — ${beats.join('; ')}.` : '';
}
/**
* The longest single shot `backend` can render for `scene` without losing song
* coverage, or null when no per-take limit applies: a performance shot is bound
* by its lip-sync provider's audio window (inside the safety margin), a Grok
* footage cutaway by the longest clip Grok delivers. Other lanes — and a
* performance shot on a lane that cannot lip-sync at all, which is blocked
* outright rather than split — have no limit here.
*/
export function shotSplitLimit(scene, backend, videoSettings = null) {
if (isPerformanceScene(scene)) {
const capability = performanceCapability(backend, { scene, videoSettings });
return capability ? round6(capability.maxAudioSec - PERFORMANCE_WINDOW_MARGIN_SEC) : null;
}
const footage = scene?.visualLayer == null || scene.visualLayer === 'footage';
return backend === 'grok' && footage ? GROK_VIDEO_DURATIONS[GROK_VIDEO_DURATIONS.length - 1] : null;
}
// Shortest piece a split may produce, so a cut never strands a sliver shot.
export const SHOT_SPLIT_MIN_PIECE_SEC = 1;
// Cut-point preference, best first: a pause between sung lines, a lyric line
// or phrase boundary, a beat; the even-split fallback ranks below all three.
const CUT_RANK = Object.freeze({ pause: 0, line: 1, beat: 2 });
const CUT_KIND = Object.freeze(['pause', 'line', 'beat']);
const timedNumber = (v) => typeof v === 'number' && Number.isFinite(v);
function splitCandidates({ lyricCues, phrases, beats }) {
const out = [];
const timedCues = (Array.isArray(lyricCues) ? lyricCues : [])
.filter((cue) => timedNumber(cue?.startSec))
.map((cue) => ({ startSec: cue.startSec, endSec: timedNumber(cue.endSec) ? cue.endSec : cue.startSec }))
.sort((a, b) => a.startSec - b.startSec);
let sungUntil = -Infinity;
for (const cue of timedCues) {
// The middle of the silence between one line and the next is the cleanest
// cut: neither take has to start or stop mid-word.
if (cue.startSec > sungUntil && Number.isFinite(sungUntil)) out.push({ t: (sungUntil + cue.startSec) / 2, rank: CUT_RANK.pause });
out.push({ t: cue.startSec, rank: CUT_RANK.line });
sungUntil = Math.max(sungUntil, cue.endSec);
}
for (const phrase of Array.isArray(phrases) ? phrases : []) {
if (timedNumber(phrase?.startSec)) out.push({ t: phrase.startSec, rank: CUT_RANK.line });
if (timedNumber(phrase?.endSec)) out.push({ t: phrase.endSec, rank: CUT_RANK.line });
}
for (const beat of Array.isArray(beats) ? beats : []) {
if (timedNumber(beat)) out.push({ t: beat, rank: CUT_RANK.beat });
}
// A point strictly inside a sung line is never a lyric/phrase boundary; a
// beat may still land there, but that cut is not clean (it splits a word).
const insideLine = (t) => timedCues.some((cue) => t > cue.startSec + 1e-6 && t < cue.endSec - 1e-6);
return out
.map((c) => ({ ...c, clean: !insideLine(c.t) }))
.filter((c) => c.rank === CUT_RANK.beat || c.clean);
}
// Cut `[startSec, endSec]` into exactly `count` pieces of at most `maxSec`,
// or null when that count cannot keep every piece at least `minPieceSec`.
function cutPieces({ startSec, endSec, maxSec, minPieceSec, count, candidates }) {
if ((endSec - startSec) / count < minPieceSec) return null;
const pieces = [];
let clean = true;
let cursor = startSec;
for (let remaining = count; remaining > 1; remaining -= 1) {
// Any cut in [lo, hi] leaves the rest coverable by `remaining - 1` pieces
// and keeps both sides at least `minPieceSec` long.
const lo = Math.max(cursor + minPieceSec, endSec - (remaining - 1) * maxSec);
const hi = Math.min(cursor + maxSec, endSec - minPieceSec);
const target = cursor + (endSec - cursor) / remaining;
let best = null;
for (const c of candidates) {
if (c.t < lo - 1e-6 || c.t > hi + 1e-6) continue;
if (!best || c.rank < best.rank || (c.rank === best.rank && Math.abs(c.t - target) < Math.abs(best.t - target))) best = c;
}
clean = clean && Boolean(best?.clean);
const at = round6(Math.min(hi, Math.max(lo, best ? best.t : target)));
pieces.push({ startSec: round6(cursor), endSec: at, cut: best ? CUT_KIND[best.rank] : 'even' });
cursor = at;
}
pieces.push({ startSec: round6(cursor), endSec: round6(endSec), cut: null });
return { pieces, clean };
}
// How many pieces beyond the minimum a split may add to avoid cutting a sung
// line mid-word. Each piece is a separate (possibly paid) take, so the search
// stays tight: one extra take for a clean cut, never more.
export const SHOT_SPLIT_MAX_EXTRA_PIECES = 1;
/**
* Split a shot spanning `[startSec, endSec]` into contiguous pieces of at most
* `maxSec` each, cutting on musical boundaries: the pause between two timed
* lyric lines first, then a line or phrase boundary, then a beat, and only
* when none lies in reach an even cut. It uses the fewest pieces unless one
* more (`SHOT_SPLIT_MAX_EXTRA_PIECES`) is what lets every cut avoid splitting
* a sung line. Pieces cover the span exactly — no gap, no overlap — so nothing
* is looped, stretched or dropped.
*
* Returns `{ ok: true, pieces: [{ startSec, endSec, cut }] }` — `cut` names what
* the piece's END was cut on (`pause` | `line` | `beat` | `even`; null for
* the last piece, which ends where the shot did) — or `{ ok: false, code,
* message }` when the shot is untimed or already fits.
*/
export function planShotSplit({ startSec, endSec, maxSec, lyricCues, phrases, beats, minPieceSec = SHOT_SPLIT_MIN_PIECE_SEC }) {
if (!timedNumber(startSec) || !timedNumber(endSec) || !(endSec > startSec)) {
return { ok: false, code: 'MUSIC_VIDEO_SPLIT_UNTIMED', message: 'Set a start and end before splitting a shot.' };
}
if (!(maxSec > 0) || endSec - startSec <= maxSec + 1e-6) {
return { ok: false, code: 'MUSIC_VIDEO_SPLIT_NOT_NEEDED', message: 'This shot already fits in a single take.' };
}
const candidates = splitCandidates({ lyricCues, phrases, beats });
// The same 1µs tolerance as the fits-one-take check above: float noise in
// a span like 29.9999999 must not demand a fourth 10s piece, and no piece
// may exceed the limit by more than that tolerance.
const fewest = Math.ceil((endSec - startSec - 1e-6) / maxSec);
const base = { startSec, endSec, maxSec, minPieceSec, candidates };
const first = cutPieces({ ...base, count: fewest });
let chosen = first;
for (let extra = 1; !chosen.clean && extra <= SHOT_SPLIT_MAX_EXTRA_PIECES; extra += 1) {
const more = cutPieces({ ...base, count: fewest + extra });
if (more?.clean) chosen = more;
}
return { ok: true, pieces: chosen.pieces };
}
/**
* The fal.ai request one take of `scene` makes on the fal lane, and its
* estimated cost — the single answer the board's cost line, the submit
* payload and the production autopilot's dollar cap all read, so what the
* director is shown is what is sent and what is charged against the cap.
*
* A performance scene renders on the lip-sync route for its planned audio
* window at `videoSettings.falLipSyncResolution` (default 1080P). A cutaway
* renders on `videoSettings.falModelId` (default Hailuo-02 image-to-video) at
* `falResolution` (default: the model's), for the pinned `falDuration` or else
* the shot's span — rounded UP to a length the model delivers, so the clip
* covers the shot. A shot with a style-break end frame renders on
* `videoSettings.falEndFrameModelId` when the project picked one that takes an
* end frame — a dearer model is an explicit upgrade for those shots only, never
* the footage default. An uncurated model keeps the legacy contract (only an
* explicit pin is sent) and reports cost unknown.
*
* `lineCues` prices a performance take that covers the whole sung line
* (planPerformanceWindow).
*
* Returns `{ performance, modelId, seconds, resolution, costUsd }`; `seconds`
* and `costUsd` are null when unknown (an untimed or unplannable performance,
* an uncurated model).
*/
export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null, lineCues = null } = {}) {
const settings = videoSettings || {};
if (isPerformanceScene(scene)) {
const capability = performanceCapability('fal', { scene, videoSettings: settings });
const model = getFalVideoModel(capability.modelId);
const pinned = capability.style === 'music' ? settings.falMusicResolution : settings.falLipSyncResolution;
const resolution = resolveFalVideoResolution(model, pinned || capability.defaultResolution);
const plan = planPerformanceWindow({
startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability, lineCues,
});
const seconds = plan.ok ? plan.windowSec : null;
return {
performance: true,
music: capability.style === 'music',
modelId: capability.modelId,
seconds,
resolution,
costUsd: seconds == null ? null : estimateFalVideoCostUsd({ modelId: capability.modelId, seconds, resolution }),
};
}
const endFrameModel = typeof scene?.endImageId === 'string' && scene.endImageId.trim()
? getFalVideoModel(settings.falEndFrameModelId) : null;
const modelId = endFrameModel?.endImage ? endFrameModel.id : settings.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL;
const model = getFalVideoModel(modelId);
if (!model) {
return { performance: false, modelId, seconds: settings.falDuration || null, resolution: settings.falResolution || null, costUsd: null };
}
const spanSec = typeof scene?.startSec === 'number' && typeof scene?.endSec === 'number' && scene.endSec > scene.startSec
? scene.endSec - scene.startSec
: null;
const seconds = coerceFalVideoSeconds(model, settings.falDuration || spanSec);
const resolution = resolveFalVideoResolution(model, settings.falResolution);
return { performance: false, modelId, seconds, resolution, costUsd: estimateFalVideoCostUsd({ modelId, seconds, resolution }) };
}
/**
* A `falSceneTake` as `/api/video-gen` fields. A performance take sends its
* resolution (and a music-driven take its model, which a shot card can pick) —
* its length follows the song slice the server cuts; a cutaway names its
* model, length and resolution. Unknown values stay absent so the server
* resolves its own defaults (an uncurated model's legacy body).
*/
export const falTakeRequestFields = (take) => (take.performance
? { ...(take.music ? { falModelId: take.modelId } : {}), ...(take.resolution ? { falResolution: take.resolution } : {}) }
: {
falModelId: take.modelId,
...(take.seconds != null ? { falDuration: take.seconds } : {}),
...(take.resolution ? { falResolution: take.resolution } : {}),
});
/**
* The Runway request one cutaway take of `scene` makes, and its estimated cost:
* `videoSettings.runwayModelId` (default Gen-4.5) at `runwayResolution`, for
* the pinned `runwayDuration` or else the shot's span rounded UP to a length the
* model delivers. A shot with an end frame stays on its model; the end frame is
* sent only when that model takes one (musicVideoEndFrame.js). Returns
* `{ performance: false, modelId, seconds, resolution, costUsd }`; Runway has
* no performance lane, so a performance scene is priced as nothing (null).
*/
export function runwaySceneTake({ scene, videoSettings = {} } = {}) {
const settings = videoSettings || {};
const modelId = settings.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL;
const model = getRunwayVideoModel(modelId);
if (isPerformanceScene(scene) || !model) {
return { performance: isPerformanceScene(scene), modelId, seconds: null, resolution: null, costUsd: null };
}
const spanSec = typeof scene?.startSec === 'number' && typeof scene?.endSec === 'number' && scene.endSec > scene.startSec
? scene.endSec - scene.startSec
: null;
const seconds = coerceRunwayVideoSeconds(model, settings.runwayDuration || spanSec);
const resolution = resolveRunwayVideoResolution(model, settings.runwayResolution);
return { performance: false, modelId, seconds, resolution, costUsd: estimateRunwayVideoCostUsd({ modelId, seconds, resolution }) };
}
/** A `runwaySceneTake` as `/api/video-gen` fields. */
export const runwayTakeRequestFields = (take) => ({
runwayModelId: take.modelId,
...(take.seconds != null ? { runwayDuration: take.seconds } : {}),
...(take.resolution ? { runwayResolution: take.resolution } : {}),
});
/**
* The priced take of `scene` on a paid lane (`fal` or `runway`), or null for a
* lane PortOS cannot price (local, Grok): the one answer the board's cost line,
* the batch estimate, the submit path and the autopilot's cap read.
*/
export function meteredSceneTake(backend, options = {}) {
if (backend === 'fal') return falSceneTake(options);
if (backend === 'runway') return runwaySceneTake(options);
return null;
}
/** The label of a paid take's model, from whichever catalog holds it. */
export const meteredModelLabel = (modelId) => getFalVideoModel(modelId)?.label || getRunwayVideoModel(modelId)?.label || null;
/** A repair needs measured complete coverage and a genuine pause between words.
* Returned times distinguish song, excerpt and provider-clip timebases.
* No padding: accepted audio must never be sent for generation again.
*/
export function planPerformanceRepair({ scene, temporal, excerptStartSec, backend, videoSettings = {} }) {
const no = (message) => ({ ok: false, message });
const instruction = selectedPerformanceInstruction(scene);
const capability = performanceCapability(backend, { scene, videoSettings });
if (!capability) return no(performanceBlockedReason(backend));
if (!instruction || instruction.songInterval?.startSec !== scene.startSec || instruction.songInterval?.endSec !== scene.endSec) return no('Review needed: the selected take no longer matches this shot');
if (scene.direction?.actionContract || scene.performanceRepair) return no('Review needed: edit the shot contract or review the existing continuation before repairing again');
if (temporal?.status !== 'verified' || !temporal.analyzer || temporal.takeId !== scene.videoHistoryId || !Number.isFinite(excerptStartSec)) return no('Review needed: complete measured temporal evidence is required');
const spans = Array.isArray(temporal.spans) ? temporal.spans : [];
if (!spans.length || spans.length > 256) return no('Review needed: temporal coverage is missing or unbounded');
let cursor = scene.startSec;
let failedAt = null;
for (const span of spans) {
const start = span.startSec + excerptStartSec;
const end = span.endSec + excerptStartSec;
if (span.status !== 'verified' || !Number.isFinite(span.offsetSec) || !Number.isFinite(span.confidence) || !(span.confidence >= 0.8 && span.confidence <= 1)
|| !Number.isFinite(end) || end <= start || Math.abs(start - cursor) > 0.001 || end > scene.endSec + 0.001) return no('Review needed: temporal coverage is incomplete or inconclusive');
const failed = Math.abs(span.offsetSec) > 0.12;
if (failed && failedAt === null) failedAt = start;
if (!failed && failedAt !== null) return no('Review needed: the failure is not confined to a suffix');
cursor = end;
}
if (Math.abs(cursor - scene.endSec) > 0.001 || failedAt === null || failedAt <= scene.startSec) return no('Review needed: no accepted prefix and failed suffix were measured');
const windowStart = instruction.audioWindow?.startSec;
// Song time of clip time 0: an aligned take's window starts `offset` into its clip.
const clipStart = windowStart - clipAudioOffsetSec(instruction);
if (!Number.isFinite(windowStart) || Math.abs(clipStart + instruction.edit.inSec - scene.startSec) > 0.001
|| Math.abs(clipStart + instruction.edit.outSec - scene.endSec) > 0.001) return no('Review needed: the source audio timebase is missing');
const cues = (instruction.cues || []).filter((cue) => cue.endSec + windowStart > scene.startSec && cue.startSec + windowStart < scene.endSec);
if (!cues.length || cues.some((cue) => !cue.words?.length)) return no('Review needed: word timings are required to choose a safe boundary');
const words = cues.flatMap((cue) => cue.words).slice().sort((a, b) => a.startSec - b.startSec);
if (words.some((word) => !Number.isFinite(word.startSec) || !Number.isFinite(word.endSec) || word.endSec <= word.startSec)) return no('Review needed: word timings are invalid');
let boundarySec = null;
for (let i = 1; i < words.length; i += 1) {
const left = words[i - 1].endSec + windowStart;
const right = words[i].startSec + windowStart;
const boundary = round6((left + right) / 2);
// A gap is not safe if another overlapping word crosses it.
if (right - left < 0.02 || words.some((word) => word.startSec + windowStart < boundary && word.endSec + windowStart > boundary)) continue;
const suffixSec = scene.endSec - boundary;
if (boundary - scene.startSec >= 0.4 && boundary <= failedAt && suffixSec >= capability.minAudioSec + PERFORMANCE_WINDOW_MARGIN_SEC
&& suffixSec <= capability.maxAudioSec - PERFORMANCE_WINDOW_MARGIN_SEC) boundarySec = boundary;
}
if (boundarySec === null) return no('Review needed: no word gap leaves a supported suffix window; choose another reviewed shot boundary');
const suffix = { ...scene, startSec: boundarySec };
const take = falSceneTake({ scene: suffix, videoSettings, songDurationSec: instruction.audio?.songDurationSec });
return { ok: true, boundarySec, endSec: scene.endSec, suffixSec: round6(scene.endSec - boundarySec),
referenceClipSec: round6(instruction.edit.inSec + boundarySec - scene.startSec),
costUsd: take?.costUsd ?? null, sourceAssetId: scene.videoHistoryId };
}