Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Lip-sync shots can render the whole sung line

Claudea7f033d57476
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 7cbbc553fe..b9094723d0 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -163,12 +163,15 @@ export default function SceneCard({
const renderOverrides = renderTargetOverrides(renderPick, falVideoSettings || {}, renderOptions,
{ models: videoModels, defaultModel: defaultVideoModel, performance: isPerformanceScene(scene), scene });
const renderSettings = renderOverrides ? { ...falVideoSettings, ...renderOverrides } : falVideoSettings;
+ // The card's input toggles (SceneRenderInputs); Whole sung line also re-plans the take below.
+ const [renderInputs, setRenderInputs] = useState({});
const renderLane = renderOverrides
? (renderSettings.backend === 'local' && renderSettings.generationMode === 'audioReactive' ? 'local' : renderSettings.backend)
: lipSyncBackend;
const timing = buildSceneTimingPresentation({
scene, layer, lipSyncBackend: renderLane, songDurationSec, falVideoSettings: renderSettings,
nextEndSec, isLast, documentComposition, hasMergeNext: !!onMergeNext,
+ lineCues: isPerformanceScene(scene) && renderInputs.wholeLine === true ? project?.lyricCues || [] : null,
});
const { performance, capability, performanceBlocked, performanceCost } = timing;
const instruction = scene.takes?.find((take) => take.kind === 'video' && take.assetId === scene.videoHistoryId)?.shotInstruction;
@@ -191,9 +194,8 @@ export default function SceneCard({
? scene.lastFailure : null;
const endFramePlan = sceneEndFramePlanFor(scene, renderSettings || {}, { models: videoModels, defaultModel: defaultVideoModel });
// Exactly what Render sends: the prompt as typed and the inputs the card toggles.
- const [renderInputs, setRenderInputs] = useState({});
const cardRequest = shotCardRenderRequest({ project, scene, settings: renderSettings || {}, performance, endFramePlan,
- models: videoModels, defaultModel: defaultVideoModel, inputs: renderInputs });
+ models: videoModels, defaultModel: defaultVideoModel, inputs: renderInputs, plan: timing.plan });
const videoRefusal = videoGenerationRefusal({
settingsSaving, speakerUnsaved: performance && (speakerDirty || speakerSaving),
referenceImageId: scene.referenceImageId, generatingVideo, performanceBlocked, lockBlocked,
@@ -201,10 +203,11 @@ export default function SceneCard({
}) || shotCardRequestProblem(cardRequest);
const generateVideo = () => {
if (videoRefusal) return;
- const { songAudio, endFrame } = cardRequest;
+ const { songAudio, endFrame, wholeLine } = cardRequest;
const card = {
prompt: cardRequest.authored,
...(endFrame ? { endFrame: endFrame.state === 'on' } : {}),
+ ...(wholeLine?.state === 'on' ? { wholeLine: true } : {}),
...(songAudio.state === 'on' || songAudio.state === 'off' ? { songAudio: songAudio.state === 'on' } : {}),
};
onGenerateVideo?.(scene, { ...(renderOverrides ? { target: renderOverrides } : {}), card });
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index a3c2b76b4e..9cac19e2dc 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -471,6 +471,29 @@ describe('Sent to the model', () => {
});
});
+// Uniquely catches a Whole sung line take priced or sent for the shot alone, and the
+// render target naming the project's cutaway model for a lip-sync shot.
+describe('Whole sung line', () => {
+ it('re-plans and prices the take over the full line, sends it with Render, and names the lip-sync model', () => {
+ const scene = { sceneId: 's19', prompt: '', shotMode: 'performance', referenceImageId: 'start.png', startSec: 40.5, endSec: 42.8,
+ takes: [{ takeId: 't1', kind: 'image', assetId: 'start.png', status: 'selected' }] };
+ const project = { id: 'p1', lyricCues: [{ text: 'the whole line', startSec: 39.2, endSec: 45.6 }] };
+ const onGenerateVideo = vi.fn();
+ render(<SceneCard scene={scene} project={project} index={0} expanded onEditLocal={() => {}} onSave={() => {}} onGenerateVideo={onGenerateVideo}
+ lipSyncBackend="fal" songDurationSec={60} falVideoSettings={{ backend: 'fal' }} />);
+ expect(screen.getByLabelText('Render on').textContent).toContain('Project setting · fal.ai · MiniMax H3 Max lip-sync');
+ expect(screen.getByLabelText('Render on').textContent).not.toContain('Hailuo');
+ const plan = () => screen.getByTestId('performance-plan').textContent;
+ expect(plan()).toContain('est. $0.81 (5.05s at 1080P)');
+ fireEvent.click(within(screen.getByTestId('render-inputs')).getByLabelText('Whole sung line'));
+ expect(within(screen.getByTestId('render-inputs')).getByText('· Song 0:39.2–0:45.6')).toBeTruthy();
+ expect(plan()).toContain('shot starts 1.30s into the take');
+ expect(plan()).toContain('est. $1.02 (6.4s at 1080P)');
+ fireEvent.click(screen.getByRole('button', { name: 'Render' }));
+ expect(onGenerateVideo).toHaveBeenCalledWith(scene, expect.objectContaining({ card: expect.objectContaining({ wholeLine: true }) }));
+ });
+});
+
// Uniquely catches the glitch picker saving the wrong field or failing to clear it.
describe('VHS glitch transitions', () => {
it('sets a glitch on the cut out of a shot and clears it with Off', () => {
diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsx
index 875c0e57b8..5d58f698b4 100644
--- a/client/src/components/musicVideo/SceneRenderInputs.jsx
+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx
@@ -23,12 +23,14 @@ const clock = (sec) => {
*
* `settings` is the renderer this render uses (the project's, or the card's
* Render on pick merged over it); `inputs` holds the card's toggles,
- * `{ endFrame, songAudio }`, each undefined until the director flips it.
- * Returns `{ authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio, contractProblem }`, where each
- * input is `{ state: 'on' | 'off' | 'required' | 'unavailable', note }`.
+ * `{ endFrame, songAudio, wholeLine }`, each undefined until the director flips it.
+ * `plan` is the performance shot's planned song window (planPerformanceWindow),
+ * which names the line a Whole sung line take covers.
+ * Returns `{ authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio, wholeLine, contractProblem }`, where each
+ * input is `{ state: 'on' | 'off' | 'required' | 'unavailable', note }` (`wholeLine` is null off a performance shot).
*/
export function shotCardRenderRequest({
- project = null, scene, settings = {}, performance = false, endFramePlan = null, models = [], defaultModel = '', inputs = {},
+ project = null, scene, settings = {}, performance = false, endFramePlan = null, models = [], defaultModel = '', inputs = {}, plan = null,
}) {
const falModel = settings.backend === 'fal' ? getFalVideoModel(falSceneTake({ scene, videoSettings: settings }).modelId) : null;
const localModel = settings.backend === 'local'
@@ -63,6 +65,17 @@ export function shotCardRenderRequest({
else endFrame = { state: 'unavailable', note: null };
}
+ // A performance take can hear the whole lyric line the shot sits in, so the
+ // clip can play the full line or be cut down to the shot.
+ let wholeLine = null;
+ if (performance) {
+ const on = inputs.wholeLine === true;
+ const note = !on || !plan?.ok || plan.lineStartSec == null ? null
+ : plan.lineTooLong ? 'Line too long for one take; renders the shot'
+ : `Song ${clock(plan.lineStartSec)}–${clock(plan.lineEndSec)}`;
+ wholeLine = { state: on ? 'on' : 'off', note };
+ }
+
return {
authored,
prompt,
@@ -71,6 +84,7 @@ export function shotCardRenderRequest({
maxChars: falModel?.promptMaxChars ?? null,
endFrame,
songAudio,
+ wholeLine,
contractProblem,
};
}
@@ -89,7 +103,7 @@ function InputToggle({ id, label, input, onChange }) {
/** The card's "Sent to the model" block: the exact prompt and the inputs going with it. */
export default function SceneRenderInputs({ id, request, onChange }) {
- const { authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio } = request;
+ const { authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio, wholeLine } = request;
const over = maxChars != null && prompt.length > maxChars;
return (
<div className="min-w-0 space-y-1 rounded border border-port-border p-2 text-xs" data-testid="render-inputs">
@@ -113,6 +127,7 @@ export default function SceneRenderInputs({ id, request, onChange }) {
<InputToggle id={`${id}-still`} label="Still" input={{ state: 'required', note: null }} />
{endFrame && <InputToggle id={`${id}-end`} label="End frame" input={endFrame} onChange={(on) => onChange({ endFrame: on })} />}
<InputToggle id={`${id}-audio`} label="Song audio" input={songAudio} onChange={(on) => onChange({ songAudio: on })} />
+ {wholeLine && <InputToggle id={`${id}-line`} label="Whole sung line" input={wholeLine} onChange={(on) => onChange({ wholeLine: on })} />}
</div>
</div>
);
diff --git a/client/src/components/musicVideo/SceneRenderTarget.jsx b/client/src/components/musicVideo/SceneRenderTarget.jsx
index 6e7849bfe6..331173ff0c 100644
--- a/client/src/components/musicVideo/SceneRenderTarget.jsx
+++ b/client/src/components/musicVideo/SceneRenderTarget.jsx
@@ -2,7 +2,7 @@ import {
FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, coerceFalVideoSeconds, describeFalVideoRate, falVideoResolutions, getFalVideoModel,
} from '../../lib/falVideoModels.js';
import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
-import { grokCoverage } from '../../lib/musicVideoShotTiming.js';
+import { grokCoverage, performanceCapability } from '../../lib/musicVideoShotTiming.js';
import { localMusicVideoModels } from '../../../../server/lib/musicVideoLocalModels.js';
import { frameOptionsForModel } from '../../lib/videoGenParams.js';
import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage.js';
@@ -30,18 +30,27 @@ const BACKEND_LABELS = { local: 'Local LTX', grok: 'Grok', fal: 'fal.ai' };
const LOCAL_FPS = 24;
const SELECT = 'min-w-0 max-w-full bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50';
-function savedLabel(saved) {
- if (saved.backend === 'fal') return `fal.ai · ${getFalVideoModel(saved.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL)?.label || 'video'}`;
+// The fal model a performance shot renders on: the lip-sync route, or the
+// project's music-driven model for a music-driven shot. Never the cutaway model.
+const falPerformanceLabel = (saved, scene) => {
+ const capability = performanceCapability('fal', { scene, videoSettings: saved });
+ return getFalVideoModel(capability?.modelId)?.label || 'lip-sync';
+};
+
+function savedLabel(saved, performance, scene) {
+ if (saved.backend === 'fal') {
+ return `fal.ai · ${performance ? falPerformanceLabel(saved, scene) : getFalVideoModel(saved.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL)?.label || 'video'}`;
+ }
return BACKEND_LABELS[saved.backend] || 'install default';
}
-export function renderTargetOptions({ performance, saved, endFrame = false }) {
+export function renderTargetOptions({ performance, saved, endFrame = false, scene = null }) {
return [
- { value: '', label: `Project setting · ${savedLabel(saved)}` },
+ { value: '', label: `Project setting · ${savedLabel(saved, performance, scene)}` },
{ value: 'local', label: performance ? 'Local LTX lip-sync (free)' : 'Local LTX (free)' },
{ value: 'grok', label: performance ? 'Grok (cutaways only)' : 'Grok', disabled: performance },
...(performance
- ? [{ value: 'fal', label: 'fal.ai (song-driven)' }]
+ ? [{ value: 'fal', label: `fal.ai · ${falPerformanceLabel(saved, scene)}` }]
: FAL_IMAGE_VIDEO_MODELS.map((model) => {
const rate = describeFalVideoRate(model.id);
return { value: `fal:${model.id}`, label: `fal.ai · ${model.label}${rate ? ` · ${rate}` : ''}${endFrame && model.endImage ? ' · ends on end frame' : ''}` };
@@ -175,7 +184,7 @@ export default function SceneRenderTarget({
<select id={id} value={value} onChange={(e) => { onChange(e.target.value); onOptionsChange?.({}); rememberRenderTarget(e.target.value); }} disabled={disabled}
title="Where this shot's clip renders. Only this shot; the project setting stays as it is."
className={`${SELECT} flex-1 sm:flex-none`}>
- {renderTargetOptions({ performance, saved, endFrame: Boolean(scene?.endImageId) }).map((option) => (
+ {renderTargetOptions({ performance, saved, endFrame: Boolean(scene?.endImageId), scene }).map((option) => (
<option key={option.value} value={option.value} disabled={option.disabled}>{option.label}</option>
))}
</select>
diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.js
index 779069077b..1bb56665b5 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.js
+++ b/client/src/hooks/useMusicVideoSceneMedia.js
@@ -322,7 +322,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const problem = shotActionContractProblem(scene.direction?.actionContract, scene);
if (problem) { toast.error(problem); return Promise.resolve({ ok: false }); }
// `card` is the shot card's Render: the director's text and its input toggles
- // ({ prompt, endFrame, songAudio }), composed as the card shows it (SceneRenderInputs).
+ // ({ prompt, endFrame, songAudio, wholeLine }), composed as the card shows it (SceneRenderInputs).
// The server composes every addition from the director's text (musicVideoShotPrompt.js).
const prompt = card ? card.prompt : authoredShotPrompt(scene);
if (!prompt) { toast.error('Add a shot prompt first'); return Promise.resolve({ ok: false }); }
@@ -387,6 +387,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
projectId: project.id, sceneId: scene.sceneId, ...(revisionId ? { revisionId } : {}),
compose: true, ...(audioReactiveSelected ? { audioReactive: true } : {}),
...(card ? { promptAsTyped: true, ...(typeof card.songAudio === 'boolean' && !performance ? { songAudio: card.songAudio } : {}) } : {}),
+ ...(card?.wholeLine && performance ? { wholeLine: true } : {}),
}),
})
.then((res) => trackVideoJob(res, scene.sceneId, onJob))
diff --git a/client/src/lib/musicVideoSceneTiming.js b/client/src/lib/musicVideoSceneTiming.js
index 781899644a..a54834f39b 100644
--- a/client/src/lib/musicVideoSceneTiming.js
+++ b/client/src/lib/musicVideoSceneTiming.js
@@ -13,7 +13,7 @@ const falEstimate = (take) => (take?.costUsd == null ? null
*/
export function buildSceneTimingPresentation({
scene, layer, lipSyncBackend, songDurationSec, falVideoSettings,
- nextEndSec, isLast, documentComposition, hasMergeNext,
+ nextEndSec, isLast, documentComposition, hasMergeNext, lineCues = null,
}) {
const performance = isPerformanceScene(scene);
const capability = performance ? performanceCapability(lipSyncBackend, { scene, videoSettings: falVideoSettings }) : null;
@@ -21,14 +21,14 @@ export function buildSceneTimingPresentation({
? scene.endSec - scene.startSec
: null;
const plan = capability
- ? planPerformanceWindow({ startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability })
+ ? planPerformanceWindow({ startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability, lineCues })
: null;
const performanceBlocked = performance
? (capability ? (plan.ok ? null : plan.message) : performanceBlockedReason(lipSyncBackend))
: null;
const grokPlan = !performance && lipSyncBackend === 'grok' && timedSpan != null ? grokCoverage(timedSpan) : null;
const falTake = lipSyncBackend === 'fal' && layer === 'footage'
- ? falSceneTake({ scene, videoSettings: falVideoSettings || {}, songDurationSec })
+ ? falSceneTake({ scene, videoSettings: falVideoSettings || {}, songDurationSec, lineCues })
: null;
const falCost = falEstimate(falTake);
const performanceCost = falTake?.performance ? (falCost || capability?.costLabel) : capability?.costLabel;
diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.js
index 4409fc4d18..4f1ebd9f6e 100644
--- a/server/lib/musicVideoShotTiming.js
+++ b/server/lib/musicVideoShotTiming.js
@@ -233,8 +233,14 @@ const round6 = (n) => Math.round(n * 1e6) / 1e6;
* editInSec, editOutSec }` — the window is absolute song time; the edit in/out
* points are relative to the generated clip (whose length follows the window).
* Returns `{ ok: false, code, message }` when the shot cannot be planned.
+ *
+ * `lineCues` (the project's lyric cues) asks for the whole sung line: the
+ * window grows to cover every timed cue the shot overlaps, so the take holds
+ * the full line and can be played whole or cut down. The edit points still
+ * frame the shot. A line too long for one take keeps the shot's own window and
+ * says so (`lineTooLong`); `lineStartSec` / `lineEndSec` are the span covered.
*/
-export function planPerformanceWindow({ startSec, endSec, songDurationSec, capability }) {
+export function planPerformanceWindow({ startSec, endSec, songDurationSec, capability, lineCues = null }) {
if (!capability) {
return { ok: false, code: 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED', message: performanceBlockedReason(null) };
}
@@ -263,12 +269,26 @@ export function planPerformanceWindow({ startSec, endSec, songDurationSec, capab
}
let windowStartSec = startSec;
let windowEndSec = endSec;
- if (spanSec < minSec) {
- // Pad a short shot to a supported contextual window, centered on the shot
- // where the song allows, then shifted to stay inside it. The edit in-point
- // records exactly where the shot starts inside the generated clip.
- const pad = minSec - spanSec;
- windowStartSec = Math.max(0, startSec - pad / 2);
+ let lineTooLong = false;
+ if (lineCues) {
+ const line = sungLineSpan(lineCues, startSec, endSec);
+ const lineStart = Math.max(0, line.startSec);
+ const lineEnd = Math.min(songDurationSec, line.endSec);
+ if (lineEnd - lineStart > maxSec) lineTooLong = true;
+ else {
+ windowStartSec = lineStart;
+ windowEndSec = lineEnd;
+ }
+ }
+ const lineStartSec = windowStartSec;
+ const lineEndSec = windowEndSec;
+ const coveredSec = windowEndSec - windowStartSec;
+ if (coveredSec < minSec) {
+ // Pad a short shot (or line) to a supported contextual window, centered on
+ // it where the song allows, then shifted to stay inside it. The edit
+ // in-point records exactly where the shot starts inside the generated clip.
+ const pad = minSec - coveredSec;
+ windowStartSec = Math.max(0, windowStartSec - pad / 2);
windowEndSec = windowStartSec + minSec;
if (windowEndSec > songDurationSec) {
windowEndSec = songDurationSec;
@@ -283,9 +303,28 @@ export function planPerformanceWindow({ startSec, endSec, songDurationSec, capab
windowSec: round6(windowEndSec - windowStartSec),
editInSec: round6(startSec - windowStartSec),
editOutSec: round6(endSec - windowStartSec),
+ ...(lineCues ? { lineStartSec: round6(lineStartSec), lineEndSec: round6(lineEndSec), lineTooLong } : {}),
};
}
+/**
+ * `[startSec, endSec]` widened to the whole of every timed lyric cue it
+ * overlaps (a cue that only touches an edge does not count). A shot over no
+ * timed cue keeps its own span.
+ */
+export function sungLineSpan(cues, startSec, endSec) {
+ let lineStart = startSec;
+ let lineEnd = endSec;
+ for (const cue of Array.isArray(cues) ? cues : []) {
+ if (!timedNumber(cue?.startSec)) continue;
+ const cueEnd = timedNumber(cue.endSec) ? cue.endSec : cue.startSec;
+ if (cueEnd <= startSec + 1e-6 || cue.startSec >= endSec - 1e-6) continue;
+ lineStart = Math.min(lineStart, cue.startSec);
+ lineEnd = Math.max(lineEnd, cueEnd);
+ }
+ return { startSec: lineStart, endSec: lineEnd };
+}
+
/**
* Lyric cues overlapping `[windowStartSec, windowEndSec]`, re-timed relative to
* the window start and clamped to it. Untimed cues are dropped.
@@ -481,11 +520,14 @@ export function planShotSplit({ startSec, endSec, maxSec, lyricCues, phrases, be
* the footage default. An uncurated model keeps the legacy contract (only an
* explicit pin is sent) and reports cost unknown.
*
+ * `lineCues` prices a performance take that covers the whole sung line
+ * (planPerformanceWindow).
+ *
* Returns `{ performance, modelId, seconds, resolution, costUsd }`; `seconds`
* and `costUsd` are null when unknown (an untimed or unplannable performance,
* an uncurated model).
*/
-export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null } = {}) {
+export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null, lineCues = null } = {}) {
const settings = videoSettings || {};
if (isPerformanceScene(scene)) {
const capability = performanceCapability('fal', { scene, videoSettings: settings });
@@ -493,7 +535,7 @@ export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null
const pinned = capability.style === 'music' ? settings.falMusicResolution : settings.falLipSyncResolution;
const resolution = resolveFalVideoResolution(model, pinned || capability.defaultResolution);
const plan = planPerformanceWindow({
- startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability,
+ startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability, lineCues,
});
const seconds = plan.ok ? plan.windowSec : null;
return {
diff --git a/server/lib/musicVideoShotTiming.test.js b/server/lib/musicVideoShotTiming.test.js
index cc31f4943f..94c622ae4b 100644
--- a/server/lib/musicVideoShotTiming.test.js
+++ b/server/lib/musicVideoShotTiming.test.js
@@ -104,6 +104,39 @@ describe('local LTX lip-sync lane', () => {
});
});
+// Whole sung line: the take hears the full line while the edit still frames
+// the shot. Catches a window that drops the line's start or end, counts a cue
+// that only touches the shot, or overruns the per-take limit.
+describe('whole sung line windows', () => {
+ const capability = performanceCapability('fal');
+ const cues = [
+ { text: 'before', startSec: 30, endSec: 39.2 },
+ { text: 'the line', startSec: 39.2, endSec: 45.6 },
+ { text: 'after', startSec: 45.6, endSec: 50 },
+ ];
+
+ it('covers every lyric line the shot overlaps and keeps the shot as the edit', () => {
+ expect(planPerformanceWindow({ startSec: 40.5, endSec: 42.8, songDurationSec: 60, capability, lineCues: cues })).toMatchObject({
+ ok: true, windowStartSec: 39.2, windowEndSec: 45.6, windowSec: 6.4, editInSec: 1.3, editOutSec: 3.6,
+ lineStartSec: 39.2, lineEndSec: 45.6, lineTooLong: false,
+ });
+ });
+
+ it('pads a line shorter than the provider minimum around the line, and keeps the shot window for an over-long line', () => {
+ const short = [{ text: 'hey', startSec: 10, endSec: 12 }];
+ expect(planPerformanceWindow({ startSec: 10.5, endSec: 11, songDurationSec: 60, capability, lineCues: short }))
+ .toMatchObject({ ok: true, windowStartSec: 8.475, windowEndSec: 13.525, lineStartSec: 10, lineEndSec: 12, editInSec: 2.025 });
+ const long = [{ text: 'a very long line', startSec: 0, endSec: 40 }];
+ expect(planPerformanceWindow({ startSec: 10, endSec: 16, songDurationSec: 60, capability, lineCues: long }))
+ .toMatchObject({ ok: true, windowStartSec: 10, windowEndSec: 16, editInSec: 0, lineTooLong: true });
+ });
+
+ it('prices the whole-line take', () => {
+ const scene = { shotMode: 'performance', startSec: 40.5, endSec: 42.8 };
+ expect(falSceneTake({ scene, songDurationSec: 60, lineCues: cues })).toMatchObject({ seconds: 6.4, costUsd: 1.024 });
+ });
+});
+
// Each failure variant is a distinct admission regression: evidence, word
// gap, provider minimum, or timebase mismatch must never resubmit a prefix.
describe('music-driven performance takes', () => {
diff --git a/server/routes/videoGen.js b/server/routes/videoGen.js
index 760b113dc5..dda07b593a 100644
--- a/server/routes/videoGen.js
+++ b/server/routes/videoGen.js
@@ -495,6 +495,9 @@ const generateBodySchema = z.object({
// The shot card's Song audio toggle: hand a cutaway its song slice (true) or
// render it silent (false), whatever the project's generation mode says.
songAudio: z.boolean().optional(),
+ // The shot card's Whole sung line: a performance take's song slice covers
+ // the full lyric line the shot sits in; its edit points still frame the shot.
+ wholeLine: z.boolean().optional(),
}).optional(),
),
// File the clip into this music video's media collection even when no shot
diff --git a/server/services/musicVideo/performanceShot.js b/server/services/musicVideo/performanceShot.js
index d983a71e72..9c75c6deac 100644
--- a/server/services/musicVideo/performanceShot.js
+++ b/server/services/musicVideo/performanceShot.js
@@ -300,7 +300,10 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
if (scene.performanceRepair) await assertCurrentPerformanceTakes(project, masterPath);
const songDurationSec = await probeVideoDuration(masterPath);
if (songDurationSec == null) throw refuse('Could not read the song duration for this performance shot', 'MUSIC_VIDEO_AUDIO_UNREADABLE');
- const plan = planPerformanceWindow({ startSec: scene.startSec, endSec: scene.endSec, songDurationSec, capability });
+ // The shot card's "Whole sung line" asks for a take covering the full line
+ // the shot sits in; a continuation only ever re-renders its own interval.
+ const lineCues = musicVideo.wholeLine === true && !scene.performanceRepair ? (project.lyricCues || []) : null;
+ const plan = planPerformanceWindow({ startSec: scene.startSec, endSec: scene.endSec, songDurationSec, capability, lineCues });
if (!plan.ok) throw refuse(plan.message, plan.code);
if (scene.performanceRepair && (plan.windowStartSec !== scene.startSec || plan.windowEndSec !== scene.endSec)) {
throw refuse('Review needed: this provider cannot continue only the remaining audio interval', 'PERFORMANCE_REPAIR_REVIEW_NEEDED', 409);
@@ -374,7 +377,10 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
},
},
songInterval: { startSec: scene.startSec, endSec: scene.endSec },
- audioWindow: { startSec: plan.windowStartSec, endSec: plan.windowEndSec, durationSec: plan.windowSec },
+ audioWindow: {
+ startSec: plan.windowStartSec, endSec: plan.windowEndSec, durationSec: plan.windowSec,
+ ...(lineCues && !plan.lineTooLong ? { wholeLine: { startSec: plan.lineStartSec, endSec: plan.lineEndSec } } : {}),
+ },
edit: { inSec: plan.editInSec, outSec: plan.editOutSec, targetSec: plan.spanSec },
cues: clipRelativeCues(project.lyricCues, plan.windowStartSec, plan.windowEndSec),
referenceImageId: typeof sourceImagePath === 'string' ? basename(sourceImagePath) : null,
diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.js
index 2dbca45702..ea3d3d5801 100644
--- a/server/services/musicVideo/performanceShot.test.js
+++ b/server/services/musicVideo/performanceShot.test.js
@@ -369,6 +369,19 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
expect((await preparePerformanceShot({ ...args, resolution: '720p' })).resolution).toBe('768P');
});
+ // Catches a Whole sung line render slicing only the shot, or a continuation widening its accepted interval.
+ it('slices the whole sung line when the card asks for it, keeping the edit on the shot', async () => {
+ getProject.mockResolvedValue({ ...project(), lyricCues: [{ id: 'c1', text: 'the whole line', startSec: 16, endSec: 24 }] });
+ const args = { musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1', wholeLine: true }, backend: 'fal', sourceImagePath: '/x/frame.png', mode: 'image' };
+ const { shotInstruction: si } = await preparePerformanceShot(args);
+ expect(si.audioWindow).toEqual({ startSec: 16, endSec: 24, durationSec: 8, wholeLine: { startSec: 16, endSec: 24 } });
+ expect(si.edit).toMatchObject({ inSec: 4, outSec: 5.5, targetSec: 1.5 });
+ expect(si.songInterval).toEqual({ startSec: 20, endSec: 21.5 });
+ const plain = await preparePerformanceShot({ ...args, musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' } });
+ expect(plain.shotInstruction.audioWindow.wholeLine).toBeUndefined();
+ expect(plain.shotInstruction.audioWindow.durationSec).toBeCloseTo(5.05, 6);
+ });
+
it('asks for transcript guidance by default and skips it when the project turns it off', async () => {
const args = { musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: '/x/frame.png', mode: 'image' };
getProject.mockResolvedValue(project());