Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Add music-driven performance shots on fal audio-to-video models

Claude3f54cb3fdac7
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 7da2ed6742..1fd98d0adb 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -123,7 +123,7 @@ export default function SceneCard({
// #10297: a code shot is drawn by the composition, so it has no frame or clip to make.
const codeShot = layered && layer === 'code';
const performance = isPerformanceScene(scene);
- const capability = performance ? performanceCapability(lipSyncBackend) : null;
+ const capability = performance ? performanceCapability(lipSyncBackend, { scene, videoSettings: falVideoSettings }) : null;
const timedSpan = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' && scene.endSec > scene.startSec
? scene.endSec - scene.startSec
: null;
@@ -139,7 +139,7 @@ export default function SceneCard({
: null;
const falCost = falEstimate(falTake);
const performanceCost = falTake?.performance ? (falCost || capability?.costLabel) : capability?.costLabel;
- const splitLimit = layer === 'footage' ? shotSplitLimit(scene, lipSyncBackend) : null;
+ const splitLimit = layer === 'footage' ? shotSplitLimit(scene, lipSyncBackend, falVideoSettings) : null;
const canSplit = splitLimit != null && timedSpan != null && timedSpan > splitLimit + 1e-6;
const mergedSpan = typeof scene.startSec === 'number' && typeof nextEndSec === 'number' ? nextEndSec - scene.startSec : null;
const canMerge = !!onMergeNext && !isLast && !documentComposition
@@ -149,7 +149,7 @@ export default function SceneCard({
excerptStartSec: performanceReview?.excerptStartSec, backend: lipSyncBackend, videoSettings: falVideoSettings || {} });
const shotModeId = `mv-shot-mode-${scene.sceneId}`;
// #10152: what needs the director's attention, visible without opening the card.
- const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec, failed: failedScenes || {}, songReview });
+ const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec, failed: failedScenes || {}, songReview, videoSettings: falVideoSettings });
const songChange = songReview?.[scene.sceneId];
const generatingLane = generatingFrame && generatingVideo ? 'Frame + clip' : generatingFrame ? 'Frame' : generatingVideo ? 'Clip' : null;
// #10154: the server-persisted failure of the last render, shown until a
@@ -278,17 +278,20 @@ export default function SceneCard({
{layer === 'footage' && (
<>
<label htmlFor={shotModeId} className="flex items-center gap-1">Shot</label>
- <select id={shotModeId} value={performance ? 'performance' : 'cutaway'}
- onChange={(e) => applyPatch({ shotMode: e.target.value })}
+ <select id={shotModeId} value={performance ? (scene.performanceStyle === 'music' ? 'music' : 'performance') : 'cutaway'}
+ onChange={(e) => applyPatch(e.target.value === 'cutaway' ? { shotMode: 'cutaway' }
+ : { shotMode: 'performance', performanceStyle: e.target.value === 'music' ? 'music' : null })}
className="bg-port-bg border border-port-border rounded px-1 py-1 min-h-[44px] sm:min-h-0"
- title="Cutaway: any video lane animates the frame under the song. Performance: a singer lip-synced to the song itself (needs a source-audio provider).">
+ title="Cutaway: any video lane animates the frame under the song. Lip-sync: a singer's mouth follows the song. Music-driven: the song slice drives the whole frame's motion (a drummer on the beat). Both need a source-audio provider.">
<option value="cutaway">Cutaway</option>
<option value="performance">Performance (lip-sync)</option>
+ <option value="music">Performance (music-driven)</option>
</select>
</>
)}
</div>
- {performance && (
+ {/* A music-driven shot follows the whole mix, not one voice. */}
+ {performance && scene.performanceStyle !== 'music' && (
<div className="flex flex-wrap items-center gap-2 text-xs">
<label htmlFor={fieldId('speaker')}>Speaker / singer</label>
<input id={fieldId('speaker')} value={scene.performanceSpeaker || ''} maxLength={120}
@@ -371,7 +374,7 @@ export default function SceneCard({
)}
{performance && !performanceBlocked && (
<p className="text-[11px] text-port-text-muted break-words" data-testid="performance-plan">
- Lip-sync via {capability.label} ({capability.modelId}) · song {formatDurationSec(plan.windowStartSec)}–{formatDurationSec(plan.windowEndSec)}
+ {capability.style === 'music' ? 'Music-driven' : 'Lip-sync'} via {capability.label} ({capability.modelId}) · song {formatDurationSec(plan.windowStartSec)}–{formatDurationSec(plan.windowEndSec)}
{plan.editInSec > 0 ? ` · shot starts ${plan.editInSec.toFixed(2)}s into the take` : ''} · {performanceCost}
</p>
)}
@@ -517,7 +520,7 @@ export default function SceneCard({
className="flex items-center gap-1 bg-port-border hover:bg-port-border/70 disabled:opacity-50 rounded px-2 py-1.5 text-xs min-h-[44px] sm:min-h-0 whitespace-nowrap"
title={videoBlockedReason || performanceBlocked
|| (!scene.referenceImageId ? 'Generate a reference frame first'
- : performance ? `Lip-sync this scene's frame to the song via ${capability.label} — ${performanceCost}`
+ : performance ? `${capability.style === 'music' ? "Animate this scene's frame to the song" : "Lip-sync this scene's frame to the song"} via ${capability.label} — ${performanceCost}`
: "Generate this scene's video from its reference frame (i2v)")}>
{generatingVideo ? <Activity size={14} className="animate-spin" /> : <Video size={14} />}
{generatingVideo ? 'Generating video…' : (scene.videoHistoryId ? 'New video take' : 'Generate video')}
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index 9bd7e7de52..e9f710fdd5 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -48,6 +48,29 @@ describe('performance speaker save boundary', () => {
});
});
+describe('music-driven performance', () => {
+ it('switches a shot between cutaway, lip-sync and music-driven, and plans the music-driven take on the audio-to-video model', () => {
+ const onSave = vi.fn();
+ function Harness() {
+ const [scene, setScene] = useState({ sceneId: 's1', shotMode: 'cutaway', startSec: 10, endSec: 13.5, referenceImageId: 'frame.png', takes: [] });
+ return <SceneCard scene={scene} index={0} lipSyncBackend="fal" songDurationSec={60} falVideoSettings={{}}
+ onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
+ }
+ render(<Harness />);
+ const shot = screen.getByLabelText('Shot');
+ fireEvent.change(shot, { target: { value: 'music' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: 'music' });
+ expect(shot.value).toBe('music');
+ // The shot's own 3.5s, not lip-sync's 5s minimum window.
+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Music-driven via fal\.ai LTX-2\.3 audio-to-video \(fal-ai\/ltx-2\.3-22b\/audio-to-video\) · song 0:10–0:13 · est\. \$0\.30 \(3\.5s at 1080p\)/);
+ fireEvent.change(shot, { target: { value: 'performance' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: null });
+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Lip-sync via/);
+ fireEvent.change(shot, { target: { value: 'cutaway' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'cutaway' });
+ });
+});
+
describe('persisted render failure chip (#10154)', () => {
const failedScene = (lane) => ({ sceneId: 's1', startSec: 0, endSec: 5, referenceImageId: 'frame.png', takes: [],
lastFailure: { lane, error: 'CUDA out of memory', at: '2026-10-05T00:00:00.000Z' } });
diff --git a/client/src/components/musicVideo/VideoRenderSettings.jsx b/client/src/components/musicVideo/VideoRenderSettings.jsx
index 7d4f049b00..e15dd27580 100644
--- a/client/src/components/musicVideo/VideoRenderSettings.jsx
+++ b/client/src/components/musicVideo/VideoRenderSettings.jsx
@@ -1,9 +1,9 @@
import RenderPoolPicker from './RenderPoolPicker.jsx';
import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
import {
- FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falVideoResolutions, getFalVideoModel,
+ FAL_AUDIO_VIDEO_MODEL, FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falVideoResolutions, getFalVideoModel,
} from '../../lib/falVideoModels.js';
-import { SOURCE_AUDIO_LIPSYNC } from '../../lib/musicVideoShotTiming.js';
+import { SOURCE_AUDIO_LIPSYNC, SOURCE_AUDIO_MUSIC, performanceCapability } from '../../lib/musicVideoShotTiming.js';
const falSelectCls = 'w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 sm:w-auto min-h-[44px] sm:min-h-0';
const falOptionLabel = (model) => {
@@ -35,6 +35,10 @@ export default function VideoRenderSettings({ videoSettings, generating }) {
const falModel = getFalVideoModel(settings.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL);
const falResolutions = falVideoResolutions(falModel);
const lipSync = SOURCE_AUDIO_LIPSYNC.fal;
+ // Music-driven performance takes (scene Shot › Performance (music-driven)):
+ // an audio-to-video model where the song slice drives the whole frame.
+ const music = performanceCapability('fal', { scene: { performanceStyle: 'music' }, videoSettings: settings });
+ const musicModels = Object.values(SOURCE_AUDIO_MUSIC.fal);
return (
<>
<RenderPoolPicker settings={settings} change={change} disabled={locked} />
@@ -240,6 +244,35 @@ export default function VideoRenderSettings({ videoSettings, generating }) {
/>
Transcript-guided lip-sync
</label>
+ <label htmlFor="mv-fal-music-model" className="sr-only">fal.ai music-driven model</label>
+ <select
+ id="mv-fal-music-model"
+ value={settings.falMusicModelId || ''}
+ onChange={(e) => change({ falMusicModelId: e.target.value || null, falMusicResolution: null })}
+ disabled={locked}
+ title="Audio-to-video model for music-driven performance shots: the scene's song slice drives the whole frame's motion, not just a mouth"
+ className={`${falSelectCls} sm:max-w-[280px]`}
+ >
+ {musicModels.map((lane) => (
+ <option key={lane.modelId} value={lane.modelId === FAL_AUDIO_VIDEO_MODEL ? '' : lane.modelId}>
+ {`Music-driven · ${getFalVideoModel(lane.modelId).label} · ${describeFalVideoRate(lane.modelId, lane.defaultResolution)}${lane.modelId === FAL_AUDIO_VIDEO_MODEL ? ' (default)' : ''}`}
+ </option>
+ ))}
+ </select>
+ <label htmlFor="mv-fal-music-resolution" className="sr-only">fal.ai music-driven resolution</label>
+ <select
+ id="mv-fal-music-resolution"
+ value={music.resolutions.includes(settings.falMusicResolution) ? settings.falMusicResolution : ''}
+ onChange={(e) => change({ falMusicResolution: e.target.value || null })}
+ disabled={locked}
+ title={`Output resolution for music-driven takes on ${music.label}`}
+ className={falSelectCls}
+ >
+ <option value="">{`Music-driven ${music.defaultResolution} (default)`}</option>
+ {music.resolutions.map((res) => (
+ <option key={res} value={res}>{`Music-driven ${res} · ${describeFalVideoRate(music.modelId, res)}`}</option>
+ ))}
+ </select>
</>
)}
</>
diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsx
index c3298ce830..74dd4b5ae9 100644
--- a/client/src/components/musicVideo/stages/BoardStage.jsx
+++ b/client/src/components/musicVideo/stages/BoardStage.jsx
@@ -41,6 +41,7 @@ export default function BoardStage({ board }) {
layered, lipSyncBackend, songDurationSec, failed: sceneMedia.failedScenes,
footageOptional: FOOTAGE_OPTIONAL_MODES.has(project.composition?.mode || 'concat'),
songReview: project.songRevision?.sceneReview || null,
+ videoSettings: videoSettings.settings,
};
const codesById = new Map(scenes.map((scene) => [scene.sceneId, sceneAttention(scene, attentionCtx)]));
const counts = { all: scenes.length, attention: 0, missing: 0 };
diff --git a/client/src/components/musicVideo/stages/ProduceStage.jsx b/client/src/components/musicVideo/stages/ProduceStage.jsx
index 9c4f58d957..94be033bdf 100644
--- a/client/src/components/musicVideo/stages/ProduceStage.jsx
+++ b/client/src/components/musicVideo/stages/ProduceStage.jsx
@@ -30,6 +30,7 @@ export default function ProduceStage({ board }) {
songDurationSec: project.audioAnalysis?.durationSec ?? null,
failed: sceneMedia.failedScenes,
songReview: project.songRevision?.sceneReview || null,
+ videoSettings: videoSettings.settings,
}} />
)}
<div className="flex justify-end"><ContactSheetButton onOpen={board.openContactSheet} /></div>
diff --git a/client/src/hooks/useMusicVideoModelSettings.js b/client/src/hooks/useMusicVideoModelSettings.js
index 00d82a8041..f56de184fb 100644
--- a/client/src/hooks/useMusicVideoModelSettings.js
+++ b/client/src/hooks/useMusicVideoModelSettings.js
@@ -74,6 +74,12 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =
falModelId: project?.videoSettings?.falModelId ?? null,
falResolution: project?.videoSettings?.falResolution ?? null,
falLipSyncResolution: project?.videoSettings?.falLipSyncResolution ?? null,
+ // false = lip-sync without transcript guidance (null/absent = on).
+ falLipSyncTranscription: project?.videoSettings?.falLipSyncTranscription ?? null,
+ // Audio-to-video model / resolution for music-driven performance takes —
+ // null means FAL_AUDIO_VIDEO_MODEL (LTX-2.3) / 1080p, resolved by falSceneTake.
+ falMusicModelId: project?.videoSettings?.falMusicModelId ?? null,
+ falMusicResolution: project?.videoSettings?.falMusicResolution ?? null,
generationMode: project?.videoSettings?.generationMode || 'image',
audioReactiveLora: project?.videoSettings?.audioReactiveLora || '',
audioReactiveScale: project?.videoSettings?.audioReactiveScale ?? 1.2,
diff --git a/client/src/lib/README.md b/client/src/lib/README.md
index 9b689f9b4e..866f9a6dca 100644
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@ -66,7 +66,7 @@ grep -i "what you want to do" client/src/lib/README.md
| `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoSceneAttention.js` | Music Video per-scene attention model (#10152): `sceneAttention(scene, ctx)` returns the codes (`missing-frame`, `missing-clip`, `under-covered`, `contract-problem`, `perf-blocked`, `candidates-pending`, `plate-unverified`, `no-span`, `last-failure`) behind the collapsed scene-row chips, the Board `?scenes=attention\|missing\|all` filter (`parseSceneFilter`, `sceneMatchesFilter`) and the Produce per-shot strip; `SCENE_ATTENTION_LABELS`. Used by `components/musicVideo/SceneCard.jsx`, `stages/BoardStage.jsx` and `stages/ProduceStage.jsx`. |
| `musicVideoCoverOverlay.js` | Cover-art lettering preview (#10345): re-export of `server/lib/musicVideoCoverOverlay.js` (the design vocabulary, `normalizeCoverDesign`, `coverOverlaySvg`) plus `canvasCoverWidths(design, texts, fonts)`, which measures text with a canvas so the live preview wraps and sizes the title the way the server's measured render does. |
-| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes, the verified source-audio lip-sync capability table, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. |
+| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes and performance styles, the verified source-audio lip-sync and music-driven capability tables, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. |
| `musicVideoStages.js` | Music Video page stage model: `MUSIC_VIDEO_STAGES` (six steps: Song `setup` → Look `cast-sets` → Storyboard `board` → Make `produce` → Final render `review` → Publish; a retired `compose` link resolves to Make), `resolveStageParam(value)` for the `:stage` route param, `deriveStages(project)` (per-stage `done`/`active`/`blocked`/`todo` + `current`, the first incomplete step; a live production run pins Make), `deriveNextAction(project, ctx)` (the one primary action the sticky header offers — approve Cast & Sets, verify lyric timing, stop/resume production, render final, …), `describeProjectStatus` (the header line and the step that needs the director), `autopilotStatus` (the header Autopilot button), `projectSpend(project)` (paid generation used vs. cap), `projectShotSummary(project)` (document shots or scene counts), `stageChecklist(stageId, project, readiness)` (what one step still needs, with the open approval's reason and a scroll anchor; an `optional` item, such as Make's animated proof, is listed but not counted), `summarizeStoryboardProblems(problems)` (folds the server's one-sentence-per-shot storyboard problems into one count line per kind), `stepNotes(project, readiness, publish)` (one fact per step for the step list), `listPreviewSources` (final, live composition, drafts, then the storyboard animatic), `projectServicesSummary(project)` (the saved image/video render services), `compareMusicVideoProjectsNewestFirst(a, b)` (sort projects newest created / updated first for project cards and pickers), and `currentProductionRun`; `productionReviewStopGuidance` separates current approval blockers from historical run stops. Used by `components/musicVideo/MusicVideoLayout.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoAttention.js` | Music Video "Needs attention" model (#9940): `deriveAttentionItems(project, live)` returns the banner rows for server-held states a client-orchestrated workflow can strand — an open/rendering revision, an interrupted Cast & Sets stage, a running auto-review waiting on its revised sections, a final render nobody is watching — derived from the persisted record so they survive a reload (healthy work this tab can see progressing is not flagged); `openRevisionOf(project)`; `ATTENTION_ANCHOR_ID` + `revealAttention()` scroll a refusal toast to the banner. Used by `components/musicVideo/NeedsAttentionBanner.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoProjectList.js` | Music Video project index model (#10167): `projectRunPill(project)` (Running / Needs you / Interrupted / Done, from the same next-action and attention logic as the header), `groupMusicVideoProjects(projects, query)` (collapse forks under their root, newest version first, name filter, groups ordered by last touch), and `compareMusicVideoProjectsRecentlyTouched`. Used by `pages/MusicVideo.jsx` and `components/musicVideo/MusicVideoProjectCard.jsx`. |
@@ -162,7 +162,7 @@ grep -i "what you want to do" client/src/lib/README.md
| `quotaBurnPatch.js` | `mergeQuotaBurnPatch(base, patch)` — mirrors the server's quota-burn config merge (top-level + per-family keys merge, a family's `jobs` array replaces) so the Quota Burn page can apply an edit optimistically and accumulate debounced edits into one PUT body. `quotaBurnJobIsSpent(job, ranAt)` — whether a `run once` step has had its one dispatch, gated on the optimistic config's own `runOnce` so a just-ticked checkbox reads as spent before the save round-trips (the client mirror of the server's `jobIsSpent`). `UNLIMITED_DISPATCHES` / `isUnlimitedDispatchCap(cap)` / `dispatchCapInput(value)` — the -1 "no dispatch cap" sentinel (the default), mirrored from the server. |
| `quotaBurnTasks.js` | `MAINTENANCE_TASK_ORDER` / `MAINTENANCE_ORDER_GUIDANCE` document the default ladder; `maintenanceSequence` populates audit and claim-drain references; `maintenancePrerequisites` lists missing saved settings and scoped setup patches for the selected mode and claim policy. The Quota Burn page's view of the SHARED scheduled-task catalog. `buildQuotaBurnTaskCatalog({ schedule, jobs, apps })` groups PortOS built-in task types and app custom jobs into the picker's two groups (dropping what a burn may never invoke, and stamping a `blockedReason` on what is merely disabled or untargetable); `searchTaskCatalog` / `flattenTaskCatalog` / `findTaskEntry` query it. `taskRefKey` / `stepFromTaskEntry` / `taskEntryNeedsApp` build and address a step's `taskRef`. `effectiveQuotaBurnSettings(step, saved)` mirrors the server's `effectiveSettings` (overrides layered over the task's saved settings, params merged key-by-key). `quotaBurnStepPayload(job)` reduces a step to what the strict PUT schema accepts, dropping the top-level override mirrors that would otherwise outrank the `overrides` bag. `taskSourceHref(entry)` / `CREATE_TASK_HREF` link out to the source task. |
| `clipboard.js` | `copyToClipboard`, `writeClipboardSilently`, `readClipboard` — safe across insecure-origin contexts. Use these instead of `navigator.clipboard.writeText` inline. |
-| `falVideoModels.js` | Re-export of `server/lib/falVideoModels.js` — the curated fal.ai video model catalog, per-model request rules and `estimateFalVideoCostUsd()`. Used by the Music Video fal model picker (`components/musicVideo/VideoRenderSettings.jsx`) and the Video Gen page's fal model suggestions. |
+| `falVideoModels.js` | Re-export of `server/lib/falVideoModels.js` — the curated fal.ai video model catalog (including the `FAL_AUDIO_VIDEO_MODELS` audio-to-video lanes), per-model request rules and `estimateFalVideoCostUsd()`. Used by the Music Video fal model picker (`components/musicVideo/VideoRenderSettings.jsx`) and the Video Gen page's fal model suggestions. |
| `falVideoHandoff.js` | `buildFalH3MaxPrompt(prompt, negativePrompt)` composes the provider-neutral shot prompt fal's free MiniMax H3 Max tool is driven with; `FAL_H3_MAX_FREE_URL` and `FAL_H3_MAX_FREE_ALLOWANCE_NOTE` carry the tool URL and current vendor allowance copy. The free web allowance is deliberately a handoff, not the separately metered fal API. |
| `compareHelpers.js` | `equalByKeys(a, b, keys)` / `equalListByKeys(a, b, keys)` — typed key-based equality for `useAutoRefetch`'s `compare`. Keys are property names, dotted paths (`'context.running'`), or `(item) => value` accessors. The typed alternative to `sameJsonShape` when a monotonic timestamp or unrendered field would break stringify-equality dedup. |
| `consoleFilters.js` | `installConsoleFilters()` — drops a small allow-list of known-noise console strings (THREE.js `Clock` deprecation, expected WebGL `Context Lost.`) from `console.{warn,log,debug}`. Idempotent; auto-installed on import. Imported for its side effect from `main.jsx`. |
diff --git a/client/src/lib/falVideoModels.js b/client/src/lib/falVideoModels.js
index da800925e4..7bc03d3137 100644
--- a/client/src/lib/falVideoModels.js
+++ b/client/src/lib/falVideoModels.js
@@ -6,6 +6,8 @@
* board's cost estimate and the server's request body cannot drift.
*/
export {
+ FAL_AUDIO_VIDEO_MODEL,
+ FAL_AUDIO_VIDEO_MODELS,
FAL_DEFAULT_IMAGE_VIDEO_MODEL,
FAL_DEFAULT_TEXT_VIDEO_MODEL,
FAL_IMAGE_VIDEO_MODELS,
diff --git a/client/src/lib/musicVideoSceneAttention.js b/client/src/lib/musicVideoSceneAttention.js
index 9fcd5eb54b..ee0c22a9bf 100644
--- a/client/src/lib/musicVideoSceneAttention.js
+++ b/client/src/lib/musicVideoSceneAttention.js
@@ -41,8 +41,8 @@ export const SCENE_FILTERS = Object.freeze(['all', 'attention', 'missing']);
/** The `?scenes=` value as a filter id; anything unknown is `all`. */
export const parseSceneFilter = (value) => (SCENE_FILTERS.includes(value) ? value : 'all');
-function performanceBlocked(scene, { lipSyncBackend, songDurationSec }) {
- const capability = performanceCapability(lipSyncBackend);
+function performanceBlocked(scene, { lipSyncBackend, songDurationSec, videoSettings }) {
+ const capability = performanceCapability(lipSyncBackend, { scene, videoSettings });
if (!capability) return performanceBlockedReason(lipSyncBackend);
const plan = planPerformanceWindow({ startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability });
return plan.ok ? null : plan.message;
@@ -59,7 +59,7 @@ function performanceBlocked(scene, { lipSyncBackend, songDurationSec }) {
*/
export function sceneAttention(scene, ctx = {}) {
if (!scene) return [];
- const { layered = false, footageOptional = false, lipSyncBackend = '', songDurationSec = null, clipSec = null, failed = {}, songReview = null } = ctx;
+ const { layered = false, footageOptional = false, lipSyncBackend = '', songDurationSec = null, clipSec = null, failed = {}, songReview = null, videoSettings = null } = ctx;
const layer = sceneVisualLayer(scene, { layered });
const codes = [];
const songChange = songReview?.[scene.sceneId];
@@ -72,7 +72,7 @@ export function sceneAttention(scene, ctx = {}) {
&& scene.endSec - scene.startSec - clipSec > COVERAGE_TOLERANCE_SEC) codes.push('under-covered');
const contract = scene.direction?.actionContract;
if (contract && shotActionContractProblem(contract, scene)) codes.push('contract-problem');
- if (layer === 'footage' && isPerformanceScene(scene) && performanceBlocked(scene, { lipSyncBackend, songDurationSec })) codes.push('perf-blocked');
+ if (layer === 'footage' && isPerformanceScene(scene) && performanceBlocked(scene, { lipSyncBackend, songDurationSec, videoSettings })) codes.push('perf-blocked');
if (sceneHasPendingDecision(scene)) codes.push('candidates-pending');
if (contract && scene.referenceImageId) {
const selected = sceneTakeList(scene, 'image').find((take) => take.assetId === scene.referenceImageId);
diff --git a/client/src/lib/musicVideoShotTiming.js b/client/src/lib/musicVideoShotTiming.js
index d68934c1dd..4a9fda7ed2 100644
--- a/client/src/lib/musicVideoShotTiming.js
+++ b/client/src/lib/musicVideoShotTiming.js
@@ -9,6 +9,7 @@
export {
MUSIC_VIDEO_SHOT_MODES,
SOURCE_AUDIO_LIPSYNC,
+ SOURCE_AUDIO_MUSIC,
approximateMotionCues,
falSceneTake,
falTakeRequestFields,
diff --git a/server/lib/README.md b/server/lib/README.md
index 201088aaea..4fc3d91db9 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -321,11 +321,11 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `cursor.js` | Cursor Agent (`cursor-agent`) provider helpers — the `CURSOR_COMMAND` binary constant, the `isCursorCommand` predicate, and the `ensureCursorHeadlessArgs` (`--print --force`, folding any pinned reasoning effort into `--model`) / `ensureCursorTuiArgs` (`--force`) argv builders. `--force` is load-bearing beyond approvals: it also clears cursor's workspace-trust gate, which otherwise EXITS a headless run before any work happens. The prompt rides raw stdin (like claude/codex, unlike grok/kimi), and cursor needs no configured-default sentinel — its `auto` router is a real model id passed straight to `--model`. |
| `grok.js` | xAI Grok Build (`grok`) provider helpers — id/endpoint constants (`GROK_API_ID`/`GROK_CLI_ID`/`GROK_TUI_ID`/`GROK_API_ENDPOINT`), `isGrokCommand`/`isGrokCliProvider`/`isGrokTuiProvider` predicates, `ensureGrokHeadlessArgs`/`ensureGrokTuiArgs` argv builders (grok reads its prompt from `--prompt-file /dev/stdin`, not raw stdin; model selection uses the `GROK_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Antigravity), and `prepareGrokPromptFile` (Windows temp-file delivery fallback; `cleanup({ throwOnError: true })` reports cleanup failure to an owning lifecycle). |
| `filmStyleGrammars.js` | Curated film style grammar catalog (`FILM_STYLE_GRAMMARS`, eight code-rendered media with rendering, colour, type, motion, camera, sound and native-move guidance), `getFilmStyleGrammar`, `summarizeFilmStyleGrammar` (picker projection) and `knownFilmStyleIdSchema` (request-field id check) and `renderFilmStyleGrammarPrompt(id, { parts })` — one prompt section capped at `FILM_STYLE_PROMPT_MAX_CHARS`; unknown id or bad parts is a 400. |
-| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. |
+| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, LTX-2.3 and OmniHuman v1.5 audio-to-video (`FAL_AUDIO_VIDEO_MODELS`, kind `a2v`: the supplied audio drives the whole frame's motion), Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. |
| `grokVideoClip.js` | Clip lengths grok's `image_to_video` actually delivers — `GROK_VIDEO_DURATIONS` (`[6, 10]`, measured in #3022: a 2s/3s request returns the same 6.04s clip), `GROK_VIDEO_DEFAULT_DURATION`, `resolveGrokDuration()` (validate an already-grok-shaped request, else default), and `nearestGrokDuration()` (round a length authored against another backend's continuous contract UP to the shortest clip that covers it — for translating a CD/commission `targetDurationSeconds`). Dependency-free so `validation.js` / `routes/videoGen.js` can derive their schemas from the same list the services gate on. |
| `fontMetadata.js` | `fontContainer(buf)` (sfnt / woff2 / null) and `fontFamilyName(buf)` — a font file's family name read from its `name` table (TrueType, OpenType, and WOFF2's brotli stream) with no font library; null for anything unreadable (#10345). |
| `musicVideoCoverOverlay.js` | Music Video cover lettering (#10345): `COVER_DESIGN_OPTIONS` / `DEFAULT_COVER_DESIGN`, `normalizeCoverDesign(design, { fonts })` (accepts every stored design; a `font:<id>` typeface only when that uploaded font is listed), `coverOverlaySvg({ title, tag, design, widths, fonts })` and the `coverFontSpec` / `fontAttrs` / `titleLinesFor` / `tagTextFor` helpers the width probe reads. Pure; the server render (`coverArtCompose.js`) and the client's live Lettering preview run this one layout. |
-| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend)`, `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. |
+| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `MUSIC_VIDEO_PERFORMANCE_STYLES` (`lipsync`/`music`) + `isLipSyncScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `SOURCE_AUDIO_MUSIC` (music-driven lanes — fal.ai audio-to-video models, picked by `videoSettings.falMusicModelId`, default LTX-2.3 at 1080p), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend, { scene, videoSettings })` (lip-sync lane, or the music-driven one for a music-driven scene), `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. |
| `reactorStartingFrame.js` | Fits a reactor.inc `starting_frame` to the fast-h3 session canvas before it is uploaded. `readImageSize(path)` → `{width,height}` or `null` (never a guessed dimension), and `prepareReactorStartingFrame(sourceImagePath, requestedAspect, outputPath)` → `{ aspect, canvas, framePath, fittedPath }`: an absent `requestedAspect` DERIVES the canvas from the frame's own shape (`nearestReactorAspect`), then centre cover-crops the frame to that canvas's exact pixel size so reactor's own "fitted to the session canvas" step is a no-op. `fittedPath` is the temp file the caller deletes; `null` means the original was uploaded unchanged. A frame sharp cannot decode (or a resize that fails) passes through untouched on the default canvas rather than failing a render PortOS merely could not measure — reactor decodes more containers than sharp does. |
| `reactorVideoClip.js` | The reactor.inc `fast-h3` clip contract, dependency-free so the route schema, the FableLoom prompt compiler and the VideoGen picker all read one copy — `REACTOR_MAX_PROMPT_LENGTH` (800; the API rejects a longer prompt rather than truncating it), `REACTOR_MIN_CLIP_SECONDS` / `REACTOR_MAX_CLIP_SECONDS` (5.167–14.375, i.e. 124–345 frames at the `REACTOR_CLIP_FPS` 24 it delivers), `REACTOR_MAX_CLIP_ID_LENGTH`, and the picker list `REACTOR_CLIP_LENGTHS` + `REACTOR_DEFAULT_CLIP_LENGTH` + `reactorClipLengthLabel()` (the two exact endpoints plus every whole second between them, since the accepted range is continuous but starts and ends off a round number). Also the CANVAS contract `set_canvas` accepts: `REACTOR_CANVASES` (`16:9` 1344×768, `4:3` 1024×768, `1:1` 768×768, `9:16` 768×1344 — every one a 768px short edge, so the aspect string is the whole choice), `REACTOR_ASPECTS`, `REACTOR_DEFAULT_ASPECT`, `reactorCanvas(aspect)` and `nearestReactorAspect(width, height)` (closest canvas in log-ratio space; anything unmeasurable answers the default rather than deriving one). Mirrored to `client/src/lib/reactorVideoClip.js`, parity pinned by `reactorVideoClip.test.js`. |
| `kimi.js` | Moonshot AI Kimi Code (`kimi`) provider helpers — id constants (`KIMI_CLI_ID`/`KIMI_TUI_ID`), `isKimiCommand`/`isKimiCliProvider`/`isKimiTuiProvider` predicates, `ensureKimiHeadlessArgs` (model flag only — non-interactive mode is implicit in `--prompt`, and kimi has no `--print`/`--afk` and refuses approval flags alongside `--prompt`) / `ensureKimiTuiArgs` (`--yolo`) argv builders (model selection uses the `KIMI_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Grok/Antigravity), and `prepareKimiPrompt` (delivers the prompt as the `--prompt <value>` argv, not stdin). |
diff --git a/server/lib/falVideoModels.js b/server/lib/falVideoModels.js
index aa13fa6e30..0307ac7c68 100644
--- a/server/lib/falVideoModels.js
+++ b/server/lib/falVideoModels.js
@@ -26,14 +26,20 @@
*/
/** Date the schemas and prices below were last verified against fal.ai. */
-export const FAL_VIDEO_PRICING_VERIFIED_AT = '2026-09-29';
+export const FAL_VIDEO_PRICING_VERIFIED_AT = '2026-10-11';
-/** What an entry renders from: text, a start frame, reference assets, or a frame + source audio. */
-export const FAL_VIDEO_KINDS = Object.freeze(['t2v', 'i2v', 'ref2v', 'lipsync']);
+/**
+ * What an entry renders from: text, a start frame, reference assets, a frame +
+ * source audio it lip-syncs to (`lipsync`), or a frame + source audio that
+ * drives the whole picture's motion (`a2v` — a drummer hitting on the beat, a
+ * dancer, a crowd), steered by a prompt.
+ */
+export const FAL_VIDEO_KINDS = Object.freeze(['t2v', 'i2v', 'ref2v', 'lipsync', 'a2v']);
export const FAL_DEFAULT_TEXT_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/text-to-video';
export const FAL_DEFAULT_IMAGE_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/image-to-video';
export const FAL_LIPSYNC_VIDEO_MODEL = 'minimax/h3-max/lip-sync/image-to-video';
+export const FAL_AUDIO_VIDEO_MODEL = 'fal-ai/ltx-2.3-22b/audio-to-video';
const H3_RESOLUTIONS = Object.freeze(['480P', '768P', '1080P']);
const SEEDANCE_RESOLUTIONS = Object.freeze(['480p', '720p', '1080p']);
@@ -46,6 +52,9 @@ const H3_MAX_RATES = Object.freeze({ '480P': 0.05, '768P': 0.08, '1080P': 0.16 }
// per-second price for a 16:9 output; those published figures are used (they
// run ~2% above the nominal-pixel formula, i.e. the provider pads the canvas).
const SEEDANCE_PER_SEC_16X9 = Object.freeze({ '480p': 0.2205, '720p': 0.473, '1080p': 1.164 });
+// LTX-2.3 takes an explicit {width, height} canvas; PortOS names it by its
+// short side and keeps the requested aspect (see `ltxCanvas`).
+const LTX_SHORT_SIDES = Object.freeze({ '480p': 480, '720p': 720, '1080p': 1080 });
const seedance = (id, kind, label, extra) => ({
id, kind, label, family: 'seedance-2.5',
@@ -145,6 +154,41 @@ export const FAL_VIDEO_MODELS = Object.freeze([
seedance('bytedance/seedance-2.5/text-to-video', 't2v', 'Seedance 2.5 (text)', {
aspectRatio: { param: 'aspect_ratio', options: WIDE_ASPECTS, auto: 'auto' },
}),
+ {
+ // LTX-2.3 22B is a joint audio-video model: the supplied audio conditions
+ // every frame, so motion lands on the music (and a singer's mouth on the
+ // vocal) rather than being only a mouth layered over a still pose.
+ id: FAL_AUDIO_VIDEO_MODEL, kind: 'a2v', label: 'LTX-2.3 audio-to-video', family: 'ltx-2.3',
+ prompt: 'required',
+ image: { param: 'image_url', required: false },
+ endImage: { param: 'end_image_url' },
+ // num_frames is 9–481 at 24fps, i.e. up to ~20s; with match_audio_length
+ // the frame count follows the audio.
+ audioInput: { param: 'audio_url', minSec: 1, maxSec: 20 },
+ duration: { kind: 'audio' },
+ resolution: { param: 'video_size', options: Object.keys(LTX_SHORT_SIDES), default: '720p', wire: 'canvas', fps: 24 },
+ aspectRatio: null,
+ negativePrompt: 'native',
+ seed: 'seed',
+ // audio_strength 1 keeps the supplied audio unmodified in the output.
+ fixed: { match_audio_length: true, fps: 24, audio_strength: 1 },
+ // "$0.001605 per megapixel of generated video data (width × height × frames)".
+ pricing: { perMegapixel: 0.001605 },
+ },
+ {
+ // Audio-driven human figure: expression, gesture and body movement follow
+ // the audio, not just the mouth. Expects a person in the frame.
+ id: 'fal-ai/bytedance/omnihuman/v1.5', kind: 'a2v', label: 'OmniHuman v1.5 (audio-driven person)', family: 'omnihuman',
+ prompt: 'optional',
+ image: { param: 'image_url', required: true },
+ // "Audio must be under 30s long for 1080p generation and under 60s long for 720p".
+ audioInput: { param: 'audio_url', minSec: 1, maxSec: 30 },
+ duration: { kind: 'audio' },
+ resolution: { param: 'resolution', options: ['720p', '1080p'], default: '1080p' },
+ aspectRatio: null,
+ negativePrompt: 'fold',
+ pricing: { perSecond: 0.16 },
+ },
seedance('bytedance/seedance-2.5/reference-to-video', 'ref2v', 'Seedance 2.5 (reference)', {
// Referenced in the prompt as @Image1 / @Audio1. Audio references let a
// singer follow supplied audio, but each clip is capped at 30.2s.
@@ -196,6 +240,9 @@ export const getFalVideoModel = (modelId) => (typeof modelId === 'string' ? BY_I
*/
export const FAL_IMAGE_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'i2v'));
+/** Models that animate a frame to supplied audio across the whole picture (the music-driven performance lane). */
+export const FAL_AUDIO_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'a2v'));
+
/** Resolution choices a model offers (empty when it has no resolution parameter). */
export const falVideoResolutions = (model) => (model?.resolution?.options ? [...model.resolution.options] : []);
@@ -215,7 +262,7 @@ export function resolveFalVideoResolution(model, requested) {
* shortest supported length that still COVERS it, else the longest — a Music
* Video shot must not be under-covered (the renderer refuses a clip more than a
* frame short) and nothing loops or stretches footage. An absent / invalid
- * request gets the model's default. Null for a lip-sync model, whose length
+ * request gets the model's default. Null for a source-audio model, whose length
* follows its audio.
*/
export function coerceFalVideoSeconds(model, seconds) {
@@ -235,6 +282,24 @@ const wireDuration = (rule, sec) => (
const ratioOf = (r) => { const [w, h] = String(r).split(':').map(Number); return w / h; };
+const round32 = (n) => Math.max(32, Math.round(n / 32) * 32);
+
+/**
+ * The explicit canvas a `wire: 'canvas'` model renders at `resolution`: the
+ * named short side, the long side from the requested width/height (16:9 when
+ * none is given), both on the 32-pixel grid the model works in.
+ */
+function ltxCanvas(resolution, { width, height } = {}) {
+ const short = LTX_SHORT_SIDES[resolution];
+ if (!short) return null;
+ const w = Number(width);
+ const h = Number(height);
+ const portrait = w > 0 && h > 0 && h > w;
+ const aspect = w > 0 && h > 0 ? Math.max(w, h) / Math.min(w, h) : 16 / 9;
+ const long = round32(short * aspect);
+ return portrait ? { width: short, height: long } : { width: long, height: short };
+}
+
function pickAspectRatio(rule, { aspectRatio, width, height }) {
if (!rule) return null;
if (rule.options.includes(aspectRatio)) return aspectRatio;
@@ -280,6 +345,15 @@ export function estimateFalVideoCostUsd({ modelId, seconds, resolution, width, h
}
const res = resolveFalVideoResolution(model, resolution);
let perSec;
+ if (model.pricing.perMegapixel) {
+ const canvas = ltxCanvas(res, { width, height });
+ if (!canvas) return null;
+ const fps = model.resolution.fps;
+ // The frames the audio needs, rounded UP to the 8n+1 counts LTX renders,
+ // so the estimate never undershoots the bill.
+ const frames = Math.ceil(Math.ceil(billedSec * fps) / 8) * 8 + 1;
+ return round4((canvas.width * canvas.height * frames / 1e6) * model.pricing.perMegapixel);
+ }
if (model.pricing.videoTokensPerSec16x9) {
perSec = model.pricing.videoTokensPerSec16x9[res] ?? null;
const w = Number(width);
@@ -299,6 +373,12 @@ export function describeFalVideoRate(modelId, resolution) {
const model = getFalVideoModel(modelId);
if (!model) return null;
const res = resolveFalVideoResolution(model, resolution);
+ if (model.pricing.perMegapixel) {
+ const canvas = ltxCanvas(res);
+ if (!canvas) return null;
+ const perSec = (canvas.width * canvas.height * model.resolution.fps / 1e6) * model.pricing.perMegapixel;
+ return `≈$${perSec.toFixed(4)}/s at ${res} (16:9)`;
+ }
if (model.pricing.videoTokensPerSec16x9) {
const perSec = model.pricing.videoTokensPerSec16x9[res];
return perSec == null ? null : `≈$${perSec.toFixed(4)}/s at ${res} (16:9)`;
@@ -313,7 +393,7 @@ export function describeFalVideoRate(modelId, resolution) {
* Build the fal queue request body for a curated model, or return null for an
* uncurated id (the caller keeps its legacy body). Returns `{ body, model,
* seconds, resolution, generateAudio, estimatedCostUsd }` — `seconds` is the
- * coerced clip length (null for lip-sync, whose length is `audioSec`).
+ * coerced clip length (null for a source-audio model, whose length is `audioSec`).
*
* Inputs a model cannot take are dropped rather than sent (fal ignores unknown
* keys, which would bill a render that silently disregarded them); a missing
@@ -367,7 +447,7 @@ export function buildFalVideoRequest({
body[model.referenceAudio.param] = audioRefs;
}
if (model.audioInput) {
- if (!audioUrl) throw refuse(`${model.label} needs the source audio to synchronize to`);
+ if (!audioUrl) throw refuse(`${model.label} needs the source audio to ${model.kind === 'a2v' ? 'drive its motion' : 'synchronize to'}`);
body[model.audioInput.param] = audioUrl;
if (model.transcription && enableTranscription) body[model.transcription] = true;
} else if (audioUrl) {
@@ -377,7 +457,7 @@ export function buildFalVideoRequest({
const coercedSec = coerceFalVideoSeconds(model, seconds);
if (coercedSec != null) body.duration = wireDuration(model.duration, coercedSec);
const res = resolveFalVideoResolution(model, resolution);
- if (res) body[model.resolution.param] = res;
+ if (res) body[model.resolution.param] = model.resolution.wire === 'canvas' ? ltxCanvas(res, { width, height }) : res;
const aspect = pickAspectRatio(model.aspectRatio, { aspectRatio, width, height });
if (aspect) body[model.aspectRatio.param] = aspect;
const wantsAudio = generateAudio === true && !!model.audio;
diff --git a/server/lib/falVideoModels.test.js b/server/lib/falVideoModels.test.js
index 69016a21a6..b9ef23cf49 100644
--- a/server/lib/falVideoModels.test.js
+++ b/server/lib/falVideoModels.test.js
@@ -1,5 +1,6 @@
import { describe, it, expect } from 'vitest';
import {
+ FAL_AUDIO_VIDEO_MODEL,
FAL_LIPSYNC_VIDEO_MODEL,
FAL_VIDEO_MODELS,
buildFalVideoRequest,
@@ -76,6 +77,28 @@ describe('buildFalVideoRequest — per-family wire bodies', () => {
expect(req.estimatedCostUsd).toBe(3.84);
});
+ it('LTX-2.3 audio-to-video: prompt + frame + audio on an explicit canvas that keeps the shot aspect; length follows the audio', () => {
+ const req = buildFalVideoRequest({
+ modelId: FAL_AUDIO_VIDEO_MODEL, prompt: 'drummer', negativePrompt: 'blur', imageUrl: 'i', audioUrl: 'a', audioSec: 3.735,
+ width: 1920, height: 1080, resolution: '1080P',
+ });
+ expect(req.body).toEqual({
+ prompt: 'drummer', negative_prompt: 'blur', image_url: 'i', audio_url: 'a', video_size: { width: 1920, height: 1080 },
+ match_audio_length: true, fps: 24, audio_strength: 1,
+ });
+ expect(req.seconds).toBeNull();
+ // 1920×1080 × 97 frames (3.735s at 24fps, up to LTX's 8n+1 count) at $0.001605/MP.
+ expect(req.estimatedCostUsd).toBe(0.3228);
+ expect(buildFalVideoRequest({ modelId: FAL_AUDIO_VIDEO_MODEL, prompt: 'x', audioUrl: 'a', width: 1080, height: 1920 }).body.video_size)
+ .toEqual({ width: 720, height: 1280 });
+ });
+
+ it('OmniHuman v1.5: frame + audio, an optional prompt and its own resolution enum', () => {
+ expect(buildFalVideoRequest({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', imageUrl: 'i', audioUrl: 'a', resolution: '720P' }).body)
+ .toEqual({ image_url: 'i', audio_url: 'a', resolution: '720p' });
+ expect(estimateFalVideoCostUsd({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', seconds: 4 })).toBe(0.64);
+ });
+
it('refuses a request the model cannot honor before anything is paid for; an uncurated id is not built', () => {
const refused = (fn) => { try { fn(); } catch (err) { return err.code; } return null; };
expect(refused(() => buildFalVideoRequest({ modelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'x' }))).toBe('FAL_MODEL_INPUT');
diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.js
index b8ed5f78a5..d5fef203ef 100644
--- a/server/lib/musicVideoDependencies.js
+++ b/server/lib/musicVideoDependencies.js
@@ -14,7 +14,9 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s
revision: checksum(Object.fromEntries(['startSec', 'endSec', 'visualLayer', 'stillMove', 'cardText', 'cardColor', 'shotMode', 'performanceSpeaker', 'loop']
.map((key) => [key, scene[key] ?? null]).concat([['actionContract', scene.direction?.actionContract || null]])
// #10589: only a planned camera joins the revision, so pre-camera evidence stays current.
- .concat(scene.camera ? [['camera', scene.camera]] : []))) });
+ .concat(scene.camera ? [['camera', scene.camera]] : [])
+ // Only a music-driven performance joins it, so lip-sync evidence stays current.
+ .concat(scene.performanceStyle === 'music' ? [['performanceStyle', 'music']] : []))) });
const snapshot = (references) => ({ version: MUSIC_VIDEO_DEPENDENCY_VERSION, references });
/** Capture the actual submitted plate, including optional crop/mask provenance. */
diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.js
index 5d2d7b9768..95d483eae0 100644
--- a/server/lib/musicVideoShotTiming.js
+++ b/server/lib/musicVideoShotTiming.js
@@ -15,6 +15,10 @@
* it is cutaway-only here until an actual adapter proves exact-source-audio
* conditioning — REST documentation for a different Grok surface is not that
* proof. Local runtimes are absent for the same reason.
+ * - `SOURCE_AUDIO_MUSIC` — the music-driven performance lanes: fal.ai
+ * audio-to-video models (kind `a2v`) where the song slice drives the whole
+ * picture's motion rather than only a singer's mouth. A scene opts in with
+ * `performanceStyle: 'music'`; it shares the window planning below.
* - `planPerformanceWindow` — the audio window a performance shot submits:
* inside the provider's limits (never relying on its silent truncation), a
* short shot padded to a supported contextual window with an explicit edit
@@ -38,6 +42,8 @@
import { GROK_VIDEO_DURATIONS, nearestGrokDuration } from './grokVideoClip.js';
import {
+ FAL_AUDIO_VIDEO_MODEL,
+ FAL_AUDIO_VIDEO_MODELS,
FAL_DEFAULT_IMAGE_VIDEO_MODEL,
FAL_LIPSYNC_VIDEO_MODEL,
coerceFalVideoSeconds,
@@ -58,6 +64,17 @@ export const MUSIC_VIDEO_SHOT_MODES = Object.freeze(['cutaway', 'performance']);
export const isPerformanceScene = (scene) => scene?.shotMode === 'performance'
&& (scene.visualLayer == null || scene.visualLayer === 'footage');
+/**
+ * How a performance shot follows the song: `lipsync` (absent on every older
+ * record) syncs a singer's mouth to the vocal; `music` lets the song slice
+ * drive the whole picture's motion — a drummer hitting on the beat, a dancer,
+ * a crowd — on an audio-to-video model (lib/falVideoModels.js kind `a2v`).
+ */
+export const MUSIC_VIDEO_PERFORMANCE_STYLES = Object.freeze(['lipsync', 'music']);
+
+/** A performance shot a singer lip-syncs (the only kind whose mouth sync is measured). */
+export const isLipSyncScene = (scene) => isPerformanceScene(scene) && scene.performanceStyle !== 'music';
+
/**
* The shot instruction of a performance scene's SELECTED video take, or null —
* for a cutaway scene (even one whose selected take was once generated as a
@@ -100,6 +117,7 @@ export const MUSIC_VIDEO_LIPSYNC_DEFAULT_RESOLUTION = '1080P';
export const SOURCE_AUDIO_LIPSYNC = Object.freeze({
fal: Object.freeze({
provider: 'fal',
+ style: 'lipsync',
label: 'fal.ai MiniMax H3 lip-sync',
modelId: FAL_LIPSYNC_VIDEO_MODEL,
minAudioSec: 5,
@@ -113,10 +131,51 @@ export const SOURCE_AUDIO_LIPSYNC = Object.freeze({
}),
});
-/** The lip-sync capability a backend offers, or null. */
-export const performanceCapability = (backend) => (
- Object.prototype.hasOwnProperty.call(SOURCE_AUDIO_LIPSYNC, backend || '') ? SOURCE_AUDIO_LIPSYNC[backend] : null
-);
+/**
+ * Default output resolution of a music-driven take. Both audio-to-video
+ * models offer 1080p, and LTX-2.3 bills by pixel at a few cents a second.
+ */
+const MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION = '1080p';
+
+const musicCapability = (model) => {
+ const resolutions = falVideoResolutions(model);
+ const defaultResolution = resolutions.includes(MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION) ? MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION : model.resolution.default;
+ return Object.freeze({
+ provider: 'fal',
+ style: 'music',
+ label: `fal.ai ${model.label}`,
+ modelId: model.id,
+ minAudioSec: model.audioInput.minSec,
+ // The same one-phrase per-take ceiling as lip-sync (see above).
+ maxAudioSec: Math.min(model.audioInput.maxSec, 30),
+ transcription: false,
+ resolutions: Object.freeze(resolutions),
+ defaultResolution,
+ costLabel: `${describeFalVideoRate(model.id, defaultResolution)} — billed to your fal.ai account`,
+ });
+};
+
+/**
+ * Verified music-driven lanes, keyed by backend then model id. The project's
+ * `videoSettings.falMusicModelId` picks one; absent (or a model this install
+ * does not curate) = `FAL_AUDIO_VIDEO_MODEL` (LTX-2.3 audio-to-video).
+ */
+export const SOURCE_AUDIO_MUSIC = Object.freeze({
+ fal: Object.freeze(Object.fromEntries(FAL_AUDIO_VIDEO_MODELS.map((model) => [model.id, musicCapability(model)]))),
+});
+
+/**
+ * The source-audio capability a backend offers a performance shot, or null.
+ * Without a scene (or for a lip-sync scene) it is the lip-sync lane; a
+ * music-driven scene gets the audio-to-video model `videoSettings` pins.
+ */
+export function performanceCapability(backend, { scene = null, videoSettings = null } = {}) {
+ if (!Object.prototype.hasOwnProperty.call(SOURCE_AUDIO_LIPSYNC, backend || '')) return null;
+ if (scene?.performanceStyle !== 'music') return SOURCE_AUDIO_LIPSYNC[backend];
+ const lanes = SOURCE_AUDIO_MUSIC[backend];
+ const pinned = videoSettings?.falMusicModelId;
+ return (pinned && Object.prototype.hasOwnProperty.call(lanes, pinned) ? lanes[pinned] : null) || lanes[FAL_AUDIO_VIDEO_MODEL];
+}
/**
* Why a performance shot cannot render on `backend`, or null when it can.
@@ -259,9 +318,9 @@ export function approximateMotionCues(phrases, startSec, endSec) {
* performance shot on a lane that cannot lip-sync at all, which is blocked
* outright rather than split — have no limit here.
*/
-export function shotSplitLimit(scene, backend) {
+export function shotSplitLimit(scene, backend, videoSettings = null) {
if (isPerformanceScene(scene)) {
- const capability = performanceCapability(backend);
+ const capability = performanceCapability(backend, { scene, videoSettings });
return capability ? round6(capability.maxAudioSec - PERFORMANCE_WINDOW_MARGIN_SEC) : null;
}
const footage = scene?.visualLayer == null || scene.visualLayer === 'footage';
@@ -396,9 +455,10 @@ export function planShotSplit({ startSec, endSec, maxSec, lyricCues, phrases, be
export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null } = {}) {
const settings = videoSettings || {};
if (isPerformanceScene(scene)) {
- const capability = SOURCE_AUDIO_LIPSYNC.fal;
+ const capability = performanceCapability('fal', { scene, videoSettings: settings });
const model = getFalVideoModel(capability.modelId);
- const resolution = resolveFalVideoResolution(model, settings.falLipSyncResolution || capability.defaultResolution);
+ const pinned = capability.style === 'music' ? settings.falMusicResolution : settings.falLipSyncResolution;
+ const resolution = resolveFalVideoResolution(model, pinned || capability.defaultResolution);
const plan = planPerformanceWindow({
startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability,
});
@@ -445,7 +505,7 @@ export const falTakeRequestFields = (take) => (take.performance
export function planPerformanceRepair({ scene, temporal, excerptStartSec, backend, videoSettings = {} }) {
const no = (message) => ({ ok: false, message });
const instruction = selectedPerformanceInstruction(scene);
- const capability = performanceCapability(backend);
+ const capability = performanceCapability(backend, { scene, videoSettings });
if (!capability) return no(performanceBlockedReason(backend));
if (!instruction || instruction.songInterval?.startSec !== scene.startSec || instruction.songInterval?.endSec !== scene.endSec) return no('Review needed: the selected take no longer matches this shot');
if (scene.direction?.actionContract || scene.performanceRepair) return no('Review needed: edit the shot contract or review the existing continuation before repairing again');
diff --git a/server/lib/musicVideoShotTiming.test.js b/server/lib/musicVideoShotTiming.test.js
index 57e2f4f3e5..37ae9f7ef9 100644
--- a/server/lib/musicVideoShotTiming.test.js
+++ b/server/lib/musicVideoShotTiming.test.js
@@ -1,5 +1,7 @@
import { describe, it, expect } from 'vitest';
-import { planShotSplit, shotSplitLimit, SOURCE_AUDIO_LIPSYNC, PERFORMANCE_WINDOW_MARGIN_SEC } from './musicVideoShotTiming.js';
+import {
+ falSceneTake, falTakeRequestFields, planShotSplit, shotSplitLimit, SOURCE_AUDIO_LIPSYNC, PERFORMANCE_WINDOW_MARGIN_SEC,
+} from './musicVideoShotTiming.js';
// The split planner is an algorithm with a real input matrix (#8977): which
// boundary wins, how many pieces, and exact coverage are each ambiguous when
@@ -87,6 +89,29 @@ describe('shotSplitLimit', () => {
// Each failure variant is a distinct admission regression: evidence, word
// gap, provider minimum, or timebase mismatch must never resubmit a prefix.
+describe('music-driven performance takes', () => {
+ // The drummer case: a 3.735s shot of the song whose motion the audio drives.
+ const drummer = { shotMode: 'performance', performanceStyle: 'music', startSec: 212.718, endSec: 216.453 };
+
+ it('render on the audio-to-video model for the shot itself, without lip-sync\'s 5s padding', () => {
+ const take = falSceneTake({ scene: drummer, videoSettings: {}, songDurationSec: 240 });
+ expect(take).toMatchObject({ performance: true, modelId: 'fal-ai/ltx-2.3-22b/audio-to-video', seconds: 3.735, resolution: '1080p' });
+ expect(take.costUsd).toBeGreaterThan(0);
+ expect(falTakeRequestFields(take)).toEqual({ falResolution: '1080p' });
+ // The same shot as lip-sync pads to the route's 5s minimum.
+ expect(falSceneTake({ scene: { ...drummer, performanceStyle: null }, songDurationSec: 240 }).seconds).toBeCloseTo(5.05, 6);
+ });
+
+ it('follow the project pin (model and resolution) and fall back to the default for an uncurated pin', () => {
+ const pinned = falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5', falMusicResolution: '720p' }, songDurationSec: 240 });
+ expect(pinned).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '720p', costUsd: 0.5976 });
+ expect(falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'example/newer-model' }, songDurationSec: 240 }).modelId)
+ .toBe('fal-ai/ltx-2.3-22b/audio-to-video');
+ expect(shotSplitLimit(drummer, 'fal')).toBeCloseTo(20 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);
+ expect(shotSplitLimit(drummer, 'fal', { falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5' })).toBeCloseTo(30 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);
+ });
+});
+
describe('accepted-prefix repair planning', () => {
const scene = { shotMode: 'performance', sceneId: 'example', videoHistoryId: 'clip', startSec: 10, endSec: 24,
takes: [{ kind: 'video', assetId: 'clip', shotInstruction: { shotMode: 'performance',
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index 7e8bd5471f..cd2916e776 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -51,7 +51,7 @@ import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';
import { MUSIC_VIDEO_MEDIA } from './musicVideoMediumPlan.js';
import { MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';
import { CAMERA_FRAMINGS, CAMERA_MOVEMENT_VALUES, CAMERA_SPEEDS } from './cameraMovements.js';
-import { MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';
+import { MUSIC_VIDEO_PERFORMANCE_STYLES, MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';
import {
MUSIC_VIDEO_AUTOMATION_BUDGET_MAX_USD,
MUSIC_VIDEO_AUTOMATION_GUIDANCE_MAX,
@@ -152,6 +152,12 @@ export const musicVideoVideoSettingsSchema = z.object({
// false = lip-sync to the audio alone, without the provider transcribing it
// first (sung words it mishears turn into the wrong mouth shapes).
falLipSyncTranscription: z.boolean().nullable().optional(),
+ // Audio-to-video model and resolution for music-driven performance takes
+ // (scene.performanceStyle 'music'). Loosely validated like falModelId: a peer
+ // on a newer catalog may pin a model this install does not curate, which then
+ // falls back to the default (LTX-2.3 audio-to-video).
+ falMusicModelId: z.string().min(1).max(200).nullable().optional(),
+ falMusicResolution: z.string().min(1).max(16).nullable().optional(),
generationMode: z.enum(['image', 'audioReactive', 'suppliedAudio']).optional(),
audioReactiveLora: z.string().max(255).regex(/^[^/\\]+\.safetensors$/i).nullable().optional(),
audioReactiveScale: z.number().min(0).max(2).optional(),
@@ -1349,6 +1355,8 @@ export const musicVideoSceneCreateSchema = z.object({
// `cutaway` = the ordinary image-to-video shot. See lib/musicVideoShotTiming.js.
shotMode: z.enum(MUSIC_VIDEO_SHOT_MODES).optional(),
performanceSpeaker: z.string().trim().max(120).nullable().optional(),
+ // How a performance follows the song: lip-sync (absent/null) or music-driven motion.
+ performanceStyle: z.enum(MUSIC_VIDEO_PERFORMANCE_STYLES).nullable().optional(),
}).strict().refine(
(s) => s.startSec == null || s.endSec == null || s.endSec >= s.startSec,
{ message: 'endSec must be >= startSec', path: ['endSec'] },
@@ -1387,6 +1395,8 @@ export const musicVideoSceneUpdateSchema = z.object({
// `cutaway` = the ordinary image-to-video shot. See lib/musicVideoShotTiming.js.
shotMode: z.enum(MUSIC_VIDEO_SHOT_MODES).optional(),
performanceSpeaker: z.string().trim().max(120).nullable().optional(),
+ // How a performance follows the song: lip-sync (absent/null) or music-driven motion.
+ performanceStyle: z.enum(MUSIC_VIDEO_PERFORMANCE_STYLES).nullable().optional(),
referenceImageId: z.string().max(256).nullable().optional(),
videoHistoryId: z.string().max(64).nullable().optional(),
// #10154: the last generation failure the render hooks persisted. The board
diff --git a/server/services/musicVideo/autoReviewService.js b/server/services/musicVideo/autoReviewService.js
index bae968177f..7e5ab94de5 100644
--- a/server/services/musicVideo/autoReviewService.js
+++ b/server/services/musicVideo/autoReviewService.js
@@ -22,7 +22,7 @@
* so an event arriving mid-step can never start a second review.
*/
-import { isPerformanceScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
+import { isLipSyncScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
import { analyzeTemporalPerformance } from './temporalPerformance.js';
import { unlink } from 'fs/promises';
import { existsSync } from 'fs';
@@ -211,11 +211,11 @@ async function reviewDraft(project, run, excerpt) {
const shots = sections.flatMap((section) => {
const scene = project.scenes?.find((s) => s.sceneId === section.sceneId);
const instruction = selectedPerformanceInstruction(scene);
- const performance = 'performance' in section ? section.performance : (isPerformanceScene(scene)
+ const performance = 'performance' in section ? section.performance : (isLipSyncScene(scene)
? { takeId: scene.videoHistoryId, speaker: instruction?.speaker || null, conditioning: instruction?.audio?.conditioning || null } : null);
return performance ? [{ sceneId: section.sceneId, ...performance, startSec: section.startSec, endSec: section.endSec }] : [];
});
- const temporal = !sections.length && project.scenes?.some(isPerformanceScene)
+ const temporal = !sections.length && project.scenes?.some(isLipSyncScene)
? { version: 1, status: 'unverified', analyzer: null, reason: 'The draft has no performance-section provenance', shots: [] }
: await analyzeTemporalPerformance({ excerptPath, shots });
const evidence = { boundaryFrames: hasSheet ? 1 : 0, continuousFrames: frameTimes.length, temporal, ...(excerpt.footageVisibility ? { footageVisibility: excerpt.footageVisibility } : {}), excerptStartSec: excerpt.startSec };
diff --git a/server/services/musicVideo/excerpt.js b/server/services/musicVideo/excerpt.js
index 10b810505b..8924976b48 100644
--- a/server/services/musicVideo/excerpt.js
+++ b/server/services/musicVideo/excerpt.js
@@ -22,7 +22,7 @@
* `mutateProjectRecord`'s write serialization without duplicating validation.
*/
-import { isPerformanceScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
+import { isLipSyncScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
import { captureMusicVideoEvidence } from '../../lib/musicVideoDependencies.js';
import { randomUUID } from 'crypto';
import { ServerError } from '../../lib/errorHandler.js';
@@ -81,7 +81,7 @@ export function startExcerptOnProject(project, { startSec, endSec, sections = nu
sections: Array.isArray(sections) ? sections.map((section) => {
const scene = performanceProject.scenes?.find((s) => s.sceneId === section.sceneId);
const instruction = selectedPerformanceInstruction(scene);
- return { ...section, performance: isPerformanceScene(scene) && (!section.layer || section.layer === 'footage')
+ return { ...section, performance: isLipSyncScene(scene) && (!section.layer || section.layer === 'footage')
? { takeId: scene.videoHistoryId || null, speaker: instruction?.speaker || null, conditioning: instruction?.audio?.conditioning || null } : null };
}) : null,
// #9280: a social cut's frame (the project's own unless re-framed) and
diff --git a/server/services/musicVideo/performanceShot.js b/server/services/musicVideo/performanceShot.js
index eabfd300aa..47c022f99b 100644
--- a/server/services/musicVideo/performanceShot.js
+++ b/server/services/musicVideo/performanceShot.js
@@ -264,8 +264,11 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
if (!sourceImagePath || basename(sourceImagePath) !== scene.referenceImageId) throw refuse('Use the accepted boundary frame for this continuation', 'PERFORMANCE_REPAIR_STALE', 409);
}
- const capability = performanceCapability(backend);
+ // A music-driven scene renders on the project's audio-to-video model; a
+ // lip-sync scene on the lip-sync route.
+ const capability = performanceCapability(backend, { scene, videoSettings: project.videoSettings });
if (!capability) throw refuse(performanceBlockedReason(backend), 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');
+ const musicDriven = capability.style === 'music';
// The lip-sync route is image + source audio; any other semantic mode would
// ask it for something it does not do.
if (mode && mode !== 'image') {
@@ -286,12 +289,16 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
}
const offered = getFalVideoModel(capability.modelId)?.resolution?.options || capability.resolutions || [];
const pick = (value) => (typeof value === 'string' ? offered.find((o) => o.toLowerCase() === value.trim().toLowerCase()) : null);
- const takeResolution = pick(resolution) || pick(project.videoSettings?.falLipSyncResolution) || capability.defaultResolution;
+ const pinnedResolution = musicDriven ? project.videoSettings?.falMusicResolution : project.videoSettings?.falLipSyncResolution;
+ const takeResolution = pick(resolution) || pick(pinnedResolution) || capability.defaultResolution;
// An optional vocal stem conditions the provider in place of the mix. It
// is re-checked here because the file could have been replaced on disk or
// arrived from a peer since it was attached.
- const conditioningSource = project.performanceConditioningSource || (project.vocalStemFilename ? 'vocal-stem' : 'master');
+ // A music-driven take always hears the full mix: the drums, bass and
+ // everything else a vocal stem leaves out are what drive its motion.
+ const conditioningSource = musicDriven ? 'master'
+ : project.performanceConditioningSource || (project.vocalStemFilename ? 'vocal-stem' : 'master');
const stemPath = conditioningSource === 'master' ? null : resolveVocalStemPath(project);
if (conditioningSource !== 'master' && !stemPath) throw refuse('Attach the selected singer stem before rendering', 'MUSIC_VIDEO_VOCAL_STEM_MISSING');
if (stemPath) assertVocalStemTimebase(await probeVideoDuration(stemPath), songDurationSec);
@@ -330,6 +337,7 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
version: SHOT_INSTRUCTION_VERSION,
...(scene.performanceRepair ? { repair: { ...scene.performanceRepair, role: 'continuation' } } : {}),
shotMode: 'performance',
+ performanceStyle: capability.style,
createdAt: new Date().toISOString(),
speaker: scene.performanceSpeaker || null,
audio: {
@@ -343,7 +351,7 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
// and then only when `windowFingerprint` says this window changed.
conditioning: {
source: conditioningSource, sha256: conditioningSha256, filename: basename(stemPath || masterPath),
- selection: project.performanceConditioningSource ? 'user' : 'legacy-default',
+ selection: musicDriven ? 'music-driven' : project.performanceConditioningSource ? 'user' : 'legacy-default',
voiceIsolation: 'unverified',
},
},
@@ -355,6 +363,7 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
performance: (scene.visualIntent || scene.prompt || '').slice(0, 2000),
capability: {
provider: capability.provider,
+ style: capability.style,
modelId: capability.modelId,
minAudioSec: capability.minAudioSec,
maxAudioSec: capability.maxAudioSec,
diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.js
index 3439bbf33f..cc20b791db 100644
--- a/server/services/musicVideo/performanceShot.test.js
+++ b/server/services/musicVideo/performanceShot.test.js
@@ -298,6 +298,73 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
expect(existsSync(PATHS.uploads) ? await readdir(PATHS.uploads) : []).toEqual([]);
});
+ it('a music-driven shot hears the full mix for exactly its span and submits frame + prompt + audio to the audio-to-video model', async () => {
+ await writeFile(join(PATHS.music, 'vocals.wav'), songWav({ markerAmp: 12000 }));
+ // A vocal stem is attached, but the drums it leaves out are what drive this shot.
+ getProject.mockResolvedValue({ ...project({ performanceStyle: 'music', prompt: 'drummer hits on the beat' }), vocalStemFilename: 'vocals.wav',
+ videoSettings: { backend: 'fal', falLipSyncTranscription: true } });
+ const prepared = await preparePerformanceShot({
+ musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'frame.png'), mode: 'image',
+ });
+ const { shotInstruction: si } = prepared;
+ // 1.5s clears the model's 1s minimum: no contextual padding, in-point 0.
+ expect(si.audioWindow).toEqual({ startSec: 20, endSec: 21.5, durationSec: 1.5 });
+ expect(si.edit).toEqual({ inSec: 0, outSec: 1.5, targetSec: 1.5 });
+ expect(si).toMatchObject({ performanceStyle: 'music', transcription: false,
+ capability: { style: 'music', modelId: 'fal-ai/ltx-2.3-22b/audio-to-video', resolution: '1080p' } });
+ expect(si.audio.conditioning).toMatchObject({ source: 'master', filename: 'song.wav', selection: 'music-driven' });
+ expect(prepared).toMatchObject({ modelId: 'fal-ai/ltx-2.3-22b/audio-to-video', resolution: '1080p', enableTranscription: false });
+ const samples = pcm(await readFile(prepared.audioFilePath));
+ expect(samples.readInt16LE(0)).toBe(30000);
+
+ await mkdir(PATHS.images, { recursive: true });
+ await writeFile(join(PATHS.images, 'frame.png'), Buffer.from('89504e470d0a1a0a', 'hex'));
+ const deliveredPath = join(PATHS.data, 'delivered-music.mp4');
+ execFileSync(ffmpeg, ['-v', 'error', '-f', 'lavfi', '-i', 'color=blue:s=64x64:r=24:d=1.5', '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-y', deliveredPath]);
+ const delivered = await readFile(deliveredPath);
+ const posts = [];
+ vi.stubGlobal('fetch', vi.fn(async (url, opts) => {
+ if (opts?.method === 'POST') {
+ posts.push({ url, body: JSON.parse(opts.body) });
+ return jsonResponse({ request_id: 'r2', status_url: 'https://queue.fal.run/s2', response_url: 'https://queue.fal.run/r2' });
+ }
+ if (url === 'https://queue.fal.run/s2') return jsonResponse({ status: 'COMPLETED' });
+ if (url === 'https://queue.fal.run/r2') return jsonResponse({ video: { url: 'https://cdn.example.com/music.mp4' } });
+ if (url === 'https://cdn.example.com/music.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(delivered).buffer };
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+ const job = await fal.generateVideo({
+ apiKey: 'test-key', modelId: prepared.modelId, resolution: prepared.resolution, prompt: 'drummer hits on the beat', width: 1920, height: 1080,
+ sourceImagePath: join(PATHS.images, 'frame.png'), audioFilePath: prepared.audioFilePath,
+ lipSync: { enableTranscription: prepared.enableTranscription }, shotInstruction: si,
+ });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
+ expect(posts).toHaveLength(1);
+ expect(posts[0].url).toBe('https://queue.fal.run/fal-ai/ltx-2.3-22b/audio-to-video');
+ expect(posts[0].body).toMatchObject({
+ prompt: 'drummer hits on the beat', video_size: { width: 1920, height: 1080 }, match_audio_length: true, fps: 24, audio_strength: 1,
+ });
+ expect(posts[0].body.image_url).toMatch(/^data:image\//);
+ expect(posts[0].body.audio_url).toMatch(/^data:audio\/wav;base64,/);
+ const { loadHistory } = await import('../videoGen/history.js');
+ // 1920×1080 × 41 frames (1.5s at 24fps, up to LTX's 8n+1 count) at $0.001605/MP.
+ expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({
+ modelId: 'fal:fal-ai/ltx-2.3-22b/audio-to-video', resolution: '1080p', estimatedCostUsd: 0.1365, audioDriven: true,
+ });
+
+ // With no requested size the canvas follows the start frame's shape (a portrait plate stays portrait).
+ execFileSync(ffmpeg, ['-v', 'error', '-f', 'lavfi', '-i', 'color=gray:s=540x960', '-frames:v', '1', '-y', join(PATHS.images, 'portrait.png')]);
+ const again = await preparePerformanceShot({
+ musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'portrait.png'), mode: 'image',
+ });
+ const portraitJob = await fal.generateVideo({
+ apiKey: 'test-key', modelId: again.modelId, resolution: again.resolution, prompt: 'drummer hits on the beat',
+ sourceImagePath: join(PATHS.images, 'portrait.png'), audioFilePath: again.audioFilePath, shotInstruction: again.shotInstruction,
+ });
+ expect(await waitForTerminal(portraitJob.jobId)).toMatchObject({ type: 'completed' });
+ expect(posts[1].body.video_size).toEqual({ width: 1080, height: 1920 });
+ }, 60_000);
+
it('uses the full span of a long shot in one take, at the project\'s lip-sync resolution pin', async () => {
// 26s: past the old 14.8s cap (fal accepts up to 15 minutes; PortOS keeps
// a take to 30s so it stays one editable phrase).
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index 5f0ed0ef95..8e61395f67 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -131,6 +131,8 @@ export function buildProjectRecord(input, { id, now }) {
falModelId: videoSettings.falModelId ?? null,
falResolution: videoSettings.falResolution ?? null,
falLipSyncResolution: videoSettings.falLipSyncResolution ?? null,
+ falMusicModelId: videoSettings.falMusicModelId ?? null,
+ falMusicResolution: videoSettings.falMusicResolution ?? null,
generationMode: videoSettings.generationMode ?? 'image',
audioReactiveLora: videoSettings.audioReactiveLora ?? null,
audioReactiveScale: videoSettings.audioReactiveScale ?? 1.2,
@@ -383,6 +385,8 @@ export function applyProjectPatch(project, patch) {
falModelId: null,
falResolution: null,
falLipSyncResolution: null,
+ falMusicModelId: null,
+ falMusicResolution: null,
generationMode: 'image',
audioReactiveLora: null,
audioReactiveScale: 1.2,
@@ -497,6 +501,7 @@ function buildScene(input, { order }) {
// lip-synced performance shot, which only a source-audio provider renders.
shotMode: input.shotMode ?? 'cutaway',
...(input.performanceSpeaker ? { performanceSpeaker: input.performanceSpeaker } : {}),
+ ...(input.performanceStyle === 'music' ? { performanceStyle: 'music' } : {}),
referenceImageId: null,
videoHistoryId: null,
// #8965 — immutable candidate takes; the two slot fields above are the
@@ -698,7 +703,7 @@ export function splitScene(project, sceneId, { backend = null } = {}) {
if (idx < 0) throw new ServerError('Scene not found', { status: 404, code: 'NOT_FOUND' });
const scene = scenes[idx];
const lane = backend || project.videoSettings?.backend || null;
- const maxSec = shotSplitLimit(scene, lane);
+ const maxSec = shotSplitLimit(scene, lane, project.videoSettings);
if (maxSec == null) {
throw new ServerError('This backend renders the shot at any length — there is nothing to split.', { status: 400, code: 'MUSIC_VIDEO_SPLIT_NOT_NEEDED' });
}
@@ -747,6 +752,7 @@ export function splitScene(project, sceneId, { backend = null } = {}) {
visualLayer: scene.visualLayer ?? 'footage',
shotMode: scene.shotMode ?? 'cutaway',
performanceSpeaker: scene.performanceSpeaker ?? null,
+ performanceStyle: scene.performanceStyle ?? null,
...(scene.camera ? { camera: scene.camera } : {}),
...timing,
}),
@@ -796,7 +802,7 @@ export function mergeNextScene(project, sceneId, { backend = null } = {}) {
if (timedSec(startSec) && timedSec(endSec) && (endSec <= startSec || (timedSec(scene.endSec) && endSec < scene.endSec))) {
throw new ServerError('The next scene ends before this one does — put the scenes in song order before merging them.', { status: 400, code: 'MUSIC_VIDEO_MERGE_OUT_OF_ORDER' });
}
- const maxSec = shotSplitLimit(scene, lane);
+ const maxSec = shotSplitLimit(scene, lane, project.videoSettings);
if (maxSec != null) {
if (!timedSec(startSec) || !timedSec(endSec)) {
throw new ServerError('Both scenes need a start and end time before they can be merged on this backend.', { status: 400, code: 'MUSIC_VIDEO_MERGE_UNTIMED' });
diff --git a/server/services/musicVideo/projectsLogic.test.js b/server/services/musicVideo/projectsLogic.test.js
index 314178e483..1a0a4915eb 100644
--- a/server/services/musicVideo/projectsLogic.test.js
+++ b/server/services/musicVideo/projectsLogic.test.js
@@ -393,6 +393,8 @@ describe('applyProjectPatch', () => {
falModelId: null,
falResolution: null,
falLipSyncResolution: null,
+ falMusicModelId: null,
+ falMusicResolution: null,
generationMode: 'image',
audioReactiveLora: null,
audioReactiveScale: 1.2,
@@ -409,6 +411,8 @@ describe('applyProjectPatch', () => {
falModelId: null,
falResolution: null,
falLipSyncResolution: null,
+ falMusicModelId: null,
+ falMusicResolution: null,
generationMode: 'image',
audioReactiveLora: null,
audioReactiveScale: 1.2,
diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.js
index 83f6ad3d93..5ba979669d 100644
--- a/server/services/videoGen/fal.js
+++ b/server/services/videoGen/fal.js
@@ -159,10 +159,10 @@ export async function generateVideo({
if (promptRequired && !prompt?.trim()) {
throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });
}
- // A lip-sync render conditions a reference frame on source audio; either
- // input alone is not a request this route can honor.
+ // A lip-sync or audio-to-video render conditions a reference frame on
+ // source audio; either input alone is not a request this route can honor.
if (audioFilePath && !sourceImagePath) {
- throw new ServerError('A fal.ai lip-sync render needs a reference frame', { status: 400, code: 'VALIDATION_ERROR' });
+ throw new ServerError('A fal.ai source-audio render needs a reference frame', { status: 400, code: 'VALIDATION_ERROR' });
}
// The queue only persists the resolved provider CONFIG a job needs (mirrors
// grokPath on the grok lane) — never a secret — so the key is re-resolved
@@ -184,13 +184,20 @@ export async function generateVideo({
// only — a curated model resolves its own aspect rule.
const effectiveAspectRatio = FAL_ASPECT_RATIOS.includes(aspectRatio) ? aspectRatio : deriveAspectRatio(width, height);
- // A lip-sync take is billed on its audio length: the shot instruction's
+ // A source-audio take is billed on its audio length: the shot instruction's
// planned window, else the staged slice itself.
const audioSec = audioFilePath
? (shotInstruction?.audioWindow?.durationSec || await probeVideoDuration(audioFilePath).catch(() => null))
: null;
+ // A model that takes an explicit canvas (LTX-2.3) keeps the shot's shape:
+ // with no requested size, read it off the start frame rather than assume 16:9.
+ let canvas = { width, height };
+ if (catalogModel?.resolution?.wire === 'canvas' && !(Number(width) > 0 && Number(height) > 0) && sourceImagePath) {
+ const { readImageSize } = await import('../../lib/reactorStartingFrame.js');
+ canvas = (await readImageSize(sourceImagePath)) || canvas;
+ }
const requestSpec = {
- modelId, prompt, negativePrompt, seconds: duration, audioSec, aspectRatio, width, height, resolution,
+ modelId, prompt, negativePrompt, seconds: duration, audioSec, aspectRatio, width: canvas.width, height: canvas.height, resolution,
generateAudio: generateAudio === true, enableTranscription: lipSync?.enableTranscription === true,
};
// Dry-run the curated body with placeholder media before anything is read or
@@ -235,14 +242,15 @@ export async function generateVideo({
filename,
createdAt: new Date().toISOString(),
mode: sourceImagePath ? 'image' : 'text',
- ...(audioFilePath ? { lipSync: true } : {}),
+ // A lip-sync take syncs a mouth; an audio-to-video take moves the whole frame to the audio.
+ ...(audioFilePath ? (catalogModel?.kind === 'a2v' ? { audioDriven: true } : { lipSync: true }) : {}),
...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),
};
const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
jobs.set(jobId, job);
const costNote = meta.estimatedCostUsd != null ? ` ~$${meta.estimatedCostUsd.toFixed(2)}` : ' cost unknown';
- console.log(`🎬 Generating video [${jobId.slice(0, 8)}] fal (${modelId}${meta.resolution ? ` ${meta.resolution}` : ''}${costNote}): ${(prompt || '(lip-sync)').slice(0, 60)}…`);
+ console.log(`🎬 Generating video [${jobId.slice(0, 8)}] fal (${modelId}${meta.resolution ? ` ${meta.resolution}` : ''}${costNote}): ${(prompt || '(source audio)').slice(0, 60)}…`);
videoGenEvents.emit('started', { generationId: jobId, totalSteps: 1, ...meta });
activeJobs.set(jobId, { ...meta, generationId: jobId, totalSteps: 1, step: 0, progress: 0 });
emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Submitting to fal.ai…');
@@ -342,7 +350,7 @@ async function runFalVideo(job, jobId, {
const measured = (await probeVideoGeometry(outputPath)) || {};
const expectedSec = meta.audioWindow?.durationSec;
if (expectedSec && measured.durationSec && measured.durationSec + 0.1 < expectedSec) {
- console.warn(`⚠️ fal lip-sync [${jobId.slice(0, 8)}] delivered ${measured.durationSec.toFixed(2)}s for a ${expectedSec.toFixed(2)}s audio window`);
+ console.warn(`⚠️ fal source-audio take [${jobId.slice(0, 8)}] delivered ${measured.durationSec.toFixed(2)}s for a ${expectedSec.toFixed(2)}s audio window`);
}
activeRequests.delete(jobId);
activeJobs.delete(jobId);
diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.js
index fcda7b05bc..8037900773 100644
--- a/server/services/videoGen/prepareParams.js
+++ b/server/services/videoGen/prepareParams.js
@@ -530,13 +530,14 @@ async function resolvePreparedParams({
{ status: 400, code: 'VIDEO_GEN_AUDIO_REQUIRED' },
);
}
- // A fal lip-sync model takes the frame plus an uploaded voice clip (no Music Video scene needed).
- const falLipSync = body.backend === VIDEO_GEN_MODE.FAL && getFalVideoModel(body.falModelId)?.kind === 'lipsync';
- if (falLipSync && !uploads.audioFile && !body.musicVideo) {
+ // A fal source-audio model (lip-sync, or audio-to-video motion) takes the frame plus an uploaded
+ // audio clip (no Music Video scene needed).
+ const falSourceAudio = body.backend === VIDEO_GEN_MODE.FAL && !!getFalVideoModel(body.falModelId)?.audioInput;
+ if (falSourceAudio && !uploads.audioFile && !body.musicVideo) {
await cleanupStaged();
- throw new ServerError('A fal.ai lip-sync model needs an audioFile upload (the voice to sync to).', { status: 400, code: 'VIDEO_GEN_AUDIO_REQUIRED' });
+ throw new ServerError('A fal.ai source-audio model needs an audioFile upload (the audio it follows).', { status: 400, code: 'VIDEO_GEN_AUDIO_REQUIRED' });
}
- if (uploads.audioFile && body.mode !== 'a2v' && !falLipSync) {
+ if (uploads.audioFile && body.mode !== 'a2v' && !falSourceAudio) {
await cleanupStaged();
throw new ServerError(
`audioFile upload is only valid with mode='a2v' (got mode='${body.mode || 'unset'}').`,
@@ -835,14 +836,14 @@ async function resolvePreparedParams({
}
}
}
- // A standalone fal lip-sync take: stage the voice clip durably like the frame. The queue removes
+ // A standalone fal source-audio take: stage the audio clip durably like the frame. The queue removes
// it if the job fails, is cancelled or is lost to a restart (uploadedTempPaths); the fal worker
// removes it once the take is published.
let hostedAudioPath = null;
- if (falLipSync && uploads.audioFile) {
+ if (falSourceAudio && uploads.audioFile) {
if (!sourceImagePath) {
await cleanupStaged();
- throw new ServerError('A fal.ai lip-sync render needs a reference frame (sourceImage or sourceImageFile).', { status: 400, code: 'VALIDATION_ERROR' });
+ throw new ServerError('A fal.ai source-audio render needs a reference frame (sourceImage or sourceImageFile).', { status: 400, code: 'VALIDATION_ERROR' });
}
hostedAudioPath = await stageUploadDurable(uploads.audioFile, 'audio');
// A paid route: refuse a clip fal would reject (unreadable, or outside the model's window)
@@ -856,7 +857,7 @@ async function resolvePreparedParams({
}
if ((window.minSec != null && clipSec < window.minSec) || (window.maxSec != null && clipSec > window.maxSec)) {
await cleanupStaged();
- throw new ServerError(`The voice clip is ${clipSec.toFixed(2)} s; this lip-sync model takes ${window.minSec}–${window.maxSec} s (pad a short line with silence).`,
+ throw new ServerError(`The voice clip is ${clipSec.toFixed(2)} s; this model takes ${window.minSec}–${window.maxSec} s (pad a short line with silence).`,
{ status: 400, code: 'VIDEO_GEN_AUDIO_LENGTH' });
}
}
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index 974a9b076b..f83c030092 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -300,7 +300,11 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
const { falSceneTake } = await import('../../lib/musicVideoShotTiming.js');
const { costUsd } = falSceneTake({
scene,
- videoSettings: { falModelId: body.falModelId, falDuration: body.falDuration, falResolution: body.falResolution, falLipSyncResolution: body.falResolution },
+ videoSettings: {
+ falMusicModelId: project.videoSettings?.falMusicModelId ?? null,
+ falModelId: body.falModelId, falDuration: body.falDuration, falResolution: body.falResolution,
+ falLipSyncResolution: body.falResolution, falMusicResolution: body.falResolution,
+ },
songDurationSec: project.audioAnalysis?.durationSec ?? null,
});
if (Number.isFinite(costUsd)) params.musicVideoCostUsd = costUsd;