Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Add music-driven performance shots on fal audio-to-video models
Claude3f54cb3fdac7
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsxindex 7da2ed6742..1fd98d0adb 100644--- a/client/src/components/musicVideo/SceneCard.jsx+++ b/client/src/components/musicVideo/SceneCard.jsx@@ -123,7 +123,7 @@ export default function SceneCard({// #10297: a code shot is drawn by the composition, so it has no frame or clip to make.const codeShot = layered && layer === 'code';const performance = isPerformanceScene(scene);- const capability = performance ? performanceCapability(lipSyncBackend) : null;+ const capability = performance ? performanceCapability(lipSyncBackend, { scene, videoSettings: falVideoSettings }) : null;const timedSpan = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' && scene.endSec > scene.startSec? scene.endSec - scene.startSec: null;@@ -139,7 +139,7 @@ export default function SceneCard({: null;const falCost = falEstimate(falTake);const performanceCost = falTake?.performance ? (falCost || capability?.costLabel) : capability?.costLabel;- const splitLimit = layer === 'footage' ? shotSplitLimit(scene, lipSyncBackend) : null;+ const splitLimit = layer === 'footage' ? shotSplitLimit(scene, lipSyncBackend, falVideoSettings) : null;const canSplit = splitLimit != null && timedSpan != null && timedSpan > splitLimit + 1e-6;const mergedSpan = typeof scene.startSec === 'number' && typeof nextEndSec === 'number' ? nextEndSec - scene.startSec : null;const canMerge = !!onMergeNext && !isLast && !documentComposition@@ -149,7 +149,7 @@ export default function SceneCard({excerptStartSec: performanceReview?.excerptStartSec, backend: lipSyncBackend, videoSettings: falVideoSettings || {} });const shotModeId = `mv-shot-mode-${scene.sceneId}`;// #10152: what needs the director's attention, visible without opening the card.- const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec, failed: failedScenes || {}, songReview });+ const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec, failed: failedScenes || {}, songReview, videoSettings: falVideoSettings });const songChange = songReview?.[scene.sceneId];const generatingLane = generatingFrame && generatingVideo ? 'Frame + clip' : generatingFrame ? 'Frame' : generatingVideo ? 'Clip' : null;// #10154: the server-persisted failure of the last render, shown until a@@ -278,17 +278,20 @@ export default function SceneCard({{layer === 'footage' && (<><label htmlFor={shotModeId} className="flex items-center gap-1">Shot</label>- <select id={shotModeId} value={performance ? 'performance' : 'cutaway'}- onChange={(e) => applyPatch({ shotMode: e.target.value })}+ <select id={shotModeId} value={performance ? (scene.performanceStyle === 'music' ? 'music' : 'performance') : 'cutaway'}+ onChange={(e) => applyPatch(e.target.value === 'cutaway' ? { shotMode: 'cutaway' }+ : { shotMode: 'performance', performanceStyle: e.target.value === 'music' ? 'music' : null })}className="bg-port-bg border border-port-border rounded px-1 py-1 min-h-[44px] sm:min-h-0"- title="Cutaway: any video lane animates the frame under the song. Performance: a singer lip-synced to the song itself (needs a source-audio provider).">+ title="Cutaway: any video lane animates the frame under the song. Lip-sync: a singer's mouth follows the song. Music-driven: the song slice drives the whole frame's motion (a drummer on the beat). Both need a source-audio provider."><option value="cutaway">Cutaway</option><option value="performance">Performance (lip-sync)</option>+ <option value="music">Performance (music-driven)</option></select></>)}</div>- {performance && (+ {/* A music-driven shot follows the whole mix, not one voice. */}+ {performance && scene.performanceStyle !== 'music' && (<div className="flex flex-wrap items-center gap-2 text-xs"><label htmlFor={fieldId('speaker')}>Speaker / singer</label><input id={fieldId('speaker')} value={scene.performanceSpeaker || ''} maxLength={120}@@ -371,7 +374,7 @@ export default function SceneCard({)}{performance && !performanceBlocked && (<p className="text-[11px] text-port-text-muted break-words" data-testid="performance-plan">- Lip-sync via {capability.label} ({capability.modelId}) · song {formatDurationSec(plan.windowStartSec)}–{formatDurationSec(plan.windowEndSec)}+ {capability.style === 'music' ? 'Music-driven' : 'Lip-sync'} via {capability.label} ({capability.modelId}) · song {formatDurationSec(plan.windowStartSec)}–{formatDurationSec(plan.windowEndSec)}{plan.editInSec > 0 ? ` · shot starts ${plan.editInSec.toFixed(2)}s into the take` : ''} · {performanceCost}</p>)}@@ -517,7 +520,7 @@ export default function SceneCard({className="flex items-center gap-1 bg-port-border hover:bg-port-border/70 disabled:opacity-50 rounded px-2 py-1.5 text-xs min-h-[44px] sm:min-h-0 whitespace-nowrap"title={videoBlockedReason || performanceBlocked|| (!scene.referenceImageId ? 'Generate a reference frame first'- : performance ? `Lip-sync this scene's frame to the song via ${capability.label} — ${performanceCost}`+ : performance ? `${capability.style === 'music' ? "Animate this scene's frame to the song" : "Lip-sync this scene's frame to the song"} via ${capability.label} — ${performanceCost}`: "Generate this scene's video from its reference frame (i2v)")}>{generatingVideo ? <Activity size={14} className="animate-spin" /> : <Video size={14} />}{generatingVideo ? 'Generating video…' : (scene.videoHistoryId ? 'New video take' : 'Generate video')}diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsxindex 9bd7e7de52..e9f710fdd5 100644--- a/client/src/components/musicVideo/SceneCard.test.jsx+++ b/client/src/components/musicVideo/SceneCard.test.jsx@@ -48,6 +48,29 @@ describe('performance speaker save boundary', () => {});});+describe('music-driven performance', () => {+ it('switches a shot between cutaway, lip-sync and music-driven, and plans the music-driven take on the audio-to-video model', () => {+ const onSave = vi.fn();+ function Harness() {+ const [scene, setScene] = useState({ sceneId: 's1', shotMode: 'cutaway', startSec: 10, endSec: 13.5, referenceImageId: 'frame.png', takes: [] });+ return <SceneCard scene={scene} index={0} lipSyncBackend="fal" songDurationSec={60} falVideoSettings={{}}+ onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;+ }+ render(<Harness />);+ const shot = screen.getByLabelText('Shot');+ fireEvent.change(shot, { target: { value: 'music' } });+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: 'music' });+ expect(shot.value).toBe('music');+ // The shot's own 3.5s, not lip-sync's 5s minimum window.+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Music-driven via fal\.ai LTX-2\.3 audio-to-video \(fal-ai\/ltx-2\.3-22b\/audio-to-video\) · song 0:10–0:13 · est\. \$0\.30 \(3\.5s at 1080p\)/);+ fireEvent.change(shot, { target: { value: 'performance' } });+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: null });+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Lip-sync via/);+ fireEvent.change(shot, { target: { value: 'cutaway' } });+ expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'cutaway' });+ });+});+describe('persisted render failure chip (#10154)', () => {const failedScene = (lane) => ({ sceneId: 's1', startSec: 0, endSec: 5, referenceImageId: 'frame.png', takes: [],lastFailure: { lane, error: 'CUDA out of memory', at: '2026-10-05T00:00:00.000Z' } });diff --git a/client/src/components/musicVideo/VideoRenderSettings.jsx b/client/src/components/musicVideo/VideoRenderSettings.jsxindex 7d4f049b00..e15dd27580 100644--- a/client/src/components/musicVideo/VideoRenderSettings.jsx+++ b/client/src/components/musicVideo/VideoRenderSettings.jsx@@ -1,9 +1,9 @@import RenderPoolPicker from './RenderPoolPicker.jsx';import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';import {- FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falVideoResolutions, getFalVideoModel,+ FAL_AUDIO_VIDEO_MODEL, FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falVideoResolutions, getFalVideoModel,} from '../../lib/falVideoModels.js';-import { SOURCE_AUDIO_LIPSYNC } from '../../lib/musicVideoShotTiming.js';+import { SOURCE_AUDIO_LIPSYNC, SOURCE_AUDIO_MUSIC, performanceCapability } from '../../lib/musicVideoShotTiming.js';const falSelectCls = 'w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 sm:w-auto min-h-[44px] sm:min-h-0';const falOptionLabel = (model) => {@@ -35,6 +35,10 @@ export default function VideoRenderSettings({ videoSettings, generating }) {const falModel = getFalVideoModel(settings.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL);const falResolutions = falVideoResolutions(falModel);const lipSync = SOURCE_AUDIO_LIPSYNC.fal;+ // Music-driven performance takes (scene Shot › Performance (music-driven)):+ // an audio-to-video model where the song slice drives the whole frame.+ const music = performanceCapability('fal', { scene: { performanceStyle: 'music' }, videoSettings: settings });+ const musicModels = Object.values(SOURCE_AUDIO_MUSIC.fal);return (<><RenderPoolPicker settings={settings} change={change} disabled={locked} />@@ -240,6 +244,35 @@ export default function VideoRenderSettings({ videoSettings, generating }) {/>Transcript-guided lip-sync</label>+ <label htmlFor="mv-fal-music-model" className="sr-only">fal.ai music-driven model</label>+ <select+ id="mv-fal-music-model"+ value={settings.falMusicModelId || ''}+ onChange={(e) => change({ falMusicModelId: e.target.value || null, falMusicResolution: null })}+ disabled={locked}+ title="Audio-to-video model for music-driven performance shots: the scene's song slice drives the whole frame's motion, not just a mouth"+ className={`${falSelectCls} sm:max-w-[280px]`}+ >+ {musicModels.map((lane) => (+ <option key={lane.modelId} value={lane.modelId === FAL_AUDIO_VIDEO_MODEL ? '' : lane.modelId}>+ {`Music-driven · ${getFalVideoModel(lane.modelId).label} · ${describeFalVideoRate(lane.modelId, lane.defaultResolution)}${lane.modelId === FAL_AUDIO_VIDEO_MODEL ? ' (default)' : ''}`}+ </option>+ ))}+ </select>+ <label htmlFor="mv-fal-music-resolution" className="sr-only">fal.ai music-driven resolution</label>+ <select+ id="mv-fal-music-resolution"+ value={music.resolutions.includes(settings.falMusicResolution) ? settings.falMusicResolution : ''}+ onChange={(e) => change({ falMusicResolution: e.target.value || null })}+ disabled={locked}+ title={`Output resolution for music-driven takes on ${music.label}`}+ className={falSelectCls}+ >+ <option value="">{`Music-driven ${music.defaultResolution} (default)`}</option>+ {music.resolutions.map((res) => (+ <option key={res} value={res}>{`Music-driven ${res} · ${describeFalVideoRate(music.modelId, res)}`}</option>+ ))}+ </select></>)}</>diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsxindex c3298ce830..74dd4b5ae9 100644--- a/client/src/components/musicVideo/stages/BoardStage.jsx+++ b/client/src/components/musicVideo/stages/BoardStage.jsx@@ -41,6 +41,7 @@ export default function BoardStage({ board }) {layered, lipSyncBackend, songDurationSec, failed: sceneMedia.failedScenes,footageOptional: FOOTAGE_OPTIONAL_MODES.has(project.composition?.mode || 'concat'),songReview: project.songRevision?.sceneReview || null,+ videoSettings: videoSettings.settings,};const codesById = new Map(scenes.map((scene) => [scene.sceneId, sceneAttention(scene, attentionCtx)]));const counts = { all: scenes.length, attention: 0, missing: 0 };diff --git a/client/src/components/musicVideo/stages/ProduceStage.jsx b/client/src/components/musicVideo/stages/ProduceStage.jsxindex 9c4f58d957..94be033bdf 100644--- a/client/src/components/musicVideo/stages/ProduceStage.jsx+++ b/client/src/components/musicVideo/stages/ProduceStage.jsx@@ -30,6 +30,7 @@ export default function ProduceStage({ board }) {songDurationSec: project.audioAnalysis?.durationSec ?? null,failed: sceneMedia.failedScenes,songReview: project.songRevision?.sceneReview || null,+ videoSettings: videoSettings.settings,}} />)}<div className="flex justify-end"><ContactSheetButton onOpen={board.openContactSheet} /></div>diff --git a/client/src/hooks/useMusicVideoModelSettings.js b/client/src/hooks/useMusicVideoModelSettings.jsindex 00d82a8041..f56de184fb 100644--- a/client/src/hooks/useMusicVideoModelSettings.js+++ b/client/src/hooks/useMusicVideoModelSettings.js@@ -74,6 +74,12 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =falModelId: project?.videoSettings?.falModelId ?? null,falResolution: project?.videoSettings?.falResolution ?? null,falLipSyncResolution: project?.videoSettings?.falLipSyncResolution ?? null,+ // false = lip-sync without transcript guidance (null/absent = on).+ falLipSyncTranscription: project?.videoSettings?.falLipSyncTranscription ?? null,+ // Audio-to-video model / resolution for music-driven performance takes —+ // null means FAL_AUDIO_VIDEO_MODEL (LTX-2.3) / 1080p, resolved by falSceneTake.+ falMusicModelId: project?.videoSettings?.falMusicModelId ?? null,+ falMusicResolution: project?.videoSettings?.falMusicResolution ?? null,generationMode: project?.videoSettings?.generationMode || 'image',audioReactiveLora: project?.videoSettings?.audioReactiveLora || '',audioReactiveScale: project?.videoSettings?.audioReactiveScale ?? 1.2,diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex 9b689f9b4e..866f9a6dca 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -66,7 +66,7 @@ grep -i "what you want to do" client/src/lib/README.md| `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. || `musicVideoSceneAttention.js` | Music Video per-scene attention model (#10152): `sceneAttention(scene, ctx)` returns the codes (`missing-frame`, `missing-clip`, `under-covered`, `contract-problem`, `perf-blocked`, `candidates-pending`, `plate-unverified`, `no-span`, `last-failure`) behind the collapsed scene-row chips, the Board `?scenes=attention\|missing\|all` filter (`parseSceneFilter`, `sceneMatchesFilter`) and the Produce per-shot strip; `SCENE_ATTENTION_LABELS`. Used by `components/musicVideo/SceneCard.jsx`, `stages/BoardStage.jsx` and `stages/ProduceStage.jsx`. || `musicVideoCoverOverlay.js` | Cover-art lettering preview (#10345): re-export of `server/lib/musicVideoCoverOverlay.js` (the design vocabulary, `normalizeCoverDesign`, `coverOverlaySvg`) plus `canvasCoverWidths(design, texts, fonts)`, which measures text with a canvas so the live preview wraps and sizes the title the way the server's measured render does. |-| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes, the verified source-audio lip-sync capability table, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. |+| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes and performance styles, the verified source-audio lip-sync and music-driven capability tables, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. || `musicVideoStages.js` | Music Video page stage model: `MUSIC_VIDEO_STAGES` (six steps: Song `setup` → Look `cast-sets` → Storyboard `board` → Make `produce` → Final render `review` → Publish; a retired `compose` link resolves to Make), `resolveStageParam(value)` for the `:stage` route param, `deriveStages(project)` (per-stage `done`/`active`/`blocked`/`todo` + `current`, the first incomplete step; a live production run pins Make), `deriveNextAction(project, ctx)` (the one primary action the sticky header offers — approve Cast & Sets, verify lyric timing, stop/resume production, render final, …), `describeProjectStatus` (the header line and the step that needs the director), `autopilotStatus` (the header Autopilot button), `projectSpend(project)` (paid generation used vs. cap), `projectShotSummary(project)` (document shots or scene counts), `stageChecklist(stageId, project, readiness)` (what one step still needs, with the open approval's reason and a scroll anchor; an `optional` item, such as Make's animated proof, is listed but not counted), `summarizeStoryboardProblems(problems)` (folds the server's one-sentence-per-shot storyboard problems into one count line per kind), `stepNotes(project, readiness, publish)` (one fact per step for the step list), `listPreviewSources` (final, live composition, drafts, then the storyboard animatic), `projectServicesSummary(project)` (the saved image/video render services), `compareMusicVideoProjectsNewestFirst(a, b)` (sort projects newest created / updated first for project cards and pickers), and `currentProductionRun`; `productionReviewStopGuidance` separates current approval blockers from historical run stops. Used by `components/musicVideo/MusicVideoLayout.jsx` and `pages/MusicVideo.jsx`. || `musicVideoAttention.js` | Music Video "Needs attention" model (#9940): `deriveAttentionItems(project, live)` returns the banner rows for server-held states a client-orchestrated workflow can strand — an open/rendering revision, an interrupted Cast & Sets stage, a running auto-review waiting on its revised sections, a final render nobody is watching — derived from the persisted record so they survive a reload (healthy work this tab can see progressing is not flagged); `openRevisionOf(project)`; `ATTENTION_ANCHOR_ID` + `revealAttention()` scroll a refusal toast to the banner. Used by `components/musicVideo/NeedsAttentionBanner.jsx` and `pages/MusicVideo.jsx`. || `musicVideoProjectList.js` | Music Video project index model (#10167): `projectRunPill(project)` (Running / Needs you / Interrupted / Done, from the same next-action and attention logic as the header), `groupMusicVideoProjects(projects, query)` (collapse forks under their root, newest version first, name filter, groups ordered by last touch), and `compareMusicVideoProjectsRecentlyTouched`. Used by `pages/MusicVideo.jsx` and `components/musicVideo/MusicVideoProjectCard.jsx`. |@@ -162,7 +162,7 @@ grep -i "what you want to do" client/src/lib/README.md| `quotaBurnPatch.js` | `mergeQuotaBurnPatch(base, patch)` — mirrors the server's quota-burn config merge (top-level + per-family keys merge, a family's `jobs` array replaces) so the Quota Burn page can apply an edit optimistically and accumulate debounced edits into one PUT body. `quotaBurnJobIsSpent(job, ranAt)` — whether a `run once` step has had its one dispatch, gated on the optimistic config's own `runOnce` so a just-ticked checkbox reads as spent before the save round-trips (the client mirror of the server's `jobIsSpent`). `UNLIMITED_DISPATCHES` / `isUnlimitedDispatchCap(cap)` / `dispatchCapInput(value)` — the -1 "no dispatch cap" sentinel (the default), mirrored from the server. || `quotaBurnTasks.js` | `MAINTENANCE_TASK_ORDER` / `MAINTENANCE_ORDER_GUIDANCE` document the default ladder; `maintenanceSequence` populates audit and claim-drain references; `maintenancePrerequisites` lists missing saved settings and scoped setup patches for the selected mode and claim policy. The Quota Burn page's view of the SHARED scheduled-task catalog. `buildQuotaBurnTaskCatalog({ schedule, jobs, apps })` groups PortOS built-in task types and app custom jobs into the picker's two groups (dropping what a burn may never invoke, and stamping a `blockedReason` on what is merely disabled or untargetable); `searchTaskCatalog` / `flattenTaskCatalog` / `findTaskEntry` query it. `taskRefKey` / `stepFromTaskEntry` / `taskEntryNeedsApp` build and address a step's `taskRef`. `effectiveQuotaBurnSettings(step, saved)` mirrors the server's `effectiveSettings` (overrides layered over the task's saved settings, params merged key-by-key). `quotaBurnStepPayload(job)` reduces a step to what the strict PUT schema accepts, dropping the top-level override mirrors that would otherwise outrank the `overrides` bag. `taskSourceHref(entry)` / `CREATE_TASK_HREF` link out to the source task. || `clipboard.js` | `copyToClipboard`, `writeClipboardSilently`, `readClipboard` — safe across insecure-origin contexts. Use these instead of `navigator.clipboard.writeText` inline. |-| `falVideoModels.js` | Re-export of `server/lib/falVideoModels.js` — the curated fal.ai video model catalog, per-model request rules and `estimateFalVideoCostUsd()`. Used by the Music Video fal model picker (`components/musicVideo/VideoRenderSettings.jsx`) and the Video Gen page's fal model suggestions. |+| `falVideoModels.js` | Re-export of `server/lib/falVideoModels.js` — the curated fal.ai video model catalog (including the `FAL_AUDIO_VIDEO_MODELS` audio-to-video lanes), per-model request rules and `estimateFalVideoCostUsd()`. Used by the Music Video fal model picker (`components/musicVideo/VideoRenderSettings.jsx`) and the Video Gen page's fal model suggestions. || `falVideoHandoff.js` | `buildFalH3MaxPrompt(prompt, negativePrompt)` composes the provider-neutral shot prompt fal's free MiniMax H3 Max tool is driven with; `FAL_H3_MAX_FREE_URL` and `FAL_H3_MAX_FREE_ALLOWANCE_NOTE` carry the tool URL and current vendor allowance copy. The free web allowance is deliberately a handoff, not the separately metered fal API. || `compareHelpers.js` | `equalByKeys(a, b, keys)` / `equalListByKeys(a, b, keys)` — typed key-based equality for `useAutoRefetch`'s `compare`. Keys are property names, dotted paths (`'context.running'`), or `(item) => value` accessors. The typed alternative to `sameJsonShape` when a monotonic timestamp or unrendered field would break stringify-equality dedup. || `consoleFilters.js` | `installConsoleFilters()` — drops a small allow-list of known-noise console strings (THREE.js `Clock` deprecation, expected WebGL `Context Lost.`) from `console.{warn,log,debug}`. Idempotent; auto-installed on import. Imported for its side effect from `main.jsx`. |diff --git a/client/src/lib/falVideoModels.js b/client/src/lib/falVideoModels.jsindex da800925e4..7bc03d3137 100644--- a/client/src/lib/falVideoModels.js+++ b/client/src/lib/falVideoModels.js@@ -6,6 +6,8 @@* board's cost estimate and the server's request body cannot drift.*/export {+ FAL_AUDIO_VIDEO_MODEL,+ FAL_AUDIO_VIDEO_MODELS,FAL_DEFAULT_IMAGE_VIDEO_MODEL,FAL_DEFAULT_TEXT_VIDEO_MODEL,FAL_IMAGE_VIDEO_MODELS,diff --git a/client/src/lib/musicVideoSceneAttention.js b/client/src/lib/musicVideoSceneAttention.jsindex 9fcd5eb54b..ee0c22a9bf 100644--- a/client/src/lib/musicVideoSceneAttention.js+++ b/client/src/lib/musicVideoSceneAttention.js@@ -41,8 +41,8 @@ export const SCENE_FILTERS = Object.freeze(['all', 'attention', 'missing']);/** The `?scenes=` value as a filter id; anything unknown is `all`. */export const parseSceneFilter = (value) => (SCENE_FILTERS.includes(value) ? value : 'all');-function performanceBlocked(scene, { lipSyncBackend, songDurationSec }) {- const capability = performanceCapability(lipSyncBackend);+function performanceBlocked(scene, { lipSyncBackend, songDurationSec, videoSettings }) {+ const capability = performanceCapability(lipSyncBackend, { scene, videoSettings });if (!capability) return performanceBlockedReason(lipSyncBackend);const plan = planPerformanceWindow({ startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability });return plan.ok ? null : plan.message;@@ -59,7 +59,7 @@ function performanceBlocked(scene, { lipSyncBackend, songDurationSec }) {*/export function sceneAttention(scene, ctx = {}) {if (!scene) return [];- const { layered = false, footageOptional = false, lipSyncBackend = '', songDurationSec = null, clipSec = null, failed = {}, songReview = null } = ctx;+ const { layered = false, footageOptional = false, lipSyncBackend = '', songDurationSec = null, clipSec = null, failed = {}, songReview = null, videoSettings = null } = ctx;const layer = sceneVisualLayer(scene, { layered });const codes = [];const songChange = songReview?.[scene.sceneId];@@ -72,7 +72,7 @@ export function sceneAttention(scene, ctx = {}) {&& scene.endSec - scene.startSec - clipSec > COVERAGE_TOLERANCE_SEC) codes.push('under-covered');const contract = scene.direction?.actionContract;if (contract && shotActionContractProblem(contract, scene)) codes.push('contract-problem');- if (layer === 'footage' && isPerformanceScene(scene) && performanceBlocked(scene, { lipSyncBackend, songDurationSec })) codes.push('perf-blocked');+ if (layer === 'footage' && isPerformanceScene(scene) && performanceBlocked(scene, { lipSyncBackend, songDurationSec, videoSettings })) codes.push('perf-blocked');if (sceneHasPendingDecision(scene)) codes.push('candidates-pending');if (contract && scene.referenceImageId) {const selected = sceneTakeList(scene, 'image').find((take) => take.assetId === scene.referenceImageId);diff --git a/client/src/lib/musicVideoShotTiming.js b/client/src/lib/musicVideoShotTiming.jsindex d68934c1dd..4a9fda7ed2 100644--- a/client/src/lib/musicVideoShotTiming.js+++ b/client/src/lib/musicVideoShotTiming.js@@ -9,6 +9,7 @@export {MUSIC_VIDEO_SHOT_MODES,SOURCE_AUDIO_LIPSYNC,+ SOURCE_AUDIO_MUSIC,approximateMotionCues,falSceneTake,falTakeRequestFields,diff --git a/server/lib/README.md b/server/lib/README.mdindex 201088aaea..4fc3d91db9 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -321,11 +321,11 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `cursor.js` | Cursor Agent (`cursor-agent`) provider helpers — the `CURSOR_COMMAND` binary constant, the `isCursorCommand` predicate, and the `ensureCursorHeadlessArgs` (`--print --force`, folding any pinned reasoning effort into `--model`) / `ensureCursorTuiArgs` (`--force`) argv builders. `--force` is load-bearing beyond approvals: it also clears cursor's workspace-trust gate, which otherwise EXITS a headless run before any work happens. The prompt rides raw stdin (like claude/codex, unlike grok/kimi), and cursor needs no configured-default sentinel — its `auto` router is a real model id passed straight to `--model`. || `grok.js` | xAI Grok Build (`grok`) provider helpers — id/endpoint constants (`GROK_API_ID`/`GROK_CLI_ID`/`GROK_TUI_ID`/`GROK_API_ENDPOINT`), `isGrokCommand`/`isGrokCliProvider`/`isGrokTuiProvider` predicates, `ensureGrokHeadlessArgs`/`ensureGrokTuiArgs` argv builders (grok reads its prompt from `--prompt-file /dev/stdin`, not raw stdin; model selection uses the `GROK_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Antigravity), and `prepareGrokPromptFile` (Windows temp-file delivery fallback; `cleanup({ throwOnError: true })` reports cleanup failure to an owning lifecycle). || `filmStyleGrammars.js` | Curated film style grammar catalog (`FILM_STYLE_GRAMMARS`, eight code-rendered media with rendering, colour, type, motion, camera, sound and native-move guidance), `getFilmStyleGrammar`, `summarizeFilmStyleGrammar` (picker projection) and `knownFilmStyleIdSchema` (request-field id check) and `renderFilmStyleGrammarPrompt(id, { parts })` — one prompt section capped at `FILM_STYLE_PROMPT_MAX_CHARS`; unknown id or bad parts is a 400. |-| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. |+| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, LTX-2.3 and OmniHuman v1.5 audio-to-video (`FAL_AUDIO_VIDEO_MODELS`, kind `a2v`: the supplied audio drives the whole frame's motion), Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. || `grokVideoClip.js` | Clip lengths grok's `image_to_video` actually delivers — `GROK_VIDEO_DURATIONS` (`[6, 10]`, measured in #3022: a 2s/3s request returns the same 6.04s clip), `GROK_VIDEO_DEFAULT_DURATION`, `resolveGrokDuration()` (validate an already-grok-shaped request, else default), and `nearestGrokDuration()` (round a length authored against another backend's continuous contract UP to the shortest clip that covers it — for translating a CD/commission `targetDurationSeconds`). Dependency-free so `validation.js` / `routes/videoGen.js` can derive their schemas from the same list the services gate on. || `fontMetadata.js` | `fontContainer(buf)` (sfnt / woff2 / null) and `fontFamilyName(buf)` — a font file's family name read from its `name` table (TrueType, OpenType, and WOFF2's brotli stream) with no font library; null for anything unreadable (#10345). || `musicVideoCoverOverlay.js` | Music Video cover lettering (#10345): `COVER_DESIGN_OPTIONS` / `DEFAULT_COVER_DESIGN`, `normalizeCoverDesign(design, { fonts })` (accepts every stored design; a `font:<id>` typeface only when that uploaded font is listed), `coverOverlaySvg({ title, tag, design, widths, fonts })` and the `coverFontSpec` / `fontAttrs` / `titleLinesFor` / `tagTextFor` helpers the width probe reads. Pure; the server render (`coverArtCompose.js`) and the client's live Lettering preview run this one layout. |-| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend)`, `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. |+| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `MUSIC_VIDEO_PERFORMANCE_STYLES` (`lipsync`/`music`) + `isLipSyncScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `SOURCE_AUDIO_MUSIC` (music-driven lanes — fal.ai audio-to-video models, picked by `videoSettings.falMusicModelId`, default LTX-2.3 at 1080p), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend, { scene, videoSettings })` (lip-sync lane, or the music-driven one for a music-driven scene), `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. || `reactorStartingFrame.js` | Fits a reactor.inc `starting_frame` to the fast-h3 session canvas before it is uploaded. `readImageSize(path)` → `{width,height}` or `null` (never a guessed dimension), and `prepareReactorStartingFrame(sourceImagePath, requestedAspect, outputPath)` → `{ aspect, canvas, framePath, fittedPath }`: an absent `requestedAspect` DERIVES the canvas from the frame's own shape (`nearestReactorAspect`), then centre cover-crops the frame to that canvas's exact pixel size so reactor's own "fitted to the session canvas" step is a no-op. `fittedPath` is the temp file the caller deletes; `null` means the original was uploaded unchanged. A frame sharp cannot decode (or a resize that fails) passes through untouched on the default canvas rather than failing a render PortOS merely could not measure — reactor decodes more containers than sharp does. || `reactorVideoClip.js` | The reactor.inc `fast-h3` clip contract, dependency-free so the route schema, the FableLoom prompt compiler and the VideoGen picker all read one copy — `REACTOR_MAX_PROMPT_LENGTH` (800; the API rejects a longer prompt rather than truncating it), `REACTOR_MIN_CLIP_SECONDS` / `REACTOR_MAX_CLIP_SECONDS` (5.167–14.375, i.e. 124–345 frames at the `REACTOR_CLIP_FPS` 24 it delivers), `REACTOR_MAX_CLIP_ID_LENGTH`, and the picker list `REACTOR_CLIP_LENGTHS` + `REACTOR_DEFAULT_CLIP_LENGTH` + `reactorClipLengthLabel()` (the two exact endpoints plus every whole second between them, since the accepted range is continuous but starts and ends off a round number). Also the CANVAS contract `set_canvas` accepts: `REACTOR_CANVASES` (`16:9` 1344×768, `4:3` 1024×768, `1:1` 768×768, `9:16` 768×1344 — every one a 768px short edge, so the aspect string is the whole choice), `REACTOR_ASPECTS`, `REACTOR_DEFAULT_ASPECT`, `reactorCanvas(aspect)` and `nearestReactorAspect(width, height)` (closest canvas in log-ratio space; anything unmeasurable answers the default rather than deriving one). Mirrored to `client/src/lib/reactorVideoClip.js`, parity pinned by `reactorVideoClip.test.js`. || `kimi.js` | Moonshot AI Kimi Code (`kimi`) provider helpers — id constants (`KIMI_CLI_ID`/`KIMI_TUI_ID`), `isKimiCommand`/`isKimiCliProvider`/`isKimiTuiProvider` predicates, `ensureKimiHeadlessArgs` (model flag only — non-interactive mode is implicit in `--prompt`, and kimi has no `--print`/`--afk` and refuses approval flags alongside `--prompt`) / `ensureKimiTuiArgs` (`--yolo`) argv builders (model selection uses the `KIMI_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Grok/Antigravity), and `prepareKimiPrompt` (delivers the prompt as the `--prompt <value>` argv, not stdin). |diff --git a/server/lib/falVideoModels.js b/server/lib/falVideoModels.jsindex aa13fa6e30..0307ac7c68 100644--- a/server/lib/falVideoModels.js+++ b/server/lib/falVideoModels.js@@ -26,14 +26,20 @@*//** Date the schemas and prices below were last verified against fal.ai. */-export const FAL_VIDEO_PRICING_VERIFIED_AT = '2026-09-29';+export const FAL_VIDEO_PRICING_VERIFIED_AT = '2026-10-11';-/** What an entry renders from: text, a start frame, reference assets, or a frame + source audio. */-export const FAL_VIDEO_KINDS = Object.freeze(['t2v', 'i2v', 'ref2v', 'lipsync']);+/**+ * What an entry renders from: text, a start frame, reference assets, a frame ++ * source audio it lip-syncs to (`lipsync`), or a frame + source audio that+ * drives the whole picture's motion (`a2v` — a drummer hitting on the beat, a+ * dancer, a crowd), steered by a prompt.+ */+export const FAL_VIDEO_KINDS = Object.freeze(['t2v', 'i2v', 'ref2v', 'lipsync', 'a2v']);export const FAL_DEFAULT_TEXT_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/text-to-video';export const FAL_DEFAULT_IMAGE_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/image-to-video';export const FAL_LIPSYNC_VIDEO_MODEL = 'minimax/h3-max/lip-sync/image-to-video';+export const FAL_AUDIO_VIDEO_MODEL = 'fal-ai/ltx-2.3-22b/audio-to-video';const H3_RESOLUTIONS = Object.freeze(['480P', '768P', '1080P']);const SEEDANCE_RESOLUTIONS = Object.freeze(['480p', '720p', '1080p']);@@ -46,6 +52,9 @@ const H3_MAX_RATES = Object.freeze({ '480P': 0.05, '768P': 0.08, '1080P': 0.16 }// per-second price for a 16:9 output; those published figures are used (they// run ~2% above the nominal-pixel formula, i.e. the provider pads the canvas).const SEEDANCE_PER_SEC_16X9 = Object.freeze({ '480p': 0.2205, '720p': 0.473, '1080p': 1.164 });+// LTX-2.3 takes an explicit {width, height} canvas; PortOS names it by its+// short side and keeps the requested aspect (see `ltxCanvas`).+const LTX_SHORT_SIDES = Object.freeze({ '480p': 480, '720p': 720, '1080p': 1080 });const seedance = (id, kind, label, extra) => ({id, kind, label, family: 'seedance-2.5',@@ -145,6 +154,41 @@ export const FAL_VIDEO_MODELS = Object.freeze([seedance('bytedance/seedance-2.5/text-to-video', 't2v', 'Seedance 2.5 (text)', {aspectRatio: { param: 'aspect_ratio', options: WIDE_ASPECTS, auto: 'auto' },}),+ {+ // LTX-2.3 22B is a joint audio-video model: the supplied audio conditions+ // every frame, so motion lands on the music (and a singer's mouth on the+ // vocal) rather than being only a mouth layered over a still pose.+ id: FAL_AUDIO_VIDEO_MODEL, kind: 'a2v', label: 'LTX-2.3 audio-to-video', family: 'ltx-2.3',+ prompt: 'required',+ image: { param: 'image_url', required: false },+ endImage: { param: 'end_image_url' },+ // num_frames is 9–481 at 24fps, i.e. up to ~20s; with match_audio_length+ // the frame count follows the audio.+ audioInput: { param: 'audio_url', minSec: 1, maxSec: 20 },+ duration: { kind: 'audio' },+ resolution: { param: 'video_size', options: Object.keys(LTX_SHORT_SIDES), default: '720p', wire: 'canvas', fps: 24 },+ aspectRatio: null,+ negativePrompt: 'native',+ seed: 'seed',+ // audio_strength 1 keeps the supplied audio unmodified in the output.+ fixed: { match_audio_length: true, fps: 24, audio_strength: 1 },+ // "$0.001605 per megapixel of generated video data (width × height × frames)".+ pricing: { perMegapixel: 0.001605 },+ },+ {+ // Audio-driven human figure: expression, gesture and body movement follow+ // the audio, not just the mouth. Expects a person in the frame.+ id: 'fal-ai/bytedance/omnihuman/v1.5', kind: 'a2v', label: 'OmniHuman v1.5 (audio-driven person)', family: 'omnihuman',+ prompt: 'optional',+ image: { param: 'image_url', required: true },+ // "Audio must be under 30s long for 1080p generation and under 60s long for 720p".+ audioInput: { param: 'audio_url', minSec: 1, maxSec: 30 },+ duration: { kind: 'audio' },+ resolution: { param: 'resolution', options: ['720p', '1080p'], default: '1080p' },+ aspectRatio: null,+ negativePrompt: 'fold',+ pricing: { perSecond: 0.16 },+ },seedance('bytedance/seedance-2.5/reference-to-video', 'ref2v', 'Seedance 2.5 (reference)', {// Referenced in the prompt as @Image1 / @Audio1. Audio references let a// singer follow supplied audio, but each clip is capped at 30.2s.@@ -196,6 +240,9 @@ export const getFalVideoModel = (modelId) => (typeof modelId === 'string' ? BY_I*/export const FAL_IMAGE_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'i2v'));+/** Models that animate a frame to supplied audio across the whole picture (the music-driven performance lane). */+export const FAL_AUDIO_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'a2v'));+/** Resolution choices a model offers (empty when it has no resolution parameter). */export const falVideoResolutions = (model) => (model?.resolution?.options ? [...model.resolution.options] : []);@@ -215,7 +262,7 @@ export function resolveFalVideoResolution(model, requested) {* shortest supported length that still COVERS it, else the longest — a Music* Video shot must not be under-covered (the renderer refuses a clip more than a* frame short) and nothing loops or stretches footage. An absent / invalid- * request gets the model's default. Null for a lip-sync model, whose length+ * request gets the model's default. Null for a source-audio model, whose length* follows its audio.*/export function coerceFalVideoSeconds(model, seconds) {@@ -235,6 +282,24 @@ const wireDuration = (rule, sec) => (const ratioOf = (r) => { const [w, h] = String(r).split(':').map(Number); return w / h; };+const round32 = (n) => Math.max(32, Math.round(n / 32) * 32);++/**+ * The explicit canvas a `wire: 'canvas'` model renders at `resolution`: the+ * named short side, the long side from the requested width/height (16:9 when+ * none is given), both on the 32-pixel grid the model works in.+ */+function ltxCanvas(resolution, { width, height } = {}) {+ const short = LTX_SHORT_SIDES[resolution];+ if (!short) return null;+ const w = Number(width);+ const h = Number(height);+ const portrait = w > 0 && h > 0 && h > w;+ const aspect = w > 0 && h > 0 ? Math.max(w, h) / Math.min(w, h) : 16 / 9;+ const long = round32(short * aspect);+ return portrait ? { width: short, height: long } : { width: long, height: short };+}+function pickAspectRatio(rule, { aspectRatio, width, height }) {if (!rule) return null;if (rule.options.includes(aspectRatio)) return aspectRatio;@@ -280,6 +345,15 @@ export function estimateFalVideoCostUsd({ modelId, seconds, resolution, width, h}const res = resolveFalVideoResolution(model, resolution);let perSec;+ if (model.pricing.perMegapixel) {+ const canvas = ltxCanvas(res, { width, height });+ if (!canvas) return null;+ const fps = model.resolution.fps;+ // The frames the audio needs, rounded UP to the 8n+1 counts LTX renders,+ // so the estimate never undershoots the bill.+ const frames = Math.ceil(Math.ceil(billedSec * fps) / 8) * 8 + 1;+ return round4((canvas.width * canvas.height * frames / 1e6) * model.pricing.perMegapixel);+ }if (model.pricing.videoTokensPerSec16x9) {perSec = model.pricing.videoTokensPerSec16x9[res] ?? null;const w = Number(width);@@ -299,6 +373,12 @@ export function describeFalVideoRate(modelId, resolution) {const model = getFalVideoModel(modelId);if (!model) return null;const res = resolveFalVideoResolution(model, resolution);+ if (model.pricing.perMegapixel) {+ const canvas = ltxCanvas(res);+ if (!canvas) return null;+ const perSec = (canvas.width * canvas.height * model.resolution.fps / 1e6) * model.pricing.perMegapixel;+ return `≈$${perSec.toFixed(4)}/s at ${res} (16:9)`;+ }if (model.pricing.videoTokensPerSec16x9) {const perSec = model.pricing.videoTokensPerSec16x9[res];return perSec == null ? null : `≈$${perSec.toFixed(4)}/s at ${res} (16:9)`;@@ -313,7 +393,7 @@ export function describeFalVideoRate(modelId, resolution) {* Build the fal queue request body for a curated model, or return null for an* uncurated id (the caller keeps its legacy body). Returns `{ body, model,* seconds, resolution, generateAudio, estimatedCostUsd }` — `seconds` is the- * coerced clip length (null for lip-sync, whose length is `audioSec`).+ * coerced clip length (null for a source-audio model, whose length is `audioSec`).** Inputs a model cannot take are dropped rather than sent (fal ignores unknown* keys, which would bill a render that silently disregarded them); a missing@@ -367,7 +447,7 @@ export function buildFalVideoRequest({body[model.referenceAudio.param] = audioRefs;}if (model.audioInput) {- if (!audioUrl) throw refuse(`${model.label} needs the source audio to synchronize to`);+ if (!audioUrl) throw refuse(`${model.label} needs the source audio to ${model.kind === 'a2v' ? 'drive its motion' : 'synchronize to'}`);body[model.audioInput.param] = audioUrl;if (model.transcription && enableTranscription) body[model.transcription] = true;} else if (audioUrl) {@@ -377,7 +457,7 @@ export function buildFalVideoRequest({const coercedSec = coerceFalVideoSeconds(model, seconds);if (coercedSec != null) body.duration = wireDuration(model.duration, coercedSec);const res = resolveFalVideoResolution(model, resolution);- if (res) body[model.resolution.param] = res;+ if (res) body[model.resolution.param] = model.resolution.wire === 'canvas' ? ltxCanvas(res, { width, height }) : res;const aspect = pickAspectRatio(model.aspectRatio, { aspectRatio, width, height });if (aspect) body[model.aspectRatio.param] = aspect;const wantsAudio = generateAudio === true && !!model.audio;diff --git a/server/lib/falVideoModels.test.js b/server/lib/falVideoModels.test.jsindex 69016a21a6..b9ef23cf49 100644--- a/server/lib/falVideoModels.test.js+++ b/server/lib/falVideoModels.test.js@@ -1,5 +1,6 @@import { describe, it, expect } from 'vitest';import {+ FAL_AUDIO_VIDEO_MODEL,FAL_LIPSYNC_VIDEO_MODEL,FAL_VIDEO_MODELS,buildFalVideoRequest,@@ -76,6 +77,28 @@ describe('buildFalVideoRequest — per-family wire bodies', () => {expect(req.estimatedCostUsd).toBe(3.84);});+ it('LTX-2.3 audio-to-video: prompt + frame + audio on an explicit canvas that keeps the shot aspect; length follows the audio', () => {+ const req = buildFalVideoRequest({+ modelId: FAL_AUDIO_VIDEO_MODEL, prompt: 'drummer', negativePrompt: 'blur', imageUrl: 'i', audioUrl: 'a', audioSec: 3.735,+ width: 1920, height: 1080, resolution: '1080P',+ });+ expect(req.body).toEqual({+ prompt: 'drummer', negative_prompt: 'blur', image_url: 'i', audio_url: 'a', video_size: { width: 1920, height: 1080 },+ match_audio_length: true, fps: 24, audio_strength: 1,+ });+ expect(req.seconds).toBeNull();+ // 1920×1080 × 97 frames (3.735s at 24fps, up to LTX's 8n+1 count) at $0.001605/MP.+ expect(req.estimatedCostUsd).toBe(0.3228);+ expect(buildFalVideoRequest({ modelId: FAL_AUDIO_VIDEO_MODEL, prompt: 'x', audioUrl: 'a', width: 1080, height: 1920 }).body.video_size)+ .toEqual({ width: 720, height: 1280 });+ });++ it('OmniHuman v1.5: frame + audio, an optional prompt and its own resolution enum', () => {+ expect(buildFalVideoRequest({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', imageUrl: 'i', audioUrl: 'a', resolution: '720P' }).body)+ .toEqual({ image_url: 'i', audio_url: 'a', resolution: '720p' });+ expect(estimateFalVideoCostUsd({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', seconds: 4 })).toBe(0.64);+ });+it('refuses a request the model cannot honor before anything is paid for; an uncurated id is not built', () => {const refused = (fn) => { try { fn(); } catch (err) { return err.code; } return null; };expect(refused(() => buildFalVideoRequest({ modelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'x' }))).toBe('FAL_MODEL_INPUT');diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.jsindex b8ed5f78a5..d5fef203ef 100644--- a/server/lib/musicVideoDependencies.js+++ b/server/lib/musicVideoDependencies.js@@ -14,7 +14,9 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: srevision: checksum(Object.fromEntries(['startSec', 'endSec', 'visualLayer', 'stillMove', 'cardText', 'cardColor', 'shotMode', 'performanceSpeaker', 'loop'].map((key) => [key, scene[key] ?? null]).concat([['actionContract', scene.direction?.actionContract || null]])// #10589: only a planned camera joins the revision, so pre-camera evidence stays current.- .concat(scene.camera ? [['camera', scene.camera]] : []))) });+ .concat(scene.camera ? [['camera', scene.camera]] : [])+ // Only a music-driven performance joins it, so lip-sync evidence stays current.+ .concat(scene.performanceStyle === 'music' ? [['performanceStyle', 'music']] : []))) });const snapshot = (references) => ({ version: MUSIC_VIDEO_DEPENDENCY_VERSION, references });/** Capture the actual submitted plate, including optional crop/mask provenance. */diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.jsindex 5d2d7b9768..95d483eae0 100644--- a/server/lib/musicVideoShotTiming.js+++ b/server/lib/musicVideoShotTiming.js@@ -15,6 +15,10 @@* it is cutaway-only here until an actual adapter proves exact-source-audio* conditioning — REST documentation for a different Grok surface is not that* proof. Local runtimes are absent for the same reason.+ * - `SOURCE_AUDIO_MUSIC` — the music-driven performance lanes: fal.ai+ * audio-to-video models (kind `a2v`) where the song slice drives the whole+ * picture's motion rather than only a singer's mouth. A scene opts in with+ * `performanceStyle: 'music'`; it shares the window planning below.* - `planPerformanceWindow` — the audio window a performance shot submits:* inside the provider's limits (never relying on its silent truncation), a* short shot padded to a supported contextual window with an explicit edit@@ -38,6 +42,8 @@import { GROK_VIDEO_DURATIONS, nearestGrokDuration } from './grokVideoClip.js';import {+ FAL_AUDIO_VIDEO_MODEL,+ FAL_AUDIO_VIDEO_MODELS,FAL_DEFAULT_IMAGE_VIDEO_MODEL,FAL_LIPSYNC_VIDEO_MODEL,coerceFalVideoSeconds,@@ -58,6 +64,17 @@ export const MUSIC_VIDEO_SHOT_MODES = Object.freeze(['cutaway', 'performance']);export const isPerformanceScene = (scene) => scene?.shotMode === 'performance'&& (scene.visualLayer == null || scene.visualLayer === 'footage');+/**+ * How a performance shot follows the song: `lipsync` (absent on every older+ * record) syncs a singer's mouth to the vocal; `music` lets the song slice+ * drive the whole picture's motion — a drummer hitting on the beat, a dancer,+ * a crowd — on an audio-to-video model (lib/falVideoModels.js kind `a2v`).+ */+export const MUSIC_VIDEO_PERFORMANCE_STYLES = Object.freeze(['lipsync', 'music']);++/** A performance shot a singer lip-syncs (the only kind whose mouth sync is measured). */+export const isLipSyncScene = (scene) => isPerformanceScene(scene) && scene.performanceStyle !== 'music';+/*** The shot instruction of a performance scene's SELECTED video take, or null —* for a cutaway scene (even one whose selected take was once generated as a@@ -100,6 +117,7 @@ export const MUSIC_VIDEO_LIPSYNC_DEFAULT_RESOLUTION = '1080P';export const SOURCE_AUDIO_LIPSYNC = Object.freeze({fal: Object.freeze({provider: 'fal',+ style: 'lipsync',label: 'fal.ai MiniMax H3 lip-sync',modelId: FAL_LIPSYNC_VIDEO_MODEL,minAudioSec: 5,@@ -113,10 +131,51 @@ export const SOURCE_AUDIO_LIPSYNC = Object.freeze({}),});-/** The lip-sync capability a backend offers, or null. */-export const performanceCapability = (backend) => (- Object.prototype.hasOwnProperty.call(SOURCE_AUDIO_LIPSYNC, backend || '') ? SOURCE_AUDIO_LIPSYNC[backend] : null-);+/**+ * Default output resolution of a music-driven take. Both audio-to-video+ * models offer 1080p, and LTX-2.3 bills by pixel at a few cents a second.+ */+const MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION = '1080p';++const musicCapability = (model) => {+ const resolutions = falVideoResolutions(model);+ const defaultResolution = resolutions.includes(MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION) ? MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION : model.resolution.default;+ return Object.freeze({+ provider: 'fal',+ style: 'music',+ label: `fal.ai ${model.label}`,+ modelId: model.id,+ minAudioSec: model.audioInput.minSec,+ // The same one-phrase per-take ceiling as lip-sync (see above).+ maxAudioSec: Math.min(model.audioInput.maxSec, 30),+ transcription: false,+ resolutions: Object.freeze(resolutions),+ defaultResolution,+ costLabel: `${describeFalVideoRate(model.id, defaultResolution)} — billed to your fal.ai account`,+ });+};++/**+ * Verified music-driven lanes, keyed by backend then model id. The project's+ * `videoSettings.falMusicModelId` picks one; absent (or a model this install+ * does not curate) = `FAL_AUDIO_VIDEO_MODEL` (LTX-2.3 audio-to-video).+ */+export const SOURCE_AUDIO_MUSIC = Object.freeze({+ fal: Object.freeze(Object.fromEntries(FAL_AUDIO_VIDEO_MODELS.map((model) => [model.id, musicCapability(model)]))),+});++/**+ * The source-audio capability a backend offers a performance shot, or null.+ * Without a scene (or for a lip-sync scene) it is the lip-sync lane; a+ * music-driven scene gets the audio-to-video model `videoSettings` pins.+ */+export function performanceCapability(backend, { scene = null, videoSettings = null } = {}) {+ if (!Object.prototype.hasOwnProperty.call(SOURCE_AUDIO_LIPSYNC, backend || '')) return null;+ if (scene?.performanceStyle !== 'music') return SOURCE_AUDIO_LIPSYNC[backend];+ const lanes = SOURCE_AUDIO_MUSIC[backend];+ const pinned = videoSettings?.falMusicModelId;+ return (pinned && Object.prototype.hasOwnProperty.call(lanes, pinned) ? lanes[pinned] : null) || lanes[FAL_AUDIO_VIDEO_MODEL];+}/*** Why a performance shot cannot render on `backend`, or null when it can.@@ -259,9 +318,9 @@ export function approximateMotionCues(phrases, startSec, endSec) {* performance shot on a lane that cannot lip-sync at all, which is blocked* outright rather than split — have no limit here.*/-export function shotSplitLimit(scene, backend) {+export function shotSplitLimit(scene, backend, videoSettings = null) {if (isPerformanceScene(scene)) {- const capability = performanceCapability(backend);+ const capability = performanceCapability(backend, { scene, videoSettings });return capability ? round6(capability.maxAudioSec - PERFORMANCE_WINDOW_MARGIN_SEC) : null;}const footage = scene?.visualLayer == null || scene.visualLayer === 'footage';@@ -396,9 +455,10 @@ export function planShotSplit({ startSec, endSec, maxSec, lyricCues, phrases, beexport function falSceneTake({ scene, videoSettings = {}, songDurationSec = null } = {}) {const settings = videoSettings || {};if (isPerformanceScene(scene)) {- const capability = SOURCE_AUDIO_LIPSYNC.fal;+ const capability = performanceCapability('fal', { scene, videoSettings: settings });const model = getFalVideoModel(capability.modelId);- const resolution = resolveFalVideoResolution(model, settings.falLipSyncResolution || capability.defaultResolution);+ const pinned = capability.style === 'music' ? settings.falMusicResolution : settings.falLipSyncResolution;+ const resolution = resolveFalVideoResolution(model, pinned || capability.defaultResolution);const plan = planPerformanceWindow({startSec: scene.startSec, endSec: scene.endSec, songDurationSec: songDurationSec ?? Infinity, capability,});@@ -445,7 +505,7 @@ export const falTakeRequestFields = (take) => (take.performanceexport function planPerformanceRepair({ scene, temporal, excerptStartSec, backend, videoSettings = {} }) {const no = (message) => ({ ok: false, message });const instruction = selectedPerformanceInstruction(scene);- const capability = performanceCapability(backend);+ const capability = performanceCapability(backend, { scene, videoSettings });if (!capability) return no(performanceBlockedReason(backend));if (!instruction || instruction.songInterval?.startSec !== scene.startSec || instruction.songInterval?.endSec !== scene.endSec) return no('Review needed: the selected take no longer matches this shot');if (scene.direction?.actionContract || scene.performanceRepair) return no('Review needed: edit the shot contract or review the existing continuation before repairing again');diff --git a/server/lib/musicVideoShotTiming.test.js b/server/lib/musicVideoShotTiming.test.jsindex 57e2f4f3e5..37ae9f7ef9 100644--- a/server/lib/musicVideoShotTiming.test.js+++ b/server/lib/musicVideoShotTiming.test.js@@ -1,5 +1,7 @@import { describe, it, expect } from 'vitest';-import { planShotSplit, shotSplitLimit, SOURCE_AUDIO_LIPSYNC, PERFORMANCE_WINDOW_MARGIN_SEC } from './musicVideoShotTiming.js';+import {+ falSceneTake, falTakeRequestFields, planShotSplit, shotSplitLimit, SOURCE_AUDIO_LIPSYNC, PERFORMANCE_WINDOW_MARGIN_SEC,+} from './musicVideoShotTiming.js';// The split planner is an algorithm with a real input matrix (#8977): which// boundary wins, how many pieces, and exact coverage are each ambiguous when@@ -87,6 +89,29 @@ describe('shotSplitLimit', () => {// Each failure variant is a distinct admission regression: evidence, word// gap, provider minimum, or timebase mismatch must never resubmit a prefix.+describe('music-driven performance takes', () => {+ // The drummer case: a 3.735s shot of the song whose motion the audio drives.+ const drummer = { shotMode: 'performance', performanceStyle: 'music', startSec: 212.718, endSec: 216.453 };++ it('render on the audio-to-video model for the shot itself, without lip-sync\'s 5s padding', () => {+ const take = falSceneTake({ scene: drummer, videoSettings: {}, songDurationSec: 240 });+ expect(take).toMatchObject({ performance: true, modelId: 'fal-ai/ltx-2.3-22b/audio-to-video', seconds: 3.735, resolution: '1080p' });+ expect(take.costUsd).toBeGreaterThan(0);+ expect(falTakeRequestFields(take)).toEqual({ falResolution: '1080p' });+ // The same shot as lip-sync pads to the route's 5s minimum.+ expect(falSceneTake({ scene: { ...drummer, performanceStyle: null }, songDurationSec: 240 }).seconds).toBeCloseTo(5.05, 6);+ });++ it('follow the project pin (model and resolution) and fall back to the default for an uncurated pin', () => {+ const pinned = falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5', falMusicResolution: '720p' }, songDurationSec: 240 });+ expect(pinned).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '720p', costUsd: 0.5976 });+ expect(falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'example/newer-model' }, songDurationSec: 240 }).modelId)+ .toBe('fal-ai/ltx-2.3-22b/audio-to-video');+ expect(shotSplitLimit(drummer, 'fal')).toBeCloseTo(20 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);+ expect(shotSplitLimit(drummer, 'fal', { falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5' })).toBeCloseTo(30 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);+ });+});+describe('accepted-prefix repair planning', () => {const scene = { shotMode: 'performance', sceneId: 'example', videoHistoryId: 'clip', startSec: 10, endSec: 24,takes: [{ kind: 'video', assetId: 'clip', shotInstruction: { shotMode: 'performance',diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.jsindex 7e8bd5471f..cd2916e776 100644--- a/server/lib/musicVideoValidation.js+++ b/server/lib/musicVideoValidation.js@@ -51,7 +51,7 @@ import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';import { MUSIC_VIDEO_MEDIA } from './musicVideoMediumPlan.js';import { MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';import { CAMERA_FRAMINGS, CAMERA_MOVEMENT_VALUES, CAMERA_SPEEDS } from './cameraMovements.js';-import { MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';+import { MUSIC_VIDEO_PERFORMANCE_STYLES, MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';import {MUSIC_VIDEO_AUTOMATION_BUDGET_MAX_USD,MUSIC_VIDEO_AUTOMATION_GUIDANCE_MAX,@@ -152,6 +152,12 @@ export const musicVideoVideoSettingsSchema = z.object({// false = lip-sync to the audio alone, without the provider transcribing it// first (sung words it mishears turn into the wrong mouth shapes).falLipSyncTranscription: z.boolean().nullable().optional(),+ // Audio-to-video model and resolution for music-driven performance takes+ // (scene.performanceStyle 'music'). Loosely validated like falModelId: a peer+ // on a newer catalog may pin a model this install does not curate, which then+ // falls back to the default (LTX-2.3 audio-to-video).+ falMusicModelId: z.string().min(1).max(200).nullable().optional(),+ falMusicResolution: z.string().min(1).max(16).nullable().optional(),generationMode: z.enum(['image', 'audioReactive', 'suppliedAudio']).optional(),audioReactiveLora: z.string().max(255).regex(/^[^/\\]+\.safetensors$/i).nullable().optional(),audioReactiveScale: z.number().min(0).max(2).optional(),@@ -1349,6 +1355,8 @@ export const musicVideoSceneCreateSchema = z.object({// `cutaway` = the ordinary image-to-video shot. See lib/musicVideoShotTiming.js.shotMode: z.enum(MUSIC_VIDEO_SHOT_MODES).optional(),performanceSpeaker: z.string().trim().max(120).nullable().optional(),+ // How a performance follows the song: lip-sync (absent/null) or music-driven motion.+ performanceStyle: z.enum(MUSIC_VIDEO_PERFORMANCE_STYLES).nullable().optional(),}).strict().refine((s) => s.startSec == null || s.endSec == null || s.endSec >= s.startSec,{ message: 'endSec must be >= startSec', path: ['endSec'] },@@ -1387,6 +1395,8 @@ export const musicVideoSceneUpdateSchema = z.object({// `cutaway` = the ordinary image-to-video shot. See lib/musicVideoShotTiming.js.shotMode: z.enum(MUSIC_VIDEO_SHOT_MODES).optional(),performanceSpeaker: z.string().trim().max(120).nullable().optional(),+ // How a performance follows the song: lip-sync (absent/null) or music-driven motion.+ performanceStyle: z.enum(MUSIC_VIDEO_PERFORMANCE_STYLES).nullable().optional(),referenceImageId: z.string().max(256).nullable().optional(),videoHistoryId: z.string().max(64).nullable().optional(),// #10154: the last generation failure the render hooks persisted. The boarddiff --git a/server/services/musicVideo/autoReviewService.js b/server/services/musicVideo/autoReviewService.jsindex bae968177f..7e5ab94de5 100644--- a/server/services/musicVideo/autoReviewService.js+++ b/server/services/musicVideo/autoReviewService.js@@ -22,7 +22,7 @@* so an event arriving mid-step can never start a second review.*/-import { isPerformanceScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';+import { isLipSyncScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';import { analyzeTemporalPerformance } from './temporalPerformance.js';import { unlink } from 'fs/promises';import { existsSync } from 'fs';@@ -211,11 +211,11 @@ async function reviewDraft(project, run, excerpt) {const shots = sections.flatMap((section) => {const scene = project.scenes?.find((s) => s.sceneId === section.sceneId);const instruction = selectedPerformanceInstruction(scene);- const performance = 'performance' in section ? section.performance : (isPerformanceScene(scene)+ const performance = 'performance' in section ? section.performance : (isLipSyncScene(scene)? { takeId: scene.videoHistoryId, speaker: instruction?.speaker || null, conditioning: instruction?.audio?.conditioning || null } : null);return performance ? [{ sceneId: section.sceneId, ...performance, startSec: section.startSec, endSec: section.endSec }] : [];});- const temporal = !sections.length && project.scenes?.some(isPerformanceScene)+ const temporal = !sections.length && project.scenes?.some(isLipSyncScene)? { version: 1, status: 'unverified', analyzer: null, reason: 'The draft has no performance-section provenance', shots: [] }: await analyzeTemporalPerformance({ excerptPath, shots });const evidence = { boundaryFrames: hasSheet ? 1 : 0, continuousFrames: frameTimes.length, temporal, ...(excerpt.footageVisibility ? { footageVisibility: excerpt.footageVisibility } : {}), excerptStartSec: excerpt.startSec };diff --git a/server/services/musicVideo/excerpt.js b/server/services/musicVideo/excerpt.jsindex 10b810505b..8924976b48 100644--- a/server/services/musicVideo/excerpt.js+++ b/server/services/musicVideo/excerpt.js@@ -22,7 +22,7 @@* `mutateProjectRecord`'s write serialization without duplicating validation.*/-import { isPerformanceScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';+import { isLipSyncScene, selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';import { captureMusicVideoEvidence } from '../../lib/musicVideoDependencies.js';import { randomUUID } from 'crypto';import { ServerError } from '../../lib/errorHandler.js';@@ -81,7 +81,7 @@ export function startExcerptOnProject(project, { startSec, endSec, sections = nusections: Array.isArray(sections) ? sections.map((section) => {const scene = performanceProject.scenes?.find((s) => s.sceneId === section.sceneId);const instruction = selectedPerformanceInstruction(scene);- return { ...section, performance: isPerformanceScene(scene) && (!section.layer || section.layer === 'footage')+ return { ...section, performance: isLipSyncScene(scene) && (!section.layer || section.layer === 'footage')? { takeId: scene.videoHistoryId || null, speaker: instruction?.speaker || null, conditioning: instruction?.audio?.conditioning || null } : null };}) : null,// #9280: a social cut's frame (the project's own unless re-framed) anddiff --git a/server/services/musicVideo/performanceShot.js b/server/services/musicVideo/performanceShot.jsindex eabfd300aa..47c022f99b 100644--- a/server/services/musicVideo/performanceShot.js+++ b/server/services/musicVideo/performanceShot.js@@ -264,8 +264,11 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImagePif (!sourceImagePath || basename(sourceImagePath) !== scene.referenceImageId) throw refuse('Use the accepted boundary frame for this continuation', 'PERFORMANCE_REPAIR_STALE', 409);}- const capability = performanceCapability(backend);+ // A music-driven scene renders on the project's audio-to-video model; a+ // lip-sync scene on the lip-sync route.+ const capability = performanceCapability(backend, { scene, videoSettings: project.videoSettings });if (!capability) throw refuse(performanceBlockedReason(backend), 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');+ const musicDriven = capability.style === 'music';// The lip-sync route is image + source audio; any other semantic mode would// ask it for something it does not do.if (mode && mode !== 'image') {@@ -286,12 +289,16 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP}const offered = getFalVideoModel(capability.modelId)?.resolution?.options || capability.resolutions || [];const pick = (value) => (typeof value === 'string' ? offered.find((o) => o.toLowerCase() === value.trim().toLowerCase()) : null);- const takeResolution = pick(resolution) || pick(project.videoSettings?.falLipSyncResolution) || capability.defaultResolution;+ const pinnedResolution = musicDriven ? project.videoSettings?.falMusicResolution : project.videoSettings?.falLipSyncResolution;+ const takeResolution = pick(resolution) || pick(pinnedResolution) || capability.defaultResolution;// An optional vocal stem conditions the provider in place of the mix. It// is re-checked here because the file could have been replaced on disk or// arrived from a peer since it was attached.- const conditioningSource = project.performanceConditioningSource || (project.vocalStemFilename ? 'vocal-stem' : 'master');+ // A music-driven take always hears the full mix: the drums, bass and+ // everything else a vocal stem leaves out are what drive its motion.+ const conditioningSource = musicDriven ? 'master'+ : project.performanceConditioningSource || (project.vocalStemFilename ? 'vocal-stem' : 'master');const stemPath = conditioningSource === 'master' ? null : resolveVocalStemPath(project);if (conditioningSource !== 'master' && !stemPath) throw refuse('Attach the selected singer stem before rendering', 'MUSIC_VIDEO_VOCAL_STEM_MISSING');if (stemPath) assertVocalStemTimebase(await probeVideoDuration(stemPath), songDurationSec);@@ -330,6 +337,7 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImagePversion: SHOT_INSTRUCTION_VERSION,...(scene.performanceRepair ? { repair: { ...scene.performanceRepair, role: 'continuation' } } : {}),shotMode: 'performance',+ performanceStyle: capability.style,createdAt: new Date().toISOString(),speaker: scene.performanceSpeaker || null,audio: {@@ -343,7 +351,7 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP// and then only when `windowFingerprint` says this window changed.conditioning: {source: conditioningSource, sha256: conditioningSha256, filename: basename(stemPath || masterPath),- selection: project.performanceConditioningSource ? 'user' : 'legacy-default',+ selection: musicDriven ? 'music-driven' : project.performanceConditioningSource ? 'user' : 'legacy-default',voiceIsolation: 'unverified',},},@@ -355,6 +363,7 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImagePperformance: (scene.visualIntent || scene.prompt || '').slice(0, 2000),capability: {provider: capability.provider,+ style: capability.style,modelId: capability.modelId,minAudioSec: capability.minAudioSec,maxAudioSec: capability.maxAudioSec,diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.jsindex 3439bbf33f..cc20b791db 100644--- a/server/services/musicVideo/performanceShot.test.js+++ b/server/services/musicVideo/performanceShot.test.js@@ -298,6 +298,73 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-syncexpect(existsSync(PATHS.uploads) ? await readdir(PATHS.uploads) : []).toEqual([]);});+ it('a music-driven shot hears the full mix for exactly its span and submits frame + prompt + audio to the audio-to-video model', async () => {+ await writeFile(join(PATHS.music, 'vocals.wav'), songWav({ markerAmp: 12000 }));+ // A vocal stem is attached, but the drums it leaves out are what drive this shot.+ getProject.mockResolvedValue({ ...project({ performanceStyle: 'music', prompt: 'drummer hits on the beat' }), vocalStemFilename: 'vocals.wav',+ videoSettings: { backend: 'fal', falLipSyncTranscription: true } });+ const prepared = await preparePerformanceShot({+ musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'frame.png'), mode: 'image',+ });+ const { shotInstruction: si } = prepared;+ // 1.5s clears the model's 1s minimum: no contextual padding, in-point 0.+ expect(si.audioWindow).toEqual({ startSec: 20, endSec: 21.5, durationSec: 1.5 });+ expect(si.edit).toEqual({ inSec: 0, outSec: 1.5, targetSec: 1.5 });+ expect(si).toMatchObject({ performanceStyle: 'music', transcription: false,+ capability: { style: 'music', modelId: 'fal-ai/ltx-2.3-22b/audio-to-video', resolution: '1080p' } });+ expect(si.audio.conditioning).toMatchObject({ source: 'master', filename: 'song.wav', selection: 'music-driven' });+ expect(prepared).toMatchObject({ modelId: 'fal-ai/ltx-2.3-22b/audio-to-video', resolution: '1080p', enableTranscription: false });+ const samples = pcm(await readFile(prepared.audioFilePath));+ expect(samples.readInt16LE(0)).toBe(30000);++ await mkdir(PATHS.images, { recursive: true });+ await writeFile(join(PATHS.images, 'frame.png'), Buffer.from('89504e470d0a1a0a', 'hex'));+ const deliveredPath = join(PATHS.data, 'delivered-music.mp4');+ execFileSync(ffmpeg, ['-v', 'error', '-f', 'lavfi', '-i', 'color=blue:s=64x64:r=24:d=1.5', '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-y', deliveredPath]);+ const delivered = await readFile(deliveredPath);+ const posts = [];+ vi.stubGlobal('fetch', vi.fn(async (url, opts) => {+ if (opts?.method === 'POST') {+ posts.push({ url, body: JSON.parse(opts.body) });+ return jsonResponse({ request_id: 'r2', status_url: 'https://queue.fal.run/s2', response_url: 'https://queue.fal.run/r2' });+ }+ if (url === 'https://queue.fal.run/s2') return jsonResponse({ status: 'COMPLETED' });+ if (url === 'https://queue.fal.run/r2') return jsonResponse({ video: { url: 'https://cdn.example.com/music.mp4' } });+ if (url === 'https://cdn.example.com/music.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(delivered).buffer };+ throw new Error(`unexpected fetch: ${url}`);+ }));+ const job = await fal.generateVideo({+ apiKey: 'test-key', modelId: prepared.modelId, resolution: prepared.resolution, prompt: 'drummer hits on the beat', width: 1920, height: 1080,+ sourceImagePath: join(PATHS.images, 'frame.png'), audioFilePath: prepared.audioFilePath,+ lipSync: { enableTranscription: prepared.enableTranscription }, shotInstruction: si,+ });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });+ expect(posts).toHaveLength(1);+ expect(posts[0].url).toBe('https://queue.fal.run/fal-ai/ltx-2.3-22b/audio-to-video');+ expect(posts[0].body).toMatchObject({+ prompt: 'drummer hits on the beat', video_size: { width: 1920, height: 1080 }, match_audio_length: true, fps: 24, audio_strength: 1,+ });+ expect(posts[0].body.image_url).toMatch(/^data:image\//);+ expect(posts[0].body.audio_url).toMatch(/^data:audio\/wav;base64,/);+ const { loadHistory } = await import('../videoGen/history.js');+ // 1920×1080 × 41 frames (1.5s at 24fps, up to LTX's 8n+1 count) at $0.001605/MP.+ expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({+ modelId: 'fal:fal-ai/ltx-2.3-22b/audio-to-video', resolution: '1080p', estimatedCostUsd: 0.1365, audioDriven: true,+ });++ // With no requested size the canvas follows the start frame's shape (a portrait plate stays portrait).+ execFileSync(ffmpeg, ['-v', 'error', '-f', 'lavfi', '-i', 'color=gray:s=540x960', '-frames:v', '1', '-y', join(PATHS.images, 'portrait.png')]);+ const again = await preparePerformanceShot({+ musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'portrait.png'), mode: 'image',+ });+ const portraitJob = await fal.generateVideo({+ apiKey: 'test-key', modelId: again.modelId, resolution: again.resolution, prompt: 'drummer hits on the beat',+ sourceImagePath: join(PATHS.images, 'portrait.png'), audioFilePath: again.audioFilePath, shotInstruction: again.shotInstruction,+ });+ expect(await waitForTerminal(portraitJob.jobId)).toMatchObject({ type: 'completed' });+ expect(posts[1].body.video_size).toEqual({ width: 1080, height: 1920 });+ }, 60_000);+it('uses the full span of a long shot in one take, at the project\'s lip-sync resolution pin', async () => {// 26s: past the old 14.8s cap (fal accepts up to 15 minutes; PortOS keeps// a take to 30s so it stays one editable phrase).diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.jsindex 5f0ed0ef95..8e61395f67 100644--- a/server/services/musicVideo/projectsLogic.js+++ b/server/services/musicVideo/projectsLogic.js@@ -131,6 +131,8 @@ export function buildProjectRecord(input, { id, now }) {falModelId: videoSettings.falModelId ?? null,falResolution: videoSettings.falResolution ?? null,falLipSyncResolution: videoSettings.falLipSyncResolution ?? null,+ falMusicModelId: videoSettings.falMusicModelId ?? null,+ falMusicResolution: videoSettings.falMusicResolution ?? null,generationMode: videoSettings.generationMode ?? 'image',audioReactiveLora: videoSettings.audioReactiveLora ?? null,audioReactiveScale: videoSettings.audioReactiveScale ?? 1.2,@@ -383,6 +385,8 @@ export function applyProjectPatch(project, patch) {falModelId: null,falResolution: null,falLipSyncResolution: null,+ falMusicModelId: null,+ falMusicResolution: null,generationMode: 'image',audioReactiveLora: null,audioReactiveScale: 1.2,@@ -497,6 +501,7 @@ function buildScene(input, { order }) {// lip-synced performance shot, which only a source-audio provider renders.shotMode: input.shotMode ?? 'cutaway',...(input.performanceSpeaker ? { performanceSpeaker: input.performanceSpeaker } : {}),+ ...(input.performanceStyle === 'music' ? { performanceStyle: 'music' } : {}),referenceImageId: null,videoHistoryId: null,// #8965 — immutable candidate takes; the two slot fields above are the@@ -698,7 +703,7 @@ export function splitScene(project, sceneId, { backend = null } = {}) {if (idx < 0) throw new ServerError('Scene not found', { status: 404, code: 'NOT_FOUND' });const scene = scenes[idx];const lane = backend || project.videoSettings?.backend || null;- const maxSec = shotSplitLimit(scene, lane);+ const maxSec = shotSplitLimit(scene, lane, project.videoSettings);if (maxSec == null) {throw new ServerError('This backend renders the shot at any length — there is nothing to split.', { status: 400, code: 'MUSIC_VIDEO_SPLIT_NOT_NEEDED' });}@@ -747,6 +752,7 @@ export function splitScene(project, sceneId, { backend = null } = {}) {visualLayer: scene.visualLayer ?? 'footage',shotMode: scene.shotMode ?? 'cutaway',performanceSpeaker: scene.performanceSpeaker ?? null,+ performanceStyle: scene.performanceStyle ?? null,...(scene.camera ? { camera: scene.camera } : {}),...timing,}),@@ -796,7 +802,7 @@ export function mergeNextScene(project, sceneId, { backend = null } = {}) {if (timedSec(startSec) && timedSec(endSec) && (endSec <= startSec || (timedSec(scene.endSec) && endSec < scene.endSec))) {throw new ServerError('The next scene ends before this one does — put the scenes in song order before merging them.', { status: 400, code: 'MUSIC_VIDEO_MERGE_OUT_OF_ORDER' });}- const maxSec = shotSplitLimit(scene, lane);+ const maxSec = shotSplitLimit(scene, lane, project.videoSettings);if (maxSec != null) {if (!timedSec(startSec) || !timedSec(endSec)) {throw new ServerError('Both scenes need a start and end time before they can be merged on this backend.', { status: 400, code: 'MUSIC_VIDEO_MERGE_UNTIMED' });diff --git a/server/services/musicVideo/projectsLogic.test.js b/server/services/musicVideo/projectsLogic.test.jsindex 314178e483..1a0a4915eb 100644--- a/server/services/musicVideo/projectsLogic.test.js+++ b/server/services/musicVideo/projectsLogic.test.js@@ -393,6 +393,8 @@ describe('applyProjectPatch', () => {falModelId: null,falResolution: null,falLipSyncResolution: null,+ falMusicModelId: null,+ falMusicResolution: null,generationMode: 'image',audioReactiveLora: null,audioReactiveScale: 1.2,@@ -409,6 +411,8 @@ describe('applyProjectPatch', () => {falModelId: null,falResolution: null,falLipSyncResolution: null,+ falMusicModelId: null,+ falMusicResolution: null,generationMode: 'image',audioReactiveLora: null,audioReactiveScale: 1.2,diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.jsindex 83f6ad3d93..5ba979669d 100644--- a/server/services/videoGen/fal.js+++ b/server/services/videoGen/fal.js@@ -159,10 +159,10 @@ export async function generateVideo({if (promptRequired && !prompt?.trim()) {throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });}- // A lip-sync render conditions a reference frame on source audio; either- // input alone is not a request this route can honor.+ // A lip-sync or audio-to-video render conditions a reference frame on+ // source audio; either input alone is not a request this route can honor.if (audioFilePath && !sourceImagePath) {- throw new ServerError('A fal.ai lip-sync render needs a reference frame', { status: 400, code: 'VALIDATION_ERROR' });+ throw new ServerError('A fal.ai source-audio render needs a reference frame', { status: 400, code: 'VALIDATION_ERROR' });}// The queue only persists the resolved provider CONFIG a job needs (mirrors// grokPath on the grok lane) — never a secret — so the key is re-resolved@@ -184,13 +184,20 @@ export async function generateVideo({// only — a curated model resolves its own aspect rule.const effectiveAspectRatio = FAL_ASPECT_RATIOS.includes(aspectRatio) ? aspectRatio : deriveAspectRatio(width, height);- // A lip-sync take is billed on its audio length: the shot instruction's+ // A source-audio take is billed on its audio length: the shot instruction's// planned window, else the staged slice itself.const audioSec = audioFilePath? (shotInstruction?.audioWindow?.durationSec || await probeVideoDuration(audioFilePath).catch(() => null)): null;+ // A model that takes an explicit canvas (LTX-2.3) keeps the shot's shape:+ // with no requested size, read it off the start frame rather than assume 16:9.+ let canvas = { width, height };+ if (catalogModel?.resolution?.wire === 'canvas' && !(Number(width) > 0 && Number(height) > 0) && sourceImagePath) {+ const { readImageSize } = await import('../../lib/reactorStartingFrame.js');+ canvas = (await readImageSize(sourceImagePath)) || canvas;+ }const requestSpec = {- modelId, prompt, negativePrompt, seconds: duration, audioSec, aspectRatio, width, height, resolution,+ modelId, prompt, negativePrompt, seconds: duration, audioSec, aspectRatio, width: canvas.width, height: canvas.height, resolution,generateAudio: generateAudio === true, enableTranscription: lipSync?.enableTranscription === true,};// Dry-run the curated body with placeholder media before anything is read or@@ -235,14 +242,15 @@ export async function generateVideo({filename,createdAt: new Date().toISOString(),mode: sourceImagePath ? 'image' : 'text',- ...(audioFilePath ? { lipSync: true } : {}),+ // A lip-sync take syncs a mouth; an audio-to-video take moves the whole frame to the audio.+ ...(audioFilePath ? (catalogModel?.kind === 'a2v' ? { audioDriven: true } : { lipSync: true }) : {}),...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),};const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };jobs.set(jobId, job);const costNote = meta.estimatedCostUsd != null ? ` ~$${meta.estimatedCostUsd.toFixed(2)}` : ' cost unknown';- console.log(`🎬 Generating video [${jobId.slice(0, 8)}] fal (${modelId}${meta.resolution ? ` ${meta.resolution}` : ''}${costNote}): ${(prompt || '(lip-sync)').slice(0, 60)}…`);+ console.log(`🎬 Generating video [${jobId.slice(0, 8)}] fal (${modelId}${meta.resolution ? ` ${meta.resolution}` : ''}${costNote}): ${(prompt || '(source audio)').slice(0, 60)}…`);videoGenEvents.emit('started', { generationId: jobId, totalSteps: 1, ...meta });activeJobs.set(jobId, { ...meta, generationId: jobId, totalSteps: 1, step: 0, progress: 0 });emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Submitting to fal.ai…');@@ -342,7 +350,7 @@ async function runFalVideo(job, jobId, {const measured = (await probeVideoGeometry(outputPath)) || {};const expectedSec = meta.audioWindow?.durationSec;if (expectedSec && measured.durationSec && measured.durationSec + 0.1 < expectedSec) {- console.warn(`⚠️ fal lip-sync [${jobId.slice(0, 8)}] delivered ${measured.durationSec.toFixed(2)}s for a ${expectedSec.toFixed(2)}s audio window`);+ console.warn(`⚠️ fal source-audio take [${jobId.slice(0, 8)}] delivered ${measured.durationSec.toFixed(2)}s for a ${expectedSec.toFixed(2)}s audio window`);}activeRequests.delete(jobId);activeJobs.delete(jobId);diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.jsindex fcda7b05bc..8037900773 100644--- a/server/services/videoGen/prepareParams.js+++ b/server/services/videoGen/prepareParams.js@@ -530,13 +530,14 @@ async function resolvePreparedParams({{ status: 400, code: 'VIDEO_GEN_AUDIO_REQUIRED' },);}- // A fal lip-sync model takes the frame plus an uploaded voice clip (no Music Video scene needed).- const falLipSync = body.backend === VIDEO_GEN_MODE.FAL && getFalVideoModel(body.falModelId)?.kind === 'lipsync';- if (falLipSync && !uploads.audioFile && !body.musicVideo) {+ // A fal source-audio model (lip-sync, or audio-to-video motion) takes the frame plus an uploaded+ // audio clip (no Music Video scene needed).+ const falSourceAudio = body.backend === VIDEO_GEN_MODE.FAL && !!getFalVideoModel(body.falModelId)?.audioInput;+ if (falSourceAudio && !uploads.audioFile && !body.musicVideo) {await cleanupStaged();- throw new ServerError('A fal.ai lip-sync model needs an audioFile upload (the voice to sync to).', { status: 400, code: 'VIDEO_GEN_AUDIO_REQUIRED' });+ throw new ServerError('A fal.ai source-audio model needs an audioFile upload (the audio it follows).', { status: 400, code: 'VIDEO_GEN_AUDIO_REQUIRED' });}- if (uploads.audioFile && body.mode !== 'a2v' && !falLipSync) {+ if (uploads.audioFile && body.mode !== 'a2v' && !falSourceAudio) {await cleanupStaged();throw new ServerError(`audioFile upload is only valid with mode='a2v' (got mode='${body.mode || 'unset'}').`,@@ -835,14 +836,14 @@ async function resolvePreparedParams({}}}- // A standalone fal lip-sync take: stage the voice clip durably like the frame. The queue removes+ // A standalone fal source-audio take: stage the audio clip durably like the frame. The queue removes// it if the job fails, is cancelled or is lost to a restart (uploadedTempPaths); the fal worker// removes it once the take is published.let hostedAudioPath = null;- if (falLipSync && uploads.audioFile) {+ if (falSourceAudio && uploads.audioFile) {if (!sourceImagePath) {await cleanupStaged();- throw new ServerError('A fal.ai lip-sync render needs a reference frame (sourceImage or sourceImageFile).', { status: 400, code: 'VALIDATION_ERROR' });+ throw new ServerError('A fal.ai source-audio render needs a reference frame (sourceImage or sourceImageFile).', { status: 400, code: 'VALIDATION_ERROR' });}hostedAudioPath = await stageUploadDurable(uploads.audioFile, 'audio');// A paid route: refuse a clip fal would reject (unreadable, or outside the model's window)@@ -856,7 +857,7 @@ async function resolvePreparedParams({}if ((window.minSec != null && clipSec < window.minSec) || (window.maxSec != null && clipSec > window.maxSec)) {await cleanupStaged();- throw new ServerError(`The voice clip is ${clipSec.toFixed(2)} s; this lip-sync model takes ${window.minSec}–${window.maxSec} s (pad a short line with silence).`,+ throw new ServerError(`The voice clip is ${clipSec.toFixed(2)} s; this model takes ${window.minSec}–${window.maxSec} s (pad a short line with silence).`,{ status: 400, code: 'VIDEO_GEN_AUDIO_LENGTH' });}}diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex 974a9b076b..f83c030092 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -300,7 +300,11 @@ const submitValidatedVideoGenJob = async (body, uploads) => {const { falSceneTake } = await import('../../lib/musicVideoShotTiming.js');const { costUsd } = falSceneTake({scene,- videoSettings: { falModelId: body.falModelId, falDuration: body.falDuration, falResolution: body.falResolution, falLipSyncResolution: body.falResolution },+ videoSettings: {+ falMusicModelId: project.videoSettings?.falMusicModelId ?? null,+ falModelId: body.falModelId, falDuration: body.falDuration, falResolution: body.falResolution,+ falLipSyncResolution: body.falResolution, falMusicResolution: body.falResolution,+ },songDurationSec: project.audioAnalysis?.durationSec ?? null,});if (Number.isFinite(costUsd)) params.musicVideoCostUsd = costUsd;