Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Music-driven shots on fal hear the song as an audio reference (H3 Max, Seedance 2.5)
Claudeb8d665b8f56b
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsxindex 62bc449864..c8306da241 100644--- a/client/src/components/musicVideo/SceneCard.test.jsx+++ b/client/src/components/musicVideo/SceneCard.test.jsx@@ -55,7 +55,7 @@ describe('performance speaker save boundary', () => {});describe('music-driven performance', () => {- it('switches a shot between cutaway, lip-sync and music-driven, and plans the music-driven take on the audio-to-video model', () => {+ it('switches a shot between cutaway, lip-sync and music-driven, and plans the music-driven take on the default music-driven model', () => {const onSave = vi.fn();function Harness() {const [scene, setScene] = useState({ sceneId: 's1', shotMode: 'cutaway', startSec: 10, endSec: 13.5, referenceImageId: 'frame.png', takes: [] });@@ -69,8 +69,8 @@ describe('music-driven performance', () => {fireEvent.change(shot, { target: { value: 'music' } });expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: 'music' });expect(shot.value).toBe('music');- // The shot's own 3.5s, not lip-sync's 5s minimum window.- expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Music-driven via fal\.ai OmniHuman v1\.5 \(audio-driven person\) \(fal-ai\/bytedance\/omnihuman\/v1\.5\) · song 0:10–0:13 · est\. \$0\.56 \(3\.5s at 1080p\)/);+ // H3 Max reference by default: the song window pads to exactly its 5s shortest billed clip.+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Music-driven via fal\.ai MiniMax H3 Max \(reference\) \(minimax\/h3-max\/reference-to-video\) · song 0:09–0:14 · shot starts 0\.75s into the take · est\. \$0\.40 \(5s at 768P\)/);fireEvent.change(shot, { target: { value: 'performance' } });expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: null });expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Lip-sync via/);diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsxindex 5d58f698b4..0dd28931b1 100644--- a/client/src/components/musicVideo/SceneRenderInputs.jsx+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx@@ -1,4 +1,4 @@-import { getFalVideoModel } from '../../lib/falVideoModels.js';+import { falHearsAudioReference, getFalVideoModel } from '../../lib/falVideoModels.js';import { falSceneTake } from '../../lib/musicVideoShotTiming.js';import { shotActionContractProblem } from '../../../../server/lib/musicVideoActionContract.js';import { authoredShotPrompt, musicVideoShotModelInput } from '../../../../server/lib/musicVideoShotPrompt.js';@@ -55,6 +55,7 @@ export function shotCardRenderRequest({const authored = authoredShotPrompt(scene);const { prompt, promptAdditions } = musicVideoShotModelInput(contractProblem ? null : project, scene, {authored, asTyped: true, hears, backend: settings.backend, audioReactive,+ audioReferenceModelId: performance && falHearsAudioReference(falModel) ? falModel.id : null,});let endFrame = null;diff --git a/client/src/components/musicVideo/SceneRenderInputs.test.jsx b/client/src/components/musicVideo/SceneRenderInputs.test.jsxindex f4f268673d..45273faa9e 100644--- a/client/src/components/musicVideo/SceneRenderInputs.test.jsx+++ b/client/src/components/musicVideo/SceneRenderInputs.test.jsx@@ -48,6 +48,26 @@ describe('shotCardRenderRequest', () => {});});+// The 80s drummer shot: a music-driven take hears the song slice as a reference its prompt names.+describe('music-driven shot on a reference model', () => {+ const drummer = scene({ shotMode: 'performance', performanceStyle: 'music', prompt: 'The drummer hits the snare' });+ it.each([+ [{}, 'Audio 1 is the song playing under this shot: time the movement to its beat.'],+ [{ falMusicModelId: 'bytedance/seedance-2.5/reference-to-video' },+ '@Image1 is the opening frame. @Audio1 is the song playing under this shot: time the movement to its beat.'],+ ])('lists the song audio reference as a chip, sent after the typed prompt (%o)', (pin, text) => {+ const request = shotCardRenderRequest({ scene: drummer, performance: true, settings: { backend: 'fal', ...pin } });+ expect(request.songAudio.state).toBe('required');+ expect(request.promptAdditions).toContainEqual({ id: 'audio-reference', label: 'Song audio reference', text });+ expect(request.prompt).toBe(`The drummer hits the snare. ${text}`);+ });++ it('adds nothing for an audio-to-video model that hears the song as its source audio', () => {+ const request = shotCardRenderRequest({ scene: drummer, performance: true, settings: { backend: 'fal', falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5' } });+ expect(request.promptAdditions.map((item) => item.id)).not.toContain('audio-reference');+ });+});+describe('SceneRenderInputs', () => {it('shows the exact prompt and reports a toggle', () => {const onChange = vi.fn();diff --git a/client/src/components/musicVideo/SceneRenderTarget.jsx b/client/src/components/musicVideo/SceneRenderTarget.jsxindex 331173ff0c..9ccc710acf 100644--- a/client/src/components/musicVideo/SceneRenderTarget.jsx+++ b/client/src/components/musicVideo/SceneRenderTarget.jsx@@ -2,7 +2,7 @@ import {FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, coerceFalVideoSeconds, describeFalVideoRate, falVideoResolutions, getFalVideoModel,} from '../../lib/falVideoModels.js';import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';-import { grokCoverage, performanceCapability } from '../../lib/musicVideoShotTiming.js';+import { SOURCE_AUDIO_MUSIC, grokCoverage, performanceCapability } from '../../lib/musicVideoShotTiming.js';import { localMusicVideoModels } from '../../../../server/lib/musicVideoLocalModels.js';import { frameOptionsForModel } from '../../lib/videoGenParams.js';import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage.js';@@ -10,9 +10,10 @@ import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage/*** Where one shot's clip renders, picked on its card: the project's saved* renderer, local LTX, Grok, or a fal.ai model. LTX is local only, so fal- * offers only its curated start-frame models; a lip-synced or music-driven- * shot renders on fal's source-audio route, so fal is one choice there, and- * Grok (no audio) renders cutaways only.+ * offers only its curated start-frame models; a lip-synced shot renders on+ * fal's lip-sync route, so fal is one choice there, a music-driven shot offers+ * each music-driven model (the song slice drives the picture), and Grok (no+ * audio) renders cutaways only.* The pick is a value string: '' (saved), 'local', 'grok', 'fal' or 'fal:<modelId>'.** A picked lane shows its own options, from the same definitions the Video Gen@@ -50,7 +51,14 @@ export function renderTargetOptions({ performance, saved, endFrame = false, scen{ value: 'local', label: performance ? 'Local LTX lip-sync (free)' : 'Local LTX (free)' },{ value: 'grok', label: performance ? 'Grok (cutaways only)' : 'Grok', disabled: performance },...(performance- ? [{ value: 'fal', label: `fal.ai · ${falPerformanceLabel(saved, scene)}` }]+ ? [+ { value: 'fal', label: `fal.ai · ${falPerformanceLabel(saved, scene)}` },+ // A music-driven shot can try another model that hears the song, priced per second.+ ...(scene?.performanceStyle === 'music' ? Object.values(SOURCE_AUDIO_MUSIC.fal).map((lane) => {+ const rate = describeFalVideoRate(lane.modelId, lane.defaultResolution);+ return { value: `fal:${lane.modelId}`, label: `fal.ai · ${getFalVideoModel(lane.modelId).label} + song audio${rate ? ` · ${rate}` : ''}` };+ }) : []),+ ]: FAL_IMAGE_VIDEO_MODELS.map((model) => {const rate = describeFalVideoRate(model.id);return { value: `fal:${model.id}`, label: `fal.ai · ${model.label}${rate ? ` · ${rate}` : ''}${endFrame && model.endImage ? ' · ends on end frame' : ''}` };@@ -141,6 +149,10 @@ export function renderTargetOverrides(value, saved, options = {}, context = {})}const falModelId = value.slice(4) || null;if (!falModelId) return { backend: 'fal' };+ // A music-driven shot's pick is its music model, at that model's own default size.+ if (context.performance) {+ return { backend: 'fal', falMusicModelId: falModelId, ...(falModelId === saved.falMusicModelId ? {} : { falMusicResolution: null }) };+ }// Each model has its own resolutions, so the saved one only carries over to the saved model.const sameModel = falModelId === (saved.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL);// A picked model is the one this render uses, end frame or not.diff --git a/client/src/components/musicVideo/SceneRenderTarget.test.jsx b/client/src/components/musicVideo/SceneRenderTarget.test.jsxindex 39776deb1c..c5a611e14c 100644--- a/client/src/components/musicVideo/SceneRenderTarget.test.jsx+++ b/client/src/components/musicVideo/SceneRenderTarget.test.jsx@@ -1,6 +1,6 @@import { beforeEach, describe, expect, it } from 'vitest';import { render, screen, within } from '@testing-library/react';-import SceneRenderTarget, { lastRenderTarget, rememberRenderTarget, renderTargetOverrides } from './SceneRenderTarget.jsx';+import SceneRenderTarget, { lastRenderTarget, rememberRenderTarget, renderTargetOptions, renderTargetOverrides } from './SceneRenderTarget.jsx';import { FAL_IMAGE_VIDEO_MODELS, falVideoResolutions } from '../../lib/falVideoModels.js';const models = [@@ -43,3 +43,21 @@ describe('a remembered Render on pick', () => {expect(lastRenderTarget({ performance: false, saved: {} }).options).toEqual({});});});++// Uniquely catches a music-driven shot losing its per-shot choice of the models that hear the song.+describe('a music-driven shot on fal', () => {+ it('offers each music-driven model with its rate, and the pick becomes that take\'s music model', () => {+ const scene = { sceneId: 's1', shotMode: 'performance', performanceStyle: 'music', startSec: 0, endSec: 2 };+ const fal = renderTargetOptions({ performance: true, saved: { backend: 'fal' }, scene }).filter((option) => option.value.startsWith('fal'));+ expect(fal[0].label).toBe('fal.ai · MiniMax H3 Max (reference)');+ expect(fal.map((option) => option.value)).toEqual(expect.arrayContaining([+ 'fal:minimax/h3-max/reference-to-video', 'fal:bytedance/seedance-2.5/reference-to-video',+ ]));+ expect(fal.find((option) => option.value === 'fal:minimax/h3-max/reference-to-video').label).toMatch(/\+ song audio · \$0\.08\/s at 768P/);+ // A lip-synced shot keeps its one lip-sync choice.+ expect(renderTargetOptions({ performance: true, saved: {}, scene: { ...scene, performanceStyle: null } })+ .filter((option) => option.value.startsWith('fal'))).toHaveLength(1);+ expect(renderTargetOverrides('fal:bytedance/seedance-2.5/reference-to-video', { falMusicResolution: '768P' }, {}, { performance: true, scene }))+ .toEqual({ backend: 'fal', falMusicModelId: 'bytedance/seedance-2.5/reference-to-video', falMusicResolution: null });+ });+});diff --git a/client/src/components/musicVideo/VideoRenderSettings.jsx b/client/src/components/musicVideo/VideoRenderSettings.jsxindex a12a00dd59..5c9d14bf6d 100644--- a/client/src/components/musicVideo/VideoRenderSettings.jsx+++ b/client/src/components/musicVideo/VideoRenderSettings.jsx@@ -1,7 +1,7 @@import RenderPoolPicker from './RenderPoolPicker.jsx';import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';import {- FAL_AUDIO_VIDEO_MODEL, FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falImageVideoQuotes, falVideoResolutions, getFalVideoModel,+ FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_MUSIC_VIDEO_DEFAULT_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falImageVideoQuotes, falVideoResolutions, getFalVideoModel,} from '../../lib/falVideoModels.js';import { projectFalSpend } from '../../lib/musicVideoStages.js';import { formatUsd } from '../../utils/formatters.js';@@ -279,8 +279,8 @@ export default function VideoRenderSettings({ videoSettings, generating, projectclassName={`${falSelectCls} sm:max-w-[280px]`}>{musicModels.map((lane) => (- <option key={lane.modelId} value={lane.modelId === FAL_AUDIO_VIDEO_MODEL ? '' : lane.modelId}>- {`Music-driven · ${getFalVideoModel(lane.modelId).label} · ${describeFalVideoRate(lane.modelId, lane.defaultResolution)}${lane.modelId === FAL_AUDIO_VIDEO_MODEL ? ' (default)' : ''}`}+ <option key={lane.modelId} value={lane.modelId === FAL_MUSIC_VIDEO_DEFAULT_MODEL ? '' : lane.modelId}>+ {`Music-driven · ${getFalVideoModel(lane.modelId).label} · ${describeFalVideoRate(lane.modelId, lane.defaultResolution)}${lane.modelId === FAL_MUSIC_VIDEO_DEFAULT_MODEL ? ' (default)' : ''}`}</option>))}</select>diff --git a/client/src/hooks/useMusicVideoModelSettings.js b/client/src/hooks/useMusicVideoModelSettings.jsindex d9cc8a6f3c..1a36a6986f 100644--- a/client/src/hooks/useMusicVideoModelSettings.js+++ b/client/src/hooks/useMusicVideoModelSettings.js@@ -79,7 +79,7 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =// false = lip-sync without transcript guidance (null/absent = on).falLipSyncTranscription: project?.videoSettings?.falLipSyncTranscription ?? null,// Audio-to-video model / resolution for music-driven performance takes —- // null means FAL_AUDIO_VIDEO_MODEL (LTX-2.3) / 1080p, resolved by falSceneTake.+ // null means FAL_MUSIC_VIDEO_DEFAULT_MODEL (MiniMax H3 Max reference) at its own default size, resolved by falSceneTake.falMusicModelId: project?.videoSettings?.falMusicModelId ?? null,falMusicResolution: project?.videoSettings?.falMusicResolution ?? null,generationMode: project?.videoSettings?.generationMode || 'image',diff --git a/client/src/lib/falVideoModels.js b/client/src/lib/falVideoModels.jsindex def3d7e0e3..c55103ffc3 100644--- a/client/src/lib/falVideoModels.js+++ b/client/src/lib/falVideoModels.js@@ -11,10 +11,12 @@ export {FAL_DEFAULT_IMAGE_VIDEO_MODEL,FAL_DEFAULT_TEXT_VIDEO_MODEL,FAL_IMAGE_VIDEO_MODELS,+ FAL_MUSIC_VIDEO_DEFAULT_MODEL,FAL_VIDEO_MODELS,coerceFalVideoSeconds,describeFalVideoRate,estimateFalVideoCostUsd,+ falHearsAudioReference,falImageVideoQuotes,falVideoClipQuote,falVideoResolutions,diff --git a/server/lib/README.md b/server/lib/README.mdindex b03c38ad16..1c1e504977 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -326,7 +326,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `cursor.js` | Cursor Agent (`cursor-agent`) provider helpers — the `CURSOR_COMMAND` binary constant, the `isCursorCommand` predicate, and the `ensureCursorHeadlessArgs` (`--print --force`, folding any pinned reasoning effort into `--model`) / `ensureCursorTuiArgs` (`--force`) argv builders. `--force` is load-bearing beyond approvals: it also clears cursor's workspace-trust gate, which otherwise EXITS a headless run before any work happens. The prompt rides raw stdin (like claude/codex, unlike grok/kimi), and cursor needs no configured-default sentinel — its `auto` router is a real model id passed straight to `--model`. || `grok.js` | xAI Grok Build (`grok`) provider helpers — id/endpoint constants (`GROK_API_ID`/`GROK_CLI_ID`/`GROK_TUI_ID`/`GROK_API_ENDPOINT`), `isGrokCommand`/`isGrokCliProvider`/`isGrokTuiProvider` predicates, `ensureGrokHeadlessArgs`/`ensureGrokTuiArgs` argv builders (grok reads its prompt from `--prompt-file /dev/stdin`, not raw stdin; model selection uses the `GROK_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Antigravity), and `prepareGrokPromptFile` (Windows temp-file delivery fallback; `cleanup({ throwOnError: true })` reports cleanup failure to an owning lifecycle). || `filmStyleGrammars.js` | Curated film style grammar catalog (`FILM_STYLE_GRAMMARS`, eight code-rendered media with rendering, colour, type, motion, camera, sound and native-move guidance), `getFilmStyleGrammar`, `summarizeFilmStyleGrammar` (picker projection) and `knownFilmStyleIdSchema` (request-field id check) and `renderFilmStyleGrammarPrompt(id, { parts })` — one prompt section capped at `FILM_STYLE_PROMPT_MAX_CHARS`; unknown id or bad parts is a 400. |-| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, OmniHuman v1.5 audio-to-video (`FAL_AUDIO_VIDEO_MODELS`, kind `a2v`: the supplied audio drives the figure's whole motion; LTX audio-to-video runs locally, never on fal), Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body) (a prompt over the model's `promptMaxChars` is cut at a clause boundary and reported as `promptClipped`), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. |+| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, OmniHuman v1.5 audio-to-video (`FAL_AUDIO_VIDEO_MODELS`: kind `a2v`, plus the H3 Max and Seedance reference models that hear a song slice as an audio reference (`falHearsAudioReference()`, default `FAL_MUSIC_VIDEO_DEFAULT_MODEL`); the supplied audio drives the whole picture's motion; LTX audio-to-video runs locally, never on fal), Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body) (a prompt over the model's `promptMaxChars` is refused, never cut), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. || `grokVideoClip.js` | Clip lengths grok's `image_to_video` actually delivers — `GROK_VIDEO_DURATIONS` (`[6, 10, 15]`; 6/10 measured in #3022, where a 2s/3s request returned the same 6.04s clip; 15 from xAI's 1–15 s API limit, confirmed per clip by the probe in `videoGen/grok.js`), `GROK_VIDEO_DEFAULT_DURATION`, `resolveGrokDuration()` (validate an already-grok-shaped request, else default), and `nearestGrokDuration()` (round a length authored against another backend's continuous contract UP to the shortest clip that covers it — for translating a CD/commission `targetDurationSeconds`). Dependency-free so `validation.js` / `routes/videoGen.js` can derive their schemas from the same list the services gate on. || `fontMetadata.js` | `fontContainer(buf)` (sfnt / woff2 / null) and `fontFamilyName(buf)` — a font file's family name read from its `name` table (TrueType, OpenType, and WOFF2's brotli stream) with no font library; null for anything unreadable (#10345). || `musicVideoCoverOverlay.js` | Music Video cover lettering (#10345): `COVER_DESIGN_OPTIONS` / `DEFAULT_COVER_DESIGN`, `normalizeCoverDesign(design, { fonts })` (accepts every stored design; a `font:<id>` typeface only when that uploaded font is listed), `coverOverlaySvg({ title, tag, design, widths, fonts })` and the `coverFontSpec` / `fontAttrs` / `titleLinesFor` / `tagTextFor` helpers the width probe reads. Pure; the server render (`coverArtCompose.js`) and the client's live Lettering preview run this one layout. |@@ -856,7 +856,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. || `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit only to a request sent before `musicVideoShotPrompt.js` composed clips. |-| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip and stills (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, repair notes) as a labeled addition; a take that hears no audio is composed without the look, song timing or on-beat camera clause. `musicVideoFrameModelInput(project, scene, { slot, fromFrame, refine, repairs, fixes })` does the same for a start or end frame (look, frame direction, plate actions, rework line, keep-the-shot line, refine wrapper). `authoredShotPrompt`, `authoredFramePromptText` and `visualDirection`. Shared by the shot card, the board, the production run and the image and video submits. |+| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip and stills (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, song audio reference, repair notes) as a labeled addition; a take that hears no audio is composed without the look, song timing or on-beat camera clause. `musicVideoFrameModelInput(project, scene, { slot, fromFrame, refine, repairs, fixes })` does the same for a start or end frame (look, frame direction, plate actions, rework line, keep-the-shot line, refine wrapper). `authoredShotPrompt`, `authoredFramePromptText` and `visualDirection`. Shared by the shot card, the board, the production run and the image and video submits. || `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. || `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |diff --git a/server/lib/falVideoModels.js b/server/lib/falVideoModels.jsindex 05e18cd22d..ee89bc3079 100644--- a/server/lib/falVideoModels.js+++ b/server/lib/falVideoModels.js@@ -40,6 +40,9 @@ export const FAL_DEFAULT_TEXT_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/texport const FAL_DEFAULT_IMAGE_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/image-to-video';export const FAL_LIPSYNC_VIDEO_MODEL = 'minimax/h3-max/lip-sync/image-to-video';export const FAL_AUDIO_VIDEO_MODEL = 'fal-ai/bytedance/omnihuman/v1.5';+// The music-driven shot default: a start frame, a prompt and the song slice as+// an audio reference (OmniHuman's takes were judged worse, 2026-10-13).+export const FAL_MUSIC_VIDEO_DEFAULT_MODEL = 'minimax/h3-max/reference-to-video';const H3_RESOLUTIONS = Object.freeze(['480P', '768P', '1080P']);const SEEDANCE_RESOLUTIONS = Object.freeze(['480p', '720p', '1080p']);@@ -119,9 +122,15 @@ export const FAL_VIDEO_MODELS = Object.freeze([id: 'minimax/h3-max/reference-to-video', kind: 'ref2v', label: 'MiniMax H3 Max (reference)', family: 'minimax-h3',prompt: 'required',promptMaxChars: 2000,+ // Reads its inputs from URLs: the frame and song slice are hosted on fal's CDN first.+ hostedInputs: true,+ // "The video opens exactly on this image": a true start frame, unlike the references.+ image: { param: 'image_url', required: false },+ endImage: { param: 'end_image_url' },// Referenced in the prompt as "Image 1", "Audio 1", … in list order.referenceImages: { param: 'reference_image_urls', max: 9 },- referenceAudio: { param: 'reference_audio_urls', max: 3, minSec: 2, maxSec: 15 },+ // 2–15 s each and at most 15 s combined.+ referenceAudio: { param: 'reference_audio_urls', max: 3, minSec: 2, maxSec: 15, promptName: 'Audio 1' },duration: { kind: 'range', min: 5, max: 15, wire: 'integer', defaultSec: 5 },resolution: { param: 'resolution', options: H3_RESOLUTIONS, default: '768P' },aspectRatio: { param: 'aspect_ratio', options: WIDE_ASPECTS, auto: 'adaptive' },@@ -178,10 +187,12 @@ export const FAL_VIDEO_MODELS = Object.freeze([pricing: { perSecond: 0.16 },},seedance('bytedance/seedance-2.5/reference-to-video', 'ref2v', 'Seedance 2.5 (reference)', {- // Referenced in the prompt as @Image1 / @Audio1. Audio references let a- // singer follow supplied audio, but each clip is capped at 30.2s.- referenceImages: { param: 'image_urls', max: 30 },- referenceAudio: { param: 'audio_urls', max: 10, minSec: 1.8, maxSec: 30.2 },+ // Referenced in the prompt as @Image1 / @Audio1. There is no start-frame+ // field: a PortOS start frame is the first reference image. Audio needs an+ // image reference beside it, and each clip is capped at 30.2s.+ hostedInputs: true,+ referenceImages: { param: 'image_urls', max: 30, promptName: '@Image1' },+ referenceAudio: { param: 'audio_urls', max: 10, minSec: 1.8, maxSec: 30.2, promptName: '@Audio1' },aspectRatio: { param: 'aspect_ratio', options: WIDE_ASPECTS, auto: 'auto' },seed: 'seed',}),@@ -233,8 +244,15 @@ export const getFalVideoModel = (modelId) => (typeof modelId === 'string' ? BY_I*/export const FAL_IMAGE_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'i2v'));-/** Models that animate a frame to supplied audio across the whole picture (the music-driven performance lane). */-export const FAL_AUDIO_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'a2v'));+/**+ * Models that move a whole frame to supplied audio (the music-driven+ * performance lane): an audio-to-video model, or a reference model that hears+ * the song slice as a reference its prompt names ("Audio 1" / @Audio1).+ */+export const FAL_AUDIO_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'a2v' || m.referenceAudio));++/** Whether `model` hears supplied audio as a prompt-named reference rather than as its source audio. */+export const falHearsAudioReference = (model) => Boolean(model?.referenceAudio && !model.audioInput);/** Resolution choices a model offers (empty when it has no resolution parameter). */export const falVideoResolutions = (model) => (model?.resolution?.options ? [...model.resolution.options] : []);@@ -438,13 +456,14 @@ export function buildFalVideoRequest({if (model.image) {if (imageUrl) body[model.image.param] = imageUrl;else if (model.image.required) throw refuse(`${model.label} renders from a start frame — supply a source image`);- } else if (model.referenceImages) {- // A PortOS start frame becomes the first reference ("Image 1" / @Image1).- const refs = [imageUrl, ...(Array.isArray(referenceImageUrls) ? referenceImageUrls : [])].filter(Boolean);+ } else if (imageUrl && !model.referenceImages) {+ throw refuse(`${model.label} is text-to-video — it cannot use a source image`);+ }+ if (model.referenceImages) {+ // Without a start-frame field, a PortOS start frame becomes the first reference ("Image 1" / @Image1).+ const refs = [model.image ? null : imageUrl, ...(Array.isArray(referenceImageUrls) ? referenceImageUrls : [])].filter(Boolean);if (refs.length > model.referenceImages.max) throw refuse(`${model.label} takes at most ${model.referenceImages.max} reference images`);if (refs.length) body[model.referenceImages.param] = refs;- } else if (imageUrl) {- throw refuse(`${model.label} is text-to-video — it cannot use a source image`);}if (endImageUrl) {if (!model.endImage) throw refuse(`${model.label} takes no end frame`);diff --git a/server/lib/falVideoModels.test.js b/server/lib/falVideoModels.test.jsindex 982b980564..28ea030f15 100644--- a/server/lib/falVideoModels.test.js+++ b/server/lib/falVideoModels.test.js@@ -65,9 +65,12 @@ describe('buildFalVideoRequest — per-family wire bodies', () => {expect(body.image_urls).toEqual(['data:image/png;base64,AA==', 'https://example.com/b.png']);expect(body.audio_urls).toEqual(['https://example.com/a.wav']);expect(body).not.toHaveProperty('image_url');- expect(build('minimax/h3-max/reference-to-video', { width: 1080, height: 1350 }).body).toMatchObject({- reference_image_urls: ['data:image/png;base64,AA=='], aspect_ratio: '3:4', duration: 5,+ // H3 Max reference opens exactly on its start frame; the song slice is "Audio 1".+ const h3 = build('minimax/h3-max/reference-to-video', { width: 1080, height: 1350, referenceAudioUrls: ['https://example.com/a.wav'] }).body;+ expect(h3).toMatchObject({+ image_url: 'data:image/png;base64,AA==', reference_audio_urls: ['https://example.com/a.wav'], aspect_ratio: '3:4', duration: 5,});+ expect(h3).not.toHaveProperty('reference_image_urls');});it('lip-sync: frame + audio + resolution only — no prompt field even when one is supplied', () => {diff --git a/server/lib/musicVideoShotPrompt.js b/server/lib/musicVideoShotPrompt.jsindex 6940ffc97f..6e0cbb0bbc 100644--- a/server/lib/musicVideoShotPrompt.js+++ b/server/lib/musicVideoShotPrompt.js@@ -15,6 +15,7 @@ import { musicVideoCreativeContext, musicVideoDirectorLook } from './musicVideoCimport { shotCameraPrompt } from './cameraMovements.js';import { shotActionPrompt } from './musicVideoActionContract.js';import { approximateMotionCues } from './musicVideoShotTiming.js';+import { falHearsAudioReference, getFalVideoModel } from './falVideoModels.js';/*** The project-wide visual direction: palette and camera rules from the visual@@ -37,6 +38,18 @@ const addition = (id, label, text) => {return trimmed ? { id, label, text: trimmed } : null;};+/**+ * What a reference model is told about the song slice it hears (and, where its+ * start frame is only a reference, that frame), in the names its prompt uses+ * for them; '' for a model that hears no audio reference.+ */+function audioReferencePrompt(modelId) {+ const model = getFalVideoModel(modelId);+ if (!falHearsAudioReference(model)) return '';+ const frame = model.image ? '' : `${model.referenceImages?.promptName || 'Image 1'} is the opening frame. `;+ return `${frame}${model.referenceAudio.promptName || 'Audio 1'} is the song playing under this shot: time the movement to its beat.`;+}+/*** `{ prompt, promptAdditions }` for one clip. `authored` is the director's text* (the shot card's box, or the scene's saved prompt). `asTyped` is the shot@@ -44,10 +57,12 @@ const addition = (id, label, text) => {* whether the take hears song audio (lip-sync, supplied song audio or an* audio-reactive LoRA); `offsetSec` re-times the shot actions to a performance* take's padded song slice; `fixes` is an autonomous run's repair note.+ * `audioReferenceModelId` is the fal model of a music-driven take that hears+ * the song slice as a reference, which its prompt then names.*/export function musicVideoShotModelInput(project, scene, {authored = authoredShotPrompt(scene), asTyped = false, hears = false, backend = null,- audioReactive = false, offsetSec = 0, fixes = '',+ audioReactive = false, offsetSec = 0, fixes = '', audioReferenceModelId = null,} = {}) {const text = typeof authored === 'string' ? authored.trim() : '';const look = !asTyped && hears ? [@@ -66,6 +81,7 @@ export function musicVideoShotModelInput(project, scene, {const tail = audioReactive? [addition('audio-reactive-guard', 'Audio-reactive guard', AUDIO_REACTIVE_PERFORMANCE_GUARD)]: [+ addition('audio-reference', 'Song audio reference', audioReferenceModelId ? audioReferencePrompt(audioReferenceModelId) : ''),addition('phrase-timing', 'Phrase timing (approximate)', backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : ''),addition('repair-notes', 'Repair notes', fixes ? `Fix these problems from the previous take while preserving the shot intent: ${fixes}` : ''),];diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.jsindex 3e5838be6f..c23b871f5d 100644--- a/server/lib/musicVideoShotTiming.js+++ b/server/lib/musicVideoShotTiming.js@@ -16,8 +16,9 @@* conditioning — REST documentation for a different Grok surface is not that* proof. Local runtimes are absent for the same reason.* - `SOURCE_AUDIO_MUSIC` — the music-driven performance lanes: fal.ai- * audio-to-video models (kind `a2v`) where the song slice drives the whole- * picture's motion rather than only a singer's mouth. A scene opts in with+ * audio-to-video models (kind `a2v`) and reference models that hear the+ * song slice as an audio reference (MiniMax H3 Max, Seedance 2.5), where+ * the song drives the whole picture's motion rather than only a singer's mouth. A scene opts in with* `performanceStyle: 'music'`; it shares the window planning below.* - `planPerformanceWindow` — the audio window a performance shot submits:* inside the provider's limits (never relying on its silent truncation), a@@ -42,10 +43,10 @@import { GROK_VIDEO_DURATIONS, nearestGrokDuration } from './grokVideoClip.js';import {- FAL_AUDIO_VIDEO_MODEL,FAL_AUDIO_VIDEO_MODELS,FAL_DEFAULT_IMAGE_VIDEO_MODEL,FAL_LIPSYNC_VIDEO_MODEL,+ FAL_MUSIC_VIDEO_DEFAULT_MODEL,coerceFalVideoSeconds,describeFalVideoRate,estimateFalVideoCostUsd,@@ -159,27 +160,27 @@ export const SOURCE_AUDIO_LIPSYNC = Object.freeze({local: localAudioLane('lipsync'),});-/**- * Default output resolution of a music-driven take (OmniHuman's own default).- * LTX audio-to-video is deliberately not a fal lane: PortOS runs LTX-2.3 /- * LTX-2.5 locally, where Make › Supplied song audio conditions a cutaway on- * its own song slice (submitJob.js `_suppliedAudio`).- */-const MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION = '1080p';-+// A music-driven take renders at its model's own default resolution.+// LTX audio-to-video is deliberately not a fal lane: PortOS runs LTX-2.3 /+// LTX-2.5 locally, where Make › Supplied song audio conditions a cutaway on+// its own song slice (submitJob.js `_suppliedAudio`).const musicCapability = (model) => {- const resolutions = falVideoResolutions(model);- const defaultResolution = resolutions.includes(MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION) ? MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION : model.resolution.default;+ const audio = model.audioInput || model.referenceAudio;+ // A reference model bills and renders whole seconds from its shortest clip+ // up: the song window starts at that length (less the planning margin, so a+ // padded window lands exactly on it) and never outruns the longest clip.+ const clip = model.duration.kind === 'range' ? model.duration : null;+ const defaultResolution = model.resolution.default;return Object.freeze({provider: 'fal',style: 'music',label: `fal.ai ${model.label}`,modelId: model.id,- minAudioSec: model.audioInput.minSec,+ minAudioSec: clip ? Math.max(audio.minSec, clip.min - PERFORMANCE_WINDOW_MARGIN_SEC) : audio.minSec,// The same one-phrase per-take ceiling as lip-sync (see above).- maxAudioSec: Math.min(model.audioInput.maxSec, 30),+ maxAudioSec: Math.min(audio.maxSec, clip ? clip.max : Infinity, 30),transcription: false,- resolutions: Object.freeze(resolutions),+ resolutions: Object.freeze(falVideoResolutions(model)),defaultResolution,costLabel: `${describeFalVideoRate(model.id, defaultResolution)} — billed to your fal.ai account`,});@@ -188,7 +189,7 @@ const musicCapability = (model) => {/*** Verified music-driven lanes, keyed by backend then model id. The project's* `videoSettings.falMusicModelId` picks one; absent (or a model this install- * does not curate) = `FAL_AUDIO_VIDEO_MODEL` (OmniHuman v1.5).+ * does not curate) = `FAL_MUSIC_VIDEO_DEFAULT_MODEL` (MiniMax H3 Max reference).*/const LOCAL_MUSIC_LANE = localAudioLane('music');@@ -207,7 +208,7 @@ export function performanceCapability(backend, { scene = null, videoSettings = nif (backend === 'local') return LOCAL_MUSIC_LANE;const lanes = SOURCE_AUDIO_MUSIC[backend];const pinned = videoSettings?.falMusicModelId;- return (pinned && Object.prototype.hasOwnProperty.call(lanes, pinned) ? lanes[pinned] : null) || lanes[FAL_AUDIO_VIDEO_MODEL];+ return (pinned && Object.prototype.hasOwnProperty.call(lanes, pinned) ? lanes[pinned] : null) || lanes[FAL_MUSIC_VIDEO_DEFAULT_MODEL];}/**@@ -541,6 +542,7 @@ export function falSceneTake({ scene, videoSettings = {}, songDurationSec = nullconst seconds = plan.ok ? plan.windowSec : null;return {performance: true,+ music: capability.style === 'music',modelId: capability.modelId,seconds,resolution,@@ -563,13 +565,14 @@ export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null}/**- * A `falSceneTake` as `/api/video-gen` fields. A performance take sends only- * its lip-sync resolution — its length follows the song slice the server cuts;- * a cutaway names its model, length and resolution. Unknown values stay absent- * so the server resolves its own defaults (an uncurated model's legacy body).+ * A `falSceneTake` as `/api/video-gen` fields. A performance take sends its+ * resolution (and a music-driven take its model, which a shot card can pick) —+ * its length follows the song slice the server cuts; a cutaway names its+ * model, length and resolution. Unknown values stay absent so the server+ * resolves its own defaults (an uncurated model's legacy body).*/export const falTakeRequestFields = (take) => (take.performance- ? (take.resolution ? { falResolution: take.resolution } : {})+ ? { ...(take.music ? { falModelId: take.modelId } : {}), ...(take.resolution ? { falResolution: take.resolution } : {}) }: {falModelId: take.modelId,...(take.seconds != null ? { falDuration: take.seconds } : {}),diff --git a/server/lib/musicVideoShotTiming.test.js b/server/lib/musicVideoShotTiming.test.jsindex 94c622ae4b..c14bae886a 100644--- a/server/lib/musicVideoShotTiming.test.js+++ b/server/lib/musicVideoShotTiming.test.js@@ -143,21 +143,25 @@ describe('music-driven performance takes', () => {// The drummer case: a 3.735s shot of the song whose motion the audio drives.const drummer = { shotMode: 'performance', performanceStyle: 'music', startSec: 212.718, endSec: 216.453 };- it('render on the audio-to-video model for the shot itself, without lip-sync\'s 5s padding', () => {+ it('render on H3 Max reference by default, the song window padded to exactly its shortest billed clip', () => {const take = falSceneTake({ scene: drummer, videoSettings: {}, songDurationSec: 240 });- expect(take).toMatchObject({ performance: true, modelId: 'fal-ai/bytedance/omnihuman/v1.5', seconds: 3.735, resolution: '1080p', costUsd: 0.5976 });- expect(falTakeRequestFields(take)).toEqual({ falResolution: '1080p' });- // The same shot as lip-sync pads to the route's 5s minimum.+ expect(take).toMatchObject({ performance: true, music: true, modelId: 'minimax/h3-max/reference-to-video', seconds: 5, resolution: '768P', costUsd: 0.4 });+ // A music-driven take names its model, so a shot card can pick another.+ expect(falTakeRequestFields(take)).toEqual({ falModelId: 'minimax/h3-max/reference-to-video', falResolution: '768P' });+ // The same shot as lip-sync pads to the route's 5s minimum plus its margin.expect(falSceneTake({ scene: { ...drummer, performanceStyle: null }, songDurationSec: 240 }).seconds).toBeCloseTo(5.05, 6);+ expect(shotSplitLimit(drummer, 'fal')).toBeCloseTo(15 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);});it('follow the project pin (model and resolution) and fall back to the default for an uncurated pin', () => {+ // An audio-to-video model hears the shot itself, with no padding.const pinned = falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5', falMusicResolution: '720p' }, songDurationSec: 240 });- expect(pinned).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '720p', costUsd: 0.5976 });+ expect(pinned).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', seconds: 3.735, resolution: '720p', costUsd: 0.5976 });+ const seedance = falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'bytedance/seedance-2.5/reference-to-video' }, songDurationSec: 240 });+ expect(seedance).toMatchObject({ seconds: 4, resolution: '720p', costUsd: 1.892 });// A pin this install no longer curates (fal LTX-2.3, removed) falls back to the default.expect(falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'fal-ai/ltx-2.3-22b/audio-to-video' }, songDurationSec: 240 }).modelId)- .toBe('fal-ai/bytedance/omnihuman/v1.5');- expect(shotSplitLimit(drummer, 'fal')).toBeCloseTo(30 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);+ .toBe('minimax/h3-max/reference-to-video');});});diff --git a/server/services/musicVideo/performanceShot.js b/server/services/musicVideo/performanceShot.jsindex 9c75c6deac..b8e9602542 100644--- a/server/services/musicVideo/performanceShot.js+++ b/server/services/musicVideo/performanceShot.js@@ -263,10 +263,11 @@ export async function assertCurrentPerformanceTakes(project, masterPath) {* for a performance scene on a capable backend. `resolution` is the take's* output resolution: the request's own when the lip-sync route offers it, else* the project's `videoSettings.falLipSyncResolution`, else the capability- * default (1080P). Throws a 4xx ServerError when the scene is a performance+ * default (1080P). `musicModelId` is a music-driven model picked for this one+ * take (ignored unless it is a curated music lane). Throws a 4xx ServerError when the scene is a performance* shot the backend or its timing cannot deliver.*/-export async function preparePerformanceShot({ musicVideo, backend, sourceImagePath, mode, resolution = null }) {+export async function preparePerformanceShot({ musicVideo, backend, sourceImagePath, mode, resolution = null, musicModelId = null }) {if (!musicVideo?.projectId || !musicVideo?.sceneId) return null;const project = await getProject(musicVideo.projectId);const scene = project?.scenes?.find((s) => s.sceneId === musicVideo.sceneId);@@ -283,7 +284,9 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP// A music-driven scene renders on the project's audio-to-video model; a// lip-sync scene on the lip-sync route.- const capability = performanceCapability(backend, { scene, videoSettings: project.videoSettings });+ // A shot card can pick another music-driven model for this take (one PortOS curates, else the project's).+ const videoSettings = musicModelId ? { ...project.videoSettings, falMusicModelId: musicModelId } : project.videoSettings;+ const capability = performanceCapability(backend, { scene, videoSettings });if (!capability) throw refuse(performanceBlockedReason(backend), 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');const musicDriven = capability.style === 'music';// The lip-sync route is image + source audio; any other semantic mode woulddiff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.jsindex 1aa5079a0a..4672f67803 100644--- a/server/services/musicVideo/performanceShot.test.js+++ b/server/services/musicVideo/performanceShot.test.js@@ -304,7 +304,7 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-syncawait writeFile(join(PATHS.music, 'vocals.wav'), songWav({ markerAmp: 12000 }));// A vocal stem is attached, but the drums it leaves out are what drive this shot.getProject.mockResolvedValue({ ...project({ performanceStyle: 'music', prompt: 'drummer hits on the beat' }), vocalStemFilename: 'vocals.wav',- videoSettings: { backend: 'fal', falLipSyncTranscription: true } });+ videoSettings: { backend: 'fal', falLipSyncTranscription: true, falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5' } });const prepared = await preparePerformanceShot({musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'frame.png'), mode: 'image',});@@ -368,6 +368,66 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync}, 60_000);+ it.each([+ // The default: H3 Max reference opens on the start frame and hears the song as "Audio 1".+ { musicModelId: null, url: 'minimax/h3-max/reference-to-video', window: { startSec: 18.25, endSec: 23.25, durationSec: 5 },+ body: { duration: 5, resolution: '768P' }, frame: 'image_url', audio: 'reference_audio_urls', costUsd: 0.4 },+ // Picked on the shot card: Seedance has no start-frame field, so the frame is @Image1.+ { musicModelId: 'bytedance/seedance-2.5/reference-to-video', url: 'bytedance/seedance-2.5/reference-to-video',+ window: { startSec: 18.75, endSec: 22.75, durationSec: 4 }, body: { duration: '4', resolution: '720p', generate_audio: false },+ frame: 'image_urls', audio: 'audio_urls', costUsd: 1.892 },+ ])('a music-driven shot on $url sends the start frame, prompt and song slice as an audio reference', async ({ musicModelId, url, window, body, frame, audio, costUsd }) => {+ getProject.mockResolvedValue({ ...project({ performanceStyle: 'music', prompt: 'drummer hits on the beat' }), videoSettings: { backend: 'fal' } });+ const prepared = await preparePerformanceShot({+ musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'frame.png'), mode: 'image', musicModelId,+ });+ // A 1.5s shot pads to exactly the model's shortest billed clip, centered on the shot.+ expect(prepared.shotInstruction.audioWindow).toEqual(window);+ expect(prepared.shotInstruction.edit).toEqual({ inSec: 20 - window.startSec, outSec: 21.5 - window.startSec, targetSec: 1.5 });+ expect(prepared.modelId).toBe(url);++ await mkdir(PATHS.images, { recursive: true });+ await writeFile(join(PATHS.images, 'frame.png'), Buffer.from('89504e470d0a1a0a', 'hex'));+ const deliveredPath = join(PATHS.data, 'delivered-reference.mp4');+ execFileSync(ffmpeg, ['-v', 'error', '-f', 'lavfi', '-i', `color=blue:s=64x64:r=24:d=${window.durationSec}`, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-y', deliveredPath]);+ const delivered = await readFile(deliveredPath);+ const posts = [];+ const uploads = new Map();+ vi.stubGlobal('fetch', vi.fn(async (target, opts) => {+ if (target.startsWith('https://rest.fal.ai/storage/upload/initiate')) {+ const { file_name: name } = JSON.parse(opts.body);+ return jsonResponse({ upload_url: `https://upload.example.com/${name}`, file_url: `https://cdn.example.com/${name}` });+ }+ if (target.startsWith('https://upload.example.com/')) {+ uploads.set(target.replace('https://upload.example.com/', 'https://cdn.example.com/'), Buffer.from(opts.body));+ return { ok: opts.method === 'PUT', status: 200 };+ }+ if (opts?.method === 'POST') {+ posts.push({ url: target, body: JSON.parse(opts.body) });+ return jsonResponse({ request_id: 'r3', status_url: 'https://queue.fal.run/s3', response_url: 'https://queue.fal.run/r3' });+ }+ if (target === 'https://queue.fal.run/s3') return jsonResponse({ status: 'COMPLETED' });+ if (target === 'https://queue.fal.run/r3') return jsonResponse({ video: { url: 'https://cdn.example.com/ref.mp4' } });+ if (target === 'https://cdn.example.com/ref.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(delivered).buffer };+ throw new Error(`unexpected fetch: ${target}`);+ }));+ const job = await fal.generateVideo({+ apiKey: 'test-key', modelId: prepared.modelId, resolution: prepared.resolution, prompt: 'drummer hits on the beat', width: 1920, height: 1080,+ sourceImagePath: join(PATHS.images, 'frame.png'), audioFilePath: prepared.audioFilePath, shotInstruction: prepared.shotInstruction,+ });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });+ expect(posts).toHaveLength(1);+ expect(posts[0].url).toBe(`https://queue.fal.run/${url}`);+ expect(posts[0].body).toMatchObject({ prompt: 'drummer hits on the beat', ...body });+ expect(posts[0].body).not.toHaveProperty('audio_url');+ const start = frame === 'image_urls' ? posts[0].body.image_urls[0] : posts[0].body.image_url;+ expect(start).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.[a-z]+$/);+ expect(posts[0].body[audio]).toHaveLength(1);+ expect(uploads.get(posts[0].body[audio][0])).toEqual(await readFile(prepared.audioFilePath));+ const { loadHistory } = await import('../videoGen/history.js');+ expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({ modelId: `fal:${url}`, estimatedCostUsd: costUsd, audioDriven: true });+ }, 60_000);+it('uses the full span of a long shot in one take, at the project\'s lip-sync resolution pin', async () => {// 26s: past the old 14.8s cap (fal accepts up to 15 minutes; PortOS keeps// a take to 30s so it stays one editable phrase).diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.jsindex 2fc87f7e9b..7a3bad494e 100644--- a/server/services/videoGen/fal.js+++ b/server/services/videoGen/fal.js@@ -34,6 +34,7 @@ import {FAL_DEFAULT_IMAGE_VIDEO_MODEL,FAL_DEFAULT_TEXT_VIDEO_MODEL,buildFalVideoRequest,+ falHearsAudioReference,getFalVideoModel,} from '../../lib/falVideoModels.js';import { probeVideoDuration, probeVideoGeometry } from '../../lib/ffmpeg.js';@@ -151,13 +152,16 @@ function buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageexport function falVideoSentPrompt({ hasImage = false, hasEndImage = false, hasAudio = false, promptAdditions = [], ...spec }) {const modelId = spec.modelId || (hasImage ? FAL_DEFAULT_IMAGE_MODEL : FAL_DEFAULT_TEXT_MODEL);const { prompt = '', negativePrompt } = spec;+ // A reference model hears the audio as a prompt-named reference ("Audio 1").+ const audioAsReference = hasAudio && falHearsAudioReference(getFalVideoModel(modelId));let planned = null;try {planned = buildFalVideoRequest({...spec, modelId,imageUrl: hasImage ? 'pending:image' : null,endImageUrl: hasEndImage ? 'pending:end-image' : null,- audioUrl: hasAudio ? 'pending:audio' : null,+ audioUrl: hasAudio && !audioAsReference ? 'pending:audio' : null,+ referenceAudioUrls: audioAsReference ? ['pending:audio'] : [],});} catch (err) {if (err?.code !== 'FAL_MODEL_INPUT') throw err;@@ -227,8 +231,10 @@ export async function generateVideo({const audioSec = audioFilePath? (shotInstruction?.audioWindow?.durationSec || await probeVideoDuration(audioFilePath).catch(() => null)): null;+ // A reference model renders a set length, not one that follows its audio: the clip covers the song slice.+ const audioAsReference = !!audioFilePath && falHearsAudioReference(catalogModel);const requestSpec = {- modelId, prompt, negativePrompt, seconds: duration, audioSec, aspectRatio, width, height, resolution,+ modelId, prompt, negativePrompt, seconds: audioAsReference ? (duration ?? audioSec) : duration, audioSec, aspectRatio, width, height, resolution,generateAudio: generateAudio === true, enableTranscription: lipSync?.enableTranscription === true,};const { planned, record: sentRecord } = falVideoSentPrompt({@@ -260,7 +266,7 @@ export async function generateVideo({createdAt: new Date().toISOString(),mode: sourceImagePath ? 'image' : 'text',// A lip-sync take syncs a mouth; an audio-to-video take moves the whole frame to the audio.- ...(audioFilePath ? (catalogModel?.kind === 'a2v' ? { audioDriven: true } : { lipSync: true }) : {}),+ ...(audioFilePath ? (catalogModel?.kind === 'a2v' || audioAsReference ? { audioDriven: true } : { lipSync: true }) : {}),...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),};const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };@@ -333,8 +339,9 @@ async function runFalVideo(job, jobId, {const [imageUrl, endImageUrl, audioUrl] = await Promise.all([host(imageDataUri, 'start'), host(endImageDataUri, 'end'), host(audioDataUri, 'audio'),]);+ const audioInputs = falHearsAudioReference(getFalVideoModel(modelId)) ? { referenceAudioUrls: audioUrl ? [audioUrl] : [] } : { audioUrl };const body = requestSpec- ? buildFalVideoRequest({ ...requestSpec, imageUrl, endImageUrl, audioUrl }).body+ ? buildFalVideoRequest({ ...requestSpec, imageUrl, endImageUrl, ...audioInputs }).body: audioFilePath? buildLipSyncRequestBody({ imageDataUri: imageUrl, audioDataUri: audioUrl, enableTranscription }): buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri: imageUrl });diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex 2a18889fa9..ac1b35ca33 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -235,6 +235,7 @@ const submitValidatedVideoGenJob = async (body, uploads) => {sourceImagePath: prepared.sourceImagePath,mode: body.mode,resolution: body.falResolution,+ musicModelId: backend === VIDEO_GEN_MODE.FAL ? body.falModelId || null : null,}).catch(async (error) => {await cleanupStaged();throw error;@@ -322,6 +323,8 @@ const submitValidatedVideoGenJob = async (body, uploads) => {const composed = musicVideoShotModelInput(project, scene, {authored: params.prompt, asTyped: body.musicVideo.promptAsTyped === true, hears: hearsAudio, backend,audioReactive: body.musicVideo.audioReactive === true, offsetSec, fixes: body._clipFixes || '',+ // A music-driven take on a reference model hears the song slice as "Audio 1" / @Audio1.+ audioReferenceModelId: backend === VIDEO_GEN_MODE.FAL ? performance?.modelId || null : null,});params.prompt = composed.prompt;composedAdditions = composed.promptAdditions;@@ -377,7 +380,8 @@ const submitValidatedVideoGenJob = async (body, uploads) => {const { costUsd, modelId: costModelId } = falSceneTake({scene,videoSettings: {- falMusicModelId: project.videoSettings?.falMusicModelId ?? null,+ falMusicModelId: (performance?.shotInstruction?.performanceStyle === 'music' ? performance.modelId : null)+ ?? project.videoSettings?.falMusicModelId ?? null,falModelId: body.falModelId, falDuration: body.falDuration, falResolution: body.falResolution,falLipSyncResolution: body.falResolution, falMusicResolution: body.falResolution,},diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.jsindex 436979827b..0406fab32c 100644--- a/server/services/videoGen/submitJob.test.js+++ b/server/services/videoGen/submitJob.test.js@@ -235,6 +235,8 @@ describe('submitVideoGenJob', () => {}, {});expect(mocks.preparePerformanceShot).toHaveBeenCalledWith({musicVideo, backend: 'fal', sourceImagePath: '/example/images/frame.png', mode: 'image', resolution: '1080p',+ // A music-driven shot's picked model; a lip-sync scene ignores it.+ musicModelId: 'minimax/h3-max/image-to-video',});const { params } = mocks.enqueueJob.mock.calls[0][0];expect(params).toMatchObject({@@ -282,6 +284,19 @@ describe('submitVideoGenJob', () => {expect(hearing.prompt).toContain('Lyric 0.500s–1.500s: hold on');expect(hearing.prompt).toContain('land the move exactly on the downbeat');expect(hearing.promptAdditions.map((item) => item.id)).toContain('project-style');++ // A music-driven take on a reference model is told which reference is the song.+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());+ mocks.preparePerformanceShot.mockResolvedValue({+ audioFilePath: '/example/uploads/mv-performance-2.wav', modelId: 'minimax/h3-max/reference-to-video', resolution: '768P',+ shotInstruction: { version: 2, shotMode: 'performance', performanceStyle: 'music', songInterval: { startSec: 0, endSec: 4 }, edit: { inSec: 0 } },+ });+ await submitVideoGenJob({ prompt: 'she turns away', backend: 'fal', mode: 'image', musicVideo: { ...musicVideo, compose: true } }, {});+ const music = mocks.enqueueJob.mock.calls[2][0].params;+ expect(music).toMatchObject({ modelId: 'minimax/h3-max/reference-to-video', audioFilePath: '/example/uploads/mv-performance-2.wav' });+ expect(music.promptAdditions).toContainEqual({ id: 'audio-reference', label: 'Song audio reference',+ text: 'Audio 1 is the song playing under this shot: time the movement to its beat.' });+ expect(music.prompt.endsWith('. Audio 1 is the song playing under this shot: time the movement to its beat.')).toBe(true);});it('leaves a clip outside a music video at its own aspect', async () => {