Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music-driven shots on fal hear the song as an audio reference (H3 Max, Seedance 2.5)

Claudeb8d665b8f56b
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index 62bc449864..c8306da241 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -55,7 +55,7 @@ describe('performance speaker save boundary', () => {
});
describe('music-driven performance', () => {
- it('switches a shot between cutaway, lip-sync and music-driven, and plans the music-driven take on the audio-to-video model', () => {
+ it('switches a shot between cutaway, lip-sync and music-driven, and plans the music-driven take on the default music-driven model', () => {
const onSave = vi.fn();
function Harness() {
const [scene, setScene] = useState({ sceneId: 's1', shotMode: 'cutaway', startSec: 10, endSec: 13.5, referenceImageId: 'frame.png', takes: [] });
@@ -69,8 +69,8 @@ describe('music-driven performance', () => {
fireEvent.change(shot, { target: { value: 'music' } });
expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: 'music' });
expect(shot.value).toBe('music');
- // The shot's own 3.5s, not lip-sync's 5s minimum window.
- expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Music-driven via fal\.ai OmniHuman v1\.5 \(audio-driven person\) \(fal-ai\/bytedance\/omnihuman\/v1\.5\) · song 0:10–0:13 · est\. \$0\.56 \(3\.5s at 1080p\)/);
+ // H3 Max reference by default: the song window pads to exactly its 5s shortest billed clip.
+ expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Music-driven via fal\.ai MiniMax H3 Max \(reference\) \(minimax\/h3-max\/reference-to-video\) · song 0:09–0:14 · shot starts 0\.75s into the take · est\. \$0\.40 \(5s at 768P\)/);
fireEvent.change(shot, { target: { value: 'performance' } });
expect(onSave).toHaveBeenLastCalledWith('s1', { shotMode: 'performance', performanceStyle: null });
expect(screen.getByTestId('performance-plan').textContent).toMatch(/^Lip-sync via/);
diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsx
index 5d58f698b4..0dd28931b1 100644
--- a/client/src/components/musicVideo/SceneRenderInputs.jsx
+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx
@@ -1,4 +1,4 @@
-import { getFalVideoModel } from '../../lib/falVideoModels.js';
+import { falHearsAudioReference, getFalVideoModel } from '../../lib/falVideoModels.js';
import { falSceneTake } from '../../lib/musicVideoShotTiming.js';
import { shotActionContractProblem } from '../../../../server/lib/musicVideoActionContract.js';
import { authoredShotPrompt, musicVideoShotModelInput } from '../../../../server/lib/musicVideoShotPrompt.js';
@@ -55,6 +55,7 @@ export function shotCardRenderRequest({
const authored = authoredShotPrompt(scene);
const { prompt, promptAdditions } = musicVideoShotModelInput(contractProblem ? null : project, scene, {
authored, asTyped: true, hears, backend: settings.backend, audioReactive,
+ audioReferenceModelId: performance && falHearsAudioReference(falModel) ? falModel.id : null,
});
let endFrame = null;
diff --git a/client/src/components/musicVideo/SceneRenderInputs.test.jsx b/client/src/components/musicVideo/SceneRenderInputs.test.jsx
index f4f268673d..45273faa9e 100644
--- a/client/src/components/musicVideo/SceneRenderInputs.test.jsx
+++ b/client/src/components/musicVideo/SceneRenderInputs.test.jsx
@@ -48,6 +48,26 @@ describe('shotCardRenderRequest', () => {
});
});
+// The 80s drummer shot: a music-driven take hears the song slice as a reference its prompt names.
+describe('music-driven shot on a reference model', () => {
+ const drummer = scene({ shotMode: 'performance', performanceStyle: 'music', prompt: 'The drummer hits the snare' });
+ it.each([
+ [{}, 'Audio 1 is the song playing under this shot: time the movement to its beat.'],
+ [{ falMusicModelId: 'bytedance/seedance-2.5/reference-to-video' },
+ '@Image1 is the opening frame. @Audio1 is the song playing under this shot: time the movement to its beat.'],
+ ])('lists the song audio reference as a chip, sent after the typed prompt (%o)', (pin, text) => {
+ const request = shotCardRenderRequest({ scene: drummer, performance: true, settings: { backend: 'fal', ...pin } });
+ expect(request.songAudio.state).toBe('required');
+ expect(request.promptAdditions).toContainEqual({ id: 'audio-reference', label: 'Song audio reference', text });
+ expect(request.prompt).toBe(`The drummer hits the snare. ${text}`);
+ });
+
+ it('adds nothing for an audio-to-video model that hears the song as its source audio', () => {
+ const request = shotCardRenderRequest({ scene: drummer, performance: true, settings: { backend: 'fal', falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5' } });
+ expect(request.promptAdditions.map((item) => item.id)).not.toContain('audio-reference');
+ });
+});
+
describe('SceneRenderInputs', () => {
it('shows the exact prompt and reports a toggle', () => {
const onChange = vi.fn();
diff --git a/client/src/components/musicVideo/SceneRenderTarget.jsx b/client/src/components/musicVideo/SceneRenderTarget.jsx
index 331173ff0c..9ccc710acf 100644
--- a/client/src/components/musicVideo/SceneRenderTarget.jsx
+++ b/client/src/components/musicVideo/SceneRenderTarget.jsx
@@ -2,7 +2,7 @@ import {
FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, coerceFalVideoSeconds, describeFalVideoRate, falVideoResolutions, getFalVideoModel,
} from '../../lib/falVideoModels.js';
import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
-import { grokCoverage, performanceCapability } from '../../lib/musicVideoShotTiming.js';
+import { SOURCE_AUDIO_MUSIC, grokCoverage, performanceCapability } from '../../lib/musicVideoShotTiming.js';
import { localMusicVideoModels } from '../../../../server/lib/musicVideoLocalModels.js';
import { frameOptionsForModel } from '../../lib/videoGenParams.js';
import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage.js';
@@ -10,9 +10,10 @@ import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage
/**
* Where one shot's clip renders, picked on its card: the project's saved
* renderer, local LTX, Grok, or a fal.ai model. LTX is local only, so fal
- * offers only its curated start-frame models; a lip-synced or music-driven
- * shot renders on fal's source-audio route, so fal is one choice there, and
- * Grok (no audio) renders cutaways only.
+ * offers only its curated start-frame models; a lip-synced shot renders on
+ * fal's lip-sync route, so fal is one choice there, a music-driven shot offers
+ * each music-driven model (the song slice drives the picture), and Grok (no
+ * audio) renders cutaways only.
* The pick is a value string: '' (saved), 'local', 'grok', 'fal' or 'fal:<modelId>'.
*
* A picked lane shows its own options, from the same definitions the Video Gen
@@ -50,7 +51,14 @@ export function renderTargetOptions({ performance, saved, endFrame = false, scen
{ value: 'local', label: performance ? 'Local LTX lip-sync (free)' : 'Local LTX (free)' },
{ value: 'grok', label: performance ? 'Grok (cutaways only)' : 'Grok', disabled: performance },
...(performance
- ? [{ value: 'fal', label: `fal.ai · ${falPerformanceLabel(saved, scene)}` }]
+ ? [
+ { value: 'fal', label: `fal.ai · ${falPerformanceLabel(saved, scene)}` },
+ // A music-driven shot can try another model that hears the song, priced per second.
+ ...(scene?.performanceStyle === 'music' ? Object.values(SOURCE_AUDIO_MUSIC.fal).map((lane) => {
+ const rate = describeFalVideoRate(lane.modelId, lane.defaultResolution);
+ return { value: `fal:${lane.modelId}`, label: `fal.ai · ${getFalVideoModel(lane.modelId).label} + song audio${rate ? ` · ${rate}` : ''}` };
+ }) : []),
+ ]
: FAL_IMAGE_VIDEO_MODELS.map((model) => {
const rate = describeFalVideoRate(model.id);
return { value: `fal:${model.id}`, label: `fal.ai · ${model.label}${rate ? ` · ${rate}` : ''}${endFrame && model.endImage ? ' · ends on end frame' : ''}` };
@@ -141,6 +149,10 @@ export function renderTargetOverrides(value, saved, options = {}, context = {})
}
const falModelId = value.slice(4) || null;
if (!falModelId) return { backend: 'fal' };
+ // A music-driven shot's pick is its music model, at that model's own default size.
+ if (context.performance) {
+ return { backend: 'fal', falMusicModelId: falModelId, ...(falModelId === saved.falMusicModelId ? {} : { falMusicResolution: null }) };
+ }
// Each model has its own resolutions, so the saved one only carries over to the saved model.
const sameModel = falModelId === (saved.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL);
// A picked model is the one this render uses, end frame or not.
diff --git a/client/src/components/musicVideo/SceneRenderTarget.test.jsx b/client/src/components/musicVideo/SceneRenderTarget.test.jsx
index 39776deb1c..c5a611e14c 100644
--- a/client/src/components/musicVideo/SceneRenderTarget.test.jsx
+++ b/client/src/components/musicVideo/SceneRenderTarget.test.jsx
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, it } from 'vitest';
import { render, screen, within } from '@testing-library/react';
-import SceneRenderTarget, { lastRenderTarget, rememberRenderTarget, renderTargetOverrides } from './SceneRenderTarget.jsx';
+import SceneRenderTarget, { lastRenderTarget, rememberRenderTarget, renderTargetOptions, renderTargetOverrides } from './SceneRenderTarget.jsx';
import { FAL_IMAGE_VIDEO_MODELS, falVideoResolutions } from '../../lib/falVideoModels.js';
const models = [
@@ -43,3 +43,21 @@ describe('a remembered Render on pick', () => {
expect(lastRenderTarget({ performance: false, saved: {} }).options).toEqual({});
});
});
+
+// Uniquely catches a music-driven shot losing its per-shot choice of the models that hear the song.
+describe('a music-driven shot on fal', () => {
+ it('offers each music-driven model with its rate, and the pick becomes that take\'s music model', () => {
+ const scene = { sceneId: 's1', shotMode: 'performance', performanceStyle: 'music', startSec: 0, endSec: 2 };
+ const fal = renderTargetOptions({ performance: true, saved: { backend: 'fal' }, scene }).filter((option) => option.value.startsWith('fal'));
+ expect(fal[0].label).toBe('fal.ai · MiniMax H3 Max (reference)');
+ expect(fal.map((option) => option.value)).toEqual(expect.arrayContaining([
+ 'fal:minimax/h3-max/reference-to-video', 'fal:bytedance/seedance-2.5/reference-to-video',
+ ]));
+ expect(fal.find((option) => option.value === 'fal:minimax/h3-max/reference-to-video').label).toMatch(/\+ song audio · \$0\.08\/s at 768P/);
+ // A lip-synced shot keeps its one lip-sync choice.
+ expect(renderTargetOptions({ performance: true, saved: {}, scene: { ...scene, performanceStyle: null } })
+ .filter((option) => option.value.startsWith('fal'))).toHaveLength(1);
+ expect(renderTargetOverrides('fal:bytedance/seedance-2.5/reference-to-video', { falMusicResolution: '768P' }, {}, { performance: true, scene }))
+ .toEqual({ backend: 'fal', falMusicModelId: 'bytedance/seedance-2.5/reference-to-video', falMusicResolution: null });
+ });
+});
diff --git a/client/src/components/musicVideo/VideoRenderSettings.jsx b/client/src/components/musicVideo/VideoRenderSettings.jsx
index a12a00dd59..5c9d14bf6d 100644
--- a/client/src/components/musicVideo/VideoRenderSettings.jsx
+++ b/client/src/components/musicVideo/VideoRenderSettings.jsx
@@ -1,7 +1,7 @@
import RenderPoolPicker from './RenderPoolPicker.jsx';
import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
import {
- FAL_AUDIO_VIDEO_MODEL, FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falImageVideoQuotes, falVideoResolutions, getFalVideoModel,
+ FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_MUSIC_VIDEO_DEFAULT_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falImageVideoQuotes, falVideoResolutions, getFalVideoModel,
} from '../../lib/falVideoModels.js';
import { projectFalSpend } from '../../lib/musicVideoStages.js';
import { formatUsd } from '../../utils/formatters.js';
@@ -279,8 +279,8 @@ export default function VideoRenderSettings({ videoSettings, generating, project
className={`${falSelectCls} sm:max-w-[280px]`}
>
{musicModels.map((lane) => (
- <option key={lane.modelId} value={lane.modelId === FAL_AUDIO_VIDEO_MODEL ? '' : lane.modelId}>
- {`Music-driven · ${getFalVideoModel(lane.modelId).label} · ${describeFalVideoRate(lane.modelId, lane.defaultResolution)}${lane.modelId === FAL_AUDIO_VIDEO_MODEL ? ' (default)' : ''}`}
+ <option key={lane.modelId} value={lane.modelId === FAL_MUSIC_VIDEO_DEFAULT_MODEL ? '' : lane.modelId}>
+ {`Music-driven · ${getFalVideoModel(lane.modelId).label} · ${describeFalVideoRate(lane.modelId, lane.defaultResolution)}${lane.modelId === FAL_MUSIC_VIDEO_DEFAULT_MODEL ? ' (default)' : ''}`}
</option>
))}
</select>
diff --git a/client/src/hooks/useMusicVideoModelSettings.js b/client/src/hooks/useMusicVideoModelSettings.js
index d9cc8a6f3c..1a36a6986f 100644
--- a/client/src/hooks/useMusicVideoModelSettings.js
+++ b/client/src/hooks/useMusicVideoModelSettings.js
@@ -79,7 +79,7 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =
// false = lip-sync without transcript guidance (null/absent = on).
falLipSyncTranscription: project?.videoSettings?.falLipSyncTranscription ?? null,
// Audio-to-video model / resolution for music-driven performance takes —
- // null means FAL_AUDIO_VIDEO_MODEL (LTX-2.3) / 1080p, resolved by falSceneTake.
+ // null means FAL_MUSIC_VIDEO_DEFAULT_MODEL (MiniMax H3 Max reference) at its own default size, resolved by falSceneTake.
falMusicModelId: project?.videoSettings?.falMusicModelId ?? null,
falMusicResolution: project?.videoSettings?.falMusicResolution ?? null,
generationMode: project?.videoSettings?.generationMode || 'image',
diff --git a/client/src/lib/falVideoModels.js b/client/src/lib/falVideoModels.js
index def3d7e0e3..c55103ffc3 100644
--- a/client/src/lib/falVideoModels.js
+++ b/client/src/lib/falVideoModels.js
@@ -11,10 +11,12 @@ export {
FAL_DEFAULT_IMAGE_VIDEO_MODEL,
FAL_DEFAULT_TEXT_VIDEO_MODEL,
FAL_IMAGE_VIDEO_MODELS,
+ FAL_MUSIC_VIDEO_DEFAULT_MODEL,
FAL_VIDEO_MODELS,
coerceFalVideoSeconds,
describeFalVideoRate,
estimateFalVideoCostUsd,
+ falHearsAudioReference,
falImageVideoQuotes,
falVideoClipQuote,
falVideoResolutions,
diff --git a/server/lib/README.md b/server/lib/README.md
index b03c38ad16..1c1e504977 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -326,7 +326,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `cursor.js` | Cursor Agent (`cursor-agent`) provider helpers — the `CURSOR_COMMAND` binary constant, the `isCursorCommand` predicate, and the `ensureCursorHeadlessArgs` (`--print --force`, folding any pinned reasoning effort into `--model`) / `ensureCursorTuiArgs` (`--force`) argv builders. `--force` is load-bearing beyond approvals: it also clears cursor's workspace-trust gate, which otherwise EXITS a headless run before any work happens. The prompt rides raw stdin (like claude/codex, unlike grok/kimi), and cursor needs no configured-default sentinel — its `auto` router is a real model id passed straight to `--model`. |
| `grok.js` | xAI Grok Build (`grok`) provider helpers — id/endpoint constants (`GROK_API_ID`/`GROK_CLI_ID`/`GROK_TUI_ID`/`GROK_API_ENDPOINT`), `isGrokCommand`/`isGrokCliProvider`/`isGrokTuiProvider` predicates, `ensureGrokHeadlessArgs`/`ensureGrokTuiArgs` argv builders (grok reads its prompt from `--prompt-file /dev/stdin`, not raw stdin; model selection uses the `GROK_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Antigravity), and `prepareGrokPromptFile` (Windows temp-file delivery fallback; `cleanup({ throwOnError: true })` reports cleanup failure to an owning lifecycle). |
| `filmStyleGrammars.js` | Curated film style grammar catalog (`FILM_STYLE_GRAMMARS`, eight code-rendered media with rendering, colour, type, motion, camera, sound and native-move guidance), `getFilmStyleGrammar`, `summarizeFilmStyleGrammar` (picker projection) and `knownFilmStyleIdSchema` (request-field id check) and `renderFilmStyleGrammarPrompt(id, { parts })` — one prompt section capped at `FILM_STYLE_PROMPT_MAX_CHARS`; unknown id or bad parts is a 400. |
-| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, OmniHuman v1.5 audio-to-video (`FAL_AUDIO_VIDEO_MODELS`, kind `a2v`: the supplied audio drives the figure's whole motion; LTX audio-to-video runs locally, never on fal), Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body) (a prompt over the model's `promptMaxChars` is cut at a clause boundary and reported as `promptClipped`), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. |
+| `falVideoModels.js` | Curated fal.ai VIDEO model catalog — `FAL_VIDEO_MODELS` (Hailuo-02 standard t2v/i2v default, MiniMax H3 Max/Max Turbo i2v, H3 reference-to-video, H3 lip-sync, OmniHuman v1.5 audio-to-video (`FAL_AUDIO_VIDEO_MODELS`: kind `a2v`, plus the H3 Max and Seedance reference models that hear a song slice as an audio reference (`falHearsAudioReference()`, default `FAL_MUSIC_VIDEO_DEFAULT_MODEL`); the supplied audio drives the whole picture's motion; LTX audio-to-video runs locally, never on fal), Seedance 2.5 t2v/i2v/reference, Kling v3 Pro i2v, Veo 3.1 Fast i2v) with per-model duration rules (enum/range/audio), resolutions, aspect ratios, optional inputs and list prices verified on `FAL_VIDEO_PRICING_VERIFIED_AT`; `getFalVideoModel()`, `coerceFalVideoSeconds()` (round UP to a covering length), `resolveFalVideoResolution()`, `buildFalVideoRequest()` (the exact queue body per model; null for an uncurated id, which keeps the legacy body) (a prompt over the model's `promptMaxChars` is refused, never cut), `estimateFalVideoCostUsd()` (null = unknown), `describeFalVideoRate()`. Dependency-free; the client re-exports it. |
| `grokVideoClip.js` | Clip lengths grok's `image_to_video` actually delivers — `GROK_VIDEO_DURATIONS` (`[6, 10, 15]`; 6/10 measured in #3022, where a 2s/3s request returned the same 6.04s clip; 15 from xAI's 1–15 s API limit, confirmed per clip by the probe in `videoGen/grok.js`), `GROK_VIDEO_DEFAULT_DURATION`, `resolveGrokDuration()` (validate an already-grok-shaped request, else default), and `nearestGrokDuration()` (round a length authored against another backend's continuous contract UP to the shortest clip that covers it — for translating a CD/commission `targetDurationSeconds`). Dependency-free so `validation.js` / `routes/videoGen.js` can derive their schemas from the same list the services gate on. |
| `fontMetadata.js` | `fontContainer(buf)` (sfnt / woff2 / null) and `fontFamilyName(buf)` — a font file's family name read from its `name` table (TrueType, OpenType, and WOFF2's brotli stream) with no font library; null for anything unreadable (#10345). |
| `musicVideoCoverOverlay.js` | Music Video cover lettering (#10345): `COVER_DESIGN_OPTIONS` / `DEFAULT_COVER_DESIGN`, `normalizeCoverDesign(design, { fonts })` (accepts every stored design; a `font:<id>` typeface only when that uploaded font is listed), `coverOverlaySvg({ title, tag, design, widths, fonts })` and the `coverFontSpec` / `fontAttrs` / `titleLinesFor` / `tagTextFor` helpers the width probe reads. Pure; the server render (`coverArtCompose.js`) and the client's live Lettering preview run this one layout. |
@@ -856,7 +856,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. |
| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit only to a request sent before `musicVideoShotPrompt.js` composed clips. |
-| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip and stills (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, repair notes) as a labeled addition; a take that hears no audio is composed without the look, song timing or on-beat camera clause. `musicVideoFrameModelInput(project, scene, { slot, fromFrame, refine, repairs, fixes })` does the same for a start or end frame (look, frame direction, plate actions, rework line, keep-the-shot line, refine wrapper). `authoredShotPrompt`, `authoredFramePromptText` and `visualDirection`. Shared by the shot card, the board, the production run and the image and video submits. |
+| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip and stills (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, song audio reference, repair notes) as a labeled addition; a take that hears no audio is composed without the look, song timing or on-beat camera clause. `musicVideoFrameModelInput(project, scene, { slot, fromFrame, refine, repairs, fixes })` does the same for a start or end frame (look, frame direction, plate actions, rework line, keep-the-shot line, refine wrapper). `authoredShotPrompt`, `authoredFramePromptText` and `visualDirection`. Shared by the shot card, the board, the production run and the image and video submits. |
| `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. |
| `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |
diff --git a/server/lib/falVideoModels.js b/server/lib/falVideoModels.js
index 05e18cd22d..ee89bc3079 100644
--- a/server/lib/falVideoModels.js
+++ b/server/lib/falVideoModels.js
@@ -40,6 +40,9 @@ export const FAL_DEFAULT_TEXT_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/t
export const FAL_DEFAULT_IMAGE_VIDEO_MODEL = 'fal-ai/minimax/hailuo-02/standard/image-to-video';
export const FAL_LIPSYNC_VIDEO_MODEL = 'minimax/h3-max/lip-sync/image-to-video';
export const FAL_AUDIO_VIDEO_MODEL = 'fal-ai/bytedance/omnihuman/v1.5';
+// The music-driven shot default: a start frame, a prompt and the song slice as
+// an audio reference (OmniHuman's takes were judged worse, 2026-10-13).
+export const FAL_MUSIC_VIDEO_DEFAULT_MODEL = 'minimax/h3-max/reference-to-video';
const H3_RESOLUTIONS = Object.freeze(['480P', '768P', '1080P']);
const SEEDANCE_RESOLUTIONS = Object.freeze(['480p', '720p', '1080p']);
@@ -119,9 +122,15 @@ export const FAL_VIDEO_MODELS = Object.freeze([
id: 'minimax/h3-max/reference-to-video', kind: 'ref2v', label: 'MiniMax H3 Max (reference)', family: 'minimax-h3',
prompt: 'required',
promptMaxChars: 2000,
+ // Reads its inputs from URLs: the frame and song slice are hosted on fal's CDN first.
+ hostedInputs: true,
+ // "The video opens exactly on this image": a true start frame, unlike the references.
+ image: { param: 'image_url', required: false },
+ endImage: { param: 'end_image_url' },
// Referenced in the prompt as "Image 1", "Audio 1", … in list order.
referenceImages: { param: 'reference_image_urls', max: 9 },
- referenceAudio: { param: 'reference_audio_urls', max: 3, minSec: 2, maxSec: 15 },
+ // 2–15 s each and at most 15 s combined.
+ referenceAudio: { param: 'reference_audio_urls', max: 3, minSec: 2, maxSec: 15, promptName: 'Audio 1' },
duration: { kind: 'range', min: 5, max: 15, wire: 'integer', defaultSec: 5 },
resolution: { param: 'resolution', options: H3_RESOLUTIONS, default: '768P' },
aspectRatio: { param: 'aspect_ratio', options: WIDE_ASPECTS, auto: 'adaptive' },
@@ -178,10 +187,12 @@ export const FAL_VIDEO_MODELS = Object.freeze([
pricing: { perSecond: 0.16 },
},
seedance('bytedance/seedance-2.5/reference-to-video', 'ref2v', 'Seedance 2.5 (reference)', {
- // Referenced in the prompt as @Image1 / @Audio1. Audio references let a
- // singer follow supplied audio, but each clip is capped at 30.2s.
- referenceImages: { param: 'image_urls', max: 30 },
- referenceAudio: { param: 'audio_urls', max: 10, minSec: 1.8, maxSec: 30.2 },
+ // Referenced in the prompt as @Image1 / @Audio1. There is no start-frame
+ // field: a PortOS start frame is the first reference image. Audio needs an
+ // image reference beside it, and each clip is capped at 30.2s.
+ hostedInputs: true,
+ referenceImages: { param: 'image_urls', max: 30, promptName: '@Image1' },
+ referenceAudio: { param: 'audio_urls', max: 10, minSec: 1.8, maxSec: 30.2, promptName: '@Audio1' },
aspectRatio: { param: 'aspect_ratio', options: WIDE_ASPECTS, auto: 'auto' },
seed: 'seed',
}),
@@ -233,8 +244,15 @@ export const getFalVideoModel = (modelId) => (typeof modelId === 'string' ? BY_I
*/
export const FAL_IMAGE_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'i2v'));
-/** Models that animate a frame to supplied audio across the whole picture (the music-driven performance lane). */
-export const FAL_AUDIO_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'a2v'));
+/**
+ * Models that move a whole frame to supplied audio (the music-driven
+ * performance lane): an audio-to-video model, or a reference model that hears
+ * the song slice as a reference its prompt names ("Audio 1" / @Audio1).
+ */
+export const FAL_AUDIO_VIDEO_MODELS = Object.freeze(FAL_VIDEO_MODELS.filter((m) => m.kind === 'a2v' || m.referenceAudio));
+
+/** Whether `model` hears supplied audio as a prompt-named reference rather than as its source audio. */
+export const falHearsAudioReference = (model) => Boolean(model?.referenceAudio && !model.audioInput);
/** Resolution choices a model offers (empty when it has no resolution parameter). */
export const falVideoResolutions = (model) => (model?.resolution?.options ? [...model.resolution.options] : []);
@@ -438,13 +456,14 @@ export function buildFalVideoRequest({
if (model.image) {
if (imageUrl) body[model.image.param] = imageUrl;
else if (model.image.required) throw refuse(`${model.label} renders from a start frame — supply a source image`);
- } else if (model.referenceImages) {
- // A PortOS start frame becomes the first reference ("Image 1" / @Image1).
- const refs = [imageUrl, ...(Array.isArray(referenceImageUrls) ? referenceImageUrls : [])].filter(Boolean);
+ } else if (imageUrl && !model.referenceImages) {
+ throw refuse(`${model.label} is text-to-video — it cannot use a source image`);
+ }
+ if (model.referenceImages) {
+ // Without a start-frame field, a PortOS start frame becomes the first reference ("Image 1" / @Image1).
+ const refs = [model.image ? null : imageUrl, ...(Array.isArray(referenceImageUrls) ? referenceImageUrls : [])].filter(Boolean);
if (refs.length > model.referenceImages.max) throw refuse(`${model.label} takes at most ${model.referenceImages.max} reference images`);
if (refs.length) body[model.referenceImages.param] = refs;
- } else if (imageUrl) {
- throw refuse(`${model.label} is text-to-video — it cannot use a source image`);
}
if (endImageUrl) {
if (!model.endImage) throw refuse(`${model.label} takes no end frame`);
diff --git a/server/lib/falVideoModels.test.js b/server/lib/falVideoModels.test.js
index 982b980564..28ea030f15 100644
--- a/server/lib/falVideoModels.test.js
+++ b/server/lib/falVideoModels.test.js
@@ -65,9 +65,12 @@ describe('buildFalVideoRequest — per-family wire bodies', () => {
expect(body.image_urls).toEqual(['data:image/png;base64,AA==', 'https://example.com/b.png']);
expect(body.audio_urls).toEqual(['https://example.com/a.wav']);
expect(body).not.toHaveProperty('image_url');
- expect(build('minimax/h3-max/reference-to-video', { width: 1080, height: 1350 }).body).toMatchObject({
- reference_image_urls: ['data:image/png;base64,AA=='], aspect_ratio: '3:4', duration: 5,
+ // H3 Max reference opens exactly on its start frame; the song slice is "Audio 1".
+ const h3 = build('minimax/h3-max/reference-to-video', { width: 1080, height: 1350, referenceAudioUrls: ['https://example.com/a.wav'] }).body;
+ expect(h3).toMatchObject({
+ image_url: 'data:image/png;base64,AA==', reference_audio_urls: ['https://example.com/a.wav'], aspect_ratio: '3:4', duration: 5,
});
+ expect(h3).not.toHaveProperty('reference_image_urls');
});
it('lip-sync: frame + audio + resolution only — no prompt field even when one is supplied', () => {
diff --git a/server/lib/musicVideoShotPrompt.js b/server/lib/musicVideoShotPrompt.js
index 6940ffc97f..6e0cbb0bbc 100644
--- a/server/lib/musicVideoShotPrompt.js
+++ b/server/lib/musicVideoShotPrompt.js
@@ -15,6 +15,7 @@ import { musicVideoCreativeContext, musicVideoDirectorLook } from './musicVideoC
import { shotCameraPrompt } from './cameraMovements.js';
import { shotActionPrompt } from './musicVideoActionContract.js';
import { approximateMotionCues } from './musicVideoShotTiming.js';
+import { falHearsAudioReference, getFalVideoModel } from './falVideoModels.js';
/**
* The project-wide visual direction: palette and camera rules from the visual
@@ -37,6 +38,18 @@ const addition = (id, label, text) => {
return trimmed ? { id, label, text: trimmed } : null;
};
+/**
+ * What a reference model is told about the song slice it hears (and, where its
+ * start frame is only a reference, that frame), in the names its prompt uses
+ * for them; '' for a model that hears no audio reference.
+ */
+function audioReferencePrompt(modelId) {
+ const model = getFalVideoModel(modelId);
+ if (!falHearsAudioReference(model)) return '';
+ const frame = model.image ? '' : `${model.referenceImages?.promptName || 'Image 1'} is the opening frame. `;
+ return `${frame}${model.referenceAudio.promptName || 'Audio 1'} is the song playing under this shot: time the movement to its beat.`;
+}
+
/**
* `{ prompt, promptAdditions }` for one clip. `authored` is the director's text
* (the shot card's box, or the scene's saved prompt). `asTyped` is the shot
@@ -44,10 +57,12 @@ const addition = (id, label, text) => {
* whether the take hears song audio (lip-sync, supplied song audio or an
* audio-reactive LoRA); `offsetSec` re-times the shot actions to a performance
* take's padded song slice; `fixes` is an autonomous run's repair note.
+ * `audioReferenceModelId` is the fal model of a music-driven take that hears
+ * the song slice as a reference, which its prompt then names.
*/
export function musicVideoShotModelInput(project, scene, {
authored = authoredShotPrompt(scene), asTyped = false, hears = false, backend = null,
- audioReactive = false, offsetSec = 0, fixes = '',
+ audioReactive = false, offsetSec = 0, fixes = '', audioReferenceModelId = null,
} = {}) {
const text = typeof authored === 'string' ? authored.trim() : '';
const look = !asTyped && hears ? [
@@ -66,6 +81,7 @@ export function musicVideoShotModelInput(project, scene, {
const tail = audioReactive
? [addition('audio-reactive-guard', 'Audio-reactive guard', AUDIO_REACTIVE_PERFORMANCE_GUARD)]
: [
+ addition('audio-reference', 'Song audio reference', audioReferenceModelId ? audioReferencePrompt(audioReferenceModelId) : ''),
addition('phrase-timing', 'Phrase timing (approximate)', backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : ''),
addition('repair-notes', 'Repair notes', fixes ? `Fix these problems from the previous take while preserving the shot intent: ${fixes}` : ''),
];
diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.js
index 3e5838be6f..c23b871f5d 100644
--- a/server/lib/musicVideoShotTiming.js
+++ b/server/lib/musicVideoShotTiming.js
@@ -16,8 +16,9 @@
* conditioning — REST documentation for a different Grok surface is not that
* proof. Local runtimes are absent for the same reason.
* - `SOURCE_AUDIO_MUSIC` — the music-driven performance lanes: fal.ai
- * audio-to-video models (kind `a2v`) where the song slice drives the whole
- * picture's motion rather than only a singer's mouth. A scene opts in with
+ * audio-to-video models (kind `a2v`) and reference models that hear the
+ * song slice as an audio reference (MiniMax H3 Max, Seedance 2.5), where
+ * the song drives the whole picture's motion rather than only a singer's mouth. A scene opts in with
* `performanceStyle: 'music'`; it shares the window planning below.
* - `planPerformanceWindow` — the audio window a performance shot submits:
* inside the provider's limits (never relying on its silent truncation), a
@@ -42,10 +43,10 @@
import { GROK_VIDEO_DURATIONS, nearestGrokDuration } from './grokVideoClip.js';
import {
- FAL_AUDIO_VIDEO_MODEL,
FAL_AUDIO_VIDEO_MODELS,
FAL_DEFAULT_IMAGE_VIDEO_MODEL,
FAL_LIPSYNC_VIDEO_MODEL,
+ FAL_MUSIC_VIDEO_DEFAULT_MODEL,
coerceFalVideoSeconds,
describeFalVideoRate,
estimateFalVideoCostUsd,
@@ -159,27 +160,27 @@ export const SOURCE_AUDIO_LIPSYNC = Object.freeze({
local: localAudioLane('lipsync'),
});
-/**
- * Default output resolution of a music-driven take (OmniHuman's own default).
- * LTX audio-to-video is deliberately not a fal lane: PortOS runs LTX-2.3 /
- * LTX-2.5 locally, where Make › Supplied song audio conditions a cutaway on
- * its own song slice (submitJob.js `_suppliedAudio`).
- */
-const MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION = '1080p';
-
+// A music-driven take renders at its model's own default resolution.
+// LTX audio-to-video is deliberately not a fal lane: PortOS runs LTX-2.3 /
+// LTX-2.5 locally, where Make › Supplied song audio conditions a cutaway on
+// its own song slice (submitJob.js `_suppliedAudio`).
const musicCapability = (model) => {
- const resolutions = falVideoResolutions(model);
- const defaultResolution = resolutions.includes(MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION) ? MUSIC_VIDEO_MUSIC_DEFAULT_RESOLUTION : model.resolution.default;
+ const audio = model.audioInput || model.referenceAudio;
+ // A reference model bills and renders whole seconds from its shortest clip
+ // up: the song window starts at that length (less the planning margin, so a
+ // padded window lands exactly on it) and never outruns the longest clip.
+ const clip = model.duration.kind === 'range' ? model.duration : null;
+ const defaultResolution = model.resolution.default;
return Object.freeze({
provider: 'fal',
style: 'music',
label: `fal.ai ${model.label}`,
modelId: model.id,
- minAudioSec: model.audioInput.minSec,
+ minAudioSec: clip ? Math.max(audio.minSec, clip.min - PERFORMANCE_WINDOW_MARGIN_SEC) : audio.minSec,
// The same one-phrase per-take ceiling as lip-sync (see above).
- maxAudioSec: Math.min(model.audioInput.maxSec, 30),
+ maxAudioSec: Math.min(audio.maxSec, clip ? clip.max : Infinity, 30),
transcription: false,
- resolutions: Object.freeze(resolutions),
+ resolutions: Object.freeze(falVideoResolutions(model)),
defaultResolution,
costLabel: `${describeFalVideoRate(model.id, defaultResolution)} — billed to your fal.ai account`,
});
@@ -188,7 +189,7 @@ const musicCapability = (model) => {
/**
* Verified music-driven lanes, keyed by backend then model id. The project's
* `videoSettings.falMusicModelId` picks one; absent (or a model this install
- * does not curate) = `FAL_AUDIO_VIDEO_MODEL` (OmniHuman v1.5).
+ * does not curate) = `FAL_MUSIC_VIDEO_DEFAULT_MODEL` (MiniMax H3 Max reference).
*/
const LOCAL_MUSIC_LANE = localAudioLane('music');
@@ -207,7 +208,7 @@ export function performanceCapability(backend, { scene = null, videoSettings = n
if (backend === 'local') return LOCAL_MUSIC_LANE;
const lanes = SOURCE_AUDIO_MUSIC[backend];
const pinned = videoSettings?.falMusicModelId;
- return (pinned && Object.prototype.hasOwnProperty.call(lanes, pinned) ? lanes[pinned] : null) || lanes[FAL_AUDIO_VIDEO_MODEL];
+ return (pinned && Object.prototype.hasOwnProperty.call(lanes, pinned) ? lanes[pinned] : null) || lanes[FAL_MUSIC_VIDEO_DEFAULT_MODEL];
}
/**
@@ -541,6 +542,7 @@ export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null
const seconds = plan.ok ? plan.windowSec : null;
return {
performance: true,
+ music: capability.style === 'music',
modelId: capability.modelId,
seconds,
resolution,
@@ -563,13 +565,14 @@ export function falSceneTake({ scene, videoSettings = {}, songDurationSec = null
}
/**
- * A `falSceneTake` as `/api/video-gen` fields. A performance take sends only
- * its lip-sync resolution — its length follows the song slice the server cuts;
- * a cutaway names its model, length and resolution. Unknown values stay absent
- * so the server resolves its own defaults (an uncurated model's legacy body).
+ * A `falSceneTake` as `/api/video-gen` fields. A performance take sends its
+ * resolution (and a music-driven take its model, which a shot card can pick) —
+ * its length follows the song slice the server cuts; a cutaway names its
+ * model, length and resolution. Unknown values stay absent so the server
+ * resolves its own defaults (an uncurated model's legacy body).
*/
export const falTakeRequestFields = (take) => (take.performance
- ? (take.resolution ? { falResolution: take.resolution } : {})
+ ? { ...(take.music ? { falModelId: take.modelId } : {}), ...(take.resolution ? { falResolution: take.resolution } : {}) }
: {
falModelId: take.modelId,
...(take.seconds != null ? { falDuration: take.seconds } : {}),
diff --git a/server/lib/musicVideoShotTiming.test.js b/server/lib/musicVideoShotTiming.test.js
index 94c622ae4b..c14bae886a 100644
--- a/server/lib/musicVideoShotTiming.test.js
+++ b/server/lib/musicVideoShotTiming.test.js
@@ -143,21 +143,25 @@ describe('music-driven performance takes', () => {
// The drummer case: a 3.735s shot of the song whose motion the audio drives.
const drummer = { shotMode: 'performance', performanceStyle: 'music', startSec: 212.718, endSec: 216.453 };
- it('render on the audio-to-video model for the shot itself, without lip-sync\'s 5s padding', () => {
+ it('render on H3 Max reference by default, the song window padded to exactly its shortest billed clip', () => {
const take = falSceneTake({ scene: drummer, videoSettings: {}, songDurationSec: 240 });
- expect(take).toMatchObject({ performance: true, modelId: 'fal-ai/bytedance/omnihuman/v1.5', seconds: 3.735, resolution: '1080p', costUsd: 0.5976 });
- expect(falTakeRequestFields(take)).toEqual({ falResolution: '1080p' });
- // The same shot as lip-sync pads to the route's 5s minimum.
+ expect(take).toMatchObject({ performance: true, music: true, modelId: 'minimax/h3-max/reference-to-video', seconds: 5, resolution: '768P', costUsd: 0.4 });
+ // A music-driven take names its model, so a shot card can pick another.
+ expect(falTakeRequestFields(take)).toEqual({ falModelId: 'minimax/h3-max/reference-to-video', falResolution: '768P' });
+ // The same shot as lip-sync pads to the route's 5s minimum plus its margin.
expect(falSceneTake({ scene: { ...drummer, performanceStyle: null }, songDurationSec: 240 }).seconds).toBeCloseTo(5.05, 6);
+ expect(shotSplitLimit(drummer, 'fal')).toBeCloseTo(15 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);
});
it('follow the project pin (model and resolution) and fall back to the default for an uncurated pin', () => {
+ // An audio-to-video model hears the shot itself, with no padding.
const pinned = falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5', falMusicResolution: '720p' }, songDurationSec: 240 });
- expect(pinned).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '720p', costUsd: 0.5976 });
+ expect(pinned).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', seconds: 3.735, resolution: '720p', costUsd: 0.5976 });
+ const seedance = falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'bytedance/seedance-2.5/reference-to-video' }, songDurationSec: 240 });
+ expect(seedance).toMatchObject({ seconds: 4, resolution: '720p', costUsd: 1.892 });
// A pin this install no longer curates (fal LTX-2.3, removed) falls back to the default.
expect(falSceneTake({ scene: drummer, videoSettings: { falMusicModelId: 'fal-ai/ltx-2.3-22b/audio-to-video' }, songDurationSec: 240 }).modelId)
- .toBe('fal-ai/bytedance/omnihuman/v1.5');
- expect(shotSplitLimit(drummer, 'fal')).toBeCloseTo(30 - PERFORMANCE_WINDOW_MARGIN_SEC, 6);
+ .toBe('minimax/h3-max/reference-to-video');
});
});
diff --git a/server/services/musicVideo/performanceShot.js b/server/services/musicVideo/performanceShot.js
index 9c75c6deac..b8e9602542 100644
--- a/server/services/musicVideo/performanceShot.js
+++ b/server/services/musicVideo/performanceShot.js
@@ -263,10 +263,11 @@ export async function assertCurrentPerformanceTakes(project, masterPath) {
* for a performance scene on a capable backend. `resolution` is the take's
* output resolution: the request's own when the lip-sync route offers it, else
* the project's `videoSettings.falLipSyncResolution`, else the capability
- * default (1080P). Throws a 4xx ServerError when the scene is a performance
+ * default (1080P). `musicModelId` is a music-driven model picked for this one
+ * take (ignored unless it is a curated music lane). Throws a 4xx ServerError when the scene is a performance
* shot the backend or its timing cannot deliver.
*/
-export async function preparePerformanceShot({ musicVideo, backend, sourceImagePath, mode, resolution = null }) {
+export async function preparePerformanceShot({ musicVideo, backend, sourceImagePath, mode, resolution = null, musicModelId = null }) {
if (!musicVideo?.projectId || !musicVideo?.sceneId) return null;
const project = await getProject(musicVideo.projectId);
const scene = project?.scenes?.find((s) => s.sceneId === musicVideo.sceneId);
@@ -283,7 +284,9 @@ export async function preparePerformanceShot({ musicVideo, backend, sourceImageP
// A music-driven scene renders on the project's audio-to-video model; a
// lip-sync scene on the lip-sync route.
- const capability = performanceCapability(backend, { scene, videoSettings: project.videoSettings });
+ // A shot card can pick another music-driven model for this take (one PortOS curates, else the project's).
+ const videoSettings = musicModelId ? { ...project.videoSettings, falMusicModelId: musicModelId } : project.videoSettings;
+ const capability = performanceCapability(backend, { scene, videoSettings });
if (!capability) throw refuse(performanceBlockedReason(backend), 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');
const musicDriven = capability.style === 'music';
// The lip-sync route is image + source audio; any other semantic mode would
diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.js
index 1aa5079a0a..4672f67803 100644
--- a/server/services/musicVideo/performanceShot.test.js
+++ b/server/services/musicVideo/performanceShot.test.js
@@ -304,7 +304,7 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
await writeFile(join(PATHS.music, 'vocals.wav'), songWav({ markerAmp: 12000 }));
// A vocal stem is attached, but the drums it leaves out are what drive this shot.
getProject.mockResolvedValue({ ...project({ performanceStyle: 'music', prompt: 'drummer hits on the beat' }), vocalStemFilename: 'vocals.wav',
- videoSettings: { backend: 'fal', falLipSyncTranscription: true } });
+ videoSettings: { backend: 'fal', falLipSyncTranscription: true, falMusicModelId: 'fal-ai/bytedance/omnihuman/v1.5' } });
const prepared = await preparePerformanceShot({
musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'frame.png'), mode: 'image',
});
@@ -368,6 +368,66 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
}, 60_000);
+ it.each([
+ // The default: H3 Max reference opens on the start frame and hears the song as "Audio 1".
+ { musicModelId: null, url: 'minimax/h3-max/reference-to-video', window: { startSec: 18.25, endSec: 23.25, durationSec: 5 },
+ body: { duration: 5, resolution: '768P' }, frame: 'image_url', audio: 'reference_audio_urls', costUsd: 0.4 },
+ // Picked on the shot card: Seedance has no start-frame field, so the frame is @Image1.
+ { musicModelId: 'bytedance/seedance-2.5/reference-to-video', url: 'bytedance/seedance-2.5/reference-to-video',
+ window: { startSec: 18.75, endSec: 22.75, durationSec: 4 }, body: { duration: '4', resolution: '720p', generate_audio: false },
+ frame: 'image_urls', audio: 'audio_urls', costUsd: 1.892 },
+ ])('a music-driven shot on $url sends the start frame, prompt and song slice as an audio reference', async ({ musicModelId, url, window, body, frame, audio, costUsd }) => {
+ getProject.mockResolvedValue({ ...project({ performanceStyle: 'music', prompt: 'drummer hits on the beat' }), videoSettings: { backend: 'fal' } });
+ const prepared = await preparePerformanceShot({
+ musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' }, backend: 'fal', sourceImagePath: join(PATHS.images, 'frame.png'), mode: 'image', musicModelId,
+ });
+ // A 1.5s shot pads to exactly the model's shortest billed clip, centered on the shot.
+ expect(prepared.shotInstruction.audioWindow).toEqual(window);
+ expect(prepared.shotInstruction.edit).toEqual({ inSec: 20 - window.startSec, outSec: 21.5 - window.startSec, targetSec: 1.5 });
+ expect(prepared.modelId).toBe(url);
+
+ await mkdir(PATHS.images, { recursive: true });
+ await writeFile(join(PATHS.images, 'frame.png'), Buffer.from('89504e470d0a1a0a', 'hex'));
+ const deliveredPath = join(PATHS.data, 'delivered-reference.mp4');
+ execFileSync(ffmpeg, ['-v', 'error', '-f', 'lavfi', '-i', `color=blue:s=64x64:r=24:d=${window.durationSec}`, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-y', deliveredPath]);
+ const delivered = await readFile(deliveredPath);
+ const posts = [];
+ const uploads = new Map();
+ vi.stubGlobal('fetch', vi.fn(async (target, opts) => {
+ if (target.startsWith('https://rest.fal.ai/storage/upload/initiate')) {
+ const { file_name: name } = JSON.parse(opts.body);
+ return jsonResponse({ upload_url: `https://upload.example.com/${name}`, file_url: `https://cdn.example.com/${name}` });
+ }
+ if (target.startsWith('https://upload.example.com/')) {
+ uploads.set(target.replace('https://upload.example.com/', 'https://cdn.example.com/'), Buffer.from(opts.body));
+ return { ok: opts.method === 'PUT', status: 200 };
+ }
+ if (opts?.method === 'POST') {
+ posts.push({ url: target, body: JSON.parse(opts.body) });
+ return jsonResponse({ request_id: 'r3', status_url: 'https://queue.fal.run/s3', response_url: 'https://queue.fal.run/r3' });
+ }
+ if (target === 'https://queue.fal.run/s3') return jsonResponse({ status: 'COMPLETED' });
+ if (target === 'https://queue.fal.run/r3') return jsonResponse({ video: { url: 'https://cdn.example.com/ref.mp4' } });
+ if (target === 'https://cdn.example.com/ref.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(delivered).buffer };
+ throw new Error(`unexpected fetch: ${target}`);
+ }));
+ const job = await fal.generateVideo({
+ apiKey: 'test-key', modelId: prepared.modelId, resolution: prepared.resolution, prompt: 'drummer hits on the beat', width: 1920, height: 1080,
+ sourceImagePath: join(PATHS.images, 'frame.png'), audioFilePath: prepared.audioFilePath, shotInstruction: prepared.shotInstruction,
+ });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
+ expect(posts).toHaveLength(1);
+ expect(posts[0].url).toBe(`https://queue.fal.run/${url}`);
+ expect(posts[0].body).toMatchObject({ prompt: 'drummer hits on the beat', ...body });
+ expect(posts[0].body).not.toHaveProperty('audio_url');
+ const start = frame === 'image_urls' ? posts[0].body.image_urls[0] : posts[0].body.image_url;
+ expect(start).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.[a-z]+$/);
+ expect(posts[0].body[audio]).toHaveLength(1);
+ expect(uploads.get(posts[0].body[audio][0])).toEqual(await readFile(prepared.audioFilePath));
+ const { loadHistory } = await import('../videoGen/history.js');
+ expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({ modelId: `fal:${url}`, estimatedCostUsd: costUsd, audioDriven: true });
+ }, 60_000);
+
it('uses the full span of a long shot in one take, at the project\'s lip-sync resolution pin', async () => {
// 26s: past the old 14.8s cap (fal accepts up to 15 minutes; PortOS keeps
// a take to 30s so it stays one editable phrase).
diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.js
index 2fc87f7e9b..7a3bad494e 100644
--- a/server/services/videoGen/fal.js
+++ b/server/services/videoGen/fal.js
@@ -34,6 +34,7 @@ import {
FAL_DEFAULT_IMAGE_VIDEO_MODEL,
FAL_DEFAULT_TEXT_VIDEO_MODEL,
buildFalVideoRequest,
+ falHearsAudioReference,
getFalVideoModel,
} from '../../lib/falVideoModels.js';
import { probeVideoDuration, probeVideoGeometry } from '../../lib/ffmpeg.js';
@@ -151,13 +152,16 @@ function buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, image
export function falVideoSentPrompt({ hasImage = false, hasEndImage = false, hasAudio = false, promptAdditions = [], ...spec }) {
const modelId = spec.modelId || (hasImage ? FAL_DEFAULT_IMAGE_MODEL : FAL_DEFAULT_TEXT_MODEL);
const { prompt = '', negativePrompt } = spec;
+ // A reference model hears the audio as a prompt-named reference ("Audio 1").
+ const audioAsReference = hasAudio && falHearsAudioReference(getFalVideoModel(modelId));
let planned = null;
try {
planned = buildFalVideoRequest({
...spec, modelId,
imageUrl: hasImage ? 'pending:image' : null,
endImageUrl: hasEndImage ? 'pending:end-image' : null,
- audioUrl: hasAudio ? 'pending:audio' : null,
+ audioUrl: hasAudio && !audioAsReference ? 'pending:audio' : null,
+ referenceAudioUrls: audioAsReference ? ['pending:audio'] : [],
});
} catch (err) {
if (err?.code !== 'FAL_MODEL_INPUT') throw err;
@@ -227,8 +231,10 @@ export async function generateVideo({
const audioSec = audioFilePath
? (shotInstruction?.audioWindow?.durationSec || await probeVideoDuration(audioFilePath).catch(() => null))
: null;
+ // A reference model renders a set length, not one that follows its audio: the clip covers the song slice.
+ const audioAsReference = !!audioFilePath && falHearsAudioReference(catalogModel);
const requestSpec = {
- modelId, prompt, negativePrompt, seconds: duration, audioSec, aspectRatio, width, height, resolution,
+ modelId, prompt, negativePrompt, seconds: audioAsReference ? (duration ?? audioSec) : duration, audioSec, aspectRatio, width, height, resolution,
generateAudio: generateAudio === true, enableTranscription: lipSync?.enableTranscription === true,
};
const { planned, record: sentRecord } = falVideoSentPrompt({
@@ -260,7 +266,7 @@ export async function generateVideo({
createdAt: new Date().toISOString(),
mode: sourceImagePath ? 'image' : 'text',
// A lip-sync take syncs a mouth; an audio-to-video take moves the whole frame to the audio.
- ...(audioFilePath ? (catalogModel?.kind === 'a2v' ? { audioDriven: true } : { lipSync: true }) : {}),
+ ...(audioFilePath ? (catalogModel?.kind === 'a2v' || audioAsReference ? { audioDriven: true } : { lipSync: true }) : {}),
...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),
};
const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
@@ -333,8 +339,9 @@ async function runFalVideo(job, jobId, {
const [imageUrl, endImageUrl, audioUrl] = await Promise.all([
host(imageDataUri, 'start'), host(endImageDataUri, 'end'), host(audioDataUri, 'audio'),
]);
+ const audioInputs = falHearsAudioReference(getFalVideoModel(modelId)) ? { referenceAudioUrls: audioUrl ? [audioUrl] : [] } : { audioUrl };
const body = requestSpec
- ? buildFalVideoRequest({ ...requestSpec, imageUrl, endImageUrl, audioUrl }).body
+ ? buildFalVideoRequest({ ...requestSpec, imageUrl, endImageUrl, ...audioInputs }).body
: audioFilePath
? buildLipSyncRequestBody({ imageDataUri: imageUrl, audioDataUri: audioUrl, enableTranscription })
: buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri: imageUrl });
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index 2a18889fa9..ac1b35ca33 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -235,6 +235,7 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
sourceImagePath: prepared.sourceImagePath,
mode: body.mode,
resolution: body.falResolution,
+ musicModelId: backend === VIDEO_GEN_MODE.FAL ? body.falModelId || null : null,
}).catch(async (error) => {
await cleanupStaged();
throw error;
@@ -322,6 +323,8 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
const composed = musicVideoShotModelInput(project, scene, {
authored: params.prompt, asTyped: body.musicVideo.promptAsTyped === true, hears: hearsAudio, backend,
audioReactive: body.musicVideo.audioReactive === true, offsetSec, fixes: body._clipFixes || '',
+ // A music-driven take on a reference model hears the song slice as "Audio 1" / @Audio1.
+ audioReferenceModelId: backend === VIDEO_GEN_MODE.FAL ? performance?.modelId || null : null,
});
params.prompt = composed.prompt;
composedAdditions = composed.promptAdditions;
@@ -377,7 +380,8 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
const { costUsd, modelId: costModelId } = falSceneTake({
scene,
videoSettings: {
- falMusicModelId: project.videoSettings?.falMusicModelId ?? null,
+ falMusicModelId: (performance?.shotInstruction?.performanceStyle === 'music' ? performance.modelId : null)
+ ?? project.videoSettings?.falMusicModelId ?? null,
falModelId: body.falModelId, falDuration: body.falDuration, falResolution: body.falResolution,
falLipSyncResolution: body.falResolution, falMusicResolution: body.falResolution,
},
diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.js
index 436979827b..0406fab32c 100644
--- a/server/services/videoGen/submitJob.test.js
+++ b/server/services/videoGen/submitJob.test.js
@@ -235,6 +235,8 @@ describe('submitVideoGenJob', () => {
}, {});
expect(mocks.preparePerformanceShot).toHaveBeenCalledWith({
musicVideo, backend: 'fal', sourceImagePath: '/example/images/frame.png', mode: 'image', resolution: '1080p',
+ // A music-driven shot's picked model; a lip-sync scene ignores it.
+ musicModelId: 'minimax/h3-max/image-to-video',
});
const { params } = mocks.enqueueJob.mock.calls[0][0];
expect(params).toMatchObject({
@@ -282,6 +284,19 @@ describe('submitVideoGenJob', () => {
expect(hearing.prompt).toContain('Lyric 0.500s–1.500s: hold on');
expect(hearing.prompt).toContain('land the move exactly on the downbeat');
expect(hearing.promptAdditions.map((item) => item.id)).toContain('project-style');
+
+ // A music-driven take on a reference model is told which reference is the song.
+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());
+ mocks.preparePerformanceShot.mockResolvedValue({
+ audioFilePath: '/example/uploads/mv-performance-2.wav', modelId: 'minimax/h3-max/reference-to-video', resolution: '768P',
+ shotInstruction: { version: 2, shotMode: 'performance', performanceStyle: 'music', songInterval: { startSec: 0, endSec: 4 }, edit: { inSec: 0 } },
+ });
+ await submitVideoGenJob({ prompt: 'she turns away', backend: 'fal', mode: 'image', musicVideo: { ...musicVideo, compose: true } }, {});
+ const music = mocks.enqueueJob.mock.calls[2][0].params;
+ expect(music).toMatchObject({ modelId: 'minimax/h3-max/reference-to-video', audioFilePath: '/example/uploads/mv-performance-2.wav' });
+ expect(music.promptAdditions).toContainEqual({ id: 'audio-reference', label: 'Song audio reference',
+ text: 'Audio 1 is the song playing under this shot: time the movement to its beat.' });
+ expect(music.prompt.endsWith('. Audio 1 is the song playing under this shot: time the movement to its beat.')).toBe(true);
});
it('leaves a clip outside a music video at its own aspect', async () => {