Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Offer each shot only the local models it can render, and let Ref2VA lip-sync

Claude817f51aaa35c
diff --git a/client/src/components/musicVideo/SceneRenderTarget.jsx b/client/src/components/musicVideo/SceneRenderTarget.jsx
index 0c42bc9087..3067a3c53e 100644
--- a/client/src/components/musicVideo/SceneRenderTarget.jsx
+++ b/client/src/components/musicVideo/SceneRenderTarget.jsx
@@ -3,7 +3,7 @@ import {
} from '../../lib/falVideoModels.js';
import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
import { grokCoverage } from '../../lib/musicVideoShotTiming.js';
-import { isAudioToVideoRuntime, isLtx2FamilyRuntime } from '../../lib/runnerFamilies';
+import { localMusicVideoModels } from '../../../../server/lib/musicVideoLocalModels.js';
import { frameOptionsForModel } from '../../lib/videoGenParams.js';
/**
@@ -47,10 +47,12 @@ export function renderTargetOptions({ performance, saved, endFrame = false }) {
const isLtx25 = (model) => /^ltx25/.test(model?.runtime || '');
-/** Local models a shot can render on: any image-to-video model, or for lip-sync only the LTX audio-to-video ones. */
-export const localModelChoices = (models, performance) => (performance
- ? models.filter((model) => isLtx2FamilyRuntime(model.runtime) && isAudioToVideoRuntime(model.runtime))
- : models);
+/**
+ * Local models a shot can render on: image-to-video models for a cutaway, and
+ * for a lip-sync or music-driven shot the audio-to-video models PortOS hands
+ * the song slice (LTX-2.x, MiniMax H3 Ref2VA). Same rule the server checks.
+ */
+export const localModelChoices = (models, performance) => localMusicVideoModels(models, performance);
/**
* The local model a shot renders on unless the director picks another: the
diff --git a/client/src/components/musicVideo/SceneRenderTarget.test.jsx b/client/src/components/musicVideo/SceneRenderTarget.test.jsx
new file mode 100644
index 0000000000..a2d7b02045
--- /dev/null
+++ b/client/src/components/musicVideo/SceneRenderTarget.test.jsx
@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest';
+import { render, screen, within } from '@testing-library/react';
+import SceneRenderTarget from './SceneRenderTarget.jsx';
+
+const models = [
+ { id: 'ltx25_example', name: 'LTX-2.5 Example', runtime: 'ltx25', supportedModes: ['text', 'image', 'fflf', 'a2v'] },
+ { id: 'ref2va_example', name: 'Ref2VA Example', runtime: 'minimax_h3_ref2va', supportedModes: ['a2v'] },
+ { id: 'wan_example', name: 'Wan Example', runtime: 'wan22', supportedModes: ['text', 'image'] },
+];
+const modelNames = (performance) => {
+ const { unmount } = render(<SceneRenderTarget id="pick" value="local" performance={performance} saved={{}} models={models}
+ scene={{ sceneId: 's1', startSec: 0, endSec: 4 }} onChange={() => {}} onOptionsChange={() => {}} />);
+ const names = within(screen.getByLabelText('Model')).getAllByRole('option').map((option) => option.textContent);
+ unmount();
+ return names;
+};
+
+// Uniquely catches the shot card offering a model its kind of shot cannot
+// render: an audio-only model for a cutaway, an image-only one for lip-sync.
+describe('the shot card local model picker', () => {
+ it('offers image models for a cutaway and audio-to-video models for a performance shot', () => {
+ expect(modelNames(false)).toEqual(['LTX-2.5 Example', 'Wan Example']);
+ expect(modelNames(true)).toEqual(['LTX-2.5 Example', 'Ref2VA Example']);
+ });
+});
diff --git a/client/src/components/musicVideo/VideoRenderSettings.jsx b/client/src/components/musicVideo/VideoRenderSettings.jsx
index fa8f0224e8..a12a00dd59 100644
--- a/client/src/components/musicVideo/VideoRenderSettings.jsx
+++ b/client/src/components/musicVideo/VideoRenderSettings.jsx
@@ -6,6 +6,7 @@ import {
import { projectFalSpend } from '../../lib/musicVideoStages.js';
import { formatUsd } from '../../utils/formatters.js';
import { SOURCE_AUDIO_LIPSYNC, SOURCE_AUDIO_MUSIC, performanceCapability } from '../../lib/musicVideoShotTiming.js';
+import { localModelRendersCutaway } from '../../../../server/lib/musicVideoLocalModels.js';
const falSelectCls = 'w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 sm:w-auto min-h-[44px] sm:min-h-0';
const falOptionLabel = (model) => {
@@ -106,7 +107,8 @@ export default function VideoRenderSettings({ videoSettings, generating, project
? `Local default · ${models.find((model) => model.id === defaultModel)?.name || defaultModel}`
: 'Local default model'}
</option>
- {(audioReactiveSelected ? audioReactiveModels : models).map((model) => (
+ {/* Cutaways animate their still, so an audio-only model (Ref2VA) is a per-shot pick for performance shots. */}
+ {(audioReactiveSelected ? audioReactiveModels : models.filter((model) => localModelRendersCutaway(model) || model.id === settings.modelId)).map((model) => (
<option key={model.id} value={model.id}>{model.name || model.id}</option>
))}
</select>
diff --git a/server/lib/README.md b/server/lib/README.md
index d5e283546d..1ac30d91de 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -862,6 +862,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `musicVideoTitleCard.js` | The MTV-style opening title card (song, artist, year over the opener): `normalizeTitleCard`, `titleCardVisible`, `titleCardPrefill` (a new card's text and a window ending before the first sung line), and `titleCardRuntimeSource` (the page script the live preview and the document render install to draw it per seek, with its fade and title-safe layout). Browser-safe. |
| `musicVideoDependencies.js` | Versioned take/evidence provenance, dependency comparisons and bounded selective-repair previews. |
| `musicVideoEndFrame.js` | Style-break end frames (#11173): `sceneEndFramePlan(scene, { backend, falModelId })` says whether a clip sends the scene's end frame (only fal models with an end image, never a performance shot) and why not; `sceneEndFrameRequestFields` turns it into `lastImageFile`. Shared by the board and production. |
+| `musicVideoLocalModels.js` | Which local video models render which Music Video shot: `localModelRendersCutaway` (image-to-video), `localModelRendersPerformance` (LTX-2 family a2v or MiniMax H3 Ref2VA, from the still + song slice) and `localMusicVideoModels(models, performance)`. Shared by the shot card and project model pickers and `submitJob.js` / `prepareParams.js`. |
| `musicVideoHandoffFrame.js` | Continuing shots: `sceneClipOutFrameSec(scene, clipDurationSec, fps)` is the time in a shot's clip of the last frame the cut shows (performance edit in/out, loop wrap, hold past the clip's end); `previousShot(project, sceneId)` is the shot before it by start time. |
| `musicVideoCrossLinks.js` | Cross-links between a release's posts: each post's public link, the `Label: url` lines a new draft adds, which links a posted record carries (`links`, or what its draft always linked), and `crossLinkBackfill(kit)` listing the links each editable post (YouTube, Suno, X, Stacker News) still lacks. Shared by the publish payloads and the Publish stage. |
| `musicVideoExcerptRange.js` | `excerptRangeFits(startSec, endSec, totalSec, fps)`: whether a code/document excerpt window fits a frame-quantized render, allowing the end up to one frame past it (scene ends follow the analysed song length; the caller clamps). The start must fall inside the render; an unknown fps allows no slack. |
diff --git a/server/lib/index.js b/server/lib/index.js
index a5cab5f04a..7dda412260 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -840,6 +840,7 @@ export * from './filmLookValidation.js';
export * from './musicVideoTitleCard.js';
export * from './musicVideoDependencies.js';
export * from './musicVideoEndFrame.js';
+export * from './musicVideoLocalModels.js';
export * from './musicVideoHandoffFrame.js';
export * from './musicVideoCrossLinks.js';
export * from './musicVideoExcerptRange.js';
diff --git a/server/lib/musicVideoLocalModels.js b/server/lib/musicVideoLocalModels.js
new file mode 100644
index 0000000000..594fba5b0a
--- /dev/null
+++ b/server/lib/musicVideoLocalModels.js
@@ -0,0 +1,23 @@
+/**
+ * Which local video models can render which Music Video shot. A cutaway
+ * animates its still, so it needs an image-to-video model; a performance shot
+ * (lip-sync or music-driven) renders from its still and the song's slice, so
+ * it needs an audio-to-video model PortOS hands that slice to: the LTX-2
+ * family, or MiniMax H3 Ref2VA, which takes image + audio only. Shared by the
+ * shot card's model picker, the project's renderer picker and the server's
+ * submission checks, so the picker never offers a model the render refuses.
+ */
+import { MINIMAX_H3_REF2VA_RUNTIME, isLtx2FamilyRuntime } from './runners.js';
+import { resolveVideoSupportedModes } from './videoModeProfiles.js';
+
+/** Whether `model` can animate a cutaway's still. */
+export const localModelRendersCutaway = (model) => !!model && resolveVideoSupportedModes(model).includes('image');
+
+/** Whether `model` can render a performance shot from its still and the song slice. */
+export const localModelRendersPerformance = (model) => !!model
+ && resolveVideoSupportedModes(model).includes('a2v')
+ && (isLtx2FamilyRuntime(model.runtime) || model.runtime === MINIMAX_H3_REF2VA_RUNTIME);
+
+/** The local models a shot can render on (performance shots, or cutaways). */
+export const localMusicVideoModels = (models, performance) => (Array.isArray(models) ? models : [])
+ .filter(performance ? localModelRendersPerformance : localModelRendersCutaway);
diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.js
index 18f51f1bf6..84df3cedb6 100644
--- a/server/services/videoGen/prepareParams.js
+++ b/server/services/videoGen/prepareParams.js
@@ -52,6 +52,8 @@ import {
import { getSettings } from '../settings.js';
import { getProject as getMusicVideoProject } from '../musicVideo/projects.js';
import { musicVideoLocalClipSize } from '../../lib/musicVideoAspect.js';
+import { localModelRendersCutaway, localModelRendersPerformance } from '../../lib/musicVideoLocalModels.js';
+import { isPerformanceScene } from '../../lib/musicVideoShotTiming.js';
import { getTrack } from '../tracks/index.js';
import { VIDEO_GEN_MODE, resolveVideoMode } from './modes.js';
import { getFalVideoModel } from '../../lib/falVideoModels.js';
@@ -328,16 +330,39 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys
{ status: 400, code: 'VIDEO_GEN_UNKNOWN_MODEL' },
);
}
- // A local Music Video clip left unsized would render the model's 768x512
- // (3:2) default, cropping its 16:9 still and leaving side gaps in a 16:9
- // video; it renders at the project's aspect instead.
- if (backend === VIDEO_GEN_MODE.LOCAL && body.musicVideo?.projectId && body.width == null && body.height == null) {
- const project = await getMusicVideoProject(body.musicVideo.projectId)
- .catch(async (error) => { await cleanupMultipartTemp(uploads); throw error; });
- if (project) {
+ // A local Music Video shot: sized to the project's aspect, and checked
+ // against the model its kind of shot needs (musicVideoLocalModels.js).
+ let musicVideoContractMode = null;
+ if (backend === VIDEO_GEN_MODE.LOCAL && body.musicVideo?.projectId) {
+ // A clip's own audio never reaches the music video (the song does), so a
+ // model that always renders its audio track is not asked to mute it.
+ if (effectiveModel?.supportsDisableAudio === false) delete body.disableAudio;
+ const unsized = body.width == null && body.height == null;
+ const audioOnlyModel = !!effectiveModel && (!body.mode || body.mode === 'image') && !localModelRendersCutaway(effectiveModel);
+ const project = unsized || audioOnlyModel
+ ? await Promise.resolve().then(() => getMusicVideoProject(body.musicVideo.projectId))
+ .catch(async (error) => { await cleanupMultipartTemp(uploads); throw error; })
+ : null;
+ // Left unsized it would render the model's 768x512 (3:2) default,
+ // cropping its 16:9 still and leaving side gaps in a 16:9 video.
+ if (project && unsized) {
const endFrame = body.mode === 'fflf' || !!body.lastImageFile || !!uploads.lastImage;
Object.assign(body, musicVideoLocalClipSize(project, effectiveModel, { endFrame }));
}
+ const scene = project?.scenes?.find((entry) => entry.sceneId === body.musicVideo.sceneId);
+ if (scene && audioOnlyModel) {
+ // An audio-only model (MiniMax H3 Ref2VA) renders a performance shot as
+ // the a2v job the performance step builds from the still and the song
+ // slice after staging, so its contract is checked as that render.
+ if (isPerformanceScene(scene) && localModelRendersPerformance(effectiveModel)) musicVideoContractMode = 'a2v';
+ else {
+ await cleanupMultipartTemp(uploads);
+ throw new ServerError(
+ `${effectiveModel.name} renders lip-sync and music-driven shots only, from the still and the song. Pick an image-to-video model such as LTX-2.5 for this cutaway.`,
+ { status: 400, code: 'MUSIC_VIDEO_MODEL_NEEDS_PERFORMANCE_SHOT' },
+ );
+ }
+ }
}
validateVideoBatch({ ...body, backend }, effectiveModel);
if (effectiveModel && !isHardwareCompatible(effectiveModel.hardwareCompatibility)) {
@@ -393,7 +418,7 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys
// rejection point at once, and stays correct as new awaits are added.
return withStagedRollback(cleanupStaged, () => resolvePreparedParams({
body, uploads, settings, backend, pythonPath,
- effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged,
+ effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode,
}));
}
@@ -588,7 +613,7 @@ async function resolveAudioFrameCount({ body, audioFilePath, effectiveModel, eff
*/
async function resolvePreparedParams({
body, uploads, settings, backend, pythonPath,
- effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged,
+ effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode = null,
}) {
// Stage a multipart upload into data/uploads so the queue worker can find
// it after a server restart — the OS temp dir gets reaped on reboot, and a
@@ -778,7 +803,7 @@ async function resolvePreparedParams({
// Pinned here rather than re-derived at the resolved pass below: once a
// declared gallery pick fails to resolve, "was this an i2v request?" can only
// be answered from what the caller declared.
- const declaredMode = body.mode || (hasDeclaredFirstImage ? 'image' : 'text');
+ const declaredMode = musicVideoContractMode || body.mode || (hasDeclaredFirstImage ? 'image' : 'text');
const modeContractError = videoModeContractError({
model: effectiveModel,
mode: declaredMode,
diff --git a/server/services/videoGen/prepareParams.test.js b/server/services/videoGen/prepareParams.test.js
index a6efebe552..f48413a734 100644
--- a/server/services/videoGen/prepareParams.test.js
+++ b/server/services/videoGen/prepareParams.test.js
@@ -275,6 +275,29 @@ describe('prepareVideoGenParams', () => {
expect([picked.width, picked.height]).toEqual([768, 512]);
});
+ // Uniquely catches an audio-only model (MiniMax H3 Ref2VA) failing a local
+ // lip-sync shot on the image-mode contract before its song slice exists,
+ // and a cutaway on it failing with the bare mode error instead of the fix.
+ it('lets Ref2VA render a lip-sync shot and points a cutaway at an image model', async () => {
+ listVideoModels.mockReturnValue([{
+ id: 'ref2va_example', name: 'MiniMax H3 Ref2VA', runtime: 'minimax_h3_ref2va', supportedModes: ['a2v'],
+ requiresSourceImageForA2v: true, frameOptions: [124], defaultFrames: 124, supportsDisableAudio: false,
+ }]);
+ const project = (shotMode) => ({ treatment: { brief: { aspectRatio: '16:9' } }, scenes: [{ sceneId: 'scene-example', shotMode }] });
+ const musicVideo = { projectId: 'project-example', sceneId: 'scene-example' };
+ const body = { prompt: 'she sings', backend: 'local', mode: 'image', modelId: 'ref2va_example', sourceImageFile: 'still.png', disableAudio: true, musicVideo };
+
+ getMusicVideoProject.mockResolvedValue(project('performance'));
+ const lipSync = { ...body };
+ const prepared = await prepareVideoGenParams({ body: lipSync, uploads: {}, localOnlyParamKeys: LOCAL_ONLY_KEYS });
+ expect(prepared.backend).toBe('local');
+ expect(lipSync.disableAudio).toBeUndefined();
+
+ getMusicVideoProject.mockResolvedValue(project('cutaway'));
+ await expect(prepareVideoGenParams({ body: { ...body }, uploads: {}, localOnlyParamKeys: LOCAL_ONLY_KEYS }))
+ .rejects.toMatchObject({ code: 'MUSIC_VIDEO_MODEL_NEEDS_PERFORMANCE_SHOT' });
+ });
+
it('short-circuits for grok without staging local-only inputs', async () => {
const prepared = await prepare({ backend: 'grok', sourceImageFile: 'still.png' });
expect(prepared.backend).toBe('grok');
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index d69fd040dc..62eee838cd 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -239,18 +239,20 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
throw error;
})
: null;
- // Local LTX lip-sync: the performance window is the a2v audio, on a frame
+ // Local lip-sync (LTX, or MiniMax H3 Ref2VA): the performance window is the a2v audio, on a frame
// canvas that covers it (the helper pads the tail; the edit points trim it).
let localPerformanceParams = null;
if (performance && backend === 'local') {
- const { sourceAudioVideoRequest, supportsSourceAudioWindow } = await import('../federatedMedia/sourceAudio.js');
+ const [{ sourceAudioVideoRequest }, { localModelRendersPerformance }] = await Promise.all([
+ import('../federatedMedia/sourceAudio.js'), import('../../lib/musicVideoLocalModels.js'),
+ ]);
const fail = async (message, code) => {
await unlink(performance.audioFilePath).catch(() => {});
await cleanupStaged();
throw new ServerError(message, { status: 400, code });
};
- if (!supportsSourceAudioWindow(prepared.effectiveModel)) {
- await fail(`${prepared.effectiveModel?.name || 'This local model'} cannot lip-sync; choose an LTX-2.5 (or LTX-2.3 dgrauet) model for performance shots`, 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');
+ if (!localModelRendersPerformance(prepared.effectiveModel)) {
+ await fail(`${prepared.effectiveModel?.name || 'This local model'} cannot lip-sync; choose LTX-2.5, LTX-2.3 dgrauet or MiniMax H3 Ref2VA for performance shots`, 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');
}
const { audioWindow } = performance.shotInstruction;
const { numFrames } = await Promise.resolve()
diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.js
index 4bb98a7068..65830eea87 100644
--- a/server/services/videoGen/submitJob.test.js
+++ b/server/services/videoGen/submitJob.test.js
@@ -295,6 +295,15 @@ describe('submitVideoGenJob', () => {
expect(params).not.toHaveProperty('lipSync');
});
+ // Uniquely catches MiniMax H3 Ref2VA (image + audio only) being refused
+ // for lip-sync, or sent a frame count off its 17n+5 grid.
+ it('renders the song window on MiniMax H3 Ref2VA at its covering frame count', async () => {
+ const ref2va = { id: 'ref2va-example', name: 'Ref2VA Example', runtime: 'minimax_h3_ref2va', supportedModes: ['a2v'], frameOptions: [107, 124, 141], fpsOptions: [24] };
+ mocks.prepareVideoGenParams.mockResolvedValue(localPrepared({ effectiveModel: ref2va, mode: 'image', sourceImagePath: '/example/images/frame.png' }));
+ await submitVideoGenJob({ prompt: 'singer', backend: 'local', modelId: ref2va.id, mode: 'image', fps: 24, musicVideo }, {});
+ expect(mocks.enqueueJob.mock.calls[0][0].params).toMatchObject({ mode: 'a2v', numFrames: 107, audioFilePath: slice, audioStartSec: 0, disableAudio: false });
+ });
+
it('renders a local cutaway silent unless the song conditions it', async () => {
mocks.preparePerformanceShot.mockResolvedValue(null);
mocks.prepareVideoGenParams.mockResolvedValue(localPrepared({ effectiveModel: ltx, mode: 'image' }));