Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Offer each shot only the local models it can render, and let Ref2VA lip-sync
Claude817f51aaa35c
diff --git a/client/src/components/musicVideo/SceneRenderTarget.jsx b/client/src/components/musicVideo/SceneRenderTarget.jsxindex 0c42bc9087..3067a3c53e 100644--- a/client/src/components/musicVideo/SceneRenderTarget.jsx+++ b/client/src/components/musicVideo/SceneRenderTarget.jsx@@ -3,7 +3,7 @@ import {} from '../../lib/falVideoModels.js';import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';import { grokCoverage } from '../../lib/musicVideoShotTiming.js';-import { isAudioToVideoRuntime, isLtx2FamilyRuntime } from '../../lib/runnerFamilies';+import { localMusicVideoModels } from '../../../../server/lib/musicVideoLocalModels.js';import { frameOptionsForModel } from '../../lib/videoGenParams.js';/**@@ -47,10 +47,12 @@ export function renderTargetOptions({ performance, saved, endFrame = false }) {const isLtx25 = (model) => /^ltx25/.test(model?.runtime || '');-/** Local models a shot can render on: any image-to-video model, or for lip-sync only the LTX audio-to-video ones. */-export const localModelChoices = (models, performance) => (performance- ? models.filter((model) => isLtx2FamilyRuntime(model.runtime) && isAudioToVideoRuntime(model.runtime))- : models);+/**+ * Local models a shot can render on: image-to-video models for a cutaway, and+ * for a lip-sync or music-driven shot the audio-to-video models PortOS hands+ * the song slice (LTX-2.x, MiniMax H3 Ref2VA). Same rule the server checks.+ */+export const localModelChoices = (models, performance) => localMusicVideoModels(models, performance);/*** The local model a shot renders on unless the director picks another: thediff --git a/client/src/components/musicVideo/SceneRenderTarget.test.jsx b/client/src/components/musicVideo/SceneRenderTarget.test.jsxnew file mode 100644index 0000000000..a2d7b02045--- /dev/null+++ b/client/src/components/musicVideo/SceneRenderTarget.test.jsx@@ -0,0 +1,25 @@+import { describe, expect, it } from 'vitest';+import { render, screen, within } from '@testing-library/react';+import SceneRenderTarget from './SceneRenderTarget.jsx';++const models = [+ { id: 'ltx25_example', name: 'LTX-2.5 Example', runtime: 'ltx25', supportedModes: ['text', 'image', 'fflf', 'a2v'] },+ { id: 'ref2va_example', name: 'Ref2VA Example', runtime: 'minimax_h3_ref2va', supportedModes: ['a2v'] },+ { id: 'wan_example', name: 'Wan Example', runtime: 'wan22', supportedModes: ['text', 'image'] },+];+const modelNames = (performance) => {+ const { unmount } = render(<SceneRenderTarget id="pick" value="local" performance={performance} saved={{}} models={models}+ scene={{ sceneId: 's1', startSec: 0, endSec: 4 }} onChange={() => {}} onOptionsChange={() => {}} />);+ const names = within(screen.getByLabelText('Model')).getAllByRole('option').map((option) => option.textContent);+ unmount();+ return names;+};++// Uniquely catches the shot card offering a model its kind of shot cannot+// render: an audio-only model for a cutaway, an image-only one for lip-sync.+describe('the shot card local model picker', () => {+ it('offers image models for a cutaway and audio-to-video models for a performance shot', () => {+ expect(modelNames(false)).toEqual(['LTX-2.5 Example', 'Wan Example']);+ expect(modelNames(true)).toEqual(['LTX-2.5 Example', 'Ref2VA Example']);+ });+});diff --git a/client/src/components/musicVideo/VideoRenderSettings.jsx b/client/src/components/musicVideo/VideoRenderSettings.jsxindex fa8f0224e8..a12a00dd59 100644--- a/client/src/components/musicVideo/VideoRenderSettings.jsx+++ b/client/src/components/musicVideo/VideoRenderSettings.jsx@@ -6,6 +6,7 @@ import {import { projectFalSpend } from '../../lib/musicVideoStages.js';import { formatUsd } from '../../utils/formatters.js';import { SOURCE_AUDIO_LIPSYNC, SOURCE_AUDIO_MUSIC, performanceCapability } from '../../lib/musicVideoShotTiming.js';+import { localModelRendersCutaway } from '../../../../server/lib/musicVideoLocalModels.js';const falSelectCls = 'w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 sm:w-auto min-h-[44px] sm:min-h-0';const falOptionLabel = (model) => {@@ -106,7 +107,8 @@ export default function VideoRenderSettings({ videoSettings, generating, project? `Local default · ${models.find((model) => model.id === defaultModel)?.name || defaultModel}`: 'Local default model'}</option>- {(audioReactiveSelected ? audioReactiveModels : models).map((model) => (+ {/* Cutaways animate their still, so an audio-only model (Ref2VA) is a per-shot pick for performance shots. */}+ {(audioReactiveSelected ? audioReactiveModels : models.filter((model) => localModelRendersCutaway(model) || model.id === settings.modelId)).map((model) => (<option key={model.id} value={model.id}>{model.name || model.id}</option>))}</select>diff --git a/server/lib/README.md b/server/lib/README.mdindex d5e283546d..1ac30d91de 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -862,6 +862,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `musicVideoTitleCard.js` | The MTV-style opening title card (song, artist, year over the opener): `normalizeTitleCard`, `titleCardVisible`, `titleCardPrefill` (a new card's text and a window ending before the first sung line), and `titleCardRuntimeSource` (the page script the live preview and the document render install to draw it per seek, with its fade and title-safe layout). Browser-safe. || `musicVideoDependencies.js` | Versioned take/evidence provenance, dependency comparisons and bounded selective-repair previews. || `musicVideoEndFrame.js` | Style-break end frames (#11173): `sceneEndFramePlan(scene, { backend, falModelId })` says whether a clip sends the scene's end frame (only fal models with an end image, never a performance shot) and why not; `sceneEndFrameRequestFields` turns it into `lastImageFile`. Shared by the board and production. |+| `musicVideoLocalModels.js` | Which local video models render which Music Video shot: `localModelRendersCutaway` (image-to-video), `localModelRendersPerformance` (LTX-2 family a2v or MiniMax H3 Ref2VA, from the still + song slice) and `localMusicVideoModels(models, performance)`. Shared by the shot card and project model pickers and `submitJob.js` / `prepareParams.js`. || `musicVideoHandoffFrame.js` | Continuing shots: `sceneClipOutFrameSec(scene, clipDurationSec, fps)` is the time in a shot's clip of the last frame the cut shows (performance edit in/out, loop wrap, hold past the clip's end); `previousShot(project, sceneId)` is the shot before it by start time. || `musicVideoCrossLinks.js` | Cross-links between a release's posts: each post's public link, the `Label: url` lines a new draft adds, which links a posted record carries (`links`, or what its draft always linked), and `crossLinkBackfill(kit)` listing the links each editable post (YouTube, Suno, X, Stacker News) still lacks. Shared by the publish payloads and the Publish stage. || `musicVideoExcerptRange.js` | `excerptRangeFits(startSec, endSec, totalSec, fps)`: whether a code/document excerpt window fits a frame-quantized render, allowing the end up to one frame past it (scene ends follow the analysed song length; the caller clamps). The start must fall inside the render; an unknown fps allows no slack. |diff --git a/server/lib/index.js b/server/lib/index.jsindex a5cab5f04a..7dda412260 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -840,6 +840,7 @@ export * from './filmLookValidation.js';export * from './musicVideoTitleCard.js';export * from './musicVideoDependencies.js';export * from './musicVideoEndFrame.js';+export * from './musicVideoLocalModels.js';export * from './musicVideoHandoffFrame.js';export * from './musicVideoCrossLinks.js';export * from './musicVideoExcerptRange.js';diff --git a/server/lib/musicVideoLocalModels.js b/server/lib/musicVideoLocalModels.jsnew file mode 100644index 0000000000..594fba5b0a--- /dev/null+++ b/server/lib/musicVideoLocalModels.js@@ -0,0 +1,23 @@+/**+ * Which local video models can render which Music Video shot. A cutaway+ * animates its still, so it needs an image-to-video model; a performance shot+ * (lip-sync or music-driven) renders from its still and the song's slice, so+ * it needs an audio-to-video model PortOS hands that slice to: the LTX-2+ * family, or MiniMax H3 Ref2VA, which takes image + audio only. Shared by the+ * shot card's model picker, the project's renderer picker and the server's+ * submission checks, so the picker never offers a model the render refuses.+ */+import { MINIMAX_H3_REF2VA_RUNTIME, isLtx2FamilyRuntime } from './runners.js';+import { resolveVideoSupportedModes } from './videoModeProfiles.js';++/** Whether `model` can animate a cutaway's still. */+export const localModelRendersCutaway = (model) => !!model && resolveVideoSupportedModes(model).includes('image');++/** Whether `model` can render a performance shot from its still and the song slice. */+export const localModelRendersPerformance = (model) => !!model+ && resolveVideoSupportedModes(model).includes('a2v')+ && (isLtx2FamilyRuntime(model.runtime) || model.runtime === MINIMAX_H3_REF2VA_RUNTIME);++/** The local models a shot can render on (performance shots, or cutaways). */+export const localMusicVideoModels = (models, performance) => (Array.isArray(models) ? models : [])+ .filter(performance ? localModelRendersPerformance : localModelRendersCutaway);diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.jsindex 18f51f1bf6..84df3cedb6 100644--- a/server/services/videoGen/prepareParams.js+++ b/server/services/videoGen/prepareParams.js@@ -52,6 +52,8 @@ import {import { getSettings } from '../settings.js';import { getProject as getMusicVideoProject } from '../musicVideo/projects.js';import { musicVideoLocalClipSize } from '../../lib/musicVideoAspect.js';+import { localModelRendersCutaway, localModelRendersPerformance } from '../../lib/musicVideoLocalModels.js';+import { isPerformanceScene } from '../../lib/musicVideoShotTiming.js';import { getTrack } from '../tracks/index.js';import { VIDEO_GEN_MODE, resolveVideoMode } from './modes.js';import { getFalVideoModel } from '../../lib/falVideoModels.js';@@ -328,16 +330,39 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys{ status: 400, code: 'VIDEO_GEN_UNKNOWN_MODEL' },);}- // A local Music Video clip left unsized would render the model's 768x512- // (3:2) default, cropping its 16:9 still and leaving side gaps in a 16:9- // video; it renders at the project's aspect instead.- if (backend === VIDEO_GEN_MODE.LOCAL && body.musicVideo?.projectId && body.width == null && body.height == null) {- const project = await getMusicVideoProject(body.musicVideo.projectId)- .catch(async (error) => { await cleanupMultipartTemp(uploads); throw error; });- if (project) {+ // A local Music Video shot: sized to the project's aspect, and checked+ // against the model its kind of shot needs (musicVideoLocalModels.js).+ let musicVideoContractMode = null;+ if (backend === VIDEO_GEN_MODE.LOCAL && body.musicVideo?.projectId) {+ // A clip's own audio never reaches the music video (the song does), so a+ // model that always renders its audio track is not asked to mute it.+ if (effectiveModel?.supportsDisableAudio === false) delete body.disableAudio;+ const unsized = body.width == null && body.height == null;+ const audioOnlyModel = !!effectiveModel && (!body.mode || body.mode === 'image') && !localModelRendersCutaway(effectiveModel);+ const project = unsized || audioOnlyModel+ ? await Promise.resolve().then(() => getMusicVideoProject(body.musicVideo.projectId))+ .catch(async (error) => { await cleanupMultipartTemp(uploads); throw error; })+ : null;+ // Left unsized it would render the model's 768x512 (3:2) default,+ // cropping its 16:9 still and leaving side gaps in a 16:9 video.+ if (project && unsized) {const endFrame = body.mode === 'fflf' || !!body.lastImageFile || !!uploads.lastImage;Object.assign(body, musicVideoLocalClipSize(project, effectiveModel, { endFrame }));}+ const scene = project?.scenes?.find((entry) => entry.sceneId === body.musicVideo.sceneId);+ if (scene && audioOnlyModel) {+ // An audio-only model (MiniMax H3 Ref2VA) renders a performance shot as+ // the a2v job the performance step builds from the still and the song+ // slice after staging, so its contract is checked as that render.+ if (isPerformanceScene(scene) && localModelRendersPerformance(effectiveModel)) musicVideoContractMode = 'a2v';+ else {+ await cleanupMultipartTemp(uploads);+ throw new ServerError(+ `${effectiveModel.name} renders lip-sync and music-driven shots only, from the still and the song. Pick an image-to-video model such as LTX-2.5 for this cutaway.`,+ { status: 400, code: 'MUSIC_VIDEO_MODEL_NEEDS_PERFORMANCE_SHOT' },+ );+ }+ }}validateVideoBatch({ ...body, backend }, effectiveModel);if (effectiveModel && !isHardwareCompatible(effectiveModel.hardwareCompatibility)) {@@ -393,7 +418,7 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys// rejection point at once, and stays correct as new awaits are added.return withStagedRollback(cleanupStaged, () => resolvePreparedParams({body, uploads, settings, backend, pythonPath,- effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged,+ effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode,}));}@@ -588,7 +613,7 @@ async function resolveAudioFrameCount({ body, audioFilePath, effectiveModel, eff*/async function resolvePreparedParams({body, uploads, settings, backend, pythonPath,- effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged,+ effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode = null,}) {// Stage a multipart upload into data/uploads so the queue worker can find// it after a server restart — the OS temp dir gets reaped on reboot, and a@@ -778,7 +803,7 @@ async function resolvePreparedParams({// Pinned here rather than re-derived at the resolved pass below: once a// declared gallery pick fails to resolve, "was this an i2v request?" can only// be answered from what the caller declared.- const declaredMode = body.mode || (hasDeclaredFirstImage ? 'image' : 'text');+ const declaredMode = musicVideoContractMode || body.mode || (hasDeclaredFirstImage ? 'image' : 'text');const modeContractError = videoModeContractError({model: effectiveModel,mode: declaredMode,diff --git a/server/services/videoGen/prepareParams.test.js b/server/services/videoGen/prepareParams.test.jsindex a6efebe552..f48413a734 100644--- a/server/services/videoGen/prepareParams.test.js+++ b/server/services/videoGen/prepareParams.test.js@@ -275,6 +275,29 @@ describe('prepareVideoGenParams', () => {expect([picked.width, picked.height]).toEqual([768, 512]);});+ // Uniquely catches an audio-only model (MiniMax H3 Ref2VA) failing a local+ // lip-sync shot on the image-mode contract before its song slice exists,+ // and a cutaway on it failing with the bare mode error instead of the fix.+ it('lets Ref2VA render a lip-sync shot and points a cutaway at an image model', async () => {+ listVideoModels.mockReturnValue([{+ id: 'ref2va_example', name: 'MiniMax H3 Ref2VA', runtime: 'minimax_h3_ref2va', supportedModes: ['a2v'],+ requiresSourceImageForA2v: true, frameOptions: [124], defaultFrames: 124, supportsDisableAudio: false,+ }]);+ const project = (shotMode) => ({ treatment: { brief: { aspectRatio: '16:9' } }, scenes: [{ sceneId: 'scene-example', shotMode }] });+ const musicVideo = { projectId: 'project-example', sceneId: 'scene-example' };+ const body = { prompt: 'she sings', backend: 'local', mode: 'image', modelId: 'ref2va_example', sourceImageFile: 'still.png', disableAudio: true, musicVideo };++ getMusicVideoProject.mockResolvedValue(project('performance'));+ const lipSync = { ...body };+ const prepared = await prepareVideoGenParams({ body: lipSync, uploads: {}, localOnlyParamKeys: LOCAL_ONLY_KEYS });+ expect(prepared.backend).toBe('local');+ expect(lipSync.disableAudio).toBeUndefined();++ getMusicVideoProject.mockResolvedValue(project('cutaway'));+ await expect(prepareVideoGenParams({ body: { ...body }, uploads: {}, localOnlyParamKeys: LOCAL_ONLY_KEYS }))+ .rejects.toMatchObject({ code: 'MUSIC_VIDEO_MODEL_NEEDS_PERFORMANCE_SHOT' });+ });+it('short-circuits for grok without staging local-only inputs', async () => {const prepared = await prepare({ backend: 'grok', sourceImageFile: 'still.png' });expect(prepared.backend).toBe('grok');diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex d69fd040dc..62eee838cd 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -239,18 +239,20 @@ const submitValidatedVideoGenJob = async (body, uploads) => {throw error;}): null;- // Local LTX lip-sync: the performance window is the a2v audio, on a frame+ // Local lip-sync (LTX, or MiniMax H3 Ref2VA): the performance window is the a2v audio, on a frame// canvas that covers it (the helper pads the tail; the edit points trim it).let localPerformanceParams = null;if (performance && backend === 'local') {- const { sourceAudioVideoRequest, supportsSourceAudioWindow } = await import('../federatedMedia/sourceAudio.js');+ const [{ sourceAudioVideoRequest }, { localModelRendersPerformance }] = await Promise.all([+ import('../federatedMedia/sourceAudio.js'), import('../../lib/musicVideoLocalModels.js'),+ ]);const fail = async (message, code) => {await unlink(performance.audioFilePath).catch(() => {});await cleanupStaged();throw new ServerError(message, { status: 400, code });};- if (!supportsSourceAudioWindow(prepared.effectiveModel)) {- await fail(`${prepared.effectiveModel?.name || 'This local model'} cannot lip-sync; choose an LTX-2.5 (or LTX-2.3 dgrauet) model for performance shots`, 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');+ if (!localModelRendersPerformance(prepared.effectiveModel)) {+ await fail(`${prepared.effectiveModel?.name || 'This local model'} cannot lip-sync; choose LTX-2.5, LTX-2.3 dgrauet or MiniMax H3 Ref2VA for performance shots`, 'MUSIC_VIDEO_PERFORMANCE_UNSUPPORTED');}const { audioWindow } = performance.shotInstruction;const { numFrames } = await Promise.resolve()diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.jsindex 4bb98a7068..65830eea87 100644--- a/server/services/videoGen/submitJob.test.js+++ b/server/services/videoGen/submitJob.test.js@@ -295,6 +295,15 @@ describe('submitVideoGenJob', () => {expect(params).not.toHaveProperty('lipSync');});+ // Uniquely catches MiniMax H3 Ref2VA (image + audio only) being refused+ // for lip-sync, or sent a frame count off its 17n+5 grid.+ it('renders the song window on MiniMax H3 Ref2VA at its covering frame count', async () => {+ const ref2va = { id: 'ref2va-example', name: 'Ref2VA Example', runtime: 'minimax_h3_ref2va', supportedModes: ['a2v'], frameOptions: [107, 124, 141], fpsOptions: [24] };+ mocks.prepareVideoGenParams.mockResolvedValue(localPrepared({ effectiveModel: ref2va, mode: 'image', sourceImagePath: '/example/images/frame.png' }));+ await submitVideoGenJob({ prompt: 'singer', backend: 'local', modelId: ref2va.id, mode: 'image', fps: 24, musicVideo }, {});+ expect(mocks.enqueueJob.mock.calls[0][0].params).toMatchObject({ mode: 'a2v', numFrames: 107, audioFilePath: slice, audioStartSec: 0, disableAudio: false });+ });+it('renders a local cutaway silent unless the song conditions it', async () => {mocks.preparePerformanceShot.mockResolvedValue(null);mocks.prepareVideoGenParams.mockResolvedValue(localPrepared({ effectiveModel: ltx, mode: 'image' }));