Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Music Video checks a video submission runs as its last steps before the queue
* write: the production and brief gates, the shot prompt, timing and action
* contract guards, dependency capture and the board cost stamp. Staging and
* FableLoom compilation can take real time, so everything here re-reads the
* project and refuses a submission whose scene or song moved meanwhile.
* Loaded on demand by `videoGen/submitJob.js`, only for a tagged submission.
*/
import { basename } from 'path';
import { ServerError } from '../../lib/errorHandler.js';
import { musicVideoBriefAllowsVideo } from '../../lib/musicVideoMediumPlan.js';
import { assertShotActionContract, withShotActionPrompt } from '../../lib/musicVideoActionContract.js';
import { captureTakeDependencies, musicVideoDependencyChanges } from '../../lib/musicVideoDependencies.js';
import { silentMotionPrompt } from '../../lib/musicVideoMotion.js';
import { musicVideoCreativeContext } from '../../lib/musicVideoCreativeContext.js';
import { musicVideoShotModelInput } from '../../lib/musicVideoShotPrompt.js';
import { falSceneTake, runwaySceneTake } from '../../lib/musicVideoShotTiming.js';
import { VIDEO_GEN_MODE } from '../videoGen/modes.js';
const SONG_SAMPLE_RATE = 48000;
const refuse = (status, code, message) => { throw new ServerError(message, { status, code }); };
/** A Music Video motion prompt for a take that hears no audio, with the project's look clauses removed. */
async function silentMusicVideoMotionPrompt(prompt, project, scene) {
const { visualDirection } = await import('./handoff.js');
return silentMotionPrompt(prompt, { styleParts: [project?.concept?.style, musicVideoCreativeContext(project?.concept, { moodBoard: false }),
visualDirection(project?.visualSpec)].filter(Boolean), keep: [scene?.direction?.motionClause] });
}
/** Rewrites `params.prompt` for the shot and returns the labeled additions the composer put around it. */
async function composeShotPrompt({ body, params, project, scene, prepared, performance, backend }) {
const { musicVideo } = body;
// A take that hears nothing (no lip-sync or supplied song audio) gets plain physical
// motion: the model would act out song cues and look words instead of reading them.
const hearsAudio = !!(performance || prepared.audio || params.audioConditioning || params.audioFilePath);
const offsetSec = performance?.shotInstruction?.edit?.inSec || 0;
if (musicVideo.compose) {
// The director's text plus each labeled block PortOS adds (#11317), from the same
// composer the shot card shows; a take that hears nothing gets no look or song timing.
const composed = musicVideoShotModelInput(project, scene, {
authored: params.prompt, asTyped: musicVideo.promptAsTyped === true, hears: hearsAudio, backend,
audioReactive: musicVideo.audioReactive === true, offsetSec, fixes: body._clipFixes || '',
// A music-driven take on a reference model hears the song slice as "Audio 1" / @Audio1.
audioReferenceModelId: backend === VIDEO_GEN_MODE.FAL ? performance?.modelId || null : null,
});
params.prompt = composed.prompt;
return composed.promptAdditions;
}
// A client from before the composer sends its prompt already composed.
// A shot card's prompt is sent as shown, its shot actions included; only a performance
// take re-times them to its padded song slice.
if (!musicVideo.promptAsTyped || performance) {
params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec });
}
if (!hearsAudio && !musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);
return [];
}
function assertShotTiming({ params, scene, performance, prepared }) {
if (params.audioConditioning && Math.round(scene?.startSec * SONG_SAMPLE_RATE) !== params.audioConditioning.startSample) {
refuse(409, 'MUSIC_VIDEO_SHOT_TIMING_CHANGED', 'Scene timing changed while source audio was prepared');
}
const interval = performance?.shotInstruction?.songInterval || prepared.audio?.songInterval;
if (interval && (interval.startSec !== scene?.startSec || interval.endSec !== scene?.endSec)) {
refuse(409, 'MUSIC_VIDEO_SHOT_TIMING_CHANGED', 'The shot timing changed while its performance audio was prepared — submit again');
}
}
/** Refuses a provider clip too short for the shot's last action, and stamps the contract on the take. */
function applyActionContract({ params, scene, performance }) {
const contract = scene?.direction?.actionContract;
if (contract == null) return;
const offsetSec = performance?.shotInstruction?.edit?.inSec || 0;
const requestedSec = performance?.shotInstruction?.audioWindow?.durationSec ?? params.duration
?? (Number.isFinite(params.numFrames) ? (params.numFrames - 1) / (params.fps || 24) : null);
const lastEventSec = Math.max(0, ...['actions', 'reactions'].flatMap((key) => (contract[key] || []).map((event) => event.endSec))) + offsetSec;
if (Number.isFinite(requestedSec) && lastEventSec > requestedSec) {
refuse(422, 'MUSIC_VIDEO_ACTION_CONTRACT_INVALID', 'Shot actions do not fit inside the requested provider clip — increase its duration');
}
params.shotInstruction = {
...(params.shotInstruction || { version: 1, shotMode: scene.shotMode || 'cutaway', songInterval: { startSec: scene.startSec, endSec: scene.endSec } }),
actionContract: structuredClone(contract),
};
}
async function assertSourceAudioCurrent({ project, prepared }) {
if (!prepared.audio) return;
if (musicVideoDependencyChanges(project, prepared.audio.audioDependencies).length) {
refuse(409, 'MUSIC_VIDEO_AUDIO_SOURCE_CHANGED', 'The source song changed while supplied audio was prepared');
}
const { assertCurrentSourceAudioWindow } = await import('../federatedMedia/sourceAudio.js');
await assertCurrentSourceAudioWindow(project, prepared.audio.audioConditioning);
}
function captureDependencies({ body, params, scene, prepared }) {
if (!scene) return;
// A style-break end frame (#11173) is a dependency too, so changing it stales the clip.
const endImageId = body.lastImageFile && body.lastImageFile === scene.endImageId ? scene.endImageId : null;
params.musicVideoDependencies = captureTakeDependencies(scene, prepared.sourceImagePath ? basename(prepared.sourceImagePath) : null, endImageId);
if (prepared.audio) params.musicVideoDependencies.references.push(...prepared.audio.audioDependencies.references);
}
// The priced hosted backends: each quotes the take it would render for a scene.
const SCENE_TAKE_BY_BACKEND = {
[VIDEO_GEN_MODE.RUNWAY]: ({ body, scene }) => runwaySceneTake({
scene,
videoSettings: { runwayModelId: body.runwayModelId, runwayDuration: body.runwayDuration, runwayResolution: body.runwayResolution },
}),
[VIDEO_GEN_MODE.FAL]: ({ body, scene, project, performance }) => falSceneTake({
scene,
videoSettings: {
falMusicModelId: (performance?.shotInstruction?.performanceStyle === 'music' ? performance.modelId : null)
?? project.videoSettings?.falMusicModelId ?? null,
falModelId: body.falModelId, falDuration: body.falDuration, falResolution: body.falResolution,
falLipSyncResolution: body.falResolution, falMusicResolution: body.falResolution,
},
songDurationSec: project.audioAnalysis?.durationSec ?? null,
}),
};
/**
* #10157: a board-started (manual or auto-review) hosted clip records the
* estimate it was priced at so project spend counts it. A production-run step
* is already charged to the run's own usage, so it is skipped.
*/
function stampSceneTakeCost({ body, params, scene, project, performance, backend }) {
const quote = SCENE_TAKE_BY_BACKEND[backend];
if (!scene || !quote || body.musicVideo.productionRunId) return;
const { costUsd, modelId } = quote({ body, scene, project, performance });
if (Number.isFinite(costUsd)) Object.assign(params, { musicVideoCostUsd: costUsd, musicVideoCostModelId: modelId });
}
/**
* Applies a tagged submission's guards to the job `params` about to be queued:
* rewrites the prompt, stamps the shot instruction, dependencies and cost, and
* throws a coded `ServerError` when the project no longer matches what was
* prepared. Returns the labeled prompt additions the composer produced.
*/
export async function applyMusicVideoSubmissionGuards({ body, params, prepared, performance, backend }) {
const { musicVideo } = body;
if (musicVideo.productionRunId) {
const { assertProductionSubmission } = await import('./productionService.js');
await assertProductionSubmission(musicVideo.projectId, musicVideo.productionRunId,
musicVideo.productionStepKey, { sceneId: musicVideo.sceneId, kind: 'video' });
}
const { getProject } = await import('./projects.js');
const project = await getProject(musicVideo.projectId);
// A brief that names tools and no video tool never dispatches generated footage;
// an approved plan or sheet stays as saved until the director replans.
if (project && !musicVideoBriefAllowsVideo(project)) {
refuse(409, 'MUSIC_VIDEO_NO_VIDEO_TOOL', 'No video tool is selected in this project\'s brief, so generated footage is not dispatched. Add a video tool, or replan the project without generated video.');
}
const scene = project?.scenes?.find((entry) => entry.sceneId === musicVideo.sceneId);
assertShotActionContract(scene);
const composedAdditions = await composeShotPrompt({ body, params, project, scene, prepared, performance, backend });
assertShotTiming({ params, scene, performance, prepared });
applyActionContract({ params, scene, performance });
await assertSourceAudioCurrent({ project, prepared });
captureDependencies({ body, params, scene, prepared });
stampSceneTakeCost({ body, params, scene, project, performance, backend });
return composedAdditions;
}