Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Music video clips list what PortOS adds to the director's prompt
Claudeeb71caf9092c
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsxindex bf91b363cc..fb04ad1c8e 100644--- a/client/src/components/musicVideo/SceneCard.jsx+++ b/client/src/components/musicVideo/SceneCard.jsx@@ -201,7 +201,7 @@ export default function SceneCard({if (videoRefusal) return;const { songAudio, endFrame } = cardRequest;const card = {- prompt: cardRequest.prompt,+ prompt: cardRequest.authored,...(endFrame ? { endFrame: endFrame.state === 'on' } : {}),...(songAudio.state === 'on' || songAudio.state === 'off' ? { songAudio: songAudio.state === 'on' } : {}),};diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsxindex 54640becd9..875c0e57b8 100644--- a/client/src/components/musicVideo/SceneRenderInputs.jsx+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx@@ -1,10 +1,10 @@import { getFalVideoModel } from '../../lib/falVideoModels.js';-import { AUDIO_REACTIVE_PERFORMANCE_GUARD, silentMotionPrompt } from '../../lib/musicVideoMotion.js';-import { approximateMotionCues, falSceneTake } from '../../lib/musicVideoShotTiming.js';-import { shotActionContractProblem, shotActionPrompt } from '../../../../server/lib/musicVideoActionContract.js';-import { shotCameraPrompt } from '../../../../server/lib/cameraMovements.js';+import { falSceneTake } from '../../lib/musicVideoShotTiming.js';+import { shotActionContractProblem } from '../../../../server/lib/musicVideoActionContract.js';+import { authoredShotPrompt, musicVideoShotModelInput } from '../../../../server/lib/musicVideoShotPrompt.js';import { localModelTakesSongAudio } from '../../../../server/lib/musicVideoLocalModels.js';import { formatCount } from '../../utils/formatters.js';+import SentPromptPanel from '../media/SentPromptPanel';const clock = (sec) => {const whole = Math.max(0, sec || 0);@@ -13,17 +13,18 @@ const clock = (sec) => {};/**- * Exactly what the shot card's Render sends: the prompt as typed, then what- * the card itself sets (its camera move, its applied shot actions, Grok's- * approximate phrase timing, the audio-reactive guard), and which inputs go- * with it. The project's style and bible are not added, and the server- * receives `promptAsTyped` and leaves the prompt alone (a performance shot's- * action times shift to its padded song slice).+ * Exactly what the shot card's Render sends: the director's text, then what+ * the card itself adds (its camera move, its applied shot actions, Grok's+ * approximate phrase timing, the audio-reactive guard) as labeled additions,+ * and which inputs go with it. The composer is the server's own+ * (lib/musicVideoShotPrompt.js): Render sends the director's text and the+ * server composes the same prompt (a performance shot's action times shift to+ * its padded song slice). The project's style and bible are not added.** `settings` is the renderer this render uses (the project's, or the card's* Render on pick merged over it); `inputs` holds the card's toggles,* `{ endFrame, songAudio }`, each undefined until the director flips it.- * Returns `{ prompt, promptUsed, maxChars, endFrame, songAudio, contractProblem }`, where each+ * Returns `{ authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio, contractProblem }`, where each* input is `{ state: 'on' | 'off' | 'required' | 'unavailable', note }`.*/export function shotCardRenderRequest({@@ -49,11 +50,10 @@ export function shotCardRenderRequest({// A saved contract can outgrow its scene (a split piece, a retimed shot); the card shows the problem rather than throwing.const contract = scene?.direction?.actionContract;const contractProblem = contract != null ? shotActionContractProblem(contract, scene) : null;- const actions = project && contract != null && !contractProblem ? shotActionPrompt(project, scene) : '';- const motionCues = settings.backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : '';- const base = [(scene?.prompt?.trim() || scene?.framePrompt?.trim() || ''), shotCameraPrompt(scene?.camera),- hears ? actions : silentMotionPrompt(actions)].filter(Boolean).join('\n');- const prompt = audioReactive ? `${base}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}` : [base, motionCues].filter(Boolean).join('. ');+ const authored = authoredShotPrompt(scene);+ const { prompt, promptAdditions } = musicVideoShotModelInput(contractProblem ? null : project, scene, {+ authored, asTyped: true, hears, backend: settings.backend, audioReactive,+ });let endFrame = null;if (scene?.endImageId) {@@ -64,7 +64,9 @@ export function shotCardRenderRequest({}return {+ authored,prompt,+ promptAdditions,promptUsed: falModel?.prompt !== 'none',maxChars: falModel?.promptMaxChars ?? null,endFrame,@@ -87,14 +89,21 @@ function InputToggle({ id, label, input, onChange }) {/** The card's "Sent to the model" block: the exact prompt and the inputs going with it. */export default function SceneRenderInputs({ id, request, onChange }) {- const { prompt, promptUsed, maxChars, endFrame, songAudio } = request;+ const { authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio } = request;const over = maxChars != null && prompt.length > maxChars;return (<div className="min-w-0 space-y-1 rounded border border-port-border p-2 text-xs" data-testid="render-inputs"><div className="text-port-text-muted">Sent to the model</div>- {promptUsed- ? <p className="whitespace-pre-wrap break-words">{prompt || <span className="text-port-text-muted">No prompt</span>}</p>- : <p className="text-port-text-muted">No prompt: this model takes the still and the song only.</p>}+ {!promptUsed+ ? <p className="text-port-text-muted">No prompt: this model takes the still and the song only.</p>+ : prompt+ ? (+ <>+ <p className="whitespace-pre-wrap break-words">{authored || prompt}</p>+ {promptAdditions.length > 0 && <SentPromptPanel prompt={authored} sentPrompt={prompt} additions={promptAdditions} />}+ </>+ )+ : <p className="text-port-text-muted">No prompt</p>}{promptUsed && maxChars != null && (<p className={over ? 'text-port-warning' : 'text-port-text-muted'}>{formatCount(prompt.length)} of {formatCount(maxChars)} characters{over ? ' · shorten the prompt to render' : ''}diff --git a/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsx b/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsxindex 014caa3ba7..8d84f105d4 100644--- a/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsx+++ b/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsx@@ -82,6 +82,6 @@ describe('useMusicVideoSceneMedia shot card Render', () => {const request = generateVideo.mock.calls[0][0];expect(request.prompt).toBe('exactly this');expect(request.mode).toBe('a2v');- expect(JSON.parse(request.musicVideo)).toEqual({ projectId: 'p1', sceneId: 's0', promptAsTyped: true, songAudio: true });+ expect(JSON.parse(request.musicVideo)).toEqual({ projectId: 'p1', sceneId: 's0', compose: true, promptAsTyped: true, songAudio: true });});});diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.jsindex 7d18776f5f..7643944011 100644--- a/client/src/hooks/useMusicVideoSceneMedia.js+++ b/client/src/hooks/useMusicVideoSceneMedia.js@@ -1,8 +1,8 @@import { shotActionContractProblem, shotActionPrompt } from '../../../server/lib/musicVideoActionContract.js';-import { shotCameraPrompt } from '../../../server/lib/cameraMovements.js';import { isOutsideMusicVideoReference, musicVideoConditioningReferences, MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES } from '../../../server/lib/musicVideoConditioning.js';import { musicVideoCreativeContext, musicVideoDirectorLook } from '../../../server/lib/musicVideoCreativeContext.js';import { sceneEndFrameRequestFields } from '../../../server/lib/musicVideoEndFrame.js';+import { authoredShotPrompt, visualDirection } from '../../../server/lib/musicVideoShotPrompt.js';import { useEffect, useRef } from 'react';import socket from '../services/socket';import toast from '../components/ui/Toast';@@ -14,9 +14,8 @@ import useSceneRenderLifecycle from './useSceneRenderLifecycle.js';import { isLtx2FamilyRuntime } from '../lib/runnerFamilies';import { isLayeredComposition, isSelfDrawnLayer, sceneVisualLayer } from '../lib/musicVideoLayers.js';import { musicVideoFrameGenSize } from '../lib/musicVideoAspect.js';-import { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE } from '../lib/musicVideoMotion.js';import {- approximateMotionCues, falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason,+ falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason,} from '../lib/musicVideoShotTiming.js';// The image backends accept at most four reference images for most models@@ -37,27 +36,8 @@ const REFERENCE_CAPABILITY_CODES = new Set(['TOO_MANY_INPUT_IMAGES',]);-/**- * The project-wide visual direction appended to every generated prompt —- * palette and camera rules from the visual spec (#8965). Kept in step with- * `visualDirection` in server/services/musicVideo/handoff.js, which composes- * the same suffix for the external-tool handoff manifest.- *- * Typography is deliberately excluded from this suffix (#8992): the visual- * spec's font/caption guidance belongs to the separately composited text- * layer (#8984), not to the image/video model's prompt — describing a font- * style there works against the "no text, letters, captions" guard the- * applied treatment adds and encourages text baked into the generated- * pixels. `spec.typography` still reaches the composition/typography lane- * directly and the external handoff manifest (`visualSpec.typography`).- */-export function visualDirection(spec) {- if (!spec) return '';- return [- spec.palette?.length ? `color palette ${spec.palette.join(' ')}` : '',- spec.cameraRules?.trim() ? `camera: ${spec.cameraRules.trim()}` : '',- ].filter(Boolean).join('; ');-}+/** The project-wide palette and camera rules, shared with the server's composers. */+export { visualDirection };/** Shared scene-aware frame conditioning, with the legacy capped fallback. */export const conditioningReferences = musicVideoConditioningReferences;@@ -179,8 +159,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applySconst style = project?.concept?.style?.trim();const direction = [musicVideoCreativeContext(project?.concept, { look: musicVideoDirectorLook(project) }), visualDirection(project?.visualSpec)].filter(Boolean).join('; ');- // The i2v prompt leaves the mood-board look out (handoff.js composePrompt): the frame already carries it.- const motionDirection = [musicVideoCreativeContext(project?.concept, { moodBoard: false }), visualDirection(project?.visualSpec)].filter(Boolean).join('; ');const conditioning = conditioningReferences(project);// The image prompt for a scene's reference frame: its frame prompt (or the// shot prompt as a fallback) suffixed with the project's global concept style@@ -191,12 +169,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS// handoff manifest (handoff.js composePrompt).const buildFramePrompt = (scene) =>[[(scene.framePrompt?.trim() || scene.prompt?.trim() || ''), style, direction, scene.direction?.frameClause?.trim()].filter(Boolean).join(', '), shotActionPrompt(project, scene, { frame: true })].filter(Boolean).join('\n');- // The i2v prompt for a scene's clip: its shot prompt (or the frame prompt as a- // fallback) suffixed the same way. The reference frame already fixes the- // look; this prompt guides the motion. A planned camera move (#10589) follows- // as the four-part camera block, same as handoff.js sceneShotPrompt.- const buildShotPrompt = (scene) =>- [[(scene.prompt?.trim() || scene.framePrompt?.trim() || ''), style, motionDirection, scene.direction?.motionClause?.trim(), MOTION_CONTINUITY_CLAUSE].filter(Boolean).join(', '), shotCameraPrompt(scene.camera), shotActionPrompt(project, scene)].filter(Boolean).join('\n');/*** Render a still reference frame for one scene from its frame prompt. The@@ -359,10 +331,11 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applySif (!scene.referenceImageId) { toast.error('Generate a reference frame first'); return Promise.resolve({ ok: false }); }const problem = shotActionContractProblem(scene.direction?.actionContract, scene);if (problem) { toast.error(problem); return Promise.resolve({ ok: false }); }- // `card` is the shot card's Render: its prompt exactly as shown and its input toggles- // ({ prompt, endFrame, songAudio }), sent with nothing added (SceneRenderInputs).- const basePrompt = card ? card.prompt : buildShotPrompt(scene);- if (!basePrompt) { toast.error('Add a shot prompt first'); return Promise.resolve({ ok: false }); }+ // `card` is the shot card's Render: the director's text and its input toggles+ // ({ prompt, endFrame, songAudio }), composed as the card shows it (SceneRenderInputs).+ // The server composes every addition from the director's text (musicVideoShotPrompt.js).+ const prompt = card ? card.prompt : authoredShotPrompt(scene);+ if (!prompt) { toast.error('Add a shot prompt first'); return Promise.resolve({ ok: false }); }// `target` renders this one shot elsewhere (a scene card's Render on pick) without changing the saved setting.const settings = target ? { ...videoSettings.settings, ...target } : videoSettings.settings;const { detectedAudioReactiveLora } = videoSettings;@@ -378,13 +351,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS: null;if (performanceBlocked) { toast.error(performanceBlocked); return Promise.resolve({ ok: false }); }const spanSec = sceneSpanSec(scene);- // Grok's CLI lane takes no timed controls: phrase intents ride as prose- // whose timing is explicitly approximate. The environmental-motion guard- // stays exclusive to the audio-reactive lane.- const motionCues = settings.backend === 'grok' ? approximateMotionCues(project?.phrases, scene.startSec, scene.endSec) : '';- const prompt = card ? basePrompt : audioReactiveSelected- ? `${basePrompt}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}`- : [basePrompt, motionCues].filter(Boolean).join('. ');const songAudio = typeof card?.songAudio === 'boolean' ? card.songAudio : settings.generationMode === 'suppliedAudio';const mode = audioReactiveSelected || songAudio ? 'a2v' : 'image';const falFields = settings.backend === 'fal'@@ -429,6 +395,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS} : {}),musicVideo: JSON.stringify({projectId: project.id, sceneId: scene.sceneId, ...(revisionId ? { revisionId } : {}),+ compose: true, ...(audioReactiveSelected ? { audioReactive: true } : {}),...(card ? { promptAsTyped: true, ...(typeof card.songAudio === 'boolean' && !performance ? { songAudio: card.songAudio } : {}) } : {}),}),})@@ -453,14 +420,14 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applySif (videoBlockedReason) { toast.error(videoBlockedReason); return; }videoLane.startScene(scene.sceneId);generateVideo({- prompt: buildShotPrompt(scene),+ prompt: authoredShotPrompt(scene),backend: 'local',modelId: effectiveModelId || undefined,disableAudio: true,mode: 'extend',extendFromVideoId: scene.videoHistoryId,sourceImageFile: scene.referenceImageId,- musicVideo: JSON.stringify({ projectId: project.id, sceneId: scene.sceneId }),+ musicVideo: JSON.stringify({ projectId: project.id, sceneId: scene.sceneId, compose: true }),}).then((res) => trackVideoJob(res, scene.sceneId)).catch((err) => handleVideoError(err, scene.sceneId, 'Shot continuation failed'));@@ -481,7 +448,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS// dropped here and reported once (#8977).const planMissingVideos = () => {const candidates = footageScenes.filter((scene) =>- scene.referenceImageId && !scene.videoHistoryId && !genVideoScenes[scene.sceneId] && buildShotPrompt(scene));+ scene.referenceImageId && !scene.videoHistoryId && !genVideoScenes[scene.sceneId] && authoredShotPrompt(scene));const { settings, audioReactiveSelected } = videoSettings;const lipSyncBlocked = performanceBlockedReason(audioReactiveSelected ? 'local' : settings.backend);const pending = lipSyncBlocked ? candidates.filter((scene) => !isPerformanceScene(scene)) : candidates;@@ -531,7 +498,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applySvideoBatch,conditioning,buildFramePrompt,- buildShotPrompt,generateFrame,generateEndFrame,generateSceneVideo,diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsxindex 365d03a405..3180199cf4 100644--- a/client/src/pages/MusicVideo.test.jsx+++ b/client/src/pages/MusicVideo.test.jsx@@ -995,7 +995,7 @@ describe('MusicVideo selective section revision (#8987)', () => {await waitFor(() => expect(resumeMusicVideoRevision).toHaveBeenCalledWith('mv-1', 'mvr-1', { silent: true }));expect(startMusicVideoRevision).toHaveBeenCalledWith('mv-1', 'mve-1', {}, { silent: true });await waitFor(() => expect(generateVideo).toHaveBeenCalledTimes(1));- expect(JSON.parse(generateVideo.mock.calls[0][0].musicVideo)).toEqual({ projectId: 'mv-1', sceneId: 's2', revisionId: 'mvr-1' });+ expect(JSON.parse(generateVideo.mock.calls[0][0].musicVideo)).toEqual({ projectId: 'mv-1', sceneId: 's2', revisionId: 'mvr-1', compose: true });const panel = screen.getByLabelText('Section revision');expect(within(panel).getByText('Kept')).toBeInTheDocument();});@@ -1131,8 +1131,9 @@ describe('MusicVideo project video renderer', () => {disableAudio: true,loraFilenames: ['audio-reactive-v2.safetensors'],loraScales: [1.2],- prompt: expect.stringMatching(/No singing, lip-sync, speaking, mouth movement/i),})));+ // The server adds the audio-reactive guard as a labeled addition.+ expect(JSON.parse(generateVideo.mock.calls.at(-1)[0].musicVideo)).toMatchObject({ compose: true, audioReactive: true });});it('lets the project pin an exact installed audio-reactive LoRA version', async () => {@@ -1180,21 +1181,6 @@ describe('MusicVideo project video renderer', () => {expect(await screen.findByLabelText('fal.ai scene clip duration')).toBeTruthy();});- it('carries applied action and reaction into a manually submitted clip prompt', async () => {- getVideoGenStatus.mockResolvedValueOnce({ connected: true, defaultModel: '', falEnabled: true, models: [] });- generateVideo.mockResolvedValue({ jobId: 'intent-job' });- const actionContract = { version: 1, purpose: 'The listener decides to stay',- actions: [{ startSec: 0, endSec: 1, subject: 'Singer', description: 'Offers a hand' }],- reactions: [{ startSec: 1, endSec: 2, subject: 'Listener', description: 'Turns back' }] };- await openProject({ ...PROJECT_NO_CLIP, videoSettings: { backend: 'fal', falDuration: 6 },- scenes: [{ ...PROJECT_NO_CLIP.scenes[0], startSec: 0, endSec: 4, direction: { actionContract } }] }, 'produce');- fireEvent.click(await findEnabledByRole('button', { name: /^Render$/ }));- await waitFor(() => expect(generateVideo).toHaveBeenCalled());- const prompt = generateVideo.mock.calls.at(-1)[0].prompt;- expect(prompt).toContain('Action 0.000s–1.000s: Singer');- expect(prompt).toContain('Reaction 1.000s–2.000s: Listener');- });-it('renders a scene image-to-video through a saved fal.ai pin (#8968)', async () => {getVideoGenStatus.mockResolvedValueOnce({connected: true,@@ -1401,8 +1387,9 @@ describe('MusicVideo project video renderer', () => {fireEvent.click(await findEnabledByRole('button', { name: /^Render$/ }));await waitFor(() => expect(generateVideo).toHaveBeenCalledWith(expect.objectContaining({backend: 'grok', grokDuration: 10,- prompt: expect.stringContaining('around 2.0s: camera pushes in'),})));+ // The server composes Grok's approximate phrase timing onto the director's text.+ expect(JSON.parse(generateVideo.mock.calls.at(-1)[0].musicVideo)).toMatchObject({ compose: true });});});diff --git a/server/lib/README.md b/server/lib/README.mdindex 9788cc8293..87211e253c 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -852,7 +852,8 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `loopInterval.js` | Loop interval grammar (`MIN_INTERVAL_MS`, `parseInterval`, `isValidLoopInterval`) shared by `services/loops.js` and `createLoopSchema` so the route rejects what the service would. || `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. |-| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. |+| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit only to a request sent before `musicVideoShotPrompt.js` composed clips. |+| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, repair notes) as a labeled addition. A take that hears no audio is composed without the look and song timing. `authoredShotPrompt(scene)` and `visualDirection(spec)`. Shared by the shot card, the board and the video submit. || `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. || `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |diff --git a/server/lib/index.js b/server/lib/index.jsindex a5a5402cf4..3382bdd7d4 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -828,6 +828,7 @@ export * from './defaultRequestBody.js';export * from './musicVideoConditioning.js';export * from './musicVideoAspect.js';export * from './musicVideoMotion.js';+export * from './musicVideoShotPrompt.js';export * from './musicVideoMediumPlan.js';export * from './musicVideoFinishedOutside.js';diff --git a/server/lib/musicVideoActionContract.js b/server/lib/musicVideoActionContract.jsindex 28b0a796fd..e738cebb01 100644--- a/server/lib/musicVideoActionContract.js+++ b/server/lib/musicVideoActionContract.js@@ -48,8 +48,12 @@ export function assertShotActionContract(scene) {}}-/** Compile intent and actual song cues to provider clip time. offsetSec is the performance edit in-point. */-export function shotActionPrompt(project, scene, { frame = false, offsetSec = 0 } = {}) {+/**+ * Compile intent and actual song cues to provider clip time. offsetSec is the+ * performance edit in-point. A take that hears no audio (`heard: false`) gets+ * no lyric, word or onset timings: it has nothing to time them against.+ */+export function shotActionPrompt(project, scene, { frame = false, offsetSec = 0, heard = true } = {}) {const contract = scene?.direction?.actionContract;if (contract == null) return '';assertShotActionContract(scene);@@ -66,7 +70,7 @@ export function shotActionPrompt(project, scene, { frame = false, offsetSec = 0for (const [key, label] of [['cameraConstraints', 'Camera'], ['continuityRequirements', 'Must preserve'], ['acceptanceCriteria', 'Acceptance']]) {for (const item of contract[key] || []) lines.push(`${label}: ${item}`);}- if (!frame) {+ if (!frame && heard) {const window = (item) => Number.isFinite(item.startSec) && Number.isFinite(item.endSec) && item.endSec > scene.startSec && item.startSec < scene.endSec;for (const cue of (project.lyricCues || []).filter(window).slice(0, 32)) {lines.push(`Lyric ${time(Math.max(0, cue.startSec - scene.startSec))}–${time(Math.min(scene.endSec, cue.endSec) - scene.startSec)}: ${cue.text}`);diff --git a/server/lib/musicVideoShotPrompt.js b/server/lib/musicVideoShotPrompt.jsnew file mode 100644index 0000000000..ae3d528bcf--- /dev/null+++ b/server/lib/musicVideoShotPrompt.js@@ -0,0 +1,75 @@+/**+ * The model input for a Music Video shot's clip (#11317): the director's own+ * text, then each block PortOS adds as a labeled addition the UI lists as a+ * chip. Shared by the shot card (its "Sent to the model" view), the board's+ * renders and the server's video submit, so what the card shows is what the+ * submit sends.+ *+ * A take that hears no audio is composed without the blocks it cannot use: no+ * project look (the reference frame already carries it, and a silent i2v model+ * acts style words out literally) and no lyric, word or onset timings. Nothing+ * is removed from the director's text.+ */+import { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE } from './musicVideoMotion.js';+import { musicVideoCreativeContext, musicVideoDirectorLook } from './musicVideoCreativeContext.js';+import { shotCameraPrompt } from './cameraMovements.js';+import { shotActionPrompt } from './musicVideoActionContract.js';+import { approximateMotionCues } from './musicVideoShotTiming.js';++/**+ * The project-wide visual direction: palette and camera rules from the visual+ * spec (#8965). Typography is excluded (#8992): it belongs to the composited+ * text layer, not to the image or video model's prompt.+ */+export function visualDirection(spec) {+ if (!spec) return '';+ return [+ spec.palette?.length ? `color palette ${spec.palette.join(' ')}` : '',+ spec.cameraRules?.trim() ? `camera: ${spec.cameraRules.trim()}` : '',+ ].filter(Boolean).join('; ');+}++/** The director's text for a shot's clip: its shot prompt, else its frame prompt. */+export const authoredShotPrompt = (scene) => scene?.prompt?.trim() || scene?.framePrompt?.trim() || '';++const addition = (id, label, text) => {+ const trimmed = typeof text === 'string' ? text.trim() : '';+ return trimmed ? { id, label, text: trimmed } : null;+};++/**+ * `{ prompt, promptAdditions }` for one clip. `authored` is the director's text+ * (the shot card's box, or the scene's saved prompt). `asTyped` is the shot+ * card's Render, which adds no project look or treatment direction. `hears` is+ * whether the take hears song audio (lip-sync, supplied song audio or an+ * audio-reactive LoRA); `offsetSec` re-times the shot actions to a performance+ * take's padded song slice; `fixes` is an autonomous run's repair note.+ */+export function musicVideoShotModelInput(project, scene, {+ authored = authoredShotPrompt(scene), asTyped = false, hears = false, backend = null,+ audioReactive = false, offsetSec = 0, fixes = '',+} = {}) {+ const text = typeof authored === 'string' ? authored.trim() : '';+ const look = !asTyped && hears ? [+ addition('project-style', 'Project style', project?.concept?.style),+ addition('creative-context', 'Production bible', musicVideoCreativeContext(project?.concept, { moodBoard: false, look: musicVideoDirectorLook(project) })),+ addition('visual-direction', 'Palette and camera rules', visualDirection(project?.visualSpec)),+ ] : [];+ const direction = asTyped ? [] : [+ addition('shot-direction', 'Shot direction', scene?.direction?.motionClause),+ addition('lighting-continuity', 'Lighting continuity', MOTION_CONTINUITY_CLAUSE),+ ];+ const camera = addition('camera-move', 'Camera move', shotCameraPrompt(scene?.camera));+ const actions = addition('shot-actions', 'Shot actions', project ? shotActionPrompt(project, scene, { offsetSec, heard: hears }) : '');+ const tail = audioReactive+ ? [addition('audio-reactive-guard', 'Audio-reactive guard', AUDIO_REACTIVE_PERFORMANCE_GUARD)]+ : [+ addition('phrase-timing', 'Phrase timing (approximate)', backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : ''),+ addition('repair-notes', 'Repair notes', fixes ? `Fix these problems from the previous take while preserving the shot intent: ${fixes}` : ''),+ ];++ const firstLine = [text, ...[...look, ...direction].filter(Boolean).map((item) => item.text)].filter(Boolean).join(', ');+ const body = [firstLine, camera?.text, actions?.text].filter(Boolean).join('\n');+ const prompt = [body, ...tail.filter(Boolean).map((item) => item.text)].filter(Boolean).join('. ');+ return { prompt, promptAdditions: [...look, ...direction, camera, actions, ...tail].filter(Boolean) };+}diff --git a/server/routes/videoGen.js b/server/routes/videoGen.jsindex 1001e35ab5..760b113dc5 100644--- a/server/routes/videoGen.js+++ b/server/routes/videoGen.js@@ -487,6 +487,11 @@ const generateBodySchema = z.object({// The shot card's Render: `prompt` is exactly what the director sees and// sends, so the shot-intent lines and the silent-model rewrite are skipped.promptAsTyped: z.boolean().optional(),+ // `prompt` is the director's own text and the server composes the rest as+ // labeled additions (lib/musicVideoShotPrompt.js, #11317); `audioReactive`+ // names an audio-reactive LoRA take, which gets its guard clause.+ compose: z.boolean().optional(),+ audioReactive: z.boolean().optional(),// The shot card's Song audio toggle: hand a cutaway its song slice (true) or// render it silent (false), whatever the project's generation mode says.songAudio: z.boolean().optional(),diff --git a/server/services/musicVideo/handoff.js b/server/services/musicVideo/handoff.jsindex f7ffe7cb06..3804544920 100644--- a/server/services/musicVideo/handoff.js+++ b/server/services/musicVideo/handoff.js@@ -3,6 +3,7 @@ import { shotCameraPrompt } from '../../lib/cameraMovements.js';import { musicVideoCreativeContext, musicVideoDirectorLook } from '../../lib/musicVideoCreativeContext.js';import { MOTION_CONTINUITY_CLAUSE } from '../../lib/musicVideoMotion.js';import { isOutsideMusicVideoReference } from '../../lib/musicVideoConditioning.js';+import { visualDirection } from '../../lib/musicVideoShotPrompt.js';/*** Music Video external-asset handoff (#8965) — provider-neutral export/import* for tools PortOS does not drive, such as Midjourney.@@ -64,24 +65,8 @@ export function matchSceneByFileTag(scenes, originalName) {return matched.size === 1 ? [...matched][0] : null;}-/**- * The project-wide visual direction appended to generated prompts — palette- * and camera rules from the visual spec. Kept in step with `visualDirection`- * in client/src/hooks/useMusicVideoSceneMedia.js, which composes the same- * suffix for PortOS's own renders.- *- * Typography is deliberately excluded from this suffix (#8992): it belongs- * to the separately composited text layer (#8984), not the image/video- * prompt handed to an external tool. `visualSpec.typography` still exports- * on the manifest below for the typography lane and other external tools.- */-export function visualDirection(spec) {- if (!spec) return '';- return [- spec.palette?.length ? `color palette ${spec.palette.join(' ')}` : '',- spec.cameraRules?.trim() ? `camera: ${spec.cameraRules.trim()}` : '',- ].filter(Boolean).join('; ');-}+/** The project-wide palette and camera rules (shared with the board and the video submit). */+export { visualDirection };// `clause` is the scene's applied treatment direction (#8980) — the composition// constraints for a frame or a motion prompt. Same order as the client'sdiff --git a/server/services/musicVideo/productionDispatch.js b/server/services/musicVideo/productionDispatch.jsindex fe1f61d435..349f46b47c 100644--- a/server/services/musicVideo/productionDispatch.js+++ b/server/services/musicVideo/productionDispatch.js@@ -29,8 +29,9 @@ import { withMusicVideoStyle } from './styleReferences.js';import { ServerError } from '../../lib/errorHandler.js';import { resolveGalleryImage } from '../../lib/pathSafety.js';import { RENDER_TARGET } from '../../lib/renderTargets.js';-import { approximateMotionCues, falSceneTake, falTakeRequestFields, grokCoverage } from '../../lib/musicVideoShotTiming.js';-import { sceneFramePrompt, sceneShotPrompt } from './handoff.js';+import { falSceneTake, falTakeRequestFields, grokCoverage } from '../../lib/musicVideoShotTiming.js';+import { sceneFramePrompt } from './handoff.js';+import { authoredShotPrompt } from '../../lib/musicVideoShotPrompt.js';import { sceneEndFrameRequestFields } from '../../lib/musicVideoEndFrame.js';import { conditioningReferences, falRouteVideoSettings } from './productionPool.js';import { musicVideoFrameGenSize } from '../../lib/musicVideoAspect.js';@@ -109,15 +110,13 @@ async function dispatchClip({ project, scene, route, tag }) {if (!scene.referenceImageId) {throw new ServerError(`"${scene.label || scene.sceneId}" has no reference frame yet`, { status: 409, code: 'PRODUCTION_FRAME_MISSING' });}- const base = sceneShotPrompt(project, scene);- if (!base) throw unprompted(scene, 'shot');+ // The director's text; the submit service composes the rest as labeled additions (#11317).+ const prompt = authoredShotPrompt(scene);+ if (!prompt) throw unprompted(scene, 'shot');const spanSec = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' && scene.endSec > scene.startSec? scene.endSec - scene.startSec : null;- const isGrok = route.mode === 'grok';// The autonomous run's pilot repair names what the last clip got wrong (autonomousService.js).const fixes = typeof scene.clipFixes?.text === 'string' ? scene.clipFixes.text : '';- const prompt = [base, isGrok ? approximateMotionCues(project.phrases, scene.startSec, scene.endSec) : '',- fixes ? `Fix these problems from the previous take while preserving the shot intent: ${fixes}` : ''].filter(Boolean).join('. ');const byBackend = {// A supplied-audio route animates the frame from its shot's own song window.local: route.audio === 'supplied' ? { modelId: route.model, mode: 'a2v', _suppliedAudio: true } : { modelId: route.model, disableAudio: true },@@ -143,7 +142,8 @@ async function dispatchClip({ project, scene, route, tag }) {...sceneEndFrameRequestFields(scene, {backend: route.mode, falModelId: byBackend.fal?.falModelId, mode: byBackend[route.mode]?.mode || 'image', localRuntime,}),- musicVideo: tag,+ musicVideo: { ...tag, compose: true },+ ...(fixes ? { _clipFixes: fixes } : {}),}, {});return { jobId };}diff --git a/server/services/musicVideo/productionDispatch.test.js b/server/services/musicVideo/productionDispatch.test.jsindex 1f8b576b3d..0e65b66145 100644--- a/server/services/musicVideo/productionDispatch.test.js+++ b/server/services/musicVideo/productionDispatch.test.js@@ -157,7 +157,7 @@ describe('production scene dispatch (#9066)', () => {const clipTag = { ...tag, productionStepKey: 'clip:mvs-a:base:1' };await dispatchProductionStep({ stepKind: 'clip', project, scene, route: { kind: 'video', mode: 'grok', model: null }, tag: clipTag, settings });expect(submitVideoGenJob).toHaveBeenCalledWith(expect.objectContaining({- backend: 'grok', mode: 'image', sourceImageFile: 'frame-example.png', grokDuration: 6, musicVideo: clipTag,+ backend: 'grok', mode: 'image', sourceImageFile: 'frame-example.png', grokDuration: 6, musicVideo: { ...clipTag, compose: true }, prompt: 'waves roll',}), {});});diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex 4370054a0c..6d73fe5413 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -283,6 +283,7 @@ const submitValidatedVideoGenJob = async (body, uploads) => {} : null;let repairReserved = false;+ let composedAdditions = [];const enqueue = (params) => withStagedRollback(async () => {if (repairReserved) {@@ -299,8 +300,8 @@ const submitValidatedVideoGenJob = async (body, uploads) => {body.musicVideo.productionStepKey, { sceneId: body.musicVideo.sceneId, kind: 'video' });}if (body.musicVideo) {- const [{ getProject }, { assertShotActionContract, withShotActionPrompt }] = await Promise.all([- import('../musicVideo/projects.js'), import('../../lib/musicVideoActionContract.js'),+ const [{ getProject }, { assertShotActionContract, withShotActionPrompt }, { musicVideoShotModelInput }] = await Promise.all([+ import('../musicVideo/projects.js'), import('../../lib/musicVideoActionContract.js'), import('../../lib/musicVideoShotPrompt.js'),]);const project = await getProject(body.musicVideo.projectId);// A brief that names tools and no video tool never dispatches generated footage;@@ -313,12 +314,25 @@ const submitValidatedVideoGenJob = async (body, uploads) => {// A take that hears nothing (no lip-sync or supplied song audio) gets plain physical// motion: the model would act out song cues and look words instead of reading them.const hearsAudio = !!(performance || prepared.audio || params.audioConditioning || params.audioFilePath);- // A shot card's prompt is sent as shown, its shot actions included; only a performance- // take re-times them to its padded song slice.- if (!body.musicVideo.promptAsTyped || performance) {- params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec: performance?.shotInstruction?.edit?.inSec || 0 });+ const offsetSec = performance?.shotInstruction?.edit?.inSec || 0;+ if (body.musicVideo.compose) {+ // The director's text plus each labeled block PortOS adds (#11317), from the same+ // composer the shot card shows; a take that hears nothing gets no look or song timing.+ const composed = musicVideoShotModelInput(project, scene, {+ authored: params.prompt, asTyped: body.musicVideo.promptAsTyped === true, hears: hearsAudio, backend,+ audioReactive: body.musicVideo.audioReactive === true, offsetSec, fixes: body._clipFixes || '',+ });+ params.prompt = composed.prompt;+ composedAdditions = composed.promptAdditions;+ } else {+ // A client from before the composer sends its prompt already composed.+ // A shot card's prompt is sent as shown, its shot actions included; only a performance+ // take re-times them to its padded song slice.+ if (!body.musicVideo.promptAsTyped || performance) {+ params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec });+ }+ if (!hearsAudio && !body.musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);}- if (!hearsAudio && !body.musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);const interval = performance?.shotInstruction?.songInterval || prepared.audio?.songInterval;if (params.audioConditioning && Math.round(scene?.startSec * 48000) !== params.audioConditioning.startSample) {throw new ServerError('Scene timing changed while source audio was prepared', { status: 409, code: 'MUSIC_VIDEO_SHOT_TIMING_CHANGED' });@@ -387,8 +401,9 @@ const submitValidatedVideoGenJob = async (body, uploads) => {params.remoteMedia.request = buildFederatedMediaRequest({ kind: 'video', params: { ...params.remoteMedia.request, prompt: params.prompt } });}if (params.remoteMedia && !params.musicVideo) delete params.prompt;- // The labeled additions the composer put around the user's text ride with the job to its record.- if (body.promptAdditions?.length) params.promptAdditions = body.promptAdditions;+ // The labeled additions the composers put around the user's text ride with the job to its record.+ const additions = [...(body.promptAdditions || []), ...composedAdditions];+ if (additions.length) params.promptAdditions = additions;return enqueueJob({ kind: 'video', params });},);diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.jsindex 4c4a588832..075dc221e8 100644--- a/server/services/videoGen/submitJob.test.js+++ b/server/services/videoGen/submitJob.test.js@@ -253,6 +253,35 @@ describe('submitVideoGenJob', () => {expect(mocks.enqueueJob.mock.calls[0][0].params).toMatchObject({ aspectRatio: '9:16', frameAspect: '9:16' });});+ // Uniquely catches the server composing a shot clip differently from the shot card, or+ // adding look words and song timing a take that hears nothing would act out (#11317).+ it('composes a board clip from the director\'s text, adding each block as a labeled addition', async () => {+ const scene = {+ sceneId: 'mvs-1', startSec: 0, endSec: 4, camera: { move: 'locked-off' },+ direction: { motionClause: 'slow drift', actionContract: { version: 1, purpose: 'She leaves',+ actions: [{ startSec: 0, endSec: 1, subject: 'Singer', description: 'Offers a hand' }] } },+ };+ const project = { id: 'mv-1', scenes: [scene], concept: { style: 'grainy 16mm' },+ lyricCues: [{ startSec: 0.5, endSec: 1.5, text: 'hold on', words: [] }] };+ mocks.getMusicVideoProject.mockResolvedValue(project);+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());+ await submitVideoGenJob({ prompt: 'she turns away', backend: 'fal', mode: 'image', musicVideo: { ...musicVideo, compose: true } }, {});+ const silent = mocks.enqueueJob.mock.calls[0][0].params;+ expect(silent.prompt.startsWith('she turns away, slow drift, ')).toBe(true);+ expect(silent.prompt).toContain('Action 0.000s–1.000s: Singer');+ expect(silent.prompt).not.toContain('grainy 16mm');+ expect(silent.prompt).not.toContain('Lyric');+ expect(silent.promptAdditions.map((item) => item.id)).toEqual(['shot-direction', 'lighting-continuity', 'camera-move', 'shot-actions']);++ // A take that hears the song gets the project look and the lyric timing, each listed.+ mocks.prepareVideoGenParams.mockResolvedValue({ ...falPrepared(), audioFilePath: '/example/uploads/song.wav' });+ await submitVideoGenJob({ prompt: 'she turns away', backend: 'fal', mode: 'image', musicVideo: { ...musicVideo, compose: true } }, {});+ const hearing = mocks.enqueueJob.mock.calls[1][0].params;+ expect(hearing.prompt).toContain('grainy 16mm');+ expect(hearing.prompt).toContain('Lyric 0.500s–1.500s: hold on');+ expect(hearing.promptAdditions.map((item) => item.id)).toContain('project-style');+ });+it('leaves a clip outside a music video at its own aspect', async () => {mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());await submitVideoGenJob({ prompt: 'a fox', backend: 'fal', mode: 'image' }, {});