Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video clips list what PortOS adds to the director's prompt

Claudeeb71caf9092c
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index bf91b363cc..fb04ad1c8e 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -201,7 +201,7 @@ export default function SceneCard({
if (videoRefusal) return;
const { songAudio, endFrame } = cardRequest;
const card = {
- prompt: cardRequest.prompt,
+ prompt: cardRequest.authored,
...(endFrame ? { endFrame: endFrame.state === 'on' } : {}),
...(songAudio.state === 'on' || songAudio.state === 'off' ? { songAudio: songAudio.state === 'on' } : {}),
};
diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsx
index 54640becd9..875c0e57b8 100644
--- a/client/src/components/musicVideo/SceneRenderInputs.jsx
+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx
@@ -1,10 +1,10 @@
import { getFalVideoModel } from '../../lib/falVideoModels.js';
-import { AUDIO_REACTIVE_PERFORMANCE_GUARD, silentMotionPrompt } from '../../lib/musicVideoMotion.js';
-import { approximateMotionCues, falSceneTake } from '../../lib/musicVideoShotTiming.js';
-import { shotActionContractProblem, shotActionPrompt } from '../../../../server/lib/musicVideoActionContract.js';
-import { shotCameraPrompt } from '../../../../server/lib/cameraMovements.js';
+import { falSceneTake } from '../../lib/musicVideoShotTiming.js';
+import { shotActionContractProblem } from '../../../../server/lib/musicVideoActionContract.js';
+import { authoredShotPrompt, musicVideoShotModelInput } from '../../../../server/lib/musicVideoShotPrompt.js';
import { localModelTakesSongAudio } from '../../../../server/lib/musicVideoLocalModels.js';
import { formatCount } from '../../utils/formatters.js';
+import SentPromptPanel from '../media/SentPromptPanel';
const clock = (sec) => {
const whole = Math.max(0, sec || 0);
@@ -13,17 +13,18 @@ const clock = (sec) => {
};
/**
- * Exactly what the shot card's Render sends: the prompt as typed, then what
- * the card itself sets (its camera move, its applied shot actions, Grok's
- * approximate phrase timing, the audio-reactive guard), and which inputs go
- * with it. The project's style and bible are not added, and the server
- * receives `promptAsTyped` and leaves the prompt alone (a performance shot's
- * action times shift to its padded song slice).
+ * Exactly what the shot card's Render sends: the director's text, then what
+ * the card itself adds (its camera move, its applied shot actions, Grok's
+ * approximate phrase timing, the audio-reactive guard) as labeled additions,
+ * and which inputs go with it. The composer is the server's own
+ * (lib/musicVideoShotPrompt.js): Render sends the director's text and the
+ * server composes the same prompt (a performance shot's action times shift to
+ * its padded song slice). The project's style and bible are not added.
*
* `settings` is the renderer this render uses (the project's, or the card's
* Render on pick merged over it); `inputs` holds the card's toggles,
* `{ endFrame, songAudio }`, each undefined until the director flips it.
- * Returns `{ prompt, promptUsed, maxChars, endFrame, songAudio, contractProblem }`, where each
+ * Returns `{ authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio, contractProblem }`, where each
* input is `{ state: 'on' | 'off' | 'required' | 'unavailable', note }`.
*/
export function shotCardRenderRequest({
@@ -49,11 +50,10 @@ export function shotCardRenderRequest({
// A saved contract can outgrow its scene (a split piece, a retimed shot); the card shows the problem rather than throwing.
const contract = scene?.direction?.actionContract;
const contractProblem = contract != null ? shotActionContractProblem(contract, scene) : null;
- const actions = project && contract != null && !contractProblem ? shotActionPrompt(project, scene) : '';
- const motionCues = settings.backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : '';
- const base = [(scene?.prompt?.trim() || scene?.framePrompt?.trim() || ''), shotCameraPrompt(scene?.camera),
- hears ? actions : silentMotionPrompt(actions)].filter(Boolean).join('\n');
- const prompt = audioReactive ? `${base}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}` : [base, motionCues].filter(Boolean).join('. ');
+ const authored = authoredShotPrompt(scene);
+ const { prompt, promptAdditions } = musicVideoShotModelInput(contractProblem ? null : project, scene, {
+ authored, asTyped: true, hears, backend: settings.backend, audioReactive,
+ });
let endFrame = null;
if (scene?.endImageId) {
@@ -64,7 +64,9 @@ export function shotCardRenderRequest({
}
return {
+ authored,
prompt,
+ promptAdditions,
promptUsed: falModel?.prompt !== 'none',
maxChars: falModel?.promptMaxChars ?? null,
endFrame,
@@ -87,14 +89,21 @@ function InputToggle({ id, label, input, onChange }) {
/** The card's "Sent to the model" block: the exact prompt and the inputs going with it. */
export default function SceneRenderInputs({ id, request, onChange }) {
- const { prompt, promptUsed, maxChars, endFrame, songAudio } = request;
+ const { authored, prompt, promptAdditions, promptUsed, maxChars, endFrame, songAudio } = request;
const over = maxChars != null && prompt.length > maxChars;
return (
<div className="min-w-0 space-y-1 rounded border border-port-border p-2 text-xs" data-testid="render-inputs">
<div className="text-port-text-muted">Sent to the model</div>
- {promptUsed
- ? <p className="whitespace-pre-wrap break-words">{prompt || <span className="text-port-text-muted">No prompt</span>}</p>
- : <p className="text-port-text-muted">No prompt: this model takes the still and the song only.</p>}
+ {!promptUsed
+ ? <p className="text-port-text-muted">No prompt: this model takes the still and the song only.</p>
+ : prompt
+ ? (
+ <>
+ <p className="whitespace-pre-wrap break-words">{authored || prompt}</p>
+ {promptAdditions.length > 0 && <SentPromptPanel prompt={authored} sentPrompt={prompt} additions={promptAdditions} />}
+ </>
+ )
+ : <p className="text-port-text-muted">No prompt</p>}
{promptUsed && maxChars != null && (
<p className={over ? 'text-port-warning' : 'text-port-text-muted'}>
{formatCount(prompt.length)} of {formatCount(maxChars)} characters{over ? ' · shorten the prompt to render' : ''}
diff --git a/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsx b/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsx
index 014caa3ba7..8d84f105d4 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsx
+++ b/client/src/hooks/useMusicVideoSceneMedia.batch.test.jsx
@@ -82,6 +82,6 @@ describe('useMusicVideoSceneMedia shot card Render', () => {
const request = generateVideo.mock.calls[0][0];
expect(request.prompt).toBe('exactly this');
expect(request.mode).toBe('a2v');
- expect(JSON.parse(request.musicVideo)).toEqual({ projectId: 'p1', sceneId: 's0', promptAsTyped: true, songAudio: true });
+ expect(JSON.parse(request.musicVideo)).toEqual({ projectId: 'p1', sceneId: 's0', compose: true, promptAsTyped: true, songAudio: true });
});
});
diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.js
index 7d18776f5f..7643944011 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.js
+++ b/client/src/hooks/useMusicVideoSceneMedia.js
@@ -1,8 +1,8 @@
import { shotActionContractProblem, shotActionPrompt } from '../../../server/lib/musicVideoActionContract.js';
-import { shotCameraPrompt } from '../../../server/lib/cameraMovements.js';
import { isOutsideMusicVideoReference, musicVideoConditioningReferences, MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES } from '../../../server/lib/musicVideoConditioning.js';
import { musicVideoCreativeContext, musicVideoDirectorLook } from '../../../server/lib/musicVideoCreativeContext.js';
import { sceneEndFrameRequestFields } from '../../../server/lib/musicVideoEndFrame.js';
+import { authoredShotPrompt, visualDirection } from '../../../server/lib/musicVideoShotPrompt.js';
import { useEffect, useRef } from 'react';
import socket from '../services/socket';
import toast from '../components/ui/Toast';
@@ -14,9 +14,8 @@ import useSceneRenderLifecycle from './useSceneRenderLifecycle.js';
import { isLtx2FamilyRuntime } from '../lib/runnerFamilies';
import { isLayeredComposition, isSelfDrawnLayer, sceneVisualLayer } from '../lib/musicVideoLayers.js';
import { musicVideoFrameGenSize } from '../lib/musicVideoAspect.js';
-import { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE } from '../lib/musicVideoMotion.js';
import {
- approximateMotionCues, falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason,
+ falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason,
} from '../lib/musicVideoShotTiming.js';
// The image backends accept at most four reference images for most models
@@ -37,27 +36,8 @@ const REFERENCE_CAPABILITY_CODES = new Set([
'TOO_MANY_INPUT_IMAGES',
]);
-/**
- * The project-wide visual direction appended to every generated prompt —
- * palette and camera rules from the visual spec (#8965). Kept in step with
- * `visualDirection` in server/services/musicVideo/handoff.js, which composes
- * the same suffix for the external-tool handoff manifest.
- *
- * Typography is deliberately excluded from this suffix (#8992): the visual
- * spec's font/caption guidance belongs to the separately composited text
- * layer (#8984), not to the image/video model's prompt — describing a font
- * style there works against the "no text, letters, captions" guard the
- * applied treatment adds and encourages text baked into the generated
- * pixels. `spec.typography` still reaches the composition/typography lane
- * directly and the external handoff manifest (`visualSpec.typography`).
- */
-export function visualDirection(spec) {
- if (!spec) return '';
- return [
- spec.palette?.length ? `color palette ${spec.palette.join(' ')}` : '',
- spec.cameraRules?.trim() ? `camera: ${spec.cameraRules.trim()}` : '',
- ].filter(Boolean).join('; ');
-}
+/** The project-wide palette and camera rules, shared with the server's composers. */
+export { visualDirection };
/** Shared scene-aware frame conditioning, with the legacy capped fallback. */
export const conditioningReferences = musicVideoConditioningReferences;
@@ -179,8 +159,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const style = project?.concept?.style?.trim();
const direction = [musicVideoCreativeContext(project?.concept, { look: musicVideoDirectorLook(project) }), visualDirection(project?.visualSpec)].filter(Boolean).join('; ');
- // The i2v prompt leaves the mood-board look out (handoff.js composePrompt): the frame already carries it.
- const motionDirection = [musicVideoCreativeContext(project?.concept, { moodBoard: false }), visualDirection(project?.visualSpec)].filter(Boolean).join('; ');
const conditioning = conditioningReferences(project);
// The image prompt for a scene's reference frame: its frame prompt (or the
// shot prompt as a fallback) suffixed with the project's global concept style
@@ -191,12 +169,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
// handoff manifest (handoff.js composePrompt).
const buildFramePrompt = (scene) =>
[[(scene.framePrompt?.trim() || scene.prompt?.trim() || ''), style, direction, scene.direction?.frameClause?.trim()].filter(Boolean).join(', '), shotActionPrompt(project, scene, { frame: true })].filter(Boolean).join('\n');
- // The i2v prompt for a scene's clip: its shot prompt (or the frame prompt as a
- // fallback) suffixed the same way. The reference frame already fixes the
- // look; this prompt guides the motion. A planned camera move (#10589) follows
- // as the four-part camera block, same as handoff.js sceneShotPrompt.
- const buildShotPrompt = (scene) =>
- [[(scene.prompt?.trim() || scene.framePrompt?.trim() || ''), style, motionDirection, scene.direction?.motionClause?.trim(), MOTION_CONTINUITY_CLAUSE].filter(Boolean).join(', '), shotCameraPrompt(scene.camera), shotActionPrompt(project, scene)].filter(Boolean).join('\n');
/**
* Render a still reference frame for one scene from its frame prompt. The
@@ -359,10 +331,11 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
if (!scene.referenceImageId) { toast.error('Generate a reference frame first'); return Promise.resolve({ ok: false }); }
const problem = shotActionContractProblem(scene.direction?.actionContract, scene);
if (problem) { toast.error(problem); return Promise.resolve({ ok: false }); }
- // `card` is the shot card's Render: its prompt exactly as shown and its input toggles
- // ({ prompt, endFrame, songAudio }), sent with nothing added (SceneRenderInputs).
- const basePrompt = card ? card.prompt : buildShotPrompt(scene);
- if (!basePrompt) { toast.error('Add a shot prompt first'); return Promise.resolve({ ok: false }); }
+ // `card` is the shot card's Render: the director's text and its input toggles
+ // ({ prompt, endFrame, songAudio }), composed as the card shows it (SceneRenderInputs).
+ // The server composes every addition from the director's text (musicVideoShotPrompt.js).
+ const prompt = card ? card.prompt : authoredShotPrompt(scene);
+ if (!prompt) { toast.error('Add a shot prompt first'); return Promise.resolve({ ok: false }); }
// `target` renders this one shot elsewhere (a scene card's Render on pick) without changing the saved setting.
const settings = target ? { ...videoSettings.settings, ...target } : videoSettings.settings;
const { detectedAudioReactiveLora } = videoSettings;
@@ -378,13 +351,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
: null;
if (performanceBlocked) { toast.error(performanceBlocked); return Promise.resolve({ ok: false }); }
const spanSec = sceneSpanSec(scene);
- // Grok's CLI lane takes no timed controls: phrase intents ride as prose
- // whose timing is explicitly approximate. The environmental-motion guard
- // stays exclusive to the audio-reactive lane.
- const motionCues = settings.backend === 'grok' ? approximateMotionCues(project?.phrases, scene.startSec, scene.endSec) : '';
- const prompt = card ? basePrompt : audioReactiveSelected
- ? `${basePrompt}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}`
- : [basePrompt, motionCues].filter(Boolean).join('. ');
const songAudio = typeof card?.songAudio === 'boolean' ? card.songAudio : settings.generationMode === 'suppliedAudio';
const mode = audioReactiveSelected || songAudio ? 'a2v' : 'image';
const falFields = settings.backend === 'fal'
@@ -429,6 +395,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
} : {}),
musicVideo: JSON.stringify({
projectId: project.id, sceneId: scene.sceneId, ...(revisionId ? { revisionId } : {}),
+ compose: true, ...(audioReactiveSelected ? { audioReactive: true } : {}),
...(card ? { promptAsTyped: true, ...(typeof card.songAudio === 'boolean' && !performance ? { songAudio: card.songAudio } : {}) } : {}),
}),
})
@@ -453,14 +420,14 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
if (videoBlockedReason) { toast.error(videoBlockedReason); return; }
videoLane.startScene(scene.sceneId);
generateVideo({
- prompt: buildShotPrompt(scene),
+ prompt: authoredShotPrompt(scene),
backend: 'local',
modelId: effectiveModelId || undefined,
disableAudio: true,
mode: 'extend',
extendFromVideoId: scene.videoHistoryId,
sourceImageFile: scene.referenceImageId,
- musicVideo: JSON.stringify({ projectId: project.id, sceneId: scene.sceneId }),
+ musicVideo: JSON.stringify({ projectId: project.id, sceneId: scene.sceneId, compose: true }),
})
.then((res) => trackVideoJob(res, scene.sceneId))
.catch((err) => handleVideoError(err, scene.sceneId, 'Shot continuation failed'));
@@ -481,7 +448,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
// dropped here and reported once (#8977).
const planMissingVideos = () => {
const candidates = footageScenes.filter((scene) =>
- scene.referenceImageId && !scene.videoHistoryId && !genVideoScenes[scene.sceneId] && buildShotPrompt(scene));
+ scene.referenceImageId && !scene.videoHistoryId && !genVideoScenes[scene.sceneId] && authoredShotPrompt(scene));
const { settings, audioReactiveSelected } = videoSettings;
const lipSyncBlocked = performanceBlockedReason(audioReactiveSelected ? 'local' : settings.backend);
const pending = lipSyncBlocked ? candidates.filter((scene) => !isPerformanceScene(scene)) : candidates;
@@ -531,7 +498,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
videoBatch,
conditioning,
buildFramePrompt,
- buildShotPrompt,
generateFrame,
generateEndFrame,
generateSceneVideo,
diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsx
index 365d03a405..3180199cf4 100644
--- a/client/src/pages/MusicVideo.test.jsx
+++ b/client/src/pages/MusicVideo.test.jsx
@@ -995,7 +995,7 @@ describe('MusicVideo selective section revision (#8987)', () => {
await waitFor(() => expect(resumeMusicVideoRevision).toHaveBeenCalledWith('mv-1', 'mvr-1', { silent: true }));
expect(startMusicVideoRevision).toHaveBeenCalledWith('mv-1', 'mve-1', {}, { silent: true });
await waitFor(() => expect(generateVideo).toHaveBeenCalledTimes(1));
- expect(JSON.parse(generateVideo.mock.calls[0][0].musicVideo)).toEqual({ projectId: 'mv-1', sceneId: 's2', revisionId: 'mvr-1' });
+ expect(JSON.parse(generateVideo.mock.calls[0][0].musicVideo)).toEqual({ projectId: 'mv-1', sceneId: 's2', revisionId: 'mvr-1', compose: true });
const panel = screen.getByLabelText('Section revision');
expect(within(panel).getByText('Kept')).toBeInTheDocument();
});
@@ -1131,8 +1131,9 @@ describe('MusicVideo project video renderer', () => {
disableAudio: true,
loraFilenames: ['audio-reactive-v2.safetensors'],
loraScales: [1.2],
- prompt: expect.stringMatching(/No singing, lip-sync, speaking, mouth movement/i),
})));
+ // The server adds the audio-reactive guard as a labeled addition.
+ expect(JSON.parse(generateVideo.mock.calls.at(-1)[0].musicVideo)).toMatchObject({ compose: true, audioReactive: true });
});
it('lets the project pin an exact installed audio-reactive LoRA version', async () => {
@@ -1180,21 +1181,6 @@ describe('MusicVideo project video renderer', () => {
expect(await screen.findByLabelText('fal.ai scene clip duration')).toBeTruthy();
});
- it('carries applied action and reaction into a manually submitted clip prompt', async () => {
- getVideoGenStatus.mockResolvedValueOnce({ connected: true, defaultModel: '', falEnabled: true, models: [] });
- generateVideo.mockResolvedValue({ jobId: 'intent-job' });
- const actionContract = { version: 1, purpose: 'The listener decides to stay',
- actions: [{ startSec: 0, endSec: 1, subject: 'Singer', description: 'Offers a hand' }],
- reactions: [{ startSec: 1, endSec: 2, subject: 'Listener', description: 'Turns back' }] };
- await openProject({ ...PROJECT_NO_CLIP, videoSettings: { backend: 'fal', falDuration: 6 },
- scenes: [{ ...PROJECT_NO_CLIP.scenes[0], startSec: 0, endSec: 4, direction: { actionContract } }] }, 'produce');
- fireEvent.click(await findEnabledByRole('button', { name: /^Render$/ }));
- await waitFor(() => expect(generateVideo).toHaveBeenCalled());
- const prompt = generateVideo.mock.calls.at(-1)[0].prompt;
- expect(prompt).toContain('Action 0.000s–1.000s: Singer');
- expect(prompt).toContain('Reaction 1.000s–2.000s: Listener');
- });
-
it('renders a scene image-to-video through a saved fal.ai pin (#8968)', async () => {
getVideoGenStatus.mockResolvedValueOnce({
connected: true,
@@ -1401,8 +1387,9 @@ describe('MusicVideo project video renderer', () => {
fireEvent.click(await findEnabledByRole('button', { name: /^Render$/ }));
await waitFor(() => expect(generateVideo).toHaveBeenCalledWith(expect.objectContaining({
backend: 'grok', grokDuration: 10,
- prompt: expect.stringContaining('around 2.0s: camera pushes in'),
})));
+ // The server composes Grok's approximate phrase timing onto the director's text.
+ expect(JSON.parse(generateVideo.mock.calls.at(-1)[0].musicVideo)).toMatchObject({ compose: true });
});
});
diff --git a/server/lib/README.md b/server/lib/README.md
index 9788cc8293..87211e253c 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -852,7 +852,8 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `loopInterval.js` | Loop interval grammar (`MIN_INTERVAL_MS`, `parseInterval`, `isValidLoopInterval`) shared by `services/loops.js` and `createLoopSchema` so the route rejects what the service would. |
| `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. |
-| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. |
+| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit only to a request sent before `musicVideoShotPrompt.js` composed clips. |
+| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, repair notes) as a labeled addition. A take that hears no audio is composed without the look and song timing. `authoredShotPrompt(scene)` and `visualDirection(spec)`. Shared by the shot card, the board and the video submit. |
| `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. |
| `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |
diff --git a/server/lib/index.js b/server/lib/index.js
index a5a5402cf4..3382bdd7d4 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -828,6 +828,7 @@ export * from './defaultRequestBody.js';
export * from './musicVideoConditioning.js';
export * from './musicVideoAspect.js';
export * from './musicVideoMotion.js';
+export * from './musicVideoShotPrompt.js';
export * from './musicVideoMediumPlan.js';
export * from './musicVideoFinishedOutside.js';
diff --git a/server/lib/musicVideoActionContract.js b/server/lib/musicVideoActionContract.js
index 28b0a796fd..e738cebb01 100644
--- a/server/lib/musicVideoActionContract.js
+++ b/server/lib/musicVideoActionContract.js
@@ -48,8 +48,12 @@ export function assertShotActionContract(scene) {
}
}
-/** Compile intent and actual song cues to provider clip time. offsetSec is the performance edit in-point. */
-export function shotActionPrompt(project, scene, { frame = false, offsetSec = 0 } = {}) {
+/**
+ * Compile intent and actual song cues to provider clip time. offsetSec is the
+ * performance edit in-point. A take that hears no audio (`heard: false`) gets
+ * no lyric, word or onset timings: it has nothing to time them against.
+ */
+export function shotActionPrompt(project, scene, { frame = false, offsetSec = 0, heard = true } = {}) {
const contract = scene?.direction?.actionContract;
if (contract == null) return '';
assertShotActionContract(scene);
@@ -66,7 +70,7 @@ export function shotActionPrompt(project, scene, { frame = false, offsetSec = 0
for (const [key, label] of [['cameraConstraints', 'Camera'], ['continuityRequirements', 'Must preserve'], ['acceptanceCriteria', 'Acceptance']]) {
for (const item of contract[key] || []) lines.push(`${label}: ${item}`);
}
- if (!frame) {
+ if (!frame && heard) {
const window = (item) => Number.isFinite(item.startSec) && Number.isFinite(item.endSec) && item.endSec > scene.startSec && item.startSec < scene.endSec;
for (const cue of (project.lyricCues || []).filter(window).slice(0, 32)) {
lines.push(`Lyric ${time(Math.max(0, cue.startSec - scene.startSec))}–${time(Math.min(scene.endSec, cue.endSec) - scene.startSec)}: ${cue.text}`);
diff --git a/server/lib/musicVideoShotPrompt.js b/server/lib/musicVideoShotPrompt.js
new file mode 100644
index 0000000000..ae3d528bcf
--- /dev/null
+++ b/server/lib/musicVideoShotPrompt.js
@@ -0,0 +1,75 @@
+/**
+ * The model input for a Music Video shot's clip (#11317): the director's own
+ * text, then each block PortOS adds as a labeled addition the UI lists as a
+ * chip. Shared by the shot card (its "Sent to the model" view), the board's
+ * renders and the server's video submit, so what the card shows is what the
+ * submit sends.
+ *
+ * A take that hears no audio is composed without the blocks it cannot use: no
+ * project look (the reference frame already carries it, and a silent i2v model
+ * acts style words out literally) and no lyric, word or onset timings. Nothing
+ * is removed from the director's text.
+ */
+import { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE } from './musicVideoMotion.js';
+import { musicVideoCreativeContext, musicVideoDirectorLook } from './musicVideoCreativeContext.js';
+import { shotCameraPrompt } from './cameraMovements.js';
+import { shotActionPrompt } from './musicVideoActionContract.js';
+import { approximateMotionCues } from './musicVideoShotTiming.js';
+
+/**
+ * The project-wide visual direction: palette and camera rules from the visual
+ * spec (#8965). Typography is excluded (#8992): it belongs to the composited
+ * text layer, not to the image or video model's prompt.
+ */
+export function visualDirection(spec) {
+ if (!spec) return '';
+ return [
+ spec.palette?.length ? `color palette ${spec.palette.join(' ')}` : '',
+ spec.cameraRules?.trim() ? `camera: ${spec.cameraRules.trim()}` : '',
+ ].filter(Boolean).join('; ');
+}
+
+/** The director's text for a shot's clip: its shot prompt, else its frame prompt. */
+export const authoredShotPrompt = (scene) => scene?.prompt?.trim() || scene?.framePrompt?.trim() || '';
+
+const addition = (id, label, text) => {
+ const trimmed = typeof text === 'string' ? text.trim() : '';
+ return trimmed ? { id, label, text: trimmed } : null;
+};
+
+/**
+ * `{ prompt, promptAdditions }` for one clip. `authored` is the director's text
+ * (the shot card's box, or the scene's saved prompt). `asTyped` is the shot
+ * card's Render, which adds no project look or treatment direction. `hears` is
+ * whether the take hears song audio (lip-sync, supplied song audio or an
+ * audio-reactive LoRA); `offsetSec` re-times the shot actions to a performance
+ * take's padded song slice; `fixes` is an autonomous run's repair note.
+ */
+export function musicVideoShotModelInput(project, scene, {
+ authored = authoredShotPrompt(scene), asTyped = false, hears = false, backend = null,
+ audioReactive = false, offsetSec = 0, fixes = '',
+} = {}) {
+ const text = typeof authored === 'string' ? authored.trim() : '';
+ const look = !asTyped && hears ? [
+ addition('project-style', 'Project style', project?.concept?.style),
+ addition('creative-context', 'Production bible', musicVideoCreativeContext(project?.concept, { moodBoard: false, look: musicVideoDirectorLook(project) })),
+ addition('visual-direction', 'Palette and camera rules', visualDirection(project?.visualSpec)),
+ ] : [];
+ const direction = asTyped ? [] : [
+ addition('shot-direction', 'Shot direction', scene?.direction?.motionClause),
+ addition('lighting-continuity', 'Lighting continuity', MOTION_CONTINUITY_CLAUSE),
+ ];
+ const camera = addition('camera-move', 'Camera move', shotCameraPrompt(scene?.camera));
+ const actions = addition('shot-actions', 'Shot actions', project ? shotActionPrompt(project, scene, { offsetSec, heard: hears }) : '');
+ const tail = audioReactive
+ ? [addition('audio-reactive-guard', 'Audio-reactive guard', AUDIO_REACTIVE_PERFORMANCE_GUARD)]
+ : [
+ addition('phrase-timing', 'Phrase timing (approximate)', backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : ''),
+ addition('repair-notes', 'Repair notes', fixes ? `Fix these problems from the previous take while preserving the shot intent: ${fixes}` : ''),
+ ];
+
+ const firstLine = [text, ...[...look, ...direction].filter(Boolean).map((item) => item.text)].filter(Boolean).join(', ');
+ const body = [firstLine, camera?.text, actions?.text].filter(Boolean).join('\n');
+ const prompt = [body, ...tail.filter(Boolean).map((item) => item.text)].filter(Boolean).join('. ');
+ return { prompt, promptAdditions: [...look, ...direction, camera, actions, ...tail].filter(Boolean) };
+}
diff --git a/server/routes/videoGen.js b/server/routes/videoGen.js
index 1001e35ab5..760b113dc5 100644
--- a/server/routes/videoGen.js
+++ b/server/routes/videoGen.js
@@ -487,6 +487,11 @@ const generateBodySchema = z.object({
// The shot card's Render: `prompt` is exactly what the director sees and
// sends, so the shot-intent lines and the silent-model rewrite are skipped.
promptAsTyped: z.boolean().optional(),
+ // `prompt` is the director's own text and the server composes the rest as
+ // labeled additions (lib/musicVideoShotPrompt.js, #11317); `audioReactive`
+ // names an audio-reactive LoRA take, which gets its guard clause.
+ compose: z.boolean().optional(),
+ audioReactive: z.boolean().optional(),
// The shot card's Song audio toggle: hand a cutaway its song slice (true) or
// render it silent (false), whatever the project's generation mode says.
songAudio: z.boolean().optional(),
diff --git a/server/services/musicVideo/handoff.js b/server/services/musicVideo/handoff.js
index f7ffe7cb06..3804544920 100644
--- a/server/services/musicVideo/handoff.js
+++ b/server/services/musicVideo/handoff.js
@@ -3,6 +3,7 @@ import { shotCameraPrompt } from '../../lib/cameraMovements.js';
import { musicVideoCreativeContext, musicVideoDirectorLook } from '../../lib/musicVideoCreativeContext.js';
import { MOTION_CONTINUITY_CLAUSE } from '../../lib/musicVideoMotion.js';
import { isOutsideMusicVideoReference } from '../../lib/musicVideoConditioning.js';
+import { visualDirection } from '../../lib/musicVideoShotPrompt.js';
/**
* Music Video external-asset handoff (#8965) — provider-neutral export/import
* for tools PortOS does not drive, such as Midjourney.
@@ -64,24 +65,8 @@ export function matchSceneByFileTag(scenes, originalName) {
return matched.size === 1 ? [...matched][0] : null;
}
-/**
- * The project-wide visual direction appended to generated prompts — palette
- * and camera rules from the visual spec. Kept in step with `visualDirection`
- * in client/src/hooks/useMusicVideoSceneMedia.js, which composes the same
- * suffix for PortOS's own renders.
- *
- * Typography is deliberately excluded from this suffix (#8992): it belongs
- * to the separately composited text layer (#8984), not the image/video
- * prompt handed to an external tool. `visualSpec.typography` still exports
- * on the manifest below for the typography lane and other external tools.
- */
-export function visualDirection(spec) {
- if (!spec) return '';
- return [
- spec.palette?.length ? `color palette ${spec.palette.join(' ')}` : '',
- spec.cameraRules?.trim() ? `camera: ${spec.cameraRules.trim()}` : '',
- ].filter(Boolean).join('; ');
-}
+/** The project-wide palette and camera rules (shared with the board and the video submit). */
+export { visualDirection };
// `clause` is the scene's applied treatment direction (#8980) — the composition
// constraints for a frame or a motion prompt. Same order as the client's
diff --git a/server/services/musicVideo/productionDispatch.js b/server/services/musicVideo/productionDispatch.js
index fe1f61d435..349f46b47c 100644
--- a/server/services/musicVideo/productionDispatch.js
+++ b/server/services/musicVideo/productionDispatch.js
@@ -29,8 +29,9 @@ import { withMusicVideoStyle } from './styleReferences.js';
import { ServerError } from '../../lib/errorHandler.js';
import { resolveGalleryImage } from '../../lib/pathSafety.js';
import { RENDER_TARGET } from '../../lib/renderTargets.js';
-import { approximateMotionCues, falSceneTake, falTakeRequestFields, grokCoverage } from '../../lib/musicVideoShotTiming.js';
-import { sceneFramePrompt, sceneShotPrompt } from './handoff.js';
+import { falSceneTake, falTakeRequestFields, grokCoverage } from '../../lib/musicVideoShotTiming.js';
+import { sceneFramePrompt } from './handoff.js';
+import { authoredShotPrompt } from '../../lib/musicVideoShotPrompt.js';
import { sceneEndFrameRequestFields } from '../../lib/musicVideoEndFrame.js';
import { conditioningReferences, falRouteVideoSettings } from './productionPool.js';
import { musicVideoFrameGenSize } from '../../lib/musicVideoAspect.js';
@@ -109,15 +110,13 @@ async function dispatchClip({ project, scene, route, tag }) {
if (!scene.referenceImageId) {
throw new ServerError(`"${scene.label || scene.sceneId}" has no reference frame yet`, { status: 409, code: 'PRODUCTION_FRAME_MISSING' });
}
- const base = sceneShotPrompt(project, scene);
- if (!base) throw unprompted(scene, 'shot');
+ // The director's text; the submit service composes the rest as labeled additions (#11317).
+ const prompt = authoredShotPrompt(scene);
+ if (!prompt) throw unprompted(scene, 'shot');
const spanSec = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' && scene.endSec > scene.startSec
? scene.endSec - scene.startSec : null;
- const isGrok = route.mode === 'grok';
// The autonomous run's pilot repair names what the last clip got wrong (autonomousService.js).
const fixes = typeof scene.clipFixes?.text === 'string' ? scene.clipFixes.text : '';
- const prompt = [base, isGrok ? approximateMotionCues(project.phrases, scene.startSec, scene.endSec) : '',
- fixes ? `Fix these problems from the previous take while preserving the shot intent: ${fixes}` : ''].filter(Boolean).join('. ');
const byBackend = {
// A supplied-audio route animates the frame from its shot's own song window.
local: route.audio === 'supplied' ? { modelId: route.model, mode: 'a2v', _suppliedAudio: true } : { modelId: route.model, disableAudio: true },
@@ -143,7 +142,8 @@ async function dispatchClip({ project, scene, route, tag }) {
...sceneEndFrameRequestFields(scene, {
backend: route.mode, falModelId: byBackend.fal?.falModelId, mode: byBackend[route.mode]?.mode || 'image', localRuntime,
}),
- musicVideo: tag,
+ musicVideo: { ...tag, compose: true },
+ ...(fixes ? { _clipFixes: fixes } : {}),
}, {});
return { jobId };
}
diff --git a/server/services/musicVideo/productionDispatch.test.js b/server/services/musicVideo/productionDispatch.test.js
index 1f8b576b3d..0e65b66145 100644
--- a/server/services/musicVideo/productionDispatch.test.js
+++ b/server/services/musicVideo/productionDispatch.test.js
@@ -157,7 +157,7 @@ describe('production scene dispatch (#9066)', () => {
const clipTag = { ...tag, productionStepKey: 'clip:mvs-a:base:1' };
await dispatchProductionStep({ stepKind: 'clip', project, scene, route: { kind: 'video', mode: 'grok', model: null }, tag: clipTag, settings });
expect(submitVideoGenJob).toHaveBeenCalledWith(expect.objectContaining({
- backend: 'grok', mode: 'image', sourceImageFile: 'frame-example.png', grokDuration: 6, musicVideo: clipTag,
+ backend: 'grok', mode: 'image', sourceImageFile: 'frame-example.png', grokDuration: 6, musicVideo: { ...clipTag, compose: true }, prompt: 'waves roll',
}), {});
});
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index 4370054a0c..6d73fe5413 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -283,6 +283,7 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
} : null;
let repairReserved = false;
+ let composedAdditions = [];
const enqueue = (params) => withStagedRollback(
async () => {
if (repairReserved) {
@@ -299,8 +300,8 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
body.musicVideo.productionStepKey, { sceneId: body.musicVideo.sceneId, kind: 'video' });
}
if (body.musicVideo) {
- const [{ getProject }, { assertShotActionContract, withShotActionPrompt }] = await Promise.all([
- import('../musicVideo/projects.js'), import('../../lib/musicVideoActionContract.js'),
+ const [{ getProject }, { assertShotActionContract, withShotActionPrompt }, { musicVideoShotModelInput }] = await Promise.all([
+ import('../musicVideo/projects.js'), import('../../lib/musicVideoActionContract.js'), import('../../lib/musicVideoShotPrompt.js'),
]);
const project = await getProject(body.musicVideo.projectId);
// A brief that names tools and no video tool never dispatches generated footage;
@@ -313,12 +314,25 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
// A take that hears nothing (no lip-sync or supplied song audio) gets plain physical
// motion: the model would act out song cues and look words instead of reading them.
const hearsAudio = !!(performance || prepared.audio || params.audioConditioning || params.audioFilePath);
- // A shot card's prompt is sent as shown, its shot actions included; only a performance
- // take re-times them to its padded song slice.
- if (!body.musicVideo.promptAsTyped || performance) {
- params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec: performance?.shotInstruction?.edit?.inSec || 0 });
+ const offsetSec = performance?.shotInstruction?.edit?.inSec || 0;
+ if (body.musicVideo.compose) {
+ // The director's text plus each labeled block PortOS adds (#11317), from the same
+ // composer the shot card shows; a take that hears nothing gets no look or song timing.
+ const composed = musicVideoShotModelInput(project, scene, {
+ authored: params.prompt, asTyped: body.musicVideo.promptAsTyped === true, hears: hearsAudio, backend,
+ audioReactive: body.musicVideo.audioReactive === true, offsetSec, fixes: body._clipFixes || '',
+ });
+ params.prompt = composed.prompt;
+ composedAdditions = composed.promptAdditions;
+ } else {
+ // A client from before the composer sends its prompt already composed.
+ // A shot card's prompt is sent as shown, its shot actions included; only a performance
+ // take re-times them to its padded song slice.
+ if (!body.musicVideo.promptAsTyped || performance) {
+ params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec });
+ }
+ if (!hearsAudio && !body.musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);
}
- if (!hearsAudio && !body.musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);
const interval = performance?.shotInstruction?.songInterval || prepared.audio?.songInterval;
if (params.audioConditioning && Math.round(scene?.startSec * 48000) !== params.audioConditioning.startSample) {
throw new ServerError('Scene timing changed while source audio was prepared', { status: 409, code: 'MUSIC_VIDEO_SHOT_TIMING_CHANGED' });
@@ -387,8 +401,9 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
params.remoteMedia.request = buildFederatedMediaRequest({ kind: 'video', params: { ...params.remoteMedia.request, prompt: params.prompt } });
}
if (params.remoteMedia && !params.musicVideo) delete params.prompt;
- // The labeled additions the composer put around the user's text ride with the job to its record.
- if (body.promptAdditions?.length) params.promptAdditions = body.promptAdditions;
+ // The labeled additions the composers put around the user's text ride with the job to its record.
+ const additions = [...(body.promptAdditions || []), ...composedAdditions];
+ if (additions.length) params.promptAdditions = additions;
return enqueueJob({ kind: 'video', params });
},
);
diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.js
index 4c4a588832..075dc221e8 100644
--- a/server/services/videoGen/submitJob.test.js
+++ b/server/services/videoGen/submitJob.test.js
@@ -253,6 +253,35 @@ describe('submitVideoGenJob', () => {
expect(mocks.enqueueJob.mock.calls[0][0].params).toMatchObject({ aspectRatio: '9:16', frameAspect: '9:16' });
});
+ // Uniquely catches the server composing a shot clip differently from the shot card, or
+ // adding look words and song timing a take that hears nothing would act out (#11317).
+ it('composes a board clip from the director\'s text, adding each block as a labeled addition', async () => {
+ const scene = {
+ sceneId: 'mvs-1', startSec: 0, endSec: 4, camera: { move: 'locked-off' },
+ direction: { motionClause: 'slow drift', actionContract: { version: 1, purpose: 'She leaves',
+ actions: [{ startSec: 0, endSec: 1, subject: 'Singer', description: 'Offers a hand' }] } },
+ };
+ const project = { id: 'mv-1', scenes: [scene], concept: { style: 'grainy 16mm' },
+ lyricCues: [{ startSec: 0.5, endSec: 1.5, text: 'hold on', words: [] }] };
+ mocks.getMusicVideoProject.mockResolvedValue(project);
+ mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());
+ await submitVideoGenJob({ prompt: 'she turns away', backend: 'fal', mode: 'image', musicVideo: { ...musicVideo, compose: true } }, {});
+ const silent = mocks.enqueueJob.mock.calls[0][0].params;
+ expect(silent.prompt.startsWith('she turns away, slow drift, ')).toBe(true);
+ expect(silent.prompt).toContain('Action 0.000s–1.000s: Singer');
+ expect(silent.prompt).not.toContain('grainy 16mm');
+ expect(silent.prompt).not.toContain('Lyric');
+ expect(silent.promptAdditions.map((item) => item.id)).toEqual(['shot-direction', 'lighting-continuity', 'camera-move', 'shot-actions']);
+
+ // A take that hears the song gets the project look and the lyric timing, each listed.
+ mocks.prepareVideoGenParams.mockResolvedValue({ ...falPrepared(), audioFilePath: '/example/uploads/song.wav' });
+ await submitVideoGenJob({ prompt: 'she turns away', backend: 'fal', mode: 'image', musicVideo: { ...musicVideo, compose: true } }, {});
+ const hearing = mocks.enqueueJob.mock.calls[1][0].params;
+ expect(hearing.prompt).toContain('grainy 16mm');
+ expect(hearing.prompt).toContain('Lyric 0.500s–1.500s: hold on');
+ expect(hearing.promptAdditions.map((item) => item.id)).toContain('project-style');
+ });
+
it('leaves a clip outside a music video at its own aspect', async () => {
mocks.prepareVideoGenParams.mockResolvedValue(falPrepared());
await submitVideoGenJob({ prompt: 'a fox', backend: 'fal', mode: 'image' }, {});