Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video stills list what PortOS adds to the director's frame prompt

Claudecb77ecf2e637
diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.js
index 7643944011..1d9f06c555 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.js
+++ b/client/src/hooks/useMusicVideoSceneMedia.js
@@ -1,8 +1,7 @@
-import { shotActionContractProblem, shotActionPrompt } from '../../../server/lib/musicVideoActionContract.js';
+import { shotActionContractProblem } from '../../../server/lib/musicVideoActionContract.js';
import { isOutsideMusicVideoReference, musicVideoConditioningReferences, MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES } from '../../../server/lib/musicVideoConditioning.js';
-import { musicVideoCreativeContext, musicVideoDirectorLook } from '../../../server/lib/musicVideoCreativeContext.js';
import { sceneEndFrameRequestFields } from '../../../server/lib/musicVideoEndFrame.js';
-import { authoredShotPrompt, visualDirection } from '../../../server/lib/musicVideoShotPrompt.js';
+import { authoredFramePromptText, authoredShotPrompt, visualDirection } from '../../../server/lib/musicVideoShotPrompt.js';
import { useEffect, useRef } from 'react';
import socket from '../services/socket';
import toast from '../components/ui/Toast';
@@ -26,9 +25,6 @@ export const MAX_CONDITIONING_REFERENCES = MUSIC_VIDEO_MAX_CONDITIONING_REFERENC
// Refusals the image route returns when the resolved backend can't consume
// reference images — surfaced as explicit capability feedback, never retried
// silently without the references.
-// Refine: edit one frame by a short instruction, keeping the rest of the picture.
-const refineFramePrompt = (instruction) => `Edit the reference image: ${instruction.trim()}. Keep everything else exactly as it is: composition, framing, subject, lighting, color and film look.`;
-
const REFERENCE_CAPABILITY_CODES = new Set([
'REFERENCE_IMAGES_FLUX2_ONLY',
'IMAGE_EDIT_UNSUPPORTED_MODE',
@@ -157,18 +153,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const genVideoScenes = videoLane.genScenes;
const failedScenes = { frame: frameLane.failedScenes, video: videoLane.failedScenes };
- const style = project?.concept?.style?.trim();
- const direction = [musicVideoCreativeContext(project?.concept, { look: musicVideoDirectorLook(project) }), visualDirection(project?.visualSpec)].filter(Boolean).join('; ');
const conditioning = conditioningReferences(project);
- // The image prompt for a scene's reference frame: its frame prompt (or the
- // shot prompt as a fallback) suffixed with the project's global concept style
- // and the visual spec's palette/camera direction (typography excluded, #8992).
- // An applied treatment (#8980) adds the scene's composition constraints last:
- // focal subject, framing, the region reserved for the composited typography,
- // and no lettering in the generated pixels. Same order as the server's
- // handoff manifest (handoff.js composePrompt).
- const buildFramePrompt = (scene) =>
- [[(scene.framePrompt?.trim() || scene.prompt?.trim() || ''), style, direction, scene.direction?.frameClause?.trim()].filter(Boolean).join(', '), shotActionPrompt(project, scene, { frame: true })].filter(Boolean).join('\n');
/**
* Render a still reference frame for one scene from its frame prompt. The
@@ -201,10 +186,11 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const conditioning = refining ? [] : conditioningReferences(project, scene);
const problem = refining ? null : shotActionContractProblem(scene.direction?.actionContract, scene);
if (problem) { toast.error(problem); return Promise.resolve({ ok: false }); }
- const framePrompt = refining ? refineFramePrompt(refine) : buildFramePrompt(scene);
- if (!framePrompt) { toast.error('Add a frame prompt or shot prompt first'); return Promise.resolve({ ok: false }); }
+ // The director's text (or a refine's instruction); the server composes the still's
+ // prompt and lists each block it adds (lib/musicVideoShotPrompt.js, #11317).
+ const prompt = refining ? refine.trim() : authoredFramePromptText(scene);
+ if (!prompt) { toast.error('Add a frame prompt or shot prompt first'); return Promise.resolve({ ok: false }); }
// A frame picked on the card (its start or end frame) leads the references, as the image this render reworks.
- const prompt = referenceImageId && !refining ? `${framePrompt}\nStart from the first reference image and rework it to match this prompt.` : framePrompt;
const references = [...(referenceImageId ? [referenceImageId] : []), ...conditioning.map((ref) => ref.imageId).filter((id) => id !== referenceImageId)];
const projectId = project.id;
frameLane.startScene(scene.sceneId);
@@ -212,7 +198,10 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
prompt,
...musicVideoFrameGenSize(project),
...(references.length ? { referenceImageFiles: references } : {}),
- musicVideo: { projectId, sceneId: scene.sceneId, ...(revisionId ? { revisionId } : {}) },
+ musicVideo: {
+ projectId, sceneId: scene.sceneId, compose: true, ...(revisionId ? { revisionId } : {}),
+ ...(refining ? { refine: true } : referenceImageId ? { fromFrame: true } : {}),
+ },
}, { silent: true })
.then((res) => {
const stillRunning = res?.status === 'queued' || res?.status === 'running';
@@ -263,14 +252,14 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const projectId = project.id;
endFrameLane.startScene(scene.sceneId);
return generateImage({
- // From a frame, keep its shot and change what the prompt asks; prompt only, render it like a start frame.
- prompt: refining ? refineFramePrompt(refine)
- : referenceImageId
- ? `${endPrompt}. Keep the reference image's composition, subject and framing; change only what this prompt describes.`
- : [endPrompt, style, direction].filter(Boolean).join(', '),
+ // From a frame the server keeps its shot and changes what the prompt asks; with no frame it renders like a start frame.
+ prompt: refining ? refine.trim() : endPrompt,
...musicVideoFrameGenSize(project),
...(referenceImageId ? { referenceImageFiles: [referenceImageId] } : {}),
- musicVideo: { projectId, sceneId: scene.sceneId, slot: 'end' },
+ musicVideo: {
+ projectId, sceneId: scene.sceneId, slot: 'end', compose: true,
+ ...(refining ? { refine: true } : referenceImageId ? { fromFrame: true } : {}),
+ },
}, { silent: true })
.then((res) => {
if (res?.status === 'queued' || res?.status === 'running') {
@@ -440,7 +429,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const frameScenes = scenes.filter((scene) => !isSelfDrawnLayer(sceneVisualLayer(scene, { layered })));
const footageScenes = scenes.filter((scene) => sceneVisualLayer(scene, { layered }) === 'footage');
const planMissingFrames = () => frameScenes.filter((scene) =>
- !scene.referenceImageId && !genScenes[scene.sceneId] && buildFramePrompt(scene));
+ !scene.referenceImageId && !genScenes[scene.sceneId] && authoredFramePromptText(scene));
// The clips a Videos batch would submit: a footage scene with a frame, no clip
// and nothing in flight. A scene still waiting for its frame never blocks the
@@ -497,7 +486,6 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
frameBatch,
videoBatch,
conditioning,
- buildFramePrompt,
generateFrame,
generateEndFrame,
generateSceneVideo,
diff --git a/client/src/pages/MusicVideo.jsx b/client/src/pages/MusicVideo.jsx
index c6d044d971..caeabf3833 100644
--- a/client/src/pages/MusicVideo.jsx
+++ b/client/src/pages/MusicVideo.jsx
@@ -779,8 +779,8 @@ export default function MusicVideo() {
// commit, same as commitSceneTiming below. Sends only the changed sub-field;
// the server merges it into the existing concept (applyProjectPatch), so a
// stale local copy can't clobber a sibling sub-field. Consumed by
- // buildScenePlanPrompt (AI Plan) and by buildFramePrompt/buildShotPrompt's
- // style suffix, both already reading concept.
+ // buildScenePlanPrompt (AI Plan) and by the server's frame and clip composers
+ // (lib/musicVideoShotPrompt.js), both already reading concept.
const commitConcept = (patch) => {
replaceProject({ ...selected, concept: { ...selected.concept, ...patch } });
updateMusicVideoProject(selected.id, { concept: patch }, { silent: true })
diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsx
index 3180199cf4..fb20891a6e 100644
--- a/client/src/pages/MusicVideo.test.jsx
+++ b/client/src/pages/MusicVideo.test.jsx
@@ -2435,20 +2435,6 @@ describe('MusicVideo visual spec, takes and handoff (#8965)', () => {
scenes: [{ sceneId: 's1', order: 0, prompt: 'waves', framePrompt: 'harbor at dawn', referenceImageId: null, videoHistoryId: null, takes: [] }],
};
- it('carries authored character identity and source styles into actual frame requests', async () => {
- await openProject({ ...SPEC_PROJECT, concept: { ...SPEC_PROJECT.concept,
- universeStyle: 'Ink silhouettes', moodBoardStyle: 'Watercolor', subjects: [
- { id: 'lead', kind: 'character', role: 'protagonist', name: 'Example singer', description: 'Silver coat' },
- ],
- } }, 'produce');
- fireEvent.click(screen.getByRole('button', { name: /^Generate frame$/ }));
- await waitFor(() => expect(generateImage).toHaveBeenCalledWith(expect.objectContaining({
- prompt: expect.stringContaining('character (protagonist): Example singer — Silver coat'),
- }), { silent: true }));
- expect(generateImage.mock.calls[0][0].prompt).toContain('Mood board look (palette, lighting and texture only; never its locations, objects or poses): Watercolor');
- expect(generateImage.mock.calls[0][0].prompt).toContain('Universe style: Ink silhouettes');
- });
-
it('sends flagged references as conditioning inputs and names the capability gap when the backend refuses them', async () => {
generateImage.mockRejectedValueOnce(Object.assign(
new Error('Reference images are only supported for FLUX.2 and Qwen Image 2.1 models on the local backend'),
@@ -2457,12 +2443,13 @@ describe('MusicVideo visual spec, takes and handoff (#8965)', () => {
await openProject(SPEC_PROJECT, 'produce');
fireEvent.click(screen.getByRole('button', { name: /^Generate frame$/ }));
await waitFor(() => expect(generateImage).toHaveBeenCalledWith({
- prompt: 'harbor at dawn, grainy 16mm, color palette #112233; camera: locked-off wides',
+ // The director's text; the server composes the look onto it (lib/musicVideoShotPrompt.js).
+ prompt: 'harbor at dawn',
// Frames are requested at the project's aspect (16:9 by default).
width: 1536,
height: 864,
referenceImageFiles: ['mood.png'],
- musicVideo: { projectId: 'mv-spec', sceneId: 's1' },
+ musicVideo: { projectId: 'mv-spec', sceneId: 's1', compose: true },
}, { silent: true }));
await waitFor(() => expect(toast.error).toHaveBeenCalledWith(
expect.stringMatching(/can't condition on 1 reference image: Reference images are only supported/),
@@ -2483,7 +2470,7 @@ describe('MusicVideo visual spec, takes and handoff (#8965)', () => {
fireEvent.click(screen.getByRole('button', { name: /^Generate frame$/ }));
await waitFor(() => expect(generateImage).toHaveBeenCalledWith(expect.objectContaining({
referenceImageFiles: ['character.png', 'roof.png'],
- musicVideo: { projectId: 'mv-spec', sceneId: 's1' },
+ musicVideo: { projectId: 'mv-spec', sceneId: 's1', compose: true },
}), { silent: true }));
});
@@ -2493,8 +2480,9 @@ describe('MusicVideo visual spec, takes and handoff (#8965)', () => {
fireEvent.change(screen.getByLabelText('From', { selector: '#mv-scene-s1-start-from' }), { target: { value: 'start' } });
fireEvent.click(screen.getByRole('button', { name: /^New frame take$/ }));
await waitFor(() => expect(generateImage).toHaveBeenCalledWith(expect.objectContaining({
- prompt: expect.stringContaining('Start from the first reference image'),
+ prompt: 'harbor at dawn',
referenceImageFiles: ['start.png', 'mood.png'],
+ musicVideo: { projectId: 'mv-spec', sceneId: 's1', compose: true, fromFrame: true },
}), { silent: true }));
});
@@ -2505,9 +2493,9 @@ describe('MusicVideo visual spec, takes and handoff (#8965)', () => {
fireEvent.change(screen.getByLabelText('What to change in the start frame'), { target: { value: 'remove the hand' } });
fireEvent.click(screen.getByRole('button', { name: 'Render refined start frame' }));
await waitFor(() => expect(generateImage).toHaveBeenCalledWith(expect.objectContaining({
- prompt: expect.stringMatching(/^Edit the reference image: remove the hand\. Keep everything else/),
+ prompt: 'remove the hand',
referenceImageFiles: ['start.png'],
- musicVideo: { projectId: 'mv-spec', sceneId: 's1' },
+ musicVideo: { projectId: 'mv-spec', sceneId: 's1', compose: true, refine: true },
}), { silent: true }));
expect(screen.queryByLabelText('What to change in the start frame')).toBeNull();
});
diff --git a/server/lib/README.md b/server/lib/README.md
index 87211e253c..3ef92ea1a7 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -853,7 +853,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. |
| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit only to a request sent before `musicVideoShotPrompt.js` composed clips. |
-| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, repair notes) as a labeled addition. A take that hears no audio is composed without the look and song timing. `authoredShotPrompt(scene)` and `visualDirection(spec)`. Shared by the shot card, the board and the video submit. |
+| `musicVideoShotPrompt.js` | The model input for a Music Video shot's clip and stills (#11317): `musicVideoShotModelInput(project, scene, opts)` → `{ prompt, promptAdditions }`, the director's text plus each block PortOS adds (project look, bible, palette/camera rules, shot direction, lighting continuity, camera move, shot actions, Grok phrase timing, audio-reactive guard, repair notes) as a labeled addition; a take that hears no audio is composed without the look, song timing or on-beat camera clause. `musicVideoFrameModelInput(project, scene, { slot, fromFrame, refine, repairs, fixes })` does the same for a start or end frame (look, frame direction, plate actions, rework line, keep-the-shot line, refine wrapper). `authoredShotPrompt`, `authoredFramePromptText` and `visualDirection`. Shared by the shot card, the board, the production run and the image and video submits. |
| `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. |
| `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |
diff --git a/server/lib/musicVideoShotPrompt.js b/server/lib/musicVideoShotPrompt.js
index 583587a487..9bfc030cf9 100644
--- a/server/lib/musicVideoShotPrompt.js
+++ b/server/lib/musicVideoShotPrompt.js
@@ -1,7 +1,7 @@
/**
- * The model input for a Music Video shot's clip (#11317): the director's own
- * text, then each block PortOS adds as a labeled addition the UI lists as a
- * chip. Shared by the shot card (its "Sent to the model" view), the board's
+ * The model input for a Music Video shot's clip and its start and end frames
+ * (#11317): the director's own text, then each block PortOS adds as a labeled
+ * addition the UI lists as a chip. Shared by the shot card (its "Sent to the model" view), the board's
* renders and the server's video submit, so what the card shows is what the
* submit sends.
*
@@ -75,3 +75,52 @@ export function musicVideoShotModelInput(project, scene, {
const prompt = [body, ...tail.filter(Boolean).map((item) => item.text)].filter(Boolean).join('. ');
return { prompt, promptAdditions: [...look, ...direction, camera, actions, ...tail].filter(Boolean) };
}
+
+/** A refine edit's wrapper around the director's instruction ("[instruction]" marks where it goes). */
+const refineWrapper = (instruction) => `Edit the reference image: ${instruction}. Keep everything else exactly as it is: composition, framing, subject, lighting, color and film look.`;
+const REWORK_FRAME = 'Start from the first reference image and rework it to match this prompt.';
+const KEEP_REFERENCE = 'Keep the reference image\'s composition, subject and framing; change only what this prompt describes.';
+
+/** The director's text for a shot's start or end frame. */
+export const authoredFramePromptText = (scene, slot = 'start') => (slot === 'end'
+ ? scene?.endFramePrompt?.trim() || ''
+ : scene?.framePrompt?.trim() || scene?.prompt?.trim() || '');
+
+/**
+ * `{ prompt, promptAdditions }` for a shot's still: its start frame (`slot`
+ * 'start') or style-break end frame ('end'). `fromFrame` is a render that
+ * reworks a picked frame (the first reference image); `refine` makes `authored`
+ * an edit instruction for that frame, wrapped and sent with nothing else.
+ * `repairs` (plate mismatches) and `fixes` (a stills review's notes) are an
+ * autonomous run's repair notes. The frame text keeps the project look the
+ * mood board's own style block joins later (imageGen/prepareParams.js).
+ */
+export function musicVideoFrameModelInput(project, scene, {
+ slot = 'start', authored = authoredFramePromptText(scene, slot), fromFrame = false, refine = false,
+ repairs = [], fixes = '',
+} = {}) {
+ const text = typeof authored === 'string' ? authored.trim() : '';
+ if (!text) return { prompt: '', promptAdditions: [] };
+ if (refine) {
+ return { prompt: refineWrapper(text), promptAdditions: [addition('edit-instructions', 'Edit instructions', refineWrapper('[instruction]'))] };
+ }
+ if (slot === 'end' && fromFrame) {
+ return { prompt: `${text}. ${KEEP_REFERENCE}`, promptAdditions: [addition('keep-reference', 'Keep the start frame', KEEP_REFERENCE)] };
+ }
+ const look = [
+ addition('project-style', 'Project style', project?.concept?.style),
+ addition('creative-context', 'Production bible', musicVideoCreativeContext(project?.concept, { look: musicVideoDirectorLook(project) })),
+ addition('visual-direction', 'Palette and camera rules', visualDirection(project?.visualSpec)),
+ ...(slot === 'end' ? [] : [addition('frame-direction', 'Frame direction', scene?.direction?.frameClause)]),
+ ].filter(Boolean);
+ const firstLine = [text, ...look.map((item) => item.text)].join(', ');
+ if (slot === 'end') return { prompt: firstLine, promptAdditions: look };
+ const unmet = [...new Set(repairs)];
+ const tail = [
+ addition('shot-actions', 'Shot actions', project ? shotActionPrompt(project, scene, { frame: true }) : ''),
+ fromFrame ? addition('rework-frame', 'Rework the picked frame', REWORK_FRAME) : null,
+ addition('plate-repairs', 'Plate repairs', unmet.length ? `Repair these visible plate mismatches while preserving the shot intent:\n${unmet.join('\n')}` : ''),
+ addition('repair-notes', 'Repair notes', fixes ? `Fix these visible problems from the previous still while preserving the shot intent:\n${fixes}` : ''),
+ ].filter(Boolean);
+ return { prompt: [firstLine, ...tail.map((item) => item.text)].join('\n'), promptAdditions: [...look, ...tail] };
+}
diff --git a/server/lib/musicVideoShotPrompt.test.js b/server/lib/musicVideoShotPrompt.test.js
new file mode 100644
index 0000000000..c1b7555833
--- /dev/null
+++ b/server/lib/musicVideoShotPrompt.test.js
@@ -0,0 +1,53 @@
+import { describe, expect, it } from 'vitest';
+import { musicVideoFrameModelInput } from './musicVideoShotPrompt.js';
+
+const project = {
+ concept: {
+ style: 'grainy 16mm', universeStyle: 'Ink silhouettes', moodBoardStyle: 'Watercolor',
+ subjects: [{ id: 'lead', kind: 'character', role: 'protagonist', name: 'Example singer', description: 'Silver coat' }],
+ },
+ visualSpec: { palette: ['#112233'], cameraRules: 'locked-off wides', typography: 'condensed sans' },
+};
+const scene = { sceneId: 's1', startSec: 0, endSec: 4, framePrompt: 'harbor at dawn', endFramePrompt: 'the same frame as a pencil sketch', direction: { frameClause: 'subject left third' } };
+const ids = (input) => input.promptAdditions.map((item) => item.id);
+
+// Uniquely catches a still's prompt drifting from the chips it lists, or a block
+// reaching a frame kind that should not carry it (#11317).
+describe('musicVideoFrameModelInput', () => {
+ it('composes a start frame from the director\'s text with each look block listed', () => {
+ const input = musicVideoFrameModelInput(project, scene);
+ expect(input.prompt.startsWith('harbor at dawn, grainy 16mm, ')).toBe(true);
+ expect(input.prompt).toContain('character (protagonist): Example singer — Silver coat');
+ expect(input.prompt).toContain('Universe style: Ink silhouettes');
+ expect(input.prompt).toContain('color palette #112233; camera: locked-off wides');
+ expect(input.prompt).not.toContain('condensed sans');
+ expect(ids(input)).toEqual(['project-style', 'creative-context', 'visual-direction', 'frame-direction']);
+ for (const item of input.promptAdditions) expect(input.prompt).toContain(item.text);
+ });
+
+ it('adds the rework line for a picked frame and the run\'s repair notes', () => {
+ const input = musicVideoFrameModelInput(project, scene, { fromFrame: true, repairs: ['hand: six fingers', 'hand: six fingers'], fixes: 'too dark' });
+ expect(input.prompt).toContain('\nStart from the first reference image and rework it to match this prompt.');
+ expect(input.prompt.match(/six fingers/g)).toHaveLength(1);
+ expect(ids(input).slice(-3)).toEqual(['rework-frame', 'plate-repairs', 'repair-notes']);
+ });
+
+ it('keeps an end frame rendered from the start frame to its own text plus the keep-the-shot line', () => {
+ const fromStart = musicVideoFrameModelInput(project, scene, { slot: 'end', fromFrame: true });
+ expect(fromStart.prompt).toBe('the same frame as a pencil sketch. Keep the reference image\'s composition, subject and framing; change only what this prompt describes.');
+ expect(ids(fromStart)).toEqual(['keep-reference']);
+ const fresh = musicVideoFrameModelInput(project, scene, { slot: 'end' });
+ expect(fresh.prompt.startsWith('the same frame as a pencil sketch, grainy 16mm, ')).toBe(true);
+ expect(ids(fresh)).toEqual(['project-style', 'creative-context', 'visual-direction']);
+ });
+
+ it('wraps a refine instruction and sends nothing else', () => {
+ const input = musicVideoFrameModelInput(project, scene, { authored: 'remove the hand', refine: true });
+ expect(input.prompt).toMatch(/^Edit the reference image: remove the hand\. Keep everything else exactly as it is/);
+ expect(input.promptAdditions).toEqual([expect.objectContaining({ id: 'edit-instructions', text: expect.stringContaining('[instruction]') })]);
+ });
+
+ it('sends nothing for a still with no text', () => {
+ expect(musicVideoFrameModelInput(project, { sceneId: 's2' })).toEqual({ prompt: '', promptAdditions: [] });
+ });
+});
diff --git a/server/routes/imageGen.js b/server/routes/imageGen.js
index 2cdbd8c58f..f21c5f3cc5 100644
--- a/server/routes/imageGen.js
+++ b/server/routes/imageGen.js
@@ -241,6 +241,12 @@ const generateSchema = z.object({
// #11173: 'end' files the render as the scene's style-break end frame
// (an `endImage` take) instead of its start frame.
slot: z.enum(['end']).optional(),
+ // #11317: the prompt is the director's text; the server composes the still's
+ // prompt and lists each block it adds. `fromFrame` reworks the first reference
+ // image (a picked frame); `refine` makes the prompt an edit instruction for it.
+ compose: z.boolean().optional(),
+ fromFrame: z.boolean().optional(),
+ refine: z.boolean().optional(),
}).optional(),
// File the render into this music video's media collection even when no shot
// keeps it (sheets, tests, alternates): musicVideoCollectionHook for the
diff --git a/server/services/imageGen/prepareParams.js b/server/services/imageGen/prepareParams.js
index d75e94257a..5a9ff1c4ab 100644
--- a/server/services/imageGen/prepareParams.js
+++ b/server/services/imageGen/prepareParams.js
@@ -29,6 +29,7 @@
import { randomUUID } from 'crypto';
import { musicVideoStyleImages, musicVideoStylePrompt, musicVideoStyleReferenceCapacity } from '../../lib/musicVideoConditioning.js';
+import { musicVideoFrameModelInput } from '../../lib/musicVideoShotPrompt.js';
import { join } from 'node:path';
import { ServerError } from '../../lib/errorHandler.js';
import { PATHS, ensureDir, resolveGalleryImage, copyFileGuarded, unlinkGuarded } from '../../lib/fileUtils.js';
@@ -231,6 +232,17 @@ export async function prepareGenerateParams({ data, files, referenceImageFields,
const localModel = mode === IMAGE_GEN_MODE.LOCAL ? selectLocalImageModelFromSettings(settings, data.modelId, getImageModels(), localModelCandidates) : null;
if (data.musicVideo?.sceneId) {
if (!musicVideoProjectLoaded) musicVideoProject = await getMusicVideoProject(data.musicVideo.projectId).catch(() => null);
+ if (data.musicVideo.compose) {
+ // The director's text plus each labeled block PortOS adds (#11317), from the
+ // composer the server's production run uses for the same still.
+ const scene = musicVideoProject?.scenes?.find((entry) => entry.sceneId === data.musicVideo.sceneId);
+ const composed = musicVideoFrameModelInput(musicVideoProject, scene, {
+ slot: data.musicVideo.slot === 'end' ? 'end' : 'start', authored: data.prompt || '',
+ fromFrame: data.musicVideo.fromFrame === true, refine: data.musicVideo.refine === true,
+ });
+ data.prompt = composed.prompt;
+ if (composed.promptAdditions.length) data.promptAdditions = [...(data.promptAdditions || []), ...composed.promptAdditions];
+ }
const reservedSlots = referenceUploads.length + (initUpload || data.initImageFile ? 1 : 0);
const capacity = Math.max(0, musicVideoStyleReferenceCapacity(mode, localModel) - reservedSlots);
const styles = musicVideoStyleImages(musicVideoProject, namedReferenceFiles, capacity);
diff --git a/server/services/imageGen/prepareParams.test.js b/server/services/imageGen/prepareParams.test.js
index 38196dca38..b0d5dc4c7d 100644
--- a/server/services/imageGen/prepareParams.test.js
+++ b/server/services/imageGen/prepareParams.test.js
@@ -284,6 +284,17 @@ it('uses moodboard caption text on a backend without image-reference support', a
expect(data.referenceImagePaths).toBeUndefined();
});
+// Uniquely catches a board still sent with the director's text but without the
+// server's composed look, or with the look and no chips for it (#11317).
+it('composes a board still from the director\'s text and lists what it adds', async () => {
+ getSettings.mockResolvedValue({ imageGen: { mode: 'external' } });
+ getProject.mockResolvedValue({ concept: { style: 'grainy 16mm' }, styleReferences: [{ imageId: 'style.png', caption: 'fine silver grain' }],
+ scenes: [{ sceneId: 's1', framePrompt: 'harbor at dawn' }] });
+ const { data } = await run({ prompt: 'harbor at dawn', musicVideo: { projectId: 'mv-1', sceneId: 's1', compose: true, fromFrame: true } });
+ expect(data.prompt).toMatch(/^harbor at dawn, grainy 16mm\nStart from the first reference image.*\. Moodboard style only/s);
+ expect(data.promptAdditions.map((item) => item.id)).toEqual(['project-style', 'rework-frame', 'mood-board-style']);
+});
+
// Pins cleaner precedence where requests are prepared, without loading a backend.
describe('prepared image cleaner policy', () => {
it('inherits saved per-mode flags and accepts explicit boolean overrides', async () => {
diff --git a/server/services/musicVideo/handoff.js b/server/services/musicVideo/handoff.js
index 3804544920..bab6c5dbc1 100644
--- a/server/services/musicVideo/handoff.js
+++ b/server/services/musicVideo/handoff.js
@@ -3,7 +3,7 @@ import { shotCameraPrompt } from '../../lib/cameraMovements.js';
import { musicVideoCreativeContext, musicVideoDirectorLook } from '../../lib/musicVideoCreativeContext.js';
import { MOTION_CONTINUITY_CLAUSE } from '../../lib/musicVideoMotion.js';
import { isOutsideMusicVideoReference } from '../../lib/musicVideoConditioning.js';
-import { visualDirection } from '../../lib/musicVideoShotPrompt.js';
+import { musicVideoFrameModelInput, visualDirection } from '../../lib/musicVideoShotPrompt.js';
/**
* Music Video external-asset handoff (#8965) — provider-neutral export/import
* for tools PortOS does not drive, such as Midjourney.
@@ -107,7 +107,7 @@ export function authoredFramePrompt(project, prompt) {
}
/** A scene's composed reference-frame prompt (also the server-owned production run's, #9066). */
-export const sceneFramePrompt = (project, scene) => [composePrompt(scene.framePrompt, scene.prompt, project, scene.direction?.frameClause), shotActionPrompt(project, scene, { frame: true })].filter(Boolean).join('\n');
+export const sceneFramePrompt = (project, scene) => musicVideoFrameModelInput(project, scene).prompt;
/**
* A scene's composed shot (motion) prompt. A planned shot (#10589) carries its
diff --git a/server/services/musicVideo/productionDispatch.js b/server/services/musicVideo/productionDispatch.js
index 349f46b47c..dceb3843f0 100644
--- a/server/services/musicVideo/productionDispatch.js
+++ b/server/services/musicVideo/productionDispatch.js
@@ -30,8 +30,7 @@ import { ServerError } from '../../lib/errorHandler.js';
import { resolveGalleryImage } from '../../lib/pathSafety.js';
import { RENDER_TARGET } from '../../lib/renderTargets.js';
import { falSceneTake, falTakeRequestFields, grokCoverage } from '../../lib/musicVideoShotTiming.js';
-import { sceneFramePrompt } from './handoff.js';
-import { authoredShotPrompt } from '../../lib/musicVideoShotPrompt.js';
+import { authoredShotPrompt, musicVideoFrameModelInput } from '../../lib/musicVideoShotPrompt.js';
import { sceneEndFrameRequestFields } from '../../lib/musicVideoEndFrame.js';
import { conditioningReferences, falRouteVideoSettings } from './productionPool.js';
import { musicVideoFrameGenSize } from '../../lib/musicVideoAspect.js';
@@ -49,13 +48,11 @@ async function guardRevision(tag, kind) {
/** Enqueue a scene's reference frame on exactly `route`. Returns `{ jobId }`. */
async function dispatchFrame({ project, scene, route, tag, settings }) {
- const basePrompt = sceneFramePrompt(project, scene);
const repairing = project.productionRuns?.find((run) => run.id === tag.productionRunId)?.steps?.find((step) => step.key === tag.productionStepKey)?.plateRepairBasis;
const unmet = repairing ? (scene.takes || []).flatMap((take) => currentPlateEvidence(scene, take)?.checks.filter((check) => check.status === 'fail').map((check) => `${check.requirement}: ${check.note}`) || []) : [];
// The autonomous run's stills review names what the last still got wrong (autonomousService.js).
const fixes = typeof scene.stillFixes?.text === 'string' ? scene.stillFixes.text : '';
- const prompt = basePrompt && [basePrompt, unmet.length ? `Repair these visible plate mismatches while preserving the shot intent:\n${[...new Set(unmet)].join('\n')}` : '',
- fixes ? `Fix these visible problems from the previous still while preserving the shot intent:\n${fixes}` : ''].filter(Boolean).join('\n');
+ const { prompt, promptAdditions } = musicVideoFrameModelInput(project, scene, { repairs: unmet, fixes });
if (!prompt) throw unprompted(scene, 'frame');
const referenceImagePaths = conditioningReferences(project, scene)
.map((ref) => resolveGalleryImage(ref.imageId, { mustExist: false })).filter(Boolean);
@@ -72,6 +69,7 @@ async function dispatchFrame({ project, scene, route, tag, settings }) {
const { cleanC2PA, denoise } = resolveImageCleaners(undefined, settings, route.mode);
const common = {
prompt,
+ ...(promptAdditions.length ? { promptAdditions } : {}),
// The project's aspect, never the backend's per-prompt choice (a portrait
// close-up pillarboxes in a 16:9 render).
...musicVideoFrameGenSize(project),
diff --git a/server/services/musicVideo/styleReferences.js b/server/services/musicVideo/styleReferences.js
index a1466bc3d7..db2433dd66 100644
--- a/server/services/musicVideo/styleReferences.js
+++ b/server/services/musicVideo/styleReferences.js
@@ -1,5 +1,6 @@
import { basename } from 'node:path';
import { resolveGalleryImage } from '../../lib/pathSafety.js';
+import { promptAddition } from '../../lib/modelInput.js';
import { musicVideoStyleImages, musicVideoStylePrompt, musicVideoStyleReferenceCapacity } from '../../lib/musicVideoConditioning.js';
/** Resolve style capacity on the actual route, preserving every explicit identity input. */
@@ -16,9 +17,11 @@ export async function withMusicVideoStyle(project, params, mode, model, settings
const capacity = musicVideoStyleReferenceCapacity(mode, localModel);
const styles = musicVideoStyleImages(project, identities.map((p) => basename(p)), capacity)
.map((id) => resolveGalleryImage(id, { mustExist: false })).filter((p) => p && !identities.includes(p));
+ const look = musicVideoStylePrompt(project, styles.length);
return {
...params,
- prompt: [params.prompt, musicVideoStylePrompt(project, styles.length)].filter(Boolean).join('. '),
+ prompt: [params.prompt, look].filter(Boolean).join('. '),
+ promptAdditions: [...(params.promptAdditions || []), promptAddition('mood-board-style', 'Mood board style', look)].filter(Boolean),
...(styles.length ? {
referenceImagePaths: [...identities, ...styles],
referenceImageStrengths: [...(params.referenceImageStrengths || identities.map(() => 1)), ...styles.map(() => 1)],