Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video: pick the character references a shot's frames send (#11483)

[._.]/2cf877f867a8
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 453ad0a674..37798027f9 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -500,7 +500,7 @@ export default function SceneCard({
<div className="grid gap-3 @3xl:grid-cols-2">
{/* Frames: the still this shot starts from and, optionally, the one it ends on (#11173),
each with its prompt and Render under it, then the tapped frame's takes. */}
- <SceneFrames scene={scene} index={index} endFrames={endFrames} generatingFrame={!!generatingFrame} generatingEndFrame={!!generatingEndFrame}
+ <SceneFrames project={project} scene={scene} index={index} endFrames={endFrames} generatingFrame={!!generatingFrame} generatingEndFrame={!!generatingEndFrame}
onGenerateFrame={onGenerateFrame} onGenerateEndFrame={onGenerateEndFrame} onEditLocal={onEditLocal} onSave={onSave}
onOpenPreview={onOpenPreview} takeBusy={takeBusy} onSelectTake={onSelectTake} onReviewTake={onReviewTake}
onSwap={onPlacePoolItem ? setSwapKind : null} onSwapFrames={onSwapFrames ? () => onSwapFrames(scene) : null} onImportTake={onImportTake}
diff --git a/client/src/components/musicVideo/SceneFrameReferences.jsx b/client/src/components/musicVideo/SceneFrameReferences.jsx
new file mode 100644
index 0000000000..c9dc975c52
--- /dev/null
+++ b/client/src/components/musicVideo/SceneFrameReferences.jsx
@@ -0,0 +1,79 @@
+import { useState } from 'react';
+import { Plus, X } from 'lucide-react';
+import { musicVideoConditioningReferences, musicVideoReferenceChoices, MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES } from '../../../../server/lib/musicVideoConditioning.js';
+import { MUSIC_VIDEO_OTHER_REFERENCES_NOTE } from '../../../../server/lib/musicVideoShotPrompt.js';
+
+const CHIP = 'inline-flex min-h-[44px] max-w-full items-center gap-1 rounded border border-port-border bg-port-bg p-0.5 text-xs sm:min-h-[32px]';
+const thumb = (ref) => <img src={`/data/images/${encodeURIComponent(ref.imageId)}`} alt="" className="h-8 w-8 shrink-0 rounded bg-black object-cover" />;
+const nameOf = (ref) => ref.label || ref.role || 'Reference';
+
+/**
+ * The character and set references sent with this shot's frame renders (start,
+ * end and both refines), shown as chips. Until the director changes them it is
+ * the automatic set (the main character, the section's set and anyone the
+ * prompt names); a change is kept on the shot (`frameReferenceIds`), and Auto
+ * puts the automatic set back.
+ */
+export default function SceneFrameReferences({ project, scene, onEditLocal, onSave, disabled = false }) {
+ const [adding, setAdding] = useState(false);
+ const choices = musicVideoReferenceChoices(project);
+ if (!choices.length) return null;
+ const chosen = musicVideoConditioningReferences(project, scene);
+ const ids = chosen.map((ref) => ref.id);
+ const others = choices.filter((ref) => !ids.includes(ref.id));
+ const auto = !Array.isArray(scene.frameReferenceIds);
+ const full = ids.length >= MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES;
+ const set = (next) => { onEditLocal(scene.sceneId, { frameReferenceIds: next }); onSave(scene.sceneId, { frameReferenceIds: next }); };
+
+ return (
+ <div role="group" aria-label="References sent with the frames" className="space-y-1">
+ <div className="flex flex-wrap items-center gap-x-2 text-[11px] text-port-text-muted">
+ <span className="font-medium text-port-text">References</span>
+ {auto ? <span>Auto: from the prompt</span> : (
+ <button type="button" onClick={() => set(null)} disabled={disabled} className="min-h-[44px] underline sm:min-h-0">Back to auto</button>
+ )}
+ </div>
+ <ul className="flex flex-wrap gap-1">
+ {chosen.map((ref) => (
+ <li key={ref.id} className={CHIP}>
+ {thumb(ref)}
+ <span className="min-w-0 max-w-[10rem] truncate">{nameOf(ref)}</span>
+ <button type="button" onClick={() => set(ids.filter((id) => id !== ref.id))} disabled={disabled} aria-label={`Don't send ${nameOf(ref)}`}
+ className="inline-flex min-h-[40px] min-w-[32px] items-center justify-center rounded hover:bg-port-border disabled:opacity-50 sm:min-h-[28px] sm:min-w-[24px]">
+ <X size={12} aria-hidden="true" />
+ </button>
+ </li>
+ ))}
+ {others.length > 0 && (
+ <li>
+ <button type="button" onClick={() => setAdding((open) => !open)} disabled={disabled || full} aria-expanded={adding}
+ title={full ? `Up to ${MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES} references` : 'Send another character or set'}
+ className={`${CHIP} px-2 disabled:opacity-50`}>
+ <Plus size={12} aria-hidden="true" /> Add
+ </button>
+ </li>
+ )}
+ </ul>
+ {full && <p className="text-[11px] text-port-text-muted">Refines and end frames send the frame plus the first {MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES - 1}.</p>}
+ {chosen.length === 0 && <p className="text-[11px] text-port-text-muted">No references: frames render from the prompt alone.</p>}
+ {adding && !full && (
+ <ul aria-label="References to add" className="grid grid-cols-2 gap-1 sm:grid-cols-3 lg:grid-cols-4">
+ {others.map((ref) => (
+ <li key={ref.id} className="min-w-0">
+ <button type="button" onClick={() => { set([...ids, ref.id]); setAdding(false); }} disabled={disabled}
+ className={`${CHIP} w-full text-left hover:bg-port-border/40`}>
+ {thumb(ref)}<span className="min-w-0 truncate">{nameOf(ref)}</span>
+ </button>
+ </li>
+ ))}
+ </ul>
+ )}
+ {chosen.length > 0 && (
+ <details className="text-[11px] text-port-text-muted">
+ <summary className="cursor-pointer">Note sent with refines and end frames</summary>
+ <p className="mt-1">{MUSIC_VIDEO_OTHER_REFERENCES_NOTE}</p>
+ </details>
+ )}
+ </div>
+ );
+}
diff --git a/client/src/components/musicVideo/SceneFrames.jsx b/client/src/components/musicVideo/SceneFrames.jsx
index e2290554bd..32714b2457 100644
--- a/client/src/components/musicVideo/SceneFrames.jsx
+++ b/client/src/components/musicVideo/SceneFrames.jsx
@@ -1,6 +1,7 @@
import { useLayoutEffect, useRef, useState } from 'react';
import { Activity, ArrowLeftRight, ArrowRight, Image as ImageIcon, ImagePlus, PencilLine, Shuffle, SkipForward, Wand2, X } from 'lucide-react';
import SceneTakeStrip from './SceneTakeStrip.jsx';
+import SceneFrameReferences from './SceneFrameReferences.jsx';
const PRIMARY_BUTTON = 'flex items-center gap-1 bg-port-border hover:bg-port-border/70 disabled:opacity-50 rounded px-2 py-1.5 text-xs min-h-[44px] sm:min-h-0 whitespace-nowrap';
const SECONDARY_BUTTON = 'flex items-center gap-1 bg-port-bg border border-port-border hover:bg-port-border/40 disabled:opacity-50 rounded px-2 py-1.5 text-xs min-h-[44px] sm:min-h-0 whitespace-nowrap';
@@ -59,9 +60,10 @@ const REFERENCE_LABELS = { '': 'Prompt only', start: 'Start frame', end: 'End fr
* video's unused stills on it. Each frame's own takes sit under it as a grid,
* so both frames' actions and takes show at once; tapping a frame opens it full size.
* With both frames set, `onSwapFrames` trades them (prompts included) in one tap.
+ * The References row above them picks the character and set references every frame render sends.
*/
export default function SceneFrames({
- scene, index, endFrames, generatingFrame = false, generatingEndFrame = false, onGenerateFrame, onGenerateEndFrame,
+ project = null, scene, index, endFrames, generatingFrame = false, generatingEndFrame = false, onGenerateFrame, onGenerateEndFrame,
onEditLocal, onSave, onOpenPreview, takeBusy = false, onSelectTake, onReviewTake, onSwap = null, onImportTake = null,
onContinueFromPrevious = null, previousHasClip = false, clipLocked = false, onSwapFrames = null, assetMarks = null, onMarkTake = null,
}) {
@@ -184,6 +186,7 @@ export default function SceneFrames({
return (
<section aria-label="Still" className="@container min-w-0 space-y-2">
<h4 className="text-xs font-medium">{endFrames ? 'Frames' : 'Still'}</h4>
+ <SceneFrameReferences project={project} scene={scene} onEditLocal={onEditLocal} onSave={onSave} />
{endFrames ? (
<div role="group" aria-label="Start and end frames" className="grid gap-3 @sm:grid-cols-[minmax(0,1fr)_auto_minmax(0,1fr)]">
{column('start')}
diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.js
index 1bb56665b5..36637d6dea 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.js
+++ b/client/src/hooks/useMusicVideoSceneMedia.js
@@ -37,6 +37,14 @@ export { visualDirection };
/** Shared scene-aware frame conditioning, with the legacy capped fallback. */
export const conditioningReferences = musicVideoConditioningReferences;
+/**
+ * A render's reference files: the frame it reworks leads, then the shot's references, kept
+ * within the backends' reference cap (a leading frame leaves room for the first 3).
+ */
+const frameReferenceFiles = (leadImageId, references) => [
+ ...(leadImageId ? [leadImageId] : []),
+ ...references.map((ref) => ref.imageId).filter((id) => id !== leadImageId),
+].slice(0, MAX_CONDITIONING_REFERENCES);
/** A mood board / Pinterest reference: look text only, never a conditioning image. */
export const isOutsideReference = isOutsideMusicVideoReference;
@@ -179,11 +187,12 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
* ahead of the kickoff is still attributed to it).
*
* `refine` (with `referenceImageId`) edits that frame by a short instruction
- * instead: the frame is the only reference and the prompt is the instruction.
+ * instead: the prompt is the instruction, and the frame leads the shot's references.
*/
const generateFrame = (scene, { revisionId, onJob, referenceImageId = null, refine = null } = {}) => {
const refining = Boolean(refine?.trim() && referenceImageId);
- const conditioning = refining ? [] : conditioningReferences(project, scene);
+ // The shot's references (the card's References row) ride along on every frame render, a refine included.
+ const conditioning = conditioningReferences(project, scene);
const problem = refining ? null : shotActionContractProblem(scene.direction?.actionContract, scene);
if (problem) { toast.error(problem); return Promise.resolve({ ok: false }); }
// The director's text (or a refine's instruction); the server composes the still's
@@ -192,7 +201,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
// A planned shot with no text still renders from its look, frame clause and actions.
if (!refining && !musicVideoFrameModelInput(project, scene).prompt) { toast.error('Add a frame prompt or shot prompt first'); return Promise.resolve({ ok: false }); }
// A frame picked on the card (its start or end frame) leads the references, as the image this render reworks.
- const references = [...(referenceImageId ? [referenceImageId] : []), ...conditioning.map((ref) => ref.imageId).filter((id) => id !== referenceImageId)];
+ const references = frameReferenceFiles(referenceImageId, conditioning);
const projectId = project.id;
frameLane.startScene(scene.sceneId);
return generateImage({
@@ -229,7 +238,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
})
.catch((err) => {
if (conditioning.length && REFERENCE_CAPABILITY_CODES.has(err?.code)) {
- toast.error(`The current image backend can't condition on ${conditioning.length} reference image${conditioning.length === 1 ? '' : 's'}: ${err.message}. Switch the image backend, or untick "Condition frames" on the visual-spec references.`);
+ toast.error(`The current image backend can't condition on ${conditioning.length} reference image${conditioning.length === 1 ? '' : 's'}: ${err.message}. Switch the image backend, or take references off the shot's References row.`);
} else {
toast.error(err?.message || 'Frame generation failed');
}
@@ -251,12 +260,14 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const endPrompt = scene.endFramePrompt?.trim();
if (!endPrompt && !refining) { toast.error('Describe the end frame first'); return Promise.resolve({ ok: false }); }
const projectId = project.id;
+ // The frame it renders from leads, then the shot's references (the card's References row).
+ const references = frameReferenceFiles(referenceImageId, conditioningReferences(project, scene));
endFrameLane.startScene(scene.sceneId);
return generateImage({
// From a frame the server keeps its shot and changes what the prompt asks; with no frame it renders like a start frame.
prompt: refining ? refine.trim() : endPrompt,
...musicVideoFrameGenSize(project),
- ...(referenceImageId ? { referenceImageFiles: [referenceImageId] } : {}),
+ ...(references.length ? { referenceImageFiles: references } : {}),
musicVideo: {
projectId, sceneId: scene.sceneId, slot: 'end', compose: true,
...(refining ? { refine: true } : referenceImageId ? { fromFrame: true } : {}),
diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsx
index e47a99f2a0..b76a872cd1 100644
--- a/client/src/pages/MusicVideo.test.jsx
+++ b/client/src/pages/MusicVideo.test.jsx
@@ -2525,20 +2525,42 @@ describe('MusicVideo visual spec, takes and handoff (#8965)', () => {
}), { silent: true }));
});
- // Uniquely catches Refine sending the shot prompt or the mood references instead of editing just this frame by the instruction.
- it('refines the start frame by a short instruction, with that frame as the only reference', async () => {
+ // Uniquely catches Refine sending the shot prompt instead of the instruction, the frame not leading, or the shot's picked references not riding along.
+ it('refines the start frame by a short instruction, sending the references picked on the shot after that frame', async () => {
await openProject({ ...SPEC_PROJECT, scenes: [{ ...SPEC_PROJECT.scenes[0], referenceImageId: 'start.png' }] }, 'produce');
+ // The automatic set shows as chips; swap the flagged Lead for the Pier set.
+ const refs = within(screen.getByRole('group', { name: 'References sent with the frames' }));
+ expect(refs.getByText('Auto: from the prompt')).toBeTruthy();
+ fireEvent.click(refs.getByRole('button', { name: "Don't send Lead" }));
+ expect(updateMusicVideoScene).toHaveBeenLastCalledWith('mv-spec', 's1', { frameReferenceIds: [] }, expect.anything());
+ fireEvent.click(refs.getByRole('button', { name: 'Add' }));
+ fireEvent.click(refs.getByRole('button', { name: /Pier/ }));
+ expect(updateMusicVideoScene).toHaveBeenLastCalledWith('mv-spec', 's1', { frameReferenceIds: ['r2'] }, expect.anything());
fireEvent.click(screen.getByRole('button', { name: 'Refine' }));
fireEvent.change(screen.getByLabelText('What to change in the start frame'), { target: { value: 'remove the hand' } });
fireEvent.click(screen.getByRole('button', { name: 'Render refined start frame' }));
await waitFor(() => expect(generateImage).toHaveBeenCalledWith(expect.objectContaining({
prompt: 'remove the hand',
- referenceImageFiles: ['start.png'],
+ referenceImageFiles: ['start.png', 'set.png'],
musicVideo: { projectId: 'mv-spec', sceneId: 's1', compose: true, refine: true },
}), { silent: true }));
expect(screen.queryByLabelText('What to change in the start frame')).toBeNull();
});
+ // Uniquely catches a refine sending the frame plus 4 references, over the backends' 4-image cap.
+ it('refines a shot with four references by sending the frame plus the first three', async () => {
+ const references = ['a', 'b', 'c', 'd'].map((key) => ({ id: `r-${key}`, imageId: `${key}.png`, role: 'character', label: key, note: '', condition: true }));
+ await openProject({ ...SPEC_PROJECT, visualSpec: { ...SPEC_PROJECT.visualSpec, references },
+ scenes: [{ ...SPEC_PROJECT.scenes[0], referenceImageId: 'start.png' }] }, 'produce');
+ expect(screen.getByText('Refines and end frames send the frame plus the first 3.')).toBeTruthy();
+ fireEvent.click(screen.getByRole('button', { name: 'Refine' }));
+ fireEvent.change(screen.getByLabelText('What to change in the start frame'), { target: { value: 'remove the hand' } });
+ fireEvent.click(screen.getByRole('button', { name: 'Render refined start frame' }));
+ await waitFor(() => expect(generateImage).toHaveBeenCalledWith(expect.objectContaining({
+ referenceImageFiles: ['start.png', 'a.png', 'b.png', 'c.png'],
+ }), { silent: true }));
+ });
+
// Uniquely catches a trimmed clip landing only in Video Gen history, never as this shot's take, or not becoming the shot's clip.
it('trims the shot clip to a later start frame and makes it the shot\'s selected clip take', async () => {
const scene = { sceneId: 's1', order: 0, prompt: 'waves', referenceImageId: 'img1', videoHistoryId: 'h1',
diff --git a/server/lib/musicVideoConditioning.js b/server/lib/musicVideoConditioning.js
index 4305c8fdfd..580c5e074f 100644
--- a/server/lib/musicVideoConditioning.js
+++ b/server/lib/musicVideoConditioning.js
@@ -37,8 +37,18 @@ function featuredAuthoredCharacters(references, scene) {
&& !isOutsideMusicVideoReference(ref) && namesIn(text, String(ref.label || '').split(/\s[—-]\s|[:(]/)[0]));
}
+/** The references a director can send with a shot's frames: every one with an image except outside inspiration. */
+export const musicVideoReferenceChoices = (project) => (project?.visualSpec?.references || [])
+ .filter((ref) => ref?.id && ref.imageId && !isOutsideMusicVideoReference(ref));
+
export function musicVideoConditioningReferences(project, scene) {
const references = project?.visualSpec?.references || [];
+ // The director's own pick for this shot (the shot card's References row) wins over the automatic set.
+ if (Array.isArray(scene?.frameReferenceIds)) {
+ const choices = musicVideoReferenceChoices(project);
+ return scene.frameReferenceIds.map((id) => choices.find((ref) => ref.id === id)).filter(Boolean)
+ .slice(0, MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES);
+ }
const stage = project?.castAndSets;
if (stage?.status === 'approved' && Number.isInteger(scene?.sectionIndex)) {
const setId = stage.direction?.songMap?.find((entry) => entry.section === scene.sectionIndex)?.setId;
diff --git a/server/lib/musicVideoShotPrompt.js b/server/lib/musicVideoShotPrompt.js
index 6e0cbb0bbc..332479f33e 100644
--- a/server/lib/musicVideoShotPrompt.js
+++ b/server/lib/musicVideoShotPrompt.js
@@ -93,9 +93,11 @@ export function musicVideoShotModelInput(project, scene, {
}
/** A refine edit's wrapper around the director's instruction ("[instruction]" marks where it goes). */
-const refineWrapper = (instruction) => `Edit the reference image: ${instruction}. Keep everything else exactly as it is: composition, framing, subject, lighting, color and film look.`;
+const refineWrapper = (instruction, first = false) => `Edit the ${first ? 'first ' : ''}reference image: ${instruction}. Keep everything else exactly as it is: composition, framing, subject, lighting, color and film look.`;
const REWORK_FRAME = 'Start from the first reference image and rework it to match this prompt.';
const KEEP_REFERENCE = 'Keep the reference image\'s composition, subject and framing; change only what this prompt describes.';
+// Sent when a refine or an end frame from a frame carries the shot's character or set references after that frame.
+export const MUSIC_VIDEO_OTHER_REFERENCES_NOTE = 'The other reference images show the characters and places in this shot: keep their faces, hair and wardrobe as shown there.';
/** The director's text for a shot's start or end frame. */
export const authoredFramePromptText = (scene, slot = 'start') => (slot === 'end'
@@ -113,17 +115,20 @@ export const authoredFramePromptText = (scene, slot = 'start') => (slot === 'end
*/
export function musicVideoFrameModelInput(project, scene, {
slot = 'start', authored = authoredFramePromptText(scene, slot), fromFrame = false, refine = false,
- repairs = [], fixes = '',
+ repairs = [], fixes = '', otherReferences = 0,
} = {}) {
+ const others = otherReferences > 0 ? addition('other-references', 'Character references', MUSIC_VIDEO_OTHER_REFERENCES_NOTE) : null;
const text = typeof authored === 'string' ? authored.trim() : '';
// A refine or an end frame from the start frame says only what changes, so it needs text;
// a planned shot with no seeded text still gets its look, frame clause and actions.
if (!text && (refine || slot === 'end')) return { prompt: '', promptAdditions: [] };
if (refine) {
- return { prompt: refineWrapper(text), promptAdditions: [addition('edit-instructions', 'Edit instructions', refineWrapper('[instruction]'))] };
+ return { prompt: [refineWrapper(text, !!others), others?.text].filter(Boolean).join(' '),
+ promptAdditions: [addition('edit-instructions', 'Edit instructions', refineWrapper('[instruction]', !!others)), others].filter(Boolean) };
}
if (slot === 'end' && fromFrame) {
- return { prompt: `${text}. ${KEEP_REFERENCE}`, promptAdditions: [addition('keep-reference', 'Keep the start frame', KEEP_REFERENCE)] };
+ return { prompt: [`${text}. ${KEEP_REFERENCE}`, others?.text].filter(Boolean).join(' '),
+ promptAdditions: [addition('keep-reference', 'Keep the start frame', KEEP_REFERENCE), others].filter(Boolean) };
}
const look = [
addition('project-style', 'Project style', project?.concept?.style),
diff --git a/server/lib/musicVideoShotPrompt.test.js b/server/lib/musicVideoShotPrompt.test.js
index f8f5683c03..c9c62b8164 100644
--- a/server/lib/musicVideoShotPrompt.test.js
+++ b/server/lib/musicVideoShotPrompt.test.js
@@ -47,6 +47,17 @@ describe('musicVideoFrameModelInput', () => {
expect(input.promptAdditions).toEqual([expect.objectContaining({ id: 'edit-instructions', text: expect.stringContaining('[instruction]') })]);
});
+ // Uniquely catches a refine or end frame that sends character references without saying what they are for.
+ it('names the first image as the one to edit, and says what the shot\'s other references are for', () => {
+ const refine = musicVideoFrameModelInput(project, scene, { authored: 'match his vest', refine: true, otherReferences: 2 });
+ expect(refine.prompt).toMatch(/^Edit the first reference image: match his vest\./);
+ expect(refine.prompt).toContain('The other reference images show the characters and places in this shot');
+ expect(ids(refine)).toEqual(['edit-instructions', 'other-references']);
+ const end = musicVideoFrameModelInput(project, scene, { slot: 'end', fromFrame: true, otherReferences: 1 });
+ expect(ids(end)).toEqual(['keep-reference', 'other-references']);
+ for (const input of [refine, end]) for (const item of input.promptAdditions.filter((a) => a.id === 'other-references')) expect(input.prompt).toContain(item.text);
+ });
+
it('still composes a planned start frame with no seeded text, and sends nothing for an empty end frame or refine', () => {
expect(musicVideoFrameModelInput(project, { sceneId: 's2' }).prompt.startsWith('grainy 16mm, ')).toBe(true);
expect(musicVideoFrameModelInput(project, { sceneId: 's2' }, { slot: 'end' })).toEqual({ prompt: '', promptAdditions: [] });
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index dd1198da4d..64930a62fd 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -602,6 +602,8 @@ export const musicVideoSceneCameraSchema = z.object({
// Per-scene visual layer (#8985) — footage, a moved still, or a title card;
// see musicVideoLayers.js. Only a composed render honors a non-footage layer.
const sceneLayerFields = {
+ // The references (visual spec ids) sent with this shot's frame renders; absent or null = the automatic set.
+ frameReferenceIds: z.array(z.string().min(1).max(128)).max(MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES).nullable().optional(),
camera: musicVideoSceneCameraSchema.nullable().optional(),
visualLayer: z.enum(MUSIC_VIDEO_VISUAL_LAYERS).optional(),
stillMove: z.enum(MUSIC_VIDEO_STILL_MOVES).optional(),
diff --git a/server/routes/musicVideoTakes.test.js b/server/routes/musicVideoTakes.test.js
index 940a97e7db..e47e78bfca 100644
--- a/server/routes/musicVideoTakes.test.js
+++ b/server/routes/musicVideoTakes.test.js
@@ -435,3 +435,26 @@ describe('the shot asset manager', () => {
expect((await getCollection(musicVideoCollectionId(project.id))).items.map((item) => item.ref).sort()).toEqual(['b-clip', 'b-used.png', 'keep.png', 'shared.png', 'used.png']);
});
});
+
+describe('a shot\'s frame references', () => {
+ // Uniquely catches the shot card's References pick failing to save, or a frame render ignoring it.
+ it('saves the director\'s pick on the shot, and frame renders send it instead of the automatic set', async () => {
+ const { musicVideoConditioningReferences } = await import('../lib/musicVideoConditioning.js');
+ await projects.mutateProjectRecord(project.id, (current) => ({ project: { ...current, visualSpec: { ...(current.visualSpec || {}), references: [
+ { id: 'lead', imageId: 'lead.png', role: 'character', label: 'Lead', condition: true },
+ { id: 'tattooist', imageId: 'tattooist.png', role: 'character', label: 'Tattoo artist', condition: false },
+ { id: 'mvr-board-1', imageId: 'board.png', role: 'style', label: 'Board pin', condition: false },
+ ] } } }));
+ const sceneOf = async () => (await reload()).scenes.find((scene) => scene.sceneId === sceneA.sceneId);
+ expect(musicVideoConditioningReferences(await reload(), await sceneOf()).map((ref) => ref.id)).toEqual(['lead']);
+
+ const saved = await request(app).patch(`${base()}/scenes/${sceneA.sceneId}`).send({ frameReferenceIds: ['tattooist', 'mvr-board-1'] });
+ expect(saved.status).toBe(200);
+ // Outside inspiration is never sent as an image, even when named.
+ expect(musicVideoConditioningReferences(await reload(), await sceneOf()).map((ref) => ref.id)).toEqual(['tattooist']);
+ expect((await request(app).patch(`${base()}/scenes/${sceneA.sceneId}`).send({ frameReferenceIds: ['a', 'b', 'c', 'd', 'e'] })).status).toBe(400);
+ // Back to auto.
+ await request(app).patch(`${base()}/scenes/${sceneA.sceneId}`).send({ frameReferenceIds: null });
+ expect(musicVideoConditioningReferences(await reload(), await sceneOf()).map((ref) => ref.id)).toEqual(['lead']);
+ });
+});
diff --git a/server/services/imageGen/prepareParams.js b/server/services/imageGen/prepareParams.js
index 5a9ff1c4ab..f4eff7fbcc 100644
--- a/server/services/imageGen/prepareParams.js
+++ b/server/services/imageGen/prepareParams.js
@@ -239,6 +239,8 @@ export async function prepareGenerateParams({ data, files, referenceImageFields,
const composed = musicVideoFrameModelInput(musicVideoProject, scene, {
slot: data.musicVideo.slot === 'end' ? 'end' : 'start', authored: data.prompt || '',
fromFrame: data.musicVideo.fromFrame === true, refine: data.musicVideo.refine === true,
+ // The picked frame comes first; any reference after it is the shot's character or set reference.
+ otherReferences: Math.max(0, namedReferenceFiles.length + referenceUploads.length - 1),
});
data.prompt = composed.prompt;
if (composed.promptAdditions.length) data.promptAdditions = [...(data.promptAdditions || []), ...composed.promptAdditions];
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index ecc0e536aa..ec3237aea0 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -633,7 +633,7 @@ export function replaceScenes(project, sceneInputs) {
// adding a shot or lyric the storyboard approval did not cover.
const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',
'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'speedFps', 'offsetFrames', 'reverse', 'transitionIn', 'transitionOut', 'glitches', 'cutEffect',
- 'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec'];
+ 'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec', 'frameReferenceIds'];
/**
* Apply a patch to a single scene. Returns `{ project, updated }`. Throws if the