Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Bring a swapped still's frame prompt onto the shot (#11288)

[._.]/36cf15cadf02
diff --git a/client/src/hooks/useMusicVideoTakes.js b/client/src/hooks/useMusicVideoTakes.js
index 86023381d5..2491bb9802 100644
--- a/client/src/hooks/useMusicVideoTakes.js
+++ b/client/src/hooks/useMusicVideoTakes.js
@@ -72,8 +72,14 @@ export default function useMusicVideoTakes({ project, applyScenePatch }) {
);
// An unused still or clip from the project's pool becomes this shot's selection (with its partner, when picked).
+ // The still's frame prompt comes with it, so the start or end prompt shows what made the picture.
const placePoolItem = (scene, pick) => runTakeOp(
- (projectId) => placeMusicVideoPoolItem(projectId, scene.sceneId, pick, { silent: true }).then((res) => res.scene),
+ (projectId) => placeMusicVideoPoolItem(projectId, scene.sceneId, pick, { silent: true }).then(({ scene: placed }) => {
+ const field = pick.kind === 'endImage' ? 'endFramePrompt' : 'framePrompt';
+ const stillPlaced = pick.kind !== 'video' || Boolean(pick.withAssetId);
+ if (stillPlaced && placed[field] !== scene[field]) applyScenePatch?.(projectId, placed.sceneId, { [field]: placed[field] ?? null });
+ return placed;
+ }),
'Failed to use that take',
);
diff --git a/client/src/hooks/useMusicVideoTakes.test.js b/client/src/hooks/useMusicVideoTakes.test.js
new file mode 100644
index 0000000000..218b806a2b
--- /dev/null
+++ b/client/src/hooks/useMusicVideoTakes.test.js
@@ -0,0 +1,30 @@
+import { describe, expect, it, vi } from 'vitest';
+import { act, renderHook } from '@testing-library/react';
+
+vi.mock('../services/apiMusicVideo.js', () => ({ placeMusicVideoPoolItem: vi.fn() }));
+vi.mock('../components/ui/Toast', () => ({ default: { error: vi.fn(), success: vi.fn() } }));
+
+import { placeMusicVideoPoolItem } from '../services/apiMusicVideo.js';
+import useMusicVideoTakes from './useMusicVideoTakes.js';
+
+describe('useMusicVideoTakes placePoolItem', () => {
+ // Uniquely catches a swapped still landing on the card while its start or end prompt keeps describing the old picture.
+ it('shows the placed still\'s frame prompt, and leaves the prompts alone for a clip on its own', async () => {
+ const applyScenePatch = vi.fn();
+ const scene = { sceneId: 's1', framePrompt: 'old frame', endFramePrompt: 'old end', takes: [] };
+ const { result } = renderHook(() => useMusicVideoTakes({ project: { id: 'mv-1' }, applyScenePatch }));
+
+ placeMusicVideoPoolItem.mockResolvedValueOnce({ scene: { ...scene, referenceImageId: 'diner.png', framePrompt: 'A neon diner' } });
+ await act(() => result.current.placePoolItem(scene, { kind: 'image', assetId: 'diner.png' }));
+ expect(applyScenePatch).toHaveBeenCalledWith('mv-1', 's1', { framePrompt: 'A neon diner' });
+
+ placeMusicVideoPoolItem.mockResolvedValueOnce({ scene: { ...scene, endImageId: 'smile.png', endFramePrompt: 'She smiles' } });
+ await act(() => result.current.placePoolItem(scene, { kind: 'endImage', assetId: 'smile.png' }));
+ expect(applyScenePatch).toHaveBeenCalledWith('mv-1', 's1', { endFramePrompt: 'She smiles' });
+
+ applyScenePatch.mockClear();
+ placeMusicVideoPoolItem.mockResolvedValueOnce({ scene: { ...scene, framePrompt: 'saved frame', videoHistoryId: 'clip-1' } });
+ await act(() => result.current.placePoolItem(scene, { kind: 'video', assetId: 'clip-1' }));
+ expect(applyScenePatch.mock.calls.every(([, , patch]) => !('framePrompt' in patch))).toBe(true);
+ });
+});
diff --git a/server/routes/musicVideoTakes.test.js b/server/routes/musicVideoTakes.test.js
index 3f9dcfacb5..11b7273e34 100644
--- a/server/routes/musicVideoTakes.test.js
+++ b/server/routes/musicVideoTakes.test.js
@@ -282,6 +282,37 @@ describe('swapping a shot from unused takes', () => {
});
});
+describe('a swapped still brings its frame prompt', () => {
+ // Uniquely catches a still placed from another shot or the floor leaving the start or end prompt describing the old picture, or carrying the look twice.
+ it('sets the start or end frame prompt to the one the placed still was made from, without the project look', async () => {
+ const prompts = {
+ 'diner.png': 'A neon diner at night, grainy 16mm\nShe enters from the left.',
+ 'smile.png': 'She smiles. Keep the reference image\'s composition, subject and framing; change only what this prompt describes.',
+ 'edit.png': 'Edit the reference image: remove the hand. Keep everything else exactly as it is.',
+ 'booth.png': 'An empty red booth, grainy 16mm',
+ };
+ for (const [name, prompt] of Object.entries(prompts)) {
+ seedImage(name);
+ writeFileSync(join(ROOT(), 'images', name.replace('.png', '.metadata.json')), JSON.stringify({ prompt }));
+ }
+ await projects.mutateProjectRecord(project.id, (current) => ({ project: { ...current, concept: { ...current.concept, style: 'grainy 16mm' } } }));
+ await addTake(sceneB.sceneId, { kind: 'image', assetId: 'booth.png', source: 'generated' });
+ for (const name of ['diner.png', 'edit.png']) await addTake(sceneB.sceneId, { kind: 'image', assetId: name, source: 'generated' });
+ await addTake(sceneB.sceneId, { kind: 'endImage', assetId: 'smile.png', source: 'generated' });
+ // Cut shot B: its stills land on the floor, where only their image records still name a prompt.
+ await request(app).delete(`${base()}/scenes/${sceneB.sceneId}`);
+ const use = (body) => request(app).post(`${base()}/scenes/${sceneA.sceneId}/takes/use`).send(body);
+
+ expect((await use({ kind: 'image', assetId: 'diner.png' })).body.scene.framePrompt).toBe('A neon diner at night');
+ expect((await use({ kind: 'endImage', assetId: 'smile.png' })).body.scene).toMatchObject({ framePrompt: 'A neon diner at night', endFramePrompt: 'She smiles' });
+ // A refined still's prompt is an edit instruction, not a frame: the shot keeps its prompt.
+ expect((await use({ kind: 'image', assetId: 'edit.png' })).body.scene.framePrompt).toBe('A neon diner at night');
+ const booth = await use({ kind: 'image', assetId: 'booth.png' });
+ expect(booth.body.scene.framePrompt).toBe('An empty red booth');
+ expect(booth.body.scene.takes.find((take) => take.assetId === 'booth.png').prompt).toBe(prompts['booth.png']);
+ });
+});
+
describe('a director\'s still change keeps the shot locked', () => {
// Uniquely catches a swapped still or edited prompt leaving the shot "changed since you locked it", so its clip can't render.
it('re-binds the still lock on Use, take select and a prompt edit, and drops a clip lock made from the old still', async () => {
diff --git a/server/services/musicVideo/handoff.js b/server/services/musicVideo/handoff.js
index 6fa633d221..f7ffe7cb06 100644
--- a/server/services/musicVideo/handoff.js
+++ b/server/services/musicVideo/handoff.js
@@ -100,6 +100,27 @@ function composePrompt(primary, fallback, project, clause, { motion = false } =
].filter(Boolean).join(', ');
}
+/**
+ * The authored frame prompt inside a still's rendered prompt: its first line,
+ * cut where the project-wide look (concept style, mood-board context, visual
+ * spec) was appended and before an end frame's keep-the-reference clause, so a
+ * still swapped onto another shot shows the prompt it was made from without
+ * the look being appended twice on the next render. Null for a refine edit
+ * (its text is an instruction, not a frame) or an empty prompt.
+ */
+export function authoredFramePrompt(project, prompt) {
+ const line = typeof prompt === 'string' ? prompt.split('\n')[0].trim() : '';
+ if (!line || line.startsWith('Edit the reference image:')) return null;
+ const look = [
+ project?.concept?.style?.trim(),
+ musicVideoCreativeContext(project?.concept, { look: musicVideoDirectorLook(project) }),
+ visualDirection(project?.visualSpec),
+ ].filter(Boolean);
+ const cuts = [...look.map((piece) => line.indexOf(`, ${piece}`)), line.indexOf('. Keep the reference image\'s composition')].filter((at) => at > 0);
+ const authored = (cuts.length ? line.slice(0, Math.min(...cuts)) : line).trim();
+ return authored || null;
+}
+
/** A scene's composed reference-frame prompt (also the server-owned production run's, #9066). */
export const sceneFramePrompt = (project, scene) => [composePrompt(scene.framePrompt, scene.prompt, project, scene.direction?.frameClause), shotActionPrompt(project, scene, { frame: true })].filter(Boolean).join('\n');
diff --git a/server/services/musicVideo/projects.js b/server/services/musicVideo/projects.js
index 017a7cf5ff..b7f3123692 100644
--- a/server/services/musicVideo/projects.js
+++ b/server/services/musicVideo/projects.js
@@ -374,14 +374,21 @@ export async function selectSceneTake(id, sceneId, takeId) {
* card's "Swap from unused"): it becomes a take of the shot and its selection.
* `withAssetId` brings its partner too: a still's clip, or a clip's still.
* `kind: 'endImage'` puts a still on the shot's end frame instead.
- * The take keeps the prompt and provider of the take it came from. A shot
+ * The take keeps the prompt and provider of the take it came from, and the
+ * shot's start or end frame prompt becomes the one that still was made from. A shot
* whose video is locked refuses, as it does any other new selection.
* Returns the updated scene.
*/
export async function placeMediaPoolItem(id, sceneId, { kind, assetId, withAssetId = null }) {
- const [{ appendSceneTakes: append, selectSceneTake: select }, { assertClipUnlocked, keepDirectorStillLock }, { projectTakeAssets }] = await Promise.all([
- import('./takes.js'), import('./productionReview.js'), import('./mediaPool.js'),
+ const [{ appendSceneTakes: append, selectSceneTake: select }, { assertClipUnlocked, keepDirectorStillLock }, { projectTakeAssets }, { authoredFramePrompt }] = await Promise.all([
+ import('./takes.js'), import('./productionReview.js'), import('./mediaPool.js'), import('./handoff.js'),
]);
+ // A still off the floor has no take left to name its prompt; its image sidecar still does.
+ const placedStill = kind === 'video' ? withAssetId : assetId;
+ const sidecarPrompt = placedStill
+ ? await import('../imageGen/local.js').then(({ readImageSidecar }) => readImageSidecar(placedStill))
+ .then(({ metadata }) => (typeof metadata?.prompt === 'string' ? metadata.prompt : null)).catch(() => null)
+ : null;
const { scene } = await mutateProjectRecord(id, (current) => {
assertClipUnlocked(current, sceneId);
const origin = new Map(projectTakeAssets(current).filter(asset => asset.take).map(asset => [`${asset.kind}:${asset.ref}`, asset.take]));
@@ -389,7 +396,8 @@ export async function placeMediaPoolItem(id, sceneId, { kind, assetId, withAsset
const clip = kind === 'video' ? assetId : withAssetId;
const input = (takeKind, ref) => {
const from = origin.get(`${takeKind === 'video' ? 'video' : 'image'}:${ref}`);
- return { kind: takeKind, assetId: ref, source: 'reused', provider: from?.provider || null, prompt: from?.prompt || null,
+ return { kind: takeKind, assetId: ref, source: 'reused', provider: from?.provider || null,
+ prompt: from?.prompt || (takeKind !== 'video' && ref === placedStill ? sidecarPrompt : null),
...(takeKind === 'video' && still ? { sourceImageId: still } : {}) };
};
const inputs = kind === 'endImage'
@@ -397,6 +405,14 @@ export async function placeMediaPoolItem(id, sceneId, { kind, assetId, withAsset
: [...(still ? [input('image', still)] : []), ...(clip ? [input('video', clip)] : [])];
let outcome = append(current, sceneId, inputs);
for (const take of outcome.appended) outcome = { ...select(outcome.project, sceneId, take.takeId), appended: outcome.appended };
+ // The placed still's frame prompt comes with it, so the start or end prompt matches the picture.
+ const stillTake = outcome.appended.find((take) => take.kind !== 'video');
+ const framePrompt = authoredFramePrompt(current, stillTake?.prompt);
+ if (framePrompt) {
+ const field = stillTake.kind === 'endImage' ? 'endFramePrompt' : 'framePrompt';
+ const scenes = outcome.project.scenes.map((scene) => (scene.sceneId === sceneId ? { ...scene, [field]: framePrompt } : scene));
+ outcome = { ...outcome, project: { ...outcome.project, scenes }, scene: scenes.find((scene) => scene.sceneId === sceneId) };
+ }
return { ...outcome, project: keepDirectorStillLock(current, outcome.project, sceneId) };
});
return scene;