Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video: bigger Swap picker, one-tap frame swap, own frames in Swap, From previous shot fixes (#11429)

[._.]/ee3f336183e7
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 2d1a833094..a7a5e94802 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -115,7 +115,7 @@ export default function SceneCard({
settingsSaving, videoBlockedReason, videoBlockedReasonFor = null, canContinueShot,
onMove, onDelete, onEditLocal, onSave,
project = null, onGenerateFrame, onGenerateVideo, onContinueVideo, onGenerateEndFrame, generatingEndFrame = false, onContinueFromPrevious, previousHasClip = false,
- onOpenPreview, onSelectTake, onReviewTake, onRejectUnusedTakes = null, onImportTake, onImportClipTake, onPlacePoolItem, onTrimClip, projectId = null, aspect = '16:9', videoModels = [], defaultVideoModel = '', shotNumbers = undefined, takeBusy = false, layered = false, documentComposition = false,
+ onOpenPreview, onSelectTake, onReviewTake, onRejectUnusedTakes = null, onSwapFrames = null, onImportTake, onImportClipTake, onPlacePoolItem, onTrimClip, projectId = null, aspect = '16:9', videoModels = [], defaultVideoModel = '', shotNumbers = undefined, takeBusy = false, layered = false, documentComposition = false,
lipSyncBackend = '', songDurationSec = null, onSplit, onMergeNext, onMoveStart = null, nextEndSec = null, falVideoSettings = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,
expanded, onToggleExpand, footageOptional = false, failedScenes = null, songReview = null,
}) {
@@ -490,7 +490,7 @@ export default function SceneCard({
<SceneFrames scene={scene} index={index} endFrames={endFrames} generatingFrame={!!generatingFrame} generatingEndFrame={!!generatingEndFrame}
onGenerateFrame={onGenerateFrame} onGenerateEndFrame={onGenerateEndFrame} onEditLocal={onEditLocal} onSave={onSave}
onOpenPreview={onOpenPreview} takeBusy={takeBusy} onSelectTake={onSelectTake} onReviewTake={onReviewTake}
- onSwap={onPlacePoolItem ? setSwapKind : null} onImportTake={onImportTake}
+ onSwap={onPlacePoolItem ? setSwapKind : null} onSwapFrames={onSwapFrames ? () => onSwapFrames(scene) : null} onImportTake={onImportTake}
onContinueFromPrevious={layer === 'footage' && index > 0 ? onContinueFromPrevious : null} previousHasClip={previousHasClip}
clipLocked={shotLock?.clip === 'locked'} />
{/* Clip: the motion prompt, where this shot renders and what it costs, then its clip (Phase 1). */}
diff --git a/client/src/components/musicVideo/SceneFrames.jsx b/client/src/components/musicVideo/SceneFrames.jsx
index 43aaa6f422..d60d1505c6 100644
--- a/client/src/components/musicVideo/SceneFrames.jsx
+++ b/client/src/components/musicVideo/SceneFrames.jsx
@@ -1,5 +1,5 @@
import { useLayoutEffect, useRef, useState } from 'react';
-import { Activity, ArrowRight, Image as ImageIcon, ImagePlus, PencilLine, Shuffle, SkipForward, Wand2, X } from 'lucide-react';
+import { Activity, ArrowLeftRight, ArrowRight, Image as ImageIcon, ImagePlus, PencilLine, Shuffle, SkipForward, Wand2, X } from 'lucide-react';
import SceneTakeStrip from './SceneTakeStrip.jsx';
const PRIMARY_BUTTON = 'flex items-center gap-1 bg-port-border hover:bg-port-border/70 disabled:opacity-50 rounded px-2 py-1.5 text-xs min-h-[44px] sm:min-h-0 whitespace-nowrap';
@@ -58,11 +58,12 @@ const REFERENCE_LABELS = { '': 'Prompt only', start: 'Start frame', end: 'End fr
* short instruction (`{ referenceImageId, refine }`), and Swap puts one of the
* video's unused stills on it. Each frame's own takes sit under it as a grid,
* so both frames' actions and takes show at once; tapping a frame opens it full size.
+ * With both frames set, `onSwapFrames` trades them (prompts included) in one tap.
*/
export default function SceneFrames({
scene, index, endFrames, generatingFrame = false, generatingEndFrame = false, onGenerateFrame, onGenerateEndFrame,
onEditLocal, onSave, onOpenPreview, takeBusy = false, onSelectTake, onReviewTake, onSwap = null, onImportTake = null,
- onContinueFromPrevious = null, previousHasClip = false, clipLocked = false,
+ onContinueFromPrevious = null, previousHasClip = false, clipLocked = false, onSwapFrames = null,
}) {
const [keepStillAsEnd, setKeepStillAsEnd] = useState(false);
// Which frame each render starts from: the start frame renders from its prompt, the end frame from the start frame.
@@ -153,7 +154,7 @@ export default function SceneFrames({
{start && onContinueFromPrevious && (
<button type="button" onClick={() => onContinueFromPrevious(scene, { keepStillAsEnd: keepStillAsEnd && Boolean(scene.referenceImageId) })}
disabled={takeBusy || !previousHasClip} className={SECONDARY_BUTTON}
- title={previousHasClip ? 'Start this shot from the last frame the previous shot shows in the cut' : 'The previous shot needs a clip first'}>
+ title={previousHasClip ? 'Start this shot from the last frame the previous shot shows in the cut' : 'The previous shot needs a clip or still first'}>
<SkipForward size={14} /> From previous shot
</button>
)}
@@ -190,6 +191,12 @@ export default function SceneFrames({
{column('end')}
</div>
) : column('start')}
+ {endFrames && onSwapFrames && frames.start && frames.end && (
+ <button type="button" onClick={onSwapFrames} disabled={takeBusy || clipLocked} className={SECONDARY_BUTTON}
+ title={clipLocked ? lockedTitle : 'The start frame becomes the end frame and the end frame the start, each with its prompt'}>
+ <ArrowLeftRight size={14} /> Swap start and end
+ </button>
+ )}
</section>
);
}
diff --git a/client/src/components/musicVideo/SceneFrames.test.jsx b/client/src/components/musicVideo/SceneFrames.test.jsx
new file mode 100644
index 0000000000..20617e1abb
--- /dev/null
+++ b/client/src/components/musicVideo/SceneFrames.test.jsx
@@ -0,0 +1,24 @@
+import { describe, expect, it, vi } from 'vitest';
+import { fireEvent, render, screen } from '@testing-library/react';
+import SceneFrames from './SceneFrames.jsx';
+
+const scene = { sceneId: 's1', referenceImageId: 'start.png', endImageId: 'end.png', framePrompt: 'Door opens', endFramePrompt: 'Door shut', takes: [] };
+const frames = (props) => render(<SceneFrames scene={scene} index={0} endFrames onEditLocal={vi.fn()} onSave={vi.fn()} {...props} />);
+
+describe('Swap start and end', () => {
+ // Uniquely catches the one-tap swap missing from a shot with both frames, offered with one frame, or open while its video is locked.
+ it('swaps a shot with both frames in one tap, and is held while the video is locked', () => {
+ const onSwapFrames = vi.fn();
+ const { unmount } = frames({ onSwapFrames });
+ fireEvent.click(screen.getByRole('button', { name: 'Swap start and end' }));
+ expect(onSwapFrames).toHaveBeenCalledTimes(1);
+ unmount();
+ frames({ onSwapFrames, clipLocked: true });
+ expect(screen.getByRole('button', { name: 'Swap start and end' }).disabled).toBe(true);
+ });
+
+ it('is not offered until both frames exist', () => {
+ render(<SceneFrames scene={{ ...scene, endImageId: null }} index={0} endFrames onEditLocal={vi.fn()} onSave={vi.fn()} onSwapFrames={vi.fn()} />);
+ expect(screen.queryByRole('button', { name: 'Swap start and end' })).toBeNull();
+ });
+});
diff --git a/client/src/components/musicVideo/SceneTakePicker.jsx b/client/src/components/musicVideo/SceneTakePicker.jsx
index dd25a1aff3..24b4aeaf98 100644
--- a/client/src/components/musicVideo/SceneTakePicker.jsx
+++ b/client/src/components/musicVideo/SceneTakePicker.jsx
@@ -1,7 +1,8 @@
-import { useEffect, useState } from 'react';
+import { useCallback, useEffect, useState } from 'react';
import { X, Search, Film, Activity, Star, Ban, Trash2 } from 'lucide-react';
import Modal from '../ui/Modal.jsx';
import TabPills from '../ui/TabPills.jsx';
+import InfiniteScrollFooter from '../ui/InfiniteScrollFooter.jsx';
import { deleteMusicVideoPoolAssets, getMusicVideoMediaPool, markMusicVideoPoolAssets } from '../../services/apiMusicVideo.js';
import { takeThumbUrl, takeClipUrl } from '../../lib/musicVideoTakes.js';
import { formatCount } from '../../utils/formatters.js';
@@ -19,6 +20,21 @@ const MARK_FILTERS = [
{ id: 'all', label: 'All' }, { id: 'ingredient', label: '★ Ingredients' }, { id: 'unsorted', label: 'Unsorted' }, { id: 'bad', label: '✕ Bad' },
];
const markOf = (item) => item.mark || 'unsorted';
+// The shot's own frame in the other slot: its end frame when swapping the start, its start when swapping the end.
+const OTHER_FRAME = { image: { field: 'endImageId', takeKind: 'endImage', prompt: 'endFramePrompt', label: 'End frame' },
+ endImage: { field: 'referenceImageId', takeKind: 'image', prompt: 'framePrompt', label: 'Start frame' } };
+
+/**
+ * The shot's frame in the other slot as a pool item, or null. The pool lists only
+ * unused stills, and that frame is in use, so it is added here, listed first on This shot.
+ */
+function otherFrameItem(scene, kind) {
+ const other = OTHER_FRAME[kind];
+ const assetId = other && scene?.[other.field];
+ if (!assetId) return null;
+ const take = scene.takes?.find((t) => t.kind === other.takeKind && t.assetId === assetId);
+ return { kind: 'image', assetId, sceneId: scene.sceneId, prompt: take?.prompt || scene[other.prompt] || null, clipIds: [], ownFrame: other.label };
+}
const poolKey = (item) => `${item.kind}:${item.assetId}`;
const CHIP = 'min-h-[44px] rounded border px-2 text-xs sm:min-h-[28px]';
const ACTION = 'inline-flex min-h-[44px] items-center gap-1 rounded border border-port-border px-3 text-xs disabled:opacity-50 sm:min-h-[32px]';
@@ -55,6 +71,8 @@ export default function SceneTakePicker({ open, onClose, projectId, scene, kind,
const [confirmDelete, setConfirmDelete] = useState(null);
const [working, setWorking] = useState(false);
+ // The shot's frame in the other slot always sits on This shot, so the picker opens there when it has one.
+ const hasOwnFrame = !!otherFrameItem(scene, kind);
useEffect(() => {
if (!open) return undefined;
let active = true;
@@ -66,13 +84,15 @@ export default function SceneTakePicker({ open, onClose, projectId, scene, kind,
setPool(items);
// Open on the first tab that has anything in it.
const groups = new Set(items.filter((item) => item.kind === POOL_KIND[kind]).map((item) => poolGroup(item, scene.sceneId)));
- setGroup(['shot', 'others', 'floor'].find((g) => groups.has(g)) || 'shot');
+ setGroup(hasOwnFrame ? 'shot' : ['shot', 'others', 'floor'].find((g) => groups.has(g)) || 'shot');
})
.catch((err) => { if (active) setError(err?.message || 'Could not load the unused takes'); });
return () => { active = false; };
- }, [open, projectId, scene.sceneId, kind]);
+ }, [open, projectId, scene.sceneId, kind, hasOwnFrame]);
- const ofKind = pool ? filterPool(pool, { kind: POOL_KIND[kind], query, shotNumbers }) : [];
+ const own = pool ? otherFrameItem(scene, kind) : null;
+ const ofKind = pool ? filterPool([...(own ? [own] : []), ...pool.filter((item) => !own || poolKey(item) !== poolKey(own))],
+ { kind: POOL_KIND[kind], query, shotNumbers }) : [];
const counts = { shot: 0, others: 0, floor: 0 };
for (const item of ofKind) counts[poolGroup(item, scene.sceneId)] += 1;
const activeGroup = group;
@@ -105,10 +125,13 @@ export default function SceneTakePicker({ open, onClose, projectId, scene, kind,
setConfirmDelete(null);
});
const badHere = markFilter === 'bad' ? items : [];
+ // The next page loads when the bottom of the grid scrolls into view.
+ const showMore = useCallback(() => setShown((n) => n + PAGE), []);
return (
- <Modal open={open} onClose={onClose} size="2xl" align="top" usePortal ariaLabel={`Swap ${noun} for ${shotName}`}
- panelClassName="bg-port-card border border-port-border rounded-lg p-3 space-y-2">
+ // Full height at every width, and nearly the full width on a desktop, so more of the contact sheet is in view.
+ <Modal open={open} onClose={onClose} size="none" usePortal ariaLabel={`Swap ${noun} for ${shotName}`}
+ panelClassName="w-full max-w-4xl lg:max-w-[min(100%,1600px)] min-h-dvh-cap bg-port-card border border-port-border rounded-lg p-3 space-y-2">
<div className="flex items-center gap-2">
<h3 className="min-w-0 flex-1 truncate text-sm font-medium">Swap {noun} · {shotName}</h3>
<button type="button" onClick={onClose} aria-label="Close" className="inline-flex min-h-[44px] min-w-[44px] items-center justify-center rounded sm:min-h-0 sm:min-w-0 sm:p-1">
@@ -148,6 +171,7 @@ export default function SceneTakePicker({ open, onClose, projectId, scene, kind,
Use {noun} + its {kind === 'image' ? 'clip' : 'still'}
</button>
)}
+ {!picked.ownFrame && (<>
<button type="button" disabled={working} onClick={() => toggleMark('ingredient')} aria-pressed={picked.mark === 'ingredient'}
title="Good, not placed yet: keep it as an ingredient" className={`${ACTION} ${picked.mark === 'ingredient' ? 'border-port-warning text-port-warning' : ''}`}>
<Star size={14} aria-hidden="true" />Ingredient
@@ -159,6 +183,7 @@ export default function SceneTakePicker({ open, onClose, projectId, scene, kind,
<button type="button" disabled={working} onClick={() => setConfirmDelete('picked')} className={ACTION}>
<Trash2 size={14} aria-hidden="true" />Delete
</button>
+ </>)}
</div>
{confirmDelete === 'picked' && (
<DeleteConfirm count={1} noun={itemNoun} working={working} onConfirm={() => remove([picked])} onCancel={() => setConfirmDelete(null)} />
@@ -178,17 +203,17 @@ export default function SceneTakePicker({ open, onClose, projectId, scene, kind,
{pool && items.length === 0 && (
<p className="text-xs text-port-text-muted">{query ? `No unused ${itemNoun}s match.` : `No unused ${itemNoun}s here.`}</p>
)}
- <ul className="grid grid-cols-3 gap-2 sm:grid-cols-4 lg:grid-cols-6" aria-label={`Unused ${itemNoun}s`}>
+ <ul className="grid grid-cols-3 gap-2 sm:grid-cols-4 lg:grid-cols-6 xl:grid-cols-8" aria-label={`Unused ${itemNoun}s`}>
{items.slice(0, shown).map((item) => {
const number = item.sceneId ? shotNumbers.get(item.sceneId) : null;
const selected = picked?.assetId === item.assetId;
return (
<li key={item.assetId}>
<button type="button" onClick={() => setPicked(item)} aria-pressed={selected}
- aria-label={`${itemNoun}${number ? ` from shot ${number}` : ''}${item.prompt ? `: ${item.prompt.slice(0, 80)}` : ''}`}
+ aria-label={`${item.ownFrame ? `This shot's ${item.ownFrame.toLowerCase()}` : `${itemNoun}${number ? ` from shot ${number}` : ''}`}${item.prompt ? `: ${item.prompt.slice(0, 80)}` : ''}`}
className={`relative block w-full overflow-hidden rounded border ${selected ? 'border-port-accent ring-2 ring-port-accent' : 'border-port-border'} ${item.mark === 'bad' ? 'opacity-50' : ''}`}>
<img src={takeThumbUrl(item)} alt="" loading="lazy" className="block aspect-video w-full bg-black object-cover" />
- {number && <span className="port-media-overlay-strong absolute left-1 top-1 rounded px-1 text-[10px] leading-tight">#{number}</span>}
+ {(item.ownFrame || number) && <span className="port-media-overlay-strong absolute left-1 top-1 rounded px-1 text-[10px] leading-tight">{item.ownFrame || `#${number}`}</span>}
{item.mark && (
<span className={`port-media-overlay-strong absolute right-1 top-1 rounded px-0.5 ${item.mark === 'bad' ? 'text-port-error' : 'text-port-warning'}`}>
{item.mark === 'bad' ? <Ban size={11} aria-label="bad" /> : <Star size={11} aria-label="ingredient" />}
@@ -205,10 +230,8 @@ export default function SceneTakePicker({ open, onClose, projectId, scene, kind,
})}
</ul>
{items.length > shown && (
- <button type="button" onClick={() => setShown((n) => n + PAGE)}
- className="w-full min-h-[44px] rounded border border-port-border text-xs sm:min-h-[32px]">
- Show more ({formatCount(items.length - shown)} left)
- </button>
+ <InfiniteScrollFooter hasMore loading={false} error={null} onLoadMore={showMore}
+ label={`Show more (${formatCount(items.length - shown)} left)`} />
)}
</Modal>
);
diff --git a/client/src/components/musicVideo/SceneTakePicker.test.jsx b/client/src/components/musicVideo/SceneTakePicker.test.jsx
index 77167c9438..f26a7cf1fa 100644
--- a/client/src/components/musicVideo/SceneTakePicker.test.jsx
+++ b/client/src/components/musicVideo/SceneTakePicker.test.jsx
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from 'vitest';
-import { fireEvent, render, screen, waitFor } from '@testing-library/react';
+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
const pool = [
{ kind: 'image', assetId: 'mine.png', sceneId: 's8', prompt: 'Film strip held to the lamp', clipIds: [] },
@@ -58,4 +58,38 @@ describe('Swap from unused', () => {
await waitFor(() => expect(api.deleteMusicVideoPoolAssets).toHaveBeenCalledWith('mv-1', [{ kind: 'image', assetId: 'blurry.png' }], { silent: true }));
expect(await screen.findByRole('button', { name: '✕ Bad 0' })).toBeTruthy();
});
+
+ // Uniquely catches the grid stopping at its first page instead of loading more as its bottom scrolls into view.
+ it('loads the next page of stills when the bottom of the grid comes into view', async () => {
+ const many = Array.from({ length: 70 }, (_, i) => ({ kind: 'image', assetId: `f${i}.png`, sceneId: null, prompt: `Frame ${i}`, clipIds: [] }));
+ api.getMusicVideoMediaPool.mockImplementationOnce(async () => ({ items: many }));
+ const observers = [];
+ vi.stubGlobal('IntersectionObserver', function IntersectionObserver(callback) {
+ const self = { observe: () => observers.push(self), disconnect: () => {}, fire: () => callback([{ isIntersecting: true }]) };
+ return self;
+ });
+ render(<SceneTakePicker open projectId="mv-1" scene={{ sceneId: 's8' }} kind="image" onPlace={vi.fn()} shotNumbers={new Map()} />);
+ await waitFor(() => expect(screen.getAllByRole('listitem')).toHaveLength(60));
+ act(() => observers.at(-1).fire());
+ await waitFor(() => expect(screen.getAllByRole('listitem')).toHaveLength(70));
+ expect(screen.queryByRole('button', { name: /Show more/ })).toBeNull();
+ vi.unstubAllGlobals();
+ });
+});
+
+describe("the shot's own other frame", () => {
+ // Uniquely catches the picker hiding the shot's end frame (in use, so not in the unused pool) when swapping its start.
+ it('lists the end frame first on This shot when swapping the start, and places it', async () => {
+ const onPlace = vi.fn(async () => {});
+ const scene = { sceneId: 's8', referenceImageId: 'start.png', endImageId: 'end.png', endFramePrompt: 'Door shut',
+ takes: [{ takeId: 't1', kind: 'endImage', assetId: 'end.png', prompt: 'Door shut', status: 'candidate' }] };
+ render(<SceneTakePicker open projectId="mv-1" scene={scene} kind="image" onClose={vi.fn()} onPlace={onPlace} shotNumbers={new Map([['s8', 8]])} />);
+ await waitFor(() => expect(screen.getAllByRole('listitem')).toHaveLength(2));
+ const first = screen.getAllByRole('listitem')[0];
+ expect(first.textContent).toContain('End frame');
+ fireEvent.click(screen.getByRole('button', { name: "This shot's end frame: Door shut" }));
+ expect(screen.queryByRole('button', { name: 'Delete' })).toBeNull();
+ fireEvent.click(screen.getByRole('button', { name: 'Use still' }));
+ expect(onPlace).toHaveBeenCalledWith({ kind: 'image', assetId: 'end.png' });
+ });
});
diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsx
index 48310adc0d..c236ff17ed 100644
--- a/client/src/components/musicVideo/stages/BoardStage.jsx
+++ b/client/src/components/musicVideo/stages/BoardStage.jsx
@@ -6,7 +6,7 @@ import SceneCard from '../SceneCard.jsx';
import ScenePoster from '../ScenePoster.jsx';
import { ShotLockContext, SHOT_LOCK_LABELS, overallShotLock } from '../ShotLock.jsx';
import { shotNumbers } from '../ShotStepper.jsx';
-import { previousShot } from '../../../../../server/lib/musicVideoHandoffFrame.js';
+import { previousShot, shotHandoffSource } from '../../../../../server/lib/musicVideoHandoffFrame.js';
import { isLayeredComposition } from '../../../lib/musicVideoLayers.js';
import { musicVideoAspect } from '../../../lib/musicVideoAspect.js';
import { FOOTAGE_OPTIONAL_MODES } from '../../../lib/musicVideoStages.js';
@@ -230,9 +230,10 @@ export default function BoardStage({ board }) {
onSelectTake={takes.selectTake}
onReviewTake={takes.reviewTake}
onRejectUnusedTakes={takes.rejectUnusedTakes}
+ onSwapFrames={takes.swapFrames}
onContinueFromPrevious={takes.continueFromPrevious}
// The server's own neighbor rule: the shot before this one by start time.
- previousHasClip={Boolean(previousShot(project, scene.sceneId)?.videoHistoryId)}
+ previousHasClip={Boolean(shotHandoffSource(project, previousShot(project, scene.sceneId)))}
onImportTake={(target) => board.setPickerTarget({ type: 'take', sceneId: target.sceneId })}
onImportClipTake={(target) => board.setPickerTarget({ type: 'clip', sceneId: target.sceneId })}
onPlacePoolItem={takes.placePoolItem}
diff --git a/client/src/hooks/useMusicVideoTakes.js b/client/src/hooks/useMusicVideoTakes.js
index 9f32fe1a5b..aaff8dde5e 100644
--- a/client/src/hooks/useMusicVideoTakes.js
+++ b/client/src/hooks/useMusicVideoTakes.js
@@ -7,6 +7,7 @@ import {
continueMusicVideoSceneFromPrevious,
reviewMusicVideoSceneTake,
rejectUnusedMusicVideoSceneTakes,
+ swapMusicVideoSceneFrames,
getMusicVideoHandoff,
getMusicVideoHandoffBundle,
importMusicVideoHandoff,
@@ -113,6 +114,15 @@ export default function useMusicVideoTakes({ project, applyScenePatch }) {
'Failed to update take',
);
+ // Start and end frames trade places, each bringing its prompt along.
+ const swapFrames = (scene) => runTakeOp(
+ (projectId) => swapMusicVideoSceneFrames(projectId, scene.sceneId, { silent: true }).then((placed) => {
+ applyScenePatch?.(projectId, placed.sceneId, { framePrompt: placed.framePrompt ?? null, endFramePrompt: placed.endFramePrompt ?? null });
+ return placed;
+ }),
+ 'Failed to swap the frames',
+ );
+
// Finalize a shot: every take other than its selected frame, end frame and clip is rejected (files kept).
const rejectUnusedTakes = (scene) => runTakeOp(
(projectId) => rejectUnusedMusicVideoSceneTakes(projectId, scene.sceneId, { silent: true }).then((res) => res.scene),
@@ -192,5 +202,5 @@ export default function useMusicVideoTakes({ project, applyScenePatch }) {
if (failures.length) toast.error(failures.slice(0, 3).join(' · '));
};
- return { busy, selectTake, placePoolItem, trimClipTake, reviewTake, rejectUnusedTakes, continueFromPrevious, importTake, importClipTake, exportHandoff, exportHandoffBundle, importHandoffFiles };
+ return { busy, selectTake, placePoolItem, trimClipTake, reviewTake, rejectUnusedTakes, swapFrames, continueFromPrevious, importTake, importClipTake, exportHandoff, exportHandoffBundle, importHandoffFiles };
}
diff --git a/client/src/services/apiMusicVideo.js b/client/src/services/apiMusicVideo.js
index 34289d695b..d560ddbba9 100644
--- a/client/src/services/apiMusicVideo.js
+++ b/client/src/services/apiMusicVideo.js
@@ -197,6 +197,9 @@ export const continueMusicVideoSceneFromPrevious = (id, sceneId, body = {}, opti
request(`/music-video/${encodeURIComponent(id)}/scenes/${encodeURIComponent(sceneId)}/continue-from-previous`, {
method: 'POST', body: JSON.stringify(body), ...options,
});
+// The shot's start frame becomes its end frame and the end frame its start, prompts included → the scene.
+export const swapMusicVideoSceneFrames = (id, sceneId, options = {}) =>
+ request(`/music-video/${encodeURIComponent(id)}/scenes/${encodeURIComponent(sceneId)}/swap-frames`, { method: 'POST', ...options });
export const rejectUnusedMusicVideoSceneTakes = (id, sceneId, options = {}) =>
request(`/music-video/${encodeURIComponent(id)}/scenes/${encodeURIComponent(sceneId)}/takes/reject-unused`, { method: 'POST', ...options });
export const reviewMusicVideoSceneTake = (id, sceneId, takeId, review, options = {}) =>
diff --git a/server/lib/musicVideoHandoffFrame.js b/server/lib/musicVideoHandoffFrame.js
index dfd4a88bac..7881bc4e68 100644
--- a/server/lib/musicVideoHandoffFrame.js
+++ b/server/lib/musicVideoHandoffFrame.js
@@ -9,9 +9,12 @@
* - a performance take starts at its edit in-point and stops at its edit out-point;
* - a looping shot wraps around the clip;
* - a non-looping shot longer than its clip holds the clip's last frame.
+ * - a shot's clip speed and frame offset move where it ends, and a reversed clip
+ * ends on an earlier frame (its first frame when the shot outlasts it).
* The returned time is the last displayed frame, one frame before the out-point.
*/
-import { sceneHasAuthoredSpan } from './musicVideoLayers.js';
+import { isEndFrameImageEffect } from './musicVideoImageEffects.js';
+import { clipOffsetSec, clipReversed, clipSpeedRate, isLayeredComposition, sceneHasAuthoredSpan, sceneVisualLayer } from './musicVideoLayers.js';
import { selectedPerformanceInstruction } from './musicVideoShotTiming.js';
const round3 = (n) => Math.round(n * 1000) / 1000;
@@ -27,15 +30,43 @@ export function sceneClipOutFrameSec(scene, clipDurationSec, fps = 24) {
const frameSec = 1 / (Number(fps) > 0 ? Number(fps) : 24);
const span = scene.endSec - scene.startSec;
const edit = selectedPerformanceInstruction(scene)?.edit ?? null;
+ // Mirrors render.js: a hand-set speed reads the clip faster or slower, and an offset starts it
+ // later (or, negative, holds its first frame that long first).
+ const rate = clipSpeedRate(scene);
+ const offset = clipOffsetSec(scene);
+ // A reversed clip is read once from its end, so the cut ends on an earlier frame of it.
+ if (!edit && clipReversed(scene)) {
+ const read = Math.max(0, offset) + Math.max(0, span + Math.min(0, offset)) * rate;
+ return round3(Math.min(duration - frameSec, Math.max(0, duration - read + frameSec * rate)));
+ }
const inSec = edit ? Math.min(Math.max(0, edit.inSec), duration) : 0;
const stopSec = edit ? Math.min(edit.outSec, duration) : duration;
- // Mirrors render.js: only a pre-#8964 scene (no `loop` key) or `loop: true`
- // repeats, and a performance take never does.
+ // Only a pre-#8964 scene (no `loop` key) or `loop: true` repeats, and a performance take never does.
const loops = !edit && scene.loop !== false;
const playable = stopSec - inSec;
if (!(playable > 0)) return round3(inSec);
- const played = loops && span > playable ? span % playable || playable : Math.min(span, playable);
- return round3(Math.max(inSec, inSec + played - frameSec));
+ const start = Math.max(0, offset);
+ const played = start + Math.max(0, span + Math.min(0, offset)) * rate;
+ const within = loops && played > playable ? played % playable || playable : Math.min(played, playable);
+ return round3(Math.max(inSec, inSec + within - frameSec * rate));
+}
+
+/**
+ * What the cut shows at the end of `shot`, for the shot after it to start from:
+ * `{ imageId }` for a still shot (its end frame when an effect plays to it, else
+ * its still), `{ clip: true }` for footage with a clip to read the out-frame
+ * from, or null when there is nothing to hand over (a drawn card or code shot,
+ * or a missing picture). A still shot's clip, left from before it became a
+ * still, is never on screen, so it is never read.
+ */
+export function shotHandoffSource(project, shot) {
+ if (!shot) return null;
+ const layer = sceneVisualLayer(shot, { layered: isLayeredComposition(project) });
+ if (layer === 'still') {
+ const imageId = isEndFrameImageEffect(shot.imageEffect) && shot.endImageId ? shot.endImageId : shot.referenceImageId;
+ return imageId ? { imageId } : null;
+ }
+ return layer === 'footage' && shot.videoHistoryId ? { clip: true } : null;
}
/**
diff --git a/server/lib/musicVideoHandoffFrame.test.js b/server/lib/musicVideoHandoffFrame.test.js
index 3d80cc788e..0ad604f9f8 100644
--- a/server/lib/musicVideoHandoffFrame.test.js
+++ b/server/lib/musicVideoHandoffFrame.test.js
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
-import { previousShot, sceneClipOutFrameSec } from './musicVideoHandoffFrame.js';
+import { previousShot, sceneClipOutFrameSec, shotHandoffSource } from './musicVideoHandoffFrame.js';
// Where a continuing shot's start frame comes from: the last frame the cut
// shows of the previous shot's clip, never the clip's own last frame.
@@ -14,6 +14,11 @@ describe('sceneClipOutFrameSec', () => {
endSec: 3, shotMode: 'performance', videoHistoryId: 'clip',
takes: [{ kind: 'video', assetId: 'clip', shotInstruction: { shotMode: 'performance', edit: { inSec: 1.5, outSec: 4.5 } } }],
}), 6, 24, 4.458],
+ // Uniquely catch a continuing shot starting from the wrong frame of a retimed or reversed previous clip.
+ ['a sped-up shot reads further into its clip', shot({ endSec: 4, speedFps: 6 }), 6, 24, 4.948],
+ ['an offset shot starts that far into its clip', shot({ endSec: 4, offsetFrames: 24 }), 6, 24, 4.958],
+ ['a reversed shot ends on an earlier frame of its clip', shot({ endSec: 4, reverse: true }), 6, 24, 2.042],
+ ['a reversed shot longer than its clip holds its first frame', shot({ endSec: 8, reverse: true }), 6, 24, 0],
])('%s', (_label, scene, durationSec, fps, expected) => {
expect(sceneClipOutFrameSec(scene, durationSec, fps)).toBe(expected);
});
@@ -31,3 +36,18 @@ describe('previousShot', () => {
expect(previousShot(project, 'a')).toBeNull();
});
});
+
+// Uniquely catches a still shot handing over a stale clip that is not on screen (the clip a still shot kept from before).
+describe('shotHandoffSource', () => {
+ const composed = { composition: { mode: 'composed' } };
+ const still = { sceneId: 's1', visualLayer: 'still', referenceImageId: 'still.png', endImageId: 'end.png', videoHistoryId: 'stale-clip' };
+ it('hands over what the cut ends on', () => {
+ expect(shotHandoffSource(composed, { ...still, imageEffect: 'end-glitch' })).toEqual({ imageId: 'end.png' });
+ expect(shotHandoffSource(composed, { ...still, imageEffect: 'flash-develop' })).toEqual({ imageId: 'still.png' });
+ expect(shotHandoffSource(composed, { ...still, imageEffect: 'end-glitch', endImageId: null })).toEqual({ imageId: 'still.png' });
+ expect(shotHandoffSource(composed, { sceneId: 's2', videoHistoryId: 'clip' })).toEqual({ clip: true });
+ expect(shotHandoffSource(composed, { sceneId: 's3', visualLayer: 'card' })).toBeNull();
+ // A plain concat render plays footage only, so a still-tagged shot there plays its clip.
+ expect(shotHandoffSource({}, still)).toEqual({ clip: true });
+ });
+});
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index 38a7c7371e..43f8a3765c 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -121,6 +121,7 @@ import {
markMediaPoolAssets,
reviewSceneTake,
rejectUnusedSceneTakes,
+ swapProjectSceneFrames,
refreshSongStyleFromSuno,
} from '../services/musicVideo/projects.js';
import { buildHandoffManifest, buildHandoffBundle, matchSceneByFileTag } from '../services/musicVideo/handoff.js';
@@ -1599,6 +1600,11 @@ router.post('/:id/scenes/:sceneId/takes/reject-unused', asyncHandler(async (req,
res.json(await rejectUnusedSceneTakes(req.params.id, req.params.sceneId));
}));
+// No body: the shot's start frame becomes its end frame and the end frame its start, prompts included.
+router.post('/:id/scenes/:sceneId/swap-frames', asyncHandler(async (req, res) => {
+ res.json(await swapProjectSceneFrames(req.params.id, req.params.sceneId));
+}));
+
router.patch('/:id/scenes/:sceneId/takes/:takeId', asyncHandler(async (req, res) => {
const review = validateRequest(musicVideoTakeReviewSchema, req.body);
res.json(await reviewSceneTake(req.params.id, req.params.sceneId, req.params.takeId, review));
diff --git a/server/services/musicVideo/continueShot.js b/server/services/musicVideo/continueShot.js
index 906dd65abd..cb1fe07a80 100644
--- a/server/services/musicVideo/continueShot.js
+++ b/server/services/musicVideo/continueShot.js
@@ -10,14 +10,15 @@
* (lib/musicVideoHandoffFrame.js) into the gallery and selects it as this
* scene's start frame. With `keepStillAsEnd`, the scene's current start frame
* becomes its style-break end frame (#11173), so a fal clip on an end-frame
- * model travels from the handoff to the planned still.
+ * model travels from the handoff to the planned still. A still shot before it
+ * hands over the picture it ends on (shotHandoffSource) instead of a clip.
*/
import { join } from 'path';
import { existsSync } from 'fs';
import { ServerError } from '../../lib/errorHandler.js';
import { ensureDir, PATHS } from '../../lib/fileUtils.js';
import { findFfmpeg, probeVideoGeometry, runFfmpegProcess, safeUnder } from '../../lib/ffmpeg.js';
-import { previousShot, sceneClipOutFrameSec } from '../../lib/musicVideoHandoffFrame.js';
+import { previousShot, sceneClipOutFrameSec, shotHandoffSource } from '../../lib/musicVideoHandoffFrame.js';
import { getHistoryItem } from '../videoGen/history.js';
import { appendSceneTakes, getProject, selectSceneTake } from './projects.js';
import { assertClipUnlocked } from './productionReview.js';
@@ -36,10 +37,14 @@ export async function continueFromPreviousShot(projectId, sceneId, { keepStillAs
assertClipUnlocked(project, sceneId);
const previous = previousShot(project, sceneId);
if (!previous) throw refuse('This is the first shot, so there is no previous shot to continue from', 'MUSIC_VIDEO_NO_PREVIOUS_SHOT');
- const entry = previous.videoHistoryId ? await getHistoryItem(previous.videoHistoryId) : null;
+ const source = shotHandoffSource(project, previous);
+ // A still shot hands over the picture the cut ends on, as it is: there is no clip on screen to read.
+ if (source?.imageId) return startFromFrame(projectId, scene, source.imageId, { keepStillAsEnd,
+ prompt: 'The picture the previous shot ends on', frame: { imageId: source.imageId, atSec: null, fromSceneId: previous.sceneId } });
+ const entry = source?.clip ? await getHistoryItem(previous.videoHistoryId) : null;
const videoPath = entry?.filename ? safeUnder(PATHS.videos, entry.filename) : null;
if (!videoPath || !existsSync(videoPath)) {
- throw refuse('The previous shot has no selected clip to continue from', 'MUSIC_VIDEO_PREVIOUS_CLIP_MISSING');
+ throw refuse('The previous shot has no selected clip or still to continue from', 'MUSIC_VIDEO_PREVIOUS_CLIP_MISSING');
}
const geometry = await probeVideoGeometry(videoPath);
const atSec = sceneClipOutFrameSec(previous, geometry?.durationSec, geometry?.fps);
@@ -62,12 +67,18 @@ export async function continueFromPreviousShot(projectId, sceneId, { keepStillAs
}
console.log(`🎞️ Handoff frame for ${sceneId}: ${previous.sceneId} clip at ${atSec}s of ${geometry.durationSec.toFixed(2)}s`);
+ return startFromFrame(projectId, scene, imageId, { keepStillAsEnd,
+ prompt: `Out-frame of the previous shot at ${atSec}s of its clip`, frame: { imageId, atSec, fromSceneId: previous.sceneId } });
+}
+
+// Select `imageId` as the scene's start frame; with `keepStillAsEnd`, its current start frame becomes its end frame.
+async function startFromFrame(projectId, scene, imageId, { keepStillAsEnd, prompt, frame }) {
const plannedStill = keepStillAsEnd && scene.referenceImageId && scene.referenceImageId !== imageId ? scene.referenceImageId : null;
- const { appended } = await appendSceneTakes(projectId, sceneId, [
- { kind: 'image', assetId: imageId, source: 'generated', provider: 'portos', prompt: `Out-frame of the previous shot at ${atSec}s of its clip` },
+ const { appended } = await appendSceneTakes(projectId, scene.sceneId, [
+ { kind: 'image', assetId: imageId, source: 'generated', provider: 'portos', prompt },
...(plannedStill ? [{ kind: 'endImage', assetId: plannedStill, source: 'imported', provider: 'portos' }] : []),
]);
- let updated = await selectSceneTake(projectId, sceneId, appended[0].takeId);
- if (plannedStill) updated = await selectSceneTake(projectId, sceneId, appended[1].takeId);
- return { scene: updated, frame: { imageId, atSec, fromSceneId: previous.sceneId } };
+ let updated = await selectSceneTake(projectId, scene.sceneId, appended[0].takeId);
+ if (plannedStill) updated = await selectSceneTake(projectId, scene.sceneId, appended[1].takeId);
+ return { scene: updated, frame };
}
diff --git a/server/services/musicVideo/continueShot.test.js b/server/services/musicVideo/continueShot.test.js
index 4707819c85..777f28b8b1 100644
--- a/server/services/musicVideo/continueShot.test.js
+++ b/server/services/musicVideo/continueShot.test.js
@@ -51,6 +51,18 @@ describe.skipIf(!ffmpeg)('continueFromPreviousShot', () => {
expect(mocks.selected).toEqual(['t0', 't1']);
});
+ // Uniquely catches a still shot handing over the stale clip it kept, which the cut never shows.
+ it('starts from a still shot\'s end frame, never its stale clip', async () => {
+ mocks.appended.length = 0; mocks.selected.length = 0;
+ mocks.project = { id: 'mv-1', composition: { mode: 'composed' }, scenes: [
+ { sceneId: 'a', startSec: 0, endSec: 1, visualLayer: 'still', imageEffect: 'end-glitch', referenceImageId: 'still.png', endImageId: 'end.png', videoHistoryId: 'clip-a' },
+ { sceneId: 'b', startSec: 1, endSec: 3 },
+ ] };
+ const { frame } = await continueFromPreviousShot('mv-1', 'b');
+ expect(frame).toEqual({ imageId: 'end.png', atSec: null, fromSceneId: 'a' });
+ expect(mocks.appended.map((t) => [t.kind, t.assetId])).toEqual([['image', 'end.png']]);
+ });
+
it('refuses the first shot and a previous shot without a clip', async () => {
mocks.project = { id: 'mv-1', scenes: [{ sceneId: 'a', startSec: 0, endSec: 1 }, { sceneId: 'b', startSec: 1, endSec: 2 }] };
await expect(continueFromPreviousShot('mv-1', 'a')).rejects.toMatchObject({ code: 'MUSIC_VIDEO_NO_PREVIOUS_SHOT' });
diff --git a/server/services/musicVideo/projects.js b/server/services/musicVideo/projects.js
index 6c7c769603..03e2a0a26e 100644
--- a/server/services/musicVideo/projects.js
+++ b/server/services/musicVideo/projects.js
@@ -452,6 +452,17 @@ export async function rejectUnusedSceneTakes(id, sceneId) {
return { scene, rejected };
}
+/** Swap a shot's start and end frames (and their prompts). A locked video refuses it. Returns the scene. */
+export async function swapProjectSceneFrames(id, sceneId) {
+ const [{ swapSceneFrames }, { assertClipUnlocked, keepDirectorStillLock }] = await Promise.all([import('./takes.js'), import('./productionReview.js')]);
+ const { scene } = await mutateProjectRecord(id, (current) => {
+ assertClipUnlocked(current, sceneId);
+ const outcome = swapSceneFrames(current, sceneId);
+ return { ...outcome, project: keepDirectorStillLock(current, outcome.project, sceneId) };
+ });
+ return scene;
+}
+
/** Reject/restore a take and/or set its note. Returns the updated scene. */
export async function reviewSceneTake(id, sceneId, takeId, review) {
const [{ reviewSceneTake: reviewTake }, { assertClipUnlocked }] = await Promise.all([import('./takes.js'), import('./productionReview.js')]);
diff --git a/server/services/musicVideo/takes.js b/server/services/musicVideo/takes.js
index 338b70c6fa..84c2b17e18 100644
--- a/server/services/musicVideo/takes.js
+++ b/server/services/musicVideo/takes.js
@@ -258,6 +258,34 @@ export function rejectUnusedTakes(project, sceneId) {
return { project: replaceScene(project, idx, next), scene: next, rejected };
}
+/**
+ * Swap a shot's selected start and end frames, with their prompts. Each still
+ * becomes a take of the other frame (an existing one is reused), so both stay
+ * selected takes and neither frame's other takes change.
+ */
+export function swapSceneFrames(project, sceneId) {
+ const idx = findSceneIndex(project, sceneId);
+ const scene = project.scenes[idx];
+ const start = scene.referenceImageId;
+ const end = scene.endImageId;
+ if (!isNonBlankStr(start) || !isNonBlankStr(end)) {
+ throw new ServerError('This shot needs both a start and an end frame to swap them.', { status: 409, code: 'MUSIC_VIDEO_SWAP_FRAMES_MISSING' });
+ }
+ const now = new Date().toISOString();
+ const takes = ensureSceneTakes(scene, now);
+ const origin = (kind, assetId) => takes.find((t) => t.kind === kind && t.assetId === assetId);
+ const { scene: withTakes } = appendToScene(project, { ...scene, takes }, [
+ { kind: 'image', assetId: end, source: 'imported', provider: 'portos', prompt: origin('endImage', end)?.prompt ?? null },
+ { kind: 'endImage', assetId: start, source: 'imported', provider: 'portos', prompt: origin('image', start)?.prompt ?? null },
+ ], now);
+ // A take the director had rejected comes back as a candidate once it is selected, as selectSceneTake does.
+ const restore = (t) => ((t.kind === 'image' && t.assetId === end) || (t.kind === 'endImage' && t.assetId === start)) && t.status === 'rejected'
+ ? { ...t, status: 'candidate' } : t;
+ const next = { ...withTakes, takes: withTakes.takes.map(restore), referenceImageId: end, endImageId: start,
+ framePrompt: scene.endFramePrompt ?? null, endFramePrompt: scene.framePrompt ?? null };
+ return { project: replaceScene(project, idx, next), scene: next };
+}
+
export function reviewSceneTake(project, sceneId, takeId, { status, note }) {
const { idx, scene, takes, take } = locateTake(project, sceneId, takeId);
const updated = {
diff --git a/server/services/musicVideo/takes.test.js b/server/services/musicVideo/takes.test.js
index 1d479524d2..a319c4aebd 100644
--- a/server/services/musicVideo/takes.test.js
+++ b/server/services/musicVideo/takes.test.js
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
-import { appendSceneTakes, rejectUnusedTakes } from './takes.js';
+import { appendSceneTakes, rejectUnusedTakes, swapSceneFrames } from './takes.js';
import { captureMusicVideoEvidence, captureTakeDependencies } from '../../lib/musicVideoDependencies.js';
const project = () => ({ id: 'mv-1', scenes: [{ sceneId: 's1' }] });
@@ -90,3 +90,19 @@ describe('reject unused takes', () => {
expect(rejectUnusedTakes(project, 's1').rejected).toBe(0);
});
});
+
+describe('swap start and end frames', () => {
+ // Uniquely catches the swap losing a prompt, duplicating a take it could reuse, or leaving a frame's take rejected while selected.
+ it('trades the selected frames and their prompts, reusing and restoring takes', () => {
+ const take = (takeId, kind, assetId, extra = {}) => ({ takeId, kind, assetId, status: 'candidate', note: null, ...extra });
+ const scene = { sceneId: 's1', referenceImageId: 'a.png', endImageId: 'e.png', framePrompt: 'Door opens', endFramePrompt: 'Door shut',
+ takes: [take('t1', 'image', 'a.png', { prompt: 'Door opens' }), take('t2', 'image', 'e.png', { status: 'rejected' }),
+ take('t3', 'endImage', 'e.png', { prompt: 'Door shut' })] };
+ const { scene: next } = swapSceneFrames({ scenes: [scene] }, 's1');
+ expect([next.referenceImageId, next.endImageId, next.framePrompt, next.endFramePrompt]).toEqual(['e.png', 'a.png', 'Door shut', 'Door opens']);
+ expect(next.takes.find((t) => t.takeId === 't2').status).toBe('candidate');
+ const added = next.takes.filter((t) => !['t1', 't2', 't3'].includes(t.takeId));
+ expect(added.map((t) => [t.kind, t.assetId, t.prompt])).toEqual([['endImage', 'a.png', 'Door opens']]);
+ expect(() => swapSceneFrames({ scenes: [{ ...scene, endImageId: null }] }, 's1')).toThrow(/both a start and an end frame/);
+ });
+});