Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

feat: revise a music video's song mid-project from the Song step

Claude83318c763862
diff --git a/client/src/components/musicVideo/ReviseSongCard.jsx b/client/src/components/musicVideo/ReviseSongCard.jsx
new file mode 100644
index 0000000000..5af4645380
--- /dev/null
+++ b/client/src/components/musicVideo/ReviseSongCard.jsx
@@ -0,0 +1,179 @@
+import { useEffect, useRef, useState } from 'react';
+import { Link } from 'react-router';
+import { Check, RefreshCw } from 'lucide-react';
+import socket from '../../services/socket';
+import { reviseMusicVideoSongFromTrack, actOnMusicVideoSongRevisionScenes } from '../../services/apiMusicVideo.js';
+import useYoutubeTrackImport from '../../hooks/useYoutubeTrackImport.js';
+import { formatCount } from '../../utils/formatters.js';
+import { trackOptionLabels } from '../../utils/trackOptionLabels.js';
+
+const buttonClass = 'min-h-[44px] rounded border border-port-border bg-port-bg px-3 py-1.5 text-sm disabled:opacity-50 sm:min-h-0';
+const primaryClass = 'min-h-[44px] rounded bg-port-accent px-3 py-1.5 text-sm text-white disabled:opacity-50 sm:min-h-0';
+
+const countLine = (parts) => parts.filter(([n]) => n > 0).map(([n, label]) => `${formatCount(n)} ${label}`).join(' · ');
+
+/** The lines the new song kept, reworded, added and cut. */
+function LyricChanges({ project, revision }) {
+ const status = revision.cueStatus || {};
+ const cues = project.lyricCues || [];
+ const changed = cues.filter((c) => status[c.id] === 'changed');
+ const added = cues.filter((c) => status[c.id] === 'added');
+ const removed = revision.removedLines || [];
+ if (!changed.length && !added.length && !removed.length) return null;
+ return (
+ <details className="text-xs">
+ <summary className="cursor-pointer py-2">What changed in the lyrics</summary>
+ <ul className="space-y-1">
+ {changed.map((c) => <li key={c.id} className="break-words"><span className="text-port-text-muted line-through">{revision.changedFrom?.[c.id]}</span> → {c.text}</li>)}
+ {added.map((c) => <li key={c.id} className="break-words text-port-success">+ {c.text}</li>)}
+ {removed.map((c) => <li key={c.id} className="break-words text-port-error">− {c.text}</li>)}
+ </ul>
+ </details>
+ );
+}
+
+/**
+ * Revise the song mid-project, top of the Song step. Paste the new Suno link
+ * (or pick a library track): a new version gets the song and this one stays
+ * as it is. The new version re-times on its own (analysis, the separated
+ * vocal, lyric alignment), moves every shot through the lines both songs
+ * share, and flags the shots the new lyrics changed, added or cut.
+ */
+export default function ReviseSongCard({ project, tracks = [], disabled = false, retime = null, onRevised, onRetime, onUpdated, onTrackImported }) {
+ const revision = project.songRevision?.status === 'selected' && project.songRevision.baseline ? project.songRevision : null;
+ const [open, setOpen] = useState(false);
+ const [url, setUrl] = useState('');
+ const [busy, setBusy] = useState(false);
+ const [error, setError] = useState('');
+ const sequence = useRef(0);
+ sequence.current = Math.max(sequence.current, project.songRevision?.sequence || 0);
+ const updated = useRef(onUpdated);
+ updated.current = onUpdated;
+
+ // Re-time progress and its result arrive as song-revision events.
+ useEffect(() => {
+ const receive = (event) => {
+ if (event.projectId !== project.id || (event.project?.songRevision?.sequence || 0) <= sequence.current) return;
+ sequence.current = event.project.songRevision.sequence;
+ updated.current?.(event.project);
+ };
+ socket.on('music-video:song-revision', receive);
+ return () => socket.off('music-video:song-revision', receive);
+ }, [project.id]);
+
+ const run = (request) => {
+ setBusy(true); setError('');
+ return request().catch((err) => setError(err?.message || 'That did not work. Try again.')).finally(() => setBusy(false));
+ };
+ const revise = (trackId) => run(() => reviseMusicVideoSongFromTrack(project.id, trackId, { silent: true }).then((result) => {
+ setOpen(false); setUrl('');
+ onRevised?.(result);
+ }));
+ const importJob = useYoutubeTrackImport({
+ onComplete: (track) => {
+ onTrackImported?.(track);
+ revise(track.id);
+ },
+ });
+ const act = (action) => run(() => actOnMusicVideoSongRevisionScenes(project.id, { revisionId: revision.id, action }, { silent: true })
+ .then(({ project: next }) => onUpdated?.(next)));
+
+ const working = busy || importJob.active;
+ const picker = (
+ <div className="space-y-2">
+ <p className="text-xs text-port-text-muted">A new version gets the new song and keeps your shots; this version stays as it is.</p>
+ <div className="flex flex-wrap gap-2">
+ <label htmlFor={`mv-revise-song-url-${project.id}`} className="sr-only">New song link</label>
+ <input id={`mv-revise-song-url-${project.id}`} type="url" value={url} onChange={(e) => setUrl(e.target.value)} disabled={working || disabled}
+ onKeyDown={(e) => { if (e.key === 'Enter' && url.trim()) { e.preventDefault(); importJob.start(url); } }}
+ placeholder="Paste the new Suno link…" className="min-h-[44px] min-w-0 flex-1 rounded border border-port-border bg-port-bg px-2 py-1.5 text-sm sm:min-h-0" />
+ {importJob.active
+ ? <button type="button" onClick={importJob.cancel} className={buttonClass}>Importing {formatCount(importJob.percent, { fallback: '0' })}% · Cancel</button>
+ : <button type="button" onClick={() => importJob.start(url)} disabled={!url.trim() || working || disabled} className={primaryClass}>Use this song</button>}
+ </div>
+ <label htmlFor={`mv-revise-song-track-${project.id}`} className="block text-xs text-port-text-muted">Or a song already in the library</label>
+ <select id={`mv-revise-song-track-${project.id}`} value="" disabled={working || disabled} onChange={(e) => e.target.value && revise(e.target.value)}
+ className="min-h-[44px] w-full rounded border border-port-border bg-port-bg px-2 py-1 text-sm sm:min-h-0">
+ <option value="">Pick a track…</option>
+ {(() => {
+ const labels = trackOptionLabels(tracks);
+ return tracks.filter((t) => t.id !== project.trackId).map((t) => <option key={t.id} value={t.id}>{labels.get(t.id)}</option>);
+ })()}
+ </select>
+ </div>
+ );
+
+ if (!revision) {
+ return (
+ <section aria-label="Revise song" className="min-w-0 space-y-2 rounded-lg border border-port-border p-2">
+ <div className="flex flex-wrap items-center justify-between gap-2">
+ <h3 className="text-sm font-medium">Changed the song?</h3>
+ <button type="button" aria-expanded={open} onClick={() => setOpen((v) => !v)} disabled={disabled}
+ className={`${buttonClass} flex items-center gap-1 text-port-accent`}><RefreshCw size={14} aria-hidden="true" /> Revise song</button>
+ </div>
+ {open && picker}
+ {error && <p role="alert" className="text-xs text-port-error">{error}</p>}
+ </section>
+ );
+ }
+
+ const review = revision.sceneReview || {};
+ const pending = (statuses) => Object.values(review).filter((entry) => statuses.includes(entry.status) && !entry.resolved).length;
+ const toReplan = pending(['changed', 'new']);
+ const toRemove = pending(['removed']);
+ const counts = revision.sceneCounts;
+ const lyrics = revision.lyricDiff || {};
+ // A stored 'running' with no live job slot is a job this page lost (a restart): offer the button again.
+ const stored = revision.retime?.status || 'pending';
+ // `retime` is the page's alignment slot, which a plain Align words run also fills.
+ const state = stored === 'done' ? 'done' : retime ? 'running' : stored === 'running' ? 'pending' : stored;
+ return (
+ <section aria-label="Revised song" className="min-w-0 space-y-2 rounded-lg border border-port-accent/50 p-2">
+ <div className="flex flex-wrap items-center justify-between gap-2">
+ <h3 className="text-sm font-medium">Revised song{revision.fields?.title ? `: ${revision.fields.title}` : ''}</h3>
+ {project.parentProjectId && <Link to={`/music-video/${encodeURIComponent(project.parentProjectId)}/setup`} className="text-xs text-port-accent">Previous version</Link>}
+ </div>
+ <p className="text-xs">Lyrics: {countLine([[lyrics.kept, 'kept'], [lyrics.changed, 'reworded'], [lyrics.added, 'new'], [lyrics.removed, 'cut']]) || 'no lines'}</p>
+ <LyricChanges project={project} revision={revision} />
+
+ {state === 'running' && (
+ <div role="status" className="flex flex-wrap items-center gap-2 text-xs text-port-text-muted">
+ <RefreshCw size={14} className="animate-spin" aria-hidden="true" />
+ <span>{retime?.label || 'Re-timing to the new song…'}{retime?.percent > 0 ? ` ${formatCount(retime.percent)}%` : ''}</span>
+ {retime?.onCancel && <button type="button" onClick={retime.onCancel} className={`${buttonClass} text-port-error`}>Cancel</button>}
+ </div>
+ )}
+ {state !== 'running' && state !== 'done' && (
+ <div className="space-y-1">
+ {state === 'failed' && <p role="alert" className="text-xs text-port-error">{revision.retime?.error || 'Re-timing failed.'} Fix the lyrics below if they differ from the vocal, then try again.</p>}
+ <button type="button" onClick={onRetime} disabled={disabled || working} className={primaryClass}>{state === 'failed' ? 'Try re-timing again' : 'Re-time lyrics and shots'}</button>
+ <p className="text-xs text-port-text-muted">Analyzes the new song, separates its vocal, aligns the lyrics and moves the shots.</p>
+ </div>
+ )}
+ {state === 'done' && counts && (
+ <div className="space-y-2">
+ <p className="flex items-center gap-1 text-xs text-port-success"><Check size={14} aria-hidden="true" /> Shots moved to the new song: {countLine([[counts.kept, 'kept'], [counts.changed, 'changed'], [counts.new, 'new'], [counts.removed, 'cut']]) || 'none'}</p>
+ {(toReplan > 0 || toRemove > 0) && (
+ <div className="flex flex-wrap gap-2">
+ {toReplan > 0 && <button type="button" onClick={() => act('replan')} disabled={disabled || working} className={primaryClass}>{busy ? 'Working…' : `Replan ${formatCount(toReplan)} shot${toReplan === 1 ? '' : 's'}`}</button>}
+ {toRemove > 0 && <button type="button" onClick={() => act('remove')} disabled={disabled || working} className={buttonClass}>Remove {formatCount(toRemove)} cut shot{toRemove === 1 ? '' : 's'}</button>}
+ <Link to={`/music-video/${encodeURIComponent(project.id)}/board?scenes=attention`} className={`${buttonClass} text-port-accent`}>Review on Storyboard</Link>
+ <button type="button" onClick={() => act('dismiss')} disabled={disabled || working} className={buttonClass}>Keep them as they are</button>
+ </div>
+ )}
+ {toReplan > 0 && <p className="text-xs text-port-text-muted">Replan writes new frame and motion prompts for the changed and new shots with the planning model.</p>}
+ {revision.compositionStale && (
+ <p className="text-xs text-port-warning">The composition document still has the old song&rsquo;s timing and lyrics. <Link to={`/music-video/${encodeURIComponent(project.id)}/produce#mv-composition`} className="text-port-accent">Regenerate or reimport it</Link>.</p>
+ )}
+ {toReplan === 0 && toRemove === 0 && (
+ <details open={open} onToggle={(e) => setOpen(e.currentTarget.open)}>
+ <summary className="cursor-pointer py-2 text-xs text-port-accent">Revise the song again</summary>
+ {picker}
+ </details>
+ )}
+ </div>
+ )}
+ {error && <p role="alert" className="text-xs text-port-error">{error}</p>}
+ </section>
+ );
+}
diff --git a/client/src/components/musicVideo/ReviseSongCard.test.jsx b/client/src/components/musicVideo/ReviseSongCard.test.jsx
new file mode 100644
index 0000000000..461905e33b
--- /dev/null
+++ b/client/src/components/musicVideo/ReviseSongCard.test.jsx
@@ -0,0 +1,66 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { render, screen, fireEvent, waitFor, act } from '@testing-library/react';
+import { MemoryRouter } from 'react-router';
+import ReviseSongCard from './ReviseSongCard.jsx';
+
+const socket = vi.hoisted(() => ({ on: vi.fn(), off: vi.fn() }));
+const api = vi.hoisted(() => ({ revise: vi.fn(), act: vi.fn() }));
+vi.mock('../../services/apiMusicVideo.js', () => ({ reviseMusicVideoSongFromTrack: api.revise, actOnMusicVideoSongRevisionScenes: api.act }));
+vi.mock('../../services/socket', () => ({ default: socket }));
+vi.mock('../../hooks/useYoutubeTrackImport.js', () => ({ default: () => ({ active: false, percent: 0, start: vi.fn(), cancel: vi.fn() }) }));
+
+const tracks = [{ id: 'track-old', title: 'Old take' }, { id: 'track-new', title: 'New take' }];
+const revised = {
+ id: 'fork', parentProjectId: 'original', trackId: 'track-new',
+ lyricCues: [{ id: 'a', text: 'Line a' }, { id: 'b', text: 'Line b again' }, { id: 'n', text: 'Brand new' }],
+ songRevision: { id: 'rev', status: 'selected', sequence: 2, baseline: { cues: [], scenes: [] }, fields: { title: 'New take' },
+ cueStatus: { a: 'kept', b: 'changed', n: 'added' }, changedFrom: { b: 'Line b' }, removedLines: [{ id: 'x', text: 'Cut line' }],
+ lyricDiff: { kept: 1, changed: 1, added: 1, removed: 1 }, retime: { status: 'done' },
+ sceneCounts: { kept: 2, changed: 1, new: 1, removed: 1 },
+ sceneReview: { s1: { status: 'kept' }, s2: { status: 'changed' }, s3: { status: 'new' }, s4: { status: 'removed' } } },
+};
+const renderCard = (props) => render(<MemoryRouter><ReviseSongCard tracks={tracks} {...props} /></MemoryRouter>);
+beforeEach(() => vi.clearAllMocks());
+
+describe('Revise song', () => {
+ it('forks a new version from a library track', async () => {
+ const result = { project: { ...revised }, retimeJobId: 'job' };
+ api.revise.mockResolvedValue(result);
+ const onRevised = vi.fn();
+ renderCard({ project: { id: 'original', trackId: 'track-old' }, onRevised });
+ fireEvent.click(screen.getByRole('button', { name: 'Revise song' }));
+ const picker = screen.getByLabelText('Or a song already in the library');
+ expect([...picker.options].map((o) => o.value)).toEqual(['', 'track-new']);
+ fireEvent.change(picker, { target: { value: 'track-new' } });
+ await waitFor(() => expect(onRevised).toHaveBeenCalledWith(result));
+ expect(api.revise).toHaveBeenCalledWith('original', 'track-new', { silent: true });
+ });
+
+ it('shows what the new lyrics changed and acts on the flagged shots', async () => {
+ const onUpdated = vi.fn();
+ api.act.mockResolvedValue({ project: { ...revised, songRevision: { ...revised.songRevision, sequence: 3 } } });
+ renderCard({ project: revised, onUpdated });
+ expect(screen.getByText('Lyrics: 1 kept · 1 reworded · 1 new · 1 cut')).toBeInTheDocument();
+ expect(screen.getByText('Cut line', { exact: false })).toBeInTheDocument();
+ expect(screen.getByRole('link', { name: 'Review on Storyboard' })).toHaveAttribute('href', '/music-video/fork/board?scenes=attention');
+ fireEvent.click(screen.getByRole('button', { name: 'Replan 2 shots' }));
+ await waitFor(() => expect(onUpdated).toHaveBeenCalled());
+ expect(api.act).toHaveBeenCalledWith('fork', { revisionId: 'rev', action: 'replan' }, { silent: true });
+ expect(screen.getByRole('button', { name: 'Remove 1 cut shot' })).toBeInTheDocument();
+ });
+
+ it('offers the re-time again after a failure and applies newer re-time results from the server', () => {
+ const onRetime = vi.fn();
+ const onUpdated = vi.fn();
+ const failed = { ...revised, songRevision: { ...revised.songRevision, retime: { status: 'failed', error: 'Some lyric words aligned to silence.' }, sceneReview: null, sceneCounts: null } };
+ renderCard({ project: failed, onRetime, onUpdated });
+ expect(screen.getByRole('alert')).toHaveTextContent('aligned to silence');
+ fireEvent.click(screen.getByRole('button', { name: 'Try re-timing again' }));
+ expect(onRetime).toHaveBeenCalled();
+ const receive = socket.on.mock.calls.find(([event]) => event === 'music-video:song-revision')[1];
+ act(() => receive({ projectId: 'fork', project: { ...revised, songRevision: { ...revised.songRevision, sequence: 1 } } }));
+ expect(onUpdated).not.toHaveBeenCalled();
+ act(() => receive({ projectId: 'fork', project: { ...revised, songRevision: { ...revised.songRevision, sequence: 9 } } }));
+ expect(onUpdated).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 0d3c731432..25f4593842 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -77,7 +77,7 @@ export default function SceneCard({
onGenerateFrame, onGenerateVideo, onContinueVideo,
onOpenPreview, onSelectTake, onReviewTake, onImportTake, onImportClipTake, takeBusy = false, layered = false, documentComposition = false,
lipSyncBackend = '', songDurationSec = null, onSplit, falVideoSettings = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,
- expanded, onToggleExpand, footageOptional = false, failedScenes = null,
+ expanded, onToggleExpand, footageOptional = false, failedScenes = null, songReview = null,
}) {
const detailsRef = useRef(null);
@@ -142,7 +142,8 @@ export default function SceneCard({
excerptStartSec: performanceReview?.excerptStartSec, backend: lipSyncBackend, videoSettings: falVideoSettings || {} });
const shotModeId = `mv-shot-mode-${scene.sceneId}`;
// #10152: what needs the director's attention, visible without opening the card.
- const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec, failed: failedScenes || {} });
+ const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec, failed: failedScenes || {}, songReview });
+ const songChange = songReview?.[scene.sceneId];
const generatingLane = generatingFrame && generatingVideo ? 'Frame + clip' : generatingFrame ? 'Frame' : generatingVideo ? 'Clip' : null;
// #10154: the server-persisted failure of the last render, shown until a
// retry is in flight (or lands a take, which clears it server-side).
@@ -221,6 +222,9 @@ export default function SceneCard({
{(scene.lyricText || scene.visualIntent || scene.direction) && (
<div className="text-[11px] text-port-text-muted space-y-0.5">
{scene.lyricText && <p className="italic break-words">♪ {scene.lyricText}</p>}
+ {songChange && !songChange.resolved && songChange.status !== 'kept' && (
+ <p className="break-words text-port-warning">{songChange.status === 'new' ? 'New shot for lines the revised song added.' : `Before the song revision: ${songChange.previousLyricText ? `♪ ${songChange.previousLyricText}` : 'instrumental'}`}</p>
+ )}
{scene.visualIntent && <p className="break-words">Intent: {scene.visualIntent}</p>}
{/* Applied treatment direction (#8980) — appended to both generated prompts. */}
{scene.direction && (
diff --git a/client/src/components/musicVideo/SongRevisionPanel.jsx b/client/src/components/musicVideo/SongRevisionPanel.jsx
index cc93efa750..64e85d7bad 100644
--- a/client/src/components/musicVideo/SongRevisionPanel.jsx
+++ b/client/src/components/musicVideo/SongRevisionPanel.jsx
@@ -92,10 +92,9 @@ export default function SongRevisionPanel({ project, tracks = [], onUpdated, onF
</div>)}
</div>
{revision.status === 'selected' && <div className="space-y-2 text-sm" role="status">
- <p>New master selected. Art and editable composition source are retained. Beat analysis, lyric alignment, shot timing, proofs and renders need fresh review.</p>
+ <p>New master selected. Art and editable composition source are retained. The Song step re-times the lyrics on the new vocal and moves the shots to the new song.</p>
<ol className="list-inside list-decimal space-y-1">
- <li><Link className="text-port-accent" to={`/music-video/${project.id}/setup#mv-lyric-timing`}>Analyze the new song and align its lyrics on the Song step; verify by listening.</Link></li>
- <li><Link className="text-port-accent" to={`/music-video/${project.id}/board`}>Review shot timing and retained or revised art on the Storyboard.</Link></li>
+ <li><Link className="text-port-accent" to={`/music-video/${project.id}/setup`}>Follow the re-time on the Song step, then replan the shots it flags.</Link></li>
<li><Link className="text-port-accent" to={`/music-video/${project.id}/produce#mv-composition`}>Regenerate or reimport the composition against the new audio.</Link> Imported source can contain a fuller timeline than the Storyboard; revise that source too.</li>
<li><Link className="text-port-accent" to={`/music-video/${project.id}/produce#mv-review-proof`}>Rebuild and review proofs before the full video.</Link></li>
</ol>
diff --git a/client/src/components/musicVideo/SongRevisionPanel.test.jsx b/client/src/components/musicVideo/SongRevisionPanel.test.jsx
index e933800c3a..13ec08d1ba 100644
--- a/client/src/components/musicVideo/SongRevisionPanel.test.jsx
+++ b/client/src/components/musicVideo/SongRevisionPanel.test.jsx
@@ -33,7 +33,7 @@ describe('song revision UI', () => {
expect(select).toBeDisabled(); fireEvent.play(screen.getByLabelText('Listen to candidate 1')); fireEvent.click(select);
expect(await screen.findByText(/New master selected/)).toBeInTheDocument();
expect(screen.getByRole('link', { name: /Regenerate or reimport/ })).toHaveAttribute('href', '/music-video/fork/produce#mv-composition');
- expect(screen.getByRole('link', { name: /align its lyrics on the Song step/ })).toHaveAttribute('href', '/music-video/fork/setup#mv-lyric-timing');
+ expect(screen.getByRole('link', { name: /Follow the re-time on the Song step/ })).toHaveAttribute('href', '/music-video/fork/setup');
expect(screen.getByRole('link', { name: /Rebuild and review proofs/ })).toHaveAttribute('href', '/music-video/fork/produce#mv-review-proof');
expect(api.act.mock.calls.map((call) => call[1])).toEqual(['generate', 'select']);
});
diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsx
index 03e45bfcc7..99d52288f6 100644
--- a/client/src/components/musicVideo/stages/BoardStage.jsx
+++ b/client/src/components/musicVideo/stages/BoardStage.jsx
@@ -40,6 +40,7 @@ export default function BoardStage({ board }) {
const attentionCtx = {
layered, lipSyncBackend, songDurationSec, failed: sceneMedia.failedScenes,
footageOptional: FOOTAGE_OPTIONAL_MODES.has(project.composition?.mode || 'concat'),
+ songReview: project.songRevision?.sceneReview || null,
};
const codesById = new Map(scenes.map((scene) => [scene.sceneId, sceneAttention(scene, attentionCtx)]));
const counts = { all: scenes.length, attention: 0, missing: 0 };
@@ -174,6 +175,7 @@ export default function BoardStage({ board }) {
songDurationSec={songDurationSec}
footageOptional={attentionCtx.footageOptional}
failedScenes={sceneMedia.failedScenes}
+ songReview={attentionCtx.songReview}
falVideoSettings={videoSettings.settings}
canContinueShot={board.canContinueShot}
onMove={board.moveScene}
diff --git a/client/src/components/musicVideo/stages/ProduceStage.jsx b/client/src/components/musicVideo/stages/ProduceStage.jsx
index e36f08980f..9c4f58d957 100644
--- a/client/src/components/musicVideo/stages/ProduceStage.jsx
+++ b/client/src/components/musicVideo/stages/ProduceStage.jsx
@@ -29,6 +29,7 @@ export default function ProduceStage({ board }) {
lipSyncBackend: videoSettings.audioReactiveSelected ? 'local' : videoSettings.settings.backend,
songDurationSec: project.audioAnalysis?.durationSec ?? null,
failed: sceneMedia.failedScenes,
+ songReview: project.songRevision?.sceneReview || null,
}} />
)}
<div className="flex justify-end"><ContactSheetButton onOpen={board.openContactSheet} /></div>
diff --git a/client/src/components/musicVideo/stages/SetupStage.jsx b/client/src/components/musicVideo/stages/SetupStage.jsx
index dfd83d90b9..f1a99e8b0c 100644
--- a/client/src/components/musicVideo/stages/SetupStage.jsx
+++ b/client/src/components/musicVideo/stages/SetupStage.jsx
@@ -3,6 +3,7 @@ import TrackPanel from '../TrackPanel.jsx';
import AnalysisPanel from '../AnalysisPanel.jsx';
import LyricsPanel from '../LyricsPanel.jsx';
import LyricTimingCheck from '../LyricTimingCheck.jsx';
+import ReviseSongCard from '../ReviseSongCard.jsx';
import { AnalyzeAction } from '../ProjectActionGroups.jsx';
import { Check } from 'lucide-react';
import { lyricSetupState, projectHasAudio } from '../../../lib/musicVideoStages.js';
@@ -53,9 +54,11 @@ function AlignWords({ disabled, aligning, status, aligned, onAlign }) {
/**
* Song: the track, its analysis, the lyrics, and their word timing — four
- * numbered parts, each marked done or to do, all open. Forking or revising the
- * song, the vocal stem, MIDI and timing revisions live in Project settings ›
- * Audio; the render style and services in Project settings › Project.
+ * numbered parts, each marked done or to do, all open. Above them, Revise song
+ * brings in a new take mid-project and carries the board across to it.
+ * Writing new lyrics for a Suno take, the vocal stem, MIDI and timing revisions
+ * live in Project settings › Audio; the render style and services in Project
+ * settings › Project.
*/
export default function SetupStage({ board }) {
const {
@@ -68,6 +71,18 @@ export default function SetupStage({ board }) {
return (
<fieldset disabled={locked} className="min-w-0">
<div id="mv-track" className="space-y-2">
+ {projectHasAudio(project) && (
+ <ReviseSongCard
+ project={project}
+ tracks={tracks}
+ disabled={renderBound || midiBound}
+ retime={board.retimeStatus}
+ onRevised={board.onSongRevised}
+ onRetime={board.onRetimeSong}
+ onUpdated={board.replaceProject}
+ onTrackImported={board.onTrackImported}
+ />
+ )}
<SongStep number={1} title="Track" done={projectHasAudio(project)}>
<TrackPanel
project={project}
diff --git a/client/src/hooks/useMusicVideoLyricAlign.js b/client/src/hooks/useMusicVideoLyricAlign.js
index 04e8b501f4..6a12be5eaf 100644
--- a/client/src/hooks/useMusicVideoLyricAlign.js
+++ b/client/src/hooks/useMusicVideoLyricAlign.js
@@ -19,6 +19,8 @@ export function lyricAlignStageLabel(frame) {
case 'downloading-model': return 'Downloading the speech model (first run only)…';
case 'transcribing': return frame.total > 1 ? `Transcribing phrase ${frame.current} of ${frame.total}…` : 'Transcribing…';
case 'saving': return 'Saving word timings…';
+ case 'analyzing': return 'Analyzing the new song…';
+ case 'remapping': return 'Moving the shots to the new song…';
default: return 'Starting…';
}
}
@@ -40,12 +42,14 @@ export default function useMusicVideoLyricAlign({ onAligned } = {}) {
if (pending) fn(pending);
};
const slot = useSseJobSlot({
- startRequest: ({ projectId, cueId, separateVocals }) => alignMusicVideoLyrics(projectId, { ...(cueId ? { cueId } : {}), ...(separateVocals ? { separateVocals: true } : {}) }, { silent: true }),
+ startRequest: ({ projectId, cueId, separateVocals, retimeSong }) => alignMusicVideoLyrics(projectId, {
+ ...(cueId ? { cueId } : {}), ...(separateVocals ? { separateVocals: true } : {}), ...(retimeSong ? { retimeSong: true } : {}),
+ }, { silent: true }),
eventsUrl: musicVideoLyricAlignEventsUrl,
cancelRequest: cancelMusicVideoLyricAlign,
- onComplete: (frame, { projectId, cueId }) => {
+ onComplete: (frame, { projectId, cueId, retimeSong }) => {
if (frame.project) onAligned?.(projectId, frame.project);
- toast.success(cueId ? 'Re-aligned that line' : 'Aligned words to the vocal');
+ toast.success(cueId ? 'Re-aligned that line' : retimeSong ? 'Re-timed lyrics and shots to the new song' : 'Aligned words to the vocal');
settle((pending) => pending.resolve(frame.project || null));
},
errorFallback: 'Could not align the words to the vocal',
@@ -71,10 +75,10 @@ export default function useMusicVideoLyricAlign({ onAligned } = {}) {
}, [slot.active]);
useEffect(() => () => settle((pending) => pending.resolve(null)), []);
- const run = (projectId, cueId = null, { separateVocals = false } = {}) => new Promise((resolve, reject) => {
+ const run = (projectId, cueId = null, { separateVocals = false, retimeSong = false } = {}) => new Promise((resolve, reject) => {
if (slot.active) { resolve(null); return; }
waiter.current = { resolve, reject, started: false };
- slot.start({ projectId, cueId, ...(separateVocals ? { separateVocals } : {}) });
+ slot.start({ projectId, cueId, ...(separateVocals ? { separateVocals } : {}), ...(retimeSong ? { retimeSong } : {}) });
});
return {
diff --git a/client/src/lib/musicVideoSceneAttention.js b/client/src/lib/musicVideoSceneAttention.js
index 6fa728cad3..9fcd5eb54b 100644
--- a/client/src/lib/musicVideoSceneAttention.js
+++ b/client/src/lib/musicVideoSceneAttention.js
@@ -25,8 +25,14 @@ export const SCENE_ATTENTION_LABELS = Object.freeze({
'plate-unverified': 'Plate unverified',
'no-span': 'No time span',
'last-failure': 'Last render failed',
+ 'song-changed': 'Lyrics changed',
+ 'song-new': 'New lyrics',
+ 'song-cut': 'Lyrics cut',
});
+// A revised song's verdict on a shot (server songRevision.js sceneReview) as its attention code.
+const SONG_REVIEW_CODES = Object.freeze({ changed: 'song-changed', new: 'song-new', removed: 'song-cut' });
+
/** Codes that mean the shot still lacks material a render needs (the `missing` filter). */
export const SCENE_MISSING_CODES = Object.freeze(['missing-frame', 'missing-clip', 'no-span']);
@@ -48,13 +54,16 @@ function performanceBlocked(scene, { lipSyncBackend, songDurationSec }) {
* `lipSyncBackend`, `songDurationSec`, `clipSec` (the measured clip length —
* only a mounted card knows it, so board-level counts omit `under-covered`),
* and `failed: { frame: {sceneId: true}, video: {sceneId: true} }` for renders
- * that failed this session.
+ * that failed this session, and `songReview` (the project's revised-song
+ * verdicts by sceneId) so a shot the new lyrics changed is flagged until acted on.
*/
export function sceneAttention(scene, ctx = {}) {
if (!scene) return [];
- const { layered = false, footageOptional = false, lipSyncBackend = '', songDurationSec = null, clipSec = null, failed = {} } = ctx;
+ const { layered = false, footageOptional = false, lipSyncBackend = '', songDurationSec = null, clipSec = null, failed = {}, songReview = null } = ctx;
const layer = sceneVisualLayer(scene, { layered });
const codes = [];
+ const songChange = songReview?.[scene.sceneId];
+ if (songChange && !songChange.resolved && SONG_REVIEW_CODES[songChange.status]) codes.push(SONG_REVIEW_CODES[songChange.status]);
if (!isSelfDrawnLayer(layer) && !scene.referenceImageId) codes.push('missing-frame');
if (layer === 'footage' && !footageOptional && !scene.videoHistoryId) codes.push('missing-clip');
if (layer !== 'footage' && !sceneHasAuthoredSpan(scene)) codes.push('no-span');
diff --git a/client/src/pages/MusicVideo.jsx b/client/src/pages/MusicVideo.jsx
index 029de65cde..212d9c2e96 100644
--- a/client/src/pages/MusicVideo.jsx
+++ b/client/src/pages/MusicVideo.jsx
@@ -783,6 +783,15 @@ export default function MusicVideo() {
// Alignment is a click, never an import side effect. The panel shows the
// whisper setup error itself, so this request stays silent.
const handleAlignLyrics = (cueId, options) => lyricAlign.run(selected.id, cueId, options);
+ // Revise song: open the new version on its Song step and follow its re-time job there.
+ const handleSongRevised = ({ project, retimeJobId }) => {
+ openProject(project);
+ navigate(`/music-video/${project.id}/setup`);
+ if (retimeJobId) lyricAlign.attach(retimeJobId, project.id);
+ toast.success(`Created ${project.name} with the new song`);
+ };
+ const handleRetimeSong = () => lyricAlign.run(selected.id, null, { retimeSong: true })
+ .catch((err) => toast.error(err?.message || 'Could not re-time to the new song'));
// The slot is page-wide but the job belongs to one project: only that
// project's Setup shows "Aligning…".
const aligningLyrics = Boolean(lyricAlign.active && selected && lyricAlign.context?.projectId === selected.id);
@@ -1134,6 +1143,10 @@ export default function MusicVideo() {
onImportLyrics: handleImportLyrics,
onImportTrackLyrics: handleImportTrackLyrics,
onAlignLyrics: handleAlignLyrics,
+ onSongRevised: handleSongRevised,
+ onRetimeSong: handleRetimeSong,
+ retimeStatus: alignStatus,
+ onTrackImported: (track) => setTracks((prev) => [...prev, track]),
onAddScene: handleAddScene,
onDeleteScene: handleDeleteScene,
onSplitScene: handleSplitScene,
diff --git a/client/src/services/apiMusicVideo.js b/client/src/services/apiMusicVideo.js
index b9258ba3de..699f2353cf 100644
--- a/client/src/services/apiMusicVideo.js
+++ b/client/src/services/apiMusicVideo.js
@@ -542,6 +542,11 @@ export const actOnMusicVideoSongRevision = (id, action, data, options = {}) => {
return Promise.reject(new Error('Unknown song revision action'));
};
+// Revise the song from a library track (an imported Suno link): forks a new version on it and starts its re-time job.
+export const reviseMusicVideoSongFromTrack = (id, trackId, options = {}) => request(`/music-video/${encodeURIComponent(id)}/song-revision/track`, { method: 'POST', body: JSON.stringify({ trackId }), ...options });
+// Act on the shots a song revision flagged: `replan` (LLM), `remove` (cut shots) or `dismiss`.
+export const actOnMusicVideoSongRevisionScenes = (id, body, options = {}) => request(`/music-video/${encodeURIComponent(id)}/song-revision/scenes`, { method: 'POST', body: JSON.stringify(body), ...options });
+
export const importMusicVideoDocumentShots = (id, body, options = {}) => request(`/music-video/${encodeURIComponent(id)}/production-review/document-shots`, { method: 'POST', body: JSON.stringify(body), ...options });
// Source-bound private sharing export; the server rejects stale download URLs.
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index d6d78c3434..562c485566 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -40,6 +40,8 @@ export const musicVideoSongDraftSchema = z.object({
}).strict().refine((v) => v.instrumental || v.lyrics.trim().length > 0, 'Lyrics are required for a vocal song');
export const musicVideoSongActionSchema = z.object({ revisionId: z.string().min(1) }).strict();
export const musicVideoSongSelectSchema = z.object({ revisionId: z.string().min(1), songId: z.string().min(1) }).strict();
+// Revise the song from a library track (an imported Suno link): forks a new version on it.
+export const musicVideoSongFromTrackSchema = z.object({ trackId: z.string().min(1).max(128) }).strict();
import { EFFORT_LEVELS } from './providerModels.js';
import { shotActionContractProblem } from './musicVideoActionContract.js';
import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';
@@ -333,6 +335,8 @@ export const musicVideoLyricsImportTrackSchema = z.object({
export const musicVideoLyricsAlignSchema = z.object({
cueId: z.string().min(1).max(64).optional(),
separateVocals: z.boolean().optional(),
+ // A revised song: analyze, separate, align and carry the board across in one job.
+ retimeSong: z.boolean().optional(),
}).strict();
// ---- Composition manifest (#8984, part of #8966) ---------------------------
@@ -1477,6 +1481,12 @@ export const musicVideoProductionFeedbackSchema = z.object({
target: z.string().trim().min(1).max(300), text: z.string().trim().min(1).max(8000),
decision: z.enum(['comment', 'structure-accepted', 'request-changes']),
}).strict();
+// Act on the shots a song revision flagged (songRevision.js resolveSongRevisionScenes).
+export const musicVideoSongSceneActionSchema = musicVideoCastAndSetsStartSchema.extend({
+ revisionId: z.string().min(1).max(128),
+ action: z.enum(['replan', 'remove', 'dismiss']),
+ sceneIds: z.array(z.string().min(1).max(64)).min(1).max(2000).optional(),
+}).strict();
export const musicVideoProductionReviseSchema = musicVideoCastAndSetsStartSchema.extend({
stage: z.enum(['art', 'storyboard', 'proof']),
}).strict();
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index b3b6236f55..e1aa60dc72 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -317,6 +317,19 @@ router.post('/:id/song-revision/select', asyncHandler(async (req, res) => {
res.json(await selectSongRevision(req.params.id, revisionId, songId));
}));
+router.post('/:id/song-revision/track', asyncHandler(async (req, res) => {
+ const { musicVideoSongFromTrackSchema } = await import('../lib/musicVideoValidation.js');
+ const { trackId } = validateRequest(musicVideoSongFromTrackSchema, req.body);
+ const { reviseSongFromTrack } = await import('../services/musicVideo/songRevision.js');
+ res.status(201).json(await reviseSongFromTrack(req.params.id, { trackId }));
+}));
+router.post('/:id/song-revision/scenes', asyncHandler(async (req, res) => {
+ const { musicVideoSongSceneActionSchema } = await import('../lib/musicVideoValidation.js');
+ const body = validateRequest(musicVideoSongSceneActionSchema, req.body);
+ const { resolveSongRevisionScenes } = await import('../services/musicVideo/songRevision.js');
+ res.json(await resolveSongRevisionScenes(req.params.id, body));
+}));
+
router.post('/:id/clone', asyncHandler(async (req, res) => {
const options = validateRequest(musicVideoProjectCloneSchema, req.body || {});
res.status(201).json(await cloneProject(req.params.id, options));
@@ -519,8 +532,8 @@ router.post('/:id/lyrics/import-track', asyncHandler(async (req, res) => {
// provisions MMS_FA only on demand. Explicit separateVocals consent creates
// a missing stem with Demucs first; older master-only callers retain Whisper.
router.post('/:id/lyrics/align', asyncHandler(async (req, res) => {
- const { cueId, separateVocals } = validateRequest(musicVideoLyricsAlignSchema, req.body || {});
- res.status(202).json(await startLyricAlign(req.params.id, { cueId, separateVocals }));
+ const { cueId, separateVocals, retimeSong } = validateRequest(musicVideoLyricsAlignSchema, req.body || {});
+ res.status(202).json(await startLyricAlign(req.params.id, { cueId, separateVocals, retimeSong }));
}));
router.get('/lyrics/align/:jobId/events', (req, res) => {
diff --git a/server/routes/musicVideoSongRevision.browser.test.js b/server/routes/musicVideoSongRevision.browser.test.js
index b692c166e8..31bf91a3c0 100644
--- a/server/routes/musicVideoSongRevision.browser.test.js
+++ b/server/routes/musicVideoSongRevision.browser.test.js
@@ -70,7 +70,8 @@ describe.skipIf(!canRun)('song revision in Chrome (client dependencies required)
const before = await store.getProject(original.id);
let creates = 0;
let generationMode = 'success';
- songs.__setSongRevisionDepsForTests({ generate: async (_fields, opts) => {
+ // Selecting a candidate starts its re-time job; this suite covers selection only.
+ songs.__setSongRevisionDepsForTests({ startRetime: async () => null, generate: async (_fields, opts) => {
if (!opts.songIds) { creates++; await opts.onSubmitted(['candidate-a', 'candidate-b']); }
if (generationMode === 'failed') throw new Error('Synthetic download failure');
if (generationMode === 'held') {
diff --git a/server/routes/musicVideoSongRevision.test.js b/server/routes/musicVideoSongRevision.test.js
index 850885b104..4ec1948955 100644
--- a/server/routes/musicVideoSongRevision.test.js
+++ b/server/routes/musicVideoSongRevision.test.js
@@ -33,7 +33,10 @@ async function fixture() {
expect(draft.status).toBe(200);
return { before, fork, revisionId: draft.body.project.songRevision.id };
}
-afterEach(async () => { await service.__testing.settle(); service.__setSongRevisionDepsForTests(); });
+// Selecting a song starts its re-time job; these suites drive re-timing themselves.
+const startRetime = vi.fn(async () => ({ jobId: 'retime-job' }));
+const setDeps = (overrides = {}) => service.__setSongRevisionDepsForTests({ startRetime, ...overrides });
+afterEach(async () => { await service.__testing.settle(); setDeps(); startRetime.mockClear(); });
afterAll(cleanupTempDataRoots);
describe('fork song revision through HTTP', () => {
@@ -44,7 +47,7 @@ describe('fork song revision through HTTP', () => {
const songId = opts.songIds?.[0] || 'take-b';
return { songId, filename: `${songId}.m4a` };
});
- service.__setSongRevisionDepsForTests({ generate });
+ setDeps({ generate });
expect((await post(before.id, '', fields)).status).toBe(409);
expect((await post(fork.id, '', { ...fields, unknown: true })).status).toBe(400);
expect(generate).not.toHaveBeenCalled();
@@ -72,7 +75,7 @@ describe('fork song revision through HTTP', () => {
it.each(['auto', 'manual'])('rejects %s analysis completing after a new candidate is selected', async (kind) => {
const { fork, revisionId } = await fixture();
await store.mutateProjectRecord(fork.id, (p) => ({ project: { ...p, audioAnalysis: null } }));
- service.__setSongRevisionDepsForTests({ generate: async (_fields, opts) => {
+ setDeps({ generate: async (_fields, opts) => {
await opts.onSubmitted(['take-new']);
return { songId: 'take-new', filename: 'take-new.m4a' };
} });
@@ -94,7 +97,7 @@ describe('fork song revision through HTTP', () => {
const { fork, revisionId } = await fixture();
let fail = true;
let submissions = 0;
- service.__setSongRevisionDepsForTests({ generate: async (_fields, opts) => {
+ setDeps({ generate: async (_fields, opts) => {
if (!opts.songIds) { submissions++; await opts.onSubmitted(['take-a', 'take-b']); }
if (fail) throw new Error('download failed');
return { songId: opts.songIds[0], filename: `${opts.songIds[0]}.m4a` };
@@ -116,7 +119,7 @@ describe('fork song revision through HTTP', () => {
const { fork, revisionId } = await fixture();
const held = Promise.withResolvers();
const entered = Promise.withResolvers();
- service.__setSongRevisionDepsForTests({ generate: async (_fields, opts) => {
+ setDeps({ generate: async (_fields, opts) => {
await opts.onSubmitted(['take-a']); entered.resolve(); await held.promise;
return { songId: 'take-a', filename: 'take-a.m4a' };
} });
@@ -126,9 +129,81 @@ describe('fork song revision through HTTP', () => {
expect((await store.getProject(fork.id)).songRevision).toMatchObject({ status: 'canceled', candidates: [] });
const next = await post(fork.id, '', fields);
const newId = next.body.project.songRevision.id;
- service.__setSongRevisionDepsForTests({ generate: async () => { throw new Error('unknown submission'); } });
+ setDeps({ generate: async () => { throw new Error('unknown submission'); } });
await post(fork.id, 'generate', { revisionId: newId }); await service.__testing.settle();
const refused = await post(fork.id, 'generate', { revisionId: newId });
expect(refused.status).toBe(409); expect(refused.body.code || refused.body.error?.code).toBe('SONG_SUBMISSION_UNKNOWN');
});
});
+
+describe('revise the song from an imported track', () => {
+ const track = { id: 'track-new', title: 'Example song v2', prompt: 'Bright synth pop', audioFilename: 'new-take.m4a',
+ lyrics: '[Verse]\nLine a\nA brand new line\nLine b' };
+ const timed = (project, times) => project.lyricCues.map((cue, i) => ({ ...cue, startSec: times[i][0], endSec: times[i][1] }));
+ async function source() {
+ const project = await store.createProject({ name: 'Example video', uploadedAudioFilename: 'original.m4a' });
+ await store.mutateProjectRecord(project.id, (p) => ({ project: { ...p, status: 'complete', audioAnalysis: { durationSec: 30 },
+ lyricCues: [{ id: 'cue-a', text: 'Line a', startSec: 10, endSec: 14 }, { id: 'cue-x', text: 'Old middle line', startSec: 14, endSec: 18 },
+ { id: 'cue-b', text: 'Line b', startSec: 20, endSec: 24 }, { id: 'cue-y', text: 'Old last line', startSec: 26, endSec: 29 }],
+ scenes: [{ sceneId: 'intro', startSec: 0, endSec: 10, prompt: 'Opening', referenceImageId: 'intro.png' },
+ { sceneId: 'middle', startSec: 10, endSec: 18, prompt: 'Middle', lyricText: 'Line a / Old middle line' },
+ { sceneId: 'b', startSec: 18, endSec: 26, prompt: 'Line b shot', lyricText: 'Line b' },
+ { sceneId: 'tail', startSec: 26, endSec: 30, prompt: 'Tail', lyricText: 'Old last line' }] } }));
+ return store.getProject(project.id);
+ }
+
+ it('forks a new version on the track, re-times the board through shared lines and acts on flagged shots', async () => {
+ const before = await source();
+ const proposeShotRevisions = vi.fn(async (_project, notes) => new Map(notes.map((note) => [note.target, { prompt: `Replanned for ${note.target}` }])));
+ setDeps({ getTrack: async (id) => (id === track.id ? track : null), proposeShotRevisions });
+ expect((await post(before.id, 'track', { trackId: 'missing' })).status).toBe(404);
+ const created = await post(before.id, 'track', { trackId: track.id });
+ expect(created.status).toBe(201);
+ const fork = created.body.project;
+ expect(fork).toMatchObject({ parentProjectId: before.id, trackId: track.id, audioAnalysis: null });
+ expect(fork.lyricCues.map((c) => c.id)).toEqual(['cue-a', expect.any(String), 'cue-b']);
+ expect(created.body.retimeJobId).toBe('retime-job');
+ expect(startRetime).toHaveBeenCalledWith(fork.id);
+ expect(fork.songRevision).toMatchObject({ source: 'track', status: 'selected', lyricDiff: { kept: 2, changed: 0, added: 1, removed: 2 }, retime: { status: 'pending' } });
+ expect(await store.getProject(before.id)).toEqual(before);
+ const revisionId = fork.songRevision.id;
+
+ // Nothing to re-time on a version without a revised song.
+ const refused = await request(app).post(`/api/music-video/${before.id}/lyrics/align`).send({ retimeSong: true });
+ expect(refused.status).toBe(409);
+
+ // The new song sings everything 2 s later and drops the last line.
+ const retimed = await service.retimeRevisedSong(fork.id, {
+ analyze: (id) => store.mutateProjectRecord(id, (p) => ({ project: { ...p, audioAnalysis: { durationSec: 32 } } })),
+ align: (id) => store.mutateProjectRecord(id, (p) => ({ project: { ...p, lyricCues: timed(p, [[12, 16], [16, 20], [22, 26]]) } })),
+ });
+ // A fork mints its own scene ids; the shots are told apart by their prompts.
+ const ids = Object.fromEntries(fork.scenes.map((s) => [s.prompt, s.sceneId]));
+ const scene = (prompt) => retimed.scenes.find((s) => s.sceneId === ids[prompt]);
+ const status = (prompt) => retimed.songRevision.sceneReview[ids[prompt]].status;
+ expect(scene('Opening')).toMatchObject({ startSec: 0, endSec: 12, referenceImageId: 'intro.png' });
+ expect(scene('Line b shot')).toMatchObject({ startSec: 20, endSec: 28 });
+ expect(['Opening', 'Middle', 'Line b shot', 'Tail'].map(status)).toEqual(['kept', 'changed', 'kept', 'removed']);
+ expect(retimed.songRevision.retime.status).toBe('done');
+ expect(scene('Middle').lyricText).toBe('Line a / A brand new line');
+
+ const replanned = await post(fork.id, 'scenes', { revisionId, action: 'replan' });
+ expect(replanned.status).toBe(200);
+ expect(proposeShotRevisions.mock.calls[0][1].map((n) => n.target)).toEqual([ids.Middle]);
+ expect(proposeShotRevisions.mock.calls[0][0].productionReview.feedback.at(-1).text).toContain('Old middle line');
+ expect(replanned.body.project.scenes.find((s) => s.sceneId === ids.Middle).prompt).toBe(`Replanned for ${ids.Middle}`);
+ const removed = await post(fork.id, 'scenes', { revisionId, action: 'remove' });
+ expect(removed.body.project.scenes.map((s) => s.sceneId)).toEqual([ids.Opening, ids.Middle, ids['Line b shot']]);
+ expect((await post(fork.id, 'scenes', { revisionId, action: 'remove' })).status).toBe(409);
+ expect(stripMusicVideoLocalRenderPins(removed.body.project)).not.toHaveProperty('songRevision');
+ });
+
+ it('keeps the current lines when the new song came without a lyric sheet', async () => {
+ const before = await source();
+ setDeps({ getTrack: async () => ({ ...track, lyrics: '' }) });
+ const created = await post(before.id, 'track', { trackId: track.id });
+ expect(created.status).toBe(201);
+ expect(created.body.project.lyricCues.map((c) => c.id)).toEqual(['cue-a', 'cue-x', 'cue-b', 'cue-y']);
+ expect(created.body.project.songRevision.lyricDiff).toEqual({ kept: 4, changed: 0, added: 0, removed: 0 });
+ });
+});
diff --git a/server/services/musicVideo/lyricAlignJob.js b/server/services/musicVideo/lyricAlignJob.js
index b2527b2264..9db70ef6b2 100644
--- a/server/services/musicVideo/lyricAlignJob.js
+++ b/server/services/musicVideo/lyricAlignJob.js
@@ -18,6 +18,9 @@ import { ServerError } from '../../lib/errorHandler.js';
import { alignProjectLyrics } from './lyricAlign.js';
import { getProject } from './projects.js';
+const reviewableRevision = (project) => project.songRevision?.status === 'selected' && Boolean(project.songRevision.baseline);
+const retimeRevisedSongLazy = async (...args) => (await import('./songRevision.js')).retimeRevisedSong(...args);
+
// jobId -> job; projectId -> running jobId
const alignJobs = new Map();
const activeByProject = new Map();
@@ -44,14 +47,18 @@ export function cancelLyricAlign(jobId) {
* line while one runs also reuses it: the client reattaches rather than
* queueing a second whisper run over the same song.
*/
-export async function startLyricAlign(projectId, { cueId = null, separateVocals = false, align = alignProjectLyrics } = {}) {
+export async function startLyricAlign(projectId, { cueId = null, separateVocals = false, retimeSong = false, align = alignProjectLyrics, retime = null } = {}) {
const running = getActiveLyricAlignJobId(projectId);
if (running) return { jobId: running, reused: true };
// Fail with a real status before a job exists, as the blocking route did.
const project = await getProject(projectId);
if (!project) throw new ServerError('Project not found', { status: 404, code: 'NOT_FOUND' });
const cues = Array.isArray(project.lyricCues) ? project.lyricCues : [];
- if (cues.length === 0) throw new ServerError('Add lyric lines before aligning words.', { status: 400, code: 'NO_LYRICS' });
+ // A revised song re-times as a whole: analysis, alignment and the board (songRevision.js).
+ if (retimeSong && !reviewableRevision(project)) {
+ throw new ServerError('This version has no revised song to re-time.', { status: 409, code: 'SONG_REVISION_CONFLICT' });
+ }
+ if (cues.length === 0 && !retimeSong) throw new ServerError('Add lyric lines before aligning words.', { status: 400, code: 'NO_LYRICS' });
if (cueId && !cues.some((cue) => cue.id === cueId)) {
throw new ServerError('That lyric line is no longer on the project.', { status: 404, code: 'NOT_FOUND' });
}
@@ -67,7 +74,10 @@ export async function startLyricAlign(projectId, { cueId = null, separateVocals
(async () => {
try {
broadcastSse(job, { type: 'progress', stage: 'preparing' });
- const project = await align(projectId, {
+ const run = retimeSong
+ ? (id, options) => (retime || retimeRevisedSongLazy)(id, { ...options, align })
+ : align;
+ const project = await run(projectId, {
cueId,
separateVocals,
isCancelled: () => job.cancelRequested,
diff --git a/server/services/musicVideo/songRemap.js b/server/services/musicVideo/songRemap.js
new file mode 100644
index 0000000000..83edf8254f
--- /dev/null
+++ b/server/services/musicVideo/songRemap.js
@@ -0,0 +1,306 @@
+/**
+ * Revising a project's song: carry the video across to the new master.
+ *
+ * Pure helpers, no I/O. `diffLyricLines` compares the old and new lyric sheets
+ * line by line so an unchanged line keeps its cue id (storyboard shots link
+ * cues by id). Once the new song is aligned, `remapSongTimeline` moves every
+ * timed shot from the old song's timeline to the new one through the lines
+ * both songs share, and says which shots kept their lyrics, which changed,
+ * which are new and which lost every line they carried.
+ */
+
+import { randomUUID } from 'crypto';
+import { lyricTokens } from './timedText.js';
+
+const EPS = 1e-6;
+const round3 = (n) => Math.round(n * 1000) / 1000;
+const isTime = (v) => typeof v === 'number' && Number.isFinite(v);
+// A changed line still shares at least this much of its words with the line it replaces.
+const CHANGED_LINE_SIMILARITY = 0.5;
+// A shot shorter than this after an insertion is carved out of it is not worth keeping split.
+const MIN_SHOT_SEC = 0.5;
+const DEFAULT_NEW_SHOT_SEC = 8;
+const LYRIC_TEXT_MAX = 2000;
+
+const lineKey = (text) => lyricTokens(text).map((t) => t.key).join(' ');
+
+function similarity(a, b) {
+ const left = lyricTokens(a).map((t) => t.key);
+ const right = lyricTokens(b).map((t) => t.key);
+ if (!left.length || !right.length) return 0;
+ const pool = [...right];
+ let shared = 0;
+ for (const word of left) {
+ const at = pool.indexOf(word);
+ if (at >= 0) { shared += 1; pool.splice(at, 1); }
+ }
+ return (2 * shared) / (left.length + right.length);
+}
+
+/** Longest common subsequence of two key lists, as matched index pairs in order. */
+function lcsPairs(a, b) {
+ const n = a.length;
+ const m = b.length;
+ const width = m + 1;
+ const table = new Uint16Array((n + 1) * width);
+ for (let i = n - 1; i >= 0; i--) {
+ for (let j = m - 1; j >= 0; j--) {
+ table[i * width + j] = a[i] && a[i] === b[j]
+ ? table[(i + 1) * width + j + 1] + 1
+ : Math.max(table[(i + 1) * width + j], table[i * width + j + 1]);
+ }
+ }
+ const pairs = [];
+ let i = 0;
+ let j = 0;
+ while (i < n && j < m) {
+ if (a[i] && a[i] === b[j]) { pairs.push([i, j]); i++; j++; } else if (table[(i + 1) * width + j] >= table[i * width + j + 1]) i++; else j++;
+ }
+ return pairs;
+}
+
+/**
+ * Compare the project's current lyric lines with the revised song's lines.
+ * Returns the new cue list (unchanged and lightly reworded lines keep the old
+ * cue id; new lines get a fresh one), each new cue's status by id, the old
+ * lines that were cut, and the counts.
+ * @param {Array<{id:string,text:string}>} oldCues
+ * @param {Array<{text:string}>} newCues — parsed, not yet id'd
+ */
+export function diffLyricLines(oldCues, newCues) {
+ const before = (oldCues || []).filter((c) => c && typeof c.text === 'string');
+ const after = (newCues || []).filter((c) => c && typeof c.text === 'string');
+ const pairs = lcsPairs(before.map((c) => lineKey(c.text)), after.map((c) => lineKey(c.text)));
+ const oldFor = new Array(after.length).fill(null);
+ const status = new Array(after.length).fill('added');
+ const used = new Set();
+ const take = (oldIndex, newIndex, kind) => { oldFor[newIndex] = oldIndex; status[newIndex] = kind; used.add(oldIndex); };
+ for (const [i, j] of pairs) take(i, j, 'kept');
+ // Inside each gap between matched lines, pair a reworded line with the old
+ // line it replaced, in order, so its shot keeps reading as the same moment.
+ const bounds = [[-1, -1], ...pairs, [before.length, after.length]];
+ for (let k = 0; k < bounds.length - 1; k++) {
+ let cursor = bounds[k][0] + 1;
+ for (let j = bounds[k][1] + 1; j < bounds[k + 1][1]; j++) {
+ let best = -1;
+ let bestScore = CHANGED_LINE_SIMILARITY - EPS;
+ for (let i = cursor; i < bounds[k + 1][0]; i++) {
+ const score = similarity(before[i].text, after[j].text);
+ if (score > bestScore) { best = i; bestScore = score; }
+ }
+ if (best >= 0) { take(best, j, 'changed'); cursor = best + 1; }
+ }
+ }
+ const cueStatus = {};
+ const changedFrom = {};
+ const cues = after.map((cue, j) => {
+ const old = oldFor[j] == null ? null : before[oldFor[j]];
+ const id = old?.id || `lc-${randomUUID()}`;
+ cueStatus[id] = status[j];
+ if (status[j] === 'changed') changedFrom[id] = old.text;
+ const { startSec: _s, endSec: _e, words: _w, matched: _m, ...rest } = cue;
+ return { ...rest, id };
+ });
+ const removed = before.filter((_, i) => !used.has(i)).map(({ id, text, startSec, endSec }) => ({ id, text, startSec: startSec ?? null, endSec: endSec ?? null }));
+ const counts = { kept: 0, changed: 0, added: 0, removed: removed.length };
+ for (const kind of status) counts[kind] += 1;
+ return { cues, cueStatus, changedFrom, removed, counts };
+}
+
+/** The old timeline's lines and shots, captured before the new song replaces them. */
+export function songBaseline(project) {
+ const cues = (project.lyricCues || []).map(({ id, text, startSec, endSec }) => ({ id, text, startSec: startSec ?? null, endSec: endSec ?? null }));
+ const scenes = (project.scenes || []).map(({ sceneId, startSec, endSec, lyricText }) => ({ sceneId, startSec: startSec ?? null, endSec: endSec ?? null, lyricText: lyricText ?? null }));
+ const ends = [...cues, ...scenes].map((e) => e.endSec).filter(isTime);
+ const durationSec = isTime(project.audioAnalysis?.durationSec) ? project.audioAnalysis.durationSec : (ends.length ? Math.max(...ends) : null);
+ return { durationSec, cues, scenes };
+}
+
+/** Longest run of anchors rising in old time and never falling in new time (a cut line closes its gap). */
+function monotoneAnchors(anchors) {
+ const sorted = [...anchors].sort((a, b) => a.old - b.old || a.next - b.next);
+ const tails = [];
+ const prev = new Array(sorted.length).fill(-1);
+ for (let i = 0; i < sorted.length; i++) {
+ let lo = 0;
+ let hi = tails.length;
+ while (lo < hi) {
+ const mid = (lo + hi) >> 1;
+ const t = sorted[tails[mid]];
+ if (t.next <= sorted[i].next + EPS && t.old < sorted[i].old - EPS) lo = mid + 1; else hi = mid;
+ }
+ if (lo > 0) prev[i] = tails[lo - 1];
+ tails[lo] = i;
+ }
+ const out = [];
+ for (let i = tails.length ? tails[tails.length - 1] : -1; i >= 0; i = prev[i]) out.unshift(sorted[i]);
+ return out;
+}
+
+/**
+ * A monotone old-time → new-time map through the line boundaries both songs
+ * share. Outside the anchors the offset of the nearest anchor carries on.
+ */
+function buildTimeMap(anchors, { oldDurationSec = null, newDurationSec = null } = {}) {
+ const points = monotoneAnchors([{ old: 0, next: 0 }, ...anchors.filter((a) => isTime(a.old) && isTime(a.next))]);
+ if (isTime(oldDurationSec) && isTime(newDurationSec)) {
+ const last = points[points.length - 1];
+ if (oldDurationSec > last.old + EPS && newDurationSec > last.next + EPS) points.push({ old: oldDurationSec, next: newDurationSec });
+ }
+ const clamp = (t) => (isTime(newDurationSec) ? Math.min(Math.max(t, 0), newDurationSec) : Math.max(t, 0));
+ return (t) => {
+ if (!isTime(t)) return null;
+ let i = 0;
+ while (i < points.length - 1 && points[i + 1].old <= t) i++;
+ const a = points[i];
+ const b = points[i + 1];
+ if (!b || b.old - a.old < EPS) return round3(clamp(t - a.old + a.next));
+ return round3(clamp(a.next + ((t - a.old) * (b.next - a.next)) / (b.old - a.old)));
+ };
+}
+
+// Same membership rule as the shot planner (shotPlan.js lyricsFor): a line
+// belongs to a shot when half the line, or half a second of it, plays there.
+function cuesIn(cues, startSec, endSec) {
+ return cues.filter((c) => {
+ if (!isTime(c.startSec) || !isTime(c.endSec)) return false;
+ const overlap = Math.min(endSec, c.endSec) - Math.max(startSec, c.startSec);
+ return overlap > EPS && (overlap >= 0.5 || overlap >= (c.endSec - c.startSec) / 2);
+ });
+}
+
+const lyricTextOf = (cues) => (cues.length ? cues.map((c) => c.text).join(' / ').slice(0, LYRIC_TEXT_MAX) : null);
+
+/** Consecutive runs of new lines, in lyric order, as `{ cues, startSec, endSec }`. */
+function addedRuns(cues, cueStatus) {
+ const runs = [];
+ let run = null;
+ for (const cue of cues) {
+ const added = cueStatus[cue.id] === 'added' && isTime(cue.startSec) && isTime(cue.endSec);
+ if (added) {
+ if (!run) { run = { cues: [] }; runs.push(run); }
+ run.cues.push(cue);
+ } else run = null;
+ }
+ return runs.map((r) => ({ cues: r.cues, startSec: r.cues[0].startSec, endSec: r.cues[r.cues.length - 1].endSec }));
+}
+
+/** Split an inserted run into shots of at most `maxSec`, cutting on line boundaries. */
+function splitRun(run, maxSec) {
+ const shots = [];
+ let current = null;
+ for (const cue of run.cues) {
+ if (current && cue.endSec - current.startSec > maxSec) current = null;
+ if (!current) { current = { startSec: cue.startSec, endSec: cue.endSec, cues: [] }; shots.push(current); }
+ current.cues.push(cue);
+ current.endSec = cue.endSec;
+ }
+ return shots;
+}
+
+/**
+ * Move the board to the new song once its lyrics are aligned.
+ * @param {object} project — the project on the new song (aligned cues, fresh analysis)
+ * @param {object} revision — its song revision (`baseline`, `cueStatus`, `changedFrom`)
+ * @returns {{ scenes: object[], newScenes: object[], storyboard: object[]|null, sceneReview: object, counts: object }}
+ * `scenes` is the remapped existing board (same ids and order), `newScenes`
+ * the inputs for shots the inserted lines need, `sceneReview` each shot's
+ * status by id (`kept` | `changed` | `new` | `removed`).
+ */
+export function remapSongTimeline(project, revision) {
+ const baseline = revision.baseline || { cues: [], scenes: [], durationSec: null };
+ const cueStatus = revision.cueStatus || {};
+ const newCues = (project.lyricCues || []).filter((c) => isTime(c.startSec) && isTime(c.endSec));
+ const newById = new Map(newCues.map((c) => [c.id, c]));
+ const newDurationSec = isTime(project.audioAnalysis?.durationSec) ? project.audioAnalysis.durationSec : null;
+ const anchorsFor = (kinds) => baseline.cues.flatMap((old) => {
+ const next = newById.get(old.id);
+ if (!next || !kinds.includes(cueStatus[old.id]) || !isTime(old.startSec) || !isTime(old.endSec)) return [];
+ return [{ old: old.startSec, next: next.startSec }, { old: old.endSec, next: next.endSec }];
+ });
+ // A reworded line is still sung at the same moment, so it anchors too; one that moved is dropped as non-monotone.
+ const map = buildTimeMap(anchorsFor(['kept', 'changed']), { oldDurationSec: baseline.durationSec, newDurationSec });
+ const oldTimed = baseline.cues.filter((c) => isTime(c.startSec) && isTime(c.endSec));
+ const oldScene = new Map(baseline.scenes.map((s) => [s.sceneId, s]));
+
+ const sceneReview = {};
+ const touched = new Set();
+ const scenes = (project.scenes || []).map((scene) => {
+ const old = oldScene.get(scene.sceneId);
+ if (!old || !isTime(old.startSec) || !isTime(old.endSec)) return scene;
+ const startSec = map(old.startSec);
+ const endSec = Math.max(startSec, map(old.endSec));
+ const before = cuesIn(oldTimed, old.startSec, old.endSec);
+ const after = cuesIn(newCues, startSec, endSec);
+ const fates = before.map((c) => (newById.has(c.id) ? cueStatus[c.id] : 'removed'));
+ const gained = after.some((c) => cueStatus[c.id] === 'added');
+ const reworded = fates.some((f) => f !== 'kept');
+ let status = 'kept';
+ if (before.length && fates.every((f) => f === 'removed') && !after.length) status = 'removed';
+ else if (reworded || gained) status = 'changed';
+ sceneReview[scene.sceneId] = { status, previousLyricText: old.lyricText ?? null, hadRemovedLines: fates.includes('removed') };
+ if (reworded) touched.add(scene.sceneId);
+ return { ...scene, startSec, endSec, ...(status === 'changed' ? { lyricText: lyricTextOf(after) } : {}) };
+ });
+
+ // Inserted lines: a run no shot covers gets its own shots; a run a shot was
+ // only stretched across (it lost no lines) is carved out of that shot.
+ const maxSec = project.pacing?.maxShotSec || DEFAULT_NEW_SHOT_SEC;
+ const newScenes = [];
+ for (const run of addedRuns(project.lyricCues || [], cueStatus)) {
+ const covering = scenes.filter((s) => sceneReview[s.sceneId] && isTime(s.startSec) && s.endSec - run.startSec > EPS && run.endSec - s.startSec > EPS);
+ if (covering.some((s) => sceneReview[s.sceneId].hadRemovedLines)) continue; // a rewrite: its shots are re-planned
+ let gap = { startSec: run.startSec, endSec: run.endSec };
+ let carved = true;
+ for (const s of covering) {
+ const keepBefore = run.startSec - s.startSec;
+ const keepAfter = s.endSec - run.endSec;
+ // Leave the shot the side that holds its own lines; the rest goes to the insert.
+ const ownLines = cuesIn(newCues, s.startSec, s.endSec).filter((c) => cueStatus[c.id] !== 'added');
+ const linesBefore = ownLines.some((c) => c.endSec <= run.startSec + EPS);
+ const side = keepBefore >= MIN_SHOT_SEC && (linesBefore || keepAfter < MIN_SHOT_SEC) ? 'before' : keepAfter >= MIN_SHOT_SEC ? 'after' : null;
+ if (!side) { carved = false; continue; }
+ const index = scenes.indexOf(s);
+ if (side === 'before') {
+ gap = { startSec: Math.min(gap.startSec, run.startSec), endSec: Math.max(gap.endSec, s.endSec) };
+ scenes[index] = { ...s, endSec: run.startSec };
+ } else {
+ gap = { startSec: Math.min(gap.startSec, s.startSec), endSec: Math.max(gap.endSec, run.endSec) };
+ scenes[index] = { ...s, startSec: run.endSec };
+ }
+ const left = scenes[index];
+ const lines = cuesIn(newCues, left.startSec, left.endSec);
+ const status = touched.has(s.sceneId) || lines.some((c) => cueStatus[c.id] !== 'kept') ? 'changed' : 'kept';
+ sceneReview[s.sceneId] = { ...sceneReview[s.sceneId], status };
+ scenes[index] = { ...left, lyricText: status === 'changed' ? lyricTextOf(lines) : (oldScene.get(s.sceneId).lyricText ?? null) };
+ }
+ if (!carved) continue; // too short to split: the shot it sits in is marked changed above
+ const pieces = splitRun(run, maxSec);
+ pieces[0].startSec = gap.startSec;
+ pieces[pieces.length - 1].endSec = gap.endSec;
+ for (const piece of pieces) {
+ newScenes.push({ label: `New: ${piece.cues[0].text}`.slice(0, 120), startSec: round3(piece.startSec), endSec: round3(piece.endSec), lyricText: lyricTextOf(piece.cues), prompt: '' });
+ }
+ }
+
+ const counts = { kept: 0, changed: 0, new: newScenes.length, removed: 0 };
+ for (const entry of Object.values(sceneReview)) counts[entry.status] += 1;
+ return { scenes, newScenes, sceneReview, counts, map };
+}
+
+/**
+ * The saved storyboard shots on the new timeline: a shot bound to a scene
+ * takes its scene's new span, any other shot moves through the time map, and
+ * each shot's lyric cue ids are re-read from the new song.
+ */
+export function remapStoryboardShots(storyboard, { scenes, cues, map }) {
+ if (!Array.isArray(storyboard)) return storyboard;
+ const byId = new Map(scenes.map((s) => [s.sceneId, s]));
+ return storyboard.map((shot) => {
+ const scene = shot.sceneId ? byId.get(shot.sceneId) : null;
+ const startSec = scene && isTime(scene.startSec) ? scene.startSec : map(shot.startSec);
+ const endSec = scene && isTime(scene.endSec) ? scene.endSec : map(shot.endSec);
+ if (!isTime(startSec) || !isTime(endSec)) return shot;
+ return { ...shot, startSec, endSec: Math.max(startSec, endSec), lyricCueIds: cuesIn(cues, startSec, endSec).map((c) => c.id) };
+ });
+}
diff --git a/server/services/musicVideo/songRemap.test.js b/server/services/musicVideo/songRemap.test.js
new file mode 100644
index 0000000000..3d1c4e004d
--- /dev/null
+++ b/server/services/musicVideo/songRemap.test.js
@@ -0,0 +1,95 @@
+import { describe, it, expect } from 'vitest';
+import { diffLyricLines, songBaseline, remapSongTimeline, remapStoryboardShots } from './songRemap.js';
+
+const cue = (id, text, startSec = null, endSec = null) => ({ id, text, startSec, endSec });
+
+describe('diffLyricLines', () => {
+ it('keeps ids for unchanged and reworded lines, mints ids for new lines and lists cut lines', () => {
+ const before = [cue('a', 'Mountains rise'), cue('b', 'Oceans fall'), cue('c', 'We were small'), cue('d', 'Hello')];
+ const after = [{ text: 'Mountains rise' }, { text: 'Oceans fall away' }, { text: 'A brand new verse' }, { text: 'Hello' }];
+ const diff = diffLyricLines(before, after);
+ expect(diff.cues.map((c) => c.id).slice(0, 2)).toEqual(['a', 'b']);
+ expect(diff.cues[3].id).toBe('d');
+ expect(diff.cueStatus).toMatchObject({ a: 'kept', b: 'changed', d: 'kept' });
+ expect(diff.cueStatus[diff.cues[2].id]).toBe('added');
+ expect(diff.changedFrom).toEqual({ b: 'Oceans fall' });
+ expect(diff.removed.map((c) => c.id)).toEqual(['c']);
+ expect(diff.counts).toEqual({ kept: 2, changed: 1, added: 1, removed: 1 });
+ });
+
+ it('matches a repeated chorus to its own occurrence', () => {
+ const before = [cue('c1', 'At the scale'), cue('v', 'Old verse line'), cue('c2', 'At the scale')];
+ const after = [{ text: 'At the scale' }, { text: 'At the scale' }];
+ const diff = diffLyricLines(before, after);
+ expect(diff.cues.map((c) => c.id)).toEqual(['c1', 'c2']);
+ expect(diff.removed.map((c) => c.id)).toEqual(['v']);
+ });
+});
+
+describe('remapSongTimeline', () => {
+ // Old song: intro 0-10, line A 10-14, line B 14-18, instrumental 18-24, line C 24-28, cut line X 28-32, line D 32-36, end 40.
+ const oldProject = {
+ audioAnalysis: { durationSec: 40 },
+ lyricCues: [cue('A', 'line a', 10, 14), cue('B', 'line b', 14, 18), cue('C', 'line c', 24, 28), cue('X', 'cut line', 28, 32), cue('D', 'line d', 32, 36)],
+ scenes: [
+ { sceneId: 'intro', startSec: 0, endSec: 10, lyricText: null },
+ { sceneId: 'ab', startSec: 10, endSec: 18, lyricText: 'line a / line b' },
+ { sceneId: 'break', startSec: 18, endSec: 24, lyricText: null },
+ { sceneId: 'c', startSec: 24, endSec: 28, lyricText: 'line c' },
+ { sceneId: 'x', startSec: 28, endSec: 32, lyricText: 'cut line' },
+ { sceneId: 'd', startSec: 32, endSec: 40, lyricText: 'line d' },
+ ],
+ };
+
+ it('carries shared lines across, carves inserted lines into new shots and flags cut and reworded shots', () => {
+ const baseline = songBaseline(oldProject);
+ // New song: everything 2 s later, a new two-line insert sung across the old break, line B reworded, line X cut.
+ const diff = diffLyricLines(oldProject.lyricCues, [
+ { text: 'line a' }, { text: 'line b changed' }, { text: 'new one' }, { text: 'new two' }, { text: 'line c' }, { text: 'line d' },
+ ]);
+ const times = [[12, 16], [16, 20], [20, 24], [24, 28], [34, 38], [38, 42]];
+ const project = {
+ audioAnalysis: { durationSec: 46 },
+ lyricCues: diff.cues.map((c, i) => ({ ...c, startSec: times[i][0], endSec: times[i][1] })),
+ scenes: oldProject.scenes,
+ };
+ const revision = { baseline, cueStatus: diff.cueStatus };
+ const result = remapSongTimeline(project, revision);
+ const byId = Object.fromEntries(result.scenes.map((s) => [s.sceneId, s]));
+ expect(byId.intro).toMatchObject({ startSec: 0, endSec: 12 });
+ expect(result.sceneReview.intro.status).toBe('kept');
+ expect(byId.ab).toMatchObject({ startSec: 12, endSec: 20, lyricText: 'line a / line b changed' });
+ expect(result.sceneReview.ab.status).toBe('changed');
+ expect(byId.c).toMatchObject({ startSec: 34, endSec: 38 });
+ expect(result.sceneReview.c.status).toBe('kept');
+ expect(result.sceneReview.x).toMatchObject({ status: 'removed', previousLyricText: 'cut line' });
+ expect(byId.x.endSec - byId.x.startSec).toBeLessThan(0.01);
+ expect(byId.d).toMatchObject({ startSec: 38, endSec: 46 });
+ // The break was stretched over the insert; the insert becomes its own shot ahead of it.
+ expect(result.newScenes).toHaveLength(1);
+ expect(result.newScenes[0]).toMatchObject({ lyricText: 'new one / new two', startSec: 20, endSec: 28 });
+ expect(byId.break).toMatchObject({ startSec: 28, endSec: 34 });
+ expect(result.sceneReview.break.status).toBe('kept');
+ expect(result.counts).toEqual({ kept: 4, changed: 1, new: 1, removed: 1 });
+
+ const storyboard = remapStoryboardShots([
+ { id: 's1', sceneId: 'c', startSec: 24, endSec: 28, lyricCueIds: ['C'] },
+ { id: 's2', sceneId: null, startSec: 0, endSec: 10, lyricCueIds: [] },
+ ], { scenes: result.scenes, cues: project.lyricCues, map: result.map });
+ expect(storyboard[0]).toMatchObject({ startSec: 34, endSec: 38, lyricCueIds: ['C'] });
+ expect(storyboard[1]).toMatchObject({ startSec: 0, endSec: 12, lyricCueIds: [] });
+ });
+
+ it('rewrites a replaced verse into the shots that sang it instead of adding new ones', () => {
+ const baseline = songBaseline(oldProject);
+ const diff = diffLyricLines(oldProject.lyricCues, [
+ { text: 'line a' }, { text: 'line b' }, { text: 'line c' }, { text: 'totally different words here' }, { text: 'line d' },
+ ]);
+ const times = [[10, 14], [14, 18], [24, 28], [28, 32], [32, 36]];
+ const project = { audioAnalysis: { durationSec: 40 }, lyricCues: diff.cues.map((c, i) => ({ ...c, startSec: times[i][0], endSec: times[i][1] })), scenes: oldProject.scenes };
+ const result = remapSongTimeline(project, { baseline, cueStatus: diff.cueStatus });
+ expect(result.newScenes).toEqual([]);
+ expect(result.sceneReview.x.status).toBe('changed');
+ expect(result.scenes.find((s) => s.sceneId === 'x').lyricText).toBe('totally different words here');
+ });
+});
diff --git a/server/services/musicVideo/songRevision.js b/server/services/musicVideo/songRevision.js
index 49cf870454..444c14e415 100644
--- a/server/services/musicVideo/songRevision.js
+++ b/server/services/musicVideo/songRevision.js
@@ -1,14 +1,31 @@
/** Explicit, fork-scoped song revision. Never advances on boot or selects audio automatically. */
import { randomUUID } from 'crypto';
import { ServerError } from '../../lib/errorHandler.js';
-import { getProject, mutateProjectRecord } from './projects.js';
-import { applyProjectPatch } from './projectsLogic.js';
+import { getProject, mutateProjectRecord, cloneProject } from './projects.js';
+import { applyProjectPatch, addScenes } from './projectsLogic.js';
import { parseLyricCues } from './timedText.js';
import { musicVideoEvents } from './events.js';
+import { diffLyricLines, songBaseline, remapSongTimeline, remapStoryboardShots } from './songRemap.js';
const inflight = new Map();
const fail = (message, code = 'SONG_REVISION_CONFLICT') => { throw new ServerError(message, { status: 409, code }); };
-const defaults = { generate: async (...args) => (await import('./autonomousSuno.js')).generateSunoSong(...args) };
+const defaults = {
+ generate: async (...args) => (await import('./autonomousSuno.js')).generateSunoSong(...args),
+ getTrack: async (...args) => (await import('../tracks/index.js')).getTrack(...args),
+ cloneProject,
+ proposeShotRevisions: async (...args) => (await import('./planner.js')).proposeShotRevisions(...args),
+ startRetime: async (id) => (await import('./lyricAlignJob.js')).startLyricAlign(id, { retimeSong: true }),
+};
+
+// The director's choice of the new song is the request to re-time it; a start
+// that fails leaves the re-time pending for the Song step's button.
+async function startRetime(out) {
+ const started = await deps.startRetime(out.project.id).catch((err) => {
+ console.error(`❌ Song re-time did not start for ${out.project.id}: ${err.message}`);
+ return null;
+ });
+ return { ...out, retimeJobId: started?.jobId || null };
+}
let deps = defaults;
export const __setSongRevisionDepsForTests = (overrides = {}) => { deps = { ...defaults, ...overrides }; };
@@ -106,20 +123,191 @@ export async function cancelSongRevision(projectId, revisionId) {
return out;
}
+const productionActive = (project) => project.status === 'rendering' || project.autonomousRun?.status === 'running'
+ || project.productionRuns?.some((r) => r.status === 'running');
+
+/**
+ * Put a revised song on the project as its master: compare the new lyric
+ * sheet with the current lines (unchanged lines keep their cue ids), snapshot
+ * the old timeline so the board can follow the song once it is re-timed, and
+ * mark everything timed against the old audio as provisional. Returns the next
+ * project and the fields the revision records.
+ */
+function applyRevisedMaster(project, { trackId = null, uploadedAudioFilename = null, lyrics = '', instrumental = false }) {
+ const baseline = songBaseline(project);
+ // No sheet came with the new song (lyrics null): keep the current lines and re-time them on it.
+ const { cues, markers } = lyrics == null
+ ? { cues: (project.lyricCues || []).map(({ text }) => ({ text })), markers: project.lyricMarkers || [] }
+ : parseLyricCues(instrumental ? '' : lyrics);
+ const diff = diffLyricLines(project.lyricCues || [], cues);
+ const next = applyProjectPatch(project, { trackId, uploadedAudioFilename, lyricCues: diff.cues, lyricMarkers: markers, phrases: [] });
+ if (next.productionReview) next.productionReview = { ...next.productionReview,
+ draft: { ...next.productionReview.draft, timingStatus: 'provisional' }, alignmentBasis: null,
+ approvals: next.productionReview.approvals?.art ? { art: next.productionReview.approvals.art } : {}, proof: null,
+ };
+ return { project: next, revision: { baseline, cueStatus: diff.cueStatus, changedFrom: diff.changedFrom,
+ removedLines: diff.removed, lyricDiff: diff.counts, retime: { status: 'pending' }, sceneReview: null, sceneCounts: null,
+ compositionStale: Boolean(project.composition?.document) } };
+}
+
export async function selectSongRevision(projectId, revisionId, songId) {
- return mutate(projectId, (project) => {
+ return startRetime(await mutate(projectId, (project) => {
const revision = requireRevision(project, revisionId);
if (inflight.has(projectId) || !['review', 'failed'].includes(revision.status)) fail('Wait for candidate generation to settle before selecting audio');
- if (project.status === 'rendering' || project.autonomousRun?.status === 'running' || project.productionRuns?.some((r) => r.status === 'running')) fail('Stop active production before replacing the master audio');
+ if (productionActive(project)) fail('Stop active production before replacing the master audio');
const candidate = revision.candidates.find((c) => c.songId === songId);
if (!candidate) fail('Import and listen to a candidate before selecting it');
- const { cues, markers } = parseLyricCues(revision.fields.instrumental ? '' : revision.fields.lyrics);
- const next = applyProjectPatch(project, { trackId: null, uploadedAudioFilename: candidate.filename, lyricCues: cues, lyricMarkers: markers, phrases: [] });
- if (next.productionReview) next.productionReview = { ...next.productionReview,
- draft: { ...next.productionReview.draft, timingStatus: 'provisional' }, alignmentBasis: null,
- approvals: next.productionReview.approvals?.art ? { art: next.productionReview.approvals.art } : {}, proof: null,
- };
- return { project: { ...next, songRevision: { ...revision, status: 'selected', selectedSongId: songId, selectedAt: new Date().toISOString() } } };
+ const applied = applyRevisedMaster(project, { uploadedAudioFilename: candidate.filename,
+ lyrics: revision.fields.lyrics, instrumental: revision.fields.instrumental });
+ return { project: { ...applied.project, songRevision: { ...revision, ...applied.revision,
+ status: 'selected', selectedSongId: songId, selectedAt: new Date().toISOString() } } };
+ }));
+}
+
+/**
+ * Revise the song from a track already in the music library (a Suno link the
+ * director imported). Forks a new version, so the current version keeps its
+ * song and video, and puts the track on the fork as its master with its lyric
+ * sheet, then starts re-timing it. Resolves `{ project, retimeJobId }`.
+ */
+export async function reviseSongFromTrack(projectId, { trackId }) {
+ const source = await getProject(projectId);
+ if (!source) throw new ServerError('Project not found', { status: 404, code: 'NOT_FOUND' });
+ if (productionActive(source)) fail('Stop active production before revising the song');
+ const track = await deps.getTrack(trackId);
+ if (!track) throw new ServerError('Track not found', { status: 404, code: 'NOT_FOUND' });
+ if (!track.audioFilename) fail('That track has no audio yet', 'SONG_REVISION_NO_AUDIO');
+ if (track.id === source.trackId) fail('That is already this version\'s song', 'SONG_REVISION_SAME_TRACK');
+ const lyrics = typeof track.lyrics === 'string' ? track.lyrics : '';
+ const fork = await deps.cloneProject(projectId, { variant: 'revision', includeGeneratedMedia: true });
+ return startRetime(await mutate(fork.id, (project) => {
+ const applied = applyRevisedMaster(project, { trackId: track.id, lyrics: lyrics.trim() ? lyrics : null });
+ const now = new Date().toISOString();
+ return { project: { ...applied.project, songRevision: { id: `mvsr-${randomUUID()}`, source: 'track', trackId: track.id,
+ fields: { title: String(track.title || project.name || 'Song').slice(0, 80), style: String(track.prompt || '').slice(0, 1000),
+ lyrics: (lyrics.trim() ? lyrics : (project.lyricCues || []).map((c) => c.text).join('\n')).slice(0, 5000), instrumental: false },
+ status: 'selected', songIds: [], candidates: [], createdAt: now, selectedAt: now, ...applied.revision } } };
+ }));
+}
+
+const reviewable = (revision) => revision?.status === 'selected' && revision.baseline;
+
+/** Mark the re-time pass started or failed; the client shows it on the Song step. */
+async function markSongRetime(projectId, retime) {
+ const project = await getProject(projectId);
+ if (!reviewable(project?.songRevision)) return null;
+ return patch(projectId, project.songRevision.id, (r) => ({ retime: { ...r.retime, ...retime } }));
+}
+
+/**
+ * Once the new song is analyzed and its lyrics aligned, move the board onto
+ * it: shared lines carry shots across, shots whose lines changed or were cut
+ * are flagged, inserted lines get new shots. Throws when the lyrics are not
+ * timed yet. A project with no revised song to carry across is returned untouched.
+ */
+async function remapSongRevisionBoard(projectId) {
+ const current = await getProject(projectId);
+ if (!reviewable(current?.songRevision)) return current;
+ const out = await mutate(projectId, (project) => {
+ const revision = project.songRevision;
+ if (!reviewable(revision)) fail('This song revision is no longer current');
+ if (!(project.lyricCues || []).some((c) => typeof c.startSec === 'number')) fail('Align the new lyrics before re-timing the shots', 'SONG_REVISION_NOT_ALIGNED');
+ const remap = remapSongTimeline(project, revision);
+ const added = addScenes({ ...project, scenes: remap.scenes }, remap.newScenes);
+ const sceneReview = { ...remap.sceneReview };
+ for (const scene of added.scenes) sceneReview[scene.sceneId] = { status: 'new', previousLyricText: null };
+ // Board order follows the song once every shot is timed.
+ const timed = added.project.scenes.every((s) => typeof s.startSec === 'number');
+ const scenes = timed ? [...added.project.scenes].sort((a, b) => a.startSec - b.startSec || a.order - b.order).map((s, order) => ({ ...s, order })) : added.project.scenes;
+ const draft = project.productionReview?.draft;
+ const storyboard = draft?.storyboard ? remapStoryboardShots(draft.storyboard, { scenes, cues: project.lyricCues, map: remap.map }) : null;
+ return { project: { ...added.project, scenes,
+ ...(storyboard ? { productionReview: { ...project.productionReview, draft: { ...draft, storyboard } } } : {}),
+ songRevision: { ...revision, sceneReview, sceneCounts: remap.counts, retime: { status: 'done', at: new Date().toISOString() }, updatedAt: new Date().toISOString() } } };
+ });
+ return out.project;
+}
+
+/**
+ * Re-time a revised song in one pass, as the lyric-alignment job's work:
+ * analyze the new master, separate its vocal and align the lyric text on it,
+ * then carry the board across. Each step is the same one the Song step runs
+ * on its own; the director's single tap is the consent for all of them.
+ */
+export async function retimeRevisedSong(projectId, { onProgress = () => {}, isCancelled = () => false, align, analyze } = {}) {
+ const checkCancel = () => {
+ if (isCancelled()) throw Object.assign(new Error('cancelled'), { canceled: true });
+ };
+ await markSongRetime(projectId, { status: 'running', error: null });
+ try {
+ onProgress({ stage: 'analyzing' });
+ await (analyze || (async (id) => (await import('./projectAudio.js')).analyzeProjectSong(id)))(projectId);
+ checkCancel();
+ const project = await getProject(projectId);
+ if ((project?.lyricCues || []).length) await align(projectId, { separateVocals: true, onProgress, isCancelled });
+ checkCancel();
+ onProgress({ stage: 'remapping' });
+ return await remapSongRevisionBoard(projectId);
+ } catch (err) {
+ await markSongRetime(projectId, err?.canceled ? { status: 'pending', error: null } : { status: 'failed', error: String(err?.message || err).slice(0, 300) })
+ .catch((writeError) => console.error(`❌ Song re-time could not store its state: ${writeError.message}`));
+ throw err;
+ }
+}
+
+const pendingScenes = (revision, statuses) => Object.entries(revision.sceneReview || {})
+ .filter(([, entry]) => statuses.includes(entry.status) && !entry.resolved).map(([sceneId]) => sceneId);
+
+/** The note the shot planner reads for one shot whose lyrics the new song changed. */
+function replanNote(scene, entry) {
+ const now = scene.lyricText ? `"${scene.lyricText}"` : 'no lyrics (instrumental)';
+ if (entry.status === 'new') return `New shot for lines the revised song added: ${now}. Plan it to fit the shots around it.`;
+ return `The song was revised. This shot's lyrics were ${entry.previousLyricText ? `"${entry.previousLyricText}"` : 'instrumental'} and are now ${now}. Re-plan it for the new lyrics; keep what still fits.`;
+}
+
+/**
+ * Act on the shots the song revision flagged: `replan` asks the shot planner
+ * (an explicit, user-triggered LLM call) for new frame/motion prompts for the
+ * changed and new shots; `remove` deletes the shots whose lines were all cut;
+ * `dismiss` keeps the named shots as they are.
+ */
+export async function resolveSongRevisionScenes(projectId, { revisionId, action, sceneIds = null, ...route }) {
+ const project = await getProject(projectId);
+ if (!project) throw new ServerError('Project not found', { status: 404, code: 'NOT_FOUND' });
+ const revision = requireRevision(project, revisionId);
+ if (!revision.sceneReview) fail('Re-time the shots to the new song first', 'SONG_REVISION_NOT_REMAPPED');
+ const statuses = action === 'remove' ? ['removed'] : action === 'replan' ? ['changed', 'new'] : ['changed', 'new', 'removed'];
+ const targets = pendingScenes(revision, statuses).filter((id) => !sceneIds || sceneIds.includes(id));
+ if (!targets.length) fail('There are no flagged shots left for that');
+ let updates = new Map();
+ if (action === 'replan') {
+ if (productionActive(project)) fail('Stop active production before re-planning shots');
+ const notes = targets.map((sceneId) => {
+ const scene = project.scenes.find((s) => s.sceneId === sceneId);
+ return scene ? { stage: 'storyboard', target: sceneId, decision: 'request-changes', text: replanNote(scene, revision.sceneReview[sceneId]) } : null;
+ }).filter(Boolean);
+ // The planner reads open review notes from the project, so the notes ride along in memory only.
+ const planning = { ...project, productionReview: { ...project.productionReview, feedback: [...(project.productionReview?.feedback || []), ...notes] } };
+ updates = await deps.proposeShotRevisions(planning, notes, route);
+ }
+ return mutate(projectId, (current) => {
+ const live = requireRevision(current, revisionId);
+ const now = new Date().toISOString();
+ const done = new Set(action === 'replan' ? targets.filter((id) => updates.has(id)) : targets);
+ let scenes = (current.scenes || []).map((scene) => (updates.has(scene.sceneId) ? { ...scene, ...updates.get(scene.sceneId) } : scene));
+ if (action === 'remove') scenes = scenes.filter((s) => !done.has(s.sceneId)).map((s, order) => ({ ...s, order }));
+ const draft = current.productionReview?.draft;
+ const storyboard = draft?.storyboard && (action === 'remove'
+ ? draft.storyboard.filter((shot) => !done.has(shot.sceneId))
+ : draft.storyboard.map((shot) => {
+ const fields = updates.get(shot.sceneId);
+ return fields ? { ...shot, ...(fields.prompt ? { action: fields.prompt } : {}), ...(fields.framePrompt ? { staging: fields.framePrompt } : {}) } : shot;
+ }));
+ const sceneReview = Object.fromEntries(Object.entries(live.sceneReview || {}).map(([id, entry]) => [id,
+ done.has(id) ? { ...entry, resolved: action === 'dismiss' ? 'kept' : action === 'remove' ? 'removed' : 'replanned', resolvedAt: now } : entry]));
+ return { project: { ...current, scenes, updatedAt: now,
+ ...(storyboard ? { productionReview: { ...current.productionReview, draft: { ...draft, storyboard } } } : {}),
+ songRevision: { ...live, sceneReview, updatedAt: now } } };
});
}