Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video: analyze and align a new song automatically, and check lyric timing on a playthrough (#10968)

[._.]/b664ef57d0a8
diff --git a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx b/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
index c9ea8b9e2d..bdb26ad691 100644
--- a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
+++ b/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
@@ -1,9 +1,18 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { Pause, Play } from 'lucide-react';
-import { fetchMusicVideoPreviewAsset, getMusicVideoCompositionPreview } from '../../services/apiMusicVideo.js';
+import { fetchMusicVideoPreviewAsset, getMusicVideoCompositionPreview, getMusicVideoLyricPlaythroughPreview } from '../../services/apiMusicVideo.js';
const buttonCls = 'flex items-center gap-1 bg-port-bg border border-port-border rounded px-2 py-1.5 text-sm min-h-[44px] sm:min-h-0 disabled:opacity-50';
+// What the lyric playthrough page is built from: the song length, the word times,
+// the sheet's sections and the shots' timing and text zones. A change rebuilds it.
+const playthroughVersion = (project) => JSON.stringify([
+ project.audioAnalysis?.durationSec ?? null,
+ (project.lyricCues || []).map((cue) => [cue.text, cue.startSec, cue.endSec, (cue.words || []).map((w) => [w.startSec, w.endSec])]),
+ (project.lyricMarkers || []).map((m) => [m.type, m.label, m.line]),
+ (project.scenes || []).map((s) => [s.startSec, s.endSec, s.textZone, s.lyricRole]),
+]);
+
/** The lyric line sung at song time `t`, from the project's timed cues. */
const lyricAt = (cues, t) => {
const line = (cues || []).find((cue) => typeof cue.startSec === 'number' && typeof cue.endSec === 'number' && t >= cue.startSec && t < cue.endSec);
@@ -45,10 +54,13 @@ const remember = (cache, url, blob) => {
* time still applies. `collapsed` hides the picture (mini-player) below `lg`
* while the transport stays usable; the iframe stays mounted so the loaded
* media and playhead survive.
+ *
+ * `lyrics` plays the lyric timing playthrough instead of the project's document:
+ * the aligned words over a plain frame, for checking timing before any picture.
*/
-export default function CompositionPreviewPlayer({ project, audioUrl, seekRequest = null, collapsed = false, draft = false }) {
- const doc = (draft ? project.composition?.documentDraft : project.composition?.document) || null;
- const scrubId = draft ? 'mv-doc-draft-scrub' : 'mv-doc-scrub';
+export default function CompositionPreviewPlayer({ project, audioUrl, seekRequest = null, collapsed = false, draft = false, lyrics = false, scrubId: givenScrubId = null }) {
+ const doc = lyrics ? { playthrough: true } : (draft ? project.composition?.documentDraft : project.composition?.document) || null;
+ const scrubId = givenScrubId || (lyrics ? 'mv-lyrics-scrub' : draft ? 'mv-doc-draft-scrub' : 'mv-doc-scrub');
const [preview, setPreview] = useState(null);
const [previewError, setPreviewError] = useState('');
const [status, setStatus] = useState('');
@@ -73,7 +85,7 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
// The document's own version (its folder + save time), not the project's
// `updatedAt`: an unrelated save must not reload a playing preview.
- const refresh = doc ? `${doc.directory}|${doc.updatedAt || ''}` : null;
+ const refresh = lyrics ? playthroughVersion(project) : doc ? `${doc.directory}|${doc.updatedAt || ''}` : null;
useEffect(() => {
let active = true;
setPreview(null);
@@ -81,11 +93,11 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
setStatus('');
seekState.current = { inFlight: false, pending: null, ready: false };
if (!refresh || !wanted) return () => { active = false; };
- getMusicVideoCompositionPreview(project.id, { silent: true, draft })
+ (lyrics ? getMusicVideoLyricPlaythroughPreview(project.id, { silent: true }) : getMusicVideoCompositionPreview(project.id, { silent: true, draft }))
.then((next) => { if (active) setPreview(next); })
.catch((err) => { if (active) setPreviewError(err?.message || 'Could not build the preview'); });
return () => { active = false; };
- }, [project.id, refresh, draft, wanted]);
+ }, [project.id, refresh, draft, lyrics, wanted]);
const postSeek = useCallback((time) => {
const frame = iframeRef.current?.contentWindow;
@@ -202,13 +214,13 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
const aspect = preview?.width && preview?.height ? `${preview.width} / ${preview.height}` : '16 / 9';
const lyric = lyricAt(project.lyricCues, t);
return (
- <div className="space-y-2" aria-label="Composition preview">
+ <div className="space-y-2" aria-label={lyrics ? 'Lyric timing playthrough' : 'Composition preview'}>
{previewError && <p className="text-xs text-port-error" role="alert">{previewError}</p>}
{status && <p className="text-xs text-port-text-muted">{status}</p>}
<div className={`overflow-hidden rounded border border-port-border bg-black mx-auto ${collapsed ? 'max-xl:hidden' : ''}`}
style={{ aspectRatio: aspect, maxHeight: '70vh', maxWidth: '100%' }}>
{preview?.html ? (
- <iframe ref={iframeRef} title={draft ? 'Composition candidate preview' : 'Composition document preview'} sandbox="allow-scripts" srcDoc={preview.html} className="h-full w-full" />
+ <iframe ref={iframeRef} title={lyrics ? 'Lyric timing playthrough' : draft ? 'Composition candidate preview' : 'Composition document preview'} sandbox="allow-scripts" srcDoc={preview.html} className="h-full w-full" />
) : (
<p className="p-3 text-xs text-port-text-muted">{previewError ? '' : 'Building the preview…'}</p>
)}
@@ -217,13 +229,13 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
<button type="button" className={buttonCls} onClick={togglePlay} disabled={!preview || !audioUrl || (!!previewError && !playing)}>
{playing ? <Pause size={14} /> : <Play size={14} />} {playing ? 'Pause' : 'Play'}
</button>
- <label htmlFor={scrubId} className="sr-only">{draft ? 'Scrub the composition candidate' : 'Scrub the composition preview'}</label>
+ <label htmlFor={scrubId} className="sr-only">{lyrics ? 'Scrub the lyric playthrough' : draft ? 'Scrub the composition candidate' : 'Scrub the composition preview'}</label>
<input disabled={!preview || !!previewError} id={scrubId} type="range" min={0} max={duration || 0} step={1 / fps} value={Math.min(t, duration || 0)}
onChange={(e) => { audioRef.current?.pause(); setPlaying(false); seek(Number(e.target.value)); }}
className="min-w-0 flex-1" />
<span className="text-xs text-port-text-muted tabular-nums">{previewError ? 'Preview unavailable' : preview ? `${t.toFixed(2)}s / ${duration.toFixed(1)}s` : !wanted ? 'Expand to load preview' : 'Loading preview…'}</span>
</div>
- {lyric && <p className="text-xs italic break-words" aria-live="off" data-testid="preview-lyric">♪ {lyric}</p>}
+ {lyric && !lyrics && <p className="text-xs italic break-words" aria-live="off" data-testid="preview-lyric">♪ {lyric}</p>}
{audioUrl && <audio ref={audioRef} src={audioUrl} preload="none" className="hidden" onEnded={() => setPlaying(false)} />}
</div>
);
diff --git a/client/src/components/musicVideo/LyricTimingCheck.jsx b/client/src/components/musicVideo/LyricTimingCheck.jsx
index 82e1d56c1a..5eb05fd676 100644
--- a/client/src/components/musicVideo/LyricTimingCheck.jsx
+++ b/client/src/components/musicVideo/LyricTimingCheck.jsx
@@ -1,5 +1,6 @@
import { useState } from 'react';
import { lyricSetupState } from '../../lib/musicVideoStages.js';
+import CompositionPreviewPlayer from './CompositionPreviewPlayer.jsx';
const EMPTY_DRAFT = { cast: '', environments: '', visualLanguage: '', motionLanguage: '', guideArtifactId: null,
lyricsMode: 'vocal', timingStatus: 'provisional', timingNotes: '', storyboard: [] };
@@ -14,8 +15,12 @@ const buttonClass = 'min-h-[44px] rounded border border-port-border bg-port-bg p
* changed) is re-verified through the server's alignment route. Notes are
* optional. `planning` is the page's unsaved planning draft pair; its copy of these
* fields is kept in step so a later planning save cannot undo this one.
+ *
+ * Timing is judged by watching, so a vocal song's check carries the lyric
+ * playthrough: the aligned words drawn with the render's own type over a plain
+ * frame, played with the song (`audioUrl`). `aligning` says alignment is running.
*/
-export default function LyricTimingCheck({ project, review, planning = null, disabled = false }) {
+export default function LyricTimingCheck({ project, review, planning = null, disabled = false, audioUrl = null, aligning = false }) {
const saved = { ...EMPTY_DRAFT, ...(project.productionReview?.draft || {}) };
const lyrics = lyricSetupState(project, review.readiness);
const [mode, setMode] = useState(saved.lyricsMode === 'instrumental' ? 'instrumental' : 'vocal');
@@ -36,6 +41,8 @@ export default function LyricTimingCheck({ project, review, planning = null, dis
};
const verified = mode === 'vocal' ? lyrics.verified && !lyrics.instrumental : lyrics.instrumental && lyrics.verified;
+ const timed = (project.lyricCues || []).some((cue) => typeof cue.startSec === 'number' && cue.text?.trim());
+ const playable = mode === 'vocal' && timed && !!project.audioAnalysis && !!audioUrl && !aligning;
return (
<div className="space-y-2" id="mv-lyric-timing">
<fieldset className="flex flex-wrap gap-3 text-sm">
@@ -55,6 +62,19 @@ export default function LyricTimingCheck({ project, review, planning = null, dis
</p>
) : (
<>
+ {mode === 'vocal' && !playable && (
+ <p role="status" className="text-sm text-port-text-muted">
+ {aligning ? 'Aligning the words to the vocal. The playthrough appears here when they are placed.'
+ : !project.audioAnalysis ? 'Analyzing the song. The playthrough appears here once the words are placed.'
+ : 'Align the words to play them through with the song.'}
+ </p>
+ )}
+ {playable && (
+ <>
+ <p className="text-sm text-port-text-muted">Play the lyrics through with the song and watch each word land on the vocal.</p>
+ <CompositionPreviewPlayer project={project} audioUrl={audioUrl} lyrics scrubId={`mv-lyrics-check-scrub-${project.id}`} />
+ </>
+ )}
<label htmlFor={notesId} className="block text-xs text-port-text-muted">
Notes (optional)
</label>
@@ -65,9 +85,9 @@ export default function LyricTimingCheck({ project, review, planning = null, dis
)}
{mode === 'vocal' ? (
<button type="button" className={buttonClass}
- disabled={busy || lyrics.lines === 0}
+ disabled={busy || lyrics.lines === 0 || !playable}
onClick={() => (stale ? reverify() : write({ lyricsMode: 'vocal', timingStatus: 'verified', timingNotes: notes.trim() }))}>
- {stale ? 'Re-verify timing' : 'Mark timing verified'}
+ {stale ? 'Timing still looks right' : 'Timing looks right'}
</button>
) : (
<button type="button" className={buttonClass} disabled={busy}
diff --git a/client/src/components/musicVideo/PreviewDock.jsx b/client/src/components/musicVideo/PreviewDock.jsx
index 7e685f3858..a7c13c49e5 100644
--- a/client/src/components/musicVideo/PreviewDock.jsx
+++ b/client/src/components/musicVideo/PreviewDock.jsx
@@ -92,11 +92,14 @@ export default function PreviewDock({ project, sources, audioUrl, seekRequest, c
</div>
{!source && (
<p className={`text-xs text-port-text-muted ${collapsed ? 'max-xl:hidden' : ''}`}>
- Attach a track and plan the shots to watch a storyboard animatic here; drafts and the final render join it as you make them.
+ {project.trackId || project.uploadedAudioFilename
+ ? 'The lyric timing playthrough plays here once the song is analyzed and its words are aligned; the storyboard, drafts and final render join it as you make them.'
+ : 'Attach a track to watch the lyric timing playthrough here; the storyboard, drafts and final render join it as you make them.'}
</p>
)}
{/* Folded below xl, the dock is one row: the player and its controls wait for the expand. */}
{source?.kind === 'document' && <div className={collapsed ? 'max-xl:hidden' : ''}><CompositionPreviewPlayer project={project} audioUrl={audioUrl} seekRequest={seekRequest} collapsed={collapsed} /></div>}
+ {source?.kind === 'lyrics' && <div className={collapsed ? 'max-xl:hidden' : ''}><CompositionPreviewPlayer project={project} audioUrl={audioUrl} seekRequest={seekRequest} collapsed={collapsed} lyrics /></div>}
{source?.kind === 'animatic' && <StoryboardAnimatic project={project} audioUrl={audioUrl} seekRequest={seekRequest} collapsed={collapsed} />}
{source?.kind === 'video' && <VideoPreview key={source.id} source={source} seekRequest={seekRequest} collapsed={collapsed} />}
</aside>
diff --git a/client/src/components/musicVideo/TrackPanel.jsx b/client/src/components/musicVideo/TrackPanel.jsx
index 4984835ca4..1130242eda 100644
--- a/client/src/components/musicVideo/TrackPanel.jsx
+++ b/client/src/components/musicVideo/TrackPanel.jsx
@@ -93,7 +93,7 @@ export default function TrackPanel({
{pending && (
<div role="alertdialog" aria-label="Confirm track change" className="mt-2 rounded border border-port-warning/50 bg-port-warning/10 p-2 text-xs space-y-2">
<p>
- {pending.kind === 'youtube' ? 'Importing new audio' : 'Changing the track'} will clear: {clearLabels.join(', ')}. Lyric text and scenes are kept. You will need to re-run Analyze and Align words.
+ {pending.kind === 'youtube' ? 'Importing new audio' : 'Changing the track'} will clear: {clearLabels.join(', ')}. Lyric text and scenes are kept. The new song is analyzed and the words re-aligned automatically.
</p>
<div className="flex flex-wrap gap-2">
<button type="button" onClick={() => runChange(pending, {})}
diff --git a/client/src/components/musicVideo/stages/SetupStage.jsx b/client/src/components/musicVideo/stages/SetupStage.jsx
index f1a99e8b0c..cc762dd07f 100644
--- a/client/src/components/musicVideo/stages/SetupStage.jsx
+++ b/client/src/components/musicVideo/stages/SetupStage.jsx
@@ -29,12 +29,14 @@ function AlignWords({ disabled, aligning, status, aligned, onAlign }) {
const [error, setError] = useState('');
const run = () => {
setError('');
- Promise.resolve(onAlign()).catch((err) => setError(err?.message || 'Could not align the words to the vocal. Try again.'));
+ // The isolated vocal is the default alignment source; a missing stem is separated first.
+ Promise.resolve(onAlign(null, { separateVocals: true })).catch((err) => setError(err?.message || 'Could not align the words to the vocal. Try again.'));
};
return (
<>
<p className="text-xs text-port-text-muted">
- {aligned ? 'Every word is placed. Listen back, fine-tune any line in step 3, then verify.' : 'Place each word on the vocal, then listen back.'}
+ {aligned ? 'Every word is placed. Watch the playthrough below, fine-tune any line in step 3, then confirm.'
+ : 'The words are placed on the isolated vocal on their own once the song has lyrics.'}
</p>
<button type="button" onClick={run} disabled={aligning || disabled}
className="min-h-[44px] rounded border border-port-border bg-port-bg px-2 py-1.5 text-sm text-port-accent disabled:opacity-50 sm:min-h-0">
@@ -129,7 +131,8 @@ export default function SetupStage({ board }) {
/>
)}
{board.productionReview && (
- <LyricTimingCheck project={project} review={board.productionReview} planning={board.planningDraft} disabled={locked} />
+ <LyricTimingCheck project={project} review={board.productionReview} planning={board.planningDraft} disabled={locked}
+ audioUrl={board.audioUrl} aligning={aligningLyrics} />
)}
</SongStep>
</div>
diff --git a/client/src/components/musicVideo/stages/SetupStage.test.jsx b/client/src/components/musicVideo/stages/SetupStage.test.jsx
index e4bb5cf8c2..fcc21608f2 100644
--- a/client/src/components/musicVideo/stages/SetupStage.test.jsx
+++ b/client/src/components/musicVideo/stages/SetupStage.test.jsx
@@ -4,10 +4,11 @@ import { fireEvent, render, screen, within } from '@testing-library/react';
import SetupStage from './SetupStage.jsx';
vi.mock('../../songs/MidiVisualization.jsx', () => ({ default: () => null }));
+vi.mock('../CompositionPreviewPlayer.jsx', () => ({ default: ({ lyrics }) => <div data-testid="playthrough">{lyrics ? 'lyrics' : 'document'}</div> }));
const review = (over = {}) => ({ readiness: null, current: true, busy: false, error: null, save: vi.fn(async () => ({})), reverifyAlignment: vi.fn(async () => ({})), ...over });
const board = (project, extra = {}) => ({
- project, tracks: [], trackName: () => 'Song', tempo: { bpm: '', setBpm: vi.fn() }, audioFilename: 'a.wav', locked: false,
+ project, tracks: [], trackName: () => 'Song', tempo: { bpm: '', setBpm: vi.fn() }, audioFilename: 'a.wav', audioUrl: '/data/music/a.wav', locked: false,
youtube: { editJob: {}, editUrl: '', setEditUrl: vi.fn(), startEdit: vi.fn() },
midi: { model: 'base', active: false }, busy: {},
onAnalyze: vi.fn(), onAlignLyrics: vi.fn(), ...extra,
@@ -40,20 +41,31 @@ describe('Song step', () => {
it('verifies the timing on the step itself, keeping an unsaved planning draft in step', async () => {
const save = vi.fn(async () => ({}));
const setPlanning = vi.fn();
- const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi', words: [{ w: 'hi' }] }],
+ const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi', startSec: 1, endSec: 2, words: [{ w: 'hi' }] }],
productionReview: { draft: { cast: 'kept', lyricsMode: 'vocal', timingStatus: 'provisional', timingNotes: '' } } };
render(<SetupStage board={board(project, { productionReview: review({ save }), planningDraft: [{ cast: 'edited', lyricsMode: 'vocal' }, setPlanning] })} />);
fireEvent.change(screen.getByLabelText('Notes (optional)'), { target: { value: 'Listened through twice' } });
- await fireEvent.click(screen.getByRole('button', { name: 'Mark timing verified' }));
+ // The timing is judged on the lyric playthrough shown beside the button.
+ expect(screen.getByTestId('playthrough').textContent).toBe('lyrics');
+ await fireEvent.click(screen.getByRole('button', { name: 'Timing looks right' }));
expect(save).toHaveBeenCalledWith(expect.objectContaining({ cast: 'kept', lyricsMode: 'vocal', timingStatus: 'verified', timingNotes: 'Listened through twice' }));
await vi.waitFor(() => expect(setPlanning).toHaveBeenCalledWith({ cast: 'edited', lyricsMode: 'vocal', timingStatus: 'verified', timingNotes: 'Listened through twice' }));
});
+ it('holds the timing check until the aligned words can be played through', () => {
+ const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi', startSec: 1, endSec: 2 }],
+ productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'provisional' } } };
+ render(<SetupStage board={board(project, { productionReview: review(), aligningLyrics: true })} />);
+ expect(screen.queryByTestId('playthrough')).toBeNull();
+ expect(screen.getByText(/Aligning the words to the vocal/)).toBeTruthy();
+ expect(screen.getByRole('button', { name: 'Timing looks right' })).toBeDisabled();
+ });
+
it('re-verifies a stale timing through the alignment route without requiring a note', () => {
const reverifyAlignment = vi.fn(async () => ({}));
- const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi' }], productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'verified' } } };
+ const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi', startSec: 1, endSec: 2 }], productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'verified' } } };
render(<SetupStage board={board(project, { productionReview: review({ reverifyAlignment, readiness: { alignment: { status: 'stale' } } }) })} />);
- const button = screen.getByRole('button', { name: 'Re-verify timing' });
+ const button = screen.getByRole('button', { name: 'Timing still looks right' });
expect(button).toBeEnabled();
fireEvent.change(screen.getByLabelText('Notes (optional)'), { target: { value: 'New master checked' } });
fireEvent.click(button);
@@ -62,13 +74,13 @@ describe('Song step', () => {
it('keeps an unsaved planning draft in step after a re-verify', async () => {
const setPlanning = vi.fn();
- const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi' }], productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'verified', timingNotes: 'old' } } };
+ const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi', startSec: 1, endSec: 2 }], productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'verified', timingNotes: 'old' } } };
render(<SetupStage board={board(project, {
productionReview: review({ readiness: { alignment: { status: 'stale' } } }),
planningDraft: [{ cast: 'edited', timingStatus: 'verified', timingNotes: 'old' }, setPlanning],
})} />);
fireEvent.change(screen.getByLabelText('Notes (optional)'), { target: { value: 'New master checked' } });
- fireEvent.click(screen.getByRole('button', { name: 'Re-verify timing' }));
+ fireEvent.click(screen.getByRole('button', { name: 'Timing still looks right' }));
await vi.waitFor(() => expect(setPlanning).toHaveBeenCalledWith({ cast: 'edited', timingStatus: 'verified', timingNotes: 'New master checked' }));
});
diff --git a/client/src/lib/musicVideoStages.js b/client/src/lib/musicVideoStages.js
index 3926d1dfa9..74866aa444 100644
--- a/client/src/lib/musicVideoStages.js
+++ b/client/src/lib/musicVideoStages.js
@@ -146,7 +146,8 @@ export function boardJobEstimate(project) {
* old final render is not what the user sees by default. A final render whose
* inputs changed is labelled out of date. Last comes the storyboard animatic —
* the song under each shot's frame or card — so a project with a track and
- * shots always has something to watch. Each entry has a stable `id` (the
+ * shots always has something to watch — and after it the lyric timing
+ * playthrough once the words are aligned. Each entry has a stable `id` (the
* `?play=` value) and a `label` for the source picker.
*/
export function listPreviewSources(project, { finalVideoSrc = null, liveFirst = false, draftsFirst = false, storyboardFirst = false } = {}) {
@@ -172,6 +173,11 @@ export function listPreviewSources(project, { finalVideoSrc = null, liveFirst =
if (storyboardFirst) sources.splice(live ? 1 : 0, 0, animatic);
else sources.push(animatic);
}
+ // The aligned words over a plain frame, for checking timing before there is any picture.
+ if (projectHasAudio(project) && project.audioAnalysis
+ && (project.lyricCues || []).some((cue) => typeof cue.startSec === 'number' && isNonBlankStr(cue.text))) {
+ sources.push({ id: 'lyrics', kind: 'lyrics', label: 'Lyric timing' });
+ }
return sources;
}
@@ -616,7 +622,7 @@ export function stageChecklist(stageId, project, readiness = project?.production
{ id: 'timing', label: lyrics.instrumental ? 'Instrumental confirmed' : 'Lyric timing verified', done: lyrics.verified || lyrics.ok,
detail: lyrics.alignment === 'stale' ? 'Word timings or the master changed since you verified them; verify again.'
: lyrics.instrumental ? 'Confirm the song is instrumental under Time and verify.'
- : 'Align the words, listen back, then mark the timing verified under Time and verify.',
+ : 'Watch the lyric playthrough under Time and verify, then confirm the timing looks right.',
action: autoSong ? null : { label: 'Verify timing', anchor: 'mv-lyric-timing' } },
];
}
diff --git a/client/src/pages/MusicVideo.jsx b/client/src/pages/MusicVideo.jsx
index 3454b57fb6..d405ea0a92 100644
--- a/client/src/pages/MusicVideo.jsx
+++ b/client/src/pages/MusicVideo.jsx
@@ -19,6 +19,7 @@ import {
updateMusicVideoProject,
deleteMusicVideoProject,
analyzeMusicVideoProject,
+ prepareMusicVideoSong,
planMusicVideoProject,
addMusicVideoScene,
updateMusicVideoScene,
@@ -349,7 +350,7 @@ export default function MusicVideo() {
trackId: track.id,
...(!f.name || tracks.some((t) => t.title === f.name) ? { name: track.title || '' } : {}),
})),
- onProjectUpdated: replaceProject,
+ onProjectUpdated: (proj) => { replaceProject(proj); prepareSong(proj.id); },
});
// "Separate vocals" (demucs) — one slot shared by the stem control and the
// autopilot kickoff; the terminal frame carries the project with its stem.
@@ -359,6 +360,18 @@ export default function MusicVideo() {
const lyricAlign = useMusicVideoLyricAlign({
onAligned: (projectId, project) => patchProject(projectId, { lyricCues: project.lyricCues, audioAnalysis: project.audioAnalysis, updatedAt: project.updatedAt }),
});
+ // A track attached or lyrics imported: analyze the song and align never-aligned
+ // lines on the server, then follow the alignment here (the Song step shows it).
+ const prepareSong = (projectId) => {
+ setAnalyzing(true);
+ return prepareMusicVideoSong(projectId, { silent: true })
+ .then(({ project, analyzed, alignJobId }) => {
+ if (analyzed) patchProject(projectId, { audioAnalysis: project.audioAnalysis, updatedAt: project.updatedAt });
+ if (alignJobId) lyricAlign.attach(alignJobId, projectId);
+ })
+ .catch((err) => toast.error(err?.message || 'Could not analyze the song'))
+ .finally(() => setAnalyzing(false));
+ };
const midi = useMusicVideoMidiJob({
onTranscribed: (projectId, midiTranscription) => patchProject(projectId, { midiTranscription }),
});
@@ -517,6 +530,7 @@ export default function MusicVideo() {
setForm(emptyCreateForm());
setCreateOpen(false);
toast.success('Project created');
+ if (proj.trackId) prepareSong(proj.id);
})
.catch((err) => toast.error(err?.message || 'Failed to create project'))
.finally(() => setCreating(false));
@@ -701,7 +715,7 @@ export default function MusicVideo() {
// Re-point the selected project at a different library track (the detail
// view's "Change track" picker — previously there was no way to relink a
// project's audio after creation at all).
- const handleChangeTrack = (trackId, { fork = false, cleared = [] } = {}) => {
+ const handleChangeTrack = (trackId, { fork = false } = {}) => {
if (!selected) return;
if (renderTargetsSelected) {
toast.error('Wait for the current render to finish before changing the track');
@@ -727,7 +741,7 @@ export default function MusicVideo() {
openProject(proj);
navigate(`/music-video/${proj.id}/setup`);
} else replaceProject(proj);
- if (cleared.length > 0) toast.success('Track changed — re-run Analyze and Align words');
+ prepareSong(proj.id);
})
.catch((err) => toast.error(err?.message || 'Failed to change track'));
};
@@ -785,6 +799,7 @@ export default function MusicVideo() {
patchProject(projectId, { lyricCues: project.lyricCues, updatedAt: project.updatedAt });
onDone?.();
toast.success(`Imported ${imported} lyric line${imported === 1 ? '' : 's'} (${format})`);
+ prepareSong(projectId);
})
.catch((err) => toast.error(err?.message || 'Lyric import failed'))
.finally(() => setImportingLyrics(false));
@@ -797,12 +812,13 @@ export default function MusicVideo() {
.then(({ project, imported, markers }) => {
patchProject(projectId, { lyricCues: project.lyricCues, lyricMarkers: project.lyricMarkers, updatedAt: project.updatedAt });
toast.success(`Imported ${imported} lyric line${imported === 1 ? '' : 's'}${markers ? ` and ${markers} section/direction marker${markers === 1 ? '' : 's'}` : ''} from the track`);
+ prepareSong(projectId);
})
.catch((err) => toast.error(err?.message || 'Could not import the track lyrics'))
.finally(() => setImportingLyrics(false));
};
- // Alignment is a click, never an import side effect. The panel shows the
- // whisper setup error itself, so this request stays silent.
+ // A re-align is a click (imports align through prepareSong). The panel shows
+ // the alignment setup error itself, so this request stays silent.
const handleAlignLyrics = (cueId, options) => lyricAlign.run(selected.id, cueId, options);
// Revise song: open the new version on its Song step and follow its re-time job there.
const handleSongRevised = ({ project, retimeJobId }) => {
diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsx
index ee5faefd47..6d2664d52a 100644
--- a/client/src/pages/MusicVideo.test.jsx
+++ b/client/src/pages/MusicVideo.test.jsx
@@ -63,6 +63,8 @@ vi.mock('../services/apiMusicVideo.js', () => ({
})),
deleteMusicVideoProject: vi.fn(),
analyzeMusicVideoProject: vi.fn(),
+ prepareMusicVideoSong: vi.fn((id) => Promise.resolve({ project: { id }, analyzed: false, alignJobId: null })),
+ getMusicVideoLyricPlaythroughPreview: vi.fn(() => new Promise(() => {})),
planMusicVideoProject: vi.fn(),
addMusicVideoScene: vi.fn(),
updateMusicVideoScene: vi.fn(),
@@ -264,7 +266,7 @@ vi.mock('../components/PageHeader', () => ({ default: ({ title, actions }) => <d
import MusicVideo from './MusicVideo.jsx';
import {
listMusicVideoProjects, listMusicVideoProjectSummaries, createMusicVideoProject, cloneMusicVideoProject, renderMusicVideoProject, planMusicVideoProject, updateMusicVideoProject,
- deleteMusicVideoProject, transcribeMusicVideoMidi, cancelMusicVideoRender, analyzeMusicVideoProject,
+ deleteMusicVideoProject, transcribeMusicVideoMidi, cancelMusicVideoRender, analyzeMusicVideoProject, prepareMusicVideoSong,
importMusicVideoLyrics, importMusicVideoTrackLyrics, separateMusicVideoVocals, alignMusicVideoLyrics, updateMusicVideoScene, splitMusicVideoScene,
selectMusicVideoSceneTake, reviewMusicVideoSceneTake, importMusicVideoHandoff,
addMusicVideoSceneTake, getMusicVideoHandoffBundle,
@@ -1630,7 +1632,19 @@ describe('MusicVideo lyrics and shot coverage (#8964)', () => {
));
});
- it('aligns words only after the button click and shows the returned timings', async () => {
+ it('aligns imported lyrics on its own and follows the alignment it started', async () => {
+ const cues = [{ id: 'lc-1', text: 'first line', startSec: null, endSec: null }];
+ importMusicVideoLyrics.mockResolvedValue({ project: { ...PROJECT_ANALYZED, lyricCues: cues }, imported: 1, format: 'text' });
+ prepareMusicVideoSong.mockResolvedValueOnce({ project: { ...PROJECT_ANALYZED, lyricCues: cues }, analyzed: false, alignJobId: 'align-auto' });
+ await openProject(PROJECT_ANALYZED, 'setup');
+ fireEvent.change(screen.getByLabelText('Lyrics to import'), { target: { value: 'first line' } });
+ fireEvent.click(screen.getByRole('button', { name: /^Import lyrics$/ }));
+ await waitFor(() => expect(prepareMusicVideoSong).toHaveBeenCalledWith('mv-3', { silent: true }));
+ expect((await screen.findAllByRole('button', { name: 'Aligning…' })).length).toBeGreaterThan(0);
+ expect(alignMusicVideoLyrics).not.toHaveBeenCalled();
+ });
+
+ it('re-aligns words from the button and shows the returned timings', async () => {
const cues = [{
id: 'lc-1', text: 'walking home', startSec: 0.5, endSec: 1.5,
words: [
@@ -2104,7 +2118,7 @@ describe('MusicVideo YouTube audio import (#1945)', () => {
prompt: 'Cyberpunk neon aesthetics',
durationSec: 180,
}]);
- createMusicVideoProject.mockResolvedValue({ ...PROJECT_NO_CLIP, id: 'mv-cool', name: 'Neon Horizon' });
+ createMusicVideoProject.mockResolvedValue({ ...PROJECT_NO_CLIP, id: 'mv-cool', name: 'Neon Horizon', trackId: 'track-cool' });
renderMV();
await openCreateForm();
@@ -2125,6 +2139,8 @@ describe('MusicVideo YouTube audio import (#1945)', () => {
name: 'Neon Horizon',
trackId: 'track-cool',
});
+ // The new song is analyzed and its lyrics aligned without another click.
+ await waitFor(() => expect(prepareMusicVideoSong).toHaveBeenCalledWith('mv-cool', { silent: true }));
});
it('changing a track in the edit view auto-seeds track concept and style if not set', async () => {
diff --git a/client/src/services/apiMusicVideo.js b/client/src/services/apiMusicVideo.js
index 3265154397..570224f5d2 100644
--- a/client/src/services/apiMusicVideo.js
+++ b/client/src/services/apiMusicVideo.js
@@ -91,6 +91,13 @@ export const importMusicVideoTrackLyrics = (id, body = {}, options = {}) => requ
export const alignMusicVideoLyrics = (id, body = {}, options = {}) => request(`/music-video/${encodeURIComponent(id)}/lyrics/align`, {
method: 'POST', body: JSON.stringify(body), ...options,
});
+// Analyze a newly attached song and start aligning never-aligned lyrics.
+// Resolves `{ project, analyzed, alignJobId }`; follow the job with the align events URL.
+export const prepareMusicVideoSong = (id, options = {}) => request(`/music-video/${encodeURIComponent(id)}/song/prepare`, {
+ method: 'POST', ...options,
+});
+// The lyric timing playthrough page: the aligned words over a plain frame (same shape as the composition preview).
+export const getMusicVideoLyricPlaythroughPreview = (id, options = {}) => request(`/music-video/${encodeURIComponent(id)}/lyrics/playthrough/preview`, options);
export const musicVideoLyricAlignEventsUrl = (jobId) =>
`/api/music-video/lyrics/align/${encodeURIComponent(jobId)}/events`;
export const cancelMusicVideoLyricAlign = (jobId, options = {}) =>
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index 58134de9c2..a098be1dc3 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -527,9 +527,9 @@ router.post('/:id/lyrics/import-track', asyncHandler(async (req, res) => {
res.json(await importTrackLyrics(req.params.id, { mode }));
}));
-// Word-level alignment (#9074) as a job (#10155). Nothing here runs until the
-// director clicks Align words or a line's Re-align (or starts an autopilot
-// run). Kickoff returns 202 + a jobId; stages (model download %, decoding,
+// Word-level alignment (#9074) as a job (#10155). It runs when the director
+// clicks Align words or a line's Re-align, when an autopilot run starts, and
+// when the song is prepared after a track or lyrics arrive (song/prepare below). Kickoff returns 202 + a jobId; stages (model download %, decoding,
// transcribing window n/m) stream over SSE with cancel, and the terminal
// `complete` frame carries the updated project. One job per project: a second
// request returns the running job (`reused: true`). The first alignment
@@ -540,6 +540,21 @@ router.post('/:id/lyrics/align', asyncHandler(async (req, res) => {
res.status(202).json(await startLyricAlign(req.params.id, { cueId, separateVocals, retimeSong }));
}));
+// After a track is attached or lyrics are imported (the director's own action),
+// analyze the song if it has no analysis and start word alignment when the
+// lines were never aligned. Returns the project and the alignment job to follow.
+router.post('/:id/song/prepare', asyncHandler(async (req, res) => {
+ const { autoPrepareSong } = await import('../services/musicVideo/songAutoPrepare.js');
+ res.json(await autoPrepareSong(req.params.id));
+}));
+
+// The lyric timing playthrough: the aligned words drawn by the shipped template over a
+// plain frame, laid out as scenes, for checking word timing before any picture exists.
+router.get('/:id/lyrics/playthrough/preview', asyncHandler(async (req, res) => {
+ const { buildLyricPlaythroughPreview } = await import('../services/musicVideo/lyricPlaythrough.js');
+ res.json(await buildLyricPlaythroughPreview(await requireProject(req.params.id)));
+}));
+
router.get('/lyrics/align/:jobId/events', (req, res) => {
if (!attachLyricAlignSseClient(req.params.jobId, res)) {
throw new ServerError('Alignment job not found or expired', { status: 404, code: 'NOT_FOUND' });
diff --git a/server/services/musicVideo/compositionDocument.js b/server/services/musicVideo/compositionDocument.js
index 04a9afc506..5537fb0678 100644
--- a/server/services/musicVideo/compositionDocument.js
+++ b/server/services/musicVideo/compositionDocument.js
@@ -356,6 +356,17 @@ export async function importDocumentTemplate(projectId, templateId = 'layered')
return storeVersion(projectId, files, { kind: 'template', name: templateId });
}
+/**
+ * A shipped template's files as `readDocumentFiles` returns them, read in place
+ * with the shared lyric-type module beside them, for a preview no project owns
+ * (the lyric timing playthrough). Nothing is copied into the project.
+ */
+export async function readTemplateDocumentFiles(templateId = 'layered') {
+ if (!MUSIC_VIDEO_DOCUMENT_TEMPLATES.includes(templateId)) throw refuse('Unknown composition template', 'VALIDATION_ERROR', 400);
+ const { files } = await collectTree(join(TEMPLATE_ROOT, templateId), { excludeTests: true });
+ return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));
+}
+
/** Stage a host-assembled generated document for review before selection. */
export async function stageGeneratedDocument(projectId, generatedFiles, { verifyCurrent, renderer = 'canvas' } = {}) {
if (renderer === 'three') {
diff --git a/server/services/musicVideo/documentPreview.js b/server/services/musicVideo/documentPreview.js
index 328bd56163..b0d215d832 100644
--- a/server/services/musicVideo/documentPreview.js
+++ b/server/services/musicVideo/documentPreview.js
@@ -209,8 +209,9 @@ const BOOTSTRAP = `(() => {
* The preview page and the assets to post into it:
* `{ html, assets: [{ key, url, bytes }], width, height, fps, durationSec }`.
*/
-export async function buildDocumentPreview(project, { draft = false } = {}) {
- const files = await readDocumentFiles(project);
+export async function buildDocumentPreview(project, { draft = false, files: givenFiles = null } = {}) {
+ // `files` stands in a shipped template for the project's own document (it has no file route).
+ const files = givenFiles || await readDocumentFiles(project);
const songDurationSec = documentSongDuration(project) || 0;
const clock = documentRenderClock(songDurationSec);
const frame = DOCUMENT_FRAME_SIZES[documentAspect(project)];
@@ -290,7 +291,7 @@ export async function buildDocumentPreview(project, { draft = false } = {}) {
const name = m.path.split(/[\\/]/).pop();
assets.push({ key: scene.media.src, url: `/data/${m.kind === 'video' ? 'videos' : 'images'}/${encodeURIComponent(name)}`, bytes: null });
}
- for (const [rel, file] of files) {
+ for (const [rel, file] of givenFiles ? [] : files) {
// Media is posted even when CSS or an <img> tag already inlined it as data:, because document code
// can also reach the same file through a relative src or a bundler URL (new URL(…, import.meta.url)).
if (!BRIDGED.has(extname(rel).toLowerCase())) continue;
diff --git a/server/services/musicVideo/lyricPlaythrough.js b/server/services/musicVideo/lyricPlaythrough.js
new file mode 100644
index 0000000000..1ddb98873f
--- /dev/null
+++ b/server/services/musicVideo/lyricPlaythrough.js
@@ -0,0 +1,82 @@
+/**
+ * Music Video — the lyric timing playthrough.
+ *
+ * Word timing is checked by watching the words land on the song, which used to
+ * wait for a first composition. This preview needs only the song, its analysis
+ * and the aligned lyrics: the shipped layered template draws the words with the
+ * shared lyricType.js (the same type and motion the final render uses) over a
+ * plain frame, with no images or footage.
+ *
+ * Its scenes are the board's own timing when the board has timed shots (so each
+ * line sits in the text zone its shot will give it), else the lyrics laid out
+ * as scenes: one per lyric-sheet section, or per four lines when the sheet has
+ * no headers. The layout is built for the preview only; the board is untouched.
+ */
+
+import { ServerError } from '../../lib/errorHandler.js';
+import { isNonBlankStr } from '../../lib/textUtils.js';
+import { buildDocumentPreview } from './documentPreview.js';
+import { readTemplateDocumentFiles } from './compositionDocument.js';
+import { documentSongDuration } from './documentRender.js';
+
+const LINES_PER_SCENE = 4;
+const finite = (n) => typeof n === 'number' && Number.isFinite(n);
+// The fields the template reads from a shot; no take, frame or clip comes along.
+const SCENE_FIELDS = ['sceneId', 'order', 'label', 'sectionLabel', 'startSec', 'endSec', 'shotMode', 'textZone', 'lyricRole', 'lyricText', 'camera', 'stillMove'];
+
+const timedShots = (scenes) => (Array.isArray(scenes) ? scenes : [])
+ .filter((scene) => scene?.sceneId && finite(scene.startSec) && finite(scene.endSec) && scene.endSec > scene.startSec)
+ .map((scene) => Object.fromEntries(SCENE_FIELDS.filter((key) => scene[key] !== undefined).map((key) => [key, scene[key]])));
+
+/**
+ * The lyrics laid out as scenes, `[{ sceneId, order, label, sectionLabel, startSec, endSec }]`.
+ * A group starts at each section header (or every four lines without headers); each
+ * scene runs from its first timed line to the next scene, the first from 0 and the
+ * last to the end of the song.
+ */
+function lyricScenes(cues, markers, durationSec) {
+ const lines = Array.isArray(cues) ? cues : [];
+ const sections = (Array.isArray(markers) ? markers : [])
+ .filter((marker) => marker?.type === 'section' && Number.isInteger(marker.line) && marker.line < lines.length)
+ .sort((a, b) => a.line - b.line);
+ const groups = [];
+ if (sections.length) {
+ if (sections[0].line > 0) groups.push({ from: 0, label: 'Intro' });
+ for (const marker of sections) {
+ if (groups.at(-1)?.from === marker.line) groups.at(-1).label = marker.label;
+ else groups.push({ from: marker.line, label: marker.label });
+ }
+ } else {
+ for (let from = 0; from < lines.length; from += LINES_PER_SCENE) {
+ groups.push({ from, label: `Lines ${from + 1}–${Math.min(from + LINES_PER_SCENE, lines.length)}` });
+ }
+ }
+ const starts = groups.map((group, index) => {
+ const until = groups[index + 1]?.from ?? lines.length;
+ const first = lines.slice(group.from, until).find((cue) => finite(cue?.startSec) && isNonBlankStr(cue?.text));
+ return first ? { label: group.label, startSec: first.startSec } : null;
+ }).filter(Boolean);
+ return starts.map((scene, index) => {
+ const startSec = index === 0 ? 0 : scene.startSec;
+ const endSec = starts[index + 1]?.startSec ?? durationSec;
+ return { sceneId: `lyric-${index + 1}`, order: index, label: scene.label, sectionLabel: scene.label, startSec, endSec };
+ }).filter((scene) => scene.endSec > scene.startSec);
+}
+
+/**
+ * The preview page for the playthrough, in the shape `buildDocumentPreview`
+ * returns. 409 until the song is analyzed and at least one line is timed.
+ */
+export async function buildLyricPlaythroughPreview(project) {
+ const durationSec = documentSongDuration(project);
+ if (!durationSec) throw new ServerError('Analyze the song before playing the lyrics through', { status: 409, code: 'NOT_ANALYZED' });
+ const cues = project.lyricCues || [];
+ if (!cues.some((cue) => finite(cue?.startSec) && isNonBlankStr(cue?.text))) {
+ throw new ServerError('Align the words before playing the lyrics through', { status: 409, code: 'NO_TIMED_LYRICS' });
+ }
+ const shots = timedShots(project.scenes);
+ const scenes = shots.length ? shots : lyricScenes(cues, project.lyricMarkers, durationSec);
+ // Only the song and the words: no composition, overlay text or narrative events of the project's own.
+ const playthrough = { ...project, scenes, composition: { mode: 'document', textCues: [] } };
+ return buildDocumentPreview(playthrough, { files: await readTemplateDocumentFiles('layered') });
+}
diff --git a/server/services/musicVideo/lyricPlaythrough.test.js b/server/services/musicVideo/lyricPlaythrough.test.js
new file mode 100644
index 0000000000..d2cc2b1e9a
--- /dev/null
+++ b/server/services/musicVideo/lyricPlaythrough.test.js
@@ -0,0 +1,81 @@
+import { afterAll, describe, expect, it, vi } from 'vitest';
+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
+
+vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await importOriginal(), {
+ dataRoot: () => lazyTempDataRoot('portos-mv-lyric-playthrough-'),
+}));
+
+const { buildLyricPlaythroughPreview } = await import('./lyricPlaythrough.js');
+
+afterAll(() => cleanupTempDataRoots());
+
+const cue = (text, startSec, endSec = startSec == null ? null : startSec + 2) => ({ id: `c-${text}`, text, startSec, endSec });
+const portosData = (html) => JSON.parse(/window\.PORTOS_MV = (\{.*?\});window\.PORTOS_MV_EVENT_STATE/s.exec(html)[1]);
+// The scenes the playthrough lays out for these lyrics, as [label, startSec, endSec].
+const lyricScenes = async (lyricCues, lyricMarkers, durationSec) => portosData((await buildLyricPlaythroughPreview({
+ id: 'mv-example', name: 'Example Song', audioAnalysis: { durationSec }, lyricCues, lyricMarkers, scenes: [],
+})).html).scenes.map(({ label, startSec, endSec }) => [label, startSec, endSec]);
+
+describe('lyric scenes', () => {
+ it('opens a scene at each sheet section, from 0 to the end of the song', async () => {
+ const cues = [cue('a', 5), cue('b', 8), cue('c', 20), cue('d', 24), cue('e', 40)];
+ const markers = [
+ { type: 'section', label: 'Verse 1', line: 1 },
+ { type: 'direction', label: 'Whispered', line: 2 },
+ { type: 'section', label: 'Chorus', line: 2 },
+ { type: 'section', label: 'Outro', line: 5 },
+ ];
+ expect(await lyricScenes(cues, markers, 60)).toEqual([
+ ['Intro', 0, 8], ['Verse 1', 8, 20], ['Chorus', 20, 60],
+ ]);
+ });
+
+ it('groups four lines a scene without headers and skips a group with no timed line', async () => {
+ const cues = [cue('a', 2), cue('b', 4), cue('c', 6), cue('d', 8), cue('e', null), cue('f', null), cue('g', 30)];
+ expect(await lyricScenes(cues, [], 50)).toEqual([
+ ['Lines 1–4', 0, 30], ['Lines 5–7', 30, 50],
+ ]);
+ });
+});
+
+describe('lyric timing playthrough', () => {
+ const project = {
+ id: 'mv-example', name: 'Example Song', audioAnalysis: { durationSec: 30, bpm: 120, sections: [] },
+ lyricCues: [
+ { ...cue('hello there', 2), words: [{ text: 'hello', startSec: 2, endSec: 2.5 }, { text: 'there', startSec: 2.6, endSec: 3.4 }] },
+ cue('second line', 10),
+ ],
+ lyricMarkers: [{ type: 'section', label: 'Chorus', line: 1 }],
+ scenes: [],
+ composition: { mode: 'document', textCues: [{ text: 'Overlay', startSec: 0, endSec: 5 }], overlay: { enabled: true } },
+ };
+
+ it('needs the analysis and at least one timed line', async () => {
+ await expect(buildLyricPlaythroughPreview({ ...project, audioAnalysis: null })).rejects.toMatchObject({ status: 409, code: 'NOT_ANALYZED' });
+ await expect(buildLyricPlaythroughPreview({ ...project, lyricCues: [cue('untimed', null)] })).rejects.toMatchObject({ status: 409, code: 'NO_TIMED_LYRICS' });
+ });
+
+ it('draws the aligned words with the shipped template over the lyrics laid out as scenes, with no project document', async () => {
+ const preview = await buildLyricPlaythroughPreview(project);
+ const data = portosData(preview.html);
+ expect(data.scenes.map(({ label, startSec, endSec, media }) => [label, startSec, endSec, media])).toEqual([
+ ['Intro', 0, 10, null], ['Chorus', 10, 30, null],
+ ]);
+ expect(data.lyrics.map((line) => line.text)).toEqual(['hello there', 'second line']);
+ // Only the words: no overlay text or HUD from the project's own composition.
+ expect(data.textCues).toEqual([]);
+ expect(data.composition.overlay).toBeNull();
+ expect(preview.html).toContain('data:text/javascript;base64,');
+ expect(preview.assets).toEqual([]);
+ expect(preview.durationSec).toBeGreaterThan(29);
+ });
+
+ it('follows the board\'s timed shots, without their takes, once there are any', async () => {
+ const scenes = [
+ { sceneId: 's1', order: 0, label: 'Wide', startSec: 0, endSec: 12, textZone: 'upper-right', referenceImageId: 'frame.png', videoHistoryId: 'v1' },
+ { sceneId: 's2', order: 1, label: 'Close', startSec: 12, endSec: 30, textZone: 'lower' },
+ ];
+ const data = portosData((await buildLyricPlaythroughPreview({ ...project, scenes })).html);
+ expect(data.scenes.map(({ sceneId, textZone, media }) => [sceneId, textZone, media])).toEqual([['s1', 'upper-right', null], ['s2', 'lower', null]]);
+ });
+});
diff --git a/server/services/musicVideo/songAutoPrepare.js b/server/services/musicVideo/songAutoPrepare.js
new file mode 100644
index 0000000000..b804db8403
--- /dev/null
+++ b/server/services/musicVideo/songAutoPrepare.js
@@ -0,0 +1,56 @@
+/**
+ * Music Video — get a newly attached song ready without a click.
+ *
+ * Creating a project with a track (an imported Suno song, the default case),
+ * attaching or changing its track, and importing its lyrics are the director's
+ * own actions; what follows from them is mechanical. This runs the offline
+ * beat/tempo/section analysis when the song has none, then starts word
+ * alignment (forced alignment against the isolated vocal, separating it first
+ * when there is no stem yet) when the lyric lines have never been aligned.
+ *
+ * Analysis is a local DSP pass of a few seconds and resolves inline; alignment
+ * is a job, so its id comes back for the page to follow. Nothing here calls an
+ * AI provider. A song whose lines already carry word times, an instrumental,
+ * and a project an autonomous run is driving are left alone.
+ */
+
+import { ServerError } from '../../lib/errorHandler.js';
+import { isNonBlankStr } from '../../lib/textUtils.js';
+import { getProject } from './projects.js';
+import { analyzeProjectSong } from './projectAudio.js';
+import { startLyricAlign } from './lyricAlignJob.js';
+
+const hasAudio = (project) => Boolean(project?.trackId || project?.uploadedAudioFilename);
+const autonomousRunActive = (project) => {
+ const run = project?.autonomousRun;
+ return Boolean(run) && !['completed', 'canceled', 'failed'].includes(run.status);
+};
+
+/** Whether the project's lyric lines are waiting for their first word alignment. */
+function needsAutoAlignment(project) {
+ if (project?.productionReview?.draft?.lyricsMode === 'instrumental') return false;
+ const cues = (project?.lyricCues || []).filter((cue) => isNonBlankStr(cue?.text));
+ return cues.length > 0 && !cues.some((cue) => Array.isArray(cue.words) && cue.words.length > 0);
+}
+
+/**
+ * Analyze and align what is missing. Resolves `{ project, analyzed, alignJobId }`:
+ * the current project, whether analysis ran, and the alignment job to follow
+ * (null when none was needed).
+ */
+export async function autoPrepareSong(projectId, { analyze = analyzeProjectSong, align = startLyricAlign } = {}) {
+ let project = await getProject(projectId);
+ if (!project) throw new ServerError('Project not found', { status: 404, code: 'NOT_FOUND' });
+ if (!hasAudio(project) || autonomousRunActive(project)) return { project, analyzed: false, alignJobId: null };
+ let analyzed = false;
+ if (!project.audioAnalysis) {
+ project = await analyze(project.id);
+ analyzed = true;
+ }
+ let alignJobId = null;
+ if (needsAutoAlignment(project)) {
+ ({ jobId: alignJobId } = await align(project.id, { separateVocals: true }));
+ console.log(`🎤 Started automatic lyric alignment for ${project.id}`);
+ }
+ return { project, analyzed, alignJobId };
+}
diff --git a/server/services/musicVideo/songAutoPrepare.test.js b/server/services/musicVideo/songAutoPrepare.test.js
new file mode 100644
index 0000000000..f224e26bfe
--- /dev/null
+++ b/server/services/musicVideo/songAutoPrepare.test.js
@@ -0,0 +1,51 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const store = new Map();
+vi.mock('./projects.js', () => ({ getProject: vi.fn(async (id) => store.get(id) || null) }));
+
+const { autoPrepareSong } = await import('./songAutoPrepare.js');
+
+const words = [{ text: 'hi', startSec: 1, endSec: 1.4 }];
+let analyze;
+let align;
+const put = (project) => store.set(project.id, { id: 'p', trackId: 't', lyricCues: [{ id: 'c', text: 'hi' }], ...project });
+
+beforeEach(() => {
+ store.clear();
+ analyze = vi.fn(async (id) => ({ ...store.get(id), audioAnalysis: { durationSec: 30 } }));
+ align = vi.fn(async () => ({ jobId: 'job-1' }));
+});
+
+describe('automatic song preparation', () => {
+ it('analyzes a new song, then aligns its never-aligned lyrics on the isolated vocal', async () => {
+ put({ id: 'p' });
+ const result = await autoPrepareSong('p', { analyze, align });
+ expect(analyze).toHaveBeenCalledWith('p');
+ expect(align).toHaveBeenCalledWith('p', { separateVocals: true });
+ expect(result).toMatchObject({ analyzed: true, alignJobId: 'job-1', project: { audioAnalysis: { durationSec: 30 } } });
+ });
+
+ it('only aligns when the song is already analyzed, and leaves aligned words alone', async () => {
+ put({ id: 'p', audioAnalysis: { durationSec: 30 } });
+ expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ analyzed: false, alignJobId: 'job-1' });
+ expect(analyze).not.toHaveBeenCalled();
+ put({ id: 'p', audioAnalysis: { durationSec: 30 }, lyricCues: [{ id: 'c', text: 'hi', words }, { id: 'd', text: 'added later' }] });
+ expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ alignJobId: null });
+ expect(align).toHaveBeenCalledTimes(1);
+ });
+
+ it('does nothing without audio, for an instrumental, or while an autonomous run drives the project', async () => {
+ put({ id: 'p', trackId: null });
+ expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ analyzed: false, alignJobId: null });
+ put({ id: 'p', productionReview: { draft: { lyricsMode: 'instrumental' } } });
+ expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ analyzed: true, alignJobId: null });
+ put({ id: 'p', autonomousRun: { status: 'running' } });
+ expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ analyzed: false, alignJobId: null });
+ expect(analyze).toHaveBeenCalledTimes(1);
+ expect(align).not.toHaveBeenCalled();
+ });
+
+ it('answers 404 for a missing project', async () => {
+ await expect(autoPrepareSong('nope', { analyze, align })).rejects.toMatchObject({ status: 404 });
+ });
+});