Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video: re-align full-mix word timings on the isolated vocal when a song is prepared (#10970)

[._.]/b42965f2057b
diff --git a/server/services/musicVideo/songAutoPrepare.js b/server/services/musicVideo/songAutoPrepare.js
index b804db8403..03d663c853 100644
--- a/server/services/musicVideo/songAutoPrepare.js
+++ b/server/services/musicVideo/songAutoPrepare.js
@@ -6,11 +6,12 @@
* own actions; what follows from them is mechanical. This runs the offline
* beat/tempo/section analysis when the song has none, then starts word
* alignment (forced alignment against the isolated vocal, separating it first
- * when there is no stem yet) when the lyric lines have never been aligned.
+ * when there is no stem yet) when the lyric lines have never been aligned or
+ * were aligned only against the full mix.
*
* Analysis is a local DSP pass of a few seconds and resolves inline; alignment
* is a job, so its id comes back for the page to follow. Nothing here calls an
- * AI provider. A song whose lines already carry word times, an instrumental,
+ * AI provider. A song whose lines already carry vocal-aligned word times, an instrumental,
* and a project an autonomous run is driving are left alone.
*/
@@ -26,11 +27,17 @@ const autonomousRunActive = (project) => {
return Boolean(run) && !['completed', 'canceled', 'failed'].includes(run.status);
};
-/** Whether the project's lyric lines are waiting for their first word alignment. */
+/**
+ * Whether the project's lyric lines are waiting for word alignment on the
+ * isolated vocal: never aligned, or aligned only against the full mix (whose
+ * word times stretch over intros and dense verses).
+ */
function needsAutoAlignment(project) {
if (project?.productionReview?.draft?.lyricsMode === 'instrumental') return false;
const cues = (project?.lyricCues || []).filter((cue) => isNonBlankStr(cue?.text));
- return cues.length > 0 && !cues.some((cue) => Array.isArray(cue.words) && cue.words.length > 0);
+ if (cues.length === 0) return false;
+ const aligned = cues.some((cue) => Array.isArray(cue.words) && cue.words.length > 0);
+ return !aligned || project.lyricAlignSource !== 'vocal-stem';
}
/**
diff --git a/server/services/musicVideo/songAutoPrepare.test.js b/server/services/musicVideo/songAutoPrepare.test.js
index f224e26bfe..8d61ec8697 100644
--- a/server/services/musicVideo/songAutoPrepare.test.js
+++ b/server/services/musicVideo/songAutoPrepare.test.js
@@ -29,11 +29,20 @@ describe('automatic song preparation', () => {
put({ id: 'p', audioAnalysis: { durationSec: 30 } });
expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ analyzed: false, alignJobId: 'job-1' });
expect(analyze).not.toHaveBeenCalled();
- put({ id: 'p', audioAnalysis: { durationSec: 30 }, lyricCues: [{ id: 'c', text: 'hi', words }, { id: 'd', text: 'added later' }] });
+ put({ id: 'p', audioAnalysis: { durationSec: 30 }, lyricAlignSource: 'vocal-stem', lyricCues: [{ id: 'c', text: 'hi', words }, { id: 'd', text: 'added later' }] });
expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ alignJobId: null });
expect(align).toHaveBeenCalledTimes(1);
});
+ it('re-aligns words that were placed against the full mix on the isolated vocal', async () => {
+ for (const lyricAlignSource of ['master', undefined]) {
+ put({ id: 'p', audioAnalysis: { durationSec: 30 }, lyricAlignSource, lyricCues: [{ id: 'c', text: 'hi', words }] });
+ expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ alignJobId: 'job-1' });
+ }
+ expect(align).toHaveBeenCalledTimes(2);
+ expect(align).toHaveBeenLastCalledWith('p', { separateVocals: true });
+ });
+
it('does nothing without audio, for an instrumental, or while an autonomous run drives the project', async () => {
put({ id: 'p', trackId: null });
expect(await autoPrepareSong('p', { analyze, align })).toMatchObject({ analyzed: false, alignJobId: null });