Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: repair compressed lyric lines and flag unresolved timing (#10973)

Adam Eivyae2392b47c20
diff --git a/client/src/components/musicVideo/LyricTimingCheck.jsx b/client/src/components/musicVideo/LyricTimingCheck.jsx
index 5eb05fd676..60565517e1 100644
--- a/client/src/components/musicVideo/LyricTimingCheck.jsx
+++ b/client/src/components/musicVideo/LyricTimingCheck.jsx
@@ -1,4 +1,5 @@
import { useState } from 'react';
+import { formatTimecode } from '../../utils/formatters.js';
import { lyricSetupState } from '../../lib/musicVideoStages.js';
import CompositionPreviewPlayer from './CompositionPreviewPlayer.jsx';
@@ -24,6 +25,7 @@ export default function LyricTimingCheck({ project, review, planning = null, dis
const saved = { ...EMPTY_DRAFT, ...(project.productionReview?.draft || {}) };
const lyrics = lyricSetupState(project, review.readiness);
const [mode, setMode] = useState(saved.lyricsMode === 'instrumental' ? 'instrumental' : 'vocal');
+ const [seekRequest, setSeekRequest] = useState(null);
const [notes, setNotes] = useState(saved.timingNotes || '');
const stale = lyrics.alignment === 'stale';
const busy = disabled || review.busy || !review.current;
@@ -42,6 +44,7 @@ export default function LyricTimingCheck({ project, review, planning = null, dis
const verified = mode === 'vocal' ? lyrics.verified && !lyrics.instrumental : lyrics.instrumental && lyrics.verified;
const timed = (project.lyricCues || []).some((cue) => typeof cue.startSec === 'number' && cue.text?.trim());
+ const suspectCues = (project.lyricCues || []).filter((cue) => cue.suspect);
const playable = mode === 'vocal' && timed && !!project.audioAnalysis && !!audioUrl && !aligning;
return (
<div className="space-y-2" id="mv-lyric-timing">
@@ -56,6 +59,26 @@ export default function LyricTimingCheck({ project, review, planning = null, dis
Instrumental
</label>
</fieldset>
+ {playable && (!verified || suspectCues.length > 0) && (
+ <>
+ <p className="text-sm text-port-text-muted">Play the lyrics through with the song and watch each word land on the vocal.</p>
+ {suspectCues.length > 0 && (
+ <div className="space-y-1 text-sm">
+ <p className="text-port-warning">These lines may play too quickly. Jump to each one and check it against the vocal.</p>
+ <ul className="space-y-1">
+ {suspectCues.map((cue) => {
+ const t = cue.words?.[0]?.startSec ?? cue.startSec ?? 0;
+ return <li key={cue.id}><button type="button" className={buttonClass}
+ onClick={() => setSeekRequest((previous) => ({ t, n: (previous?.n ?? 0) + 1, play: true }))}>
+ {formatTimecode(t)} — {cue.text}
+ </button></li>;
+ })}
+ </ul>
+ </div>
+ )}
+ <CompositionPreviewPlayer project={project} audioUrl={audioUrl} lyrics scrubId={`mv-lyrics-check-scrub-${project.id}`} seekRequest={seekRequest} />
+ </>
+ )}
{verified ? (
<p role="status" className="text-sm text-port-success">
{mode === 'vocal' ? 'Word timing verified against the current master.' : 'Marked instrumental.'}
@@ -69,12 +92,6 @@ export default function LyricTimingCheck({ project, review, planning = null, dis
: 'Align the words to play them through with the song.'}
</p>
)}
- {playable && (
- <>
- <p className="text-sm text-port-text-muted">Play the lyrics through with the song and watch each word land on the vocal.</p>
- <CompositionPreviewPlayer project={project} audioUrl={audioUrl} lyrics scrubId={`mv-lyrics-check-scrub-${project.id}`} />
- </>
- )}
<label htmlFor={notesId} className="block text-xs text-port-text-muted">
Notes (optional)
</label>
diff --git a/client/src/components/musicVideo/stages/SetupStage.test.jsx b/client/src/components/musicVideo/stages/SetupStage.test.jsx
index fcc21608f2..284a1bdcea 100644
--- a/client/src/components/musicVideo/stages/SetupStage.test.jsx
+++ b/client/src/components/musicVideo/stages/SetupStage.test.jsx
@@ -4,7 +4,7 @@ import { fireEvent, render, screen, within } from '@testing-library/react';
import SetupStage from './SetupStage.jsx';
vi.mock('../../songs/MidiVisualization.jsx', () => ({ default: () => null }));
-vi.mock('../CompositionPreviewPlayer.jsx', () => ({ default: ({ lyrics }) => <div data-testid="playthrough">{lyrics ? 'lyrics' : 'document'}</div> }));
+vi.mock('../CompositionPreviewPlayer.jsx', () => ({ default: ({ lyrics, seekRequest }) => <div data-testid="playthrough" data-seek={JSON.stringify(seekRequest)}>{lyrics ? 'lyrics' : 'document'}</div> }));
const review = (over = {}) => ({ readiness: null, current: true, busy: false, error: null, save: vi.fn(async () => ({})), reverifyAlignment: vi.fn(async () => ({})), ...over });
const board = (project, extra = {}) => ({
@@ -52,6 +52,21 @@ describe('Song step', () => {
await vi.waitFor(() => expect(setPlanning).toHaveBeenCalledWith({ cast: 'edited', lyricsMode: 'vocal', timingStatus: 'verified', timingNotes: 'Listened through twice' }));
});
+ it('lists suspect lyric text and timecodes and jumps the playthrough to that line on every click', () => {
+ const project = { id: 'p', trackId: 't', audioAnalysis: {}, productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'verified' } }, lyricCues: [
+ { id: 'c', text: 'under blue skies', startSec: 60, endSec: 65, suspect: true,
+ words: [{ w: 'under', startSec: 61.2, endSec: 61.3 }] },
+ { id: 'd', text: 'walking home', startSec: 70, endSec: 72 },
+ ] };
+ render(<SetupStage board={board(project, { productionReview: review() })} />);
+ const jump = screen.getByRole('button', { name: '1:01.20 — under blue skies' });
+ expect(screen.queryByRole('button', { name: /walking home/ })).toBeNull();
+ fireEvent.click(jump);
+ expect(JSON.parse(screen.getByTestId('playthrough').dataset.seek)).toEqual({ t: 61.2, n: 1, play: true });
+ fireEvent.click(jump);
+ expect(JSON.parse(screen.getByTestId('playthrough').dataset.seek)).toEqual({ t: 61.2, n: 2, play: true });
+ });
+
it('holds the timing check until the aligned words can be played through', () => {
const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi', startSec: 1, endSec: 2 }],
productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'provisional' } } };
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index c43979098d..d1f033aad4 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -179,6 +179,7 @@ export const musicVideoLyricCueSchema = z.object({
endSec: timedSec,
words: z.array(lyricWordSchema).max(400).optional(),
matched: z.number().min(0).max(1).optional(),
+ suspect: z.boolean().optional(),
}).strict();
// A musical-phrase annotation: a span of the song with an optional visual
diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.js
index e96e46a774..79280407b3 100644
--- a/server/lib/schemaVersions.js
+++ b/server/lib/schemaVersions.js
@@ -683,7 +683,9 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({
// v19 pins whole-workflow media policy; older peers cannot enforce it.
// v20 adds the Eidoverse render mode and scene source. Older normalizers
// would collapse the mode to footage and discard the authored scene.
- musicVideoProjects: 20,
+ // v21 adds compressed lyric timing warnings (`lyricCues[].suspect`). Older
+ // normalizers drop the warning on edits. Absent remains legacy; no backfill.
+ musicVideoProjects: 21,
// v1 = Creative Commission FEEDBACK federation (PostgreSQL `commission_feedback`)
// via the per-record peer-sync push pipeline (record kind `commissionFeedback`,
// sync category `commissionFeedback`, #2686 — split-record follow-up to #2657).
diff --git a/server/lib/schemaVersions.test.js b/server/lib/schemaVersions.test.js
index 487df19371..59ebb6306c 100644
--- a/server/lib/schemaVersions.test.js
+++ b/server/lib/schemaVersions.test.js
@@ -64,12 +64,12 @@ describe('PORTOS_SCHEMA_VERSIONS', () => {
});
describe('buildPortosMeta', () => {
- it('rejects repair, timing and narrative event metadata on older music-video peers', async () => {
+ it('rejects repair, timing, narrative event and lyric warning metadata on older music-video peers', async () => {
const meta = await buildPortosMeta();
- expect(meta.schemaVersions.musicVideoProjects).toBe(20);
- for (const receiverV of [13, 14, 15, 16, 17, 18, 19]) {
+ expect(meta.schemaVersions.musicVideoProjects).toBe(21);
+ for (const receiverV of [13, 14, 15, 16, 17, 18, 19, 20]) {
expect(compareSchemaVersions(meta.schemaVersions, { ...meta.schemaVersions, musicVideoProjects: receiverV }).ahead)
- .toContainEqual({ category: 'musicVideoProjects', senderV: 20, receiverV });
+ .toContainEqual({ category: 'musicVideoProjects', senderV: 21, receiverV });
}
});
diff --git a/server/services/musicVideo/lyricAlign.js b/server/services/musicVideo/lyricAlign.js
index b86cf29da6..faf524590c 100644
--- a/server/services/musicVideo/lyricAlign.js
+++ b/server/services/musicVideo/lyricAlign.js
@@ -64,6 +64,13 @@ function relabeledAnalysis(project, cues) {
return parsed.success ? parsed.data : null;
}
+// Measure the aligned word span, not an authored cue boundary that may be wider.
+function compressedWords(words) {
+ // CTC stores milliseconds; compare there so a span of exactly 0.15 s/word
+ // cannot become compressed through floating-point subtraction.
+ return words.length >= 3 && Math.round((words.at(-1).endSec - words[0].startSec) * 1000) < words.length * 150;
+}
+
/**
* Align one project's lyric cues. `cueId` re-aligns that line only: a timed
* line uses its own slice; an untimed line uses full-song CTC context so a
@@ -120,9 +127,27 @@ export async function alignProjectLyrics(projectId, options = {}) {
if (source === 'vocal-stem') {
onProgress({ stage: 'loading-model' });
checkCancel();
- forcedWords = await deps.forceAlign(wav, promptCues, {
- ...region, onProgress, isCancelled: options.isCancelled,
+ // Both explicit cue alignment and automatic repairs use the same runner,
+ // decoded stem, cancellation and progress callbacks. Save only once below.
+ const alignStemCues = (entries, window) => deps.forceAlign(wav, entries, {
+ ...window, onProgress, isCancelled: options.isCancelled,
});
+ forcedWords = await alignStemCues(promptCues, region);
+ if (!cueId) {
+ const wholeSongWords = forcedWords;
+ forcedWords = [...wholeSongWords];
+ for (let index = 0; index < cues.length; index++) {
+ if (!compressedWords(wholeSongWords[index])) continue;
+ checkCancel();
+ const startSec = cues[index - 1]?.endSec ?? wholeSongWords[index - 1]?.at(-1)?.endSec ?? 0;
+ const endSec = cues[index + 1]?.startSec ?? wholeSongWords[index + 1]?.[0]?.startSec ?? wavDurationSec(wav);
+ // Inconsistent authored neighbors cannot supply a useful repair window.
+ if (!(endSec > startSec)) continue;
+ const [retry] = await alignStemCues([cues[index]], { startSec, endSec });
+ checkCancel();
+ if (retry.length === wholeSongWords[index].length && !compressedWords(retry)) forcedWords[index] = retry;
+ }
+ }
kind = 'MMS_FA';
} else {
onProgress({ stage: 'loading-model' });
@@ -154,10 +179,10 @@ export async function alignProjectLyrics(projectId, options = {}) {
checkCancel();
onProgress({ stage: 'saving', percent: 100 });
const align = (entries) => {
- if (!forcedWords) return alignDirectorWords(entries, recognized);
+ if (!forcedWords) return alignDirectorWords(entries, recognized).map((entry) => ({ ...entry, suspect: false }));
return entries.map((entry, index) => {
const words = forcedWords[index];
- return { ...entry, words, matched: words.length ? 1 : 0,
+ return { ...entry, words, matched: words.length ? 1 : 0, suspect: compressedWords(words),
startSec: entry.startSec ?? words[0]?.startSec ?? null,
endSec: entry.endSec ?? words.at(-1)?.endSec ?? null };
});
diff --git a/server/services/musicVideo/lyricAlign.test.js b/server/services/musicVideo/lyricAlign.test.js
index d5da355fc4..3e5e75c0de 100644
--- a/server/services/musicVideo/lyricAlign.test.js
+++ b/server/services/musicVideo/lyricAlign.test.js
@@ -349,10 +349,57 @@ describe('alignProjectLyrics', () => {
expect(saved.lyricAlignSource).toBe('vocal-stem');
});
+ it.each([true, false])('retries only compressed whole-song cues and keeps repairs only when expanded (%s)', async (repaired) => {
+ const record = { ...project, vocalStemFilename: 'stem.wav', lyricCues: [
+ { id: 'a', text: 'walking home', startSec: null, endSec: null },
+ { id: 'b', text: 'under blue skies', startSec: null, endSec: null },
+ { id: 'c', text: 'another clear morning', startSec: null, endSec: null },
+ { id: 'd', text: 'so fast', startSec: null, endSec: null },
+ ] };
+ const words = (text, start, step) => text.split(' ').map((w, i) => ({
+ w, startSec: round3(start + i * step), endSec: round3(start + (i + 1) * step), conf: 'matched',
+ }));
+ const whole = record.lyricCues.map((cue, i) => words(cue.text, [1, 3, 7.1, 9][i], [0.5, 0.1, 0.15, 0.1][i]));
+ const retry = repaired ? words(record.lyricCues[1].text, 4, 0.6) : whole[1];
+ const forceAlign = vi.fn().mockResolvedValueOnce(whole).mockResolvedValueOnce([retry]);
+ const h = harness(vi.fn(), record, { resolveAudio: async () => ({ path: 'stem.wav', source: 'vocal-stem' }) });
+ const stem = encodePcm16Wav(16000 * 10);
+ for (let i = 0; i < 16000 * 10; i++) stem.writeInt16LE(5000, 44 + i * 2);
+ h.decodeAudio.mockResolvedValue(stem);
+ const saved = await h.run({ deps: { forceAlign } });
+ expect(forceAlign).toHaveBeenCalledTimes(2);
+ expect(forceAlign).toHaveBeenLastCalledWith(stem, [record.lyricCues[1]], expect.objectContaining({ startSec: 2, endSec: 7.1 }));
+ expect(saved.lyricCues[1]).toMatchObject({
+ id: 'b', words: retry, startSec: retry[0].startSec, endSec: retry.at(-1).endSec, suspect: !repaired,
+ });
+ for (const i of [0, 2, 3]) expect(saved.lyricCues[i].words).toEqual(whole[i]);
+ expect(h.decodeAudio).toHaveBeenCalledOnce();
+ expect(h.updateProject).toHaveBeenCalledOnce();
+ });
+
+ it.each(['cancel', 'text', 'audio'])('saves nothing when an automatic repair is interrupted by %s', async (outcome) => {
+ const record = { ...project, vocalStemFilename: 'stem.wav', lyricCues: [
+ { id: 'a', text: 'under blue skies', startSec: null, endSec: null },
+ ] };
+ const h = harness(vi.fn(), record, { resolveAudio: async () => ({ path: 'stem.wav', source: 'vocal-stem' }) });
+ let cancelled = false;
+ const collapsed = ['under', 'blue', 'skies'].map((w, i) => ({ w, startSec: i * 0.1, endSec: (i + 1) * 0.1, conf: 'matched' }));
+ const forceAlign = vi.fn().mockResolvedValueOnce([collapsed]).mockImplementationOnce(async () => {
+ if (outcome === 'cancel') cancelled = true;
+ if (outcome === 'text') record.lyricCues = [{ ...record.lyricCues[0], text: 'changed' }];
+ if (outcome === 'audio') record.trackId = 'changed';
+ return [collapsed];
+ });
+ await expect(h.run({ deps: { forceAlign }, isCancelled: () => cancelled })).rejects.toMatchObject(
+ outcome === 'cancel' ? { canceled: true } : { code: outcome === 'text' ? 'LYRIC_ALIGN_TEXT_CHANGED' : 'MUSIC_VIDEO_AUDIO_CHANGED' });
+ expect(forceAlign).toHaveBeenLastCalledWith(expect.any(Buffer), expect.any(Array), expect.objectContaining({ startSec: 0, endSec: 3 }));
+ expect(h.updateProject).not.toHaveBeenCalled();
+ });
+
it('re-aligns one stem line preserving current director edits and the other occurrence', async () => {
const record = { ...project, vocalStemFilename: 'stem.wav', lyricCues: [
- { id: 'a', text: 'walking home', startSec: 1, endSec: 2 },
- { id: 'b', text: 'walking home', startSec: 4, endSec: 5 },
+ { id: 'a', text: 'walking home', startSec: 1, endSec: 2, suspect: true },
+ { id: 'b', text: 'walking home', startSec: 4, endSec: 5, suspect: true },
] };
const h = harness(vi.fn(), record, {
resolveAudio: async () => ({ path: 'stem.wav', source: 'vocal-stem' }),
@@ -366,7 +413,7 @@ describe('alignProjectLyrics', () => {
});
const saved = await h.run({ cueId: 'a', deps: { forceAlign } });
expect(forceAlign).toHaveBeenCalledWith(stem, [expect.objectContaining({ id: 'a' })], expect.objectContaining({ startSec: 1, endSec: 2 }));
- expect(saved.lyricCues[0]).toMatchObject({ startSec: 1.2, endSec: 2.5, matched: 1 });
+ expect(saved.lyricCues[0]).toMatchObject({ startSec: 1.2, endSec: 2.5, matched: 1, suspect: false });
expect(saved.lyricCues[1]).toEqual(record.lyricCues[1]);
expect(saved.lyricAlignSource).toBeUndefined();
});
diff --git a/server/services/musicVideo/projectsFile.test.js b/server/services/musicVideo/projectsFile.test.js
index ebfaf01269..d5e012bd05 100644
--- a/server/services/musicVideo/projectsFile.test.js
+++ b/server/services/musicVideo/projectsFile.test.js
@@ -179,28 +179,31 @@ describe('projectsFile federation (#1770)', () => {
{ w: 'home', startSec: 1.55, endSec: 2, conf: 'matched' },
];
await file.updateProject(p.id, {
- lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4 }],
+ lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4, suspect: true }],
});
const saved = await file.getProject(p.id);
expect(saved.lyricCues[0].words).toEqual(nudged);
expect(saved.lyricCues[0].text).toBe('walking home');
expect(saved.lyricCues[0].matched).toBe(0.4);
+ expect(saved.lyricCues[0].suspect).toBe(true);
expect((await file.cloneProject(p.id)).lyricCues[0].words).toEqual(nudged);
const retitled = await file.updateProject(p.id, {
- lyricCues: [{ id: 'lc-1', text: 'walking house', startSec: 1, endSec: 2, words: nudged, matched: 0.4 }],
+ lyricCues: [{ id: 'lc-1', text: 'walking house', startSec: 1, endSec: 2, words: nudged, matched: 0.4, suspect: true }],
});
expect(retitled.lyricCues[0]).toMatchObject({ text: 'walking house', startSec: 1, endSec: 2 });
expect(retitled.lyricCues[0].words).toBeUndefined();
expect(retitled.lyricCues[0].matched).toBeUndefined();
+ expect(retitled.lyricCues[0].suspect).toBeUndefined();
await file.updateProject(p.id, {
- lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4 }],
+ lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4, suspect: true }],
});
const swapped = await file.updateProject(p.id, { trackId: 't2' });
expect(swapped.lyricCues[0].text).toBe('walking home');
expect(swapped.lyricCues[0].words).toBeUndefined();
expect(swapped.lyricCues[0].matched).toBeUndefined();
+ expect(swapped.lyricCues[0].suspect).toBeUndefined();
expect(swapped.lyricCues[0].startSec).toBeNull();
expect(swapped.lyricCues[0].endSec).toBeNull();
});
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index b9b446d7bb..f60ca01ac4 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -342,7 +342,10 @@ export function applyProjectPatch(project, patch) {
} : {}),
...(Array.isArray(patch.lyricCues) ? { lyricCues: normalizeLyricCues(patch.lyricCues).map((cue) => {
const previous = project.lyricCues?.find((entry) => entry.id === cue.id);
- if (previous && previous.text !== cue.text) delete cue.matched;
+ if (previous && previous.text !== cue.text) {
+ delete cue.matched;
+ delete cue.suspect;
+ }
return cue;
}) } : {}),
...(Array.isArray(patch.lyricMarkers) ? { lyricMarkers: normalizeLyricMarkers(patch.lyricMarkers) } : {}),
diff --git a/server/services/musicVideo/timedText.js b/server/services/musicVideo/timedText.js
index 4c8ea1dd38..cc5cf7573a 100644
--- a/server/services/musicVideo/timedText.js
+++ b/server/services/musicVideo/timedText.js
@@ -97,7 +97,7 @@ export const normalizeLyricCues = (cues) => normalizeList(cues, 'lc', (cue) => {
const words = normalizeWords(text, cue.words);
const matched = Number.isFinite(cue.matched) && cue.matched >= 0 && cue.matched <= 1
? { matched: cue.matched } : {};
- return { text, ...(words ? { words } : {}), ...matched };
+ return { text, ...(words ? { words, ...(cue.suspect === true ? { suspect: true } : {}) } : {}), ...matched };
});
/** Normalize an edited phrase list: keep ids, trim label/intent. */
@@ -115,6 +115,7 @@ export function invalidateTimedText(project) {
const next = { ...e, startSec: null, endSec: null };
delete next.words;
delete next.matched;
+ delete next.suspect;
return next;
}) : list);
return {