Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: repair compressed lyric lines and flag unresolved timing (#10973)
Adam Eivyae2392b47c20
diff --git a/client/src/components/musicVideo/LyricTimingCheck.jsx b/client/src/components/musicVideo/LyricTimingCheck.jsxindex 5eb05fd676..60565517e1 100644--- a/client/src/components/musicVideo/LyricTimingCheck.jsx+++ b/client/src/components/musicVideo/LyricTimingCheck.jsx@@ -1,4 +1,5 @@import { useState } from 'react';+import { formatTimecode } from '../../utils/formatters.js';import { lyricSetupState } from '../../lib/musicVideoStages.js';import CompositionPreviewPlayer from './CompositionPreviewPlayer.jsx';@@ -24,6 +25,7 @@ export default function LyricTimingCheck({ project, review, planning = null, disconst saved = { ...EMPTY_DRAFT, ...(project.productionReview?.draft || {}) };const lyrics = lyricSetupState(project, review.readiness);const [mode, setMode] = useState(saved.lyricsMode === 'instrumental' ? 'instrumental' : 'vocal');+ const [seekRequest, setSeekRequest] = useState(null);const [notes, setNotes] = useState(saved.timingNotes || '');const stale = lyrics.alignment === 'stale';const busy = disabled || review.busy || !review.current;@@ -42,6 +44,7 @@ export default function LyricTimingCheck({ project, review, planning = null, disconst verified = mode === 'vocal' ? lyrics.verified && !lyrics.instrumental : lyrics.instrumental && lyrics.verified;const timed = (project.lyricCues || []).some((cue) => typeof cue.startSec === 'number' && cue.text?.trim());+ const suspectCues = (project.lyricCues || []).filter((cue) => cue.suspect);const playable = mode === 'vocal' && timed && !!project.audioAnalysis && !!audioUrl && !aligning;return (<div className="space-y-2" id="mv-lyric-timing">@@ -56,6 +59,26 @@ export default function LyricTimingCheck({ project, review, planning = null, disInstrumental</label></fieldset>+ {playable && (!verified || suspectCues.length > 0) && (+ <>+ <p className="text-sm text-port-text-muted">Play the lyrics through with the song and watch each word land on the vocal.</p>+ {suspectCues.length > 0 && (+ <div className="space-y-1 text-sm">+ <p className="text-port-warning">These lines may play too quickly. Jump to each one and check it against the vocal.</p>+ <ul className="space-y-1">+ {suspectCues.map((cue) => {+ const t = cue.words?.[0]?.startSec ?? cue.startSec ?? 0;+ return <li key={cue.id}><button type="button" className={buttonClass}+ onClick={() => setSeekRequest((previous) => ({ t, n: (previous?.n ?? 0) + 1, play: true }))}>+ {formatTimecode(t)} — {cue.text}+ </button></li>;+ })}+ </ul>+ </div>+ )}+ <CompositionPreviewPlayer project={project} audioUrl={audioUrl} lyrics scrubId={`mv-lyrics-check-scrub-${project.id}`} seekRequest={seekRequest} />+ </>+ )}{verified ? (<p role="status" className="text-sm text-port-success">{mode === 'vocal' ? 'Word timing verified against the current master.' : 'Marked instrumental.'}@@ -69,12 +92,6 @@ export default function LyricTimingCheck({ project, review, planning = null, dis: 'Align the words to play them through with the song.'}</p>)}- {playable && (- <>- <p className="text-sm text-port-text-muted">Play the lyrics through with the song and watch each word land on the vocal.</p>- <CompositionPreviewPlayer project={project} audioUrl={audioUrl} lyrics scrubId={`mv-lyrics-check-scrub-${project.id}`} />- </>- )}<label htmlFor={notesId} className="block text-xs text-port-text-muted">Notes (optional)</label>diff --git a/client/src/components/musicVideo/stages/SetupStage.test.jsx b/client/src/components/musicVideo/stages/SetupStage.test.jsxindex fcc21608f2..284a1bdcea 100644--- a/client/src/components/musicVideo/stages/SetupStage.test.jsx+++ b/client/src/components/musicVideo/stages/SetupStage.test.jsx@@ -4,7 +4,7 @@ import { fireEvent, render, screen, within } from '@testing-library/react';import SetupStage from './SetupStage.jsx';vi.mock('../../songs/MidiVisualization.jsx', () => ({ default: () => null }));-vi.mock('../CompositionPreviewPlayer.jsx', () => ({ default: ({ lyrics }) => <div data-testid="playthrough">{lyrics ? 'lyrics' : 'document'}</div> }));+vi.mock('../CompositionPreviewPlayer.jsx', () => ({ default: ({ lyrics, seekRequest }) => <div data-testid="playthrough" data-seek={JSON.stringify(seekRequest)}>{lyrics ? 'lyrics' : 'document'}</div> }));const review = (over = {}) => ({ readiness: null, current: true, busy: false, error: null, save: vi.fn(async () => ({})), reverifyAlignment: vi.fn(async () => ({})), ...over });const board = (project, extra = {}) => ({@@ -52,6 +52,21 @@ describe('Song step', () => {await vi.waitFor(() => expect(setPlanning).toHaveBeenCalledWith({ cast: 'edited', lyricsMode: 'vocal', timingStatus: 'verified', timingNotes: 'Listened through twice' }));});+ it('lists suspect lyric text and timecodes and jumps the playthrough to that line on every click', () => {+ const project = { id: 'p', trackId: 't', audioAnalysis: {}, productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'verified' } }, lyricCues: [+ { id: 'c', text: 'under blue skies', startSec: 60, endSec: 65, suspect: true,+ words: [{ w: 'under', startSec: 61.2, endSec: 61.3 }] },+ { id: 'd', text: 'walking home', startSec: 70, endSec: 72 },+ ] };+ render(<SetupStage board={board(project, { productionReview: review() })} />);+ const jump = screen.getByRole('button', { name: '1:01.20 — under blue skies' });+ expect(screen.queryByRole('button', { name: /walking home/ })).toBeNull();+ fireEvent.click(jump);+ expect(JSON.parse(screen.getByTestId('playthrough').dataset.seek)).toEqual({ t: 61.2, n: 1, play: true });+ fireEvent.click(jump);+ expect(JSON.parse(screen.getByTestId('playthrough').dataset.seek)).toEqual({ t: 61.2, n: 2, play: true });+ });+it('holds the timing check until the aligned words can be played through', () => {const project = { id: 'p', trackId: 't', audioAnalysis: {}, lyricCues: [{ id: 'c', text: 'hi', startSec: 1, endSec: 2 }],productionReview: { draft: { lyricsMode: 'vocal', timingStatus: 'provisional' } } };diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.jsindex c43979098d..d1f033aad4 100644--- a/server/lib/musicVideoValidation.js+++ b/server/lib/musicVideoValidation.js@@ -179,6 +179,7 @@ export const musicVideoLyricCueSchema = z.object({endSec: timedSec,words: z.array(lyricWordSchema).max(400).optional(),matched: z.number().min(0).max(1).optional(),+ suspect: z.boolean().optional(),}).strict();// A musical-phrase annotation: a span of the song with an optional visualdiff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.jsindex e96e46a774..79280407b3 100644--- a/server/lib/schemaVersions.js+++ b/server/lib/schemaVersions.js@@ -683,7 +683,9 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({// v19 pins whole-workflow media policy; older peers cannot enforce it.// v20 adds the Eidoverse render mode and scene source. Older normalizers// would collapse the mode to footage and discard the authored scene.- musicVideoProjects: 20,+ // v21 adds compressed lyric timing warnings (`lyricCues[].suspect`). Older+ // normalizers drop the warning on edits. Absent remains legacy; no backfill.+ musicVideoProjects: 21,// v1 = Creative Commission FEEDBACK federation (PostgreSQL `commission_feedback`)// via the per-record peer-sync push pipeline (record kind `commissionFeedback`,// sync category `commissionFeedback`, #2686 — split-record follow-up to #2657).diff --git a/server/lib/schemaVersions.test.js b/server/lib/schemaVersions.test.jsindex 487df19371..59ebb6306c 100644--- a/server/lib/schemaVersions.test.js+++ b/server/lib/schemaVersions.test.js@@ -64,12 +64,12 @@ describe('PORTOS_SCHEMA_VERSIONS', () => {});describe('buildPortosMeta', () => {- it('rejects repair, timing and narrative event metadata on older music-video peers', async () => {+ it('rejects repair, timing, narrative event and lyric warning metadata on older music-video peers', async () => {const meta = await buildPortosMeta();- expect(meta.schemaVersions.musicVideoProjects).toBe(20);- for (const receiverV of [13, 14, 15, 16, 17, 18, 19]) {+ expect(meta.schemaVersions.musicVideoProjects).toBe(21);+ for (const receiverV of [13, 14, 15, 16, 17, 18, 19, 20]) {expect(compareSchemaVersions(meta.schemaVersions, { ...meta.schemaVersions, musicVideoProjects: receiverV }).ahead)- .toContainEqual({ category: 'musicVideoProjects', senderV: 20, receiverV });+ .toContainEqual({ category: 'musicVideoProjects', senderV: 21, receiverV });}});diff --git a/server/services/musicVideo/lyricAlign.js b/server/services/musicVideo/lyricAlign.jsindex b86cf29da6..faf524590c 100644--- a/server/services/musicVideo/lyricAlign.js+++ b/server/services/musicVideo/lyricAlign.js@@ -64,6 +64,13 @@ function relabeledAnalysis(project, cues) {return parsed.success ? parsed.data : null;}+// Measure the aligned word span, not an authored cue boundary that may be wider.+function compressedWords(words) {+ // CTC stores milliseconds; compare there so a span of exactly 0.15 s/word+ // cannot become compressed through floating-point subtraction.+ return words.length >= 3 && Math.round((words.at(-1).endSec - words[0].startSec) * 1000) < words.length * 150;+}+/*** Align one project's lyric cues. `cueId` re-aligns that line only: a timed* line uses its own slice; an untimed line uses full-song CTC context so a@@ -120,9 +127,27 @@ export async function alignProjectLyrics(projectId, options = {}) {if (source === 'vocal-stem') {onProgress({ stage: 'loading-model' });checkCancel();- forcedWords = await deps.forceAlign(wav, promptCues, {- ...region, onProgress, isCancelled: options.isCancelled,+ // Both explicit cue alignment and automatic repairs use the same runner,+ // decoded stem, cancellation and progress callbacks. Save only once below.+ const alignStemCues = (entries, window) => deps.forceAlign(wav, entries, {+ ...window, onProgress, isCancelled: options.isCancelled,});+ forcedWords = await alignStemCues(promptCues, region);+ if (!cueId) {+ const wholeSongWords = forcedWords;+ forcedWords = [...wholeSongWords];+ for (let index = 0; index < cues.length; index++) {+ if (!compressedWords(wholeSongWords[index])) continue;+ checkCancel();+ const startSec = cues[index - 1]?.endSec ?? wholeSongWords[index - 1]?.at(-1)?.endSec ?? 0;+ const endSec = cues[index + 1]?.startSec ?? wholeSongWords[index + 1]?.[0]?.startSec ?? wavDurationSec(wav);+ // Inconsistent authored neighbors cannot supply a useful repair window.+ if (!(endSec > startSec)) continue;+ const [retry] = await alignStemCues([cues[index]], { startSec, endSec });+ checkCancel();+ if (retry.length === wholeSongWords[index].length && !compressedWords(retry)) forcedWords[index] = retry;+ }+ }kind = 'MMS_FA';} else {onProgress({ stage: 'loading-model' });@@ -154,10 +179,10 @@ export async function alignProjectLyrics(projectId, options = {}) {checkCancel();onProgress({ stage: 'saving', percent: 100 });const align = (entries) => {- if (!forcedWords) return alignDirectorWords(entries, recognized);+ if (!forcedWords) return alignDirectorWords(entries, recognized).map((entry) => ({ ...entry, suspect: false }));return entries.map((entry, index) => {const words = forcedWords[index];- return { ...entry, words, matched: words.length ? 1 : 0,+ return { ...entry, words, matched: words.length ? 1 : 0, suspect: compressedWords(words),startSec: entry.startSec ?? words[0]?.startSec ?? null,endSec: entry.endSec ?? words.at(-1)?.endSec ?? null };});diff --git a/server/services/musicVideo/lyricAlign.test.js b/server/services/musicVideo/lyricAlign.test.jsindex d5da355fc4..3e5e75c0de 100644--- a/server/services/musicVideo/lyricAlign.test.js+++ b/server/services/musicVideo/lyricAlign.test.js@@ -349,10 +349,57 @@ describe('alignProjectLyrics', () => {expect(saved.lyricAlignSource).toBe('vocal-stem');});+ it.each([true, false])('retries only compressed whole-song cues and keeps repairs only when expanded (%s)', async (repaired) => {+ const record = { ...project, vocalStemFilename: 'stem.wav', lyricCues: [+ { id: 'a', text: 'walking home', startSec: null, endSec: null },+ { id: 'b', text: 'under blue skies', startSec: null, endSec: null },+ { id: 'c', text: 'another clear morning', startSec: null, endSec: null },+ { id: 'd', text: 'so fast', startSec: null, endSec: null },+ ] };+ const words = (text, start, step) => text.split(' ').map((w, i) => ({+ w, startSec: round3(start + i * step), endSec: round3(start + (i + 1) * step), conf: 'matched',+ }));+ const whole = record.lyricCues.map((cue, i) => words(cue.text, [1, 3, 7.1, 9][i], [0.5, 0.1, 0.15, 0.1][i]));+ const retry = repaired ? words(record.lyricCues[1].text, 4, 0.6) : whole[1];+ const forceAlign = vi.fn().mockResolvedValueOnce(whole).mockResolvedValueOnce([retry]);+ const h = harness(vi.fn(), record, { resolveAudio: async () => ({ path: 'stem.wav', source: 'vocal-stem' }) });+ const stem = encodePcm16Wav(16000 * 10);+ for (let i = 0; i < 16000 * 10; i++) stem.writeInt16LE(5000, 44 + i * 2);+ h.decodeAudio.mockResolvedValue(stem);+ const saved = await h.run({ deps: { forceAlign } });+ expect(forceAlign).toHaveBeenCalledTimes(2);+ expect(forceAlign).toHaveBeenLastCalledWith(stem, [record.lyricCues[1]], expect.objectContaining({ startSec: 2, endSec: 7.1 }));+ expect(saved.lyricCues[1]).toMatchObject({+ id: 'b', words: retry, startSec: retry[0].startSec, endSec: retry.at(-1).endSec, suspect: !repaired,+ });+ for (const i of [0, 2, 3]) expect(saved.lyricCues[i].words).toEqual(whole[i]);+ expect(h.decodeAudio).toHaveBeenCalledOnce();+ expect(h.updateProject).toHaveBeenCalledOnce();+ });++ it.each(['cancel', 'text', 'audio'])('saves nothing when an automatic repair is interrupted by %s', async (outcome) => {+ const record = { ...project, vocalStemFilename: 'stem.wav', lyricCues: [+ { id: 'a', text: 'under blue skies', startSec: null, endSec: null },+ ] };+ const h = harness(vi.fn(), record, { resolveAudio: async () => ({ path: 'stem.wav', source: 'vocal-stem' }) });+ let cancelled = false;+ const collapsed = ['under', 'blue', 'skies'].map((w, i) => ({ w, startSec: i * 0.1, endSec: (i + 1) * 0.1, conf: 'matched' }));+ const forceAlign = vi.fn().mockResolvedValueOnce([collapsed]).mockImplementationOnce(async () => {+ if (outcome === 'cancel') cancelled = true;+ if (outcome === 'text') record.lyricCues = [{ ...record.lyricCues[0], text: 'changed' }];+ if (outcome === 'audio') record.trackId = 'changed';+ return [collapsed];+ });+ await expect(h.run({ deps: { forceAlign }, isCancelled: () => cancelled })).rejects.toMatchObject(+ outcome === 'cancel' ? { canceled: true } : { code: outcome === 'text' ? 'LYRIC_ALIGN_TEXT_CHANGED' : 'MUSIC_VIDEO_AUDIO_CHANGED' });+ expect(forceAlign).toHaveBeenLastCalledWith(expect.any(Buffer), expect.any(Array), expect.objectContaining({ startSec: 0, endSec: 3 }));+ expect(h.updateProject).not.toHaveBeenCalled();+ });+it('re-aligns one stem line preserving current director edits and the other occurrence', async () => {const record = { ...project, vocalStemFilename: 'stem.wav', lyricCues: [- { id: 'a', text: 'walking home', startSec: 1, endSec: 2 },- { id: 'b', text: 'walking home', startSec: 4, endSec: 5 },+ { id: 'a', text: 'walking home', startSec: 1, endSec: 2, suspect: true },+ { id: 'b', text: 'walking home', startSec: 4, endSec: 5, suspect: true },] };const h = harness(vi.fn(), record, {resolveAudio: async () => ({ path: 'stem.wav', source: 'vocal-stem' }),@@ -366,7 +413,7 @@ describe('alignProjectLyrics', () => {});const saved = await h.run({ cueId: 'a', deps: { forceAlign } });expect(forceAlign).toHaveBeenCalledWith(stem, [expect.objectContaining({ id: 'a' })], expect.objectContaining({ startSec: 1, endSec: 2 }));- expect(saved.lyricCues[0]).toMatchObject({ startSec: 1.2, endSec: 2.5, matched: 1 });+ expect(saved.lyricCues[0]).toMatchObject({ startSec: 1.2, endSec: 2.5, matched: 1, suspect: false });expect(saved.lyricCues[1]).toEqual(record.lyricCues[1]);expect(saved.lyricAlignSource).toBeUndefined();});diff --git a/server/services/musicVideo/projectsFile.test.js b/server/services/musicVideo/projectsFile.test.jsindex ebfaf01269..d5e012bd05 100644--- a/server/services/musicVideo/projectsFile.test.js+++ b/server/services/musicVideo/projectsFile.test.js@@ -179,28 +179,31 @@ describe('projectsFile federation (#1770)', () => {{ w: 'home', startSec: 1.55, endSec: 2, conf: 'matched' },];await file.updateProject(p.id, {- lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4 }],+ lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4, suspect: true }],});const saved = await file.getProject(p.id);expect(saved.lyricCues[0].words).toEqual(nudged);expect(saved.lyricCues[0].text).toBe('walking home');expect(saved.lyricCues[0].matched).toBe(0.4);+ expect(saved.lyricCues[0].suspect).toBe(true);expect((await file.cloneProject(p.id)).lyricCues[0].words).toEqual(nudged);const retitled = await file.updateProject(p.id, {- lyricCues: [{ id: 'lc-1', text: 'walking house', startSec: 1, endSec: 2, words: nudged, matched: 0.4 }],+ lyricCues: [{ id: 'lc-1', text: 'walking house', startSec: 1, endSec: 2, words: nudged, matched: 0.4, suspect: true }],});expect(retitled.lyricCues[0]).toMatchObject({ text: 'walking house', startSec: 1, endSec: 2 });expect(retitled.lyricCues[0].words).toBeUndefined();expect(retitled.lyricCues[0].matched).toBeUndefined();+ expect(retitled.lyricCues[0].suspect).toBeUndefined();await file.updateProject(p.id, {- lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4 }],+ lyricCues: [{ id: 'lc-1', text: 'walking home', startSec: 1, endSec: 2, words: nudged, matched: 0.4, suspect: true }],});const swapped = await file.updateProject(p.id, { trackId: 't2' });expect(swapped.lyricCues[0].text).toBe('walking home');expect(swapped.lyricCues[0].words).toBeUndefined();expect(swapped.lyricCues[0].matched).toBeUndefined();+ expect(swapped.lyricCues[0].suspect).toBeUndefined();expect(swapped.lyricCues[0].startSec).toBeNull();expect(swapped.lyricCues[0].endSec).toBeNull();});diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.jsindex b9b446d7bb..f60ca01ac4 100644--- a/server/services/musicVideo/projectsLogic.js+++ b/server/services/musicVideo/projectsLogic.js@@ -342,7 +342,10 @@ export function applyProjectPatch(project, patch) {} : {}),...(Array.isArray(patch.lyricCues) ? { lyricCues: normalizeLyricCues(patch.lyricCues).map((cue) => {const previous = project.lyricCues?.find((entry) => entry.id === cue.id);- if (previous && previous.text !== cue.text) delete cue.matched;+ if (previous && previous.text !== cue.text) {+ delete cue.matched;+ delete cue.suspect;+ }return cue;}) } : {}),...(Array.isArray(patch.lyricMarkers) ? { lyricMarkers: normalizeLyricMarkers(patch.lyricMarkers) } : {}),diff --git a/server/services/musicVideo/timedText.js b/server/services/musicVideo/timedText.jsindex 4c8ea1dd38..cc5cf7573a 100644--- a/server/services/musicVideo/timedText.js+++ b/server/services/musicVideo/timedText.js@@ -97,7 +97,7 @@ export const normalizeLyricCues = (cues) => normalizeList(cues, 'lc', (cue) => {const words = normalizeWords(text, cue.words);const matched = Number.isFinite(cue.matched) && cue.matched >= 0 && cue.matched <= 1? { matched: cue.matched } : {};- return { text, ...(words ? { words } : {}), ...matched };+ return { text, ...(words ? { words, ...(cue.suspect === true ? { suspect: true } : {}) } : {}), ...matched };});/** Normalize an edited phrase list: keep ids, trim label/intent. */@@ -115,6 +115,7 @@ export function invalidateTimedText(project) {const next = { ...e, startSec: null, endSec: null };delete next.words;delete next.matched;+ delete next.suspect;return next;}) : list);return {