Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Lip-sync review shows the shot's own words and drops checks nobody can act on
Claudec7a80bc44576
diff --git a/client/src/components/musicVideo/PerformanceEvidence.jsx b/client/src/components/musicVideo/PerformanceEvidence.jsxindex aa9ed6f6fb..ea0adb30b9 100644--- a/client/src/components/musicVideo/PerformanceEvidence.jsx+++ b/client/src/components/musicVideo/PerformanceEvidence.jsx@@ -2,9 +2,13 @@ import { useEffect, useRef, useState } from 'react';import { formatTimecode, formatUsd } from '../../utils/formatters.js';import { clipAudioOffsetSec } from '../../lib/musicVideoShotTiming.js';+const CONDITIONING_LABELS = { 'vocal-stem': 'vocal stem', master: 'full mix' };+const formatSeconds = (sec) => (Number.isFinite(sec) ? `${sec} s` : 'a few seconds');+/** One transport: muted output picture against the take's recorded source.* Word windows are clip-relative; temporal evidence is excerpt-relative.*/+export default function PerformanceEvidence({ clipSrc, instruction, temporal = null, excerptStartSec = 0, repair = null, onRepair, repairBusy = false }) {const video = useRef(null);const audio = useRef(null);@@ -18,6 +22,22 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = nconst offset = clipAudioOffsetSec(instruction);const canSync = source && Number.isFinite(windowStart);const cues = Array.isArray(instruction?.cues) ? instruction.cues : [];+ // The provider needs a minimum length of audio (MiniMax H3: 5 s), so a short shot's+ // take holds song on either side of it; the edit points are the shot inside the clip.+ const inSec = Number.isFinite(instruction?.edit?.inSec) ? instruction.edit.inSec : 0;+ const outSec = Number.isFinite(instruction?.edit?.outSec) ? instruction.edit.outSec : null;+ const inShot = (startSec, endSec) => endSec + offset > inSec + 1e-3 && (outSec == null || startSec + offset < outSec - 1e-3);+ // Only the shot's own words, by name; takes made before the text was carried have none to show.+ const words = cues.flatMap((cue) => (cue.words?.length ? cue.words : [cue]))+ .filter((word) => Number.isFinite(word.startSec) && Number.isFinite(word.endSec) && inShot(word.startSec, word.endSec))+ .map((word) => ({ ...word, label: (word.text || word.w || '').trim() }))+ .filter((word) => word.label);+ // A measured verdict only exists when the optional local analyzer ran; without one+ // there is nothing for the user to act on, so the panel stays quiet about it.+ const measured = !!temporal?.analyzer;+ const playFrom = useRef(null);+ const windowSec = instruction?.audioWindow?.durationSec;+ const padded = outSec != null && Number.isFinite(windowSec) && outSec - inSec < windowSec - 0.05;useEffect(() => {const picture = video.current;@@ -32,7 +52,17 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = nif (Math.abs(audio.current.currentTime - target) > 0.05) audio.current.currentTime = target;};const pause = () => audio.current?.pause();+ // Playing from inside the shot stops at its out point, so the review hears only this shot.+ const stopAtOut = () => {+ sync();+ const picture = video.current;+ if (picture && outSec != null && playFrom.current != null && playFrom.current < outSec - 0.05 && picture.currentTime >= outSec) {+ playFrom.current = null;+ picture.pause();+ }+ };const play = () => {+ playFrom.current = video.current?.currentTime ?? null;if (!canSync) { setError('Recorded source unavailable — playback cannot verify sync'); video.current?.pause(); return; }sync();// Media events are outside React's request lifecycle.@@ -46,28 +76,28 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n<video ref={video} src={clipSrc} controls muted playsInline preload="metadata"className="w-full max-w-md aspect-video rounded border border-port-border"onVolumeChange={() => { if (video.current) video.current.muted = true; }}- onPlay={play} onPause={pause} onEnded={pause} onSeeking={sync} onTimeUpdate={sync}+ onPlay={play} onPause={pause} onEnded={pause} onSeeking={sync} onTimeUpdate={stopAtOut}onLoadedMetadata={() => seek(instruction?.edit?.inSec || 0)} />{source && <audio ref={audio} src={source} muted={sourceMuted} preload="metadata" onLoadedMetadata={sync}onError={() => { video.current?.pause(); setError('Recorded source unavailable — check the audio file'); }} />}{canSync && <button type="button" aria-pressed={sourceMuted} onClick={() => setSourceMuted((value) => !value)}className="rounded border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0">{sourceMuted ? 'Unmute source' : 'Mute source'}</button>}- <p>Speaker: {instruction?.speaker || 'Unspecified'} · Conditioning: {conditioning?.source || 'Unknown'}</p>- <p className="text-port-text-muted">Source and clip play together. Voice isolation is unverified; separation and equal durations do not prove correct lip-sync.</p>+ <p>Speaker: {instruction?.speaker || 'Unspecified'} · Audio: {CONDITIONING_LABELS[conditioning?.source] || conditioning?.source || 'Unknown'}</p>+ {padded && <p className="text-port-text-muted">The model needs at least {formatSeconds(instruction?.capability?.minAudioSec)} of song, so this take runs longer than the shot. The final render uses {formatTimecode(inSec)}–{formatTimecode(outSec)} of it, and play stops there.</p>}{!canSync && <p className="text-port-warning">Recorded source unavailable — review needed</p>}{error && <p role="status" className="text-port-warning">{error}</p>}- <div className="flex flex-wrap gap-1" aria-label="Word windows">- {cues.flatMap((cue, index) => (cue.words?.length ? cue.words : [cue]).map((word, i) => (- <button type="button" key={`${index}-${i}`} onClick={() => seek(word.startSec + offset)}+ {words.length > 0 && <div className="flex flex-wrap items-center gap-1" aria-label="Jump to a word">+ <span className="text-port-text-muted">Jump to</span>+ {words.map((word, i) => (+ <button type="button" key={i} onClick={() => seek(word.startSec + offset)}className="rounded border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0"- title={`${formatTimecode(word.startSec + offset)}–${formatTimecode(word.endSec + offset)} (clip time)`}>- {word.text} · {formatTimecode(word.startSec + offset)}–{formatTimecode(word.endSec + offset)}+ title={`${formatTimecode(word.startSec + offset)}–${formatTimecode(word.endSec + offset)} in the clip`}>+ {word.label}</button>- )))}- </div>- <p className="text-port-text-muted">Temporal lip-sync: {temporal?.lipSync || 'unverified'}{temporal?.analyzer ? ` · ${temporal.analyzer.id} ${temporal.analyzer.version}` : ' · review needed'}</p>- {temporal && <p className="text-port-text-muted">Measured evidence: {temporal.status}</p>}- {repair && <div className="rounded border border-port-border p-2 space-y-1" aria-label="Suffix repair">+ ))}+ </div>}+ {measured && <p className="text-port-text-muted">Measured lip-sync: {temporal.lipSync || temporal.status} · {temporal.analyzer.id} {temporal.analyzer.version}</p>}+ {repair && (repair.ok || measured) && <div className="rounded border border-port-border p-2 space-y-1" aria-label="Suffix repair">{repair.ok && onRepair ? <><p>Keep accepted prefix through {formatTimecode(repair.boundarySec)}. Continue only {formatTimecode(repair.boundarySec)}–{formatTimecode(repair.endSec)} (song time).</p><p>Next spend: {formatUsd(repair.costUsd)} · one generation · accepted boundary frame</p>@@ -75,7 +105,7 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = nclassName="rounded border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0 disabled:opacity-50">{repairBusy ? 'Repairing…' : 'Repair from here'}</button></> : <p className="text-port-warning">{repair.ok ? 'Review needed: update the authored composition edit points before continuing' : repair.message}</p>}</div>}- <ul className="space-y-1" aria-label="Temporal evidence spans">+ {measured && <ul className="space-y-1" aria-label="Temporal evidence spans">{spans.map((span, i) => (<li key={i}><button type="button" onClick={() => seek(span.startSec + excerptStartSec - (windowStart || 0))}@@ -87,7 +117,7 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n{Number.isFinite(span.confidence) && ` · confidence ${span.confidence.toFixed(2)}`}</li>))}- </ul>+ </ul>}</div>);}diff --git a/client/src/components/musicVideo/PerformanceEvidence.test.jsx b/client/src/components/musicVideo/PerformanceEvidence.test.jsxindex 55685ddb35..10245d3d91 100644--- a/client/src/components/musicVideo/PerformanceEvidence.test.jsx+++ b/client/src/components/musicVideo/PerformanceEvidence.test.jsx@@ -31,7 +31,6 @@ describe('PerformanceEvidence', () => {expect(video.currentTime).toBe(3);fireEvent.pause(video);expect(pause.mock.instances).toContain(audio);- expect(screen.getByText(/Voice isolation is unverified/)).toBeTruthy();expect(screen.getByText(/example-analyzer 1/)).toBeTruthy();play.mockRejectedValueOnce(new Error('blocked'));fireEvent.play(video);@@ -39,11 +38,41 @@ describe('PerformanceEvidence', () => {vi.restoreAllMocks();});- it('keeps legacy takes without source metadata or analyzer evidence visibly unverified', () => {+ it('says when the recorded source is missing', () => {render(<PerformanceEvidence clipSrc="/example.mp4" instruction={{}} />);- expect(screen.getByText(/Temporal lip-sync: unverified/)).toBeTruthy();expect(screen.getByText(/Recorded source unavailable/)).toBeTruthy();});++ // Shot 19 of a music video: a 2.33 s shot inside a 5 s take showed blank word chips, the+ // next line's words, and "review needed" lines no one could act on without an analyzer.+ it('lists only the shot\'s own words, explains the longer take, stops at the out point and stays quiet without an analyzer', () => {+ vi.spyOn(HTMLMediaElement.prototype, 'play').mockResolvedValue();+ const pause = vi.spyOn(HTMLMediaElement.prototype, 'pause').mockImplementation(() => {});+ const take = {+ audio: { conditioning: { source: 'vocal-stem', filename: 'singer.wav' } },+ audioWindow: { startSec: 40, durationSec: 5 }, edit: { inSec: 1, outSec: 3.33 }, capability: { minAudioSec: 5 },+ cues: [+ { text: 'this line', startSec: 0.9, endSec: 3.2, words: [{ w: 'this', startSec: 0.9, endSec: 1.6 }, { text: 'line', startSec: 1.7, endSec: 3.2 }] },+ { text: 'the next line', startSec: 4.3, endSec: 5 },+ ],+ };+ const { container } = render(<PerformanceEvidence clipSrc="/example.mp4" instruction={take}+ repair={{ ok: false, message: 'Review needed: complete measured temporal evidence is required' }} onRepair={vi.fn()} />);+ expect(screen.getByRole('button', { name: 'this' })).toBeTruthy();+ expect(screen.getByRole('button', { name: 'line' })).toBeTruthy();+ expect(screen.queryByText(/the next line/)).toBeNull();+ expect(screen.getByText(/needs at least 5 s of song/)).toBeTruthy();+ expect(screen.getByText(/Audio: vocal stem/)).toBeTruthy();+ expect(screen.queryByText(/Review needed|unverified/i)).toBeNull();+ const video = container.querySelector('video');+ fireEvent.loadedMetadata(video);+ fireEvent.play(video);+ pause.mockClear();+ video.currentTime = 3.4;+ fireEvent.timeUpdate(video);+ expect(pause.mock.instances).toContain(video);+ vi.restoreAllMocks();+ });});// This interaction verifies the spend/boundary shown before the explicit@@ -58,7 +87,8 @@ it('shows the accepted boundary and next spend, disables duplicate clicks, and rexpect(onRepair).toHaveBeenCalledTimes(1);view.rerender(<PerformanceEvidence instruction={instruction} repair={repair} onRepair={onRepair} repairBusy />);expect(screen.getByRole('button', { name: 'Repairing…' }).disabled).toBe(true);- view.rerender(<PerformanceEvidence instruction={instruction} repair={{ ok: false, message: 'Review needed: evidence is inconclusive' }} onRepair={onRepair} />);+ view.rerender(<PerformanceEvidence instruction={instruction} temporal={{ status: 'unverified', analyzer: { id: 'example-analyzer', version: '1' }, spans: [] }}+ repair={{ ok: false, message: 'Review needed: evidence is inconclusive' }} onRepair={onRepair} />);expect(screen.queryByRole('button', { name: 'Repair from here' })).toBeNull();expect(screen.getByText(/evidence is inconclusive/)).toBeTruthy();});diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.jsindex 27326d6aa0..3e5838be6f 100644--- a/server/lib/musicVideoShotTiming.js+++ b/server/lib/musicVideoShotTiming.js@@ -341,7 +341,8 @@ export function clipRelativeCues(cues, windowStartSec, windowEndSec) {endSec: round6(Math.min(windowEndSec, end) - windowStartSec),...(Array.isArray(cue.words) ? { words: cue.words.filter((word) => word && Number.isFinite(word.startSec) && Number.isFinite(word.endSec) && word.endSec > word.startSec && word.endSec > windowStartSec && word.startSec < windowEndSec)- .map((word) => ({ text: word.text, startSec: round6(Math.max(windowStartSec, word.startSec) - windowStartSec), endSec: round6(Math.min(windowEndSec, word.endSec) - windowStartSec) })) } : {}),+ // Aligned lyric words keep their text as `w` (lyricAlignCore), transcribed ones as `text`.+ .map((word) => ({ text: typeof word.text === 'string' ? word.text : typeof word.w === 'string' ? word.w : '', startSec: round6(Math.max(windowStartSec, word.startSec) - windowStartSec), endSec: round6(Math.min(windowEndSec, word.endSec) - windowStartSec) })) } : {}),});}return out;diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.jsindex ea3d3d5801..2a287cf73a 100644--- a/server/services/musicVideo/performanceShot.test.js+++ b/server/services/musicVideo/performanceShot.test.js@@ -76,7 +76,9 @@ const project = (sceneOver) => ({id: 'mv-1',uploadedAudioFilename: 'song.wav',lyricCues: [- { id: 'c1', text: 'hold on', startSec: 19, endSec: 20.5, words: [{ text: 'hold', startSec: 19, endSec: 20 }, { text: 'on', startSec: 20, endSec: 20.5 }] },+ // Aligned words carry their text as `w` (lyricAlignCore); the take records it as `text`.+ { id: 'c1', text: 'hold on', startSec: 19, endSec: 20.5,+ words: [{ w: 'hold', startSec: 19, endSec: 20, conf: 'matched' }, { text: 'on', startSec: 20, endSec: 20.5 }] },{ id: 'c2', text: 'later line', startSec: 40, endSec: 42 },{ id: 'c3', text: 'untimed', startSec: null, endSec: null },],