Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Lip-sync review shows the shot's own words and drops checks nobody can act on

Claudec7a80bc44576
diff --git a/client/src/components/musicVideo/PerformanceEvidence.jsx b/client/src/components/musicVideo/PerformanceEvidence.jsx
index aa9ed6f6fb..ea0adb30b9 100644
--- a/client/src/components/musicVideo/PerformanceEvidence.jsx
+++ b/client/src/components/musicVideo/PerformanceEvidence.jsx
@@ -2,9 +2,13 @@ import { useEffect, useRef, useState } from 'react';
import { formatTimecode, formatUsd } from '../../utils/formatters.js';
import { clipAudioOffsetSec } from '../../lib/musicVideoShotTiming.js';
+const CONDITIONING_LABELS = { 'vocal-stem': 'vocal stem', master: 'full mix' };
+const formatSeconds = (sec) => (Number.isFinite(sec) ? `${sec} s` : 'a few seconds');
+
/** One transport: muted output picture against the take's recorded source.
* Word windows are clip-relative; temporal evidence is excerpt-relative.
*/
+
export default function PerformanceEvidence({ clipSrc, instruction, temporal = null, excerptStartSec = 0, repair = null, onRepair, repairBusy = false }) {
const video = useRef(null);
const audio = useRef(null);
@@ -18,6 +22,22 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n
const offset = clipAudioOffsetSec(instruction);
const canSync = source && Number.isFinite(windowStart);
const cues = Array.isArray(instruction?.cues) ? instruction.cues : [];
+ // The provider needs a minimum length of audio (MiniMax H3: 5 s), so a short shot's
+ // take holds song on either side of it; the edit points are the shot inside the clip.
+ const inSec = Number.isFinite(instruction?.edit?.inSec) ? instruction.edit.inSec : 0;
+ const outSec = Number.isFinite(instruction?.edit?.outSec) ? instruction.edit.outSec : null;
+ const inShot = (startSec, endSec) => endSec + offset > inSec + 1e-3 && (outSec == null || startSec + offset < outSec - 1e-3);
+ // Only the shot's own words, by name; takes made before the text was carried have none to show.
+ const words = cues.flatMap((cue) => (cue.words?.length ? cue.words : [cue]))
+ .filter((word) => Number.isFinite(word.startSec) && Number.isFinite(word.endSec) && inShot(word.startSec, word.endSec))
+ .map((word) => ({ ...word, label: (word.text || word.w || '').trim() }))
+ .filter((word) => word.label);
+ // A measured verdict only exists when the optional local analyzer ran; without one
+ // there is nothing for the user to act on, so the panel stays quiet about it.
+ const measured = !!temporal?.analyzer;
+ const playFrom = useRef(null);
+ const windowSec = instruction?.audioWindow?.durationSec;
+ const padded = outSec != null && Number.isFinite(windowSec) && outSec - inSec < windowSec - 0.05;
useEffect(() => {
const picture = video.current;
@@ -32,7 +52,17 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n
if (Math.abs(audio.current.currentTime - target) > 0.05) audio.current.currentTime = target;
};
const pause = () => audio.current?.pause();
+ // Playing from inside the shot stops at its out point, so the review hears only this shot.
+ const stopAtOut = () => {
+ sync();
+ const picture = video.current;
+ if (picture && outSec != null && playFrom.current != null && playFrom.current < outSec - 0.05 && picture.currentTime >= outSec) {
+ playFrom.current = null;
+ picture.pause();
+ }
+ };
const play = () => {
+ playFrom.current = video.current?.currentTime ?? null;
if (!canSync) { setError('Recorded source unavailable — playback cannot verify sync'); video.current?.pause(); return; }
sync();
// Media events are outside React's request lifecycle.
@@ -46,28 +76,28 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n
<video ref={video} src={clipSrc} controls muted playsInline preload="metadata"
className="w-full max-w-md aspect-video rounded border border-port-border"
onVolumeChange={() => { if (video.current) video.current.muted = true; }}
- onPlay={play} onPause={pause} onEnded={pause} onSeeking={sync} onTimeUpdate={sync}
+ onPlay={play} onPause={pause} onEnded={pause} onSeeking={sync} onTimeUpdate={stopAtOut}
onLoadedMetadata={() => seek(instruction?.edit?.inSec || 0)} />
{source && <audio ref={audio} src={source} muted={sourceMuted} preload="metadata" onLoadedMetadata={sync}
onError={() => { video.current?.pause(); setError('Recorded source unavailable — check the audio file'); }} />}
{canSync && <button type="button" aria-pressed={sourceMuted} onClick={() => setSourceMuted((value) => !value)}
className="rounded border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0">{sourceMuted ? 'Unmute source' : 'Mute source'}</button>}
- <p>Speaker: {instruction?.speaker || 'Unspecified'} · Conditioning: {conditioning?.source || 'Unknown'}</p>
- <p className="text-port-text-muted">Source and clip play together. Voice isolation is unverified; separation and equal durations do not prove correct lip-sync.</p>
+ <p>Speaker: {instruction?.speaker || 'Unspecified'} · Audio: {CONDITIONING_LABELS[conditioning?.source] || conditioning?.source || 'Unknown'}</p>
+ {padded && <p className="text-port-text-muted">The model needs at least {formatSeconds(instruction?.capability?.minAudioSec)} of song, so this take runs longer than the shot. The final render uses {formatTimecode(inSec)}–{formatTimecode(outSec)} of it, and play stops there.</p>}
{!canSync && <p className="text-port-warning">Recorded source unavailable — review needed</p>}
{error && <p role="status" className="text-port-warning">{error}</p>}
- <div className="flex flex-wrap gap-1" aria-label="Word windows">
- {cues.flatMap((cue, index) => (cue.words?.length ? cue.words : [cue]).map((word, i) => (
- <button type="button" key={`${index}-${i}`} onClick={() => seek(word.startSec + offset)}
+ {words.length > 0 && <div className="flex flex-wrap items-center gap-1" aria-label="Jump to a word">
+ <span className="text-port-text-muted">Jump to</span>
+ {words.map((word, i) => (
+ <button type="button" key={i} onClick={() => seek(word.startSec + offset)}
className="rounded border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0"
- title={`${formatTimecode(word.startSec + offset)}–${formatTimecode(word.endSec + offset)} (clip time)`}>
- {word.text} · {formatTimecode(word.startSec + offset)}–{formatTimecode(word.endSec + offset)}
+ title={`${formatTimecode(word.startSec + offset)}–${formatTimecode(word.endSec + offset)} in the clip`}>
+ {word.label}
</button>
- )))}
- </div>
- <p className="text-port-text-muted">Temporal lip-sync: {temporal?.lipSync || 'unverified'}{temporal?.analyzer ? ` · ${temporal.analyzer.id} ${temporal.analyzer.version}` : ' · review needed'}</p>
- {temporal && <p className="text-port-text-muted">Measured evidence: {temporal.status}</p>}
- {repair && <div className="rounded border border-port-border p-2 space-y-1" aria-label="Suffix repair">
+ ))}
+ </div>}
+ {measured && <p className="text-port-text-muted">Measured lip-sync: {temporal.lipSync || temporal.status} · {temporal.analyzer.id} {temporal.analyzer.version}</p>}
+ {repair && (repair.ok || measured) && <div className="rounded border border-port-border p-2 space-y-1" aria-label="Suffix repair">
{repair.ok && onRepair ? <>
<p>Keep accepted prefix through {formatTimecode(repair.boundarySec)}. Continue only {formatTimecode(repair.boundarySec)}–{formatTimecode(repair.endSec)} (song time).</p>
<p>Next spend: {formatUsd(repair.costUsd)} · one generation · accepted boundary frame</p>
@@ -75,7 +105,7 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n
className="rounded border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0 disabled:opacity-50">{repairBusy ? 'Repairing…' : 'Repair from here'}</button>
</> : <p className="text-port-warning">{repair.ok ? 'Review needed: update the authored composition edit points before continuing' : repair.message}</p>}
</div>}
- <ul className="space-y-1" aria-label="Temporal evidence spans">
+ {measured && <ul className="space-y-1" aria-label="Temporal evidence spans">
{spans.map((span, i) => (
<li key={i}>
<button type="button" onClick={() => seek(span.startSec + excerptStartSec - (windowStart || 0))}
@@ -87,7 +117,7 @@ export default function PerformanceEvidence({ clipSrc, instruction, temporal = n
{Number.isFinite(span.confidence) && ` · confidence ${span.confidence.toFixed(2)}`}
</li>
))}
- </ul>
+ </ul>}
</div>
);
}
diff --git a/client/src/components/musicVideo/PerformanceEvidence.test.jsx b/client/src/components/musicVideo/PerformanceEvidence.test.jsx
index 55685ddb35..10245d3d91 100644
--- a/client/src/components/musicVideo/PerformanceEvidence.test.jsx
+++ b/client/src/components/musicVideo/PerformanceEvidence.test.jsx
@@ -31,7 +31,6 @@ describe('PerformanceEvidence', () => {
expect(video.currentTime).toBe(3);
fireEvent.pause(video);
expect(pause.mock.instances).toContain(audio);
- expect(screen.getByText(/Voice isolation is unverified/)).toBeTruthy();
expect(screen.getByText(/example-analyzer 1/)).toBeTruthy();
play.mockRejectedValueOnce(new Error('blocked'));
fireEvent.play(video);
@@ -39,11 +38,41 @@ describe('PerformanceEvidence', () => {
vi.restoreAllMocks();
});
- it('keeps legacy takes without source metadata or analyzer evidence visibly unverified', () => {
+ it('says when the recorded source is missing', () => {
render(<PerformanceEvidence clipSrc="/example.mp4" instruction={{}} />);
- expect(screen.getByText(/Temporal lip-sync: unverified/)).toBeTruthy();
expect(screen.getByText(/Recorded source unavailable/)).toBeTruthy();
});
+
+ // Shot 19 of a music video: a 2.33 s shot inside a 5 s take showed blank word chips, the
+ // next line's words, and "review needed" lines no one could act on without an analyzer.
+ it('lists only the shot\'s own words, explains the longer take, stops at the out point and stays quiet without an analyzer', () => {
+ vi.spyOn(HTMLMediaElement.prototype, 'play').mockResolvedValue();
+ const pause = vi.spyOn(HTMLMediaElement.prototype, 'pause').mockImplementation(() => {});
+ const take = {
+ audio: { conditioning: { source: 'vocal-stem', filename: 'singer.wav' } },
+ audioWindow: { startSec: 40, durationSec: 5 }, edit: { inSec: 1, outSec: 3.33 }, capability: { minAudioSec: 5 },
+ cues: [
+ { text: 'this line', startSec: 0.9, endSec: 3.2, words: [{ w: 'this', startSec: 0.9, endSec: 1.6 }, { text: 'line', startSec: 1.7, endSec: 3.2 }] },
+ { text: 'the next line', startSec: 4.3, endSec: 5 },
+ ],
+ };
+ const { container } = render(<PerformanceEvidence clipSrc="/example.mp4" instruction={take}
+ repair={{ ok: false, message: 'Review needed: complete measured temporal evidence is required' }} onRepair={vi.fn()} />);
+ expect(screen.getByRole('button', { name: 'this' })).toBeTruthy();
+ expect(screen.getByRole('button', { name: 'line' })).toBeTruthy();
+ expect(screen.queryByText(/the next line/)).toBeNull();
+ expect(screen.getByText(/needs at least 5 s of song/)).toBeTruthy();
+ expect(screen.getByText(/Audio: vocal stem/)).toBeTruthy();
+ expect(screen.queryByText(/Review needed|unverified/i)).toBeNull();
+ const video = container.querySelector('video');
+ fireEvent.loadedMetadata(video);
+ fireEvent.play(video);
+ pause.mockClear();
+ video.currentTime = 3.4;
+ fireEvent.timeUpdate(video);
+ expect(pause.mock.instances).toContain(video);
+ vi.restoreAllMocks();
+ });
});
// This interaction verifies the spend/boundary shown before the explicit
@@ -58,7 +87,8 @@ it('shows the accepted boundary and next spend, disables duplicate clicks, and r
expect(onRepair).toHaveBeenCalledTimes(1);
view.rerender(<PerformanceEvidence instruction={instruction} repair={repair} onRepair={onRepair} repairBusy />);
expect(screen.getByRole('button', { name: 'Repairing…' }).disabled).toBe(true);
- view.rerender(<PerformanceEvidence instruction={instruction} repair={{ ok: false, message: 'Review needed: evidence is inconclusive' }} onRepair={onRepair} />);
+ view.rerender(<PerformanceEvidence instruction={instruction} temporal={{ status: 'unverified', analyzer: { id: 'example-analyzer', version: '1' }, spans: [] }}
+ repair={{ ok: false, message: 'Review needed: evidence is inconclusive' }} onRepair={onRepair} />);
expect(screen.queryByRole('button', { name: 'Repair from here' })).toBeNull();
expect(screen.getByText(/evidence is inconclusive/)).toBeTruthy();
});
diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.js
index 27326d6aa0..3e5838be6f 100644
--- a/server/lib/musicVideoShotTiming.js
+++ b/server/lib/musicVideoShotTiming.js
@@ -341,7 +341,8 @@ export function clipRelativeCues(cues, windowStartSec, windowEndSec) {
endSec: round6(Math.min(windowEndSec, end) - windowStartSec),
...(Array.isArray(cue.words) ? { words: cue.words
.filter((word) => word && Number.isFinite(word.startSec) && Number.isFinite(word.endSec) && word.endSec > word.startSec && word.endSec > windowStartSec && word.startSec < windowEndSec)
- .map((word) => ({ text: word.text, startSec: round6(Math.max(windowStartSec, word.startSec) - windowStartSec), endSec: round6(Math.min(windowEndSec, word.endSec) - windowStartSec) })) } : {}),
+ // Aligned lyric words keep their text as `w` (lyricAlignCore), transcribed ones as `text`.
+ .map((word) => ({ text: typeof word.text === 'string' ? word.text : typeof word.w === 'string' ? word.w : '', startSec: round6(Math.max(windowStartSec, word.startSec) - windowStartSec), endSec: round6(Math.min(windowEndSec, word.endSec) - windowStartSec) })) } : {}),
});
}
return out;
diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.js
index ea3d3d5801..2a287cf73a 100644
--- a/server/services/musicVideo/performanceShot.test.js
+++ b/server/services/musicVideo/performanceShot.test.js
@@ -76,7 +76,9 @@ const project = (sceneOver) => ({
id: 'mv-1',
uploadedAudioFilename: 'song.wav',
lyricCues: [
- { id: 'c1', text: 'hold on', startSec: 19, endSec: 20.5, words: [{ text: 'hold', startSec: 19, endSec: 20 }, { text: 'on', startSec: 20, endSec: 20.5 }] },
+ // Aligned words carry their text as `w` (lyricAlignCore); the take records it as `text`.
+ { id: 'c1', text: 'hold on', startSec: 19, endSec: 20.5,
+ words: [{ w: 'hold', startSec: 19, endSec: 20, conf: 'matched' }, { text: 'on', startSec: 20, endSec: 20.5 }] },
{ id: 'c2', text: 'later line', startSec: 40, endSec: 42 },
{ id: 'c3', text: 'untimed', startSec: null, endSec: null },
],