Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Fix split-shot lyric alignment and preserve storyboard approval (#11784)

[._.]/84c08d13d9b4
diff --git a/server/lib/README.md b/server/lib/README.md
index 72ff3796d1..fac6defa07 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -568,7 +568,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `chiptuneRender.js` | Deterministic chiptune score → mono PCM → 16-bit WAV buffer (pure Node, no audio deps); `renderScoreToPcm` / `pcmToWavBuffer` / `renderScoreToWav`. |
| `chiptuneScore.js` | Chiptune score contract (#2911): `chiptuneScoreSchema` (Zod) + `sanitizeChiptuneScore`, `buildScoreEvents` — the pattern/order → absolute-time flatten mirrored by `client/src/lib/chiptunePlayback.js`. |
| `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `STILL_PUSH_ZOOM` / `STILL_PAN_TRAVEL` (shared render/animatic motion amounts), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. `clipSpeedRate(scene)` / `clipOffsetSec(scene)` / `clipReversed(scene)` read a footage shot's hand-set `speedFps`, `offsetFrames` and `reverse` on the 24 fps grid (`CLIP_TIMING_FPS`, `CLIP_SPEED_FPS_RANGE`, `CLIP_OFFSET_FRAMES_RANGE`). Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). |
-| `musicVideoLyricAnchors.js` | Music Video lyric anchors: `cuesSungWithin(cues, startSec, endSec)`, the lines a shot's span anchors, each counted from its first sung word to its last when it has word timing, else its line window. Shared by the storyboard anchor check and the retime anchor sync. |
+| `musicVideoLyricAnchors.js` | Music Video lyric anchors: `cuesSungWithin(cues, startSec, endSec)` and `sungLyricTextWithin(cues, startSec, endSec)` (shot text, capped at 2,000 characters), the lines a shot's span anchors, each counted from its first sung word to its last when it has word timing, else its line window. Shared by the storyboard anchor check and the retime anchor sync. |
| `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`: `flash-develop`, a camera flash then the picture develops like an instant photo over `imageEffectSec`; or a fade, flash or glitch from the start frame to the end frame). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe and import-free: document videos ship it as is. |
| `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections`, `imageEffectGlitchFrames`/`mergeGlitchFrames` (end-frame glitch effects in the same shape), `shotGlitchFrames`/`sceneShotGlitchFrames` (glitches a shot sets at moments inside it, `scene.glitches`), `blockBox`, `edgeGlows` (an effect's edge glows as fading ramps) and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Styles span VHS, film and camera looks. Browser-safe; the storyboard animatic paints the same frames. |
| `pitchMath.js` | Dependency-free scientific-pitch math shared by the LLM-authored music formats on both runtimes: `pitchToMidi("C4")` → 60 (null when unparseable) and `midiToFreq(69)` → 440. |
diff --git a/server/lib/musicVideoLyricAnchors.js b/server/lib/musicVideoLyricAnchors.js
index 9907e9a3b4..40cd442a76 100644
--- a/server/lib/musicVideoLyricAnchors.js
+++ b/server/lib/musicVideoLyricAnchors.js
@@ -21,3 +21,20 @@ export function cuesSungWithin(cues, startSec, endSec) {
return span && span.startSec < endSec && span.endSec > startSec;
});
}
+
+/** The sung lines shown on a shot, or null when its span contains no lyric. */
+export function sungLyricTextWithin(cues, startSec, endSec) {
+ const sung = new Set(cuesSungWithin(cues, startSec, endSec));
+ const lines = (Array.isArray(cues) ? cues : [])
+ .filter((cue) => {
+ if (sung.has(cue)) return true;
+ // Legacy start-only lines still label their split piece. They have no
+ // positive-duration span to anchor; valid word timing always takes priority.
+ const wordTimed = Array.isArray(cue?.words) && cue.words.some((word) => finite(word?.startSec) && finite(word?.endSec));
+ return !wordTimed && finite(cue?.startSec) && cue.endSec == null
+ && cue.startSec < endSec - 1e-6 && cue.startSec >= startSec - 1e-6;
+ })
+ .map((cue) => (typeof cue.text === 'string' ? cue.text.trim() : ''))
+ .filter(Boolean);
+ return lines.length ? lines.join(' / ').slice(0, 2000) : null;
+}
diff --git a/server/services/musicVideo/productionReviewService.js b/server/services/musicVideo/productionReviewService.js
index d38d1f730b..5c8762aa8d 100644
--- a/server/services/musicVideo/productionReviewService.js
+++ b/server/services/musicVideo/productionReviewService.js
@@ -4,7 +4,7 @@ import { getProject, mutateProjectRecord } from './projects.js';
import { musicVideoEvents } from './events.js';
import { shotCameraLabel } from '../../lib/cameraMovements.js';
import { productionReadiness, boardStoryboard, seedArtDraft, productionReviewBasis, productionAlignmentBasis, documentStoryboardBasis, approveProductionStage, assertProductionApproval, assertShotApproval, assertClipUnlocked, setShotLocks, recordProductionFeedback, resolveProductionFeedback, revertApprovedInput, anchorShotsToSungLyrics, clipLocked, dropStaleStoryboardRows, carryStoryboardApproval } from './productionReview.js';
-import { cuesSungWithin } from '../../lib/musicVideoLyricAnchors.js';
+import { sungLyricTextWithin } from '../../lib/musicVideoLyricAnchors.js';
const reviewProcessId = randomUUID();
@@ -327,8 +327,7 @@ export async function syncLyricAnchors(id, { sceneIds = null } = {}) {
const anchored = anchorShotsToSungLyrics(current, ids);
const scenes = (anchored.scenes || []).map(scene => {
if (!ids.includes(scene.sceneId) || clipLocked(current, scene.sceneId)) return scene;
- const sung = cuesSungWithin(anchored.lyricCues, scene.startSec, scene.endSec).map(cue => cue.text?.trim()).filter(Boolean);
- return { ...scene, lyricText: sung.length ? sung.join(' / ').slice(0, 2000) : null };
+ return { ...scene, lyricText: sungLyricTextWithin(anchored.lyricCues, scene.startSec, scene.endSec) };
});
return { project: { ...anchored, scenes, updatedAt: new Date().toISOString() } };
});
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index 0b0089103b..e4f02978f0 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -43,7 +43,7 @@ import { normalizeTitleCard } from '../../lib/musicVideoTitleCard.js';
import { normalizeStatsOutro } from '../../lib/musicVideoStatsOutro.js';
import { remapTreatmentForClone, scenesFingerprint } from './treatment.js';
import { anchorShotsToSungLyrics, carryStoryboardApproval, dropStaleStoryboardRows } from './productionReview.js';
-import { cuesSungWithin } from '../../lib/musicVideoLyricAnchors.js';
+import { cuesSungWithin, sungLyricTextWithin } from '../../lib/musicVideoLyricAnchors.js';
import { normalizeMusicVideoAutomation } from '../../lib/musicVideoAutomation.js';
export { sanitizeProjectForSync } from '../../lib/projectStoreKit.js';
@@ -672,8 +672,7 @@ export function applySceneUpdate(project, sceneId, patch) {
const retimed = (('startSec' in data && data.startSec !== scenes[idx].startSec) || ('endSec' in data && data.endSec !== scenes[idx].endSec))
&& (project.lyricCues || []).some((cue) => typeof cue?.startSec === 'number');
if (retimed && !('lyricText' in data)) {
- const sung = cuesSungWithin(project.lyricCues, updated.startSec, updated.endSec).map((cue) => cue.text?.trim()).filter(Boolean);
- updated.lyricText = sung.length ? sung.join(' / ').slice(0, 2000) : null;
+ updated.lyricText = sungLyricTextWithin(project.lyricCues, updated.startSec, updated.endSec);
}
assertMusicVideoMediaSelections({ ...project, scenes: [updated], visualSpec: null });
const nextScenes = scenes.slice();
@@ -760,23 +759,6 @@ export function reorderScenes(project, orderedIds) {
// Label suffix budget: the scene label schema caps at 120 characters.
const SCENE_LABEL_MAX = 120;
-/**
- * The timed lyric lines sung during `[startSec, endSec]`, joined, or null. A
- * line held across a cut belongs to both pieces.
- */
-function lyricTextWithin(cues, startSec, endSec) {
- const lines = (Array.isArray(cues) ? cues : [])
- .filter((cue) => {
- if (typeof cue?.startSec !== 'number') return false;
- const cueEnd = typeof cue.endSec === 'number' ? cue.endSec : cue.startSec;
- // Starts before the piece ends, and is still sounding (or starts) inside it.
- return cue.startSec < endSec - 1e-6 && (cueEnd > startSec + 1e-6 || cue.startSec >= startSec - 1e-6);
- })
- .map((cue) => (typeof cue.text === 'string' ? cue.text.trim() : ''))
- .filter(Boolean);
- return lines.length ? lines.join(' / ').slice(0, 2000) : null;
-}
-
/**
* Split a shot longer than its backend can render in one take (#8977) into
* contiguous scenes on musical boundaries (lib/musicVideoShotTiming.js
@@ -817,7 +799,7 @@ export function splitScene(project, sceneId, { backend = null, atSec = null } =
return `${baseLabel.slice(0, SCENE_LABEL_MAX - suffix.length)}${suffix}`;
};
const hasTimedCues = (project.lyricCues || []).some((cue) => typeof cue?.startSec === 'number');
- const pieceLyrics = (piece) => (hasTimedCues ? lyricTextWithin(project.lyricCues, piece.startSec, piece.endSec) : scene.lyricText ?? null);
+ const pieceLyrics = (piece) => (hasTimedCues ? sungLyricTextWithin(project.lyricCues, piece.startSec, piece.endSec) : scene.lyricText ?? null);
const pieces = plan.pieces.map((piece, i) => {
const timing = { label: pieceLabel(i), startSec: piece.startSec, endSec: piece.endSec, lyricText: pieceLyrics(piece) };
@@ -873,7 +855,7 @@ export function splitScene(project, sceneId, { backend = null, atSec = null } =
const storyboard = draft.storyboard.flatMap((shot) => (shot === row ? rows : [shot]));
extra.productionReview = { ...project.productionReview, draft: { ...draft, storyboard } };
}
- const next = touch(project, extra);
+ const next = carryStoryboardApproval(project, touch(project, extra), 'Split a shot');
return { project: next, scenes: nextScenes.slice(idx, idx + count) };
}
diff --git a/server/services/musicVideo/projectsLogic.test.js b/server/services/musicVideo/projectsLogic.test.js
index 96bed2300d..a94bf5d382 100644
--- a/server/services/musicVideo/projectsLogic.test.js
+++ b/server/services/musicVideo/projectsLogic.test.js
@@ -18,6 +18,8 @@ import {
mergeProjectRecord,
} from './projectsLogic.js';
+import { productionReviewBasis } from './productionReview.js';
+
const baseProject = () => buildProjectRecord({ name: 'Test MV' }, { id: 'mv-1', now: '2026-01-01T00:00:00.000Z' });
describe('buildProjectRecord', () => {
@@ -593,6 +595,55 @@ describe('scene board operations', () => {
expect(split.project.scenes.map((s) => s.stillMove)).toEqual(['push', 'push']);
});
+ // Catches the split text disagreeing with its storyboard anchors during an instrumental lead-in.
+ it('splits word-timed lyrics by their sung span and keeps retiming consistent', () => {
+ const { project, scene } = addScene(baseProject(), { prompt: 'a', startSec: 3.5, endSec: 8 });
+ project.lyricCues = [{ id: 'cue-1', text: ' Sung line ', startSec: 3.5, endSec: 8,
+ words: [{ w: 'Sung', startSec: 5, endSec: 6 }, { w: 'line', startSec: 6, endSec: 8 }] }];
+ project.productionReview = { draft: { storyboard: [{ id: 'shot-1', sceneId: scene.sceneId, startSec: 3.5, endSec: 8 }] } };
+ const split = splitScene(project, scene.sceneId, { atSec: 4.9 });
+ expect(split.scenes.map(piece => piece.lyricText)).toEqual([null, 'Sung line']);
+ expect(split.project.productionReview.draft.storyboard.map(shot => shot.lyricCueIds)).toEqual([[], ['cue-1']]);
+ expect(applySceneUpdate(split.project, scene.sceneId, { endSec: 4.8 }).updated.lyricText).toBeNull();
+ });
+
+ // Catches a shared text helper changing existing unaligned lyric formatting or the field's size cap.
+ it('keeps line-window lyrics, formatting and length limits on a split', () => {
+ const { project, scene } = addScene(baseProject(), { prompt: 'a', startSec: 0, endSec: 8 });
+ project.lyricCues = [
+ { id: 'cue-1', text: ' First ', startSec: 0, endSec: 4 },
+ { id: 'cue-2', text: 'Second', startSec: 1, endSec: 4 },
+ { id: 'cue-3', text: 'x'.repeat(2100), startSec: 4, endSec: 8 },
+ ];
+ const split = splitScene(project, scene.sceneId, { atSec: 4 });
+ expect(split.scenes.map(piece => piece.lyricText)).toEqual(['First / Second', 'x'.repeat(2000)]);
+ });
+
+ // Catches valid legacy start-only cues losing their text during a split.
+ it.each([null, undefined])('keeps a start-only lyric on its split piece (endSec: %s)', endSec => {
+ const { project, scene } = addScene(baseProject(), { prompt: 'a', startSec: 0, endSec: 8 });
+ project.lyricCues = [{ id: 'cue-1', text: 'Line', startSec: 1, endSec }];
+ const split = splitScene(project, scene.sceneId, { atSec: 4 });
+ expect(split.scenes.map(piece => piece.lyricText)).toEqual(['Line', null]);
+ });
+
+ // Catches a split staling a whole-board approval or reviving one that was already stale.
+ it.each([true, false])('carries only a current whole-board approval across a split (current: %s)', current => {
+ const { project, scene } = addScene(baseProject(), { prompt: 'a', startSec: 0, endSec: 8 });
+ const basis = productionReviewBasis(project);
+ const approval = { basis: current ? basis.storyboard : 'stale', shotsBasis: current ? basis.storyboardShots : 'stale' };
+ project.productionReview = { approvals: { storyboard: approval } };
+ const next = splitScene(project, scene.sceneId, { atSec: 4 }).project;
+ const after = next.productionReview.approvals.storyboard;
+ if (current) {
+ expect(after).toMatchObject({ basis: productionReviewBasis(next).storyboard,
+ shotsBasis: productionReviewBasis(next).storyboardShots, carriedOver: { reason: 'Split a shot' } });
+ expect(after.shotsBasis).not.toBe(basis.storyboardShots);
+ } else {
+ expect(after).toEqual(approval);
+ }
+ });
+
it('persists a shot\'s lyric-type zone and role and refuses unknown ones (#10583)', () => {
const { project, scene } = addScene(baseProject(), { prompt: 'a' });
const { updated } = applySceneUpdate(project, scene.sceneId, { textZone: 'none', lyricRole: 'hook' });