Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Draw the timed lyrics in composed final renders (#11557)

[._.]/9c40f77a1ad1
diff --git a/client/src/components/musicVideo/TypographyPanel.jsx b/client/src/components/musicVideo/TypographyPanel.jsx
index 9640b74b1f..95e3d0c865 100644
--- a/client/src/components/musicVideo/TypographyPanel.jsx
+++ b/client/src/components/musicVideo/TypographyPanel.jsx
@@ -1,5 +1,6 @@
import { Copy, Plus, Trash2 } from 'lucide-react';
import { compositionDraft, renderStyleLabel } from './compositionDraft.js';
+import { composedDrawsLyrics } from '../../lib/musicVideoLayers.js';
// Client-minted ids keep a freshly added row addressable across saves (see LyricsPanel).
const mintId = () => `mtc-${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`;
@@ -74,7 +75,7 @@ export default function TypographyPanel({ project, onEditLocal, onSave }) {
onBlur={() => save()} className={`${inputCls} w-20`} />
</div>
<button type="button" onClick={copyLyrics} disabled={timedLyrics.length === 0}
- title={timedLyrics.length === 0 ? 'Time some lyric lines first' : 'Replace the text cues with the timed lyric lines'}
+ title={timedLyrics.length === 0 ? 'Time some lyric lines first' : 'Replace the text cues with the timed lyric lines as plain subtitles (they replace the lyric type in the final render)'}
className="flex items-center gap-1 text-port-accent min-h-[44px] sm:min-h-0 disabled:opacity-50">
<Copy size={13} /> Copy timed lyrics ({timedLyrics.length})
</button>
@@ -83,7 +84,8 @@ export default function TypographyPanel({ project, onEditLocal, onSave }) {
className="flex items-center gap-1 text-port-accent min-h-[44px] sm:min-h-0"><Plus size={13} /> Text cue</button>
</div>
- {composed && cues.length === 0 && <p className="mt-2 text-port-text-muted">No text cues yet — a composed render with none renders as plain cuts.</p>}
+ {composed && composedDrawsLyrics({ ...project, composition }) && <p className="mt-2 text-port-text-muted">The final render draws your timed lyrics, as in the player. Text cues are optional; a Subtitle cue replaces the lyrics.</p>}
+ {composed && !composedDrawsLyrics({ ...project, composition }) && cues.length === 0 && <p className="mt-2 text-port-text-muted">No lyrics or text cues to draw, so the final render is plain cuts.</p>}
{footage && cues.length > 0 && <p className="mt-2 text-port-warning" role="status">Footage renders plain cuts, so these text cues will not appear. Choose Composed in Project settings to lay them over the footage.</p>}
{codeMode && <p className="mt-2 text-port-text-muted">Code-rendered draws the song in code. Text cues stay stored and come back if you switch to Composed.</p>}
{composition.mode === 'document' && <p className="mt-2 text-port-text-muted">The composition document reads these cues from <code>PORTOS_MV.textCues</code>: the layered template draws Hero cues as kinetic words on their sung times and uses Subtitle cues (or, with none, the timed lyrics) as subtitles.</p>}
diff --git a/client/src/lib/musicVideoLayers.js b/client/src/lib/musicVideoLayers.js
index 96abb21804..21017de8b2 100644
--- a/client/src/lib/musicVideoLayers.js
+++ b/client/src/lib/musicVideoLayers.js
@@ -9,6 +9,7 @@ export {
MUSIC_VIDEO_FRAME_FITS,
MUSIC_VIDEO_TEXT_ZONES,
MUSIC_VIDEO_LYRIC_ROLES,
+ composedDrawsLyrics,
isSelfDrawnLayer,
sceneVisualLayer,
sceneHasAuthoredSpan,
diff --git a/client/src/lib/musicVideoStages.js b/client/src/lib/musicVideoStages.js
index eae7189152..dc11fdb28d 100644
--- a/client/src/lib/musicVideoStages.js
+++ b/client/src/lib/musicVideoStages.js
@@ -4,7 +4,7 @@
* spend. Pure functions over the project record so the header, the progress
* strip and the tests all read the same answer.
*/
-import { isLayeredComposition, isSelfDrawnLayer, sceneRenderReady, sceneVisualLayer } from './musicVideoLayers.js';
+import { composedDrawsLyrics, isLayeredComposition, isSelfDrawnLayer, sceneRenderReady, sceneVisualLayer } from './musicVideoLayers.js';
import {
AUTONOMOUS_CHECKPOINT_LABELS, AUTONOMOUS_LYRICS_STEP_LABELS, AUTONOMOUS_SONG_STEP_LABELS, AUTONOMOUS_STATUS_LABELS,
} from './musicVideoAutonomous.js';
@@ -375,7 +375,7 @@ const codeComposed = (project) => {
};
const composeDone = (project, mode) => {
- if (mode === 'composed') return (project.composition?.textCues || []).length > 0;
+ if (mode === 'composed') return (project.composition?.textCues || []).length > 0 || composedDrawsLyrics(project);
if (mode === 'document') return !!project.composition?.document;
if (mode === 'eidoverse') return !!project.composition?.eidoverseScene?.inlineScript;
if (mode === 'code') return codeComposed(project);
@@ -1030,7 +1030,13 @@ export function stageChecklist(stageId, project, readiness = project?.production
? [{ ...proofApproval, label: proofApproval.done ? proofApproval.label : 'Animated proof (optional)', optional: true, detail: null }] : [];
const composition = project.composition || {};
let work = null;
- if (mode === 'composed') work = { id: 'composition', label: 'Timed typography added', done: (composition.textCues || []).length > 0, action: { label: 'Add typography', anchor: 'mv-typo-font' } };
+ if (mode === 'composed') {
+ const cues = (composition.textCues || []).length > 0;
+ // Timed lyrics with no subtitle cues are drawn by the render itself, as the player shows them.
+ work = composedDrawsLyrics(project)
+ ? { id: 'composition', label: 'Lyrics drawn over the footage', done: true }
+ : { id: 'composition', label: 'Timed typography added', done: cues, action: { label: 'Add typography', anchor: 'mv-typo-font' } };
+ }
else if (mode === 'document') work = { id: 'composition', label: 'Composition document attached', done: !!composition.document, action: { label: 'Attach a document', anchor: 'mv-doc-folder' } };
else if (mode === 'eidoverse') {
const saved = !!composition.eidoverseScene?.inlineScript;
diff --git a/client/src/lib/musicVideoStages.test.js b/client/src/lib/musicVideoStages.test.js
index 28d2ea9c9a..e5201f5694 100644
--- a/client/src/lib/musicVideoStages.test.js
+++ b/client/src/lib/musicVideoStages.test.js
@@ -259,6 +259,20 @@ describe('following the tab order', () => {
.toEqual([['footage', true, false, 'Footage for every shot (1 of 1)'], ['approve-storyboard', true, false, 'Every still locked'], ['composition', true, false, 'Timed typography added']]);
});
+ it('counts a composed render\'s own lyrics as its typography, so it asks for no text cues', () => {
+ const composition = { mode: 'composed', textCues: [] };
+ const lyricCues = [{ text: 'found a shot', startSec: 1, endSec: 2 }];
+ const project = { id: 'p', trackId: 't1', audioAnalysis: ANALYSIS, scenes: [scene()], composition, lyricCues };
+ const readiness = readinessFor(project, { art: true, storyboard: true, approvedComposition: JSON.stringify(composition) });
+ expect(stageChecklist('produce', project, readiness).find((i) => i.id === 'composition'))
+ .toMatchObject({ done: true, label: 'Lyrics drawn over the footage' });
+ expect(deriveNextAction(project, { readiness })).not.toMatchObject({ id: 'goto-compose' });
+ // Without timed lyrics there is nothing to draw, so the task still asks for text.
+ const bare = { ...project, lyricCues: [] };
+ expect(stageChecklist('produce', bare, readinessFor(bare, { art: true, storyboard: true })).find((i) => i.id === 'composition'))
+ .toMatchObject({ done: false, label: 'Timed typography added' });
+ });
+
it('keeps the header on the footage until it is in, unless a run is parked on its pilot proof', () => {
const project = { id: 'p', trackId: 't1', audioAnalysis: ANALYSIS, scenes: [scene({ videoHistoryId: null })] };
const readiness = readinessFor(project, { art: true, storyboard: true });
diff --git a/docs/HTML_COMPOSITIONS.md b/docs/HTML_COMPOSITIONS.md
index f0a758a63a..65575e5478 100644
--- a/docs/HTML_COMPOSITIONS.md
+++ b/docs/HTML_COMPOSITIONS.md
@@ -233,6 +233,17 @@ only audio. A cue's state at a time is a pure function of that time
The scratch directory is removed when the render ends, swept at boot, and
excluded from backups.
+With timed lyrics and no Subtitle text cues (Hero cues are titles over the
+words, not a stand-in for them), a composed render also draws the sung lyrics
+in the shared lyric type, exactly as the storyboard animatic shows them: it
+stages the animatic's lyric overlay page (`documentTemplates/lyricOverlay`)
+beside the typography page and captures it over each line's window
+(`composedLyricWindows`, the first onset to two seconds past the line's end).
+Each shot's Text zone and Lyric role apply; a title card shot already draws its
+text (else its sung line), so the overlay keeps its lines off it. A Subtitle cue hands the words
+back to the text cues, so "Copy timed lyrics" swaps the lyric type for plain
+subtitles.
+
A composed render also honors each scene's visual layer (#8985): its
generated footage (the default), its selected still frame with a deterministic
hold, push-in or pan, or a title card — a solid colour whose text is drawn by
diff --git a/server/lib/musicVideoLayers.js b/server/lib/musicVideoLayers.js
index 3833095ae0..d2f8385558 100644
--- a/server/lib/musicVideoLayers.js
+++ b/server/lib/musicVideoLayers.js
@@ -39,6 +39,21 @@ export const MUSIC_VIDEO_FRAME_FITS = ['fill', 'still', 'vhs', 'black'];
export const MUSIC_VIDEO_TEXT_ZONES = ['lower-left', 'upper-right', 'upper', 'lower', 'center', 'none'];
export const MUSIC_VIDEO_LYRIC_ROLES = ['line', 'hook', 'stamp', 'data'];
+/**
+ * Whether a composed render draws the sung lyrics itself, in the shared lyric
+ * type the storyboard animatic shows: a composed project with timed lyrics and
+ * no subtitle text cues (subtitle cues stand in for the lyrics; a hero cue is a
+ * title over them).
+ */
+export function composedDrawsLyrics(project) {
+ const composition = project?.composition;
+ if (composition?.mode !== 'composed') return false;
+ const timed = (cue) => cue?.text && Number.isFinite(cue.startSec) && Number.isFinite(cue.endSec);
+ if ((composition.textCues || []).some((cue) => timed(cue) && cue.emphasis !== 'hero')) return false;
+ return (Array.isArray(project.lyricCues) ? project.lyricCues : [])
+ .some((cue) => typeof cue?.text === 'string' && cue.text.trim() && Number.isFinite(cue.startSec));
+}
+
/**
* Whether a composition honours per-scene layers: a composed render cuts
* stills and cards itself, and a composition document (mode `document`) is
diff --git a/server/lib/musicVideoSummary.js b/server/lib/musicVideoSummary.js
index 47827d0c4a..bd07724b4d 100644
--- a/server/lib/musicVideoSummary.js
+++ b/server/lib/musicVideoSummary.js
@@ -7,7 +7,7 @@
* / spend / preview rules mirror `client/src/lib/musicVideoStages.js` and
* `musicVideoPreview.js`, which still derive them for a full record.
*/
-import { isLayeredComposition, sceneRenderReady } from './musicVideoLayers.js';
+import { composedDrawsLyrics, isLayeredComposition, sceneRenderReady } from './musicVideoLayers.js';
import { latestMusicVideoReviewDraft } from './musicVideoReviewDraft.js';
import { finishedOutsideCovers } from './musicVideoFinishedOutside.js';
@@ -54,7 +54,7 @@ function projectSpend(project, run) {
function composeDone(project, mode) {
const composition = project.composition || {};
- if (mode === 'composed') return (composition.textCues || []).length > 0;
+ if (mode === 'composed') return (composition.textCues || []).length > 0 || composedDrawsLyrics(project);
if (mode === 'document') return Boolean(composition.document);
if (mode === 'eidoverse') return Boolean(composition.eidoverseScene?.inlineScript);
if (mode === 'code') return Boolean(composition.codeVideo?.generatedAt || (composition.codeVideo?.sections || []).length > 0);
diff --git a/server/services/musicVideo/composition.js b/server/services/musicVideo/composition.js
index 8c2f2576ac..7f008d2366 100644
--- a/server/services/musicVideo/composition.js
+++ b/server/services/musicVideo/composition.js
@@ -20,6 +20,7 @@ import { normalizeMusicVideoGrade } from '../../lib/musicVideoGrade.js';
import { randomUUID } from 'crypto';
import { trimTo } from '../../lib/textUtils.js';
import { normalizeTitleCard, titleCardWindows } from '../../lib/musicVideoTitleCard.js';
+import { composedDrawsLyrics } from '../../lib/musicVideoLayers.js';
import { titleCardRuntime } from './titleCardFonts.js';
import { filmStyleIdSchema } from '../../lib/filmStyleGrammarValidation.js';
import {
@@ -247,6 +248,31 @@ function titleCardCues(project, durationSec) {
.map((w, i) => ({ id: i ? 'title-card-end' : 'title-card', titleCard: card, startSec: w.startSec, endSec: Math.min(w.endSec, durationSec) }));
}
+// How long after a lyric line ends its window stays open, so the lyric type's
+// last-word hold, exit fade and hook beat cut are captured whole.
+export const LYRIC_WINDOW_TAIL_SEC = 2;
+
+/**
+ * The windows a composed render draws the sung lyrics in, with the same shared
+ * lyric type the storyboard animatic shows (documentTemplates/lyricOverlay),
+ * when `composedDrawsLyrics` (no subtitle text cues). Each timed line opens at
+ * its first onset and closes LYRIC_WINDOW_TAIL_SEC after it ends, clipped to
+ * the video. Empty for any other render mode.
+ */
+export function composedLyricWindows(project, durationSec) {
+ if (!composedDrawsLyrics(project)) return [];
+ const finiteNum = (n) => typeof n === 'number' && Number.isFinite(n);
+ const spans = [];
+ for (const cue of Array.isArray(project?.lyricCues) ? project.lyricCues : []) {
+ if (typeof cue?.text !== 'string' || !cue.text.trim() || !finiteNum(cue.startSec) || cue.startSec >= durationSec) continue;
+ const words = Array.isArray(cue.words) ? cue.words : [];
+ const lastSec = Math.max(cue.startSec, finiteNum(cue.endSec) ? cue.endSec : cue.startSec,
+ ...words.flatMap((w) => [w?.startSec, w?.endSec]).filter(finiteNum));
+ spans.push({ startSec: cue.startSec, endSec: Math.min(durationSec, lastSec + LYRIC_WINDOW_TAIL_SEC) });
+ }
+ return overlayWindows(spans);
+}
+
/** Shared absolute-song-time cues and style for full renders and draft excerpts. */
export function projectTypographyPlan(project, clips, sections, durationSec) {
const graphicLanguage = project.treatment?.brief?.graphicLanguage;
diff --git a/server/services/musicVideo/composition.test.js b/server/services/musicVideo/composition.test.js
index 4e3afa87a9..bba2461ba6 100644
--- a/server/services/musicVideo/composition.test.js
+++ b/server/services/musicVideo/composition.test.js
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
-import { buildTypographyDocument, cueStateAt, normalizeComposition, overlayWindows, projectTypographyPlan, renderableCues, sectionCardCues } from './composition.js';
+import { buildTypographyDocument, composedLyricWindows, cueStateAt, normalizeComposition, overlayWindows, projectTypographyPlan, renderableCues, sectionCardCues } from './composition.js';
const cue = (over = {}) => ({ id: 'c', text: 'Hello world', startSec: 10, endSec: 12, template: 'fade', placement: 'lower', emphasis: 'subtitle', ...over });
@@ -88,6 +88,31 @@ describe('overlayWindows (#8984)', () => {
});
});
+describe('composedLyricWindows', () => {
+ const lyricCues = [
+ { text: 'found a shot', startSec: 1, endSec: 1.6, words: [{ w: 'found', startSec: 1, endSec: 1.2 }, { w: 'shot', startSec: 1.4, endSec: 1.9 }] },
+ { text: 'then changed the scene', startSec: 10, endSec: 11 },
+ { text: 'past the end', startSec: 30, endSec: 31 },
+ ];
+ const project = (composition) => ({ composition, lyricCues });
+
+ it('draws the timed lyrics of a composed render with no subtitle cues, holding past each line', () => {
+ expect(composedLyricWindows(project({ mode: 'composed', textCues: [] }), 20)).toEqual([
+ { startSec: 1, endSec: 3.9 }, { startSec: 10, endSec: 13 },
+ ]);
+ // A hero cue is a title over the lyrics, not a stand-in for them.
+ const hero = { id: 'h', text: 'TITLE', startSec: 0, endSec: 2, emphasis: 'hero' };
+ expect(composedLyricWindows(project({ mode: 'composed', textCues: [hero] }), 20)).toHaveLength(2);
+ });
+
+ it('leaves the lyrics to subtitle cues, and to every other render mode', () => {
+ const subtitle = { id: 's', text: 'my own line', startSec: 0, endSec: 2, emphasis: 'subtitle' };
+ expect(composedLyricWindows(project({ mode: 'composed', textCues: [subtitle] }), 20)).toEqual([]);
+ expect(composedLyricWindows(project({ mode: 'concat', textCues: [] }), 20)).toEqual([]);
+ expect(composedLyricWindows(project(undefined), 20)).toEqual([]);
+ });
+});
+
describe('card cues (#11185)', () => {
it('turns the sheet joiner into real line breaks and draws in the lyric type look', () => {
const cards = sectionCardCues(
diff --git a/server/services/musicVideo/compositionRender.js b/server/services/musicVideo/compositionRender.js
index 2e575266ad..1b91389295 100644
--- a/server/services/musicVideo/compositionRender.js
+++ b/server/services/musicVideo/compositionRender.js
@@ -14,7 +14,7 @@
* it once the final render finishes, and boot recovery sweeps any a crash left.
*/
-import { join } from 'path';
+import { dirname, join } from 'path';
import { copyFile, mkdir, rm, writeFile } from 'fs/promises';
import { fileURLToPath } from 'url';
import { spawn } from '../../lib/childProcess.js';
@@ -74,11 +74,6 @@ async function captureWindow(page, ffmpeg, { startSec, frames, fps, outputPath,
}
}
-/**
- * Render the overlay clips for `cues` (already filtered by renderableCues) on a
- * `width`×`height`, `fps` frame. Resolves `[{ path, startSec, durationSec }]` in
- * time order. `onProgress(0..1)` reports captured frames; `signal` cancels.
- */
export function musicVideoSongDocument(project) {
const analysis = project?.audioAnalysis && typeof project.audioAnalysis === 'object' && !Array.isArray(project.audioAnalysis)
? project.audioAnalysis : {};
@@ -127,45 +122,78 @@ export async function renderSongComposition({ project, directory, jobId, audioPa
});
}
-export async function renderTypographyOverlays({ jobId, cues, style, width, height, fps, durationSec, signal, onProgress }) {
+/**
+ * Stage the storyboard animatic's lyric overlay page (documentTemplates/lyricOverlay,
+ * with the shared lyricType.js and its faces) in `dir`, drawing `project`'s lyrics
+ * at the render's frame, so a composed render's words match the animatic's.
+ */
+async function stageLyricOverlay(dir, project, { width, height, fps, durationSec }) {
+ const [{ readLyricOverlayDocumentFiles }, { lyricOverlayDocumentData }] = await Promise.all([
+ import('./compositionDocument.js'), import('./lyricPlaythrough.js'),
+ ]);
+ for (const [rel, file] of await readLyricOverlayDocumentFiles()) {
+ await mkdir(dirname(join(dir, rel)), { recursive: true });
+ await copyFile(file.abs, join(dir, rel));
+ }
+ const data = lyricOverlayDocumentData(project, { width, height, fps, durationSec });
+ await writeFile(join(dir, 'portos-mv.js'), `window.PORTOS_MV = ${JSON.stringify(data).replace(/</g, '\\u003c')};\n`);
+}
+
+/**
+ * Render the overlay clips on a `width`×`height`, `fps` frame: the timed text
+ * `cues` (already filtered by renderableCues) on the typography page, and, when
+ * `lyrics` is `{ project, windows }` (composedLyricWindows), the sung lyrics in
+ * the shared lyric type over those windows. Resolves `[{ path, startSec,
+ * durationSec }]`, typography clips first, each set in time order.
+ * `onProgress(0..1)` reports captured frames; `signal` cancels.
+ */
+export async function renderTypographyOverlays({ jobId, cues, style, width, height, fps, durationSec, lyrics = null, signal, onProgress }) {
// Snap each window outward to the output frame grid so every overlay frame
// lands exactly on a footage frame, and never past the video's last frame.
const lastFrame = Math.round(durationSec * fps);
- const windows = overlayWindows(cues)
+ const onGrid = (spans) => spans
.map((w) => {
const first = Math.floor(w.startSec * fps + 1e-6);
return { startSec: first / fps, frames: Math.min(lastFrame, Math.ceil(w.endSec * fps - 1e-6)) - first };
})
.filter((w) => w.frames > 0);
- if (windows.length === 0) return [];
+ const pages = [
+ { name: 'doc', windows: onGrid(overlayWindows(cues)), stage: async (dir) => {
+ await writeFile(join(dir, 'index.html'), buildTypographyDocument({ cues, style, width, height, durationSec, fps }));
+ if (cues.some((c) => c.card || c.titleCard)) await copyFile(LYRIC_FONT_SOURCE, join(dir, LYRIC_FONT_FILE));
+ } },
+ { name: 'lyrics', windows: lyrics ? onGrid(lyrics.windows) : [], stage: (dir) => stageLyricOverlay(dir, lyrics.project, { width, height, fps, durationSec }) },
+ ].filter((p) => p.windows.length > 0);
+ if (pages.length === 0) return [];
const ffmpeg = await findFfmpeg();
if (!ffmpeg) throw new Error('ffmpeg not found on PATH');
const root = scratchPath(jobId);
- await mkdir(join(root, 'doc'), { recursive: true });
- await writeFile(join(root, 'doc', 'index.html'), buildTypographyDocument({ cues, style, width, height, durationSec, fps }));
- if (cues.some((c) => c.card || c.titleCard)) await copyFile(LYRIC_FONT_SOURCE, join(root, 'doc', LYRIC_FONT_FILE));
- signal?.throwIfAborted();
// Lazy: the composition browser drags in browserService, which only a
// composed render needs — not every suite that reaches render.js.
const { openComposition } = await import('../htmlComposition/browser.js');
- const page = await openComposition(`${COMPOSITION_SCRATCH_DIR}/${jobId}/doc`, { signal });
- try {
- await page.send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: false });
- // A transparent default background is what gives the PNG captures alpha.
- await page.send('Emulation.setDefaultBackgroundColorOverride', { color: { r: 0, g: 0, b: 0, a: 0 } });
- await page.evaluate(`globalThis.portosComposition.layout({ width: ${width}, height: ${height} })`);
- const total = windows.reduce((sum, w) => sum + w.frames, 0);
- let done = 0;
- const overlays = [];
- for (const [index, w] of windows.entries()) {
- const outputPath = join(root, `overlay-${index}.mov`);
- await captureWindow(page, ffmpeg, { startSec: w.startSec, frames: w.frames, fps, outputPath, signal,
- onFrame: () => { done += 1; onProgress?.(done / total); } });
- overlays.push({ path: outputPath, startSec: w.startSec, durationSec: w.frames / fps });
+ const total = pages.reduce((sum, p) => sum + p.windows.reduce((n, w) => n + w.frames, 0), 0);
+ let done = 0;
+ const overlays = [];
+ for (const { name, windows, stage } of pages) {
+ await mkdir(join(root, name), { recursive: true });
+ await stage(join(root, name));
+ signal?.throwIfAborted();
+ const page = await openComposition(`${COMPOSITION_SCRATCH_DIR}/${jobId}/${name}`, { signal });
+ try {
+ await page.send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 1, mobile: false });
+ // A transparent default background is what gives the PNG captures alpha.
+ await page.send('Emulation.setDefaultBackgroundColorOverride', { color: { r: 0, g: 0, b: 0, a: 0 } });
+ await page.evaluate(`globalThis.portosComposition.layout({ width: ${width}, height: ${height} })`);
+ for (const [index, w] of windows.entries()) {
+ const outputPath = join(root, `${name}-overlay-${index}.mov`);
+ await captureWindow(page, ffmpeg, { startSec: w.startSec, frames: w.frames, fps, outputPath, signal,
+ onFrame: () => { done += 1; onProgress?.(done / total); } });
+ overlays.push({ path: outputPath, startSec: w.startSec, durationSec: w.frames / fps });
+ }
+ await page.close({ verify: true });
+ } finally {
+ await page.close();
}
- await page.close({ verify: true });
- return overlays;
- } finally {
- await page.close();
}
+ return overlays;
}
diff --git a/server/services/musicVideo/compositionRender.test.js b/server/services/musicVideo/compositionRender.test.js
index 332e938e6e..18cbbbf288 100644
--- a/server/services/musicVideo/compositionRender.test.js
+++ b/server/services/musicVideo/compositionRender.test.js
@@ -9,7 +9,7 @@ import { findFfmpeg, probeVideoDuration } from '../../lib/ffmpeg.js';
import { PATHS } from '../../lib/fileUtils.js';
import { _cleanupTestBrowser } from '../htmlComposition/testBrowserCleanup.js';
import { beatSnapClips, buildMusicVideoFfmpegArgs, resolveSceneClips } from './render.js';
-import { projectTypographyPlan, renderableCues, sectionCardCues } from './composition.js';
+import { composedLyricWindows, projectTypographyPlan, renderableCues, sectionCardCues } from './composition.js';
import { COMPOSITION_SCRATCH_DIR, removeCompositionScratch, renderTypographyOverlays } from './compositionRender.js';
let endpoint;
@@ -134,6 +134,36 @@ describe.skipIf(!chrome || !ffmpeg)('music-video typography overlay with real Ch
expect(await readdir(join(PATHS.data, COMPOSITION_SCRATCH_DIR))).toEqual([]);
}, 60000);
+ it('draws a composed render\'s lyrics in the shared lyric type when it has no subtitle cues', async () => {
+ const [w, h] = [320, 180];
+ const words = (text, start) => text.split(' ').map((word, i) => ({ w: word, startSec: start + i * 0.3, endSec: start + i * 0.3 + 0.25 }));
+ const project = {
+ audioAnalysis: { durationSec: 4 }, lyricMarkers: [], scenes: [],
+ lyricCues: [{ text: 'found a shot', startSec: 1, endSec: 1.85, words: words('found a shot', 1) }],
+ composition: { mode: 'composed', textCues: [] },
+ };
+ const windows = composedLyricWindows(project, 4);
+ const overlays = await renderTypographyOverlays({ jobId: 'job-lyrics', cues: [], style: {}, width: w, height: h, fps: FPS, durationSec: 4,
+ lyrics: { project, windows } });
+ expect(overlays).toEqual([{ path: expect.stringContaining('lyrics-overlay-0.mov'), startSec: 1, durationSec: 69 / FPS }]); // 1s to 1.85s + the 2s tail, snapped out to whole frames
+ const raw = frames(overlays[0].path, 'rgba');
+ const frameBytes = w * h * 4;
+ const drawn = (f) => {
+ const box = { n: 0, minY: h };
+ for (let p = 0; p < w * h; p++) {
+ if (raw[f * frameBytes + p * 4 + 3] <= 8) continue;
+ box.n += 1; box.minY = Math.min(box.minY, Math.floor(p / w));
+ }
+ return box;
+ };
+ // The whole line is up after its last onset, in the lower-left zone, and it has left by the window's end.
+ const shown = drawn(Math.round(0.9 * FPS));
+ expect(shown.n).toBeGreaterThan(100);
+ expect(shown.minY).toBeGreaterThan(h / 2);
+ expect(drawn(raw.length / frameBytes - 1).n).toBe(0);
+ await removeCompositionScratch('job-lyrics');
+ }, 60000);
+
// #8985 acceptance: footage, a still with a push-in, a code-rendered title
// card and animated text on one 30s timebase under one continuous song.
it('renders footage, a still, a title card and text on one timebase without gaps, repeats or audio resets', async () => {
diff --git a/server/services/musicVideo/excerptRender.js b/server/services/musicVideo/excerptRender.js
index f893fc2bbb..720a00243c 100644
--- a/server/services/musicVideo/excerptRender.js
+++ b/server/services/musicVideo/excerptRender.js
@@ -38,7 +38,7 @@ import { assertCurrentPerformanceTakes, withDraftPerformanceFallbacks } from './
import { assertCurrentClipDependencies, planMusicVideoRender, buildMusicVideoFfmpegArgs, excerptBoundaryTimes, isLocalRenderMark, resolveMasterAudioPath, resolveSoundBedPath } from './render.js';
import { encodeCodeComposition, prepareCodeRender, writeCodeProofSheet } from './codeRender.js';
import { encodeDocumentComposition, prepareDocumentRender } from './documentRender.js';
-import { projectTypographyPlan } from './composition.js';
+import { composedLyricWindows, projectTypographyPlan } from './composition.js';
import { renderTypographyOverlays, removeCompositionScratch } from './compositionRender.js';
import { applyFilmLookFinish, filmLookFinishNeeded } from './filmLookFinish.js';
import { startExcerptOnProject, applyExcerptPatch } from './excerpt.js';
@@ -326,7 +326,9 @@ export async function startExcerptRender(projectId, { startSec, endSec, aspect =
const { cues: plannedCues, style } = projectTypographyPlan(project, clips, probe.sections, probe.totalDuration);
const allCues = composed ? plannedCues : [];
const cues = allCues.filter((c) => c.startSec < endClamped && c.endSec > startSec);
- const composition = cues.length > 0 ? project.composition : null;
+ const lyricWindows = (composed ? composedLyricWindows(project, endClamped) : [])
+ .filter((w) => w.endSec > startSec).map((w) => ({ ...w, startSec: Math.max(w.startSec, startSec) }));
+ const composition = cues.length > 0 || lyricWindows.length > 0 ? project.composition : null;
const totalDuration = endClamped - startSec;
// A long filter graph goes to ffmpeg in this file, removed once ffmpeg exits.
@@ -500,6 +502,7 @@ export async function startExcerptRender(projectId, { startSec, endSec, aspect =
const captureCues = cues.map((c) => ({ ...c, startSec: Math.max(c.startSec, startSec) }));
renderTypographyOverlays({
jobId, cues: captureCues, style, width: probe.canonW, height: probe.canonH, fps: probe.fps, durationSec: endClamped, signal,
+ lyrics: lyricWindows.length > 0 ? { project, windows: lyricWindows } : null,
onProgress: (fraction) => broadcastSse(job, { type: 'progress', progress: 0.5 * fraction }),
}).then(async (overlays) => {
signal.throwIfAborted();
diff --git a/server/services/musicVideo/lyricPlaythrough.js b/server/services/musicVideo/lyricPlaythrough.js
index 183478c5dd..ebde241f92 100644
--- a/server/services/musicVideo/lyricPlaythrough.js
+++ b/server/services/musicVideo/lyricPlaythrough.js
@@ -18,9 +18,10 @@
import { ServerError } from '../../lib/errorHandler.js';
import { isNonBlankStr } from '../../lib/textUtils.js';
+import { sceneVisualLayer } from '../../lib/musicVideoLayers.js';
import { buildDocumentPreview } from './documentPreview.js';
import { readLyricOverlayDocumentFiles, readTemplateDocumentFiles } from './compositionDocument.js';
-import { documentSongDuration } from './documentRender.js';
+import { buildDocumentData, documentRenderClock, documentSongDuration } from './documentRender.js';
const LINES_PER_SCENE = 4;
const finite = (n) => typeof n === 'number' && Number.isFinite(n);
@@ -78,11 +79,31 @@ export async function buildLyricPlaythroughPreview(project, { overlay = false }
if (!cues.some((cue) => finite(cue?.startSec) && isNonBlankStr(cue?.text))) {
throw new ServerError('Align the words before playing the lyrics through', { status: 409, code: 'NO_TIMED_LYRICS' });
}
+ return buildDocumentPreview(playthroughProject(project, durationSec), { files: overlay ? await readLyricOverlayDocumentFiles() : await readTemplateDocumentFiles('layered') });
+}
+
+// Only the song, the words, the shots' timing and the opening title card: no composition, overlay text,
+// narrative events or film look of the project's own. The look would filter the whole page, and on the
+// transparent overlay that dims the frames under the words.
+function playthroughProject(project, durationSec) {
const shots = timedShots(project.scenes);
- const scenes = shots.length ? shots : lyricScenes(cues, project.lyricMarkers, durationSec);
- // Only the song, the words and the opening title card: no composition, overlay text, narrative events or film look of the
- // project's own. The look would filter the whole page, and on the transparent overlay that dims the
- // storyboard frames under the words.
- const playthrough = { ...project, scenes, composition: { mode: 'document', textCues: [] }, filmLook: null };
- return buildDocumentPreview(playthrough, { files: overlay ? await readLyricOverlayDocumentFiles() : await readTemplateDocumentFiles('layered') });
+ const scenes = shots.length ? shots : lyricScenes(project.lyricCues || [], project.lyricMarkers, durationSec);
+ return { ...project, scenes, composition: { mode: 'document', textCues: [] }, filmLook: null };
+}
+
+/**
+ * The `window.PORTOS_MV` the lyric overlay page draws in a composed final render
+ * (compositionRender.js), at that render's frame: the same words, roles and text
+ * zones the storyboard animatic lays over its frames. A title card shot already
+ * draws its text (its own, else its sung line) on the typography page, so its
+ * lines are kept off it here.
+ */
+export function lyricOverlayDocumentData(project, { width, height, fps, durationSec }) {
+ const songDurationSec = documentSongDuration(project) || durationSec;
+ const cards = new Set((project.scenes || []).filter((scene) => sceneVisualLayer(scene, { layered: true }) === 'card').map((scene) => scene.sceneId));
+ const playthrough = playthroughProject(project, songDurationSec);
+ const scenes = playthrough.scenes.map((scene) => (cards.has(scene.sceneId) ? { ...scene, textZone: 'none' } : scene));
+ return buildDocumentData({ ...playthrough, scenes }, {
+ frame: { width, height }, clock: documentRenderClock(songDurationSec, fps), songDurationSec,
+ });
}
diff --git a/server/services/musicVideo/lyricPlaythrough.test.js b/server/services/musicVideo/lyricPlaythrough.test.js
index 9bf01a7d23..aeff453605 100644
--- a/server/services/musicVideo/lyricPlaythrough.test.js
+++ b/server/services/musicVideo/lyricPlaythrough.test.js
@@ -5,7 +5,7 @@ vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await imp
dataRoot: () => lazyTempDataRoot('portos-mv-lyric-playthrough-'),
}));
-const { buildLyricPlaythroughPreview } = await import('./lyricPlaythrough.js');
+const { buildLyricPlaythroughPreview, lyricOverlayDocumentData } = await import('./lyricPlaythrough.js');
afterAll(() => cleanupTempDataRoots());
@@ -97,3 +97,17 @@ describe('lyric timing playthrough', () => {
expect(preview.html).toContain('"title":"Example Song"');
});
});
+
+describe('lyric overlay for a composed render', () => {
+ it('keeps a title card shot\'s sung line off the overlay, since the card already draws it', () => {
+ const data = lyricOverlayDocumentData({
+ audioAnalysis: { durationSec: 10 }, lyricCues: [cue('found a shot', 1), cue('then changed the scene', 5)],
+ scenes: [
+ { sceneId: 's1', order: 0, startSec: 0, endSec: 4, visualLayer: 'card', lyricText: 'found a shot', textZone: 'upper' },
+ { sceneId: 's2', order: 1, startSec: 4, endSec: 10, textZone: 'upper-right' },
+ ],
+ }, { width: 320, height: 180, fps: 24, durationSec: 10 });
+ expect(data.scenes.map((scene) => [scene.sceneId, scene.textZone])).toEqual([['s1', 'none'], ['s2', 'upper-right']]);
+ expect(data.render).toMatchObject({ width: 320, height: 180, fps: 24 });
+ });
+});
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 46429e37b7..80dbf7e7c3 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -45,7 +45,7 @@ import { getProject, listProjects, updateProject, mutateProjectRecord } from './
import { applyProjectPatch } from './projectsLogic.js';
import { MUSIC_VIDEO_FRAME_FITS, clipOffsetSec, clipReversed, clipSpeedRate, isSelfDrawnLayer, sceneHasAuthoredSpan, sceneVisualLayer } from '../../lib/musicVideoLayers.js';
import { musicVideoAspect, musicVideoRenderCanvas } from '../../lib/musicVideoAspect.js';
-import { projectTypographyPlan } from './composition.js';
+import { composedLyricWindows, projectTypographyPlan } from './composition.js';
import { encodeCodeComposition, prepareCodeRender, writeCodeProofSheet } from './codeRender.js';
import { documentRenderInitScripts, encodeDocumentComposition, prepareDocumentRender, sweepDocumentScratch } from './documentRender.js';
import { renderTypographyOverlays, removeCompositionScratch, sweepCompositionScratch } from './compositionRender.js';
@@ -995,10 +995,13 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
const outputPath = join(PATHS.videos, filename);
const { args, totalDuration, canonW, canonH, fps, sections } = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, frameGrid: composed, soundBed, grade: composed ? project.composition?.grade : null, beats: project.audioAnalysis?.beats, aspect });
// #8984: a composed project lays its timed text cues over the cut, and a
- // title card's text (#8985) joins them over its own section. No renderable
- // cue (plain mode, or nothing timed) skips the overlay capture entirely.
+ // title card's text (#8985) joins them over its own section, and without
+ // subtitle cues the sung lyrics are drawn in the shared lyric type, as the
+ // storyboard animatic shows them. Nothing to draw (plain mode, or nothing
+ // timed) skips the overlay capture entirely.
const { cues, style } = projectTypographyPlan(project, clips, sections, totalDuration);
- const composition = cues.length > 0 ? project.composition : null;
+ const lyricWindows = composedLyricWindows(project, totalDuration);
+ const composition = cues.length > 0 || lyricWindows.length > 0 ? project.composition : null;
await assertCurrentRenderApproval(projectId, project, options);
// A long filter graph goes to ffmpeg in this file, removed once ffmpeg exits.
@@ -1233,6 +1236,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
const { signal } = job.overlayAbort;
renderTypographyOverlays({
jobId, cues, style, width: canonW, height: canonH, fps, durationSec: totalDuration, signal,
+ lyrics: lyricWindows.length > 0 ? { project, windows: lyricWindows } : null,
onProgress: (fraction) => broadcastSse(job, { type: 'progress', progress: 0.5 * fraction }),
}).then(async (overlays) => {
await assertCurrentRenderApproval(projectId, project, options);