Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Show the lyrics over the storyboard animatic

Claude4febe956d8e1
diff --git a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx b/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
index 0e5b861c53..bb8111b67b 100644
--- a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
+++ b/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
@@ -6,7 +6,7 @@ const buttonCls = 'flex items-center gap-1 bg-port-bg border border-port-border
// What the lyric playthrough page is built from: the song length, the word times,
// the sheet's sections and the shots' timing and text zones. A change rebuilds it.
-const playthroughVersion = (project) => JSON.stringify([
+export const playthroughVersion = (project) => JSON.stringify([
project.audioAnalysis?.durationSec ?? null,
(project.lyricCues || []).map((cue) => [cue.text, cue.startSec, cue.endSec, (cue.words || []).map((w) => [w.startSec, w.endSec])]),
(project.lyricMarkers || []).map((m) => [m.type, m.label, m.line]),
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index 3494bf7052..493b7378b2 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -1,5 +1,8 @@
-import { useEffect, useRef, useState } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { Captions } from 'lucide-react';
import { formatDurationSec } from '../../utils/formatters.js';
+import { getMusicVideoLyricOverlayPreview } from '../../services/apiMusicVideo.js';
+import { playthroughVersion } from './CompositionPreviewPlayer.jsx';
const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);
@@ -14,17 +17,128 @@ export function shotAt(scenes, t) {
return current;
}
+// The overlay page needs the song's analysis and at least one timed line (else the server answers 409).
+const hasTimedLyrics = (project) => Number(project.audioAnalysis?.durationSec) > 0
+ && (project.lyricCues || []).some((cue) => Number.isFinite(cue?.startSec) && String(cue?.text || '').trim());
+
+/**
+ * The sung words over the animatic: the server's words-only page (the final
+ * render's lyric type, in each shot's text zone, on a transparent frame) in a
+ * sandboxed iframe laid over the picture. Like the composition preview it
+ * answers the page's `portos-mv:*` bridge (it has no media to ask for) and
+ * drives it with seek messages, one in flight at a time, every animation frame
+ * while the song plays. Returns the overlay element, or null while there is
+ * nothing to draw.
+ */
+function useLyricOverlay({ project, enabled, audioRef, time }) {
+ const [preview, setPreview] = useState(null);
+ const [failed, setFailed] = useState(false);
+ const iframeRef = useRef(null);
+ const seekState = useRef({ inFlight: false, pending: null, ready: false });
+ const version = enabled ? playthroughVersion(project) : null;
+
+ useEffect(() => {
+ let active = true;
+ setPreview(null);
+ setFailed(false);
+ seekState.current = { inFlight: false, pending: null, ready: false };
+ if (!version) return () => { active = false; };
+ getMusicVideoLyricOverlayPreview(project.id, { silent: true })
+ .then((next) => { if (active) setPreview(next); })
+ .catch(() => { if (active) setFailed(true); });
+ return () => { active = false; };
+ }, [project.id, version]);
+
+ const postSeek = useCallback((t) => {
+ const frame = iframeRef.current?.contentWindow;
+ const state = seekState.current;
+ if (!frame || !state.ready) return;
+ if (state.inFlight) { state.pending = t; return; }
+ state.inFlight = true;
+ frame.postMessage({ type: 'portos-mv:seek', t }, '*');
+ }, []);
+
+ useEffect(() => {
+ if (!preview) return undefined;
+ const onMessage = (event) => {
+ if (event.source !== iframeRef.current?.contentWindow) return;
+ const message = event.data || {};
+ const state = seekState.current;
+ if (message.type === 'portos-mv:loaded') {
+ event.source.postMessage({ type: 'portos-mv:manifest', keys: [] }, '*');
+ state.ready = true;
+ postSeek(audioRef.current?.currentTime || 0);
+ } else if (message.type === 'portos-mv:request') {
+ event.source.postMessage({ type: 'portos-mv:asset', key: String(message.key), blob: null }, '*');
+ } else if (message.type === 'portos-mv:seeked') {
+ state.inFlight = false;
+ if (state.pending != null) { const next = state.pending; state.pending = null; postSeek(next); }
+ } else if (message.type === 'portos-mv:error') {
+ setFailed(true);
+ }
+ };
+ window.addEventListener('message', onMessage);
+ return () => window.removeEventListener('message', onMessage);
+ }, [preview, postSeek, audioRef]);
+
+ // A scrub, a seek or a pause lands through `time`; playback runs on animation frames,
+ // since the audio's timeupdate is far too coarse for words landing on their onsets.
+ useEffect(() => { postSeek(time); }, [time, postSeek]);
+ useEffect(() => {
+ const audio = audioRef.current;
+ if (!preview || !audio) return undefined;
+ let handle = 0;
+ const tick = () => {
+ postSeek(audio.currentTime || 0);
+ handle = requestAnimationFrame(tick);
+ };
+ const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };
+ const stop = () => cancelAnimationFrame(handle);
+ audio.addEventListener('play', start);
+ audio.addEventListener('pause', stop);
+ audio.addEventListener('ended', stop);
+ if (!audio.paused) start();
+ return () => {
+ stop();
+ audio.removeEventListener('play', start);
+ audio.removeEventListener('pause', stop);
+ audio.removeEventListener('ended', stop);
+ };
+ }, [preview, postSeek, audioRef]);
+
+ if (!preview?.html || failed) return { overlay: null, failed };
+ return {
+ failed,
+ overlay: (
+ <iframe
+ ref={iframeRef}
+ title="Lyrics over the storyboard animatic"
+ sandbox="allow-scripts"
+ srcDoc={preview.html}
+ // `color-scheme: normal` keeps the frame transparent under a dark parent page.
+ style={{ colorScheme: 'normal' }}
+ className="pointer-events-none absolute inset-0 h-full w-full border-0 bg-transparent"
+ data-testid="animatic-lyrics"
+ />
+ ),
+ };
+}
+
/**
* Something to watch before anything is rendered: the master song under the
* storyboard, showing each shot's selected frame (or, without one, its label,
- * time and intent) as the song reaches it. `seekRequest` (`{ t, n }`) is the
- * scene-card seek the other preview sources honour too.
+ * time and intent) as the song reaches it, with the aligned lyrics drawn over
+ * it once the words are timed (a toggle hides them). `seekRequest`
+ * (`{ t, n }`) is the scene-card seek the other preview sources honour too.
*/
export default function StoryboardAnimatic({ project, audioUrl, seekRequest, collapsed }) {
const audioRef = useRef(null);
const applied = useRef(null);
const [time, setTime] = useState(0);
+ const [lyricsOn, setLyricsOn] = useState(true);
const scenes = project.scenes || [];
+ const lyricsAvailable = hasTimedLyrics(project);
+ const { overlay, failed } = useLyricOverlay({ project, enabled: lyricsAvailable && lyricsOn, audioRef, time });
useEffect(() => {
if (!seekRequest || applied.current === seekRequest.n || !audioRef.current) return;
applied.current = seekRequest.n;
@@ -35,23 +149,44 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
const shot = shotAt(scenes, time);
const timed = timedShots(scenes);
const index = shot ? timed.indexOf(shot) : -1;
+ const label = shot ? shot.label || `Shot ${index + 1}` : 'No shots yet';
+ // With the lyrics drawn over the picture, a shot without a frame keeps its label and
+ // intent under the picture instead, so no stand-in text sits under the sung words.
+ const cardInFrame = !overlay;
return (
<div className={`space-y-2 ${collapsed ? 'max-xl:hidden' : ''}`}>
<figure aria-label="Storyboard animatic" className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black">
{shot?.referenceImageId ? (
<img src={`/data/images/${shot.referenceImageId}`} alt="" className="h-full w-full object-contain" />
- ) : (
+ ) : cardInFrame ? (
<div className="flex h-full w-full flex-col items-center justify-center gap-1 p-4 text-center">
- <span className="text-sm font-medium text-port-text">{shot ? shot.label || `Shot ${index + 1}` : 'No shots yet'}</span>
+ <span className="text-sm font-medium text-port-text">{label}</span>
{shot?.visualIntent && <span className="line-clamp-3 text-xs text-port-text-muted">{shot.visualIntent}</span>}
</div>
- )}
- {shot && (
- <figcaption className="port-media-overlay absolute bottom-1 left-1 rounded px-1.5 py-0.5 text-[11px]">
- Shot {index + 1} of {timed.length} · {formatDurationSec(shot.startSec)}
- </figcaption>
- )}
+ ) : null}
+ {overlay}
</figure>
+ <div className="flex min-w-0 items-start gap-2">
+ <div className="min-w-0 flex-1 text-[11px] text-port-text-muted">
+ {shot && <p>Shot {index + 1} of {timed.length} · {formatDurationSec(shot.startSec)}</p>}
+ {!cardInFrame && shot && !shot.referenceImageId && (
+ <p className="line-clamp-2 break-words">
+ <span className="font-medium text-port-text">{label}</span>{shot.visualIntent ? ` · ${shot.visualIntent}` : ''}
+ </p>
+ )}
+ {lyricsOn && failed && <p className="text-port-error">The lyrics could not be drawn over the animatic.</p>}
+ </div>
+ {lyricsAvailable && (
+ <button
+ type="button"
+ onClick={() => setLyricsOn((on) => !on)}
+ aria-pressed={lyricsOn}
+ className={`flex min-h-[44px] shrink-0 items-center gap-1 rounded border px-2 text-xs sm:min-h-0 sm:py-1 ${lyricsOn ? 'border-port-accent text-port-accent' : 'border-port-border text-port-text-muted'}`}
+ >
+ <Captions size={14} aria-hidden="true" /> Lyrics
+ </button>
+ )}
+ </div>
<audio
ref={audioRef}
src={audioUrl}
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
index 59586e9822..114b41740e 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
@@ -1,5 +1,5 @@
-import { describe, expect, it } from 'vitest';
-import { fireEvent, render, screen } from '@testing-library/react';
+import { describe, expect, it, vi } from 'vitest';
+import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import StoryboardAnimatic, { shotAt } from './StoryboardAnimatic.jsx';
const scenes = [
@@ -28,3 +28,32 @@ describe('StoryboardAnimatic', () => {
expect(screen.getByText('Shot 2 of 2 · 0:10')).toBeTruthy();
});
});
+
+vi.mock('../../services/apiMusicVideo.js', () => ({
+ getMusicVideoLyricOverlayPreview: vi.fn(async () => ({ html: '<!doctype html><p>words</p>', assets: [], width: 1920, height: 1080, fps: 24, durationSec: 20 })),
+}));
+
+describe('StoryboardAnimatic lyrics', () => {
+ const lyricProject = {
+ id: 'p', scenes, audioAnalysis: { durationSec: 20 },
+ lyricCues: [{ id: 'c1', text: 'hello there', startSec: 1, endSec: 3 }],
+ };
+
+ it('lays the lyrics over the picture by default and hides them on the toggle', async () => {
+ render(<StoryboardAnimatic project={lyricProject} audioUrl="/song.mp3" />);
+ const overlay = await screen.findByTestId('animatic-lyrics');
+ expect(overlay.getAttribute('sandbox')).toBe('allow-scripts');
+ // The frameless shot's stand-in text moves out from under the words.
+ expect(screen.getByText('Opening').closest('figure')).toBeNull();
+ const toggle = screen.getByRole('button', { name: 'Lyrics' });
+ expect(toggle.getAttribute('aria-pressed')).toBe('true');
+ fireEvent.click(toggle);
+ await waitFor(() => expect(screen.queryByTestId('animatic-lyrics')).toBeNull());
+ expect(screen.getByText('Opening').closest('figure')).toBeTruthy();
+ });
+
+ it('offers no lyrics until the words are timed', () => {
+ render(<StoryboardAnimatic project={{ ...lyricProject, lyricCues: [{ id: 'c1', text: 'hello', startSec: null }] }} audioUrl="/song.mp3" />);
+ expect(screen.queryByRole('button', { name: 'Lyrics' })).toBeNull();
+ });
+});
diff --git a/client/src/services/apiMusicVideo.js b/client/src/services/apiMusicVideo.js
index f9bdebe2f5..5133922a78 100644
--- a/client/src/services/apiMusicVideo.js
+++ b/client/src/services/apiMusicVideo.js
@@ -104,6 +104,7 @@ export const prepareMusicVideoSong = (id, options = {}) => request(`/music-video
});
// The lyric timing playthrough page: the aligned words over a plain frame (same shape as the composition preview).
export const getMusicVideoLyricPlaythroughPreview = (id, options = {}) => request(`/music-video/${encodeURIComponent(id)}/lyrics/playthrough/preview`, options);
+export const getMusicVideoLyricOverlayPreview = (id, options = {}) => request(`/music-video/${encodeURIComponent(id)}/lyrics/overlay/preview`, options);
export const musicVideoLyricAlignEventsUrl = (jobId) =>
`/api/music-video/lyrics/align/${encodeURIComponent(jobId)}/events`;
export const cancelMusicVideoLyricAlign = (jobId, options = {}) =>
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index 4af17b6f46..5fd92760f6 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -566,6 +566,12 @@ router.get('/:id/lyrics/playthrough/preview', asyncHandler(async (req, res) => {
res.json(await buildLyricPlaythroughPreview(await requireProject(req.params.id)));
}));
+// The same words alone on a transparent page, laid over the storyboard animatic's frames.
+router.get('/:id/lyrics/overlay/preview', asyncHandler(async (req, res) => {
+ const { buildLyricPlaythroughPreview } = await import('../services/musicVideo/lyricPlaythrough.js');
+ res.json(await buildLyricPlaythroughPreview(await requireProject(req.params.id), { overlay: true }));
+}));
+
router.get('/lyrics/align/:jobId/events', (req, res) => {
if (!attachLyricAlignSseClient(req.params.jobId, res)) {
throw new ServerError('Alignment job not found or expired', { status: 404, code: 'NOT_FOUND' });
diff --git a/server/services/musicVideo/compositionDocument.js b/server/services/musicVideo/compositionDocument.js
index 5537fb0678..b1d03ad8df 100644
--- a/server/services/musicVideo/compositionDocument.js
+++ b/server/services/musicVideo/compositionDocument.js
@@ -367,6 +367,16 @@ export async function readTemplateDocumentFiles(templateId = 'layered') {
return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));
}
+/**
+ * The storyboard animatic's lyric overlay page (documentTemplates/lyricOverlay),
+ * read like `readTemplateDocumentFiles`: the words alone on a transparent frame.
+ * It is not a template a project can pick.
+ */
+export async function readLyricOverlayDocumentFiles() {
+ const { files } = await collectTree(join(TEMPLATE_ROOT, 'lyricOverlay'), { excludeTests: true });
+ return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));
+}
+
/** Stage a host-assembled generated document for review before selection. */
export async function stageGeneratedDocument(projectId, generatedFiles, { verifyCurrent, renderer = 'canvas' } = {}) {
if (renderer === 'three') {
diff --git a/server/services/musicVideo/documentTemplates/lyricOverlay/engine.js b/server/services/musicVideo/documentTemplates/lyricOverlay/engine.js
new file mode 100644
index 0000000000..098b1032c7
--- /dev/null
+++ b/server/services/musicVideo/documentTemplates/lyricOverlay/engine.js
@@ -0,0 +1,48 @@
+/* Lyric overlay — the sung words alone, on a transparent frame.
+ *
+ * The storyboard animatic plays its shot frames in the PortOS page and lays
+ * this page over them, so the words are drawn exactly as the final render
+ * draws them (the shared lyricType.js, in each shot's text zone) with nothing
+ * else: no fill, grain, vignette or HUD. The palette matches the layered
+ * template's defaults. Every frame is a pure function of song time t.
+ */
+(() => {
+ 'use strict';
+ const MV = window.PORTOS_MV;
+ if (!MV || !MV.render) throw new Error('PORTOS_MV is missing');
+ const canvas = document.getElementById('stage');
+ const ctx = canvas.getContext('2d');
+ let W = MV.render.width;
+ let H = MV.render.height;
+ function resize(width, height) { W = width; H = height; canvas.width = W; canvas.height = H; }
+ resize(W, H);
+ const palette = { fill: '#eef1ea', ink: '#07090a', accent: '#ff5a1f', strike: '#ff2a2a' };
+ // lyricType.js is a module, so it runs after this classic script; it publishes
+ // itself on globalThis before DOMContentLoaded.
+ const lyricTypeModule = () => new Promise((resolve, reject) => {
+ const check = () => (globalThis.PORTOS_LYRIC_TYPE ? resolve(globalThis.PORTOS_LYRIC_TYPE)
+ : reject(new Error('lyricType.js did not load beside the lyric overlay')));
+ if (globalThis.PORTOS_LYRIC_TYPE || document.readyState !== 'loading') check();
+ else document.addEventListener('DOMContentLoaded', check, { once: true });
+ });
+ const ready = (async () => {
+ const { createLyricType } = await lyricTypeModule();
+ const lyricType = createLyricType(MV, { palette });
+ await lyricType.ready;
+ return lyricType;
+ })();
+ globalThis.portosComposition = {
+ durationSec: MV.render.durationSec,
+ fps: MV.render.fps || 24,
+ width: W,
+ height: H,
+ formats: ['1920x1080', '1080x1920', '1080x1080'],
+ layout({ width, height }) { resize(width, height); },
+ async seek(t) {
+ const lyricType = await ready;
+ ctx.clearRect(0, 0, W, H);
+ lyricType.draw(ctx, t, { width: W, height: H });
+ return true;
+ },
+ };
+})();
diff --git a/server/services/musicVideo/documentTemplates/lyricOverlay/index.html b/server/services/musicVideo/documentTemplates/lyricOverlay/index.html
new file mode 100644
index 0000000000..39c34715ea
--- /dev/null
+++ b/server/services/musicVideo/documentTemplates/lyricOverlay/index.html
@@ -0,0 +1,20 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Lyric overlay</title>
+<style>
+/* Transparent: the storyboard animatic shows its frames under this page. */
+html, body { margin: 0; height: 100%; background: transparent; overflow: hidden; }
+#stage { display: block; width: 100vw; height: 100vh; object-fit: contain; }
+</style>
+<link rel="stylesheet" href="lyricType.css">
+</head>
+<body>
+<canvas id="stage"></canvas>
+<script src="portos-mv.js"></script><!-- portos-allow-hidden-content: the render-time data file -->
+<script type="module" src="lyricType.js"></script><!-- portos-allow-hidden-content: the shared PortOS lyric-type module -->
+<script src="engine.js"></script><!-- portos-allow-hidden-content: the reviewed overlay engine beside this page -->
+</body>
+</html>
diff --git a/server/services/musicVideo/lyricPlaythrough.js b/server/services/musicVideo/lyricPlaythrough.js
index 1ddb98873f..d4e67157a7 100644
--- a/server/services/musicVideo/lyricPlaythrough.js
+++ b/server/services/musicVideo/lyricPlaythrough.js
@@ -11,12 +11,15 @@
* line sits in the text zone its shot will give it), else the lyrics laid out
* as scenes: one per lyric-sheet section, or per four lines when the sheet has
* no headers. The layout is built for the preview only; the board is untouched.
+ *
+ * `overlay` builds the same words on a transparent page with nothing else drawn,
+ * for the storyboard animatic to lay over its shot frames.
*/
import { ServerError } from '../../lib/errorHandler.js';
import { isNonBlankStr } from '../../lib/textUtils.js';
import { buildDocumentPreview } from './documentPreview.js';
-import { readTemplateDocumentFiles } from './compositionDocument.js';
+import { readLyricOverlayDocumentFiles, readTemplateDocumentFiles } from './compositionDocument.js';
import { documentSongDuration } from './documentRender.js';
const LINES_PER_SCENE = 4;
@@ -66,8 +69,9 @@ function lyricScenes(cues, markers, durationSec) {
/**
* The preview page for the playthrough, in the shape `buildDocumentPreview`
* returns. 409 until the song is analyzed and at least one line is timed.
+ * `overlay` swaps the template for the transparent words-only page.
*/
-export async function buildLyricPlaythroughPreview(project) {
+export async function buildLyricPlaythroughPreview(project, { overlay = false } = {}) {
const durationSec = documentSongDuration(project);
if (!durationSec) throw new ServerError('Analyze the song before playing the lyrics through', { status: 409, code: 'NOT_ANALYZED' });
const cues = project.lyricCues || [];
@@ -78,5 +82,5 @@ export async function buildLyricPlaythroughPreview(project) {
const scenes = shots.length ? shots : lyricScenes(cues, project.lyricMarkers, durationSec);
// Only the song and the words: no composition, overlay text or narrative events of the project's own.
const playthrough = { ...project, scenes, composition: { mode: 'document', textCues: [] } };
- return buildDocumentPreview(playthrough, { files: await readTemplateDocumentFiles('layered') });
+ return buildDocumentPreview(playthrough, { files: overlay ? await readLyricOverlayDocumentFiles() : await readTemplateDocumentFiles('layered') });
}
diff --git a/server/services/musicVideo/lyricPlaythrough.test.js b/server/services/musicVideo/lyricPlaythrough.test.js
index d2cc2b1e9a..bc509ca17c 100644
--- a/server/services/musicVideo/lyricPlaythrough.test.js
+++ b/server/services/musicVideo/lyricPlaythrough.test.js
@@ -78,4 +78,16 @@ describe('lyric timing playthrough', () => {
const data = portosData((await buildLyricPlaythroughPreview({ ...project, scenes })).html);
expect(data.scenes.map(({ sceneId, textZone, media }) => [sceneId, textZone, media])).toEqual([['s1', 'upper-right', null], ['s2', 'lower', null]]);
});
+
+ it('draws the same words alone on a transparent page for the storyboard animatic', async () => {
+ const scenes = [{ sceneId: 's1', order: 0, label: 'Wide', startSec: 0, endSec: 30, textZone: 'upper-right', referenceImageId: 'frame.png' }];
+ const preview = await buildLyricPlaythroughPreview({ ...project, scenes }, { overlay: true });
+ const data = portosData(preview.html);
+ expect(data.lyrics.map((line) => line.text)).toEqual(['hello there', 'second line']);
+ expect(data.scenes.map(({ sceneId, textZone, media }) => [sceneId, textZone, media])).toEqual([['s1', 'upper-right', null]]);
+ // The frames stay in the PortOS page: the overlay is transparent and bridges no media.
+ expect(preview.html).toMatch(/background: transparent/);
+ expect(preview.html).not.toContain('Layered music-video composition');
+ expect(preview.assets).toEqual([]);
+ });
});