Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Show the lyrics over the storyboard animatic
Claude4febe956d8e1
diff --git a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx b/client/src/components/musicVideo/CompositionPreviewPlayer.jsxindex 0e5b861c53..bb8111b67b 100644--- a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx+++ b/client/src/components/musicVideo/CompositionPreviewPlayer.jsx@@ -6,7 +6,7 @@ const buttonCls = 'flex items-center gap-1 bg-port-bg border border-port-border// What the lyric playthrough page is built from: the song length, the word times,// the sheet's sections and the shots' timing and text zones. A change rebuilds it.-const playthroughVersion = (project) => JSON.stringify([+export const playthroughVersion = (project) => JSON.stringify([project.audioAnalysis?.durationSec ?? null,(project.lyricCues || []).map((cue) => [cue.text, cue.startSec, cue.endSec, (cue.words || []).map((w) => [w.startSec, w.endSec])]),(project.lyricMarkers || []).map((m) => [m.type, m.label, m.line]),diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsxindex 3494bf7052..493b7378b2 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx@@ -1,5 +1,8 @@-import { useEffect, useRef, useState } from 'react';+import { useCallback, useEffect, useRef, useState } from 'react';+import { Captions } from 'lucide-react';import { formatDurationSec } from '../../utils/formatters.js';+import { getMusicVideoLyricOverlayPreview } from '../../services/apiMusicVideo.js';+import { playthroughVersion } from './CompositionPreviewPlayer.jsx';const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);@@ -14,17 +17,128 @@ export function shotAt(scenes, t) {return current;}+// The overlay page needs the song's analysis and at least one timed line (else the server answers 409).+const hasTimedLyrics = (project) => Number(project.audioAnalysis?.durationSec) > 0+ && (project.lyricCues || []).some((cue) => Number.isFinite(cue?.startSec) && String(cue?.text || '').trim());++/**+ * The sung words over the animatic: the server's words-only page (the final+ * render's lyric type, in each shot's text zone, on a transparent frame) in a+ * sandboxed iframe laid over the picture. Like the composition preview it+ * answers the page's `portos-mv:*` bridge (it has no media to ask for) and+ * drives it with seek messages, one in flight at a time, every animation frame+ * while the song plays. Returns the overlay element, or null while there is+ * nothing to draw.+ */+function useLyricOverlay({ project, enabled, audioRef, time }) {+ const [preview, setPreview] = useState(null);+ const [failed, setFailed] = useState(false);+ const iframeRef = useRef(null);+ const seekState = useRef({ inFlight: false, pending: null, ready: false });+ const version = enabled ? playthroughVersion(project) : null;++ useEffect(() => {+ let active = true;+ setPreview(null);+ setFailed(false);+ seekState.current = { inFlight: false, pending: null, ready: false };+ if (!version) return () => { active = false; };+ getMusicVideoLyricOverlayPreview(project.id, { silent: true })+ .then((next) => { if (active) setPreview(next); })+ .catch(() => { if (active) setFailed(true); });+ return () => { active = false; };+ }, [project.id, version]);++ const postSeek = useCallback((t) => {+ const frame = iframeRef.current?.contentWindow;+ const state = seekState.current;+ if (!frame || !state.ready) return;+ if (state.inFlight) { state.pending = t; return; }+ state.inFlight = true;+ frame.postMessage({ type: 'portos-mv:seek', t }, '*');+ }, []);++ useEffect(() => {+ if (!preview) return undefined;+ const onMessage = (event) => {+ if (event.source !== iframeRef.current?.contentWindow) return;+ const message = event.data || {};+ const state = seekState.current;+ if (message.type === 'portos-mv:loaded') {+ event.source.postMessage({ type: 'portos-mv:manifest', keys: [] }, '*');+ state.ready = true;+ postSeek(audioRef.current?.currentTime || 0);+ } else if (message.type === 'portos-mv:request') {+ event.source.postMessage({ type: 'portos-mv:asset', key: String(message.key), blob: null }, '*');+ } else if (message.type === 'portos-mv:seeked') {+ state.inFlight = false;+ if (state.pending != null) { const next = state.pending; state.pending = null; postSeek(next); }+ } else if (message.type === 'portos-mv:error') {+ setFailed(true);+ }+ };+ window.addEventListener('message', onMessage);+ return () => window.removeEventListener('message', onMessage);+ }, [preview, postSeek, audioRef]);++ // A scrub, a seek or a pause lands through `time`; playback runs on animation frames,+ // since the audio's timeupdate is far too coarse for words landing on their onsets.+ useEffect(() => { postSeek(time); }, [time, postSeek]);+ useEffect(() => {+ const audio = audioRef.current;+ if (!preview || !audio) return undefined;+ let handle = 0;+ const tick = () => {+ postSeek(audio.currentTime || 0);+ handle = requestAnimationFrame(tick);+ };+ const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };+ const stop = () => cancelAnimationFrame(handle);+ audio.addEventListener('play', start);+ audio.addEventListener('pause', stop);+ audio.addEventListener('ended', stop);+ if (!audio.paused) start();+ return () => {+ stop();+ audio.removeEventListener('play', start);+ audio.removeEventListener('pause', stop);+ audio.removeEventListener('ended', stop);+ };+ }, [preview, postSeek, audioRef]);++ if (!preview?.html || failed) return { overlay: null, failed };+ return {+ failed,+ overlay: (+ <iframe+ ref={iframeRef}+ title="Lyrics over the storyboard animatic"+ sandbox="allow-scripts"+ srcDoc={preview.html}+ // `color-scheme: normal` keeps the frame transparent under a dark parent page.+ style={{ colorScheme: 'normal' }}+ className="pointer-events-none absolute inset-0 h-full w-full border-0 bg-transparent"+ data-testid="animatic-lyrics"+ />+ ),+ };+}+/*** Something to watch before anything is rendered: the master song under the* storyboard, showing each shot's selected frame (or, without one, its label,- * time and intent) as the song reaches it. `seekRequest` (`{ t, n }`) is the- * scene-card seek the other preview sources honour too.+ * time and intent) as the song reaches it, with the aligned lyrics drawn over+ * it once the words are timed (a toggle hides them). `seekRequest`+ * (`{ t, n }`) is the scene-card seek the other preview sources honour too.*/export default function StoryboardAnimatic({ project, audioUrl, seekRequest, collapsed }) {const audioRef = useRef(null);const applied = useRef(null);const [time, setTime] = useState(0);+ const [lyricsOn, setLyricsOn] = useState(true);const scenes = project.scenes || [];+ const lyricsAvailable = hasTimedLyrics(project);+ const { overlay, failed } = useLyricOverlay({ project, enabled: lyricsAvailable && lyricsOn, audioRef, time });useEffect(() => {if (!seekRequest || applied.current === seekRequest.n || !audioRef.current) return;applied.current = seekRequest.n;@@ -35,23 +149,44 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, colconst shot = shotAt(scenes, time);const timed = timedShots(scenes);const index = shot ? timed.indexOf(shot) : -1;+ const label = shot ? shot.label || `Shot ${index + 1}` : 'No shots yet';+ // With the lyrics drawn over the picture, a shot without a frame keeps its label and+ // intent under the picture instead, so no stand-in text sits under the sung words.+ const cardInFrame = !overlay;return (<div className={`space-y-2 ${collapsed ? 'max-xl:hidden' : ''}`}><figure aria-label="Storyboard animatic" className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black">{shot?.referenceImageId ? (<img src={`/data/images/${shot.referenceImageId}`} alt="" className="h-full w-full object-contain" />- ) : (+ ) : cardInFrame ? (<div className="flex h-full w-full flex-col items-center justify-center gap-1 p-4 text-center">- <span className="text-sm font-medium text-port-text">{shot ? shot.label || `Shot ${index + 1}` : 'No shots yet'}</span>+ <span className="text-sm font-medium text-port-text">{label}</span>{shot?.visualIntent && <span className="line-clamp-3 text-xs text-port-text-muted">{shot.visualIntent}</span>}</div>- )}- {shot && (- <figcaption className="port-media-overlay absolute bottom-1 left-1 rounded px-1.5 py-0.5 text-[11px]">- Shot {index + 1} of {timed.length} · {formatDurationSec(shot.startSec)}- </figcaption>- )}+ ) : null}+ {overlay}</figure>+ <div className="flex min-w-0 items-start gap-2">+ <div className="min-w-0 flex-1 text-[11px] text-port-text-muted">+ {shot && <p>Shot {index + 1} of {timed.length} · {formatDurationSec(shot.startSec)}</p>}+ {!cardInFrame && shot && !shot.referenceImageId && (+ <p className="line-clamp-2 break-words">+ <span className="font-medium text-port-text">{label}</span>{shot.visualIntent ? ` · ${shot.visualIntent}` : ''}+ </p>+ )}+ {lyricsOn && failed && <p className="text-port-error">The lyrics could not be drawn over the animatic.</p>}+ </div>+ {lyricsAvailable && (+ <button+ type="button"+ onClick={() => setLyricsOn((on) => !on)}+ aria-pressed={lyricsOn}+ className={`flex min-h-[44px] shrink-0 items-center gap-1 rounded border px-2 text-xs sm:min-h-0 sm:py-1 ${lyricsOn ? 'border-port-accent text-port-accent' : 'border-port-border text-port-text-muted'}`}+ >+ <Captions size={14} aria-hidden="true" /> Lyrics+ </button>+ )}+ </div><audioref={audioRef}src={audioUrl}diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsxindex 59586e9822..114b41740e 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx@@ -1,5 +1,5 @@-import { describe, expect, it } from 'vitest';-import { fireEvent, render, screen } from '@testing-library/react';+import { describe, expect, it, vi } from 'vitest';+import { fireEvent, render, screen, waitFor } from '@testing-library/react';import StoryboardAnimatic, { shotAt } from './StoryboardAnimatic.jsx';const scenes = [@@ -28,3 +28,32 @@ describe('StoryboardAnimatic', () => {expect(screen.getByText('Shot 2 of 2 · 0:10')).toBeTruthy();});});++vi.mock('../../services/apiMusicVideo.js', () => ({+ getMusicVideoLyricOverlayPreview: vi.fn(async () => ({ html: '<!doctype html><p>words</p>', assets: [], width: 1920, height: 1080, fps: 24, durationSec: 20 })),+}));++describe('StoryboardAnimatic lyrics', () => {+ const lyricProject = {+ id: 'p', scenes, audioAnalysis: { durationSec: 20 },+ lyricCues: [{ id: 'c1', text: 'hello there', startSec: 1, endSec: 3 }],+ };++ it('lays the lyrics over the picture by default and hides them on the toggle', async () => {+ render(<StoryboardAnimatic project={lyricProject} audioUrl="/song.mp3" />);+ const overlay = await screen.findByTestId('animatic-lyrics');+ expect(overlay.getAttribute('sandbox')).toBe('allow-scripts');+ // The frameless shot's stand-in text moves out from under the words.+ expect(screen.getByText('Opening').closest('figure')).toBeNull();+ const toggle = screen.getByRole('button', { name: 'Lyrics' });+ expect(toggle.getAttribute('aria-pressed')).toBe('true');+ fireEvent.click(toggle);+ await waitFor(() => expect(screen.queryByTestId('animatic-lyrics')).toBeNull());+ expect(screen.getByText('Opening').closest('figure')).toBeTruthy();+ });++ it('offers no lyrics until the words are timed', () => {+ render(<StoryboardAnimatic project={{ ...lyricProject, lyricCues: [{ id: 'c1', text: 'hello', startSec: null }] }} audioUrl="/song.mp3" />);+ expect(screen.queryByRole('button', { name: 'Lyrics' })).toBeNull();+ });+});diff --git a/client/src/services/apiMusicVideo.js b/client/src/services/apiMusicVideo.jsindex f9bdebe2f5..5133922a78 100644--- a/client/src/services/apiMusicVideo.js+++ b/client/src/services/apiMusicVideo.js@@ -104,6 +104,7 @@ export const prepareMusicVideoSong = (id, options = {}) => request(`/music-video});// The lyric timing playthrough page: the aligned words over a plain frame (same shape as the composition preview).export const getMusicVideoLyricPlaythroughPreview = (id, options = {}) => request(`/music-video/${encodeURIComponent(id)}/lyrics/playthrough/preview`, options);+export const getMusicVideoLyricOverlayPreview = (id, options = {}) => request(`/music-video/${encodeURIComponent(id)}/lyrics/overlay/preview`, options);export const musicVideoLyricAlignEventsUrl = (jobId) =>`/api/music-video/lyrics/align/${encodeURIComponent(jobId)}/events`;export const cancelMusicVideoLyricAlign = (jobId, options = {}) =>diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.jsindex 4af17b6f46..5fd92760f6 100644--- a/server/routes/musicVideo.js+++ b/server/routes/musicVideo.js@@ -566,6 +566,12 @@ router.get('/:id/lyrics/playthrough/preview', asyncHandler(async (req, res) => {res.json(await buildLyricPlaythroughPreview(await requireProject(req.params.id)));}));+// The same words alone on a transparent page, laid over the storyboard animatic's frames.+router.get('/:id/lyrics/overlay/preview', asyncHandler(async (req, res) => {+ const { buildLyricPlaythroughPreview } = await import('../services/musicVideo/lyricPlaythrough.js');+ res.json(await buildLyricPlaythroughPreview(await requireProject(req.params.id), { overlay: true }));+}));+router.get('/lyrics/align/:jobId/events', (req, res) => {if (!attachLyricAlignSseClient(req.params.jobId, res)) {throw new ServerError('Alignment job not found or expired', { status: 404, code: 'NOT_FOUND' });diff --git a/server/services/musicVideo/compositionDocument.js b/server/services/musicVideo/compositionDocument.jsindex 5537fb0678..b1d03ad8df 100644--- a/server/services/musicVideo/compositionDocument.js+++ b/server/services/musicVideo/compositionDocument.js@@ -367,6 +367,16 @@ export async function readTemplateDocumentFiles(templateId = 'layered') {return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));}+/**+ * The storyboard animatic's lyric overlay page (documentTemplates/lyricOverlay),+ * read like `readTemplateDocumentFiles`: the words alone on a transparent frame.+ * It is not a template a project can pick.+ */+export async function readLyricOverlayDocumentFiles() {+ const { files } = await collectTree(join(TEMPLATE_ROOT, 'lyricOverlay'), { excludeTests: true });+ return new Map((await withSharedLyricType(files)).map((file) => [file.rel, { abs: file.abs, size: file.size }]));+}+/** Stage a host-assembled generated document for review before selection. */export async function stageGeneratedDocument(projectId, generatedFiles, { verifyCurrent, renderer = 'canvas' } = {}) {if (renderer === 'three') {diff --git a/server/services/musicVideo/documentTemplates/lyricOverlay/engine.js b/server/services/musicVideo/documentTemplates/lyricOverlay/engine.jsnew file mode 100644index 0000000000..098b1032c7--- /dev/null+++ b/server/services/musicVideo/documentTemplates/lyricOverlay/engine.js@@ -0,0 +1,48 @@+/* Lyric overlay — the sung words alone, on a transparent frame.+ *+ * The storyboard animatic plays its shot frames in the PortOS page and lays+ * this page over them, so the words are drawn exactly as the final render+ * draws them (the shared lyricType.js, in each shot's text zone) with nothing+ * else: no fill, grain, vignette or HUD. The palette matches the layered+ * template's defaults. Every frame is a pure function of song time t.+ */+(() => {+ 'use strict';+ const MV = window.PORTOS_MV;+ if (!MV || !MV.render) throw new Error('PORTOS_MV is missing');+ const canvas = document.getElementById('stage');+ const ctx = canvas.getContext('2d');+ let W = MV.render.width;+ let H = MV.render.height;+ function resize(width, height) { W = width; H = height; canvas.width = W; canvas.height = H; }+ resize(W, H);+ const palette = { fill: '#eef1ea', ink: '#07090a', accent: '#ff5a1f', strike: '#ff2a2a' };+ // lyricType.js is a module, so it runs after this classic script; it publishes+ // itself on globalThis before DOMContentLoaded.+ const lyricTypeModule = () => new Promise((resolve, reject) => {+ const check = () => (globalThis.PORTOS_LYRIC_TYPE ? resolve(globalThis.PORTOS_LYRIC_TYPE)+ : reject(new Error('lyricType.js did not load beside the lyric overlay')));+ if (globalThis.PORTOS_LYRIC_TYPE || document.readyState !== 'loading') check();+ else document.addEventListener('DOMContentLoaded', check, { once: true });+ });+ const ready = (async () => {+ const { createLyricType } = await lyricTypeModule();+ const lyricType = createLyricType(MV, { palette });+ await lyricType.ready;+ return lyricType;+ })();+ globalThis.portosComposition = {+ durationSec: MV.render.durationSec,+ fps: MV.render.fps || 24,+ width: W,+ height: H,+ formats: ['1920x1080', '1080x1920', '1080x1080'],+ layout({ width, height }) { resize(width, height); },+ async seek(t) {+ const lyricType = await ready;+ ctx.clearRect(0, 0, W, H);+ lyricType.draw(ctx, t, { width: W, height: H });+ return true;+ },+ };+})();diff --git a/server/services/musicVideo/documentTemplates/lyricOverlay/index.html b/server/services/musicVideo/documentTemplates/lyricOverlay/index.htmlnew file mode 100644index 0000000000..39c34715ea--- /dev/null+++ b/server/services/musicVideo/documentTemplates/lyricOverlay/index.html@@ -0,0 +1,20 @@+<!doctype html>+<html lang="en">+<head>+<meta charset="utf-8">+<meta name="viewport" content="width=device-width, initial-scale=1">+<title>Lyric overlay</title>+<style>+/* Transparent: the storyboard animatic shows its frames under this page. */+html, body { margin: 0; height: 100%; background: transparent; overflow: hidden; }+#stage { display: block; width: 100vw; height: 100vh; object-fit: contain; }+</style>+<link rel="stylesheet" href="lyricType.css">+</head>+<body>+<canvas id="stage"></canvas>+<script src="portos-mv.js"></script><!-- portos-allow-hidden-content: the render-time data file -->+<script type="module" src="lyricType.js"></script><!-- portos-allow-hidden-content: the shared PortOS lyric-type module -->+<script src="engine.js"></script><!-- portos-allow-hidden-content: the reviewed overlay engine beside this page -->+</body>+</html>diff --git a/server/services/musicVideo/lyricPlaythrough.js b/server/services/musicVideo/lyricPlaythrough.jsindex 1ddb98873f..d4e67157a7 100644--- a/server/services/musicVideo/lyricPlaythrough.js+++ b/server/services/musicVideo/lyricPlaythrough.js@@ -11,12 +11,15 @@* line sits in the text zone its shot will give it), else the lyrics laid out* as scenes: one per lyric-sheet section, or per four lines when the sheet has* no headers. The layout is built for the preview only; the board is untouched.+ *+ * `overlay` builds the same words on a transparent page with nothing else drawn,+ * for the storyboard animatic to lay over its shot frames.*/import { ServerError } from '../../lib/errorHandler.js';import { isNonBlankStr } from '../../lib/textUtils.js';import { buildDocumentPreview } from './documentPreview.js';-import { readTemplateDocumentFiles } from './compositionDocument.js';+import { readLyricOverlayDocumentFiles, readTemplateDocumentFiles } from './compositionDocument.js';import { documentSongDuration } from './documentRender.js';const LINES_PER_SCENE = 4;@@ -66,8 +69,9 @@ function lyricScenes(cues, markers, durationSec) {/*** The preview page for the playthrough, in the shape `buildDocumentPreview`* returns. 409 until the song is analyzed and at least one line is timed.+ * `overlay` swaps the template for the transparent words-only page.*/-export async function buildLyricPlaythroughPreview(project) {+export async function buildLyricPlaythroughPreview(project, { overlay = false } = {}) {const durationSec = documentSongDuration(project);if (!durationSec) throw new ServerError('Analyze the song before playing the lyrics through', { status: 409, code: 'NOT_ANALYZED' });const cues = project.lyricCues || [];@@ -78,5 +82,5 @@ export async function buildLyricPlaythroughPreview(project) {const scenes = shots.length ? shots : lyricScenes(cues, project.lyricMarkers, durationSec);// Only the song and the words: no composition, overlay text or narrative events of the project's own.const playthrough = { ...project, scenes, composition: { mode: 'document', textCues: [] } };- return buildDocumentPreview(playthrough, { files: await readTemplateDocumentFiles('layered') });+ return buildDocumentPreview(playthrough, { files: overlay ? await readLyricOverlayDocumentFiles() : await readTemplateDocumentFiles('layered') });}diff --git a/server/services/musicVideo/lyricPlaythrough.test.js b/server/services/musicVideo/lyricPlaythrough.test.jsindex d2cc2b1e9a..bc509ca17c 100644--- a/server/services/musicVideo/lyricPlaythrough.test.js+++ b/server/services/musicVideo/lyricPlaythrough.test.js@@ -78,4 +78,16 @@ describe('lyric timing playthrough', () => {const data = portosData((await buildLyricPlaythroughPreview({ ...project, scenes })).html);expect(data.scenes.map(({ sceneId, textZone, media }) => [sceneId, textZone, media])).toEqual([['s1', 'upper-right', null], ['s2', 'lower', null]]);});++ it('draws the same words alone on a transparent page for the storyboard animatic', async () => {+ const scenes = [{ sceneId: 's1', order: 0, label: 'Wide', startSec: 0, endSec: 30, textZone: 'upper-right', referenceImageId: 'frame.png' }];+ const preview = await buildLyricPlaythroughPreview({ ...project, scenes }, { overlay: true });+ const data = portosData(preview.html);+ expect(data.lyrics.map((line) => line.text)).toEqual(['hello there', 'second line']);+ expect(data.scenes.map(({ sceneId, textZone, media }) => [sceneId, textZone, media])).toEqual([['s1', 'upper-right', null]]);+ // The frames stay in the PortOS page: the overlay is transparent and bridges no media.+ expect(preview.html).toMatch(/background: transparent/);+ expect(preview.html).not.toContain('Layered music-video composition');+ expect(preview.assets).toEqual([]);+ });});