Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import FilmLookFrame from '../media/FilmLookFrame.jsx';
import { safeReadStorage, safeWriteStorage } from '../../lib/safeStorage.js';
import { uuidv4 } from '../../lib/uuid.js';
import { isFilmLookNeutral } from '../../lib/filmLook.js';
import { Captions, Film, Scissors } from 'lucide-react';
import { formatDurationSec } from '../../utils/formatters.js';
import { getMusicVideoLyricOverlayPreview } from '../../services/apiMusicVideo.js';
import useMusicVideoStatsOutro from '../../hooks/useMusicVideoStatsOutro.js';
import { playthroughVersion } from './CompositionPreviewPlayer.jsx';
import { useVideoFileSrc } from '../../hooks/useVideoFileSrc.js';
import { selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
import { normalizeTitleCard } from '../../../../server/lib/musicVideoTitleCard.js';
import { normalizeStatsOutro, statsOutroSeconds } from '../../../../server/lib/musicVideoStatsOutro.js';
import StatsOutroCanvas from './StatsOutroCanvas.jsx';
import ShotStepper from './ShotStepper.jsx';
import FrameStepper from './FrameStepper.jsx';
import { capturePreviewFrame, lyricFrameCaptureBridge } from '../../lib/musicVideoFrameCapture.js';
import PreviewVideo from './PreviewVideo.jsx';
import { sceneFrameFit } from './SceneFrameFit.jsx';
import TransitionGlitch from './TransitionGlitch.jsx';
import ImageEffectStill from './ImageEffectStill.jsx';
import { clipOffsetSec, clipReversed, clipSpeedRate, isLayeredComposition, sceneVisualLayer, stillFrameTransform } from '../../lib/musicVideoLayers.js';
import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
import { plannedTransitions, sceneCutEffects } from '../../lib/musicVideoCutEffects.js';
const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);
/** The shot on screen at song time `t`: the last one that has started. */
export function shotAt(scenes, t) {
const timed = timedShots(scenes);
let current = timed[0] || null;
for (const scene of timed) {
if (scene.startSec <= t) current = scene;
else break;
}
return current;
}
// How long a seek may wait for its `portos-mv:seeked` reply before the next one goes anyway.
const SEEK_REPLY_MS = 1000;
// The overlay page needs the song's analysis and at least one timed line (else the server answers 409).
const hasTimedLyrics = (project) => Number(project.audioAnalysis?.durationSec) > 0
&& (project.lyricCues || []).some((cue) => Number.isFinite(cue?.startSec) && String(cue?.text || '').trim());
/**
* The sung words over the animatic: the server's words-only page (the final
* render's lyric type, in each shot's text zone, on a transparent frame) in a
* sandboxed iframe laid over the picture. Like the composition preview it
* answers the page's `portos-mv:*` bridge (it has no media to ask for) and
* drives it with seek messages, one in flight at a time, every animation frame
* while the song plays. Returns the overlay element, or null while there is
* nothing to draw.
*/
function useLyricOverlay({ project, enabled, audioRef, time }) {
const [preview, setPreview] = useState(null);
const [failed, setFailed] = useState(false);
const iframeRef = useRef(null);
const captureRef = useRef(null);
const seekState = useRef({ inFlight: false, pending: null, ready: false, sentAt: 0 });
const version = enabled ? playthroughVersion(project) : null;
// The opening title card rides on the lyric layer, posted live so its panel's edits show at once.
const titleCard = useMemo(() => normalizeTitleCard(project.titleCard), [project.titleCard]);
const titleCardRef = useRef(titleCard);
useEffect(() => {
titleCardRef.current = titleCard;
if (seekState.current.ready) iframeRef.current?.contentWindow?.postMessage({ type: 'portos-mv:title-card', card: titleCard }, '*');
}, [titleCard]);
useEffect(() => {
let active = true;
captureRef.current?.reject(new Error('Lyric preview changed'));
setPreview(null);
setFailed(false);
seekState.current = { inFlight: false, pending: null, ready: false, sentAt: 0 };
if (!version) return () => { active = false; };
getMusicVideoLyricOverlayPreview(project.id, { silent: true })
.then((next) => { if (active) setPreview(next); })
.catch(() => { if (active) setFailed(true); });
return () => { active = false; captureRef.current?.reject(new Error('Lyric preview changed')); };
}, [project.id, version]);
const postSeek = useCallback((t) => {
const frame = iframeRef.current?.contentWindow;
const state = seekState.current;
if (!frame || !state.ready) return;
// A seek the page never answered stops holding the queue after a second.
if (state.inFlight && performance.now() - state.sentAt < SEEK_REPLY_MS) { state.pending = t; return; }
state.inFlight = true;
state.sentAt = performance.now();
frame.postMessage({ type: 'portos-mv:seek', t }, '*');
}, []);
useEffect(() => {
if (!preview) return undefined;
const onMessage = (event) => {
if (event.source !== iframeRef.current?.contentWindow) return;
const message = event.data || {};
const state = seekState.current;
if (message.type === 'portos-mv:loaded') {
event.source.postMessage({ type: 'portos-mv:manifest', keys: [] }, '*');
event.source.postMessage({ type: 'portos-mv:title-card', card: titleCardRef.current }, '*');
state.ready = true;
postSeek(audioRef.current?.currentTime || 0);
} else if (message.type === 'portos-mv:request') {
event.source.postMessage({ type: 'portos-mv:asset', key: String(message.key), blob: null }, '*');
} else if (message.type === 'portos-mv:seeked') {
state.inFlight = false;
if (state.pending != null) { const next = state.pending; state.pending = null; postSeek(next); }
else if (captureRef.current?.t === message.t) {
const capture = captureRef.current;
event.source.postMessage({ type: 'portos-mv:capture-lyrics', id: capture.id, t: capture.t }, '*');
}
} else if (message.type === 'portos-mv:captured-lyrics') {
const capture = captureRef.current;
if (!capture || message.id !== capture.id || message.t !== capture.t) return;
if (message.error || typeof message.png !== 'string' || message.png.length > 24000000 || !/^data:image\/png;base64,[A-Za-z0-9+/=]+$/.test(message.png)) capture.reject(new Error('Lyric frame unavailable'));
else capture.resolve(message.png);
} else if (message.type === 'portos-mv:error') {
captureRef.current?.reject(new Error('Lyric frame unavailable'));
setFailed(true);
}
};
window.addEventListener('message', onMessage);
return () => window.removeEventListener('message', onMessage);
}, [preview, postSeek, audioRef]);
// A scrub, a seek or a pause lands through `time`; playback runs on animation frames,
// since the audio's timeupdate is far too coarse for words landing on their onsets.
useEffect(() => { postSeek(time); }, [time, postSeek]);
useEffect(() => {
const audio = audioRef.current;
if (!preview || !audio) return undefined;
let handle = 0;
const tick = () => {
postSeek(audio.currentTime || 0);
handle = requestAnimationFrame(tick);
};
const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };
const stop = () => cancelAnimationFrame(handle);
audio.addEventListener('play', start);
audio.addEventListener('pause', stop);
audio.addEventListener('ended', stop);
if (!audio.paused) start();
return () => {
stop();
audio.removeEventListener('play', start);
audio.removeEventListener('pause', stop);
audio.removeEventListener('ended', stop);
};
}, [preview, postSeek, audioRef]);
const capture = (t) => new Promise((resolve, reject) => {
if (!seekState.current.ready || failed || !iframeRef.current) { reject(new Error('Lyric frame unavailable')); return; }
const id = uuidv4();
const finish = (done, result) => { clearTimeout(timer); captureRef.current = null; done(result); };
const timer = setTimeout(() => finish(reject, new Error('Lyric capture timed out')), 5000);
captureRef.current = { id, t, resolve: (png) => finish(resolve, png), reject: (error) => finish(reject, error) };
// The existing serialized seek bridge finishes the requested song time first.
seekState.current.pending = null;
postSeek(t);
});
if (!preview?.html || failed) return { overlay: null, failed, capture };
return {
failed,
capture,
overlay: (
<iframe
ref={iframeRef}
title="Lyrics over the storyboard animatic"
sandbox="allow-scripts"
srcDoc={`${preview.html}<script>(${lyricFrameCaptureBridge.toString()})();</script>`}
// `color-scheme: normal` keeps the frame transparent under a dark parent page.
style={{ colorScheme: 'normal' }}
className="pointer-events-none absolute inset-0 h-full w-full border-0 bg-transparent"
data-testid="animatic-lyrics"
/>
),
};
}
// How far a playing shot's clip may drift from the song before it is put back in step.
// Paused (frame stepping, scrubbing), the clip must show the exact frame, so any gap
// wider than a fraction of a frame re-seeks it.
const CLIP_DRIFT_SEC = 0.25;
const CLIP_PAUSED_DRIFT_SEC = 0.01;
// A clip still fetching after a seek gets this long before it is moved again: each seek
// restarts its download, so on a slow phone link re-seeking on every time update never lets
// it catch up.
const CLIP_RESEEK_WAIT_MS = 1500;
// A clip that fails to load is tried again this many times, a little later each time.
const CLIP_LOAD_RETRIES = 2;
const CLIP_RETRY_MS = 1500;
// A playing shot's clip whose download stalls is restarted at most this often.
const CLIP_STALL_RELOAD_MS = 5000;
/**
* Lets the browser drop a video's download and decoder. Removing the element
* alone does not: iOS Safari keeps both until the element is collected, and
* after a few dozen cuts it stops loading new clips at all.
*/
function releaseVideo(video) {
if (!video) return;
video.pause?.();
video.removeAttribute('src');
video.load?.();
}
/**
* Where song time `t` lands in a shot's clip: the take's edit-in point (a
* performance take trimmed to the sung window) plus the time into the shot. Past
* the clip's end it loops (`loop`, the render's default) or holds the last frame.
* A hand-set speed and offset (lib/musicVideoLayers.js) play as the render plays
* them: the offset moves the in-point by whole frames, a negative one past the
* clip's start holds its first frame that long, and the speed runs the clip faster
* or slower, looping round the whole file as the render's looped input does.
* `early` follows the song's clock before the shot starts too (a beat toggle
* into it, whose window opens at `fromSec`), reading the footage before its
* in-point; with less footage than that it plays on from the clip's first
* frame, as the render does, rather than holding it.
*/
export function clipTimeAt(scene, t, clipSec = null, { early = false, fromSec = null } = {}) {
const inSec = selectedPerformanceInstruction(scene)?.edit?.inSec ?? 0;
const local = early ? t - (scene.startSec || 0) : Math.max(0, t - (scene.startSec || 0));
const rate = clipSpeedRate(scene);
const start = inSec + clipOffsetSec(scene);
// A negative offset holds the first frame before the clip runs; otherwise it runs from `start`.
const position = (l) => (start >= 0 ? Math.max(0, start + l * rate) : Math.max(0, l + start) * rate);
const at = early && local < 0 && fromSec != null && fromSec <= t
? position(fromSec - (scene.startSec || 0)) + (t - fromSec) * rate
: position(local);
if (clipReversed(scene)) {
// Backwards from the file's end, as the render reads it once: past its start it holds the first frame shown last.
if (!(clipSec > 0)) return 0;
return Math.min(clipSec - 0.05, Math.max(0, clipSec - at));
}
if (rate !== 1 || start !== inSec) {
if (!(clipSec > 0) || at < clipSec) return at;
return scene.loop !== false && !(inSec > 0) ? at % clipSec : Math.max(0, clipSec - 0.05);
}
if (local < 0) return at;
if (!(clipSec > inSec)) return inSec + local;
const span = clipSec - inSec;
if (local < span) return inSec + local;
return scene.loop !== false ? inSec + (local % span) : Math.max(inSec, clipSec - 0.05);
}
// How many shots past the one on screen get their clip loaded ahead, so a short shot is
// ready to show the moment the song reaches it.
const CLIP_PRELOAD_SHOTS = 2;
// The song frame the film look's grain and drift follow; footage renders run at 24 fps.
const ANIMATIC_FPS = 24;
// Remembers, per viewer, that the animatic plays without the project's film look.
const ANIMATIC_LOOK_KEY = 'portos-mv-animatic-look';
/**
* A shot's selected video clip, muted under the master song and kept in step
* with it while its shot is `current`: it plays and pauses with the song, and a
* seek or drift puts it back at the song's place in the shot. A shot still to
* come waits paused on its first frame. The clip stays transparent until it has
* a frame to show, so the shot's still underneath covers the load and the seek.
* `preload` is 'auto' for the shot on screen and the next one, else 'metadata',
* so a phone spends its bandwidth on the clip it needs first. A clip that fails
* to load is retried, a stalled one restarted, and a removed one releases its
* download and decoder.
*/
function ShotClip({ scene, audioRef, time, current, preload = 'auto' }) {
const videoRef = useRef(null);
const [ready, setReady] = useState(false);
const [reload, setReload] = useState(0);
// -Infinity: no seek or restart yet, so the first one is never held back.
const recovery = useRef({ errors: 0, timer: 0, lastSeekAt: -Infinity, lastReloadAt: -Infinity });
const file = useVideoFileSrc(scene.videoHistoryId);
const src = file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`);
const sync = useCallback(() => {
const video = videoRef.current;
const audio = audioRef.current;
if (!video || !audio || video.readyState < 1) return;
const duration = Number.isFinite(video.duration) ? video.duration : null;
const target = clipTimeAt(scene, current ? audio.currentTime || 0 : scene.startSec || 0, duration);
// A browser cannot play a clip backwards, so a reversed clip stays paused and is stepped to each frame.
const reversed = clipReversed(scene);
const tolerance = current && (audio.paused || reversed) ? CLIP_PAUSED_DRIFT_SEC : CLIP_DRIFT_SEC;
// A clip still seeking is left to land; its `seeked` re-checks it. A clip playing forward
// that is still fetching after its last seek gets a moment too. Paused or reversed, it must
// show the exact frame, so it moves at once.
const fetching = current && !audio.paused && !reversed && video.readyState < 3
&& performance.now() - recovery.current.lastSeekAt < CLIP_RESEEK_WAIT_MS;
const settling = video.seeking || fetching;
if (Math.abs((video.currentTime || 0) - target) > tolerance && !settling) {
video.currentTime = target;
recovery.current.lastSeekAt = performance.now();
}
// The shot's hand-set speed; a negative offset's lead holds the first frame.
video.playbackRate = clipSpeedRate(scene);
const leading = (scene.startSec || 0) - clipOffsetSec(scene) - (selectedPerformanceInstruction(scene)?.edit?.inSec ?? 0) > (audio.currentTime || 0);
if (!current || audio.paused || leading || reversed) video.pause?.();
else if (video.paused) video.play?.()?.catch?.(() => {});
}, [scene, audioRef, current]);
// Shown once a frame is decoded where the clip should be; kept from then on, since a
// drift re-seek holds the last frame rather than going blank.
const markReady = useCallback(() => {
const video = videoRef.current;
if (video && video.readyState >= 2 && !video.seeking) setReady(true);
}, []);
useEffect(() => { sync(); }, [time, sync]);
// Each mounted element (a retry remounts it) lets go of its download when it leaves.
useEffect(() => {
const video = videoRef.current;
return () => releaseVideo(video);
}, [src, reload]);
useEffect(() => () => clearTimeout(recovery.current.timer), []);
const retryLoad = () => {
const state = recovery.current;
if (state.errors >= CLIP_LOAD_RETRIES) return;
state.errors += 1;
clearTimeout(state.timer);
state.timer = setTimeout(() => { file.retry(); setReload((n) => n + 1); }, CLIP_RETRY_MS * state.errors);
};
// A download that stalls while its shot plays is restarted; its metadata re-seeks it.
const restartStalled = () => {
const video = videoRef.current;
const state = recovery.current;
if (!video || !current || audioRef.current?.paused || video.readyState >= 3) return;
if (performance.now() - state.lastReloadAt < CLIP_STALL_RELOAD_MS) return;
state.lastReloadAt = performance.now();
video.load?.();
};
// The song's time updates are too coarse to step a reversed clip, so it follows every animation frame.
const reversed = clipReversed(scene);
useEffect(() => {
if (!current || !reversed) return undefined;
let handle = 0;
const tick = () => { sync(); handle = requestAnimationFrame(tick); };
handle = requestAnimationFrame(tick);
return () => cancelAnimationFrame(handle);
}, [current, reversed, sync]);
useEffect(() => {
const audio = audioRef.current;
if (!audio) return undefined;
audio.addEventListener('play', sync);
audio.addEventListener('pause', sync);
return () => { audio.removeEventListener('play', sync); audio.removeEventListener('pause', sync); };
}, [audioRef, sync]);
if (!src) return null;
// The final render's frame fit: zoomed to fill, or whole over the still, the VHS edge or black bars.
const fit = sceneFrameFit(scene);
return (
<PreviewVideo
key={`${src}:${reload}`}
videoRef={videoRef}
audioRef={audioRef}
active={current}
src={src}
muted
playsInline
preload={preload}
onLoadedMetadata={sync}
onLoadedData={markReady}
onSeeked={() => { markReady(); sync(); }}
onPlaying={markReady}
onError={retryLoad}
onStalled={restartStalled}
aria-label={`Clip for ${scene.label || 'this shot'}`}
aria-hidden={current ? undefined : true}
data-ready={ready ? 'true' : 'false'}
className={`pointer-events-none absolute inset-0 h-full w-full ${fit === 'fill' ? 'object-cover' : 'object-contain'} ${fit === 'black' ? 'bg-black' : ''} ${current && ready ? 'opacity-100' : 'opacity-0'}`}
/>
);
}
// Runs `apply(songTime)` on every animation frame while the song plays, and once more when it stops:
// the song's timeupdate is far too coarse for a beat flip or a smooth fade.
function useSongFrames(audioRef, apply) {
useEffect(() => {
const audio = audioRef.current;
if (!audio) return undefined;
let handle = 0;
const tick = () => {
apply(audio.currentTime || 0);
handle = requestAnimationFrame(tick);
};
const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };
const stop = () => { cancelAnimationFrame(handle); apply(audio.currentTime || 0); };
audio.addEventListener('play', start);
audio.addEventListener('pause', stop);
audio.addEventListener('ended', stop);
if (!audio.paused) start();
return () => {
cancelAnimationFrame(handle);
audio.removeEventListener('play', start);
audio.removeEventListener('pause', stop);
audio.removeEventListener('ended', stop);
};
}, [audioRef, apply]);
}
function StillShot({ scene, audioRef, time, effect = null, fps, startSec = scene.startSec, endSec = scene.endSec }) {
const ref = useRef(null);
const getTime = useCallback(() => audioRef.current?.currentTime || 0, [audioRef]);
const apply = useCallback((t) => {
if (ref.current) ref.current.style.transform = stillFrameTransform(scene, t, { fps, startSec, endSec });
}, [scene, fps, startSec, endSec]);
useEffect(() => { apply(time); }, [time, apply]);
useSongFrames(audioRef, apply);
return (
<div className="h-full w-full overflow-hidden">
<div ref={ref} data-testid="animatic-still" className="h-full w-full">
{effect ? <ImageEffectStill src={`/data/images/${scene.referenceImageId}`} effect={effect}
endSrc={scene.endImageId ? `/data/images/${scene.endImageId}` : null}
startSec={startSec} time={time} playing={!!audioRef.current && !audioRef.current.paused} getTime={getTime} fps={fps} />
: <img src={`/data/images/${scene.referenceImageId}`} alt="" className="h-full w-full object-cover" />}
</div>
</div>
);
}
// How much of the incoming clip shows at song time `t`: on a toggle's flips, or a
// dissolve's ramp, which reaches full on the cut as the render's alpha fade does.
const incomingOpacity = (effect, t, fps) => {
if (effect.style !== 'dissolve') return effect.incoming.some(([a, b]) => t >= a && t < b) ? 1 : 0;
if (t < effect.windowStartSec || t >= effect.cutSec) return 0;
return Math.min(1, (t - effect.windowStartSec) / (effect.cutSec - effect.windowStartSec + 1 / fps));
};
/**
* The incoming shot of a beat toggle or dissolve (lib/musicVideoCutEffects.js)
* over the outgoing one: its clip runs on the song's clock, reading the footage
* before its in-point, and shows on the toggle's frames or fades in over the
* dissolve's, as the final render overlays it. A shot without a clip shows its still.
*/
function ToggleClip({ scene, toggle, audioRef, time, layered, fps }) {
const mediaRef = useRef(null);
const still = sceneVisualLayer(scene, { layered }) === 'still';
const file = useVideoFileSrc(!still ? scene.videoHistoryId || null : null);
const src = !still && scene.videoHistoryId ? file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`) : null;
const apply = useCallback((t) => {
const el = mediaRef.current;
const audio = audioRef.current;
if (!el) return;
const surface = el.tagName === 'VIDEO' ? el.parentElement : el;
surface.style.opacity = String(incomingOpacity(toggle, t, fps));
if (el.tagName !== 'VIDEO' || el.readyState < 1) return;
const duration = Number.isFinite(el.duration) ? el.duration : null;
const target = clipTimeAt(scene, Math.min(Math.max(t, toggle.windowStartSec), toggle.cutSec), duration, { early: true, fromSec: toggle.windowStartSec });
const tolerance = audio?.paused ? CLIP_PAUSED_DRIFT_SEC : CLIP_DRIFT_SEC;
if (Math.abs((el.currentTime || 0) - target) > tolerance) el.currentTime = target;
const running = !!audio && !audio.paused && t >= toggle.windowStartSec && t < toggle.cutSec;
if (!running) el.pause?.();
else if (el.paused) el.play?.()?.catch?.(() => {});
}, [toggle, scene, audioRef, fps]);
useEffect(() => { apply(time); }, [time, apply]);
useEffect(() => {
const el = mediaRef.current;
return () => { if (el?.tagName === 'VIDEO') releaseVideo(el); };
}, [src]);
useSongFrames(audioRef, apply);
const fit = sceneFrameFit(scene);
const className = `pointer-events-none absolute inset-0 h-full w-full ${src && fit !== 'fill' ? 'object-contain' : 'object-cover'} ${fit === 'black' ? 'bg-black' : ''}`;
if (src) {
return <PreviewVideo key={src} videoRef={mediaRef} audioRef={audioRef} active={time >= toggle.windowStartSec && time < toggle.cutSec} src={src} muted playsInline preload="auto" onLoadedMetadata={() => apply(audioRef.current?.currentTime || 0)}
data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;
}
if (!scene.referenceImageId) return null;
if (layered) return <div ref={mediaRef} data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className}>
<StillShot scene={scene} audioRef={audioRef} time={time} fps={fps} effect={sceneImageEffect(scene, { layered })}
startSec={toggle.windowStartSec} endSec={toggle.cutSec} />
</div>;
return <img ref={mediaRef} src={`/data/images/${scene.referenceImageId}`} alt="" data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;
}
/**
* A fade through black (lib/musicVideoCutEffects.js): black over the picture
* that darkens into the cut and lifts after it, as the render fades the two shots.
*/
function FadeBlack({ fade, audioRef, time }) {
const ref = useRef(null);
const apply = useCallback((t) => {
const el = ref.current;
if (!el) return;
const down = t >= fade.outStartSec && t < fade.cutSec ? (t - fade.outStartSec) / (fade.cutSec - fade.outStartSec) : 0;
const up = t >= fade.cutSec && t < fade.inEndSec ? 1 - (t - fade.cutSec) / (fade.inEndSec - fade.cutSec) : 0;
el.style.opacity = String(Math.min(1, Math.max(down, up)));
}, [fade]);
useEffect(() => { apply(time); }, [time, apply]);
useSongFrames(audioRef, apply);
return <div ref={ref} data-testid="fade-black" style={{ opacity: 0 }} className="pointer-events-none absolute inset-0 bg-black" />;
}
/**
* Something to watch before anything is rendered: the master song under the
* storyboard, showing each shot's selected video clip (muted, in step with the
* song) or else its selected frame (or, without either, its label, time and
* intent) as the song reaches it, with the aligned lyrics drawn over
* it once the words are timed (a toggle hides them). `seekRequest`
* (`{ t, n }`) is the scene-card seek the other preview sources honour too.
*/
export default function StoryboardAnimatic({ project, audioUrl, seekRequest, collapsed, onEditShot = null }) {
const audioRef = useRef(null);
const figureRef = useRef(null);
const applied = useRef(null);
const [time, setTime] = useState(0);
const [lyricsOn, setLyricsOn] = useState(true);
// The look is the render's; a viewer whose phone struggles can watch without it.
const [lookOn, setLookOn] = useState(() => safeReadStorage(ANIMATIC_LOOK_KEY) !== 'off');
// While the song plays the look is the light playback version; paused, the exact render filter.
const [playing, setPlaying] = useState(false);
const captureBasis = useRef(null);
const lookAvailable = !isFilmLookNeutral(project.filmLook);
const toggleLook = () => setLookOn((on) => { safeWriteStorage(ANIMATIC_LOOK_KEY, on ? 'off' : 'on'); return !on; });
const scenes = project.scenes || [];
const layered = isLayeredComposition(project);
// The first footage source sets the composed render's frame rate, even when the current shot is a still.
const lead = scenes.slice().sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
.find((scene) => scene.videoHistoryId && sceneVisualLayer(scene, { layered }) === 'footage');
const leadFile = useVideoFileSrc(layered ? lead?.videoHistoryId : null, { geometry: true });
const stillFps = leadFile.fps || ANIMATIC_FPS;
const lyricsAvailable = hasTimedLyrics(project);
const { overlay, failed, capture: captureLyrics } = useLyricOverlay({ project, enabled: lyricsAvailable && lyricsOn, audioRef, time });
// The cutting room floor outro: it plays when the song ends, and Ending plays it straight away; playing or seeking the song leaves it.
const outro = useMusicVideoStatsOutro({ projectId: project.id, seconds: statsOutroSeconds(project.statsOutro), audioRef });
const outroLook = useMemo(() => normalizeStatsOutro(project.statsOutro), [project.statsOutro]);
const songEndSec = Number(project.audioAnalysis?.durationSec) || 0;
useEffect(() => {
if (!seekRequest || applied.current === seekRequest.n || !audioRef.current) return;
applied.current = seekRequest.n;
audioRef.current.currentTime = seekRequest.t;
setTime(seekRequest.t);
if (seekRequest.play) audioRef.current.play?.()?.catch?.(() => {});
}, [seekRequest]);
// Back and Next move the song; it keeps playing or stays paused as it was.
const seekTo = useCallback((t) => {
if (!audioRef.current) return;
audioRef.current.currentTime = t;
setTime(t);
}, []);
// Frame steps pause the song first, so the frame stays on screen to check.
const stepFrame = useCallback((t) => {
audioRef.current?.pause?.();
seekTo(t);
}, [seekTo]);
// Edit shot pauses the song so the shot stays on screen while its card opens.
const editShot = onEditShot ? (sceneId) => { audioRef.current?.pause?.(); onEditShot(sceneId); } : null;
captureBasis.current = { project, lyricsOn, lookOn, outro: outro.active, outroTime: outro.t, outroStats: outro.stats };
const captureFrame = async () => {
const basis = captureBasis.current;
const audio = audioRef.current;
audio?.pause();
outro.pause();
const at = audio?.currentTime || 0;
setTime(at);
setPlaying(false);
// Pause swaps the live film look for its exact filter and catches the shot up
// to the audio clock. Effects/clip seeks then get a paint before freezing.
await new Promise((resolve, reject) => {
let handle;
const timer = setTimeout(() => { cancelAnimationFrame(handle); reject(new Error('Preview did not settle')); }, 2000);
let paints = 0;
const settle = () => {
const clips = [...(figureRef.current?.querySelectorAll('video[data-preview-video]') || [])]
.filter((video) => Number(getComputedStyle(video.parentElement).opacity) > 0);
const failedClip = clips.find((video) => video.dataset.previewError);
if (failedClip) { clearTimeout(timer); reject(new Error(failedClip.dataset.previewError)); return; }
if (++paints >= 2 && clips.every((video) => !video.seeking && video.dataset.pausedFrame === String(video.currentTime))) {
clearTimeout(timer); resolve(); return;
}
handle = requestAnimationFrame(settle);
};
handle = requestAnimationFrame(settle);
});
// The click can land before a tick's queued setT has rendered; pause() stops
// further ticks, so the clock the settle wait observed is the one captured.
// Later outro changes (late stats, a restart) still invalidate the capture.
basis.outroTime = captureBasis.current.outroTime;
basis.outroStats = captureBasis.current.outroStats;
const unchanged = () => {
const current = captureBasis.current;
return current.project === basis.project && current.lyricsOn === basis.lyricsOn && current.lookOn === basis.lookOn
&& current.outro === basis.outro && current.outroTime === basis.outroTime && current.outroStats === basis.outroStats
&& audio?.paused && audio.currentTime === at;
};
if (!unchanged()) throw new Error('Preview changed during capture');
const selected = shotAt(scenes, at);
if (!basis.outro && selected?.videoHistoryId && sceneVisualLayer(selected, { layered }) === 'footage') {
const figure = figureRef.current;
const clip = figure?.querySelector('video[data-ready="true"]:not([aria-hidden="true"])');
const fallback = [...(figure?.querySelectorAll('img') || [])].find((image) => image.getAttribute('src') === `/data/images/${selected.referenceImageId}` && image.complete && image.naturalWidth > 0);
if (!clip && !fallback) throw new Error('Video frame is still loading');
}
if (lyricsAvailable && lyricsOn && !overlay) throw new Error('Lyric frame unavailable');
const iframeImages = new Map();
const iframe = figureRef.current?.querySelector('iframe');
if (iframe) iframeImages.set(iframe, await captureLyrics(at));
if (!unchanged()) throw new Error('Preview changed during capture');
const blob = await capturePreviewFrame(figureRef.current?.firstElementChild, { iframeImages });
if (!unchanged()) throw new Error('Preview changed during capture');
const shotLabel = basis.outro ? 'Ending' : `Shot ${timedShots(scenes).indexOf(selected) + 1}`;
return { blob, time: at, fps: ANIMATIC_FPS, label: [project.name, shotLabel].filter(Boolean).join(' · ') };
};
// Cut effects (beat toggles, dissolves, fades through black), on the frames the final
// render paints them (lib/musicVideoCutEffects.js), storyboard-planned ones included.
const beats = project.audioAnalysis?.beats;
const storyboardRows = project.productionReview?.draft?.storyboard;
const planned = useMemo(() => plannedTransitions(storyboardRows), [storyboardRows]);
const cutEffects = useMemo(() => sceneCutEffects(scenes, { fps: stillFps, beats, endSec: songEndSec || null, planned }), [scenes, stillFps, beats, songEndSec, planned]);
// A toggle or dissolve mounts its incoming clip a little ahead, so it has loaded when it starts showing.
const nearToggles = cutEffects.filter((t) => t.incoming && time >= t.windowStartSec - 3 && time < t.cutSec);
const nearFades = cutEffects.filter((t) => t.style === 'fade-black' && time >= t.outStartSec - 1 && time < t.inEndSec + 1);
const shot = shotAt(scenes, time);
const timed = timedShots(scenes);
const index = shot ? timed.indexOf(shot) : -1;
// A still shot with an image effect plays the effect over its still, as the composed render does, not its clip.
const shotEffect = shot ? sceneImageEffect(shot, { layered }) : null;
const stillShot = shot && layered && (sceneVisualLayer(shot, { layered }) === 'still' || !shot.videoHistoryId);
// The shot on screen and the next few: each keeps one mounted clip, so the cut to the
// next shot shows a clip that has already loaded and sits on its first frame.
const clipShots = timed.slice(Math.max(index, 0), Math.max(index, 0) + 1 + CLIP_PRELOAD_SHOTS)
.filter((scene) => scene.videoHistoryId && sceneVisualLayer(scene, { layered }) === 'footage');
const label = shot ? shot.label || `Shot ${index + 1}` : 'No shots yet';
// With the lyrics drawn over the picture, a shot without a frame keeps its label and
// intent under the picture instead, so no stand-in text sits under the sung words.
const cardInFrame = !overlay;
const vhsEdges = !!shot?.videoHistoryId && sceneFrameFit(shot) === 'vhs';
// The render cover-fits the still behind a still-edged clip; the preview matches it.
const stillEdges = !!shot?.videoHistoryId && sceneFrameFit(shot) === 'still';
return (
<div className={`space-y-2 ${collapsed ? 'max-xl:hidden' : ''}`}>
<figure ref={figureRef} aria-label="Storyboard animatic" className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black">
{/* The project's film look, as the final render applies it over the whole picture, lyrics included. */}
<FilmLookFrame look={lookOn ? project.filmLook : null} frame={Math.round((outro.active ? songEndSec + outro.t : time) * ANIMATIC_FPS)}
playing={playing || (outro.active && !outro.paused && outro.t < outro.seconds)} className="h-full w-full">
{shot?.referenceImageId && stillShot ? (
<StillShot key={shot.sceneId} scene={shot} audioRef={audioRef} time={time} fps={stillFps} effect={shotEffect} />
) : shot?.referenceImageId ? (
// Behind a VHS-edged clip the still stands in, blurred and faded, for the render's blur of the clip.
<img src={`/data/images/${shot.referenceImageId}`} alt=""
className={`h-full w-full ${vhsEdges ? 'object-cover blur-md saturate-50 brightness-75' : stillEdges ? 'object-cover' : 'object-contain'}`} />
) : cardInFrame && !shot?.videoHistoryId ? (
<div className="flex h-full w-full flex-col items-center justify-center gap-1 p-4 text-center">
<span className="text-sm font-medium text-port-text">{label}</span>
{shot?.visualIntent && <span className="line-clamp-3 text-xs text-port-text-muted">{shot.visualIntent}</span>}
</div>
) : null}
{clipShots.map((scene) => (
<ShotClip key={`${scene.sceneId}:${scene.videoHistoryId}`} scene={scene} audioRef={audioRef} time={time} current={scene === shot}
preload={timed.indexOf(scene) - Math.max(index, 0) <= 1 ? 'auto' : 'metadata'} />
))}
{nearToggles.map((toggle) => {
const incoming = scenes.find((scene) => scene.sceneId === toggle.toSceneId);
return incoming ? <ToggleClip key={`toggle:${toggle.toSceneId}:${incoming.videoHistoryId || ''}`} scene={incoming} toggle={toggle} audioRef={audioRef} time={time} layered={layered} fps={stillFps} /> : null;
})}
{nearFades.map((fade) => <FadeBlack key={`fade:${fade.fromSceneId}`} fade={fade} audioRef={audioRef} time={time} />)}
{/* The shots' VHS glitches on their cuts, on the frames the final render repaints. */}
<TransitionGlitch scenes={scenes} figureRef={figureRef} audioRef={audioRef} time={time} layered={layered}
endSec={Number(project.audioAnalysis?.durationSec) || null} fps={ANIMATIC_FPS} />
{overlay}
{outro.active && <StatsOutroCanvas stats={outro.stats} outro={outroLook} t={outro.t} durationSec={outro.seconds} />}
</FilmLookFrame>
</figure>
<div className="flex min-w-0 flex-wrap items-center gap-2">
<div className="min-w-0 basis-full text-[11px] text-port-text-muted @min-[32rem]/mv-preview:flex-1 @min-[32rem]/mv-preview:basis-auto">
{shot && (
<div className="flex flex-wrap items-center gap-x-2 gap-y-1">
<ShotStepper scenes={timed} time={time} onSeek={seekTo} onEditShot={editShot} keys />
<span className="tabular-nums">{formatDurationSec(shot.startSec)}</span>
</div>
)}
{!cardInFrame && shot && !shot.referenceImageId && !shot.videoHistoryId && (
<p className="line-clamp-2 break-words">
<span className="font-medium text-port-text">{label}</span>{shot.visualIntent ? ` · ${shot.visualIntent}` : ''}
</p>
)}
{lyricsOn && failed && <p className="text-port-error">The lyrics could not be drawn over the animatic.</p>}
</div>
{outro.seconds > 0 && (
<button
type="button"
onClick={() => {
if (outro.active) { outro.stop(); return; }
audioRef.current?.pause?.();
outro.start();
}}
aria-pressed={outro.active}
title={`The cutting room floor scene that plays for ${outro.seconds}s after the song`}
className={`flex min-h-[44px] shrink-0 items-center gap-1 rounded border px-2 text-xs sm:min-h-0 sm:py-1 ${outro.active ? 'border-port-accent text-port-accent' : 'border-port-border text-port-text-muted'}`}
>
<Scissors size={14} aria-hidden="true" /> Ending
</button>
)}
{lookAvailable && (
<button
type="button"
onClick={toggleLook}
aria-pressed={lookOn}
title={lookOn ? 'Showing the project film look: a light version while playing, the exact render look when paused. Tap to watch without it.' : 'Film look hidden in the animatic; the render still applies it'}
className={`flex min-h-[44px] shrink-0 items-center gap-1 rounded border px-2 text-xs sm:min-h-0 sm:py-1 ${lookOn ? 'border-port-accent text-port-accent' : 'border-port-border text-port-text-muted'}`}
>
<Film size={14} aria-hidden="true" /> Film look
</button>
)}
{lyricsAvailable && (
<button
type="button"
onClick={() => setLyricsOn((on) => !on)}
aria-pressed={lyricsOn}
className={`flex min-h-[44px] shrink-0 items-center gap-1 rounded border px-2 text-xs sm:min-h-0 sm:py-1 ${lyricsOn ? 'border-port-accent text-port-accent' : 'border-port-border text-port-text-muted'}`}
>
<Captions size={14} aria-hidden="true" /> Lyrics
</button>
)}
</div>
{shot && (
<FrameStepper
time={time}
fps={ANIMATIC_FPS}
maxSec={Number(project.audioAnalysis?.durationSec) || null}
onSeek={stepFrame}
capture={captureFrame}
label={[project.name, `Shot ${index + 1}`].filter(Boolean).join(' · ')}
keys
/>
)}
<audio
ref={audioRef}
src={audioUrl}
controls
preload="metadata"
aria-label="Song for the storyboard animatic"
onTimeUpdate={(e) => setTime(e.currentTarget.currentTime)}
onSeeked={(e) => { outro.stop(); setTime(e.currentTarget.currentTime); }}
onPlay={() => { outro.stop(); setPlaying(true); }}
onPause={() => setPlaying(false)}
onEnded={() => setPlaying(false)}
className="w-full"
/>
</div>
);
}