Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-11619]): animate still moves in storyboard playback (#11626)
[._.]/72faa0dcfa21
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsxindex 36674eaf14..736cb45967 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx@@ -17,7 +17,7 @@ import FrameStepper, { captureMediaFrame } from './FrameStepper.jsx';import { sceneFrameFit } from './SceneFrameFit.jsx';import TransitionGlitch from './TransitionGlitch.jsx';import ImageEffectStill from './ImageEffectStill.jsx';-import { clipOffsetSec, clipReversed, clipSpeedRate, isLayeredComposition } from '../../lib/musicVideoLayers.js';+import { clipOffsetSec, clipReversed, clipSpeedRate, isLayeredComposition, sceneVisualLayer, stillFrameTransform } from '../../lib/musicVideoLayers.js';import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';import { plannedTransitions, sceneCutEffects } from '../../lib/musicVideoCutEffects.js';@@ -342,7 +342,7 @@ function ShotClip({ scene, audioRef, time, current, preload = 'auto' }) {);}-// Runs `apply(songTime)` on every animation frame while the song plays, and once more when it pauses:+// Runs `apply(songTime)` on every animation frame while the song plays, and once more when it stops:// the song's timeupdate is far too coarse for a beat flip or a smooth fade.function useSongFrames(audioRef, apply) {useEffect(() => {@@ -357,21 +357,43 @@ function useSongFrames(audioRef, apply) {const stop = () => { cancelAnimationFrame(handle); apply(audio.currentTime || 0); };audio.addEventListener('play', start);audio.addEventListener('pause', stop);+ audio.addEventListener('ended', stop);if (!audio.paused) start();return () => {cancelAnimationFrame(handle);audio.removeEventListener('play', start);audio.removeEventListener('pause', stop);+ audio.removeEventListener('ended', stop);};}, [audioRef, apply]);}+function StillShot({ scene, audioRef, time, effect = null, fps, startSec = scene.startSec, endSec = scene.endSec }) {+ const ref = useRef(null);+ const getTime = useCallback(() => audioRef.current?.currentTime || 0, [audioRef]);+ const apply = useCallback((t) => {+ if (ref.current) ref.current.style.transform = stillFrameTransform(scene, t, { fps, startSec, endSec });+ }, [scene, fps, startSec, endSec]);+ useEffect(() => { apply(time); }, [time, apply]);+ useSongFrames(audioRef, apply);+ return (+ <div className="h-full w-full overflow-hidden">+ <div ref={ref} data-testid="animatic-still" className="h-full w-full">+ {effect ? <ImageEffectStill src={`/data/images/${scene.referenceImageId}`} effect={effect}+ endSrc={scene.endImageId ? `/data/images/${scene.endImageId}` : null}+ startSec={startSec} time={time} playing={!!audioRef.current && !audioRef.current.paused} getTime={getTime} fps={fps} />+ : <img src={`/data/images/${scene.referenceImageId}`} alt="" className="h-full w-full object-cover" />}+ </div>+ </div>+ );+}+// How much of the incoming clip shows at song time `t`: on a toggle's flips, or a// dissolve's ramp, which reaches full on the cut as the render's alpha fade does.-const incomingOpacity = (effect, t) => {+const incomingOpacity = (effect, t, fps) => {if (effect.style !== 'dissolve') return effect.incoming.some(([a, b]) => t >= a && t < b) ? 1 : 0;if (t < effect.windowStartSec || t >= effect.cutSec) return 0;- return Math.min(1, (t - effect.windowStartSec) / (effect.cutSec - effect.windowStartSec + 1 / ANIMATIC_FPS));+ return Math.min(1, (t - effect.windowStartSec) / (effect.cutSec - effect.windowStartSec + 1 / fps));};/**@@ -380,15 +402,16 @@ const incomingOpacity = (effect, t) => {* before its in-point, and shows on the toggle's frames or fades in over the* dissolve's, as the final render overlays it. A shot without a clip shows its still.*/-function ToggleClip({ scene, toggle, audioRef, time }) {+function ToggleClip({ scene, toggle, audioRef, time, layered, fps }) {const mediaRef = useRef(null);- const file = useVideoFileSrc(scene.videoHistoryId || null);- const src = scene.videoHistoryId ? file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`) : null;+ const still = sceneVisualLayer(scene, { layered }) === 'still';+ const file = useVideoFileSrc(!still ? scene.videoHistoryId || null : null);+ const src = !still && scene.videoHistoryId ? file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`) : null;const apply = useCallback((t) => {const el = mediaRef.current;const audio = audioRef.current;if (!el) return;- el.style.opacity = String(incomingOpacity(toggle, t));+ el.style.opacity = String(incomingOpacity(toggle, t, fps));if (el.tagName !== 'VIDEO' || el.readyState < 1) return;const duration = Number.isFinite(el.duration) ? el.duration : null;const target = clipTimeAt(scene, Math.min(Math.max(t, toggle.windowStartSec), toggle.cutSec), duration, { early: true, fromSec: toggle.windowStartSec });@@ -397,7 +420,7 @@ function ToggleClip({ scene, toggle, audioRef, time }) {const running = !!audio && !audio.paused && t >= toggle.windowStartSec && t < toggle.cutSec;if (!running) el.pause?.();else if (el.paused) el.play?.()?.catch?.(() => {});- }, [toggle, scene, audioRef]);+ }, [toggle, scene, audioRef, fps]);useEffect(() => { apply(time); }, [time, apply]);useEffect(() => {const el = mediaRef.current;@@ -411,6 +434,10 @@ function ToggleClip({ scene, toggle, audioRef, time }) {data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;}if (!scene.referenceImageId) return null;+ if (layered) return <div ref={mediaRef} data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className}>+ <StillShot scene={scene} audioRef={audioRef} time={time} fps={fps} effect={sceneImageEffect(scene, { layered })}+ startSec={toggle.windowStartSec} endSec={toggle.cutSec} />+ </div>;return <img ref={mediaRef} src={`/data/images/${scene.referenceImageId}`} alt="" data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;}@@ -451,9 +478,14 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col// While the song plays the look is the light playback version; paused, the exact render filter.const [playing, setPlaying] = useState(false);const lookAvailable = !isFilmLookNeutral(project.filmLook);- const songTime = useCallback(() => audioRef.current?.currentTime || 0, []);const toggleLook = () => setLookOn((on) => { safeWriteStorage(ANIMATIC_LOOK_KEY, on ? 'off' : 'on'); return !on; });const scenes = project.scenes || [];+ const layered = isLayeredComposition(project);+ // The first footage source sets the composed render's frame rate, even when the current shot is a still.+ const lead = scenes.slice().sort((a, b) => (a.order ?? 0) - (b.order ?? 0))+ .find((scene) => scene.videoHistoryId && sceneVisualLayer(scene, { layered }) === 'footage');+ const leadFile = useVideoFileSrc(layered ? lead?.videoHistoryId : null, { geometry: true });+ const stillFps = leadFile.fps || ANIMATIC_FPS;const lyricsAvailable = hasTimedLyrics(project);const { overlay, failed } = useLyricOverlay({ project, enabled: lyricsAvailable && lyricsOn, audioRef, time });// The cutting room floor outro: it plays when the song ends, and Ending plays it straight away; playing or seeking the song leaves it.@@ -491,7 +523,7 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, colconst beats = project.audioAnalysis?.beats;const storyboardRows = project.productionReview?.draft?.storyboard;const planned = useMemo(() => plannedTransitions(storyboardRows), [storyboardRows]);- const cutEffects = useMemo(() => sceneCutEffects(scenes, { fps: ANIMATIC_FPS, beats, endSec: songEndSec || null, planned }), [scenes, beats, songEndSec, planned]);+ const cutEffects = useMemo(() => sceneCutEffects(scenes, { fps: stillFps, beats, endSec: songEndSec || null, planned }), [scenes, stillFps, beats, songEndSec, planned]);// A toggle or dissolve mounts its incoming clip a little ahead, so it has loaded when it starts showing.const nearToggles = cutEffects.filter((t) => t.incoming && time >= t.windowStartSec - 3 && time < t.cutSec);const nearFades = cutEffects.filter((t) => t.style === 'fade-black' && time >= t.outStartSec - 1 && time < t.inEndSec + 1);@@ -499,12 +531,12 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, colconst timed = timedShots(scenes);const index = shot ? timed.indexOf(shot) : -1;// A still shot with an image effect plays the effect over its still, as the composed render does, not its clip.- const layered = isLayeredComposition(project);const shotEffect = shot ? sceneImageEffect(shot, { layered }) : null;+ const stillShot = shot && layered && (sceneVisualLayer(shot, { layered }) === 'still' || !shot.videoHistoryId);// The shot on screen and the next few: each keeps one mounted clip, so the cut to the// next shot shows a clip that has already loaded and sits on its first frame.const clipShots = timed.slice(Math.max(index, 0), Math.max(index, 0) + 1 + CLIP_PRELOAD_SHOTS)- .filter((scene) => scene.videoHistoryId && !sceneImageEffect(scene, { layered }));+ .filter((scene) => scene.videoHistoryId && sceneVisualLayer(scene, { layered }) === 'footage');const label = shot ? shot.label || `Shot ${index + 1}` : 'No shots yet';// With the lyrics drawn over the picture, a shot without a frame keeps its label and// intent under the picture instead, so no stand-in text sits under the sung words.@@ -518,10 +550,8 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col{/* The project's film look, as the final render applies it over the whole picture, lyrics included. */}<FilmLookFrame look={lookOn ? project.filmLook : null} frame={Math.round((outro.active ? songEndSec + outro.t : time) * ANIMATIC_FPS)}playing={playing || (outro.active && outro.t < outro.seconds)} className="h-full w-full">- {shot?.referenceImageId && shotEffect ? (- <ImageEffectStill key={shot.sceneId} src={`/data/images/${shot.referenceImageId}`} effect={shotEffect}- endSrc={shot.endImageId ? `/data/images/${shot.endImageId}` : null}- startSec={shot.startSec} time={time} playing={playing} getTime={songTime} fps={ANIMATIC_FPS} />+ {shot?.referenceImageId && stillShot ? (+ <StillShot key={shot.sceneId} scene={shot} audioRef={audioRef} time={time} fps={stillFps} effect={shotEffect} />) : shot?.referenceImageId ? (// Behind a VHS-edged clip the still stands in, blurred and faded, for the render's blur of the clip.<img src={`/data/images/${shot.referenceImageId}`} alt=""@@ -538,7 +568,7 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col))}{nearToggles.map((toggle) => {const incoming = scenes.find((scene) => scene.sceneId === toggle.toSceneId);- return incoming ? <ToggleClip key={`toggle:${toggle.toSceneId}:${incoming.videoHistoryId || ''}`} scene={incoming} toggle={toggle} audioRef={audioRef} time={time} /> : null;+ return incoming ? <ToggleClip key={`toggle:${toggle.toSceneId}:${incoming.videoHistoryId || ''}`} scene={incoming} toggle={toggle} audioRef={audioRef} time={time} layered={layered} fps={stillFps} /> : null;})}{nearFades.map((fade) => <FadeBlack key={`fade:${fade.fromSceneId}`} fade={fade} audioRef={audioRef} time={time} />)}{/* The shots' VHS glitches on their cuts, on the frames the final render repaints. */}diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsxindex 766eaff246..0e717a489c 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx@@ -28,10 +28,124 @@ describe('StoryboardAnimatic', () => {expect(screen.getByText('Shot 2 / 2')).toBeTruthy();});+ // Pins the rendered frame endpoints and seeks, including an explicit Hold overriding a planned camera.+ it('pushes and pans cover-fitted stills on the song clock, while Hold stays still', () => {+ const still = { sceneId: 'still', startSec: 10, endSec: 12, referenceImageId: 'frame.png', visualLayer: 'still',+ videoHistoryId: 'unused-clip', camera: { move: 'handheld' } };+ const project = (stillMove) => ({ id: 'p', composition: { mode: 'composed' }, scenes: [{ ...still, stillMove }] });+ const { rerender } = render(<StoryboardAnimatic project={project('push')} audioUrl="/song.mp3" />);+ const audio = screen.getByLabelText('Song for the storyboard animatic');+ const at = (t) => { audio.currentTime = t; fireEvent.timeUpdate(audio); };+ const transform = () => screen.getByTestId('animatic-still').style.transform;+ at(10);+ expect(transform()).toBe('scale(1)');+ expect(screen.queryByLabelText('Clip for this shot')).toBeNull();+ expect(screen.getByTestId('animatic-still').querySelector('img').className).toContain('object-cover');+ at(11);+ expect(transform()).toBe('scale(1.06)');+ at(12 - 1 / 24);+ expect(transform()).toBe('scale(1.1175)');+ at(20);+ expect(transform()).toBe('scale(1.1175)');+ at(10);+ expect(transform()).toBe('scale(1)');+ rerender(<StoryboardAnimatic project={project('pan')} audioUrl="/song.mp3" />);+ expect(transform()).toBe('translateX(7.5%) scale(1.15)');+ at(12 - 1 / 24);+ expect(transform()).toBe('translateX(-7.5%) scale(1.15)');+ rerender(<StoryboardAnimatic project={project('hold')} audioUrl="/song.mp3" />);+ expect(transform()).toBe('none');+ at(10);+ expect(transform()).toBe('none');+ });++ // A retained clip on a still cannot set the render cadence; the first actual footage source does.+ it('uses the lead footage frame rate for a mixed 30 fps render\'s still endpoint', async () => {+ const { getVideoHistoryItem } = await import('../../services/apiImageVideo.js');+ getVideoHistoryItem.mockImplementation(async (id) => ({ id, filename: `${id}.mov`, fps: id === 'lead' ? 30 : 60 }));+ render(<StoryboardAnimatic project={{ id: 'p', composition: { mode: 'composed' }, scenes: [+ { sceneId: 's', order: 0, startSec: 0, endSec: 2, visualLayer: 'still', referenceImageId: 'frame.png', stillMove: 'push', videoHistoryId: 'unused' },+ { sceneId: 'f', order: 1, startSec: 2, endSec: 4, videoHistoryId: 'lead' },+ ] }} audioUrl="/song.mp3" />);+ await waitFor(() => expect(getVideoHistoryItem).toHaveBeenCalledWith('lead', { silent: true, geometry: true }));+ await act(async () => {});+ const audio = screen.getByLabelText('Song for the storyboard animatic');+ audio.currentTime = 2 - 1 / 30;+ fireEvent.timeUpdate(audio);+ expect(screen.getByTestId('animatic-still').style.transform).toBe(`scale(${1 + 0.12 * 59 / 60})`);+ getVideoHistoryItem.mockImplementation(async (id) => ({ id, filename: `${id}.mov` }));+ });++ // timeupdate is too slow for motion: playback must read currentTime on animation frames, then stop on pause.+ it('moves a playing still between timeupdates and freezes it at the paused song time', () => {+ const frames = new Map();+ let nextFrame = 0;+ vi.stubGlobal('requestAnimationFrame', (tick) => { frames.set(++nextFrame, tick); return nextFrame; });+ vi.stubGlobal('cancelAnimationFrame', (id) => frames.delete(id));+ try {+ render(<StoryboardAnimatic project={{ id: 'p', composition: { mode: 'composed' }, scenes: [+ { sceneId: 's', startSec: 0, endSec: 2, visualLayer: 'still', referenceImageId: 'frame.png', stillMove: 'push' },+ ] }} audioUrl="/song.mp3" />);+ const audio = screen.getByLabelText('Song for the storyboard animatic');+ const still = screen.getByTestId('animatic-still');+ fireEvent.play(audio);+ audio.currentTime = 1;+ act(() => {+ const pending = [...frames.values()];+ frames.clear();+ pending.forEach((tick) => tick());+ });+ expect(still.style.transform).toBe('scale(1.06)');+ audio.currentTime = 1.5;+ fireEvent.pause(audio);+ expect(still.style.transform).toBe('scale(1.09)');+ audio.currentTime = 1.9;+ act(() => {+ const pending = [...frames.values()];+ frames.clear();+ pending.forEach((tick) => tick());+ });+ expect(still.style.transform).toBe('scale(1.09)');+ fireEvent.play(audio);+ audio.currentTime = 2;+ fireEvent.ended(audio);+ expect(still.style.transform).toBe('scale(1.1175)');+ audio.currentTime = 0;+ act(() => {+ const pending = [...frames.values()];+ frames.clear();+ pending.forEach((tick) => tick());+ });+ expect(still.style.transform).toBe('scale(1.1175)');+ } finally {+ vi.unstubAllGlobals();+ }+ });++ // A dissolve renders its incoming still's full move over the preroll window, then restarts for the shot itself.+ it('moves the incoming still over the dissolve window, then starts its own shot move', () => {+ render(<StoryboardAnimatic project={{ id: 'p', composition: { mode: 'composed' }, scenes: [+ { sceneId: 'a', startSec: 0, endSec: 4, visualLayer: 'still', referenceImageId: 'a.png',+ cutEffect: { style: 'dissolve', enabled: true, overlapSec: 1 } },+ { sceneId: 'b', startSec: 4, endSec: 8, visualLayer: 'still', referenceImageId: 'b.png', stillMove: 'pan', videoHistoryId: 'unused' },+ ] }} audioUrl="/song.mp3" />);+ const audio = screen.getByLabelText('Song for the storyboard animatic');+ const at = (t) => { audio.currentTime = t; fireEvent.timeUpdate(audio); };+ at(3);+ const incoming = screen.getByTestId('beat-toggle-clip');+ expect(incoming.querySelector('video')).toBeNull();+ expect(incoming.querySelector('[data-testid="animatic-still"]').style.transform).toBe('translateX(7.5%) scale(1.15)');+ at(4 - 1 / 24);+ expect(incoming.querySelector('[data-testid="animatic-still"]').style.transform).toBe('translateX(-7.5%) scale(1.15)');+ at(4);+ expect(screen.queryByTestId('beat-toggle-clip')).toBeNull();+ expect(screen.getByTestId('animatic-still').style.transform).toBe('translateX(7.5%) scale(1.15)');+ });+// Uniquely catches the animatic showing a clip where the composed render plays a still's image effect.it('plays a still shot\'s camera flash develop over its still, not its clip, from the shot\'s start', () => {const effectShot = { sceneId: 'fx', label: 'Burger', startSec: 0, endSec: 6, referenceImageId: 'burger.png', videoHistoryId: 'clip-1',- visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 2 };+ visualLayer: 'still', stillMove: 'push', imageEffect: 'flash-develop', imageEffectSec: 2 };render(<StoryboardAnimatic project={{ id: 'p', composition: { mode: 'composed' }, scenes: [effectShot] }} audioUrl="/song.mp3" />);const effect = screen.getByTestId('animatic-image-effect');expect(screen.queryByLabelText('Clip for Burger')).toBeNull();@@ -41,6 +155,7 @@ describe('StoryboardAnimatic', () => {Object.defineProperty(audio, 'currentTime', { value: 3, configurable: true });fireEvent.timeUpdate(audio);expect(img.style.filter).toBe('contrast(1) saturate(1)');+ expect(screen.getByTestId('animatic-still').style.transform).toBe('scale(1.06)');});});diff --git a/client/src/hooks/README.md b/client/src/hooks/README.mdindex 86ea672423..2f89256acd 100644--- a/client/src/hooks/README.md+++ b/client/src/hooks/README.md@@ -243,7 +243,7 @@ grep -i "what you want to do" client/src/hooks/README.md| `useUniverseRender` | Universe Builder batch-render settings, scoped render payload compilation, canon-selection defaults, per-entry job queue, and run-list refresh. Returns the shared `runRender`/`handleRender` action used by the Render tab and inline bucket/canon controls. | Universe Builder composition shell and presentational panels; don't fork render payload semantics per tab. || `useUniverseTabs` | `useUniverseTabs(categories)` → `{ activeTab, activeBucket, bucketsByKind, hasOtherBuckets, setTab, setBucket }`. URL-backed (`?tab=&bucket=`) tab + sub-bucket state for the Universe Builder, including the self-healing effects that `replace`-strip a tab/bucket the current categories no longer support. | Universe Builder composition shell — don't re-roll the tab/bucket search-param plumbing or the stale-param cleanup. || `useUniverseNav` | `goToWorld(id)` → navigate to `/universes/:id`, preserve `location.search`. | Any Universe Builder caller that needs to switch worlds via URL. |-| `useVideoFileSrc` | `useVideoFileSrc(jobId, { enabled })` → `{ src, resolving, retry }`. Resolves a video-history **id** to the file it really points at (`/data/videos/<filename>`) via the by-id lookup `GET /api/video-gen/history/:id`. | Any surface rendering a video by history id where the id is NOT the filename stem — notably a CD `finalVideoId` (the timeline renderer mints `timeline-*.mp4` beside a `randomUUID()` id, so `/data/videos/<id>.mp4` 404s while `<id>.jpg` poster loads). Pass the result to `<ScenePreview src=…>`. Gate autoplay on `resolving`. |+| `useVideoFileSrc` | `useVideoFileSrc(jobId, { enabled })` → `{ src, resolving, retry }`. Resolves a video-history **id** to the file it really points at (`/data/videos/<filename>`) via the by-id lookup `GET /api/video-gen/history/:id`. Returns source `fps` when known; opt-in `{ geometry: true }` measures missing dimensions/cadence through the one-record read. | Any surface rendering a video by history id where the id is NOT the filename stem — notably a CD `finalVideoId` (the timeline renderer mints `timeline-*.mp4` beside a `randomUUID()` id, so `/data/videos/<id>.mp4` 404s while `<id>.jpg` poster loads). Pass the result to `<ScenePreview src=…>`. Gate autoplay on `resolving`. || `useVoiceUiSync` | Keeps voice server's UI index in sync with current page. | Wire once at root for voice agent support. || `useMoltworldWs` | Moltworld WebSocket feed. | Moltworld surfaces only. || `useYoutubeIngest` | One YouTube brain-ingest job slot: start/cancel + SSE progress + terminal-frame handling via `POST /api/brain/youtube/ingest`. Returns `{ active, percent, stage, start(body), cancel }` — `start` takes the whole payload (`{ url, captureTranscript, downloadVideo, ingestAudio, note, agentPrompt, tags }`), not a bare URL. `onComplete(ingest)` fires with the stored ingest record; non-fatal `warnings[]` on the terminal frame are toasted automatically. | Quick Capture's YouTube path, and any other surface that ingests a video into the brain. |diff --git a/client/src/hooks/useVideoFileSrc.js b/client/src/hooks/useVideoFileSrc.jsindex 6e0f56273f..590fc9266a 100644--- a/client/src/hooks/useVideoFileSrc.js+++ b/client/src/hooks/useVideoFileSrc.js@@ -33,10 +33,13 @@ import { getVideoHistoryItem } from '../services/apiImageVideo.js';* `preload="none"` player can ignore `resolving` — nothing is fetched until the* user hits Play, by which time the lookup has long settled.*+ * `geometry: true` asks the one-record read to measure missing source geometry;+ * `fps` is null until a finite source cadence is known.+ ** `enabled` keeps this lazy: the list grid must not fetch history for N cards,* so a card resolves only once the user actually presses play.*/-export function useVideoFileSrc(jobId, { enabled = true } = {}) {+export function useVideoFileSrc(jobId, { enabled = true, geometry = false } = {}) {// Keyed by the id + attempt it was resolved FOR, so `resolving` can be derived// synchronously rather than set from an effect. An effect-set flag would// still be false for the first render after `enabled` flips true (effects run@@ -46,7 +49,7 @@ export function useVideoFileSrc(jobId, { enabled = true } = {}) {const [attempt, setAttempt] = useState(0);const [resolved, setResolved] = useState({ jobId: null, attempt: -1, src: null });const active = Boolean(enabled && jobId);- const settled = resolved.jobId === jobId && resolved.attempt === attempt;+ const settled = resolved.jobId === jobId && resolved.attempt === attempt && resolved.geometry === geometry;const resolving = active && !settled;useEffect(() => {@@ -57,15 +60,15 @@ export function useVideoFileSrc(jobId, { enabled = true } = {}) {// reconstruction fallback (and its own missing-media UI) covers both, so a// toast would be noise on a page that already degrades gracefully.// Both paths settle on THIS jobId+attempt so `resolving` can never latch on.- getVideoHistoryItem(jobId, { silent: true })+ getVideoHistoryItem(jobId, { silent: true, ...(geometry ? { geometry: true } : {}) }).then((entry) => {if (cancelled) return;const filename = typeof entry?.filename === 'string' ? entry.filename.trim() : '';- setResolved({ jobId, attempt, src: filename ? `/data/videos/${filename}` : null });+ setResolved({ jobId, attempt, geometry, src: filename ? `/data/videos/${filename}` : null, fps: entry?.fps });})- .catch(() => { if (!cancelled) setResolved({ jobId, attempt, src: null }); });+ .catch(() => { if (!cancelled) setResolved({ jobId, attempt, geometry, src: null }); });return () => { cancelled = true; };- }, [jobId, active, settled, attempt]);+ }, [jobId, active, settled, attempt, geometry]);// Redo a settled lookup. A transient 5xx on the history call would otherwise// strand a timeline final forever: `src` stays null, the caller falls back to@@ -74,5 +77,5 @@ export function useVideoFileSrc(jobId, { enabled = true } = {}) {// (`<ScenePreview onRetry={retry}>`) so one button recovers both layers.const retry = useCallback(() => setAttempt((a) => a + 1), []);- return { src: settled ? resolved.src : null, resolving, retry };+ return { src: settled ? resolved.src : null, fps: settled && Number.isFinite(resolved.fps) && resolved.fps > 0 ? resolved.fps : null, resolving, retry };}diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex 591ac15a74..885422e880 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -63,7 +63,7 @@ grep -i "what you want to do" client/src/lib/README.md| `musicVideoFiling.js` | Which music video an Image Gen / Video Gen render is for: `MUSIC_VIDEO_FILING_PARAM` (the `musicVideo` URL param), `musicVideoFilingFields(searchParams)` (the `musicVideoProjectId` request field), `musicVideoOf(item)` (a record's video). Used by `components/media/MusicVideoFilingSelect.jsx`, `hooks/useMediaPreviewActions.js` and the two generate pages. || `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`, `AUDIO_REACTIVE_PERFORMANCE_GUARD`, `silentMotionPrompt`) for board motion prompts. || `musicVideoBatchPlan.js` | Pure helpers for the Music Video missing-frame / missing-clip batches (#10153): `videoBatchPreview` (count + fal estimate summed from `falSceneTake`, unpriced count), `scenesWaitingForFrame`, `videosButtonLabel`, `batchSummary`, `batchActive`, `batchSceneName`. Used by `components/musicVideo/ProjectActionGroups.jsx`. |-| `musicVideoLayers.js` | Music-video per-scene visual layers (#8985), re-exported from `server/lib/musicVideoLayers.js`: `MUSIC_VIDEO_VISUAL_LAYERS`, `MUSIC_VIDEO_STILL_MOVES`, `MUSIC_VIDEO_TEXT_ZONES` / `MUSIC_VIDEO_LYRIC_ROLES` (#10583 lyric type per shot), `sceneVisualLayer(scene, { layered })`, `sceneHasAuthoredSpan`, `sceneRenderReady(scene, { layered })`, and a footage shot's hand-set clip speed, offset and reverse (`clipSpeedRate`, `clipOffsetSec`, `clipReversed`). Used by `components/musicVideo/SceneCard.jsx`, `ProjectActionGroups.jsx` and `hooks/useMusicVideoSceneMedia.js`. |+| `musicVideoLayers.js` | Music-video per-scene visual layers (#8985), re-exported from `server/lib/musicVideoLayers.js`: `MUSIC_VIDEO_VISUAL_LAYERS`, `MUSIC_VIDEO_STILL_MOVES`, `MUSIC_VIDEO_TEXT_ZONES` / `MUSIC_VIDEO_LYRIC_ROLES` (#10583 lyric type per shot), `sceneVisualLayer(scene, { layered })`, `sceneHasAuthoredSpan`, `sceneRenderReady(scene, { layered })`, and a footage shot's hand-set clip speed, offset and reverse (`clipSpeedRate`, `clipOffsetSec`, `clipReversed`). `stillFrameTransform(scene, songTime, { fps, startSec, endSec })` gives the animatic the composed render's frame-clamped push/pan transform (explicit Move before planned camera). Used by `components/musicVideo/SceneCard.jsx`, `StoryboardAnimatic.jsx`, `ProjectActionGroups.jsx` and `hooks/useMusicVideoSceneMedia.js`. || `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. || `musicVideoSceneAttention.js` | Music Video per-scene attention model (#10152): `sceneAttention(scene, ctx)` returns the codes (`missing-frame`, `missing-clip`, `under-covered`, `contract-problem`, `perf-blocked`, `candidates-pending`, `plate-unverified`, `no-span`, `last-failure`) behind the collapsed scene-row chips, the Board `?scenes=attention\|missing\|all` filter (`parseSceneFilter`, `sceneMatchesFilter`) and the Produce per-shot strip; `SCENE_ATTENTION_LABELS`; `sceneMediaKind` (`clip` / `image` / `code` / `card`) and `SCENE_MEDIA_LABELS` for the Board thumbnail badges. Used by `components/musicVideo/SceneCard.jsx`, `stages/BoardStage.jsx` and `stages/ProduceStage.jsx`. || `musicVideoCoverOverlay.js` | Cover-art lettering preview (#10345): re-export of `server/lib/musicVideoCoverOverlay.js` (the design vocabulary, `normalizeCoverDesign`, `coverOverlaySvg`) plus `canvasCoverWidths(design, texts, fonts)`, which measures text with a canvas so the live preview wraps and sizes the title the way the server's measured render does. |diff --git a/client/src/lib/musicVideoLayers.js b/client/src/lib/musicVideoLayers.jsindex 85eeac3829..6a0cc56822 100644--- a/client/src/lib/musicVideoLayers.js+++ b/client/src/lib/musicVideoLayers.js@@ -1,3 +1,6 @@+import { STILL_PAN_TRAVEL, STILL_PUSH_ZOOM } from '../../../server/lib/musicVideoLayers.js';+import { stillMoveForCamera } from '../../../server/lib/cameraMovements.js';+/*** Music Video per-scene visual layers (#8985) — re-export of* `server/lib/musicVideoLayers.js`, so the board's layer picker and Render@@ -24,3 +27,18 @@ export {clipOffsetSec,clipReversed,} from '../../../server/lib/musicVideoLayers.js';++/** The composed render's cover-fitted still move at a song time, clamped to its last frame. */+export function stillFrameTransform(scene, time, { fps = 24, startSec = scene.startSec, endSec = scene.endSec } = {}) {+ if (!Number.isFinite(startSec) || !Number.isFinite(endSec) || endSec <= startSec) return 'none';+ const frames = Math.max(1, Math.round(endSec * fps) - Math.round(startSec * fps));+ const frame = Math.min(frames - 1, Math.max(0, Math.floor(time * fps) - Math.round(startSec * fps)));+ const move = scene.stillMove || stillMoveForCamera(scene.camera) || 'hold';+ // zoompan divides by the frame count; crop's pan reaches the far edge on the last frame.+ if (move === 'push') return `scale(${1 + STILL_PUSH_ZOOM * frame / frames})`;+ if (move === 'pan') {+ const travel = STILL_PAN_TRAVEL * (0.5 - frame / Math.max(1, frames - 1));+ return `translateX(${Number((100 * travel).toFixed(6))}%) scale(${1 + STILL_PAN_TRAVEL})`;+ }+ return 'none';+}diff --git a/client/src/services/apiImageVideo.js b/client/src/services/apiImageVideo.jsindex c56f2873a8..fccb2e081e 100644--- a/client/src/services/apiImageVideo.js+++ b/client/src/services/apiImageVideo.js@@ -276,7 +276,8 @@ export const listVideoHistory = (options = {}) => request('/video-gen/history',// pulling the whole list to find one row is what this replaces. 404s (throwing an// Error with `.status === 404`) when the entry is gone; pass `{ silent: true }`// when the caller owns the not-found UI.-export const getVideoHistoryItem = (id, options = {}) => request(`/video-gen/history/${encodeURIComponent(id)}`, options);+// `geometry: true` resolves missing source dimensions/cadence without modifying history.+export const getVideoHistoryItem = (id, { geometry = false, ...options } = {}) => request(`/video-gen/history/${encodeURIComponent(id)}${geometry ? '?geometry=true' : ''}`, options);// Upload a video into the shared gallery (#4188) — lands under /data/videos/// with a video-history entry (peer-syncable), unlike the /api/uploads scratch// dir. Returns the history entry; feed it through normalizeVideo to display.diff --git a/server/lib/README.md b/server/lib/README.mdindex e38f944687..5f7f843757 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -565,7 +565,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `claimFlowTaskTypes.js` | `hasClaimFlowContract(task)` shares trusted claim identity across registration, completion and resume. `CLAIM_FLOW_TASK_TYPES` — the ONE list of task kinds whose prompts own their own `claim/<item>` worktree and push/PR/MR/review lifecycle (#6613). `services/cosTaskGenerator.js` stamps `metadata.claimFlow` from it (WRITE side); `services/agentPromptBuilder.js` reads it back through `isClaimFlowTask()` to pick a run's claim posture (READ side). It lived as two byte-identical copies, so a new claim type added to one silently split the writer from the reader — no error, just a claim run treated as a plain commit-only handoff; `claimFlowTaskTypes.test.js` fails if a local copy reappears. Imports nothing. || `chiptuneRender.js` | Deterministic chiptune score → mono PCM → 16-bit WAV buffer (pure Node, no audio deps); `renderScoreToPcm` / `pcmToWavBuffer` / `renderScoreToWav`. || `chiptuneScore.js` | Chiptune score contract (#2911): `chiptuneScoreSchema` (Zod) + `sanitizeChiptuneScore`, `buildScoreEvents` — the pattern/order → absolute-time flatten mirrored by `client/src/lib/chiptunePlayback.js`. |-| `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. `clipSpeedRate(scene)` / `clipOffsetSec(scene)` / `clipReversed(scene)` read a footage shot's hand-set `speedFps`, `offsetFrames` and `reverse` on the 24 fps grid (`CLIP_TIMING_FPS`, `CLIP_SPEED_FPS_RANGE`, `CLIP_OFFSET_FRAMES_RANGE`). Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). |+| `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `STILL_PUSH_ZOOM` / `STILL_PAN_TRAVEL` (shared render/animatic motion amounts), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. `clipSpeedRate(scene)` / `clipOffsetSec(scene)` / `clipReversed(scene)` read a footage shot's hand-set `speedFps`, `offsetFrames` and `reverse` on the 24 fps grid (`CLIP_TIMING_FPS`, `CLIP_SPEED_FPS_RANGE`, `CLIP_OFFSET_FRAMES_RANGE`). Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). || `musicVideoLyricAnchors.js` | Music Video lyric anchors: `cuesSungWithin(cues, startSec, endSec)`, the lines a shot's span anchors, each counted from its first sung word to its last when it has word timing, else its line window. Shared by the storyboard anchor check and the retime anchor sync. || `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`: `flash-develop`, a camera flash then the picture develops like an instant photo over `imageEffectSec`; or a fade, flash or glitch from the start frame to the end frame). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe and import-free: document videos ship it as is. || `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections`, `imageEffectGlitchFrames`/`mergeGlitchFrames` (end-frame glitch effects in the same shape), `shotGlitchFrames`/`sceneShotGlitchFrames` (glitches a shot sets at moments inside it, `scene.glitches`), `blockBox`, `edgeGlows` (an effect's edge glows as fading ramps) and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Styles span VHS, film and camera looks. Browser-safe; the storyboard animatic paints the same frames. |diff --git a/server/lib/musicVideoLayers.js b/server/lib/musicVideoLayers.jsindex e1d62d4ca8..99a8296a8f 100644--- a/server/lib/musicVideoLayers.js+++ b/server/lib/musicVideoLayers.js@@ -20,6 +20,9 @@ export const MUSIC_VIDEO_VISUAL_LAYERS = ['footage', 'still', 'card', 'code'];/** Layers the composition draws itself: no reference frame and no clip to generate. */export const isSelfDrawnLayer = (layer) => layer === 'card' || layer === 'code';export const MUSIC_VIDEO_STILL_MOVES = ['hold', 'push', 'pan'];+// Shared by the composed render and storyboard playback.+export const STILL_PUSH_ZOOM = 0.12;+export const STILL_PAN_TRAVEL = 0.15;/*** How a footage clip whose shape is not the video's fills the frame:diff --git a/server/routes/videoGen.test.js b/server/routes/videoGen.test.jsindex 72e1ab9d54..be55f5d4f9 100644--- a/server/routes/videoGen.test.js+++ b/server/routes/videoGen.test.js@@ -11,9 +11,11 @@ const compiledVisual = vi.hoisted(() => ({version: 1, compilerVersion: '1.0.0', status: 'locked', assets: [], adapters: [], omitted: [], warnings: [],}));const probeVideoDuration = vi.hoisted(() => vi.fn(async () => 41.041281));+const probeVideoGeometry = vi.hoisted(() => vi.fn());vi.mock('../lib/ffmpeg.js', async (importOriginal) => ({...(await importOriginal()),probeVideoDuration,+ probeVideoGeometry,}));const compileFableLoomVisualRequest = vi.hoisted(() => vi.fn(async ({ authoredPrompt, authoredNegativePrompt, sourceImagePath }) => ({prompt: authoredPrompt,@@ -3013,6 +3015,34 @@ describe('videoGen routes', () => {});describe('GET /history/:id', () => {+ // Hosted clips lack stored geometry; the animatic needs the cadence the renderer measures.+ it('measures geometry only when requested and missing, without changing the stored entry', async () => {+ const entry = { id: 'hosted', filename: 'hosted.mp4' };+ videoGenService.getHistoryItem.mockResolvedValue(entry);+ probeVideoGeometry.mockResolvedValue({ width: 1280, height: 720, fps: 30, numFrames: 60 });+ const plain = await request(app).get('/api/video-gen/history/hosted');+ expect(plain.body).toEqual(entry);+ expect(probeVideoGeometry).not.toHaveBeenCalled();+ const measured = await request(app).get('/api/video-gen/history/hosted?geometry=true');+ expect(measured.status).toBe(200);+ expect(measured.body).toEqual({ ...entry, width: 1280, height: 720, fps: 30, numFrames: 60 });+ expect(probeVideoGeometry).toHaveBeenCalledTimes(1);+ expect(entry).toEqual({ id: 'hosted', filename: 'hosted.mp4' });+ videoGenService.getHistoryItem.mockResolvedValue(measured.body);+ await request(app).get('/api/video-gen/history/hosted?geometry=true');+ expect(probeVideoGeometry).toHaveBeenCalledTimes(1);+ });++ it('refuses unavailable geometry, unsafe filenames and malformed geometry requests', async () => {+ videoGenService.getHistoryItem.mockResolvedValue({ id: 'hosted', filename: 'hosted.mp4' });+ probeVideoGeometry.mockResolvedValue(null);+ expect((await request(app).get('/api/video-gen/history/hosted?geometry=true')).status).toBe(422);+ videoGenService.getHistoryItem.mockResolvedValue({ id: 'hosted', filename: '../outside.mp4' });+ expect((await request(app).get('/api/video-gen/history/hosted?geometry=true')).status).toBe(422);+ expect(probeVideoGeometry).toHaveBeenCalledTimes(1);+ expect((await request(app).get('/api/video-gen/history/hosted?geometry=maybe')).status).toBe(400);+ });+// The point of the route (#4165): a timeline render's history id is a// randomUUID() that has nothing to do with its `timeline-*.mp4` filename,// so a client holding only the id learns the real file from HERE instead ofdiff --git a/server/routes/videoHistoryRead.js b/server/routes/videoHistoryRead.jsindex 4a72c0876d..de05472753 100644--- a/server/routes/videoHistoryRead.js+++ b/server/routes/videoHistoryRead.js@@ -17,6 +17,7 @@ import { asyncHandler, ServerError, failValidation } from '../lib/errorHandler.j// the only thing standing between a hostile id and an unlink loop.export const historyRecordIdSchema = z.string().trim().min(1).max(200).regex(/^[A-Za-z0-9._-]+$/, 'invalid history id');+const historyReadQuerySchema = z.object({ geometry: z.enum(['true', 'false']).optional() });// One history entry by id (#4165). A history id is NOT the filename stem — the// timeline renderer mints `timeline-<project>-<ts>.mp4` beside an independent@@ -27,8 +28,20 @@ export function createVideoHistoryItemRead(getHistoryItem = async (id) => (awaitreturn asyncHandler(async (req, res) => {const parsed = historyRecordIdSchema.safeParse(req.params.id);if (!parsed.success) failValidation(parsed);+ const query = historyReadQuerySchema.safeParse(req.query);+ if (!query.success) failValidation(query);const entry = await getHistoryItem(parsed.data);if (!entry) throw new ServerError('Not found', { status: 404, code: 'NOT_FOUND' });+ // Hosted clips may omit geometry. Opt-in playback callers measure the same source the render measures.+ if (query.data.geometry === 'true' && !(entry.numFrames && entry.fps && entry.width && entry.height)) {+ const { probeVideoGeometry, safeUnder } = await import('../lib/ffmpeg.js');+ const { PATHS } = await import('../lib/fileUtils.js');+ const path = safeUnder(PATHS.videos, entry.filename);+ const geometry = path ? await probeVideoGeometry(path) : null;+ if (!geometry) throw new ServerError('Video geometry unavailable', { status: 422, code: 'VIDEO_GEOMETRY_UNAVAILABLE' });+ res.json({ ...entry, ...geometry });+ return;+ }res.json(entry);});}diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.jsindex d593029413..28188025b5 100644--- a/server/services/musicVideo/render.js+++ b/server/services/musicVideo/render.js@@ -43,7 +43,7 @@ import { loadHistory, mutateVideoHistory } from '../videoGen/local.js';import { getTrack } from '../tracks/index.js';import { getProject, listProjects, updateProject, mutateProjectRecord } from './projects.js';import { applyProjectPatch } from './projectsLogic.js';-import { MUSIC_VIDEO_FRAME_FITS, clipOffsetSec, clipReversed, clipSpeedRate, isSelfDrawnLayer, sceneHasAuthoredSpan, sceneVisualLayer } from '../../lib/musicVideoLayers.js';+import { MUSIC_VIDEO_FRAME_FITS, STILL_PAN_TRAVEL, STILL_PUSH_ZOOM, clipOffsetSec, clipReversed, clipSpeedRate, isSelfDrawnLayer, sceneHasAuthoredSpan, sceneVisualLayer } from '../../lib/musicVideoLayers.js';import { musicVideoAspect, musicVideoRenderCanvas } from '../../lib/musicVideoAspect.js';import { composedLyricWindows, projectTypographyPlan } from './composition.js';import { encodeCodeComposition, prepareCodeRender, writeCodeProofSheet } from './codeRender.js';@@ -439,9 +439,6 @@ function footageFrameFit(scene) {// The frame for a composed render with no footage to take dimensions from.const DEFAULT_CANVAS = Object.freeze({ width: 1280, height: 720, fps: 24 });-// A still's push-in ends this much closer; a pan crosses this much extra width.-const STILL_PUSH_ZOOM = 0.12;-const STILL_PAN_TRAVEL = 0.15;const evenPx = (n) => Math.max(2, Math.round(n / 2) * 2);// One section's video chain (without its output label). A footage clip is