Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Play each shot's video clip in the storyboard animatic (#11195)
[._.]/027baea05e15
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsxindex 5211a09547..0ba4b09d70 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx@@ -3,6 +3,8 @@ import { Captions } from 'lucide-react';import { formatDurationSec } from '../../utils/formatters.js';import { getMusicVideoLyricOverlayPreview } from '../../services/apiMusicVideo.js';import { playthroughVersion } from './CompositionPreviewPlayer.jsx';+import { useVideoFileSrc } from '../../hooks/useVideoFileSrc.js';+import { selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';import { normalizeTitleCard } from '../../../../server/lib/musicVideoTitleCard.js';const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);@@ -138,10 +140,70 @@ function useLyricOverlay({ project, enabled, audioRef, time }) {};}+// How far a shot's clip may drift from the song before it is put back in step.+const CLIP_DRIFT_SEC = 0.25;++/**+ * Where song time `t` lands in a shot's clip: the take's edit-in point (a+ * performance take trimmed to the sung window) plus the time into the shot. Past+ * the clip's end it loops (`loop`, the render's default) or holds the last frame.+ */+export function clipTimeAt(scene, t, clipSec = null) {+ const inSec = selectedPerformanceInstruction(scene)?.edit?.inSec ?? 0;+ const local = Math.max(0, t - (scene.startSec || 0));+ if (!(clipSec > inSec)) return inSec + local;+ const span = clipSec - inSec;+ if (local < span) return inSec + local;+ return scene.loop !== false ? inSec + (local % span) : Math.max(inSec, clipSec - 0.05);+}++/**+ * A shot's selected video clip, muted under the master song and kept in step+ * with it: it plays and pauses with the song, and a seek or drift puts it back+ * at the song's place in the shot. The shot's frame is its poster.+ */+function ShotClip({ scene, audioRef, time }) {+ const videoRef = useRef(null);+ const file = useVideoFileSrc(scene.videoHistoryId);+ const src = file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`);+ const sync = useCallback(() => {+ const video = videoRef.current;+ const audio = audioRef.current;+ if (!video || !audio || video.readyState < 1) return;+ const target = clipTimeAt(scene, audio.currentTime || 0, Number.isFinite(video.duration) ? video.duration : null);+ if (Math.abs((video.currentTime || 0) - target) > CLIP_DRIFT_SEC) video.currentTime = target;+ if (audio.paused) video.pause?.();+ else if (video.paused) video.play?.()?.catch?.(() => {});+ }, [scene, audioRef]);+ useEffect(() => { sync(); }, [time, sync]);+ useEffect(() => {+ const audio = audioRef.current;+ if (!audio) return undefined;+ audio.addEventListener('play', sync);+ audio.addEventListener('pause', sync);+ return () => { audio.removeEventListener('play', sync); audio.removeEventListener('pause', sync); };+ }, [audioRef, sync]);+ if (!src) return null;+ return (+ <video+ ref={videoRef}+ src={src}+ poster={scene.referenceImageId ? `/data/images/${scene.referenceImageId}` : undefined}+ muted+ playsInline+ preload="auto"+ onLoadedMetadata={sync}+ aria-label={`Clip for ${scene.label || 'this shot'}`}+ className="h-full w-full object-contain"+ />+ );+}+/*** Something to watch before anything is rendered: the master song under the- * storyboard, showing each shot's selected frame (or, without one, its label,- * time and intent) as the song reaches it, with the aligned lyrics drawn over+ * storyboard, showing each shot's selected video clip (muted, in step with the+ * song) or else its selected frame (or, without either, its label, time and+ * intent) as the song reaches it, with the aligned lyrics drawn over* it once the words are timed (a toggle hides them). `seekRequest`* (`{ t, n }`) is the scene-card seek the other preview sources honour too.*/@@ -170,7 +232,9 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, colreturn (<div className={`space-y-2 ${collapsed ? 'max-xl:hidden' : ''}`}><figure aria-label="Storyboard animatic" className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black">- {shot?.referenceImageId ? (+ {shot?.videoHistoryId ? (+ <ShotClip key={`${shot.sceneId}:${shot.videoHistoryId}`} scene={shot} audioRef={audioRef} time={time} />+ ) : shot?.referenceImageId ? (<img src={`/data/images/${shot.referenceImageId}`} alt="" className="h-full w-full object-contain" />) : cardInFrame ? (<div className="flex h-full w-full flex-col items-center justify-center gap-1 p-4 text-center">@@ -183,7 +247,7 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col<div className="flex min-w-0 items-start gap-2"><div className="min-w-0 flex-1 text-[11px] text-port-text-muted">{shot && <p>Shot {index + 1} of {timed.length} · {formatDurationSec(shot.startSec)}</p>}- {!cardInFrame && shot && !shot.referenceImageId && (+ {!cardInFrame && shot && !shot.referenceImageId && !shot.videoHistoryId && (<p className="line-clamp-2 break-words"><span className="font-medium text-port-text">{label}</span>{shot.visualIntent ? ` · ${shot.visualIntent}` : ''}</p>diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsxindex 8dbe23556e..d83e07eea6 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx@@ -1,6 +1,6 @@import { describe, expect, it, vi } from 'vitest';import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';-import StoryboardAnimatic, { shotAt } from './StoryboardAnimatic.jsx';+import StoryboardAnimatic, { clipTimeAt, shotAt } from './StoryboardAnimatic.jsx';const scenes = [{ sceneId: 'b', label: 'Chorus', startSec: 10, endSec: 20, referenceImageId: 'frame-b.png' },@@ -69,3 +69,37 @@ describe('StoryboardAnimatic lyrics', () => {expect(screen.queryByRole('button', { name: 'Lyrics' })).toBeNull();});});++vi.mock('../../services/apiImageVideo.js', () => ({+ getVideoHistoryItem: vi.fn(async (id) => ({ id, filename: `${id}.mov` })),+}));++describe('StoryboardAnimatic clips', () => {+ const clipScenes = [+ { sceneId: 'a', label: 'Opening', startSec: 0, endSec: 10, referenceImageId: 'frame-a.png', videoHistoryId: 'clip-a' },+ { sceneId: 'b', label: 'Chorus', startSec: 10, endSec: 20, referenceImageId: 'frame-b.png' },+ ];++ it('plays a shot\'s clip muted where it has one and its still where it does not', async () => {+ const { container } = render(<StoryboardAnimatic project={{ id: 'p', scenes: clipScenes }} audioUrl="/song.mp3" />);+ const clip = await screen.findByLabelText('Clip for Opening');+ await waitFor(() => expect(clip.getAttribute('src')).toBe('/data/videos/clip-a.mov'));+ expect(clip.muted).toBe(true);+ expect(clip.getAttribute('poster')).toBe('/data/images/frame-a.png');+ const audio = screen.getByLabelText('Song for the storyboard animatic');+ Object.defineProperty(audio, 'currentTime', { value: 12, configurable: true });+ fireEvent.timeUpdate(audio);+ expect(screen.queryByLabelText('Clip for Opening')).toBeNull();+ expect(container.querySelector('img').getAttribute('src')).toBe('/data/images/frame-b.png');+ });++ it('maps song time into the clip: trimmed in-point, then loop or hold past its end', () => {+ const performance = {+ sceneId: 'p', startSec: 10, endSec: 20, shotMode: 'performance', videoHistoryId: 'v1',+ takes: [{ kind: 'video', assetId: 'v1', shotInstruction: { shotMode: 'performance', edit: { inSec: 2, outSec: 9 } } }],+ };+ expect(clipTimeAt(performance, 13, 9)).toBe(5);+ expect(clipTimeAt({ startSec: 10 }, 16, 4)).toBe(2);+ expect(clipTimeAt({ startSec: 10, loop: false }, 16, 4)).toBeCloseTo(3.95);+ });+});diff --git a/client/src/lib/musicVideoShotTiming.js b/client/src/lib/musicVideoShotTiming.jsindex 4a9fda7ed2..33d8cc89c3 100644--- a/client/src/lib/musicVideoShotTiming.js+++ b/client/src/lib/musicVideoShotTiming.js@@ -19,5 +19,6 @@ export {performanceCapability,planPerformanceWindow,planPerformanceRepair,+ selectedPerformanceInstruction,shotSplitLimit,} from '../../../server/lib/musicVideoShotTiming.js';diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsxindex e191c583a4..67df5fa1b6 100644--- a/client/src/pages/MusicVideo.test.jsx+++ b/client/src/pages/MusicVideo.test.jsx@@ -1688,6 +1688,9 @@ describe('MusicVideo lyrics and shot coverage (#8964)', () => {});it('flags a non-looping shot longer than its clip and trims it; a legacy scene is left alone', async () => {+ // The scene cards' players; the preview dock's storyboard animatic plays the shot's clip on its own.+ const cardPlayers = () => [...document.body.querySelectorAll('video[src^="/data/videos/h"]')]+ .filter((video) => !video.closest('figure[aria-label="Storyboard animatic"]'));const project = {...PROJECT_WITH_CLIP,scenes: [@@ -1700,7 +1703,7 @@ describe('MusicVideo lyrics and shot coverage (#8964)', () => {// The card's summary row and its editor both quote the lyric.expect(screen.getAllByText(/first line/).length).toBeGreaterThan(0);- const players = document.body.querySelectorAll('video[src^="/data/videos/h"]');+ const players = cardPlayers();expect(players).toHaveLength(1); // only the selected shot owns a playerfor (const player of players) {Object.defineProperty(player, 'duration', { configurable: true, value: 5 });@@ -1715,7 +1718,7 @@ describe('MusicVideo lyrics and shot coverage (#8964)', () => {await waitFor(() => expect(screen.queryByRole('alert')).toBeNull());await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Open Scene 2' })));- const legacyPlayers = document.body.querySelectorAll('video[src^="/data/videos/h"]');+ const legacyPlayers = cardPlayers();expect(legacyPlayers).toHaveLength(1);expect(legacyPlayers[0]).toHaveAttribute('src', '/data/videos/h2.mp4');Object.defineProperty(legacyPlayers[0], 'duration', { configurable: true, value: 5 });