Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Play each shot's video clip in the storyboard animatic (#11195)

[._.]/027baea05e15
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index 5211a09547..0ba4b09d70 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -3,6 +3,8 @@ import { Captions } from 'lucide-react';
import { formatDurationSec } from '../../utils/formatters.js';
import { getMusicVideoLyricOverlayPreview } from '../../services/apiMusicVideo.js';
import { playthroughVersion } from './CompositionPreviewPlayer.jsx';
+import { useVideoFileSrc } from '../../hooks/useVideoFileSrc.js';
+import { selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
import { normalizeTitleCard } from '../../../../server/lib/musicVideoTitleCard.js';
const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);
@@ -138,10 +140,70 @@ function useLyricOverlay({ project, enabled, audioRef, time }) {
};
}
+// How far a shot's clip may drift from the song before it is put back in step.
+const CLIP_DRIFT_SEC = 0.25;
+
+/**
+ * Where song time `t` lands in a shot's clip: the take's edit-in point (a
+ * performance take trimmed to the sung window) plus the time into the shot. Past
+ * the clip's end it loops (`loop`, the render's default) or holds the last frame.
+ */
+export function clipTimeAt(scene, t, clipSec = null) {
+ const inSec = selectedPerformanceInstruction(scene)?.edit?.inSec ?? 0;
+ const local = Math.max(0, t - (scene.startSec || 0));
+ if (!(clipSec > inSec)) return inSec + local;
+ const span = clipSec - inSec;
+ if (local < span) return inSec + local;
+ return scene.loop !== false ? inSec + (local % span) : Math.max(inSec, clipSec - 0.05);
+}
+
+/**
+ * A shot's selected video clip, muted under the master song and kept in step
+ * with it: it plays and pauses with the song, and a seek or drift puts it back
+ * at the song's place in the shot. The shot's frame is its poster.
+ */
+function ShotClip({ scene, audioRef, time }) {
+ const videoRef = useRef(null);
+ const file = useVideoFileSrc(scene.videoHistoryId);
+ const src = file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`);
+ const sync = useCallback(() => {
+ const video = videoRef.current;
+ const audio = audioRef.current;
+ if (!video || !audio || video.readyState < 1) return;
+ const target = clipTimeAt(scene, audio.currentTime || 0, Number.isFinite(video.duration) ? video.duration : null);
+ if (Math.abs((video.currentTime || 0) - target) > CLIP_DRIFT_SEC) video.currentTime = target;
+ if (audio.paused) video.pause?.();
+ else if (video.paused) video.play?.()?.catch?.(() => {});
+ }, [scene, audioRef]);
+ useEffect(() => { sync(); }, [time, sync]);
+ useEffect(() => {
+ const audio = audioRef.current;
+ if (!audio) return undefined;
+ audio.addEventListener('play', sync);
+ audio.addEventListener('pause', sync);
+ return () => { audio.removeEventListener('play', sync); audio.removeEventListener('pause', sync); };
+ }, [audioRef, sync]);
+ if (!src) return null;
+ return (
+ <video
+ ref={videoRef}
+ src={src}
+ poster={scene.referenceImageId ? `/data/images/${scene.referenceImageId}` : undefined}
+ muted
+ playsInline
+ preload="auto"
+ onLoadedMetadata={sync}
+ aria-label={`Clip for ${scene.label || 'this shot'}`}
+ className="h-full w-full object-contain"
+ />
+ );
+}
+
/**
* Something to watch before anything is rendered: the master song under the
- * storyboard, showing each shot's selected frame (or, without one, its label,
- * time and intent) as the song reaches it, with the aligned lyrics drawn over
+ * storyboard, showing each shot's selected video clip (muted, in step with the
+ * song) or else its selected frame (or, without either, its label, time and
+ * intent) as the song reaches it, with the aligned lyrics drawn over
* it once the words are timed (a toggle hides them). `seekRequest`
* (`{ t, n }`) is the scene-card seek the other preview sources honour too.
*/
@@ -170,7 +232,9 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
return (
<div className={`space-y-2 ${collapsed ? 'max-xl:hidden' : ''}`}>
<figure aria-label="Storyboard animatic" className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black">
- {shot?.referenceImageId ? (
+ {shot?.videoHistoryId ? (
+ <ShotClip key={`${shot.sceneId}:${shot.videoHistoryId}`} scene={shot} audioRef={audioRef} time={time} />
+ ) : shot?.referenceImageId ? (
<img src={`/data/images/${shot.referenceImageId}`} alt="" className="h-full w-full object-contain" />
) : cardInFrame ? (
<div className="flex h-full w-full flex-col items-center justify-center gap-1 p-4 text-center">
@@ -183,7 +247,7 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
<div className="flex min-w-0 items-start gap-2">
<div className="min-w-0 flex-1 text-[11px] text-port-text-muted">
{shot && <p>Shot {index + 1} of {timed.length} · {formatDurationSec(shot.startSec)}</p>}
- {!cardInFrame && shot && !shot.referenceImageId && (
+ {!cardInFrame && shot && !shot.referenceImageId && !shot.videoHistoryId && (
<p className="line-clamp-2 break-words">
<span className="font-medium text-port-text">{label}</span>{shot.visualIntent ? ` · ${shot.visualIntent}` : ''}
</p>
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
index 8dbe23556e..d83e07eea6 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
-import StoryboardAnimatic, { shotAt } from './StoryboardAnimatic.jsx';
+import StoryboardAnimatic, { clipTimeAt, shotAt } from './StoryboardAnimatic.jsx';
const scenes = [
{ sceneId: 'b', label: 'Chorus', startSec: 10, endSec: 20, referenceImageId: 'frame-b.png' },
@@ -69,3 +69,37 @@ describe('StoryboardAnimatic lyrics', () => {
expect(screen.queryByRole('button', { name: 'Lyrics' })).toBeNull();
});
});
+
+vi.mock('../../services/apiImageVideo.js', () => ({
+ getVideoHistoryItem: vi.fn(async (id) => ({ id, filename: `${id}.mov` })),
+}));
+
+describe('StoryboardAnimatic clips', () => {
+ const clipScenes = [
+ { sceneId: 'a', label: 'Opening', startSec: 0, endSec: 10, referenceImageId: 'frame-a.png', videoHistoryId: 'clip-a' },
+ { sceneId: 'b', label: 'Chorus', startSec: 10, endSec: 20, referenceImageId: 'frame-b.png' },
+ ];
+
+ it('plays a shot\'s clip muted where it has one and its still where it does not', async () => {
+ const { container } = render(<StoryboardAnimatic project={{ id: 'p', scenes: clipScenes }} audioUrl="/song.mp3" />);
+ const clip = await screen.findByLabelText('Clip for Opening');
+ await waitFor(() => expect(clip.getAttribute('src')).toBe('/data/videos/clip-a.mov'));
+ expect(clip.muted).toBe(true);
+ expect(clip.getAttribute('poster')).toBe('/data/images/frame-a.png');
+ const audio = screen.getByLabelText('Song for the storyboard animatic');
+ Object.defineProperty(audio, 'currentTime', { value: 12, configurable: true });
+ fireEvent.timeUpdate(audio);
+ expect(screen.queryByLabelText('Clip for Opening')).toBeNull();
+ expect(container.querySelector('img').getAttribute('src')).toBe('/data/images/frame-b.png');
+ });
+
+ it('maps song time into the clip: trimmed in-point, then loop or hold past its end', () => {
+ const performance = {
+ sceneId: 'p', startSec: 10, endSec: 20, shotMode: 'performance', videoHistoryId: 'v1',
+ takes: [{ kind: 'video', assetId: 'v1', shotInstruction: { shotMode: 'performance', edit: { inSec: 2, outSec: 9 } } }],
+ };
+ expect(clipTimeAt(performance, 13, 9)).toBe(5);
+ expect(clipTimeAt({ startSec: 10 }, 16, 4)).toBe(2);
+ expect(clipTimeAt({ startSec: 10, loop: false }, 16, 4)).toBeCloseTo(3.95);
+ });
+});
diff --git a/client/src/lib/musicVideoShotTiming.js b/client/src/lib/musicVideoShotTiming.js
index 4a9fda7ed2..33d8cc89c3 100644
--- a/client/src/lib/musicVideoShotTiming.js
+++ b/client/src/lib/musicVideoShotTiming.js
@@ -19,5 +19,6 @@ export {
performanceCapability,
planPerformanceWindow,
planPerformanceRepair,
+ selectedPerformanceInstruction,
shotSplitLimit,
} from '../../../server/lib/musicVideoShotTiming.js';
diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsx
index e191c583a4..67df5fa1b6 100644
--- a/client/src/pages/MusicVideo.test.jsx
+++ b/client/src/pages/MusicVideo.test.jsx
@@ -1688,6 +1688,9 @@ describe('MusicVideo lyrics and shot coverage (#8964)', () => {
});
it('flags a non-looping shot longer than its clip and trims it; a legacy scene is left alone', async () => {
+ // The scene cards' players; the preview dock's storyboard animatic plays the shot's clip on its own.
+ const cardPlayers = () => [...document.body.querySelectorAll('video[src^="/data/videos/h"]')]
+ .filter((video) => !video.closest('figure[aria-label="Storyboard animatic"]'));
const project = {
...PROJECT_WITH_CLIP,
scenes: [
@@ -1700,7 +1703,7 @@ describe('MusicVideo lyrics and shot coverage (#8964)', () => {
// The card's summary row and its editor both quote the lyric.
expect(screen.getAllByText(/first line/).length).toBeGreaterThan(0);
- const players = document.body.querySelectorAll('video[src^="/data/videos/h"]');
+ const players = cardPlayers();
expect(players).toHaveLength(1); // only the selected shot owns a player
for (const player of players) {
Object.defineProperty(player, 'duration', { configurable: true, value: 5 });
@@ -1715,7 +1718,7 @@ describe('MusicVideo lyrics and shot coverage (#8964)', () => {
await waitFor(() => expect(screen.queryByRole('alert')).toBeNull());
await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Open Scene 2' })));
- const legacyPlayers = document.body.querySelectorAll('video[src^="/data/videos/h"]');
+ const legacyPlayers = cardPlayers();
expect(legacyPlayers).toHaveLength(1);
expect(legacyPlayers[0]).toHaveAttribute('src', '/data/videos/h2.mp4');
Object.defineProperty(legacyPlayers[0], 'duration', { configurable: true, value: 5 });