Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Storyboards: Fit to song writes a music video's shots from its board (#13002)

[._.]/10797e4c9cdc
diff --git a/client/src/components/musicVideo/StoryboardFitPanel.jsx b/client/src/components/musicVideo/StoryboardFitPanel.jsx
new file mode 100644
index 0000000000..144fbee6db
--- /dev/null
+++ b/client/src/components/musicVideo/StoryboardFitPanel.jsx
@@ -0,0 +1,109 @@
+import { useMemo, useState } from 'react';
+import { AudioLines, Loader2 } from 'lucide-react';
+import InlineConfirmRow from '../ui/InlineConfirmRow';
+import { fitMusicVideoToStoryboard } from '../../services/api';
+import { formatCount, formatTimecode } from '../../utils/formatters';
+import { fitStoryboardToSong, STORYBOARD_FIT_MIN_SHOT_SEC, storyboardFitSong } from '../../../../server/lib/storyboardFit.js';
+
+const actionButton = 'min-h-[44px] inline-flex items-center justify-center gap-1.5 rounded-lg border border-port-border px-3 text-sm text-gray-200 hover:border-port-accent/60 hover:text-white disabled:opacity-40';
+const plural = (n, word) => `${formatCount(n)} ${word}${n === 1 ? '' : 's'}`;
+const SNAP_LABELS = { start: 'Song start', section: 'On a section', lyric: 'On a sung line', downbeat: 'On a downbeat', picked: 'Your pick' };
+const MARK_KINDS = { section: 'Section', lyric: 'Line' };
+
+/**
+ * Fit to song: the linked board's beats placed on the analyzed song, shown in
+ * place (where each beat starts, its section and how its length changes) with
+ * a start picker per beat, then written as one shot per panel. The preview
+ * and the write run the same fit, so the shots written are the ones shown.
+ */
+export default function StoryboardFitPanel({ project, board, disabled, onFitted, onBoardStale }) {
+ const [open, setOpen] = useState(false);
+ const [starts, setStarts] = useState({});
+ const [confirming, setConfirming] = useState(false);
+ const [writing, setWriting] = useState(false);
+ const [error, setError] = useState(null);
+ const song = useMemo(() => storyboardFitSong(project), [project]);
+ const fit = useMemo(() => (open ? fitStoryboardToSong(board, song, { starts }) : null), [open, board, song, starts]);
+ const existing = (project.scenes || []).length;
+
+ if (!song) return <p className="text-xs text-gray-400">Analyze the song on the Song step to fit this board to it.</p>;
+ if (!open) {
+ return (
+ <button type="button" onClick={() => setOpen(true)} disabled={disabled || !board.beats.length} className={actionButton}>
+ <AudioLines size={16} aria-hidden="true" /> Fit to song
+ </button>
+ );
+ }
+
+ const write = async () => {
+ setConfirming(false);
+ setWriting(true);
+ setError(null);
+ const res = await fitMusicVideoToStoryboard(project.id, { boardUpdatedAt: board.updatedAt, starts }, { silent: true })
+ .catch((err) => {
+ setError(err?.message || 'The shots could not be written.');
+ // The fit above is redrawn from the board as it is now.
+ if (err?.code === 'STORYBOARD_CHANGED') onBoardStale();
+ return null;
+ });
+ setWriting(false);
+ if (res?.project) {
+ onFitted(res.project);
+ setOpen(false);
+ }
+ };
+ const pick = (beatId, value) => setStarts((current) => {
+ const next = { ...current };
+ if (value === '') delete next[beatId];
+ else next[beatId] = Number(value);
+ return next;
+ });
+
+ return (
+ <section aria-label="Fit to song" className="space-y-2 rounded-lg border border-port-border p-2">
+ <p className="text-xs text-gray-400">
+ {plural(fit.beats.length, 'beat')} become {plural(fit.shots.length, 'shot')} over <span className="tabular-nums">{formatTimecode(song.durationSec)}</span>, one per panel. Pick where a beat starts to move it.
+ </p>
+ <ol className="space-y-1.5" aria-label="Beats on the song">
+ {fit.beats.map((beat, index) => (
+ <li key={beat.beatId} className="space-y-1 rounded border border-port-border/60 p-1.5 text-xs text-gray-400">
+ <div className="flex flex-wrap items-baseline gap-x-2">
+ <span className="text-sm text-white">{index + 1}. {beat.title}</span>
+ <span className="tabular-nums">{formatTimecode(beat.startSec)}–{formatTimecode(beat.endSec)}</span>
+ {beat.sectionLabel && <span>{beat.sectionLabel}</span>}
+ <span className="tabular-nums">board {formatTimecode(beat.boardSec)} → song {formatTimecode(beat.songSec)}</span>
+ </div>
+ {index > 0 && (
+ <div className="flex items-center gap-2">
+ <label htmlFor={`fit-start-${beat.beatId}`} className="shrink-0">Starts</label>
+ <select id={`fit-start-${beat.beatId}`} value={starts[beat.beatId] ?? ''} onChange={(e) => pick(beat.beatId, e.target.value)}
+ className="min-h-[44px] min-w-0 flex-1 bg-port-bg border border-port-border rounded px-2 text-sm text-white">
+ <option value="">Auto: {starts[beat.beatId] === undefined ? `${SNAP_LABELS[beat.snap] || 'In proportion'}, ` : ''}{formatTimecode(beat.startSec)}</option>
+ {beat.options.map((mark) => (
+ <option key={`${mark.kind}-${mark.at}`} value={mark.at}>{formatTimecode(mark.at)} · {MARK_KINDS[mark.kind]}: {mark.label}</option>
+ ))}
+ </select>
+ </div>
+ )}
+ {beat.noPanels && <p>No panels yet; one shot from the beat's own words.</p>}
+ {beat.short > 0 && <p className="text-port-warning">{plural(beat.short, 'shot')} under {formatCount(STORYBOARD_FIT_MIN_SHOT_SEC)} s; the song is short for this many panels.</p>}
+ </li>
+ ))}
+ </ol>
+ {error && <p role="alert" className="text-xs text-port-error">{error}</p>}
+ {confirming ? (
+ <InlineConfirmRow autoFocus aria-label="Replace the shots" tone="warning" confirmText="Write shots"
+ question={`Replace the ${plural(existing, 'shot')} with ${plural(fit.shots.length, 'shot')} from the board? A shot whose span stays the same keeps its frames and clips.`}
+ onConfirm={write} onCancel={() => setConfirming(false)} />
+ ) : (
+ <div className="flex flex-wrap gap-2">
+ <button type="button" onClick={() => (existing ? setConfirming(true) : write())} disabled={disabled || writing || !fit.shots.length}
+ className={`${actionButton} bg-port-accent/20 border-port-accent/50`}>
+ {writing ? <Loader2 size={16} className="animate-spin" aria-hidden="true" /> : <AudioLines size={16} aria-hidden="true" />} Write {plural(fit.shots.length, 'shot')}
+ </button>
+ <button type="button" onClick={() => { setOpen(false); setError(null); }} className={actionButton}>Close</button>
+ </div>
+ )}
+ </section>
+ );
+}
diff --git a/client/src/components/musicVideo/StoryboardLinkPanel.jsx b/client/src/components/musicVideo/StoryboardLinkPanel.jsx
index 60250601d4..be7536a45f 100644
--- a/client/src/components/musicVideo/StoryboardLinkPanel.jsx
+++ b/client/src/components/musicVideo/StoryboardLinkPanel.jsx
@@ -4,6 +4,7 @@ import { ExternalLink, LayoutPanelLeft, Loader2, Unlink } from 'lucide-react';
import { getStoryboard, listStoryboards, makeStoryboardFromMusicVideo } from '../../services/api';
import { formatCount, formatTimecode } from '../../utils/formatters';
import { selectedTake, takeSrc } from '../storyboards/storyboardArt';
+import StoryboardFitPanel from './StoryboardFitPanel';
const actionButton = 'min-h-[44px] inline-flex items-center justify-center gap-1.5 rounded-lg border border-port-border px-3 text-sm text-gray-200 hover:border-port-accent/60 hover:text-white disabled:opacity-40';
const plural = (n, word) => `${formatCount(n)} ${word}${n === 1 ? '' : 's'}`;
@@ -11,10 +12,10 @@ const STRIP_PANELS = 8;
/**
* The Plan step's link to a Create › Storyboards board: pick one, or make one
- * from this video's shots, then see it at a glance and open it. Linking
- * changes no shot.
+ * from this video's shots, then see it at a glance, open it, and fit it to
+ * the song. Linking changes no shot; only Fit to song writes them.
*/
-export default function StoryboardLinkPanel({ project, onLink }) {
+export default function StoryboardLinkPanel({ project, onLink, fitDisabled, onFitted }) {
const linkedId = project.storyboardId || null;
// undefined = loading, null = missing on this machine.
const [board, setBoard] = useState(undefined);
@@ -22,16 +23,18 @@ export default function StoryboardLinkPanel({ project, onLink }) {
const [boards, setBoards] = useState(null);
const [listError, setListError] = useState(false);
const [making, setMaking] = useState(false);
+ const [reads, setReads] = useState(0);
useEffect(() => {
if (!linkedId) return undefined;
let active = true;
- setBoard(undefined);
+ // A re-read of the same board keeps it on screen; another board shows as loading.
+ setBoard((current) => (current?.id === linkedId ? current : undefined));
getStoryboard(linkedId, { silent: true })
.then((next) => { if (active) setBoard(next); })
.catch(() => { if (active) setBoard(null); });
return () => { active = false; };
- }, [linkedId]);
+ }, [linkedId, reads]);
// The board list is read when the picker is first touched, not on every Plan visit.
const loadBoards = () => {
@@ -91,6 +94,7 @@ export default function StoryboardLinkPanel({ project, onLink }) {
{panels.length > STRIP_PANELS && <li className="flex h-12 shrink-0 items-center px-1 text-[11px] text-gray-500">+{formatCount(panels.length - STRIP_PANELS)}</li>}
</ol>
)}
+ <StoryboardFitPanel project={project} board={board} disabled={fitDisabled} onFitted={onFitted} onBoardStale={() => setReads((n) => n + 1)} />
</div>
);
}
diff --git a/client/src/components/musicVideo/StoryboardLinkPanel.test.jsx b/client/src/components/musicVideo/StoryboardLinkPanel.test.jsx
index 1fe9f4349c..b0aea7883c 100644
--- a/client/src/components/musicVideo/StoryboardLinkPanel.test.jsx
+++ b/client/src/components/musicVideo/StoryboardLinkPanel.test.jsx
@@ -3,15 +3,15 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import { MemoryRouter } from 'react-router';
import StoryboardLinkPanel from './StoryboardLinkPanel';
-const api = vi.hoisted(() => ({ getStoryboard: vi.fn(), listStoryboards: vi.fn(), makeStoryboardFromMusicVideo: vi.fn() }));
+const api = vi.hoisted(() => ({ getStoryboard: vi.fn(), listStoryboards: vi.fn(), makeStoryboardFromMusicVideo: vi.fn(), fitMusicVideoToStoryboard: vi.fn() }));
vi.mock('../../services/api', () => api);
const board = {
id: 'sb-1', name: 'Example Story',
beats: [{ id: 'beat-1', panels: [{ id: 'panel-1', action: 'Wide of the gym', seconds: 4, takes: [], selectedTakeId: null }] }],
};
-const renderPanel = (project, onLink = vi.fn()) => {
- render(<MemoryRouter><StoryboardLinkPanel project={project} onLink={onLink} /></MemoryRouter>);
+const renderPanel = (project, onLink = vi.fn(), onFitted = vi.fn()) => {
+ render(<MemoryRouter><StoryboardLinkPanel project={project} onLink={onLink} onFitted={onFitted} /></MemoryRouter>);
return onLink;
};
@@ -46,4 +46,41 @@ describe('StoryboardLinkPanel', () => {
fireEvent.click(screen.getByRole('button', { name: /Unlink/ }));
expect(onLink).toHaveBeenCalledWith(null);
});
+
+ it('previews the fit on the song, lets a beat start move, then writes the shots after confirming the replace', async () => {
+ const fitBoard = {
+ id: 'sb-1', name: 'Example Story', updatedAt: '2026-01-01T00:00:00.000Z',
+ beats: [
+ { id: 'beat-1', title: 'Arrival', panels: [{ id: 'panel-1', action: 'Wide of the gym', seconds: 4, takes: [], selectedTakeId: null }] },
+ { id: 'beat-2', title: 'Lift off', panels: [{ id: 'panel-2', action: 'Crowd lifts her', seconds: 4, takes: [], selectedTakeId: null }] },
+ ],
+ };
+ api.getStoryboard.mockResolvedValue(fitBoard);
+ const saved = { id: 'mv-1', scenes: [{ sceneId: 'mvs-new' }] };
+ api.fitMusicVideoToStoryboard.mockResolvedValue({ project: saved, shots: 2, replaced: 1 });
+ const onFitted = vi.fn();
+ renderPanel({
+ id: 'mv-1', storyboardId: 'sb-1', scenes: [{ sceneId: 'mvs-old' }],
+ audioAnalysis: { durationSec: 40, beats: [], downbeats: [], sections: [{ label: 'Verse', startSec: 0, endSec: 18 }, { label: 'Chorus', startSec: 18, endSec: 40 }] },
+ lyricCues: [{ id: 'c1', text: 'Here we go', startSec: 22 }],
+ }, vi.fn(), onFitted);
+
+ fireEvent.click(await screen.findByRole('button', { name: /Fit to song/ }));
+ expect(screen.getByText('0:18.00–0:40.00')).toBeTruthy();
+ const start = screen.getByLabelText('Starts');
+ fireEvent.change(start, { target: { value: '22' } });
+ expect(screen.getByText('0:22.00–0:40.00')).toBeTruthy();
+
+ fireEvent.click(screen.getByRole('button', { name: /Write 2 shots/ }));
+ expect(api.fitMusicVideoToStoryboard).not.toHaveBeenCalled();
+ fireEvent.click(screen.getByRole('button', { name: 'Write shots' }));
+ await waitFor(() => expect(onFitted).toHaveBeenCalledWith(saved));
+ expect(api.fitMusicVideoToStoryboard).toHaveBeenCalledWith('mv-1', { boardUpdatedAt: fitBoard.updatedAt, starts: { 'beat-2': 22 } }, { silent: true });
+ });
+
+ it('asks for an analyzed song before fitting', async () => {
+ api.getStoryboard.mockResolvedValue(board);
+ renderPanel({ id: 'mv-1', storyboardId: 'sb-1', scenes: [] });
+ expect(await screen.findByText(/Analyze the song on the Song step/)).toBeTruthy();
+ });
});
diff --git a/client/src/components/musicVideo/stages/PlanStage.jsx b/client/src/components/musicVideo/stages/PlanStage.jsx
index f1fc800d3c..9a83f2ba61 100644
--- a/client/src/components/musicVideo/stages/PlanStage.jsx
+++ b/client/src/components/musicVideo/stages/PlanStage.jsx
@@ -14,7 +14,7 @@ import StoryboardLinkPanel from '../StoryboardLinkPanel.jsx';
export default function PlanStage({ board }) {
const { project, locked, busy, treatment } = board;
const scenes = project.scenes || [];
- // Linking a board changes no shot, so it stays usable while shots are locked.
+ // Linking a board changes no shot, so it stays usable while shots are locked; Fit to song writes them and is not.
const linkStoryboard = (storyboardId, { saved = false } = {}) => {
board.editProjectLocal({ storyboardId });
if (!saved) board.saveProjectFields({ storyboardId });
@@ -43,7 +43,8 @@ export default function PlanStage({ board }) {
)}
</fieldset>
<StageSection id="mv-board-storyboard" title="Storyboard" summary={project.storyboardId ? 'Linked' : 'Plan the story as beats and panels'}>
- <StoryboardLinkPanel project={project} onLink={linkStoryboard} />
+ <StoryboardLinkPanel project={project} onLink={linkStoryboard} fitDisabled={locked || busy.planning || busy.arranging}
+ onFitted={(saved) => board.editProjectLocal(saved)} />
</StageSection>
</div>
);
diff --git a/client/src/services/README.md b/client/src/services/README.md
index e020bfe8a0..4c44cd9e1e 100644
--- a/client/src/services/README.md
+++ b/client/src/services/README.md
@@ -87,7 +87,7 @@ toasts on throw). **Custom catch ⇒ `silent: true`** — otherwise toasts fire
| `apiMeatspace.js` | MeatSpace health, genome, POST, memory-practice, and atomic POST training-run APIs. |
| `apiMortalLoom.js` | Mortality tracking. |
| `apiMoodBoard.js` | Mood boards (inspiration canvas + items), `listMoodBoardNames()` for pickers, `composeMoodBoardPrompt()` which stores the board's composite style prompt. |
-| `apiStoryboards.js` | Storyboards (Create › Storyboards): board/beat/panel CRUD and moves, panel art takes (add, select, remove), the panel maker (board recipe, per-panel node values, node runs, Make empty panels), beats from a plot summary (preview, then add), the saved recipe library and Make storyboard from a music video; every board write returns the whole board. |
+| `apiStoryboards.js` | Storyboards (Create › Storyboards): board/beat/panel CRUD and moves, panel art takes (add, select, remove), the panel maker (board recipe, per-panel node values, node runs, Make empty panels), beats from a plot summary (preview, then add), the saved recipe library, Make storyboard from a music video and Fit to song (write a video's shots from its linked board); every board write returns the whole board. |
| `apiTribe.js` | Tribe people (relationship rings + contacts), plus `getTribeDuplicateIdentifiers()` (`GET /tribe/duplicate-identifiers`): a non-blocking report of emails/phones shared by more than one person. |
| `apiTimeline.js` | Human-activity timeline: `/timeline/day`. |
| `apiCalendar.js` | Calendar events. |
diff --git a/client/src/services/apiStoryboards.js b/client/src/services/apiStoryboards.js
index 8323863089..62c6bef62d 100644
--- a/client/src/services/apiStoryboards.js
+++ b/client/src/services/apiStoryboards.js
@@ -78,3 +78,8 @@ export const addStoryboardBeats = (id, beats, options) =>
// A new board built from a music video's shots, linked to it → { board, project }.
export const makeStoryboardFromMusicVideo = (projectId, options) =>
request(`/storyboards/from-music-video/${enc(projectId)}`, { method: 'POST', ...options });
+
+// Fit to song: the video's shots written from its linked board, as previewed (lib/storyboardFit.js)
+// → { project, shots, replaced }. 409 when the board changed since the preview or a shot is locked.
+export const fitMusicVideoToStoryboard = (projectId, input, options) =>
+ request(`/storyboards/fit-music-video/${enc(projectId)}`, { ...json('POST', input), ...options });
diff --git a/docs/API.md b/docs/API.md
index 45dba37fa6..3cdae38dc3 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -808,7 +808,7 @@ Every mounted API prefix (see `server/index.js` for the authoritative list). Dom
| `/api/music-video` | Music video projects; [one-prompt autonomous runs](./features/music-video-autonomous.md) at `/api/music-video/autonomous` |
| `/api/music-video/character-styles` | Built-in character styles a project loads via `concept.characterStyleId`: list with each style's sheet prompt and this install's sheet (`GET /`), full style (`GET /:id`), set or clear the install's character sheet (`PUT /:id/reference` `{ imageId }`) |
| `/api/mood-boards` | Mood boards; `POST /:id/generate` writes 1–4 prompts with the chosen LLM and queues new image items using saved imagegen settings (1024×576 default). Body: `requestId` (UUID), `providerId`, optional `model`, `effort`, `direction`, `count`, `render` (`mode`, `model`, `width`, `height`), `retry`. `GET /:id/generate` restores transient status; `mood-board:generate` pushes progress. Retry reuses the original batch and only queues unfinished slots. Prompt writing uses stored text, not pixels; saved prompts and references survive navigation and server restart, while transient batch status and recent request receipts reset on restart. |
-| `/api/storyboards` | Storyboards (Create › Storyboards): `GET`/`POST /`, `GET`/`PATCH`/`DELETE /:id`; beats at `/:id/beats` (`POST`, `PATCH`/`DELETE /:beatId`, `POST /:beatId/move`); panels at `/:id/beats/:beatId/panels` (`POST`) and `/:id/panels/:panelId` (`PATCH`, `DELETE`, `POST /move` with `{ beatId?, index }`); panel art takes at `/:id/panels/:panelId/takes` (`POST` `{ source, image \| sketchKey }`, `POST /:takeId/select`, `DELETE /:takeId`); the panel maker at `PUT /:id/recipe` (`{ recipe \| null }`, null = default), `GET /:id/panels/:panelId/make` (linked catalog reference images), `PATCH /:id/panels/:panelId/make/:nodeId` (`{ text?, excludedRefs?, extraRefs? }`) and `POST /:id/panels/:panelId/make/:nodeId/run` (an LLM node answers when done; an image node once its frames are queued, with `storyboard:make` socket events after). Beats from a plot summary: `POST /:id/beats/extract` (`{ plot, providerId?, model?, effort? }` → `{ board, beats, sentPrompt, promptAdditions }`: the plot and pick are saved, the beats are a preview) and `POST /:id/beats/add` (`{ beats }`, appended). Make empty panels: `GET /:id/make-empty` (`{ batch, panelCount, steps }`: the panels with no art and each node it runs with its model) and `POST /:id/make-empty` (202, one batch per board, `storyboard:make-batch` progress). The saved recipe library: `GET`/`POST /recipes` (`{ name, recipe }`) and `DELETE /recipes/:recipeId`. The animatic's temp track is `PATCH /:id` `{ tempTrack: { filename, name?, sections?, lines? } \| null }`, naming an audio file already in `/api/uploads`, with its song marks (`sections` `{ id?, label, at }`, lyric `lines` `{ id?, text, at \| null }`); a beat starts at one with `PATCH …/beats/:beatId` `{ pin }`. Make storyboard from a music video: `POST /from-music-video/:projectId` (201 `{ board, project }`: one beat per song section, one panel per shot, and the project's `storyboardId` set; 409 while it links a live board). Every write returns the whole board; over-limit text is refused. |
+| `/api/storyboards` | Storyboards (Create › Storyboards): `GET`/`POST /`, `GET`/`PATCH`/`DELETE /:id`; beats at `/:id/beats` (`POST`, `PATCH`/`DELETE /:beatId`, `POST /:beatId/move`); panels at `/:id/beats/:beatId/panels` (`POST`) and `/:id/panels/:panelId` (`PATCH`, `DELETE`, `POST /move` with `{ beatId?, index }`); panel art takes at `/:id/panels/:panelId/takes` (`POST` `{ source, image \| sketchKey }`, `POST /:takeId/select`, `DELETE /:takeId`); the panel maker at `PUT /:id/recipe` (`{ recipe \| null }`, null = default), `GET /:id/panels/:panelId/make` (linked catalog reference images), `PATCH /:id/panels/:panelId/make/:nodeId` (`{ text?, excludedRefs?, extraRefs? }`) and `POST /:id/panels/:panelId/make/:nodeId/run` (an LLM node answers when done; an image node once its frames are queued, with `storyboard:make` socket events after). Beats from a plot summary: `POST /:id/beats/extract` (`{ plot, providerId?, model?, effort? }` → `{ board, beats, sentPrompt, promptAdditions }`: the plot and pick are saved, the beats are a preview) and `POST /:id/beats/add` (`{ beats }`, appended). Make empty panels: `GET /:id/make-empty` (`{ batch, panelCount, steps }`: the panels with no art and each node it runs with its model) and `POST /:id/make-empty` (202, one batch per board, `storyboard:make-batch` progress). The saved recipe library: `GET`/`POST /recipes` (`{ name, recipe }`) and `DELETE /recipes/:recipeId`. The animatic's temp track is `PATCH /:id` `{ tempTrack: { filename, name?, sections?, lines? } \| null }`, naming an audio file already in `/api/uploads`, with its song marks (`sections` `{ id?, label, at }`, lyric `lines` `{ id?, text, at \| null }`); a beat starts at one with `PATCH …/beats/:beatId` `{ pin }`. Make storyboard from a music video: `POST /from-music-video/:projectId` (201 `{ board, project }`: one beat per song section, one panel per shot, and the project's `storyboardId` set; 409 while it links a live board). Fit to song: `POST /fit-music-video/:projectId` (`{ boardUpdatedAt, starts? }`, `starts` = beat id → picked start seconds → `{ project, shots, replaced }`: the linked board's panels written as the video's shots, each with `panelId`/`panelRev`; 409 when the board changed since `boardUpdatedAt`, the song is not analyzed, or a shot is locked). Every write returns the whole board; over-limit text is refused. |
| `/api/decks` | Decks (playing-card / tarot designer) |
| `/api/writers-room` | Writers Room |
| `/api/universe-builder` | Universe Builder |
diff --git a/server/lib/README.md b/server/lib/README.md
index e6207375b6..b511f3aa66 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -194,6 +194,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `castIntegrityPrompt.js` | The prompt-facing render of a `characterIntegrity.js` report (#6415): `renderCastIntegrity` (gap rows first, header always kept, so a budget-truncated block can never read as a clean cast) and `INTEGRITY_DEPTH_NOTES` (the `explained` / `light` / `full` rulings in the reviewer's own words). Shared by the Series foundation judge and the FableLoom whole-series editor so the two cannot drift on what a `light` character owes. |
| `bibleLimits.js` | `BIBLE_LIMITS` — the canon field length/count caps every story-bible sanitizer, Zod schema, and catalog payload upgrade measures against. A pure leaf split out of `storyBible.js` (which pulls `crypto` + `fileUtils`) so `catalogTypes.js` and the browser bundle can read the numbers alone. |
| `storyboardBeatExtract.js` | Beats from a plot summary: `beatExtractInstructions` (the visible block sent ahead of the plot), `beatExtractPrompt`, `parseExtractedBeats` (a model's JSON answer reduced to beats and panels a board holds), `BEAT_EXTRACT_LIMITS`. Browser-safe. |
+| `storyboardFit.js` | Fit to song: `fitStoryboardToSong` places a storyboard's beats on a music video's analyzed song (snapped to sections, sung lines or downbeats; picked starts win) and returns one shot per panel with its `panelId`/`panelRev`; `storyboardFitSong` reads the song from a project; `STORYBOARD_FIT_MIN_SHOT_SEC`. Shared by the Plan step preview and the write. Browser-safe. |
| `storyboardSongBrief.js` | A storyboard's beats as a song map: `storyboardSongSections` (role, target length, energy, mood, lyric theme per beat), `storyboardSongBriefText` (the paste-ready brief), `storyboardTimeline` (the animatic's timing: panel holds, beats pinned to temp-track marks, the runs before them fitted) / `STORYBOARD_DEFAULT_HOLD_SEC`. No model call. Browser-safe. |
| `storyboardRecipe.js` | Storyboard panel maker recipes: node types and ports (`RECIPE_NODE_PORTS`: text, system, llm, references, image, panel), `defaultPanelRecipe` (Text → Prompt writer + System prompt → Image + References → Panel), `sanitizeRecipe` (edges join matching ports, no cycles), `recipeRunOrder`, `nodeInputValue` / `nodeOutputValue` (what flows into a node for one panel), `sanitizePanelMake` (per-panel run state), `STORYBOARD_RENDER_SIZES`. Browser-safe. |
| `storyboardRecord.js` | Shared Storyboard record (Create › Storyboards): `sanitizeStoryboard` (beats → panels → art takes, catalog refs by id), `sanitizeTake`, `summarizeStoryboard` (with runtime), `sanitizeSavedRecipe` (a saved recipe library entry). |
diff --git a/server/lib/index.js b/server/lib/index.js
index 2f36259f26..f5ab5301bf 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -145,6 +145,7 @@ export * from './proseExportSettings.js';
export * from './shotGrammar.js';
export * from './storyboardScenes.js';
export * from './storyboardBeatExtract.js';
+export * from './storyboardFit.js';
export * from './storyboardRecipe.js';
export * from './storyboardRecord.js';
export * from './storyboardSongBrief.js';
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index f42c5f6a25..a6d60b9d0f 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -1552,6 +1552,13 @@ export const musicVideoProductionProofSchema = z.object({
// A scene on the director board. `startSec`/`endSec` place it on the timeline;
// `prompt` drives the shot's video; `framePrompt`/`referenceImageId` are the
// reference-frame inputs the i2v generation (Phase 1b) will consume.
+// The linked storyboard panel a shot was fitted from (Fit to song), and a
+// fingerprint of the panel as it was then, so a later panel edit shows on the shot.
+const scenePanelFields = {
+ panelId: z.string().regex(/^panel-[A-Za-z0-9-]{1,80}$/, 'Invalid panel id').nullable().optional(),
+ panelRev: z.string().regex(/^[0-9a-z]{1,16}$/, 'Invalid panel revision').nullable().optional(),
+};
+
export const musicVideoSceneCreateSchema = z.object({
label: z.string().max(120).optional(),
sectionLabel: z.string().max(120).nullable().optional(),
@@ -1575,6 +1582,7 @@ export const musicVideoSceneCreateSchema = z.object({
performanceSpeaker: z.string().trim().max(120).nullable().optional(),
// How a performance follows the song: lip-sync (absent/null) or music-driven motion.
performanceStyle: z.enum(MUSIC_VIDEO_PERFORMANCE_STYLES).nullable().optional(),
+ ...scenePanelFields,
}).strict().refine(
(s) => s.startSec == null || s.endSec == null || s.endSec >= s.startSec,
{ message: 'endSec must be >= startSec', path: ['endSec'] },
@@ -1625,6 +1633,7 @@ export const musicVideoSceneUpdateSchema = z.object({
// may send `null` to dismiss it; a full object is accepted so a client that
// round-trips the scene doesn't 400.
lastFailure: musicVideoSceneLastFailureSchema.nullable().optional(),
+ ...scenePanelFields,
}).strict();
// Reorder the board: the full set of scene ids in their new order.
diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.js
index 11849bfbdc..eb452e0141 100644
--- a/server/lib/schemaVersions.js
+++ b/server/lib/schemaVersions.js
@@ -740,8 +740,11 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({
// `offsetFrames`, `reverse`, `holdLastFrame`, `filmLook`, `transitionIn`/
// `transitionOut`, `glitches`, `cutEffect`, `imageEffect`/`imageEffectSec`,
// `stillMove` and `performanceStyle`. Optional fields on the same layout; no
- // migration or backfill. A field added to these normalizers later needs v23.
- musicVideoProjects: 22,
+ // migration or backfill.
+ // v23 = a shot's storyboard panel link (`scene.panelId`/`panelRev`, Fit to
+ // song), which a v22 re-plan or split rebuilds without. Optional, no backfill.
+ // A field added to these normalizers later needs v24.
+ musicVideoProjects: 23,
// v1 = Creative Commission FEEDBACK federation (PostgreSQL `commission_feedback`)
// via the per-record peer-sync push pipeline (record kind `commissionFeedback`,
// sync category `commissionFeedback`, #2686 — split-record follow-up to #2657).
diff --git a/server/lib/schemaVersions.test.js b/server/lib/schemaVersions.test.js
index ba3d9a9414..b71b9c4d44 100644
--- a/server/lib/schemaVersions.test.js
+++ b/server/lib/schemaVersions.test.js
@@ -89,17 +89,17 @@ describe('PORTOS_SCHEMA_VERSIONS', () => {
});
describe('buildPortosMeta', () => {
- it('rejects repair, timing, narrative event, lyric warning and title/ending/subject/tool metadata on older music-video peers', async () => {
+ it('rejects repair, timing, narrative event, lyric warning, title/ending/subject/tool metadata and panel links on older music-video peers', async () => {
const meta = await buildPortosMeta();
- expect(meta.schemaVersions.musicVideoProjects).toBe(22);
- // v21 builds re-normalize the v22 fields away; the gate must stop that peer receiving them.
- for (const receiverV of [13, 14, 15, 16, 17, 18, 19, 20, 21]) {
+ expect(meta.schemaVersions.musicVideoProjects).toBe(23);
+ // Older builds re-normalize the newer fields away; the gate must stop that peer receiving them.
+ for (const receiverV of [13, 14, 15, 16, 17, 18, 19, 20, 21, 22]) {
expect(compareSchemaVersions(meta.schemaVersions, { ...meta.schemaVersions, musicVideoProjects: receiverV }).ahead)
- .toContainEqual({ category: 'musicVideoProjects', senderV: 22, receiverV });
+ .toContainEqual({ category: 'musicVideoProjects', senderV: 23, receiverV });
}
- const olderPeer = { ...meta.schemaVersions, musicVideoProjects: 21 };
+ const olderPeer = { ...meta.schemaVersions, musicVideoProjects: 22 };
expect(scopeVersionDiff(compareSchemaVersions(meta.schemaVersions, olderPeer), RECORD_KIND_SCHEMA_CATEGORIES.musicVideoProject))
- .toMatchObject({ compatible: false, ahead: [{ category: 'musicVideoProjects', senderV: 22, receiverV: 21 }] });
+ .toMatchObject({ compatible: false, ahead: [{ category: 'musicVideoProjects', senderV: 23, receiverV: 22 }] });
});
it('advertises lyric provenance to pause track transfers to caption-only peers', async () => {
diff --git a/server/lib/storyboardFit.js b/server/lib/storyboardFit.js
new file mode 100644
index 0000000000..5b68d39ff6
--- /dev/null
+++ b/server/lib/storyboardFit.js
@@ -0,0 +1,188 @@
+/**
+ * Fit to song — places a storyboard's beats on a music video's analyzed song
+ * and turns each panel into a shot. Shared by the Plan step's preview (in the
+ * browser) and the write (on the server), so the shots written are the ones
+ * previewed.
+ *
+ * Each beat keeps its share of the board's length (a panel's estimated
+ * seconds, or the default hold), stretched over the whole song. A beat's start
+ * then snaps to the nearest section start (one matching its musical role
+ * first), else a sung line, else a downbeat, when one is close. A start the
+ * director picked wins. Panels split their beat in proportion to their holds,
+ * cuts on the beat grid. A beat with no panels still gets one shot from its
+ * own words, so no beat is dropped; shots under the minimum length are named.
+ *
+ * Browser-safe: no node-only imports.
+ */
+
+import { STORYBOARD_DEFAULT_HOLD_SEC } from './storyboardSongBrief.js';
+import { STORYBOARD_MUSICAL_ROLE_LABELS } from './storyboardVocabulary.js';
+
+export const STORYBOARD_FIT_MIN_SHOT_SEC = 2;
+
+const LABEL_MAX = 120;
+const TEXT_MAX = 2000;
+const CUT_SNAP_SEC = 0.3;
+const OPTIONS_MAX = 12;
+
+const round = (t) => Math.round(t * 100) / 100;
+const clip = (text, max) => String(text || '').trim().slice(0, max);
+const holdSeconds = (panel) => (panel?.seconds > 0 ? panel.seconds : STORYBOARD_DEFAULT_HOLD_SEC);
+const nearest = (list, at, within) => list.reduce((best, item) => {
+ const gap = Math.abs(item.at - at);
+ return gap <= within && (!best || gap < Math.abs(best.at - at)) ? item : best;
+}, null);
+
+/** The song a fit lands on, from a music video's analysis and timed lyric cues; null before analysis. */
+export function storyboardFitSong(project) {
+ const analysis = project?.audioAnalysis;
+ const sections = (analysis?.sections || []).filter((s) => Number.isFinite(s.startSec) && Number.isFinite(s.endSec));
+ const end = Number.isFinite(analysis?.durationSec) && analysis.durationSec > 0 ? analysis.durationSec : sections.at(-1)?.endSec;
+ if (!(end > 0)) return null;
+ return {
+ durationSec: end,
+ sections,
+ downbeats: (analysis.downbeats || []).filter(Number.isFinite),
+ beats: (analysis.beats || []).filter(Number.isFinite),
+ lyricCues: (project.lyricCues || []).filter((cue) => Number.isFinite(cue?.startSec) && cue.text),
+ };
+}
+
+/** Which analyzed section a time falls in (the last one starting at or before it). */
+function sectionAt(sections, at) {
+ let index = -1;
+ sections.forEach((section, i) => { if (section.startSec <= at + 0.01) index = i; });
+ return index < 0 ? (sections.length ? 0 : null) : index;
+}
+
+// What a shot drew from its panel; a later change to any of it marks the shot's panel changed.
+const PANEL_REV_FIELDS = ['action', 'dialogue', 'framing', 'camera', 'castIds', 'environmentIds', 'objectIds'];
+function panelRev(panel) {
+ const text = JSON.stringify(PANEL_REV_FIELDS.map((key) => panel[key] ?? null));
+ let hash = 0x811c9dc5;
+ for (let i = 0; i < text.length; i += 1) hash = Math.imul(hash ^ text.charCodeAt(i), 0x01000193);
+ return (hash >>> 0).toString(36);
+}
+
+const roleMatches = (role, label) => {
+ const name = STORYBOARD_MUSICAL_ROLE_LABELS[role];
+ return !!name && String(label || '').toLowerCase().startsWith(name.toLowerCase());
+};
+
+/**
+ * The fit: `beats` (each beat's place, section and length change, with the
+ * starts the director can pick from) and `shots` (scene inputs in song order,
+ * each carrying its `beatId` and `panelId`). `starts` maps a beat id to a
+ * picked start time; a pick that would run time backwards is ignored.
+ */
+export function fitStoryboardToSong(board, song, { starts = {}, minShotSec = STORYBOARD_FIT_MIN_SHOT_SEC } = {}) {
+ const beats = board?.beats || [];
+ if (!song || !beats.length) return { beats: [], shots: [] };
+ const songEnd = song.durationSec;
+ const holds = beats.map((beat) => (beat.panels.length ? beat.panels.reduce((sum, panel) => sum + holdSeconds(panel), 0) : STORYBOARD_DEFAULT_HOLD_SEC));
+ const total = holds.reduce((sum, hold) => sum + hold, 0);
+ const scale = songEnd / total;
+ const mapped = [];
+ holds.reduce((at, hold) => { mapped.push(at * scale); return at + hold; }, 0);
+
+ const sectionMarks = song.sections.map((section, index) => ({ at: section.startSec, kind: 'section', label: section.label, index }));
+ const lyricMarks = song.lyricCues.map((cue) => ({ at: cue.startSec, kind: 'lyric', label: clip(cue.text, 80) }));
+ const downbeatMarks = song.downbeats.map((at) => ({ at, kind: 'downbeat', label: null }));
+ // Every shot needs its minimum, so a beat's start leaves room for the beats before and after it.
+ const shotCount = beats.map((beat) => Math.max(1, beat.panels.length));
+ const after = shotCount.map((_, i) => shotCount.slice(i).reduce((sum, n) => sum + n, 0) * minShotSec);
+
+ const placed = [];
+ beats.forEach((beat, i) => {
+ const raw = mapped[i];
+ const share = holds[i] * scale;
+ const low = i === 0 ? 0 : placed[i - 1].start + shotCount[i - 1] * minShotSec;
+ const high = Math.max(low, songEnd - after[i]);
+ const fits = (at) => at >= low - 0.001 && at <= high + 0.001;
+ const window = Math.max(1.5, Math.min(6, share * 0.35));
+ const prevShare = i > 0 ? holds[i - 1] * scale : share;
+ const options = i === 0 ? [] : [...sectionMarks, ...lyricMarks]
+ .filter((mark) => mark.at > raw - prevShare && mark.at < raw + share && fits(mark.at))
+ .sort((a, b) => Math.abs(a.at - raw) - Math.abs(b.at - raw)).slice(0, OPTIONS_MAX)
+ .sort((a, b) => a.at - b.at)
+ .map(({ at, kind, label }) => ({ at: round(at), kind, label }));
+
+ let start;
+ let snap;
+ const picked = starts[beat.id];
+ if (i === 0) {
+ start = 0;
+ snap = 'start';
+ } else if (Number.isFinite(picked) && fits(picked)) {
+ start = picked;
+ snap = 'picked';
+ } else {
+ const inRange = (marks) => marks.filter((mark) => fits(mark.at));
+ const sections = inRange(sectionMarks);
+ const found = nearest(sections.filter((mark) => roleMatches(beat.musicalRole, mark.label)), raw, window * 1.5)
+ || nearest(sections, raw, window)
+ || nearest(inRange(lyricMarks), raw, Math.min(window, 2))
+ || nearest(inRange(downbeatMarks), raw, 1);
+ start = found ? found.at : Math.min(Math.max(raw, low), high);
+ snap = found ? found.kind : null;
+ }
+ placed.push({ start: round(start), snap, options });
+ });
+
+ const cutGrid = song.beats.length ? song.beats : song.downbeats;
+ const fitBeats = [];
+ const shots = [];
+ beats.forEach((beat, i) => {
+ const start = placed[i].start;
+ const end = i + 1 < beats.length ? placed[i + 1].start : round(songEnd);
+ const sectionIndex = sectionAt(song.sections, start);
+ const sectionLabel = sectionIndex === null ? null : song.sections[sectionIndex].label;
+ const title = clip(beat.title || STORYBOARD_MUSICAL_ROLE_LABELS[beat.musicalRole] || `Beat ${i + 1}`, LABEL_MAX);
+ const panels = beat.panels.length ? beat.panels : [null];
+ const weights = panels.map((panel) => holdSeconds(panel));
+ const weightTotal = weights.reduce((sum, w) => sum + w, 0);
+ const cuts = [start];
+ weights.slice(0, -1).reduce((acc, w) => {
+ const next = acc + w;
+ const at = start + (end - start) * (next / weightTotal);
+ const snapped = nearest(cutGrid.map((t) => ({ at: t })), at, CUT_SNAP_SEC)?.at;
+ const floor = cuts.at(-1) + 0.5;
+ cuts.push(round(snapped !== undefined && snapped > floor && snapped < end - 0.5 ? snapped : at));
+ return next;
+ }, 0);
+ cuts.push(end);
+ let short = 0;
+ panels.forEach((panel, p) => {
+ const startSec = cuts[p];
+ const endSec = cuts[p + 1];
+ if (endSec - startSec < minShotSec - 0.001) short += 1;
+ const suffix = panels.length > 1 ? ` · ${p + 1}/${panels.length}` : '';
+ shots.push({
+ beatId: beat.id,
+ panelId: panel?.id || null,
+ panelRev: panel ? panelRev(panel) : null,
+ label: `${title.slice(0, LABEL_MAX - suffix.length)}${suffix}`,
+ sectionIndex,
+ sectionLabel,
+ startSec,
+ endSec,
+ visualIntent: clip(panel ? panel.action : beat.what || beat.title, TEXT_MAX) || null,
+ });
+ });
+ fitBeats.push({
+ beatId: beat.id,
+ title,
+ startSec: start,
+ endSec: end,
+ boardSec: round(holds[i]),
+ songSec: round(end - start),
+ sectionLabel,
+ snap: placed[i].snap,
+ options: placed[i].options,
+ shots: panels.length,
+ short,
+ noPanels: !beat.panels.length,
+ });
+ });
+ return { beats: fitBeats, shots };
+}
diff --git a/server/lib/storyboardFit.test.js b/server/lib/storyboardFit.test.js
new file mode 100644
index 0000000000..bf5140b724
--- /dev/null
+++ b/server/lib/storyboardFit.test.js
@@ -0,0 +1,76 @@
+import { describe, expect, it } from 'vitest';
+import { fitStoryboardToSong, storyboardFitSong } from './storyboardFit.js';
+
+const panel = (id, seconds, action = `Panel ${id}`) => ({ id: `panel-${id}`, action, dialogue: '', framing: null, camera: '', seconds, castIds: [], environmentIds: [], objectIds: [] });
+const beat = (id, musicalRole, panels, extra = {}) => ({ id: `beat-${id}`, title: '', what: '', musicalRole, panels, ...extra });
+const song = (extra = {}) => storyboardFitSong({
+ audioAnalysis: {
+ durationSec: 60, beats: [], downbeats: [],
+ sections: [{ label: 'Intro', startSec: 0, endSec: 8 }, { label: 'Verse 1', startSec: 8, endSec: 30 }, { label: 'Chorus 1', startSec: 30, endSec: 60 }],
+ ...extra,
+ },
+ lyricCues: [{ id: 'c1', text: 'First line', startSec: 9 }, { id: 'c2', text: 'Hook line', startSec: 31 }],
+});
+const board = {
+ beats: [
+ beat('a', 'intro', [panel('a1', 3)]),
+ beat('b', 'verse', [panel('b1', 3), panel('b2', 3)]),
+ beat('c', 'chorus', [panel('c1', 6)]),
+ ],
+};
+
+describe('fitStoryboardToSong', () => {
+ it('places beats on the nearest section starts and splits panels by their holds', () => {
+ const fit = fitStoryboardToSong(board, song());
+ expect(fit.beats.map((b) => [b.startSec, b.endSec, b.sectionLabel, b.snap])).toEqual([
+ [0, 8, 'Intro', 'start'], [8, 30, 'Verse 1', 'section'], [30, 60, 'Chorus 1', 'section'],
+ ]);
+ expect(fit.beats[2]).toMatchObject({ boardSec: 6, songSec: 30 });
+ expect(fit.shots.map((s) => [s.panelId, s.startSec, s.endSec, s.label, s.sectionIndex])).toEqual([
+ ['panel-a1', 0, 8, 'Intro', 0], ['panel-b1', 8, 19, 'Verse · 1/2', 1], ['panel-b2', 19, 30, 'Verse · 2/2', 1], ['panel-c1', 30, 60, 'Chorus', 2],
+ ]);
+ // The chorus can start on the hook line instead; a pick that runs time backwards is ignored.
+ expect(fit.beats[2].options).toContainEqual({ at: 31, kind: 'lyric', label: 'Hook line' });
+ const picked = fitStoryboardToSong(board, song(), { starts: { 'beat-c': 31 } });
+ expect(picked.beats.map((b) => [b.startSec, b.snap])).toEqual([[0, 'start'], [8, 'section'], [31, 'picked']]);
+ const backwards = fitStoryboardToSong(board, song(), { starts: { 'beat-c': 5 } });
+ expect(backwards.beats[2]).toMatchObject({ startSec: 30, snap: 'section' });
+ });
+
+ it('cuts panels on the beat grid and prefers a section matching the beat role', () => {
+ const fit = fitStoryboardToSong(board, song({ beats: [18.8, 19.4], sections: [
+ { label: 'Intro', startSec: 0, endSec: 10 }, { label: 'Section 2', startSec: 10, endSec: 32 }, { label: 'Chorus', startSec: 32, endSec: 60 },
+ ] }));
+ // The chorus beat lands on the section named Chorus.
+ expect(fit.beats.map((b) => b.startSec)).toEqual([0, 10, 32]);
+ expect(fit.shots[1]).toMatchObject({ startSec: 10, endSec: 21 });
+ const gridded = fitStoryboardToSong({ beats: [beat('x', null, [panel('x1', 1), panel('x2', 1)])] }, song({ beats: [29.8] }));
+ expect(gridded.shots.map((s) => [s.startSec, s.endSec])).toEqual([[0, 29.8], [29.8, 60]]);
+ });
+
+ it('keeps a beat with no panels as one shot from its words, leaves every shot its minimum, and names short ones', () => {
+ const fit = fitStoryboardToSong({
+ beats: [beat('a', null, [panel('a1', 50)]), beat('b', null, [], { title: 'Gap', what: 'The street empties' }), beat('c', null, Array.from({ length: 6 }, (_, i) => panel(`c${i}`, 1)))],
+ }, storyboardFitSong({ audioAnalysis: { durationSec: 40, sections: [], beats: [], downbeats: [] } }));
+ expect(fit.beats[1]).toMatchObject({ title: 'Gap', noPanels: true, shots: 1 });
+ expect(fit.shots.find((s) => s.beatId === 'beat-b')).toMatchObject({ panelId: null, panelRev: null, visualIntent: 'The street empties' });
+ // The long first beat would leave the last six panels under a second each; they keep two.
+ expect(fit.shots.slice(-6).every((s) => s.endSec - s.startSec >= 2)).toBe(true);
+ expect(fit.beats.map((b) => b.short)).toEqual([0, 0, 0]);
+ expect(fit.shots.at(-1).endSec).toBe(40);
+ const cramped = fitStoryboardToSong({ beats: [beat('c', null, Array.from({ length: 6 }, (_, i) => panel(`c${i}`, 1)))] },
+ storyboardFitSong({ audioAnalysis: { durationSec: 9, sections: [], beats: [], downbeats: [] } }));
+ expect(cramped.beats[0].short).toBe(6);
+ });
+
+ it('fingerprints what a shot drew from its panel, not its length', () => {
+ const rev = (p) => fitStoryboardToSong({ beats: [beat('a', null, [p])] }, song()).shots[0].panelRev;
+ expect(rev(panel('a1', 3))).toBe(rev(panel('a1', 9)));
+ expect(rev(panel('a1', 3))).not.toBe(rev(panel('a1', 3, 'Something else')));
+ });
+
+ it('needs an analyzed song', () => {
+ expect(storyboardFitSong({})).toBeNull();
+ expect(fitStoryboardToSong(board, null)).toEqual({ beats: [], shots: [] });
+ });
+});
diff --git a/server/lib/storyboardValidation.js b/server/lib/storyboardValidation.js
index c09fc5d706..27f654ef5f 100644
--- a/server/lib/storyboardValidation.js
+++ b/server/lib/storyboardValidation.js
@@ -139,3 +139,10 @@ const addedPanel = z.object({
export const storyboardBeatsAddSchema = z.object({
beats: z.array(z.object({ ...beatFields, panels: z.array(addedPanel).max(L.panelsPerBeat) }).strict()).min(1).max(L.beats),
}).strict();
+
+// Fit to song: the board revision the preview was built from, and the beat starts the user picked.
+export const storyboardFitSchema = z.object({
+ boardUpdatedAt: z.string().min(1).max(64),
+ starts: z.record(entityId, z.number().min(0).max(36000))
+ .refine((starts) => Object.keys(starts).length <= L.beats, { message: `At most ${L.beats} picked starts` }).optional(),
+}).strict();
diff --git a/server/routes/storyboards.js b/server/routes/storyboards.js
index 86656a9e83..5e0d016474 100644
--- a/server/routes/storyboards.js
+++ b/server/routes/storyboards.js
@@ -10,7 +10,7 @@ import { validateRequest } from '../lib/validation.js';
import {
storyboardBeatCreateSchema, storyboardBeatUpdateSchema, storyboardCreateSchema, storyboardMoveSchema,
storyboardPanelCreateSchema, storyboardPanelMoveSchema, storyboardPanelNodeUpdateSchema, storyboardPanelUpdateSchema,
- storyboardBeatsAddSchema, storyboardPlotExtractSchema, storyboardRecipeSchema, storyboardSavedRecipeSchema, storyboardTakeCreateSchema, storyboardUpdateSchema,
+ storyboardBeatsAddSchema, storyboardFitSchema, storyboardPlotExtractSchema, storyboardRecipeSchema, storyboardSavedRecipeSchema, storyboardTakeCreateSchema, storyboardUpdateSchema,
} from '../lib/storyboardValidation.js';
import {
addBeat, addPanel, addPanelTake, createStoryboard, deleteBeat, deletePanel, deleteStoryboard, getStoryboard,
@@ -20,6 +20,7 @@ import {
import {
getMakeEmptyPlan, getPanelMake, runPanelNode, startMakeEmpty, updatePanelNode, updateRecipe,
} from '../services/storyboards/make.js';
+import { fitMusicVideoToStoryboard } from '../services/storyboards/fitMusicVideo.js';
import { makeStoryboardFromMusicVideo } from '../services/storyboards/fromMusicVideo.js';
import { addBeats, extractBeats } from '../services/storyboards/plot.js';
import { deleteSavedRecipe, listSavedRecipes, saveRecipe } from '../services/storyboards/recipeLibrary.js';
@@ -39,6 +40,11 @@ router.post('/from-music-video/:projectId', asyncHandler(async (req, res) => {
res.status(201).json(await makeStoryboardFromMusicVideo(req.params.projectId));
}));
+// Fit to song: write the video's shots from its linked board, as previewed.
+router.post('/fit-music-video/:projectId', asyncHandler(async (req, res) => {
+ res.json(await fitMusicVideoToStoryboard(req.params.projectId, validateRequest(storyboardFitSchema, req.body || {})));
+}));
+
// The saved recipe library sits before /:id; a board id always starts "sb-".
router.get('/recipes', asyncHandler(async (_req, res) => {
res.json(await listSavedRecipes());
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index 722bd03c29..d0b9e0de54 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -549,6 +549,7 @@ function buildScene(input, { order }) {
shotMode: input.shotMode ?? 'cutaway',
...(input.performanceSpeaker ? { performanceSpeaker: input.performanceSpeaker } : {}),
...(input.performanceStyle === 'music' ? { performanceStyle: 'music' } : {}),
+ ...(input.panelId ? { panelId: input.panelId, panelRev: input.panelRev ?? null } : {}),
referenceImageId: null,
videoHistoryId: null,
// #8965 — immutable candidate takes; the two slot fields above are the
@@ -649,7 +650,7 @@ export function replaceScenes(project, sceneInputs) {
// adding a shot or lyric the storyboard approval did not cover.
const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',
'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'speedFps', 'offsetFrames', 'reverse', 'holdLastFrame', 'filmLook', 'transitionIn', 'transitionOut', 'glitches', 'cutEffect',
- 'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec', 'frameReferenceIds', 'characterVariant'];
+ 'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec', 'frameReferenceIds', 'characterVariant', 'panelId', 'panelRev'];
/**
* Apply a patch to a single scene. Returns `{ project, updated }`. Throws if the
@@ -837,6 +838,7 @@ export function splitScene(project, sceneId, { backend = null, atSec = null } =
performanceSpeaker: scene.performanceSpeaker ?? null,
performanceStyle: scene.performanceStyle ?? null,
...(scene.camera ? { camera: scene.camera } : {}),
+ ...(scene.panelId ? { panelId: scene.panelId, panelRev: scene.panelRev ?? null } : {}),
...timing,
}),
}, { order: 0 });
diff --git a/server/services/musicVideo/projectsLogic.test.js b/server/services/musicVideo/projectsLogic.test.js
index d46279bcd1..948d02d8eb 100644
--- a/server/services/musicVideo/projectsLogic.test.js
+++ b/server/services/musicVideo/projectsLogic.test.js
@@ -597,11 +597,11 @@ describe('scene board operations', () => {
expect(applySceneUpdate(still, scene.sceneId, { visualLayer: 'still' }).updated.stillMove).toBe('push');
});
- // Uniquely catches a split still's later pieces losing the Move the shot set.
- it('keeps a still\'s Move on every piece of a split', () => {
- const { project, scene } = addScene(baseProject(), { prompt: 'a', visualLayer: 'still', stillMove: 'push', camera: { move: 'handheld' }, startSec: 0, endSec: 8 });
+ // Uniquely catches a split still's later pieces losing the Move, or the storyboard panel, the shot set.
+ it('keeps a still\'s Move and its storyboard panel on every piece of a split', () => {
+ const { project, scene } = addScene(baseProject(), { prompt: 'a', visualLayer: 'still', stillMove: 'push', camera: { move: 'handheld' }, startSec: 0, endSec: 8, panelId: 'panel-example-1', panelRev: 'abc' });
const split = splitScene(project, scene.sceneId, { atSec: 4 });
- expect(split.project.scenes.map((s) => s.stillMove)).toEqual(['push', 'push']);
+ expect(split.project.scenes.map((s) => [s.stillMove, s.panelId, s.panelRev])).toEqual([['push', 'panel-example-1', 'abc'], ['push', 'panel-example-1', 'abc']]);
});
// Catches the split text disagreeing with its storyboard anchors during an instrumental lead-in.
@@ -1106,6 +1106,8 @@ describe('create-schema parity', () => {
shotMode: 'performance',
performanceSpeaker: 'Singer',
performanceStyle: 'music',
+ panelId: 'panel-example-1',
+ panelRev: 'abc123',
};
it('stores every accepted video setting key on the project record', () => {
diff --git a/server/services/storyboards/fitMusicVideo.js b/server/services/storyboards/fitMusicVideo.js
new file mode 100644
index 0000000000..b55496d171
--- /dev/null
+++ b/server/services/storyboards/fitMusicVideo.js
@@ -0,0 +1,51 @@
+/**
+ * Fit to song: writes a music video's shots from its linked storyboard, each
+ * beat placed on the analyzed song and each panel a shot that keeps the
+ * panel's id (lib/storyboardFit.js, the same fit the Plan step previews).
+ * Replaces the shots, so it refuses while any shot is locked; a shot whose
+ * span is unchanged keeps its frames and clips (replaceScenes). Runs only on
+ * the user's tap; no model call.
+ */
+
+import { ServerError } from '../../lib/errorHandler.js';
+import { sungLyricTextWithin } from '../../lib/musicVideoLyricAnchors.js';
+import { fitStoryboardToSong, storyboardFitSong } from '../../lib/storyboardFit.js';
+import { getStoryboard } from './index.js';
+
+export async function fitMusicVideoToStoryboard(projectId, { boardUpdatedAt, starts = {} } = {}) {
+ const [{ getProject, mutateProjectRecord }, { replaceScenes }, { shotLocks }] = await Promise.all([
+ import('../musicVideo/projects.js'), import('../musicVideo/projectsLogic.js'), import('../musicVideo/productionReview.js'),
+ ]);
+ const linked = await getProject(projectId);
+ if (!linked) throw new ServerError('Music video not found', { status: 404, code: 'NOT_FOUND' });
+ if (!linked.storyboardId) throw new ServerError('Link a storyboard first.', { status: 409, code: 'STORYBOARD_NOT_LINKED' });
+ const board = await getStoryboard(linked.storyboardId);
+ if (board.updatedAt !== boardUpdatedAt) {
+ throw new ServerError('The storyboard changed since this fit was shown. Check the fit again.', { status: 409, code: 'STORYBOARD_CHANGED' });
+ }
+ if (!board.beats.length) throw new ServerError('The storyboard has no beats yet.', { status: 400, code: 'STORYBOARD_EMPTY' });
+
+ const outcome = await mutateProjectRecord(projectId, (current) => {
+ if (current.storyboardId !== board.id) {
+ throw new ServerError('This video was linked to another storyboard meanwhile. Check the fit again.', { status: 409, code: 'STORYBOARD_CHANGED' });
+ }
+ const song = storyboardFitSong(current);
+ if (!song) throw new ServerError('Analyze the song first; the fit places beats on its sections.', { status: 409, code: 'SONG_NOT_ANALYZED' });
+ const locked = Object.values(shotLocks(current) || {}).filter((lock) => lock !== 'open').length;
+ if (locked) {
+ // A lock marked changed (edited after locking) still holds until unlocked.
+ throw new ServerError(`${locked} shot${locked === 1 ? ' has a lock' : 's have locks'} on Shots. Fit to song replaces every shot and never moves a locked one; unlock them first.`,
+ { status: 409, code: 'SHOTS_LOCKED' });
+ }
+ const fit = fitStoryboardToSong(board, song, { starts });
+ const timedLyrics = song.lyricCues.length > 0;
+ const inputs = fit.shots.map(({ beatId: _beatId, ...shot }) => ({
+ ...shot,
+ loop: false,
+ lyricText: timedLyrics ? sungLyricTextWithin(current.lyricCues, shot.startSec, shot.endSec) : null,
+ }));
+ return { ...replaceScenes(current, inputs), fit };
+ });
+ console.log(`🎬 Fit storyboard ${board.id} to music video ${projectId}: ${outcome.scenes.length} shot(s), replaced ${outcome.replaced}`);
+ return { project: outcome.project, shots: outcome.scenes.length, replaced: outcome.replaced };
+}
diff --git a/server/services/storyboards/fitMusicVideo.test.js b/server/services/storyboards/fitMusicVideo.test.js
new file mode 100644
index 0000000000..81a39e2d0c
--- /dev/null
+++ b/server/services/storyboards/fitMusicVideo.test.js
@@ -0,0 +1,79 @@
+/**
+ * Fit to song through the real routers and file-backed stores: the linked
+ * board's panels become shots on the analyzed song, each keeping its panel.
+ */
+import { afterAll, describe, expect, it, vi } from 'vitest';
+import express from 'express';
+import { request } from '../../lib/testHelper.js';
+import { errorMiddleware } from '../../lib/errorHandler.js';
+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
+
+const ROOT = () => lazyTempDataRoot('storyboard-fit-mv-');
+vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await importOriginal(), { dataRoot: ROOT }));
+vi.mock('../../lib/fileUtils.js', async (importOriginal) => makePathsProxy(await importOriginal(), { dataRoot: ROOT }));
+vi.mock('../settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));
+
+const { default: storyboardRoutes } = await import('../../routes/storyboards.js');
+const { default: musicVideoRoutes } = await import('../../routes/musicVideo.js');
+const projects = await import('../musicVideo/projects.js');
+
+const app = express();
+app.use(express.json());
+app.use('/api/storyboards', storyboardRoutes);
+app.use('/api/music-video', musicVideoRoutes);
+app.use(errorMiddleware);
+afterAll(cleanupTempDataRoots);
+
+const analysis = {
+ bpm: 120, beats: [], downbeats: [], durationSec: 40,
+ sections: [{ label: 'Verse', startSec: 0, endSec: 20 }, { label: 'Chorus', startSec: 20, endSec: 40 }],
+};
+
+async function linkedProject(extra = {}) {
+ const created = await request(app).post('/api/storyboards').send({ name: 'Example board' });
+ const boardId = created.body.id;
+ await request(app).post(`/api/storyboards/${boardId}/beats/add`).send({ beats: [
+ { title: 'Arrival', musicalRole: 'verse', panels: [{ action: 'She steps off the bus', seconds: 4 }, { action: 'The town square', seconds: 4 }] },
+ { title: 'Lift off', musicalRole: 'chorus', panels: [{ action: 'Crowd lifts her up', seconds: 8 }] },
+ ] });
+ const project = await projects.createProject({ name: 'Example Song' });
+ await projects.mutateProjectRecord(project.id, (record) => ({
+ project: { ...record, storyboardId: boardId, audioAnalysis: analysis, lyricCues: [{ id: 'c1', text: 'Here we go', startSec: 21, endSec: 23 }], ...extra },
+ }));
+ const board = (await request(app).get(`/api/storyboards/${boardId}`)).body;
+ return { project, board };
+}
+
+describe('fit a music video to its storyboard', () => {
+ it('writes one shot per panel on the song, tied to its panel', async () => {
+ const { project, board } = await linkedProject();
+ const res = await request(app).post(`/api/storyboards/fit-music-video/${project.id}`).send({ boardUpdatedAt: board.updatedAt });
+ expect(res.status).toBe(200);
+ expect(res.body).toMatchObject({ shots: 3, replaced: 0 });
+ const panelIds = board.beats.flatMap((beat) => beat.panels.map((panel) => panel.id));
+ expect(res.body.project.scenes.map((s) => [s.label, s.startSec, s.endSec, s.sectionLabel, s.visualIntent, s.panelId])).toEqual([
+ ['Arrival · 1/2', 0, 10, 'Verse', 'She steps off the bus', panelIds[0]],
+ ['Arrival · 2/2', 10, 20, 'Verse', 'The town square', panelIds[1]],
+ ['Lift off', 20, 40, 'Chorus', 'Crowd lifts her up', panelIds[2]],
+ ]);
+ expect(res.body.project.scenes[2]).toMatchObject({ lyricText: 'Here we go', loop: false, panelRev: expect.any(String) });
+
+ // A board edited after the preview is refused, so what was shown is what gets written.
+ expect((await request(app).post(`/api/storyboards/fit-music-video/${project.id}`).send({ boardUpdatedAt: '2000-01-01T00:00:00.000Z' })).status).toBe(409);
+ });
+
+ it('never replaces locked shots and needs an analyzed song', async () => {
+ const { project, board } = await linkedProject({
+ scenes: [{ sceneId: 'mvs-kept', order: 0, startSec: 0, endSec: 40, prompt: '', takes: [] }],
+ productionReview: { shotApprovals: { 'mvs-kept': { basis: 'example' } } },
+ });
+ const locked = await request(app).post(`/api/storyboards/fit-music-video/${project.id}`).send({ boardUpdatedAt: board.updatedAt });
+ expect(locked.status).toBe(409);
+ expect(locked.body.code).toBe('SHOTS_LOCKED');
+ expect((await projects.getProject(project.id)).scenes.map((s) => s.sceneId)).toEqual(['mvs-kept']);
+
+ const bare = await linkedProject({ audioAnalysis: null });
+ const unanalyzed = await request(app).post(`/api/storyboards/fit-music-video/${bare.project.id}`).send({ boardUpdatedAt: bare.board.updatedAt });
+ expect(unanalyzed.body.code).toBe('SONG_NOT_ANALYZED');
+ });
+});