Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Render music videos shot by shot, with a cached film look per shot (#11580)

[._.]/23f213f97f6a
diff --git a/client/src/components/musicVideo/ProjectActionGroups.jsx b/client/src/components/musicVideo/ProjectActionGroups.jsx
index 8229f1f650..d2f52cfd0a 100644
--- a/client/src/components/musicVideo/ProjectActionGroups.jsx
+++ b/client/src/components/musicVideo/ProjectActionGroups.jsx
@@ -1,4 +1,6 @@
import { useState } from 'react';
+import useMusicVideoShotRenders from '../../hooks/useMusicVideoShotRenders.js';
+import { formatCount } from '../../utils/formatters.js';
import LlmRouteNote from './LlmRouteNote.jsx';
import CancelRenderButton from './CancelRenderButton.jsx';
import { Film, Music, Activity, Image as ImageIcon, Video, Wand2, X } from 'lucide-react';
@@ -247,6 +249,8 @@ export function RenderStyleSelect({ project, onRenderStyle }) {
/** Render the complete video, or cancel the render in flight for this project. */
export function RenderFinalButton({ project, renderJob, readiness }) {
+ // A composed render reuses every shot already rendered and stitches.
+ const { status: shotRenders } = useMusicVideoShotRenders(project);
const scenes = project.scenes || [];
const sceneCount = scenes.length;
const mode = project.composition?.mode || 'concat';
@@ -311,6 +315,12 @@ export function RenderFinalButton({ project, renderJob, readiness }) {
{isDisabled && blockerReason && (
<p className="text-xs text-port-text-muted">{blockerReason}</p>
)}
+ {shotRenders?.total > 0 && (
+ <p className="text-xs text-port-text-muted" data-testid="shot-renders-ready">
+ {shotRenders.ready === shotRenders.total ? 'Every shot is ready: Render final only stitches them.'
+ : `${formatCount(shotRenders.ready)} of ${formatCount(shotRenders.total)} shots ready. Render final renders the rest, then stitches.`}
+ </p>
+ )}
</div>
);
}
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 6a0523310b..ad2892f61b 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -4,6 +4,7 @@ import SceneCameraControls from './SceneCameraControls.jsx';
import SceneStartFrame from './SceneStartFrame.jsx';
import SceneClipTiming from './SceneClipTiming.jsx';
import SceneRejectUnused from './SceneRejectUnused.jsx';
+import ShotFinalRender from './ShotFinalRender.jsx';
import { MUSIC_VIDEO_MEDIUM_LABELS } from '../../../../server/lib/musicVideoMediumPlan.js';
import { useEffect, useRef, useState } from 'react';
import { Trash2, Activity, ArrowUp, ArrowDown, ChevronLeft, ChevronRight, Video, AlertTriangle, Clapperboard, Shuffle } from 'lucide-react';
@@ -561,6 +562,8 @@ export default function SceneCard({
</>
)}
{codeShot && shotSettings}
+ {/* This shot's piece of the composed final render: cached or not, Render shot, its film look. */}
+ {project?.composition?.mode === 'composed' && <ShotFinalRender project={project} scene={scene} onChange={applyPatch} fieldId={fieldId} />}
{(scene.visualIntent || scene.direction || scene.direction?.actionContract) && (
<details className="group/direction rounded border border-port-border">
<summary className="flex min-h-[44px] cursor-pointer items-center gap-2 px-2 text-xs marker:content-none sm:min-h-[32px] [&::-webkit-details-marker]:hidden">
diff --git a/client/src/components/musicVideo/ShotFinalRender.jsx b/client/src/components/musicVideo/ShotFinalRender.jsx
new file mode 100644
index 0000000000..ba520ece7e
--- /dev/null
+++ b/client/src/components/musicVideo/ShotFinalRender.jsx
@@ -0,0 +1,95 @@
+import { useEffect, useState } from 'react';
+import { Activity, Film, Play, Square, X } from 'lucide-react';
+import { cancelMusicVideoShotRender, musicVideoShotRenderUrl, renderMusicVideoShot } from '../../services/apiMusicVideo.js';
+import useMusicVideoShotRenders from '../../hooks/useMusicVideoShotRenders.js';
+import { FILM_LOOK_PRESETS, filmLookPreset, isFilmLookNeutral, normalizeFilmLook } from '../../lib/filmLook.js';
+
+const BUTTON = 'inline-flex min-h-[44px] items-center gap-1 rounded border border-port-border bg-port-bg px-2 py-1.5 text-xs hover:bg-port-border/40 disabled:opacity-50 sm:min-h-0';
+const presetLabel = (look) => (isFilmLookNeutral(look) ? 'Clean' : FILM_LOOK_PRESETS.find((p) => p.id === look?.preset)?.label || 'Custom');
+
+/**
+ * The shot's piece of the composed final render: whether its segment is
+ * cached, Render shot to make it now (only this shot), a play of the rendered
+ * shot with its film look, and the shot's own look (kept when the video's
+ * look changes). The final render reuses every cached shot and stitches.
+ */
+export default function ShotFinalRender({ project, scene, onChange, fieldId }) {
+ const projectId = project?.id;
+ const sceneId = scene.sceneId;
+ const { status, live: liveByShot, markLive } = useMusicVideoShotRenders(project);
+ const live = liveByShot[sceneId] || null;
+ const [playing, setPlaying] = useState(false);
+ const [starting, setStarting] = useState(false);
+
+ useEffect(() => { setPlaying(false); }, [sceneId]);
+
+ if (!status?.available) return null;
+ const shot = status.shots.find((entry) => entry.sceneId === sceneId);
+ const rendering = live?.state === 'rendering' || (!live && shot?.rendering);
+ const rendered = shot?.state === 'rendered' && !rendering;
+ const file = rendered ? shot.files[shot.files.length - 1] : null;
+ const ownLook = scene.filmLook ? normalizeFilmLook(scene.filmLook) : null;
+ const videoLook = normalizeFilmLook(project.filmLook);
+ const lookValue = ownLook ? (FILM_LOOK_PRESETS.some((p) => p.id === ownLook.preset) ? ownLook.preset : 'own') : 'video';
+
+ const start = () => {
+ setStarting(true);
+ markLive(sceneId, null);
+ renderMusicVideoShot(projectId, sceneId, { silent: true })
+ .then((result) => { if (result?.state === 'rendering') markLive(sceneId, { state: 'rendering', progress: 0 }); })
+ .catch((err) => markLive(sceneId, { state: 'failed', error: err.message }))
+ .finally(() => setStarting(false));
+ };
+ const pickLook = (value) => {
+ setPlaying(false);
+ if (value === 'video') onChange({ filmLook: null });
+ else if (value === 'keep') onChange({ filmLook: videoLook || filmLookPreset('none') });
+ else if (value !== 'own') onChange({ filmLook: filmLookPreset(value) });
+ };
+
+ const state = status.error ? null : rendering ? `Rendering ${Math.round((live?.progress || 0) * 100)}%` : rendered ? 'Rendered' : shot ? 'Not rendered' : 'Not in the cut';
+ return (
+ <section aria-label="Final render" className="min-w-0 space-y-2 rounded border border-port-border p-2">
+ <div className="flex flex-wrap items-center gap-2">
+ <h4 className="inline-flex items-center gap-1 text-xs font-medium"><Film size={14} aria-hidden="true" /> Final render</h4>
+ {state && (
+ <span data-testid="shot-render-state" className={`rounded px-1.5 text-[11px] ${rendered ? 'bg-port-success/10 text-port-success' : 'bg-port-border/60 text-port-text-muted'}`}>
+ {rendering && <Activity size={11} className="mr-1 inline animate-spin" aria-hidden="true" />}{state}
+ </span>
+ )}
+ <span className="ml-auto flex flex-wrap items-center gap-2">
+ {rendering ? (
+ <button type="button" className={BUTTON} onClick={() => cancelMusicVideoShotRender(projectId, sceneId, { silent: true }).catch(() => {})}>
+ <Square size={12} aria-hidden="true" /> Stop
+ </button>
+ ) : shot && !rendered ? (
+ <button type="button" className={BUTTON} onClick={start} disabled={starting || !!status.error}>
+ <Film size={12} aria-hidden="true" /> Render shot
+ </button>
+ ) : null}
+ {file && (
+ <button type="button" className={BUTTON} onClick={() => setPlaying((on) => !on)} aria-pressed={playing}>
+ {playing ? <X size={12} aria-hidden="true" /> : <Play size={12} aria-hidden="true" />} {playing ? 'Close' : 'Play'}
+ </button>
+ )}
+ </span>
+ </div>
+ {status.error && <p role="status" className="break-words text-[11px] text-port-warning">{status.error.message}</p>}
+ {live?.state === 'failed' && live.error && <p role="alert" className="break-words text-[11px] text-port-error">{live.error}</p>}
+ {playing && file && (
+ // The rendered shot as the final render cuts it: film look and text, without the song.
+ <video key={file} src={musicVideoShotRenderUrl(projectId, file)} controls autoPlay loop muted playsInline className="w-full rounded bg-black" />
+ )}
+ <div className="flex flex-wrap items-center gap-2">
+ <label htmlFor={fieldId('film-look')} className="text-[11px] text-port-text-muted">Film look</label>
+ <select id={fieldId('film-look')} value={lookValue} onChange={(e) => pickLook(e.target.value)}
+ className="min-h-[44px] rounded border border-port-border bg-port-bg px-2 py-1 text-xs sm:min-h-0">
+ <option value="video">Video&apos;s look ({presetLabel(videoLook)})</option>
+ {!ownLook && <option value="keep">Keep the video&apos;s look on this shot</option>}
+ {lookValue === 'own' && <option value="own">This shot&apos;s look ({presetLabel(ownLook)})</option>}
+ {FILM_LOOK_PRESETS.map((preset) => <option key={preset.id} value={preset.id}>{preset.label}</option>)}
+ </select>
+ </div>
+ </section>
+ );
+}
diff --git a/client/src/components/musicVideo/ShotFinalRender.test.jsx b/client/src/components/musicVideo/ShotFinalRender.test.jsx
new file mode 100644
index 0000000000..314e0d3f51
--- /dev/null
+++ b/client/src/components/musicVideo/ShotFinalRender.test.jsx
@@ -0,0 +1,69 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
+
+const handlers = new Map();
+vi.mock('../../services/socket', () => ({
+ default: { on: vi.fn((event, fn) => handlers.set(event, fn)), off: vi.fn((event) => handlers.delete(event)) },
+}));
+vi.mock('../../services/apiMusicVideo.js', () => ({
+ getMusicVideoShotRenders: vi.fn(),
+ renderMusicVideoShot: vi.fn(async () => ({ state: 'rendering' })),
+ cancelMusicVideoShotRender: vi.fn(async () => ({ ok: true })),
+ musicVideoShotRenderUrl: (id, name) => `/api/music-video/${id}/shot-renders/${name}`,
+}));
+
+import ShotFinalRender from './ShotFinalRender.jsx';
+import { getMusicVideoShotRenders, renderMusicVideoShot } from '../../services/apiMusicVideo.js';
+
+const project = { id: 'mv-1', composition: { mode: 'composed' }, filmLook: { preset: 'vhs' }, scenes: [{ sceneId: 'a' }, { sceneId: 'b' }] };
+const fieldId = (name) => `f-${name}`;
+const shot = (sceneId, state, files = []) => ({ sceneId, state, rendering: false, files });
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ handlers.clear();
+});
+
+describe('ShotFinalRender', () => {
+ it('renders only this shot, shows its progress pushed from the server, then offers its segment to play', async () => {
+ getMusicVideoShotRenders.mockResolvedValue({ available: true, shots: [shot('a', 'stale'), shot('b', 'rendered', ['b.mp4'])], ready: 1, total: 2 });
+ render(<ShotFinalRender project={project} scene={project.scenes[0]} onChange={() => {}} fieldId={fieldId} />);
+ const button = await screen.findByRole('button', { name: /Render shot/ });
+ await act(async () => fireEvent.click(button));
+ expect(renderMusicVideoShot).toHaveBeenCalledWith('mv-1', 'a', { silent: true });
+ await waitFor(() => expect(screen.getByTestId('shot-render-state').textContent).toBe('Rendering 0%'));
+ act(() => handlers.get('music-video:shot-render')({ projectId: 'mv-1', sceneId: 'a', state: 'rendering', progress: 0.45 }));
+ await waitFor(() => expect(screen.getByTestId('shot-render-state').textContent).toBe('Rendering 45%'));
+
+ getMusicVideoShotRenders.mockResolvedValue({ available: true, shots: [shot('a', 'rendered', ['a.mp4']), shot('b', 'rendered', ['b.mp4'])], ready: 2, total: 2 });
+ await act(async () => handlers.get('music-video:shot-render')({ projectId: 'mv-1', sceneId: 'a', state: 'rendered', progress: 1 }));
+ await waitFor(() => expect(screen.getByTestId('shot-render-state').textContent).toBe('Rendered'));
+ fireEvent.click(screen.getByRole('button', { name: /Play/ }));
+ expect(document.querySelector('video').getAttribute('src')).toBe('/api/music-video/mv-1/shot-renders/a.mp4');
+ });
+
+ it("keeps the video's look on the shot, or returns it to the video's look", async () => {
+ getMusicVideoShotRenders.mockResolvedValue({ available: true, shots: [shot('a', 'rendered', ['a.mp4'])], ready: 1, total: 1 });
+ const onChange = vi.fn();
+ const { rerender } = render(<ShotFinalRender project={project} scene={project.scenes[0]} onChange={onChange} fieldId={fieldId} />);
+ const picker = await screen.findByLabelText('Film look');
+ fireEvent.change(picker, { target: { value: 'keep' } });
+ expect(onChange.mock.calls[0][0].filmLook.preset).toBe('vhs');
+ rerender(<ShotFinalRender project={project} scene={{ sceneId: 'a', filmLook: { preset: 'polaroid' } }} onChange={onChange} fieldId={fieldId} />);
+ expect(screen.getByLabelText('Film look').value).toBe('polaroid');
+ fireEvent.change(screen.getByLabelText('Film look'), { target: { value: 'video' } });
+ expect(onChange).toHaveBeenLastCalledWith({ filmLook: null });
+ });
+
+ it('names the refusal when the cut cannot be planned, and stays away from other render styles', async () => {
+ getMusicVideoShotRenders.mockResolvedValue({ available: true, error: { message: 'Shot 3: the selected clip file is missing' }, shots: [] });
+ const { unmount } = render(<ShotFinalRender project={project} scene={project.scenes[0]} onChange={() => {}} fieldId={fieldId} />);
+ expect(await screen.findByText('Shot 3: the selected clip file is missing')).toBeTruthy();
+ unmount();
+ getMusicVideoShotRenders.mockClear();
+ const { container } = render(<ShotFinalRender project={{ ...project, composition: { mode: 'concat' } }} scene={project.scenes[0]} onChange={() => {}} fieldId={fieldId} />);
+ await new Promise((r) => setTimeout(r, 700));
+ expect(getMusicVideoShotRenders).not.toHaveBeenCalled();
+ expect(container.innerHTML).toBe('');
+ });
+});
diff --git a/client/src/hooks/README.md b/client/src/hooks/README.md
index b600637ea3..19e05c56ab 100644
--- a/client/src/hooks/README.md
+++ b/client/src/hooks/README.md
@@ -264,6 +264,7 @@ grep -i "what you want to do" client/src/hooks/README.md
| `useAutonomousMusicVideo` | Fully-autonomous run (prompt → lyrics → Suno song → video): applies the project pushed on `music-video:autonomous` and toasts the outcomes that need the director (checkpoint waiting, needs you, finished, failed). Returns `{ busy, resume(edits?, { inline }?), stop(), cancel() }` (an `inline` resume rejects instead of toasting, for errors shown next to the control). | The Music Video autonomous run panel. |
| `useMusicVideoProduction` | Server-owned production run (#9066): start a browserless plan → frames → clips → reviewed-draft run with an allowed provider/model pool and explicit limits, and Stop / Resume / Cancel it; applies the project pushed on `music-video:production` and toasts terminal outcomes. Returns `{ busy, start(body), resume(runId, body?), stop(runId), cancel(runId) }`. | The Music Video production panel. |
| `useMusicVideoRenderJob` | The final-render job (#1760 Phase 2): kickoff (409 → attach to the existing job), SSE progress, terminal-frame handling and non-terminal stream-close recovery. Thin `useSseJobSlot` adapter with 0–1 progress conversion; returns `{ active, pending, context, job, progress, start(projectId), cancel, reattach(projectId), reattaching }`; a project the server marks `rendering` re-attaches on load through a READ of the live job (`GET /:id/render`), never a POST (#9940); `onSettled(reason, projectId)` fires on every terminal frame; `onRendered(projectId, result)` / `onFailed(projectId)` key on the projectId captured at kickoff. | The Music Video "Render final" control. |
+| `useMusicVideoShotRenders` | The composed final render's per-shot segment cache: `GET /:id/shot-renders` re-read as render inputs change (debounced) and when a shot render settles, plus live `music-video:shot-render` progress per shot. Returns `{ status, live, markLive(sceneId, entry), refresh }`; `status` is null outside the composed render style. | The shot card's Final render section and the Render final control. |
| `useMusicVideoStatsOutro` | The cutting room floor outro's clock: `start()` plays it for `seconds` on its own rAF clock and fetches the tally afresh; with `audioRef` it also starts when the song ends; a new project or length stops it. Returns `{ seconds, t, stats, active, start, stop }`. | The storyboard animatic's ending and the board's Ending card. |
| `useMusicVideoRevisions` | Selective section revision (#8987): open a revision from a reviewed draft excerpt (its flagged sections' takes are rejected, the rest kept), resume it from the server checkpoint — generating ONLY the sections the server reports as `needsGeneration` through the board's scene lanes, or adopting the draft re-render via `attachRender` once every revised section has a take — and cancel it. A server refusal toasts as a failure; a throw in the hook's own follow-up AFTER the server succeeded reloads the saved record instead (#9940), so an opened repair whose resume failed stays recoverable from the "Needs attention" banner. Returns `{ busy, revise(excerptId, sceneIds?), repair(basis), repairPerformance(sceneId, input), resume(revisionId), cancel(revisionId) }`. | The Music Video excerpt panel. |
| `useMusicVideoAutoReview` | Opt-in automatic review/retries (#8988): start a run over a draft window with explicit review (`maxAttempts`) and paid-generation (`maxGenerations`) limits plus a vision reviewer; applies the project pushed on `music-video:auto-review` and reports outcomes; the server generates a run's revised sections itself (#10014), so the board never submits them and a run completes with no tab open. Returns `{ busy, action, start(startSec, endSec, limits, reviewer), resume(runId, limits?), stop(runId), cancel(runId) }`. | The Music Video excerpt panel. |
diff --git a/client/src/hooks/index.js b/client/src/hooks/index.js
index 95eaede400..38990f93d2 100644
--- a/client/src/hooks/index.js
+++ b/client/src/hooks/index.js
@@ -78,6 +78,7 @@ export { default as useMusicVideoModelSettings } from './useMusicVideoModelSetti
export { default as useMusicVideoProduction } from './useMusicVideoProduction.js';
export { default as useAutonomousMusicVideo } from './useAutonomousMusicVideo.js';
export { default as useMusicVideoRenderJob } from './useMusicVideoRenderJob.js';
+export { default as useMusicVideoShotRenders } from './useMusicVideoShotRenders.js';
export { default as useMusicVideoStatsOutro } from './useMusicVideoStatsOutro.js';
export { default as useMusicVideoRevisions } from './useMusicVideoRevisions.js';
export { default as useMusicVideoSceneMedia } from './useMusicVideoSceneMedia.js';
diff --git a/client/src/hooks/useMusicVideoShotRenders.js b/client/src/hooks/useMusicVideoShotRenders.js
new file mode 100644
index 0000000000..5fdfd33b5c
--- /dev/null
+++ b/client/src/hooks/useMusicVideoShotRenders.js
@@ -0,0 +1,61 @@
+import { useEffect, useMemo, useRef, useState } from 'react';
+import socket from '../services/socket';
+import { getMusicVideoShotRenders } from '../services/apiMusicVideo.js';
+
+// The project fields that can change which cached shot segments are current.
+const renderInputs = (project) => JSON.stringify([project?.scenes, project?.filmLook, project?.composition, project?.statsOutro, project?.trackId, project?.lyricCues, project?.renderHistoryId]);
+
+/**
+ * The composed final render's per-shot cache state (`GET /:id/shot-renders`):
+ * read when the project's render inputs change (debounced, as edits land) and
+ * when a shot render settles, plus each shot's live render from the
+ * `music-video:shot-render` events. Returns `{ status, live }`, `live` keyed
+ * by sceneId (`{ state, progress, error }`). Off (null) for other render styles.
+ */
+export default function useMusicVideoShotRenders(project) {
+ const projectId = project?.id;
+ const composed = project?.composition?.mode === 'composed';
+ const inputs = useMemo(() => renderInputs(project), [project]);
+ const [status, setStatus] = useState(null);
+ const [live, setLive] = useState({});
+ const generation = useRef(0);
+
+ const refresh = useRef(null);
+ refresh.current = () => {
+ const ticket = ++generation.current;
+ getMusicVideoShotRenders(projectId, { silent: true })
+ .then((next) => { if (ticket === generation.current) setStatus(next); })
+ .catch(() => {});
+ };
+
+ useEffect(() => {
+ if (!projectId || !composed) { generation.current += 1; setStatus(null); return undefined; }
+ const timer = setTimeout(() => refresh.current(), 600);
+ return () => clearTimeout(timer);
+ }, [projectId, composed, inputs]);
+
+ useEffect(() => {
+ setLive({});
+ if (!projectId || !composed) return undefined;
+ const onEvent = (event) => {
+ if (event?.projectId !== projectId || !event.sceneId) return;
+ setLive((prev) => {
+ const next = { ...prev };
+ if (event.state === 'rendering' || event.state === 'failed') next[event.sceneId] = event;
+ else delete next[event.sceneId];
+ return next;
+ });
+ if (event.state !== 'rendering') refresh.current();
+ };
+ socket.on('music-video:shot-render', onEvent);
+ return () => socket.off('music-video:shot-render', onEvent);
+ }, [projectId, composed]);
+
+ const markLive = (sceneId, entry) => setLive((prev) => {
+ const next = { ...prev };
+ if (entry) next[sceneId] = entry;
+ else delete next[sceneId];
+ return next;
+ });
+ return { status, live, markLive, refresh: () => refresh.current() };
+}
diff --git a/client/src/services/apiMusicVideo.js b/client/src/services/apiMusicVideo.js
index eaeb18fe14..7d54639cce 100644
--- a/client/src/services/apiMusicVideo.js
+++ b/client/src/services/apiMusicVideo.js
@@ -278,6 +278,20 @@ export const regenerateMusicVideoCodeSection = (id, sectionId, body = {}, option
method: 'POST', body: JSON.stringify(body), ...options,
});
+// ---- Per-shot render segments ----
+// Every shot's cache state for the composed final render:
+// { available, error?, shots: [{ sceneId, state: 'rendered'|'stale', rendering, look, files }], outro, ready, total }.
+export const getMusicVideoShotRenders = (id, options = {}) =>
+ request(`/music-video/${encodeURIComponent(id)}/shot-renders`, options);
+// Render one shot's segments into the cache; progress arrives on `music-video:shot-render`.
+export const renderMusicVideoShot = (id, sceneId, options = {}) =>
+ request(`/music-video/${encodeURIComponent(id)}/shots/${encodeURIComponent(sceneId)}/render`, { method: 'POST', ...options });
+export const cancelMusicVideoShotRender = (id, sceneId, options = {}) =>
+ request(`/music-video/${encodeURIComponent(id)}/shots/${encodeURIComponent(sceneId)}/render/cancel`, { method: 'POST', ...options });
+// A rendered shot's silent segment, to play.
+export const musicVideoShotRenderUrl = (id, name) =>
+ `/api/music-video/${encodeURIComponent(id)}/shot-renders/${encodeURIComponent(name)}`;
+
// EventSource URL for a render job's progress stream (consumed by useSseProgress).
export const musicVideoRenderEventsUrl = (jobId) =>
`/api/music-video/render/${encodeURIComponent(jobId)}/events`;
diff --git a/docs/HTML_COMPOSITIONS.md b/docs/HTML_COMPOSITIONS.md
index ce31cd025a..d721bd4012 100644
--- a/docs/HTML_COMPOSITIONS.md
+++ b/docs/HTML_COMPOSITIONS.md
@@ -322,6 +322,19 @@ what the preview shows and what the render bakes:
audio is copied back untouched. Paid clips are never regenerated to change
the feel. The pass takes the second half of the render's progress bar and a
cancel aborts it.
+- **Composed final renders go shot by shot** (`services/musicVideo/shotSegments.js`).
+ Each section of the cut is a cached segment in two stages: the base (its
+ frames from `buildMusicVideoFfmpegArgs`' `segment` mode, which draws the same
+ chain the whole-film graph draws there, cut effects borrowing the next shot,
+ fades, grade, glitches and the typography over it) and the look (the base
+ through the shot's film look on song time). A shot may keep its own look
+ (`scene.filmLook`; `null` follows the video's). A look change redoes only the
+ look stage of the shots it reaches, a shot change only that shot (and a
+ neighbor whose cut effect reads it). The shot card's **Final render** row
+ renders one shot now and plays it silent; **Render final** renders whatever
+ is missing, joins the segments by stream copy and lays the song under them.
+ Segments live in `data/music-video-segments/<projectId>/`, machine-local and
+ excluded from backups; publishing a final render drops ones it no longer uses.
- **The composition document preview** carries playback mode with each seek
into the OS-isolated preview worker. Playing frames use the same lightweight
motion layers as the animatic: CSS focus/tone/saturation and weave on the
diff --git a/docs/STORAGE.md b/docs/STORAGE.md
index 1d7c94839f..27dbc97dd8 100644
--- a/docs/STORAGE.md
+++ b/docs/STORAGE.md
@@ -291,6 +291,7 @@ goal data together; interrupted intents remain retryable after restore.
- Music Video dependency provenance — versioned plate/clip references, optional crop/mask input revisions, composition-window checksums, and review snapshots live on existing db-primary `musicVideoProject` rows. Repairs keep historical takes and files and reuse serialized revision checkpoints; no new store, seed, or install migration. Legacy evidence without a snapshot remains unverified. The `musicVideoProjects` schema gate prevents older peers from reusing stale evidence; existing asset channels retain byte ownership.
- Music Video development artifacts ("ingredients": a Cast & Sets check-in sheet, animatic, treatment, storyboard) — each version is an immutable file under `data/music-video/{projectId}/dev/{artifactId}/v{N}.{html,md,mp4,png,jpg}`; the `musicVideoProject` record owns the metadata (`devArtifacts[]`: kind, title, review status, notes, version history with each version's data-relative path). Versions are never overwritten and deletes are soft, so a clone that carried the metadata keeps valid pointers. The metadata is **wire-local** (stripped by `stripMusicVideoLocalRenderPins`, restored over a newer remote by `mergeProjectRecord`) because the bytes are not in the project's peer-sync asset manifest; both halves are backed up (PostgreSQL dump + rsync snapshot). Adapter: `server/services/musicVideo/devArtifactStore.js`.
- Music Video narrative events (`composition.narrativeEvents`) and section reactive gain caps (`composition.reactiveSections`) persist in the existing db-primary `music_video_projects` JSONB record. Anchors are song seconds, measured band-onset indices, or aligned cue/word indices; absent fields preserve legacy behavior. Audio replacement clears anchors while retaining narrative intent. Schema v14 gates peers whose composition normalizer would drop these fields. Resolved absolute frames live in immutable, wire-local document manifests; an event revision retains selected takes and replaces only affected section functions.
+- Music Video shot segments — the composed final render's cached per-shot encodes (`data/music-video-segments/{projectId}/{key}.mp4`, a film look pass as `{key}-{lookKey}.mp4`, the outro as `outro-…`). Content-addressed by everything that reaches their pixels, so a stale one is never reused; machine-local, regenerable, excluded from backups, dropped with the project and pruned when a final render publishes. Adapter: `server/services/musicVideo/shotSegments.js`.
- Music Video composition documents (render style `document`) — each import (zip, a folder inside `data/`, or the shipped `layered` template) is one immutable version folder `data/music-video/{projectId}/composition/{versionId}/` (index.html + scripts, fonts, media); the `musicVideoProject` record's `composition.document` holds the data-relative pointer plus source/size metadata. A clone shares the pointer; a version no project (live or tombstoned) points at is pruned on the next import or detach. The pointer is **wire-local** like the development artifacts (stripped by `stripMusicVideoLocalRenderPins`, restored by `mergeProjectRecord`) — a peer keeps its own and refuses to render without one. Renders stage a job-private copy under `data/music-video-song-renders/{jobId}/` (swept at boot, excluded from backups). Adapter: `server/services/musicVideo/compositionDocument.js`. An explicit `POST /api/music-video/:id/composition/document/engine/upgrade` with the selected `directory` adopts a recognized shipped layered engine in a new immutable version, preserving every authored file. Customized engines and pending candidates are refused; prior render/review evidence becomes stale when the document pointer changes.
- Beeper attachment mirror (#37) — message media stays on disk under `data/beeper/attachments/<sha256 prefix>/<sha256>.<ext>` (content-addressed, so one forwarded photo is one file); `beeper_attachments` in PostgreSQL holds the metadata plus `local_path` / `sha256` / `byte_length` / `keep`. Machine-local and **never federated** (message content is PII — see the message-bodies ADR); a lazy CACHE rather than an archive, so it is excluded from backup by default (overridable) while the rows that describe it ride the Postgres dump. The bytes are re-fetchable from Beeper Desktop for as long as the source network still holds the media, and the surface renders a labelled reference when it does not.
- Media collections — many-to-many links over assets/universes/series/catalog media pointers (`db-primary` link tables) pointing at `asset-file-db-indexed` bytes. **Still `data/media-collections/*` JSON today** — a follow-up slice of #1000.
diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.js
index 22f6cdb3d9..350b4628d4 100644
--- a/server/lib/musicVideoDependencies.js
+++ b/server/lib/musicVideoDependencies.js
@@ -25,6 +25,8 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s
// A glitch over its cuts changes the render; a shot without one keeps its evidence.
.concat(['transitionIn', 'transitionOut'].filter((key) => scene[key]).map((key) => [key, scene[key]]))
.concat(scene.glitches?.length ? [['glitches', scene.glitches]] : [])
+ // A shot's own film look changes the render; a shot following the video's keeps its evidence.
+ .concat(scene.filmLook ? [['filmLook', scene.filmLook]] : [])
.concat(scene.cutEffect && scene.cutEffect.enabled !== false ? [['cutEffect', scene.cutEffect]] : [])
// An image effect over a still changes the render; a still without one keeps its evidence.
.concat(scene.imageEffect ? [['imageEffect', scene.imageEffect], ['imageEffectSec', scene.imageEffectSec ?? null]] : [])
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index a9d48c6faa..cfe57509fa 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -636,6 +636,8 @@ const sceneLayerFields = {
reverse: z.boolean().nullable().optional(),
// A clip shorter than its shot holds its last frame for the rest of the shot; null clears it.
holdLastFrame: z.boolean().nullable().optional(),
+ // The shot's own film look (lib/filmLook.js), kept when the video's look changes; null follows the video's.
+ filmLook: filmLookSchema.nullable().optional(),
// A VHS glitch over the cut into / out of this shot (musicVideoTransitions.js); null clears it.
transitionIn: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),
transitionOut: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index ee0d4807d1..87d70b063b 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -718,6 +718,27 @@ router.get('/:id/render', (req, res) => {
res.json({ jobId: getActiveRenderJobId(req.params.id) });
});
+// Per-shot render segments (shotSegments.js): every shot's cache state, one
+// shot's render started or stopped, and a rendered shot's silent segment.
+router.get('/:id/shot-renders', asyncHandler(async (req, res) => {
+ const { getShotRenderStatus } = await import('../services/musicVideo/shotSegments.js');
+ res.json(await getShotRenderStatus(req.params.id));
+}));
+router.post('/:id/shots/:sceneId/render', asyncHandler(async (req, res) => {
+ const { startShotRender } = await import('../services/musicVideo/shotSegments.js');
+ res.status(202).json(await startShotRender(req.params.id, req.params.sceneId));
+}));
+router.post('/:id/shots/:sceneId/render/cancel', asyncHandler(async (req, res) => {
+ const { cancelShotRender } = await import('../services/musicVideo/shotSegments.js');
+ res.json({ ok: cancelShotRender(req.params.id, req.params.sceneId) });
+}));
+router.get('/:id/shot-renders/:name', asyncHandler(async (req, res) => {
+ const { shotSegmentFile } = await import('../services/musicVideo/shotSegments.js');
+ const path = shotSegmentFile(req.params.id, req.params.name);
+ if (!path) throw new ServerError('Shot render not found', { status: 404, code: 'NOT_FOUND' });
+ res.type('video/mp4').sendFile(path);
+}));
+
// Read-only: this project's in-flight scene frame/clip renders (#10154), so a
// reloaded board restores its spinners instead of letting a duplicate be submitted.
router.get('/:id/scene-jobs', (req, res) => {
diff --git a/server/services/backup.js b/server/services/backup.js
index 2995b75b91..7f1309ae41 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -253,6 +253,7 @@ export const DEFAULT_EXCLUDES = [
{ path: '/music-video-compositions/', reason: 'In-flight music-video typography overlay scratch — removed when its render ends and swept at boot', overridable: false },
{ path: '/music-video-film-look/', reason: 'In-flight music-video film look pass (a staged link to the finished render) — removed when the pass ends', overridable: false },
{ path: '/music-video-stats-outro/', reason: 'In-flight music-video outro page (fonts and floor stills) — removed when its render ends and swept at boot', overridable: false },
+ { path: '/music-video-segments/', reason: 'Cached music-video shot segments — the final render re-renders any that are missing', overridable: true },
{ path: '/music-video-song-renders/', reason: 'In-flight music-video composition renders (a staged copy of the document plus scene media) — removed when the render ends and swept at boot', overridable: false },
{ path: '/launch-videos/*/*/proofs/', reason: 'Launch-video critique-loop contact sheets — re-rendered from the run composition', overridable: true },
{ path: '/cache/', reason: 'Remote-API metadata and licensed reading caches — regenerable on demand, and stale on restore anyway', overridable: false },
diff --git a/server/services/dataManager.js b/server/services/dataManager.js
index 55c8de7051..bb84c9bc2f 100644
--- a/server/services/dataManager.js
+++ b/server/services/dataManager.js
@@ -93,6 +93,7 @@ export const CATEGORIES = {
'music-video-compositions': { label: 'Music Video Overlay Scratch', description: 'Typography overlay frames for an in-flight composed music-video render — removed when the render ends and swept at boot', archivable: false, deletable: false },
'music-video-film-look': { label: 'Music Video Film Look Scratch', description: 'A staged link to a finished render for its film look pass — removed when the pass ends', archivable: false, deletable: false },
'music-video-stats-outro': { label: 'Music Video Outro Scratch', description: 'The cutting room floor outro page (fonts and floor stills) for an in-flight music-video render — removed when the render ends and swept at boot', archivable: false, deletable: false },
+ 'music-video-segments': { label: 'Music Video Shot Renders', description: 'Cached per-shot segments of composed music-video final renders — pruned after each final render and removed with the project; the final render re-renders any that are missing', archivable: false, deletable: false },
'music-video-song-renders': { label: 'Music Video Composition Render Scratch', description: 'A staged copy of a composition document plus its scene media for an in-flight music-video render — removed when the render ends and swept at boot', archivable: false, deletable: false },
'code-animation-workspaces': { label: 'Code Animation Contained Workers', description: 'Per-run staged input, output and scratch for a sandboxed Code Animation worker — removed when the run ends and swept on the next run', archivable: false, deletable: false },
'code-animations': { label: 'Code Animations', description: 'Generated animation HTML retained with its job history — the only copy of provider-generated work', archivable: true, deletable: false },
diff --git a/server/services/musicVideo/compositionRender.js b/server/services/musicVideo/compositionRender.js
index 1b91389295..a1aecad872 100644
--- a/server/services/musicVideo/compositionRender.js
+++ b/server/services/musicVideo/compositionRender.js
@@ -140,30 +140,47 @@ async function stageLyricOverlay(dir, project, { width, height, fps, durationSec
}
/**
- * Render the overlay clips on a `width`×`height`, `fps` frame: the timed text
- * `cues` (already filtered by renderableCues) on the typography page, and, when
- * `lyrics` is `{ project, windows }` (composedLyricWindows), the sung lyrics in
- * the shared lyric type over those windows. Resolves `[{ path, startSec,
- * durationSec }]`, typography clips first, each set in time order.
- * `onProgress(0..1)` reports captured frames; `signal` cancels.
+ * The frame windows the overlay capture draws, per page (`doc` for the timed
+ * text cues, `lyrics` for the sung lyrics), in the order renderTypographyOverlays
+ * emits its clips. Each window is snapped outward to the output frame grid and
+ * never runs past the video's last frame; `window` (`{ startFrame, endFrame }`,
+ * a per-shot segment) clips every one to that span. Pure.
*/
-export async function renderTypographyOverlays({ jobId, cues, style, width, height, fps, durationSec, lyrics = null, signal, onProgress }) {
- // Snap each window outward to the output frame grid so every overlay frame
- // lands exactly on a footage frame, and never past the video's last frame.
+export function typographyOverlayWindows({ cues, lyricWindows = [], fps, durationSec, window = null }) {
const lastFrame = Math.round(durationSec * fps);
+ const lo = window ? Math.max(0, window.startFrame) : 0;
+ const hi = window ? Math.min(lastFrame, window.endFrame) : lastFrame;
const onGrid = (spans) => spans
.map((w) => {
- const first = Math.floor(w.startSec * fps + 1e-6);
- return { startSec: first / fps, frames: Math.min(lastFrame, Math.ceil(w.endSec * fps - 1e-6)) - first };
+ const first = Math.max(lo, Math.floor(w.startSec * fps + 1e-6));
+ return { startSec: first / fps, frames: Math.min(hi, Math.ceil(w.endSec * fps - 1e-6)) - first };
})
.filter((w) => w.frames > 0);
- const pages = [
- { name: 'doc', windows: onGrid(overlayWindows(cues)), stage: async (dir) => {
- await writeFile(join(dir, 'index.html'), buildTypographyDocument({ cues, style, width, height, durationSec, fps }));
- if (cues.some((c) => c.card || c.titleCard)) await copyFile(LYRIC_FONT_SOURCE, join(dir, LYRIC_FONT_FILE));
- } },
- { name: 'lyrics', windows: lyrics ? onGrid(lyrics.windows) : [], stage: (dir) => stageLyricOverlay(dir, lyrics.project, { width, height, fps, durationSec }) },
+ return [
+ { name: 'doc', windows: onGrid(overlayWindows(cues)) },
+ { name: 'lyrics', windows: onGrid(lyricWindows) },
].filter((p) => p.windows.length > 0);
+}
+
+/**
+ * Render the overlay clips on a `width`×`height`, `fps` frame: the timed text
+ * `cues` (already filtered by renderableCues) on the typography page, and, when
+ * `lyrics` is `{ project, windows }` (composedLyricWindows), the sung lyrics in
+ * the shared lyric type over those windows. Resolves `[{ path, startSec,
+ * durationSec }]`, typography clips first, each set in time order.
+ * `window` limits the capture to one segment's frames (typographyOverlayWindows).
+ * `onProgress(0..1)` reports captured frames; `signal` cancels.
+ */
+export async function renderTypographyOverlays({ jobId, cues, style, width, height, fps, durationSec, lyrics = null, window = null, signal, onProgress }) {
+ const pages = typographyOverlayWindows({ cues, lyricWindows: lyrics?.windows || [], fps, durationSec, window }).map(({ name, windows }) => ({
+ name, windows,
+ stage: name === 'doc'
+ ? async (dir) => {
+ await writeFile(join(dir, 'index.html'), buildTypographyDocument({ cues, style, width, height, durationSec, fps }));
+ if (cues.some((c) => c.card || c.titleCard)) await copyFile(LYRIC_FONT_SOURCE, join(dir, LYRIC_FONT_FILE));
+ }
+ : (dir) => stageLyricOverlay(dir, lyrics.project, { width, height, fps, durationSec }),
+ }));
if (pages.length === 0) return [];
const ffmpeg = await findFfmpeg();
if (!ffmpeg) throw new Error('ffmpeg not found on PATH');
diff --git a/server/services/musicVideo/events.js b/server/services/musicVideo/events.js
index 1bd65ff427..d1087189f5 100644
--- a/server/services/musicVideo/events.js
+++ b/server/services/musicVideo/events.js
@@ -16,6 +16,9 @@ import { EventEmitter } from 'events';
// synchronous external-SD-API lane (which returns the image filename inline and
// lets the client add it as a take through the takes route; video renders
// always ride the queue, so the clip attach is hook-only).
+// 'shot-render' → { projectId, sceneId, state: 'rendering'|'rendered'|'failed'|'canceled', progress?, error? }
+// — a shot's cached render segment advancing (shotSegments.js), bridged to
+// `music-video:shot-render` so the shot card shows it without a refetch.
// 'scene-failure' → { projectId, sceneId, lastFailure: { lane, error, at } | null }
//
// `scene-image` / `scene-video` also carry the scene's `lastFailure` (null once a
diff --git a/server/services/musicVideo/filmLookFinish.js b/server/services/musicVideo/filmLookFinish.js
index ba09874037..84c11de822 100644
--- a/server/services/musicVideo/filmLookFinish.js
+++ b/server/services/musicVideo/filmLookFinish.js
@@ -88,12 +88,14 @@ function filmLookFinishPage({ width, height, fps, durationSec, offsetSec = 0, sr
/**
* Re-encode the finished render at `path` through the project's film look, in
* place. Resolves `true` when the pass ran, `false` when the look is neutral
- * (the file is untouched). `offsetSec` is the song time of the render's first
- * frame (an excerpt's window start) so the grain lands on song frames.
+ * (the file is untouched). `look` (a shot's own look) replaces the project's;
+ * `workers` caps the browsers a short window splits across. `offsetSec` is the
+ * song time of the render's first frame (an excerpt's window or a shot's start)
+ * so the grain lands on song frames.
* Rejects on failure or when `signal` aborts, leaving `path` as it was.
*/
-export async function applyFilmLookFinish({ project, path, jobId, signal, onProgress, offsetSec = 0, label = 'Music-video film look' }) {
- const look = normalizeFilmLook(project?.filmLook);
+export async function applyFilmLookFinish({ project, look: lookInput, path, jobId, signal, onProgress, offsetSec = 0, workers, label = 'Music-video film look' }) {
+ const look = normalizeFilmLook(lookInput === undefined ? project?.filmLook : lookInput);
if (!look || isFilmLookNeutral(look)) return false;
if (typeof jobId !== 'string' || !/^[A-Za-z0-9_-]+$/.test(jobId)) throw new Error('Invalid film look job id');
const { findFfmpeg, runFfmpegProcess, probeVideoGeometry } = await import('../../lib/ffmpeg.js');
@@ -131,7 +133,7 @@ export async function applyFilmLookFinish({ project, path, jobId, signal, onProg
});
}
const encoded = await encodeCompositionSegments(page, { width, height, fps, durationSec }, silent, {
- openPage, encode: encodeComposition, signal, offsetSec,
+ openPage, encode: encodeComposition, signal, offsetSec, ...(workers ? { workers } : {}),
onProgress: (fraction) => onProgress?.(fraction * 0.95),
onRetry: ({ frame, attempt, segment, error }) => console.warn(`⚠️ ${label} [${jobId.slice(0, 8)}] ${segment == null ? '' : `worker ${segment + 1} `}restarting its browser at song frame ${frame} (retry ${attempt}): ${error.message}`),
});
diff --git a/server/services/musicVideo/projects.js b/server/services/musicVideo/projects.js
index 99e1db21cd..f5039e4422 100644
--- a/server/services/musicVideo/projects.js
+++ b/server/services/musicVideo/projects.js
@@ -226,6 +226,10 @@ export async function deleteProject(id) {
const result = await (await selectBackend()).deleteProject(id);
// Soft-delete tombstone — push the deletion to subscribed peers immediately.
emitRecordDeleted('musicVideoProject', id);
+ // Its cached shot segments are regenerable, and only this machine's.
+ await import('./shotSegments.js').then(({ removeProjectSegments }) => removeProjectSegments(id)).catch((err) => {
+ console.warn(`⚠️ Music Video ${id.slice(0, 8)}: could not remove its cached shot segments: ${err.message}`);
+ });
return result;
}
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index c44c2f0020..047cbdf602 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -525,6 +525,7 @@ function buildScene(input, { order }) {
...(input.offsetFrames ? { offsetFrames: input.offsetFrames } : {}),
...(input.reverse ? { reverse: true } : {}),
...(input.holdLastFrame ? { holdLastFrame: true } : {}),
+ ...(input.filmLook ? { filmLook: normalizeFilmLook(input.filmLook) } : {}),
...(input.transitionIn ? { transitionIn: input.transitionIn } : {}),
...(input.transitionOut ? { transitionOut: input.transitionOut } : {}),
...(input.glitches?.length ? { glitches: input.glitches } : {}),
@@ -616,6 +617,7 @@ export function replaceScenes(project, sceneInputs) {
...(prior.offsetFrames ? { offsetFrames: prior.offsetFrames } : {}),
...(prior.reverse ? { reverse: true } : {}),
...(prior.holdLastFrame ? { holdLastFrame: true } : {}),
+ ...(prior.filmLook ? { filmLook: prior.filmLook } : {}),
...(prior.transitionIn ? { transitionIn: prior.transitionIn } : {}),
...(prior.transitionOut ? { transitionOut: prior.transitionOut } : {}),
...(prior.glitches?.length ? { glitches: prior.glitches } : {}),
@@ -635,7 +637,7 @@ export function replaceScenes(project, sceneInputs) {
// takes, and whether it plays as a clip or its still with an image effect) without
// adding a shot or lyric the storyboard approval did not cover.
const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',
- 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'speedFps', 'offsetFrames', 'reverse', 'holdLastFrame', 'transitionIn', 'transitionOut', 'glitches', 'cutEffect',
+ 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'speedFps', 'offsetFrames', 'reverse', 'holdLastFrame', 'filmLook', 'transitionIn', 'transitionOut', 'glitches', 'cutEffect',
'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec', 'frameReferenceIds'];
/**
@@ -643,7 +645,9 @@ const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'p
* scene id is unknown.
*/
export function applySceneUpdate(project, sceneId, patch) {
- const data = parseSceneOrThrow(musicVideoSceneUpdateSchema, patch);
+ const parsed = parseSceneOrThrow(musicVideoSceneUpdateSchema, patch);
+ // A shot's own look is stored complete; null returns it to the video's look.
+ const data = 'filmLook' in parsed ? { ...parsed, filmLook: parsed.filmLook ? normalizeFilmLook(parsed.filmLook) : null } : parsed;
const scenes = project.scenes || [];
const idx = scenes.findIndex((s) => s.sceneId === sceneId);
if (idx < 0) throw new ServerError('Scene not found', { status: 404, code: 'NOT_FOUND' });
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 6aafd1f157..88a5a59750 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -34,7 +34,7 @@ import { randomUUID } from 'crypto';
import { ensureDir, PATHS } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay } from '../../lib/sseUtils.js';
-import { findFfmpeg, safeUnder, generateThumbnail, probeVideoDuration, probeVideoGeometry } from '../../lib/ffmpeg.js';
+import { BT709_CONTAINER_ARGS, H264_ENCODE_ARGS, findFfmpeg, safeUnder, generateThumbnail, probeVideoDuration, probeVideoGeometry } from '../../lib/ffmpeg.js';
import { filterGraphViaFile } from '../../lib/ffmpegFilterGraph.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
import { cancelMusicVideoRenderJob, isMusicVideoRenderCanceled } from './renderCancellation.js';
@@ -503,6 +503,54 @@ function sectionChain(c, inputRefs, { canonW, canonH, fps, frames, frameGrid })
return `[${id}:v]${fit},pad=${canonW}:${canonH}:(ow-iw)/2:(oh-ih)/2,${timed}`;
}
+// One plan section of a composed render as its own silent, video-only encode
+// (shotSegments.js): the section's chain on the full film's timeline (cut
+// effects borrowing the next shot, fades either side, its grade and glitches on
+// absolute song frames, the typography windows over it), so the encoded frames
+// are the ones the whole-film render draws there. `segment` is `{ index,
+// maxFrames?, tagFilter? }`; `overlays` here are this window's captures, each
+// with its absolute song `startSec`.
+function buildSegmentArgs(segment, { plan, prerolls, addInput, sectionFilters, glitches, overlays, outputPath, grade, canonW, canonH, fps, sectionFrame, inputs, nextInput }) {
+ const index = segment.index;
+ if (!Number.isInteger(index) || !plan[index]) throw new Error(`buildMusicVideoFfmpegArgs: no section ${index}`);
+ const { c, frames, startSec, endSec } = plan[index];
+ const startFrame = sectionFrame(startSec);
+ const own = addInput(c);
+ const preroll = prerolls.find((p) => p.index === index);
+ if (preroll) preroll.input = addInput(preroll.clip);
+ const overlayBase = nextInput();
+ for (const overlay of overlays) inputs.push('-itsoffset', String(round3(overlay.startSec - startSec)), '-i', overlay.path);
+ const filters = sectionFilters(index, own, preroll, 'v0');
+ const gradeFilter = musicVideoGradeFilter(grade, [{ sceneId: c.sceneId, startSec, endSec }], { fps, offsetSec: startSec });
+ const local = glitches.filter((g) => g.frame >= startFrame && g.frame < startFrame + frames).map((g) => ({ ...g, frame: g.frame - startFrame }));
+ const glitchFilter = musicVideoTransitionFilter(local, { width: canonW, height: canonH });
+ const finishing = [gradeFilter, glitchFilter].filter(Boolean).join(',');
+ let label = 'v0';
+ if (finishing) { filters.push(`[v0]${finishing}[graded]`); label = 'graded'; }
+ overlays.forEach((_, k) => {
+ filters.push(`[${label}][${overlayBase + k}:v]overlay=eof_action=pass:format=auto[ov${k}]`);
+ label = `ov${k}`;
+ });
+ // Every segment is tagged the way the composition encoder tags its own, so
+ // shot segments, film-look passes and the outro join by stream copy.
+ filters.push(`[${label}]format=yuv420p${segment.tagFilter ? `,${segment.tagFilter}` : ''}[outv]`);
+ const outFrames = Number.isInteger(segment.maxFrames) ? Math.max(0, Math.min(frames, segment.maxFrames)) : frames;
+ const args = [
+ ...inputs,
+ '-filter_complex', filters.join(';'),
+ '-map', '[outv]',
+ '-frames:v', String(outFrames),
+ '-r', String(fps),
+ ...H264_ENCODE_ARGS,
+ ...BT709_CONTAINER_ARGS,
+ '-an',
+ '-movflags', '+faststart',
+ '-y',
+ outputPath,
+ ];
+ return { args, sceneId: c.sceneId, layer: c.layer || 'footage', frames: outFrames, startFrame, startSec, endSec: (startFrame + outFrames) / fps, canonW, canonH, fps, sections: plan.length };
+}
+
// Pure: build the ffmpeg args for the master-bed render. Concats the clips
// video-only (each scaled/padded to the canonical dims + fps and trimmed to its
// snapped out-point) and maps ONE external audio input as the sole output audio.
@@ -526,7 +574,7 @@ function sectionChain(c, inputRefs, { canonW, canonH, fps, frames, frameGrid })
// overlap the window, clipped to it and re-based to the excerpt's own timeline
// (0 = `startSec`) so a contact sheet built from the excerpt file can sample
// cut boundaries directly. `totalDuration` becomes the excerpt's own length.
-export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec = null, overlays = [], frameGrid: gridOption = false, excerpt = null, soundBed = null, grade = null, aspect = null, beats = [] } = {}) {
+export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec = null, overlays = [], frameGrid: gridOption = false, excerpt = null, soundBed = null, grade = null, aspect = null, beats = [], segment = null } = {}) {
if (!Array.isArray(clips) || clips.length === 0) throw new Error('buildMusicVideoFfmpegArgs: empty clips');
const frameGrid = gridOption || clips.some((c) => c.layer);
// Stills and cards have no dimensions of their own: the first footage clip
@@ -608,29 +656,27 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioD
};
return { ...t, clip };
});
- for (const { c } of plan) sectionInputs.push(addInput(c));
- for (const preroll of prerolls) preroll.input = addInput(preroll.clip);
- const audioIdx = nextInput; // master audio follows the section inputs
- inputs.push('-i', audioPath);
- for (const overlay of overlays) inputs.push('-itsoffset', String(overlay.startSec), '-i', overlay.path);
-
const fadeChain = (i, frames) => [
...fades.filter((f) => f.index === i - 1 && f.inFrames > 0).map((f) => `,fade=t=in:s=0:n=${f.inFrames}`),
...fades.filter((f) => f.index === i).map((f) => `,fade=t=out:s=${Math.max(0, frames - f.outFrames)}:n=${f.outFrames}`),
].join('');
- const filters = plan.map(({ c, frames }, i) => `${sectionChain(c, sectionInputs[i], { canonW, canonH, fps, frames, frameGrid })}${fadeChain(i, frames)}[${prerolls.some((p) => p.index === i) ? 'vraw' : 'v'}${i}]`);
- for (const p of prerolls) {
- const { frames, startSec } = plan[p.index];
+ // One section's chain, ending on `out`; an incoming cut effect lays the next shot's preroll over its end.
+ const sectionFilters = (i, input, preroll, out) => {
+ const { c, frames, startSec } = plan[i];
+ const head = `${sectionChain(c, input, { canonW, canonH, fps, frames, frameGrid })}${fadeChain(i, frames)}`;
+ if (!preroll) return [`${head}[${out}]`];
const start = sectionFrame(startSec);
- const pad = Math.max(0, frames - p.windowFrames);
- const shown = p.incoming.map(([a, b]) => `between(n,${a - start},${b - start - 1})`).join('+');
- filters.push(`${sectionChain(p.clip, p.input, { canonW, canonH, fps, frames: p.windowFrames, frameGrid })}[pre${p.index}]`);
+ const pad = Math.max(0, frames - preroll.windowFrames);
+ const shown = preroll.incoming.map(([a, b]) => `between(n,${a - start},${b - start - 1})`).join('+');
// A dissolve fades the incoming clip in across its window, fully in on the cut.
- const fadeIn = p.style === 'dissolve' ? `format=yuva420p,fade=t=in:s=0:n=${p.windowFrames + 1}:alpha=1,` : '';
- filters.push(`[pre${p.index}]${fadeIn}tpad=start=${pad}:color=black,setpts=N/(${fps}*TB)[prepad${p.index}]`);
- filters.push(`[vraw${p.index}][prepad${p.index}]overlay=eof_action=pass:enable='${shown}'[v${p.index}]`);
- }
- const gradeFilter = musicVideoGradeFilter(grade, plan.map(({ c, startSec, endSec }) => ({ sceneId: c.sceneId, startSec, endSec })), { fps });
+ const fadeIn = preroll.style === 'dissolve' ? `format=yuva420p,fade=t=in:s=0:n=${preroll.windowFrames + 1}:alpha=1,` : '';
+ return [
+ `${head}[vraw${i}]`,
+ `${sectionChain(preroll.clip, preroll.input, { canonW, canonH, fps, frames: preroll.windowFrames, frameGrid })}[pre${i}]`,
+ `[pre${i}]${fadeIn}tpad=start=${pad}:color=black,setpts=N/(${fps}*TB)[prepad${i}]`,
+ `[vraw${i}][prepad${i}]overlay=eof_action=pass:enable='${shown}'[${out}]`,
+ ];
+ };
// Cut transitions repaint a few frames around their cuts, under the typography, and add no time.
const pieces = plan.map(({ c, startSec, endSec }) => ({
sceneId: c.sceneId, startFrame: Math.round(startSec * fps), endFrame: Math.round(endSec * fps),
@@ -640,6 +686,19 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioD
// A still's glitch to its end frame paints with the same frames.
// So do the glitches a shot sets at moments inside it.
const glitches = mergeGlitchFrames(transitionGlitchFrames(pieces), imageEffectGlitchFrames(pieces, fps), shotGlitchFrames(pieces, fps));
+
+ if (segment) {
+ return buildSegmentArgs(segment, { plan, prerolls, addInput, sectionFilters, glitches, overlays, outputPath, grade, canonW, canonH, fps, sectionFrame, inputs, nextInput: () => nextInput });
+ }
+
+ for (const { c } of plan) sectionInputs.push(addInput(c));
+ for (const preroll of prerolls) preroll.input = addInput(preroll.clip);
+ const audioIdx = nextInput; // master audio follows the section inputs
+ inputs.push('-i', audioPath);
+ for (const overlay of overlays) inputs.push('-itsoffset', String(overlay.startSec), '-i', overlay.path);
+
+ const filters = plan.flatMap((_, i) => sectionFilters(i, sectionInputs[i], prerolls.find((p) => p.index === i), `v${i}`));
+ const gradeFilter = musicVideoGradeFilter(grade, plan.map(({ c, startSec, endSec }) => ({ sceneId: c.sceneId, startSec, endSec })), { fps });
const glitchFilter = musicVideoTransitionFilter(glitches, { width: canonW, height: canonH });
const cutLabel = overlays.length > 0 ? 'cut0' : 'outv';
const finishing = [gradeFilter, glitchFilter].filter(Boolean).join(',');
@@ -958,6 +1017,87 @@ async function renderSeekedMode(projectId, project, handOff, renderer, options =
return { jobId };
}
+// A composed footage render (shotSegments.js): each shot section, its film
+// look and the outro come from the segment cache, rendering only what is
+// missing, then join by stream copy with the song laid under them once.
+async function renderFromShotSegments(projectId, project, handOff, options) {
+ const { planShotSegments, renderMissingSegments, stitchSegments, pruneSegments } = await import('./shotSegments.js');
+ const plan = await planShotSegments(project);
+ await ensureDir(PATHS.videos);
+ await ensureDir(PATHS.videoThumbnails);
+ const renderingOn = await ensureInstanceId();
+ await assertCurrentRenderApproval(projectId, project, options);
+ const jobId = randomUUID();
+ const filename = `music-video-${projectId.slice(0, 8)}-${Date.now()}.mp4`;
+ const outputPath = join(PATHS.videos, filename);
+ const job = { id: jobId, projectId, status: 'running', clients: [], process: null, totalDuration: plan.songSec, overlayAbort: new AbortController() };
+ jobs.set(jobId, job);
+ projectRenders.set(projectId, jobId);
+ handOff();
+ const priorStatus = project.status && project.status !== 'rendering' ? project.status : 'ready';
+ const writeProject = (patch, status) => updateProject(projectId, patch).catch((err) => {
+ maintenance.markCurrentUnsettled();
+ console.error(`❌ Music-video render [${jobId.slice(0, 8)}] project ${projectId.slice(0, 8)} status→${status} write failed: ${err.message}`);
+ });
+ await writeProject({ status: 'rendering', renderingOn, renderPartialFilename: filename, renderError: null }, 'rendering');
+ const all = [...plan.segments, ...(plan.outro ? [plan.outro] : [])];
+ const cached = all.filter((segment) => existsSync(segment.path)).length;
+ console.log(`🎬 Rendering music video [${jobId.slice(0, 8)}]: project=${projectId.slice(0, 8)} segments=${all.length} cached=${cached} duration=${plan.songSec.toFixed(2)}s`);
+ const { signal } = job.overlayAbort;
+ const finish = (status) => {
+ projectRenders.delete(projectId);
+ emitRenderSettled(projectId, job);
+ closeJobAfterDelay(jobs, jobId);
+ options.maintenancePermit?.finish();
+ return status;
+ };
+ Promise.resolve().then(async () => {
+ // Shots take 95% of the bar; joining and laying the song under them the rest.
+ await renderMissingSegments(plan, { signal, onProgress: (fraction) => broadcastSse(job, { type: 'progress', progress: 0.95 * fraction }) });
+ await assertCurrentRenderApproval(projectId, project, options);
+ const { durationSec } = await stitchSegments(plan, outputPath, { signal });
+ broadcastSse(job, { type: 'progress', progress: 1 });
+ return durationSec;
+ }).then(async (durationSec) => {
+ job.overlayAbort = null;
+ job.status = 'complete';
+ const section = (project.audioAnalysis?.sections || [])
+ .filter((entry) => Number.isFinite(entry.energy) && entry.startSec >= 0 && entry.endSec > entry.startSec && entry.startSec < plan.songSec)
+ .sort((a, b) => b.energy - a.energy || a.startSec - b.startSec)[0];
+ const posterSec = project.composition?.posterSec;
+ const atSec = typeof posterSec === 'number' && posterSec < plan.songSec ? posterSec
+ : section ? (section.startSec + Math.min(section.endSec, plan.songSec)) / 2 : undefined;
+ const thumb = await withBackupAssetPublication(async () => {
+ const poster = await generateThumbnail(outputPath, jobId, { atSec });
+ await appendToVideoHistory({
+ id: jobId, prompt: `Music Video: ${project.name}`, modelId: 'music-video', seed: 0,
+ width: plan.width, height: plan.height, numFrames: Math.round(durationSec * plan.fps), fps: plan.fps,
+ filename, thumbnail: poster, createdAt: new Date().toISOString(), musicVideoProjectId: projectId,
+ });
+ await writeProject(settledRender('complete', { renderHistoryId: jobId, renderDependencies: captureMusicVideoEvidence(project) }), 'complete');
+ return poster;
+ });
+ finish('complete');
+ // Older takes of each shot are dropped once the new cut is published.
+ const pruned = await pruneSegments(plan).catch(() => 0);
+ console.log(`✅ Music video rendered [${jobId.slice(0, 8)}]: ${filename} (${all.length - cached} of ${all.length} segments rendered${pruned ? `, ${pruned} old dropped` : ''})`);
+ broadcastSse(job, { type: 'complete', result: { id: jobId, filename, thumbnail: thumb, path: `/data/videos/${filename}` } });
+ }).catch(async (err) => {
+ const canceled = signal.aborted || err?.code === 'CANCELED';
+ job.status = canceled ? 'canceled' : 'error';
+ const reason = canceled ? 'Render cancelled' : err.message;
+ job.lastError = reason;
+ const log = canceled ? console.log : console.error;
+ log(`${canceled ? '🛑' : '❌'} Music-video render ${canceled ? 'cancelled' : 'failed'} [${jobId.slice(0, 8)}]: ${reason}`);
+ await unlink(outputPath).catch(() => {});
+ broadcastSse(job, { type: canceled ? 'canceled' : 'error', error: reason });
+ const status = canceled ? priorStatus : 'failed';
+ await writeProject(settledRender(status, { renderError: canceled ? null : reason.slice(0, 2000) }), status);
+ finish(status);
+ });
+ return { jobId };
+}
+
async function assertCurrentRenderApproval(projectId, preparedProject, options) {
const current = await getProject(projectId);
// The final render needs the approved storyboard; the animated proof is optional.
@@ -998,6 +1138,10 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
if (seeked) {
return await renderSeekedMode(projectId, project, () => { handedOff = true; }, seeked, { ...options, maintenancePermit: permit });
}
+ // A composed cut renders shot by shot into the segment cache and stitches.
+ if (project.composition?.mode === 'composed') {
+ return await renderFromShotSegments(projectId, project, () => { handedOff = true; }, { ...options, maintenancePermit: permit });
+ }
const { ffmpeg, audioPath, composed, clips, audioDurationSec, soundBed, aspect } = await planMusicVideoRender(project);
await ensureDir(PATHS.videos);
await ensureDir(PATHS.videoThumbnails);
@@ -1292,6 +1436,10 @@ export async function recoverStuckMusicVideoRenders() {
await sweepStatsOutroScratch().catch((err) => {
console.warn(`⚠️ Music Video recovery: could not remove stale outro scratch: ${err.message}`);
});
+ // Nor a shot segment an interrupted render was writing (finished ones stay cached).
+ await import('./shotSegments.js').then(({ sweepPartialSegments }) => sweepPartialSegments()).catch((err) => {
+ console.warn(`⚠️ Music Video recovery: could not remove partial shot segments: ${err.message}`);
+ });
const instanceId = await ensureInstanceId();
const stuck = (await listProjects()).filter((p) => p?.status === 'rendering'
&& !projectRenders.has(p.id) && isLocalRenderMark(p.renderingOn, instanceId));
diff --git a/server/services/musicVideo/render.terminal.test.js b/server/services/musicVideo/render.terminal.test.js
index 982fad4eba..0bc123eac8 100644
--- a/server/services/musicVideo/render.terminal.test.js
+++ b/server/services/musicVideo/render.terminal.test.js
@@ -77,6 +77,25 @@ vi.mock('./compositionRender.js', () => ({
}));
vi.mock('./projects.js', () => ({ getProject: vi.fn(), listProjects: vi.fn(async () => []), updateProject: vi.fn(async () => ({})), mutateProjectRecord: vi.fn() }));
vi.mock('../instanceIdentity.js', () => ({ ensureInstanceId: vi.fn(async () => 'inst-self') }));
+// A composed render goes shot by shot (shotSegments.js): the segment work is
+// held open until a test settles it, or rejects on the render's abort signal.
+const seg = vi.hoisted(() => ({ plans: [], render: null, stitch: vi.fn(async () => ({ durationSec: 2, songSec: 2 })) }));
+vi.mock('./shotSegments.js', () => ({
+ planShotSegments: vi.fn(async (project) => {
+ // Planning refuses what the whole-film plan refuses (missing stills, untimed sections).
+ await (await import('./render.js')).planMusicVideoRender(project);
+ const plan = { project, songSec: 2, width: 910, height: 512, fps: 24, segments: [{ sceneId: 's1', path: '/data/seg/a.mp4', frames: 48 }], outro: null };
+ seg.plans.push(plan);
+ return plan;
+ }),
+ renderMissingSegments: vi.fn((plan, { signal, onProgress }) => new Promise((resolve, reject) => {
+ seg.render = { resolve, onProgress };
+ signal.addEventListener('abort', () => reject(Object.assign(new Error('Render cancelled'), { code: 'CANCELED' })), { once: true });
+ })),
+ stitchSegments: (...args) => seg.stitch(...args),
+ pruneSegments: vi.fn(async () => 0),
+ sweepPartialSegments: vi.fn(async () => {}),
+}));
import { buildTypographyDocument } from './composition.js';
import { renderMusicVideo, getRenderJobStatus, cancelRender } from './render.js';
@@ -110,6 +129,8 @@ beforeEach(() => {
vi.clearAllMocks();
h.procs.length = 0;
h.overlays.calls.length = 0;
+ seg.render = null;
+ seg.plans.length = 0;
});
describe('renderMusicVideo terminal handling (#2386)', () => {
@@ -504,43 +525,54 @@ describe('composed render lifecycle (#8984)', () => {
expect(lastProc().args).not.toContain('-itsoffset');
});
- it('cancels during overlay capture: no ffmpeg, prior status restored, scratch removed, slot released', async () => {
+ it('cancels while shots render: no stitch, prior status restored, slot released', async () => {
primeComposed('cancel-1', { mode: 'composed', textCues: [cue] });
const { jobId } = await renderMusicVideo('cancel-1');
- expect(h.overlays.calls).toHaveLength(1);
- expect(h.procs).toHaveLength(0);
+ await tick();
+ expect(seg.render).toBeTruthy();
expect(cancelRender(jobId)).toBe(true);
await tick();
await tick();
expect(getRenderJobStatus(jobId).status).toBe('canceled');
+ expect(seg.stitch).not.toHaveBeenCalled();
expect(h.procs).toHaveLength(0);
- expect(updateProject).toHaveBeenLastCalledWith('cancel-1', settled('complete'));
- expect(removeCompositionScratch).toHaveBeenCalledWith(jobId);
+ expect(updateProject).toHaveBeenLastCalledWith('cancel-1', settled('complete', { renderError: null }));
expect(cancelRender(jobId)).toBe(false);
const again = await renderMusicVideo('cancel-1');
expect(again.jobId).not.toBe(jobId);
});
- it('lays the captured overlays over the cut and honors the chosen poster frame', async () => {
+ it('renders the missing shots, stitches them into the published file and honors the chosen poster frame', async () => {
primeComposed('composed-1', { mode: 'composed', textCues: [cue], posterSec: 1 });
const { jobId } = await renderMusicVideo('composed-1');
- // The 768x512 fixture clips sit in a 16:9 video, at their height.
- expect(h.overlays.calls[0]).toMatchObject({ jobId, width: 910, height: 512, fps: 24, cues: [expect.objectContaining({ id: 'c1' })] });
- h.overlays.resolve([{ path: '/data/overlay-0.mov', startSec: 0.5, durationSec: 1 }]);
await tick();
- const { args } = lastProc();
- expect(args.slice(args.indexOf('-itsoffset'), args.indexOf('-itsoffset') + 4)).toEqual(['-itsoffset', '0.5', '-i', '/data/overlay-0.mov']);
- expect(args[args.indexOf('-filter_complex') + 1]).toContain('[cut0][2:v]overlay=eof_action=pass:format=auto[outv]');
- expect(args[args.indexOf('-map') + 1]).toBe('[outv]');
- lastProc().emit('spawn');
- lastProc().emit('close', 0, null);
+ seg.render.onProgress(0.5);
+ expect(broadcastSse).toHaveBeenCalledWith(expect.anything(), { type: 'progress', progress: 0.475 });
+ seg.render.resolve(0);
+ await tick();
+ await tick();
await tick();
- expect(generateThumbnail).toHaveBeenCalledWith(expect.any(String), jobId, { atSec: 1 });
+ const [plan, outputPath] = seg.stitch.mock.calls[0];
+ expect(plan).toBe(seg.plans.at(-1));
+ expect(outputPath).toMatch(/^\/data\/videos\/music-video-composed/);
+ expect(generateThumbnail).toHaveBeenCalledWith(outputPath, jobId, { atSec: 1 });
expect(getRenderJobStatus(jobId).status).toBe('complete');
- expect(removeCompositionScratch).toHaveBeenCalledWith(jobId);
+ expect(updateProject).toHaveBeenCalledWith('composed-1', expect.objectContaining({ status: 'complete', renderHistoryId: jobId }));
// Nothing is left to cancel once the render has finished.
expect(cancelRender(jobId)).toBe(false);
});
+
+ it('a failed shot settles the render as failed with the shot it names', async () => {
+ primeComposed('composed-2', { mode: 'composed' });
+ seg.stitch.mockRejectedValueOnce(new Error('Shot 2: ffmpeg failed: boom'));
+ const { jobId } = await renderMusicVideo('composed-2');
+ await tick();
+ seg.render.resolve(0);
+ await tick();
+ await tick();
+ expect(getRenderJobStatus(jobId)).toEqual({ status: 'error', error: 'Shot 2: ffmpeg failed: boom' });
+ expect(updateProject).toHaveBeenLastCalledWith('composed-2', settled('failed', { renderError: 'Shot 2: ffmpeg failed: boom' }));
+ });
});
// #8985 — per-scene visual layers. A composed render cuts a still (its
@@ -572,38 +604,6 @@ describe('layered sections (#8985)', () => {
expect(h.overlays.calls).toHaveLength(0);
});
- it('cuts footage, a still and a titled card on one frame grid and draws the card text over its section', async () => {
- primeLayered('layers-1', 'composed');
- const { jobId } = await renderMusicVideo('layers-1');
- // The card's title is captured for exactly its section: [2s clip][2s still][1.01s card → 24 frames].
- expect(h.overlays.calls[0]).toMatchObject({ jobId, width: 910, height: 512,
- cues: [{ id: 'card-s3', text: 'Verse two', startSec: 4, endSec: 5, placement: 'center', emphasis: 'hero' }] });
- h.overlays.resolve([{ path: '/data/overlay-0.mov', startSec: 4, durationSec: 1 }]);
- await tick();
- const { args } = lastProc();
- expect(args.slice(args.indexOf('-loop'), args.indexOf('-loop') + 6)).toEqual(['-loop', '1', '-framerate', '24', '-i', '/data/images/frame.png']);
- const graph = filterGraph(args);
- expect(graph).toContain('trim=end_frame=48,setpts=PTS-STARTPTS[v0]');
- expect(graph).toMatch(/crop=768:512:x='\(iw-ow\)\*n\/47'.*trim=end_frame=48,setpts=PTS-STARTPTS\[v1\]/);
- expect(graph).toContain('color=c=0x112233:s=768x512:r=24,setsar=1,format=yuv420p,trim=end_frame=24,setpts=PTS-STARTPTS[v2]');
- // The song is still the one and only audio: two section inputs, then the master.
- expect(args[args.lastIndexOf('-map') + 1]).toBe('2:a');
- });
-
- it.each([['HUD counters', 'pop', 'Menlo'], [undefined, 'fade', 'Georgia']])('applies shared card and font treatment %s at the full render entrypoint', async (graphicLanguage, template, font) => {
- const id = `typography-${template}`;
- primeLayered(id, 'composed');
- const project = await getProject(id);
- getProject.mockResolvedValue({ ...project, composition: { ...project.composition, style: { font: 'serif' } }, treatment: { brief: { graphicLanguage } } });
- const { jobId } = await renderMusicVideo(id);
- const options = h.overlays.calls[0];
- expect(options.cues[0]).toMatchObject({ id: 'card-s3', template });
- expect(options.style).toEqual({ font: 'serif', graphicLanguage });
- expect(buildTypographyDocument(options)).toContain(font);
- cancelRender(jobId);
- await tick();
- });
-
it('refuses a still with no reference frame and an untimed card, without spawning', async () => {
primeLayered('no-frame', 'composed', [{ sceneId: 's1', order: 0, visualLayer: 'still', referenceImageId: null, startSec: 0, endSec: 2 }]);
await expect(renderMusicVideo('no-frame')).rejects.toMatchObject({ status: 404, code: 'MISSING_STILLS', context: { sceneIds: ['s1'] } });
diff --git a/server/services/musicVideo/shotSegments.js b/server/services/musicVideo/shotSegments.js
new file mode 100644
index 0000000000..464dab09f7
--- /dev/null
+++ b/server/services/musicVideo/shotSegments.js
@@ -0,0 +1,516 @@
+/**
+ * Music Video — per-shot render segments for composed footage renders.
+ *
+ * A composed final render used to be one ffmpeg graph over every shot, one
+ * overlay capture over the whole song and one film look pass over the whole
+ * cut, so tuning one shot re-drew the full film. Here each section of the cut
+ * is its own cached segment, in two stages:
+ *
+ * base the section's frames from buildMusicVideoFfmpegArgs' segment mode
+ * (the same chain the whole-film graph draws there: cut effects from
+ * the next shot, fades, grade, glitches and the typography over it),
+ * keyed on everything that reaches those pixels.
+ * look the base re-captured through the shot's film look (its own pinned
+ * look, else the video's) on song time, keyed on the base and the look.
+ *
+ * A look change redoes only the look stage, a shot change only that shot.
+ * The final render renders what is missing, joins the segments by stream copy
+ * and lays the song under them once. Segments live under data/ per project
+ * and are machine-local (excluded from backups, regenerable).
+ */
+
+import { createHash, randomUUID } from 'crypto';
+import { existsSync } from 'fs';
+import { copyFile, mkdir, readdir, rename, rm, stat, writeFile } from 'fs/promises';
+import { availableParallelism, tmpdir } from 'os';
+import { join } from 'path';
+import { PATHS } from '../../lib/fileUtils.js';
+import { ServerError } from '../../lib/errorHandler.js';
+import { AAC_ENCODE_ARGS, runFfmpegProcess, supportsSetparamsFilter } from '../../lib/ffmpeg.js';
+import { filterGraphViaFile } from '../../lib/ffmpegFilterGraph.js';
+import { isFilmLookNeutral, normalizeFilmLook } from '../../lib/filmLook.js';
+import { normalizeStatsOutro } from '../../lib/musicVideoStatsOutro.js';
+import { AUDIO_NORM, buildAudioBedMix } from '../videoTimeline/audioBedMix.js';
+import { buildMusicVideoFfmpegArgs, planMusicVideoRender } from './render.js';
+import { composedLyricWindows, projectTypographyPlan } from './composition.js';
+import { removeCompositionScratch, renderTypographyOverlays, typographyOverlayWindows } from './compositionRender.js';
+import { lyricOverlayDocumentData } from './lyricPlaythrough.js';
+import { applyFilmLookFinish } from './filmLookFinish.js';
+import { getMusicVideoOutroStats, renderStatsOutroClip } from './statsOutro.js';
+import { nameShots } from './productionReview.js';
+import { getProject } from './projects.js';
+import { musicVideoEvents } from './events.js';
+import { maintenance } from '../../lib/maintenanceAdmission.js';
+
+export const SHOT_SEGMENT_DIR = 'music-video-segments';
+// Bump when a segment's encode changes in a way its key cannot see.
+export const SHOT_SEGMENT_VERSION = 1;
+// Timed text this far outside a segment is left out of its key: a lyric line
+// or cue edit elsewhere in the song does not re-render it.
+const KEY_MARGIN_SEC = 5;
+const SEGMENT_TAG = 'setparams=range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709';
+
+const hash = (value) => createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 32);
+const safeProjectId = (projectId) => {
+ if (typeof projectId !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(projectId)) throw new ServerError('Invalid project id', { status: 400, code: 'VALIDATION_ERROR' });
+ return projectId;
+};
+export const shotSegmentDir = (projectId) => join(PATHS.data, SHOT_SEGMENT_DIR, safeProjectId(projectId));
+
+/** A shot's own pinned look, else the video's; null when the shot plays clean. Pure. */
+function shotFilmLook(project, scene) {
+ const own = scene?.filmLook && typeof scene.filmLook === 'object' ? scene.filmLook : null;
+ const look = normalizeFilmLook(own || project?.filmLook);
+ return look && !isFilmLookNeutral(look) ? look : null;
+}
+
+/**
+ * `value` with every timed entry (an object carrying a finite `startSec`) that
+ * lies wholly outside `[lo, hi]` removed, at any depth — the part of a page's
+ * data a segment's frames can depend on, for its key. Pure.
+ */
+function windowedKeyData(value, lo, hi) {
+ const outside = (v) => v && typeof v === 'object' && Number.isFinite(v.startSec)
+ && (v.startSec > hi || (Number.isFinite(v.endSec) ? v.endSec : v.startSec) < lo);
+ if (Array.isArray(value)) return value.filter((v) => !outside(v)).map((v) => windowedKeyData(v, lo, hi));
+ if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, windowedKeyData(v, lo, hi)]));
+ return value;
+}
+
+// Each input file stands in the key as its size and modification time, so a
+// replaced clip or still re-renders while an unchanged one keeps its segment.
+async function fingerprintArgs(args) {
+ const out = [];
+ for (let i = 0; i < args.length; i++) {
+ const value = args[i];
+ if (args[i - 1] === '-i' && !value.startsWith('@')) {
+ const info = await stat(value).catch(() => null);
+ out.push(info ? `${value}|${info.size}|${Math.round(info.mtimeMs)}` : `${value}|missing`);
+ } else out.push(value);
+ }
+ return out;
+}
+
+/**
+ * Plan a composed project's segments: the same preflight and clip plan as the
+ * final render, then each section's keys and cache paths. Throws the render's
+ * own refusals (each names its shots and the fix). Reads files, writes nothing.
+ */
+export async function planShotSegments(project) {
+ if (project?.composition?.mode !== 'composed') {
+ throw new ServerError('Shot renders need the composed render style', { status: 409, code: 'SHOT_RENDER_NEEDS_COMPOSED' });
+ }
+ const { ffmpeg, audioPath, clips, audioDurationSec, soundBed, aspect } = await planMusicVideoRender(project);
+ const grade = project.composition?.grade;
+ const beats = project.audioAnalysis?.beats;
+ const common = { audioDurationSec, frameGrid: true, grade, beats, aspect };
+ const full = buildMusicVideoFfmpegArgs(clips, audioPath, join(tmpdir(), 'portos-segment-plan.mp4'), common);
+ const { sections, totalDuration, canonW: width, canonH: height, fps } = full;
+ const { cues, style } = projectTypographyPlan(project, clips, sections, totalDuration);
+ const lyricWindows = composedLyricWindows(project, totalDuration);
+ const lyricData = lyricWindows.length ? lyricOverlayDocumentData(project, { width, height, fps, durationSec: totalDuration }) : null;
+ const tagFilter = await supportsSetparamsFilter() ? SEGMENT_TAG : null;
+ const lastFrame = Math.round(totalDuration * fps);
+ const dir = shotSegmentDir(project.id);
+ const scenes = new Map((project.scenes || []).map((scene) => [scene.sceneId, scene]));
+ const segments = [];
+ for (const [index, section] of sections.entries()) {
+ const startFrame = Math.round(section.startSec * fps);
+ if (startFrame >= lastFrame) break;
+ const window = { startFrame, endFrame: Math.min(Math.round(section.endSec * fps), lastFrame) };
+ const overlayPages = typographyOverlayWindows({ cues, lyricWindows, fps, durationSec: totalDuration, window });
+ const placeholders = overlayPages.flatMap(({ name, windows }) => windows.map((w, k) => ({ path: `@overlay/${name}/${k}`, startSec: w.startSec })));
+ const segment = { index, maxFrames: window.endFrame - startFrame, tagFilter };
+ const built = buildMusicVideoFfmpegArgs(clips, audioPath, '@out', { ...common, overlays: placeholders, segment });
+ const lo = built.startSec - KEY_MARGIN_SEC;
+ const hi = built.endSec + KEY_MARGIN_SEC;
+ const nearEnd = hi >= totalDuration;
+ const overlayKey = {
+ ...(overlayPages.some((p) => p.name === 'doc') ? { doc: { cues: windowedKeyData(cues, lo, hi), style, ...(nearEnd ? { totalDuration } : {}) } } : {}),
+ ...(overlayPages.some((p) => p.name === 'lyrics') ? { lyrics: windowedKeyData({ ...lyricData, render: nearEnd ? lyricData.render : null }, lo, hi) } : {}),
+ };
+ const baseKey = hash({ v: SHOT_SEGMENT_VERSION, args: await fingerprintArgs(built.args), overlays: overlayKey, width, height, fps });
+ const look = shotFilmLook(project, scenes.get(built.sceneId));
+ const lookKey = look ? hash({ v: SHOT_SEGMENT_VERSION, baseKey, look, startSec: built.startSec, fps }) : null;
+ segments.push({
+ index, sceneId: built.sceneId, layer: built.layer, startSec: built.startSec, endSec: built.endSec,
+ startFrame, frames: built.frames, window, segment, look, baseKey, lookKey,
+ basePath: join(dir, `${baseKey}.mp4`),
+ path: join(dir, lookKey ? `${baseKey}-${lookKey}.mp4` : `${baseKey}.mp4`),
+ });
+ }
+ if (!segments.length) throw new ServerError('Nothing to render — no shot spans a frame of the song', { status: 422, code: 'NO_SCENE_CLIPS' });
+ const songFrames = segments.reduce((sum, s) => sum + s.frames, 0);
+ const songSec = songFrames / fps;
+ const outro = await planOutroSegment(project, { dir, width, height, fps, offsetSec: songSec });
+ return {
+ project, ffmpeg, audioPath, soundBed, clips, common, cues, style, lyricWindows,
+ width, height, fps, totalDuration, songSec, segments, outro, dir,
+ };
+}
+
+async function planOutroSegment(project, { dir, width, height, fps, offsetSec }) {
+ const outro = normalizeStatsOutro(project?.statsOutro);
+ if (!outro?.enabled) return null;
+ const stats = await getMusicVideoOutroStats(project);
+ const baseKey = hash({ v: SHOT_SEGMENT_VERSION, outro, stats, width, height, fps });
+ const look = shotFilmLook(project, null);
+ const lookKey = look ? hash({ v: SHOT_SEGMENT_VERSION, baseKey, look, startSec: offsetSec, fps }) : null;
+ return {
+ outro: true, stats, startSec: offsetSec, durationSec: outro.durationSec, frames: Math.round(outro.durationSec * fps), look, baseKey, lookKey,
+ basePath: join(dir, `outro-${baseKey}.mp4`),
+ path: join(dir, lookKey ? `outro-${baseKey}-${lookKey}.mp4` : `outro-${baseKey}.mp4`),
+ };
+}
+
+/** Each planned segment's state: `rendered` when its final file is cached. */
+export const segmentRendered = (segment) => existsSync(segment.path);
+
+// One render per cache file at a time: a shot render and a final render that
+// both need it share the one in flight.
+const inFlight = new Map();
+
+/** Whether a segment's file is being rendered now. */
+export const segmentRendering = (segment) => inFlight.has(segment.path);
+
+/**
+ * Render one planned segment (a shot section or the outro) into the cache,
+ * skipping whatever stage is already cached. `onProgress(0..1)`; `signal`
+ * cancels (the shared render keeps going while another caller still waits).
+ */
+export function renderSegment(plan, segment, { signal, onProgress } = {}) {
+ if (existsSync(segment.path)) return Promise.resolve(segment.path);
+ let entry = inFlight.get(segment.path);
+ if (!entry) {
+ const controller = new AbortController();
+ entry = { controller, waiters: 0, listeners: new Set() };
+ entry.promise = renderSegmentFiles(plan, segment, { signal: controller.signal, onProgress: (f) => entry.listeners.forEach((fn) => fn(f)) })
+ .finally(() => inFlight.delete(segment.path));
+ inFlight.set(segment.path, entry);
+ }
+ entry.waiters += 1;
+ if (onProgress) entry.listeners.add(onProgress);
+ let left = false;
+ const leave = () => {
+ if (left) return false;
+ left = true;
+ entry.waiters -= 1;
+ if (onProgress) entry.listeners.delete(onProgress);
+ return true;
+ };
+ return new Promise((resolve, reject) => {
+ const onAbort = () => {
+ // The last caller to give up stops the render; another still waiting keeps it.
+ if (leave() && entry.waiters <= 0) entry.controller.abort(new Error('Render cancelled'));
+ reject(Object.assign(new Error('Render cancelled'), { code: 'CANCELED' }));
+ };
+ if (signal?.aborted) { onAbort(); return; }
+ signal?.addEventListener('abort', onAbort, { once: true });
+ entry.promise.then((path) => {
+ signal?.removeEventListener('abort', onAbort);
+ if (leave()) resolve(path);
+ }, (err) => {
+ signal?.removeEventListener('abort', onAbort);
+ if (leave()) reject(err);
+ });
+ });
+}
+
+const label = (plan, segment) => (segment.outro ? 'the ending card' : nameShots(plan.project, [segment.sceneId]));
+
+async function renderSegmentFiles(plan, segment, { signal, onProgress }) {
+ await mkdir(plan.dir, { recursive: true });
+ const jobId = randomUUID();
+ const lookShare = segment.look ? 0.5 : 0;
+ if (!existsSync(segment.basePath)) {
+ const partial = `${segment.basePath}.${jobId}.partial.mp4`;
+ try {
+ if (segment.outro) {
+ await renderStatsOutroClip({ project: plan.project, outputPath: partial, width: plan.width, height: plan.height, fps: plan.fps, jobId,
+ stats: segment.stats, signal, onProgress: (f) => onProgress?.(f * (1 - lookShare)) });
+ } else {
+ await renderShotBase(plan, segment, partial, { jobId, signal, onProgress: (f) => onProgress?.(f * (1 - lookShare)) });
+ }
+ await rename(partial, segment.basePath);
+ } catch (err) {
+ await rm(partial, { force: true }).catch(() => {});
+ if (signal.aborted) throw Object.assign(new Error('Render cancelled'), { code: 'CANCELED' });
+ throw new ServerError(`${label(plan, segment)}: ${err.message}`, { status: err.status || 500, code: err.code || 'SHOT_RENDER_FAILED' });
+ }
+ }
+ if (segment.look && !existsSync(segment.path)) {
+ const partial = `${segment.path}.${jobId}.partial.mp4`;
+ try {
+ await copyFile(segment.basePath, partial);
+ await applyFilmLookFinish({ project: plan.project, look: segment.look, path: partial, jobId, signal, offsetSec: segment.startSec, workers: 1,
+ label: `Music-video film look ${segment.outro ? 'ending card' : `shot ${segment.sceneId.slice(0, 8)}`}`,
+ onProgress: (f) => onProgress?.(1 - lookShare + f * lookShare) });
+ await rename(partial, segment.path);
+ } catch (err) {
+ await rm(partial, { force: true }).catch(() => {});
+ if (signal.aborted) throw Object.assign(new Error('Render cancelled'), { code: 'CANCELED' });
+ throw new ServerError(`${label(plan, segment)}: film look failed: ${err.message}`, { status: err.status || 500, code: err.code || 'SHOT_LOOK_FAILED' });
+ }
+ }
+ onProgress?.(1);
+ return segment.path;
+}
+
+async function renderShotBase(plan, segment, outputPath, { jobId, signal, onProgress }) {
+ const needsOverlay = typographyOverlayWindows({ cues: plan.cues, lyricWindows: plan.lyricWindows, fps: plan.fps, durationSec: plan.totalDuration, window: segment.window }).length > 0;
+ const graphPath = join(tmpdir(), `portos-music-video-segment-${jobId}.filtergraph`);
+ try {
+ const overlays = needsOverlay ? await renderTypographyOverlays({
+ jobId, cues: plan.cues, style: plan.style, width: plan.width, height: plan.height, fps: plan.fps, durationSec: plan.totalDuration,
+ lyrics: plan.lyricWindows.length ? { project: plan.project, windows: plan.lyricWindows } : null, window: segment.window, signal,
+ onProgress: (f) => onProgress?.(f * 0.5),
+ }) : [];
+ signal?.throwIfAborted();
+ const built = buildMusicVideoFfmpegArgs(plan.clips, plan.audioPath, outputPath, { ...plan.common, overlays, segment: segment.segment });
+ const encode = await filterGraphViaFile(plan.ffmpeg, ['-hide_banner', '-loglevel', 'error', ...built.args], graphPath);
+ const result = await runFfmpegProcess({ bin: plan.ffmpeg, args: encode.args, signal });
+ signal?.throwIfAborted();
+ if (!result.ok) throw new Error(`ffmpeg failed: ${result.reason}`);
+ onProgress?.(1);
+ } finally {
+ await rm(graphPath, { force: true }).catch(() => {});
+ await removeCompositionScratch(jobId).catch(() => {});
+ }
+}
+
+/** How many segments render at once: each may hold a browser and an encoder. */
+export const segmentConcurrency = ({ cores = availableParallelism() } = {}) => Math.max(1, Math.min(3, Math.floor(cores / 4)));
+
+/**
+ * Render every missing segment of `plan` (`concurrency` at a time). `onProgress`
+ * reports the share of frames ready, counting cached ones as done.
+ */
+export async function renderMissingSegments(plan, { signal, onProgress, concurrency = segmentConcurrency() } = {}) {
+ const all = [...plan.segments, ...(plan.outro ? [plan.outro] : [])];
+ const total = all.reduce((sum, s) => sum + s.frames, 0) || 1;
+ const done = new Map(all.map((s) => [s, existsSync(s.path) ? 1 : 0]));
+ const report = () => onProgress?.([...done].reduce((sum, [s, f]) => sum + s.frames * f, 0) / total);
+ report();
+ const queue = all.filter((s) => done.get(s) < 1);
+ const failures = [];
+ const worker = async () => {
+ while (queue.length && !signal?.aborted) {
+ const segment = queue.shift();
+ try {
+ await renderSegment(plan, segment, { signal, onProgress: (f) => { done.set(segment, f); report(); } });
+ done.set(segment, 1);
+ report();
+ } catch (err) {
+ if (signal?.aborted || err.code === 'CANCELED') throw err;
+ failures.push(err);
+ queue.length = 0;
+ }
+ }
+ };
+ await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, worker));
+ signal?.throwIfAborted();
+ if (failures.length) throw failures[0];
+ return queue.length;
+}
+
+/**
+ * Join the cached segments by stream copy and lay the song (with any sound
+ * bed) under them once, silence under the outro, into `outputPath`. Resolves
+ * `{ durationSec, songSec }`.
+ */
+export async function stitchSegments(plan, outputPath, { signal } = {}) {
+ const parts = [...plan.segments, ...(plan.outro ? [plan.outro] : [])].map((s) => s.path);
+ const missing = parts.filter((p) => !existsSync(p));
+ if (missing.length) throw new ServerError('A shot segment went missing before stitching — render again', { status: 409, code: 'SHOT_SEGMENT_MISSING' });
+ const listPath = `${outputPath}.segments.txt`;
+ const silent = `${outputPath}.silent.mp4`;
+ const songSec = plan.songSec;
+ const durationSec = songSec + (plan.outro ? plan.outro.frames / plan.fps : 0);
+ try {
+ await writeFile(listPath, parts.map((p) => `file '${p.replaceAll("'", "'\\''")}'`).join('\n'));
+ const joined = await runFfmpegProcess({ bin: plan.ffmpeg, signal, args: ['-hide_banner', '-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i', listPath,
+ '-c', 'copy', '-y', silent] });
+ signal?.throwIfAborted();
+ if (!joined.ok) throw new ServerError(`Joining shot segments failed: ${joined.reason}`, { status: 500, code: 'SHOT_SEGMENT_JOIN' });
+ const filters = [];
+ const inputs = ['-i', silent, '-i', plan.audioPath];
+ let master = '[1:a]';
+ if (plan.soundBed?.path) {
+ filters.push(`[1:a]${AUDIO_NORM}[master]`);
+ const bed = buildAudioBedMix({
+ beds: [{ assetPath: plan.soundBed.path, offsetSec: 0, durationSec: songSec, volume: plan.soundBed.volume }],
+ firstInputIdx: 2, mainLabel: '[master]', outLabel: '[mixa]', limitPeak: true,
+ });
+ inputs.push(...bed.inputs);
+ filters.push(...bed.filters);
+ master = '[mixa]';
+ }
+ filters.push(`${master}atrim=end=${songSec},asetpts=PTS-STARTPTS,aresample=48000,aformat=sample_fmts=fltp:channel_layouts=stereo,apad=whole_dur=${durationSec}[a]`);
+ const muxed = await runFfmpegProcess({ bin: plan.ffmpeg, signal, args: ['-hide_banner', '-loglevel', 'error', ...inputs,
+ '-filter_complex', filters.join(';'), '-map', '0:v', '-map', '[a]', '-c:v', 'copy', ...AAC_ENCODE_ARGS,
+ '-t', String(durationSec), '-movflags', '+faststart', '-y', outputPath] });
+ signal?.throwIfAborted();
+ if (!muxed.ok) throw new ServerError(`Laying the song under the shots failed: ${muxed.reason}`, { status: 500, code: 'SHOT_SEGMENT_MUX' });
+ return { durationSec, songSec };
+ } finally {
+ await rm(listPath, { force: true }).catch(() => {});
+ await rm(silent, { force: true }).catch(() => {});
+ }
+}
+
+/** Remove cached files the current plan no longer uses (older takes of each shot and look). */
+export async function pruneSegments(plan) {
+ const keep = new Set([...plan.segments, ...(plan.outro ? [plan.outro] : [])].flatMap((s) => [s.path, s.basePath]));
+ const names = await readdir(plan.dir).catch(() => []);
+ let removed = 0;
+ for (const name of names) {
+ const path = join(plan.dir, name);
+ if (keep.has(path) || [...inFlight.keys()].some((p) => path.startsWith(p))) continue;
+ if (name.endsWith('.partial.mp4')) continue;
+ await rm(path, { force: true }).then(() => { removed += 1; }, () => {});
+ }
+ return removed;
+}
+
+/** Remove a project's whole segment cache (the project was deleted). */
+export const removeProjectSegments = (projectId) => rm(shotSegmentDir(projectId), { recursive: true, force: true });
+
+/** Boot sweep: partial files from renders a restart interrupted. */
+export async function sweepPartialSegments() {
+ const root = join(PATHS.data, SHOT_SEGMENT_DIR);
+ for (const project of await readdir(root).catch(() => [])) {
+ for (const name of await readdir(join(root, project)).catch(() => [])) {
+ if (name.endsWith('.partial.mp4')) await rm(join(root, project, name), { force: true }).catch(() => {});
+ }
+ }
+}
+
+/**
+ * The shot panel's view of the cache: per shot, whether every section it plays
+ * is rendered and whether one is rendering now, plus the playable files.
+ */
+function segmentStatus(plan) {
+ const byShot = new Map();
+ for (const segment of plan.segments) {
+ const entry = byShot.get(segment.sceneId) || { sceneId: segment.sceneId, sections: 0, rendered: 0, rendering: false, startSec: segment.startSec, endSec: segment.endSec, files: [] };
+ entry.sections += 1;
+ if (segmentRendered(segment)) { entry.rendered += 1; entry.files.push(segment.path.slice(plan.dir.length + 1)); }
+ if (segmentRendering(segment)) entry.rendering = true;
+ entry.endSec = Math.max(entry.endSec, segment.endSec);
+ entry.look = !!segment.look;
+ byShot.set(segment.sceneId, entry);
+ }
+ const shots = [...byShot.values()].map(({ rendered, sections, ...rest }) => ({ ...rest, state: rendered === sections ? 'rendered' : 'stale' }));
+ const all = [...plan.segments, ...(plan.outro ? [plan.outro] : [])];
+ return {
+ shots,
+ outro: plan.outro ? { state: segmentRendered(plan.outro) ? 'rendered' : 'stale', rendering: segmentRendering(plan.outro) } : null,
+ ready: all.filter(segmentRendered).length,
+ total: all.length,
+ };
+}
+
+// One shot render per shot at a time: `${projectId}:${sceneId}` → { controller }.
+const shotJobs = new Map();
+const shotJobKey = (projectId, sceneId) => `${projectId}:${sceneId}`;
+
+const emitShotRender = (payload) => musicVideoEvents.emit('shot-render', payload);
+
+async function loadProjectScene(projectId, sceneId) {
+ const project = await getProject(projectId);
+ if (!project) throw new ServerError('Project not found', { status: 404, code: 'NOT_FOUND' });
+ if (!(project.scenes || []).some((scene) => scene.sceneId === sceneId)) throw new ServerError('Shot not found', { status: 404, code: 'NOT_FOUND' });
+ return project;
+}
+
+/**
+ * The shot panel's render state for every shot of a project. A project the
+ * final render would refuse reports that refusal (it names the shots and the
+ * fix) instead of shot states, since no segment can be placed until it plans.
+ */
+export async function getShotRenderStatus(projectId) {
+ const project = await getProject(projectId);
+ if (!project) throw new ServerError('Project not found', { status: 404, code: 'NOT_FOUND' });
+ if (project.composition?.mode !== 'composed') return { available: false, shots: [], outro: null, ready: 0, total: 0 };
+ const plan = await planShotSegments(project).catch((err) => {
+ if (!err?.status || err.status >= 500) throw err;
+ return { error: { message: err.message, code: err.code || null } };
+ });
+ if (plan.error) return { available: true, error: plan.error, shots: [], outro: null, ready: 0, total: 0 };
+ const status = segmentStatus(plan);
+ const running = new Set([...shotJobs.keys()].filter((key) => key.startsWith(`${projectId}:`)).map((key) => key.slice(projectId.length + 1)));
+ return { available: true, ...status, shots: status.shots.map((shot) => (running.has(shot.sceneId) ? { ...shot, rendering: true } : shot)) };
+}
+
+/**
+ * Render one shot's segments (every section of the cut it plays, base and
+ * film look) into the cache in the background. Resolves at once with the
+ * shot's state; progress and the outcome arrive as `shot-render` events.
+ */
+export async function startShotRender(projectId, sceneId) {
+ const permit = maintenance.admit('music-video', projectId, { continuation: true, parentKinds: ['media'] });
+ let handedOff = false;
+ try {
+ return await permit.run(async () => {
+ const key = shotJobKey(projectId, sceneId);
+ if (shotJobs.has(key)) return { sceneId, state: 'rendering' };
+ const project = await loadProjectScene(projectId, sceneId);
+ const plan = await planShotSegments(project);
+ const segments = plan.segments.filter((segment) => segment.sceneId === sceneId);
+ if (!segments.length) {
+ throw new ServerError(`${nameShots(project, [sceneId])}: not in the cut yet. Give it a clip, or a still with a start and end time`, { status: 422, code: 'SHOT_NOT_IN_CUT' });
+ }
+ if (segments.every(segmentRendered)) return { sceneId, state: 'rendered' };
+ if (shotJobs.has(key)) return { sceneId, state: 'rendering' };
+ const controller = new AbortController();
+ shotJobs.set(key, { controller });
+ handedOff = true;
+ const total = segments.reduce((sum, s) => sum + s.frames, 0) || 1;
+ let reported = -1;
+ const report = (fraction) => {
+ const pct = Math.floor(fraction * 20) * 5;
+ if (pct === reported) return;
+ reported = pct;
+ emitShotRender({ projectId, sceneId, state: 'rendering', progress: pct / 100 });
+ };
+ report(0);
+ const started = Date.now();
+ Promise.resolve().then(async () => {
+ let before = 0;
+ for (const segment of segments) {
+ await renderSegment(plan, segment, { signal: controller.signal, onProgress: (f) => report((before + segment.frames * f) / total) });
+ before += segment.frames;
+ }
+ }).then(() => {
+ console.log(`🎬 Music-video ${nameShots(project, [sceneId])} rendered [${projectId.slice(0, 8)}] in ${((Date.now() - started) / 1000).toFixed(1)}s`);
+ emitShotRender({ projectId, sceneId, state: 'rendered', progress: 1 });
+ }, (err) => {
+ const canceled = controller.signal.aborted || err?.code === 'CANCELED';
+ const log = canceled ? console.log : console.error;
+ log(`${canceled ? '🛑' : '❌'} Music-video shot render ${canceled ? 'cancelled' : 'failed'} [${projectId.slice(0, 8)}]: ${err.message}`);
+ emitShotRender({ projectId, sceneId, state: canceled ? 'canceled' : 'failed', error: canceled ? null : err.message });
+ }).finally(() => {
+ shotJobs.delete(key);
+ permit.finish();
+ });
+ return { sceneId, state: 'rendering' };
+ });
+ } finally {
+ if (!handedOff) permit.finish();
+ }
+}
+
+/** Stop a shot render; resolves whether one was running. */
+export function cancelShotRender(projectId, sceneId) {
+ const job = shotJobs.get(shotJobKey(projectId, sceneId));
+ if (!job) return false;
+ job.controller.abort(new Error('Render cancelled'));
+ return true;
+}
+
+/** A cached segment file of `projectId` by name, or null when the name is not one or it is gone. */
+export function shotSegmentFile(projectId, name) {
+ if (typeof name !== 'string' || !/^(outro-)?[0-9a-f]{32}(-[0-9a-f]{32})?\.mp4$/.test(name)) return null;
+ const path = join(shotSegmentDir(projectId), name);
+ return existsSync(path) ? path : null;
+}
diff --git a/server/services/musicVideo/shotSegments.test.js b/server/services/musicVideo/shotSegments.test.js
new file mode 100644
index 0000000000..46511173e8
--- /dev/null
+++ b/server/services/musicVideo/shotSegments.test.js
@@ -0,0 +1,113 @@
+/**
+ * Per-shot render segments, against real ffmpeg on tiny generated clips: the
+ * stitched segments are the picture the whole-film graph draws (cut effects
+ * and glitches that cross a cut included), a tweak to one shot re-renders only
+ * the segments it reaches, and a film look change keeps every base segment.
+ */
+import { describe, it, expect, vi, afterAll, beforeAll } from 'vitest';
+import { execFileSync } from 'child_process';
+import { existsSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
+
+const ROOT = () => lazyTempDataRoot('mv-shot-segments-test-');
+vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await importOriginal(), { dataRoot: ROOT }));
+
+const fixture = { clips: [], audioPath: null };
+vi.mock('./render.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return {
+ ...actual,
+ planMusicVideoRender: vi.fn(async () => ({
+ ffmpeg: 'ffmpeg', audioPath: fixture.audioPath, composed: true, clips: fixture.clips.map((c) => ({ ...c })),
+ audioDurationSec: 3, soundBed: null, aspect: null,
+ })),
+ };
+});
+
+const { planShotSegments, renderMissingSegments, stitchSegments } = await import('./shotSegments.js');
+const { buildMusicVideoFfmpegArgs } = await import('./render.js');
+
+afterAll(cleanupTempDataRoots);
+
+// Runs against the installed ffmpeg; skipped where none is on PATH (Linux CI).
+const hasFfmpeg = (() => { try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); execFileSync('ffprobe', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })();
+const run = (args) => execFileSync('ffmpeg', ['-hide_banner', '-loglevel', 'error', ...args]);
+const probe = (path, entries) => execFileSync('ffprobe', ['-v', 'error', '-count_frames', '-show_entries', entries, '-of', 'default=nw=1:nk=1', path]).toString().trim().split('\n');
+
+const project = (over = {}) => ({
+ id: 'mv-segments-test', name: 'Example', composition: { mode: 'composed' },
+ scenes: [{ sceneId: 'a', order: 0 }, { sceneId: 'b', order: 1 }, { sceneId: 'c', order: 2 }],
+ ...over,
+});
+const footage = (sceneId, path, over = {}) => ({
+ sceneId, videoPath: path, width: 320, height: 180, fps: 24, duration: 1, sourceSec: 2, inSec: 0, outSec: 1, loop: false, frameFit: 'fill', ...over,
+});
+
+let media;
+beforeAll(() => {
+ if (!hasFfmpeg) return;
+ media = join(ROOT(), 'fixture');
+ mkdirSync(media, { recursive: true });
+ for (const [name, src] of [['a', 'testsrc2'], ['b', 'smptebars'], ['c', 'rgbtestsrc']]) {
+ run(['-f', 'lavfi', '-i', `${src}=s=320x180:r=24`, '-t', '2', '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-y', join(media, `${name}.mp4`)]);
+ }
+ fixture.audioPath = join(media, 'song.wav');
+ run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=3', '-y', fixture.audioPath]);
+ fixture.clips = [
+ // A dissolves into B (B's footage is read over A's end) with a glitch on the cut.
+ footage('a', join(media, 'a.mp4'), { cutEffect: { enabled: true, style: 'dissolve', overlapSec: 0.5 }, transitionOut: 'vhs-glitch' }),
+ footage('b', join(media, 'b.mp4')),
+ footage('c', join(media, 'c.mp4')),
+ ];
+});
+
+describe.skipIf(!hasFfmpeg)('shot segments', () => {
+ it('stitch to the picture the whole-film graph draws, with the song under it', async () => {
+ const plan = await planShotSegments(project());
+ expect(plan.segments.map((s) => [s.sceneId, s.frames])).toEqual([['a', 24], ['b', 24], ['c', 24]]);
+ expect(await renderMissingSegments(plan, { concurrency: 2 })).toBe(0);
+ const out = join(ROOT(), 'stitched.mp4');
+ await stitchSegments(plan, out);
+ expect(probe(out, 'stream=codec_type').sort()).toEqual(['audio', 'video']);
+ expect(probe(out, 'stream=nb_read_frames')).toContain('72');
+
+ // The whole film through the one-graph path, then compared frame by frame.
+ const whole = join(ROOT(), 'whole.mp4');
+ const { args } = buildMusicVideoFfmpegArgs(fixture.clips, fixture.audioPath, whole, { audioDurationSec: 3, frameGrid: true });
+ // Encoded at the segments' preset, and both sides given the segments' BT.709
+ // tags: ffmpeg 7 converts between differently tagged inputs before
+ // comparing, which scores matching pixels around 27 dB.
+ run(args.filter((a, i) => a !== '-progress' && args[i - 1] !== '-progress').map((a, i, all) => (all[i - 1] === '-preset' ? 'medium' : a)));
+ const tags = 'setparams=range=tv:color_primaries=bt709:color_trc=bt709:colorspace=bt709';
+ const lavfi = `[0:v]${tags}[a];[1:v]${tags}[b];[a][b]psnr`;
+ const report = execFileSync('sh', ['-c', `ffmpeg -hide_banner -i '${out}' -i '${whole}' -lavfi '${lavfi}' -f null - 2>&1 | grep -o 'average:[0-9.inf]*'`]).toString();
+ const average = report.split(':')[1].trim();
+ expect(average === 'inf' || Number(average) > 35).toBe(true);
+ }, 60000);
+
+ it('re-renders only the segments a shot tweak reaches, and keeps bases through a look change', async () => {
+ const before = await planShotSegments(project());
+ // B played faster: B changes, and so does A, which shows B's footage in its dissolve.
+ fixture.clips[1] = { ...fixture.clips[1], rate: 1.5 };
+ const tweaked = await planShotSegments(project());
+ expect(tweaked.segments.map((s, i) => s.baseKey === before.segments[i].baseKey)).toEqual([false, false, true]);
+ expect(tweaked.segments.map((s) => existsSync(s.path))).toEqual([false, false, true]);
+
+ const look = { preset: 'vhs', grain: 0.3 };
+ const looked = await planShotSegments(project({ filmLook: look }));
+ expect(looked.segments.map((s, i) => s.baseKey === tweaked.segments[i].baseKey)).toEqual([true, true, true]);
+ expect(looked.segments.every((s) => s.lookKey && s.path !== s.basePath)).toBe(true);
+
+ // A shot with its own look keeps it when the video's look changes.
+ const pinned = { preset: 'polaroid', grain: 0.2 };
+ const scenes = project().scenes.map((s) => (s.sceneId === 'c' ? { ...s, filmLook: pinned } : s));
+ const a = await planShotSegments(project({ scenes, filmLook: look }));
+ const b = await planShotSegments(project({ scenes, filmLook: { preset: 'super8' } }));
+ expect(a.segments[2].path).toBe(b.segments[2].path);
+ expect(a.segments[0].path).not.toBe(b.segments[0].path);
+ // A shot set to a clean look skips the look pass under a looked video.
+ const clean = await planShotSegments(project({ scenes: scenes.map((s) => (s.sceneId === 'c' ? { ...s, filmLook: { preset: 'none' } } : s)), filmLook: look }));
+ expect(clean.segments[2].path).toBe(clean.segments[2].basePath);
+ }, 60000);
+});
diff --git a/server/services/musicVideo/statsOutro.js b/server/services/musicVideo/statsOutro.js
index a758568f56..792639f712 100644
--- a/server/services/musicVideo/statsOutro.js
+++ b/server/services/musicVideo/statsOutro.js
@@ -52,30 +52,22 @@ html,body{margin:0;width:${width}px;height:${height}px;overflow:hidden;backgroun
}
/**
- * Append the project's outro to the finished render at `path`, in place.
- * Resolves the seconds it added (0 when the outro is off; the file is then
- * untouched). Rejects on failure or when `signal` aborts, leaving `path` as
- * it was.
+ * Capture the project's outro alone, silent, into `outputPath` on a
+ * `width`×`height` frame at `fps`. Resolves its seconds (0 when the outro is
+ * off; nothing is written). `stats` (getMusicVideoOutroStats) may be passed in
+ * when the caller already read it. Rejects on failure or when `signal` aborts.
*/
-export async function appendStatsOutro({ project, path, jobId, signal, onProgress, withFilmLook = false }) {
+export async function renderStatsOutroClip({ project, outputPath, width, height, fps, jobId, signal, onProgress, withFilmLook = false, stats: knownStats = null }) {
const outro = normalizeStatsOutro(project?.statsOutro);
if (!outro?.enabled) return 0;
if (typeof jobId !== 'string' || !/^[A-Za-z0-9_-]+$/.test(jobId)) throw new Error('Invalid outro job id');
- const { findFfmpeg, runFfmpegProcess, probeVideoGeometry } = await import('../../lib/ffmpeg.js');
const { openComposition } = await import('../htmlComposition/browser.js');
const { encodeComposition } = await import('../htmlComposition/encode.js');
const { ensureImageThumbnail } = await import('../../lib/imageThumbnail.js');
- const ffmpeg = await findFfmpeg();
- if (!ffmpeg) throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });
- const geometry = await probeVideoGeometry(path);
- if (!geometry) throw new ServerError('The rendered video could not be measured for its outro', { status: 500, code: 'STATS_OUTRO_PROBE' });
- const { width, height, fps } = geometry;
const durationSec = outro.durationSec;
- const stats = await getMusicVideoOutroStats(project);
+ const stats = knownStats || await getMusicVideoOutroStats(project);
const directory = `${STATS_OUTRO_SCRATCH_DIR}/${jobId}`;
const scratch = join(PATHS.data, directory);
- const clip = `${path}.outro.mp4`;
- const joined = `${path}.outro-joined.mp4`;
let page;
try {
await rm(scratch, { recursive: true, force: true });
@@ -103,12 +95,38 @@ export async function appendStatsOutro({ project, path, jobId, signal, onProgres
const look = withFilmLook ? normalizeFilmLook(project?.filmLook) : null;
const initScripts = look && !isFilmLookNeutral(look) ? [filmLookRuntimeSource(look)] : [];
page = await openComposition(directory, { signal, ownedBrowser: true, initScripts });
- await encodeComposition(page, { width, height, fps, durationSec }, clip, {
- signal, onProgress: (fraction) => onProgress?.(fraction * 0.6),
- });
+ await encodeComposition(page, { width, height, fps, durationSec }, outputPath, { signal, onProgress });
page.check();
await page.close({ verify: true });
page = null;
+ return durationSec;
+ } finally {
+ if (page) await page.close().catch(() => {});
+ await rm(scratch, { recursive: true, force: true }).catch(() => {});
+ }
+}
+
+/**
+ * Append the project's outro to the finished render at `path`, in place.
+ * Resolves the seconds it added (0 when the outro is off; the file is then
+ * untouched). Rejects on failure or when `signal` aborts, leaving `path` as
+ * it was.
+ */
+export async function appendStatsOutro({ project, path, jobId, signal, onProgress, withFilmLook = false }) {
+ const outro = normalizeStatsOutro(project?.statsOutro);
+ if (!outro?.enabled) return 0;
+ if (typeof jobId !== 'string' || !/^[A-Za-z0-9_-]+$/.test(jobId)) throw new Error('Invalid outro job id');
+ const { findFfmpeg, runFfmpegProcess, probeVideoGeometry } = await import('../../lib/ffmpeg.js');
+ const ffmpeg = await findFfmpeg();
+ if (!ffmpeg) throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });
+ const geometry = await probeVideoGeometry(path);
+ if (!geometry) throw new ServerError('The rendered video could not be measured for its outro', { status: 500, code: 'STATS_OUTRO_PROBE' });
+ const { width, height, fps } = geometry;
+ const clip = `${path}.outro.mp4`;
+ const joined = `${path}.outro-joined.mp4`;
+ try {
+ const durationSec = await renderStatsOutroClip({ project, outputPath: clip, width, height, fps, jobId, signal, withFilmLook,
+ onProgress: (fraction) => onProgress?.(fraction * 0.6) });
signal?.throwIfAborted();
// Join the cut and the outro, with silence under the outro. Both pictures
// are brought to the cut's frame, rate and pixel format first.
@@ -131,8 +149,6 @@ export async function appendStatsOutro({ project, path, jobId, signal, onProgres
console.log(`🎞️ Music-video outro [${jobId.slice(0, 8)}]: ${durationSec}s cutting room floor after the song`);
return durationSec;
} finally {
- if (page) await page.close().catch(() => {});
- await rm(scratch, { recursive: true, force: true }).catch(() => {});
await rm(clip, { force: true }).catch(() => {});
await rm(joined, { force: true }).catch(() => {});
}
diff --git a/server/services/socket.js b/server/services/socket.js
index 82db6f2b34..5dfc573485 100644
--- a/server/services/socket.js
+++ b/server/services/socket.js
@@ -333,6 +333,8 @@ const SIMPLE_BRIDGES = [
{ emitter: musicVideoEvents, event: 'publish-draft', channel: 'music-video:publish-draft' },
// The release cover art was composed, or its source image was queued or failed.
{ emitter: musicVideoEvents, event: 'cover-art', channel: 'music-video:cover-art' },
+ // A shot's cached render segment rendering, rendered, or failed (shotSegments.js).
+ { emitter: musicVideoEvents, event: 'shot-render', channel: 'music-video:shot-render' },
];
let forwardingRegistered = false;