Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Fade, flash or glitch a still shot into its end frame (#11349)

[._.]/38604bd84180
diff --git a/client/src/components/musicVideo/ImageEffectStill.jsx b/client/src/components/musicVideo/ImageEffectStill.jsx
index c59064749b..936f25b642 100644
--- a/client/src/components/musicVideo/ImageEffectStill.jsx
+++ b/client/src/components/musicVideo/ImageEffectStill.jsx
@@ -2,48 +2,51 @@ import { useCallback, useEffect, useRef } from 'react';
import { IMAGE_EFFECT_FOG_COLOR, imageEffectState } from '../../lib/musicVideoImageEffects.js';
/**
- * A still shot's image effect in the storyboard animatic: the still, cover-fitted
- * as the final render crops it, with the effect's flash and emulsion laid over it
- * and its contrast and colour filtered, all from the same curves the render uses
- * (`imageEffectState`). Paused it paints the frame at `time`; playing it repaints
- * every animation frame from the song's clock, writing styles directly so the
- * board does not re-render per frame.
+ * A still shot's image effect, as the final render plays it: the still (and,
+ * for an end-frame effect, the end frame over it), cover-fitted as the render
+ * crops them, with the effect's flash and emulsion laid over and its contrast
+ * and colour filtered, all from the same curves the render uses
+ * (`imageEffectState`). Stopped it paints the moment at `time` (song seconds);
+ * while `playing` it repaints every animation frame from `getTime()`, writing
+ * styles directly so its parent does not re-render per frame.
+ *
+ * The picture on top carries `data-shown`, so a glitch painted over the
+ * frame (TransitionGlitch) reads the picture that is showing.
*/
-export default function ImageEffectStill({ src, effect, startSec, audioRef, time }) {
+export default function ImageEffectStill({ src, endSrc = null, effect, startSec, time, playing = false, getTime = null, fps = 24 }) {
const imgRef = useRef(null);
+ const endRef = useRef(null);
const fogRef = useRef(null);
const flashRef = useRef(null);
const paint = useCallback((t) => {
- const state = imageEffectState(effect, t - startSec);
- if (imgRef.current) imgRef.current.style.filter = `contrast(${state.contrast}) saturate(${state.saturation})`;
+ const state = imageEffectState(effect, t - startSec, { fps });
+ if (imgRef.current) {
+ imgRef.current.style.filter = `contrast(${state.contrast}) saturate(${state.saturation})`;
+ imgRef.current.dataset.shown = state.end < 1 ? 'true' : 'false';
+ }
+ if (endRef.current) {
+ endRef.current.style.opacity = String(state.end);
+ endRef.current.dataset.shown = state.end >= 1 ? 'true' : 'false';
+ }
if (fogRef.current) fogRef.current.style.opacity = String(state.fog);
if (flashRef.current) flashRef.current.style.opacity = String(state.flash);
- }, [effect, startSec]);
+ }, [effect, startSec, fps]);
useEffect(() => { paint(time); }, [time, paint]);
useEffect(() => {
- const audio = audioRef.current;
- if (!audio) return undefined;
+ if (!playing || !getTime) return undefined;
let handle = 0;
const tick = () => {
- paint(audio.currentTime || 0);
+ paint(getTime());
handle = requestAnimationFrame(tick);
};
- const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };
- const stop = () => { cancelAnimationFrame(handle); paint(audio.currentTime || 0); };
- audio.addEventListener('play', start);
- audio.addEventListener('pause', stop);
- audio.addEventListener('ended', stop);
- if (!audio.paused) start();
- return () => {
- cancelAnimationFrame(handle);
- audio.removeEventListener('play', start);
- audio.removeEventListener('pause', stop);
- audio.removeEventListener('ended', stop);
- };
- }, [audioRef, paint]);
+ handle = requestAnimationFrame(tick);
+ return () => cancelAnimationFrame(handle);
+ }, [playing, getTime, paint]);
return (
<div className="relative h-full w-full" data-testid="animatic-image-effect">
- <img ref={imgRef} src={src} alt="" className="h-full w-full object-cover" />
+ {/* The start frame comes first, so a plain `img` lookup (Save frame) still finds the shot's still. */}
+ <img ref={imgRef} src={src} alt="" data-shown="true" className="h-full w-full object-cover" />
+ {endSrc && <img ref={endRef} src={endSrc} alt="" data-shown="false" className="pointer-events-none absolute inset-0 h-full w-full object-cover" style={{ opacity: 0 }} />}
<div ref={fogRef} className="pointer-events-none absolute inset-0" style={{ background: IMAGE_EFFECT_FOG_COLOR, opacity: 0 }} />
<div ref={flashRef} className="pointer-events-none absolute inset-0" style={{ background: '#fff', opacity: 0 }} />
</div>
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index b0f5e7d12b..803b7ecdc9 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -36,6 +36,8 @@ import SceneRenderInputs, { shotCardRenderRequest, shotCardRequestProblem } from
import SceneFrameFit, { sceneFrameFit } from './SceneFrameFit.jsx';
import SceneTransitions from './SceneTransitions.jsx';
import SceneImageEffect from './SceneImageEffect.jsx';
+import ScenePicturePreview from './ScenePicturePreview.jsx';
+import { isEndFrameImageEffect } from '../../lib/musicVideoImageEffects.js';
// The two timeline-bound scene fields rendered as identical number inputs.
const SCENE_TIME_FIELDS = [['Start', 'startSec'], ['End', 'endSec']];
@@ -208,8 +210,11 @@ export default function SceneCard({
onGenerateVideo?.(scene, { ...(renderOverrides ? { target: renderOverrides } : {}), card });
};
const renderCost = performance ? performanceCost : timing.falCost;
- // A footage shot's start and end frames show as a pair; the end frame is optional.
- const endFrames = Boolean(onGenerateEndFrame) && layer === 'footage';
+ // A composed still shows its picture (the still, or the still playing its image effect) in place of a clip.
+ const stillPicture = layered && layer === 'still';
+ // A footage shot's start and end frames show as a pair, the end frame optional; an effect
+ // from the start frame to the end frame needs both.
+ const endFrames = Boolean(onGenerateEndFrame) && (layer === 'footage' || (stillPicture && isEndFrameImageEffect(scene.imageEffect)));
const shotModeValue = performance ? (scene.performanceStyle === 'music' ? 'music' : 'performance') : 'cutaway';
// The collapsed Shot settings row says what is set, so it rarely needs opening.
const settingsSummary = [
@@ -468,6 +473,8 @@ export default function SceneCard({
<SceneImageEffect fieldId={fieldId} scene={scene} onChange={applyPatch}
disabled={shotLock?.clip === 'locked'} disabledReason={shotLock?.clip === 'locked' ? 'Video locked. Unlock the video to change it.' : null} />
)}
+ {stillPicture ? <ScenePicturePreview scene={scene} /> : (
+ <>
{scene.videoHistoryId && performance && (
<PerformanceEvidence key={scene.videoHistoryId} clipSrc={clipSrc} instruction={instruction}
temporal={performanceReview?.shot} excerptStartSec={performanceReview?.excerptStartSec || 0}
@@ -582,6 +589,8 @@ export default function SceneCard({
onSelect={(take) => onSelectTake?.(scene, take)}
onReview={(take, review) => onReviewTake?.(scene, take, review)}
onOpenPreview={onOpenPreview} />
+ </>
+ )}
{/* The render's settings sit with the render: what it covers and costs, then timing, shot mode, layer and camera. */}
{shotSettings}
</section>
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index a9b0bfe5ac..82a8b8c83d 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -499,3 +499,27 @@ describe('image effect picker', () => {
expect(screen.queryByLabelText('Picture')).toBeNull();
});
});
+
+// Uniquely catches an effect shot still showing its old clip and video controls, or an end-frame effect saving the wrong fields.
+describe('a still picture in the clip column', () => {
+ it('previews the still playing its effect instead of the old clip, and an end-frame effect asks for its end frame', () => {
+ const onSave = vi.fn();
+ function Harness() {
+ const [scene, setScene] = useState({ sceneId: 's1', startSec: 0, endSec: 3, referenceImageId: 'start.png', videoHistoryId: 'clip-1', takes: [] });
+ return <SceneCard scene={scene} index={0} layered onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
+ }
+ const { container } = render(<Harness />);
+ expect(container.querySelector('section[aria-label="Render"] video')).toBeTruthy();
+ fireEvent.change(screen.getByLabelText('Picture'), { target: { value: 'flash-develop' } });
+ expect(screen.getByTestId('scene-picture-preview')).toBeTruthy();
+ expect(container.querySelector('section[aria-label="Render"] video')).toBeNull();
+ expect(screen.queryByRole('button', { name: /^Render/ })).toBeNull();
+ fireEvent.change(screen.getByLabelText('Picture'), { target: { value: 'end-glitch' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { visualLayer: 'still', imageEffect: 'end-glitch', imageEffectSec: null });
+ expect(screen.getByText(/Needs an end frame/)).toBeTruthy();
+ const change = screen.getByLabelText('Changes at');
+ expect([...change.options].map((o) => o.textContent)).toEqual(['Middle', '0.5s', '1s', '1.5s', '2s']);
+ fireEvent.change(change, { target: { value: '1' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { imageEffectSec: 1 });
+ });
+});
diff --git a/client/src/components/musicVideo/SceneImageEffect.jsx b/client/src/components/musicVideo/SceneImageEffect.jsx
index 7833ef1510..c37fd7785a 100644
--- a/client/src/components/musicVideo/SceneImageEffect.jsx
+++ b/client/src/components/musicVideo/SceneImageEffect.jsx
@@ -1,5 +1,8 @@
import { safeReadStorage, safeWriteStorage } from '../../lib/safeStorage.js';
-import { DEFAULT_IMAGE_EFFECT_SEC, IMAGE_EFFECT_DEVELOP_SECS, MUSIC_VIDEO_IMAGE_EFFECTS, MUSIC_VIDEO_IMAGE_EFFECT_LABELS } from '../../lib/musicVideoImageEffects.js';
+import {
+ DEFAULT_IMAGE_EFFECT_SEC, IMAGE_EFFECT_CHANGE_SECS, IMAGE_EFFECT_DEVELOP_SECS, MUSIC_VIDEO_IMAGE_EFFECTS, MUSIC_VIDEO_IMAGE_EFFECT_LABELS,
+ isEndFrameImageEffect,
+} from '../../lib/musicVideoImageEffects.js';
// The develop length picked last on this device, for the next shot that takes an effect.
const LAST_SEC_KEY = 'portos-mv-image-effect-sec';
@@ -7,22 +10,30 @@ const lastDevelopSec = () => {
const sec = Number(safeReadStorage(LAST_SEC_KEY));
return IMAGE_EFFECT_DEVELOP_SECS.includes(sec) ? sec : DEFAULT_IMAGE_EFFECT_SEC;
};
+const SELECT = 'min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50';
/**
* Where a shot's picture comes from in a composed render: a rendered clip, its
- * still, or its still playing a code-drawn image effect (a camera flash that
- * develops like an instant photo), which needs no model. An effect's develop
- * length rides beside it. The effect plays inside the shot's span and adds no time.
+ * still, or its still playing a code-drawn image effect, which needs no model:
+ * a camera flash that develops like an instant photo (with its develop length),
+ * or a fade, flash or glitch from the start frame to the end frame (with when
+ * it changes, the middle of the shot by default). The effect plays inside the
+ * shot's span and adds no time.
*/
export default function SceneImageEffect({ fieldId, scene, onChange, disabled = false, disabledReason = null }) {
const effect = scene.visualLayer === 'still' && MUSIC_VIDEO_IMAGE_EFFECTS.includes(scene.imageEffect) ? scene.imageEffect : null;
+ const toEnd = isEndFrameImageEffect(effect);
const value = effect || (scene.visualLayer === 'still' ? 'still' : 'clip');
const developSec = IMAGE_EFFECT_DEVELOP_SECS.includes(scene.imageEffectSec) ? scene.imageEffectSec : DEFAULT_IMAGE_EFFECT_SEC;
const spanSec = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' ? scene.endSec - scene.startSec : null;
+ // Change points inside the shot; a saved one past a shortened shot reads Middle, as it plays.
+ const changeSecs = IMAGE_EFFECT_CHANGE_SECS.filter((sec) => spanSec == null || sec < spanSec);
const pick = (next) => {
if (next === 'clip') onChange({ visualLayer: 'footage', ...(scene.imageEffect ? { imageEffect: null } : {}) });
else if (next === 'still') onChange({ visualLayer: 'still', ...(scene.imageEffect ? { imageEffect: null } : {}) });
- else onChange({ visualLayer: 'still', imageEffect: next, imageEffectSec: scene.imageEffectSec || lastDevelopSec() });
+ // An end-frame effect changes mid-shot until a time is picked.
+ else if (isEndFrameImageEffect(next)) onChange({ visualLayer: 'still', imageEffect: next, ...(toEnd ? {} : { imageEffectSec: null }) });
+ else onChange({ visualLayer: 'still', imageEffect: next, imageEffectSec: toEnd || !scene.imageEffectSec ? lastDevelopSec() : scene.imageEffectSec });
};
const pickSec = (sec) => {
safeWriteStorage(LAST_SEC_KEY, String(sec));
@@ -34,28 +45,39 @@ export default function SceneImageEffect({ fieldId, scene, onChange, disabled =
<span className="flex min-w-0 items-center gap-1">
<label htmlFor={fieldId('picture')} className={effect ? 'text-port-accent' : 'text-port-text-muted'}>Picture</label>
<select id={fieldId('picture')} value={value} disabled={disabled} title={disabledReason || undefined}
- onChange={(e) => pick(e.target.value)}
- className="min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50">
+ onChange={(e) => pick(e.target.value)} className={SELECT}>
<option value="clip">Video clip</option>
<option value="still">Still image</option>
{MUSIC_VIDEO_IMAGE_EFFECTS.map((id) => <option key={id} value={id}>{MUSIC_VIDEO_IMAGE_EFFECT_LABELS[id]}</option>)}
</select>
</span>
- {effect && (
+ {effect && !toEnd && (
<span className="flex min-w-0 items-center gap-1">
<label htmlFor={fieldId('develop')} className="text-port-text-muted">Develops in</label>
- <select id={fieldId('develop')} value={developSec} disabled={disabled} title={disabledReason || undefined} onChange={(e) => pickSec(Number(e.target.value))}
- className="min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50">
+ <select id={fieldId('develop')} value={developSec} disabled={disabled} title={disabledReason || undefined}
+ onChange={(e) => pickSec(Number(e.target.value))} className={SELECT}>
{IMAGE_EFFECT_DEVELOP_SECS.map((sec) => <option key={sec} value={sec}>{sec}s</option>)}
</select>
</span>
)}
+ {toEnd && (
+ <span className="flex min-w-0 items-center gap-1">
+ <label htmlFor={fieldId('change')} className="text-port-text-muted">Changes at</label>
+ <select id={fieldId('change')} value={changeSecs.includes(scene.imageEffectSec) ? scene.imageEffectSec : ''} disabled={disabled}
+ title={disabledReason || undefined} onChange={(e) => onChange({ imageEffectSec: e.target.value ? Number(e.target.value) : null })} className={SELECT}>
+ <option value="">Middle</option>
+ {changeSecs.map((sec) => <option key={sec} value={sec}>{sec}s</option>)}
+ </select>
+ </span>
+ )}
</div>
{effect && (
- <p className="text-[11px] text-port-text-muted">
- {spanSec != null && developSec > spanSec
- ? `The shot ends at ${spanSec.toFixed(1)}s, before the photo finishes developing.`
- : 'No video render: the still plays the effect.'}
+ <p className={`text-[11px] ${toEnd && !scene.endImageId ? 'text-port-warning' : 'text-port-text-muted'}`}>
+ {toEnd && !scene.endImageId
+ ? 'Needs an end frame: make or pick one beside the start frame. Until then the shot plays its still.'
+ : !toEnd && spanSec != null && developSec > spanSec
+ ? `The shot ends at ${spanSec.toFixed(1)}s, before the photo finishes developing.`
+ : 'No video render: the still plays the effect.'}
</p>
)}
</div>
diff --git a/client/src/components/musicVideo/ScenePicturePreview.jsx b/client/src/components/musicVideo/ScenePicturePreview.jsx
new file mode 100644
index 0000000000..e3a329585b
--- /dev/null
+++ b/client/src/components/musicVideo/ScenePicturePreview.jsx
@@ -0,0 +1,51 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { Pause, Play } from 'lucide-react';
+import ImageEffectStill from './ImageEffectStill.jsx';
+import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
+
+// A shot with no span still previews its effect over this long.
+const FALLBACK_SPAN_SEC = 4;
+
+/**
+ * What a still shot shows in a composed render, in the shot card's Clip
+ * column instead of its old clip: the still alone, or the still playing its
+ * image effect, looped over the shot's length with a play/pause toggle.
+ */
+export default function ScenePicturePreview({ scene }) {
+ const effect = sceneImageEffect(scene, { layered: true });
+ const spanSec = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' && scene.endSec > scene.startSec
+ ? scene.endSec - scene.startSec : FALLBACK_SPAN_SEC;
+ const [playing, setPlaying] = useState(true);
+ const [pausedAt, setPausedAt] = useState(0);
+ const startedAt = useRef(0);
+ // Time into the shot, looping over its span.
+ const getTime = useCallback(() => ((performance.now() - startedAt.current) / 1000) % spanSec, [spanSec]);
+ useEffect(() => {
+ if (playing) startedAt.current = performance.now() - pausedAt * 1000;
+ // Restart the loop when the effect or its timing changes.
+ }, [playing, pausedAt, effect?.effect, effect?.developSec, effect?.changeSec]);
+ const toggle = () => {
+ if (playing) setPausedAt(getTime());
+ setPlaying((on) => !on);
+ };
+ if (!scene.referenceImageId) {
+ return <div className="flex aspect-video w-full items-center justify-center rounded border border-dashed border-port-border text-xs text-port-text-muted">No still yet</div>;
+ }
+ const src = `/data/images/${scene.referenceImageId}`;
+ return (
+ <div className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black" data-testid="scene-picture-preview">
+ {effect ? (
+ <>
+ <ImageEffectStill src={src} endSrc={scene.endImageId ? `/data/images/${scene.endImageId}` : null} effect={effect}
+ startSec={0} time={pausedAt} playing={playing} getTime={getTime} />
+ <button type="button" onClick={toggle} aria-label={playing ? 'Pause effect preview' : 'Play effect preview'}
+ className="port-media-overlay port-media-overlay-item absolute bottom-1 left-1 flex min-h-[44px] min-w-[44px] items-center justify-center rounded sm:min-h-0 sm:min-w-0 sm:p-1.5">
+ {playing ? <Pause size={14} /> : <Play size={14} />}
+ </button>
+ </>
+ ) : (
+ <img src={src} alt="" className="h-full w-full object-cover" />
+ )}
+ </div>
+ );
+}
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index 6f8ee15cf3..69005af85d 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -256,6 +256,7 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
// While the song plays the look is the light playback version; paused, the exact render filter.
const [playing, setPlaying] = useState(false);
const lookAvailable = !isFilmLookNeutral(project.filmLook);
+ const songTime = useCallback(() => audioRef.current?.currentTime || 0, []);
const toggleLook = () => setLookOn((on) => { safeWriteStorage(ANIMATIC_LOOK_KEY, on ? 'off' : 'on'); return !on; });
const scenes = project.scenes || [];
const lyricsAvailable = hasTimedLyrics(project);
@@ -308,7 +309,8 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
<FilmLookFrame look={lookOn ? project.filmLook : null} frame={Math.round(time * ANIMATIC_FPS)} playing={playing} className="h-full w-full">
{shot?.referenceImageId && shotEffect ? (
<ImageEffectStill key={shot.sceneId} src={`/data/images/${shot.referenceImageId}`} effect={shotEffect}
- startSec={shot.startSec} audioRef={audioRef} time={time} />
+ endSrc={shot.endImageId ? `/data/images/${shot.endImageId}` : null}
+ startSec={shot.startSec} time={time} playing={playing} getTime={songTime} fps={ANIMATIC_FPS} />
) : shot?.referenceImageId ? (
// Behind a VHS-edged clip the still stands in, blurred and faded, for the render's blur of the clip.
<img src={`/data/images/${shot.referenceImageId}`} alt=""
@@ -323,7 +325,7 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
<ShotClip key={`${scene.sceneId}:${scene.videoHistoryId}`} scene={scene} audioRef={audioRef} time={time} current={scene === shot} />
))}
{/* The shots' VHS glitches on their cuts, on the frames the final render repaints. */}
- <TransitionGlitch scenes={scenes} figureRef={figureRef} audioRef={audioRef} time={time}
+ <TransitionGlitch scenes={scenes} figureRef={figureRef} audioRef={audioRef} time={time} layered={layered}
endSec={Number(project.audioAnalysis?.durationSec) || null} fps={ANIMATIC_FPS} />
{overlay}
</FilmLookFrame>
diff --git a/client/src/components/musicVideo/TransitionGlitch.jsx b/client/src/components/musicVideo/TransitionGlitch.jsx
index c366a296aa..0d7d531aff 100644
--- a/client/src/components/musicVideo/TransitionGlitch.jsx
+++ b/client/src/components/musicVideo/TransitionGlitch.jsx
@@ -1,5 +1,7 @@
import { useCallback, useEffect, useMemo, useRef } from 'react';
-import { sceneTransitionSections, transitionFrameAt, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';
+import {
+ mergeGlitchFrames, sceneImageEffectGlitchFrames, sceneTransitionSections, transitionFrameAt, transitionGlitchFrames,
+} from '../../lib/musicVideoTransitions.js';
// The canvas paints at most this wide; a glitch lasts a few frames, so detail past it is never seen.
const MAX_CANVAS_PX = 960;
@@ -95,7 +97,7 @@ function paintGlitch(canvas, figure, params, frame) {
const base = cur.getContext('2d');
base.fillStyle = '#000';
base.fillRect(0, 0, width, height);
- const still = figure?.querySelector('img');
+ const still = figure?.querySelector('img[data-shown="true"]') || figure?.querySelector('img');
const clip = figure?.querySelector('video[data-ready="true"]:not([aria-hidden="true"])');
const stillReady = !!still?.complete && still.naturalWidth > 0;
const clipReady = !!clip && clip.readyState >= 2;
@@ -204,9 +206,11 @@ function paintGlitch(canvas, figure, params, frame) {
* paints the glitch of what is on screen; every other frame it stays hidden.
* Below the lyric layer, as the render keeps its typography clean.
*/
-export default function TransitionGlitch({ scenes, figureRef, audioRef, time, endSec = null, fps }) {
+export default function TransitionGlitch({ scenes, figureRef, audioRef, time, endSec = null, fps, layered = false }) {
const canvasRef = useRef(null);
- const glitches = useMemo(() => transitionGlitchFrames(sceneTransitionSections(scenes, fps, { endSec })), [scenes, fps, endSec]);
+ // A still's glitch to its end frame (an image effect) paints the same way, on the frames the render repaints.
+ const glitches = useMemo(() => mergeGlitchFrames(transitionGlitchFrames(sceneTransitionSections(scenes, fps, { endSec })),
+ sceneImageEffectGlitchFrames(scenes, fps, { layered, endSec })), [scenes, fps, endSec, layered]);
const paint = useCallback((t) => {
const canvas = canvasRef.current;
if (!canvas) return;
diff --git a/client/src/lib/musicVideoImageEffects.js b/client/src/lib/musicVideoImageEffects.js
index 923e8a7d4c..b2b28cefd5 100644
--- a/client/src/lib/musicVideoImageEffects.js
+++ b/client/src/lib/musicVideoImageEffects.js
@@ -4,11 +4,14 @@
*/
export {
DEFAULT_IMAGE_EFFECT_SEC,
+ END_FRAME_IMAGE_EFFECTS,
+ IMAGE_EFFECT_CHANGE_SECS,
IMAGE_EFFECT_DEVELOP_SECS,
IMAGE_EFFECT_FOG_COLOR,
IMAGE_EFFECT_SEC_RANGE,
MUSIC_VIDEO_IMAGE_EFFECTS,
MUSIC_VIDEO_IMAGE_EFFECT_LABELS,
imageEffectState,
+ isEndFrameImageEffect,
sceneImageEffect,
} from '../../../server/lib/musicVideoImageEffects.js';
diff --git a/client/src/lib/musicVideoTransitions.js b/client/src/lib/musicVideoTransitions.js
index b5616b1d78..f96dc2c69a 100644
--- a/client/src/lib/musicVideoTransitions.js
+++ b/client/src/lib/musicVideoTransitions.js
@@ -7,6 +7,8 @@ export {
MUSIC_VIDEO_TRANSITIONS,
MUSIC_VIDEO_TRANSITION_LABELS,
MUSIC_VIDEO_TRANSITION_STYLES,
+ mergeGlitchFrames,
+ sceneImageEffectGlitchFrames,
sceneTransitions,
sceneTransitionSections,
transitionFrameAt,
diff --git a/server/lib/README.md b/server/lib/README.md
index 03601e862d..7d7cb0e0cd 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -562,8 +562,8 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `chiptuneRender.js` | Deterministic chiptune score → mono PCM → 16-bit WAV buffer (pure Node, no audio deps); `renderScoreToPcm` / `pcmToWavBuffer` / `renderScoreToWav`. |
| `chiptuneScore.js` | Chiptune score contract (#2911): `chiptuneScoreSchema` (Zod) + `sanitizeChiptuneScore`, `buildScoreEvents` — the pattern/order → absolute-time flatten mirrored by `client/src/lib/chiptunePlayback.js`. |
| `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). |
-| `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`, first `flash-develop`: a camera flash, then the picture develops like an instant photo over `imageEffectSec`). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe. |
-| `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections` and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Browser-safe; the storyboard animatic paints the same frames. |
+| `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`: `flash-develop`, a camera flash then the picture develops like an instant photo over `imageEffectSec`; or a fade, flash or glitch from the start frame to the end frame). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe and import-free: document videos ship it as is. |
+| `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections`, `imageEffectGlitchFrames`/`mergeGlitchFrames` (end-frame glitch effects in the same shape) and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Browser-safe; the storyboard animatic paints the same frames. |
| `pitchMath.js` | Dependency-free scientific-pitch math shared by the LLM-authored music formats on both runtimes: `pitchToMidi("C4")` → 60 (null when unparseable) and `midiToFreq(69)` → 440. |
| `wavAudioFile.js` | `writeWavAudioFile(wav, dir, basename)` — land a synthesized WAV Buffer as `<basename>.ogg` via the system ffmpeg, falling back to `<basename>.wav` (and dropping a stale `.ogg`) when ffmpeg is missing or the encode fails; resolves to the filename used. Shared by the chiptune and drawn-waveform renderers and the code-engine take upload. `wavDurationMs(buffer)` reads a PCM WAV's duration from its RIFF header (0 for anything that is not a parseable WAV). `measureWavAudio(buffer)` decodes PCM 16/24/32-bit or float 32/64 (EXTENSIBLE included) into `{ channels, sampleRate, frames, durationMs, peak, rms, nonFinite }` so native renders are judged by their samples (null when undecodable). |
| `superColliderRuntime.js` | Managed SuperCollider runtime contract (#9412), pure: image/runtime/policy version constants, `buildSuperColliderRunArgs` (the one contained `docker run` for untrusted `sclang` source: no network/caps, read-only root, limits, ro input, private output), `buildSuperColliderBuildArgs`, the shipped synthetic probe, `evaluateSuperColliderStatus` (docker-missing … ready, with stale image/evidence invalidation), and the render contract (#9413): `SUPERCOLLIDER_RENDER` bounds, the trusted NRT `SUPERCOLLIDER_RENDER_WRAPPER_SOURCE`, `classifySuperColliderRenderLog` (syntax/source/pattern/score/synthesis failures) and `parseDockerContainerList` for orphan sweeps. I/O lives in `services/superColliderRuntime.js`. |
diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.js
index 61e9d6ade3..2ade85439b 100644
--- a/server/lib/musicVideoDependencies.js
+++ b/server/lib/musicVideoDependencies.js
@@ -1,5 +1,6 @@
/** Versioned Music Video provenance. Only pointers/checksums live on project rows. */
import { canonicalSnapshotChecksum } from './snapshotChecksum.js';
+import { isEndFrameImageEffect } from './musicVideoImageEffects.js';
export const MUSIC_VIDEO_DEPENDENCY_VERSION = 1;
// `endPlate` (#11173): the style-break end frame a fal clip was generated to end on.
@@ -23,6 +24,8 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s
.concat(['transitionIn', 'transitionOut'].filter((key) => scene[key]).map((key) => [key, scene[key]]))
// An image effect over a still changes the render; a still without one keeps its evidence.
.concat(scene.imageEffect ? [['imageEffect', scene.imageEffect], ['imageEffectSec', scene.imageEffectSec ?? null]] : [])
+ // An effect from the start frame to the end frame renders the end frame too.
+ .concat(scene.imageEffect && isEndFrameImageEffect(scene.imageEffect) ? [['endImageId', scene.endImageId ?? null]] : [])
// Only a music-driven performance joins it, so lip-sync evidence stays current.
.concat(scene.performanceStyle === 'music' ? [['performanceStyle', 'music']] : []))) });
const snapshot = (references) => ({ version: MUSIC_VIDEO_DEPENDENCY_VERSION, references });
diff --git a/server/lib/musicVideoImageEffects.js b/server/lib/musicVideoImageEffects.js
index e2050fd606..5fb01bc700 100644
--- a/server/lib/musicVideoImageEffects.js
+++ b/server/lib/musicVideoImageEffects.js
@@ -12,6 +12,13 @@
* for the rest of the shot. The effect never adds time: it plays inside the
* shot's own span, and a develop longer than the shot ends the shot part-way.
*
+ * Three effects play from the shot's start frame to its end frame
+ * (`endImageId`), changing `imageEffectSec` seconds in (the middle of the shot
+ * when unset): "Fade to end frame" dissolves over FADE_SEC, "Flash to end
+ * frame" cuts under a white camera flash, and "Glitch to end frame" cuts under
+ * a VHS glitch (the cut transitions' own `vhs-glitch` frames). Without an end
+ * frame the shot plays its still.
+ *
* One definition serves both players: every value is a pure function of the
* time into the shot (`imageEffectState`), the storyboard animatic paints it
* with CSS over the still, and the final render turns the same curves into
@@ -19,12 +26,25 @@
* square of a linear ramp so the two read the same frame for frame.
*/
-export const MUSIC_VIDEO_IMAGE_EFFECTS = Object.freeze(['flash-develop']);
+// No imports: document videos ship this file as is (compositionDocument.js), so it
+// stays self-contained. Glitch frames for `end-glitch` live in musicVideoTransitions.js.
+
+export const MUSIC_VIDEO_IMAGE_EFFECTS = Object.freeze(['flash-develop', 'end-fade', 'end-flash', 'end-glitch']);
export const MUSIC_VIDEO_IMAGE_EFFECT_LABELS = Object.freeze({
'flash-develop': 'Camera flash develop',
+ 'end-fade': 'Fade to end frame',
+ 'end-flash': 'Flash to end frame',
+ 'end-glitch': 'Glitch to end frame',
});
+/** Effects that play from the start frame to the end frame. */
+export const END_FRAME_IMAGE_EFFECTS = Object.freeze(['end-fade', 'end-flash', 'end-glitch']);
+export const isEndFrameImageEffect = (effect) => END_FRAME_IMAGE_EFFECTS.includes(effect);
+
+/** When an end-frame effect changes, in seconds into the shot; unset is the middle. */
+export const IMAGE_EFFECT_CHANGE_SECS = Object.freeze([0.5, 1, 1.5, 2, 3, 4, 6]);
+
/** The develop lengths the shot card offers, in seconds. */
export const IMAGE_EFFECT_DEVELOP_SECS = Object.freeze([1, 2, 3, 4, 6, 8]);
export const DEFAULT_IMAGE_EFFECT_SEC = 3;
@@ -37,37 +57,63 @@ export const IMAGE_EFFECT_FOG_COLOR = '#4a5452';
const FOG_SHARE = 0.75;
// Contrast starts here and rises to 1 as the picture develops.
const START_CONTRAST = 0.55;
+// The dissolve to the end frame takes this long; the end frame's flash rises over
+// FLASH_UP_SEC to full white on the cut and fades over FLASH_SEC after it.
+const FADE_SEC = 0.6;
+const FLASH_UP_SEC = 0.08;
+// The glitch an end-frame glitch plays on its cut (musicVideoTransitions.js).
+export const IMAGE_EFFECT_GLITCH_STYLE = 'vhs-glitch';
const isEffect = (value) => MUSIC_VIDEO_IMAGE_EFFECTS.includes(value);
const clampSec = (sec) => Math.min(IMAGE_EFFECT_SEC_RANGE.max, Math.max(IMAGE_EFFECT_SEC_RANGE.min, sec));
/**
- * The effect a scene plays, `{ effect, developSec }`, or null. Only a still
- * shot in a layered composition plays one: anywhere else the shot shows its
- * footage as before.
+ * The effect a scene plays, `{ effect, developSec }` (an end-frame effect:
+ * `{ effect, changeSec }`), or null. Only a still shot in a layered
+ * composition plays one, and an end-frame effect needs the shot's end frame:
+ * anywhere else the shot shows its footage or still as before.
*/
export function sceneImageEffect(scene, { layered = false } = {}) {
if (!layered || scene?.visualLayer !== 'still' || !isEffect(scene?.imageEffect)) return null;
const sec = Number(scene.imageEffectSec);
- return { effect: scene.imageEffect, developSec: Number.isFinite(sec) && sec > 0 ? clampSec(sec) : DEFAULT_IMAGE_EFFECT_SEC };
+ const set = Number.isFinite(sec) && sec > 0;
+ if (!isEndFrameImageEffect(scene.imageEffect)) return { effect: scene.imageEffect, developSec: set ? clampSec(sec) : DEFAULT_IMAGE_EFFECT_SEC };
+ if (!scene.endImageId) return null;
+ const spanSec = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' ? scene.endSec - scene.startSec : null;
+ const middle = spanSec != null && spanSec > 0 ? spanSec / 2 : DEFAULT_IMAGE_EFFECT_SEC;
+ return { effect: scene.imageEffect, changeSec: set && (spanSec == null || sec < spanSec) ? sec : Math.round(middle * 1000) / 1000 };
}
+/** The frame (from the shot's first, at `fps`) an end-frame effect cuts or starts dissolving on. */
+export const imageEffectChangeFrame = (effectSpec, fps) => Math.max(0, Math.round(effectSpec.changeSec * fps));
+
const ramp = (tSec, sec) => Math.min(1, Math.max(0, tSec / sec));
/**
* The effect's state `tSec` seconds into the shot: `flash` and `fog` are the
- * opacities of the white flash and the emulsion colour over the picture, and
- * `contrast` / `saturation` the picture's own (1 = as shot).
+ * opacities of the white flash and the emulsion colour over the picture,
+ * `contrast` / `saturation` the picture's own (1 = as shot), and `end` the end
+ * frame's opacity over the start frame. A cut lands on a frame of `fps`.
*/
-export function imageEffectState({ effect, developSec }, tSec) {
- if (effect !== 'flash-develop') return { flash: 0, fog: 0, contrast: 1, saturation: 1 };
+export function imageEffectState(effectSpec, tSec, { fps = 24 } = {}) {
+ const { effect, developSec } = effectSpec;
+ const clean = { flash: 0, fog: 0, contrast: 1, saturation: 1, end: 0 };
const t = Math.max(0, tSec);
+ if (isEndFrameImageEffect(effect)) {
+ const changeAt = imageEffectChangeFrame(effectSpec, fps) / fps;
+ if (effect === 'end-fade') return { ...clean, end: ramp(t - changeAt, FADE_SEC) };
+ const cut = t >= changeAt - 1e-6 ? 1 : 0;
+ if (effect === 'end-glitch') return { ...clean, end: cut };
+ return { ...clean, end: cut, flash: cut ? 1 - ramp(t - changeAt, FLASH_SEC) : ramp(t - (changeAt - FLASH_UP_SEC), FLASH_UP_SEC) };
+ }
+ if (effect !== 'flash-develop') return clean;
const progress = ramp(t, developSec);
return {
flash: 1 - ramp(t, FLASH_SEC),
fog: 1 - ramp(t, developSec * FOG_SHARE),
contrast: START_CONTRAST + (1 - START_CONTRAST) * progress,
saturation: progress * progress,
+ end: 0,
};
}
@@ -80,10 +126,43 @@ const num = (n) => String(Math.round(n * 10000) / 10000);
* so the caller's output label follows), or '' for no effect. `atSec` is how
* far into the shot the section starts (a shot cut into pieces picks up where
* its last piece left off); `key` keeps the graph's labels unique per section.
+ * An end-frame effect reads the end frame from `end`, a labelled stream the
+ * caller prepared like the still (cover-fitted, moved, trimmed). Its glitch
+ * frames are painted with the cut transitions' (`imageEffectGlitchFrames` in
+ * musicVideoTransitions.js).
*/
-export function musicVideoImageEffectFilter(effectSpec, { width, height, fps, key, atSec = 0 }) {
- if (effectSpec?.effect !== 'flash-develop') return '';
+export function musicVideoImageEffectFilter(effectSpec, { width, height, fps, key, atSec = 0, end = null }) {
const startFrame = Math.max(0, atSec) * fps;
+ if (isEndFrameImageEffect(effectSpec?.effect)) {
+ if (!end) return '';
+ // The change, counted from this section's first frame.
+ const change = imageEffectChangeFrame(effectSpec, fps) - Math.round(startFrame);
+ if (effectSpec.effect === 'end-fade') {
+ const frames = Math.max(1, Math.round(FADE_SEC * fps));
+ // The dissolve by frame number, so a piece starting mid-dissolve picks it up where it stands.
+ const mix = `clip((N-(${change}))/${frames},0,1)`;
+ return `[fxa${key}];[fxa${key}]format=yuv420p[fxs${key}];[${end}]format=yuv420p[fxb${key}];`
+ + `[fxs${key}][fxb${key}]blend=all_expr='A+(B-A)*${mix}':shortest=1,format=yuv420p`;
+ }
+ let chain = `[fxa${key}];[fxa${key}][${end}]overlay=shortest=1:format=auto:enable='gte(n,${change})'`;
+ if (effectSpec.effect === 'end-flash') {
+ const up = Math.max(1, Math.round(FLASH_UP_SEC * fps));
+ const down = Math.max(1, Math.round(FLASH_SEC * fps));
+ const rise = change - up;
+ const fades = [
+ ...(change > 0 ? [`fade=t=in:start_frame=${Math.max(0, rise)}:nb_frames=${Math.min(up, change)}:alpha=1`] : []),
+ ...(change + down > 0 ? [`fade=t=out:start_frame=${Math.max(0, change)}:nb_frames=${change >= 0 ? down : change + down}:alpha=1`] : []),
+ ];
+ // Before its rise the flash layer is clear; from the cut it fades to nothing,
+ // from where it stands when a piece starts mid-fade.
+ if (change + down > 0) {
+ chain += `[fxcut${key}];color=c=white@${num(change >= 0 ? 1 : (change + down) / down)}:s=${width}x${height}:r=${fps},format=yuva420p,${fades.join(',')}[fxflash${key}];`
+ + `[fxcut${key}][fxflash${key}]overlay=shortest=1:format=auto`;
+ }
+ }
+ return `${chain},format=yuv420p`;
+ }
+ if (effectSpec?.effect !== 'flash-develop') return '';
const progress = `min(1,(n+${num(startFrame)})/${num(Math.max(1, effectSpec.developSec * fps))})`;
let chain = `,eq=contrast='${num(START_CONTRAST)}+${num(1 - START_CONTRAST)}*${progress}':saturation='${progress}*${progress}':eval=frame`;
// A layer fading out over `sec` from the shot's start: from where it stands at `atSec`, linearly to nothing.
diff --git a/server/lib/musicVideoTransitions.js b/server/lib/musicVideoTransitions.js
index 2877fdfd67..f0e07d8d45 100644
--- a/server/lib/musicVideoTransitions.js
+++ b/server/lib/musicVideoTransitions.js
@@ -16,13 +16,16 @@
* of the frame's width or height, so a glitch reads the same at any size.
*/
-export const MUSIC_VIDEO_TRANSITIONS = Object.freeze(['vhs-glitch', 'channel-change', 'vhs-tracking', 'vhs-dropout']);
+import { IMAGE_EFFECT_GLITCH_STYLE, imageEffectChangeFrame, sceneImageEffect } from './musicVideoImageEffects.js';
+
+export const MUSIC_VIDEO_TRANSITIONS = Object.freeze(['vhs-glitch', 'channel-change', 'vhs-tracking', 'vhs-dropout', 'camera-flash']);
export const MUSIC_VIDEO_TRANSITION_LABELS = Object.freeze({
'vhs-glitch': 'VHS glitch',
'channel-change': 'Channel change',
'vhs-tracking': 'Tracking roll',
'vhs-dropout': 'Dropout flash',
+ 'camera-flash': 'Camera flash',
});
// Each style's frames, in order, and how many of them come before the cut
@@ -70,6 +73,12 @@ export const MUSIC_VIDEO_TRANSITION_STYLES = Object.freeze({
{ chroma: 0.003, noise: 0.15, lines: 0.2, bar: { y: 0.02, h: 0.04, alpha: 0.4 } },
],
},
+ // A camera flash on the cut: white rises over the outgoing shot's last two
+ // frames, peaks on the cut and fades off the incoming one (a still flashing into a clip).
+ 'camera-flash': {
+ lead: 2,
+ frames: [0.35, 0.8, 1, 0.75, 0.5, 0.3, 0.15].map((alpha) => ({ bar: { y: 0, h: 1, alpha } })),
+ },
'vhs-dropout': {
lead: 1,
frames: [
@@ -223,3 +232,37 @@ export function musicVideoTransitionFilter(glitches, { width, height }) {
}
return chain.length ? chain.join(',') : null;
}
+
+/**
+ * The glitched frames end-frame glitch effects play on their cuts, in the cut
+ * transitions' shape (`[{ frame, style, index }]`, see above)
+ * so both players paint them with the transitions' own code. `pieces` are
+ * timeline sections `{ startFrame, endFrame, effect?, effectAtSec? }`; a
+ * glitch keeps to its own section.
+ */
+export function imageEffectGlitchFrames(pieces, fps) {
+ const style = MUSIC_VIDEO_TRANSITION_STYLES[IMAGE_EFFECT_GLITCH_STYLE];
+ const out = [];
+ for (const piece of Array.isArray(pieces) ? pieces : []) {
+ if (piece?.effect?.effect !== 'end-glitch') continue;
+ const change = piece.startFrame + imageEffectChangeFrame(piece.effect, fps) - Math.round(Math.max(0, piece.effectAtSec || 0) * fps);
+ style.frames.forEach((_, index) => {
+ const frame = change - style.lead + index;
+ if (frame >= piece.startFrame && frame < piece.endFrame) out.push({ frame, style: IMAGE_EFFECT_GLITCH_STYLE, index });
+ });
+ }
+ return out;
+}
+
+/** The storyboard's end-frame glitch frames on `fps` (the animatic's counterpart of the render's pieces). */
+export function sceneImageEffectGlitchFrames(scenes, fps, { layered = false, endSec = null } = {}) {
+ const byId = new Map((Array.isArray(scenes) ? scenes : []).map((scene) => [scene?.sceneId, scene]));
+ return imageEffectGlitchFrames(sceneTransitionSections(scenes, fps, { endSec })
+ .map((section) => ({ ...section, effect: sceneImageEffect(byId.get(section.sceneId), { layered }) })), fps);
+}
+
+/** Cut-transition glitches and image-effect glitches as one list, a transition keeping a frame both reach. */
+export function mergeGlitchFrames(transitions, effects) {
+ const taken = new Set((transitions || []).map((g) => g.frame));
+ return [...(transitions || []), ...(effects || []).filter((g) => !taken.has(g.frame))].sort((a, b) => a.frame - b.frame);
+}
diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.js
index d2fd9cf777..f6fdff052b 100644
--- a/server/services/musicVideo/documentRender.js
+++ b/server/services/musicVideo/documentRender.js
@@ -1,5 +1,5 @@
import { assertMusicVideoMediaSelections, musicVideoMediaMode } from '../../lib/musicVideoMediaPolicy.js';
-import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
+import { isEndFrameImageEffect, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
import { musicVideoGradeFilter } from '../../lib/musicVideoGrade.js';
import { filmLookRuntimeSource, isFilmLookNeutral } from '../../lib/filmLook.js';
import { titleCardVisible } from '../../lib/musicVideoTitleCard.js';
@@ -58,6 +58,10 @@ const round3 = (n) => Math.round(n * 1000) / 1000;
const finite = (n) => (typeof n === 'number' && Number.isFinite(n) ? n : null);
const scriptJson = (value) => JSON.stringify(value).replace(/</g, '\\u003c');
const safeSegment = (id) => String(id || 'scene').replace(/[^A-Za-z0-9_-]+/g, '-').slice(0, 80) || 'scene';
+const documentImageEffect = (scene) => {
+ const effect = sceneImageEffect(scene, { layered: true });
+ return effect && !isEndFrameImageEffect(effect.effect) ? effect : null;
+};
/** The project's aspect ratio (brief), defaulting to 16:9 (lib/musicVideoAspect.js). */
export const documentAspect = musicVideoAspect;
@@ -191,7 +195,8 @@ export function buildDocumentData(project, { media = new Map(), frame, clock, so
endSec: start != null && end != null && end > start ? end : null,
shotMode: scene.shotMode === 'performance' ? 'performance' : 'cutaway',
visualLayer: documentSceneVisualLayer(project, scene, { generated }),
- imageEffect: sceneImageEffect(scene, { layered: true }),
+ // A document has no end-frame media yet, so a start-to-end effect plays its still.
+ imageEffect: documentImageEffect(scene),
stillMove: typeof scene.stillMove === 'string' ? scene.stillMove : null,
// #10589: the planned camera move the template's camera rig follows (only when planned).
...(typeof scene.camera?.move === 'string' ? { camera: structuredClone(scene.camera) } : {}),
diff --git a/server/services/musicVideo/imageEffects.test.js b/server/services/musicVideo/imageEffects.test.js
index ed0bca3783..1d1e979c98 100644
--- a/server/services/musicVideo/imageEffects.test.js
+++ b/server/services/musicVideo/imageEffects.test.js
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { findFfmpeg } from '../../lib/ffmpeg.js';
import { imageEffectState, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
+import { sceneImageEffectGlitchFrames } from '../../lib/musicVideoTransitions.js';
import { buildMusicVideoFfmpegArgs } from './render.js';
import { applySceneUpdate, replaceScenes } from './projectsLogic.js';
@@ -69,7 +70,7 @@ describe.skipIf(!ffmpeg)('camera flash develop in the final render', () => {
expect(Math.abs(stats[60].luma - held.luma)).toBeLessThan(1);
// The animatic paints the same moments.
expect(imageEffectState(effect, 0)).toMatchObject({ flash: 1, fog: 1, saturation: 0 });
- expect(imageEffectState(effect, 1)).toEqual({ flash: 0, fog: 0, contrast: 1, saturation: 1 });
+ expect(imageEffectState(effect, 1)).toEqual({ flash: 0, fog: 0, contrast: 1, saturation: 1, end: 0 });
});
it('picks up a shot cut into pieces where the piece before it left off', () => {
@@ -89,6 +90,67 @@ describe.skipIf(!ffmpeg)('camera flash develop in the final render', () => {
});
});
+// Uniquely catches an end-frame effect playing without its end frame, changing at the wrong time, or its glitch landing off its cut.
+describe('end-frame effects on the storyboard', () => {
+ const shot = { sceneId: 'e', startSec: 10, endSec: 13, visualLayer: 'still', imageEffect: 'end-glitch', endImageId: 'end.png' };
+ it('changes in the middle by default or at its chosen time, and needs an end frame', () => {
+ expect(sceneImageEffect(shot, { layered: true })).toEqual({ effect: 'end-glitch', changeSec: 1.5 });
+ expect(sceneImageEffect({ ...shot, imageEffectSec: 1 }, { layered: true }).changeSec).toBe(1);
+ expect(sceneImageEffect({ ...shot, imageEffectSec: 6 }, { layered: true }).changeSec).toBe(1.5);
+ expect(sceneImageEffect({ ...shot, endImageId: null }, { layered: true })).toBeNull();
+ });
+ it('places the glitch on the cut to the end frame, inside the shot', () => {
+ expect(sceneImageEffectGlitchFrames([shot], 24, { layered: true }).map((g) => g.frame)).toEqual([275, 276, 277]);
+ expect(sceneImageEffectGlitchFrames([shot], 24, { layered: false })).toEqual([]);
+ });
+});
+
+const renderStill = (name, clip, sec) => {
+ const song = join(dir, `${name}.wav`);
+ run(['-f', 'lavfi', '-i', `sine=frequency=440:duration=${sec}`, '-y', song]);
+ const output = join(dir, `${name}.mp4`);
+ const plan = buildMusicVideoFfmpegArgs([clip], song, output, { audioDurationSec: sec });
+ run(plan.args.filter((arg, i) => !(arg === '-progress' || plan.args[i - 1] === '-progress')));
+ return frameStats(output);
+};
+
+// Uniquely catches an end-frame effect's filter graph failing in ffmpeg or changing on the wrong frame.
+describe.skipIf(!ffmpeg)('end-frame effects in the final render', () => {
+ const still = (name, color) => {
+ const path = join(dir, `${name}.png`);
+ run(['-f', 'lavfi', '-i', `color=${color}:s=${W}x${H}`, '-frames:v', '1', '-y', path]);
+ return path;
+ };
+ const clipFor = (effect) => ({ sceneId: 's1', layer: 'still', imagePath: still('red', '0xd02020'), endImagePath: still('blue', '0x2040d0'), move: 'hold',
+ inSec: 0, outSec: 2, duration: 2, sourceSec: 2, effect: { effect, changeSec: 1 } });
+ const isRed = (f) => f.chroma > 40 && f.luma < 120 && f.luma > 50;
+ it.each([['end-fade'], ['end-flash'], ['end-glitch']])('%s plays the start frame, then the end frame from the change on', (effect) => {
+ const stats = renderStill(effect, clipFor(effect), 2);
+ expect(stats).toHaveLength(48);
+ const start = stats[10];
+ const end = stats[47];
+ expect(Math.abs(start.luma - end.luma) + Math.abs(start.chroma - end.chroma)).toBeGreaterThan(10);
+ if (effect === 'end-fade') expect(stats[31].luma).not.toBeCloseTo(start.luma, 0);
+ // The flash peaks white on the cut.
+ if (effect === 'end-flash') expect(stats[24].luma).toBeGreaterThan(230);
+ expect(isRed(stats[20]) || stats[20].chroma === start.chroma).toBe(true);
+ });
+ // Uniquely catches an intercut piece restarting the fade or flash instead of continuing it.
+ it('picks a fade or flash up mid-way when a piece starts just after the change', () => {
+ const solid = renderStill('fade-whole', clipFor('end-fade'), 2);
+ const fade = renderStill('fade-piece', { ...clipFor('end-fade'), effectAtSec: 1.3 }, 1);
+ const red = solid[10];
+ const blue = solid[47];
+ // Half-way through the dissolve, between the two frames, not back on the start frame.
+ expect(fade[0].luma).toBeGreaterThan(Math.min(red.luma, blue.luma) + 3);
+ expect(fade[0].luma).toBeLessThan(Math.max(red.luma, blue.luma) - 3);
+ const flash = renderStill('flash-piece', { ...clipFor('end-flash'), effectAtSec: 1.1 }, 1);
+ // Two frames into its fade the flash is fading, not a full white frame again.
+ expect(flash[0].luma).toBeLessThan(220);
+ expect(flash[0].luma).toBeGreaterThan(blue.luma + 30);
+ });
+});
+
// Uniquely catches the scene PATCH refusing the effect fields or an out-of-range develop.
describe('a shot\'s image effect on the board', () => {
const project = { id: 'mv-1', scenes: [{ sceneId: 's1', order: 0, startSec: 0, endSec: 4, takes: [] }] };
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 892efc9a64..7f27aac5ac 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -56,9 +56,9 @@ import { AUDIO_NORM, buildAudioBedMix } from '../videoTimeline/audioBedMix.js';
import { musicVideoEvents } from './events.js';
import { projectSoundBed } from './soundBed.js';
import { intercutClips } from './intercut.js';
-import { musicVideoTransitionFilter, sceneTransitions, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';
+import { imageEffectGlitchFrames, mergeGlitchFrames, musicVideoTransitionFilter, sceneTransitions, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';
import { stillMoveForCamera } from '../../lib/cameraMovements.js';
-import { musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
+import { isEndFrameImageEffect, musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
// Per-project render mutex (keyed by projectId so two projects can render in
// parallel; same-project re-entry returns 409 with the live jobId for re-attach).
@@ -253,8 +253,12 @@ export async function resolveSceneClips(project, { layered = false, stillForMiss
if (!imagePath || !existsSync(imagePath)) { missingStills.push(scene.sceneId); continue; }
// A planned camera move (#10589) picks the nearest composed still move.
// An image effect (a camera flash that develops, musicVideoImageEffects.js) plays over the still.
+ // An end-frame effect also reads the shot's end frame; without it on disk the still plays alone.
const effect = sceneImageEffect(scene, { layered });
- clips.push({ ...section, imagePath, move: stillMoveForCamera(scene.camera) || scene.stillMove || 'hold', ...(effect ? { effect } : {}) });
+ const endImagePath = effect && isEndFrameImageEffect(effect.effect) ? safeUnder(PATHS.images, scene.endImageId) : null;
+ const playable = effect && (!isEndFrameImageEffect(effect.effect) || (endImagePath && existsSync(endImagePath)));
+ clips.push({ ...section, imagePath, move: stillMoveForCamera(scene.camera) || scene.stillMove || 'hold',
+ ...(playable ? { effect, ...(endImagePath ? { endImagePath } : {}) } : {}) });
continue;
}
const entry = historyMap.get(scene.videoHistoryId);
@@ -430,9 +434,13 @@ function sectionChain(c, input, { canonW, canonH, fps, frames, frameGrid }) {
// Rendered from a 2x cover so the slow zoom does not step pixel by pixel.
? `zoompan=z='1+${STILL_PUSH_ZOOM}*on/${frames}':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=${canonW}x${canonH}:fps=${fps},setsar=1,`
: c.move === 'pan' ? `crop=${canonW}:${canonH}:x='(iw-ow)*n/${Math.max(1, frames - 1)}':y='(ih-oh)/2',` : '';
- return `[${input}:v]fps=${fps},scale=${coverW}:${coverH}:force_original_aspect_ratio=increase,crop=${coverW}:${coverH},setsar=1,`
- + `${move}format=yuv420p,${trim},setpts=PTS-STARTPTS`
- + musicVideoImageEffectFilter(c.effect, { width: canonW, height: canonH, fps, key: input, atSec: c.effectAtSec || 0 });
+ // An end frame (an end-frame image effect) is fitted and moved exactly like the still.
+ const fitted = (id) => `[${id}:v]fps=${fps},scale=${coverW}:${coverH}:force_original_aspect_ratio=increase,crop=${coverW}:${coverH},setsar=1,`
+ + `${move}format=yuv420p,${trim},setpts=PTS-STARTPTS`;
+ const id = typeof input === 'object' ? input.clip : input;
+ const end = typeof input === 'object' && input.end != null ? `${fitted(input.end)}[fxend${id}];` : '';
+ return end + fitted(id)
+ + musicVideoImageEffectFilter(c.effect, { width: canonW, height: canonH, fps, key: id, atSec: c.effectAtSec || 0, end: end ? `fxend${id}` : null });
}
// A non-looping shot a hair longer than its source holds the last frame for
// the remainder instead of ending early and drifting every later cut. On the
@@ -522,8 +530,14 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioD
for (const { c } of plan) {
if (c.layer === 'card') { inputOf.push(null); continue; }
const clipInput = nextInput++;
- if (c.layer === 'still') inputs.push('-loop', '1', '-framerate', String(fps), '-i', c.imagePath);
- else {
+ if (c.layer === 'still') {
+ inputs.push('-loop', '1', '-framerate', String(fps), '-i', c.imagePath);
+ if (c.endImagePath) {
+ inputOf.push({ clip: clipInput, end: nextInput++ });
+ inputs.push('-loop', '1', '-framerate', String(fps), '-i', c.endImagePath);
+ continue;
+ }
+ } else {
if (c.loop !== false) inputs.push('-stream_loop', '-1');
inputs.push('-i', c.videoPath);
}
@@ -540,10 +554,12 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioD
const filters = plan.map(({ c, frames }, i) => `${sectionChain(c, inputOf[i], { canonW, canonH, fps, frames, frameGrid })}[v${i}]`);
const gradeFilter = musicVideoGradeFilter(grade, plan.map(({ c, startSec, endSec }) => ({ sceneId: c.sceneId, startSec, endSec })), { fps });
// Cut transitions repaint a few frames around their cuts, under the typography, and add no time.
- const glitches = transitionGlitchFrames(plan.map(({ c, startSec, endSec }) => ({
+ const pieces = plan.map(({ c, startSec, endSec }) => ({
sceneId: c.sceneId, startFrame: Math.round(startSec * fps), endFrame: Math.round(endSec * fps),
- partner: !!c.intercutOf, transitionIn: c.transitionIn, transitionOut: c.transitionOut,
- })));
+ partner: !!c.intercutOf, transitionIn: c.transitionIn, transitionOut: c.transitionOut, effect: c.effect, effectAtSec: c.effectAtSec,
+ }));
+ // A still's glitch to its end frame paints with the same frames.
+ const glitches = mergeGlitchFrames(transitionGlitchFrames(pieces), imageEffectGlitchFrames(pieces, fps));
const glitchFilter = musicVideoTransitionFilter(glitches, { width: canonW, height: canonH });
const cutLabel = overlays.length > 0 ? 'cut0' : 'outv';
const finishing = [gradeFilter, glitchFilter].filter(Boolean).join(',');