Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

feat: image effect shots, starting with camera flash develop

Claudec1542b4c80f0
diff --git a/client/src/components/musicVideo/ImageEffectStill.jsx b/client/src/components/musicVideo/ImageEffectStill.jsx
new file mode 100644
index 0000000000..c59064749b
--- /dev/null
+++ b/client/src/components/musicVideo/ImageEffectStill.jsx
@@ -0,0 +1,51 @@
+import { useCallback, useEffect, useRef } from 'react';
+import { IMAGE_EFFECT_FOG_COLOR, imageEffectState } from '../../lib/musicVideoImageEffects.js';
+
+/**
+ * A still shot's image effect in the storyboard animatic: the still, cover-fitted
+ * as the final render crops it, with the effect's flash and emulsion laid over it
+ * and its contrast and colour filtered, all from the same curves the render uses
+ * (`imageEffectState`). Paused it paints the frame at `time`; playing it repaints
+ * every animation frame from the song's clock, writing styles directly so the
+ * board does not re-render per frame.
+ */
+export default function ImageEffectStill({ src, effect, startSec, audioRef, time }) {
+ const imgRef = useRef(null);
+ const fogRef = useRef(null);
+ const flashRef = useRef(null);
+ const paint = useCallback((t) => {
+ const state = imageEffectState(effect, t - startSec);
+ if (imgRef.current) imgRef.current.style.filter = `contrast(${state.contrast}) saturate(${state.saturation})`;
+ if (fogRef.current) fogRef.current.style.opacity = String(state.fog);
+ if (flashRef.current) flashRef.current.style.opacity = String(state.flash);
+ }, [effect, startSec]);
+ useEffect(() => { paint(time); }, [time, paint]);
+ useEffect(() => {
+ const audio = audioRef.current;
+ if (!audio) return undefined;
+ let handle = 0;
+ const tick = () => {
+ paint(audio.currentTime || 0);
+ handle = requestAnimationFrame(tick);
+ };
+ const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };
+ const stop = () => { cancelAnimationFrame(handle); paint(audio.currentTime || 0); };
+ audio.addEventListener('play', start);
+ audio.addEventListener('pause', stop);
+ audio.addEventListener('ended', stop);
+ if (!audio.paused) start();
+ return () => {
+ cancelAnimationFrame(handle);
+ audio.removeEventListener('play', start);
+ audio.removeEventListener('pause', stop);
+ audio.removeEventListener('ended', stop);
+ };
+ }, [audioRef, paint]);
+ return (
+ <div className="relative h-full w-full" data-testid="animatic-image-effect">
+ <img ref={imgRef} src={src} alt="" className="h-full w-full object-cover" />
+ <div ref={fogRef} className="pointer-events-none absolute inset-0" style={{ background: IMAGE_EFFECT_FOG_COLOR, opacity: 0 }} />
+ <div ref={flashRef} className="pointer-events-none absolute inset-0" style={{ background: '#fff', opacity: 0 }} />
+ </div>
+ );
+}
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index a9ff8c4798..bf91b363cc 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -35,6 +35,7 @@ import SceneRenderTarget, { lastRenderTarget, renderTargetOverrides } from './Sc
import SceneRenderInputs, { shotCardRenderRequest, shotCardRequestProblem } from './SceneRenderInputs.jsx';
import SceneFrameFit, { sceneFrameFit } from './SceneFrameFit.jsx';
import SceneTransitions from './SceneTransitions.jsx';
+import SceneImageEffect from './SceneImageEffect.jsx';
// The two timeline-bound scene fields rendered as identical number inputs.
const SCENE_TIME_FIELDS = [['Start', 'startSec'], ['End', 'endSec']];
@@ -461,6 +462,11 @@ export default function SceneCard({
{/* Clip: the motion prompt, where this shot renders and what it costs, then its clip (Phase 1). */}
<section aria-label="Render" className="min-w-0 space-y-2">
<h4 className="text-xs font-medium">Clip</h4>
+ {/* A clip, the still, or the still playing a code-drawn image effect (no model call). */}
+ {layered && (layer === 'footage' || layer === 'still') && (
+ <SceneImageEffect fieldId={fieldId} scene={scene} onChange={applyPatch}
+ disabled={shotLock?.clip === 'locked'} disabledReason={shotLock?.clip === 'locked' ? 'Video locked. Unlock the video to change it.' : null} />
+ )}
{scene.videoHistoryId && performance && (
<PerformanceEvidence key={scene.videoHistoryId} clipSrc={clipSrc} instruction={instruction}
temporal={performanceReview?.shot} excerptStartSec={performanceReview?.excerptStartSec || 0}
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index 29483ee070..0454234177 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -468,3 +468,26 @@ describe('VHS glitch transitions', () => {
expect(screen.getByLabelText('Glitch in').value).toBe('');
});
});
+
+// Uniquely catches the picture picker saving the wrong layer or effect fields, or offering the effect where the render ignores it.
+describe('image effect picker', () => {
+ it('turns a shot into a still playing camera flash develop, sets its develop length, and back to a clip', () => {
+ const onSave = vi.fn();
+ function Harness({ layered = true }) {
+ const [scene, setScene] = useState({ sceneId: 's1', startSec: 0, endSec: 4, referenceImageId: 'burger.png', takes: [] });
+ return <SceneCard scene={scene} index={0} layered={layered} onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
+ }
+ const { unmount } = render(<Harness />);
+ const picture = screen.getByLabelText('Picture');
+ fireEvent.change(picture, { target: { value: 'flash-develop' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 3 });
+ fireEvent.change(screen.getByLabelText('Develops in'), { target: { value: '6' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { imageEffectSec: 6 });
+ expect(screen.getByText('The shot ends at 4.0s, before the photo finishes developing.')).toBeTruthy();
+ fireEvent.change(picture, { target: { value: 'clip' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { visualLayer: 'footage', imageEffect: null });
+ unmount();
+ render(<Harness layered={false} />);
+ expect(screen.queryByLabelText('Picture')).toBeNull();
+ });
+});
diff --git a/client/src/components/musicVideo/SceneImageEffect.jsx b/client/src/components/musicVideo/SceneImageEffect.jsx
new file mode 100644
index 0000000000..2e6c2cf763
--- /dev/null
+++ b/client/src/components/musicVideo/SceneImageEffect.jsx
@@ -0,0 +1,63 @@
+import { safeReadStorage, safeWriteStorage } from '../../lib/safeStorage.js';
+import { DEFAULT_IMAGE_EFFECT_SEC, IMAGE_EFFECT_DEVELOP_SECS, MUSIC_VIDEO_IMAGE_EFFECTS, MUSIC_VIDEO_IMAGE_EFFECT_LABELS } from '../../lib/musicVideoImageEffects.js';
+
+// The develop length picked last on this device, for the next shot that takes an effect.
+const LAST_SEC_KEY = 'portos-mv-image-effect-sec';
+const lastDevelopSec = () => {
+ const sec = Number(safeReadStorage(LAST_SEC_KEY));
+ return IMAGE_EFFECT_DEVELOP_SECS.includes(sec) ? sec : DEFAULT_IMAGE_EFFECT_SEC;
+};
+
+/**
+ * Where a shot's picture comes from in a composed render: a rendered clip, its
+ * still, or its still playing a code-drawn image effect (a camera flash that
+ * develops like an instant photo), which needs no model. An effect's develop
+ * length rides beside it. The effect plays inside the shot's span and adds no time.
+ */
+export default function SceneImageEffect({ fieldId, scene, onChange, disabled = false, disabledReason = null }) {
+ const effect = scene.visualLayer === 'still' && MUSIC_VIDEO_IMAGE_EFFECTS.includes(scene.imageEffect) ? scene.imageEffect : null;
+ const value = effect || (scene.visualLayer === 'still' ? 'still' : 'clip');
+ const developSec = IMAGE_EFFECT_DEVELOP_SECS.includes(scene.imageEffectSec) ? scene.imageEffectSec : DEFAULT_IMAGE_EFFECT_SEC;
+ const spanSec = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' ? scene.endSec - scene.startSec : null;
+ const pick = (next) => {
+ if (next === 'clip') onChange({ visualLayer: 'footage', ...(scene.imageEffect ? { imageEffect: null } : {}) });
+ else if (next === 'still') onChange({ visualLayer: 'still', ...(scene.imageEffect ? { imageEffect: null } : {}) });
+ else onChange({ visualLayer: 'still', imageEffect: next, imageEffectSec: scene.imageEffectSec || lastDevelopSec() });
+ };
+ const pickSec = (sec) => {
+ safeWriteStorage(LAST_SEC_KEY, String(sec));
+ onChange({ imageEffectSec: sec });
+ };
+ return (
+ <div className="space-y-1 text-xs" data-testid="scene-image-effect">
+ <div className="flex flex-wrap items-center gap-x-3 gap-y-1">
+ <span className="flex min-w-0 items-center gap-1">
+ <label htmlFor={fieldId('picture')} className={effect ? 'text-port-accent' : 'text-port-text-muted'}>Picture</label>
+ <select id={fieldId('picture')} value={value} disabled={disabled} title={disabledReason || undefined}
+ onChange={(e) => pick(e.target.value)}
+ className="min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50">
+ <option value="clip">Video clip</option>
+ <option value="still">Still image</option>
+ {MUSIC_VIDEO_IMAGE_EFFECTS.map((id) => <option key={id} value={id}>{MUSIC_VIDEO_IMAGE_EFFECT_LABELS[id]}</option>)}
+ </select>
+ </span>
+ {effect && (
+ <span className="flex min-w-0 items-center gap-1">
+ <label htmlFor={fieldId('develop')} className="text-port-text-muted">Develops in</label>
+ <select id={fieldId('develop')} value={developSec} onChange={(e) => pickSec(Number(e.target.value))}
+ className="min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0">
+ {IMAGE_EFFECT_DEVELOP_SECS.map((sec) => <option key={sec} value={sec}>{sec}s</option>)}
+ </select>
+ </span>
+ )}
+ </div>
+ {effect && (
+ <p className="text-[11px] text-port-text-muted">
+ {spanSec != null && developSec > spanSec
+ ? `The shot ends at ${spanSec.toFixed(1)}s, before the photo finishes developing.`
+ : 'No video render: the still plays the effect.'}
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index d2045bfcf5..6f8ee15cf3 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -13,6 +13,9 @@ import ShotStepper from './ShotStepper.jsx';
import FrameStepper, { captureMediaFrame } from './FrameStepper.jsx';
import { sceneFrameFit } from './SceneFrameFit.jsx';
import TransitionGlitch from './TransitionGlitch.jsx';
+import ImageEffectStill from './ImageEffectStill.jsx';
+import { isLayeredComposition } from '../../lib/musicVideoLayers.js';
+import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);
@@ -284,9 +287,13 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
const shot = shotAt(scenes, time);
const timed = timedShots(scenes);
const index = shot ? timed.indexOf(shot) : -1;
+ // A still shot with an image effect plays the effect over its still, as the composed render does, not its clip.
+ const layered = isLayeredComposition(project);
+ const shotEffect = shot ? sceneImageEffect(shot, { layered }) : null;
// The shot on screen and the next few: each keeps one mounted clip, so the cut to the
// next shot shows a clip that has already loaded and sits on its first frame.
- const clipShots = timed.slice(Math.max(index, 0), Math.max(index, 0) + 1 + CLIP_PRELOAD_SHOTS).filter((scene) => scene.videoHistoryId);
+ const clipShots = timed.slice(Math.max(index, 0), Math.max(index, 0) + 1 + CLIP_PRELOAD_SHOTS)
+ .filter((scene) => scene.videoHistoryId && !sceneImageEffect(scene, { layered }));
const label = shot ? shot.label || `Shot ${index + 1}` : 'No shots yet';
// With the lyrics drawn over the picture, a shot without a frame keeps its label and
// intent under the picture instead, so no stand-in text sits under the sung words.
@@ -299,7 +306,10 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
<figure ref={figureRef} aria-label="Storyboard animatic" className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black">
{/* The project's film look, as the final render applies it over the whole picture, lyrics included. */}
<FilmLookFrame look={lookOn ? project.filmLook : null} frame={Math.round(time * ANIMATIC_FPS)} playing={playing} className="h-full w-full">
- {shot?.referenceImageId ? (
+ {shot?.referenceImageId && shotEffect ? (
+ <ImageEffectStill key={shot.sceneId} src={`/data/images/${shot.referenceImageId}`} effect={shotEffect}
+ startSec={shot.startSec} audioRef={audioRef} time={time} />
+ ) : shot?.referenceImageId ? (
// Behind a VHS-edged clip the still stands in, blurred and faded, for the render's blur of the clip.
<img src={`/data/images/${shot.referenceImageId}`} alt=""
className={`h-full w-full ${vhsEdges ? 'object-cover blur-md saturate-50 brightness-75' : stillEdges ? 'object-cover' : 'object-contain'}`} />
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
index 68d42554bb..97444e9269 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
@@ -27,6 +27,21 @@ describe('StoryboardAnimatic', () => {
expect(container.querySelector('img').getAttribute('src')).toBe('/data/images/frame-b.png');
expect(screen.getByText('Shot 2 / 2')).toBeTruthy();
});
+
+ // Uniquely catches the animatic showing a clip where the composed render plays a still's image effect.
+ it('plays a still shot\'s camera flash develop over its still, not its clip, from the shot\'s start', () => {
+ const effectShot = { sceneId: 'fx', label: 'Burger', startSec: 0, endSec: 6, referenceImageId: 'burger.png', videoHistoryId: 'clip-1',
+ visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 2 };
+ render(<StoryboardAnimatic project={{ id: 'p', composition: { mode: 'composed' }, scenes: [effectShot] }} audioUrl="/song.mp3" />);
+ const effect = screen.getByTestId('animatic-image-effect');
+ expect(screen.queryByLabelText('Clip for Burger')).toBeNull();
+ const img = effect.querySelector('img');
+ expect(img.style.filter).toBe('contrast(0.55) saturate(0)');
+ const audio = screen.getByLabelText('Song for the storyboard animatic');
+ Object.defineProperty(audio, 'currentTime', { value: 3, configurable: true });
+ fireEvent.timeUpdate(audio);
+ expect(img.style.filter).toBe('contrast(1) saturate(1)');
+ });
});
vi.mock('../../services/apiMusicVideo.js', () => ({
diff --git a/client/src/lib/README.md b/client/src/lib/README.md
index adf90c262f..b38f8cac67 100644
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@ -317,4 +317,5 @@ grep -i "what you want to do" client/src/lib/README.md
| `musicVideoGrade.js` | Shared Music Video grade preset and grain bounds for the composition controls. |
| `filmLook.js` | Shared film look controls, presets, normalizer, prompt vocabulary, the SVG filter markup the editor previews with and the playback-time motion layers (server `lib/filmLook.js`). |
| `musicVideoNarrativeEvents.js` | Named narrative event kinds, absolute onset/word/frame resolution and deterministic silence/reactive state for composition documents. |
+| `musicVideoImageEffects.js` | Music-video image effects (a camera flash that develops over a still shot), re-exported from `server/lib/musicVideoImageEffects.js`: the effects, labels and develop lengths, `sceneImageEffect`, `imageEffectState`. Used by `components/musicVideo/SceneImageEffect.jsx` and the storyboard animatic. |
| `musicVideoTransitions.js` | Music-video cut transitions (VHS glitches over a cut), re-exported from `server/lib/musicVideoTransitions.js`: the styles and labels, `sceneTransitionSections`, `transitionGlitchFrames`, `transitionFrameAt`. Used by `components/musicVideo/SceneTransitions.jsx` and the storyboard animatic. |
diff --git a/client/src/lib/index.js b/client/src/lib/index.js
index 14ecfcdd98..a4331c2d62 100644
--- a/client/src/lib/index.js
+++ b/client/src/lib/index.js
@@ -278,3 +278,4 @@ export * from './musicVideoGrade.js';
export * from './filmLook.js';
export * from './musicVideoNarrativeEvents.js';
export * from './musicVideoTransitions.js';
+export * from './musicVideoImageEffects.js';
diff --git a/client/src/lib/musicVideoImageEffects.js b/client/src/lib/musicVideoImageEffects.js
new file mode 100644
index 0000000000..923e8a7d4c
--- /dev/null
+++ b/client/src/lib/musicVideoImageEffects.js
@@ -0,0 +1,14 @@
+/**
+ * Music Video image effects — re-export of `server/lib/musicVideoImageEffects.js`,
+ * so the shot card's picker and the animatic paint the same effect as the final render.
+ */
+export {
+ DEFAULT_IMAGE_EFFECT_SEC,
+ IMAGE_EFFECT_DEVELOP_SECS,
+ IMAGE_EFFECT_FOG_COLOR,
+ IMAGE_EFFECT_SEC_RANGE,
+ MUSIC_VIDEO_IMAGE_EFFECTS,
+ MUSIC_VIDEO_IMAGE_EFFECT_LABELS,
+ imageEffectState,
+ sceneImageEffect,
+} from '../../../server/lib/musicVideoImageEffects.js';
diff --git a/server/lib/README.md b/server/lib/README.md
index 442c26da64..00067f6839 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -560,6 +560,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `chiptuneRender.js` | Deterministic chiptune score → mono PCM → 16-bit WAV buffer (pure Node, no audio deps); `renderScoreToPcm` / `pcmToWavBuffer` / `renderScoreToWav`. |
| `chiptuneScore.js` | Chiptune score contract (#2911): `chiptuneScoreSchema` (Zod) + `sanitizeChiptuneScore`, `buildScoreEvents` — the pattern/order → absolute-time flatten mirrored by `client/src/lib/chiptunePlayback.js`. |
| `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). |
+| `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`, first `flash-develop`: a camera flash, then the picture develops like an instant photo over `imageEffectSec`). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe. |
| `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections` and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Browser-safe; the storyboard animatic paints the same frames. |
| `pitchMath.js` | Dependency-free scientific-pitch math shared by the LLM-authored music formats on both runtimes: `pitchToMidi("C4")` → 60 (null when unparseable) and `midiToFreq(69)` → 440. |
| `wavAudioFile.js` | `writeWavAudioFile(wav, dir, basename)` — land a synthesized WAV Buffer as `<basename>.ogg` via the system ffmpeg, falling back to `<basename>.wav` (and dropping a stale `.ogg`) when ffmpeg is missing or the encode fails; resolves to the filename used. Shared by the chiptune and drawn-waveform renderers and the code-engine take upload. `wavDurationMs(buffer)` reads a PCM WAV's duration from its RIFF header (0 for anything that is not a parseable WAV). `measureWavAudio(buffer)` decodes PCM 16/24/32-bit or float 32/64 (EXTENSIBLE included) into `{ channels, sampleRate, frames, durationMs, peak, rms, nonFinite }` so native renders are judged by their samples (null when undecodable). |
diff --git a/server/lib/index.js b/server/lib/index.js
index 0e281bc763..cbde30e831 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -878,3 +878,4 @@ export * from './deepAuditCoverage.js';
export * from './auditWorkflow.js';
export * from './musicVideoTransitions.js';
+export * from './musicVideoImageEffects.js';
diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.js
index 072844a60a..61e9d6ade3 100644
--- a/server/lib/musicVideoDependencies.js
+++ b/server/lib/musicVideoDependencies.js
@@ -21,6 +21,8 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s
.concat(scene.frameFit && scene.frameFit !== 'fill' ? [['frameFit', scene.frameFit]] : [])
// A glitch over its cuts changes the render; a shot without one keeps its evidence.
.concat(['transitionIn', 'transitionOut'].filter((key) => scene[key]).map((key) => [key, scene[key]]))
+ // An image effect over a still changes the render; a still without one keeps its evidence.
+ .concat(scene.imageEffect ? [['imageEffect', scene.imageEffect], ['imageEffectSec', scene.imageEffectSec ?? null]] : [])
// Only a music-driven performance joins it, so lip-sync evidence stays current.
.concat(scene.performanceStyle === 'music' ? [['performanceStyle', 'music']] : []))) });
const snapshot = (references) => ({ version: MUSIC_VIDEO_DEPENDENCY_VERSION, references });
diff --git a/server/lib/musicVideoImageEffects.js b/server/lib/musicVideoImageEffects.js
new file mode 100644
index 0000000000..e2050fd606
--- /dev/null
+++ b/server/lib/musicVideoImageEffects.js
@@ -0,0 +1,100 @@
+/**
+ * Music Video image effects: a still shot that plays a code-drawn effect over
+ * its still instead of a generated clip, so no model is called for it. The
+ * shot is a still section (`visualLayer: 'still'`) that also names an
+ * `imageEffect`; only a layered (composed) render honours it, as it does the
+ * still itself.
+ *
+ * "Camera flash develop" (`flash-develop`): the shot opens on a white camera
+ * flash, then the picture develops like an instant photo over
+ * `imageEffectSec` seconds: a murky grey-green emulsion clears, the contrast
+ * comes up and the colour fills in last, and the still holds fully developed
+ * for the rest of the shot. The effect never adds time: it plays inside the
+ * shot's own span, and a develop longer than the shot ends the shot part-way.
+ *
+ * One definition serves both players: every value is a pure function of the
+ * time into the shot (`imageEffectState`), the storyboard animatic paints it
+ * with CSS over the still, and the final render turns the same curves into
+ * ffmpeg filters (`musicVideoImageEffectFilter`). Each curve is linear or a
+ * square of a linear ramp so the two read the same frame for frame.
+ */
+
+export const MUSIC_VIDEO_IMAGE_EFFECTS = Object.freeze(['flash-develop']);
+
+export const MUSIC_VIDEO_IMAGE_EFFECT_LABELS = Object.freeze({
+ 'flash-develop': 'Camera flash develop',
+});
+
+/** The develop lengths the shot card offers, in seconds. */
+export const IMAGE_EFFECT_DEVELOP_SECS = Object.freeze([1, 2, 3, 4, 6, 8]);
+export const DEFAULT_IMAGE_EFFECT_SEC = 3;
+export const IMAGE_EFFECT_SEC_RANGE = Object.freeze({ min: 0.5, max: 30 });
+
+// The flash fades from full white to nothing over this long.
+const FLASH_SEC = 0.25;
+// The undeveloped emulsion's colour, and the share of the develop it takes to clear.
+export const IMAGE_EFFECT_FOG_COLOR = '#4a5452';
+const FOG_SHARE = 0.75;
+// Contrast starts here and rises to 1 as the picture develops.
+const START_CONTRAST = 0.55;
+
+const isEffect = (value) => MUSIC_VIDEO_IMAGE_EFFECTS.includes(value);
+const clampSec = (sec) => Math.min(IMAGE_EFFECT_SEC_RANGE.max, Math.max(IMAGE_EFFECT_SEC_RANGE.min, sec));
+
+/**
+ * The effect a scene plays, `{ effect, developSec }`, or null. Only a still
+ * shot in a layered composition plays one: anywhere else the shot shows its
+ * footage as before.
+ */
+export function sceneImageEffect(scene, { layered = false } = {}) {
+ if (!layered || scene?.visualLayer !== 'still' || !isEffect(scene?.imageEffect)) return null;
+ const sec = Number(scene.imageEffectSec);
+ return { effect: scene.imageEffect, developSec: Number.isFinite(sec) && sec > 0 ? clampSec(sec) : DEFAULT_IMAGE_EFFECT_SEC };
+}
+
+const ramp = (tSec, sec) => Math.min(1, Math.max(0, tSec / sec));
+
+/**
+ * The effect's state `tSec` seconds into the shot: `flash` and `fog` are the
+ * opacities of the white flash and the emulsion colour over the picture, and
+ * `contrast` / `saturation` the picture's own (1 = as shot).
+ */
+export function imageEffectState({ effect, developSec }, tSec) {
+ if (effect !== 'flash-develop') return { flash: 0, fog: 0, contrast: 1, saturation: 1 };
+ const t = Math.max(0, tSec);
+ const progress = ramp(t, developSec);
+ return {
+ flash: 1 - ramp(t, FLASH_SEC),
+ fog: 1 - ramp(t, developSec * FOG_SHARE),
+ contrast: START_CONTRAST + (1 - START_CONTRAST) * progress,
+ saturation: progress * progress,
+ };
+}
+
+const num = (n) => String(Math.round(n * 10000) / 10000);
+
+/**
+ * The ffmpeg filtergraph that plays `effect` over one section, appended to the
+ * section's chain after its frames start counting from 0. Returns the text to
+ * append (it begins with a comma and ends on the effected stream, unlabelled,
+ * so the caller's output label follows), or '' for no effect. `atSec` is how
+ * far into the shot the section starts (a shot cut into pieces picks up where
+ * its last piece left off); `key` keeps the graph's labels unique per section.
+ */
+export function musicVideoImageEffectFilter(effectSpec, { width, height, fps, key, atSec = 0 }) {
+ if (effectSpec?.effect !== 'flash-develop') return '';
+ const startFrame = Math.max(0, atSec) * fps;
+ const progress = `min(1,(n+${num(startFrame)})/${num(Math.max(1, effectSpec.developSec * fps))})`;
+ let chain = `,eq=contrast='${num(START_CONTRAST)}+${num(1 - START_CONTRAST)}*${progress}':saturation='${progress}*${progress}':eval=frame`;
+ // A layer fading out over `sec` from the shot's start: from where it stands at `atSec`, linearly to nothing.
+ const layer = (name, color, sec) => {
+ const left = sec - Math.max(0, atSec);
+ const frames = Math.round(left * fps);
+ if (frames < 1) return;
+ chain += `[fxin${name}${key}];color=c=${color}@${num(left / sec)}:s=${width}x${height}:r=${fps},format=yuva420p,`
+ + `fade=t=out:start_frame=0:nb_frames=${frames}:alpha=1[fx${name}${key}];[fxin${name}${key}][fx${name}${key}]overlay=shortest=1:format=auto`;
+ };
+ layer('fog', `0x${IMAGE_EFFECT_FOG_COLOR.slice(1)}`, effectSpec.developSec * FOG_SHARE);
+ layer('flash', 'white', FLASH_SEC);
+ return `${chain},format=yuv420p`;
+}
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index 905567d63e..6100b4522e 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -53,6 +53,7 @@ import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';
import { MUSIC_VIDEO_MEDIA } from './musicVideoMediumPlan.js';
import { MUSIC_VIDEO_FRAME_FITS, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';
import { MUSIC_VIDEO_TRANSITIONS } from './musicVideoTransitions.js';
+import { IMAGE_EFFECT_SEC_RANGE, MUSIC_VIDEO_IMAGE_EFFECTS } from './musicVideoImageEffects.js';
import { CAMERA_FRAMINGS, CAMERA_MOVEMENT_VALUES, CAMERA_SPEEDS } from './cameraMovements.js';
import { MUSIC_VIDEO_PERFORMANCE_STYLES, MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';
import {
@@ -607,6 +608,10 @@ const sceneLayerFields = {
// A VHS glitch over the cut into / out of this shot (musicVideoTransitions.js); null clears it.
transitionIn: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),
transitionOut: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),
+ // A code-drawn effect a still shot plays instead of a clip (musicVideoImageEffects.js) and how long
+ // it takes to finish; null clears it.
+ imageEffect: z.enum(MUSIC_VIDEO_IMAGE_EFFECTS).nullable().optional(),
+ imageEffectSec: z.number().min(IMAGE_EFFECT_SEC_RANGE.min).max(IMAGE_EFFECT_SEC_RANGE.max).nullable().optional(),
cardText: z.string().max(500).nullable().optional(),
// #10302: footage that the composition also draws code over; still counts as footage.
codeOverlay: z.boolean().optional(),
diff --git a/server/services/musicVideo/imageEffects.test.js b/server/services/musicVideo/imageEffects.test.js
new file mode 100644
index 0000000000..226b2a42c1
--- /dev/null
+++ b/server/services/musicVideo/imageEffects.test.js
@@ -0,0 +1,102 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { findFfmpeg } from '../../lib/ffmpeg.js';
+import { imageEffectState, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
+import { buildMusicVideoFfmpegArgs } from './render.js';
+import { applySceneUpdate } from './projectsLogic.js';
+
+const ffmpeg = await findFfmpeg();
+const dir = mkdtempSync(join(tmpdir(), 'portos-mv-image-effects-'));
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+const run = (args) => execFileSync(ffmpeg, ['-v', 'error', ...args], { maxBuffer: 64 * 1024 * 1024 });
+const W = 96;
+const H = 64;
+
+// Uniquely catches an effect playing outside a layered still shot, where the render would not draw it.
+describe('sceneImageEffect', () => {
+ const still = { visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 2 };
+ it('plays only on a still shot in a layered composition, with its develop length or the default', () => {
+ expect(sceneImageEffect(still, { layered: true })).toEqual({ effect: 'flash-develop', developSec: 2 });
+ expect(sceneImageEffect({ ...still, imageEffectSec: null }, { layered: true }).developSec).toBe(3);
+ expect(sceneImageEffect(still, { layered: false })).toBeNull();
+ expect(sceneImageEffect({ ...still, visualLayer: 'footage' }, { layered: true })).toBeNull();
+ expect(sceneImageEffect({ ...still, imageEffect: 'unknown' }, { layered: true })).toBeNull();
+ });
+});
+
+// Per-frame mean luma and mean chroma distance from neutral, read back at W x H.
+function frameStats(path) {
+ const raw = run(['-i', path, '-vf', `scale=${W}:${H}`, '-f', 'rawvideo', '-pix_fmt', 'yuv420p', '-']);
+ const size = W * H;
+ const frameBytes = size * 1.5;
+ return Array.from({ length: raw.length / frameBytes }, (_, f) => {
+ const frame = raw.subarray(f * frameBytes, (f + 1) * frameBytes);
+ const luma = frame.subarray(0, size).reduce((sum, v) => sum + v, 0) / size;
+ const chroma = frame.subarray(size).reduce((sum, v) => sum + Math.abs(v - 128), 0) / (size / 2);
+ return { luma, chroma };
+ });
+}
+
+// Uniquely catches the effect's filter graph failing in ffmpeg, drifting from the
+// curves the animatic paints, or adding time to the shot.
+describe.skipIf(!ffmpeg)('camera flash develop in the final render', () => {
+ it('opens white, develops through the grey-green emulsion and holds the still, in the shot\'s own frames', () => {
+ const song = join(dir, 'song.wav');
+ const imagePath = join(dir, 'still.png');
+ run(['-f', 'lavfi', '-i', `color=0xd02020:s=${W}x${H}`, '-frames:v', '1', '-y', imagePath]);
+ run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=3', '-y', song]);
+ const effect = { effect: 'flash-develop', developSec: 1 };
+ const clip = { sceneId: 's1', layer: 'still', imagePath, move: 'hold', inSec: 0, outSec: 3, duration: 3, sourceSec: 3, effect };
+ const output = join(dir, 'develop.mp4');
+ const plan = buildMusicVideoFfmpegArgs([clip], song, output, { audioDurationSec: 3 });
+ run(plan.args.filter((arg, i) => !(arg === '-progress' || plan.args[i - 1] === '-progress')));
+ const stats = frameStats(output);
+ expect(stats).toHaveLength(3 * 24); // a render with no footage runs at 24 fps
+ const held = stats[stats.length - 1];
+ // Frame 0 is the flash: near white, no colour.
+ expect(stats[0].luma).toBeGreaterThan(230);
+ expect(stats[0].chroma).toBeLessThan(6);
+ // Just after the flash the emulsion hides the colour; it fills in as the photo develops.
+ const early = stats[Math.round(0.35 * 24)];
+ expect(early.luma).toBeLessThan(held.luma + 30);
+ expect(early.chroma).toBeLessThan(held.chroma * 0.25);
+ expect(stats[Math.round(0.7 * 24)].chroma).toBeGreaterThan(early.chroma);
+ // From the end of the develop it holds the still exactly.
+ expect(Math.abs(stats[24].chroma - held.chroma)).toBeLessThan(2);
+ expect(Math.abs(stats[60].luma - held.luma)).toBeLessThan(1);
+ // The animatic paints the same moments.
+ expect(imageEffectState(effect, 0)).toMatchObject({ flash: 1, fog: 1, saturation: 0 });
+ expect(imageEffectState(effect, 1)).toEqual({ flash: 0, fog: 0, contrast: 1, saturation: 1 });
+ });
+
+ it('picks up a shot cut into pieces where the piece before it left off', () => {
+ const song = join(dir, 'song2.wav');
+ const imagePath = join(dir, 'still2.png');
+ run(['-f', 'lavfi', '-i', `color=0x2040d0:s=${W}x${H}`, '-frames:v', '1', '-y', imagePath]);
+ run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=1', '-y', song]);
+ const effect = { effect: 'flash-develop', developSec: 2 };
+ const output = join(dir, 'piece.mp4');
+ const plan = buildMusicVideoFfmpegArgs([{ sceneId: 's1', layer: 'still', imagePath, move: 'hold', inSec: 0, outSec: 1, duration: 1, sourceSec: 1, effect, effectAtSec: 1 }],
+ song, output, { audioDurationSec: 1 });
+ run(plan.args.filter((arg, i) => !(arg === '-progress' || plan.args[i - 1] === '-progress')));
+ const first = frameStats(output)[0];
+ // A second in, the flash is long gone and the picture is part-developed.
+ expect(first.luma).toBeLessThan(200);
+ expect(first.chroma).toBeGreaterThan(2);
+ });
+});
+
+// Uniquely catches the scene PATCH refusing the effect fields or an out-of-range develop.
+describe('a shot\'s image effect on the board', () => {
+ const project = { id: 'mv-1', scenes: [{ sceneId: 's1', order: 0, startSec: 0, endSec: 4, takes: [] }] };
+ it('saves an effect on a still, clears it with null and refuses an unknown effect or develop length', () => {
+ const { project: set } = applySceneUpdate(project, 's1', { visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 4 });
+ expect(set.scenes[0]).toMatchObject({ visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 4 });
+ expect(applySceneUpdate(set, 's1', { imageEffect: null }).project.scenes[0].imageEffect).toBeNull();
+ expect(() => applySceneUpdate(project, 's1', { imageEffect: 'sparkle' })).toThrow();
+ expect(() => applySceneUpdate(project, 's1', { imageEffectSec: 120 })).toThrow();
+ });
+});
diff --git a/server/services/musicVideo/intercut.js b/server/services/musicVideo/intercut.js
index 82c90a86ed..9c1fe09ac4 100644
--- a/server/services/musicVideo/intercut.js
+++ b/server/services/musicVideo/intercut.js
@@ -184,7 +184,8 @@ function insertGraphicCards(clips, ranked, period, lyricCues, accentColor) {
function sliceClip(clip, offset, length) {
const duration = round3(length);
- if (clip.layer === 'still') return { ...clip, inSec: 0, outSec: duration, duration, sourceSec: duration };
+ // A still's image effect picks up where the piece before it left off.
+ if (clip.layer === 'still') return { ...clip, inSec: 0, outSec: duration, duration, sourceSec: duration, ...(clip.effect ? { effectAtSec: round3((clip.effectAtSec || 0) + offset) } : {}) };
const inSec = round3(clip.inSec + offset);
return { ...clip, inSec, outSec: round3(inSec + duration), duration,
...(clip.loop === false ? { sourceSec: round3(clip.inSec + (clip.sourceSec ?? clip.duration) - inSec) } : {}) };
@@ -254,7 +255,7 @@ export function intercutClips(clips, { scenes = [], sections = [], beats = [], b
const len = round3(bounds[k + 1] - a);
const partner = k % 2 === 1 ? takePartner(host, len) : null;
if (partner) { out.push(partner); continue; }
- if (still) { out.push({ ...clip, inSec: 0, outSec: len, duration: len, sourceSec: len }); continue; }
+ if (still) { out.push({ ...clip, inSec: 0, outSec: len, duration: len, sourceSec: len, ...(clip.effect ? { effectAtSec: round3(a - t0) } : {}) }); continue; }
// The host keeps its own continuous time (a performance stays in sync).
const inSec = round3(clip.inSec + (a - t0));
out.push({ ...clip, inSec, outSec: round3(inSec + len), duration: len, ...(clip.loop === false ? { sourceSec: round3(clip.inSec + (clip.sourceSec ?? clip.duration) - inSec) } : {}) });
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index ed290c277c..f68dab0401 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -517,6 +517,8 @@ function buildScene(input, { order }) {
...(input.frameFit ? { frameFit: input.frameFit } : {}),
...(input.transitionIn ? { transitionIn: input.transitionIn } : {}),
...(input.transitionOut ? { transitionOut: input.transitionOut } : {}),
+ ...(input.imageEffect ? { imageEffect: input.imageEffect } : {}),
+ ...(input.imageEffectSec ? { imageEffectSec: input.imageEffectSec } : {}),
cardColor: input.cardColor ?? null,
// #8977: cutaway (any image-to-video lane) unless the director asks for a
// lip-synced performance shot, which only a source-audio provider renders.
@@ -600,6 +602,7 @@ export function replaceScenes(project, sceneInputs) {
...(prior.frameFit ? { frameFit: prior.frameFit } : {}),
...(prior.transitionIn ? { transitionIn: prior.transitionIn } : {}),
...(prior.transitionOut ? { transitionOut: prior.transitionOut } : {}),
+ ...(prior.imageEffect ? { imageEffect: prior.imageEffect, ...(prior.imageEffectSec ? { imageEffectSec: prior.imageEffectSec } : {}) } : {}),
takes: prior.takes || [],
...(prior.direction ? { direction: prior.direction } : {}),
};
@@ -607,10 +610,12 @@ export function replaceScenes(project, sceneInputs) {
return { project: touch(project, { scenes }), scenes, replaced: existing.length };
}
-// Scene fields a director edits within an approved shot (its span, wording and
-// selected takes) without adding a shot or lyric the storyboard approval did not cover.
+// Scene fields a director edits within an approved shot (its span, wording, selected
+// takes, and whether it plays as a clip or its still with an image effect) without
+// adding a shot or lyric the storyboard approval did not cover.
const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',
- 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut'];
+ 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut',
+ 'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec'];
/**
* Apply a patch to a single scene. Returns `{ project, updated }`. Throws if the
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 8a4a1b5f22..892efc9a64 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -58,6 +58,7 @@ import { projectSoundBed } from './soundBed.js';
import { intercutClips } from './intercut.js';
import { musicVideoTransitionFilter, sceneTransitions, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';
import { stillMoveForCamera } from '../../lib/cameraMovements.js';
+import { musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
// Per-project render mutex (keyed by projectId so two projects can render in
// parallel; same-project re-entry returns 409 with the live jobId for re-attach).
@@ -251,7 +252,9 @@ export async function resolveSceneClips(project, { layered = false, stillForMiss
const imagePath = scene.referenceImageId ? safeUnder(PATHS.images, scene.referenceImageId) : null;
if (!imagePath || !existsSync(imagePath)) { missingStills.push(scene.sceneId); continue; }
// A planned camera move (#10589) picks the nearest composed still move.
- clips.push({ ...section, imagePath, move: stillMoveForCamera(scene.camera) || scene.stillMove || 'hold' });
+ // An image effect (a camera flash that develops, musicVideoImageEffects.js) plays over the still.
+ const effect = sceneImageEffect(scene, { layered });
+ clips.push({ ...section, imagePath, move: stillMoveForCamera(scene.camera) || scene.stillMove || 'hold', ...(effect ? { effect } : {}) });
continue;
}
const entry = historyMap.get(scene.videoHistoryId);
@@ -428,7 +431,8 @@ function sectionChain(c, input, { canonW, canonH, fps, frames, frameGrid }) {
? `zoompan=z='1+${STILL_PUSH_ZOOM}*on/${frames}':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=${canonW}x${canonH}:fps=${fps},setsar=1,`
: c.move === 'pan' ? `crop=${canonW}:${canonH}:x='(iw-ow)*n/${Math.max(1, frames - 1)}':y='(ih-oh)/2',` : '';
return `[${input}:v]fps=${fps},scale=${coverW}:${coverH}:force_original_aspect_ratio=increase,crop=${coverW}:${coverH},setsar=1,`
- + `${move}format=yuv420p,${trim},setpts=PTS-STARTPTS`;
+ + `${move}format=yuv420p,${trim},setpts=PTS-STARTPTS`
+ + musicVideoImageEffectFilter(c.effect, { width: canonW, height: canonH, fps, key: input, atSec: c.effectAtSec || 0 });
}
// A non-looping shot a hair longer than its source holds the last frame for
// the remainder instead of ending early and drifting every later cut. On the