Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
feat: image effect shots, starting with camera flash develop
Claudec1542b4c80f0
diff --git a/client/src/components/musicVideo/ImageEffectStill.jsx b/client/src/components/musicVideo/ImageEffectStill.jsxnew file mode 100644index 0000000000..c59064749b--- /dev/null+++ b/client/src/components/musicVideo/ImageEffectStill.jsx@@ -0,0 +1,51 @@+import { useCallback, useEffect, useRef } from 'react';+import { IMAGE_EFFECT_FOG_COLOR, imageEffectState } from '../../lib/musicVideoImageEffects.js';++/**+ * A still shot's image effect in the storyboard animatic: the still, cover-fitted+ * as the final render crops it, with the effect's flash and emulsion laid over it+ * and its contrast and colour filtered, all from the same curves the render uses+ * (`imageEffectState`). Paused it paints the frame at `time`; playing it repaints+ * every animation frame from the song's clock, writing styles directly so the+ * board does not re-render per frame.+ */+export default function ImageEffectStill({ src, effect, startSec, audioRef, time }) {+ const imgRef = useRef(null);+ const fogRef = useRef(null);+ const flashRef = useRef(null);+ const paint = useCallback((t) => {+ const state = imageEffectState(effect, t - startSec);+ if (imgRef.current) imgRef.current.style.filter = `contrast(${state.contrast}) saturate(${state.saturation})`;+ if (fogRef.current) fogRef.current.style.opacity = String(state.fog);+ if (flashRef.current) flashRef.current.style.opacity = String(state.flash);+ }, [effect, startSec]);+ useEffect(() => { paint(time); }, [time, paint]);+ useEffect(() => {+ const audio = audioRef.current;+ if (!audio) return undefined;+ let handle = 0;+ const tick = () => {+ paint(audio.currentTime || 0);+ handle = requestAnimationFrame(tick);+ };+ const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };+ const stop = () => { cancelAnimationFrame(handle); paint(audio.currentTime || 0); };+ audio.addEventListener('play', start);+ audio.addEventListener('pause', stop);+ audio.addEventListener('ended', stop);+ if (!audio.paused) start();+ return () => {+ cancelAnimationFrame(handle);+ audio.removeEventListener('play', start);+ audio.removeEventListener('pause', stop);+ audio.removeEventListener('ended', stop);+ };+ }, [audioRef, paint]);+ return (+ <div className="relative h-full w-full" data-testid="animatic-image-effect">+ <img ref={imgRef} src={src} alt="" className="h-full w-full object-cover" />+ <div ref={fogRef} className="pointer-events-none absolute inset-0" style={{ background: IMAGE_EFFECT_FOG_COLOR, opacity: 0 }} />+ <div ref={flashRef} className="pointer-events-none absolute inset-0" style={{ background: '#fff', opacity: 0 }} />+ </div>+ );+}diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsxindex a9ff8c4798..bf91b363cc 100644--- a/client/src/components/musicVideo/SceneCard.jsx+++ b/client/src/components/musicVideo/SceneCard.jsx@@ -35,6 +35,7 @@ import SceneRenderTarget, { lastRenderTarget, renderTargetOverrides } from './Scimport SceneRenderInputs, { shotCardRenderRequest, shotCardRequestProblem } from './SceneRenderInputs.jsx';import SceneFrameFit, { sceneFrameFit } from './SceneFrameFit.jsx';import SceneTransitions from './SceneTransitions.jsx';+import SceneImageEffect from './SceneImageEffect.jsx';// The two timeline-bound scene fields rendered as identical number inputs.const SCENE_TIME_FIELDS = [['Start', 'startSec'], ['End', 'endSec']];@@ -461,6 +462,11 @@ export default function SceneCard({{/* Clip: the motion prompt, where this shot renders and what it costs, then its clip (Phase 1). */}<section aria-label="Render" className="min-w-0 space-y-2"><h4 className="text-xs font-medium">Clip</h4>+ {/* A clip, the still, or the still playing a code-drawn image effect (no model call). */}+ {layered && (layer === 'footage' || layer === 'still') && (+ <SceneImageEffect fieldId={fieldId} scene={scene} onChange={applyPatch}+ disabled={shotLock?.clip === 'locked'} disabledReason={shotLock?.clip === 'locked' ? 'Video locked. Unlock the video to change it.' : null} />+ )}{scene.videoHistoryId && performance && (<PerformanceEvidence key={scene.videoHistoryId} clipSrc={clipSrc} instruction={instruction}temporal={performanceReview?.shot} excerptStartSec={performanceReview?.excerptStartSec || 0}diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsxindex 29483ee070..0454234177 100644--- a/client/src/components/musicVideo/SceneCard.test.jsx+++ b/client/src/components/musicVideo/SceneCard.test.jsx@@ -468,3 +468,26 @@ describe('VHS glitch transitions', () => {expect(screen.getByLabelText('Glitch in').value).toBe('');});});++// Uniquely catches the picture picker saving the wrong layer or effect fields, or offering the effect where the render ignores it.+describe('image effect picker', () => {+ it('turns a shot into a still playing camera flash develop, sets its develop length, and back to a clip', () => {+ const onSave = vi.fn();+ function Harness({ layered = true }) {+ const [scene, setScene] = useState({ sceneId: 's1', startSec: 0, endSec: 4, referenceImageId: 'burger.png', takes: [] });+ return <SceneCard scene={scene} index={0} layered={layered} onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;+ }+ const { unmount } = render(<Harness />);+ const picture = screen.getByLabelText('Picture');+ fireEvent.change(picture, { target: { value: 'flash-develop' } });+ expect(onSave).toHaveBeenLastCalledWith('s1', { visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 3 });+ fireEvent.change(screen.getByLabelText('Develops in'), { target: { value: '6' } });+ expect(onSave).toHaveBeenLastCalledWith('s1', { imageEffectSec: 6 });+ expect(screen.getByText('The shot ends at 4.0s, before the photo finishes developing.')).toBeTruthy();+ fireEvent.change(picture, { target: { value: 'clip' } });+ expect(onSave).toHaveBeenLastCalledWith('s1', { visualLayer: 'footage', imageEffect: null });+ unmount();+ render(<Harness layered={false} />);+ expect(screen.queryByLabelText('Picture')).toBeNull();+ });+});diff --git a/client/src/components/musicVideo/SceneImageEffect.jsx b/client/src/components/musicVideo/SceneImageEffect.jsxnew file mode 100644index 0000000000..2e6c2cf763--- /dev/null+++ b/client/src/components/musicVideo/SceneImageEffect.jsx@@ -0,0 +1,63 @@+import { safeReadStorage, safeWriteStorage } from '../../lib/safeStorage.js';+import { DEFAULT_IMAGE_EFFECT_SEC, IMAGE_EFFECT_DEVELOP_SECS, MUSIC_VIDEO_IMAGE_EFFECTS, MUSIC_VIDEO_IMAGE_EFFECT_LABELS } from '../../lib/musicVideoImageEffects.js';++// The develop length picked last on this device, for the next shot that takes an effect.+const LAST_SEC_KEY = 'portos-mv-image-effect-sec';+const lastDevelopSec = () => {+ const sec = Number(safeReadStorage(LAST_SEC_KEY));+ return IMAGE_EFFECT_DEVELOP_SECS.includes(sec) ? sec : DEFAULT_IMAGE_EFFECT_SEC;+};++/**+ * Where a shot's picture comes from in a composed render: a rendered clip, its+ * still, or its still playing a code-drawn image effect (a camera flash that+ * develops like an instant photo), which needs no model. An effect's develop+ * length rides beside it. The effect plays inside the shot's span and adds no time.+ */+export default function SceneImageEffect({ fieldId, scene, onChange, disabled = false, disabledReason = null }) {+ const effect = scene.visualLayer === 'still' && MUSIC_VIDEO_IMAGE_EFFECTS.includes(scene.imageEffect) ? scene.imageEffect : null;+ const value = effect || (scene.visualLayer === 'still' ? 'still' : 'clip');+ const developSec = IMAGE_EFFECT_DEVELOP_SECS.includes(scene.imageEffectSec) ? scene.imageEffectSec : DEFAULT_IMAGE_EFFECT_SEC;+ const spanSec = typeof scene.startSec === 'number' && typeof scene.endSec === 'number' ? scene.endSec - scene.startSec : null;+ const pick = (next) => {+ if (next === 'clip') onChange({ visualLayer: 'footage', ...(scene.imageEffect ? { imageEffect: null } : {}) });+ else if (next === 'still') onChange({ visualLayer: 'still', ...(scene.imageEffect ? { imageEffect: null } : {}) });+ else onChange({ visualLayer: 'still', imageEffect: next, imageEffectSec: scene.imageEffectSec || lastDevelopSec() });+ };+ const pickSec = (sec) => {+ safeWriteStorage(LAST_SEC_KEY, String(sec));+ onChange({ imageEffectSec: sec });+ };+ return (+ <div className="space-y-1 text-xs" data-testid="scene-image-effect">+ <div className="flex flex-wrap items-center gap-x-3 gap-y-1">+ <span className="flex min-w-0 items-center gap-1">+ <label htmlFor={fieldId('picture')} className={effect ? 'text-port-accent' : 'text-port-text-muted'}>Picture</label>+ <select id={fieldId('picture')} value={value} disabled={disabled} title={disabledReason || undefined}+ onChange={(e) => pick(e.target.value)}+ className="min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50">+ <option value="clip">Video clip</option>+ <option value="still">Still image</option>+ {MUSIC_VIDEO_IMAGE_EFFECTS.map((id) => <option key={id} value={id}>{MUSIC_VIDEO_IMAGE_EFFECT_LABELS[id]}</option>)}+ </select>+ </span>+ {effect && (+ <span className="flex min-w-0 items-center gap-1">+ <label htmlFor={fieldId('develop')} className="text-port-text-muted">Develops in</label>+ <select id={fieldId('develop')} value={developSec} onChange={(e) => pickSec(Number(e.target.value))}+ className="min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0">+ {IMAGE_EFFECT_DEVELOP_SECS.map((sec) => <option key={sec} value={sec}>{sec}s</option>)}+ </select>+ </span>+ )}+ </div>+ {effect && (+ <p className="text-[11px] text-port-text-muted">+ {spanSec != null && developSec > spanSec+ ? `The shot ends at ${spanSec.toFixed(1)}s, before the photo finishes developing.`+ : 'No video render: the still plays the effect.'}+ </p>+ )}+ </div>+ );+}diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsxindex d2045bfcf5..6f8ee15cf3 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx@@ -13,6 +13,9 @@ import ShotStepper from './ShotStepper.jsx';import FrameStepper, { captureMediaFrame } from './FrameStepper.jsx';import { sceneFrameFit } from './SceneFrameFit.jsx';import TransitionGlitch from './TransitionGlitch.jsx';+import ImageEffectStill from './ImageEffectStill.jsx';+import { isLayeredComposition } from '../../lib/musicVideoLayers.js';+import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);@@ -284,9 +287,13 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, colconst shot = shotAt(scenes, time);const timed = timedShots(scenes);const index = shot ? timed.indexOf(shot) : -1;+ // A still shot with an image effect plays the effect over its still, as the composed render does, not its clip.+ const layered = isLayeredComposition(project);+ const shotEffect = shot ? sceneImageEffect(shot, { layered }) : null;// The shot on screen and the next few: each keeps one mounted clip, so the cut to the// next shot shows a clip that has already loaded and sits on its first frame.- const clipShots = timed.slice(Math.max(index, 0), Math.max(index, 0) + 1 + CLIP_PRELOAD_SHOTS).filter((scene) => scene.videoHistoryId);+ const clipShots = timed.slice(Math.max(index, 0), Math.max(index, 0) + 1 + CLIP_PRELOAD_SHOTS)+ .filter((scene) => scene.videoHistoryId && !sceneImageEffect(scene, { layered }));const label = shot ? shot.label || `Shot ${index + 1}` : 'No shots yet';// With the lyrics drawn over the picture, a shot without a frame keeps its label and// intent under the picture instead, so no stand-in text sits under the sung words.@@ -299,7 +306,10 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col<figure ref={figureRef} aria-label="Storyboard animatic" className="relative aspect-video w-full overflow-hidden rounded border border-port-border bg-black">{/* The project's film look, as the final render applies it over the whole picture, lyrics included. */}<FilmLookFrame look={lookOn ? project.filmLook : null} frame={Math.round(time * ANIMATIC_FPS)} playing={playing} className="h-full w-full">- {shot?.referenceImageId ? (+ {shot?.referenceImageId && shotEffect ? (+ <ImageEffectStill key={shot.sceneId} src={`/data/images/${shot.referenceImageId}`} effect={shotEffect}+ startSec={shot.startSec} audioRef={audioRef} time={time} />+ ) : shot?.referenceImageId ? (// Behind a VHS-edged clip the still stands in, blurred and faded, for the render's blur of the clip.<img src={`/data/images/${shot.referenceImageId}`} alt=""className={`h-full w-full ${vhsEdges ? 'object-cover blur-md saturate-50 brightness-75' : stillEdges ? 'object-cover' : 'object-contain'}`} />diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsxindex 68d42554bb..97444e9269 100644--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx@@ -27,6 +27,21 @@ describe('StoryboardAnimatic', () => {expect(container.querySelector('img').getAttribute('src')).toBe('/data/images/frame-b.png');expect(screen.getByText('Shot 2 / 2')).toBeTruthy();});++ // Uniquely catches the animatic showing a clip where the composed render plays a still's image effect.+ it('plays a still shot\'s camera flash develop over its still, not its clip, from the shot\'s start', () => {+ const effectShot = { sceneId: 'fx', label: 'Burger', startSec: 0, endSec: 6, referenceImageId: 'burger.png', videoHistoryId: 'clip-1',+ visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 2 };+ render(<StoryboardAnimatic project={{ id: 'p', composition: { mode: 'composed' }, scenes: [effectShot] }} audioUrl="/song.mp3" />);+ const effect = screen.getByTestId('animatic-image-effect');+ expect(screen.queryByLabelText('Clip for Burger')).toBeNull();+ const img = effect.querySelector('img');+ expect(img.style.filter).toBe('contrast(0.55) saturate(0)');+ const audio = screen.getByLabelText('Song for the storyboard animatic');+ Object.defineProperty(audio, 'currentTime', { value: 3, configurable: true });+ fireEvent.timeUpdate(audio);+ expect(img.style.filter).toBe('contrast(1) saturate(1)');+ });});vi.mock('../../services/apiMusicVideo.js', () => ({diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex adf90c262f..b38f8cac67 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -317,4 +317,5 @@ grep -i "what you want to do" client/src/lib/README.md| `musicVideoGrade.js` | Shared Music Video grade preset and grain bounds for the composition controls. || `filmLook.js` | Shared film look controls, presets, normalizer, prompt vocabulary, the SVG filter markup the editor previews with and the playback-time motion layers (server `lib/filmLook.js`). || `musicVideoNarrativeEvents.js` | Named narrative event kinds, absolute onset/word/frame resolution and deterministic silence/reactive state for composition documents. |+| `musicVideoImageEffects.js` | Music-video image effects (a camera flash that develops over a still shot), re-exported from `server/lib/musicVideoImageEffects.js`: the effects, labels and develop lengths, `sceneImageEffect`, `imageEffectState`. Used by `components/musicVideo/SceneImageEffect.jsx` and the storyboard animatic. || `musicVideoTransitions.js` | Music-video cut transitions (VHS glitches over a cut), re-exported from `server/lib/musicVideoTransitions.js`: the styles and labels, `sceneTransitionSections`, `transitionGlitchFrames`, `transitionFrameAt`. Used by `components/musicVideo/SceneTransitions.jsx` and the storyboard animatic. |diff --git a/client/src/lib/index.js b/client/src/lib/index.jsindex 14ecfcdd98..a4331c2d62 100644--- a/client/src/lib/index.js+++ b/client/src/lib/index.js@@ -278,3 +278,4 @@ export * from './musicVideoGrade.js';export * from './filmLook.js';export * from './musicVideoNarrativeEvents.js';export * from './musicVideoTransitions.js';+export * from './musicVideoImageEffects.js';diff --git a/client/src/lib/musicVideoImageEffects.js b/client/src/lib/musicVideoImageEffects.jsnew file mode 100644index 0000000000..923e8a7d4c--- /dev/null+++ b/client/src/lib/musicVideoImageEffects.js@@ -0,0 +1,14 @@+/**+ * Music Video image effects — re-export of `server/lib/musicVideoImageEffects.js`,+ * so the shot card's picker and the animatic paint the same effect as the final render.+ */+export {+ DEFAULT_IMAGE_EFFECT_SEC,+ IMAGE_EFFECT_DEVELOP_SECS,+ IMAGE_EFFECT_FOG_COLOR,+ IMAGE_EFFECT_SEC_RANGE,+ MUSIC_VIDEO_IMAGE_EFFECTS,+ MUSIC_VIDEO_IMAGE_EFFECT_LABELS,+ imageEffectState,+ sceneImageEffect,+} from '../../../server/lib/musicVideoImageEffects.js';diff --git a/server/lib/README.md b/server/lib/README.mdindex 442c26da64..00067f6839 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -560,6 +560,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `chiptuneRender.js` | Deterministic chiptune score → mono PCM → 16-bit WAV buffer (pure Node, no audio deps); `renderScoreToPcm` / `pcmToWavBuffer` / `renderScoreToWav`. || `chiptuneScore.js` | Chiptune score contract (#2911): `chiptuneScoreSchema` (Zod) + `sanitizeChiptuneScore`, `buildScoreEvents` — the pattern/order → absolute-time flatten mirrored by `client/src/lib/chiptunePlayback.js`. || `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). |+| `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`, first `flash-develop`: a camera flash, then the picture develops like an instant photo over `imageEffectSec`). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe. || `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections` and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Browser-safe; the storyboard animatic paints the same frames. || `pitchMath.js` | Dependency-free scientific-pitch math shared by the LLM-authored music formats on both runtimes: `pitchToMidi("C4")` → 60 (null when unparseable) and `midiToFreq(69)` → 440. || `wavAudioFile.js` | `writeWavAudioFile(wav, dir, basename)` — land a synthesized WAV Buffer as `<basename>.ogg` via the system ffmpeg, falling back to `<basename>.wav` (and dropping a stale `.ogg`) when ffmpeg is missing or the encode fails; resolves to the filename used. Shared by the chiptune and drawn-waveform renderers and the code-engine take upload. `wavDurationMs(buffer)` reads a PCM WAV's duration from its RIFF header (0 for anything that is not a parseable WAV). `measureWavAudio(buffer)` decodes PCM 16/24/32-bit or float 32/64 (EXTENSIBLE included) into `{ channels, sampleRate, frames, durationMs, peak, rms, nonFinite }` so native renders are judged by their samples (null when undecodable). |diff --git a/server/lib/index.js b/server/lib/index.jsindex 0e281bc763..cbde30e831 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -878,3 +878,4 @@ export * from './deepAuditCoverage.js';export * from './auditWorkflow.js';export * from './musicVideoTransitions.js';+export * from './musicVideoImageEffects.js';diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.jsindex 072844a60a..61e9d6ade3 100644--- a/server/lib/musicVideoDependencies.js+++ b/server/lib/musicVideoDependencies.js@@ -21,6 +21,8 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s.concat(scene.frameFit && scene.frameFit !== 'fill' ? [['frameFit', scene.frameFit]] : [])// A glitch over its cuts changes the render; a shot without one keeps its evidence..concat(['transitionIn', 'transitionOut'].filter((key) => scene[key]).map((key) => [key, scene[key]]))+ // An image effect over a still changes the render; a still without one keeps its evidence.+ .concat(scene.imageEffect ? [['imageEffect', scene.imageEffect], ['imageEffectSec', scene.imageEffectSec ?? null]] : [])// Only a music-driven performance joins it, so lip-sync evidence stays current..concat(scene.performanceStyle === 'music' ? [['performanceStyle', 'music']] : []))) });const snapshot = (references) => ({ version: MUSIC_VIDEO_DEPENDENCY_VERSION, references });diff --git a/server/lib/musicVideoImageEffects.js b/server/lib/musicVideoImageEffects.jsnew file mode 100644index 0000000000..e2050fd606--- /dev/null+++ b/server/lib/musicVideoImageEffects.js@@ -0,0 +1,100 @@+/**+ * Music Video image effects: a still shot that plays a code-drawn effect over+ * its still instead of a generated clip, so no model is called for it. The+ * shot is a still section (`visualLayer: 'still'`) that also names an+ * `imageEffect`; only a layered (composed) render honours it, as it does the+ * still itself.+ *+ * "Camera flash develop" (`flash-develop`): the shot opens on a white camera+ * flash, then the picture develops like an instant photo over+ * `imageEffectSec` seconds: a murky grey-green emulsion clears, the contrast+ * comes up and the colour fills in last, and the still holds fully developed+ * for the rest of the shot. The effect never adds time: it plays inside the+ * shot's own span, and a develop longer than the shot ends the shot part-way.+ *+ * One definition serves both players: every value is a pure function of the+ * time into the shot (`imageEffectState`), the storyboard animatic paints it+ * with CSS over the still, and the final render turns the same curves into+ * ffmpeg filters (`musicVideoImageEffectFilter`). Each curve is linear or a+ * square of a linear ramp so the two read the same frame for frame.+ */++export const MUSIC_VIDEO_IMAGE_EFFECTS = Object.freeze(['flash-develop']);++export const MUSIC_VIDEO_IMAGE_EFFECT_LABELS = Object.freeze({+ 'flash-develop': 'Camera flash develop',+});++/** The develop lengths the shot card offers, in seconds. */+export const IMAGE_EFFECT_DEVELOP_SECS = Object.freeze([1, 2, 3, 4, 6, 8]);+export const DEFAULT_IMAGE_EFFECT_SEC = 3;+export const IMAGE_EFFECT_SEC_RANGE = Object.freeze({ min: 0.5, max: 30 });++// The flash fades from full white to nothing over this long.+const FLASH_SEC = 0.25;+// The undeveloped emulsion's colour, and the share of the develop it takes to clear.+export const IMAGE_EFFECT_FOG_COLOR = '#4a5452';+const FOG_SHARE = 0.75;+// Contrast starts here and rises to 1 as the picture develops.+const START_CONTRAST = 0.55;++const isEffect = (value) => MUSIC_VIDEO_IMAGE_EFFECTS.includes(value);+const clampSec = (sec) => Math.min(IMAGE_EFFECT_SEC_RANGE.max, Math.max(IMAGE_EFFECT_SEC_RANGE.min, sec));++/**+ * The effect a scene plays, `{ effect, developSec }`, or null. Only a still+ * shot in a layered composition plays one: anywhere else the shot shows its+ * footage as before.+ */+export function sceneImageEffect(scene, { layered = false } = {}) {+ if (!layered || scene?.visualLayer !== 'still' || !isEffect(scene?.imageEffect)) return null;+ const sec = Number(scene.imageEffectSec);+ return { effect: scene.imageEffect, developSec: Number.isFinite(sec) && sec > 0 ? clampSec(sec) : DEFAULT_IMAGE_EFFECT_SEC };+}++const ramp = (tSec, sec) => Math.min(1, Math.max(0, tSec / sec));++/**+ * The effect's state `tSec` seconds into the shot: `flash` and `fog` are the+ * opacities of the white flash and the emulsion colour over the picture, and+ * `contrast` / `saturation` the picture's own (1 = as shot).+ */+export function imageEffectState({ effect, developSec }, tSec) {+ if (effect !== 'flash-develop') return { flash: 0, fog: 0, contrast: 1, saturation: 1 };+ const t = Math.max(0, tSec);+ const progress = ramp(t, developSec);+ return {+ flash: 1 - ramp(t, FLASH_SEC),+ fog: 1 - ramp(t, developSec * FOG_SHARE),+ contrast: START_CONTRAST + (1 - START_CONTRAST) * progress,+ saturation: progress * progress,+ };+}++const num = (n) => String(Math.round(n * 10000) / 10000);++/**+ * The ffmpeg filtergraph that plays `effect` over one section, appended to the+ * section's chain after its frames start counting from 0. Returns the text to+ * append (it begins with a comma and ends on the effected stream, unlabelled,+ * so the caller's output label follows), or '' for no effect. `atSec` is how+ * far into the shot the section starts (a shot cut into pieces picks up where+ * its last piece left off); `key` keeps the graph's labels unique per section.+ */+export function musicVideoImageEffectFilter(effectSpec, { width, height, fps, key, atSec = 0 }) {+ if (effectSpec?.effect !== 'flash-develop') return '';+ const startFrame = Math.max(0, atSec) * fps;+ const progress = `min(1,(n+${num(startFrame)})/${num(Math.max(1, effectSpec.developSec * fps))})`;+ let chain = `,eq=contrast='${num(START_CONTRAST)}+${num(1 - START_CONTRAST)}*${progress}':saturation='${progress}*${progress}':eval=frame`;+ // A layer fading out over `sec` from the shot's start: from where it stands at `atSec`, linearly to nothing.+ const layer = (name, color, sec) => {+ const left = sec - Math.max(0, atSec);+ const frames = Math.round(left * fps);+ if (frames < 1) return;+ chain += `[fxin${name}${key}];color=c=${color}@${num(left / sec)}:s=${width}x${height}:r=${fps},format=yuva420p,`+ + `fade=t=out:start_frame=0:nb_frames=${frames}:alpha=1[fx${name}${key}];[fxin${name}${key}][fx${name}${key}]overlay=shortest=1:format=auto`;+ };+ layer('fog', `0x${IMAGE_EFFECT_FOG_COLOR.slice(1)}`, effectSpec.developSec * FOG_SHARE);+ layer('flash', 'white', FLASH_SEC);+ return `${chain},format=yuv420p`;+}diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.jsindex 905567d63e..6100b4522e 100644--- a/server/lib/musicVideoValidation.js+++ b/server/lib/musicVideoValidation.js@@ -53,6 +53,7 @@ import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';import { MUSIC_VIDEO_MEDIA } from './musicVideoMediumPlan.js';import { MUSIC_VIDEO_FRAME_FITS, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';import { MUSIC_VIDEO_TRANSITIONS } from './musicVideoTransitions.js';+import { IMAGE_EFFECT_SEC_RANGE, MUSIC_VIDEO_IMAGE_EFFECTS } from './musicVideoImageEffects.js';import { CAMERA_FRAMINGS, CAMERA_MOVEMENT_VALUES, CAMERA_SPEEDS } from './cameraMovements.js';import { MUSIC_VIDEO_PERFORMANCE_STYLES, MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';import {@@ -607,6 +608,10 @@ const sceneLayerFields = {// A VHS glitch over the cut into / out of this shot (musicVideoTransitions.js); null clears it.transitionIn: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),transitionOut: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),+ // A code-drawn effect a still shot plays instead of a clip (musicVideoImageEffects.js) and how long+ // it takes to finish; null clears it.+ imageEffect: z.enum(MUSIC_VIDEO_IMAGE_EFFECTS).nullable().optional(),+ imageEffectSec: z.number().min(IMAGE_EFFECT_SEC_RANGE.min).max(IMAGE_EFFECT_SEC_RANGE.max).nullable().optional(),cardText: z.string().max(500).nullable().optional(),// #10302: footage that the composition also draws code over; still counts as footage.codeOverlay: z.boolean().optional(),diff --git a/server/services/musicVideo/imageEffects.test.js b/server/services/musicVideo/imageEffects.test.jsnew file mode 100644index 0000000000..226b2a42c1--- /dev/null+++ b/server/services/musicVideo/imageEffects.test.js@@ -0,0 +1,102 @@+import { afterAll, describe, expect, it } from 'vitest';+import { execFileSync } from 'node:child_process';+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { findFfmpeg } from '../../lib/ffmpeg.js';+import { imageEffectState, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';+import { buildMusicVideoFfmpegArgs } from './render.js';+import { applySceneUpdate } from './projectsLogic.js';++const ffmpeg = await findFfmpeg();+const dir = mkdtempSync(join(tmpdir(), 'portos-mv-image-effects-'));+afterAll(() => rmSync(dir, { recursive: true, force: true }));+const run = (args) => execFileSync(ffmpeg, ['-v', 'error', ...args], { maxBuffer: 64 * 1024 * 1024 });+const W = 96;+const H = 64;++// Uniquely catches an effect playing outside a layered still shot, where the render would not draw it.+describe('sceneImageEffect', () => {+ const still = { visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 2 };+ it('plays only on a still shot in a layered composition, with its develop length or the default', () => {+ expect(sceneImageEffect(still, { layered: true })).toEqual({ effect: 'flash-develop', developSec: 2 });+ expect(sceneImageEffect({ ...still, imageEffectSec: null }, { layered: true }).developSec).toBe(3);+ expect(sceneImageEffect(still, { layered: false })).toBeNull();+ expect(sceneImageEffect({ ...still, visualLayer: 'footage' }, { layered: true })).toBeNull();+ expect(sceneImageEffect({ ...still, imageEffect: 'unknown' }, { layered: true })).toBeNull();+ });+});++// Per-frame mean luma and mean chroma distance from neutral, read back at W x H.+function frameStats(path) {+ const raw = run(['-i', path, '-vf', `scale=${W}:${H}`, '-f', 'rawvideo', '-pix_fmt', 'yuv420p', '-']);+ const size = W * H;+ const frameBytes = size * 1.5;+ return Array.from({ length: raw.length / frameBytes }, (_, f) => {+ const frame = raw.subarray(f * frameBytes, (f + 1) * frameBytes);+ const luma = frame.subarray(0, size).reduce((sum, v) => sum + v, 0) / size;+ const chroma = frame.subarray(size).reduce((sum, v) => sum + Math.abs(v - 128), 0) / (size / 2);+ return { luma, chroma };+ });+}++// Uniquely catches the effect's filter graph failing in ffmpeg, drifting from the+// curves the animatic paints, or adding time to the shot.+describe.skipIf(!ffmpeg)('camera flash develop in the final render', () => {+ it('opens white, develops through the grey-green emulsion and holds the still, in the shot\'s own frames', () => {+ const song = join(dir, 'song.wav');+ const imagePath = join(dir, 'still.png');+ run(['-f', 'lavfi', '-i', `color=0xd02020:s=${W}x${H}`, '-frames:v', '1', '-y', imagePath]);+ run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=3', '-y', song]);+ const effect = { effect: 'flash-develop', developSec: 1 };+ const clip = { sceneId: 's1', layer: 'still', imagePath, move: 'hold', inSec: 0, outSec: 3, duration: 3, sourceSec: 3, effect };+ const output = join(dir, 'develop.mp4');+ const plan = buildMusicVideoFfmpegArgs([clip], song, output, { audioDurationSec: 3 });+ run(plan.args.filter((arg, i) => !(arg === '-progress' || plan.args[i - 1] === '-progress')));+ const stats = frameStats(output);+ expect(stats).toHaveLength(3 * 24); // a render with no footage runs at 24 fps+ const held = stats[stats.length - 1];+ // Frame 0 is the flash: near white, no colour.+ expect(stats[0].luma).toBeGreaterThan(230);+ expect(stats[0].chroma).toBeLessThan(6);+ // Just after the flash the emulsion hides the colour; it fills in as the photo develops.+ const early = stats[Math.round(0.35 * 24)];+ expect(early.luma).toBeLessThan(held.luma + 30);+ expect(early.chroma).toBeLessThan(held.chroma * 0.25);+ expect(stats[Math.round(0.7 * 24)].chroma).toBeGreaterThan(early.chroma);+ // From the end of the develop it holds the still exactly.+ expect(Math.abs(stats[24].chroma - held.chroma)).toBeLessThan(2);+ expect(Math.abs(stats[60].luma - held.luma)).toBeLessThan(1);+ // The animatic paints the same moments.+ expect(imageEffectState(effect, 0)).toMatchObject({ flash: 1, fog: 1, saturation: 0 });+ expect(imageEffectState(effect, 1)).toEqual({ flash: 0, fog: 0, contrast: 1, saturation: 1 });+ });++ it('picks up a shot cut into pieces where the piece before it left off', () => {+ const song = join(dir, 'song2.wav');+ const imagePath = join(dir, 'still2.png');+ run(['-f', 'lavfi', '-i', `color=0x2040d0:s=${W}x${H}`, '-frames:v', '1', '-y', imagePath]);+ run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=1', '-y', song]);+ const effect = { effect: 'flash-develop', developSec: 2 };+ const output = join(dir, 'piece.mp4');+ const plan = buildMusicVideoFfmpegArgs([{ sceneId: 's1', layer: 'still', imagePath, move: 'hold', inSec: 0, outSec: 1, duration: 1, sourceSec: 1, effect, effectAtSec: 1 }],+ song, output, { audioDurationSec: 1 });+ run(plan.args.filter((arg, i) => !(arg === '-progress' || plan.args[i - 1] === '-progress')));+ const first = frameStats(output)[0];+ // A second in, the flash is long gone and the picture is part-developed.+ expect(first.luma).toBeLessThan(200);+ expect(first.chroma).toBeGreaterThan(2);+ });+});++// Uniquely catches the scene PATCH refusing the effect fields or an out-of-range develop.+describe('a shot\'s image effect on the board', () => {+ const project = { id: 'mv-1', scenes: [{ sceneId: 's1', order: 0, startSec: 0, endSec: 4, takes: [] }] };+ it('saves an effect on a still, clears it with null and refuses an unknown effect or develop length', () => {+ const { project: set } = applySceneUpdate(project, 's1', { visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 4 });+ expect(set.scenes[0]).toMatchObject({ visualLayer: 'still', imageEffect: 'flash-develop', imageEffectSec: 4 });+ expect(applySceneUpdate(set, 's1', { imageEffect: null }).project.scenes[0].imageEffect).toBeNull();+ expect(() => applySceneUpdate(project, 's1', { imageEffect: 'sparkle' })).toThrow();+ expect(() => applySceneUpdate(project, 's1', { imageEffectSec: 120 })).toThrow();+ });+});diff --git a/server/services/musicVideo/intercut.js b/server/services/musicVideo/intercut.jsindex 82c90a86ed..9c1fe09ac4 100644--- a/server/services/musicVideo/intercut.js+++ b/server/services/musicVideo/intercut.js@@ -184,7 +184,8 @@ function insertGraphicCards(clips, ranked, period, lyricCues, accentColor) {function sliceClip(clip, offset, length) {const duration = round3(length);- if (clip.layer === 'still') return { ...clip, inSec: 0, outSec: duration, duration, sourceSec: duration };+ // A still's image effect picks up where the piece before it left off.+ if (clip.layer === 'still') return { ...clip, inSec: 0, outSec: duration, duration, sourceSec: duration, ...(clip.effect ? { effectAtSec: round3((clip.effectAtSec || 0) + offset) } : {}) };const inSec = round3(clip.inSec + offset);return { ...clip, inSec, outSec: round3(inSec + duration), duration,...(clip.loop === false ? { sourceSec: round3(clip.inSec + (clip.sourceSec ?? clip.duration) - inSec) } : {}) };@@ -254,7 +255,7 @@ export function intercutClips(clips, { scenes = [], sections = [], beats = [], bconst len = round3(bounds[k + 1] - a);const partner = k % 2 === 1 ? takePartner(host, len) : null;if (partner) { out.push(partner); continue; }- if (still) { out.push({ ...clip, inSec: 0, outSec: len, duration: len, sourceSec: len }); continue; }+ if (still) { out.push({ ...clip, inSec: 0, outSec: len, duration: len, sourceSec: len, ...(clip.effect ? { effectAtSec: round3(a - t0) } : {}) }); continue; }// The host keeps its own continuous time (a performance stays in sync).const inSec = round3(clip.inSec + (a - t0));out.push({ ...clip, inSec, outSec: round3(inSec + len), duration: len, ...(clip.loop === false ? { sourceSec: round3(clip.inSec + (clip.sourceSec ?? clip.duration) - inSec) } : {}) });diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.jsindex ed290c277c..f68dab0401 100644--- a/server/services/musicVideo/projectsLogic.js+++ b/server/services/musicVideo/projectsLogic.js@@ -517,6 +517,8 @@ function buildScene(input, { order }) {...(input.frameFit ? { frameFit: input.frameFit } : {}),...(input.transitionIn ? { transitionIn: input.transitionIn } : {}),...(input.transitionOut ? { transitionOut: input.transitionOut } : {}),+ ...(input.imageEffect ? { imageEffect: input.imageEffect } : {}),+ ...(input.imageEffectSec ? { imageEffectSec: input.imageEffectSec } : {}),cardColor: input.cardColor ?? null,// #8977: cutaway (any image-to-video lane) unless the director asks for a// lip-synced performance shot, which only a source-audio provider renders.@@ -600,6 +602,7 @@ export function replaceScenes(project, sceneInputs) {...(prior.frameFit ? { frameFit: prior.frameFit } : {}),...(prior.transitionIn ? { transitionIn: prior.transitionIn } : {}),...(prior.transitionOut ? { transitionOut: prior.transitionOut } : {}),+ ...(prior.imageEffect ? { imageEffect: prior.imageEffect, ...(prior.imageEffectSec ? { imageEffectSec: prior.imageEffectSec } : {}) } : {}),takes: prior.takes || [],...(prior.direction ? { direction: prior.direction } : {}),};@@ -607,10 +610,12 @@ export function replaceScenes(project, sceneInputs) {return { project: touch(project, { scenes }), scenes, replaced: existing.length };}-// Scene fields a director edits within an approved shot (its span, wording and-// selected takes) without adding a shot or lyric the storyboard approval did not cover.+// Scene fields a director edits within an approved shot (its span, wording, selected+// takes, and whether it plays as a clip or its still with an image effect) without+// adding a shot or lyric the storyboard approval did not cover.const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',- 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut'];+ 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut',+ 'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec'];/*** Apply a patch to a single scene. Returns `{ project, updated }`. Throws if thediff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.jsindex 8a4a1b5f22..892efc9a64 100644--- a/server/services/musicVideo/render.js+++ b/server/services/musicVideo/render.js@@ -58,6 +58,7 @@ import { projectSoundBed } from './soundBed.js';import { intercutClips } from './intercut.js';import { musicVideoTransitionFilter, sceneTransitions, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';import { stillMoveForCamera } from '../../lib/cameraMovements.js';+import { musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';// Per-project render mutex (keyed by projectId so two projects can render in// parallel; same-project re-entry returns 409 with the live jobId for re-attach).@@ -251,7 +252,9 @@ export async function resolveSceneClips(project, { layered = false, stillForMissconst imagePath = scene.referenceImageId ? safeUnder(PATHS.images, scene.referenceImageId) : null;if (!imagePath || !existsSync(imagePath)) { missingStills.push(scene.sceneId); continue; }// A planned camera move (#10589) picks the nearest composed still move.- clips.push({ ...section, imagePath, move: stillMoveForCamera(scene.camera) || scene.stillMove || 'hold' });+ // An image effect (a camera flash that develops, musicVideoImageEffects.js) plays over the still.+ const effect = sceneImageEffect(scene, { layered });+ clips.push({ ...section, imagePath, move: stillMoveForCamera(scene.camera) || scene.stillMove || 'hold', ...(effect ? { effect } : {}) });continue;}const entry = historyMap.get(scene.videoHistoryId);@@ -428,7 +431,8 @@ function sectionChain(c, input, { canonW, canonH, fps, frames, frameGrid }) {? `zoompan=z='1+${STILL_PUSH_ZOOM}*on/${frames}':x='iw/2-(iw/zoom/2)':y='ih/2-(ih/zoom/2)':d=1:s=${canonW}x${canonH}:fps=${fps},setsar=1,`: c.move === 'pan' ? `crop=${canonW}:${canonH}:x='(iw-ow)*n/${Math.max(1, frames - 1)}':y='(ih-oh)/2',` : '';return `[${input}:v]fps=${fps},scale=${coverW}:${coverH}:force_original_aspect_ratio=increase,crop=${coverW}:${coverH},setsar=1,`- + `${move}format=yuv420p,${trim},setpts=PTS-STARTPTS`;+ + `${move}format=yuv420p,${trim},setpts=PTS-STARTPTS`+ + musicVideoImageEffectFilter(c.effect, { width: canonW, height: canonH, fps, key: input, atSec: c.effectAtSec || 0 });}// A non-looping shot a hair longer than its source holds the last frame for// the remainder instead of ending early and drifting every later cut. On the