Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Add glitches at any moment inside a music video shot (#11391)
[._.]/2130fe29d435
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsxindex b9094723d0..f2dea8776e 100644--- a/client/src/components/musicVideo/SceneCard.jsx+++ b/client/src/components/musicVideo/SceneCard.jsx@@ -34,6 +34,7 @@ import ScenePoster from './ScenePoster.jsx';import SceneRenderTarget, { lastRenderTarget, renderTargetOverrides } from './SceneRenderTarget.jsx';import SceneRenderInputs, { shotCardRenderRequest, shotCardRequestProblem } from './SceneRenderInputs.jsx';import SceneFrameFit, { sceneFrameFit } from './SceneFrameFit.jsx';+import SceneGlitches from './SceneGlitches.jsx';import SceneTransitions from './SceneTransitions.jsx';import SceneImageEffect from './SceneImageEffect.jsx';import ScenePicturePreview from './ScenePicturePreview.jsx';@@ -244,6 +245,8 @@ export default function SceneCard({span={sceneHasAuthoredSpan(scene) ? { startSec: scene.startSec, endSec: scene.endSec } : null} />{/* A VHS glitch riding on the cut into or out of this shot; never moves the cut. */}<SceneTransitions fieldId={fieldId} scene={scene} onChange={applyPatch} />+ {/* Glitches at moments inside the shot, set and moved on a slider; they add no time. */}+ <SceneGlitches fieldId={fieldId} scene={scene} onChange={applyPatch} />{/* Everything set once and rarely revisited: timing, shot mode, layer, and a non-footage shot's camera. */}<details className="group/settings rounded border border-port-border"><summary className="flex min-h-[44px] cursor-pointer items-center gap-2 px-2 text-xs marker:content-none sm:min-h-[32px] [&::-webkit-details-marker]:hidden">diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsxindex 9cac19e2dc..9d49dab9b5 100644--- a/client/src/components/musicVideo/SceneCard.test.jsx+++ b/client/src/components/musicVideo/SceneCard.test.jsx@@ -513,6 +513,29 @@ describe('VHS glitch transitions', () => {});});+// Uniquely catches the in-shot glitch control saving the wrong shape, losing a+// slider move, or leaving an empty list behind when the last glitch goes.+describe('In-shot glitches', () => {+ it('adds a color block glitch halfway through, moves it on release and removes it', () => {+ const onSave = vi.fn();+ function Harness() {+ const [scene, setScene] = useState({ sceneId: 's1', startSec: 10, endSec: 14, takes: [] });+ return <SceneCard scene={scene} index={0} onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;+ }+ render(<Harness />);+ fireEvent.click(screen.getByRole('button', { name: /Add glitch/ }));+ expect(onSave).toHaveBeenLastCalledWith('s1', { glitches: [{ atSec: 2, style: 'color-block' }] });+ const slider = screen.getByLabelText(/^at /);+ fireEvent.change(slider, { target: { value: '2.5' } });+ fireEvent.pointerUp(slider);+ expect(onSave).toHaveBeenLastCalledWith('s1', { glitches: [{ atSec: 2.5, style: 'color-block' }] });+ fireEvent.change(screen.getByLabelText('Glitch 1 length'), { target: { value: '0.5' } });+ expect(onSave).toHaveBeenLastCalledWith('s1', { glitches: [{ atSec: 2.5, style: 'color-block', lengthSec: 0.5 }] });+ fireEvent.click(screen.getByRole('button', { name: 'Remove glitch 1' }));+ expect(onSave).toHaveBeenLastCalledWith('s1', { glitches: null });+ });+});+// Uniquely catches the picture picker saving the wrong layer or effect fields, or offering the effect where the render ignores it.describe('image effect picker', () => {it('turns a shot into a still playing camera flash develop, sets its develop length, and back to a clip', () => {diff --git a/client/src/components/musicVideo/SceneGlitches.jsx b/client/src/components/musicVideo/SceneGlitches.jsxnew file mode 100644index 0000000000..30be48e63a--- /dev/null+++ b/client/src/components/musicVideo/SceneGlitches.jsx@@ -0,0 +1,83 @@+import { useEffect, useState } from 'react';+import { Plus, X } from 'lucide-react';+import {+ MUSIC_VIDEO_GLITCH_LENGTHS, MUSIC_VIDEO_SHOT_GLITCH_LIMIT, MUSIC_VIDEO_TRANSITIONS, MUSIC_VIDEO_TRANSITION_LABELS,+} from '../../lib/musicVideoTransitions.js';++// The time slider moves a frame at a time on the render's 24 fps grid.+const STEP_SEC = 1 / 24;+const LENGTH_LABELS = { 0.125: '⅛ s', 0.25: '¼ s', 0.5: '½ s', 1: '1 s', 2: '2 s' };+const round3 = (n) => Math.round(n * 1000) / 1000;++/** One glitch: its style, when it hits (a slider over the shot) and how long it lasts. */+function GlitchRow({ fieldId, index, glitch, spanSec, onCommit, onRemove, disabled }) {+ // The slider keeps its own value while dragging and saves once it is let go.+ const [atSec, setAtSec] = useState(glitch.atSec);+ useEffect(() => { setAtSec(glitch.atSec); }, [glitch.atSec]);+ const commitTime = () => { if (atSec !== glitch.atSec) onCommit({ atSec: round3(atSec) }); };+ const id = (name) => fieldId(`glitch-${index}-${name}`);+ return (+ <li className="flex flex-wrap items-center gap-x-2 gap-y-1">+ <label htmlFor={id('style')} className="sr-only">Glitch {index + 1} style</label>+ <select id={id('style')} value={glitch.style} disabled={disabled}+ onChange={(e) => onCommit({ style: e.target.value })}+ className="min-w-0 bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50">+ {MUSIC_VIDEO_TRANSITIONS.map((style) => <option key={style} value={style}>{MUSIC_VIDEO_TRANSITION_LABELS[style]}</option>)}+ </select>+ <span className="flex min-w-[8rem] flex-1 items-center gap-1">+ <label htmlFor={id('at')} className="shrink-0 tabular-nums text-port-text-muted">at {atSec.toFixed(2)}s</label>+ <input id={id('at')} type="range" min="0" max={spanSec} step={STEP_SEC} value={Math.min(atSec, spanSec)} disabled={disabled}+ onChange={(e) => setAtSec(Number(e.target.value))}+ onPointerUp={commitTime} onKeyUp={commitTime} onBlur={commitTime}+ className="min-w-0 flex-1 accent-port-accent min-h-[44px] sm:min-h-0" />+ </span>+ <label htmlFor={id('length')} className="sr-only">Glitch {index + 1} length</label>+ <select id={id('length')} value={glitch.lengthSec ?? ''} disabled={disabled} title="How long the glitch lasts"+ onChange={(e) => onCommit({ lengthSec: e.target.value === '' ? null : Number(e.target.value) })}+ className="bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50">+ <option value="">Short</option>+ {MUSIC_VIDEO_GLITCH_LENGTHS.map((sec) => <option key={sec} value={sec}>{LENGTH_LABELS[sec] || `${sec} s`}</option>)}+ </select>+ <button type="button" onClick={onRemove} disabled={disabled} aria-label={`Remove glitch ${index + 1}`}+ className="inline-flex min-h-[44px] min-w-[44px] items-center justify-center rounded text-port-text-muted hover:text-port-error sm:min-h-[28px] sm:min-w-[28px] disabled:opacity-50">+ <X size={14} aria-hidden="true" />+ </button>+ </li>+ );+}++/**+ * Glitches at moments inside a shot, none by default. Each rides over the+ * picture in the animatic and the final render alike, adds no time and never+ * changes a lock.+ */+export default function SceneGlitches({ fieldId, scene, onChange, disabled = false }) {+ const glitches = Array.isArray(scene.glitches) ? scene.glitches : [];+ const spanSec = Number.isFinite(scene.startSec) && Number.isFinite(scene.endSec) && scene.endSec > scene.startSec+ ? round3(scene.endSec - scene.startSec) : 10;+ const save = (next) => onChange({ glitches: next.length ? next : null });+ // A new glitch lands halfway through the shot, where it is easy to find and move.+ const add = () => save([...glitches, { atSec: round3(Math.round(spanSec / 2 / STEP_SEC) * STEP_SEC), style: 'color-block' }]);+ return (+ <div className="space-y-1 text-xs" data-testid="scene-glitches">+ <div className="flex items-center gap-2">+ <span className={glitches.length ? 'text-port-accent' : 'text-port-text-muted'}>Glitches</span>+ {glitches.length < MUSIC_VIDEO_SHOT_GLITCH_LIMIT && (+ <button type="button" onClick={add} disabled={disabled}+ className="inline-flex min-h-[44px] items-center gap-1 rounded border border-port-border px-2 text-port-text-muted hover:text-port-text sm:min-h-[28px] disabled:opacity-50">+ <Plus size={12} aria-hidden="true" /> Add glitch+ </button>+ )}+ </div>+ {glitches.length > 0 && (+ <ul className="space-y-1">+ {glitches.map((glitch, index) => (+ <GlitchRow key={index} fieldId={fieldId} index={index} glitch={glitch} spanSec={spanSec} disabled={disabled}+ onCommit={(patch) => save(glitches.map((g, i) => (i === index ? { ...g, ...patch } : g)))}+ onRemove={() => save(glitches.filter((_, i) => i !== index))} />+ ))}+ </ul>+ )}+ </div>+ );+}diff --git a/client/src/components/musicVideo/TransitionGlitch.jsx b/client/src/components/musicVideo/TransitionGlitch.jsxindex 0d7d531aff..bd3ddb297b 100644--- a/client/src/components/musicVideo/TransitionGlitch.jsx+++ b/client/src/components/musicVideo/TransitionGlitch.jsx@@ -1,6 +1,6 @@import { useCallback, useEffect, useMemo, useRef } from 'react';import {- mergeGlitchFrames, sceneImageEffectGlitchFrames, sceneTransitionSections, transitionFrameAt, transitionGlitchFrames,+ blockBox, mergeGlitchFrames, sceneImageEffectGlitchFrames, sceneShotGlitchFrames, sceneTransitionSections, transitionFrameAt, transitionGlitchFrames,} from '../../lib/musicVideoTransitions.js';// The canvas paints at most this wide; a glitch lasts a few frames, so detail past it is never seen.@@ -81,6 +81,17 @@ function roll(ctx, src, a, width, height) {}}+// The render's macroblock swap (one ffmpeg swaprect): two blocks trade places.+function swapBlocks(ctx, src, sw, width, height) {+ const a = blockBox(sw.x1, sw.y1, sw.w, sw.h, width, height);+ const b = blockBox(sw.x2, sw.y2, sw.w, sw.h, width, height);+ const w = Math.min(a.w, b.w);+ const h = Math.min(a.h, b.h);+ if (w < 2 || h < 2) return;+ ctx.drawImage(src, a.x, a.y, w, h, b.x, b.y, w, h);+ ctx.drawImage(src, b.x, b.y, w, h, a.x, a.y, w, h);+}+/** Paint one glitch frame (`params` from musicVideoTransitions.js) of what the figure shows. */function paintGlitch(canvas, figure, params, frame) {const box = canvas.getBoundingClientRect();@@ -111,12 +122,14 @@ function paintGlitch(canvas, figure, params, frame) {...(params.roll ? [{ roll: params.roll * height }] : []),...(params.shift ? [{ a: params.shift * width, y: 0, h: height }] : []),...(params.tears || []).map((t) => ({ a: t.shift * width, y: Math.round(t.y * height), h: Math.round(t.h * height) })),+ ...(params.swaps || []).map((sw) => ({ swap: sw })),];slips.forEach((s, i) => {const next = scratchCanvas(i % 2 ? 'a' : 'b', width, height);const ctx = next.getContext('2d');ctx.drawImage(cur, 0, 0);if (s.roll) roll(ctx, cur, s.roll, width, height);+ else if (s.swap) swapBlocks(ctx, cur, s.swap, width, height);else slip(ctx, cur, s.a, s.y, Math.min(s.h, height - s.y), width);cur = next;});@@ -168,6 +181,13 @@ function paintGlitch(canvas, figure, params, frame) {}out.globalCompositeOperation = 'source-over';out.globalAlpha = 1;+ for (const b of params.blocks || []) {+ const box = blockBox(b.x, b.y, b.w, b.h, width, height);+ out.globalAlpha = b.alpha;+ out.fillStyle = `#${b.color}`;+ out.fillRect(box.x, box.y, box.w, box.h);+ }+ out.globalAlpha = 1;if (params.bar) {out.fillStyle = `rgba(255,255,255,${params.bar.alpha})`;out.fillRect(0, Math.round(params.bar.y * height), width, Math.max(2, Math.round(params.bar.h * height)));@@ -209,8 +229,9 @@ function paintGlitch(canvas, figure, params, frame) {export default function TransitionGlitch({ scenes, figureRef, audioRef, time, endSec = null, fps, layered = false }) {const canvasRef = useRef(null);// A still's glitch to its end frame (an image effect) paints the same way, on the frames the render repaints.+ // So do the glitches a shot sets at moments inside it.const glitches = useMemo(() => mergeGlitchFrames(transitionGlitchFrames(sceneTransitionSections(scenes, fps, { endSec })),- sceneImageEffectGlitchFrames(scenes, fps, { layered, endSec })), [scenes, fps, endSec, layered]);+ sceneImageEffectGlitchFrames(scenes, fps, { layered, endSec }), sceneShotGlitchFrames(scenes, fps, { endSec })), [scenes, fps, endSec, layered]);const paint = useCallback((t) => {const canvas = canvasRef.current;if (!canvas) return;diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex 5fc938c6f6..6b3ed244aa 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -318,4 +318,4 @@ grep -i "what you want to do" client/src/lib/README.md| `filmLook.js` | Shared film look controls, presets, normalizer, prompt vocabulary, the SVG filter markup the editor previews with and the playback-time motion layers (server `lib/filmLook.js`). || `musicVideoNarrativeEvents.js` | Named narrative event kinds, absolute onset/word/frame resolution and deterministic silence/reactive state for composition documents. || `musicVideoImageEffects.js` | Music-video image effects (a camera flash that develops over a still shot), re-exported from `server/lib/musicVideoImageEffects.js`: the effects, labels and develop lengths, `sceneImageEffect`, `imageEffectState`. Used by `components/musicVideo/SceneImageEffect.jsx` and the storyboard animatic. |-| `musicVideoTransitions.js` | Music-video cut transitions (VHS glitches over a cut), re-exported from `server/lib/musicVideoTransitions.js`: the styles and labels, `sceneTransitionSections`, `transitionGlitchFrames`, `transitionFrameAt`. Used by `components/musicVideo/SceneTransitions.jsx` and the storyboard animatic. |+| `musicVideoTransitions.js` | Music-video cut transitions (VHS glitches over a cut), re-exported from `server/lib/musicVideoTransitions.js`: the styles and labels, `sceneTransitionSections`, `transitionGlitchFrames`, `transitionFrameAt`, in-shot glitches (`sceneShotGlitchFrames`). Used by `components/musicVideo/SceneTransitions.jsx`, `SceneGlitches.jsx` and the storyboard animatic. |diff --git a/client/src/lib/musicVideoTransitions.js b/client/src/lib/musicVideoTransitions.jsindex f96dc2c69a..c72de13ecc 100644--- a/client/src/lib/musicVideoTransitions.js+++ b/client/src/lib/musicVideoTransitions.js@@ -1,14 +1,18 @@/*** Music Video cut transitions — re-export of `server/lib/musicVideoTransitions.js`,- * so the shot card's picker and the animatic's glitch frames follow the same+ * so the shot card's pickers (cut transitions and in-shot glitches) and the animatic's glitch frames follow the same* styles and frame placement as the final render.*/export {+ MUSIC_VIDEO_GLITCH_LENGTHS,+ MUSIC_VIDEO_SHOT_GLITCH_LIMIT,MUSIC_VIDEO_TRANSITIONS,MUSIC_VIDEO_TRANSITION_LABELS,MUSIC_VIDEO_TRANSITION_STYLES,mergeGlitchFrames,+ blockBox,sceneImageEffectGlitchFrames,+ sceneShotGlitchFrames,sceneTransitions,sceneTransitionSections,transitionFrameAt,diff --git a/server/lib/README.md b/server/lib/README.mdindex 1335358521..24b950fcf7 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -563,7 +563,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `chiptuneScore.js` | Chiptune score contract (#2911): `chiptuneScoreSchema` (Zod) + `sanitizeChiptuneScore`, `buildScoreEvents` — the pattern/order → absolute-time flatten mirrored by `client/src/lib/chiptunePlayback.js`. || `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). || `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`: `flash-develop`, a camera flash then the picture develops like an instant photo over `imageEffectSec`; or a fade, flash or glitch from the start frame to the end frame). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe and import-free: document videos ship it as is. |-| `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections`, `imageEffectGlitchFrames`/`mergeGlitchFrames` (end-frame glitch effects in the same shape) and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Browser-safe; the storyboard animatic paints the same frames. |+| `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections`, `imageEffectGlitchFrames`/`mergeGlitchFrames` (end-frame glitch effects in the same shape), `shotGlitchFrames`/`sceneShotGlitchFrames` (glitches a shot sets at moments inside it, `scene.glitches`), `blockBox` and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Browser-safe; the storyboard animatic paints the same frames. || `pitchMath.js` | Dependency-free scientific-pitch math shared by the LLM-authored music formats on both runtimes: `pitchToMidi("C4")` → 60 (null when unparseable) and `midiToFreq(69)` → 440. || `wavAudioFile.js` | `writeWavAudioFile(wav, dir, basename)` — land a synthesized WAV Buffer as `<basename>.ogg` via the system ffmpeg, falling back to `<basename>.wav` (and dropping a stale `.ogg`) when ffmpeg is missing or the encode fails; resolves to the filename used. Shared by the chiptune and drawn-waveform renderers and the code-engine take upload. `wavDurationMs(buffer)` reads a PCM WAV's duration from its RIFF header (0 for anything that is not a parseable WAV). `measureWavAudio(buffer)` decodes PCM 16/24/32-bit or float 32/64 (EXTENSIBLE included) into `{ channels, sampleRate, frames, durationMs, peak, rms, nonFinite }` so native renders are judged by their samples (null when undecodable). || `superColliderRuntime.js` | Managed SuperCollider runtime contract (#9412), pure: image/runtime/policy version constants, `buildSuperColliderRunArgs` (the one contained `docker run` for untrusted `sclang` source: no network/caps, read-only root, limits, ro input, private output), `buildSuperColliderBuildArgs`, the shipped synthetic probe, `evaluateSuperColliderStatus` (docker-missing … ready, with stale image/evidence invalidation), and the render contract (#9413): `SUPERCOLLIDER_RENDER` bounds, the trusted NRT `SUPERCOLLIDER_RENDER_WRAPPER_SOURCE`, `classifySuperColliderRenderLog` (syntax/source/pattern/score/synthesis failures) and `parseDockerContainerList` for orphan sweeps. I/O lives in `services/superColliderRuntime.js`. |diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.jsindex 2ade85439b..ae508168d8 100644--- a/server/lib/musicVideoDependencies.js+++ b/server/lib/musicVideoDependencies.js@@ -22,6 +22,7 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s.concat(scene.frameFit && scene.frameFit !== 'fill' ? [['frameFit', scene.frameFit]] : [])// A glitch over its cuts changes the render; a shot without one keeps its evidence..concat(['transitionIn', 'transitionOut'].filter((key) => scene[key]).map((key) => [key, scene[key]]))+ .concat(scene.glitches?.length ? [['glitches', scene.glitches]] : [])// An image effect over a still changes the render; a still without one keeps its evidence..concat(scene.imageEffect ? [['imageEffect', scene.imageEffect], ['imageEffectSec', scene.imageEffectSec ?? null]] : [])// An effect from the start frame to the end frame renders the end frame too.diff --git a/server/lib/musicVideoTransitions.js b/server/lib/musicVideoTransitions.jsindex f0e07d8d45..0d9e34a93c 100644--- a/server/lib/musicVideoTransitions.js+++ b/server/lib/musicVideoTransitions.js@@ -18,7 +18,7 @@import { IMAGE_EFFECT_GLITCH_STYLE, imageEffectChangeFrame, sceneImageEffect } from './musicVideoImageEffects.js';-export const MUSIC_VIDEO_TRANSITIONS = Object.freeze(['vhs-glitch', 'channel-change', 'vhs-tracking', 'vhs-dropout', 'camera-flash']);+export const MUSIC_VIDEO_TRANSITIONS = Object.freeze(['vhs-glitch', 'channel-change', 'vhs-tracking', 'vhs-dropout', 'camera-flash', 'color-block']);export const MUSIC_VIDEO_TRANSITION_LABELS = Object.freeze({'vhs-glitch': 'VHS glitch',@@ -26,8 +26,15 @@ export const MUSIC_VIDEO_TRANSITION_LABELS = Object.freeze({'vhs-tracking': 'Tracking roll','vhs-dropout': 'Dropout flash','camera-flash': 'Camera flash',+ 'color-block': 'Color blocks',});+// A shot also carries glitches at any moment inside it (`scene.glitches`):+// `{ atSec, style, lengthSec? }`, `atSec` counted from the shot's start and+// `lengthSec` stretching the style's frames (unset plays them once).+export const MUSIC_VIDEO_SHOT_GLITCH_LIMIT = 8;+export const MUSIC_VIDEO_GLITCH_LENGTHS = Object.freeze([0.125, 0.25, 0.5, 1, 2]);+// Each style's frames, in order, and how many of them come before the cut// (`lead`, the outgoing shot's last frames). Per frame:// shift whole-frame horizontal slip (signed fraction of width)@@ -38,6 +45,8 @@ export const MUSIC_VIDEO_TRANSITION_LABELS = Object.freeze({// noise snow strength: 0..1 over the picture, up to 1.6 for a dead channel's static// lines scanline darkness, 0..1// bar a bright tracking band { y, h, alpha }+// swaps macroblocks traded places: { x1, y1, x2, y2, w, h } fractions of the frame+// blocks solid colour macroblocks: { x, y, w, h } fractions, color 'rrggbb', alphaexport const MUSIC_VIDEO_TRANSITION_STYLES = Object.freeze({'vhs-glitch': {lead: 1,@@ -86,15 +95,58 @@ export const MUSIC_VIDEO_TRANSITION_STYLES = Object.freeze({{ tears: [{ y: 0.3, h: 0.06, shift: 0.03 }], chroma: 0.01, bright: 1.2, sat: 0.6, noise: 0.5, lines: 0.35 },],},+ // A digital decoder losing the stream: macroblocks trade places and some+ // decode as flat green, magenta or grey squares, on a 16 x 9 block grid.+ 'color-block': {+ lead: 2,+ frames: [+ { swaps: [{ x1: 0.25, y1: 0.3333, x2: 0.5625, y2: 0.4444, w: 0.125, h: 0.1111 }], chroma: 0.006, sat: 1.2,+ blocks: [{ x: 0.625, y: 0.2222, w: 0.0625, h: 0.1111, color: '2fbf4f', alpha: 0.85 }, { x: 0.3125, y: 0.5556, w: 0.125, h: 0.1111, color: 'c02cc0', alpha: 0.7 }] },+ { swaps: [{ x1: 0.0625, y1: 0.1111, x2: 0.5, y2: 0.6667, w: 0.25, h: 0.2222 }, { x1: 0.6875, y1: 0.3333, x2: 0.375, y2: 0.1111, w: 0.1875, h: 0.1111 }],+ chroma: 0.012, bright: 1.05, sat: 1.4,+ blocks: [+ { x: 0.125, y: 0.4444, w: 0.375, h: 0.1111, color: '808080', alpha: 0.6 },+ { x: 0.75, y: 0.5556, w: 0.0625, h: 0.2222, color: '2fbf4f', alpha: 0.9 },+ { x: 0.5625, y: 0.2222, w: 0.125, h: 0.1111, color: 'c02cc0', alpha: 0.8 },+ { x: 0.25, y: 0.7778, w: 0.0625, h: 0.1111, color: '1fa0a0', alpha: 0.85 },+ ] },+ { swaps: [{ x1: 0.4375, y1: 0.5556, x2: 0.125, y2: 0.2222, w: 0.125, h: 0.2222 }], chroma: 0.008, sat: 1.25,+ blocks: [{ x: 0.75, y: 0.1111, w: 0.125, h: 0.1111, color: '2fbf4f', alpha: 0.75 }, { x: 0.0625, y: 0.6667, w: 0.1875, h: 0.1111, color: '808080', alpha: 0.55 }] },+ { chroma: 0.004, sat: 1.1, blocks: [{ x: 0.5, y: 0.4444, w: 0.0625, h: 0.1111, color: 'c02cc0', alpha: 0.6 }] },+ ],+ },});const isTransition = (value) => MUSIC_VIDEO_TRANSITIONS.includes(value);-/** A scene's chosen transitions, only the ones that are set: `{ transitionIn?, transitionOut? }`. */+/**+ * A shot's in-shot glitches on the song's clock, valid ones only:+ * `[{ atSec, style, lengthSec? }]` with `atSec` in song seconds, kept inside+ * the shot when its end is known.+ */+function shotGlitchesOnSong(scene) {+ if (!Array.isArray(scene?.glitches) || !Number.isFinite(scene.startSec)) return [];+ const span = Number.isFinite(scene.endSec) ? scene.endSec - scene.startSec : Infinity;+ return scene.glitches+ .filter((g) => g && isTransition(g.style) && Number.isFinite(g.atSec))+ .slice(0, MUSIC_VIDEO_SHOT_GLITCH_LIMIT)+ .map((g) => ({+ atSec: scene.startSec + Math.min(Math.max(0, g.atSec), Math.max(0, span - 0.001)),+ style: g.style,+ ...(Number.isFinite(g.lengthSec) && g.lengthSec > 0 ? { lengthSec: g.lengthSec } : {}),+ }));+}++/**+ * A scene's chosen glitches, only the ones that are set:+ * `{ transitionIn?, transitionOut?, shotGlitches? }` (`shotGlitches` on the song's clock).+ */export function sceneTransitions(scene) {+ const shotGlitches = shotGlitchesOnSong(scene);return {...(isTransition(scene?.transitionIn) ? { transitionIn: scene.transitionIn } : {}),...(isTransition(scene?.transitionOut) ? { transitionOut: scene.transitionOut } : {}),+ ...(shotGlitches.length ? { shotGlitches } : {}),};}@@ -178,6 +230,13 @@ function roll(px, width, height) {return `swaprect=w=${width}:h=${height - a}:x1=0:y1=${y1}:x2=0:y2=${y2}`;}+/** A block given in fractions of the frame as even pixels kept inside it: `{ x, y, w, h }`. */+export function blockBox(x, y, w, h, width, height) {+ const px = Math.max(0, Math.min(width - 2, even(x * width)));+ const py = Math.max(0, Math.min(height - 2, even(y * height)));+ return { x: px, y: py, w: Math.max(0, Math.min(width - px, even(w * width))), h: Math.max(0, Math.min(height - py, even(h * height))) };+}+/** Inclusive frame runs as a timeline expression: `between(n,4,6)+between(n,90,90)`. */function framesExpr(frames) {const sorted = [...new Set(frames)].sort((a, b) => a - b);@@ -215,6 +274,13 @@ export function musicVideoTransitionFilter(glitches, { width, height }) {const y = rows(tear.y);add(slip(tear.shift * width, y, Math.min(rows(tear.h), height - y), width));}+ for (const sw of p.swaps || []) {+ const box = blockBox(sw.x1, sw.y1, sw.w, sw.h, width, height);+ const to = blockBox(sw.x2, sw.y2, sw.w, sw.h, width, height);+ const w = Math.min(box.w, to.w);+ const h = Math.min(box.h, to.h);+ if (w >= 2 && h >= 2) add(`swaprect=w=${w}:h=${h}:x1=${box.x}:y1=${box.y}:x2=${to.x}:y2=${to.y}`);+ }const fringe = Math.min(255, Math.round((p.chroma || 0) * width / 2));if (fringe > 0) add(`chromashift=cbh=${fringe}:crh=${-fringe}`);const bright = p.bright ?? 1;@@ -223,6 +289,10 @@ export function musicVideoTransitionFilter(glitches, { width, height }) {const chroma = `'clip((val-128)*${num(bright * sat)}+128,0,255)'`;add(`lutyuv=y='clip(val*${num(bright)},0,255)':u=${chroma}:v=${chroma}`);}+ for (const b of p.blocks || []) {+ const box = blockBox(b.x, b.y, b.w, b.h, width, height);+ if (box.w >= 2 && box.h >= 2) add(`drawbox=x=${box.x}:y=${box.y}:w=${box.w}:h=${box.h}:color=0x${b.color}@${num(b.alpha)}:t=fill`);+ }if (p.bar) add(`drawbox=x=0:y=${rows(p.bar.y)}:w=iw:h=${Math.max(2, rows(p.bar.h))}:color=white@${num(p.bar.alpha)}:t=fill`);if (p.lines) {const gap = Math.max(2, Math.round(height / 270));@@ -261,8 +331,49 @@ export function sceneImageEffectGlitchFrames(scenes, fps, { layered = false, end.map((section) => ({ ...section, effect: sceneImageEffect(byId.get(section.sceneId), { layered }) })), fps);}-/** Cut-transition glitches and image-effect glitches as one list, a transition keeping a frame both reach. */-export function mergeGlitchFrames(transitions, effects) {- const taken = new Set((transitions || []).map((g) => g.frame));- return [...(transitions || []), ...(effects || []).filter((g) => !taken.has(g.frame))].sort((a, b) => a.frame - b.frame);+/**+ * The glitched frames of the shots' in-shot glitches, in the cut transitions'+ * shape (`[{ frame, style, index }]`). `pieces` are timeline sections carrying+ * `sceneId`, `partner?` and `shotGlitches` (song seconds, from+ * `sceneTransitions`) and `startFrame`/`endFrame`; a glitch paints only on+ * its own shot's pieces, so it never spills into the next shot or over an+ * intercut partner, and a partner borrowing the shot's footage adds none. A `lengthSec` stretches+ * the style's frames over that many frames, holding each a little longer.+ */+export function shotGlitchFrames(pieces, fps) {+ // Each shot's own pieces: a glitch paints only while the shot is on screen.+ const shots = new Map();+ for (const piece of Array.isArray(pieces) ? pieces : []) {+ if (!piece || piece.partner || !Array.isArray(piece.shotGlitches)) continue;+ if (!shots.has(piece.sceneId)) shots.set(piece.sceneId, { glitches: piece.shotGlitches, ranges: [] });+ shots.get(piece.sceneId).ranges.push([piece.startFrame, piece.endFrame]);+ }+ const out = [];+ for (const { glitches, ranges } of shots.values()) {+ const onScreen = (frame) => ranges.some(([a, b]) => frame >= a && frame < b);+ for (const g of glitches) {+ const style = MUSIC_VIDEO_TRANSITION_STYLES[g.style];+ if (!style) continue;+ const count = style.frames.length;+ const length = g.lengthSec ? Math.max(1, Math.round(g.lengthSec * fps)) : count;+ // The frame showing at that moment, as the player counts it.+ const start = Math.floor(g.atSec * fps + 1e-6);+ for (let i = 0; i < length; i++) {+ if (onScreen(start + i)) out.push({ frame: start + i, style: g.style, index: Math.floor((i * count) / length) });+ }+ }+ }+ return out;+}++/** The storyboard's in-shot glitch frames on `fps` (the animatic's counterpart of the render's pieces). */+export function sceneShotGlitchFrames(scenes, fps, { endSec = null } = {}) {+ return shotGlitchFrames(sceneTransitionSections(scenes, fps, { endSec }), fps);+}++/** Glitch frame lists as one, sorted by frame: an earlier list keeps a frame two of them reach. */+export function mergeGlitchFrames(...lists) {+ const taken = new Map();+ for (const list of lists) for (const g of list || []) if (!taken.has(g.frame)) taken.set(g.frame, g);+ return [...taken.values()].sort((a, b) => a.frame - b.frame);}diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.jsindex bdd5bda072..cc5b9797ad 100644--- a/server/lib/musicVideoValidation.js+++ b/server/lib/musicVideoValidation.js@@ -53,7 +53,7 @@ import { shotActionContractProblem } from './musicVideoActionContract.js';import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';import { MUSIC_VIDEO_MEDIA } from './musicVideoMediumPlan.js';import { MUSIC_VIDEO_FRAME_FITS, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';-import { MUSIC_VIDEO_TRANSITIONS } from './musicVideoTransitions.js';+import { MUSIC_VIDEO_SHOT_GLITCH_LIMIT, MUSIC_VIDEO_TRANSITIONS } from './musicVideoTransitions.js';import { IMAGE_EFFECT_SEC_RANGE, MUSIC_VIDEO_IMAGE_EFFECTS } from './musicVideoImageEffects.js';import { CAMERA_FRAMINGS, CAMERA_MOVEMENT_VALUES, CAMERA_SPEEDS } from './cameraMovements.js';import { MUSIC_VIDEO_PERFORMANCE_STYLES, MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';@@ -609,6 +609,12 @@ const sceneLayerFields = {// A VHS glitch over the cut into / out of this shot (musicVideoTransitions.js); null clears it.transitionIn: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),transitionOut: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),+ // Glitches at moments inside the shot: `atSec` from the shot's start, `lengthSec` stretching the style; null clears them.+ glitches: z.array(z.object({+ atSec: z.number().min(0).max(3600),+ style: z.enum(MUSIC_VIDEO_TRANSITIONS),+ lengthSec: z.number().positive().max(4).nullable().optional(),+ })).max(MUSIC_VIDEO_SHOT_GLITCH_LIMIT).nullable().optional(),// A code-drawn effect a still shot plays instead of a clip (musicVideoImageEffects.js) and how long// it takes to finish; null clears it.imageEffect: z.enum(MUSIC_VIDEO_IMAGE_EFFECTS).nullable().optional(),diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.jsindex c27f483d10..44c544fbb6 100644--- a/server/services/musicVideo/projectsLogic.js+++ b/server/services/musicVideo/projectsLogic.js@@ -522,6 +522,7 @@ function buildScene(input, { order }) {...(input.frameFit ? { frameFit: input.frameFit } : {}),...(input.transitionIn ? { transitionIn: input.transitionIn } : {}),...(input.transitionOut ? { transitionOut: input.transitionOut } : {}),+ ...(input.glitches?.length ? { glitches: input.glitches } : {}),...(input.imageEffect ? { imageEffect: input.imageEffect } : {}),...(input.imageEffectSec ? { imageEffectSec: input.imageEffectSec } : {}),cardColor: input.cardColor ?? null,@@ -607,6 +608,7 @@ export function replaceScenes(project, sceneInputs) {...(prior.frameFit ? { frameFit: prior.frameFit } : {}),...(prior.transitionIn ? { transitionIn: prior.transitionIn } : {}),...(prior.transitionOut ? { transitionOut: prior.transitionOut } : {}),+ ...(prior.glitches?.length ? { glitches: prior.glitches } : {}),// A still playing an image effect stays one: the effect is the shot's picture....(prior.imageEffect && prior.visualLayer === 'still' ? { visualLayer: 'still', imageEffect: prior.imageEffect,...(prior.imageEffectSec ? { imageEffectSec: prior.imageEffectSec } : {}) } : {}),@@ -621,7 +623,7 @@ export function replaceScenes(project, sceneInputs) {// takes, and whether it plays as a clip or its still with an image effect) without// adding a shot or lyric the storyboard approval did not cover.const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',- 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut',+ 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut', 'glitches','visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec'];/**diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.jsindex 9ea9fb3aae..aeeddb40dc 100644--- a/server/services/musicVideo/render.js+++ b/server/services/musicVideo/render.js@@ -58,7 +58,7 @@ import { AUDIO_NORM, buildAudioBedMix } from '../videoTimeline/audioBedMix.js';import { musicVideoEvents } from './events.js';import { projectSoundBed } from './soundBed.js';import { intercutClips } from './intercut.js';-import { imageEffectGlitchFrames, mergeGlitchFrames, musicVideoTransitionFilter, sceneTransitions, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';+import { imageEffectGlitchFrames, mergeGlitchFrames, musicVideoTransitionFilter, sceneTransitions, shotGlitchFrames, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';import { stillMoveForCamera } from '../../lib/cameraMovements.js';import { isEndFrameImageEffect, musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';@@ -559,9 +559,11 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDconst pieces = plan.map(({ c, startSec, endSec }) => ({sceneId: c.sceneId, startFrame: Math.round(startSec * fps), endFrame: Math.round(endSec * fps),partner: !!c.intercutOf, transitionIn: c.transitionIn, transitionOut: c.transitionOut, effect: c.effect, effectAtSec: c.effectAtSec,+ shotGlitches: c.shotGlitches,}));// A still's glitch to its end frame paints with the same frames.- const glitches = mergeGlitchFrames(transitionGlitchFrames(pieces), imageEffectGlitchFrames(pieces, fps));+ // So do the glitches a shot sets at moments inside it.+ const glitches = mergeGlitchFrames(transitionGlitchFrames(pieces), imageEffectGlitchFrames(pieces, fps), shotGlitchFrames(pieces, fps));const glitchFilter = musicVideoTransitionFilter(glitches, { width: canonW, height: canonH });const cutLabel = overlays.length > 0 ? 'cut0' : 'outv';const finishing = [gradeFilter, glitchFilter].filter(Boolean).join(',');diff --git a/server/services/musicVideo/transitions.test.js b/server/services/musicVideo/transitions.test.jsindex 541aa17d2c..68014b3c3f 100644--- a/server/services/musicVideo/transitions.test.js+++ b/server/services/musicVideo/transitions.test.js@@ -4,7 +4,7 @@ import { mkdtempSync, rmSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { findFfmpeg } from '../../lib/ffmpeg.js';-import { transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';+import { sceneTransitions, shotGlitchFrames, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';import { buildMusicVideoFfmpegArgs } from './render.js';import { applySceneUpdate } from './projectsLogic.js';@@ -43,6 +43,37 @@ describe('transitionGlitchFrames', () => {});});+// Uniquely catches where an in-shot glitch lands: its time counted from the+// shot's start, its stretch over a set length, staying inside the shot, and a+// split or intercut shot painting it once.+describe('shotGlitchFrames', () => {+ const FPS24 = 24;+ const piece = (scene, extra = {}) => ({ sceneId: scene.sceneId, startFrame: Math.round(scene.startSec * FPS24),+ endFrame: Math.round(scene.endSec * FPS24), ...sceneTransitions(scene), ...extra });+ const framesOf = (pieces) => shotGlitchFrames(pieces, FPS24).map((g) => `${g.frame}:${g.index}`);++ it('starts the glitch at its time in the shot and stretches the style over a set length', () => {+ const shot = { sceneId: 'a', startSec: 10, endSec: 14, glitches: [{ atSec: 2, style: 'color-block' }] };+ expect(framesOf([piece(shot)])).toEqual(['288:0', '289:1', '290:2', '291:3']);+ const long = { ...shot, glitches: [{ atSec: 2, style: 'color-block', lengthSec: 0.25 }] };+ expect(framesOf([piece(long)])).toEqual(['288:0', '289:0', '290:1', '291:2', '292:2', '293:3']);+ });++ it('never paints past the shot, whether timed or stretched past its end', () => {+ const late = { sceneId: 'a', startSec: 0, endSec: 1, glitches: [{ atSec: 5, style: 'vhs-dropout' }] };+ expect(framesOf([piece(late)])).toEqual(['23:0']);+ const long = { sceneId: 'a', startSec: 0, endSec: 1, glitches: [{ atSec: 0.9, style: 'color-block', lengthSec: 0.5 }] };+ expect(framesOf([piece(long), piece({ sceneId: 'b', startSec: 1, endSec: 2 })])).toEqual(['21:0', '22:0', '23:0']);+ });++ it('paints a split shot\'s glitch once, only on its own pieces, and none for an intercut partner', () => {+ const shot = { sceneId: 'a', startSec: 0, endSec: 4, glitches: [{ atSec: 1, style: 'vhs-dropout' }, { atSec: 1.5, style: 'vhs-dropout' }] };+ const other = { sceneId: 'b', startSec: 4, endSec: 8, glitches: [{ atSec: 1, style: 'vhs-dropout' }] };+ const pieces = [{ ...piece(shot), endFrame: 36 }, { ...piece(other, { partner: true }), startFrame: 36, endFrame: 48 }, { ...piece(shot), startFrame: 48, endFrame: 96 }];+ expect(framesOf(pieces)).toEqual(['24:0', '25:1']);+ });+});+// Per-frame pixel spread of a raw gray video: ~0 on a flat colour, high once glitched.function frameSpreads(path) {const raw = run(['-i', path, '-f', 'rawvideo', '-pix_fmt', 'gray', '-']);@@ -77,6 +108,28 @@ describe.skipIf(!ffmpeg)('a VHS glitch on a cut in the final render', () => {});});+// Uniquely catches the in-shot glitch filters failing in ffmpeg, landing on+// the wrong frames, or adding time to the edit.+describe.skipIf(!ffmpeg)('a color block glitch inside a shot in the final render', () => {+ it('repaints only the frames from its time in the shot and keeps every frame of the edit', () => {+ const song = join(dir, 'song.wav');+ run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=2', '-y', song]);+ const scenes = [{ sceneId: 's0', startSec: 0, endSec: 1, glitches: [{ atSec: 0.5, style: 'color-block', lengthSec: 0.25 }] }, { sceneId: 's1', startSec: 1, endSec: 2 }];+ const clips = ['0x2050a0', '0xa05020'].map((color, i) => {+ const videoPath = join(dir, `shot${i}.mp4`);+ run(['-f', 'lavfi', '-i', `color=${color}:s=${W}x${H}:r=${FPS}:d=1`, '-pix_fmt', 'yuv420p', '-y', videoPath]);+ return { sceneId: `s${i}`, videoPath, width: W, height: H, fps: FPS, duration: 1, sourceSec: 1, loop: false, inSec: 0, outSec: 1, frameFit: 'fill', ...sceneTransitions(scenes[i]) };+ });+ const output = join(dir, 'color-block.mp4');+ const plan = buildMusicVideoFfmpegArgs(clips, song, output, { audioDurationSec: 2, frameGrid: true });+ run(plan.args.filter((arg, i) => !(arg === '-progress' || plan.args[i - 1] === '-progress')));+ expect(plan.totalDuration).toBe(2);+ const spreads = frameSpreads(output);+ expect(spreads).toHaveLength(2 * FPS);+ expect(spreads.map((s, f) => (s > 1 ? f : null)).filter((f) => f != null)).toEqual([6, 7, 8]);+ });+});+// Uniquely catches the scene PATCH refusing the new fields, keeping a cleared one, or taking an unknown style.describe('a shot\'s transitions on the board', () => {const project = { id: 'mv-1', scenes: [{ sceneId: 's1', order: 0, startSec: 0, endSec: 4, takes: [] }] };@@ -87,4 +140,12 @@ describe('a shot\'s transitions on the board', () => {expect(cleared.scenes[0].transitionOut).toBeNull();expect(() => applySceneUpdate(project, 's1', { transitionOut: 'crossfade' })).toThrow();});++ it('saves in-shot glitches, clears them with null and refuses an unknown style', () => {+ const glitches = [{ atSec: 1.5, style: 'color-block', lengthSec: 0.5 }];+ const { project: set } = applySceneUpdate(project, 's1', { glitches });+ expect(set.scenes[0].glitches).toEqual(glitches);+ expect(applySceneUpdate(set, 's1', { glitches: null }).project.scenes[0].glitches).toBeNull();+ expect(() => applySceneUpdate(project, 's1', { glitches: [{ atSec: 1, style: 'crossfade' }] })).toThrow();+ });});