Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Add a beat toggle across music video cuts (#11395)

[._.]/de1a320ba7d5
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index f2dea8776e..89c26f95f8 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -34,6 +34,7 @@ import ScenePoster from './ScenePoster.jsx';
import SceneRenderTarget, { lastRenderTarget, renderTargetOverrides } from './SceneRenderTarget.jsx';
import SceneRenderInputs, { shotCardRenderRequest, shotCardRequestProblem } from './SceneRenderInputs.jsx';
import SceneFrameFit, { sceneFrameFit } from './SceneFrameFit.jsx';
+import SceneCutEffect from './SceneCutEffect.jsx';
import SceneGlitches from './SceneGlitches.jsx';
import SceneTransitions from './SceneTransitions.jsx';
import SceneImageEffect from './SceneImageEffect.jsx';
@@ -245,6 +246,8 @@ export default function SceneCard({
span={sceneHasAuthoredSpan(scene) ? { startSec: scene.startSec, endSec: scene.endSec } : null} />
{/* A VHS glitch riding on the cut into or out of this shot; never moves the cut. */}
<SceneTransitions fieldId={fieldId} scene={scene} onChange={applyPatch} />
+ {/* Both clips flipping on the beats over the end of this shot, switched on or off. */}
+ <SceneCutEffect fieldId={fieldId} scene={scene} onChange={applyPatch} />
{/* Glitches at moments inside the shot, set and moved on a slider; they add no time. */}
<SceneGlitches fieldId={fieldId} scene={scene} onChange={applyPatch} />
{/* Everything set once and rarely revisited: timing, shot mode, layer, and a non-footage shot's camera. */}
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index 9d49dab9b5..06d6fc2044 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -582,3 +582,23 @@ describe('a still picture in the clip column', () => {
expect(onSave).toHaveBeenLastCalledWith('s1', { imageEffectSec: 1 });
});
});
+
+// Uniquely catches the beat toggle control saving the wrong shape or losing its settings when switched off.
+describe('Beat toggle into the next shot', () => {
+ it('switches on with defaults, changes its settings and keeps them when switched off', () => {
+ const onSave = vi.fn();
+ function Harness() {
+ const [scene, setScene] = useState({ sceneId: 's1', startSec: 0, endSec: 4, takes: [] });
+ return <SceneCard scene={scene} index={0} onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
+ }
+ render(<Harness />);
+ const toggle = screen.getByLabelText('Beat toggle into next shot');
+ fireEvent.click(toggle);
+ expect(onSave).toHaveBeenLastCalledWith('s1', { cutEffect: { style: 'beat-toggle', enabled: true, overlapSec: 1, rate: 'beat' } });
+ fireEvent.change(screen.getByLabelText('Beat toggle speed'), { target: { value: 'half' } });
+ expect(onSave).toHaveBeenLastCalledWith('s1', { cutEffect: { style: 'beat-toggle', enabled: true, overlapSec: 1, rate: 'half' } });
+ fireEvent.click(toggle);
+ expect(onSave).toHaveBeenLastCalledWith('s1', { cutEffect: { style: 'beat-toggle', enabled: false, overlapSec: 1, rate: 'half' } });
+ expect(screen.getByLabelText('Beat toggle speed').value).toBe('half');
+ });
+});
diff --git a/client/src/components/musicVideo/SceneCutEffect.jsx b/client/src/components/musicVideo/SceneCutEffect.jsx
new file mode 100644
index 0000000000..de614593c0
--- /dev/null
+++ b/client/src/components/musicVideo/SceneCutEffect.jsx
@@ -0,0 +1,44 @@
+import { BEAT_TOGGLE_OVERLAP, BEAT_TOGGLE_RATE_LABELS, BEAT_TOGGLE_RATES, sceneCutEffect } from '../../lib/musicVideoCutEffects.js';
+
+const OVERLAPS = Object.freeze([0.5, 1, 1.5, 2]);
+const OVERLAP_LABELS = { 0.5: '½ s', 1: '1 s', 1.5: '1½ s', 2: '2 s' };
+
+/**
+ * The beat toggle across the cut out of a shot: a switch, then how long before
+ * the cut it plays and how fast it flips. Switching it off keeps the settings,
+ * so turning it back on brings them back. It never moves a cut or changes a lock.
+ */
+export default function SceneCutEffect({ fieldId, scene, onChange, disabled = false }) {
+ const effect = sceneCutEffect(scene);
+ const on = !!effect?.enabled;
+ const save = (patch) => onChange({
+ cutEffect: { style: 'beat-toggle', enabled: on, overlapSec: effect?.overlapSec ?? BEAT_TOGGLE_OVERLAP.default, rate: effect?.rate ?? 'beat', ...patch },
+ });
+ const selectClass = 'bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50';
+ const overlap = OVERLAPS.includes(effect?.overlapSec) ? effect.overlapSec : null;
+ return (
+ <div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-xs" data-testid="scene-cut-effect">
+ <label htmlFor={fieldId('beat-toggle')} className="flex min-h-[44px] items-center gap-1.5 sm:min-h-0"
+ title="Both clips play over the end of this shot and the picture flips between them on the beats, then the next shot carries on">
+ <input id={fieldId('beat-toggle')} type="checkbox" checked={on} disabled={disabled}
+ onChange={(e) => save({ enabled: e.target.checked })} className="accent-port-accent" />
+ <span className={on ? 'text-port-accent' : 'text-port-text-muted'}>Beat toggle into next shot</span>
+ </label>
+ {effect && (
+ <>
+ <label htmlFor={fieldId('beat-toggle-overlap')} className="sr-only">Beat toggle length</label>
+ <select id={fieldId('beat-toggle-overlap')} value={overlap ?? ''} disabled={disabled} title="How long before the cut it plays"
+ onChange={(e) => save({ overlapSec: Number(e.target.value) })} className={selectClass}>
+ {overlap == null && <option value="">{`${effect.overlapSec} s`}</option>}
+ {OVERLAPS.map((sec) => <option key={sec} value={sec}>{OVERLAP_LABELS[sec]}</option>)}
+ </select>
+ <label htmlFor={fieldId('beat-toggle-rate')} className="sr-only">Beat toggle speed</label>
+ <select id={fieldId('beat-toggle-rate')} value={effect.rate} disabled={disabled}
+ onChange={(e) => save({ rate: e.target.value })} className={selectClass}>
+ {BEAT_TOGGLE_RATES.map((rate) => <option key={rate} value={rate}>{BEAT_TOGGLE_RATE_LABELS[rate]}</option>)}
+ </select>
+ </>
+ )}
+ </div>
+ );
+}
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index 9cc958a6e5..6ff3cbe27d 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -18,6 +18,7 @@ import TransitionGlitch from './TransitionGlitch.jsx';
import ImageEffectStill from './ImageEffectStill.jsx';
import { isLayeredComposition } from '../../lib/musicVideoLayers.js';
import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
+import { sceneBeatToggles } from '../../lib/musicVideoCutEffects.js';
const timedShots = (scenes) => scenes.filter((scene) => Number.isFinite(scene?.startSec)).sort((a, b) => a.startSec - b.startSec);
@@ -187,8 +188,10 @@ const ANIMATIC_LOOK_KEY = 'portos-mv-animatic-look';
* come waits paused on its first frame. The clip stays transparent until it has
* a frame to show, so the shot's still underneath covers the load and the seek.
*/
-function ShotClip({ scene, audioRef, time, current }) {
+function ShotClip({ scene, audioRef, time, current, leadSec = 0 }) {
const videoRef = useRef(null);
+ // A beat toggle into this shot started its clip early: it carries on from there.
+ const timing = useMemo(() => (leadSec > 0 ? { ...scene, startSec: (scene.startSec || 0) - leadSec } : scene), [scene, leadSec]);
const [ready, setReady] = useState(false);
const file = useVideoFileSrc(scene.videoHistoryId);
const src = file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`);
@@ -197,12 +200,12 @@ function ShotClip({ scene, audioRef, time, current }) {
const audio = audioRef.current;
if (!video || !audio || video.readyState < 1) return;
const duration = Number.isFinite(video.duration) ? video.duration : null;
- const target = clipTimeAt(scene, current ? audio.currentTime || 0 : scene.startSec || 0, duration);
+ const target = clipTimeAt(timing, current ? audio.currentTime || 0 : scene.startSec || 0, duration);
const tolerance = current && audio.paused ? CLIP_PAUSED_DRIFT_SEC : CLIP_DRIFT_SEC;
if (Math.abs((video.currentTime || 0) - target) > tolerance) video.currentTime = target;
if (!current || audio.paused) video.pause?.();
else if (video.paused) video.play?.()?.catch?.(() => {});
- }, [scene, audioRef, current]);
+ }, [scene, timing, audioRef, current]);
// Shown once a frame is decoded where the clip should be; kept from then on, since a
// drift re-seek holds the last frame rather than going blank.
const markReady = useCallback(() => {
@@ -239,6 +242,62 @@ function ShotClip({ scene, audioRef, time, current }) {
);
}
+/**
+ * The incoming shot of a beat toggle (lib/musicVideoCutEffects.js) over the
+ * outgoing one: its clip plays from where the toggle starts it and shows only
+ * on the toggle's frames, as the final render overlays it. Playback flips it on
+ * animation frames, since the song's timeupdate is far too coarse for a beat.
+ * A shot without a clip shows its still.
+ */
+function ToggleClip({ scene, toggle, audioRef, time }) {
+ const mediaRef = useRef(null);
+ const file = useVideoFileSrc(scene.videoHistoryId || null);
+ const src = scene.videoHistoryId ? file.src || (file.resolving ? null : `/data/videos/${scene.videoHistoryId}.mp4`) : null;
+ const timing = useMemo(() => ({ ...scene, startSec: toggle.windowStartSec }), [scene, toggle.windowStartSec]);
+ const apply = useCallback((t) => {
+ const el = mediaRef.current;
+ const audio = audioRef.current;
+ if (!el) return;
+ el.style.opacity = toggle.incoming.some(([a, b]) => t >= a && t < b) ? '1' : '0';
+ if (el.tagName !== 'VIDEO' || el.readyState < 1) return;
+ const duration = Number.isFinite(el.duration) ? el.duration : null;
+ const target = clipTimeAt(timing, Math.max(t, toggle.windowStartSec), duration);
+ const tolerance = audio?.paused ? CLIP_PAUSED_DRIFT_SEC : CLIP_DRIFT_SEC;
+ if (Math.abs((el.currentTime || 0) - target) > tolerance) el.currentTime = target;
+ const running = !!audio && !audio.paused && t >= toggle.windowStartSec && t < toggle.cutSec;
+ if (!running) el.pause?.();
+ else if (el.paused) el.play?.()?.catch?.(() => {});
+ }, [toggle, timing, audioRef]);
+ useEffect(() => { apply(time); }, [time, apply]);
+ useEffect(() => {
+ const audio = audioRef.current;
+ if (!audio) return undefined;
+ let handle = 0;
+ const tick = () => {
+ apply(audio.currentTime || 0);
+ handle = requestAnimationFrame(tick);
+ };
+ const start = () => { cancelAnimationFrame(handle); handle = requestAnimationFrame(tick); };
+ const stop = () => { cancelAnimationFrame(handle); apply(audio.currentTime || 0); };
+ audio.addEventListener('play', start);
+ audio.addEventListener('pause', stop);
+ if (!audio.paused) start();
+ return () => {
+ cancelAnimationFrame(handle);
+ audio.removeEventListener('play', start);
+ audio.removeEventListener('pause', stop);
+ };
+ }, [audioRef, apply]);
+ const fit = sceneFrameFit(scene);
+ const className = `pointer-events-none absolute inset-0 h-full w-full ${src && fit !== 'fill' ? 'object-contain' : 'object-cover'} ${fit === 'black' ? 'bg-black' : ''}`;
+ if (src) {
+ return <video ref={mediaRef} src={src} muted playsInline preload="auto" onLoadedMetadata={() => apply(audioRef.current?.currentTime || 0)}
+ data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;
+ }
+ if (!scene.referenceImageId) return null;
+ return <img ref={mediaRef} src={`/data/images/${scene.referenceImageId}`} alt="" data-testid="beat-toggle-clip" style={{ opacity: 0 }} className={className} />;
+}
+
/**
* The cutting room floor outro after the song (lib/musicVideoStatsOutro.js):
* when the song ends it plays for its own length on a clock of its own, and
@@ -338,6 +397,13 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
const media = figure?.querySelector('video[data-ready="true"]:not([aria-hidden="true"])') || figure?.querySelector('img');
return captureMediaFrame(media);
};
+ // Beat toggles across cuts, on the frames the final render flips (lib/musicVideoCutEffects.js).
+ const beats = project.audioAnalysis?.beats;
+ const toggles = useMemo(() => sceneBeatToggles(scenes, { fps: ANIMATIC_FPS, beats, endSec: songEndSec || null,
+ isPerformance: (scene) => !!selectedPerformanceInstruction(scene)?.edit }), [scenes, beats, songEndSec]);
+ const leadOf = (scene) => toggles.find((t) => t.toSceneId === scene.sceneId)?.leadSec || 0;
+ // A toggle mounts its incoming clip a little ahead, so it has loaded when the flips start.
+ const nearToggles = toggles.filter((t) => time >= t.windowStartSec - 3 && time < t.cutSec);
const shot = shotAt(scenes, time);
const timed = timedShots(scenes);
const index = shot ? timed.indexOf(shot) : -1;
@@ -376,8 +442,12 @@ export default function StoryboardAnimatic({ project, audioUrl, seekRequest, col
</div>
) : null}
{clipShots.map((scene) => (
- <ShotClip key={`${scene.sceneId}:${scene.videoHistoryId}`} scene={scene} audioRef={audioRef} time={time} current={scene === shot} />
+ <ShotClip key={`${scene.sceneId}:${scene.videoHistoryId}`} scene={scene} audioRef={audioRef} time={time} current={scene === shot} leadSec={leadOf(scene)} />
))}
+ {nearToggles.map((toggle) => {
+ const incoming = scenes.find((scene) => scene.sceneId === toggle.toSceneId);
+ return incoming ? <ToggleClip key={`toggle:${toggle.toSceneId}:${incoming.videoHistoryId || ''}`} scene={incoming} toggle={toggle} audioRef={audioRef} time={time} /> : null;
+ })}
{/* The shots' VHS glitches on their cuts, on the frames the final render repaints. */}
<TransitionGlitch scenes={scenes} figureRef={figureRef} audioRef={audioRef} time={time} layered={layered}
endSec={Number(project.audioAnalysis?.durationSec) || null} fps={ANIMATIC_FPS} />
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
index 411c881c97..5eb7b19b4b 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
@@ -121,6 +121,36 @@ describe('StoryboardAnimatic clips', () => {
expect(container.querySelector('img').getAttribute('src')).toBe('/data/images/frame-b.png');
});
+ // Uniquely catches the animatic's beat toggle showing the incoming shot on the wrong frames,
+ // starting its clip in the wrong place, or the incoming shot jumping back to its start at the cut.
+ it('flips to the next shot on the beats before a beat toggle cut and carries its clip on through the cut', async () => {
+ const scenes = [
+ { sceneId: 'a', label: 'One', startSec: 0, endSec: 4, videoHistoryId: 'clip-a', cutEffect: { style: 'beat-toggle', enabled: true, overlapSec: 1, rate: 'beat' } },
+ { sceneId: 'b', label: 'Two', startSec: 4, endSec: 8, videoHistoryId: 'clip-b' },
+ ];
+ const project = { id: 'p', scenes, audioAnalysis: { beats: [3, 3.5, 4, 4.5] } };
+ render(<StoryboardAnimatic project={project} audioUrl="/song.mp3" />);
+ const audio = screen.getByLabelText('Song for the storyboard animatic');
+ const at = (t) => { Object.defineProperty(audio, 'currentTime', { value: t, configurable: true }); fireEvent.timeUpdate(audio); };
+ at(2.9);
+ const toggle = await screen.findByTestId('beat-toggle-clip');
+ await waitFor(() => expect(toggle.getAttribute('src')).toBe('/data/videos/clip-b.mov'));
+ Object.defineProperty(toggle, 'readyState', { value: 4, configurable: true });
+ expect(toggle.style.opacity).toBe('0');
+ at(3.2);
+ expect(toggle.style.opacity).toBe('1');
+ // The incoming clip started on the first beat.
+ expect(toggle.currentTime).toBeCloseTo(0.2, 2);
+ at(3.7);
+ expect(toggle.style.opacity).toBe('0');
+ // Past the cut the incoming shot's own clip runs on from where the toggle took it.
+ const clipB = screen.getByLabelText('Clip for Two');
+ Object.defineProperty(clipB, 'readyState', { value: 4, configurable: true });
+ Object.defineProperty(audio, 'paused', { value: true, configurable: true });
+ at(4.25);
+ expect(clipB.currentTime).toBeCloseTo(1.25, 2);
+ });
+
it('loads the next shots\' clips ahead and cuts to the same loaded clip', async () => {
const scenes = [
{ sceneId: 'a', label: 'One', startSec: 0, endSec: 1, referenceImageId: 'frame-a.png' },
diff --git a/client/src/lib/README.md b/client/src/lib/README.md
index 6b3ed244aa..7852706aa9 100644
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@ -318,4 +318,5 @@ grep -i "what you want to do" client/src/lib/README.md
| `filmLook.js` | Shared film look controls, presets, normalizer, prompt vocabulary, the SVG filter markup the editor previews with and the playback-time motion layers (server `lib/filmLook.js`). |
| `musicVideoNarrativeEvents.js` | Named narrative event kinds, absolute onset/word/frame resolution and deterministic silence/reactive state for composition documents. |
| `musicVideoImageEffects.js` | Music-video image effects (a camera flash that develops over a still shot), re-exported from `server/lib/musicVideoImageEffects.js`: the effects, labels and develop lengths, `sceneImageEffect`, `imageEffectState`. Used by `components/musicVideo/SceneImageEffect.jsx` and the storyboard animatic. |
+| `musicVideoCutEffects.js` | Music-video cut effects (the beat toggle across a cut), re-exported from `server/lib/musicVideoCutEffects.js`: settings, labels, `sceneCutEffect`, `sceneBeatToggles`. Used by `components/musicVideo/SceneCutEffect.jsx` and the storyboard animatic. |
| `musicVideoTransitions.js` | Music-video cut transitions (VHS glitches over a cut), re-exported from `server/lib/musicVideoTransitions.js`: the styles and labels, `sceneTransitionSections`, `transitionGlitchFrames`, `transitionFrameAt`, in-shot glitches (`sceneShotGlitchFrames`). Used by `components/musicVideo/SceneTransitions.jsx`, `SceneGlitches.jsx` and the storyboard animatic. |
diff --git a/client/src/lib/index.js b/client/src/lib/index.js
index a4331c2d62..813e32cb08 100644
--- a/client/src/lib/index.js
+++ b/client/src/lib/index.js
@@ -277,5 +277,6 @@ export * from './trackProvenance.js';
export * from './musicVideoGrade.js';
export * from './filmLook.js';
export * from './musicVideoNarrativeEvents.js';
+export * from './musicVideoCutEffects.js';
export * from './musicVideoTransitions.js';
export * from './musicVideoImageEffects.js';
diff --git a/client/src/lib/musicVideoCutEffects.js b/client/src/lib/musicVideoCutEffects.js
new file mode 100644
index 0000000000..738c3436b2
--- /dev/null
+++ b/client/src/lib/musicVideoCutEffects.js
@@ -0,0 +1,16 @@
+/**
+ * Music Video cut effects — re-export of `server/lib/musicVideoCutEffects.js`,
+ * so the shot card's beat toggle control and the animatic's toggling follow the
+ * same settings and frames as the final render.
+ */
+export {
+ BEAT_TOGGLE_OVERLAP,
+ BEAT_TOGGLE_RATE_LABELS,
+ BEAT_TOGGLE_RATES,
+ MUSIC_VIDEO_CUT_EFFECT_LABELS,
+ MUSIC_VIDEO_CUT_EFFECTS,
+ beatToggleCuts,
+ beatToggleSwitches,
+ sceneBeatToggles,
+ sceneCutEffect,
+} from '../../../server/lib/musicVideoCutEffects.js';
diff --git a/server/lib/README.md b/server/lib/README.md
index 24b950fcf7..e1fe52e70f 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -872,6 +872,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `musicVideoLocalModels.js` | Which local video models render which Music Video shot: `localModelRendersCutaway` (image-to-video), `localModelRendersPerformance` (LTX-2 family a2v or MiniMax H3 Ref2VA, from the still + song slice), `localMusicVideoModels(models, performance)` and `localModelTakesSongAudio` (LTX-2 family a2v: a cutaway can take its song slice). Shared by the shot card and project model pickers and `submitJob.js` / `prepareParams.js`. |
| `musicVideoHandoffFrame.js` | Continuing shots: `sceneClipOutFrameSec(scene, clipDurationSec, fps)` is the time in a shot's clip of the last frame the cut shows (performance edit in/out, loop wrap, hold past the clip's end); `previousShot(project, sceneId)` is the shot before it by start time. |
| `musicVideoCrossLinks.js` | Cross-links between a release's posts: each post's public link, the `Label: url` lines a new draft adds, which links a posted record carries (`links`, or what its draft always linked), and `crossLinkBackfill(kit)` listing the links each editable post (YouTube, Suno, X, Stacker News) still lacks. Shared by the publish payloads and the Publish stage. |
+| `musicVideoCutEffects.js` | Music Video cut effects a shot sets on its cut out (`scene.cutEffect`, kept while switched off): the beat toggle, which flips the picture between the two shots' clips on the beats before the cut. `sceneCutEffect`, `beatToggleSwitches`, `beatToggleCuts` (the render's frames) and `sceneBeatToggles` (the animatic's song times). Browser-safe. |
| `musicVideoExcerptRange.js` | `excerptRangeFits(startSec, endSec, totalSec, fps)`: whether a code/document excerpt window fits a frame-quantized render, allowing the end up to one frame past it (scene ends follow the analysed song length; the caller clamps). The start must fall inside the render; an unknown fps allows no slack. |
| `musicVideoPlateEvidence.js` | Asset-bound authored-shot plate requirements, current evidence and selected-image admission. |
diff --git a/server/lib/index.js b/server/lib/index.js
index dce1e21aab..65da717b9e 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -880,6 +880,7 @@ export * from './deepAuditCoverage.js';
export * from './auditWorkflow.js';
+export * from './musicVideoCutEffects.js';
export * from './musicVideoTransitions.js';
export * from './musicVideoImageEffects.js';
diff --git a/server/lib/musicVideoCutEffects.js b/server/lib/musicVideoCutEffects.js
new file mode 100644
index 0000000000..4cf42badd3
--- /dev/null
+++ b/server/lib/musicVideoCutEffects.js
@@ -0,0 +1,143 @@
+/**
+ * Music Video cut effects: configurable effects that play across a cut between
+ * two shots, set on the outgoing shot (`scene.cutEffect`) and switched on or
+ * off without losing their settings. Like the cut transitions
+ * (musicVideoTransitions.js) they never add time or move a cut.
+ *
+ * `beat-toggle`: over the last moments of the outgoing shot both clips play and
+ * the picture flips between them on the song's beats, incoming first
+ * (incoming, outgoing, incoming, …), then the incoming shot carries on. Its
+ * clip keeps running from where the toggle started it, so the cut lands mid-
+ * motion; a lip-synced (performance) shot instead starts from its in-point at
+ * the cut, so its mouth stays on the words.
+ *
+ * One definition serves both players: the final render overlays the incoming
+ * clip on exactly the frames `beatToggleCuts` names, and the storyboard
+ * animatic shows it on the same frames.
+ */
+
+export const MUSIC_VIDEO_CUT_EFFECTS = Object.freeze(['beat-toggle']);
+
+export const MUSIC_VIDEO_CUT_EFFECT_LABELS = Object.freeze({ 'beat-toggle': 'Beat toggle' });
+
+// How often the picture flips: on every beat, or on every half beat (twice as fast).
+export const BEAT_TOGGLE_RATES = Object.freeze(['beat', 'half']);
+export const BEAT_TOGGLE_RATE_LABELS = Object.freeze({ beat: 'Every beat', half: 'Every half beat' });
+
+// The overlap before the cut the toggle plays over, in seconds.
+export const BEAT_TOGGLE_OVERLAP = Object.freeze({ min: 0.25, max: 4, default: 1 });
+
+// With no beat grid the overlap is split into this many even steps.
+const FALLBACK_STEPS = 4;
+// A beat this close to the cut is the cut itself, not a flip before it.
+const CUT_GUARD_SEC = 0.02;
+
+/**
+ * A scene's cut effect with its settings filled in (`{ style, enabled,
+ * overlapSec, rate }`), or null when it has none. A switched-off effect keeps
+ * its settings, so turning it back on restores them.
+ */
+export function sceneCutEffect(scene) {
+ const effect = scene?.cutEffect;
+ if (!effect || !MUSIC_VIDEO_CUT_EFFECTS.includes(effect.style)) return null;
+ const overlap = Number(effect.overlapSec);
+ return {
+ style: effect.style,
+ enabled: effect.enabled !== false,
+ overlapSec: Number.isFinite(overlap) ? Math.min(BEAT_TOGGLE_OVERLAP.max, Math.max(BEAT_TOGGLE_OVERLAP.min, overlap)) : BEAT_TOGGLE_OVERLAP.default,
+ rate: BEAT_TOGGLE_RATES.includes(effect.rate) ? effect.rate : 'beat',
+ };
+}
+
+/**
+ * The song times (seconds) where a beat toggle flips the picture before a cut
+ * at `cutSec`: the beats (or half beats) in the `overlapSec` before it, else
+ * even steps when the song has no beat grid. The incoming shot shows from the
+ * first flip to the second, the third to the fourth, and so on; a flip left
+ * open runs to the cut.
+ */
+export function beatToggleSwitches({ cutSec, overlapSec, rate = 'beat', beats = [] }) {
+ const start = cutSec - overlapSec;
+ const end = cutSec - CUT_GUARD_SEC;
+ const grid = (Array.isArray(beats) ? beats : []).filter((b) => Number.isFinite(b)).sort((a, b) => a - b);
+ const points = [];
+ grid.forEach((b, i) => {
+ if (b >= start - 0.001 && b < end) points.push(b);
+ if (rate === 'half' && i + 1 < grid.length) {
+ const half = (b + grid[i + 1]) / 2;
+ if (half >= start - 0.001 && half < end) points.push(half);
+ }
+ });
+ if (!points.length) {
+ for (let k = 0; k < FALLBACK_STEPS; k++) points.push(start + (k * overlapSec) / FALLBACK_STEPS);
+ }
+ return [...new Set(points)].sort((a, b) => a - b);
+}
+
+/**
+ * The beat toggles on a cut timeline. `pieces` are its sections in order,
+ * `{ sceneId, startFrame, endFrame, partner?, cutEffect?, performance? }` on one
+ * frame grid (`cutEffect` as `sceneCutEffect` returns it). A toggle plays on the
+ * cut out of a shot's last piece into the next shot's first; an intercut
+ * partner piece on either side carries none. Returns, per toggle:
+ * `{ index, cutFrame, windowFrames, incoming: [[from, to)], leadFrames }` with
+ * `index` the outgoing piece, `incoming` the frames (on the timeline) that show
+ * the incoming clip, `windowFrames` how many frames before the cut the incoming
+ * clip starts playing, and `leadFrames` how far its clip has run by the cut
+ * (0 for a performance shot, which starts again at its in-point).
+ */
+export function beatToggleCuts(pieces, { fps, beats = [] }) {
+ const list = Array.isArray(pieces) ? pieces : [];
+ const lastOf = new Map();
+ const firstOf = new Map();
+ list.forEach((p, i) => {
+ if (!p || p.partner) return;
+ if (!firstOf.has(p.sceneId)) firstOf.set(p.sceneId, i);
+ lastOf.set(p.sceneId, i);
+ });
+ const out = [];
+ for (let i = 0; i < list.length - 1; i++) {
+ const from = list[i];
+ const to = list[i + 1];
+ const effect = from?.cutEffect;
+ if (!effect?.enabled || effect.style !== 'beat-toggle' || from.partner || to?.partner || from.sceneId === to.sceneId) continue;
+ if (lastOf.get(from.sceneId) !== i || firstOf.get(to.sceneId) !== i + 1) continue;
+ const cutFrame = to.startFrame;
+ const switches = beatToggleSwitches({ cutSec: cutFrame / fps, overlapSec: effect.overlapSec, rate: effect.rate, beats })
+ .map((sec) => Math.round(sec * fps))
+ // A flip keeps to the outgoing shot, after its first frame.
+ .filter((f, k, all) => f > from.startFrame && f < cutFrame && f !== all[k - 1]);
+ if (!switches.length) continue;
+ const incoming = [];
+ for (let k = 0; k < switches.length; k += 2) incoming.push([switches[k], k + 1 < switches.length ? switches[k + 1] : cutFrame]);
+ const windowFrames = cutFrame - switches[0];
+ out.push({ index: i, cutFrame, windowFrames, incoming, leadFrames: to.performance ? 0 : windowFrames });
+ }
+ return out;
+}
+
+/**
+ * The storyboard's beat toggles in song seconds (the animatic's counterpart of
+ * the render's): `[{ fromSceneId, toSceneId, windowStartSec, cutSec, incoming:
+ * [[fromSec, toSec)], leadSec }]`. `isPerformance(scene)` says whether a shot
+ * plays a lip-synced edit.
+ */
+export function sceneBeatToggles(scenes, { fps, beats = [], endSec = null, isPerformance = () => false }) {
+ const timed = (Array.isArray(scenes) ? scenes : []).filter((s) => Number.isFinite(s?.startSec)).sort((a, b) => a.startSec - b.startSec);
+ const pieces = timed.map((scene, i) => {
+ const next = timed[i + 1];
+ const stop = next ? next.startSec : (Number.isFinite(endSec) ? endSec : scene.endSec);
+ return {
+ sceneId: scene.sceneId, startFrame: Math.round(scene.startSec * fps), endFrame: Math.round((stop ?? scene.startSec) * fps),
+ cutEffect: sceneCutEffect(scene), performance: !!isPerformance(scene),
+ };
+ });
+ return beatToggleCuts(pieces, { fps, beats }).map((t) => ({
+ fromSceneId: pieces[t.index].sceneId,
+ toSceneId: pieces[t.index + 1].sceneId,
+ windowStartSec: (t.cutFrame - t.windowFrames) / fps,
+ cutSec: t.cutFrame / fps,
+ incoming: t.incoming.map(([a, b]) => [a / fps, b / fps]),
+ leadSec: t.leadFrames / fps,
+ }));
+}
diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.js
index ae508168d8..a7b43b72cc 100644
--- a/server/lib/musicVideoDependencies.js
+++ b/server/lib/musicVideoDependencies.js
@@ -23,6 +23,7 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s
// A glitch over its cuts changes the render; a shot without one keeps its evidence.
.concat(['transitionIn', 'transitionOut'].filter((key) => scene[key]).map((key) => [key, scene[key]]))
.concat(scene.glitches?.length ? [['glitches', scene.glitches]] : [])
+ .concat(scene.cutEffect && scene.cutEffect.enabled !== false ? [['cutEffect', scene.cutEffect]] : [])
// An image effect over a still changes the render; a still without one keeps its evidence.
.concat(scene.imageEffect ? [['imageEffect', scene.imageEffect], ['imageEffectSec', scene.imageEffectSec ?? null]] : [])
// An effect from the start frame to the end frame renders the end frame too.
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index cc5b9797ad..04e14d0eae 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -54,6 +54,7 @@ import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';
import { MUSIC_VIDEO_MEDIA } from './musicVideoMediumPlan.js';
import { MUSIC_VIDEO_FRAME_FITS, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';
import { MUSIC_VIDEO_SHOT_GLITCH_LIMIT, MUSIC_VIDEO_TRANSITIONS } from './musicVideoTransitions.js';
+import { BEAT_TOGGLE_OVERLAP, BEAT_TOGGLE_RATES, MUSIC_VIDEO_CUT_EFFECTS } from './musicVideoCutEffects.js';
import { IMAGE_EFFECT_SEC_RANGE, MUSIC_VIDEO_IMAGE_EFFECTS } from './musicVideoImageEffects.js';
import { CAMERA_FRAMINGS, CAMERA_MOVEMENT_VALUES, CAMERA_SPEEDS } from './cameraMovements.js';
import { MUSIC_VIDEO_PERFORMANCE_STYLES, MUSIC_VIDEO_SHOT_MODES, SOURCE_AUDIO_LIPSYNC } from './musicVideoShotTiming.js';
@@ -615,6 +616,13 @@ const sceneLayerFields = {
style: z.enum(MUSIC_VIDEO_TRANSITIONS),
lengthSec: z.number().positive().max(4).nullable().optional(),
})).max(MUSIC_VIDEO_SHOT_GLITCH_LIMIT).nullable().optional(),
+ // An effect across the cut out of this shot (musicVideoCutEffects.js), kept with its settings while switched off; null clears it.
+ cutEffect: z.object({
+ style: z.enum(MUSIC_VIDEO_CUT_EFFECTS),
+ enabled: z.boolean().optional(),
+ overlapSec: z.number().min(BEAT_TOGGLE_OVERLAP.min).max(BEAT_TOGGLE_OVERLAP.max).optional(),
+ rate: z.enum(BEAT_TOGGLE_RATES).optional(),
+ }).nullable().optional(),
// A code-drawn effect a still shot plays instead of a clip (musicVideoImageEffects.js) and how long
// it takes to finish; null clears it.
imageEffect: z.enum(MUSIC_VIDEO_IMAGE_EFFECTS).nullable().optional(),
diff --git a/server/services/musicVideo/cutEffects.test.js b/server/services/musicVideo/cutEffects.test.js
new file mode 100644
index 0000000000..da294e32ff
--- /dev/null
+++ b/server/services/musicVideo/cutEffects.test.js
@@ -0,0 +1,103 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { findFfmpeg } from '../../lib/ffmpeg.js';
+import { beatToggleCuts, sceneCutEffect } from '../../lib/musicVideoCutEffects.js';
+import { buildMusicVideoFfmpegArgs } from './render.js';
+import { applySceneUpdate } from './projectsLogic.js';
+
+const ffmpeg = await findFfmpeg();
+const dir = mkdtempSync(join(tmpdir(), 'portos-mv-cut-effects-'));
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+const run = (args) => execFileSync(ffmpeg, ['-v', 'error', ...args], { maxBuffer: 64 * 1024 * 1024 });
+const FPS = 12;
+const W = 64;
+const H = 48;
+const toggle = (extra = {}) => sceneCutEffect({ cutEffect: { style: 'beat-toggle', overlapSec: 0.5, ...extra } });
+
+// Uniquely catches where a beat toggle flips: which beats (or half beats) in the
+// overlap, the even fallback with no beat grid, which cuts carry one, and a
+// lip-synced shot restarting at its cut rather than running on.
+describe('beatToggleCuts', () => {
+ const piece = (sceneId, startFrame, endFrame, extra = {}) => ({ sceneId, startFrame, endFrame, ...extra });
+
+ it('flips to the incoming shot on each beat in the overlap before the cut, incoming first', () => {
+ const [cut] = beatToggleCuts([piece('a', 0, 24, { cutEffect: toggle() }), piece('b', 24, 48)], { fps: 24, beats: [0.25, 0.5, 0.75, 1] });
+ expect(cut).toEqual({ index: 0, cutFrame: 24, windowFrames: 12, incoming: [[12, 18]], leadFrames: 12 });
+ const [half] = beatToggleCuts([piece('a', 0, 24, { cutEffect: toggle({ rate: 'half' }) }), piece('b', 24, 48, { performance: true })], { fps: 24, beats: [0.25, 0.5, 0.75, 1] });
+ expect(half).toMatchObject({ incoming: [[12, 15], [18, 21]], leadFrames: 0 });
+ const [even] = beatToggleCuts([piece('a', 0, 24, { cutEffect: toggle() }), piece('b', 24, 48)], { fps: 24 });
+ expect(even.incoming).toEqual([[12, 15], [18, 21]]);
+ });
+
+ it('plays only when switched on, and never on an intercut partner or inside a split shot', () => {
+ expect(beatToggleCuts([piece('a', 0, 24, { cutEffect: toggle({ enabled: false }) }), piece('b', 24, 48)], { fps: 24 })).toEqual([]);
+ expect(beatToggleCuts([piece('a', 0, 24, { cutEffect: toggle() }), piece('b', 24, 48, { partner: true })], { fps: 24 })).toEqual([]);
+ expect(beatToggleCuts([piece('a', 0, 24, { cutEffect: toggle() }), piece('b', 24, 36), piece('a', 36, 48, { cutEffect: toggle() })], { fps: 24 })).toEqual([]);
+ });
+});
+
+// Per-frame mean luma of a raw gray video.
+function frameMeans(path) {
+ const raw = run(['-i', path, '-f', 'rawvideo', '-pix_fmt', 'gray', '-']);
+ const size = W * H;
+ return Array.from({ length: raw.length / size }, (_, f) => raw.subarray(f * size, (f + 1) * size).reduce((sum, v) => sum + v, 0) / size);
+}
+
+// Uniquely catches the toggle's filters failing in ffmpeg, flipping on the wrong
+// frames, adding time, or the incoming clip jumping back to its start at the cut.
+describe.skipIf(!ffmpeg)('a beat toggle across a cut in the final render', () => {
+ const render = (incomingExtra = {}) => {
+ const song = join(dir, 'song.wav');
+ run(['-f', 'lavfi', '-i', 'sine=frequency=440:duration=2', '-y', song]);
+ const outgoing = join(dir, 'outgoing.mp4');
+ const incoming = join(dir, 'incoming.mp4');
+ run(['-f', 'lavfi', '-i', `color=white:s=${W}x${H}:r=${FPS}:d=1`, '-pix_fmt', 'yuv420p', '-y', outgoing]);
+ // Its brightness climbs 8 levels a frame, so a frame's luma names its place in the clip.
+ run(['-f', 'lavfi', '-i', `color=black:s=${W}x${H}:r=${FPS}:d=2,geq=lum='40+8*N':cb=128:cr=128`, '-pix_fmt', 'yuv420p', '-y', incoming]);
+ const clip = (videoPath, i, extra = {}) => ({ sceneId: `s${i}`, videoPath, width: W, height: H, fps: FPS, duration: i ? 2 : 1, sourceSec: i ? 2 : 1, loop: false, inSec: 0, outSec: 1, frameFit: 'fill', ...extra });
+ const output = join(dir, 'toggle.mp4');
+ const plan = buildMusicVideoFfmpegArgs([clip(outgoing, 0, { cutEffect: toggle() }), clip(incoming, 1, incomingExtra)], song, output,
+ { audioDurationSec: 2, frameGrid: true, beats: [0.5, 0.75, 1, 1.25] });
+ run(plan.args.filter((arg, i) => !(arg === '-progress' || plan.args[i - 1] === '-progress')));
+ expect(plan.totalDuration).toBe(2);
+ return frameMeans(output);
+ };
+ // The clip's Y levels read back as full-range gray.
+ const clipFrame = (n) => ((40 + 8 * n - 16) * 255) / 219;
+
+ it('shows the incoming clip on its beats, keeps every frame and carries the clip on through the cut', () => {
+ const means = render();
+ expect(means).toHaveLength(2 * FPS);
+ const white = (f) => means[f] > 200;
+ // Outgoing until the first beat, the incoming clip from its start on frames 6-8, outgoing again on 9-11.
+ expect([0, 5, 9, 11].every(white)).toBe(true);
+ [6, 7, 8].forEach((f, k) => expect(Math.abs(means[f] - clipFrame(k))).toBeLessThan(4));
+ // At the cut the incoming clip is 6 frames in, as far as the toggle ran it.
+ expect(Math.abs(means[12] - clipFrame(6))).toBeLessThan(4);
+ });
+
+ // Uniquely catches a non-looping clip with no spare footage running out before its shot ends.
+ it('starts a clip with no footage to spare from its in-point at the cut', () => {
+ const means = render({ duration: 1, sourceSec: 1 });
+ [6, 7, 8].forEach((f, k) => expect(Math.abs(means[f] - clipFrame(k))).toBeLessThan(4));
+ expect(Math.abs(means[12] - clipFrame(0))).toBeLessThan(4);
+ expect(Math.abs(means[23] - clipFrame(11))).toBeLessThan(4);
+ });
+});
+
+// Uniquely catches the scene PATCH refusing the effect, losing its settings when switched off, or taking an unknown style.
+describe('a shot\'s cut effect on the board', () => {
+ const project = { id: 'mv-1', scenes: [{ sceneId: 's1', order: 0, startSec: 0, endSec: 4, takes: [] }] };
+ it('saves a beat toggle, keeps its settings switched off, clears it with null and refuses an unknown style', () => {
+ const cutEffect = { style: 'beat-toggle', enabled: true, overlapSec: 1, rate: 'half' };
+ const { project: set } = applySceneUpdate(project, 's1', { cutEffect });
+ expect(set.scenes[0].cutEffect).toEqual(cutEffect);
+ const { project: off } = applySceneUpdate(set, 's1', { cutEffect: { ...cutEffect, enabled: false } });
+ expect(sceneCutEffect(off.scenes[0])).toEqual({ ...cutEffect, enabled: false });
+ expect(applySceneUpdate(off, 's1', { cutEffect: null }).project.scenes[0].cutEffect).toBeNull();
+ expect(() => applySceneUpdate(project, 's1', { cutEffect: { style: 'crossfade' } })).toThrow();
+ });
+});
diff --git a/server/services/musicVideo/excerptRender.js b/server/services/musicVideo/excerptRender.js
index 199ff43bc0..cc490978f5 100644
--- a/server/services/musicVideo/excerptRender.js
+++ b/server/services/musicVideo/excerptRender.js
@@ -478,7 +478,7 @@ export async function startExcerptRender(projectId, { startSec, endSec, aspect =
};
if (!composition) {
- const { args } = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, frameGrid: composed, excerpt: { startSec, endSec: endClamped }, soundBed, grade: composed ? project.composition?.grade : null, aspect: canvasAspect });
+ const { args } = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, frameGrid: composed, excerpt: { startSec, endSec: endClamped }, soundBed, grade: composed ? project.composition?.grade : null, beats: project.audioAnalysis?.beats, aspect: canvasAspect });
startEncode(args);
return { jobId, excerptId };
}
@@ -497,7 +497,7 @@ export async function startExcerptRender(projectId, { startSec, endSec, aspect =
}).then((overlays) => {
signal.throwIfAborted();
job.overlayAbort = null;
- const layered = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, overlays, frameGrid: true, excerpt: { startSec, endSec: endClamped }, soundBed, grade: composed ? project.composition?.grade : null, aspect: canvasAspect });
+ const layered = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, overlays, frameGrid: true, excerpt: { startSec, endSec: endClamped }, soundBed, grade: composed ? project.composition?.grade : null, beats: project.audioAnalysis?.beats, aspect: canvasAspect });
startEncode(layered.args, 0.5);
}).catch(async (err) => {
const canceled = signal.aborted;
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index 44c544fbb6..597621e3f2 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -523,6 +523,7 @@ function buildScene(input, { order }) {
...(input.transitionIn ? { transitionIn: input.transitionIn } : {}),
...(input.transitionOut ? { transitionOut: input.transitionOut } : {}),
...(input.glitches?.length ? { glitches: input.glitches } : {}),
+ ...(input.cutEffect ? { cutEffect: input.cutEffect } : {}),
...(input.imageEffect ? { imageEffect: input.imageEffect } : {}),
...(input.imageEffectSec ? { imageEffectSec: input.imageEffectSec } : {}),
cardColor: input.cardColor ?? null,
@@ -609,6 +610,7 @@ export function replaceScenes(project, sceneInputs) {
...(prior.transitionIn ? { transitionIn: prior.transitionIn } : {}),
...(prior.transitionOut ? { transitionOut: prior.transitionOut } : {}),
...(prior.glitches?.length ? { glitches: prior.glitches } : {}),
+ ...(prior.cutEffect ? { cutEffect: prior.cutEffect } : {}),
// A still playing an image effect stays one: the effect is the shot's picture.
...(prior.imageEffect && prior.visualLayer === 'still' ? { visualLayer: 'still', imageEffect: prior.imageEffect,
...(prior.imageEffectSec ? { imageEffectSec: prior.imageEffectSec } : {}) } : {}),
@@ -623,7 +625,7 @@ export function replaceScenes(project, sceneInputs) {
// takes, and whether it plays as a clip or its still with an image effect) without
// adding a shot or lyric the storyboard approval did not cover.
const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',
- 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut', 'glitches',
+ 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut', 'glitches', 'cutEffect',
'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec'];
/**
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index aeeddb40dc..172ae7b46e 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -58,6 +58,7 @@ import { AUDIO_NORM, buildAudioBedMix } from '../videoTimeline/audioBedMix.js';
import { musicVideoEvents } from './events.js';
import { projectSoundBed } from './soundBed.js';
import { intercutClips } from './intercut.js';
+import { beatToggleCuts, sceneCutEffect } from '../../lib/musicVideoCutEffects.js';
import { imageEffectGlitchFrames, mergeGlitchFrames, musicVideoTransitionFilter, sceneTransitions, shotGlitchFrames, transitionGlitchFrames } from '../../lib/musicVideoTransitions.js';
import { stillMoveForCamera } from '../../lib/cameraMovements.js';
import { isEndFrameImageEffect, musicVideoImageEffectFilter, sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
@@ -119,6 +120,11 @@ const round3 = (n) => Math.round(n * 1000) / 1000;
// window was padded to the provider minimum) and ends at `edit.outSec`. It
// never loops: repeating a sung take would desync the mouth from the song.
const performanceEdit = (scene) => selectedPerformanceInstruction(scene)?.edit ?? null;
+// A shot's switched-on cut effect for its clip, or nothing.
+const cutEffectOf = (scene) => {
+ const effect = sceneCutEffect(scene);
+ return effect?.enabled ? { cutEffect: effect } : {};
+};
// A non-looping shot may run this much past its source clip; the gap holds the
// final frame (tpad) rather than repeating footage. Longer shortfalls block the
@@ -238,7 +244,7 @@ export async function resolveSceneClips(project, { layered = false, stillForMiss
if (layer !== 'footage') {
const spanSec = authoredSpan(scene);
if (spanSec == null) { untimed.push(scene.sceneId); continue; }
- const section = { sceneId: scene.sceneId, layer, inSec: 0, outSec: spanSec, duration: spanSec, sourceSec: spanSec, ...sceneTransitions(scene) };
+ const section = { sceneId: scene.sceneId, layer, inSec: 0, outSec: spanSec, duration: spanSec, sourceSec: spanSec, ...sceneTransitions(scene), ...cutEffectOf(scene) };
// A composed render has no code to run: a code shot (#10297) is a black
// ground the typography layer draws over, cut exactly like a blank card.
if (layer === 'code') {
@@ -297,6 +303,9 @@ export async function resolveSceneClips(project, { layered = false, stillForMiss
...footageFrameFit(scene),
// VHS glitches on its cuts (musicVideoTransitions.js) ride over the edit without moving it.
...sceneTransitions(scene),
+ // So does a beat toggle on its cut out (musicVideoCutEffects.js); a lip-synced shot restarts at its cut.
+ ...cutEffectOf(scene),
+ ...(edit ? { performance: true } : {}),
});
}
if (missing.length > 0) {
@@ -492,7 +501,7 @@ function sectionChain(c, input, { canonW, canonH, fps, frames, frameGrid }) {
// overlap the window, clipped to it and re-based to the excerpt's own timeline
// (0 = `startSec`) so a contact sheet built from the excerpt file can sample
// cut boundaries directly. `totalDuration` becomes the excerpt's own length.
-export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec = null, overlays = [], frameGrid: gridOption = false, excerpt = null, soundBed = null, grade = null, aspect = null } = {}) {
+export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec = null, overlays = [], frameGrid: gridOption = false, excerpt = null, soundBed = null, grade = null, aspect = null, beats = [] } = {}) {
if (!Array.isArray(clips) || clips.length === 0) throw new Error('buildMusicVideoFfmpegArgs: empty clips');
const frameGrid = gridOption || clips.some((c) => c.layer);
// Stills and cards have no dimensions of their own: the first footage clip
@@ -529,15 +538,14 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioD
// guarantees its span fits, within COVERAGE_TOLERANCE_SEC. A still loops its
// single image; a card needs no input (its colour is a filter source).
let nextInput = 0;
- for (const { c } of plan) {
- if (c.layer === 'card') { inputOf.push(null); continue; }
+ const addInput = (c) => {
+ if (c.layer === 'card') return null;
const clipInput = nextInput++;
if (c.layer === 'still') {
inputs.push('-loop', '1', '-framerate', String(fps), '-i', c.imagePath);
if (c.endImagePath) {
- inputOf.push({ clip: clipInput, end: nextInput++ });
inputs.push('-loop', '1', '-framerate', String(fps), '-i', c.endImagePath);
- continue;
+ return { clip: clipInput, end: nextInput++ };
}
} else {
if (c.loop !== false) inputs.push('-stream_loop', '-1');
@@ -545,15 +553,49 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioD
}
// A still edge reads the shot's still as one more looping input.
if (!c.layer && c.frameFit === 'still' && c.stillPath) {
- inputOf.push({ clip: clipInput, still: nextInput++ });
inputs.push('-loop', '1', '-framerate', String(fps), '-i', c.stillPath);
- } else inputOf.push(clipInput);
- }
+ return { clip: clipInput, still: nextInput++ };
+ }
+ return clipInput;
+ };
+ // A beat toggle (musicVideoCutEffects.js) flips the picture to the next shot's clip
+ // on the beats before a cut. That clip plays from where the toggle starts it, read
+ // from an input of its own, and its own section picks up where it left off.
+ const sectionFrame = (sec) => Math.round(sec * fps);
+ const toggles = beatToggleCuts(plan.map(({ c, startSec, endSec }) => ({
+ sceneId: c.sceneId, startFrame: sectionFrame(startSec), endFrame: sectionFrame(endSec),
+ partner: !!c.intercutOf, cutEffect: c.cutEffect, performance: !!c.performance,
+ })), { fps, beats });
+ const prerolls = toggles.map((t) => {
+ const incoming = plan[t.index + 1];
+ const clip = { ...incoming.c, outSec: incoming.c.inSec + t.windowFrames / fps };
+ // A clip that doesn't loop only runs on as far as its spare footage allows, so
+ // the shot never runs out early and freezes on its last frame.
+ const spare = incoming.c.loop === false
+ ? Math.max(0, (incoming.c.sourceSec ?? incoming.c.duration) - (incoming.c.outSec - incoming.c.inSec))
+ : Infinity;
+ const lead = Math.min(t.leadFrames / fps, spare);
+ if (lead > 0 && !incoming.c.layer) {
+ incoming.c = { ...incoming.c, inSec: incoming.c.inSec + lead, outSec: incoming.c.outSec + lead, sourceSec: (incoming.c.sourceSec ?? incoming.c.duration) - lead };
+ }
+ return { ...t, clip };
+ });
+ for (const { c } of plan) inputOf.push(addInput(c));
+ for (const preroll of prerolls) preroll.input = addInput(preroll.clip);
const audioIdx = nextInput; // master audio follows the section inputs
inputs.push('-i', audioPath);
for (const overlay of overlays) inputs.push('-itsoffset', String(overlay.startSec), '-i', overlay.path);
- const filters = plan.map(({ c, frames }, i) => `${sectionChain(c, inputOf[i], { canonW, canonH, fps, frames, frameGrid })}[v${i}]`);
+ const filters = plan.map(({ c, frames }, i) => `${sectionChain(c, inputOf[i], { canonW, canonH, fps, frames, frameGrid })}[${prerolls.some((p) => p.index === i) ? 'vraw' : 'v'}${i}]`);
+ for (const p of prerolls) {
+ const { frames, startSec } = plan[p.index];
+ const start = sectionFrame(startSec);
+ const pad = Math.max(0, frames - p.windowFrames);
+ const shown = p.incoming.map(([a, b]) => `between(n,${a - start},${b - start - 1})`).join('+');
+ filters.push(`${sectionChain(p.clip, p.input, { canonW, canonH, fps, frames: p.windowFrames, frameGrid })}[pre${p.index}]`);
+ filters.push(`[pre${p.index}]tpad=start=${pad}:color=black,setpts=N/(${fps}*TB)[prepad${p.index}]`);
+ filters.push(`[vraw${p.index}][prepad${p.index}]overlay=eof_action=pass:enable='${shown}'[v${p.index}]`);
+ }
const gradeFilter = musicVideoGradeFilter(grade, plan.map(({ c, startSec, endSec }) => ({ sceneId: c.sceneId, startSec, endSec })), { fps });
// Cut transitions repaint a few frames around their cuts, under the typography, and add no time.
const pieces = plan.map(({ c, startSec, endSec }) => ({
@@ -918,7 +960,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
const jobId = randomUUID();
const filename = `music-video-${projectId.slice(0, 8)}-${Date.now()}.mp4`;
const outputPath = join(PATHS.videos, filename);
- const { args, totalDuration, canonW, canonH, fps, sections } = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, frameGrid: composed, soundBed, grade: composed ? project.composition?.grade : null, aspect });
+ const { args, totalDuration, canonW, canonH, fps, sections } = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, frameGrid: composed, soundBed, grade: composed ? project.composition?.grade : null, beats: project.audioAnalysis?.beats, aspect });
// #8984: a composed project lays its timed text cues over the cut, and a
// title card's text (#8985) joins them over its own section. No renderable
// cue (plain mode, or nothing timed) skips the overlay capture entirely.
@@ -1158,7 +1200,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
signal.throwIfAborted();
// Capture is over: from here a cancel kills the encode (job.process).
job.overlayAbort = null;
- const layered = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, overlays, frameGrid: true, soundBed, grade: project.composition?.grade });
+ const layered = buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioDurationSec, overlays, frameGrid: true, soundBed, grade: project.composition?.grade, beats: project.audioAnalysis?.beats });
startEncode(layered.args, 0.5);
}).catch(async (err) => {
const canceled = signal.aborted;