Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video: start frame, clip speed and frame offset per shot (#11392)

[._.]/4db177e8b866
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 89c26f95f8..d5844441fe 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -1,6 +1,8 @@
import PlateComparison from './PlateComparison.jsx';
import ShotActionInspector from './ShotActionInspector.jsx';
import SceneCameraControls from './SceneCameraControls.jsx';
+import SceneStartFrame from './SceneStartFrame.jsx';
+import SceneClipTiming from './SceneClipTiming.jsx';
import { MUSIC_VIDEO_MEDIUM_LABELS } from '../../../../server/lib/musicVideoMediumPlan.js';
import { useEffect, useRef, useState } from 'react';
import { Trash2, Activity, ArrowUp, ArrowDown, ChevronRight, Video, Maximize2, AlertTriangle, Clapperboard, Shuffle } from 'lucide-react';
@@ -14,7 +16,7 @@ import { isPerformanceScene, planPerformanceRepair } from '../../lib/musicVideoS
import SceneTakeStrip from './SceneTakeStrip.jsx';
import { sceneEndFramePlanFor } from './SceneEndFrame.jsx';
import SceneFrames from './SceneFrames.jsx';
-import { MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS, sceneHasAuthoredSpan } from '../../lib/musicVideoLayers.js';
+import { MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS, clipOffsetSec, clipSpeedRate, sceneHasAuthoredSpan } from '../../lib/musicVideoLayers.js';
// #8985: what a composed render shows for this scene's span.
const LAYER_LABELS = { footage: 'Footage', still: 'Still image', card: 'Title card', code: 'Code-drawn' };
@@ -113,7 +115,7 @@ export default function SceneCard({
onMove, onDelete, onEditLocal, onSave,
project = null, onGenerateFrame, onGenerateVideo, onContinueVideo, onGenerateEndFrame, generatingEndFrame = false, onContinueFromPrevious, previousHasClip = false,
onOpenPreview, onSelectTake, onReviewTake, onImportTake, onImportClipTake, onPlacePoolItem, onTrimClip, projectId = null, aspect = '16:9', videoModels = [], defaultVideoModel = '', shotNumbers = undefined, takeBusy = false, layered = false, documentComposition = false,
- lipSyncBackend = '', songDurationSec = null, onSplit, onMergeNext, nextEndSec = null, falVideoSettings = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,
+ lipSyncBackend = '', songDurationSec = null, onSplit, onMergeNext, onMoveStart = null, nextEndSec = null, falVideoSettings = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,
expanded, onToggleExpand, footageOptional = false, failedScenes = null, songReview = null,
}) {
const detailsRef = useRef(null);
@@ -151,8 +153,13 @@ export default function SceneCard({
const spanSec = scene.beatAligned && typeof scene.startSec === 'number' && typeof scene.endSec === 'number'
? scene.endSec - scene.startSec
: null;
- const shortBySec = !loops && spanSec != null && clipSec != null ? spanSec - clipSec : 0;
+ // The clip's playing length with its hand-set speed and offset, as the render times it.
+ const clipRate = clipSpeedRate(scene);
+ const clipLead = clipOffsetSec(scene);
+ const playedSec = clipSec != null ? Math.max(0, -clipLead) + Math.max(0, clipSec - Math.max(0, clipLead)) / clipRate : null;
+ const shortBySec = !loops && spanSec != null && playedSec != null ? spanSec - playedSec : 0;
const underCovered = shortBySec > COVERAGE_TOLERANCE_SEC;
+ useEffect(() => { if (clipPlayerRef.current) clipPlayerRef.current.playbackRate = clipRate; }, [clipRate]);
const applyPatch = (patch) => { onEditLocal(scene.sceneId, patch); onSave(scene.sceneId, patch); };
const layer = MUSIC_VIDEO_VISUAL_LAYERS.includes(scene.visualLayer) ? scene.visualLayer : 'footage';
const fieldId = (name) => `mv-scene-${scene.sceneId}-${name}`;
@@ -187,7 +194,7 @@ export default function SceneCard({
const lockBlocked = !shotLock ? null
: shotLock.clip === 'locked' ? 'Video locked. Unlock the video to render it again.'
: shotLock.still !== 'locked' ? 'Lock the still first' : null;
- const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec, failed: failedScenes || {}, songReview, videoSettings: falVideoSettings, locked });
+ const attention = sceneAttention(scene, { layered, footageOptional, lipSyncBackend, songDurationSec, clipSec: playedSec, failed: failedScenes || {}, songReview, videoSettings: falVideoSettings, locked });
const songChange = songReview?.[scene.sceneId];
const generatingLane = generatingFrame && generatingVideo ? 'Frame + clip' : generatingFrame ? 'Frame' : generatingVideo ? 'Clip' : null;
// #10154: the server-persisted failure of the last render, shown until a
@@ -241,6 +248,11 @@ export default function SceneCard({
);
const shotSettings = (
<>
+ {/* Move the cut before this shot a frame at a time; the shot before ends where this one starts. */}
+ {onMoveStart && sceneHasAuthoredSpan(scene) && (
+ <SceneStartFrame id={fieldId('start-frame')} scene={scene} joinsPrevious={(scene.order ?? index) > 0} onMove={onMoveStart}
+ disabled={shotLock?.clip === 'locked'} disabledReason={shotLock?.clip === 'locked' ? 'Video locked. Unlock the video to retime it.' : null} />
+ )}
{/* What this shot's render covers and costs, and what blocks it, stays in view. */}
<SceneTimingPlan timing={timing} sceneId={scene.sceneId} lipSyncBackend={renderLane} onSplit={onSplit} onMergeNext={onMergeNext}
span={sceneHasAuthoredSpan(scene) ? { startSec: scene.startSec, endSec: scene.endSec } : null} />
@@ -445,10 +457,10 @@ export default function SceneCard({
<div role="alert" className="flex flex-wrap items-center gap-2 rounded border border-port-warning/40 bg-port-warning/10 px-2 py-1.5 text-xs text-port-warning">
<AlertTriangle size={13} className="shrink-0" />
<span className="min-w-0 flex-1 basis-48">
- Shot runs {spanSec.toFixed(1)}s but its clip is {clipSec.toFixed(1)}s — the render won&apos;t repeat it.
+ Shot runs {spanSec.toFixed(1)}s but its clip is {playedSec.toFixed(1)}s — the render won&apos;t repeat it.
Trim the shot, continue or regenerate a longer clip, or loop it on purpose.
</span>
- <button type="button" onClick={() => applyPatch({ endSec: Math.round((scene.startSec + clipSec) * 1000) / 1000 })}
+ <button type="button" onClick={() => applyPatch({ endSec: Math.round((scene.startSec + playedSec) * 1000) / 1000 })}
className="rounded bg-port-bg border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0 text-port-text">Trim to clip</button>
<button type="button" onClick={() => applyPatch({ loop: true })}
className="rounded bg-port-bg border border-port-border px-2 py-1 min-h-[44px] sm:min-h-0 text-port-text">Loop clip</button>
@@ -503,6 +515,7 @@ export default function SceneCard({
preload="metadata"
controls
onLoadedMetadata={(e) => {
+ e.currentTarget.playbackRate = clipRate;
const sec = e.currentTarget.duration;
const { videoWidth: width, videoHeight: height } = e.currentTarget;
if (Number.isFinite(sec) && sec > 0) setClipMeta({ id: scene.videoHistoryId, sec, width, height });
@@ -531,6 +544,11 @@ export default function SceneCard({
disabled={takeBusy || !!generatingVideo || shotLock?.clip === 'locked'}
onTrim={(startSec) => onTrimClip(scene, startSec)} />
)}
+ {/* Speed and offset by hand, from the same clip file, in the player and the final render. */}
+ {scene.videoHistoryId && layer === 'footage' && (
+ <SceneClipTiming fieldId={fieldId} scene={scene} onChange={applyPatch}
+ disabled={shotLock?.clip === 'locked'} disabledReason={shotLock?.clip === 'locked' ? 'Video locked. Unlock the video to retime it.' : null} />
+ )}
{/* A clip of another shape (a 3:2 local render) zooms to fill, or keeps its ratio with covered edges. */}
{scene.videoHistoryId && (
<SceneFrameFit id={fieldId('frame-fit')} scene={scene} clipSize={clipSize} aspect={aspect}
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index 06d6fc2044..0c0758e199 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -583,6 +583,51 @@ describe('a still picture in the clip column', () => {
});
});
+describe('clip speed and offset by hand', () => {
+ function Harness({ initial, onSave }) {
+ const [scene, setScene] = useState(initial);
+ return <SceneCard scene={scene} index={0} expanded onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
+ }
+
+ // Uniquely catches the steppers saving something other than whole fps/frames, a zero not clearing,
+ // or the inline clip ignoring the speed it will render at.
+ it('steps speed in fps and offset in frames, saves them as shot settings, and plays the clip at that speed', () => {
+ const onSave = vi.fn();
+ const { container } = render(<Harness initial={{ sceneId: 's1', prompt: 'she dances', videoHistoryId: 'clip-1', startSec: 0, endSec: 4, takes: [] }} onSave={onSave} />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play the clip one fps faster' }));
+ fireEvent.click(screen.getByRole('button', { name: 'Play the clip one fps faster' }));
+ expect(onSave).toHaveBeenLastCalledWith('s1', { speedFps: 2 });
+ expect(screen.getByText('26 fps (1.08×)')).toBeTruthy();
+ expect(container.querySelector('video').playbackRate).toBeCloseTo(26 / 24);
+ fireEvent.click(screen.getByRole('button', { name: 'Start the clip one frame later' }));
+ expect(onSave).toHaveBeenLastCalledWith('s1', { offsetFrames: -1 });
+ expect(screen.getByText('clip starts 1 frame later')).toBeTruthy();
+ const offset = screen.getByLabelText('Offset');
+ fireEvent.change(offset, { target: { value: '0' } });
+ fireEvent.blur(offset);
+ expect(onSave).toHaveBeenLastCalledWith('s1', { offsetFrames: null });
+ expect(screen.getByText('on the cut')).toBeTruthy();
+ // An emptied field is no change, not zero: the speed stays at +2.
+ const speed = screen.getByLabelText('Speed');
+ const saves = onSave.mock.calls.length;
+ fireEvent.change(speed, { target: { value: '' } });
+ fireEvent.blur(speed);
+ expect(onSave).toHaveBeenCalledTimes(saves);
+ expect(speed.value).toBe('2');
+ });
+
+ it('is off while the video is locked', async () => {
+ const { ShotLockContext } = await import('./ShotLock.jsx');
+ render(
+ <ShotLockContext.Provider value={{ locks: { s1: 'locked' }, clips: { s1: 'locked' }, toggle: () => {}, busy: false }}>
+ <SceneCard scene={{ sceneId: 's1', videoHistoryId: 'clip-1', startSec: 0, endSec: 4, speedFps: 2, takes: [] }} index={0} expanded onEditLocal={() => {}} onSave={() => {}} />
+ </ShotLockContext.Provider>,
+ );
+ expect(screen.getByRole('button', { name: 'Play the clip one fps faster' }).disabled).toBe(true);
+ expect(screen.getByLabelText('Speed').value).toBe('2');
+ });
+});
+
// Uniquely catches the beat toggle control saving the wrong shape or losing its settings when switched off.
describe('Beat toggle into the next shot', () => {
it('switches on with defaults, changes its settings and keeps them when switched off', () => {
diff --git a/client/src/components/musicVideo/SceneClipTiming.jsx b/client/src/components/musicVideo/SceneClipTiming.jsx
new file mode 100644
index 0000000000..79da21c79b
--- /dev/null
+++ b/client/src/components/musicVideo/SceneClipTiming.jsx
@@ -0,0 +1,60 @@
+import { useState } from 'react';
+import { Minus, Plus } from 'lucide-react';
+import { CLIP_OFFSET_FRAMES_RANGE, CLIP_SPEED_FPS_RANGE, CLIP_TIMING_FPS, clipSpeedRate } from '../../lib/musicVideoLayers.js';
+
+const STEP = 'flex items-center justify-center rounded bg-port-bg border border-port-border hover:bg-port-border/40 disabled:opacity-50 min-h-[44px] min-w-[44px] sm:min-h-[28px] sm:min-w-[28px]';
+const plural = (n) => `${n} frame${n === 1 ? '' : 's'}`;
+
+function Stepper({ id, label, value, range, disabled, onSet, readout, less, more }) {
+ const [draft, setDraft] = useState(null);
+ const commit = (next) => {
+ setDraft(null);
+ if (!Number.isInteger(next) || next === value || next < range.min || next > range.max) return;
+ onSet(next);
+ };
+ return (
+ <div className="flex flex-wrap items-center gap-2 text-xs">
+ <label htmlFor={id} className="w-12">{label}</label>
+ <span className="flex items-center gap-1">
+ <button type="button" onClick={() => commit(value - 1)} disabled={disabled || value <= range.min} className={STEP} aria-label={less}>
+ <Minus size={13} />
+ </button>
+ <input id={id} type="number" inputMode="numeric" min={range.min} max={range.max} step={1} value={draft ?? value} disabled={disabled}
+ onChange={(e) => setDraft(e.target.value)}
+ onBlur={(e) => (e.target.value.trim() === '' ? setDraft(null) : commit(Number(e.target.value)))}
+ onKeyDown={(e) => { if (e.key === 'Enter') e.currentTarget.blur(); }}
+ className="w-16 bg-port-bg border border-port-border rounded px-1 py-1 text-base sm:text-xs tabular-nums min-h-[44px] sm:min-h-0 disabled:opacity-50" />
+ <button type="button" onClick={() => commit(value + 1)} disabled={disabled || value >= range.max} className={STEP} aria-label={more}>
+ <Plus size={13} />
+ </button>
+ </span>
+ <span className="tabular-nums text-port-text-muted">{readout}</span>
+ </div>
+ );
+}
+
+/**
+ * A footage shot's clip timing, set by hand and played the same way by the
+ * player and the final render, from the one clip file: Speed in frames per
+ * second over the 24 fps grid, and Offset in frames against the song.
+ * `onChange(patch)` saves `{ speedFps }` or `{ offsetFrames }` (0 clears it).
+ */
+export default function SceneClipTiming({ fieldId, scene, disabled = false, disabledReason = null, onChange }) {
+ const speed = Number.isInteger(scene.speedFps) ? scene.speedFps : 0;
+ const offset = Number.isInteger(scene.offsetFrames) ? scene.offsetFrames : 0;
+ const rate = clipSpeedRate(scene);
+ const offsetText = offset > 0 ? `starts ${plural(offset)} into the clip`
+ : offset < 0 ? `clip starts ${plural(-offset)} later` : 'on the cut';
+ return (
+ <div className="space-y-1" title={disabledReason || undefined}>
+ <Stepper id={fieldId('speed')} label="Speed" value={speed} range={CLIP_SPEED_FPS_RANGE} disabled={disabled}
+ less="Play the clip one fps slower" more="Play the clip one fps faster"
+ onSet={(next) => onChange({ speedFps: next || null })}
+ readout={`${CLIP_TIMING_FPS + speed} fps (${Number(rate.toFixed(2))}×)`} />
+ <Stepper id={fieldId('offset')} label="Offset" value={offset} range={CLIP_OFFSET_FRAMES_RANGE} disabled={disabled}
+ less="Start the clip one frame later" more="Start the clip one frame further in"
+ onSet={(next) => onChange({ offsetFrames: next || null })}
+ readout={offsetText} />
+ </div>
+ );
+}
diff --git a/client/src/components/musicVideo/SceneStartFrame.jsx b/client/src/components/musicVideo/SceneStartFrame.jsx
new file mode 100644
index 0000000000..ccf9ca7d9d
--- /dev/null
+++ b/client/src/components/musicVideo/SceneStartFrame.jsx
@@ -0,0 +1,50 @@
+import { useState } from 'react';
+import { Minus, Plus } from 'lucide-react';
+import { formatTimecode } from '../../utils/formatters.js';
+
+// The cut grid: shots start on a 24 fps frame (the render's default canvas rate).
+const FPS = 24;
+const STEP = 'flex items-center justify-center rounded bg-port-bg border border-port-border hover:bg-port-border/40 disabled:opacity-50 min-h-[44px] min-w-[44px] sm:min-h-[28px] sm:min-w-[28px]';
+const frameOf = (sec) => Math.round(sec * FPS);
+const secOf = (frame) => Math.round((frame / FPS) * 1000) / 1000;
+
+/**
+ * Where a shot starts, as a frame number: step it a frame at a time or type
+ * one. `onMove(sceneId, startSec)` moves the cut, so the shot before this one
+ * (`joinsPrevious`) ends on the same frame.
+ */
+export default function SceneStartFrame({ id, scene, joinsPrevious = false, disabled = false, disabledReason = null, onMove }) {
+ const [draft, setDraft] = useState(null);
+ const [pending, setPending] = useState(null);
+ const frame = pending ?? frameOf(scene.startSec);
+ const lastFrame = Math.ceil(scene.endSec * FPS) - 1;
+ const locked = disabled || pending != null;
+ const move = (next) => {
+ setDraft(null);
+ if (!Number.isInteger(next) || next === frame || next < 0 || next > lastFrame) return;
+ setPending(next);
+ Promise.resolve(onMove(scene.sceneId, secOf(next))).finally(() => setPending(null));
+ };
+ const shown = draft != null && Number.isInteger(Number(draft)) && draft !== '' ? Number(draft) : frame;
+ return (
+ <div className="flex flex-wrap items-center gap-2 text-xs" title={disabledReason || undefined}>
+ <label htmlFor={id}>Start frame</label>
+ <span className="flex items-center gap-1">
+ <button type="button" onClick={() => move(frame - 1)} disabled={locked || frame <= 0} className={STEP} aria-label="Start one frame earlier">
+ <Minus size={13} />
+ </button>
+ <input id={id} type="number" inputMode="numeric" min={0} max={lastFrame} step={1} value={draft ?? frame} disabled={locked}
+ onChange={(e) => setDraft(e.target.value)}
+ onBlur={(e) => (e.target.value.trim() === '' ? setDraft(null) : move(Number(e.target.value)))}
+ onKeyDown={(e) => { if (e.key === 'Enter') e.currentTarget.blur(); }}
+ className="w-20 bg-port-bg border border-port-border rounded px-1 py-1 text-base sm:text-xs tabular-nums min-h-[44px] sm:min-h-0 disabled:opacity-50" />
+ <button type="button" onClick={() => move(frame + 1)} disabled={locked || frame >= lastFrame} className={STEP} aria-label="Start one frame later">
+ <Plus size={13} />
+ </button>
+ </span>
+ <span className="tabular-nums text-port-text-muted" data-testid="start-frame-time">
+ {formatTimecode(secOf(shown))} · {FPS} fps{joinsPrevious ? ' · the shot before ends here' : ''}
+ </span>
+ </div>
+ );
+}
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index 6ff3cbe27d..1df615942c 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -16,7 +16,7 @@ import FrameStepper, { captureMediaFrame } from './FrameStepper.jsx';
import { sceneFrameFit } from './SceneFrameFit.jsx';
import TransitionGlitch from './TransitionGlitch.jsx';
import ImageEffectStill from './ImageEffectStill.jsx';
-import { isLayeredComposition } from '../../lib/musicVideoLayers.js';
+import { clipOffsetSec, clipSpeedRate, isLayeredComposition } from '../../lib/musicVideoLayers.js';
import { sceneImageEffect } from '../../lib/musicVideoImageEffects.js';
import { sceneBeatToggles } from '../../lib/musicVideoCutEffects.js';
@@ -163,10 +163,22 @@ const CLIP_PAUSED_DRIFT_SEC = 0.01;
* Where song time `t` lands in a shot's clip: the take's edit-in point (a
* performance take trimmed to the sung window) plus the time into the shot. Past
* the clip's end it loops (`loop`, the render's default) or holds the last frame.
+ * A hand-set speed and offset (lib/musicVideoLayers.js) play as the render plays
+ * them: the offset moves the in-point by whole frames, a negative one past the
+ * clip's start holds its first frame that long, and the speed runs the clip faster
+ * or slower, looping round the whole file as the render's looped input does.
*/
export function clipTimeAt(scene, t, clipSec = null) {
const inSec = selectedPerformanceInstruction(scene)?.edit?.inSec ?? 0;
const local = Math.max(0, t - (scene.startSec || 0));
+ const rate = clipSpeedRate(scene);
+ const start = inSec + clipOffsetSec(scene);
+ if (rate !== 1 || start !== inSec) {
+ const sourceIn = Math.max(0, start);
+ const at = sourceIn + Math.max(0, local + Math.min(0, start)) * rate;
+ if (!(clipSec > 0) || at < clipSec) return at;
+ return scene.loop !== false && !(inSec > 0) ? at % clipSec : Math.max(0, clipSec - 0.05);
+ }
if (!(clipSec > inSec)) return inSec + local;
const span = clipSec - inSec;
if (local < span) return inSec + local;
@@ -203,7 +215,10 @@ function ShotClip({ scene, audioRef, time, current, leadSec = 0 }) {
const target = clipTimeAt(timing, current ? audio.currentTime || 0 : scene.startSec || 0, duration);
const tolerance = current && audio.paused ? CLIP_PAUSED_DRIFT_SEC : CLIP_DRIFT_SEC;
if (Math.abs((video.currentTime || 0) - target) > tolerance) video.currentTime = target;
- if (!current || audio.paused) video.pause?.();
+ // The shot's hand-set speed; a negative offset's lead holds the first frame.
+ video.playbackRate = clipSpeedRate(scene);
+ const leading = (scene.startSec || 0) - clipOffsetSec(scene) - (selectedPerformanceInstruction(scene)?.edit?.inSec ?? 0) > (audio.currentTime || 0);
+ if (!current || audio.paused || leading) video.pause?.();
else if (video.paused) video.play?.()?.catch?.(() => {});
}, [scene, timing, audioRef, current]);
// Shown once a frame is decoded where the clip should be; kept from then on, since a
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
index 5eb7b19b4b..aac7d9af9e 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
@@ -229,6 +229,19 @@ describe('StoryboardAnimatic clips', () => {
expect(clipTimeAt({ startSec: 10 }, 16, 4)).toBe(2);
expect(clipTimeAt({ startSec: 10, loop: false }, 16, 4)).toBeCloseTo(3.95);
});
+
+ // Uniquely catches the preview playing a shot's hand-set speed or offset differently from the render.
+ it('plays a hand-set speed and frame offset from the same clip, as the render does', () => {
+ // 48 fps on the 24 fps grid: twice as fast, starting 12 frames (0.5s) in.
+ const fast = { startSec: 10, speedFps: 24, offsetFrames: 12 };
+ expect(clipTimeAt(fast, 11, 6)).toBeCloseTo(2.5);
+ expect(clipTimeAt(fast, 13, 6)).toBeCloseTo(0.5);
+ expect(clipTimeAt({ ...fast, loop: false }, 13, 6)).toBeCloseTo(5.95);
+ // Six frames early: the first frame holds a quarter second, then the clip runs.
+ const early = { startSec: 10, offsetFrames: -6 };
+ expect(clipTimeAt(early, 10.2, 6)).toBe(0);
+ expect(clipTimeAt(early, 11.25, 6)).toBeCloseTo(1);
+ });
});
describe('StoryboardAnimatic film look', () => {
diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsx
index d5c9bbc9f8..e6c9b05ec1 100644
--- a/client/src/components/musicVideo/stages/BoardStage.jsx
+++ b/client/src/components/musicVideo/stages/BoardStage.jsx
@@ -213,6 +213,7 @@ export default function BoardStage({ board }) {
onDelete={board.onDeleteScene}
onSplit={board.onSplitScene}
onMergeNext={board.onMergeNextScene}
+ onMoveStart={board.onMoveSceneStart}
nextEndSec={scenes[idx + 1]?.endSec ?? null}
onRepairPerformance={['code', 'document'].includes(project.composition?.mode) ? null : board.onRepairPerformance}
repairBusy={board.repairBusy || videoSettings.saving}
diff --git a/client/src/lib/README.md b/client/src/lib/README.md
index 7852706aa9..13525e984a 100644
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@ -63,7 +63,7 @@ grep -i "what you want to do" client/src/lib/README.md
| `musicVideoFiling.js` | Which music video an Image Gen / Video Gen render is for: `MUSIC_VIDEO_FILING_PARAM` (the `musicVideo` URL param), `musicVideoFilingFields(searchParams)` (the `musicVideoProjectId` request field), `musicVideoOf(item)` (a record's video). Used by `components/media/MusicVideoFilingSelect.jsx`, `hooks/useMediaPreviewActions.js` and the two generate pages. |
| `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`, `AUDIO_REACTIVE_PERFORMANCE_GUARD`, `silentMotionPrompt`) for board motion prompts. |
| `musicVideoBatchPlan.js` | Pure helpers for the Music Video missing-frame / missing-clip batches (#10153): `videoBatchPreview` (count + fal estimate summed from `falSceneTake`, unpriced count), `scenesWaitingForFrame`, `videosButtonLabel`, `batchSummary`, `batchActive`, `batchSceneName`. Used by `components/musicVideo/ProjectActionGroups.jsx`. |
-| `musicVideoLayers.js` | Music-video per-scene visual layers (#8985), re-exported from `server/lib/musicVideoLayers.js`: `MUSIC_VIDEO_VISUAL_LAYERS`, `MUSIC_VIDEO_STILL_MOVES`, `MUSIC_VIDEO_TEXT_ZONES` / `MUSIC_VIDEO_LYRIC_ROLES` (#10583 lyric type per shot), `sceneVisualLayer(scene, { layered })`, `sceneHasAuthoredSpan`, `sceneRenderReady(scene, { layered })`. Used by `components/musicVideo/SceneCard.jsx`, `ProjectActionGroups.jsx` and `hooks/useMusicVideoSceneMedia.js`. |
+| `musicVideoLayers.js` | Music-video per-scene visual layers (#8985), re-exported from `server/lib/musicVideoLayers.js`: `MUSIC_VIDEO_VISUAL_LAYERS`, `MUSIC_VIDEO_STILL_MOVES`, `MUSIC_VIDEO_TEXT_ZONES` / `MUSIC_VIDEO_LYRIC_ROLES` (#10583 lyric type per shot), `sceneVisualLayer(scene, { layered })`, `sceneHasAuthoredSpan`, `sceneRenderReady(scene, { layered })`, and a footage shot's hand-set clip speed and offset (`clipSpeedRate`, `clipOffsetSec`). Used by `components/musicVideo/SceneCard.jsx`, `ProjectActionGroups.jsx` and `hooks/useMusicVideoSceneMedia.js`. |
| `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoSceneAttention.js` | Music Video per-scene attention model (#10152): `sceneAttention(scene, ctx)` returns the codes (`missing-frame`, `missing-clip`, `under-covered`, `contract-problem`, `perf-blocked`, `candidates-pending`, `plate-unverified`, `no-span`, `last-failure`) behind the collapsed scene-row chips, the Board `?scenes=attention\|missing\|all` filter (`parseSceneFilter`, `sceneMatchesFilter`) and the Produce per-shot strip; `SCENE_ATTENTION_LABELS`; `sceneMediaKind` (`clip` / `image` / `code` / `card`) and `SCENE_MEDIA_LABELS` for the Board thumbnail badges. Used by `components/musicVideo/SceneCard.jsx`, `stages/BoardStage.jsx` and `stages/ProduceStage.jsx`. |
| `musicVideoCoverOverlay.js` | Cover-art lettering preview (#10345): re-export of `server/lib/musicVideoCoverOverlay.js` (the design vocabulary, `normalizeCoverDesign`, `coverOverlaySvg`) plus `canvasCoverWidths(design, texts, fonts)`, which measures text with a canvas so the live preview wraps and sizes the title the way the server's measured render does. |
diff --git a/client/src/lib/musicVideoLayers.js b/client/src/lib/musicVideoLayers.js
index 9f208dfd83..34ed86f5ea 100644
--- a/client/src/lib/musicVideoLayers.js
+++ b/client/src/lib/musicVideoLayers.js
@@ -15,4 +15,9 @@ export {
sceneRenderReady,
isLayeredComposition,
LAYERED_COMPOSITION_MODES,
+ CLIP_TIMING_FPS,
+ CLIP_SPEED_FPS_RANGE,
+ CLIP_OFFSET_FRAMES_RANGE,
+ clipSpeedRate,
+ clipOffsetSec,
} from '../../../server/lib/musicVideoLayers.js';
diff --git a/client/src/lib/musicVideoSceneAttention.js b/client/src/lib/musicVideoSceneAttention.js
index 61fd132074..99e35cc9c5 100644
--- a/client/src/lib/musicVideoSceneAttention.js
+++ b/client/src/lib/musicVideoSceneAttention.js
@@ -63,7 +63,7 @@ function performanceBlocked(scene, { lipSyncBackend, songDurationSec, videoSetti
/**
* The attention codes for one scene, in display order (empty = nothing to do).
* ctx: `layered`, `footageOptional` (a mode whose shots need no clip),
- * `lipSyncBackend`, `songDurationSec`, `clipSec` (the measured clip length —
+ * `lipSyncBackend`, `songDurationSec`, `clipSec` (the measured clip length as it plays, with any hand-set speed and offset —
* only a mounted card knows it, so board-level counts omit `under-covered`),
* and `failed: { frame: {sceneId: true}, video: {sceneId: true} }` for renders
* that failed this session, and `songReview` (the project's revised-song
diff --git a/client/src/pages/MusicVideo.jsx b/client/src/pages/MusicVideo.jsx
index 40419ca8d0..5d5510cec9 100644
--- a/client/src/pages/MusicVideo.jsx
+++ b/client/src/pages/MusicVideo.jsx
@@ -25,6 +25,7 @@ import {
deleteMusicVideoScene,
splitMusicVideoScene,
mergeMusicVideoNextScene,
+ moveMusicVideoSceneStart,
reorderMusicVideoScenes,
importMusicVideoLyrics,
importMusicVideoTrackLyrics,
@@ -960,6 +961,11 @@ export default function MusicVideo() {
.catch((err) => toast.error(err?.message || 'Failed to merge scenes'));
};
+ // Move the cut before a shot: it starts at `startSec` and the shot before it ends there.
+ const handleMoveSceneStart = (sceneId, startSec) => moveMusicVideoSceneStart(selected.id, sceneId, startSec, { silent: true })
+ .then(({ project }) => replaceProject(project))
+ .catch((err) => toast.error(err?.message || 'Failed to move the shot start'));
+
const moveScene = (idx, dir) => {
const scenes = selected.scenes || [];
const target = idx + dir;
@@ -1303,6 +1309,7 @@ export default function MusicVideo() {
onDeleteScene: handleDeleteScene,
onSplitScene: handleSplitScene,
onMergeNextScene: handleMergeNextScene,
+ onMoveSceneStart: handleMoveSceneStart,
onRepairPerformance: revisions.repairPerformance,
repairBusy: revisions.busy,
onRenderStyle: (mode) => {
diff --git a/client/src/pages/MusicVideo.test.jsx b/client/src/pages/MusicVideo.test.jsx
index ffc4e17b8b..76ccfaec8c 100644
--- a/client/src/pages/MusicVideo.test.jsx
+++ b/client/src/pages/MusicVideo.test.jsx
@@ -74,6 +74,7 @@ vi.mock('../services/apiMusicVideo.js', () => ({
deleteMusicVideoScene: vi.fn(),
reorderMusicVideoScenes: vi.fn(),
splitMusicVideoScene: vi.fn(),
+ moveMusicVideoSceneStart: vi.fn(),
importMusicVideoLyrics: vi.fn(),
importMusicVideoTrackLyrics: vi.fn(),
alignMusicVideoLyrics: vi.fn(),
@@ -279,7 +280,7 @@ import MusicVideo from './MusicVideo.jsx';
import {
listMusicVideoProjects, listMusicVideoProjectSummaries, createMusicVideoProject, cloneMusicVideoProject, renderMusicVideoProject, planMusicVideoProject, updateMusicVideoProject,
deleteMusicVideoProject, transcribeMusicVideoMidi, cancelMusicVideoRender, analyzeMusicVideoProject, prepareMusicVideoSong,
- importMusicVideoLyrics, importMusicVideoTrackLyrics, separateMusicVideoVocals, alignMusicVideoLyrics, updateMusicVideoScene, splitMusicVideoScene,
+ importMusicVideoLyrics, importMusicVideoTrackLyrics, separateMusicVideoVocals, alignMusicVideoLyrics, updateMusicVideoScene, splitMusicVideoScene, moveMusicVideoSceneStart,
selectMusicVideoSceneTake, reviewMusicVideoSceneTake, importMusicVideoHandoff,
addMusicVideoSceneTake, getMusicVideoHandoffBundle,
renderMusicVideoExcerpt, deleteMusicVideoExcerpt, addMusicVideoExcerptNote,
@@ -1391,6 +1392,27 @@ describe('MusicVideo project video renderer', () => {
expect((await screen.findAllByText(/Shot · 2\/2$/, { selector: ':not(option)' })).length).toBeGreaterThan(0);
});
+ it('moves a shot start by frame number, and the shot before ends there too', async () => {
+ const first = { ...PROJECT_NO_CLIP.scenes[0], startSec: 0, endSec: 10 };
+ const second = { ...first, sceneId: 's2', order: 1, startSec: 10, endSec: 14, label: 'Second' };
+ const moved = [{ ...first, endSec: 10.042 }, { ...second, startSec: 10.042 }];
+ moveMusicVideoSceneStart.mockResolvedValue({ project: { ...PROJECT_NO_CLIP, scenes: moved }, scene: moved[1], previous: moved[0] });
+ await openProject({ ...PROJECT_NO_CLIP, scenes: [first, second] }, 'produce');
+
+ await act(async () => fireEvent.click(await screen.findByRole('button', { name: /^Open Shot 2/ })));
+ const input = screen.getByLabelText('Start frame');
+ expect(input.value).toBe('240');
+ const row = input.closest('div');
+ expect(within(row).getByTestId('start-frame-time').textContent).toMatch(/^0:10\.00 · 24 fps · the shot before ends here/);
+ fireEvent.click(within(row).getByRole('button', { name: 'Start one frame later' }));
+ await waitFor(() => expect(moveMusicVideoSceneStart).toHaveBeenCalledWith('mv-2', 's2', 10.042, expect.objectContaining({ silent: true })));
+ await waitFor(() => expect(screen.getByLabelText('Start frame').value).toBe('241'));
+ // A typed frame moves it there.
+ fireEvent.change(screen.getByLabelText('Start frame'), { target: { value: '228' } });
+ fireEvent.blur(screen.getByLabelText('Start frame'));
+ await waitFor(() => expect(moveMusicVideoSceneStart).toHaveBeenLastCalledWith('mv-2', 's2', 9.5, expect.anything()));
+ });
+
it('requests the Grok clip that covers a cutaway and adds approximate motion timing', async () => {
generateVideo.mockResolvedValue({ jobId: 'grok-job' });
await openProject({
diff --git a/client/src/services/apiMusicVideo.js b/client/src/services/apiMusicVideo.js
index dfd4679974..666f6c1877 100644
--- a/client/src/services/apiMusicVideo.js
+++ b/client/src/services/apiMusicVideo.js
@@ -141,6 +141,11 @@ export const mergeMusicVideoNextScene = (id, sceneId, backend, options = {}) =>
request(`/music-video/${encodeURIComponent(id)}/scenes/${encodeURIComponent(sceneId)}/merge-next`, {
method: 'POST', body: JSON.stringify(backend ? { backend } : {}), ...options,
});
+// Move a shot's start; the shot before ends there too → { project, scene, previous }.
+export const moveMusicVideoSceneStart = (id, sceneId, startSec, options = {}) =>
+ request(`/music-video/${encodeURIComponent(id)}/scenes/${encodeURIComponent(sceneId)}/start`, {
+ method: 'POST', body: JSON.stringify({ startSec }), ...options,
+ });
// Optional vocal stem (#8977): a full-length vocal bounce on the master's
// timebase that performance shots are conditioned on. Both → the project.
export const uploadMusicVideoVocalStem = (id, file, options = {}) => {
diff --git a/server/lib/README.md b/server/lib/README.md
index 089c2f4b67..737e5c46dc 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -562,7 +562,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `claimFlowTaskTypes.js` | `hasClaimFlowContract(task)` shares trusted claim identity across registration, completion and resume. `CLAIM_FLOW_TASK_TYPES` — the ONE list of task kinds whose prompts own their own `claim/<item>` worktree and push/PR/MR/review lifecycle (#6613). `services/cosTaskGenerator.js` stamps `metadata.claimFlow` from it (WRITE side); `services/agentPromptBuilder.js` reads it back through `isClaimFlowTask()` to pick a run's claim posture (READ side). It lived as two byte-identical copies, so a new claim type added to one silently split the writer from the reader — no error, just a claim run treated as a plain commit-only handoff; `claimFlowTaskTypes.test.js` fails if a local copy reappears. Imports nothing. |
| `chiptuneRender.js` | Deterministic chiptune score → mono PCM → 16-bit WAV buffer (pure Node, no audio deps); `renderScoreToPcm` / `pcmToWavBuffer` / `renderScoreToWav`. |
| `chiptuneScore.js` | Chiptune score contract (#2911): `chiptuneScoreSchema` (Zod) + `sanitizeChiptuneScore`, `buildScoreEvents` — the pattern/order → absolute-time flatten mirrored by `client/src/lib/chiptunePlayback.js`. |
-| `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). |
+| `musicVideoLayers.js` | Music Video per-scene visual layers (#8985): `MUSIC_VIDEO_VISUAL_LAYERS` (`footage`/`still`/`card`/`code`), `MUSIC_VIDEO_STILL_MOVES` (`hold`/`push`/`pan`), `MUSIC_VIDEO_FRAME_FITS` (`fill`/`still`/`vhs`/`black`: how an off-shape clip fills the frame), `sceneVisualLayer(scene, { layered })` (only a composed render honors a still/card), `sceneHasAuthoredSpan`, and `sceneRenderReady(scene, { layered })`. `clipSpeedRate(scene)` / `clipOffsetSec(scene)` read a footage shot's hand-set `speedFps` and `offsetFrames` on the 24 fps grid (`CLIP_TIMING_FPS`, `CLIP_SPEED_FPS_RANGE`, `CLIP_OFFSET_FRAMES_RANGE`). Pure leaf shared by `services/musicVideo/render.js` and the client board (`client/src/lib/musicVideoLayers.js`). |
| `musicVideoImageEffects.js` | Music Video image effects: a code-drawn effect a still shot plays instead of a generated clip (`imageEffect`: `flash-develop`, a camera flash then the picture develops like an instant photo over `imageEffectSec`; or a fade, flash or glitch from the start frame to the end frame). `sceneImageEffect(scene, { layered })`, `imageEffectState` (the effect at a time into the shot, painted by the storyboard animatic) and `musicVideoImageEffectFilter` (the same curves as ffmpeg filters for the final render). Browser-safe and import-free: document videos ship it as is. |
| `musicVideoTransitions.js` | Music Video cut transitions: VHS glitch styles a shot sets on the cut into (`transitionIn`) or out of (`transitionOut`) it, `transitionGlitchFrames` (the glitched frames of a cut timeline, never moving a cut), `transitionFrameAt`, `sceneTransitionSections`, `imageEffectGlitchFrames`/`mergeGlitchFrames` (end-frame glitch effects in the same shape), `shotGlitchFrames`/`sceneShotGlitchFrames` (glitches a shot sets at moments inside it, `scene.glitches`), `blockBox` and `musicVideoTransitionFilter` (per-frame ffmpeg filters for the final render). Browser-safe; the storyboard animatic paints the same frames. |
| `pitchMath.js` | Dependency-free scientific-pitch math shared by the LLM-authored music formats on both runtimes: `pitchToMidi("C4")` → 60 (null when unparseable) and `midiToFreq(69)` → 440. |
diff --git a/server/lib/musicVideoDependencies.js b/server/lib/musicVideoDependencies.js
index a7b43b72cc..be5d045444 100644
--- a/server/lib/musicVideoDependencies.js
+++ b/server/lib/musicVideoDependencies.js
@@ -20,6 +20,8 @@ const layerRef = (scene) => ({ role: 'layer', sceneId: scene.sceneId, assetId: s
.concat(scene.camera ? [['camera', scene.camera]] : [])
// A chosen frame fit changes the render; a shot left to fill keeps its evidence.
.concat(scene.frameFit && scene.frameFit !== 'fill' ? [['frameFit', scene.frameFit]] : [])
+ // A hand-set clip speed or offset changes the render; a shot left as rendered keeps its evidence.
+ .concat(['speedFps', 'offsetFrames'].filter((key) => scene[key]).map((key) => [key, scene[key]]))
// A glitch over its cuts changes the render; a shot without one keeps its evidence.
.concat(['transitionIn', 'transitionOut'].filter((key) => scene[key]).map((key) => [key, scene[key]]))
.concat(scene.glitches?.length ? [['glitches', scene.glitches]] : [])
diff --git a/server/lib/musicVideoLayers.js b/server/lib/musicVideoLayers.js
index 62cd052b04..9850769345 100644
--- a/server/lib/musicVideoLayers.js
+++ b/server/lib/musicVideoLayers.js
@@ -83,3 +83,17 @@ export function sceneRenderReady(scene, { layered = false } = {}) {
if (!sceneHasAuthoredSpan(scene)) return false;
return isSelfDrawnLayer(layer) || Boolean(scene.referenceImageId);
}
+
+// A footage shot's playback timing, set by hand on its card: `speedFps` plays
+// the clip that many frames a second faster (or slower) on the 24 fps grid, and
+// `offsetFrames` slides it against the song (positive starts that many frames
+// into the clip, negative holds its first frame that long). Both are applied
+// at playback and render, never by writing another copy of the clip.
+export const CLIP_TIMING_FPS = 24;
+export const CLIP_SPEED_FPS_RANGE = Object.freeze({ min: -18, max: 48 });
+export const CLIP_OFFSET_FRAMES_RANGE = Object.freeze({ min: -240, max: 240 });
+const whole = (value, { min, max }) => (Number.isInteger(value) ? Math.min(max, Math.max(min, value)) : 0);
+/** How fast a shot's clip plays: 1 is as rendered, 26/24 is two frames a second faster. */
+export const clipSpeedRate = (scene) => (CLIP_TIMING_FPS + whole(scene?.speedFps, CLIP_SPEED_FPS_RANGE)) / CLIP_TIMING_FPS;
+/** How far a shot's clip slides against the song, in seconds: positive skips in, negative holds the first frame. */
+export const clipOffsetSec = (scene) => whole(scene?.offsetFrames, CLIP_OFFSET_FRAMES_RANGE) / CLIP_TIMING_FPS;
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index 04e14d0eae..eb700fa3d0 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -52,7 +52,7 @@ import { EFFORT_LEVELS } from './providerModels.js';
import { shotActionContractProblem } from './musicVideoActionContract.js';
import { NARRATIVE_EVENT_KINDS } from './musicVideoNarrativeEvents.js';
import { MUSIC_VIDEO_MEDIA } from './musicVideoMediumPlan.js';
-import { MUSIC_VIDEO_FRAME_FITS, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';
+import { CLIP_OFFSET_FRAMES_RANGE, CLIP_SPEED_FPS_RANGE, MUSIC_VIDEO_FRAME_FITS, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_STILL_MOVES, MUSIC_VIDEO_TEXT_ZONES, MUSIC_VIDEO_VISUAL_LAYERS } from './musicVideoLayers.js';
import { MUSIC_VIDEO_SHOT_GLITCH_LIMIT, MUSIC_VIDEO_TRANSITIONS } from './musicVideoTransitions.js';
import { BEAT_TOGGLE_OVERLAP, BEAT_TOGGLE_RATES, MUSIC_VIDEO_CUT_EFFECTS } from './musicVideoCutEffects.js';
import { IMAGE_EFFECT_SEC_RANGE, MUSIC_VIDEO_IMAGE_EFFECTS } from './musicVideoImageEffects.js';
@@ -607,6 +607,9 @@ const sceneLayerFields = {
stillMove: z.enum(MUSIC_VIDEO_STILL_MOVES).optional(),
// How a clip of another shape fills the video's frame (musicVideoLayers.js).
frameFit: z.enum(MUSIC_VIDEO_FRAME_FITS).optional(),
+ // A footage clip played faster or slower by whole frames a second, and slid against the song by whole frames; null clears it.
+ speedFps: z.number().int().min(CLIP_SPEED_FPS_RANGE.min).max(CLIP_SPEED_FPS_RANGE.max).nullable().optional(),
+ offsetFrames: z.number().int().min(CLIP_OFFSET_FRAMES_RANGE.min).max(CLIP_OFFSET_FRAMES_RANGE.max).nullable().optional(),
// A VHS glitch over the cut into / out of this shot (musicVideoTransitions.js); null clears it.
transitionIn: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),
transitionOut: z.enum(MUSIC_VIDEO_TRANSITIONS).nullable().optional(),
@@ -1560,6 +1563,11 @@ export const musicVideoSceneSplitSchema = z.object({
atSec: z.number().finite().nonnegative().optional(),
}).strict();
+// Move where a shot starts; the shot before it ends there too, so the cut moves and nothing gaps or overlaps.
+export const musicVideoSceneStartSchema = z.object({
+ startSec: z.number().finite().nonnegative(),
+}).strict();
+
// Autonomous shot planner (#1855): propose a scene per analyzed audio section.
// `seedPrompts` (default true) additionally asks the active/given AI provider
// for a first-pass framePrompt/prompt per scene — best-effort, never fails the
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index ca6f210681..764638e587 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -28,6 +28,7 @@ import {
musicVideoSceneUpdateSchema,
musicVideoSceneReorderSchema,
musicVideoSceneSplitSchema,
+ musicVideoSceneStartSchema,
musicVideoPlanRequestSchema,
musicVideoManualAnalysisSchema,
musicVideoTranscribeMidiRequestSchema,
@@ -111,6 +112,7 @@ import {
reorderProjectScenes,
splitProjectScene,
mergeProjectNextScene,
+ moveProjectSceneStart,
setProjectMidiTranscription,
appendSceneTakes,
appendTakesAcrossScenes,
@@ -1503,6 +1505,12 @@ router.post('/:id/scenes/:sceneId/merge-next', asyncHandler(async (req, res) =>
res.json(await mergeProjectNextScene(req.params.id, req.params.sceneId, { backend }));
}));
+// Move a shot's start; the shot before it ends at the same time, so the cut between them moves.
+router.post('/:id/scenes/:sceneId/start', asyncHandler(async (req, res) => {
+ const { startSec } = validateRequest(musicVideoSceneStartSchema, req.body ?? {});
+ res.json(await moveProjectSceneStart(req.params.id, req.params.sceneId, { startSec }));
+}));
+
// --- Scene takes (#8965) ---
// Every render or import for a scene slot is kept as an immutable take; the
// slot field is the director's explicit selection among them. A take's asset
diff --git a/server/services/musicVideo/productionReview.js b/server/services/musicVideo/productionReview.js
index 7e7ebee8a5..e269c9ef9d 100644
--- a/server/services/musicVideo/productionReview.js
+++ b/server/services/musicVideo/productionReview.js
@@ -195,7 +195,7 @@ const clipLocked = (project, sceneId) => project.productionReview?.clipApprovals
/** What a locked clip freezes: the inputs it was rendered from, and the selected clip itself. */
const CLIP_LOCKED_FIELDS = ['prompt', 'framePrompt', 'visualIntent', 'camera', 'direction', 'startSec', 'endSec', 'lyricText',
- 'referenceImageId', 'endImageId', 'shotMode', 'performanceSpeaker', 'performanceStyle', 'visualLayer', 'loop', 'videoHistoryId', 'codeClip'];
+ 'referenceImageId', 'endImageId', 'shotMode', 'performanceSpeaker', 'performanceStyle', 'visualLayer', 'loop', 'videoHistoryId', 'codeClip', 'speedFps', 'offsetFrames'];
/**
* Refuse a change to a shot whose video is locked: `patch` is a scene patch
diff --git a/server/services/musicVideo/projects.js b/server/services/musicVideo/projects.js
index b7f3123692..abcfcd0550 100644
--- a/server/services/musicVideo/projects.js
+++ b/server/services/musicVideo/projects.js
@@ -312,6 +312,33 @@ export async function mergeProjectNextScene(id, sceneId, options = {}) {
});
}
+/**
+ * Move the cut before a shot: the shot starts at `startSec` and the shot
+ * before it ends there, so the two stay joined. Either shot's locked video
+ * refuses the move. Returns `{ project, scene, previous }`.
+ */
+export async function moveProjectSceneStart(id, sceneId, { startSec }) {
+ const [{ applySceneUpdate }, { assertClipUnlocked, keepDirectorStillLock }] = await Promise.all([import('./projectsLogic.js'), import('./productionReview.js')]);
+ return mutateProjectRecord(id, (current) => {
+ const scenes = current.scenes || [];
+ const index = scenes.findIndex((scene) => scene.sceneId === sceneId);
+ if (index < 0) throw new ServerError('Scene not found', { status: 404, code: 'NOT_FOUND' });
+ const scene = scenes[index];
+ const previous = index > 0 ? scenes[index - 1] : null;
+ if (typeof scene.endSec === 'number' && startSec >= scene.endSec) {
+ throw new ServerError('A shot has to start before it ends', { status: 400, code: 'VALIDATION_ERROR' });
+ }
+ if (previous && typeof previous.startSec === 'number' && startSec <= previous.startSec) {
+ throw new ServerError('The shot before this one would have no length left', { status: 400, code: 'VALIDATION_ERROR' });
+ }
+ assertClipUnlocked(current, sceneId, { startSec });
+ if (previous) assertClipUnlocked(current, previous.sceneId, { endSec: startSec });
+ let project = keepDirectorStillLock(current, applySceneUpdate(current, sceneId, { startSec }).project, sceneId);
+ if (previous) project = keepDirectorStillLock(current, applySceneUpdate(project, previous.sceneId, { endSec: startSec }).project, previous.sceneId);
+ return { project, scene: project.scenes[index], previous: previous ? project.scenes[index - 1] : null };
+ });
+}
+
/** Mark unused assets ★ ingredient or ✕ bad (null clears); returns the project's marks. */
export async function markMediaPoolAssets(id, keys, mark) {
const { markPoolAssets } = await import('./mediaPool.js');
diff --git a/server/services/musicVideo/projects.test.js b/server/services/musicVideo/projects.test.js
index 8d787f68c7..26cda2a779 100644
--- a/server/services/musicVideo/projects.test.js
+++ b/server/services/musicVideo/projects.test.js
@@ -461,3 +461,26 @@ describe('video lock transactions', () => {
expect((await projects.reviewSceneTake(p.id, scene.sceneId, appended[0].takeId, { status: 'rejected' })).videoHistoryId).toBeNull();
});
});
+
+describe('moving the cut before a shot', () => {
+ // Uniquely catches a start move that leaves a gap or overlap with the shot before, or retimes a locked video.
+ it('starts the shot at the new time and ends the shot before it there, unless either video is locked', async () => {
+ const { setShotLocks } = await import('./productionReview.js');
+ const p = await projects.createProject({ name: 'Move a cut' });
+ const { scenes: [a, b] } = await projects.addProjectScenes(p.id, [
+ { startSec: 0, endSec: 2, prompt: 'First' },
+ { startSec: 2, endSec: 4, prompt: 'Second' },
+ ]);
+ const later = await projects.moveProjectSceneStart(p.id, b.sceneId, { startSec: 2.5 });
+ expect([later.previous.endSec, later.scene.startSec, later.scene.endSec]).toEqual([2.5, 2.5, 4]);
+ const earlier = await projects.moveProjectSceneStart(p.id, b.sceneId, { startSec: 1.25 });
+ expect(earlier.project.scenes.map((scene) => [scene.startSec, scene.endSec])).toEqual([[0, 1.25], [1.25, 4]]);
+ await expect(projects.moveProjectSceneStart(p.id, b.sceneId, { startSec: 4 })).rejects.toMatchObject({ code: 'VALIDATION_ERROR' });
+ await expect(projects.moveProjectSceneStart(p.id, b.sceneId, { startSec: 0 })).rejects.toMatchObject({ code: 'VALIDATION_ERROR' });
+ await projects.updateScene(p.id, a.sceneId, { videoHistoryId: 'clip-a' });
+ await projects.mutateProjectRecord(p.id, (current) => ({ project: setShotLocks(setShotLocks(current, { sceneIds: [a.sceneId], locked: true, stage: 'still' }),
+ { sceneIds: [a.sceneId], locked: true, stage: 'video' }) }));
+ await expect(projects.moveProjectSceneStart(p.id, b.sceneId, { startSec: 1.5 })).rejects.toMatchObject({ code: 'MUSIC_VIDEO_SHOT_LOCKED' });
+ expect((await projects.getProject(p.id)).scenes.map((scene) => [scene.startSec, scene.endSec])).toEqual([[0, 1.25], [1.25, 4]]);
+ });
+});
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index 597621e3f2..2e95b4918b 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -520,6 +520,8 @@ function buildScene(input, { order }) {
...(input.codeOverlay ? { codeOverlay: true } : {}),
...(input.codeClip ? { codeClip: true } : {}),
...(input.frameFit ? { frameFit: input.frameFit } : {}),
+ ...(input.speedFps ? { speedFps: input.speedFps } : {}),
+ ...(input.offsetFrames ? { offsetFrames: input.offsetFrames } : {}),
...(input.transitionIn ? { transitionIn: input.transitionIn } : {}),
...(input.transitionOut ? { transitionOut: input.transitionOut } : {}),
...(input.glitches?.length ? { glitches: input.glitches } : {}),
@@ -607,6 +609,8 @@ export function replaceScenes(project, sceneInputs) {
...(prior.endImageId ? { endImageId: prior.endImageId } : {}),
...(prior.endFramePrompt ? { endFramePrompt: prior.endFramePrompt } : {}),
...(prior.frameFit ? { frameFit: prior.frameFit } : {}),
+ ...(prior.speedFps ? { speedFps: prior.speedFps } : {}),
+ ...(prior.offsetFrames ? { offsetFrames: prior.offsetFrames } : {}),
...(prior.transitionIn ? { transitionIn: prior.transitionIn } : {}),
...(prior.transitionOut ? { transitionOut: prior.transitionOut } : {}),
...(prior.glitches?.length ? { glitches: prior.glitches } : {}),
@@ -625,7 +629,7 @@ export function replaceScenes(project, sceneInputs) {
// takes, and whether it plays as a clip or its still with an image effect) without
// adding a shot or lyric the storyboard approval did not cover.
const SCENE_SHOT_EDIT_FIELDS = ['startSec', 'endSec', 'beatAligned', 'label', 'prompt', 'framePrompt', 'visualIntent',
- 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'transitionIn', 'transitionOut', 'glitches', 'cutEffect',
+ 'referenceImageId', 'videoHistoryId', 'endImageId', 'codeClip', 'frameFit', 'speedFps', 'offsetFrames', 'transitionIn', 'transitionOut', 'glitches', 'cutEffect',
'visualLayer', 'stillMove', 'imageEffect', 'imageEffectSec'];
/**
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 172ae7b46e..cbb70d8730 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -41,7 +41,7 @@ import { loadHistory, mutateVideoHistory } from '../videoGen/local.js';
import { getTrack } from '../tracks/index.js';
import { getProject, listProjects, updateProject, mutateProjectRecord } from './projects.js';
import { applyProjectPatch } from './projectsLogic.js';
-import { MUSIC_VIDEO_FRAME_FITS, isSelfDrawnLayer, sceneHasAuthoredSpan, sceneVisualLayer } from '../../lib/musicVideoLayers.js';
+import { MUSIC_VIDEO_FRAME_FITS, clipOffsetSec, clipSpeedRate, isSelfDrawnLayer, sceneHasAuthoredSpan, sceneVisualLayer } from '../../lib/musicVideoLayers.js';
import { musicVideoAspect, musicVideoRenderCanvas } from '../../lib/musicVideoAspect.js';
import { projectTypographyPlan } from './composition.js';
import { encodeCodeComposition, prepareCodeRender, writeCodeProofSheet } from './codeRender.js';
@@ -284,8 +284,22 @@ export async function resolveSceneClips(project, { layered = false, stillForMiss
// clip so the caller gets the clean MISSING_CLIPS 4xx, matching the duration guard.
if (!measured.width || measured.width <= 0 || !measured.height || measured.height <= 0) { missing.push(scene.videoHistoryId); continue; }
const edit = performanceEdit(scene);
- const inSec = edit ? Math.min(edit.inSec, duration) : 0;
- const outSec = edit ? Math.min(edit.outSec, duration) : duration;
+ let inSec = edit ? Math.min(edit.inSec, duration) : 0;
+ let outSec = edit ? Math.min(edit.outSec, duration) : duration;
+ let sourceSec = duration - inSec;
+ // A hand-set speed and offset (clipSpeedRate / clipOffsetSec) put the clip on the
+ // output timebase: sectionChain plays it at `rate` and holds its first frame for
+ // `leadSec` before the trim, so the in/out points here are output seconds.
+ const rate = clipSpeedRate(scene);
+ const start = inSec + clipOffsetSec(scene);
+ const timing = rate !== 1 || start !== inSec;
+ const leadSec = timing ? round3(Math.max(0, -start)) : 0;
+ if (timing) {
+ const sourceIn = Math.min(duration, Math.max(0, start));
+ outSec = round3(leadSec + Math.max(sourceIn, outSec) / rate);
+ inSec = round3(sourceIn / rate);
+ sourceSec = round3(leadSec + (duration - sourceIn) / rate);
+ }
clips.push({
sceneId: scene.sceneId,
videoPath,
@@ -296,8 +310,10 @@ export async function resolveSceneClips(project, { layered = false, stillForMiss
// The playable source length (from the in-point on) survives the snap
// below (which rewrites `duration` to the rendered span) so coverage can
// be judged against it.
- sourceSec: duration - inSec,
+ sourceSec,
loop: edit ? false : sceneLoops(scene),
+ ...(rate !== 1 ? { rate } : {}),
+ ...(leadSec > 0 ? { leadSec } : {}),
inSec,
outSec,
...footageFrameFit(scene),
@@ -459,7 +475,10 @@ function sectionChain(c, input, { canonW, canonH, fps, frames, frameGrid }) {
const holdSec = c.loop === false ? (c.outSec - c.inSec) - (c.sourceSec ?? c.duration) : 0;
const padSec = c.loop === false && frameGrid ? Math.max(0, holdSec) + 2 / fps : holdSec;
const hold = padSec > 0 ? `tpad=stop_mode=clone:stop_duration=${round3(padSec)},` : '';
- const timed = `fps=${fps},${hold}${trim},setpts=PTS-STARTPTS`;
+ // A hand-set speed retimes the clip and a negative offset holds its first frame, both before the trim.
+ const speed = c.rate && c.rate !== 1 ? `setpts=(PTS-STARTPTS)/${Number(c.rate.toFixed(6))},` : '';
+ const lead = c.leadSec > 0 ? `tpad=start_mode=clone:start_duration=${round3(c.leadSec)},` : '';
+ const timed = `${speed}${lead}fps=${fps},${hold}${trim},setpts=PTS-STARTPTS`;
const cover = `scale=${canonW}:${canonH}:force_original_aspect_ratio=increase,crop=${canonW}:${canonH},setsar=1`;
const fit = `scale=${canonW}:${canonH}:force_original_aspect_ratio=decrease,setsar=1`;
const centered = `overlay=(W-w)/2:(H-h)/2:shortest=1`;
diff --git a/server/services/musicVideo/render.test.js b/server/services/musicVideo/render.test.js
index d58daaa99c..e726226243 100644
--- a/server/services/musicVideo/render.test.js
+++ b/server/services/musicVideo/render.test.js
@@ -311,6 +311,26 @@ describe('resolveSceneClips', () => {
expect(clips[0].duration).toBe(2); // 48/24
});
+ // Uniquely catches a hand-set speed or offset that never reaches the render, or reaches it by a copied clip.
+ it('plays a hand-set speed and offset from the same clip file, on the output timebase', async () => {
+ loadHistory.mockResolvedValue([{ id: 'h1', filename: 'a.mp4', width: 768, height: 512, fps: 24, numFrames: 96 }]);
+ const [faster, held] = await resolveSceneClips({ scenes: [
+ { sceneId: 's1', order: 0, videoHistoryId: 'h1', speedFps: 24, offsetFrames: 12 },
+ { sceneId: 's2', order: 1, videoHistoryId: 'h1', offsetFrames: -6 },
+ ] });
+ // Double speed, starting half a second in: the 4s clip's last 3.5s play in 1.75s.
+ expect(faster).toMatchObject({ rate: 2, inSec: 0.25, outSec: 2, sourceSec: 1.75 });
+ expect(faster.leadSec).toBeUndefined();
+ // Six frames late: the first frame holds a quarter second, then the whole clip plays.
+ expect(held).toMatchObject({ inSec: 0, outSec: 4.25, sourceSec: 4.25, leadSec: 0.25 });
+ expect(held.rate).toBeUndefined();
+ expect(faster.videoPath).toBe(held.videoPath);
+ const fc = (clips) => { const { args } = buildMusicVideoFfmpegArgs(clips, '/music/track.wav', '/out.mp4', { audioDurationSec: 30 }); return args[args.indexOf('-filter_complex') + 1]; };
+ expect(fc([faster])).toContain('setpts=(PTS-STARTPTS)/2,fps=24');
+ expect(fc([held])).toContain('tpad=start_mode=clone:start_duration=0.25,fps=24');
+ expect(fc([{ ...faster, rate: undefined }])).not.toContain('PTS-STARTPTS)/');
+ });
+
it('cuts a code shot as a black ground in a composed render (#10297)', async () => {
loadHistory.mockResolvedValue([]);
const clips = await resolveSceneClips({ scenes: [{ sceneId: 's1', order: 0, visualLayer: 'code', startSec: 2, endSec: 5 }] }, { layered: true });