Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Music video: beat toggles play the incoming shot; a still's Move wins over its planned camera (#11618)

[._.]/1b215c7fb640
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index c35c54f337..1c347f044e 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -301,7 +301,7 @@ export default function SceneCard({
<>
<label htmlFor={fieldId('move')}>Move</label>
<select id={fieldId('move')} value={scene.stillMove || 'hold'} onChange={(e) => applyPatch({ stillMove: e.target.value })}
- disabled={!!scene.camera?.move} title={scene.camera?.move ? 'The planned camera move drives this still.' : undefined}
+ disabled={documentComposition && !!scene.camera?.move} title={documentComposition && scene.camera?.move ? 'The planned camera move drives this still.' : undefined}
className="bg-port-bg border border-port-border rounded px-1 py-1 min-h-[44px] sm:min-h-0 disabled:opacity-50">
{STILL_MOVE_LABELS.map(([value, text]) => <option key={value} value={value}>{text}</option>)}
</select>
diff --git a/client/src/components/musicVideo/SceneCard.test.jsx b/client/src/components/musicVideo/SceneCard.test.jsx
index 87fe7e82ce..9c4f8abc4d 100644
--- a/client/src/components/musicVideo/SceneCard.test.jsx
+++ b/client/src/components/musicVideo/SceneCard.test.jsx
@@ -158,14 +158,14 @@ describe('persisted render failure chip (#10154)', () => {
});
describe('planned camera move (#10589)', () => {
- function Harness({ initial, onSave }) {
+ function Harness({ initial, onSave, documentComposition = false }) {
const [scene, setScene] = useState(initial);
- return <SceneCard scene={scene} index={0} layered onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
+ return <SceneCard scene={scene} index={0} layered documentComposition={documentComposition} onEditLocal={(_id, patch) => setScene((current) => ({ ...current, ...patch }))} onSave={onSave} />;
}
it('picks a catalog move, its speed and beat, and clears it', () => {
const onSave = vi.fn();
- render(<Harness initial={{ sceneId: 's1', visualLayer: 'still', startSec: 0, endSec: 5, takes: [] }} onSave={onSave} />);
+ render(<Harness initial={{ sceneId: 's1', visualLayer: 'still', startSec: 0, endSec: 5, takes: [] }} onSave={onSave} documentComposition />);
const select = screen.getByLabelText('Camera');
expect(select.value).toBe('');
expect(screen.getByLabelText('Move').disabled).toBe(false);
@@ -179,6 +179,13 @@ describe('planned camera move (#10589)', () => {
expect(onSave).toHaveBeenLastCalledWith('s1', { camera: null });
});
+ // Uniquely catches a composed render's still Move locked behind a planned camera it no longer follows.
+ it('keeps a composed still\'s Move in the director\'s hands with a camera planned', () => {
+ render(<Harness initial={{ sceneId: 's1', visualLayer: 'still', stillMove: 'hold', startSec: 0, endSec: 5, takes: [], camera: { move: 'handheld' } }} onSave={vi.fn()} />);
+ expect(screen.getByLabelText('Move').disabled).toBe(false);
+ expect(screen.getByLabelText('Move').value).toBe('hold');
+ });
+
it('puts a footage shot\'s camera beside its motion prompt and shows the text it adds', () => {
render(<Harness initial={{ sceneId: 's1', visualLayer: 'footage', prompt: 'a hand turns the dial', startSec: 0, endSec: 5, takes: [] }} onSave={vi.fn()} />);
const camera = screen.getByLabelText('Camera');
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.jsx b/client/src/components/musicVideo/StoryboardAnimatic.jsx
index 40dc0b51ed..36674eaf14 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.jsx
@@ -190,15 +190,20 @@ function releaseVideo(video) {
* clip's start holds its first frame that long, and the speed runs the clip faster
* or slower, looping round the whole file as the render's looped input does.
* `early` follows the song's clock before the shot starts too (a beat toggle
- * into it), reading the footage before its in-point back to the clip's start.
+ * into it, whose window opens at `fromSec`), reading the footage before its
+ * in-point; with less footage than that it plays on from the clip's first
+ * frame, as the render does, rather than holding it.
*/
-export function clipTimeAt(scene, t, clipSec = null, { early = false } = {}) {
+export function clipTimeAt(scene, t, clipSec = null, { early = false, fromSec = null } = {}) {
const inSec = selectedPerformanceInstruction(scene)?.edit?.inSec ?? 0;
const local = early ? t - (scene.startSec || 0) : Math.max(0, t - (scene.startSec || 0));
const rate = clipSpeedRate(scene);
const start = inSec + clipOffsetSec(scene);
// A negative offset holds the first frame before the clip runs; otherwise it runs from `start`.
- const at = start >= 0 ? Math.max(0, start + local * rate) : Math.max(0, local + start) * rate;
+ const position = (l) => (start >= 0 ? Math.max(0, start + l * rate) : Math.max(0, l + start) * rate);
+ const at = early && local < 0 && fromSec != null && fromSec <= t
+ ? position(fromSec - (scene.startSec || 0)) + (t - fromSec) * rate
+ : position(local);
if (clipReversed(scene)) {
// Backwards from the file's end, as the render reads it once: past its start it holds the first frame shown last.
if (!(clipSec > 0)) return 0;
@@ -386,7 +391,7 @@ function ToggleClip({ scene, toggle, audioRef, time }) {
el.style.opacity = String(incomingOpacity(toggle, t));
if (el.tagName !== 'VIDEO' || el.readyState < 1) return;
const duration = Number.isFinite(el.duration) ? el.duration : null;
- const target = clipTimeAt(scene, Math.min(Math.max(t, toggle.windowStartSec), toggle.cutSec), duration, { early: true });
+ const target = clipTimeAt(scene, Math.min(Math.max(t, toggle.windowStartSec), toggle.cutSec), duration, { early: true, fromSec: toggle.windowStartSec });
const tolerance = audio?.paused ? CLIP_PAUSED_DRIFT_SEC : CLIP_DRIFT_SEC;
if (Math.abs((el.currentTime || 0) - target) > tolerance) el.currentTime = target;
const running = !!audio && !audio.paused && t >= toggle.windowStartSec && t < toggle.cutSec;
diff --git a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
index 01733eda55..766eaff246 100644
--- a/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
+++ b/client/src/components/musicVideo/StoryboardAnimatic.test.jsx
@@ -331,6 +331,18 @@ describe('StoryboardAnimatic clips', () => {
expect(clipTimeAt(early, 11.25, 6)).toBeCloseTo(1);
});
+ // Uniquely catches a beat toggle's incoming clip freezing on its first frame when it has
+ // less footage before its in-point than the toggle's window, where the render plays it on.
+ it('plays a toggle\'s incoming clip on from its first frame when it has too little footage before its in-point', () => {
+ const toggle = { early: true, fromSec: 9 };
+ expect(clipTimeAt({ startSec: 10 }, 9, 6, toggle)).toBe(0);
+ expect(clipTimeAt({ startSec: 10 }, 9.5, 6, toggle)).toBeCloseTo(0.5);
+ // A quarter second in: still from the first frame, not held for the first three quarters.
+ expect(clipTimeAt({ startSec: 10, offsetFrames: 6 }, 9.5, 6, toggle)).toBeCloseTo(0.5);
+ // With footage enough before its in-point it reads that footage.
+ expect(clipTimeAt({ startSec: 10, offsetFrames: 48 }, 9.5, 6, toggle)).toBeCloseTo(1.5);
+ });
+
// Uniquely catches a reversed shot previewing forwards, or looping where the render holds.
it('plays a reversed clip from its end, with its speed and offset, and holds once it runs out', () => {
const back = { startSec: 10, reverse: true };
diff --git a/server/routes/musicVideoTreatment.test.js b/server/routes/musicVideoTreatment.test.js
index bf61fe9836..3a649d5730 100644
--- a/server/routes/musicVideoTreatment.test.js
+++ b/server/routes/musicVideoTreatment.test.js
@@ -16,6 +16,7 @@ import { request } from '../lib/testHelper.js';
import { captureMusicVideoEvidence } from '../lib/musicVideoDependencies.js';
import { errorMiddleware } from '../lib/errorHandler.js';
import { makePathsProxy, lazyTempDataRoot, cleanupTempDataRoots } from '../lib/mockPathsDataRoot.js';
+import { stillMoveForCamera } from '../lib/cameraMovements.js';
const ROOT = () => lazyTempDataRoot('mv-treatment-route-test-');
vi.mock('../lib/paths.js', async (importOriginal) => makePathsProxy(await importOriginal(), { dataRoot: ROOT }));
@@ -421,7 +422,8 @@ describe('treatment → scene render fields (#8977 shot mode, #8985 visual layer
expect(res.status).toBe(200);
const after = await reload(project.id);
expect(after.scenes[sung]).toMatchObject({ visualLayer: 'card', cardText: scenes[sung].lyricText.split(' / ')[0] });
- expect(after.scenes[instrumental]).toMatchObject({ visualLayer: 'still', stillMove: 'push' });
+ // Its planned camera picks the still's move; with none it is pushed.
+ expect(after.scenes[instrumental]).toMatchObject({ visualLayer: 'still', stillMove: stillMoveForCamera(scenes[instrumental].camera) || 'push' });
expect(after.scenes[chosen]).toMatchObject({ visualLayer: 'still', stillMove: 'pan' });
expect(after.composition.mode).toBe('concat');
// The card draws its own line, so no lyric cue is stacked over it.
diff --git a/server/services/musicVideo/cutEffects.test.js b/server/services/musicVideo/cutEffects.test.js
index 5fbc073928..0bbb925043 100644
--- a/server/services/musicVideo/cutEffects.test.js
+++ b/server/services/musicVideo/cutEffects.test.js
@@ -122,11 +122,20 @@ describe.skipIf(!ffmpeg)('a beat toggle across a cut in the final render', () =>
showsClipFrame(means, 23, 17);
});
- it('holds the first frame where the clip has no footage before its in-point', () => {
+ it('plays on from the clip\'s first frame where it has less footage than the window before its in-point', () => {
const means = render({ sourceSec: 4 });
- [6, 7, 8].forEach((f) => showsClipFrame(means, f, 0));
+ [6, 7, 8].forEach((f, k) => showsClipFrame(means, f, k));
showsClipFrame(means, 12, 0);
showsClipFrame(means, 23, 11);
+ // A quarter second in, half the window's length: still moving from the first frame, never held.
+ const short = render({ inSec: 0.25, outSec: 1.25, sourceSec: 3.75, duration: 1 });
+ [6, 7, 8].forEach((f, k) => showsClipFrame(short, f, k));
+ showsClipFrame(short, 12, 3);
+ // A negative offset holds the shot's first frame, but its flips still move.
+ const late = render({ leadSec: 0.25, outSec: 1, sourceSec: 4.25 });
+ [6, 7, 8].forEach((f, k) => showsClipFrame(late, f, k));
+ showsClipFrame(late, 12, 0);
+ showsClipFrame(late, 15, 0);
});
it('plays the footage before the in-point at the shot\'s speed', () => {
diff --git a/server/services/musicVideo/projectsLogic.js b/server/services/musicVideo/projectsLogic.js
index e25b99a1b3..0b0089103b 100644
--- a/server/services/musicVideo/projectsLogic.js
+++ b/server/services/musicVideo/projectsLogic.js
@@ -38,6 +38,7 @@ import { ensureSceneTakes, keepCodeClipWithItsClip, TAKE_SLOT } from './takes.js
import { normalizeComposition, invalidateCompositionTiming, withStoredCompositionDocument } from './composition.js';
import { normalizeSoundBed } from './soundBed.js';
import { normalizeFilmLook } from '../../lib/filmLook.js';
+import { stillMoveForCamera } from '../../lib/cameraMovements.js';
import { normalizeTitleCard } from '../../lib/musicVideoTitleCard.js';
import { normalizeStatsOutro } from '../../lib/musicVideoStatsOutro.js';
import { remapTreatmentForClone, scenesFingerprint } from './treatment.js';
@@ -652,7 +653,12 @@ export function applySceneUpdate(project, sceneId, patch) {
const scenes = project.scenes || [];
const idx = scenes.findIndex((s) => s.sceneId === sceneId);
if (idx < 0) throw new ServerError('Scene not found', { status: 404, code: 'NOT_FOUND' });
- const merged = 'codeClip' in data ? { ...scenes[idx], ...data } : keepCodeClipWithItsClip(scenes[idx], { ...scenes[idx], ...data });
+ const kept = 'codeClip' in data ? { ...scenes[idx], ...data } : keepCodeClipWithItsClip(scenes[idx], { ...scenes[idx], ...data });
+ // A shot turning into a still takes the nearest move to its planned camera (#10589) unless
+ // it already has one; from then on its Move picker owns how the still moves.
+ const becomesStill = data.visualLayer === 'still' && scenes[idx].visualLayer !== 'still' && !('stillMove' in data) && (kept.stillMove ?? 'hold') === 'hold';
+ const seeded = becomesStill ? stillMoveForCamera(kept.camera) : null;
+ const merged = seeded ? { ...kept, stillMove: seeded } : kept;
// Setting a slot directly (an older client, or a hand edit) still records the
// asset as a take so the candidate list stays a superset of the selection.
const touchesSlot = Object.values(TAKE_SLOT).some((field) => typeof data[field] === 'string');
@@ -835,6 +841,7 @@ export function splitScene(project, sceneId, { backend = null, atSec = null } =
sectionIndex: scene.sectionIndex ?? null,
visualIntent: scene.visualIntent ?? null,
visualLayer: scene.visualLayer ?? 'footage',
+ ...(scene.stillMove ? { stillMove: scene.stillMove } : {}),
shotMode: scene.shotMode ?? 'cutaway',
performanceSpeaker: scene.performanceSpeaker ?? null,
performanceStyle: scene.performanceStyle ?? null,
diff --git a/server/services/musicVideo/projectsLogic.test.js b/server/services/musicVideo/projectsLogic.test.js
index e294818d17..96bed2300d 100644
--- a/server/services/musicVideo/projectsLogic.test.js
+++ b/server/services/musicVideo/projectsLogic.test.js
@@ -12,6 +12,7 @@ import {
reorderScenes,
mergeNextScene,
mergePreviousScene,
+ splitScene,
mirrorStatus,
sanitizeProjectForSync,
mergeProjectRecord,
@@ -575,6 +576,23 @@ describe('scene board operations', () => {
expect(() => applySceneUpdate(project, scene.sceneId, { cardColor: 'red' })).toThrow();
});
+ // Uniquely catches a planned camera move not seeding a new still's Move, or overriding one the shot sets.
+ it('seeds a shot\'s still move from its planned camera when it becomes a still, and keeps a move it is given', () => {
+ const { project, scene } = addScene(baseProject(), { prompt: 'a', camera: { move: 'truck-left' } });
+ expect(scene.stillMove).toBe('hold');
+ expect(applySceneUpdate(project, scene.sceneId, { visualLayer: 'still' }).updated.stillMove).toBe('pan');
+ expect(applySceneUpdate(project, scene.sceneId, { visualLayer: 'still', stillMove: 'hold' }).updated.stillMove).toBe('hold');
+ const still = applySceneUpdate(project, scene.sceneId, { visualLayer: 'still', stillMove: 'push' }).project;
+ expect(applySceneUpdate(still, scene.sceneId, { visualLayer: 'still' }).updated.stillMove).toBe('push');
+ });
+
+ // Uniquely catches a split still's later pieces losing the Move the shot set.
+ it('keeps a still\'s Move on every piece of a split', () => {
+ const { project, scene } = addScene(baseProject(), { prompt: 'a', visualLayer: 'still', stillMove: 'push', camera: { move: 'handheld' }, startSec: 0, endSec: 8 });
+ const split = splitScene(project, scene.sceneId, { atSec: 4 });
+ expect(split.project.scenes.map((s) => s.stillMove)).toEqual(['push', 'push']);
+ });
+
it('persists a shot\'s lyric-type zone and role and refuses unknown ones (#10583)', () => {
const { project, scene } = addScene(baseProject(), { prompt: 'a' });
const { updated } = applySceneUpdate(project, scene.sceneId, { textZone: 'none', lyricRole: 'hook' });
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index e5b00dae8e..d593029413 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -266,13 +266,13 @@ export async function resolveSceneClips(project, { layered = false, stillForMiss
}
const imagePath = scene.referenceImageId ? safeUnder(PATHS.images, scene.referenceImageId) : null;
if (!imagePath || !existsSync(imagePath)) { missingStills.push(scene.sceneId); continue; }
- // A planned camera move (#10589) picks the nearest composed still move.
+ // The shot's Move plays as set; a planned camera move (#10589) only seeds it when the shot is made.
// An image effect (a camera flash that develops, musicVideoImageEffects.js) plays over the still.
// An end-frame effect also reads the shot's end frame; without it on disk the still plays alone.
const effect = sceneImageEffect(scene, { layered });
const endImagePath = effect && isEndFrameImageEffect(effect.effect) ? safeUnder(PATHS.images, scene.endImageId) : null;
const playable = effect && (!isEndFrameImageEffect(effect.effect) || (endImagePath && existsSync(endImagePath)));
- clips.push({ ...section, imagePath, move: stillMoveForCamera(scene.camera) || scene.stillMove || 'hold',
+ clips.push({ ...section, imagePath, move: scene.stillMove || stillMoveForCamera(scene.camera) || 'hold',
...(playable ? { effect, ...(endImagePath ? { endImagePath } : {}) } : {}) });
continue;
}
@@ -649,15 +649,17 @@ export function buildMusicVideoFfmpegArgs(clips, audioPath, outputPath, { audioD
const prerolls = cutEffects.filter((t) => t.incoming).map((t) => {
const { c } = plan[t.index + 1];
// The footage just before the incoming shot's in-point, on the output timebase
- // its speed and offset already set; any part before the clip starts holds its first frame.
+ // its speed and offset already set. With less footage than the window before
+ // the in-point it plays on from the clip's first frame instead: the flips
+ // always show moving footage, never a held frame.
const windowSec = t.windowFrames / fps;
- const shortSec = Math.max(0, windowSec - c.inSec);
+ const { leadSec = 0, ...shot } = c;
+ const inSec = Math.max(0, c.inSec - windowSec);
const clip = c.layer ? { ...c, outSec: c.inSec + windowSec } : {
- ...c,
- inSec: Math.max(0, c.inSec - windowSec),
- outSec: c.inSec + shortSec,
- sourceSec: (c.sourceSec ?? c.duration) + windowSec,
- ...(shortSec > 0 ? { leadSec: round3((c.leadSec || 0) + shortSec) } : {}),
+ ...shot,
+ inSec,
+ outSec: inSec + windowSec,
+ sourceSec: (c.sourceSec ?? c.duration) - leadSec + (c.inSec - inSec),
};
return { ...t, clip };
});
diff --git a/server/services/musicVideo/render.test.js b/server/services/musicVideo/render.test.js
index 5aec1ffe69..f0ffff0f04 100644
--- a/server/services/musicVideo/render.test.js
+++ b/server/services/musicVideo/render.test.js
@@ -313,6 +313,18 @@ describe('resolveSceneClips', () => {
expect(clips[0].duration).toBe(2); // 48/24
});
+ // Uniquely catches a planned camera move overriding the Move the shot shows (the render panning a Hold still).
+ it('moves a still as its Move is set, whatever camera was planned for it', async () => {
+ loadHistory.mockResolvedValue([]);
+ const still = { order: 0, visualLayer: 'still', referenceImageId: 'frame.png', startSec: 0, endSec: 2, camera: { move: 'handheld', onBeat: true } };
+ const [held, panned, seeded] = await resolveSceneClips({ scenes: [
+ { ...still, sceneId: 's1', stillMove: 'hold' },
+ { ...still, sceneId: 's2', order: 1, startSec: 2, endSec: 4, stillMove: 'pan' },
+ { ...still, sceneId: 's3', order: 2, startSec: 4, endSec: 6 },
+ ] }, { layered: true });
+ expect([held.move, panned.move, seeded.move]).toEqual(['hold', 'pan', 'pan']);
+ });
+
// Uniquely catches a hand-set speed or offset that never reaches the render, or reaches it by a copied clip.
it('plays a hand-set speed and offset from the same clip file, on the output timebase', async () => {
loadHistory.mockResolvedValue([{ id: 'h1', filename: 'a.mp4', width: 768, height: 512, fps: 24, numFrames: 96 }]);
diff --git a/server/services/musicVideo/treatment.js b/server/services/musicVideo/treatment.js
index 85ff37d623..bd94e75e3c 100644
--- a/server/services/musicVideo/treatment.js
+++ b/server/services/musicVideo/treatment.js
@@ -59,6 +59,7 @@ import { performanceCapability } from '../../lib/musicVideoShotTiming.js';
import { musicVideoAllowsMedia } from '../../lib/musicVideoMediaPolicy.js';
import { normalizeComposition } from './composition.js';
import { hookKey, hookLines, cueWordOnsets } from './hookTypography.js';
+import { stillMoveForCamera } from '../../lib/cameraMovements.js';
const TREATMENT_VERSION = 1;
@@ -639,7 +640,8 @@ function renderFieldPatch(scene, direction, { lipSyncAvailable, imagesAllowed =
: '';
if (line) return { visualLayer: 'card', ...(isNonBlankStr(scene.cardText) ? {} : { cardText: line }) };
if (!imagesAllowed) return { visualLayer: 'code' };
- return { visualLayer: 'still', ...((scene.stillMove ?? 'hold') === 'hold' ? { stillMove: 'push' } : {}) };
+ // A held still takes the nearest move to its planned camera (#10589), else a push.
+ return { visualLayer: 'still', ...((scene.stillMove ?? 'hold') === 'hold' ? { stillMove: stillMoveForCamera(scene.camera) || 'push' } : {}) };
}
/** Code-first Apply never reinterprets layers, but a lip-sync performance footage shot must carry the Performance shot mode preflight demands. */