Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Keep a card render's shot actions, Grok timing and audio-reactive guard, shown in its preview

Claudea141963433a4
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsx
index 9abc589414..35debef506 100644
--- a/client/src/components/musicVideo/SceneCard.jsx
+++ b/client/src/components/musicVideo/SceneCard.jsx
@@ -105,7 +105,7 @@ export default function SceneCard({
scene, index, shotNumber = null, isLast, generatingFrame, generatingVideo, frameProgress = null, videoProgress = null,
settingsSaving, videoBlockedReason, videoBlockedReasonFor = null, canContinueShot,
onMove, onDelete, onEditLocal, onSave,
- onGenerateFrame, onGenerateVideo, onContinueVideo, onGenerateEndFrame, generatingEndFrame = false, onContinueFromPrevious, previousHasClip = false,
+ project = null, onGenerateFrame, onGenerateVideo, onContinueVideo, onGenerateEndFrame, generatingEndFrame = false, onContinueFromPrevious, previousHasClip = false,
onOpenPreview, onSelectTake, onReviewTake, onImportTake, onImportClipTake, onPlacePoolItem, onTrimClip, projectId = null, aspect = '16:9', videoModels = [], defaultVideoModel = '', shotNumbers = undefined, takeBusy = false, layered = false, documentComposition = false,
lipSyncBackend = '', songDurationSec = null, onSplit, onMergeNext, nextEndSec = null, falVideoSettings = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,
expanded, onToggleExpand, footageOptional = false, failedScenes = null, songReview = null,
@@ -188,7 +188,7 @@ export default function SceneCard({
const endFramePlan = sceneEndFramePlanFor(scene, renderSettings || {}, { models: videoModels, defaultModel: defaultVideoModel });
// Exactly what Render sends: the prompt as typed and the inputs the card toggles.
const [renderInputs, setRenderInputs] = useState({});
- const cardRequest = shotCardRenderRequest({ scene, settings: renderSettings || {}, performance, endFramePlan,
+ const cardRequest = shotCardRenderRequest({ project, scene, settings: renderSettings || {}, performance, endFramePlan,
models: videoModels, defaultModel: defaultVideoModel, inputs: renderInputs });
const videoRefusal = videoGenerationRefusal({
settingsSaving, speakerUnsaved: performance && (speakerDirty || speakerSaving),
diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsx
index 8721d1acd6..d430fce969 100644
--- a/client/src/components/musicVideo/SceneRenderInputs.jsx
+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx
@@ -1,5 +1,7 @@
import { getFalVideoModel } from '../../lib/falVideoModels.js';
-import { falSceneTake } from '../../lib/musicVideoShotTiming.js';
+import { AUDIO_REACTIVE_PERFORMANCE_GUARD, silentMotionPrompt } from '../../lib/musicVideoMotion.js';
+import { approximateMotionCues, falSceneTake } from '../../lib/musicVideoShotTiming.js';
+import { shotActionPrompt } from '../../../../server/lib/musicVideoActionContract.js';
import { shotCameraPrompt } from '../../../../server/lib/cameraMovements.js';
import { localModelTakesSongAudio } from '../../../../server/lib/musicVideoLocalModels.js';
import { formatCount } from '../../utils/formatters.js';
@@ -11,10 +13,12 @@ const clock = (sec) => {
};
/**
- * Exactly what the shot card's Render sends: the prompt as typed (plus the
- * camera move the card sets), and which inputs go with it. Nothing the card
- * does not show is added: the server receives `promptAsTyped` and leaves the
- * prompt alone.
+ * Exactly what the shot card's Render sends: the prompt as typed, then what
+ * the card itself sets (its camera move, its applied shot actions, Grok's
+ * approximate phrase timing, the audio-reactive guard), and which inputs go
+ * with it. The project's style and bible are not added, and the server
+ * receives `promptAsTyped` and leaves the prompt alone (a performance shot's
+ * action times shift to its padded song slice).
*
* `settings` is the renderer this render uses (the project's, or the card's
* Render on pick merged over it); `inputs` holds the card's toggles,
@@ -23,9 +27,8 @@ const clock = (sec) => {
* input is `{ state: 'on' | 'off' | 'required' | 'unavailable', note }`.
*/
export function shotCardRenderRequest({
- scene, settings = {}, performance = false, endFramePlan = null, models = [], defaultModel = '', inputs = {},
+ project = null, scene, settings = {}, performance = false, endFramePlan = null, models = [], defaultModel = '', inputs = {},
}) {
- const prompt = [(scene?.prompt?.trim() || scene?.framePrompt?.trim() || ''), shotCameraPrompt(scene?.camera)].filter(Boolean).join('\n');
const falModel = settings.backend === 'fal' ? getFalVideoModel(falSceneTake({ scene, videoSettings: settings }).modelId) : null;
const localModel = settings.backend === 'local'
? models.find((model) => model.id === (settings.modelId || defaultModel)) || null : null;
@@ -37,6 +40,17 @@ export function shotCardRenderRequest({
const on = inputs.songAudio ?? settings.generationMode === 'suppliedAudio';
songAudio = { state: on ? 'on' : 'off', note: `Song ${span}` };
} else songAudio = { state: 'unavailable', note: 'This model hears no audio' };
+ // An audio-reactive LoRA render hears the song through its own weights.
+ const audioReactive = settings.backend === 'local' && settings.generationMode === 'audioReactive' && !performance;
+ if (audioReactive) songAudio = { state: 'required', note: `Song ${span} drives the motion` };
+
+ // A take that hears nothing gets its shot actions without the lyric and beat lines it cannot hear.
+ const hears = songAudio.state === 'on' || songAudio.state === 'required';
+ const actions = project && scene?.direction?.actionContract != null ? shotActionPrompt(project, scene) : '';
+ const motionCues = settings.backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : '';
+ const base = [(scene?.prompt?.trim() || scene?.framePrompt?.trim() || ''), shotCameraPrompt(scene?.camera),
+ hears ? actions : silentMotionPrompt(actions)].filter(Boolean).join('\n');
+ const prompt = audioReactive ? `${base}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}` : [base, motionCues].filter(Boolean).join('. ');
let endFrame = null;
if (scene?.endImageId) {
diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsx
index 594a584069..e5752486d2 100644
--- a/client/src/components/musicVideo/stages/BoardStage.jsx
+++ b/client/src/components/musicVideo/stages/BoardStage.jsx
@@ -182,6 +182,7 @@ export default function BoardStage({ board }) {
<SceneCard
key={scene.sceneId}
scene={scene}
+ project={project}
expanded={activeSceneId === scene.sceneId}
onToggleExpand={toggleInspector}
performanceReview={performanceReviews.get(`${scene.sceneId}:${scene.videoHistoryId}`)}
diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.js
index 5fbe2fb260..e64b4ff81e 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.js
+++ b/client/src/hooks/useMusicVideoSceneMedia.js
@@ -14,15 +14,11 @@ import useSceneRenderLifecycle from './useSceneRenderLifecycle.js';
import { isLtx2FamilyRuntime } from '../lib/runnerFamilies';
import { isLayeredComposition, isSelfDrawnLayer, sceneVisualLayer } from '../lib/musicVideoLayers.js';
import { musicVideoFrameGenSize } from '../lib/musicVideoAspect.js';
-import { MOTION_CONTINUITY_CLAUSE } from '../lib/musicVideoMotion.js';
+import { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE } from '../lib/musicVideoMotion.js';
import {
approximateMotionCues, falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason,
} from '../lib/musicVideoShotTiming.js';
-// Audio-reactive generation conditions motion on the song itself, so the prompt
-// has to rule out anything that reads as a performance of it.
-const AUDIO_REACTIVE_PERFORMANCE_GUARD = 'The music drives only environmental motion, lighting, particles, reflections, fabric, and subtle camera accents. No singing, lip-sync, speaking, mouth movement, dancing, instruments, performers, or musical performance.';
-
// The image backends accept at most four reference images for most models
// (server imageGen/prepareParams.js), mirrored by the server's
// MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES.
diff --git a/client/src/lib/README.md b/client/src/lib/README.md
index 1ffd03d5a9..7f16026e87 100644
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@ -61,7 +61,7 @@ grep -i "what you want to do" client/src/lib/README.md
| `beatGrid.js` | Music-video beat-quantized timeline arranger (#1854). `buildBeatGridPoints(audioAnalysis)` merges beats/downbeats/section edges into one ranked snap-point list; `snapTimeToGrid(timeSec, gridPoints, toleranceSec)` finds the nearest point within tolerance; `computeSceneSpans(scenes, durationSec)` lays out scenes without a persisted `startSec`/`endSec` contiguously as a display-only fallback; `computeDragSpan({kind, startSpan, deltaSec, gridPoints, ...})` resolves a timeline drag gesture (`'move'` or `'right'` — no `'left'`, since the render can only trim from a clip's own frame 0) into a new span; `shouldMarkBeatAligned({kind, snapped, wasPersisted})` gates the `beatAligned` flag so a reposition-only drag can't silently promote an unpersisted scene's placeholder fallback duration into a "saved exactly" render duration; `autoArrangeScenes(scenes, audioAnalysis)` proposes a full `{ sceneId, startSec, endSec, beatAligned }[]` arrangement by distributing scenes across song sections weighted by each section's `energy` (#1915). Used by `components/musicVideo/BeatTimeline.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoAspect.js` | Re-export of `server/lib/musicVideoAspect.js`: `musicVideoAspect`, `musicVideoFrameGenSize`, `matchesMusicVideoAspect`. Used by `hooks/useMusicVideoSceneMedia.js` so board frames match the project aspect. |
| `musicVideoFiling.js` | Which music video an Image Gen / Video Gen render is for: `MUSIC_VIDEO_FILING_PARAM` (the `musicVideo` URL param), `musicVideoFilingFields(searchParams)` (the `musicVideoProjectId` request field), `musicVideoOf(item)` (a record's video). Used by `components/media/MusicVideoFilingSelect.jsx`, `hooks/useMediaPreviewActions.js` and the two generate pages. |
-| `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`) for board motion prompts. |
+| `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`, `AUDIO_REACTIVE_PERFORMANCE_GUARD`, `silentMotionPrompt`) for board motion prompts. |
| `musicVideoBatchPlan.js` | Pure helpers for the Music Video missing-frame / missing-clip batches (#10153): `videoBatchPreview` (count + fal estimate summed from `falSceneTake`, unpriced count), `scenesWaitingForFrame`, `videosButtonLabel`, `batchSummary`, `batchActive`, `batchSceneName`. Used by `components/musicVideo/ProjectActionGroups.jsx`. |
| `musicVideoLayers.js` | Music-video per-scene visual layers (#8985), re-exported from `server/lib/musicVideoLayers.js`: `MUSIC_VIDEO_VISUAL_LAYERS`, `MUSIC_VIDEO_STILL_MOVES`, `MUSIC_VIDEO_TEXT_ZONES` / `MUSIC_VIDEO_LYRIC_ROLES` (#10583 lyric type per shot), `sceneVisualLayer(scene, { layered })`, `sceneHasAuthoredSpan`, `sceneRenderReady(scene, { layered })`. Used by `components/musicVideo/SceneCard.jsx`, `ProjectActionGroups.jsx` and `hooks/useMusicVideoSceneMedia.js`. |
| `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. |
diff --git a/client/src/lib/musicVideoMotion.js b/client/src/lib/musicVideoMotion.js
index c0b49fb00e..2c4c7cfdf2 100644
--- a/client/src/lib/musicVideoMotion.js
+++ b/client/src/lib/musicVideoMotion.js
@@ -1,2 +1,2 @@
/** Re-export of the server's motion-continuity clause so board clips pin the frame's lighting like production clips. */
-export { MOTION_CONTINUITY_CLAUSE } from '../../../server/lib/musicVideoMotion.js';
+export { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE, silentMotionPrompt } from '../../../server/lib/musicVideoMotion.js';
diff --git a/server/lib/README.md b/server/lib/README.md
index bac3116ad4..1709d19e3a 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -848,7 +848,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `loopInterval.js` | Loop interval grammar (`MIN_INTERVAL_MS`, `parseInterval`, `isValidLoopInterval`) shared by `services/loops.js` and `createLoopSchema` so the route rejects what the service would. |
| `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. |
-| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. |
+| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. |
| `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. |
| `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |
diff --git a/server/lib/musicVideoMotion.js b/server/lib/musicVideoMotion.js
index 785dbdc561..9b43e04fd4 100644
--- a/server/lib/musicVideoMotion.js
+++ b/server/lib/musicVideoMotion.js
@@ -8,6 +8,9 @@
*/
export const MOTION_CONTINUITY_CLAUSE = "hold the reference frame's lighting, time of day and palette for the whole shot";
+/** Appended to an audio-reactive (LoRA) shot's prompt so the music moves the scene, never a performer. */
+export const AUDIO_REACTIVE_PERFORMANCE_GUARD = 'The music drives only environmental motion, lighting, particles, reflections, fabric, and subtle camera accents. No singing, lip-sync, speaking, mouth movement, dancing, instruments, performers, or musical performance.';
+
// A song reference a silent i2v model cannot hear: a quoted lyric cue ("on
// 'Sleep'"), a musical event ("on the downbeat", "in time with the snare") or
// a song timestamp ("at 1:42").
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index b6bf03deb2..a6117c326d 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -313,10 +313,12 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
// A take that hears nothing (no lip-sync or supplied song audio) gets plain physical
// motion: the model would act out song cues and look words instead of reading them.
const hearsAudio = !!(performance || prepared.audio || params.audioConditioning || params.audioFilePath);
- if (!body.musicVideo.promptAsTyped) {
+ // A shot card's prompt is sent as shown, its shot actions included; only a performance
+ // take re-times them to its padded song slice.
+ if (!body.musicVideo.promptAsTyped || performance) {
params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec: performance?.shotInstruction?.edit?.inSec || 0 });
- if (!hearsAudio) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);
}
+ if (!hearsAudio && !body.musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);
const interval = performance?.shotInstruction?.songInterval || prepared.audio?.songInterval;
if (params.audioConditioning && Math.round(scene?.startSec * 48000) !== params.audioConditioning.startSample) {
throw new ServerError('Scene timing changed while source audio was prepared', { status: 409, code: 'MUSIC_VIDEO_SHOT_TIMING_CHANGED' });