Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Keep a card render's shot actions, Grok timing and audio-reactive guard, shown in its preview
Claudea141963433a4
diff --git a/client/src/components/musicVideo/SceneCard.jsx b/client/src/components/musicVideo/SceneCard.jsxindex 9abc589414..35debef506 100644--- a/client/src/components/musicVideo/SceneCard.jsx+++ b/client/src/components/musicVideo/SceneCard.jsx@@ -105,7 +105,7 @@ export default function SceneCard({scene, index, shotNumber = null, isLast, generatingFrame, generatingVideo, frameProgress = null, videoProgress = null,settingsSaving, videoBlockedReason, videoBlockedReasonFor = null, canContinueShot,onMove, onDelete, onEditLocal, onSave,- onGenerateFrame, onGenerateVideo, onContinueVideo, onGenerateEndFrame, generatingEndFrame = false, onContinueFromPrevious, previousHasClip = false,+ project = null, onGenerateFrame, onGenerateVideo, onContinueVideo, onGenerateEndFrame, generatingEndFrame = false, onContinueFromPrevious, previousHasClip = false,onOpenPreview, onSelectTake, onReviewTake, onImportTake, onImportClipTake, onPlacePoolItem, onTrimClip, projectId = null, aspect = '16:9', videoModels = [], defaultVideoModel = '', shotNumbers = undefined, takeBusy = false, layered = false, documentComposition = false,lipSyncBackend = '', songDurationSec = null, onSplit, onMergeNext, nextEndSec = null, falVideoSettings = null, onSeek, performanceReview = null, onRepairPerformance, repairBusy = false,expanded, onToggleExpand, footageOptional = false, failedScenes = null, songReview = null,@@ -188,7 +188,7 @@ export default function SceneCard({const endFramePlan = sceneEndFramePlanFor(scene, renderSettings || {}, { models: videoModels, defaultModel: defaultVideoModel });// Exactly what Render sends: the prompt as typed and the inputs the card toggles.const [renderInputs, setRenderInputs] = useState({});- const cardRequest = shotCardRenderRequest({ scene, settings: renderSettings || {}, performance, endFramePlan,+ const cardRequest = shotCardRenderRequest({ project, scene, settings: renderSettings || {}, performance, endFramePlan,models: videoModels, defaultModel: defaultVideoModel, inputs: renderInputs });const videoRefusal = videoGenerationRefusal({settingsSaving, speakerUnsaved: performance && (speakerDirty || speakerSaving),diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsxindex 8721d1acd6..d430fce969 100644--- a/client/src/components/musicVideo/SceneRenderInputs.jsx+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx@@ -1,5 +1,7 @@import { getFalVideoModel } from '../../lib/falVideoModels.js';-import { falSceneTake } from '../../lib/musicVideoShotTiming.js';+import { AUDIO_REACTIVE_PERFORMANCE_GUARD, silentMotionPrompt } from '../../lib/musicVideoMotion.js';+import { approximateMotionCues, falSceneTake } from '../../lib/musicVideoShotTiming.js';+import { shotActionPrompt } from '../../../../server/lib/musicVideoActionContract.js';import { shotCameraPrompt } from '../../../../server/lib/cameraMovements.js';import { localModelTakesSongAudio } from '../../../../server/lib/musicVideoLocalModels.js';import { formatCount } from '../../utils/formatters.js';@@ -11,10 +13,12 @@ const clock = (sec) => {};/**- * Exactly what the shot card's Render sends: the prompt as typed (plus the- * camera move the card sets), and which inputs go with it. Nothing the card- * does not show is added: the server receives `promptAsTyped` and leaves the- * prompt alone.+ * Exactly what the shot card's Render sends: the prompt as typed, then what+ * the card itself sets (its camera move, its applied shot actions, Grok's+ * approximate phrase timing, the audio-reactive guard), and which inputs go+ * with it. The project's style and bible are not added, and the server+ * receives `promptAsTyped` and leaves the prompt alone (a performance shot's+ * action times shift to its padded song slice).** `settings` is the renderer this render uses (the project's, or the card's* Render on pick merged over it); `inputs` holds the card's toggles,@@ -23,9 +27,8 @@ const clock = (sec) => {* input is `{ state: 'on' | 'off' | 'required' | 'unavailable', note }`.*/export function shotCardRenderRequest({- scene, settings = {}, performance = false, endFramePlan = null, models = [], defaultModel = '', inputs = {},+ project = null, scene, settings = {}, performance = false, endFramePlan = null, models = [], defaultModel = '', inputs = {},}) {- const prompt = [(scene?.prompt?.trim() || scene?.framePrompt?.trim() || ''), shotCameraPrompt(scene?.camera)].filter(Boolean).join('\n');const falModel = settings.backend === 'fal' ? getFalVideoModel(falSceneTake({ scene, videoSettings: settings }).modelId) : null;const localModel = settings.backend === 'local'? models.find((model) => model.id === (settings.modelId || defaultModel)) || null : null;@@ -37,6 +40,17 @@ export function shotCardRenderRequest({const on = inputs.songAudio ?? settings.generationMode === 'suppliedAudio';songAudio = { state: on ? 'on' : 'off', note: `Song ${span}` };} else songAudio = { state: 'unavailable', note: 'This model hears no audio' };+ // An audio-reactive LoRA render hears the song through its own weights.+ const audioReactive = settings.backend === 'local' && settings.generationMode === 'audioReactive' && !performance;+ if (audioReactive) songAudio = { state: 'required', note: `Song ${span} drives the motion` };++ // A take that hears nothing gets its shot actions without the lyric and beat lines it cannot hear.+ const hears = songAudio.state === 'on' || songAudio.state === 'required';+ const actions = project && scene?.direction?.actionContract != null ? shotActionPrompt(project, scene) : '';+ const motionCues = settings.backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : '';+ const base = [(scene?.prompt?.trim() || scene?.framePrompt?.trim() || ''), shotCameraPrompt(scene?.camera),+ hears ? actions : silentMotionPrompt(actions)].filter(Boolean).join('\n');+ const prompt = audioReactive ? `${base}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}` : [base, motionCues].filter(Boolean).join('. ');let endFrame = null;if (scene?.endImageId) {diff --git a/client/src/components/musicVideo/stages/BoardStage.jsx b/client/src/components/musicVideo/stages/BoardStage.jsxindex 594a584069..e5752486d2 100644--- a/client/src/components/musicVideo/stages/BoardStage.jsx+++ b/client/src/components/musicVideo/stages/BoardStage.jsx@@ -182,6 +182,7 @@ export default function BoardStage({ board }) {<SceneCardkey={scene.sceneId}scene={scene}+ project={project}expanded={activeSceneId === scene.sceneId}onToggleExpand={toggleInspector}performanceReview={performanceReviews.get(`${scene.sceneId}:${scene.videoHistoryId}`)}diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.jsindex 5fbe2fb260..e64b4ff81e 100644--- a/client/src/hooks/useMusicVideoSceneMedia.js+++ b/client/src/hooks/useMusicVideoSceneMedia.js@@ -14,15 +14,11 @@ import useSceneRenderLifecycle from './useSceneRenderLifecycle.js';import { isLtx2FamilyRuntime } from '../lib/runnerFamilies';import { isLayeredComposition, isSelfDrawnLayer, sceneVisualLayer } from '../lib/musicVideoLayers.js';import { musicVideoFrameGenSize } from '../lib/musicVideoAspect.js';-import { MOTION_CONTINUITY_CLAUSE } from '../lib/musicVideoMotion.js';+import { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE } from '../lib/musicVideoMotion.js';import {approximateMotionCues, falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason,} from '../lib/musicVideoShotTiming.js';-// Audio-reactive generation conditions motion on the song itself, so the prompt-// has to rule out anything that reads as a performance of it.-const AUDIO_REACTIVE_PERFORMANCE_GUARD = 'The music drives only environmental motion, lighting, particles, reflections, fabric, and subtle camera accents. No singing, lip-sync, speaking, mouth movement, dancing, instruments, performers, or musical performance.';-// The image backends accept at most four reference images for most models// (server imageGen/prepareParams.js), mirrored by the server's// MUSIC_VIDEO_MAX_CONDITIONING_REFERENCES.diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex 1ffd03d5a9..7f16026e87 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -61,7 +61,7 @@ grep -i "what you want to do" client/src/lib/README.md| `beatGrid.js` | Music-video beat-quantized timeline arranger (#1854). `buildBeatGridPoints(audioAnalysis)` merges beats/downbeats/section edges into one ranked snap-point list; `snapTimeToGrid(timeSec, gridPoints, toleranceSec)` finds the nearest point within tolerance; `computeSceneSpans(scenes, durationSec)` lays out scenes without a persisted `startSec`/`endSec` contiguously as a display-only fallback; `computeDragSpan({kind, startSpan, deltaSec, gridPoints, ...})` resolves a timeline drag gesture (`'move'` or `'right'` — no `'left'`, since the render can only trim from a clip's own frame 0) into a new span; `shouldMarkBeatAligned({kind, snapped, wasPersisted})` gates the `beatAligned` flag so a reposition-only drag can't silently promote an unpersisted scene's placeholder fallback duration into a "saved exactly" render duration; `autoArrangeScenes(scenes, audioAnalysis)` proposes a full `{ sceneId, startSec, endSec, beatAligned }[]` arrangement by distributing scenes across song sections weighted by each section's `energy` (#1915). Used by `components/musicVideo/BeatTimeline.jsx` and `pages/MusicVideo.jsx`. || `musicVideoAspect.js` | Re-export of `server/lib/musicVideoAspect.js`: `musicVideoAspect`, `musicVideoFrameGenSize`, `matchesMusicVideoAspect`. Used by `hooks/useMusicVideoSceneMedia.js` so board frames match the project aspect. || `musicVideoFiling.js` | Which music video an Image Gen / Video Gen render is for: `MUSIC_VIDEO_FILING_PARAM` (the `musicVideo` URL param), `musicVideoFilingFields(searchParams)` (the `musicVideoProjectId` request field), `musicVideoOf(item)` (a record's video). Used by `components/media/MusicVideoFilingSelect.jsx`, `hooks/useMediaPreviewActions.js` and the two generate pages. |-| `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`) for board motion prompts. |+| `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`, `AUDIO_REACTIVE_PERFORMANCE_GUARD`, `silentMotionPrompt`) for board motion prompts. || `musicVideoBatchPlan.js` | Pure helpers for the Music Video missing-frame / missing-clip batches (#10153): `videoBatchPreview` (count + fal estimate summed from `falSceneTake`, unpriced count), `scenesWaitingForFrame`, `videosButtonLabel`, `batchSummary`, `batchActive`, `batchSceneName`. Used by `components/musicVideo/ProjectActionGroups.jsx`. || `musicVideoLayers.js` | Music-video per-scene visual layers (#8985), re-exported from `server/lib/musicVideoLayers.js`: `MUSIC_VIDEO_VISUAL_LAYERS`, `MUSIC_VIDEO_STILL_MOVES`, `MUSIC_VIDEO_TEXT_ZONES` / `MUSIC_VIDEO_LYRIC_ROLES` (#10583 lyric type per shot), `sceneVisualLayer(scene, { layered })`, `sceneHasAuthoredSpan`, `sceneRenderReady(scene, { layered })`. Used by `components/musicVideo/SceneCard.jsx`, `ProjectActionGroups.jsx` and `hooks/useMusicVideoSceneMedia.js`. || `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. |diff --git a/client/src/lib/musicVideoMotion.js b/client/src/lib/musicVideoMotion.jsindex c0b49fb00e..2c4c7cfdf2 100644--- a/client/src/lib/musicVideoMotion.js+++ b/client/src/lib/musicVideoMotion.js@@ -1,2 +1,2 @@/** Re-export of the server's motion-continuity clause so board clips pin the frame's lighting like production clips. */-export { MOTION_CONTINUITY_CLAUSE } from '../../../server/lib/musicVideoMotion.js';+export { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE, silentMotionPrompt } from '../../../server/lib/musicVideoMotion.js';diff --git a/server/lib/README.md b/server/lib/README.mdindex bac3116ad4..1709d19e3a 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -848,7 +848,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `loopInterval.js` | Loop interval grammar (`MIN_INTERVAL_MS`, `parseInterval`, `isValidLoopInterval`) shared by `services/loops.js` and `createLoopSchema` so the route rejects what the service would. || `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. |-| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. |+| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. || `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. || `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |diff --git a/server/lib/musicVideoMotion.js b/server/lib/musicVideoMotion.jsindex 785dbdc561..9b43e04fd4 100644--- a/server/lib/musicVideoMotion.js+++ b/server/lib/musicVideoMotion.js@@ -8,6 +8,9 @@*/export const MOTION_CONTINUITY_CLAUSE = "hold the reference frame's lighting, time of day and palette for the whole shot";+/** Appended to an audio-reactive (LoRA) shot's prompt so the music moves the scene, never a performer. */+export const AUDIO_REACTIVE_PERFORMANCE_GUARD = 'The music drives only environmental motion, lighting, particles, reflections, fabric, and subtle camera accents. No singing, lip-sync, speaking, mouth movement, dancing, instruments, performers, or musical performance.';+// A song reference a silent i2v model cannot hear: a quoted lyric cue ("on// 'Sleep'"), a musical event ("on the downbeat", "in time with the snare") or// a song timestamp ("at 1:42").diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex b6bf03deb2..a6117c326d 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -313,10 +313,12 @@ const submitValidatedVideoGenJob = async (body, uploads) => {// A take that hears nothing (no lip-sync or supplied song audio) gets plain physical// motion: the model would act out song cues and look words instead of reading them.const hearsAudio = !!(performance || prepared.audio || params.audioConditioning || params.audioFilePath);- if (!body.musicVideo.promptAsTyped) {+ // A shot card's prompt is sent as shown, its shot actions included; only a performance+ // take re-times them to its padded song slice.+ if (!body.musicVideo.promptAsTyped || performance) {params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec: performance?.shotInstruction?.edit?.inSec || 0 });- if (!hearsAudio) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);}+ if (!hearsAudio && !body.musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);const interval = performance?.shotInstruction?.songInterval || prepared.audio?.songInterval;if (params.audioConditioning && Math.round(scene?.startSec * 48000) !== params.audioConditioning.startSample) {throw new ServerError('Scene timing changed while source audio was prepared', { status: 409, code: 'MUSIC_VIDEO_SHOT_TIMING_CHANGED' });