Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Drop the send-time prompt cut; keep only the planner and treatment-draft wording
Claude6d17b40abb1a
diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsxindex 08072a2834..54640becd9 100644--- a/client/src/components/musicVideo/SceneRenderInputs.jsx+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx@@ -1,5 +1,5 @@import { getFalVideoModel } from '../../lib/falVideoModels.js';-import { AUDIO_REACTIVE_PERFORMANCE_GUARD, silentMotionPrompt, videoModelPrompt } from '../../lib/musicVideoMotion.js';+import { AUDIO_REACTIVE_PERFORMANCE_GUARD, silentMotionPrompt } from '../../lib/musicVideoMotion.js';import { approximateMotionCues, falSceneTake } from '../../lib/musicVideoShotTiming.js';import { shotActionContractProblem, shotActionPrompt } from '../../../../server/lib/musicVideoActionContract.js';import { shotCameraPrompt } from '../../../../server/lib/cameraMovements.js';@@ -13,9 +13,7 @@ const clock = (sec) => {};/**- * Exactly what the shot card's Render sends: the prompt as typed (less song- * spans, clip lengths, trim notes and, when the take hears nothing, "No- * lip-sync", which the model cannot act on), then what+ * Exactly what the shot card's Render sends: the prompt as typed, then what* the card itself sets (its camera move, its applied shot actions, Grok's* approximate phrase timing, the audio-reactive guard), and which inputs go* with it. The project's style and bible are not added, and the server@@ -55,8 +53,7 @@ export function shotCardRenderRequest({const motionCues = settings.backend === 'grok' ? approximateMotionCues(project?.phrases, scene?.startSec, scene?.endSec) : '';const base = [(scene?.prompt?.trim() || scene?.framePrompt?.trim() || ''), shotCameraPrompt(scene?.camera),hears ? actions : silentMotionPrompt(actions)].filter(Boolean).join('\n');- // The server applies the same cut at submit, so this stays what is sent.- const prompt = videoModelPrompt(audioReactive ? `${base}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}` : [base, motionCues].filter(Boolean).join('. '), { hearsAudio: hears });+ const prompt = audioReactive ? `${base}. ${AUDIO_REACTIVE_PERFORMANCE_GUARD}` : [base, motionCues].filter(Boolean).join('. ');let endFrame = null;if (scene?.endImageId) {diff --git a/client/src/components/musicVideo/SceneRenderInputs.test.jsx b/client/src/components/musicVideo/SceneRenderInputs.test.jsxindex 311d55e9c2..f4f268673d 100644--- a/client/src/components/musicVideo/SceneRenderInputs.test.jsx+++ b/client/src/components/musicVideo/SceneRenderInputs.test.jsx@@ -24,16 +24,6 @@ describe('shotCardRenderRequest', () => {expect(shotCardRenderRequest({ ...base, inputs: { endFrame: false } }).endFrame.state).toBe('off');});- // Adam: song spans, clip lengths and trim notes never reach the model, nor "No lip-sync" for a take that hears nothing.- it('leaves render bookkeeping out of the prompt it shows and sends', () => {- const typed = '26.45-28.31s (1.87 s). Her palm slams the splicer. No lip-sync. Use the first ~1.9 s.';- const base = { scene: scene({ prompt: typed }), settings: { backend: 'local', modelId: 'ltx25_example' }, models };- expect(shotCardRenderRequest(base).prompt).toBe('Her palm slams the splicer.');- expect(shotCardRenderRequest({ ...base, inputs: { songAudio: true } }).prompt).toBe('Her palm slams the splicer. No lip-sync.');- // A leading count is action, not a song span.- expect(shotCardRenderRequest({ ...base, scene: scene({ prompt: '2-3: two dancers spin' }) }).prompt).toBe('2-3: two dancers spin');- });-// Review on #11280: an audio-reactive render hears the song through its LoRA, so the card must say so and never stack a supplied window.it('shows an audio-reactive render as hearing the song, with its guard in the prompt', () => {const request = shotCardRenderRequest({ scene: scene(), settings: { backend: 'local', modelId: 'ltx25_example', generationMode: 'audioReactive' }, models,diff --git a/client/src/lib/README.md b/client/src/lib/README.mdindex 226eb48bcd..adf90c262f 100644--- a/client/src/lib/README.md+++ b/client/src/lib/README.md@@ -61,7 +61,7 @@ grep -i "what you want to do" client/src/lib/README.md| `beatGrid.js` | Music-video beat-quantized timeline arranger (#1854). `buildBeatGridPoints(audioAnalysis)` merges beats/downbeats/section edges into one ranked snap-point list; `snapTimeToGrid(timeSec, gridPoints, toleranceSec)` finds the nearest point within tolerance; `computeSceneSpans(scenes, durationSec)` lays out scenes without a persisted `startSec`/`endSec` contiguously as a display-only fallback; `computeDragSpan({kind, startSpan, deltaSec, gridPoints, ...})` resolves a timeline drag gesture (`'move'` or `'right'` — no `'left'`, since the render can only trim from a clip's own frame 0) into a new span; `shouldMarkBeatAligned({kind, snapped, wasPersisted})` gates the `beatAligned` flag so a reposition-only drag can't silently promote an unpersisted scene's placeholder fallback duration into a "saved exactly" render duration; `autoArrangeScenes(scenes, audioAnalysis)` proposes a full `{ sceneId, startSec, endSec, beatAligned }[]` arrangement by distributing scenes across song sections weighted by each section's `energy` (#1915). Used by `components/musicVideo/BeatTimeline.jsx` and `pages/MusicVideo.jsx`. || `musicVideoAspect.js` | Re-export of `server/lib/musicVideoAspect.js`: `musicVideoAspect`, `musicVideoFrameGenSize`, `matchesMusicVideoAspect`. Used by `hooks/useMusicVideoSceneMedia.js` so board frames match the project aspect. || `musicVideoFiling.js` | Which music video an Image Gen / Video Gen render is for: `MUSIC_VIDEO_FILING_PARAM` (the `musicVideo` URL param), `musicVideoFilingFields(searchParams)` (the `musicVideoProjectId` request field), `musicVideoOf(item)` (a record's video). Used by `components/media/MusicVideoFilingSelect.jsx`, `hooks/useMediaPreviewActions.js` and the two generate pages. |-| `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`, `AUDIO_REACTIVE_PERFORMANCE_GUARD`, `silentMotionPrompt`, `videoModelPrompt`) for board motion prompts. |+| `musicVideoMotion.js` | Re-export of `server/lib/musicVideoMotion.js` (`MOTION_CONTINUITY_CLAUSE`, `AUDIO_REACTIVE_PERFORMANCE_GUARD`, `silentMotionPrompt`) for board motion prompts. || `musicVideoBatchPlan.js` | Pure helpers for the Music Video missing-frame / missing-clip batches (#10153): `videoBatchPreview` (count + fal estimate summed from `falSceneTake`, unpriced count), `scenesWaitingForFrame`, `videosButtonLabel`, `batchSummary`, `batchActive`, `batchSceneName`. Used by `components/musicVideo/ProjectActionGroups.jsx`. || `musicVideoLayers.js` | Music-video per-scene visual layers (#8985), re-exported from `server/lib/musicVideoLayers.js`: `MUSIC_VIDEO_VISUAL_LAYERS`, `MUSIC_VIDEO_STILL_MOVES`, `MUSIC_VIDEO_TEXT_ZONES` / `MUSIC_VIDEO_LYRIC_ROLES` (#10583 lyric type per shot), `sceneVisualLayer(scene, { layered })`, `sceneHasAuthoredSpan`, `sceneRenderReady(scene, { layered })`. Used by `components/musicVideo/SceneCard.jsx`, `ProjectActionGroups.jsx` and `hooks/useMusicVideoSceneMedia.js`. || `musicVideoTakes.js` | Music-video scene takes view helpers (#8965): `sceneTakeList(scene, kind)` lists a scene's image/video takes with an un-materialized pre-takes selection appended as a read-only `legacy` take; `takePreviewKey`, `takeThumbUrl`, `takeClipUrl` (hover-preview clip), `sceneHasPendingDecision` (contact-sheet filter), `takeProvenance`; `TAKE_SLOT` maps a take kind to its selection field. Used by `components/musicVideo/SceneTakeStrip.jsx` and `pages/MusicVideo.jsx`. |diff --git a/client/src/lib/musicVideoMotion.js b/client/src/lib/musicVideoMotion.jsindex 49860119d9..2c4c7cfdf2 100644--- a/client/src/lib/musicVideoMotion.js+++ b/client/src/lib/musicVideoMotion.js@@ -1,2 +1,2 @@/** Re-export of the server's motion-continuity clause so board clips pin the frame's lighting like production clips. */-export { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE, silentMotionPrompt, videoModelPrompt } from '../../../server/lib/musicVideoMotion.js';+export { AUDIO_REACTIVE_PERFORMANCE_GUARD, MOTION_CONTINUITY_CLAUSE, silentMotionPrompt } from '../../../server/lib/musicVideoMotion.js';diff --git a/server/lib/README.md b/server/lib/README.mdindex 839b100cfb..442c26da64 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -850,7 +850,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `loopInterval.js` | Loop interval grammar (`MIN_INTERVAL_MS`, `parseInterval`, `isValidLoopInterval`) shared by `services/loops.js` and `createLoopSchema` so the route rejects what the service would. || `musicVideoAspect.js` | `musicVideoAspect(project)` (treatment brief aspect, default 16:9), `MUSIC_VIDEO_FRAME_GEN_SIZES` and `musicVideoFrameGenSize(project)`: the size every Music Video reference frame is requested at, so a backend never picks portrait for a 16:9 video. `musicVideoAtAspect(project, aspect)` re-frames the project for one render (a vertical social cut) without touching the record. `musicVideoLocalClipSize(project, model, { endFrame })` sizes a local clip to the aspect, `musicVideoRenderCanvas(aspect, lead)` the final render's frame, and `matchesMusicVideoAspect(w, h, aspect)` tests a frame's shape. Shared by `productionDispatch.js`, `documentRender.js` and the client board. |-| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. `videoModelPrompt(prompt, { hearsAudio })`: the prompt without render bookkeeping (a leading song span and clip length, a "use the first N s" trim note) and, for a take that hears nothing, its "No lip-sync" line; applied to every Music Video submit and the shot card preview. |+| `musicVideoMotion.js` | `MOTION_CONTINUITY_CLAUSE`: the clause every Music Video motion (i2v) prompt ends with, pinning the reference frame's lighting, time of day and palette. Shared by `handoff.js` and the client board. `AUDIO_REACTIVE_PERFORMANCE_GUARD`: the clause an audio-reactive shot's prompt ends with. `silentMotionPrompt(prompt, { styleParts })`: the same prompt as plain physical motion (no lyric/beat cues, song times or look words) for a take that hears no audio; applied at video submit. || `musicVideoConditioning.js` | `musicVideoStyleImages`, `musicVideoStylePrompt`, `musicVideoStyleBasis`, and `musicVideoStyleReferenceCapacity` supply bounded moodboard conditioning and text fallback; `musicVideoConditioningReferences(project, scene)` selects the approved character sheet, the scene's mapped set plate and the sheets of supporting cast the scene's prompt names (capped at 4), with capped project-wide fallback; shared by board and production. || `musicVideoMediumPlan.js` | `normalizeMusicVideoProductionPolicy`, `planMusicVideoMedia`, `summarizeMusicVideoMediumPlan`, and `codeFirstProductionAssets`: pinned shot media, union-of-intervals video allowance, and selected-asset preflight shared by planning and Board review. `musicVideoBriefTools`, `musicVideoBriefAllowsVideo` and `musicVideoToolPolicyConflict` read the brief's selected tools against the policy (the setup warning and the video-dispatch refusal). |diff --git a/server/lib/musicVideoMotion.js b/server/lib/musicVideoMotion.jsindex e6f8bc3255..9b43e04fd4 100644--- a/server/lib/musicVideoMotion.js+++ b/server/lib/musicVideoMotion.js@@ -80,25 +80,3 @@ export function silentMotionPrompt(prompt, { styleParts = [], keep = [] } = {}).filter((line, index, lines) => line.trim() || (index > 0 && lines[index - 1].trim())).join('\n').trim();}--// Render bookkeeping a video model cannot act on: a leading song span with its length-// ("26.45-28.31s (1.87 s)." or "0:26.4–0:28.3:"), and a trim note ("Use the first ~1.9 s.").-// A span has a decimal, a seconds unit or a m:ss clock, so a leading count ("2-3: two dancers") stays.-const SONG_SPAN_PREFIX = /^\s*(?=[\d:.]*[.:]\d|[\d\s:.–—-]*\ds)(?:\d{1,2}:)?\d+(?:\.\d+)?\s*s?\s*[-–—]\s*(?:\d{1,2}:)?\d+(?:\.\d+)?\s*s?\s*(?:\(\s*~?\d+(?:\.\d+)?\s*s(?:ec(?:onds?)?)?\s*\))?\s*[.:]\s*/;-const TRIM_NOTE = /(^|[.!?;]\s+)(?:Use|Keep)\s+(?:only\s+)?the\s+(?:first|last|opening|final)\s+~?\d+(?:\.\d+)?\s*s(?:ec(?:onds?)?)?\b(?:\s+of\s+(?:the\s+)?(?:clip|take|output|render|shot))?\s*(?:[.!?]|$)/gi;-// "No lip-sync." tells a model that hears nothing to avoid what it cannot do.-const NO_LIP_SYNC = /(^|[.!?;]\s+)No\s+(?:singing\s*(?:,|or)\s*)?lip[- ]?sync(?:ing)?(?:\s*(?:,|or)\s*(?:singing|speaking|talking))?\s*(?:[.!?]|$)/gi;--/**- * The prompt a video model receives: the authored prompt without render bookkeeping- * (a leading song span and clip length, a "use the first N s" trim note), and without- * a "No lip-sync." line when the take hears no audio. Applied at video submit and by- * the shot card's preview, so the preview shows what is sent.- */-export function videoModelPrompt(prompt, { hearsAudio = false } = {}) {- return String(prompt || '').split('\n').map((line) => {- let text = line.replace(SONG_SPAN_PREFIX, '').replace(TRIM_NOTE, '$1');- if (!hearsAudio) text = text.replace(NO_LIP_SYNC, '$1');- return text === line ? line : text.replace(/[ \t]{2,}/g, ' ').trim();- }).join('\n').trim();-}diff --git a/server/services/musicVideo/planner.js b/server/services/musicVideo/planner.jsindex 9ff6b91e0a..fabc9561ce 100644--- a/server/services/musicVideo/planner.js+++ b/server/services/musicVideo/planner.js@@ -229,7 +229,7 @@ ${shotLines}For EACH shot above, propose a timed composition for its assigned medium (code-authored worlds, selected stills or footage):- "framePrompt": the opening reference still — subject, setting, lighting, composition. Keep it concrete and visual.-- "prompt": the motion for that shot as plain physical action — who moves, how, where the camera goes and the end state — building on the frame. Higher-energy sections read more kinetic; calmer sections more lingering. The video model sees only the frame and usually hears nothing, and it acts out every word: no song times, clip lengths, trim notes ("use the first 2 s"), beats, lyric cues ("on 'Sleep'") or song references, no "no lip-sync" line, and no look words (film stock, grain, lens, grade, palette), which the frame already carries.+- "prompt": the motion for that shot as plain physical action — who moves, how, where the camera goes and the end state — building on the frame. Higher-energy sections read more kinetic; calmer sections more lingering. The video model sees only the frame and usually hears nothing, and it acts out every word: no song times or timestamps ("on the 9.6s downbeat", "at 10.573"), clip lengths, trim notes ("use the first 2 s"), beats, lyric cues ("on 'Sleep'") or song references, no "no lip-sync" line, and no look words (film stock, grain, lens, grade, palette), which the frame already carries.- "camera": the shot's camera move from the vocabulary below, with its speed, end framing, whether it lands on the downbeat, and (for a static camera) the reason.- Shots in the same section are one edited sequence: keep subject and setting continuous, but vary framing (wide / medium / close), angle, or action from shot to shot so consecutive shots cut rather than repeat.- The OPENING HOOK shot must grab attention immediately.diff --git a/server/services/musicVideo/treatmentDraft.js b/server/services/musicVideo/treatmentDraft.jsindex 3ad2ba9da1..5ac2c8b392 100644--- a/server/services/musicVideo/treatmentDraft.js+++ b/server/services/musicVideo/treatmentDraft.js@@ -417,7 +417,7 @@ ${project.styleReferences?.length ? '- "styleLook": one short sentence summarizi- Two or three recurring "motifs" (an image, object, color or gesture) with how each changes across the arc.- The balance of "performance", "cutaway" and "graphic" shots as percentages, with a rationale.- "graphicLanguage": a concise visual note for the graphic cards and typography (HUD, pictograms, counters), honoring any graphic direction in the brief.-${hasLyrics ? '- "lyricInterpretation": what the lyrics mean and how the picture interprets them (not word-for-word illustration).\n' : '- The song is instrumental: set "lyricInterpretation" to null and invent no lyrics.\n'}- For EACH shot: "mode" (performance|cutaway|graphic), "route" (generated|supplied-asset|code-2d — code-2d becomes a code-rendered title card for a sung line or a moved still; prefer it where a video model cannot hold the requirement), "focalSubject", "framing", "negativeSpace" (none|upper|center|lower — the region kept clean for the separately composited text), "typographyRole" (none|subtitle|hero; must be none on instrumental shots), "emphasis", "transitionIn", "transitionOut", "rationale", and a "framePrompt" (the opening still) and "prompt" (the motion) for the image/video model. The motion "prompt" is plain physical action: no song times, clip lengths, trim notes ("use the first 2 s") or "no lip-sync" line, which the video model cannot act on.+${hasLyrics ? '- "lyricInterpretation": what the lyrics mean and how the picture interprets them (not word-for-word illustration).\n' : '- The song is instrumental: set "lyricInterpretation" to null and invent no lyrics.\n'}- For EACH shot: "mode" (performance|cutaway|graphic), "route" (generated|supplied-asset|code-2d — code-2d becomes a code-rendered title card for a sung line or a moved still; prefer it where a video model cannot hold the requirement), "focalSubject", "framing", "negativeSpace" (none|upper|center|lower — the region kept clean for the separately composited text), "typographyRole" (none|subtitle|hero; must be none on instrumental shots), "emphasis", "transitionIn", "transitionOut", "rationale", and a "framePrompt" (the opening still) and "prompt" (the motion) for the image/video model. The motion "prompt" is plain physical action: no song times or timestamps ("on the 9.6s downbeat", "at 10.573"), clip lengths, trim notes ("use the first 2 s") or "no lip-sync" line, which the video model cannot act on.- For EACH shot also supply "medium" (procedural|still|existing-footage|generated-footage) and "mediumRationale". Medium is independent of performance/cutaway intent. Retain director pins. Tie the medium to the shared motifs, their evolution across repeated sections, and entry/exit transitions.- MEDIA MODE: ${musicVideoMediaMode(project)}. Code authors scene composition, characters, environment, camera and timing in every mode. Code-only forbids images even for visual guides; code-images forbids generated or imported footage.- For code-first, prefer procedural or still. Existing footage must be selected already; generated footage is an explicit exception within the allowance. At zero allowance never assign generated footage: report unmet performance intent rather than quietly generating it. This is planning, not a claim that procedural rendering is ready.diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex 9a6e99dcb0..a6117c326d 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -319,9 +319,6 @@ const submitValidatedVideoGenJob = async (body, uploads) => {params.prompt = withShotActionPrompt(params.prompt, project, scene, { offsetSec: performance?.shotInstruction?.edit?.inSec || 0 });}if (!hearsAudio && !body.musicVideo.promptAsTyped) params.prompt = await silentMusicVideoMotionPrompt(params.prompt, project, scene);- // Song spans, clip lengths and trim notes are render bookkeeping the model cannot act on.- const { videoModelPrompt } = await import('../../lib/musicVideoMotion.js');- params.prompt = videoModelPrompt(params.prompt, { hearsAudio });const interval = performance?.shotInstruction?.songInterval || prepared.audio?.songInterval;if (params.audioConditioning && Math.round(scene?.startSec * 48000) !== params.audioConditioning.startSample) {throw new ServerError('Scene timing changed while source audio was prepared', { status: 409, code: 'MUSIC_VIDEO_SHOT_TIMING_CHANGED' });diff --git a/server/services/videoGen/submitJob.test.js b/server/services/videoGen/submitJob.test.jsindex 8777197aee..4c4a588832 100644--- a/server/services/videoGen/submitJob.test.js+++ b/server/services/videoGen/submitJob.test.js@@ -436,21 +436,6 @@ describe('shot intent at the video submission boundary', () => {expect(params.shotInstruction.actionContract).toEqual(contract);});- // Adam: a song span, a clip length, a trim note or a "No lip-sync" for a model that hears nothing is noise the model cannot act on.- it('drops render bookkeeping from any Music Video prompt, and "No lip-sync" when the take hears no audio', async () => {- await submitVideoGenJob({ prompt: '26.45-28.31s (1.87 s). Her palm slams the splicer. No lip-sync. Use the first ~1.9 s.',- musicVideo: { projectId: 'p', sceneId: 's', promptAsTyped: true } }, {});- expect(mocks.enqueueJob.mock.calls[0][0].params.prompt.split('\n')[0]).toBe('Her palm slams the splicer.');- });-- it('keeps "No lip-sync" for a take that hears the song', async () => {- mocks.prepareVideoGenParams.mockResolvedValue({ backend: 'fal', cleanupStaged: vi.fn(), sourceImagePath: '/example/frame.png' });- mocks.preparePerformanceShot.mockResolvedValue({ audioFilePath: '/example/audio.wav', modelId: 'lip-sync', resolution: '1080P',- shotInstruction: { version: 1, shotMode: 'performance', songInterval: { startSec: 10, endSec: 14 }, edit: { inSec: 1.5, outSec: 5.5 } } });- await submitVideoGenJob({ prompt: '10-14s. She listens. No lip-sync.', backend: 'fal', musicVideo: { projectId: 'p', sceneId: 's', promptAsTyped: true } }, {});- expect(mocks.enqueueJob.mock.calls[0][0].params.prompt.split('\n')[0]).toBe('She listens. No lip-sync.');- });-it('still re-times a card-typed performance prompt to its padded song slice', async () => {mocks.prepareVideoGenParams.mockResolvedValue({ backend: 'fal', cleanupStaged: vi.fn(), sourceImagePath: '/example/frame.png' });mocks.preparePerformanceShot.mockResolvedValue({ audioFilePath: '/example/audio.wav', modelId: 'lip-sync', resolution: '1080P',