Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Code Animation: write the soundtrack with a coding model
Claude0f140be9756f
diff --git a/client/src/components/codeAnimation/CodeAnimationCodeScore.jsx b/client/src/components/codeAnimation/CodeAnimationCodeScore.jsxnew file mode 100644index 0000000000..82543d8b7b--- /dev/null+++ b/client/src/components/codeAnimation/CodeAnimationCodeScore.jsx@@ -0,0 +1,100 @@+import { useEffect, useState } from 'react';+import CodePanel from '../music/CodePanel';+import ProviderModelSelector from '../ProviderModelSelector';+import useProviderModels from '../../hooks/useProviderModels';+import { ensureCodeAnimationSoundtrackTrack } from '../../services/api';+import { formatCount } from '../../utils/formatters';++// Mirrors MAX_SCENE in server/services/musicCode.js.+const SCRIPT_MAX = 20000;++const inputClass = 'w-full rounded-lg border border-port-border bg-port-bg px-3 py-2 text-sm text-white placeholder:text-gray-600 focus:border-port-accent focus:outline-none';+const labelClass = 'mb-1 block text-xs text-gray-400';++/** The animation's own words, offered as the coding model's script. */+export function animationScript(job) {+ const input = job?.input || {};+ return [+ input.title || job?.title || '',+ input.concept || '',+ input.onScreenText ? `On-screen text:\n${input.onScreenText}` : '',+ ].map((part) => part.trim()).filter(Boolean).join('\n\n');+}++/**+ * A coding model writes the soundtrack as Strudel, Tone.js or SuperCollider+ * code from the animation script, at the film's length. The take saves onto+ * the animation's soundtrack track (created on the first save), which then+ * becomes the animation's audio.+ */+export default function CodeAnimationCodeScore({+ job, trackId, description, filmSec, pin, onPinChange, onSaved,+}) {+ const [script, setScript] = useState(() => animationScript(job));+ const [effort, setEffort] = useState(pin?.effort || '');+ const [pinApplied, setPinApplied] = useState(false);+ useEffect(() => { setScript(animationScript(job)); }, [job?.id]);++ const {+ providers, selectedProviderId, selectedModel, availableModels,+ setSelectedProviderId, setSelectedModel, loading,+ } = useProviderModels({ silent: true, withEffort: true });++ // Open on the last pick, else on the provider that wrote the animation.+ useEffect(() => {+ if (pinApplied || loading || !providers.length) return;+ setPinApplied(true);+ const wanted = pin?.providerId ? pin : { providerId: job?.providerId, model: job?.model };+ const provider = providers.find((item) => item.id === wanted.providerId);+ if (!provider) return;+ setSelectedProviderId(provider.id);+ const models = (provider.models?.length ? provider.models : [provider.defaultModel])+ .map((model) => (typeof model === 'string' ? model : model?.id)).filter(Boolean);+ if (wanted.model && models.includes(wanted.model)) setSelectedModel(wanted.model);+ }, [pinApplied, loading, providers, pin, job?.providerId, job?.model, setSelectedProviderId, setSelectedModel]);++ const remember = (patch) => onPinChange({ providerId: selectedProviderId || '', model: selectedModel || '', effort, ...patch });++ const providerPicker = (+ <ProviderModelSelector+ providers={providers}+ selectedProviderId={selectedProviderId}+ selectedModel={selectedModel}+ availableModels={availableModels}+ onProviderChange={(providerId) => { setSelectedProviderId(providerId); setEffort(''); remember({ providerId, model: '', effort: '' }); }}+ onModelChange={(model) => { setSelectedModel(model); remember({ model }); }}+ effort={effort}+ onEffortChange={(next) => { setEffort(next); remember({ effort: next }); }}+ disabled={loading}+ layout="stacked"+ />+ );++ return (+ <div className="space-y-3">+ <div>+ <label htmlFor="ca-soundtrack-script" className={labelClass}>Animation script (sent to the coding model)</label>+ <textarea id="ca-soundtrack-script" rows={4} value={script} maxLength={SCRIPT_MAX}+ onChange={(event) => setScript(event.target.value)}+ className={`${inputClass} resize-y`} />+ {script.length > SCRIPT_MAX && (+ <p role="alert" className="text-xs text-port-warning">The script has {formatCount(script.length)} characters; the limit is {formatCount(SCRIPT_MAX)}.</p>+ )}+ </div>+ <CodePanel+ trackId={trackId || null}+ draftKey={`code-animation:${job.id}`}+ resolveTrackId={async () => (await ensureCodeAnimationSoundtrackTrack(job.id, { silent: true }))?.trackId}+ description={description}+ scene={script}+ fixedLengthSec={filmSec || null}+ title=""+ providerId={selectedProviderId}+ model={selectedModel}+ effort={effort}+ providerPicker={providerPicker}+ onRendered={onSaved}+ />+ </div>+ );+}diff --git a/client/src/components/codeAnimation/CodeAnimationSoundtrack.jsx b/client/src/components/codeAnimation/CodeAnimationSoundtrack.jsxindex 7816933cb8..c95a623035 100644--- a/client/src/components/codeAnimation/CodeAnimationSoundtrack.jsx+++ b/client/src/components/codeAnimation/CodeAnimationSoundtrack.jsx@@ -1,5 +1,6 @@import { useEffect, useMemo, useRef, useState } from 'react';-import { LoaderCircle, Music2 } from 'lucide-react';+import { Code2, LoaderCircle, Music2 } from 'lucide-react';+import CodeAnimationCodeScore from './CodeAnimationCodeScore';import toast from '../ui/Toast';import ModelInputPreview from '../media/ModelInputPreview';import ListLoadError from '../ui/ListLoadError';@@ -9,13 +10,15 @@ import { useModelInputPreview } from '../../hooks/useModelInputPreview';import { generateCodeAnimationSoundtrack, listMusicEngines, previewMusicPrompt } from '../../services/api';import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage';-// The last engine/model/instrumental choice, reused for every animation.+// The last mode (music model or coding model) and each mode's picks, reused for+// every animation.export const SOUNDTRACK_CHOICE_KEY = 'portos.codeAnimation.soundtrackChoice';// Long-form engines first when nothing has been picked yet.const PREFERRED_ENGINES = ['acestep15', 'acestep', 'minimax-music3-mlx', 'minimax-music3'];const inputClass = 'w-full rounded-lg border border-port-border bg-port-bg px-3 py-2 text-sm text-white placeholder:text-gray-600 focus:border-port-accent focus:outline-none';const labelClass = 'mb-1 block text-xs text-gray-400';+const buttonGhost = 'inline-flex items-center gap-2 rounded-lg border border-port-border px-3 py-1.5 text-sm text-gray-300 hover:text-white disabled:cursor-not-allowed disabled:opacity-40';const buttonPrimary = 'inline-flex items-center gap-2 rounded-lg bg-port-accent px-3 py-1.5 text-sm text-white hover:opacity-90 disabled:cursor-not-allowed disabled:opacity-40';const readChoice = () => {@@ -33,12 +36,18 @@ function engineNote(engine, filmSec) {return null;}+const MODES = [+ { id: 'model', label: 'Music model', icon: Music2 },+ { id: 'code', label: 'Coding model', icon: Code2 },+];+/**- * Generate audio for a finished animation: pick a local music model, describe- * the score, and render a track at the film's length. The animation is not- * re-run; the track becomes its audio for the preview and the MP4 export.+ * Generate audio for a finished animation at the film's length, either with a+ * local music model or as code (Strudel / Tone.js / SuperCollider) a coding+ * model writes from the animation script. The animation is not re-run; the+ * track becomes its audio for the preview and the MP4 export.*/-export default function CodeAnimationSoundtrack({ jobId, soundtrack, audioUrl, audioTrackId, durationSeconds, onReady }) {+export default function CodeAnimationSoundtrack({ job, jobId, soundtrack, audioUrl, audioTrackId, durationSeconds, onReady }) {const filmSec = Math.ceil(Number(durationSeconds) || 0);const engineList = useListResource(fetchEngines);const engines = engineList.state === 'ready' ? engineList.items : null;@@ -67,6 +76,7 @@ export default function CodeAnimationSoundtrack({ jobId, soundtrack, audioUrl, aconst models = engine?.models || [];const modelId = models.some((model) => model.id === choice.modelId) ? choice.modelId : (engine?.defaultModelId || models[0]?.id || '');const instrumentalOnly = choice.instrumentalOnly !== false;+ const mode = choice.mode === 'code' && job ? 'code' : 'model';const engineBlocked = engine ? engineNote(engine, filmSec) : null;const updateChoice = (patch) => setChoice((current) => {@@ -77,7 +87,7 @@ export default function CodeAnimationSoundtrack({ jobId, soundtrack, audioUrl, aconst captionPreview = useModelInputPreview(previewMusicPrompt, {prompt, engine: engine?.id, instrumentalOnly, durationSec: filmSec || undefined,- }, { enabled: !!engine && !!prompt.trim() });+ }, { enabled: mode === 'model' && !!engine && !!prompt.trim() });const progress = useMediaJobProgress(mediaJobId, { kind: 'audio' });const live = progress.status === 'queued' || progress.status === 'running' || (!!mediaJobId && mediaJobId === startedId && progress.status === 'unknown');@@ -119,55 +129,83 @@ export default function CodeAnimationSoundtrack({ jobId, soundtrack, audioUrl, a<h3 id="ca-soundtrack-heading" className="flex items-center gap-2 text-sm font-semibold text-white"><Music2 className="h-4 w-4 text-port-accent" /> Soundtrack</h3>- <div className="grid gap-3 sm:grid-cols-2">- <div>- <label htmlFor="ca-soundtrack-engine" className={labelClass}>Music model</label>- <select id="ca-soundtrack-engine" value={engine?.id || ''} disabled={!engines?.length || live}- onChange={(event) => updateChoice({ engine: event.target.value, modelId: '' })} className={inputClass}>- {engines === null && <option value="">{engineList.state === 'error' ? 'Models unavailable' : 'Loading models…'}</option>}- {engines?.length === 0 && <option value="">No music models</option>}- {(engines || []).map((item) => {- const note = engineNote(item, filmSec);- return <option key={item.id} value={item.id}>{item.name}{note ? ` (${note})` : ''}</option>;- })}- </select>+ {job && (+ <div role="group" aria-label="Make the audio with" className="flex flex-wrap gap-2">+ {MODES.map(({ id, label, icon: Icon }) => (+ <button key={id} type="button" aria-pressed={mode === id} disabled={live}+ onClick={() => updateChoice({ mode: id })} className={mode === id ? buttonPrimary : buttonGhost}>+ <Icon className="h-4 w-4" /> {label}+ </button>+ ))}</div>- {models.length > 1 && (+ )}+ {mode === 'model' && (+ <div className="grid gap-3 sm:grid-cols-2"><div>- <label htmlFor="ca-soundtrack-model" className={labelClass}>Checkpoint</label>- <select id="ca-soundtrack-model" value={modelId} disabled={live}- onChange={(event) => updateChoice({ modelId: event.target.value })} className={inputClass}>- {models.map((model) => <option key={model.id} value={model.id}>{model.name || model.id}</option>)}+ <label htmlFor="ca-soundtrack-engine" className={labelClass}>Music model</label>+ <select id="ca-soundtrack-engine" value={engine?.id || ''} disabled={!engines?.length || live}+ onChange={(event) => updateChoice({ engine: event.target.value, modelId: '' })} className={inputClass}>+ {engines === null && <option value="">{engineList.state === 'error' ? 'Models unavailable' : 'Loading models…'}</option>}+ {engines?.length === 0 && <option value="">No music models</option>}+ {(engines || []).map((item) => {+ const note = engineNote(item, filmSec);+ return <option key={item.id} value={item.id}>{item.name}{note ? ` (${note})` : ''}</option>;+ })}</select></div>- )}- </div>- {engineList.state === 'error' && <ListLoadError what="music models" error={engineList.error} onRetry={engineList.retry} />}- {engine && engineBlocked === 'not installed' && (+ {models.length > 1 && (+ <div>+ <label htmlFor="ca-soundtrack-model" className={labelClass}>Checkpoint</label>+ <select id="ca-soundtrack-model" value={modelId} disabled={live}+ onChange={(event) => updateChoice({ modelId: event.target.value })} className={inputClass}>+ {models.map((model) => <option key={model.id} value={model.id}>{model.name || model.id}</option>)}+ </select>+ </div>+ )}+ </div>+ )}+ {mode === 'model' && engineList.state === 'error' && <ListLoadError what="music models" error={engineList.error} onRetry={engineList.retry} />}+ {mode === 'model' && engine && engineBlocked === 'not installed' && (<p className="text-xs text-port-warning">{engine.name} is not installed. Install it in Music › Generate, or pick another model.</p>)}- {engine && engineBlocked && engineBlocked !== 'not installed' && (+ {mode === 'model' && engine && engineBlocked && engineBlocked !== 'not installed' && (<p className="text-xs text-port-warning">The film is {filmSec}s; {engine.name} renders at most {engine.maxDurationSec}s. Pick a model marked for longer audio.</p>)}<div>- <label htmlFor="ca-soundtrack-prompt" className={labelClass}>Describe the music</label>+ <label htmlFor="ca-soundtrack-prompt" className={labelClass}>{mode === 'code' ? 'Describe the sound (optional)' : 'Describe the music'}</label><textarea id="ca-soundtrack-prompt" rows={3} value={prompt} maxLength={8000} disabled={live}onChange={(event) => setPrompt(event.target.value)}- placeholder="Warm analog synths, slow build, soft drums from 0:20, gentle fade at the end"+ placeholder={mode === 'code'+ ? 'Mostly sound effects: whirrs, blips and boops over a soft synth pad'+ : 'Warm analog synths, slow build, soft drums from 0:20, gentle fade at the end'}className={`${inputClass} resize-y`} /></div>- <ModelInputPreview preview={captionPreview} prompt={prompt} />- <label className="flex items-center gap-2 text-xs text-gray-300">- <input type="checkbox" checked={instrumentalOnly} disabled={live} onChange={(event) => updateChoice({ instrumentalOnly: event.target.checked })} />- Instrumental only- </label>- <div className="flex flex-wrap items-center gap-3">- <button type="button" onClick={handleGenerate} disabled={!canGenerate} className={buttonPrimary}>- {starting || live ? <LoaderCircle className="h-4 w-4 animate-spin" /> : <Music2 className="h-4 w-4" />}- {live ? `Generating audio ${percent}%` : hasTake ? 'Regenerate audio' : 'Generate audio'}- </button>- <span className="text-xs text-gray-500">{filmSec ? `${filmSec}s, the film's length` : ''}</span>- </div>+ {mode === 'code' ? (+ <CodeAnimationCodeScore+ job={job}+ trackId={soundtrack?.trackId}+ description={prompt}+ filmSec={filmSec}+ pin={choice.code}+ onPinChange={(code) => updateChoice({ code })}+ onSaved={() => onReady?.()}+ />+ ) : (+ <>+ <ModelInputPreview preview={captionPreview} prompt={prompt} />+ <label className="flex items-center gap-2 text-xs text-gray-300">+ <input type="checkbox" checked={instrumentalOnly} disabled={live} onChange={(event) => updateChoice({ instrumentalOnly: event.target.checked })} />+ Instrumental only+ </label>+ <div className="flex flex-wrap items-center gap-3">+ <button type="button" onClick={handleGenerate} disabled={!canGenerate} className={buttonPrimary}>+ {starting || live ? <LoaderCircle className="h-4 w-4 animate-spin" /> : <Music2 className="h-4 w-4" />}+ {live ? `Generating audio ${percent}%` : hasTake ? 'Regenerate audio' : 'Generate audio'}+ </button>+ <span className="text-xs text-gray-500">{filmSec ? `${filmSec}s, the film's length` : ''}</span>+ </div>+ </>+ )}{live && (<div className="space-y-1" role="status" aria-label="Soundtrack progress"><div className="h-1.5 overflow-hidden rounded bg-port-border">@@ -176,10 +214,10 @@ export default function CodeAnimationSoundtrack({ jobId, soundtrack, audioUrl, a<p className="text-xs text-gray-500">{progress.statusMsg || (progress.status === 'running' ? 'Rendering…' : 'Waiting in the audio queue…')}</p></div>)}- {progress.status === 'failed' && (+ {mode === 'model' && progress.status === 'failed' && (<p role="alert" className="text-xs text-port-error">Audio failed: {progress.error || 'unknown error'}. Try again, or pick another model.</p>)}- {progress.status === 'canceled' && <p className="text-xs text-gray-500">The audio render was canceled.</p>}+ {mode === 'model' && progress.status === 'canceled' && <p className="text-xs text-gray-500">The audio render was canceled.</p>}{hasTake && !live && (<div className="space-y-1"><audio src={audioUrl} controls preload="metadata" className="w-full" aria-label="Generated soundtrack" />diff --git a/client/src/components/codeAnimation/CodeAnimationSoundtrack.test.jsx b/client/src/components/codeAnimation/CodeAnimationSoundtrack.test.jsxindex 88afd5d04b..799eb94e9d 100644--- a/client/src/components/codeAnimation/CodeAnimationSoundtrack.test.jsx+++ b/client/src/components/codeAnimation/CodeAnimationSoundtrack.test.jsx@@ -7,11 +7,28 @@ vi.mock('../../services/api', () => ({listMusicEngines: vi.fn(),previewMusicPrompt: vi.fn(async ({ prompt }) => ({ sentPrompt: prompt, promptAdditions: [] })),generateCodeAnimationSoundtrack: vi.fn(),+ writeMusicCode: vi.fn(),+ previewMusicCode: vi.fn(async () => ({ sentPrompt: 'Example prompt', promptAdditions: [] })),+ ensureCodeAnimationSoundtrackTrack: vi.fn(),+ renderTrackCode: vi.fn(),}));vi.mock('../../hooks/useMediaJobProgress', () => ({ default: vi.fn(() => progress.current) }));+const providerHook = vi.hoisted(() => ({ setSelectedProviderId: vi.fn(), setSelectedModel: vi.fn() }));+vi.mock('../../hooks/useProviderModels', () => ({+ default: () => ({+ providers: [{ id: 'other', models: ['o'] }, { id: 'job-provider', models: ['m1', 'm2'] }],+ selectedProviderId: 'job-provider', selectedModel: 'm2', availableModels: ['m1', 'm2'], loading: false,+ ...providerHook,+ }),+}));+vi.mock('../ProviderModelSelector', () => ({ default: () => <div data-testid="provider-picker" /> }));+vi.mock('../music/strudelFrame', () => ({+ CODE_FRAME_SOURCE: 'portos-code-frame', STRUDEL_VERSION: '0', TONE_VERSION: '0',+ buildStrudelFrameDoc: () => '<!doctype html><title>stub</title>', buildToneFrameDoc: () => '<!doctype html><title>stub</title>',+}));import CodeAnimationSoundtrack, { SOUNDTRACK_CHOICE_KEY } from './CodeAnimationSoundtrack';-import { generateCodeAnimationSoundtrack, listMusicEngines } from '../../services/api';+import { generateCodeAnimationSoundtrack, listMusicEngines, writeMusicCode } from '../../services/api';const ENGINES = [{ id: 'musicgen', name: 'MusicGen', ready: true, maxDurationSec: 30, models: [{ id: 'mg', name: 'MG' }], defaultModelId: 'mg' },@@ -81,4 +98,29 @@ describe('CodeAnimationSoundtrack', () => {expect(screen.getByLabelText('Describe the music').disabled).toBe(false);expect(screen.queryByLabelText('Generated soundtrack')).toBeNull();});++ it('writes the audio as code from the animation script, remembering the choice', async () => {+ const user = userEvent.setup();+ writeMusicCode.mockResolvedValue({ language: 'strudel', code: 'silence', llm: {} });+ const job = {+ id: JOB_ID, title: 'Boot', providerId: 'job-provider', model: 'm2',+ input: { title: 'Station boot', concept: 'Panels light up one by one', onScreenText: 'SYSTEM ONLINE' },+ };+ render(<CodeAnimationSoundtrack job={job} jobId={JOB_ID} durationSeconds={20} />);+ await user.click(screen.getByRole('button', { name: 'Coding model' }));+ expect(JSON.parse(localStorage.getItem(SOUNDTRACK_CHOICE_KEY))).toMatchObject({ mode: 'code' });+ expect(screen.queryByLabelText('Music model')).toBeNull();+ // The animation's own words, editable before they are sent.+ expect(screen.getByLabelText(/Animation script/).value).toBe('Station boot\n\nPanels light up one by one\n\nOn-screen text:\nSYSTEM ONLINE');+ // Opens on the provider that made the animation.+ expect(providerHook.setSelectedProviderId).toHaveBeenCalledWith('job-provider');+ expect(providerHook.setSelectedModel).toHaveBeenCalledWith('m2');++ await user.type(screen.getByLabelText('Describe the sound (optional)'), 'Whirrs and blips');+ await user.click(screen.getByRole('button', { name: /Write the code/ }));+ await waitFor(() => expect(writeMusicCode).toHaveBeenCalled());+ expect(writeMusicCode.mock.calls[0][0]).toMatchObject({+ description: 'Whirrs and blips', scene: expect.stringContaining('Panels light up'), durationSec: 20, providerId: 'job-provider', model: 'm2',+ });+ });});diff --git a/client/src/components/music/CodePanel.jsx b/client/src/components/music/CodePanel.jsxindex 3b2cddba02..fc62654194 100644--- a/client/src/components/music/CodePanel.jsx+++ b/client/src/components/music/CodePanel.jsx@@ -11,6 +11,10 @@* "Save as take" records the code in the frame for the chosen length and* uploads the WAV into the track's render history (`engine: 'code'`).*+ * A Code Animation soundtrack reuses the panel: `scene` (the animation script)+ * and `fixedLengthSec` (the film's length) join the brief, and `resolveTrackId`+ * creates the soundtrack track on the first save.+ ** The LLM call fires only from the Write/Revise buttons (no cold-bootstrap LLM* calls). The latest code is remembered per viewer in localStorage, tagged* with the language it was written in, so a reload doesn't lose it.@@ -19,12 +23,14 @@import { useEffect, useMemo, useRef, useState } from 'react';import { Code2, Loader2, Play, Save, Square, Wand2 } from 'lucide-react';import toast from '../ui/Toast';+import ModelInputPreview from '../media/ModelInputPreview';+import { useModelInputPreview } from '../../hooks/useModelInputPreview';import useMounted from '../../hooks/useMounted';import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage.js';import { clamp, formatTimecode } from '../../utils/formatters';import SuperColliderStage from './SuperColliderStage';import { FIELD_CLASS, GHOST_BTN, LABEL_CLASS, PRIMARY_BTN } from './designerStyles';-import { renderTrackCode, writeMusicCode } from '../../services/api';+import { previewMusicCode, renderTrackCode, writeMusicCode } from '../../services/api';import {CODE_FRAME_SOURCE, STRUDEL_VERSION, TONE_VERSION, buildStrudelFrameDoc, buildToneFrameDoc,} from './strudelFrame';@@ -67,9 +73,9 @@ const DEFAULT_LANGUAGE = 'strudel';// Only the most recent code (and the language it was written in) is kept —// one key, tagged with its draft track.-const readStoredCode = (trackId) => {+const readStoredCode = (draftId) => {const stored = safeReadJsonStorage(CODE_KEY);- if (!trackId || stored?.trackId !== trackId) return { code: '', language: DEFAULT_LANGUAGE };+ if (!draftId || stored?.trackId !== draftId) return { code: '', language: DEFAULT_LANGUAGE };return {code: typeof stored.code === 'string' ? stored.code : '',language: LANGUAGES[stored.language] ? stored.language : DEFAULT_LANGUAGE,@@ -79,10 +85,12 @@ const readStoredCode = (trackId) => {export default function CodePanel({trackId, disabled = false, description = '', lyrics = '', title = '',providerId, model, effort, providerPicker, onRendered,+ scene = '', fixedLengthSec = null, draftKey = null, resolveTrackId = null,}) {const mountedRef = useMounted();const frameRef = useRef(null);- const initialStored = useMemo(() => readStoredCode(trackId), [trackId]);+ const draftId = draftKey || trackId;+ const initialStored = useMemo(() => readStoredCode(draftId), [draftId]);const [code, setCode] = useState(initialStored.code);const [language, setLanguage] = useState(initialStored.language);const [guidance, setGuidance] = useState('');@@ -97,13 +105,18 @@ export default function CodePanel({const [recordedSec, setRecordedSec] = useState(0);const [takeSec, setTakeSec] = useState(TAKE_SEC.DEFAULT);const saveContextRef = useRef({});- saveContextRef.current = { trackId, description, title, onRendered };+ // The track resolveTrackId created, held across re-renders until the parent passes it down.+ const createdTrackRef = useRef(null);+ saveContextRef.current = { trackId: trackId || createdTrackRef.current, description, title, onRendered };+ const brief = description.trim() || scene.trim();+ const filmTooLong = !!fixedLengthSec && fixedLengthSec > TAKE_SEC.MAX;const busy = !!writing || !!savePhase;const post = (msg) => frameRef.current?.contentWindow?.postMessage(msg, '*');const uploadTake = async (wav, durationSec) => {const { trackId: id, description: prompt, title: takeTitle, onRendered: done } = saveContextRef.current;+ if (!id) return;setSavePhase('uploading');const form = new FormData();form.append('track', new Blob([wav], { type: 'audio/wav' }), 'code-take.wav');@@ -140,7 +153,7 @@ export default function CodePanel({// Subscribed once; uploadTake reads the latest props through saveContextRef.}, []);- const rememberCode = (next, lang = language) => safeWriteJsonStorage(CODE_KEY, { trackId, code: next, language: lang });+ const rememberCode = (next, lang = language) => safeWriteJsonStorage(CODE_KEY, { trackId: draftId, code: next, language: lang });// Switching languages invalidates the code in the box (it's the wrong// dialect for the new frame), so clear it and remount the player fresh.@@ -155,11 +168,13 @@ export default function CodePanel({};const write = async (mode) => {- if (!description.trim()) { toast.error('Write the musical description first'); return; }+ if (!brief) { toast.error('Write the musical description first'); return; }setWriting(mode);const res = await writeMusicCode({- description: description.trim(),+ ...(description.trim() ? { description: description.trim() } : {}),lyrics: lyrics.trim() || undefined,+ ...(scene.trim() ? { scene } : {}),+ ...(fixedLengthSec ? { durationSec: fixedLengthSec } : {}),guidance: guidance.trim() || undefined,...(mode === 'revise' && code.trim() ? { current: code } : {}),language,@@ -185,9 +200,22 @@ export default function CodePanel({setSavePhase((phase) => (phase === 'recording' ? null : phase));};- const save = () => {- if (!code.trim() || !trackId) return;- const seconds = clamp(Number(lengthInput) || TAKE_SEC.DEFAULT, TAKE_SEC.MIN, TAKE_SEC.MAX);+ // A soundtrack's track is created on the first save, so viewing never makes one.+ const ensureTrackId = async () => {+ if (trackId || createdTrackRef.current) return trackId || createdTrackRef.current;+ const id = await resolveTrackId?.().catch((err) => { toast.error(err?.message || 'Could not create the soundtrack track'); return null; });+ if (id) {+ createdTrackRef.current = id;+ saveContextRef.current = { ...saveContextRef.current, trackId: id };+ }+ return id || null;+ };++ const save = async () => {+ if (!code.trim() || filmTooLong) return;+ // Short-circuits so a known track records in the same tick as the tap.+ if (!(trackId || createdTrackRef.current || await ensureTrackId()) || !mountedRef.current) return;+ const seconds = fixedLengthSec || clamp(Number(lengthInput) || TAKE_SEC.DEFAULT, TAKE_SEC.MIN, TAKE_SEC.MAX);setTakeSec(seconds);setFrameError('');setRecordedSec(0);@@ -199,7 +227,12 @@ export default function CodePanel({const playing = frameState === 'playing';const lang = LANGUAGES[language];const serverRendered = !!lang.serverRendered;- const takeLength = clamp(Number(lengthInput) || TAKE_SEC.DEFAULT, TAKE_SEC.MIN, TAKE_SEC.MAX);+ const takeLength = fixedLengthSec || clamp(Number(lengthInput) || TAKE_SEC.DEFAULT, TAKE_SEC.MIN, TAKE_SEC.MAX);+ const canSaveTo = !!trackId || !!resolveTrackId;+ const sentPreview = useModelInputPreview(previewMusicCode, {+ description: description.trim(), lyrics: lyrics.trim(), scene, guidance, language,+ ...(fixedLengthSec ? { durationSec: fixedLengthSec } : {}),+ }, { enabled: !!brief });return (<div className="space-y-4">@@ -237,19 +270,23 @@ export default function CodePanel({className={FIELD_CLASS}/></label>- <label htmlFor="music-code-length" className="block">- <span className={LABEL_CLASS}>{serverRendered ? 'Render length (sec)' : 'Take length (sec)'}</span>- <input- id="music-code-length"- type="number"- min={TAKE_SEC.MIN}- max={TAKE_SEC.MAX}- value={lengthInput}- onChange={(event) => setLengthInput(event.target.value)}- disabled={disabled || busy}- className={`${FIELD_CLASS} sm:w-28`}- />- </label>+ {fixedLengthSec ? (+ <p className="self-end pb-2 text-xs text-gray-500">{fixedLengthSec}s, the film's length</p>+ ) : (+ <label htmlFor="music-code-length" className="block">+ <span className={LABEL_CLASS}>{serverRendered ? 'Render length (sec)' : 'Take length (sec)'}</span>+ <input+ id="music-code-length"+ type="number"+ min={TAKE_SEC.MIN}+ max={TAKE_SEC.MAX}+ value={lengthInput}+ onChange={(event) => setLengthInput(event.target.value)}+ disabled={disabled || busy}+ className={`${FIELD_CLASS} sm:w-28`}+ />+ </label>+ )}</div><div className="rounded border border-port-border bg-port-bg/60 p-3">@@ -258,18 +295,26 @@ export default function CodePanel({</div><div className="flex flex-wrap gap-2">- <button type="button" onClick={() => write('fresh')} disabled={disabled || busy || !description.trim()} className={hasCode ? GHOST_BTN : PRIMARY_BTN}>+ <button type="button" onClick={() => write('fresh')} disabled={disabled || busy || !brief} className={hasCode ? GHOST_BTN : PRIMARY_BTN}>{writing === 'fresh' ? <Loader2 className="h-4 w-4 animate-spin" /> : <Code2 className="h-4 w-4" />}<span>{writing === 'fresh' ? 'Writing…' : hasCode ? 'Write from scratch' : 'Write the code'}</span></button>{hasCode && (- <button type="button" onClick={() => write('revise')} disabled={disabled || busy || !description.trim()} className={PRIMARY_BTN}>+ <button type="button" onClick={() => write('revise')} disabled={disabled || busy || !brief} className={PRIMARY_BTN}>{writing === 'revise' ? <Loader2 className="h-4 w-4 animate-spin" /> : <Wand2 className="h-4 w-4" />}<span>{writing === 'revise' ? 'Revising…' : 'Revise code'}</span></button>)}</div>+ <ModelInputPreview preview={sentPreview} prompt={description.trim()} />++ {filmTooLong && (+ <p role="alert" className="text-xs text-port-warning">+ The film is {fixedLengthSec}s; coded audio renders at most {TAKE_SEC.MAX}s. Use a music model for this film, or shorten it.+ </p>+ )}+<label htmlFor="music-code-editor" className="block"><span className={LABEL_CLASS}>{lang.label} code</span><textarea@@ -297,9 +342,10 @@ export default function CodePanel({code={code}durationSec={takeLength}trackId={trackId}+ resolveTrackId={resolveTrackId ? ensureTrackId : null}description={description}title={title}- disabled={disabled || !!writing}+ disabled={disabled || !!writing || filmTooLong}onRendered={onRendered}/>) : (@@ -324,7 +370,7 @@ export default function CodePanel({<Square className="h-4 w-4" /><span>Stop</span></button>- <button type="button" onClick={save} disabled={disabled || busy || !frameReady || !hasCode || !trackId} className={PRIMARY_BTN}>+ <button type="button" onClick={save} disabled={disabled || busy || !frameReady || !hasCode || !canSaveTo || filmTooLong} className={PRIMARY_BTN}>{savePhase ? <Loader2 className="h-4 w-4 animate-spin" /> : <Save className="h-4 w-4" />}<span>{savePhase === 'recording' ? 'Recording…' : savePhase === 'uploading' ? 'Saving…' : 'Save as take'}</span></button>diff --git a/client/src/components/music/CodePanel.test.jsx b/client/src/components/music/CodePanel.test.jsxindex 2959dfaef6..d47a775b27 100644--- a/client/src/components/music/CodePanel.test.jsx+++ b/client/src/components/music/CodePanel.test.jsx@@ -5,6 +5,7 @@ import * as api from '../../services/api';vi.mock('../../services/api', () => ({writeMusicCode: vi.fn(),+ previewMusicCode: vi.fn(async () => ({ sentPrompt: 'Example prompt', promptAdditions: [] })),renderTrackCode: vi.fn(),getSuperColliderStatus: vi.fn(),setupSuperCollider: vi.fn(),@@ -169,6 +170,30 @@ describe('<CodePanel>', () => {expect(form.get('title')).toBe('Night Drive');});+ it('scores an animation at the film length, creating its soundtrack track on the first save', async () => {+ const resolveTrackId = vi.fn(async () => 'soundtrack-track');+ await mountReady({ trackId: null, draftKey: 'code-animation:job-1', resolveTrackId, description: '', title: '', scene: 'Boot sequence', fixedLengthSec: 20 });+ expect(screen.queryByLabelText('Take length (sec)')).toBeNull();+ fireEvent.click(screen.getByRole('button', { name: /Write the code/ }));+ await waitFor(() => expect(screen.getByLabelText('Strudel code')).toHaveValue(CODE));+ expect(api.writeMusicCode.mock.calls[0][0]).toMatchObject({ scene: 'Boot sequence', durationSec: 20 });+ expect(api.writeMusicCode.mock.calls[0][0]).not.toHaveProperty('description');++ fireEvent.click(screen.getByRole('button', { name: /Save as take/ }));+ await waitFor(() => expect(posted).toHaveBeenLastCalledWith({ type: 'record', code: CODE, seconds: 20 }, '*'));+ await fromFrame({ type: 'recorded', wav: new ArrayBuffer(64), durationSec: 20 });+ await waitFor(() => expect(api.renderTrackCode).toHaveBeenCalled());+ expect(resolveTrackId).toHaveBeenCalledTimes(1);+ expect(api.renderTrackCode.mock.calls[0][0]).toBe('soundtrack-track');+ });++ it('refuses to record a film longer than coded audio can render', async () => {+ window.localStorage.setItem('portos.musicDesigner.strudelCode', JSON.stringify({ trackId: 'job-2', code: CODE }));+ await mountReady({ trackId: null, draftKey: 'job-2', resolveTrackId: vi.fn(), scene: 'Long film', fixedLengthSec: 150 });+ expect(screen.getByRole('alert').textContent).toContain('The film is 150s; coded audio renders at most 120s');+ expect(screen.getByRole('button', { name: /Save as take/ })).toBeDisabled();+ });+it('lets Stop cancel a recording that is waiting on the audio unlock', async () => {window.localStorage.setItem('portos.musicDesigner.strudelCode', JSON.stringify({ trackId: 'track-1', code: CODE }));await mountReady();diff --git a/client/src/components/music/SuperColliderStage.jsx b/client/src/components/music/SuperColliderStage.jsxindex 3d41cd3e67..ac08b9ff08 100644--- a/client/src/components/music/SuperColliderStage.jsx+++ b/client/src/components/music/SuperColliderStage.jsx@@ -31,7 +31,7 @@ import {const LOG_TAIL = 6;export default function SuperColliderStage({- code, durationSec, trackId, description = '', title = '', disabled = false, onRendered,+ code, durationSec, trackId, description = '', title = '', disabled = false, onRendered, resolveTrackId = null,}) {const mountedRef = useMounted();const [status, setStatus] = useState(null); // null = checking@@ -105,9 +105,12 @@ export default function SuperColliderStage({};const save = async () => {- if (!preview || !trackId) return;+ if (!preview) return;setSaving(true);- const res = await saveSuperColliderTake(trackId, {+ const id = trackId || await resolveTrackId?.();+ if (!mountedRef.current) return;+ if (!id) { setSaving(false); return; }+ const res = await saveSuperColliderTake(id, {jobId: job.jobId,...(description.trim() ? { prompt: description.trim() } : {}),...(title.trim() ? { title: title.trim() } : {}),@@ -176,7 +179,7 @@ export default function SuperColliderStage({</button>)}{preview && (- <button type="button" onClick={save} disabled={disabled || saving || !trackId || stale} className={PRIMARY_BTN}>+ <button type="button" onClick={save} disabled={disabled || saving || !(trackId || resolveTrackId) || stale} className={PRIMARY_BTN}>{saving ? <Loader2 className="h-4 w-4 animate-spin" /> : <Save className="h-4 w-4" />}<span>{saving ? 'Saving…' : 'Save as take'}</span></button>diff --git a/client/src/pages/CodeAnimation.jsx b/client/src/pages/CodeAnimation.jsxindex 786534291f..71c6f3af72 100644--- a/client/src/pages/CodeAnimation.jsx+++ b/client/src/pages/CodeAnimation.jsx@@ -833,6 +833,7 @@ function FastCodeAnimation() {)}{preview?.jobId && job?.id === preview.jobId && job.status === 'completed' && (<CodeAnimationSoundtrack+ job={job}jobId={job.id}soundtrack={job.soundtrack}audioUrl={job.audioUrl}diff --git a/client/src/services/apiCodeAnimation.js b/client/src/services/apiCodeAnimation.jsindex 88f6bb309a..fd1da4b03d 100644--- a/client/src/services/apiCodeAnimation.js+++ b/client/src/services/apiCodeAnimation.js@@ -104,3 +104,8 @@ export const generateCodeAnimationSoundtrack = (id, body, options) => request(`/body: JSON.stringify(body),...options,});+// The soundtrack track a coded score saves its take onto (created on first use).+export const ensureCodeAnimationSoundtrackTrack = (id, options) => request(`/code-animation/${encodeURIComponent(id)}/soundtrack/track`, {+ method: 'POST',+ ...options,+});diff --git a/client/src/services/apiMusic.js b/client/src/services/apiMusic.jsindex 991dcc208b..1dcb4b6488 100644--- a/client/src/services/apiMusic.js+++ b/client/src/services/apiMusic.js@@ -53,13 +53,18 @@ export const generateLyrics = (body, requestOptions = {}) => request('/music/lyr// Have the LLM write the piece as Strudel code (the Music Designer's code// engine). Nothing runs server-side; the browser plays it in a sandboxed frame.-// body: { description, lyrics?, guidance?, current? (code to revise),-// language?, providerId?, model?, effort? } → { language, code, llm }+// body: { description, lyrics?, scene? (animation script), durationSec?,+// guidance?, current? (code to revise), language?, providerId?, model?, effort? }+// → { language, code, llm }export const writeMusicCode = (body, requestOptions = {}) => request('/music/code', {method: 'POST',body: JSON.stringify(body),...requestOptions,});+// The prompt writeMusicCode would send; no provider call.+export const previewMusicCode = (body, options = {}) => request('/music/code/preview', {+ method: 'POST', body: JSON.stringify(body), ...options,+});// De-register a user-installed model (id is the HF repo id) → { removed }.export const removeAudioModel = (engine, id, requestOptions = {}) =>diff --git a/docs/API.md b/docs/API.mdindex 917877397f..caf149f934 100644--- a/docs/API.md+++ b/docs/API.md@@ -828,7 +828,7 @@ Every mounted API prefix (see `server/index.js` for the authoritative list). Dom| `/api/sprites` | Sprite catalog / export; `POST /import` is host control (it names a host source tree to copy from) || `/api/threejs-models` | Procedural Three.js models || `/api/film-styles` | Read-only film style grammar catalog: picker list (`GET /`), full grammar record (`GET /:id`) and rendered prompt-section preview (`GET /:id/prompt?parts=motion,camera`) |-| `/api/code-animation` | Code Animation: LLM-written briefs, prompt building (a `baseJobId` forks a finished animation: its HTML rides the prompt as the cut to extend, refused whole above `baseHtmlMax`), persistent jobs gallery with poster thumbnails (`GET`/`POST /jobs/:id/poster`; captured in the jailed preview browser), generated HTML retrieval, frame-exact MP4 export (`POST /:id/export` → HTML-composition media job), a generated soundtrack for a finished animation (`POST /:id/soundtrack` → one local music-engine render at the film length onto a Music-library track that becomes the job audio), the vendored three.js modules the `three` renderer's preview inlines (`GET /vendor/three`; the export stages the same files, hashed, beside the page), portable source download (`GET /:id/package`), and data-only package validation (`POST /packages/validate`; [contract](CODE_ANIMATION_PACKAGES.md)) |+| `/api/code-animation` | Code Animation: LLM-written briefs, prompt building (a `baseJobId` forks a finished animation: its HTML rides the prompt as the cut to extend, refused whole above `baseHtmlMax`), persistent jobs gallery with poster thumbnails (`GET`/`POST /jobs/:id/poster`; captured in the jailed preview browser), generated HTML retrieval, frame-exact MP4 export (`POST /:id/export` → HTML-composition media job), a generated soundtrack for a finished animation (`POST /:id/soundtrack` → one local music-engine render at the film length onto a Music-library track that becomes the job audio; `POST /:id/soundtrack/track` returns that track, created on first use, for a coded score saved with `POST /api/music/code` + the track's code-take routes), the vendored three.js modules the `three` renderer's preview inlines (`GET /vendor/three`; the export stages the same files, hashed, beside the page), portable source download (`GET /:id/package`), and data-only package validation (`POST /packages/validate`; [contract](CODE_ANIMATION_PACKAGES.md)) || `/api/code-animation/execution` | Code Animation contained production execution: platform sandbox and lane readiness (`GET /`), operator-owned tool paths (`PUT /tools`, host control) and the on-demand adversarial containment check (`POST /probe`, host control) ([contract](CODE_ANIMATION_PACKAGES.md#contained-production-execution)) || `/api/image-to-3d` | Image-to-3D conversion || `/api/rigging` | Auto-skin rigging and animation retargeting for image-to-3D models |diff --git a/server/lib/hostControlRoutes.js b/server/lib/hostControlRoutes.jsindex 1ae21b68e6..08da974fe5 100644--- a/server/lib/hostControlRoutes.js+++ b/server/lib/hostControlRoutes.js@@ -605,6 +605,7 @@ export const HOST_CONTROL_ROUTES = Object.freeze(['POST /api/music/code',// Local music generation spawns neural-audio Python on the host (#11077).'POST /api/music/generate',+ 'POST /api/code-animation/:id/soundtrack','POST /api/tracks/:id/waveform/draw','POST /api/music-video/autonomous','POST /api/music-video/:id/autonomous/resume',diff --git a/server/routes/codeAnimation.js b/server/routes/codeAnimation.jsindex 4fb1f7eb68..52a1921530 100644--- a/server/routes/codeAnimation.js+++ b/server/routes/codeAnimation.js@@ -23,7 +23,7 @@import { Router } from 'express';import { soundAssetSchema, stageProductionSoundAsset } from '../services/codeAnimation/soundAssets.js';-import { startCodeAnimationSoundtrack } from '../services/codeAnimation/soundtrack.js';+import { ensureCodeAnimationSoundtrackTrack, startCodeAnimationSoundtrack } from '../services/codeAnimation/soundtrack.js';import { preflightProductionProject } from '../services/codeAnimation/preflight.js';import { z } from 'zod';import { asyncHandler, ServerError } from '../lib/errorHandler.js';@@ -339,6 +339,13 @@ router.post('/:id/soundtrack', asyncHandler(async (req, res) => {res.status(202).json(await startCodeAnimationSoundtrack(id, validateRequest(soundtrackSchema, req.body ?? {})));}));+// The soundtrack track a coded score (written by a coding model, rendered in+// the browser or by SuperCollider) saves its take onto. Created on first use.+router.post('/:id/soundtrack/track', asyncHandler(async (req, res) => {+ const { id } = validateRequest(exportParamsSchema, req.params);+ res.json(await ensureCodeAnimationSoundtrackTrack(id));+}));+// Frame-exact export: the stored HTML renders through the HTML-composition// pipeline (seek each frame, H.264 MP4, Media History) on the media queue.// Progress streams from /api/html-composition/:jobId/events.diff --git a/server/routes/codeAnimation.test.js b/server/routes/codeAnimation.test.jsindex fc6538f307..4c19d5b356 100644--- a/server/routes/codeAnimation.test.js+++ b/server/routes/codeAnimation.test.js@@ -228,6 +228,19 @@ describe('POST /api/code-animation/:id/soundtrack', () => {expect((await request(app).post(`/api/code-animation/${id}/soundtrack`).send(body)).status).toBe(409);expect(queueMusicGeneration).not.toHaveBeenCalled();});++ it('gives a coded score one soundtrack track, created on first use and reused after', async () => {+ codeAnimationRecords.set(id, finished);+ const app = makeApp();+ const first = await request(app).post(`/api/code-animation/${id}/soundtrack/track`);+ expect(first.status).toBe(200);+ expect(first.body).toEqual({ trackId: 'track-soundtrack' });+ expect(codeAnimationRecords.get(id).soundtrack).toMatchObject({ trackId: 'track-soundtrack', mediaJobId: null });+ getTrack.mockResolvedValue({ id: 'track-soundtrack', title: 'Lantern soundtrack' });+ expect((await request(app).post(`/api/code-animation/${id}/soundtrack/track`)).body).toEqual({ trackId: 'track-soundtrack' });+ expect(createTrackWithAlbum).toHaveBeenCalledTimes(1);+ expect(queueMusicGeneration).not.toHaveBeenCalled();+ });});describe('Code Animation portable packages', () => {diff --git a/server/routes/hostControlMediaAuthoring.test.js b/server/routes/hostControlMediaAuthoring.test.jsindex 5c2e8766a6..5652bfa95c 100644--- a/server/routes/hostControlMediaAuthoring.test.js+++ b/server/routes/hostControlMediaAuthoring.test.js@@ -39,6 +39,10 @@ vi.mock('../services/codeAnimation/stages.js', () => ({startProductionStageRun: vi.fn(async () => ({ run: { status: 'running' } })),cancelProductionStageRun: vi.fn(() => ({ cancelled: true })),}));+vi.mock('../services/codeAnimation/soundtrack.js', () => ({+ startCodeAnimationSoundtrack: vi.fn(async () => ({ mediaJobId: 'example-media' })),+ ensureCodeAnimationSoundtrackTrack: vi.fn(async () => ({ trackId: 'example-track' })),+}));vi.mock('../services/codeAnimation/export.js', () => ({startCodeAnimationExport: vi.fn(async () => ({ status: 'queued' })),}));@@ -150,6 +154,7 @@ import { startProductionStageRun } from '../services/codeAnimation/stages.js';import { describeMusic, writeLyrics } from '../services/musicDesigner.js';import { drawWaveSketch, drawWaveSketchForTrack } from '../services/musicWaveform.js';import { writeMusicCode } from '../services/musicCode.js';+import { startCodeAnimationSoundtrack } from '../services/codeAnimation/soundtrack.js';import { startAutonomousVideo, resumeAutonomousVideo } from '../services/musicVideo/autonomousService.js';import { createSession, revokeSessionById } from '../services/auth.js';@@ -171,6 +176,7 @@ const authoring = [['/api/code-animation/brief', { seedIdea: 'Example scene', ...picker }, generateCodeAnimationBrief, 200],['/api/code-animation/generate', { concept: 'Example scene', ...picker }, startCodeAnimationGeneration, 202],[`/api/code-animation/projects/${id}/stage-runs`, {}, startProductionStageRun, 202],+ [`/api/code-animation/${id}/soundtrack`, { prompt: 'Example score', engine: 'example-engine' }, startCodeAnimationSoundtrack, 202],['/api/music/describe', { concept: 'Example song', ...picker }, describeMusic, 200],['/api/music/lyrics', { description: 'Example song', ...picker }, writeLyrics, 200],['/api/music/waveform', { description: 'Example song', ...picker }, drawWaveSketch, 200],diff --git a/server/routes/music.js b/server/routes/music.jsindex dc4c80d717..47d9847542 100644--- a/server/routes/music.js+++ b/server/routes/music.js@@ -46,7 +46,7 @@ import { listEngineModels, addAudioModel, removeAudioModel, isValidRepoId } fromimport { listMusicEngineCapabilities } from '../services/musicEngineCapabilities.js';import { describeMusic, writeLyrics } from '../services/musicDesigner.js';import { drawWaveSketch } from '../services/musicWaveform.js';-import { MUSIC_CODE_LANGUAGES, MUSIC_CODE_MAX, writeMusicCode } from '../services/musicCode.js';+import { MUSIC_CODE_LANGUAGES, MUSIC_CODE_MAX, previewMusicCodePrompt, writeMusicCode } from '../services/musicCode.js';import { PAINTED_CANVAS_LIMITS } from '../lib/paintedCanvas.js';import { startHfDownloadStream } from '../services/hfDownloadStream.js';import { onClientDisconnect, openSseStream } from '../lib/sseDownload.js';@@ -384,15 +384,21 @@ router.post('/waveform', asyncHandler(async (req, res) => {res.json(await drawWaveSketch(body));}));-const codeSchema = z.object({- description: z.string().trim().min(1, 'description is required').max(8000),+const codeShape = z.object({+ description: z.string().trim().max(8000).optional(),lyrics: z.string().trim().max(20000).optional(),+ // The script of an animation the piece scores, and that film's length.+ scene: z.string().max(20000).optional().transform(blankToUndefined),+ durationSec: z.number().int().min(1).max(600).optional(),// The code in the editor, to revise. It is prompt input for an authorized// authoring provider; playback uses the language's separate render boundary.current: z.string().max(MUSIC_CODE_MAX).optional().transform(blankToUndefined),language: z.enum(MUSIC_CODE_LANGUAGES).optional().default('strudel'),...designerPickerShape,}).omit({ template: true });+const codeSchema = codeShape.refine((body) => !!body.description || !!body.scene, {+ message: 'description is required', path: ['description'],+});// POST /api/music/code — have the LLM write the piece in the selected language.// The selected provider or fallback can execute a coding agent on the host;@@ -402,6 +408,11 @@ router.post('/code', asyncHandler(async (req, res) => {res.json(await writeMusicCode(body));}));+// POST /api/music/code/preview — the prompt /code would send; no provider work.+router.post('/code/preview', asyncHandler(async (req, res) => {+ res.json(previewMusicCodePrompt(validateRequest(codeShape, req.body ?? {})));+}));+const captionField = z.string().max(8000).refine((value) => value.trim().length > 0, 'prompt is required');const generateSchema = z.object({diff --git a/server/routes/music.test.js b/server/routes/music.test.jsindex 47b9c611d5..5ff731fc09 100644--- a/server/routes/music.test.js+++ b/server/routes/music.test.js@@ -408,6 +408,24 @@ describe('music routes', () => {expect(args.prompt).toContain('124 BPM');});+ it('scores an animation script at its length, and previews the exact prompt with labeled additions', async () => {+ promptRunner.runPromptThroughProvider.mockResolvedValue({ text: 'silence', model: 'fake-model' });+ const brief = { scene: 'A space station boots up', durationSec: 20, providerId: 'fake-provider' };+ const r = await request(app).post('/api/music/code').send(brief);+ expect(r.status).toBe(200);+ const sent = promptRunner.runPromptThroughProvider.mock.calls[0][0].prompt;+ expect(sent).toContain('A space station boots up');+ expect(sent).toContain('exactly 20 seconds');++ const preview = await request(app).post('/api/music/code/preview').send(brief);+ expect(preview.status).toBe(200);+ expect(preview.body.sentPrompt).toBe(sent);+ expect(preview.body.promptAdditions.map((item) => item.label)).toEqual(['Strudel writing rules', 'Animation script', 'Length']);+ expect(promptRunner.runPromptThroughProvider).toHaveBeenCalledTimes(1);++ expect((await request(app).post('/api/music/code').send({ providerId: 'fake-provider' })).status).toBe(400);+ });+it('writes Tone.js code with its own prompt contract when language is tonejs', async () => {promptRunner.runPromptThroughProvider.mockResolvedValue({text: 'const synth = new Tone.Synth().toDestination();\nTone.getTransport().start();',diff --git a/server/services/codeAnimation/soundtrack.js b/server/services/codeAnimation/soundtrack.jsindex ee8873df82..5db97b9cdd 100644--- a/server/services/codeAnimation/soundtrack.js+++ b/server/services/codeAnimation/soundtrack.js@@ -54,6 +54,39 @@ export async function withGeneratedSoundtrack(job, deps = {}) {};}+async function requireCompletedJob(id) {+ const job = isCodeAnimationJobId(id) ? await getCodeAnimationJobRecord(id) : null;+ if (!job) throw fail(404, 'NOT_FOUND', 'Code Animation job not found');+ if (job.status !== 'completed') throw fail(409, 'NOT_COMPLETED', 'Generate audio once the animation has finished');+ return job;+}++const createSoundtrackTrack = (job, deps) => deps.createTrack({ title: `${job.title || 'Code animation'} soundtrack`.slice(0, 200) });++/**+ * The animation's soundtrack track, created on first use. A coded score+ * (Strudel / Tone.js / SuperCollider) saves its take onto this track, which+ * then becomes the job's audio like a music-model render does.+ */+export async function ensureCodeAnimationSoundtrackTrack(id, overrides = {}) {+ const deps = { ...defaults, ...overrides };+ const job = await requireCompletedJob(id);+ const existing = job.soundtrack?.trackId ? await deps.getTrack(job.soundtrack.trackId).catch(() => null) : null;+ if (existing) return { trackId: existing.id };+ if (starting.has(id)) throw fail(409, 'CODE_ANIMATION_SOUNDTRACK_BUSY', 'A soundtrack for this animation is already starting');+ starting.add(id);+ try {+ const track = await createSoundtrackTrack(job, deps);+ const now = new Date().toISOString();+ const latest = await getCodeAnimationJobRecord(id) || job;+ await saveCodeAnimationJobRecord({ ...latest, soundtrack: { trackId: track.id, mediaJobId: null, requestedAt: now }, updatedAt: now });+ emitCodeAnimationChanged(id);+ return { trackId: track.id };+ } finally {+ starting.delete(id);+ }+}+/*** Queue a soundtrack render for a completed animation. Resolves to the updated* job's `soundtrack` record: `{ trackId, mediaJobId, engine, modelId, prompt,@@ -61,9 +94,7 @@ export async function withGeneratedSoundtrack(job, deps = {}) {*/export async function startCodeAnimationSoundtrack(id, request, overrides = {}) {const deps = { ...defaults, ...overrides };- const job = isCodeAnimationJobId(id) ? await getCodeAnimationJobRecord(id) : null;- if (!job) throw fail(404, 'NOT_FOUND', 'Code Animation job not found');- if (job.status !== 'completed') throw fail(409, 'NOT_COMPLETED', 'Generate audio once the animation has finished');+ const job = await requireCompletedJob(id);if (starting.has(id)) throw fail(409, 'CODE_ANIMATION_SOUNDTRACK_BUSY', 'A soundtrack for this animation is already starting');const engine = await deps.getEngine(request.engine);@@ -83,7 +114,7 @@ export async function startCodeAnimationSoundtrack(id, request, overrides = {})starting.add(id);try {const existing = job.soundtrack?.trackId ? await deps.getTrack(job.soundtrack.trackId).catch(() => null) : null;- const track = existing || await deps.createTrack({ title: `${job.title || 'Code animation'} soundtrack`.slice(0, 200) });+ const track = existing || await createSoundtrackTrack(job, deps);const queued = await deps.queueGeneration({trackId: track.id,prompt: request.prompt,diff --git a/server/services/musicCode.js b/server/services/musicCode.jsindex ae86d394a3..338154f6d9 100644--- a/server/services/musicCode.js+++ b/server/services/musicCode.js@@ -37,19 +37,26 @@ const MAX_DESCRIPTION = 8000;const MAX_LYRICS = 20000;const MAX_GUIDANCE = 4000;+const MAX_SCENE = 20000;+const section = (label, body) => (body ? `\n\n${label}:\n${body}` : '');+// The piece's brief, shared by every language's contract. `scene` is the+// script of an animation the piece scores; `durationSec` is its fixed length.+function briefSections({ description, lyrics, scene, durationSec, guidance, current }) {+ return [+ section('MUSIC TO WRITE', trimTo(description, MAX_DESCRIPTION) || (scene ? '(score the animation below)' : '(none given)')),+ section('ANIMATION THIS SCORES (a soundtrack for the film, not a standalone song: unless the description says otherwise, write sound design synced to what happens on screen, such as whirrs, blips, sweeps, risers and impacts built from synths and noise at the moments they occur, over a musical bed that follows its mood and pacing; nothing is sung)', trimTo(scene, MAX_SCENE)),+ durationSec ? section('LENGTH', `The piece plays for exactly ${durationSec} seconds. Shape intro, build, peak and ending to fit that length, and land the ending at ${durationSec}s.`) : '',+ section('LYRICS / THEME (mood only; nothing is sung, so instruments carry the melody)', trimTo(lyrics, MAX_LYRICS)),+ section('ADDITIONAL GUIDANCE FROM THE USER', trimTo(guidance, MAX_GUIDANCE)),+ current ? section('CURRENT CODE (revise it per the request above; keep what works, change what is asked)', current) : '',+ ].join('');+}+/** The Strudel writing contract sent to the LLM. */-function buildStrudelPrompt({ description, lyrics, guidance, current }) {- return `You are a composer who writes music as Strudel code (the JavaScript port of TidalCycles, strudel.cc). The code runs in a browser with the @strudel/web bundle and is the whole piece: there is no audio model and no DAW.${- section('MUSIC TO WRITE', trimTo(description, MAX_DESCRIPTION) || '(none given)')- }${- section('LYRICS / THEME (mood only; nothing is sung, so instruments carry the melody)', trimTo(lyrics, MAX_LYRICS))- }${- section('ADDITIONAL GUIDANCE FROM THE USER', trimTo(guidance, MAX_GUIDANCE))- }${- current ? section('CURRENT CODE (revise it per the request above; keep what works, change what is asked)', current) : ''- }+function buildStrudelPrompt(input) {+ return `You are a composer who writes music as Strudel code (the JavaScript port of TidalCycles, strudel.cc). The code runs in a browser with the @strudel/web bundle and is the whole piece: there is no audio model and no DAW.${briefSections(input)}Return ONLY the Strudel code: no prose and no markdown fence. It is evaluated exactly like the strudel.cc REPL:- Set the tempo first with setcps(BPM / 60 / 4), e.g. setcps(0.5) for 120 BPM in 4/4.@@ -65,16 +72,8 @@ Compose real music: a memorable melody, a bassline that moves, harmony, and perc}/** The Tone.js writing contract sent to the LLM. */-function buildTonejsPrompt({ description, lyrics, guidance, current }) {- return `You are a composer who writes music as plain JavaScript against the Tone.js library (tonejs.github.io/), running against the global \`Tone\`. The code IS the whole piece: there is no audio model and no DAW, and it runs once per play.${- section('MUSIC TO WRITE', trimTo(description, MAX_DESCRIPTION) || '(none given)')- }${- section('LYRICS / THEME (mood only; nothing is sung, so instruments carry the melody)', trimTo(lyrics, MAX_LYRICS))- }${- section('ADDITIONAL GUIDANCE FROM THE USER', trimTo(guidance, MAX_GUIDANCE))- }${- current ? section('CURRENT CODE (revise it per the request above; keep what works, change what is asked)', current) : ''- }+function buildTonejsPrompt(input) {+ return `You are a composer who writes music as plain JavaScript against the Tone.js library (tonejs.github.io/), running against the global \`Tone\`. The code IS the whole piece: there is no audio model and no DAW, and it runs once per play.${briefSections(input)}Return ONLY the JavaScript: no prose and no markdown fence. Rules for the code:- The host has already unlocked audio and reset the transport before your code runs; do not call Tone.start().@@ -90,16 +89,8 @@ Compose real music: a memorable melody, a bassline that moves, harmony, and perc}/** The SuperCollider writing contract sent to the LLM (rendered offline by a trusted, contained runner). */-function buildSupercolliderPrompt({ description, lyrics, guidance, current }) {- return `You are a composer who writes music as SuperCollider (sclang) code. The code is the whole piece: it is compiled and rendered OFFLINE by a trusted runner in a sandbox, never played live, and there is no audio model and no DAW.${- section('MUSIC TO WRITE', trimTo(description, MAX_DESCRIPTION) || '(none given)')- }${- section('LYRICS / THEME (mood only; nothing is sung, so instruments carry the melody)', trimTo(lyrics, MAX_LYRICS))- }${- section('ADDITIONAL GUIDANCE FROM THE USER', trimTo(guidance, MAX_GUIDANCE))- }${- current ? section('CURRENT CODE (revise it per the request above; keep what works, change what is asked)', current) : ''- }+function buildSupercolliderPrompt(input) {+ return `You are a composer who writes music as SuperCollider (sclang) code. The code is the whole piece: it is compiled and rendered OFFLINE by a trusted runner in a sandbox, never played live, and there is no audio model and no DAW.${briefSections(input)}Return ONLY the sclang code: no prose and no markdown fence. Rules for the code:- Define instruments with SynthDef(\\name, { |out=0, freq=440, amp=0.2, gate=1| ... }) using ONLY stock UGens (SinOsc, Saw, Pulse, LFTri, WhiteNoise, PinkNoise, RLPF, RHPF, LPF, HPF, EnvGen, Env, Pan2, FreeVerb, CombL, Splay, Mix, ...), finish each with Out.ar(out, signal) in stereo, and register it with .add (e.g. SynthDef(\\lead, { ... }).add;). Give every sustained instrument an envelope that frees it (doneAction: 2), gated by gate when it has one.@@ -113,6 +104,33 @@ Compose real music: a memorable melody, a bassline that moves, harmony, and percconst PROMPT_BUILDERS = { strudel: buildStrudelPrompt, tonejs: buildTonejsPrompt, supercollider: buildSupercolliderPrompt };+const LANGUAGE_LABELS = { strudel: 'Strudel', tonejs: 'Tone.js', supercollider: 'SuperCollider' };++/** The exact prompt writeMusicCode sends for `input`. */+function buildMusicCodePrompt({ language = 'strudel', current, ...input } = {}) {+ const buildPrompt = PROMPT_BUILDERS[language] || buildStrudelPrompt;+ return buildPrompt({ ...input, current: trimTo(current, MUSIC_CODE_MAX) });+}++/**+ * What writing the code would send (#11317): the sent prompt, plus everything+ * in it beyond the description as labeled additions. Pure; no provider call.+ */+export function previewMusicCodePrompt(input = {}) {+ const language = PROMPT_BUILDERS[input.language] ? input.language : 'strudel';+ const sentPrompt = buildMusicCodePrompt({ ...input, language });+ const rules = buildMusicCodePrompt({ language }).replace(briefSections({}), '');+ const promptAdditions = [+ ['code-rules', `${LANGUAGE_LABELS[language]} writing rules`, rules],+ ['animation-script', 'Animation script', trimTo(input.scene, MAX_SCENE)],+ ['length', 'Length', input.durationSec ? `${input.durationSec} seconds` : ''],+ ['lyrics', 'Lyrics / theme', trimTo(input.lyrics, MAX_LYRICS)],+ ['guidance', 'Code guidance', trimTo(input.guidance, MAX_GUIDANCE)],+ ['current', 'Current code (revise)', trimTo(input.current, MUSIC_CODE_MAX)],+ ].filter(([, , text]) => text).map(([id, label, text]) => ({ id, label, text }));+ return { sentPrompt, promptAdditions };+}+// A reply usually arrives bare, but a fenced block (with or without prose// around it) is common. Take the first fence's body when there is one.const FENCE_RE = /```[^\n`]*\n([\s\S]*?)\n?```/;@@ -126,16 +144,15 @@ function extractCode(text) {* Ask the chosen provider to write the piece as code. `current` (the code in* the editor) turns the call into a revision of it.*/-export async function writeMusicCode({ description, lyrics, guidance, current, language = 'strudel', providerId, model, effort } = {}) {+export async function writeMusicCode({ language = 'strudel', providerId, model, effort, ...brief } = {}) {const { provider, selectedModel } = await resolveProviderAndModel({ providerId, model });assertProvider(provider, { message: 'No AI provider available to write the music code', code: 'NO_PROVIDER' });- const buildPrompt = PROMPT_BUILDERS[language] || buildStrudelPrompt;const run = await runPromptThroughProvider({provider,model: selectedModel ?? undefined,effort,- prompt: buildPrompt({ description, lyrics, guidance, current: trimTo(current, MUSIC_CODE_MAX) }),+ prompt: buildMusicCodePrompt({ ...brief, language }),source: 'music-code',});