Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Generate audio for a finished code animation

Claude0dd3fd2c05fe
diff --git a/client/src/components/codeAnimation/CodeAnimationSoundtrack.jsx b/client/src/components/codeAnimation/CodeAnimationSoundtrack.jsx
new file mode 100644
index 0000000000..c1d25040b2
--- /dev/null
+++ b/client/src/components/codeAnimation/CodeAnimationSoundtrack.jsx
@@ -0,0 +1,184 @@
+import { useEffect, useMemo, useRef, useState } from 'react';
+import { LoaderCircle, Music2 } from 'lucide-react';
+import toast from '../ui/Toast';
+import ModelInputPreview from '../media/ModelInputPreview';
+import useMediaJobProgress from '../../hooks/useMediaJobProgress';
+import { useModelInputPreview } from '../../hooks/useModelInputPreview';
+import { generateCodeAnimationSoundtrack, listMusicEngines, previewMusicPrompt } from '../../services/api';
+import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage';
+
+// The last engine/model/instrumental choice, reused for every animation.
+export const SOUNDTRACK_CHOICE_KEY = 'portos.codeAnimation.soundtrackChoice';
+// Long-form engines first when nothing has been picked yet.
+const PREFERRED_ENGINES = ['acestep15', 'acestep', 'minimax-music3-mlx', 'minimax-music3'];
+
+const inputClass = 'w-full rounded-lg border border-port-border bg-port-bg px-3 py-2 text-sm text-white placeholder:text-gray-600 focus:border-port-accent focus:outline-none';
+const labelClass = 'mb-1 block text-xs text-gray-400';
+const buttonPrimary = 'inline-flex items-center gap-2 rounded-lg bg-port-accent px-3 py-1.5 text-sm text-white hover:opacity-90 disabled:cursor-not-allowed disabled:opacity-40';
+
+const readChoice = () => {
+ const stored = safeReadJsonStorage(SOUNDTRACK_CHOICE_KEY, null);
+ return stored && typeof stored === 'object' ? stored : {};
+};
+
+const fits = (engine, filmSec) => !Number.isFinite(engine.maxDurationSec) || engine.maxDurationSec >= filmSec;
+
+function engineNote(engine, filmSec) {
+ if (!engine.ready) return 'not installed';
+ if (!fits(engine, filmSec)) return `max ${engine.maxDurationSec}s`;
+ return null;
+}
+
+/**
+ * Generate audio for a finished animation: pick a local music model, describe
+ * the score, and render a track at the film's length. The animation is not
+ * re-run; the track becomes its audio for the preview and the MP4 export.
+ */
+export default function CodeAnimationSoundtrack({ jobId, soundtrack, audioUrl, durationSeconds, onReady }) {
+ const filmSec = Math.ceil(Number(durationSeconds) || 0);
+ const [engines, setEngines] = useState(null);
+ const [choice, setChoice] = useState(readChoice);
+ const [prompt, setPrompt] = useState(soundtrack?.prompt || '');
+ const [starting, setStarting] = useState(false);
+ const [mediaJobId, setMediaJobId] = useState(soundtrack?.mediaJobId || null);
+
+ useEffect(() => {
+ setPrompt(soundtrack?.prompt || '');
+ setMediaJobId(soundtrack?.mediaJobId || null);
+ }, [jobId, soundtrack?.mediaJobId]);
+
+ useEffect(() => {
+ listMusicEngines({ silent: true })
+ .then((data) => setEngines(Array.isArray(data?.engines) ? data.engines : []))
+ .catch(() => setEngines([]));
+ }, []);
+
+ const usable = useMemo(() => (engines || []).filter((engine) => engine.ready && fits(engine, filmSec)), [engines, filmSec]);
+ const engine = useMemo(() => {
+ const list = engines || [];
+ return list.find((item) => item.id === choice.engine)
+ || PREFERRED_ENGINES.map((id) => usable.find((item) => item.id === id)).find(Boolean)
+ || usable[0]
+ || list[0]
+ || null;
+ }, [engines, usable, choice.engine]);
+ const models = engine?.models || [];
+ const modelId = models.some((model) => model.id === choice.modelId) ? choice.modelId : (engine?.defaultModelId || models[0]?.id || '');
+ const instrumentalOnly = choice.instrumentalOnly !== false;
+ const engineBlocked = engine ? engineNote(engine, filmSec) : null;
+
+ const updateChoice = (patch) => setChoice((current) => {
+ const next = { ...current, ...patch };
+ safeWriteJsonStorage(SOUNDTRACK_CHOICE_KEY, next);
+ return next;
+ });
+
+ const captionPreview = useModelInputPreview(previewMusicPrompt, {
+ prompt, engine: engine?.id, instrumentalOnly, durationSec: filmSec || undefined,
+ }, { enabled: !!engine && !!prompt.trim() });
+
+ const progress = useMediaJobProgress(mediaJobId, { kind: 'audio' });
+ const live = progress.status === 'queued' || progress.status === 'running' || (mediaJobId && progress.status === 'unknown');
+ // Refetch the animation once the take lands so the preview and export use it.
+ const settledRef = useRef(null);
+ useEffect(() => {
+ if (!mediaJobId || progress.status !== 'completed' || settledRef.current === mediaJobId) return;
+ settledRef.current = mediaJobId;
+ onReady?.();
+ }, [mediaJobId, progress.status, onReady]);
+
+ const canGenerate = !!engine && !engineBlocked && !!prompt.trim() && !starting && !live && filmSec > 0;
+
+ const handleGenerate = async () => {
+ if (!canGenerate) return;
+ setStarting(true);
+ const started = await generateCodeAnimationSoundtrack(jobId, {
+ prompt,
+ engine: engine.id,
+ ...(modelId ? { modelId } : {}),
+ instrumentalOnly,
+ }, { silent: true }).catch((error) => {
+ toast.error(error.message || 'Failed to start the soundtrack');
+ return null;
+ });
+ setStarting(false);
+ if (started?.mediaJobId) setMediaJobId(started.mediaJobId);
+ };
+
+ const percent = Math.round((progress.progress || 0) * 100);
+ const hasTake = !!soundtrack?.trackId && !!audioUrl;
+
+ return (
+ <section className="space-y-3 rounded-lg border border-port-border bg-port-card p-3" aria-labelledby="ca-soundtrack-heading">
+ <h3 id="ca-soundtrack-heading" className="flex items-center gap-2 text-sm font-semibold text-white">
+ <Music2 className="h-4 w-4 text-port-accent" /> Soundtrack
+ </h3>
+ <div className="grid gap-3 sm:grid-cols-2">
+ <div>
+ <label htmlFor="ca-soundtrack-engine" className={labelClass}>Music model</label>
+ <select id="ca-soundtrack-engine" value={engine?.id || ''} disabled={!engines?.length || live}
+ onChange={(event) => updateChoice({ engine: event.target.value, modelId: '' })} className={inputClass}>
+ {engines === null && <option value="">Loading models…</option>}
+ {engines?.length === 0 && <option value="">No music models</option>}
+ {(engines || []).map((item) => {
+ const note = engineNote(item, filmSec);
+ return <option key={item.id} value={item.id}>{item.name}{note ? ` (${note})` : ''}</option>;
+ })}
+ </select>
+ </div>
+ {models.length > 1 && (
+ <div>
+ <label htmlFor="ca-soundtrack-model" className={labelClass}>Checkpoint</label>
+ <select id="ca-soundtrack-model" value={modelId} disabled={live}
+ onChange={(event) => updateChoice({ modelId: event.target.value })} className={inputClass}>
+ {models.map((model) => <option key={model.id} value={model.id}>{model.name || model.id}</option>)}
+ </select>
+ </div>
+ )}
+ </div>
+ {engine && engineBlocked === 'not installed' && (
+ <p className="text-xs text-port-warning">{engine.name} is not installed. Install it in Music › Generate, or pick another model.</p>
+ )}
+ {engine && engineBlocked && engineBlocked !== 'not installed' && (
+ <p className="text-xs text-port-warning">The film is {filmSec}s; {engine.name} renders at most {engine.maxDurationSec}s. Pick a model marked for longer audio.</p>
+ )}
+ <div>
+ <label htmlFor="ca-soundtrack-prompt" className={labelClass}>Describe the music</label>
+ <textarea id="ca-soundtrack-prompt" rows={3} value={prompt} maxLength={8000} disabled={live}
+ onChange={(event) => setPrompt(event.target.value)}
+ placeholder="Warm analog synths, slow build, soft drums from 0:20, gentle fade at the end"
+ className={`${inputClass} resize-y`} />
+ </div>
+ <ModelInputPreview preview={captionPreview} prompt={prompt} />
+ <label className="flex items-center gap-2 text-xs text-gray-300">
+ <input type="checkbox" checked={instrumentalOnly} disabled={live} onChange={(event) => updateChoice({ instrumentalOnly: event.target.checked })} />
+ Instrumental only
+ </label>
+ <div className="flex flex-wrap items-center gap-3">
+ <button type="button" onClick={handleGenerate} disabled={!canGenerate} className={buttonPrimary}>
+ {starting || live ? <LoaderCircle className="h-4 w-4 animate-spin" /> : <Music2 className="h-4 w-4" />}
+ {live ? `Generating audio ${percent}%` : hasTake ? 'Regenerate audio' : 'Generate audio'}
+ </button>
+ <span className="text-xs text-gray-500">{filmSec ? `${filmSec}s, the film's length` : ''}</span>
+ </div>
+ {live && (
+ <div className="space-y-1" role="status" aria-label="Soundtrack progress">
+ <div className="h-1.5 overflow-hidden rounded bg-port-border">
+ <div className="h-full bg-port-accent transition-all" style={{ width: `${percent}%` }} />
+ </div>
+ <p className="text-xs text-gray-500">{progress.statusMsg || (progress.status === 'running' ? 'Rendering…' : 'Waiting in the audio queue…')}</p>
+ </div>
+ )}
+ {progress.status === 'failed' && (
+ <p role="alert" className="text-xs text-port-error">Audio failed: {progress.error || 'unknown error'}. Try again, or pick another model.</p>
+ )}
+ {progress.status === 'canceled' && <p className="text-xs text-gray-500">The audio render was canceled.</p>}
+ {hasTake && !live && (
+ <div className="space-y-1">
+ <audio src={audioUrl} controls preload="metadata" className="w-full" aria-label="Generated soundtrack" />
+ <p className="text-xs text-gray-500">This is the animation&apos;s audio now. Export MP4 includes it.</p>
+ </div>
+ )}
+ </section>
+ );
+}
diff --git a/client/src/components/codeAnimation/CodeAnimationSoundtrack.test.jsx b/client/src/components/codeAnimation/CodeAnimationSoundtrack.test.jsx
new file mode 100644
index 0000000000..aad5e51f08
--- /dev/null
+++ b/client/src/components/codeAnimation/CodeAnimationSoundtrack.test.jsx
@@ -0,0 +1,74 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { render, screen, waitFor } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+
+const progress = vi.hoisted(() => ({ current: { status: 'unknown', progress: 0 } }));
+vi.mock('../../services/api', () => ({
+ listMusicEngines: vi.fn(),
+ previewMusicPrompt: vi.fn(async ({ prompt }) => ({ sentPrompt: prompt, promptAdditions: [] })),
+ generateCodeAnimationSoundtrack: vi.fn(),
+}));
+vi.mock('../../hooks/useMediaJobProgress', () => ({ default: vi.fn(() => progress.current) }));
+
+import CodeAnimationSoundtrack, { SOUNDTRACK_CHOICE_KEY } from './CodeAnimationSoundtrack';
+import { generateCodeAnimationSoundtrack, listMusicEngines } from '../../services/api';
+
+const ENGINES = [
+ { id: 'musicgen', name: 'MusicGen', ready: true, maxDurationSec: 30, models: [{ id: 'mg', name: 'MG' }], defaultModelId: 'mg' },
+ { id: 'audioldm2', name: 'AudioLDM2', ready: false, maxDurationSec: 120, models: [{ id: 'al' }], defaultModelId: 'al' },
+ { id: 'acestep15', name: 'ACE-Step 1.5', ready: true, maxDurationSec: 240, models: [{ id: 'ace' }], defaultModelId: 'ace' },
+];
+const JOB_ID = '00000000-0000-4000-8000-000000000003';
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ localStorage.clear();
+ progress.current = { status: 'unknown', progress: 0 };
+ listMusicEngines.mockResolvedValue({ engines: ENGINES });
+ generateCodeAnimationSoundtrack.mockResolvedValue({ mediaJobId: 'media-audio' });
+});
+
+describe('CodeAnimationSoundtrack', () => {
+ it('generates audio for a finished animation with a remembered model that fits the film', async () => {
+ const user = userEvent.setup();
+ const { unmount } = render(<CodeAnimationSoundtrack jobId={JOB_ID} durationSeconds={120} />);
+ const picker = await screen.findByLabelText('Music model');
+ // A model too short for the film, or not installed, says why in the option.
+ expect(screen.getByRole('option', { name: 'MusicGen (max 30s)' })).toBeTruthy();
+ expect(screen.getByRole('option', { name: 'AudioLDM2 (not installed)' })).toBeTruthy();
+ expect(picker.value).toBe('acestep15');
+
+ await user.selectOptions(picker, 'musicgen');
+ expect(screen.getByText(/The film is 120s; MusicGen renders at most 30s/)).toBeTruthy();
+ await user.type(screen.getByLabelText('Describe the music'), 'Warm synths');
+ expect(screen.getByRole('button', { name: /Generate audio/ }).disabled).toBe(true);
+
+ await user.selectOptions(picker, 'acestep15');
+ await user.click(screen.getByRole('button', { name: /Generate audio/ }));
+ expect(generateCodeAnimationSoundtrack).toHaveBeenCalledWith(JOB_ID,
+ { prompt: 'Warm synths', engine: 'acestep15', modelId: 'ace', instrumentalOnly: true }, { silent: true });
+ expect(JSON.parse(localStorage.getItem(SOUNDTRACK_CHOICE_KEY))).toMatchObject({ engine: 'acestep15' });
+ unmount();
+
+ // The next animation opens on the same choice.
+ localStorage.setItem(SOUNDTRACK_CHOICE_KEY, JSON.stringify({ engine: 'acestep15', instrumentalOnly: false }));
+ render(<CodeAnimationSoundtrack jobId={JOB_ID} durationSeconds={20} />);
+ await waitFor(() => expect(screen.getByLabelText('Music model').value).toBe('acestep15'));
+ expect(screen.getByLabelText('Instrumental only').checked).toBe(false);
+ });
+
+ it('shows render progress, then the take as the animation audio once it lands', async () => {
+ progress.current = { status: 'running', progress: 0.4, statusMsg: 'Rendering the score' };
+ const onReady = vi.fn();
+ const soundtrack = { trackId: 'track-1', mediaJobId: 'media-audio', prompt: 'Warm synths' };
+ const { rerender } = render(<CodeAnimationSoundtrack jobId={JOB_ID} durationSeconds={120} soundtrack={soundtrack} onReady={onReady} />);
+ expect(await screen.findByRole('button', { name: 'Generating audio 40%' })).toBeTruthy();
+ expect(screen.getByText('Rendering the score')).toBeTruthy();
+
+ progress.current = { status: 'completed', progress: 1 };
+ rerender(<CodeAnimationSoundtrack jobId={JOB_ID} durationSeconds={120} soundtrack={soundtrack} onReady={onReady} audioUrl="/data/music/take.wav" />);
+ await waitFor(() => expect(onReady).toHaveBeenCalledTimes(1));
+ expect(screen.getByLabelText('Generated soundtrack').getAttribute('src')).toBe('/data/music/take.wav');
+ expect(screen.getByRole('button', { name: 'Regenerate audio' })).toBeTruthy();
+ });
+});
diff --git a/client/src/pages/CodeAnimation.jsx b/client/src/pages/CodeAnimation.jsx
index fcb557c75c..e0f1a9f1b3 100644
--- a/client/src/pages/CodeAnimation.jsx
+++ b/client/src/pages/CodeAnimation.jsx
@@ -6,6 +6,7 @@ import RouteTabsHeader from '../components/ui/RouteTabsHeader';
import ProviderModelSelector from '../components/ProviderModelSelector';
import AlbumTrackPicker from '../components/music/AlbumTrackPicker';
import CodeAnimationPreview from '../components/codeAnimation/CodeAnimationPreview';
+import CodeAnimationSoundtrack from '../components/codeAnimation/CodeAnimationSoundtrack';
import FilmStylePicker from '../components/codeAnimation/FilmStylePicker';
import GalleryVideoPicker from '../components/videoGen/GalleryVideoPicker';
import ProductionProjects from '../components/codeAnimation/ProductionProjects';
@@ -360,8 +361,13 @@ function FastCodeAnimation() {
if (!next) return;
const requested = jobId;
setJob(next);
- if (next.status === 'completed' && next.html) setPreview({ html: next.html, audioUrl: next.audioUrl, frame: next.frame, jobId: next.id });
- else setPreview(null);
+ // Keep the same preview object when only other job fields changed (a
+ // soundtrack request), so the running preview session is not restarted.
+ if (next.status === 'completed' && next.html) {
+ setPreview((current) => (current?.jobId === next.id && current.html === next.html && current.audioUrl === next.audioUrl
+ ? current
+ : { html: next.html, audioUrl: next.audioUrl, frame: next.frame, jobId: next.id }));
+ } else setPreview(null);
if (locallyStartedJobIdRef.current !== requested
&& hydratedJobIdRef.current !== requested
&& next.input) {
@@ -692,6 +698,7 @@ function FastCodeAnimation() {
<input type="checkbox" checked={draft.soundtrack === 'procedural'} onChange={(event) => update({ soundtrack: event.target.checked ? 'procedural' : 'none' })} />
No track? Have the code compose a procedural soundtrack
</label>
+ <p className="text-xs text-gray-500">Or generate one with a music model once the preview is ready, under Soundtrack below it.</p>
</div>
)}
</div>
@@ -787,6 +794,15 @@ function FastCodeAnimation() {
title={draft.title || draft.concept.slice(0, 40)}
/>
)}
+ {preview?.jobId && job?.id === preview.jobId && job.status === 'completed' && (
+ <CodeAnimationSoundtrack
+ jobId={job.id}
+ soundtrack={job.soundtrack}
+ audioUrl={job.audioUrl}
+ durationSeconds={job.frame?.durationSeconds}
+ onReady={jobResource.refetch}
+ />
+ )}
</section>
</div>
</div>
diff --git a/client/src/services/apiCodeAnimation.js b/client/src/services/apiCodeAnimation.js
index 793a90eaf1..88f6bb309a 100644
--- a/client/src/services/apiCodeAnimation.js
+++ b/client/src/services/apiCodeAnimation.js
@@ -95,3 +95,12 @@ export const acceptCodeAnimationOutput = (id, runId, options) => request(`/code-
// full record carries each native move's "fits content like" examples.
export const listFilmStyles = (options) => request('/film-styles', options);
export const getFilmStyle = (id, options) => request(`/film-styles/${encodeURIComponent(id)}`, options);
+
+// Render a soundtrack for a finished animation with a local music engine,
+// without re-running the animation. body: { prompt, engine, modelId?,
+// instrumentalOnly } → the job's `soundtrack` record (watch `mediaJobId`).
+export const generateCodeAnimationSoundtrack = (id, body, options) => request(`/code-animation/${encodeURIComponent(id)}/soundtrack`, {
+ method: 'POST',
+ body: JSON.stringify(body),
+ ...options,
+});
diff --git a/docs/API.md b/docs/API.md
index 4a9b3d7e4e..36beb4768a 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -828,7 +828,7 @@ Every mounted API prefix (see `server/index.js` for the authoritative list). Dom
| `/api/sprites` | Sprite catalog / export; `POST /import` is host control (it names a host source tree to copy from) |
| `/api/threejs-models` | Procedural Three.js models |
| `/api/film-styles` | Read-only film style grammar catalog: picker list (`GET /`), full grammar record (`GET /:id`) and rendered prompt-section preview (`GET /:id/prompt?parts=motion,camera`) |
-| `/api/code-animation` | Code Animation: LLM-written briefs, prompt building, persistent jobs gallery with poster thumbnails (`GET`/`POST /jobs/:id/poster`; captured in the jailed preview browser), generated HTML retrieval, frame-exact MP4 export (`POST /:id/export` → HTML-composition media job), the vendored three.js modules the `three` renderer's preview inlines (`GET /vendor/three`; the export stages the same files, hashed, beside the page), portable source download (`GET /:id/package`), and data-only package validation (`POST /packages/validate`; [contract](CODE_ANIMATION_PACKAGES.md)) |
+| `/api/code-animation` | Code Animation: LLM-written briefs, prompt building, persistent jobs gallery with poster thumbnails (`GET`/`POST /jobs/:id/poster`; captured in the jailed preview browser), generated HTML retrieval, frame-exact MP4 export (`POST /:id/export` → HTML-composition media job), a generated soundtrack for a finished animation (`POST /:id/soundtrack` → one local music-engine render at the film length onto a Music-library track that becomes the job audio), the vendored three.js modules the `three` renderer's preview inlines (`GET /vendor/three`; the export stages the same files, hashed, beside the page), portable source download (`GET /:id/package`), and data-only package validation (`POST /packages/validate`; [contract](CODE_ANIMATION_PACKAGES.md)) |
| `/api/code-animation/execution` | Code Animation contained production execution: platform sandbox and lane readiness (`GET /`), operator-owned tool paths (`PUT /tools`, host control) and the on-demand adversarial containment check (`POST /probe`, host control) ([contract](CODE_ANIMATION_PACKAGES.md#contained-production-execution)) |
| `/api/image-to-3d` | Image-to-3D conversion |
| `/api/rigging` | Auto-skin rigging and animation retargeting for image-to-3D models |
diff --git a/server/routes/codeAnimation.js b/server/routes/codeAnimation.js
index 3be6a94f7d..2cbfd416e3 100644
--- a/server/routes/codeAnimation.js
+++ b/server/routes/codeAnimation.js
@@ -23,6 +23,7 @@
import { Router } from 'express';
import { soundAssetSchema, stageProductionSoundAsset } from '../services/codeAnimation/soundAssets.js';
+import { startCodeAnimationSoundtrack } from '../services/codeAnimation/soundtrack.js';
import { preflightProductionProject } from '../services/codeAnimation/preflight.js';
import { z } from 'zod';
import { asyncHandler, ServerError } from '../lib/errorHandler.js';
@@ -321,6 +322,21 @@ router.get('/:id/package', asyncHandler(async (req, res) => {
res.json(pkg);
}));
+// Generate (or regenerate) a soundtrack for a finished animation with a local
+// music engine, without re-running the animation code. 202: the render runs on
+// the audio media lane; progress rides audio-gen:* for `mediaJobId`.
+const soundtrackSchema = z.object({
+ prompt: z.string().max(8000).refine((value) => value.trim().length > 0, 'Describe the soundtrack'),
+ engine: z.string().trim().min(1).max(80),
+ modelId: z.string().trim().min(1).max(256).optional(),
+ instrumentalOnly: z.boolean().default(true),
+}).strict();
+
+router.post('/:id/soundtrack', asyncHandler(async (req, res) => {
+ const { id } = validateRequest(exportParamsSchema, req.params);
+ res.status(202).json(await startCodeAnimationSoundtrack(id, validateRequest(soundtrackSchema, req.body ?? {})));
+}));
+
// Frame-exact export: the stored HTML renders through the HTML-composition
// pipeline (seek each frame, H.264 MP4, Media History) on the media queue.
// Progress streams from /api/html-composition/:jobId/events.
diff --git a/server/routes/codeAnimation.test.js b/server/routes/codeAnimation.test.js
index 4e4d0909f9..01d1cf8965 100644
--- a/server/routes/codeAnimation.test.js
+++ b/server/routes/codeAnimation.test.js
@@ -69,7 +69,21 @@ vi.mock('../services/mediaJobQueue/index.js', () => ({
listJobs: vi.fn(() => []),
}));
+vi.mock('../services/pipeline/musicGen.js', async (importOriginal) => ({
+ ...await importOriginal(),
+ isEngineHealthy: vi.fn(async () => true),
+}));
+vi.mock('../services/musicGeneration.js', () => ({
+ queueMusicGeneration: vi.fn(async () => ({ jobId: 'media-audio', position: 1, status: 'queued' })),
+}));
+vi.mock('../services/trackAlbumMembership.js', () => ({
+ createTrackWithAlbum: vi.fn(async (input) => ({ track: { id: 'track-soundtrack', ...input } })),
+}));
+
import { emitCodeAnimationChanged } from '../services/socket.js';
+import { isEngineHealthy } from '../services/pipeline/musicGen.js';
+import { queueMusicGeneration } from '../services/musicGeneration.js';
+import { createTrackWithAlbum } from '../services/trackAlbumMembership.js';
import { PATHS } from '../lib/paths.js';
import { getUniverse } from '../services/universeBuilder/crud.js';
import { getBoard } from '../services/moodBoard/db.js';
@@ -169,6 +183,53 @@ describe('Code Animation fast-export audio notes', () => {
});
});
+describe('POST /api/code-animation/:id/soundtrack', () => {
+ const id = '00000000-0000-4000-8000-000000000002';
+ const finished = { id, status: 'completed', title: 'Lantern', input: { soundtrack: 'none' },
+ frame: { width: 1920, height: 1080, fps: 24, durationSeconds: 120 }, audioUrl: null };
+ const body = { prompt: 'Warm analog synths', engine: 'acestep15' };
+
+ it('renders a film-length track for a finished animation, which then becomes its audio', async () => {
+ codeAnimationRecords.set(id, finished);
+ codeAnimationHtml.set(id, '<!doctype html><html><head></head><body></body></html>');
+ const app = makeApp();
+ const started = await request(app).post(`/api/code-animation/${id}/soundtrack`).send(body);
+ expect(started.status).toBe(202);
+ expect(started.body).toMatchObject({ trackId: 'track-soundtrack', mediaJobId: 'media-audio', engine: 'acestep15', durationSec: 120, instrumentalOnly: true });
+ expect(createTrackWithAlbum).toHaveBeenCalledWith({ title: 'Lantern soundtrack' }, { preserveUnassigned: true });
+ expect(queueMusicGeneration).toHaveBeenCalledWith({ trackId: 'track-soundtrack', prompt: 'Warm analog synths', instrumentalOnly: true, engine: 'acestep15', durationSec: 120 });
+ expect(codeAnimationRecords.get(id).soundtrack).toMatchObject({ trackId: 'track-soundtrack', mediaJobId: 'media-audio' });
+ expect(emitCodeAnimationChanged).toHaveBeenCalledWith(id);
+
+ // Until the render lands the original (silent) audio stands.
+ getTrack.mockResolvedValue({ id: 'track-soundtrack', title: 'Lantern soundtrack' });
+ expect((await request(app).get(`/api/code-animation/generate/${id}`)).body.audioUrl).toBeNull();
+ getTrack.mockResolvedValue({ id: 'track-soundtrack', title: 'Lantern soundtrack', audioFilename: 'take 1.wav', durationSec: 120 });
+ const read = await request(app).get(`/api/code-animation/generate/${id}`);
+ expect(read.body.audioUrl).toBe('/data/music/take%201.wav');
+ expect(read.body.input.audio).toMatchObject({ source: 'track', trackId: 'track-soundtrack' });
+
+ // Regenerating renders a new take on the same track.
+ expect((await request(app).post(`/api/code-animation/${id}/soundtrack`).send({ ...body, prompt: 'Softer' })).status).toBe(202);
+ expect(createTrackWithAlbum).toHaveBeenCalledTimes(1);
+ expect(queueMusicGeneration).toHaveBeenLastCalledWith(expect.objectContaining({ trackId: 'track-soundtrack', prompt: 'Softer' }));
+ });
+
+ it('refuses a model that cannot render the whole film, an uninstalled one, and an unfinished animation', async () => {
+ codeAnimationRecords.set(id, finished);
+ const app = makeApp();
+ const short = await request(app).post(`/api/code-animation/${id}/soundtrack`).send({ ...body, engine: 'musicgen' });
+ expect(short.status).toBe(400);
+ expect(short.body.code).toBe('CODE_ANIMATION_SOUNDTRACK_TOO_LONG');
+ expect(short.body.error).toMatch(/120s.*at most 30s/);
+ isEngineHealthy.mockResolvedValueOnce(false);
+ expect((await request(app).post(`/api/code-animation/${id}/soundtrack`).send(body)).body.code).toBe('CODE_ANIMATION_SOUNDTRACK_ENGINE_NOT_READY');
+ codeAnimationRecords.set(id, { ...finished, status: 'running' });
+ expect((await request(app).post(`/api/code-animation/${id}/soundtrack`).send(body)).status).toBe(409);
+ expect(queueMusicGeneration).not.toHaveBeenCalled();
+ });
+});
+
describe('Code Animation portable packages', () => {
const id = '00000000-0000-4000-8000-000000000001';
const html = '<!doctype html><html><body><script>window.renderFrame = t => t;</script></body></html>\n';
diff --git a/server/services/codeAnimation/export.js b/server/services/codeAnimation/export.js
index b3e1eebc4e..7594b7fbb7 100644
--- a/server/services/codeAnimation/export.js
+++ b/server/services/codeAnimation/export.js
@@ -131,7 +131,9 @@ async function startAdmittedExport(id, deps) {
return getBeatGrid(path).catch(() => null);
},
} = deps;
- const job = isCodeAnimationJobId(id) ? await getCodeAnimationJobRecord(id) : null;
+ const record = isCodeAnimationJobId(id) ? await getCodeAnimationJobRecord(id) : null;
+ // A soundtrack generated after the animation is the audio the export muxes.
+ const job = record ? await (await import('./soundtrack.js')).withGeneratedSoundtrack(record) : null;
if (!job) throw new ServerError('Code Animation job not found', { status: 404, code: 'NOT_FOUND' });
if (job.status !== 'completed') throw new ServerError('Only a completed animation can be exported', { status: 409, code: 'NOT_COMPLETED' });
const { width, height, durationSeconds } = job.frame || {};
diff --git a/server/services/codeAnimation/index.js b/server/services/codeAnimation/index.js
index 8003d2e010..3de28715da 100644
--- a/server/services/codeAnimation/index.js
+++ b/server/services/codeAnimation/index.js
@@ -548,7 +548,8 @@ export async function getCodeAnimationJob(id) {
let job = await reconcileJob(record);
if (job.status !== 'completed') return { ...job, html: null };
try {
- return { ...job, html: await readCodeAnimationHtml(job.id) };
+ const { withGeneratedSoundtrack } = await import('./soundtrack.js');
+ return { ...await withGeneratedSoundtrack(job), html: await readCodeAnimationHtml(job.id) };
} catch (error) {
if (error?.code !== 'CODE_ANIMATION_OUTPUT_MISSING') throw error;
const completedAt = new Date().toISOString();
diff --git a/server/services/codeAnimation/soundtrack.js b/server/services/codeAnimation/soundtrack.js
new file mode 100644
index 0000000000..ee8873df82
--- /dev/null
+++ b/server/services/codeAnimation/soundtrack.js
@@ -0,0 +1,115 @@
+/**
+ * Generated soundtrack for a finished Fast HTML animation. The animation code is
+ * never re-run: one Music-library track is rendered by a local music engine at
+ * the film's length and becomes the job's audio, which the preview and the
+ * frame-exact export (muxed) already read from `job.audioUrl`.
+ *
+ * The track is created before the render so regenerating reuses it (each take
+ * lands as a render on that track) and the Music Studio completion hook files
+ * the audio on it. The job's audio is resolved from the track on read, so no
+ * second completion hook has to order itself after the Music Studio one.
+ */
+
+import { ServerError } from '../../lib/errorHandler.js';
+import { emitCodeAnimationChanged } from '../socket.js';
+import { getCodeAnimationJobRecord, isCodeAnimationJobId, saveCodeAnimationJobRecord } from './jobStore.js';
+
+const fail = (status, code, message, context) => new ServerError(message, { status, code, ...(context ? { context } : {}) });
+
+const defaults = {
+ getEngine: async (id) => (await import('../pipeline/musicGen.js')).ENGINES[id] || null,
+ isEngineHealthy: async (id) => (await import('../pipeline/musicGen.js')).isEngineHealthy(id),
+ getTrack: async (id) => (await import('../tracks/index.js')).getTrack(id),
+ createTrack: async (input) => (await (await import('../trackAlbumMembership.js')).createTrackWithAlbum(input, { preserveUnassigned: true })).track,
+ queueGeneration: async (body) => (await import('../musicGeneration.js')).queueMusicGeneration(body),
+};
+
+// One request per animation at a time; the queue's own per-track duplicate
+// check covers a render that is already queued.
+const starting = new Set();
+
+const trackAudioUrl = (track) => `/data/music/${encodeURIComponent(track.audioFilename)}`;
+
+/** The job with its generated soundtrack (once rendered) as the audio. */
+export async function withGeneratedSoundtrack(job, deps = {}) {
+ const trackId = job?.soundtrack?.trackId;
+ if (!trackId) return job;
+ const { getTrack } = { ...defaults, ...deps };
+ const track = await getTrack(trackId).catch(() => null);
+ if (!track?.audioFilename) return job;
+ return {
+ ...job,
+ audioUrl: trackAudioUrl(track),
+ input: {
+ ...job.input,
+ audio: {
+ source: 'track',
+ trackId: track.id,
+ label: track.title || 'Soundtrack',
+ durationSeconds: track.durationSec ?? null,
+ notes: job.input?.audio?.trackId === track.id ? job.input.audio.notes || '' : '',
+ },
+ soundtrack: 'none',
+ },
+ };
+}
+
+/**
+ * Queue a soundtrack render for a completed animation. Resolves to the updated
+ * job's `soundtrack` record: `{ trackId, mediaJobId, engine, modelId, prompt,
+ * instrumentalOnly, durationSec, requestedAt }`.
+ */
+export async function startCodeAnimationSoundtrack(id, request, overrides = {}) {
+ const deps = { ...defaults, ...overrides };
+ const job = isCodeAnimationJobId(id) ? await getCodeAnimationJobRecord(id) : null;
+ if (!job) throw fail(404, 'NOT_FOUND', 'Code Animation job not found');
+ if (job.status !== 'completed') throw fail(409, 'NOT_COMPLETED', 'Generate audio once the animation has finished');
+ if (starting.has(id)) throw fail(409, 'CODE_ANIMATION_SOUNDTRACK_BUSY', 'A soundtrack for this animation is already starting');
+
+ const engine = await deps.getEngine(request.engine);
+ if (!engine) throw fail(400, 'PIPELINE_MUSIC_UNKNOWN_ENGINE', `Unknown music engine: ${request.engine}`);
+ const filmSec = Math.ceil(Number(job.frame?.durationSeconds) || 0);
+ if (!filmSec) throw fail(409, 'CODE_ANIMATION_NO_DURATION', 'This animation has no recorded duration');
+ if (Number.isFinite(engine.maxDurationSec) && filmSec > engine.maxDurationSec) {
+ throw fail(400, 'CODE_ANIMATION_SOUNDTRACK_TOO_LONG',
+ `The film is ${filmSec}s; ${engine.name} renders at most ${engine.maxDurationSec}s. Pick a model that renders ${filmSec}s or more.`,
+ { filmSec, maxDurationSec: engine.maxDurationSec });
+ }
+ if (!await deps.isEngineHealthy(engine.id)) {
+ throw fail(409, 'CODE_ANIMATION_SOUNDTRACK_ENGINE_NOT_READY', `${engine.name} is not installed yet. Install it in Music › Generate, or pick another model.`);
+ }
+ const durationSec = Math.max(engine.minDurationSec ?? 1, filmSec);
+
+ starting.add(id);
+ try {
+ const existing = job.soundtrack?.trackId ? await deps.getTrack(job.soundtrack.trackId).catch(() => null) : null;
+ const track = existing || await deps.createTrack({ title: `${job.title || 'Code animation'} soundtrack`.slice(0, 200) });
+ const queued = await deps.queueGeneration({
+ trackId: track.id,
+ prompt: request.prompt,
+ instrumentalOnly: request.instrumentalOnly,
+ engine: engine.id,
+ ...(request.modelId ? { modelId: request.modelId } : {}),
+ durationSec,
+ });
+ const now = new Date().toISOString();
+ const soundtrack = {
+ trackId: track.id,
+ mediaJobId: queued.jobId,
+ engine: engine.id,
+ modelId: request.modelId || null,
+ prompt: request.prompt,
+ instrumentalOnly: request.instrumentalOnly,
+ durationSec,
+ requestedAt: now,
+ };
+ // Re-read so a field written while the render was being queued survives.
+ const latest = await getCodeAnimationJobRecord(id) || job;
+ await saveCodeAnimationJobRecord({ ...latest, soundtrack, updatedAt: now });
+ emitCodeAnimationChanged(id);
+ console.log(`🎵 Code animation ${id.slice(0, 8)} soundtrack queued on ${engine.id} (${durationSec}s) as ${queued.jobId}`);
+ return soundtrack;
+ } finally {
+ starting.delete(id);
+ }
+}