Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Music generation routes (the Music studio's on-device generator surface).
*
* GET /api/music/engines → { engines, defaultEngine }
* POST /api/music/describe → { description, llm }
* POST /api/music/lyrics → { lyrics, llm }
* POST /api/music/waveform → { sketch, llm }
* POST /api/music/code → { language, code, llm }
* POST /api/music/generate → { jobId, position, status }
*
* `describe`/`lyrics` are the Generate tab's stepped designer (#4305): an LLM
* expands a short reference/vibe into a rich conditioning prompt, then writes
* lyrics from it. Both are one-shot, user-triggered calls — the studio never
* fires them on its own (AI Provider Usage Policy). `waveform` is the same
* designer's "Drawn waveform" engine: the LLM paints strokes on a stereo
* spectrogram canvas (server/lib/paintedCanvas.js) that the browser plays directly.
* `code` is its "Code" engine: the LLM writes the piece for a sandboxed browser
* player or contained server renderer. Authoring itself can launch a
* coding CLI/TUI agent (including via fallback), so all four designer routes
* require operator authority through the central host-control gate.
*
* Generation (`queueMusicGeneration`, services/musicGeneration.js — shared with the
* autonomous Music Video run) runs the engine-agnostic `generateMusic` (server/services/pipeline/
* musicGen.js) through the unified audio media-job lane. The completion hook
* lands the WAV in the shared music library and creates or updates the Track,
* independently of whether the requesting browser remains mounted. The pipeline
* audio stage has its own generator routes; this is the studio's standalone path.
*
* Generation is acknowledged immediately and drained by the audio media-job
* lane, so a long render never holds the HTTP request open.
*/
import { Router } from 'express';
import { existsSync } from 'fs';
import { z } from 'zod';
import { failValidation, asyncHandler, ServerError } from '../lib/errorHandler.js';
import { validateRequest } from '../lib/validation.js';
import { createLineReader } from '../lib/streamLines.js';
import { SETUP_IMAGE_VIDEO_SCRIPT, spawnSetupScript, stopSetupScript } from '../lib/setupScriptRunner.js';
import {
ENGINES, composeMusicLyrics, isEngineHealthy, isEnginePlatformSupported,
enginePlatformLabel, resolveEngineVramReadiness, MUSIC_VRAM_READINESS,
formatEngineVramReadinessMessage,
} from '../services/pipeline/musicGen.js';
import { listEngineModels, addAudioModel, removeAudioModel, isValidRepoId } from '../services/audioModels.js';
import { listMusicEngineCapabilities } from '../services/musicEngineCapabilities.js';
import { describeMusic, writeLyrics } from '../services/musicDesigner.js';
import { drawWaveSketch } from '../services/musicWaveform.js';
import { MUSIC_CODE_LANGUAGES, MUSIC_CODE_MAX, previewMusicCodePrompt, writeMusicCode } from '../services/musicCode.js';
import { PAINTED_CANVAS_LIMITS } from '../lib/paintedCanvas.js';
import { startHfDownloadStream } from '../services/hfDownloadStream.js';
import { onClientDisconnect, openSseStream } from '../lib/sseDownload.js';
import { createInstallLogger } from '../lib/installLogger.js';
import { inspectModelCache } from '../lib/hfCache.js';
import { getCudaCapability } from '../lib/cudaCapability.js';
import { federatedMediaAudioProfileSchema } from '../lib/federatedMediaWire.js';
import { composeMusicPrompt, queueMusicGeneration } from '../services/musicGeneration.js';
import { musicDescribeInputSchema, musicLyricsInputSchema, musicDesignerFields } from '../lib/musicDesignerValidation.js';
const router = Router();
// GET /api/music/engines — every selectable backend with its models (shipped +
// user-installed, merged), duration window, lyric capability, auto-duration
// support, and a `ready` flag (the opt-in venv is provisioned). The UI gates its
// Generate affordance + shows the install hint from this.
router.get('/engines', asyncHandler(async (_req, res) => {
res.json(await listMusicEngineCapabilities());
}));
// --- Install music runtime venvs -------------------------------------------
// Mirrors the image/video in-app setup flow: the client opens a POST fetch stream
// and we shell out to the canonical setup script with the selected engine's
// INSTALL_* env var set. Keeping the bash path as the single installer source
// avoids a second Node implementation drifting from scripts/setup-image-video.sh.
const requireMusicEngine = (runtime) => {
const engine = ENGINES[runtime];
if (!engine) {
throw new ServerError(
`Unknown music runtime: ${runtime}. Expected one of: ${Object.keys(ENGINES).join(', ')}`,
{ status: 400, code: 'UNKNOWN_MUSIC_RUNTIME' },
);
}
return engine;
};
const sendRuntimeStatus = async (req, res) => {
const runtime = String(req.query?.runtime || '');
const engine = requireMusicEngine(runtime);
res.json({
runtime: engine.id,
label: engine.name,
installed: await isEngineHealthy(engine.id),
venvPath: engine.resolvePython() || null,
expectedVenvPath: engine.venvDefault,
installEnvVar: engine.installEnv,
});
};
router.get('/setup/runtime-status', asyncHandler(sendRuntimeStatus));
// Backward-compatible read surface for stale clients and status pollers. GET
// reports readiness only; installing is restricted to the POST route below.
router.get('/setup/runtime-install', asyncHandler(sendRuntimeStatus));
const runtimeInstallInFlight = new Map();
router.post('/setup/runtime-install', asyncHandler(async (req, res) => {
const runtime = String(req.query?.runtime || '');
// Validate before flushing SSE headers so callers receive the standard
// { error, code, timestamp } response for a bad runtime.
const engine = requireMusicEngine(runtime);
const { send, safeEnd } = openSseStream(res);
if (runtimeInstallInFlight.has(engine.id)) {
send({ type: 'error', message: `Another ${engine.name} install is already running. Wait for it to finish or restart PortOS.` });
return safeEnd();
}
// Claim synchronously before platform/CUDA probes can yield, so concurrent
// requests cannot both pass the guard while readiness is being checked.
runtimeInstallInFlight.set(engine.id, null);
if (!isEnginePlatformSupported(engine.id)) {
runtimeInstallInFlight.delete(engine.id);
send({ type: 'error', message: `${engine.name} requires ${enginePlatformLabel(engine.id)} and cannot be installed on this host.` });
return safeEnd();
}
if (engine.cudaRequired) {
let cuda;
try {
cuda = await getCudaCapability();
} catch (error) {
runtimeInstallInFlight.delete(engine.id);
throw error;
}
if (cuda.status !== 'available') {
send({ type: 'error', message: cuda.status === 'unknown'
? `${engine.name} cannot be installed because CUDA availability could not be determined.`
: `${engine.name} requires an NVIDIA CUDA GPU and cannot be installed on this host.` });
runtimeInstallInFlight.delete(engine.id);
return safeEnd();
}
const vram = resolveEngineVramReadiness(engine.id, cuda);
if (vram.state !== MUSIC_VRAM_READINESS.SUFFICIENT) {
send({ type: 'error', message: formatEngineVramReadinessMessage(engine.id, vram, 'installed') });
runtimeInstallInFlight.delete(engine.id);
return safeEnd();
}
}
// Force a fresh probe: a cached `true` from before the venv broke would
// short-circuit the very install meant to repair it.
if (await isEngineHealthy(engine.id, { refresh: true })) {
runtimeInstallInFlight.delete(engine.id);
send({ type: 'log', message: `${engine.name} already installed at ${engine.resolvePython() || engine.venvDefault}` });
send({ type: 'complete', message: 'Already installed - nothing to do.' });
return safeEnd();
}
if (!existsSync(SETUP_IMAGE_VIDEO_SCRIPT)) {
runtimeInstallInFlight.delete(engine.id);
send({ type: 'error', message: `Installer script not found at ${SETUP_IMAGE_VIDEO_SCRIPT}` });
return safeEnd();
}
send({ type: 'log', message: `Starting ${engine.name} install.` });
// Server-console visibility for the multi-GB install (start / heartbeat /
// outcome) — the SSE stream otherwise surfaces progress only in the browser.
const installLog = createInstallLogger({ installer: engine.name, target: engine.venvDefault });
const emit = (ev) => { installLog.onEvent(ev); send(ev); };
installLog.start();
const child = spawnSetupScript({ [engine.installEnv]: '1' });
runtimeInstallInFlight.set(engine.id, child);
let finished = false;
// `splitRe: /[\r\n]+/` so a bash/pip/tqdm progress bar that redraws with a
// bare `\r` surfaces each redraw as its own log line; the carry buffer
// stitches a line split across chunk boundaries (flushed on close).
const onLine = (line) => {
const t = line.trimEnd();
if (t) emit({ type: 'log', message: t });
};
const stdoutReader = createLineReader(onLine, { splitRe: /[\r\n]+/ });
const stderrReader = createLineReader(onLine, { splitRe: /[\r\n]+/ });
child.stdout.on('data', stdoutReader.push);
child.stderr.on('data', stderrReader.push);
child.on('error', (err) => {
finished = true;
runtimeInstallInFlight.delete(engine.id);
emit({ type: 'error', message: `Installer failed to spawn: ${err.message}` });
safeEnd();
});
// Async because the completion verdict re-probes the venv. try/catch because
// this runs outside the request lifecycle — an uncaught throw here would take
// the process down instead of reaching the error middleware.
child.on('close', async (code) => {
try {
stdoutReader.flush();
stderrReader.flush();
finished = true;
runtimeInstallInFlight.delete(engine.id);
// The venv just changed on disk, so the cached verdict is stale by
// definition — re-probe rather than trusting it.
const healthy = code === 0 && await isEngineHealthy(engine.id, { refresh: true });
if (healthy) {
emit({ type: 'complete', message: `${engine.name} ready: ${engine.resolvePython() || engine.venvDefault}` });
} else if (code === 0) {
emit({ type: 'error', message: `Installer exited 0 but ${engine.name} is still not available. Check the log above for setup errors.` });
} else {
emit({ type: 'error', message: `Installer exited with code ${code}.` });
}
safeEnd();
} catch (err) {
console.error(`❌ ${engine.name} install completion check failed: ${err.message}`);
emit({ type: 'error', message: `Install completion check failed: ${err.message}` });
safeEnd();
}
});
onClientDisconnect(req, res, () => {
if (finished) return;
installLog.cancel();
stopSetupScript(child);
safeEnd();
});
}));
// --- Install additional audio models from HuggingFace -----------------------
// The shipped per-engine model lists (musicGen.js) cover the common checkpoints;
// these endpoints let a user add more HF repos (e.g. a larger MusicGen, an
// AudioLDM2 variant) using the SAME HF-download path as the image/video model
// installer. A registered model's id is its repo id, which the sidecar passes
// to --model, so it's selectable for generation immediately.
const installSchema = z.object({
engine: z.string().trim().min(1).max(60),
repo: z.string().trim().min(1).max(200),
name: z.string().trim().max(200).optional(),
});
// GET /api/music/models/:engine → the merged shipped+user model list for one
// engine (also exposed via /engines, but handy for a focused refresh).
router.get('/models/:engine', asyncHandler(async (req, res) => {
if (!ENGINES[req.params.engine]) throw new ServerError('Unknown audio engine', { status: 404, code: 'AUDIO_MODEL_UNKNOWN_ENGINE' });
res.json({ models: await listEngineModels(req.params.engine) });
}));
// POST /api/music/models — register the repo, STREAM its HF download as SSE
// (text/event-stream), then ROLL BACK the registration if the download didn't
// actually land. Registering up front (before the stream's `complete` frame)
// means the model is durably persisted by the time the client's post-stream
// `/engines` refresh runs — no race. The rollback (de-register when the cache
// is still empty after a failed/cancelled download) means a typo / private /
// gated / auth-failed repo doesn't linger as a bogus "installed" model. Net:
// a completed install is always registered, a failed one never is.
router.post('/models', asyncHandler(async (req, res) => {
const body = validateRequest(installSchema, req.body ?? {});
if (!ENGINES[body.engine]) throw new ServerError('Unknown audio engine', { status: 400, code: 'AUDIO_MODEL_UNKNOWN_ENGINE' });
// Reject install for engines that can't render a custom HF checkpoint (e.g.
// ACE-Step, which uses a fixed checkpoint_dir) — otherwise a downloaded repo
// would register as selectable but the sidecar would ignore it.
const engine = ENGINES[body.engine];
const fixedModel = engine.fixedModelInstall ? engine.models.find((model) => model.repo === body.repo) || null : null;
if (!engine.customModels && !fixedModel) {
throw new ServerError(`${ENGINES[body.engine].name} does not support custom HuggingFace models`, { status: 400, code: 'AUDIO_MODEL_ENGINE_FIXED' });
}
if (!isValidRepoId(body.repo)) throw new ServerError('Invalid HuggingFace repo id', { status: 400, code: 'AUDIO_MODEL_INVALID_REPO' });
// Register first so it's durable before the stream signals completion.
if (!fixedModel) await addAudioModel({ engine: body.engine, repo: body.repo, name: body.name });
// Hand the response to the shared SSE driver — it owns writeHead/end + the
// in-flight dedupe + client-disconnect kill. Resolves after the stream ends.
// A shipped fixed model may declare `downloadIgnore` to skip repo paths its
// runtime never loads (see MINIMAX_MUSIC3_MODELS) — a user-added repo has no
// such contract, so it always gets the full snapshot.
const downloadTarget = { repo: body.repo, ignore: fixedModel?.downloadIgnore ?? [] };
if (fixedModel?.revision) downloadTarget.revision = fixedModel.revision;
await startHfDownloadStream({
req,
res,
repos: [downloadTarget],
});
// Roll back if the weights aren't actually present now (failed/cancelled
// download) so a bogus repo doesn't persist. Best-effort: a rollback failure
// is logged by the service, not surfaced (the response already closed).
const cacheOptions = fixedModel?.revision ? { revision: fixedModel.revision } : {};
const cached = await inspectModelCache(body.repo, cacheOptions).catch(() => ({ cached: false }));
if (!cached.cached) {
if (!fixedModel) await removeAudioModel({ engine: body.engine, id: body.repo }).catch(() => {});
}
}));
// DELETE /api/music/models/:engine/*id — de-register a user-added model. The id
// is an HF repo id (contains a slash), so it's captured as a named trailing
// wildcard (`*id`, path-to-regexp v8) rather than a single `:id` segment.
// path-to-regexp returns the splat as an array of path segments; rejoin with
// `/` to reconstruct the repo id. Cached weights are left to the HF cache;
// shipped defaults can't be removed here (no-op → 200 {removed:false}).
router.delete('/models/:engine/*id', asyncHandler(async (req, res) => {
const splat = req.params.id;
const id = Array.isArray(splat) ? splat.join('/') : String(splat || '');
const removed = await removeAudioModel({ engine: req.params.engine, id });
res.json({ removed });
}));
// --- Stepped designer (#4305) ----------------------------------------------
// Empty/whitespace picker values normalize to undefined so a blank <select>
// means "use the install's active provider / the provider's default model"
// instead of reaching the runner as a whitespace string (same preprocessing as
// `refinePromptSchema` in routes/mediaJobs.js).
const blankToUndefined = (s) => {
const v = (s ?? '').trim();
return v.length > 0 ? v : undefined;
};
// Shared picker + meta-prompt-override fields for both designer routes. The
// template cap matches the description cap — an override is an instruction
// block, not a document.
const designerPickerShape = {
...musicDesignerFields,
providerId: z.string().max(128).optional().transform(blankToUndefined),
model: z.string().max(256).optional().transform(blankToUndefined),
effort: z.string().max(64).optional().transform(blankToUndefined),
};
// Caps mirror the Generate form's own limits: a concept/description feeds the
// prompt field (≤8000), lyrics feed the lyrics field (≤20000).
const describeSchema = musicDescribeInputSchema.extend(designerPickerShape);
const lyricsSchema = musicLyricsInputSchema.extend(designerPickerShape);
// POST /api/music/describe — expand a short reference/vibe into a rich musical
// description the audio engine can condition on. One explicit user action per
// call; nothing here runs unprompted.
router.post('/describe', asyncHandler(async (req, res) => {
const parsed = describeSchema.safeParse(req.body ?? {});
if (!parsed.success) failValidation(parsed);
const body = parsed.data;
const { description, llm, sentPrompt, promptAdditions } = await describeMusic({
concept: body.concept,
guidance: body.guidance,
template: body.template,
providerId: body.providerId,
model: body.model,
effort: body.effort,
});
res.json({ description, llm, sentPrompt, promptAdditions });
}));
// POST /api/music/lyrics — write original lyrics from that description plus the
// user's extra guidance, in the `[verse]`/`[chorus]` syntax the lyric-aware
// engines expect.
router.post('/lyrics', asyncHandler(async (req, res) => {
const parsed = lyricsSchema.safeParse(req.body ?? {});
if (!parsed.success) failValidation(parsed);
const body = parsed.data;
const { lyrics, llm, sentPrompt, promptAdditions } = await writeLyrics({
description: body.description,
request: body.request,
targetSeconds: body.targetSeconds,
guidance: body.guidance,
template: body.template,
providerId: body.providerId,
model: body.model,
effort: body.effort,
});
res.json({ lyrics, llm, sentPrompt, promptAdditions });
}));
const waveformSchema = z.object({
description: z.string().trim().min(1, 'description is required').max(8000),
lyrics: z.string().trim().max(20000).optional(),
durationSec: z.number().min(PAINTED_CANVAS_LIMITS.DURATION_MIN_SEC).max(PAINTED_CANVAS_LIMITS.DURATION_MAX_SEC).optional(),
// The painting to revise — normalized (and size-bounded) by the service.
current: z.record(z.string(), z.unknown()).optional(),
review: z.boolean().optional(),
// The drawing contract is fixed (no meta-prompt override like describe/lyrics).
...designerPickerShape,
}).omit({ template: true });
// POST /api/music/waveform — stateless: have the LLM PAINT the music on a
// spectrogram canvas (lib/paintedCanvas.js) without storing it. The
// designer draws via POST /api/tracks/:id/waveform/draw, which persists it.
// One explicit user action per call.
router.post('/waveform', asyncHandler(async (req, res) => {
const body = validateRequest(waveformSchema, req.body ?? {});
res.json(await drawWaveSketch(body));
}));
const codeShape = z.object({
description: z.string().trim().max(8000).optional(),
lyrics: z.string().trim().max(20000).optional(),
// The script of an animation the piece scores, and that film's length.
scene: z.string().max(20000).optional().transform(blankToUndefined),
durationSec: z.number().int().min(1).max(600).optional(),
// The code in the editor, to revise. It is prompt input for an authorized
// authoring provider; playback uses the language's separate render boundary.
current: z.string().max(MUSIC_CODE_MAX).optional().transform(blankToUndefined),
language: z.enum(MUSIC_CODE_LANGUAGES).optional().default('strudel'),
...designerPickerShape,
}).omit({ template: true });
const codeSchema = codeShape.refine((body) => !!body.description || !!body.scene, {
message: 'description is required', path: ['description'],
});
// POST /api/music/code — have the LLM write the piece in the selected language.
// The selected provider or fallback can execute a coding agent on the host;
// the central host-control gate requires operator authority.
router.post('/code', asyncHandler(async (req, res) => {
const body = validateRequest(codeSchema, req.body ?? {});
res.json(await writeMusicCode(body));
}));
// POST /api/music/code/preview — the prompt /code would send; no provider work.
router.post('/code/preview', asyncHandler(async (req, res) => {
res.json(previewMusicCodePrompt(validateRequest(codeShape, req.body ?? {})));
}));
const captionField = z.string().max(8000).refine((value) => value.trim().length > 0, 'prompt is required');
const generateSchema = z.object({
prompt: captionField,
// No default: distinguish ABSENT (don't touch the track's lyrics) from a
// present '' (the user cleared them and generated without — persist the clear).
lyrics: z.string().max(20000).optional(),
// A render-level override: ignore even supplied lyrics without clearing the
// track's saved lyric field when the completed audio is attached.
instrumentalOnly: z.boolean().optional().default(false),
engine: z.string().trim().min(1).max(80).optional(),
modelId: z.string().trim().min(1).max(256).optional(),
// The peer record id is machine-local routing intent. It never crosses the
// provider boundary. Free-form text stays local too: remote execution uses a
// fixed-vocabulary instrumental profile rendered by the worker.
// Why audio alone is restricted that way:
// docs/decisions/2026-08-20-federated-visual-prompts.md
mediaProviderPeerId: z.string().uuid().optional(),
remoteMusicProfile: federatedMediaAudioProfileSchema.optional(),
durationSec: z.number().positive().max(600).optional(),
durationMode: z.enum(['auto', 'manual']).optional(),
// Attach the result to an existing track (else a new one is created). The
// title seeds a freshly-created track; ignored when trackId is given.
trackId: z.string().trim().max(80).optional(),
title: z.string().trim().max(200).optional(),
artistId: z.string().trim().max(80).optional().default(''),
artist: z.string().trim().max(120).optional().default(''),
albumId: z.string().trim().max(80).optional().default(''),
}).superRefine((value, ctx) => {
if (!value.mediaProviderPeerId) {
if (value.remoteMusicProfile) {
ctx.addIssue({
code: 'custom',
path: ['remoteMusicProfile'],
message: 'remoteMusicProfile requires a remote media provider',
});
}
return;
}
if (!value.engine) {
ctx.addIssue({ code: 'custom', path: ['engine'], message: 'engine is required for a remote media provider' });
}
if (!value.modelId) {
ctx.addIssue({ code: 'custom', path: ['modelId'], message: 'modelId is required for a remote media provider' });
}
if (!value.remoteMusicProfile) {
ctx.addIssue({ code: 'custom', path: ['remoteMusicProfile'], message: 'remoteMusicProfile is required for a remote media provider' });
}
// Lyrics used to be refused outright here. They now cross to an allowlisted
// peer (ADR docs/decisions/2026-08-22-federated-media-input-assets.md rule 2),
// but only when the resolved capability says the model sings AND the peer's
// build accepts them on the wire — neither of which this schema can see.
// The check moved into the handler, below the capability resolve.
});
// Local caption preview: pure composition, no queue, peer probe or provider work.
router.post('/prompt-preview', asyncHandler(async (req, res) => {
const body = validateRequest(generateSchema, req.body ?? {});
if (body.engine !== undefined && !ENGINES[body.engine]) {
throw new ServerError(`Unknown audio engine: ${body.engine}`, { status: 400, code: 'PIPELINE_MUSIC_UNKNOWN_ENGINE' });
}
res.json({
...composeMusicPrompt(body),
...composeMusicLyrics({ ...body, lyrics: body.instrumentalOnly ? '' : body.lyrics }),
});
}));
router.post('/generate', asyncHandler(async (req, res) => {
const body = validateRequest(generateSchema, req.body ?? {});
res.status(202).json(await queueMusicGeneration(body));
}));
export default router;