Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { observeModelResource, observeModelMutations } from '../services/modelObservation.js';
/**
* Voice Routes
*
* REST endpoints for voice configuration, profiles, and local voice-stack health.
* Actual audio streaming happens over Socket.IO (see server/sockets/voice.js).
*/
import { Router } from 'express';
import { TTS_ENGINE_IDS } from '../lib/voiceEngines.js';
import { z } from 'zod';
import { asyncHandler, ServerError } from '../lib/errorHandler.js';
import { validateRequest, voicePiperFetchBodySchema } from '../lib/validation.js';
import { MAX_BASE64_UPLOAD_BYTES } from '../lib/uploadLimits.js';
import { getVoiceConfig, updateVoiceConfig } from '../services/voice/config.js';
import { requestHasHostControl } from '../services/authGate.js';
import { checkAll, invalidateHealthCache } from '../services/voice/health.js';
import * as facetimeBridge from '../services/voice/facetimeBridge.js';
import { reconcile, verifyBinaries, verifyModels, downloadPiperVoice, startWhisper, stopWhisper } from '../services/voice/bootstrap.js';
import { synthesize, listVoices, listVoiceEngines, VALID_ENGINES } from '../services/voice/tts.js';
import {
listVoiceProfiles,
listStudioProfiles,
getVoiceProfileRequired,
parsePresetVoiceId,
promotePresetProfile,
createVoiceDesignCandidate,
createClonedVoiceCandidate,
promoteVoiceProfile,
} from '../services/voice/profiles.js';
import { renderProfileBenchmark, benchmarkProfileInteractive, completeProfileInteractiveBenchmark } from '../services/voice/profileBenchmarks.js';
import { getQwen3RuntimeStatus, downloadQwen3Model, DEFAULT_DESIGN_MODEL } from '../services/voice/qwen3TtsRuntime.js';
import {
startFineTuningJob,
listFineTuningJobs,
getFineTuningJobStatus,
cancelFineTuningJob,
promoteCheckpoint,
} from '../services/voice/fineTuning.js';
import { findPiperVoice } from '../services/voice/piper-voices.js';
import { speakProactive, HHMM_RE, MAX_PROACTIVE_TEXT_LEN } from '../services/voice/proactiveSpeech.js';
const router = Router();
router.use(observeModelMutations('voice-readiness'));
const facetimeActionSchema = z.object({}).strict();
const validEngine = (v) => VALID_ENGINES.has(v) ? v : undefined;
const MAX_VOICE_TEXT_LEN = MAX_PROACTIVE_TEXT_LEN;
const voiceConfigPatchSchema = z.object({
enabled: z.boolean().optional(),
trigger: z.enum(['push-to-talk', 'hotword', 'vad']).optional(),
hotkey: z.string().max(32).optional(),
facetime: z.object({
maxCallMinutes: z.number().int().min(1).max(120).optional(),
targetHandle: z.string().trim().max(254).refine((value) => value === '' || /^\+?[1-9]\d{6,14}$/.test(value) || z.string().email().safeParse(value).success, 'Must be an E.164 phone number or email address').optional(),
targetName: z.string().trim().max(120).optional(),
blackHole2chLabel: z.string().trim().min(1).max(120).optional(),
blackHole16chLabel: z.string().trim().min(1).max(120).optional(),
escalateCritical: z.boolean().optional(),
escalateAfterMinutes: z.number().int().min(1).max(1440).optional(),
autoAnswer: z.boolean().optional(),
}).partial().optional(),
stt: z.object({
engine: z.enum(['whisper', 'web-speech']).optional(),
endpoint: z.string().url().optional(),
model: z.string().max(64).optional(),
modelPath: z.string().max(512).optional(),
language: z.string().max(16).optional(),
coreml: z.boolean().optional(),
vocabularyPrompt: z.string().max(4000).optional(),
}).partial().optional(),
tts: z.object({
engine: z.enum([...TTS_ENGINE_IDS, 'kokoro']).optional(),
retiredEngine: z.literal('kokoro').optional(),
rate: z.number().min(0.25).max(4).optional(),
kokoro: z.object({
modelId: z.string().max(128).optional(),
dtype: z.enum(['fp32', 'fp16', 'q8', 'q4', 'q4f16']).optional(),
voice: z.string().max(64).optional(),
}).partial().optional(),
piper: z.object({
voice: z.string().max(128).optional(),
voicePath: z.string().max(512).optional(),
speakerId: z.number().int().nullable().optional(),
}).partial().optional(),
qwen3: z.object({
modelId: z.string().max(128).optional(),
voice: z.string().max(128).optional(),
}).partial().optional(),
}).partial().optional(),
llm: z.object({
provider: z.string().max(80).optional(),
model: z.string().max(128).optional(),
visionModel: z.string().max(128).optional(),
systemPrompt: z.string().max(4000).optional(),
usePersonality: z.boolean().optional(),
personality: z.object({
name: z.string().max(64).optional(),
role: z.string().max(128).optional(),
traits: z.array(z.string().max(64)).max(20).optional(),
speechStyle: z.string().max(256).optional(),
customPrompt: z.string().max(2000).optional(),
}).partial().optional(),
tools: z.object({
enabled: z.boolean().optional(),
maxIterations: z.number().int().min(1).max(10).optional(),
}).partial().optional(),
codeAgent: z.object({
enabled: z.boolean().optional(),
provider: z.string().max(80).optional(),
model: z.string().max(128).optional(),
announceOnComplete: z.boolean().optional(),
}).partial().optional(),
proactive: z.object({
enabled: z.boolean().optional(),
quietHours: z.object({
enabled: z.boolean().optional(),
start: z.string().regex(HHMM_RE).optional(),
end: z.string().regex(HHMM_RE).optional(),
}).partial().optional(),
}).partial().optional(),
fastPath: z.object({
enabled: z.boolean().optional(),
triggers: z.boolean().optional(),
browserLlm: z.boolean().optional(),
browser: z.object({
temperature: z.number().min(0).max(2).optional(),
topK: z.number().int().min(1).max(128).optional(),
}).partial().optional(),
}).partial().optional(),
}).partial().optional(),
vad: z.object({
endOfSpeechMs: z.number().int().min(100).max(5000).optional(),
minUtteranceMs: z.number().int().min(50).max(5000).optional(),
}).partial().optional(),
}).strict();
// GET /api/voice/config — current merged voice settings
router.get('/config', asyncHandler(async (_req, res) => {
res.json(await getVoiceConfig());
}));
// PUT /api/voice/config — deep-merge patch, save, and reconcile PM2 state
router.put('/config', asyncHandler(async (req, res) => {
const parsed = voiceConfigPatchSchema.safeParse(req.body || {});
if (!parsed.success) {
throw new ServerError(
`Invalid voice config: ${parsed.error.issues.map((i) => `${i.path.join('.')} ${i.message}`).join('; ')}`,
{ status: 400, code: 'VALIDATION_ERROR' },
);
}
let next = await updateVoiceConfig(parsed.data);
invalidateHealthCache();
// Reconcile starts/stops whisper-server under PM2 and can run the setup
// script, so it is host work (#13055). hostControlBodyGate already refused a
// caller without host control that changed a runtime key, so skipping it
// leaves nothing that caller changed un-applied.
const reconciliation = requestHasHostControl(req)
? await reconcile(next).catch((err) => ({ error: err.message }))
: { skipped: 'host-control' };
if (next.enabled && next.tts?.retiredEngine && !reconciliation?.skipped && !reconciliation?.error && !reconciliation?.setupRequired) {
next = await updateVoiceConfig({ tts: { retiredEngine: null } });
}
req.app.get('io')?.emit('voice:config:changed', {
enabled: next.enabled,
sttEngine: next.stt?.engine,
sttLanguage: next.stt?.language,
ttsEngine: next.tts?.engine,
ttsVoice: next.tts?.[next.tts?.engine]?.voice,
ttsRate: next.tts?.rate,
hotkey: next.hotkey,
fastPath: next.llm?.fastPath || null,
});
res.json({ config: next, reconciliation });
}));
// GET /api/voice/status — reachability + enabled flag + binary/model presence
const voiceObservation = observeModelResource('voice-readiness', async () => {
const cfg = await getVoiceConfig();
const [services, bins] = await Promise.all([checkAll(cfg), verifyBinaries(cfg)]);
const models = verifyModels(cfg);
return {
enabled: cfg.enabled,
sttEngine: cfg.stt.engine,
ttsEngine: cfg.tts.engine,
services,
binaries: bins,
models,
};
}, 5000);
router.get('/status', asyncHandler(async (_req, res) => {
res.json(await voiceObservation.read());
}));
router.get('/facetime/status', asyncHandler(async (_req, res) => {
res.json(await facetimeBridge.checkSetup());
}));
const facetimeActions = { probe: facetimeBridge.probe, call: facetimeBridge.call, hangup: facetimeBridge.hangup };
for (const command of ['probe', 'call', 'hangup']) {
router.post(`/facetime/${command}`, asyncHandler(async (req, res) => {
validateRequest(facetimeActionSchema, req.body || {});
res.json(await facetimeActions[command]());
}));
}
router.get('/voices', asyncHandler(async (req, res) => {
res.json(await listVoices(validEngine(req.query?.engine)));
}));
const voiceProfileListSchema = z.object({
universeId: z.string().trim().min(1).max(160).optional(),
characterId: z.string().trim().min(1).max(160).optional(),
}).strict();
const promotePresetProfileSchema = z.object({
universeId: z.string().trim().min(1).max(160),
characterId: z.string().trim().min(1).max(160),
characterName: z.string().trim().max(160).optional(),
voiceId: z.string().trim().min(1).max(160),
}).strict();
const voiceDesignCandidateSchema = z.object({
universeId: z.string().trim().min(1).max(160),
characterId: z.string().trim().min(1).max(160),
characterName: z.string().trim().max(160).optional(),
instructions: z.string().trim().max(2000).optional(),
seed: z.number().int().optional(),
modelId: z.string().trim().max(160).optional(),
rate: z.number().min(0.25).max(4).optional(),
}).strict();
const voiceCloneCandidateSchema = z.object({
universeId: z.string().trim().min(1).max(160),
characterId: z.string().trim().min(1).max(160),
characterName: z.string().trim().max(160).optional(),
filename: z.string().trim().min(1).max(160),
audioBase64: z.string().min(1),
transcript: z.string().trim().max(4000).optional(),
performerConsentConfirmed: z.boolean(),
licensePosture: z.string().trim().max(160).optional(),
modelId: z.string().trim().max(160).optional(),
rate: z.number().min(0.25).max(4).optional(),
}).strict();
const promoteProfileSchema = z.object({
routes: z.object({
studio: z.object({ enabled: z.boolean().optional() }).partial().optional(),
interactive: z.object({
enabled: z.boolean().optional(),
maxFirstAudioMs: z.number().min(50).max(5000).optional(),
}).partial().optional(),
}).partial().optional(),
}).strict();
const interactiveBenchmarkSchema = z.object({
maxFirstAudioMs: z.number().min(50).max(5000).optional(),
}).strict();
const interactivePlaybackReceiptSchema = z.object({
benchmarkId: z.string().uuid(),
renderRequestLatencyMs: z.number().min(0).max(3600000),
playbackStartupMs: z.number().min(0).max(30000),
}).strict();
const fineTuneStartSchema = z.object({
epochs: z.number().int().min(1).max(50).optional(),
checkpointInterval: z.number().int().min(10).max(500).optional(),
baseModel: z.string().trim().max(160).optional(),
}).strict();
const fineTuneJobParamsSchema = z.object({
id: z.string().trim().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
// Job ids are minted with randomUUID and are joined into a filesystem path
// when the run is resolved from its on-disk record, so anything that could
// carry a separator or `..` is rejected here.
jobId: z.string().trim().uuid(),
}).strict();
const fineTunePromoteSchema = z.object({
checkpointId: z.string().trim().min(1).max(160),
}).strict();
const downloadModelSchema = z.object({
modelId: z.string().trim().min(1).max(160),
}).strict();
const profileIdParamsSchema = z.object({
id: z.string().trim().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
}).strict();
const studioDesignSchema = z.object({
label: z.string().trim().min(1).max(160),
instructions: z.string().trim().min(1).max(2000),
text: z.string().trim().min(1).max(400),
seed: z.number().int().min(0).max(2147483647).default(42),
rate: z.number().min(0.5).max(2).default(1),
pitchSemitones: z.number().int().min(-12).max(12).default(0),
genSeconds: z.number().min(2).max(6).default(4),
}).strict();
const studioAssignmentSchema = z.object({
enableInteractive: z.boolean().default(false),
universeId: z.string().trim().min(1).max(160),
characterId: z.string().trim().min(1).max(160),
}).strict();
const notifyStudio = req => req.app.get('io')?.emit('voice-studio:changed', {});
// Include legacy character-editor mutations so an open library stays current.
router.use('/profiles', (req, res, next) => {
if (req.method !== 'GET' && req.method !== 'HEAD') res.once('finish', () => {
if (res.statusCode < 400) notifyStudio(req);
});
next();
});
router.get('/studio/profiles', asyncHandler(async (req, res) => {
const filters = validateRequest(z.object({
limit: z.coerce.number().int().min(1).max(100).default(30),
cursor: z.string().max(512).optional(),
}).strict(), req.query);
res.json(await listStudioProfiles(filters));
}));
router.get('/profiles/:id', asyncHandler(async (req, res) => {
const { id } = validateRequest(profileIdParamsSchema, req.params);
res.json({ profile: await getVoiceProfileRequired(id) });
}));
router.get('/studio/status', asyncHandler(async (_req, res) => {
const { getAukStatus } = await import('../services/voice/aukRuntime.js');
res.json(await getAukStatus());
}));
router.post('/studio/setup', asyncHandler(async (req, res) => {
validateRequest(facetimeActionSchema, req.body || {});
const { startAukSetup } = await import('../services/voice/aukRuntime.js');
res.status(202).json(startAukSetup(() => notifyStudio(req)));
}));
router.post('/studio/unload', asyncHandler(async (req, res) => {
validateRequest(facetimeActionSchema, req.body || {});
const { unloadAuk } = await import('../services/voice/aukRuntime.js');
unloadAuk();
notifyStudio(req);
res.json({ ok: true });
}));
router.post('/studio/design', asyncHandler(async (req, res) => {
const body = validateRequest(studioDesignSchema, req.body || {});
const { createStudioVoice } = await import('../services/voice/studio.js');
const profile = await createStudioVoice(body);
notifyStudio(req);
res.status(201).json({ profile });
}));
router.post('/profiles/:id/assign', asyncHandler(async (req, res) => {
const { id } = validateRequest(profileIdParamsSchema, req.params);
const body = validateRequest(studioAssignmentSchema, req.body || {});
const { assignStudioVoice } = await import('../services/voice/studio.js');
const profile = await assignStudioVoice(id, body);
res.json({ profile });
}));
// GET /api/voice/engines
router.get('/engines', asyncHandler(async (_req, res) => {
res.json({ engines: await listVoiceEngines() });
}));
// GET /api/voice/profiles
router.get('/profiles', asyncHandler(async (req, res) => {
const filters = validateRequest(voiceProfileListSchema, req.query || {});
res.json({ profiles: await listVoiceProfiles(filters) });
}));
// POST /api/voice/profiles/preset
router.post('/profiles/preset', asyncHandler(async (req, res) => {
const body = validateRequest(promotePresetProfileSchema, req.body || {});
const cfg = await getVoiceConfig();
const preset = parsePresetVoiceId(body.voiceId);
if (preset?.engine === 'kokoro') {
throw new ServerError('Kokoro has been retired. Select a Piper preset to create a voice profile.', {
status: 409, code: 'VOICE_ENGINE_RETIRED',
});
}
const modelRevision = preset?.engine === 'qwen3-tts' ? DEFAULT_DESIGN_MODEL : `piper:${preset?.voice || ''}`;
const profile = await promotePresetProfile({
...body,
modelRevision,
delivery: { rate: cfg.tts?.rate },
});
res.status(201).json({ profile });
}));
// POST /api/voice/profiles/design — create voice design candidate profile
router.post('/profiles/design', asyncHandler(async (req, res) => {
const body = validateRequest(voiceDesignCandidateSchema, req.body || {});
const profile = await createVoiceDesignCandidate(body);
res.status(201).json({ profile });
}));
// POST /api/voice/profiles/clone — create consented clone candidate profile
router.post('/profiles/clone', asyncHandler(async (req, res) => {
const body = validateRequest(voiceCloneCandidateSchema, req.body || {});
const audioBuffer = Buffer.from(body.audioBase64, 'base64');
if (audioBuffer.length > MAX_BASE64_UPLOAD_BYTES) {
throw new ServerError('FILE_TOO_LARGE', `Audio file exceeds maximum size limit of ${MAX_BASE64_UPLOAD_BYTES} bytes`);
}
const profile = await createClonedVoiceCandidate({
...body,
audioBuffer,
});
res.status(201).json({ profile });
}));
// POST /api/voice/profiles/:id/promote — explicitly promote candidate profile
router.post('/profiles/:id/promote', asyncHandler(async (req, res) => {
const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
const body = validateRequest(promoteProfileSchema, req.body || {});
const profile = await promoteVoiceProfile(profileId, body);
res.json({ profile });
}));
// POST /api/voice/profiles/:id/benchmark
router.post('/profiles/:id/benchmark', asyncHandler(async (req, res) => {
const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
const profile = await renderProfileBenchmark(profileId);
res.json({ profile });
}));
// POST /api/voice/profiles/:id/benchmark-interactive — qualify interactive route
router.post('/profiles/:id/benchmark-interactive', asyncHandler(async (req, res) => {
const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
const body = validateRequest(interactiveBenchmarkSchema, req.body || {});
const benchmark = await benchmarkProfileInteractive(profileId, body);
res.json({ benchmark });
}));
router.post('/profiles/:id/benchmark-interactive/complete', asyncHandler(async (req, res) => {
const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
const body = validateRequest(interactivePlaybackReceiptSchema, req.body || {});
const profile = await completeProfileInteractiveBenchmark(profileId, body);
res.json({ profile });
}));
// Fine-tuning endpoints
router.post('/profiles/:id/fine-tune/start', asyncHandler(async (req, res) => {
const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
const body = validateRequest(fineTuneStartSchema, req.body || {});
const result = await startFineTuningJob({ profileId, ...body });
res.status(202).json(result);
}));
// Newest-first runs from their job.json sidecars, so the Voice Lab recovers a
// run (and its checkpoints) after a reload; live changes arrive on
// `voice:fine-tune:updated`.
router.get('/profiles/:id/fine-tune', asyncHandler(async (req, res) => {
const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
res.json({ jobs: await listFineTuningJobs(profileId) });
}));
router.get('/profiles/:id/fine-tune/:jobId', asyncHandler(async (req, res) => {
const { id: profileId, jobId } = validateRequest(fineTuneJobParamsSchema, req.params);
const result = await getFineTuningJobStatus(jobId, profileId);
res.json(result);
}));
router.post('/profiles/:id/fine-tune/:jobId/cancel', asyncHandler(async (req, res) => {
const { id: profileId, jobId } = validateRequest(fineTuneJobParamsSchema, req.params);
const result = cancelFineTuningJob(jobId, profileId);
res.json(result);
}));
router.post('/profiles/:id/fine-tune/:jobId/promote', asyncHandler(async (req, res) => {
const { id: profileId, jobId } = validateRequest(fineTuneJobParamsSchema, req.params);
const body = validateRequest(fineTunePromoteSchema, req.body || {});
const profile = await promoteCheckpoint({
profileId,
jobId,
checkpointId: body.checkpointId,
});
res.json({ profile });
}));
// Qwen3 runtime status and model management
router.get('/qwen3/status', asyncHandler(async (_req, res) => {
res.json(await getQwen3RuntimeStatus());
}));
router.post('/qwen3/download-model', asyncHandler(async (req, res) => {
const body = validateRequest(downloadModelSchema, req.body || {});
const result = await downloadQwen3Model(body.modelId);
res.json(result);
}));
// POST /api/voice/piper/fetch
router.post('/piper/fetch', asyncHandler(async (req, res) => {
const { voice } = validateRequest(voicePiperFetchBodySchema, req.body ?? {});
if (!findPiperVoice(voice)) {
throw new ServerError(`unknown piper voice: ${voice}`, { status: 400 });
}
const cfg = await getVoiceConfig();
const result = await downloadPiperVoice(voice, cfg);
res.json({ voice, ...result });
}));
// POST /api/voice/test
const voiceTestBodySchema = z.object({
text: z.string().trim().min(1).max(MAX_VOICE_TEXT_LEN),
voice: z.string().trim().default(''),
engine: z.string().trim().default(''),
});
router.post('/test', asyncHandler(async (req, res) => {
const { text, voice: requestedVoice, engine: requestedEngine } = validateRequest(voiceTestBodySchema, req.body ?? {});
const voice = requestedVoice || undefined;
const engine = validEngine(requestedEngine);
const { wav, latencyMs } = await synthesize(text, { voice, engine });
res.setHeader('Content-Type', 'audio/wav');
res.setHeader('X-TTS-Latency-Ms', String(latencyMs));
res.send(wav);
}));
const speakBodySchema = z.object({
text: z.string().trim().min(1).max(MAX_VOICE_TEXT_LEN),
priority: z.enum(['low', 'normal', 'high']).optional(),
source: z.string().max(64).optional()
.transform((s) => {
const t = (s ?? '').trim();
return t === '' ? undefined : t;
}),
});
router.post('/speak', asyncHandler(async (req, res) => {
const parsed = speakBodySchema.safeParse(req.body || {});
if (!parsed.success) {
throw new ServerError(
`Invalid speak payload: ${parsed.error.issues.map((i) => `${i.path.join('.')} ${i.message}`).join('; ')}`,
{ status: 400, code: 'VALIDATION_ERROR' },
);
}
const io = req.app.get('io');
if (!io) {
throw new ServerError(
'voice subsystem misconfigured: io not attached',
{ status: 500, code: 'VOICE_IO_UNAVAILABLE' },
);
}
const result = await speakProactive({ io, ...parsed.data });
res.json(result);
}));
// GET /api/voice/tts/status
router.get('/tts/status', asyncHandler(async (_req, res) => {
res.json({
kokoro: {
state: 'lazy',
retired: true,
loadedKey: null,
},
});
}));
// POST /api/voice/tts/unload
router.post('/tts/unload', asyncHandler(async (_req, res) => {
res.json({ unloaded: false, retired: true });
}));
const whisperActionSchema = z.object({
action: z.enum(['start', 'stop']),
});
router.post('/whisper', asyncHandler(async (req, res) => {
const parsed = whisperActionSchema.safeParse(req.body || {});
if (!parsed.success) {
throw new ServerError(
`Invalid whisper payload: ${parsed.error.issues.map((i) => `${i.path.join('.')} ${i.message}`).join('; ')}`,
{ status: 400, code: 'VALIDATION_ERROR' },
);
}
const { action } = parsed.data;
if (action === 'stop') {
await stopWhisper();
invalidateHealthCache();
return res.json({ success: true, action: 'stop' });
}
const cfg = await getVoiceConfig();
const result = await startWhisper(cfg);
invalidateHealthCache();
res.json({ success: true, action: 'start', ...result });
}));
export default router;