Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.0 KB
Raw Blame
import { observeModelResource, observeModelMutations } from '../services/modelObservation.js';
/**
 * Voice Routes
 *
 * REST endpoints for voice configuration, profiles, and local voice-stack health.
 * Actual audio streaming happens over Socket.IO (see server/sockets/voice.js).
 */

import { Router } from 'express';
import { TTS_ENGINE_IDS } from '../lib/voiceEngines.js';
import { z } from 'zod';
import { asyncHandler, ServerError } from '../lib/errorHandler.js';
import { validateRequest, voicePiperFetchBodySchema } from '../lib/validation.js';
import { MAX_BASE64_UPLOAD_BYTES } from '../lib/uploadLimits.js';
import { getVoiceConfig, updateVoiceConfig } from '../services/voice/config.js';
import { requestHasHostControl } from '../services/authGate.js';
import { checkAll, invalidateHealthCache } from '../services/voice/health.js';
import * as facetimeBridge from '../services/voice/facetimeBridge.js';
import { reconcile, verifyBinaries, verifyModels, downloadPiperVoice, startWhisper, stopWhisper } from '../services/voice/bootstrap.js';
import { synthesize, listVoices, listVoiceEngines, VALID_ENGINES } from '../services/voice/tts.js';
import {
  listVoiceProfiles,
  listStudioProfiles,
  getVoiceProfileRequired,
  parsePresetVoiceId,
  promotePresetProfile,
  createVoiceDesignCandidate,
  createClonedVoiceCandidate,
  promoteVoiceProfile,
} from '../services/voice/profiles.js';
import { renderProfileBenchmark, benchmarkProfileInteractive, completeProfileInteractiveBenchmark } from '../services/voice/profileBenchmarks.js';
import { getQwen3RuntimeStatus, downloadQwen3Model, DEFAULT_DESIGN_MODEL } from '../services/voice/qwen3TtsRuntime.js';
import {
  startFineTuningJob,
  listFineTuningJobs,
  getFineTuningJobStatus,
  cancelFineTuningJob,
  promoteCheckpoint,
} from '../services/voice/fineTuning.js';
import { findPiperVoice } from '../services/voice/piper-voices.js';
import { speakProactive, HHMM_RE, MAX_PROACTIVE_TEXT_LEN } from '../services/voice/proactiveSpeech.js';

const router = Router();
router.use(observeModelMutations('voice-readiness'));

const facetimeActionSchema = z.object({}).strict();

const validEngine = (v) => VALID_ENGINES.has(v) ? v : undefined;

const MAX_VOICE_TEXT_LEN = MAX_PROACTIVE_TEXT_LEN;

const voiceConfigPatchSchema = z.object({
  enabled: z.boolean().optional(),
  trigger: z.enum(['push-to-talk', 'hotword', 'vad']).optional(),
  hotkey: z.string().max(32).optional(),
  facetime: z.object({
    maxCallMinutes: z.number().int().min(1).max(120).optional(),
    targetHandle: z.string().trim().max(254).refine((value) => value === '' || /^\+?[1-9]\d{6,14}$/.test(value) || z.string().email().safeParse(value).success, 'Must be an E.164 phone number or email address').optional(),
    targetName: z.string().trim().max(120).optional(),
    blackHole2chLabel: z.string().trim().min(1).max(120).optional(),
    blackHole16chLabel: z.string().trim().min(1).max(120).optional(),
    escalateCritical: z.boolean().optional(),
    escalateAfterMinutes: z.number().int().min(1).max(1440).optional(),
    autoAnswer: z.boolean().optional(),
  }).partial().optional(),
  stt: z.object({
    engine: z.enum(['whisper', 'web-speech']).optional(),
    endpoint: z.string().url().optional(),
    model: z.string().max(64).optional(),
    modelPath: z.string().max(512).optional(),
    language: z.string().max(16).optional(),
    coreml: z.boolean().optional(),
    vocabularyPrompt: z.string().max(4000).optional(),
  }).partial().optional(),
  tts: z.object({
    engine: z.enum([...TTS_ENGINE_IDS, 'kokoro']).optional(),
    retiredEngine: z.literal('kokoro').optional(),
    rate: z.number().min(0.25).max(4).optional(),
    kokoro: z.object({
      modelId: z.string().max(128).optional(),
      dtype: z.enum(['fp32', 'fp16', 'q8', 'q4', 'q4f16']).optional(),
      voice: z.string().max(64).optional(),
    }).partial().optional(),
    piper: z.object({
      voice: z.string().max(128).optional(),
      voicePath: z.string().max(512).optional(),
      speakerId: z.number().int().nullable().optional(),
    }).partial().optional(),
    qwen3: z.object({
      modelId: z.string().max(128).optional(),
      voice: z.string().max(128).optional(),
    }).partial().optional(),
  }).partial().optional(),
  llm: z.object({
    provider: z.string().max(80).optional(),
    model: z.string().max(128).optional(),
    visionModel: z.string().max(128).optional(),
    systemPrompt: z.string().max(4000).optional(),
    usePersonality: z.boolean().optional(),
    personality: z.object({
      name: z.string().max(64).optional(),
      role: z.string().max(128).optional(),
      traits: z.array(z.string().max(64)).max(20).optional(),
      speechStyle: z.string().max(256).optional(),
      customPrompt: z.string().max(2000).optional(),
    }).partial().optional(),
    tools: z.object({
      enabled: z.boolean().optional(),
      maxIterations: z.number().int().min(1).max(10).optional(),
    }).partial().optional(),
    codeAgent: z.object({
      enabled: z.boolean().optional(),
      provider: z.string().max(80).optional(),
      model: z.string().max(128).optional(),
      announceOnComplete: z.boolean().optional(),
    }).partial().optional(),
    proactive: z.object({
      enabled: z.boolean().optional(),
      quietHours: z.object({
        enabled: z.boolean().optional(),
        start: z.string().regex(HHMM_RE).optional(),
        end: z.string().regex(HHMM_RE).optional(),
      }).partial().optional(),
    }).partial().optional(),
    fastPath: z.object({
      enabled: z.boolean().optional(),
      triggers: z.boolean().optional(),
      browserLlm: z.boolean().optional(),
      browser: z.object({
        temperature: z.number().min(0).max(2).optional(),
        topK: z.number().int().min(1).max(128).optional(),
      }).partial().optional(),
    }).partial().optional(),
  }).partial().optional(),
  vad: z.object({
    endOfSpeechMs: z.number().int().min(100).max(5000).optional(),
    minUtteranceMs: z.number().int().min(50).max(5000).optional(),
  }).partial().optional(),
}).strict();

// GET /api/voice/config — current merged voice settings
router.get('/config', asyncHandler(async (_req, res) => {
  res.json(await getVoiceConfig());
}));

// PUT /api/voice/config — deep-merge patch, save, and reconcile PM2 state
router.put('/config', asyncHandler(async (req, res) => {
  const parsed = voiceConfigPatchSchema.safeParse(req.body || {});
  if (!parsed.success) {
    throw new ServerError(
      `Invalid voice config: ${parsed.error.issues.map((i) => `${i.path.join('.')} ${i.message}`).join('; ')}`,
      { status: 400, code: 'VALIDATION_ERROR' },
    );
  }
  let next = await updateVoiceConfig(parsed.data);
  invalidateHealthCache();
  // Reconcile starts/stops whisper-server under PM2 and can run the setup
  // script, so it is host work (#13055). hostControlBodyGate already refused a
  // caller without host control that changed a runtime key, so skipping it
  // leaves nothing that caller changed un-applied.
  const reconciliation = requestHasHostControl(req)
    ? await reconcile(next).catch((err) => ({ error: err.message }))
    : { skipped: 'host-control' };
  if (next.enabled && next.tts?.retiredEngine && !reconciliation?.skipped && !reconciliation?.error && !reconciliation?.setupRequired) {
    next = await updateVoiceConfig({ tts: { retiredEngine: null } });
  }
  req.app.get('io')?.emit('voice:config:changed', {
    enabled: next.enabled,
    sttEngine: next.stt?.engine,
    sttLanguage: next.stt?.language,
    ttsEngine: next.tts?.engine,
    ttsVoice: next.tts?.[next.tts?.engine]?.voice,
    ttsRate: next.tts?.rate,
    hotkey: next.hotkey,
    fastPath: next.llm?.fastPath || null,
  });
  res.json({ config: next, reconciliation });
}));

// GET /api/voice/status — reachability + enabled flag + binary/model presence
const voiceObservation = observeModelResource('voice-readiness', async () => {
  const cfg = await getVoiceConfig();
  const [services, bins] = await Promise.all([checkAll(cfg), verifyBinaries(cfg)]);
  const models = verifyModels(cfg);
  return {
    enabled: cfg.enabled,
    sttEngine: cfg.stt.engine,
    ttsEngine: cfg.tts.engine,
    services,
    binaries: bins,
    models,
  };
}, 5000);
router.get('/status', asyncHandler(async (_req, res) => {
  res.json(await voiceObservation.read());
}));

router.get('/facetime/status', asyncHandler(async (_req, res) => {
  res.json(await facetimeBridge.checkSetup());
}));

const facetimeActions = { probe: facetimeBridge.probe, call: facetimeBridge.call, hangup: facetimeBridge.hangup };
for (const command of ['probe', 'call', 'hangup']) {
  router.post(`/facetime/${command}`, asyncHandler(async (req, res) => {
    validateRequest(facetimeActionSchema, req.body || {});
    res.json(await facetimeActions[command]());
  }));
}

router.get('/voices', asyncHandler(async (req, res) => {
  res.json(await listVoices(validEngine(req.query?.engine)));
}));

const voiceProfileListSchema = z.object({
  universeId: z.string().trim().min(1).max(160).optional(),
  characterId: z.string().trim().min(1).max(160).optional(),
}).strict();

const promotePresetProfileSchema = z.object({
  universeId: z.string().trim().min(1).max(160),
  characterId: z.string().trim().min(1).max(160),
  characterName: z.string().trim().max(160).optional(),
  voiceId: z.string().trim().min(1).max(160),
}).strict();

const voiceDesignCandidateSchema = z.object({
  universeId: z.string().trim().min(1).max(160),
  characterId: z.string().trim().min(1).max(160),
  characterName: z.string().trim().max(160).optional(),
  instructions: z.string().trim().max(2000).optional(),
  seed: z.number().int().optional(),
  modelId: z.string().trim().max(160).optional(),
  rate: z.number().min(0.25).max(4).optional(),
}).strict();

const voiceCloneCandidateSchema = z.object({
  universeId: z.string().trim().min(1).max(160),
  characterId: z.string().trim().min(1).max(160),
  characterName: z.string().trim().max(160).optional(),
  filename: z.string().trim().min(1).max(160),
  audioBase64: z.string().min(1),
  transcript: z.string().trim().max(4000).optional(),
  performerConsentConfirmed: z.boolean(),
  licensePosture: z.string().trim().max(160).optional(),
  modelId: z.string().trim().max(160).optional(),
  rate: z.number().min(0.25).max(4).optional(),
}).strict();

const promoteProfileSchema = z.object({
  routes: z.object({
    studio: z.object({ enabled: z.boolean().optional() }).partial().optional(),
    interactive: z.object({
      enabled: z.boolean().optional(),
      maxFirstAudioMs: z.number().min(50).max(5000).optional(),
    }).partial().optional(),
  }).partial().optional(),
}).strict();

const interactiveBenchmarkSchema = z.object({
  maxFirstAudioMs: z.number().min(50).max(5000).optional(),
}).strict();

const interactivePlaybackReceiptSchema = z.object({
  benchmarkId: z.string().uuid(),
  renderRequestLatencyMs: z.number().min(0).max(3600000),
  playbackStartupMs: z.number().min(0).max(30000),
}).strict();

const fineTuneStartSchema = z.object({
  epochs: z.number().int().min(1).max(50).optional(),
  checkpointInterval: z.number().int().min(10).max(500).optional(),
  baseModel: z.string().trim().max(160).optional(),
}).strict();

const fineTuneJobParamsSchema = z.object({
  id: z.string().trim().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
  // Job ids are minted with randomUUID and are joined into a filesystem path
  // when the run is resolved from its on-disk record, so anything that could
  // carry a separator or `..` is rejected here.
  jobId: z.string().trim().uuid(),
}).strict();

const fineTunePromoteSchema = z.object({
  checkpointId: z.string().trim().min(1).max(160),
}).strict();

const downloadModelSchema = z.object({
  modelId: z.string().trim().min(1).max(160),
}).strict();

const profileIdParamsSchema = z.object({
  id: z.string().trim().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
}).strict();

const studioDesignSchema = z.object({
  label: z.string().trim().min(1).max(160),
  instructions: z.string().trim().min(1).max(2000),
  text: z.string().trim().min(1).max(400),
  seed: z.number().int().min(0).max(2147483647).default(42),
  rate: z.number().min(0.5).max(2).default(1),
  pitchSemitones: z.number().int().min(-12).max(12).default(0),
  genSeconds: z.number().min(2).max(6).default(4),
}).strict();
const studioAssignmentSchema = z.object({
  enableInteractive: z.boolean().default(false),
  universeId: z.string().trim().min(1).max(160),
  characterId: z.string().trim().min(1).max(160),
}).strict();
const notifyStudio = req => req.app.get('io')?.emit('voice-studio:changed', {});
// Include legacy character-editor mutations so an open library stays current.
router.use('/profiles', (req, res, next) => {
  if (req.method !== 'GET' && req.method !== 'HEAD') res.once('finish', () => {
    if (res.statusCode < 400) notifyStudio(req);
  });
  next();
});

router.get('/studio/profiles', asyncHandler(async (req, res) => {
  const filters = validateRequest(z.object({
    limit: z.coerce.number().int().min(1).max(100).default(30),
    cursor: z.string().max(512).optional(),
  }).strict(), req.query);
  res.json(await listStudioProfiles(filters));
}));
router.get('/profiles/:id', asyncHandler(async (req, res) => {
  const { id } = validateRequest(profileIdParamsSchema, req.params);
  res.json({ profile: await getVoiceProfileRequired(id) });
}));
router.get('/studio/status', asyncHandler(async (_req, res) => {
  const { getAukStatus } = await import('../services/voice/aukRuntime.js');
  res.json(await getAukStatus());
}));
router.post('/studio/setup', asyncHandler(async (req, res) => {
  validateRequest(facetimeActionSchema, req.body || {});
  const { startAukSetup } = await import('../services/voice/aukRuntime.js');
  res.status(202).json(startAukSetup(() => notifyStudio(req)));
}));
router.post('/studio/unload', asyncHandler(async (req, res) => {
  validateRequest(facetimeActionSchema, req.body || {});
  const { unloadAuk } = await import('../services/voice/aukRuntime.js');
  unloadAuk();
  notifyStudio(req);
  res.json({ ok: true });
}));
router.post('/studio/design', asyncHandler(async (req, res) => {
  const body = validateRequest(studioDesignSchema, req.body || {});
  const { createStudioVoice } = await import('../services/voice/studio.js');
  const profile = await createStudioVoice(body);
  notifyStudio(req);
  res.status(201).json({ profile });
}));
router.post('/profiles/:id/assign', asyncHandler(async (req, res) => {
  const { id } = validateRequest(profileIdParamsSchema, req.params);
  const body = validateRequest(studioAssignmentSchema, req.body || {});
  const { assignStudioVoice } = await import('../services/voice/studio.js');
  const profile = await assignStudioVoice(id, body);
  res.json({ profile });
}));

// GET /api/voice/engines
router.get('/engines', asyncHandler(async (_req, res) => {
  res.json({ engines: await listVoiceEngines() });
}));

// GET /api/voice/profiles
router.get('/profiles', asyncHandler(async (req, res) => {
  const filters = validateRequest(voiceProfileListSchema, req.query || {});
  res.json({ profiles: await listVoiceProfiles(filters) });
}));

// POST /api/voice/profiles/preset
router.post('/profiles/preset', asyncHandler(async (req, res) => {
  const body = validateRequest(promotePresetProfileSchema, req.body || {});
  const cfg = await getVoiceConfig();
  const preset = parsePresetVoiceId(body.voiceId);
  if (preset?.engine === 'kokoro') {
    throw new ServerError('Kokoro has been retired. Select a Piper preset to create a voice profile.', {
      status: 409, code: 'VOICE_ENGINE_RETIRED',
    });
  }
  const modelRevision = preset?.engine === 'qwen3-tts' ? DEFAULT_DESIGN_MODEL : `piper:${preset?.voice || ''}`;
  const profile = await promotePresetProfile({
    ...body,
    modelRevision,
    delivery: { rate: cfg.tts?.rate },
  });
  res.status(201).json({ profile });
}));

// POST /api/voice/profiles/design — create voice design candidate profile
router.post('/profiles/design', asyncHandler(async (req, res) => {
  const body = validateRequest(voiceDesignCandidateSchema, req.body || {});
  const profile = await createVoiceDesignCandidate(body);
  res.status(201).json({ profile });
}));

// POST /api/voice/profiles/clone — create consented clone candidate profile
router.post('/profiles/clone', asyncHandler(async (req, res) => {
  const body = validateRequest(voiceCloneCandidateSchema, req.body || {});
  const audioBuffer = Buffer.from(body.audioBase64, 'base64');
  if (audioBuffer.length > MAX_BASE64_UPLOAD_BYTES) {
    throw new ServerError('FILE_TOO_LARGE', `Audio file exceeds maximum size limit of ${MAX_BASE64_UPLOAD_BYTES} bytes`);
  }
  const profile = await createClonedVoiceCandidate({
    ...body,
    audioBuffer,
  });
  res.status(201).json({ profile });
}));

// POST /api/voice/profiles/:id/promote — explicitly promote candidate profile
router.post('/profiles/:id/promote', asyncHandler(async (req, res) => {
  const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
  const body = validateRequest(promoteProfileSchema, req.body || {});
  const profile = await promoteVoiceProfile(profileId, body);
  res.json({ profile });
}));

// POST /api/voice/profiles/:id/benchmark
router.post('/profiles/:id/benchmark', asyncHandler(async (req, res) => {
  const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
  const profile = await renderProfileBenchmark(profileId);
  res.json({ profile });
}));

// POST /api/voice/profiles/:id/benchmark-interactive — qualify interactive route
router.post('/profiles/:id/benchmark-interactive', asyncHandler(async (req, res) => {
  const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
  const body = validateRequest(interactiveBenchmarkSchema, req.body || {});
  const benchmark = await benchmarkProfileInteractive(profileId, body);
  res.json({ benchmark });
}));

router.post('/profiles/:id/benchmark-interactive/complete', asyncHandler(async (req, res) => {
  const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
  const body = validateRequest(interactivePlaybackReceiptSchema, req.body || {});
  const profile = await completeProfileInteractiveBenchmark(profileId, body);
  res.json({ profile });
}));

// Fine-tuning endpoints
router.post('/profiles/:id/fine-tune/start', asyncHandler(async (req, res) => {
  const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
  const body = validateRequest(fineTuneStartSchema, req.body || {});
  const result = await startFineTuningJob({ profileId, ...body });
  res.status(202).json(result);
}));

// Newest-first runs from their job.json sidecars, so the Voice Lab recovers a
// run (and its checkpoints) after a reload; live changes arrive on
// `voice:fine-tune:updated`.
router.get('/profiles/:id/fine-tune', asyncHandler(async (req, res) => {
  const { id: profileId } = validateRequest(profileIdParamsSchema, req.params);
  res.json({ jobs: await listFineTuningJobs(profileId) });
}));

router.get('/profiles/:id/fine-tune/:jobId', asyncHandler(async (req, res) => {
  const { id: profileId, jobId } = validateRequest(fineTuneJobParamsSchema, req.params);
  const result = await getFineTuningJobStatus(jobId, profileId);
  res.json(result);
}));

router.post('/profiles/:id/fine-tune/:jobId/cancel', asyncHandler(async (req, res) => {
  const { id: profileId, jobId } = validateRequest(fineTuneJobParamsSchema, req.params);
  const result = cancelFineTuningJob(jobId, profileId);
  res.json(result);
}));

router.post('/profiles/:id/fine-tune/:jobId/promote', asyncHandler(async (req, res) => {
  const { id: profileId, jobId } = validateRequest(fineTuneJobParamsSchema, req.params);
  const body = validateRequest(fineTunePromoteSchema, req.body || {});
  const profile = await promoteCheckpoint({
    profileId,
    jobId,
    checkpointId: body.checkpointId,
  });
  res.json({ profile });
}));

// Qwen3 runtime status and model management
router.get('/qwen3/status', asyncHandler(async (_req, res) => {
  res.json(await getQwen3RuntimeStatus());
}));

router.post('/qwen3/download-model', asyncHandler(async (req, res) => {
  const body = validateRequest(downloadModelSchema, req.body || {});
  const result = await downloadQwen3Model(body.modelId);
  res.json(result);
}));

// POST /api/voice/piper/fetch
router.post('/piper/fetch', asyncHandler(async (req, res) => {
  const { voice } = validateRequest(voicePiperFetchBodySchema, req.body ?? {});
  if (!findPiperVoice(voice)) {
    throw new ServerError(`unknown piper voice: ${voice}`, { status: 400 });
  }
  const cfg = await getVoiceConfig();
  const result = await downloadPiperVoice(voice, cfg);
  res.json({ voice, ...result });
}));

// POST /api/voice/test
const voiceTestBodySchema = z.object({
  text: z.string().trim().min(1).max(MAX_VOICE_TEXT_LEN),
  voice: z.string().trim().default(''),
  engine: z.string().trim().default(''),
});
router.post('/test', asyncHandler(async (req, res) => {
  const { text, voice: requestedVoice, engine: requestedEngine } = validateRequest(voiceTestBodySchema, req.body ?? {});
  const voice = requestedVoice || undefined;
  const engine = validEngine(requestedEngine);
  const { wav, latencyMs } = await synthesize(text, { voice, engine });
  res.setHeader('Content-Type', 'audio/wav');
  res.setHeader('X-TTS-Latency-Ms', String(latencyMs));
  res.send(wav);
}));

const speakBodySchema = z.object({
  text: z.string().trim().min(1).max(MAX_VOICE_TEXT_LEN),
  priority: z.enum(['low', 'normal', 'high']).optional(),
  source: z.string().max(64).optional()
    .transform((s) => {
      const t = (s ?? '').trim();
      return t === '' ? undefined : t;
    }),
});
router.post('/speak', asyncHandler(async (req, res) => {
  const parsed = speakBodySchema.safeParse(req.body || {});
  if (!parsed.success) {
    throw new ServerError(
      `Invalid speak payload: ${parsed.error.issues.map((i) => `${i.path.join('.')} ${i.message}`).join('; ')}`,
      { status: 400, code: 'VALIDATION_ERROR' },
    );
  }
  const io = req.app.get('io');
  if (!io) {
    throw new ServerError(
      'voice subsystem misconfigured: io not attached',
      { status: 500, code: 'VOICE_IO_UNAVAILABLE' },
    );
  }
  const result = await speakProactive({ io, ...parsed.data });
  res.json(result);
}));

// GET /api/voice/tts/status
router.get('/tts/status', asyncHandler(async (_req, res) => {
  res.json({
    kokoro: {
      state: 'lazy',
      retired: true,
      loadedKey: null,
    },
  });
}));

// POST /api/voice/tts/unload
router.post('/tts/unload', asyncHandler(async (_req, res) => {
  res.json({ unloaded: false, retired: true });
}));

const whisperActionSchema = z.object({
  action: z.enum(['start', 'stop']),
});
router.post('/whisper', asyncHandler(async (req, res) => {
  const parsed = whisperActionSchema.safeParse(req.body || {});
  if (!parsed.success) {
    throw new ServerError(
      `Invalid whisper payload: ${parsed.error.issues.map((i) => `${i.path.join('.')} ${i.message}`).join('; ')}`,
      { status: 400, code: 'VALIDATION_ERROR' },
    );
  }
  const { action } = parsed.data;
  if (action === 'stop') {
    await stopWhisper();
    invalidateHealthCache();
    return res.json({ success: true, action: 'stop' });
  }
  const cfg = await getVoiceConfig();
  const result = await startWhisper(cfg);
  invalidateHealthCache();
  res.json({ success: true, action: 'start', ...result });
}));

export default router;