Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
// Voice stack lifecycle — owns the whisper-server PM2 app and model/binary
// provisioning. Piper (TTS) is spawned per-request in services/voice/tts.js.
import { existsSync } from 'fs';
import { execFile } from '../../lib/childProcess.js';
import { promisify } from 'util';
import { basename, join } from 'path';
import { createServer } from 'net';
import { PATHS, sleep } from '../../lib/fileUtils.js';
import { execPm2, getAppStatus } from '../pm2.js';
import { expandPath, piperVoiceTildePath, voiceHome, IS_WIN, PIPER_BIN_NAME } from './config.js';
import { isReasoningModel } from './llm.js';
import {
getVoiceProvisioner, DEFAULT_VOICE_BACKEND, sortPreferred, sizeOf, FAST_VOICE_MODEL_MAX_B,
} from './modelProvisioners.js';
import { getProviderById } from '../providers.js';
import { fetchWithTimeout } from '../../lib/fetchWithTimeout.js';
import { whichFirst } from '../../lib/processEnv.js';
import { PORTS } from '../../lib/ports.js';
import { inspectVoiceAsset, isManagedVoiceAsset, isVoiceAssetUsable } from '../../lib/voiceModelAssets.js';
export const pexec = promisify(execFile);
export const WHISPER_APP = 'portos-whisper';
// Thin re-export of the shared PATH probe so voice callers keep the local name.
export const which = (bin) => whichFirst(bin);
export const verifyBinaries = async (cfg) => {
const piperLocal = join(voiceHome(), 'piper', PIPER_BIN_NAME);
const piperResolved = existsSync(piperLocal) ? piperLocal : null;
// Only search PATH when piper isn't locally installed — spawning `where`/`which` is expensive.
const [whisper, piperOnPath] = await Promise.all([
which('whisper-server'),
piperResolved ? Promise.resolve(null) : which('piper'),
]);
const piperRequired = cfg?.tts?.engine === 'piper';
return { whisper, piper: piperResolved ?? piperOnPath, piperRequired };
};
// A model is ready only when it passes the completion contract
// (lib/voiceModelAssets.js) — a file that merely exists may be a truncated
// download. `*State` carries the verdict ('verified' | 'unverified' |
// 'incomplete' | 'missing') so callers can tell "repair me" from "install me".
export const verifyModels = (cfg) => {
const modelPath = expandPath(cfg.stt.modelPath);
const sttAsset = inspectVoiceAsset('whisper', modelPath);
const out = {
sttModel: isVoiceAssetUsable(sttAsset.state) ? modelPath : null,
sttModelState: sttAsset.state,
};
if (cfg.tts.engine === 'piper') {
const voicePath = expandPath(cfg.tts.piper.voicePath);
const voiceAsset = inspectVoiceAsset('piper', voicePath);
out.ttsVoice = isVoiceAssetUsable(voiceAsset.state) ? voicePath : null;
out.ttsVoiceState = voiceAsset.state;
} else {
// Qwen3 runtime owns its model readiness.
out.ttsVoice = null;
}
if (cfg.stt.coreml) {
const mlPath = modelPath.replace(/\.bin$/, '-encoder.mlmodelc');
out.coreml = existsSync(mlPath) ? mlPath : null;
}
return out;
};
const parseVoiceName = (voicePath) => basename(voicePath).replace(/\.onnx$/, '');
export const runSetupScript = async (cfg) => {
const modelName = basename(expandPath(cfg.stt.modelPath));
const voiceName = cfg.tts.engine === 'piper' ? parseVoiceName(expandPath(cfg.tts.piper.voicePath)) : '';
const sttEngine = cfg.stt?.engine || 'whisper';
const env = {
...process.env,
MODEL_NAME: modelName,
VOICE_NAME: voiceName,
STT_ENGINE: sttEngine,
TTS_ENGINE: cfg.tts.engine || 'piper',
INSTALL_COREML: cfg.stt.coreml ? '1' : '0',
};
console.log(`🔧 voice: setup-voice (stt=${sttEngine}/${modelName}, tts=${cfg.tts.engine}, coreml=${env.INSTALL_COREML})`);
// 10-minute cap — large models + slow network can legitimately take several
// minutes, but a hung curl must not pin the HTTP request that triggered us.
// On Windows prefer pwsh (PowerShell 7+) but fall back to the always-present
// Windows PowerShell when pwsh isn't installed.
let cmd;
let args;
if (IS_WIN) {
const psBin = (await which('pwsh')) ? 'pwsh' : 'powershell';
cmd = psBin;
args = ['-ExecutionPolicy', 'Bypass', '-File', 'scripts\\setup-voice.ps1'];
} else {
cmd = 'bash';
args = ['scripts/setup-voice.sh'];
}
const { stdout, stderr } = await pexec(cmd, args, {
cwd: PATHS.root,
env,
maxBuffer: 64 * 1024 * 1024,
timeout: 10 * 60 * 1000,
});
return { stdout, stderr };
};
/**
* Download a single Piper voice without touching whisper/STT state. Used by
* the Settings voice-picker so users can audition voices as they browse the
* catalog rather than waiting for Save & Reconcile.
*/
export const downloadPiperVoice = async (voiceId, currentCfg) => {
if (!voiceId || typeof voiceId !== 'string') throw new Error('voiceId required');
const voicePath = piperVoiceTildePath(voiceId);
const absVoicePath = expandPath(voicePath);
// Only a receipted pair is done. A bare .onnx may be a truncated download
// (or lack its .onnx.json), and an unreceipted pair gets its receipt from the
// script — so both go through setup instead of being skipped.
if (inspectVoiceAsset('piper', absVoicePath).state === 'verified') return { skipped: true, voicePath };
// Re-use the existing setup script but force it into Piper-only mode. The
// script already short-circuits whisper steps when the model/binary are
// present, so this is cheap on repeat invocations.
await runSetupScript({
...currentCfg,
tts: { engine: 'piper', piper: { voicePath } },
});
const after = inspectVoiceAsset('piper', absVoicePath);
if (after.state !== 'verified') {
throw new Error(`Piper voice ${voiceId} is still not usable after setup (${after.reason || after.state})`);
}
return { downloaded: true, voicePath };
};
const isWhisperRunning = async () => {
const status = await getAppStatus(WHISPER_APP).catch(() => null);
return status?.status === 'online';
};
// Returns null if the port is free, else a short description of who's there.
// `port` MUST be coerced to a number — `net.Server.listen(stringPort)` is
// interpreted as a pipe path and silently misses real TCP port collisions.
// Any listen() error other than EADDRINUSE (EACCES, EADDRNOTAVAIL, EINVAL…)
// indicates endpoint misconfiguration — surface it instead of silently
// proceeding to a more confusing PM2 failure downstream.
const probePortInUse = (host, port) => new Promise((resolve) => {
const portNum = Number(port);
const s = createServer();
s.once('error', (err) => {
s.close();
if (err.code === 'EADDRINUSE') {
resolve(`port ${portNum} in use (${err.code})`);
} else {
resolve(`cannot bind ${host}:${portNum} (${err.code || err.message})`);
}
});
s.once('listening', () => s.close(() => resolve(null)));
s.listen(portNum, host);
});
// Poll until whisper's /inference endpoint answers (any HTTP status = bound),
// or give up after `timeoutMs`. Distinguishes "bound but slow" from "crashed".
// Each probe has its own abort-based timeout so a hung connect (firewall,
// half-open socket) can't stall the loop past the overall deadline.
const waitForWhisper = async (host, port, timeoutMs = 8000) => {
const deadline = Date.now() + timeoutMs;
const url = `http://${host}:${port}/`;
while (Date.now() < deadline) {
const remaining = deadline - Date.now();
const probeTimeout = Math.max(1, Math.min(1000, remaining));
const ok = await fetchWithTimeout(url, { method: 'GET' }, probeTimeout)
.then(() => true)
.catch(() => false);
if (ok) return true;
const waitMs = Math.min(250, Math.max(0, deadline - Date.now()));
if (waitMs > 0) await sleep(waitMs);
}
return false;
};
export const startWhisper = async (cfg) => {
const whisperBin = await which('whisper-server');
if (!whisperBin) throw new Error('whisper-server not on PATH — run scripts/setup-voice.sh');
const modelPath = expandPath(cfg.stt.modelPath);
const modelAsset = inspectVoiceAsset('whisper', modelPath);
if (modelAsset.state === 'missing') throw new Error(`whisper model missing: ${modelPath}`);
if (!isVoiceAssetUsable(modelAsset.state)) {
throw new Error(`whisper model incomplete: ${modelPath} (${modelAsset.reason}) — run Save & Reconcile in Settings → Voice to repair it`);
}
const url = new URL(cfg.stt.endpoint);
const host = url.hostname;
const port = url.port || String(PORTS.WHISPER);
// Delete stale PM2 entry so our own previous instance doesn't count as a collision.
await execPm2(['delete', WHISPER_APP]).catch(() => {});
// Pre-flight: refuse to start if something ELSE is already on the port —
// whisper-server crashes on bind failure and takes the model with it.
// Distinguish "port collision" (use a different port) from "bind error"
// (EACCES / EINVAL / EADDRNOTAVAIL → host/IP itself is wrong).
const occupied = await probePortInUse(host, port);
if (occupied) {
if (/EADDRINUSE|in use/i.test(occupied)) {
throw new Error(`${occupied} — another service is bound to ${host}:${port}. Change voice.stt.endpoint (e.g. http://127.0.0.1:5563) under Settings → Voice.`);
}
throw new Error(`${occupied} — voice.stt.endpoint is misconfigured for ${host}:${port}. Check Settings → Voice and ensure the host/IP is valid and bindable on this machine.`);
}
await execPm2([
'start', whisperBin,
'--name', WHISPER_APP,
'--interpreter', 'none',
'--no-autorestart',
'--',
'--host', host, '--port', port, '--model', modelPath,
]);
// Verify the server actually bound. whisper-server returns 0 to PM2 even
// when it aborts on bind failure, so we can't trust pm2 exit status alone.
const bound = await waitForWhisper(host, port);
if (!bound) {
await execPm2(['delete', WHISPER_APP]).catch(() => {});
throw new Error(`whisper-server failed to bind on ${host}:${port} within 8s — check pm2 logs ${WHISPER_APP}`);
}
console.log(`🎙️ voice: ${WHISPER_APP} up on ${host}:${port} (model=${modelPath})`);
return { name: WHISPER_APP, host, port, modelPath };
};
export const stopWhisper = async () => {
if (!(await isWhisperRunning())) return { skipped: true };
await execPm2(['delete', WHISPER_APP]).catch(() => {});
console.log(`🛑 voice: ${WHISPER_APP} stopped`);
return { stopped: true };
};
/**
* The first model in `ids` that is fast, non-reasoning and tool-capable.
*
* Sequential on purpose. The capability check can be a per-model HTTP probe
* (Ollama's /api/show), and we only need the FIRST match — a `Promise.all`
* here would probe every installed model on an install with dozens of them to
* answer a question the first hit already settles.
*/
const firstFastCapable = async (backend, ids) => {
for (const id of ids) {
if (isReasoningModel(id) || sizeOf(id) > FAST_VOICE_MODEL_MAX_B) continue;
if (await backend.isToolCapable(id)) return id;
}
return null;
};
/**
* The local backend voice should PROVISION models against for `cfg`, or null
* when this provider needs no provisioning.
*
* Mirrors `resolveLlmEndpoint` in llm.js: voice falls back to the default
* backend whenever the configured provider is missing, not api-type, or has no
* endpoint — so a half-configured install still gets a working local model.
* A provider that DOES resolve to a usable api-type endpoint serves its own
* models (OpenAI, Groq, a remote vLLM), so there is nothing to install or
* pre-warm and we return null rather than provisioning the wrong backend.
*/
const resolveVoiceProvisioner = async (cfg) => {
const providerId = cfg?.llm?.provider || DEFAULT_VOICE_BACKEND;
const local = getVoiceProvisioner(providerId);
if (local) return local;
const provider = await getProviderById(providerId).catch(() => null);
// A usable remote provider needs no local provisioning.
if (provider && provider.type === 'api' && provider.endpoint) return null;
return getVoiceProvisioner(DEFAULT_VOICE_BACKEND);
};
/**
* Ensure a fast, tool-capable model exists on the configured local backend,
* installing one from the catalog-derived chain when none does.
*
* Only intervenes when the user opted in: voice on, tools on AND model is
* 'auto'. An explicit model id means they know what they want — respect it
* even if incompatible.
*
* The `enabled` gate is deliberately duplicated from `reconcile`'s early
* return rather than left to the caller: this function downloads multi-GB
* weights, so an install with voice OFF must never reach it even if a future
* caller forgets the gate. `preloadModel` guards itself the same way.
*/
export const ensureToolCapableModel = async (cfg) => {
if (!cfg?.enabled) return { skipped: 'voice-disabled' };
if (!cfg?.llm?.tools?.enabled) return { skipped: 'tools-disabled' };
if (cfg?.llm?.model && cfg.llm.model !== 'auto') return { skipped: 'explicit-model' };
const backend = await resolveVoiceProvisioner(cfg);
if (!backend) return { skipped: 'remote-provider', provider: cfg?.llm?.provider };
const installed = await backend.listModels();
// `null` = the backend could not be reached. Every entry in the install
// chain would fail for that one shared reason, so say so ONCE and stop —
// walking the chain here is what spawned four futile multi-GB downloads on
// every boot of a machine whose local server simply wasn't running.
if (installed === null) {
console.warn(`🎙️ voice: ${backend.label} is not reachable — ${backend.remedy}`);
return { skipped: 'backend-unreachable', backend: backend.id };
}
// Tool-capable AND non-reasoning AND under the size cap. The size cap
// matters: a user with only `mistral-small-24B` installed gets a model that
// thrashes VRAM on every turn; we'd rather pull a small one and give them
// snappy responses out of the box.
const fastCapable = await firstFastCapable(backend, installed);
if (fastCapable) return { skipped: 'already-capable', model: fastCapable };
// Snapshot the model set BEFORE install so we can detect which id the
// backend actually registered the download under (ids get case-normalized or
// gain a quant suffix). Without this snapshot we mis-attributed success to
// whichever existing tool-capable model happened to match — including the
// slow reasoning model we were trying to escape.
const before = new Set(installed);
const chain = backend.chain();
if (!chain.length) {
console.warn(`🎙️ voice: no catalog model with tool support is installable on ${backend.label} — set voice.llm.model explicitly in Settings`);
return { skipped: 'no-install-target', backend: backend.id };
}
for (const target of chain) {
console.log(`🎙️ voice: installing fast tool-capable model ${target} via ${backend.label} (this may take a few minutes)`);
const outcome = await backend.install(target);
const after = await backend.listModels();
// The backend going away mid-chain is the same shared cause as above —
// stop rather than retrying every remaining entry against a dead server.
if (after === null) {
console.warn(`🎙️ voice: ${backend.label} became unreachable during install — ${backend.remedy}`);
return { skipped: 'backend-unreachable', backend: backend.id };
}
const fastNew = await firstFastCapable(backend, after.filter((id) => !before.has(id)));
if (fastNew) {
console.log(`🎙️ voice: fast tool-capable model ready — ${fastNew}`);
return { installed: fastNew };
}
console.log(`🎙️ voice: ${target} unavailable (${String(outcome?.reason || 'unknown').slice(0, 160)}) — trying next`);
}
console.warn(`🎙️ voice: exhausted ${backend.label} install chain ${chain.join(', ')} — set voice.llm.model explicitly in Settings`);
return { failed: chain };
};
// Pre-warm the model that 'auto' will pick on the first turn so the user
// doesn't pay a 5–30 s cold-load on their first question. Skip when the chosen
// model is already resident: on LM Studio a second `lms load` spawns another
// INSTANCE of the same model (3 copies of qwen3-4b reported in the wild),
// eating multiples of its VRAM and producing "Model loading was stopped due to
// insufficient system resources" when nothing is actually wrong.
export const preloadModel = async (cfg) => {
if (!cfg?.enabled) return { skipped: 'voice-disabled' };
if (cfg?.llm?.model && cfg.llm.model !== 'auto') return { skipped: 'explicit-model' };
const backend = await resolveVoiceProvisioner(cfg);
if (!backend) return { skipped: 'remote-provider', provider: cfg?.llm?.provider };
const installed = await backend.listModels();
if (installed === null) return { skipped: 'backend-unreachable', backend: backend.id };
if (!installed.length) return { skipped: 'no-models', backend: backend.id };
// Rank first, then walk — so the tool-capability probe runs on the models we
// would actually pick and stops at the winner, rather than on every model.
const ranked = sortPreferred(installed);
let target = ranked[0] || null;
if (cfg.llm?.tools?.enabled) {
for (const id of ranked) {
if (await backend.isToolCapable(id)) { target = id; break; }
}
}
if (!target) return { skipped: 'no-candidate' };
const loaded = await backend.loadedModels();
if (loaded.has(target)) {
console.log(`🎙️ voice: ${target} already loaded — skipping preload`);
return { skipped: 'already-loaded', model: target };
}
// The load blocks until ready (5–30 s cold). Run fire-and-forget so
// reconcile returns immediately; whisper/TTS can come up in parallel.
console.log(`🎙️ voice: preloading ${target} (warming GPU/cache for first turn)`);
backend.load(target)
.then((result) => (result?.ok
? console.log(`🎙️ voice: ${target} loaded and ready`)
: console.warn(`🎙️ voice: preload ${target} failed: ${result?.reason || 'unknown'}`)))
.catch((err) => console.warn(`🎙️ voice: preload ${target} failed: ${err.message}`));
return { preloading: target };
};
/**
* Reconcile PM2 state with desired voice.enabled. Called from
* PUT /api/voice/config and at server boot.
*/
export const reconcile = async (cfg, { allowSetup = true } = {}) => {
if (!cfg.enabled) return stopWhisper();
// A retired-engine upgrade must wait for the user's Save & Reconcile action.
// Do not turn a normal restart into a new binary/model download.
if (!allowSetup && cfg.tts?.retiredEngine === 'kokoro') {
const bins = await verifyBinaries(cfg);
const models = verifyModels(cfg);
if (bins.piperRequired && (!bins.piper || !models.ttsVoice)) {
// Keep an already-provisioned speech recognizer available while TTS
// waits for consent. No setup scripts or LLM preloads run on this path.
if (cfg.stt?.engine === 'web-speech') {
await stopWhisper().catch(() => null);
} else if (bins.whisper && models.sttModel && (!cfg.stt.coreml || models.coreml)) {
return { ...await startWhisper(cfg), setupRequired: 'piper' };
}
return { setupRequired: 'piper' };
}
}
// Don't block reconcile on this — it can take minutes on first install.
// The user will see a clear log line and their first turn may fail with the
// new `voice:error` hint until the model finishes downloading, but voice
// STT + TTS + whisper are ready immediately. Once install resolves (or
// immediately if no install was needed), pre-warm the chosen model so the
// first voice turn doesn't pay a 5-30s cold-load.
ensureToolCapableModel(cfg)
.catch((err) => {
console.warn(`🎙️ voice: ensureToolCapableModel failed: ${err.message}`);
})
.then(() => preloadModel(cfg))
.catch((err) => {
console.warn(`🎙️ voice: preloadModel failed: ${err.message}`);
});
const bins = await verifyBinaries(cfg);
const models = verifyModels(cfg);
const piperMissing = bins.piperRequired && (!bins.piper || !models.ttsVoice);
const webSpeech = cfg.stt?.engine === 'web-speech';
const coremlMissing = cfg.stt.coreml && !models.coreml;
// Boot is read-only: an asset that is present but incomplete is repaired by
// the user's next Save & Reconcile / voice download, never by a restart
// kicking off a surprise multi-hundred-MB transfer. Missing assets keep their
// existing first-install behavior.
if (!allowSetup) {
const repair = [];
if (bins.piperRequired && models.ttsVoiceState === 'incomplete') repair.push('piper');
if (!webSpeech && models.sttModelState === 'incomplete') repair.push('whisper');
if (repair.length) {
console.warn(`🎙️ voice: incomplete model download (${repair.join(', ')}) — run Save & Reconcile in Settings → Voice to repair`);
const setupRequired = repair.join(',');
if (webSpeech) {
await stopWhisper().catch(() => null);
} else if (!repair.includes('whisper') && bins.whisper && models.sttModel && !coremlMissing) {
return { ...await startWhisper(cfg), setupRequired };
}
return { setupRequired };
}
}
// Models that predate completion receipts are usable as-is, but only the
// setup script can establish their receipt (it may need a remote size check).
// Do that on a user-authorized pass, for files the script manages — a model
// at a user-chosen path is validated, never touched.
const sttNeedsReceipt = !webSpeech && models.sttModelState === 'unverified'
&& isManagedVoiceAsset(expandPath(cfg.stt.modelPath), join(voiceHome(), 'models'));
const ttsNeedsReceipt = bins.piperRequired && models.ttsVoiceState === 'unverified'
&& isManagedVoiceAsset(expandPath(cfg.tts.piper.voicePath), join(voiceHome(), 'voices'));
const adoptReceipts = async () => {
if (!allowSetup || !(sttNeedsReceipt || ttsNeedsReceipt)) return;
await runSetupScript(cfg).catch((err) => {
console.warn(`🎙️ voice: could not record model completion receipts: ${err.message}`);
});
};
// Web Speech STT runs entirely in the browser — stop any leftover whisper
// instance and skip STT provisioning. Piper voice provisioning still runs.
if (webSpeech) {
if (piperMissing) await runSetupScript(cfg);
else await adoptReceipts();
await stopWhisper().catch(() => null);
return { skipped: 'web-speech', piperProvisioned: piperMissing };
}
const sttMissing = !bins.whisper || !models.sttModel || coremlMissing;
if (piperMissing || sttMissing) await runSetupScript(cfg);
else await adoptReceipts();
return startWhisper(cfg);
};