Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Speech-to-text for Music Video lyric alignment.
*
* Voice chat's STT is off by default (the browser does it) and its whisper
* model is `base.en`, which is too weak for a sung vocal. Alignment therefore
* brings its own runner and never asks the director to enable voice:
*
* 1. `whisper-cli` on PATH → one-shot run over each requested audio window with a
* music-grade model (large-v3-turbo), word timestamps from its `-ojf`
* JSON. Phrase-window orchestration lives in lyricAlign.js.
* 2. The configured voice STT endpoint, when something answers there.
* 3. `whisper-server` on PATH → a temporary loopback server on a free port
* with the music-grade model, stopped when the alignment finishes.
* 4. None of these → 503 with how to install whisper.cpp.
*
* The model downloads on the first alignment that needs it (a user click or
* an autopilot run the user started — never at boot) into the same folder
* voice uses, through a `.partial` file renamed into place when complete.
*
* A transcriber is `{ kind, transcribe(wav, { startSec, endSec, prompt }), release() }`
* where `transcribe` returns words on the SONG clock.
*/
import { existsSync } from 'fs';
import { mkdtemp, readFile, rm, writeFile } from 'fs/promises';
import { createServer } from 'net';
import { homedir, tmpdir } from 'os';
import { basename, join } from 'path';
import { ServerError } from '../../lib/errorHandler.js';
import { spawn } from '../../lib/childProcess.js';
import { whichFirst } from '../../lib/processEnv.js';
import { runStreamingCommand } from '../../lib/streamingSpawn.js';
import { streamResumableDownload } from '../../lib/downloadPreflight.js';
import { fetchWithTimeout } from '../../lib/fetchWithTimeout.js';
import { killWithEscalation } from '../../lib/killWithEscalation.js';
import { getVoiceConfig, expandPath } from '../voice/config.js';
import { transcribe as sttTranscribe } from '../voice/stt.js';
import {
explainSttFailure,
lyricAlignChunkSec,
mergeChunkWords,
parseWhisperCliWords,
planAudioChunks,
sliceWav,
wavDurationSec,
} from './lyricAlignCore.js';
export const ALIGN_MODEL_FILE = 'ggml-large-v3-turbo.bin';
export const ALIGN_MODEL_URL = `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${ALIGN_MODEL_FILE}`;
export const alignModelDir = () => join(homedir(), '.portos', 'voice', 'models');
// Homebrew's whisper-cpp ships `whisper-cli`; older builds named it `whisper-cpp`.
const WHISPER_CLI_NAMES = ['whisper-cli', 'whisper-cpp'];
const CLI_TIMEOUT_MS = 15 * 60 * 1000;
const SERVER_BOOT_MS = 90_000;
const ENDPOINT_PROBE_MS = 1500;
export const WHISPER_MISSING_MESSAGE = 'Aligning words needs whisper.cpp, and none was found on this machine. '
+ 'Install it (macOS: `brew install whisper-cpp`; Linux/Windows: build whisper.cpp so `whisper-cli` is on PATH, '
+ 'or run scripts/setup-voice.sh), then click Align words again.';
let modelDownload = null;
// Callers sharing the one in-flight download each watch its progress.
const modelDownloadListeners = new Set();
async function downloadAlignmentModel(target, { download = streamResumableDownload } = {}) {
console.log(`⬇️ Downloading whisper alignment model ${ALIGN_MODEL_FILE} (about 1.6 GB, first alignment only)`);
let lastDecile = -1;
await download({
url: ALIGN_MODEL_URL,
destPath: target,
onBytes: (received, total) => {
if (!(total > 0)) return;
for (const listener of modelDownloadListeners) listener({ received, total, percent: Math.round((received / total) * 100) });
const decile = Math.floor((received / total) * 10);
if (decile === lastDecile) return;
lastDecile = decile;
console.log(`⬇️ Whisper alignment model ${decile * 10}% (${Math.round(received / 1e6)} of ${Math.round(total / 1e6)} MB)`);
},
});
console.log(`✅ Whisper alignment model ready at ${target}`);
}
/**
* Path of a music-grade whisper model, downloading large-v3-turbo on first
* use. A `large` model the voice settings already point at is used as is.
* Concurrent callers share one download.
*/
async function ensureAlignmentModel({ configuredModelPath = null, download, onDownloadProgress = null } = {}) {
const target = join(alignModelDir(), ALIGN_MODEL_FILE);
if (existsSync(target)) return target;
if (configuredModelPath && /large/i.test(basename(configuredModelPath)) && existsSync(configuredModelPath)) {
return configuredModelPath;
}
if (!modelDownload) {
modelDownload = downloadAlignmentModel(target, { download }).finally(() => { modelDownload = null; });
}
if (onDownloadProgress) modelDownloadListeners.add(onDownloadProgress);
try {
await modelDownload;
} catch (err) {
console.error(`❌ Whisper alignment model download failed: ${err.message}`);
throw new ServerError(`Could not download the whisper alignment model: ${err.message}`, { status: 502, code: 'LYRIC_ALIGN_MODEL_DOWNLOAD_FAILED' });
} finally {
if (onDownloadProgress) modelDownloadListeners.delete(onDownloadProgress);
}
return target;
}
/** whisper-cli argv for one WAV file; JSON lands at `${outBase}.json`. */
function whisperCliArgs({ modelPath, wavPath, outBase, language = 'en', prompt = '' }) {
const args = ['-m', modelPath, '-f', wavPath, '-l', language, '-ml', '1', '-sow', '-ojf', '-of', outBase, '-np'];
if (prompt) args.push('--prompt', prompt);
return args;
}
function regionOf(wav, { startSec, endSec } = {}) {
const duration = wavDurationSec(wav);
if (!(duration > 0)) {
throw new ServerError('Could not read the decoded vocal.', { status: 422, code: 'LYRIC_ALIGN_DECODE_FAILED' });
}
const start = Math.max(0, startSec || 0);
const end = Math.min(duration, endSec ?? duration);
if (!(end > start)) {
throw new ServerError('That lyric line has no audio window to align.', { status: 422, code: 'LYRIC_ALIGN_NO_WINDOW' });
}
return { startSec: start, endSec: end, whole: start === 0 && end === duration };
}
/** One-shot whisper-cli transcriber. */
function createCliTranscriber({ bin, modelPath, language = 'en', run = runStreamingCommand }) {
return {
kind: 'whisper-cli',
async transcribe(wav, { startSec, endSec, prompt = '' } = {}) {
const region = regionOf(wav, { startSec, endSec });
const dir = await mkdtemp(join(tmpdir(), 'lyric-align-cli-'));
try {
const wavPath = join(dir, 'audio.wav');
const outBase = join(dir, 'words');
await writeFile(wavPath, region.whole ? wav : sliceWav(wav, region.startSec, region.endSec));
const result = await run(bin, whisperCliArgs({ modelPath, wavPath, outBase, language, prompt }), null, { timeoutMs: CLI_TIMEOUT_MS });
if (!result?.success) {
console.error(`❌ whisper-cli failed: ${result?.error || 'unknown error'}`);
throw new ServerError(`whisper-cli could not transcribe the song: ${result?.error || 'unknown error'}`, { status: 502, code: 'LYRIC_ALIGN_WHISPER_FAILED' });
}
const raw = await readFile(`${outBase}.json`, 'utf8').catch(() => null);
const parsed = raw ? (() => { try { return JSON.parse(raw); } catch { return null; } })() : null;
if (!parsed) {
throw new ServerError('whisper-cli wrote no readable word timings.', { status: 502, code: 'LYRIC_ALIGN_WHISPER_FAILED' });
}
return parseWhisperCliWords(parsed).map((word) => ({
...word,
startSec: word.startSec + region.startSec,
endSec: word.endSec + region.startSec,
}));
} finally {
await rm(dir, { recursive: true, force: true }).catch(() => {});
}
},
release: async () => {},
};
}
/**
* Transcriber over a whisper.cpp HTTP server (the voice endpoint or a
* temporary one): the region is chunked so each request fits the STT timeout.
*/
function createEndpointTranscriber({ endpoint = null, kind = 'stt-endpoint', transcribe = sttTranscribe, release = async () => {} }) {
return {
kind,
async transcribe(wav, { startSec, endSec, prompt = '' } = {}) {
const region = regionOf(wav, { startSec, endSec });
const chunks = planAudioChunks(region.endSec - region.startSec, { chunkSec: lyricAlignChunkSec() }).map((chunk) => ({
startSec: region.startSec + chunk.startSec,
endSec: region.startSec + chunk.endSec,
}));
console.log(`🎤 Aligning lyric words (${chunks.length} whisper ${chunks.length === 1 ? 'chunk' : 'chunks'} via ${kind})`);
const results = [];
for (const chunk of chunks) {
let result;
try {
result = await transcribe(sliceWav(wav, chunk.startSec, chunk.endSec), {
verbose: true,
mimeType: 'audio/wav',
filename: 'lyric-align.wav',
prompt,
...(endpoint ? { endpoint } : {}),
});
} catch (err) {
console.error(`❌ Lyric alignment speech-to-text failed: ${err.message}`);
throw new ServerError(explainSttFailure(err), { status: 503, code: 'LYRIC_ALIGN_STT_UNAVAILABLE' });
}
results.push({
startSec: chunk.startSec,
endSec: chunk.endSec,
words: (result?.words || []).map((word) => ({ text: word.text, startSec: word.startSec, endSec: word.endSec })),
});
}
return mergeChunkWords(results);
},
release,
};
}
/** True when anything answers HTTP at the endpoint (any status = bound). */
async function probeSttEndpoint(endpoint) {
if (!endpoint) return false;
return fetchWithTimeout(`${String(endpoint).replace(/\/+$/, '')}/`, { method: 'GET' }, ENDPOINT_PROBE_MS)
.then(() => true)
.catch(() => false);
}
function freeLoopbackPort() {
return new Promise((resolve, reject) => {
const server = createServer();
server.unref();
server.on('error', reject);
server.listen(0, '127.0.0.1', () => {
const { port } = server.address();
server.close(() => resolve(port));
});
});
}
/** Start a private whisper-server for this alignment; resolves `{ endpoint, stop }`. */
async function startTemporaryWhisperServer({ bin, modelPath }) {
const port = await freeLoopbackPort();
const endpoint = `http://127.0.0.1:${port}`;
const child = spawn(bin, ['--host', '127.0.0.1', '--port', String(port), '--model', modelPath], { stdio: 'ignore' });
let exited = false;
child.on('exit', () => { exited = true; });
child.on('error', (err) => {
exited = true;
console.error(`❌ Temporary whisper-server failed: ${err.message}`);
});
const stop = async () => {
if (exited) return;
killWithEscalation(child, { label: 'temporary whisper-server', stillRunning: () => !exited });
};
const deadline = Date.now() + SERVER_BOOT_MS;
while (Date.now() < deadline && !exited) {
if (await probeSttEndpoint(endpoint)) {
console.log(`🎙️ Temporary whisper-server up on ${endpoint} for lyric alignment`);
return { endpoint, stop };
}
await new Promise((resolve) => setTimeout(resolve, 300));
}
await stop();
throw new ServerError('The temporary whisper server did not start. Check that whisper-server runs from a terminal.', { status: 503, code: 'LYRIC_ALIGN_STT_UNAVAILABLE' });
}
async function firstOnPath(names, which) {
for (const name of names) {
const found = await which(name);
if (found) return found;
}
return null;
}
/**
* Pick the runner for this alignment (see the module header for the order).
* Every dependency is injectable so selection is testable without binaries.
*/
export async function resolveAlignmentTranscriber(deps = {}) {
const {
which = whichFirst,
voiceConfig = getVoiceConfig,
probeEndpoint = probeSttEndpoint,
ensureModel = ensureAlignmentModel,
runCommand = runStreamingCommand,
startServer = startTemporaryWhisperServer,
transcribe = sttTranscribe,
onDownloadProgress = null,
} = deps;
const cfg = await voiceConfig().catch(() => null);
const language = cfg?.stt?.language || 'en';
const configuredModelPath = cfg?.stt?.modelPath ? expandPath(cfg.stt.modelPath) : null;
const cli = await firstOnPath(WHISPER_CLI_NAMES, which);
if (cli) {
const modelPath = await ensureModel({ configuredModelPath, onDownloadProgress });
return createCliTranscriber({ bin: cli, modelPath, language, run: runCommand });
}
const endpoint = cfg?.stt?.endpoint || null;
if (endpoint && await probeEndpoint(endpoint)) {
return createEndpointTranscriber({ endpoint, kind: 'stt-endpoint', transcribe });
}
const serverBin = await which('whisper-server');
if (serverBin) {
const modelPath = await ensureModel({ configuredModelPath, onDownloadProgress });
const server = await startServer({ bin: serverBin, modelPath });
return createEndpointTranscriber({ endpoint: server.endpoint, kind: 'whisper-server', transcribe, release: server.stop });
}
throw new ServerError(WHISPER_MISSING_MESSAGE, { status: 503, code: 'LYRIC_ALIGN_STT_UNAVAILABLE' });
}