Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Pipeline audio helpers — voice ID namespace + thin wrappers around the
* existing voice-agent TTS (`server/services/voice/tts.js`) for pipeline
* artifacts that need to land on disk instead of streaming over Socket.IO.
*
* Provider strategy: **always-available local OSS first.** Kokoro and Piper
* ship in-process via ONNX/transformers.js — every install can render voice
* lines without network or API keys. Premium providers (ElevenLabs, etc.)
* add as sibling engines via `server/services/voice/tts.js`'s engine
* dispatch; the pipeline doesn't care which one rendered the bytes.
*
* Voice ID namespace: `engine:voiceName` — `kokoro:af_heart`,
* `piper:en_GB-northern_english_male`, `qwen3-tts:warm-narrator`. A bare
* voice name without the `engine:` prefix is interpreted as the active
* engine's voice; legacy `qwen3:` ids normalize to `qwen3-tts:` on read.
*/
import { join } from 'path';
import { randomUUID } from 'crypto';
import { PATHS, ensureDir, atomicWrite } from '../../lib/fileUtils.js';
import {
synthesize,
listVoices,
normalizeVoiceEngine,
VALID_ENGINES,
} from '../voice/tts.js';
import { ServerError } from '../../lib/errorHandler.js';
import { wavDurationMs } from '../../lib/wavAudioFile.js';
import { compareSpeech } from '../../lib/speechMatch.js';
import { transcribe } from '../voice/stt.js';
const VOICE_ID_RE = /^([a-z][a-z0-9-]*):(.+)$/i;
/**
* Split a voice id into `{ engine, voice }`. Returns `{ engine: null, voice: id }`
* when no engine prefix is present so the caller can fall back to the active
* engine — same shape as the legacy single-engine code path.
*/
export function parseVoiceId(voiceId) {
if (typeof voiceId !== 'string' || !voiceId.trim()) return { engine: null, voice: null };
const trimmed = voiceId.trim();
const m = trimmed.match(VOICE_ID_RE);
if (!m) return { engine: null, voice: trimmed };
const engine = normalizeVoiceEngine(m[1].toLowerCase());
if (engine !== 'kokoro' && !VALID_ENGINES.has(engine)) return { engine: null, voice: trimmed };
return { engine, voice: m[2] };
}
/**
* List every voice exposed by every supported engine, namespaced with
* `engine:voiceName` so the character voice picker can present a single
* flat list across providers. Failures from one engine never block the
* others — a missing Piper binary, for instance, just drops piper voices
* from the list.
*/
export async function listAllVoices() {
const engines = [...VALID_ENGINES];
const results = await Promise.all(engines.map(async (engine) => {
try {
const { voices } = await listVoices(engine);
return voices.map((v) => {
const voice = v.voice || v.name;
return {
// Carry through any metadata the engine surfaces (gender, language,
// accent, etc.) without locking the shape — the UI renders what's
// present.
...v,
// Namespace fields are authoritative. Engine adapters may expose a
// legacy id, but it must not override the catalog contract.
id: `${engine}:${voice}`,
engine,
voice,
label: v.label ?? v.name,
};
});
} catch (err) {
console.warn(`⚠️ listAllVoices: ${engine} unavailable — ${err?.message || err}`);
return [];
}
}));
return results.flat();
}
/**
* Synthesize text to a WAV file under PATHS.audio and return the saved
* filename. The route layer + the audio stage line records correlate by
* filename, same convention as the image pipeline.
*
* `voiceId` may be the namespaced form (`kokoro:af_heart`) or a bare voice
* name (interpreted against the active engine). Empty / falsy `voiceId`
* uses the configured default voice.
*/
/**
* Walk an issue's already-extracted storyboard scenes and emit a flat
* `lines[]` ready to write onto `stages.audio.lines`. The scene extractor
* already parsed dialogue into `{ character, line }` per scene, so we don't
* re-parse markdown here — we just flatten + bind each speaker to a canon
* character (by name, case-insensitive) so the per-line render can resolve
* a voice id.
*
* Lines without a matching canon character still get persisted (the user
* may want narrator / un-named-character lines synthesized via the project
* default voice); `characterId` stays null in that case.
*/
/**
* Voice resolution for a single VO line. Priority:
* 1. `explicit` (per-request body override)
* 2. `line.voiceIdOverride` (per-line override saved on the issue)
* 3. `character.voiceId` (canon character binding by line.characterId)
* 4. `null` (caller falls through to the configured default voice)
*
* `canon` is `{ characters }` — typically loaded from the linked universe
* via `getSeriesCanon(series)`. Pure function — no I/O — so the route,
* a future "render all" flow, and unit tests all use the same priority.
*/
export function resolveVoiceForLine(line, canon, { explicit } = {}) {
if (typeof explicit === 'string' && explicit.trim()) return explicit.trim();
if (line?.voiceIdOverride) return line.voiceIdOverride;
if (line?.characterId) {
const char = (canon?.characters || []).find((c) => c?.id === line.characterId);
if (char?.voiceId) return char.voiceId;
}
return null;
}
/**
* Strip parenthetical performance hints from a screenplay speaker label so
* character matching works on the bare name. Handles trailing single
* (`JEAN (O.S.)`), stacked trailing (`JEAN (O.S.)(angry)`), and leading
* (`(O.S.) JEAN`) parens — all three appear in real teleplays.
*/
function stripSpeakerParens(raw) {
if (typeof raw !== 'string') return '';
return raw
.replace(/^\s*\([^)]*\)\s*/, '')
.replace(/\s*\([^)]*\)\s*/g, ' ')
.trim();
}
export function extractDialogueLines(issue, canon, { preserveFrom = [] } = {}) {
const scenes = Array.isArray(issue?.stages?.storyboards?.scenes)
? issue.stages.storyboards.scenes
: [];
// First-registration wins so duplicate character names / aliased collisions
// resolve deterministically. Last-writer-wins on a Map would silently flip
// a binding when a later character shares a name.
const charactersByKey = new Map();
const rememberKey = (key, c) => {
if (key && !charactersByKey.has(key)) charactersByKey.set(key, c);
};
for (const c of (canon?.characters || [])) {
if (!c || typeof c !== 'object') continue;
const name = typeof c.name === 'string' ? c.name.trim() : '';
if (!name) continue;
rememberKey(name.toLowerCase(), c);
if (Array.isArray(c.aliases)) {
for (const alias of c.aliases) {
if (typeof alias === 'string' && alias.trim()) rememberKey(alias.trim().toLowerCase(), c);
}
}
}
// Build a (speakerKey + text) → prior line index so re-extraction can carry
// forward rendered audio for lines that haven't changed. The match is
// intentionally strict — any edit to text or character invalidates the
// existing render.
const preservedByKey = new Map();
if (Array.isArray(preserveFrom)) {
for (const prev of preserveFrom) {
if (!prev || typeof prev !== 'object') continue;
if (!prev.audioFilename && !prev.audioJobId) continue;
const key = `${(prev.characterName || '').toLowerCase()}|${prev.text || ''}`;
if (!preservedByKey.has(key)) preservedByKey.set(key, prev);
}
}
const lines = [];
let preservedCount = 0;
let lineNumber = 1;
for (let sIdx = 0; sIdx < scenes.length; sIdx += 1) {
const scene = scenes[sIdx];
const dialogue = Array.isArray(scene?.dialogue) ? scene.dialogue : [];
for (let dIdx = 0; dIdx < dialogue.length; dIdx += 1) {
const d = dialogue[dIdx];
const text = typeof d?.line === 'string' ? d.line.trim() : '';
if (!text) continue;
const rawSpeaker = typeof d?.character === 'string' ? d.character.trim() : '';
const bareSpeaker = stripSpeakerParens(rawSpeaker);
const match = bareSpeaker ? charactersByKey.get(bareSpeaker.toLowerCase()) : null;
const preserveKey = `${rawSpeaker.toLowerCase()}|${text}`;
const carryover = preservedByKey.get(preserveKey);
if (carryover) preservedCount += 1;
lines.push({
id: `line-${String(lineNumber).padStart(3, '0')}`,
characterId: match?.id || null,
characterName: rawSpeaker || null,
text,
voiceIdOverride: carryover?.voiceIdOverride || null,
audioJobId: carryover?.audioJobId || null,
audioFilename: carryover?.audioFilename || null,
verification: carryover?.verification || null,
expectedSpeech: carryover?.expectedSpeech || null,
});
lineNumber += 1;
}
}
return { lines, preservedCount };
}
// wavDurationMs moved to lib/wavAudioFile.js (shared with the Music Designer code engine).
export { wavDurationMs };
/**
* Round-trip a rendered WAV through speech-to-text and compare it with the
* script (#10250). The empty decoder prompt keeps vocabulary bias from masking
* a mispronunciation. `expectedSpeech` is the user's accepted recognizer
* spelling and replaces the script text as the comparison target. STT being
* down or slow yields `unverified` — never an error, never blocks a render.
* @returns {Promise<{ status: 'matched'|'mismatch'|'unverified', similarity: number|null, heard: string }>}
*/
async function verifyRenderedLine({ wav, text, expectedSpeech, signal } = {}) {
const heardResult = await transcribe(wav, { prompt: '', signal }).catch((err) => {
console.warn(`⚠️ voice-over verification skipped: ${err?.message || err}`);
return null;
});
if (!heardResult) return { status: 'unverified', similarity: null, heard: '' };
const heard = heardResult.text;
const { status, similarity } = compareSpeech(expectedSpeech || text, heard);
return { status, similarity, heard };
}
export async function synthesizeToFile({ text, voiceId, profileId, route = 'studio', signal, expectedSpeech } = {}) {
const trimmed = (text || '').trim();
if (!trimmed) {
throw new ServerError('text is required', { status: 400, code: 'PIPELINE_AUDIO_EMPTY_TEXT' });
}
const { engine, voice } = parseVoiceId(voiceId);
const opts = { signal };
if (profileId) {
opts.profileId = profileId;
opts.route = route;
}
if (engine) opts.engine = engine;
if (voice) opts.voice = voice;
const {
wav, latencyMs, engine: usedEngine,
profileId: usedProfileId, profileRevision, provenance,
} = await synthesize(trimmed, opts);
await ensureDir(PATHS.audio);
// UUID filename keeps two simultaneous renders from colliding; the line's
// audioJobId-or-audioFilename binding lives in stages.audio.lines[].
const filename = `vo-${randomUUID()}.wav`;
await atomicWrite(join(PATHS.audio, filename), wav);
const verification = await verifyRenderedLine({ wav, text: trimmed, expectedSpeech, signal });
return {
filename,
verification,
latencyMs,
durationMs: wavDurationMs(wav),
engine: usedEngine,
voiceId: voice ? `${usedEngine}:${voice}` : null,
profileId: usedProfileId || null,
profileRevision: profileRevision || null,
provenance: provenance || null,
};
}