Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
const fs = require("fs");
const fsPromises = require("fs").promises;
const path = require("path");
const { spawn } = require("child_process");
const debugLogger = require("./debugLogger");
const { runSystemTar } = require("./systemTar");
const { downloadFile, createDownloadSignal, checkDiskSpace } = require("./downloadUtils");
const { resolveBinaryPath, gracefulStopProcess } = require("../utils/serverUtils");
const { getModelsDirForService } = require("./modelDirUtils");
const { convertToWav } = require("./ffmpegUtils");
const { getSafeTempDir } = require("./safeTempDir");
const { applyConfirmedSpeaker } = require("./speakerAssignmentPolicy");
const { buildSpeakerRenumber } = require("./speakerIndexBase");
const sidecarPidFile = require("./sidecarPidFile");
const {
transcriptsOverlap,
transcriptsLooselyOverlap,
buildMergedCandidates,
} = require("./transcriptText");
const {
computeTranscriptionTimeoutMs,
PCM16_MONO_16K_BYTES_PER_SECOND,
} = require("./transcriptionTimeout");
const DIARIZATION_TIMEOUT_MS = 3600000; // 60 minutes
// splitMicEchoDuplicates runs on timestamps already normalized to seconds from
// the start of the recording, so this window is in seconds too, though
// buildMergedCandidates calls it windowMs.
const ECHO_DUPLICATE_WINDOW_S = 6;
const ECHO_DUPLICATE_MERGE_LIMIT = 3;
const SEGMENTATION_MODEL_URL =
"https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2";
const EMBEDDING_MODEL_URL =
"https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
const SILERO_VAD_MODEL_URL =
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx";
const SEGMENTATION_DIR = "sherpa-onnx-pyannote-segmentation-3-0";
const SEGMENTATION_ONNX = path.join(SEGMENTATION_DIR, "model.onnx");
const EMBEDDING_ONNX = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
const SILERO_VAD_ONNX = "silero_vad.onnx";
class DiarizationManager {
constructor() {
// Meeting post-processing and upload/batch diarization can overlap, so
// track every live process, not a single slot.
this._processes = new Set();
this.currentDownloadProcess = null;
this.cachedBinaryPath = null;
}
getBinaryPath() {
if (this.cachedBinaryPath) return this.cachedBinaryPath;
const platformArch = `${process.platform}-${process.arch}`;
const binaryName =
process.platform === "win32"
? `sherpa-onnx-diarize-${platformArch}.exe`
: `sherpa-onnx-diarize-${platformArch}`;
const resolved = resolveBinaryPath(binaryName);
if (resolved) this.cachedBinaryPath = resolved;
return resolved;
}
isAvailable() {
return this.getBinaryPath() !== null && this.isModelDownloaded();
}
getModelsDir() {
return getModelsDirForService("diarization");
}
getBundledModelsDir() {
if (!process.resourcesPath) {
return null;
}
return path.join(process.resourcesPath, "bin", "diarization-models");
}
_resolveModelPath(relativePath) {
const bundledModelsDir = this.getBundledModelsDir();
if (bundledModelsDir) {
const bundledPath = path.join(bundledModelsDir, relativePath);
if (fs.existsSync(bundledPath)) {
return bundledPath;
}
}
return path.join(this.getModelsDir(), relativePath);
}
isModelDownloaded() {
const segPath = this._resolveModelPath(SEGMENTATION_ONNX);
const embPath = this._resolveModelPath(EMBEDDING_ONNX);
return fs.existsSync(segPath) && fs.existsSync(embPath);
}
getVadModelPath() {
return this._resolveModelPath(SILERO_VAD_ONNX);
}
isVadModelDownloaded() {
return fs.existsSync(this.getVadModelPath());
}
async downloadModels(progressCallback = null) {
const modelsDir = this.getModelsDir();
await fsPromises.mkdir(modelsDir, { recursive: true });
const modelsReady = this.isModelDownloaded();
const vadReady = this.isVadModelDownloaded();
if (modelsReady && vadReady) {
return { success: true, path: modelsDir };
}
const requiredBytes = modelsReady ? 2 * 1_000_000 : 37 * 1_000_000;
const spaceCheck = await checkDiskSpace(modelsDir, requiredBytes * 2.5);
if (!spaceCheck.ok) {
throw new Error(
`Not enough disk space. Need ~${Math.round((requiredBytes * 2.5) / 1_000_000)}MB, ` +
`only ${Math.round(spaceCheck.availableBytes / 1_000_000)}MB available.`
);
}
const { signal, abort } = createDownloadSignal();
this.currentDownloadProcess = { abort };
try {
// Download segmentation model (tar.bz2)
const segArchivePath = path.join(modelsDir, `${SEGMENTATION_DIR}.tar.bz2`);
const segModelPath = path.join(modelsDir, SEGMENTATION_ONNX);
if (!fs.existsSync(segModelPath)) {
await downloadFile(SEGMENTATION_MODEL_URL, segArchivePath, {
timeout: 600000,
signal,
onProgress: (downloadedBytes, totalBytes) => {
if (progressCallback) {
progressCallback({
type: "progress",
stage: "segmentation",
downloaded_bytes: downloadedBytes,
total_bytes: totalBytes,
percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,
});
}
},
});
// Extract tar.bz2
if (progressCallback) {
progressCallback({ type: "progress", stage: "extracting", percentage: 100 });
}
await this._extractTarBz2(segArchivePath, modelsDir);
await fsPromises.unlink(segArchivePath).catch(() => {});
if (!fs.existsSync(segModelPath)) {
throw new Error("Segmentation model extraction failed: model.onnx not found");
}
}
// Download embedding model (.onnx directly)
const embModelPath = path.join(modelsDir, EMBEDDING_ONNX);
if (!fs.existsSync(embModelPath)) {
await downloadFile(EMBEDDING_MODEL_URL, embModelPath, {
timeout: 600000,
signal,
onProgress: (downloadedBytes, totalBytes) => {
if (progressCallback) {
progressCallback({
type: "progress",
stage: "embedding",
downloaded_bytes: downloadedBytes,
total_bytes: totalBytes,
percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,
});
}
},
});
}
if (!this.isVadModelDownloaded()) {
try {
await downloadFile(SILERO_VAD_MODEL_URL, this.getVadModelPath(), {
timeout: 600000,
signal,
onProgress: (downloadedBytes, totalBytes) => {
if (progressCallback) {
progressCallback({
type: "progress",
stage: "vad",
downloaded_bytes: downloadedBytes,
total_bytes: totalBytes,
percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,
});
}
},
});
} catch (error) {
if (error.isAbort) {
throw new Error("Download interrupted by user");
}
debugLogger.warn("Silero VAD model download failed", {
error: error.message,
modelsDir,
});
}
}
if (progressCallback) {
progressCallback({ type: "complete", percentage: 100 });
}
debugLogger.info("Diarization models downloaded", { modelsDir });
return { success: true, path: modelsDir };
} catch (error) {
if (error.isAbort) {
throw new Error("Download interrupted by user");
}
if (progressCallback) {
progressCallback({ type: "error", error: error.message });
}
throw error;
} finally {
this.currentDownloadProcess = null;
}
}
async _extractTarBz2(archivePath, destDir) {
try {
await this._runSystemTar(archivePath, destDir);
return;
} catch (err) {
debugLogger.debug("System tar failed, falling back to JS extraction", {
error: err.message,
});
}
const unbzip2 = require("unbzip2-stream");
const tar = require("tar");
const { pipeline } = require("stream/promises");
await pipeline(fs.createReadStream(archivePath), unbzip2(), tar.x({ cwd: destDir }));
}
_runSystemTar(archivePath, destDir) {
return runSystemTar(archivePath, destDir);
}
async cancelDownload() {
if (this.currentDownloadProcess) {
this.currentDownloadProcess.abort();
this.currentDownloadProcess = null;
return { success: true, message: "Download cancelled" };
}
return { success: false, error: "No active download to cancel" };
}
async diarize(wavPath, options = {}) {
const { numSpeakers = -1, threshold = 0.55, signal = null } = options;
if (signal?.aborted) return [];
const binaryPath = this.getBinaryPath();
if (!binaryPath) {
debugLogger.warn("Diarization binary not found");
return [];
}
if (!this.isModelDownloaded()) {
debugLogger.warn("Diarization models not downloaded");
return [];
}
if (!fs.existsSync(wavPath)) {
debugLogger.warn("Diarization input file not found", { wavPath });
return [];
}
const segPath = this._resolveModelPath(SEGMENTATION_ONNX);
const embPath = this._resolveModelPath(EMBEDDING_ONNX);
const args = [
`--segmentation.pyannote-model=${segPath}`,
`--embedding.model=${embPath}`,
`--clustering.num-clusters=${numSpeakers}`,
`--clustering.cluster-threshold=${threshold}`,
"--min-duration-on=0.2",
"--min-duration-off=0.5",
wavPath,
];
debugLogger.info("Starting diarization", {
binaryPath,
numSpeakers,
threshold,
wavPath,
});
// Scale with the recording length, but never below the 60-minute floor.
let timeoutMs = DIARIZATION_TIMEOUT_MS;
try {
timeoutMs = Math.max(
DIARIZATION_TIMEOUT_MS,
computeTranscriptionTimeoutMs(fs.statSync(wavPath).size / PCM16_MONO_16K_BYTES_PER_SECOND)
);
} catch {
// Unreadable WAV: keep the flat cap.
}
return new Promise((resolve) => {
let stdout = "";
let stderr = "";
const proc = spawn(binaryPath, args, {
stdio: ["ignore", "pipe", "pipe"],
windowsHide: true,
detached: process.platform !== "win32",
});
this._processes.add(proc);
sidecarPidFile.write("diarization", proc.pid);
// The single pid-file slot tracks whichever process is still alive; only
// the reaper consumes it, and the Set is the real shutdown source.
const untrack = () => {
this._processes.delete(proc);
const survivor = this._processes.values().next().value;
if (survivor) sidecarPidFile.write("diarization", survivor.pid);
else sidecarPidFile.clear("diarization");
};
const timeout = setTimeout(() => {
debugLogger.warn("Diarization timed out", { timeoutMs });
gracefulStopProcess(proc);
resolve([]);
}, timeoutMs);
// A cancelled upload kills the child immediately (same stop the timeout
// path uses); the close handler below still fires and cleans up.
const onAbort = () => {
debugLogger.info("Diarization cancelled, stopping process");
gracefulStopProcess(proc);
resolve([]);
};
signal?.addEventListener("abort", onAbort, { once: true });
proc.stdout.on("data", (data) => {
stdout += data.toString();
});
proc.stderr.on("data", (data) => {
stderr += data.toString();
});
proc.on("close", (code) => {
clearTimeout(timeout);
signal?.removeEventListener("abort", onAbort);
untrack();
if (code !== 0) {
debugLogger.warn("Diarization process exited with error", {
code,
stderr: stderr.slice(-500).trim(),
});
resolve([]);
return;
}
const segments = this._parseOutput(stdout);
debugLogger.info("Diarization complete", { segmentCount: segments.length });
resolve(segments);
});
proc.on("error", (err) => {
clearTimeout(timeout);
signal?.removeEventListener("abort", onAbort);
untrack();
debugLogger.warn("Diarization process error", { error: err.message });
resolve([]);
});
});
}
_parseOutput(stdout) {
const segments = [];
const lineRegex = /^(\d+\.?\d*)\s+--\s+(\d+\.?\d*)\s+(speaker_\d+)$/;
for (const line of stdout.split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
const match = trimmed.match(lineRegex);
if (match) {
segments.push({
start: parseFloat(match[1]),
end: parseFloat(match[2]),
speaker: match[3],
});
}
}
return segments;
}
// Splits out echo-flagged mic segments whose text matches nearby system audio.
// `removed` holds the same objects that were passed in, so callers can map
// them back to their original rows.
splitMicEchoDuplicates(segments) {
const systemSegments = segments.filter((seg) => seg.source === "system" && seg.text);
if (!systemSegments.length) return { kept: segments, removed: [] };
const kept = [];
const removed = [];
for (const seg of segments) {
if (
seg.source !== "mic" ||
!seg.text ||
(!seg.likelyRenderBleed && !seg.hasBleedEvidence && seg.suppressionReason !== "double_talk")
) {
kept.push(seg);
continue;
}
const matcher =
seg.suppressionReason === "double_talk" ? transcriptsLooselyOverlap : transcriptsOverlap;
const candidates = buildMergedCandidates({
segments: systemSegments,
timestamp: seg.timestamp,
windowMs: ECHO_DUPLICATE_WINDOW_S,
mergeLimit: ECHO_DUPLICATE_MERGE_LIMIT,
});
const isEcho = candidates.some((candidateText) => matcher(seg.text, candidateText));
(isEcho ? removed : kept).push(seg);
}
return { kept, removed };
}
capSpeakerClusters(segments, cap) {
if (!cap || !segments?.length) return segments;
const totals = new Map();
for (const s of segments) {
totals.set(s.speaker, (totals.get(s.speaker) || 0) + (s.end - s.start));
}
if (totals.size <= cap) return segments;
const ranked = [...totals.entries()].sort((a, b) => b[1] - a[1]);
const keep = new Set(ranked.slice(0, cap).map(([sp]) => sp));
const primary = ranked[0][0];
return segments.map((s) => (keep.has(s.speaker) ? s : { ...s, speaker: primary }));
}
// Expects echo duplicates already removed with splitMicEchoDuplicates, so
// the caller can report them to the renderer.
mergeWithTranscript(
transcriptSegments,
diarizationSegments,
{ diarizedSource = "system", speakerIndexBase = 0 } = {}
) {
if (!transcriptSegments || transcriptSegments.length === 0) return [];
if (!diarizationSegments || diarizationSegments.length === 0) {
return transcriptSegments.map((seg) => ({ ...seg }));
}
// Renumber clusters (speaker_00 → speaker_0), offset past a resumed note's ids.
const speakerSet = new Set(diarizationSegments.map((d) => d.speaker));
const speakerMap = buildSpeakerRenumber(speakerSet, speakerIndexBase);
// Mic-mode softening: a single cluster on a mic-only session is the user
// talking alone (e.g. a call where the remote never spoke audibly), so it
// stays "you" instead of becoming a stranger's speaker_0.
const micSingleCluster = diarizedSource === "mic" && speakerSet.size === 1;
const nextTimestampAt = (startIndex, source) => {
for (let i = startIndex + 1; i < transcriptSegments.length; i += 1) {
const candidate = transcriptSegments[i];
if (candidate.source === source && candidate.timestamp != null) {
return candidate.timestamp;
}
}
return null;
};
return transcriptSegments.map((seg, index) => {
const enriched = { ...seg };
if (seg.source === "mic" && (diarizedSource !== "mic" || micSingleCluster)) {
applyConfirmedSpeaker(enriched, {
speaker: "you",
speakerIsPlaceholder: false,
});
return enriched;
}
if (seg.source === diarizedSource && seg.timestamp != null) {
const segStart = seg.timestamp;
const segEnd = nextTimestampAt(index, diarizedSource) ?? segStart + 2.5;
const midpoint = segStart + (segEnd - segStart) / 2;
let overlapSpeaker = null;
let nearestSpeaker = null;
let bestOverlap = 0;
let bestDistance = Number.POSITIVE_INFINITY;
for (const dSeg of diarizationSegments) {
const overlap = Math.min(segEnd, dSeg.end) - Math.max(segStart, dSeg.start);
if (overlap > bestOverlap) {
bestOverlap = overlap;
overlapSpeaker = dSeg.speaker;
}
const distance =
midpoint < dSeg.start
? dSeg.start - midpoint
: midpoint > dSeg.end
? midpoint - dSeg.end
: 0;
if (distance < bestDistance) {
bestDistance = distance;
nearestSpeaker = dSeg.speaker;
}
}
// Tracked separately so the between-clusters fallback compares every
// distance instead of latching onto the first cluster it sees.
const bestSpeaker = overlapSpeaker ?? nearestSpeaker;
if (bestSpeaker) {
applyConfirmedSpeaker(enriched, {
speaker: speakerMap.get(bestSpeaker) || bestSpeaker,
speakerIsPlaceholder: false,
});
}
}
return enriched;
});
}
async convertRawPcmToWav(rawPcmPath, inputSampleRate) {
const stat = await fsPromises.stat(rawPcmPath);
if (stat.size === 0) {
throw new Error("Raw PCM file is empty");
}
const tempDir = getSafeTempDir();
// Random suffix: concurrent conversions (meeting + upload) must never collide.
const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const inputWavPath = path.join(tempDir, `ow-diarize-${runId}-input.wav`);
const wavPath = path.join(tempDir, `ow-diarize-${runId}.wav`);
// Stream: write 44-byte WAV header, then pipe raw PCM — avoids loading entire file into memory
const header = this._createWavHeader(stat.size, inputSampleRate, 1);
await new Promise((resolve, reject) => {
const out = fs.createWriteStream(inputWavPath);
out.write(header);
const pcmStream = fs.createReadStream(rawPcmPath);
pcmStream.pipe(out);
out.on("finish", resolve);
out.on("error", reject);
pcmStream.on("error", reject);
});
try {
await convertToWav(inputWavPath, wavPath, { sampleRate: 16000, channels: 1 });
} finally {
await fsPromises.unlink(inputWavPath).catch(() => {});
}
debugLogger.debug("Raw PCM converted to WAV for diarization", {
wavPath,
rawPcmBytes: stat.size,
});
return wavPath;
}
_createWavHeader(dataSize, sampleRate, channels) {
const bytesPerSample = 2;
const blockAlign = channels * bytesPerSample;
const byteRate = sampleRate * blockAlign;
const header = Buffer.alloc(44);
header.write("RIFF", 0);
header.writeUInt32LE(36 + dataSize, 4);
header.write("WAVE", 8);
header.write("fmt ", 12);
header.writeUInt32LE(16, 16);
header.writeUInt16LE(1, 20);
header.writeUInt16LE(channels, 22);
header.writeUInt32LE(sampleRate, 24);
header.writeUInt32LE(byteRate, 28);
header.writeUInt16LE(blockAlign, 32);
header.writeUInt16LE(bytesPerSample * 8, 34);
header.write("data", 36);
header.writeUInt32LE(dataSize, 40);
return header;
}
async deleteModels() {
const modelsDir = this.getModelsDir();
const segDir = path.join(modelsDir, SEGMENTATION_DIR);
const embPath = path.join(modelsDir, EMBEDDING_ONNX);
const vadPath = this.getVadModelPath();
if (fs.existsSync(segDir)) {
await fsPromises.rm(segDir, { recursive: true, force: true });
}
if (fs.existsSync(embPath)) {
await fsPromises.unlink(embPath);
}
if (fs.existsSync(vadPath)) {
await fsPromises.unlink(vadPath);
}
debugLogger.info("Diarization models deleted", { modelsDir });
return { success: true };
}
async shutdown() {
const procs = [...this._processes];
this._processes.clear();
await Promise.all(procs.map((p) => gracefulStopProcess(p)));
}
}
module.exports = DiarizationManager;