Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

20.5 KB
Raw Blame
const fs = require("fs");
const fsPromises = require("fs").promises;
const path = require("path");
const { spawn } = require("child_process");
const debugLogger = require("./debugLogger");
const { runSystemTar } = require("./systemTar");
const { downloadFile, createDownloadSignal, checkDiskSpace } = require("./downloadUtils");
const { resolveBinaryPath, gracefulStopProcess } = require("../utils/serverUtils");
const { getModelsDirForService } = require("./modelDirUtils");
const { convertToWav } = require("./ffmpegUtils");
const { getSafeTempDir } = require("./safeTempDir");
const { applyConfirmedSpeaker } = require("./speakerAssignmentPolicy");
const { buildSpeakerRenumber } = require("./speakerIndexBase");
const sidecarPidFile = require("./sidecarPidFile");
const {
  transcriptsOverlap,
  transcriptsLooselyOverlap,
  buildMergedCandidates,
} = require("./transcriptText");
const {
  computeTranscriptionTimeoutMs,
  PCM16_MONO_16K_BYTES_PER_SECOND,
} = require("./transcriptionTimeout");

const DIARIZATION_TIMEOUT_MS = 3600000; // 60 minutes
// splitMicEchoDuplicates runs on timestamps already normalized to seconds from
// the start of the recording, so this window is in seconds too, though
// buildMergedCandidates calls it windowMs.
const ECHO_DUPLICATE_WINDOW_S = 6;
const ECHO_DUPLICATE_MERGE_LIMIT = 3;

const SEGMENTATION_MODEL_URL =
  "https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2";
const EMBEDDING_MODEL_URL =
  "https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
const SILERO_VAD_MODEL_URL =
  "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx";

const SEGMENTATION_DIR = "sherpa-onnx-pyannote-segmentation-3-0";
const SEGMENTATION_ONNX = path.join(SEGMENTATION_DIR, "model.onnx");
const EMBEDDING_ONNX = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
const SILERO_VAD_ONNX = "silero_vad.onnx";

class DiarizationManager {
  constructor() {
    // Meeting post-processing and upload/batch diarization can overlap, so
    // track every live process, not a single slot.
    this._processes = new Set();
    this.currentDownloadProcess = null;
    this.cachedBinaryPath = null;
  }

  getBinaryPath() {
    if (this.cachedBinaryPath) return this.cachedBinaryPath;

    const platformArch = `${process.platform}-${process.arch}`;
    const binaryName =
      process.platform === "win32"
        ? `sherpa-onnx-diarize-${platformArch}.exe`
        : `sherpa-onnx-diarize-${platformArch}`;

    const resolved = resolveBinaryPath(binaryName);
    if (resolved) this.cachedBinaryPath = resolved;
    return resolved;
  }

  isAvailable() {
    return this.getBinaryPath() !== null && this.isModelDownloaded();
  }

  getModelsDir() {
    return getModelsDirForService("diarization");
  }

  getBundledModelsDir() {
    if (!process.resourcesPath) {
      return null;
    }

    return path.join(process.resourcesPath, "bin", "diarization-models");
  }

  _resolveModelPath(relativePath) {
    const bundledModelsDir = this.getBundledModelsDir();
    if (bundledModelsDir) {
      const bundledPath = path.join(bundledModelsDir, relativePath);
      if (fs.existsSync(bundledPath)) {
        return bundledPath;
      }
    }

    return path.join(this.getModelsDir(), relativePath);
  }

  isModelDownloaded() {
    const segPath = this._resolveModelPath(SEGMENTATION_ONNX);
    const embPath = this._resolveModelPath(EMBEDDING_ONNX);
    return fs.existsSync(segPath) && fs.existsSync(embPath);
  }

  getVadModelPath() {
    return this._resolveModelPath(SILERO_VAD_ONNX);
  }

  isVadModelDownloaded() {
    return fs.existsSync(this.getVadModelPath());
  }

  async downloadModels(progressCallback = null) {
    const modelsDir = this.getModelsDir();
    await fsPromises.mkdir(modelsDir, { recursive: true });

    const modelsReady = this.isModelDownloaded();
    const vadReady = this.isVadModelDownloaded();

    if (modelsReady && vadReady) {
      return { success: true, path: modelsDir };
    }

    const requiredBytes = modelsReady ? 2 * 1_000_000 : 37 * 1_000_000;
    const spaceCheck = await checkDiskSpace(modelsDir, requiredBytes * 2.5);
    if (!spaceCheck.ok) {
      throw new Error(
        `Not enough disk space. Need ~${Math.round((requiredBytes * 2.5) / 1_000_000)}MB, ` +
          `only ${Math.round(spaceCheck.availableBytes / 1_000_000)}MB available.`
      );
    }

    const { signal, abort } = createDownloadSignal();
    this.currentDownloadProcess = { abort };

    try {
      // Download segmentation model (tar.bz2)
      const segArchivePath = path.join(modelsDir, `${SEGMENTATION_DIR}.tar.bz2`);
      const segModelPath = path.join(modelsDir, SEGMENTATION_ONNX);

      if (!fs.existsSync(segModelPath)) {
        await downloadFile(SEGMENTATION_MODEL_URL, segArchivePath, {
          timeout: 600000,
          signal,
          onProgress: (downloadedBytes, totalBytes) => {
            if (progressCallback) {
              progressCallback({
                type: "progress",
                stage: "segmentation",
                downloaded_bytes: downloadedBytes,
                total_bytes: totalBytes,
                percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,
              });
            }
          },
        });

        // Extract tar.bz2
        if (progressCallback) {
          progressCallback({ type: "progress", stage: "extracting", percentage: 100 });
        }

        await this._extractTarBz2(segArchivePath, modelsDir);
        await fsPromises.unlink(segArchivePath).catch(() => {});

        if (!fs.existsSync(segModelPath)) {
          throw new Error("Segmentation model extraction failed: model.onnx not found");
        }
      }

      // Download embedding model (.onnx directly)
      const embModelPath = path.join(modelsDir, EMBEDDING_ONNX);

      if (!fs.existsSync(embModelPath)) {
        await downloadFile(EMBEDDING_MODEL_URL, embModelPath, {
          timeout: 600000,
          signal,
          onProgress: (downloadedBytes, totalBytes) => {
            if (progressCallback) {
              progressCallback({
                type: "progress",
                stage: "embedding",
                downloaded_bytes: downloadedBytes,
                total_bytes: totalBytes,
                percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,
              });
            }
          },
        });
      }

      if (!this.isVadModelDownloaded()) {
        try {
          await downloadFile(SILERO_VAD_MODEL_URL, this.getVadModelPath(), {
            timeout: 600000,
            signal,
            onProgress: (downloadedBytes, totalBytes) => {
              if (progressCallback) {
                progressCallback({
                  type: "progress",
                  stage: "vad",
                  downloaded_bytes: downloadedBytes,
                  total_bytes: totalBytes,
                  percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,
                });
              }
            },
          });
        } catch (error) {
          if (error.isAbort) {
            throw new Error("Download interrupted by user");
          }
          debugLogger.warn("Silero VAD model download failed", {
            error: error.message,
            modelsDir,
          });
        }
      }

      if (progressCallback) {
        progressCallback({ type: "complete", percentage: 100 });
      }

      debugLogger.info("Diarization models downloaded", { modelsDir });
      return { success: true, path: modelsDir };
    } catch (error) {
      if (error.isAbort) {
        throw new Error("Download interrupted by user");
      }
      if (progressCallback) {
        progressCallback({ type: "error", error: error.message });
      }
      throw error;
    } finally {
      this.currentDownloadProcess = null;
    }
  }

  async _extractTarBz2(archivePath, destDir) {
    try {
      await this._runSystemTar(archivePath, destDir);
      return;
    } catch (err) {
      debugLogger.debug("System tar failed, falling back to JS extraction", {
        error: err.message,
      });
    }

    const unbzip2 = require("unbzip2-stream");
    const tar = require("tar");
    const { pipeline } = require("stream/promises");
    await pipeline(fs.createReadStream(archivePath), unbzip2(), tar.x({ cwd: destDir }));
  }

  _runSystemTar(archivePath, destDir) {
    return runSystemTar(archivePath, destDir);
  }

  async cancelDownload() {
    if (this.currentDownloadProcess) {
      this.currentDownloadProcess.abort();
      this.currentDownloadProcess = null;
      return { success: true, message: "Download cancelled" };
    }
    return { success: false, error: "No active download to cancel" };
  }

  async diarize(wavPath, options = {}) {
    const { numSpeakers = -1, threshold = 0.55, signal = null } = options;

    if (signal?.aborted) return [];

    const binaryPath = this.getBinaryPath();
    if (!binaryPath) {
      debugLogger.warn("Diarization binary not found");
      return [];
    }

    if (!this.isModelDownloaded()) {
      debugLogger.warn("Diarization models not downloaded");
      return [];
    }

    if (!fs.existsSync(wavPath)) {
      debugLogger.warn("Diarization input file not found", { wavPath });
      return [];
    }

    const segPath = this._resolveModelPath(SEGMENTATION_ONNX);
    const embPath = this._resolveModelPath(EMBEDDING_ONNX);

    const args = [
      `--segmentation.pyannote-model=${segPath}`,
      `--embedding.model=${embPath}`,
      `--clustering.num-clusters=${numSpeakers}`,
      `--clustering.cluster-threshold=${threshold}`,
      "--min-duration-on=0.2",
      "--min-duration-off=0.5",
      wavPath,
    ];

    debugLogger.info("Starting diarization", {
      binaryPath,
      numSpeakers,
      threshold,
      wavPath,
    });

    // Scale with the recording length, but never below the 60-minute floor.
    let timeoutMs = DIARIZATION_TIMEOUT_MS;
    try {
      timeoutMs = Math.max(
        DIARIZATION_TIMEOUT_MS,
        computeTranscriptionTimeoutMs(fs.statSync(wavPath).size / PCM16_MONO_16K_BYTES_PER_SECOND)
      );
    } catch {
      // Unreadable WAV: keep the flat cap.
    }

    return new Promise((resolve) => {
      let stdout = "";
      let stderr = "";

      const proc = spawn(binaryPath, args, {
        stdio: ["ignore", "pipe", "pipe"],
        windowsHide: true,
        detached: process.platform !== "win32",
      });

      this._processes.add(proc);
      sidecarPidFile.write("diarization", proc.pid);

      // The single pid-file slot tracks whichever process is still alive; only
      // the reaper consumes it, and the Set is the real shutdown source.
      const untrack = () => {
        this._processes.delete(proc);
        const survivor = this._processes.values().next().value;
        if (survivor) sidecarPidFile.write("diarization", survivor.pid);
        else sidecarPidFile.clear("diarization");
      };

      const timeout = setTimeout(() => {
        debugLogger.warn("Diarization timed out", { timeoutMs });
        gracefulStopProcess(proc);
        resolve([]);
      }, timeoutMs);

      // A cancelled upload kills the child immediately (same stop the timeout
      // path uses); the close handler below still fires and cleans up.
      const onAbort = () => {
        debugLogger.info("Diarization cancelled, stopping process");
        gracefulStopProcess(proc);
        resolve([]);
      };
      signal?.addEventListener("abort", onAbort, { once: true });

      proc.stdout.on("data", (data) => {
        stdout += data.toString();
      });

      proc.stderr.on("data", (data) => {
        stderr += data.toString();
      });

      proc.on("close", (code) => {
        clearTimeout(timeout);
        signal?.removeEventListener("abort", onAbort);
        untrack();

        if (code !== 0) {
          debugLogger.warn("Diarization process exited with error", {
            code,
            stderr: stderr.slice(-500).trim(),
          });
          resolve([]);
          return;
        }

        const segments = this._parseOutput(stdout);
        debugLogger.info("Diarization complete", { segmentCount: segments.length });
        resolve(segments);
      });

      proc.on("error", (err) => {
        clearTimeout(timeout);
        signal?.removeEventListener("abort", onAbort);
        untrack();
        debugLogger.warn("Diarization process error", { error: err.message });
        resolve([]);
      });
    });
  }

  _parseOutput(stdout) {
    const segments = [];
    const lineRegex = /^(\d+\.?\d*)\s+--\s+(\d+\.?\d*)\s+(speaker_\d+)$/;

    for (const line of stdout.split("\n")) {
      const trimmed = line.trim();
      if (!trimmed) continue;
      const match = trimmed.match(lineRegex);
      if (match) {
        segments.push({
          start: parseFloat(match[1]),
          end: parseFloat(match[2]),
          speaker: match[3],
        });
      }
    }

    return segments;
  }

  // Splits out echo-flagged mic segments whose text matches nearby system audio.
  // `removed` holds the same objects that were passed in, so callers can map
  // them back to their original rows.
  splitMicEchoDuplicates(segments) {
    const systemSegments = segments.filter((seg) => seg.source === "system" && seg.text);
    if (!systemSegments.length) return { kept: segments, removed: [] };

    const kept = [];
    const removed = [];
    for (const seg of segments) {
      if (
        seg.source !== "mic" ||
        !seg.text ||
        (!seg.likelyRenderBleed && !seg.hasBleedEvidence && seg.suppressionReason !== "double_talk")
      ) {
        kept.push(seg);
        continue;
      }

      const matcher =
        seg.suppressionReason === "double_talk" ? transcriptsLooselyOverlap : transcriptsOverlap;
      const candidates = buildMergedCandidates({
        segments: systemSegments,
        timestamp: seg.timestamp,
        windowMs: ECHO_DUPLICATE_WINDOW_S,
        mergeLimit: ECHO_DUPLICATE_MERGE_LIMIT,
      });
      const isEcho = candidates.some((candidateText) => matcher(seg.text, candidateText));
      (isEcho ? removed : kept).push(seg);
    }
    return { kept, removed };
  }

  capSpeakerClusters(segments, cap) {
    if (!cap || !segments?.length) return segments;
    const totals = new Map();
    for (const s of segments) {
      totals.set(s.speaker, (totals.get(s.speaker) || 0) + (s.end - s.start));
    }
    if (totals.size <= cap) return segments;

    const ranked = [...totals.entries()].sort((a, b) => b[1] - a[1]);
    const keep = new Set(ranked.slice(0, cap).map(([sp]) => sp));
    const primary = ranked[0][0];
    return segments.map((s) => (keep.has(s.speaker) ? s : { ...s, speaker: primary }));
  }

  // Expects echo duplicates already removed with splitMicEchoDuplicates, so
  // the caller can report them to the renderer.
  mergeWithTranscript(
    transcriptSegments,
    diarizationSegments,
    { diarizedSource = "system", speakerIndexBase = 0 } = {}
  ) {
    if (!transcriptSegments || transcriptSegments.length === 0) return [];
    if (!diarizationSegments || diarizationSegments.length === 0) {
      return transcriptSegments.map((seg) => ({ ...seg }));
    }

    // Renumber clusters (speaker_00 → speaker_0), offset past a resumed note's ids.
    const speakerSet = new Set(diarizationSegments.map((d) => d.speaker));
    const speakerMap = buildSpeakerRenumber(speakerSet, speakerIndexBase);

    // Mic-mode softening: a single cluster on a mic-only session is the user
    // talking alone (e.g. a call where the remote never spoke audibly), so it
    // stays "you" instead of becoming a stranger's speaker_0.
    const micSingleCluster = diarizedSource === "mic" && speakerSet.size === 1;

    const nextTimestampAt = (startIndex, source) => {
      for (let i = startIndex + 1; i < transcriptSegments.length; i += 1) {
        const candidate = transcriptSegments[i];
        if (candidate.source === source && candidate.timestamp != null) {
          return candidate.timestamp;
        }
      }
      return null;
    };

    return transcriptSegments.map((seg, index) => {
      const enriched = { ...seg };

      if (seg.source === "mic" && (diarizedSource !== "mic" || micSingleCluster)) {
        applyConfirmedSpeaker(enriched, {
          speaker: "you",
          speakerIsPlaceholder: false,
        });
        return enriched;
      }

      if (seg.source === diarizedSource && seg.timestamp != null) {
        const segStart = seg.timestamp;
        const segEnd = nextTimestampAt(index, diarizedSource) ?? segStart + 2.5;
        const midpoint = segStart + (segEnd - segStart) / 2;
        let overlapSpeaker = null;
        let nearestSpeaker = null;
        let bestOverlap = 0;
        let bestDistance = Number.POSITIVE_INFINITY;

        for (const dSeg of diarizationSegments) {
          const overlap = Math.min(segEnd, dSeg.end) - Math.max(segStart, dSeg.start);
          if (overlap > bestOverlap) {
            bestOverlap = overlap;
            overlapSpeaker = dSeg.speaker;
          }

          const distance =
            midpoint < dSeg.start
              ? dSeg.start - midpoint
              : midpoint > dSeg.end
                ? midpoint - dSeg.end
                : 0;

          if (distance < bestDistance) {
            bestDistance = distance;
            nearestSpeaker = dSeg.speaker;
          }
        }

        // Tracked separately so the between-clusters fallback compares every
        // distance instead of latching onto the first cluster it sees.
        const bestSpeaker = overlapSpeaker ?? nearestSpeaker;

        if (bestSpeaker) {
          applyConfirmedSpeaker(enriched, {
            speaker: speakerMap.get(bestSpeaker) || bestSpeaker,
            speakerIsPlaceholder: false,
          });
        }
      }

      return enriched;
    });
  }

  async convertRawPcmToWav(rawPcmPath, inputSampleRate) {
    const stat = await fsPromises.stat(rawPcmPath);
    if (stat.size === 0) {
      throw new Error("Raw PCM file is empty");
    }

    const tempDir = getSafeTempDir();
    // Random suffix: concurrent conversions (meeting + upload) must never collide.
    const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
    const inputWavPath = path.join(tempDir, `ow-diarize-${runId}-input.wav`);
    const wavPath = path.join(tempDir, `ow-diarize-${runId}.wav`);

    // Stream: write 44-byte WAV header, then pipe raw PCM — avoids loading entire file into memory
    const header = this._createWavHeader(stat.size, inputSampleRate, 1);
    await new Promise((resolve, reject) => {
      const out = fs.createWriteStream(inputWavPath);
      out.write(header);
      const pcmStream = fs.createReadStream(rawPcmPath);
      pcmStream.pipe(out);
      out.on("finish", resolve);
      out.on("error", reject);
      pcmStream.on("error", reject);
    });

    try {
      await convertToWav(inputWavPath, wavPath, { sampleRate: 16000, channels: 1 });
    } finally {
      await fsPromises.unlink(inputWavPath).catch(() => {});
    }

    debugLogger.debug("Raw PCM converted to WAV for diarization", {
      wavPath,
      rawPcmBytes: stat.size,
    });

    return wavPath;
  }

  _createWavHeader(dataSize, sampleRate, channels) {
    const bytesPerSample = 2;
    const blockAlign = channels * bytesPerSample;
    const byteRate = sampleRate * blockAlign;
    const header = Buffer.alloc(44);

    header.write("RIFF", 0);
    header.writeUInt32LE(36 + dataSize, 4);
    header.write("WAVE", 8);
    header.write("fmt ", 12);
    header.writeUInt32LE(16, 16);
    header.writeUInt16LE(1, 20);
    header.writeUInt16LE(channels, 22);
    header.writeUInt32LE(sampleRate, 24);
    header.writeUInt32LE(byteRate, 28);
    header.writeUInt16LE(blockAlign, 32);
    header.writeUInt16LE(bytesPerSample * 8, 34);
    header.write("data", 36);
    header.writeUInt32LE(dataSize, 40);

    return header;
  }

  async deleteModels() {
    const modelsDir = this.getModelsDir();
    const segDir = path.join(modelsDir, SEGMENTATION_DIR);
    const embPath = path.join(modelsDir, EMBEDDING_ONNX);
    const vadPath = this.getVadModelPath();

    if (fs.existsSync(segDir)) {
      await fsPromises.rm(segDir, { recursive: true, force: true });
    }
    if (fs.existsSync(embPath)) {
      await fsPromises.unlink(embPath);
    }
    if (fs.existsSync(vadPath)) {
      await fsPromises.unlink(vadPath);
    }

    debugLogger.info("Diarization models deleted", { modelsDir });
    return { success: true };
  }

  async shutdown() {
    const procs = [...this._processes];
    this._processes.clear();
    await Promise.all(procs.map((p) => gracefulStopProcess(p)));
  }
}

module.exports = DiarizationManager;