Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(diarization): live speaker identification pipeline bugs
Gabriel Stein458aa50954cb
diff --git a/electron-builder.json b/electron-builder.jsonindex 4502e751f..47a1a7b02 100644--- a/electron-builder.json+++ b/electron-builder.json@@ -109,6 +109,7 @@"windows-text-monitor*","windows-fast-paste*","all-MiniLM-L6-v2/**/*",+ "diarization-models/**/*","*.dylib","*.dll","*.so*"diff --git a/main.js b/main.jsindex e156a79fb..4a587035e 100644--- a/main.js+++ b/main.js@@ -685,7 +685,10 @@ async function startApp() {}// Auto-download diarization models if binary is available- if (diarizationManager.getBinaryPath() && !diarizationManager.isModelDownloaded()) {+ if (+ diarizationManager.getBinaryPath() &&+ (!diarizationManager.isModelDownloaded() || !diarizationManager.isVadModelDownloaded())+ ) {diarizationManager.downloadModels().catch((err) => {debugLogger.debug("Diarization model auto-download error (non-fatal)", {error: err.message,diff --git a/package.json b/package.jsonindex 9851c0703..a3ab8d6e0 100644--- a/package.json+++ b/package.json@@ -25,25 +25,25 @@"predev:main": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model","dev:main": "cross-env NODE_ENV=development node scripts/run-electron.js --dev","dev:renderer": "cd src && vite",- "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model -- --for-build",+ "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model -- --for-build && npm run download:diarization-models -- --output-dir resources/bin/diarization-models","build": "cd src && vite build && cd .. && electron-builder","postinstall": "electron-builder install-app-deps","build:renderer": "cd src && vite build",- "prebuild:mac": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",+ "prebuild:mac": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models","build:mac": "npm run build:renderer && electron-builder --mac","build:mac:arm64": "npm run build:renderer && electron-builder --mac --arm64","build:mac:x64": "npm run build:renderer && electron-builder --mac --x64",- "prebuild:win": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:nircmd && npm run download:windows-fast-paste && npm run download:windows-key-listener && npm run download:windows-mic-listener",+ "prebuild:win": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models && npm run download:nircmd && npm run download:windows-fast-paste && npm run download:windows-key-listener && npm run download:windows-mic-listener","build:win": "npm run build:renderer && electron-builder --win",- "prebuild:linux": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",+ "prebuild:linux": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models","build:linux": "npm run build:renderer && electron-builder --linux","build:linux:appimage": "npm run build:renderer && electron-builder --linux AppImage","build:linux:deb": "npm run build:renderer && electron-builder --linux deb","build:linux:rpm": "npm run build:renderer && electron-builder --linux rpm","build:linux:tar": "npm run build:renderer && electron-builder --linux tar.gz",- "prepack": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",+ "prepack": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models","pack": "npm run build:renderer && cross-env CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --dir",- "predist": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",+ "predist": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models","dist": "npm run build:renderer && electron-builder","download:whisper-cpp": "node scripts/download-whisper-cpp.js --current","download:whisper-cpp:all": "node scripts/download-whisper-cpp.js --all",diff --git a/preload.js b/preload.jsindex a297c95cd..f5540ba68 100644--- a/preload.js+++ b/preload.js@@ -514,6 +514,10 @@ contextBridge.exposeInMainWorld("electronAPI", {"meeting-transcription-segment",(callback) => (_event, data) => callback(data)),+ onMeetingSpeakerIdentified: registerListener(+ "meeting-speaker-identified",+ (callback) => (_event, data) => callback(data)+ ),onMeetingTranscriptionError: registerListener("meeting-transcription-error",(callback) => (_event, data) => callback(data)diff --git a/scripts/download-diarization-models.js b/scripts/download-diarization-models.jsindex b19c029c2..4451deecc 100644--- a/scripts/download-diarization-models.js+++ b/scripts/download-diarization-models.js@@ -5,8 +5,6 @@ const path = require("path");const { execFileSync } = require("child_process");const { downloadFile, parseArgs } = require("./lib/download-utils");-const MODEL_DIR = path.join(os.homedir(), ".cache", "openwhispr", "diarization-models");-const SEGMENTATION_URL ="https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2";const SEGMENTATION_DIR = "sherpa-onnx-pyannote-segmentation-3-0";@@ -15,6 +13,22 @@ const SEGMENTATION_FILE = "model.onnx";const EMBEDDING_URL ="https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";const EMBEDDING_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";+const VAD_URL =+ "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx";+const VAD_FILE = "silero_vad.onnx";++function getModelDir() {+ const outputDirIndex = process.argv.indexOf("--output-dir");+ if (outputDirIndex !== -1 && process.argv[outputDirIndex + 1]) {+ return path.resolve(process.argv[outputDirIndex + 1]);+ }++ if (process.env.DIARIZATION_MODEL_DIR) {+ return path.resolve(process.env.DIARIZATION_MODEL_DIR);+ }++ return path.join(os.homedir(), ".cache", "openwhispr", "diarization-models");+}function extractTarBz2(archivePath, destDir) {fs.mkdirSync(destDir, { recursive: true });@@ -29,28 +43,31 @@ async function main() {console.log("\n[diarization-models] Downloading diarization models...\n");const args = parseArgs();+ const modelDir = getModelDir();- const segModelPath = path.join(MODEL_DIR, SEGMENTATION_DIR, SEGMENTATION_FILE);- const embModelPath = path.join(MODEL_DIR, EMBEDDING_FILE);+ const segModelPath = path.join(modelDir, SEGMENTATION_DIR, SEGMENTATION_FILE);+ const embModelPath = path.join(modelDir, EMBEDDING_FILE);+ const vadModelPath = path.join(modelDir, VAD_FILE);- const allExist = fs.existsSync(segModelPath) && fs.existsSync(embModelPath);+ const allExist =+ fs.existsSync(segModelPath) && fs.existsSync(embModelPath) && fs.existsSync(vadModelPath);if (allExist && !args.isForce) {console.log("[diarization-models] Model files already exist (use --force to re-download)\n");return;}- fs.mkdirSync(MODEL_DIR, { recursive: true });+ fs.mkdirSync(modelDir, { recursive: true });// Download segmentation model (tar.bz2 archive)if (!fs.existsSync(segModelPath) || args.isForce) {console.log(`[diarization-models] Downloading segmentation model from ${SEGMENTATION_URL}`);- const archivePath = path.join(MODEL_DIR, `${SEGMENTATION_DIR}.tar.bz2`);+ const archivePath = path.join(modelDir, `${SEGMENTATION_DIR}.tar.bz2`);try {await downloadFile(SEGMENTATION_URL, archivePath);- const extractDir = path.join(MODEL_DIR, "temp-segmentation");+ const extractDir = path.join(modelDir, "temp-segmentation");fs.mkdirSync(extractDir, { recursive: true });extractTarBz2(archivePath, extractDir);@@ -62,7 +79,7 @@ async function main() {return;}- const destDir = path.join(MODEL_DIR, SEGMENTATION_DIR);+ const destDir = path.join(modelDir, SEGMENTATION_DIR);fs.mkdirSync(destDir, { recursive: true });fs.copyFileSync(extractedModelPath, segModelPath);@@ -102,7 +119,25 @@ async function main() {console.log(`[diarization-models] Embedding model already exists, skipping`);}- console.log(`\n[diarization-models] Models ready at ${MODEL_DIR}\n`);+ if (!fs.existsSync(vadModelPath) || args.isForce) {+ console.log(`[diarization-models] Downloading VAD model from ${VAD_URL}`);++ try {+ await downloadFile(VAD_URL, vadModelPath);+ const stats = fs.statSync(vadModelPath);+ const sizeMB = (stats.size / 1024 / 1024).toFixed(1);+ console.log(`[diarization-models] VAD model downloaded (${sizeMB}MB)`);+ } catch (error) {+ console.error(`[diarization-models] Failed to download VAD model: ${error.message}`);+ if (fs.existsSync(vadModelPath)) fs.unlinkSync(vadModelPath);+ process.exitCode = 1;+ return;+ }+ } else {+ console.log("[diarization-models] VAD model already exists, skipping");+ }++ console.log(`\n[diarization-models] Models ready at ${modelDir}\n`);}main().catch(console.error);diff --git a/src/helpers/diarization.js b/src/helpers/diarization.jsindex 59f40d948..ae8d5522f 100644--- a/src/helpers/diarization.js+++ b/src/helpers/diarization.js@@ -15,10 +15,13 @@ const SEGMENTATION_MODEL_URL ="https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2";const EMBEDDING_MODEL_URL ="https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";+const SILERO_VAD_MODEL_URL =+ "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx";const SEGMENTATION_DIR = "sherpa-onnx-pyannote-segmentation-3-0";const SEGMENTATION_ONNX = path.join(SEGMENTATION_DIR, "model.onnx");const EMBEDDING_ONNX = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";+const SILERO_VAD_ONNX = "silero_vad.onnx";class DiarizationManager {constructor() {@@ -49,22 +52,52 @@ class DiarizationManager {return getModelsDirForService("diarization");}+ getBundledModelsDir() {+ if (!process.resourcesPath) {+ return null;+ }++ return path.join(process.resourcesPath, "bin", "diarization-models");+ }++ _resolveModelPath(relativePath) {+ const bundledModelsDir = this.getBundledModelsDir();+ if (bundledModelsDir) {+ const bundledPath = path.join(bundledModelsDir, relativePath);+ if (fs.existsSync(bundledPath)) {+ return bundledPath;+ }+ }++ return path.join(this.getModelsDir(), relativePath);+ }+isModelDownloaded() {- const modelsDir = this.getModelsDir();- const segPath = path.join(modelsDir, SEGMENTATION_ONNX);- const embPath = path.join(modelsDir, EMBEDDING_ONNX);+ const segPath = this._resolveModelPath(SEGMENTATION_ONNX);+ const embPath = this._resolveModelPath(EMBEDDING_ONNX);return fs.existsSync(segPath) && fs.existsSync(embPath);}+ getVadModelPath() {+ return this._resolveModelPath(SILERO_VAD_ONNX);+ }++ isVadModelDownloaded() {+ return fs.existsSync(this.getVadModelPath());+ }+async downloadModels(progressCallback = null) {const modelsDir = this.getModelsDir();await fsPromises.mkdir(modelsDir, { recursive: true });- if (this.isModelDownloaded()) {+ const modelsReady = this.isModelDownloaded();+ const vadReady = this.isVadModelDownloaded();++ if (modelsReady && vadReady) {return { success: true, path: modelsDir };}- const requiredBytes = 35 * 1_000_000; // ~35MB total+ const requiredBytes = modelsReady ? 2 * 1_000_000 : 37 * 1_000_000;const spaceCheck = await checkDiskSpace(modelsDir, requiredBytes * 2.5);if (!spaceCheck.ok) {throw new Error(@@ -132,6 +165,34 @@ class DiarizationManager {});}+ if (!this.isVadModelDownloaded()) {+ try {+ await downloadFile(SILERO_VAD_MODEL_URL, this.getVadModelPath(), {+ timeout: 600000,+ signal,+ onProgress: (downloadedBytes, totalBytes) => {+ if (progressCallback) {+ progressCallback({+ type: "progress",+ stage: "vad",+ downloaded_bytes: downloadedBytes,+ total_bytes: totalBytes,+ percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,+ });+ }+ },+ });+ } catch (error) {+ if (error.isAbort) {+ throw new Error("Download interrupted by user");+ }+ debugLogger.warn("Silero VAD model download failed", {+ error: error.message,+ modelsDir,+ });+ }+ }+if (progressCallback) {progressCallback({ type: "complete", percentage: 100 });}@@ -226,9 +287,8 @@ class DiarizationManager {return [];}- const modelsDir = this.getModelsDir();- const segPath = path.join(modelsDir, SEGMENTATION_ONNX);- const embPath = path.join(modelsDir, EMBEDDING_ONNX);+ const segPath = this._resolveModelPath(SEGMENTATION_ONNX);+ const embPath = this._resolveModelPath(EMBEDDING_ONNX);const args = [`--segmentation.pyannote-model=${segPath}`,@@ -431,6 +491,7 @@ class DiarizationManager {const modelsDir = this.getModelsDir();const segDir = path.join(modelsDir, SEGMENTATION_DIR);const embPath = path.join(modelsDir, EMBEDDING_ONNX);+ const vadPath = this.getVadModelPath();if (fs.existsSync(segDir)) {await fsPromises.rm(segDir, { recursive: true, force: true });@@ -438,6 +499,9 @@ class DiarizationManager {if (fs.existsSync(embPath)) {await fsPromises.unlink(embPath);}+ if (fs.existsSync(vadPath)) {+ await fsPromises.unlink(vadPath);+ }debugLogger.info("Diarization models deleted", { modelsDir });return { success: true };diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 91453cccf..0848556e9 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -11,6 +11,8 @@ const { i18nMain, changeLanguage } = require("./i18nMain");const DeepgramStreaming = require("./deepgramStreaming");const OpenAIRealtimeStreaming = require("./openaiRealtimeStreaming");const AudioStorageManager = require("./audioStorage");+const liveSpeakerIdentifier = require("./liveSpeakerIdentifier");+const { downsample24kTo16k, pcm16ToWav } = require("../utils/audioUtils");const MISTRAL_TRANSCRIPTION_URL = "https://api.mistral.ai/v1/audio/transcriptions";@@ -119,6 +121,7 @@ class IPCHandlers {this.audioStorageManager = new AudioStorageManager();this._audioCleanupInterval = null;this._noteFilesEnabled = false;+ liveSpeakerIdentifier.setDiarizationManager(this.diarizationManager);this._setupTextEditMonitor();this._setupAudioCleanup();this._logDetectedGpus();@@ -1490,7 +1493,9 @@ class IPCHandlers {ipcMain.handle("get-diarization-model-status", async () => {return {available: this.diarizationManager?.isAvailable() ?? false,- modelsDownloaded: this.diarizationManager?.isModelDownloaded() ?? false,+ modelsDownloaded:+ (this.diarizationManager?.isModelDownloaded() ?? false) &&+ (this.diarizationManager?.isVadModelDownloaded() ?? false),};});@@ -2933,47 +2938,12 @@ class IPCHandlers {let meetingSendCounts = { mic: 0, system: 0 };- // Meeting mic captures at 24kHz (for OpenAI Realtime), but local engines- // expect 16kHz. Downsample with linear interpolation (3:2 ratio).- const downsample24kTo16k = (pcmBuffer) => {- const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);- const ratio = 1.5; // 24000 / 16000- const outputLength = Math.floor(input.length / ratio);- const output = new Int16Array(outputLength);- for (let i = 0; i < outputLength; i++) {- const srcIdx = i * ratio;- const idx = Math.floor(srcIdx);- const frac = srcIdx - idx;- const s0 = input[idx];- const s1 = idx + 1 < input.length ? input[idx + 1] : s0;- output[i] = Math.round(s0 + frac * (s1 - s0));- }- return Buffer.from(output.buffer);- };-- const pcm16ToWav = (pcmBuffer, sampleRate = 16000, channels = 1) => {- const dataSize = pcmBuffer.length;- const header = Buffer.alloc(44);- header.write("RIFF", 0);- header.writeUInt32LE(36 + dataSize, 4);- header.write("WAVE", 8);- header.write("fmt ", 12);- header.writeUInt32LE(16, 16);- header.writeUInt16LE(1, 20);- header.writeUInt16LE(channels, 22);- header.writeUInt32LE(sampleRate, 24);- header.writeUInt32LE(sampleRate * channels * 2, 28);- header.writeUInt16LE(channels * 2, 32);- header.writeUInt16LE(16, 34);- header.write("data", 36);- header.writeUInt32LE(dataSize, 40);- return Buffer.concat([header, pcmBuffer]);- };-const fs = require("fs");let meetingDiarizationStream = null;let meetingDiarizationPath = null;let meetingDiarizationSegments = [];+ let meetingLiveSpeakerActive = false;+ let meetingLiveSpeakerState = null;let meetingLocalMode = false;let meetingLocalBuffers = { mic: [], system: [] };@@ -2984,6 +2954,47 @@ class IPCHandlers {let meetingLocalModel = null;let meetingLocalTranscribing = false;+ const getLiveSpeakerProfiles = () => this.databaseManager.getSpeakerProfiles(true);++ const stopLiveSpeakerIdentification = async () => {+ if (!meetingLiveSpeakerActive) {+ return null;+ }++ meetingLiveSpeakerState = liveSpeakerIdentifier.getTransientMap();+ meetingLiveSpeakerActive = false;+ liveSpeakerIdentifier.stop();+ return meetingLiveSpeakerState;+ };++ const startLiveSpeakerIdentification = async (win, systemAudioMode) => {+ await stopLiveSpeakerIdentification();++ if (systemAudioMode !== "native" || !liveSpeakerIdentifier.isAvailable()) {+ return false;+ }++ meetingLiveSpeakerState = null;+ const started = await liveSpeakerIdentifier.start(+ (identification) => {+ if (!win || win.isDestroyed()) {+ return;+ }++ win.webContents.send("meeting-speaker-identified", identification);+ },+ {+ getSpeakerProfiles: getLiveSpeakerProfiles,+ }+ );++ if (started) {+ meetingLiveSpeakerActive = true;+ }++ return started;+ };+const transcribeLocalMeetingChunk = async (source) => {const chunks = meetingLocalBuffers[source];if (!chunks.length) return;@@ -3060,6 +3071,8 @@ class IPCHandlers {clearInterval(meetingLocalTimer);meetingLocalTimer = null;}+ void stopLiveSpeakerIdentification();+ meetingLiveSpeakerState = null;meetingLocalMode = false;meetingLocalBuffers = { mic: [], system: [] };if (meetingDiarizationStream) {@@ -3102,6 +3115,7 @@ class IPCHandlers {if (this.audioTapManager) {await this.audioTapManager.stop().catch(() => {});}+ await stopLiveSpeakerIdentification().catch(() => {});resetMeetingLocalState();await disconnectMeetingStreaming().catch(() => {});};@@ -3192,6 +3206,7 @@ class IPCHandlers {const win = BrowserWindow.fromWebContents(event.sender);attachMeetingStreamingHandlers(this._meetingMicStreaming, win, "mic");if (systemAudioMode === "native") {+ await startLiveSpeakerIdentification(win, systemAudioMode);attachMeetingStreamingHandlers(this._meetingSystemStreaming, win, "system");await startNativeMeetingSystemAudio(event);}@@ -3206,6 +3221,8 @@ class IPCHandlers {meetingLocalBuffers = { mic: [], system: [] };meetingLocalTranscript = "";+ await startLiveSpeakerIdentification(meetingLocalWin, systemAudioMode);+meetingLocalTimer = setInterval(() => {transcribeAllLocalBuffers();}, 5000);@@ -3227,6 +3244,7 @@ class IPCHandlers {}await connectRealtimeStreaming(event, options);+ await startLiveSpeakerIdentification(BrowserWindow.fromWebContents(event.sender), systemAudioMode);if (systemAudioMode === "native") {await startNativeMeetingSystemAudio(event);}@@ -3242,14 +3260,18 @@ class IPCHandlers {const sendMeetingAudio = (audioBuffer, source) => {if (source === "system") {+ const buf = Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer);++ if (meetingLiveSpeakerActive) {+ void liveSpeakerIdentifier.feedAudio(buf);+ }+if (!meetingDiarizationStream) {const os = require("os");meetingDiarizationPath = path.join(os.tmpdir(), `ow-diarize-raw-${Date.now()}.pcm`);meetingDiarizationStream = fs.createWriteStream(meetingDiarizationPath);}- meetingDiarizationStream.write(- Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer)- );+ meetingDiarizationStream.write(buf);}if (meetingLocalMode) {@@ -3304,6 +3326,8 @@ class IPCHandlers {await this.audioTapManager.stop();}+ const liveSpeakerState = await stopLiveSpeakerIdentification().catch(() => null);+const diarizationSessionId = `diar-${Date.now()}`;const diarizationPcmPath = meetingDiarizationPath;const diarizationSegments = meetingDiarizationSegments;@@ -3329,7 +3353,13 @@ class IPCHandlers {resetMeetingLocalState();// Fire-and-forget background diarization (or notify skip)- this._startOrSkipDiarization(diarizationSessionId, diarizationPcmPath, diarizationSegments, diarizationWin);+ this._startOrSkipDiarization(+ diarizationSessionId,+ diarizationPcmPath,+ diarizationSegments,+ diarizationWin,+ liveSpeakerState+ );return { success: true, transcript, diarizationSessionId };}@@ -3338,7 +3368,13 @@ class IPCHandlers {const transcript = [results[0]?.text, results[1]?.text].filter(Boolean).join(" ");// Fire-and-forget background diarization (or notify skip)- this._startOrSkipDiarization(diarizationSessionId, diarizationPcmPath, diarizationSegments, diarizationWin);+ this._startOrSkipDiarization(+ diarizationSessionId,+ diarizationPcmPath,+ diarizationSegments,+ diarizationWin,+ liveSpeakerState+ );return { success: true, transcript, diarizationSessionId };} catch (error) {@@ -5093,14 +5129,17 @@ class IPCHandlers {"set-speaker-mapping",async (_event, noteId, speakerId, displayName, email, profileId) => {const embeddings = this.databaseManager.getNoteSpeakerEmbeddings(noteId);- const speakerEmb = embeddings.find((e) => e.speaker_id === speakerId);+ const noteSpeakerEmbedding = embeddings.find((e) => e.speaker_id === speakerId);+ const liveSpeakerEmbedding = liveSpeakerIdentifier.getSpeakerEmbedding(speakerId);+ const speakerEmbeddingBuffer = noteSpeakerEmbedding?.embedding+ || (liveSpeakerEmbedding ? Buffer.from(liveSpeakerEmbedding.buffer) : null);let resolvedProfileId = profileId ?? null;- if (speakerEmb) {+ if (speakerEmbeddingBuffer) {const profile = this.databaseManager.upsertSpeakerProfile(displayName,email || null,- speakerEmb.embedding,+ speakerEmbeddingBuffer,resolvedProfileId);resolvedProfileId = profile.id;@@ -5108,6 +5147,7 @@ class IPCHandlers {}this.databaseManager.setSpeakerMapping(noteId, speakerId, resolvedProfileId, displayName);+ liveSpeakerIdentifier.mapSpeaker(speakerId, resolvedProfileId, displayName, noteId);return { success: true, profileId: resolvedProfileId };});@@ -5177,7 +5217,99 @@ class IPCHandlers {});}- _startOrSkipDiarization(sessionId, rawPcmPath, transcriptSegments, win) {+ _applySpeakerName(segments, speakerId, displayName) {+ if (!displayName) {+ return;+ }++ for (const segment of segments) {+ if (segment.speaker !== speakerId) {+ continue;+ }++ segment.speakerName = displayName;+ segment.suggestedName = undefined;+ segment.suggestedProfileId = undefined;+ }+ }++ _reconcileLiveSpeakerState(liveSpeakerState, speakerEmbeddingsMap, enrichedSegments) {+ if (!liveSpeakerState || !speakerEmbeddingsMap) {+ return new Set();+ }++ const speakerEmbeddings = require("./speakerEmbeddings");+ const reconciledSpeakers = new Set();+ const usedLiveSpeakers = new Set();+ const noteMappings = new Map();++ const liveEntries = Object.entries(liveSpeakerState)+ .map(([speakerId, data]) => ({+ speakerId,+ displayName: data?.displayName || null,+ profileId: data?.profileId ?? null,+ noteId: data?.noteId ?? null,+ embedding: Array.isArray(data?.embedding) ? new Float32Array(data.embedding) : null,+ }))+ .filter((entry) => entry.embedding);++ const getMappingsForNote = (noteId) => {+ if (!noteMappings.has(noteId)) {+ noteMappings.set(noteId, this.databaseManager.getSpeakerMappings(noteId));+ }+ return noteMappings.get(noteId);+ };++ for (const [mappedId, embeddingArray] of Object.entries(speakerEmbeddingsMap)) {+ let bestEntry = null;+ let bestSimilarity = 0;++ for (const entry of liveEntries) {+ if (usedLiveSpeakers.has(entry.speakerId)) {+ continue;+ }++ const similarity = speakerEmbeddings.cosineSimilarity(+ new Float32Array(embeddingArray),+ entry.embedding+ );+ if (similarity > bestSimilarity) {+ bestSimilarity = similarity;+ bestEntry = entry;+ }+ }++ if (!bestEntry || bestSimilarity <= 0.7) {+ continue;+ }++ usedLiveSpeakers.add(bestEntry.speakerId);+ reconciledSpeakers.add(mappedId);++ let displayName = bestEntry.displayName;+ let profileId = bestEntry.profileId;++ if (bestEntry.noteId) {+ const liveMapping = getMappingsForNote(bestEntry.noteId).find(+ (mapping) => mapping.speaker_id === bestEntry.speakerId+ );+ if (liveMapping) {+ displayName = liveMapping.display_name || displayName;+ profileId = liveMapping.profile_id ?? profileId;+ this.databaseManager.setSpeakerMapping(bestEntry.noteId, mappedId, profileId, displayName);+ this.databaseManager.removeSpeakerMapping(bestEntry.noteId, bestEntry.speakerId);+ } else if (displayName) {+ this.databaseManager.setSpeakerMapping(bestEntry.noteId, mappedId, profileId, displayName);+ }+ }++ this._applySpeakerName(enrichedSegments, mappedId, displayName);+ }++ return reconciledSpeakers;+ }++ _startOrSkipDiarization(sessionId, rawPcmPath, transcriptSegments, win, liveSpeakerState = null) {const send = (payload) => {if (win && !win.isDestroyed()) {win.webContents.send("meeting-diarization-complete", { sessionId, ...payload });@@ -5235,7 +5367,7 @@ class IPCHandlers {const sorted = segs.sort((a, b) => (b.end - b.start) - (a.end - a.start)).slice(0, 3);const embeddings = [];for (const seg of sorted) {- if (seg.end - seg.start < 2) continue;+ if (seg.end - seg.start < 1.5) continue;const emb = await speakerEmb.extractEmbedding(tmpWav, seg.start, seg.end);if (emb) embeddings.push(emb);}@@ -5250,12 +5382,25 @@ class IPCHandlers {debugLogger.debug("Speaker embedding extraction skipped", { error: err.message });}+ const reconciledSpeakers = this._reconcileLiveSpeakerState(+ liveSpeakerState,+ speakerEmbeddingsMap,+ enrichedSegments+ );+if (speakerEmbeddingsMap) {try {const profiles = this.databaseManager.getSpeakerProfiles(true);if (profiles.length > 0) {for (const [mappedId, embArr] of Object.entries(speakerEmbeddingsMap)) {+ const alreadyMapped = enrichedSegments.some(+ (segment) => segment.speaker === mappedId && segment.speakerName+ );+ if (reconciledSpeakers.has(mappedId) || alreadyMapped) {+ continue;+ }+const emb = new Float32Array(embArr);let bestProfile = null;let bestSim = 0;diff --git a/src/helpers/liveSpeakerIdentifier.js b/src/helpers/liveSpeakerIdentifier.jsnew file mode 100644index 000000000..09b3ceedd--- /dev/null+++ b/src/helpers/liveSpeakerIdentifier.js@@ -0,0 +1,495 @@+const fs = require("fs");+const debugLogger = require("./debugLogger");+const speakerEmbeddings = require("./speakerEmbeddings");+const { downsample24kTo16k } = require("../utils/audioUtils");++const SAMPLE_RATE = 16000;+const VAD_WINDOW_SIZE = 512;+const MIN_SEGMENT_SECONDS = 1.5;+const MIN_SEGMENT_SAMPLES = Math.round(SAMPLE_RATE * MIN_SEGMENT_SECONDS);+const SPEECH_THRESHOLD = 0.15;+const SILENCE_THRESHOLD = 0.08;+const SILENCE_WINDOWS_TO_END = 4;+const MATCH_THRESHOLD = 0.7;+const DEFAULT_VAD_STATE_SHAPE = [2, 1, 64];++function pcm16BufferToFloat32(buffer) {+ const input = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);+ const output = new Float32Array(input.length);++ for (let i = 0; i < input.length; i += 1) {+ output[i] = input[i] / 32768;+ }++ return output;+}++function appendFloat32(existing, next) {+ if (!existing.length) return next;+ if (!next.length) return existing;++ const merged = new Float32Array(existing.length + next.length);+ merged.set(existing, 0);+ merged.set(next, existing.length);+ return merged;+}++function concatFloat32Arrays(chunks) {+ if (chunks.length === 0) return new Float32Array(0);+ if (chunks.length === 1) return chunks[0];++ let totalLength = 0;+ for (const chunk of chunks) {+ totalLength += chunk.length;+ }++ const merged = new Float32Array(totalLength);+ let offset = 0;+ for (const chunk of chunks) {+ merged.set(chunk, offset);+ offset += chunk.length;+ }++ return merged;+}++function cloneFloat32Array(value) {+ return new Float32Array(value);+}++function normalizeVadShape(shape) {+ if (!Array.isArray(shape) || shape.length === 0) {+ return DEFAULT_VAD_STATE_SHAPE;+ }++ return shape.map((dim) => (typeof dim === "number" && dim > 0 ? dim : 1));+}++function normalizeVadStateName(name) {+ return String(name || "").replace(/[^a-z0-9]/gi, "").toLowerCase();+}++function getBufferFloat32View(buffer) {+ return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);+}++class LiveSpeakerIdentifier {+ constructor() {+ this.session = null;+ this.running = false;+ this.queue = Promise.resolve();+ this.onSpeakerIdentified = null;+ this.getSpeakerProfiles = null;+ this.audioRemainder = new Float32Array(0);+ this.vadStateInputs = [];+ this.vadStateOutputs = [];+ this.vadStates = new Map();+ this.speechChunks = [];+ this.speechActive = false;+ this.silenceWindows = 0;+ this.sampleCursor = 0;+ this.segmentStartSample = 0;+ this.segmentEndSample = 0;+ this.transientEmbeddings = new Map();+ this.transientCounts = new Map();+ this.transientDisplayNames = new Map();+ this.transientProfileIds = new Map();+ this.nextLiveIndex = 0;+ this._diarizationManager = null;+ }++ setDiarizationManager(manager) {+ this._diarizationManager = manager;+ }++ isAvailable() {+ return this._diarizationManager?.isVadModelDownloaded() && speakerEmbeddings.isAvailable();+ }++ getTransientMap() {+ const map = new Map();+ for (const [speakerId, embedding] of this.transientEmbeddings.entries()) {+ map.set(speakerId, cloneFloat32Array(embedding));+ }+ return map;+ }++ getSpeakerEmbedding(speakerId) {+ const embedding = this.transientEmbeddings.get(speakerId);+ return embedding ? cloneFloat32Array(embedding) : null;+ }++ async start(options = {}) {+ const {+ onSpeakerIdentified = null,+ getSpeakerProfiles = null,+ } = typeof options === "function" ? { onSpeakerIdentified: options } : options;++ this.onSpeakerIdentified =+ typeof onSpeakerIdentified === "function" ? onSpeakerIdentified : null;+ this.getSpeakerProfiles = typeof getSpeakerProfiles === "function" ? getSpeakerProfiles : null;+ this._resetMeetingState();++ if (!this.isAvailable()) {+ debugLogger.warn("Live speaker identifier unavailable", {+ vadModelPath: this._diarizationManager?.getVadModelPath(),+ embeddingModelAvailable: speakerEmbeddings.isAvailable(),+ });+ return false;+ }++ await this._ensureLoaded();+ this.running = !!this.session;+ return this.running;+ }++ async stop() {+ this.running = false;+ await this.queue;++ if (this.speechActive && this.audioRemainder.length > 0) {+ this.speechChunks.push(this.audioRemainder);+ this.audioRemainder = new Float32Array(0);+ }++ if (this.speechActive) {+ await this._finalizeSpeechSegment();+ }++ const transientMap = this.getTransientMap();+ this._resetMeetingState();+ this.onSpeakerIdentified = null;+ this.getSpeakerProfiles = null;+ return transientMap;+ }++ feedAudio(pcmBuffer) {+ if (!this.running || !pcmBuffer?.length) {+ return Promise.resolve();+ }++ this.queue = this.queue+ .then(() => this._processAudio(pcmBuffer))+ .catch((error) => {+ debugLogger.warn("Live speaker identification failed", { error: error.message });+ });++ return this.queue;+ }++ mapSpeaker(liveId, profileId, displayName) {+ if (!liveId || !this.transientEmbeddings.has(liveId)) {+ return false;+ }++ if (profileId != null) {+ this.transientProfileIds.set(liveId, profileId);+ }++ if (displayName) {+ this.transientDisplayNames.set(liveId, displayName);+ }++ return true;+ }++ async _ensureLoaded() {+ if (this.session) return;++ const vadModelPath = this._diarizationManager?.getVadModelPath();+ if (!fs.existsSync(vadModelPath)) {+ return;+ }++ const ort = require("onnxruntime-node");+ this.session = await ort.InferenceSession.create(vadModelPath);+ this.vadStateInputs = (this.session.inputNames || []).filter((name) =>+ /state|h|c/i.test(name)+ );+ this.vadStateOutputs = (this.session.outputNames || []).filter((name) =>+ /state|h|c/i.test(name)+ );+ this._resetVadRuntimeState();+ }++ _resetMeetingState() {+ this.queue = Promise.resolve();+ this.audioRemainder = new Float32Array(0);+ this.speechChunks = [];+ this.speechActive = false;+ this.silenceWindows = 0;+ this.sampleCursor = 0;+ this.segmentStartSample = 0;+ this.segmentEndSample = 0;+ this.transientEmbeddings = new Map();+ this.transientCounts = new Map();+ this.transientDisplayNames = new Map();+ this.transientProfileIds = new Map();+ this.nextLiveIndex = 0;+ this._resetVadRuntimeState();+ }++ _resetVadRuntimeState() {+ this.vadStates = new Map();++ if (!this.session) {+ return;+ }++ for (const name of this.vadStateInputs) {+ const metadata = this.session.inputMetadata?.[name];+ const shape = normalizeVadShape(metadata?.dimensions || metadata?.shape);+ const size = shape.reduce((total, dim) => total * dim, 1);+ this.vadStates.set(name, new Float32Array(size));+ }+ }++ async _processAudio(pcmBuffer) {+ await this._ensureLoaded();+ if (!this.session) return;++ const downsampled = downsample24kTo16k(+ Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer)+ );+ if (!downsampled.length) return;++ this.audioRemainder = appendFloat32(this.audioRemainder, pcm16BufferToFloat32(downsampled));++ while (this.audioRemainder.length >= VAD_WINDOW_SIZE) {+ const window = this.audioRemainder.subarray(0, VAD_WINDOW_SIZE);+ this.audioRemainder = this.audioRemainder.slice(VAD_WINDOW_SIZE);++ const windowStartSample = this.sampleCursor;+ this.sampleCursor += window.length;++ await this._processWindow(window, windowStartSample, this.sampleCursor);+ }+ }++ async _processWindow(window, windowStartSample, windowEndSample) {+ const probability = await this._getVadProbability(window);++ if (this.speechActive) {+ this.speechChunks.push(cloneFloat32Array(window));+ this.segmentEndSample = windowEndSample;++ if (probability >= SILENCE_THRESHOLD) {+ this.silenceWindows = 0;+ return;+ }++ this.silenceWindows += 1;+ if (this.silenceWindows >= SILENCE_WINDOWS_TO_END) {+ await this._finalizeSpeechSegment();+ }+ return;+ }++ if (probability < SPEECH_THRESHOLD) {+ return;+ }++ this.speechActive = true;+ this.segmentStartSample = windowStartSample;+ this.segmentEndSample = windowEndSample;+ this.speechChunks = [cloneFloat32Array(window)];+ this.silenceWindows = 0;+ }++ async _getVadProbability(window) {+ if (!this.session) return 0;++ const ort = require("onnxruntime-node");+ const feeds = {};+ const audioInputName = (this.session.inputNames || []).find(+ (name) => !this.vadStateInputs.includes(name) && !/sr|sample.?rate/i.test(name)+ );++ if (!audioInputName) {+ return 0;+ }++ feeds[audioInputName] = new ort.Tensor("float32", window, [1, window.length]);++ const sampleRateInputName = (this.session.inputNames || []).find((name) =>+ /sr|sample.?rate/i.test(name)+ );+ if (sampleRateInputName) {+ feeds[sampleRateInputName] = new ort.Tensor(+ "int64",+ BigInt64Array.from([BigInt(SAMPLE_RATE)]),+ [1]+ );+ }++ for (const stateName of this.vadStateInputs) {+ const metadata = this.session.inputMetadata?.[stateName];+ const shape = normalizeVadShape(metadata?.dimensions || metadata?.shape);+ const state =+ this.vadStates.get(stateName) || new Float32Array(shape.reduce((a, b) => a * b, 1));++ this.vadStates.set(stateName, state);+ feeds[stateName] = new ort.Tensor("float32", state, shape);+ }++ const results = await this.session.run(feeds);+ this._updateVadState(results);++ const outputName = (this.session.outputNames || []).find(+ (name) => !this.vadStateOutputs.includes(name)+ );+ const output = (outputName && results[outputName]) || Object.values(results)[0];+ const value = output?.data?.[0];+ return typeof value === "number" ? value : 0;+ }++ _updateVadState(results) {+ if (!results) {+ return;+ }++ for (const inputName of this.vadStateInputs) {+ const expectedName = normalizeVadStateName(inputName);+ const matchingOutput = this.vadStateOutputs.find(+ (outputName) => normalizeVadStateName(outputName).startsWith(expectedName)+ );+ const output = (matchingOutput && results[matchingOutput]) || results[inputName];++ if (output?.data) {+ this.vadStates.set(inputName, new Float32Array(output.data));+ }+ }+ }++ async _finalizeSpeechSegment() {+ const samples = concatFloat32Arrays(this.speechChunks);+ this.speechChunks = [];+ this.speechActive = false;+ this.silenceWindows = 0;++ if (samples.length < MIN_SEGMENT_SAMPLES) {+ return;+ }++ const embedding = await speakerEmbeddings.extractEmbeddingFromSamples(samples);+ if (!embedding) {+ return;+ }++ let speakerId = this._findTransientMatch(embedding);+ if (speakerId) {+ this._updateCentroid(speakerId, embedding);+ } else {+ const matchedProfile = this._findStoredProfileMatch(embedding);+ if (matchedProfile) {+ speakerId = this._findTransientSpeakerForProfile(matchedProfile.id);+ if (!speakerId) {+ speakerId = this._assignSpeakerId(embedding);+ } else {+ this._updateCentroid(speakerId, embedding);+ }++ this.transientProfileIds.set(speakerId, matchedProfile.id);+ this.transientDisplayNames.set(speakerId, matchedProfile.display_name);+ } else {+ speakerId = this._assignSpeakerId(embedding);+ }+ }++ this.onSpeakerIdentified?.({+ speakerId,+ displayName: this.transientDisplayNames.get(speakerId) || null,+ startTime: this.segmentStartSample / SAMPLE_RATE,+ endTime: this.segmentEndSample / SAMPLE_RATE,+ });+ }++ _findTransientMatch(embedding) {+ let bestSpeakerId = null;+ let bestSimilarity = 0;++ for (const [speakerId, centroid] of this.transientEmbeddings.entries()) {+ const similarity = speakerEmbeddings.cosineSimilarity(embedding, centroid);+ if (similarity > bestSimilarity) {+ bestSimilarity = similarity;+ bestSpeakerId = speakerId;+ }+ }++ return bestSimilarity >= MATCH_THRESHOLD ? bestSpeakerId : null;+ }++ _findStoredProfileMatch(embedding) {+ let profiles = [];++ try {+ profiles = this.getSpeakerProfiles?.() || [];+ } catch (error) {+ debugLogger.debug("Live speaker profile lookup failed", { error: error.message });+ return null;+ }++ let bestProfile = null;+ let bestSimilarity = 0;++ for (const profile of profiles) {+ if (!profile?.embedding) continue;++ const profileEmbedding =+ profile.embedding instanceof Float32Array+ ? profile.embedding+ : Array.isArray(profile.embedding)+ ? new Float32Array(profile.embedding)+ : getBufferFloat32View(profile.embedding);++ if (!profileEmbedding.length) continue;++ const similarity = speakerEmbeddings.cosineSimilarity(embedding, profileEmbedding);+ if (similarity > bestSimilarity) {+ bestSimilarity = similarity;+ bestProfile = profile;+ }+ }++ return bestSimilarity >= MATCH_THRESHOLD ? bestProfile : null;+ }++ _findTransientSpeakerForProfile(profileId) {+ for (const [speakerId, candidateProfileId] of this.transientProfileIds.entries()) {+ if (candidateProfileId === profileId) {+ return speakerId;+ }+ }++ return null;+ }++ _assignSpeakerId(embedding) {+ const speakerId = `live_${this.nextLiveIndex}`;+ this.nextLiveIndex += 1;+ this.transientEmbeddings.set(speakerId, cloneFloat32Array(embedding));+ this.transientCounts.set(speakerId, 1);+ return speakerId;+ }++ _updateCentroid(speakerId, embedding) {+ const centroid = this.transientEmbeddings.get(speakerId);+ if (!centroid) {+ return;+ }++ const count = this.transientCounts.get(speakerId) || 1;+ const nextCentroid = new Float32Array(embedding.length);++ for (let i = 0; i < embedding.length; i += 1) {+ nextCentroid[i] = (centroid[i] * count + embedding[i]) / (count + 1);+ }++ this.transientEmbeddings.set(speakerId, nextCentroid);+ this.transientCounts.set(speakerId, count + 1);+ }+}++const instance = new LiveSpeakerIdentifier();+module.exports = instance;+module.exports.LiveSpeakerIdentifier = LiveSpeakerIdentifier;diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.jsindex bcc2d4856..65e49301e 100644--- a/src/helpers/speakerEmbeddings.js+++ b/src/helpers/speakerEmbeddings.js@@ -4,39 +4,197 @@ const debugLogger = require("./debugLogger");const { getModelsDirForService } = require("./modelDirUtils");const EMBEDDING_DIM = 512;-const MIN_SEGMENT_SECONDS = 2;+const MIN_SEGMENT_SECONDS = 1.5;+const MIN_SEGMENT_SAMPLES = 16000 * MIN_SEGMENT_SECONDS;const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";+const FBANK_SAMPLE_RATE = 16000;+const FBANK_FRAME_LENGTH_MS = 25;+const FBANK_FRAME_SHIFT_MS = 10;+const FBANK_NUM_MELS = 80;+const FBANK_FRAME_LENGTH = Math.round(FBANK_SAMPLE_RATE * FBANK_FRAME_LENGTH_MS / 1000);+const FBANK_FRAME_SHIFT = Math.round(FBANK_SAMPLE_RATE * FBANK_FRAME_SHIFT_MS / 1000);+const FBANK_FFT_SIZE = 512;++let _melFilterbank = null;++function getMelFilterbank() {+ if (_melFilterbank) return _melFilterbank;++ const numBins = FBANK_FFT_SIZE / 2 + 1;+ const lowFreq = 20;+ const highFreq = FBANK_SAMPLE_RATE / 2;++ const melLow = 1127 * Math.log(1 + lowFreq / 700);+ const melHigh = 1127 * Math.log(1 + highFreq / 700);++ const melPoints = new Float64Array(FBANK_NUM_MELS + 2);+ for (let i = 0; i < melPoints.length; i++) {+ const mel = melLow + (melHigh - melLow) * i / (FBANK_NUM_MELS + 1);+ melPoints[i] = 700 * (Math.exp(mel / 1127) - 1);+ }++ const binPoints = new Float64Array(melPoints.length);+ for (let i = 0; i < melPoints.length; i++) {+ binPoints[i] = Math.floor((FBANK_FFT_SIZE + 1) * melPoints[i] / FBANK_SAMPLE_RATE);+ }++ _melFilterbank = new Array(FBANK_NUM_MELS);+ for (let m = 0; m < FBANK_NUM_MELS; m++) {+ const filter = new Float32Array(numBins);+ const left = binPoints[m];+ const center = binPoints[m + 1];+ const right = binPoints[m + 2];++ for (let k = 0; k < numBins; k++) {+ if (k >= left && k <= center && center > left) {+ filter[k] = (k - left) / (center - left);+ } else if (k > center && k <= right && right > center) {+ filter[k] = (right - k) / (right - center);+ }+ }+ _melFilterbank[m] = filter;+ }++ return _melFilterbank;+}++function realFFT(frame) {+ const n = FBANK_FFT_SIZE;+ const re = new Float64Array(n);+ const im = new Float64Array(n);+ for (let i = 0; i < frame.length && i < n; i++) re[i] = frame[i];++ for (let i = 1, j = 0; i < n; i++) {+ let bit = n >> 1;+ while (j & bit) { j ^= bit; bit >>= 1; }+ j ^= bit;+ if (i < j) { [re[i], re[j]] = [re[j], re[i]]; }+ }++ for (let len = 2; len <= n; len <<= 1) {+ const angle = -2 * Math.PI / len;+ const wRe = Math.cos(angle);+ const wIm = Math.sin(angle);+ for (let i = 0; i < n; i += len) {+ let curRe = 1, curIm = 0;+ for (let j = 0; j < len / 2; j++) {+ const uRe = re[i + j], uIm = im[i + j];+ const vRe = re[i + j + len / 2] * curRe - im[i + j + len / 2] * curIm;+ const vIm = re[i + j + len / 2] * curIm + im[i + j + len / 2] * curRe;+ re[i + j] = uRe + vRe; im[i + j] = uIm + vIm;+ re[i + j + len / 2] = uRe - vRe; im[i + j + len / 2] = uIm - vIm;+ const tmpRe = curRe * wRe - curIm * wIm;+ curIm = curRe * wIm + curIm * wRe;+ curRe = tmpRe;+ }+ }+ }++ const numBins = n / 2 + 1;+ const powerSpectrum = new Float32Array(numBins);+ for (let i = 0; i < numBins; i++) {+ powerSpectrum[i] = re[i] * re[i] + im[i] * im[i];+ }+ return powerSpectrum;+}++function computeFbank(samples) {+ const numFrames = Math.max(0, Math.floor((samples.length - FBANK_FRAME_LENGTH) / FBANK_FRAME_SHIFT) + 1);+ if (numFrames === 0) return null;++ const hamming = new Float32Array(FBANK_FRAME_LENGTH);+ for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {+ hamming[i] = 0.54 - 0.46 * Math.cos(2 * Math.PI * i / (FBANK_FRAME_LENGTH - 1));+ }++ const melBank = getMelFilterbank();+ const features = new Float32Array(numFrames * FBANK_NUM_MELS);++ for (let f = 0; f < numFrames; f++) {+ const start = f * FBANK_FRAME_SHIFT;+ const frame = new Float32Array(FBANK_FRAME_LENGTH);+ for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {+ frame[i] = (samples[start + i] || 0) * hamming[i];+ }++ const power = realFFT(frame);++ for (let m = 0; m < FBANK_NUM_MELS; m++) {+ let energy = 0;+ const filter = melBank[m];+ for (let k = 0; k < power.length; k++) {+ energy += filter[k] * power[k];+ }+ features[f * FBANK_NUM_MELS + m] = Math.log(Math.max(energy, 1e-10));+ }+ }++ return { features, numFrames };+}+class SpeakerEmbeddings {constructor() {this.session = null;- this.modelPath = path.join(getModelsDirForService("diarization"), MODEL_FILE);+ }++ getModelPath() {+ if (process.resourcesPath) {+ const bundledPath = path.join(process.resourcesPath, "bin", "diarization-models", MODEL_FILE);+ if (fs.existsSync(bundledPath)) {+ return bundledPath;+ }+ }++ return path.join(getModelsDirForService("diarization"), MODEL_FILE);}isAvailable() {- return fs.existsSync(this.modelPath);+ return fs.existsSync(this.getModelPath());}async _ensureLoaded() {if (this.session) return;if (!this.isAvailable()) {- throw new Error(`Speaker embedding model not found at ${this.modelPath}`);+ throw new Error(`Speaker embedding model not found at ${this.getModelPath()}`);}- debugLogger.debug("speaker-embeddings loading model", { modelPath: this.modelPath });+ const modelPath = this.getModelPath();+ debugLogger.debug("speaker-embeddings loading model", { modelPath });const ort = require("onnxruntime-node");- this.session = await ort.InferenceSession.create(this.modelPath);+ this.session = await ort.InferenceSession.create(modelPath);debugLogger.debug("speaker-embeddings model loaded");}+ async _extractEmbeddingFromSamples(samples) {+ await this._ensureLoaded();++ const fbank = computeFbank(samples);+ if (!fbank) return null;++ const ort = require("onnxruntime-node");+ const inputName = this.session.inputNames[0];+ const feeds = {+ [inputName]: new ort.Tensor("float32", fbank.features, [1, fbank.numFrames, FBANK_NUM_MELS]),+ };++ const results = await this.session.run(feeds);+ const output = results[Object.keys(results)[0]];++ return new Float32Array(output.data);+ }++ async extractEmbeddingFromSamples(samples) {+ if (samples.length < MIN_SEGMENT_SAMPLES) return null;+ return this._extractEmbeddingFromSamples(samples);+ }+async extractEmbedding(wavPath, startSec, endSec) {if (endSec - startSec < MIN_SEGMENT_SECONDS) return null;- await this._ensureLoaded();-const buf = fs.readFileSync(wavPath);const { sampleRate, dataOffset } = this._parseWavHeader(buf);@@ -55,16 +213,7 @@ class SpeakerEmbeddings {samples[i] = int16 / 32768;}- const ort = require("onnxruntime-node");- const inputName = this.session.inputNames[0];- const feeds = {- [inputName]: new ort.Tensor("float32", samples, [1, numSamples]),- };-- const results = await this.session.run(feeds);- const output = results[Object.keys(results)[0]];-- return new Float32Array(output.data);+ return this._extractEmbeddingFromSamples(samples);}_parseWavHeader(buf) {diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.tsindex ca59814b1..6d898903f 100644--- a/src/hooks/useMeetingTranscription.ts+++ b/src/hooks/useMeetingTranscription.ts@@ -32,6 +32,7 @@ const MEETING_AUDIO_BUFFER_SIZE = 800;const MEETING_STOP_FLUSH_TIMEOUT_MS = 50;const REALTIME_MODELS = new Set(["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]);+const SPEAKER_IDENTIFICATION_RETENTION_MS = 30_000;const getMeetingTranscriptionOptions = () => {const {@@ -256,6 +257,31 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {const preparePromiseRef = useRef<Promise<void> | null>(null);const ipcCleanupsRef = useRef<Array<() => void>>([]);const pendingCleanupRef = useRef<ReturnType<typeof setTimeout> | null>(null);+ const speakerIdentificationsRef = useRef<+ Array<{ speakerId: string; displayName?: string; startTime: number; endTime: number }>+ >([]);++ const applySpeakerIdentification = useCallback(+ (+ segment: TranscriptSegment,+ identification: { speakerId: string; displayName?: string; startTime: number; endTime: number }+ ): TranscriptSegment => {+ if (segment.source !== "system" || segment.timestamp == null) {+ return segment;+ }++ if (segment.timestamp < identification.startTime || segment.timestamp > identification.endTime) {+ return segment;+ }++ return {+ ...segment,+ speaker: identification.speakerId,+ speakerName: identification.displayName ?? segment.speakerName,+ };+ },+ []+ );const cleanup = useCallback(async () => {await flushAndDisconnectProcessor(micProcessorRef.current);@@ -361,6 +387,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {setMicPartial("");setSystemPartial("");setError(null);+ speakerIdentificationsRef.current = [];// Set recording state immediately for instant UI feedbackisRecordingRef.current = true;@@ -450,12 +477,13 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {setPartialForSource(data.text);setPartialTranscript(data.text);} else {- const seg: TranscriptSegment = {+ const rawSegment: TranscriptSegment = {id: `seg-${++segmentCounter}`,text: data.text,source: data.source,timestamp: data.timestamp,};+ const seg = speakerIdentificationsRef.current.reduce(applySpeakerIdentification, rawSegment);setSegments((prev) => {// Insert in chronological order — scan from the end since most// segments arrive in order and this is O(1) in the common case.@@ -473,6 +501,20 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {);if (segmentCleanup) ipcCleanupsRef.current.push(segmentCleanup);+ const speakerCleanup = window.electronAPI?.onMeetingSpeakerIdentified?.((data) => {+ speakerIdentificationsRef.current = [+ ...speakerIdentificationsRef.current.filter(+ (identification) => identification.endTime >= data.endTime - SPEAKER_IDENTIFICATION_RETENTION_MS+ ),+ data,+ ];++ setSegments((prev) =>+ prev.map((segment) => applySpeakerIdentification(segment, data))+ );+ });+ if (speakerCleanup) ipcCleanupsRef.current.push(speakerCleanup);+const errorCleanup = window.electronAPI?.onMeetingTranscriptionError?.((err) => {setError(err);logger.error("Meeting transcription stream error", { error: err }, "meeting");@@ -564,7 +606,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {setIsRecording(false);await cleanup();}- }, [cleanup]);+ }, [applySpeakerIdentification, cleanup]);useEffect(() => {getMeetingWorkletBlobUrl();diff --git a/src/types/electron.ts b/src/types/electron.tsindex 29b246e67..e2b9ae61a 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -1306,6 +1306,14 @@ declare global {type: "partial" | "final";}) => void) => () => void;+ onMeetingSpeakerIdentified?: (+ callback: (data: {+ speakerId: string;+ displayName?: string;+ startTime: number;+ endTime: number;+ }) => void+ ) => () => void;onMeetingTranscriptionError?: (callback: (error: string) => void) => () => void;// Speaker diarizationdiff --git a/src/utils/audioUtils.js b/src/utils/audioUtils.jsnew file mode 100644index 000000000..fb7d751be--- /dev/null+++ b/src/utils/audioUtils.js@@ -0,0 +1,41 @@+function downsample24kTo16k(pcmBuffer) {+ const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);+ const ratio = 1.5;+ const outputLength = Math.floor(input.length / ratio);+ const output = new Int16Array(outputLength);++ for (let i = 0; i < outputLength; i++) {+ const srcIdx = i * ratio;+ const idx = Math.floor(srcIdx);+ const frac = srcIdx - idx;+ const s0 = input[idx];+ const s1 = idx + 1 < input.length ? input[idx + 1] : s0;+ output[i] = Math.round(s0 + frac * (s1 - s0));+ }++ return Buffer.from(output.buffer);+}++function pcm16ToWav(pcmBuffer, sampleRate = 16000, channels = 1) {+ const dataSize = pcmBuffer.length;+ const header = Buffer.alloc(44);+ header.write("RIFF", 0);+ header.writeUInt32LE(36 + dataSize, 4);+ header.write("WAVE", 8);+ header.write("fmt ", 12);+ header.writeUInt32LE(16, 16);+ header.writeUInt16LE(1, 20);+ header.writeUInt16LE(channels, 22);+ header.writeUInt32LE(sampleRate, 24);+ header.writeUInt32LE(sampleRate * channels * 2, 28);+ header.writeUInt16LE(channels * 2, 32);+ header.writeUInt16LE(16, 34);+ header.write("data", 36);+ header.writeUInt32LE(dataSize, 40);+ return Buffer.concat([header, pcmBuffer]);+}++module.exports = {+ downsample24kTo16k,+ pcm16ToWav,+};