Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(diarization): live speaker identification pipeline bugs

Gabriel Stein458aa50954cb
diff --git a/electron-builder.json b/electron-builder.json
index 4502e751f..47a1a7b02 100644
--- a/electron-builder.json
+++ b/electron-builder.json
@@ -109,6 +109,7 @@
"windows-text-monitor*",
"windows-fast-paste*",
"all-MiniLM-L6-v2/**/*",
+ "diarization-models/**/*",
"*.dylib",
"*.dll",
"*.so*"
diff --git a/main.js b/main.js
index e156a79fb..4a587035e 100644
--- a/main.js
+++ b/main.js
@@ -685,7 +685,10 @@ async function startApp() {
}
// Auto-download diarization models if binary is available
- if (diarizationManager.getBinaryPath() && !diarizationManager.isModelDownloaded()) {
+ if (
+ diarizationManager.getBinaryPath() &&
+ (!diarizationManager.isModelDownloaded() || !diarizationManager.isVadModelDownloaded())
+ ) {
diarizationManager.downloadModels().catch((err) => {
debugLogger.debug("Diarization model auto-download error (non-fatal)", {
error: err.message,
diff --git a/package.json b/package.json
index 9851c0703..a3ab8d6e0 100644
--- a/package.json
+++ b/package.json
@@ -25,25 +25,25 @@
"predev:main": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model",
"dev:main": "cross-env NODE_ENV=development node scripts/run-electron.js --dev",
"dev:renderer": "cd src && vite",
- "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model -- --for-build",
+ "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model -- --for-build && npm run download:diarization-models -- --output-dir resources/bin/diarization-models",
"build": "cd src && vite build && cd .. && electron-builder",
"postinstall": "electron-builder install-app-deps",
"build:renderer": "cd src && vite build",
- "prebuild:mac": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",
+ "prebuild:mac": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models",
"build:mac": "npm run build:renderer && electron-builder --mac",
"build:mac:arm64": "npm run build:renderer && electron-builder --mac --arm64",
"build:mac:x64": "npm run build:renderer && electron-builder --mac --x64",
- "prebuild:win": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:nircmd && npm run download:windows-fast-paste && npm run download:windows-key-listener && npm run download:windows-mic-listener",
+ "prebuild:win": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models && npm run download:nircmd && npm run download:windows-fast-paste && npm run download:windows-key-listener && npm run download:windows-mic-listener",
"build:win": "npm run build:renderer && electron-builder --win",
- "prebuild:linux": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",
+ "prebuild:linux": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models",
"build:linux": "npm run build:renderer && electron-builder --linux",
"build:linux:appimage": "npm run build:renderer && electron-builder --linux AppImage",
"build:linux:deb": "npm run build:renderer && electron-builder --linux deb",
"build:linux:rpm": "npm run build:renderer && electron-builder --linux rpm",
"build:linux:tar": "npm run build:renderer && electron-builder --linux tar.gz",
- "prepack": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",
+ "prepack": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models",
"pack": "npm run build:renderer && cross-env CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --dir",
- "predist": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",
+ "predist": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:diarization-models -- --output-dir resources/bin/diarization-models",
"dist": "npm run build:renderer && electron-builder",
"download:whisper-cpp": "node scripts/download-whisper-cpp.js --current",
"download:whisper-cpp:all": "node scripts/download-whisper-cpp.js --all",
diff --git a/preload.js b/preload.js
index a297c95cd..f5540ba68 100644
--- a/preload.js
+++ b/preload.js
@@ -514,6 +514,10 @@ contextBridge.exposeInMainWorld("electronAPI", {
"meeting-transcription-segment",
(callback) => (_event, data) => callback(data)
),
+ onMeetingSpeakerIdentified: registerListener(
+ "meeting-speaker-identified",
+ (callback) => (_event, data) => callback(data)
+ ),
onMeetingTranscriptionError: registerListener(
"meeting-transcription-error",
(callback) => (_event, data) => callback(data)
diff --git a/scripts/download-diarization-models.js b/scripts/download-diarization-models.js
index b19c029c2..4451deecc 100644
--- a/scripts/download-diarization-models.js
+++ b/scripts/download-diarization-models.js
@@ -5,8 +5,6 @@ const path = require("path");
const { execFileSync } = require("child_process");
const { downloadFile, parseArgs } = require("./lib/download-utils");
-const MODEL_DIR = path.join(os.homedir(), ".cache", "openwhispr", "diarization-models");
-
const SEGMENTATION_URL =
"https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2";
const SEGMENTATION_DIR = "sherpa-onnx-pyannote-segmentation-3-0";
@@ -15,6 +13,22 @@ const SEGMENTATION_FILE = "model.onnx";
const EMBEDDING_URL =
"https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
const EMBEDDING_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
+const VAD_URL =
+ "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx";
+const VAD_FILE = "silero_vad.onnx";
+
+function getModelDir() {
+ const outputDirIndex = process.argv.indexOf("--output-dir");
+ if (outputDirIndex !== -1 && process.argv[outputDirIndex + 1]) {
+ return path.resolve(process.argv[outputDirIndex + 1]);
+ }
+
+ if (process.env.DIARIZATION_MODEL_DIR) {
+ return path.resolve(process.env.DIARIZATION_MODEL_DIR);
+ }
+
+ return path.join(os.homedir(), ".cache", "openwhispr", "diarization-models");
+}
function extractTarBz2(archivePath, destDir) {
fs.mkdirSync(destDir, { recursive: true });
@@ -29,28 +43,31 @@ async function main() {
console.log("\n[diarization-models] Downloading diarization models...\n");
const args = parseArgs();
+ const modelDir = getModelDir();
- const segModelPath = path.join(MODEL_DIR, SEGMENTATION_DIR, SEGMENTATION_FILE);
- const embModelPath = path.join(MODEL_DIR, EMBEDDING_FILE);
+ const segModelPath = path.join(modelDir, SEGMENTATION_DIR, SEGMENTATION_FILE);
+ const embModelPath = path.join(modelDir, EMBEDDING_FILE);
+ const vadModelPath = path.join(modelDir, VAD_FILE);
- const allExist = fs.existsSync(segModelPath) && fs.existsSync(embModelPath);
+ const allExist =
+ fs.existsSync(segModelPath) && fs.existsSync(embModelPath) && fs.existsSync(vadModelPath);
if (allExist && !args.isForce) {
console.log("[diarization-models] Model files already exist (use --force to re-download)\n");
return;
}
- fs.mkdirSync(MODEL_DIR, { recursive: true });
+ fs.mkdirSync(modelDir, { recursive: true });
// Download segmentation model (tar.bz2 archive)
if (!fs.existsSync(segModelPath) || args.isForce) {
console.log(`[diarization-models] Downloading segmentation model from ${SEGMENTATION_URL}`);
- const archivePath = path.join(MODEL_DIR, `${SEGMENTATION_DIR}.tar.bz2`);
+ const archivePath = path.join(modelDir, `${SEGMENTATION_DIR}.tar.bz2`);
try {
await downloadFile(SEGMENTATION_URL, archivePath);
- const extractDir = path.join(MODEL_DIR, "temp-segmentation");
+ const extractDir = path.join(modelDir, "temp-segmentation");
fs.mkdirSync(extractDir, { recursive: true });
extractTarBz2(archivePath, extractDir);
@@ -62,7 +79,7 @@ async function main() {
return;
}
- const destDir = path.join(MODEL_DIR, SEGMENTATION_DIR);
+ const destDir = path.join(modelDir, SEGMENTATION_DIR);
fs.mkdirSync(destDir, { recursive: true });
fs.copyFileSync(extractedModelPath, segModelPath);
@@ -102,7 +119,25 @@ async function main() {
console.log(`[diarization-models] Embedding model already exists, skipping`);
}
- console.log(`\n[diarization-models] Models ready at ${MODEL_DIR}\n`);
+ if (!fs.existsSync(vadModelPath) || args.isForce) {
+ console.log(`[diarization-models] Downloading VAD model from ${VAD_URL}`);
+
+ try {
+ await downloadFile(VAD_URL, vadModelPath);
+ const stats = fs.statSync(vadModelPath);
+ const sizeMB = (stats.size / 1024 / 1024).toFixed(1);
+ console.log(`[diarization-models] VAD model downloaded (${sizeMB}MB)`);
+ } catch (error) {
+ console.error(`[diarization-models] Failed to download VAD model: ${error.message}`);
+ if (fs.existsSync(vadModelPath)) fs.unlinkSync(vadModelPath);
+ process.exitCode = 1;
+ return;
+ }
+ } else {
+ console.log("[diarization-models] VAD model already exists, skipping");
+ }
+
+ console.log(`\n[diarization-models] Models ready at ${modelDir}\n`);
}
main().catch(console.error);
diff --git a/src/helpers/diarization.js b/src/helpers/diarization.js
index 59f40d948..ae8d5522f 100644
--- a/src/helpers/diarization.js
+++ b/src/helpers/diarization.js
@@ -15,10 +15,13 @@ const SEGMENTATION_MODEL_URL =
"https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2";
const EMBEDDING_MODEL_URL =
"https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
+const SILERO_VAD_MODEL_URL =
+ "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx";
const SEGMENTATION_DIR = "sherpa-onnx-pyannote-segmentation-3-0";
const SEGMENTATION_ONNX = path.join(SEGMENTATION_DIR, "model.onnx");
const EMBEDDING_ONNX = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
+const SILERO_VAD_ONNX = "silero_vad.onnx";
class DiarizationManager {
constructor() {
@@ -49,22 +52,52 @@ class DiarizationManager {
return getModelsDirForService("diarization");
}
+ getBundledModelsDir() {
+ if (!process.resourcesPath) {
+ return null;
+ }
+
+ return path.join(process.resourcesPath, "bin", "diarization-models");
+ }
+
+ _resolveModelPath(relativePath) {
+ const bundledModelsDir = this.getBundledModelsDir();
+ if (bundledModelsDir) {
+ const bundledPath = path.join(bundledModelsDir, relativePath);
+ if (fs.existsSync(bundledPath)) {
+ return bundledPath;
+ }
+ }
+
+ return path.join(this.getModelsDir(), relativePath);
+ }
+
isModelDownloaded() {
- const modelsDir = this.getModelsDir();
- const segPath = path.join(modelsDir, SEGMENTATION_ONNX);
- const embPath = path.join(modelsDir, EMBEDDING_ONNX);
+ const segPath = this._resolveModelPath(SEGMENTATION_ONNX);
+ const embPath = this._resolveModelPath(EMBEDDING_ONNX);
return fs.existsSync(segPath) && fs.existsSync(embPath);
}
+ getVadModelPath() {
+ return this._resolveModelPath(SILERO_VAD_ONNX);
+ }
+
+ isVadModelDownloaded() {
+ return fs.existsSync(this.getVadModelPath());
+ }
+
async downloadModels(progressCallback = null) {
const modelsDir = this.getModelsDir();
await fsPromises.mkdir(modelsDir, { recursive: true });
- if (this.isModelDownloaded()) {
+ const modelsReady = this.isModelDownloaded();
+ const vadReady = this.isVadModelDownloaded();
+
+ if (modelsReady && vadReady) {
return { success: true, path: modelsDir };
}
- const requiredBytes = 35 * 1_000_000; // ~35MB total
+ const requiredBytes = modelsReady ? 2 * 1_000_000 : 37 * 1_000_000;
const spaceCheck = await checkDiskSpace(modelsDir, requiredBytes * 2.5);
if (!spaceCheck.ok) {
throw new Error(
@@ -132,6 +165,34 @@ class DiarizationManager {
});
}
+ if (!this.isVadModelDownloaded()) {
+ try {
+ await downloadFile(SILERO_VAD_MODEL_URL, this.getVadModelPath(), {
+ timeout: 600000,
+ signal,
+ onProgress: (downloadedBytes, totalBytes) => {
+ if (progressCallback) {
+ progressCallback({
+ type: "progress",
+ stage: "vad",
+ downloaded_bytes: downloadedBytes,
+ total_bytes: totalBytes,
+ percentage: totalBytes > 0 ? Math.round((downloadedBytes / totalBytes) * 100) : 0,
+ });
+ }
+ },
+ });
+ } catch (error) {
+ if (error.isAbort) {
+ throw new Error("Download interrupted by user");
+ }
+ debugLogger.warn("Silero VAD model download failed", {
+ error: error.message,
+ modelsDir,
+ });
+ }
+ }
+
if (progressCallback) {
progressCallback({ type: "complete", percentage: 100 });
}
@@ -226,9 +287,8 @@ class DiarizationManager {
return [];
}
- const modelsDir = this.getModelsDir();
- const segPath = path.join(modelsDir, SEGMENTATION_ONNX);
- const embPath = path.join(modelsDir, EMBEDDING_ONNX);
+ const segPath = this._resolveModelPath(SEGMENTATION_ONNX);
+ const embPath = this._resolveModelPath(EMBEDDING_ONNX);
const args = [
`--segmentation.pyannote-model=${segPath}`,
@@ -431,6 +491,7 @@ class DiarizationManager {
const modelsDir = this.getModelsDir();
const segDir = path.join(modelsDir, SEGMENTATION_DIR);
const embPath = path.join(modelsDir, EMBEDDING_ONNX);
+ const vadPath = this.getVadModelPath();
if (fs.existsSync(segDir)) {
await fsPromises.rm(segDir, { recursive: true, force: true });
@@ -438,6 +499,9 @@ class DiarizationManager {
if (fs.existsSync(embPath)) {
await fsPromises.unlink(embPath);
}
+ if (fs.existsSync(vadPath)) {
+ await fsPromises.unlink(vadPath);
+ }
debugLogger.info("Diarization models deleted", { modelsDir });
return { success: true };
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 91453cccf..0848556e9 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -11,6 +11,8 @@ const { i18nMain, changeLanguage } = require("./i18nMain");
const DeepgramStreaming = require("./deepgramStreaming");
const OpenAIRealtimeStreaming = require("./openaiRealtimeStreaming");
const AudioStorageManager = require("./audioStorage");
+const liveSpeakerIdentifier = require("./liveSpeakerIdentifier");
+const { downsample24kTo16k, pcm16ToWav } = require("../utils/audioUtils");
const MISTRAL_TRANSCRIPTION_URL = "https://api.mistral.ai/v1/audio/transcriptions";
@@ -119,6 +121,7 @@ class IPCHandlers {
this.audioStorageManager = new AudioStorageManager();
this._audioCleanupInterval = null;
this._noteFilesEnabled = false;
+ liveSpeakerIdentifier.setDiarizationManager(this.diarizationManager);
this._setupTextEditMonitor();
this._setupAudioCleanup();
this._logDetectedGpus();
@@ -1490,7 +1493,9 @@ class IPCHandlers {
ipcMain.handle("get-diarization-model-status", async () => {
return {
available: this.diarizationManager?.isAvailable() ?? false,
- modelsDownloaded: this.diarizationManager?.isModelDownloaded() ?? false,
+ modelsDownloaded:
+ (this.diarizationManager?.isModelDownloaded() ?? false) &&
+ (this.diarizationManager?.isVadModelDownloaded() ?? false),
};
});
@@ -2933,47 +2938,12 @@ class IPCHandlers {
let meetingSendCounts = { mic: 0, system: 0 };
- // Meeting mic captures at 24kHz (for OpenAI Realtime), but local engines
- // expect 16kHz. Downsample with linear interpolation (3:2 ratio).
- const downsample24kTo16k = (pcmBuffer) => {
- const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);
- const ratio = 1.5; // 24000 / 16000
- const outputLength = Math.floor(input.length / ratio);
- const output = new Int16Array(outputLength);
- for (let i = 0; i < outputLength; i++) {
- const srcIdx = i * ratio;
- const idx = Math.floor(srcIdx);
- const frac = srcIdx - idx;
- const s0 = input[idx];
- const s1 = idx + 1 < input.length ? input[idx + 1] : s0;
- output[i] = Math.round(s0 + frac * (s1 - s0));
- }
- return Buffer.from(output.buffer);
- };
-
- const pcm16ToWav = (pcmBuffer, sampleRate = 16000, channels = 1) => {
- const dataSize = pcmBuffer.length;
- const header = Buffer.alloc(44);
- header.write("RIFF", 0);
- header.writeUInt32LE(36 + dataSize, 4);
- header.write("WAVE", 8);
- header.write("fmt ", 12);
- header.writeUInt32LE(16, 16);
- header.writeUInt16LE(1, 20);
- header.writeUInt16LE(channels, 22);
- header.writeUInt32LE(sampleRate, 24);
- header.writeUInt32LE(sampleRate * channels * 2, 28);
- header.writeUInt16LE(channels * 2, 32);
- header.writeUInt16LE(16, 34);
- header.write("data", 36);
- header.writeUInt32LE(dataSize, 40);
- return Buffer.concat([header, pcmBuffer]);
- };
-
const fs = require("fs");
let meetingDiarizationStream = null;
let meetingDiarizationPath = null;
let meetingDiarizationSegments = [];
+ let meetingLiveSpeakerActive = false;
+ let meetingLiveSpeakerState = null;
let meetingLocalMode = false;
let meetingLocalBuffers = { mic: [], system: [] };
@@ -2984,6 +2954,47 @@ class IPCHandlers {
let meetingLocalModel = null;
let meetingLocalTranscribing = false;
+ const getLiveSpeakerProfiles = () => this.databaseManager.getSpeakerProfiles(true);
+
+ const stopLiveSpeakerIdentification = async () => {
+ if (!meetingLiveSpeakerActive) {
+ return null;
+ }
+
+ meetingLiveSpeakerState = liveSpeakerIdentifier.getTransientMap();
+ meetingLiveSpeakerActive = false;
+ liveSpeakerIdentifier.stop();
+ return meetingLiveSpeakerState;
+ };
+
+ const startLiveSpeakerIdentification = async (win, systemAudioMode) => {
+ await stopLiveSpeakerIdentification();
+
+ if (systemAudioMode !== "native" || !liveSpeakerIdentifier.isAvailable()) {
+ return false;
+ }
+
+ meetingLiveSpeakerState = null;
+ const started = await liveSpeakerIdentifier.start(
+ (identification) => {
+ if (!win || win.isDestroyed()) {
+ return;
+ }
+
+ win.webContents.send("meeting-speaker-identified", identification);
+ },
+ {
+ getSpeakerProfiles: getLiveSpeakerProfiles,
+ }
+ );
+
+ if (started) {
+ meetingLiveSpeakerActive = true;
+ }
+
+ return started;
+ };
+
const transcribeLocalMeetingChunk = async (source) => {
const chunks = meetingLocalBuffers[source];
if (!chunks.length) return;
@@ -3060,6 +3071,8 @@ class IPCHandlers {
clearInterval(meetingLocalTimer);
meetingLocalTimer = null;
}
+ void stopLiveSpeakerIdentification();
+ meetingLiveSpeakerState = null;
meetingLocalMode = false;
meetingLocalBuffers = { mic: [], system: [] };
if (meetingDiarizationStream) {
@@ -3102,6 +3115,7 @@ class IPCHandlers {
if (this.audioTapManager) {
await this.audioTapManager.stop().catch(() => {});
}
+ await stopLiveSpeakerIdentification().catch(() => {});
resetMeetingLocalState();
await disconnectMeetingStreaming().catch(() => {});
};
@@ -3192,6 +3206,7 @@ class IPCHandlers {
const win = BrowserWindow.fromWebContents(event.sender);
attachMeetingStreamingHandlers(this._meetingMicStreaming, win, "mic");
if (systemAudioMode === "native") {
+ await startLiveSpeakerIdentification(win, systemAudioMode);
attachMeetingStreamingHandlers(this._meetingSystemStreaming, win, "system");
await startNativeMeetingSystemAudio(event);
}
@@ -3206,6 +3221,8 @@ class IPCHandlers {
meetingLocalBuffers = { mic: [], system: [] };
meetingLocalTranscript = "";
+ await startLiveSpeakerIdentification(meetingLocalWin, systemAudioMode);
+
meetingLocalTimer = setInterval(() => {
transcribeAllLocalBuffers();
}, 5000);
@@ -3227,6 +3244,7 @@ class IPCHandlers {
}
await connectRealtimeStreaming(event, options);
+ await startLiveSpeakerIdentification(BrowserWindow.fromWebContents(event.sender), systemAudioMode);
if (systemAudioMode === "native") {
await startNativeMeetingSystemAudio(event);
}
@@ -3242,14 +3260,18 @@ class IPCHandlers {
const sendMeetingAudio = (audioBuffer, source) => {
if (source === "system") {
+ const buf = Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer);
+
+ if (meetingLiveSpeakerActive) {
+ void liveSpeakerIdentifier.feedAudio(buf);
+ }
+
if (!meetingDiarizationStream) {
const os = require("os");
meetingDiarizationPath = path.join(os.tmpdir(), `ow-diarize-raw-${Date.now()}.pcm`);
meetingDiarizationStream = fs.createWriteStream(meetingDiarizationPath);
}
- meetingDiarizationStream.write(
- Buffer.isBuffer(audioBuffer) ? audioBuffer : Buffer.from(audioBuffer)
- );
+ meetingDiarizationStream.write(buf);
}
if (meetingLocalMode) {
@@ -3304,6 +3326,8 @@ class IPCHandlers {
await this.audioTapManager.stop();
}
+ const liveSpeakerState = await stopLiveSpeakerIdentification().catch(() => null);
+
const diarizationSessionId = `diar-${Date.now()}`;
const diarizationPcmPath = meetingDiarizationPath;
const diarizationSegments = meetingDiarizationSegments;
@@ -3329,7 +3353,13 @@ class IPCHandlers {
resetMeetingLocalState();
// Fire-and-forget background diarization (or notify skip)
- this._startOrSkipDiarization(diarizationSessionId, diarizationPcmPath, diarizationSegments, diarizationWin);
+ this._startOrSkipDiarization(
+ diarizationSessionId,
+ diarizationPcmPath,
+ diarizationSegments,
+ diarizationWin,
+ liveSpeakerState
+ );
return { success: true, transcript, diarizationSessionId };
}
@@ -3338,7 +3368,13 @@ class IPCHandlers {
const transcript = [results[0]?.text, results[1]?.text].filter(Boolean).join(" ");
// Fire-and-forget background diarization (or notify skip)
- this._startOrSkipDiarization(diarizationSessionId, diarizationPcmPath, diarizationSegments, diarizationWin);
+ this._startOrSkipDiarization(
+ diarizationSessionId,
+ diarizationPcmPath,
+ diarizationSegments,
+ diarizationWin,
+ liveSpeakerState
+ );
return { success: true, transcript, diarizationSessionId };
} catch (error) {
@@ -5093,14 +5129,17 @@ class IPCHandlers {
"set-speaker-mapping",
async (_event, noteId, speakerId, displayName, email, profileId) => {
const embeddings = this.databaseManager.getNoteSpeakerEmbeddings(noteId);
- const speakerEmb = embeddings.find((e) => e.speaker_id === speakerId);
+ const noteSpeakerEmbedding = embeddings.find((e) => e.speaker_id === speakerId);
+ const liveSpeakerEmbedding = liveSpeakerIdentifier.getSpeakerEmbedding(speakerId);
+ const speakerEmbeddingBuffer = noteSpeakerEmbedding?.embedding
+ || (liveSpeakerEmbedding ? Buffer.from(liveSpeakerEmbedding.buffer) : null);
let resolvedProfileId = profileId ?? null;
- if (speakerEmb) {
+ if (speakerEmbeddingBuffer) {
const profile = this.databaseManager.upsertSpeakerProfile(
displayName,
email || null,
- speakerEmb.embedding,
+ speakerEmbeddingBuffer,
resolvedProfileId
);
resolvedProfileId = profile.id;
@@ -5108,6 +5147,7 @@ class IPCHandlers {
}
this.databaseManager.setSpeakerMapping(noteId, speakerId, resolvedProfileId, displayName);
+ liveSpeakerIdentifier.mapSpeaker(speakerId, resolvedProfileId, displayName, noteId);
return { success: true, profileId: resolvedProfileId };
}
);
@@ -5177,7 +5217,99 @@ class IPCHandlers {
});
}
- _startOrSkipDiarization(sessionId, rawPcmPath, transcriptSegments, win) {
+ _applySpeakerName(segments, speakerId, displayName) {
+ if (!displayName) {
+ return;
+ }
+
+ for (const segment of segments) {
+ if (segment.speaker !== speakerId) {
+ continue;
+ }
+
+ segment.speakerName = displayName;
+ segment.suggestedName = undefined;
+ segment.suggestedProfileId = undefined;
+ }
+ }
+
+ _reconcileLiveSpeakerState(liveSpeakerState, speakerEmbeddingsMap, enrichedSegments) {
+ if (!liveSpeakerState || !speakerEmbeddingsMap) {
+ return new Set();
+ }
+
+ const speakerEmbeddings = require("./speakerEmbeddings");
+ const reconciledSpeakers = new Set();
+ const usedLiveSpeakers = new Set();
+ const noteMappings = new Map();
+
+ const liveEntries = Object.entries(liveSpeakerState)
+ .map(([speakerId, data]) => ({
+ speakerId,
+ displayName: data?.displayName || null,
+ profileId: data?.profileId ?? null,
+ noteId: data?.noteId ?? null,
+ embedding: Array.isArray(data?.embedding) ? new Float32Array(data.embedding) : null,
+ }))
+ .filter((entry) => entry.embedding);
+
+ const getMappingsForNote = (noteId) => {
+ if (!noteMappings.has(noteId)) {
+ noteMappings.set(noteId, this.databaseManager.getSpeakerMappings(noteId));
+ }
+ return noteMappings.get(noteId);
+ };
+
+ for (const [mappedId, embeddingArray] of Object.entries(speakerEmbeddingsMap)) {
+ let bestEntry = null;
+ let bestSimilarity = 0;
+
+ for (const entry of liveEntries) {
+ if (usedLiveSpeakers.has(entry.speakerId)) {
+ continue;
+ }
+
+ const similarity = speakerEmbeddings.cosineSimilarity(
+ new Float32Array(embeddingArray),
+ entry.embedding
+ );
+ if (similarity > bestSimilarity) {
+ bestSimilarity = similarity;
+ bestEntry = entry;
+ }
+ }
+
+ if (!bestEntry || bestSimilarity <= 0.7) {
+ continue;
+ }
+
+ usedLiveSpeakers.add(bestEntry.speakerId);
+ reconciledSpeakers.add(mappedId);
+
+ let displayName = bestEntry.displayName;
+ let profileId = bestEntry.profileId;
+
+ if (bestEntry.noteId) {
+ const liveMapping = getMappingsForNote(bestEntry.noteId).find(
+ (mapping) => mapping.speaker_id === bestEntry.speakerId
+ );
+ if (liveMapping) {
+ displayName = liveMapping.display_name || displayName;
+ profileId = liveMapping.profile_id ?? profileId;
+ this.databaseManager.setSpeakerMapping(bestEntry.noteId, mappedId, profileId, displayName);
+ this.databaseManager.removeSpeakerMapping(bestEntry.noteId, bestEntry.speakerId);
+ } else if (displayName) {
+ this.databaseManager.setSpeakerMapping(bestEntry.noteId, mappedId, profileId, displayName);
+ }
+ }
+
+ this._applySpeakerName(enrichedSegments, mappedId, displayName);
+ }
+
+ return reconciledSpeakers;
+ }
+
+ _startOrSkipDiarization(sessionId, rawPcmPath, transcriptSegments, win, liveSpeakerState = null) {
const send = (payload) => {
if (win && !win.isDestroyed()) {
win.webContents.send("meeting-diarization-complete", { sessionId, ...payload });
@@ -5235,7 +5367,7 @@ class IPCHandlers {
const sorted = segs.sort((a, b) => (b.end - b.start) - (a.end - a.start)).slice(0, 3);
const embeddings = [];
for (const seg of sorted) {
- if (seg.end - seg.start < 2) continue;
+ if (seg.end - seg.start < 1.5) continue;
const emb = await speakerEmb.extractEmbedding(tmpWav, seg.start, seg.end);
if (emb) embeddings.push(emb);
}
@@ -5250,12 +5382,25 @@ class IPCHandlers {
debugLogger.debug("Speaker embedding extraction skipped", { error: err.message });
}
+ const reconciledSpeakers = this._reconcileLiveSpeakerState(
+ liveSpeakerState,
+ speakerEmbeddingsMap,
+ enrichedSegments
+ );
+
if (speakerEmbeddingsMap) {
try {
const profiles = this.databaseManager.getSpeakerProfiles(true);
if (profiles.length > 0) {
for (const [mappedId, embArr] of Object.entries(speakerEmbeddingsMap)) {
+ const alreadyMapped = enrichedSegments.some(
+ (segment) => segment.speaker === mappedId && segment.speakerName
+ );
+ if (reconciledSpeakers.has(mappedId) || alreadyMapped) {
+ continue;
+ }
+
const emb = new Float32Array(embArr);
let bestProfile = null;
let bestSim = 0;
diff --git a/src/helpers/liveSpeakerIdentifier.js b/src/helpers/liveSpeakerIdentifier.js
new file mode 100644
index 000000000..09b3ceedd
--- /dev/null
+++ b/src/helpers/liveSpeakerIdentifier.js
@@ -0,0 +1,495 @@
+const fs = require("fs");
+const debugLogger = require("./debugLogger");
+const speakerEmbeddings = require("./speakerEmbeddings");
+const { downsample24kTo16k } = require("../utils/audioUtils");
+
+const SAMPLE_RATE = 16000;
+const VAD_WINDOW_SIZE = 512;
+const MIN_SEGMENT_SECONDS = 1.5;
+const MIN_SEGMENT_SAMPLES = Math.round(SAMPLE_RATE * MIN_SEGMENT_SECONDS);
+const SPEECH_THRESHOLD = 0.15;
+const SILENCE_THRESHOLD = 0.08;
+const SILENCE_WINDOWS_TO_END = 4;
+const MATCH_THRESHOLD = 0.7;
+const DEFAULT_VAD_STATE_SHAPE = [2, 1, 64];
+
+function pcm16BufferToFloat32(buffer) {
+ const input = new Int16Array(buffer.buffer, buffer.byteOffset, buffer.length / 2);
+ const output = new Float32Array(input.length);
+
+ for (let i = 0; i < input.length; i += 1) {
+ output[i] = input[i] / 32768;
+ }
+
+ return output;
+}
+
+function appendFloat32(existing, next) {
+ if (!existing.length) return next;
+ if (!next.length) return existing;
+
+ const merged = new Float32Array(existing.length + next.length);
+ merged.set(existing, 0);
+ merged.set(next, existing.length);
+ return merged;
+}
+
+function concatFloat32Arrays(chunks) {
+ if (chunks.length === 0) return new Float32Array(0);
+ if (chunks.length === 1) return chunks[0];
+
+ let totalLength = 0;
+ for (const chunk of chunks) {
+ totalLength += chunk.length;
+ }
+
+ const merged = new Float32Array(totalLength);
+ let offset = 0;
+ for (const chunk of chunks) {
+ merged.set(chunk, offset);
+ offset += chunk.length;
+ }
+
+ return merged;
+}
+
+function cloneFloat32Array(value) {
+ return new Float32Array(value);
+}
+
+function normalizeVadShape(shape) {
+ if (!Array.isArray(shape) || shape.length === 0) {
+ return DEFAULT_VAD_STATE_SHAPE;
+ }
+
+ return shape.map((dim) => (typeof dim === "number" && dim > 0 ? dim : 1));
+}
+
+function normalizeVadStateName(name) {
+ return String(name || "").replace(/[^a-z0-9]/gi, "").toLowerCase();
+}
+
+function getBufferFloat32View(buffer) {
+ return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
+}
+
+class LiveSpeakerIdentifier {
+ constructor() {
+ this.session = null;
+ this.running = false;
+ this.queue = Promise.resolve();
+ this.onSpeakerIdentified = null;
+ this.getSpeakerProfiles = null;
+ this.audioRemainder = new Float32Array(0);
+ this.vadStateInputs = [];
+ this.vadStateOutputs = [];
+ this.vadStates = new Map();
+ this.speechChunks = [];
+ this.speechActive = false;
+ this.silenceWindows = 0;
+ this.sampleCursor = 0;
+ this.segmentStartSample = 0;
+ this.segmentEndSample = 0;
+ this.transientEmbeddings = new Map();
+ this.transientCounts = new Map();
+ this.transientDisplayNames = new Map();
+ this.transientProfileIds = new Map();
+ this.nextLiveIndex = 0;
+ this._diarizationManager = null;
+ }
+
+ setDiarizationManager(manager) {
+ this._diarizationManager = manager;
+ }
+
+ isAvailable() {
+ return this._diarizationManager?.isVadModelDownloaded() && speakerEmbeddings.isAvailable();
+ }
+
+ getTransientMap() {
+ const map = new Map();
+ for (const [speakerId, embedding] of this.transientEmbeddings.entries()) {
+ map.set(speakerId, cloneFloat32Array(embedding));
+ }
+ return map;
+ }
+
+ getSpeakerEmbedding(speakerId) {
+ const embedding = this.transientEmbeddings.get(speakerId);
+ return embedding ? cloneFloat32Array(embedding) : null;
+ }
+
+ async start(options = {}) {
+ const {
+ onSpeakerIdentified = null,
+ getSpeakerProfiles = null,
+ } = typeof options === "function" ? { onSpeakerIdentified: options } : options;
+
+ this.onSpeakerIdentified =
+ typeof onSpeakerIdentified === "function" ? onSpeakerIdentified : null;
+ this.getSpeakerProfiles = typeof getSpeakerProfiles === "function" ? getSpeakerProfiles : null;
+ this._resetMeetingState();
+
+ if (!this.isAvailable()) {
+ debugLogger.warn("Live speaker identifier unavailable", {
+ vadModelPath: this._diarizationManager?.getVadModelPath(),
+ embeddingModelAvailable: speakerEmbeddings.isAvailable(),
+ });
+ return false;
+ }
+
+ await this._ensureLoaded();
+ this.running = !!this.session;
+ return this.running;
+ }
+
+ async stop() {
+ this.running = false;
+ await this.queue;
+
+ if (this.speechActive && this.audioRemainder.length > 0) {
+ this.speechChunks.push(this.audioRemainder);
+ this.audioRemainder = new Float32Array(0);
+ }
+
+ if (this.speechActive) {
+ await this._finalizeSpeechSegment();
+ }
+
+ const transientMap = this.getTransientMap();
+ this._resetMeetingState();
+ this.onSpeakerIdentified = null;
+ this.getSpeakerProfiles = null;
+ return transientMap;
+ }
+
+ feedAudio(pcmBuffer) {
+ if (!this.running || !pcmBuffer?.length) {
+ return Promise.resolve();
+ }
+
+ this.queue = this.queue
+ .then(() => this._processAudio(pcmBuffer))
+ .catch((error) => {
+ debugLogger.warn("Live speaker identification failed", { error: error.message });
+ });
+
+ return this.queue;
+ }
+
+ mapSpeaker(liveId, profileId, displayName) {
+ if (!liveId || !this.transientEmbeddings.has(liveId)) {
+ return false;
+ }
+
+ if (profileId != null) {
+ this.transientProfileIds.set(liveId, profileId);
+ }
+
+ if (displayName) {
+ this.transientDisplayNames.set(liveId, displayName);
+ }
+
+ return true;
+ }
+
+ async _ensureLoaded() {
+ if (this.session) return;
+
+ const vadModelPath = this._diarizationManager?.getVadModelPath();
+ if (!fs.existsSync(vadModelPath)) {
+ return;
+ }
+
+ const ort = require("onnxruntime-node");
+ this.session = await ort.InferenceSession.create(vadModelPath);
+ this.vadStateInputs = (this.session.inputNames || []).filter((name) =>
+ /state|h|c/i.test(name)
+ );
+ this.vadStateOutputs = (this.session.outputNames || []).filter((name) =>
+ /state|h|c/i.test(name)
+ );
+ this._resetVadRuntimeState();
+ }
+
+ _resetMeetingState() {
+ this.queue = Promise.resolve();
+ this.audioRemainder = new Float32Array(0);
+ this.speechChunks = [];
+ this.speechActive = false;
+ this.silenceWindows = 0;
+ this.sampleCursor = 0;
+ this.segmentStartSample = 0;
+ this.segmentEndSample = 0;
+ this.transientEmbeddings = new Map();
+ this.transientCounts = new Map();
+ this.transientDisplayNames = new Map();
+ this.transientProfileIds = new Map();
+ this.nextLiveIndex = 0;
+ this._resetVadRuntimeState();
+ }
+
+ _resetVadRuntimeState() {
+ this.vadStates = new Map();
+
+ if (!this.session) {
+ return;
+ }
+
+ for (const name of this.vadStateInputs) {
+ const metadata = this.session.inputMetadata?.[name];
+ const shape = normalizeVadShape(metadata?.dimensions || metadata?.shape);
+ const size = shape.reduce((total, dim) => total * dim, 1);
+ this.vadStates.set(name, new Float32Array(size));
+ }
+ }
+
+ async _processAudio(pcmBuffer) {
+ await this._ensureLoaded();
+ if (!this.session) return;
+
+ const downsampled = downsample24kTo16k(
+ Buffer.isBuffer(pcmBuffer) ? pcmBuffer : Buffer.from(pcmBuffer)
+ );
+ if (!downsampled.length) return;
+
+ this.audioRemainder = appendFloat32(this.audioRemainder, pcm16BufferToFloat32(downsampled));
+
+ while (this.audioRemainder.length >= VAD_WINDOW_SIZE) {
+ const window = this.audioRemainder.subarray(0, VAD_WINDOW_SIZE);
+ this.audioRemainder = this.audioRemainder.slice(VAD_WINDOW_SIZE);
+
+ const windowStartSample = this.sampleCursor;
+ this.sampleCursor += window.length;
+
+ await this._processWindow(window, windowStartSample, this.sampleCursor);
+ }
+ }
+
+ async _processWindow(window, windowStartSample, windowEndSample) {
+ const probability = await this._getVadProbability(window);
+
+ if (this.speechActive) {
+ this.speechChunks.push(cloneFloat32Array(window));
+ this.segmentEndSample = windowEndSample;
+
+ if (probability >= SILENCE_THRESHOLD) {
+ this.silenceWindows = 0;
+ return;
+ }
+
+ this.silenceWindows += 1;
+ if (this.silenceWindows >= SILENCE_WINDOWS_TO_END) {
+ await this._finalizeSpeechSegment();
+ }
+ return;
+ }
+
+ if (probability < SPEECH_THRESHOLD) {
+ return;
+ }
+
+ this.speechActive = true;
+ this.segmentStartSample = windowStartSample;
+ this.segmentEndSample = windowEndSample;
+ this.speechChunks = [cloneFloat32Array(window)];
+ this.silenceWindows = 0;
+ }
+
+ async _getVadProbability(window) {
+ if (!this.session) return 0;
+
+ const ort = require("onnxruntime-node");
+ const feeds = {};
+ const audioInputName = (this.session.inputNames || []).find(
+ (name) => !this.vadStateInputs.includes(name) && !/sr|sample.?rate/i.test(name)
+ );
+
+ if (!audioInputName) {
+ return 0;
+ }
+
+ feeds[audioInputName] = new ort.Tensor("float32", window, [1, window.length]);
+
+ const sampleRateInputName = (this.session.inputNames || []).find((name) =>
+ /sr|sample.?rate/i.test(name)
+ );
+ if (sampleRateInputName) {
+ feeds[sampleRateInputName] = new ort.Tensor(
+ "int64",
+ BigInt64Array.from([BigInt(SAMPLE_RATE)]),
+ [1]
+ );
+ }
+
+ for (const stateName of this.vadStateInputs) {
+ const metadata = this.session.inputMetadata?.[stateName];
+ const shape = normalizeVadShape(metadata?.dimensions || metadata?.shape);
+ const state =
+ this.vadStates.get(stateName) || new Float32Array(shape.reduce((a, b) => a * b, 1));
+
+ this.vadStates.set(stateName, state);
+ feeds[stateName] = new ort.Tensor("float32", state, shape);
+ }
+
+ const results = await this.session.run(feeds);
+ this._updateVadState(results);
+
+ const outputName = (this.session.outputNames || []).find(
+ (name) => !this.vadStateOutputs.includes(name)
+ );
+ const output = (outputName && results[outputName]) || Object.values(results)[0];
+ const value = output?.data?.[0];
+ return typeof value === "number" ? value : 0;
+ }
+
+ _updateVadState(results) {
+ if (!results) {
+ return;
+ }
+
+ for (const inputName of this.vadStateInputs) {
+ const expectedName = normalizeVadStateName(inputName);
+ const matchingOutput = this.vadStateOutputs.find(
+ (outputName) => normalizeVadStateName(outputName).startsWith(expectedName)
+ );
+ const output = (matchingOutput && results[matchingOutput]) || results[inputName];
+
+ if (output?.data) {
+ this.vadStates.set(inputName, new Float32Array(output.data));
+ }
+ }
+ }
+
+ async _finalizeSpeechSegment() {
+ const samples = concatFloat32Arrays(this.speechChunks);
+ this.speechChunks = [];
+ this.speechActive = false;
+ this.silenceWindows = 0;
+
+ if (samples.length < MIN_SEGMENT_SAMPLES) {
+ return;
+ }
+
+ const embedding = await speakerEmbeddings.extractEmbeddingFromSamples(samples);
+ if (!embedding) {
+ return;
+ }
+
+ let speakerId = this._findTransientMatch(embedding);
+ if (speakerId) {
+ this._updateCentroid(speakerId, embedding);
+ } else {
+ const matchedProfile = this._findStoredProfileMatch(embedding);
+ if (matchedProfile) {
+ speakerId = this._findTransientSpeakerForProfile(matchedProfile.id);
+ if (!speakerId) {
+ speakerId = this._assignSpeakerId(embedding);
+ } else {
+ this._updateCentroid(speakerId, embedding);
+ }
+
+ this.transientProfileIds.set(speakerId, matchedProfile.id);
+ this.transientDisplayNames.set(speakerId, matchedProfile.display_name);
+ } else {
+ speakerId = this._assignSpeakerId(embedding);
+ }
+ }
+
+ this.onSpeakerIdentified?.({
+ speakerId,
+ displayName: this.transientDisplayNames.get(speakerId) || null,
+ startTime: this.segmentStartSample / SAMPLE_RATE,
+ endTime: this.segmentEndSample / SAMPLE_RATE,
+ });
+ }
+
+ _findTransientMatch(embedding) {
+ let bestSpeakerId = null;
+ let bestSimilarity = 0;
+
+ for (const [speakerId, centroid] of this.transientEmbeddings.entries()) {
+ const similarity = speakerEmbeddings.cosineSimilarity(embedding, centroid);
+ if (similarity > bestSimilarity) {
+ bestSimilarity = similarity;
+ bestSpeakerId = speakerId;
+ }
+ }
+
+ return bestSimilarity >= MATCH_THRESHOLD ? bestSpeakerId : null;
+ }
+
+ _findStoredProfileMatch(embedding) {
+ let profiles = [];
+
+ try {
+ profiles = this.getSpeakerProfiles?.() || [];
+ } catch (error) {
+ debugLogger.debug("Live speaker profile lookup failed", { error: error.message });
+ return null;
+ }
+
+ let bestProfile = null;
+ let bestSimilarity = 0;
+
+ for (const profile of profiles) {
+ if (!profile?.embedding) continue;
+
+ const profileEmbedding =
+ profile.embedding instanceof Float32Array
+ ? profile.embedding
+ : Array.isArray(profile.embedding)
+ ? new Float32Array(profile.embedding)
+ : getBufferFloat32View(profile.embedding);
+
+ if (!profileEmbedding.length) continue;
+
+ const similarity = speakerEmbeddings.cosineSimilarity(embedding, profileEmbedding);
+ if (similarity > bestSimilarity) {
+ bestSimilarity = similarity;
+ bestProfile = profile;
+ }
+ }
+
+ return bestSimilarity >= MATCH_THRESHOLD ? bestProfile : null;
+ }
+
+ _findTransientSpeakerForProfile(profileId) {
+ for (const [speakerId, candidateProfileId] of this.transientProfileIds.entries()) {
+ if (candidateProfileId === profileId) {
+ return speakerId;
+ }
+ }
+
+ return null;
+ }
+
+ _assignSpeakerId(embedding) {
+ const speakerId = `live_${this.nextLiveIndex}`;
+ this.nextLiveIndex += 1;
+ this.transientEmbeddings.set(speakerId, cloneFloat32Array(embedding));
+ this.transientCounts.set(speakerId, 1);
+ return speakerId;
+ }
+
+ _updateCentroid(speakerId, embedding) {
+ const centroid = this.transientEmbeddings.get(speakerId);
+ if (!centroid) {
+ return;
+ }
+
+ const count = this.transientCounts.get(speakerId) || 1;
+ const nextCentroid = new Float32Array(embedding.length);
+
+ for (let i = 0; i < embedding.length; i += 1) {
+ nextCentroid[i] = (centroid[i] * count + embedding[i]) / (count + 1);
+ }
+
+ this.transientEmbeddings.set(speakerId, nextCentroid);
+ this.transientCounts.set(speakerId, count + 1);
+ }
+}
+
+const instance = new LiveSpeakerIdentifier();
+module.exports = instance;
+module.exports.LiveSpeakerIdentifier = LiveSpeakerIdentifier;
diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.js
index bcc2d4856..65e49301e 100644
--- a/src/helpers/speakerEmbeddings.js
+++ b/src/helpers/speakerEmbeddings.js
@@ -4,39 +4,197 @@ const debugLogger = require("./debugLogger");
const { getModelsDirForService } = require("./modelDirUtils");
const EMBEDDING_DIM = 512;
-const MIN_SEGMENT_SECONDS = 2;
+const MIN_SEGMENT_SECONDS = 1.5;
+const MIN_SEGMENT_SAMPLES = 16000 * MIN_SEGMENT_SECONDS;
const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
+const FBANK_SAMPLE_RATE = 16000;
+const FBANK_FRAME_LENGTH_MS = 25;
+const FBANK_FRAME_SHIFT_MS = 10;
+const FBANK_NUM_MELS = 80;
+const FBANK_FRAME_LENGTH = Math.round(FBANK_SAMPLE_RATE * FBANK_FRAME_LENGTH_MS / 1000);
+const FBANK_FRAME_SHIFT = Math.round(FBANK_SAMPLE_RATE * FBANK_FRAME_SHIFT_MS / 1000);
+const FBANK_FFT_SIZE = 512;
+
+let _melFilterbank = null;
+
+function getMelFilterbank() {
+ if (_melFilterbank) return _melFilterbank;
+
+ const numBins = FBANK_FFT_SIZE / 2 + 1;
+ const lowFreq = 20;
+ const highFreq = FBANK_SAMPLE_RATE / 2;
+
+ const melLow = 1127 * Math.log(1 + lowFreq / 700);
+ const melHigh = 1127 * Math.log(1 + highFreq / 700);
+
+ const melPoints = new Float64Array(FBANK_NUM_MELS + 2);
+ for (let i = 0; i < melPoints.length; i++) {
+ const mel = melLow + (melHigh - melLow) * i / (FBANK_NUM_MELS + 1);
+ melPoints[i] = 700 * (Math.exp(mel / 1127) - 1);
+ }
+
+ const binPoints = new Float64Array(melPoints.length);
+ for (let i = 0; i < melPoints.length; i++) {
+ binPoints[i] = Math.floor((FBANK_FFT_SIZE + 1) * melPoints[i] / FBANK_SAMPLE_RATE);
+ }
+
+ _melFilterbank = new Array(FBANK_NUM_MELS);
+ for (let m = 0; m < FBANK_NUM_MELS; m++) {
+ const filter = new Float32Array(numBins);
+ const left = binPoints[m];
+ const center = binPoints[m + 1];
+ const right = binPoints[m + 2];
+
+ for (let k = 0; k < numBins; k++) {
+ if (k >= left && k <= center && center > left) {
+ filter[k] = (k - left) / (center - left);
+ } else if (k > center && k <= right && right > center) {
+ filter[k] = (right - k) / (right - center);
+ }
+ }
+ _melFilterbank[m] = filter;
+ }
+
+ return _melFilterbank;
+}
+
+function realFFT(frame) {
+ const n = FBANK_FFT_SIZE;
+ const re = new Float64Array(n);
+ const im = new Float64Array(n);
+ for (let i = 0; i < frame.length && i < n; i++) re[i] = frame[i];
+
+ for (let i = 1, j = 0; i < n; i++) {
+ let bit = n >> 1;
+ while (j & bit) { j ^= bit; bit >>= 1; }
+ j ^= bit;
+ if (i < j) { [re[i], re[j]] = [re[j], re[i]]; }
+ }
+
+ for (let len = 2; len <= n; len <<= 1) {
+ const angle = -2 * Math.PI / len;
+ const wRe = Math.cos(angle);
+ const wIm = Math.sin(angle);
+ for (let i = 0; i < n; i += len) {
+ let curRe = 1, curIm = 0;
+ for (let j = 0; j < len / 2; j++) {
+ const uRe = re[i + j], uIm = im[i + j];
+ const vRe = re[i + j + len / 2] * curRe - im[i + j + len / 2] * curIm;
+ const vIm = re[i + j + len / 2] * curIm + im[i + j + len / 2] * curRe;
+ re[i + j] = uRe + vRe; im[i + j] = uIm + vIm;
+ re[i + j + len / 2] = uRe - vRe; im[i + j + len / 2] = uIm - vIm;
+ const tmpRe = curRe * wRe - curIm * wIm;
+ curIm = curRe * wIm + curIm * wRe;
+ curRe = tmpRe;
+ }
+ }
+ }
+
+ const numBins = n / 2 + 1;
+ const powerSpectrum = new Float32Array(numBins);
+ for (let i = 0; i < numBins; i++) {
+ powerSpectrum[i] = re[i] * re[i] + im[i] * im[i];
+ }
+ return powerSpectrum;
+}
+
+function computeFbank(samples) {
+ const numFrames = Math.max(0, Math.floor((samples.length - FBANK_FRAME_LENGTH) / FBANK_FRAME_SHIFT) + 1);
+ if (numFrames === 0) return null;
+
+ const hamming = new Float32Array(FBANK_FRAME_LENGTH);
+ for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {
+ hamming[i] = 0.54 - 0.46 * Math.cos(2 * Math.PI * i / (FBANK_FRAME_LENGTH - 1));
+ }
+
+ const melBank = getMelFilterbank();
+ const features = new Float32Array(numFrames * FBANK_NUM_MELS);
+
+ for (let f = 0; f < numFrames; f++) {
+ const start = f * FBANK_FRAME_SHIFT;
+ const frame = new Float32Array(FBANK_FRAME_LENGTH);
+ for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {
+ frame[i] = (samples[start + i] || 0) * hamming[i];
+ }
+
+ const power = realFFT(frame);
+
+ for (let m = 0; m < FBANK_NUM_MELS; m++) {
+ let energy = 0;
+ const filter = melBank[m];
+ for (let k = 0; k < power.length; k++) {
+ energy += filter[k] * power[k];
+ }
+ features[f * FBANK_NUM_MELS + m] = Math.log(Math.max(energy, 1e-10));
+ }
+ }
+
+ return { features, numFrames };
+}
+
class SpeakerEmbeddings {
constructor() {
this.session = null;
- this.modelPath = path.join(getModelsDirForService("diarization"), MODEL_FILE);
+ }
+
+ getModelPath() {
+ if (process.resourcesPath) {
+ const bundledPath = path.join(process.resourcesPath, "bin", "diarization-models", MODEL_FILE);
+ if (fs.existsSync(bundledPath)) {
+ return bundledPath;
+ }
+ }
+
+ return path.join(getModelsDirForService("diarization"), MODEL_FILE);
}
isAvailable() {
- return fs.existsSync(this.modelPath);
+ return fs.existsSync(this.getModelPath());
}
async _ensureLoaded() {
if (this.session) return;
if (!this.isAvailable()) {
- throw new Error(`Speaker embedding model not found at ${this.modelPath}`);
+ throw new Error(`Speaker embedding model not found at ${this.getModelPath()}`);
}
- debugLogger.debug("speaker-embeddings loading model", { modelPath: this.modelPath });
+ const modelPath = this.getModelPath();
+ debugLogger.debug("speaker-embeddings loading model", { modelPath });
const ort = require("onnxruntime-node");
- this.session = await ort.InferenceSession.create(this.modelPath);
+ this.session = await ort.InferenceSession.create(modelPath);
debugLogger.debug("speaker-embeddings model loaded");
}
+ async _extractEmbeddingFromSamples(samples) {
+ await this._ensureLoaded();
+
+ const fbank = computeFbank(samples);
+ if (!fbank) return null;
+
+ const ort = require("onnxruntime-node");
+ const inputName = this.session.inputNames[0];
+ const feeds = {
+ [inputName]: new ort.Tensor("float32", fbank.features, [1, fbank.numFrames, FBANK_NUM_MELS]),
+ };
+
+ const results = await this.session.run(feeds);
+ const output = results[Object.keys(results)[0]];
+
+ return new Float32Array(output.data);
+ }
+
+ async extractEmbeddingFromSamples(samples) {
+ if (samples.length < MIN_SEGMENT_SAMPLES) return null;
+ return this._extractEmbeddingFromSamples(samples);
+ }
+
async extractEmbedding(wavPath, startSec, endSec) {
if (endSec - startSec < MIN_SEGMENT_SECONDS) return null;
- await this._ensureLoaded();
-
const buf = fs.readFileSync(wavPath);
const { sampleRate, dataOffset } = this._parseWavHeader(buf);
@@ -55,16 +213,7 @@ class SpeakerEmbeddings {
samples[i] = int16 / 32768;
}
- const ort = require("onnxruntime-node");
- const inputName = this.session.inputNames[0];
- const feeds = {
- [inputName]: new ort.Tensor("float32", samples, [1, numSamples]),
- };
-
- const results = await this.session.run(feeds);
- const output = results[Object.keys(results)[0]];
-
- return new Float32Array(output.data);
+ return this._extractEmbeddingFromSamples(samples);
}
_parseWavHeader(buf) {
diff --git a/src/hooks/useMeetingTranscription.ts b/src/hooks/useMeetingTranscription.ts
index ca59814b1..6d898903f 100644
--- a/src/hooks/useMeetingTranscription.ts
+++ b/src/hooks/useMeetingTranscription.ts
@@ -32,6 +32,7 @@ const MEETING_AUDIO_BUFFER_SIZE = 800;
const MEETING_STOP_FLUSH_TIMEOUT_MS = 50;
const REALTIME_MODELS = new Set(["gpt-4o-mini-transcribe", "gpt-4o-transcribe"]);
+const SPEAKER_IDENTIFICATION_RETENTION_MS = 30_000;
const getMeetingTranscriptionOptions = () => {
const {
@@ -256,6 +257,31 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
const preparePromiseRef = useRef<Promise<void> | null>(null);
const ipcCleanupsRef = useRef<Array<() => void>>([]);
const pendingCleanupRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+ const speakerIdentificationsRef = useRef<
+ Array<{ speakerId: string; displayName?: string; startTime: number; endTime: number }>
+ >([]);
+
+ const applySpeakerIdentification = useCallback(
+ (
+ segment: TranscriptSegment,
+ identification: { speakerId: string; displayName?: string; startTime: number; endTime: number }
+ ): TranscriptSegment => {
+ if (segment.source !== "system" || segment.timestamp == null) {
+ return segment;
+ }
+
+ if (segment.timestamp < identification.startTime || segment.timestamp > identification.endTime) {
+ return segment;
+ }
+
+ return {
+ ...segment,
+ speaker: identification.speakerId,
+ speakerName: identification.displayName ?? segment.speakerName,
+ };
+ },
+ []
+ );
const cleanup = useCallback(async () => {
await flushAndDisconnectProcessor(micProcessorRef.current);
@@ -361,6 +387,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
setMicPartial("");
setSystemPartial("");
setError(null);
+ speakerIdentificationsRef.current = [];
// Set recording state immediately for instant UI feedback
isRecordingRef.current = true;
@@ -450,12 +477,13 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
setPartialForSource(data.text);
setPartialTranscript(data.text);
} else {
- const seg: TranscriptSegment = {
+ const rawSegment: TranscriptSegment = {
id: `seg-${++segmentCounter}`,
text: data.text,
source: data.source,
timestamp: data.timestamp,
};
+ const seg = speakerIdentificationsRef.current.reduce(applySpeakerIdentification, rawSegment);
setSegments((prev) => {
// Insert in chronological order — scan from the end since most
// segments arrive in order and this is O(1) in the common case.
@@ -473,6 +501,20 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
);
if (segmentCleanup) ipcCleanupsRef.current.push(segmentCleanup);
+ const speakerCleanup = window.electronAPI?.onMeetingSpeakerIdentified?.((data) => {
+ speakerIdentificationsRef.current = [
+ ...speakerIdentificationsRef.current.filter(
+ (identification) => identification.endTime >= data.endTime - SPEAKER_IDENTIFICATION_RETENTION_MS
+ ),
+ data,
+ ];
+
+ setSegments((prev) =>
+ prev.map((segment) => applySpeakerIdentification(segment, data))
+ );
+ });
+ if (speakerCleanup) ipcCleanupsRef.current.push(speakerCleanup);
+
const errorCleanup = window.electronAPI?.onMeetingTranscriptionError?.((err) => {
setError(err);
logger.error("Meeting transcription stream error", { error: err }, "meeting");
@@ -564,7 +606,7 @@ export function useMeetingTranscription(): UseMeetingTranscriptionReturn {
setIsRecording(false);
await cleanup();
}
- }, [cleanup]);
+ }, [applySpeakerIdentification, cleanup]);
useEffect(() => {
getMeetingWorkletBlobUrl();
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 29b246e67..e2b9ae61a 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -1306,6 +1306,14 @@ declare global {
type: "partial" | "final";
}) => void
) => () => void;
+ onMeetingSpeakerIdentified?: (
+ callback: (data: {
+ speakerId: string;
+ displayName?: string;
+ startTime: number;
+ endTime: number;
+ }) => void
+ ) => () => void;
onMeetingTranscriptionError?: (callback: (error: string) => void) => () => void;
// Speaker diarization
diff --git a/src/utils/audioUtils.js b/src/utils/audioUtils.js
new file mode 100644
index 000000000..fb7d751be
--- /dev/null
+++ b/src/utils/audioUtils.js
@@ -0,0 +1,41 @@
+function downsample24kTo16k(pcmBuffer) {
+ const input = new Int16Array(pcmBuffer.buffer, pcmBuffer.byteOffset, pcmBuffer.length / 2);
+ const ratio = 1.5;
+ const outputLength = Math.floor(input.length / ratio);
+ const output = new Int16Array(outputLength);
+
+ for (let i = 0; i < outputLength; i++) {
+ const srcIdx = i * ratio;
+ const idx = Math.floor(srcIdx);
+ const frac = srcIdx - idx;
+ const s0 = input[idx];
+ const s1 = idx + 1 < input.length ? input[idx + 1] : s0;
+ output[i] = Math.round(s0 + frac * (s1 - s0));
+ }
+
+ return Buffer.from(output.buffer);
+}
+
+function pcm16ToWav(pcmBuffer, sampleRate = 16000, channels = 1) {
+ const dataSize = pcmBuffer.length;
+ const header = Buffer.alloc(44);
+ header.write("RIFF", 0);
+ header.writeUInt32LE(36 + dataSize, 4);
+ header.write("WAVE", 8);
+ header.write("fmt ", 12);
+ header.writeUInt32LE(16, 16);
+ header.writeUInt16LE(1, 20);
+ header.writeUInt16LE(channels, 22);
+ header.writeUInt32LE(sampleRate, 24);
+ header.writeUInt32LE(sampleRate * channels * 2, 28);
+ header.writeUInt16LE(channels * 2, 32);
+ header.writeUInt16LE(16, 34);
+ header.write("data", 36);
+ header.writeUInt32LE(dataSize, 40);
+ return Buffer.concat([header, pcmBuffer]);
+}
+
+module.exports = {
+ downsample24kTo16k,
+ pcm16ToWav,
+};