Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(onnx): cap embedding segments and isolate inference in utility process (#693)

Gabriel Stein1d7fe1e82ea5
diff --git a/CLAUDE.md b/CLAUDE.md
index f53731e6e..db09ae384 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -28,6 +28,7 @@ OpenWhispr is an Electron-based desktop dictation application that uses whisper.
- Main Process: Electron main, IPC handlers, database operations
- Renderer Process: React app with context isolation
- Preload Script: Secure bridge between processes
+ - ONNX Utility Process: hosts all `onnxruntime-node` inference (text embeddings, speaker embeddings, fbank). Lazy-spawned on first use via `src/helpers/onnxWorkerClient.js` → `src/workers/onnxWorker.js`. Native crashes (e.g., ORT `bad_alloc`) confine to the worker; main process rejects in-flight requests and respawns with backoff. Stopped in `will-quit`.
3. **Audio Pipeline**:
- MediaRecorder API → Blob → ArrayBuffer → IPC → File → whisper.cpp
diff --git a/main.js b/main.js
index 511544781..2676e28d8 100644
--- a/main.js
+++ b/main.js
@@ -1437,5 +1437,7 @@ if (gotSingleInstanceLock) {
if (qdrantManager) {
qdrantManager.stop().catch(() => {});
}
+ const onnxWorkerClient = require("./src/helpers/onnxWorkerClient");
+ onnxWorkerClient.stop().catch(() => {});
});
}
diff --git a/src/helpers/liveSpeakerIdentifier.js b/src/helpers/liveSpeakerIdentifier.js
index 4a8aad7ce..5059c1d8d 100644
--- a/src/helpers/liveSpeakerIdentifier.js
+++ b/src/helpers/liveSpeakerIdentifier.js
@@ -1,6 +1,7 @@
const fs = require("fs");
const debugLogger = require("./debugLogger");
const speakerEmbeddings = require("./speakerEmbeddings");
+const { MAX_EMBEDDING_SECONDS } = speakerEmbeddings;
const { downsample24kTo16k } = require("../utils/audioUtils");
const { MAX_SPEAKER_COUNT } = require("../constants/speakerDetection.json");
@@ -20,6 +21,8 @@ const LIVE_IDENTIFICATION_INTERVAL_SECONDS = 1.0;
const LIVE_IDENTIFICATION_INTERVAL_SAMPLES = Math.round(
SAMPLE_RATE * LIVE_IDENTIFICATION_INTERVAL_SECONDS
);
+const MAX_EMBEDDING_SAMPLES = SAMPLE_RATE * MAX_EMBEDDING_SECONDS;
+const SPEECH_CHUNKS_MAX_SAMPLES = MAX_EMBEDDING_SAMPLES * 4;
const SPEECH_THRESHOLD = 0.15;
const SILENCE_THRESHOLD = 0.08;
const SILENCE_WINDOWS_TO_END = 24;
@@ -68,6 +71,36 @@ function concatFloat32Arrays(chunks) {
return merged;
}
+function trimChunksToMaxSamples(chunks, maxSamples) {
+ let total = 0;
+ for (const chunk of chunks) total += chunk.length;
+ while (total > maxSamples && chunks.length > 0) {
+ total -= chunks[0].length;
+ chunks.shift();
+ }
+}
+
+function selectBestEmbeddingWindow(samples) {
+ if (samples.length <= MAX_EMBEDDING_SAMPLES) return samples;
+
+ const stride = SAMPLE_RATE;
+ let bestStart = 0;
+ let bestEnergy = -Infinity;
+
+ for (let start = 0; start + MAX_EMBEDDING_SAMPLES <= samples.length; start += stride) {
+ let energy = 0;
+ for (let i = start; i < start + MAX_EMBEDDING_SAMPLES; i++) {
+ energy += Math.abs(samples[i]);
+ }
+ if (energy > bestEnergy) {
+ bestEnergy = energy;
+ bestStart = start;
+ }
+ }
+
+ return samples.subarray(bestStart, bestStart + MAX_EMBEDDING_SAMPLES);
+}
+
function cloneFloat32Array(value) {
return new Float32Array(value);
}
@@ -409,6 +442,7 @@ class LiveSpeakerIdentifier {
if (this.speechActive) {
this.speechChunks.push(cloneFloat32Array(window));
+ trimChunksToMaxSamples(this.speechChunks, SPEECH_CHUNKS_MAX_SAMPLES);
this.segmentEndSample = windowEndSample;
if (probability >= SILENCE_THRESHOLD) {
@@ -504,10 +538,14 @@ class LiveSpeakerIdentifier {
}
async _identifyActiveSpeechSegment(force = false) {
- const currentSamples = concatFloat32Arrays(this.speechChunks);
- if (currentSamples.length < LIVE_IDENTIFICATION_MIN_SAMPLES) {
+ const allSamples = concatFloat32Arrays(this.speechChunks);
+ if (allSamples.length < LIVE_IDENTIFICATION_MIN_SAMPLES) {
return;
}
+ const currentSamples =
+ allSamples.length > MAX_EMBEDDING_SAMPLES
+ ? allSamples.subarray(allSamples.length - MAX_EMBEDDING_SAMPLES)
+ : allSamples;
if (
!force &&
@@ -552,7 +590,9 @@ class LiveSpeakerIdentifier {
return;
}
- const embedding = await speakerEmbeddings.extractEmbeddingFromSamples(samples);
+ const embedding = await speakerEmbeddings.extractEmbeddingFromSamples(
+ selectBestEmbeddingWindow(samples)
+ );
if (!embedding) {
return;
}
diff --git a/src/helpers/localEmbeddings.js b/src/helpers/localEmbeddings.js
index 5732fc556..25d7e71ee 100644
--- a/src/helpers/localEmbeddings.js
+++ b/src/helpers/localEmbeddings.js
@@ -2,16 +2,13 @@ const fs = require("fs");
const os = require("os");
const path = require("path");
const debugLogger = require("./debugLogger");
-
-const MAX_TOKENS = 256;
-const EMBEDDING_DIM = 384;
+const onnxWorkerClient = require("./onnxWorkerClient");
const MODEL_SUBDIR = "all-MiniLM-L6-v2";
class LocalEmbeddings {
constructor() {
- this.session = null;
- this.tokenizer = null;
+ this.loadPromise = null;
this.modelDir = this._resolveModelDir();
}
@@ -52,111 +49,28 @@ class LocalEmbeddings {
);
}
- async _ensureLoaded() {
- if (this.session && this.tokenizer) return;
-
+ _ensureLoaded() {
+ if (this.loadPromise) return this.loadPromise;
if (!this.isAvailable()) {
- throw new Error("Embedding model not found. Run: node scripts/download-minilm.js");
+ return Promise.reject(
+ new Error("Embedding model not found. Run: node scripts/download-minilm.js")
+ );
}
-
debugLogger.debug("local-embeddings loading model", { modelDir: this.modelDir });
-
- const tokenizerPath = path.join(this.modelDir, "tokenizer.json");
- const tokenizerData = JSON.parse(fs.readFileSync(tokenizerPath, "utf-8"));
- this.tokenizer = this._buildTokenizer(tokenizerData);
-
- const ort = require("onnxruntime-node");
- const modelPath = path.join(this.modelDir, "model.onnx");
- this.session = await ort.InferenceSession.create(modelPath);
-
- debugLogger.debug("local-embeddings model loaded");
- }
-
- _buildTokenizer(tokenizerData) {
- const tokenToId = new Map();
- for (const [token, id] of Object.entries(tokenizerData.model.vocab)) {
- tokenToId.set(token, id);
- }
-
- return {
- tokenToId,
- clsId: tokenToId.get("[CLS]") ?? 101,
- sepId: tokenToId.get("[SEP]") ?? 102,
- unkId: tokenToId.get("[UNK]") ?? 100,
- };
- }
-
- _tokenize(text) {
- const { tokenToId, clsId, sepId, unkId } = this.tokenizer;
- const words = text.toLowerCase().match(/[a-z0-9]+|[^\s\w]/g) || [];
- const tokenIds = [clsId];
-
- for (const word of words) {
- if (tokenIds.length >= MAX_TOKENS - 1) break;
-
- if (tokenToId.has(word)) {
- tokenIds.push(tokenToId.get(word));
- continue;
- }
-
- // WordPiece: greedily match longest subword
- let start = 0;
- while (start < word.length) {
- if (tokenIds.length >= MAX_TOKENS - 1) break;
-
- let end = word.length;
- let matched = false;
-
- while (end > start) {
- const subword = start === 0 ? word.slice(start, end) : `##${word.slice(start, end)}`;
- if (tokenToId.has(subword)) {
- tokenIds.push(tokenToId.get(subword));
- start = end;
- matched = true;
- break;
- }
- end--;
- }
-
- if (!matched) {
- tokenIds.push(unkId);
- start++;
- }
- }
- }
-
- tokenIds.push(sepId);
-
- const length = tokenIds.length;
- const inputIds = new BigInt64Array(length);
- const attentionMask = new BigInt64Array(length);
- const tokenTypeIds = new BigInt64Array(length);
-
- for (let i = 0; i < length; i++) {
- inputIds[i] = BigInt(tokenIds[i]);
- attentionMask[i] = 1n;
- tokenTypeIds[i] = 0n;
- }
-
- return { inputIds, attentionMask, tokenTypeIds, length };
+ this.loadPromise = onnxWorkerClient
+ .request("text.load", { modelDir: this.modelDir })
+ .then(() => debugLogger.debug("local-embeddings model loaded"))
+ .catch((err) => {
+ this.loadPromise = null;
+ throw err;
+ });
+ return this.loadPromise;
}
async embedText(text) {
await this._ensureLoaded();
-
- const { inputIds, attentionMask, tokenTypeIds, length } = this._tokenize(text);
-
- const ort = require("onnxruntime-node");
- const feeds = {
- input_ids: new ort.Tensor("int64", inputIds, [1, length]),
- attention_mask: new ort.Tensor("int64", attentionMask, [1, length]),
- token_type_ids: new ort.Tensor("int64", tokenTypeIds, [1, length]),
- };
-
- const results = await this.session.run(feeds);
- const output = results.last_hidden_state ?? results.output_0;
-
- return this._meanPoolAndNormalize(output.data, length, EMBEDDING_DIM);
+ const { embeddingBuffer } = await onnxWorkerClient.request("text.embed", { text });
+ return new Float32Array(embeddingBuffer);
}
async embedTexts(texts) {
@@ -167,36 +81,6 @@ class LocalEmbeddings {
return results;
}
- // Mean pooling over token embeddings, then L2 normalize to unit vector
- _meanPoolAndNormalize(data, tokenCount, dim) {
- const embedding = new Float32Array(dim);
-
- for (let t = 0; t < tokenCount; t++) {
- const offset = t * dim;
- for (let d = 0; d < dim; d++) {
- embedding[d] += data[offset + d];
- }
- }
-
- for (let d = 0; d < dim; d++) {
- embedding[d] /= tokenCount;
- }
-
- let norm = 0;
- for (let d = 0; d < dim; d++) {
- norm += embedding[d] * embedding[d];
- }
- norm = Math.sqrt(norm);
-
- if (norm > 0) {
- for (let d = 0; d < dim; d++) {
- embedding[d] /= norm;
- }
- }
-
- return embedding;
- }
-
static noteEmbedText(title, content, enhancedContent) {
return `${title}\n${enhancedContent || content}`.slice(0, 1500);
}
diff --git a/src/helpers/onnxWorkerClient.js b/src/helpers/onnxWorkerClient.js
new file mode 100644
index 000000000..253d15004
--- /dev/null
+++ b/src/helpers/onnxWorkerClient.js
@@ -0,0 +1,226 @@
+const path = require("path");
+const { app, utilityProcess, MessageChannelMain } = require("electron");
+const debugLogger = require("./debugLogger");
+
+const REQUEST_TIMEOUT_MS = 30_000;
+const MAX_PENDING_REQUESTS = 1000;
+const RESPAWN_BACKOFF_MS = [1000, 2000, 4000, 8000, 16000, 30000];
+const SHUTDOWN_TIMEOUT_MS = 5000;
+
+const WORKER_SCRIPT = path.join(__dirname, "..", "workers", "onnxWorker.js");
+
+class WorkerCrashedError extends Error {
+ constructor(message = "ONNX worker crashed") {
+ super(message);
+ this.name = "WorkerCrashedError";
+ }
+}
+
+class WorkerOverloadedError extends Error {
+ constructor() {
+ super("ONNX worker request queue full");
+ this.name = "WorkerOverloadedError";
+ }
+}
+
+class OnnxWorkerClient {
+ constructor() {
+ this.child = null;
+ this.port = null;
+ this.pending = new Map();
+ this.nextRequestId = 1;
+ this.crashCount = 0;
+ this.shuttingDown = false;
+ this.spawnPromise = null;
+ this.respawnTimer = null;
+ }
+
+ _logPath() {
+ try {
+ return path.join(app.getPath("userData"), "logs", "onnx-worker.log");
+ } catch {
+ return null;
+ }
+ }
+
+ async _spawn() {
+ if (this.child) return;
+ if (this.spawnPromise) return this.spawnPromise;
+
+ this.spawnPromise = (async () => {
+ const env = { ...process.env };
+ const logPath = this._logPath();
+ if (logPath) env.OPENWHISPR_ONNX_WORKER_LOG = logPath;
+
+ const child = utilityProcess.fork(WORKER_SCRIPT, [], {
+ serviceName: "openwhispr-onnx",
+ stdio: "ignore",
+ env,
+ execArgv: ["--max-old-space-size=512"],
+ });
+
+ await new Promise((resolve, reject) => {
+ const onSpawn = () => {
+ child.removeListener("error", onError);
+ resolve();
+ };
+ const onError = (err) => {
+ child.removeListener("spawn", onSpawn);
+ reject(err);
+ };
+ child.once("spawn", onSpawn);
+ child.once("error", onError);
+ });
+
+ const { port1, port2 } = new MessageChannelMain();
+ port1.on("message", (event) => this._onMessage(event.data));
+ port1.start();
+
+ child.postMessage("init", [port2]);
+
+ child.on("exit", (code) => this._onExit(code));
+
+ this.child = child;
+ this.port = port1;
+ debugLogger.info("onnx worker spawned", { pid: child.pid });
+ })();
+
+ try {
+ await this.spawnPromise;
+ } finally {
+ this.spawnPromise = null;
+ }
+ }
+
+ _onMessage(reply) {
+ const entry = this.pending.get(reply.id);
+ if (!entry) return;
+ this.pending.delete(reply.id);
+ clearTimeout(entry.timeout);
+ if (reply.error) {
+ entry.reject(new Error(reply.error.message));
+ } else {
+ this.crashCount = 0;
+ entry.resolve(reply.result);
+ }
+ }
+
+ _onExit(code) {
+ debugLogger.warn("onnx worker exited", {
+ code,
+ pending: this.pending.size,
+ shuttingDown: this.shuttingDown,
+ });
+
+ this.child = null;
+ if (this.port) {
+ try {
+ this.port.close();
+ } catch {
+ // already closed
+ }
+ this.port = null;
+ }
+
+ const err = new WorkerCrashedError();
+ for (const entry of this.pending.values()) {
+ clearTimeout(entry.timeout);
+ entry.reject(err);
+ }
+ this.pending.clear();
+
+ if (this.shuttingDown) return;
+
+ if (code !== 0) {
+ this.crashCount += 1;
+ const delay =
+ RESPAWN_BACKOFF_MS[Math.min(this.crashCount - 1, RESPAWN_BACKOFF_MS.length - 1)];
+ debugLogger.info("onnx worker respawn scheduled", {
+ delayMs: delay,
+ crashCount: this.crashCount,
+ });
+ this.respawnTimer = setTimeout(() => {
+ this.respawnTimer = null;
+ this._spawn().catch((spawnErr) => {
+ debugLogger.error("onnx worker respawn failed", { error: spawnErr?.message });
+ });
+ }, delay);
+ }
+ }
+
+ async request(method, payload, transferList) {
+ if (this.shuttingDown) {
+ throw new WorkerCrashedError("worker shutting down");
+ }
+
+ if (this.respawnTimer) {
+ throw new WorkerCrashedError("worker restarting");
+ }
+
+ if (this.pending.size >= MAX_PENDING_REQUESTS) {
+ const oldestId = this.pending.keys().next().value;
+ const oldest = this.pending.get(oldestId);
+ if (oldest) {
+ this.pending.delete(oldestId);
+ clearTimeout(oldest.timeout);
+ oldest.reject(new WorkerOverloadedError());
+ }
+ }
+
+ await this._spawn();
+
+ const id = this.nextRequestId++;
+ return new Promise((resolve, reject) => {
+ const timeout = setTimeout(() => {
+ if (this.pending.delete(id)) {
+ reject(new Error(`onnx worker request timeout: ${method}`));
+ }
+ }, REQUEST_TIMEOUT_MS);
+ this.pending.set(id, { resolve, reject, timeout });
+ try {
+ this.port.postMessage({ id, method, payload }, transferList || []);
+ } catch (err) {
+ clearTimeout(timeout);
+ this.pending.delete(id);
+ reject(err);
+ }
+ });
+ }
+
+ async stop() {
+ this.shuttingDown = true;
+ if (this.respawnTimer) {
+ clearTimeout(this.respawnTimer);
+ this.respawnTimer = null;
+ }
+ if (!this.child) return;
+
+ const child = this.child;
+ try {
+ this.port?.postMessage({ id: 0, method: "shutdown", payload: {} });
+ } catch {
+ // worker may already be gone
+ }
+
+ await new Promise((resolve) => {
+ const timer = setTimeout(() => {
+ try {
+ child.kill();
+ } catch {
+ // already dead
+ }
+ resolve();
+ }, SHUTDOWN_TIMEOUT_MS);
+ child.once("exit", () => {
+ clearTimeout(timer);
+ resolve();
+ });
+ });
+ }
+}
+
+const instance = new OnnxWorkerClient();
+module.exports = instance;
+module.exports.OnnxWorkerClient = OnnxWorkerClient;
+module.exports.WorkerCrashedError = WorkerCrashedError;
+module.exports.WorkerOverloadedError = WorkerOverloadedError;
diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.js
index e7e23f9df..16ba39c31 100644
--- a/src/helpers/speakerEmbeddings.js
+++ b/src/helpers/speakerEmbeddings.js
@@ -2,152 +2,19 @@ const fs = require("fs");
const path = require("path");
const debugLogger = require("./debugLogger");
const { getModelsDirForService } = require("./modelDirUtils");
+const onnxWorkerClient = require("./onnxWorkerClient");
+const SAMPLE_RATE = 16000;
const EMBEDDING_DIM = 512;
const MIN_SEGMENT_SECONDS = 1.5;
-const MIN_SEGMENT_SAMPLES = 16000 * MIN_SEGMENT_SECONDS;
+const MIN_SEGMENT_SAMPLES = SAMPLE_RATE * MIN_SEGMENT_SECONDS;
+const MAX_EMBEDDING_SECONDS = 8;
+const MAX_EMBEDDING_SAMPLES = SAMPLE_RATE * MAX_EMBEDDING_SECONDS;
const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
-const FBANK_SAMPLE_RATE = 16000;
-const FBANK_FRAME_LENGTH_MS = 25;
-const FBANK_FRAME_SHIFT_MS = 10;
-const FBANK_NUM_MELS = 80;
-const FBANK_FRAME_LENGTH = Math.round((FBANK_SAMPLE_RATE * FBANK_FRAME_LENGTH_MS) / 1000);
-const FBANK_FRAME_SHIFT = Math.round((FBANK_SAMPLE_RATE * FBANK_FRAME_SHIFT_MS) / 1000);
-const FBANK_FFT_SIZE = 512;
-
-let _melFilterbank = null;
-
-function getMelFilterbank() {
- if (_melFilterbank) return _melFilterbank;
-
- const numBins = FBANK_FFT_SIZE / 2 + 1;
- const lowFreq = 20;
- const highFreq = FBANK_SAMPLE_RATE / 2;
-
- const melLow = 1127 * Math.log(1 + lowFreq / 700);
- const melHigh = 1127 * Math.log(1 + highFreq / 700);
-
- const melPoints = new Float64Array(FBANK_NUM_MELS + 2);
- for (let i = 0; i < melPoints.length; i++) {
- const mel = melLow + ((melHigh - melLow) * i) / (FBANK_NUM_MELS + 1);
- melPoints[i] = 700 * (Math.exp(mel / 1127) - 1);
- }
-
- const binPoints = new Float64Array(melPoints.length);
- for (let i = 0; i < melPoints.length; i++) {
- binPoints[i] = Math.floor(((FBANK_FFT_SIZE + 1) * melPoints[i]) / FBANK_SAMPLE_RATE);
- }
-
- _melFilterbank = new Array(FBANK_NUM_MELS);
- for (let m = 0; m < FBANK_NUM_MELS; m++) {
- const filter = new Float32Array(numBins);
- const left = binPoints[m];
- const center = binPoints[m + 1];
- const right = binPoints[m + 2];
-
- for (let k = 0; k < numBins; k++) {
- if (k >= left && k <= center && center > left) {
- filter[k] = (k - left) / (center - left);
- } else if (k > center && k <= right && right > center) {
- filter[k] = (right - k) / (right - center);
- }
- }
- _melFilterbank[m] = filter;
- }
-
- return _melFilterbank;
-}
-
-function realFFT(frame) {
- const n = FBANK_FFT_SIZE;
- const re = new Float64Array(n);
- const im = new Float64Array(n);
- for (let i = 0; i < frame.length && i < n; i++) re[i] = frame[i];
-
- for (let i = 1, j = 0; i < n; i++) {
- let bit = n >> 1;
- while (j & bit) {
- j ^= bit;
- bit >>= 1;
- }
- j ^= bit;
- if (i < j) {
- [re[i], re[j]] = [re[j], re[i]];
- }
- }
-
- for (let len = 2; len <= n; len <<= 1) {
- const angle = (-2 * Math.PI) / len;
- const wRe = Math.cos(angle);
- const wIm = Math.sin(angle);
- for (let i = 0; i < n; i += len) {
- let curRe = 1,
- curIm = 0;
- for (let j = 0; j < len / 2; j++) {
- const uRe = re[i + j],
- uIm = im[i + j];
- const vRe = re[i + j + len / 2] * curRe - im[i + j + len / 2] * curIm;
- const vIm = re[i + j + len / 2] * curIm + im[i + j + len / 2] * curRe;
- re[i + j] = uRe + vRe;
- im[i + j] = uIm + vIm;
- re[i + j + len / 2] = uRe - vRe;
- im[i + j + len / 2] = uIm - vIm;
- const tmpRe = curRe * wRe - curIm * wIm;
- curIm = curRe * wIm + curIm * wRe;
- curRe = tmpRe;
- }
- }
- }
-
- const numBins = n / 2 + 1;
- const powerSpectrum = new Float32Array(numBins);
- for (let i = 0; i < numBins; i++) {
- powerSpectrum[i] = re[i] * re[i] + im[i] * im[i];
- }
- return powerSpectrum;
-}
-
-function computeFbank(samples) {
- const numFrames = Math.max(
- 0,
- Math.floor((samples.length - FBANK_FRAME_LENGTH) / FBANK_FRAME_SHIFT) + 1
- );
- if (numFrames === 0) return null;
-
- const hamming = new Float32Array(FBANK_FRAME_LENGTH);
- for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {
- hamming[i] = 0.54 - 0.46 * Math.cos((2 * Math.PI * i) / (FBANK_FRAME_LENGTH - 1));
- }
-
- const melBank = getMelFilterbank();
- const features = new Float32Array(numFrames * FBANK_NUM_MELS);
-
- for (let f = 0; f < numFrames; f++) {
- const start = f * FBANK_FRAME_SHIFT;
- const frame = new Float32Array(FBANK_FRAME_LENGTH);
- for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {
- frame[i] = (samples[start + i] || 0) * hamming[i];
- }
-
- const power = realFFT(frame);
-
- for (let m = 0; m < FBANK_NUM_MELS; m++) {
- let energy = 0;
- const filter = melBank[m];
- for (let k = 0; k < power.length; k++) {
- energy += filter[k] * power[k];
- }
- features[f * FBANK_NUM_MELS + m] = Math.log(Math.max(energy, 1e-10));
- }
- }
-
- return { features, numFrames };
-}
-
class SpeakerEmbeddings {
constructor() {
- this.session = null;
+ this.loadPromise = null;
}
getModelPath() {
@@ -165,43 +32,50 @@ class SpeakerEmbeddings {
return fs.existsSync(this.getModelPath());
}
- async _ensureLoaded() {
- if (this.session) return;
-
+ _ensureLoaded() {
+ if (this.loadPromise) return this.loadPromise;
if (!this.isAvailable()) {
- throw new Error(`Speaker embedding model not found at ${this.getModelPath()}`);
+ return Promise.reject(
+ new Error(`Speaker embedding model not found at ${this.getModelPath()}`)
+ );
}
-
const modelPath = this.getModelPath();
debugLogger.debug("speaker-embeddings loading model", { modelPath });
-
- const ort = require("onnxruntime-node");
- this.session = await ort.InferenceSession.create(modelPath);
-
- debugLogger.debug("speaker-embeddings model loaded");
+ this.loadPromise = onnxWorkerClient
+ .request("speaker.load", { modelPath })
+ .then(() => debugLogger.debug("speaker-embeddings model loaded"))
+ .catch((err) => {
+ this.loadPromise = null;
+ throw err;
+ });
+ return this.loadPromise;
}
async _extractEmbeddingFromSamples(samples) {
await this._ensureLoaded();
- const fbank = computeFbank(samples);
- if (!fbank) return null;
+ const samplesBuffer = samples.buffer.slice(
+ samples.byteOffset,
+ samples.byteOffset + samples.byteLength
+ );
- const ort = require("onnxruntime-node");
- const inputName = this.session.inputNames[0];
- const feeds = {
- [inputName]: new ort.Tensor("float32", fbank.features, [1, fbank.numFrames, FBANK_NUM_MELS]),
- };
+ const { embeddingBuffer } = await onnxWorkerClient.request(
+ "speaker.extract",
+ { samplesBuffer },
+ [samplesBuffer]
+ );
- const results = await this.session.run(feeds);
- const output = results[Object.keys(results)[0]];
-
- return new Float32Array(output.data);
+ if (!embeddingBuffer) return null;
+ return new Float32Array(embeddingBuffer);
}
async extractEmbeddingFromSamples(samples) {
if (samples.length < MIN_SEGMENT_SAMPLES) return null;
- return this._extractEmbeddingFromSamples(samples);
+ const capped =
+ samples.length > MAX_EMBEDDING_SAMPLES
+ ? samples.subarray(samples.length - MAX_EMBEDDING_SAMPLES)
+ : samples;
+ return this._extractEmbeddingFromSamples(capped);
}
async extractEmbedding(wavPath, startSec, endSec) {
@@ -210,7 +84,9 @@ class SpeakerEmbeddings {
const buf = fs.readFileSync(wavPath);
const { sampleRate, dataOffset } = this._parseWavHeader(buf);
- const startSample = Math.floor(startSec * sampleRate);
+ const cappedSeconds = Math.min(endSec - startSec, MAX_EMBEDDING_SECONDS);
+ const cappedStartSec = endSec - cappedSeconds;
+ const startSample = Math.floor(cappedStartSec * sampleRate);
const endSample = Math.floor(endSec * sampleRate);
const numSamples = endSample - startSample;
@@ -282,3 +158,4 @@ class SpeakerEmbeddings {
const instance = new SpeakerEmbeddings();
module.exports = instance;
module.exports.SpeakerEmbeddings = SpeakerEmbeddings;
+module.exports.MAX_EMBEDDING_SECONDS = MAX_EMBEDDING_SECONDS;
diff --git a/src/workers/onnxWorker.js b/src/workers/onnxWorker.js
new file mode 100644
index 000000000..20adc281e
--- /dev/null
+++ b/src/workers/onnxWorker.js
@@ -0,0 +1,393 @@
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+
+const FBANK_SAMPLE_RATE = 16000;
+const FBANK_FRAME_LENGTH_MS = 25;
+const FBANK_FRAME_SHIFT_MS = 10;
+const FBANK_NUM_MELS = 80;
+const FBANK_FRAME_LENGTH = Math.round((FBANK_SAMPLE_RATE * FBANK_FRAME_LENGTH_MS) / 1000);
+const FBANK_FRAME_SHIFT = Math.round((FBANK_SAMPLE_RATE * FBANK_FRAME_SHIFT_MS) / 1000);
+const FBANK_FFT_SIZE = 512;
+
+const SPEAKER_MAX_SAMPLES = FBANK_SAMPLE_RATE * 8;
+
+const TEXT_EMBED_MAX_TOKENS = 256;
+const TEXT_EMBED_DIM = 384;
+
+const intraOpNumThreads = Math.min(4, Math.max(2, Math.floor((os.cpus()?.length || 4) / 2)));
+
+let port = null;
+let ort = null;
+let speakerSession = null;
+let speakerInputName = null;
+let textSession = null;
+let textTokenizer = null;
+let logStream = null;
+
+function log(level, message, extra) {
+ if (!logStream) return;
+ const line = JSON.stringify({
+ ts: new Date().toISOString(),
+ level,
+ pid: process.pid,
+ message,
+ ...(extra || {}),
+ });
+ try {
+ logStream.write(line + "\n");
+ } catch {
+ // Best-effort logging; never throw from log path.
+ }
+}
+
+function openLog() {
+ const logPath = process.env.OPENWHISPR_ONNX_WORKER_LOG;
+ if (!logPath) return;
+ try {
+ fs.mkdirSync(path.dirname(logPath), { recursive: true });
+ logStream = fs.createWriteStream(logPath, { flags: "a" });
+ } catch {
+ logStream = null;
+ }
+}
+
+function loadOrt() {
+ if (ort) return;
+ ort = require("onnxruntime-node");
+ log("info", "ort loaded");
+}
+
+const SESSION_OPTIONS = { intraOpNumThreads, executionMode: "sequential" };
+
+let _melFilterbank = null;
+function getMelFilterbank() {
+ if (_melFilterbank) return _melFilterbank;
+
+ const numBins = FBANK_FFT_SIZE / 2 + 1;
+ const lowFreq = 20;
+ const highFreq = FBANK_SAMPLE_RATE / 2;
+ const melLow = 1127 * Math.log(1 + lowFreq / 700);
+ const melHigh = 1127 * Math.log(1 + highFreq / 700);
+
+ const melPoints = new Float64Array(FBANK_NUM_MELS + 2);
+ for (let i = 0; i < melPoints.length; i++) {
+ const mel = melLow + ((melHigh - melLow) * i) / (FBANK_NUM_MELS + 1);
+ melPoints[i] = 700 * (Math.exp(mel / 1127) - 1);
+ }
+
+ const binPoints = new Float64Array(melPoints.length);
+ for (let i = 0; i < melPoints.length; i++) {
+ binPoints[i] = Math.floor(((FBANK_FFT_SIZE + 1) * melPoints[i]) / FBANK_SAMPLE_RATE);
+ }
+
+ _melFilterbank = new Array(FBANK_NUM_MELS);
+ for (let m = 0; m < FBANK_NUM_MELS; m++) {
+ const filter = new Float32Array(numBins);
+ const left = binPoints[m];
+ const center = binPoints[m + 1];
+ const right = binPoints[m + 2];
+ for (let k = 0; k < numBins; k++) {
+ if (k >= left && k <= center && center > left) {
+ filter[k] = (k - left) / (center - left);
+ } else if (k > center && k <= right && right > center) {
+ filter[k] = (right - k) / (right - center);
+ }
+ }
+ _melFilterbank[m] = filter;
+ }
+
+ return _melFilterbank;
+}
+
+function realFFT(frame) {
+ const n = FBANK_FFT_SIZE;
+ const re = new Float64Array(n);
+ const im = new Float64Array(n);
+ for (let i = 0; i < frame.length && i < n; i++) re[i] = frame[i];
+
+ for (let i = 1, j = 0; i < n; i++) {
+ let bit = n >> 1;
+ while (j & bit) {
+ j ^= bit;
+ bit >>= 1;
+ }
+ j ^= bit;
+ if (i < j) {
+ [re[i], re[j]] = [re[j], re[i]];
+ }
+ }
+
+ for (let len = 2; len <= n; len <<= 1) {
+ const angle = (-2 * Math.PI) / len;
+ const wRe = Math.cos(angle);
+ const wIm = Math.sin(angle);
+ for (let i = 0; i < n; i += len) {
+ let curRe = 1;
+ let curIm = 0;
+ for (let j = 0; j < len / 2; j++) {
+ const uRe = re[i + j];
+ const uIm = im[i + j];
+ const vRe = re[i + j + len / 2] * curRe - im[i + j + len / 2] * curIm;
+ const vIm = re[i + j + len / 2] * curIm + im[i + j + len / 2] * curRe;
+ re[i + j] = uRe + vRe;
+ im[i + j] = uIm + vIm;
+ re[i + j + len / 2] = uRe - vRe;
+ im[i + j + len / 2] = uIm - vIm;
+ const tmpRe = curRe * wRe - curIm * wIm;
+ curIm = curRe * wIm + curIm * wRe;
+ curRe = tmpRe;
+ }
+ }
+ }
+
+ const numBins = n / 2 + 1;
+ const powerSpectrum = new Float32Array(numBins);
+ for (let i = 0; i < numBins; i++) {
+ powerSpectrum[i] = re[i] * re[i] + im[i] * im[i];
+ }
+ return powerSpectrum;
+}
+
+function computeFbank(samples) {
+ const numFrames = Math.max(
+ 0,
+ Math.floor((samples.length - FBANK_FRAME_LENGTH) / FBANK_FRAME_SHIFT) + 1
+ );
+ if (numFrames === 0) return null;
+
+ const hamming = new Float32Array(FBANK_FRAME_LENGTH);
+ for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {
+ hamming[i] = 0.54 - 0.46 * Math.cos((2 * Math.PI * i) / (FBANK_FRAME_LENGTH - 1));
+ }
+
+ const melBank = getMelFilterbank();
+ const features = new Float32Array(numFrames * FBANK_NUM_MELS);
+
+ for (let f = 0; f < numFrames; f++) {
+ const start = f * FBANK_FRAME_SHIFT;
+ const frame = new Float32Array(FBANK_FRAME_LENGTH);
+ for (let i = 0; i < FBANK_FRAME_LENGTH; i++) {
+ frame[i] = (samples[start + i] || 0) * hamming[i];
+ }
+ const power = realFFT(frame);
+ for (let m = 0; m < FBANK_NUM_MELS; m++) {
+ let energy = 0;
+ const filter = melBank[m];
+ for (let k = 0; k < power.length; k++) {
+ energy += filter[k] * power[k];
+ }
+ features[f * FBANK_NUM_MELS + m] = Math.log(Math.max(energy, 1e-10));
+ }
+ }
+
+ return { features, numFrames };
+}
+
+async function speakerLoad({ modelPath }) {
+ if (speakerSession) return { ok: true };
+ loadOrt();
+ speakerSession = await ort.InferenceSession.create(modelPath, SESSION_OPTIONS);
+ speakerInputName = speakerSession.inputNames[0];
+ log("info", "speaker session loaded", { modelPath });
+ return { ok: true };
+}
+
+async function speakerExtract({ samplesBuffer }) {
+ if (!speakerSession) throw new Error("speaker session not loaded");
+
+ const allSamples = new Float32Array(samplesBuffer);
+ const samples =
+ allSamples.length > SPEAKER_MAX_SAMPLES
+ ? allSamples.subarray(allSamples.length - SPEAKER_MAX_SAMPLES)
+ : allSamples;
+
+ const fbank = computeFbank(samples);
+ if (!fbank) return { embeddingBuffer: null };
+
+ const feeds = {
+ [speakerInputName]: new ort.Tensor("float32", fbank.features, [
+ 1,
+ fbank.numFrames,
+ FBANK_NUM_MELS,
+ ]),
+ };
+ const results = await speakerSession.run(feeds);
+ const output = results[Object.keys(results)[0]];
+ const data = new Float32Array(output.data);
+ return { embeddingBuffer: data.buffer };
+}
+
+function buildTextTokenizer(tokenizerData) {
+ const tokenToId = new Map();
+ for (const [token, id] of Object.entries(tokenizerData.model.vocab)) {
+ tokenToId.set(token, id);
+ }
+ return {
+ tokenToId,
+ clsId: tokenToId.get("[CLS]") ?? 101,
+ sepId: tokenToId.get("[SEP]") ?? 102,
+ unkId: tokenToId.get("[UNK]") ?? 100,
+ };
+}
+
+function tokenizeText(text) {
+ const { tokenToId, clsId, sepId, unkId } = textTokenizer;
+ const words = text.toLowerCase().match(/[a-z0-9]+|[^\s\w]/g) || [];
+ const tokenIds = [clsId];
+
+ for (const word of words) {
+ if (tokenIds.length >= TEXT_EMBED_MAX_TOKENS - 1) break;
+
+ if (tokenToId.has(word)) {
+ tokenIds.push(tokenToId.get(word));
+ continue;
+ }
+
+ let start = 0;
+ while (start < word.length) {
+ if (tokenIds.length >= TEXT_EMBED_MAX_TOKENS - 1) break;
+ let end = word.length;
+ let matched = false;
+ while (end > start) {
+ const subword = start === 0 ? word.slice(start, end) : `##${word.slice(start, end)}`;
+ if (tokenToId.has(subword)) {
+ tokenIds.push(tokenToId.get(subword));
+ start = end;
+ matched = true;
+ break;
+ }
+ end--;
+ }
+ if (!matched) {
+ tokenIds.push(unkId);
+ start++;
+ }
+ }
+ }
+
+ tokenIds.push(sepId);
+
+ const length = tokenIds.length;
+ const inputIds = new BigInt64Array(length);
+ const attentionMask = new BigInt64Array(length);
+ const tokenTypeIds = new BigInt64Array(length);
+ for (let i = 0; i < length; i++) {
+ inputIds[i] = BigInt(tokenIds[i]);
+ attentionMask[i] = 1n;
+ tokenTypeIds[i] = 0n;
+ }
+ return { inputIds, attentionMask, tokenTypeIds, length };
+}
+
+function meanPoolAndNormalize(data, tokenCount, dim) {
+ const embedding = new Float32Array(dim);
+ for (let t = 0; t < tokenCount; t++) {
+ const offset = t * dim;
+ for (let d = 0; d < dim; d++) {
+ embedding[d] += data[offset + d];
+ }
+ }
+ for (let d = 0; d < dim; d++) {
+ embedding[d] /= tokenCount;
+ }
+
+ let norm = 0;
+ for (let d = 0; d < dim; d++) {
+ norm += embedding[d] * embedding[d];
+ }
+ norm = Math.sqrt(norm);
+ if (norm > 0) {
+ for (let d = 0; d < dim; d++) {
+ embedding[d] /= norm;
+ }
+ }
+ return embedding;
+}
+
+async function textLoad({ modelDir }) {
+ if (textSession && textTokenizer) return { ok: true };
+ loadOrt();
+
+ const tokenizerData = JSON.parse(fs.readFileSync(path.join(modelDir, "tokenizer.json"), "utf-8"));
+ textTokenizer = buildTextTokenizer(tokenizerData);
+
+ textSession = await ort.InferenceSession.create(
+ path.join(modelDir, "model.onnx"),
+ SESSION_OPTIONS
+ );
+ log("info", "text session loaded", { modelDir });
+ return { ok: true };
+}
+
+async function textEmbed({ text }) {
+ if (!textSession) throw new Error("text session not loaded");
+
+ const { inputIds, attentionMask, tokenTypeIds, length } = tokenizeText(text);
+ const feeds = {
+ input_ids: new ort.Tensor("int64", inputIds, [1, length]),
+ attention_mask: new ort.Tensor("int64", attentionMask, [1, length]),
+ token_type_ids: new ort.Tensor("int64", tokenTypeIds, [1, length]),
+ };
+ const results = await textSession.run(feeds);
+ const output = results.last_hidden_state ?? results.output_0;
+ const embedding = meanPoolAndNormalize(output.data, length, TEXT_EMBED_DIM);
+ return { embeddingBuffer: embedding.buffer };
+}
+
+const handlers = {
+ ping: () => ({ ok: true, sessions: { speaker: !!speakerSession, text: !!textSession } }),
+ "speaker.load": speakerLoad,
+ "speaker.extract": speakerExtract,
+ "text.load": textLoad,
+ "text.embed": textEmbed,
+ shutdown: () => {
+ log("info", "shutdown requested");
+ setImmediate(() => process.exit(0));
+ return { ok: true };
+ },
+};
+
+async function dispatch({ id, method, payload }) {
+ const handler = handlers[method];
+ if (!handler) {
+ return { reply: { id, error: { message: `unknown method: ${method}` } }, transferList: [] };
+ }
+ try {
+ const result = await handler(payload || {});
+ const transferList = [];
+ if (result?.embeddingBuffer) transferList.push(result.embeddingBuffer);
+ return { reply: { id, result }, transferList };
+ } catch (err) {
+ log("error", "handler threw", { method, error: err?.message, stack: err?.stack });
+ return { reply: { id, error: { message: err?.message || String(err) } }, transferList: [] };
+ }
+}
+
+process.parentPort.once("message", ({ data, ports }) => {
+ if (data === "init" && ports?.length) {
+ port = ports[0];
+ port.on("message", async (event) => {
+ const message = event.data;
+ const { reply, transferList } = await dispatch(message);
+ port.postMessage(reply, transferList);
+ });
+ port.on("close", () => {
+ log("info", "port closed");
+ process.exit(0);
+ });
+ port.start();
+ log("info", "worker initialized", { intraOpNumThreads });
+ }
+});
+
+process.on("uncaughtException", (err) => {
+ log("fatal", "uncaughtException", { error: err?.message, stack: err?.stack });
+ process.exit(1);
+});
+process.on("unhandledRejection", (err) => {
+ log("fatal", "unhandledRejection", { error: err?.message, stack: err?.stack });
+ process.exit(1);
+});
+
+openLog();