Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(onnx): unload the idle speaker session so the ONNX worker can exit (#2371)

Chad Pihaacc806b240ff
diff --git a/CLAUDE.md b/CLAUDE.md
index d603ff63a..51898a6f8 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -33,7 +33,7 @@ OpenWhispr is an Electron-based desktop dictation application that uses whisper.
- Main Process: Electron main, IPC handlers, database operations
- Renderer Process: React app with context isolation
- Preload Script: Secure bridge between processes
- - ONNX Utility Process: hosts all `onnxruntime-node` inference (text embeddings, speaker embeddings, fbank). Lazy-spawned on first use via `src/helpers/onnxWorkerClient.js` → `src/workers/onnxWorker.js`. Native crashes (e.g., ORT `bad_alloc`) confine to the worker; main process rejects in-flight requests and respawns with backoff. A request that times out kills the worker so it respawns the same way; the embedding clients reload their sessions on the new worker. Exits once idle with no session loaded (`releaseIfIdle`, after semantic search releases its model). Stopped in `will-quit`.
+ - ONNX Utility Process: hosts all `onnxruntime-node` inference (text embeddings, speaker embeddings, fbank). Lazy-spawned on first use via `src/helpers/onnxWorkerClient.js` → `src/workers/onnxWorker.js`. Native crashes (e.g., ORT `bad_alloc`) confine to the worker; main process rejects in-flight requests, waits out a backoff, and respawns the worker on the next request (never eagerly, so a crash can't leave an empty worker running). A request that times out kills the worker so it respawns the same way; the embedding clients reload their sessions on the new worker. Exits once idle with no session loaded (`releaseIfIdle`, called after semantic search releases its model and after `speakerEmbeddings.js` unloads the speaker model 5 minutes after its last extract). Each session's calls are serialized in the worker, so an unload never releases a session mid-inference. Stopped in `will-quit`.
3. **Audio Pipeline**:
- MediaRecorder API → Blob → ArrayBuffer → IPC → File → whisper.cpp
diff --git a/src/helpers/localEmbeddings.js b/src/helpers/localEmbeddings.js
index fd8857354..bf547280a 100644
--- a/src/helpers/localEmbeddings.js
+++ b/src/helpers/localEmbeddings.js
@@ -105,8 +105,8 @@ class LocalEmbeddings {
} finally {
this.loadPromise = null;
this.loadedGeneration = null;
+ await onnxWorkerClient.releaseIfIdle();
}
- await onnxWorkerClient.releaseIfIdle();
});
}
diff --git a/src/helpers/onnxWorkerClient.js b/src/helpers/onnxWorkerClient.js
index a68e59491..a92a94472 100644
--- a/src/helpers/onnxWorkerClient.js
+++ b/src/helpers/onnxWorkerClient.js
@@ -36,7 +36,7 @@ class OnnxWorkerClient {
this.shuttingDown = false;
this.gaveUp = false;
this.spawnPromise = null;
- this.respawnTimer = null;
+ this.backoffTimer = null;
this.generation = 0;
this.killedForTimeout = false;
}
@@ -167,20 +167,19 @@ class OnnxWorkerClient {
}
const delay =
RESPAWN_BACKOFF_MS[Math.min(this.crashCount - 1, RESPAWN_BACKOFF_MS.length - 1)];
- debugLogger.info("onnx worker respawn scheduled", {
+ debugLogger.info("onnx worker respawn backoff", {
delayMs: delay,
crashCount: this.crashCount,
});
- this.respawnTimer = setTimeout(() => {
- this.respawnTimer = null;
- this._spawn().catch((spawnErr) => {
- debugLogger.error("onnx worker respawn failed", { error: spawnErr?.message });
- });
+ // The next request respawns the worker; spawning here would leave an empty worker
+ // running (e.g. after a crash during an unload) that nothing ever releases.
+ this.backoffTimer = setTimeout(() => {
+ this.backoffTimer = null;
}, delay);
}
}
- // Exits the worker once no session is loaded, so an idle text unload also frees
+ // Exits the worker once no session is loaded, so an idle unload also frees
// the onnxruntime arena. Detaches before the kill so a racing request spawns fresh.
async releaseIfIdle() {
if (!this.child || this.shuttingDown || this.pending.size) return false;
@@ -205,9 +204,9 @@ class OnnxWorkerClient {
}
async request(method, payload, transferList) {
- // Releasing text must never start a worker just to free an absent session; a
- // worker that is shutting down takes its session with it.
- if (method === "text.unload" && (!this.child || this.shuttingDown)) return { ok: true };
+ // An unload must never start a worker just to free an absent session; a
+ // worker that is shutting down takes its sessions with it.
+ if (method.endsWith(".unload") && (!this.child || this.shuttingDown)) return { ok: true };
if (this.shuttingDown) {
throw new WorkerCrashedError("worker shutting down");
}
@@ -216,8 +215,8 @@ class OnnxWorkerClient {
throw new WorkerCrashedError("worker unavailable");
}
- if (this.respawnTimer) {
- throw new WorkerCrashedError("worker restarting");
+ if (this.backoffTimer) {
+ throw new WorkerCrashedError("worker in crash backoff");
}
if (this.pending.size >= MAX_PENDING_REQUESTS) {
@@ -237,7 +236,7 @@ class OnnxWorkerClient {
const timeout = setTimeout(() => {
if (!this.pending.delete(id)) return;
reject(new Error(`onnx worker request timeout: ${method}`));
- // The worker serializes text work, so one hung request wedges every later one.
+ // The worker serializes each session's calls, so one hung request blocks every later call on that session.
debugLogger.warn("onnx worker request timeout; killing worker", { method });
this.killedForTimeout = true;
try {
@@ -259,9 +258,9 @@ class OnnxWorkerClient {
async stop() {
this.shuttingDown = true;
- if (this.respawnTimer) {
- clearTimeout(this.respawnTimer);
- this.respawnTimer = null;
+ if (this.backoffTimer) {
+ clearTimeout(this.backoffTimer);
+ this.backoffTimer = null;
}
if (!this.child) return;
diff --git a/src/helpers/speakerEmbeddings.js b/src/helpers/speakerEmbeddings.js
index e402b066f..736bed0dd 100644
--- a/src/helpers/speakerEmbeddings.js
+++ b/src/helpers/speakerEmbeddings.js
@@ -11,11 +11,17 @@ const MIN_SEGMENT_SAMPLES = SAMPLE_RATE * MIN_SEGMENT_SECONDS;
const MAX_EMBEDDING_SECONDS = 8;
const MAX_EMBEDDING_SAMPLES = SAMPLE_RATE * MAX_EMBEDDING_SECONDS;
const MODEL_FILE = "3dspeaker_speech_campplus_sv_en_voxceleb_16k.onnx";
+// Live meetings extract during remote speech, so this normally fires once a meeting and its
+// post-meeting diarization are done (a longer silence mid-meeting costs one reload). The
+// unload lets the idle ONNX worker exit.
+const IDLE_TIMEOUT_MS = 5 * 60 * 1000;
class SpeakerEmbeddings {
constructor() {
this.loadPromise = null;
this.loadedGeneration = null;
+ this.operationQueue = Promise.resolve();
+ this.idleTimer = null;
}
getModelPath() {
@@ -55,21 +61,63 @@ class SpeakerEmbeddings {
return this.loadPromise;
}
- async _extractEmbeddingFromSamples(samples) {
- await this._ensureLoaded();
+ _enqueue(operation) {
+ const result = this.operationQueue.then(operation);
+ this.operationQueue = result.catch(() => {});
+ return result;
+ }
+
+ _clearIdleTimer() {
+ if (this.idleTimer !== null) clearTimeout(this.idleTimer);
+ this.idleTimer = null;
+ }
- const samplesBuffer = samples.buffer.slice(
- samples.byteOffset,
- samples.byteOffset + samples.byteLength
- );
+ _armIdleTimer() {
+ this._clearIdleTimer();
+ this.idleTimer = setTimeout(() => {
+ this.idleTimer = null;
+ this.unload().catch((err) =>
+ debugLogger.warn("speaker-embeddings idle unload failed", { error: err?.message })
+ );
+ }, IDLE_TIMEOUT_MS);
+ this.idleTimer.unref();
+ }
- // No transfer-list: MessagePortMain can't transfer ArrayBuffers, so the samples are cloned.
- const { embeddingBuffer } = await onnxWorkerClient.request("speaker.extract", {
- samplesBuffer,
+ _extractEmbeddingFromSamples(samples) {
+ // A timer armed by an earlier extract must not queue an unload behind this one.
+ this._clearIdleTimer();
+ return this._enqueue(async () => {
+ try {
+ await this._ensureLoaded();
+
+ const samplesBuffer = samples.buffer.slice(
+ samples.byteOffset,
+ samples.byteOffset + samples.byteLength
+ );
+
+ // No transfer-list: MessagePortMain can't transfer ArrayBuffers, so the samples are cloned.
+ const { embeddingBuffer } = await onnxWorkerClient.request("speaker.extract", {
+ samplesBuffer,
+ });
+
+ if (!embeddingBuffer) return null;
+ return new Float32Array(embeddingBuffer);
+ } finally {
+ this._armIdleTimer();
+ }
});
+ }
- if (!embeddingBuffer) return null;
- return new Float32Array(embeddingBuffer);
+ unload() {
+ return this._enqueue(async () => {
+ try {
+ await onnxWorkerClient.request("speaker.unload", {});
+ } finally {
+ this.loadPromise = null;
+ this.loadedGeneration = null;
+ await onnxWorkerClient.releaseIfIdle();
+ }
+ });
}
async extractEmbeddingFromSamples(samples) {
diff --git a/src/workers/onnxWorker.js b/src/workers/onnxWorker.js
index 832e1b174..e6b4af462 100644
--- a/src/workers/onnxWorker.js
+++ b/src/workers/onnxWorker.js
@@ -38,7 +38,8 @@ let speakerSession = null;
let speakerInputName = null;
let textSession = null;
let textTokenizer = null;
-let textQueue = Promise.resolve();
+// One queue per session, keyed by method prefix ("text.", "speaker.").
+const sessionQueues = { text: Promise.resolve(), speaker: Promise.resolve() };
function log(level, message, extra) {
if (!logStream) return;
@@ -222,6 +223,17 @@ async function speakerExtract({ samplesBuffer }) {
return { embeddingBuffer: data.buffer };
}
+// Unloads clear the session before releasing it, so a release that throws can't leave ping
+// reporting a loaded session and keep releaseIfIdle from ever exiting the worker.
+async function speakerUnload() {
+ const session = speakerSession;
+ speakerSession = null;
+ speakerInputName = null;
+ if (session) await session.release();
+ log("info", "speaker session unloaded");
+ return { ok: true };
+}
+
function buildTextTokenizer(tokenizerData) {
const tokenToId = new Map();
for (const [token, id] of Object.entries(tokenizerData.model.vocab)) {
@@ -341,9 +353,10 @@ async function textEmbed({ text }) {
}
async function textUnload() {
- if (textSession) await textSession.release();
+ const session = textSession;
textSession = null;
textTokenizer = null;
+ if (session) await session.release();
log("info", "text session unloaded");
return { ok: true };
}
@@ -352,6 +365,7 @@ const handlers = {
ping: () => ({ ok: true, sessions: { speaker: !!speakerSession, text: !!textSession } }),
"speaker.load": speakerLoad,
"speaker.extract": speakerExtract,
+ "speaker.unload": speakerUnload,
"text.load": textLoad,
"text.embed": textEmbed,
"text.unload": textUnload,
@@ -369,10 +383,11 @@ async function dispatch({ id, method, payload }) {
}
try {
let result;
- if (method.startsWith("text.")) {
+ const queueKey = method.split(".")[0];
+ if (Object.hasOwn(sessionQueues, queueKey)) {
// Message callbacks overlap; never release a session during native inference.
- const operation = textQueue.then(() => handler(payload || {}));
- textQueue = operation.catch(() => {});
+ const operation = sessionQueues[queueKey].then(() => handler(payload || {}));
+ sessionQueues[queueKey] = operation.catch(() => {});
result = await operation;
} else {
result = await handler(payload || {});
diff --git a/test/helpers/localEmbeddingsLifecycle.test.js b/test/helpers/localEmbeddingsLifecycle.test.js
index 4cbe6b2c1..ec1f1c18e 100644
--- a/test/helpers/localEmbeddingsLifecycle.test.js
+++ b/test/helpers/localEmbeddingsLifecycle.test.js
@@ -8,13 +8,16 @@ const vm = require("node:vm");
const WORKER_SOURCE = fs.readFileSync(path.resolve("src/workers/onnxWorker.js"), "utf8");
const CLIENT_SOURCE = fs.readFileSync(path.resolve("src/helpers/onnxWorkerClient.js"), "utf8");
const EMBEDDINGS_SOURCE = fs.readFileSync(path.resolve("src/helpers/localEmbeddings.js"), "utf8");
+const SPEAKER_SOURCE = fs.readFileSync(path.resolve("src/helpers/speakerEmbeddings.js"), "utf8");
+const SPEAKER_SAMPLES = new Float32Array(16000 * 2).fill(0.1);
function fakeOrt(events, nativeSession) {
return {
InferenceSession: {
create: async (file) => {
- events.push(`${file === "speaker" ? "speaker" : "text"}.create`);
- return nativeSession(file === "speaker" ? "speaker" : "text");
+ const name = path.basename(file) === "model.onnx" ? "text" : "speaker";
+ events.push(`${name}.create`);
+ return nativeSession(name);
},
},
Tensor: class {},
@@ -37,6 +40,30 @@ function loadEmbeddings(client) {
return localContext.module.exports;
}
+function loadSpeakerEmbeddings(client, idleTimers) {
+ const context = vm.createContext({
+ module: { exports: {} },
+ process: {},
+ setTimeout(callback) {
+ const timer = { callback, cleared: false, unref() {} };
+ idleTimers.push(timer);
+ return timer;
+ },
+ clearTimeout(timer) {
+ if (timer) timer.cleared = true;
+ },
+ require(name) {
+ if (name === "fs") return { existsSync: () => true };
+ if (name === "./debugLogger") return { debug() {}, warn() {} };
+ if (name === "./modelDirUtils") return { getModelsDirForService: () => "/models" };
+ if (name === "./onnxWorkerClient") return client;
+ return require(name);
+ },
+ });
+ vm.runInContext(SPEAKER_SOURCE, context);
+ return context.module.exports;
+}
+
function createHarness() {
const events = [];
let releaseInference;
@@ -171,38 +198,6 @@ test("reloads after the shared worker restarts", async () => {
assert.equal(events.filter((event) => event === "text.load").length, 2);
});
-test("speaker embeddings reload after the shared worker restarts", async () => {
- const methods = [];
- const client = {
- generation: 0,
- async request(method) {
- methods.push(method);
- return method === "speaker.extract" ? { embeddingBuffer: new ArrayBuffer(4) } : { ok: true };
- },
- };
- const context = vm.createContext({
- module: { exports: {} },
- process: {},
- require(name) {
- if (name === "fs") return { existsSync: () => true };
- if (name === "./debugLogger") return { debug() {} };
- if (name === "./modelDirUtils") return { getModelsDirForService: () => "/models" };
- if (name === "./onnxWorkerClient") return client;
- return require(name);
- },
- });
- vm.runInContext(
- fs.readFileSync(path.resolve("src/helpers/speakerEmbeddings.js"), "utf8"),
- context
- );
- const speakerEmbeddings = context.module.exports;
- const samples = new Float32Array(16000 * 2);
- await speakerEmbeddings.extractEmbeddingFromSamples(samples);
- client.generation += 1;
- await speakerEmbeddings.extractEmbeddingFromSamples(samples);
- assert.equal(methods.filter((method) => method === "speaker.load").length, 2);
-});
-
test("a failed load does not block unloading or the next load attempt", async () => {
const { embeddings, client, events } = createHarness();
const request = client.request.bind(client);
@@ -264,9 +259,10 @@ class FakePort extends EventEmitter {
}
// The real client driving the real worker source: each fork boots a fresh worker context.
-function createIntegratedHarness() {
+function createIntegratedHarness({ failRelease = null } = {}) {
const events = [];
const workers = [];
+ const idleTimers = [];
const nativeSession = (name) => ({
inputNames: ["input"],
async run() {
@@ -274,6 +270,7 @@ function createIntegratedHarness() {
},
async release() {
events.push(`${name}.release`);
+ if (name === failRelease) throw new Error("release failed");
},
});
const fork = () => {
@@ -339,7 +336,17 @@ function createIntegratedHarness() {
});
vm.runInContext(CLIENT_SOURCE, clientContext);
const client = clientContext.module.exports;
- return { client, embeddings: loadEmbeddings(client), events, workers };
+ return {
+ client,
+ embeddings: loadEmbeddings(client),
+ speaker: loadSpeakerEmbeddings(client, idleTimers),
+ events,
+ workers,
+ async fireSpeakerIdleUnload() {
+ idleTimers.findLast((timer) => !timer.cleared).callback();
+ for (let i = 0; i < 10; i++) await nextTurn();
+ },
+ };
}
test("unloading the last session exits the worker and a later embedding respawns and reloads it", async () => {
@@ -368,3 +375,42 @@ test("unloading text keeps the worker alive while diarization holds a speaker se
assert.equal(sessions.speaker, true);
assert.equal(sessions.text, false);
});
+
+test("the speaker idle unload exits the worker once the text session is gone", async () => {
+ const h = createIntegratedHarness();
+ await h.speaker.extractEmbeddingFromSamples(SPEAKER_SAMPLES);
+ await h.embeddings.embedText("");
+ await h.embeddings.unload();
+ assert.equal(h.workers[0].child.killed, false);
+ await h.fireSpeakerIdleUnload();
+ assert.equal(h.workers[0].child.killed, true);
+ assert.deepEqual(h.events, ["speaker.create", "text.create", "text.release", "speaker.release"]);
+ assert.ok(await h.speaker.extractEmbeddingFromSamples(SPEAKER_SAMPLES));
+ assert.equal(h.workers.length, 2);
+});
+
+test("the speaker idle unload keeps the worker for a loaded text session", async () => {
+ const h = createIntegratedHarness();
+ await h.embeddings.embedText("");
+ await h.speaker.extractEmbeddingFromSamples(SPEAKER_SAMPLES);
+ await h.fireSpeakerIdleUnload();
+ assert.equal(h.workers[0].child.killed, false);
+ await h.embeddings.unload();
+ assert.equal(h.workers[0].child.killed, true);
+ assert.equal(h.workers.length, 1);
+});
+
+for (const session of ["speaker", "text"]) {
+ test(`a ${session} release that throws still lets the worker exit`, async () => {
+ const h = createIntegratedHarness({ failRelease: session });
+ if (session === "speaker") {
+ await h.speaker.extractEmbeddingFromSamples(SPEAKER_SAMPLES);
+ await assert.rejects(h.speaker.unload(), /release failed/);
+ } else {
+ await h.embeddings.embedText("");
+ await assert.rejects(h.embeddings.unload(), /release failed/);
+ }
+ assert.equal(h.workers[0].child.killed, true);
+ assert.equal(h.client.child, null);
+ });
+}
diff --git a/test/helpers/onnxWorkerClientLifecycle.test.js b/test/helpers/onnxWorkerClientLifecycle.test.js
index 0705ca975..992317d18 100644
--- a/test/helpers/onnxWorkerClientLifecycle.test.js
+++ b/test/helpers/onnxWorkerClientLifecycle.test.js
@@ -133,23 +133,43 @@ function createHarness({ killExitCode = 0 } = {}) {
};
}
-test("unloading unused text does not spawn a worker", async () => {
- const h = createHarness();
- await h.client.request("text.unload", {});
- assert.equal(h.client.child, null);
- assert.equal(h.forks.length, 0);
-});
+for (const method of ["text.unload", "speaker.unload"]) {
+ test(`${method} without a loaded session does not spawn a worker`, async () => {
+ const h = createHarness();
+ await h.client.request(method, {});
+ assert.equal(h.client.child, null);
+ assert.equal(h.forks.length, 0);
+ });
-test("unloading after worker exit succeeds without restarting it", async () => {
- const h = createHarness();
- await h.client.request("ping", {});
- const generation = h.client.generation;
- h.forks[0].emit("exit", 0);
- assert.equal(h.client.generation, generation + 1);
- await h.client.request("text.unload", {});
- assert.equal(h.client.child, null);
- assert.equal(h.forks.length, 1);
-});
+ test(`${method} after worker exit succeeds without restarting it`, async () => {
+ const h = createHarness();
+ await h.client.request("ping", {});
+ const generation = h.client.generation;
+ h.forks[0].emit("exit", 0);
+ assert.equal(h.client.generation, generation + 1);
+ await h.client.request(method, {});
+ assert.equal(h.client.child, null);
+ assert.equal(h.forks.length, 1);
+ });
+
+ test(`${method} resolves while the worker is shutting down`, async () => {
+ const h = createHarness();
+ await h.client.request("ping", {});
+ const stopping = h.client.stop();
+ assert.equal((await h.client.request(method, {})).ok, true);
+ h.forks[0].emit("exit", 0);
+ await stopping;
+ });
+
+ test(`${method} resolves during the crash backoff`, async () => {
+ const h = createHarness();
+ await h.client.request("ping", {});
+ h.forks[0].emit("exit", 1);
+ assert.ok(h.client.backoffTimer);
+ assert.equal((await h.client.request(method, {})).ok, true);
+ assert.equal(h.forks.length, 1);
+ });
+}
test("releasing an idle worker kills it without counting a crash, and the next request respawns it", async () => {
const h = createHarness({ killExitCode: 15 });
@@ -162,7 +182,7 @@ test("releasing an idle worker kills it without counting a crash, and the next r
assert.equal(h.client.child, null);
assert.equal(h.client.generation, generation + 1);
assert.equal(h.client.crashCount, 0);
- assert.equal(h.client.respawnTimer, null);
+ assert.equal(h.client.backoffTimer, null);
assert.equal(h.client.gaveUp, false);
assert.equal(h.client.shuttingDown, false);
assert.ok(h.logs.some((entry) => entry.level === "info" && /released/.test(entry.message)));
@@ -206,25 +226,7 @@ test("releasing does nothing without a worker", async () => {
assert.equal(h.forks.length, 0);
});
-test("text unload resolves while the worker is shutting down", async () => {
- const h = createHarness();
- await h.client.request("ping", {});
- const stopping = h.client.stop();
- assert.equal((await h.client.request("text.unload", {})).ok, true);
- h.forks[0].emit("exit", 0);
- await stopping;
-});
-
-test("text unload resolves while a crash respawn is pending", async () => {
- const h = createHarness();
- await h.client.request("ping", {});
- h.forks[0].emit("exit", 1);
- assert.ok(h.client.respawnTimer);
- assert.equal((await h.client.request("text.unload", {})).ok, true);
- assert.equal(h.forks.length, 1);
-});
-
-test("a request timeout kills the worker so the crash path respawns it", async () => {
+test("a request timeout kills the worker and starts the crash backoff", async () => {
const h = createHarness({ killExitCode: 0 });
h.hang("text.embed");
await h.client.request("ping", {});
@@ -237,5 +239,19 @@ test("a request timeout kills the worker so the crash path respawns it", async (
await nextTurn();
assert.equal(h.client.child, null);
assert.equal(h.client.crashCount, 1);
- assert.ok(h.client.respawnTimer);
+ assert.ok(h.client.backoffTimer);
+});
+
+test("a crash respawns the worker on the next request after the backoff, never eagerly", async () => {
+ const h = createHarness();
+ await h.client.request("ping", {});
+ h.forks[0].emit("exit", 1);
+ await assert.rejects(h.client.request("ping", {}), /crash backoff/);
+ h.timers.fire(1000);
+ await nextTurn();
+ assert.equal(h.client.backoffTimer, null);
+ assert.equal(h.client.child, null);
+ assert.equal(h.forks.length, 1);
+ await h.client.request("ping", {});
+ assert.equal(h.forks.length, 2);
});
diff --git a/test/helpers/speakerEmbeddingsLifecycle.test.js b/test/helpers/speakerEmbeddingsLifecycle.test.js
new file mode 100644
index 000000000..513ce1246
--- /dev/null
+++ b/test/helpers/speakerEmbeddingsLifecycle.test.js
@@ -0,0 +1,243 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const vm = require("node:vm");
+
+const WORKER_SOURCE = fs.readFileSync(path.resolve("src/workers/onnxWorker.js"), "utf8");
+const SPEAKER_SOURCE = fs.readFileSync(path.resolve("src/helpers/speakerEmbeddings.js"), "utf8");
+
+const SAMPLES = new Float32Array(16000 * 2).fill(0.1);
+
+function createTimers() {
+ const timers = [];
+ return {
+ timers,
+ active: () => timers.filter((timer) => !timer.cleared && !timer.fired),
+ setTimeout(callback) {
+ const timer = { callback, cleared: false, fired: false, unref() {} };
+ timers.push(timer);
+ return timer;
+ },
+ clearTimeout(timer) {
+ if (timer) timer.cleared = true;
+ },
+ fire(timer) {
+ timer.fired = true;
+ timer.callback();
+ },
+ };
+}
+
+function createHarness() {
+ const events = [];
+ const logs = [];
+ const timers = createTimers();
+ let releaseInference;
+ let gateInference = false;
+ const nativeSession = (name) => ({
+ inputNames: ["input"],
+ async run() {
+ events.push(`${name}.run`);
+ if (gateInference) {
+ await new Promise((resolve) => {
+ releaseInference = resolve;
+ });
+ }
+ events.push(`${name}.done`);
+ return { output: { data: new Float32Array(512).fill(1) } };
+ },
+ async release() {
+ events.push(`${name}.release`);
+ },
+ });
+ const workerContext = vm.createContext({
+ require(name) {
+ if (name === "onnxruntime-node")
+ return {
+ InferenceSession: { create: async () => nativeSession("speaker") },
+ Tensor: class {},
+ };
+ return require(name);
+ },
+ process: { env: {}, on() {}, parentPort: { once() {} } },
+ setImmediate,
+ });
+ vm.runInContext(WORKER_SOURCE, workerContext);
+ const dispatch = vm.runInContext("dispatch", workerContext);
+ const client = {
+ generation: 0,
+ failNext: null,
+ async request(method, payload) {
+ events.push(method);
+ if (this.failNext === method) {
+ this.failNext = null;
+ throw new Error("worker unavailable");
+ }
+ const { reply } = await dispatch({ id: 1, method, payload });
+ if (reply.error) throw new Error(reply.error.message);
+ return reply.result;
+ },
+ async releaseIfIdle() {
+ events.push("releaseIfIdle");
+ return true;
+ },
+ };
+ const context = vm.createContext({
+ module: { exports: {} },
+ process: {},
+ setTimeout: timers.setTimeout,
+ clearTimeout: timers.clearTimeout,
+ require(name) {
+ if (name === "fs") return { existsSync: () => true };
+ if (name === "./debugLogger")
+ return Object.fromEntries(
+ ["debug", "warn"].map((level) => [
+ level,
+ (message, meta) => logs.push({ level, message, meta }),
+ ])
+ );
+ if (name === "./modelDirUtils") return { getModelsDirForService: () => "/models" };
+ if (name === "./onnxWorkerClient") return client;
+ return require(name);
+ },
+ });
+ vm.runInContext(SPEAKER_SOURCE, context);
+ return {
+ speaker: context.module.exports,
+ client,
+ events,
+ logs,
+ timers,
+ gateInference() {
+ gateInference = true;
+ },
+ releaseInference() {
+ gateInference = false;
+ releaseInference();
+ },
+ };
+}
+
+async function flush() {
+ for (let i = 0; i < 10; i++) await new Promise((resolve) => setImmediate(resolve));
+}
+
+test("an extract arms an idle unload that releases the speaker session, then the worker", async () => {
+ const h = createHarness();
+ assert.ok(await h.speaker.extractEmbeddingFromSamples(SAMPLES));
+ const [timer] = h.timers.active();
+ h.timers.fire(timer);
+ await flush();
+ const { sessions } = await h.client.request("ping", {});
+ assert.equal(sessions.speaker, false);
+ assert.ok(h.events.indexOf("speaker.release") < h.events.indexOf("releaseIfIdle"));
+});
+
+test("each extract restarts the idle window", async () => {
+ const h = createHarness();
+ await h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ await h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ assert.equal(h.timers.active().length, 1);
+});
+
+test("extracts queued together leave a single idle timer", async () => {
+ const h = createHarness();
+ h.gateInference();
+ const first = h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ const second = h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ await flush();
+ h.releaseInference();
+ await Promise.all([first, second]);
+ assert.equal(h.timers.active().length, 1);
+});
+
+test("a failed extract still arms the idle unload", async () => {
+ const h = createHarness();
+ h.client.failNext = "speaker.extract";
+ await assert.rejects(h.speaker.extractEmbeddingFromSamples(SAMPLES), /worker unavailable/);
+ assert.equal(h.timers.active().length, 1);
+});
+
+test("an extract in flight holds off the idle unload until it finishes", async () => {
+ const h = createHarness();
+ await h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ h.gateInference();
+ const second = h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ assert.equal(h.timers.active().length, 0);
+ await flush();
+ h.releaseInference();
+ await second;
+ assert.equal(h.timers.active().length, 1);
+ assert.ok(!h.events.includes("speaker.unload"));
+});
+
+test("a segment too short to embed neither loads the model nor arms the timer", async () => {
+ const h = createHarness();
+ assert.equal(await h.speaker.extractEmbeddingFromSamples(new Float32Array(1000)), null);
+ assert.deepEqual(h.events, []);
+ assert.equal(h.timers.timers.length, 0);
+});
+
+test("unload waits for an in-flight extract and a racing extract reloads after release", async () => {
+ const h = createHarness();
+ h.gateInference();
+ const first = h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ await flush();
+ const unloading = h.speaker.unload();
+ const second = h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ await flush();
+ assert.ok(!h.events.includes("speaker.release"));
+ h.releaseInference();
+ await Promise.all([first, unloading, second]);
+ assert.deepEqual(
+ h.events.filter((event) => event.startsWith("speaker")),
+ [
+ "speaker.load",
+ "speaker.extract",
+ "speaker.run",
+ "speaker.done",
+ "speaker.unload",
+ "speaker.release",
+ "speaker.load",
+ "speaker.extract",
+ "speaker.run",
+ "speaker.done",
+ ]
+ );
+});
+
+test("worker serializes speaker unload behind in-flight native inference", async () => {
+ const h = createHarness();
+ await h.client.request("speaker.load", { modelPath: "speaker" });
+ h.gateInference();
+ const extracting = h.client.request("speaker.extract", { samplesBuffer: SAMPLES.buffer });
+ await flush();
+ const unloading = h.client.request("speaker.unload", {});
+ await flush();
+ assert.ok(!h.events.includes("speaker.release"));
+ h.releaseInference();
+ await Promise.all([extracting, unloading]);
+ assert.ok(h.events.indexOf("speaker.done") < h.events.indexOf("speaker.release"));
+});
+
+test("reloads after the shared worker restarts", async () => {
+ const h = createHarness();
+ await h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ h.client.generation += 1;
+ await h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ assert.equal(h.events.filter((event) => event === "speaker.load").length, 2);
+});
+
+test("a failed idle unload is logged and the next extract still reloads", async () => {
+ const h = createHarness();
+ await h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ h.client.failNext = "speaker.unload";
+ h.timers.fire(h.timers.active()[0]);
+ await flush();
+ assert.ok(
+ h.logs.some((entry) => entry.level === "warn" && /idle unload failed/.test(entry.message))
+ );
+ await h.speaker.extractEmbeddingFromSamples(SAMPLES);
+ assert.equal(h.events.filter((event) => event === "speaker.load").length, 2);
+});