Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

fix(whisper): fall back to CPU when the CUDA server crashes mid-request (#1192)

Alcahest Malus Dienheimd5f840a08f23
diff --git a/src/App.jsx b/src/App.jsx
index 4b31bfc2b..a75345e1f 100644
--- a/src/App.jsx
+++ b/src/App.jsx
@@ -124,6 +124,18 @@ export default function App() {
});
});
+ const showGpuFallbackToast = () => {
+ toast({
+ title: t("app.toasts.gpuFallback.title"),
+ description: t("app.toasts.gpuFallback.description"),
+ duration: 10000,
+ });
+ };
+ const unsubscribeCudaFallback =
+ window.electronAPI?.onCudaFallbackNotification?.(showGpuFallbackToast);
+ const unsubscribeGpuFallback =
+ window.electronAPI?.onGpuFallbackNotification?.(showGpuFallbackToast);
+
const unsubscribeCorrections = window.electronAPI?.onCorrectionsLearned?.((words) => {
if (words && words.length > 0) {
const wordList = words.map((w) => `\u201c${w}\u201d`).join(", ");
@@ -160,6 +172,8 @@ export default function App() {
return () => {
unsubscribeFallback?.();
unsubscribeFailed?.();
+ unsubscribeCudaFallback?.();
+ unsubscribeGpuFallback?.();
unsubscribeCorrections?.();
};
}, [toast, dismiss, t]);
diff --git a/src/helpers/whisperServer.js b/src/helpers/whisperServer.js
index d83001438..caa67c354 100644
--- a/src/helpers/whisperServer.js
+++ b/src/helpers/whisperServer.js
@@ -20,6 +20,8 @@ const STARTUP_TIMEOUT_MS = 30000;
const VULKAN_STARTUP_TIMEOUT_MS = 120000;
const HEALTH_CHECK_INTERVAL_MS = 5000;
const HEALTH_CHECK_TIMEOUT_MS = 2000;
+const PROCESS_EXIT_WAIT_MS = 2000;
+const PROCESS_EXIT_POLL_INTERVAL_MS = 50;
const DEFAULT_WHISPER_THREADS = 4;
const MAX_AUTO_WHISPER_THREADS = 12;
const MAX_MANUAL_WHISPER_THREADS = 64;
@@ -161,6 +163,39 @@ function buildWhisperServerArgs({
return args;
}
+function shouldFallbackToCpuAfterRequestError({
+ isConnectionError,
+ useGpu,
+ isRemote,
+ stopRequested,
+ generationChanged,
+ processExited,
+}) {
+ // A local GPU whisper-server that drops the connection and dies mid-request crashed
+ // (e.g. CUDA aborting on an unsupported GPU at the first kernel launch): retry on CPU.
+ // Skip remote/CPU servers, intentional stops, and restarts.
+ return (
+ !!isConnectionError &&
+ !!useGpu &&
+ !isRemote &&
+ !stopRequested &&
+ !generationChanged &&
+ !!processExited
+ );
+}
+
+function shouldRetryAfterServerReplaced({
+ isConnectionError,
+ isRemote,
+ stopRequested,
+ ready,
+ sameModel,
+}) {
+ // A concurrent caller already restarted the server; retry only if it is up
+ // and still serving the same model (a model switch must not answer for it).
+ return !!isConnectionError && !isRemote && !stopRequested && !!ready && !!sameModel;
+}
+
class WhisperServerManager extends EventEmitter {
constructor() {
super();
@@ -177,7 +212,8 @@ class WhisperServerManager extends EventEmitter {
this.canConvert = false;
this.useCuda = false;
this.useVulkan = false;
- this._abortStartup = false;
+ this.startGeneration = 0;
+ this._stopRequested = false;
this.vadSignature = "vad:off";
this.threadSignature = "threads:default";
this.lastStartOptions = {};
@@ -412,6 +448,8 @@ class WhisperServerManager extends EventEmitter {
}
async _doStart(modelPath, options = {}) {
+ this.startGeneration += 1;
+ this._stopRequested = false;
const usingCuda = options.useCuda || false;
const usingVulkan = !usingCuda && (options.useVulkan || false);
const threadResolution = options.threadResolution || resolveWhisperThreads(options);
@@ -425,7 +463,6 @@ class WhisperServerManager extends EventEmitter {
this.modelPath = modelPath;
this.useCuda = usingCuda;
this.useVulkan = usingVulkan;
- this._abortStartup = false;
// Check for FFmpeg first - only use --convert flag if FFmpeg is available
const ffmpegPath = this.getFFmpegPath();
@@ -589,7 +626,7 @@ class WhisperServerManager extends EventEmitter {
const STARTUP_POLL_INTERVAL_MS = 100;
while (Date.now() - startTime < timeoutMs) {
- if (this._abortStartup) {
+ if (this._stopRequested) {
throw Object.assign(new Error("whisper-server startup interrupted by stop"), {
isStopped: true,
});
@@ -730,6 +767,17 @@ class WhisperServerManager extends EventEmitter {
const bodyParts = parts.map((part) => (typeof part === "string" ? Buffer.from(part) : part));
const body = Buffer.concat(bodyParts);
+ const generation = this.startGeneration;
+ const modelPath = this.modelPath;
+
+ try {
+ return await this._postInference(body, boundary);
+ } catch (err) {
+ return await this._retryAfterRequestFailure(err, body, boundary, generation, modelPath);
+ }
+ }
+
+ _postInference(body, boundary) {
return new Promise((resolve, reject) => {
const startTime = Date.now();
@@ -773,7 +821,10 @@ class WhisperServerManager extends EventEmitter {
);
req.on("error", (error) => {
- reject(new Error(`whisper-server request failed: ${error.message}`));
+ const err = new Error(`whisper-server request failed: ${error.message}`);
+ err.isConnectionError = true;
+ err.code = error.code;
+ reject(err);
});
req.on("timeout", () => {
req.destroy();
@@ -785,6 +836,85 @@ class WhisperServerManager extends EventEmitter {
});
}
+ async _retryAfterRequestFailure(err, body, boundary, generation, modelPath) {
+ if (!err?.isConnectionError || this.isRemote || this._stopRequested) throw err;
+
+ if (this.startGeneration === generation) {
+ if (!this.useCuda && !this.useVulkan) throw err;
+
+ // The child's close handler clears this.process; wait for it so a crash is told
+ // apart from a server that merely refused this one request.
+ const processExited = await this._waitForProcessExit(PROCESS_EXIT_WAIT_MS);
+ if (
+ this.startGeneration === generation &&
+ shouldFallbackToCpuAfterRequestError({
+ isConnectionError: true,
+ useGpu: this.useCuda || this.useVulkan,
+ isRemote: this.isRemote,
+ stopRequested: this._stopRequested,
+ generationChanged: false,
+ processExited,
+ })
+ ) {
+ return await this._fallbackToCpuAndRetry(body, boundary, modelPath);
+ }
+ if (this.startGeneration === generation) throw err;
+ }
+
+ // Another start already replaced the crashed server (concurrent fallback or
+ // model reload): retry only against a ready server holding the same model.
+ const pending = this.startupPromise;
+ if (pending) await pending.catch(() => {});
+ if (
+ !shouldRetryAfterServerReplaced({
+ isConnectionError: true,
+ isRemote: this.isRemote,
+ stopRequested: this._stopRequested,
+ ready: this.ready,
+ sameModel: this.modelPath === modelPath,
+ })
+ ) {
+ throw err;
+ }
+ try {
+ return await this._postInference(body, boundary);
+ } catch (retryErr) {
+ // The replacement can be another doomed GPU server (a peer restarted with the
+ // GPU flags still set): give it the same one-shot crash check before giving up.
+ if (
+ !retryErr?.isConnectionError ||
+ this._stopRequested ||
+ (!this.useCuda && !this.useVulkan)
+ ) {
+ throw retryErr;
+ }
+ const exited = await this._waitForProcessExit(PROCESS_EXIT_WAIT_MS);
+ if (!exited || this._stopRequested) throw retryErr;
+ return await this._fallbackToCpuAndRetry(body, boundary, modelPath);
+ }
+ }
+
+ async _fallbackToCpuAndRetry(body, boundary, modelPath) {
+ const backend = this.useCuda ? "cuda" : "vulkan";
+ debugLogger.warn(`${backend} whisper-server died during transcription, falling back to CPU`, {
+ port: this.port,
+ model: modelPath ? path.basename(modelPath) : null,
+ });
+ await this.start(modelPath, { ...this.lastStartOptions, useCuda: false, useVulkan: false });
+ // Emit only once the CPU server is up — the notification tells the user CPU is in use
+ this.emit(backend === "cuda" ? "cuda-fallback" : "gpu-fallback");
+ return await this._postInference(body, boundary);
+ }
+
+ async _waitForProcessExit(timeoutMs) {
+ const deadline = Date.now() + timeoutMs;
+ while (this.process) {
+ if (Date.now() >= deadline) return false;
+ await new Promise((resolve) => setTimeout(resolve, PROCESS_EXIT_POLL_INTERVAL_MS));
+ }
+ return true;
+ }
+
async _convertToWav(audioBuffer) {
const tempDir = getSafeTempDir();
const timestamp = Date.now();
@@ -807,7 +937,7 @@ class WhisperServerManager extends EventEmitter {
}
async stop() {
- this._abortStartup = true;
+ this._stopRequested = true;
this.stopHealthCheck();
if (this.isRemote) {
@@ -874,3 +1004,5 @@ module.exports = WhisperServerManager;
module.exports.buildWhisperServerArgs = buildWhisperServerArgs;
module.exports.getVadSignature = getVadSignature;
module.exports.resolveWhisperThreads = resolveWhisperThreads;
+module.exports.shouldFallbackToCpuAfterRequestError = shouldFallbackToCpuAfterRequestError;
+module.exports.shouldRetryAfterServerReplaced = shouldRetryAfterServerReplaced;
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index ac5da96fe..75df050f1 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -2502,6 +2502,10 @@
"title": "Tastenkombination nicht verfügbar",
"description": "Die ausgewählte Tastenkombination konnte nicht registriert werden. Bitte wähle eine andere in den Einstellungen."
},
+ "gpuFallback": {
+ "title": "GPU-Beschleunigung nicht verfügbar",
+ "description": "Die Transkription verwendet stattdessen die CPU. Du kannst die GPU-Beschleunigung in den Einstellungen deaktivieren."
+ },
"addedToDict": "{{words}} zu deinem Wörterbuch hinzugefügt",
"undo": "Rückgängig",
"accessibilityMissing": {
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index fa3d30f9e..471b324be 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -2638,6 +2638,10 @@
"title": "Hotkey unavailable",
"description": "The selected hotkey could not be registered. Please choose a different one in settings."
},
+ "gpuFallback": {
+ "title": "GPU acceleration unavailable",
+ "description": "Transcription is using the CPU instead. You can disable GPU acceleration in Settings."
+ },
"addedToDict": "Added {{words}} to your dictionary",
"undo": "Undo",
"accessibilityMissing": {
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 55bf80218..92d5d61f3 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -2550,6 +2550,10 @@
"title": "Atajo no disponible",
"description": "No se pudo registrar el atajo seleccionado. Elige otro en la configuración."
},
+ "gpuFallback": {
+ "title": "Aceleración GPU no disponible",
+ "description": "La transcripción está usando la CPU. Puedes desactivar la aceleración GPU en Configuración."
+ },
"addedToDict": "{{words}} añadido a tu diccionario",
"undo": "Deshacer",
"accessibilityMissing": {
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index aed248c01..155c95777 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -2550,6 +2550,10 @@
"title": "Raccourci indisponible",
"description": "Le raccourci sélectionné n’a pas pu être enregistré. Veuillez en choisir un autre dans les paramètres."
},
+ "gpuFallback": {
+ "title": "Accélération GPU indisponible",
+ "description": "La transcription utilise le CPU à la place. Vous pouvez désactiver l'accélération GPU dans les paramètres."
+ },
"addedToDict": "{{words}} ajouté à votre dictionnaire",
"undo": "Annuler",
"accessibilityMissing": {
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index 1cc6bb310..d4555951a 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -2502,6 +2502,10 @@
"title": "Scorciatoia non disponibile",
"description": "Non è stato possibile registrare la scorciatoia selezionata. Scegline un’altra nelle impostazioni."
},
+ "gpuFallback": {
+ "title": "Accelerazione GPU non disponibile",
+ "description": "La trascrizione sta usando la CPU. Puoi disattivare l'accelerazione GPU nelle Impostazioni."
+ },
"addedToDict": "{{words}} aggiunto al tuo dizionario",
"undo": "Annulla",
"accessibilityMissing": {
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index 438c618f0..8795482f1 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -2502,6 +2502,10 @@
"title": "ホットキーが使用できません",
"description": "選択したホットキーを登録できませんでした。設定から別のホットキーを選択してください。"
},
+ "gpuFallback": {
+ "title": "GPU アクセラレーションを利用できません",
+ "description": "文字起こしは代わりに CPU を使用しています。設定で GPU アクセラレーションを無効にできます。"
+ },
"addedToDict": "{{words}} を辞書に追加しました",
"undo": "元に戻す",
"accessibilityMissing": {
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index 8a364169b..6e2f352b9 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -2502,6 +2502,10 @@
"title": "Atalho indisponível",
"description": "Não foi possível registrar o atalho selecionado. Escolha outro nas configurações."
},
+ "gpuFallback": {
+ "title": "Aceleração GPU indisponível",
+ "description": "A transcrição está usando a CPU. Você pode desativar a aceleração GPU nas Configurações."
+ },
"addedToDict": "{{words}} adicionado ao seu dicionário",
"undo": "Desfazer",
"accessibilityMissing": {
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index 5d3081cdb..8f2dbb82c 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -2506,6 +2506,10 @@
"title": "Горячая клавиша недоступна",
"description": "Не удалось зарегистрировать выбранную комбинацию. Выберите другую в настройках."
},
+ "gpuFallback": {
+ "title": "Ускорение GPU недоступно",
+ "description": "Транскрипция выполняется на CPU. Вы можете отключить ускорение GPU в настройках."
+ },
"addedToDict": "{{words}} добавлено в ваш словарь",
"undo": "Отменить",
"accessibilityMissing": {
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index 35100d5b1..167fee06f 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -2497,6 +2497,10 @@
"title": "快捷键不可用",
"description": "无法注册所选快捷键。请在设置中选择其他快捷键。"
},
+ "gpuFallback": {
+ "title": "GPU 加速不可用",
+ "description": "转录正在使用 CPU。你可以在设置中关闭 GPU 加速。"
+ },
"addedToDict": "已将 {{words}} 添加到你的词典",
"undo": "撤消",
"accessibilityMissing": {
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index 0550aa0e4..9bad9bdf1 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -2497,6 +2497,10 @@
"title": "快捷鍵無法使用",
"description": "無法註冊所選快捷鍵。請在設定中選擇其他快捷鍵。"
},
+ "gpuFallback": {
+ "title": "GPU 加速無法使用",
+ "description": "轉錄正在使用 CPU。你可以在設定中關閉 GPU 加速。"
+ },
"addedToDict": "已將 {{words}} 加入到你的詞典",
"undo": "復原",
"accessibilityMissing": {
diff --git a/test/helpers/whisperCudaRequestFallback.test.js b/test/helpers/whisperCudaRequestFallback.test.js
new file mode 100644
index 000000000..65f4c4cd5
--- /dev/null
+++ b/test/helpers/whisperCudaRequestFallback.test.js
@@ -0,0 +1,642 @@
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const http = require("node:http");
+
+const WhisperServerManager = require("../../src/helpers/whisperServer");
+const {
+ shouldFallbackToCpuAfterRequestError,
+ shouldRetryAfterServerReplaced,
+} = require("../../src/helpers/whisperServer");
+
+const base = {
+ isConnectionError: true,
+ useGpu: true,
+ isRemote: false,
+ stopRequested: false,
+ generationChanged: false,
+ processExited: true,
+};
+
+test("falls back when a local GPU server drops the connection and exits", () => {
+ assert.equal(shouldFallbackToCpuAfterRequestError(base), true);
+});
+
+test("skips a failure that is not a connection error (server answered)", () => {
+ assert.equal(shouldFallbackToCpuAfterRequestError({ ...base, isConnectionError: false }), false);
+});
+
+test("skips a CPU server (nothing to fall back to)", () => {
+ assert.equal(shouldFallbackToCpuAfterRequestError({ ...base, useGpu: false }), false);
+});
+
+test("skips a remote server (its process is not ours to restart)", () => {
+ assert.equal(shouldFallbackToCpuAfterRequestError({ ...base, isRemote: true }), false);
+});
+
+test("skips when the server was stopped on purpose", () => {
+ assert.equal(shouldFallbackToCpuAfterRequestError({ ...base, stopRequested: true }), false);
+});
+
+test("skips when another start replaced the server mid-request", () => {
+ assert.equal(shouldFallbackToCpuAfterRequestError({ ...base, generationChanged: true }), false);
+});
+
+test("skips when the server process is still alive (request refused, not a crash)", () => {
+ assert.equal(shouldFallbackToCpuAfterRequestError({ ...base, processExited: false }), false);
+});
+
+const replacedBase = {
+ isConnectionError: true,
+ isRemote: false,
+ stopRequested: false,
+ ready: true,
+ sameModel: true,
+};
+
+test("retries against the server a concurrent caller already restarted", () => {
+ assert.equal(shouldRetryAfterServerReplaced(replacedBase), true);
+});
+
+test("skips the replaced-server retry for a failure that is not a connection error", () => {
+ assert.equal(
+ shouldRetryAfterServerReplaced({ ...replacedBase, isConnectionError: false }),
+ false
+ );
+});
+
+test("skips the replaced-server retry when the server is now remote", () => {
+ assert.equal(shouldRetryAfterServerReplaced({ ...replacedBase, isRemote: true }), false);
+});
+
+test("skips the replaced-server retry after an intentional stop", () => {
+ assert.equal(shouldRetryAfterServerReplaced({ ...replacedBase, stopRequested: true }), false);
+});
+
+test("skips the replaced-server retry while the replacement is not ready", () => {
+ assert.equal(shouldRetryAfterServerReplaced({ ...replacedBase, ready: false }), false);
+});
+
+test("skips the replaced-server retry when the replacement switched model", () => {
+ assert.equal(shouldRetryAfterServerReplaced({ ...replacedBase, sameModel: false }), false);
+});
+
+function startServer(handler) {
+ const server = http.createServer(handler);
+ return new Promise((resolve) => {
+ server.listen(0, "127.0.0.1", () => resolve({ server, port: server.address().port }));
+ });
+}
+
+function createDeferred() {
+ let resolve;
+ let reject;
+ const promise = new Promise((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+ return { promise, resolve, reject };
+}
+
+function createManager(port, { useCuda, useVulkan = false }) {
+ const manager = new WhisperServerManager();
+ manager.ready = true;
+ manager.hostname = "127.0.0.1";
+ manager.port = port;
+ manager.useCuda = useCuda;
+ manager.useVulkan = useVulkan;
+ manager.canConvert = true;
+ manager.process = {};
+ manager.modelPath = "/tmp/model.bin";
+ manager.lastStartOptions = { useCuda, useVulkan };
+ manager._convertToWav = async (buffer) => buffer;
+ return manager;
+}
+
+test("falls back to CPU and retries once when the CUDA server dies mid-request", async (t) => {
+ let manager;
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req, res) => {
+ requestCount += 1;
+ if (requestCount === 1) {
+ // The real child's close handler clears this before the socket drops.
+ manager.process = null;
+ req.socket.destroy();
+ return;
+ }
+ res.writeHead(200, { "Content-Type": "application/json" });
+ res.end(JSON.stringify({ text: "hello" }));
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+
+ const startCalls = [];
+ manager.start = async (modelPath, options) => {
+ startCalls.push({ modelPath, options });
+ manager.useCuda = false;
+ manager.ready = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ const result = await manager.transcribe(Buffer.from("audio"));
+
+ assert.equal(result.text, "hello");
+ assert.equal(requestCount, 2);
+ assert.equal(fallbackEvents, 1);
+ assert.equal(startCalls.length, 1);
+ assert.equal(startCalls[0].modelPath, "/tmp/model.bin");
+ assert.equal(startCalls[0].options.useCuda, false);
+ assert.equal(startCalls[0].options.useVulkan, false);
+});
+
+test("falls back to CPU and emits gpu-fallback when a Vulkan server dies mid-request", async (t) => {
+ let manager;
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req, res) => {
+ requestCount += 1;
+ if (requestCount === 1) {
+ manager.process = null;
+ req.socket.destroy();
+ return;
+ }
+ res.writeHead(200, { "Content-Type": "application/json" });
+ res.end(JSON.stringify({ text: "hello" }));
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: false, useVulkan: true });
+
+ const startCalls = [];
+ manager.start = async (modelPath, options) => {
+ startCalls.push({ modelPath, options });
+ manager.useVulkan = false;
+ manager.ready = true;
+ };
+
+ const events = [];
+ manager.on("cuda-fallback", () => events.push("cuda-fallback"));
+ manager.on("gpu-fallback", () => events.push("gpu-fallback"));
+
+ const result = await manager.transcribe(Buffer.from("audio"));
+
+ assert.equal(result.text, "hello");
+ assert.equal(requestCount, 2);
+ assert.deepEqual(events, ["gpu-fallback"]);
+ assert.equal(startCalls.length, 1);
+ assert.equal(startCalls[0].options.useCuda, false);
+ assert.equal(startCalls[0].options.useVulkan, false);
+});
+
+test("falls back to CPU when a peer's replacement is another doomed CUDA server", async (t) => {
+ let manager;
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req, res) => {
+ requestCount += 1;
+ if (requestCount === 1) {
+ // Crash, then simulate a peer restarting with CUDA still enabled.
+ manager.process = null;
+ manager.startGeneration += 1;
+ manager.ready = true;
+ req.socket.destroy();
+ return;
+ }
+ if (requestCount === 2) {
+ // The replacement CUDA server aborts at its first kernel launch too.
+ manager.process = null;
+ req.socket.destroy();
+ return;
+ }
+ res.writeHead(200, { "Content-Type": "application/json" });
+ res.end(JSON.stringify({ text: "hello" }));
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+
+ const startCalls = [];
+ manager.start = async (modelPath, options) => {
+ startCalls.push({ modelPath, options });
+ manager.useCuda = false;
+ manager.ready = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ const result = await manager.transcribe(Buffer.from("audio"));
+
+ assert.equal(result.text, "hello");
+ assert.equal(requestCount, 3);
+ assert.equal(fallbackEvents, 1);
+ assert.equal(startCalls.length, 1);
+ assert.equal(startCalls[0].options.useCuda, false);
+});
+
+test("does not fall back when the request fails on a CPU server", async (t) => {
+ let manager;
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req) => {
+ requestCount += 1;
+ manager.process = null;
+ req.socket.destroy();
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: false });
+
+ let startCalled = false;
+ manager.start = async () => {
+ startCalled = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ await assert.rejects(
+ () => manager.transcribe(Buffer.from("audio")),
+ (err) => {
+ assert.ok(err.message.startsWith("whisper-server request failed"));
+ return true;
+ }
+ );
+
+ assert.equal(requestCount, 1);
+ assert.equal(startCalled, false);
+ assert.equal(fallbackEvents, 0);
+});
+
+test("recovers both callers when two concurrent requests hit the same crash", async (t) => {
+ let manager;
+ let requestCount = 0;
+ const crashing = [];
+
+ const { server, port } = await startServer((req, res) => {
+ requestCount += 1;
+ if (requestCount <= 2) {
+ // Drop both sockets only once both requests are in flight against the CUDA server.
+ crashing.push(req);
+ if (crashing.length === 2) {
+ manager.process = null;
+ for (const pending of crashing) pending.socket.destroy();
+ }
+ return;
+ }
+ res.writeHead(200, { "Content-Type": "application/json" });
+ res.end(JSON.stringify({ text: "hello" }));
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+
+ const startCalls = [];
+ manager.start = async (mp, options) => {
+ startCalls.push({ modelPath: mp, options });
+ manager.startGeneration += 1;
+ manager.useCuda = false;
+ manager.ready = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ const results = await Promise.all([
+ manager.transcribe(Buffer.from("audio-a")),
+ manager.transcribe(Buffer.from("audio-b")),
+ ]);
+
+ assert.equal(results[0].text, "hello");
+ assert.equal(results[1].text, "hello");
+ assert.equal(fallbackEvents, 1);
+ assert.equal(startCalls.length, 1);
+ assert.equal(startCalls[0].options.useCuda, false);
+ assert.equal(requestCount, 4);
+});
+
+test("waits for a peer's in-flight restart before retrying", async (t) => {
+ let manager;
+ let requestCount = 0;
+ let restartFinished = false;
+ let restartFinishedWhenRetried = null;
+ const restarted = createDeferred();
+
+ const { server, port } = await startServer((req, res) => {
+ requestCount += 1;
+ if (requestCount === 1) {
+ manager.process = null;
+ manager.startGeneration += 1;
+ manager.ready = false;
+ manager.startupPromise = restarted.promise;
+ setTimeout(() => {
+ manager.ready = true;
+ manager.useCuda = false;
+ manager.startupPromise = null;
+ restartFinished = true;
+ restarted.resolve();
+ }, 100);
+ req.socket.destroy();
+ return;
+ }
+ restartFinishedWhenRetried = restartFinished;
+ res.writeHead(200, { "Content-Type": "application/json" });
+ res.end(JSON.stringify({ text: "hello" }));
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+
+ const startCalls = [];
+ manager.start = async (modelPath, options) => {
+ startCalls.push({ modelPath, options });
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ const result = await manager.transcribe(Buffer.from("audio"));
+
+ assert.equal(result.text, "hello");
+ assert.equal(requestCount, 2);
+ assert.equal(restartFinishedWhenRetried, true);
+ assert.equal(startCalls.length, 0);
+ assert.equal(fallbackEvents, 0);
+});
+
+test("throws the original error when the peer's restart fails", async (t) => {
+ let manager;
+ let requestCount = 0;
+ const restarted = createDeferred();
+
+ const { server, port } = await startServer((req) => {
+ requestCount += 1;
+ manager.process = null;
+ manager.startGeneration += 1;
+ manager.ready = false;
+ manager.startupPromise = restarted.promise;
+ setTimeout(() => {
+ manager.startupPromise = null;
+ restarted.reject(new Error("boom"));
+ }, 100);
+ req.socket.destroy();
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+
+ const startCalls = [];
+ manager.start = async (modelPath, options) => {
+ startCalls.push({ modelPath, options });
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ await assert.rejects(
+ () => manager.transcribe(Buffer.from("audio")),
+ (err) => {
+ assert.ok(err.message.startsWith("whisper-server request failed"));
+ return true;
+ }
+ );
+
+ assert.equal(requestCount, 1);
+ assert.equal(startCalls.length, 0);
+ assert.equal(fallbackEvents, 0);
+});
+
+test("aborts the fallback when a stop request lands during the exit wait", async (t) => {
+ let manager;
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req) => {
+ requestCount += 1;
+ setTimeout(() => {
+ manager._stopRequested = true;
+ manager.process = null;
+ }, 100);
+ req.socket.destroy();
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+
+ let startCalled = false;
+ manager.start = async () => {
+ startCalled = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ await assert.rejects(
+ () => manager.transcribe(Buffer.from("audio")),
+ (err) => {
+ assert.ok(err.message.startsWith("whisper-server request failed"));
+ return true;
+ }
+ );
+
+ assert.equal(requestCount, 1);
+ assert.equal(startCalled, false);
+ assert.equal(fallbackEvents, 0);
+});
+
+test("does not fall back while the CUDA server process stays alive", async (t) => {
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req) => {
+ requestCount += 1;
+ req.socket.destroy();
+ });
+ t.after(() => server.close());
+
+ const manager = createManager(port, { useCuda: true });
+
+ let startCalled = false;
+ manager.start = async () => {
+ startCalled = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ await assert.rejects(
+ () => manager.transcribe(Buffer.from("audio")),
+ (err) => {
+ assert.ok(err.message.startsWith("whisper-server request failed"));
+ return true;
+ }
+ );
+
+ assert.equal(requestCount, 1);
+ assert.equal(manager.process !== null, true);
+ assert.equal(startCalled, false);
+ assert.equal(fallbackEvents, 0);
+});
+
+test("_waitForProcessExit resolves true immediately when no process is tracked", async () => {
+ const manager = new WhisperServerManager();
+ manager.process = null;
+
+ const startedAt = Date.now();
+ const exited = await manager._waitForProcessExit(500);
+ const elapsed = Date.now() - startedAt;
+
+ assert.equal(exited, true);
+ assert.ok(elapsed < 100, `expected no wait, took ${elapsed}ms`);
+});
+
+test("_waitForProcessExit resolves false after the deadline while the process stays alive", async () => {
+ const manager = new WhisperServerManager();
+ manager.process = {};
+
+ const startedAt = Date.now();
+ const exited = await manager._waitForProcessExit(120);
+ const elapsed = Date.now() - startedAt;
+
+ assert.equal(exited, false);
+ assert.ok(elapsed >= 120, `expected the full deadline, took ${elapsed}ms`);
+ assert.ok(elapsed < 1500, `expected the deadline to bound the wait, took ${elapsed}ms`);
+});
+
+test("_waitForProcessExit resolves true as soon as the process exits mid-wait", async () => {
+ const manager = new WhisperServerManager();
+ manager.process = {};
+ setTimeout(() => {
+ manager.process = null;
+ }, 60);
+
+ const startedAt = Date.now();
+ const exited = await manager._waitForProcessExit(1000);
+ const elapsed = Date.now() - startedAt;
+
+ assert.equal(exited, true);
+ assert.ok(elapsed < 800, `expected an early exit, took ${elapsed}ms`);
+});
+
+test("propagates the CPU restart failure without notifying (CPU never came up)", async (t) => {
+ let manager;
+
+ const { server, port } = await startServer((req) => {
+ manager.process = null;
+ req.socket.destroy();
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+ manager.start = async () => {
+ throw new Error("boom");
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ await assert.rejects(
+ () => manager.transcribe(Buffer.from("audio")),
+ (err) => {
+ assert.equal(err.message, "boom");
+ return true;
+ }
+ );
+
+ assert.equal(fallbackEvents, 0);
+});
+
+test("rejects immediately for a CPU server without waiting for process exit", async (t) => {
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req) => {
+ requestCount += 1;
+ req.socket.destroy();
+ });
+ t.after(() => server.close());
+
+ // The process stays alive: the old code polled it for the full exit timeout.
+ const manager = createManager(port, { useCuda: false });
+
+ let startCalled = false;
+ manager.start = async () => {
+ startCalled = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ const startedAt = Date.now();
+ await assert.rejects(
+ () => manager.transcribe(Buffer.from("audio")),
+ (err) => {
+ assert.ok(err.message.startsWith("whisper-server request failed"));
+ return true;
+ }
+ );
+ const elapsed = Date.now() - startedAt;
+
+ assert.ok(elapsed < 1000, `expected an immediate rejection, took ${elapsed}ms`);
+ assert.equal(requestCount, 1);
+ assert.equal(manager.process !== null, true);
+ assert.equal(startCalled, false);
+ assert.equal(fallbackEvents, 0);
+});
+
+test("does not fall back after an intentional stop", async (t) => {
+ let manager;
+ let requestCount = 0;
+
+ const { server, port } = await startServer((req) => {
+ requestCount += 1;
+ manager.process = null;
+ req.socket.destroy();
+ });
+ t.after(() => server.close());
+
+ manager = createManager(port, { useCuda: true });
+ manager._stopRequested = true;
+
+ let startCalled = false;
+ manager.start = async () => {
+ startCalled = true;
+ };
+
+ let fallbackEvents = 0;
+ manager.on("cuda-fallback", () => {
+ fallbackEvents += 1;
+ });
+
+ await assert.rejects(
+ () => manager.transcribe(Buffer.from("audio")),
+ (err) => {
+ assert.ok(err.message.startsWith("whisper-server request failed"));
+ return true;
+ }
+ );
+
+ assert.equal(requestCount, 1);
+ assert.equal(startCalled, false);
+ assert.equal(fallbackEvents, 0);
+});