Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix(whisper): save and show why GPU acceleration fell back to CPU (#2317)
Joshua58e8cb00d18f
diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.mdindex f78dc7621..b056577e9 100644--- a/TROUBLESHOOTING.md+++ b/TROUBLESHOOTING.md@@ -102,7 +102,7 @@4. Clear model cache: `rm -rf ~/.cache/openwhispr/whisper-models`5. Try cloud transcription as fallback-**GPU acceleration (CUDA / Vulkan):** If the GPU-accelerated whisper-server crashes at startup (unsupported GPU, out of VRAM), OpenWhispr automatically restarts it on CPU, retries the same request, and shows a "using CPU instead" notice — the dictation still completes. GPU acceleration can be toggled off from the GPU card in the transcription model picker.+**GPU acceleration (CUDA / Vulkan):** If the GPU-accelerated whisper-server crashes at startup (unsupported GPU, out of VRAM), OpenWhispr automatically restarts it on CPU, retries the same request, and shows a "using CPU instead" notice — the dictation still completes. GPU acceleration can be toggled off from the GPU card in the transcription model picker. That card also shows the error line that caused the fallback (for example `vk::PhysicalDevice::createDevice: ErrorDeviceLost`); include it, or a screenshot of the card, when you report a GPU problem.### Wayland Clipboard Issues (Linux)diff --git a/src/components/TranscriptionModelPicker.tsx b/src/components/TranscriptionModelPicker.tsxindex ec2f8ca3c..5536d2b42 100644--- a/src/components/TranscriptionModelPicker.tsx+++ b/src/components/TranscriptionModelPicker.tsx@@ -1,4 +1,4 @@-import { useState, useEffect, useCallback, useMemo, useRef } from "react";+import { useState, useEffect, useCallback, useMemo, useRef, type ReactNode } from "react";import { useTranslation } from "react-i18next";import { Button } from "./ui/button";import { Input } from "./ui/input";@@ -48,7 +48,11 @@ import { GetApiKeyLink } from "./ui/GetApiKeyLink";import { getCachedPlatform } from "../utils/platform";import { pickWhisperGpuBackend } from "../utils/whisperGpuPack";import logger from "../utils/logger";-import type { ParakeetCheckResult } from "../types/electron";+import type {+ CudaWhisperStatus,+ ParakeetCheckResult,+ VulkanWhisperStatus,+} from "../types/electron";interface LocalModel {model: string;@@ -383,6 +387,7 @@ interface GpuWarningRowProps {actionLabel: string;onAction: () => void;onRemove: () => void;+ children?: ReactNode;}function GpuWarningRow({@@ -391,6 +396,7 @@ function GpuWarningRow({actionLabel,onAction,onRemove,+ children,}: GpuWarningRowProps) {const { t } = useTranslation();return (@@ -400,6 +406,7 @@ function GpuWarningRow({<div className="min-w-0"><p className="text-xs font-medium text-foreground">{title}</p><p className="mt-0.5 text-xs leading-snug text-muted-foreground">{description}</p>+ {children}<Button onClick={onAction} size="sm" className="mt-2 h-7 px-3 text-xs">{actionLabel}</Button>@@ -488,6 +495,8 @@ export default function TranscriptionModelPicker({const [gpuDismissed, setGpuDismissed] = useState(false);// The pack fell back to CPU on this machine (persisted by main until retried)const [gpuFailed, setGpuFailed] = useState(false);+ // The whisper-server error line main saved with that failure (#1736)+ const [gpuFailReason, setGpuFailReason] = useState<string | null>(null);// An older release installed the pack and this version can't use it (#2424)const [gpuNeedsUpdate, setGpuNeedsUpdate] = useState(false);// Why the last pack download failed (cleared by the next attempt)@@ -770,6 +779,7 @@ export default function TranscriptionModelPicker({setGpuBackend(backend);setGpuDownloaded(!!status?.downloaded);setGpuFailed(!!status?.gpuFailed);+ setGpuFailReason(status?.gpuFailReason ?? null);setGpuNeedsUpdate(!!status?.needsUpdate);// A download this card started before Settings was closed is still runningif (status?.downloading) {@@ -838,13 +848,26 @@ export default function TranscriptionModelPicker({// Main falls back to CPU (and remembers it) when a GPU server crashesuseEffect(() => {- const onFallback = () => {- setGpuFailed(true);- setGpuActivating(false);- setGpuActive(false);- };- const disposeCuda = window.electronAPI?.onCudaFallbackNotification?.(onFallback);- const disposeVulkan = window.electronAPI?.onGpuFallbackNotification?.(onFallback);+ // Main saves the reason before it notifies. Read it from the backend that+ // failed, not the installed-pack preference above: with both packs+ // installed, the card can show CUDA while the server ran Vulkan (#1736).+ const onFallback =+ (readStatus: () => Promise<CudaWhisperStatus | VulkanWhisperStatus>) => () => {+ setGpuFailed(true);+ // Never show the previous failure's reason while the new one loads+ setGpuFailReason(null);+ setGpuActivating(false);+ setGpuActive(false);+ readStatus()+ .then((status) => setGpuFailReason(status.gpuFailReason ?? null))+ .catch(() => {});+ };+ const disposeCuda = window.electronAPI?.onCudaFallbackNotification?.(+ onFallback(window.electronAPI.getCudaWhisperStatus)+ );+ const disposeVulkan = window.electronAPI?.onGpuFallbackNotification?.(+ onFallback(window.electronAPI.getVulkanWhisperStatus)+ );return () => {disposeCuda?.();disposeVulkan?.();@@ -1439,7 +1462,16 @@ export default function TranscriptionModelPicker({actionLabel={t("gpu.retryActivation")}onAction={handleGpuRetry}onRemove={handleGpuDelete}- />+ >+ {gpuFailReason && (+ <p+ dir="ltr"+ className="mt-1 wrap-break-word font-mono text-[11px] leading-snug text-muted-foreground"+ >+ {gpuFailReason}+ </p>+ )}+ </GpuWarningRow>) : (<div className="flex items-center justify-between"><div className="flex items-center gap-1.5">diff --git a/src/helpers/environment.js b/src/helpers/environment.jsindex d717e9afa..ce06240bc 100644--- a/src/helpers/environment.js+++ b/src/helpers/environment.js@@ -47,6 +47,8 @@ const PERSISTED_KEYS = ["WHISPER_VULKAN_ENABLED","WHISPER_VULKAN_DEVICE","WHISPER_GPU_FAILED",+ "WHISPER_GPU_FAILED_REASON_CUDA",+ "WHISPER_GPU_FAILED_REASON_VULKAN","WHISPER_THREADS","TRANSCRIPTION_GPU_UUID","INTELLIGENCE_GPU_UUID",diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 1d776733f..3ba2c0802 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -13,6 +13,7 @@ const { getModelType, isSherpaLocalProvider } = require("./parakeetModelInfo");const { broadcastToWindows } = require("./windowBroadcast");const { openExternalUrl } = require("./externalUrlOpener");const { resolveFailedGpuBackends } = require("./whisper");+const { WHISPER_GPU_FAILURE_REASON_KEYS } = require("./whisperGpuFailureReason");const { BYOK_API_KEYS } = require("../config/secretKeys");const tokenStore = require("./tokenStore");const accountScopeBinding = require("./accountScopeBinding");@@ -685,24 +686,7 @@ class IPCHandlers {tokenStore.subscribe((state) => this._handleAuthTokenChange(state));if (this.whisperManager?.serverManager) {- // Remember the failed backend so it isn't re-attempted (and its model- // reload re-paid) on every launch; cleared by retry, re-download, delete.- this.whisperManager.serverManager.on("cuda-fallback", () => {- this._recordWhisperGpuFailure("cuda");- broadcastToWindows("cuda-fallback-notification", {});- });- this.whisperManager.serverManager.on("gpu-fallback", () => {- this._recordWhisperGpuFailure("vulkan");- broadcastToWindows("gpu-fallback-notification", {});- });- // Persist the discrete-GPU pin so later launches spawn pinned directly- // instead of paying a second Vulkan cold start. See #1606.- this.whisperManager.serverManager.on("vulkan-device-pinned", ({ index }) => {- this._syncStartupEnv({ WHISPER_VULKAN_DEVICE: String(index) });- });- this.whisperManager.serverManager.on("vulkan-device-pin-cleared", () => {- this._syncStartupEnv({}, ["WHISPER_VULKAN_DEVICE"]);- });+ this._attachWhisperServerListeners(this.whisperManager.serverManager);}}@@ -1256,21 +1240,57 @@ class IPCHandlers {return resolveFailedGpuBackends(process.env.WHISPER_GPU_FAILED);}- _recordWhisperGpuFailure(backend) {+ _attachWhisperServerListeners(serverManager) {+ serverManager.on("cuda-fallback", ({ reason }) => {+ this._recordWhisperGpuFailure("cuda", reason);+ broadcastToWindows("cuda-fallback-notification", {});+ });+ serverManager.on("gpu-fallback", ({ reason }) => {+ this._recordWhisperGpuFailure("vulkan", reason);+ broadcastToWindows("gpu-fallback-notification", {});+ });+ // Persist the discrete-GPU pin so later launches spawn pinned directly+ // instead of paying a second Vulkan cold start. See #1606.+ serverManager.on("vulkan-device-pinned", ({ index }) => {+ this._syncStartupEnv({ WHISPER_VULKAN_DEVICE: String(index) });+ });+ serverManager.on("vulkan-device-pin-cleared", () => {+ this._syncStartupEnv({}, ["WHISPER_VULKAN_DEVICE"]);+ });+ }++ // Remember the failed backend, and the error line that explains it, so the+ // backend isn't re-attempted (and its model reload re-paid) on every launch+ // and the settings card can say why (#1736). Cleared by retry, re-download,+ // delete, and the once-per-upgrade reset. The reason is written in the same+ // .env write as the flag. A failure with no readable reason clears the older+ // one, so a stale cause is never shown.+ _recordWhisperGpuFailure(backend, reason) {const failed = this._whisperGpuFailedBackends();if (!failed.includes(backend)) failed.push(backend);- this._syncStartupEnv({ WHISPER_GPU_FAILED: failed.join(",") });+ const reasonKey = WHISPER_GPU_FAILURE_REASON_KEYS[backend];+ this._syncStartupEnv(+ { WHISPER_GPU_FAILED: failed.join(","), ...(reason ? { [reasonKey]: reason } : {}) },+ reason ? [] : [reasonKey]+ );}_clearWhisperGpuFailure(backend) {const failed = this._whisperGpuFailedBackends().filter((b) => b !== backend);+ const reasonKey = WHISPER_GPU_FAILURE_REASON_KEYS[backend];if (failed.length > 0) {- this._syncStartupEnv({ WHISPER_GPU_FAILED: failed.join(",") });+ this._syncStartupEnv({ WHISPER_GPU_FAILED: failed.join(",") }, [reasonKey]);} else {- this._syncStartupEnv({}, ["WHISPER_GPU_FAILED"]);+ this._syncStartupEnv({}, ["WHISPER_GPU_FAILED", reasonKey]);}}+ _whisperGpuFailureStatus(backend) {+ const gpuFailed = this._whisperGpuFailedBackends().includes(backend);+ const reason = gpuFailed ? process.env[WHISPER_GPU_FAILURE_REASON_KEYS[backend]] : null;+ return { gpuFailed, gpuFailReason: reason || null };+ }+// Captured before a handler stops the server to touch pack files (stopServer// clears currentServerModel); tells _applyWhisperGpuPreference what to reload._whisperReloadModel() {@@ -3440,7 +3460,7 @@ class IPCHandlers {downloading: this.whisperCudaManager.isDownloading(),path: this.whisperCudaManager.getCudaBinaryPath(),gpuInfo,- gpuFailed: this._whisperGpuFailedBackends().includes("cuda"),+ ...this._whisperGpuFailureStatus("cuda"),};});@@ -3503,7 +3523,7 @@ class IPCHandlers {downloading: this.whisperVulkanManager?.isDownloading() ?? false,vulkan,hasNvidiaGpu: gpuInfo.hasNvidiaGpu,- gpuFailed: this._whisperGpuFailedBackends().includes("vulkan"),+ ...this._whisperGpuFailureStatus("vulkan"),};});@@ -3567,7 +3587,10 @@ class IPCHandlers {// Clears the remembered GPU failure and reloads the server with the GPU// backend re-enabled (Retry on the "GPU could not be activated" state)ipcMain.handle("whisper-gpu-retry", async () => {- this._syncStartupEnv({}, ["WHISPER_GPU_FAILED"]);+ this._syncStartupEnv({}, [+ "WHISPER_GPU_FAILED",+ ...Object.values(WHISPER_GPU_FAILURE_REASON_KEYS),+ ]);return {success: true,willRestart: this._applyWhisperGpuPreference(this._whisperReloadModel()),diff --git a/src/helpers/whisperGpuFailureReason.js b/src/helpers/whisperGpuFailureReason.jsnew file mode 100644index 000000000..aca55d8db--- /dev/null+++ b/src/helpers/whisperGpuFailureReason.js@@ -0,0 +1,116 @@+const os = require("os");++// When a GPU whisper-server falls back to CPU, this picks the stderr line that+// explains why, for the settings card and bug reports (#1736).++const MAX_REASON_LENGTH = 240;++// Where the reason is saved: one .env key per backend beside WHISPER_GPU_FAILED,+// set and cleared with it.+const WHISPER_GPU_FAILURE_REASON_KEYS = Object.freeze({+ cuda: "WHISPER_GPU_FAILED_REASON_CUDA",+ vulkan: "WHISPER_GPU_FAILED_REASON_VULKAN",+});++// Most specific first; the capture group is the reason. Formats are from the+// pinned OpenWhispr/whisper.cpp tag.+const CAUSE_PATTERNS = [+ // src/whisper.cpp (whisper_init_with_params_no_state) catches the backend's+ // C++ exception around model load, e.g. ggml-vulkan's createDevice throwing+ // vk::DeviceLostError: "...: exception during model load: <what()>"+ /exception during model load: (.+)/,+ // ggml-vulkan's VK_CHECK and fence wait print this and exit(1), e.g.+ // "ggml_vulkan: error ErrorDeviceLost at .../ggml-vulkan.cpp:2209"+ /(ggml_vulkan: .*\berror Error\w+.*)/,+ // vulkan-hpp's exception text wherever else it surfaces, e.g. the "what():"+ // line of an exception nothing caught. \berror\b cannot see "ErrorDeviceLost".+ /(vk::\S+: Error\w+)/,+ // ggml-cuda.cu ggml_cuda_error: "CUDA error: <cudaGetErrorString>"+ /(CUDA error: .+)/,+];+// Any other error line; the first one is the closest to the cause.+const ERROR_LINE = /\b(?:error|failed|failure|exception|abort(?:ed)?)\b/i;+// ggml_abort and the assert macros print "<source file>:<line>: <message>" and+// abort, e.g. "…/ggml-vulkan.cpp:8412: Requested preallocation size is too large".+// Often no error word, and after an error line it is only the consequence.+const ABORT_LINE = /^(?:WHISPER_ASSERT: )?\S+\.(?:c|cpp|cu|cuh|h):\d+: /;+// What whisper.cpp and whisper-server print after any failed load. They say+// that loading failed, never why, so they are the answer of last resort.+const LOAD_FAILURE_ECHO = /failed to load model|failed to initialize whisper context/;+// Warnings the backends log and then carry on from (a CPU-side buffer instead+// of pinned memory, a copy instead of an imported host pointer, the Vulkan+// loader skipping one of several drivers). They carry error text, so they would+// otherwise outrank the line that killed it. The non-pinned abort is fatal.+const RECOVERED_WARNING =+ /(?<!non-)pinned memory|^WARNING:|\[Loader Message\]|Failed getMemoryHostPointerPropertiesEXT|Failed ggml_vk_create_buffer/;++function findCauseLine(lines) {+ for (const pattern of CAUSE_PATTERNS) {+ for (const line of lines) {+ const match = line.match(pattern);+ if (match) return match[1];+ }+ }+ return (+ lines.find((line) => ERROR_LINE.test(line) && !LOAD_FAILURE_ECHO.test(line)) ||+ lines.find((line) => ABORT_LINE.test(line)) ||+ lines.find((line) => LOAD_FAILURE_ECHO.test(line)) ||+ null+ );+}++// One line that is safe to show and to save. EnvironmentManager writes .env+// values raw (KEY=value), and dotenv reads "#" as a comment and a leading+// quote as the start of a quoted value that can run over later keys. A trailing+// quote can close one an earlier line left open (a backtick hotkey).+function sanitizeReason(text, homeDir) {+ let reason = text;+ if (homeDir) {+ // People screenshot this into public issues, and a home folder is often+ // named after its owner.+ for (const home of new Set([homeDir, homeDir.replace(/\\/g, "/")])) {+ reason = reason.replace(new RegExp(home.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "gi"), "~");+ }+ }+ reason = reason+ .replace(/[\p{Cc}\u2028\u2029]/gu, " ")+ .replace(/#/g, "")+ .replace(/\s+/g, " ")+ .trim()+ .replace(/^['"`\s]+|['"`\s]+$/g, "");+ if (reason.length > MAX_REASON_LENGTH) {+ reason = `${reason.slice(0, MAX_REASON_LENGTH - 1).trimEnd()}…`;+ }+ return reason || null;+}++/**+ * The key line explaining why a GPU whisper-server failed, or how the process+ * ended when its output names no cause. One line of at most MAX_REASON_LENGTH+ * characters, or null when there is nothing to report.+ */+function extractWhisperGpuFailureReason({+ stderr = "",+ exitCode = null,+ signal = null,+ timeoutMs = null,+ homeDir = os.homedir(),+}) {+ const lines = stderr+ .split(/\r\n|\r|\n/)+ .map((line) => line.trim())+ .filter((line) => line && !RECOVERED_WARNING.test(line));+ const cause = findCauseLine(lines);+ const reason = cause ? sanitizeReason(cause, homeDir) : null;+ if (reason) return reason;+ // No line names the cause (a driver crash, a missing DLL, a hang): say how it ended+ if (signal) return `terminated by ${signal}`;+ if (exitCode !== null && exitCode !== undefined) return `exit code ${exitCode}`;+ if (timeoutMs) return `startup timed out after ${Math.round(timeoutMs / 1000)} s`;+ return null;+}++module.exports = {+ WHISPER_GPU_FAILURE_REASON_KEYS,+ extractWhisperGpuFailureReason,+};diff --git a/src/helpers/whisperGpuUpgradeReset.js b/src/helpers/whisperGpuUpgradeReset.jsindex d804a9a5a..b31e8fba8 100644--- a/src/helpers/whisperGpuUpgradeReset.js+++ b/src/helpers/whisperGpuUpgradeReset.js@@ -2,6 +2,7 @@ const fs = require("fs");const path = require("path");const { app } = require("electron");const debugLogger = require("./debugLogger");+const { WHISPER_GPU_FAILURE_REASON_KEYS } = require("./whisperGpuFailureReason");// WHISPER_GPU_FAILED persists until a manual Retry, so a GPU knocked out once// (a driver hiccup, or 1.8.3's pack deletion) stays on CPU whisper forever.@@ -11,6 +12,8 @@ const debugLogger = require("./debugLogger");// next server start. Must run before the whisper server pre-warm resolves its// GPU options.const SENTINEL_FILENAME = ".whisper-gpu-retry-version";+// The failure flag and the reasons saved with it (#1736) are one record+const GPU_FAILURE_KEYS = ["WHISPER_GPU_FAILED", ...Object.values(WHISPER_GPU_FAILURE_REASON_KEYS)];function getSentinelPath() {return path.join(app.getPath("userData"), SENTINEL_FILENAME);@@ -28,19 +31,23 @@ function resetWhisperGpuFailureOnUpgrade(environmentManager) {} catch {// Best-effort: without the sentinel the flag is cleared again next launch.}- if (!process.env.WHISPER_GPU_FAILED) return false;- delete process.env.WHISPER_GPU_FAILED;+ const storedKeys = GPU_FAILURE_KEYS.filter((key) => process.env[key]);+ if (storedKeys.length === 0) return false;+ for (const key of storedKeys) delete process.env[key];debugLogger.info("Cleared remembered whisper GPU failure for a fresh attempt after upgrade", {from: lastRun,to: version,});// Targeted removal: a full saveAllKeysToEnvFile() rewrite would drop// hand-added .env lines (e.g. OPENWHISPR_LOG_LEVEL=debug).- environmentManager.removeKeyFromEnvFile("WHISPER_GPU_FAILED").catch((err) => {- debugLogger.error("Failed to persist WHISPER_GPU_FAILED clear to .env", {- error: err.message,+ for (const key of storedKeys) {+ environmentManager.removeKeyFromEnvFile(key).catch((err) => {+ debugLogger.error("Failed to persist whisper GPU failure clear to .env", {+ key,+ error: err.message,+ });});- });+ }return true;}diff --git a/src/helpers/whisperServer.js b/src/helpers/whisperServer.jsindex 2c45a86fb..e789cf162 100644--- a/src/helpers/whisperServer.js+++ b/src/helpers/whisperServer.js@@ -18,6 +18,7 @@ const {computeTranscriptionTimeoutMs,PCM16_MONO_16K_BYTES_PER_SECOND,} = require("./transcriptionTimeout");+const { extractWhisperGpuFailureReason } = require("./whisperGpuFailureReason");const PORT_RANGE_START = 8178;const PORT_RANGE_END = 8199;@@ -305,6 +306,9 @@ class WhisperServerManager extends EventEmitter {this.gpuSignature = "gpu:cpu";this.gpuFallbackActive = false;this.lastStartOptions = {};+ // Output and exit of the last spawned server, read when a GPU server that+ // started fine dies mid-transcription (_fallbackToCpuAndRetry). See #1736.+ this._lastProcessInfo = null;}getFFmpegPath() {@@ -642,6 +646,9 @@ class WhisperServerManager extends EventEmitter {let stderrBuffer = "";let exitCode = null;+ let exitSignal = null;+ const getProcessInfo = () => ({ stderr: stderrBuffer, exitCode, signal: exitSignal });+ this._lastProcessInfo = getProcessInfo;this.process.stdout.on("data", (data) => {debugLogger.debug("whisper-server stdout", { data: data.toString().trim() });@@ -657,20 +664,19 @@ class WhisperServerManager extends EventEmitter {this.ready = false;});- this.process.on("close", (code) => {+ this.process.on("close", (code, signal) => {exitCode = code;- debugLogger.debug("whisper-server process exited", { code });+ exitSignal = signal;+ debugLogger.debug("whisper-server process exited", { code, signal });this.ready = false;this.process = null;this.stopHealthCheck();sidecarPidFile.clear("whisper");});+ const startupTimeoutMs = usingVulkan ? VULKAN_STARTUP_TIMEOUT_MS : STARTUP_TIMEOUT_MS;try {- await this.waitForReady(- () => ({ stderr: stderrBuffer, exitCode }),- usingVulkan ? VULKAN_STARTUP_TIMEOUT_MS : STARTUP_TIMEOUT_MS- );+ await this.waitForReady(getProcessInfo, startupTimeoutMs);} catch (err) {// An intentional stop() during startup is not a GPU/thread failureif (err.isStopped) throw err;@@ -678,16 +684,17 @@ class WhisperServerManager extends EventEmitter {// Fall back on ANY startup rejection — a GPU server can exit early// (missing kernels), die late (VRAM OOM mid-model-load), or hang, and// in every case the CPU binary is the working answer. stop() reaps a- // hung process before the CPU restart.+ // hung process before the CPU restart. The reason travels with the+ // event so it is saved beside the failure and shown on the GPU card (#1736).+ const reason = extractWhisperGpuFailureReason({+ ...getProcessInfo(),+ timeoutMs: startupTimeoutMs,+ });debugLogger.warn(`${usingCuda ? "CUDA" : "Vulkan"} whisper-server failed, falling back to CPU`,- {- error: err.message,- exitCode,- stderr: stderrBuffer.slice(0, 200),- }+ { error: err.message, exitCode, reason });- this.emit(usingCuda ? "cuda-fallback" : "gpu-fallback");+ this.emit(usingCuda ? "cuda-fallback" : "gpu-fallback", { reason });await this.stop();this.gpuFallbackActive = true;return this._doStart(modelPath, { ...options, useCuda: false, useVulkan: false });@@ -928,6 +935,12 @@ class WhisperServerManager extends EventEmitter {const generation = this.startGeneration;const modelPath = this.modelPath;+ // Where this request's output starts in the server's stderr: a crash is+ // explained by what the failing request printed, not by earlier requests+ const stderrMark = {+ processInfo: this._lastProcessInfo,+ offset: this._lastProcessInfo?.().stderr.length ?? 0,+ };try {return await this._postInference(body, boundary, signal);@@ -935,7 +948,14 @@ class WhisperServerManager extends EventEmitter {// A cancel is not a server failure: rethrow before the retry/CPU-fallback// logic so it never triggers a server restart.if (err?.name === "AbortError") throw err;- return await this._retryAfterRequestFailure(err, body, boundary, generation, modelPath);+ return await this._retryAfterRequestFailure(+ err,+ body,+ boundary,+ generation,+ modelPath,+ stderrMark+ );}}@@ -1019,7 +1039,7 @@ class WhisperServerManager extends EventEmitter {});}- async _retryAfterRequestFailure(err, body, boundary, generation, modelPath) {+ async _retryAfterRequestFailure(err, body, boundary, generation, modelPath, stderrMark) {if (!err?.isConnectionError || this.isRemote || this._stopRequested) throw err;if (this.startGeneration === generation) {@@ -1039,7 +1059,7 @@ class WhisperServerManager extends EventEmitter {processExited,})) {- return await this._fallbackToCpuAndRetry(body, boundary, modelPath);+ return await this._fallbackToCpuAndRetry(body, boundary, modelPath, stderrMark);}if (this.startGeneration === generation) throw err;}@@ -1073,20 +1093,30 @@ class WhisperServerManager extends EventEmitter {}const exited = await this._waitForProcessExit(PROCESS_EXIT_WAIT_MS);if (!exited || this._stopRequested) throw retryErr;- return await this._fallbackToCpuAndRetry(body, boundary, modelPath);+ return await this._fallbackToCpuAndRetry(body, boundary, modelPath, stderrMark);}}- async _fallbackToCpuAndRetry(body, boundary, modelPath) {+ async _fallbackToCpuAndRetry(body, boundary, modelPath, stderrMark) {const backend = this.useCuda ? "cuda" : "vulkan";+ // Read the crashed server's output now (the CPU start below replaces it),+ // from where the failing request began. A peer's replacement server is a+ // different process, so its output is read whole.+ const processInfo = this._lastProcessInfo();+ const offset = stderrMark.processInfo === this._lastProcessInfo ? stderrMark.offset : 0;+ const reason = extractWhisperGpuFailureReason({+ ...processInfo,+ stderr: processInfo.stderr.slice(offset),+ });debugLogger.warn(`${backend} whisper-server died during transcription, falling back to CPU`, {port: this.port,model: modelPath ? path.basename(modelPath) : null,+ reason,});await this.start(modelPath, { ...this.lastStartOptions, useCuda: false, useVulkan: false });this.gpuFallbackActive = true;// Emit only once the CPU server is up — the notification tells the user CPU is in use- this.emit(backend === "cuda" ? "cuda-fallback" : "gpu-fallback");+ this.emit(backend === "cuda" ? "cuda-fallback" : "gpu-fallback", { reason });return await this._postInference(body, boundary);}diff --git a/src/types/electron.ts b/src/types/electron.tsindex ada25ccc6..3fe08da65 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -755,6 +755,8 @@ export interface CudaWhisperStatus {gpuInfo: GpuInfo;/** CUDA fell back to CPU on this machine and stays off until retried. */gpuFailed?: boolean;+ /** The whisper-server error line saved with that failure; null when none was readable. */+ gpuFailReason?: string | null;/** An older release installed the pack and this version can't use it. */needsUpdate?: boolean;}@@ -766,6 +768,8 @@ export interface VulkanWhisperStatus {hasNvidiaGpu: boolean;/** Vulkan fell back to CPU on this machine and stays off until retried. */gpuFailed?: boolean;+ /** The whisper-server error line saved with that failure; null when none was readable. */+ gpuFailReason?: string | null;/** An older release installed the pack and this version can't use it. */needsUpdate?: boolean;}diff --git a/test/components/directionalContentPolicy.test.js b/test/components/directionalContentPolicy.test.jsindex 8bdf6f138..6f5856476 100644--- a/test/components/directionalContentPolicy.test.js+++ b/test/components/directionalContentPolicy.test.js@@ -112,6 +112,7 @@ test("technical output values remain LTR inside an Arabic document", () => {"src/components/onboarding/RequiredModelDownloadStep.tsx",/<span\s+dir="ltr"[\s\S]*?\{info\?\.name \?\? modelId\}/,],+ ["src/components/TranscriptionModelPicker.tsx", /<p\s+dir="ltr"[^>]*>\s*\{gpuFailReason\}/],];for (const [file, pattern] of expectations) {diff --git a/test/components/gpuFailureReasonCard.test.js b/test/components/gpuFailureReasonCard.test.jsnew file mode 100644index 000000000..371da303e--- /dev/null+++ b/test/components/gpuFailureReasonCard.test.js@@ -0,0 +1,217 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const React = require("react");+const path = require("node:path");+const { createRoot } = require("react-dom/client");+const {+ createRendererServer,+ installBrowserGlobals,+ installHookDom,+} = require("../lib/rendererTestHarness");++// The GPU card in the transcription model picker shows the whisper-server error+// line that main saved with a GPU->CPU fallback (#1736).+const DEVICE_LOST = "vk::PhysicalDevice::createDevice: ErrorDeviceLost";+const OUT_OF_DEVICE_MEMORY = "vk::Device::allocateMemory: ErrorOutOfDeviceMemory";+const KERNEL_IMAGE = "CUDA error: no kernel image is available for execution on the device";+const noop = () => {};++const vulkanPack = (overrides = {}) => ({+ downloaded: true,+ downloading: false,+ vulkan: { available: true },+ hasNvidiaGpu: false,+ gpuFailed: false,+ gpuFailReason: null,+ ...overrides,+});++const cudaPack = (overrides = {}) => ({+ downloaded: false,+ downloading: false,+ path: null,+ gpuInfo: { hasNvidiaGpu: false },+ gpuFailed: false,+ gpuFailReason: null,+ ...overrides,+});++function findElement(node, predicate) {+ if (Array.isArray(node)) {+ for (const child of node) {+ const found = findElement(child, predicate);+ if (found) return found;+ }+ return null;+ }+ if (!node || typeof node !== "object") return null;+ if (predicate(node)) return node;+ return findElement(node.props?.children, predicate);+}++const isFailedCard = (node) => String(node.props?.className ?? "").includes("border-warning/40");+const hasText = (text) => (node) => node.props?.children === text;++// Lets the picker's IPC reads (status on mount, re-read after a fallback) land+function settle() {+ return React.act(async () => {+ await new Promise((resolve) => setTimeout(resolve, 20));+ });+}++async function mountPicker(t, vulkanStatus, cudaStatus = cudaPack()) {+ installBrowserGlobals(t, {+ window: { location: { search: "" }, electronAPI: { getPlatform: () => "win32" } },+ });+ const vite = await createRendererServer(t, {+ cachePrefix: "openwhispr-gpu-failure-reason-",+ resolveAlias: { "@": path.resolve(__dirname, "../../src") },+ });+ const container = installHookDom(t);+ const statuses = { cuda: cudaStatus, vulkan: vulkanStatus };+ const fallbackListeners = { cuda: [], vulkan: [] };+ Object.assign(globalThis.window.electronAPI, {+ checkParakeetInstallation: async () => ({ supported: true }),+ listParakeetModels: async () => ({ success: true, models: [] }),+ listWhisperModels: async () => ({ success: true, models: [] }),+ onWhisperDownloadProgress: () => noop,+ onParakeetDownloadProgress: () => noop,+ getCudaWhisperStatus: async () => statuses.cuda,+ getVulkanWhisperStatus: async () => statuses.vulkan,+ whisperServerStatus: async () => ({ gpuAccelerated: false }),+ onCudaFallbackNotification: (callback) => {+ fallbackListeners.cuda.push(callback);+ return noop;+ },+ onGpuFallbackNotification: (callback) => {+ fallbackListeners.vulkan.push(callback);+ return noop;+ },+ });+ const { default: Picker } = await vite.ssrLoadModule("/components/TranscriptionModelPicker.tsx");+ const { ToastContext } = await vite.ssrLoadModule("/components/ui/useToast.ts");+ let tree;+ function Harness() {+ tree = Picker({+ selectedLocalProvider: "whisper",+ selectedLocalModel: "base",+ useLocalWhisper: true,+ onLocalModelSelect: noop,+ onModeChange: noop,+ });+ return null;+ }+ const root = createRoot(container);+ await React.act(async () => {+ root.render(+ React.createElement(+ ToastContext.Provider,+ { value: { toast: noop } },+ React.createElement(Harness)+ )+ );+ });+ await settle();+ return {+ find: (predicate) => findElement(tree, predicate),+ // Main has already saved the new failure when it sends this notification+ fireFallback: async (backend, nextStatus) => {+ statuses[backend] = nextStatus;+ await React.act(async () => {+ for (const listener of fallbackListeners[backend]) listener();+ });+ await settle();+ },+ unmount: () => React.act(async () => root.unmount()),+ };+}++test("the failed card shows the saved reason as its own left-to-right line", async (t) => {+ const picker = await mountPicker(t, vulkanPack({ gpuFailed: true, gpuFailReason: DEVICE_LOST }));+ try {+ const card = picker.find(isFailedCard);+ const line = findElement(card, hasText(DEVICE_LOST));+ assert.ok(line, "the reason is on the failed card");+ assert.equal(line.props.dir, "ltr");+ } finally {+ await picker.unmount();+ }+});++test("a failure saved before this change renders the card exactly as before", async (t) => {+ const picker = await mountPicker(t, vulkanPack({ gpuFailed: true, gpuFailReason: undefined }));+ try {+ const card = picker.find(isFailedCard);+ assert.ok(card, "the failed card still shows");+ assert.equal(+ findElement(card, (node) => node.props?.dir === "ltr"),+ null,+ "no empty line"+ );+ } finally {+ await picker.unmount();+ }+});++test("a reason is never shown while the pack is not marked failed", async (t) => {+ const stale = "a reason left over from an old failure";+ const picker = await mountPicker(t, vulkanPack({ gpuFailReason: stale }));+ try {+ assert.equal(picker.find(isFailedCard), null);+ assert.equal(picker.find(hasText(stale)), null);+ } finally {+ await picker.unmount();+ }+});++test("a live fallback re-reads the status: the new reason shows and replaces the old one", async (t) => {+ const picker = await mountPicker(t, vulkanPack());+ try {+ assert.equal(picker.find(isFailedCard), null);++ await picker.fireFallback(+ "vulkan",+ vulkanPack({ gpuFailed: true, gpuFailReason: DEVICE_LOST })+ );+ assert.ok(picker.find(hasText(DEVICE_LOST)), "shown without reopening settings");++ // A later failure (e.g. after Retry) replaces the line; the old reason never lingers+ await picker.fireFallback(+ "vulkan",+ vulkanPack({ gpuFailed: true, gpuFailReason: OUT_OF_DEVICE_MEMORY })+ );+ assert.ok(picker.find(hasText(OUT_OF_DEVICE_MEMORY)));+ assert.equal(picker.find(hasText(DEVICE_LOST)), null);+ } finally {+ await picker.unmount();+ }+});++test("a Vulkan fallback shows as failed even while the card shows the installed CUDA pack", async (t) => {+ // Both packs installed on an NVIDIA machine: the card shows CUDA, but main ran+ // Vulkan (CUDA opted out with WHISPER_CUDA_ENABLED=false, or failed before)+ const installedCuda = { downloaded: true, gpuInfo: { hasNvidiaGpu: true, cudaSupported: true } };+ const picker = await mountPicker(t, vulkanPack({ hasNvidiaGpu: true }), cudaPack(installedCuda));+ try {+ assert.equal(picker.find(isFailedCard), null);++ await picker.fireFallback(+ "vulkan",+ vulkanPack({ hasNvidiaGpu: true, gpuFailed: true, gpuFailReason: DEVICE_LOST })+ );++ const card = picker.find(isFailedCard);+ assert.ok(card, "the fallback that just happened stays visible");+ assert.ok(findElement(card, hasText(DEVICE_LOST)), "with the reason of the pack that failed");++ // A CUDA fallback reads CUDA's own reason+ await picker.fireFallback(+ "cuda",+ cudaPack({ ...installedCuda, gpuFailed: true, gpuFailReason: KERNEL_IMAGE })+ );+ assert.ok(picker.find(hasText(KERNEL_IMAGE)));+ assert.equal(picker.find(hasText(DEVICE_LOST)), null);+ } finally {+ await picker.unmount();+ }+});diff --git a/test/helpers/harness/whisperServerStderr.js b/test/helpers/harness/whisperServerStderr.jsnew file mode 100644index 000000000..9c526fa3b--- /dev/null+++ b/test/helpers/harness/whisperServerStderr.js@@ -0,0 +1,63 @@+// whisper-server stderr, rebuilt line by line from the pinned OpenWhispr/whisper.cpp+// tag (src/whisper.cpp, ggml/src/ggml-vulkan, ggml/src/ggml-cuda, ggml-backend.cpp,+// examples/server/server.cpp) and the logs on #1340 and #1606.++// #1340: RX 9070 XT on the AMD proprietary driver (Windows). The Vulkan backend+// prints its device banner when whisper-server registers it, then createDevice+// throws inside model load; whisper.cpp logs the exception and the server exits+// 3. The cause sits far past the first 200 characters.+const VULKAN_DEVICE_LOST_STDERR = [+ "ggml_vulkan: Found 1 Vulkan devices:",+ "ggml_vulkan: 0 = AMD Radeon RX 9070 XT (AMD proprietary driver) | uma: 0 | fp16: 1 | bf16: 1 | warp size: 64 | shared memory: 32768 | int dot: 1 | matrix cores: KHR_coopmat",+ "whisper_init_from_file_with_params_no_state: loading model from 'C:\\Users\\Mika\\.cache\\openwhispr\\whisper-models\\ggml-large-v3-turbo.bin'",+ "whisper_init_with_params_no_state: use gpu = 1",+ "whisper_init_with_params_no_state: flash attn = 1",+ "whisper_init_with_params_no_state: gpu_device = 0",+ "whisper_model_load: loading model",+ "whisper_model_load: n_vocab = 51866",+ "whisper_model_load: n_audio_ctx = 1500",+ "whisper_model_load: n_audio_state = 1280",+ "whisper_model_load: n_mels = 128",+ "whisper_model_load: type = 5 (large v3)",+ "whisper_model_load: adding 1609 extra tokens",+ "whisper_model_load: n_langs = 100",+ "whisper_init_with_params_no_state: exception during model load: vk::PhysicalDevice::createDevice: ErrorDeviceLost",+ "whisper_init_with_params_no_state: failed to load model",+ "error: failed to initialize whisper context",+ "",+].join("\r\n");++// A CUDA pack on a card below its kernel floor (the Maxwell case the model+// picker steers to Vulkan): the server starts, then aborts at the first+// kernel launch, mid-transcription.+const CUDA_KERNEL_IMAGE_STDERR = [+ "ggml_cuda_init: found 1 CUDA devices (Total VRAM: 4096 MiB):",+ " Device 0: NVIDIA GeForce GTX 970, compute capability 5.2, VMM: yes, VRAM: 4096 MiB",+ "whisper_init_from_file_with_params_no_state: loading model from '/home/ana/.cache/openwhispr/whisper-models/ggml-base.bin'",+ "whisper_backend_init_gpu: using CUDA0 backend",+ "whisper_model_load: CUDA0 total size = 147.37 MB",+ "CUDA error: no kernel image is available for execution on the device",+ " current device: 0, in function ggml_cuda_compute_forward at /home/runner/work/whisper.cpp/whisper.cpp/ggml/src/ggml-cuda/ggml-cuda.cu:2503",+ " err",+ "/home/runner/work/whisper.cpp/whisper.cpp/ggml/src/ggml-cuda/ggml-cuda.cu:88: CUDA error",+ "",+].join("\n");++// A model larger than free VRAM: the weight buffer allocation fails and the+// load then aborts on the unallocated tensor.+const CUDA_OUT_OF_MEMORY_STDERR = [+ "ggml_cuda_init: found 1 CUDA devices (Total VRAM: 2048 MiB):",+ " Device 0: NVIDIA GeForce GTX 1050, compute capability 6.1, VMM: yes, VRAM: 2048 MiB",+ "whisper_init_from_file_with_params_no_state: loading model from 'C:\\Users\\Ana\\.cache\\openwhispr\\whisper-models\\ggml-large-v3-turbo.bin'",+ "whisper_backend_init_gpu: using CUDA0 backend",+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 1533.14 MiB on device 0: cudaMalloc failed: out of memory",+ "alloc_tensor_range: failed to allocate CUDA0 buffer of size 1607598080",+ 'D:\\a\\whisper.cpp\\whisper.cpp\\ggml\\src\\ggml-backend.cpp:327: GGML_ASSERT(buf != NULL && "tensor buffer not set") failed',+ "",+].join("\r\n");++module.exports = {+ VULKAN_DEVICE_LOST_STDERR,+ CUDA_KERNEL_IMAGE_STDERR,+ CUDA_OUT_OF_MEMORY_STDERR,+};diff --git a/test/helpers/whisperCudaRequestFallback.test.js b/test/helpers/whisperCudaRequestFallback.test.jsindex c8aa88789..0f76cc4b2 100644--- a/test/helpers/whisperCudaRequestFallback.test.js+++ b/test/helpers/whisperCudaRequestFallback.test.js@@ -7,6 +7,7 @@ const {shouldFallbackToCpuAfterRequestError,shouldRetryAfterServerReplaced,} = require("../../src/helpers/whisperServer");+const { CUDA_KERNEL_IMAGE_STDERR } = require("./harness/whisperServerStderr");const base = {isConnectionError: true,@@ -106,6 +107,7 @@ function createManager(port, { useCuda, useVulkan = false }) {manager.useVulkan = useVulkan;manager.canConvert = true;manager.process = {};+ manager._lastProcessInfo = () => ({ stderr: "", exitCode: null, signal: null });manager.modelPath = "/tmp/model.bin";manager.lastStartOptions = { useCuda, useVulkan };manager._convertToWav = async (buffer) => buffer;@@ -193,6 +195,82 @@ test("falls back to CPU and emits gpu-fallback when a Vulkan server dies mid-reqassert.equal(startCalls[0].options.useVulkan, false);});+test("the mid-transcription fallback carries the crashed server's error line", async (t) => {+ let manager;+ let requestCount = 0;+ // An earlier request's output; the abort is printed while serving this one+ let stderr = "whisper_print_timings: total time = 640.12 ms\n";++ const { server, port } = await startServer((req, res) => {+ requestCount += 1;+ if (requestCount === 1) {+ stderr += CUDA_KERNEL_IMAGE_STDERR;+ manager.process = null;+ req.socket.destroy();+ return;+ }+ res.writeHead(200, { "Content-Type": "application/json" });+ res.end(JSON.stringify({ text: "hello" }));+ });+ t.after(() => server.close());++ manager = createManager(port, { useCuda: true });+ // What _doStart records for the server it spawned: its output and how it ended+ manager._lastProcessInfo = () => ({ stderr, exitCode: null, signal: "SIGABRT" });+ manager.start = async () => {+ // The CPU restart spawns a new process, so the reason must be read before it+ manager._lastProcessInfo = () => ({ stderr: "", exitCode: null, signal: null });+ manager.useCuda = false;+ manager.ready = true;+ };++ const events = [];+ manager.on("cuda-fallback", (payload) => events.push(payload));++ const result = await manager.transcribe(Buffer.from("audio"));++ assert.equal(result.text, "hello");+ assert.deepEqual(events, [+ { reason: "CUDA error: no kernel image is available for execution on the device" },+ ]);+});++test("the mid-transcription reason ignores what earlier requests printed", async (t) => {+ let manager;+ let requestCount = 0;+ // A bad earlier request left an error line in the long-running server's stderr+ const stderr =+ "error: failed to read audio data\nwhisper_print_timings: total time = 9.12 ms\n";++ const { server, port } = await startServer((req, res) => {+ requestCount += 1;+ if (requestCount === 1) {+ // A driver crash: the process dies without printing anything+ manager.process = null;+ req.socket.destroy();+ return;+ }+ res.writeHead(200, { "Content-Type": "application/json" });+ res.end(JSON.stringify({ text: "hello" }));+ });+ t.after(() => server.close());++ manager = createManager(port, { useCuda: false, useVulkan: true });+ manager._lastProcessInfo = () => ({ stderr, exitCode: 3221225477, signal: null });+ manager.start = async () => {+ manager._lastProcessInfo = () => ({ stderr: "", exitCode: null, signal: null });+ manager.useVulkan = false;+ manager.ready = true;+ };++ const events = [];+ manager.on("gpu-fallback", (payload) => events.push(payload));++ await manager.transcribe(Buffer.from("audio"));++ assert.deepEqual(events, [{ reason: "exit code 3221225477" }]);+});+test("falls back to CPU when a peer's replacement is another doomed CUDA server", async (t) => {let manager;let requestCount = 0;@@ -204,6 +282,12 @@ test("falls back to CPU when a peer's replacement is another doomed CUDA server"manager.process = null;manager.startGeneration += 1;manager.ready = true;+ // The replacement is a new process: its output is read from the start+ manager._lastProcessInfo = () => ({+ stderr: CUDA_KERNEL_IMAGE_STDERR,+ exitCode: null,+ signal: "SIGABRT",+ });req.socket.destroy();return;}@@ -219,6 +303,9 @@ test("falls back to CPU when a peer's replacement is another doomed CUDA server"t.after(() => server.close());manager = createManager(port, { useCuda: true });+ // The crashed server's long output, longer than all of the replacement's+ const crashedStderr = "whisper_print_timings: total time = 640.12 ms\n".repeat(40);+ manager._lastProcessInfo = () => ({ stderr: crashedStderr, exitCode: null, signal: null });const startCalls = [];manager.start = async (modelPath, options) => {@@ -227,16 +314,16 @@ test("falls back to CPU when a peer's replacement is another doomed CUDA server"manager.ready = true;};- let fallbackEvents = 0;- manager.on("cuda-fallback", () => {- fallbackEvents += 1;- });+ const fallbackEvents = [];+ manager.on("cuda-fallback", (payload) => fallbackEvents.push(payload));const result = await manager.transcribe(Buffer.from("audio"));assert.equal(result.text, "hello");assert.equal(requestCount, 3);- assert.equal(fallbackEvents, 1);+ assert.deepEqual(fallbackEvents, [+ { reason: "CUDA error: no kernel image is available for execution on the device" },+ ]);assert.equal(startCalls.length, 1);assert.equal(startCalls[0].options.useCuda, false);});diff --git a/test/helpers/whisperGpuFailureReason.test.js b/test/helpers/whisperGpuFailureReason.test.jsnew file mode 100644index 000000000..2b14ad366--- /dev/null+++ b/test/helpers/whisperGpuFailureReason.test.js@@ -0,0 +1,201 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const { parse: parseDotenv } = require("dotenv");++const {+ extractWhisperGpuFailureReason: extractReason,+} = require("../../src/helpers/whisperGpuFailureReason");+const {+ VULKAN_DEVICE_LOST_STDERR,+ CUDA_KERNEL_IMAGE_STDERR,+ CUDA_OUT_OF_MEMORY_STDERR,+} = require("./harness/whisperServerStderr");++const DEVICE_LOST = "vk::PhysicalDevice::createDevice: ErrorDeviceLost";++test("#1340: finds the createDevice error that a 200-character slice of stderr cuts off", () => {+ // The warn log kept stderrBuffer.slice(0, 200): all banner, no cause+ assert.equal(VULKAN_DEVICE_LOST_STDERR.slice(0, 200).includes("ErrorDeviceLost"), false);+ assert.equal(extractReason({ stderr: VULKAN_DEVICE_LOST_STDERR, exitCode: 3 }), DEVICE_LOST);+});++test("CUDA: the kernel-image error wins over the abort lines after it", () => {+ assert.equal(+ extractReason({ stderr: CUDA_KERNEL_IMAGE_STDERR, signal: "SIGABRT" }),+ "CUDA error: no kernel image is available for execution on the device"+ );+});++test("CUDA: an out-of-memory load reports the first failed allocation, not the assert", () => {+ assert.equal(+ extractReason({ stderr: CUDA_OUT_OF_MEMORY_STDERR, exitCode: 3 }),+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 1533.14 MiB on device 0: cudaMalloc failed: out of memory"+ );+});++test("reads an uncaught Vulkan exception from its what() line", () => {+ const stderr = [+ "terminate called after throwing an instance of 'vk::DeviceLostError'",+ " what(): vk::Queue::submit: ErrorDeviceLost",+ ].join("\n");+ assert.equal(extractReason({ stderr, signal: "SIGABRT" }), "vk::Queue::submit: ErrorDeviceLost");+});++test("a warning the backend recovered from never outranks the error that killed it", () => {+ // ggml-vulkan logs the first two and carries on with a CPU-side buffer; the+ // fence wait in ggml_vk_wait_for_fence then prints the fatal line and exits+ const stderr = [+ "ggml_vulkan: Failed to allocate pinned memory (vk::Device::allocateMemory: ErrorOutOfHostMemory)",+ "WARNING: failed to allocate 512.00 MB of pinned memory",+ "whisper_print_timings: total time = 812.44 ms",+ "ggml_vulkan: error ErrorDeviceLost at D:\\a\\whisper.cpp\\ggml\\src\\ggml-vulkan\\ggml-vulkan.cpp:2209",+ ].join("\r\n");+ assert.equal(+ extractReason({ stderr, exitCode: 1 }),+ "ggml_vulkan: error ErrorDeviceLost at D:\\a\\whisper.cpp\\ggml\\src\\ggml-vulkan\\ggml-vulkan.cpp:2209"+ );+});++test("a VK_CHECK exit names the Vulkan call that failed", () => {+ const stderr =+ "ggml_vulkan: ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX) error ErrorDeviceLost at ggml-vulkan.cpp:2200\n";+ assert.equal(+ extractReason({ stderr, exitCode: 1 }),+ "ggml_vulkan: ctx->device->device.waitForFences({ ctx->almost_ready_fence }, true, UINT64_MAX) error ErrorDeviceLost at ggml-vulkan.cpp:2200"+ );+});++test("recovered warnings alone are not a cause: the exit is reported instead", () => {+ const stderr = [+ "ggml_vulkan: Failed to allocate pinned memory (vk::Device::allocateMemory: ErrorOutOfHostMemory)",+ "ggml_vulkan: Failed getMemoryHostPointerPropertiesEXT (vk::Device::getMemoryHostPointerPropertiesEXT: ErrorInvalidExternalHandle)",+ "ggml_vulkan: Failed ggml_vk_create_buffer (vk::Device::allocateMemory: ErrorOutOfDeviceMemory)",+ "ggml_cuda_host_malloc: failed to allocate 512.00 MiB of pinned memory: out of memory",+ ].join("\n");+ assert.equal(extractReason({ stderr, exitCode: 3221225477 }), "exit code 3221225477");+});++test("a ggml abort with no error word in it is reported, not the exit it caused", () => {+ // ggml_abort prints "<file>:<line>: <message>", then a backtrace, then aborts.+ // The paths are the build machine's, so no home folder is redacted from them.+ const preallocation =+ "D:\\a\\whisper.cpp\\whisper.cpp\\ggml\\src\\ggml-vulkan\\ggml-vulkan.cpp:8412: Requested preallocation size is too large";+ assert.equal(+ extractReason({+ stderr: `ggml_vulkan: Found 1 Vulkan devices:\n${preallocation}\n`,+ exitCode: 3,+ homeDir: null,+ }),+ preallocation+ );+ // It mentions pinned memory but, unlike the warnings above, it is fatal+ const nonPinned =+ "/home/runner/work/whisper.cpp/whisper.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp:7507: Asynchronous write to non-pinned memory not supported";+ assert.equal(+ extractReason({ stderr: `${nonPinned}\n`, signal: "SIGABRT", homeDir: null }),+ nonPinned+ );+});++test("the Vulkan loader's notes about drivers it skipped are not a cause", () => {+ // Common with several drivers installed; the loader skips that one and carries on+ const stderr = [+ "ERROR: [Loader Message] Code 0 : loader_scanned_icd_add: Could not get 'vkCreateInstance' via 'vk_icdGetInstanceProcAddr' for ICD libGLX_nvidia.so.0",+ "ggml_vulkan: Found 1 Vulkan devices:",+ ].join("\n");+ assert.equal(extractReason({ stderr, signal: "SIGSEGV" }), "terminated by SIGSEGV");+ assert.equal(extractReason({ stderr, timeoutMs: 120000 }), "startup timed out after 120 s");+});++test("an unrecognised error line beats the generic load-failure lines after it", () => {+ const stderr = [+ "ggml_vulkan: Found 1 Vulkan devices:",+ "ggml_vulkan: Error: Vulkan 1.2 required.",+ "whisper_init_with_params_no_state: failed to load model",+ "error: failed to initialize whisper context",+ ].join("\n");+ assert.equal(extractReason({ stderr, exitCode: 3 }), "ggml_vulkan: Error: Vulkan 1.2 required.");+});++test("with only the generic load-failure lines, the first one is kept", () => {+ const stderr = [+ "whisper_model_load: invalid model data (bad magic)",+ "whisper_init_with_params_no_state: failed to load model",+ "error: failed to initialize whisper context",+ ].join("\n");+ assert.equal(+ extractReason({ stderr, exitCode: 3 }),+ "whisper_init_with_params_no_state: failed to load model"+ );+});++test("without an error line, reports how the process ended", () => {+ const banner = "ggml_vulkan: Found 1 Vulkan devices:\n";+ // A driver crash or a missing DLL on Windows prints nothing and exits with an NTSTATUS code+ assert.equal(extractReason({ stderr: banner, exitCode: 3221225477 }), "exit code 3221225477");+ assert.equal(extractReason({ stderr: banner, signal: "SIGSEGV" }), "terminated by SIGSEGV");+ assert.equal(+ extractReason({ stderr: banner, timeoutMs: 120000 }),+ "startup timed out after 120 s"+ );+ assert.equal(extractReason({ stderr: "", exitCode: 3, timeoutMs: 120000 }), "exit code 3");+ assert.equal(extractReason({}), null);+});++test("returns one line, capped at 240 characters", () => {+ const long = extractReason({ stderr: `error: ${"x".repeat(500)}` });+ assert.equal(long.length, 240);+ assert.ok(long.endsWith("…"));+ assert.equal(extractReason({ stderr: "error:\tfirst \u0007 second" }), "error: first second");+});++test("replaces the user's home folder with ~ in any letter case or slash style", () => {+ const homeDir = "C:\\Users\\Mika";+ assert.equal(+ extractReason({ stderr: "error: failed to read 'c:\\users\\MIKA\\a.wav'", homeDir }),+ "error: failed to read '~\\a.wav"+ );+ assert.equal(+ extractReason({ stderr: "error: failed to read 'C:/Users/Mika/a.wav'", homeDir }),+ "error: failed to read '~/a.wav"+ );+});++test("the reason survives the raw KEY=value line it is saved as in .env", () => {+ // EnvironmentManager writes KEY=value unquoted. dotenv reads "#" as a comment,+ // and a value that starts with a quote runs on to the next line ending in that+ // quote, swallowing the keys between. Each later line ends in a quote to prove it.+ const laterLines = "WHISPER_THREADS=4\nA=1'\nB=2\"\nC=3`\n";+ for (const line of [+ "error: bad #5",+ "'error: single quoted'",+ '"error: C:\\new\\folder',+ "`error` in backticks",+ "error: a=b: c",+ "error: $HOME",+ ]) {+ const reason = extractReason({ stderr: line, homeDir: null });+ const env = parseDotenv(`WHISPER_GPU_FAILED_REASON_VULKAN=${reason}\n${laterLines}`);+ assert.equal(env.WHISPER_GPU_FAILED_REASON_VULKAN, reason, line);+ assert.equal(env.WHISPER_THREADS, "4", `${line} swallowed the next key`);+ }+});++test("the reason never closes a quote that an earlier .env line left open", () => {+ // A backtick hotkey is saved as DICTATION_KEY=`. A reason whose only backtick+ // ends the line would close that quote, and the hotkey would swallow every key+ // down to the reason, the failure flag included.+ for (const quote of ["'", '"', "`"]) {+ const reason = extractReason({+ stderr: `exception during model load: ${quote}vkCreateDevice failed${quote}`,+ homeDir: null,+ });+ const env = parseDotenv(+ `DICTATION_KEY=${quote}\nWHISPER_GPU_FAILED=vulkan\nWHISPER_GPU_FAILED_REASON_VULKAN=${reason}\n`+ );+ assert.equal(reason, "vkCreateDevice failed", quote);+ assert.equal(env.DICTATION_KEY, quote, `${quote}: the hotkey swallowed the keys after it`);+ assert.equal(env.WHISPER_GPU_FAILED, "vulkan", quote);+ assert.equal(env.WHISPER_GPU_FAILED_REASON_VULKAN, reason, quote);+ }+});diff --git a/test/helpers/whisperGpuFailureRecordIpc.test.js b/test/helpers/whisperGpuFailureRecordIpc.test.jsnew file mode 100644index 000000000..0f32a4796--- /dev/null+++ b/test/helpers/whisperGpuFailureRecordIpc.test.js@@ -0,0 +1,253 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const Module = require("node:module");+const fs = require("node:fs");+const os = require("node:os");+const path = require("node:path");+const { EventEmitter } = require("node:events");++// Runs the real GPU-failure handlers from ipcHandlers.js outside Electron. The+// saved reason must follow WHISPER_GPU_FAILED everywhere the flag is set,+// cleared or reported (#1736).+const handlersModulePath = require.resolve("../../src/helpers/ipcHandlers");+const originalLoad = Module._load;+const handlers = new Map();+const broadcasts = [];+// A private userData, so nothing (e.g. tokenStore's auth-token.bin) is read from a shared temp dir+const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "whisper-gpu-ipc-"));+const electronStub = {+ app: {+ getPath: () => userDataDir,+ getName: () => "test",+ getVersion: () => "0.0.0",+ isPackaged: false,+ on() {},+ requestSingleInstanceLock: () => true,+ },+ ipcMain: {+ handle: (channel, handler) => handlers.set(channel, handler),+ on() {},+ removeHandler() {},+ },+ net: { fetch: async () => ({ ok: true, status: 200, json: async () => ({}) }) },+ BrowserWindow: class {+ static getAllWindows() {+ return [+ {+ isDestroyed: () => false,+ webContents: { send: (channel, data) => broadcasts.push({ channel, data }) },+ },+ ];+ }+ static fromWebContents() {+ return null;+ }+ },+ shell: {},+ dialog: {},+ screen: { getPrimaryDisplay: () => ({ workAreaSize: { width: 0, height: 0 } }) },+ systemPreferences: { getMediaAccessStatus: () => "granted" },+ session: { fromPartition: () => ({}) },+ clipboard: {},+ nativeImage: {},+ globalShortcut: {},+ utilityProcess: {},+ MessageChannelMain: class {},+};++Module._load = function loadWithMocks(request, parent, isMain) {+ if (request === "electron") return electronStub;+ // Never reach the OS keychain (tokenStore and environment.js load secretCrypto)+ if (request === "./secretCrypto") return { isAvailable: () => false };+ if (parent?.filename === handlersModulePath) {+ if (request === "./debugLogger") return new Proxy({}, { get: () => () => {} });+ // The status handlers probe the machine's GPUs; the answer is irrelevant here+ if (request === "../utils/gpuDetection") {+ return { detectNvidiaGpu: async () => ({ hasNvidiaGpu: false }) };+ }+ if (request === "../utils/vulkanDetection") {+ return { detectVulkanGpu: async () => ({ available: true }) };+ }+ }+ return originalLoad.call(this, request, parent, isMain);+};+test.after(() => {+ Module._load = originalLoad;+ fs.rmSync(userDataDir, { recursive: true, force: true });+});++const FAILURE_KEYS = [+ "WHISPER_GPU_FAILED",+ "WHISPER_GPU_FAILED_REASON_CUDA",+ "WHISPER_GPU_FAILED_REASON_VULKAN",+];+const ENV_KEYS = [+ ...FAILURE_KEYS,+ "WHISPER_CUDA_ENABLED",+ "WHISPER_VULKAN_ENABLED",+ "WHISPER_VULKAN_DEVICE",+];+const savedEnv = {};+test.beforeEach(() => {+ for (const key of ENV_KEYS) {+ savedEnv[key] = process.env[key];+ delete process.env[key];+ }+ broadcasts.length = 0;+});+test.afterEach(() => {+ for (const key of ENV_KEYS) {+ if (savedEnv[key] === undefined) delete process.env[key];+ else process.env[key] = savedEnv[key];+ }+});++const DEVICE_LOST = "vk::PhysicalDevice::createDevice: ErrorDeviceLost";+const KERNEL_IMAGE = "CUDA error: no kernel image is available for execution on the device";++function anything() {+ return new Proxy(function () {}, {+ get: (_target, property) => {+ if (property === Symbol.toPrimitive || property === "toString") return () => "";+ if (property === "then") return undefined;+ return anything();+ },+ apply: () => anything(),+ });+}++function createHandlers() {+ const IPCHandlers = require(handlersModulePath);+ const serverManager = new EventEmitter();+ serverManager.isRemote = false;+ // What each .env rewrite would persist, captured at the moment of the write+ const envWrites = [];+ const target = Object.assign(Object.create(IPCHandlers.prototype), {+ environmentManager: {+ saveAllKeysToEnvFile: async () => {+ envWrites.push(Object.fromEntries(FAILURE_KEYS.map((key) => [key, process.env[key]])));+ return { success: true };+ },+ },+ whisperManager: {+ serverManager,+ currentServerModel: null,+ stopServer: async () => {},+ restartServerWithGpuPreference: async () => ({ success: true, restarted: false }),+ },+ whisperCudaManager: {+ isDownloaded: () => true,+ needsUpdate: () => false,+ isDownloading: () => false,+ getCudaBinaryPath: () => null,+ download: async () => {},+ delete: async () => ({ success: true }),+ },+ whisperVulkanManager: {+ isDownloaded: () => true,+ needsUpdate: () => false,+ isDownloading: () => false,+ download: async () => {},+ delete: async () => ({ success: true, deletedCount: 1 }),+ },+ });+ const context = new Proxy(target, {+ get: (value, property) => (property in value ? value[property] : anything()),+ });+ IPCHandlers.prototype.setupHandlers.call(context);+ context._attachWhisperServerListeners(serverManager);+ const invoke = (channel) => handlers.get(channel)({ sender: { isDestroyed: () => true } });+ return { serverManager, invoke, envWrites };+}++test("a Vulkan fallback saves its reason with the flag", () => {+ const { serverManager, envWrites } = createHandlers();++ serverManager.emit("gpu-fallback", { reason: DEVICE_LOST });++ assert.equal(process.env.WHISPER_GPU_FAILED, "vulkan");+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_VULKAN, DEVICE_LOST);+ assert.deepEqual(envWrites.at(-1), {+ WHISPER_GPU_FAILED: "vulkan",+ WHISPER_GPU_FAILED_REASON_CUDA: undefined,+ WHISPER_GPU_FAILED_REASON_VULKAN: DEVICE_LOST,+ });+ assert.deepEqual(broadcasts, [{ channel: "gpu-fallback-notification", data: {} }]);+});++test("the status IPC reports each backend's own saved reason", async () => {+ const { serverManager, invoke } = createHandlers();+ serverManager.emit("gpu-fallback", { reason: DEVICE_LOST });+ serverManager.emit("cuda-fallback", { reason: KERNEL_IMAGE });++ const vulkan = await invoke("get-vulkan-whisper-status");+ const cuda = await invoke("get-cuda-whisper-status");++ assert.equal(vulkan.gpuFailed, true);+ assert.equal(vulkan.gpuFailReason, DEVICE_LOST);+ assert.equal(cuda.gpuFailed, true);+ assert.equal(cuda.gpuFailReason, KERNEL_IMAGE);+});++test("a failure with no readable reason clears the older one instead of showing it", async () => {+ const { serverManager, invoke } = createHandlers();+ process.env.WHISPER_GPU_FAILED_REASON_CUDA = KERNEL_IMAGE;++ serverManager.emit("cuda-fallback", { reason: null });++ assert.equal(process.env.WHISPER_GPU_FAILED, "cuda");+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_CUDA, undefined);+ assert.equal((await invoke("get-cuda-whisper-status")).gpuFailReason, null);+});++test("no reason is reported for a backend that is not marked failed", async () => {+ const { invoke } = createHandlers();+ // A leftover reason without its flag, e.g. from a hand-edited .env+ process.env.WHISPER_GPU_FAILED_REASON_VULKAN = DEVICE_LOST;++ const vulkan = await invoke("get-vulkan-whisper-status");++ assert.equal(vulkan.gpuFailed, false);+ assert.equal(vulkan.gpuFailReason, null);+});++test("Retry clears every saved reason with the flag", async () => {+ const { serverManager, invoke } = createHandlers();+ serverManager.emit("gpu-fallback", { reason: DEVICE_LOST });+ serverManager.emit("cuda-fallback", { reason: KERNEL_IMAGE });+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_VULKAN, DEVICE_LOST);+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_CUDA, KERNEL_IMAGE);++ await invoke("whisper-gpu-retry");++ for (const key of FAILURE_KEYS) assert.equal(process.env[key], undefined, key);+});++test("deleting or re-downloading one pack clears only that pack's reason", async () => {+ const { serverManager, invoke } = createHandlers();+ serverManager.emit("gpu-fallback", { reason: DEVICE_LOST });+ serverManager.emit("cuda-fallback", { reason: KERNEL_IMAGE });++ await invoke("delete-vulkan-whisper-binary");+ assert.equal(process.env.WHISPER_GPU_FAILED, "cuda");+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_VULKAN, undefined);+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_CUDA, KERNEL_IMAGE);++ await invoke("download-cuda-whisper-binary");+ assert.equal(process.env.WHISPER_GPU_FAILED, undefined);+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_CUDA, undefined);+});++test("deleting the CUDA pack and re-downloading Vulkan clear their reasons too", async () => {+ const { serverManager, invoke } = createHandlers();++ serverManager.emit("cuda-fallback", { reason: KERNEL_IMAGE });+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_CUDA, KERNEL_IMAGE);+ await invoke("delete-cuda-whisper-binary");+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_CUDA, undefined);++ serverManager.emit("gpu-fallback", { reason: DEVICE_LOST });+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_VULKAN, DEVICE_LOST);+ await invoke("download-vulkan-whisper-binary");+ assert.equal(process.env.WHISPER_GPU_FAILED_REASON_VULKAN, undefined);+});diff --git a/test/helpers/whisperGpuUpgradeReset.test.js b/test/helpers/whisperGpuUpgradeReset.test.jsindex e71936577..0f81e9ca2 100644--- a/test/helpers/whisperGpuUpgradeReset.test.js+++ b/test/helpers/whisperGpuUpgradeReset.test.js@@ -3,6 +3,8 @@ const assert = require("node:assert/strict");const fs = require("fs");const os = require("os");const path = require("path");+// The real parser, taken before any test stubs dotenv's loader+const { parse: parseDotenv } = require("dotenv");// Runs outside Electron: stub the app userData path and version before loading.let userDataDir = null;@@ -12,8 +14,27 @@ require.cache[require.resolve("electron")] = {exports: { app: { getPath: () => userDataDir, getVersion: () => appVersion } },};+// Never touch the OS keychain: EnvironmentManager's .env writer asks secretCrypto+// whether encryption is available, which opens the real keychain.+const secretCryptoPath = require.resolve("../../src/helpers/secretCrypto.js");+require.cache[secretCryptoPath] = {+ id: secretCryptoPath,+ filename: secretCryptoPath,+ loaded: true,+ exports: { isAvailable: () => false },+};+const { resetWhisperGpuFailureOnUpgrade } = require("../../src/helpers/whisperGpuUpgradeReset.js");+const DEVICE_LOST = "vk::PhysicalDevice::createDevice: ErrorDeviceLost";+const KERNEL_IMAGE = "CUDA error: no kernel image is available for execution on the device";+// The failure flag and the reasons saved with it (#1736) are one record+const FAILURE_KEYS = [+ "WHISPER_GPU_FAILED",+ "WHISPER_GPU_FAILED_REASON_CUDA",+ "WHISPER_GPU_FAILED_REASON_VULKAN",+];+function makeEnvManager() {const manager = { removals: [] };manager.removeKeyFromEnvFile = async (key) => {@@ -22,17 +43,52 @@ function makeEnvManager() {return manager;}+function clearFailureKeys() {+ for (const key of FAILURE_KEYS) delete process.env[key];+}+test.beforeEach(() => {userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "gpu-upgrade-reset-"));appVersion = "1.9.1";- delete process.env.WHISPER_GPU_FAILED;+ clearFailureKeys();});test.afterEach(() => {fs.rmSync(userDataDir, { recursive: true, force: true });- delete process.env.WHISPER_GPU_FAILED;+ clearFailureKeys();});+// Real EnvironmentManager (electron and secretCrypto stubbed above) with+// dotenv's loader stubbed and resourcesPath pinned, so the test owns the .env.+async function withRealEnvironmentManager(run) {+ const dotenvPath = require.resolve("dotenv");+ const originalDotenv = require.cache[dotenvPath];+ require.cache[dotenvPath] = {+ id: dotenvPath,+ filename: dotenvPath,+ loaded: true,+ exports: { config: () => ({ parsed: {} }) },+ };+ const originalResourcesPath = process.resourcesPath;+ process.resourcesPath = userDataDir;+ try {+ const EnvironmentManager = require("../../src/helpers/environment.js");+ await run(new EnvironmentManager(), path.join(userDataDir, ".env"));+ } finally {+ if (originalDotenv) require.cache[dotenvPath] = originalDotenv;+ else delete require.cache[dotenvPath];+ process.resourcesPath = originalResourcesPath;+ }+}++// Removals are queued one after another, so the last one settles after all+function trackRemovals(envManager) {+ const realRemove = envManager.removeKeyFromEnvFile.bind(envManager);+ const tracked = { last: null };+ envManager.removeKeyFromEnvFile = (key) => (tracked.last = realRemove(key));+ return tracked;+}+test("clears the remembered GPU failure exactly once per version change", () => {process.env.WHISPER_GPU_FAILED = "cuda";const envManager = makeEnvManager();@@ -67,23 +123,20 @@ test("records the running version without persisting when no failure is stored",assert.deepEqual(envManager.removals, []);});-test("reset removes only the WHISPER_GPU_FAILED line; hand-added .env lines survive", async () => {- // Real EnvironmentManager (electron already stubbed above; dotenv stubbed and- // resourcesPath pinned) so the on-disk .env edit is pinned, not a stub.- const dotenvPath = require.resolve("dotenv");- const originalDotenv = require.cache[dotenvPath];- require.cache[dotenvPath] = {- id: dotenvPath,- filename: dotenvPath,- loaded: true,- exports: { config: () => ({ parsed: {} }) },- };- const originalResourcesPath = process.resourcesPath;- process.resourcesPath = userDataDir;+test("an upgrade clears the saved reasons together with the flag", () => {+ process.env.WHISPER_GPU_FAILED = "cuda,vulkan";+ process.env.WHISPER_GPU_FAILED_REASON_CUDA = KERNEL_IMAGE;+ process.env.WHISPER_GPU_FAILED_REASON_VULKAN = DEVICE_LOST;+ const envManager = makeEnvManager();- try {- const EnvironmentManager = require("../../src/helpers/environment.js");- const envPath = path.join(userDataDir, ".env");+ assert.equal(resetWhisperGpuFailureOnUpgrade(envManager), true);++ for (const key of FAILURE_KEYS) assert.equal(process.env[key], undefined, key);+ assert.deepEqual(envManager.removals, FAILURE_KEYS);+});++test("reset removes only the WHISPER_GPU_FAILED line; hand-added .env lines survive", async () => {+ await withRealEnvironmentManager(async (envManager, envPath) => {fs.writeFileSync(envPath,[@@ -95,16 +148,12 @@ test("reset removes only the WHISPER_GPU_FAILED line; hand-added .env lines surv].join("\n"));process.env.WHISPER_GPU_FAILED = "cuda";-- const envManager = new EnvironmentManager();- const realRemove = envManager.removeKeyFromEnvFile.bind(envManager);- let persistence = null;- envManager.removeKeyFromEnvFile = (key) => (persistence = realRemove(key));+ const removals = trackRemovals(envManager);assert.equal(resetWhisperGpuFailureOnUpgrade(envManager), true);assert.equal(process.env.WHISPER_GPU_FAILED, undefined);- assert.ok(persistence);- await persistence;+ assert.ok(removals.last);+ await removals.last;assert.equal(fs.readFileSync(envPath, "utf8"),@@ -121,9 +170,53 @@ test("reset removes only the WHISPER_GPU_FAILED line; hand-added .env lines survfs.unlinkSync(envPath);await envManager.removeKeyFromEnvFile("WHISPER_GPU_FAILED");assert.equal(fs.existsSync(envPath), false);- } finally {- if (originalDotenv) require.cache[dotenvPath] = originalDotenv;- else delete require.cache[dotenvPath];- process.resourcesPath = originalResourcesPath;- }+ });+});++test("an upgrade removes the reason lines from .env too; hand-added lines survive", async () => {+ await withRealEnvironmentManager(async (envManager, envPath) => {+ fs.writeFileSync(+ envPath,+ [+ "# OpenWhispr Environment Variables",+ "OPENWHISPR_LOG_LEVEL=debug",+ "WHISPER_GPU_FAILED=vulkan",+ `WHISPER_GPU_FAILED_REASON_VULKAN=${DEVICE_LOST}`,+ "WHISPER_VULKAN_ENABLED=true",+ "",+ ].join("\n")+ );+ process.env.WHISPER_GPU_FAILED = "vulkan";+ process.env.WHISPER_GPU_FAILED_REASON_VULKAN = DEVICE_LOST;+ const removals = trackRemovals(envManager);++ assert.equal(resetWhisperGpuFailureOnUpgrade(envManager), true);+ await removals.last;++ assert.equal(+ fs.readFileSync(envPath, "utf8"),+ [+ "# OpenWhispr Environment Variables",+ "OPENWHISPR_LOG_LEVEL=debug",+ "WHISPER_VULKAN_ENABLED=true",+ "",+ ].join("\n")+ );+ });+});++test("a saved reason survives a full .env rewrite and reads back unchanged", async () => {+ await withRealEnvironmentManager(async (envManager, envPath) => {+ process.env.WHISPER_GPU_FAILED = "cuda,vulkan";+ process.env.WHISPER_GPU_FAILED_REASON_CUDA = KERNEL_IMAGE;+ process.env.WHISPER_GPU_FAILED_REASON_VULKAN = DEVICE_LOST;++ // _syncStartupEnv rewrites the whole file from PERSISTED_KEYS+ await envManager.saveAllKeysToEnvFile();++ const saved = parseDotenv(fs.readFileSync(envPath, "utf8"));+ assert.equal(saved.WHISPER_GPU_FAILED, "cuda,vulkan");+ assert.equal(saved.WHISPER_GPU_FAILED_REASON_CUDA, KERNEL_IMAGE);+ assert.equal(saved.WHISPER_GPU_FAILED_REASON_VULKAN, DEVICE_LOST);+ });});diff --git a/test/helpers/whisperServerGpuFailureReason.test.js b/test/helpers/whisperServerGpuFailureReason.test.jsnew file mode 100644index 000000000..025719bef--- /dev/null+++ b/test/helpers/whisperServerGpuFailureReason.test.js@@ -0,0 +1,158 @@+const test = require("node:test");+const assert = require("node:assert/strict");+const Module = require("node:module");+const { EventEmitter } = require("node:events");+const childProcess = require("node:child_process");+const fs = require("node:fs");+const os = require("node:os");+const path = require("node:path");+const {+ VULKAN_DEVICE_LOST_STDERR,+ CUDA_OUT_OF_MEMORY_STDERR,+} = require("./harness/whisperServerStderr");++// Drives the real _doStart against a fake whisper-server process: a GPU start+// that fails must hand the fallback its key error line, where the warn log used+// to keep only the first 200 characters of the device banner (#1736, #1340).+const serverModulePath = require.resolve("../../src/helpers/whisperServer");+const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "whisper-gpu-reason-"));+const modelPath = path.join(userDataDir, "ggml-large-v3-turbo.bin");+fs.writeFileSync(modelPath, "model");+test.after(() => fs.rmSync(userDataDir, { recursive: true, force: true }));++const BINARY = {+ cpu: "/fake/whisper-server",+ cuda: "/fake/whisper-server-cuda",+ vulkan: "/fake/whisper-server-vulkan",+};+// Binary path -> how its fake process behaves+const behaviours = new Map();+const warnings = [];++function fakeWhisperServer({ stderr = "", exit = null, healthy = false } = {}) {+ const child = new EventEmitter();+ child.pid = 4242;+ child.killed = false;+ child.exitCode = null;+ child.healthy = healthy;+ child.stdout = new EventEmitter();+ child.stderr = new EventEmitter();+ let closed = false;+ const close = (code, signal) => {+ if (closed) return;+ closed = true;+ child.exitCode = code;+ child.emit("exit", code, signal);+ child.emit("close", code, signal);+ };+ child.kill = (signal = "SIGTERM") => {+ child.killed = true;+ setImmediate(() => close(null, signal));+ return true;+ };+ // Output lands after _doStart attaches its handlers; then the process ends.+ setImmediate(() => {+ if (stderr) child.stderr.emit("data", Buffer.from(stderr));+ if (exit) setImmediate(() => close(exit.code ?? null, exit.signal ?? null));+ });+ return child;+}++const originalLoad = Module._load;+Module._load = function loadWithFakeProcess(request, parent, isMain) {+ if (request === "electron") {+ return { app: { getPath: () => userDataDir, isReady: () => false } };+ }+ if (parent?.filename === serverModulePath) {+ if (request === "child_process") {+ return { ...childProcess, spawn: (binary) => fakeWhisperServer(behaviours.get(binary)) };+ }+ if (request === "./debugLogger") {+ const ignore = () => {};+ return {+ debug: ignore,+ info: ignore,+ error: ignore,+ warn: (message, meta) => warnings.push({ message, meta }),+ };+ }+ }+ return originalLoad.call(this, request, parent, isMain);+};+let WhisperServerManager;+try {+ WhisperServerManager = require(serverModulePath);+} finally {+ Module._load = originalLoad;+}++function createManager(t) {+ behaviours.clear();+ warnings.length = 0;+ behaviours.set(BINARY.cpu, { healthy: true });+ const manager = new WhisperServerManager();+ manager.getServerBinaryPath = (options = {}) =>+ options.preferCuda ? BINARY.cuda : options.preferVulkan ? BINARY.vulkan : BINARY.cpu;+ manager.findAvailablePort = async () => 8199;+ manager.getFFmpegPath = () => null;+ manager.checkHealth = async () => Boolean(manager.process?.healthy);+ t.after(() => manager.stop());+ return manager;+}++function fallbackWarning() {+ return warnings.find(({ message }) =>+ message.endsWith("whisper-server failed, falling back to CPU")+ );+}++test("#1340: a Vulkan server that dies at startup reports the createDevice error", async (t) => {+ const manager = createManager(t);+ behaviours.set(BINARY.vulkan, { stderr: VULKAN_DEVICE_LOST_STDERR, exit: { code: 3 } });+ const events = [];+ manager.on("gpu-fallback", (payload) => events.push(payload));++ await manager.start(modelPath, { useVulkan: true });++ assert.equal(manager.useVulkan, false, "the CPU server took over");+ assert.deepEqual(events, [{ reason: "vk::PhysicalDevice::createDevice: ErrorDeviceLost" }]);+ const warning = fallbackWarning();+ assert.equal(warning.meta.reason, "vk::PhysicalDevice::createDevice: ErrorDeviceLost");+});++test("a CUDA server that runs out of memory at startup reports the failed allocation", async (t) => {+ const manager = createManager(t);+ behaviours.set(BINARY.cuda, { stderr: CUDA_OUT_OF_MEMORY_STDERR, exit: { code: 3 } });+ const events = [];+ manager.on("cuda-fallback", (payload) => events.push(payload));++ await manager.start(modelPath, { useCuda: true });++ assert.equal(manager.useCuda, false);+ assert.deepEqual(events, [+ {+ reason:+ "ggml_backend_cuda_buffer_type_alloc_buffer: allocating 1533.14 MiB on device 0: cudaMalloc failed: out of memory",+ },+ ]);+});++test("a GPU server that never answers is reported as a startup timeout", async (t) => {+ const manager = createManager(t);+ // Prints its banner and hangs+ behaviours.set(BINARY.vulkan, { stderr: "ggml_vulkan: Found 1 Vulkan devices:\n" });+ // Stands in for the real 120 s wait, which throws exactly this at its deadline+ const waitForReady = manager.waitForReady.bind(manager);+ manager.waitForReady = async (getProcessInfo, timeoutMs) => {+ if (manager.useVulkan) {+ throw new Error(`whisper-server failed to start within ${timeoutMs}ms`);+ }+ return waitForReady(getProcessInfo, timeoutMs);+ };+ const events = [];+ manager.on("gpu-fallback", (payload) => events.push(payload));++ await manager.start(modelPath, { useVulkan: true });++ assert.deepEqual(events, [{ reason: "startup timed out after 120 s" }]);+});