Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat: GPU-accelerated local inference (Vulkan/Metal) (#315)
Gabriel Stein898887628666
diff --git a/preload.js b/preload.jsindex aa002f517..d52819c18 100644--- a/preload.js+++ b/preload.js@@ -296,6 +296,18 @@ contextBridge.exposeInMainWorld("electronAPI", {llamaServerStart: (modelId) => ipcRenderer.invoke("llama-server-start", modelId),llamaServerStop: () => ipcRenderer.invoke("llama-server-stop"),llamaServerStatus: () => ipcRenderer.invoke("llama-server-status"),+ llamaGpuReset: () => ipcRenderer.invoke("llama-gpu-reset"),++ // Vulkan GPU acceleration+ detectVulkanGpu: () => ipcRenderer.invoke("detect-vulkan-gpu"),+ getLlamaVulkanStatus: () => ipcRenderer.invoke("get-llama-vulkan-status"),+ downloadLlamaVulkanBinary: () => ipcRenderer.invoke("download-llama-vulkan-binary"),+ cancelLlamaVulkanDownload: () => ipcRenderer.invoke("cancel-llama-vulkan-download"),+ deleteLlamaVulkanBinary: () => ipcRenderer.invoke("delete-llama-vulkan-binary"),+ onLlamaVulkanDownloadProgress: registerListener(+ "llama-vulkan-download-progress",+ (callback) => (_event, data) => callback(data)+ ),getLogLevel: () => ipcRenderer.invoke("get-log-level"),log: (entry) => ipcRenderer.invoke("app-log", entry),diff --git a/scripts/download-llama-server.js b/scripts/download-llama-server.jsindex a87219153..64120323c 100644--- a/scripts/download-llama-server.js+++ b/scripts/download-llama-server.js@@ -16,37 +16,39 @@ const LLAMA_CPP_REPO = "ggerganov/llama.cpp";// Version can be pinned via environment variable for reproducible buildsconst VERSION_OVERRIDE = process.env.LLAMA_CPP_VERSION || null;-// Asset name patterns to match in the release (version-independent)const BINARIES = {"darwin-arm64": {+ platformArch: "darwin-arm64",assetPattern: /^llama-.*-bin-macos-arm64\.tar\.gz$/,binaryPath: "build/bin/llama-server",outputName: "llama-server-darwin-arm64",libPattern: "*.dylib",},"darwin-x64": {+ platformArch: "darwin-x64",assetPattern: /^llama-.*-bin-macos-x64\.tar\.gz$/,binaryPath: "build/bin/llama-server",outputName: "llama-server-darwin-x64",libPattern: "*.dylib",},- "win32-x64": {+ "win32-x64-cpu": {+ platformArch: "win32-x64",assetPattern: /^llama-.*-bin-win-cpu-x64\.zip$/,binaryPath: "build/bin/llama-server.exe",- outputName: "llama-server-win32-x64.exe",+ outputName: "llama-server-win32-x64-cpu.exe",libPattern: "*.dll",},- "linux-x64": {+ "linux-x64-cpu": {+ platformArch: "linux-x64",assetPattern: /^llama-.*-bin-ubuntu-x64\.tar\.gz$/,binaryPath: "build/bin/llama-server",- outputName: "llama-server-linux-x64",+ outputName: "llama-server-linux-x64-cpu",libPattern: "*.so*",},};const BIN_DIR = path.join(__dirname, "..", "resources", "bin");-// Cache the release info to avoid multiple API callslet cachedRelease = null;async function getRelease() {@@ -84,45 +86,43 @@ function findLibrariesInDir(dir, pattern, maxDepth = 5, currentDepth = 0) {}function matchesPattern(filename, pattern) {- // Handle patterns like "*.dylib", "*.dll", "*.so*"if (pattern === "*.dylib") {return filename.endsWith(".dylib");} else if (pattern === "*.dll") {return filename.endsWith(".dll");} else if (pattern === "*.so*") {- // Match .so files with optional version suffix (e.g., libfoo.so, libfoo.so.1, libfoo.so.1.2.3)return /\.so(\.\d+)*$/.test(filename) || filename.endsWith(".so");}return false;}-async function downloadBinary(platformArch, config, release, isForce = false) {+async function downloadBinary(key, config, release, isForce = false) {if (!config) {- console.log(` ${platformArch}: Not supported`);+ console.log(` ${key}: Not supported`);return false;}const outputPath = path.join(BIN_DIR, config.outputName);if (fs.existsSync(outputPath) && !isForce) {- console.log(` ${platformArch}: Already exists (use --force to re-download)`);+ console.log(` ${key}: Already exists (use --force to re-download)`);return true;}const asset = findAsset(release, config.assetPattern);if (!asset) {- console.error(` ${platformArch}: No matching asset found for pattern ${config.assetPattern}`);+ console.error(` ${key}: No matching asset found for pattern ${config.assetPattern}`);return false;}- console.log(` ${platformArch}: Downloading from ${asset.url}`);+ console.log(` ${key}: Downloading from ${asset.url}`);const zipPath = path.join(BIN_DIR, asset.name);try {await downloadFile(asset.url, zipPath);- const extractDir = path.join(BIN_DIR, `temp-llama-${platformArch}`);+ const extractDir = path.join(BIN_DIR, `temp-llama-${key}`);fs.mkdirSync(extractDir, { recursive: true });await extractArchive(zipPath, extractDir);@@ -136,10 +136,8 @@ async function downloadBinary(platformArch, config, release, isForce = false) {if (binaryPath && fs.existsSync(binaryPath)) {fs.copyFileSync(binaryPath, outputPath);setExecutable(outputPath);- console.log(` ${platformArch}: Extracted to ${config.outputName}`);+ console.log(` ${key}: Extracted to ${config.outputName}`);- // Copy shared libraries (dylib/dll/so files)- // Always overwrite — libraries are architecture-specific (e.g. arm64 vs x64 dylibs)if (config.libPattern) {const libraries = findLibrariesInDir(extractDir, config.libPattern);@@ -149,11 +147,11 @@ async function downloadBinary(platformArch, config, release, isForce = false) {fs.copyFileSync(libPath, destPath);setExecutable(destPath);- console.log(` ${platformArch}: Copied library ${libName}`);+ console.log(` ${key}: Copied library ${libName}`);}}} else {- console.error(` ${platformArch}: Binary '${binaryName}' not found in archive`);+ console.error(` ${key}: Binary '${binaryName}' not found in archive`);return false;}@@ -161,12 +159,16 @@ async function downloadBinary(platformArch, config, release, isForce = false) {if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath);return true;} catch (error) {- console.error(` ${platformArch}: Failed - ${error.message}`);+ console.error(` ${key}: Failed - ${error.message}`);if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath);return false;}}+function getEntriesForPlatformArch(platformArch) {+ return Object.entries(BINARIES).filter(([, config]) => config.platformArch === platformArch);+}+async function main() {if (VERSION_OVERRIDE) {console.log(`\n[llama-server] Using pinned version: ${VERSION_OVERRIDE}`);@@ -189,18 +191,25 @@ async function main() {const args = parseArgs();if (args.isCurrent) {- if (!BINARIES[args.platformArch]) {+ const entries = getEntriesForPlatformArch(args.platformArch);++ if (entries.length === 0) {console.error(`Unsupported platform/arch: ${args.platformArch}`);process.exitCode = 1;return;}console.log(`Downloading for target platform (${args.platformArch}):`);- const ok = await downloadBinary(args.platformArch, BINARIES[args.platformArch], release, args.isForce);- if (!ok) {- console.error(`Failed to download binaries for ${args.platformArch}`);- process.exitCode = 1;- return;++ for (const [key, config] of entries) {+ const ok = await downloadBinary(key, config, release, args.isForce);+ if (!ok && config.optional) {+ console.warn(` ${key}: Skipping optional variant`);+ } else if (!ok) {+ console.error(`Failed to download binaries for ${key}`);+ process.exitCode = 1;+ return;+ }}if (args.shouldCleanup) {@@ -208,8 +217,11 @@ async function main() {}} else {console.log("Downloading binaries for all platforms:");- for (const platformArch of Object.keys(BINARIES)) {- await downloadBinary(platformArch, BINARIES[platformArch], release, args.isForce);+ for (const [key, config] of Object.entries(BINARIES)) {+ const ok = await downloadBinary(key, config, release, args.isForce);+ if (!ok && config.optional) {+ console.warn(` ${key}: Skipping optional variant`);+ }}}diff --git a/src/components/ControlPanel.tsx b/src/components/ControlPanel.tsxindex 1bc89686e..7f25455e0 100644--- a/src/components/ControlPanel.tsx+++ b/src/components/ControlPanel.tsx@@ -45,6 +45,9 @@ export default function ControlPanel() {);const [showReferrals, setShowReferrals] = useState(false);const [showCloudMigrationBanner, setShowCloudMigrationBanner] = useState(false);+ const [vulkanBanner, setVulkanBanner] = useState<{ show: boolean; deviceName?: string }>({+ show: false,+ });const [activeView, setActiveView] = useState<ControlPanelView>("home");const [cudaStatus, setCudaStatus] = useState<CudaWhisperStatus | null>(null);const [gpuBannerDismissed, setGpuBannerDismissed] = useState(@@ -153,6 +156,22 @@ export default function ControlPanel() {setShowCloudMigrationBanner(true);}, [authLoaded, isSignedIn, setUseLocalWhisper, setCloudTranscriptionMode]);+ useEffect(() => {+ if (platform === "darwin" || !useReasoningModel) return;+ if (localStorage.getItem("llamaVulkanBannerDismissed") === "true") return;++ Promise.all([+ window.electronAPI?.detectVulkanGpu?.(),+ window.electronAPI?.getLlamaVulkanStatus?.(),+ ])+ .then(([gpu, vulkan]) => {+ if (gpu?.available && !vulkan?.downloaded) {+ setVulkanBanner({ show: true, deviceName: gpu.deviceName });+ }+ })+ .catch(() => {});+ }, [useReasoningModel]);+const fetchCudaStatus = useCallback(() => {if (platform === "darwin" || gpuBannerDismissed) return;if (!useLocalWhisper || localTranscriptionProvider !== "whisper") return;@@ -431,6 +450,41 @@ export default function ControlPanel() {</div></div>)}+ {vulkanBanner.show && activeView === "home" && (+ <div className="mx-4 mt-3 p-3 rounded-lg bg-purple-500/10 border border-purple-500/20">+ <div className="flex items-start gap-2">+ <Zap className="w-4 h-4 text-purple-400 shrink-0 mt-0.5" />+ <div className="flex-1 min-w-0">+ <p className="text-sm text-foreground">+ {vulkanBanner.deviceName || "GPU"} detected — speed up local AI with Vulkan+ acceleration+ </p>+ <div className="flex items-center gap-3 mt-2">+ <button+ type="button"+ onClick={() => {+ setSettingsSection("reasoning");+ setShowSettings(true);+ }}+ className="text-xs font-medium text-purple-400 hover:text-purple-300 transition-colors"+ >+ Enable GPU+ </button>+ <button+ type="button"+ onClick={() => {+ localStorage.setItem("llamaVulkanBannerDismissed", "true");+ setVulkanBanner({ show: false });+ }}+ className="text-xs text-muted-foreground hover:text-foreground transition-colors"+ >+ Not now+ </button>+ </div>+ </div>+ </div>+ </div>+ )}{showGpuBanner && (<div className="max-w-3xl mx-auto w-full mb-3"><div className="rounded-lg border border-primary/20 dark:border-primary/15 bg-primary/5 p-3">diff --git a/src/components/ReasoningModelSelector.tsx b/src/components/ReasoningModelSelector.tsxindex a6f9c490d..4d343732a 100644--- a/src/components/ReasoningModelSelector.tsx+++ b/src/components/ReasoningModelSelector.tsx@@ -1,5 +1,11 @@import { useState, useEffect, useCallback, useMemo, useRef } from "react";import { useTranslation } from "react-i18next";+import type {+ LlamaServerStatus,+ LlamaVulkanStatus,+ VulkanGpuResult,+ LlamaVulkanDownloadProgress,+} from "../types/electron";import { Button } from "./ui/button";import { Input } from "./ui/input";import { Cloud, Lock } from "lucide-react";@@ -14,6 +20,7 @@ import { modelRegistry } from "../models/ModelRegistry";import { getProviderIcon, isMonochromeProvider } from "../utils/providerIcons";import { isSecureEndpoint } from "../utils/urlUtils";import { createExternalLinkHandler } from "../utils/externalLinks";+import { getCachedPlatform } from "../utils/platform";type CloudModelOption = {value: string;@@ -68,6 +75,173 @@ interface ReasoningModelSelectorProps {setCustomReasoningApiKey?: (key: string) => void;}+function GpuStatusBadge() {+ const [serverStatus, setServerStatus] = useState<LlamaServerStatus | null>(null);+ const [vulkanStatus, setVulkanStatus] = useState<LlamaVulkanStatus | null>(null);+ const [gpuResult, setGpuResult] = useState<VulkanGpuResult | null>(null);+ const [progress, setProgress] = useState<LlamaVulkanDownloadProgress | null>(null);+ const [downloading, setDownloading] = useState(false);+ const [error, setError] = useState<string | null>(null);+ const platform = getCachedPlatform();++ useEffect(() => {+ const poll = () => {+ window.electronAPI+ ?.llamaServerStatus?.()+ .then(setServerStatus)+ .catch(() => {});+ if (platform !== "darwin") {+ window.electronAPI+ ?.getLlamaVulkanStatus?.()+ .then(setVulkanStatus)+ .catch(() => {});+ }+ };+ poll();+ const id = setInterval(poll, 5000);+ return () => clearInterval(id);+ }, [platform]);++ useEffect(() => {+ if (platform !== "darwin") {+ window.electronAPI+ ?.detectVulkanGpu?.()+ .then(setGpuResult)+ .catch(() => {});+ }+ }, [platform]);++ useEffect(() => {+ const cleanup = window.electronAPI?.onLlamaVulkanDownloadProgress?.((data) => {+ setProgress(data);+ });+ return () => cleanup?.();+ }, []);++ const handleDownload = async () => {+ setDownloading(true);+ setError(null);+ try {+ const result = await window.electronAPI?.downloadLlamaVulkanBinary?.();+ if (result?.success) {+ setVulkanStatus((prev) => (prev ? { ...prev, downloaded: true } : prev));+ window.electronAPI?.llamaGpuReset?.();+ } else if (result && !result.cancelled) {+ setError(result.error || "Download failed");+ }+ } catch (err) {+ setError(err instanceof Error ? err.message : "Download failed");+ } finally {+ setDownloading(false);+ setProgress(null);+ }+ };++ const handleDelete = async () => {+ await window.electronAPI?.deleteLlamaVulkanBinary?.();+ setVulkanStatus((prev) => (prev ? { ...prev, downloaded: false } : prev));+ };++ if (platform === "darwin") {+ if (!serverStatus?.running) return null;+ return (+ <div className="flex items-center gap-1.5 mt-2 px-1">+ <span+ className="inline-block w-1.5 h-1.5 rounded-full shrink-0"+ style={{ backgroundColor: "#22c55e" }}+ />+ <span className="text-xs text-muted-foreground">GPU Accelerated (Metal)</span>+ </div>+ );+ }++ if (downloading && progress) {+ return (+ <div className="flex items-center gap-2 mt-2 px-1">+ <div className="flex-1 h-1 bg-muted rounded-full overflow-hidden">+ <div+ className="h-full bg-purple-500 transition-all"+ style={{ width: `${progress.percentage}%` }}+ />+ </div>+ <span className="text-xs text-muted-foreground tabular-nums">{progress.percentage}%</span>+ <button+ type="button"+ onClick={() => window.electronAPI?.cancelLlamaVulkanDownload?.()}+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors"+ >+ Cancel+ </button>+ </div>+ );+ }++ if (error) {+ return (+ <div className="flex items-center gap-1.5 mt-2 px-1">+ <span className="text-xs text-destructive">{error}</span>+ <button+ type="button"+ onClick={() => setError(null)}+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-1"+ >+ Dismiss+ </button>+ </div>+ );+ }++ if (vulkanStatus?.downloaded) {+ const isGpu = serverStatus?.gpuAccelerated && serverStatus?.backend === "vulkan";+ return (+ <div className="flex items-center gap-1.5 mt-2 px-1">+ <span+ className="inline-block w-1.5 h-1.5 rounded-full shrink-0"+ style={{ backgroundColor: isGpu ? "#22c55e" : "#f59e0b" }}+ />+ <span className="text-xs text-muted-foreground">+ {isGpu ? "GPU Accelerated (Vulkan)" : "CPU Only"}+ </span>+ {!isGpu && serverStatus?.running && (+ <button+ type="button"+ onClick={() => window.electronAPI?.llamaGpuReset?.()}+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-1"+ >+ Re-detect GPU+ </button>+ )}+ <button+ type="button"+ onClick={handleDelete}+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-auto"+ >+ Remove+ </button>+ </div>+ );+ }++ if (gpuResult?.available) {+ return (+ <div className="flex items-center gap-1.5 mt-2 px-1">+ <span className="text-xs text-muted-foreground">+ {gpuResult.deviceName || "GPU"} detected+ </span>+ <button+ type="button"+ onClick={handleDownload}+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-1"+ >+ Download Vulkan (~40MB)+ </button>+ </div>+ );+ }++ return null;+}+export default function ReasoningModelSelector({reasoningModel,setReasoningModel,@@ -746,16 +920,19 @@ export default function ReasoningModelSelector({</div></div>) : (- <LocalModelPicker- providers={localProviders}- selectedModel={reasoningModel}- selectedProvider={selectedLocalProvider}- onModelSelect={setReasoningModel}- onProviderSelect={handleLocalProviderChange}- modelType="llm"- colorScheme="purple"- onDownloadComplete={loadDownloadedModels}- />+ <>+ <LocalModelPicker+ providers={localProviders}+ selectedModel={reasoningModel}+ selectedProvider={selectedLocalProvider}+ onModelSelect={setReasoningModel}+ onProviderSelect={handleLocalProviderChange}+ modelType="llm"+ colorScheme="purple"+ onDownloadComplete={loadDownloadedModels}+ />+ <GpuStatusBadge />+ </>)}</div>);diff --git a/src/components/notes/UploadAudioView.tsx b/src/components/notes/UploadAudioView.tsxindex e62a0d7b0..8165881b8 100644--- a/src/components/notes/UploadAudioView.tsx+++ b/src/components/notes/UploadAudioView.tsx@@ -32,7 +32,7 @@ import { useSettings } from "../../hooks/useSettings";import { withSessionRefresh } from "../../lib/neonAuth";import reasoningService from "../../services/ReasoningService";import { getAllReasoningModels } from "../../models/ModelRegistry";-import { useSettingsStore } from "../../stores/settingsStore";+import { useSettingsStore, selectIsCloudReasoningMode } from "../../stores/settingsStore";const TranscriptionModelPicker = React.lazy(() => import("../TranscriptionModelPicker"));@@ -110,7 +110,10 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: UploaupdateTranscriptionSettings,} = useSettings();- const effectiveReasoningModel = useSettingsStore((s) => s.reasoningModel);+ const isCloudReasoning = useSettingsStore(selectIsCloudReasoningMode);+ const effectiveReasoningModel = useSettingsStore((s) =>+ selectIsCloudReasoningMode(s) ? "" : s.reasoningModel+ );const useReasoningModel = useSettingsStore((s) => s.useReasoningModel);const isOpenWhisprCloud =@@ -213,8 +216,10 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploaconst generateTitle = async (text: string): Promise<string> => {if (!useReasoningModel) return "";- const model = effectiveReasoningModel || getAllReasoningModels()[0]?.value;- if (!model) return "";+ const model = isCloudReasoning+ ? ""+ : effectiveReasoningModel || getAllReasoningModels()[0]?.value;+ if (!model && !isCloudReasoning) return "";try {const title = await reasoningService.processText(text.slice(0, 2000), model, null, {systemPrompt: TITLE_SYSTEM_PROMPT,diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.jsindex 71f609a77..4b60a4459 100644--- a/src/helpers/audioManager.js+++ b/src/helpers/audioManager.js@@ -5,7 +5,11 @@ import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";import { isSecureEndpoint } from "../utils/urlUtils";import { withSessionRefresh } from "../lib/neonAuth";import { getBaseLanguageCode, validateLanguageForModel } from "../utils/languageSupport";-import { getSettings, getEffectiveReasoningModel } from "../stores/settingsStore";+import {+ getSettings,+ getEffectiveReasoningModel,+ isCloudReasoningMode,+} from "../stores/settingsStore";const SHORT_CLIP_DURATION_SECONDS = 2.5;const REASONING_CACHE_TTL = 30000; // 30 seconds@@ -804,6 +808,15 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);return false;}+ if (isCloudReasoningMode()) {+ this.reasoningAvailabilityCache = {+ value: true,+ expiresAt: now + REASONING_CACHE_TTL,+ };+ this.cachedReasoningPreference = useReasoning;+ return true;+ }+try {const isAvailable = await ReasoningService.isAvailable();@@ -846,12 +859,13 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);});const reasoningModel = getEffectiveReasoningModel();+ const isCloud = isCloudReasoningMode();const reasoningProvider = getSettings().reasoningProvider || "auto";const agentName =typeof window !== "undefined" && window.localStorage? localStorage.getItem("agentName") || null: null;- if (!reasoningModel) {+ if (!reasoningModel && !isCloud) {logger.logReasoning("REASONING_SKIPPED", {reason: "No reasoning model selected",});diff --git a/src/helpers/environment.js b/src/helpers/environment.jsindex 7b983861e..e9ca5e741 100644--- a/src/helpers/environment.js+++ b/src/helpers/environment.js@@ -17,6 +17,8 @@ const PERSISTED_KEYS = ["LOCAL_WHISPER_MODEL","REASONING_PROVIDER","LOCAL_REASONING_MODEL",+ "LLAMA_GPU_BACKEND",+ "LLAMA_VULKAN_ENABLED","DICTATION_KEY","ACTIVATION_MODE","FLOATING_ICON_AUTO_HIDE",diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 2f67d5a57..c28314afe 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -1332,9 +1332,11 @@ class IPCHandlers {await modelManager.serverManager.start(modelPath, {contextSize: modelInfo.model.contextLength || 4096,threads: 4,+ gpuLayers: 99,});modelManager.currentServerModelId = modelId;+ this.environmentManager.saveAllKeysToEnvFile().catch(() => {});return { success: true, port: modelManager.serverManager.port };} catch (error) {return { success: false, error: error.message };@@ -1360,6 +1362,101 @@ class IPCHandlers {}});+ ipcMain.handle("llama-gpu-reset", async () => {+ try {+ const modelManager = require("./modelManagerBridge").default;+ modelManager.serverManager.resetGpuDetection();+ await modelManager.stopServer();+ return { success: true };+ } catch (error) {+ return { success: false, error: error.message };+ }+ });++ ipcMain.handle("detect-vulkan-gpu", async () => {+ try {+ const { detectVulkanGpu } = require("../utils/vulkanDetection");+ return await detectVulkanGpu();+ } catch (error) {+ return { available: false, error: error.message };+ }+ });++ ipcMain.handle("get-llama-vulkan-status", async () => {+ try {+ if (!this._llamaVulkanManager) {+ const LlamaVulkanManager = require("./llamaVulkanManager");+ this._llamaVulkanManager = new LlamaVulkanManager();+ }+ return this._llamaVulkanManager.getStatus();+ } catch (error) {+ return { supported: false, downloaded: false, error: error.message };+ }+ });++ ipcMain.handle("download-llama-vulkan-binary", async (event) => {+ try {+ if (!this._llamaVulkanManager) {+ const LlamaVulkanManager = require("./llamaVulkanManager");+ this._llamaVulkanManager = new LlamaVulkanManager();+ }++ const result = await this._llamaVulkanManager.download((downloaded, total) => {+ if (!event.sender.isDestroyed()) {+ event.sender.send("llama-vulkan-download-progress", {+ downloaded,+ total,+ percentage: total > 0 ? Math.round((downloaded / total) * 100) : 0,+ });+ }+ });++ if (result.success) {+ process.env.LLAMA_VULKAN_ENABLED = "true";+ delete process.env.LLAMA_GPU_BACKEND;+ const modelManager = require("./modelManagerBridge").default;+ modelManager.serverManager.cachedServerBinaryPaths = null;+ this.environmentManager.saveAllKeysToEnvFile().catch(() => {});+ }++ return result;+ } catch (error) {+ return { success: false, error: error.message };+ }+ });++ ipcMain.handle("cancel-llama-vulkan-download", async () => {+ if (this._llamaVulkanManager) {+ return { success: this._llamaVulkanManager.cancelDownload() };+ }+ return { success: false };+ });++ ipcMain.handle("delete-llama-vulkan-binary", async () => {+ try {+ if (!this._llamaVulkanManager) {+ const LlamaVulkanManager = require("./llamaVulkanManager");+ this._llamaVulkanManager = new LlamaVulkanManager();+ }++ const modelManager = require("./modelManagerBridge").default;+ if (modelManager.serverManager.activeBackend === "vulkan") {+ await modelManager.stopServer();+ }++ const result = await this._llamaVulkanManager.deleteBinary();++ delete process.env.LLAMA_VULKAN_ENABLED;+ delete process.env.LLAMA_GPU_BACKEND;+ modelManager.serverManager.cachedServerBinaryPaths = null;+ this.environmentManager.saveAllKeysToEnvFile().catch(() => {});++ return result;+ } catch (error) {+ return { success: false, error: error.message };+ }+ });+ipcMain.handle("get-log-level", async () => {return debugLogger.getLevel();});diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.jsindex 2d4512c01..0e9737666 100644--- a/src/helpers/llamaServer.js+++ b/src/helpers/llamaServer.js@@ -6,10 +6,12 @@ const http = require("http");const debugLogger = require("./debugLogger");const { killProcess } = require("../utils/process");const { getSafeTempDir } = require("./safeTempDir");+const { app } = require("electron");const PORT_RANGE_START = 8200;const PORT_RANGE_END = 8220;const STARTUP_TIMEOUT_MS = 60000;+const VULKAN_STARTUP_TIMEOUT_MS = 15000;const HEALTH_CHECK_INTERVAL_MS = 5000;const HEALTH_CHECK_TIMEOUT_MS = 2000;const STARTUP_POLL_INTERVAL_MS = 500;@@ -24,52 +26,70 @@ class LlamaServerManager {this.startupPromise = null;this.healthCheckInterval = null;this.healthCheckFailures = 0;- this.cachedServerBinaryPath = null;+ this.cachedServerBinaryPaths = null;+ this.activeBackend = null;}- getServerBinaryPath() {- if (this.cachedServerBinaryPath) return this.cachedServerBinaryPath;+ getServerBinaryPaths() {+ if (this.cachedServerBinaryPaths) return this.cachedServerBinaryPaths;const platform = process.platform;const arch = process.arch;const platformArch = `${platform}-${arch}`;- const binaryName =- platform === "win32" ? `llama-server-${platformArch}.exe` : `llama-server-${platformArch}`;- const genericName = platform === "win32" ? "llama-server.exe" : "llama-server";+ const ext = platform === "win32" ? ".exe" : "";- const candidates = [];-- // Production: check resourcesPath- if (process.resourcesPath) {- candidates.push(- path.join(process.resourcesPath, "bin", binaryName),- path.join(process.resourcesPath, "bin", genericName)- );- }-- // Development: check relative to this file- candidates.push(- path.join(__dirname, "..", "..", "resources", "bin", binaryName),- path.join(__dirname, "..", "..", "resources", "bin", genericName)- );+ const resolveBinary = (name) => {+ const candidates = [];+ if (process.resourcesPath) {+ candidates.push(path.join(process.resourcesPath, "bin", name));+ }+ candidates.push(path.join(__dirname, "..", "..", "resources", "bin", name));- for (const candidate of candidates) {- if (fs.existsSync(candidate)) {+ for (const candidate of candidates) {try {- fs.statSync(candidate);- this.cachedServerBinaryPath = candidate;- return candidate;+ if (fs.existsSync(candidate)) {+ fs.statSync(candidate);+ return candidate;+ }} catch {// Can't access binary}}+ return null;+ };++ let paths;++ if (platform === "darwin") {+ const defaultBin =+ resolveBinary(`llama-server-${platformArch}`) || resolveBinary(`llama-server${ext}`);+ paths = defaultBin ? { default: defaultBin } : {};+ } else {+ const userBinDir = path.join(app.getPath("userData"), "bin");+ const vulkanName = `llama-server-vulkan${ext}`;+ let vulkanBin = null;+ try {+ const vulkanPath = path.join(userBinDir, vulkanName);+ if (fs.existsSync(vulkanPath)) vulkanBin = vulkanPath;+ } catch {}++ const cpuBin =+ resolveBinary(`llama-server-${platformArch}-cpu${ext}`) ||+ resolveBinary(`llama-server-${platformArch}${ext}`) ||+ resolveBinary(`llama-server${ext}`);++ paths = {};+ if (vulkanBin) paths.vulkan = vulkanBin;+ if (cpuBin) paths.cpu = cpuBin;}- return null;+ this.cachedServerBinaryPaths = paths;+ return paths;}isAvailable() {- return this.getServerBinaryPath() !== null;+ const paths = this.getServerBinaryPaths();+ return Object.keys(paths).length > 0;}async findAvailablePort() {@@ -94,10 +114,8 @@ class LlamaServerManager {async start(modelPath, options = {}) {if (this.startupPromise) return this.startupPromise;- // Already running with same modelif (this.ready && this.modelPath === modelPath) return;- // Stop existing server if runningif (this.process) {await this.stop();}@@ -111,14 +129,14 @@ class LlamaServerManager {}async _doStart(modelPath, options = {}) {- const serverBinary = this.getServerBinaryPath();- if (!serverBinary) throw new Error("llama-server binary not found");+ const binaryPaths = this.getServerBinaryPaths();+ if (Object.keys(binaryPaths).length === 0) throw new Error("llama-server binary not found");if (!fs.existsSync(modelPath)) throw new Error(`Model file not found: ${modelPath}`);this.port = await this.findAvailablePort();this.modelPath = modelPath;- const args = [+ const baseArgs = ["--model",modelPath,"--host",@@ -131,105 +149,219 @@ class LlamaServerManager {String(options.threads || 4),];- // Add GPU layers if specified- if (options.gpuLayers) {- args.push("--n-gpu-layers", String(options.gpuLayers));+ if (process.platform === "darwin") {+ const args = [...baseArgs, "--n-gpu-layers", "99"];+ await this._startWithBinary(+ binaryPaths.default,+ args,+ this._buildEnv(binaryPaths.default),+ STARTUP_TIMEOUT_MS+ );+ this.activeBackend = "metal";+ this._saveGpuBackend("metal");+ } else {+ await this._startWithGpuFallback(binaryPaths, baseArgs, options);}- debugLogger.debug("Starting llama-server", { port: this.port, modelPath, args });+ this.startHealthCheck();+ debugLogger.info("llama-server started successfully", {+ port: this.port,+ model: path.basename(modelPath),+ backend: this.activeBackend,+ });+ }++ async _startWithGpuFallback(binaryPaths, baseArgs, options) {+ const cached = this._loadGpuBackend();+ const gpuArgs = [...baseArgs, "--n-gpu-layers", "99"];+ const cpuArgs = baseArgs;- // Set library path for dynamic library loading- const binDir = path.dirname(serverBinary);+ if (cached !== "cpu" && binaryPaths.vulkan) {+ try {+ debugLogger.debug("Attempting Vulkan backend startup");+ await this._startWithBinary(+ binaryPaths.vulkan,+ gpuArgs,+ this._buildEnv(binaryPaths.vulkan),+ VULKAN_STARTUP_TIMEOUT_MS+ );+ this.activeBackend = "vulkan";+ this._saveGpuBackend("vulkan");+ return;+ } catch (err) {+ debugLogger.warn("Vulkan backend failed, falling back to CPU", { error: err.message });+ await this._killCurrentProcess();+ this.port = await this.findAvailablePort();+ }+ }++ if (!binaryPaths.cpu) throw new Error("No CPU llama-server binary available");++ debugLogger.debug("Starting with CPU backend");+ await this._startWithBinary(+ binaryPaths.cpu,+ cpuArgs,+ this._buildEnv(binaryPaths.cpu),+ STARTUP_TIMEOUT_MS+ );+ this.activeBackend = "cpu";+ this._saveGpuBackend("cpu");+ }++ _buildEnv(binaryPath) {+ const binDir = path.dirname(binaryPath);const env = { ...process.env };if (process.platform === "darwin") {- // macOS: Set DYLD_LIBRARY_PATH to find .dylib filesenv.DYLD_LIBRARY_PATH = binDir + (env.DYLD_LIBRARY_PATH ? `:${env.DYLD_LIBRARY_PATH}` : "");} else if (process.platform === "linux") {- // Linux: Set LD_LIBRARY_PATH to find .so filesenv.LD_LIBRARY_PATH = binDir + (env.LD_LIBRARY_PATH ? `:${env.LD_LIBRARY_PATH}` : "");+ } else if (process.platform === "win32") {+ env.PATH = binDir + (env.PATH ? `;${env.PATH}` : "");}- this.process = spawn(serverBinary, args, {- stdio: ["ignore", "pipe", "pipe"],- windowsHide: true,- cwd: getSafeTempDir(),- env,- });+ return env;+ }- let stderrBuffer = "";- let exitCode = null;- let exitSignal = null;+ _startWithBinary(binaryPath, args, env, timeoutMs) {+ return new Promise((resolve, reject) => {+ debugLogger.debug("Spawning llama-server", { binary: binaryPath, port: this.port, args });- this.process.stdout.on("data", (data) => {- debugLogger.debug("llama-server stdout", { data: data.toString().trim() });- });+ this.process = spawn(binaryPath, args, {+ stdio: ["ignore", "pipe", "pipe"],+ windowsHide: true,+ cwd: getSafeTempDir(),+ env,+ });- this.process.stderr.on("data", (data) => {- stderrBuffer += data.toString();- debugLogger.debug("llama-server stderr", { data: data.toString().trim() });- });+ let stderrBuffer = "";+ let exitCode = null;+ let exitSignal = null;+ let settled = false;- this.process.on("error", (error) => {- debugLogger.error("llama-server process error", { error: error.message });- this.ready = false;- });+ const settle = (fn) => {+ if (settled) return;+ settled = true;+ fn();+ };- this.process.on("close", (code, signal) => {- exitCode = code;- exitSignal = signal;- debugLogger.debug("llama-server process exited", { code, signal });- this.ready = false;- this.process = null;- this.stopHealthCheck();- });+ this.process.stdout.on("data", (data) => {+ debugLogger.debug("llama-server stdout", { data: data.toString().trim() });+ });- await this.waitForReady(() => ({ stderr: stderrBuffer, exitCode, exitSignal }));- this.startHealthCheck();+ this.process.stderr.on("data", (data) => {+ stderrBuffer += data.toString();+ debugLogger.debug("llama-server stderr", { data: data.toString().trim() });+ });- debugLogger.info("llama-server started successfully", {- port: this.port,- model: path.basename(modelPath),+ this.process.on("error", (error) => {+ debugLogger.error("llama-server process error", { error: error.message });+ this.ready = false;+ settle(() => reject(new Error(`Failed to spawn llama-server: ${error.message}`)));+ });++ this.process.on("close", (code, signal) => {+ exitCode = code;+ exitSignal = signal;+ debugLogger.debug("llama-server process exited", { code, signal });+ this.ready = false;+ this.process = null;+ this.stopHealthCheck();+ });++ const getProcessInfo = () => ({ stderr: stderrBuffer, exitCode, exitSignal });++ const startTime = Date.now();+ let pollCount = 0;++ const poll = async () => {+ if (settled) return;++ if (!this.process || this.process.killed) {+ const info = getProcessInfo();+ const signal = info.exitSignal;+ const diagParts = [];+ if (signal) diagParts.push(`signal: ${signal}`);+ else if (info.exitCode !== null && info.exitCode !== undefined)+ diagParts.push(`exit code: ${info.exitCode}`);+ const oomHint =+ signal === "SIGKILL"+ ? " — the process was killed by the OS, likely due to insufficient memory. Try a smaller/more quantized model, or reduce the context size."+ : "";+ const stderr = info.stderr ? info.stderr.trim().slice(-800) : "";+ const diagStr = diagParts.length ? ` (${diagParts.join(", ")})` : "";+ settle(() =>+ reject(+ new Error(+ `llama-server process died during startup${diagStr}${oomHint}${stderr ? `\nProcess output: ${stderr}` : ""}`+ )+ )+ );+ return;+ }++ pollCount++;+ if (await this.checkHealth()) {+ this.ready = true;+ debugLogger.debug("llama-server ready", {+ startupTimeMs: Date.now() - startTime,+ pollCount,+ });+ settle(() => resolve());+ return;+ }++ if (Date.now() - startTime >= timeoutMs) {+ settle(() => reject(new Error(`llama-server failed to start within ${timeoutMs}ms`)));+ return;+ }++ setTimeout(poll, STARTUP_POLL_INTERVAL_MS);+ };++ poll();});}- async waitForReady(getProcessInfo) {- const startTime = Date.now();- let pollCount = 0;-- while (Date.now() - startTime < STARTUP_TIMEOUT_MS) {- if (!this.process || this.process.killed) {- const info = getProcessInfo ? getProcessInfo() : {};- const signal = info.exitSignal;- const diagParts = [];- if (signal) diagParts.push(`signal: ${signal}`);- else if (info.exitCode !== null && info.exitCode !== undefined)- diagParts.push(`exit code: ${info.exitCode}`);- const oomHint =- signal === "SIGKILL"- ? " — the process was killed by the OS, likely due to insufficient memory. Try a smaller/more quantized model, or reduce the context size."- : "";- const stderr = info.stderr ? info.stderr.trim().slice(-800) : "";- const diagStr = diagParts.length ? ` (${diagParts.join(", ")})` : "";- throw new Error(- `llama-server process died during startup${diagStr}${oomHint}${stderr ? `\nProcess output: ${stderr}` : ""}`- );- }+ async _killCurrentProcess() {+ if (!this.process) return;- pollCount++;- if (await this.checkHealth()) {- this.ready = true;- debugLogger.debug("llama-server ready", {- startupTimeMs: Date.now() - startTime,- pollCount,- });- return;- }+ this.stopHealthCheck();- await new Promise((resolve) => setTimeout(resolve, STARTUP_POLL_INTERVAL_MS));+ try {+ killProcess(this.process, "SIGTERM");+ await new Promise((resolve) => {+ const timeout = setTimeout(() => {+ if (this.process) killProcess(this.process, "SIGKILL");+ resolve();+ }, 5000);++ if (this.process) {+ this.process.once("close", () => {+ clearTimeout(timeout);+ resolve();+ });+ } else {+ clearTimeout(timeout);+ resolve();+ }+ });+ } catch (error) {+ debugLogger.error("Error killing llama-server process", { error: error.message });}- throw new Error(`llama-server failed to start within ${STARTUP_TIMEOUT_MS}ms`);+ this.process = null;+ this.ready = false;+ }++ _loadGpuBackend() {+ const val = process.env.LLAMA_GPU_BACKEND;+ if (val === "vulkan" || val === "cpu" || val === "metal") return val;+ return null;+ }++ _saveGpuBackend(backend) {+ process.env.LLAMA_GPU_BACKEND = backend;}checkHealth() {@@ -315,7 +447,7 @@ class LlamaServerManager {"Content-Type": "application/json","Content-Length": Buffer.byteLength(body),},- timeout: 300000, // 5 minute timeout for inference+ timeout: 300000,},(res) => {let data = "";@@ -335,7 +467,6 @@ class LlamaServerManager {try {const response = JSON.parse(data);- // Extract text from OpenAI-compatible responseconst text = response.choices?.[0]?.message?.content || "";resolve(text.trim());} catch (e) {@@ -397,6 +528,7 @@ class LlamaServerManager {this.ready = false;this.port = null;this.modelPath = null;+ this.activeBackend = null;}getStatus() {@@ -406,8 +538,16 @@ class LlamaServerManager {port: this.port,modelPath: this.modelPath,modelName: this.modelPath ? path.basename(this.modelPath, ".gguf") : null,+ backend: this.activeBackend,+ gpuAccelerated: this.activeBackend === "vulkan" || this.activeBackend === "metal",};}++ resetGpuDetection() {+ delete process.env.LLAMA_GPU_BACKEND;+ this.activeBackend = null;+ this.cachedServerBinaryPaths = null;+ }}module.exports = LlamaServerManager;diff --git a/src/helpers/llamaVulkanManager.js b/src/helpers/llamaVulkanManager.jsnew file mode 100644index 000000000..0e040ff76--- /dev/null+++ b/src/helpers/llamaVulkanManager.js@@ -0,0 +1,266 @@+const fs = require("fs");+const { promises: fsPromises } = require("fs");+const path = require("path");+const { execFile } = require("child_process");+const { app } = require("electron");+const debugLogger = require("./debugLogger");+const {+ downloadFile,+ createDownloadSignal,+ checkDiskSpace,+ cleanupStaleDownloads,+} = require("./downloadUtils");++const GITHUB_RELEASE_URL = "https://api.github.com/repos/ggerganov/llama.cpp/releases/latest";++const VULKAN_ASSETS = {+ "win32-x64": {+ assetPattern: /^llama-.*-bin-win-vulkan-x64\.zip$/,+ binaryName: "llama-server.exe",+ outputName: "llama-server-vulkan.exe",+ libPattern: /\.dll$/i,+ },+ "linux-x64": {+ assetPattern: /^llama-.*-bin-ubuntu-vulkan-x64\.tar\.gz$/,+ binaryName: "llama-server",+ outputName: "llama-server-vulkan",+ libPattern: /\.so(\.\d+)*$/,+ },+};++class LlamaVulkanManager {+ constructor() {+ this._binDir = null;+ this._downloadSignal = null;+ this._downloading = false;+ }++ get binDir() {+ if (!this._binDir) {+ this._binDir = path.join(app.getPath("userData"), "bin");+ }+ return this._binDir;+ }++ _getConfig() {+ return VULKAN_ASSETS[`${process.platform}-${process.arch}`] || null;+ }++ isSupported() {+ return this._getConfig() !== null;+ }++ getBinaryPath() {+ const config = this._getConfig();+ if (!config) return null;+ const p = path.join(this.binDir, config.outputName);+ try {+ if (fs.existsSync(p)) return p;+ } catch {}+ return null;+ }++ isDownloaded() {+ return this.getBinaryPath() !== null;+ }++ getStatus() {+ return {+ supported: this.isSupported(),+ downloaded: this.isDownloaded(),+ downloading: this._downloading,+ };+ }++ async download(onProgress) {+ if (this._downloading) throw new Error("Download already in progress");+ if (!this.isSupported()) throw new Error("Vulkan not available for this platform");++ this._downloading = true;+ const { signal, abort } = createDownloadSignal();+ this._downloadSignal = { abort };++ try {+ await fsPromises.mkdir(this.binDir, { recursive: true });+ await cleanupStaleDownloads(this.binDir);++ const release = await this._fetchJson(GITHUB_RELEASE_URL);+ if (!release?.assets) throw new Error("Could not fetch llama.cpp release info");++ const config = this._getConfig();+ const asset = release.assets.find((a) => config.assetPattern.test(a.name));+ if (!asset) throw new Error("Vulkan binary not found in latest release");++ const spaceCheck = await checkDiskSpace(this.binDir, (asset.size || 100_000_000) * 2.5);+ if (!spaceCheck.ok) {+ throw new Error(+ `Not enough disk space. Need ~${Math.round(((asset.size || 100_000_000) * 2.5) / 1_000_000)}MB, ` ++ `only ${Math.round(spaceCheck.availableBytes / 1_000_000)}MB available.`+ );+ }++ const archivePath = path.join(this.binDir, asset.name);+ await downloadFile(asset.browser_download_url, archivePath, {+ signal,+ expectedSize: asset.size,+ onProgress,+ });++ const extractDir = path.join(this.binDir, `temp-vulkan-${Date.now()}`);+ await fsPromises.mkdir(extractDir, { recursive: true });++ try {+ await this._extract(archivePath, extractDir);++ const binaryPath = this._findFile(extractDir, config.binaryName);+ if (!binaryPath) throw new Error(`${config.binaryName} not found in archive`);++ const outputPath = path.join(this.binDir, config.outputName);+ await fsPromises.copyFile(binaryPath, outputPath);+ if (process.platform !== "win32") await fsPromises.chmod(outputPath, 0o755);++ const libs = this._findFiles(extractDir, config.libPattern);+ for (const lib of libs) {+ const dest = path.join(this.binDir, path.basename(lib));+ await fsPromises.copyFile(lib, dest);+ if (process.platform !== "win32") await fsPromises.chmod(dest, 0o755);+ }++ debugLogger.info("Vulkan llama-server installed", { path: outputPath });+ } finally {+ await fsPromises.rm(extractDir, { recursive: true, force: true }).catch(() => {});+ await fsPromises.unlink(archivePath).catch(() => {});+ }++ return { success: true };+ } catch (error) {+ if (error.isAbort) return { success: false, cancelled: true };+ throw error;+ } finally {+ this._downloading = false;+ this._downloadSignal = null;+ }+ }++ cancelDownload() {+ if (this._downloadSignal) {+ this._downloadSignal.abort();+ this._downloadSignal = null;+ return true;+ }+ return false;+ }++ async deleteBinary() {+ const config = this._getConfig();+ if (!config) return { success: true };++ let deletedCount = 0;+ try {+ const entries = await fsPromises.readdir(this.binDir);+ for (const entry of entries) {+ if (entry === config.outputName || config.libPattern.test(entry)) {+ await fsPromises.unlink(path.join(this.binDir, entry)).catch(() => {});+ deletedCount++;+ }+ }+ } catch {}++ debugLogger.info("Vulkan llama-server deleted", { deletedCount });+ return { success: true, deletedCount };+ }++ _fetchJson(url) {+ const https = require("https");+ return new Promise((resolve, reject) => {+ const headers = {+ "User-Agent": "OpenWhispr/1.0",+ Accept: "application/vnd.github+json",+ };+ const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;+ if (token) headers.Authorization = `Bearer ${token}`;++ https+ .get(url, { headers, timeout: 15000 }, (res) => {+ if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {+ res.resume();+ this._fetchJson(res.headers.location).then(resolve, reject);+ return;+ }+ if (res.statusCode !== 200) {+ res.resume();+ reject(new Error(`GitHub API returned ${res.statusCode}`));+ return;+ }+ let data = "";+ res.on("data", (chunk) => (data += chunk));+ res.on("end", () => {+ try {+ resolve(JSON.parse(data));+ } catch {+ reject(new Error("Failed to parse GitHub release JSON"));+ }+ });+ })+ .on("error", reject);+ });+ }++ _extract(archivePath, destDir) {+ return new Promise((resolve, reject) => {+ if (archivePath.endsWith(".tar.gz") || archivePath.endsWith(".tgz")) {+ execFile("tar", ["-xzf", archivePath, "-C", destDir], (err) => {+ err ? reject(new Error(`Extraction failed: ${err.message}`)) : resolve();+ });+ } else if (process.platform === "win32") {+ execFile(+ "powershell",+ [+ "-NoProfile",+ "-Command",+ `Expand-Archive -Force -Path '${archivePath}' -DestinationPath '${destDir}'`,+ ],+ (err) => {+ err ? reject(new Error(`Extraction failed: ${err.message}`)) : resolve();+ }+ );+ } else {+ execFile("unzip", ["-o", archivePath, "-d", destDir], (err) => {+ err ? reject(new Error(`Extraction failed: ${err.message}`)) : resolve();+ });+ }+ });+ }++ _findFile(dir, name, maxDepth = 5, depth = 0) {+ if (depth >= maxDepth) return null;+ const entries = fs.readdirSync(dir, { withFileTypes: true });+ for (const entry of entries) {+ const full = path.join(dir, entry.name);+ if (entry.isDirectory()) {+ const found = this._findFile(full, name, maxDepth, depth + 1);+ if (found) return found;+ } else if (entry.name === name) {+ return full;+ }+ }+ return null;+ }++ _findFiles(dir, pattern, maxDepth = 5, depth = 0) {+ if (depth >= maxDepth) return [];+ const results = [];+ const entries = fs.readdirSync(dir, { withFileTypes: true });+ for (const entry of entries) {+ const full = path.join(dir, entry.name);+ if (entry.isDirectory()) {+ results.push(...this._findFiles(full, pattern, maxDepth, depth + 1));+ } else if (pattern.test(entry.name)) {+ results.push(full);+ }+ }+ return results;+ }+}++module.exports = LlamaVulkanManager;diff --git a/src/helpers/modelManagerBridge.js b/src/helpers/modelManagerBridge.jsindex 16d3f906b..7ce5785c8 100644--- a/src/helpers/modelManagerBridge.js+++ b/src/helpers/modelManagerBridge.js@@ -369,6 +369,7 @@ class ModelManager {await this.serverManager.start(modelPath, {contextSize: options.contextSize || modelInfo.model.contextLength || 4096,threads: options.threads || 4,+ gpuLayers: 99,});this.currentServerModelId = modelId;@@ -441,6 +442,7 @@ class ModelManager {await this.serverManager.start(modelPath, {contextSize: modelInfo.model.contextLength || 4096,threads: 4,+ gpuLayers: 99,});this.currentServerModelId = modelId;debugLogger.info("llama-server pre-warmed", { modelId });diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.tsindex da311fe5e..4e9c53d3a 100644--- a/src/services/ReasoningService.ts+++ b/src/services/ReasoningService.ts@@ -1034,10 +1034,8 @@ class ReasoningService extends BaseReasoningService {const language = this.getPreferredLanguage();const locale = this.getUiLanguage();- // Use withSessionRefresh to handle AUTH_EXPIRED automaticallyconst result = await withSessionRefresh(async () => {const res = await (window as any).electronAPI.cloudReason(text, {- model,agentName,customDictionary,customPrompt: this.getCustomPrompt(),@@ -1086,6 +1084,11 @@ class ReasoningService extends BaseReasoningService {async isAvailable(): Promise<boolean> {try {+ if (isCloudReasoningMode()) {+ logger.logReasoning("API_KEY_CHECK", { cloudReasoningMode: true });+ return true;+ }+const openaiKey = await window.electronAPI?.getOpenAIKey?.();const anthropicKey = await window.electronAPI?.getAnthropicKey?.();const geminiKey = await window.electronAPI?.getGeminiKey?.();diff --git a/src/stores/settingsStore.ts b/src/stores/settingsStore.tsindex fb2e205cb..fae211f3f 100644--- a/src/stores/settingsStore.ts+++ b/src/stores/settingsStore.ts@@ -16,12 +16,6 @@ import type {} from "../hooks/useSettings";let _ReasoningService: typeof import("../services/ReasoningService").default | null = null;-function getReasoningService() {- if (!_ReasoningService) {- _ReasoningService = require("../services/ReasoningService").default;- }- return _ReasoningService!;-}const isBrowser = typeof window !== "undefined";@@ -171,7 +165,16 @@ function invalidateApiKeyCaches(provider?: "openai" | "anthropic" | "gemini" | "groq" | "mistral" | "custom") {if (provider) {- getReasoningService().clearApiKeyCache(provider);+ if (_ReasoningService) {+ _ReasoningService.clearApiKeyCache(provider);+ } else {+ import("../services/ReasoningService")+ .then((mod) => {+ _ReasoningService = mod.default;+ _ReasoningService.clearApiKeyCache(provider);+ })+ .catch(() => {});+ }}if (isBrowser) window.dispatchEvent(new Event("api-key-changed"));debouncedPersistToEnv();@@ -441,7 +444,11 @@ export function getSettings() {}export function getEffectiveReasoningModel() {- return useSettingsStore.getState().reasoningModel;+ const state = useSettingsStore.getState();+ if (selectIsCloudReasoningMode(state)) {+ return "";+ }+ return state.reasoningModel;}export function isCloudReasoningMode() {diff --git a/src/types/electron.ts b/src/types/electron.tsindex 4e3296092..60e02d070 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -218,6 +218,38 @@ export interface PasteToolsResult {recommendedInstall?: string;}+export type GpuBackend = "vulkan" | "cpu" | "metal" | null;++export interface LlamaServerStatus {+ available: boolean;+ running: boolean;+ port: number | null;+ modelPath: string | null;+ modelName: string | null;+ backend: GpuBackend;+ gpuAccelerated: boolean;+}++export interface VulkanGpuResult {+ available: boolean;+ deviceName?: string;+ reason?: string;+ error?: string;+}++export interface LlamaVulkanStatus {+ supported: boolean;+ downloaded: boolean;+ downloading?: boolean;+ error?: string;+}++export interface LlamaVulkanDownloadProgress {+ downloaded: number;+ total: number;+ percentage: number;+}+export interface ReferralItem {id: string;email: string;@@ -484,6 +516,30 @@ declare global {llamaCppInstall: () => Promise<{ success: boolean; error?: string }>;llamaCppUninstall: () => Promise<{ success: boolean; error?: string }>;+ // llama-server+ llamaServerStart: (+ modelId: string+ ) => Promise<{ success: boolean; port?: number; error?: string }>;+ llamaServerStop: () => Promise<{ success: boolean; error?: string }>;+ llamaServerStatus: () => Promise<LlamaServerStatus>;+ llamaGpuReset: () => Promise<{ success: boolean; error?: string }>;+ detectVulkanGpu?: () => Promise<VulkanGpuResult>;+ getLlamaVulkanStatus?: () => Promise<LlamaVulkanStatus>;+ downloadLlamaVulkanBinary?: () => Promise<{+ success: boolean;+ cancelled?: boolean;+ error?: string;+ }>;+ cancelLlamaVulkanDownload?: () => Promise<{ success: boolean }>;+ deleteLlamaVulkanBinary?: () => Promise<{+ success: boolean;+ deletedCount?: number;+ error?: string;+ }>;+ onLlamaVulkanDownloadProgress?: (+ callback: (data: LlamaVulkanDownloadProgress) => void+ ) => () => void;+// Window control operationswindowMinimize: () => Promise<void>;windowMaximize: () => Promise<void>;diff --git a/src/utils/vulkanDetection.js b/src/utils/vulkanDetection.jsnew file mode 100644index 000000000..9540716ac--- /dev/null+++ b/src/utils/vulkanDetection.js@@ -0,0 +1,61 @@+const { app } = require("electron");++let cachedResult = null;++const VULKAN_VENDOR_IDS = new Set([+ 0x10de, // NVIDIA+ 0x1002, // AMD+ 0x8086, // Intel+]);++const VENDOR_NAMES = {+ 0x10de: "NVIDIA",+ 0x1002: "AMD",+ 0x8086: "Intel",+};++function parseDeviceName(gpuInfo, activeGpu) {+ const renderer = gpuInfo.auxAttributes?.glRenderer;+ if (renderer) {+ const match = renderer.match(/ANGLE\s*\([^,]*,\s*([^,)]+)/);+ if (match) return match[1].trim();+ }+ if (activeGpu?.vendorId) {+ return VENDOR_NAMES[activeGpu.vendorId] || `GPU (vendor ${activeGpu.vendorId})`;+ }+ return null;+}++async function detectVulkanGpu() {+ if (cachedResult) return cachedResult;++ if (process.platform === "darwin") {+ cachedResult = { available: false };+ return cachedResult;+ }++ try {+ const gpuInfo = await app.getGPUInfo("complete");+ const activeGpu = gpuInfo.gpuDevice?.find((d) => d.active) || gpuInfo.gpuDevice?.[0];++ if (!activeGpu || !VULKAN_VENDOR_IDS.has(activeGpu.vendorId)) {+ cachedResult = { available: false };+ return cachedResult;+ }++ cachedResult = {+ available: true,+ deviceName: parseDeviceName(gpuInfo, activeGpu),+ };+ return cachedResult;+ } catch {+ cachedResult = { available: false };+ return cachedResult;+ }+}++function clearCache() {+ cachedResult = null;+}++module.exports = { detectVulkanGpu, clearCache };