Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat: GPU-accelerated local inference (Vulkan/Metal) (#315)

Gabriel Stein898887628666
diff --git a/preload.js b/preload.js
index aa002f517..d52819c18 100644
--- a/preload.js
+++ b/preload.js
@@ -296,6 +296,18 @@ contextBridge.exposeInMainWorld("electronAPI", {
llamaServerStart: (modelId) => ipcRenderer.invoke("llama-server-start", modelId),
llamaServerStop: () => ipcRenderer.invoke("llama-server-stop"),
llamaServerStatus: () => ipcRenderer.invoke("llama-server-status"),
+ llamaGpuReset: () => ipcRenderer.invoke("llama-gpu-reset"),
+
+ // Vulkan GPU acceleration
+ detectVulkanGpu: () => ipcRenderer.invoke("detect-vulkan-gpu"),
+ getLlamaVulkanStatus: () => ipcRenderer.invoke("get-llama-vulkan-status"),
+ downloadLlamaVulkanBinary: () => ipcRenderer.invoke("download-llama-vulkan-binary"),
+ cancelLlamaVulkanDownload: () => ipcRenderer.invoke("cancel-llama-vulkan-download"),
+ deleteLlamaVulkanBinary: () => ipcRenderer.invoke("delete-llama-vulkan-binary"),
+ onLlamaVulkanDownloadProgress: registerListener(
+ "llama-vulkan-download-progress",
+ (callback) => (_event, data) => callback(data)
+ ),
getLogLevel: () => ipcRenderer.invoke("get-log-level"),
log: (entry) => ipcRenderer.invoke("app-log", entry),
diff --git a/scripts/download-llama-server.js b/scripts/download-llama-server.js
index a87219153..64120323c 100644
--- a/scripts/download-llama-server.js
+++ b/scripts/download-llama-server.js
@@ -16,37 +16,39 @@ const LLAMA_CPP_REPO = "ggerganov/llama.cpp";
// Version can be pinned via environment variable for reproducible builds
const VERSION_OVERRIDE = process.env.LLAMA_CPP_VERSION || null;
-// Asset name patterns to match in the release (version-independent)
const BINARIES = {
"darwin-arm64": {
+ platformArch: "darwin-arm64",
assetPattern: /^llama-.*-bin-macos-arm64\.tar\.gz$/,
binaryPath: "build/bin/llama-server",
outputName: "llama-server-darwin-arm64",
libPattern: "*.dylib",
},
"darwin-x64": {
+ platformArch: "darwin-x64",
assetPattern: /^llama-.*-bin-macos-x64\.tar\.gz$/,
binaryPath: "build/bin/llama-server",
outputName: "llama-server-darwin-x64",
libPattern: "*.dylib",
},
- "win32-x64": {
+ "win32-x64-cpu": {
+ platformArch: "win32-x64",
assetPattern: /^llama-.*-bin-win-cpu-x64\.zip$/,
binaryPath: "build/bin/llama-server.exe",
- outputName: "llama-server-win32-x64.exe",
+ outputName: "llama-server-win32-x64-cpu.exe",
libPattern: "*.dll",
},
- "linux-x64": {
+ "linux-x64-cpu": {
+ platformArch: "linux-x64",
assetPattern: /^llama-.*-bin-ubuntu-x64\.tar\.gz$/,
binaryPath: "build/bin/llama-server",
- outputName: "llama-server-linux-x64",
+ outputName: "llama-server-linux-x64-cpu",
libPattern: "*.so*",
},
};
const BIN_DIR = path.join(__dirname, "..", "resources", "bin");
-// Cache the release info to avoid multiple API calls
let cachedRelease = null;
async function getRelease() {
@@ -84,45 +86,43 @@ function findLibrariesInDir(dir, pattern, maxDepth = 5, currentDepth = 0) {
}
function matchesPattern(filename, pattern) {
- // Handle patterns like "*.dylib", "*.dll", "*.so*"
if (pattern === "*.dylib") {
return filename.endsWith(".dylib");
} else if (pattern === "*.dll") {
return filename.endsWith(".dll");
} else if (pattern === "*.so*") {
- // Match .so files with optional version suffix (e.g., libfoo.so, libfoo.so.1, libfoo.so.1.2.3)
return /\.so(\.\d+)*$/.test(filename) || filename.endsWith(".so");
}
return false;
}
-async function downloadBinary(platformArch, config, release, isForce = false) {
+async function downloadBinary(key, config, release, isForce = false) {
if (!config) {
- console.log(` ${platformArch}: Not supported`);
+ console.log(` ${key}: Not supported`);
return false;
}
const outputPath = path.join(BIN_DIR, config.outputName);
if (fs.existsSync(outputPath) && !isForce) {
- console.log(` ${platformArch}: Already exists (use --force to re-download)`);
+ console.log(` ${key}: Already exists (use --force to re-download)`);
return true;
}
const asset = findAsset(release, config.assetPattern);
if (!asset) {
- console.error(` ${platformArch}: No matching asset found for pattern ${config.assetPattern}`);
+ console.error(` ${key}: No matching asset found for pattern ${config.assetPattern}`);
return false;
}
- console.log(` ${platformArch}: Downloading from ${asset.url}`);
+ console.log(` ${key}: Downloading from ${asset.url}`);
const zipPath = path.join(BIN_DIR, asset.name);
try {
await downloadFile(asset.url, zipPath);
- const extractDir = path.join(BIN_DIR, `temp-llama-${platformArch}`);
+ const extractDir = path.join(BIN_DIR, `temp-llama-${key}`);
fs.mkdirSync(extractDir, { recursive: true });
await extractArchive(zipPath, extractDir);
@@ -136,10 +136,8 @@ async function downloadBinary(platformArch, config, release, isForce = false) {
if (binaryPath && fs.existsSync(binaryPath)) {
fs.copyFileSync(binaryPath, outputPath);
setExecutable(outputPath);
- console.log(` ${platformArch}: Extracted to ${config.outputName}`);
+ console.log(` ${key}: Extracted to ${config.outputName}`);
- // Copy shared libraries (dylib/dll/so files)
- // Always overwrite — libraries are architecture-specific (e.g. arm64 vs x64 dylibs)
if (config.libPattern) {
const libraries = findLibrariesInDir(extractDir, config.libPattern);
@@ -149,11 +147,11 @@ async function downloadBinary(platformArch, config, release, isForce = false) {
fs.copyFileSync(libPath, destPath);
setExecutable(destPath);
- console.log(` ${platformArch}: Copied library ${libName}`);
+ console.log(` ${key}: Copied library ${libName}`);
}
}
} else {
- console.error(` ${platformArch}: Binary '${binaryName}' not found in archive`);
+ console.error(` ${key}: Binary '${binaryName}' not found in archive`);
return false;
}
@@ -161,12 +159,16 @@ async function downloadBinary(platformArch, config, release, isForce = false) {
if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath);
return true;
} catch (error) {
- console.error(` ${platformArch}: Failed - ${error.message}`);
+ console.error(` ${key}: Failed - ${error.message}`);
if (fs.existsSync(zipPath)) fs.unlinkSync(zipPath);
return false;
}
}
+function getEntriesForPlatformArch(platformArch) {
+ return Object.entries(BINARIES).filter(([, config]) => config.platformArch === platformArch);
+}
+
async function main() {
if (VERSION_OVERRIDE) {
console.log(`\n[llama-server] Using pinned version: ${VERSION_OVERRIDE}`);
@@ -189,18 +191,25 @@ async function main() {
const args = parseArgs();
if (args.isCurrent) {
- if (!BINARIES[args.platformArch]) {
+ const entries = getEntriesForPlatformArch(args.platformArch);
+
+ if (entries.length === 0) {
console.error(`Unsupported platform/arch: ${args.platformArch}`);
process.exitCode = 1;
return;
}
console.log(`Downloading for target platform (${args.platformArch}):`);
- const ok = await downloadBinary(args.platformArch, BINARIES[args.platformArch], release, args.isForce);
- if (!ok) {
- console.error(`Failed to download binaries for ${args.platformArch}`);
- process.exitCode = 1;
- return;
+
+ for (const [key, config] of entries) {
+ const ok = await downloadBinary(key, config, release, args.isForce);
+ if (!ok && config.optional) {
+ console.warn(` ${key}: Skipping optional variant`);
+ } else if (!ok) {
+ console.error(`Failed to download binaries for ${key}`);
+ process.exitCode = 1;
+ return;
+ }
}
if (args.shouldCleanup) {
@@ -208,8 +217,11 @@ async function main() {
}
} else {
console.log("Downloading binaries for all platforms:");
- for (const platformArch of Object.keys(BINARIES)) {
- await downloadBinary(platformArch, BINARIES[platformArch], release, args.isForce);
+ for (const [key, config] of Object.entries(BINARIES)) {
+ const ok = await downloadBinary(key, config, release, args.isForce);
+ if (!ok && config.optional) {
+ console.warn(` ${key}: Skipping optional variant`);
+ }
}
}
diff --git a/src/components/ControlPanel.tsx b/src/components/ControlPanel.tsx
index 1bc89686e..7f25455e0 100644
--- a/src/components/ControlPanel.tsx
+++ b/src/components/ControlPanel.tsx
@@ -45,6 +45,9 @@ export default function ControlPanel() {
);
const [showReferrals, setShowReferrals] = useState(false);
const [showCloudMigrationBanner, setShowCloudMigrationBanner] = useState(false);
+ const [vulkanBanner, setVulkanBanner] = useState<{ show: boolean; deviceName?: string }>({
+ show: false,
+ });
const [activeView, setActiveView] = useState<ControlPanelView>("home");
const [cudaStatus, setCudaStatus] = useState<CudaWhisperStatus | null>(null);
const [gpuBannerDismissed, setGpuBannerDismissed] = useState(
@@ -153,6 +156,22 @@ export default function ControlPanel() {
setShowCloudMigrationBanner(true);
}, [authLoaded, isSignedIn, setUseLocalWhisper, setCloudTranscriptionMode]);
+ useEffect(() => {
+ if (platform === "darwin" || !useReasoningModel) return;
+ if (localStorage.getItem("llamaVulkanBannerDismissed") === "true") return;
+
+ Promise.all([
+ window.electronAPI?.detectVulkanGpu?.(),
+ window.electronAPI?.getLlamaVulkanStatus?.(),
+ ])
+ .then(([gpu, vulkan]) => {
+ if (gpu?.available && !vulkan?.downloaded) {
+ setVulkanBanner({ show: true, deviceName: gpu.deviceName });
+ }
+ })
+ .catch(() => {});
+ }, [useReasoningModel]);
+
const fetchCudaStatus = useCallback(() => {
if (platform === "darwin" || gpuBannerDismissed) return;
if (!useLocalWhisper || localTranscriptionProvider !== "whisper") return;
@@ -431,6 +450,41 @@ export default function ControlPanel() {
</div>
</div>
)}
+ {vulkanBanner.show && activeView === "home" && (
+ <div className="mx-4 mt-3 p-3 rounded-lg bg-purple-500/10 border border-purple-500/20">
+ <div className="flex items-start gap-2">
+ <Zap className="w-4 h-4 text-purple-400 shrink-0 mt-0.5" />
+ <div className="flex-1 min-w-0">
+ <p className="text-sm text-foreground">
+ {vulkanBanner.deviceName || "GPU"} detected — speed up local AI with Vulkan
+ acceleration
+ </p>
+ <div className="flex items-center gap-3 mt-2">
+ <button
+ type="button"
+ onClick={() => {
+ setSettingsSection("reasoning");
+ setShowSettings(true);
+ }}
+ className="text-xs font-medium text-purple-400 hover:text-purple-300 transition-colors"
+ >
+ Enable GPU
+ </button>
+ <button
+ type="button"
+ onClick={() => {
+ localStorage.setItem("llamaVulkanBannerDismissed", "true");
+ setVulkanBanner({ show: false });
+ }}
+ className="text-xs text-muted-foreground hover:text-foreground transition-colors"
+ >
+ Not now
+ </button>
+ </div>
+ </div>
+ </div>
+ </div>
+ )}
{showGpuBanner && (
<div className="max-w-3xl mx-auto w-full mb-3">
<div className="rounded-lg border border-primary/20 dark:border-primary/15 bg-primary/5 p-3">
diff --git a/src/components/ReasoningModelSelector.tsx b/src/components/ReasoningModelSelector.tsx
index a6f9c490d..4d343732a 100644
--- a/src/components/ReasoningModelSelector.tsx
+++ b/src/components/ReasoningModelSelector.tsx
@@ -1,5 +1,11 @@
import { useState, useEffect, useCallback, useMemo, useRef } from "react";
import { useTranslation } from "react-i18next";
+import type {
+ LlamaServerStatus,
+ LlamaVulkanStatus,
+ VulkanGpuResult,
+ LlamaVulkanDownloadProgress,
+} from "../types/electron";
import { Button } from "./ui/button";
import { Input } from "./ui/input";
import { Cloud, Lock } from "lucide-react";
@@ -14,6 +20,7 @@ import { modelRegistry } from "../models/ModelRegistry";
import { getProviderIcon, isMonochromeProvider } from "../utils/providerIcons";
import { isSecureEndpoint } from "../utils/urlUtils";
import { createExternalLinkHandler } from "../utils/externalLinks";
+import { getCachedPlatform } from "../utils/platform";
type CloudModelOption = {
value: string;
@@ -68,6 +75,173 @@ interface ReasoningModelSelectorProps {
setCustomReasoningApiKey?: (key: string) => void;
}
+function GpuStatusBadge() {
+ const [serverStatus, setServerStatus] = useState<LlamaServerStatus | null>(null);
+ const [vulkanStatus, setVulkanStatus] = useState<LlamaVulkanStatus | null>(null);
+ const [gpuResult, setGpuResult] = useState<VulkanGpuResult | null>(null);
+ const [progress, setProgress] = useState<LlamaVulkanDownloadProgress | null>(null);
+ const [downloading, setDownloading] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+ const platform = getCachedPlatform();
+
+ useEffect(() => {
+ const poll = () => {
+ window.electronAPI
+ ?.llamaServerStatus?.()
+ .then(setServerStatus)
+ .catch(() => {});
+ if (platform !== "darwin") {
+ window.electronAPI
+ ?.getLlamaVulkanStatus?.()
+ .then(setVulkanStatus)
+ .catch(() => {});
+ }
+ };
+ poll();
+ const id = setInterval(poll, 5000);
+ return () => clearInterval(id);
+ }, [platform]);
+
+ useEffect(() => {
+ if (platform !== "darwin") {
+ window.electronAPI
+ ?.detectVulkanGpu?.()
+ .then(setGpuResult)
+ .catch(() => {});
+ }
+ }, [platform]);
+
+ useEffect(() => {
+ const cleanup = window.electronAPI?.onLlamaVulkanDownloadProgress?.((data) => {
+ setProgress(data);
+ });
+ return () => cleanup?.();
+ }, []);
+
+ const handleDownload = async () => {
+ setDownloading(true);
+ setError(null);
+ try {
+ const result = await window.electronAPI?.downloadLlamaVulkanBinary?.();
+ if (result?.success) {
+ setVulkanStatus((prev) => (prev ? { ...prev, downloaded: true } : prev));
+ window.electronAPI?.llamaGpuReset?.();
+ } else if (result && !result.cancelled) {
+ setError(result.error || "Download failed");
+ }
+ } catch (err) {
+ setError(err instanceof Error ? err.message : "Download failed");
+ } finally {
+ setDownloading(false);
+ setProgress(null);
+ }
+ };
+
+ const handleDelete = async () => {
+ await window.electronAPI?.deleteLlamaVulkanBinary?.();
+ setVulkanStatus((prev) => (prev ? { ...prev, downloaded: false } : prev));
+ };
+
+ if (platform === "darwin") {
+ if (!serverStatus?.running) return null;
+ return (
+ <div className="flex items-center gap-1.5 mt-2 px-1">
+ <span
+ className="inline-block w-1.5 h-1.5 rounded-full shrink-0"
+ style={{ backgroundColor: "#22c55e" }}
+ />
+ <span className="text-xs text-muted-foreground">GPU Accelerated (Metal)</span>
+ </div>
+ );
+ }
+
+ if (downloading && progress) {
+ return (
+ <div className="flex items-center gap-2 mt-2 px-1">
+ <div className="flex-1 h-1 bg-muted rounded-full overflow-hidden">
+ <div
+ className="h-full bg-purple-500 transition-all"
+ style={{ width: `${progress.percentage}%` }}
+ />
+ </div>
+ <span className="text-xs text-muted-foreground tabular-nums">{progress.percentage}%</span>
+ <button
+ type="button"
+ onClick={() => window.electronAPI?.cancelLlamaVulkanDownload?.()}
+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors"
+ >
+ Cancel
+ </button>
+ </div>
+ );
+ }
+
+ if (error) {
+ return (
+ <div className="flex items-center gap-1.5 mt-2 px-1">
+ <span className="text-xs text-destructive">{error}</span>
+ <button
+ type="button"
+ onClick={() => setError(null)}
+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-1"
+ >
+ Dismiss
+ </button>
+ </div>
+ );
+ }
+
+ if (vulkanStatus?.downloaded) {
+ const isGpu = serverStatus?.gpuAccelerated && serverStatus?.backend === "vulkan";
+ return (
+ <div className="flex items-center gap-1.5 mt-2 px-1">
+ <span
+ className="inline-block w-1.5 h-1.5 rounded-full shrink-0"
+ style={{ backgroundColor: isGpu ? "#22c55e" : "#f59e0b" }}
+ />
+ <span className="text-xs text-muted-foreground">
+ {isGpu ? "GPU Accelerated (Vulkan)" : "CPU Only"}
+ </span>
+ {!isGpu && serverStatus?.running && (
+ <button
+ type="button"
+ onClick={() => window.electronAPI?.llamaGpuReset?.()}
+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-1"
+ >
+ Re-detect GPU
+ </button>
+ )}
+ <button
+ type="button"
+ onClick={handleDelete}
+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-auto"
+ >
+ Remove
+ </button>
+ </div>
+ );
+ }
+
+ if (gpuResult?.available) {
+ return (
+ <div className="flex items-center gap-1.5 mt-2 px-1">
+ <span className="text-xs text-muted-foreground">
+ {gpuResult.deviceName || "GPU"} detected
+ </span>
+ <button
+ type="button"
+ onClick={handleDownload}
+ className="text-xs text-muted-foreground underline decoration-muted-foreground/30 hover:decoration-muted-foreground/60 hover:text-foreground transition-colors ml-1"
+ >
+ Download Vulkan (~40MB)
+ </button>
+ </div>
+ );
+ }
+
+ return null;
+}
+
export default function ReasoningModelSelector({
reasoningModel,
setReasoningModel,
@@ -746,16 +920,19 @@ export default function ReasoningModelSelector({
</div>
</div>
) : (
- <LocalModelPicker
- providers={localProviders}
- selectedModel={reasoningModel}
- selectedProvider={selectedLocalProvider}
- onModelSelect={setReasoningModel}
- onProviderSelect={handleLocalProviderChange}
- modelType="llm"
- colorScheme="purple"
- onDownloadComplete={loadDownloadedModels}
- />
+ <>
+ <LocalModelPicker
+ providers={localProviders}
+ selectedModel={reasoningModel}
+ selectedProvider={selectedLocalProvider}
+ onModelSelect={setReasoningModel}
+ onProviderSelect={handleLocalProviderChange}
+ modelType="llm"
+ colorScheme="purple"
+ onDownloadComplete={loadDownloadedModels}
+ />
+ <GpuStatusBadge />
+ </>
)}
</div>
);
diff --git a/src/components/notes/UploadAudioView.tsx b/src/components/notes/UploadAudioView.tsx
index e62a0d7b0..8165881b8 100644
--- a/src/components/notes/UploadAudioView.tsx
+++ b/src/components/notes/UploadAudioView.tsx
@@ -32,7 +32,7 @@ import { useSettings } from "../../hooks/useSettings";
import { withSessionRefresh } from "../../lib/neonAuth";
import reasoningService from "../../services/ReasoningService";
import { getAllReasoningModels } from "../../models/ModelRegistry";
-import { useSettingsStore } from "../../stores/settingsStore";
+import { useSettingsStore, selectIsCloudReasoningMode } from "../../stores/settingsStore";
const TranscriptionModelPicker = React.lazy(() => import("../TranscriptionModelPicker"));
@@ -110,7 +110,10 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
updateTranscriptionSettings,
} = useSettings();
- const effectiveReasoningModel = useSettingsStore((s) => s.reasoningModel);
+ const isCloudReasoning = useSettingsStore(selectIsCloudReasoningMode);
+ const effectiveReasoningModel = useSettingsStore((s) =>
+ selectIsCloudReasoningMode(s) ? "" : s.reasoningModel
+ );
const useReasoningModel = useSettingsStore((s) => s.useReasoningModel);
const isOpenWhisprCloud =
@@ -213,8 +216,10 @@ export default function UploadAudioView({ onNoteCreated, onOpenSettings }: Uploa
const generateTitle = async (text: string): Promise<string> => {
if (!useReasoningModel) return "";
- const model = effectiveReasoningModel || getAllReasoningModels()[0]?.value;
- if (!model) return "";
+ const model = isCloudReasoning
+ ? ""
+ : effectiveReasoningModel || getAllReasoningModels()[0]?.value;
+ if (!model && !isCloudReasoning) return "";
try {
const title = await reasoningService.processText(text.slice(0, 2000), model, null, {
systemPrompt: TITLE_SYSTEM_PROMPT,
diff --git a/src/helpers/audioManager.js b/src/helpers/audioManager.js
index 71f609a77..4b60a4459 100644
--- a/src/helpers/audioManager.js
+++ b/src/helpers/audioManager.js
@@ -5,7 +5,11 @@ import { isBuiltInMicrophone } from "../utils/audioDeviceUtils";
import { isSecureEndpoint } from "../utils/urlUtils";
import { withSessionRefresh } from "../lib/neonAuth";
import { getBaseLanguageCode, validateLanguageForModel } from "../utils/languageSupport";
-import { getSettings, getEffectiveReasoningModel } from "../stores/settingsStore";
+import {
+ getSettings,
+ getEffectiveReasoningModel,
+ isCloudReasoningMode,
+} from "../stores/settingsStore";
const SHORT_CLIP_DURATION_SECONDS = 2.5;
const REASONING_CACHE_TTL = 30000; // 30 seconds
@@ -804,6 +808,15 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
return false;
}
+ if (isCloudReasoningMode()) {
+ this.reasoningAvailabilityCache = {
+ value: true,
+ expiresAt: now + REASONING_CACHE_TTL,
+ };
+ this.cachedReasoningPreference = useReasoning;
+ return true;
+ }
+
try {
const isAvailable = await ReasoningService.isAvailable();
@@ -846,12 +859,13 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
});
const reasoningModel = getEffectiveReasoningModel();
+ const isCloud = isCloudReasoningMode();
const reasoningProvider = getSettings().reasoningProvider || "auto";
const agentName =
typeof window !== "undefined" && window.localStorage
? localStorage.getItem("agentName") || null
: null;
- if (!reasoningModel) {
+ if (!reasoningModel && !isCloud) {
logger.logReasoning("REASONING_SKIPPED", {
reason: "No reasoning model selected",
});
diff --git a/src/helpers/environment.js b/src/helpers/environment.js
index 7b983861e..e9ca5e741 100644
--- a/src/helpers/environment.js
+++ b/src/helpers/environment.js
@@ -17,6 +17,8 @@ const PERSISTED_KEYS = [
"LOCAL_WHISPER_MODEL",
"REASONING_PROVIDER",
"LOCAL_REASONING_MODEL",
+ "LLAMA_GPU_BACKEND",
+ "LLAMA_VULKAN_ENABLED",
"DICTATION_KEY",
"ACTIVATION_MODE",
"FLOATING_ICON_AUTO_HIDE",
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 2f67d5a57..c28314afe 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -1332,9 +1332,11 @@ class IPCHandlers {
await modelManager.serverManager.start(modelPath, {
contextSize: modelInfo.model.contextLength || 4096,
threads: 4,
+ gpuLayers: 99,
});
modelManager.currentServerModelId = modelId;
+ this.environmentManager.saveAllKeysToEnvFile().catch(() => {});
return { success: true, port: modelManager.serverManager.port };
} catch (error) {
return { success: false, error: error.message };
@@ -1360,6 +1362,101 @@ class IPCHandlers {
}
});
+ ipcMain.handle("llama-gpu-reset", async () => {
+ try {
+ const modelManager = require("./modelManagerBridge").default;
+ modelManager.serverManager.resetGpuDetection();
+ await modelManager.stopServer();
+ return { success: true };
+ } catch (error) {
+ return { success: false, error: error.message };
+ }
+ });
+
+ ipcMain.handle("detect-vulkan-gpu", async () => {
+ try {
+ const { detectVulkanGpu } = require("../utils/vulkanDetection");
+ return await detectVulkanGpu();
+ } catch (error) {
+ return { available: false, error: error.message };
+ }
+ });
+
+ ipcMain.handle("get-llama-vulkan-status", async () => {
+ try {
+ if (!this._llamaVulkanManager) {
+ const LlamaVulkanManager = require("./llamaVulkanManager");
+ this._llamaVulkanManager = new LlamaVulkanManager();
+ }
+ return this._llamaVulkanManager.getStatus();
+ } catch (error) {
+ return { supported: false, downloaded: false, error: error.message };
+ }
+ });
+
+ ipcMain.handle("download-llama-vulkan-binary", async (event) => {
+ try {
+ if (!this._llamaVulkanManager) {
+ const LlamaVulkanManager = require("./llamaVulkanManager");
+ this._llamaVulkanManager = new LlamaVulkanManager();
+ }
+
+ const result = await this._llamaVulkanManager.download((downloaded, total) => {
+ if (!event.sender.isDestroyed()) {
+ event.sender.send("llama-vulkan-download-progress", {
+ downloaded,
+ total,
+ percentage: total > 0 ? Math.round((downloaded / total) * 100) : 0,
+ });
+ }
+ });
+
+ if (result.success) {
+ process.env.LLAMA_VULKAN_ENABLED = "true";
+ delete process.env.LLAMA_GPU_BACKEND;
+ const modelManager = require("./modelManagerBridge").default;
+ modelManager.serverManager.cachedServerBinaryPaths = null;
+ this.environmentManager.saveAllKeysToEnvFile().catch(() => {});
+ }
+
+ return result;
+ } catch (error) {
+ return { success: false, error: error.message };
+ }
+ });
+
+ ipcMain.handle("cancel-llama-vulkan-download", async () => {
+ if (this._llamaVulkanManager) {
+ return { success: this._llamaVulkanManager.cancelDownload() };
+ }
+ return { success: false };
+ });
+
+ ipcMain.handle("delete-llama-vulkan-binary", async () => {
+ try {
+ if (!this._llamaVulkanManager) {
+ const LlamaVulkanManager = require("./llamaVulkanManager");
+ this._llamaVulkanManager = new LlamaVulkanManager();
+ }
+
+ const modelManager = require("./modelManagerBridge").default;
+ if (modelManager.serverManager.activeBackend === "vulkan") {
+ await modelManager.stopServer();
+ }
+
+ const result = await this._llamaVulkanManager.deleteBinary();
+
+ delete process.env.LLAMA_VULKAN_ENABLED;
+ delete process.env.LLAMA_GPU_BACKEND;
+ modelManager.serverManager.cachedServerBinaryPaths = null;
+ this.environmentManager.saveAllKeysToEnvFile().catch(() => {});
+
+ return result;
+ } catch (error) {
+ return { success: false, error: error.message };
+ }
+ });
+
ipcMain.handle("get-log-level", async () => {
return debugLogger.getLevel();
});
diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.js
index 2d4512c01..0e9737666 100644
--- a/src/helpers/llamaServer.js
+++ b/src/helpers/llamaServer.js
@@ -6,10 +6,12 @@ const http = require("http");
const debugLogger = require("./debugLogger");
const { killProcess } = require("../utils/process");
const { getSafeTempDir } = require("./safeTempDir");
+const { app } = require("electron");
const PORT_RANGE_START = 8200;
const PORT_RANGE_END = 8220;
const STARTUP_TIMEOUT_MS = 60000;
+const VULKAN_STARTUP_TIMEOUT_MS = 15000;
const HEALTH_CHECK_INTERVAL_MS = 5000;
const HEALTH_CHECK_TIMEOUT_MS = 2000;
const STARTUP_POLL_INTERVAL_MS = 500;
@@ -24,52 +26,70 @@ class LlamaServerManager {
this.startupPromise = null;
this.healthCheckInterval = null;
this.healthCheckFailures = 0;
- this.cachedServerBinaryPath = null;
+ this.cachedServerBinaryPaths = null;
+ this.activeBackend = null;
}
- getServerBinaryPath() {
- if (this.cachedServerBinaryPath) return this.cachedServerBinaryPath;
+ getServerBinaryPaths() {
+ if (this.cachedServerBinaryPaths) return this.cachedServerBinaryPaths;
const platform = process.platform;
const arch = process.arch;
const platformArch = `${platform}-${arch}`;
- const binaryName =
- platform === "win32" ? `llama-server-${platformArch}.exe` : `llama-server-${platformArch}`;
- const genericName = platform === "win32" ? "llama-server.exe" : "llama-server";
+ const ext = platform === "win32" ? ".exe" : "";
- const candidates = [];
-
- // Production: check resourcesPath
- if (process.resourcesPath) {
- candidates.push(
- path.join(process.resourcesPath, "bin", binaryName),
- path.join(process.resourcesPath, "bin", genericName)
- );
- }
-
- // Development: check relative to this file
- candidates.push(
- path.join(__dirname, "..", "..", "resources", "bin", binaryName),
- path.join(__dirname, "..", "..", "resources", "bin", genericName)
- );
+ const resolveBinary = (name) => {
+ const candidates = [];
+ if (process.resourcesPath) {
+ candidates.push(path.join(process.resourcesPath, "bin", name));
+ }
+ candidates.push(path.join(__dirname, "..", "..", "resources", "bin", name));
- for (const candidate of candidates) {
- if (fs.existsSync(candidate)) {
+ for (const candidate of candidates) {
try {
- fs.statSync(candidate);
- this.cachedServerBinaryPath = candidate;
- return candidate;
+ if (fs.existsSync(candidate)) {
+ fs.statSync(candidate);
+ return candidate;
+ }
} catch {
// Can't access binary
}
}
+ return null;
+ };
+
+ let paths;
+
+ if (platform === "darwin") {
+ const defaultBin =
+ resolveBinary(`llama-server-${platformArch}`) || resolveBinary(`llama-server${ext}`);
+ paths = defaultBin ? { default: defaultBin } : {};
+ } else {
+ const userBinDir = path.join(app.getPath("userData"), "bin");
+ const vulkanName = `llama-server-vulkan${ext}`;
+ let vulkanBin = null;
+ try {
+ const vulkanPath = path.join(userBinDir, vulkanName);
+ if (fs.existsSync(vulkanPath)) vulkanBin = vulkanPath;
+ } catch {}
+
+ const cpuBin =
+ resolveBinary(`llama-server-${platformArch}-cpu${ext}`) ||
+ resolveBinary(`llama-server-${platformArch}${ext}`) ||
+ resolveBinary(`llama-server${ext}`);
+
+ paths = {};
+ if (vulkanBin) paths.vulkan = vulkanBin;
+ if (cpuBin) paths.cpu = cpuBin;
}
- return null;
+ this.cachedServerBinaryPaths = paths;
+ return paths;
}
isAvailable() {
- return this.getServerBinaryPath() !== null;
+ const paths = this.getServerBinaryPaths();
+ return Object.keys(paths).length > 0;
}
async findAvailablePort() {
@@ -94,10 +114,8 @@ class LlamaServerManager {
async start(modelPath, options = {}) {
if (this.startupPromise) return this.startupPromise;
- // Already running with same model
if (this.ready && this.modelPath === modelPath) return;
- // Stop existing server if running
if (this.process) {
await this.stop();
}
@@ -111,14 +129,14 @@ class LlamaServerManager {
}
async _doStart(modelPath, options = {}) {
- const serverBinary = this.getServerBinaryPath();
- if (!serverBinary) throw new Error("llama-server binary not found");
+ const binaryPaths = this.getServerBinaryPaths();
+ if (Object.keys(binaryPaths).length === 0) throw new Error("llama-server binary not found");
if (!fs.existsSync(modelPath)) throw new Error(`Model file not found: ${modelPath}`);
this.port = await this.findAvailablePort();
this.modelPath = modelPath;
- const args = [
+ const baseArgs = [
"--model",
modelPath,
"--host",
@@ -131,105 +149,219 @@ class LlamaServerManager {
String(options.threads || 4),
];
- // Add GPU layers if specified
- if (options.gpuLayers) {
- args.push("--n-gpu-layers", String(options.gpuLayers));
+ if (process.platform === "darwin") {
+ const args = [...baseArgs, "--n-gpu-layers", "99"];
+ await this._startWithBinary(
+ binaryPaths.default,
+ args,
+ this._buildEnv(binaryPaths.default),
+ STARTUP_TIMEOUT_MS
+ );
+ this.activeBackend = "metal";
+ this._saveGpuBackend("metal");
+ } else {
+ await this._startWithGpuFallback(binaryPaths, baseArgs, options);
}
- debugLogger.debug("Starting llama-server", { port: this.port, modelPath, args });
+ this.startHealthCheck();
+ debugLogger.info("llama-server started successfully", {
+ port: this.port,
+ model: path.basename(modelPath),
+ backend: this.activeBackend,
+ });
+ }
+
+ async _startWithGpuFallback(binaryPaths, baseArgs, options) {
+ const cached = this._loadGpuBackend();
+ const gpuArgs = [...baseArgs, "--n-gpu-layers", "99"];
+ const cpuArgs = baseArgs;
- // Set library path for dynamic library loading
- const binDir = path.dirname(serverBinary);
+ if (cached !== "cpu" && binaryPaths.vulkan) {
+ try {
+ debugLogger.debug("Attempting Vulkan backend startup");
+ await this._startWithBinary(
+ binaryPaths.vulkan,
+ gpuArgs,
+ this._buildEnv(binaryPaths.vulkan),
+ VULKAN_STARTUP_TIMEOUT_MS
+ );
+ this.activeBackend = "vulkan";
+ this._saveGpuBackend("vulkan");
+ return;
+ } catch (err) {
+ debugLogger.warn("Vulkan backend failed, falling back to CPU", { error: err.message });
+ await this._killCurrentProcess();
+ this.port = await this.findAvailablePort();
+ }
+ }
+
+ if (!binaryPaths.cpu) throw new Error("No CPU llama-server binary available");
+
+ debugLogger.debug("Starting with CPU backend");
+ await this._startWithBinary(
+ binaryPaths.cpu,
+ cpuArgs,
+ this._buildEnv(binaryPaths.cpu),
+ STARTUP_TIMEOUT_MS
+ );
+ this.activeBackend = "cpu";
+ this._saveGpuBackend("cpu");
+ }
+
+ _buildEnv(binaryPath) {
+ const binDir = path.dirname(binaryPath);
const env = { ...process.env };
if (process.platform === "darwin") {
- // macOS: Set DYLD_LIBRARY_PATH to find .dylib files
env.DYLD_LIBRARY_PATH = binDir + (env.DYLD_LIBRARY_PATH ? `:${env.DYLD_LIBRARY_PATH}` : "");
} else if (process.platform === "linux") {
- // Linux: Set LD_LIBRARY_PATH to find .so files
env.LD_LIBRARY_PATH = binDir + (env.LD_LIBRARY_PATH ? `:${env.LD_LIBRARY_PATH}` : "");
+ } else if (process.platform === "win32") {
+ env.PATH = binDir + (env.PATH ? `;${env.PATH}` : "");
}
- this.process = spawn(serverBinary, args, {
- stdio: ["ignore", "pipe", "pipe"],
- windowsHide: true,
- cwd: getSafeTempDir(),
- env,
- });
+ return env;
+ }
- let stderrBuffer = "";
- let exitCode = null;
- let exitSignal = null;
+ _startWithBinary(binaryPath, args, env, timeoutMs) {
+ return new Promise((resolve, reject) => {
+ debugLogger.debug("Spawning llama-server", { binary: binaryPath, port: this.port, args });
- this.process.stdout.on("data", (data) => {
- debugLogger.debug("llama-server stdout", { data: data.toString().trim() });
- });
+ this.process = spawn(binaryPath, args, {
+ stdio: ["ignore", "pipe", "pipe"],
+ windowsHide: true,
+ cwd: getSafeTempDir(),
+ env,
+ });
- this.process.stderr.on("data", (data) => {
- stderrBuffer += data.toString();
- debugLogger.debug("llama-server stderr", { data: data.toString().trim() });
- });
+ let stderrBuffer = "";
+ let exitCode = null;
+ let exitSignal = null;
+ let settled = false;
- this.process.on("error", (error) => {
- debugLogger.error("llama-server process error", { error: error.message });
- this.ready = false;
- });
+ const settle = (fn) => {
+ if (settled) return;
+ settled = true;
+ fn();
+ };
- this.process.on("close", (code, signal) => {
- exitCode = code;
- exitSignal = signal;
- debugLogger.debug("llama-server process exited", { code, signal });
- this.ready = false;
- this.process = null;
- this.stopHealthCheck();
- });
+ this.process.stdout.on("data", (data) => {
+ debugLogger.debug("llama-server stdout", { data: data.toString().trim() });
+ });
- await this.waitForReady(() => ({ stderr: stderrBuffer, exitCode, exitSignal }));
- this.startHealthCheck();
+ this.process.stderr.on("data", (data) => {
+ stderrBuffer += data.toString();
+ debugLogger.debug("llama-server stderr", { data: data.toString().trim() });
+ });
- debugLogger.info("llama-server started successfully", {
- port: this.port,
- model: path.basename(modelPath),
+ this.process.on("error", (error) => {
+ debugLogger.error("llama-server process error", { error: error.message });
+ this.ready = false;
+ settle(() => reject(new Error(`Failed to spawn llama-server: ${error.message}`)));
+ });
+
+ this.process.on("close", (code, signal) => {
+ exitCode = code;
+ exitSignal = signal;
+ debugLogger.debug("llama-server process exited", { code, signal });
+ this.ready = false;
+ this.process = null;
+ this.stopHealthCheck();
+ });
+
+ const getProcessInfo = () => ({ stderr: stderrBuffer, exitCode, exitSignal });
+
+ const startTime = Date.now();
+ let pollCount = 0;
+
+ const poll = async () => {
+ if (settled) return;
+
+ if (!this.process || this.process.killed) {
+ const info = getProcessInfo();
+ const signal = info.exitSignal;
+ const diagParts = [];
+ if (signal) diagParts.push(`signal: ${signal}`);
+ else if (info.exitCode !== null && info.exitCode !== undefined)
+ diagParts.push(`exit code: ${info.exitCode}`);
+ const oomHint =
+ signal === "SIGKILL"
+ ? " — the process was killed by the OS, likely due to insufficient memory. Try a smaller/more quantized model, or reduce the context size."
+ : "";
+ const stderr = info.stderr ? info.stderr.trim().slice(-800) : "";
+ const diagStr = diagParts.length ? ` (${diagParts.join(", ")})` : "";
+ settle(() =>
+ reject(
+ new Error(
+ `llama-server process died during startup${diagStr}${oomHint}${stderr ? `\nProcess output: ${stderr}` : ""}`
+ )
+ )
+ );
+ return;
+ }
+
+ pollCount++;
+ if (await this.checkHealth()) {
+ this.ready = true;
+ debugLogger.debug("llama-server ready", {
+ startupTimeMs: Date.now() - startTime,
+ pollCount,
+ });
+ settle(() => resolve());
+ return;
+ }
+
+ if (Date.now() - startTime >= timeoutMs) {
+ settle(() => reject(new Error(`llama-server failed to start within ${timeoutMs}ms`)));
+ return;
+ }
+
+ setTimeout(poll, STARTUP_POLL_INTERVAL_MS);
+ };
+
+ poll();
});
}
- async waitForReady(getProcessInfo) {
- const startTime = Date.now();
- let pollCount = 0;
-
- while (Date.now() - startTime < STARTUP_TIMEOUT_MS) {
- if (!this.process || this.process.killed) {
- const info = getProcessInfo ? getProcessInfo() : {};
- const signal = info.exitSignal;
- const diagParts = [];
- if (signal) diagParts.push(`signal: ${signal}`);
- else if (info.exitCode !== null && info.exitCode !== undefined)
- diagParts.push(`exit code: ${info.exitCode}`);
- const oomHint =
- signal === "SIGKILL"
- ? " — the process was killed by the OS, likely due to insufficient memory. Try a smaller/more quantized model, or reduce the context size."
- : "";
- const stderr = info.stderr ? info.stderr.trim().slice(-800) : "";
- const diagStr = diagParts.length ? ` (${diagParts.join(", ")})` : "";
- throw new Error(
- `llama-server process died during startup${diagStr}${oomHint}${stderr ? `\nProcess output: ${stderr}` : ""}`
- );
- }
+ async _killCurrentProcess() {
+ if (!this.process) return;
- pollCount++;
- if (await this.checkHealth()) {
- this.ready = true;
- debugLogger.debug("llama-server ready", {
- startupTimeMs: Date.now() - startTime,
- pollCount,
- });
- return;
- }
+ this.stopHealthCheck();
- await new Promise((resolve) => setTimeout(resolve, STARTUP_POLL_INTERVAL_MS));
+ try {
+ killProcess(this.process, "SIGTERM");
+ await new Promise((resolve) => {
+ const timeout = setTimeout(() => {
+ if (this.process) killProcess(this.process, "SIGKILL");
+ resolve();
+ }, 5000);
+
+ if (this.process) {
+ this.process.once("close", () => {
+ clearTimeout(timeout);
+ resolve();
+ });
+ } else {
+ clearTimeout(timeout);
+ resolve();
+ }
+ });
+ } catch (error) {
+ debugLogger.error("Error killing llama-server process", { error: error.message });
}
- throw new Error(`llama-server failed to start within ${STARTUP_TIMEOUT_MS}ms`);
+ this.process = null;
+ this.ready = false;
+ }
+
+ _loadGpuBackend() {
+ const val = process.env.LLAMA_GPU_BACKEND;
+ if (val === "vulkan" || val === "cpu" || val === "metal") return val;
+ return null;
+ }
+
+ _saveGpuBackend(backend) {
+ process.env.LLAMA_GPU_BACKEND = backend;
}
checkHealth() {
@@ -315,7 +447,7 @@ class LlamaServerManager {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(body),
},
- timeout: 300000, // 5 minute timeout for inference
+ timeout: 300000,
},
(res) => {
let data = "";
@@ -335,7 +467,6 @@ class LlamaServerManager {
try {
const response = JSON.parse(data);
- // Extract text from OpenAI-compatible response
const text = response.choices?.[0]?.message?.content || "";
resolve(text.trim());
} catch (e) {
@@ -397,6 +528,7 @@ class LlamaServerManager {
this.ready = false;
this.port = null;
this.modelPath = null;
+ this.activeBackend = null;
}
getStatus() {
@@ -406,8 +538,16 @@ class LlamaServerManager {
port: this.port,
modelPath: this.modelPath,
modelName: this.modelPath ? path.basename(this.modelPath, ".gguf") : null,
+ backend: this.activeBackend,
+ gpuAccelerated: this.activeBackend === "vulkan" || this.activeBackend === "metal",
};
}
+
+ resetGpuDetection() {
+ delete process.env.LLAMA_GPU_BACKEND;
+ this.activeBackend = null;
+ this.cachedServerBinaryPaths = null;
+ }
}
module.exports = LlamaServerManager;
diff --git a/src/helpers/llamaVulkanManager.js b/src/helpers/llamaVulkanManager.js
new file mode 100644
index 000000000..0e040ff76
--- /dev/null
+++ b/src/helpers/llamaVulkanManager.js
@@ -0,0 +1,266 @@
+const fs = require("fs");
+const { promises: fsPromises } = require("fs");
+const path = require("path");
+const { execFile } = require("child_process");
+const { app } = require("electron");
+const debugLogger = require("./debugLogger");
+const {
+ downloadFile,
+ createDownloadSignal,
+ checkDiskSpace,
+ cleanupStaleDownloads,
+} = require("./downloadUtils");
+
+const GITHUB_RELEASE_URL = "https://api.github.com/repos/ggerganov/llama.cpp/releases/latest";
+
+const VULKAN_ASSETS = {
+ "win32-x64": {
+ assetPattern: /^llama-.*-bin-win-vulkan-x64\.zip$/,
+ binaryName: "llama-server.exe",
+ outputName: "llama-server-vulkan.exe",
+ libPattern: /\.dll$/i,
+ },
+ "linux-x64": {
+ assetPattern: /^llama-.*-bin-ubuntu-vulkan-x64\.tar\.gz$/,
+ binaryName: "llama-server",
+ outputName: "llama-server-vulkan",
+ libPattern: /\.so(\.\d+)*$/,
+ },
+};
+
+class LlamaVulkanManager {
+ constructor() {
+ this._binDir = null;
+ this._downloadSignal = null;
+ this._downloading = false;
+ }
+
+ get binDir() {
+ if (!this._binDir) {
+ this._binDir = path.join(app.getPath("userData"), "bin");
+ }
+ return this._binDir;
+ }
+
+ _getConfig() {
+ return VULKAN_ASSETS[`${process.platform}-${process.arch}`] || null;
+ }
+
+ isSupported() {
+ return this._getConfig() !== null;
+ }
+
+ getBinaryPath() {
+ const config = this._getConfig();
+ if (!config) return null;
+ const p = path.join(this.binDir, config.outputName);
+ try {
+ if (fs.existsSync(p)) return p;
+ } catch {}
+ return null;
+ }
+
+ isDownloaded() {
+ return this.getBinaryPath() !== null;
+ }
+
+ getStatus() {
+ return {
+ supported: this.isSupported(),
+ downloaded: this.isDownloaded(),
+ downloading: this._downloading,
+ };
+ }
+
+ async download(onProgress) {
+ if (this._downloading) throw new Error("Download already in progress");
+ if (!this.isSupported()) throw new Error("Vulkan not available for this platform");
+
+ this._downloading = true;
+ const { signal, abort } = createDownloadSignal();
+ this._downloadSignal = { abort };
+
+ try {
+ await fsPromises.mkdir(this.binDir, { recursive: true });
+ await cleanupStaleDownloads(this.binDir);
+
+ const release = await this._fetchJson(GITHUB_RELEASE_URL);
+ if (!release?.assets) throw new Error("Could not fetch llama.cpp release info");
+
+ const config = this._getConfig();
+ const asset = release.assets.find((a) => config.assetPattern.test(a.name));
+ if (!asset) throw new Error("Vulkan binary not found in latest release");
+
+ const spaceCheck = await checkDiskSpace(this.binDir, (asset.size || 100_000_000) * 2.5);
+ if (!spaceCheck.ok) {
+ throw new Error(
+ `Not enough disk space. Need ~${Math.round(((asset.size || 100_000_000) * 2.5) / 1_000_000)}MB, ` +
+ `only ${Math.round(spaceCheck.availableBytes / 1_000_000)}MB available.`
+ );
+ }
+
+ const archivePath = path.join(this.binDir, asset.name);
+ await downloadFile(asset.browser_download_url, archivePath, {
+ signal,
+ expectedSize: asset.size,
+ onProgress,
+ });
+
+ const extractDir = path.join(this.binDir, `temp-vulkan-${Date.now()}`);
+ await fsPromises.mkdir(extractDir, { recursive: true });
+
+ try {
+ await this._extract(archivePath, extractDir);
+
+ const binaryPath = this._findFile(extractDir, config.binaryName);
+ if (!binaryPath) throw new Error(`${config.binaryName} not found in archive`);
+
+ const outputPath = path.join(this.binDir, config.outputName);
+ await fsPromises.copyFile(binaryPath, outputPath);
+ if (process.platform !== "win32") await fsPromises.chmod(outputPath, 0o755);
+
+ const libs = this._findFiles(extractDir, config.libPattern);
+ for (const lib of libs) {
+ const dest = path.join(this.binDir, path.basename(lib));
+ await fsPromises.copyFile(lib, dest);
+ if (process.platform !== "win32") await fsPromises.chmod(dest, 0o755);
+ }
+
+ debugLogger.info("Vulkan llama-server installed", { path: outputPath });
+ } finally {
+ await fsPromises.rm(extractDir, { recursive: true, force: true }).catch(() => {});
+ await fsPromises.unlink(archivePath).catch(() => {});
+ }
+
+ return { success: true };
+ } catch (error) {
+ if (error.isAbort) return { success: false, cancelled: true };
+ throw error;
+ } finally {
+ this._downloading = false;
+ this._downloadSignal = null;
+ }
+ }
+
+ cancelDownload() {
+ if (this._downloadSignal) {
+ this._downloadSignal.abort();
+ this._downloadSignal = null;
+ return true;
+ }
+ return false;
+ }
+
+ async deleteBinary() {
+ const config = this._getConfig();
+ if (!config) return { success: true };
+
+ let deletedCount = 0;
+ try {
+ const entries = await fsPromises.readdir(this.binDir);
+ for (const entry of entries) {
+ if (entry === config.outputName || config.libPattern.test(entry)) {
+ await fsPromises.unlink(path.join(this.binDir, entry)).catch(() => {});
+ deletedCount++;
+ }
+ }
+ } catch {}
+
+ debugLogger.info("Vulkan llama-server deleted", { deletedCount });
+ return { success: true, deletedCount };
+ }
+
+ _fetchJson(url) {
+ const https = require("https");
+ return new Promise((resolve, reject) => {
+ const headers = {
+ "User-Agent": "OpenWhispr/1.0",
+ Accept: "application/vnd.github+json",
+ };
+ const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+ if (token) headers.Authorization = `Bearer ${token}`;
+
+ https
+ .get(url, { headers, timeout: 15000 }, (res) => {
+ if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+ res.resume();
+ this._fetchJson(res.headers.location).then(resolve, reject);
+ return;
+ }
+ if (res.statusCode !== 200) {
+ res.resume();
+ reject(new Error(`GitHub API returned ${res.statusCode}`));
+ return;
+ }
+ let data = "";
+ res.on("data", (chunk) => (data += chunk));
+ res.on("end", () => {
+ try {
+ resolve(JSON.parse(data));
+ } catch {
+ reject(new Error("Failed to parse GitHub release JSON"));
+ }
+ });
+ })
+ .on("error", reject);
+ });
+ }
+
+ _extract(archivePath, destDir) {
+ return new Promise((resolve, reject) => {
+ if (archivePath.endsWith(".tar.gz") || archivePath.endsWith(".tgz")) {
+ execFile("tar", ["-xzf", archivePath, "-C", destDir], (err) => {
+ err ? reject(new Error(`Extraction failed: ${err.message}`)) : resolve();
+ });
+ } else if (process.platform === "win32") {
+ execFile(
+ "powershell",
+ [
+ "-NoProfile",
+ "-Command",
+ `Expand-Archive -Force -Path '${archivePath}' -DestinationPath '${destDir}'`,
+ ],
+ (err) => {
+ err ? reject(new Error(`Extraction failed: ${err.message}`)) : resolve();
+ }
+ );
+ } else {
+ execFile("unzip", ["-o", archivePath, "-d", destDir], (err) => {
+ err ? reject(new Error(`Extraction failed: ${err.message}`)) : resolve();
+ });
+ }
+ });
+ }
+
+ _findFile(dir, name, maxDepth = 5, depth = 0) {
+ if (depth >= maxDepth) return null;
+ const entries = fs.readdirSync(dir, { withFileTypes: true });
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ const found = this._findFile(full, name, maxDepth, depth + 1);
+ if (found) return found;
+ } else if (entry.name === name) {
+ return full;
+ }
+ }
+ return null;
+ }
+
+ _findFiles(dir, pattern, maxDepth = 5, depth = 0) {
+ if (depth >= maxDepth) return [];
+ const results = [];
+ const entries = fs.readdirSync(dir, { withFileTypes: true });
+ for (const entry of entries) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ results.push(...this._findFiles(full, pattern, maxDepth, depth + 1));
+ } else if (pattern.test(entry.name)) {
+ results.push(full);
+ }
+ }
+ return results;
+ }
+}
+
+module.exports = LlamaVulkanManager;
diff --git a/src/helpers/modelManagerBridge.js b/src/helpers/modelManagerBridge.js
index 16d3f906b..7ce5785c8 100644
--- a/src/helpers/modelManagerBridge.js
+++ b/src/helpers/modelManagerBridge.js
@@ -369,6 +369,7 @@ class ModelManager {
await this.serverManager.start(modelPath, {
contextSize: options.contextSize || modelInfo.model.contextLength || 4096,
threads: options.threads || 4,
+ gpuLayers: 99,
});
this.currentServerModelId = modelId;
@@ -441,6 +442,7 @@ class ModelManager {
await this.serverManager.start(modelPath, {
contextSize: modelInfo.model.contextLength || 4096,
threads: 4,
+ gpuLayers: 99,
});
this.currentServerModelId = modelId;
debugLogger.info("llama-server pre-warmed", { modelId });
diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.ts
index da311fe5e..4e9c53d3a 100644
--- a/src/services/ReasoningService.ts
+++ b/src/services/ReasoningService.ts
@@ -1034,10 +1034,8 @@ class ReasoningService extends BaseReasoningService {
const language = this.getPreferredLanguage();
const locale = this.getUiLanguage();
- // Use withSessionRefresh to handle AUTH_EXPIRED automatically
const result = await withSessionRefresh(async () => {
const res = await (window as any).electronAPI.cloudReason(text, {
- model,
agentName,
customDictionary,
customPrompt: this.getCustomPrompt(),
@@ -1086,6 +1084,11 @@ class ReasoningService extends BaseReasoningService {
async isAvailable(): Promise<boolean> {
try {
+ if (isCloudReasoningMode()) {
+ logger.logReasoning("API_KEY_CHECK", { cloudReasoningMode: true });
+ return true;
+ }
+
const openaiKey = await window.electronAPI?.getOpenAIKey?.();
const anthropicKey = await window.electronAPI?.getAnthropicKey?.();
const geminiKey = await window.electronAPI?.getGeminiKey?.();
diff --git a/src/stores/settingsStore.ts b/src/stores/settingsStore.ts
index fb2e205cb..fae211f3f 100644
--- a/src/stores/settingsStore.ts
+++ b/src/stores/settingsStore.ts
@@ -16,12 +16,6 @@ import type {
} from "../hooks/useSettings";
let _ReasoningService: typeof import("../services/ReasoningService").default | null = null;
-function getReasoningService() {
- if (!_ReasoningService) {
- _ReasoningService = require("../services/ReasoningService").default;
- }
- return _ReasoningService!;
-}
const isBrowser = typeof window !== "undefined";
@@ -171,7 +165,16 @@ function invalidateApiKeyCaches(
provider?: "openai" | "anthropic" | "gemini" | "groq" | "mistral" | "custom"
) {
if (provider) {
- getReasoningService().clearApiKeyCache(provider);
+ if (_ReasoningService) {
+ _ReasoningService.clearApiKeyCache(provider);
+ } else {
+ import("../services/ReasoningService")
+ .then((mod) => {
+ _ReasoningService = mod.default;
+ _ReasoningService.clearApiKeyCache(provider);
+ })
+ .catch(() => {});
+ }
}
if (isBrowser) window.dispatchEvent(new Event("api-key-changed"));
debouncedPersistToEnv();
@@ -441,7 +444,11 @@ export function getSettings() {
}
export function getEffectiveReasoningModel() {
- return useSettingsStore.getState().reasoningModel;
+ const state = useSettingsStore.getState();
+ if (selectIsCloudReasoningMode(state)) {
+ return "";
+ }
+ return state.reasoningModel;
}
export function isCloudReasoningMode() {
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 4e3296092..60e02d070 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -218,6 +218,38 @@ export interface PasteToolsResult {
recommendedInstall?: string;
}
+export type GpuBackend = "vulkan" | "cpu" | "metal" | null;
+
+export interface LlamaServerStatus {
+ available: boolean;
+ running: boolean;
+ port: number | null;
+ modelPath: string | null;
+ modelName: string | null;
+ backend: GpuBackend;
+ gpuAccelerated: boolean;
+}
+
+export interface VulkanGpuResult {
+ available: boolean;
+ deviceName?: string;
+ reason?: string;
+ error?: string;
+}
+
+export interface LlamaVulkanStatus {
+ supported: boolean;
+ downloaded: boolean;
+ downloading?: boolean;
+ error?: string;
+}
+
+export interface LlamaVulkanDownloadProgress {
+ downloaded: number;
+ total: number;
+ percentage: number;
+}
+
export interface ReferralItem {
id: string;
email: string;
@@ -484,6 +516,30 @@ declare global {
llamaCppInstall: () => Promise<{ success: boolean; error?: string }>;
llamaCppUninstall: () => Promise<{ success: boolean; error?: string }>;
+ // llama-server
+ llamaServerStart: (
+ modelId: string
+ ) => Promise<{ success: boolean; port?: number; error?: string }>;
+ llamaServerStop: () => Promise<{ success: boolean; error?: string }>;
+ llamaServerStatus: () => Promise<LlamaServerStatus>;
+ llamaGpuReset: () => Promise<{ success: boolean; error?: string }>;
+ detectVulkanGpu?: () => Promise<VulkanGpuResult>;
+ getLlamaVulkanStatus?: () => Promise<LlamaVulkanStatus>;
+ downloadLlamaVulkanBinary?: () => Promise<{
+ success: boolean;
+ cancelled?: boolean;
+ error?: string;
+ }>;
+ cancelLlamaVulkanDownload?: () => Promise<{ success: boolean }>;
+ deleteLlamaVulkanBinary?: () => Promise<{
+ success: boolean;
+ deletedCount?: number;
+ error?: string;
+ }>;
+ onLlamaVulkanDownloadProgress?: (
+ callback: (data: LlamaVulkanDownloadProgress) => void
+ ) => () => void;
+
// Window control operations
windowMinimize: () => Promise<void>;
windowMaximize: () => Promise<void>;
diff --git a/src/utils/vulkanDetection.js b/src/utils/vulkanDetection.js
new file mode 100644
index 000000000..9540716ac
--- /dev/null
+++ b/src/utils/vulkanDetection.js
@@ -0,0 +1,61 @@
+const { app } = require("electron");
+
+let cachedResult = null;
+
+const VULKAN_VENDOR_IDS = new Set([
+ 0x10de, // NVIDIA
+ 0x1002, // AMD
+ 0x8086, // Intel
+]);
+
+const VENDOR_NAMES = {
+ 0x10de: "NVIDIA",
+ 0x1002: "AMD",
+ 0x8086: "Intel",
+};
+
+function parseDeviceName(gpuInfo, activeGpu) {
+ const renderer = gpuInfo.auxAttributes?.glRenderer;
+ if (renderer) {
+ const match = renderer.match(/ANGLE\s*\([^,]*,\s*([^,)]+)/);
+ if (match) return match[1].trim();
+ }
+ if (activeGpu?.vendorId) {
+ return VENDOR_NAMES[activeGpu.vendorId] || `GPU (vendor ${activeGpu.vendorId})`;
+ }
+ return null;
+}
+
+async function detectVulkanGpu() {
+ if (cachedResult) return cachedResult;
+
+ if (process.platform === "darwin") {
+ cachedResult = { available: false };
+ return cachedResult;
+ }
+
+ try {
+ const gpuInfo = await app.getGPUInfo("complete");
+ const activeGpu = gpuInfo.gpuDevice?.find((d) => d.active) || gpuInfo.gpuDevice?.[0];
+
+ if (!activeGpu || !VULKAN_VENDOR_IDS.has(activeGpu.vendorId)) {
+ cachedResult = { available: false };
+ return cachedResult;
+ }
+
+ cachedResult = {
+ available: true,
+ deviceName: parseDeviceName(gpuInfo, activeGpu),
+ };
+ return cachedResult;
+ } catch {
+ cachedResult = { available: false };
+ return cachedResult;
+ }
+}
+
+function clearCache() {
+ cachedResult = null;
+}
+
+module.exports = { detectVulkanGpu, clearCache };