Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
fix: support local models in agent streaming and fix Metal OOM crash
Gabriel Stein80aaf8327ec2
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex e64e324c0..5b544ade1 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -1687,11 +1687,7 @@ class IPCHandlers {const modelPath = require("path").join(modelManager.modelsDir, modelInfo.model.fileName);- await modelManager.serverManager.start(modelPath, {- contextSize: modelInfo.model.contextLength || 4096,- threads: 4,- gpuLayers: 99,- });+ await modelManager.serverManager.start(modelPath, { threads: 4 });modelManager.currentServerModelId = modelId;this.environmentManager.saveAllKeysToEnvFile().catch(() => {});diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.jsindex b0b58fc51..381b36d62 100644--- a/src/helpers/llamaServer.js+++ b/src/helpers/llamaServer.js@@ -136,6 +136,7 @@ class LlamaServerManager {this.port = await this.findAvailablePort();this.modelPath = modelPath;+ // Let llama.cpp's auto-fit system choose --ctx-size based on available GPU memoryconst baseArgs = ["--model",modelPath,@@ -143,8 +144,6 @@ class LlamaServerManager {"127.0.0.1","--port",String(this.port),- "--ctx-size",- String(options.contextSize || 4096),"--threads",String(options.threads || 4),];diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.tsindex cbaa9b26f..fcaa0c2db 100644--- a/src/services/ReasoningService.ts+++ b/src/services/ReasoningService.ts@@ -1093,21 +1093,35 @@ class ReasoningService extends BaseReasoningService {provider: string,config: ReasoningConfig & { systemPrompt: string }): AsyncGenerator<string, void, unknown> {- const providerKey = provider as "openai" | "groq" | "gemini" | "anthropic" | "custom";- const apiKey = await this.getApiKey(providerKey);+ const cloudProviders = ["openai", "groq", "gemini", "anthropic", "custom"];+ const isLocalProvider = !cloudProviders.includes(provider);let endpoint: string;- switch (providerKey) {- case "groq":- endpoint = buildApiUrl(API_ENDPOINTS.GROQ_BASE, "/chat/completions");- break;- case "openai":- case "custom":- endpoint = buildApiUrl(this.getConfiguredOpenAIBase(), "/chat/completions");- break;- default:- endpoint = buildApiUrl(API_ENDPOINTS.OPENAI_BASE, "/chat/completions");- break;+ let apiKey = "";++ if (isLocalProvider) {+ // Local model via llama.cpp server+ const serverResult = await window.electronAPI.llamaServerStart(model);+ if (!serverResult.success || !serverResult.port) {+ throw new Error(serverResult.error || "Failed to start local model server");+ }+ endpoint = `http://127.0.0.1:${serverResult.port}/v1/chat/completions`;+ } else {+ const providerKey = provider as "openai" | "groq" | "gemini" | "anthropic" | "custom";+ apiKey = await this.getApiKey(providerKey);++ switch (providerKey) {+ case "groq":+ endpoint = buildApiUrl(API_ENDPOINTS.GROQ_BASE, "/chat/completions");+ break;+ case "openai":+ case "custom":+ endpoint = buildApiUrl(this.getConfiguredOpenAIBase(), "/chat/completions");+ break;+ default:+ endpoint = buildApiUrl(API_ENDPOINTS.OPENAI_BASE, "/chat/completions");+ break;+ }}const requestBody: Record<string, unknown> = {@@ -1122,15 +1136,20 @@ class ReasoningService extends BaseReasoningService {endpoint,model,provider,+ isLocal: isLocalProvider,messageCount: messages.length,});+ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ };+ if (apiKey) {+ headers["Authorization"] = `Bearer ${apiKey}`;+ }+const response = await fetch(endpoint, {method: "POST",- headers: {- "Content-Type": "application/json",- Authorization: `Bearer ${apiKey}`,- },+ headers,body: JSON.stringify(requestBody),});@@ -1156,6 +1175,7 @@ class ReasoningService extends BaseReasoningService {const decoder = new TextDecoder();let buffer = "";+ let insideThinkBlock = false;try {while (true) {@@ -1175,8 +1195,36 @@ class ReasoningService extends BaseReasoningService {try {const parsed = JSON.parse(data);- const content = parsed.choices?.[0]?.delta?.content;- if (content) yield content;+ let content = parsed.choices?.[0]?.delta?.content;+ if (!content) continue;++ // Strip Qwen3 <think> blocks from streamed output+ if (isLocalProvider) {+ if (insideThinkBlock) {+ const endIdx = content.indexOf("</think>");+ if (endIdx !== -1) {+ insideThinkBlock = false;+ content = content.slice(endIdx + 8);+ } else {+ continue;+ }+ }+ const startIdx = content.indexOf("<think>");+ if (startIdx !== -1) {+ const before = content.slice(0, startIdx);+ const after = content.slice(startIdx + 7);+ const endIdx = after.indexOf("</think>");+ if (endIdx !== -1) {+ content = before + after.slice(endIdx + 8);+ } else {+ insideThinkBlock = true;+ content = before;+ }+ }+ if (!content) continue;+ }++ yield content;} catch {// skip malformed SSE chunks}