Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat(agent): enable local model tool calling + RAG context injection
Gabriel Steina893037d5b56
diff --git a/electron-builder.json b/electron-builder.jsonindex 0a51f5e3b..f4c011cdd 100644--- a/electron-builder.json+++ b/electron-builder.json@@ -107,6 +107,7 @@"windows-mic-listener*","windows-text-monitor*","windows-fast-paste*",+ "all-MiniLM-L6-v2/**/*","*.dylib","*.dll","*.so*"diff --git a/package.json b/package.jsonindex 684fef926..76eca4135 100644--- a/package.json+++ b/package.json@@ -18,14 +18,14 @@"compile:mic-listener": "node scripts/build-macos-mic-listener.js","compile:audio-tap": "node scripts/build-macos-audio-tap.js","compile:native": "npm run compile:globe && npm run compile:fast-paste && npm run compile:winkeys && npm run compile:winpaste && npm run compile:linux-paste && npm run compile:text-monitor && npm run compile:media-remote && npm run compile:mic-listener && npm run compile:audio-tap",- "prestart": "npm run compile:native && npm run download:qdrant",+ "prestart": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model","start": "electron .",- "predev": "npm run compile:native && npm run download:qdrant",+ "predev": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model","dev": "concurrently -k -r \"npm:dev:renderer\" \"npm:dev:main\"",- "predev:main": "npm run compile:native && npm run download:qdrant",+ "predev:main": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model","dev:main": "cross-env NODE_ENV=development node scripts/run-electron.js --dev","dev:renderer": "cd src && vite",- "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",+ "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model -- --for-build","build": "cd src && vite build && cd .. && electron-builder","postinstall": "electron-builder install-app-deps","build:renderer": "cd src && vite build",diff --git a/scripts/download-minilm.js b/scripts/download-minilm.jsindex 270ba15cd..a590d10dd 100644--- a/scripts/download-minilm.js+++ b/scripts/download-minilm.js@@ -4,13 +4,10 @@ const os = require("os");const path = require("path");const { downloadFile, parseArgs } = require("./lib/download-utils");-const MODEL_DIR = path.join(- os.homedir(),- ".cache",- "openwhispr",- "embedding-models",- "all-MiniLM-L6-v2"-);+const forBuild = process.argv.includes("--for-build");+const MODEL_DIR = forBuild+ ? path.join(__dirname, "..", "resources", "bin", "all-MiniLM-L6-v2")+ : path.join(os.homedir(), ".cache", "openwhispr", "embedding-models", "all-MiniLM-L6-v2");const FILES = [{diff --git a/scripts/lib/download-utils.js b/scripts/lib/download-utils.jsindex 1135858c9..18d0018b1 100644--- a/scripts/lib/download-utils.js+++ b/scripts/lib/download-utils.js@@ -39,12 +39,15 @@ function fetchJson(url, redirectCount = 0) {https.get(url, options, (res) => {- if (res.statusCode === 301 || res.statusCode === 302) {- const redirectUrl = res.headers.location;- if (!redirectUrl) {+ if ([301, 302, 303, 307, 308].includes(res.statusCode)) {+ const location = res.headers.location;+ if (!location) {reject(new Error("Redirect without location header"));return;}+ const redirectUrl = location.startsWith("/")+ ? new URL(location, url).href+ : location;fetchJson(redirectUrl, redirectCount + 1).then(resolve).catch(reject);@@ -156,13 +159,16 @@ function downloadFile(url, dest, retryCount = 0) {}activeRequest = https.get(currentUrl, (response) => {- if (response.statusCode === 302 || response.statusCode === 301) {- const redirectUrl = response.headers.location;- if (!redirectUrl) {+ if ([301, 302, 303, 307, 308].includes(response.statusCode)) {+ const location = response.headers.location;+ if (!location) {cleanup();reject(new Error("Redirect without location header"));return;}+ const redirectUrl = location.startsWith("/")+ ? new URL(location, currentUrl).href+ : location;request(redirectUrl, redirectCount + 1);return;}diff --git a/src/components/chat/useChatStreaming.ts b/src/components/chat/useChatStreaming.tsindex 17702e7ef..de179c8fb 100644--- a/src/components/chat/useChatStreaming.ts+++ b/src/components/chat/useChatStreaming.ts@@ -7,6 +7,37 @@ import { createToolRegistry } from "../../services/tools";import type { ToolRegistry } from "../../services/tools/ToolRegistry";import type { Message, AgentState, ToolCallInfo } from "./types";+const RAG_NOTE_LIMIT = 5;+const RAG_NOTE_SNIPPET_LENGTH = 500;++const LOCAL_TOOL_MIN_PARAMS_B = 4;++function estimateModelSizeB(modelId: string): number {+ const match = modelId.match(/-([\d.]+)[bB]/);+ return match ? parseFloat(match[1]) : 0;+}++async function buildRAGContext(userText: string): Promise<string> {+ if (!window.electronAPI?.semanticSearchNotes) return "";+ try {+ const results = await window.electronAPI.semanticSearchNotes(userText, RAG_NOTE_LIMIT);+ if (!results || results.length === 0) return "";++ const snippets = await Promise.all(+ results.map(async (r: { id: number; title: string; score?: number }) => {+ const note = await window.electronAPI.getNote(r.id);+ if (!note) return null;+ const content = (note.content || "").slice(0, RAG_NOTE_SNIPPET_LENGTH);+ return `<note id="${note.id}" title="${note.title}">\n${content}\n</note>`;+ })+ );++ return snippets.filter(Boolean).join("\n\n");+ } catch {+ return "";+ }+}+interface UseChatStreamingOptions {messages: Message[];setMessages: React.Dispatch<React.SetStateAction<Message[]>>;@@ -59,8 +90,12 @@ export function useChatStreaming({const settings = getSettings();const isCloudAgent = settings.isSignedIn && settings.cloudAgentMode === "openwhispr";- const toolSupportedProviders = ["openai", "groq", "custom", "anthropic", "gemini"];- const supportsTools = isCloudAgent || toolSupportedProviders.includes(settings.agentProvider);+ const isLocalProvider = !["openai", "groq", "custom", "anthropic", "gemini"].includes(+ settings.agentProvider+ );+ const localModelCanUseTool =+ isLocalProvider && estimateModelSizeB(settings.agentModel) >= LOCAL_TOOL_MIN_PARAMS_B;+ const supportsTools = isCloudAgent || !isLocalProvider || localModelCanUseTool;let registry: ToolRegistry | null = null;if (supportsTools) {@@ -76,7 +111,12 @@ export function useChatStreaming({toolRegistryRef.current = { key: cacheKey, registry };}}- const systemPrompt = getAgentSystemPrompt(registry?.getAll().map((t) => t.name));++ const noteContext = await buildRAGContext(userText);+ const systemPrompt = getAgentSystemPrompt(+ registry?.getAll().map((t) => t.name),+ noteContext+ );const llmMessages = [{ role: "system", content: systemPrompt },diff --git a/src/config/prompts.ts b/src/config/prompts.tsindex 1a5cbc9d9..82d03f585 100644--- a/src/config/prompts.ts+++ b/src/config/prompts.ts@@ -160,7 +160,7 @@ const TOOL_INSTRUCTIONS: Record<string, string> = {"Use get_calendar_events to check the user's schedule, upcoming meetings, or calendar events.",};-export function getAgentSystemPrompt(availableTools?: string[]): string {+export function getAgentSystemPrompt(availableTools?: string[], noteContext?: string): string {if (typeof window !== "undefined" && window.localStorage) {const custom = window.localStorage.getItem("agentSystemPrompt");if (custom) return custom;@@ -175,5 +175,12 @@ export function getAgentSystemPrompt(availableTools?: string[]): string {}}+ if (noteContext) {+ prompt +=+ "\n\nBelow are notes from the user's library that may be relevant. " ++ "Reference them naturally if they help answer the question.\n\n" ++ noteContext;+ }+return prompt;}diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.jsindex 381b36d62..4feeed810 100644--- a/src/helpers/llamaServer.js+++ b/src/helpers/llamaServer.js@@ -136,7 +136,6 @@ class LlamaServerManager {this.port = await this.findAvailablePort();this.modelPath = modelPath;- // Let llama.cpp's auto-fit system choose --ctx-size based on available GPU memoryconst baseArgs = ["--model",modelPath,@@ -146,6 +145,7 @@ class LlamaServerManager {String(this.port),"--threads",String(options.threads || 4),+ "--jinja",];if (process.platform === "darwin") {diff --git a/src/helpers/localEmbeddings.js b/src/helpers/localEmbeddings.jsindex 343a42191..5732fc556 100644--- a/src/helpers/localEmbeddings.js+++ b/src/helpers/localEmbeddings.js@@ -6,17 +6,43 @@ const debugLogger = require("./debugLogger");const MAX_TOKENS = 256;const EMBEDDING_DIM = 384;+const MODEL_SUBDIR = "all-MiniLM-L6-v2";+class LocalEmbeddings {constructor() {this.session = null;this.tokenizer = null;- this.modelDir = path.join(+ this.modelDir = this._resolveModelDir();+ }++ _resolveModelDir() {+ const cacheDir = path.join(os.homedir(),".cache","openwhispr","embedding-models",- "all-MiniLM-L6-v2"+ MODEL_SUBDIR);++ if (process.resourcesPath) {+ const bundled = path.join(process.resourcesPath, "bin", MODEL_SUBDIR);+ if (+ fs.existsSync(path.join(bundled, "model.onnx")) &&+ fs.existsSync(path.join(bundled, "tokenizer.json"))+ ) {+ return bundled;+ }+ }++ const projectBin = path.resolve(__dirname, "..", "..", "resources", "bin", MODEL_SUBDIR);+ if (+ fs.existsSync(path.join(projectBin, "model.onnx")) &&+ fs.existsSync(path.join(projectBin, "tokenizer.json"))+ ) {+ return projectBin;+ }++ return cacheDir;}isAvailable() {diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.tsindex 317846630..ba6f43a3c 100644--- a/src/services/ReasoningService.ts+++ b/src/services/ReasoningService.ts@@ -1325,7 +1325,7 @@ class ReasoningService extends BaseReasoningService {const cloudProviders = ["openai", "groq", "gemini", "anthropic", "custom"];const isLocalProvider = !cloudProviders.includes(provider);- if (isLocalProvider) {+ if (isLocalProvider && !tools) {const contentGen = this.processTextStreaming(messages, model, provider, config);for await (const text of contentGen) {yield { type: "content", text };@@ -1334,12 +1334,24 @@ class ReasoningService extends BaseReasoningService {return;}- const providerKey = provider as "openai" | "groq" | "gemini" | "anthropic" | "custom";- const apiKey = await this.getApiKey(providerKey);- const baseURL = provider === "custom" ? this.getConfiguredOpenAIBase() : undefined;+ let apiKey = "";+ let baseURL: string | undefined;++ if (isLocalProvider) {+ const serverResult = await window.electronAPI.llamaServerStart(model);+ if (!serverResult.success || !serverResult.port) {+ throw new Error(serverResult.error || "Failed to start local model server");+ }+ baseURL = `http://127.0.0.1:${serverResult.port}/v1`;+ } else {+ const providerKey = provider as "openai" | "groq" | "gemini" | "anthropic" | "custom";+ apiKey = await this.getApiKey(providerKey);+ baseURL = provider === "custom" ? this.getConfiguredOpenAIBase() : undefined;+ }const apiConfig = getOpenAiApiConfig(model);- const aiModel = getAIModel(provider, model, apiKey, baseURL);+ const aiProvider = isLocalProvider ? "local" : provider;+ const aiModel = getAIModel(aiProvider, model, apiKey, baseURL);const modelDef = getCloudModel(model);const needsDisableThinking = provider === "groq" && modelDef?.disableThinking;@@ -1352,6 +1364,8 @@ class ReasoningService extends BaseReasoningService {messageCount: messages.length,});+ const useTemperature = isLocalProvider || apiConfig.supportsTemperature;+const result = streamText({model: aiModel,messages: messages.map((m) => ({@@ -1360,7 +1374,7 @@ class ReasoningService extends BaseReasoningService {})),tools: tools || undefined,stopWhen: stepCountIs(tools ? ReasoningService.MAX_TOOL_STEPS : 1),- ...(apiConfig.supportsTemperature ? { temperature: config.temperature ?? 0.3 } : {}),+ ...(useTemperature ? { temperature: config.temperature ?? 0.3 } : {}),maxOutputTokens: config.maxTokens || 4096,...(needsDisableThinking ? { providerOptions: { groq: { reasoningEffort: "none" } } } : {}),});diff --git a/src/services/ai/providers.ts b/src/services/ai/providers.tsindex b760cf958..e0b3c1461 100644--- a/src/services/ai/providers.ts+++ b/src/services/ai/providers.ts@@ -21,6 +21,11 @@ export function getAIModel(return createGoogleGenerativeAI({ apiKey })(model);case "custom":return createOpenAI({ apiKey, baseURL })(model);+ case "local":+ return createOpenAI({+ apiKey: "no-key",+ baseURL,+ }).chat(model);default:throw new Error(`Unsupported AI SDK provider: ${provider}`);}