Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat(agent): enable local model tool calling + RAG context injection

Gabriel Steina893037d5b56
diff --git a/electron-builder.json b/electron-builder.json
index 0a51f5e3b..f4c011cdd 100644
--- a/electron-builder.json
+++ b/electron-builder.json
@@ -107,6 +107,7 @@
"windows-mic-listener*",
"windows-text-monitor*",
"windows-fast-paste*",
+ "all-MiniLM-L6-v2/**/*",
"*.dylib",
"*.dll",
"*.so*"
diff --git a/package.json b/package.json
index 684fef926..76eca4135 100644
--- a/package.json
+++ b/package.json
@@ -18,14 +18,14 @@
"compile:mic-listener": "node scripts/build-macos-mic-listener.js",
"compile:audio-tap": "node scripts/build-macos-audio-tap.js",
"compile:native": "npm run compile:globe && npm run compile:fast-paste && npm run compile:winkeys && npm run compile:winpaste && npm run compile:linux-paste && npm run compile:text-monitor && npm run compile:media-remote && npm run compile:mic-listener && npm run compile:audio-tap",
- "prestart": "npm run compile:native && npm run download:qdrant",
+ "prestart": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model",
"start": "electron .",
- "predev": "npm run compile:native && npm run download:qdrant",
+ "predev": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model",
"dev": "concurrently -k -r \"npm:dev:renderer\" \"npm:dev:main\"",
- "predev:main": "npm run compile:native && npm run download:qdrant",
+ "predev:main": "npm run compile:native && npm run download:qdrant && npm run download:embedding-model",
"dev:main": "cross-env NODE_ENV=development node scripts/run-electron.js --dev",
"dev:renderer": "cd src && vite",
- "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant",
+ "prebuild": "npm run compile:native && npm run download:whisper-cpp && npm run download:llama-server && npm run download:sherpa-onnx && npm run download:qdrant && npm run download:embedding-model -- --for-build",
"build": "cd src && vite build && cd .. && electron-builder",
"postinstall": "electron-builder install-app-deps",
"build:renderer": "cd src && vite build",
diff --git a/scripts/download-minilm.js b/scripts/download-minilm.js
index 270ba15cd..a590d10dd 100644
--- a/scripts/download-minilm.js
+++ b/scripts/download-minilm.js
@@ -4,13 +4,10 @@ const os = require("os");
const path = require("path");
const { downloadFile, parseArgs } = require("./lib/download-utils");
-const MODEL_DIR = path.join(
- os.homedir(),
- ".cache",
- "openwhispr",
- "embedding-models",
- "all-MiniLM-L6-v2"
-);
+const forBuild = process.argv.includes("--for-build");
+const MODEL_DIR = forBuild
+ ? path.join(__dirname, "..", "resources", "bin", "all-MiniLM-L6-v2")
+ : path.join(os.homedir(), ".cache", "openwhispr", "embedding-models", "all-MiniLM-L6-v2");
const FILES = [
{
diff --git a/scripts/lib/download-utils.js b/scripts/lib/download-utils.js
index 1135858c9..18d0018b1 100644
--- a/scripts/lib/download-utils.js
+++ b/scripts/lib/download-utils.js
@@ -39,12 +39,15 @@ function fetchJson(url, redirectCount = 0) {
https
.get(url, options, (res) => {
- if (res.statusCode === 301 || res.statusCode === 302) {
- const redirectUrl = res.headers.location;
- if (!redirectUrl) {
+ if ([301, 302, 303, 307, 308].includes(res.statusCode)) {
+ const location = res.headers.location;
+ if (!location) {
reject(new Error("Redirect without location header"));
return;
}
+ const redirectUrl = location.startsWith("/")
+ ? new URL(location, url).href
+ : location;
fetchJson(redirectUrl, redirectCount + 1)
.then(resolve)
.catch(reject);
@@ -156,13 +159,16 @@ function downloadFile(url, dest, retryCount = 0) {
}
activeRequest = https.get(currentUrl, (response) => {
- if (response.statusCode === 302 || response.statusCode === 301) {
- const redirectUrl = response.headers.location;
- if (!redirectUrl) {
+ if ([301, 302, 303, 307, 308].includes(response.statusCode)) {
+ const location = response.headers.location;
+ if (!location) {
cleanup();
reject(new Error("Redirect without location header"));
return;
}
+ const redirectUrl = location.startsWith("/")
+ ? new URL(location, currentUrl).href
+ : location;
request(redirectUrl, redirectCount + 1);
return;
}
diff --git a/src/components/chat/useChatStreaming.ts b/src/components/chat/useChatStreaming.ts
index 17702e7ef..de179c8fb 100644
--- a/src/components/chat/useChatStreaming.ts
+++ b/src/components/chat/useChatStreaming.ts
@@ -7,6 +7,37 @@ import { createToolRegistry } from "../../services/tools";
import type { ToolRegistry } from "../../services/tools/ToolRegistry";
import type { Message, AgentState, ToolCallInfo } from "./types";
+const RAG_NOTE_LIMIT = 5;
+const RAG_NOTE_SNIPPET_LENGTH = 500;
+
+const LOCAL_TOOL_MIN_PARAMS_B = 4;
+
+function estimateModelSizeB(modelId: string): number {
+ const match = modelId.match(/-([\d.]+)[bB]/);
+ return match ? parseFloat(match[1]) : 0;
+}
+
+async function buildRAGContext(userText: string): Promise<string> {
+ if (!window.electronAPI?.semanticSearchNotes) return "";
+ try {
+ const results = await window.electronAPI.semanticSearchNotes(userText, RAG_NOTE_LIMIT);
+ if (!results || results.length === 0) return "";
+
+ const snippets = await Promise.all(
+ results.map(async (r: { id: number; title: string; score?: number }) => {
+ const note = await window.electronAPI.getNote(r.id);
+ if (!note) return null;
+ const content = (note.content || "").slice(0, RAG_NOTE_SNIPPET_LENGTH);
+ return `<note id="${note.id}" title="${note.title}">\n${content}\n</note>`;
+ })
+ );
+
+ return snippets.filter(Boolean).join("\n\n");
+ } catch {
+ return "";
+ }
+}
+
interface UseChatStreamingOptions {
messages: Message[];
setMessages: React.Dispatch<React.SetStateAction<Message[]>>;
@@ -59,8 +90,12 @@ export function useChatStreaming({
const settings = getSettings();
const isCloudAgent = settings.isSignedIn && settings.cloudAgentMode === "openwhispr";
- const toolSupportedProviders = ["openai", "groq", "custom", "anthropic", "gemini"];
- const supportsTools = isCloudAgent || toolSupportedProviders.includes(settings.agentProvider);
+ const isLocalProvider = !["openai", "groq", "custom", "anthropic", "gemini"].includes(
+ settings.agentProvider
+ );
+ const localModelCanUseTool =
+ isLocalProvider && estimateModelSizeB(settings.agentModel) >= LOCAL_TOOL_MIN_PARAMS_B;
+ const supportsTools = isCloudAgent || !isLocalProvider || localModelCanUseTool;
let registry: ToolRegistry | null = null;
if (supportsTools) {
@@ -76,7 +111,12 @@ export function useChatStreaming({
toolRegistryRef.current = { key: cacheKey, registry };
}
}
- const systemPrompt = getAgentSystemPrompt(registry?.getAll().map((t) => t.name));
+
+ const noteContext = await buildRAGContext(userText);
+ const systemPrompt = getAgentSystemPrompt(
+ registry?.getAll().map((t) => t.name),
+ noteContext
+ );
const llmMessages = [
{ role: "system", content: systemPrompt },
diff --git a/src/config/prompts.ts b/src/config/prompts.ts
index 1a5cbc9d9..82d03f585 100644
--- a/src/config/prompts.ts
+++ b/src/config/prompts.ts
@@ -160,7 +160,7 @@ const TOOL_INSTRUCTIONS: Record<string, string> = {
"Use get_calendar_events to check the user's schedule, upcoming meetings, or calendar events.",
};
-export function getAgentSystemPrompt(availableTools?: string[]): string {
+export function getAgentSystemPrompt(availableTools?: string[], noteContext?: string): string {
if (typeof window !== "undefined" && window.localStorage) {
const custom = window.localStorage.getItem("agentSystemPrompt");
if (custom) return custom;
@@ -175,5 +175,12 @@ export function getAgentSystemPrompt(availableTools?: string[]): string {
}
}
+ if (noteContext) {
+ prompt +=
+ "\n\nBelow are notes from the user's library that may be relevant. " +
+ "Reference them naturally if they help answer the question.\n\n" +
+ noteContext;
+ }
+
return prompt;
}
diff --git a/src/helpers/llamaServer.js b/src/helpers/llamaServer.js
index 381b36d62..4feeed810 100644
--- a/src/helpers/llamaServer.js
+++ b/src/helpers/llamaServer.js
@@ -136,7 +136,6 @@ class LlamaServerManager {
this.port = await this.findAvailablePort();
this.modelPath = modelPath;
- // Let llama.cpp's auto-fit system choose --ctx-size based on available GPU memory
const baseArgs = [
"--model",
modelPath,
@@ -146,6 +145,7 @@ class LlamaServerManager {
String(this.port),
"--threads",
String(options.threads || 4),
+ "--jinja",
];
if (process.platform === "darwin") {
diff --git a/src/helpers/localEmbeddings.js b/src/helpers/localEmbeddings.js
index 343a42191..5732fc556 100644
--- a/src/helpers/localEmbeddings.js
+++ b/src/helpers/localEmbeddings.js
@@ -6,17 +6,43 @@ const debugLogger = require("./debugLogger");
const MAX_TOKENS = 256;
const EMBEDDING_DIM = 384;
+const MODEL_SUBDIR = "all-MiniLM-L6-v2";
+
class LocalEmbeddings {
constructor() {
this.session = null;
this.tokenizer = null;
- this.modelDir = path.join(
+ this.modelDir = this._resolveModelDir();
+ }
+
+ _resolveModelDir() {
+ const cacheDir = path.join(
os.homedir(),
".cache",
"openwhispr",
"embedding-models",
- "all-MiniLM-L6-v2"
+ MODEL_SUBDIR
);
+
+ if (process.resourcesPath) {
+ const bundled = path.join(process.resourcesPath, "bin", MODEL_SUBDIR);
+ if (
+ fs.existsSync(path.join(bundled, "model.onnx")) &&
+ fs.existsSync(path.join(bundled, "tokenizer.json"))
+ ) {
+ return bundled;
+ }
+ }
+
+ const projectBin = path.resolve(__dirname, "..", "..", "resources", "bin", MODEL_SUBDIR);
+ if (
+ fs.existsSync(path.join(projectBin, "model.onnx")) &&
+ fs.existsSync(path.join(projectBin, "tokenizer.json"))
+ ) {
+ return projectBin;
+ }
+
+ return cacheDir;
}
isAvailable() {
diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.ts
index 317846630..ba6f43a3c 100644
--- a/src/services/ReasoningService.ts
+++ b/src/services/ReasoningService.ts
@@ -1325,7 +1325,7 @@ class ReasoningService extends BaseReasoningService {
const cloudProviders = ["openai", "groq", "gemini", "anthropic", "custom"];
const isLocalProvider = !cloudProviders.includes(provider);
- if (isLocalProvider) {
+ if (isLocalProvider && !tools) {
const contentGen = this.processTextStreaming(messages, model, provider, config);
for await (const text of contentGen) {
yield { type: "content", text };
@@ -1334,12 +1334,24 @@ class ReasoningService extends BaseReasoningService {
return;
}
- const providerKey = provider as "openai" | "groq" | "gemini" | "anthropic" | "custom";
- const apiKey = await this.getApiKey(providerKey);
- const baseURL = provider === "custom" ? this.getConfiguredOpenAIBase() : undefined;
+ let apiKey = "";
+ let baseURL: string | undefined;
+
+ if (isLocalProvider) {
+ const serverResult = await window.electronAPI.llamaServerStart(model);
+ if (!serverResult.success || !serverResult.port) {
+ throw new Error(serverResult.error || "Failed to start local model server");
+ }
+ baseURL = `http://127.0.0.1:${serverResult.port}/v1`;
+ } else {
+ const providerKey = provider as "openai" | "groq" | "gemini" | "anthropic" | "custom";
+ apiKey = await this.getApiKey(providerKey);
+ baseURL = provider === "custom" ? this.getConfiguredOpenAIBase() : undefined;
+ }
const apiConfig = getOpenAiApiConfig(model);
- const aiModel = getAIModel(provider, model, apiKey, baseURL);
+ const aiProvider = isLocalProvider ? "local" : provider;
+ const aiModel = getAIModel(aiProvider, model, apiKey, baseURL);
const modelDef = getCloudModel(model);
const needsDisableThinking = provider === "groq" && modelDef?.disableThinking;
@@ -1352,6 +1364,8 @@ class ReasoningService extends BaseReasoningService {
messageCount: messages.length,
});
+ const useTemperature = isLocalProvider || apiConfig.supportsTemperature;
+
const result = streamText({
model: aiModel,
messages: messages.map((m) => ({
@@ -1360,7 +1374,7 @@ class ReasoningService extends BaseReasoningService {
})),
tools: tools || undefined,
stopWhen: stepCountIs(tools ? ReasoningService.MAX_TOOL_STEPS : 1),
- ...(apiConfig.supportsTemperature ? { temperature: config.temperature ?? 0.3 } : {}),
+ ...(useTemperature ? { temperature: config.temperature ?? 0.3 } : {}),
maxOutputTokens: config.maxTokens || 4096,
...(needsDisableThinking ? { providerOptions: { groq: { reasoningEffort: "none" } } } : {}),
});
diff --git a/src/services/ai/providers.ts b/src/services/ai/providers.ts
index b760cf958..e0b3c1461 100644
--- a/src/services/ai/providers.ts
+++ b/src/services/ai/providers.ts
@@ -21,6 +21,11 @@ export function getAIModel(
return createGoogleGenerativeAI({ apiKey })(model);
case "custom":
return createOpenAI({ apiKey, baseURL })(model);
+ case "local":
+ return createOpenAI({
+ apiKey: "no-key",
+ baseURL,
+ }).chat(model);
default:
throw new Error(`Unsupported AI SDK provider: ${provider}`);
}