Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.
JavaScript aianthropiccross-platformgeminigroqlinuxmacosnvidiaopen-sourceopenaiparakeetspeech-to-texttranscribewhisperwindows
feat: add cloud agent tool support with NDJSON streaming
Gabriel Steinbbc7748f4af5
diff --git a/preload.js b/preload.jsindex 4ba03b1d1..d5545f15b 100644--- a/preload.js+++ b/preload.js@@ -580,11 +580,6 @@ contextBridge.exposeInMainWorld("electronAPI", {// Agent cloud streamingcloudAgentStream: (messages, opts) => ipcRenderer.invoke("cloud-agent-stream", messages, opts),- onAgentStreamChunk: registerListener(- "agent-stream-chunk",- (callback) => (_event, chunk) => callback(chunk)- ),- onAgentStreamDone: registerListener("agent-stream-done", (callback) => () => callback()),agentWebSearch: (query, numResults) => ipcRenderer.invoke("agent-web-search", query, numResults),// Agent conversation persistencediff --git a/src/components/AgentOverlay.tsx b/src/components/AgentOverlay.tsxindex 451037862..a3975f101 100644--- a/src/components/AgentOverlay.tsx+++ b/src/components/AgentOverlay.tsx@@ -5,7 +5,7 @@ import { AgentTitleBar } from "./agent/AgentTitleBar";import { AgentChat } from "./agent/AgentChat";import { AgentInput } from "./agent/AgentInput";import AudioManager from "../helpers/audioManager";-import ReasoningService from "../services/ReasoningService";+import ReasoningService, { type AgentStreamChunk } from "../services/ReasoningService";import { getSettings } from "../stores/settingsStore";import { getAgentSystemPrompt } from "../config/prompts";import { createToolRegistry } from "../services/tools";@@ -58,6 +58,7 @@ export default function AgentOverlay() {}, [agentState]);useEffect(() => {+ mountedRef.current = true;return () => {mountedRef.current = false;};@@ -108,8 +109,7 @@ export default function AgentOverlay() {const isCloudAgent = settings.isSignedIn && settings.cloudAgentMode === "openwhispr";const toolSupportedProviders = ["openai", "groq", "custom", "anthropic", "gemini"];- const supportsTools =- !isCloudAgent && toolSupportedProviders.includes(settings.agentProvider);+ const supportsTools = isCloudAgent || toolSupportedProviders.includes(settings.agentProvider);const registry = supportsTools? createToolRegistry({@@ -133,94 +133,89 @@ export default function AgentOverlay() {try {let fullContent = "";+ let stream: AsyncGenerator<AgentStreamChunk>;++ if (isCloudAgent) {+ const executeToolCall = registry+ ? async (name: string, argsJson: string) => {+ const tool = registry.get(name);+ if (!tool) return `Unknown tool: ${name}`;+ const args = JSON.parse(argsJson);+ const result = await tool.execute(args);+ if (!result.success) return result.displayText;+ return typeof result.data === "string" ? result.data : JSON.stringify(result.data);+ }+ : undefined;- if (supportsTools && registry) {- const aiTools = registry.toAISDKFormat();- const stream = ReasoningService.processTextStreamingAI(+ stream = ReasoningService.processTextStreamingCloud(llmMessages, {+ systemPrompt,+ tools: registry?.getAll().map((t) => ({+ name: t.name,+ description: t.description,+ parameters: t.parameters,+ })),+ executeToolCall,+ });+ } else {+ const aiTools = registry?.toAISDKFormat();+ stream = ReasoningService.processTextStreamingAI(llmMessages,settings.agentModel,settings.agentProvider,{ systemPrompt },aiTools);+ }- for await (const chunk of stream) {- if (!mountedRef.current) break;- if (chunk.type === "content") {- fullContent += chunk.text;- setMessages((prev) =>- prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))+ for await (const chunk of stream) {+ if (!mountedRef.current) break;+ if (chunk.type === "content") {+ fullContent += chunk.text;+ setMessages((prev) =>+ prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))+ );+ } else if (chunk.type === "tool_calls") {+ for (const call of chunk.calls) {+ setAgentState("tool-executing");+ setToolStatus(+ t(`agentMode.tools.${call.name}Status`, { defaultValue: `Using ${call.name}...` }));- } else if (chunk.type === "tool_calls") {- for (const call of chunk.calls) {- setAgentState("tool-executing");- setToolStatus(- t(`agentMode.tools.${call.name}Status`, { defaultValue: `Using ${call.name}...` })- );- setMessages((prev) =>- prev.map((m) =>- m.id === assistantId- ? {- ...m,- toolCalls: [- ...(m.toolCalls || []),- {- id: call.id,- name: call.name,- arguments: call.arguments,- status: "executing" as const,- },- ],- }- : m- )- );- }- } else if (chunk.type === "tool_result") {setMessages((prev) =>prev.map((m) =>- m.id === assistantId && m.toolCalls+ m.id === assistantId? {...m,- toolCalls: m.toolCalls.map((tc) =>- tc.id === chunk.callId- ? { ...tc, status: "completed" as const, result: chunk.displayText }- : tc- ),+ toolCalls: [+ ...(m.toolCalls || []),+ {+ id: call.id,+ name: call.name,+ arguments: call.arguments,+ status: "executing" as const,+ },+ ],}: m));- setAgentState("streaming");- setToolStatus("");}- }- } else if (isCloudAgent) {- const streamSource = ReasoningService.processTextStreamingCloud(llmMessages, {- systemPrompt,- });- for await (const chunk of streamSource) {- if (!mountedRef.current) break;- fullContent += chunk;+ } else if (chunk.type === "tool_result") {setMessages((prev) =>- prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))+ prev.map((m) =>+ m.id === assistantId && m.toolCalls+ ? {+ ...m,+ toolCalls: m.toolCalls.map((tc) =>+ tc.id === chunk.callId+ ? { ...tc, status: "completed" as const, result: chunk.displayText }+ : tc+ ),+ }+ : m+ ));- }- } else {- const stream = ReasoningService.processTextStreamingAI(- llmMessages,- settings.agentModel,- settings.agentProvider,- { systemPrompt }- );- for await (const chunk of stream) {- if (!mountedRef.current) break;- if (chunk.type === "content") {- fullContent += chunk.text;- setMessages((prev) =>- prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))- );- }+ setAgentState("streaming");+ setToolStatus("");}}diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.jsindex 0721414cb..bfe0e08d7 100644--- a/src/helpers/ipcHandlers.js+++ b/src/helpers/ipcHandlers.js@@ -2658,7 +2658,7 @@ class IPCHandlers {debugLogger.debug("Cloud agent stream request",- { messageCount: messages?.length || 0 },+ { messageCount: messages?.length || 0, toolCount: opts.tools?.length || 0 },"cloud-api");@@ -2671,6 +2671,7 @@ class IPCHandlers {body: JSON.stringify({messages,systemPrompt: opts.systemPrompt,+ tools: opts.tools,sessionId: this.sessionId,clientType: "desktop",appVersion: app.getVersion(),@@ -2690,24 +2691,42 @@ class IPCHandlers {const reader = response.body.getReader();const decoder = new TextDecoder();+ let buffer = "";+ const events = [];try {while (true) {const { done, value } = await reader.read();if (done) break;- const text = decoder.decode(value, { stream: true });- if (text) event.sender.send("agent-stream-chunk", text);+ buffer += decoder.decode(value, { stream: true });+ const lines = buffer.split("\n");+ buffer = lines.pop() || "";++ for (const line of lines) {+ if (!line.trim()) continue;+ try {+ events.push(JSON.parse(line));+ } catch {+ // Non-JSON line, skip+ }+ }+ }+ if (buffer.trim()) {+ try {+ events.push(JSON.parse(buffer));+ } catch {+ // Non-JSON remainder, skip+ }}} finally {reader.releaseLock();}- event.sender.send("agent-stream-done");- return { success: true };+ debugLogger.debug("Agent stream complete", { eventCount: events.length }, "cloud-api");+ return { success: true, events };} catch (error) {debugLogger.error("Cloud agent stream error:", error);- event.sender.send("agent-stream-done");return { success: false, error: error.message };}});diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.tsindex 1704dc0c2..c6bffdf53 100644--- a/src/services/ReasoningService.ts+++ b/src/services/ReasoningService.ts@@ -20,6 +20,7 @@ class ReasoningService extends BaseReasoningService {private apiKeyCache: SecureCache<string>;private openAiEndpointPreference = new Map<string, "responses" | "chat">();private static readonly OPENAI_ENDPOINT_PREF_STORAGE_KEY = "openAiEndpointPreference";+ private static readonly MAX_TOOL_STEPS = 5;private cacheCleanupStop: (() => void) | undefined;constructor() {@@ -1280,7 +1281,6 @@ class ReasoningService extends BaseReasoningService {const cloudProviders = ["openai", "groq", "gemini", "anthropic", "custom"];const isLocalProvider = !cloudProviders.includes(provider);- // Local models don't work with AI SDK — fall back to content-only streamingif (isLocalProvider) {const contentGen = this.processTextStreaming(messages, model, provider, config);for await (const text of contentGen) {@@ -1315,7 +1315,7 @@ class ReasoningService extends BaseReasoningService {content: m.content,})),tools: tools || undefined,- stopWhen: stepCountIs(tools ? 5 : 1),+ stopWhen: stepCountIs(tools ? ReasoningService.MAX_TOOL_STEPS : 1),...(apiConfig.supportsTemperature ? { temperature: config.temperature ?? 0.3 } : {}),maxOutputTokens: config.maxTokens || 4096,...(needsDisableThinking ? { providerOptions: { groq: { reasoningEffort: "none" } } } : {}),@@ -1352,45 +1352,91 @@ class ReasoningService extends BaseReasoningService {}async *processTextStreamingCloud(- messages: Array<{ role: string; content: string }>,- config: { systemPrompt: string }- ): AsyncGenerator<string, void, unknown> {- const chunks: string[] = [];- let done = false;- let waiting: (() => void) | null = null;-- const unsubChunk = window.electronAPI?.onAgentStreamChunk?.((chunk: string) => {- chunks.push(chunk);- waiting?.();- });- const unsubDone = window.electronAPI?.onAgentStreamDone?.(() => {- done = true;- waiting?.();- });+ messages: Array<{ role: string; content: string | Array<unknown> }>,+ config: {+ systemPrompt: string;+ tools?: Array<{ name: string; description: string; parameters: Record<string, unknown> }>;+ executeToolCall?: (name: string, args: string) => Promise<string>;+ }+ ): AsyncGenerator<AgentStreamChunk, void, unknown> {+ const maxSteps = config.tools?.length ? ReasoningService.MAX_TOOL_STEPS : 1;+ let currentMessages = [...messages];- try {- const result = await window.electronAPI?.cloudAgentStream?.(messages, {+ for (let step = 0; step < maxSteps; step++) {+ const result = await window.electronAPI?.cloudAgentStream?.(currentMessages, {systemPrompt: config.systemPrompt,+ tools: config.tools,});- if (result && !result.success) {- throw new Error(result.error || "Cloud agent streaming failed");+ if (!result || !result.success) {+ throw new Error(result?.error || "Cloud agent streaming failed");}- while (!done || chunks.length > 0) {- if (chunks.length > 0) {- yield chunks.shift()!;- } else if (!done) {- await new Promise<void>((r) => {- waiting = r;- });- waiting = null;+ const events = result.events || [];+ const pendingToolCalls: Array<{ id: string; name: string; arguments: string }> = [];++ for (const ev of events) {+ if (ev.type === "content") {+ yield { type: "content", text: ev.text as string };+ } else if (ev.type === "tool_call") {+ const call = {+ id: ev.id as string,+ name: ev.name as string,+ arguments: ev.arguments as string,+ };+ pendingToolCalls.push(call);+ yield { type: "tool_calls", calls: [call] };}}- } finally {- unsubChunk?.();- unsubDone?.();++ if (pendingToolCalls.length === 0 || !config.executeToolCall) {+ yield { type: "done", finishReason: "stop" };+ return;+ }++ for (const call of pendingToolCalls) {+ let toolResult: string;+ try {+ toolResult = await config.executeToolCall(call.name, call.arguments);+ } catch (error) {+ toolResult = `Error: ${(error as Error).message}`;+ }+ yield {+ type: "tool_result",+ callId: call.id,+ toolName: call.name,+ displayText: toolResult,+ };++ currentMessages = [+ ...currentMessages,+ {+ role: "assistant",+ content: [+ {+ type: "tool-call",+ toolCallId: call.id,+ toolName: call.name,+ input: JSON.parse(call.arguments),+ },+ ],+ },+ {+ role: "tool",+ content: [+ {+ type: "tool-result",+ toolCallId: call.id,+ toolName: call.name,+ output: { type: "text", value: toolResult },+ },+ ],+ },+ ];+ }}++ yield { type: "done", finishReason: "stop" };}async isAvailable(): Promise<boolean> {diff --git a/src/services/tools/webSearchTool.ts b/src/services/tools/webSearchTool.tsindex b5d681760..42db80cca 100644--- a/src/services/tools/webSearchTool.ts+++ b/src/services/tools/webSearchTool.ts@@ -3,7 +3,7 @@ import type { ToolDefinition, ToolResult } from "./ToolRegistry";export const webSearchTool: ToolDefinition = {name: "web_search",description:- "Search the web for current information. Returns relevant web results with titles and snippets.",+ "Search the web for current information. Returns relevant web results with titles, URLs, and article text.",parameters: {type: "object",properties: {diff --git a/src/types/electron.ts b/src/types/electron.tsindex b0dfb125e..9106971ba 100644--- a/src/types/electron.ts+++ b/src/types/electron.ts@@ -1073,11 +1073,24 @@ declare global {// Agent cloud streamingcloudAgentStream?: (- messages: Array<{ role: string; content: string }>,- opts?: { systemPrompt?: string }- ) => Promise<{ success: boolean; error?: string; code?: string }>;- onAgentStreamChunk?: (callback: (chunk: string) => void) => () => void;- onAgentStreamDone?: (callback: () => void) => () => void;+ messages: Array<{ role: string; content: string | Array<unknown> }>,+ opts?: {+ systemPrompt?: string;+ tools?: Array<{ name: string; description: string; parameters: Record<string, unknown> }>;+ }+ ) => Promise<{+ success: boolean;+ error?: string;+ code?: string;+ events?: Array<{+ type: "content" | "tool_call" | "done";+ text?: string;+ id?: string;+ name?: string;+ arguments?: string;+ finishReason?: string;+ }>;+ }>;agentWebSearch?: (query: string,numResults?: number