Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat: add cloud agent tool support with NDJSON streaming

Gabriel Steinbbc7748f4af5
diff --git a/preload.js b/preload.js
index 4ba03b1d1..d5545f15b 100644
--- a/preload.js
+++ b/preload.js
@@ -580,11 +580,6 @@ contextBridge.exposeInMainWorld("electronAPI", {
// Agent cloud streaming
cloudAgentStream: (messages, opts) => ipcRenderer.invoke("cloud-agent-stream", messages, opts),
- onAgentStreamChunk: registerListener(
- "agent-stream-chunk",
- (callback) => (_event, chunk) => callback(chunk)
- ),
- onAgentStreamDone: registerListener("agent-stream-done", (callback) => () => callback()),
agentWebSearch: (query, numResults) => ipcRenderer.invoke("agent-web-search", query, numResults),
// Agent conversation persistence
diff --git a/src/components/AgentOverlay.tsx b/src/components/AgentOverlay.tsx
index 451037862..a3975f101 100644
--- a/src/components/AgentOverlay.tsx
+++ b/src/components/AgentOverlay.tsx
@@ -5,7 +5,7 @@ import { AgentTitleBar } from "./agent/AgentTitleBar";
import { AgentChat } from "./agent/AgentChat";
import { AgentInput } from "./agent/AgentInput";
import AudioManager from "../helpers/audioManager";
-import ReasoningService from "../services/ReasoningService";
+import ReasoningService, { type AgentStreamChunk } from "../services/ReasoningService";
import { getSettings } from "../stores/settingsStore";
import { getAgentSystemPrompt } from "../config/prompts";
import { createToolRegistry } from "../services/tools";
@@ -58,6 +58,7 @@ export default function AgentOverlay() {
}, [agentState]);
useEffect(() => {
+ mountedRef.current = true;
return () => {
mountedRef.current = false;
};
@@ -108,8 +109,7 @@ export default function AgentOverlay() {
const isCloudAgent = settings.isSignedIn && settings.cloudAgentMode === "openwhispr";
const toolSupportedProviders = ["openai", "groq", "custom", "anthropic", "gemini"];
- const supportsTools =
- !isCloudAgent && toolSupportedProviders.includes(settings.agentProvider);
+ const supportsTools = isCloudAgent || toolSupportedProviders.includes(settings.agentProvider);
const registry = supportsTools
? createToolRegistry({
@@ -133,94 +133,89 @@ export default function AgentOverlay() {
try {
let fullContent = "";
+ let stream: AsyncGenerator<AgentStreamChunk>;
+
+ if (isCloudAgent) {
+ const executeToolCall = registry
+ ? async (name: string, argsJson: string) => {
+ const tool = registry.get(name);
+ if (!tool) return `Unknown tool: ${name}`;
+ const args = JSON.parse(argsJson);
+ const result = await tool.execute(args);
+ if (!result.success) return result.displayText;
+ return typeof result.data === "string" ? result.data : JSON.stringify(result.data);
+ }
+ : undefined;
- if (supportsTools && registry) {
- const aiTools = registry.toAISDKFormat();
- const stream = ReasoningService.processTextStreamingAI(
+ stream = ReasoningService.processTextStreamingCloud(llmMessages, {
+ systemPrompt,
+ tools: registry?.getAll().map((t) => ({
+ name: t.name,
+ description: t.description,
+ parameters: t.parameters,
+ })),
+ executeToolCall,
+ });
+ } else {
+ const aiTools = registry?.toAISDKFormat();
+ stream = ReasoningService.processTextStreamingAI(
llmMessages,
settings.agentModel,
settings.agentProvider,
{ systemPrompt },
aiTools
);
+ }
- for await (const chunk of stream) {
- if (!mountedRef.current) break;
- if (chunk.type === "content") {
- fullContent += chunk.text;
- setMessages((prev) =>
- prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
+ for await (const chunk of stream) {
+ if (!mountedRef.current) break;
+ if (chunk.type === "content") {
+ fullContent += chunk.text;
+ setMessages((prev) =>
+ prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
+ );
+ } else if (chunk.type === "tool_calls") {
+ for (const call of chunk.calls) {
+ setAgentState("tool-executing");
+ setToolStatus(
+ t(`agentMode.tools.${call.name}Status`, { defaultValue: `Using ${call.name}...` })
);
- } else if (chunk.type === "tool_calls") {
- for (const call of chunk.calls) {
- setAgentState("tool-executing");
- setToolStatus(
- t(`agentMode.tools.${call.name}Status`, { defaultValue: `Using ${call.name}...` })
- );
- setMessages((prev) =>
- prev.map((m) =>
- m.id === assistantId
- ? {
- ...m,
- toolCalls: [
- ...(m.toolCalls || []),
- {
- id: call.id,
- name: call.name,
- arguments: call.arguments,
- status: "executing" as const,
- },
- ],
- }
- : m
- )
- );
- }
- } else if (chunk.type === "tool_result") {
setMessages((prev) =>
prev.map((m) =>
- m.id === assistantId && m.toolCalls
+ m.id === assistantId
? {
...m,
- toolCalls: m.toolCalls.map((tc) =>
- tc.id === chunk.callId
- ? { ...tc, status: "completed" as const, result: chunk.displayText }
- : tc
- ),
+ toolCalls: [
+ ...(m.toolCalls || []),
+ {
+ id: call.id,
+ name: call.name,
+ arguments: call.arguments,
+ status: "executing" as const,
+ },
+ ],
}
: m
)
);
- setAgentState("streaming");
- setToolStatus("");
}
- }
- } else if (isCloudAgent) {
- const streamSource = ReasoningService.processTextStreamingCloud(llmMessages, {
- systemPrompt,
- });
- for await (const chunk of streamSource) {
- if (!mountedRef.current) break;
- fullContent += chunk;
+ } else if (chunk.type === "tool_result") {
setMessages((prev) =>
- prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
+ prev.map((m) =>
+ m.id === assistantId && m.toolCalls
+ ? {
+ ...m,
+ toolCalls: m.toolCalls.map((tc) =>
+ tc.id === chunk.callId
+ ? { ...tc, status: "completed" as const, result: chunk.displayText }
+ : tc
+ ),
+ }
+ : m
+ )
);
- }
- } else {
- const stream = ReasoningService.processTextStreamingAI(
- llmMessages,
- settings.agentModel,
- settings.agentProvider,
- { systemPrompt }
- );
- for await (const chunk of stream) {
- if (!mountedRef.current) break;
- if (chunk.type === "content") {
- fullContent += chunk.text;
- setMessages((prev) =>
- prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
- );
- }
+ setAgentState("streaming");
+ setToolStatus("");
}
}
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index 0721414cb..bfe0e08d7 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -2658,7 +2658,7 @@ class IPCHandlers {
debugLogger.debug(
"Cloud agent stream request",
- { messageCount: messages?.length || 0 },
+ { messageCount: messages?.length || 0, toolCount: opts.tools?.length || 0 },
"cloud-api"
);
@@ -2671,6 +2671,7 @@ class IPCHandlers {
body: JSON.stringify({
messages,
systemPrompt: opts.systemPrompt,
+ tools: opts.tools,
sessionId: this.sessionId,
clientType: "desktop",
appVersion: app.getVersion(),
@@ -2690,24 +2691,42 @@ class IPCHandlers {
const reader = response.body.getReader();
const decoder = new TextDecoder();
+ let buffer = "";
+ const events = [];
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
- const text = decoder.decode(value, { stream: true });
- if (text) event.sender.send("agent-stream-chunk", text);
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split("\n");
+ buffer = lines.pop() || "";
+
+ for (const line of lines) {
+ if (!line.trim()) continue;
+ try {
+ events.push(JSON.parse(line));
+ } catch {
+ // Non-JSON line, skip
+ }
+ }
+ }
+ if (buffer.trim()) {
+ try {
+ events.push(JSON.parse(buffer));
+ } catch {
+ // Non-JSON remainder, skip
+ }
}
} finally {
reader.releaseLock();
}
- event.sender.send("agent-stream-done");
- return { success: true };
+ debugLogger.debug("Agent stream complete", { eventCount: events.length }, "cloud-api");
+ return { success: true, events };
} catch (error) {
debugLogger.error("Cloud agent stream error:", error);
- event.sender.send("agent-stream-done");
return { success: false, error: error.message };
}
});
diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.ts
index 1704dc0c2..c6bffdf53 100644
--- a/src/services/ReasoningService.ts
+++ b/src/services/ReasoningService.ts
@@ -20,6 +20,7 @@ class ReasoningService extends BaseReasoningService {
private apiKeyCache: SecureCache<string>;
private openAiEndpointPreference = new Map<string, "responses" | "chat">();
private static readonly OPENAI_ENDPOINT_PREF_STORAGE_KEY = "openAiEndpointPreference";
+ private static readonly MAX_TOOL_STEPS = 5;
private cacheCleanupStop: (() => void) | undefined;
constructor() {
@@ -1280,7 +1281,6 @@ class ReasoningService extends BaseReasoningService {
const cloudProviders = ["openai", "groq", "gemini", "anthropic", "custom"];
const isLocalProvider = !cloudProviders.includes(provider);
- // Local models don't work with AI SDK — fall back to content-only streaming
if (isLocalProvider) {
const contentGen = this.processTextStreaming(messages, model, provider, config);
for await (const text of contentGen) {
@@ -1315,7 +1315,7 @@ class ReasoningService extends BaseReasoningService {
content: m.content,
})),
tools: tools || undefined,
- stopWhen: stepCountIs(tools ? 5 : 1),
+ stopWhen: stepCountIs(tools ? ReasoningService.MAX_TOOL_STEPS : 1),
...(apiConfig.supportsTemperature ? { temperature: config.temperature ?? 0.3 } : {}),
maxOutputTokens: config.maxTokens || 4096,
...(needsDisableThinking ? { providerOptions: { groq: { reasoningEffort: "none" } } } : {}),
@@ -1352,45 +1352,91 @@ class ReasoningService extends BaseReasoningService {
}
async *processTextStreamingCloud(
- messages: Array<{ role: string; content: string }>,
- config: { systemPrompt: string }
- ): AsyncGenerator<string, void, unknown> {
- const chunks: string[] = [];
- let done = false;
- let waiting: (() => void) | null = null;
-
- const unsubChunk = window.electronAPI?.onAgentStreamChunk?.((chunk: string) => {
- chunks.push(chunk);
- waiting?.();
- });
- const unsubDone = window.electronAPI?.onAgentStreamDone?.(() => {
- done = true;
- waiting?.();
- });
+ messages: Array<{ role: string; content: string | Array<unknown> }>,
+ config: {
+ systemPrompt: string;
+ tools?: Array<{ name: string; description: string; parameters: Record<string, unknown> }>;
+ executeToolCall?: (name: string, args: string) => Promise<string>;
+ }
+ ): AsyncGenerator<AgentStreamChunk, void, unknown> {
+ const maxSteps = config.tools?.length ? ReasoningService.MAX_TOOL_STEPS : 1;
+ let currentMessages = [...messages];
- try {
- const result = await window.electronAPI?.cloudAgentStream?.(messages, {
+ for (let step = 0; step < maxSteps; step++) {
+ const result = await window.electronAPI?.cloudAgentStream?.(currentMessages, {
systemPrompt: config.systemPrompt,
+ tools: config.tools,
});
- if (result && !result.success) {
- throw new Error(result.error || "Cloud agent streaming failed");
+ if (!result || !result.success) {
+ throw new Error(result?.error || "Cloud agent streaming failed");
}
- while (!done || chunks.length > 0) {
- if (chunks.length > 0) {
- yield chunks.shift()!;
- } else if (!done) {
- await new Promise<void>((r) => {
- waiting = r;
- });
- waiting = null;
+ const events = result.events || [];
+ const pendingToolCalls: Array<{ id: string; name: string; arguments: string }> = [];
+
+ for (const ev of events) {
+ if (ev.type === "content") {
+ yield { type: "content", text: ev.text as string };
+ } else if (ev.type === "tool_call") {
+ const call = {
+ id: ev.id as string,
+ name: ev.name as string,
+ arguments: ev.arguments as string,
+ };
+ pendingToolCalls.push(call);
+ yield { type: "tool_calls", calls: [call] };
}
}
- } finally {
- unsubChunk?.();
- unsubDone?.();
+
+ if (pendingToolCalls.length === 0 || !config.executeToolCall) {
+ yield { type: "done", finishReason: "stop" };
+ return;
+ }
+
+ for (const call of pendingToolCalls) {
+ let toolResult: string;
+ try {
+ toolResult = await config.executeToolCall(call.name, call.arguments);
+ } catch (error) {
+ toolResult = `Error: ${(error as Error).message}`;
+ }
+ yield {
+ type: "tool_result",
+ callId: call.id,
+ toolName: call.name,
+ displayText: toolResult,
+ };
+
+ currentMessages = [
+ ...currentMessages,
+ {
+ role: "assistant",
+ content: [
+ {
+ type: "tool-call",
+ toolCallId: call.id,
+ toolName: call.name,
+ input: JSON.parse(call.arguments),
+ },
+ ],
+ },
+ {
+ role: "tool",
+ content: [
+ {
+ type: "tool-result",
+ toolCallId: call.id,
+ toolName: call.name,
+ output: { type: "text", value: toolResult },
+ },
+ ],
+ },
+ ];
+ }
}
+
+ yield { type: "done", finishReason: "stop" };
}
async isAvailable(): Promise<boolean> {
diff --git a/src/services/tools/webSearchTool.ts b/src/services/tools/webSearchTool.ts
index b5d681760..42db80cca 100644
--- a/src/services/tools/webSearchTool.ts
+++ b/src/services/tools/webSearchTool.ts
@@ -3,7 +3,7 @@ import type { ToolDefinition, ToolResult } from "./ToolRegistry";
export const webSearchTool: ToolDefinition = {
name: "web_search",
description:
- "Search the web for current information. Returns relevant web results with titles and snippets.",
+ "Search the web for current information. Returns relevant web results with titles, URLs, and article text.",
parameters: {
type: "object",
properties: {
diff --git a/src/types/electron.ts b/src/types/electron.ts
index b0dfb125e..9106971ba 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -1073,11 +1073,24 @@ declare global {
// Agent cloud streaming
cloudAgentStream?: (
- messages: Array<{ role: string; content: string }>,
- opts?: { systemPrompt?: string }
- ) => Promise<{ success: boolean; error?: string; code?: string }>;
- onAgentStreamChunk?: (callback: (chunk: string) => void) => () => void;
- onAgentStreamDone?: (callback: () => void) => () => void;
+ messages: Array<{ role: string; content: string | Array<unknown> }>,
+ opts?: {
+ systemPrompt?: string;
+ tools?: Array<{ name: string; description: string; parameters: Record<string, unknown> }>;
+ }
+ ) => Promise<{
+ success: boolean;
+ error?: string;
+ code?: string;
+ events?: Array<{
+ type: "content" | "tool_call" | "done";
+ text?: string;
+ id?: string;
+ name?: string;
+ arguments?: string;
+ finishReason?: string;
+ }>;
+ }>;
agentWebSearch?: (
query: string,
numResults?: number