Skip to content

Voice-to-text dictation app with local (Nvidia Parakeet/Whisper) and cloud models (BYOK). Privacy-first and available cross-platform.

feat: add agentic tool-calling system to Agent Mode

Gabriel Steinde141c5944bc
diff --git a/preload.js b/preload.js
index 5575aec20..4ba03b1d1 100644
--- a/preload.js
+++ b/preload.js
@@ -585,6 +585,7 @@ contextBridge.exposeInMainWorld("electronAPI", {
(callback) => (_event, chunk) => callback(chunk)
),
onAgentStreamDone: registerListener("agent-stream-done", (callback) => () => callback()),
+ agentWebSearch: (query, numResults) => ipcRenderer.invoke("agent-web-search", query, numResults),
// Agent conversation persistence
createAgentConversation: (title) => ipcRenderer.invoke("db-create-agent-conversation", title),
diff --git a/src/components/AgentOverlay.tsx b/src/components/AgentOverlay.tsx
index ec0e44329..0687de051 100644
--- a/src/components/AgentOverlay.tsx
+++ b/src/components/AgentOverlay.tsx
@@ -1,4 +1,5 @@
import { useState, useEffect, useRef, useCallback } from "react";
+import { useTranslation } from "react-i18next";
import { cn } from "./lib/utils";
import { AgentTitleBar } from "./agent/AgentTitleBar";
import { AgentChat } from "./agent/AgentChat";
@@ -7,23 +8,42 @@ import AudioManager from "../helpers/audioManager";
import ReasoningService from "../services/ReasoningService";
import { getSettings } from "../stores/settingsStore";
import { getAgentSystemPrompt } from "../config/prompts";
-
-type AgentState = "idle" | "listening" | "transcribing" | "thinking" | "streaming";
+import { run as runAgentLoop } from "../services/AgentLoop";
+import { createToolRegistry } from "../services/tools";
+
+type AgentState =
+ | "idle"
+ | "listening"
+ | "transcribing"
+ | "thinking"
+ | "streaming"
+ | "tool-executing";
+
+interface ToolCallEntry {
+ id: string;
+ name: string;
+ arguments: string;
+ status: "executing" | "completed" | "error";
+ result?: string;
+}
interface Message {
id: string;
- role: "user" | "assistant";
+ role: "user" | "assistant" | "tool";
content: string;
isStreaming: boolean;
+ toolCalls?: ToolCallEntry[];
}
const MIN_HEIGHT = 200;
const MIN_WIDTH = 360;
export default function AgentOverlay() {
+ const { t } = useTranslation();
const [messages, setMessages] = useState<Message[]>([]);
const [agentState, setAgentState] = useState<AgentState>("idle");
const [partialTranscript, setPartialTranscript] = useState("");
+ const [toolStatus, setToolStatus] = useState("");
const audioManagerRef = useRef<InstanceType<typeof AudioManager> | null>(null);
const messagesRef = useRef<Message[]>([]);
const agentStateRef = useRef<AgentState>("idle");
@@ -44,94 +64,175 @@ export default function AgentOverlay() {
]);
}, []);
- const handleTranscriptionComplete = useCallback(async (text: string) => {
- if (!text.trim()) {
- setAgentState("idle");
- return;
- }
-
- // Create conversation on first message
- if (!conversationIdRef.current) {
- const conv = await window.electronAPI?.createAgentConversation?.("New conversation");
- conversationIdRef.current = conv?.id ?? null;
- }
-
- const userMsg: Message = {
- id: crypto.randomUUID(),
- role: "user",
- content: text,
- isStreaming: false,
- };
- setMessages((prev) => [...prev, userMsg]);
-
- if (conversationIdRef.current) {
- window.electronAPI?.addAgentMessage?.(conversationIdRef.current, "user", text);
- }
-
- // Auto-title after first user message
- const allMessages = messagesRef.current;
- if (conversationIdRef.current && allMessages.length === 0) {
- const title = text.slice(0, 50) + (text.length > 50 ? "..." : "");
- window.electronAPI?.updateAgentConversationTitle?.(conversationIdRef.current, title);
- }
+ const handleTranscriptionComplete = useCallback(
+ async (text: string) => {
+ if (!text.trim()) {
+ setAgentState("idle");
+ return;
+ }
- setAgentState("thinking");
+ // Create conversation on first message
+ if (!conversationIdRef.current) {
+ const conv = await window.electronAPI?.createAgentConversation?.("New conversation");
+ conversationIdRef.current = conv?.id ?? null;
+ }
- const settings = getSettings();
- const systemPrompt = getAgentSystemPrompt();
+ const userMsg: Message = {
+ id: crypto.randomUUID(),
+ role: "user",
+ content: text,
+ isStreaming: false,
+ };
+ setMessages((prev) => [...prev, userMsg]);
- const llmMessages = [
- { role: "system", content: systemPrompt },
- ...[...allMessages, userMsg].slice(-20).map((m) => ({ role: m.role, content: m.content })),
- ];
+ if (conversationIdRef.current) {
+ window.electronAPI?.addAgentMessage?.(conversationIdRef.current, "user", text);
+ }
- const assistantId = crypto.randomUUID();
- setMessages((prev) => [
- ...prev,
- { id: assistantId, role: "assistant", content: "", isStreaming: true },
- ]);
- setAgentState("streaming");
+ // Auto-title after first user message
+ const allMessages = messagesRef.current;
+ if (conversationIdRef.current && allMessages.length === 0) {
+ const title = text.slice(0, 50) + (text.length > 50 ? "..." : "");
+ window.electronAPI?.updateAgentConversationTitle?.(conversationIdRef.current, title);
+ }
- const isCloudAgent = settings.isSignedIn && settings.cloudAgentMode === "openwhispr";
+ setAgentState("thinking");
+
+ const settings = getSettings();
+
+ const isCloudAgent = settings.isSignedIn && settings.cloudAgentMode === "openwhispr";
+ const toolSupportedProviders = ["openai", "groq", "custom"];
+ const supportsTools =
+ !isCloudAgent && toolSupportedProviders.includes(settings.agentProvider);
+
+ const registry = supportsTools
+ ? createToolRegistry({
+ isSignedIn: settings.isSignedIn,
+ gcalConnected: settings.gcalConnected,
+ })
+ : null;
+ const systemPrompt = getAgentSystemPrompt(registry?.getAll().map((t) => t.name));
+
+ const llmMessages = [
+ { role: "system", content: systemPrompt },
+ ...[...allMessages, userMsg].slice(-20).map((m) => ({ role: m.role, content: m.content })),
+ ];
+
+ const assistantId = crypto.randomUUID();
+ setMessages((prev) => [
+ ...prev,
+ { id: assistantId, role: "assistant", content: "", isStreaming: true },
+ ]);
+ setAgentState("streaming");
+
+ try {
+ let fullContent = "";
+
+ if (supportsTools && registry) {
+ await runAgentLoop({
+ messages: llmMessages,
+ model: settings.agentModel,
+ provider: settings.agentProvider,
+ systemPrompt,
+ tools: registry,
+ callbacks: {
+ onContentChunk: (text) => {
+ fullContent += text;
+ setMessages((prev) =>
+ prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
+ );
+ },
+ onToolStart: (toolName) => {
+ setAgentState("tool-executing");
+ setToolStatus(
+ t(`agentMode.tools.${toolName}Status`, { defaultValue: `Using ${toolName}...` })
+ );
+ setMessages((prev) =>
+ prev.map((m) =>
+ m.id === assistantId
+ ? {
+ ...m,
+ toolCalls: [
+ ...(m.toolCalls || []),
+ {
+ id: crypto.randomUUID(),
+ name: toolName,
+ arguments: "",
+ status: "executing" as const,
+ },
+ ],
+ }
+ : m
+ )
+ );
+ },
+ onToolComplete: (toolName, result) => {
+ setAgentState("streaming");
+ setToolStatus("");
+ setMessages((prev) =>
+ prev.map((m) =>
+ m.id === assistantId
+ ? {
+ ...m,
+ toolCalls: m.toolCalls?.map((tc) =>
+ tc.name === toolName && tc.status === "executing"
+ ? { ...tc, status: "completed" as const, result: result.displayText }
+ : tc
+ ),
+ }
+ : m
+ )
+ );
+ },
+ onError: (error) => {
+ console.error("Agent loop error:", error);
+ },
+ },
+ maxIterations: 5,
+ });
+ } else {
+ const streamSource = isCloudAgent
+ ? ReasoningService.processTextStreamingCloud(llmMessages, { systemPrompt })
+ : ReasoningService.processTextStreaming(
+ llmMessages,
+ settings.agentModel,
+ settings.agentProvider,
+ { systemPrompt }
+ );
+
+ for await (const chunk of streamSource) {
+ fullContent += chunk;
+ setMessages((prev) =>
+ prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
+ );
+ }
+ }
- try {
- let fullContent = "";
+ setMessages((prev) =>
+ prev.map((m) => (m.id === assistantId ? { ...m, isStreaming: false } : m))
+ );
- const streamSource = isCloudAgent
- ? ReasoningService.processTextStreamingCloud(llmMessages, { systemPrompt })
- : ReasoningService.processTextStreaming(
- llmMessages,
- settings.agentModel,
- settings.agentProvider,
- { systemPrompt }
+ if (conversationIdRef.current) {
+ window.electronAPI?.addAgentMessage?.(
+ conversationIdRef.current,
+ "assistant",
+ fullContent
);
-
- for await (const chunk of streamSource) {
- fullContent += chunk;
+ }
+ } catch (error) {
setMessages((prev) =>
- prev.map((m) => (m.id === assistantId ? { ...m, content: fullContent } : m))
+ prev.map((m) =>
+ m.id === assistantId
+ ? { ...m, content: `Error: ${(error as Error).message}`, isStreaming: false }
+ : m
+ )
);
}
- setMessages((prev) =>
- prev.map((m) => (m.id === assistantId ? { ...m, isStreaming: false } : m))
- );
-
- if (conversationIdRef.current) {
- window.electronAPI?.addAgentMessage?.(conversationIdRef.current, "assistant", fullContent);
- }
- } catch (error) {
- setMessages((prev) =>
- prev.map((m) =>
- m.id === assistantId
- ? { ...m, content: `Error: ${(error as Error).message}`, isStreaming: false }
- : m
- )
- );
- }
-
- setAgentState("idle");
- }, []);
+ setAgentState("idle");
+ },
+ [t]
+ );
useEffect(() => {
const am = new AudioManager();
@@ -244,10 +345,18 @@ export default function AgentOverlay() {
};
}, []);
+ const handleTextSubmit = useCallback(
+ (text: string) => {
+ handleTranscriptionComplete(text);
+ },
+ [handleTranscriptionComplete]
+ );
+
const handleNewChat = useCallback(() => {
setMessages([]);
setAgentState("idle");
setPartialTranscript("");
+ setToolStatus("");
conversationIdRef.current = null;
}, []);
@@ -268,7 +377,12 @@ export default function AgentOverlay() {
>
<AgentTitleBar onNewChat={handleNewChat} onClose={handleClose} />
<AgentChat messages={messages} />
- <AgentInput agentState={agentState} partialTranscript={partialTranscript} />
+ <AgentInput
+ agentState={agentState}
+ partialTranscript={partialTranscript}
+ toolStatus={toolStatus}
+ onTextSubmit={handleTextSubmit}
+ />
</div>
{/* Resize handles — edges */}
diff --git a/src/components/agent/AgentChat.tsx b/src/components/agent/AgentChat.tsx
index 923b5d54f..b91991e14 100644
--- a/src/components/agent/AgentChat.tsx
+++ b/src/components/agent/AgentChat.tsx
@@ -5,11 +5,20 @@ import { AgentMessage } from "./AgentMessage";
import { useSettingsStore } from "../../stores/settingsStore";
import { formatHotkeyLabel } from "../../utils/hotkeys";
-interface Message {
+export interface ToolCallInfo {
id: string;
- role: "user" | "assistant";
+ name: string;
+ arguments: string;
+ status: "executing" | "completed" | "error";
+ result?: string;
+}
+
+export interface Message {
+ id: string;
+ role: "user" | "assistant" | "tool";
content: string;
isStreaming: boolean;
+ toolCalls?: ToolCallInfo[];
}
interface AgentChatProps {
@@ -39,14 +48,17 @@ export function AgentChat({ messages }: AgentChatProps) {
</div>
) : (
<div className="flex flex-col gap-2">
- {messages.map((msg) => (
- <AgentMessage
- key={msg.id}
- role={msg.role}
- content={msg.content}
- isStreaming={msg.isStreaming}
- />
- ))}
+ {messages
+ .filter((msg) => msg.role !== "tool")
+ .map((msg) => (
+ <AgentMessage
+ key={msg.id}
+ role={msg.role as "user" | "assistant"}
+ content={msg.content}
+ isStreaming={msg.isStreaming}
+ toolCalls={msg.toolCalls}
+ />
+ ))}
</div>
)}
</div>
diff --git a/src/components/agent/AgentInput.tsx b/src/components/agent/AgentInput.tsx
index a55c7b5cf..f0f86807a 100644
--- a/src/components/agent/AgentInput.tsx
+++ b/src/components/agent/AgentInput.tsx
@@ -1,14 +1,23 @@
-import { Mic } from "lucide-react";
+import { useState, useRef, useCallback } from "react";
+import { Mic, SendHorizontal } from "lucide-react";
import { useTranslation } from "react-i18next";
import { cn } from "../lib/utils";
import { useSettingsStore } from "../../stores/settingsStore";
import { formatHotkeyLabel, isGlobeLikeHotkey } from "../../utils/hotkeys";
-type AgentState = "idle" | "listening" | "transcribing" | "thinking" | "streaming";
+type AgentState =
+ | "idle"
+ | "listening"
+ | "transcribing"
+ | "thinking"
+ | "streaming"
+ | "tool-executing";
interface AgentInputProps {
agentState: AgentState;
partialTranscript: string;
+ toolStatus?: string;
+ onTextSubmit?: (text: string) => void;
}
function Kbd({ children }: { children: React.ReactNode }) {
@@ -80,37 +89,81 @@ function InputLoadingDots() {
);
}
-export function AgentInput({ agentState, partialTranscript }: AgentInputProps) {
+export function AgentInput({
+ agentState,
+ partialTranscript,
+ toolStatus,
+ onTextSubmit,
+}: AgentInputProps) {
const { t } = useTranslation();
const agentKey = useSettingsStore((s) => s.agentKey);
+ const [inputText, setInputText] = useState("");
+ const inputRef = useRef<HTMLInputElement>(null);
+
+ const handleSubmit = useCallback(() => {
+ const text = inputText.trim();
+ if (!text || !onTextSubmit) return;
+ onTextSubmit(text);
+ setInputText("");
+ }, [inputText, onTextSubmit]);
+
+ const handleKeyDown = useCallback(
+ (e: React.KeyboardEvent<HTMLInputElement>) => {
+ if (e.key === "Enter" && !e.shiftKey) {
+ e.preventDefault();
+ handleSubmit();
+ }
+ },
+ [handleSubmit]
+ );
+
+ const isIdle = agentState === "idle";
return (
<div
className={cn(
- "flex items-center gap-3 h-12 px-3 py-2 shrink-0",
+ "flex items-center gap-2 min-h-12 px-3 py-2 shrink-0",
"bg-surface-1 border-t border-border/30"
)}
>
- {agentState === "idle" && (
- <div className="flex items-center justify-between w-full">
- <div className="flex items-center gap-2">
- <div
- className="text-muted-foreground/50"
- style={{ animation: "agent-mic-pulse 2.5s ease-in-out infinite" }}
+ {isIdle && (
+ <div className="flex items-center gap-2 w-full">
+ <input
+ ref={inputRef}
+ type="text"
+ value={inputText}
+ onChange={(e) => setInputText(e.target.value)}
+ onKeyDown={handleKeyDown}
+ placeholder={t("agentMode.input.typeMessage")}
+ className={cn(
+ "bg-transparent border-none outline-none flex-1",
+ "text-[13px] text-foreground placeholder:text-muted-foreground/40",
+ "min-w-0"
+ )}
+ />
+ {inputText.trim() ? (
+ <button
+ onClick={handleSubmit}
+ className={cn(
+ "p-1 rounded-sm shrink-0",
+ "text-primary hover:text-primary/80",
+ "focus:outline-none focus-visible:ring-1 focus-visible:ring-ring/30",
+ "transition-colors duration-100"
+ )}
>
- <Mic size={14} />
+ <SendHorizontal size={14} />
+ </button>
+ ) : (
+ <div className="flex items-center gap-2 shrink-0">
+ <div
+ className="text-muted-foreground/50"
+ style={{ animation: "agent-mic-pulse 2.5s ease-in-out infinite" }}
+ >
+ <Mic size={14} />
+ </div>
+ <HotkeyKeys hotkey={agentKey} />
</div>
- <span className="text-[11px] text-muted-foreground/50 select-none">
- {t("agentMode.input.holdToSpeak")}
- </span>
- <HotkeyKeys hotkey={agentKey} />
- </div>
- <div className="flex items-center gap-1.5">
- <Kbd>Esc</Kbd>
- <span className="text-[10px] text-muted-foreground/35 select-none">
- {t("agentMode.input.toClose")}
- </span>
- </div>
+ )}
</div>
)}
@@ -140,6 +193,15 @@ export function AgentInput({ agentState, partialTranscript }: AgentInputProps) {
</span>
</>
)}
+
+ {agentState === "tool-executing" && (
+ <>
+ <InputLoadingDots />
+ <span className="text-[12px] text-muted-foreground select-none">
+ {toolStatus || t("agentMode.input.thinking")}
+ </span>
+ </>
+ )}
</div>
);
}
diff --git a/src/components/agent/AgentMessage.tsx b/src/components/agent/AgentMessage.tsx
index 1a569c06a..ecfe55eb9 100644
--- a/src/components/agent/AgentMessage.tsx
+++ b/src/components/agent/AgentMessage.tsx
@@ -1,15 +1,74 @@
import { useState } from "react";
-import { Copy, Check } from "lucide-react";
+import { Copy, Check, Search, Globe, ClipboardCopy, Calendar } from "lucide-react";
import { cn } from "../lib/utils";
import { MarkdownRenderer } from "../ui/MarkdownRenderer";
+import type { ToolCallInfo } from "./AgentChat";
interface AgentMessageProps {
role: "user" | "assistant";
content: string;
isStreaming: boolean;
+ toolCalls?: ToolCallInfo[];
}
-export function AgentMessage({ role, content, isStreaming }: AgentMessageProps) {
+const toolIcons: Record<string, typeof Search> = {
+ search_notes: Search,
+ web_search: Globe,
+ copy_to_clipboard: ClipboardCopy,
+ get_calendar_events: Calendar,
+};
+
+function ToolCallPill({ toolCall }: { toolCall: ToolCallInfo }) {
+ const [expanded, setExpanded] = useState(false);
+ const Icon = toolIcons[toolCall.name] || Search;
+ const isExecuting = toolCall.status === "executing";
+ const isError = toolCall.status === "error";
+ const resultLines = toolCall.result?.split("\n") ?? [];
+ const isExpandable = resultLines.length > 3;
+
+ return (
+ <div
+ className={cn(
+ "bg-surface-1 border border-border/20 rounded-md px-2.5 py-1.5 mb-1.5",
+ isExpandable && !isExecuting && "cursor-pointer"
+ )}
+ onClick={isExpandable && !isExecuting ? () => setExpanded((v) => !v) : undefined}
+ >
+ <div className="flex items-center gap-1.5">
+ <Icon size={12} className="text-muted-foreground/60 shrink-0" />
+ {isExecuting ? (
+ <div className="flex items-center gap-1">
+ {[0, 1, 2].map((i) => (
+ <div
+ key={i}
+ className="w-1 h-1 rounded-full bg-muted-foreground/50"
+ style={{ animation: `agent-loading-dot 1.2s ease-in-out ${i * 0.2}s infinite` }}
+ />
+ ))}
+ </div>
+ ) : (
+ <span
+ className={cn("text-[11px]", isError ? "text-destructive/70" : "text-muted-foreground")}
+ >
+ {toolCall.result || toolCall.name}
+ </span>
+ )}
+ </div>
+ {!isExecuting && isExpandable && (
+ <div
+ className="overflow-hidden transition-all duration-200"
+ style={{ maxHeight: expanded ? `${resultLines.length * 18 + 8}px` : "0px" }}
+ >
+ <pre className="text-[10px] text-muted-foreground/70 mt-1 whitespace-pre-wrap">
+ {toolCall.result}
+ </pre>
+ </div>
+ )}
+ </div>
+ );
+}
+
+export function AgentMessage({ role, content, isStreaming, toolCalls }: AgentMessageProps) {
const [copied, setCopied] = useState(false);
const handleCopy = async () => {
@@ -65,6 +124,14 @@ export function AgentMessage({ role, content, isStreaming }: AgentMessageProps)
{copied ? <Check size={14} /> : <Copy size={14} />}
</button>
+ {toolCalls && toolCalls.length > 0 && (
+ <div className="mb-1">
+ {toolCalls.map((tc) => (
+ <ToolCallPill key={tc.id} toolCall={tc} />
+ ))}
+ </div>
+ )}
+
<MarkdownRenderer
content={content}
className="text-[13px] leading-relaxed [&_p]:text-[13px] [&_li]:text-[13px]"
diff --git a/src/config/prompts.ts b/src/config/prompts.ts
index 257da3341..ab1b37b86 100644
--- a/src/config/prompts.ts
+++ b/src/config/prompts.ts
@@ -144,10 +144,31 @@ const DEFAULT_AGENT_SYSTEM_PROMPT =
"Keep answers brief unless the user asks for detail. " +
"You may be given a transcription of spoken input, so handle informal phrasing gracefully.";
-export function getAgentSystemPrompt(): string {
+const TOOL_INSTRUCTIONS: Record<string, string> = {
+ search_notes:
+ "Use search_notes to find information from the user's past meetings, discussions, or personal notes before answering from memory.",
+ web_search:
+ "Use web_search for questions about current events, facts you're unsure about, or anything requiring up-to-date information.",
+ copy_to_clipboard:
+ "Use copy_to_clipboard when the user asks you to copy something to their clipboard.",
+ get_calendar_events:
+ "Use get_calendar_events to check the user's schedule, upcoming meetings, or calendar events.",
+};
+
+export function getAgentSystemPrompt(availableTools?: string[]): string {
if (typeof window !== "undefined" && window.localStorage) {
const custom = window.localStorage.getItem("agentSystemPrompt");
if (custom) return custom;
}
- return DEFAULT_AGENT_SYSTEM_PROMPT;
+
+ let prompt = DEFAULT_AGENT_SYSTEM_PROMPT;
+
+ if (availableTools && availableTools.length > 0) {
+ const toolLines = availableTools.map((name) => TOOL_INSTRUCTIONS[name]).filter(Boolean);
+ if (toolLines.length > 0) {
+ prompt += "\n\nYou have access to tools. " + toolLines.join(" ");
+ }
+ }
+
+ return prompt;
}
diff --git a/src/helpers/database.js b/src/helpers/database.js
index 32284dfc1..b4dc348d4 100644
--- a/src/helpers/database.js
+++ b/src/helpers/database.js
@@ -230,6 +230,12 @@ class DatabaseManager {
"CREATE INDEX IF NOT EXISTS idx_agent_messages_conversation ON agent_messages(conversation_id)"
);
+ try {
+ this.db.exec("ALTER TABLE agent_messages ADD COLUMN metadata TEXT");
+ } catch (err) {
+ if (!err.message.includes("duplicate column")) throw err;
+ }
+
const actionCount = this.db.prepare("SELECT COUNT(*) as count FROM actions").get();
if (actionCount.count === 0) {
this.db
diff --git a/src/helpers/ipcHandlers.js b/src/helpers/ipcHandlers.js
index d7adf6364..6f1b20801 100644
--- a/src/helpers/ipcHandlers.js
+++ b/src/helpers/ipcHandlers.js
@@ -2712,6 +2712,43 @@ class IPCHandlers {
}
});
+ ipcMain.handle("agent-web-search", async (event, query, numResults = 5) => {
+ try {
+ const apiUrl = getApiUrl();
+ if (!apiUrl) throw new Error("OpenWhispr API URL not configured");
+
+ const cookieHeader = await getSessionCookies(event);
+ if (!cookieHeader) throw new Error("No session cookies available");
+
+ debugLogger.debug("Agent web search request", { query, numResults }, "cloud-api");
+
+ const response = await fetch(`${apiUrl}/api/agent/web-search`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ Cookie: cookieHeader,
+ },
+ body: JSON.stringify({ query, numResults }),
+ });
+
+ if (!response.ok) {
+ if (response.status === 401) {
+ return { success: false, error: "Session expired", code: "AUTH_EXPIRED" };
+ }
+ const errorData = await response.json().catch(() => ({}));
+ return {
+ success: false,
+ error: errorData.error || `API error: ${response.status}`,
+ };
+ }
+
+ return await response.json();
+ } catch (error) {
+ debugLogger.error("Agent web search error:", error);
+ return { success: false, error: error.message };
+ }
+ });
+
ipcMain.handle(
"cloud-streaming-usage",
async (event, text, audioDurationSeconds, opts = {}) => {
diff --git a/src/locales/de/translation.json b/src/locales/de/translation.json
index f737682f2..6802ee828 100644
--- a/src/locales/de/translation.json
+++ b/src/locales/de/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "schließen",
"listening": "Hört zu...",
"transcribing": "Transkribiert...",
- "thinking": "Denkt nach..."
+ "thinking": "Denkt nach...",
+ "typeMessage": "Nachricht eingeben...",
+ "send": "Senden"
+ },
+ "tools": {
+ "search_notesStatus": "Notizen werden durchsucht...",
+ "web_searchStatus": "Websuche...",
+ "copy_to_clipboardStatus": "In Zwischenablage kopieren...",
+ "get_calendar_eventsStatus": "Kalender wird geprüft...",
+ "foundNotes": "{{count}} Notiz gefunden",
+ "foundNotes_plural": "{{count}} Notizen gefunden",
+ "searchResults": "{{count}} Ergebnis gefunden",
+ "searchResults_plural": "{{count}} Ergebnisse gefunden",
+ "copiedToClipboard": "In Zwischenablage kopiert",
+ "calendarEvents": "{{count}} Termin gefunden",
+ "calendarEvents_plural": "{{count}} Termine gefunden",
+ "noResults": "Keine Ergebnisse",
+ "error": "Werkzeugfehler"
}
},
"integrations": {
diff --git a/src/locales/en/translation.json b/src/locales/en/translation.json
index e18649dd2..6f151f837 100644
--- a/src/locales/en/translation.json
+++ b/src/locales/en/translation.json
@@ -1882,7 +1882,24 @@
"toClose": "close",
"listening": "Listening...",
"transcribing": "Transcribing...",
- "thinking": "Thinking..."
+ "thinking": "Thinking...",
+ "typeMessage": "Type a message...",
+ "send": "Send"
+ },
+ "tools": {
+ "search_notesStatus": "Searching notes...",
+ "web_searchStatus": "Searching the web...",
+ "copy_to_clipboardStatus": "Copying to clipboard...",
+ "get_calendar_eventsStatus": "Checking calendar...",
+ "foundNotes": "Found {{count}} note",
+ "foundNotes_plural": "Found {{count}} notes",
+ "searchResults": "Found {{count}} result",
+ "searchResults_plural": "Found {{count}} results",
+ "copiedToClipboard": "Copied to clipboard",
+ "calendarEvents": "Found {{count}} event",
+ "calendarEvents_plural": "Found {{count}} events",
+ "noResults": "No results found",
+ "error": "Tool error"
}
},
"integrations": {
diff --git a/src/locales/es/translation.json b/src/locales/es/translation.json
index 272560202..b49d92978 100644
--- a/src/locales/es/translation.json
+++ b/src/locales/es/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "cerrar",
"listening": "Escuchando...",
"transcribing": "Transcribiendo...",
- "thinking": "Pensando..."
+ "thinking": "Pensando...",
+ "typeMessage": "Escribe un mensaje...",
+ "send": "Enviar"
+ },
+ "tools": {
+ "search_notesStatus": "Buscando notas...",
+ "web_searchStatus": "Buscando en la web...",
+ "copy_to_clipboardStatus": "Copiando al portapapeles...",
+ "get_calendar_eventsStatus": "Consultando calendario...",
+ "foundNotes": "{{count}} nota encontrada",
+ "foundNotes_plural": "{{count}} notas encontradas",
+ "searchResults": "{{count}} resultado encontrado",
+ "searchResults_plural": "{{count}} resultados encontrados",
+ "copiedToClipboard": "Copiado al portapapeles",
+ "calendarEvents": "{{count}} evento encontrado",
+ "calendarEvents_plural": "{{count}} eventos encontrados",
+ "noResults": "Sin resultados",
+ "error": "Error de herramienta"
}
},
"integrations": {
diff --git a/src/locales/fr/translation.json b/src/locales/fr/translation.json
index 9ddd4e6fb..f50dac182 100644
--- a/src/locales/fr/translation.json
+++ b/src/locales/fr/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "fermer",
"listening": "Écoute...",
"transcribing": "Transcription...",
- "thinking": "Réflexion..."
+ "thinking": "Réflexion...",
+ "typeMessage": "Écrivez un message...",
+ "send": "Envoyer"
+ },
+ "tools": {
+ "search_notesStatus": "Recherche de notes...",
+ "web_searchStatus": "Recherche sur le web...",
+ "copy_to_clipboardStatus": "Copie dans le presse-papiers...",
+ "get_calendar_eventsStatus": "Vérification du calendrier...",
+ "foundNotes": "{{count}} note trouvée",
+ "foundNotes_plural": "{{count}} notes trouvées",
+ "searchResults": "{{count}} résultat trouvé",
+ "searchResults_plural": "{{count}} résultats trouvés",
+ "copiedToClipboard": "Copié dans le presse-papiers",
+ "calendarEvents": "{{count}} événement trouvé",
+ "calendarEvents_plural": "{{count}} événements trouvés",
+ "noResults": "Aucun résultat",
+ "error": "Erreur d'outil"
}
},
"integrations": {
diff --git a/src/locales/it/translation.json b/src/locales/it/translation.json
index 107aaa675..835c5e5a1 100644
--- a/src/locales/it/translation.json
+++ b/src/locales/it/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "chiudi",
"listening": "Ascolto...",
"transcribing": "Trascrizione...",
- "thinking": "Pensiero..."
+ "thinking": "Pensiero...",
+ "typeMessage": "Scrivi un messaggio...",
+ "send": "Invia"
+ },
+ "tools": {
+ "search_notesStatus": "Ricerca nelle note...",
+ "web_searchStatus": "Ricerca sul web...",
+ "copy_to_clipboardStatus": "Copia negli appunti...",
+ "get_calendar_eventsStatus": "Controllo del calendario...",
+ "foundNotes": "{{count}} nota trovata",
+ "foundNotes_plural": "{{count}} note trovate",
+ "searchResults": "{{count}} risultato trovato",
+ "searchResults_plural": "{{count}} risultati trovati",
+ "copiedToClipboard": "Copiato negli appunti",
+ "calendarEvents": "{{count}} evento trovato",
+ "calendarEvents_plural": "{{count}} eventi trovati",
+ "noResults": "Nessun risultato",
+ "error": "Errore dello strumento"
}
},
"integrations": {
diff --git a/src/locales/ja/translation.json b/src/locales/ja/translation.json
index c75fa6c78..4250f34f7 100644
--- a/src/locales/ja/translation.json
+++ b/src/locales/ja/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "閉じる",
"listening": "聞いています...",
"transcribing": "文字起こし中...",
- "thinking": "考え中..."
+ "thinking": "考え中...",
+ "typeMessage": "メッセージを入力...",
+ "send": "送信"
+ },
+ "tools": {
+ "search_notesStatus": "ノートを検索中...",
+ "web_searchStatus": "ウェブを検索中...",
+ "copy_to_clipboardStatus": "クリップボードにコピー中...",
+ "get_calendar_eventsStatus": "カレンダーを確認中...",
+ "foundNotes": "{{count}}件のノートが見つかりました",
+ "foundNotes_plural": "{{count}}件のノートが見つかりました",
+ "searchResults": "{{count}}件の結果が見つかりました",
+ "searchResults_plural": "{{count}}件の結果が見つかりました",
+ "copiedToClipboard": "クリップボードにコピーしました",
+ "calendarEvents": "{{count}}件の予定が見つかりました",
+ "calendarEvents_plural": "{{count}}件の予定が見つかりました",
+ "noResults": "結果が見つかりません",
+ "error": "ツールエラー"
}
},
"integrations": {
diff --git a/src/locales/pt/translation.json b/src/locales/pt/translation.json
index 2fdece6cf..9cd5a679c 100644
--- a/src/locales/pt/translation.json
+++ b/src/locales/pt/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "fechar",
"listening": "Ouvindo...",
"transcribing": "Transcrevendo...",
- "thinking": "Pensando..."
+ "thinking": "Pensando...",
+ "typeMessage": "Digite uma mensagem...",
+ "send": "Enviar"
+ },
+ "tools": {
+ "search_notesStatus": "Pesquisando notas...",
+ "web_searchStatus": "Pesquisando na web...",
+ "copy_to_clipboardStatus": "Copiando para a área de transferência...",
+ "get_calendar_eventsStatus": "Verificando calendário...",
+ "foundNotes": "{{count}} nota encontrada",
+ "foundNotes_plural": "{{count}} notas encontradas",
+ "searchResults": "{{count}} resultado encontrado",
+ "searchResults_plural": "{{count}} resultados encontrados",
+ "copiedToClipboard": "Copiado para a área de transferência",
+ "calendarEvents": "{{count}} evento encontrado",
+ "calendarEvents_plural": "{{count}} eventos encontrados",
+ "noResults": "Nenhum resultado",
+ "error": "Erro de ferramenta"
}
},
"integrations": {
diff --git a/src/locales/ru/translation.json b/src/locales/ru/translation.json
index fc14875ec..88fa191d1 100644
--- a/src/locales/ru/translation.json
+++ b/src/locales/ru/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "закрыть",
"listening": "Слушаю...",
"transcribing": "Транскрибирую...",
- "thinking": "Думаю..."
+ "thinking": "Думаю...",
+ "typeMessage": "Введите сообщение...",
+ "send": "Отправить"
+ },
+ "tools": {
+ "search_notesStatus": "Поиск в заметках...",
+ "web_searchStatus": "Поиск в интернете...",
+ "copy_to_clipboardStatus": "Копирование в буфер обмена...",
+ "get_calendar_eventsStatus": "Проверка календаря...",
+ "foundNotes": "Найдена {{count}} заметка",
+ "foundNotes_plural": "Найдено {{count}} заметок",
+ "searchResults": "Найден {{count}} результат",
+ "searchResults_plural": "Найдено {{count}} результатов",
+ "copiedToClipboard": "Скопировано в буфер обмена",
+ "calendarEvents": "Найдено {{count}} событие",
+ "calendarEvents_plural": "Найдено {{count}} событий",
+ "noResults": "Ничего не найдено",
+ "error": "Ошибка инструмента"
}
},
"integrations": {
diff --git a/src/locales/zh-CN/translation.json b/src/locales/zh-CN/translation.json
index f11cef94b..bb8c79b3e 100644
--- a/src/locales/zh-CN/translation.json
+++ b/src/locales/zh-CN/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "关闭",
"listening": "正在聆听...",
"transcribing": "正在转录...",
- "thinking": "正在思考..."
+ "thinking": "正在思考...",
+ "typeMessage": "输入消息...",
+ "send": "发送"
+ },
+ "tools": {
+ "search_notesStatus": "正在搜索笔记...",
+ "web_searchStatus": "正在搜索网页...",
+ "copy_to_clipboardStatus": "正在复制到剪贴板...",
+ "get_calendar_eventsStatus": "正在查看日历...",
+ "foundNotes": "找到 {{count}} 条笔记",
+ "foundNotes_plural": "找到 {{count}} 条笔记",
+ "searchResults": "找到 {{count}} 条结果",
+ "searchResults_plural": "找到 {{count}} 条结果",
+ "copiedToClipboard": "已复制到剪贴板",
+ "calendarEvents": "找到 {{count}} 个日程",
+ "calendarEvents_plural": "找到 {{count}} 个日程",
+ "noResults": "未找到结果",
+ "error": "工具错误"
}
},
"integrations": {
diff --git a/src/locales/zh-TW/translation.json b/src/locales/zh-TW/translation.json
index 8353a2505..7631b3517 100644
--- a/src/locales/zh-TW/translation.json
+++ b/src/locales/zh-TW/translation.json
@@ -1812,7 +1812,24 @@
"toClose": "關閉",
"listening": "正在聆聽...",
"transcribing": "正在轉錄...",
- "thinking": "正在思考..."
+ "thinking": "正在思考...",
+ "typeMessage": "輸入訊息...",
+ "send": "傳送"
+ },
+ "tools": {
+ "search_notesStatus": "正在搜尋筆記...",
+ "web_searchStatus": "正在搜尋網頁...",
+ "copy_to_clipboardStatus": "正在複製到剪貼簿...",
+ "get_calendar_eventsStatus": "正在查看行事曆...",
+ "foundNotes": "找到 {{count}} 則筆記",
+ "foundNotes_plural": "找到 {{count}} 則筆記",
+ "searchResults": "找到 {{count}} 項結果",
+ "searchResults_plural": "找到 {{count}} 項結果",
+ "copiedToClipboard": "已複製到剪貼簿",
+ "calendarEvents": "找到 {{count}} 個行程",
+ "calendarEvents_plural": "找到 {{count}} 個行程",
+ "noResults": "未找到結果",
+ "error": "工具錯誤"
}
},
"integrations": {
diff --git a/src/services/AgentLoop.ts b/src/services/AgentLoop.ts
new file mode 100644
index 000000000..6b6f37ab6
--- /dev/null
+++ b/src/services/AgentLoop.ts
@@ -0,0 +1,185 @@
+import ReasoningService, { type AgentStreamChunk } from "./ReasoningService";
+import type { ToolRegistry, ToolResult } from "./tools";
+import logger from "../utils/logger";
+
+export interface AgentLoopCallbacks {
+ onContentChunk: (text: string) => void;
+ onToolStart: (toolName: string, args: Record<string, unknown>) => void;
+ onToolComplete: (toolName: string, result: ToolResult) => void;
+ onError: (error: Error) => void;
+}
+
+export interface AgentLoopOptions {
+ messages: Array<{ role: string; content: string }>;
+ model: string;
+ provider: string;
+ systemPrompt: string;
+ tools: ToolRegistry;
+ callbacks: AgentLoopCallbacks;
+ isCloud?: boolean;
+ maxIterations?: number;
+}
+
+interface ToolCallMessage {
+ role: "assistant";
+ tool_calls: Array<{
+ id: string;
+ type: "function";
+ function: { name: string; arguments: string };
+ }>;
+}
+
+interface ToolResultMessage {
+ role: "tool";
+ tool_call_id: string;
+ content: string;
+}
+
+type Message = { role: string; content: string } | ToolCallMessage | ToolResultMessage;
+
+async function executeTool(
+ tools: ToolRegistry,
+ name: string,
+ rawArgs: string,
+ callbacks: AgentLoopCallbacks
+): Promise<ToolResult> {
+ const tool = tools.get(name);
+ if (!tool) {
+ const result: ToolResult = {
+ success: false,
+ data: null,
+ displayText: `Unknown tool: ${name}`,
+ };
+ return result;
+ }
+
+ let args: Record<string, unknown>;
+ try {
+ args = JSON.parse(rawArgs);
+ } catch {
+ const result: ToolResult = {
+ success: false,
+ data: null,
+ displayText: `Invalid arguments for ${name}: failed to parse JSON`,
+ };
+ return result;
+ }
+
+ callbacks.onToolStart(name, args);
+
+ try {
+ const result = await tool.execute(args);
+ callbacks.onToolComplete(name, result);
+ return result;
+ } catch (error) {
+ const result: ToolResult = {
+ success: false,
+ data: null,
+ displayText: `Tool ${name} error: ${(error as Error).message}`,
+ };
+ callbacks.onToolComplete(name, result);
+ return result;
+ }
+}
+
+export async function run(options: AgentLoopOptions): Promise<void> {
+ const {
+ messages: initialMessages,
+ model,
+ provider,
+ systemPrompt,
+ tools,
+ callbacks,
+ maxIterations = 5,
+ } = options;
+
+ const conversationMessages: Message[] = [
+ { role: "system", content: systemPrompt },
+ ...initialMessages,
+ ];
+
+ const toolDefs = tools.toOpenAIFormat();
+
+ for (let iteration = 0; iteration < maxIterations; iteration++) {
+ let pendingToolCalls: Array<{ id: string; name: string; arguments: string }> = [];
+
+ try {
+ const stream = ReasoningService.processTextStreamingWithTools(
+ conversationMessages as Array<{ role: string; content: string }>,
+ model,
+ provider,
+ { systemPrompt },
+ toolDefs
+ ) as AsyncGenerator<AgentStreamChunk, void, unknown>;
+
+ for await (const chunk of stream) {
+ if (chunk.type === "content") {
+ callbacks.onContentChunk(chunk.text);
+ } else if (chunk.type === "tool_calls") {
+ pendingToolCalls = chunk.calls;
+ } else if (chunk.type === "done") {
+ if (pendingToolCalls.length === 0) return;
+ }
+ }
+ } catch (error) {
+ callbacks.onError(error as Error);
+ return;
+ }
+
+ if (pendingToolCalls.length === 0) return;
+
+ // Append assistant message with tool_calls
+ const assistantMessage: ToolCallMessage = {
+ role: "assistant",
+ tool_calls: pendingToolCalls.map((call) => ({
+ id: call.id,
+ type: "function",
+ function: { name: call.name, arguments: call.arguments },
+ })),
+ };
+ conversationMessages.push(assistantMessage);
+
+ // Classify calls: read-only tools run in parallel, write tools run sequentially
+ const readOnlyCalls = pendingToolCalls.filter((c) => tools.get(c.name)?.readOnly);
+ const writeCalls = pendingToolCalls.filter((c) => !tools.get(c.name)?.readOnly);
+
+ const toolResults: ToolResultMessage[] = [];
+
+ // Execute read-only tools in parallel
+ if (readOnlyCalls.length > 0) {
+ const results = await Promise.all(
+ readOnlyCalls.map(async (call) => {
+ const result = await executeTool(tools, call.name, call.arguments, callbacks);
+ return { id: call.id, result };
+ })
+ );
+ for (const { id, result } of results) {
+ toolResults.push({
+ role: "tool",
+ tool_call_id: id,
+ content: JSON.stringify({ success: result.success, data: result.data }),
+ });
+ }
+ }
+
+ // Execute write tools sequentially
+ for (const call of writeCalls) {
+ const result = await executeTool(tools, call.name, call.arguments, callbacks);
+ toolResults.push({
+ role: "tool",
+ tool_call_id: call.id,
+ content: JSON.stringify({ success: result.success, data: result.data }),
+ });
+ }
+
+ conversationMessages.push(...toolResults);
+
+ logger.logReasoning("AGENT_LOOP_ITERATION", {
+ iteration: iteration + 1,
+ toolCallCount: pendingToolCalls.length,
+ toolNames: pendingToolCalls.map((c) => c.name),
+ });
+ }
+
+ logger.warn("Agent loop reached max iterations", { maxIterations }, "agent");
+}
diff --git a/src/services/ReasoningService.ts b/src/services/ReasoningService.ts
index f822d99e7..de79c9e88 100644
--- a/src/services/ReasoningService.ts
+++ b/src/services/ReasoningService.ts
@@ -8,6 +8,19 @@ import { isSecureEndpoint } from "../utils/urlUtils";
import { withSessionRefresh } from "../lib/neonAuth";
import { getSettings, isCloudReasoningMode } from "../stores/settingsStore";
+export type AgentStreamChunk =
+ | { type: "content"; text: string }
+ | { type: "tool_calls"; calls: Array<{ id: string; name: string; arguments: string }> }
+ | { type: "done"; finishReason?: string };
+
+interface ResolvedEndpoint {
+ endpoint: string;
+ apiKey: string;
+ isLocalProvider: boolean;
+ apiConfig: ReturnType<typeof getOpenAiApiConfig>;
+ useOldTokenParam: boolean;
+}
+
class ReasoningService extends BaseReasoningService {
private apiKeyCache: SecureCache<string>;
private openAiEndpointPreference = new Map<string, "responses" | "chat">();
@@ -250,6 +263,43 @@ class ReasoningService extends BaseReasoningService {
return apiKey;
}
+ private async resolveEndpoint(model: string, provider: string): Promise<ResolvedEndpoint> {
+ const cloudProviders = ["openai", "groq", "gemini", "anthropic", "custom"];
+ const isLocalProvider = !cloudProviders.includes(provider);
+
+ let endpoint: string;
+ let apiKey = "";
+
+ if (isLocalProvider) {
+ const serverResult = await window.electronAPI.llamaServerStart(model);
+ if (!serverResult.success || !serverResult.port) {
+ throw new Error(serverResult.error || "Failed to start local model server");
+ }
+ endpoint = `http://127.0.0.1:${serverResult.port}/v1/chat/completions`;
+ } else {
+ const providerKey = provider as "openai" | "groq" | "gemini" | "anthropic" | "custom";
+ apiKey = await this.getApiKey(providerKey);
+
+ switch (providerKey) {
+ case "groq":
+ endpoint = buildApiUrl(API_ENDPOINTS.GROQ_BASE, "/chat/completions");
+ break;
+ case "openai":
+ case "custom":
+ endpoint = buildApiUrl(this.getConfiguredOpenAIBase(), "/chat/completions");
+ break;
+ default:
+ endpoint = buildApiUrl(API_ENDPOINTS.OPENAI_BASE, "/chat/completions");
+ break;
+ }
+ }
+
+ const apiConfig = getOpenAiApiConfig(model);
+ const useOldTokenParam = isLocalProvider || provider === "groq";
+
+ return { endpoint, apiKey, isLocalProvider, apiConfig, useOldTokenParam };
+ }
+
private async callChatCompletionsApi(
endpoint: string,
apiKey: string,
@@ -1262,6 +1312,184 @@ class ReasoningService extends BaseReasoningService {
}
}
+ async *processTextStreamingWithTools(
+ messages: Array<{ role: string; content: string }>,
+ model: string,
+ provider: string,
+ config: ReasoningConfig & { systemPrompt: string },
+ tools: Array<{
+ type: "function";
+ function: { name: string; description: string; parameters: Record<string, unknown> };
+ }>
+ ): AsyncGenerator<AgentStreamChunk, void, unknown> {
+ const { endpoint, apiKey, isLocalProvider, apiConfig, useOldTokenParam } =
+ await this.resolveEndpoint(model, provider);
+
+ // Local providers don't support tools — fall back to content-only streaming
+ if (isLocalProvider) {
+ const contentGen = this.processTextStreaming(messages, model, provider, config);
+ for await (const text of contentGen) {
+ yield { type: "content", text };
+ }
+ yield { type: "done", finishReason: "stop" };
+ return;
+ }
+
+ const maxTokens = config.maxTokens || Math.max(4096, TOKEN_LIMITS.MAX_TOKENS);
+
+ const requestBody: Record<string, unknown> = {
+ model,
+ messages,
+ stream: true,
+ tools,
+ tool_choice: "auto",
+ };
+
+ if (useOldTokenParam) {
+ requestBody.temperature = config.temperature ?? 0.3;
+ requestBody.max_tokens = maxTokens;
+ } else {
+ requestBody[apiConfig.tokenParam] = maxTokens;
+ if (apiConfig.supportsTemperature) {
+ requestBody.temperature = config.temperature ?? 0.3;
+ }
+ }
+
+ logger.logReasoning("AGENT_TOOL_STREAM_REQUEST", {
+ endpoint,
+ model,
+ provider,
+ toolCount: tools.length,
+ messageCount: messages.length,
+ });
+
+ const headers: Record<string, string> = {
+ "Content-Type": "application/json",
+ };
+ if (apiKey) {
+ headers["Authorization"] = `Bearer ${apiKey}`;
+ }
+
+ const response = await fetch(endpoint, {
+ method: "POST",
+ headers,
+ body: JSON.stringify(requestBody),
+ });
+
+ if (!response.ok) {
+ const errorText = await response.text();
+ let errorMessage: string;
+ try {
+ const errorData = JSON.parse(errorText);
+ errorMessage =
+ errorData.error?.message ||
+ errorData.message ||
+ errorData.error ||
+ `API error: ${response.status}`;
+ } catch {
+ errorMessage = errorText || `API error: ${response.status}`;
+ }
+ logger.logReasoning("AGENT_TOOL_STREAM_ERROR", { status: response.status, errorMessage });
+ throw new Error(errorMessage);
+ }
+
+ const reader = response.body?.getReader();
+ if (!reader) throw new Error("No response body");
+
+ const decoder = new TextDecoder();
+ let buffer = "";
+ // Accumulate tool calls by index — OpenAI streams arguments incrementally
+ const toolCallAccumulator = new Map<number, { id: string; name: string; arguments: string }>();
+
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split("\n");
+ buffer = lines.pop() || "";
+
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed || !trimmed.startsWith("data: ")) continue;
+
+ const data = trimmed.slice(6);
+ if (data === "[DONE]") {
+ // Flush any remaining accumulated tool calls
+ if (toolCallAccumulator.size > 0) {
+ const sortedCalls = Array.from(toolCallAccumulator.entries())
+ .sort(([a], [b]) => a - b)
+ .map(([, call]) => call);
+ yield { type: "tool_calls", calls: sortedCalls };
+ toolCallAccumulator.clear();
+ }
+ yield { type: "done" };
+ return;
+ }
+
+ try {
+ const parsed = JSON.parse(data);
+ const choice = parsed.choices?.[0];
+ if (!choice) continue;
+
+ const delta = choice.delta;
+ const finishReason = choice.finish_reason;
+
+ // Accumulate text content
+ if (delta?.content) {
+ yield { type: "content", text: delta.content };
+ }
+
+ // Accumulate tool call deltas
+ if (delta?.tool_calls) {
+ for (const toolCallDelta of delta.tool_calls) {
+ const idx = toolCallDelta.index;
+ const existing = toolCallAccumulator.get(idx);
+
+ if (existing) {
+ // Append incremental argument fragments
+ if (toolCallDelta.function?.arguments) {
+ existing.arguments += toolCallDelta.function.arguments;
+ }
+ } else {
+ // First chunk for this tool call — initialize
+ toolCallAccumulator.set(idx, {
+ id: toolCallDelta.id || "",
+ name: toolCallDelta.function?.name || "",
+ arguments: toolCallDelta.function?.arguments || "",
+ });
+ }
+ }
+ }
+
+ // When finish_reason arrives, yield accumulated state
+ if (finishReason === "tool_calls") {
+ if (toolCallAccumulator.size > 0) {
+ const sortedCalls = Array.from(toolCallAccumulator.entries())
+ .sort(([a], [b]) => a - b)
+ .map(([, call]) => call);
+ yield { type: "tool_calls", calls: sortedCalls };
+ toolCallAccumulator.clear();
+ }
+ yield { type: "done", finishReason: "tool_calls" };
+ return;
+ }
+
+ if (finishReason === "stop") {
+ yield { type: "done", finishReason: "stop" };
+ return;
+ }
+ } catch {
+ // skip malformed SSE chunks
+ }
+ }
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ }
+
async *processTextStreamingCloud(
messages: Array<{ role: string; content: string }>,
config: { systemPrompt: string }
diff --git a/src/services/tools/ToolRegistry.ts b/src/services/tools/ToolRegistry.ts
new file mode 100644
index 000000000..80f8a0039
--- /dev/null
+++ b/src/services/tools/ToolRegistry.ts
@@ -0,0 +1,51 @@
+export interface ToolResult {
+ success: boolean;
+ data: unknown;
+ displayText: string;
+}
+
+export interface ToolDefinition {
+ name: string;
+ description: string;
+ parameters: Record<string, unknown>;
+ readOnly: boolean;
+ execute: (args: Record<string, unknown>) => Promise<ToolResult>;
+}
+
+interface OpenAIFunctionTool {
+ type: "function";
+ function: {
+ name: string;
+ description: string;
+ parameters: Record<string, unknown>;
+ strict: true;
+ };
+}
+
+export class ToolRegistry {
+ private tools = new Map<string, ToolDefinition>();
+
+ register(tool: ToolDefinition): void {
+ this.tools.set(tool.name, tool);
+ }
+
+ get(name: string): ToolDefinition | undefined {
+ return this.tools.get(name);
+ }
+
+ getAll(): ToolDefinition[] {
+ return Array.from(this.tools.values());
+ }
+
+ toOpenAIFormat(): OpenAIFunctionTool[] {
+ return this.getAll().map((tool) => ({
+ type: "function",
+ function: {
+ name: tool.name,
+ description: tool.description,
+ parameters: tool.parameters,
+ strict: true,
+ },
+ }));
+ }
+}
diff --git a/src/services/tools/calendarTool.ts b/src/services/tools/calendarTool.ts
new file mode 100644
index 000000000..56a665748
--- /dev/null
+++ b/src/services/tools/calendarTool.ts
@@ -0,0 +1,77 @@
+import type { ToolDefinition, ToolResult } from "./ToolRegistry";
+
+type TimeRange = "today" | "tomorrow" | "week";
+
+function getWindowMinutes(timeRange: TimeRange): number {
+ if (timeRange === "week") return 10080;
+ if (timeRange === "tomorrow") return 2880;
+
+ // "today": remaining minutes until midnight
+ const now = new Date();
+ const midnight = new Date(now);
+ midnight.setHours(23, 59, 59, 999);
+ return Math.max(1, Math.ceil((midnight.getTime() - now.getTime()) / 60000));
+}
+
+export const calendarTool: ToolDefinition = {
+ name: "get_calendar_events",
+ description:
+ "Get upcoming Google Calendar events for a given time range. Returns event summaries, times, and locations.",
+ parameters: {
+ type: "object",
+ properties: {
+ timeRange: {
+ type: "string",
+ enum: ["today", "tomorrow", "week"],
+ description: 'Time range to fetch events for (default "today")',
+ },
+ },
+ required: [],
+ additionalProperties: false,
+ },
+ readOnly: true,
+
+ async execute(args: Record<string, unknown>): Promise<ToolResult> {
+ const timeRange = (args.timeRange as TimeRange) || "today";
+ const windowMinutes = getWindowMinutes(timeRange);
+
+ try {
+ const response = await window.electronAPI.gcalGetUpcomingEvents!(windowMinutes);
+
+ if (!response.success) {
+ return {
+ success: false,
+ data: null,
+ displayText: "Failed to fetch calendar events",
+ };
+ }
+
+ const events = response.events.map((event: Record<string, unknown>) => ({
+ summary: event.summary || "(No title)",
+ start: event.start,
+ end: event.end,
+ location: event.location || null,
+ }));
+
+ if (events.length === 0) {
+ return {
+ success: true,
+ data: [],
+ displayText: `No events found for ${timeRange}`,
+ };
+ }
+
+ return {
+ success: true,
+ data: events,
+ displayText: `Found ${events.length} event${events.length === 1 ? "" : "s"} for ${timeRange}`,
+ };
+ } catch (error) {
+ return {
+ success: false,
+ data: null,
+ displayText: `Failed to fetch calendar events: ${(error as Error).message}`,
+ };
+ }
+ },
+};
diff --git a/src/services/tools/clipboardTool.ts b/src/services/tools/clipboardTool.ts
new file mode 100644
index 000000000..09328e096
--- /dev/null
+++ b/src/services/tools/clipboardTool.ts
@@ -0,0 +1,39 @@
+import type { ToolDefinition, ToolResult } from "./ToolRegistry";
+
+export const clipboardTool: ToolDefinition = {
+ name: "copy_to_clipboard",
+ description: "Copy text to the user's system clipboard.",
+ parameters: {
+ type: "object",
+ properties: {
+ text: {
+ type: "string",
+ description: "The text to copy to the clipboard",
+ },
+ },
+ required: ["text"],
+ additionalProperties: false,
+ },
+ readOnly: false,
+
+ async execute(args: Record<string, unknown>): Promise<ToolResult> {
+ const text = args.text as string;
+
+ try {
+ await window.electronAPI.writeClipboard(text);
+
+ const preview = text.length > 100 ? text.slice(0, 100) + "..." : text;
+ return {
+ success: true,
+ data: null,
+ displayText: `Copied to clipboard: "${preview}"`,
+ };
+ } catch (error) {
+ return {
+ success: false,
+ data: null,
+ displayText: `Failed to copy to clipboard: ${(error as Error).message}`,
+ };
+ }
+ },
+};
diff --git a/src/services/tools/index.ts b/src/services/tools/index.ts
new file mode 100644
index 000000000..ea782ed7b
--- /dev/null
+++ b/src/services/tools/index.ts
@@ -0,0 +1,30 @@
+import { ToolRegistry } from "./ToolRegistry";
+import { searchNotesTool } from "./searchNotesTool";
+import { clipboardTool } from "./clipboardTool";
+import { webSearchTool } from "./webSearchTool";
+import { calendarTool } from "./calendarTool";
+
+export { ToolRegistry } from "./ToolRegistry";
+export type { ToolDefinition, ToolResult } from "./ToolRegistry";
+
+interface ToolRegistrySettings {
+ isSignedIn: boolean;
+ gcalConnected: boolean;
+}
+
+export function createToolRegistry(settings: ToolRegistrySettings): ToolRegistry {
+ const registry = new ToolRegistry();
+
+ registry.register(searchNotesTool);
+ registry.register(clipboardTool);
+
+ if (settings.isSignedIn) {
+ registry.register(webSearchTool);
+ }
+
+ if (settings.gcalConnected) {
+ registry.register(calendarTool);
+ }
+
+ return registry;
+}
diff --git a/src/services/tools/searchNotesTool.ts b/src/services/tools/searchNotesTool.ts
new file mode 100644
index 000000000..fd4cd59be
--- /dev/null
+++ b/src/services/tools/searchNotesTool.ts
@@ -0,0 +1,61 @@
+import type { ToolDefinition, ToolResult } from "./ToolRegistry";
+
+const MAX_CONTENT_LENGTH = 500;
+
+export const searchNotesTool: ToolDefinition = {
+ name: "search_notes",
+ description:
+ "Search the user's notes by keyword or phrase. Returns matching notes with title, date, and a preview of content.",
+ parameters: {
+ type: "object",
+ properties: {
+ query: {
+ type: "string",
+ description: "The search query to find relevant notes",
+ },
+ limit: {
+ type: "number",
+ description: "Maximum number of results to return (default 5)",
+ },
+ },
+ required: ["query"],
+ additionalProperties: false,
+ },
+ readOnly: true,
+
+ async execute(args: Record<string, unknown>): Promise<ToolResult> {
+ const query = args.query as string;
+ const limit = typeof args.limit === "number" ? args.limit : 5;
+
+ try {
+ const notes = await window.electronAPI.searchNotes(query, limit);
+
+ if (notes.length === 0) {
+ return {
+ success: true,
+ data: [],
+ displayText: `No notes found for "${query}"`,
+ };
+ }
+
+ const results = notes.map((note) => ({
+ title: note.title,
+ date: note.created_at,
+ type: note.note_type,
+ content: (note.enhanced_content || note.content).slice(0, MAX_CONTENT_LENGTH),
+ }));
+
+ return {
+ success: true,
+ data: results,
+ displayText: `Found ${results.length} note${results.length === 1 ? "" : "s"} for "${query}"`,
+ };
+ } catch (error) {
+ return {
+ success: false,
+ data: null,
+ displayText: `Failed to search notes: ${(error as Error).message}`,
+ };
+ }
+ },
+};
diff --git a/src/services/tools/webSearchTool.ts b/src/services/tools/webSearchTool.ts
new file mode 100644
index 000000000..b5d681760
--- /dev/null
+++ b/src/services/tools/webSearchTool.ts
@@ -0,0 +1,44 @@
+import type { ToolDefinition, ToolResult } from "./ToolRegistry";
+
+export const webSearchTool: ToolDefinition = {
+ name: "web_search",
+ description:
+ "Search the web for current information. Returns relevant web results with titles and snippets.",
+ parameters: {
+ type: "object",
+ properties: {
+ query: {
+ type: "string",
+ description: "The search query",
+ },
+ numResults: {
+ type: "number",
+ description: "Number of results to return (default 5)",
+ },
+ },
+ required: ["query"],
+ additionalProperties: false,
+ },
+ readOnly: true,
+
+ async execute(args: Record<string, unknown>): Promise<ToolResult> {
+ const query = args.query as string;
+ const numResults = typeof args.numResults === "number" ? args.numResults : 5;
+
+ try {
+ const results = await window.electronAPI.agentWebSearch!(query, numResults);
+
+ return {
+ success: true,
+ data: results,
+ displayText: `Found web results for "${query}"`,
+ };
+ } catch (error) {
+ return {
+ success: false,
+ data: null,
+ displayText: `Web search failed: ${(error as Error).message}`,
+ };
+ }
+ },
+};
diff --git a/src/types/electron.ts b/src/types/electron.ts
index 663f423d7..b0dfb125e 100644
--- a/src/types/electron.ts
+++ b/src/types/electron.ts
@@ -1078,6 +1078,19 @@ declare global {
) => Promise<{ success: boolean; error?: string; code?: string }>;
onAgentStreamChunk?: (callback: (chunk: string) => void) => () => void;
onAgentStreamDone?: (callback: () => void) => () => void;
+ agentWebSearch?: (
+ query: string,
+ numResults?: number
+ ) => Promise<{
+ success: boolean;
+ results?: Array<{
+ title: string;
+ url: string;
+ text: string;
+ publishedDate?: string;
+ }>;
+ error?: string;
+ }>;
// Google Calendar
gcalStartOAuth?: () => Promise<{ success: boolean; email?: string; error?: string }>;