A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.
Swift ai-agentsanthropicantigravityclaudeclaude-codecodexcursordynamic-islandgemini-cliiosiphonlinuxmacosmacos-appmenubar-appnotchopen-sourceswiftswiftuiwindows
voice: Claude as Coucou's brain (the user's Anthropic key), with Coucou's tools
Claudeaacd077fb409
diff --git a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxprojindex 89ec9d01..ef367c43 100644--- a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj+++ b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj@@ -138,6 +138,7 @@6FC65C190ED614C664C9BC17 /* SafeWebURL.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2FDED98F7F6C755A72C97A3 /* SafeWebURL.swift */; };714B5BB7FFB0C8491AF67AD1 /* VoiceTranscriptHistory.swift in Sources */ = {isa = PBXBuildFile; fileRef = DE925A6844FE3E3C09131CE0 /* VoiceTranscriptHistory.swift */; };718410ED630AE3AC5DF59A95 /* VoiceSpeaker.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */; };+ 50C33F44687D70690514FBBB /* ClaudeVoiceBrain.swift in Sources */ = {isa = PBXBuildFile; fileRef = ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */; };9BAD96406059597681D89A92 /* VoiceGenderNames.swift in Sources */ = {isa = PBXBuildFile; fileRef = 16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */; };71CBD1D39A6668BF454CA1C5 /* IslandTypes.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0365DDE3C554F5E8570055F4 /* IslandTypes.swift */; };724E602A05FE5A6D58218E27 /* CoucouControl.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C406C508A86E215B3CBE3FC /* CoucouControl.swift */; };@@ -310,6 +311,7 @@F6CA56F362902839EF9A3E2E /* PhoneAssets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 60401D1023D17C3FD30D7113 /* PhoneAssets.xcassets */; };F6D57EEBD7E937EFD2B715CF /* ClaudeHost.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F0D0F67E7C56B9E37E7648 /* ClaudeHost.swift */; };F7162101509D8CAE59711602 /* VoiceSpeaker.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */; };+ 70F221444F3D0B11EEA0E758 /* ClaudeVoiceBrain.swift in Sources */ = {isa = PBXBuildFile; fileRef = ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */; };D166CDFE3CBAF1310E7EC1F7 /* VoiceGenderNames.swift in Sources */ = {isa = PBXBuildFile; fileRef = 16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */; };F8F95EAD88F70E0A147D1662 /* MacDictation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6544ED02D945CF9F00647AD1 /* MacDictation.swift */; };F9445A71BBF9E4D20224238E /* QuestionPayload.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6C6993B34A0FFF8E3D454ADC /* QuestionPayload.swift */; };@@ -475,6 +477,7 @@C563E55884A7750B40EC9A01 /* VoiceBrain.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceBrain.swift; sourceTree = "<group>"; };C581F7B1A9F14B281B60063E /* GreetingCanvasView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GreetingCanvasView.swift; sourceTree = "<group>"; };CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceSpeaker.swift; sourceTree = "<group>"; };+ ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClaudeVoiceBrain.swift; sourceTree = "<group>"; };16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceGenderNames.swift; sourceTree = "<group>"; };D0F0D0F67E7C56B9E37E7648 /* ClaudeHost.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClaudeHost.swift; sourceTree = "<group>"; };D1A7C62058BB0C9D0DF58DCB /* VoiceIntent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceIntent.swift; sourceTree = "<group>"; };@@ -752,6 +755,7 @@7C432FE1286945E42D3E11D7 /* VoiceResultView.swift */,D4543CB771E0B723783F529E /* VoiceSettings.swift */,CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */,+ ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */,16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */,DE925A6844FE3E3C09131CE0 /* VoiceTranscriptHistory.swift */,ABD7A2C29B15EFFAF67FF2DE /* VoiceWakeFilter.swift */,@@ -1067,6 +1071,7 @@CE5ACDD856C76B717ABA295F /* VoiceResultView.swift in Sources */,502F2B0D607C511DD3B836B3 /* VoiceSettings.swift in Sources */,718410ED630AE3AC5DF59A95 /* VoiceSpeaker.swift in Sources */,+ 50C33F44687D70690514FBBB /* ClaudeVoiceBrain.swift in Sources */,9BAD96406059597681D89A92 /* VoiceGenderNames.swift in Sources */,189811F4DB3C4F6030043643 /* VoiceTranscriptHistory.swift in Sources */,BBCA973A1AE1385BBE79A834 /* VoiceWakeFilter.swift in Sources */,@@ -1258,6 +1263,7 @@F9685A58729FE64B23FCCFC5 /* VoiceResultView.swift in Sources */,035F9A85BF71B21F9AA5E004 /* VoiceSettings.swift in Sources */,F7162101509D8CAE59711602 /* VoiceSpeaker.swift in Sources */,+ 70F221444F3D0B11EEA0E758 /* ClaudeVoiceBrain.swift in Sources */,D166CDFE3CBAF1310E7EC1F7 /* VoiceGenderNames.swift in Sources */,714B5BB7FFB0C8491AF67AD1 /* VoiceTranscriptHistory.swift in Sources */,8DE75D1E260A1314D1C47885 /* VoiceWakeFilter.swift in Sources */,diff --git a/NotchBuddy/Resources/Localizable.xcstrings b/NotchBuddy/Resources/Localizable.xcstringsindex 8d1f0ca5..9b27c6d9 100644--- a/NotchBuddy/Resources/Localizable.xcstrings+++ b/NotchBuddy/Resources/Localizable.xcstrings@@ -25953,61 +25953,61 @@"ar": {"stringUnit": {"state": "translated",- "value": "الإجابة عن الأسئلة ببحث على الويب"+ "value": "Claude عقل Coucou"}},"bn": {"stringUnit": {"state": "translated",- "value": "ওয়েবে খুঁজে প্রশ্নের উত্তর দাও"+ "value": "Coucou-র মস্তিষ্ক: Claude"}},"en": {"stringUnit": {"state": "translated",- "value": "Answer questions with a web search"+ "value": "Claude as Coucou's brain"}},"es": {"stringUnit": {"state": "translated",- "value": "Responder preguntas buscando en la web"+ "value": "Claude como cerebro de Coucou"}},"fr": {"stringUnit": {"state": "translated",- "value": "Répondre aux questions avec une recherche web"+ "value": "Claude comme cerveau de Coucou"}},"hi": {"stringUnit": {"state": "translated",- "value": "वेब खोज से सवालों के जवाब"+ "value": "Coucou का दिमाग: Claude"}},"id": {"stringUnit": {"state": "translated",- "value": "Jawab pertanyaan dengan pencarian web"+ "value": "Claude sebagai otak Coucou"}},"pt-BR": {"stringUnit": {"state": "translated",- "value": "Responder perguntas com uma busca na web"+ "value": "Claude como cérebro do Coucou"}},"ru": {"stringUnit": {"state": "translated",- "value": "Отвечать на вопросы с поиском в интернете"+ "value": "Claude — мозг Coucou"}},"zh-Hans": {"stringUnit": {"state": "translated",- "value": "用网页搜索回答问题"+ "value": "由 Claude 担任 Coucou 的大脑"}}}@@ -26018,61 +26018,61 @@"ar": {"stringUnit": {"state": "translated",- "value": "الأسئلة التي لا تجيب عنها حبوبك تُرسل إلى Claude مع بحث على الويب، بمفتاح Anthropic API الخاص بك. يُرسل السؤال فقط."+ "value": "بعد «OK Coucou»، يُرسل ما تقوله إلى Claude بمفتاح Anthropic API الخاص بك: ينفّذ (الحبوب، الموسيقى، البريد، خدماتك) ويبحث على الويب. تُرسل كلماتك فقط، وليس الصوت أبدًا."}},"bn": {"stringUnit": {"state": "translated",- "value": "যে প্রশ্নের উত্তর তোমার পিলগুলো দিতে পারে না, সেগুলো তোমার Anthropic API কী দিয়ে ওয়েব সার্চসহ Claude-এর কাছে যায়। শুধু প্রশ্নটাই পাঠানো হয়।"+ "value": "«OK Coucou»-এর পরে তুমি যা বলো তা তোমার Anthropic API কী দিয়ে Claude-এর কাছে যায়: সে কাজ করে (পিল, গান, মেইল, তোমার সার্ভিস) আর ওয়েবে খোঁজে। শুধু তোমার কথা পাঠানো হয়, কখনো অডিও নয়।"}},"en": {"stringUnit": {"state": "translated",- "value": "Questions your pills can't answer go to Claude with a web search, using your Anthropic API key. Only the question is sent."+ "value": "After «OK Coucou», what you say goes to Claude with your Anthropic API key: it acts (pills, music, email, your services) and searches the web. Only your words are sent, never audio."}},"es": {"stringUnit": {"state": "translated",- "value": "Las preguntas que tus píldoras no responden van a Claude con una búsqueda web, con tu clave de API de Anthropic. Solo se envía la pregunta."+ "value": "Tras «OK Coucou», lo que dices va a Claude con tu clave de API de Anthropic: actúa (píldoras, música, correo, tus servicios) y busca en la web. Solo se envían tus palabras, nunca el audio."}},"fr": {"stringUnit": {"state": "translated",- "value": "Les questions auxquelles tes pilules ne répondent pas partent chez Claude avec une recherche web, avec ta clé API Anthropic. Seule la question est envoyée."+ "value": "Après « OK Coucou », ce que tu dis part chez Claude avec ta clé API Anthropic : il agit (pilules, musique, mail, tes services) et cherche sur le web. Seuls tes mots sont envoyés, jamais l'audio."}},"hi": {"stringUnit": {"state": "translated",- "value": "जिन सवालों का जवाब तुम्हारी पिल्स नहीं दे सकतीं, वे तुम्हारी Anthropic API कुंजी से वेब खोज के साथ Claude को जाते हैं। सिर्फ़ सवाल भेजा जाता है।"+ "value": "«OK Coucou» के बाद तुम जो कहते हो वह तुम्हारी Anthropic API कुंजी से Claude को जाता है: वह काम करता है (पिल्स, संगीत, ईमेल, तुम्हारी सेवाएँ) और वेब पर खोजता है। सिर्फ़ तुम्हारे शब्द भेजे जाते हैं, ऑडियो कभी नहीं।"}},"id": {"stringUnit": {"state": "translated",- "value": "Pertanyaan yang tidak bisa dijawab pil-mu dikirim ke Claude dengan pencarian web, memakai kunci API Anthropic-mu. Hanya pertanyaannya yang dikirim."+ "value": "Setelah «OK Coucou», ucapanmu dikirim ke Claude dengan kunci API Anthropic-mu: ia bertindak (pil, musik, email, layananmu) dan mencari di web. Hanya kata-katamu yang dikirim, tidak pernah audio."}},"pt-BR": {"stringUnit": {"state": "translated",- "value": "Perguntas que suas pílulas não respondem vão para o Claude com uma busca na web, usando sua chave de API da Anthropic. Só a pergunta é enviada."+ "value": "Depois de «OK Coucou», o que você diz vai para o Claude com sua chave de API da Anthropic: ele age (pílulas, música, e-mail, seus serviços) e pesquisa na web. Só suas palavras são enviadas, nunca o áudio."}},"ru": {"stringUnit": {"state": "translated",- "value": "Вопросы, на которые не отвечают твои пилюли, уходят в Claude с поиском в интернете, по твоему ключу Anthropic API. Отправляется только вопрос."+ "value": "После «OK Coucou» сказанное уходит в Claude по твоему ключу Anthropic API: он действует (пилюли, музыка, почта, твои сервисы) и ищет в интернете. Отправляются только слова, никогда не звук."}},"zh-Hans": {"stringUnit": {"state": "translated",- "value": "药丸无法回答的问题会用你的 Anthropic API 密钥发送给 Claude 并进行网页搜索。只发送问题本身。"+ "value": "说「OK Coucou」后,你说的话会用你的 Anthropic API 密钥发送给 Claude:它会执行操作(药丸、音乐、邮件、你的服务)并进行网页搜索。只发送文字,从不发送音频。"}}}diff --git a/NotchBuddy/Sources/App/ClaudeService.swift b/NotchBuddy/Sources/App/ClaudeService.swiftindex 24c26ccd..cbbb7c19 100644--- a/NotchBuddy/Sources/App/ClaudeService.swift+++ b/NotchBuddy/Sources/App/ClaudeService.swift@@ -502,6 +502,31 @@ final class ClaudeService {}#if !APPSTORE+ // MARK: - Voice brain turn (Claude with Coucou's tools)++ /// One Messages API call for the voice brain: returns the assistant content blocks and+ /// the stop reason. The fast model first, the Settings model if the key can't use it.+ func voiceTurn(system: String, messages: [[String: Any]], tools: [[String: Any]])+ async -> (content: [[String: Any]], stopReason: String)? {+ guard let key = apiKey, !key.isEmpty else { return nil }+ func body(_ model: String) -> [String: Any] {+ ["model": model, "max_tokens": 600, "system": system, "tools": tools, "messages": messages]+ }+ let data: Data+ do {+ data = try await callAPI(body: body(Self.voiceModel), key: key, beta: "web-search-2025-03-05")+ } catch {+ guard let retry = try? await callAPI(body: body(model), key: key, beta: "web-search-2025-03-05") else {+ appendAppLog("nb.log", "[Voice] Claude turn failed: \(error.localizedDescription)")+ return nil+ }+ data = retry+ }+ guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],+ let content = json["content"] as? [[String: Any]] else { return nil }+ return (content, json["stop_reason"] as? String ?? "end_turn")+ }+// MARK: - Voice answer (Coucou's voice, web search)/// Fast model for spoken answers; the chat model from Settings when it is unavailable.diff --git a/NotchBuddy/Sources/App/IslandWindowController.swift b/NotchBuddy/Sources/App/IslandWindowController.swiftindex 19d2f4ed..fbe8f420 100644--- a/NotchBuddy/Sources/App/IslandWindowController.swift+++ b/NotchBuddy/Sources/App/IslandWindowController.swift@@ -128,6 +128,11 @@ final class IslandWindowController: NSWindowController {Task { @MainActor inlet iLoc = self?.windowToIsland(loc) ?? CGPoint(x: 320, y: 88)AppState.shared.fileDragOver = true+ // The voice mail card stays on screen: the file will be its attachment.+ if AppState.shared.voiceMailDraft != nil && AppState.shared.view == .mail {+ NotificationCenter.default.post(name: .botMorphTo, object: CGFloat(1))+ return+ }// enterZone sets isActive=true BEFORE hookExpand triggers re-render,// so IslandContainer sees isActive=true when state.view becomes .upload.UploadSequenceEngine.shared.enterZone(x: iLoc.x, y: iLoc.y)@@ -158,6 +163,11 @@ final class IslandWindowController: NSWindowController {await self?.attachVoiceMailFile(url)return}+ // The mail card prepared by voice (Claude) is open: the file is its attachment.+ if AppState.shared.voiceMailDraft != nil, let url = urls.first {+ self?.attachToVoiceMailCard(url)+ return+ }#endifawait FileDropHandler.handle(urls: urls, state: AppState.shared)}@@ -1441,6 +1451,14 @@ extension IslandWindowController {return}+ // ── Claude as the brain (Settings → Voice, the user's Anthropic key) ───────+ // Every phrase goes to Claude, which acts with Coucou's tools. Only when Claude+ // can't be reached does the phrase parser below take over.+ if runner.pendingQuestion == nil, ClaudeVoiceBrain.isActive {+ if await handleWithClaude(transcript) { return }+ appendAppLog("nb.log", "[Voice] Claude unreachable, phrase parser used")+ }+// ── Short noise / spurious activation guard (conversation mode only) ────────// A transcript shorter than 2 words that isn't a pill name or known command// is almost certainly a false activation. Silently re-listen without feedback.@@ -1805,6 +1823,52 @@ extension IslandWindowController {}}+ /// One turn with Claude: caption, spoken answer, and a re-listen when Claude asked+ /// something (its answer ends with "?"). False when Claude couldn't be reached.+ @MainActor+ private func handleWithClaude(_ transcript: String) async -> Bool {+ VoiceCaptionManager.shared.setUserLine(transcript)+ // A stale "finished speaking" handler must not reopen the mic during the wait.+ VoiceSpeaker.shared.onDidFinish = nil+ let t0 = Date()+ guard let reply = await ClaudeVoiceBrain.shared.respond(to: transcript), !reply.failed else {+ return false+ }+ appendAppLog("nb.log", "[Voice] Claude turn \(Int(Date().timeIntervalSince(t0) * 1000)) ms\(reply.acted ? ", acted" : "")")+ let fr = VoiceSettings.language == "fr"+ let text = reply.text.isEmpty ? (fr ? "C'est fait." : "Done.") : reply.text+ VoiceTranscriptHistory.shared.record(transcript: transcript, note: text, origin: .brain)+ consecutiveFailures = 0+ if reply.acted { NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy) }+ VoiceCaptionManager.shared.appendResponse(text)+ let result = VoiceActionResult(outcome: .success, message: text)+ AppState.shared.voiceResult = result+ speakAndContinueConversation(result)+ return true+ }++ /// A file dropped on the notch while the voice email card is open: it becomes the+ /// attachment, and Coucou says so.+ @MainActor+ func attachToVoiceMailCard(_ url: URL) {+ let state = AppState.shared+ state.fileDragOver = false+ state.droppedFile = DroppedFile(url: url, name: url.lastPathComponent)+ NotificationCenter.default.post(name: .botGulp, object: nil)+ NotificationCenter.default.post(name: .botMorphTo, object: CGFloat(0))+ NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)+ if state.soundEnabled { SoundEngine.shared.play("approve") }+ fsm.openedExternally()+ expand(to: .mail)+ let fr = VoiceSettings.language == "fr"+ let text = fr ? "\(url.lastPathComponent) est en pièce jointe. Tu n'as plus qu'à cliquer sur Envoyer."+ : "\(url.lastPathComponent) is attached. Just click Send."+ if VoiceSettings.speakEnabled {+ VoiceSpeaker.shared.onDidFinish = nil+ VoiceSpeaker.shared.speak(text, locale: VoiceSettings.answerLocale)+ }+ }+/// A voice command that failed: a small "huh?" from Mochi. Not .botDizzy, which is the/// slap reaction and opened the "Too many hits at once" card in the middle of a mail.@MainActor@@ -1851,6 +1915,7 @@ extension IslandWindowController {Task { @MainActor inself?.conversationContext.reset()VoiceActionRunner.shared.resetWebThread()+ ClaudeVoiceBrain.shared.reset()VoiceBrain.shared.endConversation()}}diff --git a/NotchBuddy/Sources/App/SettingsView.swift b/NotchBuddy/Sources/App/SettingsView.swiftindex 3db6b22e..445ba9cf 100644--- a/NotchBuddy/Sources/App/SettingsView.swift+++ b/NotchBuddy/Sources/App/SettingsView.swift@@ -1268,7 +1268,7 @@ struct SettingsView: View {@AppStorage("voiceElevenGender") private var elevenGender: String = "female"@AppStorage("voicePitchFemale") private var pitchFemale: Double = VoiceSettings.defaultPitchFemale@AppStorage("voicePitchMale") private var pitchMale: Double = VoiceSettings.defaultPitchMale- @AppStorage("voiceWebSearchEnabled") private var webSearchEnabled: Bool = false+ @AppStorage("voiceClaudeBrain") private var webSearchEnabled: Bool = false@State private var webKeySaved: Bool = !(KeychainStore.shared.get("anthropic-api-key") ?? "").isEmpty@State private var elevenKeyDraft: String = ""@State private var elevenKeySaved: Bool = KeychainStore.shared.get(ElevenLabsTTS.keyName) != nildiff --git a/NotchBuddy/Sources/App/Voice/ClaudeVoiceBrain.swift b/NotchBuddy/Sources/App/Voice/ClaudeVoiceBrain.swiftnew file mode 100644index 00000000..dd0c25f9--- /dev/null+++ b/NotchBuddy/Sources/App/Voice/ClaudeVoiceBrain.swift@@ -0,0 +1,294 @@+#if !APPSTORE+import Foundation++// MARK: - ClaudeVoiceBrain+//+// "OK Coucou, …" handled by Claude (the user's Anthropic key, Settings → Voice), not by+// the phrase parser or the on-device model: Claude understands any wording, keeps the+// thread of the conversation and calls Coucou's tools to act.+//+// Tools: pills, music, service_info (Stripe, GitHub, agents…), open_app, prepare_email+// (the island's mail card opens filled in, the user clicks Send) and Anthropic's+// web_search. The conversation lives in memory only and is forgotten with the rest of+// the voice context (90 s after the last turn). Nothing is sent, approved or deleted+// without a click: the tools can't.+@MainActor+final class ClaudeVoiceBrain {+ static let shared = ClaudeVoiceBrain()++ struct Reply {+ let text: String // spoken and shown+ let acted: Bool // a tool changed something (pills, music, mail…)+ let failed: Bool // nothing usable came back+ }++ /// On when the setting is on and an Anthropic key is saved.+ static var isActive: Bool {+ VoiceSettings.claudeBrainEnabled && !(ClaudeService.shared.apiKey ?? "").isEmpty+ }++ private var messages: [[String: Any]] = []++ func reset() { messages = [] }++ // MARK: Turn++ func respond(to transcript: String) async -> Reply? {+ messages.append(["role": "user", "content": transcript])+ trimHistory()+ var acted = false+ var spoken = ""++ // Claude may call several tools in a row (look up, then act): at most 5 rounds.+ for _ in 0..<5 {+ guard let turn = await ClaudeService.shared.voiceTurn(system: Self.systemPrompt(),+ messages: messages,+ tools: Self.tools) else {+ // Leave the history coherent: drop the question that got no answer.+ if let last = messages.last, last["role"] as? String == "user",+ last["content"] is String { messages.removeLast() }+ return nil+ }+ messages.append(["role": "assistant", "content": turn.content])+ if let text = claudeResponseText(fromContent: turn.content) { spoken = text }++ let uses = turn.content.filter { $0["type"] as? String == "tool_use" }+ if turn.stopReason == "pause_turn" { continue } // long web search: go on+ guard turn.stopReason == "tool_use", !uses.isEmpty else { break }++ var results: [[String: Any]] = []+ for use in uses {+ let id = use["id"] as? String ?? ""+ let name = use["name"] as? String ?? ""+ let input = use["input"] as? [String: Any] ?? [:]+ let (result, didAct) = await run(tool: name, input: input)+ if didAct { acted = true }+ results.append(["type": "tool_result", "tool_use_id": id, "content": result])+ }+ messages.append(["role": "user", "content": results])+ spoken = ""+ }++ let text = VoiceQuery.spokenText(spoken)+ return Reply(text: text, acted: acted, failed: text.isEmpty && !acted)+ }++ /// Keeps the last exchanges, never cutting between a tool call and its result.+ private func trimHistory() {+ guard messages.count > 24 else { return }+ var cut = messages.count - 24+ // Start on a plain user message (a string), not on tool results.+ while cut < messages.count,+ !(messages[cut]["role"] as? String == "user" && messages[cut]["content"] is String) {+ cut += 1+ }+ messages.removeFirst(min(cut, messages.count - 1))+ }++ // MARK: Prompt++ private static func systemPrompt() -> String {+ let fr = VoiceSettings.language == "fr"+ let today = Date().formatted(.dateTime.weekday(.wide).day().month(.wide).year().hour().minute()+ .locale(Locale(identifier: fr ? "fr_FR" : "en_US")))+ let state = AppState.shared+ let active = ([state.mainPillId] + Array(state.activeIntegrations))+ .compactMap { PillCatalog.definition(for: $0)?.name }+ let available = PillCatalog.available.map(\.name).joined(separator: ", ")+ let language = fr ? "French" : "English"+ return """+ You are Coucou, a voice assistant living in the notch of the user's Mac, a bit like Jarvis: \+ quick, warm, a little playful. It is \(today).+ Everything you write is read aloud: answer in \(language) (the user may speak French or English), \+ in one or two short sentences, no markdown, no lists, no emojis, no links.+ Act with the tools instead of describing: pills, music, service_info for the user's own data \+ (Stripe sales, GitHub, Vercel, Resend, n8n, Notion, Cal.com, agent sessions like Claude Code, \+ plan usage, music, weather), open_app, prepare_email, and web_search for anything current \+ (news, scores, prices, facts you are not sure of). Never invent the user's data.+ Email: gather the recipient, the subject and what to say (write the text yourself from what \+ the user wants to say, in the language they used), asking for what is missing one short \+ question at a time, then call prepare_email. The email is never sent by you: the user checks \+ the card and clicks Send. If they want an attachment and haven't named the file, set \+ wants_attachment and tell them to drop the file on the notch.+ When you need an answer from the user, end with one short question. Otherwise never end \+ with a question.+ Pills in the notch (at most 4 besides the main one). Active: \(active.joined(separator: ", ")). \+ Available: \(available).+ """+ }++ // MARK: Tools++ private static let tools: [[String: Any]] = [+ [+ "name": "pills",+ "description": "Add, remove, set as main or list the pills shown in the notch. At most 4 pills besides the main one: remove one first if needed.",+ "input_schema": [+ "type": "object",+ "properties": [+ "action": ["type": "string", "enum": ["add", "remove", "set_main", "list"]],+ "names": ["type": "array", "items": ["type": "string"],+ "description": "Pill names as listed (e.g. Stripe, GitHub, Cursor). Empty for list."],+ ],+ "required": ["action"],+ ],+ ],+ [+ "name": "music",+ "description": "Control Apple Music or Spotify: play, pause, next, previous, volume, play a song/artist search or a playlist.",+ "input_schema": [+ "type": "object",+ "properties": [+ "action": ["type": "string", "enum": ["play", "pause", "next", "previous", "volume_up",+ "volume_down", "set_volume", "play_search", "play_playlist"]],+ "query": ["type": "string", "description": "Song, artist or playlist name for play_search / play_playlist."],+ "volume": ["type": "integer", "description": "0-100 for set_volume."],+ ],+ "required": ["action"],+ ],+ ],+ [+ "name": "service_info",+ "description": "Read the user's real data that Coucou already has: Stripe sales and balance, GitHub stars/PRs/CI, Vercel deployments, Resend emails, n8n runs, Notion pages, Cal.com bookings, agent sessions (Claude Code, Codex…), Claude or Codex plan usage, music now playing, active pills, weather today or tomorrow.",+ "input_schema": [+ "type": "object",+ "properties": [+ "topic": ["type": "string", "enum": VoiceTopic.allCases.map(\.rawValue)],+ ],+ "required": ["topic"],+ ],+ ],+ [+ "name": "open_app",+ "description": "Open an app on the Mac by name (Figma, Safari, VS Code, Notion…).",+ "input_schema": [+ "type": "object",+ "properties": ["name": ["type": "string"]],+ "required": ["name"],+ ],+ ],+ [+ "name": "prepare_email",+ "description": "Open the email card in the notch, filled in, for the user to check and send with a click (it is never sent automatically). The recipient is a contact name or an email address.",+ "input_schema": [+ "type": "object",+ "properties": [+ "recipient": ["type": "string"],+ "subject": ["type": "string"],+ "body": ["type": "string", "description": "The full email text, written for the user."],+ "attachment_file": ["type": "string", "description": "File name to attach if the user named one (searched in Downloads, Desktop, Documents, Pictures)."],+ "wants_attachment": ["type": "boolean", "description": "True if the user wants to attach a file without naming it: they drop it on the notch."],+ ],+ "required": ["recipient", "subject", "body"],+ ],+ ],+ ["type": "web_search_20250305", "name": "web_search", "max_uses": 3],+ ]++ // MARK: Running a tool++ /// Returns the text sent back to Claude, and whether something changed.+ private func run(tool: String, input: [String: Any]) async -> (String, Bool) {+ let runner = VoiceActionRunner.shared+ let pills = PillCatalog.available+ runner.commandLocale = VoiceSettings.answerLocale+ func perform(_ intent: VoiceIntent) async -> VoiceActionResult {+ let r = await runner.run(intent, availablePills: pills)+ // The runner may ask its own follow-up (4-pill limit…): Claude asks instead.+ runner.pendingQuestion = nil+ return r+ }++ switch tool {+ case "pills":+ let action = input["action"] as? String ?? ""+ let names = input["names"] as? [String] ?? []+ if action == "list" {+ return (await LiveVoiceInfo.shared.answer(.pills, locale: VoiceSettings.answerLocale), false)+ }+ var lines: [String] = []+ var acted = false+ for name in names {+ guard let id = EntityResolver.resolve(name, from: pills) else {+ lines.append("\(name): unknown pill"); continue+ }+ let intent: VoiceIntent+ switch action {+ case "add": intent = .pillAdd(id: id)+ case "remove": intent = .pillRemove(id: id)+ case "set_main": intent = .pillSetMain(id: id)+ default: lines.append("unknown action \(action)"); continue+ }+ let r = await perform(intent)+ if case .question = r.outcome {+ lines.append("\(name): the notch is full (4 pills). Ask which pill to remove, then remove it and add \(name).")+ } else {+ if r.outcome == .success { acted = true }+ lines.append("\(name): \(r.message)")+ }+ }+ return (lines.isEmpty ? "nothing done" : lines.joined(separator: "\n"), acted)++ case "music":+ let action = input["action"] as? String ?? ""+ let query = (input["query"] as? String ?? "").trimmingCharacters(in: .whitespaces)+ let intent: VoiceIntent?+ switch action {+ case "play": intent = .musicPlay(target: nil)+ case "pause": intent = .musicPause+ case "next": intent = .musicNext+ case "previous": intent = .musicPrevious+ case "volume_up": intent = .musicVolumeUp+ case "volume_down": intent = .musicVolumeDown+ case "set_volume": intent = .musicSetVolume(max(0, min(100, input["volume"] as? Int ?? 50)))+ case "play_search": intent = query.isEmpty ? nil : .musicPlaySearch(name: query)+ case "play_playlist": intent = query.isEmpty ? nil : .musicPlayPlaylist(name: query)+ default: intent = nil+ }+ guard let intent else { return ("missing query or unknown action", false) }+ let r = await perform(intent)+ return (r.message, r.outcome == .success)++ case "service_info":+ guard let topic = VoiceTopic(rawValue: input["topic"] as? String ?? "") else {+ return ("unknown topic", false)+ }+ let text = await LiveVoiceInfo.shared.answer(topic, locale: VoiceSettings.answerLocale)+ return (text.isEmpty ? "no data" : text, false)++ case "open_app":+ let name = input["name"] as? String ?? ""+ let r = LiveVoiceInfo.shared.openApp(name, locale: VoiceSettings.answerLocale)+ return (r.message, r.outcome == .success)++ case "prepare_email":+ return await prepareEmail(input)++ default:+ return ("unknown tool \(tool)", false)+ }+ }++ private func prepareEmail(_ input: [String: Any]) async -> (String, Bool) {+ let info = LiveVoiceInfo.shared+ let recipient = (input["recipient"] as? String ?? "").trimmingCharacters(in: .whitespacesAndNewlines)+ let subject = input["subject"] as? String ?? ""+ let body = input["body"] as? String ?? ""+ guard !recipient.isEmpty else { return ("no recipient: ask who it is for", false) }+ guard let address = await info.resolveEmail(recipient) else {+ return ("No email address found for \(recipient) in the contacts. Ask the user to spell the address.", false)+ }+ var file: URL? = nil+ var note = ""+ if let name = input["attachment_file"] as? String, !name.trimmingCharacters(in: .whitespaces).isEmpty {+ file = info.findFile(name, folder: nil)+ if file == nil { note = " The file \(name) was not found: the user can drop it on the notch." }+ }+ info.showMailCard(to: address, subject: subject, body: body, file: file)+ let attach = (input["wants_attachment"] as? Bool ?? false) && file == nil+ ? " The user wants an attachment: tell them to drop the file on the notch." : ""+ let with = file.map { " with \($0.lastPathComponent) attached" } ?? ""+ return ("Email card shown to \(address)\(with). Nothing is sent until the user clicks Send.\(note)\(attach)", true)+ }+}+#endifdiff --git a/NotchBuddy/Sources/App/Voice/VoiceSettings.swift b/NotchBuddy/Sources/App/Voice/VoiceSettings.swiftindex fd09b344..f46e88af 100644--- a/NotchBuddy/Sources/App/Voice/VoiceSettings.swift+++ b/NotchBuddy/Sources/App/Voice/VoiceSettings.swift@@ -77,12 +77,15 @@ enum VoiceSettings {return v == 0 ? fallback : min(1.6, max(0.8, v))}- /// Questions answered with a web search by Claude, with the user's own Anthropic API- /// key. Off by default: the question leaves the Mac only once this is turned on.- static var webSearchEnabled: Bool {- get { UserDefaults.standard.bool(forKey: "voiceWebSearchEnabled") }- set { UserDefaults.standard.set(newValue, forKey: "voiceWebSearchEnabled") }+ /// Claude is Coucou's brain (the user's Anthropic API key): what I say after+ /// "OK Coucou" goes to Claude, which acts with Coucou's tools and searches the web.+ /// Off by default: my words leave the Mac only once this is turned on.+ static var claudeBrainEnabled: Bool {+ get { UserDefaults.standard.bool(forKey: "voiceClaudeBrain") }+ set { UserDefaults.standard.set(newValue, forKey: "voiceClaudeBrain") }}+ /// The older web-search-only path follows the same switch.+ static var webSearchEnabled: Bool { claudeBrainEnabled }/// Weather by voice (Open-Meteo, no key). Off by default: network only when the user/// turned it on and set a city.