Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

voice: Claude as Coucou's brain (the user's Anthropic key), with Coucou's tools

Claudeaacd077fb409
diff --git a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
index 89ec9d01..ef367c43 100644
--- a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
+++ b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
@@ -138,6 +138,7 @@
6FC65C190ED614C664C9BC17 /* SafeWebURL.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2FDED98F7F6C755A72C97A3 /* SafeWebURL.swift */; };
714B5BB7FFB0C8491AF67AD1 /* VoiceTranscriptHistory.swift in Sources */ = {isa = PBXBuildFile; fileRef = DE925A6844FE3E3C09131CE0 /* VoiceTranscriptHistory.swift */; };
718410ED630AE3AC5DF59A95 /* VoiceSpeaker.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */; };
+ 50C33F44687D70690514FBBB /* ClaudeVoiceBrain.swift in Sources */ = {isa = PBXBuildFile; fileRef = ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */; };
9BAD96406059597681D89A92 /* VoiceGenderNames.swift in Sources */ = {isa = PBXBuildFile; fileRef = 16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */; };
71CBD1D39A6668BF454CA1C5 /* IslandTypes.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0365DDE3C554F5E8570055F4 /* IslandTypes.swift */; };
724E602A05FE5A6D58218E27 /* CoucouControl.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5C406C508A86E215B3CBE3FC /* CoucouControl.swift */; };
@@ -310,6 +311,7 @@
F6CA56F362902839EF9A3E2E /* PhoneAssets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 60401D1023D17C3FD30D7113 /* PhoneAssets.xcassets */; };
F6D57EEBD7E937EFD2B715CF /* ClaudeHost.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F0D0F67E7C56B9E37E7648 /* ClaudeHost.swift */; };
F7162101509D8CAE59711602 /* VoiceSpeaker.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */; };
+ 70F221444F3D0B11EEA0E758 /* ClaudeVoiceBrain.swift in Sources */ = {isa = PBXBuildFile; fileRef = ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */; };
D166CDFE3CBAF1310E7EC1F7 /* VoiceGenderNames.swift in Sources */ = {isa = PBXBuildFile; fileRef = 16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */; };
F8F95EAD88F70E0A147D1662 /* MacDictation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6544ED02D945CF9F00647AD1 /* MacDictation.swift */; };
F9445A71BBF9E4D20224238E /* QuestionPayload.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6C6993B34A0FFF8E3D454ADC /* QuestionPayload.swift */; };
@@ -475,6 +477,7 @@
C563E55884A7750B40EC9A01 /* VoiceBrain.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceBrain.swift; sourceTree = "<group>"; };
C581F7B1A9F14B281B60063E /* GreetingCanvasView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GreetingCanvasView.swift; sourceTree = "<group>"; };
CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceSpeaker.swift; sourceTree = "<group>"; };
+ ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClaudeVoiceBrain.swift; sourceTree = "<group>"; };
16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceGenderNames.swift; sourceTree = "<group>"; };
D0F0D0F67E7C56B9E37E7648 /* ClaudeHost.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClaudeHost.swift; sourceTree = "<group>"; };
D1A7C62058BB0C9D0DF58DCB /* VoiceIntent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceIntent.swift; sourceTree = "<group>"; };
@@ -752,6 +755,7 @@
7C432FE1286945E42D3E11D7 /* VoiceResultView.swift */,
D4543CB771E0B723783F529E /* VoiceSettings.swift */,
CD11C7925B0A7831BFC2199C /* VoiceSpeaker.swift */,
+ ADBD590972D9536A1DA7B6EF /* ClaudeVoiceBrain.swift */,
16BF6539981E9EEF2397D061 /* VoiceGenderNames.swift */,
DE925A6844FE3E3C09131CE0 /* VoiceTranscriptHistory.swift */,
ABD7A2C29B15EFFAF67FF2DE /* VoiceWakeFilter.swift */,
@@ -1067,6 +1071,7 @@
CE5ACDD856C76B717ABA295F /* VoiceResultView.swift in Sources */,
502F2B0D607C511DD3B836B3 /* VoiceSettings.swift in Sources */,
718410ED630AE3AC5DF59A95 /* VoiceSpeaker.swift in Sources */,
+ 50C33F44687D70690514FBBB /* ClaudeVoiceBrain.swift in Sources */,
9BAD96406059597681D89A92 /* VoiceGenderNames.swift in Sources */,
189811F4DB3C4F6030043643 /* VoiceTranscriptHistory.swift in Sources */,
BBCA973A1AE1385BBE79A834 /* VoiceWakeFilter.swift in Sources */,
@@ -1258,6 +1263,7 @@
F9685A58729FE64B23FCCFC5 /* VoiceResultView.swift in Sources */,
035F9A85BF71B21F9AA5E004 /* VoiceSettings.swift in Sources */,
F7162101509D8CAE59711602 /* VoiceSpeaker.swift in Sources */,
+ 70F221444F3D0B11EEA0E758 /* ClaudeVoiceBrain.swift in Sources */,
D166CDFE3CBAF1310E7EC1F7 /* VoiceGenderNames.swift in Sources */,
714B5BB7FFB0C8491AF67AD1 /* VoiceTranscriptHistory.swift in Sources */,
8DE75D1E260A1314D1C47885 /* VoiceWakeFilter.swift in Sources */,
diff --git a/NotchBuddy/Resources/Localizable.xcstrings b/NotchBuddy/Resources/Localizable.xcstrings
index 8d1f0ca5..9b27c6d9 100644
--- a/NotchBuddy/Resources/Localizable.xcstrings
+++ b/NotchBuddy/Resources/Localizable.xcstrings
@@ -25953,61 +25953,61 @@
"ar": {
"stringUnit": {
"state": "translated",
- "value": "الإجابة عن الأسئلة ببحث على الويب"
+ "value": "Claude عقل Coucou"
}
},
"bn": {
"stringUnit": {
"state": "translated",
- "value": "ওয়েবে খুঁজে প্রশ্নের উত্তর দাও"
+ "value": "Coucou-র মস্তিষ্ক: Claude"
}
},
"en": {
"stringUnit": {
"state": "translated",
- "value": "Answer questions with a web search"
+ "value": "Claude as Coucou's brain"
}
},
"es": {
"stringUnit": {
"state": "translated",
- "value": "Responder preguntas buscando en la web"
+ "value": "Claude como cerebro de Coucou"
}
},
"fr": {
"stringUnit": {
"state": "translated",
- "value": "Répondre aux questions avec une recherche web"
+ "value": "Claude comme cerveau de Coucou"
}
},
"hi": {
"stringUnit": {
"state": "translated",
- "value": "वेब खोज से सवालों के जवाब"
+ "value": "Coucou का दिमाग: Claude"
}
},
"id": {
"stringUnit": {
"state": "translated",
- "value": "Jawab pertanyaan dengan pencarian web"
+ "value": "Claude sebagai otak Coucou"
}
},
"pt-BR": {
"stringUnit": {
"state": "translated",
- "value": "Responder perguntas com uma busca na web"
+ "value": "Claude como cérebro do Coucou"
}
},
"ru": {
"stringUnit": {
"state": "translated",
- "value": "Отвечать на вопросы с поиском в интернете"
+ "value": "Claude — мозг Coucou"
}
},
"zh-Hans": {
"stringUnit": {
"state": "translated",
- "value": "用网页搜索回答问题"
+ "value": "由 Claude 担任 Coucou 的大脑"
}
}
}
@@ -26018,61 +26018,61 @@
"ar": {
"stringUnit": {
"state": "translated",
- "value": "الأسئلة التي لا تجيب عنها حبوبك تُرسل إلى Claude مع بحث على الويب، بمفتاح Anthropic API الخاص بك. يُرسل السؤال فقط."
+ "value": "بعد «OK Coucou»، يُرسل ما تقوله إلى Claude بمفتاح Anthropic API الخاص بك: ينفّذ (الحبوب، الموسيقى، البريد، خدماتك) ويبحث على الويب. تُرسل كلماتك فقط، وليس الصوت أبدًا."
}
},
"bn": {
"stringUnit": {
"state": "translated",
- "value": "যে প্রশ্নের উত্তর তোমার পিলগুলো দিতে পারে না, সেগুলো তোমার Anthropic API কী দিয়ে ওয়েব সার্চসহ Claude-এর কাছে যায়। শুধু প্রশ্নটাই পাঠানো হয়।"
+ "value": "«OK Coucou»-এর পরে তুমি যা বলো তা তোমার Anthropic API কী দিয়ে Claude-এর কাছে যায়: সে কাজ করে (পিল, গান, মেইল, তোমার সার্ভিস) আর ওয়েবে খোঁজে। শুধু তোমার কথা পাঠানো হয়, কখনো অডিও নয়।"
}
},
"en": {
"stringUnit": {
"state": "translated",
- "value": "Questions your pills can't answer go to Claude with a web search, using your Anthropic API key. Only the question is sent."
+ "value": "After «OK Coucou», what you say goes to Claude with your Anthropic API key: it acts (pills, music, email, your services) and searches the web. Only your words are sent, never audio."
}
},
"es": {
"stringUnit": {
"state": "translated",
- "value": "Las preguntas que tus píldoras no responden van a Claude con una búsqueda web, con tu clave de API de Anthropic. Solo se envía la pregunta."
+ "value": "Tras «OK Coucou», lo que dices va a Claude con tu clave de API de Anthropic: actúa (píldoras, música, correo, tus servicios) y busca en la web. Solo se envían tus palabras, nunca el audio."
}
},
"fr": {
"stringUnit": {
"state": "translated",
- "value": "Les questions auxquelles tes pilules ne répondent pas partent chez Claude avec une recherche web, avec ta clé API Anthropic. Seule la question est envoyée."
+ "value": "Après « OK Coucou », ce que tu dis part chez Claude avec ta clé API Anthropic : il agit (pilules, musique, mail, tes services) et cherche sur le web. Seuls tes mots sont envoyés, jamais l'audio."
}
},
"hi": {
"stringUnit": {
"state": "translated",
- "value": "जिन सवालों का जवाब तुम्हारी पिल्स नहीं दे सकतीं, वे तुम्हारी Anthropic API कुंजी से वेब खोज के साथ Claude को जाते हैं। सिर्फ़ सवाल भेजा जाता है।"
+ "value": "«OK Coucou» के बाद तुम जो कहते हो वह तुम्हारी Anthropic API कुंजी से Claude को जाता है: वह काम करता है (पिल्स, संगीत, ईमेल, तुम्हारी सेवाएँ) और वेब पर खोजता है। सिर्फ़ तुम्हारे शब्द भेजे जाते हैं, ऑडियो कभी नहीं।"
}
},
"id": {
"stringUnit": {
"state": "translated",
- "value": "Pertanyaan yang tidak bisa dijawab pil-mu dikirim ke Claude dengan pencarian web, memakai kunci API Anthropic-mu. Hanya pertanyaannya yang dikirim."
+ "value": "Setelah «OK Coucou», ucapanmu dikirim ke Claude dengan kunci API Anthropic-mu: ia bertindak (pil, musik, email, layananmu) dan mencari di web. Hanya kata-katamu yang dikirim, tidak pernah audio."
}
},
"pt-BR": {
"stringUnit": {
"state": "translated",
- "value": "Perguntas que suas pílulas não respondem vão para o Claude com uma busca na web, usando sua chave de API da Anthropic. Só a pergunta é enviada."
+ "value": "Depois de «OK Coucou», o que você diz vai para o Claude com sua chave de API da Anthropic: ele age (pílulas, música, e-mail, seus serviços) e pesquisa na web. Só suas palavras são enviadas, nunca o áudio."
}
},
"ru": {
"stringUnit": {
"state": "translated",
- "value": "Вопросы, на которые не отвечают твои пилюли, уходят в Claude с поиском в интернете, по твоему ключу Anthropic API. Отправляется только вопрос."
+ "value": "После «OK Coucou» сказанное уходит в Claude по твоему ключу Anthropic API: он действует (пилюли, музыка, почта, твои сервисы) и ищет в интернете. Отправляются только слова, никогда не звук."
}
},
"zh-Hans": {
"stringUnit": {
"state": "translated",
- "value": "药丸无法回答的问题会用你的 Anthropic API 密钥发送给 Claude 并进行网页搜索。只发送问题本身。"
+ "value": "说「OK Coucou」后,你说的话会用你的 Anthropic API 密钥发送给 Claude:它会执行操作(药丸、音乐、邮件、你的服务)并进行网页搜索。只发送文字,从不发送音频。"
}
}
}
diff --git a/NotchBuddy/Sources/App/ClaudeService.swift b/NotchBuddy/Sources/App/ClaudeService.swift
index 24c26ccd..cbbb7c19 100644
--- a/NotchBuddy/Sources/App/ClaudeService.swift
+++ b/NotchBuddy/Sources/App/ClaudeService.swift
@@ -502,6 +502,31 @@ final class ClaudeService {
}
#if !APPSTORE
+ // MARK: - Voice brain turn (Claude with Coucou's tools)
+
+ /// One Messages API call for the voice brain: returns the assistant content blocks and
+ /// the stop reason. The fast model first, the Settings model if the key can't use it.
+ func voiceTurn(system: String, messages: [[String: Any]], tools: [[String: Any]])
+ async -> (content: [[String: Any]], stopReason: String)? {
+ guard let key = apiKey, !key.isEmpty else { return nil }
+ func body(_ model: String) -> [String: Any] {
+ ["model": model, "max_tokens": 600, "system": system, "tools": tools, "messages": messages]
+ }
+ let data: Data
+ do {
+ data = try await callAPI(body: body(Self.voiceModel), key: key, beta: "web-search-2025-03-05")
+ } catch {
+ guard let retry = try? await callAPI(body: body(model), key: key, beta: "web-search-2025-03-05") else {
+ appendAppLog("nb.log", "[Voice] Claude turn failed: \(error.localizedDescription)")
+ return nil
+ }
+ data = retry
+ }
+ guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
+ let content = json["content"] as? [[String: Any]] else { return nil }
+ return (content, json["stop_reason"] as? String ?? "end_turn")
+ }
+
// MARK: - Voice answer (Coucou's voice, web search)
/// Fast model for spoken answers; the chat model from Settings when it is unavailable.
diff --git a/NotchBuddy/Sources/App/IslandWindowController.swift b/NotchBuddy/Sources/App/IslandWindowController.swift
index 19d2f4ed..fbe8f420 100644
--- a/NotchBuddy/Sources/App/IslandWindowController.swift
+++ b/NotchBuddy/Sources/App/IslandWindowController.swift
@@ -128,6 +128,11 @@ final class IslandWindowController: NSWindowController {
Task { @MainActor in
let iLoc = self?.windowToIsland(loc) ?? CGPoint(x: 320, y: 88)
AppState.shared.fileDragOver = true
+ // The voice mail card stays on screen: the file will be its attachment.
+ if AppState.shared.voiceMailDraft != nil && AppState.shared.view == .mail {
+ NotificationCenter.default.post(name: .botMorphTo, object: CGFloat(1))
+ return
+ }
// enterZone sets isActive=true BEFORE hookExpand triggers re-render,
// so IslandContainer sees isActive=true when state.view becomes .upload.
UploadSequenceEngine.shared.enterZone(x: iLoc.x, y: iLoc.y)
@@ -158,6 +163,11 @@ final class IslandWindowController: NSWindowController {
await self?.attachVoiceMailFile(url)
return
}
+ // The mail card prepared by voice (Claude) is open: the file is its attachment.
+ if AppState.shared.voiceMailDraft != nil, let url = urls.first {
+ self?.attachToVoiceMailCard(url)
+ return
+ }
#endif
await FileDropHandler.handle(urls: urls, state: AppState.shared)
}
@@ -1441,6 +1451,14 @@ extension IslandWindowController {
return
}
+ // ── Claude as the brain (Settings → Voice, the user's Anthropic key) ───────
+ // Every phrase goes to Claude, which acts with Coucou's tools. Only when Claude
+ // can't be reached does the phrase parser below take over.
+ if runner.pendingQuestion == nil, ClaudeVoiceBrain.isActive {
+ if await handleWithClaude(transcript) { return }
+ appendAppLog("nb.log", "[Voice] Claude unreachable, phrase parser used")
+ }
+
// ── Short noise / spurious activation guard (conversation mode only) ────────
// A transcript shorter than 2 words that isn't a pill name or known command
// is almost certainly a false activation. Silently re-listen without feedback.
@@ -1805,6 +1823,52 @@ extension IslandWindowController {
}
}
+ /// One turn with Claude: caption, spoken answer, and a re-listen when Claude asked
+ /// something (its answer ends with "?"). False when Claude couldn't be reached.
+ @MainActor
+ private func handleWithClaude(_ transcript: String) async -> Bool {
+ VoiceCaptionManager.shared.setUserLine(transcript)
+ // A stale "finished speaking" handler must not reopen the mic during the wait.
+ VoiceSpeaker.shared.onDidFinish = nil
+ let t0 = Date()
+ guard let reply = await ClaudeVoiceBrain.shared.respond(to: transcript), !reply.failed else {
+ return false
+ }
+ appendAppLog("nb.log", "[Voice] Claude turn \(Int(Date().timeIntervalSince(t0) * 1000)) ms\(reply.acted ? ", acted" : "")")
+ let fr = VoiceSettings.language == "fr"
+ let text = reply.text.isEmpty ? (fr ? "C'est fait." : "Done.") : reply.text
+ VoiceTranscriptHistory.shared.record(transcript: transcript, note: text, origin: .brain)
+ consecutiveFailures = 0
+ if reply.acted { NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy) }
+ VoiceCaptionManager.shared.appendResponse(text)
+ let result = VoiceActionResult(outcome: .success, message: text)
+ AppState.shared.voiceResult = result
+ speakAndContinueConversation(result)
+ return true
+ }
+
+ /// A file dropped on the notch while the voice email card is open: it becomes the
+ /// attachment, and Coucou says so.
+ @MainActor
+ func attachToVoiceMailCard(_ url: URL) {
+ let state = AppState.shared
+ state.fileDragOver = false
+ state.droppedFile = DroppedFile(url: url, name: url.lastPathComponent)
+ NotificationCenter.default.post(name: .botGulp, object: nil)
+ NotificationCenter.default.post(name: .botMorphTo, object: CGFloat(0))
+ NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
+ if state.soundEnabled { SoundEngine.shared.play("approve") }
+ fsm.openedExternally()
+ expand(to: .mail)
+ let fr = VoiceSettings.language == "fr"
+ let text = fr ? "\(url.lastPathComponent) est en pièce jointe. Tu n'as plus qu'à cliquer sur Envoyer."
+ : "\(url.lastPathComponent) is attached. Just click Send."
+ if VoiceSettings.speakEnabled {
+ VoiceSpeaker.shared.onDidFinish = nil
+ VoiceSpeaker.shared.speak(text, locale: VoiceSettings.answerLocale)
+ }
+ }
+
/// A voice command that failed: a small "huh?" from Mochi. Not .botDizzy, which is the
/// slap reaction and opened the "Too many hits at once" card in the middle of a mail.
@MainActor
@@ -1851,6 +1915,7 @@ extension IslandWindowController {
Task { @MainActor in
self?.conversationContext.reset()
VoiceActionRunner.shared.resetWebThread()
+ ClaudeVoiceBrain.shared.reset()
VoiceBrain.shared.endConversation()
}
}
diff --git a/NotchBuddy/Sources/App/SettingsView.swift b/NotchBuddy/Sources/App/SettingsView.swift
index 3db6b22e..445ba9cf 100644
--- a/NotchBuddy/Sources/App/SettingsView.swift
+++ b/NotchBuddy/Sources/App/SettingsView.swift
@@ -1268,7 +1268,7 @@ struct SettingsView: View {
@AppStorage("voiceElevenGender") private var elevenGender: String = "female"
@AppStorage("voicePitchFemale") private var pitchFemale: Double = VoiceSettings.defaultPitchFemale
@AppStorage("voicePitchMale") private var pitchMale: Double = VoiceSettings.defaultPitchMale
- @AppStorage("voiceWebSearchEnabled") private var webSearchEnabled: Bool = false
+ @AppStorage("voiceClaudeBrain") private var webSearchEnabled: Bool = false
@State private var webKeySaved: Bool = !(KeychainStore.shared.get("anthropic-api-key") ?? "").isEmpty
@State private var elevenKeyDraft: String = ""
@State private var elevenKeySaved: Bool = KeychainStore.shared.get(ElevenLabsTTS.keyName) != nil
diff --git a/NotchBuddy/Sources/App/Voice/ClaudeVoiceBrain.swift b/NotchBuddy/Sources/App/Voice/ClaudeVoiceBrain.swift
new file mode 100644
index 00000000..dd0c25f9
--- /dev/null
+++ b/NotchBuddy/Sources/App/Voice/ClaudeVoiceBrain.swift
@@ -0,0 +1,294 @@
+#if !APPSTORE
+import Foundation
+
+// MARK: - ClaudeVoiceBrain
+//
+// "OK Coucou, …" handled by Claude (the user's Anthropic key, Settings → Voice), not by
+// the phrase parser or the on-device model: Claude understands any wording, keeps the
+// thread of the conversation and calls Coucou's tools to act.
+//
+// Tools: pills, music, service_info (Stripe, GitHub, agents…), open_app, prepare_email
+// (the island's mail card opens filled in, the user clicks Send) and Anthropic's
+// web_search. The conversation lives in memory only and is forgotten with the rest of
+// the voice context (90 s after the last turn). Nothing is sent, approved or deleted
+// without a click: the tools can't.
+@MainActor
+final class ClaudeVoiceBrain {
+ static let shared = ClaudeVoiceBrain()
+
+ struct Reply {
+ let text: String // spoken and shown
+ let acted: Bool // a tool changed something (pills, music, mail…)
+ let failed: Bool // nothing usable came back
+ }
+
+ /// On when the setting is on and an Anthropic key is saved.
+ static var isActive: Bool {
+ VoiceSettings.claudeBrainEnabled && !(ClaudeService.shared.apiKey ?? "").isEmpty
+ }
+
+ private var messages: [[String: Any]] = []
+
+ func reset() { messages = [] }
+
+ // MARK: Turn
+
+ func respond(to transcript: String) async -> Reply? {
+ messages.append(["role": "user", "content": transcript])
+ trimHistory()
+ var acted = false
+ var spoken = ""
+
+ // Claude may call several tools in a row (look up, then act): at most 5 rounds.
+ for _ in 0..<5 {
+ guard let turn = await ClaudeService.shared.voiceTurn(system: Self.systemPrompt(),
+ messages: messages,
+ tools: Self.tools) else {
+ // Leave the history coherent: drop the question that got no answer.
+ if let last = messages.last, last["role"] as? String == "user",
+ last["content"] is String { messages.removeLast() }
+ return nil
+ }
+ messages.append(["role": "assistant", "content": turn.content])
+ if let text = claudeResponseText(fromContent: turn.content) { spoken = text }
+
+ let uses = turn.content.filter { $0["type"] as? String == "tool_use" }
+ if turn.stopReason == "pause_turn" { continue } // long web search: go on
+ guard turn.stopReason == "tool_use", !uses.isEmpty else { break }
+
+ var results: [[String: Any]] = []
+ for use in uses {
+ let id = use["id"] as? String ?? ""
+ let name = use["name"] as? String ?? ""
+ let input = use["input"] as? [String: Any] ?? [:]
+ let (result, didAct) = await run(tool: name, input: input)
+ if didAct { acted = true }
+ results.append(["type": "tool_result", "tool_use_id": id, "content": result])
+ }
+ messages.append(["role": "user", "content": results])
+ spoken = ""
+ }
+
+ let text = VoiceQuery.spokenText(spoken)
+ return Reply(text: text, acted: acted, failed: text.isEmpty && !acted)
+ }
+
+ /// Keeps the last exchanges, never cutting between a tool call and its result.
+ private func trimHistory() {
+ guard messages.count > 24 else { return }
+ var cut = messages.count - 24
+ // Start on a plain user message (a string), not on tool results.
+ while cut < messages.count,
+ !(messages[cut]["role"] as? String == "user" && messages[cut]["content"] is String) {
+ cut += 1
+ }
+ messages.removeFirst(min(cut, messages.count - 1))
+ }
+
+ // MARK: Prompt
+
+ private static func systemPrompt() -> String {
+ let fr = VoiceSettings.language == "fr"
+ let today = Date().formatted(.dateTime.weekday(.wide).day().month(.wide).year().hour().minute()
+ .locale(Locale(identifier: fr ? "fr_FR" : "en_US")))
+ let state = AppState.shared
+ let active = ([state.mainPillId] + Array(state.activeIntegrations))
+ .compactMap { PillCatalog.definition(for: $0)?.name }
+ let available = PillCatalog.available.map(\.name).joined(separator: ", ")
+ let language = fr ? "French" : "English"
+ return """
+ You are Coucou, a voice assistant living in the notch of the user's Mac, a bit like Jarvis: \
+ quick, warm, a little playful. It is \(today).
+ Everything you write is read aloud: answer in \(language) (the user may speak French or English), \
+ in one or two short sentences, no markdown, no lists, no emojis, no links.
+ Act with the tools instead of describing: pills, music, service_info for the user's own data \
+ (Stripe sales, GitHub, Vercel, Resend, n8n, Notion, Cal.com, agent sessions like Claude Code, \
+ plan usage, music, weather), open_app, prepare_email, and web_search for anything current \
+ (news, scores, prices, facts you are not sure of). Never invent the user's data.
+ Email: gather the recipient, the subject and what to say (write the text yourself from what \
+ the user wants to say, in the language they used), asking for what is missing one short \
+ question at a time, then call prepare_email. The email is never sent by you: the user checks \
+ the card and clicks Send. If they want an attachment and haven't named the file, set \
+ wants_attachment and tell them to drop the file on the notch.
+ When you need an answer from the user, end with one short question. Otherwise never end \
+ with a question.
+ Pills in the notch (at most 4 besides the main one). Active: \(active.joined(separator: ", ")). \
+ Available: \(available).
+ """
+ }
+
+ // MARK: Tools
+
+ private static let tools: [[String: Any]] = [
+ [
+ "name": "pills",
+ "description": "Add, remove, set as main or list the pills shown in the notch. At most 4 pills besides the main one: remove one first if needed.",
+ "input_schema": [
+ "type": "object",
+ "properties": [
+ "action": ["type": "string", "enum": ["add", "remove", "set_main", "list"]],
+ "names": ["type": "array", "items": ["type": "string"],
+ "description": "Pill names as listed (e.g. Stripe, GitHub, Cursor). Empty for list."],
+ ],
+ "required": ["action"],
+ ],
+ ],
+ [
+ "name": "music",
+ "description": "Control Apple Music or Spotify: play, pause, next, previous, volume, play a song/artist search or a playlist.",
+ "input_schema": [
+ "type": "object",
+ "properties": [
+ "action": ["type": "string", "enum": ["play", "pause", "next", "previous", "volume_up",
+ "volume_down", "set_volume", "play_search", "play_playlist"]],
+ "query": ["type": "string", "description": "Song, artist or playlist name for play_search / play_playlist."],
+ "volume": ["type": "integer", "description": "0-100 for set_volume."],
+ ],
+ "required": ["action"],
+ ],
+ ],
+ [
+ "name": "service_info",
+ "description": "Read the user's real data that Coucou already has: Stripe sales and balance, GitHub stars/PRs/CI, Vercel deployments, Resend emails, n8n runs, Notion pages, Cal.com bookings, agent sessions (Claude Code, Codex…), Claude or Codex plan usage, music now playing, active pills, weather today or tomorrow.",
+ "input_schema": [
+ "type": "object",
+ "properties": [
+ "topic": ["type": "string", "enum": VoiceTopic.allCases.map(\.rawValue)],
+ ],
+ "required": ["topic"],
+ ],
+ ],
+ [
+ "name": "open_app",
+ "description": "Open an app on the Mac by name (Figma, Safari, VS Code, Notion…).",
+ "input_schema": [
+ "type": "object",
+ "properties": ["name": ["type": "string"]],
+ "required": ["name"],
+ ],
+ ],
+ [
+ "name": "prepare_email",
+ "description": "Open the email card in the notch, filled in, for the user to check and send with a click (it is never sent automatically). The recipient is a contact name or an email address.",
+ "input_schema": [
+ "type": "object",
+ "properties": [
+ "recipient": ["type": "string"],
+ "subject": ["type": "string"],
+ "body": ["type": "string", "description": "The full email text, written for the user."],
+ "attachment_file": ["type": "string", "description": "File name to attach if the user named one (searched in Downloads, Desktop, Documents, Pictures)."],
+ "wants_attachment": ["type": "boolean", "description": "True if the user wants to attach a file without naming it: they drop it on the notch."],
+ ],
+ "required": ["recipient", "subject", "body"],
+ ],
+ ],
+ ["type": "web_search_20250305", "name": "web_search", "max_uses": 3],
+ ]
+
+ // MARK: Running a tool
+
+ /// Returns the text sent back to Claude, and whether something changed.
+ private func run(tool: String, input: [String: Any]) async -> (String, Bool) {
+ let runner = VoiceActionRunner.shared
+ let pills = PillCatalog.available
+ runner.commandLocale = VoiceSettings.answerLocale
+ func perform(_ intent: VoiceIntent) async -> VoiceActionResult {
+ let r = await runner.run(intent, availablePills: pills)
+ // The runner may ask its own follow-up (4-pill limit…): Claude asks instead.
+ runner.pendingQuestion = nil
+ return r
+ }
+
+ switch tool {
+ case "pills":
+ let action = input["action"] as? String ?? ""
+ let names = input["names"] as? [String] ?? []
+ if action == "list" {
+ return (await LiveVoiceInfo.shared.answer(.pills, locale: VoiceSettings.answerLocale), false)
+ }
+ var lines: [String] = []
+ var acted = false
+ for name in names {
+ guard let id = EntityResolver.resolve(name, from: pills) else {
+ lines.append("\(name): unknown pill"); continue
+ }
+ let intent: VoiceIntent
+ switch action {
+ case "add": intent = .pillAdd(id: id)
+ case "remove": intent = .pillRemove(id: id)
+ case "set_main": intent = .pillSetMain(id: id)
+ default: lines.append("unknown action \(action)"); continue
+ }
+ let r = await perform(intent)
+ if case .question = r.outcome {
+ lines.append("\(name): the notch is full (4 pills). Ask which pill to remove, then remove it and add \(name).")
+ } else {
+ if r.outcome == .success { acted = true }
+ lines.append("\(name): \(r.message)")
+ }
+ }
+ return (lines.isEmpty ? "nothing done" : lines.joined(separator: "\n"), acted)
+
+ case "music":
+ let action = input["action"] as? String ?? ""
+ let query = (input["query"] as? String ?? "").trimmingCharacters(in: .whitespaces)
+ let intent: VoiceIntent?
+ switch action {
+ case "play": intent = .musicPlay(target: nil)
+ case "pause": intent = .musicPause
+ case "next": intent = .musicNext
+ case "previous": intent = .musicPrevious
+ case "volume_up": intent = .musicVolumeUp
+ case "volume_down": intent = .musicVolumeDown
+ case "set_volume": intent = .musicSetVolume(max(0, min(100, input["volume"] as? Int ?? 50)))
+ case "play_search": intent = query.isEmpty ? nil : .musicPlaySearch(name: query)
+ case "play_playlist": intent = query.isEmpty ? nil : .musicPlayPlaylist(name: query)
+ default: intent = nil
+ }
+ guard let intent else { return ("missing query or unknown action", false) }
+ let r = await perform(intent)
+ return (r.message, r.outcome == .success)
+
+ case "service_info":
+ guard let topic = VoiceTopic(rawValue: input["topic"] as? String ?? "") else {
+ return ("unknown topic", false)
+ }
+ let text = await LiveVoiceInfo.shared.answer(topic, locale: VoiceSettings.answerLocale)
+ return (text.isEmpty ? "no data" : text, false)
+
+ case "open_app":
+ let name = input["name"] as? String ?? ""
+ let r = LiveVoiceInfo.shared.openApp(name, locale: VoiceSettings.answerLocale)
+ return (r.message, r.outcome == .success)
+
+ case "prepare_email":
+ return await prepareEmail(input)
+
+ default:
+ return ("unknown tool \(tool)", false)
+ }
+ }
+
+ private func prepareEmail(_ input: [String: Any]) async -> (String, Bool) {
+ let info = LiveVoiceInfo.shared
+ let recipient = (input["recipient"] as? String ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
+ let subject = input["subject"] as? String ?? ""
+ let body = input["body"] as? String ?? ""
+ guard !recipient.isEmpty else { return ("no recipient: ask who it is for", false) }
+ guard let address = await info.resolveEmail(recipient) else {
+ return ("No email address found for \(recipient) in the contacts. Ask the user to spell the address.", false)
+ }
+ var file: URL? = nil
+ var note = ""
+ if let name = input["attachment_file"] as? String, !name.trimmingCharacters(in: .whitespaces).isEmpty {
+ file = info.findFile(name, folder: nil)
+ if file == nil { note = " The file \(name) was not found: the user can drop it on the notch." }
+ }
+ info.showMailCard(to: address, subject: subject, body: body, file: file)
+ let attach = (input["wants_attachment"] as? Bool ?? false) && file == nil
+ ? " The user wants an attachment: tell them to drop the file on the notch." : ""
+ let with = file.map { " with \($0.lastPathComponent) attached" } ?? ""
+ return ("Email card shown to \(address)\(with). Nothing is sent until the user clicks Send.\(note)\(attach)", true)
+ }
+}
+#endif
diff --git a/NotchBuddy/Sources/App/Voice/VoiceSettings.swift b/NotchBuddy/Sources/App/Voice/VoiceSettings.swift
index fd09b344..f46e88af 100644
--- a/NotchBuddy/Sources/App/Voice/VoiceSettings.swift
+++ b/NotchBuddy/Sources/App/Voice/VoiceSettings.swift
@@ -77,12 +77,15 @@ enum VoiceSettings {
return v == 0 ? fallback : min(1.6, max(0.8, v))
}
- /// Questions answered with a web search by Claude, with the user's own Anthropic API
- /// key. Off by default: the question leaves the Mac only once this is turned on.
- static var webSearchEnabled: Bool {
- get { UserDefaults.standard.bool(forKey: "voiceWebSearchEnabled") }
- set { UserDefaults.standard.set(newValue, forKey: "voiceWebSearchEnabled") }
+ /// Claude is Coucou's brain (the user's Anthropic API key): what I say after
+ /// "OK Coucou" goes to Claude, which acts with Coucou's tools and searches the web.
+ /// Off by default: my words leave the Mac only once this is turned on.
+ static var claudeBrainEnabled: Bool {
+ get { UserDefaults.standard.bool(forKey: "voiceClaudeBrain") }
+ set { UserDefaults.standard.set(newValue, forKey: "voiceClaudeBrain") }
}
+ /// The older web-search-only path follows the same switch.
+ static var webSearchEnabled: Bool { claudeBrainEnabled }
/// Weather by voice (Open-Meteo, no key). Off by default: network only when the user
/// turned it on and set a city.