Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

feat/voice-jarvis: Jarvis-style fast conversation mode

Louis Railleb6b40d83be78
diff --git a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
index 6cece4ea..9b4fbe3a 100644
--- a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
+++ b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj
@@ -70,6 +70,7 @@
2F36E3C6C073E968A682AFA4 /* NotionPoller.swift in Sources */ = {isa = PBXBuildFile; fileRef = BE03E03D1804B0B92C5C2DAD /* NotionPoller.swift */; };
326971DDA23F331116E5A628 /* VoiceBrain.swift in Sources */ = {isa = PBXBuildFile; fileRef = C563E55884A7750B40EC9A01 /* VoiceBrain.swift */; };
3451DB0668FAEEF09503D808 /* WakePhrase.swift in Sources */ = {isa = PBXBuildFile; fileRef = A12EC8F605BBC37F8223D7AA /* WakePhrase.swift */; };
+ 36EBFB9A047DEBAC43E5D0CC /* VoiceCaptionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3A8FB03801C49690C3789EFD /* VoiceCaptionManager.swift */; };
3756B7E1A55D9CEC8C4D34A4 /* IslandStateMachine.swift in Sources */ = {isa = PBXBuildFile; fileRef = D8212018F09EA1E96FBD63CD /* IslandStateMachine.swift */; };
38EEB2D3A6EC538958D92575 /* GitHubPulse.swift in Sources */ = {isa = PBXBuildFile; fileRef = F8D3AD61802A32C4833F3EDA /* GitHubPulse.swift */; };
390486F7CDC01C1BCDBB27F9 /* VoiceActionRunner.swift in Sources */ = {isa = PBXBuildFile; fileRef = A2737EC4885ADB8FDDA1C610 /* VoiceActionRunner.swift */; };
@@ -94,6 +95,7 @@
4F1E0B3C838DA3CD6F864BC0 /* CalcomPoller.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3104222B3F17BA5E8DEA5277 /* CalcomPoller.swift */; };
4F4176856B544D8C620B6204 /* ConversationContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A67DA3B8EEF063C44E07F7E /* ConversationContext.swift */; };
502F2B0D607C511DD3B836B3 /* VoiceSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = D4543CB771E0B723783F529E /* VoiceSettings.swift */; };
+ 525FB188F09F68D26A9D587D /* VoiceCaptionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3A8FB03801C49690C3789EFD /* VoiceCaptionManager.swift */; };
52F0DF1FBCB522951910390E /* InstructionRunner.swift in Sources */ = {isa = PBXBuildFile; fileRef = FADC1FE5696B1075932D0B7C /* InstructionRunner.swift */; };
530443F46F896F037A56F663 /* HotKeyCenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1218B86E249691AF44F819AC /* HotKeyCenter.swift */; };
535EC818E1DAF0FA588545D8 /* QuestionPayload.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6C6993B34A0FFF8E3D454ADC /* QuestionPayload.swift */; };
@@ -380,6 +382,7 @@
37564F5F041B38C976DF3DEF /* PillColors.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PillColors.swift; sourceTree = "<group>"; };
38BEAB5F20A13E5806A42EC9 /* ApprovalCard.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ApprovalCard.swift; sourceTree = "<group>"; };
3A27178AA6007F7B56901750 /* SessionDetailView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SessionDetailView.swift; sourceTree = "<group>"; };
+ 3A8FB03801C49690C3789EFD /* VoiceCaptionManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceCaptionManager.swift; sourceTree = "<group>"; };
3B238F6E7B20EFDED5CD3E8E /* ClaudeSettingsFile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClaudeSettingsFile.swift; sourceTree = "<group>"; };
444604C7E2C530B664849EF9 /* ClaudeResponseText.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ClaudeResponseText.swift; sourceTree = "<group>"; };
49E39852175E81C9632A59C9 /* KitPreviewView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KitPreviewView.swift; sourceTree = "<group>"; };
@@ -727,6 +730,7 @@
A2737EC4885ADB8FDDA1C610 /* VoiceActionRunner.swift */,
A87F39805C78722F614295B4 /* VoiceAudio.swift */,
C563E55884A7750B40EC9A01 /* VoiceBrain.swift */,
+ 3A8FB03801C49690C3789EFD /* VoiceCaptionManager.swift */,
B328028D18F590D6680B8A2B /* VoiceEngine.swift */,
D1A7C62058BB0C9D0DF58DCB /* VoiceIntent.swift */,
22E03C8E267399FA25C75877 /* VoiceIslandView.swift */,
@@ -1037,6 +1041,7 @@
390486F7CDC01C1BCDBB27F9 /* VoiceActionRunner.swift in Sources */,
88A8F2E4E754FAFABF6B1960 /* VoiceAudio.swift in Sources */,
326971DDA23F331116E5A628 /* VoiceBrain.swift in Sources */,
+ 525FB188F09F68D26A9D587D /* VoiceCaptionManager.swift in Sources */,
BDEFF4223DA949E2B8AB85A7 /* VoiceEngine.swift in Sources */,
9C11C226915DBC97B5CD7953 /* VoiceIntent.swift in Sources */,
9088C7BADF166685E471D248 /* VoiceIslandView.swift in Sources */,
@@ -1223,6 +1228,7 @@
BF3F437B1FE9C246E9014F76 /* VoiceActionRunner.swift in Sources */,
64FE8055CCAD30B44FE398E7 /* VoiceAudio.swift in Sources */,
273B404F55739D661D9489D8 /* VoiceBrain.swift in Sources */,
+ 36EBFB9A047DEBAC43E5D0CC /* VoiceCaptionManager.swift in Sources */,
80B0B3C777A9DC8C82952FD2 /* VoiceEngine.swift in Sources */,
15D69752806E1D1D589B20A7 /* VoiceIntent.swift in Sources */,
C3BA3FA2A4025194FC088C5A /* VoiceIslandView.swift in Sources */,
diff --git a/NotchBuddy/Sources/App/IslandWindowController.swift b/NotchBuddy/Sources/App/IslandWindowController.swift
index 6e5c45db..2f83f7e9 100644
--- a/NotchBuddy/Sources/App/IslandWindowController.swift
+++ b/NotchBuddy/Sources/App/IslandWindowController.swift
@@ -286,7 +286,12 @@ final class IslandWindowController: NSWindowController {
self.expand(to: .greeting)
case .listening:
- self.expand(to: .listening)
+ // Island stays compact; caption panel handles display.
+ #if !APPSTORE
+ let screen = IslandWindowController.islandScreen()
+ VoiceCaptionManager.shared.show(on: screen, notchHeight: AppState.shared.notchHeight)
+ VoiceBrain.shared.prewarmSession()
+ #endif
}
}
@@ -1333,6 +1338,7 @@ extension IslandWindowController {
/// Run the intent derived from `transcript`, show VoiceResultView, then continue conversation or collapse.
@MainActor
func handleVoiceCommand(_ transcript: String) async {
+ let t0 = Date()
let pills = PillCatalog.available
let runner = VoiceActionRunner.shared
@@ -1374,8 +1380,9 @@ extension IslandWindowController {
} else {
NotificationCenter.default.post(name: .botDizzy, object: nil)
}
+ VoiceCaptionManager.shared.setUserLine(transcript)
+ VoiceCaptionManager.shared.appendResponse(result.message)
AppState.shared.voiceResult = result
- expand(to: .voiceResult)
speakAndContinueConversation(result)
return
}
@@ -1421,8 +1428,9 @@ extension IslandWindowController {
} else {
NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
}
+ VoiceCaptionManager.shared.setUserLine(transcript)
+ VoiceCaptionManager.shared.appendResponse(combined.message)
AppState.shared.voiceResult = combined
- expand(to: .voiceResult)
speakAndContinueConversation(combined)
return
}
@@ -1440,40 +1448,75 @@ extension IslandWindowController {
}
// ── Single command ─────────────────────────────────────────────────────────
+ let locale = VoiceEngine.shared.speechLocale
+ let tParseEnd = Date()
+ let parseMs = Int(tParseEnd.timeIntervalSince(t0) * 1000)
+
+ // Update caption user line immediately
+ VoiceCaptionManager.shared.setUserLine(transcript)
+
var result = await runner.run(intent, availablePills: pills, rawTranscript: transcript)
var effectiveIntent = intent
VoiceTranscriptHistory.shared.record(transcript: transcript, intent: intent, origin: transcriptOrigin)
- // If unknown, try VoiceBrain (macOS 26 + Apple Intelligence).
- if case .unknown = intent,
- let brain = await VoiceBrain.shared.resolve(transcript, pills: pills) {
- if !brain.intents.isEmpty {
- // Run all brain intents: removals before additions (same rule as parseMultiAction).
- var sorted = brain.intents
- sorted.sort { a, b in
- let ra: Bool = { switch a { case .pillRemove, .pillRemoveMultiple: return true; default: return false } }()
- let rb: Bool = { switch b { case .pillRemove, .pillRemoveMultiple: return true; default: return false } }()
- return ra && !rb
- }
- var parts: [String] = []
- var anyFailure = false
- for bi in sorted {
- let r = await runner.run(bi, availablePills: pills, rawTranscript: transcript)
- parts.append(r.message)
- if case .failure = r.outcome { anyFailure = true }
- if case .question = r.outcome { anyFailure = true }
- if case .success = r.outcome {
- effectiveIntent = bi
- VoiceTranscriptHistory.shared.record(transcript: transcript, intent: bi, origin: .brain)
+ let tActionEnd = Date()
+ let actionMs = Int(tActionEnd.timeIntervalSince(tParseEnd) * 1000)
+
+ // If unknown, try VoiceBrain (macOS 26 + Apple Intelligence) with streaming TTS.
+ if case .unknown = intent {
+ let tBrain0 = Date()
+ let brainWarm = VoiceBrain.shared.isSessionReady
+ var brainUsed = false
+
+ let brain = await VoiceBrain.shared.resolveWithStreaming(
+ transcript, pills: pills
+ ) { sentence in
+ VoiceSpeaker.shared.enqueue(sentence, locale: locale)
+ VoiceCaptionManager.shared.appendResponse(sentence)
+ }
+
+ let brainMs = Int(Date().timeIntervalSince(tBrain0) * 1000)
+ appendAppLog("nb.log",
+ "[Voice] turn: parse=\(parseMs)ms action=\(actionMs)ms brain=\(brainMs)ms (\(brainWarm ? "warm" : "cold"))")
+
+ if let brain {
+ if !brain.intents.isEmpty {
+ // Run brain intents; model already voiced confirmation via streaming TTS.
+ var sorted = brain.intents
+ sorted.sort { a, b in
+ let ra: Bool = { switch a { case .pillRemove, .pillRemoveMultiple: return true; default: return false } }()
+ let rb: Bool = { switch b { case .pillRemove, .pillRemoveMultiple: return true; default: return false } }()
+ return ra && !rb
}
+ for bi in sorted {
+ let r = await runner.run(bi, availablePills: pills, rawTranscript: transcript)
+ if case .success = r.outcome {
+ effectiveIntent = bi
+ VoiceTranscriptHistory.shared.record(transcript: transcript, intent: bi, origin: .brain)
+ }
+ }
+ result = VoiceActionResult(outcome: .success, message: brain.text)
+ brainUsed = true
+ } else if !brain.text.isEmpty {
+ result = VoiceActionResult(outcome: .success, message: brain.text)
+ brainUsed = true
}
- result = VoiceActionResult(
- outcome: anyFailure ? .failure : .success,
- message: parts.joined(separator: " · ")
- )
- } else if !brain.text.isEmpty {
- result = VoiceActionResult(outcome: .success, message: brain.text)
}
+
+ if brainUsed {
+ // TTS sentences already enqueued via streaming. Enter conversation and
+ // continue once the speaker queue drains.
+ NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
+ conversationContext.update(effectiveIntent)
+ consecutiveFailures = 0
+ AppState.shared.voiceResult = result
+ _enterConversationAfterStreamedSpeech()
+ return
+ }
+
+ appendAppLog("nb.log", "[Voice] turn: parse=\(parseMs)ms action=\(actionMs)ms brain=\(brainMs)ms (\(brainWarm ? "warm" : "cold")) — no result")
+ } else {
+ appendAppLog("nb.log", "[Voice] turn: parse=\(parseMs)ms action=\(actionMs)ms (parser)")
}
// Mid-conversation, a phrase with no command in it (talking to someone else,
@@ -1511,16 +1554,16 @@ extension IslandWindowController {
return
}
- // Show result view
+ // Update caption with result; island stays compact (no expand).
+ VoiceCaptionManager.shared.appendResponse(result.message)
AppState.shared.voiceResult = result
- expand(to: .voiceResult)
if case .question = result.outcome {
// Speak the question aloud, then re-listen once speech finishes.
// Give 5 s initial silence so the user has time to read/hear the question.
let speaker = VoiceSpeaker.shared
if VoiceSettings.speakEnabled {
- speaker.speak(result.message, locale: VoiceEngine.shared.speechLocale)
+ speaker.speak(result.message, locale: locale)
speaker.onDidFinish = { [weak self] in
Task { @MainActor in
guard self != nil else { return }
@@ -1575,15 +1618,36 @@ extension IslandWindowController {
speaker.speak(result.message, locale: locale)
speaker.onDidFinish = { [weak self] in
Task { @MainActor in
- self?.scheduleConversationContinue(delay: 0.3)
+ self?.scheduleConversationContinue(delay: 0.15)
}
}
} else {
- // No speech: continue after showing result for 1.5 s
+ // No speech: continue after 1.5 s
scheduleConversationContinue(delay: 1.5)
}
}
+ /// Enter/stay in conversation mode after brain streaming TTS queues drain.
+ @MainActor
+ private func _enterConversationAfterStreamedSpeech() {
+ if !isInConversation {
+ VoiceBrain.shared.beginConversation()
+ consecutiveFailures = 0
+ hasSpokenPasCompris = false
+ }
+ isInConversation = true
+ let speaker = VoiceSpeaker.shared
+ if speaker.isSpeaking {
+ speaker.onDidFinish = { [weak self] in
+ Task { @MainActor in
+ self?.scheduleConversationContinue(delay: 0.15)
+ }
+ }
+ } else {
+ scheduleConversationContinue(delay: 0.15)
+ }
+ }
+
/// Schedule the next conversation listening turn.
@MainActor
private func scheduleConversationContinue(delay: TimeInterval) {
@@ -1612,6 +1676,7 @@ extension IslandWindowController {
voiceResultWork = nil
VoiceEngine.shared.endConversation()
VoiceBrain.shared.endConversation()
+ VoiceCaptionManager.shared.endConversation()
if speaking {
// Brief farewell emote
NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
@@ -1622,6 +1687,7 @@ extension IslandWindowController {
@MainActor
private func scheduleVoiceDismiss(delay: TimeInterval) {
voiceResultWork?.cancel()
+ VoiceCaptionManager.shared.hide(after: delay)
let item = DispatchWorkItem { [weak self] in
guard let self else { return }
AppState.shared.voiceResult = nil
diff --git a/NotchBuddy/Sources/App/SettingsView.swift b/NotchBuddy/Sources/App/SettingsView.swift
index 46517327..90a03838 100644
--- a/NotchBuddy/Sources/App/SettingsView.swift
+++ b/NotchBuddy/Sources/App/SettingsView.swift
@@ -1258,7 +1258,8 @@ struct SettingsView: View {
#if !APPSTORE
@State private var voicePermissionsGranted: Bool = false
@ObservedObject private var voiceEngine = VoiceEngine.shared
- @AppStorage("voiceSpeakEnabled") private var speakEnabled: Bool = true
+ @AppStorage("voiceSpeakEnabled") private var speakEnabled: Bool = true
+ @AppStorage("voiceCaptionEnabled") private var captionEnabled: Bool = true
@ViewBuilder private var voiceSection: some View {
GroupBox(String(localized: "«\u{202F}OK Coucou\u{202F}» — voice wake word")) {
@@ -1268,6 +1269,8 @@ struct SettingsView: View {
Toggle(String(localized: "voice.setting-speak"), isOn: $speakEnabled)
+ Toggle(String(localized: "Sous-titres"), isOn: $captionEnabled)
+
Text("When enabled, Coucou listens for the wake word «\u{202F}OK Coucou\u{202F}». Speech recognition runs entirely on-device — no audio or transcript leaves your Mac.")
.font(.system(size: 11))
.foregroundColor(.secondary)
diff --git a/NotchBuddy/Sources/App/Voice/IntentParser.swift b/NotchBuddy/Sources/App/Voice/IntentParser.swift
index 43ebdbce..2047cb5e 100644
--- a/NotchBuddy/Sources/App/Voice/IntentParser.swift
+++ b/NotchBuddy/Sources/App/Voice/IntentParser.swift
@@ -623,6 +623,7 @@ enum IntentParser {
private static let pausePrefixes: [[String]] = [
["pause"], ["stop"], ["stoppe"], ["coupe"],
+ ["stop", "la", "musique"], ["stop", "la", "chanson"],
["mets", "en", "pause"], ["met", "en", "pause"],
["arrete", "la", "musique"], ["arrete", "la", "chanson"],
["arrete", "la", "lecture"],
@@ -691,7 +692,7 @@ enum IntentParser {
// Bare music verbs (single word → musicPlay)
private static let bareMusicVerbs: Set<String> = [
- "joue", "lance", "play", "start", "demarre", "balance",
+ "joue", "lance", "lecture", "play", "start", "demarre", "balance",
"reprends", "resume",
]
diff --git a/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift b/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift
index 1b6a927b..20756c58 100644
--- a/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift
+++ b/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift
@@ -6,9 +6,9 @@ import Foundation
// Decides silence duration before ending a command turn.
// Standalone compilable (used by test scripts via swiftc).
enum TurnEndPolicy {
- static let baseSilence: TimeInterval = 1.5
- static let extendedSilence: TimeInterval = 3.0
- static let maxTurnTime: TimeInterval = 20.0
+ static let baseSilence: TimeInterval = 0.8
+ static let extendedSilence: TimeInterval = 1.6
+ static let maxTurnTime: TimeInterval = 15.0
/// Returns the silence timeout given the current *normalised* partial transcript.
/// Call with `IntentParser.normalise(rawTranscript)`.
diff --git a/NotchBuddy/Sources/App/Voice/VoiceBrain.swift b/NotchBuddy/Sources/App/Voice/VoiceBrain.swift
index df34f27b..4d910827 100644
--- a/NotchBuddy/Sources/App/Voice/VoiceBrain.swift
+++ b/NotchBuddy/Sources/App/Voice/VoiceBrain.swift
@@ -50,9 +50,18 @@ final class VoiceBrain {
sessionBox = nil
}
+ /// Ensures a session exists and is warmed up. Call on wake detection.
+ func prewarmSession() {
+ if sessionBox == nil { sessionBox = VoiceBrain._makeSession() }
+ // prewarm already fired in _makeSession via Task.detached
+ }
+
+ /// True when a session is already created (warm), false on first call (cold).
+ var isSessionReady: Bool { sessionBox != nil }
+
// MARK: - Intent resolution
- /// Try to resolve `transcript` using the language model.
+ /// Try to resolve `transcript` using the language model (non-streaming).
/// Returns nil if the model is unavailable or if the model cannot map to any intent.
func resolve(_ transcript: String, pills: [PillDefinition]) async -> BrainResult? {
// Lazily create session on first call so the model is available
@@ -61,6 +70,19 @@ final class VoiceBrain {
return await VoiceBrain._resolve(transcript, pills: pills, sessionBox: sessionBox)
}
+ /// Streaming variant: calls onSentence for each complete sentence as it arrives.
+ /// Returns the full BrainResult when the stream ends. 8s total timeout.
+ func resolveWithStreaming(
+ _ transcript: String,
+ pills: [PillDefinition],
+ onSentence: @escaping @MainActor (String) -> Void
+ ) async -> BrainResult? {
+ if sessionBox == nil { sessionBox = VoiceBrain._makeSession() }
+ return await VoiceBrain._resolveWithStreaming(
+ transcript, pills: pills, sessionBox: sessionBox, onSentence: onSentence
+ )
+ }
+
// MARK: - Static impl helpers
static func _checkStatus() -> LocalModelStatus {
@@ -96,12 +118,17 @@ final class VoiceBrain {
instructions: """
Tu es Coucou, un assistant dans le notch du MacBook.
Réponds toujours dans la langue de l'utilisateur.
- Réponds avec une phrase courte et directe.
+ Réponds avec 1 à 2 phrases maximum. Sois direct et concis.
Utilise les outils disponibles pour exécuter des commandes sur les pills et la musique.
Pour les noms de pilules, utilise le nom exact fourni par l'utilisateur.
"""
)
- return SessionContainer(session: session, collector: collector)
+ let container = SessionContainer(session: session, collector: collector)
+ // Prewarm in background — warms the attention cache without blocking the caller.
+ Task.detached {
+ session.prewarm()
+ }
+ return container
}
#endif
return nil
@@ -140,6 +167,71 @@ final class VoiceBrain {
#endif
return nil
}
+
+ static func _resolveWithStreaming(
+ _ transcript: String,
+ pills: [PillDefinition],
+ sessionBox: AnyObject?,
+ onSentence: @escaping @MainActor (String) -> Void
+ ) async -> BrainResult? {
+ #if canImport(FoundationModels)
+ if #available(macOS 26, *) {
+ guard SystemLanguageModel.default.availability == .available else { return nil }
+ guard let container = sessionBox as? SessionContainer else { return nil }
+ let collector = container.collector
+ collector.reset()
+
+ let pillNames = pills.map { $0.name }.joined(separator: ", ")
+ let prompt = """
+ User said: "\(transcript)"
+ Available pill names: \(pillNames)
+ Use a tool if this is a command. Otherwise answer naturally in the user's language.
+ """
+
+ do {
+ let fullText = try await withBrainTimeout(seconds: 8) { () -> String in
+ let stream = container.session.streamResponse(to: prompt)
+ var sentUpTo = 0
+ var accumulated = ""
+
+ for try await snapshot in stream {
+ accumulated = snapshot.content
+ guard sentUpTo < accumulated.count else { continue }
+
+ let startIdx = accumulated.index(accumulated.startIndex, offsetBy: sentUpTo)
+ let slice = accumulated[startIdx...]
+ let sentenceEnders: Set<Character> = [".", "!", "?", "\n"]
+
+ if let boundary = slice.lastIndex(where: { sentenceEnders.contains($0) }) {
+ let afterBoundary = accumulated.index(after: boundary)
+ let sentence = String(accumulated[startIdx..<afterBoundary])
+ .trimmingCharacters(in: .whitespaces)
+ if !sentence.isEmpty {
+ await MainActor.run { onSentence(sentence) }
+ }
+ sentUpTo = accumulated.distance(from: accumulated.startIndex, to: afterBoundary)
+ }
+ }
+
+ // Flush any trailing text without sentence terminator
+ if sentUpTo < accumulated.count {
+ let startIdx = accumulated.index(accumulated.startIndex, offsetBy: sentUpTo)
+ let remaining = String(accumulated[startIdx...]).trimmingCharacters(in: .whitespaces)
+ if !remaining.isEmpty {
+ await MainActor.run { onSentence(remaining) }
+ }
+ }
+
+ return accumulated
+ }
+ return BrainResult(intents: collector.intents, text: fullText)
+ } catch {
+ return nil
+ }
+ }
+ #endif
+ return nil
+ }
}
// MARK: - Timeout helper (file-private)
diff --git a/NotchBuddy/Sources/App/Voice/VoiceCaptionManager.swift b/NotchBuddy/Sources/App/Voice/VoiceCaptionManager.swift
new file mode 100644
index 00000000..65ec7a24
--- /dev/null
+++ b/NotchBuddy/Sources/App/Voice/VoiceCaptionManager.swift
@@ -0,0 +1,174 @@
+#if !APPSTORE
+import AppKit
+import SwiftUI
+
+// MARK: - VoiceCaptionState
+
+/// Holds the two text lines shown in the caption capsule.
+/// Mutated only from @MainActor (VoiceCaptionManager).
+final class VoiceCaptionState: ObservableObject {
+ @Published var userLine: String = ""
+ @Published var responseLine: String = ""
+ @Published var isVisible: Bool = false
+}
+
+// MARK: - VoiceCaptionManager
+//
+// Manages a borderless, non-activating NSPanel positioned below the notch center.
+// Shows a compact capsule with:
+// – user transcript (gray, top line)
+// – AI response (white, bottom line, streams in)
+// Fades in/out in 0.2s. Auto-hides 2s after endConversation().
+//
+// Usage:
+// VoiceCaptionManager.shared.show(on: screen, notchHeight: 36)
+// VoiceCaptionManager.shared.setUserLine("Ajoute GitHub")
+// VoiceCaptionManager.shared.appendResponse("D'accord.")
+// VoiceCaptionManager.shared.endConversation()
+
+@MainActor
+final class VoiceCaptionManager {
+ static let shared = VoiceCaptionManager()
+
+ let state = VoiceCaptionState()
+
+ private var panel: NSPanel?
+ private var hideTask: Task<Void, Never>?
+
+ private let captionWidth: CGFloat = 360
+ private let captionHeight: CGFloat = 52
+ private let notchGap: CGFloat = 6
+
+ private init() {}
+
+ // MARK: - Show / hide
+
+ func show(on screen: NSScreen, notchHeight: CGFloat) {
+ guard VoiceSettings.captionEnabled else { return }
+ hideTask?.cancel()
+ hideTask = nil
+
+ if panel == nil { _buildPanel() }
+ _position(on: screen, notchHeight: notchHeight)
+
+ guard let p = panel else { return }
+ if !p.isVisible { p.alphaValue = 0; p.orderFrontRegardless() }
+
+ state.isVisible = true
+ NSAnimationContext.runAnimationGroup { ctx in
+ ctx.duration = 0.2
+ p.animator().alphaValue = 1
+ }
+ }
+
+ func hide(after delay: TimeInterval = 0) {
+ hideTask?.cancel()
+ hideTask = Task { [weak self] in
+ guard let self else { return }
+ if delay > 0 {
+ try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
+ }
+ guard !Task.isCancelled else { return }
+ await MainActor.run { self._fadeOut() }
+ }
+ }
+
+ // MARK: - Content updates
+
+ func setUserLine(_ text: String) {
+ state.userLine = text
+ state.responseLine = ""
+ }
+
+ func appendResponse(_ chunk: String) {
+ if state.responseLine.isEmpty {
+ state.responseLine = chunk
+ } else {
+ state.responseLine += " " + chunk
+ }
+ }
+
+ func clearResponse() {
+ state.responseLine = ""
+ }
+
+ /// Call when the conversation ends. Panel fades out after 2s.
+ func endConversation() {
+ hide(after: 2.0)
+ }
+
+ // MARK: - Private
+
+ private func _buildPanel() {
+ let view = NSHostingView(rootView: VoiceCaptionView(state: state))
+ view.frame = NSRect(x: 0, y: 0, width: captionWidth, height: captionHeight)
+
+ let p = NSPanel(
+ contentRect: NSRect(x: 0, y: 0, width: captionWidth, height: captionHeight),
+ styleMask: [.borderless, .nonactivatingPanel],
+ backing: .buffered,
+ defer: false
+ )
+ p.isOpaque = false
+ p.backgroundColor = .clear
+ p.level = .statusBar
+ p.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle]
+ p.contentView = view
+ p.alphaValue = 0
+ panel = p
+ }
+
+ private func _position(on screen: NSScreen, notchHeight: CGFloat) {
+ guard let p = panel else { return }
+ let sf = screen.frame
+ let x = sf.midX - captionWidth / 2
+ // AppKit origin is bottom-left; notch is at top of screen
+ let y = sf.maxY - notchHeight - notchGap - captionHeight
+ p.setFrameOrigin(NSPoint(x: x, y: y))
+ }
+
+ private func _fadeOut() {
+ state.isVisible = false
+ guard let p = panel, p.isVisible else { return }
+ NSAnimationContext.runAnimationGroup({ ctx in
+ ctx.duration = 0.2
+ p.animator().alphaValue = 0
+ }, completionHandler: {
+ Task { @MainActor in self.panel?.orderOut(nil) }
+ })
+ }
+}
+
+// MARK: - VoiceCaptionView
+
+struct VoiceCaptionView: View {
+ @ObservedObject var state: VoiceCaptionState
+
+ var body: some View {
+ VStack(alignment: .leading, spacing: 2) {
+ if !state.userLine.isEmpty {
+ Text(state.userLine)
+ .font(.system(size: 12))
+ .foregroundStyle(.secondary)
+ .lineLimit(1)
+ .truncationMode(.tail)
+ }
+ if !state.responseLine.isEmpty {
+ Text(state.responseLine)
+ .font(.system(size: 12))
+ .foregroundStyle(.white)
+ .lineLimit(2)
+ .truncationMode(.tail)
+ }
+ }
+ .padding(.horizontal, 14)
+ .padding(.vertical, 8)
+ .frame(maxWidth: .infinity, alignment: .leading)
+ .background(
+ RoundedRectangle(cornerRadius: 12, style: .continuous)
+ .fill(Color.black.opacity(0.88))
+ )
+ .frame(width: 360)
+ }
+}
+#endif
diff --git a/NotchBuddy/Sources/App/Voice/VoiceSettings.swift b/NotchBuddy/Sources/App/Voice/VoiceSettings.swift
index 350d2b5e..8eaa9beb 100644
--- a/NotchBuddy/Sources/App/Voice/VoiceSettings.swift
+++ b/NotchBuddy/Sources/App/Voice/VoiceSettings.swift
@@ -6,8 +6,9 @@ import Speech
/// Persisted voice feature settings and permission helpers.
enum VoiceSettings {
- static let enabledKey = "voiceEnabled"
+ static let enabledKey = "voiceEnabled"
static let speakEnabledKey = "voiceSpeakEnabled"
+ static let captionEnabledKey = "voiceCaptionEnabled"
static var isEnabled: Bool {
get { UserDefaults.standard.bool(forKey: enabledKey) }
@@ -24,6 +25,16 @@ enum VoiceSettings {
set { UserDefaults.standard.set(newValue, forKey: speakEnabledKey) }
}
+ /// Show caption capsule below notch during voice. Default: on.
+ static var captionEnabled: Bool {
+ get {
+ let d = UserDefaults.standard
+ if d.object(forKey: captionEnabledKey) == nil { return true } // default on
+ return d.bool(forKey: captionEnabledKey)
+ }
+ set { UserDefaults.standard.set(newValue, forKey: captionEnabledKey) }
+ }
+
// MARK: - Permissions
enum PermissionStatus { case granted, denied, undetermined }
diff --git a/NotchBuddy/Sources/App/Voice/VoiceSpeaker.swift b/NotchBuddy/Sources/App/Voice/VoiceSpeaker.swift
index eee1c031..1bcb7c49 100644
--- a/NotchBuddy/Sources/App/Voice/VoiceSpeaker.swift
+++ b/NotchBuddy/Sources/App/Voice/VoiceSpeaker.swift
@@ -3,9 +3,9 @@ import AVFoundation
// MARK: - VoiceSpeaker
//
-// Gives Mochi a voice: wraps AVSpeechSynthesizer with a Mochi-like pitch.
-// Semi-duplex: VoiceEngine pauses recognition while this is speaking.
-// Setting: VoiceSettings.speakEnabled (default on).
+// Gives Mochi a voice: wraps AVSpeechSynthesizer with a sentence queue.
+// Sentence queue: enqueue(text:locale:) adds to queue, playback starts automatically
+// and continues through the queue. onDidFinish fires when the queue drains.
//
// All methods: @MainActor.
@MainActor
@@ -15,8 +15,9 @@ final class VoiceSpeaker: NSObject, AVSpeechSynthesizerDelegate, @unchecked Send
private let synth = AVSpeechSynthesizer()
private(set) var isSpeaking = false
private var currentUtterance: AVSpeechUtterance? = nil
+ private var queue: [(text: String, locale: Locale?)] = []
- /// Called ~300 ms after the utterance finishes. VoiceEngine uses this to resume listening.
+ /// Called ~150 ms after the last queued utterance finishes.
var onDidFinish: (() -> Void)?
override init() {
@@ -24,26 +25,61 @@ final class VoiceSpeaker: NSObject, AVSpeechSynthesizerDelegate, @unchecked Send
synth.delegate = self
}
+ // MARK: - Public API
+
+ /// Clears the queue, stops current speech, and speaks text immediately.
func speak(_ text: String, locale: Locale?) {
guard VoiceSettings.speakEnabled else { return }
+ queue.removeAll()
synth.stopSpeaking(at: .immediate)
isSpeaking = false
- let utt = AVSpeechUtterance(string: text)
- utt.pitchMultiplier = 1.25
- utt.rate = AVSpeechUtteranceDefaultSpeechRate * 1.05
- if let loc = locale {
- let lang = loc.language.languageCode?.identifier ?? ""
- if !lang.isEmpty { utt.voice = AVSpeechSynthesisVoice(language: lang) }
+ currentUtterance = nil
+ _enqueueAndPlay(text: text, locale: locale)
+ }
+
+ /// Adds text to the end of the playback queue. Starts playing if not already.
+ func enqueue(_ text: String, locale: Locale?) {
+ guard VoiceSettings.speakEnabled else { return }
+ if isSpeaking {
+ queue.append((text, locale))
+ } else {
+ _enqueueAndPlay(text: text, locale: locale)
}
- currentUtterance = utt
- isSpeaking = true
- synth.speak(utt)
}
func stop() {
+ queue.removeAll()
guard isSpeaking else { return }
synth.stopSpeaking(at: .immediate)
- // isSpeaking set to false in didCancel/didFinish delegate callbacks
+ }
+
+ // MARK: - Private
+
+ private func _enqueueAndPlay(text: String, locale: Locale?) {
+ let utt = AVSpeechUtterance(string: text)
+ utt.pitchMultiplier = 1.15
+ utt.rate = AVSpeechUtteranceDefaultSpeechRate * 1.1
+ utt.voice = _bestVoice(for: locale)
+ currentUtterance = utt
+ isSpeaking = true
+ synth.speak(utt)
+ }
+
+ /// Picks the highest quality installed voice for the given locale.
+ /// Falls back to .premium → .enhanced → default language voice.
+ private func _bestVoice(for locale: Locale?) -> AVSpeechSynthesisVoice? {
+ guard let loc = locale else { return nil }
+ let lang = loc.language.languageCode?.identifier ?? ""
+ guard !lang.isEmpty else { return nil }
+
+ let voices = AVSpeechSynthesisVoice.speechVoices().filter {
+ $0.language.hasPrefix(lang)
+ }
+ if voices.isEmpty { return AVSpeechSynthesisVoice(language: lang) }
+
+ // Sort by quality descending (.premium > .enhanced > default)
+ let sorted = voices.sorted { $0.quality.rawValue > $1.quality.rawValue }
+ return sorted.first ?? AVSpeechSynthesisVoice(language: lang)
}
// MARK: - AVSpeechSynthesizerDelegate
@@ -53,10 +89,15 @@ final class VoiceSpeaker: NSObject, AVSpeechSynthesizerDelegate, @unchecked Send
let id = ObjectIdentifier(utterance)
Task { @MainActor in
guard let curr = self.currentUtterance, ObjectIdentifier(curr) == id else { return }
- self.isSpeaking = false
self.currentUtterance = nil
- try? await Task.sleep(nanoseconds: 300_000_000) // 300 ms gap
- self.onDidFinish?()
+ if self.queue.isEmpty {
+ self.isSpeaking = false
+ try? await Task.sleep(nanoseconds: 150_000_000) // 150 ms gap
+ self.onDidFinish?()
+ } else {
+ let next = self.queue.removeFirst()
+ self._enqueueAndPlay(text: next.text, locale: next.locale)
+ }
}
}
@@ -67,6 +108,7 @@ final class VoiceSpeaker: NSObject, AVSpeechSynthesizerDelegate, @unchecked Send
guard let curr = self.currentUtterance, ObjectIdentifier(curr) == id else { return }
self.isSpeaking = false
self.currentUtterance = nil
+ self.queue.removeAll()
}
}
}
diff --git a/tests/IntentParserTests.swift b/tests/IntentParserTests.swift
index 0dff281b..ea148436 100644
--- a/tests/IntentParserTests.swift
+++ b/tests/IntentParserTests.swift
@@ -317,6 +317,11 @@ enum IntentParserTests {
checkNilMultiAction("mets Gemini et blahblah",
parseMulti("mets Gemini et blahblah"))
+ // ── Music: lecture / stop la musique (feat/voice-jarvis) ─────────────
+ check("lecture", parse("lecture"), .musicPlay(target: nil))
+ check("stop la musique", parse("stop la musique"), .musicPause)
+ check("stop la chanson", parse("stop la chanson"), .musicPause)
+
// Summary
let total = pass + fail
if fail == 0 { print("\n\(total)/\(total) passed.") }
diff --git a/tests/TurnEndPolicyTests.swift b/tests/TurnEndPolicyTests.swift
index 740d62fc..472fdcd2 100644
--- a/tests/TurnEndPolicyTests.swift
+++ b/tests/TurnEndPolicyTests.swift
@@ -32,9 +32,9 @@ enum TurnEndPolicyTests {
"complete command → baseSilence")
// Constants
- precondition(TurnEndPolicy.baseSilence == 1.5, "baseSilence == 1.5")
- precondition(TurnEndPolicy.extendedSilence == 3.0, "extendedSilence == 3.0")
- precondition(TurnEndPolicy.maxTurnTime == 20.0, "maxTurnTime == 20.0")
+ precondition(TurnEndPolicy.baseSilence == 0.8, "baseSilence == 0.8")
+ precondition(TurnEndPolicy.extendedSilence == 1.6, "extendedSilence == 1.6")
+ precondition(TurnEndPolicy.maxTurnTime == 15.0, "maxTurnTime == 15.0")
print("TurnEndPolicyTests: all cases passed")
}