Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

voice: take turns (no open mic after answers), calm Mochi, more time to speak

Claudea0c030d986e4
diff --git a/NotchBuddy/Sources/App/AppState.swift b/NotchBuddy/Sources/App/AppState.swift
index 9aee4833..47c1a075 100644
--- a/NotchBuddy/Sources/App/AppState.swift
+++ b/NotchBuddy/Sources/App/AppState.swift
@@ -425,6 +425,9 @@ final class AppState: ObservableObject {
// Voice command result (shown in VoiceResultView for ~2 s, then cleared by IWC)
@Published var voiceResult: VoiceActionResult? = nil
+ /// True from "OK Coucou" until Coucou has finished answering (island stays compact).
+ /// Drives Mochi's single attentive pose for the whole exchange.
+ @Published var voiceActive = false
#endif
// Claude plan gauge (from statusline hook)
diff --git a/NotchBuddy/Sources/App/BotCanvasView.swift b/NotchBuddy/Sources/App/BotCanvasView.swift
index 7f7a4e86..50dcc041 100644
--- a/NotchBuddy/Sources/App/BotCanvasView.swift
+++ b/NotchBuddy/Sources/App/BotCanvasView.swift
@@ -74,7 +74,10 @@ struct BotCanvasView: View {
#if !APPSTORE
if state.view == .listening {
- engine.listeningLevel = CGFloat(VoiceEngine.shared.micLevel)
+ // Expanded listening view only (no longer used by voice, kept for the
+ // view itself). Smoothed so the eyes do not follow every mic frame.
+ let target = CGFloat(VoiceEngine.shared.micLevel)
+ engine.listeningLevel += (target - engine.listeningLevel) * 0.12
engine.listeningHasWords = !VoiceEngine.shared.commandTranscript.isEmpty
}
#endif
@@ -108,6 +111,11 @@ struct BotCanvasView: View {
.onChange(of: state.effectiveState) { _, newState in
engine.setState(newState)
}
+ #if !APPSTORE
+ .onChange(of: state.voiceActive) { _, on in
+ if on { engine.enterVoiceCompact() } else { engine.exitVoiceCompact() }
+ }
+ #endif
.onChange(of: state.view) { oldView, newView in
// Morph up when upload view is active
if state.mode == .expanded && newView == .upload {
diff --git a/NotchBuddy/Sources/App/IslandWindowController.swift b/NotchBuddy/Sources/App/IslandWindowController.swift
index 716db884..e79635fa 100644
--- a/NotchBuddy/Sources/App/IslandWindowController.swift
+++ b/NotchBuddy/Sources/App/IslandWindowController.swift
@@ -25,7 +25,11 @@ final class IslandWindowController: NSWindowController {
// Voice result auto-dismiss timer
#if !APPSTORE
private var voiceResultWork: DispatchWorkItem?
+ /// True only while Coucou listens for the answer to a question it asked.
private var isInConversation = false
+ /// Context (last action, on-device model session) is kept a little after a turn so
+ /// "OK Coucou, et Stripe aussi" still works; this resets it.
+ private var voiceContextExpiry: DispatchWorkItem?
private var conversationContext = ConversationContext()
private var consecutiveFailures = 0
private var hasSpokenPasCompris = false
@@ -311,6 +315,8 @@ final class IslandWindowController: NSWindowController {
let isDirect = (note.object as? String) == "direct"
Task { @MainActor [weak self] in
// Genuine wake phrase (not programmatic re-listen) → clear any pending question
+ AppState.shared.voiceActive = true
+ self?.voiceContextExpiry?.cancel()
if !isDirect {
VoiceActionRunner.shared.pendingQuestion = nil
// The island stays compact now: the tick says "I heard OK Coucou".
@@ -335,14 +341,8 @@ final class IslandWindowController: NSWindowController {
self.expand(to: .voiceResult)
self.scheduleVoiceDismiss(delay: 1.5)
} else if self.isInConversation {
- // Conversation ended by 8-second silence
- self.isInConversation = false
- self.conversationContext.reset()
- self.voiceResultWork?.cancel()
- self.voiceResultWork = nil
- VoiceEngine.shared.endConversation()
- VoiceBrain.shared.endConversation()
- self.scheduleVoiceDismiss(delay: 0.3)
+ // No answer to Coucou's question: stop listening.
+ self.closeVoiceTurn()
} else {
self.fsm.voiceFinished()
}
@@ -539,12 +539,16 @@ final class IslandWindowController: NSWindowController {
finishedPinTimer?.cancel()
#if !APPSTORE
VoiceSpeaker.shared.stop()
- if isInConversation {
+ if isInConversation || AppState.shared.voiceActive {
+ // Closing the island ends the voice exchange (speech was just cut, so its
+ // "finished" callback will not come): reset everything that it would have.
isInConversation = false
voiceResultWork?.cancel()
voiceResultWork = nil
VoiceEngine.shared.endConversation()
- VoiceBrain.shared.endConversation()
+ VoiceCaptionManager.shared.hide(after: 0)
+ AppState.shared.voiceResult = nil
+ AppState.shared.voiceActive = false
}
#endif
// Keep the FSM in step with what is on screen (home/coucou → petit now).
@@ -1352,7 +1356,7 @@ extension IslandWindowController {
let normTranscript = WakePhrase.normalise(transcript)
if isInConversation && TurnEndPolicy.conversationEndPhrases.contains(normTranscript) {
VoiceTranscriptHistory.shared.record(transcript: transcript, note: "end", origin: .end)
- endConversation(speaking: true)
+ closeVoiceTurn()
return
}
@@ -1368,7 +1372,7 @@ extension IslandWindowController {
// Never log the words themselves (VOICE.md: no transcript on disk).
appendAppLog("nb.log", "[Voice] ignoring short spurious transcript")
VoiceTranscriptHistory.shared.record(transcript: transcript, note: "—", origin: .ignored)
- scheduleConversationContinue(delay: 0.2)
+ closeVoiceTurn()
return
}
}
@@ -1378,9 +1382,7 @@ extension IslandWindowController {
if runner.pendingQuestion != nil {
let result = await runner.handleAnswer(transcript, availablePills: pills)
VoiceTranscriptHistory.shared.record(transcript: transcript, note: result.message, origin: .answer)
- if result.outcome == .success {
- NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
- } else {
+ if result.outcome != .success {
NotificationCenter.default.post(name: .botDizzy, object: nil)
}
VoiceCaptionManager.shared.setUserLine(transcript)
@@ -1428,8 +1430,6 @@ extension IslandWindowController {
)
if anyFailure {
NotificationCenter.default.post(name: .botDizzy, object: nil)
- } else {
- NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
}
VoiceCaptionManager.shared.setUserLine(transcript)
VoiceCaptionManager.shared.appendResponse(combined.message)
@@ -1441,7 +1441,7 @@ extension IslandWindowController {
// ── Relative context resolution ────────────────────────────────────────────
let intent: VoiceIntent
let transcriptOrigin: TranscriptOrigin
- if isInConversation,
+ if conversationContext.lastIntent != nil,
let resolved = conversationContext.resolveRelative(transcript, pills: pills) {
intent = resolved
transcriptOrigin = .context
@@ -1530,7 +1530,6 @@ extension IslandWindowController {
if brainUsed {
// TTS sentences already enqueued via streaming. Enter conversation and
// continue once the speaker queue drains.
- NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
conversationContext.update(effectiveIntent)
consecutiveFailures = 0
AppState.shared.voiceResult = result
@@ -1547,21 +1546,14 @@ extension IslandWindowController {
// "on s'en fout c'est"…) is dropped silently: no dizzy Mochi, no "pas compris".
// Two in a row end the conversation.
if isInConversation, case .unknown = effectiveIntent, result.outcome == .failure {
- consecutiveFailures += 1
- appendAppLog("nb.log", "[Voice] conversation: no command in phrase, ignored")
- if consecutiveFailures >= 2 {
- consecutiveFailures = 0
- endConversation(speaking: false)
- } else {
- scheduleConversationContinue(delay: 0.2)
- }
+ appendAppLog("nb.log", "[Voice] answer had no command in it, ignored")
+ closeVoiceTurn()
return
}
// Mochi reaction + consecutive failure tracking
switch result.outcome {
case .success:
- NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
conversationContext.update(effectiveIntent)
consecutiveFailures = 0
case .failure:
@@ -1619,93 +1611,91 @@ extension IslandWindowController {
scheduleVoiceDismiss(delay: 2.0)
}
- /// Speak the result message (if enabled) then start next conversation turn.
+ /// Speak the result, then stop listening — unless the answer is a question, in which
+ /// case listen once for the reply. Coucou cannot tell whether I am talking to it or
+ /// to someone else, so it only keeps the mic open when it asked something.
@MainActor
private func speakAndContinueConversation(_ result: VoiceActionResult) {
- if !isInConversation {
- VoiceBrain.shared.beginConversation()
- consecutiveFailures = 0
- hasSpokenPasCompris = false
- }
- isInConversation = true
-
+ let asks = Self.isQuestion(result)
let speaker = VoiceSpeaker.shared
- let shouldSpeak: Bool
- if result.outcome == .failure && hasSpokenPasCompris {
- shouldSpeak = false // max 1 "pas compris" per conversation
- } else {
- shouldSpeak = VoiceSettings.speakEnabled
- }
- if result.outcome == .failure { hasSpokenPasCompris = true }
- if shouldSpeak {
- let locale = VoiceEngine.shared.speechLocale
- speaker.speak(result.message, locale: locale)
+ if VoiceSettings.speakEnabled {
+ speaker.speak(result.message, locale: VoiceEngine.shared.speechLocale)
speaker.onDidFinish = { [weak self] in
- Task { @MainActor in
- self?.scheduleConversationContinue(delay: 0.15)
- }
+ Task { @MainActor in self?.finishVoiceTurn(expectAnswer: asks) }
}
} else {
- // No speech: continue after 1.5 s
- scheduleConversationContinue(delay: 1.5)
+ // No speech: leave the caption up a moment, then finish.
+ voiceResultWork?.cancel()
+ let item = DispatchWorkItem { [weak self] in
+ Task { @MainActor in self?.finishVoiceTurn(expectAnswer: asks) }
+ }
+ voiceResultWork = item
+ DispatchQueue.main.asyncAfter(deadline: .now() + 1.5, execute: item)
}
}
- /// Enter/stay in conversation mode after brain streaming TTS queues drain.
+ /// After the on-device model's streamed answer: same rule once the speech drains.
@MainActor
private func _enterConversationAfterStreamedSpeech() {
- if !isInConversation {
- VoiceBrain.shared.beginConversation()
- consecutiveFailures = 0
- hasSpokenPasCompris = false
- }
- isInConversation = true
+ let asks = Self.isQuestion(AppState.shared.voiceResult)
let speaker = VoiceSpeaker.shared
if speaker.isSpeaking {
speaker.onDidFinish = { [weak self] in
- Task { @MainActor in
- self?.scheduleConversationContinue(delay: 0.15)
- }
+ Task { @MainActor in self?.finishVoiceTurn(expectAnswer: asks) }
}
} else {
- scheduleConversationContinue(delay: 0.15)
+ finishVoiceTurn(expectAnswer: asks)
}
}
- /// Schedule the next conversation listening turn.
+ /// A question Coucou asked: an explicit follow-up, or an answer ending with "?".
+ private static func isQuestion(_ result: VoiceActionResult?) -> Bool {
+ guard let result else { return false }
+ if case .question = result.outcome { return true }
+ let t = result.message.trimmingCharacters(in: .whitespacesAndNewlines)
+ return t.hasSuffix("?") || t.hasSuffix("?") || t.hasSuffix("؟")
+ }
+
+ /// Listen once for the reply to Coucou's question, or close the turn.
@MainActor
- private func scheduleConversationContinue(delay: TimeInterval) {
- voiceResultWork?.cancel()
- let item = DispatchWorkItem { [weak self] in
- Task { @MainActor in
- guard let self, self.isInConversation else { return }
- // Tick sound + listening emote signal that Mochi is ready to hear the next command.
- SoundEngine.shared.play("tick")
- NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.listening)
- VoiceEngine.shared.startConversationTurn()
- }
+ private func finishVoiceTurn(expectAnswer: Bool) {
+ // One follow-up only: the reply to a question closes the turn after it is handled
+ // (unless that reply leads to another question, e.g. "which one do I remove?").
+ if expectAnswer && VoiceEngine.shared.isEnabled {
+ isInConversation = true
+ VoiceBrain.shared.beginConversation()
+ if AppState.shared.soundEnabled { SoundEngine.shared.play("tick") }
+ VoiceEngine.shared.startConversationTurn()
+ } else {
+ closeVoiceTurn()
}
- voiceResultWork = item
- DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: item)
}
- /// End conversation window, optionally trigger a farewell emote, then collapse.
+ /// Stop listening and let the island settle. The context (last action, model session)
+ /// stays 90 s so a new "OK Coucou, et Stripe aussi" still understands "aussi".
@MainActor
- private func endConversation(speaking: Bool) {
+ private func closeVoiceTurn() {
isInConversation = false
- conversationContext.reset()
consecutiveFailures = 0
hasSpokenPasCompris = false
- voiceResultWork?.cancel()
- voiceResultWork = nil
VoiceEngine.shared.endConversation()
- VoiceBrain.shared.endConversation()
VoiceCaptionManager.shared.endConversation()
- if speaking {
- // Brief farewell emote
- NotificationCenter.default.post(name: .triggerEmote, object: BotEmote.happy)
+ scheduleVoiceDismiss(delay: 0.3)
+ voiceContextExpiry?.cancel()
+ let expiry = DispatchWorkItem { [weak self] in
+ Task { @MainActor in
+ self?.conversationContext.reset()
+ VoiceBrain.shared.endConversation()
+ }
}
- scheduleVoiceDismiss(delay: 0.5)
+ voiceContextExpiry = expiry
+ DispatchQueue.main.asyncAfter(deadline: .now() + 90, execute: expiry)
+ }
+
+ /// Kept for the paths that still call it (collapse, end phrase): close and forget.
+ @MainActor
+ private func endConversation(speaking: Bool) {
+ closeVoiceTurn()
}
@MainActor
@@ -1715,6 +1705,7 @@ extension IslandWindowController {
let item = DispatchWorkItem { [weak self] in
guard let self else { return }
AppState.shared.voiceResult = nil
+ AppState.shared.voiceActive = false
// Reset view before collapsing so shouldIgnoreWake never sees a stale .voiceResult.
AppState.shared.view = self.defaultView()
self.fsm.voiceFinished()
diff --git a/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift b/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift
index 20756c58..4131c0a0 100644
--- a/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift
+++ b/NotchBuddy/Sources/App/Voice/TurnEndPolicy.swift
@@ -6,8 +6,8 @@ import Foundation
// Decides silence duration before ending a command turn.
// Standalone compilable (used by test scripts via swiftc).
enum TurnEndPolicy {
- static let baseSilence: TimeInterval = 0.8
- static let extendedSilence: TimeInterval = 1.6
+ static let baseSilence: TimeInterval = 1.2
+ static let extendedSilence: TimeInterval = 2.2
static let maxTurnTime: TimeInterval = 15.0
/// Returns the silence timeout given the current *normalised* partial transcript.
diff --git a/NotchBuddy/Sources/App/Voice/VoiceBrain.swift b/NotchBuddy/Sources/App/Voice/VoiceBrain.swift
index 2a47833c..6cbfe4dc 100644
--- a/NotchBuddy/Sources/App/Voice/VoiceBrain.swift
+++ b/NotchBuddy/Sources/App/Voice/VoiceBrain.swift
@@ -121,6 +121,7 @@ final class VoiceBrain {
Tu es Coucou, un assistant dans le notch du MacBook.
Réponds toujours dans la langue de l'utilisateur.
Réponds avec 1 à 2 phrases maximum. Sois direct et concis.
+ Ne pose une question que si tu as vraiment besoin d'une précision pour agir : une seule, courte, qui finit par « ? ». Sinon, ne finis jamais par une question.
Utilise les outils disponibles pour exécuter des commandes sur les pills et la musique.
Pour les noms de pilules, utilise le nom exact fourni par l'utilisateur.
"""
diff --git a/NotchBuddy/Sources/CoucouKit/BotEngine.swift b/NotchBuddy/Sources/CoucouKit/BotEngine.swift
index 715d1c08..f69a7211 100644
--- a/NotchBuddy/Sources/CoucouKit/BotEngine.swift
+++ b/NotchBuddy/Sources/CoucouKit/BotEngine.swift
@@ -628,6 +628,34 @@ final class BotEngine: ObservableObject {
nextBlink = CACurrentMediaTime() + 2.5 + Double.random(in: 0...1.0)
}
+ // MARK: - Voice in the compact island
+
+ private var voicePrevPermanentEmote: BotEmote? = nil
+ private(set) var isInVoiceCompact = false
+
+ /// Voice session while the island stays compact (wake → answer spoken).
+ /// One calm attentive pose for the whole exchange: wide eyes eased in, a slight
+ /// tilt, no jump, no hands, no mic-driven motion — in the notch every extra
+ /// movement reads as jitter.
+ func enterVoiceCompact() {
+ guard !isInVoiceCompact else { return }
+ isInVoiceCompact = true
+ voicePrevPermanentEmote = permanentEmote
+ setPermanentEmote(.listening)
+ anim("es", keys: [TweenKey(target: 1.06, duration: 260, ease: Ease.inOut)])
+ anim("tilt", keys: [TweenKey(target: -0.06, duration: 260, ease: Ease.inOut)])
+ }
+
+ /// End of the voice exchange: back to the previous look, eased.
+ func exitVoiceCompact() {
+ guard isInVoiceCompact else { return }
+ isInVoiceCompact = false
+ setPermanentEmote(voicePrevPermanentEmote)
+ voicePrevPermanentEmote = nil
+ anim("es", keys: [TweenKey(target: 1, duration: 260, ease: Ease.inOut)])
+ anim("tilt", keys: [TweenKey(target: 0, duration: 260, ease: Ease.inOut)])
+ }
+
/// Call when the island leaves listening state.
/// `hadCommand` = commandTranscript was non-empty when the session ended.
func exitListening(hadCommand: Bool) {
diff --git a/tests/TurnEndPolicyTests.swift b/tests/TurnEndPolicyTests.swift
index 472fdcd2..0b8cea33 100644
--- a/tests/TurnEndPolicyTests.swift
+++ b/tests/TurnEndPolicyTests.swift
@@ -32,8 +32,8 @@ enum TurnEndPolicyTests {
"complete command → baseSilence")
// Constants
- precondition(TurnEndPolicy.baseSilence == 0.8, "baseSilence == 0.8")
- precondition(TurnEndPolicy.extendedSilence == 1.6, "extendedSilence == 1.6")
+ precondition(TurnEndPolicy.baseSilence == 1.2, "baseSilence == 1.2")
+ precondition(TurnEndPolicy.extendedSilence == 2.2, "extendedSilence == 2.2")
precondition(TurnEndPolicy.maxTurnTime == 15.0, "maxTurnTime == 15.0")
print("TurnEndPolicyTests: all cases passed")