Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

Mac dictation: understands your languages, not just the app's

Claudec475a2105f52
diff --git a/NotchBuddy/Sources/App/IslandViewContent.swift b/NotchBuddy/Sources/App/IslandViewContent.swift
index 74a23826..3d161ae5 100644
--- a/NotchBuddy/Sources/App/IslandViewContent.swift
+++ b/NotchBuddy/Sources/App/IslandViewContent.swift
@@ -1173,6 +1173,40 @@ struct MailView: View {
// MARK: - Prompt (chat)
+#if !APPSTORE
+/// The mic's right-click menu: Automatic (the user's languages) or one fixed language.
+private struct DictationLanguageMenu: View {
+ @Bindable var dictation: MacDictation
+
+ var body: some View {
+ let automatic = MacDictation.automaticLocales().map(\.identifier)
+ Picker(String(localized: "Dictation language"), selection: $dictation.language) {
+ Text(String(localized: "Automatic") + " (" + automatic.map(MacDictation.name(of:)).joined(separator: ", ") + ")")
+ .tag(MacDictation.automatic)
+ ForEach(quickChoices(automatic), id: \.self) { id in
+ Text(MacDictation.name(of: id)).tag(id)
+ }
+ }
+ .pickerStyle(.inline)
+ Menu(String(localized: "Other languages")) {
+ Picker(String(localized: "Dictation language"), selection: $dictation.language) {
+ ForEach(MacDictation.allLanguages, id: \.self) { id in
+ Text(MacDictation.name(of: id)).tag(id)
+ }
+ }
+ .pickerStyle(.inline)
+ .labelsHidden()
+ }
+ }
+
+ /// Automatic's languages, plus the fixed one when it is another.
+ private func quickChoices(_ automatic: [String]) -> [String] {
+ let chosen = dictation.language
+ return chosen == MacDictation.automatic || automatic.contains(chosen) ? automatic : automatic + [chosen]
+ }
+}
+#endif
+
struct PromptView: View {
@ObservedObject var state: AppState
@State private var text: String = ""
@@ -1269,7 +1303,10 @@ struct PromptView: View {
#if !APPSTORE
// Dictate instead of typing (on-device speech recognition when available)
Button {
- Task { await dictation.toggle(startingFrom: text) }
+ Task {
+ if dictation.isRecording { text = await dictation.finish() }
+ else { await dictation.start(from: text) }
+ }
} label: {
Image(systemName: dictation.isRecording ? "mic.fill" : "mic")
.font(.system(size: 11, weight: .semibold))
@@ -1278,7 +1315,8 @@ struct PromptView: View {
.contentShape(Rectangle())
}
.buttonStyle(.plain)
- .help(dictation.isRecording ? String(localized: "Stop dictation") : String(localized: "Dictate"))
+ .help(dictation.isRecording ? String(localized: "Stop dictation") : String(localized: "Dictate (right-click to choose the language)"))
+ .contextMenu { DictationLanguageMenu(dictation: dictation) }
.onChange(of: dictation.transcript) { _, _ in
if dictation.isRecording { text = dictation.text }
}
@@ -1329,7 +1367,14 @@ struct PromptView: View {
private func sendMessage() {
#if !APPSTORE
- dictation.stop()
+ // Still dictating: take the final words (and the right language) before sending.
+ if dictation.isRecording {
+ Task {
+ text = await dictation.finish()
+ sendMessage()
+ }
+ return
+ }
#endif
let query = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !query.isEmpty else { return }
diff --git a/NotchBuddy/Sources/App/MacDictation.swift b/NotchBuddy/Sources/App/MacDictation.swift
index 1a00fe65..12e462e6 100644
--- a/NotchBuddy/Sources/App/MacDictation.swift
+++ b/NotchBuddy/Sources/App/MacDictation.swift
@@ -1,71 +1,282 @@
#if !APPSTORE
import AVFoundation
+import Carbon
import Speech
/// Speech to text for the notch chat (GitHub build), on-device when the Mac supports it.
-/// Same model as the iPhone's Dictation: not main-actor bound, because the audio tap and the
-/// recognizer call back on their own threads; the published values are set on the main actor.
+///
+/// Apple's recognizer only understands one language per request, so "Automatic" (the default)
+/// listens with up to three recognizers at once — the user's languages (macOS languages,
+/// keyboard layouts, then English) — shows the one that won last time while you speak, and when
+/// you stop keeps the transcript the recognizers were the most confident about. A fixed language
+/// can be chosen from the mic's right-click menu.
+///
+/// Not main-actor bound, because the audio tap and the recognizers call back on their own
+/// threads; the published values are set on the main actor.
@Observable
final class MacDictation: @unchecked Sendable {
+ static let automatic = "auto"
+ private static let languageKey = "dictationLanguage"
+ /// The language that won the last automatic dictation: shown first next time.
+ private static let lastHeardKey = "dictationLastLocale"
+ private static let maxCandidates = 3
+
var isRecording = false
var transcript = ""
/// What was typed before dictating, kept in front of the words.
var prefix = ""
+ /// `automatic` or a locale identifier (the mic's right-click menu).
+ var language: String = UserDefaults.standard.string(forKey: MacDictation.languageKey) ?? MacDictation.automatic {
+ didSet { UserDefaults.standard.set(language, forKey: Self.languageKey) }
+ }
@ObservationIgnored private let engine = AVAudioEngine()
- @ObservationIgnored private var request: SFSpeechAudioBufferRecognitionRequest?
- @ObservationIgnored private var task: SFSpeechRecognitionTask?
+ @ObservationIgnored private var candidates: [Candidate] = []
+ /// The candidate whose words are shown while speaking.
+ @ObservationIgnored private var lead = 0
+ @ObservationIgnored private var finishing: CheckedContinuation<Void, Never>?
+ @ObservationIgnored private var isFinishing = false
+ @ObservationIgnored private var tapInstalled = false
/// The text field's content: what was typed, then the words heard so far.
var text: String { prefix + transcript }
- @MainActor func toggle(startingFrom text: String) async {
- if isRecording { stop(); return }
- guard await Self.authorized() else { return }
+ /// One recognizer listening in one language.
+ private final class Candidate {
+ let locale: Locale
+ let recognizer: SFSpeechRecognizer
+ let request = SFSpeechAudioBufferRecognitionRequest()
+ var task: SFSpeechRecognitionTask?
+ var words = ""
+ /// Mean confidence of the words (Apple fills it in on the final result).
+ var confidence: Float = 0
+ var finished = false
+
+ init(locale: Locale, recognizer: SFSpeechRecognizer) {
+ self.locale = locale
+ self.recognizer = recognizer
+ request.shouldReportPartialResults = true
+ if recognizer.supportsOnDeviceRecognition { request.requiresOnDeviceRecognition = true }
+ }
+ }
+
+ // MARK: Start / finish
+
+ @MainActor func start(from text: String) async {
+ guard !isRecording, await Self.authorized() else { return }
+ let built = Self.makeCandidates(language == Self.automatic ? Self.automaticLocales() : [Locale(identifier: language)])
+ guard !built.isEmpty, Self.startEngine(engine, feeding: built.map(\.request)) else { return }
+ tapInstalled = true
prefix = text.isEmpty || text.hasSuffix(" ") ? text : text + " "
transcript = ""
- start()
+ candidates = built
+ lead = 0
+ isRecording = true
+ for (index, candidate) in built.enumerated() {
+ candidate.task = Self.recognize(candidate) { [weak self] words, confidence, final, failed in
+ Task { @MainActor in
+ self?.update(index, words: words, confidence: confidence, final: final, failed: failed)
+ }
+ }
+ }
}
- nonisolated private func start() {
- // The app's language first (Settings → General → Language), then the system's.
- let localeID = Bundle.main.preferredLocalizations.first ?? Locale.preferredLanguages.first ?? "en-US"
- guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeID)) ?? SFSpeechRecognizer(),
- recognizer.isAvailable else { return }
- let request = SFSpeechAudioBufferRecognitionRequest()
- request.shouldReportPartialResults = true
- if recognizer.supportsOnDeviceRecognition { request.requiresOnDeviceRecognition = true }
+ /// Stops listening and returns the field's text with the most likely transcript.
+ @MainActor func finish() async -> String {
+ guard isRecording else { return text }
+ isFinishing = true
+ stopEngine()
+ candidates.forEach { $0.request.endAudio() }
+ // With several languages, wait a moment for the final results: they carry the confidence.
+ if candidates.count > 1, !candidates.allSatisfy(\.finished) {
+ await withCheckedContinuation { continuation in
+ finishing = continuation
+ Task { @MainActor [weak self] in
+ try? await Task.sleep(for: .milliseconds(1200))
+ self?.resumeFinishing()
+ }
+ }
+ }
+ if let best = bestCandidate() {
+ transcript = best.words
+ if candidates.count > 1 {
+ UserDefaults.standard.set(best.locale.identifier, forKey: Self.lastHeardKey)
+ }
+ }
+ tearDown()
+ return text
+ }
+
+ /// Cancels without waiting (the view went away).
+ @MainActor func stop() {
+ guard isRecording || !candidates.isEmpty else { return }
+ tearDown()
+ }
+
+ @MainActor private func stopEngine() {
+ engine.stop()
+ if tapInstalled { engine.inputNode.removeTap(onBus: 0) }
+ tapInstalled = false
+ }
+
+ @MainActor private func tearDown() {
+ stopEngine()
+ candidates.forEach { $0.request.endAudio(); $0.task?.cancel() }
+ candidates = []
+ lead = 0
+ isRecording = false
+ isFinishing = false
+ resumeFinishing()
+ }
+
+ @MainActor private func resumeFinishing() {
+ finishing?.resume()
+ finishing = nil
+ }
+
+ // MARK: Results
+
+ @MainActor private func update(_ index: Int, words: String?, confidence: Float, final: Bool, failed: Bool) {
+ guard index < candidates.count else { return }
+ let candidate = candidates[index]
+ if let words { candidate.words = words }
+ if confidence > 0 { candidate.confidence = confidence }
+ if final || failed { candidate.finished = true }
+
+ if isFinishing {
+ if candidates.allSatisfy(\.finished) { resumeFinishing() }
+ return
+ }
+ // A recognizer that gives up while you speak (no model, busy…) hands over to the next one.
+ if index == lead, candidate.finished, candidate.words.isEmpty,
+ let next = candidates.firstIndex(where: { !$0.finished }) {
+ lead = next
+ transcript = candidates[next].words
+ return
+ }
+ // When Apple reports confidences while speaking, follow the clearly better language.
+ let current = candidates[lead]
+ if index != lead, candidate.confidence > current.confidence + 0.15,
+ candidate.words.split(separator: " ").count >= 3 {
+ lead = index
+ }
+ if index == lead { transcript = candidate.words }
+ if candidates.allSatisfy(\.finished) { tearDown() }
+ }
+
+ @MainActor private func bestCandidate() -> Candidate? {
+ guard lead < candidates.count else { return nil }
+ let leader = candidates[lead]
+ let best = candidates
+ .filter { !$0.words.isEmpty }
+ .max { $0.confidence < $1.confidence }
+ guard let best, best.confidence > leader.confidence else { return leader }
+ return best
+ }
+
+ // MARK: Audio and recognizers (off the main actor: they call back on their own threads)
+
+ nonisolated private static func startEngine(_ engine: AVAudioEngine, feeding requests: [SFSpeechAudioBufferRecognitionRequest]) -> Bool {
let input = engine.inputNode
input.installTap(onBus: 0, bufferSize: 1024, format: input.outputFormat(forBus: 0)) { buffer, _ in
- request.append(buffer)
+ for request in requests { request.append(buffer) }
}
engine.prepare()
do { try engine.start() } catch {
input.removeTap(onBus: 0)
- return
+ return false
}
- self.request = request
- Task { @MainActor in self.isRecording = true }
- task = recognizer.recognitionTask(with: request) { [weak self] result, error in
+ return true
+ }
+
+ nonisolated private static func recognize(
+ _ candidate: Candidate,
+ report: @escaping @Sendable (String?, Float, Bool, Bool) -> Void
+ ) -> SFSpeechRecognitionTask {
+ candidate.recognizer.recognitionTask(with: candidate.request) { result, error in
let words = result?.bestTranscription.formattedString
- let done = error != nil || (result?.isFinal ?? false)
- Task { @MainActor in
- guard let self else { return }
- if let words { self.transcript = words }
- if done { self.stop() }
- }
+ let segments = result?.bestTranscription.segments ?? []
+ let confidence = segments.isEmpty ? 0 : segments.map(\.confidence).reduce(0, +) / Float(segments.count)
+ report(words, confidence, result?.isFinal ?? false, error != nil)
}
}
- @MainActor func stop() {
- guard isRecording || request != nil else { return }
- engine.stop()
- engine.inputNode.removeTap(onBus: 0)
- request?.endAudio()
- task?.cancel()
- request = nil
- task = nil
- isRecording = false
+ /// Recognizers that can run now. Extra languages in Automatic only listen on-device,
+ /// so the audio never goes to Apple in a language you didn't speak.
+ private static func makeCandidates(_ locales: [Locale]) -> [Candidate] {
+ var out: [Candidate] = []
+ for locale in locales {
+ guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.isAvailable else { continue }
+ if !out.isEmpty && !recognizer.supportsOnDeviceRecognition { continue }
+ out.append(Candidate(locale: locale, recognizer: recognizer))
+ }
+ if out.isEmpty, let fallback = SFSpeechRecognizer(), fallback.isAvailable {
+ out.append(Candidate(locale: fallback.locale, recognizer: fallback))
+ }
+ return out
+ }
+
+ // MARK: Languages
+
+ /// The languages Automatic listens for: last winner, macOS languages, keyboard layouts,
+ /// Coucou's own language, then English — one per language, up to three.
+ static func automaticLocales() -> [Locale] {
+ var ids: [String] = []
+ if let last = UserDefaults.standard.string(forKey: lastHeardKey) { ids.append(last) }
+ // The Mac's languages (not Coucou's override in Settings → General → Language).
+ ids += (CFPreferencesCopyValue("AppleLanguages" as CFString, kCFPreferencesAnyApplication,
+ kCFPreferencesCurrentUser, kCFPreferencesAnyHost) as? [String]) ?? []
+ ids += keyboardLanguages()
+ ids += Locale.preferredLanguages
+ ids.append("en-US")
+
+ let supported = Array(SFSpeechRecognizer.supportedLocales())
+ var seen = Set<String>()
+ var out: [Locale] = []
+ for id in ids {
+ guard let locale = match(id, in: supported) else { continue }
+ let language = locale.language.languageCode?.identifier ?? locale.identifier
+ guard seen.insert(language).inserted else { continue }
+ out.append(locale)
+ if out.count == maxCandidates { break }
+ }
+ return out
+ }
+
+ /// Every language the recognizer knows, by name.
+ static var allLanguages: [String] {
+ SFSpeechRecognizer.supportedLocales()
+ .map(\.identifier)
+ .sorted { name(of: $0).localizedCaseInsensitiveCompare(name(of: $1)) == .orderedAscending }
+ }
+
+ static func name(of identifier: String) -> String {
+ let name = Locale.current.localizedString(forIdentifier: identifier) ?? identifier
+ return name.prefix(1).uppercased() + name.dropFirst()
+ }
+
+ /// "fr" or "fr_FR" → the recognizer locale for it, preferring the Mac's region.
+ private static func match(_ id: String, in supported: [Locale]) -> Locale? {
+ let wanted = id.replacingOccurrences(of: "_", with: "-").lowercased()
+ if let exact = supported.first(where: { $0.identifier.replacingOccurrences(of: "_", with: "-").lowercased() == wanted }) {
+ return exact
+ }
+ let language = Locale(identifier: id).language.languageCode?.identifier
+ let sameLanguage = supported.filter { $0.language.languageCode?.identifier == language }
+ let region = Locale(identifier: id).region ?? Locale.current.region
+ return sameLanguage.first { $0.region == region }
+ ?? sameLanguage.min { $0.identifier < $1.identifier }
+ }
+
+ /// The main language of each enabled keyboard layout ("fr" for AZERTY, "en" for ABC…).
+ private static func keyboardLanguages() -> [String] {
+ let filter = ["TISPropertyInputSourceCategory": "TISCategoryKeyboardInputSource"] as CFDictionary
+ guard let list = TISCreateInputSourceList(filter, false)?.takeRetainedValue() as? [TISInputSource] else { return [] }
+ return list.compactMap { source in
+ guard let raw = TISGetInputSourceProperty(source, "TISPropertyInputSourceLanguages" as CFString) else { return nil }
+ let languages = Unmanaged<CFArray>.fromOpaque(raw).takeUnretainedValue() as? [String]
+ return languages?.first
+ }
}
private static func authorized() async -> Bool {
diff --git a/README.md b/README.md
index 41f3c39b..b475eb92 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ Meet **Mochi**: a soft little squircle with big eyes that pops out of your notch
- ⌨️ **Keyboard shortcuts** — open the chat, jump to an alert or a terminal, switch pills, mute, send Mochi to the desktop or open the wardrobe from anywhere; all customizable in Settings → Shortcuts.
- 🎭 **A real character** — idle breathing, blinks, eyes on a sphere that follow your mouse, emotes, 28 handcrafted sounds, a greeting on launch.
- 🔊 **Your own sounds** *(macOS)* — drop a file named like one of Mochi's sounds (`finish.wav`, `approval.mp3`…) in Settings → General → Sound → **Open sounds folder** to replace it.
-- 🎙️ **Dictate in the chat** *(macOS, GitHub build)* — click the mic next to the chat field and talk; speech recognition runs on your Mac when it supports it.
+- 🎙️ **Dictate in the chat** *(macOS, GitHub build)* — click the mic next to the chat field and talk in any of your languages (Coucou picks the one you spoke, or right-click the mic to choose); speech recognition runs on your Mac when it supports it.
- 🫥 **Invisible when idle** — hides away when nothing is running, peeks out when you hover the notch (the top edge of the screen on Windows and Linux).
- 🖥️ **Any Mac, notch or not** — on an iMac, a Mac mini, or a MacBook with its lid closed on an external display, Mochi sits in a small bar at the top of the screen. With several displays, pick his screen in Settings → General, or let him follow your mouse *(macOS)*.
- 📱 **Coucou on iPhone** — your sessions, approvals and services in your pocket, with Live Activities, widgets and Siri. See [Coucou on iPhone](#coucou-on-iphone).