A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.
Swift ai-agentsanthropicantigravityclaudeclaude-codecodexcursordynamic-islandgemini-cliiosiphonlinuxmacosmacos-appmenubar-appnotchopen-sourceswiftswiftuiwindows
Mac dictation: understands your languages, not just the app's
Claudec475a2105f52
diff --git a/NotchBuddy/Sources/App/IslandViewContent.swift b/NotchBuddy/Sources/App/IslandViewContent.swiftindex 74a23826..3d161ae5 100644--- a/NotchBuddy/Sources/App/IslandViewContent.swift+++ b/NotchBuddy/Sources/App/IslandViewContent.swift@@ -1173,6 +1173,40 @@ struct MailView: View {// MARK: - Prompt (chat)+#if !APPSTORE+/// The mic's right-click menu: Automatic (the user's languages) or one fixed language.+private struct DictationLanguageMenu: View {+ @Bindable var dictation: MacDictation++ var body: some View {+ let automatic = MacDictation.automaticLocales().map(\.identifier)+ Picker(String(localized: "Dictation language"), selection: $dictation.language) {+ Text(String(localized: "Automatic") + " (" + automatic.map(MacDictation.name(of:)).joined(separator: ", ") + ")")+ .tag(MacDictation.automatic)+ ForEach(quickChoices(automatic), id: \.self) { id in+ Text(MacDictation.name(of: id)).tag(id)+ }+ }+ .pickerStyle(.inline)+ Menu(String(localized: "Other languages")) {+ Picker(String(localized: "Dictation language"), selection: $dictation.language) {+ ForEach(MacDictation.allLanguages, id: \.self) { id in+ Text(MacDictation.name(of: id)).tag(id)+ }+ }+ .pickerStyle(.inline)+ .labelsHidden()+ }+ }++ /// Automatic's languages, plus the fixed one when it is another.+ private func quickChoices(_ automatic: [String]) -> [String] {+ let chosen = dictation.language+ return chosen == MacDictation.automatic || automatic.contains(chosen) ? automatic : automatic + [chosen]+ }+}+#endif+struct PromptView: View {@ObservedObject var state: AppState@State private var text: String = ""@@ -1269,7 +1303,10 @@ struct PromptView: View {#if !APPSTORE// Dictate instead of typing (on-device speech recognition when available)Button {- Task { await dictation.toggle(startingFrom: text) }+ Task {+ if dictation.isRecording { text = await dictation.finish() }+ else { await dictation.start(from: text) }+ }} label: {Image(systemName: dictation.isRecording ? "mic.fill" : "mic").font(.system(size: 11, weight: .semibold))@@ -1278,7 +1315,8 @@ struct PromptView: View {.contentShape(Rectangle())}.buttonStyle(.plain)- .help(dictation.isRecording ? String(localized: "Stop dictation") : String(localized: "Dictate"))+ .help(dictation.isRecording ? String(localized: "Stop dictation") : String(localized: "Dictate (right-click to choose the language)"))+ .contextMenu { DictationLanguageMenu(dictation: dictation) }.onChange(of: dictation.transcript) { _, _ inif dictation.isRecording { text = dictation.text }}@@ -1329,7 +1367,14 @@ struct PromptView: View {private func sendMessage() {#if !APPSTORE- dictation.stop()+ // Still dictating: take the final words (and the right language) before sending.+ if dictation.isRecording {+ Task {+ text = await dictation.finish()+ sendMessage()+ }+ return+ }#endiflet query = text.trimmingCharacters(in: .whitespacesAndNewlines)guard !query.isEmpty else { return }diff --git a/NotchBuddy/Sources/App/MacDictation.swift b/NotchBuddy/Sources/App/MacDictation.swiftindex 1a00fe65..12e462e6 100644--- a/NotchBuddy/Sources/App/MacDictation.swift+++ b/NotchBuddy/Sources/App/MacDictation.swift@@ -1,71 +1,282 @@#if !APPSTOREimport AVFoundation+import Carbonimport Speech/// Speech to text for the notch chat (GitHub build), on-device when the Mac supports it.-/// Same model as the iPhone's Dictation: not main-actor bound, because the audio tap and the-/// recognizer call back on their own threads; the published values are set on the main actor.+///+/// Apple's recognizer only understands one language per request, so "Automatic" (the default)+/// listens with up to three recognizers at once — the user's languages (macOS languages,+/// keyboard layouts, then English) — shows the one that won last time while you speak, and when+/// you stop keeps the transcript the recognizers were the most confident about. A fixed language+/// can be chosen from the mic's right-click menu.+///+/// Not main-actor bound, because the audio tap and the recognizers call back on their own+/// threads; the published values are set on the main actor.@Observablefinal class MacDictation: @unchecked Sendable {+ static let automatic = "auto"+ private static let languageKey = "dictationLanguage"+ /// The language that won the last automatic dictation: shown first next time.+ private static let lastHeardKey = "dictationLastLocale"+ private static let maxCandidates = 3+var isRecording = falsevar transcript = ""/// What was typed before dictating, kept in front of the words.var prefix = ""+ /// `automatic` or a locale identifier (the mic's right-click menu).+ var language: String = UserDefaults.standard.string(forKey: MacDictation.languageKey) ?? MacDictation.automatic {+ didSet { UserDefaults.standard.set(language, forKey: Self.languageKey) }+ }@ObservationIgnored private let engine = AVAudioEngine()- @ObservationIgnored private var request: SFSpeechAudioBufferRecognitionRequest?- @ObservationIgnored private var task: SFSpeechRecognitionTask?+ @ObservationIgnored private var candidates: [Candidate] = []+ /// The candidate whose words are shown while speaking.+ @ObservationIgnored private var lead = 0+ @ObservationIgnored private var finishing: CheckedContinuation<Void, Never>?+ @ObservationIgnored private var isFinishing = false+ @ObservationIgnored private var tapInstalled = false/// The text field's content: what was typed, then the words heard so far.var text: String { prefix + transcript }- @MainActor func toggle(startingFrom text: String) async {- if isRecording { stop(); return }- guard await Self.authorized() else { return }+ /// One recognizer listening in one language.+ private final class Candidate {+ let locale: Locale+ let recognizer: SFSpeechRecognizer+ let request = SFSpeechAudioBufferRecognitionRequest()+ var task: SFSpeechRecognitionTask?+ var words = ""+ /// Mean confidence of the words (Apple fills it in on the final result).+ var confidence: Float = 0+ var finished = false++ init(locale: Locale, recognizer: SFSpeechRecognizer) {+ self.locale = locale+ self.recognizer = recognizer+ request.shouldReportPartialResults = true+ if recognizer.supportsOnDeviceRecognition { request.requiresOnDeviceRecognition = true }+ }+ }++ // MARK: Start / finish++ @MainActor func start(from text: String) async {+ guard !isRecording, await Self.authorized() else { return }+ let built = Self.makeCandidates(language == Self.automatic ? Self.automaticLocales() : [Locale(identifier: language)])+ guard !built.isEmpty, Self.startEngine(engine, feeding: built.map(\.request)) else { return }+ tapInstalled = trueprefix = text.isEmpty || text.hasSuffix(" ") ? text : text + " "transcript = ""- start()+ candidates = built+ lead = 0+ isRecording = true+ for (index, candidate) in built.enumerated() {+ candidate.task = Self.recognize(candidate) { [weak self] words, confidence, final, failed in+ Task { @MainActor in+ self?.update(index, words: words, confidence: confidence, final: final, failed: failed)+ }+ }+ }}- nonisolated private func start() {- // The app's language first (Settings → General → Language), then the system's.- let localeID = Bundle.main.preferredLocalizations.first ?? Locale.preferredLanguages.first ?? "en-US"- guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: localeID)) ?? SFSpeechRecognizer(),- recognizer.isAvailable else { return }- let request = SFSpeechAudioBufferRecognitionRequest()- request.shouldReportPartialResults = true- if recognizer.supportsOnDeviceRecognition { request.requiresOnDeviceRecognition = true }+ /// Stops listening and returns the field's text with the most likely transcript.+ @MainActor func finish() async -> String {+ guard isRecording else { return text }+ isFinishing = true+ stopEngine()+ candidates.forEach { $0.request.endAudio() }+ // With several languages, wait a moment for the final results: they carry the confidence.+ if candidates.count > 1, !candidates.allSatisfy(\.finished) {+ await withCheckedContinuation { continuation in+ finishing = continuation+ Task { @MainActor [weak self] in+ try? await Task.sleep(for: .milliseconds(1200))+ self?.resumeFinishing()+ }+ }+ }+ if let best = bestCandidate() {+ transcript = best.words+ if candidates.count > 1 {+ UserDefaults.standard.set(best.locale.identifier, forKey: Self.lastHeardKey)+ }+ }+ tearDown()+ return text+ }++ /// Cancels without waiting (the view went away).+ @MainActor func stop() {+ guard isRecording || !candidates.isEmpty else { return }+ tearDown()+ }++ @MainActor private func stopEngine() {+ engine.stop()+ if tapInstalled { engine.inputNode.removeTap(onBus: 0) }+ tapInstalled = false+ }++ @MainActor private func tearDown() {+ stopEngine()+ candidates.forEach { $0.request.endAudio(); $0.task?.cancel() }+ candidates = []+ lead = 0+ isRecording = false+ isFinishing = false+ resumeFinishing()+ }++ @MainActor private func resumeFinishing() {+ finishing?.resume()+ finishing = nil+ }++ // MARK: Results++ @MainActor private func update(_ index: Int, words: String?, confidence: Float, final: Bool, failed: Bool) {+ guard index < candidates.count else { return }+ let candidate = candidates[index]+ if let words { candidate.words = words }+ if confidence > 0 { candidate.confidence = confidence }+ if final || failed { candidate.finished = true }++ if isFinishing {+ if candidates.allSatisfy(\.finished) { resumeFinishing() }+ return+ }+ // A recognizer that gives up while you speak (no model, busy…) hands over to the next one.+ if index == lead, candidate.finished, candidate.words.isEmpty,+ let next = candidates.firstIndex(where: { !$0.finished }) {+ lead = next+ transcript = candidates[next].words+ return+ }+ // When Apple reports confidences while speaking, follow the clearly better language.+ let current = candidates[lead]+ if index != lead, candidate.confidence > current.confidence + 0.15,+ candidate.words.split(separator: " ").count >= 3 {+ lead = index+ }+ if index == lead { transcript = candidate.words }+ if candidates.allSatisfy(\.finished) { tearDown() }+ }++ @MainActor private func bestCandidate() -> Candidate? {+ guard lead < candidates.count else { return nil }+ let leader = candidates[lead]+ let best = candidates+ .filter { !$0.words.isEmpty }+ .max { $0.confidence < $1.confidence }+ guard let best, best.confidence > leader.confidence else { return leader }+ return best+ }++ // MARK: Audio and recognizers (off the main actor: they call back on their own threads)++ nonisolated private static func startEngine(_ engine: AVAudioEngine, feeding requests: [SFSpeechAudioBufferRecognitionRequest]) -> Bool {let input = engine.inputNodeinput.installTap(onBus: 0, bufferSize: 1024, format: input.outputFormat(forBus: 0)) { buffer, _ in- request.append(buffer)+ for request in requests { request.append(buffer) }}engine.prepare()do { try engine.start() } catch {input.removeTap(onBus: 0)- return+ return false}- self.request = request- Task { @MainActor in self.isRecording = true }- task = recognizer.recognitionTask(with: request) { [weak self] result, error in+ return true+ }++ nonisolated private static func recognize(+ _ candidate: Candidate,+ report: @escaping @Sendable (String?, Float, Bool, Bool) -> Void+ ) -> SFSpeechRecognitionTask {+ candidate.recognizer.recognitionTask(with: candidate.request) { result, error inlet words = result?.bestTranscription.formattedString- let done = error != nil || (result?.isFinal ?? false)- Task { @MainActor in- guard let self else { return }- if let words { self.transcript = words }- if done { self.stop() }- }+ let segments = result?.bestTranscription.segments ?? []+ let confidence = segments.isEmpty ? 0 : segments.map(\.confidence).reduce(0, +) / Float(segments.count)+ report(words, confidence, result?.isFinal ?? false, error != nil)}}- @MainActor func stop() {- guard isRecording || request != nil else { return }- engine.stop()- engine.inputNode.removeTap(onBus: 0)- request?.endAudio()- task?.cancel()- request = nil- task = nil- isRecording = false+ /// Recognizers that can run now. Extra languages in Automatic only listen on-device,+ /// so the audio never goes to Apple in a language you didn't speak.+ private static func makeCandidates(_ locales: [Locale]) -> [Candidate] {+ var out: [Candidate] = []+ for locale in locales {+ guard let recognizer = SFSpeechRecognizer(locale: locale), recognizer.isAvailable else { continue }+ if !out.isEmpty && !recognizer.supportsOnDeviceRecognition { continue }+ out.append(Candidate(locale: locale, recognizer: recognizer))+ }+ if out.isEmpty, let fallback = SFSpeechRecognizer(), fallback.isAvailable {+ out.append(Candidate(locale: fallback.locale, recognizer: fallback))+ }+ return out+ }++ // MARK: Languages++ /// The languages Automatic listens for: last winner, macOS languages, keyboard layouts,+ /// Coucou's own language, then English — one per language, up to three.+ static func automaticLocales() -> [Locale] {+ var ids: [String] = []+ if let last = UserDefaults.standard.string(forKey: lastHeardKey) { ids.append(last) }+ // The Mac's languages (not Coucou's override in Settings → General → Language).+ ids += (CFPreferencesCopyValue("AppleLanguages" as CFString, kCFPreferencesAnyApplication,+ kCFPreferencesCurrentUser, kCFPreferencesAnyHost) as? [String]) ?? []+ ids += keyboardLanguages()+ ids += Locale.preferredLanguages+ ids.append("en-US")++ let supported = Array(SFSpeechRecognizer.supportedLocales())+ var seen = Set<String>()+ var out: [Locale] = []+ for id in ids {+ guard let locale = match(id, in: supported) else { continue }+ let language = locale.language.languageCode?.identifier ?? locale.identifier+ guard seen.insert(language).inserted else { continue }+ out.append(locale)+ if out.count == maxCandidates { break }+ }+ return out+ }++ /// Every language the recognizer knows, by name.+ static var allLanguages: [String] {+ SFSpeechRecognizer.supportedLocales()+ .map(\.identifier)+ .sorted { name(of: $0).localizedCaseInsensitiveCompare(name(of: $1)) == .orderedAscending }+ }++ static func name(of identifier: String) -> String {+ let name = Locale.current.localizedString(forIdentifier: identifier) ?? identifier+ return name.prefix(1).uppercased() + name.dropFirst()+ }++ /// "fr" or "fr_FR" → the recognizer locale for it, preferring the Mac's region.+ private static func match(_ id: String, in supported: [Locale]) -> Locale? {+ let wanted = id.replacingOccurrences(of: "_", with: "-").lowercased()+ if let exact = supported.first(where: { $0.identifier.replacingOccurrences(of: "_", with: "-").lowercased() == wanted }) {+ return exact+ }+ let language = Locale(identifier: id).language.languageCode?.identifier+ let sameLanguage = supported.filter { $0.language.languageCode?.identifier == language }+ let region = Locale(identifier: id).region ?? Locale.current.region+ return sameLanguage.first { $0.region == region }+ ?? sameLanguage.min { $0.identifier < $1.identifier }+ }++ /// The main language of each enabled keyboard layout ("fr" for AZERTY, "en" for ABC…).+ private static func keyboardLanguages() -> [String] {+ let filter = ["TISPropertyInputSourceCategory": "TISCategoryKeyboardInputSource"] as CFDictionary+ guard let list = TISCreateInputSourceList(filter, false)?.takeRetainedValue() as? [TISInputSource] else { return [] }+ return list.compactMap { source in+ guard let raw = TISGetInputSourceProperty(source, "TISPropertyInputSourceLanguages" as CFString) else { return nil }+ let languages = Unmanaged<CFArray>.fromOpaque(raw).takeUnretainedValue() as? [String]+ return languages?.first+ }}private static func authorized() async -> Bool {diff --git a/README.md b/README.mdindex 41f3c39b..b475eb92 100644--- a/README.md+++ b/README.md@@ -52,7 +52,7 @@ Meet **Mochi**: a soft little squircle with big eyes that pops out of your notch- ⌨️ **Keyboard shortcuts** — open the chat, jump to an alert or a terminal, switch pills, mute, send Mochi to the desktop or open the wardrobe from anywhere; all customizable in Settings → Shortcuts.- 🎭 **A real character** — idle breathing, blinks, eyes on a sphere that follow your mouse, emotes, 28 handcrafted sounds, a greeting on launch.- 🔊 **Your own sounds** *(macOS)* — drop a file named like one of Mochi's sounds (`finish.wav`, `approval.mp3`…) in Settings → General → Sound → **Open sounds folder** to replace it.-- 🎙️ **Dictate in the chat** *(macOS, GitHub build)* — click the mic next to the chat field and talk; speech recognition runs on your Mac when it supports it.+- 🎙️ **Dictate in the chat** *(macOS, GitHub build)* — click the mic next to the chat field and talk in any of your languages (Coucou picks the one you spoke, or right-click the mic to choose); speech recognition runs on your Mac when it supports it.- 🫥 **Invisible when idle** — hides away when nothing is running, peeks out when you hover the notch (the top edge of the screen on Windows and Linux).- 🖥️ **Any Mac, notch or not** — on an iMac, a Mac mini, or a MacBook with its lid closed on an external display, Mochi sits in a small bar at the top of the screen. With several displays, pick his screen in Settings → General, or let him follow your mouse *(macOS)*.- 📱 **Coucou on iPhone** — your sessions, approvals and services in your pocket, with Live Activities, widgets and Siri. See [Coucou on iPhone](#coucou-on-iphone).