A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.
Swift ai-agentsanthropicantigravityclaudeclaude-codecodexcursordynamic-islandgemini-cliiosiphonlinuxmacosmacos-appmenubar-appnotchopen-sourceswiftswiftuiwindows
Flow dictation: keep the last word and the whole sentence
Claude362c7a797d33
diff --git a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxprojindex 171a3401..869124c6 100644--- a/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj+++ b/NotchBuddy/NotchBuddy.xcodeproj/project.pbxproj@@ -47,6 +47,7 @@1C2BB61FA83015C42B321E0E /* PillColors.swift in Sources */ = {isa = PBXBuildFile; fileRef = 37564F5F041B38C976DF3DEF /* PillColors.swift */; };1D23613B270FE8B11C5E27A7 /* IslandDisplayChoice.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A09E73F1E58019EF6EA2A62 /* IslandDisplayChoice.swift */; };1D4CE0CCD2B5A0FFE50C7F06 /* TerminalTarget.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */; };+ 5663BA3BFBE656C2F5C00290 /* DictationTranscript.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */; };0AF77F5721986BE11151CE87 /* FlowSettingsBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */; };614F9783757FE777EF4CABB5 /* FlowPill.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79D06C32348E26EAA1087BFC /* FlowPill.swift */; };F05CD98E6BF8679F12880345 /* FlowDictation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17870B121906BE24A935C407 /* FlowDictation.swift */; };@@ -132,6 +133,7 @@65CB53E20B2289CA3AC3E8FE /* TurnSnapshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A1C4152EBDBD7C24CABFFD7 /* TurnSnapshot.swift */; };6613EB914852E6EEB1FF37BF /* IntentParser.swift in Sources */ = {isa = PBXBuildFile; fileRef = F39753CFF90B494D449CCD95 /* IntentParser.swift */; };6726F74966EDE7F82BFE89B2 /* TerminalTarget.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */; };+ AAC6AB081F959CEE5FE628C8 /* DictationTranscript.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */; };98863553DD449BD791A1E00C /* FlowSettingsBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */; };15C9E266B443EFE70908510B /* FlowPill.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79D06C32348E26EAA1087BFC /* FlowPill.swift */; };8BF7CB586067C38AB5C4738C /* FlowDictation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17870B121906BE24A935C407 /* FlowDictation.swift */; };@@ -470,6 +472,7 @@871CA2BADC67D97C97C7009D /* error.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = error.wav; sourceTree = "<group>"; };8790ACE9F73BD74640A0F20C /* LocalChat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalChat.swift; sourceTree = "<group>"; };87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalTarget.swift; sourceTree = "<group>"; };+ 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DictationTranscript.swift; sourceTree = "<group>"; };0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowSettingsBox.swift; sourceTree = "<group>"; };79D06C32348E26EAA1087BFC /* FlowPill.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowPill.swift; sourceTree = "<group>"; };17870B121906BE24A935C407 /* FlowDictation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlowDictation.swift; sourceTree = "<group>"; };@@ -630,6 +633,7 @@914A792788D403F836BEB570 /* SpotifyViews.swift */,2EF78999D68D56ADE4CEFBE2 /* StripePoller.swift */,87AF6F7A027AB1D46309DD49 /* TerminalTarget.swift */,+ 78597C8DAC0BA94D4AD6E870 /* DictationTranscript.swift */,0565D4D4D369E3ADD9CEF697 /* FlowSettingsBox.swift */,79D06C32348E26EAA1087BFC /* FlowPill.swift */,17870B121906BE24A935C407 /* FlowDictation.swift */,@@ -1118,6 +1122,7 @@1558BB00C530B721172F6E65 /* StripePoller.swift in Sources */,9EA178EA395B116336C1FA10 /* TerminalTarget+AppKit.swift in Sources */,1D4CE0CCD2B5A0FFE50C7F06 /* TerminalTarget.swift in Sources */,+ 5663BA3BFBE656C2F5C00290 /* DictationTranscript.swift in Sources */,0AF77F5721986BE11151CE87 /* FlowSettingsBox.swift in Sources */,614F9783757FE777EF4CABB5 /* FlowPill.swift in Sources */,F05CD98E6BF8679F12880345 /* FlowDictation.swift in Sources */,@@ -1324,6 +1329,7 @@E4B9986674F4D425826F15A8 /* StripePoller.swift in Sources */,11F14DB97C93CF84DF9D5C43 /* TerminalTarget+AppKit.swift in Sources */,6726F74966EDE7F82BFE89B2 /* TerminalTarget.swift in Sources */,+ AAC6AB081F959CEE5FE628C8 /* DictationTranscript.swift in Sources */,98863553DD449BD791A1E00C /* FlowSettingsBox.swift in Sources */,15C9E266B443EFE70908510B /* FlowPill.swift in Sources */,8BF7CB586067C38AB5C4738C /* FlowDictation.swift in Sources */,diff --git a/NotchBuddy/Sources/App/DictationTranscript.swift b/NotchBuddy/Sources/App/DictationTranscript.swiftnew file mode 100644index 00000000..cc85469d--- /dev/null+++ b/NotchBuddy/Sources/App/DictationTranscript.swift@@ -0,0 +1,84 @@+import Foundation++/// The words of one dictation, kept whole when Apple's recognizer loses some of them+/// (tests/FlowKeyLogicTests.swift).+///+/// On a long dictation the recognizer can:+/// - start over after a pause: the next result holds only the new words;+/// - drop a stretch it had already given, in the middle of a later result.+/// Taken as is, either loses what was said. So each result is merged with the words seen before:+/// a result that looks like a fresh start closes the text so far, and a word heard earlier that no+/// newer word covers in time is kept where it was.+struct DictationTranscript: Equatable {+ /// One word (or a few) with its place in the audio, in seconds.+ struct Piece: Equatable, Sendable {+ var start: Double+ var end: Double+ var text: String+ }++ /// Text closed by a fresh start of the recognizer.+ private(set) var committed = ""+ /// The words of the stretch in progress.+ private(set) var pieces: [Piece] = []+ /// Their text: the recognizer's own (formatted) string, unless words had to be put back.+ private(set) var current = ""++ var text: String { Self.join(committed, current) }++ /// A newer word closer than this to an older one, in time, replaces it.+ static let tolerance = 0.25++ /// - Parameters:+ /// - formatted: the result's text as the recognizer wrote it.+ /// - pieces: its words with their times (all zero when the recognizer gives no times).+ mutating func absorb(_ formatted: String, pieces new: [Piece]) {+ if Self.isRestart(previous: current, new: formatted) {+ committed = Self.join(committed, current)+ pieces = new+ current = formatted+ return+ }+ let timed = !new.isEmpty && new.contains { $0.end > $0.start }+ guard timed, let last = new.last else {+ pieces = new+ current = formatted+ return+ }+ // Earlier words the new result left out, that no new word covers: put them back.+ let kept = pieces.filter { old in+ old.end <= last.start + Self.tolerance+ && !new.contains { Self.close(old, $0) }+ }+ if kept.isEmpty {+ pieces = new+ current = formatted+ } else {+ pieces = (kept + new).sorted { $0.start < $1.start }+ current = pieces.map(\.text).joined(separator: " ")+ }+ }++ /// Plain text only (no times): a fresh start closes the text so far, anything else replaces it.+ mutating func absorb(_ formatted: String) { absorb(formatted, pieces: []) }++ /// A result is a fresh start when the text so far had a few words, the new one has less than half+ /// as many, and it does not start with the same word. A correction of the last words, or a+ /// recognizer that keeps everything, never looks like that.+ static func isRestart(previous: String, new: String) -> Bool {+ let p = previous.split(whereSeparator: \.isWhitespace)+ let n = new.split(whereSeparator: \.isWhitespace)+ guard p.count >= 3, !n.isEmpty, n.count * 2 <= p.count else { return false }+ return p[0].lowercased() != n[0].lowercased()+ }++ private static func close(_ a: Piece, _ b: Piece) -> Bool {+ a.start < b.end + tolerance && b.start < a.end + tolerance+ }++ private static func join(_ a: String, _ b: String) -> String {+ if a.isEmpty { return b }+ if b.isEmpty { return a }+ return a + " " + b+ }+}diff --git a/NotchBuddy/Sources/App/FlowDictation.swift b/NotchBuddy/Sources/App/FlowDictation.swiftindex 7c4c1529..a78cea40 100644--- a/NotchBuddy/Sources/App/FlowDictation.swift+++ b/NotchBuddy/Sources/App/FlowDictation.swift@@ -59,6 +59,8 @@ final class FlowDictation: ObservableObject {static let shared = FlowDictation()static let enabledKey = "flowDictationEnabled"+ /// How long the microphone stays open after the key comes up.+ static let tailMilliseconds = 600static let keyKey = "flowDictationKey"@Published var isEnabled: Bool = UserDefaults.standard.bool(forKey: FlowDictation.enabledKey) {@@ -237,6 +239,10 @@ final class FlowDictation: ObservableObject {let opening = startTaskTask { @MainActor [weak self] inawait opening?.value+ // The key often comes up while the last word is still being said: keep listening a+ // moment so it makes it in.+ try? await Task.sleep(for: .milliseconds(Self.tailMilliseconds))+ guard !listen.cancelled else { return }let text = await listen.dictation.finish().trimmingCharacters(in: .whitespacesAndNewlines)guard let self, !listen.cancelled else { return }// A new listen may have begun meanwhile: its pill is left alone, but these words still go in.diff --git a/NotchBuddy/Sources/App/MacDictation.swift b/NotchBuddy/Sources/App/MacDictation.swiftindex f9bc29b8..07838bc7 100644--- a/NotchBuddy/Sources/App/MacDictation.swift+++ b/NotchBuddy/Sources/App/MacDictation.swift@@ -49,7 +49,9 @@ final class MacDictation: @unchecked Sendable {let recognizer: SFSpeechRecognizerlet request = SFSpeechAudioBufferRecognitionRequest()var task: SFSpeechRecognitionTask?- var words = ""+ /// What was heard, kept whole when the recognizer starts over after a pause.+ var heard = DictationTranscript()+ var words: String { heard.text }/// Mean confidence of the words (Apple fills it in on the final result).var confidence: Float = 0var finished = false@@ -76,9 +78,9 @@ final class MacDictation: @unchecked Sendable {isRecording = trueNotificationCenter.default.post(name: .dictationDidStart, object: nil)for (index, candidate) in built.enumerated() {- candidate.task = Self.recognize(candidate) { [weak self] words, confidence, final, failed in+ candidate.task = Self.recognize(candidate) { [weak self] words, pieces, confidence, final, failed inTask { @MainActor in- self?.update(index, words: words, confidence: confidence, final: final, failed: failed)+ self?.update(index, words: words, pieces: pieces, confidence: confidence, final: final, failed: failed)}}}@@ -90,12 +92,13 @@ final class MacDictation: @unchecked Sendable {isFinishing = truestopEngine()candidates.forEach { $0.request.endAudio() }- // With several languages, wait a moment for the final results: they carry the confidence.- if candidates.count > 1, !candidates.allSatisfy(\.finished) {+ // Wait a moment for the final results: they carry the last words (and, with several+ // languages, the confidence that picks one).+ if !candidates.allSatisfy(\.finished) {await withCheckedContinuation { continuation infinishing = continuationTask { @MainActor [weak self] in- try? await Task.sleep(for: .milliseconds(1200))+ try? await Task.sleep(for: .milliseconds(1500))self?.resumeFinishing()}}@@ -140,10 +143,10 @@ final class MacDictation: @unchecked Sendable {// MARK: Results- @MainActor private func update(_ index: Int, words: String?, confidence: Float, final: Bool, failed: Bool) {+ @MainActor private func update(_ index: Int, words: String?, pieces: [DictationTranscript.Piece], confidence: Float, final: Bool, failed: Bool) {guard index < candidates.count else { return }let candidate = candidates[index]- if let words { candidate.words = words }+ if let words { candidate.heard.absorb(words, pieces: pieces) }if confidence > 0 { candidate.confidence = confidence }if final || failed { candidate.finished = true }@@ -197,13 +200,16 @@ final class MacDictation: @unchecked Sendable {nonisolated private static func recognize(_ candidate: Candidate,- report: @escaping @Sendable (String?, Float, Bool, Bool) -> Void+ report: @escaping @Sendable (String?, [DictationTranscript.Piece], Float, Bool, Bool) -> Void) -> SFSpeechRecognitionTask {candidate.recognizer.recognitionTask(with: candidate.request) { result, error inlet words = result?.bestTranscription.formattedStringlet segments = result?.bestTranscription.segments ?? []let confidence = segments.isEmpty ? 0 : segments.map(\.confidence).reduce(0, +) / Float(segments.count)- report(words, confidence, result?.isFinal ?? false, error != nil)+ let pieces = segments.map {+ DictationTranscript.Piece(start: $0.timestamp, end: $0.timestamp + $0.duration, text: $0.substring)+ }+ report(words, pieces, confidence, result?.isFinal ?? false, error != nil)}}diff --git a/scripts/test-flow-keys.sh b/scripts/test-flow-keys.shindex 91b09ca5..765f8886 100755--- a/scripts/test-flow-keys.sh+++ b/scripts/test-flow-keys.sh@@ -3,6 +3,6 @@ set -euo pipefailcd "$(dirname "$0")/.."TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/coucou-flow-keys.XXXXXX")"trap 'rm -rf "$TEST_DIR"' EXIT-swiftc NotchBuddy/Sources/App/FlowKeyLogic.swift \+swiftc NotchBuddy/Sources/App/FlowKeyLogic.swift NotchBuddy/Sources/App/DictationTranscript.swift \tests/FlowKeyLogicTests.swift -o "$TEST_DIR/flow-keys-tests""$TEST_DIR/flow-keys-tests"diff --git a/tests/FlowKeyLogicTests.swift b/tests/FlowKeyLogicTests.swiftindex 6328bc94..746af5e9 100644--- a/tests/FlowKeyLogicTests.swift+++ b/tests/FlowKeyLogicTests.swift@@ -89,6 +89,54 @@ enum FlowKeyLogicTests {check("in between", FlowBands.level(db: -54) > 0.3 && FlowBands.level(db: -54) < 0.7)}+ print("\nTranscript kept whole")+ do {+ typealias P = DictationTranscript.Piece+ func words(_ list: [(Double, String)]) -> [P] { list.map { P(start: $0.0, end: $0.0 + 0.3, text: $0.1) } }++ var t = DictationTranscript()+ t.absorb("Ok mais", pieces: words([(0, "Ok"), (0.4, "mais")]))+ t.absorb("Ok mais c'est au top", pieces: words([(0, "Ok"), (0.4, "mais"), (0.8, "c'est"), (1.2, "au"), (1.6, "top")]))+ check("a growing result is taken as is", t.text == "Ok mais c'est au top")+ t.absorb("Ok mais c'est au top.", pieces: words([(0, "Ok"), (0.4, "mais"), (0.8, "c'est"), (1.2, "au"), (1.6, "top.")]))+ check("the recognizer's own formatting wins", t.text == "Ok mais c'est au top.")++ // The recognizer drops the middle stretch from a later result.+ var m = DictationTranscript()+ m.absorb("un deux trois quatre cinq", pieces: words([(0, "un"), (1, "deux"), (2, "trois"), (3, "quatre"), (4, "cinq")]))+ m.absorb("un deux cinq six", pieces: words([(0, "un"), (1, "deux"), (4, "cinq"), (5, "six")]))+ check("a dropped middle stretch is put back", m.text == "un deux trois quatre cinq six")++ // A correction of the last words replaces them, nothing is doubled.+ var c = DictationTranscript()+ c.absorb("je vais à la plage", pieces: words([(0, "je"), (0.5, "vais"), (1, "à"), (1.3, "la"), (1.6, "plage")]))+ c.absorb("je vais à Laplace", pieces: words([(0, "je"), (0.5, "vais"), (1, "à"), (1.35, "Laplace")]))+ check("a correction replaces the old words", c.text == "je vais à Laplace")++ // Times that shift a little between results do not double a word.+ var s = DictationTranscript()+ s.absorb("bonjour à tous", pieces: words([(1.0, "bonjour"), (1.4, "à"), (1.7, "tous")]))+ s.absorb("Bonjour à tous", pieces: words([(1.15, "Bonjour"), (1.5, "à"), (1.8, "tous")]))+ check("shifted times replace, not double", s.text == "Bonjour à tous")++ // The recognizer starts over after a pause, times from zero again.+ var r = DictationTranscript()+ r.absorb("la première phrase est assez longue", pieces: words([(0, "la"), (0.4, "première"), (0.9, "phrase"), (1.3, "est"), (1.6, "assez"), (2.0, "longue")]))+ r.absorb("ensuite", pieces: words([(0, "ensuite")]))+ r.absorb("ensuite la suite", pieces: words([(0, "ensuite"), (0.5, "la"), (0.8, "suite")]))+ check("a fresh start keeps what came before", r.text == "la première phrase est assez longue ensuite la suite")++ // No times at all: plain replace, fresh starts still kept.+ var u = DictationTranscript()+ u.absorb("hello there my friend")+ u.absorb("hello there my friend how are you")+ check("untimed growth", u.text == "hello there my friend how are you")+ u.absorb("great")+ check("untimed fresh start", u.text == "hello there my friend how are you great")+ check("a short correction is not a fresh start", !DictationTranscript.isRestart(previous: "one two three four", new: "one two"))+ check("too little text to call it a fresh start", !DictationTranscript.isRestart(previous: "one two", new: "three"))+ }+if failures == 0 {print("\nAll tests passed.")exit(0)